From 690ff3e41573cae60a640d88b994d4e7e4fdbe49 Mon Sep 17 00:00:00 2001 From: Shanmukh Pawan Date: Fri, 23 Oct 2026 17:23:06 -0400 Subject: [PATCH] fix(resolver): honor configured prebuilt wheel resolvers Resolve and download fresh prebuilt wheels through the selected source resolver. Preserve local wheel cache reuse, legacy prebuilt downloads, prebuilt hooks, and the test-mode fallback. Add focused coverage for both configured profiles, build-order replay, and legacy behavior. Closes #1208 Co-Authored-By: Codex Signed-off-by: Shanmukh Pawan --- docs/concepts/bootstrapper-architecture.rst | 6 + docs/concepts/package-settings.rst | 10 + docs/concepts/resolver-architecture.rst | 6 +- .../bootstrap_requirement_resolver.py | 57 +-- src/fromager/bootstrapper/_bootstrapper.py | 7 +- src/fromager/bootstrapper/_cache.py | 22 +- src/fromager/commands/build.py | 34 +- src/fromager/commands/download_sequence.py | 22 +- src/fromager/packagesettings/_pbi.py | 5 +- src/fromager/wheels.py | 25 ++ tests/test_bootstrap_requirement_resolver.py | 144 +++++++- tests/test_bootstrapper.py | 39 +- tests/test_commands.py | 333 +++++++++++++++++- tests/test_packagesettings.py | 25 ++ tests/test_wheels.py | 116 +++++- 15 files changed, 810 insertions(+), 41 deletions(-) diff --git a/docs/concepts/bootstrapper-architecture.rst b/docs/concepts/bootstrapper-architecture.rst index 37d7d5493..86a2cc91b 100644 --- a/docs/concepts/bootstrapper-architecture.rst +++ b/docs/concepts/bootstrapper-architecture.rst @@ -16,6 +16,12 @@ Bootstrap Phase Flow Every package passes through a sequence of phases. Source packages traverse the full pipeline; prebuilt wheels skip the build phases. +``pypi-prebuilt`` and ``hook-prebuilt`` use the selected source resolver to +download a fresh wheel in ``PrepareSource``. The legacy ``pre_built`` setting +keeps its existing wheel download path. Both routes update the local mirror +and record the package as prebuilt in the graph and build order. +When a local wheel is already available, bootstrap reuses it before asking +the selected provider to resolve a fresh wheel. .. code-block:: text diff --git a/docs/concepts/package-settings.rst b/docs/concepts/package-settings.rst index c516ff93a..e588b7f96 100644 --- a/docs/concepts/package-settings.rst +++ b/docs/concepts/package-settings.rst @@ -42,6 +42,16 @@ cannot be combined with package-level ``download_source`` or ``resolver_dist``. An explicit ``pre_built: false`` also conflicts; an omitted ``pre_built`` does not. +``PackageBuildInfo.pre_built`` follows the selected ``source`` resolver's +``resolves_prebuilt_wheel`` capability. Without ``source``, it reads the +legacy variant ``pre_built`` setting. Configured prebuilt resolvers download +fresh wheels through their ``download()`` contract; legacy prebuilt packages +keep the wheel downloader. + +.. versionchanged:: 0.102.0 + Configured prebuilt source resolvers now select the prebuilt pipeline + during bootstrap and later builds. + .. versionchanged:: 0.102.0 Source and legacy settings mixed across package and variant levels now raise a validation error when package settings load. diff --git a/docs/concepts/resolver-architecture.rst b/docs/concepts/resolver-architecture.rst index 4702b957b..3eaa8d630 100644 --- a/docs/concepts/resolver-architecture.rst +++ b/docs/concepts/resolver-architecture.rst @@ -43,6 +43,8 @@ the cache already contains a matching version. The configured wheel cache server is not a version-resolution fallback. After a version has been selected, the bootstrapper checks the cache server separately for a previously built wheel. +For prebuilt packages, the local wheel server can supply a cached wheel +before a fresh resolution through the selected source provider. Provider Hierarchy ------------------ @@ -132,7 +134,9 @@ The separate ``step prepare-source`` invocation accepts ``--source-kind`` and ``step download-source-archive --output-format json`` exposes that result's actual path and kind without changing its default path-only output. Without ``--source-kind``, it keeps the legacy archive behavior. -Integrating prebuilt detection into the bootstrapper remains a separate step. +Configured prebuilt resolvers select the prebuilt bootstrap path. Fresh +wheels use their ``download()`` contract; local cached wheels and the +test-mode fallback use the legacy wheel downloader. See :ref:`download_source_hook` for the return contract and compatibility rules. .. versionchanged:: 0.102.0 diff --git a/src/fromager/bootstrap_requirement_resolver.py b/src/fromager/bootstrap_requirement_resolver.py index 8ee267511..d64453adb 100644 --- a/src/fromager/bootstrap_requirement_resolver.py +++ b/src/fromager/bootstrap_requirement_resolver.py @@ -171,12 +171,16 @@ def _resolve_and_extend( parent_req=parent_req, ) + source_resolver = self.ctx.package_build_info(req).source_resolver + configured_prebuilt = ( + source_resolver is not None and source_resolver.resolves_prebuilt_wheel + ) if cached_resolution: logger.debug( f"resolved from previous bootstrap: {len(cached_resolution)} version(s)" ) results = cached_resolution - elif pre_built: + elif pre_built and not configured_prebuilt: # Resolve prebuilt wheel wheel_server_urls = wheels.get_wheel_server_urls( self.ctx, req, cache_wheel_server_url=resolver.PYPI_SERVER_URL @@ -188,27 +192,40 @@ def _resolve_and_extend( req_type=req_type, ) else: - # Resolve source (sdist) - provider = sources.get_source_provider( - ctx=self.ctx, - req=req, - sdist_server_url=resolver.PYPI_SERVER_URL, - req_type=req_type, - ) - max_age_cutoff = resolver._compute_max_age_cutoff(self.ctx) - age_fallback = ( - resolver.AgeFallback.NEWEST - if self.multiple_versions - else resolver.AgeFallback.ALL - ) - results = resolver.find_all_matching_from_provider( - provider, - req, - max_age_cutoff=max_age_cutoff, - age_fallback=age_fallback, - ) + results = [] + if pre_built and self.ctx.wheel_server_url: + try: + results = wheels.resolve_all_cached_wheels( + ctx=self.ctx, + req=req, + cache_server_url=self.ctx.wheel_server_url, + ) + except Exception: + logger.debug("no matching wheel in local cache", exc_info=True) if not results: + # The configured source provider resolves either source archives + # or prebuilt wheels, according to its selected capability. + provider = sources.get_source_provider( + ctx=self.ctx, + req=req, + sdist_server_url=resolver.PYPI_SERVER_URL, + req_type=req_type, + ) + max_age_cutoff = resolver._compute_max_age_cutoff(self.ctx) + age_fallback = ( + resolver.AgeFallback.NEWEST + if self.multiple_versions + else resolver.AgeFallback.ALL + ) + results = resolver.find_all_matching_from_provider( + provider, + req, + max_age_cutoff=max_age_cutoff, + age_fallback=age_fallback, + ) + + if not results and not pre_built: logger.warning( "resolver returned no results (%s version mode)", "multiple" if self.multiple_versions else "single", diff --git a/src/fromager/bootstrapper/_bootstrapper.py b/src/fromager/bootstrapper/_bootstrapper.py index c524324c7..1682c9155 100644 --- a/src/fromager/bootstrapper/_bootstrapper.py +++ b/src/fromager/bootstrapper/_bootstrapper.py @@ -471,7 +471,12 @@ def _download_prebuilt( wheel_url: str, ) -> tuple[pathlib.Path, pathlib.Path]: result = _cache.bg_prepare_prebuilt( - self.ctx, req, req_type, resolved_version, wheel_url + self.ctx, + req, + req_type, + resolved_version, + wheel_url, + force_legacy_download=True, ) assert result.wheel_filename is not None assert result.unpack_dir is not None diff --git a/src/fromager/bootstrapper/_cache.py b/src/fromager/bootstrapper/_cache.py index 089895f5b..8e5fafd1f 100644 --- a/src/fromager/bootstrapper/_cache.py +++ b/src/fromager/bootstrapper/_cache.py @@ -245,12 +245,30 @@ def bg_prepare_prebuilt( req_type: RequirementType, resolved_version: Version, wheel_url: str, + *, + force_legacy_download: bool = False, ) -> PreparedSourceData: - """Background-safe prebuilt download: no Bootstrapper state accessed.""" + """Download a prebuilt wheel, reusing local cache or test-mode fallback.""" # Thread-safe: paths include {req.name}-{resolved_version} (unique per package), # mkdir uses exist_ok=True (atomic), and update_wheel_mirror() is already locked. logger.info(f"using pre-built wheel for {req_type} requirement") - wheel_filename = wheels.download_wheel(req, wheel_url, ctx.wheels_prebuilt) + source_resolver = ctx.package_build_info(req).source_resolver + local_wheel_url = bool(ctx.wheel_server_url) and wheel_url.startswith( + ctx.wheel_server_url.rstrip("/") + "/" + ) + if force_legacy_download: + wheel_filename = wheels.download_wheel(req, wheel_url, ctx.wheels_prebuilt) + elif source_resolver is not None and not source_resolver.resolves_prebuilt_wheel: + raise ValueError( + f"{req}: source resolver {source_resolver.provider!r} " + "does not resolve prebuilt wheels" + ) + elif local_wheel_url or source_resolver is None: + wheel_filename = wheels.download_wheel(req, wheel_url, ctx.wheels_prebuilt) + else: + wheel_filename = wheels.download_prebuilt_wheel( + ctx=ctx, req=req, version=resolved_version, download_url=wheel_url + ) unpack_dir = _create_unpack_dir(ctx.work_dir, req, resolved_version) server.update_wheel_mirror(ctx) return PreparedSourceData(wheel_filename=wheel_filename, unpack_dir=unpack_dir) diff --git a/src/fromager/commands/build.py b/src/fromager/commands/build.py index 70d549484..3dc89868e 100644 --- a/src/fromager/commands/build.py +++ b/src/fromager/commands/build.py @@ -362,17 +362,30 @@ def _build( if prebuilt: if not wheel_filename: logger.info("downloading prebuilt wheel") - wheel_filename = wheels.download_wheel( - req=req, - wheel_url=source_download_url, - output_directory=wkctx.wheels_build, - ) + source_resolver = pbi.source_resolver + if source_resolver is not None and source_resolver.resolves_prebuilt_wheel: + wheel_filename = wheels.download_prebuilt_wheel( + ctx=wkctx, + req=req, + version=resolved_version, + download_url=source_download_url, + ) + elif source_resolver is not None: + raise ValueError( + f"{req}: source resolver {source_resolver.provider!r} " + "does not resolve prebuilt wheels" + ) + else: + wheel_filename = wheels.download_wheel( + req=req, + wheel_url=source_download_url, + output_directory=wkctx.wheels_build, + ) else: # already downloaded prebuilt wheel use_exiting_wheel = True - # Run hooks for prebuilt wheels. At this point wheel_filename should - # be set either from _is_wheel_built() or download_wheel(). + # Run hooks for both cached and freshly downloaded prebuilt wheels. hooks.run_prebuilt_wheel_hooks( ctx=wkctx, req=req, @@ -448,9 +461,10 @@ def _build( server.update_wheel_mirror(wkctx) - # After we update the wheel mirror, the built file has - # moved to a new directory. - wheel_filename = wkctx.wheels_downloads / wheel_filename.name + # The mirror moves wheels from the build directory to downloads; + # configured prebuilt wheels stay in the prebuilt directory. + if wheel_filename.parent != wkctx.wheels_prebuilt: + wheel_filename = wkctx.wheels_downloads / wheel_filename.name return BuildSequenceEntry( name=canonicalize_name(req.name), diff --git a/src/fromager/commands/download_sequence.py b/src/fromager/commands/download_sequence.py index 177440fd5..98f7c3e97 100644 --- a/src/fromager/commands/download_sequence.py +++ b/src/fromager/commands/download_sequence.py @@ -69,7 +69,27 @@ def download_one(entry: dict[str, typing.Any]) -> None: if entry["prebuilt"]: if include_wheels: - wheels.download_wheel(req, entry["source_url"], wkctx.wheels_prebuilt) + source_resolver = wkctx.package_build_info(req).source_resolver + if ( + source_resolver is not None + and source_resolver.resolves_prebuilt_wheel + ): + wheels.download_prebuilt_wheel( + ctx=wkctx, + req=req, + version=Version(entry["version"]), + download_url=entry["source_url"], + ) + elif source_resolver is not None: + raise ValueError( + f"{req}: build order marks a prebuilt wheel, but " + f"source resolver {source_resolver.provider!r} " + "does not resolve prebuilt wheels" + ) + else: + wheels.download_wheel( + req, entry["source_url"], wkctx.wheels_prebuilt + ) else: logger.info(f"{entry['dist']}: uses a pre-built wheel, skipping") return diff --git a/src/fromager/packagesettings/_pbi.py b/src/fromager/packagesettings/_pbi.py index ba62f9bf9..74d55a45b 100644 --- a/src/fromager/packagesettings/_pbi.py +++ b/src/fromager/packagesettings/_pbi.py @@ -163,7 +163,10 @@ def has_customizations(self) -> bool: @property def pre_built(self) -> bool: - """Does the variant use pre-build wheels?""" + """Whether the selected source or legacy variant uses prebuilt wheels.""" + source_resolver = self.source_resolver + if source_resolver is not None: + return source_resolver.resolves_prebuilt_wheel vi = self._ps.variants.get(self.variant) if vi is not None: return vi.pre_built diff --git a/src/fromager/wheels.py b/src/fromager/wheels.py index 943edea8d..151e4d963 100644 --- a/src/fromager/wheels.py +++ b/src/fromager/wheels.py @@ -32,6 +32,7 @@ requirements_file, resolver, sbom, + sources, ) from .pkgmetadata.pep376 import verbatim_dist_name @@ -520,6 +521,30 @@ def download_wheel( return wheel_filename +def download_prebuilt_wheel( + *, + ctx: context.WorkContext, + req: Requirement, + version: Version, + download_url: str, +) -> pathlib.Path: + """Download a configured prebuilt wheel and stage it for the wheel mirror.""" + downloaded: packagesettings.DownloadedSource = sources.download_source( + ctx=ctx, req=req, version=version, download_url=download_url + ) + if downloaded.kind is not packagesettings.DownloadKind.prebuilt_wheel: + raise ValueError( + f"{req}: configured prebuilt source returned {downloaded.kind.value!r}; " + "expected 'prebuilt_wheel'" + ) + wheel_path = downloaded.path + if wheel_path.parent != ctx.wheels_prebuilt: + staged_path = ctx.wheels_prebuilt / wheel_path.name + shutil.copy2(wheel_path, staged_path) + return staged_path + return wheel_path + + def get_wheel_server_urls( ctx: context.WorkContext, req: Requirement, *, cache_wheel_server_url: str | None ) -> list[str]: diff --git a/tests/test_bootstrap_requirement_resolver.py b/tests/test_bootstrap_requirement_resolver.py index af359313c..11778a970 100644 --- a/tests/test_bootstrap_requirement_resolver.py +++ b/tests/test_bootstrap_requirement_resolver.py @@ -6,7 +6,7 @@ from packaging.utils import canonicalize_name from packaging.version import Version -from fromager import resolver +from fromager import packagesettings, resolver from fromager.bootstrap_requirement_resolver import BootstrapRequirementResolver from fromager.context import WorkContext from fromager.dependency_graph import DependencyGraph @@ -433,6 +433,147 @@ def test_resolve_from_graph_name_fallback_returns_none_for_missing_package( assert result is None +def test_configured_pypi_prebuilt_resolves_from_selected_index( + tmp_context: WorkContext, +) -> None: + """Fresh prebuilt resolution uses the configured index provider.""" + req = Requirement("pkg==1.0") + tmp_context.settings._package_settings[canonicalize_name("pkg")] = ( + packagesettings.PackageSettings.from_string( + "pkg", + "source:\n provider: pypi-prebuilt\n" + " index_url: https://custom.test/simple/\n", + ) + ) + expected = [("https://custom.test/pkg-1.0-py3-none-any.whl", Version("1.0"))] + with ( + patch( + "fromager.bootstrap_requirement_resolver.resolver.find_all_matching_from_provider", + return_value=expected, + ) as find_matches, + patch( + "fromager.bootstrap_requirement_resolver.wheels.resolve_all_prebuilt_wheels" + ) as legacy, + ): + result = BootstrapRequirementResolver(tmp_context).resolve( + req, RequirementType.TOP_LEVEL + ) + + assert result == expected + provider = find_matches.call_args.args[0] + assert isinstance(provider, resolver.PyPIProvider) + assert provider.sdist_server_url == "https://custom.test/simple/" + legacy.assert_not_called() + + +def test_configured_hook_prebuilt_uses_hook_provider( + tmp_context: WorkContext, +) -> None: + """Fresh prebuilt resolution calls the selected hook provider.""" + req = Requirement("pkg==1.0") + tmp_context.settings._package_settings[canonicalize_name("pkg")] = ( + packagesettings.PackageSettings.from_string( + "pkg", "source:\n provider: hook-prebuilt\n" + ) + ) + provider = resolver.PyPIProvider(sdist_server_url="https://hook.test/simple/") + expected = [("https://hook.test/pkg-1.0-py3-none-any.whl", Version("1.0"))] + with ( + patch.object( + packagesettings.HookPrebuiltResolver, + "resolver_provider", + return_value=provider, + ) as configured, + patch( + "fromager.bootstrap_requirement_resolver.resolver.find_all_matching_from_provider", + return_value=expected, + ) as find_matches, + patch( + "fromager.bootstrap_requirement_resolver.wheels.resolve_all_prebuilt_wheels" + ) as legacy, + ): + result = BootstrapRequirementResolver(tmp_context).resolve( + req, RequirementType.TOP_LEVEL + ) + + assert result == expected + configured.assert_called_once() + assert find_matches.call_args.args[0] is provider + legacy.assert_not_called() + + +def test_configured_prebuilt_reuses_local_wheel_cache( + tmp_context: WorkContext, +) -> None: + """A local wheel is resolved before the configured upstream provider.""" + req = Requirement("pkg==1.0") + tmp_context.settings._package_settings[canonicalize_name("pkg")] = ( + packagesettings.PackageSettings.from_string( + "pkg", "source:\n provider: pypi-prebuilt\n" + ) + ) + tmp_context.wheel_server_url = "https://local.test/simple/" + expected = [("https://local.test/pkg-1.0-py3-none-any.whl", Version("1.0"))] + with ( + patch( + "fromager.bootstrap_requirement_resolver.wheels.resolve_all_cached_wheels", + return_value=expected, + ) as cache, + patch( + "fromager.bootstrap_requirement_resolver.sources.get_source_provider" + ) as configured, + patch( + "fromager.bootstrap_requirement_resolver.wheels.resolve_all_prebuilt_wheels", + return_value=[], + ) as legacy, + ): + result = BootstrapRequirementResolver(tmp_context).resolve( + req, RequirementType.TOP_LEVEL + ) + + assert result == expected + cache.assert_called_once_with( + ctx=tmp_context, req=req, cache_server_url=tmp_context.wheel_server_url + ) + configured.assert_not_called() + legacy.assert_not_called() + + +def test_configured_prebuilt_cache_miss_uses_selected_provider( + tmp_context: WorkContext, +) -> None: + """A local cache miss falls through to the configured upstream index.""" + req = Requirement("pkg==1.0") + tmp_context.settings._package_settings[canonicalize_name("pkg")] = ( + packagesettings.PackageSettings.from_string( + "pkg", + "source:\n provider: pypi-prebuilt\n" + " index_url: https://custom.test/simple/\n", + ) + ) + tmp_context.wheel_server_url = "https://local.test/simple/" + expected = [("https://custom.test/pkg-1.0-py3-none-any.whl", Version("1.0"))] + with ( + patch( + "fromager.bootstrap_requirement_resolver.wheels.resolve_all_cached_wheels", + side_effect=ValueError("cache miss"), + ) as cache, + patch( + "fromager.bootstrap_requirement_resolver.resolver.find_all_matching_from_provider", + return_value=expected, + ) as find_matches, + ): + result = BootstrapRequirementResolver(tmp_context).resolve( + req, RequirementType.TOP_LEVEL + ) + + assert result == expected + cache.assert_called_once() + assert find_matches.call_args.args[0].sdist_server_url == ( + "https://custom.test/simple/" + ) + + @patch("fromager.resolver.find_all_matching_from_provider") def test_resolve_auto_routes_to_prebuilt( mock_resolve: MagicMock, @@ -444,6 +585,7 @@ def test_resolve_auto_routes_to_prebuilt( # Mock package build info to return pre_built=True mock_pbi = MagicMock() mock_pbi.pre_built = True + mock_pbi.source_resolver = None mock_pbi.wheel_server_url = None mock_pbi.resolver_min_release_age = None diff --git a/tests/test_bootstrapper.py b/tests/test_bootstrapper.py index 0d775339c..5c5402ca1 100644 --- a/tests/test_bootstrapper.py +++ b/tests/test_bootstrapper.py @@ -38,7 +38,12 @@ from fromager.bootstrapper._work_item import WorkItem from fromager.context import WorkContext from fromager.dependency_graph import ROOT -from fromager.packagesettings import DownloadedSource, DownloadKind, WheelSettings +from fromager.packagesettings import ( + DownloadedSource, + DownloadKind, + PackageSettings, + WheelSettings, +) from fromager.requirements_file import RequirementType, SourceType @@ -63,6 +68,38 @@ def helper(self) -> None: ... assert _AbstractMiddle.__abstractmethods__ +def test_test_mode_source_resolver_uses_legacy_prebuilt_fallback( + tmp_context: WorkContext, +) -> None: + """A failed source build can use the forced legacy wheel fallback.""" + req = Requirement("pkg==1.0") + version = Version("1.0") + url = "https://fallback.test/pkg-1.0-py3-none-any.whl" + wheel = tmp_context.wheels_prebuilt / "pkg-1.0-py3-none-any.whl" + tmp_context.settings._package_settings[canonicalize_name("pkg")] = ( + PackageSettings.from_string("pkg", "source:\n provider: pypi-sdist\n") + ) + bt = bootstrapper.Bootstrapper(tmp_context, test_mode=True) + with ( + patch.object(bt._resolver, "resolve", return_value=[(url, version)]), + patch( + "fromager.bootstrapper._cache.wheels.download_wheel", return_value=wheel + ) as legacy, + patch("fromager.bootstrapper._cache.server.update_wheel_mirror"), + ): + result = bt._handle_test_mode_failure( + req=req, + resolved_version=version, + req_type=RequirementType.TOP_LEVEL, + build_error=RuntimeError("source build failed"), + ) + + assert result is not None + assert result.source_type is SourceType.PREBUILT + assert result.wheel_filename == wheel + legacy.assert_called_once_with(req, url, tmp_context.wheels_prebuilt) + + def test_seen(tmp_context: WorkContext) -> None: bt = bootstrapper.Bootstrapper(tmp_context) req = Requirement("testdist") diff --git a/tests/test_commands.py b/tests/test_commands.py index 6b4ae406c..29670785b 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -1,3 +1,4 @@ +import concurrent.futures import json import pathlib import subprocess @@ -5,13 +6,341 @@ from unittest.mock import Mock, patch import click +import pytest from click.testing import CliRunner from packaging.requirements import Requirement from packaging.version import Version -from fromager import packagesettings -from fromager.commands import bootstrap, build, step +from fromager import bootstrapper, packagesettings, sources, wheels +from fromager.bootstrapper._cache import bg_prepare_prebuilt +from fromager.bootstrapper._prepare_source import PrepareSource +from fromager.bootstrapper._process_install_deps import ProcessInstallDeps +from fromager.bootstrapper._start import Start +from fromager.bootstrapper._work_item import WorkItem +from fromager.commands import bootstrap, build, download_sequence, step from fromager.context import WorkContext +from fromager.requirements_file import RequirementType, SourceType + + +def _configure_package(ctx: WorkContext, yaml_str: str) -> None: + """Apply one package setting to the test context.""" + ps = packagesettings.PackageSettings.from_string("pkg", yaml_str) + ctx.settings._package_settings[ps.name] = ps + + +@pytest.mark.parametrize("provider", ["pypi-prebuilt", "hook-prebuilt"]) +def test_bootstrap_prebuilt_uses_configured_download( + tmp_context: WorkContext, provider: str +) -> None: + """Bootstrap consumes the configured wheel path and updates its mirror.""" + _configure_package(tmp_context, f"source:\n provider: {provider}\n") + req = Requirement("pkg==1.0") + version = Version("1.0") + url = "https://pkg.test/pkg-1.0-py3-none-any.whl" + wheel = tmp_context.wheels_prebuilt / "pkg-1.0-py3-none-any.whl" + with ( + patch.object(wheels, "download_prebuilt_wheel", return_value=wheel) as download, + patch("fromager.bootstrapper._cache.server.update_wheel_mirror") as mirror, + ): + result = bg_prepare_prebuilt( + tmp_context, req, RequirementType.TOP_LEVEL, version, url + ) + + assert result.wheel_filename == wheel + download.assert_called_once_with( + ctx=tmp_context, req=req, version=version, download_url=url + ) + mirror.assert_called_once_with(tmp_context) + + +def test_bootstrap_configured_prebuilt_reuses_local_wheel( + tmp_context: WorkContext, +) -> None: + """A local wheel cache hit bypasses the configured download hook.""" + _configure_package(tmp_context, "source:\n provider: hook-prebuilt\n") + req = Requirement("pkg==1.0") + version = Version("1.0") + tmp_context.wheel_server_url = "https://local.test/simple/" + url = "https://local.test/simple/pkg/pkg-1.0-py3-none-any.whl" + wheel = tmp_context.wheels_prebuilt / "pkg-1.0-py3-none-any.whl" + with ( + patch.object(wheels, "download_prebuilt_wheel") as configured, + patch.object(wheels, "download_wheel", return_value=wheel) as cached, + patch("fromager.bootstrapper._cache.server.update_wheel_mirror"), + ): + result = bg_prepare_prebuilt( + tmp_context, req, RequirementType.TOP_LEVEL, version, url + ) + + assert result.wheel_filename == wheel + cached.assert_called_once_with(req, url, tmp_context.wheels_prebuilt) + configured.assert_not_called() + + +@pytest.mark.parametrize("local_cache", [False, True]) +def test_bootstrap_prebuilt_rejects_source_only_resolver( + tmp_context: WorkContext, local_cache: bool +) -> None: + """Do not treat a configured sdist resolver as a legacy wheel source.""" + _configure_package(tmp_context, "source:\n provider: pypi-sdist\n") + req = Requirement("pkg==1.0") + if local_cache: + tmp_context.wheel_server_url = "https://local.test/simple/" + url = ( + "https://local.test/simple/pkg/pkg-1.0-py3-none-any.whl" + if local_cache + else "https://pkg.test/pkg-1.0-py3-none-any.whl" + ) + with ( + patch.object(wheels, "download_prebuilt_wheel") as configured, + patch("fromager.bootstrapper._cache.wheels.download_wheel") as legacy, + pytest.raises(ValueError, match="does not resolve prebuilt wheels"), + ): + bg_prepare_prebuilt( + tmp_context, + req, + RequirementType.TOP_LEVEL, + Version("1.0"), + url, + ) + + configured.assert_not_called() + legacy.assert_not_called() + + +@pytest.mark.parametrize("provider", ["pypi-prebuilt", "hook-prebuilt"]) +def test_build_order_replay_uses_configured_download( + tmp_context: WorkContext, provider: str +) -> None: + """A later build run selects the configured wheel download contract.""" + _configure_package(tmp_context, f"source:\n provider: {provider}\n") + req = Requirement("pkg==1.0") + version = Version("1.0") + url = "https://pkg.test/pkg-1.0-py3-none-any.whl" + wheel = tmp_context.wheels_prebuilt / "pkg-1.0-py3-none-any.whl" + with ( + patch.object(build, "_is_wheel_built", return_value=None), + patch.object(wheels, "download_prebuilt_wheel", return_value=wheel) as download, + patch.object(build.wheels, "download_wheel") as legacy, + patch.object(build.hooks, "run_prebuilt_wheel_hooks") as hooks, + patch.object(build.server, "update_wheel_mirror"), + ): + result = build._build(tmp_context, version, req, url, False, None) + + assert result.prebuilt is True + assert result.wheel_filename == wheel + download.assert_called_once_with( + ctx=tmp_context, req=req, version=version, download_url=url + ) + legacy.assert_not_called() + hooks.assert_called_once() + + +def test_configured_prebuilt_reuses_cached_wheel(tmp_context: WorkContext) -> None: + """A cache hit avoids a fresh resolver download while retaining hooks.""" + _configure_package(tmp_context, "source:\n provider: hook-prebuilt\n") + req = Requirement("pkg==1.0") + version = Version("1.0") + url = "https://pkg.test/pkg-1.0-py3-none-any.whl" + cached = tmp_context.wheels_downloads / "pkg-1.0-py3-none-any.whl" + with ( + patch.object(build, "_is_wheel_built", return_value=cached), + patch.object(wheels, "download_prebuilt_wheel") as download, + patch.object(build.hooks, "run_prebuilt_wheel_hooks") as hooks, + patch.object(build.server, "update_wheel_mirror"), + ): + result = build._build(tmp_context, version, req, url, False, None) + + assert result.wheel_filename == cached + assert result.skipped is True + download.assert_not_called() + hooks.assert_called_once() + + +def test_download_sequence_replays_configured_prebuilt( + cli_runner: CliRunner, tmp_context: WorkContext, tmp_path: pathlib.Path +) -> None: + """Download sequence uses the saved URL with the selected source resolver.""" + _configure_package(tmp_context, "source:\n provider: hook-prebuilt\n") + url = "https://pkg.test/pkg-1.0-py3-none-any.whl" + build_order = tmp_path / "build-order.json" + build_order.write_text( + json.dumps( + [{"dist": "pkg", "version": "1.0", "prebuilt": True, "source_url": url}] + ) + ) + with ( + patch.object(wheels, "download_prebuilt_wheel") as download, + patch.object(download_sequence.wheels, "download_wheel") as legacy, + ): + result = cli_runner.invoke( + download_sequence.download_sequence, + [ + str(build_order), + "https://index.test/simple", + "--include-wheels", + "--num-threads", + "1", + ], + obj=tmp_context, + ) + + assert result.exit_code == 0, result.output + download.assert_called_once_with( + ctx=tmp_context, + req=Requirement("pkg==1.0"), + version=Version("1.0"), + download_url=url, + ) + legacy.assert_not_called() + + +def test_download_sequence_rejects_stale_prebuilt_flag( + cli_runner: CliRunner, tmp_context: WorkContext, tmp_path: pathlib.Path +) -> None: + """A source-only resolver cannot replay a prebuilt build-order entry.""" + _configure_package(tmp_context, "source:\n provider: pypi-sdist\n") + build_order = tmp_path / "build-order.json" + build_order.write_text( + json.dumps( + [ + { + "dist": "pkg", + "version": "1.0", + "prebuilt": True, + "source_url": "https://pkg.test/pkg-1.0-py3-none-any.whl", + } + ] + ) + ) + with ( + patch.object(wheels, "download_prebuilt_wheel") as configured, + patch.object(download_sequence.wheels, "download_wheel") as legacy, + ): + result = cli_runner.invoke( + download_sequence.download_sequence, + [ + str(build_order), + "https://index.test/simple", + "--include-wheels", + "--num-threads", + "1", + ], + obj=tmp_context, + ) + + assert isinstance(result.exception, ValueError) + assert "does not resolve prebuilt wheels" in str(result.exception) + configured.assert_not_called() + legacy.assert_not_called() + + +def test_legacy_prebuilt_keeps_direct_download(tmp_context: WorkContext) -> None: + """Legacy pre_built still downloads into the build wheel directory.""" + _configure_package(tmp_context, "variants:\n cpu:\n pre_built: true\n") + req = Requirement("pkg==1.0") + version = Version("1.0") + url = "https://pkg.test/pkg-1.0-py3-none-any.whl" + wheel = tmp_context.wheels_build / "pkg-1.0-py3-none-any.whl" + with ( + patch.object(build, "_is_wheel_built", return_value=None), + patch.object(build.wheels, "download_wheel", return_value=wheel) as legacy, + patch.object(wheels, "download_prebuilt_wheel") as configured, + patch.object(build.hooks, "run_prebuilt_wheel_hooks"), + patch.object(build.server, "update_wheel_mirror"), + ): + result = build._build(tmp_context, version, req, url, False, None) + + assert result.prebuilt is True + assert result.wheel_filename == tmp_context.wheels_downloads / wheel.name + legacy.assert_called_once_with( + req=req, wheel_url=url, output_directory=tmp_context.wheels_build + ) + configured.assert_not_called() + + +def test_legacy_bootstrap_keeps_direct_download(tmp_context: WorkContext) -> None: + """Legacy bootstrap still uses the wheel downloader.""" + _configure_package(tmp_context, "variants:\n cpu:\n pre_built: true\n") + req = Requirement("pkg==1.0") + version = Version("1.0") + url = "https://pkg.test/pkg-1.0-py3-none-any.whl" + wheel = tmp_context.wheels_prebuilt / "pkg-1.0-py3-none-any.whl" + with ( + patch( + "fromager.bootstrapper._cache.wheels.download_wheel", return_value=wheel + ) as legacy, + patch.object(wheels, "download_prebuilt_wheel") as configured, + patch("fromager.bootstrapper._cache.server.update_wheel_mirror"), + ): + result = bg_prepare_prebuilt( + tmp_context, req, RequirementType.TOP_LEVEL, version, url + ) + + assert result.wheel_filename == wheel + legacy.assert_called_once_with(req, url, tmp_context.wheels_prebuilt) + configured.assert_not_called() + + +@pytest.mark.parametrize("provider", ["pypi-prebuilt", "hook-prebuilt"]) +def test_bootstrap_records_configured_prebuilt_in_graph_and_order( + tmp_context: WorkContext, provider: str +) -> None: + """The selected source makes bootstrap skip build phases and persist its flag.""" + _configure_package(tmp_context, f"source:\n provider: {provider}\n") + bt = bootstrapper.Bootstrapper(tmp_context) + req = Requirement("pkg==1.0") + version = Version("1.0") + url = "https://pkg.test/pkg-1.0-py3-none-any.whl" + with patch.object(bt, "resolve_versions", return_value=[(url, version)]): + bt._resolve_and_add_top_level(req) + + wi = WorkItem( + req=req, + req_type=RequirementType.TOP_LEVEL, + why_snapshot=[], + source_url=url, + resolved_version=version, + ) + next_phases = Start(wi).run(bt) + assert wi.pbi_pre_built is True + assert len(next_phases) == 1 + assert isinstance(next_phases[0], PrepareSource) + wheel = tmp_context.wheels_prebuilt / "pkg-1.0-py3-none-any.whl" + with ( + patch.object(wheels, "download_prebuilt_wheel", return_value=wheel), + patch.object(sources, "prepare_source") as prepare_source, + patch("fromager.bootstrapper._cache.server.update_wheel_mirror"), + ): + background_work = next_phases[0].background_work(bt) + assert background_work is not None + prepared = background_work() + prepare_source.assert_not_called() + future: concurrent.futures.Future[typing.Any] = concurrent.futures.Future() + future.set_result(prepared) + next_phases[0].bg_future = future + install_phases = next_phases[0].run(bt) + assert len(install_phases) == 1 + assert isinstance(install_phases[0], ProcessInstallDeps) + assert wi.build_result is not None + assert wi.build_result.source_type is SourceType.PREBUILT + with ( + patch( + "fromager.bootstrapper._process_install_deps.hooks.run_post_bootstrap_hooks" + ), + patch( + "fromager.bootstrapper._process_install_deps._get_install_dependencies", + return_value=[], + ), + patch.object(bt, "create_unresolved_work_items", return_value=[]), + ): + install_phases[0].run(bt) + bt.finalize() + + assert tmp_context.dependency_graph.nodes["pkg==1.0"].pre_built is True + build_order = json.loads(bt._build_order_filename.read_text()) + assert build_order[0]["prebuilt"] is True + assert build_order[0]["source_url"] == url def get_option_names(cmd: click.Command) -> typing.Iterable[str]: diff --git a/tests/test_packagesettings.py b/tests/test_packagesettings.py index 610f2097f..ce18db756 100644 --- a/tests/test_packagesettings.py +++ b/tests/test_packagesettings.py @@ -1255,6 +1255,31 @@ def test_pbi_source_resolver_variant_overrides_package( assert isinstance(pbi.source_resolver, PyPIPrebuiltResolver) +@pytest.mark.parametrize( + ("yaml_str", "expected"), + [ + ("source:\n provider: pypi-prebuilt\n", True), + ("source:\n provider: hook-prebuilt\n", True), + ("source:\n provider: pypi-sdist\n", False), + ("variants:\n cpu:\n pre_built: true\n", True), + ], +) +def test_pbi_prebuilt_uses_selected_source_or_legacy_variant( + tmp_path: pathlib.Path, yaml_str: str, expected: bool +) -> None: + """Read configured source capability before the legacy variant flag.""" + ps = PackageSettings.from_string("test-pkg", yaml_str) + settings = Settings( + settings=SettingsFile(), + package_settings=[ps], + variant="cpu", + patches_dir=tmp_path, + max_jobs=1, + ) + + assert settings.package_build_info("test-pkg").pre_built is expected + + # -- source vs legacy mutual exclusivity validation -------------------------- diff --git a/tests/test_wheels.py b/tests/test_wheels.py index 91bb698e2..d622940b3 100644 --- a/tests/test_wheels.py +++ b/tests/test_wheels.py @@ -9,10 +9,124 @@ from packaging.tags import Tag from packaging.version import Version -from fromager import build_environment, context, downloads, wheels +from fromager import build_environment, context, downloads, packagesettings, wheels from fromager.packagesettings import SbomSettings, Settings, SettingsFile, WheelSettings +def _configure_package(ctx: context.WorkContext, yaml_str: str) -> None: + """Apply one package setting to the test context.""" + ps = packagesettings.PackageSettings.from_string("pkg", yaml_str) + ctx.settings._package_settings[ps.name] = ps + + +@pytest.mark.parametrize("provider", ["pypi-prebuilt", "hook-prebuilt"]) +def test_configured_prebuilt_download_uses_resolver_default( + tmp_context: context.WorkContext, provider: str +) -> None: + """Both profiles use their own default when no download hook exists.""" + _configure_package(tmp_context, f"source:\n provider: {provider}\n") + req = Requirement("pkg==1.0") + url = "https://pkg.test/pkg-1.0-py3-none-any.whl" + expected = tmp_context.wheels_prebuilt / "pkg-1.0-py3-none-any.whl" + with ( + patch("fromager.overrides.find_override_method", return_value=None), + patch("fromager.downloads.download_wheel", return_value=expected) as default, + ): + result = wheels.download_prebuilt_wheel( + ctx=tmp_context, req=req, version=Version("1.0"), download_url=url + ) + + assert result == expected + default.assert_called_once_with( + destination_dir=tmp_context.wheels_prebuilt, url=url + ) + + +def test_hook_prebuilt_download_stages_hook_result( + tmp_context: context.WorkContext, +) -> None: + """Use the hook's path and make its wheel visible to the local mirror.""" + _configure_package(tmp_context, "source:\n provider: hook-prebuilt\n") + req = Requirement("pkg==1.0") + url = "https://pkg.test/pkg-1.0-py3-none-any.whl" + hook_path = tmp_context.work_dir / "pkg-1.0-py3-none-any.whl" + hook_path.write_bytes(b"wheel") + calls: list[str] = [] + + def hook(download_url: str) -> packagesettings.DownloadedSource: + calls.append(download_url) + return packagesettings.DownloadedSource( + path=hook_path, kind=packagesettings.DownloadKind.prebuilt_wheel + ) + + with ( + patch("fromager.overrides.find_override_method", return_value=hook), + patch("fromager.downloads.download_wheel") as default, + ): + result = wheels.download_prebuilt_wheel( + ctx=tmp_context, req=req, version=Version("1.0"), download_url=url + ) + + assert result == tmp_context.wheels_prebuilt / hook_path.name + assert result.read_bytes() == b"wheel" + assert calls == [url] + default.assert_not_called() + + +def test_configured_prebuilt_rejects_other_kind( + tmp_context: context.WorkContext, +) -> None: + """An invalid hook result fails without trying a different downloader.""" + _configure_package(tmp_context, "source:\n provider: hook-prebuilt\n") + req = Requirement("pkg==1.0") + + def hook() -> packagesettings.DownloadedSource: + return packagesettings.DownloadedSource( + path=tmp_context.work_dir / "pkg-1.0.tar.gz", + kind=packagesettings.DownloadKind.sdist, + ) + + with ( + patch("fromager.overrides.find_override_method", return_value=hook), + patch("fromager.downloads.download_wheel") as default, + pytest.raises(ValueError, match="unsupported kind 'sdist'"), + ): + wheels.download_prebuilt_wheel( + ctx=tmp_context, + req=req, + version=Version("1.0"), + download_url="https://pkg.test/pkg-1.0-py3-none-any.whl", + ) + default.assert_not_called() + + +def test_hook_prebuilt_failure_does_not_fall_back( + tmp_context: context.WorkContext, +) -> None: + """Keep the hook failure visible and avoid the default wheel fetch.""" + _configure_package(tmp_context, "source:\n provider: hook-prebuilt\n") + req = Requirement("pkg==1.0") + failure = OSError("hook download failed") + + def hook() -> pathlib.Path: + raise failure + + with ( + patch("fromager.overrides.find_override_method", return_value=hook), + patch("fromager.downloads.download_wheel") as default, + pytest.raises(RuntimeError, match="download_source hook failed") as exc_info, + ): + wheels.download_prebuilt_wheel( + ctx=tmp_context, + req=req, + version=Version("1.0"), + download_url="https://pkg.test/pkg-1.0-py3-none-any.whl", + ) + + assert exc_info.value.__cause__ is failure + default.assert_not_called() + + @patch("pyproject_hooks.BuildBackendHookCaller.build_wheel") def test_default_build_wheel( mock_build_wheel: Mock,