Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions docs/concepts/bootstrapper-architecture.rst
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,12 @@ Bootstrap Phase Flow

Every package passes through a sequence of phases. Source packages
traverse the full pipeline; prebuilt wheels skip the build phases.
``pypi-prebuilt`` and ``hook-prebuilt`` use the selected source resolver to
download a fresh wheel in ``PrepareSource``. The legacy ``pre_built`` setting
keeps its existing wheel download path. Both routes update the local mirror
and record the package as prebuilt in the graph and build order.
When a local wheel is already available, bootstrap reuses it before asking
the selected provider to resolve a fresh wheel.

.. code-block:: text

Expand Down
10 changes: 10 additions & 0 deletions docs/concepts/package-settings.rst
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,16 @@ cannot be combined with package-level ``download_source`` or
``resolver_dist``. An explicit ``pre_built: false`` also conflicts;
an omitted ``pre_built`` does not.

``PackageBuildInfo.pre_built`` follows the selected ``source`` resolver's
``resolves_prebuilt_wheel`` capability. Without ``source``, it reads the
legacy variant ``pre_built`` setting. Configured prebuilt resolvers download
fresh wheels through their ``download()`` contract; legacy prebuilt packages
keep the wheel downloader.

.. versionchanged:: 0.102.0
Configured prebuilt source resolvers now select the prebuilt pipeline
during bootstrap and later builds.

.. versionchanged:: 0.102.0
Source and legacy settings mixed across package and variant levels
now raise a validation error when package settings load.
Expand Down
6 changes: 5 additions & 1 deletion docs/concepts/resolver-architecture.rst
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,8 @@ the cache already contains a matching version.
The configured wheel cache server is not a version-resolution fallback.
After a version has been selected, the bootstrapper checks the cache
server separately for a previously built wheel.
For prebuilt packages, the local wheel server can supply a cached wheel
before a fresh resolution through the selected source provider.

Provider Hierarchy
------------------
Expand Down Expand Up @@ -132,7 +134,9 @@ The separate ``step prepare-source`` invocation accepts ``--source-kind`` and
``step download-source-archive --output-format json`` exposes that result's
actual path and kind without changing its default path-only output.
Without ``--source-kind``, it keeps the legacy archive behavior.
Integrating prebuilt detection into the bootstrapper remains a separate step.
Configured prebuilt resolvers select the prebuilt bootstrap path. Fresh
wheels use their ``download()`` contract; local cached wheels and the
test-mode fallback use the legacy wheel downloader.
See :ref:`download_source_hook` for the return contract and compatibility rules.

.. versionchanged:: 0.102.0
Expand Down
57 changes: 37 additions & 20 deletions src/fromager/bootstrap_requirement_resolver.py
Original file line number Diff line number Diff line change
Expand Up @@ -171,12 +171,16 @@ def _resolve_and_extend(
parent_req=parent_req,
)

source_resolver = self.ctx.package_build_info(req).source_resolver
configured_prebuilt = (
source_resolver is not None and source_resolver.resolves_prebuilt_wheel
)
if cached_resolution:
logger.debug(
f"resolved from previous bootstrap: {len(cached_resolution)} version(s)"
)
results = cached_resolution
elif pre_built:
elif pre_built and not configured_prebuilt:
# Resolve prebuilt wheel
wheel_server_urls = wheels.get_wheel_server_urls(
self.ctx, req, cache_wheel_server_url=resolver.PYPI_SERVER_URL
Expand All @@ -188,27 +192,40 @@ def _resolve_and_extend(
req_type=req_type,
)
else:
# Resolve source (sdist)
provider = sources.get_source_provider(
ctx=self.ctx,
req=req,
sdist_server_url=resolver.PYPI_SERVER_URL,
req_type=req_type,
)
max_age_cutoff = resolver._compute_max_age_cutoff(self.ctx)
age_fallback = (
resolver.AgeFallback.NEWEST
if self.multiple_versions
else resolver.AgeFallback.ALL
)
results = resolver.find_all_matching_from_provider(
provider,
req,
max_age_cutoff=max_age_cutoff,
age_fallback=age_fallback,
)
results = []
if pre_built and self.ctx.wheel_server_url:
try:
results = wheels.resolve_all_cached_wheels(
ctx=self.ctx,
req=req,
cache_server_url=self.ctx.wheel_server_url,
)
except Exception:
logger.debug("no matching wheel in local cache", exc_info=True)

if not results:
# The configured source provider resolves either source archives
# or prebuilt wheels, according to its selected capability.
provider = sources.get_source_provider(
ctx=self.ctx,
req=req,
sdist_server_url=resolver.PYPI_SERVER_URL,
req_type=req_type,
)
max_age_cutoff = resolver._compute_max_age_cutoff(self.ctx)
age_fallback = (
resolver.AgeFallback.NEWEST
if self.multiple_versions
else resolver.AgeFallback.ALL
)
results = resolver.find_all_matching_from_provider(
provider,
req,
max_age_cutoff=max_age_cutoff,
age_fallback=age_fallback,
)

if not results and not pre_built:
logger.warning(
"resolver returned no results (%s version mode)",
"multiple" if self.multiple_versions else "single",
Expand Down
7 changes: 6 additions & 1 deletion src/fromager/bootstrapper/_bootstrapper.py
Original file line number Diff line number Diff line change
Expand Up @@ -471,7 +471,12 @@ def _download_prebuilt(
wheel_url: str,
) -> tuple[pathlib.Path, pathlib.Path]:
result = _cache.bg_prepare_prebuilt(
self.ctx, req, req_type, resolved_version, wheel_url
self.ctx,
req,
req_type,
resolved_version,
wheel_url,
force_legacy_download=True,
)
assert result.wheel_filename is not None
assert result.unpack_dir is not None
Expand Down
22 changes: 20 additions & 2 deletions src/fromager/bootstrapper/_cache.py
Original file line number Diff line number Diff line change
Expand Up @@ -245,12 +245,30 @@ def bg_prepare_prebuilt(
req_type: RequirementType,
resolved_version: Version,
wheel_url: str,
*,
force_legacy_download: bool = False,
) -> PreparedSourceData:
"""Background-safe prebuilt download: no Bootstrapper state accessed."""
"""Download a prebuilt wheel, reusing local cache or test-mode fallback."""
# Thread-safe: paths include {req.name}-{resolved_version} (unique per package),
# mkdir uses exist_ok=True (atomic), and update_wheel_mirror() is already locked.
logger.info(f"using pre-built wheel for {req_type} requirement")
wheel_filename = wheels.download_wheel(req, wheel_url, ctx.wheels_prebuilt)
source_resolver = ctx.package_build_info(req).source_resolver
local_wheel_url = bool(ctx.wheel_server_url) and wheel_url.startswith(
ctx.wheel_server_url.rstrip("/") + "/"
)
if force_legacy_download:
wheel_filename = wheels.download_wheel(req, wheel_url, ctx.wheels_prebuilt)
elif source_resolver is not None and not source_resolver.resolves_prebuilt_wheel:
raise ValueError(
f"{req}: source resolver {source_resolver.provider!r} "
"does not resolve prebuilt wheels"
)
elif local_wheel_url or source_resolver is None:
wheel_filename = wheels.download_wheel(req, wheel_url, ctx.wheels_prebuilt)
else:
wheel_filename = wheels.download_prebuilt_wheel(
ctx=ctx, req=req, version=resolved_version, download_url=wheel_url
)
unpack_dir = _create_unpack_dir(ctx.work_dir, req, resolved_version)
server.update_wheel_mirror(ctx)
return PreparedSourceData(wheel_filename=wheel_filename, unpack_dir=unpack_dir)
34 changes: 24 additions & 10 deletions src/fromager/commands/build.py
Original file line number Diff line number Diff line change
Expand Up @@ -362,17 +362,30 @@ def _build(
if prebuilt:
if not wheel_filename:
logger.info("downloading prebuilt wheel")
wheel_filename = wheels.download_wheel(
req=req,
wheel_url=source_download_url,
output_directory=wkctx.wheels_build,
)
source_resolver = pbi.source_resolver
if source_resolver is not None and source_resolver.resolves_prebuilt_wheel:
wheel_filename = wheels.download_prebuilt_wheel(
ctx=wkctx,
req=req,
version=resolved_version,
download_url=source_download_url,
)
elif source_resolver is not None:
raise ValueError(
f"{req}: source resolver {source_resolver.provider!r} "
"does not resolve prebuilt wheels"
)
else:
wheel_filename = wheels.download_wheel(
req=req,
wheel_url=source_download_url,
output_directory=wkctx.wheels_build,
)
else:
# already downloaded prebuilt wheel
use_exiting_wheel = True

# Run hooks for prebuilt wheels. At this point wheel_filename should
# be set either from _is_wheel_built() or download_wheel().
# Run hooks for both cached and freshly downloaded prebuilt wheels.
hooks.run_prebuilt_wheel_hooks(
ctx=wkctx,
req=req,
Expand Down Expand Up @@ -448,9 +461,10 @@ def _build(

server.update_wheel_mirror(wkctx)

# After we update the wheel mirror, the built file has
# moved to a new directory.
wheel_filename = wkctx.wheels_downloads / wheel_filename.name
# The mirror moves wheels from the build directory to downloads;
# configured prebuilt wheels stay in the prebuilt directory.
if wheel_filename.parent != wkctx.wheels_prebuilt:
wheel_filename = wkctx.wheels_downloads / wheel_filename.name

return BuildSequenceEntry(
name=canonicalize_name(req.name),
Expand Down
22 changes: 21 additions & 1 deletion src/fromager/commands/download_sequence.py
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,27 @@ def download_one(entry: dict[str, typing.Any]) -> None:

if entry["prebuilt"]:
if include_wheels:
wheels.download_wheel(req, entry["source_url"], wkctx.wheels_prebuilt)
source_resolver = wkctx.package_build_info(req).source_resolver
if (
source_resolver is not None
and source_resolver.resolves_prebuilt_wheel
):
wheels.download_prebuilt_wheel(
ctx=wkctx,
req=req,
version=Version(entry["version"]),
download_url=entry["source_url"],
)
elif source_resolver is not None:
raise ValueError(
f"{req}: build order marks a prebuilt wheel, but "
f"source resolver {source_resolver.provider!r} "
"does not resolve prebuilt wheels"
)
else:
wheels.download_wheel(
req, entry["source_url"], wkctx.wheels_prebuilt
)
else:
logger.info(f"{entry['dist']}: uses a pre-built wheel, skipping")
return
Expand Down
5 changes: 4 additions & 1 deletion src/fromager/packagesettings/_pbi.py
Original file line number Diff line number Diff line change
Expand Up @@ -163,7 +163,10 @@ def has_customizations(self) -> bool:

@property
def pre_built(self) -> bool:
"""Does the variant use pre-build wheels?"""
"""Whether the selected source or legacy variant uses prebuilt wheels."""
source_resolver = self.source_resolver
if source_resolver is not None:
return source_resolver.resolves_prebuilt_wheel
vi = self._ps.variants.get(self.variant)
if vi is not None:
return vi.pre_built
Expand Down
25 changes: 25 additions & 0 deletions src/fromager/wheels.py
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@
requirements_file,
resolver,
sbom,
sources,
)
from .pkgmetadata.pep376 import verbatim_dist_name

Expand Down Expand Up @@ -520,6 +521,30 @@ def download_wheel(
return wheel_filename


def download_prebuilt_wheel(
*,
ctx: context.WorkContext,
req: Requirement,
version: Version,
download_url: str,
) -> pathlib.Path:
"""Download a configured prebuilt wheel and stage it for the wheel mirror."""
downloaded: packagesettings.DownloadedSource = sources.download_source(
ctx=ctx, req=req, version=version, download_url=download_url
)
if downloaded.kind is not packagesettings.DownloadKind.prebuilt_wheel:
raise ValueError(
f"{req}: configured prebuilt source returned {downloaded.kind.value!r}; "
"expected 'prebuilt_wheel'"
)
wheel_path = downloaded.path
if wheel_path.parent != ctx.wheels_prebuilt:
staged_path = ctx.wheels_prebuilt / wheel_path.name
shutil.copy2(wheel_path, staged_path)
return staged_path
return wheel_path


def get_wheel_server_urls(
ctx: context.WorkContext, req: Requirement, *, cache_wheel_server_url: str | None
) -> list[str]:
Expand Down
Loading
Loading