fix(pypi): fetch retained wheel archives lazily

.bazelrc.deleted_packages

@@ -45,6 +45,7 @@ common --deleted_packages=tests/integration/unified_pypi
 common --deleted_packages=tests/integration/uv_lock
 common --deleted_packages=tests/integration/validate_test_main
 common --deleted_packages=tests/integration/whl_library
+common --deleted_packages=tests/integration/whl_storage
 common --deleted_packages=tests/modules/another_module
 common --deleted_packages=tests/modules/other
 common --deleted_packages=tests/modules/other/nspkg_delta

news/lazy-wheel-archives.changed.md

@@ -0,0 +1,4 @@
+Downloaded wheels are removed after extraction when their original archive can
+be fetched independently. The existing `whl` and `whl_file` targets fetch the
+archive from a separate repository on demand. Patched wheels and wheels built
+from source retain their existing archive behavior.

python/private/pypi/BUILD.bazel

@@ -481,6 +481,7 @@ bzl_library(
     deps = [
         ":pip_archive",
         ":whl_archive",
+        ":whl_download",
     ],
 )
 
@@ -555,8 +556,19 @@ bzl_library(
     deps = [
         ":attrs",
         ":patch_and_extract_whl",
-        ":urllib",
         ":whl_deps_repo",
+        ":whl_download",
+        "//python/private:auth",
+        "//python/private:repo_utils",
+    ],
+)
+
+bzl_library(
+    name = "whl_download",
+    srcs = ["whl_download.bzl"],
+    deps = [
+        ":attrs",
+        ":urllib",
         "//python/private:auth",
         "//python/private:repo_utils",
     ],

python/private/pypi/patch_and_extract_whl.bzl

@@ -69,7 +69,7 @@ def _remove_files(rctx, *basenames):
         elif path.is_dir:
             paths.extend(path.readdir())
 
-def patch_and_extract_whl(rctx, *, whl_path, logger, sdist_filename = None):
+def patch_and_extract_whl(rctx, *, whl_path, logger, sdist_filename = None, whl_file = None):
     """Extract the wheel, apply patches and generate BUILD.bazel files.
 
     Reused in pip and http wheel download code.
@@ -79,6 +79,7 @@ def patch_and_extract_whl(rctx, *, whl_path, logger, sdist_filename = None):
         whl_path: the whl path to extract.
         logger: The logger to use
         sdist_filename: The filename to ignore in the BUILD.bazel files as sources.
+        whl_file: The label of an externally stored wheel, if any.
 
     Returns:
         The repository metadata if the extraction is reproducible
@@ -91,15 +92,21 @@ def patch_and_extract_whl(rctx, *, whl_path, logger, sdist_filename = None):
                 patches[patch_file] = patch_dst.patch_strip
 
         if patches:
+            # patch_whl deletes its input before repacking. Never delete an
+            # archive owned by another repository.
+            if whl_path.dirname != rctx.path("."):
+                rctx.symlink(whl_path, whl_path.basename)
+                whl_path = rctx.path(whl_path.basename)
             whl_path = patch_whl(
                 rctx,
                 whl_path = whl_path,
                 patches = patches,
             )
+            whl_file = None
 
     whl_extract(rctx, whl_path = whl_path, logger = logger)
 
-    install_dir_path = whl_path.dirname.get_child("site-packages")
+    install_dir_path = rctx.path("site-packages")
     metadata = whl_metadata(
         install_dir = install_dir_path,
         read_fn = rctx.read,
@@ -139,8 +146,14 @@ def patch_and_extract_whl(rctx, *, whl_path, logger, sdist_filename = None):
             metadata = metadata,
             filename = sdist_filename or whl_path.basename,
         ),
+        **({"whl_file": str(whl_file)} if whl_file else {})
     )
 
+    # The generated wheel target can obtain the original archive independently.
+    # Do not retain the temporary download or delete a caller-owned wheel.
+    if whl_file and whl_path.dirname == rctx.path("."):
+        rctx.delete(whl_path)
+
     # Delete these in case the wheel had them. They generally don't cause
     # a problem, but let's avoid the chance of that happening.
     rctx.file("WORKSPACE")

python/private/pypi/whl_archive.bzl

@@ -1,11 +1,11 @@
 ""
 
-load("//python/private:auth.bzl", "AUTH_ATTRS", "get_auth")
+load("//python/private:auth.bzl", "AUTH_ATTRS")
 load("//python/private:repo_utils.bzl", "REPO_DEBUG_ENV_VAR", "repo_utils")
 load(":attrs.bzl", "ATTRS")
 load(":patch_and_extract_whl.bzl", "patch_and_extract_whl")
-load(":urllib.bzl", "urllib")
 load(":whl_deps_repo.bzl", "whl_deps_attrs")
+load(":whl_download.bzl", "download_whl")
 
 def _whl_archive_impl(rctx):
     logger = repo_utils.logger(rctx)
@@ -14,47 +14,17 @@ def _whl_archive_impl(rctx):
     if rctx.attr.whl_file:
         rctx.watch(rctx.attr.whl_file)
         whl_path = rctx.path(rctx.attr.whl_file)
-
-        # Simulate the behaviour where the whl is present in the current directory.
-        rctx.symlink(whl_path, whl_path.basename)
-        whl_path = rctx.path(whl_path.basename)
     elif rctx.attr.urls and rctx.attr.filename:
-        filename = rctx.attr.filename
-        urls = rctx.attr.urls
-        urls = [
-            urllib.absolute_url(
-                rctx.attr.index_url,
-                url,
-                envsubst = rctx.attr.envsubst,
-                getenv = rctx.getenv,
-            )
-            for url in urls
-        ]
-        result = rctx.download(
-            url = urls,
-            output = filename,
-            sha256 = rctx.attr.sha256,
-            integrity = rctx.attr.integrity if not rctx.attr.sha256 else "",
-            auth = get_auth(rctx, urls),
-        )
-        if not rctx.attr.sha256 and not rctx.attr.integrity:
-            # this is only seen when there is a direct URL reference without a hash
-            logger.warn("Please update the requirement line to include the hash:\n{} \\\n    --hash=sha256:{}".format(
-                rctx.attr.requirement,
-                result.sha256,
-            ))
-
-        if not result.success:
-            fail("could not download the '{}' from {}:\n{}".format(filename, urls, result))
-
-        if filename.endswith(".whl"):
-            whl_path = rctx.path(filename)
-        else:
-            fail("Only wheels are supported")
+        whl_path = download_whl(rctx, logger = logger)
     else:
         fail("Either 'whl_file' or 'urls' and 'filename' needs to be specified")
 
-    return patch_and_extract_whl(rctx, whl_path = whl_path, logger = logger)
+    return patch_and_extract_whl(
+        rctx,
+        whl_path = whl_path,
+        whl_file = rctx.attr.whl_file or rctx.attr.downloaded_whl,
+        logger = logger,
+    )
 
 whl_archive_attrs = whl_deps_attrs | {
     "annotation": attr.label(
@@ -64,6 +34,9 @@ whl_archive_attrs = whl_deps_attrs | {
         ),
         allow_files = True,
     ),
+    "downloaded_whl": attr.string(
+        doc = "Label of the lazy archive target, emitted in BUILD.bazel without fetching it. INTERNAL USE ONLY.",
+    ),
     "filename": attr.string(
         doc = "Download the whl file to this filename. Only used when the `urls` is passed. If not specified, will be auto-detected from the `urls`.",
     ),

python/private/pypi/whl_download.bzl

@@ -0,0 +1,69 @@
+"""Download a wheel without retaining it in the extracted repository."""
+
+load("//python/private:auth.bzl", "AUTH_ATTRS", "get_auth")
+load("//python/private:repo_utils.bzl", "REPO_DEBUG_ENV_VAR", "repo_utils")
+load(":attrs.bzl", "ATTRS")
+load(":urllib.bzl", "urllib")
+
+def download_whl(rctx, *, logger):
+    """Download a wheel and return its path, preserving index authentication.
+
+    Args:
+        rctx: The repository context.
+        logger: The repository logger.
+
+    Returns:
+        The path of the downloaded wheel.
+    """
+    filename = rctx.attr.filename
+    if not filename.endswith(".whl"):
+        fail("Only wheels are supported")
+    urls = [
+        urllib.absolute_url(
+            rctx.attr.index_url,
+            url,
+            envsubst = rctx.attr.envsubst,
+            getenv = rctx.getenv,
+        )
+        for url in rctx.attr.urls
+    ]
+    result = rctx.download(
+        url = urls,
+        output = filename,
+        sha256 = rctx.attr.sha256,
+        integrity = rctx.attr.integrity if not rctx.attr.sha256 else "",
+        auth = get_auth(rctx, urls),
+    )
+    if not rctx.attr.sha256 and not rctx.attr.integrity:
+        logger.warn("Please update the requirement line to include the hash:\n{} \\\n    --hash=sha256:{}".format(
+            rctx.attr.requirement,
+            result.sha256,
+        ))
+    if not result.success:
+        fail("could not download the '{}' from {}:\n{}".format(filename, urls, result))
+    return rctx.path(filename)
+
+whl_download_attrs = {
+    "envsubst": ATTRS["envsubst"],
+    "filename": attr.string(doc = "The filename of the downloaded wheel."),
+    "index_url": attr.string(doc = "The index URL used to resolve relative wheel URLs."),
+    "integrity": attr.string(doc = "The expected SRI checksum, used when sha256 is unset."),
+    "requirement": attr.string(doc = "The requirement used in download diagnostics."),
+    "sha256": attr.string(doc = "The expected SHA256 checksum."),
+    "urls": attr.string_list(doc = "The wheel download URLs."),
+} | AUTH_ATTRS
+
+def _whl_download_impl(rctx):
+    whl_path = download_whl(rctx, logger = repo_utils.logger(rctx))
+    rctx.file("BUILD.bazel", "exports_files([{}])\n".format(repr(whl_path.basename)))
+    if hasattr(rctx, "repo_metadata"):
+        return rctx.repo_metadata(reproducible = True)
+    return None
+
+whl_download = repository_rule(
+    implementation = _whl_download_impl,
+    attrs = whl_download_attrs | {
+        "_rule_name": attr.string(default = "whl_download"),
+    },
+    environ = [REPO_DEBUG_ENV_VAR],
+)

python/private/pypi/whl_library.bzl

@@ -16,6 +16,7 @@
 
 load(":pip_archive.bzl", "pip_archive", "pip_archive_attrs")
 load(":whl_archive.bzl", "whl_archive", "whl_archive_attrs")
+load(":whl_download.bzl", "whl_download", "whl_download_attrs")
 
 def _filter(kwargs, subset, debug = False):
     dropped = {}
@@ -38,6 +39,10 @@ def whl_library(name, repo = None, **kwargs):
     * {obj}`whl_archive`
     * {obj}`pip_archive`
 
+    URL wheels are downloaded temporarily for extraction, then removed.
+    A sibling `<name>_whl` repository supplies the original archive lazily,
+    when a wheel target is requested. Extraction does not depend on it.
+
     Args:
         name: {type}`str` The name of the repo.
         repo: Unused, will be dropped in the next major release.
@@ -56,6 +61,9 @@ def whl_library(name, repo = None, **kwargs):
     kwargs.setdefault("dep_template", "@{}{{name}}//:{{target}}".format(kwargs.pop("repo_prefix", "")))
 
     if whl_file or (urls and filename and filename.endswith(".whl")):
+        if not whl_file:
+            whl_download(name = name + "_whl", **_filter(kwargs, whl_download_attrs))
+            kwargs["downloaded_whl"] = "@{}//:{}".format(name + "_whl", filename)
         whl_archive(name = name, **_filter(kwargs, whl_archive_attrs))
     else:
         pip_archive(name = name, **_filter(kwargs, pip_archive_attrs))

python/private/pypi/whl_library_srcs.bzl

@@ -31,6 +31,7 @@ _VENV_SITE_PACKAGES_FLAG = Label("//python/config_settings:venvs_site_packages")
 def whl_library_srcs(
         *,
         name,
+        whl_file = None,
         sdist_filename = None,
         data_exclude = [],
         srcs_exclude = [],
@@ -57,6 +58,7 @@ def whl_library_srcs(
     Args:
         name: {type}`str` The file to match for including it into the `whl`
             filegroup. This may be also parsed to generate extra metadata.
+        whl_file: {type}`str | None` The label of an externally stored wheel.
         sdist_filename: {type}`str | None` If the wheel was built from an sdist,
             the filename of the sdist.
         visibility: {type}`list[str]` The visibility of the source targets.
@@ -178,7 +180,7 @@ def whl_library_srcs(
     if hasattr(native, "filegroup"):
         native.filegroup(
             name = WHEEL_FILE,
-            srcs = [name],
+            srcs = [whl_file or name],
             visibility = visibility,
         )
 

python/private/pypi/whl_library_targets.bzl

@@ -23,6 +23,7 @@ whl_library_srcs = _whl_library_srcs
 def whl_library_targets(
         *,
         name,
+        whl_file = None,
         metadata_name = "",
         requires_dist = [],
         extras = [],
@@ -46,6 +47,7 @@ def whl_library_targets(
 
     Args:
         name: {type}`str` The wheel filename
+        whl_file: {type}`str | None` The label of a wheel stored in another repository.
         metadata_name: {type}`str` The package name as written in wheel `METADATA`.
         group_deps: {type}`list[str]` names of fellow members of the group (if
             any). These will be excluded from generated deps lists so as to avoid
@@ -72,6 +74,7 @@ def whl_library_targets(
     """
     whl_library_srcs(
         name = name,
+        whl_file = whl_file,
         sdist_filename = sdist_filename,
         data_exclude = data_exclude,
         srcs_exclude = srcs_exclude,

tests/integration/BUILD.bazel

@@ -143,6 +143,11 @@ rules_python_integration_test(
     name = "whl_library_test",
 )
 
+rules_python_integration_test(
+    name = "whl_storage_test",
+    py_main = "whl_storage_test.py",
+)
+
 py_library(
     name = "runner_lib",
     srcs = ["runner.py"],

tests/integration/whl_storage/BUILD.bazel

@@ -0,0 +1,33 @@
+load("@rules_python//python:py_binary.bzl", "py_binary")
+
+exports_files([
+    "storage_pkg-1.0-py3-none-any.whl",
+    "wheel.sha256",
+    "modify_pkg.patch",
+    "uv.lock",
+])
+
+[
+    py_binary(
+        name = name,
+        srcs = ["main.py"],
+        legacy_create_init = 0,
+        main = "main.py",
+        deps = ["@{}//:pkg".format(name)],
+    )
+    for name in [
+        "plain",
+        "patched",
+        "nonmatching",
+        "provided",
+        "provided_patched",
+    ]
+]
+
+py_binary(
+    name = "uv_locked",
+    srcs = ["main.py"],
+    legacy_create_init = 0,
+    main = "main.py",
+    deps = ["@uv_pip//storage_pkg"],
+)

tests/integration/whl_storage/MODULE.bazel

@@ -0,0 +1,22 @@
+module(name = "integration_test")
+
+bazel_dep(name = "package_metadata", version = "0.0.7")
+bazel_dep(name = "rules_python", version = "0.0.0")
+local_path_override(
+    module_name = "rules_python",
+    path = "../../..",
+)
+
+python = use_extension("@rules_python//python/extensions:python.bzl", "python")
+python.toolchain(python_version = "3.14")
+
+wheels = use_extension("//:wheels.bzl", "wheels")
+use_repo(wheels, "nonmatching", "patched", "plain", "provided", "provided_patched")
+
+pip = use_extension("@rules_python//python/extensions:pip.bzl", "pip")
+pip.parse(
+    hub_name = "uv_pip",
+    python_version = "3.14",
+    uv_lock = "//:uv.lock",
+)
+use_repo(pip, "uv_pip")

tests/integration/whl_storage/main.py

@@ -0,0 +1,3 @@
+import storage_pkg
+
+print(f"PATCHED={storage_pkg.PATCHED}")