From 420ac8d5cf254a9f6fca7e96f95fcbdbe495c3bf Mon Sep 17 00:00:00 2001 From: Charley Saint Date: Thu, 22 Oct 2026 11:28:09 -0700 Subject: [PATCH 1/5] fix(pypi): fetch retained wheel archives lazily --- .bazelrc.deleted_packages | 1 + news/lazy-wheel-archives.changed.md | 4 + python/private/pypi/BUILD.bazel | 14 ++- python/private/pypi/patch_and_extract_whl.bzl | 17 ++- python/private/pypi/whl_archive.bzl | 51 +++------ python/private/pypi/whl_download.bzl | 69 ++++++++++++ python/private/pypi/whl_library.bzl | 8 ++ python/private/pypi/whl_library_srcs.bzl | 4 +- python/private/pypi/whl_library_targets.bzl | 3 + tests/integration/BUILD.bazel | 5 + tests/integration/whl_storage/BUILD.bazel | 24 +++++ tests/integration/whl_storage/MODULE.bazel | 14 +++ tests/integration/whl_storage/WORKSPACE | 0 tests/integration/whl_storage/main.py | 3 + .../integration/whl_storage/modify_pkg.patch | 5 + tests/integration/whl_storage/wheels.bzl | 38 +++++++ tests/integration/whl_storage_test.py | 101 ++++++++++++++++++ tests/pypi/integration/BUILD.bazel | 3 + tests/pypi/integration/whl_storage_tests.bzl | 44 ++++++++ .../whl_library_srcs_tests.bzl | 20 ++++ 20 files changed, 385 insertions(+), 43 deletions(-) create mode 100644 news/lazy-wheel-archives.changed.md create mode 100644 python/private/pypi/whl_download.bzl create mode 100644 tests/integration/whl_storage/BUILD.bazel create mode 100644 tests/integration/whl_storage/MODULE.bazel create mode 100644 tests/integration/whl_storage/WORKSPACE create mode 100644 tests/integration/whl_storage/main.py create mode 100644 tests/integration/whl_storage/modify_pkg.patch create mode 100644 tests/integration/whl_storage/wheels.bzl create mode 100644 tests/integration/whl_storage_test.py create mode 100644 tests/pypi/integration/whl_storage_tests.bzl diff --git a/.bazelrc.deleted_packages b/.bazelrc.deleted_packages index 3eec28298e..147d57a222 100644 --- a/.bazelrc.deleted_packages +++ b/.bazelrc.deleted_packages @@ -45,6 +45,7 @@ common --deleted_packages=tests/integration/unified_pypi common --deleted_packages=tests/integration/uv_lock common --deleted_packages=tests/integration/validate_test_main common --deleted_packages=tests/integration/whl_library +common --deleted_packages=tests/integration/whl_storage common --deleted_packages=tests/modules/another_module common --deleted_packages=tests/modules/other common --deleted_packages=tests/modules/other/nspkg_delta diff --git a/news/lazy-wheel-archives.changed.md b/news/lazy-wheel-archives.changed.md new file mode 100644 index 0000000000..493ff56178 --- /dev/null +++ b/news/lazy-wheel-archives.changed.md @@ -0,0 +1,4 @@ +Downloaded wheels are removed after extraction when their original archive can +be fetched independently. The existing `whl` and `whl_file` targets fetch the +archive from a separate repository on demand. Patched wheels and wheels built +from source retain their existing archive behavior. diff --git a/python/private/pypi/BUILD.bazel b/python/private/pypi/BUILD.bazel index 96b68e7796..032a2caafa 100644 --- a/python/private/pypi/BUILD.bazel +++ b/python/private/pypi/BUILD.bazel @@ -481,6 +481,7 @@ bzl_library( deps = [ ":pip_archive", ":whl_archive", + ":whl_download", ], ) @@ -555,8 +556,19 @@ bzl_library( deps = [ ":attrs", ":patch_and_extract_whl", - ":urllib", ":whl_deps_repo", + ":whl_download", + "//python/private:auth", + "//python/private:repo_utils", + ], +) + +bzl_library( + name = "whl_download", + srcs = ["whl_download.bzl"], + deps = [ + ":attrs", + ":urllib", "//python/private:auth", "//python/private:repo_utils", ], diff --git a/python/private/pypi/patch_and_extract_whl.bzl b/python/private/pypi/patch_and_extract_whl.bzl index b4ecb2f5b5..594840601b 100644 --- a/python/private/pypi/patch_and_extract_whl.bzl +++ b/python/private/pypi/patch_and_extract_whl.bzl @@ -69,7 +69,7 @@ def _remove_files(rctx, *basenames): elif path.is_dir: paths.extend(path.readdir()) -def patch_and_extract_whl(rctx, *, whl_path, logger, sdist_filename = None): +def patch_and_extract_whl(rctx, *, whl_path, logger, sdist_filename = None, whl_file = None): """Extract the wheel, apply patches and generate BUILD.bazel files. Reused in pip and http wheel download code. @@ -79,6 +79,7 @@ def patch_and_extract_whl(rctx, *, whl_path, logger, sdist_filename = None): whl_path: the whl path to extract. logger: The logger to use sdist_filename: The filename to ignore in the BUILD.bazel files as sources. + whl_file: The label of an externally stored wheel, if any. Returns: The repository metadata if the extraction is reproducible @@ -91,15 +92,21 @@ def patch_and_extract_whl(rctx, *, whl_path, logger, sdist_filename = None): patches[patch_file] = patch_dst.patch_strip if patches: + # patch_whl deletes its input before repacking. Never delete an + # archive owned by another repository. + if whl_path.dirname != rctx.path("."): + rctx.symlink(whl_path, whl_path.basename) + whl_path = rctx.path(whl_path.basename) whl_path = patch_whl( rctx, whl_path = whl_path, patches = patches, ) + whl_file = None whl_extract(rctx, whl_path = whl_path, logger = logger) - install_dir_path = whl_path.dirname.get_child("site-packages") + install_dir_path = rctx.path("site-packages") metadata = whl_metadata( install_dir = install_dir_path, read_fn = rctx.read, @@ -139,8 +146,14 @@ def patch_and_extract_whl(rctx, *, whl_path, logger, sdist_filename = None): metadata = metadata, filename = sdist_filename or whl_path.basename, ), + **({"whl_file": str(whl_file)} if whl_file else {}) ) + # The generated wheel target can obtain the original archive independently. + # Do not retain the temporary download or delete a caller-owned wheel. + if whl_file and whl_path.dirname == rctx.path("."): + rctx.delete(whl_path) + # Delete these in case the wheel had them. They generally don't cause # a problem, but let's avoid the chance of that happening. rctx.file("WORKSPACE") diff --git a/python/private/pypi/whl_archive.bzl b/python/private/pypi/whl_archive.bzl index 242104f3c2..9c0e8d559c 100644 --- a/python/private/pypi/whl_archive.bzl +++ b/python/private/pypi/whl_archive.bzl @@ -1,11 +1,11 @@ "" -load("//python/private:auth.bzl", "AUTH_ATTRS", "get_auth") +load("//python/private:auth.bzl", "AUTH_ATTRS") load("//python/private:repo_utils.bzl", "REPO_DEBUG_ENV_VAR", "repo_utils") load(":attrs.bzl", "ATTRS") load(":patch_and_extract_whl.bzl", "patch_and_extract_whl") -load(":urllib.bzl", "urllib") load(":whl_deps_repo.bzl", "whl_deps_attrs") +load(":whl_download.bzl", "download_whl") def _whl_archive_impl(rctx): logger = repo_utils.logger(rctx) @@ -14,47 +14,17 @@ def _whl_archive_impl(rctx): if rctx.attr.whl_file: rctx.watch(rctx.attr.whl_file) whl_path = rctx.path(rctx.attr.whl_file) - - # Simulate the behaviour where the whl is present in the current directory. - rctx.symlink(whl_path, whl_path.basename) - whl_path = rctx.path(whl_path.basename) elif rctx.attr.urls and rctx.attr.filename: - filename = rctx.attr.filename - urls = rctx.attr.urls - urls = [ - urllib.absolute_url( - rctx.attr.index_url, - url, - envsubst = rctx.attr.envsubst, - getenv = rctx.getenv, - ) - for url in urls - ] - result = rctx.download( - url = urls, - output = filename, - sha256 = rctx.attr.sha256, - integrity = rctx.attr.integrity if not rctx.attr.sha256 else "", - auth = get_auth(rctx, urls), - ) - if not rctx.attr.sha256 and not rctx.attr.integrity: - # this is only seen when there is a direct URL reference without a hash - logger.warn("Please update the requirement line to include the hash:\n{} \\\n --hash=sha256:{}".format( - rctx.attr.requirement, - result.sha256, - )) - - if not result.success: - fail("could not download the '{}' from {}:\n{}".format(filename, urls, result)) - - if filename.endswith(".whl"): - whl_path = rctx.path(filename) - else: - fail("Only wheels are supported") + whl_path = download_whl(rctx, logger = logger) else: fail("Either 'whl_file' or 'urls' and 'filename' needs to be specified") - return patch_and_extract_whl(rctx, whl_path = whl_path, logger = logger) + return patch_and_extract_whl( + rctx, + whl_path = whl_path, + whl_file = rctx.attr.whl_file or rctx.attr.downloaded_whl, + logger = logger, + ) whl_archive_attrs = whl_deps_attrs | { "annotation": attr.label( @@ -64,6 +34,9 @@ whl_archive_attrs = whl_deps_attrs | { ), allow_files = True, ), + "downloaded_whl": attr.string( + doc = "Label of the lazy archive target, emitted in BUILD.bazel without fetching it. INTERNAL USE ONLY.", + ), "filename": attr.string( doc = "Download the whl file to this filename. Only used when the `urls` is passed. If not specified, will be auto-detected from the `urls`.", ), diff --git a/python/private/pypi/whl_download.bzl b/python/private/pypi/whl_download.bzl new file mode 100644 index 0000000000..1017f75532 --- /dev/null +++ b/python/private/pypi/whl_download.bzl @@ -0,0 +1,69 @@ +"""Download a wheel without retaining it in the extracted repository.""" + +load("//python/private:auth.bzl", "AUTH_ATTRS", "get_auth") +load("//python/private:repo_utils.bzl", "REPO_DEBUG_ENV_VAR", "repo_utils") +load(":attrs.bzl", "ATTRS") +load(":urllib.bzl", "urllib") + +def download_whl(rctx, *, logger): + """Download a wheel and return its path, preserving index authentication. + + Args: + rctx: The repository context. + logger: The repository logger. + + Returns: + The path of the downloaded wheel. + """ + filename = rctx.attr.filename + if not filename.endswith(".whl"): + fail("Only wheels are supported") + urls = [ + urllib.absolute_url( + rctx.attr.index_url, + url, + envsubst = rctx.attr.envsubst, + getenv = rctx.getenv, + ) + for url in rctx.attr.urls + ] + result = rctx.download( + url = urls, + output = filename, + sha256 = rctx.attr.sha256, + integrity = rctx.attr.integrity if not rctx.attr.sha256 else "", + auth = get_auth(rctx, urls), + ) + if not rctx.attr.sha256 and not rctx.attr.integrity: + logger.warn("Please update the requirement line to include the hash:\n{} \\\n --hash=sha256:{}".format( + rctx.attr.requirement, + result.sha256, + )) + if not result.success: + fail("could not download the '{}' from {}:\n{}".format(filename, urls, result)) + return rctx.path(filename) + +whl_download_attrs = { + "envsubst": ATTRS["envsubst"], + "filename": attr.string(doc = "The filename of the downloaded wheel."), + "index_url": attr.string(doc = "The index URL used to resolve relative wheel URLs."), + "integrity": attr.string(doc = "The expected SRI checksum, used when sha256 is unset."), + "requirement": attr.string(doc = "The requirement used in download diagnostics."), + "sha256": attr.string(doc = "The expected SHA256 checksum."), + "urls": attr.string_list(doc = "The wheel download URLs."), +} | AUTH_ATTRS + +def _whl_download_impl(rctx): + whl_path = download_whl(rctx, logger = repo_utils.logger(rctx)) + rctx.file("BUILD.bazel", "exports_files([{}])\n".format(repr(whl_path.basename))) + if hasattr(rctx, "repo_metadata"): + return rctx.repo_metadata(reproducible = True) + return None + +whl_download = repository_rule( + implementation = _whl_download_impl, + attrs = whl_download_attrs | { + "_rule_name": attr.string(default = "whl_download"), + }, + environ = [REPO_DEBUG_ENV_VAR], +) diff --git a/python/private/pypi/whl_library.bzl b/python/private/pypi/whl_library.bzl index 3f874b464b..399a55b5d2 100644 --- a/python/private/pypi/whl_library.bzl +++ b/python/private/pypi/whl_library.bzl @@ -16,6 +16,7 @@ load(":pip_archive.bzl", "pip_archive", "pip_archive_attrs") load(":whl_archive.bzl", "whl_archive", "whl_archive_attrs") +load(":whl_download.bzl", "whl_download", "whl_download_attrs") def _filter(kwargs, subset, debug = False): dropped = {} @@ -38,6 +39,10 @@ def whl_library(name, repo = None, **kwargs): * {obj}`whl_archive` * {obj}`pip_archive` + URL wheels are downloaded temporarily for extraction, then removed. + A sibling `_whl` repository supplies the original archive lazily, + when a wheel target is requested. Extraction does not depend on it. + Args: name: {type}`str` The name of the repo. repo: Unused, will be dropped in the next major release. @@ -56,6 +61,9 @@ def whl_library(name, repo = None, **kwargs): kwargs.setdefault("dep_template", "@{}{{name}}//:{{target}}".format(kwargs.pop("repo_prefix", ""))) if whl_file or (urls and filename and filename.endswith(".whl")): + if not whl_file: + whl_download(name = name + "_whl", **_filter(kwargs, whl_download_attrs)) + kwargs["downloaded_whl"] = "@{}//:{}".format(name + "_whl", filename) whl_archive(name = name, **_filter(kwargs, whl_archive_attrs)) else: pip_archive(name = name, **_filter(kwargs, pip_archive_attrs)) diff --git a/python/private/pypi/whl_library_srcs.bzl b/python/private/pypi/whl_library_srcs.bzl index 0556219c93..44d47eb250 100644 --- a/python/private/pypi/whl_library_srcs.bzl +++ b/python/private/pypi/whl_library_srcs.bzl @@ -31,6 +31,7 @@ _VENV_SITE_PACKAGES_FLAG = Label("//python/config_settings:venvs_site_packages") def whl_library_srcs( *, name, + whl_file = None, sdist_filename = None, data_exclude = [], srcs_exclude = [], @@ -57,6 +58,7 @@ def whl_library_srcs( Args: name: {type}`str` The file to match for including it into the `whl` filegroup. This may be also parsed to generate extra metadata. + whl_file: {type}`str | None` The label of an externally stored wheel. sdist_filename: {type}`str | None` If the wheel was built from an sdist, the filename of the sdist. visibility: {type}`list[str]` The visibility of the source targets. @@ -178,7 +180,7 @@ def whl_library_srcs( if hasattr(native, "filegroup"): native.filegroup( name = WHEEL_FILE, - srcs = [name], + srcs = [whl_file or name], visibility = visibility, ) diff --git a/python/private/pypi/whl_library_targets.bzl b/python/private/pypi/whl_library_targets.bzl index 4f7bf8e98a..a1cc727460 100644 --- a/python/private/pypi/whl_library_targets.bzl +++ b/python/private/pypi/whl_library_targets.bzl @@ -23,6 +23,7 @@ whl_library_srcs = _whl_library_srcs def whl_library_targets( *, name, + whl_file = None, metadata_name = "", requires_dist = [], extras = [], @@ -46,6 +47,7 @@ def whl_library_targets( Args: name: {type}`str` The wheel filename + whl_file: {type}`str | None` The label of a wheel stored in another repository. metadata_name: {type}`str` The package name as written in wheel `METADATA`. group_deps: {type}`list[str]` names of fellow members of the group (if any). These will be excluded from generated deps lists so as to avoid @@ -72,6 +74,7 @@ def whl_library_targets( """ whl_library_srcs( name = name, + whl_file = whl_file, sdist_filename = sdist_filename, data_exclude = data_exclude, srcs_exclude = srcs_exclude, diff --git a/tests/integration/BUILD.bazel b/tests/integration/BUILD.bazel index 2eae8dfecb..93df40967d 100644 --- a/tests/integration/BUILD.bazel +++ b/tests/integration/BUILD.bazel @@ -143,6 +143,11 @@ rules_python_integration_test( name = "whl_library_test", ) +rules_python_integration_test( + name = "whl_storage_test", + py_main = "whl_storage_test.py", +) + py_library( name = "runner_lib", srcs = ["runner.py"], diff --git a/tests/integration/whl_storage/BUILD.bazel b/tests/integration/whl_storage/BUILD.bazel new file mode 100644 index 0000000000..4692f14633 --- /dev/null +++ b/tests/integration/whl_storage/BUILD.bazel @@ -0,0 +1,24 @@ +load("@rules_python//python:py_binary.bzl", "py_binary") + +exports_files([ + "pkg-1.0-py3-none-any.whl", + "wheel.sha256", + "modify_pkg.patch", +]) + +[ + py_binary( + name = name, + srcs = ["main.py"], + legacy_create_init = 0, + main = "main.py", + deps = ["@{}//:pkg".format(name)], + ) + for name in [ + "plain", + "patched", + "nonmatching", + "provided", + "provided_patched", + ] +] diff --git a/tests/integration/whl_storage/MODULE.bazel b/tests/integration/whl_storage/MODULE.bazel new file mode 100644 index 0000000000..10377254e8 --- /dev/null +++ b/tests/integration/whl_storage/MODULE.bazel @@ -0,0 +1,14 @@ +module(name = "integration_test") + +bazel_dep(name = "package_metadata", version = "0.0.7") +bazel_dep(name = "rules_python", version = "0.0.0") +local_path_override( + module_name = "rules_python", + path = "../../..", +) + +python = use_extension("@rules_python//python/extensions:python.bzl", "python") +python.toolchain(python_version = "3.14") + +wheels = use_extension("//:wheels.bzl", "wheels") +use_repo(wheels, "nonmatching", "patched", "plain", "provided", "provided_patched") diff --git a/tests/integration/whl_storage/WORKSPACE b/tests/integration/whl_storage/WORKSPACE new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/integration/whl_storage/main.py b/tests/integration/whl_storage/main.py new file mode 100644 index 0000000000..1c2ee07529 --- /dev/null +++ b/tests/integration/whl_storage/main.py @@ -0,0 +1,3 @@ +import pkg + +print(f"PATCHED={pkg.PATCHED}") diff --git a/tests/integration/whl_storage/modify_pkg.patch b/tests/integration/whl_storage/modify_pkg.patch new file mode 100644 index 0000000000..25c7533820 --- /dev/null +++ b/tests/integration/whl_storage/modify_pkg.patch @@ -0,0 +1,5 @@ +--- pkg/__init__.py ++++ pkg/__init__.py +@@ -1 +1 @@ +-PATCHED = False ++PATCHED = True diff --git a/tests/integration/whl_storage/wheels.bzl b/tests/integration/whl_storage/wheels.bzl new file mode 100644 index 0000000000..ed027dd538 --- /dev/null +++ b/tests/integration/whl_storage/wheels.bzl @@ -0,0 +1,38 @@ +"""Local HTTP and caller-provided wheel fixtures.""" + +load("@rules_python//python/private/pypi:whl_library.bzl", "whl_library") # buildifier: disable=bzl-visibility + +_FILENAME = "pkg-1.0-py3-none-any.whl" + +def _patches(filename): + return { + Label("//:modify_pkg.patch"): json.encode({ + "patch_strip": 0, + "whls": [filename], + }), + } + +def _wheels_impl(mrctx): + common = { + "envsubst": ["WHEEL_TEST_ADDRESS"], + "filename": _FILENAME, + "requirement": "pkg", + "sha256": mrctx.read(Label("//:wheel.sha256")).strip(), + "urls": ["http://${WHEEL_TEST_ADDRESS}/" + _FILENAME], + } + whl_library(name = "plain", **common) + whl_library(name = "patched", whl_patches = _patches(_FILENAME), **common) + whl_library(name = "nonmatching", whl_patches = _patches("other.whl"), **common) + whl_library( + name = "provided", + requirement = "pkg", + whl_file = Label("//:" + _FILENAME), + ) + whl_library( + name = "provided_patched", + requirement = "pkg", + whl_file = Label("//:" + _FILENAME), + whl_patches = _patches(_FILENAME), + ) + +wheels = module_extension(implementation = _wheels_impl) diff --git a/tests/integration/whl_storage_test.py b/tests/integration/whl_storage_test.py new file mode 100644 index 0000000000..a7fdf81ab8 --- /dev/null +++ b/tests/integration/whl_storage_test.py @@ -0,0 +1,101 @@ +"""Exercise lazy wheel retention across separate Bazel invocations.""" + +import functools +import hashlib +import http.server +import threading +import unittest +import zipfile +from pathlib import Path + +from tests.integration import runner + + +class WheelStorageTest(runner.TestCase): + def setUp(self): + super().setUp() + self.wheel = self.repo_root / "pkg-1.0-py3-none-any.whl" + with zipfile.ZipFile(self.wheel, "w") as wheel: + wheel.writestr("pkg/__init__.py", "PATCHED = False\n") + wheel.writestr( + "pkg-1.0.dist-info/METADATA", + "Metadata-Version: 2.1\nName: pkg\nVersion: 1.0\n", + ) + wheel.writestr( + "pkg-1.0.dist-info/WHEEL", + "Wheel-Version: 1.0\nRoot-Is-Purelib: true\nTag: py3-none-any\n", + ) + wheel.writestr("pkg-1.0.dist-info/RECORD", "") + self.original_wheel = self.wheel.read_bytes() + (self.repo_root / "wheel.sha256").write_text( + hashlib.sha256(self.original_wheel).hexdigest() + ) + handler = functools.partial( + http.server.SimpleHTTPRequestHandler, directory=str(self.repo_root) + ) + server = http.server.ThreadingHTTPServer(("127.0.0.1", 0), handler) + self.addCleanup(server.server_close) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + self.addCleanup(thread.join) + self.addCleanup(server.shutdown) + self.bazel_env["WHEEL_TEST_ADDRESS"] = f"127.0.0.1:{server.server_port}" + output_base = self.run_bazel("info", "output_base").stdout.strip() + self.external = Path(output_base) / "external" + + def repositories(self, name): + return [ + path + for path in self.external.iterdir() + if path.is_dir() and path.name.endswith(("+" + name, "~" + name)) + ] + + def repository(self, name): + repositories = self.repositories(name) + self.assertEqual(len(repositories), 1, repositories) + return repositories[0].resolve() + + def assert_runs(self, name, *, patched): + result = self.run_bazel("run", "//:" + name) + self.assert_result_matches(result, f"PATCHED={patched}") + + def test_lazy_archive(self): + self.assert_runs("plain", patched=False) + extracted = self.repository("plain") + self.assertEqual(list(extracted.glob("*.whl")), []) + self.assertEqual(self.repositories("plain_whl"), []) + + self.run_bazel("build", "@plain//:whl", "@plain//:whl_file") + archive = self.repository("plain_whl") / self.wheel.name + self.assertEqual(archive.read_bytes(), self.original_wheel) + self.assertEqual(list(extracted.glob("*.whl")), []) + + def test_matching_and_nonmatching_patches(self): + self.assert_runs("patched", patched=True) + patched = self.repository("patched") + self.assertFalse((patched / self.wheel.name).exists()) + self.assertEqual(self.repositories("patched_whl"), []) + self.run_bazel("build", "@patched//:whl", "@patched//:whl_file") + with zipfile.ZipFile(patched / "pkg-1.0+patched-py3-none-any.whl") as wheel: + self.assertEqual(wheel.read("pkg/__init__.py"), b"PATCHED = True\n") + self.assertEqual(self.repositories("patched_whl"), []) + + self.assert_runs("nonmatching", patched=False) + self.assertEqual(list(self.repository("nonmatching").glob("*.whl")), []) + self.assertEqual(self.repositories("nonmatching_whl"), []) + self.run_bazel("build", "@nonmatching//:whl") + archive = self.repository("nonmatching_whl") / self.wheel.name + self.assertEqual(archive.read_bytes(), self.original_wheel) + + def test_caller_owned_wheel_is_preserved(self): + for name, patched in [("provided", False), ("provided_patched", True)]: + with self.subTest(name=name): + self.assert_runs(name, patched=patched) + self.run_bazel("build", f"@{name}//:whl", f"@{name}//:whl_file") + self.assertEqual(self.wheel.read_bytes(), self.original_wheel) + self.assertFalse((self.repository(name) / self.wheel.name).exists()) + self.assertEqual(self.repositories(name + "_whl"), []) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/pypi/integration/BUILD.bazel b/tests/pypi/integration/BUILD.bazel index 316abe8271..6a060cf7b0 100644 --- a/tests/pypi/integration/BUILD.bazel +++ b/tests/pypi/integration/BUILD.bazel @@ -1,5 +1,8 @@ load("@bazel_skylib//rules:build_test.bzl", "build_test") load("@rules_python_publish_deps//:requirements.bzl", "all_requirements") +load(":whl_storage_tests.bzl", "whl_storage_test_suite") + +whl_storage_test_suite(name = "whl_storage_tests") build_test( name = "all_requirements_build_test", diff --git a/tests/pypi/integration/whl_storage_tests.bzl b/tests/pypi/integration/whl_storage_tests.bzl new file mode 100644 index 0000000000..950dc8b85e --- /dev/null +++ b/tests/pypi/integration/whl_storage_tests.bzl @@ -0,0 +1,44 @@ +"""Check archive storage and runtime dependencies for a downloaded wheel.""" + +load("@rules_testing//lib:analysis_test.bzl", "analysis_test") +load("@rules_testing//lib:test_suite.bzl", "test_suite") + +def _test_downloaded_archive(name): + analysis_test( + name = name, + target = "@rules_python_publish_deps//packaging:whl", + impl = _test_downloaded_archive_impl, + ) + +def _test_downloaded_archive_impl(env, target): + files = target[DefaultInfo].files.to_list() + env.expect.that_collection(files).has_size(1) + for file in files: + env.expect.that_bool(file.basename.endswith(".whl")).equals(True) + env.expect.that_bool(file.owner.workspace_name.endswith("_whl")).equals(True) + +def _test_runtime_excludes_archive(name): + analysis_test( + name = name, + target = "@rules_python_publish_deps//packaging:pkg", + impl = _test_runtime_excludes_archive_impl, + ) + +def _test_runtime_excludes_archive_impl(env, target): + files = target[DefaultInfo].default_runfiles.files.to_list() + env.expect.that_bool(bool(files)).equals(True) + env.expect.that_collection([ + file.short_path + for file in files + if file.extension == "whl" + ]).contains_exactly([]) + +def whl_storage_test_suite(name): + """Create tests for separate archive storage and wheel target compatibility.""" + test_suite( + name = name, + tests = [ + _test_downloaded_archive, + _test_runtime_excludes_archive, + ], + ) diff --git a/tests/pypi/whl_library_srcs/whl_library_srcs_tests.bzl b/tests/pypi/whl_library_srcs/whl_library_srcs_tests.bzl index fe4342b564..740d70e3fb 100644 --- a/tests/pypi/whl_library_srcs/whl_library_srcs_tests.bzl +++ b/tests/pypi/whl_library_srcs/whl_library_srcs_tests.bzl @@ -67,6 +67,26 @@ def _test_filegroups(env): _tests.append(_test_filegroups) +def _test_external_whl_file(env): + calls = [] + whl_library_srcs( + name = "example-1.0-py3-none-any.whl", + whl_file = "@example_whl//:example-1.0-py3-none-any.whl", + native = struct( + filegroup = lambda **kwargs: calls.append(kwargs), + glob = lambda *args, **kwargs: [], + ), + rules = struct(), + ) + wheel = [call for call in calls if call["name"] == "whl_file"][0] + env.expect.that_collection(wheel["srcs"]).contains_exactly([ + "@example_whl//:example-1.0-py3-none-any.whl", + ]) + extracted = [call for call in calls if call["name"] == "extracted_whl_files"][0] + env.expect.that_collection(extracted["srcs"]).contains_exactly([]) + +_tests.append(_test_external_whl_file) + def _test_copy(env): calls = [] From 611bc1e8705f56d3032f4019c3aafb90962142c4 Mon Sep 17 00:00:00 2001 From: Charley Saint Date: Thu, 22 Oct 2026 11:42:33 -0700 Subject: [PATCH 2/5] test(pypi): restrict lazy archive ownership check to bzlmod --- tests/pypi/integration/whl_storage_tests.bzl | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/pypi/integration/whl_storage_tests.bzl b/tests/pypi/integration/whl_storage_tests.bzl index 950dc8b85e..de469b10df 100644 --- a/tests/pypi/integration/whl_storage_tests.bzl +++ b/tests/pypi/integration/whl_storage_tests.bzl @@ -2,12 +2,16 @@ load("@rules_testing//lib:analysis_test.bzl", "analysis_test") load("@rules_testing//lib:test_suite.bzl", "test_suite") +load("//tests/support:support.bzl", "SUPPORTS_BZLMOD") def _test_downloaded_archive(name): analysis_test( name = name, target = "@rules_python_publish_deps//packaging:whl", impl = _test_downloaded_archive_impl, + # WORKSPACE resolves this fixture through pip_archive, which retains + # its existing archive layout. Only bzlmod supplies a known wheel URL. + attr_values = {"target_compatible_with": SUPPORTS_BZLMOD}, ) def _test_downloaded_archive_impl(env, target): From f66057c9fa063b745cd8ee0ed9c1c11fcfe0be4e Mon Sep 17 00:00:00 2001 From: Charley Saint Date: Thu, 22 Oct 2026 12:59:12 -0700 Subject: [PATCH 3/5] test(pypi): cover lazy wheel storage with uv locks --- tests/integration/whl_storage/BUILD.bazel | 11 +++- tests/integration/whl_storage/MODULE.bazel | 8 +++ tests/integration/whl_storage/main.py | 4 +- .../integration/whl_storage/modify_pkg.patch | 4 +- tests/integration/whl_storage/wheels.bzl | 8 +-- tests/integration/whl_storage_test.py | 55 ++++++++++++++++--- 6 files changed, 73 insertions(+), 17 deletions(-) diff --git a/tests/integration/whl_storage/BUILD.bazel b/tests/integration/whl_storage/BUILD.bazel index 4692f14633..443e6803c0 100644 --- a/tests/integration/whl_storage/BUILD.bazel +++ b/tests/integration/whl_storage/BUILD.bazel @@ -1,9 +1,10 @@ load("@rules_python//python:py_binary.bzl", "py_binary") exports_files([ - "pkg-1.0-py3-none-any.whl", + "storage_pkg-1.0-py3-none-any.whl", "wheel.sha256", "modify_pkg.patch", + "uv.lock", ]) [ @@ -22,3 +23,11 @@ exports_files([ "provided_patched", ] ] + +py_binary( + name = "uv_locked", + srcs = ["main.py"], + legacy_create_init = 0, + main = "main.py", + deps = ["@uv_pip//storage_pkg"], +) diff --git a/tests/integration/whl_storage/MODULE.bazel b/tests/integration/whl_storage/MODULE.bazel index 10377254e8..13a4e5611b 100644 --- a/tests/integration/whl_storage/MODULE.bazel +++ b/tests/integration/whl_storage/MODULE.bazel @@ -12,3 +12,11 @@ python.toolchain(python_version = "3.14") wheels = use_extension("//:wheels.bzl", "wheels") use_repo(wheels, "nonmatching", "patched", "plain", "provided", "provided_patched") + +pip = use_extension("@rules_python//python/extensions:pip.bzl", "pip") +pip.parse( + hub_name = "uv_pip", + python_version = "3.14", + uv_lock = "//:uv.lock", +) +use_repo(pip, "uv_pip") diff --git a/tests/integration/whl_storage/main.py b/tests/integration/whl_storage/main.py index 1c2ee07529..ba55731016 100644 --- a/tests/integration/whl_storage/main.py +++ b/tests/integration/whl_storage/main.py @@ -1,3 +1,3 @@ -import pkg +import storage_pkg -print(f"PATCHED={pkg.PATCHED}") +print(f"PATCHED={storage_pkg.PATCHED}") diff --git a/tests/integration/whl_storage/modify_pkg.patch b/tests/integration/whl_storage/modify_pkg.patch index 25c7533820..61c310b579 100644 --- a/tests/integration/whl_storage/modify_pkg.patch +++ b/tests/integration/whl_storage/modify_pkg.patch @@ -1,5 +1,5 @@ ---- pkg/__init__.py -+++ pkg/__init__.py +--- storage_pkg/__init__.py ++++ storage_pkg/__init__.py @@ -1 +1 @@ -PATCHED = False +PATCHED = True diff --git a/tests/integration/whl_storage/wheels.bzl b/tests/integration/whl_storage/wheels.bzl index ed027dd538..b82f0c0479 100644 --- a/tests/integration/whl_storage/wheels.bzl +++ b/tests/integration/whl_storage/wheels.bzl @@ -2,7 +2,7 @@ load("@rules_python//python/private/pypi:whl_library.bzl", "whl_library") # buildifier: disable=bzl-visibility -_FILENAME = "pkg-1.0-py3-none-any.whl" +_FILENAME = "storage_pkg-1.0-py3-none-any.whl" def _patches(filename): return { @@ -16,7 +16,7 @@ def _wheels_impl(mrctx): common = { "envsubst": ["WHEEL_TEST_ADDRESS"], "filename": _FILENAME, - "requirement": "pkg", + "requirement": "storage-pkg", "sha256": mrctx.read(Label("//:wheel.sha256")).strip(), "urls": ["http://${WHEEL_TEST_ADDRESS}/" + _FILENAME], } @@ -25,12 +25,12 @@ def _wheels_impl(mrctx): whl_library(name = "nonmatching", whl_patches = _patches("other.whl"), **common) whl_library( name = "provided", - requirement = "pkg", + requirement = "storage-pkg", whl_file = Label("//:" + _FILENAME), ) whl_library( name = "provided_patched", - requirement = "pkg", + requirement = "storage-pkg", whl_file = Label("//:" + _FILENAME), whl_patches = _patches(_FILENAME), ) diff --git a/tests/integration/whl_storage_test.py b/tests/integration/whl_storage_test.py index a7fdf81ab8..703a9b597e 100644 --- a/tests/integration/whl_storage_test.py +++ b/tests/integration/whl_storage_test.py @@ -14,18 +14,18 @@ class WheelStorageTest(runner.TestCase): def setUp(self): super().setUp() - self.wheel = self.repo_root / "pkg-1.0-py3-none-any.whl" + self.wheel = self.repo_root / "storage_pkg-1.0-py3-none-any.whl" with zipfile.ZipFile(self.wheel, "w") as wheel: - wheel.writestr("pkg/__init__.py", "PATCHED = False\n") + wheel.writestr("storage_pkg/__init__.py", "PATCHED = False\n") wheel.writestr( - "pkg-1.0.dist-info/METADATA", - "Metadata-Version: 2.1\nName: pkg\nVersion: 1.0\n", + "storage_pkg-1.0.dist-info/METADATA", + "Metadata-Version: 2.1\nName: storage-pkg\nVersion: 1.0\n", ) wheel.writestr( - "pkg-1.0.dist-info/WHEEL", + "storage_pkg-1.0.dist-info/WHEEL", "Wheel-Version: 1.0\nRoot-Is-Purelib: true\nTag: py3-none-any\n", ) - wheel.writestr("pkg-1.0.dist-info/RECORD", "") + wheel.writestr("storage_pkg-1.0.dist-info/RECORD", "") self.original_wheel = self.wheel.read_bytes() (self.repo_root / "wheel.sha256").write_text( hashlib.sha256(self.original_wheel).hexdigest() @@ -40,6 +40,23 @@ def setUp(self): self.addCleanup(thread.join) self.addCleanup(server.shutdown) self.bazel_env["WHEEL_TEST_ADDRESS"] = f"127.0.0.1:{server.server_port}" + base_url = f"http://127.0.0.1:{server.server_port}" + digest = hashlib.sha256(self.original_wheel).hexdigest() + (self.repo_root / "uv.lock").write_text( + f"""\ +version = 1 +revision = 3 +requires-python = ">=3.14" + +[[package]] +name = "storage-pkg" +version = "1.0" +source = {{ registry = "{base_url}/simple" }} +wheels = [ + {{ url = "{base_url}/{self.wheel.name}", hash = "sha256:{digest}", size = {len(self.original_wheel)} }}, +] +""" + ) output_base = self.run_bazel("info", "output_base").stdout.strip() self.external = Path(output_base) / "external" @@ -76,8 +93,10 @@ def test_matching_and_nonmatching_patches(self): self.assertFalse((patched / self.wheel.name).exists()) self.assertEqual(self.repositories("patched_whl"), []) self.run_bazel("build", "@patched//:whl", "@patched//:whl_file") - with zipfile.ZipFile(patched / "pkg-1.0+patched-py3-none-any.whl") as wheel: - self.assertEqual(wheel.read("pkg/__init__.py"), b"PATCHED = True\n") + with zipfile.ZipFile( + patched / "storage_pkg-1.0+patched-py3-none-any.whl" + ) as wheel: + self.assertEqual(wheel.read("storage_pkg/__init__.py"), b"PATCHED = True\n") self.assertEqual(self.repositories("patched_whl"), []) self.assert_runs("nonmatching", patched=False) @@ -96,6 +115,26 @@ def test_caller_owned_wheel_is_preserved(self): self.assertFalse((self.repository(name) / self.wheel.name).exists()) self.assertEqual(self.repositories(name + "_whl"), []) + def test_uv_lock_lazy_archive(self): + self.assert_runs("uv_locked", patched=False) + repositories = [ + path + for path in self.external.glob("*uv_pip_*") + if path.is_dir() and not path.name.endswith("_whl") + ] + self.assertEqual(len(repositories), 1, repositories) + extracted = repositories[0].resolve() + archive_repo = self.external / (repositories[0].name + "_whl") + self.assertEqual(list(extracted.glob("*.whl")), []) + self.assertFalse(archive_repo.exists()) + self.assertFalse(archive_repo.is_symlink()) + + self.run_bazel("build", "@uv_pip//storage_pkg:whl") + self.assertEqual( + (archive_repo / self.wheel.name).read_bytes(), self.original_wheel + ) + self.assertEqual(list(extracted.glob("*.whl")), []) + if __name__ == "__main__": unittest.main() From 5726e6dc4f5985c42ec144a31fa25f146ae64992 Mon Sep 17 00:00:00 2001 From: Charley Saint Date: Fri, 23 Oct 2026 10:22:45 -0700 Subject: [PATCH 4/5] fix(pypi): isolate patched wheels and consolidate integration tests --- .bazelrc.deleted_packages | 1 - news/lazy-wheel-archives.changed.md | 10 +++-- python/private/pypi/BUILD.bazel | 10 +++-- python/private/pypi/patch_and_extract_whl.bzl | 8 +--- python/private/pypi/patch_whl.bzl | 17 ++++++++ python/private/pypi/whl_archive.bzl | 2 +- .../{whl_download.bzl => whl_file_repo.bzl} | 40 +++++++++++++++---- python/private/pypi/whl_library.bzl | 22 ++++++++-- tests/integration/BUILD.bazel | 6 +-- tests/integration/whl_library/BUILD.bazel | 33 +++++++++++++++ tests/integration/whl_library/MODULE.bazel | 11 +++++ .../{whl_storage => whl_library}/main.py | 0 .../modify_pkg.patch | 0 .../{whl_storage => whl_library}/wheels.bzl | 0 ...hl_storage_test.py => whl_library_test.py} | 26 +++++++++--- tests/integration/whl_storage/BUILD.bazel | 33 --------------- tests/integration/whl_storage/MODULE.bazel | 22 ---------- tests/integration/whl_storage/WORKSPACE | 0 18 files changed, 148 insertions(+), 93 deletions(-) rename python/private/pypi/{whl_download.bzl => whl_file_repo.bzl} (59%) rename tests/integration/{whl_storage => whl_library}/main.py (100%) rename tests/integration/{whl_storage => whl_library}/modify_pkg.patch (100%) rename tests/integration/{whl_storage => whl_library}/wheels.bzl (100%) rename tests/integration/{whl_storage_test.py => whl_library_test.py} (82%) delete mode 100644 tests/integration/whl_storage/BUILD.bazel delete mode 100644 tests/integration/whl_storage/MODULE.bazel delete mode 100644 tests/integration/whl_storage/WORKSPACE diff --git a/.bazelrc.deleted_packages b/.bazelrc.deleted_packages index 147d57a222..3eec28298e 100644 --- a/.bazelrc.deleted_packages +++ b/.bazelrc.deleted_packages @@ -45,7 +45,6 @@ common --deleted_packages=tests/integration/unified_pypi common --deleted_packages=tests/integration/uv_lock common --deleted_packages=tests/integration/validate_test_main common --deleted_packages=tests/integration/whl_library -common --deleted_packages=tests/integration/whl_storage common --deleted_packages=tests/modules/another_module common --deleted_packages=tests/modules/other common --deleted_packages=tests/modules/other/nspkg_delta diff --git a/news/lazy-wheel-archives.changed.md b/news/lazy-wheel-archives.changed.md index 493ff56178..08226ef0e5 100644 --- a/news/lazy-wheel-archives.changed.md +++ b/news/lazy-wheel-archives.changed.md @@ -1,4 +1,6 @@ -Downloaded wheels are removed after extraction when their original archive can -be fetched independently. The existing `whl` and `whl_file` targets fetch the -archive from a separate repository on demand. Patched wheels and wheels built -from source retain their existing archive behavior. +(pypi) Downloaded wheels are removed after extraction when their original +archive can be fetched independently. The existing `whl` and `whl_file` targets +fetch the archive from a separate repository on demand. Matching patches are +applied in the separate wheel-file repository, which also supplies the patched +archive for extraction. Wheels built from source retain their existing archive +behavior. diff --git a/python/private/pypi/BUILD.bazel b/python/private/pypi/BUILD.bazel index 032a2caafa..149074dd3f 100644 --- a/python/private/pypi/BUILD.bazel +++ b/python/private/pypi/BUILD.bazel @@ -479,9 +479,10 @@ bzl_library( name = "whl_library", srcs = ["whl_library.bzl"], deps = [ + ":patch_whl", ":pip_archive", ":whl_archive", - ":whl_download", + ":whl_file_repo", ], ) @@ -557,17 +558,18 @@ bzl_library( ":attrs", ":patch_and_extract_whl", ":whl_deps_repo", - ":whl_download", + ":whl_file_repo", "//python/private:auth", "//python/private:repo_utils", ], ) bzl_library( - name = "whl_download", - srcs = ["whl_download.bzl"], + name = "whl_file_repo", + srcs = ["whl_file_repo.bzl"], deps = [ ":attrs", + ":patch_whl", ":urllib", "//python/private:auth", "//python/private:repo_utils", diff --git a/python/private/pypi/patch_and_extract_whl.bzl b/python/private/pypi/patch_and_extract_whl.bzl index 594840601b..37c90e3221 100644 --- a/python/private/pypi/patch_and_extract_whl.bzl +++ b/python/private/pypi/patch_and_extract_whl.bzl @@ -3,7 +3,7 @@ load("//python/private:normalize_name.bzl", "normalize_name") load("//python/private:repo_utils.bzl", "repo_utils") load(":generate_whl_library_build_bazel.bzl", "generate_whl_library_build_bazel") -load(":patch_whl.bzl", "patch_whl") +load(":patch_whl.bzl", "get_whl_patches", "patch_whl") load(":pep508_requirement.bzl", "requirement") load(":pypi_repo_utils.bzl", "pypi_repo_utils") load(":whl_extract.bzl", "whl_extract") @@ -85,11 +85,7 @@ def patch_and_extract_whl(rctx, *, whl_path, logger, sdist_filename = None, whl_ The repository metadata if the extraction is reproducible """ if rctx.attr.whl_patches: - patches = {} - for patch_file, json_args in rctx.attr.whl_patches.items(): - patch_dst = struct(**json.decode(json_args)) - if whl_path.basename in patch_dst.whls: - patches[patch_file] = patch_dst.patch_strip + patches = get_whl_patches(rctx.attr.whl_patches, whl_path.basename) if patches: # patch_whl deletes its input before repacking. Never delete an diff --git a/python/private/pypi/patch_whl.bzl b/python/private/pypi/patch_whl.bzl index e8d76eeba5..b257f2b4fe 100644 --- a/python/private/pypi/patch_whl.bzl +++ b/python/private/pypi/patch_whl.bzl @@ -58,6 +58,23 @@ def patched_whl_name(original_whl_name): platform_tag = parsed_whl.platform_tag, ) +def get_whl_patches(whl_patches, filename): + """Select the patches that apply to a wheel filename. + + Args: + whl_patches: Patch labels mapped to JSON-encoded wheel names and strip levels. + filename: The wheel filename to match. + + Returns: + A dictionary of matching patch labels mapped to strip levels. + """ + patches = {} + for patch_file, json_args in whl_patches.items(): + patch_dst = struct(**json.decode(json_args)) + if filename in patch_dst.whls: + patches[patch_file] = patch_dst.patch_strip + return patches + def patch_whl(rctx, *, whl_path, patches): """Patch a whl file and repack it natively. diff --git a/python/private/pypi/whl_archive.bzl b/python/private/pypi/whl_archive.bzl index 9c0e8d559c..3ba32b1aa1 100644 --- a/python/private/pypi/whl_archive.bzl +++ b/python/private/pypi/whl_archive.bzl @@ -5,7 +5,7 @@ load("//python/private:repo_utils.bzl", "REPO_DEBUG_ENV_VAR", "repo_utils") load(":attrs.bzl", "ATTRS") load(":patch_and_extract_whl.bzl", "patch_and_extract_whl") load(":whl_deps_repo.bzl", "whl_deps_attrs") -load(":whl_download.bzl", "download_whl") +load(":whl_file_repo.bzl", "download_whl") def _whl_archive_impl(rctx): logger = repo_utils.logger(rctx) diff --git a/python/private/pypi/whl_download.bzl b/python/private/pypi/whl_file_repo.bzl similarity index 59% rename from python/private/pypi/whl_download.bzl rename to python/private/pypi/whl_file_repo.bzl index 1017f75532..40d79ac10b 100644 --- a/python/private/pypi/whl_download.bzl +++ b/python/private/pypi/whl_file_repo.bzl @@ -1,8 +1,9 @@ -"""Download a wheel without retaining it in the extracted repository.""" +"""Store an original or patched wheel separately from its extracted files.""" load("//python/private:auth.bzl", "AUTH_ATTRS", "get_auth") load("//python/private:repo_utils.bzl", "REPO_DEBUG_ENV_VAR", "repo_utils") load(":attrs.bzl", "ATTRS") +load(":patch_whl.bzl", "patch_whl") load(":urllib.bzl", "urllib") def download_whl(rctx, *, logger): @@ -43,27 +44,50 @@ def download_whl(rctx, *, logger): fail("could not download the '{}' from {}:\n{}".format(filename, urls, result)) return rctx.path(filename) -whl_download_attrs = { +whl_file_repo_attrs = { "envsubst": ATTRS["envsubst"], "filename": attr.string(doc = "The filename of the downloaded wheel."), "index_url": attr.string(doc = "The index URL used to resolve relative wheel URLs."), "integrity": attr.string(doc = "The expected SRI checksum, used when sha256 is unset."), + "patches": attr.label_keyed_string_dict(doc = "Patch labels mapped to their strip levels."), "requirement": attr.string(doc = "The requirement used in download diagnostics."), "sha256": attr.string(doc = "The expected SHA256 checksum."), "urls": attr.string_list(doc = "The wheel download URLs."), + "whl_file": attr.label(doc = "A caller-owned wheel to use instead of downloading."), } | AUTH_ATTRS -def _whl_download_impl(rctx): - whl_path = download_whl(rctx, logger = repo_utils.logger(rctx)) +def _whl_file_repo_impl(rctx): + if rctx.attr.whl_file: + rctx.watch(rctx.attr.whl_file) + whl_path = rctx.path(rctx.attr.whl_file) + + # patch_whl deletes its input before repacking. Keep caller-owned + # wheels intact by giving it a repository-local symlink. + rctx.symlink(whl_path, whl_path.basename) + whl_path = rctx.path(whl_path.basename) + else: + whl_path = download_whl(rctx, logger = repo_utils.logger(rctx)) + if rctx.attr.patches: + whl_path = patch_whl( + rctx, + whl_path = whl_path, + patches = {patch: int(strip) for patch, strip in rctx.attr.patches.items()}, + ) + + # Patching temporarily unpacks the input. Retain only the replacement + # wheel; the installation is extracted in the consuming repository. + for path in rctx.path(".").readdir(): + if path.basename != whl_path.basename: + rctx.delete(path) rctx.file("BUILD.bazel", "exports_files([{}])\n".format(repr(whl_path.basename))) if hasattr(rctx, "repo_metadata"): return rctx.repo_metadata(reproducible = True) return None -whl_download = repository_rule( - implementation = _whl_download_impl, - attrs = whl_download_attrs | { - "_rule_name": attr.string(default = "whl_download"), +whl_file_repo = repository_rule( + implementation = _whl_file_repo_impl, + attrs = whl_file_repo_attrs | { + "_rule_name": attr.string(default = "whl_file_repo"), }, environ = [REPO_DEBUG_ENV_VAR], ) diff --git a/python/private/pypi/whl_library.bzl b/python/private/pypi/whl_library.bzl index 399a55b5d2..3441a2c907 100644 --- a/python/private/pypi/whl_library.bzl +++ b/python/private/pypi/whl_library.bzl @@ -14,9 +14,10 @@ "" +load(":patch_whl.bzl", "get_whl_patches", "patched_whl_name") load(":pip_archive.bzl", "pip_archive", "pip_archive_attrs") load(":whl_archive.bzl", "whl_archive", "whl_archive_attrs") -load(":whl_download.bzl", "whl_download", "whl_download_attrs") +load(":whl_file_repo.bzl", "whl_file_repo", "whl_file_repo_attrs") def _filter(kwargs, subset, debug = False): dropped = {} @@ -42,6 +43,8 @@ def whl_library(name, repo = None, **kwargs): URL wheels are downloaded temporarily for extraction, then removed. A sibling `_whl` repository supplies the original archive lazily, when a wheel target is requested. Extraction does not depend on it. + Matching patches are applied in the wheel-file repository instead; + extraction and wheel targets then consume the same patched archive. Args: name: {type}`str` The name of the repo. @@ -61,8 +64,21 @@ def whl_library(name, repo = None, **kwargs): kwargs.setdefault("dep_template", "@{}{{name}}//:{{target}}".format(kwargs.pop("repo_prefix", ""))) if whl_file or (urls and filename and filename.endswith(".whl")): - if not whl_file: - whl_download(name = name + "_whl", **_filter(kwargs, whl_download_attrs)) + wheel_filename = filename or str(whl_file).split(":")[-1].split("/")[-1] + patches = get_whl_patches(kwargs.get("whl_patches", {}), wheel_filename) + if patches: + # Caller-owned wheels are often already supplied by _whl. + # Give their patched output a distinct repository name. + file_repo = name + ("_patched_whl" if whl_file else "_whl") + whl_file_repo( + name = file_repo, + patches = {patch: str(strip) for patch, strip in patches.items()}, + **_filter(kwargs, whl_file_repo_attrs) + ) + kwargs["whl_file"] = "@{}//:{}".format(file_repo, patched_whl_name(wheel_filename)) + kwargs["whl_patches"] = {} + elif not whl_file: + whl_file_repo(name = name + "_whl", **_filter(kwargs, whl_file_repo_attrs)) kwargs["downloaded_whl"] = "@{}//:{}".format(name + "_whl", filename) whl_archive(name = name, **_filter(kwargs, whl_archive_attrs)) else: diff --git a/tests/integration/BUILD.bazel b/tests/integration/BUILD.bazel index 93df40967d..312eb42ecd 100644 --- a/tests/integration/BUILD.bazel +++ b/tests/integration/BUILD.bazel @@ -141,11 +141,7 @@ rules_python_integration_test( rules_python_integration_test( name = "whl_library_test", -) - -rules_python_integration_test( - name = "whl_storage_test", - py_main = "whl_storage_test.py", + py_main = "whl_library_test.py", ) py_library( diff --git a/tests/integration/whl_library/BUILD.bazel b/tests/integration/whl_library/BUILD.bazel index a53bcc7258..0d2085b407 100644 --- a/tests/integration/whl_library/BUILD.bazel +++ b/tests/integration/whl_library/BUILD.bazel @@ -1,5 +1,38 @@ +load("@rules_python//python:py_binary.bzl", "py_binary") load("@rules_python//python:py_test.bzl", "py_test") +exports_files([ + "storage_pkg-1.0-py3-none-any.whl", + "wheel.sha256", + "modify_pkg.patch", + "uv.lock", +]) + +[ + py_binary( + name = name, + srcs = ["main.py"], + legacy_create_init = 0, + main = "main.py", + deps = ["@{}//:pkg".format(name)], + ) + for name in [ + "plain", + "patched", + "nonmatching", + "provided", + "provided_patched", + ] +] + +py_binary( + name = "uv_locked", + srcs = ["main.py"], + legacy_create_init = 0, + main = "main.py", + deps = ["@uv_pip//storage_pkg"], +) + [ alias( name = "{}_pkg".format(pkg), diff --git a/tests/integration/whl_library/MODULE.bazel b/tests/integration/whl_library/MODULE.bazel index d3886d7650..9059879187 100644 --- a/tests/integration/whl_library/MODULE.bazel +++ b/tests/integration/whl_library/MODULE.bazel @@ -59,3 +59,14 @@ whl_deps_repo( metadata_file = "@whl_archive//:metadata.json", requirement = "requests", ) + +wheels = use_extension("//:wheels.bzl", "wheels") +use_repo(wheels, "nonmatching", "patched", "plain", "provided", "provided_patched") + +pip = use_extension("@rules_python//python/extensions:pip.bzl", "pip") +pip.parse( + hub_name = "uv_pip", + python_version = "3.14", + uv_lock = "//:uv.lock", +) +use_repo(pip, "uv_pip") diff --git a/tests/integration/whl_storage/main.py b/tests/integration/whl_library/main.py similarity index 100% rename from tests/integration/whl_storage/main.py rename to tests/integration/whl_library/main.py diff --git a/tests/integration/whl_storage/modify_pkg.patch b/tests/integration/whl_library/modify_pkg.patch similarity index 100% rename from tests/integration/whl_storage/modify_pkg.patch rename to tests/integration/whl_library/modify_pkg.patch diff --git a/tests/integration/whl_storage/wheels.bzl b/tests/integration/whl_library/wheels.bzl similarity index 100% rename from tests/integration/whl_storage/wheels.bzl rename to tests/integration/whl_library/wheels.bzl diff --git a/tests/integration/whl_storage_test.py b/tests/integration/whl_library_test.py similarity index 82% rename from tests/integration/whl_storage_test.py rename to tests/integration/whl_library_test.py index 703a9b597e..f61ad215c8 100644 --- a/tests/integration/whl_storage_test.py +++ b/tests/integration/whl_library_test.py @@ -11,7 +11,7 @@ from tests.integration import runner -class WheelStorageTest(runner.TestCase): +class WheelLibraryTest(runner.TestCase): def setUp(self): super().setUp() self.wheel = self.repo_root / "storage_pkg-1.0-py3-none-any.whl" @@ -90,14 +90,15 @@ def test_lazy_archive(self): def test_matching_and_nonmatching_patches(self): self.assert_runs("patched", patched=True) patched = self.repository("patched") - self.assertFalse((patched / self.wheel.name).exists()) - self.assertEqual(self.repositories("patched_whl"), []) + self.assertEqual(list(patched.glob("*.whl")), []) self.run_bazel("build", "@patched//:whl", "@patched//:whl_file") with zipfile.ZipFile( - patched / "storage_pkg-1.0+patched-py3-none-any.whl" + self.repository("patched_whl") / "storage_pkg-1.0+patched-py3-none-any.whl" ) as wheel: self.assertEqual(wheel.read("storage_pkg/__init__.py"), b"PATCHED = True\n") - self.assertEqual(self.repositories("patched_whl"), []) + self.assertFalse((self.repository("patched_whl") / self.wheel.name).exists()) + self.assertFalse((self.repository("patched_whl") / "storage_pkg").exists()) + self.assertEqual(list(patched.glob("*.whl")), []) self.assert_runs("nonmatching", patched=False) self.assertEqual(list(self.repository("nonmatching").glob("*.whl")), []) @@ -112,8 +113,21 @@ def test_caller_owned_wheel_is_preserved(self): self.assert_runs(name, patched=patched) self.run_bazel("build", f"@{name}//:whl", f"@{name}//:whl_file") self.assertEqual(self.wheel.read_bytes(), self.original_wheel) - self.assertFalse((self.repository(name) / self.wheel.name).exists()) + self.assertEqual(list(self.repository(name).glob("*.whl")), []) self.assertEqual(self.repositories(name + "_whl"), []) + if patched: + with zipfile.ZipFile( + self.repository(name + "_patched_whl") + / "storage_pkg-1.0+patched-py3-none-any.whl" + ) as wheel: + self.assertEqual( + wheel.read("storage_pkg/__init__.py"), b"PATCHED = True\n" + ) + + def test_repository_contents(self): + # Retain the existing direct whl_archive, pip_archive, sdist, and + # dependency-wiring assertions in the same nested Bazel test. + self.run_bazel("test", "//:test_contents") def test_uv_lock_lazy_archive(self): self.assert_runs("uv_locked", patched=False) diff --git a/tests/integration/whl_storage/BUILD.bazel b/tests/integration/whl_storage/BUILD.bazel deleted file mode 100644 index 443e6803c0..0000000000 --- a/tests/integration/whl_storage/BUILD.bazel +++ /dev/null @@ -1,33 +0,0 @@ -load("@rules_python//python:py_binary.bzl", "py_binary") - -exports_files([ - "storage_pkg-1.0-py3-none-any.whl", - "wheel.sha256", - "modify_pkg.patch", - "uv.lock", -]) - -[ - py_binary( - name = name, - srcs = ["main.py"], - legacy_create_init = 0, - main = "main.py", - deps = ["@{}//:pkg".format(name)], - ) - for name in [ - "plain", - "patched", - "nonmatching", - "provided", - "provided_patched", - ] -] - -py_binary( - name = "uv_locked", - srcs = ["main.py"], - legacy_create_init = 0, - main = "main.py", - deps = ["@uv_pip//storage_pkg"], -) diff --git a/tests/integration/whl_storage/MODULE.bazel b/tests/integration/whl_storage/MODULE.bazel deleted file mode 100644 index 13a4e5611b..0000000000 --- a/tests/integration/whl_storage/MODULE.bazel +++ /dev/null @@ -1,22 +0,0 @@ -module(name = "integration_test") - -bazel_dep(name = "package_metadata", version = "0.0.7") -bazel_dep(name = "rules_python", version = "0.0.0") -local_path_override( - module_name = "rules_python", - path = "../../..", -) - -python = use_extension("@rules_python//python/extensions:python.bzl", "python") -python.toolchain(python_version = "3.14") - -wheels = use_extension("//:wheels.bzl", "wheels") -use_repo(wheels, "nonmatching", "patched", "plain", "provided", "provided_patched") - -pip = use_extension("@rules_python//python/extensions:pip.bzl", "pip") -pip.parse( - hub_name = "uv_pip", - python_version = "3.14", - uv_lock = "//:uv.lock", -) -use_repo(pip, "uv_pip") diff --git a/tests/integration/whl_storage/WORKSPACE b/tests/integration/whl_storage/WORKSPACE deleted file mode 100644 index e69de29bb2..0000000000 From ceffc049f1db275aa6cc66a4a8ed81d3c623f93c Mon Sep 17 00:00:00 2001 From: Charley Saint Date: Fri, 23 Oct 2026 10:49:16 -0700 Subject: [PATCH 5/5] fix(examples): access patched archives through the wheel target --- examples/bzlmod/whl_mods/BUILD.bazel | 2 ++ examples/bzlmod/whl_mods/appended_build_content.BUILD | 10 ++++------ examples/bzlmod/whl_mods/pip_whl_mods_test.py | 9 +++++++++ 3 files changed, 15 insertions(+), 6 deletions(-) diff --git a/examples/bzlmod/whl_mods/BUILD.bazel b/examples/bzlmod/whl_mods/BUILD.bazel index 7c5ab5056e..d141561338 100644 --- a/examples/bzlmod/whl_mods/BUILD.bazel +++ b/examples/bzlmod/whl_mods/BUILD.bazel @@ -8,8 +8,10 @@ exports_files( py_test( name = "pip_whl_mods_test", srcs = ["pip_whl_mods_test.py"], + data = ["@pip//requests:whl"], env = { "REQUESTS_PKG": "$(rlocationpaths @pip//requests:pkg)", + "REQUESTS_WHL": "$(rlocationpath @pip//requests:whl)", "WHEEL_PKG": "$(rlocationpaths @pip//wheel:pkg)", }, main = "pip_whl_mods_test.py", diff --git a/examples/bzlmod/whl_mods/appended_build_content.BUILD b/examples/bzlmod/whl_mods/appended_build_content.BUILD index 0ca118d7b6..e751dcbb30 100644 --- a/examples/bzlmod/whl_mods/appended_build_content.BUILD +++ b/examples/bzlmod/whl_mods/appended_build_content.BUILD @@ -7,10 +7,8 @@ write_file( ) filegroup( - name = "whl_orig", - srcs = glob( - ["*.whl"], - allow_empty = False, - exclude = ["*-patched-*.whl"], - ), + name = "whl_archive", + # The archive may live in a separate repository. Use the wheel target + # instead of assuming that the archive is beside the extracted files. + srcs = [":whl_file"], ) diff --git a/examples/bzlmod/whl_mods/pip_whl_mods_test.py b/examples/bzlmod/whl_mods/pip_whl_mods_test.py index 0bd7bca110..f5afa5214d 100644 --- a/examples/bzlmod/whl_mods/pip_whl_mods_test.py +++ b/examples/bzlmod/whl_mods/pip_whl_mods_test.py @@ -19,6 +19,7 @@ import subprocess import sys import unittest +import zipfile from pathlib import Path from python.runfiles import runfiles @@ -145,6 +146,14 @@ def test_patches(self): "Summary: Python HTTP for Humans. Patched.", metadata.read_text().splitlines(), ) + # The public wheel target must expose the same patches as the library, + # even when its archive is stored outside the extracted repository. + archive = Path(r.Rlocation(os.environ["REQUESTS_WHL"])) + with zipfile.ZipFile(archive) as wheel: + self.assertEqual( + wheel.read(f"requests-{current_wheel_version}.dist-info/METADATA"), + metadata.read_bytes(), + ) if __name__ == "__main__":