diff --git a/examples/bzlmod/whl_mods/BUILD.bazel b/examples/bzlmod/whl_mods/BUILD.bazel index 7c5ab5056e..d141561338 100644 --- a/examples/bzlmod/whl_mods/BUILD.bazel +++ b/examples/bzlmod/whl_mods/BUILD.bazel @@ -8,8 +8,10 @@ exports_files( py_test( name = "pip_whl_mods_test", srcs = ["pip_whl_mods_test.py"], + data = ["@pip//requests:whl"], env = { "REQUESTS_PKG": "$(rlocationpaths @pip//requests:pkg)", + "REQUESTS_WHL": "$(rlocationpath @pip//requests:whl)", "WHEEL_PKG": "$(rlocationpaths @pip//wheel:pkg)", }, main = "pip_whl_mods_test.py", diff --git a/examples/bzlmod/whl_mods/appended_build_content.BUILD b/examples/bzlmod/whl_mods/appended_build_content.BUILD index 0ca118d7b6..e751dcbb30 100644 --- a/examples/bzlmod/whl_mods/appended_build_content.BUILD +++ b/examples/bzlmod/whl_mods/appended_build_content.BUILD @@ -7,10 +7,8 @@ write_file( ) filegroup( - name = "whl_orig", - srcs = glob( - ["*.whl"], - allow_empty = False, - exclude = ["*-patched-*.whl"], - ), + name = "whl_archive", + # The archive may live in a separate repository. Use the wheel target + # instead of assuming that the archive is beside the extracted files. + srcs = [":whl_file"], ) diff --git a/examples/bzlmod/whl_mods/pip_whl_mods_test.py b/examples/bzlmod/whl_mods/pip_whl_mods_test.py index 8cd723224a..972a7df411 100644 --- a/examples/bzlmod/whl_mods/pip_whl_mods_test.py +++ b/examples/bzlmod/whl_mods/pip_whl_mods_test.py @@ -19,6 +19,7 @@ import subprocess import sys import unittest +import zipfile from pathlib import Path from python.runfiles import runfiles @@ -145,6 +146,14 @@ def test_patches(self): "Summary: Python HTTP for Humans. Patched.", metadata.read_text().splitlines(), ) + # The public wheel target must expose the same patches as the library, + # even when its archive is stored outside the extracted repository. + archive = Path(r.Rlocation(os.environ["REQUESTS_WHL"])) + with zipfile.ZipFile(archive) as wheel: + self.assertEqual( + wheel.read(f"requests-{current_wheel_version}.dist-info/METADATA"), + metadata.read_bytes(), + ) if __name__ == "__main__": diff --git a/news/lazy-wheel-archives.changed.md b/news/lazy-wheel-archives.changed.md new file mode 100644 index 0000000000..08226ef0e5 --- /dev/null +++ b/news/lazy-wheel-archives.changed.md @@ -0,0 +1,6 @@ +(pypi) Downloaded wheels are removed after extraction when their original +archive can be fetched independently. The existing `whl` and `whl_file` targets +fetch the archive from a separate repository on demand. Matching patches are +applied in the separate wheel-file repository, which also supplies the patched +archive for extraction. Wheels built from source retain their existing archive +behavior. diff --git a/python/private/pypi/BUILD.bazel b/python/private/pypi/BUILD.bazel index 96b68e7796..149074dd3f 100644 --- a/python/private/pypi/BUILD.bazel +++ b/python/private/pypi/BUILD.bazel @@ -479,8 +479,10 @@ bzl_library( name = "whl_library", srcs = ["whl_library.bzl"], deps = [ + ":patch_whl", ":pip_archive", ":whl_archive", + ":whl_file_repo", ], ) @@ -555,8 +557,20 @@ bzl_library( deps = [ ":attrs", ":patch_and_extract_whl", - ":urllib", ":whl_deps_repo", + ":whl_file_repo", + "//python/private:auth", + "//python/private:repo_utils", + ], +) + +bzl_library( + name = "whl_file_repo", + srcs = ["whl_file_repo.bzl"], + deps = [ + ":attrs", + ":patch_whl", + ":urllib", "//python/private:auth", "//python/private:repo_utils", ], diff --git a/python/private/pypi/patch_and_extract_whl.bzl b/python/private/pypi/patch_and_extract_whl.bzl index b4ecb2f5b5..37c90e3221 100644 --- a/python/private/pypi/patch_and_extract_whl.bzl +++ b/python/private/pypi/patch_and_extract_whl.bzl @@ -3,7 +3,7 @@ load("//python/private:normalize_name.bzl", "normalize_name") load("//python/private:repo_utils.bzl", "repo_utils") load(":generate_whl_library_build_bazel.bzl", "generate_whl_library_build_bazel") -load(":patch_whl.bzl", "patch_whl") +load(":patch_whl.bzl", "get_whl_patches", "patch_whl") load(":pep508_requirement.bzl", "requirement") load(":pypi_repo_utils.bzl", "pypi_repo_utils") load(":whl_extract.bzl", "whl_extract") @@ -69,7 +69,7 @@ def _remove_files(rctx, *basenames): elif path.is_dir: paths.extend(path.readdir()) -def patch_and_extract_whl(rctx, *, whl_path, logger, sdist_filename = None): +def patch_and_extract_whl(rctx, *, whl_path, logger, sdist_filename = None, whl_file = None): """Extract the wheel, apply patches and generate BUILD.bazel files. Reused in pip and http wheel download code. @@ -79,27 +79,30 @@ def patch_and_extract_whl(rctx, *, whl_path, logger, sdist_filename = None): whl_path: the whl path to extract. logger: The logger to use sdist_filename: The filename to ignore in the BUILD.bazel files as sources. + whl_file: The label of an externally stored wheel, if any. Returns: The repository metadata if the extraction is reproducible """ if rctx.attr.whl_patches: - patches = {} - for patch_file, json_args in rctx.attr.whl_patches.items(): - patch_dst = struct(**json.decode(json_args)) - if whl_path.basename in patch_dst.whls: - patches[patch_file] = patch_dst.patch_strip + patches = get_whl_patches(rctx.attr.whl_patches, whl_path.basename) if patches: + # patch_whl deletes its input before repacking. Never delete an + # archive owned by another repository. + if whl_path.dirname != rctx.path("."): + rctx.symlink(whl_path, whl_path.basename) + whl_path = rctx.path(whl_path.basename) whl_path = patch_whl( rctx, whl_path = whl_path, patches = patches, ) + whl_file = None whl_extract(rctx, whl_path = whl_path, logger = logger) - install_dir_path = whl_path.dirname.get_child("site-packages") + install_dir_path = rctx.path("site-packages") metadata = whl_metadata( install_dir = install_dir_path, read_fn = rctx.read, @@ -139,8 +142,14 @@ def patch_and_extract_whl(rctx, *, whl_path, logger, sdist_filename = None): metadata = metadata, filename = sdist_filename or whl_path.basename, ), + **({"whl_file": str(whl_file)} if whl_file else {}) ) + # The generated wheel target can obtain the original archive independently. + # Do not retain the temporary download or delete a caller-owned wheel. + if whl_file and whl_path.dirname == rctx.path("."): + rctx.delete(whl_path) + # Delete these in case the wheel had them. They generally don't cause # a problem, but let's avoid the chance of that happening. rctx.file("WORKSPACE") diff --git a/python/private/pypi/patch_whl.bzl b/python/private/pypi/patch_whl.bzl index e8d76eeba5..b257f2b4fe 100644 --- a/python/private/pypi/patch_whl.bzl +++ b/python/private/pypi/patch_whl.bzl @@ -58,6 +58,23 @@ def patched_whl_name(original_whl_name): platform_tag = parsed_whl.platform_tag, ) +def get_whl_patches(whl_patches, filename): + """Select the patches that apply to a wheel filename. + + Args: + whl_patches: Patch labels mapped to JSON-encoded wheel names and strip levels. + filename: The wheel filename to match. + + Returns: + A dictionary of matching patch labels mapped to strip levels. + """ + patches = {} + for patch_file, json_args in whl_patches.items(): + patch_dst = struct(**json.decode(json_args)) + if filename in patch_dst.whls: + patches[patch_file] = patch_dst.patch_strip + return patches + def patch_whl(rctx, *, whl_path, patches): """Patch a whl file and repack it natively. diff --git a/python/private/pypi/whl_archive.bzl b/python/private/pypi/whl_archive.bzl index 242104f3c2..3ba32b1aa1 100644 --- a/python/private/pypi/whl_archive.bzl +++ b/python/private/pypi/whl_archive.bzl @@ -1,11 +1,11 @@ "" -load("//python/private:auth.bzl", "AUTH_ATTRS", "get_auth") +load("//python/private:auth.bzl", "AUTH_ATTRS") load("//python/private:repo_utils.bzl", "REPO_DEBUG_ENV_VAR", "repo_utils") load(":attrs.bzl", "ATTRS") load(":patch_and_extract_whl.bzl", "patch_and_extract_whl") -load(":urllib.bzl", "urllib") load(":whl_deps_repo.bzl", "whl_deps_attrs") +load(":whl_file_repo.bzl", "download_whl") def _whl_archive_impl(rctx): logger = repo_utils.logger(rctx) @@ -14,47 +14,17 @@ def _whl_archive_impl(rctx): if rctx.attr.whl_file: rctx.watch(rctx.attr.whl_file) whl_path = rctx.path(rctx.attr.whl_file) - - # Simulate the behaviour where the whl is present in the current directory. - rctx.symlink(whl_path, whl_path.basename) - whl_path = rctx.path(whl_path.basename) elif rctx.attr.urls and rctx.attr.filename: - filename = rctx.attr.filename - urls = rctx.attr.urls - urls = [ - urllib.absolute_url( - rctx.attr.index_url, - url, - envsubst = rctx.attr.envsubst, - getenv = rctx.getenv, - ) - for url in urls - ] - result = rctx.download( - url = urls, - output = filename, - sha256 = rctx.attr.sha256, - integrity = rctx.attr.integrity if not rctx.attr.sha256 else "", - auth = get_auth(rctx, urls), - ) - if not rctx.attr.sha256 and not rctx.attr.integrity: - # this is only seen when there is a direct URL reference without a hash - logger.warn("Please update the requirement line to include the hash:\n{} \\\n --hash=sha256:{}".format( - rctx.attr.requirement, - result.sha256, - )) - - if not result.success: - fail("could not download the '{}' from {}:\n{}".format(filename, urls, result)) - - if filename.endswith(".whl"): - whl_path = rctx.path(filename) - else: - fail("Only wheels are supported") + whl_path = download_whl(rctx, logger = logger) else: fail("Either 'whl_file' or 'urls' and 'filename' needs to be specified") - return patch_and_extract_whl(rctx, whl_path = whl_path, logger = logger) + return patch_and_extract_whl( + rctx, + whl_path = whl_path, + whl_file = rctx.attr.whl_file or rctx.attr.downloaded_whl, + logger = logger, + ) whl_archive_attrs = whl_deps_attrs | { "annotation": attr.label( @@ -64,6 +34,9 @@ whl_archive_attrs = whl_deps_attrs | { ), allow_files = True, ), + "downloaded_whl": attr.string( + doc = "Label of the lazy archive target, emitted in BUILD.bazel without fetching it. INTERNAL USE ONLY.", + ), "filename": attr.string( doc = "Download the whl file to this filename. Only used when the `urls` is passed. If not specified, will be auto-detected from the `urls`.", ), diff --git a/python/private/pypi/whl_file_repo.bzl b/python/private/pypi/whl_file_repo.bzl new file mode 100644 index 0000000000..40d79ac10b --- /dev/null +++ b/python/private/pypi/whl_file_repo.bzl @@ -0,0 +1,93 @@ +"""Store an original or patched wheel separately from its extracted files.""" + +load("//python/private:auth.bzl", "AUTH_ATTRS", "get_auth") +load("//python/private:repo_utils.bzl", "REPO_DEBUG_ENV_VAR", "repo_utils") +load(":attrs.bzl", "ATTRS") +load(":patch_whl.bzl", "patch_whl") +load(":urllib.bzl", "urllib") + +def download_whl(rctx, *, logger): + """Download a wheel and return its path, preserving index authentication. + + Args: + rctx: The repository context. + logger: The repository logger. + + Returns: + The path of the downloaded wheel. + """ + filename = rctx.attr.filename + if not filename.endswith(".whl"): + fail("Only wheels are supported") + urls = [ + urllib.absolute_url( + rctx.attr.index_url, + url, + envsubst = rctx.attr.envsubst, + getenv = rctx.getenv, + ) + for url in rctx.attr.urls + ] + result = rctx.download( + url = urls, + output = filename, + sha256 = rctx.attr.sha256, + integrity = rctx.attr.integrity if not rctx.attr.sha256 else "", + auth = get_auth(rctx, urls), + ) + if not rctx.attr.sha256 and not rctx.attr.integrity: + logger.warn("Please update the requirement line to include the hash:\n{} \\\n --hash=sha256:{}".format( + rctx.attr.requirement, + result.sha256, + )) + if not result.success: + fail("could not download the '{}' from {}:\n{}".format(filename, urls, result)) + return rctx.path(filename) + +whl_file_repo_attrs = { + "envsubst": ATTRS["envsubst"], + "filename": attr.string(doc = "The filename of the downloaded wheel."), + "index_url": attr.string(doc = "The index URL used to resolve relative wheel URLs."), + "integrity": attr.string(doc = "The expected SRI checksum, used when sha256 is unset."), + "patches": attr.label_keyed_string_dict(doc = "Patch labels mapped to their strip levels."), + "requirement": attr.string(doc = "The requirement used in download diagnostics."), + "sha256": attr.string(doc = "The expected SHA256 checksum."), + "urls": attr.string_list(doc = "The wheel download URLs."), + "whl_file": attr.label(doc = "A caller-owned wheel to use instead of downloading."), +} | AUTH_ATTRS + +def _whl_file_repo_impl(rctx): + if rctx.attr.whl_file: + rctx.watch(rctx.attr.whl_file) + whl_path = rctx.path(rctx.attr.whl_file) + + # patch_whl deletes its input before repacking. Keep caller-owned + # wheels intact by giving it a repository-local symlink. + rctx.symlink(whl_path, whl_path.basename) + whl_path = rctx.path(whl_path.basename) + else: + whl_path = download_whl(rctx, logger = repo_utils.logger(rctx)) + if rctx.attr.patches: + whl_path = patch_whl( + rctx, + whl_path = whl_path, + patches = {patch: int(strip) for patch, strip in rctx.attr.patches.items()}, + ) + + # Patching temporarily unpacks the input. Retain only the replacement + # wheel; the installation is extracted in the consuming repository. + for path in rctx.path(".").readdir(): + if path.basename != whl_path.basename: + rctx.delete(path) + rctx.file("BUILD.bazel", "exports_files([{}])\n".format(repr(whl_path.basename))) + if hasattr(rctx, "repo_metadata"): + return rctx.repo_metadata(reproducible = True) + return None + +whl_file_repo = repository_rule( + implementation = _whl_file_repo_impl, + attrs = whl_file_repo_attrs | { + "_rule_name": attr.string(default = "whl_file_repo"), + }, + environ = [REPO_DEBUG_ENV_VAR], +) diff --git a/python/private/pypi/whl_library.bzl b/python/private/pypi/whl_library.bzl index 3f874b464b..3441a2c907 100644 --- a/python/private/pypi/whl_library.bzl +++ b/python/private/pypi/whl_library.bzl @@ -14,8 +14,10 @@ "" +load(":patch_whl.bzl", "get_whl_patches", "patched_whl_name") load(":pip_archive.bzl", "pip_archive", "pip_archive_attrs") load(":whl_archive.bzl", "whl_archive", "whl_archive_attrs") +load(":whl_file_repo.bzl", "whl_file_repo", "whl_file_repo_attrs") def _filter(kwargs, subset, debug = False): dropped = {} @@ -38,6 +40,12 @@ def whl_library(name, repo = None, **kwargs): * {obj}`whl_archive` * {obj}`pip_archive` + URL wheels are downloaded temporarily for extraction, then removed. + A sibling `_whl` repository supplies the original archive lazily, + when a wheel target is requested. Extraction does not depend on it. + Matching patches are applied in the wheel-file repository instead; + extraction and wheel targets then consume the same patched archive. + Args: name: {type}`str` The name of the repo. repo: Unused, will be dropped in the next major release. @@ -56,6 +64,22 @@ def whl_library(name, repo = None, **kwargs): kwargs.setdefault("dep_template", "@{}{{name}}//:{{target}}".format(kwargs.pop("repo_prefix", ""))) if whl_file or (urls and filename and filename.endswith(".whl")): + wheel_filename = filename or str(whl_file).split(":")[-1].split("/")[-1] + patches = get_whl_patches(kwargs.get("whl_patches", {}), wheel_filename) + if patches: + # Caller-owned wheels are often already supplied by _whl. + # Give their patched output a distinct repository name. + file_repo = name + ("_patched_whl" if whl_file else "_whl") + whl_file_repo( + name = file_repo, + patches = {patch: str(strip) for patch, strip in patches.items()}, + **_filter(kwargs, whl_file_repo_attrs) + ) + kwargs["whl_file"] = "@{}//:{}".format(file_repo, patched_whl_name(wheel_filename)) + kwargs["whl_patches"] = {} + elif not whl_file: + whl_file_repo(name = name + "_whl", **_filter(kwargs, whl_file_repo_attrs)) + kwargs["downloaded_whl"] = "@{}//:{}".format(name + "_whl", filename) whl_archive(name = name, **_filter(kwargs, whl_archive_attrs)) else: pip_archive(name = name, **_filter(kwargs, pip_archive_attrs)) diff --git a/python/private/pypi/whl_library_srcs.bzl b/python/private/pypi/whl_library_srcs.bzl index 0556219c93..44d47eb250 100644 --- a/python/private/pypi/whl_library_srcs.bzl +++ b/python/private/pypi/whl_library_srcs.bzl @@ -31,6 +31,7 @@ _VENV_SITE_PACKAGES_FLAG = Label("//python/config_settings:venvs_site_packages") def whl_library_srcs( *, name, + whl_file = None, sdist_filename = None, data_exclude = [], srcs_exclude = [], @@ -57,6 +58,7 @@ def whl_library_srcs( Args: name: {type}`str` The file to match for including it into the `whl` filegroup. This may be also parsed to generate extra metadata. + whl_file: {type}`str | None` The label of an externally stored wheel. sdist_filename: {type}`str | None` If the wheel was built from an sdist, the filename of the sdist. visibility: {type}`list[str]` The visibility of the source targets. @@ -178,7 +180,7 @@ def whl_library_srcs( if hasattr(native, "filegroup"): native.filegroup( name = WHEEL_FILE, - srcs = [name], + srcs = [whl_file or name], visibility = visibility, ) diff --git a/python/private/pypi/whl_library_targets.bzl b/python/private/pypi/whl_library_targets.bzl index 4f7bf8e98a..a1cc727460 100644 --- a/python/private/pypi/whl_library_targets.bzl +++ b/python/private/pypi/whl_library_targets.bzl @@ -23,6 +23,7 @@ whl_library_srcs = _whl_library_srcs def whl_library_targets( *, name, + whl_file = None, metadata_name = "", requires_dist = [], extras = [], @@ -46,6 +47,7 @@ def whl_library_targets( Args: name: {type}`str` The wheel filename + whl_file: {type}`str | None` The label of a wheel stored in another repository. metadata_name: {type}`str` The package name as written in wheel `METADATA`. group_deps: {type}`list[str]` names of fellow members of the group (if any). These will be excluded from generated deps lists so as to avoid @@ -72,6 +74,7 @@ def whl_library_targets( """ whl_library_srcs( name = name, + whl_file = whl_file, sdist_filename = sdist_filename, data_exclude = data_exclude, srcs_exclude = srcs_exclude, diff --git a/tests/integration/BUILD.bazel b/tests/integration/BUILD.bazel index 6c5d248202..21ef8fad16 100644 --- a/tests/integration/BUILD.bazel +++ b/tests/integration/BUILD.bazel @@ -149,6 +149,7 @@ rules_python_integration_test( rules_python_integration_test( name = "whl_library_test", + py_main = "whl_library_test.py", ) py_library( diff --git a/tests/integration/whl_library/BUILD.bazel b/tests/integration/whl_library/BUILD.bazel index a53bcc7258..0d2085b407 100644 --- a/tests/integration/whl_library/BUILD.bazel +++ b/tests/integration/whl_library/BUILD.bazel @@ -1,5 +1,38 @@ +load("@rules_python//python:py_binary.bzl", "py_binary") load("@rules_python//python:py_test.bzl", "py_test") +exports_files([ + "storage_pkg-1.0-py3-none-any.whl", + "wheel.sha256", + "modify_pkg.patch", + "uv.lock", +]) + +[ + py_binary( + name = name, + srcs = ["main.py"], + legacy_create_init = 0, + main = "main.py", + deps = ["@{}//:pkg".format(name)], + ) + for name in [ + "plain", + "patched", + "nonmatching", + "provided", + "provided_patched", + ] +] + +py_binary( + name = "uv_locked", + srcs = ["main.py"], + legacy_create_init = 0, + main = "main.py", + deps = ["@uv_pip//storage_pkg"], +) + [ alias( name = "{}_pkg".format(pkg), diff --git a/tests/integration/whl_library/MODULE.bazel b/tests/integration/whl_library/MODULE.bazel index d3886d7650..9059879187 100644 --- a/tests/integration/whl_library/MODULE.bazel +++ b/tests/integration/whl_library/MODULE.bazel @@ -59,3 +59,14 @@ whl_deps_repo( metadata_file = "@whl_archive//:metadata.json", requirement = "requests", ) + +wheels = use_extension("//:wheels.bzl", "wheels") +use_repo(wheels, "nonmatching", "patched", "plain", "provided", "provided_patched") + +pip = use_extension("@rules_python//python/extensions:pip.bzl", "pip") +pip.parse( + hub_name = "uv_pip", + python_version = "3.14", + uv_lock = "//:uv.lock", +) +use_repo(pip, "uv_pip") diff --git a/tests/integration/whl_library/main.py b/tests/integration/whl_library/main.py new file mode 100644 index 0000000000..ba55731016 --- /dev/null +++ b/tests/integration/whl_library/main.py @@ -0,0 +1,3 @@ +import storage_pkg + +print(f"PATCHED={storage_pkg.PATCHED}") diff --git a/tests/integration/whl_library/modify_pkg.patch b/tests/integration/whl_library/modify_pkg.patch new file mode 100644 index 0000000000..61c310b579 --- /dev/null +++ b/tests/integration/whl_library/modify_pkg.patch @@ -0,0 +1,5 @@ +--- storage_pkg/__init__.py ++++ storage_pkg/__init__.py +@@ -1 +1 @@ +-PATCHED = False ++PATCHED = True diff --git a/tests/integration/whl_library/wheels.bzl b/tests/integration/whl_library/wheels.bzl new file mode 100644 index 0000000000..b82f0c0479 --- /dev/null +++ b/tests/integration/whl_library/wheels.bzl @@ -0,0 +1,38 @@ +"""Local HTTP and caller-provided wheel fixtures.""" + +load("@rules_python//python/private/pypi:whl_library.bzl", "whl_library") # buildifier: disable=bzl-visibility + +_FILENAME = "storage_pkg-1.0-py3-none-any.whl" + +def _patches(filename): + return { + Label("//:modify_pkg.patch"): json.encode({ + "patch_strip": 0, + "whls": [filename], + }), + } + +def _wheels_impl(mrctx): + common = { + "envsubst": ["WHEEL_TEST_ADDRESS"], + "filename": _FILENAME, + "requirement": "storage-pkg", + "sha256": mrctx.read(Label("//:wheel.sha256")).strip(), + "urls": ["http://${WHEEL_TEST_ADDRESS}/" + _FILENAME], + } + whl_library(name = "plain", **common) + whl_library(name = "patched", whl_patches = _patches(_FILENAME), **common) + whl_library(name = "nonmatching", whl_patches = _patches("other.whl"), **common) + whl_library( + name = "provided", + requirement = "storage-pkg", + whl_file = Label("//:" + _FILENAME), + ) + whl_library( + name = "provided_patched", + requirement = "storage-pkg", + whl_file = Label("//:" + _FILENAME), + whl_patches = _patches(_FILENAME), + ) + +wheels = module_extension(implementation = _wheels_impl) diff --git a/tests/integration/whl_library_test.py b/tests/integration/whl_library_test.py new file mode 100644 index 0000000000..f61ad215c8 --- /dev/null +++ b/tests/integration/whl_library_test.py @@ -0,0 +1,154 @@ +"""Exercise lazy wheel retention across separate Bazel invocations.""" + +import functools +import hashlib +import http.server +import threading +import unittest +import zipfile +from pathlib import Path + +from tests.integration import runner + + +class WheelLibraryTest(runner.TestCase): + def setUp(self): + super().setUp() + self.wheel = self.repo_root / "storage_pkg-1.0-py3-none-any.whl" + with zipfile.ZipFile(self.wheel, "w") as wheel: + wheel.writestr("storage_pkg/__init__.py", "PATCHED = False\n") + wheel.writestr( + "storage_pkg-1.0.dist-info/METADATA", + "Metadata-Version: 2.1\nName: storage-pkg\nVersion: 1.0\n", + ) + wheel.writestr( + "storage_pkg-1.0.dist-info/WHEEL", + "Wheel-Version: 1.0\nRoot-Is-Purelib: true\nTag: py3-none-any\n", + ) + wheel.writestr("storage_pkg-1.0.dist-info/RECORD", "") + self.original_wheel = self.wheel.read_bytes() + (self.repo_root / "wheel.sha256").write_text( + hashlib.sha256(self.original_wheel).hexdigest() + ) + handler = functools.partial( + http.server.SimpleHTTPRequestHandler, directory=str(self.repo_root) + ) + server = http.server.ThreadingHTTPServer(("127.0.0.1", 0), handler) + self.addCleanup(server.server_close) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + self.addCleanup(thread.join) + self.addCleanup(server.shutdown) + self.bazel_env["WHEEL_TEST_ADDRESS"] = f"127.0.0.1:{server.server_port}" + base_url = f"http://127.0.0.1:{server.server_port}" + digest = hashlib.sha256(self.original_wheel).hexdigest() + (self.repo_root / "uv.lock").write_text( + f"""\ +version = 1 +revision = 3 +requires-python = ">=3.14" + +[[package]] +name = "storage-pkg" +version = "1.0" +source = {{ registry = "{base_url}/simple" }} +wheels = [ + {{ url = "{base_url}/{self.wheel.name}", hash = "sha256:{digest}", size = {len(self.original_wheel)} }}, +] +""" + ) + output_base = self.run_bazel("info", "output_base").stdout.strip() + self.external = Path(output_base) / "external" + + def repositories(self, name): + return [ + path + for path in self.external.iterdir() + if path.is_dir() and path.name.endswith(("+" + name, "~" + name)) + ] + + def repository(self, name): + repositories = self.repositories(name) + self.assertEqual(len(repositories), 1, repositories) + return repositories[0].resolve() + + def assert_runs(self, name, *, patched): + result = self.run_bazel("run", "//:" + name) + self.assert_result_matches(result, f"PATCHED={patched}") + + def test_lazy_archive(self): + self.assert_runs("plain", patched=False) + extracted = self.repository("plain") + self.assertEqual(list(extracted.glob("*.whl")), []) + self.assertEqual(self.repositories("plain_whl"), []) + + self.run_bazel("build", "@plain//:whl", "@plain//:whl_file") + archive = self.repository("plain_whl") / self.wheel.name + self.assertEqual(archive.read_bytes(), self.original_wheel) + self.assertEqual(list(extracted.glob("*.whl")), []) + + def test_matching_and_nonmatching_patches(self): + self.assert_runs("patched", patched=True) + patched = self.repository("patched") + self.assertEqual(list(patched.glob("*.whl")), []) + self.run_bazel("build", "@patched//:whl", "@patched//:whl_file") + with zipfile.ZipFile( + self.repository("patched_whl") / "storage_pkg-1.0+patched-py3-none-any.whl" + ) as wheel: + self.assertEqual(wheel.read("storage_pkg/__init__.py"), b"PATCHED = True\n") + self.assertFalse((self.repository("patched_whl") / self.wheel.name).exists()) + self.assertFalse((self.repository("patched_whl") / "storage_pkg").exists()) + self.assertEqual(list(patched.glob("*.whl")), []) + + self.assert_runs("nonmatching", patched=False) + self.assertEqual(list(self.repository("nonmatching").glob("*.whl")), []) + self.assertEqual(self.repositories("nonmatching_whl"), []) + self.run_bazel("build", "@nonmatching//:whl") + archive = self.repository("nonmatching_whl") / self.wheel.name + self.assertEqual(archive.read_bytes(), self.original_wheel) + + def test_caller_owned_wheel_is_preserved(self): + for name, patched in [("provided", False), ("provided_patched", True)]: + with self.subTest(name=name): + self.assert_runs(name, patched=patched) + self.run_bazel("build", f"@{name}//:whl", f"@{name}//:whl_file") + self.assertEqual(self.wheel.read_bytes(), self.original_wheel) + self.assertEqual(list(self.repository(name).glob("*.whl")), []) + self.assertEqual(self.repositories(name + "_whl"), []) + if patched: + with zipfile.ZipFile( + self.repository(name + "_patched_whl") + / "storage_pkg-1.0+patched-py3-none-any.whl" + ) as wheel: + self.assertEqual( + wheel.read("storage_pkg/__init__.py"), b"PATCHED = True\n" + ) + + def test_repository_contents(self): + # Retain the existing direct whl_archive, pip_archive, sdist, and + # dependency-wiring assertions in the same nested Bazel test. + self.run_bazel("test", "//:test_contents") + + def test_uv_lock_lazy_archive(self): + self.assert_runs("uv_locked", patched=False) + repositories = [ + path + for path in self.external.glob("*uv_pip_*") + if path.is_dir() and not path.name.endswith("_whl") + ] + self.assertEqual(len(repositories), 1, repositories) + extracted = repositories[0].resolve() + archive_repo = self.external / (repositories[0].name + "_whl") + self.assertEqual(list(extracted.glob("*.whl")), []) + self.assertFalse(archive_repo.exists()) + self.assertFalse(archive_repo.is_symlink()) + + self.run_bazel("build", "@uv_pip//storage_pkg:whl") + self.assertEqual( + (archive_repo / self.wheel.name).read_bytes(), self.original_wheel + ) + self.assertEqual(list(extracted.glob("*.whl")), []) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/pypi/integration/BUILD.bazel b/tests/pypi/integration/BUILD.bazel index 316abe8271..6a060cf7b0 100644 --- a/tests/pypi/integration/BUILD.bazel +++ b/tests/pypi/integration/BUILD.bazel @@ -1,5 +1,8 @@ load("@bazel_skylib//rules:build_test.bzl", "build_test") load("@rules_python_publish_deps//:requirements.bzl", "all_requirements") +load(":whl_storage_tests.bzl", "whl_storage_test_suite") + +whl_storage_test_suite(name = "whl_storage_tests") build_test( name = "all_requirements_build_test", diff --git a/tests/pypi/integration/whl_storage_tests.bzl b/tests/pypi/integration/whl_storage_tests.bzl new file mode 100644 index 0000000000..de469b10df --- /dev/null +++ b/tests/pypi/integration/whl_storage_tests.bzl @@ -0,0 +1,48 @@ +"""Check archive storage and runtime dependencies for a downloaded wheel.""" + +load("@rules_testing//lib:analysis_test.bzl", "analysis_test") +load("@rules_testing//lib:test_suite.bzl", "test_suite") +load("//tests/support:support.bzl", "SUPPORTS_BZLMOD") + +def _test_downloaded_archive(name): + analysis_test( + name = name, + target = "@rules_python_publish_deps//packaging:whl", + impl = _test_downloaded_archive_impl, + # WORKSPACE resolves this fixture through pip_archive, which retains + # its existing archive layout. Only bzlmod supplies a known wheel URL. + attr_values = {"target_compatible_with": SUPPORTS_BZLMOD}, + ) + +def _test_downloaded_archive_impl(env, target): + files = target[DefaultInfo].files.to_list() + env.expect.that_collection(files).has_size(1) + for file in files: + env.expect.that_bool(file.basename.endswith(".whl")).equals(True) + env.expect.that_bool(file.owner.workspace_name.endswith("_whl")).equals(True) + +def _test_runtime_excludes_archive(name): + analysis_test( + name = name, + target = "@rules_python_publish_deps//packaging:pkg", + impl = _test_runtime_excludes_archive_impl, + ) + +def _test_runtime_excludes_archive_impl(env, target): + files = target[DefaultInfo].default_runfiles.files.to_list() + env.expect.that_bool(bool(files)).equals(True) + env.expect.that_collection([ + file.short_path + for file in files + if file.extension == "whl" + ]).contains_exactly([]) + +def whl_storage_test_suite(name): + """Create tests for separate archive storage and wheel target compatibility.""" + test_suite( + name = name, + tests = [ + _test_downloaded_archive, + _test_runtime_excludes_archive, + ], + ) diff --git a/tests/pypi/whl_library_srcs/whl_library_srcs_tests.bzl b/tests/pypi/whl_library_srcs/whl_library_srcs_tests.bzl index fe4342b564..740d70e3fb 100644 --- a/tests/pypi/whl_library_srcs/whl_library_srcs_tests.bzl +++ b/tests/pypi/whl_library_srcs/whl_library_srcs_tests.bzl @@ -67,6 +67,26 @@ def _test_filegroups(env): _tests.append(_test_filegroups) +def _test_external_whl_file(env): + calls = [] + whl_library_srcs( + name = "example-1.0-py3-none-any.whl", + whl_file = "@example_whl//:example-1.0-py3-none-any.whl", + native = struct( + filegroup = lambda **kwargs: calls.append(kwargs), + glob = lambda *args, **kwargs: [], + ), + rules = struct(), + ) + wheel = [call for call in calls if call["name"] == "whl_file"][0] + env.expect.that_collection(wheel["srcs"]).contains_exactly([ + "@example_whl//:example-1.0-py3-none-any.whl", + ]) + extracted = [call for call in calls if call["name"] == "extracted_whl_files"][0] + env.expect.that_collection(extracted["srcs"]).contains_exactly([]) + +_tests.append(_test_external_whl_file) + def _test_copy(env): calls = []