feat(rules): describe runfiles groups for rules_runfiles_group

.bazelrc

@@ -91,3 +91,7 @@ import %workspace%/specialized_configs.bazelrc
 import %workspace%/buildbuddy.bazelrc
 
 try-import user.bazelrc
+
+# TODO: Delete before merging. rules_runfiles_group's describer API isn't in the
+# BCR yet; see https://github.com/bazelbuild/rules_java/pull/368.
+common --registry=https://bazel-contrib.github.io/rules_runfiles_group --registry=https://bcr.bazel.build

MODULE.bazel

@@ -9,6 +9,7 @@ bazel_dep(name = "bazel_skylib", version = "1.8.2")
 bazel_dep(name = "package_metadata", version = "0.0.7")
 bazel_dep(name = "platforms", version = "0.0.11")
 bazel_dep(name = "rules_cc", version = "0.2.17")
+bazel_dep(name = "rules_runfiles_group", version = "99.0.0-dev.20261019.1.d105dfa")
 bazel_dep(name = "toml.bzl", version = "0.4.1")
 
 # Those are loaded only when using py_proto_library

examples/bzlmod/.bazelrc

@@ -24,3 +24,6 @@ test --test_output=errors --enable_runfiles
 # Windows requires these for multi-python support:
 build --enable_runfiles
 common:bazel7.x --incompatible_python_disallow_native_rules
+
+# TODO: Delete before merging; see the root .bazelrc.
+common --registry=https://bazel-contrib.github.io/rules_runfiles_group --registry=https://bcr.bazel.build

examples/multi_python_versions/.bazelrc

@@ -5,3 +5,6 @@ build --enable_runfiles
 
 coverage --java_runtime_version=remotejdk_11
 common:bazel7.x --incompatible_python_disallow_native_rules
+
+# TODO: Delete before merging; see the root .bazelrc.
+common --registry=https://bazel-contrib.github.io/rules_runfiles_group --registry=https://bcr.bazel.build

examples/pip_parse/.bazelrc

@@ -1,3 +1,6 @@
 # https://docs.bazel.build/versions/main/best-practices.html#using-the-bazelrc-file
 try-import %workspace%/user.bazelrc
 common --incompatible_python_disallow_native_rules
+
+# TODO: Delete before merging; see the root .bazelrc.
+common --registry=https://bazel-contrib.github.io/rules_runfiles_group --registry=https://bcr.bazel.build

gazelle/.bazelrc

@@ -20,3 +20,6 @@ build --@rules_python//python/config_settings:incompatible_default_to_explicit_i
 build --enable_runfiles
 
 common:bazel7.x --incompatible_python_disallow_native_rules
+
+# TODO: Delete before merging; see the root .bazelrc.
+common --registry=https://bazel-contrib.github.io/rules_runfiles_group --registry=https://bcr.bazel.build

gazelle/examples/bzlmod_build_file_generation/.bazelrc

@@ -14,3 +14,6 @@ common:bazel7.x --incompatible_python_disallow_native_rules
 # rules_python code. In the BCR presubmits, this override is removed
 # and the bazel_dep version of rules_python is used.
 common --override_module=rules_python=../../../
+
+# TODO: Delete before merging; see the root .bazelrc.
+common --registry=https://bazel-contrib.github.io/rules_runfiles_group --registry=https://bcr.bazel.build

internal_dev_deps.bzl

@@ -252,9 +252,9 @@ def rules_python_internal_deps():
 
     http_archive(
         name = "bazel_features",
-        sha256 = "d7787da289a7fb497352211ad200ec9f698822a9e0757a4976fd9f713ff372b3",
-        strip_prefix = "bazel_features-1.9.1",
-        url = "https://github.com/bazel-contrib/bazel_features/releases/download/v1.9.1/bazel_features-v1.9.1.tar.gz",
+        sha256 = "5450bfb2c8b4bc961c75368838f86156f563cc9adef1be7d504fc5619d54daab",
+        strip_prefix = "bazel_features-1.51.0",
+        url = "https://github.com/bazel-contrib/bazel_features/releases/download/v1.51.0/bazel_features-v1.51.0.tar.gz",
     )
 
     http_archive(

news/3757-provider-fields.added.md

@@ -0,0 +1,2 @@
+(rules) Added {obj}`PyInfo.direct_implicit_pyc_files` and
+{obj}`PyExecutableInfo.venv_runfiles`.

news/3757.added.md

@@ -0,0 +1,5 @@
+(rules) `py_binary` and `py_test` describe their runfiles groups for
+[rules_runfiles_group](https://github.com/bazel-contrib/rules_runfiles_group),
+so packaging rules like rules_img's `image_from_binary` can put the runtime,
+each PyPI package and each library in a layer of its own. Needs Bazel 8.8 or
+later ([#3757](https://github.com/bazel-contrib/rules_python/issues/3757)).

python/private/BUILD.bazel

@@ -20,6 +20,12 @@ load(":bazel_config_mode.bzl", "bazel_config_mode")
 load(":distribution_filegroup.bzl", "distribution_filegroup")
 load(":py_exec_tools_toolchain.bzl", "current_interpreter_executable")
 load(":py_interpreter_program.bzl", "py_interpreter_program")
+load(
+    ":runfiles_groups.bzl",
+    "py_executable_runfiles_group_describer",
+    "py_library_runfiles_group_describer",
+    "py_runtime_pair_runfiles_group_describer",
+)
 load(":sentinel_impl.bzl", "sentinel")
 load(":stamp_impl.bzl", "stamp_build_setting")
 load(":uncachable_version_file.bzl", "define_uncachable_version_file")
@@ -845,6 +851,23 @@ bzl_library(
     ],
 )
 
+bzl_library(
+    name = "runfiles_groups",
+    srcs = ["runfiles_groups.bzl"],
+    deps = [
+        ":attributes",
+        ":common",
+        ":flags",
+        ":py_executable",
+        ":py_executable_info",
+        ":py_info",
+        ":py_internal",
+        ":py_library",
+        ":reexports",
+        "@rules_runfiles_group//runfiles_group:describe",
+    ],
+)
+
 bzl_library(
     name = "runtime_env_repo",
     srcs = ["runtime_env_repo.bzl"],
@@ -1040,3 +1063,21 @@ bzl_library(
     name = "visibility",
     srcs = ["visibility.bzl"],
 )
+
+# Runfiles group describers of the Python rules (see runfiles_groups.bzl), held
+# by their _aspect_rule_hints attributes. Not actually public; only public
+# because they're implicit dependencies of rules built via the rule builders.
+py_library_runfiles_group_describer(
+    name = "py_library_runfiles_group_describer",
+    visibility = ["//visibility:public"],
+)
+
+py_executable_runfiles_group_describer(
+    name = "py_executable_runfiles_group_describer",
+    visibility = ["//visibility:public"],
+)
+
+py_runtime_pair_runfiles_group_describer(
+    name = "py_runtime_pair_runfiles_group_describer",
+    visibility = ["//visibility:public"],
+)

python/private/common.bzl

@@ -283,6 +283,28 @@ def get_imports(ctx):
         result.append(import_path)
     return result
 
+def py_info_runfiles_files(info, pyc_collection_enabled):
+    """The files a binary puts into its runfiles for a dependency's PyInfo.
+
+    Args:
+        info: PyInfo or BuiltinPyInfo of a dependency.
+        pyc_collection_enabled: bool; whether the binary includes implicit pyc
+            files instead of their sources.
+
+    Returns:
+        list of depset[File]
+    """
+    files = [info.transitive_sources]
+
+    # BuiltinPyInfo doesn't have the pyc fields
+    if hasattr(info, "transitive_pyc_files"):
+        files.append(info.transitive_pyc_files)
+        if pyc_collection_enabled:
+            files.append(info.transitive_implicit_pyc_files)
+        else:
+            files.append(info.transitive_implicit_pyc_source_files)
+    return files
+
 def collect_runfiles(ctx, files = depset()):
     """Collects the necessary files from the rule's context.
 
@@ -392,6 +414,7 @@ def create_py_info(
     py_info.transitive_original_sources.add(original_sources)
     py_info.transitive_pyc_files.add(required_pyc_files)
     py_info.transitive_pyi_files.add(ctx.files.pyi_srcs)
+    py_info.direct_implicit_pyc_files.add(implicit_pyc_files)
     py_info.transitive_implicit_pyc_files.add(implicit_pyc_files)
     py_info.transitive_implicit_pyc_source_files.add(implicit_pyc_source_files)
     py_info.imports.add(imports)

python/private/py_executable.bzl

@@ -55,6 +55,7 @@ load(
     "is_bool",
     "is_windows_platform",
     "maybe_create_repo_mapping",
+    "py_info_runfiles_files",
     "relative_path",
     "runfiles_root_path",
 )
@@ -92,10 +93,17 @@ EXECUTABLE_ATTRS = dicts.add(
     WINDOWS_CONSTRAINTS_ATTRS,
     # starlark flags attributes
     {
+        # Read by rules_runfiles_group's packaging aspects; see runfiles_groups.bzl.
+        "_aspect_rule_hints": attr.label_list(default = [
+            "//python/private:py_executable_runfiles_group_describer",
+            # What //python:toolchain_type is an alias of.
+            "@bazel_tools//tools/python:toolchain_type",
+        ]),
         "_build_python_zip_flag": attr.label(default = "//python/config_settings:build_python_zip"),
         "_default_to_explicit_init_py_flag": attr.label(default = "//python/config_settings:incompatible_default_to_explicit_init_py"),
         "_python_import_all_repositories_flag": attr.label(default = "//python/config_settings:experimental_python_import_all_repositories"),
         "_python_path_flag": attr.label(default = "//python/config_settings:python_path"),
+        "_runfiles_group_attrs": attr.string_list(default = ["_debugger_flag", "data", "deps", "srcs"]),
     },
     {
         "interpreter_args": lambda: attrb.StringList(
@@ -364,11 +372,8 @@ def _create_executable(
             venv = venv,
             build_data_file = runfiles_details.build_data_file,
         )
-        extra_runfiles = ctx.runfiles(
-            [stage2_bootstrap] + (
-                venv.files_without_interpreter if venv else []
-            ),
-        ).merge(venv.lib_runfiles)
+        venv_runfiles = ctx.runfiles(venv.files_without_interpreter).merge(venv.lib_runfiles)
+        extra_runfiles = ctx.runfiles([stage2_bootstrap]).merge(venv_runfiles)
         zip_main = _create_zip_main(
             ctx,
             stage2_bootstrap = stage2_bootstrap,
@@ -377,6 +382,7 @@ def _create_executable(
         )
     else:
         stage2_bootstrap = None
+        venv_runfiles = None
         extra_runfiles = ctx.runfiles()
         zip_main = ctx.actions.declare_file(base_executable_name + ".temp", sibling = executable)
         _create_stage1_bootstrap(
@@ -495,9 +501,8 @@ WARNING: Target: {}
 
     app_runfiles = builders.RunfilesBuilder()
     app_runfiles.add(runfiles_details.app_runfiles)
-    if venv:
-        app_runfiles.add(venv.files_without_interpreter)
-        app_runfiles.add(venv.lib_runfiles)
+    if venv_runfiles:
+        app_runfiles.add(venv_runfiles)
 
     # The interpreter is added this late in the process so that it isn't
     # added to the zipped files.
@@ -524,6 +529,8 @@ WARNING: Target: {}
         venv_python_exe = venv.interpreter if venv else None,
         # runfiles|None; runfiles in the venv for the interpreter
         venv_interpreter_runfiles = venv.interpreter_runfiles if venv else None,
+        # runfiles|None; runfiles in the venv for the application
+        venv_runfiles = venv_runfiles,
         # depset[ExplicitSymlink]|None; symlinks that should be created
         venv_interpreter_symlinks = venv.interpreter_symlinks if venv else None,
     )
@@ -1317,6 +1324,7 @@ def py_executable_base_impl(ctx, *, semantics, is_test, inherited_environment =
         venv_interpreter_runfiles = exec_result.venv_interpreter_runfiles,
         venv_interpreter_symlinks = exec_result.venv_interpreter_symlinks,
         venv_python_exe = exec_result.venv_python_exe,
+        venv_runfiles = exec_result.venv_runfiles,
     )
     _add_provider_py_runtime_info(providers, runtime_details)
     _add_provider_py_cc_link_params_info(providers, cc_details.cc_info_for_propagating)
@@ -1444,19 +1452,7 @@ def _get_runtime_details(ctx):
         fail("'python_path' must be an absolute path or a name to be resolved from the system PATH (e.g., 'python', 'python3').")
 
     effective_runtime = _maybe_get_runtime_from_ctx(ctx)
-
-    direct = []  # List of files
-    transitive = []  # List of depsets
-    if effective_runtime.interpreter:
-        direct.append(effective_runtime.interpreter)
-        transitive.append(effective_runtime.files)
-
-    if ctx.configuration.coverage_enabled:
-        if effective_runtime.coverage_tool:
-            direct.append(effective_runtime.coverage_tool)
-        if effective_runtime.coverage_files:
-            transitive.append(effective_runtime.coverage_files)
-    runtime_files = depset(direct = direct, transitive = transitive)
+    runtime_files = get_runtime_files(ctx, effective_runtime)
 
     executable_interpreter_path = _get_interpreter_path(
         ctx,
@@ -1478,6 +1474,29 @@ def _get_runtime_details(ctx):
         runfiles = ctx.runfiles(transitive_files = runtime_files),
     )
 
+def get_runtime_files(ctx, runtime):
+    """Returns the runtime files an executable puts into its runfiles.
+
+    Args:
+        ctx: The rule or aspect ctx.
+        runtime: PyRuntimeInfo; the runtime of the executable.
+
+    Returns:
+        depset[File]
+    """
+    direct = []  # List of files
+    transitive = []  # List of depsets
+    if runtime.interpreter:
+        direct.append(runtime.interpreter)
+        transitive.append(runtime.files)
+
+    if ctx.configuration.coverage_enabled:
+        if runtime.coverage_tool:
+            direct.append(runtime.coverage_tool)
+        if runtime.coverage_files:
+            transitive.append(runtime.coverage_files)
+    return depset(direct = direct, transitive = transitive)
+
 def _maybe_get_runtime_from_ctx(ctx):
     """Finds the PyRuntimeInfo from the toolchain or attribute, if available.
 
@@ -1556,17 +1575,7 @@ def _get_base_runfiles_for_binary(
         if not (PyInfo in dep or (BuiltinPyInfo != None and BuiltinPyInfo in dep)):
             continue
         info = dep[PyInfo] if PyInfo in dep else dep[BuiltinPyInfo]
-        app_runfiles.files.add(info.transitive_sources)
-
-        # Everything past this won't work with BuiltinPyInfo
-        if not hasattr(info, "transitive_pyc_files"):
-            continue
-
-        app_runfiles.files.add(info.transitive_pyc_files)
-        if pyc_collection_enabled:
-            app_runfiles.files.add(info.transitive_implicit_pyc_files)
-        else:
-            app_runfiles.files.add(info.transitive_implicit_pyc_source_files)
+        app_runfiles.files.add(py_info_runfiles_files(info, pyc_collection_enabled))
 
     app_runfiles.runfiles.append(collect_runfiles(ctx))
     if extra_deps:
@@ -2017,7 +2026,8 @@ def _add_provider_py_executable_info(
         venv_app_symlinks,
         venv_interpreter_runfiles,
         venv_interpreter_symlinks,
-        venv_python_exe):
+        venv_python_exe,
+        venv_runfiles):
     """Adds the PyExecutableInfo provider.
 
     Args:
@@ -2034,6 +2044,7 @@ def _add_provider_py_executable_info(
         venv_interpreter_runfiles: runfiles; runfiles specific to the interpreter for the venv.
         venv_interpreter_symlinks: depset[ExplicitSymlink]; interpreter-specific symlinks to create for the venv.
         venv_python_exe: File; the python executable in the venv.
+        venv_runfiles: runfiles; the venv's files for the application.
     """
     providers.append(PyExecutableInfo(
         app_runfiles = app_runfiles,
@@ -2047,6 +2058,7 @@ def _add_provider_py_executable_info(
         venv_interpreter_runfiles = venv_interpreter_runfiles,
         venv_interpreter_symlinks = venv_interpreter_symlinks,
         venv_python_exe = venv_python_exe,
+        venv_runfiles = venv_runfiles,
     ))
 
 def _add_provider_py_runtime_info(providers, runtime_details):

python/private/py_executable_info.bzl

@@ -119,6 +119,16 @@ mode is not enabled.
 
 :::{versionadded} 1.9.0
 :::
+""",
+        "venv_runfiles": """
+:type: runfiles | None
+
+Runfiles that are specific to the application within the venv: its `site-packages`
+files and the symlinks to the application's dependencies. A subset of
+{obj}`app_runfiles`. May be None if venv mode is not enabled.
+
+:::{versionadded} VERSION_NEXT_FEATURE
+:::
 """,
     },
 )