From 01e85628da2abf42c1a07f98fa5d6d16083184cc Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Wed, 21 Oct 2026 23:06:23 -0700 Subject: [PATCH 1/2] build(release): cross-build all Linux release artifacts in one RBE job Spinning up a separate GitHub Actions runner for each of the six Linux platforms is unnecessary now that Linux release builds run on BuildBuddy RBE. Replace the six Linux matrix entries with a single `linux` job that builds `//dev/release_artifacts:cross_compiled_artifacts`, and include `x86_64-unknown-linux-gnu` in that target along with a manifest file so one runner builds and verifies all Linux binaries in a single Bazel invocation. Work towards #4216 --- .../workflows/release_build_artifacts.yaml | 17 ++++--------- .github/workflows/release_publish.yaml | 4 +--- dev/release_artifacts/BUILD.bazel | 1 + dev/release_artifacts/build.sh | 20 +++++++++++----- dev/release_artifacts/release_files.bzl | 24 ++++++++++++++++--- dev/release_artifacts/verify.sh | 16 +++++++++++-- .../workflows/release_build_artifacts_test.py | 5 +++- 7 files changed, 59 insertions(+), 28 deletions(-) diff --git a/.github/workflows/release_build_artifacts.yaml b/.github/workflows/release_build_artifacts.yaml index ca2cdcdbe7..073c863237 100644 --- a/.github/workflows/release_build_artifacts.yaml +++ b/.github/workflows/release_build_artifacts.yaml @@ -55,20 +55,11 @@ jobs: runner: macos-15-intel - triple: x86_64-pc-windows-msvc runner: windows-2022 - - triple: x86_64-unknown-linux-gnu - runner: ubuntu-24.04 - # Cross-compiled; see dev/platforms/BUILD.bazel. - - triple: aarch64-unknown-linux-musl - runner: ubuntu-24.04 - - triple: armv7-unknown-linux-musleabihf - runner: ubuntu-24.04 + # Cross-compiled; see dev/platforms/BUILD.bazel and + # dev/release_artifacts/BUILD.bazel. - triple: i686-pc-windows-msvc runner: windows-2022 - - triple: i686-unknown-linux-musl - runner: ubuntu-24.04 - - triple: powerpc64le-unknown-linux-musl - runner: ubuntu-24.04 - - triple: riscv64gc-unknown-linux-musl + - triple: linux runner: ubuntu-24.04 env: REF: ${{ inputs.ref || github.sha }} @@ -198,7 +189,7 @@ jobs: if: hashFiles('artifacts_manifest.json') != '' uses: actions/upload-artifact@v7 with: - name: release-artifacts-manifest + name: release-manifest path: artifacts_manifest.json if-no-files-found: error diff --git a/.github/workflows/release_publish.yaml b/.github/workflows/release_publish.yaml index 0e17bbb8ce..49bb174f42 100644 --- a/.github/workflows/release_publish.yaml +++ b/.github/workflows/release_publish.yaml @@ -72,9 +72,7 @@ jobs: - name: Download built artifacts uses: actions/download-artifact@v8 with: - # Match platform triples (--), excluding - # release-artifacts-manifest. - pattern: release-artifacts-*-* + pattern: release-artifacts-* path: release-artifacts merge-multiple: true diff --git a/dev/release_artifacts/BUILD.bazel b/dev/release_artifacts/BUILD.bazel index 8f5f0c673e..26895e1190 100644 --- a/dev/release_artifacts/BUILD.bazel +++ b/dev/release_artifacts/BUILD.bazel @@ -40,6 +40,7 @@ cross_compiled_files( "//dev/platforms:i686-unknown-linux-musl", "//dev/platforms:powerpc64le-unknown-linux-musl", "//dev/platforms:riscv64gc-unknown-linux-musl", + "//dev/platforms:x86_64-unknown-linux-gnu", ], ":is_macos_aarch64": ["//dev/platforms:x86_64-apple-darwin"], ":is_windows_x86_64": ["//dev/platforms:i686-pc-windows-msvc"], diff --git a/dev/release_artifacts/build.sh b/dev/release_artifacts/build.sh index 3ceb0c8b03..4059bd01c0 100755 --- a/dev/release_artifacts/build.sh +++ b/dev/release_artifacts/build.sh @@ -14,19 +14,27 @@ triple="$2" # The paths below are relative to the workspace root. cd "$(dirname "$0")/../.." -# Fail for a triple without a platform: it's the caller's mistake, unlike a -# failed build, which only warns. -if ! grep -qF "name = \"$triple\"," dev/platforms/BUILD.bazel; then +if [[ "$triple" == "linux" ]]; then + build_args=(//dev/release_artifacts:cross_compiled_artifacts) + manifest="bazel-bin/dev/release_artifacts/cross_compiled_artifacts.txt" +elif ! grep -qF "name = \"$triple\"," dev/platforms/BUILD.bazel; then + # Fail for a triple without a platform: it's the caller's mistake, unlike a + # failed build, which only warns. echo "::error::dev/platforms has no $triple platform" exit 1 +else + build_args=( + --platforms="//dev/platforms:$triple" + //dev/release_artifacts:artifacts_for_release + ) + manifest="bazel-bin/dev/release_artifacts/artifacts_for_release.txt" fi # Keep Git Bash on Windows from rewriting `//foo` labels into paths. export MSYS_NO_PATHCONV=1 MSYS2_ARG_CONV_EXCL="*" if ! bazel build --verbose_failures --compilation_mode=opt --stamp \ - --embed_label="$embed_label" --platforms="//dev/platforms:$triple" \ - //dev/release_artifacts:artifacts_for_release; then + --embed_label="$embed_label" "${build_args[@]}"; then # Releases don't use the files yet, so don't fail the release over them. echo "::warning::Building the release artifacts failed. Releases don't use them yet." exit 0 @@ -36,7 +44,7 @@ fi if [[ -n "${GITHUB_OUTPUT:-}" ]]; then { echo "files<> "$GITHUB_OUTPUT" fi diff --git a/dev/release_artifacts/release_files.bzl b/dev/release_artifacts/release_files.bzl index 32a84b3c91..e9604cd9a3 100644 --- a/dev/release_artifacts/release_files.bzl +++ b/dev/release_artifacts/release_files.bzl @@ -5,6 +5,11 @@ load("//python/private:common_labels.bzl", "labels") # buildifier: disable=bzl- _FEATURES = "//command_line_option:features" _RUSTC_FLAGS = "@rules_rust//rust/settings:extra_rustc_flags" +_ReleaseFilesInfo = provider( + doc = "Files built by _release_files, excluding the .txt manifest.", + fields = ["files"], +) + def _release_transition_impl(settings, attr): features = [] rustc_flags = [] @@ -82,7 +87,10 @@ def _release_files_impl(ctx): # an unterminated last line, and build.sh adds a line after the paths. content = "".join([file.path + "\n" for file in files]), ) - return [DefaultInfo(files = depset([manifest] + files))] + return [ + DefaultInfo(files = depset([manifest] + files)), + _ReleaseFilesInfo(files = files), + ] _release_files = rule( implementation = _release_files_impl, @@ -168,8 +176,17 @@ def _cross_compiled_files_impl(ctx): return [DefaultInfo()] # With a transition, ctx.attr.src is a list: one target per platform. - files = [src[DefaultInfo].files for src in ctx.attr.src] - return [DefaultInfo(files = depset(transitive = files))] + files = [] + for src in ctx.attr.src: + files.extend(src[_ReleaseFilesInfo].files) + manifest = ctx.actions.declare_file(ctx.label.name + ".txt") + ctx.actions.write( + output = manifest, + # End every line with "\n", including the last: `while read` loops skip + # an unterminated last line, and build.sh adds a line after the paths. + content = "".join([file.path + "\n" for file in files]), + ) + return [DefaultInfo(files = depset([manifest] + files))] cross_compiled_files = rule( implementation = _cross_compiled_files_impl, @@ -188,6 +205,7 @@ this builds nothing. cfg = _platforms_transition, doc = "The target to build.", mandatory = True, + providers = [_ReleaseFilesInfo], ), }, ) diff --git a/dev/release_artifacts/verify.sh b/dev/release_artifacts/verify.sh index 0c55db7caa..bce09cf97a 100755 --- a/dev/release_artifacts/verify.sh +++ b/dev/release_artifacts/verify.sh @@ -7,13 +7,25 @@ # Usage: dev/release_artifacts/verify.sh TRIPLE set -euo pipefail -triple="$1" +triple_arg="$1" # The paths below are relative to the workspace root. cd "$(dirname "$0")/../.." +if [[ "$triple_arg" == "linux" ]]; then + manifest="bazel-bin/dev/release_artifacts/cross_compiled_artifacts.txt" +else + manifest="bazel-bin/dev/release_artifacts/artifacts_for_release.txt" +fi + # The target lists the paths of its files in a manifest; see release_files.bzl. while IFS= read -r bin; do + if [[ "$triple_arg" == "linux" ]]; then + name="${bin##*/}" + triple="${name#*-}" + else + triple="$triple_arg" + fi case "$triple" in *-linux-*) info="$(file -L -b "$bin")" @@ -84,4 +96,4 @@ while IFS= read -r bin; do exit 1 ;; esac -done < bazel-bin/dev/release_artifacts/artifacts_for_release.txt +done < "$manifest" diff --git a/tests/workflows/release_build_artifacts_test.py b/tests/workflows/release_build_artifacts_test.py index c3306b2bac..de4dd35fde 100644 --- a/tests/workflows/release_build_artifacts_test.py +++ b/tests/workflows/release_build_artifacts_test.py @@ -39,8 +39,11 @@ def test_matrix_matches_dev_platforms(): # e.g. because of a comment. calls = re.findall(r"^platform\(", build_file, re.MULTILINE) assert len(platforms) == len(calls), "Couldn't parse every platform's name" + expected = { + "linux" if "-linux-" in platform else platform for platform in platforms + } # Sets, so that failures list the triples that differ. - assert set(triples) == set(platforms), "Matrix (left) vs. dev/platforms (right)" + assert set(triples) == expected, "Matrix (left) vs. dev/platforms (right)" duplicates = sorted({triple for triple in triples if triples.count(triple) > 1}) assert not duplicates, f"Triples with more than one job: {duplicates}" From b00565b67270755bb93194b13237074f3ec3ee57 Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Wed, 21 Oct 2026 23:48:34 -0700 Subject: [PATCH 2/2] build(release): stage built artifacts into one directory before upload Cross-compiled Linux outputs live in separate bazel-out/ directories, causing actions/upload-artifact to preserve those subdirectories when uploading raw manifest paths. Copy the manifest files into release-artifacts/ so they are uploaded flat. --- dev/release_artifacts/build.sh | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/dev/release_artifacts/build.sh b/dev/release_artifacts/build.sh index 4059bd01c0..b84d128d2f 100755 --- a/dev/release_artifacts/build.sh +++ b/dev/release_artifacts/build.sh @@ -41,10 +41,13 @@ if ! bazel build --verbose_failures --compilation_mode=opt --stamp \ fi # The target lists the paths of its files in a manifest; see release_files.bzl. +# Copy them into a single directory so upload-artifact strips the directory +# prefix instead of preserving per-platform bazel-out/ subdirectories. if [[ -n "${GITHUB_OUTPUT:-}" ]]; then - { - echo "files<> "$GITHUB_OUTPUT" + rm -rf release-artifacts + mkdir -p release-artifacts + while IFS= read -r file; do + cp "$file" release-artifacts/ + done < "$manifest" + echo "files=release-artifacts/*" >> "$GITHUB_OUTPUT" fi