From b44eb2f02f488035f2aaf90a9b5062eb1be8bdbf Mon Sep 17 00:00:00 2001 From: sdf-jkl Date: Thu, 29 Oct 2026 12:28:07 -0400 Subject: [PATCH] ci: prototype required validation for a merge queue --- .asf.yaml | 33 ++++++- .github/workflows/README.md | 52 ++++++++++ .github/workflows/archery.yml | 14 ++- .github/workflows/cpp.yml | 4 +- .github/workflows/cpp_extra.yml | 1 + .github/workflows/cuda_extra.yml | 1 + .github/workflows/dev.yml | 4 +- .github/workflows/docs.yml | 2 + .github/workflows/docs_light.yml | 1 + .github/workflows/integration.yml | 4 +- .github/workflows/matlab.yml | 4 +- .github/workflows/merge_queue.yml | 122 ++++++++++++++++++++++++ .github/workflows/package_linux.yml | 1 + .github/workflows/python.yml | 4 +- .github/workflows/r.yml | 4 +- .github/workflows/r_extra.yml | 1 + .github/workflows/release_candidate.yml | 1 + .github/workflows/ruby.yml | 4 +- dev/test_merge_queue.py | 84 ++++++++++++++++ 19 files changed, 328 insertions(+), 13 deletions(-) create mode 100644 .github/workflows/README.md create mode 100644 .github/workflows/merge_queue.yml create mode 100644 dev/test_merge_queue.py diff --git a/.asf.yaml b/.asf.yaml index 64a06da6b9c1..516025977ef3 100644 --- a/.asf.yaml +++ b/.asf.yaml @@ -28,8 +28,37 @@ github: discussions: true protected_branches: - # protect main against force push - main: {} + main: + required_status_checks: + # The queue tests the combined commit against current main. + strict: false + contexts: + - Merge Queue + + rulesets: + - name: Merge Queue + target: branch + enforcement: active + bypass_actors: + # apache/root (ASF Infra) can recover a blocked queue. + - actor_id: 118420 + actor_type: Team + bypass_mode: always + conditions: + ref_name: + include: [refs/heads/main] + exclude: [] + rules: + - type: merge_queue + parameters: + merge_method: SQUASH + grouping_strategy: ALLGREEN + # Start with one candidate at a time to bound runner demand. + max_entries_to_build: 1 + min_entries_to_merge: 1 + max_entries_to_merge: 1 + min_entries_to_merge_wait_minutes: 0 + check_response_timeout_minutes: 360 collaborators: - alinaliBQ diff --git a/.github/workflows/README.md b/.github/workflows/README.md new file mode 100644 index 000000000000..5a2b5610b21c --- /dev/null +++ b/.github/workflows/README.md @@ -0,0 +1,52 @@ + + +# Merge queue + +`merge_queue.yml` supplies the required **Merge Queue** check for `main`. +On pull requests it permits entry into the queue without repeating the existing +PR workflows. On `merge_group`, it runs every regular language and tooling +workflow against the candidate containing current `main` and the queued change. +Every called workflow must succeed; failure, cancellation, and skipped suites +block merging. Extra, packaging, and release workflows retain their separate +triggers and are not part of this gate. + +The `.asf.yaml` configuration activates the queue with one candidate build at a +time and a six-hour check timeout, including runner scheduling. It leaves the +existing PR and post-merge CI in place. This first step adds validation before +merge; it does not reduce CI usage. Queue branch pushes are excluded so they +cannot start duplicate builds. + +Use GitHub's merge queue UI to enqueue a reviewed PR. `dev/merge_arrow_pr.py` +uses the direct merge API and cannot enqueue or wait for a queued merge. Close +the associated issue and set its milestone after the queued PR actually merges. +Maintenance branches retain their existing merge procedure. + +To exercise the same build suite on a fork with the workflow installed: + +```shell +gh workflow run merge_queue.yml --repo OWNER/arrow --ref BRANCH +``` + +GitHub requires an organization-owned repository for a native merge queue. +A manual fork run checks the suite and result aggregation, but does not test +queue ordering, stale-candidate replacement, or automatic merging. Validate +those behaviors and runner availability before enabling the proposed rules in +Apache Arrow. Queue settings follow the +[GitHub merge queue documentation](https://docs.github.com/en/repositories/configuring-branches-and-merges-in-your-repository/configuring-pull-request-merges/managing-a-merge-queue). diff --git a/.github/workflows/archery.yml b/.github/workflows/archery.yml index 6ae257e0adad..908b69b69b65 100644 --- a/.github/workflows/archery.yml +++ b/.github/workflows/archery.yml @@ -18,22 +18,28 @@ name: Archery & Crossbow on: + workflow_call: push: branches: - '**' - '!dependabot/**' + - '!gh-readonly-queue/**' tags: - '**' paths: - '.dockerignore' - - '.github/workflows/archery.yml' + - '.asf.yaml' + - '.github/workflows/**' + - 'dev/test_merge_queue.py' - 'dev/archery/**' - 'dev/tasks/**' - 'compose.yaml' pull_request: paths: - '.dockerignore' - - '.github/workflows/archery.yml' + - '.asf.yaml' + - '.github/workflows/**' + - 'dev/test_merge_queue.py' - 'dev/archery/**' - 'dev/tasks/**' - 'compose.yaml' @@ -43,7 +49,7 @@ env: ARCHERY_DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} concurrency: - group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} + group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }}-archery cancel-in-progress: true permissions: @@ -77,7 +83,7 @@ jobs: pip install -r dev/archery/requirements-test.txt - name: Archery Unittests working-directory: dev/archery - run: pytest -v archery + run: pytest -v archery ../test_merge_queue.py - name: Archery Docker Validation run: archery docker check-config - name: Crossbow Check Config diff --git a/.github/workflows/cpp.yml b/.github/workflows/cpp.yml index 21bf615c2ff4..f66e1400c207 100644 --- a/.github/workflows/cpp.yml +++ b/.github/workflows/cpp.yml @@ -18,10 +18,12 @@ name: C++ on: + workflow_call: push: branches: - '**' - '!dependabot/**' + - '!gh-readonly-queue/**' tags: - '**' paths: @@ -61,7 +63,7 @@ on: - 'testing' concurrency: - group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} + group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }}-cpp cancel-in-progress: true permissions: diff --git a/.github/workflows/cpp_extra.yml b/.github/workflows/cpp_extra.yml index 1aac6a28d680..2043af20c263 100644 --- a/.github/workflows/cpp_extra.yml +++ b/.github/workflows/cpp_extra.yml @@ -22,6 +22,7 @@ on: branches: - '**' - '!dependabot/**' + - '!gh-readonly-queue/**' paths: - '.dockerignore' - '.github/workflows/check_labels.yml' diff --git a/.github/workflows/cuda_extra.yml b/.github/workflows/cuda_extra.yml index e9896666730f..544ad4cdd448 100644 --- a/.github/workflows/cuda_extra.yml +++ b/.github/workflows/cuda_extra.yml @@ -22,6 +22,7 @@ on: branches: - '**' - '!dependabot/**' + - '!gh-readonly-queue/**' paths: - '**/*cuda*' - '**/*gpu*' diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml index 7c3607bd1df3..b11abc8e8704 100644 --- a/.github/workflows/dev.yml +++ b/.github/workflows/dev.yml @@ -18,17 +18,19 @@ name: Dev on: + workflow_call: # always trigger except Dependabot "push" push: branches: - '**' - '!dependabot/**' + - '!gh-readonly-queue/**' tags: - '**' pull_request: concurrency: - group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} + group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }}-dev cancel-in-progress: true permissions: diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index d176095739fb..ffd17ed58ccc 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -18,10 +18,12 @@ name: Docs on: + workflow_call: push: branches: - '**' - '!dependabot/**' + - '!gh-readonly-queue/**' tags: - '**' pull_request: diff --git a/.github/workflows/docs_light.yml b/.github/workflows/docs_light.yml index 9e62aacfa741..b312d17f18a0 100644 --- a/.github/workflows/docs_light.yml +++ b/.github/workflows/docs_light.yml @@ -18,6 +18,7 @@ name: Docs on: + workflow_call: pull_request: paths: - '.dockerignore' diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 197eb19b1510..ba24c7530731 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -18,10 +18,12 @@ name: Integration on: + workflow_call: push: branches: - '**' - '!dependabot/**' + - '!gh-readonly-queue/**' tags: - '**' paths: @@ -47,7 +49,7 @@ on: - 'ruby/red-arrow-format/**' concurrency: - group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} + group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }}-integration cancel-in-progress: true permissions: diff --git a/.github/workflows/matlab.yml b/.github/workflows/matlab.yml index da36b83bbec9..1d2b12482ba3 100644 --- a/.github/workflows/matlab.yml +++ b/.github/workflows/matlab.yml @@ -18,10 +18,12 @@ name: MATLAB on: + workflow_call: push: branches: - '**' - '!dependabot/**' + - '!gh-readonly-queue/**' tags: - '**' paths: @@ -37,7 +39,7 @@ on: - 'cpp/src/arrow/**' concurrency: - group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} + group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }}-matlab cancel-in-progress: true permissions: diff --git a/.github/workflows/merge_queue.yml b/.github/workflows/merge_queue.yml new file mode 100644 index 000000000000..6767dd694cf9 --- /dev/null +++ b/.github/workflows/merge_queue.yml @@ -0,0 +1,122 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: Merge Queue CI + +on: + pull_request: + branches: [main] + merge_group: + types: [checks_requested] + branches: [main] + workflow_dispatch: + +concurrency: + group: merge-queue-${{ github.ref }} + cancel-in-progress: true + +permissions: + actions: read + contents: read + +jobs: + + archery: + if: github.event_name != 'pull_request' + uses: ./.github/workflows/archery.yml + secrets: inherit + + cpp: + if: github.event_name != 'pull_request' + uses: ./.github/workflows/cpp.yml + secrets: inherit + + dev: + if: github.event_name != 'pull_request' + uses: ./.github/workflows/dev.yml + secrets: inherit + + docs: + if: github.event_name != 'pull_request' + uses: ./.github/workflows/docs.yml + secrets: inherit + + docs_light: + if: github.event_name != 'pull_request' + uses: ./.github/workflows/docs_light.yml + secrets: inherit + + integration: + if: github.event_name != 'pull_request' + uses: ./.github/workflows/integration.yml + secrets: inherit + + matlab: + if: github.event_name != 'pull_request' + uses: ./.github/workflows/matlab.yml + secrets: inherit + + python: + if: github.event_name != 'pull_request' + uses: ./.github/workflows/python.yml + secrets: inherit + + r: + if: github.event_name != 'pull_request' + uses: ./.github/workflows/r.yml + secrets: inherit + + ruby: + if: github.event_name != 'pull_request' + uses: ./.github/workflows/ruby.yml + secrets: inherit + permissions: + actions: read + contents: read + # The existing Windows job caches vcpkg packages in GitHub Packages. + packages: write + + required: + name: Merge Queue + if: always() + needs: [archery, cpp, dev, docs, docs_light, integration, matlab, python, r, ruby] + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: {} + steps: + - name: Check every suite result + env: + EVENT_NAME: ${{ github.event_name }} + NEEDS: ${{ toJSON(needs) }} + shell: python + run: | + import json + import os + import sys + + event = os.environ["EVENT_NAME"] + results = json.loads(os.environ["NEEDS"]) + # PRs can enter the queue; the combined commit must pass every suite. + if event == "pull_request": + expected = "skipped" + elif event in ("merge_group", "workflow_dispatch"): + expected = "success" + else: + sys.exit(f"Unexpected event: {event}") + print(json.dumps(results, indent=2)) + if not results or any(job["result"] != expected for job in results.values()): + sys.exit(f"Every suite must report {expected}") diff --git a/.github/workflows/package_linux.yml b/.github/workflows/package_linux.yml index b1c93fd16792..43f5d806b2b0 100644 --- a/.github/workflows/package_linux.yml +++ b/.github/workflows/package_linux.yml @@ -22,6 +22,7 @@ on: branches: - '**' - '!dependabot/**' + - '!gh-readonly-queue/**' - '!release-*' paths: - '.github/workflows/check_labels.yml' diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index bf7285182a3b..4db3814b1316 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -18,10 +18,12 @@ name: Python on: + workflow_call: push: branches: - '**' - '!dependabot/**' + - '!gh-readonly-queue/**' tags: - '**' paths: @@ -43,7 +45,7 @@ on: - 'python/**' concurrency: - group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} + group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }}-python cancel-in-progress: true permissions: diff --git a/.github/workflows/r.yml b/.github/workflows/r.yml index 98b301d095dc..a9986fdc255d 100644 --- a/.github/workflows/r.yml +++ b/.github/workflows/r.yml @@ -18,10 +18,12 @@ name: R on: + workflow_call: push: branches: - '**' - '!dependabot/**' + - '!gh-readonly-queue/**' tags: - '**' paths: @@ -51,7 +53,7 @@ on: - "r/**" concurrency: - group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} + group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }}-r cancel-in-progress: true permissions: diff --git a/.github/workflows/r_extra.yml b/.github/workflows/r_extra.yml index 4abcc5383698..4750778db052 100644 --- a/.github/workflows/r_extra.yml +++ b/.github/workflows/r_extra.yml @@ -22,6 +22,7 @@ on: branches: - '**' - '!dependabot/**' + - '!gh-readonly-queue/**' paths: - '.dockerignore' - '.github/workflows/check_labels.yml' diff --git a/.github/workflows/release_candidate.yml b/.github/workflows/release_candidate.yml index f34d7fadb029..1b60444bbd7b 100644 --- a/.github/workflows/release_candidate.yml +++ b/.github/workflows/release_candidate.yml @@ -22,6 +22,7 @@ on: branches: - '**' - '!dependabot/**' + - '!gh-readonly-queue/**' tags: - "apache-arrow-*-rc*" paths: diff --git a/.github/workflows/ruby.yml b/.github/workflows/ruby.yml index 0d4655a24115..5991f760c3be 100644 --- a/.github/workflows/ruby.yml +++ b/.github/workflows/ruby.yml @@ -18,10 +18,12 @@ name: C GLib & Ruby on: + workflow_call: push: branches: - '**' - '!dependabot/**' + - '!gh-readonly-queue/**' tags: - '**' paths: @@ -55,7 +57,7 @@ on: - 'ruby/**' concurrency: - group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} + group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }}-ruby cancel-in-progress: true permissions: diff --git a/dev/test_merge_queue.py b/dev/test_merge_queue.py new file mode 100644 index 000000000000..99d208a96802 --- /dev/null +++ b/dev/test_merge_queue.py @@ -0,0 +1,84 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import json +import os +from pathlib import Path +import subprocess +import sys + +import pytest +from ruamel.yaml import YAML + + +ROOT = Path(__file__).resolve().parents[1] + + +def read_yaml(path): + return YAML(typ="safe").load((ROOT / path).read_text()) + + +@pytest.mark.parametrize("event,result,passes", [ + ("pull_request", "skipped", True), + ("merge_group", "success", True), + ("merge_group", "failure", False), + ("merge_group", "cancelled", False), + ("merge_group", "skipped", False), + ("workflow_dispatch", "success", True), + ("workflow_dispatch", "failure", False), + ("push", "success", False), + ("merge_group", None, False), +]) +def test_required_result(event, result, passes): + gate = read_yaml(".github/workflows/merge_queue.yml")["jobs"]["required"] + expected = "skipped" if event == "pull_request" else "success" + needs = {name: {"result": expected} for name in gate["needs"]} + if result is None: + needs = {} + else: + needs[gate["needs"][0]]["result"] = result + process = subprocess.run( + [sys.executable, "-c", gate["steps"][0]["run"]], + env={**os.environ, "EVENT_NAME": event, "NEEDS": json.dumps(needs)}, + capture_output=True, text=True, + ) + assert (process.returncode == 0) == passes, process.stdout + process.stderr + + +def test_required_workflow_wiring(): + workflow = read_yaml(".github/workflows/merge_queue.yml") + jobs = workflow["jobs"] + gate = jobs["required"] + assert set(gate["needs"]) == set(jobs) - {"required"} + assert gate["if"] == "always()" + assert "push" not in workflow["on"] + protection = read_yaml(".asf.yaml")["github"]["protected_branches"]["main"] + assert protection["required_status_checks"]["contexts"] == [gate["name"]] + groups = [workflow["concurrency"]["group"]] + for name in gate["needs"]: + child = read_yaml(jobs[name]["uses"]) + assert "workflow_call" in child["on"] + if "concurrency" in child: + groups.append(child["concurrency"]["group"]) + # Called workflows share github.workflow with the caller. + assert len(groups) == len(set(groups)) + for path in (ROOT / ".github/workflows").glob("*.yml"): + triggers = read_yaml(path)["on"] + if isinstance(triggers, dict): + push = triggers.get("push") or {} + if "**" in push.get("branches", []): + assert "!gh-readonly-queue/**" in push["branches"], path