From 96c0fd8b52a01716825d1a444e0f581e8946387c Mon Sep 17 00:00:00 2001 From: imbajin Date: Mon, 5 Oct 2026 02:39:50 +0800 Subject: [PATCH 01/10] chore(ci): cancel superseded PR runs - cancel older runs of each PR workflow - verify retry source before and after the delay - test retry freshness and document CI behavior --- .github/scripts/check-rerun.py | 92 ++++++++++++++++ .github/scripts/test_check_rerun.py | 132 +++++++++++++++++++++++ .github/workflows/client-ci.yml | 4 + .github/workflows/client-go-ci.yml | 4 + .github/workflows/codeql-analysis.yml | 4 + .github/workflows/hubble-ci.yml | 4 + .github/workflows/labeler.yml | 4 + .github/workflows/license-checker.yml | 6 ++ .github/workflows/loader-ci.yml | 4 + .github/workflows/rerun-ci.yml | 76 ++++++------- .github/workflows/spark-connector-ci.yml | 4 + .github/workflows/tools-ci.yml | 4 + README.md | 2 + docs/ci.md | 18 ++++ 14 files changed, 312 insertions(+), 46 deletions(-) create mode 100644 .github/scripts/check-rerun.py create mode 100644 .github/scripts/test_check_rerun.py create mode 100644 docs/ci.md diff --git a/.github/scripts/check-rerun.py b/.github/scripts/check-rerun.py new file mode 100644 index 000000000..c0b501474 --- /dev/null +++ b/.github/scripts/check-rerun.py @@ -0,0 +1,92 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Check the failed run again immediately before an automatic retry.""" + +import argparse +import json +import os +import subprocess +from urllib.parse import quote, urlencode + + +def api(path): + result = subprocess.run( + ["gh", "api", "--method", "GET", path], check=True, + capture_output=True, text=True, + ) + return json.loads(result.stdout) + + +def decide(repository, run_id, expected_attempt, max_reruns, fetch=api): + run = fetch(f"repos/{repository}/actions/runs/{run_id}") + if (run.get("status") != "completed" or run.get("conclusion") != "failure" + or run.get("run_attempt") != expected_attempt): + return "skip", "source run changed or is no longer a completed failure" + if expected_attempt > max_reruns: + return "skip", "retry limit reached" + sha = run.get("head_sha") + if not sha: + return "skip", "missing head SHA" + if run.get("event") == "pull_request": + def local(candidate): + base_repository = candidate.get("base", {}).get("repo", {}) + return (base_repository.get("full_name") == repository + or base_repository.get("url") == f"https://api.github.com/repos/{repository}") + + candidates = [pr for pr in run.get("pull_requests", []) if local(pr)] + if not candidates: + candidates = [pr for pr in fetch( + f"repos/{repository}/commits/{quote(sha, safe='')}/pulls") if local(pr)] + for candidate in candidates: + number = candidate.get("number") + if not isinstance(number, int): + continue + pr = fetch(f"repos/{repository}/pulls/{number}") + if (pr.get("base", {}).get("repo", {}).get("full_name") == repository + and pr.get("state") == "open" and pr.get("head", {}).get("sha") == sha): + return "rerun", "open PR head unchanged" + return "skip", "no open PR in this repository with the failed head" + if run.get("event") == "push": + branch = run.get("head_branch") + if not branch: + return "skip", "missing push branch" + # Read from the workflow repository, never a similarly named fork branch. + commits = fetch(f"repos/{repository}/commits?" + urlencode({"sha": branch, "per_page": 1})) + if commits and commits[0].get("sha") == sha: + return "rerun", "push branch head unchanged" + return "skip", "push branch head moved" + return "skip", "unsupported source event" + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--repository", required=True) + parser.add_argument("--run-id", required=True, type=int) + parser.add_argument("--run-attempt", required=True, type=int) + parser.add_argument("--max-reruns", required=True, type=int) + args = parser.parse_args() + try: + action, reason = decide(args.repository, args.run_id, args.run_attempt, args.max_reruns) + except (subprocess.SubprocessError, OSError, ValueError, KeyError, TypeError, AttributeError): + action, reason = "skip", "unable to verify current run and source metadata" + print(f"{action}: {reason}") + with open(os.environ["GITHUB_OUTPUT"], "a", encoding="utf-8") as output: + output.write(f"action={action}\nreason={reason}\n") + + +if __name__ == "__main__": + main() diff --git a/.github/scripts/test_check_rerun.py b/.github/scripts/test_check_rerun.py new file mode 100644 index 000000000..06676ef73 --- /dev/null +++ b/.github/scripts/test_check_rerun.py @@ -0,0 +1,132 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Behavioral tests for retry freshness, without credentials or network.""" +import copy +import importlib.util +from pathlib import Path +import unittest +import os +import subprocess +import tempfile +from unittest.mock import patch + +spec = importlib.util.spec_from_file_location("check_rerun", Path(__file__).with_name("check-rerun.py")) +checker = importlib.util.module_from_spec(spec) +spec.loader.exec_module(checker) + + +class FreshnessTest(unittest.TestCase): + def setUp(self): + self.repository = "hugegraph/hugegraph-toolchain" + self.run = {"status": "completed", "conclusion": "failure", "run_attempt": 1, + "event": "pull_request", "head_sha": "old", "head_branch": "feature"} + self.pr = {"number": 7, "state": "open", "head": {"sha": "old"}, + "base": {"repo": {"full_name": self.repository}}} + self.run["pull_requests"] = [copy.deepcopy(self.pr)] + self.calls = [] + + def fetch(self, path): + self.calls.append(path) + if "/actions/runs/" in path: + return self.run + if "/commits/" in path: + return [self.pr] + if "/commits?" in path: + return [{"sha": self.pr["head"]["sha"]}] + if "/pulls/" in path: + return self.pr + raise AssertionError(path) + + def decide(self): + return checker.decide(self.repository, 42, 1, 2, self.fetch)[0] + + def test_current_open_pr(self): + self.assertEqual("rerun", self.decide()) + + def test_slim_workflow_run_pr_association(self): + self.run["pull_requests"] = [{"number": 7, "base": {"repo": {"id": 1, "name": "hugegraph-toolchain", + "url": f"https://api.github.com/repos/{self.repository}"}}}] + self.assertEqual("rerun", self.decide()) + + def test_api_failure_fails_closed(self): + with tempfile.TemporaryDirectory() as directory: + output = Path(directory) / "output" + with patch.dict(os.environ, {"GITHUB_OUTPUT": str(output)}), patch( + "sys.argv", ["check-rerun.py", "--repository", self.repository, + "--run-id", "42", "--run-attempt", "1", "--max-reruns", "2"]), patch.object( + checker, "decide", side_effect=subprocess.CalledProcessError(1, "gh")): + checker.main() + self.assertIn("action=skip\n", output.read_text()) + + def test_closed_pr(self): + self.pr["state"] = "closed" + self.assertEqual("skip", self.decide()) + + def test_updated_pr(self): + self.pr["head"]["sha"] = "new" + self.assertEqual("skip", self.decide()) + + def test_empty_association_falls_back_to_commit(self): + self.run["pull_requests"] = [] + self.assertEqual("rerun", self.decide()) + self.assertTrue(any("/commits/old/pulls" in p for p in self.calls)) + + def test_other_repository_pr_is_ignored(self): + self.run["pull_requests"][0]["base"]["repo"]["full_name"] = "apache/hugegraph-toolchain" + self.assertEqual("rerun", self.decide()) + self.assertTrue(any("/commits/old/pulls" in p for p in self.calls)) + + def test_foreign_pr_fallback_still_rejects_foreign_pr(self): + self.run["pull_requests"][0]["base"]["repo"]["full_name"] = "apache/hugegraph-toolchain" + self.pr["base"]["repo"]["full_name"] = "apache/hugegraph-toolchain" + self.assertEqual("skip", self.decide()) + + def test_changed_attempt(self): + self.run["run_attempt"] = 2 + self.assertEqual("skip", self.decide()) + + def test_current_run_must_remain_failed_and_completed(self): + for key, value in (("status", "in_progress"), ("conclusion", "success"), + ("conclusion", "cancelled")): + with self.subTest(key=key, value=value): + old = self.run[key] + self.run[key] = value + self.assertEqual("skip", self.decide()) + self.run[key] = old + + def test_retry_limit(self): + self.run["run_attempt"] = 3 + self.assertEqual("skip", checker.decide(self.repository, 42, 3, 2, self.fetch)[0]) + + def test_push_checks_repository_and_encodes_branch(self): + self.run.update(event="push", head_branch="release/test") + self.assertEqual("rerun", self.decide()) + self.assertIn(f"repos/{self.repository}/commits?sha=release%2Ftest&per_page=1", self.calls) + self.pr["head"]["sha"] = "new" + self.assertEqual("skip", self.decide()) + + def test_unsupported_event(self): + self.run["event"] = "workflow_dispatch" + self.assertEqual("skip", self.decide()) + + def test_missing_metadata(self): + self.run["head_sha"] = None + self.assertEqual("skip", self.decide()) + + +if __name__ == "__main__": + unittest.main() diff --git a/.github/workflows/client-ci.yml b/.github/workflows/client-ci.yml index 90d7e8110..2e8807fd1 100644 --- a/.github/workflows/client-ci.yml +++ b/.github/workflows/client-ci.yml @@ -18,6 +18,10 @@ on: - .github/workflows/** - pom.xml +concurrency: + group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + jobs: client-ci: runs-on: ubuntu-latest diff --git a/.github/workflows/client-go-ci.yml b/.github/workflows/client-go-ci.yml index 520d92d95..388f156f2 100644 --- a/.github/workflows/client-go-ci.yml +++ b/.github/workflows/client-go-ci.yml @@ -17,6 +17,10 @@ on: - .github/workflows/** - pom.xml +concurrency: + group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + jobs: client-go-ci: runs-on: ubuntu-latest diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 83c3e4d9d..8b77fd699 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -19,6 +19,10 @@ on: schedule: - cron: '42 20 * * 6' +concurrency: + group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + jobs: analyze: env: diff --git a/.github/workflows/hubble-ci.yml b/.github/workflows/hubble-ci.yml index 8a753b01e..72bd362ba 100644 --- a/.github/workflows/hubble-ci.yml +++ b/.github/workflows/hubble-ci.yml @@ -26,6 +26,10 @@ env: HUGEGRAPH_SERVER_GIT_URL: https://github.com/apache/hugegraph.git HUGEGRAPH_SERVER_BRANCH: master +concurrency: + group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + jobs: hubble-ci: runs-on: ubuntu-latest diff --git a/.github/workflows/labeler.yml b/.github/workflows/labeler.yml index 26285abb5..0222c1ae6 100644 --- a/.github/workflows/labeler.yml +++ b/.github/workflows/labeler.yml @@ -21,6 +21,10 @@ name: "Pull Request Labeler" on: - pull_request_target +concurrency: + group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request_target' }} + jobs: triage: permissions: diff --git a/.github/workflows/license-checker.yml b/.github/workflows/license-checker.yml index 3b2d3d701..3fbe54acb 100644 --- a/.github/workflows/license-checker.yml +++ b/.github/workflows/license-checker.yml @@ -26,12 +26,18 @@ on: - 'release-*' pull_request: +concurrency: + group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + jobs: check-license-header: runs-on: ubuntu-latest steps: - name: Fetch Code uses: actions/checkout@v7 + - name: Test automatic retry freshness rules + run: python3 -m unittest discover -s .github/scripts -p 'test_*.py' # More info could refer to: https://github.com/apache/skywalking-eyes - name: Check License Header uses: apache/skywalking-eyes@29bd646002b63bb4c0ed1648d1654c35fb8ff027 # main 2026-07-12 diff --git a/.github/workflows/loader-ci.yml b/.github/workflows/loader-ci.yml index 8389ecab6..ef273d48b 100644 --- a/.github/workflows/loader-ci.yml +++ b/.github/workflows/loader-ci.yml @@ -19,6 +19,10 @@ on: - .github/workflows/** - pom.xml +concurrency: + group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + jobs: loader-ci: runs-on: ubuntu-latest diff --git a/.github/workflows/rerun-ci.yml b/.github/workflows/rerun-ci.yml index 5b3202226..831a67860 100644 --- a/.github/workflows/rerun-ci.yml +++ b/.github/workflows/rerun-ci.yml @@ -73,61 +73,45 @@ jobs: permissions: actions: write contents: read + pull-requests: read runs-on: ubuntu-latest steps: - - name: Wait before rerun + # workflow_run is privileged: only load code from the default-branch commit. + - name: Checkout trusted retry checker + uses: actions/checkout@v7 + with: + ref: ${{ github.sha }} + persist-credentials: false + sparse-checkout: .github/scripts + + - name: Check source freshness before waiting + id: initial-freshness + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GH_REPO: ${{ github.repository }} + RUN_ID: ${{ github.event.workflow_run.id }} + RUN_ATTEMPT: ${{ github.event.workflow_run.run_attempt }} run: | - sleep "$RETRY_DELAY_SECONDS" + python3 .github/scripts/check-rerun.py \ + --repository "$GH_REPO" --run-id "$RUN_ID" \ + --run-attempt "$RUN_ATTEMPT" --max-reruns "$MAX_RERUNS" + + - name: Wait before rerun + if: steps.initial-freshness.outputs.action == 'rerun' + run: sleep "$RETRY_DELAY_SECONDS" - - name: Check source branch freshness + - name: Check source freshness after waiting id: freshness + if: steps.initial-freshness.outputs.action == 'rerun' env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - HEAD_REPOSITORY: ${{ github.event.workflow_run.head_repository.full_name }} - HEAD_BRANCH: ${{ github.event.workflow_run.head_branch }} - HEAD_SHA: ${{ github.event.workflow_run.head_sha }} - WORKFLOW_NAME: ${{ github.event.workflow_run.name }} + GH_REPO: ${{ github.repository }} RUN_ID: ${{ github.event.workflow_run.id }} + RUN_ATTEMPT: ${{ github.event.workflow_run.run_attempt }} run: | - set -euo pipefail - - action="rerun" - latest_head_sha="$HEAD_SHA" - reason="head unchanged" - - if [[ -z "$HEAD_REPOSITORY" || -z "$HEAD_BRANCH" || -z "$HEAD_SHA" ]]; then - action="skip" - reason="missing head repository metadata" - else - latest_head_sha="$(gh api "repos/$HEAD_REPOSITORY/commits" -f sha="$HEAD_BRANCH" -F per_page=1 --jq '.[0].sha' 2>/dev/null || true)" - - if [[ -z "$latest_head_sha" || "$latest_head_sha" == "null" ]]; then - action="skip" - reason="head branch no longer available" - elif [[ "$latest_head_sha" != "$HEAD_SHA" ]]; then - action="skip" - reason="head moved to $latest_head_sha" - fi - fi - - { - echo "action=$action" - echo "latest_head_sha=$latest_head_sha" - echo "reason=$reason" - } >> "$GITHUB_OUTPUT" - - { - echo "### Rerun CI freshness check" - echo "" - echo "- Workflow: $WORKFLOW_NAME" - echo "- Run ID: $RUN_ID" - echo "- Head repository: $HEAD_REPOSITORY" - echo "- Head branch: $HEAD_BRANCH" - echo "- Failed run head SHA: $HEAD_SHA" - echo "- Current branch tip SHA: $latest_head_sha" - echo "- Action: $action" - echo "- Reason: $reason" - } >> "$GITHUB_STEP_SUMMARY" + python3 .github/scripts/check-rerun.py \ + --repository "$GH_REPO" --run-id "$RUN_ID" \ + --run-attempt "$RUN_ATTEMPT" --max-reruns "$MAX_RERUNS" - name: Rerun failed jobs if: steps.freshness.outputs.action == 'rerun' diff --git a/.github/workflows/spark-connector-ci.yml b/.github/workflows/spark-connector-ci.yml index 50693024a..8ed3c31a0 100644 --- a/.github/workflows/spark-connector-ci.yml +++ b/.github/workflows/spark-connector-ci.yml @@ -19,6 +19,10 @@ on: - .github/workflows/** - pom.xml +concurrency: + group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + jobs: spark-connector-ci: runs-on: ubuntu-latest diff --git a/.github/workflows/tools-ci.yml b/.github/workflows/tools-ci.yml index 4613bcd0f..756521820 100644 --- a/.github/workflows/tools-ci.yml +++ b/.github/workflows/tools-ci.yml @@ -19,6 +19,10 @@ on: - .github/workflows/** - pom.xml +concurrency: + group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + jobs: tools-ci: runs-on: ubuntu-latest diff --git a/README.md b/README.md index 1b5b181d9..7d13b508e 100644 --- a/README.md +++ b/README.md @@ -482,3 +482,5 @@ Thank you to all the people who already contributed to HugeGraph! ## License hugegraph-toolchain is licensed under [Apache 2.0 License](https://github.com/apache/hugegraph-toolchain/blob/master/LICENSE). + +See [CI behavior](docs/ci.md) for PR cancellation and automatic retry rules. diff --git a/docs/ci.md b/docs/ci.md new file mode 100644 index 000000000..c28bd6273 --- /dev/null +++ b/docs/ci.md @@ -0,0 +1,18 @@ +# Continuous integration + +PR workflows cancel older runs of the same workflow and PR when a new run arrives. +Pushes, scheduled jobs and manual runs use separate concurrency groups and are not +cancelled by PR updates. The labeler applies the same rule to `pull_request_target`. + +The automatic retry workflow retries failed jobs at most twice, retaining its +existing delay. Before and after the delay it checks the current run and source +through the GitHub API. Only the same completed failed attempt can be retried, with either an +open PR in this repository at the same head SHA or an unchanged push branch tip. +Closed PRs, superseded commits, changed attempts and unverifiable API responses +are skipped. The retry checker loads only trusted default-branch code. + +Run its behavioral tests locally with: + +```bash +python3 -m unittest discover -s .github/scripts -p 'test_*.py' +``` From 5b375de8810aa7c836d0101af121250578f5e3b5 Mon Sep 17 00:00:00 2001 From: imbajin Date: Mon, 5 Oct 2026 02:46:14 +0800 Subject: [PATCH 02/10] chore(ci): align cancellation comments - explain PR concurrency alongside workflow policy - match comments on active target branches - avoid conflicts when syncing master into PRs --- .github/workflows/client-ci.yml | 1 + .github/workflows/client-go-ci.yml | 1 + .github/workflows/codeql-analysis.yml | 1 + .github/workflows/hubble-ci.yml | 1 + .github/workflows/license-checker.yml | 1 + .github/workflows/loader-ci.yml | 1 + .github/workflows/spark-connector-ci.yml | 1 + .github/workflows/tools-ci.yml | 1 + 8 files changed, 8 insertions(+) diff --git a/.github/workflows/client-ci.yml b/.github/workflows/client-ci.yml index 2e8807fd1..d96f24bf0 100644 --- a/.github/workflows/client-ci.yml +++ b/.github/workflows/client-ci.yml @@ -18,6 +18,7 @@ on: - .github/workflows/** - pom.xml +# Keep the newest PR run; preserve independent push, release and manual runs. concurrency: group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.run_id }} cancel-in-progress: ${{ github.event_name == 'pull_request' }} diff --git a/.github/workflows/client-go-ci.yml b/.github/workflows/client-go-ci.yml index 388f156f2..6c2fa3c04 100644 --- a/.github/workflows/client-go-ci.yml +++ b/.github/workflows/client-go-ci.yml @@ -17,6 +17,7 @@ on: - .github/workflows/** - pom.xml +# Keep the newest PR run; preserve independent push, release and manual runs. concurrency: group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.run_id }} cancel-in-progress: ${{ github.event_name == 'pull_request' }} diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 8b77fd699..2ee2cf91c 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -19,6 +19,7 @@ on: schedule: - cron: '42 20 * * 6' +# Keep the newest PR run; preserve independent push, release and manual runs. concurrency: group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.run_id }} cancel-in-progress: ${{ github.event_name == 'pull_request' }} diff --git a/.github/workflows/hubble-ci.yml b/.github/workflows/hubble-ci.yml index 72bd362ba..b802ecebb 100644 --- a/.github/workflows/hubble-ci.yml +++ b/.github/workflows/hubble-ci.yml @@ -26,6 +26,7 @@ env: HUGEGRAPH_SERVER_GIT_URL: https://github.com/apache/hugegraph.git HUGEGRAPH_SERVER_BRANCH: master +# Keep the newest PR run; preserve independent push, release and manual runs. concurrency: group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.run_id }} cancel-in-progress: ${{ github.event_name == 'pull_request' }} diff --git a/.github/workflows/license-checker.yml b/.github/workflows/license-checker.yml index 3fbe54acb..e4fe77e67 100644 --- a/.github/workflows/license-checker.yml +++ b/.github/workflows/license-checker.yml @@ -26,6 +26,7 @@ on: - 'release-*' pull_request: +# Keep the newest PR run; preserve independent push, release and manual runs. concurrency: group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.run_id }} cancel-in-progress: ${{ github.event_name == 'pull_request' }} diff --git a/.github/workflows/loader-ci.yml b/.github/workflows/loader-ci.yml index ef273d48b..1a37f46e0 100644 --- a/.github/workflows/loader-ci.yml +++ b/.github/workflows/loader-ci.yml @@ -19,6 +19,7 @@ on: - .github/workflows/** - pom.xml +# Keep the newest PR run; preserve independent push, release and manual runs. concurrency: group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.run_id }} cancel-in-progress: ${{ github.event_name == 'pull_request' }} diff --git a/.github/workflows/spark-connector-ci.yml b/.github/workflows/spark-connector-ci.yml index 8ed3c31a0..996fefea6 100644 --- a/.github/workflows/spark-connector-ci.yml +++ b/.github/workflows/spark-connector-ci.yml @@ -19,6 +19,7 @@ on: - .github/workflows/** - pom.xml +# Keep the newest PR run; preserve independent push, release and manual runs. concurrency: group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.run_id }} cancel-in-progress: ${{ github.event_name == 'pull_request' }} diff --git a/.github/workflows/tools-ci.yml b/.github/workflows/tools-ci.yml index 756521820..347820de3 100644 --- a/.github/workflows/tools-ci.yml +++ b/.github/workflows/tools-ci.yml @@ -19,6 +19,7 @@ on: - .github/workflows/** - pom.xml +# Keep the newest PR run; preserve independent push, release and manual runs. concurrency: group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.run_id }} cancel-in-progress: ${{ github.event_name == 'pull_request' }} From a5780f38631bc3658f9cc43e3035a0d8b3b64243 Mon Sep 17 00:00:00 2001 From: imbajin Date: Tue, 6 Oct 2026 00:36:05 +0800 Subject: [PATCH 03/10] chore(ci): gate affected module tests - select related modules and verify reusable test receipts - share immutable server fixtures with separate runtime services - isolate HDFS setup and bound downloads and readiness - defer security checks and preserve protection migration --- .asf.yaml | 2 +- .../actions/setup-hugegraph-server/action.yml | 80 +++- .../actions/setup-hugegraph-server/fixture.sh | 81 ++++ .../setup-hugegraph-server/manifest.py | 71 +++ .../setup-hugegraph-server/runtime_test.py | 148 ++++++ .../setup-hugegraph-server/service-wait.sh | 28 ++ .github/actions/setup-java-env/action.yml | 4 +- .github/scripts/check-rerun.py | 65 ++- .github/scripts/ci-policy.py | 443 ++++++++++++++++++ .github/scripts/test_check_rerun.py | 56 ++- .github/scripts/test_ci_policy.py | 327 +++++++++++++ .github/workflows/ci.yml | 273 +++++++++++ .github/workflows/client-ci.yml | 41 +- .github/workflows/client-go-ci.yml | 40 +- .github/workflows/codeql-analysis.yml | 21 +- .github/workflows/hubble-ci.yml | 68 +-- .github/workflows/labeler.yml | 2 +- .github/workflows/license-checker.yml | 25 +- .github/workflows/loader-ci.yml | 128 +++-- .github/workflows/rerun-ci.yml | 7 +- .github/workflows/spark-connector-ci.yml | 42 +- .github/workflows/tools-ci.yml | 42 +- README.md | 2 +- docs/ci.md | 46 +- .../assembly/travis/checkout-server.sh | 47 ++ .../assembly/travis/checkout-server.test.cjs | 58 +++ .../travis/install-hugegraph-from-source.sh | 69 +-- .../assembly/travis/install-server.test.cjs | 356 ++++++++++++++ .../travis/start-hugegraph-servers.sh | 72 +++ .../assembly/travis/download-hugegraph.sh | 39 +- .../hugegraph-server1/install-hugegraph.sh | 3 + .../hugegraph-server2/install-hugegraph.sh | 3 + .../assembly/travis/install-hadoop.sh | 60 ++- .../travis/install-hugegraph-from-source.sh | 115 +++-- .../travis/install-hugegraph-from-source.sh | 47 +- .../travis/install-hugegraph-from-source.sh | 47 +- 36 files changed, 2472 insertions(+), 486 deletions(-) create mode 100644 .github/actions/setup-hugegraph-server/fixture.sh create mode 100644 .github/actions/setup-hugegraph-server/manifest.py create mode 100644 .github/actions/setup-hugegraph-server/runtime_test.py create mode 100644 .github/actions/setup-hugegraph-server/service-wait.sh create mode 100644 .github/scripts/ci-policy.py create mode 100644 .github/scripts/test_ci_policy.py create mode 100644 .github/workflows/ci.yml create mode 100644 hugegraph-client/assembly/travis/checkout-server.sh create mode 100644 hugegraph-client/assembly/travis/checkout-server.test.cjs create mode 100644 hugegraph-client/assembly/travis/install-server.test.cjs create mode 100644 hugegraph-client/assembly/travis/start-hugegraph-servers.sh diff --git a/.asf.yaml b/.asf.yaml index 1d9d56e20..ee867caa4 100644 --- a/.asf.yaml +++ b/.asf.yaml @@ -45,7 +45,7 @@ github: # contexts are the names of checks that must pass contexts: - check-license-header - - Analyze (java) + - affected-module-tests required_pull_request_reviews: dismiss_stale_reviews: true require_code_owner_reviews: false diff --git a/.github/actions/setup-hugegraph-server/action.yml b/.github/actions/setup-hugegraph-server/action.yml index ff142ed1a..c6786682e 100644 --- a/.github/actions/setup-hugegraph-server/action.yml +++ b/.github/actions/setup-hugegraph-server/action.yml @@ -1,18 +1,78 @@ -name: 'Setup HugeGraph Server' -description: 'Prepare environment and install HugeGraph server from source' - +name: Setup HugeGraph Server +description: Build an immutable fixture once, or start an independent test service inputs: - travis-dir: - description: 'Path to the Travis assembly directory containing install scripts' - required: true + mode: + description: 'standalone (legacy), build (upload), or start (download)' + default: standalone + artifact-name: + description: Workflow-local immutable fixture artifact name + default: '' + repository: + description: GitHub owner/repository containing the Server source + default: apache/hugegraph + fetch-ref: + description: Fetchable source ref (must resolve to commit-id) + default: '' commit-id: - description: 'HugeGraph commit SHA to install' + description: Exact Server commit SHA required: true - + java-version: + description: JDK major used for the build and runtime + default: '11' + travis-dir: + description: Module service setup directory + default: hugegraph-loader/assembly/travis runs: using: composite steps: - - name: Prepare env and service + - name: Download shared Server fixture + if: inputs.mode == 'start' + uses: actions/download-artifact@v4 + with: + name: ${{ inputs.artifact-name }} + path: ${{ runner.temp }}/hugegraph-fixture + - name: Restore exact Server build cache + if: inputs.mode == 'build' + uses: actions/cache@v4 + with: + path: ~/hugegraph-fixture-cache + key: server-fixture-v1-${{ runner.os }}-${{ runner.arch }}-${{ inputs.repository }}-${{ inputs.commit-id }}-jdk${{ inputs.java-version }}-${{ hashFiles('.github/actions/setup-hugegraph-server/action.yml', '.github/actions/setup-hugegraph-server/fixture.sh', '.github/actions/setup-hugegraph-server/manifest.py', '.github/actions/setup-java-env/action.yml', '.github/configs/settings.xml', 'hugegraph-client/assembly/travis/checkout-server.sh', 'hugegraph-client/assembly/travis/start-hugegraph-servers.sh', 'hugegraph-*/assembly/travis/install-hugegraph-from-source.sh', 'hugegraph-hubble/hubble-dist/assembly/travis/download-hugegraph.sh') }} + - name: Prepare fixture or start service shell: bash + env: + FIXTURE_MODE: ${{ inputs.mode }} + FIXTURE_REPOSITORY: ${{ inputs.repository }} + FIXTURE_COMMIT: ${{ inputs.commit-id }} + FIXTURE_REF: ${{ inputs.fetch-ref }} + FIXTURE_BUILD_INPUTS: ${{ hashFiles('.github/actions/setup-hugegraph-server/action.yml', '.github/actions/setup-hugegraph-server/fixture.sh', '.github/actions/setup-hugegraph-server/manifest.py', '.github/actions/setup-java-env/action.yml', '.github/configs/settings.xml', 'hugegraph-client/assembly/travis/checkout-server.sh', 'hugegraph-client/assembly/travis/start-hugegraph-servers.sh', 'hugegraph-*/assembly/travis/install-hugegraph-from-source.sh', 'hugegraph-hubble/hubble-dist/assembly/travis/download-hugegraph.sh') }} + FIXTURE_PLATFORM: ${{ runner.os }}-${{ runner.arch }} + FIXTURE_JAVA: ${{ inputs.java-version }} + FIXTURE_DIR: ${{ runner.temp }}/hugegraph-fixture + SERVER_REPOSITORY: ${{ inputs.repository }} + SERVER_FETCH_REF: ${{ inputs.fetch-ref }} + TRAVIS_DIR: ${{ inputs.travis-dir }} + FIXTURE_ACTION: ${{ github.action_path }} + FIXTURE_ARTIFACT: ${{ inputs.artifact-name }} run: | - ${{ inputs.travis-dir }}/install-hugegraph-from-source.sh ${{ inputs.commit-id }} + set -euo pipefail + if [[ "$FIXTURE_MODE" != standalone && -z "$FIXTURE_ARTIFACT" ]]; then + echo 'artifact-name is required for build/start mode' >&2 + exit 1 + fi + if [[ "$FIXTURE_MODE" == build ]]; then + bash "$FIXTURE_ACTION/fixture.sh" + elif [[ -f "$TRAVIS_DIR/install-hugegraph-from-source.sh" ]]; then + bash "$TRAVIS_DIR/install-hugegraph-from-source.sh" "$FIXTURE_COMMIT" + else + bash "$TRAVIS_DIR/install-hugegraph.sh" "$FIXTURE_COMMIT" "$FIXTURE_REF" + fi + - name: Upload shared Server fixture + if: inputs.mode == 'build' + uses: actions/upload-artifact@v4 + with: + name: ${{ inputs.artifact-name }} + path: | + ${{ runner.temp }}/hugegraph-fixture/server.tar.gz + ${{ runner.temp }}/hugegraph-fixture/manifest.json + if-no-files-found: error + retention-days: 1 diff --git a/.github/actions/setup-hugegraph-server/fixture.sh b/.github/actions/setup-hugegraph-server/fixture.sh new file mode 100644 index 000000000..9fc5f1850 --- /dev/null +++ b/.github/actions/setup-hugegraph-server/fixture.sh @@ -0,0 +1,81 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with this +# work for additional information regarding copyright ownership. The ASF +# licenses this file to You under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +set -euo pipefail + +# Identity includes every build input. Never restore an archive from another source. +FIXTURE_REPOSITORY=${FIXTURE_REPOSITORY:-${SERVER_REPOSITORY:-apache/hugegraph}} +FIXTURE_COMMIT=${FIXTURE_COMMIT:-${1:-}} +FIXTURE_REF=${FIXTURE_REF:-$FIXTURE_COMMIT} +FIXTURE_JAVA=${FIXTURE_JAVA:-11} +FIXTURE_DIR=${FIXTURE_DIR:-${RUNNER_TEMP:-${TMPDIR:-/tmp}}/hugegraph-fixture} +FIXTURE_HELPER=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/manifest.py +FIXTURE_BUILD_INPUTS=${FIXTURE_BUILD_INPUTS:-$(python3 "$FIXTURE_HELPER" inputs)} +FIXTURE_PLATFORM=${FIXTURE_PLATFORM:-$(uname -sm)} +FIXTURE_CONFIG="package -DskipTests -Dmaven.javadoc.skip=true -ntp;schema=1;inputs=$FIXTURE_BUILD_INPUTS;platform=$FIXTURE_PLATFORM" +export FIXTURE_REPOSITORY FIXTURE_COMMIT FIXTURE_JAVA FIXTURE_CONFIG FIXTURE_DIR +[[ "$FIXTURE_REPOSITORY" =~ ^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$ ]] +[[ "$FIXTURE_COMMIT" =~ ^[0-9a-f]{40}$ ]] +[[ "$FIXTURE_JAVA" =~ ^[0-9]+$ ]] + +fixture_validate_java() { + actual_java=$(java -version 2>&1 | sed -n 's/.*version "\([0-9][0-9]*\).*".*/\1/p' | head -1) + [[ "$actual_java" == "$FIXTURE_JAVA" ]] || { echo 'Fixture JDK mismatch' >&2; return 1; } +} + +fixture_verify() { python3 "$FIXTURE_HELPER" verify; } +fixture_build() { + local cache_key cache_dir source_dir archive + cache_key=$(printf '%s\n' "$FIXTURE_REPOSITORY" "$FIXTURE_COMMIT" "$FIXTURE_JAVA" "$FIXTURE_CONFIG" | shasum -a 256 | cut -d ' ' -f 1) + cache_dir="${HOME}/hugegraph-fixture-cache/$cache_key" + mkdir -p "$FIXTURE_DIR" + if [[ -f "$cache_dir/server.tar.gz" && ! -L "$cache_dir/server.tar.gz" && + -f "$cache_dir/manifest.json" && ! -L "$cache_dir/manifest.json" ]] && + cp "$cache_dir"/server.tar.gz "$cache_dir"/manifest.json "$FIXTURE_DIR/"; then + if fixture_verify; then return; fi + fi + rm -f "$FIXTURE_DIR/server.tar.gz" "$FIXTURE_DIR/manifest.json" + fixture_validate_java + source_dir="$(mktemp -d "${TMPDIR:-/tmp}/hg-fixture-source.XXXXXX")/source" + SERVER_REPOSITORY=$FIXTURE_REPOSITORY SERVER_FETCH_REF=$FIXTURE_REF \ + bash "${FIXTURE_HELPER%/*}/../../../hugegraph-client/assembly/travis/checkout-server.sh" "$FIXTURE_COMMIT" "$source_dir" + (cd "$source_dir" && mvn package -DskipTests -Dmaven.javadoc.skip=true -ntp) + archive=$(find "$source_dir/hugegraph-server" -maxdepth 1 -name 'apache-hugegraph-*.tar.gz' -print) + [[ -n "$archive" && -f "$archive" ]] + cp "$archive" "$FIXTURE_DIR/server.tar.gz" + export FIXTURE_ARCHIVE_NAME=${archive##*/} + python3 "$FIXTURE_HELPER" create + fixture_verify + mkdir -p "$cache_dir" + cp "$FIXTURE_DIR/server.tar.gz" "$FIXTURE_DIR/manifest.json" "$cache_dir/" + rm -rf "$source_dir" +} +fixture_copy_archive() { + fixture_validate_java + fixture_verify + local name + name=$(python3 "$FIXTURE_HELPER" name) + cp "$FIXTURE_DIR/server.tar.gz" "./$name" +} +source "${FIXTURE_HELPER%/*}/service-wait.sh" + +case "${FIXTURE_MODE:-standalone}" in + build) fixture_build ;; + start) fixture_copy_archive ;; + standalone) fixture_build; fixture_copy_archive ;; + *) echo 'Invalid fixture mode' >&2; exit 1 ;; +esac diff --git a/.github/actions/setup-hugegraph-server/manifest.py b/.github/actions/setup-hugegraph-server/manifest.py new file mode 100644 index 000000000..c7446cecf --- /dev/null +++ b/.github/actions/setup-hugegraph-server/manifest.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python3 +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with this +# work for additional information regarding copyright ownership. The ASF +# licenses this file to You under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +"""Verify immutable CI fixture identity and archive bytes with the standard library.""" +import hashlib +import json +import os +import pathlib +import re +import sys + +if sys.argv[1] == "inputs": + repo = pathlib.Path(__file__).resolve().parents[3] + patterns = ( + '.github/actions/setup-hugegraph-server/action.yml', + '.github/actions/setup-hugegraph-server/fixture.sh', + '.github/actions/setup-hugegraph-server/manifest.py', + '.github/actions/setup-java-env/action.yml', '.github/configs/settings.xml', + 'hugegraph-client/assembly/travis/checkout-server.sh', + 'hugegraph-client/assembly/travis/start-hugegraph-servers.sh', + 'hugegraph-*/assembly/travis/install-hugegraph-from-source.sh', + 'hugegraph-hubble/hubble-dist/assembly/travis/download-hugegraph.sh', + ) + inputs = hashlib.sha256() + files = sorted({p for pattern in patterns for p in repo.glob(pattern)}) + for path in files: + inputs.update(str(path.relative_to(repo)).encode() + b"\0" + path.read_bytes()) + print(inputs.hexdigest()) + sys.exit(0) + +root = pathlib.Path(os.environ["FIXTURE_DIR"]) +identity = {key.lower(): os.environ["FIXTURE_" + key] for key in + ("REPOSITORY", "COMMIT", "JAVA", "CONFIG")} +def digest(): + h = hashlib.sha256() + with (root / "server.tar.gz").open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + h.update(chunk) + return h.hexdigest() +try: + if sys.argv[1] == "create": + data = dict(identity, archive_name=os.environ["FIXTURE_ARCHIVE_NAME"], sha256=digest()) + (root / "manifest.json").write_text(json.dumps(data, sort_keys=True) + "\n") + else: + if (root / "manifest.json").is_symlink() or (root / "server.tar.gz").is_symlink(): + raise ValueError("fixture must contain regular files") + data = json.loads((root / "manifest.json").read_text()) + if any(data.get(key) != value for key, value in identity.items()): + raise ValueError("fixture origin, commit, JDK or build configuration mismatch") + if not re.fullmatch(r"apache-hugegraph-[A-Za-z0-9_.-]+\.tar\.gz", data.get("archive_name", "")): + raise ValueError("invalid archive name") + if data.get("sha256") != digest(): + raise ValueError("fixture checksum mismatch") + if sys.argv[1] == "name": + print(data["archive_name"]) +except (OSError, ValueError, KeyError) as error: + sys.exit(str(error)) diff --git a/.github/actions/setup-hugegraph-server/runtime_test.py b/.github/actions/setup-hugegraph-server/runtime_test.py new file mode 100644 index 000000000..d35580bfd --- /dev/null +++ b/.github/actions/setup-hugegraph-server/runtime_test.py @@ -0,0 +1,148 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with this +# work for additional information regarding copyright ownership. The ASF +# licenses this file to You under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +"""Exercise fixture identity, bounded downloads and service readiness offline.""" +import hashlib +import json +import os +from pathlib import Path +import subprocess +import tempfile +import unittest + +HERE = Path(__file__).resolve().parent +ROOT = HERE.parents[2] +SHA = 'a' * 40 + +class RuntimeTests(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.dir = Path(self.temp.name) + self.bin = self.dir / 'bin' + self.bin.mkdir() + (self.bin/'java').write_text("#!/bin/bash\necho 'openjdk version \"11.0.1\"' >&2\n") + (self.bin/'java').chmod(0o755) + self.env = dict(os.environ, HOME=str(self.dir), PATH=f'{self.bin}:{os.environ["PATH"]}', + FIXTURE_DIR=str(self.dir / 'fixture'), FIXTURE_REPOSITORY='apache/hugegraph', + FIXTURE_COMMIT=SHA, FIXTURE_JAVA='11', + FIXTURE_BUILD_INPUTS='test-build-contract', FIXTURE_PLATFORM='Linux-X64', + FIXTURE_CONFIG='package -DskipTests -Dmaven.javadoc.skip=true -ntp;schema=1;inputs=test-build-contract;platform=Linux-X64') + Path(self.env['FIXTURE_DIR']).mkdir() + def command(self, name, body): + script = self.bin / name + script.write_text('#!/bin/bash\n' + body) + script.chmod(0o755) + def run_script(self, script, *args, env=None): + return subprocess.run(['bash', str(script), *args], env=env or self.env, + cwd=self.dir, text=True, capture_output=True, timeout=20) + def manifest(self, **extra): + root = Path(self.env['FIXTURE_DIR']) + data = {k.lower(): self.env['FIXTURE_'+k] for k in ('REPOSITORY','COMMIT','JAVA','CONFIG')} + (root/'server.tar.gz').write_bytes(b'archive') + data.update(archive_name='apache-hugegraph-fixture.tar.gz', sha256=hashlib.sha256(b'archive').hexdigest()) + data.update(extra) + (root/'manifest.json').write_text(json.dumps(data)) + def test_identity_checksum_and_regular_files(self): + self.manifest() + env = dict(self.env, FIXTURE_MODE='start') + self.assertEqual(self.run_script(HERE/'fixture.sh', env=env).returncode, 0) + for field, wrong in [('repository','fork/hugegraph'), ('commit','b'*40), ('java','17'), ('config','different'), ('sha256','bad')]: + self.manifest(**{field:wrong}) + self.assertNotEqual(self.run_script(HERE/'fixture.sh', env=env).returncode, 0, field) + self.manifest() + self.command('java', "echo 'openjdk version \"17.0.1\"' >&2\n") + self.assertNotEqual(self.run_script(HERE/'fixture.sh', env=env).returncode, 0) + self.command('java', "echo 'openjdk version \"11.0.1\"' >&2\n") + self.manifest(archive_name='../escape.tar.gz') + self.assertNotEqual(self.run_script(HERE/'fixture.sh', env=env).returncode, 0) + self.manifest() + archive = Path(self.env['FIXTURE_DIR'])/'server.tar.gz' + archive.unlink() + archive.symlink_to(self.dir/'missing') + self.assertNotEqual(self.run_script(HERE/'fixture.sh', env=env).returncode, 0) + def test_legacy_bash_service_deadline_and_success(self): + helper = str(HERE/'service-wait.sh') + self.command('curl', 'exit 1\n') + result = subprocess.run( + ['bash','-c',f'source "{helper}"; fixture_wait http://localhost:1'], + env=dict(self.env, SERVICE_READY_TIMEOUT='0'), text=True, capture_output=True, timeout=2) + self.assertNotEqual(result.returncode, 0) + self.assertIn('timed out', result.stderr) + self.command('curl', 'printf "%s\\n" "$@" > "$HOME/curl-args"; exit 0\n') + result = subprocess.run(['bash','-c',f'source "{helper}"; fixture_wait http://localhost:1'], + env=self.env, capture_output=True, timeout=2) + self.assertEqual(result.returncode, 0) + args = (self.dir/'curl-args').read_text() + self.assertIn('--max-time\n5', args) + self.assertIn('admin:pa', args) + def test_hadoop_invalid_cache_retries_at_most_twice_and_stops(self): + (self.dir/'hadoop-3.3.6.tar.gz').write_text('tampered') + self.command('curl', 'printf "%s\\n" "$@" >> "$HOME/download-args"; exit 22\n') + self.command('sudo', 'echo unexpected-install >> "$HOME/installed"; exit 99\n') + result = self.run_script(ROOT/'hugegraph-loader/assembly/travis/install-hadoop.sh') + self.assertNotEqual(result.returncode, 0) + args = (self.dir/'download-args').read_text() + self.assertEqual(args.count('--max-time\n900'), 2) + self.assertFalse((self.dir/'installed').exists()) + self.assertFalse((self.dir/'hadoop-3.3.6.tar.gz').exists()) + def test_real_http_readiness_requires_successful_auth_response(self): + from http.server import BaseHTTPRequestHandler, HTTPServer + from threading import Thread + class Handler(BaseHTTPRequestHandler): + def do_GET(self): + if self.path == '/graphs' and self.headers.get('Authorization') == 'Basic YWRtaW46cGE=': + self.send_response(200) + self.end_headers() + self.wfile.write(b'{"graphs":[]}') + else: + self.send_response(401) + self.end_headers() + def log_message(self, *args): + pass + server = HTTPServer(('127.0.0.1', 0), Handler) + self.addCleanup(server.server_close) + self.addCleanup(server.shutdown) + Thread(target=server.serve_forever, daemon=True).start() + helper = str(HERE/'service-wait.sh') + result = subprocess.run(['bash','-c',f'source "{helper}"; fixture_wait http://127.0.0.1:{server.server_port}'], + env=self.env, capture_output=True, timeout=5) + self.assertEqual(result.returncode, 0, result.stderr) + + def test_exact_cache_hit_never_builds_or_starts(self): + self.manifest() + identity = '\n'.join(self.env['FIXTURE_'+k] for k in ('REPOSITORY','COMMIT','JAVA','CONFIG'))+'\n' + cache = self.dir/'hugegraph-fixture-cache'/hashlib.sha256(identity.encode()).hexdigest() + cache.mkdir(parents=True) + for name in ('server.tar.gz','manifest.json'): + (cache/name).write_bytes((Path(self.env['FIXTURE_DIR'])/name).read_bytes()) + (Path(self.env['FIXTURE_DIR'])/name).unlink() + self.command('git', 'echo forbidden-git >> "$HOME/forbidden"; exit 99\n') + self.command('mvn', 'echo forbidden-build >> "$HOME/forbidden"; exit 99\n') + result = self.run_script(HERE/'fixture.sh', env=dict(self.env, FIXTURE_MODE='build')) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertFalse((self.dir/'forbidden').exists()) + self.assertFalse((self.dir/'apache-hugegraph-fixture.tar.gz').exists()) + # An incomplete exact cache must rebuild, rather than poison future runs. + (cache/'manifest.json').unlink() + result = self.run_script(HERE/'fixture.sh', env=dict(self.env, FIXTURE_MODE='build')) + self.assertNotEqual(result.returncode, 0) + self.assertTrue((self.dir/'forbidden').exists()) + + +if __name__ == '__main__': + unittest.main() diff --git a/.github/actions/setup-hugegraph-server/service-wait.sh b/.github/actions/setup-hugegraph-server/service-wait.sh new file mode 100644 index 000000000..7fb7feea2 --- /dev/null +++ b/.github/actions/setup-hugegraph-server/service-wait.sh @@ -0,0 +1,28 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with this +# work for additional information regarding copyright ownership. The ASF +# licenses this file to You under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +fixture_wait() { + local url=$1 deadline=$((SECONDS + ${SERVICE_READY_TIMEOUT:-300})) + while (( SECONDS < deadline )); do + # Use an authenticated API response, rather than a listening socket. + if curl --fail --silent --insecure --connect-timeout 2 --max-time 5 -u admin:pa "$url/graphs" >/dev/null; then return; fi + sleep 2 + done + echo "HugeGraph readiness timed out: $url" >&2 + return 1 +} + diff --git a/.github/actions/setup-java-env/action.yml b/.github/actions/setup-java-env/action.yml index 9d897bfd0..1d364b388 100644 --- a/.github/actions/setup-java-env/action.yml +++ b/.github/actions/setup-java-env/action.yml @@ -29,5 +29,5 @@ runs: if: ${{ inputs.use-stage == 'true' }} shell: bash run: | - cp $HOME/.m2/settings.xml /tmp/settings.xml - mv -vf .github/configs/settings.xml $HOME/.m2/settings.xml + cp "$HOME/.m2/settings.xml" /tmp/settings.xml + cp .github/configs/settings.xml "$HOME/.m2/settings.xml" diff --git a/.github/scripts/check-rerun.py b/.github/scripts/check-rerun.py index c0b501474..24fff7b72 100644 --- a/.github/scripts/check-rerun.py +++ b/.github/scripts/check-rerun.py @@ -17,18 +17,41 @@ """Check the failed run again immediately before an automatic retry.""" import argparse +import io import json +import zipfile import os import subprocess from urllib.parse import quote, urlencode -def api(path): +def api(path, binary=False): result = subprocess.run( ["gh", "api", "--method", "GET", path], check=True, - capture_output=True, text=True, + capture_output=True, timeout=30, ) - return json.loads(result.stdout) + return result.stdout if binary else json.loads(result.stdout) + + +def tested_base_matches(repository, run_id, pr, sha, fetch): + artifacts = fetch(f"repos/{repository}/actions/runs/{run_id}/artifacts?per_page=100")["artifacts"] + candidates = [a for a in artifacts if a.get("name") == "ci-plan" and not a.get("expired")] + for artifact in candidates: + payload = fetch(f"repos/{repository}/actions/artifacts/{artifact['id']}/zip", binary=True) + with zipfile.ZipFile(io.BytesIO(payload)) as archive: + info = archive.getinfo("plan.json") + if info.file_size > 1048576: + continue + plan = json.loads(archive.read(info)) + base = pr.get("base", {}).get("sha") + if (plan.get("repository") != repository or plan.get("pr") != pr.get("number") + or plan.get("head") != sha or not base or plan.get("base") != base + or not plan.get("testedMergeSHA")): + continue + merge = fetch(f"repos/{repository}/git/commits/{plan['testedMergeSHA']}") + if [parent["sha"] for parent in merge.get("parents", [])] == [base, sha]: + return True + return False def decide(repository, run_id, expected_attempt, max_reruns, fetch=api): @@ -47,18 +70,50 @@ def local(candidate): return (base_repository.get("full_name") == repository or base_repository.get("url") == f"https://api.github.com/repos/{repository}") + source = run.get("head_repository") or {} + source_owner = (source.get("owner") or {}).get("login") + source_name = source.get("full_name") + source_branch = run.get("head_branch") + if not source_owner or not source_name or not source_branch: + return "skip", "missing source repository or branch metadata" + def source_matches(pr): + head = pr.get("head") or {} + repo = head.get("repo") or {} + return (head.get("ref") == source_branch and repo.get("full_name") == source_name + and (repo.get("owner") or {}).get("login") == source_owner) + fallback_owner = None candidates = [pr for pr in run.get("pull_requests", []) if local(pr)] if not candidates: candidates = [pr for pr in fetch( f"repos/{repository}/commits/{quote(sha, safe='')}/pulls") if local(pr)] + if not candidates: + owner = run.get("head_repository", {}).get("owner", {}).get("login") + branch = run.get("head_branch") + if owner and branch: + fallback_owner = owner + # Fork runs can have no commit/PR association in either endpoint. + # Use the qualified fork owner, never the branch name alone. + for page in range(1, 101): + batch = fetch(f"repos/{repository}/pulls?" + urlencode({ + "state": "open", "head": f"{owner}:{branch}", + "per_page": 100, "page": page})) + candidates.extend(pr for pr in batch if local(pr) + and pr.get("head", {}).get("ref") == branch + and pr.get("head", {}).get("repo", {}).get("owner", {}).get("login") == owner) + if len(batch) < 100: + break for candidate in candidates: number = candidate.get("number") if not isinstance(number, int): continue pr = fetch(f"repos/{repository}/pulls/{number}") + if not source_matches(pr): + continue if (pr.get("base", {}).get("repo", {}).get("full_name") == repository and pr.get("state") == "open" and pr.get("head", {}).get("sha") == sha): - return "rerun", "open PR head unchanged" + if tested_base_matches(repository, run_id, pr, sha, fetch): + return "rerun", "open PR head and tested base unchanged" + return "skip", "tested base moved or immutable CI plan proof unavailable" return "skip", "no open PR in this repository with the failed head" if run.get("event") == "push": branch = run.get("head_branch") @@ -81,7 +136,7 @@ def main(): args = parser.parse_args() try: action, reason = decide(args.repository, args.run_id, args.run_attempt, args.max_reruns) - except (subprocess.SubprocessError, OSError, ValueError, KeyError, TypeError, AttributeError): + except (subprocess.SubprocessError, OSError, ValueError, KeyError, TypeError, AttributeError, zipfile.BadZipFile): action, reason = "skip", "unable to verify current run and source metadata" print(f"{action}: {reason}") with open(os.environ["GITHUB_OUTPUT"], "a", encoding="utf-8") as output: diff --git a/.github/scripts/ci-policy.py b/.github/scripts/ci-policy.py new file mode 100644 index 000000000..821b8f1ab --- /dev/null +++ b/.github/scripts/ci-policy.py @@ -0,0 +1,443 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Conservative CI selection and GitHub-verified test receipt reuse.""" + +import argparse +import hashlib +import io +import json +import os +from pathlib import Path +import subprocess +import zipfile +from urllib.parse import urlencode + +MODULES = { + "server": ["server", "commons", "struct", "pd", "store", "hstore", "cluster", "docker", "helm", "dependency_license"], + "toolchain": ["client", "loader", "tools", "spark", "hubble", "go"], +} +PREFIXES = { + "server": {"hugegraph-server/": "server", "hugegraph-commons/": "commons", + "hugegraph-struct/": "struct", "hugegraph-pd/": "pd", "hugegraph-store/": "store", + "hugegraph-cluster-test/": "cluster", "docker/": "docker", "helm/": "helm"}, + "toolchain": {"hugegraph-client/": "client", "hugegraph-client-go/": "go", + "hugegraph-loader/": "loader", "hugegraph-tools/": "tools", + "hugegraph-spark-connector/": "spark", "hugegraph-hubble/": "hubble"}, +} +WORKFLOWS = { + "server": {"server-ci.yml": ["server"], "commons-ci.yml": ["commons"], + "pd-store-ci.yml": ["pd", "store", "hstore"], "cluster-test-ci.yml": ["cluster"], + "docker-build-ci.yml": ["docker"], "helm-chart-ci.yml": ["helm"], + "codeql-analysis.yml": [], "riscv64-ci.yml": ["server"], "check-dependencies.yml": ["dependency_license"]}, + "toolchain": {"client-ci.yml": ["client"], "client-go-ci.yml": ["go"], + "loader-ci.yml": ["loader"], "tools-ci.yml": ["tools"], + "spark-connector-ci.yml": ["spark"], "hubble-ci.yml": ["hubble"], + "codeql-analysis.yml": []}, +} +DEPENDENTS = { + "server": {"commons": ["server", "pd", "store", "hstore", "cluster"], + "struct": ["server", "pd", "store", "hstore", "cluster"], + "server": ["cluster"], "pd": ["store", "hstore", "cluster"], + "store": ["hstore", "cluster"], "hstore": ["cluster"]}, + "toolchain": {"client": ["loader", "tools", "spark", "hubble"], "loader": ["hubble"]}, +} + + +def git(*args): + return subprocess.check_output(["git", *args], text=True).strip() + + +def api(path, binary=False): + result = subprocess.run(["gh", "api", "--method", "GET", path], check=True, capture_output=True, timeout=30) + return result.stdout if binary else json.loads(result.stdout) + + +def documentation(path, mode="100644"): + """Only prose and static documentation assets; never source or config.""" + if mode != "100644": + return False + p = Path(path) + prose_names = {"README.md", "README_CN.md", "README_ZH.md", "AGENTS.md"} + module_roots = {prefix.rstrip("/") for mapping in PREFIXES.values() for prefix in mapping} + module_roots.update("hugegraph-server/" + module for module in [ + "hugegraph-api", "hugegraph-core", "hugegraph-dist", "hugegraph-example", + "hugegraph-hbase", "hugegraph-hstore", "hugegraph-rocksdb", "hugegraph-test"]) + if p.name in prose_names and (str(p.parent) == "." or str(p.parent) in module_roots): + return True + return path.startswith("docs/") and p.suffix.lower() in {".md", ".rst", ".txt", ".png", ".jpg", ".jpeg", ".svg"} + + +def unsafe_documentation(ref, paths): + entries = subprocess.check_output(["git", "ls-tree", "-rz", "--full-tree", ref]).split(b"\0") + for entry in entries: + if not entry: + continue + metadata, name = entry.split(b"\t", 1) + path = name.decode("utf-8", "surrogateescape") + if path in paths and documentation(path) and metadata.split(b" ", 1)[0] != b"100644": + return True + return False + + +def select(project, paths): + selected = set() + for path in paths: + if documentation(path): + continue + if project == "server" and (Path(path).name == "pom.xml" or path.startswith("install-dist/")): + selected.add("dependency_license") + if path.startswith(".github/workflows/") and Path(path).name in WORKFLOWS[project]: + selected.update(WORKFLOWS[project][Path(path).name]) + continue + if path.endswith(".proto"): + return set(MODULES[project]) + if project == "server": + if path.startswith("hugegraph-server/hugegraph-hstore/"): + selected.add("hstore") + continue + if path.startswith("hugegraph-server/hugegraph-dist/"): + selected.update(["server", "pd", "store", "hstore", "docker", "cluster"]) + continue + if path.startswith(("hugegraph-server/hugegraph-core/", "hugegraph-server/hugegraph-api/", + "hugegraph-server/hugegraph-test/")): + selected.update(["server", "hstore"]) + continue + module = next((value for prefix, value in PREFIXES[project].items() if path.startswith(prefix)), None) + if module is None or path.endswith(".proto"): + return set(MODULES[project]) + selected.add(module) + if project == "server" and ("Dockerfile" in Path(path).name or "docker-entrypoint" in path): + selected.add("docker") + changed = True + while changed: + before = set(selected) + for module in before: + selected.update(DEPENDENTS[project].get(module, [])) + changed = before != selected + return selected + + +def fingerprint(ref, policy=False): + """Hash blob identities, including paths; exclude only the strict prose whitelist.""" + entries = subprocess.check_output(["git", "ls-tree", "-rz", "--full-tree", ref]).split(b"\0") + digest = hashlib.sha256() + for entry in sorted(entries, key=lambda e: e.split(b"\t", 1)[-1]): + if not entry: + continue + metadata, name = entry.split(b"\t", 1) + path = name.decode("utf-8", "surrogateescape") + if policy: + if not (path.startswith(".github/") or path == ".asf.yaml"): + continue + elif documentation(path, metadata.split(b" ", 1)[0].decode()): + continue + digest.update(metadata + b"\t" + name + b"\0") + return digest.hexdigest() + + +def remote_fingerprint(repository, sha, policy_only=False, fetch=api): + tree = fetch(f"repos/{repository}/git/trees/{sha}?recursive=1") + if tree.get("truncated"): + raise ValueError("truncated proof tree") + digest = hashlib.sha256() + for entry in sorted(tree["tree"], key=lambda item: item["path"].encode()): + path = entry["path"] + if entry["type"] == "tree": + continue + if policy_only: + if not (path.startswith(".github/") or path == ".asf.yaml"): + continue + elif documentation(path, entry["mode"]): + continue + digest.update((entry["mode"] + " " + entry["type"] + " " + entry["sha"] + + "\t" + path + "\0").encode()) + return digest.hexdigest() + + +def proof_pr(plan, run, fetch): + associations = run.get("pull_requests", []) + if any(pr.get("number") == plan["pr"] for pr in associations): + return True + owner = run.get("head_repository", {}).get("owner", {}).get("login") + branch = run.get("head_branch") + if not owner or not branch: + return False + candidates = pages(f"repos/{plan['repository']}/pulls?" + urlencode({ + "state": "open", "head": f"{owner}:{branch}"}), fetch) + return any(pr.get("number") == plan["pr"] + and pr.get("base", {}).get("repo", {}).get("full_name") == plan["repository"] + and pr.get("head", {}).get("ref") == branch + and pr.get("head", {}).get("repo", {}).get("full_name") == plan["source"] + for pr in candidates) + + +def suites(project, selected): + if project == "toolchain": + return sorted(selected) + result = [] + if "server" in selected: + result.extend(["server_memory", "server_rocksdb"]) + for module in ["commons", "cluster", "docker", "helm", "dependency_license"]: + if module in selected: + result.append(module) + if selected.intersection({"pd", "store", "hstore", "struct"}): + result.append("pd_store") + return sorted(result) + + +def pages(path, fetch=api): + result = [] + for page in range(1, 11): + value = fetch(path + ("&" if "?" in path else "?") + urlencode({"per_page": 100, "page": page})) + batch = value["jobs"] if isinstance(value, dict) else value + result.extend(batch) + if len(batch) < 100: + return result + raise ValueError("API pagination exceeded verification bound") + + +def successful_jobs(repository, run_id, suite, fetch=api): + jobs = pages(f"repos/{repository}/actions/runs/{run_id}/jobs", fetch) + matching = [job for job in jobs if job.get("name") == suite or job.get("name", "").startswith((suite + " / ", suite + " ("))] + if not matching or any(job.get("conclusion") != "success" for job in matching): + raise ValueError("missing or unsuccessful suite jobs") + return sorted(job["id"] for job in matching) + + +def validate_receipt(plan, receipt, fetch=api): + for key in ["schema", "repository", "project", "pr", "source", "branch", "base", "inputFingerprint", "policyFingerprint"]: + if receipt.get(key) != plan.get(key): + raise ValueError("receipt provenance differs: " + key) + proof = receipt.get("proofs", {}) + required = plan["expected"] + if not required or not set(required).issubset(proof): + raise ValueError("receipt lacks selected suites") + verified = {} + for suite in required: + item = proof[suite] + run = fetch(f"repos/{plan['repository']}/actions/runs/{int(item['runID'])}") + if (run.get("status") != "completed" + or run.get("event") != "pull_request" or run.get("head_sha") != item.get("head") + or run.get("workflow_id") != plan["workflowID"] + or run.get("head_repository", {}).get("full_name") != plan["source"] + or run.get("head_branch") != plan["branch"]): + raise ValueError("proof run is not a successful trusted workflow") + successful_jobs(plan["repository"], item["runID"], "affected-module-tests", fetch) + if not proof_pr(plan, run, fetch): + raise ValueError("proof run lacks matching PR association") + merge = fetch(f"repos/{plan['repository']}/git/commits/{item['testedMergeSHA']}") + if [parent["sha"] for parent in merge.get("parents", [])] != [plan["base"], item["head"]]: + raise ValueError("tested merge parents differ") + actual_input = remote_fingerprint(plan["repository"], item["testedMergeSHA"], fetch=fetch) + if plan["project"] == "toolchain": + context = json.dumps(plan.get("externalInputs", {}), sort_keys=True, separators=(",", ":")) + actual_input = hashlib.sha256((actual_input + context).encode()).hexdigest() + if (actual_input != plan["inputFingerprint"] or remote_fingerprint( + plan["repository"], item["testedMergeSHA"], policy_only=True, fetch=fetch) + != plan["policyFingerprint"]): + raise ValueError("actual tested inputs differ from claimed receipt") + ids = successful_jobs(plan["repository"], item["runID"], suite, fetch) + if ids != item.get("jobIDs"): + raise ValueError("actual job evidence differs") + verified[suite] = item + return verified + + +def find_reuse(plan, fetch=api): + """Receipts are uploaded by this workflow, not read from the PR tree.""" + repository = plan["repository"] + current = fetch(f"repos/{repository}/actions/runs/{os.environ['GITHUB_RUN_ID']}") + plan["workflowID"] = current["workflow_id"] + runs = fetch(f"repos/{repository}/actions/workflows/{plan['workflowID']}/runs?" + urlencode({ + "event": "pull_request", "status": "completed", "per_page": 30}))["workflow_runs"] + for run in runs: + if run["id"] == int(os.environ["GITHUB_RUN_ID"]): + continue + if (run.get("head_repository", {}).get("full_name") != plan["source"] + or run.get("head_branch") != plan["branch"]): + continue + if not proof_pr(plan, run, fetch): + continue + artifacts = fetch(f"repos/{repository}/actions/runs/{run['id']}/artifacts?per_page=100")["artifacts"] + for artifact in artifacts: + size = artifact.get("size_in_bytes") + if (artifact.get("name") != "ci-test-receipt" or artifact.get("expired") + or type(size) is not int or not 0 < size <= 1048576): + continue + try: + payload = fetch(f"repos/{repository}/actions/artifacts/{artifact['id']}/zip", binary=True) + with zipfile.ZipFile(io.BytesIO(payload)) as archive: + entries = archive.infolist() + if len(entries) != 1 or entries[0].filename != "receipt.json": + raise ValueError("unexpected receipt archive entries") + info = entries[0] + mode = (info.external_attr >> 16) & 0o170000 + if (info.is_dir() or info.flag_bits & 1 or mode not in {0, 0o100000} + or info.file_size > 1048576): + raise ValueError("oversized receipt") + receipt = json.loads(archive.read(info)) + if receipt.get("runID") != run["id"]: + raise ValueError("receipt not uploaded by claimed run") + successful_jobs(repository, run["id"], "affected-module-tests", fetch) + verified = validate_receipt(plan, receipt, fetch) + jobs = pages(f"repos/{repository}/actions/runs/{run['id']}/jobs", fetch) + def good_matches(predicate, minimum=1): + matching = [job for job in jobs if predicate(job.get("name", "").lower())] + return len(matching) >= minimum and all(job.get("conclusion") == "success" for job in matching) + plan["optionalSuccess"] = { + "compatibility": (good_matches(lambda n: "hbase compatibility" in n) + and good_matches(lambda n: "macos" in n, 2) + and good_matches(lambda n: "riscv64" in n)), + "security": good_matches(lambda n: n.startswith(("codeql /", "security /"))), + } + return verified + except (ValueError, KeyError, TypeError, AttributeError, zipfile.BadZipFile): + continue + return {} + + +def create_plan(project, event, repository, fetch=api, external_inputs=None): + plan = {"schema": 1, "project": project, "repository": repository, "pr": 0, + "source": repository, "branch": "", "base": "", "head": git("rev-parse", "HEAD"), + "reused": {}, "workflowID": 0, "reason": "affected inputs", + "externalInputs": external_inputs or {}, "testedMergeSHA": git("rev-parse", "HEAD")} + try: + pr = event.get("pull_request") + if pr: + # Refresh PR metadata: event payload may predate a new push or base movement. + live = fetch(f"repos/{repository}/pulls/{pr['number']}") + if (live.get("state") != "open" or live["head"]["sha"] != pr["head"]["sha"] + or live["base"]["repo"]["full_name"] != repository): + raise ValueError("PR metadata changed") + plan.update(pr=pr["number"], source=live["head"]["repo"]["full_name"], + base=pr["base"]["sha"], head=live["head"]["sha"], branch=live["head"]["ref"]) + parents = git("show", "-s", "--format=%P", plan["testedMergeSHA"]).split() + if parents != [plan["base"], plan["head"]]: + raise ValueError("checkout is not the event PR merge") + if live["base"]["sha"] != plan["base"]: + raise ValueError("base advanced after event checkout") + # head/base objects must exist locally; workflow fetches both before planning. + ancestor = git("merge-base", plan["base"], plan["head"]) + paths = git("diff", "--no-renames", "--name-only", ancestor, plan["head"]).splitlines() + else: + plan["base"] = event.get("before", "") + paths = git("diff", "--no-renames", "--name-only", plan["base"], plan["head"]).splitlines() + selected = select(project, paths) + if unsafe_documentation(plan["base"], paths) or unsafe_documentation(plan["testedMergeSHA"], paths): + selected = set(MODULES[project]) + plan["inputFingerprint"] = fingerprint(plan["testedMergeSHA"]) + if project == "toolchain": + context = json.dumps(external_inputs or {}, sort_keys=True, separators=(",", ":")) + plan["inputFingerprint"] = hashlib.sha256((plan["inputFingerprint"] + context).encode()).hexdigest() + plan["policyFingerprint"] = fingerprint(plan["testedMergeSHA"], policy=True) + plan["expected"] = suites(project, selected) + if pr and plan["expected"] and (project != "toolchain" or external_inputs): + plan["reused"] = find_reuse(plan, fetch) + except (subprocess.SubprocessError, OSError, ValueError, KeyError, TypeError, AttributeError): + selected = set(MODULES[project]) + plan.update(reused={}, reason="verification unavailable: full required coverage") + plan.setdefault("inputFingerprint", "") + plan.setdefault("policyFingerprint", "") + plan["expected"] = suites(project, selected) + if not selected: + plan["reason"] = "cumulative PR diff contains only plain prose documentation; modules unaffected" + elif plan["reused"]: + plan["reason"] = "affected inputs match independently verified successful tests" + plan["selected"] = sorted(selected) + for module in MODULES[project]: + module_suites = suites(project, {module}) + plan[module] = module in selected and not (module_suites and all(s in plan["reused"] for s in module_suites)) + plan["pd_store"] = any(plan.get(m, False) for m in ["pd", "store", "hstore", "struct"]) + plan["needsFixture"] = project == "toolchain" and any(plan.get(m, False) for m in MODULES[project]) + plan["fixture"] = plan["needsFixture"] + plan["compatibility"] = not plan.get("optionalSuccess", {}).get("compatibility", False) and project == "server" and bool(selected.intersection({"server", "commons", "struct", "pd", "store", "hstore", "cluster"})) + plan["security"] = (bool(selected) and not plan.get("optionalSuccess", {}).get("security", False)) or any(p.startswith(".github/workflows/codeql") for p in locals().get("paths", [])) + plan["security_languages"] = json.dumps(["java"]) + return plan + + +def gate(plan, results, fetch=api): + if results.get("plan", {}).get("result") != "success": + raise ValueError("planner did not succeed") + proofs = dict(plan["reused"]) + executed = [] + for suite in plan["expected"]: + if suite in proofs: + continue + if results.get(suite, {}).get("result") != "success": + raise ValueError("selected suite did not succeed: " + suite) + executed.append(suite) + if plan["project"] == "toolchain": + if set(executed).intersection({"client", "loader", "tools", "spark", "go"}): + if results.get("fixture", {}).get("result") != "success": + raise ValueError("selected tests lack successful fixture") + if "hubble" in executed and results.get("hubble-fixture", {}).get("result") != "success": + raise ValueError("selected Hubble tests lack successful baseline fixture") + receipt = {key: plan[key] for key in ["schema", "repository", "project", "pr", "source", "branch", "base", + "head", "testedMergeSHA", "inputFingerprint", "policyFingerprint", "workflowID"]} + receipt["runID"] = int(os.environ.get("GITHUB_RUN_ID", "0")) + # Successful tests can gate even if receipt publication is unavailable. Never reuse + # unverified evidence: a missing API proof only disables future receipt reuse. + for suite in executed: + try: + ids = successful_jobs(plan["repository"], receipt["runID"], suite, fetch) + proofs[suite] = {"runID": receipt["runID"], "head": plan["head"], "testedMergeSHA": plan["testedMergeSHA"], "jobIDs": ids} + except (subprocess.SubprocessError, OSError, ValueError, KeyError, TypeError, AttributeError): + pass + receipt["proofs"] = proofs + receipt["executed"] = executed + receipt["reused"] = sorted(plan["reused"]) + summary = "Selection: " + plan.get("reason", "affected inputs") + "\nExecuted: " + ", ".join(executed) + "\nReused: " + ", ".join(receipt["reused"]) + if os.environ.get("GITHUB_STEP_SUMMARY"): + with open(os.environ["GITHUB_STEP_SUMMARY"], "a", encoding="utf-8") as out: + out.write("## Affected module tests\n" + summary + "\n\nUnselected modules: " + + ", ".join(set(MODULES[plan["project"]]) - set(plan["selected"])) + "\n") + print(summary) + return receipt + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + sub = parser.add_subparsers(dest="command", required=True) + p = sub.add_parser("plan") + p.add_argument("--project", choices=MODULES, required=True) + p.add_argument("--event-path", required=True) + p.add_argument("--repository", required=True) + p.add_argument("--output", required=True) + p.add_argument("--inputs-json") + p = sub.add_parser("gate") + p.add_argument("--plan", required=True) + p.add_argument("--results", required=True) + p.add_argument("--output", required=True) + args = parser.parse_args() + if args.command == "plan": + external = json.loads(Path(args.inputs_json).read_text()) if args.inputs_json else None + value = create_plan(args.project, json.loads(Path(args.event_path).read_text()), args.repository, + external_inputs=external) + if os.environ.get("GITHUB_OUTPUT"): + with open(os.environ["GITHUB_OUTPUT"], "a", encoding="utf-8") as output: + for key, item in value.items(): + if isinstance(item, bool): + output.write(f"{key}={str(item).lower()}\n") + output.write(f"security_languages={value['security_languages']}\nplan_file={args.output}\n") + else: + value = gate(json.loads(Path(args.plan).read_text()), json.loads(Path(args.results).read_text())) + Path(args.output).write_text(json.dumps(value, indent=2) + "\n") + + +if __name__ == "__main__": + main() diff --git a/.github/scripts/test_check_rerun.py b/.github/scripts/test_check_rerun.py index 06676ef73..225a75cb8 100644 --- a/.github/scripts/test_check_rerun.py +++ b/.github/scripts/test_check_rerun.py @@ -16,6 +16,9 @@ """Behavioral tests for retry freshness, without credentials or network.""" import copy +import io +import json +import zipfile import importlib.util from pathlib import Path import unittest @@ -33,13 +36,24 @@ class FreshnessTest(unittest.TestCase): def setUp(self): self.repository = "hugegraph/hugegraph-toolchain" self.run = {"status": "completed", "conclusion": "failure", "run_attempt": 1, - "event": "pull_request", "head_sha": "old", "head_branch": "feature"} - self.pr = {"number": 7, "state": "open", "head": {"sha": "old"}, - "base": {"repo": {"full_name": self.repository}}} + "event": "pull_request", "head_sha": "old", "head_branch": "feature", + "head_repository": {"full_name": "alice/toolchain", "owner": {"login": "alice"}}} + self.pr = {"number": 7, "state": "open", "head": {"sha": "old", "ref": "feature", "repo": {"full_name": "alice/toolchain", "owner": {"login": "alice"}}}, + "base": {"sha": "base", "repo": {"full_name": self.repository}}} self.run["pull_requests"] = [copy.deepcopy(self.pr)] self.calls = [] - def fetch(self, path): + def fetch(self, path, binary=False): + if "/artifacts?" in path: + return {"artifacts": [{"name": "ci-plan", "id": 99, "expired": False}]} + if "/actions/artifacts/" in path: + data = io.BytesIO() + with zipfile.ZipFile(data, "w") as archive: + archive.writestr("plan.json", json.dumps({"repository": self.repository, "pr": 7, + "head": "old", "base": "base", "testedMergeSHA": "merge"})) + return data.getvalue() + if "/git/commits/" in path: + return {"parents": [{"sha": "base"}, {"sha": "old"}]} self.calls.append(path) if "/actions/runs/" in path: return self.run @@ -47,6 +61,8 @@ def fetch(self, path): return [self.pr] if "/commits?" in path: return [{"sha": self.pr["head"]["sha"]}] + if "/pulls?" in path: + return [self.pr] if "/pulls/" in path: return self.pr raise AssertionError(path) @@ -85,6 +101,20 @@ def test_empty_association_falls_back_to_commit(self): self.assertEqual("rerun", self.decide()) self.assertTrue(any("/commits/old/pulls" in p for p in self.calls)) + def test_fork_empty_associations_owner_qualified_fallback(self): + self.run.update(pull_requests=[], head_repository={"full_name": "alice/toolchain", "owner": {"login": "alice"}}) + self.pr["head"].update(ref="feature", repo={"full_name": "alice/toolchain", "owner": {"login": "alice"}}) + def fetch(path, **kwargs): + if "/commits/old/pulls" in path: + return [] + if "/pulls?" in path: + self.assertIn("head=alice%3Afeature", path) + return [self.pr] + return self.fetch(path, **kwargs) + self.assertEqual("rerun", checker.decide(self.repository, 42, 1, 2, fetch)[0]) + self.pr["head"]["repo"]["owner"]["login"] = "bob" + self.assertEqual("skip", checker.decide(self.repository, 42, 1, 2, fetch)[0]) + def test_other_repository_pr_is_ignored(self): self.run["pull_requests"][0]["base"]["repo"]["full_name"] = "apache/hugegraph-toolchain" self.assertEqual("rerun", self.decide()) @@ -95,6 +125,24 @@ def test_foreign_pr_fallback_still_rejects_foreign_pr(self): self.pr["base"]["repo"]["full_name"] = "apache/hugegraph-toolchain" self.assertEqual("skip", self.decide()) + def test_same_head_moved_base_is_stale(self): + self.pr["base"]["sha"] = "new-base" + self.assertEqual("skip", self.decide()) + + def test_missing_immutable_plan_refuses_retry(self): + def fetch(path, **kwargs): + if "/artifacts?" in path: + return {"artifacts": []} + return self.fetch(path, **kwargs) + self.assertEqual("skip", checker.decide(self.repository, 42, 1, 2, fetch)[0]) + + def test_same_sha_other_fork_cannot_authorize_retry(self): + self.pr["head"]["repo"].update(full_name="bob/toolchain", owner={"login": "bob"}) + self.assertEqual("skip", self.decide()) + self.pr["head"]["repo"].update(full_name="alice/toolchain", owner={"login": "alice"}) + self.pr["head"]["ref"] = "other" + self.assertEqual("skip", self.decide()) + def test_changed_attempt(self): self.run["run_attempt"] = 2 self.assertEqual("skip", self.decide()) diff --git a/.github/scripts/test_ci_policy.py b/.github/scripts/test_ci_policy.py new file mode 100644 index 000000000..6a4682904 --- /dev/null +++ b/.github/scripts/test_ci_policy.py @@ -0,0 +1,327 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Executable Git fixtures and API proof tests for CI selection.""" +import hashlib +import importlib.util +import io +import zipfile +import json +import os +from pathlib import Path +import subprocess +import tempfile +import unittest +from unittest.mock import patch + +spec = importlib.util.spec_from_file_location("policy", Path(__file__).with_name("ci-policy.py")) +policy = importlib.util.module_from_spec(spec) +spec.loader.exec_module(policy) + + +class PolicyTest(unittest.TestCase): + def setUp(self): + mock = patch.object(policy, "remote_fingerprint", side_effect=lambda *a, **k: "policy" if k.get("policy_only") else "inputs") + mock.start() + self.addCleanup(mock.stop) + + def test_dependency_expansion(self): + self.assertEqual({"client", "loader", "tools", "spark", "hubble"}, + policy.select("toolchain", ["hugegraph-client/src/A.java"])) + self.assertEqual({"loader", "hubble"}, policy.select("toolchain", ["hugegraph-loader/src/A.java"])) + self.assertEqual({"server", "cluster"}, policy.select("server", ["hugegraph-server/A.java"])) + + def test_server_backend_and_startup_dependents(self): + selected = policy.select("server", ["hugegraph-server/hugegraph-hstore/src/test/HstoreTableTest.java"]) + self.assertIn("hstore", selected) + self.assertIn("pd_store", policy.suites("server", selected)) + selected = policy.select("server", ["hugegraph-server/hugegraph-dist/src/assembly/travis/test-start-hugegraph-pd.sh"]) + self.assertIn("pd_store", policy.suites("server", selected)) + self.assertIn("docker", selected) + for module in ["hugegraph-core", "hugegraph-api", "hugegraph-test"]: + selected = policy.select("server", [f"hugegraph-server/{module}/src/main/A.java"]) + self.assertIn("hstore", selected) + self.assertIn("server", selected) + self.assertEqual(set(), policy.select("server", ["hugegraph-server/hugegraph-hstore/README.md"])) + self.assertNotIn("hstore", policy.select("server", ["hugegraph-server/hugegraph-rocksdb/src/main/A.java"])) + + def test_single_workflow_has_no_global_fanout(self): + self.assertEqual({"hubble"}, policy.select("toolchain", [".github/workflows/hubble-ci.yml"])) + self.assertEqual({"docker"}, policy.select("server", [".github/workflows/docker-build-ci.yml"])) + + def test_unknown_and_proto_fail_conservative(self): + for path in ["pom.xml", ".github/scripts/new.py", "hugegraph-pd/api.proto", "mystery"]: + self.assertEqual(set(policy.MODULES["server"]), policy.select("server", [path])) + + def test_strict_docs_allowlist(self): + self.assertEqual(set(), policy.select("server", ["README.md", "docs/guide.md"])) + for path in ["hugegraph-client/src/test/resources/README.md", "docs/type.ts", "docs/config.yml", "docs/example.java", "hugegraph-server/type.ts"]: + self.assertTrue(policy.select("server", [path])) + + def plan(self): + return {"schema": 1, "project": "toolchain", "repository": "apache/t", "pr": 7, + "source": "alice/t", "branch": "feature", "base": "base", "head": "new", "testedMergeSHA": "merge", "inputFingerprint": hashlib.sha256(b"inputs{}").hexdigest(), + "policyFingerprint": "policy", "workflowID": 99, "expected": ["client"], + "selected": ["client"], "reused": {}} + + def receipt(self): + r = self.plan() + r["proofs"] = {"client": {"runID": 42, "head": "old", "testedMergeSHA": "oldmerge", "jobIDs": [123]}} + return r + + def fetch(self, path): + if "/git/commits/" in path: + return {"parents": [{"sha": "base"}, {"sha": "old"}]} + if path.endswith("/jobs?per_page=100&page=1"): + return {"jobs": [{"name": "client / client-ci", "id": 123, "conclusion": "success"}, + {"name": "affected-module-tests", "id": 124, "conclusion": "success"}]} + return {"status": "completed", "conclusion": "success", "event": "pull_request", + "head_sha": "old", "head_branch": "feature", "workflow_id": 99, "head_repository": {"full_name": "alice/t"}, + "pull_requests": [{"number": 7, "base": {"sha": "base"}}]} + + def test_valid_original_proof(self): + with patch.object(policy, "remote_fingerprint", side_effect=lambda *a, **k: "policy" if k.get("policy_only") else "inputs"): + self.assertIn("client", policy.validate_receipt(self.plan(), self.receipt(), self.fetch)) + + def test_empty_fork_association_owner_branch_fallback(self): + def fetch(path): + if "/pulls?" in path: + self.assertIn("head=alice%3Afeature", path) + return [{"number": 7, "head": {"ref": "feature", "repo": {"full_name": "alice/t"}}, + "base": {"repo": {"full_name": "apache/t"}}}] + value = self.fetch(path) + if "/actions/runs/" in path and "/jobs?" not in path: + value.update(pull_requests=[], head_branch="feature", + head_repository={"full_name": "alice/t", "owner": {"login": "alice"}}) + return value + self.assertIn("client", policy.validate_receipt(self.plan(), self.receipt(), fetch)) + + def test_nonblocking_failure_keeps_core_proof_but_bad_gate_rejects(self): + def fetch(path): + value = self.fetch(path) + if "/jobs?" not in path and "/actions/runs/" in path: + value["conclusion"] = "failure" + return value + self.assertIn("client", policy.validate_receipt(self.plan(), self.receipt(), fetch)) + def bad_gate(path): + value = fetch(path) + if "/jobs?" in path: + value["jobs"][1]["conclusion"] = "failure" + return value + with self.assertRaises(ValueError): + policy.validate_receipt(self.plan(), self.receipt(), bad_gate) + + def test_artifact_core_reuse_with_failed_compatibility(self): + receipt = self.receipt() + receipt["runID"] = 42 + data = io.BytesIO() + with zipfile.ZipFile(data, "w") as archive: + archive.writestr("receipt.json", json.dumps(receipt)) + artifact_size = 256 + def fetch(path, binary=False): + if "/actions/workflows/" in path: + self.assertIn("status=completed", path) + return {"workflow_runs": [{"id": 42, "head_repository": {"full_name": "alice/t"}, "head_branch": "feature", "pull_requests": [{"number": 7}]}]} + if "/artifacts?" in path: + return {"artifacts": [{"name": "ci-test-receipt", "id": 3, "expired": False, "size_in_bytes": artifact_size}]} + if "/actions/artifacts/" in path: + return data.getvalue() + value = self.fetch(path) + if "/jobs?" in path: + value["jobs"].append({"name": "HBase compatibility", "id": 9, "conclusion": "failure"}) + elif "/actions/runs/" in path: + value["conclusion"] = "failure" + return value + plan = self.plan() + with patch.dict(os.environ, {"GITHUB_RUN_ID": "99"}): + proof = policy.find_reuse(plan, fetch) + self.assertIn("client", proof) + self.assertFalse(plan["optionalSuccess"]["compatibility"]) + self.assertFalse(plan["optionalSuccess"]["security"]) + for artifact_size in [None, 0, 1048577, "256"]: + with patch.dict(os.environ, {"GITHUB_RUN_ID": "99"}): + self.assertEqual({}, policy.find_reuse(self.plan(), fetch)) + artifact_size = 256 + data = io.BytesIO() + with zipfile.ZipFile(data, "w") as archive: + archive.writestr("receipt.json", json.dumps(receipt)) + archive.writestr("unexpected.txt", "extra") + with patch.dict(os.environ, {"GITHUB_RUN_ID": "99"}): + self.assertEqual({}, policy.find_reuse(self.plan(), fetch)) + + def test_tested_merge_parent_mismatch_rejects(self): + def fetch(path): + if "/git/commits/" in path: + return {"parents": [{"sha": "advanced-base"}, {"sha": "old"}]} + return self.fetch(path) + with self.assertRaises(ValueError): + policy.validate_receipt(self.plan(), self.receipt(), fetch) + + def test_actual_tree_fingerprint_mismatch_rejects(self): + with patch.object(policy, "remote_fingerprint", return_value="forged"): + with self.assertRaises(ValueError): + policy.validate_receipt(self.plan(), self.receipt(), self.fetch) + + def test_provenance_changes_refuse_reuse(self): + for key in ["base", "inputFingerprint", "policyFingerprint", "source", "repository", "pr", "schema"]: + receipt = self.receipt() + receipt[key] = "changed" + with self.subTest(key=key), self.assertRaises(ValueError): + policy.validate_receipt(self.plan(), receipt, self.fetch) + + def test_original_run_and_actual_job_proof_required(self): + for failure in ["failure", "cancelled", "skipped"]: + def fetch(path): + value = self.fetch(path) + if "jobs?" in path: + value["jobs"][0]["conclusion"] = failure + return value + with self.subTest(failure=failure), self.assertRaises(ValueError): + policy.validate_receipt(self.plan(), self.receipt(), fetch) + receipt = self.receipt() + receipt["proofs"]["client"]["jobIDs"] = [456] + with self.assertRaises(ValueError): + policy.validate_receipt(self.plan(), receipt, self.fetch) + + def test_gate_rejects_missing_failed_cancelled_skipped(self): + for result in [None, "failure", "cancelled", "skipped"]: + with self.subTest(result=result), self.assertRaises(ValueError): + policy.gate(self.plan(), {"plan": {"result": "success"}, "client": {"result": result}}, self.fetch) + + def test_gate_success_receipt_and_verified_reuse(self): + with patch.dict(os.environ, {"GITHUB_RUN_ID": "42"}): + result = policy.gate(self.plan(), {"plan": {"result": "success"}, "client": {"result": "success"}, "fixture": {"result": "success"}}, self.fetch) + self.assertEqual([123], result["proofs"]["client"]["jobIDs"]) + plan = self.plan() + plan["reused"] = result["proofs"] + self.assertEqual(["client"], policy.gate(plan, {"plan": {"result": "success"}, "client": {"result": "skipped"}}, self.fetch)["reused"]) + + def test_gate_refuses_failed_planner_even_empty_or_reused(self): + for result in ["failure", "skipped", "cancelled", None]: + plan = self.plan() + plan["expected"] = [] + with self.subTest(result=result), self.assertRaises(ValueError): + policy.gate(plan, {"plan": {"result": result}}, self.fetch) + + def test_fixture_missing_rejects_successful_test(self): + with self.assertRaises(ValueError): + policy.gate(self.plan(), {"plan": {"result": "success"}, "client": {"result": "success"}}, self.fetch) + plan = self.plan() + plan["expected"] = ["hubble"] + with self.assertRaises(ValueError): + policy.gate(plan, {"plan": {"result": "success"}, "hubble": {"result": "success"}}, self.fetch) + + def test_proof_failure_disables_receipt_without_falsifying_gate(self): + def failure(path): + raise subprocess.CalledProcessError(1, "gh") + result = policy.gate(self.plan(), {"plan": {"result": "success"}, "client": {"result": "success"}, + "fixture": {"result": "success"}}, failure) + self.assertEqual({}, result["proofs"]) + self.assertEqual(["client"], result["executed"]) + + def test_doc_update_reuses_only_with_valid_receipt(self): + event = {"pull_request": {"number": 7, "head": {"sha": "new"}, "base": {"sha": "base"}}} + live = {"state": "open", "head": {"sha": "new", "ref": "feature", "repo": {"full_name": "alice/t"}}, + "base": {"sha": "base", "repo": {"full_name": "apache/t"}}} + def git(*args): + if args[0] == "show": + return "base new" + if args[0] == "diff": + return "hugegraph-client/src/A.java\nREADME.md" + return "new" + with patch.object(policy, "unsafe_documentation", return_value=False), patch.object(policy, "git", side_effect=git), patch.object(policy, "fingerprint", return_value="hash"), patch.object( + policy, "find_reuse", return_value={m: {"runID": 42} for m in ["client", "loader", "tools", "spark", "hubble"]}): + plan = policy.create_plan("toolchain", event, "apache/t", lambda p: live, + external_inputs={"serverSHA": "immutable"}) + self.assertFalse(plan["client"]) + self.assertFalse(plan["needsFixture"]) + self.assertTrue(plan["security"]) + with patch.object(policy, "unsafe_documentation", return_value=False), patch.object(policy, "git", side_effect=git), patch.object(policy, "fingerprint", return_value="hash"), patch.object( + policy, "find_reuse", return_value={}): + plan = policy.create_plan("toolchain", event, "apache/t", lambda p: live, + external_inputs={"serverSHA": "immutable"}) + self.assertTrue(plan["client"]) + self.assertTrue(plan["needsFixture"]) + + def test_real_git_fingerprint_and_cli(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + def git(*args): + return subprocess.check_output(["git", *args], cwd=root, text=True).strip() + git("init", "-q") + git("config", "user.email", "ci@example.invalid") + git("config", "user.name", "CI") + (root / "README.md").write_text("initial") + git("add", ".") + git("commit", "-qm", "base") + base = git("rev-parse", "HEAD") + (root / "README.md").write_text("docs change") + git("commit", "-qam", "docs") + head = git("rev-parse", "HEAD") + old = os.getcwd() + try: + os.chdir(root) + self.assertEqual(policy.fingerprint(base), policy.fingerprint(head)) + (root / "type.ts").write_text("type changed") + git("add", ".") + git("commit", "-qm", "types") + self.assertNotEqual(policy.fingerprint(base), policy.fingerprint("HEAD")) + (root / "README.md").chmod(0o755) + git("add", "README.md") + git("commit", "-qm", "executable prose") + self.assertTrue(policy.unsafe_documentation("HEAD", ["README.md"])) + self.assertNotEqual(policy.fingerprint(base), policy.fingerprint("HEAD")) + (root / "README.md").unlink() + (root / "README.md").symlink_to("type.ts") + git("add", "README.md") + git("commit", "-qm", "symlink prose") + self.assertTrue(policy.unsafe_documentation("HEAD", ["README.md"])) + (root / "hugegraph-server").mkdir() + (root / "hugegraph-server" / "A.java").write_text("source") + git("add", ".") + git("commit", "-qm", "source") + before_rename = git("rev-parse", "HEAD") + (root / "docs").mkdir() + git("mv", "hugegraph-server/A.java", "docs/renamed.md") + git("commit", "-qm", "rename code to prose") + changed = git("diff", "--no-renames", "--name-only", before_rename, "HEAD").splitlines() + self.assertIn("server", policy.select("server", changed)) + finally: + os.chdir(old) + event = root / "event.json" + event.write_text(json.dumps({"before": base})) + output = root / "output" + subprocess.run(["python3", str(Path(policy.__file__).resolve()), "plan", "--project", "server", + "--repository", "apache/server", "--event-path", str(event), + "--output", str(root / "plan.json")], cwd=root, check=True, + env={**os.environ, "GITHUB_OUTPUT": str(output)}) + self.assertIn("server=true\n", output.read_text()) + self.assertEqual(policy.suites("server", set(policy.MODULES["server"])), + json.loads((root / "plan.json").read_text())["expected"]) + + def test_api_failure_runs_full_required(self): + event = {"pull_request": {"number": 7, "head": {"sha": "old"}}} + with patch.object(policy, "git", return_value="head"): + def fail(path): + raise subprocess.CalledProcessError(1, "gh") + plan = policy.create_plan("toolchain", event, "apache/t", fail) + self.assertEqual(set(policy.MODULES["toolchain"]), set(plan["selected"])) + self.assertTrue(plan["security"]) + self.assertEqual({}, plan["reused"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 000000000..758291548 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,273 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +name: Required CI + +on: + pull_request: + push: + branches: [master, 'release-*'] + workflow_dispatch: + inputs: + server-pr-number: + description: HugeGraph Server PR integration baseline (empty uses latest release) + required: false + type: string + +permissions: + contents: read + actions: read + pull-requests: read + security-events: write + +concurrency: + group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.run_id }}-${{ github.run_attempt == 1 && 'head' || format('rerun-{0}', github.run_id) }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +jobs: + plan: + runs-on: ubuntu-latest + outputs: + server-commit: ${{ steps.baseline.outputs.commit_id }} + server-ref: ${{ steps.baseline.outputs.fetch_ref }} + hubble-commit: ${{ steps.hubble-baseline.outputs.commit }} + client: ${{ steps.select.outputs.client }} + loader: ${{ steps.select.outputs.loader }} + tools: ${{ steps.select.outputs.tools }} + spark: ${{ steps.select.outputs.spark }} + hubble: ${{ steps.select.outputs.hubble }} + go: ${{ steps.select.outputs.go }} + needsFixture: ${{ steps.select.outputs.needsFixture }} + security: ${{ steps.select.outputs.security }} + steps: + - uses: actions/checkout@v7 + with: + fetch-depth: 0 + persist-credentials: false + - name: Test CI policy and retry guards + run: | + python3 -m unittest discover -s .github/scripts -p 'test_*.py' + python3 .github/actions/setup-hugegraph-server/runtime_test.py + node --test hugegraph-client/assembly/travis/checkout-server.test.cjs hugegraph-client/assembly/travis/install-server.test.cjs + - name: Resolve immutable Server release baseline + id: baseline + uses: ./.github/actions/get-hugegraph-commit + with: + server-pr-number: ${{ inputs.server-pr-number || '' }} + - name: Resolve Hubble's existing Server master baseline + id: hubble-baseline + env: + SERVER_REPOSITORY: https://github.com/apache/hugegraph.git + run: | + commit=$(git ls-remote "$SERVER_REPOSITORY" refs/heads/master | cut -f1) + [[ "$commit" =~ ^[0-9a-f]{40}$ ]] + echo "commit=$commit" >> "$GITHUB_OUTPUT" + - name: Record external test inputs + env: + SERVER_COMMIT: ${{ steps.baseline.outputs.commit_id }} + SERVER_REF: ${{ steps.baseline.outputs.fetch_ref }} + HUBBLE_COMMIT: ${{ steps.hubble-baseline.outputs.commit }} + run: | + python3 - <<'PYINPUTS' + import json, os + inputs = { + 'server': {'repository': 'apache/hugegraph', 'commit': os.environ['SERVER_COMMIT'], + 'ref': os.environ['SERVER_REF']}, + 'hubble': {'repository': 'apache/hugegraph', 'commit': os.environ['HUBBLE_COMMIT'], + 'ref': os.environ['HUBBLE_COMMIT'], 'branch': 'master'}, + 'jdk': '11', 'config': 'rocksdb-ci-v1' + } + with open('inputs.json', 'w') as stream: + json.dump(inputs, stream, sort_keys=True) + PYINPUTS + - name: Select affected modules and verified successes + id: select + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: >- + python3 .github/scripts/ci-policy.py plan --project toolchain + --event-path "$GITHUB_EVENT_PATH" --repository "$GITHUB_REPOSITORY" + --inputs-json inputs.json --output plan.json + - uses: actions/upload-artifact@v7 + with: + name: ci-plan + path: plan.json + overwrite: true + retention-days: 14 + + fixture: + needs: plan + if: needs.plan.outputs.client == 'true' || needs.plan.outputs.loader == 'true' || needs.plan.outputs.tools == 'true' || needs.plan.outputs.spark == 'true' || needs.plan.outputs.go == 'true' + runs-on: ubuntu-latest + outputs: + artifact: ${{ steps.identity.outputs.artifact }} + steps: + - uses: actions/checkout@v7 + with: + persist-credentials: false + - name: Name immutable Server artifact + id: identity + run: echo "artifact=fixture-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" >> "$GITHUB_OUTPUT" + - uses: ./.github/actions/setup-java-env + with: + java-version: '11' + distribution: temurin + use-stage: 'true' + - name: Build reusable Server package + uses: ./.github/actions/setup-hugegraph-server + with: + mode: build + artifact-name: ${{ steps.identity.outputs.artifact }} + commit-id: ${{ needs.plan.outputs.server-commit }} + fetch-ref: ${{ needs.plan.outputs.server-ref }} + java-version: '11' + + hubble-fixture: + needs: plan + if: needs.plan.outputs.hubble == 'true' + runs-on: ubuntu-latest + outputs: + artifact: ${{ steps.identity.outputs.artifact }} + steps: + - uses: actions/checkout@v7 + with: + persist-credentials: false + - name: Name immutable Server artifact + id: identity + run: echo "artifact=hubble-fixture-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" >> "$GITHUB_OUTPUT" + - uses: ./.github/actions/setup-java-env + with: + java-version: '11' + distribution: temurin + use-stage: 'true' + - name: Build reusable Server package + uses: ./.github/actions/setup-hugegraph-server + with: + mode: build + artifact-name: ${{ steps.identity.outputs.artifact }} + commit-id: ${{ needs.plan.outputs.hubble-commit }} + fetch-ref: ${{ needs.plan.outputs.hubble-commit }} + java-version: '11' + + client: + needs: [plan, fixture] + if: ${{ needs.plan.outputs.client == 'true' }} + uses: ./.github/workflows/client-ci.yml + with: + artifact-name: ${{ needs.fixture.outputs.artifact }} + server-commit: ${{ needs.plan.outputs.server-commit }} + server-ref: ${{ needs.plan.outputs.server-ref }} + secrets: inherit + + loader: + needs: [plan, fixture] + if: ${{ needs.plan.outputs.loader == 'true' }} + uses: ./.github/workflows/loader-ci.yml + with: + artifact-name: ${{ needs.fixture.outputs.artifact }} + server-commit: ${{ needs.plan.outputs.server-commit }} + server-ref: ${{ needs.plan.outputs.server-ref }} + secrets: inherit + + tools: + needs: [plan, fixture] + if: ${{ needs.plan.outputs.tools == 'true' }} + uses: ./.github/workflows/tools-ci.yml + with: + artifact-name: ${{ needs.fixture.outputs.artifact }} + server-commit: ${{ needs.plan.outputs.server-commit }} + server-ref: ${{ needs.plan.outputs.server-ref }} + secrets: inherit + + spark: + needs: [plan, fixture] + if: ${{ needs.plan.outputs.spark == 'true' }} + uses: ./.github/workflows/spark-connector-ci.yml + with: + artifact-name: ${{ needs.fixture.outputs.artifact }} + server-commit: ${{ needs.plan.outputs.server-commit }} + server-ref: ${{ needs.plan.outputs.server-ref }} + secrets: inherit + + hubble: + needs: [plan, hubble-fixture] + if: ${{ needs.plan.outputs.hubble == 'true' }} + uses: ./.github/workflows/hubble-ci.yml + with: + artifact-name: ${{ needs.hubble-fixture.outputs.artifact }} + server-commit: ${{ needs.plan.outputs.hubble-commit }} + server-ref: ${{ needs.plan.outputs.hubble-commit }} + secrets: inherit + + go: + needs: [plan, fixture] + if: ${{ needs.plan.outputs.go == 'true' }} + uses: ./.github/workflows/client-go-ci.yml + with: + artifact-name: ${{ needs.fixture.outputs.artifact }} + server-commit: ${{ needs.plan.outputs.server-commit }} + server-ref: ${{ needs.plan.outputs.server-ref }} + secrets: inherit + + affected-module-tests: + name: affected-module-tests + needs: [plan, fixture, hubble-fixture, client, loader, tools, spark, hubble, go] + if: ${{ !cancelled() }} + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + with: + persist-credentials: false + - uses: actions/download-artifact@v7 + with: + name: ci-plan + - name: Require every selected module to succeed + env: + RESULTS: ${{ toJSON(needs) }} + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + printf '%s' "$RESULTS" > results.json + python3 .github/scripts/ci-policy.py gate --plan plan.json --results results.json --output receipt.json + - name: Publish verified test receipt + uses: actions/upload-artifact@v7 + with: + name: ci-test-receipt + path: receipt.json + overwrite: true + retention-days: 14 + + security: + needs: [plan, affected-module-tests] + if: >- + needs.affected-module-tests.result == 'success' && + (needs.plan.outputs.security == 'true' || + (github.event_name == 'pull_request' && vars.CI_OPTIMIZED_REQUIRED != 'true')) + uses: ./.github/workflows/codeql-analysis.yml + secrets: inherit + + legacy-java-analysis: + name: Analyze (java) + needs: security + if: ${{ !cancelled() && vars.CI_OPTIMIZED_REQUIRED != 'true' }} + runs-on: ubuntu-latest + steps: + - name: Require actual Java security analysis during migration + env: + SECURITY_RESULT: ${{ needs.security.result }} + run: test "$SECURITY_RESULT" = success diff --git a/.github/workflows/client-ci.yml b/.github/workflows/client-ci.yml index d96f24bf0..f10d6d67b 100644 --- a/.github/workflows/client-ci.yml +++ b/.github/workflows/client-ci.yml @@ -1,27 +1,17 @@ name: "java-client-ci" on: - workflow_dispatch: - push: - branches: - - master - - 'release-*' - paths: - - hugegraph-client/** - - hugegraph-dist/** - - .github/workflows/** - - pom.xml - pull_request: - paths: - - hugegraph-client/** - - hugegraph-dist/** - - .github/workflows/** - - pom.xml - -# Keep the newest PR run; preserve independent push, release and manual runs. -concurrency: - group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.run_id }} - cancel-in-progress: ${{ github.event_name == 'pull_request' }} + workflow_call: + inputs: + artifact-name: + type: string + required: true + server-commit: + type: string + required: true + server-ref: + type: string + required: true jobs: client-ci: @@ -41,10 +31,6 @@ jobs: with: fetch-depth: 2 - - name: Get HugeGraph stable commit id - id: get-commit - uses: ./.github/actions/get-hugegraph-commit - - name: Setup Java environment uses: ./.github/actions/setup-java-env with: @@ -56,7 +42,10 @@ jobs: uses: ./.github/actions/setup-hugegraph-server with: travis-dir: ${{ env.TRAVIS_DIR }} - commit-id: ${{ steps.get-commit.outputs.commit_id }} + commit-id: ${{ inputs.server-commit }} + fetch-ref: ${{ inputs.server-ref }} + artifact-name: ${{ inputs.artifact-name }} + mode: start - name: Compile run: | diff --git a/.github/workflows/client-go-ci.yml b/.github/workflows/client-go-ci.yml index 6c2fa3c04..cfbebe953 100644 --- a/.github/workflows/client-go-ci.yml +++ b/.github/workflows/client-go-ci.yml @@ -1,26 +1,17 @@ name: "go-client-ci" on: - push: - branches: - - master - - 'release-*' - paths: - - hugegraph-client-go/** - - hugegraph-dist/** - - .github/workflows/** - - pom.xml - pull_request: - paths: - - hugegraph-client-go/** - - hugegraph-dist/** - - .github/workflows/** - - pom.xml - -# Keep the newest PR run; preserve independent push, release and manual runs. -concurrency: - group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.run_id }} - cancel-in-progress: ${{ github.event_name == 'pull_request' }} + workflow_call: + inputs: + artifact-name: + type: string + required: true + server-commit: + type: string + required: true + server-ref: + type: string + required: true jobs: client-go-ci: @@ -39,10 +30,6 @@ jobs: with: fetch-depth: 2 - - name: Get HugeGraph stable commit id - id: get-commit - uses: ./.github/actions/get-hugegraph-commit - - name: Setup Java environment uses: ./.github/actions/setup-java-env with: @@ -54,7 +41,10 @@ jobs: uses: ./.github/actions/setup-hugegraph-server with: travis-dir: ${{ env.TRAVIS_DIR }} - commit-id: ${{ steps.get-commit.outputs.commit_id }} + commit-id: ${{ inputs.server-commit }} + fetch-ref: ${{ inputs.server-ref }} + artifact-name: ${{ inputs.artifact-name }} + mode: start - name: Init Go env uses: actions/setup-go@v6 diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 2ee2cf91c..49486b451 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -1,29 +1,11 @@ -# For most projects, this workflow file will not need changing; you simply need -# to commit it to your repository. -# -# You may wish to alter this file to override the set of languages analyzed, -# or to provide custom queries or build logic. -# -# ******** NOTE ******** -# We have attempted to detect the languages in your repository. Please check -# the `language` matrix defined below to confirm you have the correct set of -# supported CodeQL languages. -# name: "CodeQL" on: workflow_dispatch: - pull_request: - # The branches below must be a subset of the branches above - # branches: [ master ] # enable in all PR + workflow_call: schedule: - cron: '42 20 * * 6' -# Keep the newest PR run; preserve independent push, release and manual runs. -concurrency: - group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.run_id }} - cancel-in-progress: ${{ github.event_name == 'pull_request' }} - jobs: analyze: env: @@ -99,6 +81,7 @@ jobs: uses: github/codeql-action/analyze@v4 dependency-review: + if: github.event_name == 'pull_request' runs-on: ubuntu-latest steps: - name: 'Checkout Repository' diff --git a/.github/workflows/hubble-ci.yml b/.github/workflows/hubble-ci.yml index b802ecebb..1cf6bebbe 100644 --- a/.github/workflows/hubble-ci.yml +++ b/.github/workflows/hubble-ci.yml @@ -1,35 +1,20 @@ name: "hubble-ci" on: - workflow_dispatch: - push: - branches: - - master - - 'release-*' - paths: - - hugegraph-hubble/** - - hugegraph-dist/** - - hugegraph-loader/** - - .github/workflows/** - - pom.xml - pull_request: - paths: - - hugegraph-hubble/** - - hugegraph-dist/** - - hugegraph-loader/** - - hugegraph-client/** - - .github/workflows/** - - pom.xml + workflow_call: + inputs: + artifact-name: + type: string + required: true + server-commit: + type: string + required: true + server-ref: + type: string + required: true env: TRAVIS_DIR: hugegraph-hubble/hubble-dist/assembly/travis - HUGEGRAPH_SERVER_GIT_URL: https://github.com/apache/hugegraph.git - HUGEGRAPH_SERVER_BRANCH: master - -# Keep the newest PR run; preserve independent push, release and manual runs. -concurrency: - group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.run_id }} - cancel-in-progress: ${{ github.event_name == 'pull_request' }} jobs: hubble-ci: @@ -92,23 +77,6 @@ jobs: path: ~/.cache/ms-playwright key: ${{ runner.os }}-playwright-${{ hashFiles('hugegraph-hubble/hubble-fe/yarn.lock') }} - - name: Resolve HugeGraph Server revision - id: server-ref - run: | - SERVER_COMMIT="$(git ls-remote "$HUGEGRAPH_SERVER_GIT_URL" \ - "refs/heads/$HUGEGRAPH_SERVER_BRANCH" | cut -f1)" - if [[ -z "$SERVER_COMMIT" ]]; then - echo "Unable to resolve HugeGraph Server branch" >&2 - exit 1 - fi - echo "commit=$SERVER_COMMIT" >> "$GITHUB_OUTPUT" - - - name: Cache HugeGraph Server - uses: actions/cache@v6 - with: - path: ~/hugegraph-cache-${{ steps.server-ref.outputs.commit }} - key: ${{ runner.os }}-hugegraph-server-${{ steps.server-ref.outputs.commit }} - - name: Frontend i18n check working-directory: hugegraph-hubble/hubble-fe env: @@ -132,8 +100,8 @@ jobs: - name: Prepare env and service env: - COMMIT_ID: ${{ steps.server-ref.outputs.commit }} - COMMIT_REF: ${{ steps.server-ref.outputs.commit }} + COMMIT_ID: ${{ inputs.server-commit }} + COMMIT_REF: ${{ inputs.server-commit }} NODE_OPTIONS: --max-old-space-size=4096 run: | echo "=== Environment Info ===" @@ -154,7 +122,15 @@ jobs: ./stop-hubble.sh cd ../../../ pwd - $TRAVIS_DIR/install-hugegraph.sh $COMMIT_ID $COMMIT_REF + + - name: Setup HugeGraph server + uses: ./.github/actions/setup-hugegraph-server + with: + mode: start + artifact-name: ${{ inputs.artifact-name }} + commit-id: ${{ inputs.server-commit }} + fetch-ref: ${{ inputs.server-ref }} + travis-dir: ${{ env.TRAVIS_DIR }} - name: Release package audit env: diff --git a/.github/workflows/labeler.yml b/.github/workflows/labeler.yml index 0222c1ae6..ab9e1e34d 100644 --- a/.github/workflows/labeler.yml +++ b/.github/workflows/labeler.yml @@ -22,7 +22,7 @@ on: - pull_request_target concurrency: - group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.run_id }} + group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.run_id }}-${{ github.run_attempt == 1 && 'head' || format('rerun-{0}', github.run_id) }} cancel-in-progress: ${{ github.event_name == 'pull_request_target' }} jobs: diff --git a/.github/workflows/license-checker.yml b/.github/workflows/license-checker.yml index e4fe77e67..7dda8e16a 100644 --- a/.github/workflows/license-checker.yml +++ b/.github/workflows/license-checker.yml @@ -28,15 +28,36 @@ on: # Keep the newest PR run; preserve independent push, release and manual runs. concurrency: - group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.run_id }} + group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.run_id }}-${{ github.run_attempt == 1 && 'head' || format('rerun-{0}', github.run_id) }} cancel-in-progress: ${{ github.event_name == 'pull_request' }} jobs: check-license-header: runs-on: ubuntu-latest + outputs: + dependencies: ${{ steps.dependencies.outputs.changed }} steps: - name: Fetch Code uses: actions/checkout@v7 + with: + fetch-depth: 0 + - name: Select dependency license audit + id: dependencies + env: + EVENT_NAME: ${{ github.event_name }} + BASE_SHA: ${{ github.event.pull_request.base.sha }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + run: | + changed=true + if [[ "$EVENT_NAME" == pull_request ]]; then + if git diff --name-only "$BASE_SHA...$HEAD_SHA" > "$RUNNER_TEMP/license-paths"; then + changed=false + if grep -Eq '(^|/)(pom.xml|package.json|yarn.lock|package-lock.json)$|(^|/)(assembly|licenses)/|(^|/)(LICENSE|NOTICE)$|dependency/|^\.github/(configs/|workflows/license-checker.yml)' "$RUNNER_TEMP/license-paths"; then + changed=true + fi + fi + fi + echo "changed=$changed" >> "$GITHUB_OUTPUT" - name: Test automatic retry freshness rules run: python3 -m unittest discover -s .github/scripts -p 'test_*.py' # More info could refer to: https://github.com/apache/skywalking-eyes @@ -55,6 +76,8 @@ jobs: grep "Binaries" merged-rat.txt -C 3 && cat merged-rat.txt check-dependency-license: + needs: check-license-header + if: needs.check-license-header.outputs.dependencies == 'true' runs-on: ubuntu-latest env: SCRIPT_DEPENDENCY: hugegraph-dist/scripts/dependency diff --git a/.github/workflows/loader-ci.yml b/.github/workflows/loader-ci.yml index 1a37f46e0..d5adf1160 100644 --- a/.github/workflows/loader-ci.yml +++ b/.github/workflows/loader-ci.yml @@ -1,31 +1,21 @@ name: "loader-ci" on: - workflow_dispatch: - push: - branches: - - master - - 'release-*' - paths: - - hugegraph-loader/** - - hugegraph-dist/** - - .github/workflows/** - - pom.xml - pull_request: - paths: - - hugegraph-loader/** - - hugegraph-dist/** - - hugegraph-client/** - - .github/workflows/** - - pom.xml - -# Keep the newest PR run; preserve independent push, release and manual runs. -concurrency: - group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.run_id }} - cancel-in-progress: ${{ github.event_name == 'pull_request' }} + workflow_call: + inputs: + artifact-name: + type: string + required: true + server-commit: + type: string + required: true + server-ref: + type: string + required: true jobs: - loader-ci: + profiles: + name: profiles runs-on: ubuntu-latest env: USE_STAGE: 'true' # Whether to include the stage repository. @@ -52,10 +42,6 @@ jobs: with: fetch-depth: 2 - - name: Get HugeGraph stable commit id - id: get-commit - uses: ./.github/actions/get-hugegraph-commit - - name: Setup Java environment uses: ./.github/actions/setup-java-env with: @@ -63,35 +49,24 @@ jobs: distribution: 'temurin' use-stage: ${{ env.USE_STAGE }} - - name: Cache Hadoop - uses: actions/cache@v6 - with: - path: ~/hadoop-3.3.6.tar.gz - key: ${{ runner.os }}-hadoop-3.3.6 - - - name: Cache HugeGraph Server - uses: actions/cache@v6 - with: - path: ~/hugegraph-cache-${{ steps.get-commit.outputs.commit_id }} - key: ${{ runner.os }}-hugegraph-server-${{ steps.get-commit.outputs.commit_id }} - - name: Compile run: | mvn install -pl hugegraph-client,hugegraph-loader -am -Dmaven.javadoc.skip=true -DskipTests -ntp - - name: Prepare env and service - env: - COMMIT_ID: ${{ steps.get-commit.outputs.commit_id }} - run: | - $TRAVIS_DIR/install-hadoop.sh - $TRAVIS_DIR/install-hugegraph-from-source.sh $COMMIT_ID + - name: Setup HugeGraph server + uses: ./.github/actions/setup-hugegraph-server + with: + mode: start + artifact-name: ${{ inputs.artifact-name }} + commit-id: ${{ inputs.server-commit }} + fetch-ref: ${{ inputs.server-ref }} + travis-dir: ${{ env.TRAVIS_DIR }} - name: Run test run: | cd hugegraph-loader && ls mvn test -P unit -ntp mvn test -P file - mvn test -P hdfs mvn test -P jdbc mvn test -P kafka @@ -100,3 +75,64 @@ jobs: with: token: ${{ secrets.CODECOV_TOKEN }} file: hugegraph-loader/target/jacoco.xml + + hdfs: + name: hdfs + runs-on: ubuntu-latest + env: + USE_STAGE: 'true' # Whether to include the stage repository. + TRAVIS_DIR: hugegraph-loader/assembly/travis + STATIC_DIR: hugegraph-loader/assembly/static + DB_USER: root + DB_PASS: root + DB_DATABASE: load_test + strategy: + matrix: + JAVA_VERSION: ['11'] + steps: + - name: Fetch Code + uses: actions/checkout@v7 + with: + fetch-depth: 2 + + - name: Setup Java environment + uses: ./.github/actions/setup-java-env + with: + java-version: ${{ matrix.JAVA_VERSION }} + distribution: 'temurin' + use-stage: ${{ env.USE_STAGE }} + + - name: Compile + run: | + mvn install -pl hugegraph-client,hugegraph-loader -am -Dmaven.javadoc.skip=true -DskipTests -ntp + + - name: Setup HugeGraph server + uses: ./.github/actions/setup-hugegraph-server + with: + mode: start + artifact-name: ${{ inputs.artifact-name }} + commit-id: ${{ inputs.server-commit }} + fetch-ref: ${{ inputs.server-ref }} + travis-dir: ${{ env.TRAVIS_DIR }} + + - name: Cache verified Hadoop archive + uses: actions/cache@v6 + with: + path: | + ~/hadoop-3.3.6.tar.gz + ~/hadoop-3.3.6.tar.gz.sha512 + key: ${{ runner.os }}-hadoop-3.3.6-sha512-v1 + + - name: Start Hadoop + timeout-minutes: 35 + run: | + "$TRAVIS_DIR/install-hadoop.sh" + + - name: Run HDFS tests + run: mvn test -P hdfs -f hugegraph-loader/pom.xml -ntp + + - name: Upload coverage to Codecov + uses: ./.github/actions/upload-coverage + with: + token: ${{ secrets.CODECOV_TOKEN }} + file: hugegraph-loader/target/jacoco.xml diff --git a/.github/workflows/rerun-ci.yml b/.github/workflows/rerun-ci.yml index 831a67860..ce451adfd 100644 --- a/.github/workflows/rerun-ci.yml +++ b/.github/workflows/rerun-ci.yml @@ -3,12 +3,7 @@ name: "Rerun CI" on: workflow_run: workflows: - - "java-client-ci" - - "loader-ci" - - "tools-ci" - - "hubble-ci" - - "spark-connector-ci" - - "go-client-ci" + - "Required CI" types: - completed diff --git a/.github/workflows/spark-connector-ci.yml b/.github/workflows/spark-connector-ci.yml index 996fefea6..29bbe1d46 100644 --- a/.github/workflows/spark-connector-ci.yml +++ b/.github/workflows/spark-connector-ci.yml @@ -1,28 +1,17 @@ name: "spark-connector-ci" on: - workflow_dispatch: - push: - branches: - - master - - 'release-*' - paths: - - hugegraph-spark-connector/** - - hugegraph-dist/** - - .github/workflows/** - - pom.xml - pull_request: - paths: - - hugegraph-spark-connector/** - - hugegraph-dist/** - - hugegraph-client/** - - .github/workflows/** - - pom.xml - -# Keep the newest PR run; preserve independent push, release and manual runs. -concurrency: - group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.run_id }} - cancel-in-progress: ${{ github.event_name == 'pull_request' }} + workflow_call: + inputs: + artifact-name: + type: string + required: true + server-commit: + type: string + required: true + server-ref: + type: string + required: true jobs: spark-connector-ci: @@ -40,10 +29,6 @@ jobs: with: fetch-depth: 2 - - name: Get HugeGraph stable commit id - id: get-commit - uses: ./.github/actions/get-hugegraph-commit - - name: Setup Java environment uses: ./.github/actions/setup-java-env with: @@ -59,7 +44,10 @@ jobs: uses: ./.github/actions/setup-hugegraph-server with: travis-dir: ${{ env.TRAVIS_DIR }} - commit-id: ${{ steps.get-commit.outputs.commit_id }} + commit-id: ${{ inputs.server-commit }} + fetch-ref: ${{ inputs.server-ref }} + artifact-name: ${{ inputs.artifact-name }} + mode: start - name: Run test run: | diff --git a/.github/workflows/tools-ci.yml b/.github/workflows/tools-ci.yml index 347820de3..a37c9f0d8 100644 --- a/.github/workflows/tools-ci.yml +++ b/.github/workflows/tools-ci.yml @@ -1,28 +1,17 @@ name: "tools-ci" on: - workflow_dispatch: - push: - branches: - - master - - 'release-*' - paths: - - hugegraph-tools/** - - hugegraph-dist/** - - .github/workflows/** - - pom.xml - pull_request: - paths: - - hugegraph-tools/** - - hugegraph-dist/** - - hugegraph-client/** - - .github/workflows/** - - pom.xml - -# Keep the newest PR run; preserve independent push, release and manual runs. -concurrency: - group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.run_id }} - cancel-in-progress: ${{ github.event_name == 'pull_request' }} + workflow_call: + inputs: + artifact-name: + type: string + required: true + server-commit: + type: string + required: true + server-ref: + type: string + required: true jobs: tools-ci: @@ -40,10 +29,6 @@ jobs: with: fetch-depth: 2 - - name: Get HugeGraph stable commit id - id: get-commit - uses: ./.github/actions/get-hugegraph-commit - - name: Setup Java environment uses: ./.github/actions/setup-java-env with: @@ -59,7 +44,10 @@ jobs: uses: ./.github/actions/setup-hugegraph-server with: travis-dir: ${{ env.TRAVIS_DIR }} - commit-id: ${{ steps.get-commit.outputs.commit_id }} + commit-id: ${{ inputs.server-commit }} + fetch-ref: ${{ inputs.server-ref }} + artifact-name: ${{ inputs.artifact-name }} + mode: start - name: Run test run: | diff --git a/README.md b/README.md index 7d13b508e..d1fa40d20 100644 --- a/README.md +++ b/README.md @@ -483,4 +483,4 @@ Thank you to all the people who already contributed to HugeGraph! hugegraph-toolchain is licensed under [Apache 2.0 License](https://github.com/apache/hugegraph-toolchain/blob/master/LICENSE). -See [CI behavior](docs/ci.md) for PR cancellation and automatic retry rules. +See [CI behavior](docs/ci.md) for affected-module tests, verified result reuse and retry rules. diff --git a/docs/ci.md b/docs/ci.md index c28bd6273..3d37026fd 100644 --- a/docs/ci.md +++ b/docs/ci.md @@ -1,18 +1,44 @@ # Continuous integration -PR workflows cancel older runs of the same workflow and PR when a new run arrives. -Pushes, scheduled jobs and manual runs use separate concurrency groups and are not -cancelled by PR updates. The labeler applies the same rule to `pull_request_target`. +Every PR runs license checks and `Required CI`. The stable +`affected-module-tests` check requires all selected Linux tests to pass; a failed, +cancelled or unexpectedly skipped module cannot satisfy it. -The automatic retry workflow retries failed jobs at most twice, retaining its -existing delay. Before and after the delay it checks the current run and source -through the GitHub API. Only the same completed failed attempt can be retried, with either an -open PR in this repository at the same head SHA or an unchanged push branch tip. -Closed PRs, superseded commits, changed attempts and unverifiable API responses -are skipped. The retry checker loads only trusted default-branch code. +| Change | Tests | +| --- | --- | +| Java Client | Client, Loader, Tools, Spark, Hubble | +| Loader | Loader, Hubble | +| Tools, Spark, Go or Hubble | The affected module | +| Shared build inputs or unrecognized paths | All modules | +| One workflow | Its tests and shared dependencies | -Run its behavioral tests locally with: +Loader's HDFS tests run separately, so other profiles do not wait for Hadoop. +Client, Loader, Tools, Spark and Go share one verified Server package, but each +starts its own service. Hubble retains its Server master baseline. Package reuse +checks source, commit, JDK, build configuration and archive checksum. + +Documentation paths use an explicit allowlist. A documentation-only update may +reuse a successful receipt from the same PR only when the base, non-document +inputs and CI policy match. The latest commit receives a new gate result that +identifies reused tests. Missing or unverifiable evidence runs the tests again; +source, type definitions, tests and CI configuration are never treated as docs. + +CodeQL follows the module gate and retains its weekly scan. During migration, +`Analyze (java)` still runs on every PR. After ASF branch protection actually +requires `check-license-header` and `affected-module-tests`, an administrator can +set `CI_OPTIMIZED_REQUIRED=true` to skip security scans for documentation-only +changes. This switch is disabled by default and does not alter branch protection. + +A new PR head cancels older first attempts. Reruns use separate concurrency groups, +so retrying an old commit cannot cancel the current head. Automatic retries run +failed jobs at most twice, checking the open PR/current branch and unchanged run +attempt both before and after the 180-second delay. A moved base or missing +immutable plan also prevents an obsolete retry. Retry code comes from the trusted +default branch. + +Validate policy and retry behavior locally: ```bash python3 -m unittest discover -s .github/scripts -p 'test_*.py' +actionlint ``` diff --git a/hugegraph-client/assembly/travis/checkout-server.sh b/hugegraph-client/assembly/travis/checkout-server.sh new file mode 100644 index 000000000..53b543f0f --- /dev/null +++ b/hugegraph-client/assembly/travis/checkout-server.sh @@ -0,0 +1,47 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with this +# work for additional information regarding copyright ownership. The ASF +# licenses this file to You under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +set -euo pipefail + +if [[ $# -ne 2 ]]; then + echo "Usage: checkout-server.sh " >&2 + exit 1 +fi + +COMMIT_ID=$1 +CHECKOUT_DIR=$2 +SERVER_REPOSITORY=${SERVER_REPOSITORY:-apache/hugegraph} +SERVER_FETCH_REF=${SERVER_FETCH_REF:-$COMMIT_ID} +if [[ ! "$COMMIT_ID" =~ ^[0-9a-fA-F]{40}$ ]] || + [[ ! "$SERVER_REPOSITORY" =~ ^[a-zA-Z0-9][a-zA-Z0-9_.-]*/[a-zA-Z0-9][a-zA-Z0-9_.-]*$ ]]; then + echo "Expected a full commit SHA and GitHub owner/repository" >&2 + exit 1 +fi + +# Fetch a reachable ref, then verify it against the resolved SHA before building. +# A moving branch or PR must fail closed rather than silently changing the baseline. +[[ ! -e "$CHECKOUT_DIR" ]] || { echo "Checkout directory already exists" >&2; exit 1; } +git init "$CHECKOUT_DIR" +git -C "$CHECKOUT_DIR" remote add origin "https://github.com/${SERVER_REPOSITORY}.git" +git -C "$CHECKOUT_DIR" fetch --depth 1 -- origin "$SERVER_FETCH_REF" +git -C "$CHECKOUT_DIR" checkout --detach FETCH_HEAD +ACTUAL_COMMIT=$(git -C "$CHECKOUT_DIR" rev-parse HEAD) +if [[ "$ACTUAL_COMMIT" != "$(printf '%s' "$COMMIT_ID" | tr '[:upper:]' '[:lower:]')" ]]; then + echo "Server checkout mismatch: expected $COMMIT_ID, got $ACTUAL_COMMIT" >&2 + exit 1 +fi +printf 'Server source: %s@%s\n' "$SERVER_REPOSITORY" "$ACTUAL_COMMIT" diff --git a/hugegraph-client/assembly/travis/checkout-server.test.cjs b/hugegraph-client/assembly/travis/checkout-server.test.cjs new file mode 100644 index 000000000..36a83e6e1 --- /dev/null +++ b/hugegraph-client/assembly/travis/checkout-server.test.cjs @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +const test = require('node:test'); +const assert = require('node:assert/strict'); +const { execFileSync, spawnSync } = require('node:child_process'); +const { mkdtempSync, rmSync, writeFileSync, existsSync } = require('node:fs'); +const { tmpdir } = require('node:os'); +const { join } = require('node:path'); + +test('checkout verifies actual git HEAD and refuses missing/moving refs and existing directories', t => { + const root = mkdtempSync(join(tmpdir(), 'server-checkout-')); + t.after(() => rmSync(root, { recursive: true, force: true })); + const source = join(root, 'source'); + const git = (...args) => execFileSync('git', args, { encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'] }).trim(); + git('init', source); + git('-C', source, 'config', 'user.name', 'Fixture'); + git('-C', source, 'config', 'user.email', 'fixture@example.invalid'); + writeFileSync(join(source, 'file'), 'one'); + git('-C', source, 'add', 'file'); + git('-C', source, 'commit', '-m', 'initial'); + git('-C', source, 'branch', '-M', 'candidate'); + const first = git('-C', source, 'rev-parse', 'HEAD'); + const env = { ...process.env, SERVER_REPOSITORY: 'example/server', SERVER_FETCH_REF: 'candidate', + GIT_CONFIG_COUNT: '1', GIT_CONFIG_KEY_0: `url.file://${source}.insteadOf`, + GIT_CONFIG_VALUE_0: 'https://github.com/example/server.git' }; + const checkout = (name, sha, extra = {}) => spawnSync('bash', + [join(__dirname, 'checkout-server.sh'), sha, join(root, name)], + { env: { ...env, ...extra }, encoding: 'utf8' }); + const result = checkout('success', first); + assert.equal(result.status, 0, result.stderr); + assert.equal(git('-C', join(root, 'success'), 'rev-parse', 'HEAD'), first); + assert.notEqual(checkout('success', first).status, 0); + assert.notEqual(checkout('missing', first, { SERVER_FETCH_REF: 'absent' }).status, 0); + writeFileSync(join(source, 'file'), 'two'); + git('-C', source, 'commit', '-am', 'advance'); + const moved = checkout('moved', first); + assert.notEqual(moved.status, 0); + assert.match(moved.stderr, /Server checkout mismatch/); + assert.notEqual(checkout('invalid', 'short').status, 0); + assert.equal(existsSync(join(root, 'invalid')), false); + assert.notEqual(checkout('bad-repo', first, { SERVER_REPOSITORY: '../server' }).status, 0); + assert.equal(existsSync(join(root, 'bad-repo')), false); +}); diff --git a/hugegraph-client/assembly/travis/install-hugegraph-from-source.sh b/hugegraph-client/assembly/travis/install-hugegraph-from-source.sh index d7b74aff9..4f7b058da 100755 --- a/hugegraph-client/assembly/travis/install-hugegraph-from-source.sh +++ b/hugegraph-client/assembly/travis/install-hugegraph-from-source.sh @@ -15,51 +15,30 @@ # License for the specific language governing permissions and limitations # under the License. # -set -ev +set -euo pipefail -if [[ $# -ne 1 ]]; then - echo "Must input an existing commit id of hugegraph server" && exit 1 +if [[ $# -lt 1 || $# -gt 2 || ( $# -eq 2 && "$2" != "--build-only" ) ]]; then + echo "Usage: install-hugegraph-from-source.sh [--build-only]" >&2 + exit 1 fi -COMMIT_ID=$1 -HUGEGRAPH_GIT_URL="https://github.com/apache/hugegraph.git" - -git clone --depth 150 ${HUGEGRAPH_GIT_URL} hugegraph -cd hugegraph -git checkout "${COMMIT_ID}" -mvn package -DskipTests -Dmaven.javadoc.skip=true -ntp -cd hugegraph-server -mv apache-hugegraph-*.tar.gz ../../ -cd ../../ -rm -rf hugegraph -tar zxf apache-hugegraph-*.tar.gz - -HTTPS_SERVER_DIR="hugegraph_https" -mkdir ${HTTPS_SERVER_DIR} -cp -r apache-hugegraph-*/. ${HTTPS_SERVER_DIR} -cd "$(find apache-hugegraph-* | head -1)" -# start HugeGraphServer with http protocol -sed -i 's|gremlin.graph=org.apache.hugegraph.HugeFactory|gremlin.graph=org.apache.hugegraph.auth.HugeFactoryAuthProxy|' conf/graphs/hugegraph.properties -sed -i 's|#auth.authenticator=.*|auth.authenticator=org.apache.hugegraph.auth.StandardAuthenticator|' conf/rest-server.properties -sed -i 's|#auth.admin_pa=.*|auth.admin_pa=pa|' conf/rest-server.properties -echo -e "pa" | bin/init-store.sh || exit 1 -bin/start-hugegraph.sh || exit 1 - -# Wait for server to initialize -echo "Waiting 5 seconds for HugeGraph server to initialize..." -sleep 5 - -cd ../${HTTPS_SERVER_DIR} -REST_SERVER_CONFIG="conf/rest-server.properties" -GREMLIN_SERVER_CONFIG="conf/gremlin-server.yaml" -sed -i "s?http://127.0.0.1:8080?https://127.0.0.1:8443?g" "$REST_SERVER_CONFIG" -sed -i "s/#port: 8182/port: 8282/g" "$GREMLIN_SERVER_CONFIG" -echo "gremlinserver.url=http://127.0.0.1:8282" >> ${REST_SERVER_CONFIG} - -# start HugeGraphServer with https protocol -sed -i 's|gremlin.graph=org.apache.hugegraph.HugeFactory|gremlin.graph=org.apache.hugegraph.auth.HugeFactoryAuthProxy|' conf/graphs/hugegraph.properties -sed -i 's|#auth.authenticator=.*|auth.authenticator=org.apache.hugegraph.auth.StandardAuthenticator|' conf/rest-server.properties -sed -i 's|#auth.admin_pa=.*|auth.admin_pa=pa|' conf/rest-server.properties -echo -e "pa" | bin/init-store.sh || exit 1 -bin/start-hugegraph.sh -cd ../ +if [[ "${FIXTURE_MODE:-standalone}" == start ]]; then + FIXTURE_COMMIT=$1 + ROOT_DIR=$(cd "$(dirname "$0")/../../.." && pwd) + source "$ROOT_DIR/.github/actions/setup-hugegraph-server/fixture.sh" + ARCHIVE=$(python3 "$ROOT_DIR/.github/actions/setup-hugegraph-server/manifest.py" name) + exec bash "$ROOT_DIR/hugegraph-client/assembly/travis/start-hugegraph-servers.sh" "$ARCHIVE" +fi +SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd) +bash "$SCRIPT_DIR/checkout-server.sh" "$1" hugegraph +# Build the server distribution without replacing published client dependencies +# in the caller's Maven repository. --build-only skips server startup. +(cd hugegraph && mvn package -DskipTests -Dmaven.javadoc.skip=true -ntp) +if [[ "${2:-}" != "--build-only" ]]; then + ARCHIVES=(hugegraph/hugegraph-server/apache-hugegraph-*.tar.gz) + if [[ ${#ARCHIVES[@]} -ne 1 || ! -f "${ARCHIVES[0]}" ]]; then + printf 'Expected exactly one server archive, found: %s\n' "${ARCHIVES[*]}" >&2 + exit 1 + fi + bash "$SCRIPT_DIR/start-hugegraph-servers.sh" "${ARCHIVES[0]}" +fi diff --git a/hugegraph-client/assembly/travis/install-server.test.cjs b/hugegraph-client/assembly/travis/install-server.test.cjs new file mode 100644 index 000000000..cc3d32ae8 --- /dev/null +++ b/hugegraph-client/assembly/travis/install-server.test.cjs @@ -0,0 +1,356 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +const test = require('node:test'); +const assert = require('node:assert/strict'); +const { execFileSync, spawnSync } = require('node:child_process'); +const { mkdtempSync, mkdirSync, rmSync, writeFileSync, readFileSync, readdirSync, existsSync } = require('node:fs'); +const { tmpdir } = require('node:os'); +const { join } = require('node:path'); +const { createHash } = require('node:crypto'); + +function cacheStamp(cache, repository, sha, archive) { + const digest = createHash('sha256').update(readFileSync(join(cache, archive))).digest('hex'); + writeFileSync(join(cache, 'server-provenance'), `${repository}\n${sha}\n${archive}\n${digest}\n`); +} + +function temp(t) { + const root = mkdtempSync(join(tmpdir(), 'server-install-')); + t.after(() => rmSync(root, { recursive: true, force: true })); + return root; +} + +for (const installer of [ + join(__dirname, 'install-hugegraph-from-source.sh'), + join(__dirname, '../../../hugegraph-tools/assembly/travis/install-hugegraph-from-source.sh'), + join(__dirname, '../../../hugegraph-spark-connector/assembly/travis/install-hugegraph-from-source.sh') +]) { +test(`build-only ${installer} preserves source selection and packages without dependency installation`, t => { + const root = temp(t); + const source = join(root, 'source'); + const git = (...args) => execFileSync('git', args, { encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'] }).trim(); + git('init', source); + git('-C', source, 'config', 'user.name', 'Fixture'); + git('-C', source, 'config', 'user.email', 'fixture@example.invalid'); + writeFileSync(join(source, 'pom.xml'), 'fixture'); + git('-C', source, 'add', 'pom.xml'); + git('-C', source, 'commit', '-m', 'initial'); + const sha = git('-C', source, 'rev-parse', 'HEAD'); + const bin = join(root, 'bin'); + mkdirSync(bin); + writeFileSync(join(bin, 'curl'), '#!/bin/bash\nexit 0\n', { mode: 0o755 }); + writeFileSync(join(bin, 'mvn'), '#!/bin/bash\nprintf "%s\\n" "$@" > "$BUILD_ARGS"\nprintf "%s" "$MAVEN_ARGS" > "$BUILD_REPO"\n', { mode: 0o755 }); + const env = { ...process.env, SERVER_REPOSITORY: 'example/server', SERVER_FETCH_REF: sha, + PATH: `${bin}:${process.env.PATH}`, BUILD_ARGS: join(root, 'args'), BUILD_REPO: join(root, 'repo'), + MAVEN_ARGS: `-Dmaven.repo.local=${root}/isolated-m2`, GIT_CONFIG_COUNT: '1', + GIT_CONFIG_KEY_0: `url.file://${source}.insteadOf`, GIT_CONFIG_VALUE_0: 'https://github.com/example/server.git' }; + const run = args => spawnSync('bash', [installer, ...args], + { cwd: root, env, encoding: 'utf8' }); + assert.notEqual(run([sha, '--unknown']).status, 0); + assert.equal(existsSync(join(root, 'hugegraph')), false); + const result = run([sha, '--build-only']); + assert.equal(result.status, 0, result.stderr); + assert.deepEqual(readFileSync(env.BUILD_ARGS, 'utf8').trim().split('\n'), + ['package', '-DskipTests', '-Dmaven.javadoc.skip=true', '-ntp']); + assert.equal(readFileSync(env.BUILD_REPO, 'utf8'), env.MAVEN_ARGS); + // No archive exists: success proves build-only did not attempt extraction/startup. + assert.equal(readdirSync(root).some(name => name.startsWith('hugegraph-servers.')), false); +}); +} + +function serverFixture(fixture) { + mkdirSync(join(fixture, 'conf/graphs'), { recursive: true }); + mkdirSync(join(fixture, 'bin')); + writeFileSync(join(fixture, 'conf/graphs/hugegraph.properties'), + 'gremlin.graph=org.apache.hugegraph.HugeFactory\n'); + writeFileSync(join(fixture, 'conf/rest-server.properties'), + 'restserver.url=http://127.0.0.1:8080\nrpc.server_port=8091\n#auth.authenticator=none\n#auth.admin_pa=none\n'); + writeFileSync(join(fixture, 'conf/gremlin-server.yaml'), '#port: 8182\n'); + writeFileSync(join(fixture, 'bin/init-store.sh'), + '#!/bin/bash\nset -e\nshopt -s dotglob\nread -r password\n[[ "$password" == pa ]]\n' + + 'for file in conf/graphs/*.properties; do\n' + + ' name=$(basename "$file" .properties)\n' + + ' if [[ ! "$name" =~ ^[a-zA-Z][a-zA-Z0-9_]{0,47}$ ]]; then\n' + + ' echo "Invalid graph name: $name" >&2\n exit 1\n fi\n' + + 'done\nprintf "%s" "$JAVA_HOME" > initialized-with-jvm\n', { mode: 0o755 }); + writeFileSync(join(fixture, 'bin/start-hugegraph.sh'), + '#!/bin/bash\nprintf "%s" "$JAVA_HOME" > started-with-jvm\n', { mode: 0o755 }); +} + +function runStarter(root, entries) { + const archive = join(root, 'server.tar.gz'); + // Keep metadata entries as files on macOS too, matching the Linux CI extractor. + const env = { ...process.env, COPYFILE_DISABLE: '1', RUNNER_TEMP: root, JAVA_HOME: '/fixture/java11' }; + const bin = join(root, 'tar-bin'); + mkdirSync(bin); + writeFileSync(join(bin, 'curl'), '#!/bin/bash\nexit 0\n', { mode: 0o755 }); + env.PATH = `${bin}:${env.PATH}`; + if (process.platform === 'darwin') { + writeFileSync(join(bin, 'tar'), + '#!/bin/bash\n[[ "$1" == -* ]] || set -- "-$1" "${@:2}"\n' + + 'exec /usr/bin/tar --no-mac-metadata "$@"\n', { mode: 0o755 }); + env.PATH = `${bin}:${env.PATH}`; + } + execFileSync('tar', ['czf', archive, '-C', root, ...entries], { env }); + const result = spawnSync('bash', [join(__dirname, 'start-hugegraph-servers.sh'), archive], + { env, encoding: 'utf8' }); + const serverRoot = join(root, readdirSync(root).find(name => name.startsWith('hugegraph-servers.'))); + return { result, serverRoot }; +} + +for (const layout of [ + { name: 'direct candidate', top: 'apache-hugegraph-fixture', server: 'apache-hugegraph-fixture' }, + { name: 'official aggregate', top: 'apache-hugegraph-incubating-1.7.0', + server: 'apache-hugegraph-incubating-1.7.0/apache-hugegraph-server-incubating-1.7.0' } +]) { + test(`shared starter launches ${layout.name} HTTP/auth and HTTPS with caller JVM`, t => { + const root = temp(t); + serverFixture(join(root, layout.server)); + if (layout.server !== layout.top) { + // Sibling components in the real release are not server candidates. + mkdirSync(join(root, layout.top, 'apache-hugegraph-pd-incubating-1.7.0')); + mkdirSync(join(root, layout.top, 'apache-hugegraph-store-incubating-1.7.0')); + } + const { result, serverRoot } = runStarter(root, [layout.top]); + assert.equal(result.status, 0, result.stderr); + for (const dir of [layout.server, 'hugegraph_https']) { + const deployed = join(serverRoot, dir); + assert.equal(readFileSync(join(deployed, 'started-with-jvm'), 'utf8'), '/fixture/java11'); + assert.equal(readFileSync(join(deployed, 'initialized-with-jvm'), 'utf8'), '/fixture/java11'); + assert.match(readFileSync(join(deployed, 'conf/graphs/hugegraph.properties'), 'utf8'), /HugeFactoryAuthProxy/); + const rest = readFileSync(join(deployed, 'conf/rest-server.properties'), 'utf8'); + assert.match(rest, /auth.authenticator=org.apache.hugegraph.auth.StandardAuthenticator/); + assert.match(rest, /auth.admin_pa=pa/); + assert.match(rest, /batch.max_vertices_per_batch=500/); + assert.match(rest, /batch.max_edges_per_batch=500/); + assert.match(rest, dir === 'hugegraph_https' ? /rpc.server_port=8092/ : /rpc.server_port=8091/); + assert.match(rest, dir === 'hugegraph_https' ? /https:\/\/127.0.0.1:8443/ : /http:\/\/127.0.0.1:8080/); + } + assert.match(readFileSync(join(serverRoot, 'hugegraph_https/conf/gremlin-server.yaml'), 'utf8'), /^port: 8282/m); + assert.equal(existsSync(join(serverRoot, 'hugegraph_https/apache-hugegraph-pd-incubating-1.7.0')), false); + }); +} + +test('shared starter rejects multiple runnable server distributions before starting either', t => { + const root = temp(t); + const top = 'apache-hugegraph-ambiguous'; + const paths = [top, `${top}/apache-hugegraph-server-nested`]; + paths.forEach(path => serverFixture(join(root, path))); + const { result, serverRoot } = runStarter(root, [top]); + assert.notEqual(result.status, 0); + assert.match(result.stderr, /Expected exactly one runnable server distribution, found 2/); + for (const path of paths) { + assert.equal(existsSync(join(serverRoot, path, 'initialized-with-jvm')), false); + assert.equal(existsSync(join(serverRoot, path, 'started-with-jvm')), false); + } + assert.equal(existsSync(join(serverRoot, 'hugegraph_https')), false); +}); + +test('shared starter rejects an incomplete server before initialization', t => { + const root = temp(t); + const top = 'apache-hugegraph-incomplete'; + serverFixture(join(root, top)); + rmSync(join(root, top, 'conf/graphs/hugegraph.properties')); + const { result, serverRoot } = runStarter(root, [top]); + assert.notEqual(result.status, 0); + assert.match(result.stderr, /Expected exactly one runnable server distribution, found 0/); + assert.equal(existsSync(join(serverRoot, top, 'initialized-with-jvm')), false); + assert.equal(existsSync(join(serverRoot, top, 'started-with-jvm')), false); + assert.equal(existsSync(join(serverRoot, 'hugegraph_https')), false); +}); + + +test('shared starter removes AppleDouble sidecars without replacing released content', t => { + const root = temp(t); + const top = 'apache-hugegraph-incubating-1.7.0'; + const server = `${top}/apache-hugegraph-server-incubating-1.7.0`; + serverFixture(join(root, server)); + mkdirSync(join(root, server, 'lib')); + // GNU tar retains these macOS sidecars; ConfigUtil scans every *.properties. + writeFileSync(join(root, server, 'conf/graphs/._hugegraph.properties'), 'metadata sidecar'); + writeFileSync(join(root, server, 'lib/._released.jar'), 'metadata sidecar'); + writeFileSync(join(root, server, 'lib/released.jar'), 'original released binary'); + writeFileSync(join(root, server, 'conf/.keep'), 'ordinary dotfile'); + writeFileSync(join(root, '._outside.properties'), 'outside the extracted distribution'); + const { result, serverRoot } = runStarter(root, [top]); + assert.equal(result.status, 0, result.stderr); + for (const dir of [server, 'hugegraph_https']) { + const deployed = join(serverRoot, dir); + assert.equal(readFileSync(join(deployed, 'started-with-jvm'), 'utf8'), '/fixture/java11'); + assert.equal(existsSync(join(deployed, 'conf/graphs/._hugegraph.properties')), false); + assert.equal(existsSync(join(deployed, 'lib/._released.jar')), false); + assert.equal(readFileSync(join(deployed, 'lib/released.jar'), 'utf8'), 'original released binary'); + assert.equal(readFileSync(join(deployed, 'conf/.keep'), 'utf8'), 'ordinary dotfile'); + assert.match(readFileSync(join(deployed, 'conf/graphs/hugegraph.properties'), 'utf8'), /HugeFactoryAuthProxy/); + } + assert.equal(readFileSync(join(root, '._outside.properties'), 'utf8'), 'outside the extracted distribution'); +}); + +test('shared starter still rejects invalid non-metadata graph names', t => { + const root = temp(t); + const top = 'apache-hugegraph-invalid-graph'; + serverFixture(join(root, top)); + writeFileSync(join(root, top, 'conf/graphs/9invalid.properties'), 'invalid graph configuration'); + const { result, serverRoot } = runStarter(root, [top]); + assert.notEqual(result.status, 0); + assert.equal(existsSync(join(serverRoot, top, 'conf/graphs/9invalid.properties')), true); + assert.equal(existsSync(join(serverRoot, top, 'started-with-jvm')), false); +}); + + +test('Loader reports ambiguous archives before shared startup', t => { + const root = temp(t); + const cache = join(root, 'cache'); + mkdirSync(cache); + writeFileSync(join(cache, 'apache-hugegraph-cached.tar.gz'), 'cached archive'); + cacheStamp(cache, 'apache/hugegraph', 'a'.repeat(40), 'apache-hugegraph-cached.tar.gz'); + writeFileSync(join(root, 'apache-hugegraph-leftover.tar.gz'), 'leftover archive'); + const installer = join(__dirname, '../../../hugegraph-loader/assembly/travis/install-hugegraph-from-source.sh'); + const result = spawnSync('bash', [installer, 'a'.repeat(40)], { + cwd: root, env: { ...process.env, SERVER_CACHE_DIR: cache }, encoding: 'utf8' + }); + assert.equal(result.status, 1); + assert.match(result.stderr, /Expected exactly one server archive, found:/); + assert.match(result.stderr, /apache-hugegraph-cached\.tar\.gz/); + assert.match(result.stderr, /apache-hugegraph-leftover\.tar\.gz/); + assert.equal(readdirSync(root).some(name => name.startsWith('hugegraph-servers.')), false); +}); + +test('Loader retains its source cache while using the selected repository and shared startup', t => { + const root = temp(t); + const source = join(root, 'source'); + const git = (...args) => execFileSync('git', args, { encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'] }).trim(); + git('init', source); + git('-C', source, 'config', 'user.name', 'Fixture'); + git('-C', source, 'config', 'user.email', 'fixture@example.invalid'); + writeFileSync(join(source, 'pom.xml'), 'fixture'); + git('-C', source, 'add', 'pom.xml'); + git('-C', source, 'commit', '-m', 'initial'); + const sha = git('-C', source, 'rev-parse', 'HEAD'); + const top = 'apache-hugegraph-loader-fixture'; + writeFileSync(join(source, 'pom.xml'), 'another commit'); + git('-C', source, 'commit', '-am', 'another commit'); + const anotherSha = git('-C', source, 'rev-parse', 'HEAD'); + serverFixture(join(root, top)); + const archive = join(root, 'fixture.tar.gz'); + execFileSync('tar', ['czf', archive, '-C', root, top], + { env: { ...process.env, COPYFILE_DISABLE: '1' } }); + const bin = join(root, 'bin'); + mkdirSync(bin); + writeFileSync(join(bin, 'curl'), '#!/bin/bash\nexit 0\n', { mode: 0o755 }); + writeFileSync(join(bin, 'mvn'), '#!/bin/bash\nprintf "%s\\n" "$@" > "$BUILD_ARGS"\n' + + 'mkdir -p hugegraph-server\ncp "$FIXTURE_ARCHIVE" hugegraph-server/apache-hugegraph-fixture.tar.gz\n' + + 'if [[ "$DUPLICATE_ARCHIVE" == true ]]; then\n' + + ' cp "$FIXTURE_ARCHIVE" hugegraph-server/apache-hugegraph-other.tar.gz\nfi\n', + { mode: 0o755 }); + const env = { ...process.env, SERVER_REPOSITORY: 'example/server', SERVER_FETCH_REF: sha, + SERVER_CACHE_DIR: join(root, 'cache'), FIXTURE_ARCHIVE: archive, BUILD_ARGS: join(root, 'args'), + PATH: `${bin}:${process.env.PATH}`, RUNNER_TEMP: root, JAVA_HOME: '/fixture/java11', + GIT_CONFIG_COUNT: '1', GIT_CONFIG_KEY_0: `url.file://${source}.insteadOf`, + GIT_CONFIG_VALUE_0: 'https://github.com/example/server.git' }; + const installer = join(__dirname, '../../../hugegraph-loader/assembly/travis/install-hugegraph-from-source.sh'); + for (const failure of [ + { name: 'checkout', extra: { SERVER_FETCH_REF: anotherSha }, error: /Server checkout mismatch/ }, + { name: 'build', extra: { DUPLICATE_ARCHIVE: 'true' }, error: /Expected exactly one built server archive/ } + ]) { + const cwd = join(root, `run-failed-${failure.name}`); + const cache = join(root, `cache-failed-${failure.name}`); + mkdirSync(cwd); + const result = spawnSync('bash', [installer, sha], { + cwd, env: { ...env, ...failure.extra, SERVER_CACHE_DIR: cache }, encoding: 'utf8' + }); + assert.equal(result.status, 1); + assert.match(result.stderr, failure.error); + assert.deepEqual(readdirSync(cache), []); + assert.equal(readdirSync(root).some(name => name.startsWith('hugegraph-servers.')), false); + } + for (const pass of ['build', 'cache']) { + const cwd = join(root, `run-${pass}`); + mkdirSync(cwd); + const result = spawnSync('bash', [installer, pass === 'cache' ? sha.toUpperCase() : sha], + { cwd, env, encoding: 'utf8' }); + assert.equal(result.status, 0, result.stderr); + assert.deepEqual(readFileSync(env.BUILD_ARGS, 'utf8').trim().split('\n'), + ['package', '-DskipTests', '-Dmaven.javadoc.skip=true', '-ntp']); + assert.equal(existsSync(join(cwd, 'hugegraph')), false); + if (pass === 'build') { + assert.deepEqual(readdirSync(env.SERVER_CACHE_DIR).sort(), + ['apache-hugegraph-fixture.tar.gz', 'server-provenance']); + assert.equal(readFileSync(join(env.SERVER_CACHE_DIR, 'server-provenance'), 'utf8'), + `example/server\n${sha}\napache-hugegraph-fixture.tar.gz\n` + + `${createHash('sha256').update(readFileSync(archive)).digest('hex')}\n`); + // A cache hit must not execute Maven or reach the source repository again. + writeFileSync(join(bin, 'mvn'), '#!/bin/bash\nexit 91\n', { mode: 0o755 }); + rmSync(source, { recursive: true, force: true }); + } + } + const otherCwd = join(root, 'run-other-commit'); + mkdirSync(otherCwd); + const other = spawnSync('bash', [installer, anotherSha], { cwd: otherCwd, env, encoding: 'utf8' }); + assert.equal(other.status, 1); + assert.match(other.stderr, /source or archive provenance mismatch/); + assert.deepEqual(readdirSync(otherCwd), []); + const deployments = readdirSync(root).filter(name => name.startsWith('hugegraph-servers.')); + assert.equal(deployments.length, 2); + for (const deployment of deployments) { + for (const dir of [top, 'hugegraph_https']) { + assert.equal(readFileSync(join(root, deployment, dir, 'started-with-jvm'), 'utf8'), '/fixture/java11'); + } + } +}); + +for (const scenario of [ + { name: 'another commit', commit: 'b'.repeat(40), error: /provenance mismatch/ }, + { name: 'another repository', repository: 'example/other', error: /provenance mismatch/ }, + { name: 'a missing stamp', mutate: cache => rmSync(join(cache, 'server-provenance')), + error: /missing provenance/ }, + { name: 'a tampered archive', mutate: cache => writeFileSync(join(cache, 'apache-hugegraph-fixture.tar.gz'), + 'modified archive'), error: /provenance mismatch/ }, + { name: 'multiple cached archives', mutate: cache => writeFileSync(join(cache, 'apache-hugegraph-other.tar.gz'), + 'another archive'), error: /exactly one regular server archive/ }, + { name: 'extra provenance data', mutate: cache => writeFileSync(join(cache, 'server-provenance'), + readFileSync(join(cache, 'server-provenance'), 'utf8') + 'unexpected'), error: /provenance mismatch/ }, + { name: 'binary provenance data', mutate: cache => writeFileSync(join(cache, 'server-provenance'), + readFileSync(join(cache, 'server-provenance'), 'utf8').replace('example/server', 'example/\0server')), + error: /provenance mismatch/ }, + { name: 'executable metadata', mutate: cache => writeFileSync(join(cache, 'server-provenance'), + '$(touch should-not-exist)\n' + 'a'.repeat(40) + '\napache-hugegraph-fixture.tar.gz\n' + '0'.repeat(64) + '\n'), + error: /provenance mismatch/ } +]) { + test(`Loader rejects ${scenario.name} without altering the custom cache or starting a server`, t => { + const root = temp(t); + const cache = join(root, 'cache'); + mkdirSync(cache); + writeFileSync(join(cache, 'apache-hugegraph-fixture.tar.gz'), 'original archive'); + writeFileSync(join(cache, 'unrelated-file'), 'preserve this file'); + cacheStamp(cache, 'example/server', 'a'.repeat(40), 'apache-hugegraph-fixture.tar.gz'); + if (scenario.mutate) scenario.mutate(cache); + const files = readdirSync(cache).map(name => [name, readFileSync(join(cache, name), 'utf8')]); + const installer = join(__dirname, '../../../hugegraph-loader/assembly/travis/install-hugegraph-from-source.sh'); + const result = spawnSync('bash', [installer, scenario.commit || 'a'.repeat(40)], { + cwd: root, env: { ...process.env, SERVER_CACHE_DIR: cache, + SERVER_REPOSITORY: scenario.repository || 'example/server' }, encoding: 'utf8' + }); + assert.equal(result.status, 1); + assert.match(result.stderr, scenario.error); + assert.match(result.stderr, /Use a fresh or separate SERVER_CACHE_DIR/); + assert.deepEqual(readdirSync(cache).map(name => [name, readFileSync(join(cache, name), 'utf8')]), files); + assert.equal(existsSync(join(root, 'hugegraph')), false); + assert.equal(existsSync(join(root, 'should-not-exist')), false); + assert.equal(readdirSync(root).some(name => name.startsWith('hugegraph-servers.')), false); + }); +} diff --git a/hugegraph-client/assembly/travis/start-hugegraph-servers.sh b/hugegraph-client/assembly/travis/start-hugegraph-servers.sh new file mode 100644 index 000000000..9d9ba5c98 --- /dev/null +++ b/hugegraph-client/assembly/travis/start-hugegraph-servers.sh @@ -0,0 +1,72 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with this +# work for additional information regarding copyright ownership. The ASF +# licenses this file to You under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +set -euo pipefail +source "$(cd "$(dirname "$0")/../../.." && pwd)/.github/actions/setup-hugegraph-server/service-wait.sh" + +if [[ $# -ne 1 || ! -f "$1" ]]; then + echo "Usage: start-hugegraph-servers.sh " >&2 + exit 1 +fi + +# Both baselines use the same authenticated HTTP and HTTPS test configuration. +# JAVA_HOME belongs to the server process only; callers retain their client JVM. +SERVER_ROOT=$(mktemp -d "${RUNNER_TEMP:-${TMPDIR:-/tmp}}/hugegraph-servers.XXXXXX") +tar zxf "$1" -C "$SERVER_ROOT" +# macOS-built release archives include AppleDouble sidecars. GNU tar retains +# them, and graph discovery mistakes ._hugegraph.properties for a graph. +find "$SERVER_ROOT" -type f -name '._*' -delete +# Candidate archives contain the server directly; official aggregate releases +# put it one level below the distribution root, alongside PD and Store. +SERVER_DIRS=() +for dir in "$SERVER_ROOT"/apache-hugegraph-*/ "$SERVER_ROOT"/apache-hugegraph-*/*/; do + if [[ -f "$dir/conf/graphs/hugegraph.properties" && + -f "$dir/conf/rest-server.properties" && + -f "$dir/conf/gremlin-server.yaml" && + -x "$dir/bin/init-store.sh" && -x "$dir/bin/start-hugegraph.sh" ]]; then + SERVER_DIRS+=("$dir") + fi +done +if [[ ${#SERVER_DIRS[@]} -ne 1 ]]; then + printf 'Expected exactly one runnable server distribution, found %s\n' "${#SERVER_DIRS[@]}" >&2 + exit 1 +fi +cp -r "${SERVER_DIRS[0]}" "$SERVER_ROOT/hugegraph_https" +printf 'Server directories: %s\n' "$SERVER_ROOT" + +start_server() ( + cd "$1" + sed -i.bak 's|gremlin.graph=org.apache.hugegraph.HugeFactory|gremlin.graph=org.apache.hugegraph.auth.HugeFactoryAuthProxy|' conf/graphs/hugegraph.properties + sed -i.bak 's|#auth.authenticator=.*|auth.authenticator=org.apache.hugegraph.auth.StandardAuthenticator|' conf/rest-server.properties + sed -i.bak 's|#auth.admin_pa=.*|auth.admin_pa=pa|' conf/rest-server.properties + # Boundary tests submit 501 edges / 1000 vertices and expect rejection. + # Pin their fixture limit instead of depending on changing server defaults. + printf '\nbatch.max_vertices_per_batch=500\nbatch.max_edges_per_batch=500\n' >> conf/rest-server.properties + printf 'pa\n' | bin/init-store.sh + bin/start-hugegraph.sh +) + +start_server "${SERVER_DIRS[0]}" +fixture_wait http://127.0.0.1:8080 +HTTPS_DIR="$SERVER_ROOT/hugegraph_https" +sed -i.bak 's?http://127.0.0.1:8080?https://127.0.0.1:8443?g' "$HTTPS_DIR/conf/rest-server.properties" +sed -i.bak 's/rpc.server_port=8091/rpc.server_port=8092/g' "$HTTPS_DIR/conf/rest-server.properties" +sed -i.bak 's/#port: 8182/port: 8282/g' "$HTTPS_DIR/conf/gremlin-server.yaml" +printf '\ngremlinserver.url=http://127.0.0.1:8282\n' >> "$HTTPS_DIR/conf/rest-server.properties" +start_server "$HTTPS_DIR" + +fixture_wait https://127.0.0.1:8443 diff --git a/hugegraph-hubble/hubble-dist/assembly/travis/download-hugegraph.sh b/hugegraph-hubble/hubble-dist/assembly/travis/download-hugegraph.sh index df4abd4e2..b69391c49 100755 --- a/hugegraph-hubble/hubble-dist/assembly/travis/download-hugegraph.sh +++ b/hugegraph-hubble/hubble-dist/assembly/travis/download-hugegraph.sh @@ -23,39 +23,6 @@ if [[ $# -lt 1 || $# -gt 2 ]]; then exit 1 fi -COMMIT_ID=$1 -COMMIT_REF=${2:-} -HUGEGRAPH_GIT_URL="https://github.com/apache/hugegraph.git" -GIT_DIR=hugegraph -CACHE_DIR="${HOME}/hugegraph-cache-${COMMIT_ID}" - -mkdir -p "${CACHE_DIR}" -CACHED_TARBALL=$(find "${CACHE_DIR}" -maxdepth 1 \ - -name "apache-hugegraph-*.tar.gz" -print -quit) - -if [[ -f "${CACHED_TARBALL}" ]]; then - echo "Found HugeGraph server tarball cached for commit ${COMMIT_ID}." - cp "${CACHED_TARBALL}" ./ - exit 0 -fi - -# download code and compile -git clone --depth 150 $HUGEGRAPH_GIT_URL $GIT_DIR -cd "${GIT_DIR}" -if [[ -n "${COMMIT_REF}" ]]; then - git fetch --depth 1 origin "${COMMIT_REF}" -fi -git checkout "${COMMIT_ID}" -ACTUAL_COMMIT_ID=$(git rev-parse HEAD) -if [[ "${ACTUAL_COMMIT_ID}" != "${COMMIT_ID}" ]]; then - echo "HugeGraph checkout mismatch: expected ${COMMIT_ID}, got ${ACTUAL_COMMIT_ID}" >&2 - exit 1 -fi -mvn package -DskipTests -Dmaven.javadoc.skip=true -ntp - -cd hugegraph-server -TAR=$(echo apache-hugegraph-*.tar.gz) -cp apache-hugegraph-*.tar.gz ../../ -cd ../../ -rm -rf "${GIT_DIR}" -cp apache-hugegraph-*.tar.gz "${CACHE_DIR}"/ +FIXTURE_COMMIT=$1 +FIXTURE_REF=${2:-$1} +source "$(cd "$(dirname "$0")/../../../.." && pwd)/.github/actions/setup-hugegraph-server/fixture.sh" diff --git a/hugegraph-hubble/hubble-dist/assembly/travis/hugegraph-server1/install-hugegraph.sh b/hugegraph-hubble/hubble-dist/assembly/travis/hugegraph-server1/install-hugegraph.sh index dbc7f8e77..10ca1b925 100755 --- a/hugegraph-hubble/hubble-dist/assembly/travis/hugegraph-server1/install-hugegraph.sh +++ b/hugegraph-hubble/hubble-dist/assembly/travis/hugegraph-server1/install-hugegraph.sh @@ -18,6 +18,7 @@ set -ev SERVER_CONFIG_DIR=$(dirname "$0") +source "$(cd "$SERVER_CONFIG_DIR/../../../../.." && pwd)/.github/actions/setup-hugegraph-server/service-wait.sh" SERVER_PARENT_DIR="hugegraph-server1" mkdir ${SERVER_PARENT_DIR} @@ -37,3 +38,5 @@ cd "${SERVER_DIR}" && pwd echo -e "pa" | bin/init-store.sh || exit 1 bin/start-hugegraph.sh || exit 1 + +fixture_wait http://127.0.0.1:8080 diff --git a/hugegraph-hubble/hubble-dist/assembly/travis/hugegraph-server2/install-hugegraph.sh b/hugegraph-hubble/hubble-dist/assembly/travis/hugegraph-server2/install-hugegraph.sh index 71ae5c4cf..6ba4bcacd 100755 --- a/hugegraph-hubble/hubble-dist/assembly/travis/hugegraph-server2/install-hugegraph.sh +++ b/hugegraph-hubble/hubble-dist/assembly/travis/hugegraph-server2/install-hugegraph.sh @@ -18,6 +18,7 @@ set -ev SERVER_CONFIG_DIR=$(dirname "$0") +source "$(cd "$SERVER_CONFIG_DIR/../../../../.." && pwd)/.github/actions/setup-hugegraph-server/service-wait.sh" SERVER_PARENT_DIR="hugegraph-server2" mkdir ${SERVER_PARENT_DIR} @@ -37,3 +38,5 @@ cd "${SERVER_DIR}" && pwd echo -e "pa" | bin/init-store.sh || exit 1 bin/start-hugegraph.sh || exit 1 + +fixture_wait http://127.0.0.1:8081 diff --git a/hugegraph-loader/assembly/travis/install-hadoop.sh b/hugegraph-loader/assembly/travis/install-hadoop.sh index e439411c7..54087f9a7 100755 --- a/hugegraph-loader/assembly/travis/install-hadoop.sh +++ b/hugegraph-loader/assembly/travis/install-hadoop.sh @@ -15,29 +15,36 @@ # License for the specific language governing permissions and limitations # under the License. # -set -ev +set -euo pipefail -# Upgrade stable version to 3.3.6 HADOOP_VERSION="3.3.6" HADOOP_TARBALL="hadoop-${HADOOP_VERSION}.tar.gz" -HADOOP_HOME="/usr/local/hadoop" +HADOOP_HOME=${HADOOP_HOME:-/usr/local/hadoop} HADOOP_TARBALL_PATH="${HOME}/${HADOOP_TARBALL}" - -if [[ ! -d "${HADOOP_HOME}" ]]; then - if [[ ! -f "${HADOOP_TARBALL_PATH}" ]]; then - echo "Downloading Hadoop ${HADOOP_VERSION}..." - wget -O "${HADOOP_TARBALL_PATH}" "https://archive.apache.org/dist/hadoop/common/hadoop-${HADOOP_VERSION}/${HADOOP_TARBALL}" - else - echo "Using cached Hadoop tarball at ${HADOOP_TARBALL_PATH}" - fi - echo "Extracting Hadoop to ${HADOOP_HOME}..." - sudo tar -zxf "${HADOOP_TARBALL_PATH}" -C /usr/local - cd /usr/local - sudo mv "hadoop-${HADOOP_VERSION}" hadoop - sudo chown -R "$(whoami):$(whoami)" "${HADOOP_HOME}" -else - echo "Hadoop already installed at ${HADOOP_HOME}, skipping download and extraction." +# Official archive.apache.org Hadoop 3.3.6 .sha512 release checksum. +HADOOP_SHA512=de3eaca2e0517e4b569a88b63c89fae19cb8ac6c01ff990f1ff8f0cc0f3128c8e8a23db01577ca562a0e0bb1b4a3889f8c74384e609cd55e537aada3dcaa9f8a +hadoop_verify() { + [[ -f "$HADOOP_TARBALL_PATH" ]] && + [[ "$(shasum -a 512 "$HADOOP_TARBALL_PATH" | cut -d ' ' -f 1)" == "$HADOOP_SHA512" ]] +} +if ! hadoop_verify; then + rm -f "$HADOOP_TARBALL_PATH" + for attempt in 1 2; do + echo "Downloading Hadoop ${HADOOP_VERSION} (attempt $attempt)..." + if curl --fail --location --connect-timeout 30 --max-time 900 \ + --speed-time 60 --speed-limit 1024 \ + --output "$HADOOP_TARBALL_PATH" \ + "https://archive.apache.org/dist/hadoop/common/hadoop-${HADOOP_VERSION}/${HADOOP_TARBALL}" && hadoop_verify; then + break + fi + rm -f "$HADOOP_TARBALL_PATH" + done fi +hadoop_verify || { echo 'Hadoop archive download/checksum failed' >&2; exit 1; } +# Extract a verified archive even on cache hits; no unverified installed directory reuse. +sudo mkdir -p "$HADOOP_HOME" +sudo tar -zxf "$HADOOP_TARBALL_PATH" --strip-components=1 -C "$HADOOP_HOME" +sudo chown -R "$(whoami):$(id -gn)" "$HADOOP_HOME" cd "${HADOOP_HOME}" pwd @@ -59,7 +66,9 @@ if ! grep -qxF "export PATH=\$PATH:${HADOOP_HOME}/bin:${HADOOP_HOME}/sbin" ~/.ba echo "export PATH=\$PATH:${HADOOP_HOME}/bin:${HADOOP_HOME}/sbin" >> ~/.bashrc fi -source ~/.bashrc +export HADOOP_HOME +export HADOOP_COMMON_LIB_NATIVE_DIR="${HADOOP_HOME}/lib/native" +export PATH="${PATH}:${HADOOP_HOME}/bin:${HADOOP_HOME}/sbin" if [[ ! -f etc/hadoop/core-site.xml ]] || ! grep -q "hdfs://localhost:8020" etc/hadoop/core-site.xml; then sudo tee etc/hadoop/core-site.xml > /dev/null <0 for b in json.load(sys.stdin)["beans"]))'; then + exit 0 + fi + sleep 2 +done +echo 'Hadoop readiness timed out' >&2 +exit 1 diff --git a/hugegraph-loader/assembly/travis/install-hugegraph-from-source.sh b/hugegraph-loader/assembly/travis/install-hugegraph-from-source.sh index 8884c1268..bc8f6bad2 100755 --- a/hugegraph-loader/assembly/travis/install-hugegraph-from-source.sh +++ b/hugegraph-loader/assembly/travis/install-hugegraph-from-source.sh @@ -15,61 +15,94 @@ # License for the specific language governing permissions and limitations # under the License. # -set -ev +set -euo pipefail if [[ $# -ne 1 ]]; then echo "Must input an existing commit id of hugegraph server" && exit 1 fi +if [[ "${FIXTURE_MODE:-standalone}" == start ]]; then + FIXTURE_COMMIT=$1 + ROOT_DIR=$(cd "$(dirname "$0")/../../.." && pwd) + source "$ROOT_DIR/.github/actions/setup-hugegraph-server/fixture.sh" + ARCHIVE=$(python3 "$ROOT_DIR/.github/actions/setup-hugegraph-server/manifest.py" name) + exec bash "$ROOT_DIR/hugegraph-client/assembly/travis/start-hugegraph-servers.sh" "$ARCHIVE" +fi COMMIT_ID=$1 -HUGEGRAPH_GIT_URL="https://github.com/apache/hugegraph.git" -CACHE_DIR="${HOME}/hugegraph-cache-${COMMIT_ID}" +SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd) +SHARED_DIR="$SCRIPT_DIR/../../../hugegraph-client/assembly/travis" +[[ "$COMMIT_ID" =~ ^[0-9a-fA-F]{40}$ ]] || { echo "Expected a full commit SHA" >&2; exit 1; } +CACHE_DIR=${SERVER_CACHE_DIR:-"${HOME}/hugegraph-cache-${COMMIT_ID}"} +COMMIT_ID=$(printf '%s' "$COMMIT_ID" | tr '[:upper:]' '[:lower:]') +SERVER_REPOSITORY=${SERVER_REPOSITORY:-apache/hugegraph} +[[ "$SERVER_REPOSITORY" =~ ^[a-zA-Z0-9][a-zA-Z0-9_.-]*/[a-zA-Z0-9][a-zA-Z0-9_.-]*$ ]] || { + echo "Expected a GitHub owner/repository" >&2 + exit 1 +} +PROVENANCE_FILE="${CACHE_DIR}/server-provenance" -mkdir -p "${CACHE_DIR}" +cache_error() { + printf 'Cannot reuse server cache %s: %s. Use a fresh or separate SERVER_CACHE_DIR.\n' "$CACHE_DIR" "$1" >&2 + exit 1 +} -CACHED_TARBALL=$(find "${CACHE_DIR}" -maxdepth 1 -name "apache-hugegraph-*.tar.gz" -print -quit) +archive_hash() { + if command -v sha256sum >/dev/null 2>&1; then + sha256sum "$1" + else + shasum -a 256 "$1" + fi +} -if [[ -f "${CACHED_TARBALL}" ]]; then +mkdir -p "${CACHE_DIR}" +shopt -s nullglob dotglob +CACHE_ENTRIES=("${CACHE_DIR}"/*) +shopt -u dotglob +CACHED_ARCHIVES=("${CACHE_DIR}"/apache-hugegraph-*.tar.gz) + +if [[ ${#CACHE_ENTRIES[@]} -ne 0 ]]; then + # Legacy archives have no verified source identity, even in SHA-specific directories. + [[ -f "$PROVENANCE_FILE" && ! -L "$PROVENANCE_FILE" ]] || cache_error "missing provenance" + [[ ${#CACHED_ARCHIVES[@]} -eq 1 && -f "${CACHED_ARCHIVES[0]}" && ! -L "${CACHED_ARCHIVES[0]}" ]] || + cache_error "expected exactly one regular server archive" + CACHED_ARCHIVE=${CACHED_ARCHIVES[0]##*/} + [[ "$CACHED_ARCHIVE" =~ ^apache-hugegraph-[a-zA-Z0-9_.-]+\.tar\.gz$ ]] || cache_error "invalid archive name" + ACTUAL_HASH=$(archive_hash "${CACHED_ARCHIVES[0]}") + # Compare exact bytes to canonical metadata; never execute or interpret the stamp. + printf '%s\n' "$SERVER_REPOSITORY" "$COMMIT_ID" "$CACHED_ARCHIVE" "${ACTUAL_HASH%% *}" | + cmp -s "$PROVENANCE_FILE" - || cache_error "source or archive provenance mismatch" echo "Found cached HugeGraph server tarball, skipping build." - cp "${CACHED_TARBALL}" ./ + cp "${CACHED_ARCHIVES[0]}" ./ else echo "Building HugeGraph server from source (commit ${COMMIT_ID})..." - git clone --depth 150 ${HUGEGRAPH_GIT_URL} hugegraph + bash "$SHARED_DIR/checkout-server.sh" "$COMMIT_ID" hugegraph cd hugegraph - git checkout "${COMMIT_ID}" mvn package -DskipTests -Dmaven.javadoc.skip=true -ntp - cd hugegraph-server - mv apache-hugegraph-*.tar.gz ../../ - cd ../../ + cd ../ + BUILD_ARCHIVES=(hugegraph/hugegraph-server/apache-hugegraph-*.tar.gz) + if [[ ${#BUILD_ARCHIVES[@]} -ne 1 || ! -f "${BUILD_ARCHIVES[0]}" ]]; then + printf 'Expected exactly one built server archive, found: %s\n' "${BUILD_ARCHIVES[*]}" >&2 + exit 1 + fi + mv "${BUILD_ARCHIVES[0]}" ./ rm -rf hugegraph - cp apache-hugegraph-*.tar.gz "${CACHE_DIR}"/ fi -tar zxf apache-hugegraph-*.tar.gz - -HTTPS_SERVER_DIR="hugegraph_https" -mkdir ${HTTPS_SERVER_DIR} -cp -r apache-hugegraph-*/. ${HTTPS_SERVER_DIR} -cd "$(find apache-hugegraph-* | head -1)" -# start HugeGraphServer with http protocol -sed -i 's|gremlin.graph=org.apache.hugegraph.HugeFactory|gremlin.graph=org.apache.hugegraph.auth.HugeFactoryAuthProxy|' conf/graphs/hugegraph.properties -sed -i 's|#auth.authenticator=.*|auth.authenticator=org.apache.hugegraph.auth.StandardAuthenticator|' conf/rest-server.properties -sed -i 's|#auth.admin_pa=.*|auth.admin_pa=pa|' conf/rest-server.properties -echo -e "pa" | bin/init-store.sh || exit 1 -bin/start-hugegraph.sh || exit 1 - -cd ../${HTTPS_SERVER_DIR} -REST_SERVER_CONFIG="conf/rest-server.properties" -GREMLIN_SERVER_CONFIG="conf/gremlin-server.yaml" -sed -i "s?http://127.0.0.1:8080?https://127.0.0.1:8443?g" "$REST_SERVER_CONFIG" -sed -i "s/rpc.server_port=8091/rpc.server_port=8092/g" "$REST_SERVER_CONFIG" -sed -i "s/#port: 8182/port: 8282/g" "$GREMLIN_SERVER_CONFIG" -echo "gremlinserver.url=http://127.0.0.1:8282" >> ${REST_SERVER_CONFIG} - -# start HugeGraphServer with https protocol -sed -i 's|gremlin.graph=org.apache.hugegraph.HugeFactory|gremlin.graph=org.apache.hugegraph.auth.HugeFactoryAuthProxy|' conf/graphs/hugegraph.properties -sed -i 's|#auth.authenticator=.*|auth.authenticator=org.apache.hugegraph.auth.StandardAuthenticator|' conf/rest-server.properties -sed -i 's|#auth.admin_pa=.*|auth.admin_pa=pa|' conf/rest-server.properties -echo -e "pa" | bin/init-store.sh || exit 1 -bin/start-hugegraph.sh -cd ../ +ARCHIVES=(apache-hugegraph-*.tar.gz) +if [[ ${#ARCHIVES[@]} -ne 1 || ! -f "${ARCHIVES[0]}" ]]; then + printf 'Expected exactly one server archive, found: %s\n' "${ARCHIVES[*]}" >&2 + exit 1 +fi +if [[ ${#CACHE_ENTRIES[@]} -eq 0 ]]; then + [[ "${ARCHIVES[0]}" =~ ^apache-hugegraph-[a-zA-Z0-9_.-]+\.tar\.gz$ ]] || + cache_error "invalid archive name" + ACTUAL_HASH=$(archive_hash "${ARCHIVES[0]}") + cp "${ARCHIVES[0]}" "${CACHE_DIR}"/ + # Publish provenance only after the verified checkout and unique archive succeed. + STAMP_TEMP=$(mktemp "${CACHE_DIR}/.server-provenance.XXXXXX") + trap 'rm -f "$STAMP_TEMP"' EXIT + printf '%s\n' "$SERVER_REPOSITORY" "$COMMIT_ID" "${ARCHIVES[0]}" "${ACTUAL_HASH%% *}" > "$STAMP_TEMP" + mv "$STAMP_TEMP" "$PROVENANCE_FILE" + trap - EXIT +fi +exec bash "$SHARED_DIR/start-hugegraph-servers.sh" "${ARCHIVES[0]}" diff --git a/hugegraph-spark-connector/assembly/travis/install-hugegraph-from-source.sh b/hugegraph-spark-connector/assembly/travis/install-hugegraph-from-source.sh index 354acd958..6988aa291 100755 --- a/hugegraph-spark-connector/assembly/travis/install-hugegraph-from-source.sh +++ b/hugegraph-spark-connector/assembly/travis/install-hugegraph-from-source.sh @@ -15,48 +15,7 @@ # License for the specific language governing permissions and limitations # under the License. # -set -ev +set -euo pipefail -if [[ $# -ne 1 ]]; then - echo "Must input an existing commit id of hugegraph server" && exit 1 -fi - -COMMIT_ID=$1 -HUGEGRAPH_GIT_URL="https://github.com/apache/hugegraph.git" - -git clone --depth 150 ${HUGEGRAPH_GIT_URL} hugegraph -cd hugegraph -git checkout "${COMMIT_ID}" -mvn package -DskipTests -Dmaven.javadoc.skip=true -ntp -cd hugegraph-server -mv apache-hugegraph-*.tar.gz ../../ -cd ../../ -rm -rf hugegraph -tar zxf apache-hugegraph-*.tar.gz - -HTTPS_SERVER_DIR="hugegraph_https" -mkdir ${HTTPS_SERVER_DIR} -cp -r apache-hugegraph-*/. ${HTTPS_SERVER_DIR} -cd "$(find apache-hugegraph-* | head -1)" -# start HugeGraphServer with http protocol -sed -i 's|gremlin.graph=org.apache.hugegraph.HugeFactory|gremlin.graph=org.apache.hugegraph.auth.HugeFactoryAuthProxy|' conf/graphs/hugegraph.properties -sed -i 's|#auth.authenticator=.*|auth.authenticator=org.apache.hugegraph.auth.StandardAuthenticator|' conf/rest-server.properties -sed -i 's|#auth.admin_pa=.*|auth.admin_pa=pa|' conf/rest-server.properties -echo -e "pa" | bin/init-store.sh || exit 1 -bin/start-hugegraph.sh || exit 1 - -cd ../${HTTPS_SERVER_DIR} -REST_SERVER_CONFIG="conf/rest-server.properties" -GREMLIN_SERVER_CONFIG="conf/gremlin-server.yaml" -sed -i "s?http://127.0.0.1:8080?https://127.0.0.1:8443?g" "$REST_SERVER_CONFIG" -sed -i "s/rpc.server_port=8091/rpc.server_port=8092/g" "$REST_SERVER_CONFIG" -sed -i "s/#port: 8182/port: 8282/g" "$GREMLIN_SERVER_CONFIG" -echo "gremlinserver.url=http://127.0.0.1:8282" >> ${REST_SERVER_CONFIG} - -# start HugeGraphServer with https protocol -sed -i 's|gremlin.graph=org.apache.hugegraph.HugeFactory|gremlin.graph=org.apache.hugegraph.auth.HugeFactoryAuthProxy|' conf/graphs/hugegraph.properties -sed -i 's|#auth.authenticator=.*|auth.authenticator=org.apache.hugegraph.auth.StandardAuthenticator|' conf/rest-server.properties -sed -i 's|#auth.admin_pa=.*|auth.admin_pa=pa|' conf/rest-server.properties -echo -e "pa" | bin/init-store.sh || exit 1 -bin/start-hugegraph.sh -cd ../ +SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd) +exec bash "$SCRIPT_DIR/../../../hugegraph-client/assembly/travis/install-hugegraph-from-source.sh" "$@" diff --git a/hugegraph-tools/assembly/travis/install-hugegraph-from-source.sh b/hugegraph-tools/assembly/travis/install-hugegraph-from-source.sh index 354acd958..6988aa291 100755 --- a/hugegraph-tools/assembly/travis/install-hugegraph-from-source.sh +++ b/hugegraph-tools/assembly/travis/install-hugegraph-from-source.sh @@ -15,48 +15,7 @@ # License for the specific language governing permissions and limitations # under the License. # -set -ev +set -euo pipefail -if [[ $# -ne 1 ]]; then - echo "Must input an existing commit id of hugegraph server" && exit 1 -fi - -COMMIT_ID=$1 -HUGEGRAPH_GIT_URL="https://github.com/apache/hugegraph.git" - -git clone --depth 150 ${HUGEGRAPH_GIT_URL} hugegraph -cd hugegraph -git checkout "${COMMIT_ID}" -mvn package -DskipTests -Dmaven.javadoc.skip=true -ntp -cd hugegraph-server -mv apache-hugegraph-*.tar.gz ../../ -cd ../../ -rm -rf hugegraph -tar zxf apache-hugegraph-*.tar.gz - -HTTPS_SERVER_DIR="hugegraph_https" -mkdir ${HTTPS_SERVER_DIR} -cp -r apache-hugegraph-*/. ${HTTPS_SERVER_DIR} -cd "$(find apache-hugegraph-* | head -1)" -# start HugeGraphServer with http protocol -sed -i 's|gremlin.graph=org.apache.hugegraph.HugeFactory|gremlin.graph=org.apache.hugegraph.auth.HugeFactoryAuthProxy|' conf/graphs/hugegraph.properties -sed -i 's|#auth.authenticator=.*|auth.authenticator=org.apache.hugegraph.auth.StandardAuthenticator|' conf/rest-server.properties -sed -i 's|#auth.admin_pa=.*|auth.admin_pa=pa|' conf/rest-server.properties -echo -e "pa" | bin/init-store.sh || exit 1 -bin/start-hugegraph.sh || exit 1 - -cd ../${HTTPS_SERVER_DIR} -REST_SERVER_CONFIG="conf/rest-server.properties" -GREMLIN_SERVER_CONFIG="conf/gremlin-server.yaml" -sed -i "s?http://127.0.0.1:8080?https://127.0.0.1:8443?g" "$REST_SERVER_CONFIG" -sed -i "s/rpc.server_port=8091/rpc.server_port=8092/g" "$REST_SERVER_CONFIG" -sed -i "s/#port: 8182/port: 8282/g" "$GREMLIN_SERVER_CONFIG" -echo "gremlinserver.url=http://127.0.0.1:8282" >> ${REST_SERVER_CONFIG} - -# start HugeGraphServer with https protocol -sed -i 's|gremlin.graph=org.apache.hugegraph.HugeFactory|gremlin.graph=org.apache.hugegraph.auth.HugeFactoryAuthProxy|' conf/graphs/hugegraph.properties -sed -i 's|#auth.authenticator=.*|auth.authenticator=org.apache.hugegraph.auth.StandardAuthenticator|' conf/rest-server.properties -sed -i 's|#auth.admin_pa=.*|auth.admin_pa=pa|' conf/rest-server.properties -echo -e "pa" | bin/init-store.sh || exit 1 -bin/start-hugegraph.sh -cd ../ +SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd) +exec bash "$SCRIPT_DIR/../../../hugegraph-client/assembly/travis/install-hugegraph-from-source.sh" "$@" From 245424af5617dd932d3c9c7d1d69ef3adad6eaf7 Mon Sep 17 00:00:00 2001 From: imbajin Date: Tue, 6 Oct 2026 00:46:35 +0800 Subject: [PATCH 04/10] fix(ci): reuse verified Hadoop caches - restore existing version-specific archives - validate restored data with the pinned checksum - retain bounded download and readiness deadlines --- .github/workflows/loader-ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/loader-ci.yml b/.github/workflows/loader-ci.yml index 4eb9fc008..5d23f3ced 100644 --- a/.github/workflows/loader-ci.yml +++ b/.github/workflows/loader-ci.yml @@ -127,6 +127,8 @@ jobs: ~/hadoop-3.3.6.tar.gz ~/hadoop-3.3.6.tar.gz.sha512 key: ${{ runner.os }}-hadoop-3.3.6-sha512-v1 + # Legacy archives are accepted only after the installer's fixed SHA-512 check. + restore-keys: ${{ runner.os }}-hadoop-3.3.6 - name: Start Hadoop timeout-minutes: 35 From 1386b63c570b2c653511f1f267d4d0536f65e798 Mon Sep 17 00:00:00 2001 From: imbajin Date: Tue, 6 Oct 2026 00:47:16 +0800 Subject: [PATCH 05/10] fix(ci): trim readiness script whitespace - remove the extra trailing blank line - keep bounded service readiness behavior - pass the full pull request whitespace check --- .github/actions/setup-hugegraph-server/service-wait.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/actions/setup-hugegraph-server/service-wait.sh b/.github/actions/setup-hugegraph-server/service-wait.sh index 7fb7feea2..21f13e2a5 100644 --- a/.github/actions/setup-hugegraph-server/service-wait.sh +++ b/.github/actions/setup-hugegraph-server/service-wait.sh @@ -25,4 +25,3 @@ fixture_wait() { echo "HugeGraph readiness timed out: $url" >&2 return 1 } - From c5a2524e538769b38f4641256fa6392306f0652d Mon Sep 17 00:00:00 2001 From: imbajin Date: Tue, 6 Oct 2026 02:39:20 +0800 Subject: [PATCH 06/10] fix(ci): keep JVM checks within CI scope - retain affected suites and verified shared fixtures - test Toolchain on Java 17 with isolated Server JVMs - inherit existing Surefire and JaCoCo test-tool versions - gate both current and released Hubble fixtures --- .../actions/setup-hugegraph-server/action.yml | 18 +++++++ .../setup-hugegraph-server/runtime_test.py | 35 +++++++++++++ .github/scripts/ci-policy.py | 2 +- .github/scripts/test_ci_policy.py | 11 ++++ .github/workflows/ci.yml | 34 ++++++------- .github/workflows/client-ci.yml | 14 ++++- .github/workflows/client-go-ci.yml | 12 ++++- .github/workflows/codeql-analysis.yml | 6 +-- .github/workflows/hubble-ci.yml | 51 ++++++++++++++----- .github/workflows/license-checker.yml | 12 ++--- .github/workflows/loader-ci.yml | 24 ++++++++- .github/workflows/spark-connector-ci.yml | 17 +++---- .github/workflows/tools-ci.yml | 12 ++++- docs/ci.md | 11 ++-- hugegraph-hubble/hubble-be/pom.xml | 4 +- 15 files changed, 201 insertions(+), 62 deletions(-) diff --git a/.github/actions/setup-hugegraph-server/action.yml b/.github/actions/setup-hugegraph-server/action.yml index 5994e38f3..469b6853d 100644 --- a/.github/actions/setup-hugegraph-server/action.yml +++ b/.github/actions/setup-hugegraph-server/action.yml @@ -19,6 +19,12 @@ inputs: java-version: description: JDK major used for the build and runtime default: '11' + java-home: + description: Optional fixture JDK scoped to this subprocess; keeps Toolchain JDK unchanged + default: '' + isolate-maven-repo: + description: Build fixtures without replacing Toolchain Maven dependencies + default: 'false' travis-dir: description: Module service setup directory default: hugegraph-loader/assembly/travis @@ -47,6 +53,8 @@ runs: FIXTURE_BUILD_INPUTS: ${{ hashFiles('.github/actions/setup-hugegraph-server/action.yml', '.github/actions/setup-hugegraph-server/fixture.sh', '.github/actions/setup-hugegraph-server/manifest.py', '.github/actions/setup-java-env/action.yml', '.github/configs/settings.xml', 'hugegraph-client/assembly/travis/checkout-server.sh', 'hugegraph-client/assembly/travis/start-hugegraph-servers.sh', 'hugegraph-*/assembly/travis/install-hugegraph-from-source.sh', 'hugegraph-hubble/hubble-dist/assembly/travis/download-hugegraph.sh') }} FIXTURE_PLATFORM: ${{ runner.os }}-${{ runner.arch }} FIXTURE_JAVA: ${{ inputs.java-version }} + FIXTURE_JAVA_HOME: ${{ inputs.java-home }} + ISOLATE_FIXTURE_REPO: ${{ inputs.isolate-maven-repo }} FIXTURE_DIR: ${{ runner.temp }}/hugegraph-fixture SERVER_REPOSITORY: ${{ inputs.repository }} SERVER_FETCH_REF: ${{ inputs.fetch-ref }} @@ -55,6 +63,16 @@ runs: FIXTURE_ARTIFACT: ${{ inputs.artifact-name }} run: | set -euo pipefail + if [[ -n "$FIXTURE_JAVA_HOME" ]]; then + [[ -x "$FIXTURE_JAVA_HOME/bin/java" && -x "$FIXTURE_JAVA_HOME/bin/javac" ]] + export JAVA_HOME="$FIXTURE_JAVA_HOME" + export PATH="$JAVA_HOME/bin:$PATH" + fi + if [[ "$ISOLATE_FIXTURE_REPO" == true && "$FIXTURE_MODE" != start ]]; then + fixture_repo=$(mktemp -d "$RUNNER_TEMP/hugegraph-fixture-m2.XXXXXX") + export MAVEN_ARGS="-Dmaven.repo.local=$fixture_repo" + export MAVEN_OPTS="${MAVEN_OPTS:-} -Dmaven.repo.local=$fixture_repo" + fi if [[ "$FIXTURE_MODE" != standalone && -z "$FIXTURE_ARTIFACT" ]]; then echo 'artifact-name is required for build/start mode' >&2 exit 1 diff --git a/.github/actions/setup-hugegraph-server/runtime_test.py b/.github/actions/setup-hugegraph-server/runtime_test.py index d35580bfd..73c41db66 100644 --- a/.github/actions/setup-hugegraph-server/runtime_test.py +++ b/.github/actions/setup-hugegraph-server/runtime_test.py @@ -75,6 +75,41 @@ def test_identity_checksum_and_regular_files(self): archive.unlink() archive.symlink_to(self.dir/'missing') self.assertNotEqual(self.run_script(HERE/'fixture.sh', env=env).returncode, 0) + def test_composite_step_isolates_released_jdk_from_java17_caller(self): + action = (HERE / 'action.yml').read_text() + block = action.split(' run: |\n', 1)[1].split(' - name:', 1)[0] + step = self.dir / 'action-step.sh' + step.write_text('\n'.join(line[8:] for line in block.splitlines()) + '\n') + jdk11 = self.dir / 'jdk11' + jdk17 = self.dir / 'jdk17' + for version, root in ((11, jdk11), (17, jdk17)): + (root / 'bin').mkdir(parents=True) + for name, body in [('java', f'echo \'openjdk version "{version}.0.1"\' >&2'), ('javac', 'exit 0')]: + binary = root / 'bin' / name + binary.write_text('#!/bin/bash\n' + body + '\n') + binary.chmod(0o755) + travis = self.dir / 'service' + travis.mkdir() + installer = travis / 'install-hugegraph-from-source.sh' + installer.write_text('#!/bin/bash\nset -e\nsource "$FIXTURE_ACTION/fixture.sh"\n' + 'java -version 2> "$HOME/started-java"\n') + self.manifest() + env = dict(self.env, JAVA_HOME=str(jdk17), PATH=f'{jdk17}/bin:{self.env["PATH"]}', + FIXTURE_JAVA_HOME=str(jdk11), FIXTURE_MODE='start', FIXTURE_ARTIFACT='release', + FIXTURE_ACTION=str(HERE), ISOLATE_FIXTURE_REPO='false', TRAVIS_DIR=str(travis), + RUNNER_TEMP=str(self.dir)) + result = subprocess.run(['bash', '-c', 'bash "$HOME/action-step.sh" && java -version 2> "$HOME/caller-java"'], + cwd=self.dir, env=env, text=True, capture_output=True, timeout=20) + self.assertEqual(0, result.returncode, result.stderr) + self.assertIn('11.0.1', (self.dir / 'started-java').read_text()) + self.assertIn('17.0.1', (self.dir / 'caller-java').read_text()) + # A declared Java 11 fixture cannot start under an accidentally selected Java 17 runtime. + env['FIXTURE_JAVA_HOME'] = str(jdk17) + result = subprocess.run(['bash', str(step)], cwd=self.dir, env=env, + text=True, capture_output=True, timeout=20) + self.assertNotEqual(0, result.returncode) + self.assertIn('Fixture JDK mismatch', result.stderr) + def test_legacy_bash_service_deadline_and_success(self): helper = str(HERE/'service-wait.sh') self.command('curl', 'exit 1\n') diff --git a/.github/scripts/ci-policy.py b/.github/scripts/ci-policy.py index 821b8f1ab..0d8be2f17 100644 --- a/.github/scripts/ci-policy.py +++ b/.github/scripts/ci-policy.py @@ -382,7 +382,7 @@ def gate(plan, results, fetch=api): raise ValueError("selected suite did not succeed: " + suite) executed.append(suite) if plan["project"] == "toolchain": - if set(executed).intersection({"client", "loader", "tools", "spark", "go"}): + if set(executed).intersection({"client", "loader", "tools", "spark", "go", "hubble"}): if results.get("fixture", {}).get("result") != "success": raise ValueError("selected tests lack successful fixture") if "hubble" in executed and results.get("hubble-fixture", {}).get("result") != "success": diff --git a/.github/scripts/test_ci_policy.py b/.github/scripts/test_ci_policy.py index 6a4682904..ee57fb039 100644 --- a/.github/scripts/test_ci_policy.py +++ b/.github/scripts/test_ci_policy.py @@ -224,6 +224,17 @@ def test_fixture_missing_rejects_successful_test(self): with self.assertRaises(ValueError): policy.gate(plan, {"plan": {"result": "success"}, "hubble": {"result": "success"}}, self.fetch) + def test_hubble_gate_requires_current_and_released_fixtures(self): + plan = self.plan() + plan['expected'] = ['hubble'] + for producer in ('fixture', 'hubble-fixture'): + for state in (None, 'failure', 'cancelled', 'skipped'): + results = {'plan': {'result': 'success'}, 'hubble': {'result': 'success'}, + 'fixture': {'result': 'success'}, 'hubble-fixture': {'result': 'success'}} + results[producer] = {'result': state} + with self.subTest(producer=producer, state=state), self.assertRaises(ValueError): + policy.gate(plan, results, self.fetch) + def test_proof_failure_disables_receipt_without_falsifying_gate(self): def failure(path): raise subprocess.CalledProcessError(1, "gh") diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3efdac41c..7999d3a03 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -24,11 +24,6 @@ on: push: branches: [master, 'release-*'] workflow_dispatch: - inputs: - server-pr-number: - description: HugeGraph Server PR integration baseline (empty uses latest release) - required: false - type: string permissions: contents: read @@ -66,11 +61,13 @@ jobs: python3 -m unittest discover -s .github/scripts -p 'test_*.py' python3 .github/actions/setup-hugegraph-server/runtime_test.py node --test .github/actions/get-hugegraph-commit/resolve.test.cjs hugegraph-client/assembly/travis/checkout-server.test.cjs hugegraph-client/assembly/travis/install-server.test.cjs - - name: Resolve immutable Server release baseline + - name: Resolve historical Server 1.7 baseline id: baseline uses: ./.github/actions/get-hugegraph-commit with: - server-pr-number: ${{ inputs.server-pr-number || '' }} + repository: apache/hugegraph + ref: refs/tags/1.7.0 + expected-commit: b12425c2032bf0d21a97b8221f42a18055c2982f - name: Resolve Hubble's existing Server master baseline id: hubble-baseline env: @@ -90,10 +87,10 @@ jobs: import json, os inputs = { 'server': {'repository': os.environ['SERVER_REPOSITORY'], 'commit': os.environ['SERVER_COMMIT'], - 'ref': os.environ['SERVER_REF']}, + 'ref': os.environ['SERVER_REF'], 'jdk': '11'}, 'hubble': {'repository': 'apache/hugegraph', 'commit': os.environ['HUBBLE_COMMIT'], - 'ref': os.environ['HUBBLE_COMMIT'], 'branch': 'master'}, - 'jdk': '11', 'config': 'rocksdb-ci-v1' + 'ref': os.environ['HUBBLE_COMMIT'], 'branch': 'master', 'jdk': '17'}, + 'toolchainJDK': '17', 'config': 'release-current-ci-v2' } with open('inputs.json', 'w') as stream: json.dump(inputs, stream, sort_keys=True) @@ -115,7 +112,7 @@ jobs: fixture: needs: plan - if: needs.plan.outputs.client == 'true' || needs.plan.outputs.loader == 'true' || needs.plan.outputs.tools == 'true' || needs.plan.outputs.spark == 'true' || needs.plan.outputs.go == 'true' + if: needs.plan.outputs.needsFixture == 'true' runs-on: ubuntu-latest outputs: artifact: ${{ steps.identity.outputs.artifact }} @@ -156,7 +153,7 @@ jobs: run: echo "artifact=hubble-fixture-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" >> "$GITHUB_OUTPUT" - uses: ./.github/actions/setup-java-env with: - java-version: '11' + java-version: '17' distribution: temurin use-stage: 'true' - name: Build reusable Server package @@ -166,7 +163,7 @@ jobs: artifact-name: ${{ steps.identity.outputs.artifact }} commit-id: ${{ needs.plan.outputs.hubble-commit }} fetch-ref: ${{ needs.plan.outputs.hubble-commit }} - java-version: '11' + java-version: '17' client: needs: [plan, fixture] @@ -213,13 +210,16 @@ jobs: secrets: inherit hubble: - needs: [plan, hubble-fixture] + needs: [plan, fixture, hubble-fixture] if: ${{ needs.plan.outputs.hubble == 'true' }} uses: ./.github/workflows/hubble-ci.yml with: - artifact-name: ${{ needs.hubble-fixture.outputs.artifact }} - server-commit: ${{ needs.plan.outputs.hubble-commit }} - server-ref: ${{ needs.plan.outputs.hubble-commit }} + current-artifact-name: ${{ needs.hubble-fixture.outputs.artifact }} + current-commit: ${{ needs.plan.outputs.hubble-commit }} + artifact-name: ${{ needs.fixture.outputs.artifact }} + server-repository: ${{ needs.plan.outputs.server-repository }} + server-commit: ${{ needs.plan.outputs.server-commit }} + server-ref: ${{ needs.plan.outputs.server-ref }} secrets: inherit go: diff --git a/.github/workflows/client-ci.yml b/.github/workflows/client-ci.yml index 74467b21b..9907142c1 100644 --- a/.github/workflows/client-ci.yml +++ b/.github/workflows/client-ci.yml @@ -26,7 +26,7 @@ jobs: fail-fast: false matrix: # released pd package is compiled in version 55.0, which requires java 11 to use - JAVA_VERSION: [ '11' ] + JAVA_VERSION: ['17'] steps: - name: Fetch code @@ -34,6 +34,14 @@ jobs: with: fetch-depth: 2 + - name: Install released Server JVM + id: server-java + uses: actions/setup-java@v5 + with: + java-version: '11' + distribution: temurin + overwrite-settings: false + - name: Setup Java environment uses: ./.github/actions/setup-java-env with: @@ -50,6 +58,8 @@ jobs: fetch-ref: ${{ inputs.server-ref }} artifact-name: ${{ inputs.artifact-name }} mode: start + java-version: '11' + java-home: ${{ steps.server-java.outputs.path }} - name: Compile run: | @@ -57,7 +67,7 @@ jobs: - name: Run test run: | - cd hugegraph-client && ls * + cd hugegraph-client && ls -- * mvn test -Dtest=UnitTestSuite -ntp mvn test -Dtest=ApiTestSuite mvn test -Dtest=FuncTestSuite diff --git a/.github/workflows/client-go-ci.yml b/.github/workflows/client-go-ci.yml index 081e1bf6c..1ec11903c 100644 --- a/.github/workflows/client-go-ci.yml +++ b/.github/workflows/client-go-ci.yml @@ -25,7 +25,7 @@ jobs: strategy: fail-fast: false matrix: - JAVA_VERSION: ['11'] + JAVA_VERSION: ['17'] steps: - name: Fetch Code @@ -33,6 +33,14 @@ jobs: with: fetch-depth: 2 + - name: Install released Server JVM + id: server-java + uses: actions/setup-java@v5 + with: + java-version: '11' + distribution: temurin + overwrite-settings: false + - name: Setup Java environment uses: ./.github/actions/setup-java-env with: @@ -49,6 +57,8 @@ jobs: fetch-ref: ${{ inputs.server-ref }} artifact-name: ${{ inputs.artifact-name }} mode: start + java-version: '11' + java-home: ${{ steps.server-java.outputs.path }} - name: Init Go env uses: actions/setup-go@v6 diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 49486b451..abba60c91 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -31,13 +31,13 @@ jobs: uses: actions/setup-java@v5 with: distribution: 'zulu' - java-version: '11' + java-version: '17' - name: use staged maven repo settings if: ${{ env.USE_STAGE == 'true' }} run: | - cp $HOME/.m2/settings.xml /tmp/settings.xml - mv -vf .github/configs/settings.xml $HOME/.m2/settings.xml + cp "$HOME/.m2/settings.xml" /tmp/settings.xml + mv -vf .github/configs/settings.xml "$HOME/.m2/settings.xml" - name: Use Node.js 16 uses: actions/setup-node@v6 diff --git a/.github/workflows/hubble-ci.yml b/.github/workflows/hubble-ci.yml index bd04ead30..278908aae 100644 --- a/.github/workflows/hubble-ci.yml +++ b/.github/workflows/hubble-ci.yml @@ -3,6 +3,12 @@ name: "hubble-ci" on: workflow_call: inputs: + current-artifact-name: + type: string + required: true + current-commit: + type: string + required: true server-repository: type: string default: apache/hugegraph @@ -26,8 +32,10 @@ jobs: USE_STAGE: 'true' # Whether to include the stage repository. STATIC_DIR: hugegraph-hubble/hubble-dist/assembly/static strategy: + fail-fast: false matrix: - JAVA_VERSION: ['11'] + JAVA_VERSION: ['17'] + SERVER_BASELINE: ['current', 'release-1.7'] python-version: ["3.11"] steps: @@ -36,6 +44,15 @@ jobs: with: fetch-depth: 2 + - name: Install released Server JVM + if: matrix.SERVER_BASELINE == 'release-1.7' + id: server-java + uses: actions/setup-java@v5 + with: + java-version: '11' + distribution: temurin + overwrite-settings: false + - name: Setup Java environment uses: ./.github/actions/setup-java-env with: @@ -101,13 +118,13 @@ jobs: - name: Compile run: | mvn install -pl hugegraph-client,hugegraph-loader -am -Dmaven.javadoc.skip=true -DskipTests -ntp - cd hugegraph-hubble && ls * + cd hugegraph-hubble && ls -- * mvn -e compile -Dmaven.javadoc.skip=true -ntp - name: Prepare env and service env: - COMMIT_ID: ${{ inputs.server-commit }} - COMMIT_REF: ${{ inputs.server-commit }} + COMMIT_ID: ${{ matrix.SERVER_BASELINE == 'current' && inputs.current-commit || inputs.server-commit }} + COMMIT_REF: ${{ matrix.SERVER_BASELINE == 'current' && inputs.current-commit || inputs.server-commit }} NODE_OPTIONS: --max-old-space-size=4096 run: | echo "=== Environment Info ===" @@ -119,7 +136,7 @@ jobs: free -h || true echo "=======================" - python -m pip install -r ${TRAVIS_DIR}/requirements.txt + python -m pip install -r "${TRAVIS_DIR}/requirements.txt" cd hugegraph-hubble mvn package -Dmaven.test.skip=true cd apache-hugegraph-hubble-* @@ -133,21 +150,24 @@ jobs: uses: ./.github/actions/setup-hugegraph-server with: mode: start - artifact-name: ${{ inputs.artifact-name }} - commit-id: ${{ inputs.server-commit }} + java-version: ${{ matrix.SERVER_BASELINE == 'current' && '17' || '11' }} + java-home: ${{ matrix.SERVER_BASELINE == 'release-1.7' && steps.server-java.outputs.path || '' }} + artifact-name: ${{ matrix.SERVER_BASELINE == 'current' && inputs.current-artifact-name || inputs.artifact-name }} + commit-id: ${{ matrix.SERVER_BASELINE == 'current' && inputs.current-commit || inputs.server-commit }} repository: ${{ inputs.server-repository }} - fetch-ref: ${{ inputs.server-ref }} + fetch-ref: ${{ matrix.SERVER_BASELINE == 'current' && inputs.current-commit || inputs.server-ref }} travis-dir: ${{ env.TRAVIS_DIR }} - name: Release package audit env: GNUPGHOME: ${{ runner.temp }}/hubble-release-gate-gpghome run: | - HUBBLE_TARBALL="$(ls hugegraph-hubble/target/apache-hugegraph-hubble-*.tar.gz | head -n 1)" - if [[ -z "${HUBBLE_TARBALL}" ]]; then - echo "Hubble tarball not found" >&2 + archives=(hugegraph-hubble/target/apache-hugegraph-hubble-*.tar.gz) + if [[ ${#archives[@]} -ne 1 || ! -f "${archives[0]}" ]]; then + echo "Expected exactly one Hubble tarball" >&2 exit 1 fi + HUBBLE_TARBALL="${archives[0]}" mkdir -p "${GNUPGHOME}" chmod 700 "${GNUPGHOME}" @@ -175,7 +195,12 @@ jobs: - name: API test run: | cd hugegraph-hubble - HUBBLE_TARBALL="$(ls target/apache-hugegraph-hubble-*.tar.gz | head -n 1)" + archives=(target/apache-hugegraph-hubble-*.tar.gz) + if [[ ${#archives[@]} -ne 1 || ! -f "${archives[0]}" ]]; then + echo "Expected exactly one Hubble tarball" >&2 + exit 1 + fi + HUBBLE_TARBALL="${archives[0]}" HUBBLE_694_EVIDENCE_DIR=target/hubble-live-acceptance \ hubble-dist/assembly/travis/verify-hubble-issue-694.sh \ "${HUBBLE_TARBALL}" http://127.0.0.1:8080 @@ -184,7 +209,7 @@ jobs: if: ${{ always() }} uses: actions/upload-artifact@v7 with: - name: hubble-smoke-java-${{ matrix.JAVA_VERSION }}-${{ github.run_attempt }} + name: hubble-smoke-java-${{ matrix.JAVA_VERSION }}-${{ matrix.SERVER_BASELINE }}-${{ github.run_attempt }} path: | hugegraph-hubble/target/hubble-dist-check.json hugegraph-hubble/target/hubble-live-acceptance/**/*.json diff --git a/.github/workflows/license-checker.yml b/.github/workflows/license-checker.yml index 7dda8e16a..feb050b14 100644 --- a/.github/workflows/license-checker.yml +++ b/.github/workflows/license-checker.yml @@ -85,10 +85,10 @@ jobs: steps: - name: Checkout source uses: actions/checkout@v7 - - name: Set up JDK 11 + - name: Set up JDK 17 uses: actions/setup-java@v5 with: - java-version: '11' + java-version: '17' distribution: 'temurin' - name: Use Node.js 16 uses: actions/setup-node@v6 @@ -98,15 +98,15 @@ jobs: - name: use staged maven repo settings if: ${{ env.USE_STAGE == 'true' }} run: | - cp $HOME/.m2/settings.xml /tmp/settings.xml - mv -vf .github/configs/settings.xml $HOME/.m2/settings.xml + cp "$HOME/.m2/settings.xml" /tmp/settings.xml + mv -vf .github/configs/settings.xml "$HOME/.m2/settings.xml" - name: mvn install run: | mvn install -DskipTests=true -ntp - name: generate current dependencies run: | - bash $SCRIPT_DEPENDENCY/regenerate_known_dependencies.sh current-dependencies.txt + bash "$SCRIPT_DEPENDENCY/regenerate_known_dependencies.sh" current-dependencies.txt - name: check third dependencies run: | - bash $SCRIPT_DEPENDENCY/check_dependencies.sh + bash "$SCRIPT_DEPENDENCY/check_dependencies.sh" diff --git a/.github/workflows/loader-ci.yml b/.github/workflows/loader-ci.yml index 5d23f3ced..57f9098a3 100644 --- a/.github/workflows/loader-ci.yml +++ b/.github/workflows/loader-ci.yml @@ -29,7 +29,7 @@ jobs: DB_DATABASE: load_test strategy: matrix: - JAVA_VERSION: ['11'] + JAVA_VERSION: ['17'] services: mysql: image: mysql:8.0 @@ -45,6 +45,14 @@ jobs: with: fetch-depth: 2 + - name: Install released Server JVM + id: server-java + uses: actions/setup-java@v5 + with: + java-version: '11' + distribution: temurin + overwrite-settings: false + - name: Setup Java environment uses: ./.github/actions/setup-java-env with: @@ -60,6 +68,8 @@ jobs: uses: ./.github/actions/setup-hugegraph-server with: mode: start + java-version: '11' + java-home: ${{ steps.server-java.outputs.path }} artifact-name: ${{ inputs.artifact-name }} commit-id: ${{ inputs.server-commit }} repository: ${{ inputs.server-repository }} @@ -92,13 +102,21 @@ jobs: DB_DATABASE: load_test strategy: matrix: - JAVA_VERSION: ['11'] + JAVA_VERSION: ['17'] steps: - name: Fetch Code uses: actions/checkout@v7 with: fetch-depth: 2 + - name: Install released Server JVM + id: server-java + uses: actions/setup-java@v5 + with: + java-version: '11' + distribution: temurin + overwrite-settings: false + - name: Setup Java environment uses: ./.github/actions/setup-java-env with: @@ -114,6 +132,8 @@ jobs: uses: ./.github/actions/setup-hugegraph-server with: mode: start + java-version: '11' + java-home: ${{ steps.server-java.outputs.path }} artifact-name: ${{ inputs.artifact-name }} commit-id: ${{ inputs.server-commit }} repository: ${{ inputs.server-repository }} diff --git a/.github/workflows/spark-connector-ci.yml b/.github/workflows/spark-connector-ci.yml index 2b6047546..399531e22 100644 --- a/.github/workflows/spark-connector-ci.yml +++ b/.github/workflows/spark-connector-ci.yml @@ -25,7 +25,7 @@ jobs: strategy: fail-fast: false matrix: - JAVA_VERSION: [ '11', '17' ] + JAVA_VERSION: ['17'] steps: - name: Fetch Code @@ -33,17 +33,13 @@ jobs: with: fetch-depth: 2 - # The published HugeGraph 1.7 authentication fixture requires Java 11. - - name: Setup server Java + - name: Install released Server JVM id: server-java uses: actions/setup-java@v5 with: java-version: '11' - distribution: 'temurin' - - - name: Save server Java path - id: server-path - run: echo "path=$JAVA_HOME/bin:$PATH" >> "$GITHUB_OUTPUT" + distribution: temurin + overwrite-settings: false - name: Setup Java environment uses: ./.github/actions/setup-java-env @@ -58,9 +54,6 @@ jobs: - name: Setup HugeGraph server uses: ./.github/actions/setup-hugegraph-server - env: - JAVA_HOME: ${{ steps.server-java.outputs.path }} - PATH: ${{ steps.server-path.outputs.path }} with: travis-dir: ${{ env.TRAVIS_DIR }} commit-id: ${{ inputs.server-commit }} @@ -68,6 +61,8 @@ jobs: fetch-ref: ${{ inputs.server-ref }} artifact-name: ${{ inputs.artifact-name }} mode: start + java-version: '11' + java-home: ${{ steps.server-java.outputs.path }} - name: Run test run: | diff --git a/.github/workflows/tools-ci.yml b/.github/workflows/tools-ci.yml index 036a87256..85745ecb7 100644 --- a/.github/workflows/tools-ci.yml +++ b/.github/workflows/tools-ci.yml @@ -24,7 +24,7 @@ jobs: TRAVIS_DIR: hugegraph-tools/assembly/travis strategy: matrix: - JAVA_VERSION: [ '11' ] + JAVA_VERSION: ['17'] steps: - name: Fetch Code @@ -32,6 +32,14 @@ jobs: with: fetch-depth: 2 + - name: Install released Server JVM + id: server-java + uses: actions/setup-java@v5 + with: + java-version: '11' + distribution: temurin + overwrite-settings: false + - name: Setup Java environment uses: ./.github/actions/setup-java-env with: @@ -52,6 +60,8 @@ jobs: fetch-ref: ${{ inputs.server-ref }} artifact-name: ${{ inputs.artifact-name }} mode: start + java-version: '11' + java-home: ${{ steps.server-java.outputs.path }} - name: Run test run: | diff --git a/docs/ci.md b/docs/ci.md index 3d37026fd..33f01171e 100644 --- a/docs/ci.md +++ b/docs/ci.md @@ -12,10 +12,15 @@ cancelled or unexpectedly skipped module cannot satisfy it. | Shared build inputs or unrecognized paths | All modules | | One workflow | Its tests and shared dependencies | +Toolchain compilation and tests use Java 17 without changing application +dependencies. Client, Loader, Tools, Spark and Go test the released Server 1.7 +fixture on Java 11. Hubble tests both this release and the current Server master +snapshot on Java 17; the resolved commit stays fixed while the run is queued. + +Each Server package is built once and shared, with independent services per job. +Reuse verifies source, commit, JDK, build inputs and archive checksum. The Server +JVM is scoped to service startup, so Toolchain compilation keeps Java 17. Loader's HDFS tests run separately, so other profiles do not wait for Hadoop. -Client, Loader, Tools, Spark and Go share one verified Server package, but each -starts its own service. Hubble retains its Server master baseline. Package reuse -checks source, commit, JDK, build configuration and archive checksum. Documentation paths use an explicit allowlist. A documentation-only update may reuse a successful receipt from the same PR only when the base, non-document diff --git a/hugegraph-hubble/hubble-be/pom.xml b/hugegraph-hubble/hubble-be/pom.xml index 162f24a1d..70331bf38 100644 --- a/hugegraph-hubble/hubble-be/pom.xml +++ b/hugegraph-hubble/hubble-be/pom.xml @@ -385,7 +385,7 @@ org.apache.maven.plugins maven-surefire-plugin - 2.20 + ${maven.surefire.plugin.version} org.apache.maven.plugins @@ -404,7 +404,7 @@ org.jacoco jacoco-maven-plugin - 0.8.4 + ${jacoco.version} pre-unit-test From 72d0d599cf4218cbeab8b1949170cd6f8230e29e Mon Sep 17 00:00:00 2001 From: imbajin Date: Tue, 6 Oct 2026 03:59:02 +0800 Subject: [PATCH 07/10] fix(ci): restore coverage and verified reuse - include Go consumers of shared fixture scripts - verify packaged docs and optional proof provenance - restrict legacy security aliases and write permissions - document JDK coverage and pending runtime validation --- .github/scripts/ci-policy.py | 205 ++++++++++++++++++++++------ .github/scripts/test_ci_policy.py | 220 +++++++++++++++++++++++++++++- .github/workflows/ci.yml | 8 +- docs/ci.md | 18 ++- 4 files changed, 405 insertions(+), 46 deletions(-) diff --git a/.github/scripts/ci-policy.py b/.github/scripts/ci-policy.py index 0d8be2f17..149f3a31f 100644 --- a/.github/scripts/ci-policy.py +++ b/.github/scripts/ci-policy.py @@ -17,6 +17,7 @@ """Conservative CI selection and GitHub-verified test receipt reuse.""" import argparse +import base64 import hashlib import io import json @@ -39,7 +40,7 @@ "hugegraph-spark-connector/": "spark", "hugegraph-hubble/": "hubble"}, } WORKFLOWS = { - "server": {"server-ci.yml": ["server"], "commons-ci.yml": ["commons"], + "server": {"server-ci.yml": MODULES["server"], "commons-ci.yml": ["commons"], "pd-store-ci.yml": ["pd", "store", "hstore"], "cluster-test-ci.yml": ["cluster"], "docker-build-ci.yml": ["docker"], "helm-chart-ci.yml": ["helm"], "codeql-analysis.yml": [], "riscv64-ci.yml": ["server"], "check-dependencies.yml": ["dependency_license"]}, @@ -51,7 +52,7 @@ DEPENDENTS = { "server": {"commons": ["server", "pd", "store", "hstore", "cluster"], "struct": ["server", "pd", "store", "hstore", "cluster"], - "server": ["cluster"], "pd": ["store", "hstore", "cluster"], + "server": ["pd", "store", "hstore", "cluster"], "pd": ["store", "hstore", "cluster"], "store": ["hstore", "cluster"], "hstore": ["cluster"]}, "toolchain": {"client": ["loader", "tools", "spark", "hubble"], "loader": ["hubble"]}, } @@ -105,9 +106,18 @@ def select(project, paths): continue if path.endswith(".proto"): return set(MODULES[project]) + if project == "server" and path == "hugegraph-commons/hugegraph-common/src/main/resources/version.properties": + selected.update(["commons", "docker"]) + continue + if project == "server" and path == "hugegraph-store/hg-store-dist/src/assembly/static/bin/util.sh": + selected.update(["store", "docker"]) + continue + if project == "toolchain" and path.startswith("hugegraph-client/assembly/travis/"): + selected.update(["client", "go"]) + continue if project == "server": if path.startswith("hugegraph-server/hugegraph-hstore/"): - selected.add("hstore") + selected.update(["server", "hstore"]) continue if path.startswith("hugegraph-server/hugegraph-dist/"): selected.update(["server", "pd", "store", "hstore", "docker", "cluster"]) @@ -131,7 +141,26 @@ def select(project, paths): return selected -def fingerprint(ref, policy=False): +def external_context(inputs, selected): + context = dict(inputs or {}) + if "hubble" not in selected: + context.pop("hubble", None) + return json.dumps(context, sort_keys=True, separators=(",", ":")) + + +def packaged_readme_state(ref): + path = "hugegraph-hubble/README.md" + entry = git("ls-tree", ref, "--", path) + if not entry: + return "missing" + mode, kind, blob = entry.split("\t", 1)[0].split() + if mode != "100644" or kind != "blob": + return "invalid-mode:" + mode + ":" + blob + content = subprocess.check_output(["git", "cat-file", "blob", blob]) + return "valid" if content.strip() else "empty:" + blob + + +def fingerprint(ref, policy=False, project=None): """Hash blob identities, including paths; exclude only the strict prose whitelist.""" entries = subprocess.check_output(["git", "ls-tree", "-rz", "--full-tree", ref]).split(b"\0") digest = hashlib.sha256() @@ -146,10 +175,14 @@ def fingerprint(ref, policy=False): elif documentation(path, metadata.split(b" ", 1)[0].decode()): continue digest.update(metadata + b"\t" + name + b"\0") + if project == "toolchain" and not policy: + state = packaged_readme_state(ref) + if state != "valid": + digest.update(("required-packaged-readme:" + state).encode()) return digest.hexdigest() -def remote_fingerprint(repository, sha, policy_only=False, fetch=api): +def remote_fingerprint(repository, sha, policy_only=False, fetch=api, project=None): tree = fetch(f"repos/{repository}/git/trees/{sha}?recursive=1") if tree.get("truncated"): raise ValueError("truncated proof tree") @@ -165,6 +198,17 @@ def remote_fingerprint(repository, sha, policy_only=False, fetch=api): continue digest.update((entry["mode"] + " " + entry["type"] + " " + entry["sha"] + "\t" + path + "\0").encode()) + if project == "toolchain" and not policy_only: + entry = next((item for item in tree["tree"] if item["path"] == "hugegraph-hubble/README.md"), None) + state = "missing" + if entry: + if entry["mode"] != "100644" or entry["type"] != "blob": + state = "invalid-mode:" + entry["mode"] + ":" + entry["sha"] + else: + blob = fetch(f"repos/{repository}/git/blobs/{entry['sha']}") + state = "valid" if base64.b64decode(blob["content"]).strip() else "empty:" + entry["sha"] + if state != "valid": + digest.update(("required-packaged-readme:" + state).encode()) return digest.hexdigest() @@ -218,6 +262,29 @@ def successful_jobs(repository, run_id, suite, fetch=api): return sorted(job["id"] for job in matching) +def validate_proof_run(plan, item, fetch=api): + run = fetch(f"repos/{plan['repository']}/actions/runs/{int(item['runID'])}") + if (run.get("status") != "completed" + or run.get("event") != "pull_request" or run.get("head_sha") != item.get("head") + or run.get("workflow_id") != plan["workflowID"] + or run.get("head_repository", {}).get("full_name") != plan["source"] + or run.get("head_branch") != plan["branch"]): + raise ValueError("proof run is not a successful trusted workflow") + if not proof_pr(plan, run, fetch): + raise ValueError("proof run lacks matching PR association") + merge = fetch(f"repos/{plan['repository']}/git/commits/{item['testedMergeSHA']}") + if [parent["sha"] for parent in merge.get("parents", [])] != [plan["base"], item["head"]]: + raise ValueError("tested merge parents differ") + actual_input = remote_fingerprint(plan["repository"], item["testedMergeSHA"], fetch=fetch, project=plan["project"]) + if plan["project"] == "toolchain": + context = external_context(plan.get("externalInputs", {}), plan["selected"]) + actual_input = hashlib.sha256((actual_input + context).encode()).hexdigest() + if (actual_input != plan["inputFingerprint"] or remote_fingerprint( + plan["repository"], item["testedMergeSHA"], policy_only=True, fetch=fetch) + != plan["policyFingerprint"]): + raise ValueError("actual tested inputs differ from claimed receipt") + + def validate_receipt(plan, receipt, fetch=api): for key in ["schema", "repository", "project", "pr", "source", "branch", "base", "inputFingerprint", "policyFingerprint"]: if receipt.get(key) != plan.get(key): @@ -229,27 +296,8 @@ def validate_receipt(plan, receipt, fetch=api): verified = {} for suite in required: item = proof[suite] - run = fetch(f"repos/{plan['repository']}/actions/runs/{int(item['runID'])}") - if (run.get("status") != "completed" - or run.get("event") != "pull_request" or run.get("head_sha") != item.get("head") - or run.get("workflow_id") != plan["workflowID"] - or run.get("head_repository", {}).get("full_name") != plan["source"] - or run.get("head_branch") != plan["branch"]): - raise ValueError("proof run is not a successful trusted workflow") + validate_proof_run(plan, item, fetch) successful_jobs(plan["repository"], item["runID"], "affected-module-tests", fetch) - if not proof_pr(plan, run, fetch): - raise ValueError("proof run lacks matching PR association") - merge = fetch(f"repos/{plan['repository']}/git/commits/{item['testedMergeSHA']}") - if [parent["sha"] for parent in merge.get("parents", [])] != [plan["base"], item["head"]]: - raise ValueError("tested merge parents differ") - actual_input = remote_fingerprint(plan["repository"], item["testedMergeSHA"], fetch=fetch) - if plan["project"] == "toolchain": - context = json.dumps(plan.get("externalInputs", {}), sort_keys=True, separators=(",", ":")) - actual_input = hashlib.sha256((actual_input + context).encode()).hexdigest() - if (actual_input != plan["inputFingerprint"] or remote_fingerprint( - plan["repository"], item["testedMergeSHA"], policy_only=True, fetch=fetch) - != plan["policyFingerprint"]): - raise ValueError("actual tested inputs differ from claimed receipt") ids = successful_jobs(plan["repository"], item["runID"], suite, fetch) if ids != item.get("jobIDs"): raise ValueError("actual job evidence differs") @@ -257,13 +305,94 @@ def validate_receipt(plan, receipt, fetch=api): return verified +PROVENANCE = ["schema", "repository", "project", "pr", "source", "branch", "base", + "inputFingerprint", "policyFingerprint"] + + +def optional_groups(plan): + if plan["project"] == "toolchain": + return {"security": {"security / Analyze (java)", "security / Analyze (javascript)", + "security / Analyze (python)", "security / dependency-review"}} + runtime = (Path(__file__).resolve().parents[1] / "workflows/.java-version").read_text().strip() + return {"security": {"codeql / Analyze (java)"}, + "compatibility": {f"HBase compatibility (Java {runtime})", + "build-server-macos-rocksdb (macos-15-intel)", + "build-server-macos-rocksdb (macos-15, -Xms512m -Xmx2g)", + "build-server-riscv64 / build-server-riscv64"}} + + +def optional_group_jobs(jobs, group): + def belongs(name): + if group == "security": + return name.startswith(("codeql /", "security /")) + return name.startswith(("HBase compatibility", "build-server-macos-rocksdb", "build-server-riscv64 /")) + return [job for job in jobs if belongs(job.get("name", ""))] + + +def optional_jobs(jobs, names): + # Reject incomplete/duplicate matrices; proof identities must describe the full group. + matching = [job for job in jobs if job.get("name") in names] + if len(matching) != len(names) or {job["name"] for job in matching} != names: + raise ValueError("optional group is incomplete") + if any(job.get("conclusion") != "success" for job in matching): + raise ValueError("optional group did not succeed") + return sorted(matching, key=lambda job: job["name"]) + + +def optional_proofs(plan, receipt, jobs, blocked, fetch=api): + proofs = {} + for group, names in optional_groups(plan).items(): + if group in blocked: + continue + try: + current = optional_group_jobs(jobs, group) + if any(job.get("name") not in names for job in current): + raise ValueError("unknown optional group job") + # Skipped wrapper jobs can forward proof. A partial or failed execution cannot. + executed = [job for job in current if job.get("conclusion") != "skipped"] + if executed: + matching = optional_jobs(jobs, names) + item = {key: receipt[key] for key in PROVENANCE} + item.update(runID=receipt["runID"], head=receipt["head"], + testedMergeSHA=receipt["testedMergeSHA"]) + validate_proof_run(plan, item, fetch) + item.update(jobIDs=[job["id"] for job in matching], jobNames=[job["name"] for job in matching]) + else: + item = receipt.get("optionalProofs", {}).get(group) + if not item: + continue + if any(item.get(key) != plan.get(key) for key in PROVENANCE): + raise ValueError("optional provenance differs") + validate_proof_run(plan, item, fetch) + successful_jobs(plan["repository"], item["runID"], "affected-module-tests", fetch) + original = optional_group_jobs( + pages(f"repos/{plan['repository']}/actions/runs/{item['runID']}/jobs", fetch), group) + if any(job.get("name") not in names for job in original): + raise ValueError("unknown original optional group job") + matching = optional_jobs(original, names) + if (item.get("jobIDs") != [job["id"] for job in matching] + or item.get("jobNames") != [job["name"] for job in matching]): + raise ValueError("optional job evidence differs") + proofs[group] = item + except (subprocess.SubprocessError, OSError, ValueError, KeyError, TypeError, AttributeError): + continue + return proofs + + def find_reuse(plan, fetch=api): """Receipts are uploaded by this workflow, not read from the PR tree.""" repository = plan["repository"] current = fetch(f"repos/{repository}/actions/runs/{os.environ['GITHUB_RUN_ID']}") plan["workflowID"] = current["workflow_id"] - runs = fetch(f"repos/{repository}/actions/workflows/{plan['workflowID']}/runs?" + urlencode({ - "event": "pull_request", "status": "completed", "per_page": 30}))["workflow_runs"] + runs = [] + for page in range(1, 6): + batch = fetch(f"repos/{repository}/actions/workflows/{plan['workflowID']}/runs?" + urlencode({ + "event": "pull_request", "status": "completed", "branch": plan["branch"], + "per_page": 30, "page": page}))["workflow_runs"] + runs.extend(batch) + if len(batch) < 30: + break + blocked_optional = set() for run in runs: if run["id"] == int(os.environ["GITHUB_RUN_ID"]): continue @@ -295,18 +424,14 @@ def find_reuse(plan, fetch=api): successful_jobs(repository, run["id"], "affected-module-tests", fetch) verified = validate_receipt(plan, receipt, fetch) jobs = pages(f"repos/{repository}/actions/runs/{run['id']}/jobs", fetch) - def good_matches(predicate, minimum=1): - matching = [job for job in jobs if predicate(job.get("name", "").lower())] - return len(matching) >= minimum and all(job.get("conclusion") == "success" for job in matching) - plan["optionalSuccess"] = { - "compatibility": (good_matches(lambda n: "hbase compatibility" in n) - and good_matches(lambda n: "macos" in n, 2) - and good_matches(lambda n: "riscv64" in n)), - "security": good_matches(lambda n: n.startswith(("codeql /", "security /"))), - } + plan["optionalProofs"] = optional_proofs(plan, receipt, jobs, blocked_optional, fetch) + plan["optionalSuccess"] = {group: group in plan["optionalProofs"] + for group in ["compatibility", "security"]} return verified except (ValueError, KeyError, TypeError, AttributeError, zipfile.BadZipFile): continue + # Missing/unverifiable evidence from a newer same-PR run is conservative for optional lanes. + blocked_optional.update(optional_groups(plan)) return {} @@ -339,9 +464,12 @@ def create_plan(project, event, repository, fetch=api, external_inputs=None): selected = select(project, paths) if unsafe_documentation(plan["base"], paths) or unsafe_documentation(plan["testedMergeSHA"], paths): selected = set(MODULES[project]) - plan["inputFingerprint"] = fingerprint(plan["testedMergeSHA"]) + if project == "toolchain" and packaged_readme_state(plan["testedMergeSHA"]) != "valid": + selected = set(MODULES[project]) + plan["selected"] = sorted(selected) + plan["inputFingerprint"] = fingerprint(plan["testedMergeSHA"], project=project) if project == "toolchain": - context = json.dumps(external_inputs or {}, sort_keys=True, separators=(",", ":")) + context = external_context(external_inputs, selected) plan["inputFingerprint"] = hashlib.sha256((plan["inputFingerprint"] + context).encode()).hexdigest() plan["policyFingerprint"] = fingerprint(plan["testedMergeSHA"], policy=True) plan["expected"] = suites(project, selected) @@ -399,6 +527,7 @@ def gate(plan, results, fetch=api): except (subprocess.SubprocessError, OSError, ValueError, KeyError, TypeError, AttributeError): pass receipt["proofs"] = proofs + receipt["optionalProofs"] = plan.get("optionalProofs", {}) receipt["executed"] = executed receipt["reused"] = sorted(plan["reused"]) summary = "Selection: " + plan.get("reason", "affected inputs") + "\nExecuted: " + ", ".join(executed) + "\nReused: " + ", ".join(receipt["reused"]) diff --git a/.github/scripts/test_ci_policy.py b/.github/scripts/test_ci_policy.py index ee57fb039..24e4011e6 100644 --- a/.github/scripts/test_ci_policy.py +++ b/.github/scripts/test_ci_policy.py @@ -42,7 +42,7 @@ def test_dependency_expansion(self): self.assertEqual({"client", "loader", "tools", "spark", "hubble"}, policy.select("toolchain", ["hugegraph-client/src/A.java"])) self.assertEqual({"loader", "hubble"}, policy.select("toolchain", ["hugegraph-loader/src/A.java"])) - self.assertEqual({"server", "cluster"}, policy.select("server", ["hugegraph-server/A.java"])) + self.assertEqual({"server", "pd", "store", "hstore", "cluster"}, policy.select("server", ["hugegraph-server/A.java"])) def test_server_backend_and_startup_dependents(self): selected = policy.select("server", ["hugegraph-server/hugegraph-hstore/src/test/HstoreTableTest.java"]) @@ -56,7 +56,80 @@ def test_server_backend_and_startup_dependents(self): self.assertIn("hstore", selected) self.assertIn("server", selected) self.assertEqual(set(), policy.select("server", ["hugegraph-server/hugegraph-hstore/README.md"])) - self.assertNotIn("hstore", policy.select("server", ["hugegraph-server/hugegraph-rocksdb/src/main/A.java"])) + self.assertIn("hstore", policy.select("server", ["hugegraph-server/hugegraph-rocksdb/src/main/A.java"])) + + def test_audited_consumer_edges(self): + expected = {"server", "pd", "store", "hstore", "cluster"} + for path in ["hugegraph-server/hugegraph-hstore/src/A.java", "hugegraph-server/hugegraph-rocksdb/A.java"]: + self.assertTrue(expected.issubset(policy.select("server", [path]))) + self.assertNotIn("helm", policy.select("server", [path])) + self.assertEqual(set(policy.MODULES["server"]), policy.select("server", [".github/workflows/server-ci.yml"])) + self.assertTrue({"commons", "docker"}.issubset(policy.select("server", [ + "hugegraph-commons/hugegraph-common/src/main/resources/version.properties"]))) + self.assertIn("go", policy.select("toolchain", ["hugegraph-client/assembly/travis/start-hugegraph-servers.sh"])) + self.assertNotIn("go", policy.select("toolchain", ["hugegraph-client/src/A.java"])) + self.assertNotIn("server", policy.select("server", ["hugegraph-pd/src/A.java"])) + self.assertTrue({"store", "docker"}.issubset(policy.select("server", [ + "hugegraph-store/hg-store-dist/src/assembly/static/bin/util.sh"]))) + + def test_version_resource_follows_real_docker_consumer(self): + root = Path(__file__).resolve().parents[2] + consumer = root / ".github/workflows/docker-build-ci.yml" + if not consumer.exists(): + self.skipTest("Server Docker consumer is not in the Toolchain repository") + resource = next(line.strip().rstrip(")") for line in consumer.read_text().splitlines() + if line.strip().startswith("hugegraph-commons/") and "version.properties" in line) + self.assertTrue((root / resource).is_file()) + self.assertTrue({"commons", "docker"}.issubset(policy.select("server", [resource]))) + + def test_external_hubble_baseline_only_affects_hubble(self): + a = {"server": {"sha": "release"}, "hubble": {"sha": "old"}} + b = {"server": {"sha": "release"}, "hubble": {"sha": "new"}} + self.assertEqual(policy.external_context(a, ["go"]), policy.external_context(b, ["go"])) + self.assertNotEqual(policy.external_context(a, ["hubble"]), policy.external_context(b, ["hubble"])) + + def test_packaged_readme_content_contract(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + def git(*args): + return subprocess.check_output(["git", *args], cwd=root, text=True).strip() + git("init", "-q") + git("config", "user.email", "ci@example.invalid") + git("config", "user.name", "CI") + (root / "hugegraph-hubble").mkdir() + readme = root / "hugegraph-hubble/README.md" + readme.write_text("nonempty docs") + git("add", ".") + git("commit", "-qm", "valid readme") + old = os.getcwd() + try: + os.chdir(root) + initial = policy.fingerprint("HEAD", project="toolchain") + readme.write_text("changed nonempty docs") + git("commit", "-qam", "valid docs edit") + self.assertEqual(initial, policy.fingerprint("HEAD", project="toolchain")) + for content in ["", " \t\n"]: + readme.write_text(content) + git("commit", "-qam", "empty docs") + self.assertNotEqual("valid", policy.packaged_readme_state("HEAD")) + self.assertNotEqual(initial, policy.fingerprint("HEAD", project="toolchain")) + readme.write_text("nonempty") + readme.chmod(0o755) + git("add", ".") + git("commit", "-qm", "executable docs") + self.assertNotEqual("valid", policy.packaged_readme_state("HEAD")) + readme.unlink() + readme.symlink_to("missing-target") + git("add", ".") + git("commit", "-qm", "symlink docs") + self.assertNotEqual("valid", policy.packaged_readme_state("HEAD")) + readme.unlink() + git("add", ".") + git("commit", "-qm", "deleted docs") + self.assertEqual("missing", policy.packaged_readme_state("HEAD")) + self.assertNotEqual(initial, policy.fingerprint("HEAD", project="toolchain")) + finally: + os.chdir(old) def test_single_workflow_has_no_global_fanout(self): self.assertEqual({"hubble"}, policy.select("toolchain", [".github/workflows/hubble-ci.yml"])) @@ -162,6 +235,145 @@ def fetch(path, binary=False): with patch.dict(os.environ, {"GITHUB_RUN_ID": "99"}): self.assertEqual({}, policy.find_reuse(self.plan(), fetch)) + def optional_fixture(self, receipts, optional_results=None): + optional_results = optional_results or {} + def fetch(path, binary=False): + if "/actions/workflows/" in path: + return {"workflow_runs": [{"id": run_id, "head_repository": {"full_name": "alice/t"}, + "head_branch": "feature", "pull_requests": [{"number": 7}]} + for run_id in sorted(receipts, reverse=True)]} + if "/artifacts?" in path: + run_id = int(path.split("/runs/")[1].split("/")[0]) + return {"artifacts": [{"name": "ci-test-receipt", "id": run_id, + "expired": False, "size_in_bytes": 512}]} + if "/actions/artifacts/" in path: + run_id = int(path.split("/artifacts/")[1].split("/")[0]) + data = io.BytesIO() + with zipfile.ZipFile(data, "w") as archive: + archive.writestr("receipt.json", json.dumps(receipts[run_id])) + return data.getvalue() + if "/git/commits/" in path: + head = "new" if path.endswith("/merge") else "old" + return {"parents": [{"sha": "base"}, {"sha": head}]} + if "/jobs?" in path: + run_id = int(path.split("/runs/")[1].split("/")[0]) + jobs = [{"name": "affected-module-tests", "id": run_id * 10, "conclusion": "success"}] + if run_id == 42: + jobs.append({"name": "client / client-ci", "id": 123, "conclusion": "success"}) + for index, name in enumerate(sorted(policy.optional_groups(self.plan())["security"])): + result = optional_results.get(run_id, "success" if run_id == 42 else "skipped") + jobs.append({"name": name, "id": run_id * 100 + index, "conclusion": result}) + return {"jobs": jobs} + value = self.fetch(path) + if "/actions/runs/43" in path: + value["head_sha"] = "new" + return value + return fetch + + def find_optional(self, receipts, optional_results=None): + plan = self.plan() + with patch.dict(os.environ, {"GITHUB_RUN_ID": "99"}), patch.object( + policy, "remote_fingerprint", side_effect=lambda *a, **k: "policy" if k.get("policy_only") else "inputs"): + plan["reused"] = policy.find_reuse(plan, self.optional_fixture(receipts, optional_results)) + return plan + + def test_optional_original_proof_survives_two_documentation_updates(self): + original = self.receipt() + original.update(runID=42, head="old", testedMergeSHA="oldmerge") + second = self.find_optional({42: original}) + self.assertTrue(second["optionalSuccess"]["security"]) + self.assertEqual(42, second["optionalProofs"]["security"]["runID"]) + with patch.dict(os.environ, {"GITHUB_RUN_ID": "43"}): + forwarded = policy.gate(second, {"plan": {"result": "success"}}, self.fetch) + third = self.find_optional({43: forwarded, 42: original}) + self.assertTrue(third["optionalSuccess"]["security"]) + self.assertEqual(second["optionalProofs"], third["optionalProofs"]) + self.assertEqual(second["reused"], third["reused"]) + for result in ["failure", "cancelled", "timed_out", None]: + with self.subTest(result=result): + latest = self.find_optional({43: forwarded, 42: original}, {43: result}) + self.assertTrue(latest["reused"]) + self.assertFalse(latest["optionalSuccess"]["security"]) + + def test_newer_unverifiable_receipt_blocks_older_optional_green(self): + original = self.receipt() + original.update(runID=42, head="old", testedMergeSHA="oldmerge") + invalid = dict(original, runID=43, source="other/t") + plan = self.find_optional({43: invalid, 42: original}) + self.assertTrue(plan["reused"]) + self.assertFalse(plan["optionalSuccess"]["security"]) + + def test_server_optional_matrix_follows_project_runtime(self): + plan = dict(self.plan(), project="server") + runtime_file = Path(__file__).resolve().parents[1] / "workflows/.java-version" + if not runtime_file.exists(): + self.skipTest("Server runtime configuration is not in the Toolchain repository") + runtime = runtime_file.read_text().strip() + names = policy.optional_groups(plan)["compatibility"] + self.assertIn(f"HBase compatibility (Java {runtime})", names) + self.assertEqual(4, len(names)) + jobs = [{"name": name, "id": index, "conclusion": "success"} + for index, name in enumerate(sorted(names))] + self.assertEqual(4, len(policy.optional_jobs(jobs, names))) + for missing in range(len(jobs)): + with self.assertRaises(ValueError): + policy.optional_jobs(jobs[:missing] + jobs[missing + 1:], names) + + def test_optional_forwarded_forgery_and_original_failure_reject(self): + original = self.receipt() + original.update(runID=42, head="old", testedMergeSHA="oldmerge") + second = self.find_optional({42: original}) + with patch.dict(os.environ, {"GITHUB_RUN_ID": "43"}): + forwarded = policy.gate(second, {"plan": {"result": "success"}}, self.fetch) + for field in ["jobIDs", "jobNames", "source", "base", "inputFingerprint", "policyFingerprint", "head"]: + changed = json.loads(json.dumps(forwarded)) + changed["optionalProofs"]["security"][field] = [999] if field == "jobIDs" else "forged" + with self.subTest(field=field): + latest = self.find_optional({43: changed, 42: original}) + self.assertTrue(latest["reused"]) + self.assertFalse(latest["optionalSuccess"]["security"]) + latest = self.find_optional({43: forwarded, 42: original}, {42: "failure"}) + self.assertFalse(latest["optionalSuccess"]["security"]) + + def test_optional_group_requires_complete_known_matrix(self): + plan = self.plan() + original = self.receipt() + original.update(runID=42, head="old", testedMergeSHA="oldmerge") + fetch = self.optional_fixture({42: original}) + for mutation in ["missing", "unknown", "duplicate", "api-error"]: + def bad_fetch(path, binary=False): + if "/actions/runs/" in path and mutation == "api-error": + raise OSError("unavailable") + value = fetch(path, binary) + if "/jobs?" in path: + if mutation == "missing": + value["jobs"].pop() + elif mutation == "unknown": + value["jobs"].append({"name": "security / Analyze (unknown)", "id": 999, "conclusion": "success"}) + elif mutation == "duplicate": + value["jobs"].append(dict(value["jobs"][-1])) + return value + jobs = fetch("repos/apache/t/actions/runs/42/jobs?per_page=100&page=1")["jobs"] + if mutation != "api-error": + jobs = bad_fetch("repos/apache/t/actions/runs/42/jobs?per_page=100&page=1")["jobs"] + with patch.object(policy, "remote_fingerprint", side_effect=lambda *a, **k: "policy" if k.get("policy_only") else "inputs"): + self.assertEqual({}, policy.optional_proofs(plan, original, jobs, set(), bad_fetch)) + + def test_branch_filtered_bounded_run_pagination(self): + calls = [] + def fetch(path, binary=False): + if "/actions/workflows/" in path: + calls.append(path) + self.assertIn("branch=feature", path) + if "page=1" in path: + return {"workflow_runs": [{"id": n, "head_repository": {"full_name": "alice/t"}, + "head_branch": "other"} for n in range(30)]} + return {"workflow_runs": []} + return self.fetch(path) + with patch.dict(os.environ, {"GITHUB_RUN_ID": "99"}): + self.assertEqual({}, policy.find_reuse(self.plan(), fetch)) + self.assertEqual(2, len(calls)) + def test_tested_merge_parent_mismatch_rejects(self): def fetch(path): if "/git/commits/" in path: @@ -253,14 +465,14 @@ def git(*args): if args[0] == "diff": return "hugegraph-client/src/A.java\nREADME.md" return "new" - with patch.object(policy, "unsafe_documentation", return_value=False), patch.object(policy, "git", side_effect=git), patch.object(policy, "fingerprint", return_value="hash"), patch.object( + with patch.object(policy, "packaged_readme_state", return_value="valid"), patch.object(policy, "unsafe_documentation", return_value=False), patch.object(policy, "git", side_effect=git), patch.object(policy, "fingerprint", return_value="hash"), patch.object( policy, "find_reuse", return_value={m: {"runID": 42} for m in ["client", "loader", "tools", "spark", "hubble"]}): plan = policy.create_plan("toolchain", event, "apache/t", lambda p: live, external_inputs={"serverSHA": "immutable"}) self.assertFalse(plan["client"]) self.assertFalse(plan["needsFixture"]) self.assertTrue(plan["security"]) - with patch.object(policy, "unsafe_documentation", return_value=False), patch.object(policy, "git", side_effect=git), patch.object(policy, "fingerprint", return_value="hash"), patch.object( + with patch.object(policy, "packaged_readme_state", return_value="valid"), patch.object(policy, "unsafe_documentation", return_value=False), patch.object(policy, "git", side_effect=git), patch.object(policy, "fingerprint", return_value="hash"), patch.object( policy, "find_reuse", return_value={}): plan = policy.create_plan("toolchain", event, "apache/t", lambda p: live, external_inputs={"serverSHA": "immutable"}) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7999d3a03..eead8208c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -29,7 +29,6 @@ permissions: contents: read actions: read pull-requests: read - security-events: write concurrency: group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.run_id }}-${{ github.run_attempt == 1 && 'head' || format('rerun-{0}', github.run_id) }} @@ -262,7 +261,12 @@ jobs: security: needs: [plan, affected-module-tests] + permissions: + contents: read + actions: read + security-events: write if: >- + !cancelled() && needs.plan.result == 'success' && needs.affected-module-tests.result == 'success' && (needs.plan.outputs.security == 'true' || (github.event_name == 'pull_request' && vars.CI_OPTIMIZED_REQUIRED != 'true')) @@ -272,7 +276,7 @@ jobs: legacy-java-analysis: name: Analyze (java) needs: security - if: ${{ !cancelled() && vars.CI_OPTIMIZED_REQUIRED != 'true' }} + if: ${{ !cancelled() && github.event_name == 'pull_request' && vars.CI_OPTIMIZED_REQUIRED != 'true' }} runs-on: ubuntu-latest steps: - name: Require actual Java security analysis during migration diff --git a/docs/ci.md b/docs/ci.md index 33f01171e..f538baf33 100644 --- a/docs/ci.md +++ b/docs/ci.md @@ -8,12 +8,16 @@ cancelled or unexpectedly skipped module cannot satisfy it. | --- | --- | | Java Client | Client, Loader, Tools, Spark, Hubble | | Loader | Loader, Hubble | +| Shared Client Server checkout/install/start scripts | Client downstreams and Go | | Tools, Spark, Go or Hubble | The affected module | | Shared build inputs or unrecognized paths | All modules | | One workflow | Its tests and shared dependencies | Toolchain compilation and tests use Java 17 without changing application -dependencies. Client, Loader, Tools, Spark and Go test the released Server 1.7 +dependencies. This matrix validates Toolchain on Java 17; it does not retain +a Toolchain Java 11 runtime lane. The historical Server Java 11 fixture validates +client/server compatibility, not Toolchain execution on Java 11. +Client, Loader, Tools, Spark and Go test the released Server 1.7 fixture on Java 11. Hubble tests both this release and the current Server master snapshot on Java 17; the resolved commit stays fixed while the run is queued. @@ -27,9 +31,19 @@ reuse a successful receipt from the same PR only when the base, non-document inputs and CI policy match. The latest commit receives a new gate result that identifies reused tests. Missing or unverifiable evidence runs the tests again; source, type definitions, tests and CI configuration are never treated as docs. +Hubble's packaged README must remain a regular, nonempty file; deletion, invalid +mode or empty content forces fresh validation. Receipt searches filter by PR +branch and use bounded pagination. The current Server input only affects reuse +when Hubble is selected. +For consecutive documentation updates, optional-check proofs retain their original +run and exact job identities and are reverified before forwarding. A newer failed +check or unverifiable evidence prevents reuse of an older optional success. CodeQL follows the module gate and retains its weekly scan. During migration, -`Analyze (java)` still runs on every PR. After ASF branch protection actually +`Analyze (java)` still runs on every PR, after a successful module gate. +The legacy alias is PR-only; documentation pushes do not force scanning or fail +because a deliberately unselected scan was skipped. Scan write permissions are +limited to the security job. After ASF branch protection actually requires `check-license-header` and `affected-module-tests`, an administrator can set `CI_OPTIMIZED_REQUIRED=true` to skip security scans for documentation-only changes. This switch is disabled by default and does not alter branch protection. From 0613a577fb986a4a9895e307e6d5bab9a5c8e3b7 Mon Sep 17 00:00:00 2001 From: imbajin Date: Tue, 6 Oct 2026 04:30:34 +0800 Subject: [PATCH 08/10] fix(ci): verify images and retain rerun inputs - retain necessary fixture and plan artifacts for seven days - select image startup checks from packaging inputs - bind image reuse to complete source and job evidence - preserve module coverage and document retry limits --- .../actions/setup-hugegraph-server/action.yml | 2 +- .github/scripts/check-toolchain-image.py | 99 +++++++++++++ .github/scripts/ci-policy.py | 103 +++++++++++-- .github/scripts/test_ci_policy.py | 135 ++++++++++++++++++ .github/scripts/test_toolchain_image.py | 56 ++++++++ .github/workflows/ci.yml | 24 +++- .github/workflows/image-ci.yml | 44 ++++++ docs/ci.md | 40 +++++- 8 files changed, 486 insertions(+), 17 deletions(-) create mode 100644 .github/scripts/check-toolchain-image.py create mode 100644 .github/scripts/test_toolchain_image.py create mode 100644 .github/workflows/image-ci.yml diff --git a/.github/actions/setup-hugegraph-server/action.yml b/.github/actions/setup-hugegraph-server/action.yml index 469b6853d..00a3247a1 100644 --- a/.github/actions/setup-hugegraph-server/action.yml +++ b/.github/actions/setup-hugegraph-server/action.yml @@ -100,4 +100,4 @@ runs: ${{ runner.temp }}/hugegraph-fixture/server.tar.gz ${{ runner.temp }}/hugegraph-fixture/manifest.json if-no-files-found: error - retention-days: 1 + retention-days: 7 diff --git a/.github/scripts/check-toolchain-image.py b/.github/scripts/check-toolchain-image.py new file mode 100644 index 000000000..84a215141 --- /dev/null +++ b/.github/scripts/check-toolchain-image.py @@ -0,0 +1,99 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Build and start the selected Toolchain Dockerfile from this checkout.""" + +import json +import os +from pathlib import Path +import subprocess +import sys +import tempfile +import time +import urllib.error +import urllib.request + + +def output(*args): + return subprocess.check_output(args, text=True, timeout=60).strip() + + +def verify_image(container, built): + actual = output("docker", "inspect", "--format", "{{.Image}}", container) + if not built or actual != built: + raise RuntimeError(f"Container {container} uses {actual}; expected built image {built}") + + +def valid_hubble_about(body): + if not isinstance(body, dict) or body.get("status") != 200: + return False + data = body.get("data") + return (isinstance(data, dict) and data.get("name") == "hugegraph-hubble" + and isinstance(data.get("version"), str) and bool(data["version"].strip())) + + +def wait_hubble(container): + port = output("docker", "port", container, "8088/tcp").splitlines()[0].rsplit(":", 1)[1] + deadline = time.monotonic() + 120 + last_error = "no response" + while time.monotonic() < deadline: + if output("docker", "inspect", "--format", "{{.State.Running}}", container) != "true": + raise RuntimeError("Hubble exited before becoming ready") + try: + with urllib.request.urlopen(f"http://127.0.0.1:{port}/about", timeout=5) as response: + body = json.load(response) + if valid_hubble_about(body): + return + last_error = str(body) + except (OSError, ValueError, urllib.error.HTTPError) as error: + last_error = str(error) + time.sleep(2) + raise RuntimeError("Hubble readiness failed: " + last_error) + + +def main(module): + if module not in {"loader", "hubble"}: + raise ValueError("Select loader or hubble") + tag = f"hugegraph/{module}:ci-{os.environ['GITHUB_RUN_ID']}-{os.environ['GITHUB_RUN_ATTEMPT']}" + container = f"hg-{module}-{os.environ['GITHUB_RUN_ID']}-{os.environ['GITHUB_RUN_ATTEMPT']}" + with tempfile.TemporaryDirectory() as temporary: + iidfile = Path(temporary) / "image-id" + subprocess.run(["docker", "build", "--progress=plain", "--iidfile", str(iidfile), + "-t", tag, "-f", f"hugegraph-{module}/Dockerfile", "."], check=True, timeout=2400) + built = iidfile.read_text().strip() + try: + args = ["docker", "run", "-d", "--pull", "never", "--name", container] + if module == "hubble": + args += ["-p", "127.0.0.1::8088"] + subprocess.run(args + [tag], check=True, timeout=60) + verify_image(container, built) + if module == "loader": + help_text = output("docker", "exec", container, "bash", "bin/hugegraph-loader.sh", "--help") + if "--help" not in help_text or "Usage:" not in help_text: + raise RuntimeError("Loader CLI did not produce its usage contract") + else: + wait_hubble(container) + if output("docker", "inspect", "--format", "{{.State.Running}}", container) != "true": + raise RuntimeError(f"{module} exited after its runtime check") + except BaseException: + subprocess.run(["docker", "logs", "--tail", "200", container], check=False, timeout=30) + raise + finally: + subprocess.run(["docker", "rm", "-f", container], check=False, timeout=60) + + +if __name__ == "__main__": + main(sys.argv[1]) diff --git a/.github/scripts/ci-policy.py b/.github/scripts/ci-policy.py index 149f3a31f..8b96095d7 100644 --- a/.github/scripts/ci-policy.py +++ b/.github/scripts/ci-policy.py @@ -49,6 +49,9 @@ "spark-connector-ci.yml": ["spark"], "hubble-ci.yml": ["hubble"], "codeql-analysis.yml": []}, } +IMAGES = {"loader_image", "hubble_image"} + + DEPENDENTS = { "server": {"commons": ["server", "pd", "store", "hstore", "cluster"], "struct": ["server", "pd", "store", "hstore", "cluster"], @@ -94,9 +97,67 @@ def unsafe_documentation(ref, paths): return False +def packaged_image_document(path): + p = Path(path) + return (str(p.parent) in {"hugegraph-loader", "hugegraph-hubble"} + and p.name.startswith("README")) + + +def image_only_path(path): + return path in {".dockerignore", "hugegraph-loader/Dockerfile", "hugegraph-hubble/Dockerfile", + ".github/workflows/image-ci.yml", ".github/scripts/check-toolchain-image.py", + ".github/scripts/test_toolchain_image.py"} + + +def select_images(paths): + """Image packaging selection is independent of the Java consumer closure.""" + selected = set() + for path in paths: + p = Path(path) + if path == "hugegraph-loader/Dockerfile": + selected.add("loader_image") + elif path == "hugegraph-hubble/Dockerfile": + selected.add("hubble_image") + elif (path in {"pom.xml", ".dockerignore", "hugegraph-client/pom.xml", + "hugegraph-tools/pom.xml", "hugegraph-spark-connector/pom.xml", + "hugegraph-dist/pom.xml"} + or path.startswith((".mvn/", "hugegraph-dist/release-docs/"))): + selected.update(IMAGES) + elif path.startswith("hugegraph-loader/") and ( + p.name == "pom.xml" or path.startswith(("hugegraph-loader/assembly/descriptor/", + "hugegraph-loader/assembly/static/")) + or packaged_image_document(path) or p.name.startswith(("LICENSE", "NOTICE"))): + # Hubble's Dockerfile first builds and installs Client and Loader. + selected.update(IMAGES) + elif path.startswith("hugegraph-hubble/") and ( + p.name == "pom.xml" or "/assembly/descriptor/" in path or "/assembly/static/" in path + or packaged_image_document(path) + or path == "hugegraph-hubble/hubble-dist/assembly/travis/check-hubble-dist.sh" + or path.startswith(("hugegraph-hubble/.mvn/", "hugegraph-hubble/hubble-fe/scripts/")) + or (path.startswith("hugegraph-hubble/hubble-be/src/main/resources/") + and p.name in {"application.properties", "hugegraph-hubble.properties", "log4j2.xml"}) + or p.name in {"package.json", "yarn.lock", "package-lock.json", "webpack.config.js", + "tsconfig.json", "tsconfig.eslint.json", ".yarnrc", ".npmrc", + "config-overrides.js", ".babelrc", "babel.config.js"}): + selected.add("hubble_image") + elif documentation(path): + continue + elif path.startswith(".github/workflows/") and p.name in WORKFLOWS["toolchain"]: + continue + elif any(path.startswith(prefix) for prefix in PREFIXES["toolchain"]): + # Ordinary application/test edits stay on their module behavior lanes. + continue + else: + # Unknown shared inputs (including CI implementation) fail conservative. + selected.update(IMAGES) + return selected + + def select(project, paths): selected = set() for path in paths: + if project == "toolchain" and image_only_path(path): + continue if documentation(path): continue if project == "server" and (Path(path).name == "pom.xml" or path.startswith("install-dist/")): @@ -160,7 +221,7 @@ def packaged_readme_state(ref): return "valid" if content.strip() else "empty:" + blob -def fingerprint(ref, policy=False, project=None): +def fingerprint(ref, policy=False, project=None, image=False): """Hash blob identities, including paths; exclude only the strict prose whitelist.""" entries = subprocess.check_output(["git", "ls-tree", "-rz", "--full-tree", ref]).split(b"\0") digest = hashlib.sha256() @@ -172,7 +233,7 @@ def fingerprint(ref, policy=False, project=None): if policy: if not (path.startswith(".github/") or path == ".asf.yaml"): continue - elif documentation(path, metadata.split(b" ", 1)[0].decode()): + elif documentation(path, metadata.split(b" ", 1)[0].decode()) and not (image and packaged_image_document(path)): continue digest.update(metadata + b"\t" + name + b"\0") if project == "toolchain" and not policy: @@ -182,7 +243,7 @@ def fingerprint(ref, policy=False, project=None): return digest.hexdigest() -def remote_fingerprint(repository, sha, policy_only=False, fetch=api, project=None): +def remote_fingerprint(repository, sha, policy_only=False, fetch=api, project=None, image=False): tree = fetch(f"repos/{repository}/git/trees/{sha}?recursive=1") if tree.get("truncated"): raise ValueError("truncated proof tree") @@ -194,7 +255,7 @@ def remote_fingerprint(repository, sha, policy_only=False, fetch=api, project=No if policy_only: if not (path.startswith(".github/") or path == ".asf.yaml"): continue - elif documentation(path, entry["mode"]): + elif documentation(path, entry["mode"]) and not (image and packaged_image_document(path)): continue digest.update((entry["mode"] + " " + entry["type"] + " " + entry["sha"] + "\t" + path + "\0").encode()) @@ -262,7 +323,7 @@ def successful_jobs(repository, run_id, suite, fetch=api): return sorted(job["id"] for job in matching) -def validate_proof_run(plan, item, fetch=api): +def validate_proof_run(plan, item, fetch=api, suite=None): run = fetch(f"repos/{plan['repository']}/actions/runs/{int(item['runID'])}") if (run.get("status") != "completed" or run.get("event") != "pull_request" or run.get("head_sha") != item.get("head") @@ -283,6 +344,11 @@ def validate_proof_run(plan, item, fetch=api): plan["repository"], item["testedMergeSHA"], policy_only=True, fetch=fetch) != plan["policyFingerprint"]): raise ValueError("actual tested inputs differ from claimed receipt") + if suite in IMAGES: + if (not plan.get("imageFingerprint") or item.get("imageFingerprint") != plan["imageFingerprint"] + or remote_fingerprint(plan["repository"], item["testedMergeSHA"], fetch=fetch, + project="toolchain", image=True) != plan["imageFingerprint"]): + raise ValueError("actual Docker build inputs differ") def validate_receipt(plan, receipt, fetch=api): @@ -291,17 +357,22 @@ def validate_receipt(plan, receipt, fetch=api): raise ValueError("receipt provenance differs: " + key) proof = receipt.get("proofs", {}) required = plan["expected"] - if not required or not set(required).issubset(proof): + if not required or not (set(required) - IMAGES).issubset(proof): raise ValueError("receipt lacks selected suites") verified = {} for suite in required: + if suite in IMAGES and (suite not in proof or not plan.get("imageFingerprint") + or proof[suite].get("imageFingerprint") != plan["imageFingerprint"]): + continue item = proof[suite] - validate_proof_run(plan, item, fetch) + validate_proof_run(plan, item, fetch, suite=suite) successful_jobs(plan["repository"], item["runID"], "affected-module-tests", fetch) ids = successful_jobs(plan["repository"], item["runID"], suite, fetch) if ids != item.get("jobIDs"): raise ValueError("actual job evidence differs") verified[suite] = item + if not verified: + raise ValueError("receipt contains no reusable selected suite") return verified @@ -440,6 +511,7 @@ def create_plan(project, event, repository, fetch=api, external_inputs=None): "source": repository, "branch": "", "base": "", "head": git("rev-parse", "HEAD"), "reused": {}, "workflowID": 0, "reason": "affected inputs", "externalInputs": external_inputs or {}, "testedMergeSHA": git("rev-parse", "HEAD")} + selected_images = set() try: pr = event.get("pull_request") if pr: @@ -462,6 +534,7 @@ def create_plan(project, event, repository, fetch=api, external_inputs=None): plan["base"] = event.get("before", "") paths = git("diff", "--no-renames", "--name-only", plan["base"], plan["head"]).splitlines() selected = select(project, paths) + selected_images = select_images(paths) if project == "toolchain" else set() if unsafe_documentation(plan["base"], paths) or unsafe_documentation(plan["testedMergeSHA"], paths): selected = set(MODULES[project]) if project == "toolchain" and packaged_readme_state(plan["testedMergeSHA"]) != "valid": @@ -472,16 +545,19 @@ def create_plan(project, event, repository, fetch=api, external_inputs=None): context = external_context(external_inputs, selected) plan["inputFingerprint"] = hashlib.sha256((plan["inputFingerprint"] + context).encode()).hexdigest() plan["policyFingerprint"] = fingerprint(plan["testedMergeSHA"], policy=True) - plan["expected"] = suites(project, selected) + if project == "toolchain": + plan["imageFingerprint"] = fingerprint(plan["testedMergeSHA"], project=project, image=True) + plan["expected"] = sorted(suites(project, selected) + list(selected_images)) if pr and plan["expected"] and (project != "toolchain" or external_inputs): plan["reused"] = find_reuse(plan, fetch) except (subprocess.SubprocessError, OSError, ValueError, KeyError, TypeError, AttributeError): selected = set(MODULES[project]) + selected_images = set(IMAGES) if project == "toolchain" else set() plan.update(reused={}, reason="verification unavailable: full required coverage") plan.setdefault("inputFingerprint", "") plan.setdefault("policyFingerprint", "") - plan["expected"] = suites(project, selected) - if not selected: + plan["expected"] = sorted(suites(project, selected) + list(selected_images)) + if not selected and not selected_images: plan["reason"] = "cumulative PR diff contains only plain prose documentation; modules unaffected" elif plan["reused"]: plan["reason"] = "affected inputs match independently verified successful tests" @@ -489,6 +565,9 @@ def create_plan(project, event, repository, fetch=api, external_inputs=None): for module in MODULES[project]: module_suites = suites(project, {module}) plan[module] = module in selected and not (module_suites and all(s in plan["reused"] for s in module_suites)) + plan["selectedImages"] = sorted(selected_images) + for image in IMAGES: + plan[image] = image in selected_images and image not in plan["reused"] plan["pd_store"] = any(plan.get(m, False) for m in ["pd", "store", "hstore", "struct"]) plan["needsFixture"] = project == "toolchain" and any(plan.get(m, False) for m in MODULES[project]) plan["fixture"] = plan["needsFixture"] @@ -517,6 +596,8 @@ def gate(plan, results, fetch=api): raise ValueError("selected Hubble tests lack successful baseline fixture") receipt = {key: plan[key] for key in ["schema", "repository", "project", "pr", "source", "branch", "base", "head", "testedMergeSHA", "inputFingerprint", "policyFingerprint", "workflowID"]} + if "imageFingerprint" in plan: + receipt["imageFingerprint"] = plan["imageFingerprint"] receipt["runID"] = int(os.environ.get("GITHUB_RUN_ID", "0")) # Successful tests can gate even if receipt publication is unavailable. Never reuse # unverified evidence: a missing API proof only disables future receipt reuse. @@ -524,6 +605,8 @@ def gate(plan, results, fetch=api): try: ids = successful_jobs(plan["repository"], receipt["runID"], suite, fetch) proofs[suite] = {"runID": receipt["runID"], "head": plan["head"], "testedMergeSHA": plan["testedMergeSHA"], "jobIDs": ids} + if suite in IMAGES: + proofs[suite]["imageFingerprint"] = plan.get("imageFingerprint", "") except (subprocess.SubprocessError, OSError, ValueError, KeyError, TypeError, AttributeError): pass receipt["proofs"] = proofs diff --git a/.github/scripts/test_ci_policy.py b/.github/scripts/test_ci_policy.py index 24e4011e6..ea39832ae 100644 --- a/.github/scripts/test_ci_policy.py +++ b/.github/scripts/test_ci_policy.py @@ -15,6 +15,7 @@ # limitations under the License. """Executable Git fixtures and API proof tests for CI selection.""" +import base64 import hashlib import importlib.util import io @@ -30,6 +31,7 @@ spec = importlib.util.spec_from_file_location("policy", Path(__file__).with_name("ci-policy.py")) policy = importlib.util.module_from_spec(spec) spec.loader.exec_module(policy) +ACTUAL_REMOTE_FINGERPRINT = policy.remote_fingerprint class PolicyTest(unittest.TestCase): @@ -44,6 +46,108 @@ def test_dependency_expansion(self): self.assertEqual({"loader", "hubble"}, policy.select("toolchain", ["hugegraph-loader/src/A.java"])) self.assertEqual({"server", "pd", "store", "hstore", "cluster"}, policy.select("server", ["hugegraph-server/A.java"])) + def test_image_selection_is_separate_from_module_dependencies(self): + for path in ["hugegraph-client/src/A.java", "hugegraph-loader/src/A.java", + "hugegraph-hubble/hubble-be/src/main/java/App.java", + "hugegraph-hubble/hubble-fe/src/App.js", "docs/ci.md"]: + self.assertEqual(set(), policy.select_images([path]), path) + for path, images in [ + ("hugegraph-loader/Dockerfile", {"loader_image"}), + ("hugegraph-hubble/Dockerfile", {"hubble_image"}), + (".dockerignore", policy.IMAGES), + (".github/workflows/image-ci.yml", policy.IMAGES)]: + self.assertEqual(images, policy.select_images([path]), path) + self.assertEqual(set(), policy.select("toolchain", [path]), path) + + def test_image_packaging_and_shared_build_inputs(self): + for path in ["pom.xml", ".mvn/maven.config", "hugegraph-client/pom.xml", + "hugegraph-dist/release-docs/licenses/dependency.txt", + "hugegraph-loader/pom.xml", "hugegraph-loader/assembly/descriptor/assembly.xml", + "hugegraph-loader/assembly/static/bin/hugegraph-loader.sh", + "hugegraph-loader/README_CN.md", ".github/scripts/new-input.py"]: + self.assertEqual(policy.IMAGES, policy.select_images([path]), path) + for path in ["hugegraph-hubble/pom.xml", "hugegraph-hubble/hubble-be/pom.xml", + "hugegraph-hubble/hubble-dist/assembly/static/conf/hugegraph-hubble.properties", + "hugegraph-hubble/hubble-dist/assembly/descriptor/assembly.xml", + "hugegraph-hubble/hubble-dist/assembly/travis/check-hubble-dist.sh", + "hugegraph-hubble/hubble-fe/yarn.lock", "hugegraph-hubble/README.md", + "hugegraph-hubble/hubble-be/src/main/resources/application.properties"]: + self.assertEqual({"hubble_image"}, policy.select_images([path]), path) + + def test_image_only_plan_and_gate_require_no_server_fixture(self): + with patch.object(policy, "git", side_effect=lambda *a: ( + "hugegraph-loader/Dockerfile" if a[0] == "diff" else "head")), patch.object( + policy, "fingerprint", return_value="hash"), patch.object( + policy, "unsafe_documentation", return_value=False), patch.object( + policy, "packaged_readme_state", return_value="valid"): + plan = policy.create_plan("toolchain", {"before": "base"}, "apache/t") + self.assertEqual(["loader_image"], plan["expected"]) + self.assertFalse(plan["needsFixture"]) + self.assertFalse(any(plan[m] for m in policy.MODULES["toolchain"])) + results = {"plan": {"result": "success"}, "loader_image": {"result": "success"}} + receipt = policy.gate(plan, results, lambda _: {"jobs": [ + {"name": "loader_image / build-and-start", "id": 55, "conclusion": "success"}]}) + self.assertEqual("hash", receipt["proofs"]["loader_image"]["imageFingerprint"]) + for state in [None, "skipped", "cancelled", "failure"]: + results["loader_image"]["result"] = state + with self.subTest(state=state), self.assertRaises(ValueError): + policy.gate(plan, results, self.fetch) + + def test_cumulative_packaging_edit_reuses_java_without_fixture(self): + event = {"pull_request": {"number": 7, "head": {"sha": "new"}, "base": {"sha": "base"}}} + live = {"state": "open", "head": {"sha": "new", "ref": "feature", "repo": {"full_name": "alice/t"}}, + "base": {"sha": "base", "repo": {"full_name": "apache/t"}}} + def git(*args): + if args[0] == "show": + return "base new" + if args[0] == "diff": + return "hugegraph-client/src/A.java\nhugegraph-hubble/README.md" + return "new" + verified = {module: {"runID": 42} for module in ["client", "loader", "tools", "spark", "hubble"]} + with patch.object(policy, "git", side_effect=git), patch.object(policy, "fingerprint", return_value="hash"), \ + patch.object(policy, "unsafe_documentation", return_value=False), \ + patch.object(policy, "packaged_readme_state", return_value="valid"), \ + patch.object(policy, "find_reuse", return_value=verified): + plan = policy.create_plan("toolchain", event, "apache/t", lambda _: live, external_inputs={"sha": "server"}) + self.assertTrue(plan["hubble_image"]) + self.assertFalse(plan["loader_image"]) + self.assertFalse(plan["needsFixture"]) + self.assertFalse(any(plan[module] for module in verified)) + self.assertEqual(set(verified) | {"hubble_image"}, set(plan["expected"])) + + def test_missing_or_changed_image_proof_preserves_strict_java_proof(self): + plan = dict(self.plan(), expected=["client", "loader_image"], imageFingerprint="current") + receipt = self.receipt() + self.assertEqual({"client"}, set(policy.validate_receipt(plan, receipt, self.fetch))) + receipt["proofs"]["loader_image"] = dict(receipt["proofs"]["client"], imageFingerprint="old") + self.assertEqual({"client"}, set(policy.validate_receipt(plan, receipt, self.fetch))) + receipt["proofs"]["client"]["jobIDs"] = [999] + with self.assertRaises(ValueError): + policy.validate_receipt(plan, receipt, self.fetch) + + def test_image_proof_requires_actual_build_tree_and_exact_job_ids(self): + plan = dict(self.plan(), expected=["loader_image"], imageFingerprint="docker-inputs") + receipt = self.receipt() + receipt["proofs"] = {"loader_image": dict(receipt["proofs"]["client"], + imageFingerprint="docker-inputs")} + def fetch(path): + value = self.fetch(path) + if "/jobs?" in path: + value["jobs"][0]["name"] = "loader_image / build-and-start" + return value + def fingerprint(*args, **kwargs): + return "policy" if kwargs.get("policy_only") else "docker-inputs" if kwargs.get("image") else "inputs" + with patch.object(policy, "remote_fingerprint", side_effect=fingerprint): + self.assertEqual({"loader_image"}, set(policy.validate_receipt(plan, receipt, fetch))) + receipt["proofs"]["loader_image"]["jobIDs"] = [999] + with self.assertRaises(ValueError): + policy.validate_receipt(plan, receipt, fetch) + receipt["proofs"]["loader_image"]["jobIDs"] = [123] + with patch.object(policy, "remote_fingerprint", side_effect=lambda *a, **k: ( + "policy" if k.get("policy_only") else "forged" if k.get("image") else "inputs")): + with self.assertRaises(ValueError): + policy.validate_receipt(plan, receipt, fetch) + def test_server_backend_and_startup_dependents(self): selected = policy.select("server", ["hugegraph-server/hugegraph-hstore/src/test/HstoreTableTest.java"]) self.assertIn("hstore", selected) @@ -105,9 +209,40 @@ def git(*args): try: os.chdir(root) initial = policy.fingerprint("HEAD", project="toolchain") + initial_image = policy.fingerprint("HEAD", project="toolchain", image=True) readme.write_text("changed nonempty docs") git("commit", "-qam", "valid docs edit") self.assertEqual(initial, policy.fingerprint("HEAD", project="toolchain")) + changed_image = policy.fingerprint("HEAD", project="toolchain", image=True) + def fetch_tree(path): + if "/git/blobs/" in path: + blob = subprocess.check_output(["git", "cat-file", "blob", path.rsplit("/", 1)[1]]) + return {"content": base64.b64encode(blob).decode()} + entries = [] + for entry in subprocess.check_output(["git", "ls-tree", "-rz", "HEAD"]).split(b"\0"): + if entry: + metadata, name = entry.split(b"\t", 1) + mode, kind, sha = metadata.decode().split() + entries.append({"path": name.decode(), "mode": mode, "type": kind, "sha": sha}) + return {"tree": entries} + # Local plan hashing and API verification must agree on actual Git objects. + with patch.object(policy, "remote_fingerprint", ACTUAL_REMOTE_FINGERPRINT): + self.assertEqual(changed_image, policy.remote_fingerprint( + "apache/t", "HEAD", fetch=fetch_tree, project="toolchain", image=True)) + self.assertNotEqual(initial_image, changed_image) + source = root / "hugegraph-hubble/A.java" + source.write_text("first source") + git("add", ".") + git("commit", "-qm", "application source") + source_image = policy.fingerprint("HEAD", project="toolchain", image=True) + self.assertNotEqual(changed_image, source_image) + source.write_text("changed source") + git("commit", "-qam", "changed application") + self.assertNotEqual(source_image, policy.fingerprint("HEAD", project="toolchain", image=True)) + # Restore the original module inputs before exercising the README contract. + source.unlink() + git("add", ".") + git("commit", "-qm", "remove test source") for content in ["", " \t\n"]: readme.write_text(content) git("commit", "-qam", "empty docs") diff --git a/.github/scripts/test_toolchain_image.py b/.github/scripts/test_toolchain_image.py new file mode 100644 index 000000000..e2eb33eef --- /dev/null +++ b/.github/scripts/test_toolchain_image.py @@ -0,0 +1,56 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Prevent a tag resolving to an unrelated image from providing runtime evidence.""" + +import importlib.util +from pathlib import Path +import unittest +from unittest.mock import patch + +spec = importlib.util.spec_from_file_location("images", Path(__file__).with_name("check-toolchain-image.py")) +images = importlib.util.module_from_spec(spec) +spec.loader.exec_module(images) + + +class RuntimeImageTest(unittest.TestCase): + def test_rejects_container_using_a_different_image(self): + with patch.object(images, "output", return_value="sha256:old"): + with self.assertRaisesRegex(RuntimeError, "expected built image sha256:pr"): + images.verify_image("hubble", "sha256:pr") + + def test_rejects_empty_build_result(self): + with patch.object(images, "output", return_value=""): + with self.assertRaises(RuntimeError): + images.verify_image("loader", "") + + def test_accepts_built_image(self): + with patch.object(images, "output", return_value="sha256:pr"): + images.verify_image("hubble", "sha256:pr") + + def test_requires_application_identity_and_version_not_just_http_success(self): + for body in [None, [], {}, {"status": 200}, {"status": 200, "data": []}, + {"status": 200, "data": {"name": "hugegraph-hubble"}}, + {"status": 401, "data": {"name": "hugegraph-hubble", "version": "3.0"}}, + {"status": 200, "data": {"name": "other", "version": "3.0"}}]: + self.assertFalse(images.valid_hubble_about(body), body) + self.assertTrue(images.valid_hubble_about( + {"status": 200, "data": {"name": "hugegraph-hubble", "version": "3.0.0"}})) + + def test_hubble_exit_is_not_readiness_success(self): + with patch.object(images, "output", side_effect=["127.0.0.1:12345", "false"]): + with self.assertRaisesRegex(RuntimeError, "exited before becoming ready"): + images.wait_hubble("hubble") diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index eead8208c..2ee2b8089 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -49,6 +49,8 @@ jobs: hubble: ${{ steps.select.outputs.hubble }} go: ${{ steps.select.outputs.go }} needsFixture: ${{ steps.select.outputs.needsFixture }} + loader-image: ${{ steps.select.outputs.loader_image }} + hubble-image: ${{ steps.select.outputs.hubble_image }} security: ${{ steps.select.outputs.security }} steps: - uses: actions/checkout@v7 @@ -107,7 +109,7 @@ jobs: name: ci-plan path: plan.json overwrite: true - retention-days: 14 + retention-days: 7 fixture: needs: plan @@ -232,9 +234,23 @@ jobs: server-ref: ${{ needs.plan.outputs.server-ref }} secrets: inherit + loader_image: + needs: plan + if: needs.plan.outputs.loader-image == 'true' + uses: ./.github/workflows/image-ci.yml + with: + module: loader + + hubble_image: + needs: plan + if: needs.plan.outputs.hubble-image == 'true' + uses: ./.github/workflows/image-ci.yml + with: + module: hubble + affected-module-tests: name: affected-module-tests - needs: [plan, fixture, hubble-fixture, client, loader, tools, spark, hubble, go] + needs: [plan, fixture, hubble-fixture, client, loader, tools, spark, hubble, go, loader_image, hubble_image] if: ${{ !cancelled() }} runs-on: ubuntu-latest steps: @@ -244,7 +260,7 @@ jobs: - uses: actions/download-artifact@v7 with: name: ci-plan - - name: Require every selected module to succeed + - name: Require every selected module and image to succeed env: RESULTS: ${{ toJSON(needs) }} GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -257,7 +273,7 @@ jobs: name: ci-test-receipt path: receipt.json overwrite: true - retention-days: 14 + retention-days: 7 security: needs: [plan, affected-module-tests] diff --git a/.github/workflows/image-ci.yml b/.github/workflows/image-ci.yml new file mode 100644 index 000000000..70f5826e3 --- /dev/null +++ b/.github/workflows/image-ci.yml @@ -0,0 +1,44 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +name: Toolchain image startup + +on: + workflow_call: + inputs: + module: + required: true + type: string + +permissions: + contents: read + +jobs: + image: + name: build-and-start + runs-on: ubuntu-latest + timeout-minutes: 50 + steps: + - uses: actions/checkout@v7 + with: + persist-credentials: false + - name: Build and start this checkout's image + env: + MODULE: ${{ inputs.module }} + run: python3 .github/scripts/check-toolchain-image.py "$MODULE" diff --git a/docs/ci.md b/docs/ci.md index f538baf33..96e8b14b8 100644 --- a/docs/ci.md +++ b/docs/ci.md @@ -1,8 +1,9 @@ # Continuous integration Every PR runs license checks and `Required CI`. The stable -`affected-module-tests` check requires all selected Linux tests to pass; a failed, -cancelled or unexpectedly skipped module cannot satisfy it. +`affected-module-tests` check requires all selected Linux tests and image startup +checks to pass; a failed, cancelled or unexpectedly skipped selected job cannot +satisfy it. | Change | Tests | | --- | --- | @@ -13,6 +14,35 @@ cancelled or unexpectedly skipped module cannot satisfy it. | Shared build inputs or unrecognized paths | All modules | | One workflow | Its tests and shared dependencies | +Image selection has its own matrix and does not follow the Client → Loader → +Hubble module-test dependency closure: + +| Image input change | Images built and started | +| --- | --- | +| Loader Dockerfile | Loader | +| Hubble Dockerfile | Hubble | +| Loader POM, assembly descriptor/static files or packaged README/LICENSE/NOTICE | Loader and Hubble | +| Hubble POMs, assembly descriptor/static files, distribution checker, backend startup properties or frontend build configuration | Hubble | +| Root/Client/reactor POMs, `.mvn`, `.dockerignore`, shared release docs or unknown shared inputs | Loader and Hubble | +| Ordinary Java/frontend application source or tests | None; the module behavior tests above still run | + +Dockerfile-only changes run the image check without building Server fixtures. +Hubble's Dockerfile builds Client and Loader before Hubble, so Loader packaging +changes also select Hubble. Packaged README changes select images; the existing +nonempty Hubble README content contract still permits reuse of module tests. +Once an image is selected, its proof binds the complete current source inputs, +including application source and packaged README content, plus the checked base +and CI policy. An old or missing image proof reruns that image while independently +verified module proofs remain reusable. + +Each image job builds the checkout's actual Dockerfile, records its image ID and +starts a local run-scoped tag with pulling disabled. The container's image ID must +match the build. Loader explicitly runs its CLI with `--help`; its default idle +container alone cannot pass. Hubble must remain running and return a valid `/about` +JSON response with the application name and version. These checks validate image +build, packaging and startup; Loader data ingestion and Hubble browser workflows +remain covered by module tests. They do not publish an image. + Toolchain compilation and tests use Java 17 without changing application dependencies. This matrix validates Toolchain on Java 17; it does not retain a Toolchain Java 11 runtime lane. The historical Server Java 11 fixture validates @@ -25,6 +55,12 @@ Each Server package is built once and shared, with independent services per job. Reuse verifies source, commit, JDK, build inputs and archive checksum. The Server JVM is scoped to service startup, so Toolchain compilation keeps Java 17. Loader's HDFS tests run separately, so other profiles do not wait for Hadoop. +The immutable Server packages, `ci-plan` and successful `ci-test-receipt` artifacts +are retained for seven days from their upload. Partial reruns need the original +plan and fixture within that window. After an artifact expires, rerun the whole +workflow to rebuild its plan and fixtures; extending only the receipt cannot +restore an expired package. Test reports and coverage artifacts keep their +existing retention settings. Documentation paths use an explicit allowlist. A documentation-only update may reuse a successful receipt from the same PR only when the base, non-document From 9526e103aebc525df2775449f5ab5bb02b82e611 Mon Sep 17 00:00:00 2001 From: imbajin Date: Tue, 6 Oct 2026 15:55:08 +0800 Subject: [PATCH 09/10] fix(ci): preserve runtime and product checks - retain supported JVM validation after the Hubble merge - check final image entry points and public UI assets - remove untrusted result reuse and PR artifact retries - resolve consumed baselines and reject stale PR inputs --- .../setup-hugegraph-server/runtime_test.py | 39 +- .github/scripts/check-rerun.py | 82 +- .github/scripts/check-toolchain-image.py | 61 +- .github/scripts/ci-policy.py | 395 ++-------- .github/scripts/test_check_rerun.py | 120 +-- .github/scripts/test_ci_policy.py | 743 ++++++------------ .github/scripts/test_toolchain_image.py | 101 +++ .github/workflows/ci.yml | 44 +- .github/workflows/client-ci.yml | 2 +- .github/workflows/hubble-ci.yml | 7 +- .github/workflows/loader-ci.yml | 4 +- .github/workflows/rerun-ci.yml | 9 +- .github/workflows/spark-connector-ci.yml | 2 +- .github/workflows/tools-ci.yml | 2 +- README.md | 2 +- docs/ci.md | 80 +- 16 files changed, 571 insertions(+), 1122 deletions(-) diff --git a/.github/actions/setup-hugegraph-server/runtime_test.py b/.github/actions/setup-hugegraph-server/runtime_test.py index 73c41db66..6d9f54966 100644 --- a/.github/actions/setup-hugegraph-server/runtime_test.py +++ b/.github/actions/setup-hugegraph-server/runtime_test.py @@ -75,7 +75,7 @@ def test_identity_checksum_and_regular_files(self): archive.unlink() archive.symlink_to(self.dir/'missing') self.assertNotEqual(self.run_script(HERE/'fixture.sh', env=env).returncode, 0) - def test_composite_step_isolates_released_jdk_from_java17_caller(self): + def test_composite_step_isolates_fixture_jdk_from_caller(self): action = (HERE / 'action.yml').read_text() block = action.split(' run: |\n', 1)[1].split(' - name:', 1)[0] step = self.dir / 'action-step.sh' @@ -93,22 +93,27 @@ def test_composite_step_isolates_released_jdk_from_java17_caller(self): installer = travis / 'install-hugegraph-from-source.sh' installer.write_text('#!/bin/bash\nset -e\nsource "$FIXTURE_ACTION/fixture.sh"\n' 'java -version 2> "$HOME/started-java"\n') - self.manifest() - env = dict(self.env, JAVA_HOME=str(jdk17), PATH=f'{jdk17}/bin:{self.env["PATH"]}', - FIXTURE_JAVA_HOME=str(jdk11), FIXTURE_MODE='start', FIXTURE_ARTIFACT='release', - FIXTURE_ACTION=str(HERE), ISOLATE_FIXTURE_REPO='false', TRAVIS_DIR=str(travis), - RUNNER_TEMP=str(self.dir)) - result = subprocess.run(['bash', '-c', 'bash "$HOME/action-step.sh" && java -version 2> "$HOME/caller-java"'], - cwd=self.dir, env=env, text=True, capture_output=True, timeout=20) - self.assertEqual(0, result.returncode, result.stderr) - self.assertIn('11.0.1', (self.dir / 'started-java').read_text()) - self.assertIn('17.0.1', (self.dir / 'caller-java').read_text()) - # A declared Java 11 fixture cannot start under an accidentally selected Java 17 runtime. - env['FIXTURE_JAVA_HOME'] = str(jdk17) - result = subprocess.run(['bash', str(step)], cwd=self.dir, env=env, - text=True, capture_output=True, timeout=20) - self.assertNotEqual(0, result.returncode) - self.assertIn('Fixture JDK mismatch', result.stderr) + for fixture_version, fixture_jdk, caller_version, caller_jdk in ( + (11, jdk11, 17, jdk17), (17, jdk17, 11, jdk11)): + with self.subTest(fixture=fixture_version, caller=caller_version): + self.manifest(java=str(fixture_version)) + env = dict(self.env, JAVA_HOME=str(caller_jdk), + PATH=f'{caller_jdk}/bin:{self.env["PATH"]}', + FIXTURE_JAVA=str(fixture_version), FIXTURE_JAVA_HOME=str(fixture_jdk), + FIXTURE_MODE='start', FIXTURE_ARTIFACT='shared', FIXTURE_ACTION=str(HERE), + ISOLATE_FIXTURE_REPO='false', TRAVIS_DIR=str(travis), RUNNER_TEMP=str(self.dir)) + result = subprocess.run( + ['bash', '-c', 'bash "$HOME/action-step.sh" && java -version 2> "$HOME/caller-java"'], + cwd=self.dir, env=env, text=True, capture_output=True, timeout=20) + self.assertEqual(0, result.returncode, result.stderr) + self.assertIn(f'{fixture_version}.0.1', (self.dir / 'started-java').read_text()) + self.assertIn(f'{caller_version}.0.1', (self.dir / 'caller-java').read_text()) + # A fixture must reject accidentally inheriting the caller's different JDK. + env['FIXTURE_JAVA_HOME'] = str(caller_jdk) + result = subprocess.run(['bash', str(step)], cwd=self.dir, env=env, + text=True, capture_output=True, timeout=20) + self.assertNotEqual(0, result.returncode) + self.assertIn('Fixture JDK mismatch', result.stderr) def test_legacy_bash_service_deadline_and_success(self): helper = str(HERE/'service-wait.sh') diff --git a/.github/scripts/check-rerun.py b/.github/scripts/check-rerun.py index 24fff7b72..408b5655b 100644 --- a/.github/scripts/check-rerun.py +++ b/.github/scripts/check-rerun.py @@ -17,41 +17,18 @@ """Check the failed run again immediately before an automatic retry.""" import argparse -import io import json -import zipfile import os import subprocess -from urllib.parse import quote, urlencode +from urllib.parse import urlencode -def api(path, binary=False): +def api(path): result = subprocess.run( ["gh", "api", "--method", "GET", path], check=True, capture_output=True, timeout=30, ) - return result.stdout if binary else json.loads(result.stdout) - - -def tested_base_matches(repository, run_id, pr, sha, fetch): - artifacts = fetch(f"repos/{repository}/actions/runs/{run_id}/artifacts?per_page=100")["artifacts"] - candidates = [a for a in artifacts if a.get("name") == "ci-plan" and not a.get("expired")] - for artifact in candidates: - payload = fetch(f"repos/{repository}/actions/artifacts/{artifact['id']}/zip", binary=True) - with zipfile.ZipFile(io.BytesIO(payload)) as archive: - info = archive.getinfo("plan.json") - if info.file_size > 1048576: - continue - plan = json.loads(archive.read(info)) - base = pr.get("base", {}).get("sha") - if (plan.get("repository") != repository or plan.get("pr") != pr.get("number") - or plan.get("head") != sha or not base or plan.get("base") != base - or not plan.get("testedMergeSHA")): - continue - merge = fetch(f"repos/{repository}/git/commits/{plan['testedMergeSHA']}") - if [parent["sha"] for parent in merge.get("parents", [])] == [base, sha]: - return True - return False + return json.loads(result.stdout) def decide(repository, run_id, expected_attempt, max_reruns, fetch=api): @@ -65,56 +42,7 @@ def decide(repository, run_id, expected_attempt, max_reruns, fetch=api): if not sha: return "skip", "missing head SHA" if run.get("event") == "pull_request": - def local(candidate): - base_repository = candidate.get("base", {}).get("repo", {}) - return (base_repository.get("full_name") == repository - or base_repository.get("url") == f"https://api.github.com/repos/{repository}") - - source = run.get("head_repository") or {} - source_owner = (source.get("owner") or {}).get("login") - source_name = source.get("full_name") - source_branch = run.get("head_branch") - if not source_owner or not source_name or not source_branch: - return "skip", "missing source repository or branch metadata" - def source_matches(pr): - head = pr.get("head") or {} - repo = head.get("repo") or {} - return (head.get("ref") == source_branch and repo.get("full_name") == source_name - and (repo.get("owner") or {}).get("login") == source_owner) - fallback_owner = None - candidates = [pr for pr in run.get("pull_requests", []) if local(pr)] - if not candidates: - candidates = [pr for pr in fetch( - f"repos/{repository}/commits/{quote(sha, safe='')}/pulls") if local(pr)] - if not candidates: - owner = run.get("head_repository", {}).get("owner", {}).get("login") - branch = run.get("head_branch") - if owner and branch: - fallback_owner = owner - # Fork runs can have no commit/PR association in either endpoint. - # Use the qualified fork owner, never the branch name alone. - for page in range(1, 101): - batch = fetch(f"repos/{repository}/pulls?" + urlencode({ - "state": "open", "head": f"{owner}:{branch}", - "per_page": 100, "page": page})) - candidates.extend(pr for pr in batch if local(pr) - and pr.get("head", {}).get("ref") == branch - and pr.get("head", {}).get("repo", {}).get("owner", {}).get("login") == owner) - if len(batch) < 100: - break - for candidate in candidates: - number = candidate.get("number") - if not isinstance(number, int): - continue - pr = fetch(f"repos/{repository}/pulls/{number}") - if not source_matches(pr): - continue - if (pr.get("base", {}).get("repo", {}).get("full_name") == repository - and pr.get("state") == "open" and pr.get("head", {}).get("sha") == sha): - if tested_base_matches(repository, run_id, pr, sha, fetch): - return "rerun", "open PR head and tested base unchanged" - return "skip", "tested base moved or immutable CI plan proof unavailable" - return "skip", "no open PR in this repository with the failed head" + return "skip", "PR automatic retries are deferred pending trusted execution evidence" if run.get("event") == "push": branch = run.get("head_branch") if not branch: @@ -136,7 +64,7 @@ def main(): args = parser.parse_args() try: action, reason = decide(args.repository, args.run_id, args.run_attempt, args.max_reruns) - except (subprocess.SubprocessError, OSError, ValueError, KeyError, TypeError, AttributeError, zipfile.BadZipFile): + except (subprocess.SubprocessError, OSError, ValueError, KeyError, TypeError, AttributeError): action, reason = "skip", "unable to verify current run and source metadata" print(f"{action}: {reason}") with open(os.environ["GITHUB_OUTPUT"], "a", encoding="utf-8") as output: diff --git a/.github/scripts/check-toolchain-image.py b/.github/scripts/check-toolchain-image.py index 84a215141..cfb100fd5 100644 --- a/.github/scripts/check-toolchain-image.py +++ b/.github/scripts/check-toolchain-image.py @@ -17,6 +17,7 @@ """Build and start the selected Toolchain Dockerfile from this checkout.""" import json +from html.parser import HTMLParser import os from pathlib import Path import subprocess @@ -24,6 +25,7 @@ import tempfile import time import urllib.error +from urllib.parse import urljoin, urlparse import urllib.request @@ -45,6 +47,60 @@ def valid_hubble_about(body): and isinstance(data.get("version"), str) and bool(data["version"].strip())) +def check_loader(container): + help_text = output("docker", "exec", container, "./bin/hugegraph-loader.sh", "--help") + if "--help" not in help_text or "Usage:" not in help_text: + raise RuntimeError("Loader CLI did not produce its usage contract") + + +class HubblePage(HTMLParser): + def __init__(self): + super().__init__() + self.has_root = False + self.scripts = [] + self.styles = [] + + def handle_starttag(self, tag, attrs): + attrs = dict(attrs) + if tag == "div" and attrs.get("id") == "root": + self.has_root = True + if tag == "script" and attrs.get("src"): + self.scripts.append(attrs["src"]) + if tag == "link" and "stylesheet" in attrs.get("rel", "").lower().split() and attrs.get("href"): + self.styles.append(attrs["href"]) + + +class NoRedirect(urllib.request.HTTPRedirectHandler): + def redirect_request(self, request, response, code, message, headers, newurl): + return None + + +def check_hubble_ui(origin): + page_url = origin + "/" + opener = urllib.request.build_opener(NoRedirect) + with opener.open(page_url, timeout=5) as response: + if response.headers.get_content_type() != "text/html": + raise RuntimeError("Hubble UI did not return HTML") + page = HubblePage() + page.feed(response.read().decode("utf-8")) + if not page.has_root: + raise RuntimeError("Hubble UI is missing its React root") + def local(ref): + parsed = urlparse(urljoin(page_url, ref)) + return (parsed.scheme, parsed.netloc) == (urlparse(origin).scheme, urlparse(origin).netloc) + + scripts = [urljoin(page_url, ref) for ref in page.scripts if local(ref)] + styles = [urljoin(page_url, ref) for ref in page.styles if local(ref)] + if not scripts: + raise RuntimeError("Hubble UI is missing a local JavaScript bundle") + for resource, types in ([(url, {"text/javascript", "application/javascript", "application/x-javascript"}) + for url in scripts] + [(url, {"text/css"}) for url in styles]): + with opener.open(resource, timeout=5) as response: + body = response.read() + if response.headers.get_content_type() not in types or not body.strip() or body.lstrip().startswith(b"<"): + raise RuntimeError("Hubble UI asset is empty or has invalid content: " + resource) + + def wait_hubble(container): port = output("docker", "port", container, "8088/tcp").splitlines()[0].rsplit(":", 1)[1] deadline = time.monotonic() + 120 @@ -56,6 +112,7 @@ def wait_hubble(container): with urllib.request.urlopen(f"http://127.0.0.1:{port}/about", timeout=5) as response: body = json.load(response) if valid_hubble_about(body): + check_hubble_ui(f"http://127.0.0.1:{port}") return last_error = str(body) except (OSError, ValueError, urllib.error.HTTPError) as error: @@ -81,9 +138,7 @@ def main(module): subprocess.run(args + [tag], check=True, timeout=60) verify_image(container, built) if module == "loader": - help_text = output("docker", "exec", container, "bash", "bin/hugegraph-loader.sh", "--help") - if "--help" not in help_text or "Usage:" not in help_text: - raise RuntimeError("Loader CLI did not produce its usage contract") + check_loader(container) else: wait_hubble(container) if output("docker", "inspect", "--format", "{{.State.Running}}", container) != "true": diff --git a/.github/scripts/ci-policy.py b/.github/scripts/ci-policy.py index 8b96095d7..1e1a8532d 100644 --- a/.github/scripts/ci-policy.py +++ b/.github/scripts/ci-policy.py @@ -14,18 +14,13 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Conservative CI selection and GitHub-verified test receipt reuse.""" +"""Conservative affected-module selection and current-run CI gating.""" import argparse -import base64 -import hashlib -import io import json import os from pathlib import Path import subprocess -import zipfile -from urllib.parse import urlencode MODULES = { "server": ["server", "commons", "struct", "pd", "store", "hstore", "cluster", "docker", "helm", "dependency_license"], @@ -65,9 +60,9 @@ def git(*args): return subprocess.check_output(["git", *args], text=True).strip() -def api(path, binary=False): +def api(path): result = subprocess.run(["gh", "api", "--method", "GET", path], check=True, capture_output=True, timeout=30) - return result.stdout if binary else json.loads(result.stdout) + return json.loads(result.stdout) def documentation(path, mode="100644"): @@ -202,13 +197,6 @@ def select(project, paths): return selected -def external_context(inputs, selected): - context = dict(inputs or {}) - if "hubble" not in selected: - context.pop("hubble", None) - return json.dumps(context, sort_keys=True, separators=(",", ":")) - - def packaged_readme_state(ref): path = "hugegraph-hubble/README.md" entry = git("ls-tree", ref, "--", path) @@ -221,75 +209,6 @@ def packaged_readme_state(ref): return "valid" if content.strip() else "empty:" + blob -def fingerprint(ref, policy=False, project=None, image=False): - """Hash blob identities, including paths; exclude only the strict prose whitelist.""" - entries = subprocess.check_output(["git", "ls-tree", "-rz", "--full-tree", ref]).split(b"\0") - digest = hashlib.sha256() - for entry in sorted(entries, key=lambda e: e.split(b"\t", 1)[-1]): - if not entry: - continue - metadata, name = entry.split(b"\t", 1) - path = name.decode("utf-8", "surrogateescape") - if policy: - if not (path.startswith(".github/") or path == ".asf.yaml"): - continue - elif documentation(path, metadata.split(b" ", 1)[0].decode()) and not (image and packaged_image_document(path)): - continue - digest.update(metadata + b"\t" + name + b"\0") - if project == "toolchain" and not policy: - state = packaged_readme_state(ref) - if state != "valid": - digest.update(("required-packaged-readme:" + state).encode()) - return digest.hexdigest() - - -def remote_fingerprint(repository, sha, policy_only=False, fetch=api, project=None, image=False): - tree = fetch(f"repos/{repository}/git/trees/{sha}?recursive=1") - if tree.get("truncated"): - raise ValueError("truncated proof tree") - digest = hashlib.sha256() - for entry in sorted(tree["tree"], key=lambda item: item["path"].encode()): - path = entry["path"] - if entry["type"] == "tree": - continue - if policy_only: - if not (path.startswith(".github/") or path == ".asf.yaml"): - continue - elif documentation(path, entry["mode"]) and not (image and packaged_image_document(path)): - continue - digest.update((entry["mode"] + " " + entry["type"] + " " + entry["sha"] - + "\t" + path + "\0").encode()) - if project == "toolchain" and not policy_only: - entry = next((item for item in tree["tree"] if item["path"] == "hugegraph-hubble/README.md"), None) - state = "missing" - if entry: - if entry["mode"] != "100644" or entry["type"] != "blob": - state = "invalid-mode:" + entry["mode"] + ":" + entry["sha"] - else: - blob = fetch(f"repos/{repository}/git/blobs/{entry['sha']}") - state = "valid" if base64.b64decode(blob["content"]).strip() else "empty:" + entry["sha"] - if state != "valid": - digest.update(("required-packaged-readme:" + state).encode()) - return digest.hexdigest() - - -def proof_pr(plan, run, fetch): - associations = run.get("pull_requests", []) - if any(pr.get("number") == plan["pr"] for pr in associations): - return True - owner = run.get("head_repository", {}).get("owner", {}).get("login") - branch = run.get("head_branch") - if not owner or not branch: - return False - candidates = pages(f"repos/{plan['repository']}/pulls?" + urlencode({ - "state": "open", "head": f"{owner}:{branch}"}), fetch) - return any(pr.get("number") == plan["pr"] - and pr.get("base", {}).get("repo", {}).get("full_name") == plan["repository"] - and pr.get("head", {}).get("ref") == branch - and pr.get("head", {}).get("repo", {}).get("full_name") == plan["source"] - for pr in candidates) - - def suites(project, selected): if project == "toolchain": return sorted(selected) @@ -304,229 +223,46 @@ def suites(project, selected): return sorted(result) -def pages(path, fetch=api): - result = [] - for page in range(1, 11): - value = fetch(path + ("&" if "?" in path else "?") + urlencode({"per_page": 100, "page": page})) - batch = value["jobs"] if isinstance(value, dict) else value - result.extend(batch) - if len(batch) < 100: - return result - raise ValueError("API pagination exceeded verification bound") - - -def successful_jobs(repository, run_id, suite, fetch=api): - jobs = pages(f"repos/{repository}/actions/runs/{run_id}/jobs", fetch) - matching = [job for job in jobs if job.get("name") == suite or job.get("name", "").startswith((suite + " / ", suite + " ("))] - if not matching or any(job.get("conclusion") != "success" for job in matching): - raise ValueError("missing or unsuccessful suite jobs") - return sorted(job["id"] for job in matching) - - -def validate_proof_run(plan, item, fetch=api, suite=None): - run = fetch(f"repos/{plan['repository']}/actions/runs/{int(item['runID'])}") - if (run.get("status") != "completed" - or run.get("event") != "pull_request" or run.get("head_sha") != item.get("head") - or run.get("workflow_id") != plan["workflowID"] - or run.get("head_repository", {}).get("full_name") != plan["source"] - or run.get("head_branch") != plan["branch"]): - raise ValueError("proof run is not a successful trusted workflow") - if not proof_pr(plan, run, fetch): - raise ValueError("proof run lacks matching PR association") - merge = fetch(f"repos/{plan['repository']}/git/commits/{item['testedMergeSHA']}") - if [parent["sha"] for parent in merge.get("parents", [])] != [plan["base"], item["head"]]: - raise ValueError("tested merge parents differ") - actual_input = remote_fingerprint(plan["repository"], item["testedMergeSHA"], fetch=fetch, project=plan["project"]) - if plan["project"] == "toolchain": - context = external_context(plan.get("externalInputs", {}), plan["selected"]) - actual_input = hashlib.sha256((actual_input + context).encode()).hexdigest() - if (actual_input != plan["inputFingerprint"] or remote_fingerprint( - plan["repository"], item["testedMergeSHA"], policy_only=True, fetch=fetch) - != plan["policyFingerprint"]): - raise ValueError("actual tested inputs differ from claimed receipt") - if suite in IMAGES: - if (not plan.get("imageFingerprint") or item.get("imageFingerprint") != plan["imageFingerprint"] - or remote_fingerprint(plan["repository"], item["testedMergeSHA"], fetch=fetch, - project="toolchain", image=True) != plan["imageFingerprint"]): - raise ValueError("actual Docker build inputs differ") - - -def validate_receipt(plan, receipt, fetch=api): - for key in ["schema", "repository", "project", "pr", "source", "branch", "base", "inputFingerprint", "policyFingerprint"]: - if receipt.get(key) != plan.get(key): - raise ValueError("receipt provenance differs: " + key) - proof = receipt.get("proofs", {}) - required = plan["expected"] - if not required or not (set(required) - IMAGES).issubset(proof): - raise ValueError("receipt lacks selected suites") - verified = {} - for suite in required: - if suite in IMAGES and (suite not in proof or not plan.get("imageFingerprint") - or proof[suite].get("imageFingerprint") != plan["imageFingerprint"]): - continue - item = proof[suite] - validate_proof_run(plan, item, fetch, suite=suite) - successful_jobs(plan["repository"], item["runID"], "affected-module-tests", fetch) - ids = successful_jobs(plan["repository"], item["runID"], suite, fetch) - if ids != item.get("jobIDs"): - raise ValueError("actual job evidence differs") - verified[suite] = item - if not verified: - raise ValueError("receipt contains no reusable selected suite") - return verified - +class StaleInputError(RuntimeError): + """The checkout no longer represents the current PR inputs.""" -PROVENANCE = ["schema", "repository", "project", "pr", "source", "branch", "base", - "inputFingerprint", "policyFingerprint"] - -def optional_groups(plan): - if plan["project"] == "toolchain": - return {"security": {"security / Analyze (java)", "security / Analyze (javascript)", - "security / Analyze (python)", "security / dependency-review"}} - runtime = (Path(__file__).resolve().parents[1] / "workflows/.java-version").read_text().strip() - return {"security": {"codeql / Analyze (java)"}, - "compatibility": {f"HBase compatibility (Java {runtime})", - "build-server-macos-rocksdb (macos-15-intel)", - "build-server-macos-rocksdb (macos-15, -Xms512m -Xmx2g)", - "build-server-riscv64 / build-server-riscv64"}} - - -def optional_group_jobs(jobs, group): - def belongs(name): - if group == "security": - return name.startswith(("codeql /", "security /")) - return name.startswith(("HBase compatibility", "build-server-macos-rocksdb", "build-server-riscv64 /")) - return [job for job in jobs if belongs(job.get("name", ""))] - - -def optional_jobs(jobs, names): - # Reject incomplete/duplicate matrices; proof identities must describe the full group. - matching = [job for job in jobs if job.get("name") in names] - if len(matching) != len(names) or {job["name"] for job in matching} != names: - raise ValueError("optional group is incomplete") - if any(job.get("conclusion") != "success" for job in matching): - raise ValueError("optional group did not succeed") - return sorted(matching, key=lambda job: job["name"]) - - -def optional_proofs(plan, receipt, jobs, blocked, fetch=api): - proofs = {} - for group, names in optional_groups(plan).items(): - if group in blocked: - continue - try: - current = optional_group_jobs(jobs, group) - if any(job.get("name") not in names for job in current): - raise ValueError("unknown optional group job") - # Skipped wrapper jobs can forward proof. A partial or failed execution cannot. - executed = [job for job in current if job.get("conclusion") != "skipped"] - if executed: - matching = optional_jobs(jobs, names) - item = {key: receipt[key] for key in PROVENANCE} - item.update(runID=receipt["runID"], head=receipt["head"], - testedMergeSHA=receipt["testedMergeSHA"]) - validate_proof_run(plan, item, fetch) - item.update(jobIDs=[job["id"] for job in matching], jobNames=[job["name"] for job in matching]) - else: - item = receipt.get("optionalProofs", {}).get(group) - if not item: - continue - if any(item.get(key) != plan.get(key) for key in PROVENANCE): - raise ValueError("optional provenance differs") - validate_proof_run(plan, item, fetch) - successful_jobs(plan["repository"], item["runID"], "affected-module-tests", fetch) - original = optional_group_jobs( - pages(f"repos/{plan['repository']}/actions/runs/{item['runID']}/jobs", fetch), group) - if any(job.get("name") not in names for job in original): - raise ValueError("unknown original optional group job") - matching = optional_jobs(original, names) - if (item.get("jobIDs") != [job["id"] for job in matching] - or item.get("jobNames") != [job["name"] for job in matching]): - raise ValueError("optional job evidence differs") - proofs[group] = item - except (subprocess.SubprocessError, OSError, ValueError, KeyError, TypeError, AttributeError): - continue - return proofs - - -def find_reuse(plan, fetch=api): - """Receipts are uploaded by this workflow, not read from the PR tree.""" - repository = plan["repository"] - current = fetch(f"repos/{repository}/actions/runs/{os.environ['GITHUB_RUN_ID']}") - plan["workflowID"] = current["workflow_id"] - runs = [] - for page in range(1, 6): - batch = fetch(f"repos/{repository}/actions/workflows/{plan['workflowID']}/runs?" + urlencode({ - "event": "pull_request", "status": "completed", "branch": plan["branch"], - "per_page": 30, "page": page}))["workflow_runs"] - runs.extend(batch) - if len(batch) < 30: - break - blocked_optional = set() - for run in runs: - if run["id"] == int(os.environ["GITHUB_RUN_ID"]): - continue - if (run.get("head_repository", {}).get("full_name") != plan["source"] - or run.get("head_branch") != plan["branch"]): - continue - if not proof_pr(plan, run, fetch): - continue - artifacts = fetch(f"repos/{repository}/actions/runs/{run['id']}/artifacts?per_page=100")["artifacts"] - for artifact in artifacts: - size = artifact.get("size_in_bytes") - if (artifact.get("name") != "ci-test-receipt" or artifact.get("expired") - or type(size) is not int or not 0 < size <= 1048576): - continue - try: - payload = fetch(f"repos/{repository}/actions/artifacts/{artifact['id']}/zip", binary=True) - with zipfile.ZipFile(io.BytesIO(payload)) as archive: - entries = archive.infolist() - if len(entries) != 1 or entries[0].filename != "receipt.json": - raise ValueError("unexpected receipt archive entries") - info = entries[0] - mode = (info.external_attr >> 16) & 0o170000 - if (info.is_dir() or info.flag_bits & 1 or mode not in {0, 0o100000} - or info.file_size > 1048576): - raise ValueError("oversized receipt") - receipt = json.loads(archive.read(info)) - if receipt.get("runID") != run["id"]: - raise ValueError("receipt not uploaded by claimed run") - successful_jobs(repository, run["id"], "affected-module-tests", fetch) - verified = validate_receipt(plan, receipt, fetch) - jobs = pages(f"repos/{repository}/actions/runs/{run['id']}/jobs", fetch) - plan["optionalProofs"] = optional_proofs(plan, receipt, jobs, blocked_optional, fetch) - plan["optionalSuccess"] = {group: group in plan["optionalProofs"] - for group in ["compatibility", "security"]} - return verified - except (ValueError, KeyError, TypeError, AttributeError, zipfile.BadZipFile): - continue - # Missing/unverifiable evidence from a newer same-PR run is conservative for optional lanes. - blocked_optional.update(optional_groups(plan)) - return {} +def require_current_pr(plan, fetch): + if not plan["pr"]: + return + live = fetch(f"repos/{plan['repository']}/pulls/{plan['pr']}") + if (live.get("state") != "open" or live["head"]["sha"] != plan["head"] + or live["base"]["sha"] != plan["base"] + or live["head"]["repo"]["full_name"] != plan["source"] + or live["head"]["ref"] != plan["branch"] + or live["base"]["repo"]["full_name"] != plan["repository"]): + raise StaleInputError("PR inputs changed; refresh the branch and start a new PR run") def create_plan(project, event, repository, fetch=api, external_inputs=None): plan = {"schema": 1, "project": project, "repository": repository, "pr": 0, "source": repository, "branch": "", "base": "", "head": git("rev-parse", "HEAD"), - "reused": {}, "workflowID": 0, "reason": "affected inputs", + "reason": "affected inputs from the cumulative PR diff", "externalInputs": external_inputs or {}, "testedMergeSHA": git("rev-parse", "HEAD")} selected_images = set() try: pr = event.get("pull_request") - if pr: - # Refresh PR metadata: event payload may predate a new push or base movement. - live = fetch(f"repos/{repository}/pulls/{pr['number']}") - if (live.get("state") != "open" or live["head"]["sha"] != pr["head"]["sha"] - or live["base"]["repo"]["full_name"] != repository): - raise ValueError("PR metadata changed") - plan.update(pr=pr["number"], source=live["head"]["repo"]["full_name"], - base=pr["base"]["sha"], head=live["head"]["sha"], branch=live["head"]["ref"]) + if "pull_request" in event: + if not isinstance(pr, dict) or type(pr.get("number")) is not int or pr["number"] <= 0: + raise StaleInputError("PR event lacks a valid number") + plan["pr"] = pr["number"] + # An incomplete PR event must never fall back to a push plan with pr=0. + try: + plan.update(source=pr["head"]["repo"]["full_name"], base=pr["base"]["sha"], + head=pr["head"]["sha"], branch=pr["head"]["ref"]) + except (KeyError, TypeError) as error: + raise StaleInputError("PR event lacks required input identity") from error + if not all(isinstance(plan[key], str) and plan[key] for key in ("source", "base", "head", "branch")): + raise StaleInputError("PR event has an empty input identity") parents = git("show", "-s", "--format=%P", plan["testedMergeSHA"]).split() if parents != [plan["base"], plan["head"]]: - raise ValueError("checkout is not the event PR merge") - if live["base"]["sha"] != plan["base"]: - raise ValueError("base advanced after event checkout") + raise StaleInputError("checkout is not the event PR merge; start a new PR run") + require_current_pr(plan, fetch) # head/base objects must exist locally; workflow fetches both before planning. ancestor = git("merge-base", plan["base"], plan["head"]) paths = git("diff", "--no-renames", "--name-only", ancestor, plan["head"]).splitlines() @@ -539,87 +275,54 @@ def create_plan(project, event, repository, fetch=api, external_inputs=None): selected = set(MODULES[project]) if project == "toolchain" and packaged_readme_state(plan["testedMergeSHA"]) != "valid": selected = set(MODULES[project]) - plan["selected"] = sorted(selected) - plan["inputFingerprint"] = fingerprint(plan["testedMergeSHA"], project=project) - if project == "toolchain": - context = external_context(external_inputs, selected) - plan["inputFingerprint"] = hashlib.sha256((plan["inputFingerprint"] + context).encode()).hexdigest() - plan["policyFingerprint"] = fingerprint(plan["testedMergeSHA"], policy=True) - if project == "toolchain": - plan["imageFingerprint"] = fingerprint(plan["testedMergeSHA"], project=project, image=True) - plan["expected"] = sorted(suites(project, selected) + list(selected_images)) - if pr and plan["expected"] and (project != "toolchain" or external_inputs): - plan["reused"] = find_reuse(plan, fetch) except (subprocess.SubprocessError, OSError, ValueError, KeyError, TypeError, AttributeError): selected = set(MODULES[project]) selected_images = set(IMAGES) if project == "toolchain" else set() - plan.update(reused={}, reason="verification unavailable: full required coverage") - plan.setdefault("inputFingerprint", "") - plan.setdefault("policyFingerprint", "") - plan["expected"] = sorted(suites(project, selected) + list(selected_images)) + plan["reason"] = "selection unavailable: full required coverage" if not selected and not selected_images: plan["reason"] = "cumulative PR diff contains only plain prose documentation; modules unaffected" - elif plan["reused"]: - plan["reason"] = "affected inputs match independently verified successful tests" plan["selected"] = sorted(selected) for module in MODULES[project]: - module_suites = suites(project, {module}) - plan[module] = module in selected and not (module_suites and all(s in plan["reused"] for s in module_suites)) + plan[module] = module in selected + plan["expected"] = sorted(suites(project, selected) + list(selected_images)) plan["selectedImages"] = sorted(selected_images) for image in IMAGES: - plan[image] = image in selected_images and image not in plan["reused"] + plan[image] = image in selected_images plan["pd_store"] = any(plan.get(m, False) for m in ["pd", "store", "hstore", "struct"]) plan["needsFixture"] = project == "toolchain" and any(plan.get(m, False) for m in MODULES[project]) plan["fixture"] = plan["needsFixture"] - plan["compatibility"] = not plan.get("optionalSuccess", {}).get("compatibility", False) and project == "server" and bool(selected.intersection({"server", "commons", "struct", "pd", "store", "hstore", "cluster"})) - plan["security"] = (bool(selected) and not plan.get("optionalSuccess", {}).get("security", False)) or any(p.startswith(".github/workflows/codeql") for p in locals().get("paths", [])) + plan["compatibility"] = project == "server" and bool(selected.intersection({"server", "commons", "struct", "pd", "store", "hstore", "cluster"})) + plan["security"] = bool(selected) or any(p.startswith(".github/workflows/codeql") for p in locals().get("paths", [])) plan["security_languages"] = json.dumps(["java"]) return plan -def gate(plan, results, fetch=api): +def gate(plan, results, fetch=None): if results.get("plan", {}).get("result") != "success": raise ValueError("planner did not succeed") - proofs = dict(plan["reused"]) - executed = [] for suite in plan["expected"]: - if suite in proofs: - continue if results.get(suite, {}).get("result") != "success": raise ValueError("selected suite did not succeed: " + suite) - executed.append(suite) if plan["project"] == "toolchain": - if set(executed).intersection({"client", "loader", "tools", "spark", "go", "hubble"}): + if set(plan["expected"]).intersection(MODULES["toolchain"]): if results.get("fixture", {}).get("result") != "success": raise ValueError("selected tests lack successful fixture") - if "hubble" in executed and results.get("hubble-fixture", {}).get("result") != "success": + if "hubble" in plan["expected"] and results.get("hubble-fixture", {}).get("result") != "success": raise ValueError("selected Hubble tests lack successful baseline fixture") - receipt = {key: plan[key] for key in ["schema", "repository", "project", "pr", "source", "branch", "base", - "head", "testedMergeSHA", "inputFingerprint", "policyFingerprint", "workflowID"]} - if "imageFingerprint" in plan: - receipt["imageFingerprint"] = plan["imageFingerprint"] - receipt["runID"] = int(os.environ.get("GITHUB_RUN_ID", "0")) - # Successful tests can gate even if receipt publication is unavailable. Never reuse - # unverified evidence: a missing API proof only disables future receipt reuse. - for suite in executed: - try: - ids = successful_jobs(plan["repository"], receipt["runID"], suite, fetch) - proofs[suite] = {"runID": receipt["runID"], "head": plan["head"], "testedMergeSHA": plan["testedMergeSHA"], "jobIDs": ids} - if suite in IMAGES: - proofs[suite]["imageFingerprint"] = plan.get("imageFingerprint", "") - except (subprocess.SubprocessError, OSError, ValueError, KeyError, TypeError, AttributeError): - pass - receipt["proofs"] = proofs - receipt["optionalProofs"] = plan.get("optionalProofs", {}) - receipt["executed"] = executed - receipt["reused"] = sorted(plan["reused"]) - summary = "Selection: " + plan.get("reason", "affected inputs") + "\nExecuted: " + ", ".join(executed) + "\nReused: " + ", ".join(receipt["reused"]) + require_current_pr(plan, fetch or api) + report = {key: plan[key] for key in ["schema", "repository", "project", "pr", "source", "branch", "base", + "head", "testedMergeSHA", "externalInputs"]} + report.update(runID=int(os.environ.get("GITHUB_RUN_ID", "0")), + runAttempt=int(os.environ.get("GITHUB_RUN_ATTEMPT", "1")), + executed=plan["expected"], results=results) + # These fields describe this run; they do not authorize reuse by another run. + summary = "Selection: " + plan.get("reason", "affected inputs") + "\nExecuted: " + ", ".join(plan["expected"]) if os.environ.get("GITHUB_STEP_SUMMARY"): with open(os.environ["GITHUB_STEP_SUMMARY"], "a", encoding="utf-8") as out: out.write("## Affected module tests\n" + summary + "\n\nUnselected modules: " - + ", ".join(set(MODULES[plan["project"]]) - set(plan["selected"])) + "\n") + + ", ".join(sorted(set(MODULES[plan["project"]]) - set(plan["selected"]))) + "\n") print(summary) - return receipt + return report def main(): diff --git a/.github/scripts/test_check_rerun.py b/.github/scripts/test_check_rerun.py index 225a75cb8..241210c74 100644 --- a/.github/scripts/test_check_rerun.py +++ b/.github/scripts/test_check_rerun.py @@ -14,11 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Behavioral tests for retry freshness, without credentials or network.""" -import copy -import io -import json -import zipfile +"""Behavioral tests for push retry freshness, without credentials or network.""" import importlib.util from pathlib import Path import unittest @@ -36,47 +32,31 @@ class FreshnessTest(unittest.TestCase): def setUp(self): self.repository = "hugegraph/hugegraph-toolchain" self.run = {"status": "completed", "conclusion": "failure", "run_attempt": 1, - "event": "pull_request", "head_sha": "old", "head_branch": "feature", - "head_repository": {"full_name": "alice/toolchain", "owner": {"login": "alice"}}} - self.pr = {"number": 7, "state": "open", "head": {"sha": "old", "ref": "feature", "repo": {"full_name": "alice/toolchain", "owner": {"login": "alice"}}}, - "base": {"sha": "base", "repo": {"full_name": self.repository}}} - self.run["pull_requests"] = [copy.deepcopy(self.pr)] + "event": "push", "head_sha": "old", "head_branch": "release/test"} + self.head = "old" self.calls = [] - def fetch(self, path, binary=False): - if "/artifacts?" in path: - return {"artifacts": [{"name": "ci-plan", "id": 99, "expired": False}]} - if "/actions/artifacts/" in path: - data = io.BytesIO() - with zipfile.ZipFile(data, "w") as archive: - archive.writestr("plan.json", json.dumps({"repository": self.repository, "pr": 7, - "head": "old", "base": "base", "testedMergeSHA": "merge"})) - return data.getvalue() - if "/git/commits/" in path: - return {"parents": [{"sha": "base"}, {"sha": "old"}]} + def fetch(self, path): self.calls.append(path) if "/actions/runs/" in path: return self.run - if "/commits/" in path: - return [self.pr] if "/commits?" in path: - return [{"sha": self.pr["head"]["sha"]}] - if "/pulls?" in path: - return [self.pr] - if "/pulls/" in path: - return self.pr - raise AssertionError(path) + return [{"sha": self.head}] + raise AssertionError("unnecessary external lookup: " + path) def decide(self): return checker.decide(self.repository, 42, 1, 2, self.fetch)[0] - def test_current_open_pr(self): + def test_push_checks_repository_and_encodes_branch(self): self.assertEqual("rerun", self.decide()) + self.assertIn(f"repos/{self.repository}/commits?sha=release%2Ftest&per_page=1", self.calls) + self.head = "new" + self.assertEqual("skip", self.decide()) - def test_slim_workflow_run_pr_association(self): - self.run["pull_requests"] = [{"number": 7, "base": {"repo": {"id": 1, "name": "hugegraph-toolchain", - "url": f"https://api.github.com/repos/{self.repository}"}}}] - self.assertEqual("rerun", self.decide()) + def test_pr_skips_without_pr_or_artifact_queries(self): + self.run["event"] = "pull_request" + self.assertEqual("skip", self.decide()) + self.assertEqual([f"repos/{self.repository}/actions/runs/42"], self.calls) def test_api_failure_fails_closed(self): with tempfile.TemporaryDirectory() as directory: @@ -88,61 +68,6 @@ def test_api_failure_fails_closed(self): checker.main() self.assertIn("action=skip\n", output.read_text()) - def test_closed_pr(self): - self.pr["state"] = "closed" - self.assertEqual("skip", self.decide()) - - def test_updated_pr(self): - self.pr["head"]["sha"] = "new" - self.assertEqual("skip", self.decide()) - - def test_empty_association_falls_back_to_commit(self): - self.run["pull_requests"] = [] - self.assertEqual("rerun", self.decide()) - self.assertTrue(any("/commits/old/pulls" in p for p in self.calls)) - - def test_fork_empty_associations_owner_qualified_fallback(self): - self.run.update(pull_requests=[], head_repository={"full_name": "alice/toolchain", "owner": {"login": "alice"}}) - self.pr["head"].update(ref="feature", repo={"full_name": "alice/toolchain", "owner": {"login": "alice"}}) - def fetch(path, **kwargs): - if "/commits/old/pulls" in path: - return [] - if "/pulls?" in path: - self.assertIn("head=alice%3Afeature", path) - return [self.pr] - return self.fetch(path, **kwargs) - self.assertEqual("rerun", checker.decide(self.repository, 42, 1, 2, fetch)[0]) - self.pr["head"]["repo"]["owner"]["login"] = "bob" - self.assertEqual("skip", checker.decide(self.repository, 42, 1, 2, fetch)[0]) - - def test_other_repository_pr_is_ignored(self): - self.run["pull_requests"][0]["base"]["repo"]["full_name"] = "apache/hugegraph-toolchain" - self.assertEqual("rerun", self.decide()) - self.assertTrue(any("/commits/old/pulls" in p for p in self.calls)) - - def test_foreign_pr_fallback_still_rejects_foreign_pr(self): - self.run["pull_requests"][0]["base"]["repo"]["full_name"] = "apache/hugegraph-toolchain" - self.pr["base"]["repo"]["full_name"] = "apache/hugegraph-toolchain" - self.assertEqual("skip", self.decide()) - - def test_same_head_moved_base_is_stale(self): - self.pr["base"]["sha"] = "new-base" - self.assertEqual("skip", self.decide()) - - def test_missing_immutable_plan_refuses_retry(self): - def fetch(path, **kwargs): - if "/artifacts?" in path: - return {"artifacts": []} - return self.fetch(path, **kwargs) - self.assertEqual("skip", checker.decide(self.repository, 42, 1, 2, fetch)[0]) - - def test_same_sha_other_fork_cannot_authorize_retry(self): - self.pr["head"]["repo"].update(full_name="bob/toolchain", owner={"login": "bob"}) - self.assertEqual("skip", self.decide()) - self.pr["head"]["repo"].update(full_name="alice/toolchain", owner={"login": "alice"}) - self.pr["head"]["ref"] = "other" - self.assertEqual("skip", self.decide()) - def test_changed_attempt(self): self.run["run_attempt"] = 2 self.assertEqual("skip", self.decide()) @@ -157,23 +82,22 @@ def test_current_run_must_remain_failed_and_completed(self): self.run[key] = old def test_retry_limit(self): + self.run["run_attempt"] = 2 + self.assertEqual("rerun", checker.decide(self.repository, 42, 2, 2, self.fetch)[0]) self.run["run_attempt"] = 3 self.assertEqual("skip", checker.decide(self.repository, 42, 3, 2, self.fetch)[0]) - def test_push_checks_repository_and_encodes_branch(self): - self.run.update(event="push", head_branch="release/test") - self.assertEqual("rerun", self.decide()) - self.assertIn(f"repos/{self.repository}/commits?sha=release%2Ftest&per_page=1", self.calls) - self.pr["head"]["sha"] = "new" - self.assertEqual("skip", self.decide()) - def test_unsupported_event(self): self.run["event"] = "workflow_dispatch" self.assertEqual("skip", self.decide()) def test_missing_metadata(self): - self.run["head_sha"] = None - self.assertEqual("skip", self.decide()) + for key in ["head_sha", "head_branch"]: + with self.subTest(key=key): + old = self.run[key] + self.run[key] = None + self.assertEqual("skip", self.decide()) + self.run[key] = old if __name__ == "__main__": diff --git a/.github/scripts/test_ci_policy.py b/.github/scripts/test_ci_policy.py index ea39832ae..902430492 100644 --- a/.github/scripts/test_ci_policy.py +++ b/.github/scripts/test_ci_policy.py @@ -14,12 +14,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Executable Git fixtures and API proof tests for CI selection.""" -import base64 -import hashlib +"""Executable Git fixtures for affected-module selection and current-run gating.""" import importlib.util -import io -import zipfile import json import os from pathlib import Path @@ -31,14 +27,18 @@ spec = importlib.util.spec_from_file_location("policy", Path(__file__).with_name("ci-policy.py")) policy = importlib.util.module_from_spec(spec) spec.loader.exec_module(policy) -ACTUAL_REMOTE_FINGERPRINT = policy.remote_fingerprint class PolicyTest(unittest.TestCase): + def live_pr(self): + return {"state": "open", "head": {"sha": "new", "ref": "feature", "repo": {"full_name": "alice/t"}}, + "base": {"sha": "base", "repo": {"full_name": "apache/t"}}} + def setUp(self): - mock = patch.object(policy, "remote_fingerprint", side_effect=lambda *a, **k: "policy" if k.get("policy_only") else "inputs") - mock.start() - self.addCleanup(mock.stop) + # All tests stay local; successful PR gates query only this current-PR fixture. + api_mock = patch.object(policy, "api", return_value=self.live_pr()) + self.addCleanup(api_mock.stop) + self.api_mock = api_mock.start() def test_dependency_expansion(self): self.assertEqual({"client", "loader", "tools", "spark", "hubble"}, @@ -74,80 +74,6 @@ def test_image_packaging_and_shared_build_inputs(self): "hugegraph-hubble/hubble-be/src/main/resources/application.properties"]: self.assertEqual({"hubble_image"}, policy.select_images([path]), path) - def test_image_only_plan_and_gate_require_no_server_fixture(self): - with patch.object(policy, "git", side_effect=lambda *a: ( - "hugegraph-loader/Dockerfile" if a[0] == "diff" else "head")), patch.object( - policy, "fingerprint", return_value="hash"), patch.object( - policy, "unsafe_documentation", return_value=False), patch.object( - policy, "packaged_readme_state", return_value="valid"): - plan = policy.create_plan("toolchain", {"before": "base"}, "apache/t") - self.assertEqual(["loader_image"], plan["expected"]) - self.assertFalse(plan["needsFixture"]) - self.assertFalse(any(plan[m] for m in policy.MODULES["toolchain"])) - results = {"plan": {"result": "success"}, "loader_image": {"result": "success"}} - receipt = policy.gate(plan, results, lambda _: {"jobs": [ - {"name": "loader_image / build-and-start", "id": 55, "conclusion": "success"}]}) - self.assertEqual("hash", receipt["proofs"]["loader_image"]["imageFingerprint"]) - for state in [None, "skipped", "cancelled", "failure"]: - results["loader_image"]["result"] = state - with self.subTest(state=state), self.assertRaises(ValueError): - policy.gate(plan, results, self.fetch) - - def test_cumulative_packaging_edit_reuses_java_without_fixture(self): - event = {"pull_request": {"number": 7, "head": {"sha": "new"}, "base": {"sha": "base"}}} - live = {"state": "open", "head": {"sha": "new", "ref": "feature", "repo": {"full_name": "alice/t"}}, - "base": {"sha": "base", "repo": {"full_name": "apache/t"}}} - def git(*args): - if args[0] == "show": - return "base new" - if args[0] == "diff": - return "hugegraph-client/src/A.java\nhugegraph-hubble/README.md" - return "new" - verified = {module: {"runID": 42} for module in ["client", "loader", "tools", "spark", "hubble"]} - with patch.object(policy, "git", side_effect=git), patch.object(policy, "fingerprint", return_value="hash"), \ - patch.object(policy, "unsafe_documentation", return_value=False), \ - patch.object(policy, "packaged_readme_state", return_value="valid"), \ - patch.object(policy, "find_reuse", return_value=verified): - plan = policy.create_plan("toolchain", event, "apache/t", lambda _: live, external_inputs={"sha": "server"}) - self.assertTrue(plan["hubble_image"]) - self.assertFalse(plan["loader_image"]) - self.assertFalse(plan["needsFixture"]) - self.assertFalse(any(plan[module] for module in verified)) - self.assertEqual(set(verified) | {"hubble_image"}, set(plan["expected"])) - - def test_missing_or_changed_image_proof_preserves_strict_java_proof(self): - plan = dict(self.plan(), expected=["client", "loader_image"], imageFingerprint="current") - receipt = self.receipt() - self.assertEqual({"client"}, set(policy.validate_receipt(plan, receipt, self.fetch))) - receipt["proofs"]["loader_image"] = dict(receipt["proofs"]["client"], imageFingerprint="old") - self.assertEqual({"client"}, set(policy.validate_receipt(plan, receipt, self.fetch))) - receipt["proofs"]["client"]["jobIDs"] = [999] - with self.assertRaises(ValueError): - policy.validate_receipt(plan, receipt, self.fetch) - - def test_image_proof_requires_actual_build_tree_and_exact_job_ids(self): - plan = dict(self.plan(), expected=["loader_image"], imageFingerprint="docker-inputs") - receipt = self.receipt() - receipt["proofs"] = {"loader_image": dict(receipt["proofs"]["client"], - imageFingerprint="docker-inputs")} - def fetch(path): - value = self.fetch(path) - if "/jobs?" in path: - value["jobs"][0]["name"] = "loader_image / build-and-start" - return value - def fingerprint(*args, **kwargs): - return "policy" if kwargs.get("policy_only") else "docker-inputs" if kwargs.get("image") else "inputs" - with patch.object(policy, "remote_fingerprint", side_effect=fingerprint): - self.assertEqual({"loader_image"}, set(policy.validate_receipt(plan, receipt, fetch))) - receipt["proofs"]["loader_image"]["jobIDs"] = [999] - with self.assertRaises(ValueError): - policy.validate_receipt(plan, receipt, fetch) - receipt["proofs"]["loader_image"]["jobIDs"] = [123] - with patch.object(policy, "remote_fingerprint", side_effect=lambda *a, **k: ( - "policy" if k.get("policy_only") else "forged" if k.get("image") else "inputs")): - with self.assertRaises(ValueError): - policy.validate_receipt(plan, receipt, fetch) - def test_server_backend_and_startup_dependents(self): selected = policy.select("server", ["hugegraph-server/hugegraph-hstore/src/test/HstoreTableTest.java"]) self.assertIn("hstore", selected) @@ -186,86 +112,6 @@ def test_version_resource_follows_real_docker_consumer(self): self.assertTrue((root / resource).is_file()) self.assertTrue({"commons", "docker"}.issubset(policy.select("server", [resource]))) - def test_external_hubble_baseline_only_affects_hubble(self): - a = {"server": {"sha": "release"}, "hubble": {"sha": "old"}} - b = {"server": {"sha": "release"}, "hubble": {"sha": "new"}} - self.assertEqual(policy.external_context(a, ["go"]), policy.external_context(b, ["go"])) - self.assertNotEqual(policy.external_context(a, ["hubble"]), policy.external_context(b, ["hubble"])) - - def test_packaged_readme_content_contract(self): - with tempfile.TemporaryDirectory() as directory: - root = Path(directory) - def git(*args): - return subprocess.check_output(["git", *args], cwd=root, text=True).strip() - git("init", "-q") - git("config", "user.email", "ci@example.invalid") - git("config", "user.name", "CI") - (root / "hugegraph-hubble").mkdir() - readme = root / "hugegraph-hubble/README.md" - readme.write_text("nonempty docs") - git("add", ".") - git("commit", "-qm", "valid readme") - old = os.getcwd() - try: - os.chdir(root) - initial = policy.fingerprint("HEAD", project="toolchain") - initial_image = policy.fingerprint("HEAD", project="toolchain", image=True) - readme.write_text("changed nonempty docs") - git("commit", "-qam", "valid docs edit") - self.assertEqual(initial, policy.fingerprint("HEAD", project="toolchain")) - changed_image = policy.fingerprint("HEAD", project="toolchain", image=True) - def fetch_tree(path): - if "/git/blobs/" in path: - blob = subprocess.check_output(["git", "cat-file", "blob", path.rsplit("/", 1)[1]]) - return {"content": base64.b64encode(blob).decode()} - entries = [] - for entry in subprocess.check_output(["git", "ls-tree", "-rz", "HEAD"]).split(b"\0"): - if entry: - metadata, name = entry.split(b"\t", 1) - mode, kind, sha = metadata.decode().split() - entries.append({"path": name.decode(), "mode": mode, "type": kind, "sha": sha}) - return {"tree": entries} - # Local plan hashing and API verification must agree on actual Git objects. - with patch.object(policy, "remote_fingerprint", ACTUAL_REMOTE_FINGERPRINT): - self.assertEqual(changed_image, policy.remote_fingerprint( - "apache/t", "HEAD", fetch=fetch_tree, project="toolchain", image=True)) - self.assertNotEqual(initial_image, changed_image) - source = root / "hugegraph-hubble/A.java" - source.write_text("first source") - git("add", ".") - git("commit", "-qm", "application source") - source_image = policy.fingerprint("HEAD", project="toolchain", image=True) - self.assertNotEqual(changed_image, source_image) - source.write_text("changed source") - git("commit", "-qam", "changed application") - self.assertNotEqual(source_image, policy.fingerprint("HEAD", project="toolchain", image=True)) - # Restore the original module inputs before exercising the README contract. - source.unlink() - git("add", ".") - git("commit", "-qm", "remove test source") - for content in ["", " \t\n"]: - readme.write_text(content) - git("commit", "-qam", "empty docs") - self.assertNotEqual("valid", policy.packaged_readme_state("HEAD")) - self.assertNotEqual(initial, policy.fingerprint("HEAD", project="toolchain")) - readme.write_text("nonempty") - readme.chmod(0o755) - git("add", ".") - git("commit", "-qm", "executable docs") - self.assertNotEqual("valid", policy.packaged_readme_state("HEAD")) - readme.unlink() - readme.symlink_to("missing-target") - git("add", ".") - git("commit", "-qm", "symlink docs") - self.assertNotEqual("valid", policy.packaged_readme_state("HEAD")) - readme.unlink() - git("add", ".") - git("commit", "-qm", "deleted docs") - self.assertEqual("missing", policy.packaged_readme_state("HEAD")) - self.assertNotEqual(initial, policy.fingerprint("HEAD", project="toolchain")) - finally: - os.chdir(old) - def test_single_workflow_has_no_global_fanout(self): self.assertEqual({"hubble"}, policy.select("toolchain", [".github/workflows/hubble-ci.yml"])) self.assertEqual({"docker"}, policy.select("server", [".github/workflows/docker-build-ci.yml"])) @@ -279,342 +125,129 @@ def test_strict_docs_allowlist(self): for path in ["hugegraph-client/src/test/resources/README.md", "docs/type.ts", "docs/config.yml", "docs/example.java", "hugegraph-server/type.ts"]: self.assertTrue(policy.select("server", [path])) + def plan(self): return {"schema": 1, "project": "toolchain", "repository": "apache/t", "pr": 7, - "source": "alice/t", "branch": "feature", "base": "base", "head": "new", "testedMergeSHA": "merge", "inputFingerprint": hashlib.sha256(b"inputs{}").hexdigest(), - "policyFingerprint": "policy", "workflowID": 99, "expected": ["client"], - "selected": ["client"], "reused": {}} - - def receipt(self): - r = self.plan() - r["proofs"] = {"client": {"runID": 42, "head": "old", "testedMergeSHA": "oldmerge", "jobIDs": [123]}} - return r - - def fetch(self, path): - if "/git/commits/" in path: - return {"parents": [{"sha": "base"}, {"sha": "old"}]} - if path.endswith("/jobs?per_page=100&page=1"): - return {"jobs": [{"name": "client / client-ci", "id": 123, "conclusion": "success"}, - {"name": "affected-module-tests", "id": 124, "conclusion": "success"}]} - return {"status": "completed", "conclusion": "success", "event": "pull_request", - "head_sha": "old", "head_branch": "feature", "workflow_id": 99, "head_repository": {"full_name": "alice/t"}, - "pull_requests": [{"number": 7, "base": {"sha": "base"}}]} - - def test_valid_original_proof(self): - with patch.object(policy, "remote_fingerprint", side_effect=lambda *a, **k: "policy" if k.get("policy_only") else "inputs"): - self.assertIn("client", policy.validate_receipt(self.plan(), self.receipt(), self.fetch)) - - def test_empty_fork_association_owner_branch_fallback(self): - def fetch(path): - if "/pulls?" in path: - self.assertIn("head=alice%3Afeature", path) - return [{"number": 7, "head": {"ref": "feature", "repo": {"full_name": "alice/t"}}, - "base": {"repo": {"full_name": "apache/t"}}}] - value = self.fetch(path) - if "/actions/runs/" in path and "/jobs?" not in path: - value.update(pull_requests=[], head_branch="feature", - head_repository={"full_name": "alice/t", "owner": {"login": "alice"}}) - return value - self.assertIn("client", policy.validate_receipt(self.plan(), self.receipt(), fetch)) - - def test_nonblocking_failure_keeps_core_proof_but_bad_gate_rejects(self): - def fetch(path): - value = self.fetch(path) - if "/jobs?" not in path and "/actions/runs/" in path: - value["conclusion"] = "failure" - return value - self.assertIn("client", policy.validate_receipt(self.plan(), self.receipt(), fetch)) - def bad_gate(path): - value = fetch(path) - if "/jobs?" in path: - value["jobs"][1]["conclusion"] = "failure" - return value - with self.assertRaises(ValueError): - policy.validate_receipt(self.plan(), self.receipt(), bad_gate) - - def test_artifact_core_reuse_with_failed_compatibility(self): - receipt = self.receipt() - receipt["runID"] = 42 - data = io.BytesIO() - with zipfile.ZipFile(data, "w") as archive: - archive.writestr("receipt.json", json.dumps(receipt)) - artifact_size = 256 - def fetch(path, binary=False): - if "/actions/workflows/" in path: - self.assertIn("status=completed", path) - return {"workflow_runs": [{"id": 42, "head_repository": {"full_name": "alice/t"}, "head_branch": "feature", "pull_requests": [{"number": 7}]}]} - if "/artifacts?" in path: - return {"artifacts": [{"name": "ci-test-receipt", "id": 3, "expired": False, "size_in_bytes": artifact_size}]} - if "/actions/artifacts/" in path: - return data.getvalue() - value = self.fetch(path) - if "/jobs?" in path: - value["jobs"].append({"name": "HBase compatibility", "id": 9, "conclusion": "failure"}) - elif "/actions/runs/" in path: - value["conclusion"] = "failure" - return value - plan = self.plan() - with patch.dict(os.environ, {"GITHUB_RUN_ID": "99"}): - proof = policy.find_reuse(plan, fetch) - self.assertIn("client", proof) - self.assertFalse(plan["optionalSuccess"]["compatibility"]) - self.assertFalse(plan["optionalSuccess"]["security"]) - for artifact_size in [None, 0, 1048577, "256"]: - with patch.dict(os.environ, {"GITHUB_RUN_ID": "99"}): - self.assertEqual({}, policy.find_reuse(self.plan(), fetch)) - artifact_size = 256 - data = io.BytesIO() - with zipfile.ZipFile(data, "w") as archive: - archive.writestr("receipt.json", json.dumps(receipt)) - archive.writestr("unexpected.txt", "extra") - with patch.dict(os.environ, {"GITHUB_RUN_ID": "99"}): - self.assertEqual({}, policy.find_reuse(self.plan(), fetch)) - - def optional_fixture(self, receipts, optional_results=None): - optional_results = optional_results or {} - def fetch(path, binary=False): - if "/actions/workflows/" in path: - return {"workflow_runs": [{"id": run_id, "head_repository": {"full_name": "alice/t"}, - "head_branch": "feature", "pull_requests": [{"number": 7}]} - for run_id in sorted(receipts, reverse=True)]} - if "/artifacts?" in path: - run_id = int(path.split("/runs/")[1].split("/")[0]) - return {"artifacts": [{"name": "ci-test-receipt", "id": run_id, - "expired": False, "size_in_bytes": 512}]} - if "/actions/artifacts/" in path: - run_id = int(path.split("/artifacts/")[1].split("/")[0]) - data = io.BytesIO() - with zipfile.ZipFile(data, "w") as archive: - archive.writestr("receipt.json", json.dumps(receipts[run_id])) - return data.getvalue() - if "/git/commits/" in path: - head = "new" if path.endswith("/merge") else "old" - return {"parents": [{"sha": "base"}, {"sha": head}]} - if "/jobs?" in path: - run_id = int(path.split("/runs/")[1].split("/")[0]) - jobs = [{"name": "affected-module-tests", "id": run_id * 10, "conclusion": "success"}] - if run_id == 42: - jobs.append({"name": "client / client-ci", "id": 123, "conclusion": "success"}) - for index, name in enumerate(sorted(policy.optional_groups(self.plan())["security"])): - result = optional_results.get(run_id, "success" if run_id == 42 else "skipped") - jobs.append({"name": name, "id": run_id * 100 + index, "conclusion": result}) - return {"jobs": jobs} - value = self.fetch(path) - if "/actions/runs/43" in path: - value["head_sha"] = "new" - return value - return fetch - - def find_optional(self, receipts, optional_results=None): - plan = self.plan() - with patch.dict(os.environ, {"GITHUB_RUN_ID": "99"}), patch.object( - policy, "remote_fingerprint", side_effect=lambda *a, **k: "policy" if k.get("policy_only") else "inputs"): - plan["reused"] = policy.find_reuse(plan, self.optional_fixture(receipts, optional_results)) - return plan - - def test_optional_original_proof_survives_two_documentation_updates(self): - original = self.receipt() - original.update(runID=42, head="old", testedMergeSHA="oldmerge") - second = self.find_optional({42: original}) - self.assertTrue(second["optionalSuccess"]["security"]) - self.assertEqual(42, second["optionalProofs"]["security"]["runID"]) - with patch.dict(os.environ, {"GITHUB_RUN_ID": "43"}): - forwarded = policy.gate(second, {"plan": {"result": "success"}}, self.fetch) - third = self.find_optional({43: forwarded, 42: original}) - self.assertTrue(third["optionalSuccess"]["security"]) - self.assertEqual(second["optionalProofs"], third["optionalProofs"]) - self.assertEqual(second["reused"], third["reused"]) - for result in ["failure", "cancelled", "timed_out", None]: - with self.subTest(result=result): - latest = self.find_optional({43: forwarded, 42: original}, {43: result}) - self.assertTrue(latest["reused"]) - self.assertFalse(latest["optionalSuccess"]["security"]) - - def test_newer_unverifiable_receipt_blocks_older_optional_green(self): - original = self.receipt() - original.update(runID=42, head="old", testedMergeSHA="oldmerge") - invalid = dict(original, runID=43, source="other/t") - plan = self.find_optional({43: invalid, 42: original}) - self.assertTrue(plan["reused"]) - self.assertFalse(plan["optionalSuccess"]["security"]) - - def test_server_optional_matrix_follows_project_runtime(self): - plan = dict(self.plan(), project="server") - runtime_file = Path(__file__).resolve().parents[1] / "workflows/.java-version" - if not runtime_file.exists(): - self.skipTest("Server runtime configuration is not in the Toolchain repository") - runtime = runtime_file.read_text().strip() - names = policy.optional_groups(plan)["compatibility"] - self.assertIn(f"HBase compatibility (Java {runtime})", names) - self.assertEqual(4, len(names)) - jobs = [{"name": name, "id": index, "conclusion": "success"} - for index, name in enumerate(sorted(names))] - self.assertEqual(4, len(policy.optional_jobs(jobs, names))) - for missing in range(len(jobs)): - with self.assertRaises(ValueError): - policy.optional_jobs(jobs[:missing] + jobs[missing + 1:], names) - - def test_optional_forwarded_forgery_and_original_failure_reject(self): - original = self.receipt() - original.update(runID=42, head="old", testedMergeSHA="oldmerge") - second = self.find_optional({42: original}) - with patch.dict(os.environ, {"GITHUB_RUN_ID": "43"}): - forwarded = policy.gate(second, {"plan": {"result": "success"}}, self.fetch) - for field in ["jobIDs", "jobNames", "source", "base", "inputFingerprint", "policyFingerprint", "head"]: - changed = json.loads(json.dumps(forwarded)) - changed["optionalProofs"]["security"][field] = [999] if field == "jobIDs" else "forged" - with self.subTest(field=field): - latest = self.find_optional({43: changed, 42: original}) - self.assertTrue(latest["reused"]) - self.assertFalse(latest["optionalSuccess"]["security"]) - latest = self.find_optional({43: forwarded, 42: original}, {42: "failure"}) - self.assertFalse(latest["optionalSuccess"]["security"]) - - def test_optional_group_requires_complete_known_matrix(self): - plan = self.plan() - original = self.receipt() - original.update(runID=42, head="old", testedMergeSHA="oldmerge") - fetch = self.optional_fixture({42: original}) - for mutation in ["missing", "unknown", "duplicate", "api-error"]: - def bad_fetch(path, binary=False): - if "/actions/runs/" in path and mutation == "api-error": - raise OSError("unavailable") - value = fetch(path, binary) - if "/jobs?" in path: - if mutation == "missing": - value["jobs"].pop() - elif mutation == "unknown": - value["jobs"].append({"name": "security / Analyze (unknown)", "id": 999, "conclusion": "success"}) - elif mutation == "duplicate": - value["jobs"].append(dict(value["jobs"][-1])) - return value - jobs = fetch("repos/apache/t/actions/runs/42/jobs?per_page=100&page=1")["jobs"] - if mutation != "api-error": - jobs = bad_fetch("repos/apache/t/actions/runs/42/jobs?per_page=100&page=1")["jobs"] - with patch.object(policy, "remote_fingerprint", side_effect=lambda *a, **k: "policy" if k.get("policy_only") else "inputs"): - self.assertEqual({}, policy.optional_proofs(plan, original, jobs, set(), bad_fetch)) - - def test_branch_filtered_bounded_run_pagination(self): - calls = [] - def fetch(path, binary=False): - if "/actions/workflows/" in path: - calls.append(path) - self.assertIn("branch=feature", path) - if "page=1" in path: - return {"workflow_runs": [{"id": n, "head_repository": {"full_name": "alice/t"}, - "head_branch": "other"} for n in range(30)]} - return {"workflow_runs": []} - return self.fetch(path) - with patch.dict(os.environ, {"GITHUB_RUN_ID": "99"}): - self.assertEqual({}, policy.find_reuse(self.plan(), fetch)) - self.assertEqual(2, len(calls)) - - def test_tested_merge_parent_mismatch_rejects(self): + "source": "alice/t", "branch": "feature", "base": "base", "head": "new", + "testedMergeSHA": "merge", "expected": ["client"], "selected": ["client"], + "externalInputs": {"toolchainJDK": ["11", "17"]}} + + def pr_plan(self, paths): + live = self.live_pr() + event = {"pull_request": dict(live, number=7)} + def git(*args): + if args[0] == "show": + return "base new" + if args[0] == "diff": + return "\n".join(paths) + return "merge" def fetch(path): - if "/git/commits/" in path: - return {"parents": [{"sha": "advanced-base"}, {"sha": "old"}]} - return self.fetch(path) - with self.assertRaises(ValueError): - policy.validate_receipt(self.plan(), self.receipt(), fetch) - - def test_actual_tree_fingerprint_mismatch_rejects(self): - with patch.object(policy, "remote_fingerprint", return_value="forged"): - with self.assertRaises(ValueError): - policy.validate_receipt(self.plan(), self.receipt(), self.fetch) - - def test_provenance_changes_refuse_reuse(self): - for key in ["base", "inputFingerprint", "policyFingerprint", "source", "repository", "pr", "schema"]: - receipt = self.receipt() - receipt[key] = "changed" - with self.subTest(key=key), self.assertRaises(ValueError): - policy.validate_receipt(self.plan(), receipt, self.fetch) - - def test_original_run_and_actual_job_proof_required(self): - for failure in ["failure", "cancelled", "skipped"]: - def fetch(path): - value = self.fetch(path) - if "jobs?" in path: - value["jobs"][0]["conclusion"] = failure - return value - with self.subTest(failure=failure), self.assertRaises(ValueError): - policy.validate_receipt(self.plan(), self.receipt(), fetch) - receipt = self.receipt() - receipt["proofs"]["client"]["jobIDs"] = [456] - with self.assertRaises(ValueError): - policy.validate_receipt(self.plan(), receipt, self.fetch) + self.assertEqual("repos/apache/t/pulls/7", path) + return live + with patch.object(policy, "git", side_effect=git), patch.object( + policy, "unsafe_documentation", return_value=False), patch.object( + policy, "packaged_readme_state", return_value="valid"): + return policy.create_plan("toolchain", event, "apache/t", fetch) + + def test_cumulative_pr_docs_update_runs_affected_consumers_again(self): + for document in ["README.md", "hugegraph-hubble/README.md"]: + with self.subTest(document=document): + plan = self.pr_plan(["hugegraph-client/src/A.java", document]) + self.assertEqual({"client", "loader", "tools", "spark", "hubble"}, set(plan["selected"])) + self.assertTrue(plan["needsFixture"]) + self.assertTrue(all(plan[module] for module in plan["selected"])) + self.assertEqual(document.startswith("hugegraph-hubble/"), plan["hubble_image"]) + + def test_pure_documentation_pr_has_no_modules_or_fixtures(self): + plan = self.pr_plan(["README.md", "docs/ci.md"]) + self.assertEqual([], plan["expected"]) + self.assertFalse(plan["needsFixture"]) + policy.gate(plan, {"plan": {"result": "success"}}) + + def test_packaged_readme_selects_image_on_plain_documentation_pr(self): + plan = self.pr_plan(["hugegraph-hubble/README.md"]) + self.assertEqual(["hubble_image"], plan["expected"]) + self.assertFalse(plan["needsFixture"]) + + def test_image_only_plan_and_gate_require_no_server_fixture(self): + plan = self.pr_plan(["hugegraph-loader/Dockerfile"]) + self.assertEqual(["loader_image"], plan["expected"]) + self.assertFalse(plan["needsFixture"]) + results = {"plan": {"result": "success"}, "loader_image": {"result": "success"}} + self.assertEqual(["loader_image"], policy.gate(plan, results)["executed"]) + for state in [None, "skipped", "cancelled", "failure"]: + results["loader_image"]["result"] = state + with self.subTest(state=state), self.assertRaises(ValueError): + policy.gate(plan, results) def test_gate_rejects_missing_failed_cancelled_skipped(self): for result in [None, "failure", "cancelled", "skipped"]: with self.subTest(result=result), self.assertRaises(ValueError): - policy.gate(self.plan(), {"plan": {"result": "success"}, "client": {"result": result}}, self.fetch) - - def test_gate_success_receipt_and_verified_reuse(self): - with patch.dict(os.environ, {"GITHUB_RUN_ID": "42"}): - result = policy.gate(self.plan(), {"plan": {"result": "success"}, "client": {"result": "success"}, "fixture": {"result": "success"}}, self.fetch) - self.assertEqual([123], result["proofs"]["client"]["jobIDs"]) - plan = self.plan() - plan["reused"] = result["proofs"] - self.assertEqual(["client"], policy.gate(plan, {"plan": {"result": "success"}, "client": {"result": "skipped"}}, self.fetch)["reused"]) - - def test_gate_refuses_failed_planner_even_empty_or_reused(self): + policy.gate(self.plan(), {"plan": {"result": "success"}, "client": {"result": result}, + "fixture": {"result": "success"}}) + + def test_gate_records_current_attempt_without_external_proof(self): + results = {"plan": {"result": "success"}, "client": {"result": "success"}, + "fixture": {"result": "success"}} + with patch.dict(os.environ, {"GITHUB_RUN_ID": "42", "GITHUB_RUN_ATTEMPT": "2"}), patch.object( + policy, "api", return_value=self.live_pr()) as fetch: + report = policy.gate(self.plan(), results) + fetch.assert_called_once_with("repos/apache/t/pulls/7") + self.assertEqual(["client"], report["executed"]) + self.assertEqual(results, report["results"]) + self.assertEqual(42, report["runID"]) + self.assertEqual(2, report["runAttempt"]) + + def test_old_receipt_cannot_satisfy_current_selected_failure(self): + plan = dict(self.plan(), reused={"client": {"runID": 41, "jobIDs": [123]}}) + for state in ["failure", "skipped"]: + with self.subTest(state=state), self.assertRaises(ValueError): + policy.gate(plan, {"plan": {"result": "success"}, "client": {"result": state}, + "fixture": {"result": "success"}}) + + def test_incomplete_pr_event_never_becomes_a_push_plan(self): + event = {"pull_request": dict(self.live_pr(), number=7)} + cases = [] + for change in ("null_repo", "missing_repo", "missing_number", "empty_ref"): + altered = json.loads(json.dumps(event)) + pr = altered["pull_request"] + if change == "null_repo": + pr["head"]["repo"] = None + elif change == "missing_repo": + del pr["head"]["repo"] + elif change == "missing_number": + del pr["number"] + else: + pr["head"]["ref"] = "" + cases.append(altered) + cases.extend([{"pull_request": {}}, {"pull_request": None}]) + with patch.object(policy, "git", return_value="merge"): + for altered in cases: + with self.subTest(event=altered), self.assertRaises(policy.StaleInputError): + policy.create_plan('toolchain', altered, 'apache/t', + lambda _: self.fail("Incomplete identity must fail before any API query")) + + def test_gate_refuses_failed_planner_even_empty_selection(self): for result in ["failure", "skipped", "cancelled", None]: - plan = self.plan() - plan["expected"] = [] + plan = dict(self.plan(), expected=[]) with self.subTest(result=result), self.assertRaises(ValueError): - policy.gate(plan, {"plan": {"result": result}}, self.fetch) + policy.gate(plan, {"plan": {"result": result}}) def test_fixture_missing_rejects_successful_test(self): with self.assertRaises(ValueError): - policy.gate(self.plan(), {"plan": {"result": "success"}, "client": {"result": "success"}}, self.fetch) - plan = self.plan() - plan["expected"] = ["hubble"] - with self.assertRaises(ValueError): - policy.gate(plan, {"plan": {"result": "success"}, "hubble": {"result": "success"}}, self.fetch) + policy.gate(self.plan(), {"plan": {"result": "success"}, "client": {"result": "success"}}) def test_hubble_gate_requires_current_and_released_fixtures(self): - plan = self.plan() - plan['expected'] = ['hubble'] - for producer in ('fixture', 'hubble-fixture'): - for state in (None, 'failure', 'cancelled', 'skipped'): - results = {'plan': {'result': 'success'}, 'hubble': {'result': 'success'}, - 'fixture': {'result': 'success'}, 'hubble-fixture': {'result': 'success'}} - results[producer] = {'result': state} + plan = dict(self.plan(), expected=["hubble"]) + for producer in ("fixture", "hubble-fixture"): + for state in (None, "failure", "cancelled", "skipped"): + results = {"plan": {"result": "success"}, "hubble": {"result": "success"}, + "fixture": {"result": "success"}, "hubble-fixture": {"result": "success"}} + results[producer] = {"result": state} with self.subTest(producer=producer, state=state), self.assertRaises(ValueError): - policy.gate(plan, results, self.fetch) + policy.gate(plan, results) - def test_proof_failure_disables_receipt_without_falsifying_gate(self): - def failure(path): - raise subprocess.CalledProcessError(1, "gh") - result = policy.gate(self.plan(), {"plan": {"result": "success"}, "client": {"result": "success"}, - "fixture": {"result": "success"}}, failure) - self.assertEqual({}, result["proofs"]) - self.assertEqual(["client"], result["executed"]) - - def test_doc_update_reuses_only_with_valid_receipt(self): - event = {"pull_request": {"number": 7, "head": {"sha": "new"}, "base": {"sha": "base"}}} - live = {"state": "open", "head": {"sha": "new", "ref": "feature", "repo": {"full_name": "alice/t"}}, - "base": {"sha": "base", "repo": {"full_name": "apache/t"}}} - def git(*args): - if args[0] == "show": - return "base new" - if args[0] == "diff": - return "hugegraph-client/src/A.java\nREADME.md" - return "new" - with patch.object(policy, "packaged_readme_state", return_value="valid"), patch.object(policy, "unsafe_documentation", return_value=False), patch.object(policy, "git", side_effect=git), patch.object(policy, "fingerprint", return_value="hash"), patch.object( - policy, "find_reuse", return_value={m: {"runID": 42} for m in ["client", "loader", "tools", "spark", "hubble"]}): - plan = policy.create_plan("toolchain", event, "apache/t", lambda p: live, - external_inputs={"serverSHA": "immutable"}) - self.assertFalse(plan["client"]) - self.assertFalse(plan["needsFixture"]) - self.assertTrue(plan["security"]) - with patch.object(policy, "packaged_readme_state", return_value="valid"), patch.object(policy, "unsafe_documentation", return_value=False), patch.object(policy, "git", side_effect=git), patch.object(policy, "fingerprint", return_value="hash"), patch.object( - policy, "find_reuse", return_value={}): - plan = policy.create_plan("toolchain", event, "apache/t", lambda p: live, - external_inputs={"serverSHA": "immutable"}) - self.assertTrue(plan["client"]) - self.assertTrue(plan["needsFixture"]) - - def test_real_git_fingerprint_and_cli(self): + def test_real_git_doc_modes_and_packaged_readme_contract(self): with tempfile.TemporaryDirectory() as directory: root = Path(directory) def git(*args): @@ -623,62 +256,126 @@ def git(*args): git("config", "user.email", "ci@example.invalid") git("config", "user.name", "CI") (root / "README.md").write_text("initial") + (root / "hugegraph-hubble").mkdir() + readme = root / "hugegraph-hubble/README.md" + readme.write_text("nonempty docs") git("add", ".") git("commit", "-qm", "base") base = git("rev-parse", "HEAD") - (root / "README.md").write_text("docs change") - git("commit", "-qam", "docs") - head = git("rev-parse", "HEAD") old = os.getcwd() try: os.chdir(root) - self.assertEqual(policy.fingerprint(base), policy.fingerprint(head)) - (root / "type.ts").write_text("type changed") + self.assertEqual("valid", policy.packaged_readme_state("HEAD")) + for content in ["changed docs", "", " \t\n"]: + readme.write_text(content) + git("commit", "-qam", "docs") + self.assertEqual(content.strip() != "", policy.packaged_readme_state("HEAD") == "valid") + readme.write_text("nonempty") + readme.chmod(0o755) git("add", ".") - git("commit", "-qm", "types") - self.assertNotEqual(policy.fingerprint(base), policy.fingerprint("HEAD")) - (root / "README.md").chmod(0o755) - git("add", "README.md") - git("commit", "-qm", "executable prose") - self.assertTrue(policy.unsafe_documentation("HEAD", ["README.md"])) - self.assertNotEqual(policy.fingerprint(base), policy.fingerprint("HEAD")) - (root / "README.md").unlink() - (root / "README.md").symlink_to("type.ts") - git("add", "README.md") - git("commit", "-qm", "symlink prose") - self.assertTrue(policy.unsafe_documentation("HEAD", ["README.md"])) - (root / "hugegraph-server").mkdir() - (root / "hugegraph-server" / "A.java").write_text("source") + git("commit", "-qm", "executable docs") + self.assertTrue(policy.unsafe_documentation("HEAD", ["hugegraph-hubble/README.md"])) + self.assertNotEqual("valid", policy.packaged_readme_state("HEAD")) + readme.unlink() + readme.symlink_to("missing-target") + git("add", ".") + git("commit", "-qm", "symlink docs") + self.assertTrue(policy.unsafe_documentation("HEAD", ["hugegraph-hubble/README.md"])) + self.assertNotEqual("valid", policy.packaged_readme_state("HEAD")) + readme.unlink() git("add", ".") - git("commit", "-qm", "source") - before_rename = git("rev-parse", "HEAD") - (root / "docs").mkdir() - git("mv", "hugegraph-server/A.java", "docs/renamed.md") - git("commit", "-qm", "rename code to prose") - changed = git("diff", "--no-renames", "--name-only", before_rename, "HEAD").splitlines() - self.assertIn("server", policy.select("server", changed)) + git("commit", "-qm", "deleted docs") + self.assertEqual("missing", policy.packaged_readme_state("HEAD")) finally: os.chdir(old) event = root / "event.json" event.write_text(json.dumps({"before": base})) output = root / "output" - subprocess.run(["python3", str(Path(policy.__file__).resolve()), "plan", "--project", "server", - "--repository", "apache/server", "--event-path", str(event), + subprocess.run(["python3", str(Path(policy.__file__).resolve()), "plan", "--project", "toolchain", + "--repository", "apache/t", "--event-path", str(event), "--output", str(root / "plan.json")], cwd=root, check=True, env={**os.environ, "GITHUB_OUTPUT": str(output)}) - self.assertIn("server=true\n", output.read_text()) - self.assertEqual(policy.suites("server", set(policy.MODULES["server"])), - json.loads((root / "plan.json").read_text())["expected"]) - - def test_api_failure_runs_full_required(self): - event = {"pull_request": {"number": 7, "head": {"sha": "old"}}} - with patch.object(policy, "git", return_value="head"): - def fail(path): - raise subprocess.CalledProcessError(1, "gh") - plan = policy.create_plan("toolchain", event, "apache/t", fail) + self.assertIn("client=true\n", output.read_text()) + plan = json.loads((root / "plan.json").read_text()) self.assertEqual(set(policy.MODULES["toolchain"]), set(plan["selected"])) - self.assertTrue(plan["security"]) - self.assertEqual({}, plan["reused"]) + self.assertEqual({"hubble_image"}, set(plan["selectedImages"])) + + def test_api_failure_runs_full_required_without_losing_pr_identity(self): + event = {"pull_request": dict(self.live_pr(), number=7)} + def git(*args): + return "base new" if args[0] == "show" else "merge" + def fail(path): + raise subprocess.CalledProcessError(1, "gh") + with patch.object(policy, "git", side_effect=git): + plan = policy.create_plan("toolchain", event, "apache/t", fail) + self.assertEqual(set(policy.MODULES["toolchain"]), set(plan["selected"])) + self.assertEqual((7, "alice/t", "feature", "base", "new"), + tuple(plan[key] for key in ["pr", "source", "branch", "base", "head"])) + results = {suite: {"result": "success"} for suite in plan["expected"]} + results.update(plan={"result": "success"}, fixture={"result": "success"}, + **{"hubble-fixture": {"result": "success"}}) + with self.assertRaises(subprocess.CalledProcessError): + policy.gate(plan, results, fail) + self.assertEqual(plan["expected"], policy.gate(plan, results)["executed"]) + self.api_mock.assert_called_once_with("repos/apache/t/pulls/7") + + def test_real_pr_git_snapshot_and_final_gate_freshness(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + def git(*args): + return subprocess.check_output(["git", *args], cwd=root, text=True).strip() + git("init", "-q") + git("config", "user.email", "ci@example.invalid") + git("config", "user.name", "CI") + (root / "README.md").write_text("base") + git("add", ".") + git("commit", "-qm", "base") + base = git("rev-parse", "HEAD") + git("checkout", "-qb", "feature") + (root / "hugegraph-client").mkdir() + (root / "hugegraph-client/A.java").write_text("source") + git("add", ".") + git("commit", "-qm", "source") + head = git("rev-parse", "HEAD") + tree = git("rev-parse", "HEAD^{tree}") + merge = git("commit-tree", tree, "-p", base, "-p", head, "-m", "PR merge") + advanced_head = git("commit-tree", tree, "-p", head, "-m", "new source head") + advanced_base = git("commit-tree", git("rev-parse", base + "^{tree}"), + "-p", base, "-m", "new target base") + git("checkout", "--detach", "-q", merge) + live = self.live_pr() + live["head"]["sha"], live["base"]["sha"] = head, base + event = {"pull_request": dict(live, number=7)} + old = os.getcwd() + try: + os.chdir(root) + plan = policy.create_plan("toolchain", event, "apache/t", lambda _: live) + results = {suite: {"result": "success"} for suite in plan["expected"]} + results.update(plan={"result": "success"}, fixture={"result": "success"}, + **{"hubble-fixture": {"result": "success"}}) + self.assertEqual(plan["expected"], policy.gate(plan, results, lambda _: live)["executed"]) + for section, field, value in [("head", "sha", advanced_head), ("base", "sha", advanced_base), + ("head", "ref", "other-branch"), + ("head", "repo", {"full_name": "other/fork"}), + ("base", "repo", {"full_name": "other/target"})]: + changed = dict(live, **{section: dict(live[section], **{field: value})}) + with self.subTest(section=section, field=field): + with self.assertRaises(policy.StaleInputError): + policy.create_plan("toolchain", event, "apache/t", lambda _: changed) + with self.assertRaises(policy.StaleInputError): + policy.gate(plan, results, lambda _: changed) + with self.assertRaises(policy.StaleInputError): + policy.gate(plan, results, lambda _: dict(live, state="closed")) + git("checkout", "--detach", "-q", head) + with self.assertRaises(policy.StaleInputError): + policy.create_plan("toolchain", event, "apache/t", lambda _: live) + finally: + os.chdir(old) + + def test_push_gate_does_not_query_pr(self): + plan = dict(self.plan(), pr=0, expected=[], selected=[]) + policy.gate(plan, {"plan": {"result": "success"}}, + lambda _: self.fail("push gate must not query a PR")) if __name__ == "__main__": diff --git a/.github/scripts/test_toolchain_image.py b/.github/scripts/test_toolchain_image.py index e2eb33eef..3649062e3 100644 --- a/.github/scripts/test_toolchain_image.py +++ b/.github/scripts/test_toolchain_image.py @@ -17,7 +17,11 @@ """Prevent a tag resolving to an unrelated image from providing runtime evidence.""" import importlib.util +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from pathlib import Path +import subprocess +import tempfile +import threading import unittest from unittest.mock import patch @@ -54,3 +58,100 @@ def test_hubble_exit_is_not_readiness_success(self): with patch.object(images, "output", side_effect=["127.0.0.1:12345", "false"]): with self.assertRaisesRegex(RuntimeError, "exited before becoming ready"): images.wait_hubble("hubble") + + def test_loader_executes_the_public_script_with_its_permissions_and_shebang(self): + with tempfile.TemporaryDirectory() as temporary: + script = Path(temporary) / "bin/hugegraph-loader.sh" + script.parent.mkdir() + script.write_text('#!/bin/sh\nprintf "Usage: loader --help\\n"\n') + + def execute(*args): + self.assertEqual(args[:3], ("docker", "exec", "loader")) + return subprocess.check_output(args[3:], cwd=temporary, text=True) + + with patch.object(images, "output", side_effect=execute): + script.chmod(0o755) + images.check_loader("loader") + script.chmod(0o644) + with self.assertRaises(PermissionError): + images.check_loader("loader") + script.chmod(0o755) + script.write_text('#!/missing/interpreter\nprintf "Usage: loader --help\\n"\n') + with self.assertRaises(FileNotFoundError): + images.check_loader("loader") + + +class HubbleEntryTest(unittest.TestCase): + def setUp(self): + self.routes = { + "/": ("text/html", '
' + ''), + "/main.js": ("application/javascript", 'document.getElementById("root");'), + "/main.css": ("text/css", "body { margin: 0; }"), + } + self.requested = [] + routes, requested = self.routes, self.requested + + class Handler(BaseHTTPRequestHandler): + def do_GET(self): + requested.append(self.path) + content_type, body = routes.get(self.path, ("text/html", "")) + if content_type == "redirect": + self.send_response(302) + self.send_header("Location", body) + self.end_headers() + return + self.send_response(200 if self.path in routes else 404) + self.send_header("Content-Type", content_type) + self.end_headers() + self.wfile.write(body.encode()) + + def log_message(self, *args): + pass + + self.server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + self.thread = threading.Thread(target=self.server.serve_forever, daemon=True) + self.thread.start() + self.origin = f"http://127.0.0.1:{self.server.server_port}" + + def tearDown(self): + self.server.shutdown() + self.server.server_close() + self.thread.join() + + def test_checks_root_javascript_and_stylesheets(self): + images.check_hubble_ui(self.origin) + self.assertEqual(self.requested, ["/", "/main.js", "/main.css"]) + + def test_rejects_a_backend_only_page_and_an_unbuilt_template(self): + for page in ['backend ready', '
']: + self.routes["/"] = ("text/html", page) + with self.subTest(page=page), self.assertRaises(RuntimeError): + images.check_hubble_ui(self.origin) + + def test_rejects_missing_assets_including_spa_html_fallback_and_empty_success(self): + for path in ["/main.js", "/main.css"]: + original = self.routes[path] + for content_type, body in [("text/html", self.routes["/"][1]), + (original[0], " \n"), + (original[0], "fallback")]: + self.routes[path] = (content_type, body) + with self.subTest(path=path, content_type=content_type, body=body), self.assertRaises(RuntimeError): + images.check_hubble_ui(self.origin) + del self.routes[path] + with self.assertRaises(images.urllib.error.HTTPError): + images.check_hubble_ui(self.origin) + self.routes[path] = original + + def test_does_not_fetch_external_scripts(self): + self.routes["/"] = ("text/html", '
' + '') + with self.assertRaisesRegex(RuntimeError, "missing a local JavaScript bundle"): + images.check_hubble_ui(self.origin) + self.assertEqual(self.requested, ["/"]) + + def test_does_not_follow_asset_redirects_outside_the_container(self): + self.routes["/main.js"] = ("redirect", "https://external.invalid/app.js") + with self.assertRaises(images.urllib.error.HTTPError): + images.check_hubble_ui(self.origin) + self.assertEqual(self.requested, ["/", "/main.js"]) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2ee2b8089..3ceeafb60 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -62,8 +62,17 @@ jobs: python3 -m unittest discover -s .github/scripts -p 'test_*.py' python3 .github/actions/setup-hugegraph-server/runtime_test.py node --test .github/actions/get-hugegraph-commit/resolve.test.cjs hugegraph-client/assembly/travis/checkout-server.test.cjs hugegraph-client/assembly/travis/install-server.test.cjs + - name: Determine external test input scope + id: scope + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: >- + python3 .github/scripts/ci-policy.py plan --project toolchain + --event-path "$GITHUB_EVENT_PATH" --repository "$GITHUB_REPOSITORY" + --output scope.json - name: Resolve historical Server 1.7 baseline id: baseline + if: steps.scope.outputs.needsFixture == 'true' uses: ./.github/actions/get-hugegraph-commit with: repository: apache/hugegraph @@ -71,6 +80,7 @@ jobs: expected-commit: b12425c2032bf0d21a97b8221f42a18055c2982f - name: Resolve Hubble's existing Server master baseline id: hubble-baseline + if: steps.scope.outputs.hubble == 'true' env: SERVER_REPOSITORY: https://github.com/apache/hugegraph.git run: | @@ -79,6 +89,8 @@ jobs: echo "commit=$commit" >> "$GITHUB_OUTPUT" - name: Record external test inputs env: + NEEDS_SERVER: ${{ steps.scope.outputs.needsFixture }} + NEEDS_HUBBLE: ${{ steps.scope.outputs.hubble }} SERVER_REPOSITORY: ${{ steps.baseline.outputs.repository }} SERVER_COMMIT: ${{ steps.baseline.outputs.commit_id }} SERVER_REF: ${{ steps.baseline.outputs.fetch_ref }} @@ -86,17 +98,23 @@ jobs: run: | python3 - <<'PYINPUTS' import json, os - inputs = { - 'server': {'repository': os.environ['SERVER_REPOSITORY'], 'commit': os.environ['SERVER_COMMIT'], - 'ref': os.environ['SERVER_REF'], 'jdk': '11'}, - 'hubble': {'repository': 'apache/hugegraph', 'commit': os.environ['HUBBLE_COMMIT'], - 'ref': os.environ['HUBBLE_COMMIT'], 'branch': 'master', 'jdk': '17'}, - 'toolchainJDK': '17', 'config': 'release-current-ci-v2' - } + inputs = {} + if os.environ['NEEDS_SERVER'] == 'true': + with open('scope.json') as stream: + selected = json.load(stream)['selected'] + inputs['toolchainJDK'] = {module: ['17'] if module == 'hubble' else ['11', '17'] + for module in selected if module != 'go'} + inputs['config'] = 'release-current-ci-v2' + inputs['server'] = {'repository': os.environ['SERVER_REPOSITORY'], + 'commit': os.environ['SERVER_COMMIT'], + 'ref': os.environ['SERVER_REF'], 'jdk': '11'} + if os.environ['NEEDS_HUBBLE'] == 'true': + inputs['hubble'] = {'repository': 'apache/hugegraph', 'commit': os.environ['HUBBLE_COMMIT'], + 'ref': os.environ['HUBBLE_COMMIT'], 'branch': 'master', 'jdk': '17'} with open('inputs.json', 'w') as stream: json.dump(inputs, stream, sort_keys=True) PYINPUTS - - name: Select affected modules and verified successes + - name: Select affected modules and images id: select env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -262,16 +280,16 @@ jobs: name: ci-plan - name: Require every selected module and image to succeed env: + GH_TOKEN: ${{ github.token }} RESULTS: ${{ toJSON(needs) }} - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | printf '%s' "$RESULTS" > results.json - python3 .github/scripts/ci-policy.py gate --plan plan.json --results results.json --output receipt.json - - name: Publish verified test receipt + python3 .github/scripts/ci-policy.py gate --plan plan.json --results results.json --output test-results.json + - name: Publish current-run test results uses: actions/upload-artifact@v7 with: - name: ci-test-receipt - path: receipt.json + name: ci-test-results + path: test-results.json overwrite: true retention-days: 7 diff --git a/.github/workflows/client-ci.yml b/.github/workflows/client-ci.yml index 9907142c1..fc68cd4f6 100644 --- a/.github/workflows/client-ci.yml +++ b/.github/workflows/client-ci.yml @@ -26,7 +26,7 @@ jobs: fail-fast: false matrix: # released pd package is compiled in version 55.0, which requires java 11 to use - JAVA_VERSION: ['17'] + JAVA_VERSION: ['11', '17'] steps: - name: Fetch code diff --git a/.github/workflows/hubble-ci.yml b/.github/workflows/hubble-ci.yml index 47454f8df..84788eab4 100644 --- a/.github/workflows/hubble-ci.yml +++ b/.github/workflows/hubble-ci.yml @@ -44,12 +44,11 @@ jobs: with: fetch-depth: 2 - - name: Install released Server JVM - if: matrix.SERVER_BASELINE == 'release-1.7' + - name: Install Server fixture JVM id: server-java uses: actions/setup-java@v5 with: - java-version: '11' + java-version: ${{ matrix.SERVER_BASELINE == 'current' && '17' || '11' }} distribution: temurin overwrite-settings: false @@ -151,7 +150,7 @@ jobs: with: mode: start java-version: ${{ matrix.SERVER_BASELINE == 'current' && '17' || '11' }} - java-home: ${{ matrix.SERVER_BASELINE == 'release-1.7' && steps.server-java.outputs.path || '' }} + java-home: ${{ steps.server-java.outputs.path }} artifact-name: ${{ matrix.SERVER_BASELINE == 'current' && inputs.current-artifact-name || inputs.artifact-name }} commit-id: ${{ matrix.SERVER_BASELINE == 'current' && inputs.current-commit || inputs.server-commit }} repository: ${{ inputs.server-repository }} diff --git a/.github/workflows/loader-ci.yml b/.github/workflows/loader-ci.yml index 57f9098a3..9409206ba 100644 --- a/.github/workflows/loader-ci.yml +++ b/.github/workflows/loader-ci.yml @@ -29,7 +29,7 @@ jobs: DB_DATABASE: load_test strategy: matrix: - JAVA_VERSION: ['17'] + JAVA_VERSION: ['11', '17'] services: mysql: image: mysql:8.0 @@ -102,7 +102,7 @@ jobs: DB_DATABASE: load_test strategy: matrix: - JAVA_VERSION: ['17'] + JAVA_VERSION: ['11', '17'] steps: - name: Fetch Code uses: actions/checkout@v7 diff --git a/.github/workflows/rerun-ci.yml b/.github/workflows/rerun-ci.yml index ce451adfd..5c1fe8d93 100644 --- a/.github/workflows/rerun-ci.yml +++ b/.github/workflows/rerun-ci.yml @@ -9,13 +9,17 @@ on: permissions: {} +concurrency: + group: ${{ github.workflow }}-${{ github.event.workflow_run.id }} + cancel-in-progress: false + env: MAX_RERUNS: '2' RETRY_DELAY_SECONDS: '180' jobs: decide-rerun-action: - if: ${{ github.event.workflow_run.conclusion == 'failure' }} + if: ${{ github.event.workflow_run.conclusion == 'failure' && github.event.workflow_run.event == 'push' }} runs-on: ubuntu-latest outputs: action: ${{ steps.decision.outputs.action }} @@ -34,7 +38,7 @@ jobs: action="skip" reason="unsupported event: $EVENT_NAME" - if [[ "$EVENT_NAME" == "push" || "$EVENT_NAME" == "pull_request" ]]; then + if [[ "$EVENT_NAME" == "push" ]]; then if (( RUN_ATTEMPT > MAX_RERUNS )); then reason="retry limit reached" else @@ -68,7 +72,6 @@ jobs: permissions: actions: write contents: read - pull-requests: read runs-on: ubuntu-latest steps: # workflow_run is privileged: only load code from the default-branch commit. diff --git a/.github/workflows/spark-connector-ci.yml b/.github/workflows/spark-connector-ci.yml index 399531e22..cd1c55e47 100644 --- a/.github/workflows/spark-connector-ci.yml +++ b/.github/workflows/spark-connector-ci.yml @@ -25,7 +25,7 @@ jobs: strategy: fail-fast: false matrix: - JAVA_VERSION: ['17'] + JAVA_VERSION: ['11', '17'] steps: - name: Fetch Code diff --git a/.github/workflows/tools-ci.yml b/.github/workflows/tools-ci.yml index 85745ecb7..f9d11239c 100644 --- a/.github/workflows/tools-ci.yml +++ b/.github/workflows/tools-ci.yml @@ -24,7 +24,7 @@ jobs: TRAVIS_DIR: hugegraph-tools/assembly/travis strategy: matrix: - JAVA_VERSION: ['17'] + JAVA_VERSION: ['11', '17'] steps: - name: Fetch Code diff --git a/README.md b/README.md index c674535fd..16c208508 100644 --- a/README.md +++ b/README.md @@ -483,4 +483,4 @@ Thank you to all the people who already contributed to HugeGraph! hugegraph-toolchain is licensed under [Apache 2.0 License](https://github.com/apache/hugegraph-toolchain/blob/master/LICENSE). -See [CI behavior](docs/ci.md) for affected-module tests, verified result reuse and retry rules. +See [CI behavior](docs/ci.md) for affected-module tests, shared fixtures and retry rules. diff --git a/docs/ci.md b/docs/ci.md index 96e8b14b8..b1b89f009 100644 --- a/docs/ci.md +++ b/docs/ci.md @@ -29,51 +29,66 @@ Hubble module-test dependency closure: Dockerfile-only changes run the image check without building Server fixtures. Hubble's Dockerfile builds Client and Loader before Hubble, so Loader packaging changes also select Hubble. Packaged README changes select images; the existing -nonempty Hubble README content contract still permits reuse of module tests. -Once an image is selected, its proof binds the complete current source inputs, -including application source and packaged README content, plus the checked base -and CI policy. An old or missing image proof reruns that image while independently -verified module proofs remain reusable. +nonempty Hubble README content contract still requires fresh validation if the +file is missing, empty or has an invalid mode. Every selected module and image +runs in the current workflow; successful results from another run are not reused. Each image job builds the checkout's actual Dockerfile, records its image ID and starts a local run-scoped tag with pulling disabled. The container's image ID must -match the build. Loader explicitly runs its CLI with `--help`; its default idle +match the build. Loader directly executes `./bin/hugegraph-loader.sh --help`, +including its executable permission and shebang; its default idle container alone cannot pass. Hubble must remain running and return a valid `/about` -JSON response with the application name and version. These checks validate image -build, packaging and startup; Loader data ingestion and Hubble browser workflows -remain covered by module tests. They do not publish an image. +JSON response with the application name and version. Its public root page must +contain the React root and serve nonempty local JavaScript bundles and referenced +stylesheets with the correct content types, rejecting the HTML fallback for missing +assets. These checks validate image build, packaging, startup and public entry +points; Loader data ingestion and Hubble browser workflows remain covered by +module tests. They do not publish an image. -Toolchain compilation and tests use Java 17 without changing application -dependencies. This matrix validates Toolchain on Java 17; it does not retain -a Toolchain Java 11 runtime lane. The historical Server Java 11 fixture validates -client/server compatibility, not Toolchain execution on Java 11. +Selected Client, Loader, Tools and Spark module tests compile and run on Java 11 +and Java 17. Both matrix entries must pass, including Loader's separate HDFS tests. +Hubble compiles and runs on Java 17, matching its merged Boot 3 runtime requirement, +and tests against each Server baseline. These runtime checks supplement the +existing affected-module selection; unrelated modules are not added. Client, Loader, Tools, Spark and Go test the released Server 1.7 fixture on Java 11. Hubble tests both this release and the current Server master snapshot on Java 17; the resolved commit stays fixed while the run is queued. Each Server package is built once and shared, with independent services per job. Reuse verifies source, commit, JDK, build inputs and archive checksum. The Server -JVM is scoped to service startup, so Toolchain compilation keeps Java 17. +JVM is scoped to service startup, so Toolchain compilation and tests keep their +selected Java 11 or Java 17 runtime. Both Toolchain lanes reuse the same release +Server fixture; Hubble uses the separate Java 17 current Server fixture. Loader's HDFS tests run separately, so other profiles do not wait for Hadoop. -The immutable Server packages, `ci-plan` and successful `ci-test-receipt` artifacts +The immutable Server packages, `ci-plan` and successful `ci-test-results` artifacts are retained for seven days from their upload. Partial reruns need the original plan and fixture within that window. After an artifact expires, rerun the whole -workflow to rebuild its plan and fixtures; extending only the receipt cannot +workflow to rebuild its plan and fixtures; extending only the result report cannot restore an expired package. Test reports and coverage artifacts keep their existing retention settings. -Documentation paths use an explicit allowlist. A documentation-only update may -reuse a successful receipt from the same PR only when the base, non-document -inputs and CI policy match. The latest commit receives a new gate result that -identifies reused tests. Missing or unverifiable evidence runs the tests again; -source, type definitions, tests and CI configuration are never treated as docs. -Hubble's packaged README must remain a regular, nonempty file; deletion, invalid -mode or empty content forces fresh validation. Receipt searches filter by PR -branch and use bounded pagination. The current Server input only affects reuse -when Hubble is selected. -For consecutive documentation updates, optional-check proofs retain their original -run and exact job identities and are reverified before forwarding. A newer failed -check or unverifiable evidence prevents reuse of an older optional success. +Documentation paths use an explicit allowlist. A PR containing only plain prose +and static documentation assets has no module tests. Selection uses the cumulative +PR diff, so a documentation update to a PR that also changes source runs its +selected consumers again. Source, type definitions, tests and CI configuration +are never treated as docs. The gate checks the current workflow's actual selected +job results and fixture producers. Its result report records the run, attempt and +input metadata for diagnostics; it is not a reusable success receipt. +The planner determines affected consumers before resolving external Server inputs. +Go-only changes resolve the release fixture but skip the Hubble master lookup; +image-only and plain documentation changes need neither baseline. Unverifiable +selection conservatively resolves both. A failed required baseline lookup still +fails planning; without a published plan, recovery requires a full workflow rerun. + +PR planning records the event's head, base and source before querying live metadata. +A known mismatch with the checkout merge or current PR fails planning; expanding +coverage cannot make an outdated checkout current. After all selected jobs and +fixtures succeed, the gate rechecks the open PR's head, base and source using a +read-only API request. Changed inputs or unavailable metadata fail the gate. +Refresh the branch and start a new PR event after head or base movement: GitHub +reruns retain the original commit and event, so rerunning alone cannot refresh the +base. An unknown API or selection failure falls back to full coverage; a manual +full rerun can recover a transient failure while the recorded inputs remain current. CodeQL follows the module gate and retains its weekly scan. During migration, `Analyze (java)` still runs on every PR, after a successful module gate. @@ -86,10 +101,11 @@ changes. This switch is disabled by default and does not alter branch protection A new PR head cancels older first attempts. Reruns use separate concurrency groups, so retrying an old commit cannot cancel the current head. Automatic retries run -failed jobs at most twice, checking the open PR/current branch and unchanged run -attempt both before and after the 180-second delay. A moved base or missing -immutable plan also prevents an obsolete retry. Retry code comes from the trusted -default branch. +failed **push** jobs at most twice, checking the repository branch head and +unchanged completed-failure run attempt both before and after the 180-second delay. +Retry code comes from the trusted default branch. PR automatic retries and +cross-run success reuse are deferred until execution evidence can be verified +independently; manual workflow reruns remain available. Validate policy and retry behavior locally: From bc609f941e8cceab518b282a56fbe6444ed7ab67 Mon Sep 17 00:00:00 2001 From: imbajin Date: Tue, 6 Oct 2026 17:02:30 +0800 Subject: [PATCH 10/10] fix(ci): start HDFS from the official image Use pinned Hadoop containers with reachable block endpoints Keep Java 11 and Java 17 consumer tests and bounded readiness Collect HDFS logs and clean up containers after testing Preserve historical CodeQL configuration identity --- .../setup-hugegraph-server/runtime_test.py | 24 +++++++ .github/workflows/codeql-analysis.yml | 4 ++ .github/workflows/loader-ci.yml | 25 ++++--- docs/ci.md | 6 ++ .../assembly/travis/start-hdfs-ci.sh | 65 +++++++++++++++++++ 5 files changed, 111 insertions(+), 13 deletions(-) create mode 100755 hugegraph-loader/assembly/travis/start-hdfs-ci.sh diff --git a/.github/actions/setup-hugegraph-server/runtime_test.py b/.github/actions/setup-hugegraph-server/runtime_test.py index 6d9f54966..91ce54282 100644 --- a/.github/actions/setup-hugegraph-server/runtime_test.py +++ b/.github/actions/setup-hugegraph-server/runtime_test.py @@ -140,6 +140,30 @@ def test_hadoop_invalid_cache_retries_at_most_twice_and_stops(self): self.assertEqual(args.count('--max-time\n900'), 2) self.assertFalse((self.dir/'installed').exists()) self.assertFalse((self.dir/'hadoop-3.3.6.tar.gz').exists()) + def test_hdfs_container_exit_fails_before_tests(self): + self.command('timeout', 'shift; exec "$@"\n') + self.command('docker', ''' +case "$1" in + inspect) echo false ;; + logs) echo 'NameNode startup failed' >&2 ;; +esac +''') + result = self.run_script(ROOT/'hugegraph-loader/assembly/travis/start-hdfs-ci.sh') + self.assertNotEqual(result.returncode, 0) + self.assertIn('NameNode startup failed', result.stderr) + self.assertIn('exited before readiness', result.stderr) + def test_hdfs_live_datanode_with_failed_write_is_not_ready(self): + self.command('timeout', 'shift; exec "$@"\n') + self.command('curl', '''echo '{"beans":[{"NumLiveDataNodes":1}]}'\n''') + self.command('docker', ''' +case "$1" in + inspect) echo true ;; + exec) if [[ "$*" == *"-put"* ]]; then echo 'Block write failed' >&2; exit 1; fi ;; +esac +''') + result = self.run_script(ROOT/'hugegraph-loader/assembly/travis/start-hdfs-ci.sh') + self.assertNotEqual(result.returncode, 0) + self.assertIn('Block write failed', result.stderr) def test_real_http_readiness_requires_successful_auth_response(self): from http.server import BaseHTTPRequestHandler, HTTPServer from threading import Thread diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index abba60c91..6b3fda654 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -10,6 +10,8 @@ jobs: analyze: env: USE_STAGE: 'true' # Whether to include the stage repository. + # Preserve the master scan identity when invoked by a reusable-workflow caller. + CODEQL_ACTION_ANALYSIS_KEY: .github/workflows/codeql-analysis.yml:analyze name: Analyze runs-on: ubuntu-latest permissions: @@ -79,6 +81,8 @@ jobs: - name: Perform CodeQL Analysis uses: github/codeql-action/analyze@v4 + with: + category: .github/workflows/codeql-analysis.yml:analyze/language:${{ matrix.language }} dependency-review: if: github.event_name == 'pull_request' diff --git a/.github/workflows/loader-ci.yml b/.github/workflows/loader-ci.yml index 9409206ba..46133f557 100644 --- a/.github/workflows/loader-ci.yml +++ b/.github/workflows/loader-ci.yml @@ -140,24 +140,23 @@ jobs: fetch-ref: ${{ inputs.server-ref }} travis-dir: ${{ env.TRAVIS_DIR }} - - name: Cache verified Hadoop archive - uses: actions/cache@v6 - with: - path: | - ~/hadoop-3.3.6.tar.gz - ~/hadoop-3.3.6.tar.gz.sha512 - key: ${{ runner.os }}-hadoop-3.3.6-sha512-v1 - # Legacy archives are accepted only after the installer's fixed SHA-512 check. - restore-keys: ${{ runner.os }}-hadoop-3.3.6 - - - name: Start Hadoop - timeout-minutes: 35 + - name: Start HDFS from the official Hadoop image + timeout-minutes: 18 run: | - "$TRAVIS_DIR/install-hadoop.sh" + "$TRAVIS_DIR/start-hdfs-ci.sh" - name: Run HDFS tests run: mvn test -P hdfs -f hugegraph-loader/pom.xml -ntp + - name: Collect HDFS logs and remove containers + if: ${{ always() }} + run: | + for service in namenode datanode; do + name="${HDFS_CONTAINER_PREFIX:-hugegraph-ci-hdfs}-$service" + docker logs "$name" || true + docker rm -f "$name" || true + done + - name: Upload coverage to Codecov uses: ./.github/actions/upload-coverage with: diff --git a/docs/ci.md b/docs/ci.md index b1b89f009..5926ef268 100644 --- a/docs/ci.md +++ b/docs/ci.md @@ -60,6 +60,12 @@ JVM is scoped to service startup, so Toolchain compilation and tests keep their selected Java 11 or Java 17 runtime. Both Toolchain lanes reuse the same release Server fixture; Hubble uses the separate Java 17 current Server fixture. Loader's HDFS tests run separately, so other profiles do not wait for Hadoop. +The Ubuntu HDFS jobs use the ASF Hadoop 3.3.6 image pinned by digest, with separate +NameNode and DataNode containers on the host network. Tests still run on the +selected Java 11/17 host JVM and use `localhost:8020`. Startup checks live DataNode +registration and a real block write/read before testing; containers are removed +afterward. Image pulling and readiness are bounded. This avoids installing the +large archive on each runner, but cold runners still need to pull the image. The immutable Server packages, `ci-plan` and successful `ci-test-results` artifacts are retained for seven days from their upload. Partial reruns need the original plan and fixture within that window. After an artifact expires, rerun the whole diff --git a/hugegraph-loader/assembly/travis/start-hdfs-ci.sh b/hugegraph-loader/assembly/travis/start-hdfs-ci.sh new file mode 100755 index 000000000..4df703e2f --- /dev/null +++ b/hugegraph-loader/assembly/travis/start-hdfs-ci.sh @@ -0,0 +1,65 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with this +# work for additional information regarding copyright ownership. The ASF +# licenses this file to You under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +set -euo pipefail + +# ASF's Hadoop 3.3.6 linux/amd64 image; keep the existing HDFS server version. +image=apache/hadoop:3.3.6@sha256:62a22627f35a1bbf29c721192c02fe311c93749cc43d51df01792b2ac40ff144 +prefix=${HDFS_CONTAINER_PREFIX:-hugegraph-ci-hdfs} +timeout 600 docker pull --platform linux/amd64 "$image" + +# The tests run on the Ubuntu host and connect to localhost:8020. Host networking +# also makes the DataNode block address reachable without changing test inputs. +config=( + --env CORE-SITE.XML_fs.defaultFS=hdfs://localhost:8020 + --env HDFS-SITE.XML_dfs.namenode.rpc-address=localhost:8020 + --env HDFS-SITE.XML_dfs.replication=1 + --env HDFS-SITE.XML_dfs.permissions.enabled=false + --env HDFS-SITE.XML_dfs.datanode.address=127.0.0.1:9866 +) +docker run --detach --platform linux/amd64 --network host --name "$prefix-namenode" \ + "${config[@]}" \ + --env HDFS-SITE.XML_dfs.namenode.name.dir=file:///tmp/hugegraph-hdfs/name \ + --env ENSURE_NAMENODE_DIR=/tmp/hugegraph-hdfs/name \ + "$image" hdfs namenode +docker run --detach --platform linux/amd64 --network host --name "$prefix-datanode" \ + "${config[@]}" "$image" hdfs datanode + +deadline=$((SECONDS + ${SERVICE_READY_TIMEOUT:-180})) +while (( SECONDS < deadline )); do + for service in namenode datanode; do + if [[ $(docker inspect --format '{{.State.Running}}' "$prefix-$service") != true ]]; then + docker logs "$prefix-$service" >&2 + echo "HDFS $service exited before readiness" >&2 + exit 1 + fi + done + if curl --fail --silent --connect-timeout 2 --max-time 5 \ + 'http://localhost:9870/jmx?qry=Hadoop:service=NameNode,name=FSNamesystemState' | + python3 -c 'import json,sys; sys.exit(not any(b.get("NumLiveDataNodes",0)>0 for b in json.load(sys.stdin)["beans"]))'; then + # Registration alone does not prove the DataNode accepts block writes. + timeout 30 docker exec "$prefix-namenode" hdfs dfs -mkdir -p /files + printf 'hugegraph-hdfs-ready\n' | + timeout 30 docker exec -i "$prefix-namenode" hdfs dfs -put - /files/.ci-ready + [[ $(timeout 30 docker exec "$prefix-namenode" hdfs dfs -cat /files/.ci-ready) == hugegraph-hdfs-ready ]] + timeout 30 docker exec "$prefix-namenode" hdfs dfs -rm /files/.ci-ready + exit 0 + fi + sleep 2 +done +echo 'HDFS readiness timed out' >&2 +exit 1