From 504c238f5faa62c414ae8383adc876b3cf6efb7d Mon Sep 17 00:00:00 2001 From: imbajin Date: Wed, 14 Oct 2026 01:54:47 +0800 Subject: [PATCH 1/5] fix: reduce CI runner usage - consolidate preflight and reusable workflow selection - scope costly platform and distributed tests to related inputs - use ARM startup and persistence smoke for routine changes - schedule weekly full validation and keep CI advisory - remove duplicate runner jobs and automatic retries --- .asf.yaml | 7 +- .github/scripts/check-rerun.py | 119 --- .github/scripts/ci-policy.py | 334 +++--- .github/scripts/test_check_rerun.py | 203 ---- .github/scripts/test_ci_policy.py | 964 ++++++++---------- .github/workflows/check-dependencies.yml | 7 +- .github/workflows/cluster-test-ci.yml | 3 +- .github/workflows/codeql-analysis.yml | 67 +- .github/workflows/commons-ci.yml | 19 +- .github/workflows/docker-build-ci.yml | 123 ++- .github/workflows/helm-chart-ci.yml | 3 +- .github/workflows/java-runtime.yml | 69 -- .github/workflows/pd-store-ci.yml | 47 +- .github/workflows/rerun-ci.yml | 116 --- .github/workflows/riscv64-ci.yml | 9 +- .github/workflows/server-ci.yml | 326 +++--- .github/workflows/server-compatibility-ci.yml | 145 +-- .github/workflows/server-tests.yml | 7 + docs/ci.md | 195 ++-- .../travis/run-native-runtime-smoke-test.sh | 18 +- .../travis/test-riscv64-java-runtime.sh | 136 ++- 21 files changed, 1151 insertions(+), 1766 deletions(-) delete mode 100644 .github/scripts/check-rerun.py delete mode 100644 .github/scripts/test_check_rerun.py delete mode 100644 .github/workflows/java-runtime.yml delete mode 100644 .github/workflows/rerun-ci.yml diff --git a/.asf.yaml b/.asf.yaml index 234918b6e3..a2e0b4aa67 100644 --- a/.asf.yaml +++ b/.asf.yaml @@ -33,13 +33,10 @@ github: protected_branches: master: required_status_checks: - # Keep workflows providing these checks unfiltered; path skips can leave required checks pending. # strict means "Require PR to be up-to-date before merging". (enable when branch unstable) strict: false - # contexts are the names of checks that must pass (now only enable the basic check) - contexts: - - check-license - - Server memory tests + # CI remains visible but advisory while shared runner queues are constrained. + contexts: [] required_pull_request_reviews: dismiss_stale_reviews: true require_code_owner_reviews: false diff --git a/.github/scripts/check-rerun.py b/.github/scripts/check-rerun.py deleted file mode 100644 index fbc2c6a6a3..0000000000 --- a/.github/scripts/check-rerun.py +++ /dev/null @@ -1,119 +0,0 @@ -#!/usr/bin/env python3 -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Check the failed run again immediately before an automatic retry.""" - -import argparse -import json -import os -import re -import subprocess -from urllib.parse import urlencode - - -def api(path): - result = subprocess.run( - ["gh", "api", "--method", "GET", path], check=True, - capture_output=True, timeout=30, - ) - return json.loads(result.stdout) - - -def decide(repository, run_id, expected_attempt, max_reruns, fetch=api): - run = fetch(f"repos/{repository}/actions/runs/{run_id}") - if run.get("path") != ".github/workflows/server-ci.yml": - return "skip", "only Server CI required jobs automatically retry", None - if (run.get("status") != "completed" or run.get("conclusion") != "failure" - or run.get("run_attempt") != expected_attempt): - return "skip", "source run changed or is no longer a completed failure", None - if expected_attempt > max_reruns: - return "skip", "retry limit reached", None - sha = run.get("head_sha") - if not sha: - return "skip", "missing head SHA", None - if run.get("event") == "pull_request": - return "skip", "PR automatic retry paused: CI artifacts cannot independently prove the tested code", None - if run.get("event") == "push": - if (run.get("repository", {}).get("full_name") != repository - or run.get("head_repository", {}).get("full_name") != repository): - return "skip", "push source repository differs", None - branch = run.get("head_branch") - if not branch: - return "skip", "missing push branch", None - # Read from the workflow repository, never a similarly named fork branch. - commits = fetch(f"repos/{repository}/commits?" + urlencode({"sha": branch, "per_page": 1})) - if not commits or commits[0].get("sha") != sha: - return "skip", "push branch head moved", None - # A native job rerun includes its dependent jobs. Prefer a failed producer - # over its failed reporter; never select the advisory branch of Server CI. - targets = [] - page = 1 - while True: - jobs = fetch(f"repos/{repository}/actions/runs/{run_id}/attempts/{expected_attempt}/jobs" - f"?per_page=100&page={page}")["jobs"] - for job in jobs: - if job.get("status") != "completed" or job.get("conclusion") not in {"failure", "timed_out"}: - continue - name = job.get("name", "") - if name == "check-license": - priority = 0 - elif name == "memory-plan": - priority = 1 - elif name == "memory-java-runtime / read-version": - priority = 2 - elif re.fullmatch(r"build-server / server_memory \(Java [1-9][0-9]*(?:\.[0-9]+)*\)", name): - priority = 3 - elif name == "Server memory tests": - priority = 4 - else: - continue - # Only failed reporters retry themselves; a timed-out producer - # must be retried before its dependent failure can recover. - if priority == 4 and job.get("conclusion") != "failure": - continue - if job["run_id"] != run_id or job["head_sha"] != sha: - return "skip", "job does not match source run", None - job_id = job["id"] - if not isinstance(job_id, int) or job_id <= 0: - return "skip", "invalid job ID", None - targets.append((priority, job_id)) - if len(jobs) < 100: - break - page += 1 - if targets: - return "rerun", "failed required job and its dependencies", min(targets)[1] - return "skip", "no failed required job in this attempt", None - return "skip", "unsupported source event", None - - -def main(): - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--repository", required=True) - parser.add_argument("--run-id", required=True, type=int) - parser.add_argument("--run-attempt", required=True, type=int) - parser.add_argument("--max-reruns", required=True, type=int) - args = parser.parse_args() - try: - action, reason, job_id = decide(args.repository, args.run_id, args.run_attempt, args.max_reruns) - except (subprocess.SubprocessError, OSError, ValueError, KeyError, TypeError, AttributeError): - action, reason, job_id = "skip", "unable to verify current run and source metadata", None - print(f"{action}: {reason}") - with open(os.environ["GITHUB_OUTPUT"], "a", encoding="utf-8") as output: - output.write(f"action={action}\nreason={reason}\njob_id={job_id or ''}\n") - - -if __name__ == "__main__": - main() diff --git a/.github/scripts/ci-policy.py b/.github/scripts/ci-policy.py index 26190be79c..3bd3b7b9a4 100644 --- a/.github/scripts/ci-policy.py +++ b/.github/scripts/ci-policy.py @@ -14,7 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Conservative CI selection and gating of this run's selected tests.""" +"""Select affected CI checks from the current event and cumulative PR diff.""" import argparse import json @@ -22,8 +22,10 @@ from pathlib import Path import subprocess +MAINLINE = {"server", "commons", "struct", "pd", "store", "hstore", "cluster", "docker", "helm", + "dependency_license", "macos", "security", "distributed_smoke"} MODULES = { - "server": ["server", "commons", "struct", "pd", "store", "hstore", "cluster", "docker", "helm", "dependency_license"], + "server": sorted(MAINLINE | {"macos_full", "hbase", "riscv64"}), "toolchain": ["client", "loader", "tools", "spark", "hubble", "go"], } PREFIXES = { @@ -35,34 +37,57 @@ "hugegraph-spark-connector/": "spark", "hugegraph-hubble/": "hubble"}, } WORKFLOWS = { - "server": {"server-ci.yml": MODULES["server"], "commons-ci.yml": ["commons"], - "pd-store-ci.yml": ["pd", "store", "hstore"], "cluster-test-ci.yml": ["cluster"], - "docker-build-ci.yml": ["docker"], "helm-chart-ci.yml": ["helm"], - "codeql-analysis.yml": [], "server-memory-ci.yml": ["server"], - "server-tests.yml": ["server"], "server-compatibility-ci.yml": ["server", "docker"], - "riscv64-ci.yml": ["server"], "check-dependencies.yml": ["dependency_license"]}, - "toolchain": {"client-ci.yml": ["client"], "client-go-ci.yml": ["go"], - "loader-ci.yml": ["loader"], "tools-ci.yml": ["tools"], - "spark-connector-ci.yml": ["spark"], "hubble-ci.yml": ["hubble"], - "codeql-analysis.yml": []}, + "server": {"server-ci.yml": MAINLINE, "commons-ci.yml": {"commons"}, + "pd-store-ci.yml": {"struct", "pd", "store", "hstore"}, "cluster-test-ci.yml": {"cluster"}, + "docker-build-ci.yml": {"docker"}, "helm-chart-ci.yml": {"helm"}, + "codeql-analysis.yml": {"security"}, "server-memory-ci.yml": {"server"}, + "server-tests.yml": {"server", "hbase"}, + "server-compatibility-ci.yml": {"macos", "macos_full", "hbase", "riscv64"}, + "riscv64-ci.yml": {"riscv64"}, "check-dependencies.yml": {"dependency_license"}}, + "toolchain": {"client-ci.yml": {"client"}, "client-go-ci.yml": {"go"}, + "loader-ci.yml": {"loader"}, "tools-ci.yml": {"tools"}, + "spark-connector-ci.yml": {"spark"}, "hubble-ci.yml": {"hubble"}, + "codeql-analysis.yml": set()}, +} +# Shared runtime consumers; ordinary Server edits do not fan out to PD/Store. +DEPENDENTS = { + "server": {"commons": {"server", "pd", "store", "hstore", "cluster"}, + "struct": {"server", "pd", "store", "hstore", "cluster"}, + "pd": {"store", "hstore", "cluster"}, "store": {"hstore", "cluster"}, "hstore": {"cluster"}}, + "toolchain": {"client": {"loader", "tools", "spark", "hubble"}, "loader": {"hubble"}}, } -# Exact maintenance inputs have no product consumers; their checks run in the planner. MAINTENANCE_INPUTS = { - ".github/PULL_REQUEST_TEMPLATE.md": [], - ".github/dependabot.yml": [], - ".github/scripts/check-rerun.py": [], - ".github/scripts/test_check_rerun.py": [], - ".github/scripts/check-docker-images.sh": ["docker"], - ".github/scripts/docker-deployment.py": ["docker"], - ".github/scripts/test_docker_deployment.py": ["docker"], + ".asf.yaml": set(), ".editorconfig": set(), ".licenserc.yaml": set(), + ".github/PULL_REQUEST_TEMPLATE.md": set(), ".github/dependabot.yml": set(), + ".github/workflows/.java-version": MAINLINE | {"macos_full", "hbase", "riscv64"}, + ".github/scripts/check-docker-images.sh": {"docker"}, + ".github/scripts/docker-deployment.py": {"docker"}, + ".github/scripts/test_docker_deployment.py": {"docker"}, } -DEPENDENTS = { - "server": {"commons": ["server", "pd", "store", "hstore", "cluster"], - "struct": ["server", "pd", "store", "hstore", "cluster"], - "server": ["pd", "store", "hstore", "cluster"], "pd": ["store", "hstore", "cluster"], - "store": ["hstore", "cluster"], "hstore": ["cluster"]}, - "toolchain": {"client": ["loader", "tools", "spark", "hubble"], "loader": ["hubble"]}, +PRESETS = { + "mainline": MAINLINE, + "distributed": {"struct", "pd", "store", "hstore", "cluster", "distributed_smoke"}, + "macos-arm": {"macos", "macos_full"}, "hbase": {"hbase"}, + "riscv64": {"riscv64"}, "codeql": {"security"}, "full": set(MODULES["server"]), } +SERVER_CORE = "hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/" +SERVER_API = "hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/" +DISTRIBUTED_SERVER_INPUTS = ( + "hugegraph-server/hugegraph-hstore/", SERVER_CORE + "backend/", SERVER_CORE + "meta/", + SERVER_CORE + "space/", SERVER_CORE + "task/", SERVER_CORE + "masterelection/", SERVER_CORE + "HugeGraph", + SERVER_CORE + "StandardHugeGraph", SERVER_API + "core/", SERVER_API + "rpc/", + SERVER_API + "api/raft/", SERVER_API + "api/space/", + "hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/store/", + "hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/meta/", +) +# Test/resources/templates can contain executable examples and runtime fixtures. +RUNTIME_PARTS = {"src", "resources", "fixtures", "templates", "static", "bin", "conf", "configs"} +# These build files declare native JNI dependencies/classifiers or the RISC-V protobuf profile. +PLATFORM_POMS = {"pom.xml", "hugegraph-server/pom.xml", "hugegraph-server/hugegraph-core/pom.xml", + "hugegraph-server/hugegraph-rocksdb/pom.xml", "hugegraph-server/hugegraph-dist/pom.xml", + "hugegraph-pd/hg-pd-grpc/pom.xml"} +RISCV_PROTO_ROOTS = ("hugegraph-server/hugegraph-core/", "hugegraph-pd/hg-pd-grpc/") +TRAVIS = "hugegraph-server/hugegraph-dist/src/assembly/travis/" def git(*args): @@ -75,18 +100,17 @@ def api(path): def documentation(path, mode="100644"): - """Only prose and static documentation assets; never source or config.""" + """Only plain prose/static assets outside runtime and fixture directories.""" if mode != "100644": return False p = Path(path) - prose_names = {"README.md", "README_CN.md", "README_ZH.md", "AGENTS.md"} - module_roots = {prefix.rstrip("/") for mapping in PREFIXES.values() for prefix in mapping} - module_roots.update("hugegraph-server/" + module for module in [ - "hugegraph-api", "hugegraph-core", "hugegraph-dist", "hugegraph-example", - "hugegraph-hbase", "hugegraph-hstore", "hugegraph-rocksdb", "hugegraph-test"]) - if p.name in prose_names and (str(p.parent) == "." or str(p.parent) in module_roots): + if RUNTIME_PARTS.intersection(p.parts[:-1]): + return False + if p.suffix.lower() in {".md", ".rst"}: return True - return path.startswith("docs/") and p.suffix.lower() in {".md", ".rst", ".txt", ".png", ".jpg", ".jpeg", ".svg"} + if len(p.parts) == 1 and (p.suffix.lower() == ".txt" or p.name in {"LICENSE", "NOTICE"}): + return True + return "docs" in p.parts[:-1] and p.suffix.lower() in {".md", ".rst", ".txt", ".png", ".jpg", ".jpeg", ".svg"} def unsafe_documentation(ref, paths): @@ -106,64 +130,120 @@ def select(project, paths): for path in paths: if documentation(path): continue - if project == "server" and path in MAINTENANCE_INPUTS: - selected.update(MAINTENANCE_INPUTS[path]) - continue - if project == "server" and (Path(path).name == "pom.xml" or path.startswith("install-dist/")): - selected.add("dependency_license") - if path.startswith(".github/workflows/") and Path(path).name in WORKFLOWS[project]: - selected.update(WORKFLOWS[project][Path(path).name]) - continue - if path.endswith(".proto"): - return set(MODULES[project]) - if project == "server" and path == "hugegraph-commons/hugegraph-common/src/main/resources/version.properties": - selected.update(["commons", "docker"]) - continue - if project == "toolchain" and path.startswith("hugegraph-client/assembly/travis/"): - selected.update(["client", "go"]) - continue + p = Path(path) if project == "server": - if path in {"hugegraph-server/pom.xml", "hugegraph-server/hugegraph-api/pom.xml"}: - selected.add("docker") - if path.startswith(("hugegraph-pd/hg-pd-dist/", "hugegraph-store/hg-store-dist/")): + if path in MAINTENANCE_INPUTS: + selected.update(MAINTENANCE_INPUTS[path]) + continue + if p.name == "pom.xml": + selected.update({"dependency_license", "security"}) + if path in PLATFORM_POMS: + selected.update({"macos", "macos_full", "riscv64"}) + if path.startswith("install-dist/"): + selected.add("dependency_license") + continue + if ({"auth", "security", "permission", "permissions"}.intersection(p.parts) + or p.stem.startswith(("Authentication", "Authorization")) + or p.stem.endswith(("Authentication", "Authenticator", "Permission", "WithAuth")) + or p.name in {"AuthOptions.java", "OkHttpBasicAuthInterceptor.java", "java-security.properties", + "test-java-security-properties.sh", "enable-auth.sh"}): + selected.add("security") + if path.startswith(".github/workflows/") and p.name in WORKFLOWS[project]: + selected.update(WORKFLOWS[project][p.name]) + continue + if path == TRAVIS + "install-backend.sh": + selected.update({"hbase", "hstore"}) + if path.startswith("docker/hbase/"): + selected.update({"docker", "hbase"}) + continue + if path.startswith("hugegraph-server/hugegraph-hbase/") or "hbase" in p.name.lower(): + selected.add("hbase") + continue + if "riscv" in path.lower(): + selected.add("riscv64") + continue + if path.startswith(DISTRIBUTED_SERVER_INPUTS) or path.endswith(".proto"): + selected.update({"hstore", "cluster", "distributed_smoke"}) + if path.endswith(".proto") and path.startswith(RISCV_PROTO_ROOTS): + selected.add("riscv64") + if path.startswith("hugegraph-server/hugegraph-dist/docker/"): selected.add("docker") - if path.startswith("hugegraph-server/hugegraph-hstore/"): - selected.update(["server", "hstore"]) continue if path.startswith("hugegraph-server/hugegraph-dist/"): - selected.update(["server", "pd", "store", "hstore", "docker", "cluster"]) + selected.update({"server", "docker", "macos", "macos_full"}) + if (path.startswith("hugegraph-server/hugegraph-dist/src/assembly/static/bin/") + or "native-runtime" in p.name or "rocksdb-jni" in p.name + or p.name == "check-rocksdb-only-dist.sh"): + selected.add("riscv64") + if any(word in path for word in ("pd", "store", "distributed")): + selected.update({"hstore", "cluster", "distributed_smoke"}) continue - if path.startswith(("hugegraph-server/hugegraph-core/", "hugegraph-server/hugegraph-api/", - "hugegraph-server/hugegraph-test/")): - selected.update(["server", "hstore"]) + if path.startswith("hugegraph-server/hugegraph-rocksdb/"): + selected.update({"server", "macos", "macos_full"}) continue + if path in {"hugegraph-server/pom.xml", "hugegraph-server/hugegraph-api/pom.xml"}: + selected.add("docker") + if path.startswith(("hugegraph-pd/hg-pd-dist/", "hugegraph-store/hg-store-dist/")): + selected.add("docker") + if path == "hugegraph-commons/hugegraph-common/src/main/resources/version.properties": + selected.add("docker") + elif path.endswith(".proto"): + selected.update(MODULES[project]) + continue + elif path.startswith(".github/workflows/") and p.name in WORKFLOWS[project]: + selected.update(WORKFLOWS[project][p.name]) + continue + elif path.startswith("hugegraph-client/assembly/travis/"): + selected.update({"client", "go"}) + continue module = next((value for prefix, value in PREFIXES[project].items() if path.startswith(prefix)), None) - if module is None or path.endswith(".proto"): - return set(MODULES[project]) - selected.add(module) - if project == "server" and ("Dockerfile" in Path(path).name or "docker-entrypoint" in path): - selected.add("docker") + if module is None: + selected.update(MAINLINE if project == "server" else MODULES[project]) + else: + selected.add(module) changed = True while changed: before = set(selected) for module in before: - selected.update(DEPENDENTS[project].get(module, [])) + selected.update(DEPENDENTS[project].get(module, set())) changed = before != selected + if project == "server": + if "server" in selected: + selected.add("macos") + if selected.intersection({"pd", "store", "hstore", "struct", "cluster"}): + selected.add("distributed_smoke") + return selected + + +def event_selection(project, paths, event, event_name): + """Presets and post-merge smoke share the same affected-input classifier.""" + if project != "server": + return select(project, paths) + if event_name == "schedule": + return set(PRESETS["full"]) + if event_name == "workflow_dispatch": + preset = event.get("inputs", {}).get("test-set", "mainline") + if preset not in PRESETS: + raise ValueError("unknown test-set: " + str(preset)) + return set(PRESETS[preset]) + selected = select(project, paths) + if event_name == "push" and event.get("ref") == "refs/heads/master": + smoke = selected.intersection({"docker", "helm", "dependency_license", "macos", "distributed_smoke"}) + if selected.intersection({"server", "commons", "struct", "pd", "store", "hstore", "cluster"}): + smoke.add("distributed_smoke") + return smoke return selected def suites(project, selected): if project == "toolchain": return sorted(selected) - result = [] + result = selected.intersection({"commons", "cluster", "docker", "helm", "dependency_license", "macos", + "hbase", "riscv64", "security", "distributed_smoke"}) if "server" in selected: - result.extend(["server_memory", "server_rocksdb"]) - # Third-party inventory/review remains visible without becoming a merge gate. - for module in ["commons", "cluster", "docker", "helm"]: - if module in selected: - result.append(module) + result.update({"server_memory", "server_rocksdb"}) if selected.intersection({"pd", "store", "hstore", "struct"}): - result.append("pd_store") + result.add("pd_store") return sorted(result) @@ -182,18 +262,19 @@ def require_current_pr(plan, fetch): raise StaleInputError("PR inputs changed; refresh the branch and start a new PR run") -def create_plan(project, event, repository, fetch=api): +def create_plan(project, event, repository, fetch=api, event_name=None): + event_name = event_name or os.environ.get("GITHUB_EVENT_NAME") or ( + "pull_request" if "pull_request" in event else "push") paths = [] plan = {"schema": 1, "project": project, "repository": repository, "pr": 0, "source": repository, "branch": "", "base": "", "head": git("rev-parse", "HEAD"), - "reason": "affected inputs", "testedMergeSHA": git("rev-parse", "HEAD")} + "reason": "affected inputs", "testedMergeSHA": git("rev-parse", "HEAD"), "event": event_name} try: pr = event.get("pull_request") if "pull_request" in event: if not isinstance(pr, dict) or type(pr.get("number")) is not int or pr["number"] <= 0: raise StaleInputError("PR event lacks a valid number") plan["pr"] = pr["number"] - # An incomplete PR event must never fall back to a push plan with pr=0. try: plan.update(source=pr["head"]["repo"]["full_name"], base=pr["base"]["sha"], head=pr["head"]["sha"], branch=pr["head"]["ref"]) @@ -205,52 +286,49 @@ def create_plan(project, event, repository, fetch=api): if (plan["testedMergeSHA"] != os.environ.get("GITHUB_SHA") or len(parents) != 2 or parents[1] != plan["head"]): raise StaleInputError("checkout is not the event PR merge; start a new PR run") - # The event base may lag the synthetic merge after the target branch advances. plan["base"] = parents[0] require_current_pr(plan, fetch) - # head/base objects must exist locally; workflow fetches both before planning. ancestor = git("merge-base", plan["base"], plan["head"]) paths = git("diff", "--no-renames", "--name-only", ancestor, plan["head"]).splitlines() - else: + elif event_name == "push": plan["base"] = event.get("before", "") + plan["branch"] = event.get("ref", "").removeprefix("refs/heads/") paths = git("diff", "--no-renames", "--name-only", plan["base"], plan["head"]).splitlines() - selected = select(project, paths) - if unsafe_documentation(plan["base"], paths) or unsafe_documentation(plan["testedMergeSHA"], paths): - selected = set(MODULES[project]) - plan["reason"] = "non-regular documentation input: conservative full coverage" - plan["expected"] = suites(project, selected) + selected = event_selection(project, paths, event, event_name) + if paths and (unsafe_documentation(plan["base"], paths) + or unsafe_documentation(plan["testedMergeSHA"], paths)): + selected = event_selection(project, ["unknown"], event, event_name) + plan["reason"] = "non-regular documentation input: conservative mainline coverage" + if event_name in {"schedule", "workflow_dispatch"}: + plan["reason"] = "weekly full coverage" if event_name == "schedule" else "manual " + event.get( + "inputs", {}).get("test-set", "mainline") except (subprocess.SubprocessError, OSError, ValueError, KeyError, TypeError, AttributeError): - selected = set(MODULES[project]) - plan["reason"] = "verification unavailable: conservative full coverage" - plan["expected"] = suites(project, selected) + selected = event_selection(project, ["unknown"], {"ref": event.get("ref")}, + "push" if event_name == "push" else "pull_request") + plan["reason"] = "verification unavailable: conservative mainline coverage" if not selected: - plan["reason"] = "cumulative PR diff contains only plain prose documentation; modules unaffected" + plan["reason"] = "plain prose documentation or maintenance inputs; tests unaffected" plan["selected"] = sorted(selected) + plan["expected"] = suites(project, selected) for module in MODULES[project]: plan[module] = module in selected plan["pd_store"] = any(plan.get(m, False) for m in ["pd", "store", "hstore", "struct"]) - plan["needsFixture"] = project == "toolchain" and any(plan.get(m, False) for m in MODULES[project]) + plan["needsFixture"] = project == "toolchain" and bool(selected) plan["fixture"] = plan["needsFixture"] - plan["compatibility"] = project == "server" and bool(selected.intersection({"server", "commons", "struct", "pd", "store", "hstore", "cluster"})) - plan["smoke"] = project == "server" and bool(selected.intersection({"server", "docker"})) - plan["security"] = bool(selected - {"docker", "helm"}) or any( - p.startswith(".github/workflows/codeql") for p in locals().get("paths", [])) - plan["required"] = ["server_memory"] if project == "server" and "server" in selected else [] + if project == "toolchain": + plan["security"] = bool(selected) or any(p.startswith(".github/workflows/codeql") for p in paths) + plan["required"] = [] plan["security_languages"] = json.dumps(["java"]) plan["changedPaths"] = paths plan["selectionReasons"] = { - suite: [path for path in paths if suite in suites(project, select(project, [path]))] + suite: [path for path in paths if suite in suites(project, event_selection(project, [path], event, event_name))] for suite in plan["expected"] } - if "dependency_license" in selected: - plan["selectionReasons"]["dependency_license"] = [ - path for path in paths if "dependency_license" in select(project, [path])] if os.environ.get("GITHUB_STEP_SUMMARY"): with open(os.environ["GITHUB_STEP_SUMMARY"], "a", encoding="utf-8") as out: out.write("## Selected CI checks\n" + selection_table(plan) + "\n") return plan - def selection_reason(plan, suite): paths = plan.get("selectionReasons", {}).get(suite, []) if not paths: @@ -266,11 +344,9 @@ def selection_reason(plan, suite): def selection_table(plan): summary = "| Check | Selected | Merge requirement | Changed inputs |\n| --- | --- | --- | --- |\n" checks = suites(plan["project"], set(MODULES[plan["project"]])) - if "dependency_license" in plan["selected"]: - checks.append("dependency_license") for suite in checks: - chosen = suite in plan["expected"] or suite == "dependency_license" - role = "required" if suite == "server_memory" else "advisory" + chosen = suite in plan["expected"] + role = "advisory" summary += f"| {suite} | {'yes' if chosen else 'no'} | {role} | {selection_reason(plan, suite)} |\n" return summary @@ -281,39 +357,15 @@ def __init__(self, message, report): self.report = report -def gate(plan, results, fetch=None, mode="all"): - """Report actual outcomes; Memory and advisory checks have independent dependencies.""" +def gate(plan, results, fetch=None): + """Keep the Toolchain result summary and current-PR gate contract.""" expected = list(plan["expected"]) - if mode == "memory": - expected = [suite for suite in expected if suite == "server_memory"] - elif mode == "advisory": - expected = [suite for suite in expected if suite != "server_memory"] report = {key: plan[key] for key in ["schema", "repository", "project", "pr", "source", "branch", "base", "head", "testedMergeSHA", "selected"]} report["selectionReasons"] = plan.get("selectionReasons", {}) report.update(runID=int(os.environ.get("GITHUB_RUN_ID", "0")), runAttempt=int(os.environ.get("GITHUB_RUN_ATTEMPT", "1")), executed=expected, results={suite: results.get(suite, {}).get("result", "missing") for suite in expected}) - if mode != "memory" and "dependency_license" in plan["selected"]: - report["results"]["dependency_license"] = results.get("dependency_license", {}).get("result", "missing") - summary = "Selection: " + plan.get("reason", "affected inputs") + "\n" - summary += "Tested base: `" + plan["base"] + "`; merge: `" + plan["testedMergeSHA"] + "`\n\n" - summary += "| Check | Selected | Merge requirement | Changed inputs | Result |\n| --- | --- | --- | --- | --- |\n" - for suite in suites(plan["project"], set(MODULES[plan["project"]])): - if mode == "memory" and suite != "server_memory": - continue - if mode == "advisory" and suite == "server_memory": - continue - chosen = suite in expected - role = "required" if suite == "server_memory" else "advisory" - status = report["results"].get(suite, "not selected") - summary += f"| {suite} | {'yes' if chosen else 'no'} | {role} | {selection_reason(plan, suite)} | {status} |\n" - if "dependency_license" in report["results"]: - summary += "| dependency_license | yes | advisory | " + selection_reason(plan, "dependency_license") + " | " + str(report["results"]["dependency_license"]) + " |\n" - if os.environ.get("GITHUB_STEP_SUMMARY"): - with open(os.environ["GITHUB_STEP_SUMMARY"], "a", encoding="utf-8") as out: - out.write("## " + ("Memory result" if mode == "memory" else "Affected module results") + "\n" + summary + "\n") - print(summary) errors = [] if results.get("plan", {}).get("result") != "success": errors.append("planner did not succeed") @@ -325,15 +377,21 @@ def gate(plan, results, fetch=None, mode="all"): errors.append("selected tests lack successful fixture") if "hubble" in expected and results.get("hubble-fixture", {}).get("result") != "success": errors.append("selected Hubble tests lack successful baseline fixture") + summary = "Selection: " + plan.get("reason", "affected inputs") + "\n" + summary += "Tested base: `" + plan["base"] + "`; merge: `" + plan["testedMergeSHA"] + "`\n\n" + summary += "| Check | Selected | Merge requirement | Changed inputs | Result |\n| --- | --- | --- | --- | --- |\n" + for suite in suites(plan["project"], set(MODULES[plan["project"]])): + chosen = suite in expected + status = report["results"].get(suite, "not selected") + summary += f"| {suite} | {'yes' if chosen else 'no'} | advisory | {selection_reason(plan, suite)} | {status} |\n" + if os.environ.get("GITHUB_STEP_SUMMARY"): + with open(os.environ["GITHUB_STEP_SUMMARY"], "a", encoding="utf-8") as out: + out.write("## Affected module results\n" + summary + "\n") + print(summary) if errors: raise GateFailure("; ".join(errors), report) - # Checks are attached to their actual commit; target-branch advancement alone is not stale. - # A transient API outage cannot invalidate a genuine completed Memory result. - try: + if plan["project"] == "toolchain": require_current_pr(plan, fetch or api) - except (subprocess.SubprocessError, OSError): - if mode != "memory": - raise return report @@ -345,14 +403,16 @@ def main(): p.add_argument("--event-path", required=True) p.add_argument("--repository", required=True) p.add_argument("--output", required=True) + p.add_argument("--event-name") p = sub.add_parser("gate") p.add_argument("--plan", required=True) p.add_argument("--results", required=True) - p.add_argument("--mode", choices=["all", "memory", "advisory"], default="all") + p.add_argument("--mode", choices=["all"], default="all") p.add_argument("--output", required=True) args = parser.parse_args() if args.command == "plan": - value = create_plan(args.project, json.loads(Path(args.event_path).read_text()), args.repository) + value = create_plan(args.project, json.loads(Path(args.event_path).read_text()), args.repository, + event_name=args.event_name) if os.environ.get("GITHUB_OUTPUT"): with open(os.environ["GITHUB_OUTPUT"], "a", encoding="utf-8") as output: for key, item in value.items(): @@ -361,7 +421,7 @@ def main(): output.write(f"security_languages={value['security_languages']}\nplan_file={args.output}\n") else: try: - value = gate(json.loads(Path(args.plan).read_text()), json.loads(Path(args.results).read_text()), mode=args.mode) + value = gate(json.loads(Path(args.plan).read_text()), json.loads(Path(args.results).read_text())) except GateFailure as error: Path(args.output).write_text(json.dumps(error.report, indent=2) + "\n") raise diff --git a/.github/scripts/test_check_rerun.py b/.github/scripts/test_check_rerun.py deleted file mode 100644 index e9e7f5444f..0000000000 --- a/.github/scripts/test_check_rerun.py +++ /dev/null @@ -1,203 +0,0 @@ -#!/usr/bin/env python3 -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Behavioral tests for push retry freshness, without credentials or network.""" -import importlib.util -from pathlib import Path -import unittest -import os -import subprocess -import tempfile -import json -from unittest.mock import patch - -spec = importlib.util.spec_from_file_location("check_rerun", Path(__file__).with_name("check-rerun.py")) -checker = importlib.util.module_from_spec(spec) -spec.loader.exec_module(checker) - - -class FreshnessTest(unittest.TestCase): - def setUp(self): - self.repository = "apache/hugegraph" - self.run = {"status": "completed", "conclusion": "failure", "run_attempt": 1, - "path": ".github/workflows/server-ci.yml", "event": "push", "head_sha": "old", "head_branch": "release/test", - "repository": {"full_name": self.repository}, - "head_repository": {"full_name": self.repository}} - self.current_head = "old" - self.calls = [] - self.jobs = [self.job(117, "build-server / server_memory (Java 17)"), - self.job(118, "Server memory tests")] - - def job(self, job_id, name, conclusion="failure"): - return {"id": job_id, "name": name, "status": "completed", "conclusion": conclusion, - "run_id": 42, "head_sha": "old"} - - def fetch(self, path): - self.calls.append(path) - if path == f"repos/{self.repository}/actions/runs/42": - return self.run - if path == f"repos/{self.repository}/commits?sha=release%2Ftest&per_page=1": - return [{"sha": self.current_head}] if self.current_head else [] - if path.startswith(f"repos/{self.repository}/actions/runs/42/attempts/"): - page = int(path.rsplit("page=", 1)[1]) - return {"jobs": self.jobs[(page - 1) * 100:page * 100]} - raise AssertionError("Unexpected lookup: " + path) - - def decide(self): - return checker.decide(self.repository, 42, 1, 2, self.fetch)[0] - - def test_only_server_required_jobs_retry(self): - for path in [".github/workflows/server-memory-ci.yml", ".github/workflows/licence-checker.yml", - ".github/workflows/codeql-analysis.yml", ".github/workflows/server-compatibility-ci.yml"]: - self.run["path"] = path - self.assertEqual("skip", self.decide()) - self.run["path"] = ".github/workflows/server-ci.yml" - self.assertEqual("rerun", self.decide()) - - def test_failed_producer_is_selected_before_reporter(self): - for name in ["check-license", "memory-plan", "memory-java-runtime / read-version", - "build-server / server_memory (Java 17)"]: - with self.subTest(name=name): - self.jobs = [self.job(118, "Server memory tests"), self.job(117, name)] - self.assertEqual(117, checker.decide(self.repository, 42, 1, 2, self.fetch)[2]) - self.jobs = [self.job(118, "Server memory tests")] - self.assertEqual(118, checker.decide(self.repository, 42, 1, 2, self.fetch)[2]) - - def test_timed_out_producer_is_retried_only_inside_failed_run(self): - self.jobs = [self.job(118, "Server memory tests"), - self.job(117, "build-server / server_memory (Java 17)", "timed_out")] - self.assertEqual(117, checker.decide(self.repository, 42, 1, 2, self.fetch)[2]) - self.run["conclusion"] = "timed_out" - self.assertEqual("skip", self.decide()) - - def test_timed_out_reporter_alone_is_not_retried(self): - self.jobs = [self.job(118, "Server memory tests", "timed_out")] - self.assertEqual("skip", self.decide()) - - def test_advisory_success_and_non_failure_jobs_are_not_retried(self): - for name in ["plan", "java-runtime / read-version", "server-rocksdb / server_rocksdb (Java 17)", - "pd_store / struct", "affected-module-tests", "build-server / server_memory (Java 17) / extra"]: - self.jobs = [self.job(118, name)] - self.assertEqual("skip", self.decide()) - for conclusion in ["success", "cancelled", "skipped", None]: - self.jobs = [self.job(118, "Server memory tests", conclusion)] - self.assertEqual("skip", self.decide()) - - def test_job_selection_paginates_the_exact_attempt(self): - self.jobs = [self.job(i + 1, "advisory") for i in range(100)] + [self.job(117, "memory-plan")] - self.run["run_attempt"] = 2 - self.assertEqual(117, checker.decide(self.repository, 42, 2, 2, self.fetch)[2]) - self.assertIn(f"repos/{self.repository}/actions/runs/42/attempts/2/jobs?per_page=100&page=2", self.calls) - self.assertFalse(any("/attempts/1/" in path for path in self.calls)) - - def test_job_identity_and_metadata_must_match(self): - for key, value in [("run_id", 99), ("head_sha", "different"), ("id", "117"), ("id", -1)]: - with self.subTest(key=key): - self.jobs = [dict(self.job(117, "memory-plan"), **{key: value})] - self.assertEqual("skip", self.decide()) - - def test_workflow_posts_native_rerun_for_checker_selected_job(self): - workflow = Path(__file__).resolve().parents[1] / "workflows/rerun-ci.yml" - source = workflow.read_text() - self.assertIn(' - "HugeGraph-Server CI"', source) - self.assertIn("github.event.workflow_run.event == 'push'", source) - self.assertIn("ref: ${{ github.sha }}", source) - command = source.split(" - name: Rerun the required job and its dependencies", 1)[1] - command = command.split(" run: |\n", 1)[1].strip() - with tempfile.TemporaryDirectory() as directory: - root = Path(directory) - output = root / "output" - decide = checker.decide - with patch.dict(os.environ, {"GITHUB_OUTPUT": str(output)}), patch( - "sys.argv", ["check-rerun.py", "--repository", self.repository, - "--run-id", "42", "--run-attempt", "1", "--max-reruns", "2"]), patch.object( - checker, "decide", side_effect=lambda *args: decide(*args, fetch=self.fetch)): - checker.main() - selected = dict(line.split("=", 1) for line in output.read_text().splitlines()) - self.assertEqual("rerun", selected["action"]) - command = command.replace("${{ steps.freshness.outputs.job_id }}", selected["job_id"]) - gh = root / "gh" - gh.write_text('#!/usr/bin/env python3\nimport json, os, sys\n' - 'open(os.environ["ARGS_PATH"], "w").write(json.dumps(sys.argv[1:]))\n') - gh.chmod(0o700) - args_path = root / "args.json" - subprocess.run(["bash", "-eu", "-c", command], check=True, - env={**os.environ, "GH_REPO": self.repository, "ARGS_PATH": str(args_path), - "PATH": str(root) + os.pathsep + os.environ["PATH"]}) - self.assertEqual(["api", "--method", "POST", f"repos/{self.repository}/actions/jobs/117/rerun"], - json.loads(args_path.read_text())) - - def test_api_failure_fails_closed(self): - with tempfile.TemporaryDirectory() as directory: - output = Path(directory) / "output" - with patch.dict(os.environ, {"GITHUB_OUTPUT": str(output)}), patch( - "sys.argv", ["check-rerun.py", "--repository", self.repository, - "--run-id", "42", "--run-attempt", "1", "--max-reruns", "2"]), patch.object( - checker, "decide", side_effect=subprocess.CalledProcessError(1, "gh")): - checker.main() - self.assertIn("action=skip\n", output.read_text()) - - def test_pr_is_paused_without_reading_artifacts_or_candidates(self): - self.run.update(event="pull_request", head_repository={"full_name": "alice/hugegraph"}) - self.assertEqual("skip", self.decide()) - self.assertEqual([f"repos/{self.repository}/actions/runs/42"], self.calls) - - def test_changed_attempt(self): - self.run["run_attempt"] = 2 - self.assertEqual("skip", self.decide()) - - def test_current_run_must_remain_failed_and_completed(self): - for key, value in (("status", "in_progress"), ("conclusion", "success"), - ("conclusion", "cancelled")): - with self.subTest(key=key, value=value): - old = self.run[key] - self.run[key] = value - self.assertEqual("skip", self.decide()) - self.run[key] = old - - def test_retry_limit(self): - self.run["run_attempt"] = 3 - self.assertEqual("skip", checker.decide(self.repository, 42, 3, 2, self.fetch)[0]) - self.run["run_attempt"] = 2 - self.assertEqual("rerun", checker.decide(self.repository, 42, 2, 2, self.fetch)[0]) - - def test_push_checks_repository_and_encodes_branch(self): - self.assertEqual("rerun", self.decide()) - self.assertIn(f"repos/{self.repository}/commits?sha=release%2Ftest&per_page=1", self.calls) - self.current_head = "new" - self.assertEqual("skip", self.decide()) - self.current_head = None - self.assertEqual("skip", self.decide()) - - def test_push_rejects_other_repository(self): - for field in ["repository", "head_repository"]: - with self.subTest(field=field): - self.run[field] = {"full_name": "alice/hugegraph"} - self.assertEqual("skip", self.decide()) - self.run[field] = {"full_name": self.repository} - - def test_missing_metadata_and_unsupported_event(self): - for field in ["head_sha", "head_branch", "repository", "head_repository"]: - old = self.run.pop(field) - with self.subTest(field=field): - self.assertEqual("skip", self.decide()) - self.run[field] = old - self.run["event"] = "workflow_dispatch" - self.assertEqual("skip", self.decide()) - - -if __name__ == "__main__": - unittest.main() diff --git a/.github/scripts/test_ci_policy.py b/.github/scripts/test_ci_policy.py index 28aa746835..a76d6d6dc9 100644 --- a/.github/scripts/test_ci_policy.py +++ b/.github/scripts/test_ci_policy.py @@ -14,11 +14,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Executable Git fixtures and fail-closed gate tests for CI selection.""" +"""Path/event selection, real PR snapshots, and coordinator scheduling contracts.""" import importlib.util import json import os from pathlib import Path +import re import subprocess import tempfile import textwrap @@ -29,590 +30,429 @@ policy = importlib.util.module_from_spec(spec) spec.loader.exec_module(policy) -class PolicyTest(unittest.TestCase): - - def live_pr(self): - return {"state": "open", "head": {"sha": "head", "ref": "feature", "repo": {"full_name": "alice/server"}}, - "base": {"sha": "base", "repo": {"full_name": "apache/server"}}} - - def setUp(self): - event_sha = patch.dict(os.environ, {"GITHUB_SHA": "merge"}) - self.addCleanup(event_sha.stop) - event_sha.start() - # All tests stay local; successful PR gates query only this current-PR fixture. - api_mock = patch.object(policy, "api", return_value=self.live_pr()) - self.addCleanup(api_mock.stop) - self.api_mock = api_mock.start() - - def test_dependency_expansion(self): - self.assertEqual({"client", "loader", "tools", "spark", "hubble"}, - policy.select("toolchain", ["hugegraph-client/src/A.java"])) - self.assertEqual({"loader", "hubble"}, policy.select("toolchain", ["hugegraph-loader/src/A.java"])) - self.assertEqual({"server", "pd", "store", "hstore", "cluster"}, policy.select("server", ["hugegraph-server/A.java"])) - - def test_server_backend_and_startup_dependents(self): - selected = policy.select("server", ["hugegraph-server/hugegraph-hstore/src/test/HstoreTableTest.java"]) - self.assertIn("hstore", selected) - self.assertIn("pd_store", policy.suites("server", selected)) - selected = policy.select("server", ["hugegraph-server/hugegraph-dist/src/assembly/travis/test-start-hugegraph-pd.sh"]) - self.assertIn("pd_store", policy.suites("server", selected)) - self.assertIn("docker", selected) - for module in ["hugegraph-core", "hugegraph-api", "hugegraph-test"]: - selected = policy.select("server", [f"hugegraph-server/{module}/src/main/A.java"]) - self.assertIn("hstore", selected) - self.assertIn("server", selected) - self.assertEqual(set(), policy.select("server", ["hugegraph-server/hugegraph-hstore/README.md"])) - self.assertIn("hstore", policy.select("server", ["hugegraph-server/hugegraph-rocksdb/src/main/A.java"])) - - def test_audited_consumer_edges(self): - expected = {"server", "pd", "store", "hstore", "cluster"} - for path in ["hugegraph-server/hugegraph-hstore/src/A.java", "hugegraph-server/hugegraph-rocksdb/A.java"]: - self.assertTrue(expected.issubset(policy.select("server", [path]))) - self.assertNotIn("helm", policy.select("server", [path])) - self.assertEqual(set(policy.MODULES["server"]), policy.select("server", [".github/workflows/server-ci.yml"])) - self.assertTrue({"commons", "docker"}.issubset(policy.select("server", [ - "hugegraph-commons/hugegraph-common/src/main/resources/version.properties"]))) - self.assertIn("go", policy.select("toolchain", ["hugegraph-client/assembly/travis/start-hugegraph-servers.sh"])) - self.assertNotIn("go", policy.select("toolchain", ["hugegraph-client/src/A.java"])) - self.assertNotIn("server", policy.select("server", ["hugegraph-pd/src/A.java"])) - self.assertTrue({"store", "docker"}.issubset(policy.select("server", [ - "hugegraph-store/hg-store-dist/src/assembly/static/bin/util.sh"]))) - - def test_pd_store_distribution_inputs_include_docker_consumers(self): - for module, distribution, startup in [ - ("pd", "hg-pd-dist", "start-hugegraph-pd.sh"), - ("store", "hg-store-dist", "start-hugegraph-store.sh"), - ]: - for relative in [f"src/assembly/static/bin/{startup}", - "src/assembly/static/conf/application.yml", - "src/assembly/descriptor/server-assembly.xml", "pom.xml"]: - path = f"hugegraph-{module}/{distribution}/{relative}" - with self.subTest(path=path): - selected = policy.select("server", [path]) - self.assertTrue({module, "hstore", "cluster", "docker"}.issubset(selected)) - self.assertIn("pd_store", policy.suites("server", selected)) - self.assertNotIn("server", selected) - self.assertNotIn("helm", selected) - self.assertEqual(relative == "pom.xml", "dependency_license" in selected) - source = f"hugegraph-{module}/hg-{module}-core/src/main/java/Example.java" - with self.subTest(path=source): - selected = policy.select("server", [source]) - self.assertTrue({module, "hstore", "cluster"}.issubset(selected)) - self.assertNotIn("docker", selected) - - def test_version_resource_follows_real_docker_consumer(self): - root = Path(__file__).resolve().parents[2] - consumer = root / ".github/scripts/check-docker-images.sh" - if not consumer.exists(): - self.skipTest("Server Docker consumer is not in the Toolchain repository") - resource = next(line.strip().rstrip(")") for line in consumer.read_text().splitlines() - if line.strip().startswith("hugegraph-commons/") and "version.properties" in line) - self.assertTrue((root / resource).is_file()) - self.assertTrue({"commons", "docker"}.issubset(policy.select("server", [resource]))) - def test_single_workflow_has_no_global_fanout(self): - self.assertEqual({"hubble"}, policy.select("toolchain", [".github/workflows/hubble-ci.yml"])) - self.assertEqual({"docker"}, policy.select("server", [".github/workflows/docker-build-ci.yml"])) +class GitFixture: + def __init__(self, root): + self.root = root + self.git("init", "-q") + self.git("config", "user.email", "ci@example.invalid") + self.git("config", "user.name", "CI") - def test_known_maintenance_inputs_have_specific_owners(self): - for path in [".github/PULL_REQUEST_TEMPLATE.md", ".github/dependabot.yml", - ".github/scripts/check-rerun.py", ".github/scripts/test_check_rerun.py"]: - with self.subTest(path=path): - self.assertEqual(set(), policy.select("server", [path])) - for path in [".github/scripts/check-docker-images.sh", ".github/scripts/docker-deployment.py", - ".github/scripts/test_docker_deployment.py"]: - with self.subTest(path=path): - self.assertEqual({"docker"}, policy.select("server", [path])) - selected = policy.select("server", [path, "hugegraph-pd/hg-pd-core/src/A.java"]) - self.assertTrue({"docker", "pd", "store", "hstore", "cluster"}.issubset(selected)) - for path in [".github/scripts/ci-policy.py", ".github/scripts/test_ci_policy.py", - ".github/workflows/rerun-ci.yml", ".github/dependabot-unknown.yml"]: - with self.subTest(path=path): - self.assertEqual(set(policy.MODULES["server"]), policy.select("server", [path])) + def git(self, *args): + return subprocess.check_output(["git", *args], cwd=self.root, text=True).strip() - def test_unknown_and_proto_fail_conservative(self): - for path in ["pom.xml", ".github/scripts/new.py", "hugegraph-pd/api.proto", "mystery"]: - self.assertEqual(set(policy.MODULES["server"]), policy.select("server", [path])) + def commit(self, path, content, message): + file = self.root / path + file.parent.mkdir(parents=True, exist_ok=True) + file.write_text(content) + self.git("add", ".") + self.git("commit", "-qm", message) + return self.git("rev-parse", "HEAD") - def test_strict_docs_allowlist(self): - self.assertEqual(set(), policy.select("server", ["README.md", "docs/guide.md"])) - for path in ["hugegraph-client/src/test/resources/README.md", "docs/type.ts", "docs/config.yml", "docs/example.java", "hugegraph-server/type.ts"]: - self.assertTrue(policy.select("server", [path])) - - def test_real_git_modes_rename_and_cli(self): - with tempfile.TemporaryDirectory() as directory: - root = Path(directory) - def git(*args): - return subprocess.check_output(["git", *args], cwd=root, text=True).strip() - git("init", "-q") - git("config", "user.email", "ci@example.invalid") - git("config", "user.name", "CI") - (root / "README.md").write_text("initial") - git("add", ".") - git("commit", "-qm", "base") - base = git("rev-parse", "HEAD") - (root / "README.md").write_text("docs change") - git("commit", "-qam", "docs") - head = git("rev-parse", "HEAD") + def plan(self, event, fetch=None, event_name=None): + with patch.object(policy, "git", side_effect=self.git), patch.dict( + os.environ, {"GITHUB_SHA": self.git("rev-parse", "HEAD"), "GITHUB_EVENT_NAME": ""}): old = os.getcwd() try: - os.chdir(root) - (root / "type.ts").write_text("type changed") - git("add", ".") - git("commit", "-qm", "types") - (root / "README.md").chmod(0o755) - git("add", "README.md") - git("commit", "-qm", "executable prose") - self.assertTrue(policy.unsafe_documentation("HEAD", ["README.md"])) - (root / "README.md").unlink() - (root / "README.md").symlink_to("type.ts") - git("add", "README.md") - git("commit", "-qm", "symlink prose") - self.assertTrue(policy.unsafe_documentation("HEAD", ["README.md"])) - (root / "hugegraph-server").mkdir() - (root / "hugegraph-server" / "A.java").write_text("source") - git("add", ".") - git("commit", "-qm", "source") - before_rename = git("rev-parse", "HEAD") - (root / "docs").mkdir() - git("mv", "hugegraph-server/A.java", "docs/renamed.md") - git("commit", "-qm", "rename code to prose") - changed = git("diff", "--no-renames", "--name-only", before_rename, "HEAD").splitlines() - self.assertIn("server", policy.select("server", changed)) + os.chdir(self.root) + return policy.create_plan("server", event, "apache/server", fetch=fetch, event_name=event_name) finally: os.chdir(old) - event = root / "event.json" - event.write_text(json.dumps({"before": base})) - output = root / "output" - subprocess.run(["python3", str(Path(policy.__file__).resolve()), "plan", "--project", "server", - "--repository", "apache/server", "--event-path", str(event), - "--output", str(root / "plan.json")], cwd=root, check=True, - env={**os.environ, "GITHUB_OUTPUT": str(output)}) - self.assertIn("server=true\n", output.read_text()) - self.assertEqual(policy.suites("server", set(policy.MODULES["server"])), - json.loads((root / "plan.json").read_text())["expected"]) - def plan(self): - return {"schema": 1, "project": "server", "repository": "apache/server", "pr": 7, - "source": "alice/server", "branch": "feature", "base": "base", "head": "head", - "testedMergeSHA": "merge", "expected": ["server_memory", "server_rocksdb", "pd_store", "cluster"], - "selected": ["server", "pd", "store", "hstore", "cluster"]} - def memory_workflow_results(self, planner="success", runtime="success", build="success", selected=True): - workflow = Path(__file__).resolve().parents[1] / "workflows/server-ci.yml" - reporter = workflow.read_text().split("\n server-memory-required:\n", 1)[1] - code = reporter.split("python3 - <<'PYGATE'\n", 1)[1].split("\n PYGATE", 1)[0] - # Start with the actual needs keys, without generic gate aliases. - needs = {"memory-plan": {"result": planner, "outputs": {"server": str(selected).lower()}}, - "memory-java-runtime": {"result": runtime}, "build-server": {"result": build}} - with tempfile.TemporaryDirectory() as directory: - outcome = subprocess.run(["python3", "-c", textwrap.dedent(code)], cwd=directory, - env={**os.environ, "JOB_RESULTS": json.dumps(needs)}, - capture_output=True, text=True) - if outcome.returncode: - return outcome, None - return outcome, json.loads((Path(directory) / "results.json").read_text()) - - def test_memory_workflow_results_feed_real_gate(self): - for state in ["success", "failure", "cancelled", "skipped", None]: - with self.subTest(build=state): - outcome, results = self.memory_workflow_results(build=state) - self.assertEqual(0, outcome.returncode, outcome.stderr) - if state == "success": - self.assertEqual(["server_memory"], policy.gate(self.plan(), results, mode="memory")["executed"]) - else: - with self.assertRaisesRegex(policy.GateFailure, "server_memory"): - policy.gate(self.plan(), results, mode="memory") - outcome, results = self.memory_workflow_results(runtime="skipped", build="skipped", selected=False) - self.assertEqual(0, outcome.returncode, outcome.stderr) - docs = dict(self.plan(), expected=[], selected=[]) - self.assertEqual([], policy.gate(docs, results, mode="memory")["executed"]) - for state in ["failure", "cancelled", "skipped"]: - with self.subTest(planner=state): - outcome, results = self.memory_workflow_results(planner=state) - self.assertNotEqual(0, outcome.returncode) - self.assertIn("CI planning failed", outcome.stderr) - with self.subTest(runtime=state): - outcome, results = self.memory_workflow_results(runtime=state) - self.assertNotEqual(0, outcome.returncode) - self.assertIn("Selected Java runtime preparation failed", outcome.stderr) - - def test_memory_gate_ignores_advisory_cancellation(self): - plan = self.plan() - results = {"plan": {"result": "success"}, "server_memory": {"result": "success"}, - "cluster": {"result": "cancelled"}, "server_rocksdb": {"result": "failure"}} - self.assertEqual(["server_memory"], policy.gate(plan, results, mode="memory")["executed"]) - for state in ["failure", "cancelled", "skipped", None]: - results["server_memory"]["result"] = state - with self.assertRaisesRegex(ValueError, "server_memory"): - policy.gate(plan, results, mode="memory") - with self.assertRaises(policy.GateFailure) as failure: - policy.gate(plan, results, mode="advisory") - self.assertEqual("cancelled", failure.exception.report["results"]["cluster"]) - self.assertNotIn("server_memory", failure.exception.report["results"]) - - def test_server_and_api_poms_are_docker_inputs(self): - for path in ["hugegraph-server/pom.xml", "hugegraph-server/hugegraph-api/pom.xml"]: +class PolicyTest(unittest.TestCase): + def plan_for(self, paths, event_name="pull_request", event=None): + # Identity checks have their own real-Git fixtures; isolate event selection here. + def git(*args): + if args[0] == "diff": + return "\n".join(paths) + if args[0] == "show": + return "base head" + return "merge" + event = event or {"before": "base"} + if event_name == "pull_request": + event = {"pull_request": {"number": 7, "head": {"sha": "head", "ref": "feature", + "repo": {"full_name": "alice/server"}}, "base": {"sha": "base"}}} + with patch.object(policy, "git", side_effect=git), patch.object( + policy, "unsafe_documentation", return_value=False), patch.object( + policy, "require_current_pr"), patch.dict(os.environ, {"GITHUB_SHA": "merge"}): + return policy.create_plan("server", event, "apache/server", event_name=event_name) + + def test_ordinary_server_changes_only_run_mainline_server_and_arm_smoke(self): + for path in ["hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/traversal/A.java", + "hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/api/traversers/A.java", + "hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/traversal/A.java"]: with self.subTest(path=path): - self.assertIn("docker", policy.select("server", [path])) - self.assertNotIn("docker", policy.select("server", ["hugegraph-server/hugegraph-api/src/main/A.java"])) - - def test_codeql_and_smoke_follow_affected_inputs(self): - def plan_for(path): - def git(*args): - if args[0] == "diff": - return path - return "base" - with patch.object(policy, "git", side_effect=git), patch.object(policy, "unsafe_documentation", return_value=False): - return policy.create_plan("server", {"before": "base"}, "apache/server") - for path in ["README.md", "docs/guide.md", "helm/templates/service.yaml"]: + plan = self.plan_for([path]) + self.assertEqual({"server", "macos"}, set(plan["selected"])) + self.assertEqual({"server_memory", "server_rocksdb", "macos"}, set(plan["expected"])) + self.assertFalse(plan["macos_full"]) + self.assertFalse(plan["pd_store"]) + + def test_distributed_server_contracts_keep_hstore_cluster_without_pd_store_units(self): + for path in [policy.SERVER_CORE + "backend/store/BackendStore.java", + policy.SERVER_CORE + "meta/MetaManager.java", policy.SERVER_CORE + "StandardHugeGraph.java", + policy.SERVER_CORE + "task/DistributedTaskScheduler.java", policy.SERVER_CORE + "task/TaskManager.java", + policy.SERVER_CORE + "masterelection/GlobalMasterInfo.java", + policy.SERVER_API + "core/GraphManager.java", + "hugegraph-server/hugegraph-hstore/src/main/A.java"]: with self.subTest(path=path): - plan = plan_for(path) - self.assertFalse(plan["security"]) - self.assertFalse(plan["smoke"]) - plan = plan_for("docker/Dockerfile") - self.assertFalse(plan["security"]) - self.assertTrue(plan["smoke"]) - for path in [".mvn/maven.config", "pom.xml", "hugegraph-pd/src/main/A.java", - ".github/workflows/codeql-analysis.yml"]: + plan = self.plan_for([path]) + self.assertTrue({"server", "macos", "hstore", "cluster", "distributed_smoke"}.issubset(plan["selected"])) + self.assertTrue(plan["pd_store"]) + self.assertFalse(plan["pd"]) + self.assertFalse(plan["store"]) + self.assertFalse(plan["hbase"]) + self.assertFalse(plan["riscv64"]) + + def test_shared_runtime_consumers_and_protocols(self): + for module in ["commons", "struct"]: + selected = policy.select("server", [f"hugegraph-{module}/src/main/A.java"]) + self.assertTrue({"server", "pd", "store", "hstore", "cluster", "macos"}.issubset(selected)) + self.assertNotIn("hbase", selected) + self.assertNotIn("riscv64", selected) + for module in ["pd", "store"]: + plan = self.plan_for([f"hugegraph-{module}/src/main/A.java"]) + self.assertTrue(plan["pd_store"]) + self.assertTrue(plan["hstore"]) + self.assertTrue(plan["cluster"]) + self.assertFalse(plan["server"]) + self.assertFalse(plan["macos"]) + self.assertTrue({"pd", "store", "hstore", "cluster"}.issubset(policy.select( + "server", ["hugegraph-pd/hg-pd-grpc/src/main/proto/common.proto"]))) + + def test_noncore_platforms_only_follow_direct_inputs(self): + for path in ["hugegraph-server/hugegraph-hbase/src/main/A.java", + "hugegraph-server/hugegraph-dist/src/assembly/travis/install-hbase.sh", + "hugegraph-server/hugegraph-dist/src/assembly/travis/hbase-ci-site.xml"]: + self.assertEqual({"hbase"}, policy.select("server", [path])) + for path in [".github/workflows/riscv64-ci.yml", + "hugegraph-server/hugegraph-dist/src/assembly/travis/test-riscv64-java-runtime.sh"]: + self.assertEqual({"riscv64"}, policy.select("server", [path])) + self.assertTrue(self.plan_for([".github/workflows/server-compatibility-ci.yml"])["riscv64"]) + self.assertTrue(self.plan_for([".github/workflows/server-tests.yml"])["hbase"]) + self.assertTrue(self.plan_for(["hugegraph-server/hugegraph-rocksdb/pom.xml"])["riscv64"]) + self.assertTrue(self.plan_for(["hugegraph-server/hugegraph-dist/pom.xml"])["riscv64"]) + for path in ["hugegraph-server/hugegraph-rocksdb/src/main/A.java", + "hugegraph-server/hugegraph-dist/src/assembly/static/bin/start-hugegraph.sh"]: + plan = self.plan_for([path]) + self.assertTrue(plan["macos"]) + self.assertTrue(plan["macos_full"]) + self.assertFalse(plan["hbase"]) + self.assertEqual(path.startswith("hugegraph-server/hugegraph-dist/src/assembly/static/bin/"), + plan["riscv64"]) + for path in ["docker/hbase/Dockerfile", "docker/hbase/entrypoint.sh", policy.TRAVIS + "install-backend.sh"]: + self.assertTrue(self.plan_for([path])["hbase"]) + install = self.plan_for([policy.TRAVIS + "install-backend.sh"]) + self.assertTrue(install["hstore"]) + java = self.plan_for([".github/workflows/.java-version"]) + self.assertTrue(java["macos_full"]) + self.assertTrue(java["hbase"]) + self.assertTrue(java["riscv64"]) + for path in policy.PLATFORM_POMS: + self.assertTrue(self.plan_for([path])["riscv64"]) + self.assertTrue(self.plan_for([path])["macos_full"]) + for path in ["hugegraph-server/hugegraph-core/src/main/resources/proto/A.proto", + "hugegraph-pd/hg-pd-grpc/src/main/proto/A.proto"]: + self.assertTrue(self.plan_for([path])["riscv64"]) + self.assertFalse(self.plan_for(["hugegraph-store/hg-store-grpc/src/main/proto/A.proto"])["riscv64"]) + self.assertFalse(self.plan_for(["hugegraph-pd/hg-pd-core/pom.xml"])["riscv64"]) + native = self.plan_for(["hugegraph-server/hugegraph-dist/src/assembly/travis/run-native-runtime-smoke-test.sh"]) + self.assertTrue(native["macos_full"]) + self.assertTrue(native["riscv64"]) + self.assertTrue(self.plan_for([policy.TRAVIS + "check-rocksdb-only-dist.sh"])["riscv64"]) + + def test_security_tracks_security_and_dependencies_only(self): + paths = [policy.SERVER_CORE + "auth/AuthManager.java", policy.SERVER_CORE + "security/HugeSecurityManager.java", + policy.SERVER_CORE + "config/AuthOptions.java", + policy.SERVER_API + "api/filter/AuthenticationFilter.java", "hugegraph-pd/hg-pd-core/pom.xml", + ".github/workflows/codeql-analysis.yml"] + for path in paths: with self.subTest(path=path): - self.assertTrue(plan_for(path)["security"]) - - def test_memory_result_survives_metadata_outage_and_base_movement(self): - plan = self.plan() - results = {"plan": {"result": "success"}, "server_memory": {"result": "success"}} - def unavailable(_): - raise subprocess.CalledProcessError(1, "gh") - self.assertEqual(["server_memory"], policy.gate(plan, results, unavailable, mode="memory")["executed"]) - advanced = self.live_pr() - advanced["base"]["sha"] = "advanced" - policy.gate(plan, results, lambda _: advanced, mode="memory") - advanced["head"]["sha"] = "new-head" - with self.assertRaises(policy.StaleInputError): - policy.gate(plan, results, lambda _: advanced, mode="memory") - - def test_plan_explains_consumer_selection_and_docs_skip(self): - source = "hugegraph-server/hugegraph-api/pom.xml" - def git(*args): - return source if args[0] == "diff" else "base" - with tempfile.TemporaryDirectory() as directory: - summary = Path(directory) / "summary.md" - with patch.dict(os.environ, {"GITHUB_STEP_SUMMARY": str(summary)}), patch.object( - policy, "git", side_effect=git), patch.object(policy, "unsafe_documentation", return_value=False): - plan = policy.create_plan("server", {"before": "base"}, "apache/server") - self.assertEqual([source], plan["selectionReasons"]["docker"]) - self.assertEqual([source], plan["selectionReasons"]["server_memory"]) - self.assertIn(source, summary.read_text()) - self.assertIn("| server_memory | yes | required |", summary.read_text()) - self.assertIn("| docker | yes | advisory |", summary.read_text()) - source = "README.md" - docs = policy.create_plan("server", {"before": "base"}, "apache/server") - self.assertEqual([], docs["expected"]) - self.assertEqual({}, docs["selectionReasons"]) - self.assertIn("| server_memory | no | required | no affected inputs |", summary.read_text()) - - def test_memory_rejects_confirmed_source_and_malformed_metadata(self): - plan = self.plan() - results = {"plan": {"result": "success"}, "server_memory": {"result": "success"}} - live = self.live_pr() - live["head"]["repo"]["full_name"] = "another/fork" - with self.assertRaises(policy.StaleInputError): - policy.gate(plan, results, lambda _: live, mode="memory") - for malformed in [{"state": "open"}, {"state": "open", "head": None}]: - with self.subTest(metadata=malformed), self.assertRaises((KeyError, TypeError)): - policy.gate(plan, results, lambda _: malformed, mode="memory") - - def test_gate_rejects_missing_failed_cancelled_skipped(self): - plan = self.plan() - results = {suite: {"result": "success"} for suite in plan["expected"]} - results["plan"] = {"result": "success"} - # Even an old plan with claimed successes cannot exempt any current suite. - plan["reused"] = {suite: {"runID": 42, "testedMergeSHA": "forged"} for suite in plan["expected"]} - for suite in plan["expected"]: - for result in [None, "failure", "cancelled", "skipped"]: - with self.subTest(suite=suite, result=result), self.assertRaisesRegex(ValueError, suite): - policy.gate(plan, dict(results, **{suite: {"result": result}})) - self.assertEqual(plan["expected"], policy.gate(plan, results)["executed"]) - - def test_third_party_failure_does_not_exempt_required_tests(self): - selected = policy.select("server", ["pom.xml"]) - self.assertIn("dependency_license", selected) - expected = {"server_memory", "server_rocksdb", "commons", "pd_store", "cluster", "docker", "helm"} - self.assertEqual(expected, set(policy.suites("server", selected))) - plan = dict(self.plan(), selected=sorted(selected), expected=sorted(expected)) - results = {suite: {"result": "success"} for suite in expected} - results["plan"] = {"result": "success"} - for result in ["failure", "cancelled", "skipped", None]: - results["dependency_license"] = {"result": result} - self.assertEqual(expected, set(policy.gate(plan, results)["executed"])) - results["cluster"] = {"result": "failure"} - with self.assertRaisesRegex(ValueError, "cluster"): - policy.gate(plan, results) - - def test_incomplete_pr_event_never_becomes_a_push_plan(self): - event = {"pull_request": dict(self.live_pr(), number=7)} - cases = [] - for change in ("null_repo", "missing_repo", "missing_number", "empty_ref"): - altered = json.loads(json.dumps(event)) - pr = altered["pull_request"] - if change == "null_repo": - pr["head"]["repo"] = None - elif change == "missing_repo": - del pr["head"]["repo"] - elif change == "missing_number": - del pr["number"] - else: - pr["head"]["ref"] = "" - cases.append(altered) - cases.extend([{"pull_request": {}}, {"pull_request": None}]) - with patch.object(policy, "git", return_value="merge"): - for altered in cases: - with self.subTest(event=altered), self.assertRaises(policy.StaleInputError): - policy.create_plan('server', altered, 'apache/server', - lambda _: self.fail("Incomplete identity must fail before any API query")) - - def test_gate_refuses_failed_planner_even_empty(self): - plan = dict(self.plan(), expected=[], selected=[]) - for result in ["failure", "skipped", "cancelled", None]: - with self.subTest(result=result), self.assertRaises(ValueError): - policy.gate(plan, {"plan": {"result": result}}) - report = policy.gate(plan, {"plan": {"result": "success"}}) - self.assertEqual([], report["executed"]) - self.assertNotIn("proofs", report) - - def test_cumulative_source_and_docs_always_run_current_tests_without_historical_apis(self): + self.assertTrue(self.plan_for([path])["security"]) + for path in [policy.SERVER_CORE + "api/Author.java", "hugegraph-pd/hg-pd-core/src/main/A.java", + "docker/Dockerfile", "helm/templates/service.yaml"]: + self.assertFalse(self.plan_for([path])["security"]) + + def test_documentation_and_nested_readmes_skip_every_test_lane(self): + paths = ["README.md", "CONTRIBUTING.md", "notes.txt", "LICENSE", "NOTICE", "docs/ci.md", + ".github/outdated/APACHE_PROPOSAL.md", "hugegraph-pd/design.md", + "hugegraph-pd/docs/raft.md", "hugegraph-store/docs/guide.txt", "hugegraph-server/hugegraph-hstore/README.md", + "hugegraph-server/nested/component/README.md", "hugegraph-pd/hg-pd-core/AGENTS.md", "docker/README.md"] + for event_name, event in [("pull_request", {}), ("push", {"ref": "refs/heads/master"})]: + plan = self.plan_for(paths, event_name, event) + self.assertEqual([], plan["selected"]) + self.assertEqual([], plan["expected"]) + self.assertFalse(any(plan[module] for module in policy.MODULES["server"])) + for path in ["hugegraph-server/hugegraph-test/src/test/resources/README.md", "docs/example.java", + "docs/config.yml", "hugegraph-pd/docs/fixtures/example.txt", "helm/templates/README.md", + "hugegraph-store/hg-store-dist/src/assembly/static/conf/startup.txt"]: + with self.subTest(path=path): + self.assertTrue(policy.select("server", [path])) + + def test_master_push_runs_smoke_without_repeated_full_suites(self): + event = {"before": "base", "ref": "refs/heads/master"} + generic = self.plan_for(["hugegraph-server/hugegraph-core/src/main/A.java"], "push", event) + self.assertEqual({"macos", "distributed_smoke"}, set(generic["selected"])) + pd = self.plan_for(["hugegraph-pd/hg-pd-core/src/main/A.java"], "push", event) + self.assertEqual({"distributed_smoke"}, set(pd["selected"])) + for path in ["hugegraph-server/hugegraph-hbase/src/main/A.java", ".github/workflows/riscv64-ci.yml", + ".github/workflows/codeql-analysis.yml"]: + self.assertEqual([], self.plan_for([path], "push", event)["expected"]) + native = self.plan_for(["hugegraph-server/hugegraph-rocksdb/src/main/A.java"], "push", event) + self.assertTrue(native["macos"]) + self.assertFalse(native["macos_full"]) + self.assertTrue(self.plan_for(["docker/Dockerfile"], "push", event)["docker"]) + + def test_manual_presets_weekly_and_release_branches(self): + for preset, selected in policy.PRESETS.items(): + with self.subTest(preset=preset): + plan = self.plan_for([], "workflow_dispatch", {"inputs": {"test-set": preset}}) + self.assertEqual(selected, set(plan["selected"])) + default = self.plan_for([], "workflow_dispatch") + self.assertEqual(policy.MAINLINE, set(default["selected"])) + self.assertFalse(default["macos_full"]) + weekly = self.plan_for([], "schedule") + self.assertEqual(set(policy.MODULES["server"]), set(weekly["selected"])) + self.assertTrue(weekly["macos_full"]) + for branch in ["release-1.7", "test-ci", "tinkerpop-3.8"]: + plan = self.plan_for(["hugegraph-server/hugegraph-core/src/main/A.java"], "push", + {"before": "base", "ref": "refs/heads/" + branch}) + self.assertTrue(plan["server"]) + self.assertFalse(plan["hbase"]) + self.assertFalse(plan["riscv64"]) + + def test_unknown_inputs_fall_back_to_mainline_without_expensive_noncore(self): + for path in [".github/scripts/ci-policy.py", ".mvn/maven.config", "unknown"]: + self.assertEqual(policy.MAINLINE, policy.select("server", [path])) + with patch.object(policy, "git", side_effect=lambda *args: "merge" if args[0] == "rev-parse" else + (_ for _ in ()).throw(subprocess.CalledProcessError(1, "git"))): + plan = policy.create_plan("server", {"before": "base"}, "apache/server", event_name="push") + self.assertEqual(policy.MAINLINE, set(plan["selected"])) + self.assertFalse(plan["macos_full"]) + self.assertFalse(plan["hbase"]) + self.assertFalse(plan["riscv64"]) + + def test_deployment_and_specific_workflows_keep_specific_owners(self): + for path in [".github/workflows/docker-build-ci.yml", ".github/scripts/check-docker-images.sh", + ".github/scripts/docker-deployment.py", ".github/scripts/test_docker_deployment.py", + "hugegraph-server/hugegraph-dist/docker/Dockerfile"]: + self.assertEqual({"docker"}, policy.select("server", [path])) + for path in [".asf.yaml", ".github/dependabot.yml", ".editorconfig"]: + self.assertEqual(set(), policy.select("server", [path])) + for module in ["pd", "store"]: + selected = policy.select("server", [f"hugegraph-{module}/hg-{module}-dist/src/assembly/static/bin/start.sh"]) + self.assertTrue({module, "hstore", "cluster", "docker", "distributed_smoke"}.issubset(selected)) + self.assertNotIn("server", selected) + version = "hugegraph-commons/hugegraph-common/src/main/resources/version.properties" + self.assertTrue({"commons", "docker"}.issubset(policy.select("server", [version]))) + + def test_toolchain_selection_and_fixture_gate_contract_remain_compatible(self): + quiet = patch("builtins.print") + quiet.start() + self.addCleanup(quiet.stop) + self.assertEqual({"client", "loader", "tools", "spark", "hubble"}, policy.select( + "toolchain", ["hugegraph-client/src/A.java"])) + self.assertEqual({"loader", "hubble"}, policy.select("toolchain", ["hugegraph-loader/src/A.java"])) + self.assertEqual(set(policy.MODULES["toolchain"]), policy.select("toolchain", ["hugegraph-client/api.proto"])) + self.assertIn("go", policy.select("toolchain", ["hugegraph-client/assembly/travis/start-hugegraph-servers.sh"])) + plan = {"schema": 1, "repository": "apache/toolchain", "project": "toolchain", "pr": 0, + "source": "apache/toolchain", "branch": "master", "base": "base", "head": "head", + "testedMergeSHA": "head", "selected": ["client"], "expected": ["client"]} + results = {"plan": {"result": "success"}, "client": {"result": "success"}, "fixture": {"result": "success"}} + self.assertEqual(["client"], policy.gate(plan, results, lambda _: self.fail("No post-run API query"))["executed"]) + for state in ["failure", "cancelled", "skipped", None]: + for job in ["plan", "client", "fixture"]: + with self.subTest(state=state, job=job), self.assertRaises(policy.GateFailure): + policy.gate(plan, dict(results, **{job: {"result": state}})) + + def test_toolchain_gate_keeps_summary_current_pr_and_cli_all(self): + plan = {"schema": 1, "repository": "apache/toolchain", "project": "toolchain", "pr": 7, + "source": "alice/toolchain", "branch": "feature", "base": "base", "head": "head", + "testedMergeSHA": "merge", "selected": ["client"], "expected": ["client"]} + live = {"state": "open", "head": {"sha": "head", "ref": "feature", "repo": {"full_name": "alice/toolchain"}}, + "base": {"repo": {"full_name": "apache/toolchain"}}} + results = {"plan": {"result": "success"}, "client": {"result": "success"}, "fixture": {"result": "success"}} + with tempfile.TemporaryDirectory() as directory, patch("builtins.print"): + root = Path(directory) + summary = root / "summary.md" + with patch.dict(os.environ, {"GITHUB_STEP_SUMMARY": str(summary)}): + policy.gate(plan, results, lambda _: live) + self.assertIn("| client | yes | advisory | affected inputs | success |", summary.read_text()) + with self.assertRaises(policy.StaleInputError): + policy.gate(plan, results, lambda _: dict(live, head=dict(live["head"], sha="new-head"))) + (root / "plan.json").write_text(json.dumps(dict(plan, pr=0))) + (root / "results.json").write_text(json.dumps(results)) + outcome = subprocess.run(["python3", str(Path(policy.__file__).resolve()), "gate", "--mode", "all", + "--plan", str(root / "plan.json"), "--results", str(root / "results.json"), + "--output", str(root / "report.json")], capture_output=True, text=True) + self.assertEqual(0, outcome.returncode, outcome.stderr) + self.assertEqual(["client"], json.loads((root / "report.json").read_text())["executed"]) + + def test_summary_has_only_advisory_checks(self): + with tempfile.TemporaryDirectory() as directory, patch.dict( + os.environ, {"GITHUB_STEP_SUMMARY": str(Path(directory) / "summary.md")}): + plan = self.plan_for(["pom.xml"]) + summary = (Path(directory) / "summary.md").read_text() + self.assertIn("| server_memory | yes | advisory |", summary) + self.assertNotIn("| required |", summary) + self.assertEqual([], plan["required"]) + self.assertEqual(1, summary.count("| dependency_license |")) + + def coordinator_jobs(self, plan, selection="success", java="success", result="success", cancelled=False): + workflow = Path(__file__).resolve().parents[1] / "workflows/server-ci.yml" + text = workflow.read_text().split("\njobs:\n", 1)[1] + jobs = {} + for match in re.finditer(r"^ ([\w-]+):\n(.*?)(?=^ [\w-]+:\n|\Z)", text, re.M | re.S): + name, body = match.groups() + if name == "preflight": + continue + condition = re.search(r"^ if: \$\{\{ (.+) \}\}$", body, re.M) + self.assertIsNotNone(condition, "Caller needs an explicit cancellation/selection condition: " + name) + expression = condition.group(1) + # GitHub otherwise injects success(), which would hide tests after a license failure. + self.assertIn("cancelled()", expression) + outputs = {key: str(value).lower() for key, value in plan.items() if isinstance(value, bool)} + outputs.update(selection_result=selection, java_result=java) + expression = re.sub(r"needs\.preflight\.outputs\.([\w]+)", + lambda match: repr(outputs.get(match.group(1), "")), expression) + expression = expression.replace("needs.preflight.result", repr(result)) + expression = expression.replace("!cancelled()", "not cancelled") + expression = expression.replace("&&", "and").replace("||", "or") + jobs[name] = bool(eval(expression, {"__builtins__": {}}, {"cancelled": cancelled})) + return {job for job, run in jobs.items() if run} + + def test_real_coordinator_keeps_selected_jobs_after_license_or_selftest_failure(self): + full = self.plan_for([], "workflow_dispatch", {"inputs": {"test-set": "full"}}) + expected = {"build-server", "server-rocksdb", "commons", "pd_store", "cluster", "docker", "helm", + "dependency_license", "compatibility", "codeql"} + self.assertEqual(expected, self.coordinator_jobs(full)) + self.assertEqual(expected, self.coordinator_jobs(full, result="failure")) + docs = self.plan_for(["hugegraph-pd/docs/guide.txt", "docker/README.md"]) + self.assertEqual(set(), self.coordinator_jobs(docs, result="failure")) + source = self.plan_for(["hugegraph-server/hugegraph-core/src/main/A.java"]) + self.assertEqual({"build-server", "server-rocksdb", "compatibility"}, self.coordinator_jobs(source)) + master = self.plan_for(["hugegraph-server/hugegraph-core/src/main/A.java"], "push", + {"before": "base", "ref": "refs/heads/master"}) + self.assertEqual({"docker", "compatibility"}, self.coordinator_jobs(master)) + + def test_real_coordinator_blocks_selection_runtime_failure_and_cancellation(self): + full = self.plan_for([], "schedule") + for state in ["failure", "cancelled", "skipped", ""]: + with self.subTest(selection=state): + self.assertEqual(set(), self.coordinator_jobs(full, selection=state, result="failure")) + with self.subTest(java=state): + self.assertEqual({"helm"}, self.coordinator_jobs(full, java=state, result="failure")) + self.assertEqual(set(), self.coordinator_jobs(full, cancelled=True)) + + def test_coordinator_reads_and_validates_real_java_input(self): + workflow = (Path(__file__).resolve().parents[1] / "workflows/server-ci.yml").read_text() + code = workflow.split("python3 - <<'PYTHON'\n", 1)[1].split("\n PYTHON", 1)[0] with tempfile.TemporaryDirectory() as directory: root = Path(directory) - def git(*args): - return subprocess.check_output(["git", *args], cwd=root, text=True).strip() - git("init", "-q") - git("config", "user.email", "ci@example.invalid") - git("config", "user.name", "CI") - (root / "README.md").write_text("base") - git("add", ".") - git("commit", "-qm", "base") - base = git("rev-parse", "HEAD") - git("checkout", "-qb", "feature") - (root / "hugegraph-server").mkdir() - source = root / "hugegraph-server/A.java" - source.write_text("source A") - git("add", ".") - git("commit", "-qm", "source") - for text in ["docs B after test failure", "docs C after claimed old success"]: - (root / "README.md").write_text(text) - git("commit", "-qam", "docs") - head = git("rev-parse", "HEAD") - # Synthetic PR merge; a different tree with the same parents also exists. - tree = git("rev-parse", "HEAD^{tree}") - merge = git("commit-tree", tree, "-p", base, "-p", head, "-m", "PR merge") - false_merge = git("commit-tree", git("rev-parse", base + "^{tree}"), - "-p", base, "-p", head, "-m", "unexecuted tree") - self.assertNotEqual(merge, false_merge) - git("checkout", "--detach", "-q", merge) - live = {"state": "open", "head": {"sha": head, "ref": "feature", "repo": {"full_name": "alice/server"}}, - "base": {"sha": base, "repo": {"full_name": "apache/server"}}} - event = {"pull_request": dict(live, number=7)} - calls = [] - def fetch(path): - calls.append(path) - self.assertEqual("repos/apache/server/pulls/7", path) - return live - old = os.getcwd() - try: - os.chdir(root) - os.environ["GITHUB_SHA"] = merge - plan = policy.create_plan("server", event, "apache/server", fetch) - git("checkout", "--detach", "-q", false_merge) - with self.assertRaises(policy.StaleInputError): - policy.create_plan("server", event, "apache/server", fetch) - git("checkout", "--detach", "-q", merge) - finally: - os.chdir(old) - self.assertEqual(["repos/apache/server/pulls/7"], calls) - self.assertTrue(plan["server"]) - self.assertTrue(plan["pd_store"]) - self.assertTrue(plan["cluster"]) - self.assertTrue(plan["compatibility"]) - self.assertTrue(plan["security"]) - self.assertNotIn("reused", plan) - results = {suite: {"result": "success"} for suite in plan["expected"]} - results["plan"] = {"result": "success"} - results["server_memory"] = {"result": "skipped"} - with self.assertRaisesRegex(ValueError, "server_memory"): - policy.gate(plan, results) - git("checkout", "-q", "feature") - - def test_queued_pr_uses_actual_merge_base_and_rejects_changed_identity(self): + java = root / ".github/workflows/.java-version" + java.parent.mkdir(parents=True) + output = root / "outputs.txt" + for version in ["17", "21.0.1", "", "bad", '17\n21']: + java.write_text(version) + result = subprocess.run(["python3", "-c", textwrap.dedent(code)], cwd=root, + env={**os.environ, "GITHUB_OUTPUT": str(output)}, capture_output=True, text=True) + with self.subTest(version=version): + self.assertEqual(version in {"17", "21.0.1"}, result.returncode == 0, result.stderr) + self.assertEqual(['versions=["17"]', 'versions=["21.0.1"]'], output.read_text().splitlines()) + + def test_real_git_document_modes_renames_and_cli_outputs(self): with tempfile.TemporaryDirectory() as directory: root = Path(directory) - def git(*args): - return subprocess.check_output(["git", *args], cwd=root, text=True).strip() - git("init", "-q") - git("config", "user.email", "ci@example.invalid") - git("config", "user.name", "CI") - (root / "README.md").write_text("base") - git("add", ".") - git("commit", "-qm", "event base") - event_base = git("rev-parse", "HEAD") - git("checkout", "-qb", "feature") - (root / "hugegraph-server").mkdir() - (root / "hugegraph-server/A.java").write_text("PR source") - git("add", ".") - git("commit", "-qm", "PR head") - head = git("rev-parse", "HEAD") - event_live = self.live_pr() - event_live["head"]["sha"], event_live["base"]["sha"] = head, event_base - event = {"pull_request": dict(event_live, number=7)} - git("checkout", "--detach", "-q", event_base) - (root / "README.md").write_text("target advanced while run was queued") - git("commit", "-qam", "new target base") - tested_base = git("rev-parse", "HEAD") - git("merge", "--no-ff", "-qm", "current PR merge", "feature") - merge = git("rev-parse", "HEAD") - live = dict(event_live, base=dict(event_live["base"], sha=tested_base)) - old = os.getcwd() - try: - os.chdir(root) - os.environ["GITHUB_SHA"] = merge - plan = policy.create_plan("server", event, "apache/server", lambda _: live) - self.assertEqual(tested_base, plan["base"]) - self.assertNotEqual(event_base, plan["base"]) - self.assertEqual(head, plan["head"]) - self.assertEqual(merge, plan["testedMergeSHA"]) - self.assertEqual(["hugegraph-server/A.java"], plan["changedPaths"]) - self.assertIn("server_memory", plan["expected"]) - results = {suite: {"result": "success"} for suite in plan["expected"]} - results["plan"] = {"result": "success"} - self.assertEqual(plan["expected"], policy.gate(plan, results, lambda _: live)["executed"]) - changed = dict(live, head=dict(live["head"], sha=tested_base)) - with self.assertRaises(policy.StaleInputError): - policy.create_plan("server", event, "apache/server", lambda _: changed) - with self.assertRaises(policy.StaleInputError): - policy.gate(plan, results, lambda _: changed) - wrong_merge = git("commit-tree", git("rev-parse", merge + "^{tree}"), - "-p", tested_base, "-p", event_base, "-m", "wrong PR head") - unrelated_base = git("commit-tree", git("rev-parse", tested_base + "^{tree}"), - "-m", "unrelated target history") - unrelated_merge = git("commit-tree", git("rev-parse", merge + "^{tree}"), - "-p", unrelated_base, "-p", head, "-m", "unrelated target merge") - for checkout in [wrong_merge, head, unrelated_merge]: - git("checkout", "--detach", "-q", checkout) - with self.subTest(checkout=checkout), self.assertRaises(policy.StaleInputError): - policy.create_plan("server", event, "apache/server", lambda _: live) - finally: - os.chdir(old) + fixture = GitFixture(root) + base = fixture.commit("README.md", "base", "base") + fixture.commit("README.md", "docs", "docs") + self.assertEqual([], fixture.plan({"before": base, "ref": "refs/heads/master"})["expected"]) + (root / "README.md").chmod(0o755) + fixture.git("add", ".") + fixture.git("commit", "-qm", "executable prose") + unsafe = fixture.plan({"before": base, "ref": "refs/heads/feature"}) + self.assertEqual(policy.MAINLINE, set(unsafe["selected"])) + (root / "README.md").unlink() + (root / "README.md").symlink_to("target.txt") + fixture.git("add", ".") + fixture.git("commit", "-qm", "symlink prose") + self.assertEqual(policy.MAINLINE, set(fixture.plan({"before": base})["selected"])) + before = fixture.commit("hugegraph-server/A.java", "source", "source") + (root / "docs").mkdir() + fixture.git("mv", "hugegraph-server/A.java", "docs/renamed.md") + fixture.git("commit", "-qm", "rename source to prose") + self.assertTrue(fixture.plan({"before": before})["server"]) + event = root / "event.json" + event.write_text("{}") + output = root / "outputs.txt" + subprocess.run(["python3", str(Path(policy.__file__).resolve()), "plan", "--project", "server", + "--repository", "apache/server", "--event-path", str(event), "--event-name", "schedule", + "--output", str(root / "plan.json")], cwd=root, check=True, + env={**os.environ, "GITHUB_OUTPUT": str(output)}) + self.assertIn("macos_full=true\n", output.read_text()) + self.assertIn("pd_store=true\n", output.read_text()) + self.assertNotIn("compatibility=", output.read_text()) + self.assertEqual(set(policy.MODULES["server"]), set(json.loads((root / "plan.json").read_text())["selected"])) - def test_pure_docs_pr_and_metadata_fail_closed(self): - live = self.live_pr() - event = {"pull_request": dict(live, number=7)} + def test_cumulative_pr_snapshot_keeps_source_after_document_commit_and_checks_current_head(self): + with tempfile.TemporaryDirectory() as directory: + fixture = GitFixture(Path(directory)) + base = fixture.commit("README.md", "base", "base") + fixture.git("checkout", "-qb", "feature") + fixture.commit("hugegraph-server/A.java", "source", "source") + head = fixture.commit("README.md", "docs after source", "docs") + merge = fixture.git("commit-tree", fixture.git("rev-parse", "HEAD^{tree}"), + "-p", base, "-p", head, "-m", "PR merge") + fixture.git("checkout", "--detach", "-q", merge) + live = {"number": 7, "state": "open", "head": {"sha": head, "ref": "feature", + "repo": {"full_name": "alice/server"}}, "base": {"sha": base, "repo": {"full_name": "apache/server"}}} + calls = [] + def fetch(path): + calls.append(path) + return live + plan = fixture.plan({"pull_request": live}, fetch) + self.assertEqual(["repos/apache/server/pulls/7"], calls) + self.assertTrue(plan["server"]) + self.assertFalse(plan["pd_store"]) + self.assertFalse(plan["security"]) + self.assertIn("hugegraph-server/A.java", plan["changedPaths"]) + for section, field, value in [("head", "sha", "new-head"), ("head", "ref", "other"), + ("head", "repo", {"full_name": "other/fork"}), + ("base", "repo", {"full_name": "other/target"})]: + changed = dict(live, **{section: dict(live[section], **{field: value})}) + with self.subTest(section=section, field=field), self.assertRaises(policy.StaleInputError): + fixture.plan({"pull_request": live}, lambda _: changed) + with self.assertRaises(policy.StaleInputError): + fixture.plan({"pull_request": live}, lambda _: dict(live, state="closed")) + fixture.git("checkout", "--detach", "-q", head) + with self.assertRaises(policy.StaleInputError): + fixture.plan({"pull_request": live}, fetch) + + def test_queued_merge_uses_actual_target_parent_and_api_outage_keeps_pr_identity(self): + live = {"number": 7, "state": "open", "head": {"sha": "head", "ref": "feature", + "repo": {"full_name": "alice/server"}}, "base": {"sha": "event-base", "repo": {"full_name": "apache/server"}}} def git(*args): if args[0] == "show": - return "base head" + return "current-base head" if args[0] == "diff": - return "README.md\ndocs/ci.md" + return "README.md" return "merge" - with patch.object(policy, "git", side_effect=git), patch.object(policy, "unsafe_documentation", return_value=False): - plan = policy.create_plan("server", event, "apache/server", lambda p: live) + with patch.dict(os.environ, {"GITHUB_SHA": "merge", "GITHUB_EVENT_NAME": ""}), patch.object( + policy, "git", side_effect=git), patch.object(policy, "unsafe_documentation", return_value=False): + plan = policy.create_plan("server", {"pull_request": live}, "apache/server", lambda _: live) + self.assertEqual("current-base", plan["base"]) self.assertEqual([], plan["expected"]) - self.assertFalse(plan["server"]) - self.assertFalse(plan["security"]) - advanced = dict(live, base={"sha": "new-base", "repo": {"full_name": "apache/server"}}) - self.assertFalse(policy.create_plan("server", event, "apache/server", lambda p: advanced)["server"]) - with patch.object(policy, "git", side_effect=lambda *a: "wrong parents" if a[0] == "show" else git(*a)): - with self.assertRaises(policy.StaleInputError): - policy.create_plan("server", event, "apache/server", lambda p: live) - - def test_api_failure_runs_full_required_without_losing_pr_identity(self): - event = {"pull_request": dict(self.live_pr(), number=7)} - def git(*args): - return "base head" if args[0] == "show" else "merge" - def fail(path): - raise subprocess.CalledProcessError(1, "gh") - with patch.object(policy, "git", side_effect=git): - plan = policy.create_plan("server", event, "apache/server", fail) - self.assertEqual(set(policy.MODULES["server"]), set(plan["selected"])) - self.assertEqual(plan["reason"], policy.selection_reason(plan, "dependency_license")) - self.assertEqual((7, "alice/server", "feature", "base", "head"), - tuple(plan[key] for key in ["pr", "source", "branch", "base", "head"])) - results = {suite: {"result": "success"} for suite in plan["expected"]} - results.update(plan={"result": "success"}, fixture={"result": "success"}, - **{"hubble-fixture": {"result": "success"}}) - with self.assertRaises(subprocess.CalledProcessError): - policy.gate(plan, results, fail) - self.assertEqual(plan["expected"], policy.gate(plan, results)["executed"]) - self.api_mock.assert_called_once_with("repos/apache/server/pulls/7") - - def test_real_pr_git_snapshot_and_final_gate_freshness(self): - with tempfile.TemporaryDirectory() as directory: - root = Path(directory) - def git(*args): - return subprocess.check_output(["git", *args], cwd=root, text=True).strip() - git("init", "-q") - git("config", "user.email", "ci@example.invalid") - git("config", "user.name", "CI") - (root / "README.md").write_text("base") - git("add", ".") - git("commit", "-qm", "base") - base = git("rev-parse", "HEAD") - git("checkout", "-qb", "feature") - (root / "hugegraph-client").mkdir() - (root / "hugegraph-client/A.java").write_text("source") - git("add", ".") - git("commit", "-qm", "source") - head = git("rev-parse", "HEAD") - tree = git("rev-parse", "HEAD^{tree}") - merge = git("commit-tree", tree, "-p", base, "-p", head, "-m", "PR merge") - advanced_head = git("commit-tree", tree, "-p", head, "-m", "new source head") - advanced_base = git("commit-tree", git("rev-parse", base + "^{tree}"), - "-p", base, "-m", "new target base") - git("checkout", "--detach", "-q", merge) - live = self.live_pr() - live["head"]["sha"], live["base"]["sha"] = head, base - event = {"pull_request": dict(live, number=7)} - old = os.getcwd() - try: - os.chdir(root) - os.environ["GITHUB_SHA"] = merge - plan = policy.create_plan("server", event, "apache/server", lambda _: live) - results = {suite: {"result": "success"} for suite in plan["expected"]} - results.update(plan={"result": "success"}, fixture={"result": "success"}, - **{"hubble-fixture": {"result": "success"}}) - self.assertEqual(plan["expected"], policy.gate(plan, results, lambda _: live)["executed"]) - for section, field, value in [("head", "sha", advanced_head), - ("head", "ref", "other-branch"), - ("head", "repo", {"full_name": "other/fork"}), - ("base", "repo", {"full_name": "other/target"})]: - changed = dict(live, **{section: dict(live[section], **{field: value})}) - with self.subTest(section=section, field=field): - with self.assertRaises(policy.StaleInputError): - policy.create_plan("server", event, "apache/server", lambda _: changed) - with self.assertRaises(policy.StaleInputError): - policy.gate(plan, results, lambda _: changed) - advanced_merge = git("commit-tree", tree, "-p", advanced_base, "-p", head, - "-m", "PR merge after master advanced") - git("checkout", "--detach", "-q", advanced_merge) - os.environ["GITHUB_SHA"] = advanced_merge - advanced_plan = policy.create_plan("server", event, "apache/server", lambda _: live) - self.assertEqual(base, event["pull_request"]["base"]["sha"]) - self.assertEqual(advanced_base, advanced_plan["base"]) - self.assertEqual(advanced_merge, advanced_plan["testedMergeSHA"]) - self.assertEqual(plan["changedPaths"], advanced_plan["changedPaths"]) - policy.gate(advanced_plan, results, lambda _: live) - with patch.dict(os.environ, {"GITHUB_SHA": ""}): - with self.assertRaises(policy.StaleInputError): - policy.create_plan("server", event, "apache/server", lambda _: live) - with self.assertRaises(policy.StaleInputError): - policy.gate(plan, results, lambda _: dict(live, state="closed")) - git("checkout", "--detach", "-q", head) - with self.assertRaises(policy.StaleInputError): - policy.create_plan("server", event, "apache/server", lambda _: live) - finally: - os.chdir(old) - - def test_push_gate_does_not_query_pr(self): - plan = dict(self.plan(), pr=0, expected=[], selected=[]) - policy.gate(plan, {"plan": {"result": "success"}}, - lambda _: self.fail("push gate must not query a PR")) + def unavailable(_): + raise subprocess.CalledProcessError(1, "gh") + fallback = policy.create_plan("server", {"pull_request": live}, "apache/server", unavailable) + self.assertEqual(policy.MAINLINE, set(fallback["selected"])) + self.assertEqual((7, "alice/server", "feature", "current-base", "head"), + tuple(fallback[key] for key in ["pr", "source", "branch", "base", "head"])) + for malformed in [{"pull_request": None}, {"pull_request": {}}, + {"pull_request": dict(live, head=dict(live["head"], repo=None))}]: + with self.subTest(event=malformed), self.assertRaises(policy.StaleInputError): + policy.create_plan("server", malformed, "apache/server", lambda _: self.fail("Invalid identity")) if __name__ == "__main__": diff --git a/.github/workflows/check-dependencies.yml b/.github/workflows/check-dependencies.yml index db2ec66aa0..71da915301 100644 --- a/.github/workflows/check-dependencies.yml +++ b/.github/workflows/check-dependencies.yml @@ -3,14 +3,13 @@ name: "3rd-party" on: workflow_call: - workflow_dispatch: permissions: contents: read jobs: dependency-check: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 env: USE_STAGE: 'false' # Whether to include the stage repository. SCRIPT_DEPENDENCY: install-dist/scripts/dependency @@ -53,12 +52,12 @@ jobs: "$SCRIPT_DEPENDENCY/known-dependencies.txt" "$PWD/target/dependency/current-dependencies.txt" dependency-review: - runs-on: ubuntu-latest + if: github.event_name == 'pull_request' + runs-on: ubuntu-24.04 steps: - name: 'Checkout Repository' uses: actions/checkout@v7 - name: 'Dependency Review' - if: github.event_name == 'pull_request' uses: actions/dependency-review-action@v5 # Refer: https://github.com/actions/dependency-review-action with: diff --git a/.github/workflows/cluster-test-ci.yml b/.github/workflows/cluster-test-ci.yml index 09ac85f193..29c7c5bf66 100644 --- a/.github/workflows/cluster-test-ci.yml +++ b/.github/workflows/cluster-test-ci.yml @@ -2,7 +2,6 @@ name: "Cluster Test CI" on: workflow_call: - workflow_dispatch: permissions: contents: read @@ -10,7 +9,7 @@ permissions: jobs: cluster-test: timeout-minutes: 120 - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 env: USE_STAGE: 'false' # Whether to include the stage repository. diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index e0e0aedd42..3f9b906933 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -1,79 +1,36 @@ name: "CodeQL" on: - pull_request: - push: - branches: [master, 'release-*', 'test-*'] - workflow_dispatch: - schedule: - - cron: '33 0 * * 5' - -concurrency: - group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.run_id }}-${{ github.run_attempt == 1 && 'head' || format('rerun-{0}', github.run_id) }} - cancel-in-progress: ${{ github.event_name == 'pull_request' }} + workflow_call: + inputs: + enabled: + type: boolean + default: true permissions: contents: read - pull-requests: read actions: read security-events: write jobs: - plan: - runs-on: ubuntu-24.04 - outputs: - server: ${{ steps.select.outputs.server }} - commons: ${{ steps.select.outputs.commons }} - pd_store: ${{ steps.select.outputs.pd_store }} - cluster: ${{ steps.select.outputs.cluster }} - docker: ${{ steps.select.outputs.docker }} - helm: ${{ steps.select.outputs.helm }} - dependency_license: ${{ steps.select.outputs.dependency_license }} - compatibility: ${{ steps.select.outputs.compatibility }} - smoke: ${{ steps.select.outputs.smoke }} - security: ${{ steps.select.outputs.security }} - steps: - - uses: actions/checkout@v7 - with: - fetch-depth: 0 - persist-credentials: false - - name: Select affected tests - id: select - env: - GH_TOKEN: ${{ github.token }} - run: >- - python3 .github/scripts/ci-policy.py plan --project server - --event-path "$GITHUB_EVENT_PATH" --repository "$GITHUB_REPOSITORY" - --output plan.json - - uses: actions/upload-artifact@v4 - with: - name: ci-plan - path: plan.json - retention-days: 7 - overwrite: true - analyze: - needs: plan - if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || needs.plan.outputs.security == 'true' + if: inputs.enabled env: USE_STAGE: 'false' # Whether to include the stage repository. # Preserve the master scan identity when invoked by a reusable-workflow caller. CODEQL_ACTION_ANALYSIS_KEY: .github/workflows/codeql-analysis.yml:analyze - name: Analyze (${{ matrix.language }}) - runs-on: ubuntu-latest + name: Analyze (java) + runs-on: ubuntu-24.04 permissions: actions: read contents: read security-events: write - strategy: - fail-fast: false - matrix: - language: [ 'java' ] - steps: - name: Checkout repository uses: actions/checkout@v7 + with: + persist-credentials: false - name: Setup Java JDK uses: actions/setup-java@v6 @@ -91,7 +48,7 @@ jobs: - name: Initialize CodeQL uses: github/codeql-action/init@v4 with: - languages: ${{ matrix.language }} + languages: java # If you wish to specify custom queries, you can do so here or in a config file. # By default, queries listed here will override any specified in a config file. # Prefix the list here with "+" to use these queries and those in the config file. @@ -116,4 +73,4 @@ jobs: - name: Perform CodeQL Analysis uses: github/codeql-action/analyze@v4 with: - category: .github/workflows/codeql-analysis.yml:analyze/language:${{ matrix.language }} + category: .github/workflows/codeql-analysis.yml:analyze/language:java diff --git a/.github/workflows/commons-ci.yml b/.github/workflows/commons-ci.yml index 63d86230a3..d84c937af4 100644 --- a/.github/workflows/commons-ci.yml +++ b/.github/workflows/commons-ci.yml @@ -2,29 +2,26 @@ name: "HugeGraph-Commons CI" on: workflow_call: - workflow_dispatch: + inputs: + java-versions: + type: string + required: true permissions: contents: read jobs: - java-runtime: - uses: ./.github/workflows/java-runtime.yml - permissions: - contents: read - build-commons: timeout-minutes: 20 name: Commons tests (Java ${{ matrix.JAVA_VERSION }}) - needs: java-runtime - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 env: USE_STAGE: 'false' # Whether to include the stage repository. strategy: fail-fast: false matrix: - JAVA_VERSION: ${{ fromJSON(needs.java-runtime.outputs.versions || '["unavailable"]') }} + JAVA_VERSION: ${{ fromJSON(inputs.java-versions) }} steps: - name: Checkout @@ -43,8 +40,8 @@ jobs: - name: Use staged maven repo settings if: ${{ env.USE_STAGE == 'true' }} run: | - cp $HOME/.m2/settings.xml /tmp/settings.xml - cp -vf .github/configs/settings.xml $HOME/.m2/settings.xml && cat $HOME/.m2/settings.xml + cp "$HOME/.m2/settings.xml" /tmp/settings.xml + cp -vf .github/configs/settings.xml "$HOME/.m2/settings.xml" && cat "$HOME/.m2/settings.xml" - name: Install run: | diff --git a/.github/workflows/docker-build-ci.yml b/.github/workflows/docker-build-ci.yml index 55a9045141..5ed8fd5acc 100644 --- a/.github/workflows/docker-build-ci.yml +++ b/.github/workflows/docker-build-ci.yml @@ -19,19 +19,37 @@ name: "Docker Build CI" on: workflow_call: - workflow_dispatch: - + inputs: + smoke-only: + type: boolean + default: false permissions: contents: read jobs: - docker-bake-check: - runs-on: ubuntu-latest + docker-build: + runs-on: ubuntu-24.04 + timeout-minutes: 90 + env: + IMAGE_TAG: ci-${{ github.run_id }}-${{ github.run_attempt }} + SOURCE_REVISION: ${{ github.sha }} steps: - name: Checkout + id: checkout uses: actions/checkout@v7 + - name: Install project JDK for deployment contracts + id: java + if: ${{ !cancelled() && steps.checkout.outcome == 'success' && !inputs.smoke-only }} + uses: actions/setup-java@v6 + with: + java-version-file: .github/workflows/.java-version + distribution: 'zulu' + cache: maven + cache-jdk: false + - name: Verify shared Maven stages stay identical + if: ${{ !cancelled() && steps.checkout.outcome == 'success' && !inputs.smoke-only }} shell: bash run: | set -euo pipefail @@ -66,6 +84,7 @@ jobs: done - name: Validate shared image build graph + if: ${{ !cancelled() && steps.checkout.outcome == 'success' && !inputs.smoke-only }} run: | set -euo pipefail @@ -88,58 +107,60 @@ jobs: all(. == "docker")) ' /tmp/hugegraph-bake.json - deployment-contracts: - runs-on: ubuntu-latest - timeout-minutes: 25 - steps: - - uses: actions/checkout@v7 - - uses: actions/setup-java@v6 - with: - java-version: '17' - distribution: 'zulu' - cache: maven - cache-jdk: false - - name: Build the pinned configuration reader - run: | - mvn -B -ntp -pl hugegraph-commons/hugegraph-common -am clean compile dependency:build-classpath \ - -Dmdep.outputFile=target/config-readback-classpath.txt -Dmaven.javadoc.skip=true -DskipTests - - name: Verify deployment shell contracts + - name: Verify deployment shell and Compose contracts + if: ${{ !cancelled() && steps.checkout.outcome == 'success' && steps.java.outcome == 'success' }} + timeout-minutes: 25 run: | - bash hugegraph-server/hugegraph-dist/docker/test/test-docker-entrypoint.sh - HG_CONFIG_TEST_CLASSPATH="$PWD/hugegraph-commons/hugegraph-common/target/classes" - HG_CONFIG_TEST_CLASSPATH+=":$(cat hugegraph-commons/hugegraph-common/target/config-readback-classpath.txt)" - export HG_CONFIG_TEST_CLASSPATH - bash hugegraph-server/hugegraph-dist/docker/docker-entrypoint-test.sh - bash hugegraph-server/hugegraph-dist/src/assembly/travis/test-check-port.sh \ - hugegraph-server/hugegraph-dist/src/assembly/static \ - hugegraph-store/hg-store-dist/src/assembly/static/bin/util.sh - - name: Verify Compose contracts - run: bash docker/test-compose.sh render - - docker-build: - runs-on: ubuntu-latest - timeout-minutes: 45 - env: - IMAGE_TAG: ci-${{ github.run_id }}-${{ github.run_attempt }} - SOURCE_REVISION: ${{ github.sha }} - steps: - - name: Checkout - uses: actions/checkout@v7 + set -euo pipefail + status=0 + if mvn -B -ntp -pl hugegraph-commons/hugegraph-common -am clean compile dependency:build-classpath \ + -Dmdep.outputFile=target/config-readback-classpath.txt -Dmaven.javadoc.skip=true -DskipTests; then + HG_CONFIG_TEST_CLASSPATH="$PWD/hugegraph-commons/hugegraph-common/target/classes" + HG_CONFIG_TEST_CLASSPATH+=":$(cat hugegraph-commons/hugegraph-common/target/config-readback-classpath.txt)" + export HG_CONFIG_TEST_CLASSPATH + if ! bash hugegraph-server/hugegraph-dist/docker/docker-entrypoint-test.sh; then + status=1 + fi + else + status=1 + fi + if ! bash hugegraph-server/hugegraph-dist/docker/test/test-docker-entrypoint.sh; then + status=1 + fi + if ! bash hugegraph-server/hugegraph-dist/src/assembly/travis/test-check-port.sh \ + hugegraph-server/hugegraph-dist/src/assembly/static \ + hugegraph-store/hg-store-dist/src/assembly/static/bin/util.sh; then + status=1 + fi + if ! bash docker/test-compose.sh render; then + status=1 + fi + exit "$status" - - name: Build and load all four PR images with a shared Maven stage + - name: Build current images and verify authentication, readiness and graph read/write + if: ${{ !cancelled() && steps.checkout.outcome == 'success' }} + timeout-minutes: 45 + env: + SMOKE_ONLY: ${{ inputs.smoke-only }} run: | + set -euo pipefail docker buildx create --driver docker-container --use docker buildx bake --file docker/bake.hcl --progress=plain \ --set '*.platform=linux/amd64' \ pd store server-standalone server-hstore - - name: Verify image runtime contracts - run: | - bash .github/scripts/check-docker-images.sh - bash hugegraph-server/hugegraph-dist/docker/test/test-docker-entrypoint.sh - - - name: Verify deployment checker contracts - run: python3 -m unittest discover -s .github/scripts -p 'test_docker_deployment.py' - - - name: Verify exact PR images, authentication and graph read/write - run: python3 .github/scripts/docker-deployment.py "$IMAGE_TAG" + # Each runtime check uses successfully built images. Preserve a failed + # check while allowing the independent deployment check to collect results. + status=0 + if ! bash .github/scripts/check-docker-images.sh; then + status=1 + fi + if [[ "$SMOKE_ONLY" != true ]]; then + if ! python3 -m unittest discover -s .github/scripts -p 'test_docker_deployment.py'; then + status=1 + fi + fi + if ! python3 .github/scripts/docker-deployment.py "$IMAGE_TAG"; then + status=1 + fi + exit "$status" diff --git a/.github/workflows/helm-chart-ci.yml b/.github/workflows/helm-chart-ci.yml index 5ecfaa2074..6e425a3dca 100644 --- a/.github/workflows/helm-chart-ci.yml +++ b/.github/workflows/helm-chart-ci.yml @@ -19,14 +19,13 @@ name: "Helm Chart CI" on: workflow_call: - workflow_dispatch: permissions: contents: read jobs: lint-and-render: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - uses: actions/checkout@v4 with: diff --git a/.github/workflows/java-runtime.yml b/.github/workflows/java-runtime.yml deleted file mode 100644 index 3dbb207090..0000000000 --- a/.github/workflows/java-runtime.yml +++ /dev/null @@ -1,69 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -name: CI Java runtime - -on: - workflow_call: - inputs: - extra-java-versions: - description: Additional JDKs to test alongside the configured project runtime (JSON array). - type: string - default: '[]' - outputs: - versions: - description: Project runtime followed by any additional test JDKs (JSON array). - value: ${{ jobs.read-version.outputs.versions }} - -permissions: - contents: read - -jobs: - read-version: - runs-on: ubuntu-24.04 - outputs: - versions: ${{ steps.java.outputs.versions }} - steps: - - name: Checkout - uses: actions/checkout@v7 - with: - persist-credentials: false - - - name: Read the project runtime and optional test JDKs - id: java - env: - EXTRA_JAVA_VERSIONS: ${{ inputs.extra-java-versions }} - run: | - python3 - <<'PYTHON' - import json - import os - import re - from pathlib import Path - - runtime = Path('.github/workflows/.java-version').read_text().strip() - extra = json.loads(os.environ['EXTRA_JAVA_VERSIONS']) - if not isinstance(extra, list): - raise ValueError('extra-java-versions must be a JSON array') - versions = [] - for version in [runtime, *extra]: - if not isinstance(version, str) or not re.fullmatch(r'[1-9][0-9]*(\.[0-9]+)*', version): - raise ValueError('Java versions must be non-empty numeric strings') - if version not in versions: - versions.append(version) - with open(os.environ['GITHUB_OUTPUT'], 'a') as output: - print('versions=' + json.dumps(versions), file=output) - PYTHON diff --git a/.github/workflows/pd-store-ci.yml b/.github/workflows/pd-store-ci.yml index 20e9f19dc7..00adccfe64 100644 --- a/.github/workflows/pd-store-ci.yml +++ b/.github/workflows/pd-store-ci.yml @@ -2,26 +2,25 @@ name: "HugeGraph-PD & Store & Hstore CI" on: workflow_call: - workflow_dispatch: - + inputs: + run-struct: + type: boolean + default: true + run-pd: + type: boolean + default: true + run-store: + type: boolean + default: true + run-hstore: + type: boolean + default: true permissions: contents: read jobs: - store-stop-shell-test: - permissions: - contents: read - runs-on: ubuntu-24.04 - steps: - - name: Checkout - uses: actions/checkout@v7 - with: - persist-credentials: false - - - name: Verify Store stop failure propagation and PID retention - run: bash hugegraph-server/hugegraph-dist/src/assembly/travis/test-stop-hugegraph-store.sh - rocksdb-compatibility: + if: inputs.run-pd || inputs.run-store || inputs.run-hstore permissions: contents: read runs-on: ubuntu-24.04 @@ -77,7 +76,8 @@ jobs: ${{ runner.temp }}/rocksdb-compatibility/cases/ struct: - runs-on: ubuntu-latest + if: inputs.run-struct || inputs.run-store + runs-on: ubuntu-24.04 env: USE_STAGE: 'false' steps: @@ -94,6 +94,9 @@ jobs: cache: maven cache-jdk: false + - name: Verify Store stop failure propagation and PID retention + run: bash hugegraph-server/hugegraph-dist/src/assembly/travis/test-stop-hugegraph-store.sh + - name: Run JaCoCo report validator tests run: hugegraph-server/hugegraph-dist/src/assembly/travis/test-check-jacoco-report.sh @@ -129,9 +132,9 @@ jobs: mvn -U -ntp -pl hugegraph-struct test pd: + if: inputs.run-pd timeout-minutes: 30 - needs: struct - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 env: # TODO: avoid duplicated env setup in pd & store USE_STAGE: 'false' # Whether to include the stage repository. @@ -311,9 +314,9 @@ jobs: fail_ci_if_error: false store: + if: inputs.run-store timeout-minutes: 45 - needs: struct - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 env: USE_STAGE: 'false' # Whether to include the stage repository. # TODO: remove outdated env @@ -513,8 +516,8 @@ jobs: fail_ci_if_error: false hstore: - needs: struct - runs-on: ubuntu-latest + if: inputs.run-hstore + runs-on: ubuntu-24.04 env: USE_STAGE: 'false' # Whether to include the stage repository. TRAVIS_DIR: hugegraph-server/hugegraph-dist/src/assembly/travis diff --git a/.github/workflows/rerun-ci.yml b/.github/workflows/rerun-ci.yml deleted file mode 100644 index 1e21f28ef4..0000000000 --- a/.github/workflows/rerun-ci.yml +++ /dev/null @@ -1,116 +0,0 @@ -name: "Rerun CI" - -on: - workflow_run: - workflows: - - "HugeGraph-Server CI" - types: - - completed - -permissions: {} - -env: - MAX_RERUNS: '2' - RETRY_DELAY_SECONDS: '180' - -jobs: - decide-rerun-action: - if: ${{ github.event.workflow_run.conclusion == 'failure' && github.event.workflow_run.event == 'push' }} - runs-on: ubuntu-latest - outputs: - action: ${{ steps.decision.outputs.action }} - steps: - - name: Decide rerun action - id: decision - env: - WORKFLOW_NAME: ${{ github.event.workflow_run.name }} - RUN_ID: ${{ github.event.workflow_run.id }} - RUN_ATTEMPT: ${{ github.event.workflow_run.run_attempt }} - EVENT_NAME: ${{ github.event.workflow_run.event }} - HEAD_BRANCH: ${{ github.event.workflow_run.head_branch }} - run: | - set -euo pipefail - - action="skip" - reason="unsupported event: $EVENT_NAME" - - if [[ "$EVENT_NAME" == "push" ]]; then - if (( RUN_ATTEMPT > MAX_RERUNS )); then - reason="retry limit reached" - else - action="rerun" - reason="within retry limit" - fi - fi - - { - echo "action=$action" - echo "reason=$reason" - } >> "$GITHUB_OUTPUT" - - { - echo "### Rerun CI decision" - echo "" - echo "- Workflow: $WORKFLOW_NAME" - echo "- Source event: $EVENT_NAME" - echo "- Head branch: $HEAD_BRANCH" - echo "- Run ID: $RUN_ID" - echo "- Current attempt: $RUN_ATTEMPT" - echo "- Max automatic reruns: $MAX_RERUNS" - echo "- Delay seconds: $RETRY_DELAY_SECONDS" - echo "- Action: $action" - echo "- Reason: $reason" - } >> "$GITHUB_STEP_SUMMARY" - - rerun-failed-jobs: - needs: decide-rerun-action - if: needs.decide-rerun-action.outputs.action == 'rerun' - permissions: - actions: write - contents: read - runs-on: ubuntu-latest - steps: - # workflow_run is privileged: only load code from the default-branch commit. - - name: Checkout trusted retry checker - uses: actions/checkout@v7 - with: - ref: ${{ github.sha }} - persist-credentials: false - sparse-checkout: .github/scripts - - - name: Check source freshness before waiting - id: initial-freshness - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GH_REPO: ${{ github.repository }} - RUN_ID: ${{ github.event.workflow_run.id }} - RUN_ATTEMPT: ${{ github.event.workflow_run.run_attempt }} - run: | - python3 .github/scripts/check-rerun.py \ - --repository "$GH_REPO" --run-id "$RUN_ID" \ - --run-attempt "$RUN_ATTEMPT" --max-reruns "$MAX_RERUNS" - - - name: Wait before rerun - if: steps.initial-freshness.outputs.action == 'rerun' - run: sleep "$RETRY_DELAY_SECONDS" - - - name: Check source freshness after waiting - id: freshness - if: steps.initial-freshness.outputs.action == 'rerun' - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GH_REPO: ${{ github.repository }} - RUN_ID: ${{ github.event.workflow_run.id }} - RUN_ATTEMPT: ${{ github.event.workflow_run.run_attempt }} - run: | - python3 .github/scripts/check-rerun.py \ - --repository "$GH_REPO" --run-id "$RUN_ID" \ - --run-attempt "$RUN_ATTEMPT" --max-reruns "$MAX_RERUNS" - - - name: Rerun the required job and its dependencies - if: steps.freshness.outputs.action == 'rerun' - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GH_REPO: ${{ github.repository }} - run: | - gh api --method POST "repos/$GH_REPO/actions/jobs/${{ steps.freshness.outputs.job_id }}/rerun" diff --git a/.github/workflows/riscv64-ci.yml b/.github/workflows/riscv64-ci.yml index 9812ecfa99..634251fb4e 100644 --- a/.github/workflows/riscv64-ci.yml +++ b/.github/workflows/riscv64-ci.yml @@ -44,7 +44,6 @@ env: jobs: build-server-riscv64: runs-on: ubuntu-24.04 - continue-on-error: true timeout-minutes: 90 steps: @@ -107,10 +106,10 @@ jobs: -P riscv64-protobuf-tools \ -pl hugegraph-server/hugegraph-dist -am \ -Dmaven.test.skip=true -Dmaven.javadoc.skip=true - hugegraph-server/hugegraph-dist/src/assembly/travis/check-rocksdb-only-dist.sh \ - hugegraph-server/apache-hugegraph-server-*/ - hugegraph-server/hugegraph-dist/src/assembly/travis/run-native-runtime-smoke-test.sh \ - hugegraph-server/apache-hugegraph-server-*/ + VERSION=$(mvn help:evaluate -Dexpression=project.version -q -DforceStdout) + SERVER_DIR="hugegraph-server/apache-hugegraph-server-$VERSION" + hugegraph-server/hugegraph-dist/src/assembly/travis/check-rocksdb-only-dist.sh "$SERVER_DIR" + hugegraph-server/hugegraph-dist/src/assembly/travis/run-native-runtime-smoke-test.sh "$SERVER_DIR" ' - name: Show server log on failure diff --git a/.github/workflows/server-ci.yml b/.github/workflows/server-ci.yml index 84e29fc3c4..61867828a3 100644 --- a/.github/workflows/server-ci.yml +++ b/.github/workflows/server-ci.yml @@ -2,15 +2,20 @@ name: "HugeGraph-Server CI" on: push: - branches: - - master - - 'release-*' - - 'test-*' - - 'tinkerpop-*' + branches: [master, 'release-*', 'test-*', 'tinkerpop-*'] pull_request: + schedule: + - cron: '0 6 * * 1' + timezone: Asia/Taipei workflow_dispatch: - -# Keep the newest PR run; preserve independent push, release and manual runs. + inputs: + test-set: + description: Tests to run on the selected branch + type: choice + default: mainline + options: [mainline, distributed, macos-arm, hbase, riscv64, codeql, full] + +# A new PR head replaces its old run; scheduled and manual runs stay independent. concurrency: group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.run_id }}-${{ github.run_attempt == 1 && 'head' || format('rerun-{0}', github.run_id) }} cancel-in-progress: ${{ github.event_name == 'pull_request' }} @@ -20,262 +25,171 @@ permissions: pull-requests: read jobs: - check-license: - runs-on: ubuntu-latest - env: - USE_STAGE: 'false' # Whether to include the stage repository. - steps: - - uses: actions/checkout@v7 - - - name: Check License Header - uses: apache/skywalking-eyes@main - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - log: info - token: ${{ github.token }} - config: .licenserc.yaml - - - name: License Check (RAT) - run: | - mvn apache-rat:check -ntp - find ./ -name rat.txt -print0 | xargs -0 -I file cat file > merged-rat.txt - grep "Binaries" merged-rat.txt -C 3 && cat merged-rat.txt - - plan: - needs: check-license + preflight: runs-on: ubuntu-24.04 + timeout-minutes: 15 + env: + PYTHONDONTWRITEBYTECODE: '1' outputs: + selection_result: ${{ steps.select.outcome }} + java_result: ${{ steps.java.outcome }} + java_versions: ${{ steps.java.outputs.versions }} server: ${{ steps.select.outputs.server }} commons: ${{ steps.select.outputs.commons }} + struct: ${{ steps.select.outputs.struct }} + pd: ${{ steps.select.outputs.pd }} + store: ${{ steps.select.outputs.store }} + hstore: ${{ steps.select.outputs.hstore }} pd_store: ${{ steps.select.outputs.pd_store }} cluster: ${{ steps.select.outputs.cluster }} docker: ${{ steps.select.outputs.docker }} helm: ${{ steps.select.outputs.helm }} dependency_license: ${{ steps.select.outputs.dependency_license }} - compatibility: ${{ steps.select.outputs.compatibility }} - smoke: ${{ steps.select.outputs.smoke }} + macos: ${{ steps.select.outputs.macos }} + macos_full: ${{ steps.select.outputs.macos_full }} + hbase: ${{ steps.select.outputs.hbase }} + riscv64: ${{ steps.select.outputs.riscv64 }} security: ${{ steps.select.outputs.security }} + distributed_smoke: ${{ steps.select.outputs.distributed_smoke }} steps: - - uses: actions/checkout@v7 + - name: Checkout + id: checkout + uses: actions/checkout@v7 with: fetch-depth: 0 persist-credentials: false - - name: Test selection, gate and retry policy - run: python3 -m unittest discover -s .github/scripts -p 'test_*.py' + - name: Select affected tests id: select env: GH_TOKEN: ${{ github.token }} run: >- python3 .github/scripts/ci-policy.py plan --project server - --event-path "$GITHUB_EVENT_PATH" --repository "$GITHUB_REPOSITORY" - --output plan.json - - uses: actions/upload-artifact@v4 + --event-name "$GITHUB_EVENT_NAME" --event-path "$GITHUB_EVENT_PATH" + --repository "$GITHUB_REPOSITORY" --output "$RUNNER_TEMP/ci-plan.json" + + - name: Read project Java runtime + id: java + if: ${{ !cancelled() && steps.checkout.outcome == 'success' }} + run: | + python3 - <<'PYTHON' + import json + import os + import re + from pathlib import Path + + runtime = Path('.github/workflows/.java-version').read_text().strip() + if not re.fullmatch(r'[1-9][0-9]*(\.[0-9]+)*', runtime): + raise ValueError('Project Java runtime must be a non-empty numeric version') + with open(os.environ['GITHUB_OUTPUT'], 'a') as output: + print('versions=' + json.dumps([runtime]), file=output) + PYTHON + + - name: Record selection + if: ${{ !cancelled() && steps.select.outcome == 'success' }} + uses: actions/upload-artifact@v4 with: name: ci-plan - path: plan.json + path: ${{ runner.temp }}/ci-plan.json retention-days: 7 overwrite: true - java-runtime: - needs: check-license - uses: ./.github/workflows/java-runtime.yml - permissions: - contents: read + - name: Test CI policy and deployment checks + if: ${{ !cancelled() && steps.checkout.outcome == 'success' }} + run: python3 -m unittest discover -s .github/scripts -p 'test_*.py' - wait-storage-shell-test: - name: Startup contracts - needs: plan - if: needs.plan.outputs.server == 'true' - permissions: - contents: read - runs-on: ubuntu-24.04 - steps: - - name: Checkout - uses: actions/checkout@v7 + - name: Check license headers + if: ${{ !cancelled() && steps.checkout.outcome == 'success' }} + uses: apache/skywalking-eyes@main with: - persist-credentials: false - - - name: Test CI retry freshness checks - run: python3 -m unittest discover -s .github/scripts -p 'test_check_rerun.py' - - - name: Run wait-storage.sh peer failover tests - run: hugegraph-server/hugegraph-dist/src/assembly/travis/test-wait-storage.sh + log: info + token: ${{ github.token }} + config: .licenserc.yaml - - name: Run RISC-V Java runtime contract tests - run: hugegraph-server/hugegraph-dist/src/assembly/travis/test-riscv64-java-runtime.sh + - name: License Check (RAT) + if: ${{ !cancelled() && steps.checkout.outcome == 'success' }} + run: | + mvn apache-rat:check -ntp + find . -name rat.txt -print0 | xargs -0 -I file cat file > "$RUNNER_TEMP/merged-rat.txt" + cat "$RUNNER_TEMP/merged-rat.txt" - - name: Check TinkerPop report execution gates - run: bash hugegraph-server/hugegraph-dist/src/assembly/travis/test-tinkerpop-reports.sh + build-server: + needs: preflight + if: ${{ !cancelled() && needs.preflight.outputs.selection_result == 'success' && needs.preflight.outputs.java_result == 'success' && needs.preflight.outputs.server == 'true' }} + uses: ./.github/workflows/server-tests.yml + with: + backend: memory + java-versions: ${{ needs.preflight.outputs.java_versions }} + secrets: inherit - # RocksDB coverage uses the project runtime and any additional test JDKs. - # The shared runtime comes from .github/workflows/.java-version, independent of bytecode targets. - # Keep the job ID for the Codecov configuration validator. server-rocksdb: - needs: [plan, java-runtime] - if: needs.plan.outputs.server == 'true' + needs: preflight + if: ${{ !cancelled() && needs.preflight.outputs.selection_result == 'success' && needs.preflight.outputs.java_result == 'success' && needs.preflight.outputs.server == 'true' }} uses: ./.github/workflows/server-tests.yml with: backend: rocksdb - java-versions: ${{ needs.java-runtime.outputs.versions }} + java-versions: ${{ needs.preflight.outputs.java_versions }} secrets: inherit commons: - name: commons - needs: plan - if: needs.plan.outputs.commons == 'true' + needs: preflight + if: ${{ !cancelled() && needs.preflight.outputs.selection_result == 'success' && needs.preflight.outputs.java_result == 'success' && needs.preflight.outputs.commons == 'true' }} uses: ./.github/workflows/commons-ci.yml + with: + java-versions: ${{ needs.preflight.outputs.java_versions }} secrets: inherit pd_store: - name: pd_store - needs: plan - if: needs.plan.outputs.pd_store == 'true' + needs: preflight + if: ${{ !cancelled() && needs.preflight.outputs.selection_result == 'success' && needs.preflight.outputs.java_result == 'success' && needs.preflight.outputs.pd_store == 'true' }} uses: ./.github/workflows/pd-store-ci.yml + with: + run-struct: ${{ needs.preflight.outputs.struct == 'true' }} + run-pd: ${{ needs.preflight.outputs.pd == 'true' }} + run-store: ${{ needs.preflight.outputs.store == 'true' }} + run-hstore: ${{ needs.preflight.outputs.hstore == 'true' }} secrets: inherit cluster: - name: cluster - needs: plan - if: needs.plan.outputs.cluster == 'true' + needs: preflight + if: ${{ !cancelled() && needs.preflight.outputs.selection_result == 'success' && needs.preflight.outputs.java_result == 'success' && needs.preflight.outputs.cluster == 'true' }} uses: ./.github/workflows/cluster-test-ci.yml secrets: inherit docker: - name: docker - needs: plan - if: needs.plan.outputs.docker == 'true' + needs: preflight + if: ${{ !cancelled() && needs.preflight.outputs.selection_result == 'success' && needs.preflight.outputs.java_result == 'success' && (needs.preflight.outputs.docker == 'true' || needs.preflight.outputs.distributed_smoke == 'true') }} uses: ./.github/workflows/docker-build-ci.yml - secrets: inherit + with: + smoke-only: ${{ needs.preflight.outputs.distributed_smoke == 'true' && needs.preflight.outputs.docker != 'true' }} helm: - name: helm - needs: plan - if: needs.plan.outputs.helm == 'true' + needs: preflight + if: ${{ !cancelled() && needs.preflight.outputs.selection_result == 'success' && needs.preflight.outputs.helm == 'true' }} uses: ./.github/workflows/helm-chart-ci.yml dependency_license: - name: dependency_license - needs: plan - if: needs.plan.outputs.dependency_license == 'true' + needs: preflight + if: ${{ !cancelled() && needs.preflight.outputs.selection_result == 'success' && needs.preflight.outputs.java_result == 'success' && needs.preflight.outputs.dependency_license == 'true' }} uses: ./.github/workflows/check-dependencies.yml - affected-module-tests: - name: affected-module-tests - needs: [plan, java-runtime, wait-storage-shell-test, server-rocksdb, commons, pd_store, cluster, docker, helm, dependency_license] - if: ${{ !cancelled() }} - runs-on: ubuntu-24.04 - timeout-minutes: 5 - steps: - - uses: actions/checkout@v7 - with: - persist-credentials: false - - name: Check planning and runtime results - env: - JOB_RESULTS: ${{ toJSON(needs) }} - run: | - python3 - <<'PYGATE' - import json, os - results = json.loads(os.environ['JOB_RESULTS']) - if results['plan']['result'] != 'success': - raise SystemExit('CI planning failed') - if results['plan']['outputs'].get('server') == 'true': - for job in ['java-runtime', 'wait-storage-shell-test']: - if results[job]['result'] != 'success': - raise SystemExit('Selected prerequisite failed: ' + job) - results['server_rocksdb'] = results['server-rocksdb'] - with open('results.json', 'w') as output: - json.dump(results, output) - PYGATE - - uses: actions/download-artifact@v4 - with: - name: ci-plan - - name: Report selected suites and their real results - env: - GH_TOKEN: ${{ github.token }} - run: python3 .github/scripts/ci-policy.py gate --mode advisory --plan plan.json --results results.json --output test-results.json - - memory-plan: - needs: check-license - runs-on: ubuntu-24.04 - outputs: - server: ${{ steps.select.outputs.server }} - commons: ${{ steps.select.outputs.commons }} - pd_store: ${{ steps.select.outputs.pd_store }} - cluster: ${{ steps.select.outputs.cluster }} - docker: ${{ steps.select.outputs.docker }} - helm: ${{ steps.select.outputs.helm }} - dependency_license: ${{ steps.select.outputs.dependency_license }} - compatibility: ${{ steps.select.outputs.compatibility }} - smoke: ${{ steps.select.outputs.smoke }} - security: ${{ steps.select.outputs.security }} - steps: - - uses: actions/checkout@v7 - with: - fetch-depth: 0 - persist-credentials: false - - name: Select affected tests - id: select - env: - GH_TOKEN: ${{ github.token }} - run: >- - python3 .github/scripts/ci-policy.py plan --project server - --event-path "$GITHUB_EVENT_PATH" --repository "$GITHUB_REPOSITORY" - --output plan.json - - uses: actions/upload-artifact@v4 - with: - name: ci-memory-plan - path: plan.json - retention-days: 7 - overwrite: true - - memory-java-runtime: - needs: memory-plan - if: needs.memory-plan.outputs.server == 'true' - uses: ./.github/workflows/java-runtime.yml - permissions: - contents: read - - build-server: - needs: [memory-plan, memory-java-runtime] - if: needs.memory-plan.outputs.server == 'true' - uses: ./.github/workflows/server-tests.yml + compatibility: + needs: preflight + if: ${{ !cancelled() && needs.preflight.outputs.selection_result == 'success' && needs.preflight.outputs.java_result == 'success' && (needs.preflight.outputs.macos == 'true' || needs.preflight.outputs.hbase == 'true' || needs.preflight.outputs.riscv64 == 'true') }} + uses: ./.github/workflows/server-compatibility-ci.yml with: - backend: memory - java-versions: ${{ needs.memory-java-runtime.outputs.versions }} + java-versions: ${{ needs.preflight.outputs.java_versions }} + macos: ${{ needs.preflight.outputs.macos == 'true' }} + macos-full: ${{ needs.preflight.outputs.macos_full == 'true' }} + hbase: ${{ needs.preflight.outputs.hbase == 'true' }} + riscv64: ${{ needs.preflight.outputs.riscv64 == 'true' }} secrets: inherit - server-memory-required: - name: Server memory tests - needs: [memory-plan, memory-java-runtime, build-server] - if: ${{ always() }} - runs-on: ubuntu-24.04 - timeout-minutes: 5 - steps: - - uses: actions/checkout@v7 - with: - persist-credentials: false - - name: Check planning and runtime results - env: - JOB_RESULTS: ${{ toJSON(needs) }} - run: | - python3 - <<'PYGATE' - import json, os - results = json.loads(os.environ['JOB_RESULTS']) - if results['memory-plan']['result'] != 'success': - raise SystemExit('CI planning failed') - if results['memory-plan']['outputs'].get('server') == 'true' and results['memory-java-runtime']['result'] != 'success': - raise SystemExit('Selected Java runtime preparation failed') - results['plan'] = results['memory-plan'] - results['server_memory'] = results['build-server'] - with open('results.json', 'w') as output: - json.dump(results, output) - PYGATE - - uses: actions/download-artifact@v4 - with: - name: ci-memory-plan - - name: Report selected suites and their real results - env: - GH_TOKEN: ${{ github.token }} - run: python3 .github/scripts/ci-policy.py gate --mode memory --plan plan.json --results results.json --output test-results.json + codeql: + needs: preflight + if: ${{ !cancelled() && needs.preflight.outputs.selection_result == 'success' && needs.preflight.outputs.java_result == 'success' && needs.preflight.outputs.security == 'true' }} + uses: ./.github/workflows/codeql-analysis.yml + permissions: + contents: read + actions: read + security-events: write diff --git a/.github/workflows/server-compatibility-ci.yml b/.github/workflows/server-compatibility-ci.yml index 81b9d3cdfc..422182fdd5 100644 --- a/.github/workflows/server-compatibility-ci.yml +++ b/.github/workflows/server-compatibility-ci.yml @@ -1,95 +1,56 @@ name: "Server Compatibility CI" on: - push: - branches: - - master - - 'release-*' - - 'test-*' - pull_request: - workflow_dispatch: - -# Keep the newest PR run; preserve independent push, release and manual runs. -concurrency: - group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.run_id }}-${{ github.run_attempt == 1 && 'head' || format('rerun-{0}', github.run_id) }} - cancel-in-progress: ${{ github.event_name == 'pull_request' }} + workflow_call: + inputs: + java-versions: + type: string + required: true + hbase: + type: boolean + default: false + riscv64: + type: boolean + default: false + macos: + type: boolean + default: false + macos-full: + type: boolean + default: false permissions: contents: read - pull-requests: read jobs: - plan: - runs-on: ubuntu-24.04 - outputs: - server: ${{ steps.select.outputs.server }} - commons: ${{ steps.select.outputs.commons }} - pd_store: ${{ steps.select.outputs.pd_store }} - cluster: ${{ steps.select.outputs.cluster }} - docker: ${{ steps.select.outputs.docker }} - helm: ${{ steps.select.outputs.helm }} - dependency_license: ${{ steps.select.outputs.dependency_license }} - compatibility: ${{ steps.select.outputs.compatibility }} - smoke: ${{ steps.select.outputs.smoke }} - security: ${{ steps.select.outputs.security }} - steps: - - uses: actions/checkout@v7 - with: - fetch-depth: 0 - persist-credentials: false - - name: Select affected tests - id: select - env: - GH_TOKEN: ${{ github.token }} - run: >- - python3 .github/scripts/ci-policy.py plan --project server - --event-path "$GITHUB_EVENT_PATH" --repository "$GITHUB_REPOSITORY" - --output plan.json - - uses: actions/upload-artifact@v4 - with: - name: ci-plan - path: plan.json - retention-days: 7 - overwrite: true - - java-runtime: - uses: ./.github/workflows/java-runtime.yml - permissions: - contents: read - build-server-backends: - needs: [plan, java-runtime] - if: needs.plan.outputs.compatibility == 'true' + if: inputs.hbase uses: ./.github/workflows/server-tests.yml with: backend: hbase - java-versions: ${{ needs.java-runtime.outputs.versions }} + java-versions: ${{ inputs.java-versions }} secrets: inherit build-server-macos-rocksdb: - needs: plan - if: ${{ !cancelled() && needs.plan.result == 'success' && needs.plan.outputs.compatibility == 'true' }} - runs-on: ${{ matrix.os }} - strategy: - fail-fast: false - matrix: - include: - - os: macos-15-intel - server_java_options: '' - - os: macos-15 - server_java_options: '-Xms512m -Xmx2g' + name: macOS ARM (${{ inputs.macos-full && 'full' || 'smoke' }}) + if: inputs.macos + runs-on: macos-15 + timeout-minutes: 60 env: USE_STAGE: 'false' # Whether to include the stage repository. TRAVIS_DIR: hugegraph-server/hugegraph-dist/src/assembly/travis REPORT_DIR: target/site/jacoco BACKEND: rocksdb - SERVER_JAVA_OPTIONS: ${{ matrix.server_java_options }} + EXPECTED_ARCH: arm64 + EXPECTED_JAVA_MAJOR: ${{ fromJSON(inputs.java-versions)[0] }} + SERVER_JAVA_OPTIONS: '-Xms512m -Xmx2g' steps: - name: Checkout uses: actions/checkout@v7 with: fetch-depth: 5 + persist-credentials: false - name: Install project JDK uses: actions/setup-java@v6 @@ -102,33 +63,48 @@ jobs: - name: Use staged maven repo settings if: ${{ env.USE_STAGE == 'true' }} run: | - cp $HOME/.m2/settings.xml /tmp/settings.xml - cp -vf .github/configs/settings.xml $HOME/.m2/settings.xml && cat $HOME/.m2/settings.xml + cp "$HOME/.m2/settings.xml" /tmp/settings.xml + cp -vf .github/configs/settings.xml "$HOME/.m2/settings.xml" && cat "$HOME/.m2/settings.xml" - - name: Compile + - name: Prepare bounded runtime commands run: | - mvn clean compile -pl hugegraph-server/hugegraph-test -am -U -Dmaven.javadoc.skip=true -ntp + if ! command -v timeout >/dev/null 2>&1 && ! command -v gtimeout >/dev/null 2>&1; then + brew install coreutils + fi + + - name: Package RocksDB distribution + run: | + mvn clean package -pl hugegraph-server/hugegraph-dist -am -U -Drocksdb-only \ + -Dmaven.test.skip=true -Dmaven.javadoc.skip=true -ntp - name: Run check_port unit tests # Validates the ss/netstat port preflight that replaced lsof run: | - $TRAVIS_DIR/test-check-port.sh \ + "$TRAVIS_DIR/test-check-port.sh" \ hugegraph-server/hugegraph-dist/src/assembly/static \ hugegraph-store/hg-store-dist/src/assembly/static/bin/util.sh + - name: Run native startup, JNI and restart smoke + run: | + VERSION=$(mvn help:evaluate -Dexpression=project.version -q -DforceStdout) + SERVER_DIR="hugegraph-server/apache-hugegraph-server-$VERSION" + "$TRAVIS_DIR/run-native-runtime-smoke-test.sh" "$SERVER_DIR" + - name: Run RocksDB core test + if: inputs.macos-full run: | - $TRAVIS_DIR/run-core-test.sh $BACKEND + "$TRAVIS_DIR/run-core-test.sh" "$BACKEND" - name: Run RocksDB API test + if: inputs.macos-full run: | - $TRAVIS_DIR/run-api-test.sh $BACKEND $REPORT_DIR true + "$TRAVIS_DIR/run-api-test.sh" "$BACKEND" "$REPORT_DIR" true - name: Show server log on failure if: failure() run: | VERSION=$(mvn help:evaluate -Dexpression=project.version -q -DforceStdout) - SERVER_DIR=hugegraph-server/apache-hugegraph-server-$VERSION/ + SERVER_DIR="hugegraph-server/apache-hugegraph-server-$VERSION" if [ -f "$SERVER_DIR/logs/hugegraph-server.log" ]; then tail -n 200 "$SERVER_DIR/logs/hugegraph-server.log" fi @@ -137,28 +113,13 @@ jobs: if: always() run: | VERSION=$(mvn help:evaluate -Dexpression=project.version -q -DforceStdout) - SERVER_DIR=hugegraph-server/apache-hugegraph-server-$VERSION/ + SERVER_DIR="hugegraph-server/apache-hugegraph-server-$VERSION" if [ -f "$SERVER_DIR/bin/pid" ]; then - $TRAVIS_DIR/stop-server.sh $SERVER_DIR || true + "$TRAVIS_DIR/stop-server.sh" "$SERVER_DIR" || true fi build-server-riscv64: - # Keep the emulated Java 17 RISC-V native-runtime signal visible while the - # platform lane stabilizes, but do not use it as a gate yet. - needs: plan - if: ${{ !cancelled() && needs.plan.result == 'success' && needs.plan.outputs.compatibility == 'true' }} + if: inputs.riscv64 uses: ./.github/workflows/riscv64-ci.yml permissions: contents: read - - compose-smoke: - name: Published Compose smoke - needs: plan - if: needs.plan.outputs.smoke == 'true' - runs-on: ubuntu-24.04 - timeout-minutes: 25 - steps: - - uses: actions/checkout@v7 - with: - persist-credentials: false - - run: bash docker/test-compose.sh smoke diff --git a/.github/workflows/server-tests.yml b/.github/workflows/server-tests.yml index 05ac0862bb..f86d0904bb 100644 --- a/.github/workflows/server-tests.yml +++ b/.github/workflows/server-tests.yml @@ -73,6 +73,13 @@ jobs: run: | mvn clean compile -pl hugegraph-server/hugegraph-test -am -U -Dmaven.javadoc.skip=true -ntp + - name: Check startup and native runtime contracts + if: inputs.backend == 'rocksdb' + run: | + "$TRAVIS_DIR/test-wait-storage.sh" + "$TRAVIS_DIR/test-riscv64-java-runtime.sh" + bash "$TRAVIS_DIR/test-tinkerpop-reports.sh" + - name: Validate Docker integration if: ${{ env.BACKEND == 'rocksdb' }} run: | diff --git a/docs/ci.md b/docs/ci.md index 511f8721a2..c299dfefc1 100644 --- a/docs/ci.md +++ b/docs/ci.md @@ -1,101 +1,100 @@ # CI policy -CI selects tests from the cumulative pull-request diff and known consumer dependencies. -Selection controls what runs; branch protection controls what must pass before merging. - -| Workflow | Purpose | Merge requirement | -| --- | --- | --- | -| HugeGraph-Server CI | License headers and RAT, then Memory and related modules, images and dependency audits | `check-license`, `Server memory tests`; other product checks advisory | -| CodeQL | Source analysis, independently selected and scheduled | Advisory | -| Server Compatibility CI | HBase, macOS, RISC-V and published-image Compose smoke | Advisory | - -HugeGraph-Server CI runs `check-license` before both Memory and advisory planning or -runtime preparation. A failed license check skips product jobs and fails the Memory required -result. Both required check names stay unchanged. Memory keeps its own planner and result -check so advisory failures cannot block a passing Memory result. Compatibility and CodeQL -remain independently scheduled. - -Jobs within a workflow share its cancellation scope. Cancelling HugeGraph-Server CI also -cancels its required validation, which must complete before merging. It does not cancel -compatibility or prevent CodeQL from starting. Failed advisory checks retain their real failure -result (except the existing RISC-V job, which still uses `continue-on-error`); -maintainers decide whether they need another run or a requested change. - -## Affected inputs - -| Changed inputs | Selected validation | +[HugeGraph-Server CI](../.github/workflows/server-ci.yml) is the single entry point for +PRs, supported branch pushes, weekly coverage and manual validation. Its preflight job +selects affected tests, reads the project Java version, checks CI policy and verifies +license headers and RAT. Selected suites run through reusable workflows in the same run. + +All CI results are advisory. [`.asf.yaml`](../.asf.yaml) requests no required CI contexts +and retains one approving review. This protection change takes effect after the configuration +is merged and synchronized by Gitbox; verify live protection then. Failed tests keep their +real failed results. Valid selection and Java outputs allow selected tests to run even when +the preflight license checks fail. + +## What runs + +| Validation | Ordinary PR | Push to master | Weekly full run | +| --- | --- | --- | --- | +| Memory and RocksDB | Full unit, core and API tests when affected | No repeated full tests | Full tests | +| macOS ARM | Startup, JNI, graph read/write and restart smoke when affected; full core/API for platform inputs | Affected smoke | Full core/API | +| HBase and RISC-V | Directly related backend/platform or validation inputs only | No repeated tests | Full validation | +| PD, Store, HStore and Cluster | Affected modules and known consumers | Current-commit distributed smoke | Full group | +| CodeQL | Authentication, authorization, security, POM or CodeQL configuration inputs | No repeated scan | Full scan | +| Docker, Helm and dependency audit | Corresponding inputs | Corresponding deployment smoke or checks | Full validation | + +macOS uses one `macos-15` ARM job for either smoke or full validation. Intel macOS is removed. +The weekly run uses `cron: '0 6 * * 1'` with `timezone: Asia/Taipei`: Monday at 06:00, +on the latest default-branch commit. GitHub may delay scheduled runs during high load; +see [schedule behavior](https://docs.github.com/en/actions/reference/workflows-and-actions/events-that-trigger-workflows#schedule). + +### Affected inputs + +The rules live in [ci-policy.py](../.github/scripts/ci-policy.py). Ordinary Server changes +select both Memory and RocksDB, plus ARM smoke. They do not select the complete PD/Store group. +PD and Store changes select their own tests and downstream consumers; Commons and Struct +select their known shared consumers. HStore and Cluster additionally follow distributed +backend contracts, protocols, metadata, graph lifecycle and scheduling inputs. Distribution, +startup and RocksDB inputs select ARM full validation and their deployment consumers. + +Unclassified paths or an unavailable diff/API conservatively select mainline coverage. +HBase and RISC-V remain limited to explicit related inputs, weekly coverage or manual selection. +The classifier uses paths, not PR titles or semantic interpretation of changed text. + +### Documentation + +Documentation-only ordinary PRs and pushes allocate just the preflight CI runner. This count +excludes the non-test [Auto PR Commenter](../.github/workflows/auto-pr-review.yml), which runs +once when a PR opens. Scheduled and manual selections still run their requested suites. + +Plain documentation includes regular-file root prose, README/AGENTS files and supported prose +or static images under `docs/`, including nested module documentation. Runtime resources, +configuration, source, fixtures, templates and executable or symbolic-link files do not qualify; +for example, a test fixture ending in `.txt` still selects its module's tests. + +## Manual validation and reruns + +Use **Actions → HugeGraph-Server CI → Run workflow**, select a branch and choose `test-set`: + +| Test set | Validation | | --- | --- | -| Server, including RocksDB and HStore | Memory, RocksDB, PD/Store/HStore and Cluster | -| Server distribution or shared startup scripts | Server, PD/Store/HStore, Docker and Cluster | -| Commons or Struct | Their tests and affected Server, PD, Store, HStore and Cluster tests | -| PD or Store distribution scripts, config, assembly or POM | PD/Store/HStore, Cluster and Docker | -| Other PD or Store inputs | PD/Store/HStore and Cluster | -| Server parent/API POM or Commons version resource | Normal module coverage plus Docker artifact checks | -| Cluster, Docker or Helm | Their suite and known consumers | -| PR template, Dependabot configuration or retry checker/tests | Planner checks and license checks; no product suites | -| Docker image/deployment checkers and their tests | Docker | -| Dependencies, shared build inputs or unknown paths | Conservative full coverage | - -The Memory required result depends only on its planner, runtime preparation and real Memory -matrix. A selected test that fails, is cancelled or unexpectedly skipped cannot pass it. -When Memory is not selected, the result explains that no Memory test was needed. -The advisory `affected-module-tests` summary lists selection, required status and actual results. -It does not stand in for Memory or control CodeQL and compatibility checks. - -The Docker suite builds the current four production images through a shared Maven build. -Checks consume those final images, verify identity, health, version and authentication, and -exercise graph writes, reads and Gremlin queries on standalone and PD/Store/Server topologies. Published-image Hubble Compose smoke runs once in -compatibility CI when RocksDB or Docker is selected; it does not verify the new PR images. - -HBase compatibility uses the existing HBase version in one standalone container with local -storage. Its image is pinned by digest; no separate HDFS service or CI image build is needed. -Pull, startup and readiness preparation have a five-minute total budget. Preparation failure -ends the advisory check without falling back to a tar download. Existing HBase behavior tests -remain. This changes CI preparation, not product support or backend deprecation policy. - -TinkerPop suites run only when the source, target or push/manual branch starts with -`release-`, `test-` or `tinkerpop-`. PRs run no TP suites when neither source nor target -branch matches these prefixes, including `upgrade/1.8.0` targeting `master`. -Executed TP suites require non-empty reports with actual executed tests. - -PD, Store and Commons have total job budgets of 30, 45 and 20 minutes, respectively. -Cluster has 120 minutes for its two 45-minute test windows and build/diagnostic time. -Server and HStore budgets are defined in their workflow job settings. - -## Documentation and freshness - -Only explicitly allowed prose and static documentation assets qualify as plain documentation. -Packaged resources, configuration, source, tests and CI inputs never qualify. Documentation-only -PRs skip compilation, backend services, images and PR CodeQL; lightweight selection and license -checks still report. A source PR with a later documentation commit still tests its cumulative -source changes. No result is reused from a previous run. - -Plans record the event head and the actual merge commit and base tested. Checkout must match -the event merge SHA, whose second parent must match the current PR head. Its first parent is -the tested base; the event base may lag when master advances before checkout. A new source -head invalidates the old run; target-branch advancement alone -does not, matching non-strict branch protection. A selection/API failure conservatively selects -all suites. A final metadata outage alone cannot invalidate completed Memory tests. -Plans and actual results are diagnostics, not execution credentials for later runs. - -## Protection and retries - -`.asf.yaml` requests only `check-license` and `Server memory tests`. Keep these required check -names stable. The Memory jobs run real unit, core and API tests on the project runtime -and report their results through `Server memory tests`. Commons tests remain advisory -and install only the Commons reactor and its upstream modules. -`affected-module-tests` and CodeQL remain advisory. Verify live branch protection -when changing required checks; do not forge a successful result or leave retired contexts required. - -Push runs whose overall conclusion is `failure` keep automatic retries for license and Memory -validation: wait 180 seconds, then retry at most twice, sharing the combined run's attempt limit. -Whole-run `timed_out` conclusions do not trigger retries; PR retries remain paused. -The trusted default-branch checker verifies the source workflow, failed completed attempt, -repository and unchanged branch head before and after the delay. It selects one license or -Memory producer with a `failure` or `timed_out` conclusion, or its failed result reporter, -from that attempt and uses -[GitHub's native job rerun](https://docs.github.com/en/rest/actions/workflow-runs#re-run-a-job-from-a-workflow-run), -which also reruns dependent jobs. A Memory retry updates its required result without restarting unrelated product jobs. -A license retry can unblock its previously skipped product dependencies. Advisory failures alone -do not trigger automatic retries. Use a manual rerun when appropriate; it retains the original -commit and event. +| `mainline` (default) | Full mainline modules, deployment/dependency checks and CodeQL; ARM smoke | +| `distributed` | Struct, PD, Store, HStore and Cluster, plus distributed smoke | +| `macos-arm` | ARM smoke and full RocksDB core/API tests | +| `hbase`, `riscv64`, `codeql` | The corresponding complete specialist validation | +| `full` | All retained suites, identical to weekly coverage | + +Fork contributors run additional validation in their fork's Actions. The manual entry point +does not accept a PR number, apply labels or reuse another workflow's results. +Automatic retries are disabled. Use GitHub's native rerun controls for failed jobs or a run; +a rerun retains its original commit and event, subject to the preflight freshness checks. + +## Freshness, runner use and budgets + +PR selection uses the cumulative diff against the checked-out merge commit's base. A source +PR followed by a documentation-only commit still tests the cumulative source changes. +Preflight verifies that the event merge commit's second parent matches the current PR head. +A stale PR head fails preflight. Target-branch advancement alone does not invalidate a run. +Plans and selection summaries describe this run; previous results are never reused. + +A new PR head cancels the previous PR run, including its called workflows. Manual and weekly +runs remain independent. Each job reports its own result; there are no separate post-test +aggregation runners or end-of-run metadata checks. + +Startup contracts run inside RocksDB validation, and Store stop contracts run inside Struct. +Docker's static checks, deployment contracts and current-image validation share one runner. +The Docker job has a 90-minute total budget, with 25 minutes for deployment contracts and +45 minutes for image build/runtime validation. Distributed smoke builds current PD, Store +and Server images and checks authentication, readiness and graph read/write. It does not +substitute published images for the current commit. + +PD, Store and Commons have total job budgets of 30, 45 and 20 minutes. Cluster has 120 minutes +for its two 45-minute test windows and build/diagnostics. Other budgets are in the workflows. +TinkerPop suites keep their dedicated branch policy: the source, target or push/manual branch +must start with `release-`, `test-` or `tinkerpop-`. Executed suites require non-empty reports +with actual tests. + +Each separate job or matrix item requests a runner; steps within a job share one. +[Standard hosted runners are free for public repositories](https://docs.github.com/en/actions/reference/runners/github-hosted-runners#standard-github-hosted-runners-for-public-repositories), +but [plan-dependent total and macOS concurrency limits](https://docs.github.com/en/actions/reference/limits#job-concurrency-limits-for-github-hosted-runners) +still apply. Reducing job allocations and long executions lowers shared-pool demand; +it does not guarantee a specific queue delay. diff --git a/hugegraph-server/hugegraph-dist/src/assembly/travis/run-native-runtime-smoke-test.sh b/hugegraph-server/hugegraph-dist/src/assembly/travis/run-native-runtime-smoke-test.sh index 8c42ecf7c1..524b478846 100755 --- a/hugegraph-server/hugegraph-dist/src/assembly/travis/run-native-runtime-smoke-test.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/travis/run-native-runtime-smoke-test.sh @@ -47,13 +47,21 @@ cleanup() { trap cleanup EXIT start_server() { - if ! command -v timeout >/dev/null 2>&1; then - echo "Required command is unavailable: timeout" >&2 + if command -v timeout >/dev/null 2>&1; then + TIMEOUT_COMMAND=timeout + elif command -v gtimeout >/dev/null 2>&1; then + TIMEOUT_COMMAND=gtimeout + else + echo "Required command is unavailable: timeout or gtimeout" >&2 return 1 fi - timeout --foreground --kill-after=15s \ + local start_options=(-t "$SERVER_STARTUP_TIMEOUT") + if [[ -n "${SERVER_JAVA_OPTIONS:-}" ]]; then + start_options+=(-j "$SERVER_JAVA_OPTIONS") + fi + "$TIMEOUT_COMMAND" --foreground --kill-after=15s \ "${SERVER_START_COMMAND_TIMEOUT}s" \ - "$SERVER_DIR/bin/start-hugegraph.sh" -t "$SERVER_STARTUP_TIMEOUT" + "$SERVER_DIR/bin/start-hugegraph.sh" "${start_options[@]}" } "$TRAVIS_DIR/run-rocksdb-jni-smoke-test.sh" "$SERVER_DIR" @@ -66,7 +74,7 @@ start_server grep -q '^server-e2e-smoke-create-ok$' "$WORK_DIR/create.log" "$SERVER_DIR/bin/stop-hugegraph.sh" -m false -timeout --foreground --kill-after=15s "${STORE_DUMP_TIMEOUT}s" \ +"$TIMEOUT_COMMAND" --foreground --kill-after=15s "${STORE_DUMP_TIMEOUT}s" \ "$SERVER_DIR/bin/dump-store.sh" | tee "$WORK_DIR/store-dump.log" grep -q 'Dump table VERTEX (offset 0 limit 20):' "$WORK_DIR/store-dump.log" grep -Fq "riscv-smoke-v1-$RUN_ID" "$WORK_DIR/store-dump.log" diff --git a/hugegraph-server/hugegraph-dist/src/assembly/travis/test-riscv64-java-runtime.sh b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-riscv64-java-runtime.sh index aa3131543b..84c29b9316 100755 --- a/hugegraph-server/hugegraph-dist/src/assembly/travis/test-riscv64-java-runtime.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-riscv64-java-runtime.sh @@ -34,7 +34,7 @@ mkdir -p "$MOCK_BIN" "$MOCK_JAVA_HOME/bin" "$SERVER_DIR/bin" "$SERVER_DIR/lib" cat > "$MOCK_BIN/uname" <<'EOF' #!/bin/bash -echo riscv64 +echo "${MOCK_ARCH:-riscv64}" EOF cat > "$MOCK_JAVA_HOME/bin/java" <<'EOF' @@ -56,6 +56,10 @@ case "${1:-}" in echo "openjdk version \"$JAVA_VERSION\"" >&2 ;; -cp) + if [[ "${MOCK_ARCH:-riscv64}" == arm64 && -n "${LD_PRELOAD:-}" ]]; then + echo "ARM JNI smoke inherited RISC-V native preload" >&2 + exit 1 + fi echo "rocksdb-jni-smoke-ok" ;; *) @@ -68,7 +72,9 @@ EOF cat > "$SERVER_DIR/bin/util.sh" <<'EOF' #!/bin/bash configure_riscv64_libatomic() { - LD_PRELOAD=libatomic.so.1 + if [[ "$(uname -m)" == riscv64 ]]; then + LD_PRELOAD=libatomic.so.1 + fi } EOF @@ -152,4 +158,130 @@ grep -Fq \ 'Expected RISC-V Java vendor Eclipse Adoptium, got Unknown Vendor' \ <<< "$VENDOR_OUTPUT" +# Exercise the native wrapper through restart using controlled service commands. +# An isolated PATH makes the gtimeout fallback independent of the host platform. +RUNTIME_BIN="$WORK_DIR/runtime-bin" +RUNTIME_TRAVIS="$WORK_DIR/travis" +RUNTIME_LOG="$WORK_DIR/runtime.log" +START_LOG="$WORK_DIR/start.log" +STOP_LOG="$WORK_DIR/stop.log" +PHASE_LOG="$WORK_DIR/phase.log" +RUN_ID_FILE="$WORK_DIR/run-id" +mkdir -p "$RUNTIME_BIN" "$RUNTIME_TRAVIS" "$WORK_DIR/tmp" +for command in dirname date mktemp rm tee grep awk cut getconf; do + ln -s "$(command -v "$command")" "$RUNTIME_BIN/$command" +done +cp "$MOCK_BIN/uname" "$RUNTIME_BIN/uname" +cp "$NATIVE_SMOKE_SCRIPT" "$SMOKE_SCRIPT" "$RUNTIME_TRAVIS/" + +cat > "$RUNTIME_BIN/timeout" <<'EOF' +#!/bin/bash +set -euo pipefail +printf '%s' "${0##*/}" >> "$RUNTIME_LOG" +printf ' <%s>' "$@" >> "$RUNTIME_LOG" +printf '\n' >> "$RUNTIME_LOG" +if [[ "${MOCK_START_TIMEOUT:-false}" == true && "$4" == */start-hugegraph.sh ]]; then + exit 124 +fi +shift 3 +exec "$@" +EOF + +cat > "$SERVER_DIR/bin/init-store.sh" <<'EOF' +#!/bin/bash +exit 0 +EOF + +cat > "$SERVER_DIR/bin/start-hugegraph.sh" <<'EOF' +#!/bin/bash +set -euo pipefail +printf 'arch=%s' "$EXPECTED_ARCH" >> "$START_LOG" +printf ' <%s>' "$@" >> "$START_LOG" +printf '\n' >> "$START_LOG" +EOF + +cat > "$SERVER_DIR/bin/stop-hugegraph.sh" <<'EOF' +#!/bin/bash +printf '%s\n' "$*" >> "$STOP_LOG" +exit "${MOCK_STOP_STATUS:-0}" +EOF + +cat > "$SERVER_DIR/bin/dump-store.sh" <<'EOF' +#!/bin/bash +set -euo pipefail +RUN_ID=$(cat "$RUN_ID_FILE") +printf '%s\n' 'Dump table VERTEX (offset 0 limit 20):' \ + "riscv-smoke-v1-$RUN_ID" "riscv-smoke-v2-$RUN_ID" +EOF + +cat > "$RUNTIME_TRAVIS/run-server-e2e-smoke-test.sh" <<'EOF' +#!/bin/bash +set -euo pipefail +printf '%s\n' "$2" >> "$PHASE_LOG" +printf '%s\n' "$3" > "$RUN_ID_FILE" +printf 'server-e2e-smoke-%s-ok\n' "$2" +EOF + +ln -s "$(command -v cat)" "$RUNTIME_BIN/cat" +chmod +x "$RUNTIME_BIN/timeout" "$SERVER_DIR/bin/"*.sh \ + "$RUNTIME_TRAVIS/"*.sh + +run_restart_smoke() { + rm -f "$RUNTIME_LOG" "$START_LOG" "$STOP_LOG" "$PHASE_LOG" "$RUN_ID_FILE" + env -u LD_PRELOAD -u SERVER_JAVA_OPTIONS \ + PATH="$RUNTIME_BIN" JAVA_HOME="$MOCK_JAVA_HOME" TMPDIR="$WORK_DIR/tmp" \ + RUNTIME_LOG="$RUNTIME_LOG" START_LOG="$START_LOG" STOP_LOG="$STOP_LOG" \ + PHASE_LOG="$PHASE_LOG" RUN_ID_FILE="$RUN_ID_FILE" \ + "$@" "$RUNTIME_TRAVIS/run-native-runtime-smoke-test.sh" "$SERVER_DIR" +} + +if ! RESTART_OUTPUT=$(run_restart_smoke SERVER_STARTUP_TIMEOUT=7 STORE_DUMP_TIMEOUT=9 2>&1); then + echo "$RESTART_OUTPUT" >&2 + echo "RISC-V native smoke failed the controlled restart" >&2 + exit 1 +fi +grep -q '^native-runtime-smoke-ok$' <<< "$RESTART_OUTPUT" +[[ $(wc -l < "$START_LOG") -eq 2 ]] +[[ $(wc -l < "$STOP_LOG") -eq 2 ]] +grep -q '^arch=riscv64 <-t> <7>$' "$START_LOG" +grep -Fq 'timeout <--foreground> <--kill-after=15s> <37s>' "$RUNTIME_LOG" +grep -Fq 'timeout <--foreground> <--kill-after=15s> <9s>' "$RUNTIME_LOG" +[[ $(cat "$PHASE_LOG") == $'create\nverify' ]] + +mv "$RUNTIME_BIN/timeout" "$RUNTIME_BIN/gtimeout" +if ! ARM_OUTPUT=$(run_restart_smoke MOCK_ARCH=arm64 EXPECTED_ARCH=arm64 \ + 'SERVER_JAVA_OPTIONS=-Xms512m -Xmx2g' 2>&1); then + echo "$ARM_OUTPUT" >&2 + echo "ARM native smoke failed the controlled restart" >&2 + exit 1 +fi +grep -q '^native-runtime-smoke-ok$' <<< "$ARM_OUTPUT" +[[ $(grep -c '^arch=arm64 <-t> <300> <-j> <-Xms512m -Xmx2g>$' "$START_LOG") -eq 2 ]] +[[ $(grep -c '^gtimeout ' "$RUNTIME_LOG") -eq 3 ]] +[[ $(cat "$PHASE_LOG") == $'create\nverify' ]] + +set +e +TIMEOUT_OUTPUT=$(run_restart_smoke MOCK_ARCH=arm64 EXPECTED_ARCH=arm64 \ + MOCK_START_TIMEOUT=true 2>&1) +TIMEOUT_STATUS=$? +set -e +if [[ $TIMEOUT_STATUS -ne 124 || -e "$PHASE_LOG" || $(wc -l < "$STOP_LOG") -ne 1 ]]; then + echo "$TIMEOUT_OUTPUT" >&2 + echo "Native startup timeout did not preserve failure and clean up" >&2 + exit 1 +fi + +rm "$RUNTIME_BIN/gtimeout" +set +e +MISSING_TIMEOUT_OUTPUT=$(run_restart_smoke 2>&1) +MISSING_TIMEOUT_STATUS=$? +set -e +[[ $MISSING_TIMEOUT_STATUS -eq 1 ]] +grep -Fq 'Required command is unavailable: timeout or gtimeout' <<< "$MISSING_TIMEOUT_OUTPUT" +[[ $(wc -l < "$STOP_LOG") -eq 1 ]] +for TEMP_DIR in "$WORK_DIR/tmp"/hugegraph-native-runtime-smoke.*; do + [[ ! -e "$TEMP_DIR" ]] +done + echo "PASS: RISC-V Java runtime contract" +echo "PASS: native restart, ARM heap, timeout and cleanup contracts" From 35bb279c82e9abb2cb57f46d7d2246477ab0469f Mon Sep 17 00:00:00 2001 From: imbajin Date: Wed, 14 Oct 2026 02:00:29 +0800 Subject: [PATCH 2/5] fix: use Singapore time for weekly CI - select Asia/Singapore for the weekly workflow - retain the Monday 06:00 execution schedule - align CI documentation with the configured timezone --- .github/workflows/server-ci.yml | 2 +- docs/ci.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/server-ci.yml b/.github/workflows/server-ci.yml index 61867828a3..b3114047f6 100644 --- a/.github/workflows/server-ci.yml +++ b/.github/workflows/server-ci.yml @@ -6,7 +6,7 @@ on: pull_request: schedule: - cron: '0 6 * * 1' - timezone: Asia/Taipei + timezone: Asia/Singapore workflow_dispatch: inputs: test-set: diff --git a/docs/ci.md b/docs/ci.md index c299dfefc1..5cf4190610 100644 --- a/docs/ci.md +++ b/docs/ci.md @@ -23,7 +23,7 @@ the preflight license checks fail. | Docker, Helm and dependency audit | Corresponding inputs | Corresponding deployment smoke or checks | Full validation | macOS uses one `macos-15` ARM job for either smoke or full validation. Intel macOS is removed. -The weekly run uses `cron: '0 6 * * 1'` with `timezone: Asia/Taipei`: Monday at 06:00, +The weekly run uses `cron: '0 6 * * 1'` with `timezone: Asia/Singapore`: Monday at 06:00, on the latest default-branch commit. GitHub may delay scheduled runs during high load; see [schedule behavior](https://docs.github.com/en/actions/reference/workflows-and-actions/events-that-trigger-workflows#schedule). From cd0e2ff782080f11a807c400aecae863cc3b0770 Mon Sep 17 00:00:00 2001 From: imbajin Date: Wed, 14 Oct 2026 02:56:02 +0800 Subject: [PATCH 3/5] fix: select contract test owners - map contract scripts to their executing suites - verify caller inputs and inner job conditions - preserve consumer selection for runtime changes --- .github/scripts/ci-policy.py | 12 ++++++++ .github/scripts/test_ci_policy.py | 48 +++++++++++++++++++++++++++++-- docs/ci.md | 1 + 3 files changed, 58 insertions(+), 3 deletions(-) diff --git a/.github/scripts/ci-policy.py b/.github/scripts/ci-policy.py index 3bd3b7b9a4..dd2fd27331 100644 --- a/.github/scripts/ci-policy.py +++ b/.github/scripts/ci-policy.py @@ -88,6 +88,13 @@ "hugegraph-pd/hg-pd-grpc/pom.xml"} RISCV_PROTO_ROOTS = ("hugegraph-server/hugegraph-core/", "hugegraph-pd/hg-pd-grpc/") TRAVIS = "hugegraph-server/hugegraph-dist/src/assembly/travis/" +# Contract tests follow their executing suites, without runtime consumer fan-out. +CONTRACT_SUITES = { + TRAVIS + "test-stop-hugegraph-store.sh": {"struct"}, + TRAVIS + "test-check-jacoco-report.sh": {"struct"}, + TRAVIS + "test-codecov-upload-config.sh": {"struct"}, + TRAVIS + "test-riscv64-java-runtime.sh": {"server", "riscv64"}, +} def git(*args): @@ -127,11 +134,15 @@ def unsafe_documentation(ref, paths): def select(project, paths): selected = set() + contracts = set() for path in paths: if documentation(path): continue p = Path(path) if project == "server": + if path in CONTRACT_SUITES: + contracts.update(CONTRACT_SUITES[path]) + continue if path in MAINTENANCE_INPUTS: selected.update(MAINTENANCE_INPUTS[path]) continue @@ -212,6 +223,7 @@ def select(project, paths): selected.add("macos") if selected.intersection({"pd", "store", "hstore", "struct", "cluster"}): selected.add("distributed_smoke") + selected.update(contracts) return selected diff --git a/.github/scripts/test_ci_policy.py b/.github/scripts/test_ci_policy.py index a76d6d6dc9..deb30fc618 100644 --- a/.github/scripts/test_ci_policy.py +++ b/.github/scripts/test_ci_policy.py @@ -126,9 +126,7 @@ def test_noncore_platforms_only_follow_direct_inputs(self): "hugegraph-server/hugegraph-dist/src/assembly/travis/install-hbase.sh", "hugegraph-server/hugegraph-dist/src/assembly/travis/hbase-ci-site.xml"]: self.assertEqual({"hbase"}, policy.select("server", [path])) - for path in [".github/workflows/riscv64-ci.yml", - "hugegraph-server/hugegraph-dist/src/assembly/travis/test-riscv64-java-runtime.sh"]: - self.assertEqual({"riscv64"}, policy.select("server", [path])) + self.assertEqual({"riscv64"}, policy.select("server", [".github/workflows/riscv64-ci.yml"])) self.assertTrue(self.plan_for([".github/workflows/server-compatibility-ci.yml"])["riscv64"]) self.assertTrue(self.plan_for([".github/workflows/server-tests.yml"])["hbase"]) self.assertTrue(self.plan_for(["hugegraph-server/hugegraph-rocksdb/pom.xml"])["riscv64"]) @@ -345,6 +343,50 @@ def test_real_coordinator_blocks_selection_runtime_failure_and_cancellation(self self.assertEqual({"helm"}, self.coordinator_jobs(full, java=state, result="failure")) self.assertEqual(set(), self.coordinator_jobs(full, cancelled=True)) + def test_contract_scripts_reach_their_actual_inner_jobs_and_steps(self): + workflows = Path(__file__).resolve().parents[1] / "workflows" + cases = [("test-stop-hugegraph-store.sh", {"struct"}), + ("test-check-jacoco-report.sh", {"struct"}), + ("test-codecov-upload-config.sh", {"struct"}), + ("test-riscv64-java-runtime.sh", {"server", "riscv64"})] + for name, suites in cases: + script = policy.TRAVIS + name + with self.subTest(script=script): + plan = self.plan_for([script]) + self.assertEqual(suites, set(plan["selected"])) + callers = self.coordinator_jobs(plan) + if "struct" in suites: + self.assertEqual({"pd_store"}, callers) + root = (workflows / "server-ci.yml").read_text() + caller = re.search(r"^ pd_store:\n(.*?)(?=^ [\w-]+:|\Z)", root, re.M | re.S).group(1) + inputs = {} + for key in ["struct", "store"]: + binding = re.search(r"^ run-" + key + r": \$\{\{ (.+) \}\}$", caller, re.M).group(1) + binding = binding.replace("needs.preflight.outputs." + key, repr(str(plan[key]).lower())) + inputs[key] = eval(binding, {"__builtins__": {}}) + workflow = (workflows / "pd-store-ci.yml").read_text() + struct = workflow.split("\n struct:\n", 1)[1].split("\n pd:\n", 1)[0] + condition = re.search(r"^ if: (.+)$", struct, re.M).group(1) + condition = condition.replace("inputs.run-struct", str(inputs["struct"])) + condition = condition.replace("inputs.run-store", str(inputs["store"])) + self.assertTrue(eval(condition.replace("||", "or"), {"__builtins__": {}})) + step = next(step for step in struct.split("\n - ") if script in step) + self.assertNotRegex(step, r"(?m)^ if:") + else: + self.assertIn("server-rocksdb", callers) + workflow = (workflows / "server-ci.yml").read_text() + # Read the backend from the actual caller rather than assuming a preset. + caller = re.search(r"^ server-rocksdb:\n(.*?)(?=^ [\w-]+:|\Z)", + workflow, re.M | re.S).group(1) + backend = re.search(r"^ backend: (\w+)$", caller, re.M).group(1) + inner = (workflows / "server-tests.yml").read_text() + step = next(step for step in inner.split("\n - ") if Path(script).name in step) + condition = re.search(r"^ if: (.+)$", step, re.M).group(1) + self.assertTrue(eval(condition.replace("inputs.backend", repr(backend)), + {"__builtins__": {}})) + mixed = self.plan_for([policy.TRAVIS + "test-stop-hugegraph-store.sh", "hugegraph-pd/src/A.java"]) + self.assertTrue({"struct", "pd", "store", "hstore", "cluster"}.issubset(mixed["selected"])) + def test_coordinator_reads_and_validates_real_java_input(self): workflow = (Path(__file__).resolve().parents[1] / "workflows/server-ci.yml").read_text() code = workflow.split("python3 - <<'PYTHON'\n", 1)[1].split("\n PYTHON", 1)[0] diff --git a/docs/ci.md b/docs/ci.md index 5cf4190610..17a8f2331e 100644 --- a/docs/ci.md +++ b/docs/ci.md @@ -39,6 +39,7 @@ startup and RocksDB inputs select ARM full validation and their deployment consu Unclassified paths or an unavailable diff/API conservatively select mainline coverage. HBase and RISC-V remain limited to explicit related inputs, weekly coverage or manual selection. The classifier uses paths, not PR titles or semantic interpretation of changed text. +Contract scripts explicitly select their executing suites without adding runtime consumers. ### Documentation From 3c6792a81996f6954780bdb4b3bc0af4ab88ba46 Mon Sep 17 00:00:00 2001 From: imbajin Date: Wed, 14 Oct 2026 03:13:08 +0800 Subject: [PATCH 4/5] fix: preserve license check and normalize Java - preserve the license context on the existing preflight - normalize full runtime versions before major checks - cover accepted versions and mismatched majors - document the required Infra migration step --- .github/scripts/test_ci_policy.py | 10 +++++--- .github/workflows/server-ci.yml | 1 + docs/ci.md | 7 +++++- .../travis/run-rocksdb-jni-smoke-test.sh | 2 ++ .../travis/test-riscv64-java-runtime.sh | 23 +++++++++++++++++++ 5 files changed, 39 insertions(+), 4 deletions(-) diff --git a/.github/scripts/test_ci_policy.py b/.github/scripts/test_ci_policy.py index deb30fc618..a5e9e5301c 100644 --- a/.github/scripts/test_ci_policy.py +++ b/.github/scripts/test_ci_policy.py @@ -321,6 +321,9 @@ def coordinator_jobs(self, plan, selection="success", java="success", result="su return {job for job, run in jobs.items() if run} def test_real_coordinator_keeps_selected_jobs_after_license_or_selftest_failure(self): + workflow = (Path(__file__).resolve().parents[1] / "workflows/server-ci.yml").read_text() + preflight = re.search(r"^ preflight:\n(.*?)(?=^ [\w-]+:|\Z)", workflow, re.M | re.S).group(1) + self.assertEqual("check-license", re.search(r"^ name: (.+)$", preflight, re.M).group(1)) full = self.plan_for([], "workflow_dispatch", {"inputs": {"test-set": "full"}}) expected = {"build-server", "server-rocksdb", "commons", "pd_store", "cluster", "docker", "helm", "dependency_license", "compatibility", "codeql"} @@ -395,13 +398,14 @@ def test_coordinator_reads_and_validates_real_java_input(self): java = root / ".github/workflows/.java-version" java.parent.mkdir(parents=True) output = root / "outputs.txt" - for version in ["17", "21.0.1", "", "bad", '17\n21']: + for version in ["17", "17.0.20", "21.0.1", "", "bad", '17\n21']: java.write_text(version) result = subprocess.run(["python3", "-c", textwrap.dedent(code)], cwd=root, env={**os.environ, "GITHUB_OUTPUT": str(output)}, capture_output=True, text=True) with self.subTest(version=version): - self.assertEqual(version in {"17", "21.0.1"}, result.returncode == 0, result.stderr) - self.assertEqual(['versions=["17"]', 'versions=["21.0.1"]'], output.read_text().splitlines()) + self.assertEqual(version in {"17", "17.0.20", "21.0.1"}, result.returncode == 0, result.stderr) + self.assertEqual(['versions=["17"]', 'versions=["17.0.20"]', 'versions=["21.0.1"]'], + output.read_text().splitlines()) def test_real_git_document_modes_renames_and_cli_outputs(self): with tempfile.TemporaryDirectory() as directory: diff --git a/.github/workflows/server-ci.yml b/.github/workflows/server-ci.yml index b3114047f6..f21f1f73a1 100644 --- a/.github/workflows/server-ci.yml +++ b/.github/workflows/server-ci.yml @@ -26,6 +26,7 @@ permissions: jobs: preflight: + name: check-license runs-on: ubuntu-24.04 timeout-minutes: 15 env: diff --git a/docs/ci.md b/docs/ci.md index 17a8f2331e..8e8759a412 100644 --- a/docs/ci.md +++ b/docs/ci.md @@ -5,12 +5,17 @@ PRs, supported branch pushes, weekly coverage and manual validation. Its preflig selects affected tests, reads the project Java version, checks CI policy and verifies license headers and RAT. Selected suites run through reusable workflows in the same run. -All CI results are advisory. [`.asf.yaml`](../.asf.yaml) requests no required CI contexts +CI results become advisory after [`.asf.yaml`](../.asf.yaml) is applied. It requests no required CI contexts and retains one approving review. This protection change takes effect after the configuration is merged and synchronized by Gitbox; verify live protection then. Failed tests keep their real failed results. Valid selection and Java outputs allow selected tests to run even when the preflight license checks fail. +For migration, preflight keeps the existing `check-license` check name on the same runner. +Before merging this workflow change, ASF Infra must remove only the retired `Server memory tests` +context from live protection, keeping `check-license` and all review settings unchanged. +See [ASF's required-check migration guidance](https://github.com/apache/infrastructure-asfyaml#branch-protection). + ## What runs | Validation | Ordinary PR | Push to master | Weekly full run | diff --git a/hugegraph-server/hugegraph-dist/src/assembly/travis/run-rocksdb-jni-smoke-test.sh b/hugegraph-server/hugegraph-dist/src/assembly/travis/run-rocksdb-jni-smoke-test.sh index 81d88796d0..0e1575f6a4 100755 --- a/hugegraph-server/hugegraph-dist/src/assembly/travis/run-rocksdb-jni-smoke-test.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/travis/run-rocksdb-jni-smoke-test.sh @@ -26,6 +26,8 @@ TRAVIS_DIR=$(cd "$(dirname "$0")" && pwd) SERVER_DIR=$(cd "$1" && pwd) EXPECTED_ARCH=${EXPECTED_ARCH:-} EXPECTED_JAVA_MAJOR=${EXPECTED_JAVA_MAJOR:-17} +# Callers may supply a full setup-java version; this check enforces its major. +EXPECTED_JAVA_MAJOR=${EXPECTED_JAVA_MAJOR%%.*} EXPECTED_RISCV64_JAVA_VERSION=${EXPECTED_RISCV64_JAVA_VERSION:-} EXPECTED_RISCV64_JAVA_VENDOR=${EXPECTED_RISCV64_JAVA_VENDOR:-} ACTUAL_ARCH=$(uname -m) diff --git a/hugegraph-server/hugegraph-dist/src/assembly/travis/test-riscv64-java-runtime.sh b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-riscv64-java-runtime.sh index 84c29b9316..cc33c98d17 100755 --- a/hugegraph-server/hugegraph-dist/src/assembly/travis/test-riscv64-java-runtime.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-riscv64-java-runtime.sh @@ -118,6 +118,29 @@ if [[ $NATIVE_STATUS -ne 42 ]]; then fi grep -q '^rocksdb-jni-smoke-ok$' <<< "$NATIVE_OUTPUT" +# macOS passes the setup-java version through the native wrapper unchanged. +for EXPECTED_VERSION in 17 17.0.20 21.0.1; do + set +e + VERSION_OUTPUT=$(run_native_smoke MOCK_ARCH=arm64 EXPECTED_ARCH=arm64 \ + "EXPECTED_JAVA_MAJOR=$EXPECTED_VERSION" \ + "MOCK_JAVA_VERSION=$EXPECTED_VERSION" 2>&1) + VERSION_STATUS=$? + set -e + if [[ $VERSION_STATUS -ne 42 ]]; then + echo "$VERSION_OUTPUT" >&2 + echo "ARM native smoke rejected expected version $EXPECTED_VERSION" >&2 + exit 1 + fi + grep -q '^rocksdb-jni-smoke-ok$' <<< "$VERSION_OUTPUT" +done +if MAJOR_OUTPUT=$(run_smoke MOCK_ARCH=arm64 EXPECTED_ARCH=arm64 \ + EXPECTED_JAVA_MAJOR=21.0.1 MOCK_JAVA_VERSION=17.0.20 2>&1); then + echo "$MAJOR_OUTPUT" >&2 + echo "ARM smoke accepted a mismatched Java major" >&2 + exit 1 +fi +grep -Fq 'Expected Java 21, got 17.0.20' <<< "$MAJOR_OUTPUT" + for JAVA_MAJOR_MISMATCH in 11.0.31 21.0.8; do if MAJOR_OUTPUT=$(run_smoke \ "MOCK_JAVA_VERSION=$JAVA_MAJOR_MISMATCH" 2>&1); then From 50bcccc08be7e79a2880dad3016039ed25c4ec3b Mon Sep 17 00:00:00 2001 From: imbajin Date: Wed, 14 Oct 2026 03:21:20 +0800 Subject: [PATCH 5/5] fix: bridge required checks during migration - verify the legacy context from real Memory results - allocate a lightweight runner only while required - stop the bridge after Gitbox synchronizes protection - cover failures, skips and bridge retirement --- .github/scripts/test_ci_policy.py | 24 ++++++++++++++++++++++ .github/workflows/server-ci.yml | 34 +++++++++++++++++++++++++++++++ docs/ci.md | 9 ++++---- 3 files changed, 63 insertions(+), 4 deletions(-) diff --git a/.github/scripts/test_ci_policy.py b/.github/scripts/test_ci_policy.py index a5e9e5301c..a60486ebec 100644 --- a/.github/scripts/test_ci_policy.py +++ b/.github/scripts/test_ci_policy.py @@ -346,6 +346,30 @@ def test_real_coordinator_blocks_selection_runtime_failure_and_cancellation(self self.assertEqual({"helm"}, self.coordinator_jobs(full, java=state, result="failure")) self.assertEqual(set(), self.coordinator_jobs(full, cancelled=True)) + def test_legacy_memory_bridge_requires_real_results_and_retires_without_a_runner(self): + docs = self.plan_for(["README.md"]) + self.assertEqual(set(), self.coordinator_jobs(docs)) + docs["legacy_memory_required"] = True + self.assertEqual({"legacy-server-memory"}, self.coordinator_jobs(docs)) + self.assertEqual(set(), self.coordinator_jobs(docs, cancelled=True)) + workflow = (Path(__file__).resolve().parents[1] / "workflows/server-ci.yml").read_text() + bridge = re.search(r"^ legacy-server-memory:\n(.*?)(?=^ [\w-]+:|\Z)", workflow, re.M | re.S).group(1) + self.assertEqual("Server memory tests", re.search(r"^ name: (.+)$", bridge, re.M).group(1)) + script = textwrap.dedent(bridge.split(" run: |\n", 1)[1]) + cases = [("success", "true", "success", "success", True), + ("success", "false", "success", "skipped", True), + ("failure", "false", "success", "skipped", False), + ("success", "", "success", "skipped", False), + ("success", "true", "failure", "skipped", False)] + cases += [("success", "true", "success", result, False) + for result in ["failure", "cancelled", "skipped", ""]] + for selection, selected, java, memory, passes in cases: + with self.subTest(selection=selection, selected=selected, java=java, memory=memory): + result = subprocess.run(["bash", "-eo", "pipefail", "-c", script], capture_output=True, + env={**os.environ, "SELECTION_RESULT": selection, "SERVER_SELECTED": selected, + "JAVA_RESULT": java, "MEMORY_RESULT": memory}) + self.assertEqual(passes, result.returncode == 0, result.stderr) + def test_contract_scripts_reach_their_actual_inner_jobs_and_steps(self): workflows = Path(__file__).resolve().parents[1] / "workflows" cases = [("test-stop-hugegraph-store.sh", {"struct"}), diff --git a/.github/workflows/server-ci.yml b/.github/workflows/server-ci.yml index f21f1f73a1..d27a83774e 100644 --- a/.github/workflows/server-ci.yml +++ b/.github/workflows/server-ci.yml @@ -35,6 +35,7 @@ jobs: selection_result: ${{ steps.select.outcome }} java_result: ${{ steps.java.outcome }} java_versions: ${{ steps.java.outputs.versions }} + legacy_memory_required: ${{ steps.migration.outputs.required }} server: ${{ steps.select.outputs.server }} commons: ${{ steps.select.outputs.commons }} struct: ${{ steps.select.outputs.struct }} @@ -95,6 +96,18 @@ jobs: retention-days: 7 overwrite: true + - name: Check legacy required context + id: migration + if: ${{ !cancelled() && github.event_name == 'pull_request' && steps.checkout.outcome == 'success' }} + env: + GH_TOKEN: ${{ github.token }} + BASE_REF: ${{ github.event.pull_request.base.ref }} + run: | + BASE_REF=$(python3 -c 'import os, urllib.parse; print(urllib.parse.quote(os.environ["BASE_REF"], safe=""))') + REQUIRED=$(gh api "repos/$GITHUB_REPOSITORY/branches/$BASE_REF" \ + --jq '.protection.required_status_checks.contexts // [] | index("Server memory tests") != null') + echo "required=$REQUIRED" >> "$GITHUB_OUTPUT" + - name: Test CI policy and deployment checks if: ${{ !cancelled() && steps.checkout.outcome == 'success' }} run: python3 -m unittest discover -s .github/scripts -p 'test_*.py' @@ -132,6 +145,27 @@ jobs: java-versions: ${{ needs.preflight.outputs.java_versions }} secrets: inherit + legacy-server-memory: + name: Server memory tests + needs: [preflight, build-server] + if: ${{ !cancelled() && needs.preflight.outputs.legacy_memory_required == 'true' }} + runs-on: ubuntu-slim + timeout-minutes: 5 + env: + SELECTION_RESULT: ${{ needs.preflight.outputs.selection_result }} + SERVER_SELECTED: ${{ needs.preflight.outputs.server }} + JAVA_RESULT: ${{ needs.preflight.outputs.java_result }} + MEMORY_RESULT: ${{ needs.build-server.result }} + steps: + - name: Verify the selected Memory result during protection migration + run: | + test "$SELECTION_RESULT" = success + case "$SERVER_SELECTED" in + true) test "$JAVA_RESULT" = success && test "$MEMORY_RESULT" = success ;; + false) test "$MEMORY_RESULT" = skipped ;; + *) exit 1 ;; + esac + commons: needs: preflight if: ${{ !cancelled() && needs.preflight.outputs.selection_result == 'success' && needs.preflight.outputs.java_result == 'success' && needs.preflight.outputs.commons == 'true' }} diff --git a/docs/ci.md b/docs/ci.md index 8e8759a412..a6dd9fc1e5 100644 --- a/docs/ci.md +++ b/docs/ci.md @@ -12,9 +12,10 @@ real failed results. Valid selection and Java outputs allow selected tests to ru the preflight license checks fail. For migration, preflight keeps the existing `check-license` check name on the same runner. -Before merging this workflow change, ASF Infra must remove only the retired `Server memory tests` -context from live protection, keeping `check-license` and all review settings unchanged. -See [ASF's required-check migration guidance](https://github.com/apache/infrastructure-asfyaml#branch-protection). +A temporary `Server memory tests` check verifies the selected Memory result while live protection +still requires that name. It uses one lightweight `ubuntu-slim` runner and repeats no tests. +After Gitbox applies the new configuration, this bridge skips without allocating a runner; +no Infra intervention or change to review protection is needed. ## What runs @@ -84,7 +85,7 @@ Plans and selection summaries describe this run; previous results are never reus A new PR head cancels the previous PR run, including its called workflows. Manual and weekly runs remain independent. Each job reports its own result; there are no separate post-test -aggregation runners or end-of-run metadata checks. +aggregation runners after the protection migration or end-of-run metadata checks. Startup contracts run inside RocksDB validation, and Store stop contracts run inside Struct. Docker's static checks, deployment contracts and current-image validation share one runner.