diff --git a/.asf.yaml b/.asf.yaml index 234918b6e3..a2e0b4aa67 100644 --- a/.asf.yaml +++ b/.asf.yaml @@ -33,13 +33,10 @@ github: protected_branches: master: required_status_checks: - # Keep workflows providing these checks unfiltered; path skips can leave required checks pending. # strict means "Require PR to be up-to-date before merging". (enable when branch unstable) strict: false - # contexts are the names of checks that must pass (now only enable the basic check) - contexts: - - check-license - - Server memory tests + # CI remains visible but advisory while shared runner queues are constrained. + contexts: [] required_pull_request_reviews: dismiss_stale_reviews: true require_code_owner_reviews: false diff --git a/.github/scripts/check-rerun.py b/.github/scripts/check-rerun.py deleted file mode 100644 index fbc2c6a6a3..0000000000 --- a/.github/scripts/check-rerun.py +++ /dev/null @@ -1,119 +0,0 @@ -#!/usr/bin/env python3 -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Check the failed run again immediately before an automatic retry.""" - -import argparse -import json -import os -import re -import subprocess -from urllib.parse import urlencode - - -def api(path): - result = subprocess.run( - ["gh", "api", "--method", "GET", path], check=True, - capture_output=True, timeout=30, - ) - return json.loads(result.stdout) - - -def decide(repository, run_id, expected_attempt, max_reruns, fetch=api): - run = fetch(f"repos/{repository}/actions/runs/{run_id}") - if run.get("path") != ".github/workflows/server-ci.yml": - return "skip", "only Server CI required jobs automatically retry", None - if (run.get("status") != "completed" or run.get("conclusion") != "failure" - or run.get("run_attempt") != expected_attempt): - return "skip", "source run changed or is no longer a completed failure", None - if expected_attempt > max_reruns: - return "skip", "retry limit reached", None - sha = run.get("head_sha") - if not sha: - return "skip", "missing head SHA", None - if run.get("event") == "pull_request": - return "skip", "PR automatic retry paused: CI artifacts cannot independently prove the tested code", None - if run.get("event") == "push": - if (run.get("repository", {}).get("full_name") != repository - or run.get("head_repository", {}).get("full_name") != repository): - return "skip", "push source repository differs", None - branch = run.get("head_branch") - if not branch: - return "skip", "missing push branch", None - # Read from the workflow repository, never a similarly named fork branch. - commits = fetch(f"repos/{repository}/commits?" + urlencode({"sha": branch, "per_page": 1})) - if not commits or commits[0].get("sha") != sha: - return "skip", "push branch head moved", None - # A native job rerun includes its dependent jobs. Prefer a failed producer - # over its failed reporter; never select the advisory branch of Server CI. - targets = [] - page = 1 - while True: - jobs = fetch(f"repos/{repository}/actions/runs/{run_id}/attempts/{expected_attempt}/jobs" - f"?per_page=100&page={page}")["jobs"] - for job in jobs: - if job.get("status") != "completed" or job.get("conclusion") not in {"failure", "timed_out"}: - continue - name = job.get("name", "") - if name == "check-license": - priority = 0 - elif name == "memory-plan": - priority = 1 - elif name == "memory-java-runtime / read-version": - priority = 2 - elif re.fullmatch(r"build-server / server_memory \(Java [1-9][0-9]*(?:\.[0-9]+)*\)", name): - priority = 3 - elif name == "Server memory tests": - priority = 4 - else: - continue - # Only failed reporters retry themselves; a timed-out producer - # must be retried before its dependent failure can recover. - if priority == 4 and job.get("conclusion") != "failure": - continue - if job["run_id"] != run_id or job["head_sha"] != sha: - return "skip", "job does not match source run", None - job_id = job["id"] - if not isinstance(job_id, int) or job_id <= 0: - return "skip", "invalid job ID", None - targets.append((priority, job_id)) - if len(jobs) < 100: - break - page += 1 - if targets: - return "rerun", "failed required job and its dependencies", min(targets)[1] - return "skip", "no failed required job in this attempt", None - return "skip", "unsupported source event", None - - -def main(): - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--repository", required=True) - parser.add_argument("--run-id", required=True, type=int) - parser.add_argument("--run-attempt", required=True, type=int) - parser.add_argument("--max-reruns", required=True, type=int) - args = parser.parse_args() - try: - action, reason, job_id = decide(args.repository, args.run_id, args.run_attempt, args.max_reruns) - except (subprocess.SubprocessError, OSError, ValueError, KeyError, TypeError, AttributeError): - action, reason, job_id = "skip", "unable to verify current run and source metadata", None - print(f"{action}: {reason}") - with open(os.environ["GITHUB_OUTPUT"], "a", encoding="utf-8") as output: - output.write(f"action={action}\nreason={reason}\njob_id={job_id or ''}\n") - - -if __name__ == "__main__": - main() diff --git a/.github/scripts/ci-policy.py b/.github/scripts/ci-policy.py index 26190be79c..dd2fd27331 100644 --- a/.github/scripts/ci-policy.py +++ b/.github/scripts/ci-policy.py @@ -14,7 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Conservative CI selection and gating of this run's selected tests.""" +"""Select affected CI checks from the current event and cumulative PR diff.""" import argparse import json @@ -22,8 +22,10 @@ from pathlib import Path import subprocess +MAINLINE = {"server", "commons", "struct", "pd", "store", "hstore", "cluster", "docker", "helm", + "dependency_license", "macos", "security", "distributed_smoke"} MODULES = { - "server": ["server", "commons", "struct", "pd", "store", "hstore", "cluster", "docker", "helm", "dependency_license"], + "server": sorted(MAINLINE | {"macos_full", "hbase", "riscv64"}), "toolchain": ["client", "loader", "tools", "spark", "hubble", "go"], } PREFIXES = { @@ -35,33 +37,63 @@ "hugegraph-spark-connector/": "spark", "hugegraph-hubble/": "hubble"}, } WORKFLOWS = { - "server": {"server-ci.yml": MODULES["server"], "commons-ci.yml": ["commons"], - "pd-store-ci.yml": ["pd", "store", "hstore"], "cluster-test-ci.yml": ["cluster"], - "docker-build-ci.yml": ["docker"], "helm-chart-ci.yml": ["helm"], - "codeql-analysis.yml": [], "server-memory-ci.yml": ["server"], - "server-tests.yml": ["server"], "server-compatibility-ci.yml": ["server", "docker"], - "riscv64-ci.yml": ["server"], "check-dependencies.yml": ["dependency_license"]}, - "toolchain": {"client-ci.yml": ["client"], "client-go-ci.yml": ["go"], - "loader-ci.yml": ["loader"], "tools-ci.yml": ["tools"], - "spark-connector-ci.yml": ["spark"], "hubble-ci.yml": ["hubble"], - "codeql-analysis.yml": []}, + "server": {"server-ci.yml": MAINLINE, "commons-ci.yml": {"commons"}, + "pd-store-ci.yml": {"struct", "pd", "store", "hstore"}, "cluster-test-ci.yml": {"cluster"}, + "docker-build-ci.yml": {"docker"}, "helm-chart-ci.yml": {"helm"}, + "codeql-analysis.yml": {"security"}, "server-memory-ci.yml": {"server"}, + "server-tests.yml": {"server", "hbase"}, + "server-compatibility-ci.yml": {"macos", "macos_full", "hbase", "riscv64"}, + "riscv64-ci.yml": {"riscv64"}, "check-dependencies.yml": {"dependency_license"}}, + "toolchain": {"client-ci.yml": {"client"}, "client-go-ci.yml": {"go"}, + "loader-ci.yml": {"loader"}, "tools-ci.yml": {"tools"}, + "spark-connector-ci.yml": {"spark"}, "hubble-ci.yml": {"hubble"}, + "codeql-analysis.yml": set()}, +} +# Shared runtime consumers; ordinary Server edits do not fan out to PD/Store. +DEPENDENTS = { + "server": {"commons": {"server", "pd", "store", "hstore", "cluster"}, + "struct": {"server", "pd", "store", "hstore", "cluster"}, + "pd": {"store", "hstore", "cluster"}, "store": {"hstore", "cluster"}, "hstore": {"cluster"}}, + "toolchain": {"client": {"loader", "tools", "spark", "hubble"}, "loader": {"hubble"}}, } -# Exact maintenance inputs have no product consumers; their checks run in the planner. MAINTENANCE_INPUTS = { - ".github/PULL_REQUEST_TEMPLATE.md": [], - ".github/dependabot.yml": [], - ".github/scripts/check-rerun.py": [], - ".github/scripts/test_check_rerun.py": [], - ".github/scripts/check-docker-images.sh": ["docker"], - ".github/scripts/docker-deployment.py": ["docker"], - ".github/scripts/test_docker_deployment.py": ["docker"], + ".asf.yaml": set(), ".editorconfig": set(), ".licenserc.yaml": set(), + ".github/PULL_REQUEST_TEMPLATE.md": set(), ".github/dependabot.yml": set(), + ".github/workflows/.java-version": MAINLINE | {"macos_full", "hbase", "riscv64"}, + ".github/scripts/check-docker-images.sh": {"docker"}, + ".github/scripts/docker-deployment.py": {"docker"}, + ".github/scripts/test_docker_deployment.py": {"docker"}, } -DEPENDENTS = { - "server": {"commons": ["server", "pd", "store", "hstore", "cluster"], - "struct": ["server", "pd", "store", "hstore", "cluster"], - "server": ["pd", "store", "hstore", "cluster"], "pd": ["store", "hstore", "cluster"], - "store": ["hstore", "cluster"], "hstore": ["cluster"]}, - "toolchain": {"client": ["loader", "tools", "spark", "hubble"], "loader": ["hubble"]}, +PRESETS = { + "mainline": MAINLINE, + "distributed": {"struct", "pd", "store", "hstore", "cluster", "distributed_smoke"}, + "macos-arm": {"macos", "macos_full"}, "hbase": {"hbase"}, + "riscv64": {"riscv64"}, "codeql": {"security"}, "full": set(MODULES["server"]), +} +SERVER_CORE = "hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/" +SERVER_API = "hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/" +DISTRIBUTED_SERVER_INPUTS = ( + "hugegraph-server/hugegraph-hstore/", SERVER_CORE + "backend/", SERVER_CORE + "meta/", + SERVER_CORE + "space/", SERVER_CORE + "task/", SERVER_CORE + "masterelection/", SERVER_CORE + "HugeGraph", + SERVER_CORE + "StandardHugeGraph", SERVER_API + "core/", SERVER_API + "rpc/", + SERVER_API + "api/raft/", SERVER_API + "api/space/", + "hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/store/", + "hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/meta/", +) +# Test/resources/templates can contain executable examples and runtime fixtures. +RUNTIME_PARTS = {"src", "resources", "fixtures", "templates", "static", "bin", "conf", "configs"} +# These build files declare native JNI dependencies/classifiers or the RISC-V protobuf profile. +PLATFORM_POMS = {"pom.xml", "hugegraph-server/pom.xml", "hugegraph-server/hugegraph-core/pom.xml", + "hugegraph-server/hugegraph-rocksdb/pom.xml", "hugegraph-server/hugegraph-dist/pom.xml", + "hugegraph-pd/hg-pd-grpc/pom.xml"} +RISCV_PROTO_ROOTS = ("hugegraph-server/hugegraph-core/", "hugegraph-pd/hg-pd-grpc/") +TRAVIS = "hugegraph-server/hugegraph-dist/src/assembly/travis/" +# Contract tests follow their executing suites, without runtime consumer fan-out. +CONTRACT_SUITES = { + TRAVIS + "test-stop-hugegraph-store.sh": {"struct"}, + TRAVIS + "test-check-jacoco-report.sh": {"struct"}, + TRAVIS + "test-codecov-upload-config.sh": {"struct"}, + TRAVIS + "test-riscv64-java-runtime.sh": {"server", "riscv64"}, } @@ -75,18 +107,17 @@ def api(path): def documentation(path, mode="100644"): - """Only prose and static documentation assets; never source or config.""" + """Only plain prose/static assets outside runtime and fixture directories.""" if mode != "100644": return False p = Path(path) - prose_names = {"README.md", "README_CN.md", "README_ZH.md", "AGENTS.md"} - module_roots = {prefix.rstrip("/") for mapping in PREFIXES.values() for prefix in mapping} - module_roots.update("hugegraph-server/" + module for module in [ - "hugegraph-api", "hugegraph-core", "hugegraph-dist", "hugegraph-example", - "hugegraph-hbase", "hugegraph-hstore", "hugegraph-rocksdb", "hugegraph-test"]) - if p.name in prose_names and (str(p.parent) == "." or str(p.parent) in module_roots): + if RUNTIME_PARTS.intersection(p.parts[:-1]): + return False + if p.suffix.lower() in {".md", ".rst"}: + return True + if len(p.parts) == 1 and (p.suffix.lower() == ".txt" or p.name in {"LICENSE", "NOTICE"}): return True - return path.startswith("docs/") and p.suffix.lower() in {".md", ".rst", ".txt", ".png", ".jpg", ".jpeg", ".svg"} + return "docs" in p.parts[:-1] and p.suffix.lower() in {".md", ".rst", ".txt", ".png", ".jpg", ".jpeg", ".svg"} def unsafe_documentation(ref, paths): @@ -103,67 +134,128 @@ def unsafe_documentation(ref, paths): def select(project, paths): selected = set() + contracts = set() for path in paths: if documentation(path): continue - if project == "server" and path in MAINTENANCE_INPUTS: - selected.update(MAINTENANCE_INPUTS[path]) - continue - if project == "server" and (Path(path).name == "pom.xml" or path.startswith("install-dist/")): - selected.add("dependency_license") - if path.startswith(".github/workflows/") and Path(path).name in WORKFLOWS[project]: - selected.update(WORKFLOWS[project][Path(path).name]) - continue - if path.endswith(".proto"): - return set(MODULES[project]) - if project == "server" and path == "hugegraph-commons/hugegraph-common/src/main/resources/version.properties": - selected.update(["commons", "docker"]) - continue - if project == "toolchain" and path.startswith("hugegraph-client/assembly/travis/"): - selected.update(["client", "go"]) - continue + p = Path(path) if project == "server": - if path in {"hugegraph-server/pom.xml", "hugegraph-server/hugegraph-api/pom.xml"}: - selected.add("docker") - if path.startswith(("hugegraph-pd/hg-pd-dist/", "hugegraph-store/hg-store-dist/")): + if path in CONTRACT_SUITES: + contracts.update(CONTRACT_SUITES[path]) + continue + if path in MAINTENANCE_INPUTS: + selected.update(MAINTENANCE_INPUTS[path]) + continue + if p.name == "pom.xml": + selected.update({"dependency_license", "security"}) + if path in PLATFORM_POMS: + selected.update({"macos", "macos_full", "riscv64"}) + if path.startswith("install-dist/"): + selected.add("dependency_license") + continue + if ({"auth", "security", "permission", "permissions"}.intersection(p.parts) + or p.stem.startswith(("Authentication", "Authorization")) + or p.stem.endswith(("Authentication", "Authenticator", "Permission", "WithAuth")) + or p.name in {"AuthOptions.java", "OkHttpBasicAuthInterceptor.java", "java-security.properties", + "test-java-security-properties.sh", "enable-auth.sh"}): + selected.add("security") + if path.startswith(".github/workflows/") and p.name in WORKFLOWS[project]: + selected.update(WORKFLOWS[project][p.name]) + continue + if path == TRAVIS + "install-backend.sh": + selected.update({"hbase", "hstore"}) + if path.startswith("docker/hbase/"): + selected.update({"docker", "hbase"}) + continue + if path.startswith("hugegraph-server/hugegraph-hbase/") or "hbase" in p.name.lower(): + selected.add("hbase") + continue + if "riscv" in path.lower(): + selected.add("riscv64") + continue + if path.startswith(DISTRIBUTED_SERVER_INPUTS) or path.endswith(".proto"): + selected.update({"hstore", "cluster", "distributed_smoke"}) + if path.endswith(".proto") and path.startswith(RISCV_PROTO_ROOTS): + selected.add("riscv64") + if path.startswith("hugegraph-server/hugegraph-dist/docker/"): selected.add("docker") - if path.startswith("hugegraph-server/hugegraph-hstore/"): - selected.update(["server", "hstore"]) continue if path.startswith("hugegraph-server/hugegraph-dist/"): - selected.update(["server", "pd", "store", "hstore", "docker", "cluster"]) + selected.update({"server", "docker", "macos", "macos_full"}) + if (path.startswith("hugegraph-server/hugegraph-dist/src/assembly/static/bin/") + or "native-runtime" in p.name or "rocksdb-jni" in p.name + or p.name == "check-rocksdb-only-dist.sh"): + selected.add("riscv64") + if any(word in path for word in ("pd", "store", "distributed")): + selected.update({"hstore", "cluster", "distributed_smoke"}) continue - if path.startswith(("hugegraph-server/hugegraph-core/", "hugegraph-server/hugegraph-api/", - "hugegraph-server/hugegraph-test/")): - selected.update(["server", "hstore"]) + if path.startswith("hugegraph-server/hugegraph-rocksdb/"): + selected.update({"server", "macos", "macos_full"}) continue + if path in {"hugegraph-server/pom.xml", "hugegraph-server/hugegraph-api/pom.xml"}: + selected.add("docker") + if path.startswith(("hugegraph-pd/hg-pd-dist/", "hugegraph-store/hg-store-dist/")): + selected.add("docker") + if path == "hugegraph-commons/hugegraph-common/src/main/resources/version.properties": + selected.add("docker") + elif path.endswith(".proto"): + selected.update(MODULES[project]) + continue + elif path.startswith(".github/workflows/") and p.name in WORKFLOWS[project]: + selected.update(WORKFLOWS[project][p.name]) + continue + elif path.startswith("hugegraph-client/assembly/travis/"): + selected.update({"client", "go"}) + continue module = next((value for prefix, value in PREFIXES[project].items() if path.startswith(prefix)), None) - if module is None or path.endswith(".proto"): - return set(MODULES[project]) - selected.add(module) - if project == "server" and ("Dockerfile" in Path(path).name or "docker-entrypoint" in path): - selected.add("docker") + if module is None: + selected.update(MAINLINE if project == "server" else MODULES[project]) + else: + selected.add(module) changed = True while changed: before = set(selected) for module in before: - selected.update(DEPENDENTS[project].get(module, [])) + selected.update(DEPENDENTS[project].get(module, set())) changed = before != selected + if project == "server": + if "server" in selected: + selected.add("macos") + if selected.intersection({"pd", "store", "hstore", "struct", "cluster"}): + selected.add("distributed_smoke") + selected.update(contracts) + return selected + + +def event_selection(project, paths, event, event_name): + """Presets and post-merge smoke share the same affected-input classifier.""" + if project != "server": + return select(project, paths) + if event_name == "schedule": + return set(PRESETS["full"]) + if event_name == "workflow_dispatch": + preset = event.get("inputs", {}).get("test-set", "mainline") + if preset not in PRESETS: + raise ValueError("unknown test-set: " + str(preset)) + return set(PRESETS[preset]) + selected = select(project, paths) + if event_name == "push" and event.get("ref") == "refs/heads/master": + smoke = selected.intersection({"docker", "helm", "dependency_license", "macos", "distributed_smoke"}) + if selected.intersection({"server", "commons", "struct", "pd", "store", "hstore", "cluster"}): + smoke.add("distributed_smoke") + return smoke return selected def suites(project, selected): if project == "toolchain": return sorted(selected) - result = [] + result = selected.intersection({"commons", "cluster", "docker", "helm", "dependency_license", "macos", + "hbase", "riscv64", "security", "distributed_smoke"}) if "server" in selected: - result.extend(["server_memory", "server_rocksdb"]) - # Third-party inventory/review remains visible without becoming a merge gate. - for module in ["commons", "cluster", "docker", "helm"]: - if module in selected: - result.append(module) + result.update({"server_memory", "server_rocksdb"}) if selected.intersection({"pd", "store", "hstore", "struct"}): - result.append("pd_store") + result.add("pd_store") return sorted(result) @@ -182,18 +274,19 @@ def require_current_pr(plan, fetch): raise StaleInputError("PR inputs changed; refresh the branch and start a new PR run") -def create_plan(project, event, repository, fetch=api): +def create_plan(project, event, repository, fetch=api, event_name=None): + event_name = event_name or os.environ.get("GITHUB_EVENT_NAME") or ( + "pull_request" if "pull_request" in event else "push") paths = [] plan = {"schema": 1, "project": project, "repository": repository, "pr": 0, "source": repository, "branch": "", "base": "", "head": git("rev-parse", "HEAD"), - "reason": "affected inputs", "testedMergeSHA": git("rev-parse", "HEAD")} + "reason": "affected inputs", "testedMergeSHA": git("rev-parse", "HEAD"), "event": event_name} try: pr = event.get("pull_request") if "pull_request" in event: if not isinstance(pr, dict) or type(pr.get("number")) is not int or pr["number"] <= 0: raise StaleInputError("PR event lacks a valid number") plan["pr"] = pr["number"] - # An incomplete PR event must never fall back to a push plan with pr=0. try: plan.update(source=pr["head"]["repo"]["full_name"], base=pr["base"]["sha"], head=pr["head"]["sha"], branch=pr["head"]["ref"]) @@ -205,52 +298,49 @@ def create_plan(project, event, repository, fetch=api): if (plan["testedMergeSHA"] != os.environ.get("GITHUB_SHA") or len(parents) != 2 or parents[1] != plan["head"]): raise StaleInputError("checkout is not the event PR merge; start a new PR run") - # The event base may lag the synthetic merge after the target branch advances. plan["base"] = parents[0] require_current_pr(plan, fetch) - # head/base objects must exist locally; workflow fetches both before planning. ancestor = git("merge-base", plan["base"], plan["head"]) paths = git("diff", "--no-renames", "--name-only", ancestor, plan["head"]).splitlines() - else: + elif event_name == "push": plan["base"] = event.get("before", "") + plan["branch"] = event.get("ref", "").removeprefix("refs/heads/") paths = git("diff", "--no-renames", "--name-only", plan["base"], plan["head"]).splitlines() - selected = select(project, paths) - if unsafe_documentation(plan["base"], paths) or unsafe_documentation(plan["testedMergeSHA"], paths): - selected = set(MODULES[project]) - plan["reason"] = "non-regular documentation input: conservative full coverage" - plan["expected"] = suites(project, selected) + selected = event_selection(project, paths, event, event_name) + if paths and (unsafe_documentation(plan["base"], paths) + or unsafe_documentation(plan["testedMergeSHA"], paths)): + selected = event_selection(project, ["unknown"], event, event_name) + plan["reason"] = "non-regular documentation input: conservative mainline coverage" + if event_name in {"schedule", "workflow_dispatch"}: + plan["reason"] = "weekly full coverage" if event_name == "schedule" else "manual " + event.get( + "inputs", {}).get("test-set", "mainline") except (subprocess.SubprocessError, OSError, ValueError, KeyError, TypeError, AttributeError): - selected = set(MODULES[project]) - plan["reason"] = "verification unavailable: conservative full coverage" - plan["expected"] = suites(project, selected) + selected = event_selection(project, ["unknown"], {"ref": event.get("ref")}, + "push" if event_name == "push" else "pull_request") + plan["reason"] = "verification unavailable: conservative mainline coverage" if not selected: - plan["reason"] = "cumulative PR diff contains only plain prose documentation; modules unaffected" + plan["reason"] = "plain prose documentation or maintenance inputs; tests unaffected" plan["selected"] = sorted(selected) + plan["expected"] = suites(project, selected) for module in MODULES[project]: plan[module] = module in selected plan["pd_store"] = any(plan.get(m, False) for m in ["pd", "store", "hstore", "struct"]) - plan["needsFixture"] = project == "toolchain" and any(plan.get(m, False) for m in MODULES[project]) + plan["needsFixture"] = project == "toolchain" and bool(selected) plan["fixture"] = plan["needsFixture"] - plan["compatibility"] = project == "server" and bool(selected.intersection({"server", "commons", "struct", "pd", "store", "hstore", "cluster"})) - plan["smoke"] = project == "server" and bool(selected.intersection({"server", "docker"})) - plan["security"] = bool(selected - {"docker", "helm"}) or any( - p.startswith(".github/workflows/codeql") for p in locals().get("paths", [])) - plan["required"] = ["server_memory"] if project == "server" and "server" in selected else [] + if project == "toolchain": + plan["security"] = bool(selected) or any(p.startswith(".github/workflows/codeql") for p in paths) + plan["required"] = [] plan["security_languages"] = json.dumps(["java"]) plan["changedPaths"] = paths plan["selectionReasons"] = { - suite: [path for path in paths if suite in suites(project, select(project, [path]))] + suite: [path for path in paths if suite in suites(project, event_selection(project, [path], event, event_name))] for suite in plan["expected"] } - if "dependency_license" in selected: - plan["selectionReasons"]["dependency_license"] = [ - path for path in paths if "dependency_license" in select(project, [path])] if os.environ.get("GITHUB_STEP_SUMMARY"): with open(os.environ["GITHUB_STEP_SUMMARY"], "a", encoding="utf-8") as out: out.write("## Selected CI checks\n" + selection_table(plan) + "\n") return plan - def selection_reason(plan, suite): paths = plan.get("selectionReasons", {}).get(suite, []) if not paths: @@ -266,11 +356,9 @@ def selection_reason(plan, suite): def selection_table(plan): summary = "| Check | Selected | Merge requirement | Changed inputs |\n| --- | --- | --- | --- |\n" checks = suites(plan["project"], set(MODULES[plan["project"]])) - if "dependency_license" in plan["selected"]: - checks.append("dependency_license") for suite in checks: - chosen = suite in plan["expected"] or suite == "dependency_license" - role = "required" if suite == "server_memory" else "advisory" + chosen = suite in plan["expected"] + role = "advisory" summary += f"| {suite} | {'yes' if chosen else 'no'} | {role} | {selection_reason(plan, suite)} |\n" return summary @@ -281,39 +369,15 @@ def __init__(self, message, report): self.report = report -def gate(plan, results, fetch=None, mode="all"): - """Report actual outcomes; Memory and advisory checks have independent dependencies.""" +def gate(plan, results, fetch=None): + """Keep the Toolchain result summary and current-PR gate contract.""" expected = list(plan["expected"]) - if mode == "memory": - expected = [suite for suite in expected if suite == "server_memory"] - elif mode == "advisory": - expected = [suite for suite in expected if suite != "server_memory"] report = {key: plan[key] for key in ["schema", "repository", "project", "pr", "source", "branch", "base", "head", "testedMergeSHA", "selected"]} report["selectionReasons"] = plan.get("selectionReasons", {}) report.update(runID=int(os.environ.get("GITHUB_RUN_ID", "0")), runAttempt=int(os.environ.get("GITHUB_RUN_ATTEMPT", "1")), executed=expected, results={suite: results.get(suite, {}).get("result", "missing") for suite in expected}) - if mode != "memory" and "dependency_license" in plan["selected"]: - report["results"]["dependency_license"] = results.get("dependency_license", {}).get("result", "missing") - summary = "Selection: " + plan.get("reason", "affected inputs") + "\n" - summary += "Tested base: `" + plan["base"] + "`; merge: `" + plan["testedMergeSHA"] + "`\n\n" - summary += "| Check | Selected | Merge requirement | Changed inputs | Result |\n| --- | --- | --- | --- | --- |\n" - for suite in suites(plan["project"], set(MODULES[plan["project"]])): - if mode == "memory" and suite != "server_memory": - continue - if mode == "advisory" and suite == "server_memory": - continue - chosen = suite in expected - role = "required" if suite == "server_memory" else "advisory" - status = report["results"].get(suite, "not selected") - summary += f"| {suite} | {'yes' if chosen else 'no'} | {role} | {selection_reason(plan, suite)} | {status} |\n" - if "dependency_license" in report["results"]: - summary += "| dependency_license | yes | advisory | " + selection_reason(plan, "dependency_license") + " | " + str(report["results"]["dependency_license"]) + " |\n" - if os.environ.get("GITHUB_STEP_SUMMARY"): - with open(os.environ["GITHUB_STEP_SUMMARY"], "a", encoding="utf-8") as out: - out.write("## " + ("Memory result" if mode == "memory" else "Affected module results") + "\n" + summary + "\n") - print(summary) errors = [] if results.get("plan", {}).get("result") != "success": errors.append("planner did not succeed") @@ -325,15 +389,21 @@ def gate(plan, results, fetch=None, mode="all"): errors.append("selected tests lack successful fixture") if "hubble" in expected and results.get("hubble-fixture", {}).get("result") != "success": errors.append("selected Hubble tests lack successful baseline fixture") + summary = "Selection: " + plan.get("reason", "affected inputs") + "\n" + summary += "Tested base: `" + plan["base"] + "`; merge: `" + plan["testedMergeSHA"] + "`\n\n" + summary += "| Check | Selected | Merge requirement | Changed inputs | Result |\n| --- | --- | --- | --- | --- |\n" + for suite in suites(plan["project"], set(MODULES[plan["project"]])): + chosen = suite in expected + status = report["results"].get(suite, "not selected") + summary += f"| {suite} | {'yes' if chosen else 'no'} | advisory | {selection_reason(plan, suite)} | {status} |\n" + if os.environ.get("GITHUB_STEP_SUMMARY"): + with open(os.environ["GITHUB_STEP_SUMMARY"], "a", encoding="utf-8") as out: + out.write("## Affected module results\n" + summary + "\n") + print(summary) if errors: raise GateFailure("; ".join(errors), report) - # Checks are attached to their actual commit; target-branch advancement alone is not stale. - # A transient API outage cannot invalidate a genuine completed Memory result. - try: + if plan["project"] == "toolchain": require_current_pr(plan, fetch or api) - except (subprocess.SubprocessError, OSError): - if mode != "memory": - raise return report @@ -345,14 +415,16 @@ def main(): p.add_argument("--event-path", required=True) p.add_argument("--repository", required=True) p.add_argument("--output", required=True) + p.add_argument("--event-name") p = sub.add_parser("gate") p.add_argument("--plan", required=True) p.add_argument("--results", required=True) - p.add_argument("--mode", choices=["all", "memory", "advisory"], default="all") + p.add_argument("--mode", choices=["all"], default="all") p.add_argument("--output", required=True) args = parser.parse_args() if args.command == "plan": - value = create_plan(args.project, json.loads(Path(args.event_path).read_text()), args.repository) + value = create_plan(args.project, json.loads(Path(args.event_path).read_text()), args.repository, + event_name=args.event_name) if os.environ.get("GITHUB_OUTPUT"): with open(os.environ["GITHUB_OUTPUT"], "a", encoding="utf-8") as output: for key, item in value.items(): @@ -361,7 +433,7 @@ def main(): output.write(f"security_languages={value['security_languages']}\nplan_file={args.output}\n") else: try: - value = gate(json.loads(Path(args.plan).read_text()), json.loads(Path(args.results).read_text()), mode=args.mode) + value = gate(json.loads(Path(args.plan).read_text()), json.loads(Path(args.results).read_text())) except GateFailure as error: Path(args.output).write_text(json.dumps(error.report, indent=2) + "\n") raise diff --git a/.github/scripts/test_check_rerun.py b/.github/scripts/test_check_rerun.py deleted file mode 100644 index e9e7f5444f..0000000000 --- a/.github/scripts/test_check_rerun.py +++ /dev/null @@ -1,203 +0,0 @@ -#!/usr/bin/env python3 -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Behavioral tests for push retry freshness, without credentials or network.""" -import importlib.util -from pathlib import Path -import unittest -import os -import subprocess -import tempfile -import json -from unittest.mock import patch - -spec = importlib.util.spec_from_file_location("check_rerun", Path(__file__).with_name("check-rerun.py")) -checker = importlib.util.module_from_spec(spec) -spec.loader.exec_module(checker) - - -class FreshnessTest(unittest.TestCase): - def setUp(self): - self.repository = "apache/hugegraph" - self.run = {"status": "completed", "conclusion": "failure", "run_attempt": 1, - "path": ".github/workflows/server-ci.yml", "event": "push", "head_sha": "old", "head_branch": "release/test", - "repository": {"full_name": self.repository}, - "head_repository": {"full_name": self.repository}} - self.current_head = "old" - self.calls = [] - self.jobs = [self.job(117, "build-server / server_memory (Java 17)"), - self.job(118, "Server memory tests")] - - def job(self, job_id, name, conclusion="failure"): - return {"id": job_id, "name": name, "status": "completed", "conclusion": conclusion, - "run_id": 42, "head_sha": "old"} - - def fetch(self, path): - self.calls.append(path) - if path == f"repos/{self.repository}/actions/runs/42": - return self.run - if path == f"repos/{self.repository}/commits?sha=release%2Ftest&per_page=1": - return [{"sha": self.current_head}] if self.current_head else [] - if path.startswith(f"repos/{self.repository}/actions/runs/42/attempts/"): - page = int(path.rsplit("page=", 1)[1]) - return {"jobs": self.jobs[(page - 1) * 100:page * 100]} - raise AssertionError("Unexpected lookup: " + path) - - def decide(self): - return checker.decide(self.repository, 42, 1, 2, self.fetch)[0] - - def test_only_server_required_jobs_retry(self): - for path in [".github/workflows/server-memory-ci.yml", ".github/workflows/licence-checker.yml", - ".github/workflows/codeql-analysis.yml", ".github/workflows/server-compatibility-ci.yml"]: - self.run["path"] = path - self.assertEqual("skip", self.decide()) - self.run["path"] = ".github/workflows/server-ci.yml" - self.assertEqual("rerun", self.decide()) - - def test_failed_producer_is_selected_before_reporter(self): - for name in ["check-license", "memory-plan", "memory-java-runtime / read-version", - "build-server / server_memory (Java 17)"]: - with self.subTest(name=name): - self.jobs = [self.job(118, "Server memory tests"), self.job(117, name)] - self.assertEqual(117, checker.decide(self.repository, 42, 1, 2, self.fetch)[2]) - self.jobs = [self.job(118, "Server memory tests")] - self.assertEqual(118, checker.decide(self.repository, 42, 1, 2, self.fetch)[2]) - - def test_timed_out_producer_is_retried_only_inside_failed_run(self): - self.jobs = [self.job(118, "Server memory tests"), - self.job(117, "build-server / server_memory (Java 17)", "timed_out")] - self.assertEqual(117, checker.decide(self.repository, 42, 1, 2, self.fetch)[2]) - self.run["conclusion"] = "timed_out" - self.assertEqual("skip", self.decide()) - - def test_timed_out_reporter_alone_is_not_retried(self): - self.jobs = [self.job(118, "Server memory tests", "timed_out")] - self.assertEqual("skip", self.decide()) - - def test_advisory_success_and_non_failure_jobs_are_not_retried(self): - for name in ["plan", "java-runtime / read-version", "server-rocksdb / server_rocksdb (Java 17)", - "pd_store / struct", "affected-module-tests", "build-server / server_memory (Java 17) / extra"]: - self.jobs = [self.job(118, name)] - self.assertEqual("skip", self.decide()) - for conclusion in ["success", "cancelled", "skipped", None]: - self.jobs = [self.job(118, "Server memory tests", conclusion)] - self.assertEqual("skip", self.decide()) - - def test_job_selection_paginates_the_exact_attempt(self): - self.jobs = [self.job(i + 1, "advisory") for i in range(100)] + [self.job(117, "memory-plan")] - self.run["run_attempt"] = 2 - self.assertEqual(117, checker.decide(self.repository, 42, 2, 2, self.fetch)[2]) - self.assertIn(f"repos/{self.repository}/actions/runs/42/attempts/2/jobs?per_page=100&page=2", self.calls) - self.assertFalse(any("/attempts/1/" in path for path in self.calls)) - - def test_job_identity_and_metadata_must_match(self): - for key, value in [("run_id", 99), ("head_sha", "different"), ("id", "117"), ("id", -1)]: - with self.subTest(key=key): - self.jobs = [dict(self.job(117, "memory-plan"), **{key: value})] - self.assertEqual("skip", self.decide()) - - def test_workflow_posts_native_rerun_for_checker_selected_job(self): - workflow = Path(__file__).resolve().parents[1] / "workflows/rerun-ci.yml" - source = workflow.read_text() - self.assertIn(' - "HugeGraph-Server CI"', source) - self.assertIn("github.event.workflow_run.event == 'push'", source) - self.assertIn("ref: ${{ github.sha }}", source) - command = source.split(" - name: Rerun the required job and its dependencies", 1)[1] - command = command.split(" run: |\n", 1)[1].strip() - with tempfile.TemporaryDirectory() as directory: - root = Path(directory) - output = root / "output" - decide = checker.decide - with patch.dict(os.environ, {"GITHUB_OUTPUT": str(output)}), patch( - "sys.argv", ["check-rerun.py", "--repository", self.repository, - "--run-id", "42", "--run-attempt", "1", "--max-reruns", "2"]), patch.object( - checker, "decide", side_effect=lambda *args: decide(*args, fetch=self.fetch)): - checker.main() - selected = dict(line.split("=", 1) for line in output.read_text().splitlines()) - self.assertEqual("rerun", selected["action"]) - command = command.replace("${{ steps.freshness.outputs.job_id }}", selected["job_id"]) - gh = root / "gh" - gh.write_text('#!/usr/bin/env python3\nimport json, os, sys\n' - 'open(os.environ["ARGS_PATH"], "w").write(json.dumps(sys.argv[1:]))\n') - gh.chmod(0o700) - args_path = root / "args.json" - subprocess.run(["bash", "-eu", "-c", command], check=True, - env={**os.environ, "GH_REPO": self.repository, "ARGS_PATH": str(args_path), - "PATH": str(root) + os.pathsep + os.environ["PATH"]}) - self.assertEqual(["api", "--method", "POST", f"repos/{self.repository}/actions/jobs/117/rerun"], - json.loads(args_path.read_text())) - - def test_api_failure_fails_closed(self): - with tempfile.TemporaryDirectory() as directory: - output = Path(directory) / "output" - with patch.dict(os.environ, {"GITHUB_OUTPUT": str(output)}), patch( - "sys.argv", ["check-rerun.py", "--repository", self.repository, - "--run-id", "42", "--run-attempt", "1", "--max-reruns", "2"]), patch.object( - checker, "decide", side_effect=subprocess.CalledProcessError(1, "gh")): - checker.main() - self.assertIn("action=skip\n", output.read_text()) - - def test_pr_is_paused_without_reading_artifacts_or_candidates(self): - self.run.update(event="pull_request", head_repository={"full_name": "alice/hugegraph"}) - self.assertEqual("skip", self.decide()) - self.assertEqual([f"repos/{self.repository}/actions/runs/42"], self.calls) - - def test_changed_attempt(self): - self.run["run_attempt"] = 2 - self.assertEqual("skip", self.decide()) - - def test_current_run_must_remain_failed_and_completed(self): - for key, value in (("status", "in_progress"), ("conclusion", "success"), - ("conclusion", "cancelled")): - with self.subTest(key=key, value=value): - old = self.run[key] - self.run[key] = value - self.assertEqual("skip", self.decide()) - self.run[key] = old - - def test_retry_limit(self): - self.run["run_attempt"] = 3 - self.assertEqual("skip", checker.decide(self.repository, 42, 3, 2, self.fetch)[0]) - self.run["run_attempt"] = 2 - self.assertEqual("rerun", checker.decide(self.repository, 42, 2, 2, self.fetch)[0]) - - def test_push_checks_repository_and_encodes_branch(self): - self.assertEqual("rerun", self.decide()) - self.assertIn(f"repos/{self.repository}/commits?sha=release%2Ftest&per_page=1", self.calls) - self.current_head = "new" - self.assertEqual("skip", self.decide()) - self.current_head = None - self.assertEqual("skip", self.decide()) - - def test_push_rejects_other_repository(self): - for field in ["repository", "head_repository"]: - with self.subTest(field=field): - self.run[field] = {"full_name": "alice/hugegraph"} - self.assertEqual("skip", self.decide()) - self.run[field] = {"full_name": self.repository} - - def test_missing_metadata_and_unsupported_event(self): - for field in ["head_sha", "head_branch", "repository", "head_repository"]: - old = self.run.pop(field) - with self.subTest(field=field): - self.assertEqual("skip", self.decide()) - self.run[field] = old - self.run["event"] = "workflow_dispatch" - self.assertEqual("skip", self.decide()) - - -if __name__ == "__main__": - unittest.main() diff --git a/.github/scripts/test_ci_policy.py b/.github/scripts/test_ci_policy.py index 28aa746835..a60486ebec 100644 --- a/.github/scripts/test_ci_policy.py +++ b/.github/scripts/test_ci_policy.py @@ -14,11 +14,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Executable Git fixtures and fail-closed gate tests for CI selection.""" +"""Path/event selection, real PR snapshots, and coordinator scheduling contracts.""" import importlib.util import json import os from pathlib import Path +import re import subprocess import tempfile import textwrap @@ -29,590 +30,499 @@ policy = importlib.util.module_from_spec(spec) spec.loader.exec_module(policy) -class PolicyTest(unittest.TestCase): - def live_pr(self): - return {"state": "open", "head": {"sha": "head", "ref": "feature", "repo": {"full_name": "alice/server"}}, - "base": {"sha": "base", "repo": {"full_name": "apache/server"}}} - - def setUp(self): - event_sha = patch.dict(os.environ, {"GITHUB_SHA": "merge"}) - self.addCleanup(event_sha.stop) - event_sha.start() - # All tests stay local; successful PR gates query only this current-PR fixture. - api_mock = patch.object(policy, "api", return_value=self.live_pr()) - self.addCleanup(api_mock.stop) - self.api_mock = api_mock.start() - - def test_dependency_expansion(self): - self.assertEqual({"client", "loader", "tools", "spark", "hubble"}, - policy.select("toolchain", ["hugegraph-client/src/A.java"])) - self.assertEqual({"loader", "hubble"}, policy.select("toolchain", ["hugegraph-loader/src/A.java"])) - self.assertEqual({"server", "pd", "store", "hstore", "cluster"}, policy.select("server", ["hugegraph-server/A.java"])) - - def test_server_backend_and_startup_dependents(self): - selected = policy.select("server", ["hugegraph-server/hugegraph-hstore/src/test/HstoreTableTest.java"]) - self.assertIn("hstore", selected) - self.assertIn("pd_store", policy.suites("server", selected)) - selected = policy.select("server", ["hugegraph-server/hugegraph-dist/src/assembly/travis/test-start-hugegraph-pd.sh"]) - self.assertIn("pd_store", policy.suites("server", selected)) - self.assertIn("docker", selected) - for module in ["hugegraph-core", "hugegraph-api", "hugegraph-test"]: - selected = policy.select("server", [f"hugegraph-server/{module}/src/main/A.java"]) - self.assertIn("hstore", selected) - self.assertIn("server", selected) - self.assertEqual(set(), policy.select("server", ["hugegraph-server/hugegraph-hstore/README.md"])) - self.assertIn("hstore", policy.select("server", ["hugegraph-server/hugegraph-rocksdb/src/main/A.java"])) - - def test_audited_consumer_edges(self): - expected = {"server", "pd", "store", "hstore", "cluster"} - for path in ["hugegraph-server/hugegraph-hstore/src/A.java", "hugegraph-server/hugegraph-rocksdb/A.java"]: - self.assertTrue(expected.issubset(policy.select("server", [path]))) - self.assertNotIn("helm", policy.select("server", [path])) - self.assertEqual(set(policy.MODULES["server"]), policy.select("server", [".github/workflows/server-ci.yml"])) - self.assertTrue({"commons", "docker"}.issubset(policy.select("server", [ - "hugegraph-commons/hugegraph-common/src/main/resources/version.properties"]))) - self.assertIn("go", policy.select("toolchain", ["hugegraph-client/assembly/travis/start-hugegraph-servers.sh"])) - self.assertNotIn("go", policy.select("toolchain", ["hugegraph-client/src/A.java"])) - self.assertNotIn("server", policy.select("server", ["hugegraph-pd/src/A.java"])) - self.assertTrue({"store", "docker"}.issubset(policy.select("server", [ - "hugegraph-store/hg-store-dist/src/assembly/static/bin/util.sh"]))) - - def test_pd_store_distribution_inputs_include_docker_consumers(self): - for module, distribution, startup in [ - ("pd", "hg-pd-dist", "start-hugegraph-pd.sh"), - ("store", "hg-store-dist", "start-hugegraph-store.sh"), - ]: - for relative in [f"src/assembly/static/bin/{startup}", - "src/assembly/static/conf/application.yml", - "src/assembly/descriptor/server-assembly.xml", "pom.xml"]: - path = f"hugegraph-{module}/{distribution}/{relative}" - with self.subTest(path=path): - selected = policy.select("server", [path]) - self.assertTrue({module, "hstore", "cluster", "docker"}.issubset(selected)) - self.assertIn("pd_store", policy.suites("server", selected)) - self.assertNotIn("server", selected) - self.assertNotIn("helm", selected) - self.assertEqual(relative == "pom.xml", "dependency_license" in selected) - source = f"hugegraph-{module}/hg-{module}-core/src/main/java/Example.java" - with self.subTest(path=source): - selected = policy.select("server", [source]) - self.assertTrue({module, "hstore", "cluster"}.issubset(selected)) - self.assertNotIn("docker", selected) - - def test_version_resource_follows_real_docker_consumer(self): - root = Path(__file__).resolve().parents[2] - consumer = root / ".github/scripts/check-docker-images.sh" - if not consumer.exists(): - self.skipTest("Server Docker consumer is not in the Toolchain repository") - resource = next(line.strip().rstrip(")") for line in consumer.read_text().splitlines() - if line.strip().startswith("hugegraph-commons/") and "version.properties" in line) - self.assertTrue((root / resource).is_file()) - self.assertTrue({"commons", "docker"}.issubset(policy.select("server", [resource]))) - - def test_single_workflow_has_no_global_fanout(self): - self.assertEqual({"hubble"}, policy.select("toolchain", [".github/workflows/hubble-ci.yml"])) - self.assertEqual({"docker"}, policy.select("server", [".github/workflows/docker-build-ci.yml"])) - - def test_known_maintenance_inputs_have_specific_owners(self): - for path in [".github/PULL_REQUEST_TEMPLATE.md", ".github/dependabot.yml", - ".github/scripts/check-rerun.py", ".github/scripts/test_check_rerun.py"]: - with self.subTest(path=path): - self.assertEqual(set(), policy.select("server", [path])) - for path in [".github/scripts/check-docker-images.sh", ".github/scripts/docker-deployment.py", - ".github/scripts/test_docker_deployment.py"]: - with self.subTest(path=path): - self.assertEqual({"docker"}, policy.select("server", [path])) - selected = policy.select("server", [path, "hugegraph-pd/hg-pd-core/src/A.java"]) - self.assertTrue({"docker", "pd", "store", "hstore", "cluster"}.issubset(selected)) - for path in [".github/scripts/ci-policy.py", ".github/scripts/test_ci_policy.py", - ".github/workflows/rerun-ci.yml", ".github/dependabot-unknown.yml"]: - with self.subTest(path=path): - self.assertEqual(set(policy.MODULES["server"]), policy.select("server", [path])) +class GitFixture: + def __init__(self, root): + self.root = root + self.git("init", "-q") + self.git("config", "user.email", "ci@example.invalid") + self.git("config", "user.name", "CI") - def test_unknown_and_proto_fail_conservative(self): - for path in ["pom.xml", ".github/scripts/new.py", "hugegraph-pd/api.proto", "mystery"]: - self.assertEqual(set(policy.MODULES["server"]), policy.select("server", [path])) + def git(self, *args): + return subprocess.check_output(["git", *args], cwd=self.root, text=True).strip() - def test_strict_docs_allowlist(self): - self.assertEqual(set(), policy.select("server", ["README.md", "docs/guide.md"])) - for path in ["hugegraph-client/src/test/resources/README.md", "docs/type.ts", "docs/config.yml", "docs/example.java", "hugegraph-server/type.ts"]: - self.assertTrue(policy.select("server", [path])) + def commit(self, path, content, message): + file = self.root / path + file.parent.mkdir(parents=True, exist_ok=True) + file.write_text(content) + self.git("add", ".") + self.git("commit", "-qm", message) + return self.git("rev-parse", "HEAD") - def test_real_git_modes_rename_and_cli(self): - with tempfile.TemporaryDirectory() as directory: - root = Path(directory) - def git(*args): - return subprocess.check_output(["git", *args], cwd=root, text=True).strip() - git("init", "-q") - git("config", "user.email", "ci@example.invalid") - git("config", "user.name", "CI") - (root / "README.md").write_text("initial") - git("add", ".") - git("commit", "-qm", "base") - base = git("rev-parse", "HEAD") - (root / "README.md").write_text("docs change") - git("commit", "-qam", "docs") - head = git("rev-parse", "HEAD") + def plan(self, event, fetch=None, event_name=None): + with patch.object(policy, "git", side_effect=self.git), patch.dict( + os.environ, {"GITHUB_SHA": self.git("rev-parse", "HEAD"), "GITHUB_EVENT_NAME": ""}): old = os.getcwd() try: - os.chdir(root) - (root / "type.ts").write_text("type changed") - git("add", ".") - git("commit", "-qm", "types") - (root / "README.md").chmod(0o755) - git("add", "README.md") - git("commit", "-qm", "executable prose") - self.assertTrue(policy.unsafe_documentation("HEAD", ["README.md"])) - (root / "README.md").unlink() - (root / "README.md").symlink_to("type.ts") - git("add", "README.md") - git("commit", "-qm", "symlink prose") - self.assertTrue(policy.unsafe_documentation("HEAD", ["README.md"])) - (root / "hugegraph-server").mkdir() - (root / "hugegraph-server" / "A.java").write_text("source") - git("add", ".") - git("commit", "-qm", "source") - before_rename = git("rev-parse", "HEAD") - (root / "docs").mkdir() - git("mv", "hugegraph-server/A.java", "docs/renamed.md") - git("commit", "-qm", "rename code to prose") - changed = git("diff", "--no-renames", "--name-only", before_rename, "HEAD").splitlines() - self.assertIn("server", policy.select("server", changed)) + os.chdir(self.root) + return policy.create_plan("server", event, "apache/server", fetch=fetch, event_name=event_name) finally: os.chdir(old) - event = root / "event.json" - event.write_text(json.dumps({"before": base})) - output = root / "output" - subprocess.run(["python3", str(Path(policy.__file__).resolve()), "plan", "--project", "server", - "--repository", "apache/server", "--event-path", str(event), - "--output", str(root / "plan.json")], cwd=root, check=True, - env={**os.environ, "GITHUB_OUTPUT": str(output)}) - self.assertIn("server=true\n", output.read_text()) - self.assertEqual(policy.suites("server", set(policy.MODULES["server"])), - json.loads((root / "plan.json").read_text())["expected"]) - def plan(self): - return {"schema": 1, "project": "server", "repository": "apache/server", "pr": 7, - "source": "alice/server", "branch": "feature", "base": "base", "head": "head", - "testedMergeSHA": "merge", "expected": ["server_memory", "server_rocksdb", "pd_store", "cluster"], - "selected": ["server", "pd", "store", "hstore", "cluster"]} - def memory_workflow_results(self, planner="success", runtime="success", build="success", selected=True): - workflow = Path(__file__).resolve().parents[1] / "workflows/server-ci.yml" - reporter = workflow.read_text().split("\n server-memory-required:\n", 1)[1] - code = reporter.split("python3 - <<'PYGATE'\n", 1)[1].split("\n PYGATE", 1)[0] - # Start with the actual needs keys, without generic gate aliases. - needs = {"memory-plan": {"result": planner, "outputs": {"server": str(selected).lower()}}, - "memory-java-runtime": {"result": runtime}, "build-server": {"result": build}} - with tempfile.TemporaryDirectory() as directory: - outcome = subprocess.run(["python3", "-c", textwrap.dedent(code)], cwd=directory, - env={**os.environ, "JOB_RESULTS": json.dumps(needs)}, - capture_output=True, text=True) - if outcome.returncode: - return outcome, None - return outcome, json.loads((Path(directory) / "results.json").read_text()) - - def test_memory_workflow_results_feed_real_gate(self): - for state in ["success", "failure", "cancelled", "skipped", None]: - with self.subTest(build=state): - outcome, results = self.memory_workflow_results(build=state) - self.assertEqual(0, outcome.returncode, outcome.stderr) - if state == "success": - self.assertEqual(["server_memory"], policy.gate(self.plan(), results, mode="memory")["executed"]) - else: - with self.assertRaisesRegex(policy.GateFailure, "server_memory"): - policy.gate(self.plan(), results, mode="memory") - outcome, results = self.memory_workflow_results(runtime="skipped", build="skipped", selected=False) - self.assertEqual(0, outcome.returncode, outcome.stderr) - docs = dict(self.plan(), expected=[], selected=[]) - self.assertEqual([], policy.gate(docs, results, mode="memory")["executed"]) - for state in ["failure", "cancelled", "skipped"]: - with self.subTest(planner=state): - outcome, results = self.memory_workflow_results(planner=state) - self.assertNotEqual(0, outcome.returncode) - self.assertIn("CI planning failed", outcome.stderr) - with self.subTest(runtime=state): - outcome, results = self.memory_workflow_results(runtime=state) - self.assertNotEqual(0, outcome.returncode) - self.assertIn("Selected Java runtime preparation failed", outcome.stderr) - - def test_memory_gate_ignores_advisory_cancellation(self): - plan = self.plan() - results = {"plan": {"result": "success"}, "server_memory": {"result": "success"}, - "cluster": {"result": "cancelled"}, "server_rocksdb": {"result": "failure"}} - self.assertEqual(["server_memory"], policy.gate(plan, results, mode="memory")["executed"]) - for state in ["failure", "cancelled", "skipped", None]: - results["server_memory"]["result"] = state - with self.assertRaisesRegex(ValueError, "server_memory"): - policy.gate(plan, results, mode="memory") - with self.assertRaises(policy.GateFailure) as failure: - policy.gate(plan, results, mode="advisory") - self.assertEqual("cancelled", failure.exception.report["results"]["cluster"]) - self.assertNotIn("server_memory", failure.exception.report["results"]) - - def test_server_and_api_poms_are_docker_inputs(self): - for path in ["hugegraph-server/pom.xml", "hugegraph-server/hugegraph-api/pom.xml"]: +class PolicyTest(unittest.TestCase): + def plan_for(self, paths, event_name="pull_request", event=None): + # Identity checks have their own real-Git fixtures; isolate event selection here. + def git(*args): + if args[0] == "diff": + return "\n".join(paths) + if args[0] == "show": + return "base head" + return "merge" + event = event or {"before": "base"} + if event_name == "pull_request": + event = {"pull_request": {"number": 7, "head": {"sha": "head", "ref": "feature", + "repo": {"full_name": "alice/server"}}, "base": {"sha": "base"}}} + with patch.object(policy, "git", side_effect=git), patch.object( + policy, "unsafe_documentation", return_value=False), patch.object( + policy, "require_current_pr"), patch.dict(os.environ, {"GITHUB_SHA": "merge"}): + return policy.create_plan("server", event, "apache/server", event_name=event_name) + + def test_ordinary_server_changes_only_run_mainline_server_and_arm_smoke(self): + for path in ["hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/traversal/A.java", + "hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/api/traversers/A.java", + "hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/traversal/A.java"]: with self.subTest(path=path): - self.assertIn("docker", policy.select("server", [path])) - self.assertNotIn("docker", policy.select("server", ["hugegraph-server/hugegraph-api/src/main/A.java"])) - - def test_codeql_and_smoke_follow_affected_inputs(self): - def plan_for(path): - def git(*args): - if args[0] == "diff": - return path - return "base" - with patch.object(policy, "git", side_effect=git), patch.object(policy, "unsafe_documentation", return_value=False): - return policy.create_plan("server", {"before": "base"}, "apache/server") - for path in ["README.md", "docs/guide.md", "helm/templates/service.yaml"]: + plan = self.plan_for([path]) + self.assertEqual({"server", "macos"}, set(plan["selected"])) + self.assertEqual({"server_memory", "server_rocksdb", "macos"}, set(plan["expected"])) + self.assertFalse(plan["macos_full"]) + self.assertFalse(plan["pd_store"]) + + def test_distributed_server_contracts_keep_hstore_cluster_without_pd_store_units(self): + for path in [policy.SERVER_CORE + "backend/store/BackendStore.java", + policy.SERVER_CORE + "meta/MetaManager.java", policy.SERVER_CORE + "StandardHugeGraph.java", + policy.SERVER_CORE + "task/DistributedTaskScheduler.java", policy.SERVER_CORE + "task/TaskManager.java", + policy.SERVER_CORE + "masterelection/GlobalMasterInfo.java", + policy.SERVER_API + "core/GraphManager.java", + "hugegraph-server/hugegraph-hstore/src/main/A.java"]: with self.subTest(path=path): - plan = plan_for(path) - self.assertFalse(plan["security"]) - self.assertFalse(plan["smoke"]) - plan = plan_for("docker/Dockerfile") - self.assertFalse(plan["security"]) - self.assertTrue(plan["smoke"]) - for path in [".mvn/maven.config", "pom.xml", "hugegraph-pd/src/main/A.java", - ".github/workflows/codeql-analysis.yml"]: + plan = self.plan_for([path]) + self.assertTrue({"server", "macos", "hstore", "cluster", "distributed_smoke"}.issubset(plan["selected"])) + self.assertTrue(plan["pd_store"]) + self.assertFalse(plan["pd"]) + self.assertFalse(plan["store"]) + self.assertFalse(plan["hbase"]) + self.assertFalse(plan["riscv64"]) + + def test_shared_runtime_consumers_and_protocols(self): + for module in ["commons", "struct"]: + selected = policy.select("server", [f"hugegraph-{module}/src/main/A.java"]) + self.assertTrue({"server", "pd", "store", "hstore", "cluster", "macos"}.issubset(selected)) + self.assertNotIn("hbase", selected) + self.assertNotIn("riscv64", selected) + for module in ["pd", "store"]: + plan = self.plan_for([f"hugegraph-{module}/src/main/A.java"]) + self.assertTrue(plan["pd_store"]) + self.assertTrue(plan["hstore"]) + self.assertTrue(plan["cluster"]) + self.assertFalse(plan["server"]) + self.assertFalse(plan["macos"]) + self.assertTrue({"pd", "store", "hstore", "cluster"}.issubset(policy.select( + "server", ["hugegraph-pd/hg-pd-grpc/src/main/proto/common.proto"]))) + + def test_noncore_platforms_only_follow_direct_inputs(self): + for path in ["hugegraph-server/hugegraph-hbase/src/main/A.java", + "hugegraph-server/hugegraph-dist/src/assembly/travis/install-hbase.sh", + "hugegraph-server/hugegraph-dist/src/assembly/travis/hbase-ci-site.xml"]: + self.assertEqual({"hbase"}, policy.select("server", [path])) + self.assertEqual({"riscv64"}, policy.select("server", [".github/workflows/riscv64-ci.yml"])) + self.assertTrue(self.plan_for([".github/workflows/server-compatibility-ci.yml"])["riscv64"]) + self.assertTrue(self.plan_for([".github/workflows/server-tests.yml"])["hbase"]) + self.assertTrue(self.plan_for(["hugegraph-server/hugegraph-rocksdb/pom.xml"])["riscv64"]) + self.assertTrue(self.plan_for(["hugegraph-server/hugegraph-dist/pom.xml"])["riscv64"]) + for path in ["hugegraph-server/hugegraph-rocksdb/src/main/A.java", + "hugegraph-server/hugegraph-dist/src/assembly/static/bin/start-hugegraph.sh"]: + plan = self.plan_for([path]) + self.assertTrue(plan["macos"]) + self.assertTrue(plan["macos_full"]) + self.assertFalse(plan["hbase"]) + self.assertEqual(path.startswith("hugegraph-server/hugegraph-dist/src/assembly/static/bin/"), + plan["riscv64"]) + for path in ["docker/hbase/Dockerfile", "docker/hbase/entrypoint.sh", policy.TRAVIS + "install-backend.sh"]: + self.assertTrue(self.plan_for([path])["hbase"]) + install = self.plan_for([policy.TRAVIS + "install-backend.sh"]) + self.assertTrue(install["hstore"]) + java = self.plan_for([".github/workflows/.java-version"]) + self.assertTrue(java["macos_full"]) + self.assertTrue(java["hbase"]) + self.assertTrue(java["riscv64"]) + for path in policy.PLATFORM_POMS: + self.assertTrue(self.plan_for([path])["riscv64"]) + self.assertTrue(self.plan_for([path])["macos_full"]) + for path in ["hugegraph-server/hugegraph-core/src/main/resources/proto/A.proto", + "hugegraph-pd/hg-pd-grpc/src/main/proto/A.proto"]: + self.assertTrue(self.plan_for([path])["riscv64"]) + self.assertFalse(self.plan_for(["hugegraph-store/hg-store-grpc/src/main/proto/A.proto"])["riscv64"]) + self.assertFalse(self.plan_for(["hugegraph-pd/hg-pd-core/pom.xml"])["riscv64"]) + native = self.plan_for(["hugegraph-server/hugegraph-dist/src/assembly/travis/run-native-runtime-smoke-test.sh"]) + self.assertTrue(native["macos_full"]) + self.assertTrue(native["riscv64"]) + self.assertTrue(self.plan_for([policy.TRAVIS + "check-rocksdb-only-dist.sh"])["riscv64"]) + + def test_security_tracks_security_and_dependencies_only(self): + paths = [policy.SERVER_CORE + "auth/AuthManager.java", policy.SERVER_CORE + "security/HugeSecurityManager.java", + policy.SERVER_CORE + "config/AuthOptions.java", + policy.SERVER_API + "api/filter/AuthenticationFilter.java", "hugegraph-pd/hg-pd-core/pom.xml", + ".github/workflows/codeql-analysis.yml"] + for path in paths: with self.subTest(path=path): - self.assertTrue(plan_for(path)["security"]) - - def test_memory_result_survives_metadata_outage_and_base_movement(self): - plan = self.plan() - results = {"plan": {"result": "success"}, "server_memory": {"result": "success"}} - def unavailable(_): - raise subprocess.CalledProcessError(1, "gh") - self.assertEqual(["server_memory"], policy.gate(plan, results, unavailable, mode="memory")["executed"]) - advanced = self.live_pr() - advanced["base"]["sha"] = "advanced" - policy.gate(plan, results, lambda _: advanced, mode="memory") - advanced["head"]["sha"] = "new-head" - with self.assertRaises(policy.StaleInputError): - policy.gate(plan, results, lambda _: advanced, mode="memory") - - def test_plan_explains_consumer_selection_and_docs_skip(self): - source = "hugegraph-server/hugegraph-api/pom.xml" - def git(*args): - return source if args[0] == "diff" else "base" - with tempfile.TemporaryDirectory() as directory: - summary = Path(directory) / "summary.md" - with patch.dict(os.environ, {"GITHUB_STEP_SUMMARY": str(summary)}), patch.object( - policy, "git", side_effect=git), patch.object(policy, "unsafe_documentation", return_value=False): - plan = policy.create_plan("server", {"before": "base"}, "apache/server") - self.assertEqual([source], plan["selectionReasons"]["docker"]) - self.assertEqual([source], plan["selectionReasons"]["server_memory"]) - self.assertIn(source, summary.read_text()) - self.assertIn("| server_memory | yes | required |", summary.read_text()) - self.assertIn("| docker | yes | advisory |", summary.read_text()) - source = "README.md" - docs = policy.create_plan("server", {"before": "base"}, "apache/server") - self.assertEqual([], docs["expected"]) - self.assertEqual({}, docs["selectionReasons"]) - self.assertIn("| server_memory | no | required | no affected inputs |", summary.read_text()) - - def test_memory_rejects_confirmed_source_and_malformed_metadata(self): - plan = self.plan() - results = {"plan": {"result": "success"}, "server_memory": {"result": "success"}} - live = self.live_pr() - live["head"]["repo"]["full_name"] = "another/fork" - with self.assertRaises(policy.StaleInputError): - policy.gate(plan, results, lambda _: live, mode="memory") - for malformed in [{"state": "open"}, {"state": "open", "head": None}]: - with self.subTest(metadata=malformed), self.assertRaises((KeyError, TypeError)): - policy.gate(plan, results, lambda _: malformed, mode="memory") - - def test_gate_rejects_missing_failed_cancelled_skipped(self): - plan = self.plan() - results = {suite: {"result": "success"} for suite in plan["expected"]} - results["plan"] = {"result": "success"} - # Even an old plan with claimed successes cannot exempt any current suite. - plan["reused"] = {suite: {"runID": 42, "testedMergeSHA": "forged"} for suite in plan["expected"]} - for suite in plan["expected"]: - for result in [None, "failure", "cancelled", "skipped"]: - with self.subTest(suite=suite, result=result), self.assertRaisesRegex(ValueError, suite): - policy.gate(plan, dict(results, **{suite: {"result": result}})) - self.assertEqual(plan["expected"], policy.gate(plan, results)["executed"]) - - def test_third_party_failure_does_not_exempt_required_tests(self): - selected = policy.select("server", ["pom.xml"]) - self.assertIn("dependency_license", selected) - expected = {"server_memory", "server_rocksdb", "commons", "pd_store", "cluster", "docker", "helm"} - self.assertEqual(expected, set(policy.suites("server", selected))) - plan = dict(self.plan(), selected=sorted(selected), expected=sorted(expected)) - results = {suite: {"result": "success"} for suite in expected} - results["plan"] = {"result": "success"} - for result in ["failure", "cancelled", "skipped", None]: - results["dependency_license"] = {"result": result} - self.assertEqual(expected, set(policy.gate(plan, results)["executed"])) - results["cluster"] = {"result": "failure"} - with self.assertRaisesRegex(ValueError, "cluster"): - policy.gate(plan, results) - - def test_incomplete_pr_event_never_becomes_a_push_plan(self): - event = {"pull_request": dict(self.live_pr(), number=7)} - cases = [] - for change in ("null_repo", "missing_repo", "missing_number", "empty_ref"): - altered = json.loads(json.dumps(event)) - pr = altered["pull_request"] - if change == "null_repo": - pr["head"]["repo"] = None - elif change == "missing_repo": - del pr["head"]["repo"] - elif change == "missing_number": - del pr["number"] - else: - pr["head"]["ref"] = "" - cases.append(altered) - cases.extend([{"pull_request": {}}, {"pull_request": None}]) - with patch.object(policy, "git", return_value="merge"): - for altered in cases: - with self.subTest(event=altered), self.assertRaises(policy.StaleInputError): - policy.create_plan('server', altered, 'apache/server', - lambda _: self.fail("Incomplete identity must fail before any API query")) - - def test_gate_refuses_failed_planner_even_empty(self): - plan = dict(self.plan(), expected=[], selected=[]) - for result in ["failure", "skipped", "cancelled", None]: - with self.subTest(result=result), self.assertRaises(ValueError): - policy.gate(plan, {"plan": {"result": result}}) - report = policy.gate(plan, {"plan": {"result": "success"}}) - self.assertEqual([], report["executed"]) - self.assertNotIn("proofs", report) - - def test_cumulative_source_and_docs_always_run_current_tests_without_historical_apis(self): + self.assertTrue(self.plan_for([path])["security"]) + for path in [policy.SERVER_CORE + "api/Author.java", "hugegraph-pd/hg-pd-core/src/main/A.java", + "docker/Dockerfile", "helm/templates/service.yaml"]: + self.assertFalse(self.plan_for([path])["security"]) + + def test_documentation_and_nested_readmes_skip_every_test_lane(self): + paths = ["README.md", "CONTRIBUTING.md", "notes.txt", "LICENSE", "NOTICE", "docs/ci.md", + ".github/outdated/APACHE_PROPOSAL.md", "hugegraph-pd/design.md", + "hugegraph-pd/docs/raft.md", "hugegraph-store/docs/guide.txt", "hugegraph-server/hugegraph-hstore/README.md", + "hugegraph-server/nested/component/README.md", "hugegraph-pd/hg-pd-core/AGENTS.md", "docker/README.md"] + for event_name, event in [("pull_request", {}), ("push", {"ref": "refs/heads/master"})]: + plan = self.plan_for(paths, event_name, event) + self.assertEqual([], plan["selected"]) + self.assertEqual([], plan["expected"]) + self.assertFalse(any(plan[module] for module in policy.MODULES["server"])) + for path in ["hugegraph-server/hugegraph-test/src/test/resources/README.md", "docs/example.java", + "docs/config.yml", "hugegraph-pd/docs/fixtures/example.txt", "helm/templates/README.md", + "hugegraph-store/hg-store-dist/src/assembly/static/conf/startup.txt"]: + with self.subTest(path=path): + self.assertTrue(policy.select("server", [path])) + + def test_master_push_runs_smoke_without_repeated_full_suites(self): + event = {"before": "base", "ref": "refs/heads/master"} + generic = self.plan_for(["hugegraph-server/hugegraph-core/src/main/A.java"], "push", event) + self.assertEqual({"macos", "distributed_smoke"}, set(generic["selected"])) + pd = self.plan_for(["hugegraph-pd/hg-pd-core/src/main/A.java"], "push", event) + self.assertEqual({"distributed_smoke"}, set(pd["selected"])) + for path in ["hugegraph-server/hugegraph-hbase/src/main/A.java", ".github/workflows/riscv64-ci.yml", + ".github/workflows/codeql-analysis.yml"]: + self.assertEqual([], self.plan_for([path], "push", event)["expected"]) + native = self.plan_for(["hugegraph-server/hugegraph-rocksdb/src/main/A.java"], "push", event) + self.assertTrue(native["macos"]) + self.assertFalse(native["macos_full"]) + self.assertTrue(self.plan_for(["docker/Dockerfile"], "push", event)["docker"]) + + def test_manual_presets_weekly_and_release_branches(self): + for preset, selected in policy.PRESETS.items(): + with self.subTest(preset=preset): + plan = self.plan_for([], "workflow_dispatch", {"inputs": {"test-set": preset}}) + self.assertEqual(selected, set(plan["selected"])) + default = self.plan_for([], "workflow_dispatch") + self.assertEqual(policy.MAINLINE, set(default["selected"])) + self.assertFalse(default["macos_full"]) + weekly = self.plan_for([], "schedule") + self.assertEqual(set(policy.MODULES["server"]), set(weekly["selected"])) + self.assertTrue(weekly["macos_full"]) + for branch in ["release-1.7", "test-ci", "tinkerpop-3.8"]: + plan = self.plan_for(["hugegraph-server/hugegraph-core/src/main/A.java"], "push", + {"before": "base", "ref": "refs/heads/" + branch}) + self.assertTrue(plan["server"]) + self.assertFalse(plan["hbase"]) + self.assertFalse(plan["riscv64"]) + + def test_unknown_inputs_fall_back_to_mainline_without_expensive_noncore(self): + for path in [".github/scripts/ci-policy.py", ".mvn/maven.config", "unknown"]: + self.assertEqual(policy.MAINLINE, policy.select("server", [path])) + with patch.object(policy, "git", side_effect=lambda *args: "merge" if args[0] == "rev-parse" else + (_ for _ in ()).throw(subprocess.CalledProcessError(1, "git"))): + plan = policy.create_plan("server", {"before": "base"}, "apache/server", event_name="push") + self.assertEqual(policy.MAINLINE, set(plan["selected"])) + self.assertFalse(plan["macos_full"]) + self.assertFalse(plan["hbase"]) + self.assertFalse(plan["riscv64"]) + + def test_deployment_and_specific_workflows_keep_specific_owners(self): + for path in [".github/workflows/docker-build-ci.yml", ".github/scripts/check-docker-images.sh", + ".github/scripts/docker-deployment.py", ".github/scripts/test_docker_deployment.py", + "hugegraph-server/hugegraph-dist/docker/Dockerfile"]: + self.assertEqual({"docker"}, policy.select("server", [path])) + for path in [".asf.yaml", ".github/dependabot.yml", ".editorconfig"]: + self.assertEqual(set(), policy.select("server", [path])) + for module in ["pd", "store"]: + selected = policy.select("server", [f"hugegraph-{module}/hg-{module}-dist/src/assembly/static/bin/start.sh"]) + self.assertTrue({module, "hstore", "cluster", "docker", "distributed_smoke"}.issubset(selected)) + self.assertNotIn("server", selected) + version = "hugegraph-commons/hugegraph-common/src/main/resources/version.properties" + self.assertTrue({"commons", "docker"}.issubset(policy.select("server", [version]))) + + def test_toolchain_selection_and_fixture_gate_contract_remain_compatible(self): + quiet = patch("builtins.print") + quiet.start() + self.addCleanup(quiet.stop) + self.assertEqual({"client", "loader", "tools", "spark", "hubble"}, policy.select( + "toolchain", ["hugegraph-client/src/A.java"])) + self.assertEqual({"loader", "hubble"}, policy.select("toolchain", ["hugegraph-loader/src/A.java"])) + self.assertEqual(set(policy.MODULES["toolchain"]), policy.select("toolchain", ["hugegraph-client/api.proto"])) + self.assertIn("go", policy.select("toolchain", ["hugegraph-client/assembly/travis/start-hugegraph-servers.sh"])) + plan = {"schema": 1, "repository": "apache/toolchain", "project": "toolchain", "pr": 0, + "source": "apache/toolchain", "branch": "master", "base": "base", "head": "head", + "testedMergeSHA": "head", "selected": ["client"], "expected": ["client"]} + results = {"plan": {"result": "success"}, "client": {"result": "success"}, "fixture": {"result": "success"}} + self.assertEqual(["client"], policy.gate(plan, results, lambda _: self.fail("No post-run API query"))["executed"]) + for state in ["failure", "cancelled", "skipped", None]: + for job in ["plan", "client", "fixture"]: + with self.subTest(state=state, job=job), self.assertRaises(policy.GateFailure): + policy.gate(plan, dict(results, **{job: {"result": state}})) + + def test_toolchain_gate_keeps_summary_current_pr_and_cli_all(self): + plan = {"schema": 1, "repository": "apache/toolchain", "project": "toolchain", "pr": 7, + "source": "alice/toolchain", "branch": "feature", "base": "base", "head": "head", + "testedMergeSHA": "merge", "selected": ["client"], "expected": ["client"]} + live = {"state": "open", "head": {"sha": "head", "ref": "feature", "repo": {"full_name": "alice/toolchain"}}, + "base": {"repo": {"full_name": "apache/toolchain"}}} + results = {"plan": {"result": "success"}, "client": {"result": "success"}, "fixture": {"result": "success"}} + with tempfile.TemporaryDirectory() as directory, patch("builtins.print"): + root = Path(directory) + summary = root / "summary.md" + with patch.dict(os.environ, {"GITHUB_STEP_SUMMARY": str(summary)}): + policy.gate(plan, results, lambda _: live) + self.assertIn("| client | yes | advisory | affected inputs | success |", summary.read_text()) + with self.assertRaises(policy.StaleInputError): + policy.gate(plan, results, lambda _: dict(live, head=dict(live["head"], sha="new-head"))) + (root / "plan.json").write_text(json.dumps(dict(plan, pr=0))) + (root / "results.json").write_text(json.dumps(results)) + outcome = subprocess.run(["python3", str(Path(policy.__file__).resolve()), "gate", "--mode", "all", + "--plan", str(root / "plan.json"), "--results", str(root / "results.json"), + "--output", str(root / "report.json")], capture_output=True, text=True) + self.assertEqual(0, outcome.returncode, outcome.stderr) + self.assertEqual(["client"], json.loads((root / "report.json").read_text())["executed"]) + + def test_summary_has_only_advisory_checks(self): + with tempfile.TemporaryDirectory() as directory, patch.dict( + os.environ, {"GITHUB_STEP_SUMMARY": str(Path(directory) / "summary.md")}): + plan = self.plan_for(["pom.xml"]) + summary = (Path(directory) / "summary.md").read_text() + self.assertIn("| server_memory | yes | advisory |", summary) + self.assertNotIn("| required |", summary) + self.assertEqual([], plan["required"]) + self.assertEqual(1, summary.count("| dependency_license |")) + + def coordinator_jobs(self, plan, selection="success", java="success", result="success", cancelled=False): + workflow = Path(__file__).resolve().parents[1] / "workflows/server-ci.yml" + text = workflow.read_text().split("\njobs:\n", 1)[1] + jobs = {} + for match in re.finditer(r"^ ([\w-]+):\n(.*?)(?=^ [\w-]+:\n|\Z)", text, re.M | re.S): + name, body = match.groups() + if name == "preflight": + continue + condition = re.search(r"^ if: \$\{\{ (.+) \}\}$", body, re.M) + self.assertIsNotNone(condition, "Caller needs an explicit cancellation/selection condition: " + name) + expression = condition.group(1) + # GitHub otherwise injects success(), which would hide tests after a license failure. + self.assertIn("cancelled()", expression) + outputs = {key: str(value).lower() for key, value in plan.items() if isinstance(value, bool)} + outputs.update(selection_result=selection, java_result=java) + expression = re.sub(r"needs\.preflight\.outputs\.([\w]+)", + lambda match: repr(outputs.get(match.group(1), "")), expression) + expression = expression.replace("needs.preflight.result", repr(result)) + expression = expression.replace("!cancelled()", "not cancelled") + expression = expression.replace("&&", "and").replace("||", "or") + jobs[name] = bool(eval(expression, {"__builtins__": {}}, {"cancelled": cancelled})) + return {job for job, run in jobs.items() if run} + + def test_real_coordinator_keeps_selected_jobs_after_license_or_selftest_failure(self): + workflow = (Path(__file__).resolve().parents[1] / "workflows/server-ci.yml").read_text() + preflight = re.search(r"^ preflight:\n(.*?)(?=^ [\w-]+:|\Z)", workflow, re.M | re.S).group(1) + self.assertEqual("check-license", re.search(r"^ name: (.+)$", preflight, re.M).group(1)) + full = self.plan_for([], "workflow_dispatch", {"inputs": {"test-set": "full"}}) + expected = {"build-server", "server-rocksdb", "commons", "pd_store", "cluster", "docker", "helm", + "dependency_license", "compatibility", "codeql"} + self.assertEqual(expected, self.coordinator_jobs(full)) + self.assertEqual(expected, self.coordinator_jobs(full, result="failure")) + docs = self.plan_for(["hugegraph-pd/docs/guide.txt", "docker/README.md"]) + self.assertEqual(set(), self.coordinator_jobs(docs, result="failure")) + source = self.plan_for(["hugegraph-server/hugegraph-core/src/main/A.java"]) + self.assertEqual({"build-server", "server-rocksdb", "compatibility"}, self.coordinator_jobs(source)) + master = self.plan_for(["hugegraph-server/hugegraph-core/src/main/A.java"], "push", + {"before": "base", "ref": "refs/heads/master"}) + self.assertEqual({"docker", "compatibility"}, self.coordinator_jobs(master)) + + def test_real_coordinator_blocks_selection_runtime_failure_and_cancellation(self): + full = self.plan_for([], "schedule") + for state in ["failure", "cancelled", "skipped", ""]: + with self.subTest(selection=state): + self.assertEqual(set(), self.coordinator_jobs(full, selection=state, result="failure")) + with self.subTest(java=state): + self.assertEqual({"helm"}, self.coordinator_jobs(full, java=state, result="failure")) + self.assertEqual(set(), self.coordinator_jobs(full, cancelled=True)) + + def test_legacy_memory_bridge_requires_real_results_and_retires_without_a_runner(self): + docs = self.plan_for(["README.md"]) + self.assertEqual(set(), self.coordinator_jobs(docs)) + docs["legacy_memory_required"] = True + self.assertEqual({"legacy-server-memory"}, self.coordinator_jobs(docs)) + self.assertEqual(set(), self.coordinator_jobs(docs, cancelled=True)) + workflow = (Path(__file__).resolve().parents[1] / "workflows/server-ci.yml").read_text() + bridge = re.search(r"^ legacy-server-memory:\n(.*?)(?=^ [\w-]+:|\Z)", workflow, re.M | re.S).group(1) + self.assertEqual("Server memory tests", re.search(r"^ name: (.+)$", bridge, re.M).group(1)) + script = textwrap.dedent(bridge.split(" run: |\n", 1)[1]) + cases = [("success", "true", "success", "success", True), + ("success", "false", "success", "skipped", True), + ("failure", "false", "success", "skipped", False), + ("success", "", "success", "skipped", False), + ("success", "true", "failure", "skipped", False)] + cases += [("success", "true", "success", result, False) + for result in ["failure", "cancelled", "skipped", ""]] + for selection, selected, java, memory, passes in cases: + with self.subTest(selection=selection, selected=selected, java=java, memory=memory): + result = subprocess.run(["bash", "-eo", "pipefail", "-c", script], capture_output=True, + env={**os.environ, "SELECTION_RESULT": selection, "SERVER_SELECTED": selected, + "JAVA_RESULT": java, "MEMORY_RESULT": memory}) + self.assertEqual(passes, result.returncode == 0, result.stderr) + + def test_contract_scripts_reach_their_actual_inner_jobs_and_steps(self): + workflows = Path(__file__).resolve().parents[1] / "workflows" + cases = [("test-stop-hugegraph-store.sh", {"struct"}), + ("test-check-jacoco-report.sh", {"struct"}), + ("test-codecov-upload-config.sh", {"struct"}), + ("test-riscv64-java-runtime.sh", {"server", "riscv64"})] + for name, suites in cases: + script = policy.TRAVIS + name + with self.subTest(script=script): + plan = self.plan_for([script]) + self.assertEqual(suites, set(plan["selected"])) + callers = self.coordinator_jobs(plan) + if "struct" in suites: + self.assertEqual({"pd_store"}, callers) + root = (workflows / "server-ci.yml").read_text() + caller = re.search(r"^ pd_store:\n(.*?)(?=^ [\w-]+:|\Z)", root, re.M | re.S).group(1) + inputs = {} + for key in ["struct", "store"]: + binding = re.search(r"^ run-" + key + r": \$\{\{ (.+) \}\}$", caller, re.M).group(1) + binding = binding.replace("needs.preflight.outputs." + key, repr(str(plan[key]).lower())) + inputs[key] = eval(binding, {"__builtins__": {}}) + workflow = (workflows / "pd-store-ci.yml").read_text() + struct = workflow.split("\n struct:\n", 1)[1].split("\n pd:\n", 1)[0] + condition = re.search(r"^ if: (.+)$", struct, re.M).group(1) + condition = condition.replace("inputs.run-struct", str(inputs["struct"])) + condition = condition.replace("inputs.run-store", str(inputs["store"])) + self.assertTrue(eval(condition.replace("||", "or"), {"__builtins__": {}})) + step = next(step for step in struct.split("\n - ") if script in step) + self.assertNotRegex(step, r"(?m)^ if:") + else: + self.assertIn("server-rocksdb", callers) + workflow = (workflows / "server-ci.yml").read_text() + # Read the backend from the actual caller rather than assuming a preset. + caller = re.search(r"^ server-rocksdb:\n(.*?)(?=^ [\w-]+:|\Z)", + workflow, re.M | re.S).group(1) + backend = re.search(r"^ backend: (\w+)$", caller, re.M).group(1) + inner = (workflows / "server-tests.yml").read_text() + step = next(step for step in inner.split("\n - ") if Path(script).name in step) + condition = re.search(r"^ if: (.+)$", step, re.M).group(1) + self.assertTrue(eval(condition.replace("inputs.backend", repr(backend)), + {"__builtins__": {}})) + mixed = self.plan_for([policy.TRAVIS + "test-stop-hugegraph-store.sh", "hugegraph-pd/src/A.java"]) + self.assertTrue({"struct", "pd", "store", "hstore", "cluster"}.issubset(mixed["selected"])) + + def test_coordinator_reads_and_validates_real_java_input(self): + workflow = (Path(__file__).resolve().parents[1] / "workflows/server-ci.yml").read_text() + code = workflow.split("python3 - <<'PYTHON'\n", 1)[1].split("\n PYTHON", 1)[0] with tempfile.TemporaryDirectory() as directory: root = Path(directory) - def git(*args): - return subprocess.check_output(["git", *args], cwd=root, text=True).strip() - git("init", "-q") - git("config", "user.email", "ci@example.invalid") - git("config", "user.name", "CI") - (root / "README.md").write_text("base") - git("add", ".") - git("commit", "-qm", "base") - base = git("rev-parse", "HEAD") - git("checkout", "-qb", "feature") - (root / "hugegraph-server").mkdir() - source = root / "hugegraph-server/A.java" - source.write_text("source A") - git("add", ".") - git("commit", "-qm", "source") - for text in ["docs B after test failure", "docs C after claimed old success"]: - (root / "README.md").write_text(text) - git("commit", "-qam", "docs") - head = git("rev-parse", "HEAD") - # Synthetic PR merge; a different tree with the same parents also exists. - tree = git("rev-parse", "HEAD^{tree}") - merge = git("commit-tree", tree, "-p", base, "-p", head, "-m", "PR merge") - false_merge = git("commit-tree", git("rev-parse", base + "^{tree}"), - "-p", base, "-p", head, "-m", "unexecuted tree") - self.assertNotEqual(merge, false_merge) - git("checkout", "--detach", "-q", merge) - live = {"state": "open", "head": {"sha": head, "ref": "feature", "repo": {"full_name": "alice/server"}}, - "base": {"sha": base, "repo": {"full_name": "apache/server"}}} - event = {"pull_request": dict(live, number=7)} - calls = [] - def fetch(path): - calls.append(path) - self.assertEqual("repos/apache/server/pulls/7", path) - return live - old = os.getcwd() - try: - os.chdir(root) - os.environ["GITHUB_SHA"] = merge - plan = policy.create_plan("server", event, "apache/server", fetch) - git("checkout", "--detach", "-q", false_merge) - with self.assertRaises(policy.StaleInputError): - policy.create_plan("server", event, "apache/server", fetch) - git("checkout", "--detach", "-q", merge) - finally: - os.chdir(old) - self.assertEqual(["repos/apache/server/pulls/7"], calls) - self.assertTrue(plan["server"]) - self.assertTrue(plan["pd_store"]) - self.assertTrue(plan["cluster"]) - self.assertTrue(plan["compatibility"]) - self.assertTrue(plan["security"]) - self.assertNotIn("reused", plan) - results = {suite: {"result": "success"} for suite in plan["expected"]} - results["plan"] = {"result": "success"} - results["server_memory"] = {"result": "skipped"} - with self.assertRaisesRegex(ValueError, "server_memory"): - policy.gate(plan, results) - git("checkout", "-q", "feature") - - def test_queued_pr_uses_actual_merge_base_and_rejects_changed_identity(self): + java = root / ".github/workflows/.java-version" + java.parent.mkdir(parents=True) + output = root / "outputs.txt" + for version in ["17", "17.0.20", "21.0.1", "", "bad", '17\n21']: + java.write_text(version) + result = subprocess.run(["python3", "-c", textwrap.dedent(code)], cwd=root, + env={**os.environ, "GITHUB_OUTPUT": str(output)}, capture_output=True, text=True) + with self.subTest(version=version): + self.assertEqual(version in {"17", "17.0.20", "21.0.1"}, result.returncode == 0, result.stderr) + self.assertEqual(['versions=["17"]', 'versions=["17.0.20"]', 'versions=["21.0.1"]'], + output.read_text().splitlines()) + + def test_real_git_document_modes_renames_and_cli_outputs(self): with tempfile.TemporaryDirectory() as directory: root = Path(directory) - def git(*args): - return subprocess.check_output(["git", *args], cwd=root, text=True).strip() - git("init", "-q") - git("config", "user.email", "ci@example.invalid") - git("config", "user.name", "CI") - (root / "README.md").write_text("base") - git("add", ".") - git("commit", "-qm", "event base") - event_base = git("rev-parse", "HEAD") - git("checkout", "-qb", "feature") - (root / "hugegraph-server").mkdir() - (root / "hugegraph-server/A.java").write_text("PR source") - git("add", ".") - git("commit", "-qm", "PR head") - head = git("rev-parse", "HEAD") - event_live = self.live_pr() - event_live["head"]["sha"], event_live["base"]["sha"] = head, event_base - event = {"pull_request": dict(event_live, number=7)} - git("checkout", "--detach", "-q", event_base) - (root / "README.md").write_text("target advanced while run was queued") - git("commit", "-qam", "new target base") - tested_base = git("rev-parse", "HEAD") - git("merge", "--no-ff", "-qm", "current PR merge", "feature") - merge = git("rev-parse", "HEAD") - live = dict(event_live, base=dict(event_live["base"], sha=tested_base)) - old = os.getcwd() - try: - os.chdir(root) - os.environ["GITHUB_SHA"] = merge - plan = policy.create_plan("server", event, "apache/server", lambda _: live) - self.assertEqual(tested_base, plan["base"]) - self.assertNotEqual(event_base, plan["base"]) - self.assertEqual(head, plan["head"]) - self.assertEqual(merge, plan["testedMergeSHA"]) - self.assertEqual(["hugegraph-server/A.java"], plan["changedPaths"]) - self.assertIn("server_memory", plan["expected"]) - results = {suite: {"result": "success"} for suite in plan["expected"]} - results["plan"] = {"result": "success"} - self.assertEqual(plan["expected"], policy.gate(plan, results, lambda _: live)["executed"]) - changed = dict(live, head=dict(live["head"], sha=tested_base)) - with self.assertRaises(policy.StaleInputError): - policy.create_plan("server", event, "apache/server", lambda _: changed) - with self.assertRaises(policy.StaleInputError): - policy.gate(plan, results, lambda _: changed) - wrong_merge = git("commit-tree", git("rev-parse", merge + "^{tree}"), - "-p", tested_base, "-p", event_base, "-m", "wrong PR head") - unrelated_base = git("commit-tree", git("rev-parse", tested_base + "^{tree}"), - "-m", "unrelated target history") - unrelated_merge = git("commit-tree", git("rev-parse", merge + "^{tree}"), - "-p", unrelated_base, "-p", head, "-m", "unrelated target merge") - for checkout in [wrong_merge, head, unrelated_merge]: - git("checkout", "--detach", "-q", checkout) - with self.subTest(checkout=checkout), self.assertRaises(policy.StaleInputError): - policy.create_plan("server", event, "apache/server", lambda _: live) - finally: - os.chdir(old) + fixture = GitFixture(root) + base = fixture.commit("README.md", "base", "base") + fixture.commit("README.md", "docs", "docs") + self.assertEqual([], fixture.plan({"before": base, "ref": "refs/heads/master"})["expected"]) + (root / "README.md").chmod(0o755) + fixture.git("add", ".") + fixture.git("commit", "-qm", "executable prose") + unsafe = fixture.plan({"before": base, "ref": "refs/heads/feature"}) + self.assertEqual(policy.MAINLINE, set(unsafe["selected"])) + (root / "README.md").unlink() + (root / "README.md").symlink_to("target.txt") + fixture.git("add", ".") + fixture.git("commit", "-qm", "symlink prose") + self.assertEqual(policy.MAINLINE, set(fixture.plan({"before": base})["selected"])) + before = fixture.commit("hugegraph-server/A.java", "source", "source") + (root / "docs").mkdir() + fixture.git("mv", "hugegraph-server/A.java", "docs/renamed.md") + fixture.git("commit", "-qm", "rename source to prose") + self.assertTrue(fixture.plan({"before": before})["server"]) + event = root / "event.json" + event.write_text("{}") + output = root / "outputs.txt" + subprocess.run(["python3", str(Path(policy.__file__).resolve()), "plan", "--project", "server", + "--repository", "apache/server", "--event-path", str(event), "--event-name", "schedule", + "--output", str(root / "plan.json")], cwd=root, check=True, + env={**os.environ, "GITHUB_OUTPUT": str(output)}) + self.assertIn("macos_full=true\n", output.read_text()) + self.assertIn("pd_store=true\n", output.read_text()) + self.assertNotIn("compatibility=", output.read_text()) + self.assertEqual(set(policy.MODULES["server"]), set(json.loads((root / "plan.json").read_text())["selected"])) - def test_pure_docs_pr_and_metadata_fail_closed(self): - live = self.live_pr() - event = {"pull_request": dict(live, number=7)} + def test_cumulative_pr_snapshot_keeps_source_after_document_commit_and_checks_current_head(self): + with tempfile.TemporaryDirectory() as directory: + fixture = GitFixture(Path(directory)) + base = fixture.commit("README.md", "base", "base") + fixture.git("checkout", "-qb", "feature") + fixture.commit("hugegraph-server/A.java", "source", "source") + head = fixture.commit("README.md", "docs after source", "docs") + merge = fixture.git("commit-tree", fixture.git("rev-parse", "HEAD^{tree}"), + "-p", base, "-p", head, "-m", "PR merge") + fixture.git("checkout", "--detach", "-q", merge) + live = {"number": 7, "state": "open", "head": {"sha": head, "ref": "feature", + "repo": {"full_name": "alice/server"}}, "base": {"sha": base, "repo": {"full_name": "apache/server"}}} + calls = [] + def fetch(path): + calls.append(path) + return live + plan = fixture.plan({"pull_request": live}, fetch) + self.assertEqual(["repos/apache/server/pulls/7"], calls) + self.assertTrue(plan["server"]) + self.assertFalse(plan["pd_store"]) + self.assertFalse(plan["security"]) + self.assertIn("hugegraph-server/A.java", plan["changedPaths"]) + for section, field, value in [("head", "sha", "new-head"), ("head", "ref", "other"), + ("head", "repo", {"full_name": "other/fork"}), + ("base", "repo", {"full_name": "other/target"})]: + changed = dict(live, **{section: dict(live[section], **{field: value})}) + with self.subTest(section=section, field=field), self.assertRaises(policy.StaleInputError): + fixture.plan({"pull_request": live}, lambda _: changed) + with self.assertRaises(policy.StaleInputError): + fixture.plan({"pull_request": live}, lambda _: dict(live, state="closed")) + fixture.git("checkout", "--detach", "-q", head) + with self.assertRaises(policy.StaleInputError): + fixture.plan({"pull_request": live}, fetch) + + def test_queued_merge_uses_actual_target_parent_and_api_outage_keeps_pr_identity(self): + live = {"number": 7, "state": "open", "head": {"sha": "head", "ref": "feature", + "repo": {"full_name": "alice/server"}}, "base": {"sha": "event-base", "repo": {"full_name": "apache/server"}}} def git(*args): if args[0] == "show": - return "base head" + return "current-base head" if args[0] == "diff": - return "README.md\ndocs/ci.md" + return "README.md" return "merge" - with patch.object(policy, "git", side_effect=git), patch.object(policy, "unsafe_documentation", return_value=False): - plan = policy.create_plan("server", event, "apache/server", lambda p: live) + with patch.dict(os.environ, {"GITHUB_SHA": "merge", "GITHUB_EVENT_NAME": ""}), patch.object( + policy, "git", side_effect=git), patch.object(policy, "unsafe_documentation", return_value=False): + plan = policy.create_plan("server", {"pull_request": live}, "apache/server", lambda _: live) + self.assertEqual("current-base", plan["base"]) self.assertEqual([], plan["expected"]) - self.assertFalse(plan["server"]) - self.assertFalse(plan["security"]) - advanced = dict(live, base={"sha": "new-base", "repo": {"full_name": "apache/server"}}) - self.assertFalse(policy.create_plan("server", event, "apache/server", lambda p: advanced)["server"]) - with patch.object(policy, "git", side_effect=lambda *a: "wrong parents" if a[0] == "show" else git(*a)): - with self.assertRaises(policy.StaleInputError): - policy.create_plan("server", event, "apache/server", lambda p: live) - - def test_api_failure_runs_full_required_without_losing_pr_identity(self): - event = {"pull_request": dict(self.live_pr(), number=7)} - def git(*args): - return "base head" if args[0] == "show" else "merge" - def fail(path): - raise subprocess.CalledProcessError(1, "gh") - with patch.object(policy, "git", side_effect=git): - plan = policy.create_plan("server", event, "apache/server", fail) - self.assertEqual(set(policy.MODULES["server"]), set(plan["selected"])) - self.assertEqual(plan["reason"], policy.selection_reason(plan, "dependency_license")) - self.assertEqual((7, "alice/server", "feature", "base", "head"), - tuple(plan[key] for key in ["pr", "source", "branch", "base", "head"])) - results = {suite: {"result": "success"} for suite in plan["expected"]} - results.update(plan={"result": "success"}, fixture={"result": "success"}, - **{"hubble-fixture": {"result": "success"}}) - with self.assertRaises(subprocess.CalledProcessError): - policy.gate(plan, results, fail) - self.assertEqual(plan["expected"], policy.gate(plan, results)["executed"]) - self.api_mock.assert_called_once_with("repos/apache/server/pulls/7") - - def test_real_pr_git_snapshot_and_final_gate_freshness(self): - with tempfile.TemporaryDirectory() as directory: - root = Path(directory) - def git(*args): - return subprocess.check_output(["git", *args], cwd=root, text=True).strip() - git("init", "-q") - git("config", "user.email", "ci@example.invalid") - git("config", "user.name", "CI") - (root / "README.md").write_text("base") - git("add", ".") - git("commit", "-qm", "base") - base = git("rev-parse", "HEAD") - git("checkout", "-qb", "feature") - (root / "hugegraph-client").mkdir() - (root / "hugegraph-client/A.java").write_text("source") - git("add", ".") - git("commit", "-qm", "source") - head = git("rev-parse", "HEAD") - tree = git("rev-parse", "HEAD^{tree}") - merge = git("commit-tree", tree, "-p", base, "-p", head, "-m", "PR merge") - advanced_head = git("commit-tree", tree, "-p", head, "-m", "new source head") - advanced_base = git("commit-tree", git("rev-parse", base + "^{tree}"), - "-p", base, "-m", "new target base") - git("checkout", "--detach", "-q", merge) - live = self.live_pr() - live["head"]["sha"], live["base"]["sha"] = head, base - event = {"pull_request": dict(live, number=7)} - old = os.getcwd() - try: - os.chdir(root) - os.environ["GITHUB_SHA"] = merge - plan = policy.create_plan("server", event, "apache/server", lambda _: live) - results = {suite: {"result": "success"} for suite in plan["expected"]} - results.update(plan={"result": "success"}, fixture={"result": "success"}, - **{"hubble-fixture": {"result": "success"}}) - self.assertEqual(plan["expected"], policy.gate(plan, results, lambda _: live)["executed"]) - for section, field, value in [("head", "sha", advanced_head), - ("head", "ref", "other-branch"), - ("head", "repo", {"full_name": "other/fork"}), - ("base", "repo", {"full_name": "other/target"})]: - changed = dict(live, **{section: dict(live[section], **{field: value})}) - with self.subTest(section=section, field=field): - with self.assertRaises(policy.StaleInputError): - policy.create_plan("server", event, "apache/server", lambda _: changed) - with self.assertRaises(policy.StaleInputError): - policy.gate(plan, results, lambda _: changed) - advanced_merge = git("commit-tree", tree, "-p", advanced_base, "-p", head, - "-m", "PR merge after master advanced") - git("checkout", "--detach", "-q", advanced_merge) - os.environ["GITHUB_SHA"] = advanced_merge - advanced_plan = policy.create_plan("server", event, "apache/server", lambda _: live) - self.assertEqual(base, event["pull_request"]["base"]["sha"]) - self.assertEqual(advanced_base, advanced_plan["base"]) - self.assertEqual(advanced_merge, advanced_plan["testedMergeSHA"]) - self.assertEqual(plan["changedPaths"], advanced_plan["changedPaths"]) - policy.gate(advanced_plan, results, lambda _: live) - with patch.dict(os.environ, {"GITHUB_SHA": ""}): - with self.assertRaises(policy.StaleInputError): - policy.create_plan("server", event, "apache/server", lambda _: live) - with self.assertRaises(policy.StaleInputError): - policy.gate(plan, results, lambda _: dict(live, state="closed")) - git("checkout", "--detach", "-q", head) - with self.assertRaises(policy.StaleInputError): - policy.create_plan("server", event, "apache/server", lambda _: live) - finally: - os.chdir(old) - - def test_push_gate_does_not_query_pr(self): - plan = dict(self.plan(), pr=0, expected=[], selected=[]) - policy.gate(plan, {"plan": {"result": "success"}}, - lambda _: self.fail("push gate must not query a PR")) + def unavailable(_): + raise subprocess.CalledProcessError(1, "gh") + fallback = policy.create_plan("server", {"pull_request": live}, "apache/server", unavailable) + self.assertEqual(policy.MAINLINE, set(fallback["selected"])) + self.assertEqual((7, "alice/server", "feature", "current-base", "head"), + tuple(fallback[key] for key in ["pr", "source", "branch", "base", "head"])) + for malformed in [{"pull_request": None}, {"pull_request": {}}, + {"pull_request": dict(live, head=dict(live["head"], repo=None))}]: + with self.subTest(event=malformed), self.assertRaises(policy.StaleInputError): + policy.create_plan("server", malformed, "apache/server", lambda _: self.fail("Invalid identity")) if __name__ == "__main__": diff --git a/.github/workflows/check-dependencies.yml b/.github/workflows/check-dependencies.yml index db2ec66aa0..71da915301 100644 --- a/.github/workflows/check-dependencies.yml +++ b/.github/workflows/check-dependencies.yml @@ -3,14 +3,13 @@ name: "3rd-party" on: workflow_call: - workflow_dispatch: permissions: contents: read jobs: dependency-check: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 env: USE_STAGE: 'false' # Whether to include the stage repository. SCRIPT_DEPENDENCY: install-dist/scripts/dependency @@ -53,12 +52,12 @@ jobs: "$SCRIPT_DEPENDENCY/known-dependencies.txt" "$PWD/target/dependency/current-dependencies.txt" dependency-review: - runs-on: ubuntu-latest + if: github.event_name == 'pull_request' + runs-on: ubuntu-24.04 steps: - name: 'Checkout Repository' uses: actions/checkout@v7 - name: 'Dependency Review' - if: github.event_name == 'pull_request' uses: actions/dependency-review-action@v5 # Refer: https://github.com/actions/dependency-review-action with: diff --git a/.github/workflows/cluster-test-ci.yml b/.github/workflows/cluster-test-ci.yml index 09ac85f193..29c7c5bf66 100644 --- a/.github/workflows/cluster-test-ci.yml +++ b/.github/workflows/cluster-test-ci.yml @@ -2,7 +2,6 @@ name: "Cluster Test CI" on: workflow_call: - workflow_dispatch: permissions: contents: read @@ -10,7 +9,7 @@ permissions: jobs: cluster-test: timeout-minutes: 120 - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 env: USE_STAGE: 'false' # Whether to include the stage repository. diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index e0e0aedd42..3f9b906933 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -1,79 +1,36 @@ name: "CodeQL" on: - pull_request: - push: - branches: [master, 'release-*', 'test-*'] - workflow_dispatch: - schedule: - - cron: '33 0 * * 5' - -concurrency: - group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.run_id }}-${{ github.run_attempt == 1 && 'head' || format('rerun-{0}', github.run_id) }} - cancel-in-progress: ${{ github.event_name == 'pull_request' }} + workflow_call: + inputs: + enabled: + type: boolean + default: true permissions: contents: read - pull-requests: read actions: read security-events: write jobs: - plan: - runs-on: ubuntu-24.04 - outputs: - server: ${{ steps.select.outputs.server }} - commons: ${{ steps.select.outputs.commons }} - pd_store: ${{ steps.select.outputs.pd_store }} - cluster: ${{ steps.select.outputs.cluster }} - docker: ${{ steps.select.outputs.docker }} - helm: ${{ steps.select.outputs.helm }} - dependency_license: ${{ steps.select.outputs.dependency_license }} - compatibility: ${{ steps.select.outputs.compatibility }} - smoke: ${{ steps.select.outputs.smoke }} - security: ${{ steps.select.outputs.security }} - steps: - - uses: actions/checkout@v7 - with: - fetch-depth: 0 - persist-credentials: false - - name: Select affected tests - id: select - env: - GH_TOKEN: ${{ github.token }} - run: >- - python3 .github/scripts/ci-policy.py plan --project server - --event-path "$GITHUB_EVENT_PATH" --repository "$GITHUB_REPOSITORY" - --output plan.json - - uses: actions/upload-artifact@v4 - with: - name: ci-plan - path: plan.json - retention-days: 7 - overwrite: true - analyze: - needs: plan - if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || needs.plan.outputs.security == 'true' + if: inputs.enabled env: USE_STAGE: 'false' # Whether to include the stage repository. # Preserve the master scan identity when invoked by a reusable-workflow caller. CODEQL_ACTION_ANALYSIS_KEY: .github/workflows/codeql-analysis.yml:analyze - name: Analyze (${{ matrix.language }}) - runs-on: ubuntu-latest + name: Analyze (java) + runs-on: ubuntu-24.04 permissions: actions: read contents: read security-events: write - strategy: - fail-fast: false - matrix: - language: [ 'java' ] - steps: - name: Checkout repository uses: actions/checkout@v7 + with: + persist-credentials: false - name: Setup Java JDK uses: actions/setup-java@v6 @@ -91,7 +48,7 @@ jobs: - name: Initialize CodeQL uses: github/codeql-action/init@v4 with: - languages: ${{ matrix.language }} + languages: java # If you wish to specify custom queries, you can do so here or in a config file. # By default, queries listed here will override any specified in a config file. # Prefix the list here with "+" to use these queries and those in the config file. @@ -116,4 +73,4 @@ jobs: - name: Perform CodeQL Analysis uses: github/codeql-action/analyze@v4 with: - category: .github/workflows/codeql-analysis.yml:analyze/language:${{ matrix.language }} + category: .github/workflows/codeql-analysis.yml:analyze/language:java diff --git a/.github/workflows/commons-ci.yml b/.github/workflows/commons-ci.yml index 63d86230a3..d84c937af4 100644 --- a/.github/workflows/commons-ci.yml +++ b/.github/workflows/commons-ci.yml @@ -2,29 +2,26 @@ name: "HugeGraph-Commons CI" on: workflow_call: - workflow_dispatch: + inputs: + java-versions: + type: string + required: true permissions: contents: read jobs: - java-runtime: - uses: ./.github/workflows/java-runtime.yml - permissions: - contents: read - build-commons: timeout-minutes: 20 name: Commons tests (Java ${{ matrix.JAVA_VERSION }}) - needs: java-runtime - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 env: USE_STAGE: 'false' # Whether to include the stage repository. strategy: fail-fast: false matrix: - JAVA_VERSION: ${{ fromJSON(needs.java-runtime.outputs.versions || '["unavailable"]') }} + JAVA_VERSION: ${{ fromJSON(inputs.java-versions) }} steps: - name: Checkout @@ -43,8 +40,8 @@ jobs: - name: Use staged maven repo settings if: ${{ env.USE_STAGE == 'true' }} run: | - cp $HOME/.m2/settings.xml /tmp/settings.xml - cp -vf .github/configs/settings.xml $HOME/.m2/settings.xml && cat $HOME/.m2/settings.xml + cp "$HOME/.m2/settings.xml" /tmp/settings.xml + cp -vf .github/configs/settings.xml "$HOME/.m2/settings.xml" && cat "$HOME/.m2/settings.xml" - name: Install run: | diff --git a/.github/workflows/docker-build-ci.yml b/.github/workflows/docker-build-ci.yml index 55a9045141..5ed8fd5acc 100644 --- a/.github/workflows/docker-build-ci.yml +++ b/.github/workflows/docker-build-ci.yml @@ -19,19 +19,37 @@ name: "Docker Build CI" on: workflow_call: - workflow_dispatch: - + inputs: + smoke-only: + type: boolean + default: false permissions: contents: read jobs: - docker-bake-check: - runs-on: ubuntu-latest + docker-build: + runs-on: ubuntu-24.04 + timeout-minutes: 90 + env: + IMAGE_TAG: ci-${{ github.run_id }}-${{ github.run_attempt }} + SOURCE_REVISION: ${{ github.sha }} steps: - name: Checkout + id: checkout uses: actions/checkout@v7 + - name: Install project JDK for deployment contracts + id: java + if: ${{ !cancelled() && steps.checkout.outcome == 'success' && !inputs.smoke-only }} + uses: actions/setup-java@v6 + with: + java-version-file: .github/workflows/.java-version + distribution: 'zulu' + cache: maven + cache-jdk: false + - name: Verify shared Maven stages stay identical + if: ${{ !cancelled() && steps.checkout.outcome == 'success' && !inputs.smoke-only }} shell: bash run: | set -euo pipefail @@ -66,6 +84,7 @@ jobs: done - name: Validate shared image build graph + if: ${{ !cancelled() && steps.checkout.outcome == 'success' && !inputs.smoke-only }} run: | set -euo pipefail @@ -88,58 +107,60 @@ jobs: all(. == "docker")) ' /tmp/hugegraph-bake.json - deployment-contracts: - runs-on: ubuntu-latest - timeout-minutes: 25 - steps: - - uses: actions/checkout@v7 - - uses: actions/setup-java@v6 - with: - java-version: '17' - distribution: 'zulu' - cache: maven - cache-jdk: false - - name: Build the pinned configuration reader - run: | - mvn -B -ntp -pl hugegraph-commons/hugegraph-common -am clean compile dependency:build-classpath \ - -Dmdep.outputFile=target/config-readback-classpath.txt -Dmaven.javadoc.skip=true -DskipTests - - name: Verify deployment shell contracts + - name: Verify deployment shell and Compose contracts + if: ${{ !cancelled() && steps.checkout.outcome == 'success' && steps.java.outcome == 'success' }} + timeout-minutes: 25 run: | - bash hugegraph-server/hugegraph-dist/docker/test/test-docker-entrypoint.sh - HG_CONFIG_TEST_CLASSPATH="$PWD/hugegraph-commons/hugegraph-common/target/classes" - HG_CONFIG_TEST_CLASSPATH+=":$(cat hugegraph-commons/hugegraph-common/target/config-readback-classpath.txt)" - export HG_CONFIG_TEST_CLASSPATH - bash hugegraph-server/hugegraph-dist/docker/docker-entrypoint-test.sh - bash hugegraph-server/hugegraph-dist/src/assembly/travis/test-check-port.sh \ - hugegraph-server/hugegraph-dist/src/assembly/static \ - hugegraph-store/hg-store-dist/src/assembly/static/bin/util.sh - - name: Verify Compose contracts - run: bash docker/test-compose.sh render - - docker-build: - runs-on: ubuntu-latest - timeout-minutes: 45 - env: - IMAGE_TAG: ci-${{ github.run_id }}-${{ github.run_attempt }} - SOURCE_REVISION: ${{ github.sha }} - steps: - - name: Checkout - uses: actions/checkout@v7 + set -euo pipefail + status=0 + if mvn -B -ntp -pl hugegraph-commons/hugegraph-common -am clean compile dependency:build-classpath \ + -Dmdep.outputFile=target/config-readback-classpath.txt -Dmaven.javadoc.skip=true -DskipTests; then + HG_CONFIG_TEST_CLASSPATH="$PWD/hugegraph-commons/hugegraph-common/target/classes" + HG_CONFIG_TEST_CLASSPATH+=":$(cat hugegraph-commons/hugegraph-common/target/config-readback-classpath.txt)" + export HG_CONFIG_TEST_CLASSPATH + if ! bash hugegraph-server/hugegraph-dist/docker/docker-entrypoint-test.sh; then + status=1 + fi + else + status=1 + fi + if ! bash hugegraph-server/hugegraph-dist/docker/test/test-docker-entrypoint.sh; then + status=1 + fi + if ! bash hugegraph-server/hugegraph-dist/src/assembly/travis/test-check-port.sh \ + hugegraph-server/hugegraph-dist/src/assembly/static \ + hugegraph-store/hg-store-dist/src/assembly/static/bin/util.sh; then + status=1 + fi + if ! bash docker/test-compose.sh render; then + status=1 + fi + exit "$status" - - name: Build and load all four PR images with a shared Maven stage + - name: Build current images and verify authentication, readiness and graph read/write + if: ${{ !cancelled() && steps.checkout.outcome == 'success' }} + timeout-minutes: 45 + env: + SMOKE_ONLY: ${{ inputs.smoke-only }} run: | + set -euo pipefail docker buildx create --driver docker-container --use docker buildx bake --file docker/bake.hcl --progress=plain \ --set '*.platform=linux/amd64' \ pd store server-standalone server-hstore - - name: Verify image runtime contracts - run: | - bash .github/scripts/check-docker-images.sh - bash hugegraph-server/hugegraph-dist/docker/test/test-docker-entrypoint.sh - - - name: Verify deployment checker contracts - run: python3 -m unittest discover -s .github/scripts -p 'test_docker_deployment.py' - - - name: Verify exact PR images, authentication and graph read/write - run: python3 .github/scripts/docker-deployment.py "$IMAGE_TAG" + # Each runtime check uses successfully built images. Preserve a failed + # check while allowing the independent deployment check to collect results. + status=0 + if ! bash .github/scripts/check-docker-images.sh; then + status=1 + fi + if [[ "$SMOKE_ONLY" != true ]]; then + if ! python3 -m unittest discover -s .github/scripts -p 'test_docker_deployment.py'; then + status=1 + fi + fi + if ! python3 .github/scripts/docker-deployment.py "$IMAGE_TAG"; then + status=1 + fi + exit "$status" diff --git a/.github/workflows/helm-chart-ci.yml b/.github/workflows/helm-chart-ci.yml index 5ecfaa2074..6e425a3dca 100644 --- a/.github/workflows/helm-chart-ci.yml +++ b/.github/workflows/helm-chart-ci.yml @@ -19,14 +19,13 @@ name: "Helm Chart CI" on: workflow_call: - workflow_dispatch: permissions: contents: read jobs: lint-and-render: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - uses: actions/checkout@v4 with: diff --git a/.github/workflows/java-runtime.yml b/.github/workflows/java-runtime.yml deleted file mode 100644 index 3dbb207090..0000000000 --- a/.github/workflows/java-runtime.yml +++ /dev/null @@ -1,69 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -name: CI Java runtime - -on: - workflow_call: - inputs: - extra-java-versions: - description: Additional JDKs to test alongside the configured project runtime (JSON array). - type: string - default: '[]' - outputs: - versions: - description: Project runtime followed by any additional test JDKs (JSON array). - value: ${{ jobs.read-version.outputs.versions }} - -permissions: - contents: read - -jobs: - read-version: - runs-on: ubuntu-24.04 - outputs: - versions: ${{ steps.java.outputs.versions }} - steps: - - name: Checkout - uses: actions/checkout@v7 - with: - persist-credentials: false - - - name: Read the project runtime and optional test JDKs - id: java - env: - EXTRA_JAVA_VERSIONS: ${{ inputs.extra-java-versions }} - run: | - python3 - <<'PYTHON' - import json - import os - import re - from pathlib import Path - - runtime = Path('.github/workflows/.java-version').read_text().strip() - extra = json.loads(os.environ['EXTRA_JAVA_VERSIONS']) - if not isinstance(extra, list): - raise ValueError('extra-java-versions must be a JSON array') - versions = [] - for version in [runtime, *extra]: - if not isinstance(version, str) or not re.fullmatch(r'[1-9][0-9]*(\.[0-9]+)*', version): - raise ValueError('Java versions must be non-empty numeric strings') - if version not in versions: - versions.append(version) - with open(os.environ['GITHUB_OUTPUT'], 'a') as output: - print('versions=' + json.dumps(versions), file=output) - PYTHON diff --git a/.github/workflows/pd-store-ci.yml b/.github/workflows/pd-store-ci.yml index 20e9f19dc7..00adccfe64 100644 --- a/.github/workflows/pd-store-ci.yml +++ b/.github/workflows/pd-store-ci.yml @@ -2,26 +2,25 @@ name: "HugeGraph-PD & Store & Hstore CI" on: workflow_call: - workflow_dispatch: - + inputs: + run-struct: + type: boolean + default: true + run-pd: + type: boolean + default: true + run-store: + type: boolean + default: true + run-hstore: + type: boolean + default: true permissions: contents: read jobs: - store-stop-shell-test: - permissions: - contents: read - runs-on: ubuntu-24.04 - steps: - - name: Checkout - uses: actions/checkout@v7 - with: - persist-credentials: false - - - name: Verify Store stop failure propagation and PID retention - run: bash hugegraph-server/hugegraph-dist/src/assembly/travis/test-stop-hugegraph-store.sh - rocksdb-compatibility: + if: inputs.run-pd || inputs.run-store || inputs.run-hstore permissions: contents: read runs-on: ubuntu-24.04 @@ -77,7 +76,8 @@ jobs: ${{ runner.temp }}/rocksdb-compatibility/cases/ struct: - runs-on: ubuntu-latest + if: inputs.run-struct || inputs.run-store + runs-on: ubuntu-24.04 env: USE_STAGE: 'false' steps: @@ -94,6 +94,9 @@ jobs: cache: maven cache-jdk: false + - name: Verify Store stop failure propagation and PID retention + run: bash hugegraph-server/hugegraph-dist/src/assembly/travis/test-stop-hugegraph-store.sh + - name: Run JaCoCo report validator tests run: hugegraph-server/hugegraph-dist/src/assembly/travis/test-check-jacoco-report.sh @@ -129,9 +132,9 @@ jobs: mvn -U -ntp -pl hugegraph-struct test pd: + if: inputs.run-pd timeout-minutes: 30 - needs: struct - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 env: # TODO: avoid duplicated env setup in pd & store USE_STAGE: 'false' # Whether to include the stage repository. @@ -311,9 +314,9 @@ jobs: fail_ci_if_error: false store: + if: inputs.run-store timeout-minutes: 45 - needs: struct - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 env: USE_STAGE: 'false' # Whether to include the stage repository. # TODO: remove outdated env @@ -513,8 +516,8 @@ jobs: fail_ci_if_error: false hstore: - needs: struct - runs-on: ubuntu-latest + if: inputs.run-hstore + runs-on: ubuntu-24.04 env: USE_STAGE: 'false' # Whether to include the stage repository. TRAVIS_DIR: hugegraph-server/hugegraph-dist/src/assembly/travis diff --git a/.github/workflows/rerun-ci.yml b/.github/workflows/rerun-ci.yml deleted file mode 100644 index 1e21f28ef4..0000000000 --- a/.github/workflows/rerun-ci.yml +++ /dev/null @@ -1,116 +0,0 @@ -name: "Rerun CI" - -on: - workflow_run: - workflows: - - "HugeGraph-Server CI" - types: - - completed - -permissions: {} - -env: - MAX_RERUNS: '2' - RETRY_DELAY_SECONDS: '180' - -jobs: - decide-rerun-action: - if: ${{ github.event.workflow_run.conclusion == 'failure' && github.event.workflow_run.event == 'push' }} - runs-on: ubuntu-latest - outputs: - action: ${{ steps.decision.outputs.action }} - steps: - - name: Decide rerun action - id: decision - env: - WORKFLOW_NAME: ${{ github.event.workflow_run.name }} - RUN_ID: ${{ github.event.workflow_run.id }} - RUN_ATTEMPT: ${{ github.event.workflow_run.run_attempt }} - EVENT_NAME: ${{ github.event.workflow_run.event }} - HEAD_BRANCH: ${{ github.event.workflow_run.head_branch }} - run: | - set -euo pipefail - - action="skip" - reason="unsupported event: $EVENT_NAME" - - if [[ "$EVENT_NAME" == "push" ]]; then - if (( RUN_ATTEMPT > MAX_RERUNS )); then - reason="retry limit reached" - else - action="rerun" - reason="within retry limit" - fi - fi - - { - echo "action=$action" - echo "reason=$reason" - } >> "$GITHUB_OUTPUT" - - { - echo "### Rerun CI decision" - echo "" - echo "- Workflow: $WORKFLOW_NAME" - echo "- Source event: $EVENT_NAME" - echo "- Head branch: $HEAD_BRANCH" - echo "- Run ID: $RUN_ID" - echo "- Current attempt: $RUN_ATTEMPT" - echo "- Max automatic reruns: $MAX_RERUNS" - echo "- Delay seconds: $RETRY_DELAY_SECONDS" - echo "- Action: $action" - echo "- Reason: $reason" - } >> "$GITHUB_STEP_SUMMARY" - - rerun-failed-jobs: - needs: decide-rerun-action - if: needs.decide-rerun-action.outputs.action == 'rerun' - permissions: - actions: write - contents: read - runs-on: ubuntu-latest - steps: - # workflow_run is privileged: only load code from the default-branch commit. - - name: Checkout trusted retry checker - uses: actions/checkout@v7 - with: - ref: ${{ github.sha }} - persist-credentials: false - sparse-checkout: .github/scripts - - - name: Check source freshness before waiting - id: initial-freshness - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GH_REPO: ${{ github.repository }} - RUN_ID: ${{ github.event.workflow_run.id }} - RUN_ATTEMPT: ${{ github.event.workflow_run.run_attempt }} - run: | - python3 .github/scripts/check-rerun.py \ - --repository "$GH_REPO" --run-id "$RUN_ID" \ - --run-attempt "$RUN_ATTEMPT" --max-reruns "$MAX_RERUNS" - - - name: Wait before rerun - if: steps.initial-freshness.outputs.action == 'rerun' - run: sleep "$RETRY_DELAY_SECONDS" - - - name: Check source freshness after waiting - id: freshness - if: steps.initial-freshness.outputs.action == 'rerun' - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GH_REPO: ${{ github.repository }} - RUN_ID: ${{ github.event.workflow_run.id }} - RUN_ATTEMPT: ${{ github.event.workflow_run.run_attempt }} - run: | - python3 .github/scripts/check-rerun.py \ - --repository "$GH_REPO" --run-id "$RUN_ID" \ - --run-attempt "$RUN_ATTEMPT" --max-reruns "$MAX_RERUNS" - - - name: Rerun the required job and its dependencies - if: steps.freshness.outputs.action == 'rerun' - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GH_REPO: ${{ github.repository }} - run: | - gh api --method POST "repos/$GH_REPO/actions/jobs/${{ steps.freshness.outputs.job_id }}/rerun" diff --git a/.github/workflows/riscv64-ci.yml b/.github/workflows/riscv64-ci.yml index 9812ecfa99..634251fb4e 100644 --- a/.github/workflows/riscv64-ci.yml +++ b/.github/workflows/riscv64-ci.yml @@ -44,7 +44,6 @@ env: jobs: build-server-riscv64: runs-on: ubuntu-24.04 - continue-on-error: true timeout-minutes: 90 steps: @@ -107,10 +106,10 @@ jobs: -P riscv64-protobuf-tools \ -pl hugegraph-server/hugegraph-dist -am \ -Dmaven.test.skip=true -Dmaven.javadoc.skip=true - hugegraph-server/hugegraph-dist/src/assembly/travis/check-rocksdb-only-dist.sh \ - hugegraph-server/apache-hugegraph-server-*/ - hugegraph-server/hugegraph-dist/src/assembly/travis/run-native-runtime-smoke-test.sh \ - hugegraph-server/apache-hugegraph-server-*/ + VERSION=$(mvn help:evaluate -Dexpression=project.version -q -DforceStdout) + SERVER_DIR="hugegraph-server/apache-hugegraph-server-$VERSION" + hugegraph-server/hugegraph-dist/src/assembly/travis/check-rocksdb-only-dist.sh "$SERVER_DIR" + hugegraph-server/hugegraph-dist/src/assembly/travis/run-native-runtime-smoke-test.sh "$SERVER_DIR" ' - name: Show server log on failure diff --git a/.github/workflows/server-ci.yml b/.github/workflows/server-ci.yml index 84e29fc3c4..d27a83774e 100644 --- a/.github/workflows/server-ci.yml +++ b/.github/workflows/server-ci.yml @@ -2,15 +2,20 @@ name: "HugeGraph-Server CI" on: push: - branches: - - master - - 'release-*' - - 'test-*' - - 'tinkerpop-*' + branches: [master, 'release-*', 'test-*', 'tinkerpop-*'] pull_request: + schedule: + - cron: '0 6 * * 1' + timezone: Asia/Singapore workflow_dispatch: - -# Keep the newest PR run; preserve independent push, release and manual runs. + inputs: + test-set: + description: Tests to run on the selected branch + type: choice + default: mainline + options: [mainline, distributed, macos-arm, hbase, riscv64, codeql, full] + +# A new PR head replaces its old run; scheduled and manual runs stay independent. concurrency: group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.run_id }}-${{ github.run_attempt == 1 && 'head' || format('rerun-{0}', github.run_id) }} cancel-in-progress: ${{ github.event_name == 'pull_request' }} @@ -20,262 +25,206 @@ permissions: pull-requests: read jobs: - check-license: - runs-on: ubuntu-latest - env: - USE_STAGE: 'false' # Whether to include the stage repository. - steps: - - uses: actions/checkout@v7 - - - name: Check License Header - uses: apache/skywalking-eyes@main - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - log: info - token: ${{ github.token }} - config: .licenserc.yaml - - - name: License Check (RAT) - run: | - mvn apache-rat:check -ntp - find ./ -name rat.txt -print0 | xargs -0 -I file cat file > merged-rat.txt - grep "Binaries" merged-rat.txt -C 3 && cat merged-rat.txt - - plan: - needs: check-license + preflight: + name: check-license runs-on: ubuntu-24.04 + timeout-minutes: 15 + env: + PYTHONDONTWRITEBYTECODE: '1' outputs: + selection_result: ${{ steps.select.outcome }} + java_result: ${{ steps.java.outcome }} + java_versions: ${{ steps.java.outputs.versions }} + legacy_memory_required: ${{ steps.migration.outputs.required }} server: ${{ steps.select.outputs.server }} commons: ${{ steps.select.outputs.commons }} + struct: ${{ steps.select.outputs.struct }} + pd: ${{ steps.select.outputs.pd }} + store: ${{ steps.select.outputs.store }} + hstore: ${{ steps.select.outputs.hstore }} pd_store: ${{ steps.select.outputs.pd_store }} cluster: ${{ steps.select.outputs.cluster }} docker: ${{ steps.select.outputs.docker }} helm: ${{ steps.select.outputs.helm }} dependency_license: ${{ steps.select.outputs.dependency_license }} - compatibility: ${{ steps.select.outputs.compatibility }} - smoke: ${{ steps.select.outputs.smoke }} + macos: ${{ steps.select.outputs.macos }} + macos_full: ${{ steps.select.outputs.macos_full }} + hbase: ${{ steps.select.outputs.hbase }} + riscv64: ${{ steps.select.outputs.riscv64 }} security: ${{ steps.select.outputs.security }} + distributed_smoke: ${{ steps.select.outputs.distributed_smoke }} steps: - - uses: actions/checkout@v7 + - name: Checkout + id: checkout + uses: actions/checkout@v7 with: fetch-depth: 0 persist-credentials: false - - name: Test selection, gate and retry policy - run: python3 -m unittest discover -s .github/scripts -p 'test_*.py' + - name: Select affected tests id: select env: GH_TOKEN: ${{ github.token }} run: >- python3 .github/scripts/ci-policy.py plan --project server - --event-path "$GITHUB_EVENT_PATH" --repository "$GITHUB_REPOSITORY" - --output plan.json - - uses: actions/upload-artifact@v4 + --event-name "$GITHUB_EVENT_NAME" --event-path "$GITHUB_EVENT_PATH" + --repository "$GITHUB_REPOSITORY" --output "$RUNNER_TEMP/ci-plan.json" + + - name: Read project Java runtime + id: java + if: ${{ !cancelled() && steps.checkout.outcome == 'success' }} + run: | + python3 - <<'PYTHON' + import json + import os + import re + from pathlib import Path + + runtime = Path('.github/workflows/.java-version').read_text().strip() + if not re.fullmatch(r'[1-9][0-9]*(\.[0-9]+)*', runtime): + raise ValueError('Project Java runtime must be a non-empty numeric version') + with open(os.environ['GITHUB_OUTPUT'], 'a') as output: + print('versions=' + json.dumps([runtime]), file=output) + PYTHON + + - name: Record selection + if: ${{ !cancelled() && steps.select.outcome == 'success' }} + uses: actions/upload-artifact@v4 with: name: ci-plan - path: plan.json + path: ${{ runner.temp }}/ci-plan.json retention-days: 7 overwrite: true - java-runtime: - needs: check-license - uses: ./.github/workflows/java-runtime.yml - permissions: - contents: read - - wait-storage-shell-test: - name: Startup contracts - needs: plan - if: needs.plan.outputs.server == 'true' - permissions: - contents: read - runs-on: ubuntu-24.04 - steps: - - name: Checkout - uses: actions/checkout@v7 - with: - persist-credentials: false + - name: Check legacy required context + id: migration + if: ${{ !cancelled() && github.event_name == 'pull_request' && steps.checkout.outcome == 'success' }} + env: + GH_TOKEN: ${{ github.token }} + BASE_REF: ${{ github.event.pull_request.base.ref }} + run: | + BASE_REF=$(python3 -c 'import os, urllib.parse; print(urllib.parse.quote(os.environ["BASE_REF"], safe=""))') + REQUIRED=$(gh api "repos/$GITHUB_REPOSITORY/branches/$BASE_REF" \ + --jq '.protection.required_status_checks.contexts // [] | index("Server memory tests") != null') + echo "required=$REQUIRED" >> "$GITHUB_OUTPUT" - - name: Test CI retry freshness checks - run: python3 -m unittest discover -s .github/scripts -p 'test_check_rerun.py' + - name: Test CI policy and deployment checks + if: ${{ !cancelled() && steps.checkout.outcome == 'success' }} + run: python3 -m unittest discover -s .github/scripts -p 'test_*.py' - - name: Run wait-storage.sh peer failover tests - run: hugegraph-server/hugegraph-dist/src/assembly/travis/test-wait-storage.sh + - name: Check license headers + if: ${{ !cancelled() && steps.checkout.outcome == 'success' }} + uses: apache/skywalking-eyes@main + with: + log: info + token: ${{ github.token }} + config: .licenserc.yaml - - name: Run RISC-V Java runtime contract tests - run: hugegraph-server/hugegraph-dist/src/assembly/travis/test-riscv64-java-runtime.sh + - name: License Check (RAT) + if: ${{ !cancelled() && steps.checkout.outcome == 'success' }} + run: | + mvn apache-rat:check -ntp + find . -name rat.txt -print0 | xargs -0 -I file cat file > "$RUNNER_TEMP/merged-rat.txt" + cat "$RUNNER_TEMP/merged-rat.txt" - - name: Check TinkerPop report execution gates - run: bash hugegraph-server/hugegraph-dist/src/assembly/travis/test-tinkerpop-reports.sh + build-server: + needs: preflight + if: ${{ !cancelled() && needs.preflight.outputs.selection_result == 'success' && needs.preflight.outputs.java_result == 'success' && needs.preflight.outputs.server == 'true' }} + uses: ./.github/workflows/server-tests.yml + with: + backend: memory + java-versions: ${{ needs.preflight.outputs.java_versions }} + secrets: inherit - # RocksDB coverage uses the project runtime and any additional test JDKs. - # The shared runtime comes from .github/workflows/.java-version, independent of bytecode targets. - # Keep the job ID for the Codecov configuration validator. server-rocksdb: - needs: [plan, java-runtime] - if: needs.plan.outputs.server == 'true' + needs: preflight + if: ${{ !cancelled() && needs.preflight.outputs.selection_result == 'success' && needs.preflight.outputs.java_result == 'success' && needs.preflight.outputs.server == 'true' }} uses: ./.github/workflows/server-tests.yml with: backend: rocksdb - java-versions: ${{ needs.java-runtime.outputs.versions }} + java-versions: ${{ needs.preflight.outputs.java_versions }} secrets: inherit + legacy-server-memory: + name: Server memory tests + needs: [preflight, build-server] + if: ${{ !cancelled() && needs.preflight.outputs.legacy_memory_required == 'true' }} + runs-on: ubuntu-slim + timeout-minutes: 5 + env: + SELECTION_RESULT: ${{ needs.preflight.outputs.selection_result }} + SERVER_SELECTED: ${{ needs.preflight.outputs.server }} + JAVA_RESULT: ${{ needs.preflight.outputs.java_result }} + MEMORY_RESULT: ${{ needs.build-server.result }} + steps: + - name: Verify the selected Memory result during protection migration + run: | + test "$SELECTION_RESULT" = success + case "$SERVER_SELECTED" in + true) test "$JAVA_RESULT" = success && test "$MEMORY_RESULT" = success ;; + false) test "$MEMORY_RESULT" = skipped ;; + *) exit 1 ;; + esac + commons: - name: commons - needs: plan - if: needs.plan.outputs.commons == 'true' + needs: preflight + if: ${{ !cancelled() && needs.preflight.outputs.selection_result == 'success' && needs.preflight.outputs.java_result == 'success' && needs.preflight.outputs.commons == 'true' }} uses: ./.github/workflows/commons-ci.yml + with: + java-versions: ${{ needs.preflight.outputs.java_versions }} secrets: inherit pd_store: - name: pd_store - needs: plan - if: needs.plan.outputs.pd_store == 'true' + needs: preflight + if: ${{ !cancelled() && needs.preflight.outputs.selection_result == 'success' && needs.preflight.outputs.java_result == 'success' && needs.preflight.outputs.pd_store == 'true' }} uses: ./.github/workflows/pd-store-ci.yml + with: + run-struct: ${{ needs.preflight.outputs.struct == 'true' }} + run-pd: ${{ needs.preflight.outputs.pd == 'true' }} + run-store: ${{ needs.preflight.outputs.store == 'true' }} + run-hstore: ${{ needs.preflight.outputs.hstore == 'true' }} secrets: inherit cluster: - name: cluster - needs: plan - if: needs.plan.outputs.cluster == 'true' + needs: preflight + if: ${{ !cancelled() && needs.preflight.outputs.selection_result == 'success' && needs.preflight.outputs.java_result == 'success' && needs.preflight.outputs.cluster == 'true' }} uses: ./.github/workflows/cluster-test-ci.yml secrets: inherit docker: - name: docker - needs: plan - if: needs.plan.outputs.docker == 'true' + needs: preflight + if: ${{ !cancelled() && needs.preflight.outputs.selection_result == 'success' && needs.preflight.outputs.java_result == 'success' && (needs.preflight.outputs.docker == 'true' || needs.preflight.outputs.distributed_smoke == 'true') }} uses: ./.github/workflows/docker-build-ci.yml - secrets: inherit + with: + smoke-only: ${{ needs.preflight.outputs.distributed_smoke == 'true' && needs.preflight.outputs.docker != 'true' }} helm: - name: helm - needs: plan - if: needs.plan.outputs.helm == 'true' + needs: preflight + if: ${{ !cancelled() && needs.preflight.outputs.selection_result == 'success' && needs.preflight.outputs.helm == 'true' }} uses: ./.github/workflows/helm-chart-ci.yml dependency_license: - name: dependency_license - needs: plan - if: needs.plan.outputs.dependency_license == 'true' + needs: preflight + if: ${{ !cancelled() && needs.preflight.outputs.selection_result == 'success' && needs.preflight.outputs.java_result == 'success' && needs.preflight.outputs.dependency_license == 'true' }} uses: ./.github/workflows/check-dependencies.yml - affected-module-tests: - name: affected-module-tests - needs: [plan, java-runtime, wait-storage-shell-test, server-rocksdb, commons, pd_store, cluster, docker, helm, dependency_license] - if: ${{ !cancelled() }} - runs-on: ubuntu-24.04 - timeout-minutes: 5 - steps: - - uses: actions/checkout@v7 - with: - persist-credentials: false - - name: Check planning and runtime results - env: - JOB_RESULTS: ${{ toJSON(needs) }} - run: | - python3 - <<'PYGATE' - import json, os - results = json.loads(os.environ['JOB_RESULTS']) - if results['plan']['result'] != 'success': - raise SystemExit('CI planning failed') - if results['plan']['outputs'].get('server') == 'true': - for job in ['java-runtime', 'wait-storage-shell-test']: - if results[job]['result'] != 'success': - raise SystemExit('Selected prerequisite failed: ' + job) - results['server_rocksdb'] = results['server-rocksdb'] - with open('results.json', 'w') as output: - json.dump(results, output) - PYGATE - - uses: actions/download-artifact@v4 - with: - name: ci-plan - - name: Report selected suites and their real results - env: - GH_TOKEN: ${{ github.token }} - run: python3 .github/scripts/ci-policy.py gate --mode advisory --plan plan.json --results results.json --output test-results.json - - memory-plan: - needs: check-license - runs-on: ubuntu-24.04 - outputs: - server: ${{ steps.select.outputs.server }} - commons: ${{ steps.select.outputs.commons }} - pd_store: ${{ steps.select.outputs.pd_store }} - cluster: ${{ steps.select.outputs.cluster }} - docker: ${{ steps.select.outputs.docker }} - helm: ${{ steps.select.outputs.helm }} - dependency_license: ${{ steps.select.outputs.dependency_license }} - compatibility: ${{ steps.select.outputs.compatibility }} - smoke: ${{ steps.select.outputs.smoke }} - security: ${{ steps.select.outputs.security }} - steps: - - uses: actions/checkout@v7 - with: - fetch-depth: 0 - persist-credentials: false - - name: Select affected tests - id: select - env: - GH_TOKEN: ${{ github.token }} - run: >- - python3 .github/scripts/ci-policy.py plan --project server - --event-path "$GITHUB_EVENT_PATH" --repository "$GITHUB_REPOSITORY" - --output plan.json - - uses: actions/upload-artifact@v4 - with: - name: ci-memory-plan - path: plan.json - retention-days: 7 - overwrite: true - - memory-java-runtime: - needs: memory-plan - if: needs.memory-plan.outputs.server == 'true' - uses: ./.github/workflows/java-runtime.yml - permissions: - contents: read - - build-server: - needs: [memory-plan, memory-java-runtime] - if: needs.memory-plan.outputs.server == 'true' - uses: ./.github/workflows/server-tests.yml + compatibility: + needs: preflight + if: ${{ !cancelled() && needs.preflight.outputs.selection_result == 'success' && needs.preflight.outputs.java_result == 'success' && (needs.preflight.outputs.macos == 'true' || needs.preflight.outputs.hbase == 'true' || needs.preflight.outputs.riscv64 == 'true') }} + uses: ./.github/workflows/server-compatibility-ci.yml with: - backend: memory - java-versions: ${{ needs.memory-java-runtime.outputs.versions }} + java-versions: ${{ needs.preflight.outputs.java_versions }} + macos: ${{ needs.preflight.outputs.macos == 'true' }} + macos-full: ${{ needs.preflight.outputs.macos_full == 'true' }} + hbase: ${{ needs.preflight.outputs.hbase == 'true' }} + riscv64: ${{ needs.preflight.outputs.riscv64 == 'true' }} secrets: inherit - server-memory-required: - name: Server memory tests - needs: [memory-plan, memory-java-runtime, build-server] - if: ${{ always() }} - runs-on: ubuntu-24.04 - timeout-minutes: 5 - steps: - - uses: actions/checkout@v7 - with: - persist-credentials: false - - name: Check planning and runtime results - env: - JOB_RESULTS: ${{ toJSON(needs) }} - run: | - python3 - <<'PYGATE' - import json, os - results = json.loads(os.environ['JOB_RESULTS']) - if results['memory-plan']['result'] != 'success': - raise SystemExit('CI planning failed') - if results['memory-plan']['outputs'].get('server') == 'true' and results['memory-java-runtime']['result'] != 'success': - raise SystemExit('Selected Java runtime preparation failed') - results['plan'] = results['memory-plan'] - results['server_memory'] = results['build-server'] - with open('results.json', 'w') as output: - json.dump(results, output) - PYGATE - - uses: actions/download-artifact@v4 - with: - name: ci-memory-plan - - name: Report selected suites and their real results - env: - GH_TOKEN: ${{ github.token }} - run: python3 .github/scripts/ci-policy.py gate --mode memory --plan plan.json --results results.json --output test-results.json + codeql: + needs: preflight + if: ${{ !cancelled() && needs.preflight.outputs.selection_result == 'success' && needs.preflight.outputs.java_result == 'success' && needs.preflight.outputs.security == 'true' }} + uses: ./.github/workflows/codeql-analysis.yml + permissions: + contents: read + actions: read + security-events: write diff --git a/.github/workflows/server-compatibility-ci.yml b/.github/workflows/server-compatibility-ci.yml index 81b9d3cdfc..422182fdd5 100644 --- a/.github/workflows/server-compatibility-ci.yml +++ b/.github/workflows/server-compatibility-ci.yml @@ -1,95 +1,56 @@ name: "Server Compatibility CI" on: - push: - branches: - - master - - 'release-*' - - 'test-*' - pull_request: - workflow_dispatch: - -# Keep the newest PR run; preserve independent push, release and manual runs. -concurrency: - group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.run_id }}-${{ github.run_attempt == 1 && 'head' || format('rerun-{0}', github.run_id) }} - cancel-in-progress: ${{ github.event_name == 'pull_request' }} + workflow_call: + inputs: + java-versions: + type: string + required: true + hbase: + type: boolean + default: false + riscv64: + type: boolean + default: false + macos: + type: boolean + default: false + macos-full: + type: boolean + default: false permissions: contents: read - pull-requests: read jobs: - plan: - runs-on: ubuntu-24.04 - outputs: - server: ${{ steps.select.outputs.server }} - commons: ${{ steps.select.outputs.commons }} - pd_store: ${{ steps.select.outputs.pd_store }} - cluster: ${{ steps.select.outputs.cluster }} - docker: ${{ steps.select.outputs.docker }} - helm: ${{ steps.select.outputs.helm }} - dependency_license: ${{ steps.select.outputs.dependency_license }} - compatibility: ${{ steps.select.outputs.compatibility }} - smoke: ${{ steps.select.outputs.smoke }} - security: ${{ steps.select.outputs.security }} - steps: - - uses: actions/checkout@v7 - with: - fetch-depth: 0 - persist-credentials: false - - name: Select affected tests - id: select - env: - GH_TOKEN: ${{ github.token }} - run: >- - python3 .github/scripts/ci-policy.py plan --project server - --event-path "$GITHUB_EVENT_PATH" --repository "$GITHUB_REPOSITORY" - --output plan.json - - uses: actions/upload-artifact@v4 - with: - name: ci-plan - path: plan.json - retention-days: 7 - overwrite: true - - java-runtime: - uses: ./.github/workflows/java-runtime.yml - permissions: - contents: read - build-server-backends: - needs: [plan, java-runtime] - if: needs.plan.outputs.compatibility == 'true' + if: inputs.hbase uses: ./.github/workflows/server-tests.yml with: backend: hbase - java-versions: ${{ needs.java-runtime.outputs.versions }} + java-versions: ${{ inputs.java-versions }} secrets: inherit build-server-macos-rocksdb: - needs: plan - if: ${{ !cancelled() && needs.plan.result == 'success' && needs.plan.outputs.compatibility == 'true' }} - runs-on: ${{ matrix.os }} - strategy: - fail-fast: false - matrix: - include: - - os: macos-15-intel - server_java_options: '' - - os: macos-15 - server_java_options: '-Xms512m -Xmx2g' + name: macOS ARM (${{ inputs.macos-full && 'full' || 'smoke' }}) + if: inputs.macos + runs-on: macos-15 + timeout-minutes: 60 env: USE_STAGE: 'false' # Whether to include the stage repository. TRAVIS_DIR: hugegraph-server/hugegraph-dist/src/assembly/travis REPORT_DIR: target/site/jacoco BACKEND: rocksdb - SERVER_JAVA_OPTIONS: ${{ matrix.server_java_options }} + EXPECTED_ARCH: arm64 + EXPECTED_JAVA_MAJOR: ${{ fromJSON(inputs.java-versions)[0] }} + SERVER_JAVA_OPTIONS: '-Xms512m -Xmx2g' steps: - name: Checkout uses: actions/checkout@v7 with: fetch-depth: 5 + persist-credentials: false - name: Install project JDK uses: actions/setup-java@v6 @@ -102,33 +63,48 @@ jobs: - name: Use staged maven repo settings if: ${{ env.USE_STAGE == 'true' }} run: | - cp $HOME/.m2/settings.xml /tmp/settings.xml - cp -vf .github/configs/settings.xml $HOME/.m2/settings.xml && cat $HOME/.m2/settings.xml + cp "$HOME/.m2/settings.xml" /tmp/settings.xml + cp -vf .github/configs/settings.xml "$HOME/.m2/settings.xml" && cat "$HOME/.m2/settings.xml" - - name: Compile + - name: Prepare bounded runtime commands run: | - mvn clean compile -pl hugegraph-server/hugegraph-test -am -U -Dmaven.javadoc.skip=true -ntp + if ! command -v timeout >/dev/null 2>&1 && ! command -v gtimeout >/dev/null 2>&1; then + brew install coreutils + fi + + - name: Package RocksDB distribution + run: | + mvn clean package -pl hugegraph-server/hugegraph-dist -am -U -Drocksdb-only \ + -Dmaven.test.skip=true -Dmaven.javadoc.skip=true -ntp - name: Run check_port unit tests # Validates the ss/netstat port preflight that replaced lsof run: | - $TRAVIS_DIR/test-check-port.sh \ + "$TRAVIS_DIR/test-check-port.sh" \ hugegraph-server/hugegraph-dist/src/assembly/static \ hugegraph-store/hg-store-dist/src/assembly/static/bin/util.sh + - name: Run native startup, JNI and restart smoke + run: | + VERSION=$(mvn help:evaluate -Dexpression=project.version -q -DforceStdout) + SERVER_DIR="hugegraph-server/apache-hugegraph-server-$VERSION" + "$TRAVIS_DIR/run-native-runtime-smoke-test.sh" "$SERVER_DIR" + - name: Run RocksDB core test + if: inputs.macos-full run: | - $TRAVIS_DIR/run-core-test.sh $BACKEND + "$TRAVIS_DIR/run-core-test.sh" "$BACKEND" - name: Run RocksDB API test + if: inputs.macos-full run: | - $TRAVIS_DIR/run-api-test.sh $BACKEND $REPORT_DIR true + "$TRAVIS_DIR/run-api-test.sh" "$BACKEND" "$REPORT_DIR" true - name: Show server log on failure if: failure() run: | VERSION=$(mvn help:evaluate -Dexpression=project.version -q -DforceStdout) - SERVER_DIR=hugegraph-server/apache-hugegraph-server-$VERSION/ + SERVER_DIR="hugegraph-server/apache-hugegraph-server-$VERSION" if [ -f "$SERVER_DIR/logs/hugegraph-server.log" ]; then tail -n 200 "$SERVER_DIR/logs/hugegraph-server.log" fi @@ -137,28 +113,13 @@ jobs: if: always() run: | VERSION=$(mvn help:evaluate -Dexpression=project.version -q -DforceStdout) - SERVER_DIR=hugegraph-server/apache-hugegraph-server-$VERSION/ + SERVER_DIR="hugegraph-server/apache-hugegraph-server-$VERSION" if [ -f "$SERVER_DIR/bin/pid" ]; then - $TRAVIS_DIR/stop-server.sh $SERVER_DIR || true + "$TRAVIS_DIR/stop-server.sh" "$SERVER_DIR" || true fi build-server-riscv64: - # Keep the emulated Java 17 RISC-V native-runtime signal visible while the - # platform lane stabilizes, but do not use it as a gate yet. - needs: plan - if: ${{ !cancelled() && needs.plan.result == 'success' && needs.plan.outputs.compatibility == 'true' }} + if: inputs.riscv64 uses: ./.github/workflows/riscv64-ci.yml permissions: contents: read - - compose-smoke: - name: Published Compose smoke - needs: plan - if: needs.plan.outputs.smoke == 'true' - runs-on: ubuntu-24.04 - timeout-minutes: 25 - steps: - - uses: actions/checkout@v7 - with: - persist-credentials: false - - run: bash docker/test-compose.sh smoke diff --git a/.github/workflows/server-tests.yml b/.github/workflows/server-tests.yml index 05ac0862bb..f86d0904bb 100644 --- a/.github/workflows/server-tests.yml +++ b/.github/workflows/server-tests.yml @@ -73,6 +73,13 @@ jobs: run: | mvn clean compile -pl hugegraph-server/hugegraph-test -am -U -Dmaven.javadoc.skip=true -ntp + - name: Check startup and native runtime contracts + if: inputs.backend == 'rocksdb' + run: | + "$TRAVIS_DIR/test-wait-storage.sh" + "$TRAVIS_DIR/test-riscv64-java-runtime.sh" + bash "$TRAVIS_DIR/test-tinkerpop-reports.sh" + - name: Validate Docker integration if: ${{ env.BACKEND == 'rocksdb' }} run: | diff --git a/docs/ci.md b/docs/ci.md index 511f8721a2..a6dd9fc1e5 100644 --- a/docs/ci.md +++ b/docs/ci.md @@ -1,101 +1,107 @@ # CI policy -CI selects tests from the cumulative pull-request diff and known consumer dependencies. -Selection controls what runs; branch protection controls what must pass before merging. - -| Workflow | Purpose | Merge requirement | -| --- | --- | --- | -| HugeGraph-Server CI | License headers and RAT, then Memory and related modules, images and dependency audits | `check-license`, `Server memory tests`; other product checks advisory | -| CodeQL | Source analysis, independently selected and scheduled | Advisory | -| Server Compatibility CI | HBase, macOS, RISC-V and published-image Compose smoke | Advisory | - -HugeGraph-Server CI runs `check-license` before both Memory and advisory planning or -runtime preparation. A failed license check skips product jobs and fails the Memory required -result. Both required check names stay unchanged. Memory keeps its own planner and result -check so advisory failures cannot block a passing Memory result. Compatibility and CodeQL -remain independently scheduled. - -Jobs within a workflow share its cancellation scope. Cancelling HugeGraph-Server CI also -cancels its required validation, which must complete before merging. It does not cancel -compatibility or prevent CodeQL from starting. Failed advisory checks retain their real failure -result (except the existing RISC-V job, which still uses `continue-on-error`); -maintainers decide whether they need another run or a requested change. - -## Affected inputs - -| Changed inputs | Selected validation | +[HugeGraph-Server CI](../.github/workflows/server-ci.yml) is the single entry point for +PRs, supported branch pushes, weekly coverage and manual validation. Its preflight job +selects affected tests, reads the project Java version, checks CI policy and verifies +license headers and RAT. Selected suites run through reusable workflows in the same run. + +CI results become advisory after [`.asf.yaml`](../.asf.yaml) is applied. It requests no required CI contexts +and retains one approving review. This protection change takes effect after the configuration +is merged and synchronized by Gitbox; verify live protection then. Failed tests keep their +real failed results. Valid selection and Java outputs allow selected tests to run even when +the preflight license checks fail. + +For migration, preflight keeps the existing `check-license` check name on the same runner. +A temporary `Server memory tests` check verifies the selected Memory result while live protection +still requires that name. It uses one lightweight `ubuntu-slim` runner and repeats no tests. +After Gitbox applies the new configuration, this bridge skips without allocating a runner; +no Infra intervention or change to review protection is needed. + +## What runs + +| Validation | Ordinary PR | Push to master | Weekly full run | +| --- | --- | --- | --- | +| Memory and RocksDB | Full unit, core and API tests when affected | No repeated full tests | Full tests | +| macOS ARM | Startup, JNI, graph read/write and restart smoke when affected; full core/API for platform inputs | Affected smoke | Full core/API | +| HBase and RISC-V | Directly related backend/platform or validation inputs only | No repeated tests | Full validation | +| PD, Store, HStore and Cluster | Affected modules and known consumers | Current-commit distributed smoke | Full group | +| CodeQL | Authentication, authorization, security, POM or CodeQL configuration inputs | No repeated scan | Full scan | +| Docker, Helm and dependency audit | Corresponding inputs | Corresponding deployment smoke or checks | Full validation | + +macOS uses one `macos-15` ARM job for either smoke or full validation. Intel macOS is removed. +The weekly run uses `cron: '0 6 * * 1'` with `timezone: Asia/Singapore`: Monday at 06:00, +on the latest default-branch commit. GitHub may delay scheduled runs during high load; +see [schedule behavior](https://docs.github.com/en/actions/reference/workflows-and-actions/events-that-trigger-workflows#schedule). + +### Affected inputs + +The rules live in [ci-policy.py](../.github/scripts/ci-policy.py). Ordinary Server changes +select both Memory and RocksDB, plus ARM smoke. They do not select the complete PD/Store group. +PD and Store changes select their own tests and downstream consumers; Commons and Struct +select their known shared consumers. HStore and Cluster additionally follow distributed +backend contracts, protocols, metadata, graph lifecycle and scheduling inputs. Distribution, +startup and RocksDB inputs select ARM full validation and their deployment consumers. + +Unclassified paths or an unavailable diff/API conservatively select mainline coverage. +HBase and RISC-V remain limited to explicit related inputs, weekly coverage or manual selection. +The classifier uses paths, not PR titles or semantic interpretation of changed text. +Contract scripts explicitly select their executing suites without adding runtime consumers. + +### Documentation + +Documentation-only ordinary PRs and pushes allocate just the preflight CI runner. This count +excludes the non-test [Auto PR Commenter](../.github/workflows/auto-pr-review.yml), which runs +once when a PR opens. Scheduled and manual selections still run their requested suites. + +Plain documentation includes regular-file root prose, README/AGENTS files and supported prose +or static images under `docs/`, including nested module documentation. Runtime resources, +configuration, source, fixtures, templates and executable or symbolic-link files do not qualify; +for example, a test fixture ending in `.txt` still selects its module's tests. + +## Manual validation and reruns + +Use **Actions → HugeGraph-Server CI → Run workflow**, select a branch and choose `test-set`: + +| Test set | Validation | | --- | --- | -| Server, including RocksDB and HStore | Memory, RocksDB, PD/Store/HStore and Cluster | -| Server distribution or shared startup scripts | Server, PD/Store/HStore, Docker and Cluster | -| Commons or Struct | Their tests and affected Server, PD, Store, HStore and Cluster tests | -| PD or Store distribution scripts, config, assembly or POM | PD/Store/HStore, Cluster and Docker | -| Other PD or Store inputs | PD/Store/HStore and Cluster | -| Server parent/API POM or Commons version resource | Normal module coverage plus Docker artifact checks | -| Cluster, Docker or Helm | Their suite and known consumers | -| PR template, Dependabot configuration or retry checker/tests | Planner checks and license checks; no product suites | -| Docker image/deployment checkers and their tests | Docker | -| Dependencies, shared build inputs or unknown paths | Conservative full coverage | - -The Memory required result depends only on its planner, runtime preparation and real Memory -matrix. A selected test that fails, is cancelled or unexpectedly skipped cannot pass it. -When Memory is not selected, the result explains that no Memory test was needed. -The advisory `affected-module-tests` summary lists selection, required status and actual results. -It does not stand in for Memory or control CodeQL and compatibility checks. - -The Docker suite builds the current four production images through a shared Maven build. -Checks consume those final images, verify identity, health, version and authentication, and -exercise graph writes, reads and Gremlin queries on standalone and PD/Store/Server topologies. Published-image Hubble Compose smoke runs once in -compatibility CI when RocksDB or Docker is selected; it does not verify the new PR images. - -HBase compatibility uses the existing HBase version in one standalone container with local -storage. Its image is pinned by digest; no separate HDFS service or CI image build is needed. -Pull, startup and readiness preparation have a five-minute total budget. Preparation failure -ends the advisory check without falling back to a tar download. Existing HBase behavior tests -remain. This changes CI preparation, not product support or backend deprecation policy. - -TinkerPop suites run only when the source, target or push/manual branch starts with -`release-`, `test-` or `tinkerpop-`. PRs run no TP suites when neither source nor target -branch matches these prefixes, including `upgrade/1.8.0` targeting `master`. -Executed TP suites require non-empty reports with actual executed tests. - -PD, Store and Commons have total job budgets of 30, 45 and 20 minutes, respectively. -Cluster has 120 minutes for its two 45-minute test windows and build/diagnostic time. -Server and HStore budgets are defined in their workflow job settings. - -## Documentation and freshness - -Only explicitly allowed prose and static documentation assets qualify as plain documentation. -Packaged resources, configuration, source, tests and CI inputs never qualify. Documentation-only -PRs skip compilation, backend services, images and PR CodeQL; lightweight selection and license -checks still report. A source PR with a later documentation commit still tests its cumulative -source changes. No result is reused from a previous run. - -Plans record the event head and the actual merge commit and base tested. Checkout must match -the event merge SHA, whose second parent must match the current PR head. Its first parent is -the tested base; the event base may lag when master advances before checkout. A new source -head invalidates the old run; target-branch advancement alone -does not, matching non-strict branch protection. A selection/API failure conservatively selects -all suites. A final metadata outage alone cannot invalidate completed Memory tests. -Plans and actual results are diagnostics, not execution credentials for later runs. - -## Protection and retries - -`.asf.yaml` requests only `check-license` and `Server memory tests`. Keep these required check -names stable. The Memory jobs run real unit, core and API tests on the project runtime -and report their results through `Server memory tests`. Commons tests remain advisory -and install only the Commons reactor and its upstream modules. -`affected-module-tests` and CodeQL remain advisory. Verify live branch protection -when changing required checks; do not forge a successful result or leave retired contexts required. - -Push runs whose overall conclusion is `failure` keep automatic retries for license and Memory -validation: wait 180 seconds, then retry at most twice, sharing the combined run's attempt limit. -Whole-run `timed_out` conclusions do not trigger retries; PR retries remain paused. -The trusted default-branch checker verifies the source workflow, failed completed attempt, -repository and unchanged branch head before and after the delay. It selects one license or -Memory producer with a `failure` or `timed_out` conclusion, or its failed result reporter, -from that attempt and uses -[GitHub's native job rerun](https://docs.github.com/en/rest/actions/workflow-runs#re-run-a-job-from-a-workflow-run), -which also reruns dependent jobs. A Memory retry updates its required result without restarting unrelated product jobs. -A license retry can unblock its previously skipped product dependencies. Advisory failures alone -do not trigger automatic retries. Use a manual rerun when appropriate; it retains the original -commit and event. +| `mainline` (default) | Full mainline modules, deployment/dependency checks and CodeQL; ARM smoke | +| `distributed` | Struct, PD, Store, HStore and Cluster, plus distributed smoke | +| `macos-arm` | ARM smoke and full RocksDB core/API tests | +| `hbase`, `riscv64`, `codeql` | The corresponding complete specialist validation | +| `full` | All retained suites, identical to weekly coverage | + +Fork contributors run additional validation in their fork's Actions. The manual entry point +does not accept a PR number, apply labels or reuse another workflow's results. +Automatic retries are disabled. Use GitHub's native rerun controls for failed jobs or a run; +a rerun retains its original commit and event, subject to the preflight freshness checks. + +## Freshness, runner use and budgets + +PR selection uses the cumulative diff against the checked-out merge commit's base. A source +PR followed by a documentation-only commit still tests the cumulative source changes. +Preflight verifies that the event merge commit's second parent matches the current PR head. +A stale PR head fails preflight. Target-branch advancement alone does not invalidate a run. +Plans and selection summaries describe this run; previous results are never reused. + +A new PR head cancels the previous PR run, including its called workflows. Manual and weekly +runs remain independent. Each job reports its own result; there are no separate post-test +aggregation runners after the protection migration or end-of-run metadata checks. + +Startup contracts run inside RocksDB validation, and Store stop contracts run inside Struct. +Docker's static checks, deployment contracts and current-image validation share one runner. +The Docker job has a 90-minute total budget, with 25 minutes for deployment contracts and +45 minutes for image build/runtime validation. Distributed smoke builds current PD, Store +and Server images and checks authentication, readiness and graph read/write. It does not +substitute published images for the current commit. + +PD, Store and Commons have total job budgets of 30, 45 and 20 minutes. Cluster has 120 minutes +for its two 45-minute test windows and build/diagnostics. Other budgets are in the workflows. +TinkerPop suites keep their dedicated branch policy: the source, target or push/manual branch +must start with `release-`, `test-` or `tinkerpop-`. Executed suites require non-empty reports +with actual tests. + +Each separate job or matrix item requests a runner; steps within a job share one. +[Standard hosted runners are free for public repositories](https://docs.github.com/en/actions/reference/runners/github-hosted-runners#standard-github-hosted-runners-for-public-repositories), +but [plan-dependent total and macOS concurrency limits](https://docs.github.com/en/actions/reference/limits#job-concurrency-limits-for-github-hosted-runners) +still apply. Reducing job allocations and long executions lowers shared-pool demand; +it does not guarantee a specific queue delay. diff --git a/hugegraph-server/hugegraph-dist/src/assembly/travis/run-native-runtime-smoke-test.sh b/hugegraph-server/hugegraph-dist/src/assembly/travis/run-native-runtime-smoke-test.sh index 8c42ecf7c1..524b478846 100755 --- a/hugegraph-server/hugegraph-dist/src/assembly/travis/run-native-runtime-smoke-test.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/travis/run-native-runtime-smoke-test.sh @@ -47,13 +47,21 @@ cleanup() { trap cleanup EXIT start_server() { - if ! command -v timeout >/dev/null 2>&1; then - echo "Required command is unavailable: timeout" >&2 + if command -v timeout >/dev/null 2>&1; then + TIMEOUT_COMMAND=timeout + elif command -v gtimeout >/dev/null 2>&1; then + TIMEOUT_COMMAND=gtimeout + else + echo "Required command is unavailable: timeout or gtimeout" >&2 return 1 fi - timeout --foreground --kill-after=15s \ + local start_options=(-t "$SERVER_STARTUP_TIMEOUT") + if [[ -n "${SERVER_JAVA_OPTIONS:-}" ]]; then + start_options+=(-j "$SERVER_JAVA_OPTIONS") + fi + "$TIMEOUT_COMMAND" --foreground --kill-after=15s \ "${SERVER_START_COMMAND_TIMEOUT}s" \ - "$SERVER_DIR/bin/start-hugegraph.sh" -t "$SERVER_STARTUP_TIMEOUT" + "$SERVER_DIR/bin/start-hugegraph.sh" "${start_options[@]}" } "$TRAVIS_DIR/run-rocksdb-jni-smoke-test.sh" "$SERVER_DIR" @@ -66,7 +74,7 @@ start_server grep -q '^server-e2e-smoke-create-ok$' "$WORK_DIR/create.log" "$SERVER_DIR/bin/stop-hugegraph.sh" -m false -timeout --foreground --kill-after=15s "${STORE_DUMP_TIMEOUT}s" \ +"$TIMEOUT_COMMAND" --foreground --kill-after=15s "${STORE_DUMP_TIMEOUT}s" \ "$SERVER_DIR/bin/dump-store.sh" | tee "$WORK_DIR/store-dump.log" grep -q 'Dump table VERTEX (offset 0 limit 20):' "$WORK_DIR/store-dump.log" grep -Fq "riscv-smoke-v1-$RUN_ID" "$WORK_DIR/store-dump.log" diff --git a/hugegraph-server/hugegraph-dist/src/assembly/travis/run-rocksdb-jni-smoke-test.sh b/hugegraph-server/hugegraph-dist/src/assembly/travis/run-rocksdb-jni-smoke-test.sh index 81d88796d0..0e1575f6a4 100755 --- a/hugegraph-server/hugegraph-dist/src/assembly/travis/run-rocksdb-jni-smoke-test.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/travis/run-rocksdb-jni-smoke-test.sh @@ -26,6 +26,8 @@ TRAVIS_DIR=$(cd "$(dirname "$0")" && pwd) SERVER_DIR=$(cd "$1" && pwd) EXPECTED_ARCH=${EXPECTED_ARCH:-} EXPECTED_JAVA_MAJOR=${EXPECTED_JAVA_MAJOR:-17} +# Callers may supply a full setup-java version; this check enforces its major. +EXPECTED_JAVA_MAJOR=${EXPECTED_JAVA_MAJOR%%.*} EXPECTED_RISCV64_JAVA_VERSION=${EXPECTED_RISCV64_JAVA_VERSION:-} EXPECTED_RISCV64_JAVA_VENDOR=${EXPECTED_RISCV64_JAVA_VENDOR:-} ACTUAL_ARCH=$(uname -m) diff --git a/hugegraph-server/hugegraph-dist/src/assembly/travis/test-riscv64-java-runtime.sh b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-riscv64-java-runtime.sh index aa3131543b..cc33c98d17 100755 --- a/hugegraph-server/hugegraph-dist/src/assembly/travis/test-riscv64-java-runtime.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-riscv64-java-runtime.sh @@ -34,7 +34,7 @@ mkdir -p "$MOCK_BIN" "$MOCK_JAVA_HOME/bin" "$SERVER_DIR/bin" "$SERVER_DIR/lib" cat > "$MOCK_BIN/uname" <<'EOF' #!/bin/bash -echo riscv64 +echo "${MOCK_ARCH:-riscv64}" EOF cat > "$MOCK_JAVA_HOME/bin/java" <<'EOF' @@ -56,6 +56,10 @@ case "${1:-}" in echo "openjdk version \"$JAVA_VERSION\"" >&2 ;; -cp) + if [[ "${MOCK_ARCH:-riscv64}" == arm64 && -n "${LD_PRELOAD:-}" ]]; then + echo "ARM JNI smoke inherited RISC-V native preload" >&2 + exit 1 + fi echo "rocksdb-jni-smoke-ok" ;; *) @@ -68,7 +72,9 @@ EOF cat > "$SERVER_DIR/bin/util.sh" <<'EOF' #!/bin/bash configure_riscv64_libatomic() { - LD_PRELOAD=libatomic.so.1 + if [[ "$(uname -m)" == riscv64 ]]; then + LD_PRELOAD=libatomic.so.1 + fi } EOF @@ -112,6 +118,29 @@ if [[ $NATIVE_STATUS -ne 42 ]]; then fi grep -q '^rocksdb-jni-smoke-ok$' <<< "$NATIVE_OUTPUT" +# macOS passes the setup-java version through the native wrapper unchanged. +for EXPECTED_VERSION in 17 17.0.20 21.0.1; do + set +e + VERSION_OUTPUT=$(run_native_smoke MOCK_ARCH=arm64 EXPECTED_ARCH=arm64 \ + "EXPECTED_JAVA_MAJOR=$EXPECTED_VERSION" \ + "MOCK_JAVA_VERSION=$EXPECTED_VERSION" 2>&1) + VERSION_STATUS=$? + set -e + if [[ $VERSION_STATUS -ne 42 ]]; then + echo "$VERSION_OUTPUT" >&2 + echo "ARM native smoke rejected expected version $EXPECTED_VERSION" >&2 + exit 1 + fi + grep -q '^rocksdb-jni-smoke-ok$' <<< "$VERSION_OUTPUT" +done +if MAJOR_OUTPUT=$(run_smoke MOCK_ARCH=arm64 EXPECTED_ARCH=arm64 \ + EXPECTED_JAVA_MAJOR=21.0.1 MOCK_JAVA_VERSION=17.0.20 2>&1); then + echo "$MAJOR_OUTPUT" >&2 + echo "ARM smoke accepted a mismatched Java major" >&2 + exit 1 +fi +grep -Fq 'Expected Java 21, got 17.0.20' <<< "$MAJOR_OUTPUT" + for JAVA_MAJOR_MISMATCH in 11.0.31 21.0.8; do if MAJOR_OUTPUT=$(run_smoke \ "MOCK_JAVA_VERSION=$JAVA_MAJOR_MISMATCH" 2>&1); then @@ -152,4 +181,130 @@ grep -Fq \ 'Expected RISC-V Java vendor Eclipse Adoptium, got Unknown Vendor' \ <<< "$VENDOR_OUTPUT" +# Exercise the native wrapper through restart using controlled service commands. +# An isolated PATH makes the gtimeout fallback independent of the host platform. +RUNTIME_BIN="$WORK_DIR/runtime-bin" +RUNTIME_TRAVIS="$WORK_DIR/travis" +RUNTIME_LOG="$WORK_DIR/runtime.log" +START_LOG="$WORK_DIR/start.log" +STOP_LOG="$WORK_DIR/stop.log" +PHASE_LOG="$WORK_DIR/phase.log" +RUN_ID_FILE="$WORK_DIR/run-id" +mkdir -p "$RUNTIME_BIN" "$RUNTIME_TRAVIS" "$WORK_DIR/tmp" +for command in dirname date mktemp rm tee grep awk cut getconf; do + ln -s "$(command -v "$command")" "$RUNTIME_BIN/$command" +done +cp "$MOCK_BIN/uname" "$RUNTIME_BIN/uname" +cp "$NATIVE_SMOKE_SCRIPT" "$SMOKE_SCRIPT" "$RUNTIME_TRAVIS/" + +cat > "$RUNTIME_BIN/timeout" <<'EOF' +#!/bin/bash +set -euo pipefail +printf '%s' "${0##*/}" >> "$RUNTIME_LOG" +printf ' <%s>' "$@" >> "$RUNTIME_LOG" +printf '\n' >> "$RUNTIME_LOG" +if [[ "${MOCK_START_TIMEOUT:-false}" == true && "$4" == */start-hugegraph.sh ]]; then + exit 124 +fi +shift 3 +exec "$@" +EOF + +cat > "$SERVER_DIR/bin/init-store.sh" <<'EOF' +#!/bin/bash +exit 0 +EOF + +cat > "$SERVER_DIR/bin/start-hugegraph.sh" <<'EOF' +#!/bin/bash +set -euo pipefail +printf 'arch=%s' "$EXPECTED_ARCH" >> "$START_LOG" +printf ' <%s>' "$@" >> "$START_LOG" +printf '\n' >> "$START_LOG" +EOF + +cat > "$SERVER_DIR/bin/stop-hugegraph.sh" <<'EOF' +#!/bin/bash +printf '%s\n' "$*" >> "$STOP_LOG" +exit "${MOCK_STOP_STATUS:-0}" +EOF + +cat > "$SERVER_DIR/bin/dump-store.sh" <<'EOF' +#!/bin/bash +set -euo pipefail +RUN_ID=$(cat "$RUN_ID_FILE") +printf '%s\n' 'Dump table VERTEX (offset 0 limit 20):' \ + "riscv-smoke-v1-$RUN_ID" "riscv-smoke-v2-$RUN_ID" +EOF + +cat > "$RUNTIME_TRAVIS/run-server-e2e-smoke-test.sh" <<'EOF' +#!/bin/bash +set -euo pipefail +printf '%s\n' "$2" >> "$PHASE_LOG" +printf '%s\n' "$3" > "$RUN_ID_FILE" +printf 'server-e2e-smoke-%s-ok\n' "$2" +EOF + +ln -s "$(command -v cat)" "$RUNTIME_BIN/cat" +chmod +x "$RUNTIME_BIN/timeout" "$SERVER_DIR/bin/"*.sh \ + "$RUNTIME_TRAVIS/"*.sh + +run_restart_smoke() { + rm -f "$RUNTIME_LOG" "$START_LOG" "$STOP_LOG" "$PHASE_LOG" "$RUN_ID_FILE" + env -u LD_PRELOAD -u SERVER_JAVA_OPTIONS \ + PATH="$RUNTIME_BIN" JAVA_HOME="$MOCK_JAVA_HOME" TMPDIR="$WORK_DIR/tmp" \ + RUNTIME_LOG="$RUNTIME_LOG" START_LOG="$START_LOG" STOP_LOG="$STOP_LOG" \ + PHASE_LOG="$PHASE_LOG" RUN_ID_FILE="$RUN_ID_FILE" \ + "$@" "$RUNTIME_TRAVIS/run-native-runtime-smoke-test.sh" "$SERVER_DIR" +} + +if ! RESTART_OUTPUT=$(run_restart_smoke SERVER_STARTUP_TIMEOUT=7 STORE_DUMP_TIMEOUT=9 2>&1); then + echo "$RESTART_OUTPUT" >&2 + echo "RISC-V native smoke failed the controlled restart" >&2 + exit 1 +fi +grep -q '^native-runtime-smoke-ok$' <<< "$RESTART_OUTPUT" +[[ $(wc -l < "$START_LOG") -eq 2 ]] +[[ $(wc -l < "$STOP_LOG") -eq 2 ]] +grep -q '^arch=riscv64 <-t> <7>$' "$START_LOG" +grep -Fq 'timeout <--foreground> <--kill-after=15s> <37s>' "$RUNTIME_LOG" +grep -Fq 'timeout <--foreground> <--kill-after=15s> <9s>' "$RUNTIME_LOG" +[[ $(cat "$PHASE_LOG") == $'create\nverify' ]] + +mv "$RUNTIME_BIN/timeout" "$RUNTIME_BIN/gtimeout" +if ! ARM_OUTPUT=$(run_restart_smoke MOCK_ARCH=arm64 EXPECTED_ARCH=arm64 \ + 'SERVER_JAVA_OPTIONS=-Xms512m -Xmx2g' 2>&1); then + echo "$ARM_OUTPUT" >&2 + echo "ARM native smoke failed the controlled restart" >&2 + exit 1 +fi +grep -q '^native-runtime-smoke-ok$' <<< "$ARM_OUTPUT" +[[ $(grep -c '^arch=arm64 <-t> <300> <-j> <-Xms512m -Xmx2g>$' "$START_LOG") -eq 2 ]] +[[ $(grep -c '^gtimeout ' "$RUNTIME_LOG") -eq 3 ]] +[[ $(cat "$PHASE_LOG") == $'create\nverify' ]] + +set +e +TIMEOUT_OUTPUT=$(run_restart_smoke MOCK_ARCH=arm64 EXPECTED_ARCH=arm64 \ + MOCK_START_TIMEOUT=true 2>&1) +TIMEOUT_STATUS=$? +set -e +if [[ $TIMEOUT_STATUS -ne 124 || -e "$PHASE_LOG" || $(wc -l < "$STOP_LOG") -ne 1 ]]; then + echo "$TIMEOUT_OUTPUT" >&2 + echo "Native startup timeout did not preserve failure and clean up" >&2 + exit 1 +fi + +rm "$RUNTIME_BIN/gtimeout" +set +e +MISSING_TIMEOUT_OUTPUT=$(run_restart_smoke 2>&1) +MISSING_TIMEOUT_STATUS=$? +set -e +[[ $MISSING_TIMEOUT_STATUS -eq 1 ]] +grep -Fq 'Required command is unavailable: timeout or gtimeout' <<< "$MISSING_TIMEOUT_OUTPUT" +[[ $(wc -l < "$STOP_LOG") -eq 1 ]] +for TEMP_DIR in "$WORK_DIR/tmp"/hugegraph-native-runtime-smoke.*; do + [[ ! -e "$TEMP_DIR" ]] +done + echo "PASS: RISC-V Java runtime contract" +echo "PASS: native restart, ARM heap, timeout and cleanup contracts"