diff --git a/.asf.yaml b/.asf.yaml index 39b9cb32bf..086a7bf515 100644 --- a/.asf.yaml +++ b/.asf.yaml @@ -38,12 +38,8 @@ github: strict: false # contexts are the names of checks that must pass (now only enable the basic check) contexts: - - Analyze (java) - - CodeQL - check-license - # Require the fixed memory result gate, independent of the runtime JDK list. - # Commons CI continues to run without blocking merges. - - Server memory tests + - affected-module-tests required_pull_request_reviews: dismiss_stale_reviews: true require_code_owner_reviews: false diff --git a/.github/scripts/check-docker-images.sh b/.github/scripts/check-docker-images.sh new file mode 100644 index 0000000000..15ac6f9195 --- /dev/null +++ b/.github/scripts/check-docker-images.sh @@ -0,0 +1,81 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +set -euo pipefail +: "${IMAGE_TAG:?set the unique CI image tag}" +for pair in \ + hugegraph-pd/Dockerfile=hugegraph/pd \ + hugegraph-store/Dockerfile=hugegraph/store \ + hugegraph-server/Dockerfile=hugegraph/hugegraph \ + hugegraph-server/Dockerfile-hstore=hugegraph/server; do + ( + dockerfile="${pair%%=*}" + IMAGE_ID=$(docker image inspect --format '{{.Id}}' "${pair#*=}:${IMAGE_TAG}") + echo "Checking ${pair#*=}:${IMAGE_TAG} ($IMAGE_ID)" + HC=$(docker image inspect --format '{{json .Config.Healthcheck}}' "$IMAGE_ID") + [[ "$HC" != null ]] || { echo "Missing HEALTHCHECK: $dockerfile"; exit 1; } + + JAVA_VERSION=$(docker run --rm --pull never --entrypoint java "$IMAGE_ID" -version 2>&1) + echo "$JAVA_VERSION" + grep -Eq 'version "17\.' <<< "$JAVA_VERSION" || { + echo "ERROR: expected a Java 17 runtime in $dockerfile" + exit 1 + } + + if [[ "$dockerfile" == hugegraph-server/* ]]; then + STATE=$(docker run --rm --pull never "$IMAGE_ID" bash -c \ + 'source /hugegraph-server/bin/util.sh && port_listen_state 8080') + echo "port_listen_state 8080 -> $STATE" + # Require a usable answer from the socket-table tool in the image. + [[ "$STATE" == "free" || "$STATE" == "busy" ]] || { + echo "ERROR: no usable socket-table tool (ss/netstat) in $dockerfile" + exit 1 + } + fi + + if [[ "$dockerfile" == hugegraph-server/* ]]; then + CHECK_DIR=$(mktemp -d) + trap 'rm -rf "$CHECK_DIR"' EXIT + docker run --rm --pull never --entrypoint bash \ + -v "$CHECK_DIR:/check" "$IMAGE_ID" -c \ + 'cp /hugegraph-server/lib/hugegraph-api-*.jar \ + /hugegraph-server/lib/hugegraph-common-*.jar /check/' + + API_JAR=$(find "$CHECK_DIR" -name 'hugegraph-api-*.jar' -print -quit) + COMMON_JAR=$(find "$CHECK_DIR" -name 'hugegraph-common-*.jar' -print -quit) + EXPECTED_MANIFEST=$(sed -n \ + 's|.*\([^<]*\).*|\1|p' \ + hugegraph-server/hugegraph-api/pom.xml) + ACTUAL_MANIFEST=$(unzip -p "$API_JAR" META-INF/MANIFEST.MF | + sed -n 's/^Implementation-Version: *//p' | tr -d '\r') + EXPECTED_PROPERTY=$(sed -n 's/^ApiVersion=//p' \ + hugegraph-commons/hugegraph-common/src/main/resources/version.properties) + ACTUAL_PROPERTY=$(unzip -p "$COMMON_JAR" version.properties | + sed -n 's/^ApiVersion=//p' | tr -d '\r') + + [[ "$ACTUAL_MANIFEST" == "$EXPECTED_MANIFEST" ]] || { + echo "ERROR: API manifest is $ACTUAL_MANIFEST; expected $EXPECTED_MANIFEST" + exit 1 + } + [[ "$ACTUAL_PROPERTY" == "$EXPECTED_PROPERTY" ]] || { + echo "ERROR: API property is $ACTUAL_PROPERTY; expected $EXPECTED_PROPERTY" + exit 1 + } + fi + ) +done diff --git a/.github/scripts/check-rerun.py b/.github/scripts/check-rerun.py new file mode 100644 index 0000000000..e5134b72ea --- /dev/null +++ b/.github/scripts/check-rerun.py @@ -0,0 +1,78 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Check the failed run again immediately before an automatic retry.""" + +import argparse +import json +import os +import subprocess +from urllib.parse import urlencode + + +def api(path): + result = subprocess.run( + ["gh", "api", "--method", "GET", path], check=True, + capture_output=True, timeout=30, + ) + return json.loads(result.stdout) + + +def decide(repository, run_id, expected_attempt, max_reruns, fetch=api): + run = fetch(f"repos/{repository}/actions/runs/{run_id}") + if (run.get("status") != "completed" or run.get("conclusion") != "failure" + or run.get("run_attempt") != expected_attempt): + return "skip", "source run changed or is no longer a completed failure" + if expected_attempt > max_reruns: + return "skip", "retry limit reached" + sha = run.get("head_sha") + if not sha: + return "skip", "missing head SHA" + if run.get("event") == "pull_request": + return "skip", "PR automatic retry paused: CI artifacts cannot independently prove the tested code" + if run.get("event") == "push": + if (run.get("repository", {}).get("full_name") != repository + or run.get("head_repository", {}).get("full_name") != repository): + return "skip", "push source repository differs" + branch = run.get("head_branch") + if not branch: + return "skip", "missing push branch" + # Read from the workflow repository, never a similarly named fork branch. + commits = fetch(f"repos/{repository}/commits?" + urlencode({"sha": branch, "per_page": 1})) + if commits and commits[0].get("sha") == sha: + return "rerun", "push branch head unchanged" + return "skip", "push branch head moved" + return "skip", "unsupported source event" + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--repository", required=True) + parser.add_argument("--run-id", required=True, type=int) + parser.add_argument("--run-attempt", required=True, type=int) + parser.add_argument("--max-reruns", required=True, type=int) + args = parser.parse_args() + try: + action, reason = decide(args.repository, args.run_id, args.run_attempt, args.max_reruns) + except (subprocess.SubprocessError, OSError, ValueError, KeyError, TypeError, AttributeError): + action, reason = "skip", "unable to verify current run and source metadata" + print(f"{action}: {reason}") + with open(os.environ["GITHUB_OUTPUT"], "a", encoding="utf-8") as output: + output.write(f"action={action}\nreason={reason}\n") + + +if __name__ == "__main__": + main() diff --git a/.github/scripts/ci-policy.py b/.github/scripts/ci-policy.py new file mode 100644 index 0000000000..b35b20d2bd --- /dev/null +++ b/.github/scripts/ci-policy.py @@ -0,0 +1,274 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Conservative CI selection and gating of this run's selected tests.""" + +import argparse +import json +import os +from pathlib import Path +import subprocess + +MODULES = { + "server": ["server", "commons", "struct", "pd", "store", "hstore", "cluster", "docker", "helm", "dependency_license"], + "toolchain": ["client", "loader", "tools", "spark", "hubble", "go"], +} +PREFIXES = { + "server": {"hugegraph-server/": "server", "hugegraph-commons/": "commons", + "hugegraph-struct/": "struct", "hugegraph-pd/": "pd", "hugegraph-store/": "store", + "hugegraph-cluster-test/": "cluster", "docker/": "docker", "helm/": "helm"}, + "toolchain": {"hugegraph-client/": "client", "hugegraph-client-go/": "go", + "hugegraph-loader/": "loader", "hugegraph-tools/": "tools", + "hugegraph-spark-connector/": "spark", "hugegraph-hubble/": "hubble"}, +} +WORKFLOWS = { + "server": {"server-ci.yml": MODULES["server"], "commons-ci.yml": ["commons"], + "pd-store-ci.yml": ["pd", "store", "hstore"], "cluster-test-ci.yml": ["cluster"], + "docker-build-ci.yml": ["docker"], "helm-chart-ci.yml": ["helm"], + "codeql-analysis.yml": [], "riscv64-ci.yml": ["server"], "check-dependencies.yml": ["dependency_license"]}, + "toolchain": {"client-ci.yml": ["client"], "client-go-ci.yml": ["go"], + "loader-ci.yml": ["loader"], "tools-ci.yml": ["tools"], + "spark-connector-ci.yml": ["spark"], "hubble-ci.yml": ["hubble"], + "codeql-analysis.yml": []}, +} +DEPENDENTS = { + "server": {"commons": ["server", "pd", "store", "hstore", "cluster"], + "struct": ["server", "pd", "store", "hstore", "cluster"], + "server": ["pd", "store", "hstore", "cluster"], "pd": ["store", "hstore", "cluster"], + "store": ["hstore", "cluster"], "hstore": ["cluster"]}, + "toolchain": {"client": ["loader", "tools", "spark", "hubble"], "loader": ["hubble"]}, +} + + +def git(*args): + return subprocess.check_output(["git", *args], text=True).strip() + + +def api(path): + result = subprocess.run(["gh", "api", "--method", "GET", path], check=True, capture_output=True, timeout=30) + return json.loads(result.stdout) + + +def documentation(path, mode="100644"): + """Only prose and static documentation assets; never source or config.""" + if mode != "100644": + return False + p = Path(path) + prose_names = {"README.md", "README_CN.md", "README_ZH.md", "AGENTS.md"} + module_roots = {prefix.rstrip("/") for mapping in PREFIXES.values() for prefix in mapping} + module_roots.update("hugegraph-server/" + module for module in [ + "hugegraph-api", "hugegraph-core", "hugegraph-dist", "hugegraph-example", + "hugegraph-hbase", "hugegraph-hstore", "hugegraph-rocksdb", "hugegraph-test"]) + if p.name in prose_names and (str(p.parent) == "." or str(p.parent) in module_roots): + return True + return path.startswith("docs/") and p.suffix.lower() in {".md", ".rst", ".txt", ".png", ".jpg", ".jpeg", ".svg"} + + +def unsafe_documentation(ref, paths): + entries = subprocess.check_output(["git", "ls-tree", "-rz", "--full-tree", ref]).split(b"\0") + for entry in entries: + if not entry: + continue + metadata, name = entry.split(b"\t", 1) + path = name.decode("utf-8", "surrogateescape") + if path in paths and documentation(path) and metadata.split(b" ", 1)[0] != b"100644": + return True + return False + + +def select(project, paths): + selected = set() + for path in paths: + if documentation(path): + continue + if project == "server" and (Path(path).name == "pom.xml" or path.startswith("install-dist/")): + selected.add("dependency_license") + if path.startswith(".github/workflows/") and Path(path).name in WORKFLOWS[project]: + selected.update(WORKFLOWS[project][Path(path).name]) + continue + if path.endswith(".proto"): + return set(MODULES[project]) + if project == "server" and path == "hugegraph-commons/hugegraph-common/src/main/resources/version.properties": + selected.update(["commons", "docker"]) + continue + if project == "toolchain" and path.startswith("hugegraph-client/assembly/travis/"): + selected.update(["client", "go"]) + continue + if project == "server": + if path.startswith(("hugegraph-pd/hg-pd-dist/", "hugegraph-store/hg-store-dist/")): + selected.add("docker") + if path.startswith("hugegraph-server/hugegraph-hstore/"): + selected.update(["server", "hstore"]) + continue + if path.startswith("hugegraph-server/hugegraph-dist/"): + selected.update(["server", "pd", "store", "hstore", "docker", "cluster"]) + continue + if path.startswith(("hugegraph-server/hugegraph-core/", "hugegraph-server/hugegraph-api/", + "hugegraph-server/hugegraph-test/")): + selected.update(["server", "hstore"]) + continue + module = next((value for prefix, value in PREFIXES[project].items() if path.startswith(prefix)), None) + if module is None or path.endswith(".proto"): + return set(MODULES[project]) + selected.add(module) + if project == "server" and ("Dockerfile" in Path(path).name or "docker-entrypoint" in path): + selected.add("docker") + changed = True + while changed: + before = set(selected) + for module in before: + selected.update(DEPENDENTS[project].get(module, [])) + changed = before != selected + return selected + + +def suites(project, selected): + if project == "toolchain": + return sorted(selected) + result = [] + if "server" in selected: + result.extend(["server_memory", "server_rocksdb"]) + # Third-party inventory/review remains visible without becoming a merge gate. + for module in ["commons", "cluster", "docker", "helm"]: + if module in selected: + result.append(module) + if selected.intersection({"pd", "store", "hstore", "struct"}): + result.append("pd_store") + return sorted(result) + + +class StaleInputError(RuntimeError): + """The checkout no longer represents the current PR inputs.""" + + +def require_current_pr(plan, fetch): + if not plan["pr"]: + return + live = fetch(f"repos/{plan['repository']}/pulls/{plan['pr']}") + if (live.get("state") != "open" or live["head"]["sha"] != plan["head"] + or live["base"]["sha"] != plan["base"] + or live["head"]["repo"]["full_name"] != plan["source"] + or live["head"]["ref"] != plan["branch"] + or live["base"]["repo"]["full_name"] != plan["repository"]): + raise StaleInputError("PR inputs changed; refresh the branch and start a new PR run") + + +def create_plan(project, event, repository, fetch=api): + plan = {"schema": 1, "project": project, "repository": repository, "pr": 0, + "source": repository, "branch": "", "base": "", "head": git("rev-parse", "HEAD"), + "reason": "affected inputs", "testedMergeSHA": git("rev-parse", "HEAD")} + try: + pr = event.get("pull_request") + if "pull_request" in event: + if not isinstance(pr, dict) or type(pr.get("number")) is not int or pr["number"] <= 0: + raise StaleInputError("PR event lacks a valid number") + plan["pr"] = pr["number"] + # An incomplete PR event must never fall back to a push plan with pr=0. + try: + plan.update(source=pr["head"]["repo"]["full_name"], base=pr["base"]["sha"], + head=pr["head"]["sha"], branch=pr["head"]["ref"]) + except (KeyError, TypeError) as error: + raise StaleInputError("PR event lacks required input identity") from error + if not all(isinstance(plan[key], str) and plan[key] for key in ("source", "base", "head", "branch")): + raise StaleInputError("PR event has an empty input identity") + parents = git("show", "-s", "--format=%P", plan["testedMergeSHA"]).split() + if parents != [plan["base"], plan["head"]]: + raise StaleInputError("checkout is not the event PR merge; start a new PR run") + require_current_pr(plan, fetch) + # head/base objects must exist locally; workflow fetches both before planning. + ancestor = git("merge-base", plan["base"], plan["head"]) + paths = git("diff", "--no-renames", "--name-only", ancestor, plan["head"]).splitlines() + else: + plan["base"] = event.get("before", "") + paths = git("diff", "--no-renames", "--name-only", plan["base"], plan["head"]).splitlines() + selected = select(project, paths) + if unsafe_documentation(plan["base"], paths) or unsafe_documentation(plan["testedMergeSHA"], paths): + selected = set(MODULES[project]) + plan["expected"] = suites(project, selected) + except (subprocess.SubprocessError, OSError, ValueError, KeyError, TypeError, AttributeError): + selected = set(MODULES[project]) + plan["reason"] = "verification unavailable: full required coverage" + plan["expected"] = suites(project, selected) + if not selected: + plan["reason"] = "cumulative PR diff contains only plain prose documentation; modules unaffected" + plan["selected"] = sorted(selected) + for module in MODULES[project]: + plan[module] = module in selected + plan["pd_store"] = any(plan.get(m, False) for m in ["pd", "store", "hstore", "struct"]) + plan["needsFixture"] = project == "toolchain" and any(plan.get(m, False) for m in MODULES[project]) + plan["fixture"] = plan["needsFixture"] + plan["compatibility"] = project == "server" and bool(selected.intersection({"server", "commons", "struct", "pd", "store", "hstore", "cluster"})) + plan["security"] = bool(selected) or any(p.startswith(".github/workflows/codeql") for p in locals().get("paths", [])) + plan["security_languages"] = json.dumps(["java"]) + return plan + + +def gate(plan, results, fetch=None): + if results.get("plan", {}).get("result") != "success": + raise ValueError("planner did not succeed") + for suite in plan["expected"]: + if results.get(suite, {}).get("result") != "success": + raise ValueError("selected suite did not succeed: " + suite) + if plan["project"] == "toolchain": + if set(plan["expected"]).intersection({"client", "loader", "tools", "spark", "go"}): + if results.get("fixture", {}).get("result") != "success": + raise ValueError("selected tests lack successful fixture") + if "hubble" in plan["expected"] and results.get("hubble-fixture", {}).get("result") != "success": + raise ValueError("selected Hubble tests lack successful baseline fixture") + require_current_pr(plan, fetch or api) + report = {key: plan[key] for key in ["schema", "repository", "project", "pr", "source", "branch", "base", + "head", "testedMergeSHA", "selected"]} + report["runID"] = int(os.environ.get("GITHUB_RUN_ID", "0")) + report["runAttempt"] = int(os.environ.get("GITHUB_RUN_ATTEMPT", "1")) + report["executed"] = plan["expected"] + report["results"] = {suite: results[suite]["result"] for suite in plan["expected"]} + summary = "Selection: " + plan.get("reason", "affected inputs") + "\nExecuted: " + ", ".join(plan["expected"]) + if os.environ.get("GITHUB_STEP_SUMMARY"): + with open(os.environ["GITHUB_STEP_SUMMARY"], "a", encoding="utf-8") as out: + out.write("## Affected module tests\n" + summary + "\n\nUnselected modules: " + + ", ".join(sorted(set(MODULES[plan["project"]]) - set(plan["selected"]))) + "\n") + print(summary) + return report + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + sub = parser.add_subparsers(dest="command", required=True) + p = sub.add_parser("plan") + p.add_argument("--project", choices=MODULES, required=True) + p.add_argument("--event-path", required=True) + p.add_argument("--repository", required=True) + p.add_argument("--output", required=True) + p = sub.add_parser("gate") + p.add_argument("--plan", required=True) + p.add_argument("--results", required=True) + p.add_argument("--output", required=True) + args = parser.parse_args() + if args.command == "plan": + value = create_plan(args.project, json.loads(Path(args.event_path).read_text()), args.repository) + if os.environ.get("GITHUB_OUTPUT"): + with open(os.environ["GITHUB_OUTPUT"], "a", encoding="utf-8") as output: + for key, item in value.items(): + if isinstance(item, bool): + output.write(f"{key}={str(item).lower()}\n") + output.write(f"security_languages={value['security_languages']}\nplan_file={args.output}\n") + else: + value = gate(json.loads(Path(args.plan).read_text()), json.loads(Path(args.results).read_text())) + Path(args.output).write_text(json.dumps(value, indent=2) + "\n") + + +if __name__ == "__main__": + main() diff --git a/.github/scripts/docker-deployment.py b/.github/scripts/docker-deployment.py new file mode 100644 index 0000000000..3b1283b1e5 --- /dev/null +++ b/.github/scripts/docker-deployment.py @@ -0,0 +1,146 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Start the four locally built PR images using the shipped Compose files.""" + +import base64 +import json +import os +from pathlib import Path +import subprocess +import sys +import urllib.error +import urllib.request + + +ADMIN_PASSWORD = "ci-compose-password" +PD_PASSWORD = "ci-compose-pd-secret" + + +def command(*args): + return subprocess.check_output(args, text=True, timeout=60).strip() + + +def verify_image(container, image, expected): + actual = command("docker", "inspect", "--format", "{{.Image}}", container) + if not expected or actual != expected: + raise RuntimeError(f"Container {container} uses {actual}; expected {image} ({expected})") + + +def response(url, credentials=None): + request = urllib.request.Request(url) + if credentials: + encoded = base64.b64encode(credentials.encode()).decode() + request.add_header("Authorization", "Basic " + encoded) + try: + with urllib.request.urlopen(request, timeout=15) as result: + return result.status, result.read().decode() + except urllib.error.HTTPError as error: + return error.code, error.read().decode() + + +def expect_response(url, expected=200, credentials=None): + actual, body = response(url, credentials) + if actual != expected: + raise RuntimeError(f"{url}: HTTP {actual}, expected {expected}") + return body + + +def verify_server(): + url = "http://localhost:8080" + expect_response(url + "/graphspaces/DEFAULT/graphs", 401) + graphs = json.loads(expect_response(url + "/graphspaces/DEFAULT/graphs", + credentials=f"admin:{ADMIN_PASSWORD}")) + names = graphs.get("graphs") if isinstance(graphs, dict) else None + if not isinstance(names, list) or "hugegraph" not in names: + raise RuntimeError("Server did not return its initialized hugegraph in the graphs array") + payload = json.loads(expect_response(url + "/versions")) + versions = payload.get("versions") if isinstance(payload, dict) else None + if not isinstance(versions, dict) or versions.get("version") != "v1" or any( + not isinstance(versions.get(key), str) or not versions[key] + for key in ("core", "gremlin", "api")): + raise RuntimeError("Server did not return the expected versions object") + + +def verify_storage(): + for port in (8620, 8520): + if expect_response(f"http://localhost:{port}/v1/health") != "": + raise RuntimeError(f"Port {port} did not return the PD/Store health response") + ready = json.loads(expect_response("http://localhost:8620/v1/ready")) + if not isinstance(ready, dict) or ready.get("ready") is not True: + raise RuntimeError("PD did not report ready=true") + url = "http://localhost:8620/v1/stores" + expect_response(url, 401) + payload = json.loads(expect_response(url, credentials=f"hg:{PD_PASSWORD}")) + data = payload.get("data") if isinstance(payload, dict) else None + if (not isinstance(data, dict) or payload.get("status") != 0 or + not isinstance(data.get("stores"), list) or not data["stores"]): + raise RuntimeError("Authenticated PD did not return its registered stores array") + + +def smoke(tag, topology, images): + root = Path(__file__).resolve().parents[2] + project = f"hg-pr-{tag}-{Path(topology).stem}" + compose = ["docker", "compose", "-p", project, "-f", str(root / "docker" / topology)] + # Capture identities before startup so a tag change cannot bless an unrelated container. + expected = {service: command("docker", "image", "inspect", "--format", "{{.Id}}", + f"{repository}:{tag}") for service, repository in images.items()} + failed = False + try: + subprocess.run(compose + ["up", "-d", "--pull", "never", "--wait", + "--wait-timeout", "600"] + list(images), check=True, timeout=660) + for service, repository in images.items(): + container = command(*compose, "ps", "-q", service) + if not container or "\n" in container: + raise RuntimeError(f"Expected exactly one {service} container") + verify_image(container, f"{repository}:{tag}", expected[service]) + health = command("docker", "inspect", "--format", "{{.State.Health.Status}}", container) + if health != "healthy": + raise RuntimeError(f"{service} is {health}") + verify_server() + if "pd" in images: + verify_storage() + except BaseException: + failed = True + for args in (["ps"], ["logs", "--no-color", "--tail", "200"]): + try: + subprocess.run(compose + args, check=False, timeout=60) + except (OSError, subprocess.SubprocessError) as error: + print(f"Could not collect Compose diagnostics: {error}", file=sys.stderr) + raise + finally: + try: + subprocess.run(compose + ["down", "-v", "--remove-orphans"], check=True, timeout=120) + except (OSError, subprocess.SubprocessError) as error: + if not failed: + raise + print(f"Compose cleanup also failed: {error}", file=sys.stderr) + + +def main(tag): + if not tag.startswith("ci-") or any(c not in "abcdefghijklmnopqrstuvwxyz0123456789-" for c in tag): + raise ValueError("Use a unique ci- image tag") + os.environ.update(HUGEGRAPH_VERSION=tag, HUGEGRAPH_PULL_POLICY="never", + HUGEGRAPH_ADMIN_PASSWORD=ADMIN_PASSWORD, + HUGEGRAPH_AUTH_TOKEN_SECRET="0123456789abcdef" * 4, + HG_PD_AUTH_SECRET_KEY=PD_PASSWORD) + smoke(tag, "docker-compose.yml", {"server": "hugegraph/hugegraph"}) + smoke(tag, "docker-compose-hstore.yml", {"pd": "hugegraph/pd", "store": "hugegraph/store", + "server": "hugegraph/server"}) + + +if __name__ == "__main__": + main(sys.argv[1]) diff --git a/.github/scripts/test_check_rerun.py b/.github/scripts/test_check_rerun.py new file mode 100644 index 0000000000..f2823ce581 --- /dev/null +++ b/.github/scripts/test_check_rerun.py @@ -0,0 +1,112 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Behavioral tests for push retry freshness, without credentials or network.""" +import importlib.util +from pathlib import Path +import unittest +import os +import subprocess +import tempfile +from unittest.mock import patch + +spec = importlib.util.spec_from_file_location("check_rerun", Path(__file__).with_name("check-rerun.py")) +checker = importlib.util.module_from_spec(spec) +spec.loader.exec_module(checker) + + +class FreshnessTest(unittest.TestCase): + def setUp(self): + self.repository = "apache/hugegraph" + self.run = {"status": "completed", "conclusion": "failure", "run_attempt": 1, + "event": "push", "head_sha": "old", "head_branch": "release/test", + "repository": {"full_name": self.repository}, + "head_repository": {"full_name": self.repository}} + self.current_head = "old" + self.calls = [] + + def fetch(self, path): + self.calls.append(path) + if path == f"repos/{self.repository}/actions/runs/42": + return self.run + if path == f"repos/{self.repository}/commits?sha=release%2Ftest&per_page=1": + return [{"sha": self.current_head}] if self.current_head else [] + raise AssertionError("Unexpected lookup: " + path) + + def decide(self): + return checker.decide(self.repository, 42, 1, 2, self.fetch)[0] + + def test_api_failure_fails_closed(self): + with tempfile.TemporaryDirectory() as directory: + output = Path(directory) / "output" + with patch.dict(os.environ, {"GITHUB_OUTPUT": str(output)}), patch( + "sys.argv", ["check-rerun.py", "--repository", self.repository, + "--run-id", "42", "--run-attempt", "1", "--max-reruns", "2"]), patch.object( + checker, "decide", side_effect=subprocess.CalledProcessError(1, "gh")): + checker.main() + self.assertIn("action=skip\n", output.read_text()) + + def test_pr_is_paused_without_reading_artifacts_or_candidates(self): + self.run.update(event="pull_request", head_repository={"full_name": "alice/hugegraph"}) + self.assertEqual("skip", self.decide()) + self.assertEqual([f"repos/{self.repository}/actions/runs/42"], self.calls) + + def test_changed_attempt(self): + self.run["run_attempt"] = 2 + self.assertEqual("skip", self.decide()) + + def test_current_run_must_remain_failed_and_completed(self): + for key, value in (("status", "in_progress"), ("conclusion", "success"), + ("conclusion", "cancelled")): + with self.subTest(key=key, value=value): + old = self.run[key] + self.run[key] = value + self.assertEqual("skip", self.decide()) + self.run[key] = old + + def test_retry_limit(self): + self.run["run_attempt"] = 3 + self.assertEqual("skip", checker.decide(self.repository, 42, 3, 2, self.fetch)[0]) + self.run["run_attempt"] = 2 + self.assertEqual("rerun", checker.decide(self.repository, 42, 2, 2, self.fetch)[0]) + + def test_push_checks_repository_and_encodes_branch(self): + self.assertEqual("rerun", self.decide()) + self.assertIn(f"repos/{self.repository}/commits?sha=release%2Ftest&per_page=1", self.calls) + self.current_head = "new" + self.assertEqual("skip", self.decide()) + self.current_head = None + self.assertEqual("skip", self.decide()) + + def test_push_rejects_other_repository(self): + for field in ["repository", "head_repository"]: + with self.subTest(field=field): + self.run[field] = {"full_name": "alice/hugegraph"} + self.assertEqual("skip", self.decide()) + self.run[field] = {"full_name": self.repository} + + def test_missing_metadata_and_unsupported_event(self): + for field in ["head_sha", "head_branch", "repository", "head_repository"]: + old = self.run.pop(field) + with self.subTest(field=field): + self.assertEqual("skip", self.decide()) + self.run[field] = old + self.run["event"] = "workflow_dispatch" + self.assertEqual("skip", self.decide()) + + +if __name__ == "__main__": + unittest.main() diff --git a/.github/scripts/test_ci_policy.py b/.github/scripts/test_ci_policy.py new file mode 100644 index 0000000000..78a822902e --- /dev/null +++ b/.github/scripts/test_ci_policy.py @@ -0,0 +1,396 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Executable Git fixtures and fail-closed gate tests for CI selection.""" +import importlib.util +import json +import os +from pathlib import Path +import subprocess +import tempfile +import unittest +from unittest.mock import patch + +spec = importlib.util.spec_from_file_location("policy", Path(__file__).with_name("ci-policy.py")) +policy = importlib.util.module_from_spec(spec) +spec.loader.exec_module(policy) + +class PolicyTest(unittest.TestCase): + + def live_pr(self): + return {"state": "open", "head": {"sha": "head", "ref": "feature", "repo": {"full_name": "alice/server"}}, + "base": {"sha": "base", "repo": {"full_name": "apache/server"}}} + + def setUp(self): + # All tests stay local; successful PR gates query only this current-PR fixture. + api_mock = patch.object(policy, "api", return_value=self.live_pr()) + self.addCleanup(api_mock.stop) + self.api_mock = api_mock.start() + + def test_dependency_expansion(self): + self.assertEqual({"client", "loader", "tools", "spark", "hubble"}, + policy.select("toolchain", ["hugegraph-client/src/A.java"])) + self.assertEqual({"loader", "hubble"}, policy.select("toolchain", ["hugegraph-loader/src/A.java"])) + self.assertEqual({"server", "pd", "store", "hstore", "cluster"}, policy.select("server", ["hugegraph-server/A.java"])) + + def test_server_backend_and_startup_dependents(self): + selected = policy.select("server", ["hugegraph-server/hugegraph-hstore/src/test/HstoreTableTest.java"]) + self.assertIn("hstore", selected) + self.assertIn("pd_store", policy.suites("server", selected)) + selected = policy.select("server", ["hugegraph-server/hugegraph-dist/src/assembly/travis/test-start-hugegraph-pd.sh"]) + self.assertIn("pd_store", policy.suites("server", selected)) + self.assertIn("docker", selected) + for module in ["hugegraph-core", "hugegraph-api", "hugegraph-test"]: + selected = policy.select("server", [f"hugegraph-server/{module}/src/main/A.java"]) + self.assertIn("hstore", selected) + self.assertIn("server", selected) + self.assertEqual(set(), policy.select("server", ["hugegraph-server/hugegraph-hstore/README.md"])) + self.assertIn("hstore", policy.select("server", ["hugegraph-server/hugegraph-rocksdb/src/main/A.java"])) + + def test_audited_consumer_edges(self): + expected = {"server", "pd", "store", "hstore", "cluster"} + for path in ["hugegraph-server/hugegraph-hstore/src/A.java", "hugegraph-server/hugegraph-rocksdb/A.java"]: + self.assertTrue(expected.issubset(policy.select("server", [path]))) + self.assertNotIn("helm", policy.select("server", [path])) + self.assertEqual(set(policy.MODULES["server"]), policy.select("server", [".github/workflows/server-ci.yml"])) + self.assertTrue({"commons", "docker"}.issubset(policy.select("server", [ + "hugegraph-commons/hugegraph-common/src/main/resources/version.properties"]))) + self.assertIn("go", policy.select("toolchain", ["hugegraph-client/assembly/travis/start-hugegraph-servers.sh"])) + self.assertNotIn("go", policy.select("toolchain", ["hugegraph-client/src/A.java"])) + self.assertNotIn("server", policy.select("server", ["hugegraph-pd/src/A.java"])) + self.assertTrue({"store", "docker"}.issubset(policy.select("server", [ + "hugegraph-store/hg-store-dist/src/assembly/static/bin/util.sh"]))) + + def test_pd_store_distribution_inputs_include_docker_consumers(self): + for module, distribution, startup in [ + ("pd", "hg-pd-dist", "start-hugegraph-pd.sh"), + ("store", "hg-store-dist", "start-hugegraph-store.sh"), + ]: + for relative in [f"src/assembly/static/bin/{startup}", + "src/assembly/static/conf/application.yml", + "src/assembly/descriptor/server-assembly.xml", "pom.xml"]: + path = f"hugegraph-{module}/{distribution}/{relative}" + with self.subTest(path=path): + selected = policy.select("server", [path]) + self.assertTrue({module, "hstore", "cluster", "docker"}.issubset(selected)) + self.assertIn("pd_store", policy.suites("server", selected)) + self.assertNotIn("server", selected) + self.assertNotIn("helm", selected) + self.assertEqual(relative == "pom.xml", "dependency_license" in selected) + source = f"hugegraph-{module}/hg-{module}-core/src/main/java/Example.java" + with self.subTest(path=source): + selected = policy.select("server", [source]) + self.assertTrue({module, "hstore", "cluster"}.issubset(selected)) + self.assertNotIn("docker", selected) + + def test_version_resource_follows_real_docker_consumer(self): + root = Path(__file__).resolve().parents[2] + consumer = root / ".github/scripts/check-docker-images.sh" + if not consumer.exists(): + self.skipTest("Server Docker consumer is not in the Toolchain repository") + resource = next(line.strip().rstrip(")") for line in consumer.read_text().splitlines() + if line.strip().startswith("hugegraph-commons/") and "version.properties" in line) + self.assertTrue((root / resource).is_file()) + self.assertTrue({"commons", "docker"}.issubset(policy.select("server", [resource]))) + + def test_single_workflow_has_no_global_fanout(self): + self.assertEqual({"hubble"}, policy.select("toolchain", [".github/workflows/hubble-ci.yml"])) + self.assertEqual({"docker"}, policy.select("server", [".github/workflows/docker-build-ci.yml"])) + + def test_unknown_and_proto_fail_conservative(self): + for path in ["pom.xml", ".github/scripts/new.py", "hugegraph-pd/api.proto", "mystery"]: + self.assertEqual(set(policy.MODULES["server"]), policy.select("server", [path])) + + def test_strict_docs_allowlist(self): + self.assertEqual(set(), policy.select("server", ["README.md", "docs/guide.md"])) + for path in ["hugegraph-client/src/test/resources/README.md", "docs/type.ts", "docs/config.yml", "docs/example.java", "hugegraph-server/type.ts"]: + self.assertTrue(policy.select("server", [path])) + + def test_real_git_modes_rename_and_cli(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + def git(*args): + return subprocess.check_output(["git", *args], cwd=root, text=True).strip() + git("init", "-q") + git("config", "user.email", "ci@example.invalid") + git("config", "user.name", "CI") + (root / "README.md").write_text("initial") + git("add", ".") + git("commit", "-qm", "base") + base = git("rev-parse", "HEAD") + (root / "README.md").write_text("docs change") + git("commit", "-qam", "docs") + head = git("rev-parse", "HEAD") + old = os.getcwd() + try: + os.chdir(root) + (root / "type.ts").write_text("type changed") + git("add", ".") + git("commit", "-qm", "types") + (root / "README.md").chmod(0o755) + git("add", "README.md") + git("commit", "-qm", "executable prose") + self.assertTrue(policy.unsafe_documentation("HEAD", ["README.md"])) + (root / "README.md").unlink() + (root / "README.md").symlink_to("type.ts") + git("add", "README.md") + git("commit", "-qm", "symlink prose") + self.assertTrue(policy.unsafe_documentation("HEAD", ["README.md"])) + (root / "hugegraph-server").mkdir() + (root / "hugegraph-server" / "A.java").write_text("source") + git("add", ".") + git("commit", "-qm", "source") + before_rename = git("rev-parse", "HEAD") + (root / "docs").mkdir() + git("mv", "hugegraph-server/A.java", "docs/renamed.md") + git("commit", "-qm", "rename code to prose") + changed = git("diff", "--no-renames", "--name-only", before_rename, "HEAD").splitlines() + self.assertIn("server", policy.select("server", changed)) + finally: + os.chdir(old) + event = root / "event.json" + event.write_text(json.dumps({"before": base})) + output = root / "output" + subprocess.run(["python3", str(Path(policy.__file__).resolve()), "plan", "--project", "server", + "--repository", "apache/server", "--event-path", str(event), + "--output", str(root / "plan.json")], cwd=root, check=True, + env={**os.environ, "GITHUB_OUTPUT": str(output)}) + self.assertIn("server=true\n", output.read_text()) + self.assertEqual(policy.suites("server", set(policy.MODULES["server"])), + json.loads((root / "plan.json").read_text())["expected"]) + + def plan(self): + return {"schema": 1, "project": "server", "repository": "apache/server", "pr": 7, + "source": "alice/server", "branch": "feature", "base": "base", "head": "head", + "testedMergeSHA": "merge", "expected": ["server_memory", "server_rocksdb", "pd_store", "cluster"], + "selected": ["server", "pd", "store", "hstore", "cluster"]} + + def test_gate_rejects_missing_failed_cancelled_skipped(self): + plan = self.plan() + results = {suite: {"result": "success"} for suite in plan["expected"]} + results["plan"] = {"result": "success"} + # Even an old plan with claimed successes cannot exempt any current suite. + plan["reused"] = {suite: {"runID": 42, "testedMergeSHA": "forged"} for suite in plan["expected"]} + for suite in plan["expected"]: + for result in [None, "failure", "cancelled", "skipped"]: + with self.subTest(suite=suite, result=result), self.assertRaisesRegex(ValueError, suite): + policy.gate(plan, dict(results, **{suite: {"result": result}})) + self.assertEqual(plan["expected"], policy.gate(plan, results)["executed"]) + + def test_third_party_failure_does_not_exempt_required_tests(self): + selected = policy.select("server", ["pom.xml"]) + self.assertIn("dependency_license", selected) + expected = {"server_memory", "server_rocksdb", "commons", "pd_store", "cluster", "docker", "helm"} + self.assertEqual(expected, set(policy.suites("server", selected))) + plan = dict(self.plan(), selected=sorted(selected), expected=sorted(expected)) + results = {suite: {"result": "success"} for suite in expected} + results["plan"] = {"result": "success"} + for result in ["failure", "cancelled", "skipped", None]: + results["dependency_license"] = {"result": result} + self.assertEqual(expected, set(policy.gate(plan, results)["executed"])) + results["cluster"] = {"result": "failure"} + with self.assertRaisesRegex(ValueError, "cluster"): + policy.gate(plan, results) + + def test_incomplete_pr_event_never_becomes_a_push_plan(self): + event = {"pull_request": dict(self.live_pr(), number=7)} + cases = [] + for change in ("null_repo", "missing_repo", "missing_number", "empty_ref"): + altered = json.loads(json.dumps(event)) + pr = altered["pull_request"] + if change == "null_repo": + pr["head"]["repo"] = None + elif change == "missing_repo": + del pr["head"]["repo"] + elif change == "missing_number": + del pr["number"] + else: + pr["head"]["ref"] = "" + cases.append(altered) + cases.extend([{"pull_request": {}}, {"pull_request": None}]) + with patch.object(policy, "git", return_value="merge"): + for altered in cases: + with self.subTest(event=altered), self.assertRaises(policy.StaleInputError): + policy.create_plan('server', altered, 'apache/server', + lambda _: self.fail("Incomplete identity must fail before any API query")) + + def test_gate_refuses_failed_planner_even_empty(self): + plan = dict(self.plan(), expected=[], selected=[]) + for result in ["failure", "skipped", "cancelled", None]: + with self.subTest(result=result), self.assertRaises(ValueError): + policy.gate(plan, {"plan": {"result": result}}) + report = policy.gate(plan, {"plan": {"result": "success"}}) + self.assertEqual([], report["executed"]) + self.assertNotIn("proofs", report) + + def test_cumulative_source_and_docs_always_run_current_tests_without_historical_apis(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + def git(*args): + return subprocess.check_output(["git", *args], cwd=root, text=True).strip() + git("init", "-q") + git("config", "user.email", "ci@example.invalid") + git("config", "user.name", "CI") + (root / "README.md").write_text("base") + git("add", ".") + git("commit", "-qm", "base") + base = git("rev-parse", "HEAD") + git("checkout", "-qb", "feature") + (root / "hugegraph-server").mkdir() + source = root / "hugegraph-server/A.java" + source.write_text("source A") + git("add", ".") + git("commit", "-qm", "source") + for text in ["docs B after test failure", "docs C after claimed old success"]: + (root / "README.md").write_text(text) + git("commit", "-qam", "docs") + head = git("rev-parse", "HEAD") + # Synthetic PR merge; a different tree with the same parents also exists. + tree = git("rev-parse", "HEAD^{tree}") + merge = git("commit-tree", tree, "-p", base, "-p", head, "-m", "PR merge") + false_merge = git("commit-tree", git("rev-parse", base + "^{tree}"), + "-p", base, "-p", head, "-m", "unexecuted tree") + self.assertNotEqual(merge, false_merge) + git("checkout", "--detach", "-q", merge) + live = {"state": "open", "head": {"sha": head, "ref": "feature", "repo": {"full_name": "alice/server"}}, + "base": {"sha": base, "repo": {"full_name": "apache/server"}}} + event = {"pull_request": dict(live, number=7)} + calls = [] + def fetch(path): + calls.append(path) + self.assertEqual("repos/apache/server/pulls/7", path) + return live + old = os.getcwd() + try: + os.chdir(root) + plan = policy.create_plan("server", event, "apache/server", fetch) + finally: + os.chdir(old) + self.assertEqual(["repos/apache/server/pulls/7"], calls) + self.assertTrue(plan["server"]) + self.assertTrue(plan["pd_store"]) + self.assertTrue(plan["cluster"]) + self.assertTrue(plan["compatibility"]) + self.assertTrue(plan["security"]) + self.assertNotIn("reused", plan) + results = {suite: {"result": "success"} for suite in plan["expected"]} + results["plan"] = {"result": "success"} + results["server_memory"] = {"result": "skipped"} + with self.assertRaisesRegex(ValueError, "server_memory"): + policy.gate(plan, results) + git("checkout", "-q", "feature") + + def test_pure_docs_pr_and_metadata_fail_closed(self): + live = self.live_pr() + event = {"pull_request": dict(live, number=7)} + def git(*args): + if args[0] == "show": + return "base head" + if args[0] == "diff": + return "README.md\ndocs/ci.md" + return "merge" + with patch.object(policy, "git", side_effect=git), patch.object(policy, "unsafe_documentation", return_value=False): + plan = policy.create_plan("server", event, "apache/server", lambda p: live) + self.assertEqual([], plan["expected"]) + self.assertFalse(plan["server"]) + self.assertFalse(plan["security"]) + advanced = dict(live, base={"sha": "new-base", "repo": {"full_name": "apache/server"}}) + with self.assertRaises(policy.StaleInputError): + policy.create_plan("server", event, "apache/server", lambda p: advanced) + with patch.object(policy, "git", side_effect=lambda *a: "wrong parents" if a[0] == "show" else git(*a)): + with self.assertRaises(policy.StaleInputError): + policy.create_plan("server", event, "apache/server", lambda p: live) + + def test_api_failure_runs_full_required_without_losing_pr_identity(self): + event = {"pull_request": dict(self.live_pr(), number=7)} + def git(*args): + return "base head" if args[0] == "show" else "merge" + def fail(path): + raise subprocess.CalledProcessError(1, "gh") + with patch.object(policy, "git", side_effect=git): + plan = policy.create_plan("server", event, "apache/server", fail) + self.assertEqual(set(policy.MODULES["server"]), set(plan["selected"])) + self.assertEqual((7, "alice/server", "feature", "base", "head"), + tuple(plan[key] for key in ["pr", "source", "branch", "base", "head"])) + results = {suite: {"result": "success"} for suite in plan["expected"]} + results.update(plan={"result": "success"}, fixture={"result": "success"}, + **{"hubble-fixture": {"result": "success"}}) + with self.assertRaises(subprocess.CalledProcessError): + policy.gate(plan, results, fail) + self.assertEqual(plan["expected"], policy.gate(plan, results)["executed"]) + self.api_mock.assert_called_once_with("repos/apache/server/pulls/7") + + def test_real_pr_git_snapshot_and_final_gate_freshness(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + def git(*args): + return subprocess.check_output(["git", *args], cwd=root, text=True).strip() + git("init", "-q") + git("config", "user.email", "ci@example.invalid") + git("config", "user.name", "CI") + (root / "README.md").write_text("base") + git("add", ".") + git("commit", "-qm", "base") + base = git("rev-parse", "HEAD") + git("checkout", "-qb", "feature") + (root / "hugegraph-client").mkdir() + (root / "hugegraph-client/A.java").write_text("source") + git("add", ".") + git("commit", "-qm", "source") + head = git("rev-parse", "HEAD") + tree = git("rev-parse", "HEAD^{tree}") + merge = git("commit-tree", tree, "-p", base, "-p", head, "-m", "PR merge") + advanced_head = git("commit-tree", tree, "-p", head, "-m", "new source head") + advanced_base = git("commit-tree", git("rev-parse", base + "^{tree}"), + "-p", base, "-m", "new target base") + git("checkout", "--detach", "-q", merge) + live = self.live_pr() + live["head"]["sha"], live["base"]["sha"] = head, base + event = {"pull_request": dict(live, number=7)} + old = os.getcwd() + try: + os.chdir(root) + plan = policy.create_plan("server", event, "apache/server", lambda _: live) + results = {suite: {"result": "success"} for suite in plan["expected"]} + results.update(plan={"result": "success"}, fixture={"result": "success"}, + **{"hubble-fixture": {"result": "success"}}) + self.assertEqual(plan["expected"], policy.gate(plan, results, lambda _: live)["executed"]) + for section, field, value in [("head", "sha", advanced_head), ("base", "sha", advanced_base), + ("head", "ref", "other-branch"), + ("head", "repo", {"full_name": "other/fork"}), + ("base", "repo", {"full_name": "other/target"})]: + changed = dict(live, **{section: dict(live[section], **{field: value})}) + with self.subTest(section=section, field=field): + with self.assertRaises(policy.StaleInputError): + policy.create_plan("server", event, "apache/server", lambda _: changed) + with self.assertRaises(policy.StaleInputError): + policy.gate(plan, results, lambda _: changed) + with self.assertRaises(policy.StaleInputError): + policy.gate(plan, results, lambda _: dict(live, state="closed")) + git("checkout", "--detach", "-q", head) + with self.assertRaises(policy.StaleInputError): + policy.create_plan("server", event, "apache/server", lambda _: live) + finally: + os.chdir(old) + + def test_push_gate_does_not_query_pr(self): + plan = dict(self.plan(), pr=0, expected=[], selected=[]) + policy.gate(plan, {"plan": {"result": "success"}}, + lambda _: self.fail("push gate must not query a PR")) + + +if __name__ == "__main__": + unittest.main() diff --git a/.github/scripts/test_docker_deployment.py b/.github/scripts/test_docker_deployment.py new file mode 100644 index 0000000000..6595c13490 --- /dev/null +++ b/.github/scripts/test_docker_deployment.py @@ -0,0 +1,145 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Reject successful service checks performed against an unrelated image.""" + +import importlib.util +from pathlib import Path +import unittest +from unittest.mock import patch + +spec = importlib.util.spec_from_file_location("deployment", Path(__file__).with_name("docker-deployment.py")) +deployment = importlib.util.module_from_spec(spec) +spec.loader.exec_module(deployment) + + +class ImageIdentityTest(unittest.TestCase): + def test_rejects_old_image_even_when_service_started(self): + with patch.object(deployment, "command", return_value="sha256:old"): + with self.assertRaisesRegex(RuntimeError, "uses sha256:old"): + deployment.verify_image("container", "hugegraph/server:ci-1-1", "sha256:pr") + + def test_rejects_missing_image_identity(self): + with patch.object(deployment, "command", return_value=""): + with self.assertRaises(RuntimeError): + deployment.verify_image("container", "hugegraph/server:ci-1-1", "") + + def test_accepts_exact_built_image(self): + with patch.object(deployment, "command", return_value="sha256:pr"): + deployment.verify_image("container", "hugegraph/server:ci-1-1", "sha256:pr") + + def test_identity_failure_cleans_compose_and_never_checks_http(self): + with patch.object(deployment, "command", side_effect=["sha256:pr", "container"]), \ + patch.object(deployment, "verify_image", side_effect=RuntimeError("wrong image")), \ + patch.object(deployment.subprocess, "run") as run, \ + patch.object(deployment, "response") as response: + with self.assertRaisesRegex(RuntimeError, "wrong image"): + deployment.smoke("ci-1-1", "docker-compose.yml", {"server": "hugegraph/hugegraph"}) + calls = [call.args[0] for call in run.call_args_list] + startup = next(args for args in calls if "up" in args) + self.assertEqual(startup[-1], "server") + self.assertEqual(startup[startup.index("--pull") + 1], "never") + self.assertNotIn("hubble", startup) + self.assertTrue(any("logs" in args for args in calls)) + self.assertEqual(calls[-1][-3:], ["down", "-v", "--remove-orphans"]) + response.assert_not_called() + + def test_startup_timeout_preserves_failure_and_cleans_compose(self): + timeout = deployment.subprocess.TimeoutExpired("up", 660) + cleanup = deployment.subprocess.CalledProcessError(1, "down") + with patch.object(deployment, "command", return_value="sha256:pr"), \ + patch.object(deployment.subprocess, "run", side_effect=[timeout, None, None, cleanup]) as run: + with self.assertRaises(deployment.subprocess.TimeoutExpired): + deployment.smoke("ci-1-1", "docker-compose.yml", {"server": "hugegraph/hugegraph"}) + self.assertEqual(run.call_args_list[-1].args[0][-3:], ["down", "-v", "--remove-orphans"]) + + def test_hstore_starts_only_pr_services_and_cleans_after_checks(self): + images = {"pd": "hugegraph/pd", "store": "hugegraph/store", "server": "hugegraph/server"} + identities = ["sha256:pd", "sha256:store", "sha256:server"] + containers = [value for service in images for value in (service, f"sha256:{service}", "healthy")] + with patch.object(deployment, "command", side_effect=identities + containers), \ + patch.object(deployment.subprocess, "run") as run, \ + patch.object(deployment, "verify_server") as server, \ + patch.object(deployment, "verify_storage") as storage: + deployment.smoke("ci-1-1", "docker-compose-hstore.yml", images) + startup = run.call_args_list[0].args[0] + self.assertEqual(startup[-3:], ["pd", "store", "server"]) + self.assertNotIn("hubble", startup) + server.assert_called_once_with() + storage.assert_called_once_with() + self.assertEqual(run.call_args_list[-1].args[0][-3:], ["down", "-v", "--remove-orphans"]) + + +class PayloadTest(unittest.TestCase): + VERSIONS = '{"versions":{"version":"v1","core":"1.7.0","gremlin":"3.7.3","api":"0.74"}}' + GRAPHS = '{"graphs":["hugegraph"]}' + + def test_accepts_public_versions_and_authenticated_graphs(self): + with patch.object(deployment, "response", side_effect=[(401, ""), (200, self.GRAPHS), + (200, self.VERSIONS)]) as response: + deployment.verify_server() + calls = response.call_args_list + self.assertIsNone(calls[0].args[1]) + self.assertEqual(calls[1].args[1], "admin:ci-compose-password") + self.assertIsNone(calls[2].args[1]) + + def test_rejects_graph_error_body_even_when_http_status_is_200(self): + for body in ('{"status":-1,"error":"Unauthorized"}', '{"graphs":"hugegraph"}', + '{"graphs":[]}'): + with self.subTest(body=body), \ + patch.object(deployment, "response", side_effect=[(401, ""), (200, body)]): + with self.assertRaisesRegex(RuntimeError, "graphs array"): + deployment.verify_server() + + def test_rejects_invalid_versions_even_when_http_status_is_200(self): + for body in ('{"status":200}', '{"versions":{"version":"v1","core":"1.7.0"}}', + '{"versions":{"version":"v1","core":"1","api":"","gremlin":"3"}}'): + with self.subTest(body=body), \ + patch.object(deployment, "response", side_effect=[(401, ""), (200, self.GRAPHS), (200, body)]): + with self.assertRaisesRegex(RuntimeError, "versions object"): + deployment.verify_server() + + def test_rejects_unauthenticated_server_access(self): + with patch.object(deployment, "response", return_value=(200, self.GRAPHS)): + with self.assertRaisesRegex(RuntimeError, "expected 401"): + deployment.verify_server() + + def test_accepts_pd_store_health_readiness_and_authenticated_registration(self): + responses = [(200, ""), (200, ""), (200, '{"ready":true}'), (401, ""), + (200, '{"status":0,"data":{"stores":[{"address":"store:8500"}]}}')] + with patch.object(deployment, "response", side_effect=responses) as response: + deployment.verify_storage() + self.assertEqual(response.call_args_list[-1].args[1], "hg:ci-compose-pd-secret") + + def test_rejects_pd_false_readiness_or_error_body(self): + for body in ('{"ready":false}', '{"ready":"true"}', '{"status":-1}'): + with self.subTest(body=body), \ + patch.object(deployment, "response", side_effect=[(200, ""), (200, ""), (200, body)]): + with self.assertRaisesRegex(RuntimeError, "ready=true"): + deployment.verify_storage() + + def test_rejects_health_error_body_even_when_http_status_is_200(self): + with patch.object(deployment, "response", return_value=(200, '{"error":"Unauthorized"}')): + with self.assertRaisesRegex(RuntimeError, "health response"): + deployment.verify_storage() + + def test_rejects_unregistered_store_or_pd_error(self): + for body in ('{"status":0,"data":{"stores":[]}}', + '{"status":-1,"data":{"stores":[{}]}}'): + responses = [(200, ""), (200, ""), (200, '{"ready":true}'), (401, ""), (200, body)] + with self.subTest(body=body), patch.object(deployment, "response", side_effect=responses): + with self.assertRaisesRegex(RuntimeError, "registered stores array"): + deployment.verify_storage() diff --git a/.github/workflows/check-dependencies.yml b/.github/workflows/check-dependencies.yml index ef9ff04fc3..73ef769c67 100644 --- a/.github/workflows/check-dependencies.yml +++ b/.github/workflows/check-dependencies.yml @@ -2,31 +2,12 @@ name: "3rd-party" on: - push: - branches: - - 'release-*' - pull_request: - # Keep source/resources and shared build inputs; skip named documentation files. - paths: - - '.github/workflows/.java-version' - - '**/pom.xml' - - '.mvn/**' - - '.github/configs/**' - - 'install-dist/**' - - '**/*.jar' - - '.github/workflows/check-dependencies.yml' - - '!**/README*' - - '!**/AGENTS.md' - - '!**/CLAUDE.md' + workflow_call: + workflow_dispatch: permissions: contents: read -# Keep the newest PR run; preserve independent push, release and manual runs. -concurrency: - group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.run_id }} - cancel-in-progress: ${{ github.event_name == 'pull_request' }} - jobs: dependency-check: runs-on: ubuntu-latest @@ -64,6 +45,7 @@ jobs: - name: 'Checkout Repository' uses: actions/checkout@v7 - name: 'Dependency Review' + if: github.event_name == 'pull_request' uses: actions/dependency-review-action@v5 # Refer: https://github.com/actions/dependency-review-action with: diff --git a/.github/workflows/cluster-test-ci.yml b/.github/workflows/cluster-test-ci.yml index c545ff8c80..91d63afdbb 100644 --- a/.github/workflows/cluster-test-ci.yml +++ b/.github/workflows/cluster-test-ci.yml @@ -1,36 +1,11 @@ name: "Cluster Test CI" on: - push: - branches: - - master - - 'release-*' - - 'test-*' - pull_request: - # Keep source/resources and shared build inputs; skip named documentation files. - paths: - - '.github/workflows/.java-version' - - 'pom.xml' - - '.mvn/**' - - 'style/**' - - '.editorconfig' - - '.github/configs/**' - - 'hugegraph-commons/**' - - 'hugegraph-struct/**' - - 'hugegraph-pd/**' - - 'hugegraph-store/**' - - 'hugegraph-server/**' - - 'hugegraph-cluster-test/**' - - 'install-dist/**' - - '.github/workflows/cluster-test-ci.yml' - - '!**/README*' - - '!**/AGENTS.md' - - '!**/CLAUDE.md' + workflow_call: + workflow_dispatch: -# Keep the newest PR run; preserve independent push, release and manual runs. -concurrency: - group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.run_id }} - cancel-in-progress: ${{ github.event_name == 'pull_request' }} +permissions: + contents: read jobs: cluster-test: diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 5b017919cf..ba82f833f4 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -1,23 +1,22 @@ name: "CodeQL" on: - push: - branches: [ master, release-* ] - pull_request: - # The branches below must be a subset of the branches above - # branches: [ master ] # enable in all PRs + workflow_call: + workflow_dispatch: schedule: - cron: '33 0 * * 5' -# Keep the newest PR run; preserve independent push, release and manual runs. -concurrency: - group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.run_id }} - cancel-in-progress: ${{ github.event_name == 'pull_request' }} +permissions: + contents: read + actions: read + security-events: write jobs: analyze: env: USE_STAGE: 'false' # Whether to include the stage repository. + # Preserve the master scan identity when invoked by a reusable-workflow caller. + CODEQL_ACTION_ANALYSIS_KEY: .github/workflows/codeql-analysis.yml:analyze name: Analyze runs-on: ubuntu-latest permissions: @@ -43,8 +42,8 @@ jobs: - name: use staged maven repo settings if: ${{ env.USE_STAGE == 'true' }} run: | - cp $HOME/.m2/settings.xml /tmp/settings.xml - mv -vf .github/configs/settings.xml $HOME/.m2/settings.xml + cp "$HOME/.m2/settings.xml" /tmp/settings.xml + mv -vf .github/configs/settings.xml "$HOME/.m2/settings.xml" # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL @@ -74,3 +73,5 @@ jobs: - name: Perform CodeQL Analysis uses: github/codeql-action/analyze@v4 + with: + category: .github/workflows/codeql-analysis.yml:analyze/language:${{ matrix.language }} diff --git a/.github/workflows/commons-ci.yml b/.github/workflows/commons-ci.yml index cdfea36d8b..7a60c038ab 100644 --- a/.github/workflows/commons-ci.yml +++ b/.github/workflows/commons-ci.yml @@ -1,18 +1,11 @@ name: "HugeGraph-Commons CI" on: + workflow_call: workflow_dispatch: - push: - branches: - - master - - 'release-*' - - 'test-*' - pull_request: -# Keep the newest PR run; preserve independent push, release and manual runs. -concurrency: - group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.run_id }} - cancel-in-progress: ${{ github.event_name == 'pull_request' }} +permissions: + contents: read jobs: java-runtime: diff --git a/.github/workflows/docker-build-ci.yml b/.github/workflows/docker-build-ci.yml index 3c87855b0c..56883b88f8 100644 --- a/.github/workflows/docker-build-ci.yml +++ b/.github/workflows/docker-build-ci.yml @@ -18,36 +18,11 @@ name: "Docker Build CI" on: - push: - branches: - - master - - 'release-*' - pull_request: - # Keep source/resources and shared build inputs; skip named documentation files. - paths: - - 'pom.xml' - - '.mvn/**' - - 'style/**' - - '.editorconfig' - - '.github/configs/**' - - 'hugegraph-commons/**' - - 'hugegraph-struct/**' - - 'hugegraph-pd/**' - - 'hugegraph-store/**' - - 'hugegraph-server/**' - - 'hugegraph-cluster-test/**' - - 'install-dist/**' - - 'docker/**' - - '.dockerignore' - - '.github/workflows/docker-build-ci.yml' - - '!**/README*' - - '!**/AGENTS.md' - - '!**/CLAUDE.md' + workflow_call: + workflow_dispatch: -# Keep the newest PR run; preserve independent push, release and manual runs. -concurrency: - group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.run_id }} - cancel-in-progress: ${{ github.event_name == 'pull_request' }} +permissions: + contents: read jobs: docker-bake-check: @@ -113,96 +88,47 @@ jobs: all(. == "docker")) ' /tmp/hugegraph-bake.json - docker-build: + deployment-contracts: runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - dockerfile: - - hugegraph-pd/Dockerfile - - hugegraph-store/Dockerfile - - hugegraph-server/Dockerfile - - hugegraph-server/Dockerfile-hstore + timeout-minutes: 25 + steps: + - uses: actions/checkout@v7 + - name: Verify deployment shell contracts + run: | + bash hugegraph-server/hugegraph-dist/docker/test/test-docker-entrypoint.sh + bash hugegraph-server/hugegraph-dist/docker/docker-entrypoint-test.sh + bash hugegraph-server/hugegraph-dist/src/assembly/travis/test-check-port.sh \ + hugegraph-server/hugegraph-dist/src/assembly/static \ + hugegraph-store/hg-store-dist/src/assembly/static/bin/util.sh + - name: Verify Compose contracts + run: bash docker/test-compose.sh render + - name: Run Compose auth-on smoke tests + run: bash docker/test-compose.sh smoke + docker-build: + runs-on: ubuntu-latest + timeout-minutes: 45 + env: + IMAGE_TAG: ci-${{ github.run_id }}-${{ github.run_attempt }} + SOURCE_REVISION: ${{ github.sha }} steps: - name: Checkout uses: actions/checkout@v7 - - name: Build ${{ matrix.dockerfile }} - run: | - # Keep the build log visible: -q would swallow Maven's output and - # leave only "exit code: 1" when the build stage fails. - docker build --progress=plain --iidfile "$RUNNER_TEMP/iid" \ - --build-arg SOURCE_REVISION="$GITHUB_SHA" \ - -f ${{ matrix.dockerfile }} . - IMAGE_ID=$(cat "$RUNNER_TEMP/iid") - echo "Built: $IMAGE_ID" - echo "IMAGE_ID=$IMAGE_ID" >> "$GITHUB_ENV" - HC=$(docker inspect --format='{{json .Config.Healthcheck}}' "$IMAGE_ID") - echo "Healthcheck: $HC" - [[ "$HC" != "null" ]] || { echo "ERROR: HEALTHCHECK missing in ${{ matrix.dockerfile }}"; exit 1; } - - - name: Verify Java 17 runtime in ${{ matrix.dockerfile }} - run: | - JAVA_VERSION=$(docker run --rm --entrypoint java "$IMAGE_ID" -version 2>&1) - echo "$JAVA_VERSION" - grep -Eq 'version "17\.' <<< "$JAVA_VERSION" || { - echo "ERROR: expected a Java 17 runtime in ${{ matrix.dockerfile }}" - exit 1 - } - - - name: Test server entrypoint property mapping - if: matrix.dockerfile == 'hugegraph-server/Dockerfile' - run: bash hugegraph-server/hugegraph-dist/docker/test/test-docker-entrypoint.sh - - # The startup preflight needs a socket-table tool, and the base image - # ships none of its own. Without one every start reports "unknown" and - # a duplicate start is no longer refused, so assert the image can - # actually answer. Only the server images run check_port. - # TODO(docker-ci): this pins the probe dependency, not the behaviour it - # protects. A full duplicate-start/stop check needs a booted server with - # a backend, which belongs with the e2e job rather than the image build. - - name: Port preflight can answer inside ${{ matrix.dockerfile }} - if: ${{ startsWith(matrix.dockerfile, 'hugegraph-server/') }} + - name: Build and load all four PR images with a shared Maven stage run: | - STATE=$(docker run --rm "$IMAGE_ID" bash -c \ - 'source /hugegraph-server/bin/util.sh && port_listen_state 8080') - echo "port_listen_state 8080 -> $STATE" - # Assert a positive answer rather than "not unknown": an empty $STATE - # would otherwise satisfy the check. The step's default `bash -e` - # already aborts on a failed `docker run`, so this only states intent. - [[ "$STATE" == "free" || "$STATE" == "busy" ]] || { - echo "ERROR: no usable socket-table tool (ss/netstat) in ${{ matrix.dockerfile }}" - exit 1 - } + docker buildx create --driver docker-container --use + docker buildx bake --file docker/bake.hcl --progress=plain \ + --set '*.platform=linux/amd64' \ + pd store server-standalone server-hstore - - name: Server image API versions match source - if: ${{ startsWith(matrix.dockerfile, 'hugegraph-server/') }} + - name: Verify image runtime contracts run: | - CHECK_DIR=$(mktemp -d) - trap 'rm -rf "$CHECK_DIR"' EXIT - docker run --rm --entrypoint bash \ - -v "$CHECK_DIR:/check" "$IMAGE_ID" -c \ - 'cp /hugegraph-server/lib/hugegraph-api-*.jar \ - /hugegraph-server/lib/hugegraph-common-*.jar /check/' + bash .github/scripts/check-docker-images.sh + bash hugegraph-server/hugegraph-dist/docker/test/test-docker-entrypoint.sh - API_JAR=$(find "$CHECK_DIR" -name 'hugegraph-api-*.jar' -print -quit) - COMMON_JAR=$(find "$CHECK_DIR" -name 'hugegraph-common-*.jar' -print -quit) - EXPECTED_MANIFEST=$(sed -n \ - 's|.*\([^<]*\).*|\1|p' \ - hugegraph-server/hugegraph-api/pom.xml) - ACTUAL_MANIFEST=$(unzip -p "$API_JAR" META-INF/MANIFEST.MF | - sed -n 's/^Implementation-Version: *//p' | tr -d '\r') - EXPECTED_PROPERTY=$(sed -n 's/^ApiVersion=//p' \ - hugegraph-commons/hugegraph-common/src/main/resources/version.properties) - ACTUAL_PROPERTY=$(unzip -p "$COMMON_JAR" version.properties | - sed -n 's/^ApiVersion=//p' | tr -d '\r') + - name: Verify deployment checker contracts + run: python3 -m unittest discover -s .github/scripts -p 'test_docker_deployment.py' - [[ "$ACTUAL_MANIFEST" == "$EXPECTED_MANIFEST" ]] || { - echo "ERROR: API manifest is $ACTUAL_MANIFEST; expected $EXPECTED_MANIFEST" - exit 1 - } - [[ "$ACTUAL_PROPERTY" == "$EXPECTED_PROPERTY" ]] || { - echo "ERROR: API property is $ACTUAL_PROPERTY; expected $EXPECTED_PROPERTY" - exit 1 - } + - name: Start the exact PR images and verify health and authentication + run: python3 .github/scripts/docker-deployment.py "$IMAGE_TAG" diff --git a/.github/workflows/helm-chart-ci.yml b/.github/workflows/helm-chart-ci.yml index e78dd03655..28df8f5230 100644 --- a/.github/workflows/helm-chart-ci.yml +++ b/.github/workflows/helm-chart-ci.yml @@ -18,20 +18,12 @@ name: "Helm Chart CI" on: - push: - branches: [ master, 'release-*' ] - paths: [ 'helm/**', '.github/workflows/helm-chart-ci.yml' ] - pull_request: - paths: [ 'helm/**', '.github/workflows/helm-chart-ci.yml' ] + workflow_call: + workflow_dispatch: permissions: contents: read -# Keep the newest PR run; preserve independent push, release and manual runs. -concurrency: - group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.run_id }} - cancel-in-progress: ${{ github.event_name == 'pull_request' }} - jobs: lint-and-render: runs-on: ubuntu-latest diff --git a/.github/workflows/licence-checker.yml b/.github/workflows/licence-checker.yml index 491ccf57b5..c62e9437fa 100644 --- a/.github/workflows/licence-checker.yml +++ b/.github/workflows/licence-checker.yml @@ -9,7 +9,7 @@ on: # Keep the newest PR run; preserve independent push, release and manual runs. concurrency: - group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.run_id }} + group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.run_id }}-${{ github.run_attempt == 1 && 'head' || format('rerun-{0}', github.run_id) }} cancel-in-progress: ${{ github.event_name == 'pull_request' }} jobs: diff --git a/.github/workflows/pd-store-ci.yml b/.github/workflows/pd-store-ci.yml index a61c5673ad..e7aee5d23c 100644 --- a/.github/workflows/pd-store-ci.yml +++ b/.github/workflows/pd-store-ci.yml @@ -1,37 +1,11 @@ name: "HugeGraph-PD & Store & Hstore CI" on: - push: - branches: - - master - - 'release-*' - - 'test-*' - pull_request: - # Keep source/resources and shared build inputs; skip named documentation files. - paths: - - '.github/workflows/.java-version' - - 'pom.xml' - - '.mvn/**' - - 'style/**' - - '.editorconfig' - - '.github/configs/**' - - 'hugegraph-commons/**' - - 'hugegraph-struct/**' - - 'hugegraph-pd/**' - - 'hugegraph-store/**' - - 'hugegraph-server/**' - - '.github/workflows/pd-store-ci.yml' - - 'docker/**' - - 'install-dist/**' - - '!**/README*' - - '!**/AGENTS.md' - - '!**/CLAUDE.md' - -# TODO: consider merge to one ci.yml file -# Keep the newest PR run; preserve independent push, release and manual runs. -concurrency: - group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.run_id }} - cancel-in-progress: ${{ github.event_name == 'pull_request' }} + workflow_call: + workflow_dispatch: + +permissions: + contents: read jobs: rocksdb-compatibility: @@ -39,8 +13,8 @@ jobs: contents: read runs-on: ubuntu-24.04 env: - BEFORE_SHA: ${{ github.event.pull_request.base.sha || github.event.before }} - AFTER_SHA: ${{ github.event.pull_request.head.sha || github.event.after }} + BEFORE_SHA: ${{ github.event.pull_request.base.sha || github.event.before || github.sha }} + AFTER_SHA: ${{ github.event.pull_request.head.sha || github.event.after || github.sha }} steps: - name: Require exact comparison revisions run: | @@ -223,7 +197,7 @@ jobs: if ! command -v "$tool" >/dev/null 2>&1; then echo "can_run=false" >> "$GITHUB_OUTPUT" echo "skip_reason=missing tool: $tool" >> "$GITHUB_OUTPUT" - exit 0 + exit 1 fi done echo "can_run=true" >> "$GITHUB_OUTPUT" @@ -386,14 +360,14 @@ jobs: if ! command -v "$tool" >/dev/null 2>&1; then echo "can_run=false" >> "$GITHUB_OUTPUT" echo "skip_reason=missing tool: $tool" >> "$GITHUB_OUTPUT" - exit 0 + exit 1 fi done LIMIT_N=$(ulimit -n) if [[ "$LIMIT_N" != "unlimited" ]] && (( LIMIT_N < 1024 )); then echo "can_run=false" >> "$GITHUB_OUTPUT" echo "skip_reason=ulimit -n is $LIMIT_N (store requires >= 1024)" >> "$GITHUB_OUTPUT" - exit 0 + exit 1 fi echo "can_run=true" >> "$GITHUB_OUTPUT" diff --git a/.github/workflows/rerun-ci.yml b/.github/workflows/rerun-ci.yml index 3fdd4c0f56..d76223d5b1 100644 --- a/.github/workflows/rerun-ci.yml +++ b/.github/workflows/rerun-ci.yml @@ -18,7 +18,7 @@ env: jobs: decide-rerun-action: - if: ${{ github.event.workflow_run.conclusion == 'failure' }} + if: ${{ github.event.workflow_run.conclusion == 'failure' && github.event.workflow_run.event == 'push' }} runs-on: ubuntu-latest outputs: action: ${{ steps.decision.outputs.action }} @@ -37,7 +37,7 @@ jobs: action="skip" reason="unsupported event: $EVENT_NAME" - if [[ "$EVENT_NAME" == "push" || "$EVENT_NAME" == "pull_request" ]]; then + if [[ "$EVENT_NAME" == "push" ]]; then if (( RUN_ATTEMPT > MAX_RERUNS )); then reason="retry limit reached" else @@ -73,11 +73,45 @@ jobs: contents: read runs-on: ubuntu-latest steps: + # workflow_run is privileged: only load code from the default-branch commit. + - name: Checkout trusted retry checker + uses: actions/checkout@v7 + with: + ref: ${{ github.sha }} + persist-credentials: false + sparse-checkout: .github/scripts + + - name: Check source freshness before waiting + id: initial-freshness + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GH_REPO: ${{ github.repository }} + RUN_ID: ${{ github.event.workflow_run.id }} + RUN_ATTEMPT: ${{ github.event.workflow_run.run_attempt }} + run: | + python3 .github/scripts/check-rerun.py \ + --repository "$GH_REPO" --run-id "$RUN_ID" \ + --run-attempt "$RUN_ATTEMPT" --max-reruns "$MAX_RERUNS" + - name: Wait before rerun + if: steps.initial-freshness.outputs.action == 'rerun' + run: sleep "$RETRY_DELAY_SECONDS" + + - name: Check source freshness after waiting + id: freshness + if: steps.initial-freshness.outputs.action == 'rerun' + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GH_REPO: ${{ github.repository }} + RUN_ID: ${{ github.event.workflow_run.id }} + RUN_ATTEMPT: ${{ github.event.workflow_run.run_attempt }} run: | - sleep "$RETRY_DELAY_SECONDS" + python3 .github/scripts/check-rerun.py \ + --repository "$GH_REPO" --run-id "$RUN_ID" \ + --run-attempt "$RUN_ATTEMPT" --max-reruns "$MAX_RERUNS" - name: Rerun failed jobs + if: steps.freshness.outputs.action == 'rerun' env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} GH_REPO: ${{ github.repository }} diff --git a/.github/workflows/server-ci.yml b/.github/workflows/server-ci.yml index 3c0e3f8492..2bc2dce581 100644 --- a/.github/workflows/server-ci.yml +++ b/.github/workflows/server-ci.yml @@ -7,19 +7,61 @@ on: - 'release-*' - 'test-*' pull_request: + workflow_dispatch: # Keep the newest PR run; preserve independent push, release and manual runs. concurrency: - group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.run_id }} + group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.run_id }}-${{ github.run_attempt == 1 && 'head' || format('rerun-{0}', github.run_id) }} cancel-in-progress: ${{ github.event_name == 'pull_request' }} +permissions: + contents: read + pull-requests: read + jobs: + plan: + runs-on: ubuntu-24.04 + outputs: + server: ${{ steps.select.outputs.server }} + commons: ${{ steps.select.outputs.commons }} + pd_store: ${{ steps.select.outputs.pd_store }} + cluster: ${{ steps.select.outputs.cluster }} + docker: ${{ steps.select.outputs.docker }} + helm: ${{ steps.select.outputs.helm }} + dependency_license: ${{ steps.select.outputs.dependency_license }} + compatibility: ${{ steps.select.outputs.compatibility }} + security: ${{ steps.select.outputs.security }} + steps: + - uses: actions/checkout@v7 + with: + fetch-depth: 0 + persist-credentials: false + - name: Test selection, gate and retry policy + run: python3 -m unittest discover -s .github/scripts -p 'test_*.py' + - name: Select affected tests + id: select + env: + GH_TOKEN: ${{ github.token }} + run: >- + python3 .github/scripts/ci-policy.py plan --project server + --event-path "$GITHUB_EVENT_PATH" --repository "$GITHUB_REPOSITORY" + --output plan.json + - uses: actions/upload-artifact@v4 + with: + name: ci-plan + path: plan.json + retention-days: 7 + overwrite: true + java-runtime: uses: ./.github/workflows/java-runtime.yml permissions: contents: read wait-storage-shell-test: + name: server_memory / startup contracts + needs: plan + if: needs.plan.outputs.server == 'true' permissions: contents: read runs-on: ubuntu-24.04 @@ -29,6 +71,9 @@ jobs: with: persist-credentials: false + - name: Test CI retry freshness checks + run: python3 -m unittest discover -s .github/scripts -p 'test_check_rerun.py' + - name: Run wait-storage.sh peer failover tests run: hugegraph-server/hugegraph-dist/src/assembly/travis/test-wait-storage.sh @@ -40,8 +85,9 @@ jobs: # Keep the job ID for the Codecov configuration validator. # The old required check name is provided by the temporary result job below. build-server: - needs: java-runtime - name: Server memory (Java ${{ matrix.JAVA_VERSION }}) + needs: [plan, java-runtime] + if: needs.plan.outputs.server == 'true' + name: server_memory (Java ${{ matrix.JAVA_VERSION }}) runs-on: ubuntu-24.04 env: &linux-server-env USE_STAGE: 'false' # Whether to include the stage repository. @@ -162,13 +208,13 @@ jobs: if ! command -v "$tool" >/dev/null 2>&1; then echo "can_run=false" >> "$GITHUB_OUTPUT" echo "skip_reason=missing tool: $tool" >> "$GITHUB_OUTPUT" - exit 0 + exit 1 fi done if ! command -v fuser >/dev/null 2>&1; then echo "can_run=false" >> "$GITHUB_OUTPUT" echo "skip_reason=missing tool: fuser (required for Linux port cleanup)" >> "$GITHUB_OUTPUT" - exit 0 + exit 1 fi echo "can_run=true" >> "$GITHUB_OUTPUT" @@ -223,51 +269,133 @@ jobs: disable_search: true fail_ci_if_error: false + server-rocksdb: + name: server_rocksdb (Java ${{ matrix.JAVA_VERSION }}) + needs: [plan, java-runtime] + if: needs.plan.outputs.server == 'true' + runs-on: ubuntu-24.04 + env: *linux-server-env + strategy: + fail-fast: false + matrix: + BACKEND: [ rocksdb ] + JAVA_VERSION: ${{ fromJSON(needs.java-runtime.outputs.versions || '["unavailable"]') }} + steps: *linux-server-steps + + commons: + name: commons + needs: plan + if: needs.plan.outputs.commons == 'true' + uses: ./.github/workflows/commons-ci.yml + secrets: inherit + + pd_store: + name: pd_store + needs: plan + if: needs.plan.outputs.pd_store == 'true' + uses: ./.github/workflows/pd-store-ci.yml + secrets: inherit + + cluster: + name: cluster + needs: plan + if: needs.plan.outputs.cluster == 'true' + uses: ./.github/workflows/cluster-test-ci.yml + secrets: inherit + + docker: + name: docker + needs: plan + if: needs.plan.outputs.docker == 'true' + uses: ./.github/workflows/docker-build-ci.yml + secrets: inherit + + helm: + name: helm + needs: plan + if: needs.plan.outputs.helm == 'true' + uses: ./.github/workflows/helm-chart-ci.yml + + dependency_license: + name: dependency_license + needs: plan + if: needs.plan.outputs.dependency_license == 'true' + uses: ./.github/workflows/check-dependencies.yml + + affected-module-tests: + name: affected-module-tests + needs: [plan, java-runtime, wait-storage-shell-test, build-server, server-rocksdb, commons, pd_store, cluster, docker, helm] + if: ${{ !cancelled() }} + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v7 + with: + persist-credentials: false + - uses: actions/download-artifact@v4 + with: + name: ci-plan + - name: Validate selected suites and report results + env: + GH_TOKEN: ${{ github.token }} + JOB_RESULTS: ${{ toJSON(needs) }} + run: | + python3 - <<'PYGATE' + import json, os + results = json.loads(os.environ['JOB_RESULTS']) + if results['plan']['result'] != 'success': + raise SystemExit('CI planning failed') + if results['plan']['outputs'].get('server') == 'true': + for job in ['java-runtime', 'wait-storage-shell-test']: + if results[job]['result'] != 'success': + raise SystemExit('Selected prerequisite failed: ' + job) + results['server_memory'] = results['build-server'] + results['server_rocksdb'] = results['server-rocksdb'] + with open('results.json', 'w') as output: + json.dump(results, output) + PYGATE + python3 .github/scripts/ci-policy.py gate --plan plan.json --results results.json --output test-results.json + - uses: actions/upload-artifact@v4 + with: + name: ci-test-results + path: test-results.json + retention-days: 14 + overwrite: true + + # Temporary protection aliases; skipped modules pass only through the current-run gate. server-memory-required: - # This is the only Server check referenced by .asf.yaml; keep its name stable. name: Server memory tests - needs: [ java-runtime, build-server ] - # Always evaluate results, including dependency failures and unexpected skips. - if: ${{ always() }} + needs: affected-module-tests + if: ${{ !cancelled() }} runs-on: ubuntu-24.04 - permissions: {} - env: &memory-results - RUNTIME_RESULT: ${{ needs.java-runtime.result }} - MEMORY_RESULT: ${{ needs.build-server.result }} - steps: &require-memory-success - - name: Require the runtime configuration and all memory tests to pass - run: | - if [ "$RUNTIME_RESULT" != "success" ] || [ "$MEMORY_RESULT" != "success" ]; then - echo "::error::Runtime configuration: $RUNTIME_RESULT; memory tests: $MEMORY_RESULT" - exit 1 - fi + env: &gate-result + GATE_RESULT: ${{ needs.affected-module-tests.result }} + steps: &require-gate + - run: test "$GATE_RESULT" = success - # Temporary alias for live protection. Remove after ASF activates the stable context. - # Reuse the real result check so a skipped matrix cannot satisfy the old requirement. legacy-server-memory-required: name: build-server (memory, 11) - needs: [ java-runtime, build-server ] - if: ${{ always() }} + needs: affected-module-tests + if: ${{ !cancelled() }} runs-on: ubuntu-24.04 - permissions: {} - env: *memory-results - steps: *require-memory-success + env: *gate-result + steps: *require-gate - # Optional coverage uses the same Linux environment, resolved JDKs and test steps. - # Keep it separate so backend failures do not become required memory failures. build-server-backends: - needs: java-runtime - name: build-server (${{ matrix.BACKEND }}, ${{ matrix.JAVA_VERSION }}) + name: HBase compatibility (Java ${{ matrix.JAVA_VERSION }}) + needs: [plan, java-runtime, affected-module-tests] + if: ${{ !cancelled() && needs.plan.result == 'success' && needs.affected-module-tests.result == 'success' && needs.plan.outputs.compatibility == 'true' }} runs-on: ubuntu-24.04 env: *linux-server-env strategy: fail-fast: false matrix: - BACKEND: [ rocksdb, hbase ] + BACKEND: [ hbase ] JAVA_VERSION: ${{ fromJSON(needs.java-runtime.outputs.versions || '["unavailable"]') }} steps: *linux-server-steps build-server-macos-rocksdb: + needs: [plan, affected-module-tests] + if: ${{ !cancelled() && needs.plan.result == 'success' && needs.affected-module-tests.result == 'success' && needs.plan.outputs.compatibility == 'true' }} runs-on: ${{ matrix.os }} strategy: fail-fast: false @@ -344,6 +472,28 @@ jobs: build-server-riscv64: # Keep the emulated Java 17 RISC-V native-runtime signal visible while the # platform lane stabilizes, but do not use it as a gate yet. + needs: [plan, affected-module-tests] + if: ${{ !cancelled() && needs.plan.result == 'success' && needs.affected-module-tests.result == 'success' && needs.plan.outputs.compatibility == 'true' }} uses: ./.github/workflows/riscv64-ci.yml permissions: contents: read + + codeql: + needs: [plan, affected-module-tests] + if: ${{ !cancelled() && needs.plan.result == 'success' && needs.affected-module-tests.result == 'success' && (vars.CI_OPTIMIZED_REQUIRED != 'true' || needs.plan.outputs.security == 'true') }} + uses: ./.github/workflows/codeql-analysis.yml + permissions: + contents: read + actions: read + security-events: write + secrets: inherit + + legacy-analyze-required: + name: Analyze (java) + needs: codeql + if: ${{ !cancelled() && vars.CI_OPTIMIZED_REQUIRED != 'true' }} + runs-on: ubuntu-24.04 + env: + SCAN_RESULT: ${{ needs.codeql.result }} + steps: + - run: test "$SCAN_RESULT" = success diff --git a/docs/CONTRIBUTING.md b/docs/CONTRIBUTING.md index bfa9d7e66f..af3fa2cd2d 100644 --- a/docs/CONTRIBUTING.md +++ b/docs/CONTRIBUTING.md @@ -1,187 +1,68 @@ -# How to Contribute to HugeGraph +# Contributing to HugeGraph -> Refer [website-doc](https://hugegraph.apache.org/docs/contribution-guidelines/) for the latest information. +Keep each contribution focused on one problem, with enough context and validation for review. +Report reproducible bugs and proposed features through [GitHub issues](https://github.com/apache/hugegraph/issues). -Thanks for taking the time to contribute! -As an open source project, HugeGraph is looking forward to being contributed from everyone, and we are also grateful to all the contributors. +## Prepare a change -The following is a contribution guide for HugeGraph: +- Read [AGENTS.md](../AGENTS.md) and the guidance for the modules you will change. + The same repository rules apply to changes made by people and coding agents. +- Work on a topic branch or isolated worktree based on the intended target branch. + Preserve existing uncommitted work; stage only files belonging to your change. +- Check existing implementations and dependencies before adding new abstractions or libraries. + Keep unrelated cleanup out of the PR. +- Follow [.editorconfig](../.editorconfig) and [Checkstyle](../style/checkstyle.xml): + four-space indentation, 120-column Java lines, and no wildcard imports. -image +## Build and validate -## 1. Preparation +Use Java 17 and Maven 3.6.3+. Compiler settings and project versions are defined in +[pom.xml](../pom.xml); module guidance describes test profiles and service prerequisites. +Run commands from the repository root. -**Recommended**: You can use [GitHub desktop](https://desktop.github.com/) to greatly simplify the PR process. +```bash +# Build the server and its dependencies without running tests +mvn clean install -pl hugegraph-server -am -DskipTests -We can contribute by reporting issues, submitting code patches or any other feedback. - -Before submitting the code, we need to do some preparation: - -1. Sign up or login to GitHub: [https://github.com](https://github.com) - -2. Fork HugeGraph repo from GitHub: [https://github.com/apache/hugegraph/fork](https://github.com/apache/hugegraph/fork) - -3. Clone code from fork repo to local: [https://github.com/${GITHUB_USER_NAME}/hugegraph](https://github.com/${GITHUB_USER_NAME}/hugegraph) - - ```shell - # clone code from remote to local repo - git clone https://github.com/${GITHUB_USER_NAME}/hugegraph.git hugegraph - ``` - -4. Configure local HugeGraph repo - - ```shell - cd hugegraph - - # add upstream to synchronize the latest code - git remote add hugegraph https://github.com/apache/hugegraph - - # set name and email to push code to github - git config user.name "{full-name}" # like "Jermy Li" - git config user.email "{email-address-of-github}" # like "jermy@apache.org" - ``` - -## 2. Create an Issue on GitHub - -If you encounter bugs or have any questions, please go to [GitHub Issues](https://github.com/apache/hugegraph/issues) to report them and feel free to [create an issue](https://github.com/apache/hugegraph/issues/new). - -## 3. Make changes of code locally - -#### 3.1 Create a new branch - -Please don't use master branch for development. Instead, we should create a new branch: - -```shell -# checkout master branch -git checkout master -# pull the latest code from official hugegraph -git pull hugegraph -# create new branch: bugfix-branch -git checkout -b bugfix-branch -``` - -#### 3.2 Change the code - -Assume that we need to modify some files like "HugeGraph.java" and "HugeFactory.java": - -```shell -# modify code to fix a bug -vim hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/HugeGraph.java -vim hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/HugeFactory.java -# run test locally (optional) -mvn test -Pcore-test,memory -``` -Note: Code style is defined by the `.editorconfig` file at the repository root. Checkstyle rules are defined in `style/checkstyle.xml`. Configure your IDE accordingly. - -##### 3.2.1 Check licenses -If we want to add new third-party dependencies to the `HugeGraph` project, we need to do the following things: -1. Find the third-party dependent repository, put the dependent `license` file into [./install-dist/release-docs/licenses/](https://github.com/apache/hugegraph/tree/master/install-dist/release-docs/licenses) path. -2. Declare the dependency in [./install-dist/release-docs/LICENSE](https://github.com/apache/hugegraph/blob/master/install-dist/release-docs/LICENSE) `LICENSE` information. -3. Find the NOTICE file in the repository and append it to [./install-dist/release-docs/NOTICE](https://github.com/apache/hugegraph/blob/master/install-dist/release-docs/NOTICE) file (skip this step if there is no NOTICE file). -4. Execute locally [./install-dist/scripts/dependency/regenerate_known_dependencies.sh](https://github.com/apache/hugegraph/blob/master/install-dist/scripts/dependency/regenerate_known_dependencies.sh) to update the dependency list [known-dependencies.txt](https://github.com/apache/hugegraph/blob/master/install-dist/scripts/dependency/known-dependencies.txt) (or manually update). - -**Example**: A new third-party dependency is introduced into the project -> `ant-1.9.1.jar` -- The project source code is located at: https://github.com/apache/ant/tree/rel/1.9.1 -- LICENSE file: https://github.com/apache/ant/blob/rel/1.9.1/LICENSE -- NOTICE file: https://github.com/apache/ant/blob/rel/1.9.1/NOTICE - -The license information of `ant-1.9.1.jar` needs to be specified in the LICENSE file, and the notice information needs to be specified in the NOTICE file. The detailed LICENSE file corresponding to ant-1.9.1.jar needs to be copied to our licenses/ directory. Finally, update the known-dependencies.txt file. - -#### 3.3 Commit changes to git repo - -After the code has been completed, we submit them to the local git repo: - -```shell -# add files to local git index -git add hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/HugeGraph.java -git add hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/HugeFactory.java -# commit to local git repo -git commit -``` - -Please edit the commit message after running `git commit`, we can explain what and how to fix a bug or implement a feature, the following is an example: - -```sh -Fix bug: run deploy multiple times - -fix #ISSUE_ID -``` - -> Please remember to fill in the issue id, which GitHub generated after issue creation. - -#### 3.4 Push commit to GitHub fork repo - -Push the local commit to GitHub fork repo: - -```shell -# push the local commit to fork repo -git push origin bugfix-branch:bugfix-branch +# Before pushing code, format and compile +mvn editorconfig:format +mvn clean compile -Dmaven.javadoc.skip=true ``` -Note that since GitHub requires submitting code through `username + token` (instead of using `username + password` directly), you need to create a GitHub token from https://github.com/settings/tokens: -image - -## 4. Create a Pull Request - -Go to the web page of GitHub fork repo, there would be a chance to create a Pull Request after pushing to a new branch, click the button "Compare & pull request" to do it. Then edit the description for proposed changes, which can just be copied from the commit message. - -User-visible feature, configuration or deployment behavior changes must ship with matching -documentation. Include repository documentation in the same PR. When website documentation -is affected, link a paired [hugegraph-doc](https://github.com/apache/hugegraph-doc) PR and -coordinate both merges. A follow-up issue alone does not replace required documentation. -In the PR template, mark pending documentation as `Doc - TODO` until it is ready; -use `Doc - Done` with the documentation files or PR link, or `Doc - No Need` for internal-only changes. - -Note: please make sure the email address you used to submit the code is bound to the GitHub account. For how to bind the email address, please refer to https://github.com/settings/emails: -image - -## 5. Code review - - Maintainers will start the code review after all the **automatic** checks are passed: - -- Check: Contributor License Agreement is signed -- Check: Travis CI builds are passed (automatically Test and Deploy) - -The commit will be accepted and merged if there is no problem after review. - -Please click on "Details" to find the problem if any check does not pass. - -If there are checks not passed or changes requested, then continue to modify the code and push again. - -## 6. Further changes after review - -If we have not passed the review, don't be discouraged. Usually a commit needs to be reviewed several times before being accepted! Please follow the review comments and make further changes. - -After the further changes, we submit them to the local repo: - -```shell -# commit all updated files in a new commit, -# please feel free to enter any appropriate commit message, note that -# we will squash all commits in the pull request as one commit when -# merging into the master branch. -git commit -a -``` - -> If there are conflicts that prevent the code from being merged, we need to rebase on master branch: -> -> ```shell -> # synchronize the latest code -> git checkout master -> git pull hugegraph -> # rebase on master -> git checkout bugfix-branch -> git rebase -i master -> ``` - -And push it to GitHub fork repo again: - -```shell -# force push the local commit to fork repo -git push -f origin bugfix-branch:bugfix-branch -``` - -GitHub will automatically update the Pull Request after we push it, wait for code review. - -For Any question, please contact us through [dev@hugegraph.apache.org](mailto:dev@hugegraph.apache.org) ([subscriber](https://hugegraph.apache.org/docs/contribution-guidelines/subscribe/) only) - - +Run the affected module tests and add regression coverage when behavior changes. +A successful build with skipped tests does not validate behavior; Commons tests require +`-DskipCommonsTests=false`. For documentation-only changes, verify links and paths and run +`git diff --check`. Inspect formatter output before staging to avoid unrelated edits. + +See [CI policy](ci.md) for affected-module selection, required checks, result reuse, +and automatic cancellation and retries. Report what you actually validated and any limitations. + +## Documentation and dependencies + +- User-visible features, configuration and deployment changes need matching repository + documentation in the same PR. When website documentation is affected, link a paired + [apache/hugegraph-doc](https://github.com/apache/hugegraph-doc) PR and coordinate both merges. + A follow-up issue does not replace the required documentation. +- In the PR template, use `Doc - TODO` while documentation is pending, `Doc - Done` with + its location when ready, or `Doc - No Need` for internal-only changes. +- New or updated dependencies need appropriate license and notice updates in the root + and [release documentation](../install-dist/release-docs/), including bundled license files. + Update the [dependency inventory](../install-dist/scripts/dependency/known-dependencies.txt) + using [the regeneration script](../install-dist/scripts/dependency/regenerate_known_dependencies.sh). + +## Submit and review + +Use `type(scope): description` for commits; omit the scope when it is unclear. +Types are `feat`, `fix`, `chore`, `docs`, `refactor`, or `BREAKING CHANGE`. +Start the description with a lowercase verb, keep it under 50 characters, and include +three to five core change points in the body. For example: `fix(server): handle missing vertices`. + +Push your topic branch and open or update a PR against the intended upstream branch. +Use the [PR template](../.github/PULL_REQUEST_TEMPLATE.md) to explain the problem, +the resulting behavior, relevant validation, and documentation status. Link the related +issue when applicable. A short example or diagram is useful when it clarifies the change; +commit histories and raw test logs are unnecessary. + +Address review findings within the agreed scope and validate the affected paths again. +Resolve a discussion after its issue is fixed or clarified. Inspect failing CI jobs before +retrying; a rerun is appropriate for a temporary infrastructure failure, not a reproducible bug. diff --git a/docs/ci.md b/docs/ci.md new file mode 100644 index 0000000000..9c5cd9fb80 --- /dev/null +++ b/docs/ci.md @@ -0,0 +1,84 @@ +# CI policy + +Pull requests and supported branch pushes enter `HugeGraph-Server CI`. The workflow +always reports `affected-module-tests`; the required `check-license` runs independently. Module +workflows are reusable and can also be started manually. + +| Changed inputs | Required Linux coverage | +| --- | --- | +| Any non-document Server module input, including RocksDB and HStore | Memory, RocksDB, PD/Store/HStore and Cluster | +| Server distribution or shared startup scripts | Server, PD/Store/HStore, Docker and Cluster | +| Commons or Struct | Their tests and affected Server, PD, Store, HStore and Cluster tests | +| PD or Store distribution inputs (`hg-pd-dist/`, `hg-store-dist/`), including scripts, configs, assembly descriptors and POMs | PD/Store/HStore, Cluster and Docker | +| Other PD or Store inputs | PD/Store/HStore suite and Cluster | +| Cluster, Docker or Helm | The corresponding suite; Docker includes Compose render/smoke and startup contracts | +| Central `server-ci.yml` | All Linux suites controlled by the caller | +| A module workflow | Its suite and known downstream suites | +| Dependencies, shared build inputs or unknown paths | Conservative full coverage | + +The selector follows dependency edges. PD, Store, HStore and Struct share one suite +in this first stage. Selected tests must succeed; failure, cancellation or an +unexpected skip cannot satisfy the gate. Startup prerequisites are enforced. + +Third-party dependency inventory and vulnerability review retain their existing +non-blocking policy. They run when selected, but are excluded from the core gate +and current-run test results. Their failure does not prevent affected module tests or +post-gate checks from reporting results; the required license check is unchanged. + +The Docker suite builds and loads the four production images from the current +checkout on one Linux runner, sharing their Maven build through BuildKit Bake. +It verifies healthcheck presence and Java contracts, then starts standalone Server +and the PD/Store/Server topology with unique run tags and pulling disabled. +Runtime checks match each container's image ID to the build, require healthy +services and validate Server version and authenticated graph-list responses. +PD must report readiness and a registered Store; unauthenticated graph and PD +metadata access must be rejected. The existing Hubble Compose smoke +remains a separate compatibility check; it does not verify the new PR images. + +HBase, macOS, RISC-V and CodeQL run after the core gate. Their results remain visible, +but they are outside `affected-module-tests`. Post-gate conditions explicitly +require a successful plan and gate, including when unrelated modules were skipped. This orders work within a PR; it does +not grant runner priority over other PRs. Existing scheduled CodeQL scanning remains. + +## Documentation updates + +Only explicitly allowed prose and static documentation assets qualify. Source, +types, tests, dependencies and CI configuration never qualify as documentation. +A PR containing only allowed documentation changes can omit unaffected modules. +A source PR that later receives a documentation update still selects its cumulative +source changes and runs the necessary module tests, compatibility and security +checks on every update. Cross-run success reuse is paused: an artifact's claimed +merge SHA, parents and job IDs do not independently prove the original execution tree. +A trustworthy execution-proof design is separate work. + +The plan and current-run results are saved as `ci-plan` and `ci-test-results` +artifacts. Results describe only this run's successful selected suites; they are +not authentication evidence and never authorize a later run to skip tests. + +PR planning records the event's head, base and source before querying live metadata. +A known mismatch with the checkout merge or current PR fails planning; expanding +coverage cannot make an outdated checkout current. After all selected jobs and +fixtures succeed, the gate rechecks the open PR's head, base and source using a +read-only API request. Changed inputs or unavailable metadata fail the gate. +Refresh the branch and start a new PR event after head or base movement: GitHub +reruns retain the original commit and event, so rerunning alone cannot refresh the +base. An unknown API or selection failure falls back to full coverage; a manual +full rerun can recover a transient failure while the recorded inputs remain current. + +## Protection migration + +`.asf.yaml` requests `check-license` and `affected-module-tests`. Temporary memory +and Java analysis aliases preserve the previous protection names. Leave +`CI_OPTIMIZED_REQUIRED` unset until the live branch protection uses the new gate; +CodeQL continues scanning every update during this transition. Setting the +repository variable to `true` then permits unchanged documentation updates to +avoid redundant security scanning. Enabling the variable is a separate rollout +operation. + +Automatic retry is limited to failed pushes. It verifies the run attempt, source +repository and current branch head before and after its existing wait; at most two +automatic reruns are allowed. Only failed jobs are retried. PR automatic retry is +paused because it relied on the PR's CI plan artifact to attest the tested base. +For a transient PR failure with unchanged inputs, use a manual full rerun so shared +build and fixture artifacts are rebuilt together with their consumers. A retry retains its separate concurrency +group and cannot cancel a newer PR run.