Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 6 additions & 2 deletions .github/scripts/ci-policy.py
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,9 @@
TRAVIS = "hugegraph-server/hugegraph-dist/src/assembly/travis/"
# Contract tests follow their executing suites, without runtime consumer fan-out.
CONTRACT_SUITES = {
TRAVIS + "test-check-java-runtime-logs.sh": {"struct"},
TRAVIS + "check-java-runtime-logs.sh": {"server", "commons", "struct", "pd", "store", "hstore", "cluster",
"macos", "macos_full", "hbase", "riscv64"},
TRAVIS + "test-stop-hugegraph-store.sh": {"struct"},
TRAVIS + "test-check-jacoco-report.sh": {"struct"},
TRAVIS + "test-codecov-upload-config.sh": {"struct"},
Expand Down Expand Up @@ -238,13 +241,14 @@ def event_selection(project, paths, event, event_name):
if preset not in PRESETS:
raise ValueError("unknown test-set: " + str(preset))
return set(PRESETS[preset])
selected = select(project, paths)
if event_name == "push" and event.get("ref") == "refs/heads/master":
# Test contracts do not change deployed runtime inputs.
selected = select(project, [path for path in paths if path not in CONTRACT_SUITES])
smoke = selected.intersection({"docker", "helm", "dependency_license", "macos", "distributed_smoke"})
if selected.intersection({"server", "commons", "struct", "pd", "store", "hstore", "cluster"}):
smoke.add("distributed_smoke")
return smoke
return selected
return select(project, paths)


def suites(project, selected):
Expand Down
14 changes: 13 additions & 1 deletion .github/scripts/test_ci_policy.py
Original file line number Diff line number Diff line change
Expand Up @@ -373,6 +373,7 @@ def test_legacy_memory_bridge_requires_real_results_and_retires_without_a_runner
def test_contract_scripts_reach_their_actual_inner_jobs_and_steps(self):
workflows = Path(__file__).resolve().parents[1] / "workflows"
cases = [("test-stop-hugegraph-store.sh", {"struct"}),
("test-check-java-runtime-logs.sh", {"struct"}),
("test-check-jacoco-report.sh", {"struct"}),
("test-codecov-upload-config.sh", {"struct"}),
("test-riscv64-java-runtime.sh", {"server", "riscv64"})]
Expand All @@ -397,7 +398,7 @@ def test_contract_scripts_reach_their_actual_inner_jobs_and_steps(self):
condition = condition.replace("inputs.run-struct", str(inputs["struct"]))
condition = condition.replace("inputs.run-store", str(inputs["store"]))
self.assertTrue(eval(condition.replace("||", "or"), {"__builtins__": {}}))
step = next(step for step in struct.split("\n - ") if script in step)
step = next(step for step in struct.split("\n - ") if Path(script).name in step)
self.assertNotRegex(step, r"(?m)^ if:")
else:
self.assertIn("server-rocksdb", callers)
Expand All @@ -413,6 +414,17 @@ def test_contract_scripts_reach_their_actual_inner_jobs_and_steps(self):
{"__builtins__": {}}))
mixed = self.plan_for([policy.TRAVIS + "test-stop-hugegraph-store.sh", "hugegraph-pd/src/A.java"])
self.assertTrue({"struct", "pd", "store", "hstore", "cluster"}.issubset(mixed["selected"]))
checker = self.plan_for([policy.TRAVIS + "check-java-runtime-logs.sh"])
self.assertEqual({"build-server", "server-rocksdb", "commons", "pd_store", "cluster", "compatibility"},
self.coordinator_jobs(checker))
self.assertTrue(checker["hbase"] and checker["riscv64"] and checker["macos_full"])
master = {"ref": "refs/heads/master"}
for script in policy.CONTRACT_SUITES:
self.assertEqual([], self.plan_for([script], "push", master)["expected"])
runtime = "hugegraph-server/hugegraph-core/src/main/A.java"
self.assertEqual(self.plan_for([runtime], "push", master)["selected"],
self.plan_for([runtime, policy.TRAVIS + "check-java-runtime-logs.sh"],
"push", master)["selected"])

def test_coordinator_reads_and_validates_real_java_input(self):
workflow = (Path(__file__).resolve().parents[1] / "workflows/server-ci.yml").read_text()
Expand Down
10 changes: 8 additions & 2 deletions .github/workflows/cluster-test-ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -39,13 +39,19 @@ jobs:

- name: Run simple cluster test
run: |
set -o pipefail
suite_log="$RUNNER_TEMP/simple-cluster-test.log"
timeout 45m mvn test -pl hugegraph-cluster-test/hugegraph-clustertest-test \
-am -P simple-cluster-test
-am -P simple-cluster-test 2>&1 | tee "$suite_log"
hugegraph-server/hugegraph-dist/src/assembly/travis/check-java-runtime-logs.sh "$suite_log"

- name: Run multi cluster test
run: |
set -o pipefail
suite_log="$RUNNER_TEMP/multi-cluster-test.log"
timeout 45m mvn test -pl hugegraph-cluster-test/hugegraph-clustertest-test \
-am -P multi-cluster-test
-am -P multi-cluster-test 2>&1 | tee "$suite_log"
hugegraph-server/hugegraph-dist/src/assembly/travis/check-java-runtime-logs.sh "$suite_log"

- name: Show cluster diagnostics on failure
if: failure()
Expand Down
11 changes: 9 additions & 2 deletions .github/workflows/commons-ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ jobs:
runs-on: ubuntu-24.04
env:
USE_STAGE: 'false' # Whether to include the stage repository.
TRAVIS_DIR: hugegraph-server/hugegraph-dist/src/assembly/travis

strategy:
fail-fast: false
Expand Down Expand Up @@ -50,13 +51,19 @@ jobs:

- name: Run common test
run: |
set -o pipefail
RUNTIME_LOG=$(mktemp "$RUNNER_TEMP/java-runtime.XXXXXX")
mvn test -pl hugegraph-commons/hugegraph-common -Dtest=UnitTestSuite \
-DskipCommonsTests=false -Dsurefire.failIfNoSpecifiedTests=true
-DskipCommonsTests=false -Dsurefire.failIfNoSpecifiedTests=true 2>&1 | tee "$RUNTIME_LOG"
bash "$TRAVIS_DIR/check-java-runtime-logs.sh" "$RUNTIME_LOG"

- name: Run rpc test
run: |
set -o pipefail
RUNTIME_LOG=$(mktemp "$RUNNER_TEMP/java-runtime.XXXXXX")
mvn test -pl hugegraph-commons/hugegraph-rpc -Dtest=UnitTestSuite \
-DskipCommonsTests=false -Dsurefire.failIfNoSpecifiedTests=true
-DskipCommonsTests=false -Dsurefire.failIfNoSpecifiedTests=true 2>&1 | tee "$RUNTIME_LOG"
bash "$TRAVIS_DIR/check-java-runtime-logs.sh" "$RUNTIME_LOG"

- name: Upload coverage to Codecov
uses: codecov/codecov-action@v7
Expand Down
117 changes: 96 additions & 21 deletions .github/workflows/pd-store-ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,7 @@ jobs:
runs-on: ubuntu-24.04
env:
USE_STAGE: 'false'
TRAVIS_DIR: hugegraph-server/hugegraph-dist/src/assembly/travis
steps:
- name: Checkout
uses: actions/checkout@v7
Expand All @@ -97,6 +98,9 @@ jobs:
- name: Verify Store stop failure propagation and PID retention
run: bash hugegraph-server/hugegraph-dist/src/assembly/travis/test-stop-hugegraph-store.sh

- name: Run Java runtime log checker tests
run: bash "$TRAVIS_DIR/test-check-java-runtime-logs.sh"

- name: Run JaCoCo report validator tests
run: hugegraph-server/hugegraph-dist/src/assembly/travis/test-check-jacoco-report.sh

Expand Down Expand Up @@ -129,7 +133,10 @@ jobs:
- name: Run hugegraph-struct test
if: ${{ hashFiles('hugegraph-struct/pom.xml') != '' }}
run: |
mvn -U -ntp -pl hugegraph-struct test
set -o pipefail
RUNTIME_LOG=$(mktemp "$RUNNER_TEMP/java-runtime.XXXXXX")
mvn -U -ntp -pl hugegraph-struct test 2>&1 | tee "$RUNTIME_LOG"
bash "$TRAVIS_DIR/check-java-runtime-logs.sh" "$RUNTIME_LOG"

pd:
if: inputs.run-pd
Expand Down Expand Up @@ -176,21 +183,30 @@ jobs:
# clean package so their JaCoCo data survives until the final report.
- name: Run common test
run: |
set -o pipefail
RUNTIME_LOG=$(mktemp "$RUNNER_TEMP/java-runtime.XXXXXX")
mvn test -pl hugegraph-pd/hg-pd-test -am \
-P pd-common-test -Djacoco.sessionId=pd-common-test
-P pd-common-test -Djacoco.sessionId=pd-common-test 2>&1 | tee "$RUNTIME_LOG"
bash "$TRAVIS_DIR/check-java-runtime-logs.sh" "$RUNTIME_LOG"

- name: Run core test
run: |
set -o pipefail
RUNTIME_LOG=$(mktemp "$RUNNER_TEMP/java-runtime.XXXXXX")
mvn test -pl hugegraph-pd/hg-pd-test -am \
-P pd-core-test -Djacoco.sessionId=pd-core-test
-P pd-core-test -Djacoco.sessionId=pd-core-test 2>&1 | tee "$RUNTIME_LOG"
bash "$TRAVIS_DIR/check-java-runtime-logs.sh" "$RUNTIME_LOG"

- name: Verify Surefire single-class routing
run: |
SUREFIRE_DIR=hugegraph-pd/hg-pd-test/target/surefire-reports
REPORT="$SUREFIRE_DIR/TEST-org.apache.hugegraph.pd.core.PDConfigTest.xml"
rm -f "$REPORT"
set -o pipefail
RUNTIME_LOG=$(mktemp "$RUNNER_TEMP/java-runtime.XXXXXX")
mvn test -pl hugegraph-pd/hg-pd-test -am \
-P pd-core-test -Dtest=PDConfigTest -ntp
-P pd-core-test -Dtest=PDConfigTest -ntp 2>&1 | tee "$RUNTIME_LOG"
bash "$TRAVIS_DIR/check-java-runtime-logs.sh" "$RUNTIME_LOG"
test -s "$REPORT" || {
echo "::error::PD single-class test produced no report"
exit 1
Expand Down Expand Up @@ -251,16 +267,24 @@ jobs:
- name: Prepare env and service
run: |
$TRAVIS_DIR/start-pd.sh
VERSION=$(mvn help:evaluate -Dexpression=project.version -q -DforceStdout)
echo "PD_DIR=hugegraph-pd/apache-hugegraph-pd-$VERSION" >> "$GITHUB_ENV"

- name: Run client test
run: |
set -o pipefail
RUNTIME_LOG=$(mktemp "$RUNNER_TEMP/java-runtime.XXXXXX")
mvn test -pl hugegraph-pd/hg-pd-test -am \
-P pd-client-test -Djacoco.sessionId=pd-client-test
-P pd-client-test -Djacoco.sessionId=pd-client-test 2>&1 | tee "$RUNTIME_LOG"
bash "$TRAVIS_DIR/check-java-runtime-logs.sh" "$RUNTIME_LOG" "$PD_DIR/logs"

- name: Run rest test
run: |
set -o pipefail
RUNTIME_LOG=$(mktemp "$RUNNER_TEMP/java-runtime.XXXXXX")
mvn test -pl hugegraph-pd/hg-pd-test -am \
-P pd-rest-test -Djacoco.sessionId=pd-rest-test
-P pd-rest-test -Djacoco.sessionId=pd-rest-test 2>&1 | tee "$RUNTIME_LOG"
bash "$TRAVIS_DIR/check-java-runtime-logs.sh" "$RUNTIME_LOG" "$PD_DIR/logs"

- name: Generate aggregate coverage report
run: |
Expand Down Expand Up @@ -354,8 +378,11 @@ jobs:
SUREFIRE_DIR=hugegraph-store/hg-store-test/target/surefire-reports
REPORT="$SUREFIRE_DIR/TEST-org.apache.hugegraph.store.raftcore.ZeroByteStringHelperTest.xml"
rm -f "$REPORT"
set -o pipefail
RUNTIME_LOG=$(mktemp "$RUNNER_TEMP/java-runtime.XXXXXX")
mvn test -pl hugegraph-store/hg-store-test -am \
-P store-raftcore-test -Dtest=ZeroByteStringHelperTest -ntp
-P store-raftcore-test -Dtest=ZeroByteStringHelperTest -ntp 2>&1 | tee "$RUNTIME_LOG"
bash "$TRAVIS_DIR/check-java-runtime-logs.sh" "$RUNTIME_LOG"
test -s "$REPORT" || {
echo "::error::Store single-class test produced no report"
exit 1
Expand Down Expand Up @@ -414,43 +441,67 @@ jobs:
run: |
$TRAVIS_DIR/start-pd.sh
$TRAVIS_DIR/start-store.sh
VERSION=$(mvn help:evaluate -Dexpression=project.version -q -DforceStdout)
echo "PD_DIR=hugegraph-pd/apache-hugegraph-pd-$VERSION" >> "$GITHUB_ENV"
echo "STORE_DIR=hugegraph-store/apache-hugegraph-store-$VERSION" >> "$GITHUB_ENV"

- name: Run historical Store ingress tests
run: |
set -o pipefail
RUNTIME_LOG=$(mktemp "$RUNNER_TEMP/java-runtime.XXXXXX")
mvn test -pl hugegraph-store/hg-store-node -am \
-Dtest=StoredRowIngressTest -Dsurefire.failIfNoSpecifiedTests=false \
-Djacoco.sessionId=store-node-test \
-Djacoco.destFile="$PWD/hugegraph-store/hg-store-test/target/store-node-test.exec"
-Djacoco.destFile="$PWD/hugegraph-store/hg-store-test/target/store-node-test.exec" 2>&1 | tee "$RUNTIME_LOG"
bash "$TRAVIS_DIR/check-java-runtime-logs.sh" "$RUNTIME_LOG" "$PD_DIR/logs" "$STORE_DIR/logs"

- name: Run common test
run: |
set -o pipefail
RUNTIME_LOG=$(mktemp "$RUNNER_TEMP/java-runtime.XXXXXX")
mvn test -pl hugegraph-store/hg-store-test -am \
-P store-common-test -Djacoco.sessionId=store-common-test
-P store-common-test -Djacoco.sessionId=store-common-test 2>&1 | tee "$RUNTIME_LOG"
bash "$TRAVIS_DIR/check-java-runtime-logs.sh" "$RUNTIME_LOG" "$PD_DIR/logs" "$STORE_DIR/logs"

- name: Run client test
run: |
set -o pipefail
RUNTIME_LOG=$(mktemp "$RUNNER_TEMP/java-runtime.XXXXXX")
mvn test -pl hugegraph-store/hg-store-test -am \
-P store-client-test -Djacoco.sessionId=store-client-test
-P store-client-test -Djacoco.sessionId=store-client-test 2>&1 | tee "$RUNTIME_LOG"
bash "$TRAVIS_DIR/check-java-runtime-logs.sh" "$RUNTIME_LOG" "$PD_DIR/logs" "$STORE_DIR/logs"

- name: Run rocksdb test
run: |
set -o pipefail
RUNTIME_LOG=$(mktemp "$RUNNER_TEMP/java-runtime.XXXXXX")
mvn test -pl hugegraph-store/hg-store-test -am \
-P store-rocksdb-test -Djacoco.sessionId=store-rocksdb-test
-P store-rocksdb-test -Djacoco.sessionId=store-rocksdb-test 2>&1 | tee "$RUNTIME_LOG"
bash "$TRAVIS_DIR/check-java-runtime-logs.sh" "$RUNTIME_LOG" "$PD_DIR/logs" "$STORE_DIR/logs"

- name: Run raft-core test
run: |
set -o pipefail
RUNTIME_LOG=$(mktemp "$RUNNER_TEMP/java-runtime.XXXXXX")
mvn test -pl hugegraph-store/hg-store-test -am \
-P store-raftcore-test -Djacoco.sessionId=store-raftcore-test
-P store-raftcore-test -Djacoco.sessionId=store-raftcore-test 2>&1 | tee "$RUNTIME_LOG"
bash "$TRAVIS_DIR/check-java-runtime-logs.sh" "$RUNTIME_LOG" "$PD_DIR/logs" "$STORE_DIR/logs"

- name: Run core test
run: |
set -o pipefail
RUNTIME_LOG=$(mktemp "$RUNNER_TEMP/java-runtime.XXXXXX")
mvn test -pl hugegraph-store/hg-store-test -am \
-P store-core-test -Djacoco.sessionId=store-core-test
-P store-core-test -Djacoco.sessionId=store-core-test 2>&1 | tee "$RUNTIME_LOG"
bash "$TRAVIS_DIR/check-java-runtime-logs.sh" "$RUNTIME_LOG" "$PD_DIR/logs" "$STORE_DIR/logs"

- name: Run server lifecycle and shutdown tests
run: |
set -o pipefail
RUNTIME_LOG=$(mktemp "$RUNNER_TEMP/java-runtime.XXXXXX")
mvn test -pl hugegraph-store/hg-store-test -am \
-P store-server-test -Djacoco.sessionId=store-server-test
-P store-server-test -Djacoco.sessionId=store-server-test 2>&1 | tee "$RUNTIME_LOG"
bash "$TRAVIS_DIR/check-java-runtime-logs.sh" "$RUNTIME_LOG" "$PD_DIR/logs" "$STORE_DIR/logs"

- name: Generate aggregate coverage report
run: |
Expand Down Expand Up @@ -598,10 +649,16 @@ jobs:
echo "[ci] open-file limit for HStore:" \
"soft=$(ulimit -Sn), hard=$(ulimit -Hn)"
$TRAVIS_DIR/install-backend.sh $BACKEND
VERSION=$(mvn help:evaluate -Dexpression=project.version -q -DforceStdout)
echo "PD_DIR=hugegraph-pd/apache-hugegraph-pd-$VERSION" >> "$GITHUB_ENV"
echo "STORE_DIR=hugegraph-store/apache-hugegraph-store-$VERSION" >> "$GITHUB_ENV"

- name: Run unit test
run: |
$TRAVIS_DIR/run-unit-test.sh $BACKEND
set -o pipefail
RUNTIME_LOG=$(mktemp "$RUNNER_TEMP/java-runtime.XXXXXX")
$TRAVIS_DIR/run-unit-test.sh $BACKEND 2>&1 | tee "$RUNTIME_LOG"
bash "$TRAVIS_DIR/check-java-runtime-logs.sh" "$RUNTIME_LOG" "$PD_DIR/logs" "$STORE_DIR/logs"

- name: Run core test
# Bound stuck HStore scans so the diagnostic steps below can run
Expand All @@ -610,16 +667,25 @@ jobs:
env:
HUGEGRAPH_TEST_TASK_LOG_LEVEL: WARN
run: |
$TRAVIS_DIR/run-core-test.sh $BACKEND
set -o pipefail
RUNTIME_LOG=$(mktemp "$RUNNER_TEMP/java-runtime.XXXXXX")
$TRAVIS_DIR/run-core-test.sh $BACKEND 2>&1 | tee "$RUNTIME_LOG"
bash "$TRAVIS_DIR/check-java-runtime-logs.sh" "$RUNTIME_LOG" "$PD_DIR/logs" "$STORE_DIR/logs"

- name: Run api test
run: |
$TRAVIS_DIR/run-api-test.sh $BACKEND $REPORT_DIR
set -o pipefail
RUNTIME_LOG=$(mktemp "$RUNNER_TEMP/java-runtime.XXXXXX")
$TRAVIS_DIR/run-api-test.sh $BACKEND $REPORT_DIR 2>&1 | tee "$RUNTIME_LOG"
bash "$TRAVIS_DIR/check-java-runtime-logs.sh" "$RUNTIME_LOG" "$PD_DIR/logs" "$STORE_DIR/logs"

- name: Run raft test
if: ${{ env.BACKEND == 'rocksdb' }}
run: |
$TRAVIS_DIR/run-api-test-for-raft.sh $BACKEND $REPORT_DIR
set -o pipefail
RUNTIME_LOG=$(mktemp "$RUNNER_TEMP/java-runtime.XXXXXX")
$TRAVIS_DIR/run-api-test-for-raft.sh $BACKEND $REPORT_DIR 2>&1 | tee "$RUNTIME_LOG"
bash "$TRAVIS_DIR/check-java-runtime-logs.sh" "$RUNTIME_LOG" "$PD_DIR/logs" "$STORE_DIR/logs"

# Include HStore in the upgrade compatibility gate. The dedicated HStore
# service setup above makes its structure/process suites exercise the
Expand All @@ -632,9 +698,11 @@ jobs:
env:
HUGEGRAPH_TEST_TASK_LOG_LEVEL: WARN
run: |
set -o pipefail
RUNTIME_LOG=$(mktemp "$RUNNER_TEMP/java-runtime.XXXXXX")
# Emit lightweight samples during execution: a lost runner cannot
# upload the failure-only diagnostics below.
python3 - "$TRAVIS_DIR/run-tinkerpop-test.sh" "$BACKEND" structure <<'PYTHON'
python3 - "$TRAVIS_DIR/run-tinkerpop-test.sh" "$BACKEND" structure 2>&1 <<'PYTHON' | tee "$RUNTIME_LOG"
import datetime
import os
import pathlib
Expand Down Expand Up @@ -677,18 +745,25 @@ jobs:
result = test.returncode
sys.exit(result if result >= 0 else 128 - result)
PYTHON
bash "$TRAVIS_DIR/check-java-runtime-logs.sh" "$RUNTIME_LOG" "$PD_DIR/logs" "$STORE_DIR/logs"

- name: Run TinkerPop process standard test
if: ${{ env.RUN_TINKERPOP_TESTS == 'true' }}
timeout-minutes: 120
run: |
$TRAVIS_DIR/run-tinkerpop-test.sh $BACKEND process-standard
set -o pipefail
RUNTIME_LOG=$(mktemp "$RUNNER_TEMP/java-runtime.XXXXXX")
$TRAVIS_DIR/run-tinkerpop-test.sh $BACKEND process-standard 2>&1 | tee "$RUNTIME_LOG"
bash "$TRAVIS_DIR/check-java-runtime-logs.sh" "$RUNTIME_LOG" "$PD_DIR/logs" "$STORE_DIR/logs"

- name: Run TinkerPop process feature test
if: ${{ env.RUN_TINKERPOP_TESTS == 'true' }}
timeout-minutes: 120
run: |
$TRAVIS_DIR/run-tinkerpop-test.sh $BACKEND process-feature
set -o pipefail
RUNTIME_LOG=$(mktemp "$RUNNER_TEMP/java-runtime.XXXXXX")
$TRAVIS_DIR/run-tinkerpop-test.sh $BACKEND process-feature 2>&1 | tee "$RUNTIME_LOG"
bash "$TRAVIS_DIR/check-java-runtime-logs.sh" "$RUNTIME_LOG" "$PD_DIR/logs" "$STORE_DIR/logs"

- name: Collect HStore diagnostics on failure or timeout
if: ${{ failure() || cancelled() }}
Expand Down
Loading
Loading