diff --git a/.asf.yaml b/.asf.yaml
index 1d9d56e20..ee867caa4 100644
--- a/.asf.yaml
+++ b/.asf.yaml
@@ -45,7 +45,7 @@ github:
# contexts are the names of checks that must pass
contexts:
- check-license-header
- - Analyze (java)
+ - affected-module-tests
required_pull_request_reviews:
dismiss_stale_reviews: true
require_code_owner_reviews: false
diff --git a/.github/actions/setup-hugegraph-server/action.yml b/.github/actions/setup-hugegraph-server/action.yml
index cabd8f8b0..00a3247a1 100644
--- a/.github/actions/setup-hugegraph-server/action.yml
+++ b/.github/actions/setup-hugegraph-server/action.yml
@@ -1,33 +1,103 @@
-name: 'Setup HugeGraph Server'
-description: 'Prepare environment and install HugeGraph server from source'
-
+name: Setup HugeGraph Server
+description: Build an immutable fixture once, or start an independent test service
inputs:
- travis-dir:
- description: 'Path to the Travis assembly directory containing install scripts'
- required: true
- commit-id:
- description: 'HugeGraph commit SHA to install'
- required: true
-
+ mode:
+ description: 'standalone (legacy), build (upload), or start (download)'
+ default: standalone
+ artifact-name:
+ description: Workflow-local immutable fixture artifact name
+ default: ''
repository:
- description: 'GitHub owner/repository containing the server source'
- default: 'apache/hugegraph'
+ description: GitHub owner/repository containing the Server source
+ default: apache/hugegraph
fetch-ref:
- description: 'Fetchable ref; the installer verifies its HEAD against commit-id'
+ description: Fetchable source ref (must resolve to commit-id)
default: ''
-
+ commit-id:
+ description: Exact Server commit SHA
+ required: true
+ java-version:
+ description: JDK major used for the build and runtime
+ default: '11'
+ java-home:
+ description: Optional fixture JDK scoped to this subprocess; keeps Toolchain JDK unchanged
+ default: ''
+ isolate-maven-repo:
+ description: Build fixtures without replacing Toolchain Maven dependencies
+ default: 'false'
+ travis-dir:
+ description: Module service setup directory
+ default: hugegraph-loader/assembly/travis
runs:
using: composite
steps:
- - name: Prepare env and service
+ - name: Download shared Server fixture
+ if: inputs.mode == 'start'
+ uses: actions/download-artifact@v4
+ with:
+ name: ${{ inputs.artifact-name }}
+ path: ${{ runner.temp }}/hugegraph-fixture
+ - name: Restore exact Server build cache
+ if: inputs.mode == 'build'
+ uses: actions/cache@v4
+ with:
+ path: ~/hugegraph-fixture-cache
+ key: server-fixture-v1-${{ runner.os }}-${{ runner.arch }}-${{ inputs.repository }}-${{ inputs.commit-id }}-jdk${{ inputs.java-version }}-${{ hashFiles('.github/actions/setup-hugegraph-server/action.yml', '.github/actions/setup-hugegraph-server/fixture.sh', '.github/actions/setup-hugegraph-server/manifest.py', '.github/actions/setup-java-env/action.yml', '.github/configs/settings.xml', 'hugegraph-client/assembly/travis/checkout-server.sh', 'hugegraph-client/assembly/travis/start-hugegraph-servers.sh', 'hugegraph-*/assembly/travis/install-hugegraph-from-source.sh', 'hugegraph-hubble/hubble-dist/assembly/travis/download-hugegraph.sh') }}
+ - name: Prepare fixture or start service
shell: bash
env:
- TRAVIS_SCRIPT_DIR: ${{ inputs.travis-dir }}
- SERVER_COMMIT: ${{ inputs.commit-id }}
+ FIXTURE_MODE: ${{ inputs.mode }}
+ FIXTURE_REPOSITORY: ${{ inputs.repository }}
+ FIXTURE_COMMIT: ${{ inputs.commit-id }}
+ FIXTURE_REF: ${{ inputs.fetch-ref }}
+ FIXTURE_BUILD_INPUTS: ${{ hashFiles('.github/actions/setup-hugegraph-server/action.yml', '.github/actions/setup-hugegraph-server/fixture.sh', '.github/actions/setup-hugegraph-server/manifest.py', '.github/actions/setup-java-env/action.yml', '.github/configs/settings.xml', 'hugegraph-client/assembly/travis/checkout-server.sh', 'hugegraph-client/assembly/travis/start-hugegraph-servers.sh', 'hugegraph-*/assembly/travis/install-hugegraph-from-source.sh', 'hugegraph-hubble/hubble-dist/assembly/travis/download-hugegraph.sh') }}
+ FIXTURE_PLATFORM: ${{ runner.os }}-${{ runner.arch }}
+ FIXTURE_JAVA: ${{ inputs.java-version }}
+ FIXTURE_JAVA_HOME: ${{ inputs.java-home }}
+ ISOLATE_FIXTURE_REPO: ${{ inputs.isolate-maven-repo }}
+ FIXTURE_DIR: ${{ runner.temp }}/hugegraph-fixture
SERVER_REPOSITORY: ${{ inputs.repository }}
SERVER_FETCH_REF: ${{ inputs.fetch-ref }}
+ TRAVIS_DIR: ${{ inputs.travis-dir }}
+ FIXTURE_ACTION: ${{ github.action_path }}
+ FIXTURE_ARTIFACT: ${{ inputs.artifact-name }}
run: |
- installer=$(realpath "${TRAVIS_SCRIPT_DIR}/install-hugegraph-from-source.sh")
- server_build=$(mktemp -d "$RUNNER_TEMP/hugegraph-build.XXXXXX")
- cd "$server_build"
- bash "$installer" "$SERVER_COMMIT"
+ set -euo pipefail
+ if [[ -n "$FIXTURE_JAVA_HOME" ]]; then
+ [[ -x "$FIXTURE_JAVA_HOME/bin/java" && -x "$FIXTURE_JAVA_HOME/bin/javac" ]]
+ export JAVA_HOME="$FIXTURE_JAVA_HOME"
+ export PATH="$JAVA_HOME/bin:$PATH"
+ fi
+ if [[ "$ISOLATE_FIXTURE_REPO" == true && "$FIXTURE_MODE" != start ]]; then
+ fixture_repo=$(mktemp -d "$RUNNER_TEMP/hugegraph-fixture-m2.XXXXXX")
+ export MAVEN_ARGS="-Dmaven.repo.local=$fixture_repo"
+ export MAVEN_OPTS="${MAVEN_OPTS:-} -Dmaven.repo.local=$fixture_repo"
+ fi
+ if [[ "$FIXTURE_MODE" != standalone && -z "$FIXTURE_ARTIFACT" ]]; then
+ echo 'artifact-name is required for build/start mode' >&2
+ exit 1
+ fi
+ if [[ "$FIXTURE_MODE" == build ]]; then
+ bash "$FIXTURE_ACTION/fixture.sh"
+ else
+ # Keep downloaded archives and legacy source checkouts out of the caller's tree.
+ TRAVIS_DIR=$(cd "$TRAVIS_DIR" && pwd)
+ export TRAVIS_DIR
+ server_build=$(mktemp -d "${RUNNER_TEMP:-${TMPDIR:-/tmp}}/hugegraph-build.XXXXXX")
+ cd "$server_build"
+ if [[ -f "$TRAVIS_DIR/install-hugegraph-from-source.sh" ]]; then
+ bash "$TRAVIS_DIR/install-hugegraph-from-source.sh" "$FIXTURE_COMMIT"
+ else
+ bash "$TRAVIS_DIR/install-hugegraph.sh" "$FIXTURE_COMMIT" "$FIXTURE_REF"
+ fi
+ fi
+ - name: Upload shared Server fixture
+ if: inputs.mode == 'build'
+ uses: actions/upload-artifact@v4
+ with:
+ name: ${{ inputs.artifact-name }}
+ path: |
+ ${{ runner.temp }}/hugegraph-fixture/server.tar.gz
+ ${{ runner.temp }}/hugegraph-fixture/manifest.json
+ if-no-files-found: error
+ retention-days: 7
diff --git a/.github/actions/setup-hugegraph-server/fixture.sh b/.github/actions/setup-hugegraph-server/fixture.sh
new file mode 100644
index 000000000..9fc5f1850
--- /dev/null
+++ b/.github/actions/setup-hugegraph-server/fixture.sh
@@ -0,0 +1,81 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with this
+# work for additional information regarding copyright ownership. The ASF
+# licenses this file to You under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+#
+set -euo pipefail
+
+# Identity includes every build input. Never restore an archive from another source.
+FIXTURE_REPOSITORY=${FIXTURE_REPOSITORY:-${SERVER_REPOSITORY:-apache/hugegraph}}
+FIXTURE_COMMIT=${FIXTURE_COMMIT:-${1:-}}
+FIXTURE_REF=${FIXTURE_REF:-$FIXTURE_COMMIT}
+FIXTURE_JAVA=${FIXTURE_JAVA:-11}
+FIXTURE_DIR=${FIXTURE_DIR:-${RUNNER_TEMP:-${TMPDIR:-/tmp}}/hugegraph-fixture}
+FIXTURE_HELPER=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/manifest.py
+FIXTURE_BUILD_INPUTS=${FIXTURE_BUILD_INPUTS:-$(python3 "$FIXTURE_HELPER" inputs)}
+FIXTURE_PLATFORM=${FIXTURE_PLATFORM:-$(uname -sm)}
+FIXTURE_CONFIG="package -DskipTests -Dmaven.javadoc.skip=true -ntp;schema=1;inputs=$FIXTURE_BUILD_INPUTS;platform=$FIXTURE_PLATFORM"
+export FIXTURE_REPOSITORY FIXTURE_COMMIT FIXTURE_JAVA FIXTURE_CONFIG FIXTURE_DIR
+[[ "$FIXTURE_REPOSITORY" =~ ^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$ ]]
+[[ "$FIXTURE_COMMIT" =~ ^[0-9a-f]{40}$ ]]
+[[ "$FIXTURE_JAVA" =~ ^[0-9]+$ ]]
+
+fixture_validate_java() {
+ actual_java=$(java -version 2>&1 | sed -n 's/.*version "\([0-9][0-9]*\).*".*/\1/p' | head -1)
+ [[ "$actual_java" == "$FIXTURE_JAVA" ]] || { echo 'Fixture JDK mismatch' >&2; return 1; }
+}
+
+fixture_verify() { python3 "$FIXTURE_HELPER" verify; }
+fixture_build() {
+ local cache_key cache_dir source_dir archive
+ cache_key=$(printf '%s\n' "$FIXTURE_REPOSITORY" "$FIXTURE_COMMIT" "$FIXTURE_JAVA" "$FIXTURE_CONFIG" | shasum -a 256 | cut -d ' ' -f 1)
+ cache_dir="${HOME}/hugegraph-fixture-cache/$cache_key"
+ mkdir -p "$FIXTURE_DIR"
+ if [[ -f "$cache_dir/server.tar.gz" && ! -L "$cache_dir/server.tar.gz" &&
+ -f "$cache_dir/manifest.json" && ! -L "$cache_dir/manifest.json" ]] &&
+ cp "$cache_dir"/server.tar.gz "$cache_dir"/manifest.json "$FIXTURE_DIR/"; then
+ if fixture_verify; then return; fi
+ fi
+ rm -f "$FIXTURE_DIR/server.tar.gz" "$FIXTURE_DIR/manifest.json"
+ fixture_validate_java
+ source_dir="$(mktemp -d "${TMPDIR:-/tmp}/hg-fixture-source.XXXXXX")/source"
+ SERVER_REPOSITORY=$FIXTURE_REPOSITORY SERVER_FETCH_REF=$FIXTURE_REF \
+ bash "${FIXTURE_HELPER%/*}/../../../hugegraph-client/assembly/travis/checkout-server.sh" "$FIXTURE_COMMIT" "$source_dir"
+ (cd "$source_dir" && mvn package -DskipTests -Dmaven.javadoc.skip=true -ntp)
+ archive=$(find "$source_dir/hugegraph-server" -maxdepth 1 -name 'apache-hugegraph-*.tar.gz' -print)
+ [[ -n "$archive" && -f "$archive" ]]
+ cp "$archive" "$FIXTURE_DIR/server.tar.gz"
+ export FIXTURE_ARCHIVE_NAME=${archive##*/}
+ python3 "$FIXTURE_HELPER" create
+ fixture_verify
+ mkdir -p "$cache_dir"
+ cp "$FIXTURE_DIR/server.tar.gz" "$FIXTURE_DIR/manifest.json" "$cache_dir/"
+ rm -rf "$source_dir"
+}
+fixture_copy_archive() {
+ fixture_validate_java
+ fixture_verify
+ local name
+ name=$(python3 "$FIXTURE_HELPER" name)
+ cp "$FIXTURE_DIR/server.tar.gz" "./$name"
+}
+source "${FIXTURE_HELPER%/*}/service-wait.sh"
+
+case "${FIXTURE_MODE:-standalone}" in
+ build) fixture_build ;;
+ start) fixture_copy_archive ;;
+ standalone) fixture_build; fixture_copy_archive ;;
+ *) echo 'Invalid fixture mode' >&2; exit 1 ;;
+esac
diff --git a/.github/actions/setup-hugegraph-server/manifest.py b/.github/actions/setup-hugegraph-server/manifest.py
new file mode 100644
index 000000000..c7446cecf
--- /dev/null
+++ b/.github/actions/setup-hugegraph-server/manifest.py
@@ -0,0 +1,71 @@
+#!/usr/bin/env python3
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with this
+# work for additional information regarding copyright ownership. The ASF
+# licenses this file to You under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+#
+"""Verify immutable CI fixture identity and archive bytes with the standard library."""
+import hashlib
+import json
+import os
+import pathlib
+import re
+import sys
+
+if sys.argv[1] == "inputs":
+ repo = pathlib.Path(__file__).resolve().parents[3]
+ patterns = (
+ '.github/actions/setup-hugegraph-server/action.yml',
+ '.github/actions/setup-hugegraph-server/fixture.sh',
+ '.github/actions/setup-hugegraph-server/manifest.py',
+ '.github/actions/setup-java-env/action.yml', '.github/configs/settings.xml',
+ 'hugegraph-client/assembly/travis/checkout-server.sh',
+ 'hugegraph-client/assembly/travis/start-hugegraph-servers.sh',
+ 'hugegraph-*/assembly/travis/install-hugegraph-from-source.sh',
+ 'hugegraph-hubble/hubble-dist/assembly/travis/download-hugegraph.sh',
+ )
+ inputs = hashlib.sha256()
+ files = sorted({p for pattern in patterns for p in repo.glob(pattern)})
+ for path in files:
+ inputs.update(str(path.relative_to(repo)).encode() + b"\0" + path.read_bytes())
+ print(inputs.hexdigest())
+ sys.exit(0)
+
+root = pathlib.Path(os.environ["FIXTURE_DIR"])
+identity = {key.lower(): os.environ["FIXTURE_" + key] for key in
+ ("REPOSITORY", "COMMIT", "JAVA", "CONFIG")}
+def digest():
+ h = hashlib.sha256()
+ with (root / "server.tar.gz").open("rb") as stream:
+ for chunk in iter(lambda: stream.read(1024 * 1024), b""):
+ h.update(chunk)
+ return h.hexdigest()
+try:
+ if sys.argv[1] == "create":
+ data = dict(identity, archive_name=os.environ["FIXTURE_ARCHIVE_NAME"], sha256=digest())
+ (root / "manifest.json").write_text(json.dumps(data, sort_keys=True) + "\n")
+ else:
+ if (root / "manifest.json").is_symlink() or (root / "server.tar.gz").is_symlink():
+ raise ValueError("fixture must contain regular files")
+ data = json.loads((root / "manifest.json").read_text())
+ if any(data.get(key) != value for key, value in identity.items()):
+ raise ValueError("fixture origin, commit, JDK or build configuration mismatch")
+ if not re.fullmatch(r"apache-hugegraph-[A-Za-z0-9_.-]+\.tar\.gz", data.get("archive_name", "")):
+ raise ValueError("invalid archive name")
+ if data.get("sha256") != digest():
+ raise ValueError("fixture checksum mismatch")
+ if sys.argv[1] == "name":
+ print(data["archive_name"])
+except (OSError, ValueError, KeyError) as error:
+ sys.exit(str(error))
diff --git a/.github/actions/setup-hugegraph-server/runtime_test.py b/.github/actions/setup-hugegraph-server/runtime_test.py
new file mode 100644
index 000000000..91ce54282
--- /dev/null
+++ b/.github/actions/setup-hugegraph-server/runtime_test.py
@@ -0,0 +1,212 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with this
+# work for additional information regarding copyright ownership. The ASF
+# licenses this file to You under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+#
+"""Exercise fixture identity, bounded downloads and service readiness offline."""
+import hashlib
+import json
+import os
+from pathlib import Path
+import subprocess
+import tempfile
+import unittest
+
+HERE = Path(__file__).resolve().parent
+ROOT = HERE.parents[2]
+SHA = 'a' * 40
+
+class RuntimeTests(unittest.TestCase):
+ def setUp(self):
+ self.temp = tempfile.TemporaryDirectory()
+ self.addCleanup(self.temp.cleanup)
+ self.dir = Path(self.temp.name)
+ self.bin = self.dir / 'bin'
+ self.bin.mkdir()
+ (self.bin/'java').write_text("#!/bin/bash\necho 'openjdk version \"11.0.1\"' >&2\n")
+ (self.bin/'java').chmod(0o755)
+ self.env = dict(os.environ, HOME=str(self.dir), PATH=f'{self.bin}:{os.environ["PATH"]}',
+ FIXTURE_DIR=str(self.dir / 'fixture'), FIXTURE_REPOSITORY='apache/hugegraph',
+ FIXTURE_COMMIT=SHA, FIXTURE_JAVA='11',
+ FIXTURE_BUILD_INPUTS='test-build-contract', FIXTURE_PLATFORM='Linux-X64',
+ FIXTURE_CONFIG='package -DskipTests -Dmaven.javadoc.skip=true -ntp;schema=1;inputs=test-build-contract;platform=Linux-X64')
+ Path(self.env['FIXTURE_DIR']).mkdir()
+ def command(self, name, body):
+ script = self.bin / name
+ script.write_text('#!/bin/bash\n' + body)
+ script.chmod(0o755)
+ def run_script(self, script, *args, env=None):
+ return subprocess.run(['bash', str(script), *args], env=env or self.env,
+ cwd=self.dir, text=True, capture_output=True, timeout=20)
+ def manifest(self, **extra):
+ root = Path(self.env['FIXTURE_DIR'])
+ data = {k.lower(): self.env['FIXTURE_'+k] for k in ('REPOSITORY','COMMIT','JAVA','CONFIG')}
+ (root/'server.tar.gz').write_bytes(b'archive')
+ data.update(archive_name='apache-hugegraph-fixture.tar.gz', sha256=hashlib.sha256(b'archive').hexdigest())
+ data.update(extra)
+ (root/'manifest.json').write_text(json.dumps(data))
+ def test_identity_checksum_and_regular_files(self):
+ self.manifest()
+ env = dict(self.env, FIXTURE_MODE='start')
+ self.assertEqual(self.run_script(HERE/'fixture.sh', env=env).returncode, 0)
+ for field, wrong in [('repository','fork/hugegraph'), ('commit','b'*40), ('java','17'), ('config','different'), ('sha256','bad')]:
+ self.manifest(**{field:wrong})
+ self.assertNotEqual(self.run_script(HERE/'fixture.sh', env=env).returncode, 0, field)
+ self.manifest()
+ self.command('java', "echo 'openjdk version \"17.0.1\"' >&2\n")
+ self.assertNotEqual(self.run_script(HERE/'fixture.sh', env=env).returncode, 0)
+ self.command('java', "echo 'openjdk version \"11.0.1\"' >&2\n")
+ self.manifest(archive_name='../escape.tar.gz')
+ self.assertNotEqual(self.run_script(HERE/'fixture.sh', env=env).returncode, 0)
+ self.manifest()
+ archive = Path(self.env['FIXTURE_DIR'])/'server.tar.gz'
+ archive.unlink()
+ archive.symlink_to(self.dir/'missing')
+ self.assertNotEqual(self.run_script(HERE/'fixture.sh', env=env).returncode, 0)
+ def test_composite_step_isolates_fixture_jdk_from_caller(self):
+ action = (HERE / 'action.yml').read_text()
+ block = action.split(' run: |\n', 1)[1].split(' - name:', 1)[0]
+ step = self.dir / 'action-step.sh'
+ step.write_text('\n'.join(line[8:] for line in block.splitlines()) + '\n')
+ jdk11 = self.dir / 'jdk11'
+ jdk17 = self.dir / 'jdk17'
+ for version, root in ((11, jdk11), (17, jdk17)):
+ (root / 'bin').mkdir(parents=True)
+ for name, body in [('java', f'echo \'openjdk version "{version}.0.1"\' >&2'), ('javac', 'exit 0')]:
+ binary = root / 'bin' / name
+ binary.write_text('#!/bin/bash\n' + body + '\n')
+ binary.chmod(0o755)
+ travis = self.dir / 'service'
+ travis.mkdir()
+ installer = travis / 'install-hugegraph-from-source.sh'
+ installer.write_text('#!/bin/bash\nset -e\nsource "$FIXTURE_ACTION/fixture.sh"\n'
+ 'java -version 2> "$HOME/started-java"\n')
+ for fixture_version, fixture_jdk, caller_version, caller_jdk in (
+ (11, jdk11, 17, jdk17), (17, jdk17, 11, jdk11)):
+ with self.subTest(fixture=fixture_version, caller=caller_version):
+ self.manifest(java=str(fixture_version))
+ env = dict(self.env, JAVA_HOME=str(caller_jdk),
+ PATH=f'{caller_jdk}/bin:{self.env["PATH"]}',
+ FIXTURE_JAVA=str(fixture_version), FIXTURE_JAVA_HOME=str(fixture_jdk),
+ FIXTURE_MODE='start', FIXTURE_ARTIFACT='shared', FIXTURE_ACTION=str(HERE),
+ ISOLATE_FIXTURE_REPO='false', TRAVIS_DIR=str(travis), RUNNER_TEMP=str(self.dir))
+ result = subprocess.run(
+ ['bash', '-c', 'bash "$HOME/action-step.sh" && java -version 2> "$HOME/caller-java"'],
+ cwd=self.dir, env=env, text=True, capture_output=True, timeout=20)
+ self.assertEqual(0, result.returncode, result.stderr)
+ self.assertIn(f'{fixture_version}.0.1', (self.dir / 'started-java').read_text())
+ self.assertIn(f'{caller_version}.0.1', (self.dir / 'caller-java').read_text())
+ # A fixture must reject accidentally inheriting the caller's different JDK.
+ env['FIXTURE_JAVA_HOME'] = str(caller_jdk)
+ result = subprocess.run(['bash', str(step)], cwd=self.dir, env=env,
+ text=True, capture_output=True, timeout=20)
+ self.assertNotEqual(0, result.returncode)
+ self.assertIn('Fixture JDK mismatch', result.stderr)
+
+ def test_legacy_bash_service_deadline_and_success(self):
+ helper = str(HERE/'service-wait.sh')
+ self.command('curl', 'exit 1\n')
+ result = subprocess.run(
+ ['bash','-c',f'source "{helper}"; fixture_wait http://localhost:1'],
+ env=dict(self.env, SERVICE_READY_TIMEOUT='0'), text=True, capture_output=True, timeout=2)
+ self.assertNotEqual(result.returncode, 0)
+ self.assertIn('timed out', result.stderr)
+ self.command('curl', 'printf "%s\\n" "$@" > "$HOME/curl-args"; exit 0\n')
+ result = subprocess.run(['bash','-c',f'source "{helper}"; fixture_wait http://localhost:1'],
+ env=self.env, capture_output=True, timeout=2)
+ self.assertEqual(result.returncode, 0)
+ args = (self.dir/'curl-args').read_text()
+ self.assertIn('--max-time\n5', args)
+ self.assertIn('admin:pa', args)
+ def test_hadoop_invalid_cache_retries_at_most_twice_and_stops(self):
+ (self.dir/'hadoop-3.3.6.tar.gz').write_text('tampered')
+ self.command('curl', 'printf "%s\\n" "$@" >> "$HOME/download-args"; exit 22\n')
+ self.command('sudo', 'echo unexpected-install >> "$HOME/installed"; exit 99\n')
+ result = self.run_script(ROOT/'hugegraph-loader/assembly/travis/install-hadoop.sh')
+ self.assertNotEqual(result.returncode, 0)
+ args = (self.dir/'download-args').read_text()
+ self.assertEqual(args.count('--max-time\n900'), 2)
+ self.assertFalse((self.dir/'installed').exists())
+ self.assertFalse((self.dir/'hadoop-3.3.6.tar.gz').exists())
+ def test_hdfs_container_exit_fails_before_tests(self):
+ self.command('timeout', 'shift; exec "$@"\n')
+ self.command('docker', '''
+case "$1" in
+ inspect) echo false ;;
+ logs) echo 'NameNode startup failed' >&2 ;;
+esac
+''')
+ result = self.run_script(ROOT/'hugegraph-loader/assembly/travis/start-hdfs-ci.sh')
+ self.assertNotEqual(result.returncode, 0)
+ self.assertIn('NameNode startup failed', result.stderr)
+ self.assertIn('exited before readiness', result.stderr)
+ def test_hdfs_live_datanode_with_failed_write_is_not_ready(self):
+ self.command('timeout', 'shift; exec "$@"\n')
+ self.command('curl', '''echo '{"beans":[{"NumLiveDataNodes":1}]}'\n''')
+ self.command('docker', '''
+case "$1" in
+ inspect) echo true ;;
+ exec) if [[ "$*" == *"-put"* ]]; then echo 'Block write failed' >&2; exit 1; fi ;;
+esac
+''')
+ result = self.run_script(ROOT/'hugegraph-loader/assembly/travis/start-hdfs-ci.sh')
+ self.assertNotEqual(result.returncode, 0)
+ self.assertIn('Block write failed', result.stderr)
+ def test_real_http_readiness_requires_successful_auth_response(self):
+ from http.server import BaseHTTPRequestHandler, HTTPServer
+ from threading import Thread
+ class Handler(BaseHTTPRequestHandler):
+ def do_GET(self):
+ if self.path == '/graphs' and self.headers.get('Authorization') == 'Basic YWRtaW46cGE=':
+ self.send_response(200)
+ self.end_headers()
+ self.wfile.write(b'{"graphs":[]}')
+ else:
+ self.send_response(401)
+ self.end_headers()
+ def log_message(self, *args):
+ pass
+ server = HTTPServer(('127.0.0.1', 0), Handler)
+ self.addCleanup(server.server_close)
+ self.addCleanup(server.shutdown)
+ Thread(target=server.serve_forever, daemon=True).start()
+ helper = str(HERE/'service-wait.sh')
+ result = subprocess.run(['bash','-c',f'source "{helper}"; fixture_wait http://127.0.0.1:{server.server_port}'],
+ env=self.env, capture_output=True, timeout=5)
+ self.assertEqual(result.returncode, 0, result.stderr)
+
+ def test_exact_cache_hit_never_builds_or_starts(self):
+ self.manifest()
+ identity = '\n'.join(self.env['FIXTURE_'+k] for k in ('REPOSITORY','COMMIT','JAVA','CONFIG'))+'\n'
+ cache = self.dir/'hugegraph-fixture-cache'/hashlib.sha256(identity.encode()).hexdigest()
+ cache.mkdir(parents=True)
+ for name in ('server.tar.gz','manifest.json'):
+ (cache/name).write_bytes((Path(self.env['FIXTURE_DIR'])/name).read_bytes())
+ (Path(self.env['FIXTURE_DIR'])/name).unlink()
+ self.command('git', 'echo forbidden-git >> "$HOME/forbidden"; exit 99\n')
+ self.command('mvn', 'echo forbidden-build >> "$HOME/forbidden"; exit 99\n')
+ result = self.run_script(HERE/'fixture.sh', env=dict(self.env, FIXTURE_MODE='build'))
+ self.assertEqual(result.returncode, 0, result.stderr)
+ self.assertFalse((self.dir/'forbidden').exists())
+ self.assertFalse((self.dir/'apache-hugegraph-fixture.tar.gz').exists())
+ # An incomplete exact cache must rebuild, rather than poison future runs.
+ (cache/'manifest.json').unlink()
+ result = self.run_script(HERE/'fixture.sh', env=dict(self.env, FIXTURE_MODE='build'))
+ self.assertNotEqual(result.returncode, 0)
+ self.assertTrue((self.dir/'forbidden').exists())
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/.github/actions/setup-hugegraph-server/service-wait.sh b/.github/actions/setup-hugegraph-server/service-wait.sh
new file mode 100644
index 000000000..21f13e2a5
--- /dev/null
+++ b/.github/actions/setup-hugegraph-server/service-wait.sh
@@ -0,0 +1,27 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with this
+# work for additional information regarding copyright ownership. The ASF
+# licenses this file to You under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+#
+fixture_wait() {
+ local url=$1 deadline=$((SECONDS + ${SERVICE_READY_TIMEOUT:-300}))
+ while (( SECONDS < deadline )); do
+ # Use an authenticated API response, rather than a listening socket.
+ if curl --fail --silent --insecure --connect-timeout 2 --max-time 5 -u admin:pa "$url/graphs" >/dev/null; then return; fi
+ sleep 2
+ done
+ echo "HugeGraph readiness timed out: $url" >&2
+ return 1
+}
diff --git a/.github/actions/setup-java-env/action.yml b/.github/actions/setup-java-env/action.yml
index 9d897bfd0..1d364b388 100644
--- a/.github/actions/setup-java-env/action.yml
+++ b/.github/actions/setup-java-env/action.yml
@@ -29,5 +29,5 @@ runs:
if: ${{ inputs.use-stage == 'true' }}
shell: bash
run: |
- cp $HOME/.m2/settings.xml /tmp/settings.xml
- mv -vf .github/configs/settings.xml $HOME/.m2/settings.xml
+ cp "$HOME/.m2/settings.xml" /tmp/settings.xml
+ cp .github/configs/settings.xml "$HOME/.m2/settings.xml"
diff --git a/.github/scripts/check-rerun.py b/.github/scripts/check-rerun.py
new file mode 100644
index 000000000..408b5655b
--- /dev/null
+++ b/.github/scripts/check-rerun.py
@@ -0,0 +1,75 @@
+#!/usr/bin/env python3
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Check the failed run again immediately before an automatic retry."""
+
+import argparse
+import json
+import os
+import subprocess
+from urllib.parse import urlencode
+
+
+def api(path):
+ result = subprocess.run(
+ ["gh", "api", "--method", "GET", path], check=True,
+ capture_output=True, timeout=30,
+ )
+ return json.loads(result.stdout)
+
+
+def decide(repository, run_id, expected_attempt, max_reruns, fetch=api):
+ run = fetch(f"repos/{repository}/actions/runs/{run_id}")
+ if (run.get("status") != "completed" or run.get("conclusion") != "failure"
+ or run.get("run_attempt") != expected_attempt):
+ return "skip", "source run changed or is no longer a completed failure"
+ if expected_attempt > max_reruns:
+ return "skip", "retry limit reached"
+ sha = run.get("head_sha")
+ if not sha:
+ return "skip", "missing head SHA"
+ if run.get("event") == "pull_request":
+ return "skip", "PR automatic retries are deferred pending trusted execution evidence"
+ if run.get("event") == "push":
+ branch = run.get("head_branch")
+ if not branch:
+ return "skip", "missing push branch"
+ # Read from the workflow repository, never a similarly named fork branch.
+ commits = fetch(f"repos/{repository}/commits?" + urlencode({"sha": branch, "per_page": 1}))
+ if commits and commits[0].get("sha") == sha:
+ return "rerun", "push branch head unchanged"
+ return "skip", "push branch head moved"
+ return "skip", "unsupported source event"
+
+
+def main():
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument("--repository", required=True)
+ parser.add_argument("--run-id", required=True, type=int)
+ parser.add_argument("--run-attempt", required=True, type=int)
+ parser.add_argument("--max-reruns", required=True, type=int)
+ args = parser.parse_args()
+ try:
+ action, reason = decide(args.repository, args.run_id, args.run_attempt, args.max_reruns)
+ except (subprocess.SubprocessError, OSError, ValueError, KeyError, TypeError, AttributeError):
+ action, reason = "skip", "unable to verify current run and source metadata"
+ print(f"{action}: {reason}")
+ with open(os.environ["GITHUB_OUTPUT"], "a", encoding="utf-8") as output:
+ output.write(f"action={action}\nreason={reason}\n")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/.github/scripts/check-toolchain-image.py b/.github/scripts/check-toolchain-image.py
new file mode 100644
index 000000000..cfb100fd5
--- /dev/null
+++ b/.github/scripts/check-toolchain-image.py
@@ -0,0 +1,154 @@
+#!/usr/bin/env python3
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Build and start the selected Toolchain Dockerfile from this checkout."""
+
+import json
+from html.parser import HTMLParser
+import os
+from pathlib import Path
+import subprocess
+import sys
+import tempfile
+import time
+import urllib.error
+from urllib.parse import urljoin, urlparse
+import urllib.request
+
+
+def output(*args):
+ return subprocess.check_output(args, text=True, timeout=60).strip()
+
+
+def verify_image(container, built):
+ actual = output("docker", "inspect", "--format", "{{.Image}}", container)
+ if not built or actual != built:
+ raise RuntimeError(f"Container {container} uses {actual}; expected built image {built}")
+
+
+def valid_hubble_about(body):
+ if not isinstance(body, dict) or body.get("status") != 200:
+ return False
+ data = body.get("data")
+ return (isinstance(data, dict) and data.get("name") == "hugegraph-hubble"
+ and isinstance(data.get("version"), str) and bool(data["version"].strip()))
+
+
+def check_loader(container):
+ help_text = output("docker", "exec", container, "./bin/hugegraph-loader.sh", "--help")
+ if "--help" not in help_text or "Usage:" not in help_text:
+ raise RuntimeError("Loader CLI did not produce its usage contract")
+
+
+class HubblePage(HTMLParser):
+ def __init__(self):
+ super().__init__()
+ self.has_root = False
+ self.scripts = []
+ self.styles = []
+
+ def handle_starttag(self, tag, attrs):
+ attrs = dict(attrs)
+ if tag == "div" and attrs.get("id") == "root":
+ self.has_root = True
+ if tag == "script" and attrs.get("src"):
+ self.scripts.append(attrs["src"])
+ if tag == "link" and "stylesheet" in attrs.get("rel", "").lower().split() and attrs.get("href"):
+ self.styles.append(attrs["href"])
+
+
+class NoRedirect(urllib.request.HTTPRedirectHandler):
+ def redirect_request(self, request, response, code, message, headers, newurl):
+ return None
+
+
+def check_hubble_ui(origin):
+ page_url = origin + "/"
+ opener = urllib.request.build_opener(NoRedirect)
+ with opener.open(page_url, timeout=5) as response:
+ if response.headers.get_content_type() != "text/html":
+ raise RuntimeError("Hubble UI did not return HTML")
+ page = HubblePage()
+ page.feed(response.read().decode("utf-8"))
+ if not page.has_root:
+ raise RuntimeError("Hubble UI is missing its React root")
+ def local(ref):
+ parsed = urlparse(urljoin(page_url, ref))
+ return (parsed.scheme, parsed.netloc) == (urlparse(origin).scheme, urlparse(origin).netloc)
+
+ scripts = [urljoin(page_url, ref) for ref in page.scripts if local(ref)]
+ styles = [urljoin(page_url, ref) for ref in page.styles if local(ref)]
+ if not scripts:
+ raise RuntimeError("Hubble UI is missing a local JavaScript bundle")
+ for resource, types in ([(url, {"text/javascript", "application/javascript", "application/x-javascript"})
+ for url in scripts] + [(url, {"text/css"}) for url in styles]):
+ with opener.open(resource, timeout=5) as response:
+ body = response.read()
+ if response.headers.get_content_type() not in types or not body.strip() or body.lstrip().startswith(b"<"):
+ raise RuntimeError("Hubble UI asset is empty or has invalid content: " + resource)
+
+
+def wait_hubble(container):
+ port = output("docker", "port", container, "8088/tcp").splitlines()[0].rsplit(":", 1)[1]
+ deadline = time.monotonic() + 120
+ last_error = "no response"
+ while time.monotonic() < deadline:
+ if output("docker", "inspect", "--format", "{{.State.Running}}", container) != "true":
+ raise RuntimeError("Hubble exited before becoming ready")
+ try:
+ with urllib.request.urlopen(f"http://127.0.0.1:{port}/about", timeout=5) as response:
+ body = json.load(response)
+ if valid_hubble_about(body):
+ check_hubble_ui(f"http://127.0.0.1:{port}")
+ return
+ last_error = str(body)
+ except (OSError, ValueError, urllib.error.HTTPError) as error:
+ last_error = str(error)
+ time.sleep(2)
+ raise RuntimeError("Hubble readiness failed: " + last_error)
+
+
+def main(module):
+ if module not in {"loader", "hubble"}:
+ raise ValueError("Select loader or hubble")
+ tag = f"hugegraph/{module}:ci-{os.environ['GITHUB_RUN_ID']}-{os.environ['GITHUB_RUN_ATTEMPT']}"
+ container = f"hg-{module}-{os.environ['GITHUB_RUN_ID']}-{os.environ['GITHUB_RUN_ATTEMPT']}"
+ with tempfile.TemporaryDirectory() as temporary:
+ iidfile = Path(temporary) / "image-id"
+ subprocess.run(["docker", "build", "--progress=plain", "--iidfile", str(iidfile),
+ "-t", tag, "-f", f"hugegraph-{module}/Dockerfile", "."], check=True, timeout=2400)
+ built = iidfile.read_text().strip()
+ try:
+ args = ["docker", "run", "-d", "--pull", "never", "--name", container]
+ if module == "hubble":
+ args += ["-p", "127.0.0.1::8088"]
+ subprocess.run(args + [tag], check=True, timeout=60)
+ verify_image(container, built)
+ if module == "loader":
+ check_loader(container)
+ else:
+ wait_hubble(container)
+ if output("docker", "inspect", "--format", "{{.State.Running}}", container) != "true":
+ raise RuntimeError(f"{module} exited after its runtime check")
+ except BaseException:
+ subprocess.run(["docker", "logs", "--tail", "200", container], check=False, timeout=30)
+ raise
+ finally:
+ subprocess.run(["docker", "rm", "-f", container], check=False, timeout=60)
+
+
+if __name__ == "__main__":
+ main(sys.argv[1])
diff --git a/.github/scripts/ci-policy.py b/.github/scripts/ci-policy.py
new file mode 100644
index 000000000..1e1a8532d
--- /dev/null
+++ b/.github/scripts/ci-policy.py
@@ -0,0 +1,358 @@
+#!/usr/bin/env python3
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Conservative affected-module selection and current-run CI gating."""
+
+import argparse
+import json
+import os
+from pathlib import Path
+import subprocess
+
+MODULES = {
+ "server": ["server", "commons", "struct", "pd", "store", "hstore", "cluster", "docker", "helm", "dependency_license"],
+ "toolchain": ["client", "loader", "tools", "spark", "hubble", "go"],
+}
+PREFIXES = {
+ "server": {"hugegraph-server/": "server", "hugegraph-commons/": "commons",
+ "hugegraph-struct/": "struct", "hugegraph-pd/": "pd", "hugegraph-store/": "store",
+ "hugegraph-cluster-test/": "cluster", "docker/": "docker", "helm/": "helm"},
+ "toolchain": {"hugegraph-client/": "client", "hugegraph-client-go/": "go",
+ "hugegraph-loader/": "loader", "hugegraph-tools/": "tools",
+ "hugegraph-spark-connector/": "spark", "hugegraph-hubble/": "hubble"},
+}
+WORKFLOWS = {
+ "server": {"server-ci.yml": MODULES["server"], "commons-ci.yml": ["commons"],
+ "pd-store-ci.yml": ["pd", "store", "hstore"], "cluster-test-ci.yml": ["cluster"],
+ "docker-build-ci.yml": ["docker"], "helm-chart-ci.yml": ["helm"],
+ "codeql-analysis.yml": [], "riscv64-ci.yml": ["server"], "check-dependencies.yml": ["dependency_license"]},
+ "toolchain": {"client-ci.yml": ["client"], "client-go-ci.yml": ["go"],
+ "loader-ci.yml": ["loader"], "tools-ci.yml": ["tools"],
+ "spark-connector-ci.yml": ["spark"], "hubble-ci.yml": ["hubble"],
+ "codeql-analysis.yml": []},
+}
+IMAGES = {"loader_image", "hubble_image"}
+
+
+DEPENDENTS = {
+ "server": {"commons": ["server", "pd", "store", "hstore", "cluster"],
+ "struct": ["server", "pd", "store", "hstore", "cluster"],
+ "server": ["pd", "store", "hstore", "cluster"], "pd": ["store", "hstore", "cluster"],
+ "store": ["hstore", "cluster"], "hstore": ["cluster"]},
+ "toolchain": {"client": ["loader", "tools", "spark", "hubble"], "loader": ["hubble"]},
+}
+
+
+def git(*args):
+ return subprocess.check_output(["git", *args], text=True).strip()
+
+
+def api(path):
+ result = subprocess.run(["gh", "api", "--method", "GET", path], check=True, capture_output=True, timeout=30)
+ return json.loads(result.stdout)
+
+
+def documentation(path, mode="100644"):
+ """Only prose and static documentation assets; never source or config."""
+ if mode != "100644":
+ return False
+ p = Path(path)
+ prose_names = {"README.md", "README_CN.md", "README_ZH.md", "AGENTS.md"}
+ module_roots = {prefix.rstrip("/") for mapping in PREFIXES.values() for prefix in mapping}
+ module_roots.update("hugegraph-server/" + module for module in [
+ "hugegraph-api", "hugegraph-core", "hugegraph-dist", "hugegraph-example",
+ "hugegraph-hbase", "hugegraph-hstore", "hugegraph-rocksdb", "hugegraph-test"])
+ if p.name in prose_names and (str(p.parent) == "." or str(p.parent) in module_roots):
+ return True
+ return path.startswith("docs/") and p.suffix.lower() in {".md", ".rst", ".txt", ".png", ".jpg", ".jpeg", ".svg"}
+
+
+def unsafe_documentation(ref, paths):
+ entries = subprocess.check_output(["git", "ls-tree", "-rz", "--full-tree", ref]).split(b"\0")
+ for entry in entries:
+ if not entry:
+ continue
+ metadata, name = entry.split(b"\t", 1)
+ path = name.decode("utf-8", "surrogateescape")
+ if path in paths and documentation(path) and metadata.split(b" ", 1)[0] != b"100644":
+ return True
+ return False
+
+
+def packaged_image_document(path):
+ p = Path(path)
+ return (str(p.parent) in {"hugegraph-loader", "hugegraph-hubble"}
+ and p.name.startswith("README"))
+
+
+def image_only_path(path):
+ return path in {".dockerignore", "hugegraph-loader/Dockerfile", "hugegraph-hubble/Dockerfile",
+ ".github/workflows/image-ci.yml", ".github/scripts/check-toolchain-image.py",
+ ".github/scripts/test_toolchain_image.py"}
+
+
+def select_images(paths):
+ """Image packaging selection is independent of the Java consumer closure."""
+ selected = set()
+ for path in paths:
+ p = Path(path)
+ if path == "hugegraph-loader/Dockerfile":
+ selected.add("loader_image")
+ elif path == "hugegraph-hubble/Dockerfile":
+ selected.add("hubble_image")
+ elif (path in {"pom.xml", ".dockerignore", "hugegraph-client/pom.xml",
+ "hugegraph-tools/pom.xml", "hugegraph-spark-connector/pom.xml",
+ "hugegraph-dist/pom.xml"}
+ or path.startswith((".mvn/", "hugegraph-dist/release-docs/"))):
+ selected.update(IMAGES)
+ elif path.startswith("hugegraph-loader/") and (
+ p.name == "pom.xml" or path.startswith(("hugegraph-loader/assembly/descriptor/",
+ "hugegraph-loader/assembly/static/"))
+ or packaged_image_document(path) or p.name.startswith(("LICENSE", "NOTICE"))):
+ # Hubble's Dockerfile first builds and installs Client and Loader.
+ selected.update(IMAGES)
+ elif path.startswith("hugegraph-hubble/") and (
+ p.name == "pom.xml" or "/assembly/descriptor/" in path or "/assembly/static/" in path
+ or packaged_image_document(path)
+ or path == "hugegraph-hubble/hubble-dist/assembly/travis/check-hubble-dist.sh"
+ or path.startswith(("hugegraph-hubble/.mvn/", "hugegraph-hubble/hubble-fe/scripts/"))
+ or (path.startswith("hugegraph-hubble/hubble-be/src/main/resources/")
+ and p.name in {"application.properties", "hugegraph-hubble.properties", "log4j2.xml"})
+ or p.name in {"package.json", "yarn.lock", "package-lock.json", "webpack.config.js",
+ "tsconfig.json", "tsconfig.eslint.json", ".yarnrc", ".npmrc",
+ "config-overrides.js", ".babelrc", "babel.config.js"}):
+ selected.add("hubble_image")
+ elif documentation(path):
+ continue
+ elif path.startswith(".github/workflows/") and p.name in WORKFLOWS["toolchain"]:
+ continue
+ elif any(path.startswith(prefix) for prefix in PREFIXES["toolchain"]):
+ # Ordinary application/test edits stay on their module behavior lanes.
+ continue
+ else:
+ # Unknown shared inputs (including CI implementation) fail conservative.
+ selected.update(IMAGES)
+ return selected
+
+
+def select(project, paths):
+ selected = set()
+ for path in paths:
+ if project == "toolchain" and image_only_path(path):
+ continue
+ if documentation(path):
+ continue
+ if project == "server" and (Path(path).name == "pom.xml" or path.startswith("install-dist/")):
+ selected.add("dependency_license")
+ if path.startswith(".github/workflows/") and Path(path).name in WORKFLOWS[project]:
+ selected.update(WORKFLOWS[project][Path(path).name])
+ continue
+ if path.endswith(".proto"):
+ return set(MODULES[project])
+ if project == "server" and path == "hugegraph-commons/hugegraph-common/src/main/resources/version.properties":
+ selected.update(["commons", "docker"])
+ continue
+ if project == "server" and path == "hugegraph-store/hg-store-dist/src/assembly/static/bin/util.sh":
+ selected.update(["store", "docker"])
+ continue
+ if project == "toolchain" and path.startswith("hugegraph-client/assembly/travis/"):
+ selected.update(["client", "go"])
+ continue
+ if project == "server":
+ if path.startswith("hugegraph-server/hugegraph-hstore/"):
+ selected.update(["server", "hstore"])
+ continue
+ if path.startswith("hugegraph-server/hugegraph-dist/"):
+ selected.update(["server", "pd", "store", "hstore", "docker", "cluster"])
+ continue
+ if path.startswith(("hugegraph-server/hugegraph-core/", "hugegraph-server/hugegraph-api/",
+ "hugegraph-server/hugegraph-test/")):
+ selected.update(["server", "hstore"])
+ continue
+ module = next((value for prefix, value in PREFIXES[project].items() if path.startswith(prefix)), None)
+ if module is None or path.endswith(".proto"):
+ return set(MODULES[project])
+ selected.add(module)
+ if project == "server" and ("Dockerfile" in Path(path).name or "docker-entrypoint" in path):
+ selected.add("docker")
+ changed = True
+ while changed:
+ before = set(selected)
+ for module in before:
+ selected.update(DEPENDENTS[project].get(module, []))
+ changed = before != selected
+ return selected
+
+
+def packaged_readme_state(ref):
+ path = "hugegraph-hubble/README.md"
+ entry = git("ls-tree", ref, "--", path)
+ if not entry:
+ return "missing"
+ mode, kind, blob = entry.split("\t", 1)[0].split()
+ if mode != "100644" or kind != "blob":
+ return "invalid-mode:" + mode + ":" + blob
+ content = subprocess.check_output(["git", "cat-file", "blob", blob])
+ return "valid" if content.strip() else "empty:" + blob
+
+
+def suites(project, selected):
+ if project == "toolchain":
+ return sorted(selected)
+ result = []
+ if "server" in selected:
+ result.extend(["server_memory", "server_rocksdb"])
+ for module in ["commons", "cluster", "docker", "helm", "dependency_license"]:
+ if module in selected:
+ result.append(module)
+ if selected.intersection({"pd", "store", "hstore", "struct"}):
+ result.append("pd_store")
+ return sorted(result)
+
+
+class StaleInputError(RuntimeError):
+ """The checkout no longer represents the current PR inputs."""
+
+
+def require_current_pr(plan, fetch):
+ if not plan["pr"]:
+ return
+ live = fetch(f"repos/{plan['repository']}/pulls/{plan['pr']}")
+ if (live.get("state") != "open" or live["head"]["sha"] != plan["head"]
+ or live["base"]["sha"] != plan["base"]
+ or live["head"]["repo"]["full_name"] != plan["source"]
+ or live["head"]["ref"] != plan["branch"]
+ or live["base"]["repo"]["full_name"] != plan["repository"]):
+ raise StaleInputError("PR inputs changed; refresh the branch and start a new PR run")
+
+
+def create_plan(project, event, repository, fetch=api, external_inputs=None):
+ plan = {"schema": 1, "project": project, "repository": repository, "pr": 0,
+ "source": repository, "branch": "", "base": "", "head": git("rev-parse", "HEAD"),
+ "reason": "affected inputs from the cumulative PR diff",
+ "externalInputs": external_inputs or {}, "testedMergeSHA": git("rev-parse", "HEAD")}
+ selected_images = set()
+ try:
+ pr = event.get("pull_request")
+ if "pull_request" in event:
+ if not isinstance(pr, dict) or type(pr.get("number")) is not int or pr["number"] <= 0:
+ raise StaleInputError("PR event lacks a valid number")
+ plan["pr"] = pr["number"]
+ # An incomplete PR event must never fall back to a push plan with pr=0.
+ try:
+ plan.update(source=pr["head"]["repo"]["full_name"], base=pr["base"]["sha"],
+ head=pr["head"]["sha"], branch=pr["head"]["ref"])
+ except (KeyError, TypeError) as error:
+ raise StaleInputError("PR event lacks required input identity") from error
+ if not all(isinstance(plan[key], str) and plan[key] for key in ("source", "base", "head", "branch")):
+ raise StaleInputError("PR event has an empty input identity")
+ parents = git("show", "-s", "--format=%P", plan["testedMergeSHA"]).split()
+ if parents != [plan["base"], plan["head"]]:
+ raise StaleInputError("checkout is not the event PR merge; start a new PR run")
+ require_current_pr(plan, fetch)
+ # head/base objects must exist locally; workflow fetches both before planning.
+ ancestor = git("merge-base", plan["base"], plan["head"])
+ paths = git("diff", "--no-renames", "--name-only", ancestor, plan["head"]).splitlines()
+ else:
+ plan["base"] = event.get("before", "")
+ paths = git("diff", "--no-renames", "--name-only", plan["base"], plan["head"]).splitlines()
+ selected = select(project, paths)
+ selected_images = select_images(paths) if project == "toolchain" else set()
+ if unsafe_documentation(plan["base"], paths) or unsafe_documentation(plan["testedMergeSHA"], paths):
+ selected = set(MODULES[project])
+ if project == "toolchain" and packaged_readme_state(plan["testedMergeSHA"]) != "valid":
+ selected = set(MODULES[project])
+ except (subprocess.SubprocessError, OSError, ValueError, KeyError, TypeError, AttributeError):
+ selected = set(MODULES[project])
+ selected_images = set(IMAGES) if project == "toolchain" else set()
+ plan["reason"] = "selection unavailable: full required coverage"
+ if not selected and not selected_images:
+ plan["reason"] = "cumulative PR diff contains only plain prose documentation; modules unaffected"
+ plan["selected"] = sorted(selected)
+ for module in MODULES[project]:
+ plan[module] = module in selected
+ plan["expected"] = sorted(suites(project, selected) + list(selected_images))
+ plan["selectedImages"] = sorted(selected_images)
+ for image in IMAGES:
+ plan[image] = image in selected_images
+ plan["pd_store"] = any(plan.get(m, False) for m in ["pd", "store", "hstore", "struct"])
+ plan["needsFixture"] = project == "toolchain" and any(plan.get(m, False) for m in MODULES[project])
+ plan["fixture"] = plan["needsFixture"]
+ plan["compatibility"] = project == "server" and bool(selected.intersection({"server", "commons", "struct", "pd", "store", "hstore", "cluster"}))
+ plan["security"] = bool(selected) or any(p.startswith(".github/workflows/codeql") for p in locals().get("paths", []))
+ plan["security_languages"] = json.dumps(["java"])
+ return plan
+
+
+def gate(plan, results, fetch=None):
+ if results.get("plan", {}).get("result") != "success":
+ raise ValueError("planner did not succeed")
+ for suite in plan["expected"]:
+ if results.get(suite, {}).get("result") != "success":
+ raise ValueError("selected suite did not succeed: " + suite)
+ if plan["project"] == "toolchain":
+ if set(plan["expected"]).intersection(MODULES["toolchain"]):
+ if results.get("fixture", {}).get("result") != "success":
+ raise ValueError("selected tests lack successful fixture")
+ if "hubble" in plan["expected"] and results.get("hubble-fixture", {}).get("result") != "success":
+ raise ValueError("selected Hubble tests lack successful baseline fixture")
+ require_current_pr(plan, fetch or api)
+ report = {key: plan[key] for key in ["schema", "repository", "project", "pr", "source", "branch", "base",
+ "head", "testedMergeSHA", "externalInputs"]}
+ report.update(runID=int(os.environ.get("GITHUB_RUN_ID", "0")),
+ runAttempt=int(os.environ.get("GITHUB_RUN_ATTEMPT", "1")),
+ executed=plan["expected"], results=results)
+ # These fields describe this run; they do not authorize reuse by another run.
+ summary = "Selection: " + plan.get("reason", "affected inputs") + "\nExecuted: " + ", ".join(plan["expected"])
+ if os.environ.get("GITHUB_STEP_SUMMARY"):
+ with open(os.environ["GITHUB_STEP_SUMMARY"], "a", encoding="utf-8") as out:
+ out.write("## Affected module tests\n" + summary + "\n\nUnselected modules: "
+ + ", ".join(sorted(set(MODULES[plan["project"]]) - set(plan["selected"]))) + "\n")
+ print(summary)
+ return report
+
+
+def main():
+ parser = argparse.ArgumentParser(description=__doc__)
+ sub = parser.add_subparsers(dest="command", required=True)
+ p = sub.add_parser("plan")
+ p.add_argument("--project", choices=MODULES, required=True)
+ p.add_argument("--event-path", required=True)
+ p.add_argument("--repository", required=True)
+ p.add_argument("--output", required=True)
+ p.add_argument("--inputs-json")
+ p = sub.add_parser("gate")
+ p.add_argument("--plan", required=True)
+ p.add_argument("--results", required=True)
+ p.add_argument("--output", required=True)
+ args = parser.parse_args()
+ if args.command == "plan":
+ external = json.loads(Path(args.inputs_json).read_text()) if args.inputs_json else None
+ value = create_plan(args.project, json.loads(Path(args.event_path).read_text()), args.repository,
+ external_inputs=external)
+ if os.environ.get("GITHUB_OUTPUT"):
+ with open(os.environ["GITHUB_OUTPUT"], "a", encoding="utf-8") as output:
+ for key, item in value.items():
+ if isinstance(item, bool):
+ output.write(f"{key}={str(item).lower()}\n")
+ output.write(f"security_languages={value['security_languages']}\nplan_file={args.output}\n")
+ else:
+ value = gate(json.loads(Path(args.plan).read_text()), json.loads(Path(args.results).read_text()))
+ Path(args.output).write_text(json.dumps(value, indent=2) + "\n")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/.github/scripts/test_check_rerun.py b/.github/scripts/test_check_rerun.py
new file mode 100644
index 000000000..241210c74
--- /dev/null
+++ b/.github/scripts/test_check_rerun.py
@@ -0,0 +1,104 @@
+#!/usr/bin/env python3
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Behavioral tests for push retry freshness, without credentials or network."""
+import importlib.util
+from pathlib import Path
+import unittest
+import os
+import subprocess
+import tempfile
+from unittest.mock import patch
+
+spec = importlib.util.spec_from_file_location("check_rerun", Path(__file__).with_name("check-rerun.py"))
+checker = importlib.util.module_from_spec(spec)
+spec.loader.exec_module(checker)
+
+
+class FreshnessTest(unittest.TestCase):
+ def setUp(self):
+ self.repository = "hugegraph/hugegraph-toolchain"
+ self.run = {"status": "completed", "conclusion": "failure", "run_attempt": 1,
+ "event": "push", "head_sha": "old", "head_branch": "release/test"}
+ self.head = "old"
+ self.calls = []
+
+ def fetch(self, path):
+ self.calls.append(path)
+ if "/actions/runs/" in path:
+ return self.run
+ if "/commits?" in path:
+ return [{"sha": self.head}]
+ raise AssertionError("unnecessary external lookup: " + path)
+
+ def decide(self):
+ return checker.decide(self.repository, 42, 1, 2, self.fetch)[0]
+
+ def test_push_checks_repository_and_encodes_branch(self):
+ self.assertEqual("rerun", self.decide())
+ self.assertIn(f"repos/{self.repository}/commits?sha=release%2Ftest&per_page=1", self.calls)
+ self.head = "new"
+ self.assertEqual("skip", self.decide())
+
+ def test_pr_skips_without_pr_or_artifact_queries(self):
+ self.run["event"] = "pull_request"
+ self.assertEqual("skip", self.decide())
+ self.assertEqual([f"repos/{self.repository}/actions/runs/42"], self.calls)
+
+ def test_api_failure_fails_closed(self):
+ with tempfile.TemporaryDirectory() as directory:
+ output = Path(directory) / "output"
+ with patch.dict(os.environ, {"GITHUB_OUTPUT": str(output)}), patch(
+ "sys.argv", ["check-rerun.py", "--repository", self.repository,
+ "--run-id", "42", "--run-attempt", "1", "--max-reruns", "2"]), patch.object(
+ checker, "decide", side_effect=subprocess.CalledProcessError(1, "gh")):
+ checker.main()
+ self.assertIn("action=skip\n", output.read_text())
+
+ def test_changed_attempt(self):
+ self.run["run_attempt"] = 2
+ self.assertEqual("skip", self.decide())
+
+ def test_current_run_must_remain_failed_and_completed(self):
+ for key, value in (("status", "in_progress"), ("conclusion", "success"),
+ ("conclusion", "cancelled")):
+ with self.subTest(key=key, value=value):
+ old = self.run[key]
+ self.run[key] = value
+ self.assertEqual("skip", self.decide())
+ self.run[key] = old
+
+ def test_retry_limit(self):
+ self.run["run_attempt"] = 2
+ self.assertEqual("rerun", checker.decide(self.repository, 42, 2, 2, self.fetch)[0])
+ self.run["run_attempt"] = 3
+ self.assertEqual("skip", checker.decide(self.repository, 42, 3, 2, self.fetch)[0])
+
+ def test_unsupported_event(self):
+ self.run["event"] = "workflow_dispatch"
+ self.assertEqual("skip", self.decide())
+
+ def test_missing_metadata(self):
+ for key in ["head_sha", "head_branch"]:
+ with self.subTest(key=key):
+ old = self.run[key]
+ self.run[key] = None
+ self.assertEqual("skip", self.decide())
+ self.run[key] = old
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/.github/scripts/test_ci_policy.py b/.github/scripts/test_ci_policy.py
new file mode 100644
index 000000000..902430492
--- /dev/null
+++ b/.github/scripts/test_ci_policy.py
@@ -0,0 +1,382 @@
+#!/usr/bin/env python3
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Executable Git fixtures for affected-module selection and current-run gating."""
+import importlib.util
+import json
+import os
+from pathlib import Path
+import subprocess
+import tempfile
+import unittest
+from unittest.mock import patch
+
+spec = importlib.util.spec_from_file_location("policy", Path(__file__).with_name("ci-policy.py"))
+policy = importlib.util.module_from_spec(spec)
+spec.loader.exec_module(policy)
+
+
+class PolicyTest(unittest.TestCase):
+ def live_pr(self):
+ return {"state": "open", "head": {"sha": "new", "ref": "feature", "repo": {"full_name": "alice/t"}},
+ "base": {"sha": "base", "repo": {"full_name": "apache/t"}}}
+
+ def setUp(self):
+ # All tests stay local; successful PR gates query only this current-PR fixture.
+ api_mock = patch.object(policy, "api", return_value=self.live_pr())
+ self.addCleanup(api_mock.stop)
+ self.api_mock = api_mock.start()
+
+ def test_dependency_expansion(self):
+ self.assertEqual({"client", "loader", "tools", "spark", "hubble"},
+ policy.select("toolchain", ["hugegraph-client/src/A.java"]))
+ self.assertEqual({"loader", "hubble"}, policy.select("toolchain", ["hugegraph-loader/src/A.java"]))
+ self.assertEqual({"server", "pd", "store", "hstore", "cluster"}, policy.select("server", ["hugegraph-server/A.java"]))
+
+ def test_image_selection_is_separate_from_module_dependencies(self):
+ for path in ["hugegraph-client/src/A.java", "hugegraph-loader/src/A.java",
+ "hugegraph-hubble/hubble-be/src/main/java/App.java",
+ "hugegraph-hubble/hubble-fe/src/App.js", "docs/ci.md"]:
+ self.assertEqual(set(), policy.select_images([path]), path)
+ for path, images in [
+ ("hugegraph-loader/Dockerfile", {"loader_image"}),
+ ("hugegraph-hubble/Dockerfile", {"hubble_image"}),
+ (".dockerignore", policy.IMAGES),
+ (".github/workflows/image-ci.yml", policy.IMAGES)]:
+ self.assertEqual(images, policy.select_images([path]), path)
+ self.assertEqual(set(), policy.select("toolchain", [path]), path)
+
+ def test_image_packaging_and_shared_build_inputs(self):
+ for path in ["pom.xml", ".mvn/maven.config", "hugegraph-client/pom.xml",
+ "hugegraph-dist/release-docs/licenses/dependency.txt",
+ "hugegraph-loader/pom.xml", "hugegraph-loader/assembly/descriptor/assembly.xml",
+ "hugegraph-loader/assembly/static/bin/hugegraph-loader.sh",
+ "hugegraph-loader/README_CN.md", ".github/scripts/new-input.py"]:
+ self.assertEqual(policy.IMAGES, policy.select_images([path]), path)
+ for path in ["hugegraph-hubble/pom.xml", "hugegraph-hubble/hubble-be/pom.xml",
+ "hugegraph-hubble/hubble-dist/assembly/static/conf/hugegraph-hubble.properties",
+ "hugegraph-hubble/hubble-dist/assembly/descriptor/assembly.xml",
+ "hugegraph-hubble/hubble-dist/assembly/travis/check-hubble-dist.sh",
+ "hugegraph-hubble/hubble-fe/yarn.lock", "hugegraph-hubble/README.md",
+ "hugegraph-hubble/hubble-be/src/main/resources/application.properties"]:
+ self.assertEqual({"hubble_image"}, policy.select_images([path]), path)
+
+ def test_server_backend_and_startup_dependents(self):
+ selected = policy.select("server", ["hugegraph-server/hugegraph-hstore/src/test/HstoreTableTest.java"])
+ self.assertIn("hstore", selected)
+ self.assertIn("pd_store", policy.suites("server", selected))
+ selected = policy.select("server", ["hugegraph-server/hugegraph-dist/src/assembly/travis/test-start-hugegraph-pd.sh"])
+ self.assertIn("pd_store", policy.suites("server", selected))
+ self.assertIn("docker", selected)
+ for module in ["hugegraph-core", "hugegraph-api", "hugegraph-test"]:
+ selected = policy.select("server", [f"hugegraph-server/{module}/src/main/A.java"])
+ self.assertIn("hstore", selected)
+ self.assertIn("server", selected)
+ self.assertEqual(set(), policy.select("server", ["hugegraph-server/hugegraph-hstore/README.md"]))
+ self.assertIn("hstore", policy.select("server", ["hugegraph-server/hugegraph-rocksdb/src/main/A.java"]))
+
+ def test_audited_consumer_edges(self):
+ expected = {"server", "pd", "store", "hstore", "cluster"}
+ for path in ["hugegraph-server/hugegraph-hstore/src/A.java", "hugegraph-server/hugegraph-rocksdb/A.java"]:
+ self.assertTrue(expected.issubset(policy.select("server", [path])))
+ self.assertNotIn("helm", policy.select("server", [path]))
+ self.assertEqual(set(policy.MODULES["server"]), policy.select("server", [".github/workflows/server-ci.yml"]))
+ self.assertTrue({"commons", "docker"}.issubset(policy.select("server", [
+ "hugegraph-commons/hugegraph-common/src/main/resources/version.properties"])))
+ self.assertIn("go", policy.select("toolchain", ["hugegraph-client/assembly/travis/start-hugegraph-servers.sh"]))
+ self.assertNotIn("go", policy.select("toolchain", ["hugegraph-client/src/A.java"]))
+ self.assertNotIn("server", policy.select("server", ["hugegraph-pd/src/A.java"]))
+ self.assertTrue({"store", "docker"}.issubset(policy.select("server", [
+ "hugegraph-store/hg-store-dist/src/assembly/static/bin/util.sh"])))
+
+ def test_version_resource_follows_real_docker_consumer(self):
+ root = Path(__file__).resolve().parents[2]
+ consumer = root / ".github/workflows/docker-build-ci.yml"
+ if not consumer.exists():
+ self.skipTest("Server Docker consumer is not in the Toolchain repository")
+ resource = next(line.strip().rstrip(")") for line in consumer.read_text().splitlines()
+ if line.strip().startswith("hugegraph-commons/") and "version.properties" in line)
+ self.assertTrue((root / resource).is_file())
+ self.assertTrue({"commons", "docker"}.issubset(policy.select("server", [resource])))
+
+ def test_single_workflow_has_no_global_fanout(self):
+ self.assertEqual({"hubble"}, policy.select("toolchain", [".github/workflows/hubble-ci.yml"]))
+ self.assertEqual({"docker"}, policy.select("server", [".github/workflows/docker-build-ci.yml"]))
+
+ def test_unknown_and_proto_fail_conservative(self):
+ for path in ["pom.xml", ".github/scripts/new.py", "hugegraph-pd/api.proto", "mystery"]:
+ self.assertEqual(set(policy.MODULES["server"]), policy.select("server", [path]))
+
+ def test_strict_docs_allowlist(self):
+ self.assertEqual(set(), policy.select("server", ["README.md", "docs/guide.md"]))
+ for path in ["hugegraph-client/src/test/resources/README.md", "docs/type.ts", "docs/config.yml", "docs/example.java", "hugegraph-server/type.ts"]:
+ self.assertTrue(policy.select("server", [path]))
+
+
+ def plan(self):
+ return {"schema": 1, "project": "toolchain", "repository": "apache/t", "pr": 7,
+ "source": "alice/t", "branch": "feature", "base": "base", "head": "new",
+ "testedMergeSHA": "merge", "expected": ["client"], "selected": ["client"],
+ "externalInputs": {"toolchainJDK": ["11", "17"]}}
+
+ def pr_plan(self, paths):
+ live = self.live_pr()
+ event = {"pull_request": dict(live, number=7)}
+ def git(*args):
+ if args[0] == "show":
+ return "base new"
+ if args[0] == "diff":
+ return "\n".join(paths)
+ return "merge"
+ def fetch(path):
+ self.assertEqual("repos/apache/t/pulls/7", path)
+ return live
+ with patch.object(policy, "git", side_effect=git), patch.object(
+ policy, "unsafe_documentation", return_value=False), patch.object(
+ policy, "packaged_readme_state", return_value="valid"):
+ return policy.create_plan("toolchain", event, "apache/t", fetch)
+
+ def test_cumulative_pr_docs_update_runs_affected_consumers_again(self):
+ for document in ["README.md", "hugegraph-hubble/README.md"]:
+ with self.subTest(document=document):
+ plan = self.pr_plan(["hugegraph-client/src/A.java", document])
+ self.assertEqual({"client", "loader", "tools", "spark", "hubble"}, set(plan["selected"]))
+ self.assertTrue(plan["needsFixture"])
+ self.assertTrue(all(plan[module] for module in plan["selected"]))
+ self.assertEqual(document.startswith("hugegraph-hubble/"), plan["hubble_image"])
+
+ def test_pure_documentation_pr_has_no_modules_or_fixtures(self):
+ plan = self.pr_plan(["README.md", "docs/ci.md"])
+ self.assertEqual([], plan["expected"])
+ self.assertFalse(plan["needsFixture"])
+ policy.gate(plan, {"plan": {"result": "success"}})
+
+ def test_packaged_readme_selects_image_on_plain_documentation_pr(self):
+ plan = self.pr_plan(["hugegraph-hubble/README.md"])
+ self.assertEqual(["hubble_image"], plan["expected"])
+ self.assertFalse(plan["needsFixture"])
+
+ def test_image_only_plan_and_gate_require_no_server_fixture(self):
+ plan = self.pr_plan(["hugegraph-loader/Dockerfile"])
+ self.assertEqual(["loader_image"], plan["expected"])
+ self.assertFalse(plan["needsFixture"])
+ results = {"plan": {"result": "success"}, "loader_image": {"result": "success"}}
+ self.assertEqual(["loader_image"], policy.gate(plan, results)["executed"])
+ for state in [None, "skipped", "cancelled", "failure"]:
+ results["loader_image"]["result"] = state
+ with self.subTest(state=state), self.assertRaises(ValueError):
+ policy.gate(plan, results)
+
+ def test_gate_rejects_missing_failed_cancelled_skipped(self):
+ for result in [None, "failure", "cancelled", "skipped"]:
+ with self.subTest(result=result), self.assertRaises(ValueError):
+ policy.gate(self.plan(), {"plan": {"result": "success"}, "client": {"result": result},
+ "fixture": {"result": "success"}})
+
+ def test_gate_records_current_attempt_without_external_proof(self):
+ results = {"plan": {"result": "success"}, "client": {"result": "success"},
+ "fixture": {"result": "success"}}
+ with patch.dict(os.environ, {"GITHUB_RUN_ID": "42", "GITHUB_RUN_ATTEMPT": "2"}), patch.object(
+ policy, "api", return_value=self.live_pr()) as fetch:
+ report = policy.gate(self.plan(), results)
+ fetch.assert_called_once_with("repos/apache/t/pulls/7")
+ self.assertEqual(["client"], report["executed"])
+ self.assertEqual(results, report["results"])
+ self.assertEqual(42, report["runID"])
+ self.assertEqual(2, report["runAttempt"])
+
+ def test_old_receipt_cannot_satisfy_current_selected_failure(self):
+ plan = dict(self.plan(), reused={"client": {"runID": 41, "jobIDs": [123]}})
+ for state in ["failure", "skipped"]:
+ with self.subTest(state=state), self.assertRaises(ValueError):
+ policy.gate(plan, {"plan": {"result": "success"}, "client": {"result": state},
+ "fixture": {"result": "success"}})
+
+ def test_incomplete_pr_event_never_becomes_a_push_plan(self):
+ event = {"pull_request": dict(self.live_pr(), number=7)}
+ cases = []
+ for change in ("null_repo", "missing_repo", "missing_number", "empty_ref"):
+ altered = json.loads(json.dumps(event))
+ pr = altered["pull_request"]
+ if change == "null_repo":
+ pr["head"]["repo"] = None
+ elif change == "missing_repo":
+ del pr["head"]["repo"]
+ elif change == "missing_number":
+ del pr["number"]
+ else:
+ pr["head"]["ref"] = ""
+ cases.append(altered)
+ cases.extend([{"pull_request": {}}, {"pull_request": None}])
+ with patch.object(policy, "git", return_value="merge"):
+ for altered in cases:
+ with self.subTest(event=altered), self.assertRaises(policy.StaleInputError):
+ policy.create_plan('toolchain', altered, 'apache/t',
+ lambda _: self.fail("Incomplete identity must fail before any API query"))
+
+ def test_gate_refuses_failed_planner_even_empty_selection(self):
+ for result in ["failure", "skipped", "cancelled", None]:
+ plan = dict(self.plan(), expected=[])
+ with self.subTest(result=result), self.assertRaises(ValueError):
+ policy.gate(plan, {"plan": {"result": result}})
+
+ def test_fixture_missing_rejects_successful_test(self):
+ with self.assertRaises(ValueError):
+ policy.gate(self.plan(), {"plan": {"result": "success"}, "client": {"result": "success"}})
+
+ def test_hubble_gate_requires_current_and_released_fixtures(self):
+ plan = dict(self.plan(), expected=["hubble"])
+ for producer in ("fixture", "hubble-fixture"):
+ for state in (None, "failure", "cancelled", "skipped"):
+ results = {"plan": {"result": "success"}, "hubble": {"result": "success"},
+ "fixture": {"result": "success"}, "hubble-fixture": {"result": "success"}}
+ results[producer] = {"result": state}
+ with self.subTest(producer=producer, state=state), self.assertRaises(ValueError):
+ policy.gate(plan, results)
+
+ def test_real_git_doc_modes_and_packaged_readme_contract(self):
+ with tempfile.TemporaryDirectory() as directory:
+ root = Path(directory)
+ def git(*args):
+ return subprocess.check_output(["git", *args], cwd=root, text=True).strip()
+ git("init", "-q")
+ git("config", "user.email", "ci@example.invalid")
+ git("config", "user.name", "CI")
+ (root / "README.md").write_text("initial")
+ (root / "hugegraph-hubble").mkdir()
+ readme = root / "hugegraph-hubble/README.md"
+ readme.write_text("nonempty docs")
+ git("add", ".")
+ git("commit", "-qm", "base")
+ base = git("rev-parse", "HEAD")
+ old = os.getcwd()
+ try:
+ os.chdir(root)
+ self.assertEqual("valid", policy.packaged_readme_state("HEAD"))
+ for content in ["changed docs", "", " \t\n"]:
+ readme.write_text(content)
+ git("commit", "-qam", "docs")
+ self.assertEqual(content.strip() != "", policy.packaged_readme_state("HEAD") == "valid")
+ readme.write_text("nonempty")
+ readme.chmod(0o755)
+ git("add", ".")
+ git("commit", "-qm", "executable docs")
+ self.assertTrue(policy.unsafe_documentation("HEAD", ["hugegraph-hubble/README.md"]))
+ self.assertNotEqual("valid", policy.packaged_readme_state("HEAD"))
+ readme.unlink()
+ readme.symlink_to("missing-target")
+ git("add", ".")
+ git("commit", "-qm", "symlink docs")
+ self.assertTrue(policy.unsafe_documentation("HEAD", ["hugegraph-hubble/README.md"]))
+ self.assertNotEqual("valid", policy.packaged_readme_state("HEAD"))
+ readme.unlink()
+ git("add", ".")
+ git("commit", "-qm", "deleted docs")
+ self.assertEqual("missing", policy.packaged_readme_state("HEAD"))
+ finally:
+ os.chdir(old)
+ event = root / "event.json"
+ event.write_text(json.dumps({"before": base}))
+ output = root / "output"
+ subprocess.run(["python3", str(Path(policy.__file__).resolve()), "plan", "--project", "toolchain",
+ "--repository", "apache/t", "--event-path", str(event),
+ "--output", str(root / "plan.json")], cwd=root, check=True,
+ env={**os.environ, "GITHUB_OUTPUT": str(output)})
+ self.assertIn("client=true\n", output.read_text())
+ plan = json.loads((root / "plan.json").read_text())
+ self.assertEqual(set(policy.MODULES["toolchain"]), set(plan["selected"]))
+ self.assertEqual({"hubble_image"}, set(plan["selectedImages"]))
+
+ def test_api_failure_runs_full_required_without_losing_pr_identity(self):
+ event = {"pull_request": dict(self.live_pr(), number=7)}
+ def git(*args):
+ return "base new" if args[0] == "show" else "merge"
+ def fail(path):
+ raise subprocess.CalledProcessError(1, "gh")
+ with patch.object(policy, "git", side_effect=git):
+ plan = policy.create_plan("toolchain", event, "apache/t", fail)
+ self.assertEqual(set(policy.MODULES["toolchain"]), set(plan["selected"]))
+ self.assertEqual((7, "alice/t", "feature", "base", "new"),
+ tuple(plan[key] for key in ["pr", "source", "branch", "base", "head"]))
+ results = {suite: {"result": "success"} for suite in plan["expected"]}
+ results.update(plan={"result": "success"}, fixture={"result": "success"},
+ **{"hubble-fixture": {"result": "success"}})
+ with self.assertRaises(subprocess.CalledProcessError):
+ policy.gate(plan, results, fail)
+ self.assertEqual(plan["expected"], policy.gate(plan, results)["executed"])
+ self.api_mock.assert_called_once_with("repos/apache/t/pulls/7")
+
+ def test_real_pr_git_snapshot_and_final_gate_freshness(self):
+ with tempfile.TemporaryDirectory() as directory:
+ root = Path(directory)
+ def git(*args):
+ return subprocess.check_output(["git", *args], cwd=root, text=True).strip()
+ git("init", "-q")
+ git("config", "user.email", "ci@example.invalid")
+ git("config", "user.name", "CI")
+ (root / "README.md").write_text("base")
+ git("add", ".")
+ git("commit", "-qm", "base")
+ base = git("rev-parse", "HEAD")
+ git("checkout", "-qb", "feature")
+ (root / "hugegraph-client").mkdir()
+ (root / "hugegraph-client/A.java").write_text("source")
+ git("add", ".")
+ git("commit", "-qm", "source")
+ head = git("rev-parse", "HEAD")
+ tree = git("rev-parse", "HEAD^{tree}")
+ merge = git("commit-tree", tree, "-p", base, "-p", head, "-m", "PR merge")
+ advanced_head = git("commit-tree", tree, "-p", head, "-m", "new source head")
+ advanced_base = git("commit-tree", git("rev-parse", base + "^{tree}"),
+ "-p", base, "-m", "new target base")
+ git("checkout", "--detach", "-q", merge)
+ live = self.live_pr()
+ live["head"]["sha"], live["base"]["sha"] = head, base
+ event = {"pull_request": dict(live, number=7)}
+ old = os.getcwd()
+ try:
+ os.chdir(root)
+ plan = policy.create_plan("toolchain", event, "apache/t", lambda _: live)
+ results = {suite: {"result": "success"} for suite in plan["expected"]}
+ results.update(plan={"result": "success"}, fixture={"result": "success"},
+ **{"hubble-fixture": {"result": "success"}})
+ self.assertEqual(plan["expected"], policy.gate(plan, results, lambda _: live)["executed"])
+ for section, field, value in [("head", "sha", advanced_head), ("base", "sha", advanced_base),
+ ("head", "ref", "other-branch"),
+ ("head", "repo", {"full_name": "other/fork"}),
+ ("base", "repo", {"full_name": "other/target"})]:
+ changed = dict(live, **{section: dict(live[section], **{field: value})})
+ with self.subTest(section=section, field=field):
+ with self.assertRaises(policy.StaleInputError):
+ policy.create_plan("toolchain", event, "apache/t", lambda _: changed)
+ with self.assertRaises(policy.StaleInputError):
+ policy.gate(plan, results, lambda _: changed)
+ with self.assertRaises(policy.StaleInputError):
+ policy.gate(plan, results, lambda _: dict(live, state="closed"))
+ git("checkout", "--detach", "-q", head)
+ with self.assertRaises(policy.StaleInputError):
+ policy.create_plan("toolchain", event, "apache/t", lambda _: live)
+ finally:
+ os.chdir(old)
+
+ def test_push_gate_does_not_query_pr(self):
+ plan = dict(self.plan(), pr=0, expected=[], selected=[])
+ policy.gate(plan, {"plan": {"result": "success"}},
+ lambda _: self.fail("push gate must not query a PR"))
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/.github/scripts/test_toolchain_image.py b/.github/scripts/test_toolchain_image.py
new file mode 100644
index 000000000..3649062e3
--- /dev/null
+++ b/.github/scripts/test_toolchain_image.py
@@ -0,0 +1,157 @@
+#!/usr/bin/env python3
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Prevent a tag resolving to an unrelated image from providing runtime evidence."""
+
+import importlib.util
+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
+from pathlib import Path
+import subprocess
+import tempfile
+import threading
+import unittest
+from unittest.mock import patch
+
+spec = importlib.util.spec_from_file_location("images", Path(__file__).with_name("check-toolchain-image.py"))
+images = importlib.util.module_from_spec(spec)
+spec.loader.exec_module(images)
+
+
+class RuntimeImageTest(unittest.TestCase):
+ def test_rejects_container_using_a_different_image(self):
+ with patch.object(images, "output", return_value="sha256:old"):
+ with self.assertRaisesRegex(RuntimeError, "expected built image sha256:pr"):
+ images.verify_image("hubble", "sha256:pr")
+
+ def test_rejects_empty_build_result(self):
+ with patch.object(images, "output", return_value=""):
+ with self.assertRaises(RuntimeError):
+ images.verify_image("loader", "")
+
+ def test_accepts_built_image(self):
+ with patch.object(images, "output", return_value="sha256:pr"):
+ images.verify_image("hubble", "sha256:pr")
+
+ def test_requires_application_identity_and_version_not_just_http_success(self):
+ for body in [None, [], {}, {"status": 200}, {"status": 200, "data": []},
+ {"status": 200, "data": {"name": "hugegraph-hubble"}},
+ {"status": 401, "data": {"name": "hugegraph-hubble", "version": "3.0"}},
+ {"status": 200, "data": {"name": "other", "version": "3.0"}}]:
+ self.assertFalse(images.valid_hubble_about(body), body)
+ self.assertTrue(images.valid_hubble_about(
+ {"status": 200, "data": {"name": "hugegraph-hubble", "version": "3.0.0"}}))
+
+ def test_hubble_exit_is_not_readiness_success(self):
+ with patch.object(images, "output", side_effect=["127.0.0.1:12345", "false"]):
+ with self.assertRaisesRegex(RuntimeError, "exited before becoming ready"):
+ images.wait_hubble("hubble")
+
+ def test_loader_executes_the_public_script_with_its_permissions_and_shebang(self):
+ with tempfile.TemporaryDirectory() as temporary:
+ script = Path(temporary) / "bin/hugegraph-loader.sh"
+ script.parent.mkdir()
+ script.write_text('#!/bin/sh\nprintf "Usage: loader --help\\n"\n')
+
+ def execute(*args):
+ self.assertEqual(args[:3], ("docker", "exec", "loader"))
+ return subprocess.check_output(args[3:], cwd=temporary, text=True)
+
+ with patch.object(images, "output", side_effect=execute):
+ script.chmod(0o755)
+ images.check_loader("loader")
+ script.chmod(0o644)
+ with self.assertRaises(PermissionError):
+ images.check_loader("loader")
+ script.chmod(0o755)
+ script.write_text('#!/missing/interpreter\nprintf "Usage: loader --help\\n"\n')
+ with self.assertRaises(FileNotFoundError):
+ images.check_loader("loader")
+
+
+class HubbleEntryTest(unittest.TestCase):
+ def setUp(self):
+ self.routes = {
+ "/": ("text/html", ''
+ ''),
+ "/main.js": ("application/javascript", 'document.getElementById("root");'),
+ "/main.css": ("text/css", "body { margin: 0; }"),
+ }
+ self.requested = []
+ routes, requested = self.routes, self.requested
+
+ class Handler(BaseHTTPRequestHandler):
+ def do_GET(self):
+ requested.append(self.path)
+ content_type, body = routes.get(self.path, ("text/html", ""))
+ if content_type == "redirect":
+ self.send_response(302)
+ self.send_header("Location", body)
+ self.end_headers()
+ return
+ self.send_response(200 if self.path in routes else 404)
+ self.send_header("Content-Type", content_type)
+ self.end_headers()
+ self.wfile.write(body.encode())
+
+ def log_message(self, *args):
+ pass
+
+ self.server = ThreadingHTTPServer(("127.0.0.1", 0), Handler)
+ self.thread = threading.Thread(target=self.server.serve_forever, daemon=True)
+ self.thread.start()
+ self.origin = f"http://127.0.0.1:{self.server.server_port}"
+
+ def tearDown(self):
+ self.server.shutdown()
+ self.server.server_close()
+ self.thread.join()
+
+ def test_checks_root_javascript_and_stylesheets(self):
+ images.check_hubble_ui(self.origin)
+ self.assertEqual(self.requested, ["/", "/main.js", "/main.css"])
+
+ def test_rejects_a_backend_only_page_and_an_unbuilt_template(self):
+ for page in ['backend ready', '']:
+ self.routes["/"] = ("text/html", page)
+ with self.subTest(page=page), self.assertRaises(RuntimeError):
+ images.check_hubble_ui(self.origin)
+
+ def test_rejects_missing_assets_including_spa_html_fallback_and_empty_success(self):
+ for path in ["/main.js", "/main.css"]:
+ original = self.routes[path]
+ for content_type, body in [("text/html", self.routes["/"][1]),
+ (original[0], " \n"),
+ (original[0], "fallback")]:
+ self.routes[path] = (content_type, body)
+ with self.subTest(path=path, content_type=content_type, body=body), self.assertRaises(RuntimeError):
+ images.check_hubble_ui(self.origin)
+ del self.routes[path]
+ with self.assertRaises(images.urllib.error.HTTPError):
+ images.check_hubble_ui(self.origin)
+ self.routes[path] = original
+
+ def test_does_not_fetch_external_scripts(self):
+ self.routes["/"] = ("text/html", ''
+ '')
+ with self.assertRaisesRegex(RuntimeError, "missing a local JavaScript bundle"):
+ images.check_hubble_ui(self.origin)
+ self.assertEqual(self.requested, ["/"])
+
+ def test_does_not_follow_asset_redirects_outside_the_container(self):
+ self.routes["/main.js"] = ("redirect", "https://external.invalid/app.js")
+ with self.assertRaises(images.urllib.error.HTTPError):
+ images.check_hubble_ui(self.origin)
+ self.assertEqual(self.requested, ["/", "/main.js"])
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
new file mode 100644
index 000000000..3ceeafb60
--- /dev/null
+++ b/.github/workflows/ci.yml
@@ -0,0 +1,319 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+name: Required CI
+
+on:
+ pull_request:
+ push:
+ branches: [master, 'release-*']
+ workflow_dispatch:
+
+permissions:
+ contents: read
+ actions: read
+ pull-requests: read
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.run_id }}-${{ github.run_attempt == 1 && 'head' || format('rerun-{0}', github.run_id) }}
+ cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+
+jobs:
+ plan:
+ runs-on: ubuntu-latest
+ outputs:
+ server-repository: ${{ steps.baseline.outputs.repository }}
+ server-commit: ${{ steps.baseline.outputs.commit_id }}
+ server-ref: ${{ steps.baseline.outputs.fetch_ref }}
+ hubble-commit: ${{ steps.hubble-baseline.outputs.commit }}
+ client: ${{ steps.select.outputs.client }}
+ loader: ${{ steps.select.outputs.loader }}
+ tools: ${{ steps.select.outputs.tools }}
+ spark: ${{ steps.select.outputs.spark }}
+ hubble: ${{ steps.select.outputs.hubble }}
+ go: ${{ steps.select.outputs.go }}
+ needsFixture: ${{ steps.select.outputs.needsFixture }}
+ loader-image: ${{ steps.select.outputs.loader_image }}
+ hubble-image: ${{ steps.select.outputs.hubble_image }}
+ security: ${{ steps.select.outputs.security }}
+ steps:
+ - uses: actions/checkout@v7
+ with:
+ fetch-depth: 0
+ persist-credentials: false
+ - name: Test CI policy and retry guards
+ run: |
+ python3 -m unittest discover -s .github/scripts -p 'test_*.py'
+ python3 .github/actions/setup-hugegraph-server/runtime_test.py
+ node --test .github/actions/get-hugegraph-commit/resolve.test.cjs hugegraph-client/assembly/travis/checkout-server.test.cjs hugegraph-client/assembly/travis/install-server.test.cjs
+ - name: Determine external test input scope
+ id: scope
+ env:
+ GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+ run: >-
+ python3 .github/scripts/ci-policy.py plan --project toolchain
+ --event-path "$GITHUB_EVENT_PATH" --repository "$GITHUB_REPOSITORY"
+ --output scope.json
+ - name: Resolve historical Server 1.7 baseline
+ id: baseline
+ if: steps.scope.outputs.needsFixture == 'true'
+ uses: ./.github/actions/get-hugegraph-commit
+ with:
+ repository: apache/hugegraph
+ ref: refs/tags/1.7.0
+ expected-commit: b12425c2032bf0d21a97b8221f42a18055c2982f
+ - name: Resolve Hubble's existing Server master baseline
+ id: hubble-baseline
+ if: steps.scope.outputs.hubble == 'true'
+ env:
+ SERVER_REPOSITORY: https://github.com/apache/hugegraph.git
+ run: |
+ commit=$(git ls-remote "$SERVER_REPOSITORY" refs/heads/master | cut -f1)
+ [[ "$commit" =~ ^[0-9a-f]{40}$ ]]
+ echo "commit=$commit" >> "$GITHUB_OUTPUT"
+ - name: Record external test inputs
+ env:
+ NEEDS_SERVER: ${{ steps.scope.outputs.needsFixture }}
+ NEEDS_HUBBLE: ${{ steps.scope.outputs.hubble }}
+ SERVER_REPOSITORY: ${{ steps.baseline.outputs.repository }}
+ SERVER_COMMIT: ${{ steps.baseline.outputs.commit_id }}
+ SERVER_REF: ${{ steps.baseline.outputs.fetch_ref }}
+ HUBBLE_COMMIT: ${{ steps.hubble-baseline.outputs.commit }}
+ run: |
+ python3 - <<'PYINPUTS'
+ import json, os
+ inputs = {}
+ if os.environ['NEEDS_SERVER'] == 'true':
+ with open('scope.json') as stream:
+ selected = json.load(stream)['selected']
+ inputs['toolchainJDK'] = {module: ['17'] if module == 'hubble' else ['11', '17']
+ for module in selected if module != 'go'}
+ inputs['config'] = 'release-current-ci-v2'
+ inputs['server'] = {'repository': os.environ['SERVER_REPOSITORY'],
+ 'commit': os.environ['SERVER_COMMIT'],
+ 'ref': os.environ['SERVER_REF'], 'jdk': '11'}
+ if os.environ['NEEDS_HUBBLE'] == 'true':
+ inputs['hubble'] = {'repository': 'apache/hugegraph', 'commit': os.environ['HUBBLE_COMMIT'],
+ 'ref': os.environ['HUBBLE_COMMIT'], 'branch': 'master', 'jdk': '17'}
+ with open('inputs.json', 'w') as stream:
+ json.dump(inputs, stream, sort_keys=True)
+ PYINPUTS
+ - name: Select affected modules and images
+ id: select
+ env:
+ GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+ run: >-
+ python3 .github/scripts/ci-policy.py plan --project toolchain
+ --event-path "$GITHUB_EVENT_PATH" --repository "$GITHUB_REPOSITORY"
+ --inputs-json inputs.json --output plan.json
+ - uses: actions/upload-artifact@v7
+ with:
+ name: ci-plan
+ path: plan.json
+ overwrite: true
+ retention-days: 7
+
+ fixture:
+ needs: plan
+ if: needs.plan.outputs.needsFixture == 'true'
+ runs-on: ubuntu-latest
+ outputs:
+ artifact: ${{ steps.identity.outputs.artifact }}
+ steps:
+ - uses: actions/checkout@v7
+ with:
+ persist-credentials: false
+ - name: Name immutable Server artifact
+ id: identity
+ run: echo "artifact=fixture-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" >> "$GITHUB_OUTPUT"
+ - uses: ./.github/actions/setup-java-env
+ with:
+ java-version: '11'
+ distribution: temurin
+ use-stage: 'true'
+ - name: Build reusable Server package
+ uses: ./.github/actions/setup-hugegraph-server
+ with:
+ mode: build
+ artifact-name: ${{ steps.identity.outputs.artifact }}
+ repository: ${{ needs.plan.outputs.server-repository }}
+ commit-id: ${{ needs.plan.outputs.server-commit }}
+ fetch-ref: ${{ needs.plan.outputs.server-ref }}
+ java-version: '11'
+
+ hubble-fixture:
+ needs: plan
+ if: needs.plan.outputs.hubble == 'true'
+ runs-on: ubuntu-latest
+ outputs:
+ artifact: ${{ steps.identity.outputs.artifact }}
+ steps:
+ - uses: actions/checkout@v7
+ with:
+ persist-credentials: false
+ - name: Name immutable Server artifact
+ id: identity
+ run: echo "artifact=hubble-fixture-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" >> "$GITHUB_OUTPUT"
+ - uses: ./.github/actions/setup-java-env
+ with:
+ java-version: '17'
+ distribution: temurin
+ use-stage: 'true'
+ - name: Build reusable Server package
+ uses: ./.github/actions/setup-hugegraph-server
+ with:
+ mode: build
+ artifact-name: ${{ steps.identity.outputs.artifact }}
+ commit-id: ${{ needs.plan.outputs.hubble-commit }}
+ fetch-ref: ${{ needs.plan.outputs.hubble-commit }}
+ java-version: '17'
+
+ client:
+ needs: [plan, fixture]
+ if: ${{ needs.plan.outputs.client == 'true' }}
+ uses: ./.github/workflows/client-ci.yml
+ with:
+ artifact-name: ${{ needs.fixture.outputs.artifact }}
+ server-repository: ${{ needs.plan.outputs.server-repository }}
+ server-commit: ${{ needs.plan.outputs.server-commit }}
+ server-ref: ${{ needs.plan.outputs.server-ref }}
+ secrets: inherit
+
+ loader:
+ needs: [plan, fixture]
+ if: ${{ needs.plan.outputs.loader == 'true' }}
+ uses: ./.github/workflows/loader-ci.yml
+ with:
+ artifact-name: ${{ needs.fixture.outputs.artifact }}
+ server-repository: ${{ needs.plan.outputs.server-repository }}
+ server-commit: ${{ needs.plan.outputs.server-commit }}
+ server-ref: ${{ needs.plan.outputs.server-ref }}
+ secrets: inherit
+
+ tools:
+ needs: [plan, fixture]
+ if: ${{ needs.plan.outputs.tools == 'true' }}
+ uses: ./.github/workflows/tools-ci.yml
+ with:
+ artifact-name: ${{ needs.fixture.outputs.artifact }}
+ server-repository: ${{ needs.plan.outputs.server-repository }}
+ server-commit: ${{ needs.plan.outputs.server-commit }}
+ server-ref: ${{ needs.plan.outputs.server-ref }}
+ secrets: inherit
+
+ spark:
+ needs: [plan, fixture]
+ if: ${{ needs.plan.outputs.spark == 'true' }}
+ uses: ./.github/workflows/spark-connector-ci.yml
+ with:
+ artifact-name: ${{ needs.fixture.outputs.artifact }}
+ server-repository: ${{ needs.plan.outputs.server-repository }}
+ server-commit: ${{ needs.plan.outputs.server-commit }}
+ server-ref: ${{ needs.plan.outputs.server-ref }}
+ secrets: inherit
+
+ hubble:
+ needs: [plan, fixture, hubble-fixture]
+ if: ${{ needs.plan.outputs.hubble == 'true' }}
+ uses: ./.github/workflows/hubble-ci.yml
+ with:
+ current-artifact-name: ${{ needs.hubble-fixture.outputs.artifact }}
+ current-commit: ${{ needs.plan.outputs.hubble-commit }}
+ artifact-name: ${{ needs.fixture.outputs.artifact }}
+ server-repository: ${{ needs.plan.outputs.server-repository }}
+ server-commit: ${{ needs.plan.outputs.server-commit }}
+ server-ref: ${{ needs.plan.outputs.server-ref }}
+ secrets: inherit
+
+ go:
+ needs: [plan, fixture]
+ if: ${{ needs.plan.outputs.go == 'true' }}
+ uses: ./.github/workflows/client-go-ci.yml
+ with:
+ artifact-name: ${{ needs.fixture.outputs.artifact }}
+ server-repository: ${{ needs.plan.outputs.server-repository }}
+ server-commit: ${{ needs.plan.outputs.server-commit }}
+ server-ref: ${{ needs.plan.outputs.server-ref }}
+ secrets: inherit
+
+ loader_image:
+ needs: plan
+ if: needs.plan.outputs.loader-image == 'true'
+ uses: ./.github/workflows/image-ci.yml
+ with:
+ module: loader
+
+ hubble_image:
+ needs: plan
+ if: needs.plan.outputs.hubble-image == 'true'
+ uses: ./.github/workflows/image-ci.yml
+ with:
+ module: hubble
+
+ affected-module-tests:
+ name: affected-module-tests
+ needs: [plan, fixture, hubble-fixture, client, loader, tools, spark, hubble, go, loader_image, hubble_image]
+ if: ${{ !cancelled() }}
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v7
+ with:
+ persist-credentials: false
+ - uses: actions/download-artifact@v7
+ with:
+ name: ci-plan
+ - name: Require every selected module and image to succeed
+ env:
+ GH_TOKEN: ${{ github.token }}
+ RESULTS: ${{ toJSON(needs) }}
+ run: |
+ printf '%s' "$RESULTS" > results.json
+ python3 .github/scripts/ci-policy.py gate --plan plan.json --results results.json --output test-results.json
+ - name: Publish current-run test results
+ uses: actions/upload-artifact@v7
+ with:
+ name: ci-test-results
+ path: test-results.json
+ overwrite: true
+ retention-days: 7
+
+ security:
+ needs: [plan, affected-module-tests]
+ permissions:
+ contents: read
+ actions: read
+ security-events: write
+ if: >-
+ !cancelled() && needs.plan.result == 'success' &&
+ needs.affected-module-tests.result == 'success' &&
+ (needs.plan.outputs.security == 'true' ||
+ (github.event_name == 'pull_request' && vars.CI_OPTIMIZED_REQUIRED != 'true'))
+ uses: ./.github/workflows/codeql-analysis.yml
+ secrets: inherit
+
+ legacy-java-analysis:
+ name: Analyze (java)
+ needs: security
+ if: ${{ !cancelled() && github.event_name == 'pull_request' && vars.CI_OPTIMIZED_REQUIRED != 'true' }}
+ runs-on: ubuntu-latest
+ steps:
+ - name: Require actual Java security analysis during migration
+ env:
+ SECURITY_RESULT: ${{ needs.security.result }}
+ run: test "$SECURITY_RESULT" = success
diff --git a/.github/workflows/client-ci.yml b/.github/workflows/client-ci.yml
index f8f8b930c..fc68cd4f6 100644
--- a/.github/workflows/client-ci.yml
+++ b/.github/workflows/client-ci.yml
@@ -1,27 +1,20 @@
name: "java-client-ci"
on:
- workflow_dispatch:
- push:
- branches:
- - master
- - 'release-*'
- paths:
- - hugegraph-client/**
- - hugegraph-dist/**
- - .github/workflows/**
- - pom.xml
- pull_request:
- paths:
- - hugegraph-client/**
- - hugegraph-dist/**
- - .github/workflows/**
- - pom.xml
-
-# Keep the newest PR run; preserve independent push, release and manual runs.
-concurrency:
- group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.run_id }}
- cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+ workflow_call:
+ inputs:
+ server-repository:
+ type: string
+ default: apache/hugegraph
+ artifact-name:
+ type: string
+ required: true
+ server-commit:
+ type: string
+ required: true
+ server-ref:
+ type: string
+ required: true
jobs:
client-ci:
@@ -33,7 +26,7 @@ jobs:
fail-fast: false
matrix:
# released pd package is compiled in version 55.0, which requires java 11 to use
- JAVA_VERSION: [ '11' ]
+ JAVA_VERSION: ['11', '17']
steps:
- name: Fetch code
@@ -41,9 +34,13 @@ jobs:
with:
fetch-depth: 2
- - name: Get HugeGraph stable commit id
- id: get-commit
- uses: ./.github/actions/get-hugegraph-commit
+ - name: Install released Server JVM
+ id: server-java
+ uses: actions/setup-java@v5
+ with:
+ java-version: '11'
+ distribution: temurin
+ overwrite-settings: false
- name: Setup Java environment
uses: ./.github/actions/setup-java-env
@@ -56,9 +53,13 @@ jobs:
uses: ./.github/actions/setup-hugegraph-server
with:
travis-dir: ${{ env.TRAVIS_DIR }}
- commit-id: ${{ steps.get-commit.outputs.commit_id }}
- repository: ${{ steps.get-commit.outputs.repository }}
- fetch-ref: ${{ steps.get-commit.outputs.fetch_ref }}
+ commit-id: ${{ inputs.server-commit }}
+ repository: ${{ inputs.server-repository }}
+ fetch-ref: ${{ inputs.server-ref }}
+ artifact-name: ${{ inputs.artifact-name }}
+ mode: start
+ java-version: '11'
+ java-home: ${{ steps.server-java.outputs.path }}
- name: Compile
run: |
@@ -66,7 +67,7 @@ jobs:
- name: Run test
run: |
- cd hugegraph-client && ls *
+ cd hugegraph-client && ls -- *
mvn test -Dtest=UnitTestSuite -ntp
mvn test -Dtest=ApiTestSuite
mvn test -Dtest=FuncTestSuite
diff --git a/.github/workflows/client-go-ci.yml b/.github/workflows/client-go-ci.yml
index 87c6663f5..1ec11903c 100644
--- a/.github/workflows/client-go-ci.yml
+++ b/.github/workflows/client-go-ci.yml
@@ -1,26 +1,20 @@
name: "go-client-ci"
on:
- push:
- branches:
- - master
- - 'release-*'
- paths:
- - hugegraph-client-go/**
- - hugegraph-dist/**
- - .github/workflows/**
- - pom.xml
- pull_request:
- paths:
- - hugegraph-client-go/**
- - hugegraph-dist/**
- - .github/workflows/**
- - pom.xml
-
-# Keep the newest PR run; preserve independent push, release and manual runs.
-concurrency:
- group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.run_id }}
- cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+ workflow_call:
+ inputs:
+ server-repository:
+ type: string
+ default: apache/hugegraph
+ artifact-name:
+ type: string
+ required: true
+ server-commit:
+ type: string
+ required: true
+ server-ref:
+ type: string
+ required: true
jobs:
client-go-ci:
@@ -31,7 +25,7 @@ jobs:
strategy:
fail-fast: false
matrix:
- JAVA_VERSION: ['11']
+ JAVA_VERSION: ['17']
steps:
- name: Fetch Code
@@ -39,9 +33,13 @@ jobs:
with:
fetch-depth: 2
- - name: Get HugeGraph stable commit id
- id: get-commit
- uses: ./.github/actions/get-hugegraph-commit
+ - name: Install released Server JVM
+ id: server-java
+ uses: actions/setup-java@v5
+ with:
+ java-version: '11'
+ distribution: temurin
+ overwrite-settings: false
- name: Setup Java environment
uses: ./.github/actions/setup-java-env
@@ -54,9 +52,13 @@ jobs:
uses: ./.github/actions/setup-hugegraph-server
with:
travis-dir: ${{ env.TRAVIS_DIR }}
- commit-id: ${{ steps.get-commit.outputs.commit_id }}
- repository: ${{ steps.get-commit.outputs.repository }}
- fetch-ref: ${{ steps.get-commit.outputs.fetch_ref }}
+ commit-id: ${{ inputs.server-commit }}
+ repository: ${{ inputs.server-repository }}
+ fetch-ref: ${{ inputs.server-ref }}
+ artifact-name: ${{ inputs.artifact-name }}
+ mode: start
+ java-version: '11'
+ java-home: ${{ steps.server-java.outputs.path }}
- name: Init Go env
uses: actions/setup-go@v6
diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml
index 98dd82919..6b3fda654 100644
--- a/.github/workflows/codeql-analysis.yml
+++ b/.github/workflows/codeql-analysis.yml
@@ -1,33 +1,17 @@
-# For most projects, this workflow file will not need changing; you simply need
-# to commit it to your repository.
-#
-# You may wish to alter this file to override the set of languages analyzed,
-# or to provide custom queries or build logic.
-#
-# ******** NOTE ********
-# We have attempted to detect the languages in your repository. Please check
-# the `language` matrix defined below to confirm you have the correct set of
-# supported CodeQL languages.
-#
name: "CodeQL"
on:
workflow_dispatch:
- pull_request:
- # The branches below must be a subset of the branches above
- # branches: [ master ] # enable in all PR
+ workflow_call:
schedule:
- cron: '42 20 * * 6'
-# Keep the newest PR run; preserve independent push, release and manual runs.
-concurrency:
- group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.run_id }}
- cancel-in-progress: ${{ github.event_name == 'pull_request' }}
-
jobs:
analyze:
env:
USE_STAGE: 'true' # Whether to include the stage repository.
+ # Preserve the master scan identity when invoked by a reusable-workflow caller.
+ CODEQL_ACTION_ANALYSIS_KEY: .github/workflows/codeql-analysis.yml:analyze
name: Analyze
runs-on: ubuntu-latest
permissions:
@@ -54,8 +38,8 @@ jobs:
- name: use staged maven repo settings
if: ${{ env.USE_STAGE == 'true' }}
run: |
- cp $HOME/.m2/settings.xml /tmp/settings.xml
- mv -vf .github/configs/settings.xml $HOME/.m2/settings.xml
+ cp "$HOME/.m2/settings.xml" /tmp/settings.xml
+ mv -vf .github/configs/settings.xml "$HOME/.m2/settings.xml"
- name: Use Node.js 16
uses: actions/setup-node@v6
@@ -97,8 +81,11 @@ jobs:
- name: Perform CodeQL Analysis
uses: github/codeql-action/analyze@v4
+ with:
+ category: .github/workflows/codeql-analysis.yml:analyze/language:${{ matrix.language }}
dependency-review:
+ if: github.event_name == 'pull_request'
runs-on: ubuntu-latest
steps:
- name: 'Checkout Repository'
diff --git a/.github/workflows/hubble-ci.yml b/.github/workflows/hubble-ci.yml
index 733e09212..84788eab4 100644
--- a/.github/workflows/hubble-ci.yml
+++ b/.github/workflows/hubble-ci.yml
@@ -1,34 +1,30 @@
name: "hubble-ci"
on:
- workflow_dispatch:
- push:
- branches:
- - master
- - 'release-*'
- paths:
- - hugegraph-hubble/**
- - hugegraph-dist/**
- - hugegraph-loader/**
- - .github/workflows/**
- - pom.xml
- pull_request:
- paths:
- - hugegraph-hubble/**
- - hugegraph-dist/**
- - hugegraph-loader/**
- - hugegraph-client/**
- - .github/workflows/**
- - pom.xml
+ workflow_call:
+ inputs:
+ current-artifact-name:
+ type: string
+ required: true
+ current-commit:
+ type: string
+ required: true
+ server-repository:
+ type: string
+ default: apache/hugegraph
+ artifact-name:
+ type: string
+ required: true
+ server-commit:
+ type: string
+ required: true
+ server-ref:
+ type: string
+ required: true
env:
TRAVIS_DIR: hugegraph-hubble/hubble-dist/assembly/travis
-# Keep the newest PR run; preserve independent push, release and manual runs.
-concurrency:
- group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.run_id }}
- cancel-in-progress: ${{ github.event_name == 'pull_request' }}
-
jobs:
hubble-ci:
runs-on: ubuntu-latest
@@ -36,8 +32,10 @@ jobs:
USE_STAGE: 'false' # Whether to include the stage repository.
STATIC_DIR: hugegraph-hubble/hubble-dist/assembly/static
strategy:
+ fail-fast: false
matrix:
JAVA_VERSION: ['17']
+ SERVER_BASELINE: ['current', 'release-1.7']
python-version: ["3.11"]
steps:
@@ -46,14 +44,13 @@ jobs:
with:
fetch-depth: 2
- - name: Install released Server JDK
+ - name: Install Server fixture JVM
+ id: server-java
uses: actions/setup-java@v5
with:
- java-version: '11'
- distribution: 'temurin'
-
- - name: Record released Server JDK
- run: echo "SERVER_JAVA_HOME=$JAVA_HOME" >> "$GITHUB_ENV"
+ java-version: ${{ matrix.SERVER_BASELINE == 'current' && '17' || '11' }}
+ distribution: temurin
+ overwrite-settings: false
- name: Setup Java environment
uses: ./.github/actions/setup-java-env
@@ -99,19 +96,6 @@ jobs:
path: ~/.cache/ms-playwright
key: ${{ runner.os }}-playwright-${{ hashFiles('hugegraph-hubble/hubble-fe/yarn.lock') }}
- - name: Resolve released HugeGraph Server 1.7
- id: server-ref
- uses: ./.github/actions/get-hugegraph-commit
- with:
- ref: 'refs/tags/1.7.0'
- expected-commit: 'b12425c2032bf0d21a97b8221f42a18055c2982f'
-
- - name: Cache HugeGraph Server
- uses: actions/cache@v6
- with:
- path: ~/hugegraph-cache-${{ steps.server-ref.outputs.commit_id }}
- key: ${{ runner.os }}-hugegraph-server-${{ steps.server-ref.outputs.commit_id }}
-
- name: Frontend i18n check
working-directory: hugegraph-hubble/hubble-fe
env:
@@ -133,13 +117,13 @@ jobs:
- name: Compile
run: |
mvn install -pl hugegraph-client,hugegraph-loader -am -Dmaven.javadoc.skip=true -DskipTests -ntp
- cd hugegraph-hubble && ls *
+ cd hugegraph-hubble && ls -- *
mvn -e compile -Dmaven.javadoc.skip=true -ntp
- name: Prepare env and service
env:
- COMMIT_ID: ${{ steps.server-ref.outputs.commit_id }}
- COMMIT_REF: ${{ steps.server-ref.outputs.fetch_ref }}
+ COMMIT_ID: ${{ matrix.SERVER_BASELINE == 'current' && inputs.current-commit || inputs.server-commit }}
+ COMMIT_REF: ${{ matrix.SERVER_BASELINE == 'current' && inputs.current-commit || inputs.server-commit }}
NODE_OPTIONS: --max-old-space-size=4096
run: |
echo "=== Environment Info ==="
@@ -151,7 +135,7 @@ jobs:
free -h || true
echo "======================="
- python -m pip install -r ${TRAVIS_DIR}/requirements.txt
+ python -m pip install -r "${TRAVIS_DIR}/requirements.txt"
cd hugegraph-hubble
mvn package -Dmaven.test.skip=true
cd apache-hugegraph-hubble-*
@@ -160,8 +144,18 @@ jobs:
./stop-hubble.sh
cd ../../../
pwd
- JAVA_HOME="$SERVER_JAVA_HOME" PATH="$SERVER_JAVA_HOME/bin:$PATH" \
- "$TRAVIS_DIR/install-hugegraph.sh" "$COMMIT_ID" "$COMMIT_REF"
+
+ - name: Setup HugeGraph server
+ uses: ./.github/actions/setup-hugegraph-server
+ with:
+ mode: start
+ java-version: ${{ matrix.SERVER_BASELINE == 'current' && '17' || '11' }}
+ java-home: ${{ steps.server-java.outputs.path }}
+ artifact-name: ${{ matrix.SERVER_BASELINE == 'current' && inputs.current-artifact-name || inputs.artifact-name }}
+ commit-id: ${{ matrix.SERVER_BASELINE == 'current' && inputs.current-commit || inputs.server-commit }}
+ repository: ${{ inputs.server-repository }}
+ fetch-ref: ${{ matrix.SERVER_BASELINE == 'current' && inputs.current-commit || inputs.server-ref }}
+ travis-dir: ${{ env.TRAVIS_DIR }}
- name: Archive preservation test
run: python3 "$TRAVIS_DIR/test_archive_hubble.py"
@@ -170,11 +164,12 @@ jobs:
env:
GNUPGHOME: ${{ runner.temp }}/hubble-release-gate-gpghome
run: |
- HUBBLE_TARBALL="$(ls hugegraph-hubble/target/apache-hugegraph-hubble-*.tar.gz | head -n 1)"
- if [[ -z "${HUBBLE_TARBALL}" ]]; then
- echo "Hubble tarball not found" >&2
+ archives=(hugegraph-hubble/target/apache-hugegraph-hubble-*.tar.gz)
+ if [[ ${#archives[@]} -ne 1 || ! -f "${archives[0]}" ]]; then
+ echo "Expected exactly one Hubble tarball" >&2
exit 1
fi
+ HUBBLE_TARBALL="${archives[0]}"
mkdir -p "${GNUPGHOME}"
chmod 700 "${GNUPGHOME}"
@@ -202,7 +197,12 @@ jobs:
- name: API test
run: |
cd hugegraph-hubble
- HUBBLE_TARBALL="$(ls target/apache-hugegraph-hubble-*.tar.gz | head -n 1)"
+ archives=(target/apache-hugegraph-hubble-*.tar.gz)
+ if [[ ${#archives[@]} -ne 1 || ! -f "${archives[0]}" ]]; then
+ echo "Expected exactly one Hubble tarball" >&2
+ exit 1
+ fi
+ HUBBLE_TARBALL="${archives[0]}"
HUBBLE_694_EVIDENCE_DIR=target/hubble-live-acceptance \
hubble-dist/assembly/travis/verify-hubble-issue-694.sh \
"${HUBBLE_TARBALL}" http://127.0.0.1:8080
@@ -211,7 +211,7 @@ jobs:
if: ${{ always() }}
uses: actions/upload-artifact@v7
with:
- name: hubble-smoke-java-${{ matrix.JAVA_VERSION }}-${{ github.run_attempt }}
+ name: hubble-smoke-java-${{ matrix.JAVA_VERSION }}-${{ matrix.SERVER_BASELINE }}-${{ github.run_attempt }}
path: |
hugegraph-hubble/target/hubble-dist-check.json
hugegraph-hubble/target/hubble-live-acceptance/**/*.json
diff --git a/.github/workflows/image-ci.yml b/.github/workflows/image-ci.yml
new file mode 100644
index 000000000..70f5826e3
--- /dev/null
+++ b/.github/workflows/image-ci.yml
@@ -0,0 +1,44 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+name: Toolchain image startup
+
+on:
+ workflow_call:
+ inputs:
+ module:
+ required: true
+ type: string
+
+permissions:
+ contents: read
+
+jobs:
+ image:
+ name: build-and-start
+ runs-on: ubuntu-latest
+ timeout-minutes: 50
+ steps:
+ - uses: actions/checkout@v7
+ with:
+ persist-credentials: false
+ - name: Build and start this checkout's image
+ env:
+ MODULE: ${{ inputs.module }}
+ run: python3 .github/scripts/check-toolchain-image.py "$MODULE"
diff --git a/.github/workflows/labeler.yml b/.github/workflows/labeler.yml
index 0222c1ae6..ab9e1e34d 100644
--- a/.github/workflows/labeler.yml
+++ b/.github/workflows/labeler.yml
@@ -22,7 +22,7 @@ on:
- pull_request_target
concurrency:
- group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.run_id }}
+ group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.run_id }}-${{ github.run_attempt == 1 && 'head' || format('rerun-{0}', github.run_id) }}
cancel-in-progress: ${{ github.event_name == 'pull_request_target' }}
jobs:
diff --git a/.github/workflows/license-checker.yml b/.github/workflows/license-checker.yml
index 7e1981a34..feb050b14 100644
--- a/.github/workflows/license-checker.yml
+++ b/.github/workflows/license-checker.yml
@@ -28,15 +28,38 @@ on:
# Keep the newest PR run; preserve independent push, release and manual runs.
concurrency:
- group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.run_id }}
+ group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.run_id }}-${{ github.run_attempt == 1 && 'head' || format('rerun-{0}', github.run_id) }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
jobs:
check-license-header:
runs-on: ubuntu-latest
+ outputs:
+ dependencies: ${{ steps.dependencies.outputs.changed }}
steps:
- name: Fetch Code
uses: actions/checkout@v7
+ with:
+ fetch-depth: 0
+ - name: Select dependency license audit
+ id: dependencies
+ env:
+ EVENT_NAME: ${{ github.event_name }}
+ BASE_SHA: ${{ github.event.pull_request.base.sha }}
+ HEAD_SHA: ${{ github.event.pull_request.head.sha }}
+ run: |
+ changed=true
+ if [[ "$EVENT_NAME" == pull_request ]]; then
+ if git diff --name-only "$BASE_SHA...$HEAD_SHA" > "$RUNNER_TEMP/license-paths"; then
+ changed=false
+ if grep -Eq '(^|/)(pom.xml|package.json|yarn.lock|package-lock.json)$|(^|/)(assembly|licenses)/|(^|/)(LICENSE|NOTICE)$|dependency/|^\.github/(configs/|workflows/license-checker.yml)' "$RUNNER_TEMP/license-paths"; then
+ changed=true
+ fi
+ fi
+ fi
+ echo "changed=$changed" >> "$GITHUB_OUTPUT"
+ - name: Test automatic retry freshness rules
+ run: python3 -m unittest discover -s .github/scripts -p 'test_*.py'
# More info could refer to: https://github.com/apache/skywalking-eyes
- name: Check License Header
uses: apache/skywalking-eyes@29bd646002b63bb4c0ed1648d1654c35fb8ff027 # main 2026-07-12
@@ -53,6 +76,8 @@ jobs:
grep "Binaries" merged-rat.txt -C 3 && cat merged-rat.txt
check-dependency-license:
+ needs: check-license-header
+ if: needs.check-license-header.outputs.dependencies == 'true'
runs-on: ubuntu-latest
env:
SCRIPT_DEPENDENCY: hugegraph-dist/scripts/dependency
@@ -73,15 +98,15 @@ jobs:
- name: use staged maven repo settings
if: ${{ env.USE_STAGE == 'true' }}
run: |
- cp $HOME/.m2/settings.xml /tmp/settings.xml
- mv -vf .github/configs/settings.xml $HOME/.m2/settings.xml
+ cp "$HOME/.m2/settings.xml" /tmp/settings.xml
+ mv -vf .github/configs/settings.xml "$HOME/.m2/settings.xml"
- name: mvn install
run: |
mvn install -DskipTests=true -ntp
- name: generate current dependencies
run: |
- bash $SCRIPT_DEPENDENCY/regenerate_known_dependencies.sh current-dependencies.txt
+ bash "$SCRIPT_DEPENDENCY/regenerate_known_dependencies.sh" current-dependencies.txt
- name: check third dependencies
run: |
- bash $SCRIPT_DEPENDENCY/check_dependencies.sh
+ bash "$SCRIPT_DEPENDENCY/check_dependencies.sh"
diff --git a/.github/workflows/loader-ci.yml b/.github/workflows/loader-ci.yml
index 8eca79f31..46133f557 100644
--- a/.github/workflows/loader-ci.yml
+++ b/.github/workflows/loader-ci.yml
@@ -1,31 +1,24 @@
name: "loader-ci"
on:
- workflow_dispatch:
- push:
- branches:
- - master
- - 'release-*'
- paths:
- - hugegraph-loader/**
- - hugegraph-dist/**
- - .github/workflows/**
- - pom.xml
- pull_request:
- paths:
- - hugegraph-loader/**
- - hugegraph-dist/**
- - hugegraph-client/**
- - .github/workflows/**
- - pom.xml
-
-# Keep the newest PR run; preserve independent push, release and manual runs.
-concurrency:
- group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.run_id }}
- cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+ workflow_call:
+ inputs:
+ server-repository:
+ type: string
+ default: apache/hugegraph
+ artifact-name:
+ type: string
+ required: true
+ server-commit:
+ type: string
+ required: true
+ server-ref:
+ type: string
+ required: true
jobs:
- loader-ci:
+ profiles:
+ name: profiles
runs-on: ubuntu-latest
env:
USE_STAGE: 'true' # Whether to include the stage repository.
@@ -36,7 +29,7 @@ jobs:
DB_DATABASE: load_test
strategy:
matrix:
- JAVA_VERSION: ['11']
+ JAVA_VERSION: ['11', '17']
services:
mysql:
image: mysql:8.0
@@ -52,9 +45,13 @@ jobs:
with:
fetch-depth: 2
- - name: Get HugeGraph stable commit id
- id: get-commit
- uses: ./.github/actions/get-hugegraph-commit
+ - name: Install released Server JVM
+ id: server-java
+ uses: actions/setup-java@v5
+ with:
+ java-version: '11'
+ distribution: temurin
+ overwrite-settings: false
- name: Setup Java environment
uses: ./.github/actions/setup-java-env
@@ -63,40 +60,27 @@ jobs:
distribution: 'temurin'
use-stage: ${{ env.USE_STAGE }}
- - name: Cache Hadoop
- uses: actions/cache@v6
- with:
- path: ~/hadoop-3.3.6.tar.gz
- key: ${{ runner.os }}-hadoop-3.3.6
-
- - name: Cache HugeGraph Server
- uses: actions/cache@v6
- with:
- path: ~/hugegraph-cache-${{ steps.get-commit.outputs.commit_id }}
- key: ${{ runner.os }}-hugegraph-server-provenance-v1-${{ steps.get-commit.outputs.commit_id }}
-
- name: Compile
run: |
mvn install -pl hugegraph-client,hugegraph-loader -am -Dmaven.javadoc.skip=true -DskipTests -ntp
- - name: Prepare env and service
- env:
- COMMIT_ID: ${{ steps.get-commit.outputs.commit_id }}
- SERVER_REPOSITORY: ${{ steps.get-commit.outputs.repository }}
- SERVER_FETCH_REF: ${{ steps.get-commit.outputs.fetch_ref }}
- run: |
- $TRAVIS_DIR/install-hadoop.sh
- installer=$(realpath "$TRAVIS_DIR/install-hugegraph-from-source.sh")
- server_build=$(mktemp -d "$RUNNER_TEMP/hugegraph-build.XXXXXX")
- cd "$server_build"
- bash "$installer" "$COMMIT_ID"
+ - name: Setup HugeGraph server
+ uses: ./.github/actions/setup-hugegraph-server
+ with:
+ mode: start
+ java-version: '11'
+ java-home: ${{ steps.server-java.outputs.path }}
+ artifact-name: ${{ inputs.artifact-name }}
+ commit-id: ${{ inputs.server-commit }}
+ repository: ${{ inputs.server-repository }}
+ fetch-ref: ${{ inputs.server-ref }}
+ travis-dir: ${{ env.TRAVIS_DIR }}
- name: Run test
run: |
cd hugegraph-loader && ls
mvn test -P unit -ntp
mvn test -P file
- mvn test -P hdfs
mvn test -P jdbc
mvn test -P kafka
@@ -105,3 +89,76 @@ jobs:
with:
token: ${{ secrets.CODECOV_TOKEN }}
file: hugegraph-loader/target/jacoco.xml
+
+ hdfs:
+ name: hdfs
+ runs-on: ubuntu-latest
+ env:
+ USE_STAGE: 'true' # Whether to include the stage repository.
+ TRAVIS_DIR: hugegraph-loader/assembly/travis
+ STATIC_DIR: hugegraph-loader/assembly/static
+ DB_USER: root
+ DB_PASS: root
+ DB_DATABASE: load_test
+ strategy:
+ matrix:
+ JAVA_VERSION: ['11', '17']
+ steps:
+ - name: Fetch Code
+ uses: actions/checkout@v7
+ with:
+ fetch-depth: 2
+
+ - name: Install released Server JVM
+ id: server-java
+ uses: actions/setup-java@v5
+ with:
+ java-version: '11'
+ distribution: temurin
+ overwrite-settings: false
+
+ - name: Setup Java environment
+ uses: ./.github/actions/setup-java-env
+ with:
+ java-version: ${{ matrix.JAVA_VERSION }}
+ distribution: 'temurin'
+ use-stage: ${{ env.USE_STAGE }}
+
+ - name: Compile
+ run: |
+ mvn install -pl hugegraph-client,hugegraph-loader -am -Dmaven.javadoc.skip=true -DskipTests -ntp
+
+ - name: Setup HugeGraph server
+ uses: ./.github/actions/setup-hugegraph-server
+ with:
+ mode: start
+ java-version: '11'
+ java-home: ${{ steps.server-java.outputs.path }}
+ artifact-name: ${{ inputs.artifact-name }}
+ commit-id: ${{ inputs.server-commit }}
+ repository: ${{ inputs.server-repository }}
+ fetch-ref: ${{ inputs.server-ref }}
+ travis-dir: ${{ env.TRAVIS_DIR }}
+
+ - name: Start HDFS from the official Hadoop image
+ timeout-minutes: 18
+ run: |
+ "$TRAVIS_DIR/start-hdfs-ci.sh"
+
+ - name: Run HDFS tests
+ run: mvn test -P hdfs -f hugegraph-loader/pom.xml -ntp
+
+ - name: Collect HDFS logs and remove containers
+ if: ${{ always() }}
+ run: |
+ for service in namenode datanode; do
+ name="${HDFS_CONTAINER_PREFIX:-hugegraph-ci-hdfs}-$service"
+ docker logs "$name" || true
+ docker rm -f "$name" || true
+ done
+
+ - name: Upload coverage to Codecov
+ uses: ./.github/actions/upload-coverage
+ with:
+ token: ${{ secrets.CODECOV_TOKEN }}
+ file: hugegraph-loader/target/jacoco.xml
diff --git a/.github/workflows/rerun-ci.yml b/.github/workflows/rerun-ci.yml
index 5b3202226..5c1fe8d93 100644
--- a/.github/workflows/rerun-ci.yml
+++ b/.github/workflows/rerun-ci.yml
@@ -3,24 +3,23 @@ name: "Rerun CI"
on:
workflow_run:
workflows:
- - "java-client-ci"
- - "loader-ci"
- - "tools-ci"
- - "hubble-ci"
- - "spark-connector-ci"
- - "go-client-ci"
+ - "Required CI"
types:
- completed
permissions: {}
+concurrency:
+ group: ${{ github.workflow }}-${{ github.event.workflow_run.id }}
+ cancel-in-progress: false
+
env:
MAX_RERUNS: '2'
RETRY_DELAY_SECONDS: '180'
jobs:
decide-rerun-action:
- if: ${{ github.event.workflow_run.conclusion == 'failure' }}
+ if: ${{ github.event.workflow_run.conclusion == 'failure' && github.event.workflow_run.event == 'push' }}
runs-on: ubuntu-latest
outputs:
action: ${{ steps.decision.outputs.action }}
@@ -39,7 +38,7 @@ jobs:
action="skip"
reason="unsupported event: $EVENT_NAME"
- if [[ "$EVENT_NAME" == "push" || "$EVENT_NAME" == "pull_request" ]]; then
+ if [[ "$EVENT_NAME" == "push" ]]; then
if (( RUN_ATTEMPT > MAX_RERUNS )); then
reason="retry limit reached"
else
@@ -75,59 +74,42 @@ jobs:
contents: read
runs-on: ubuntu-latest
steps:
- - name: Wait before rerun
+ # workflow_run is privileged: only load code from the default-branch commit.
+ - name: Checkout trusted retry checker
+ uses: actions/checkout@v7
+ with:
+ ref: ${{ github.sha }}
+ persist-credentials: false
+ sparse-checkout: .github/scripts
+
+ - name: Check source freshness before waiting
+ id: initial-freshness
+ env:
+ GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+ GH_REPO: ${{ github.repository }}
+ RUN_ID: ${{ github.event.workflow_run.id }}
+ RUN_ATTEMPT: ${{ github.event.workflow_run.run_attempt }}
run: |
- sleep "$RETRY_DELAY_SECONDS"
+ python3 .github/scripts/check-rerun.py \
+ --repository "$GH_REPO" --run-id "$RUN_ID" \
+ --run-attempt "$RUN_ATTEMPT" --max-reruns "$MAX_RERUNS"
+
+ - name: Wait before rerun
+ if: steps.initial-freshness.outputs.action == 'rerun'
+ run: sleep "$RETRY_DELAY_SECONDS"
- - name: Check source branch freshness
+ - name: Check source freshness after waiting
id: freshness
+ if: steps.initial-freshness.outputs.action == 'rerun'
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- HEAD_REPOSITORY: ${{ github.event.workflow_run.head_repository.full_name }}
- HEAD_BRANCH: ${{ github.event.workflow_run.head_branch }}
- HEAD_SHA: ${{ github.event.workflow_run.head_sha }}
- WORKFLOW_NAME: ${{ github.event.workflow_run.name }}
+ GH_REPO: ${{ github.repository }}
RUN_ID: ${{ github.event.workflow_run.id }}
+ RUN_ATTEMPT: ${{ github.event.workflow_run.run_attempt }}
run: |
- set -euo pipefail
-
- action="rerun"
- latest_head_sha="$HEAD_SHA"
- reason="head unchanged"
-
- if [[ -z "$HEAD_REPOSITORY" || -z "$HEAD_BRANCH" || -z "$HEAD_SHA" ]]; then
- action="skip"
- reason="missing head repository metadata"
- else
- latest_head_sha="$(gh api "repos/$HEAD_REPOSITORY/commits" -f sha="$HEAD_BRANCH" -F per_page=1 --jq '.[0].sha' 2>/dev/null || true)"
-
- if [[ -z "$latest_head_sha" || "$latest_head_sha" == "null" ]]; then
- action="skip"
- reason="head branch no longer available"
- elif [[ "$latest_head_sha" != "$HEAD_SHA" ]]; then
- action="skip"
- reason="head moved to $latest_head_sha"
- fi
- fi
-
- {
- echo "action=$action"
- echo "latest_head_sha=$latest_head_sha"
- echo "reason=$reason"
- } >> "$GITHUB_OUTPUT"
-
- {
- echo "### Rerun CI freshness check"
- echo ""
- echo "- Workflow: $WORKFLOW_NAME"
- echo "- Run ID: $RUN_ID"
- echo "- Head repository: $HEAD_REPOSITORY"
- echo "- Head branch: $HEAD_BRANCH"
- echo "- Failed run head SHA: $HEAD_SHA"
- echo "- Current branch tip SHA: $latest_head_sha"
- echo "- Action: $action"
- echo "- Reason: $reason"
- } >> "$GITHUB_STEP_SUMMARY"
+ python3 .github/scripts/check-rerun.py \
+ --repository "$GH_REPO" --run-id "$RUN_ID" \
+ --run-attempt "$RUN_ATTEMPT" --max-reruns "$MAX_RERUNS"
- name: Rerun failed jobs
if: steps.freshness.outputs.action == 'rerun'
diff --git a/.github/workflows/server-fixture-ci.yml b/.github/workflows/server-fixture-ci.yml
index 014c30bd0..1c2b384c1 100644
--- a/.github/workflows/server-fixture-ci.yml
+++ b/.github/workflows/server-fixture-ci.yml
@@ -19,40 +19,13 @@
name: "server-fixture-ci"
+# Automatic PR validation runs these contracts in Required CI.
on:
workflow_dispatch:
- push:
- branches:
- - master
- - 'release-*'
- paths:
- - .github/actions/get-hugegraph-commit/**
- - .github/actions/setup-hugegraph-server/**
- - .github/workflows/server-fixture-ci.yml
- - hugegraph-client/assembly/travis/*server*
- - hugegraph-client/assembly/travis/install-hugegraph-from-source.sh
- - hugegraph-loader/assembly/travis/install-hugegraph-from-source.sh
- - hugegraph-tools/assembly/travis/install-hugegraph-from-source.sh
- - hugegraph-spark-connector/assembly/travis/install-hugegraph-from-source.sh
- pull_request:
- paths:
- - .github/actions/get-hugegraph-commit/**
- - .github/actions/setup-hugegraph-server/**
- - .github/workflows/server-fixture-ci.yml
- - hugegraph-client/assembly/travis/*server*
- - hugegraph-client/assembly/travis/install-hugegraph-from-source.sh
- - hugegraph-loader/assembly/travis/install-hugegraph-from-source.sh
- - hugegraph-tools/assembly/travis/install-hugegraph-from-source.sh
- - hugegraph-spark-connector/assembly/travis/install-hugegraph-from-source.sh
permissions:
contents: read
-# Keep the newest PR run; preserve independent push, release and manual runs.
-concurrency:
- group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.run_id }}
- cancel-in-progress: ${{ github.event_name == 'pull_request' }}
-
jobs:
server-fixture-contracts:
runs-on: ubuntu-latest
diff --git a/.github/workflows/spark-connector-ci.yml b/.github/workflows/spark-connector-ci.yml
index 1a3b52d3f..cd1c55e47 100644
--- a/.github/workflows/spark-connector-ci.yml
+++ b/.github/workflows/spark-connector-ci.yml
@@ -1,28 +1,20 @@
name: "spark-connector-ci"
on:
- workflow_dispatch:
- push:
- branches:
- - master
- - 'release-*'
- paths:
- - hugegraph-spark-connector/**
- - hugegraph-dist/**
- - .github/workflows/**
- - pom.xml
- pull_request:
- paths:
- - hugegraph-spark-connector/**
- - hugegraph-dist/**
- - hugegraph-client/**
- - .github/workflows/**
- - pom.xml
-
-# Keep the newest PR run; preserve independent push, release and manual runs.
-concurrency:
- group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.run_id }}
- cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+ workflow_call:
+ inputs:
+ server-repository:
+ type: string
+ default: apache/hugegraph
+ artifact-name:
+ type: string
+ required: true
+ server-commit:
+ type: string
+ required: true
+ server-ref:
+ type: string
+ required: true
jobs:
spark-connector-ci:
@@ -33,7 +25,7 @@ jobs:
strategy:
fail-fast: false
matrix:
- JAVA_VERSION: [ '11', '17' ]
+ JAVA_VERSION: ['11', '17']
steps:
- name: Fetch Code
@@ -41,21 +33,13 @@ jobs:
with:
fetch-depth: 2
- - name: Get HugeGraph stable commit id
- id: get-commit
- uses: ./.github/actions/get-hugegraph-commit
-
- # The published HugeGraph 1.7 authentication fixture requires Java 11.
- - name: Setup server Java
+ - name: Install released Server JVM
id: server-java
uses: actions/setup-java@v5
with:
java-version: '11'
- distribution: 'temurin'
-
- - name: Save server Java path
- id: server-path
- run: echo "path=$JAVA_HOME/bin:$PATH" >> "$GITHUB_OUTPUT"
+ distribution: temurin
+ overwrite-settings: false
- name: Setup Java environment
uses: ./.github/actions/setup-java-env
@@ -70,14 +54,15 @@ jobs:
- name: Setup HugeGraph server
uses: ./.github/actions/setup-hugegraph-server
- env:
- JAVA_HOME: ${{ steps.server-java.outputs.path }}
- PATH: ${{ steps.server-path.outputs.path }}
with:
travis-dir: ${{ env.TRAVIS_DIR }}
- commit-id: ${{ steps.get-commit.outputs.commit_id }}
- repository: ${{ steps.get-commit.outputs.repository }}
- fetch-ref: ${{ steps.get-commit.outputs.fetch_ref }}
+ commit-id: ${{ inputs.server-commit }}
+ repository: ${{ inputs.server-repository }}
+ fetch-ref: ${{ inputs.server-ref }}
+ artifact-name: ${{ inputs.artifact-name }}
+ mode: start
+ java-version: '11'
+ java-home: ${{ steps.server-java.outputs.path }}
- name: Run test
run: |
diff --git a/.github/workflows/tools-ci.yml b/.github/workflows/tools-ci.yml
index 18de5cb6e..f9d11239c 100644
--- a/.github/workflows/tools-ci.yml
+++ b/.github/workflows/tools-ci.yml
@@ -1,28 +1,20 @@
name: "tools-ci"
on:
- workflow_dispatch:
- push:
- branches:
- - master
- - 'release-*'
- paths:
- - hugegraph-tools/**
- - hugegraph-dist/**
- - .github/workflows/**
- - pom.xml
- pull_request:
- paths:
- - hugegraph-tools/**
- - hugegraph-dist/**
- - hugegraph-client/**
- - .github/workflows/**
- - pom.xml
-
-# Keep the newest PR run; preserve independent push, release and manual runs.
-concurrency:
- group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.run_id }}
- cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+ workflow_call:
+ inputs:
+ server-repository:
+ type: string
+ default: apache/hugegraph
+ artifact-name:
+ type: string
+ required: true
+ server-commit:
+ type: string
+ required: true
+ server-ref:
+ type: string
+ required: true
jobs:
tools-ci:
@@ -32,7 +24,7 @@ jobs:
TRAVIS_DIR: hugegraph-tools/assembly/travis
strategy:
matrix:
- JAVA_VERSION: [ '11' ]
+ JAVA_VERSION: ['11', '17']
steps:
- name: Fetch Code
@@ -40,9 +32,13 @@ jobs:
with:
fetch-depth: 2
- - name: Get HugeGraph stable commit id
- id: get-commit
- uses: ./.github/actions/get-hugegraph-commit
+ - name: Install released Server JVM
+ id: server-java
+ uses: actions/setup-java@v5
+ with:
+ java-version: '11'
+ distribution: temurin
+ overwrite-settings: false
- name: Setup Java environment
uses: ./.github/actions/setup-java-env
@@ -59,9 +55,13 @@ jobs:
uses: ./.github/actions/setup-hugegraph-server
with:
travis-dir: ${{ env.TRAVIS_DIR }}
- commit-id: ${{ steps.get-commit.outputs.commit_id }}
- repository: ${{ steps.get-commit.outputs.repository }}
- fetch-ref: ${{ steps.get-commit.outputs.fetch_ref }}
+ commit-id: ${{ inputs.server-commit }}
+ repository: ${{ inputs.server-repository }}
+ fetch-ref: ${{ inputs.server-ref }}
+ artifact-name: ${{ inputs.artifact-name }}
+ mode: start
+ java-version: '11'
+ java-home: ${{ steps.server-java.outputs.path }}
- name: Run test
run: |
diff --git a/README.md b/README.md
index 536f365d2..16c208508 100644
--- a/README.md
+++ b/README.md
@@ -482,3 +482,5 @@ Thank you to all the people who already contributed to HugeGraph!
## License
hugegraph-toolchain is licensed under [Apache 2.0 License](https://github.com/apache/hugegraph-toolchain/blob/master/LICENSE).
+
+See [CI behavior](docs/ci.md) for affected-module tests, shared fixtures and retry rules.
diff --git a/docs/ci.md b/docs/ci.md
new file mode 100644
index 000000000..5926ef268
--- /dev/null
+++ b/docs/ci.md
@@ -0,0 +1,121 @@
+# Continuous integration
+
+Every PR runs license checks and `Required CI`. The stable
+`affected-module-tests` check requires all selected Linux tests and image startup
+checks to pass; a failed, cancelled or unexpectedly skipped selected job cannot
+satisfy it.
+
+| Change | Tests |
+| --- | --- |
+| Java Client | Client, Loader, Tools, Spark, Hubble |
+| Loader | Loader, Hubble |
+| Shared Client Server checkout/install/start scripts | Client downstreams and Go |
+| Tools, Spark, Go or Hubble | The affected module |
+| Shared build inputs or unrecognized paths | All modules |
+| One workflow | Its tests and shared dependencies |
+
+Image selection has its own matrix and does not follow the Client → Loader →
+Hubble module-test dependency closure:
+
+| Image input change | Images built and started |
+| --- | --- |
+| Loader Dockerfile | Loader |
+| Hubble Dockerfile | Hubble |
+| Loader POM, assembly descriptor/static files or packaged README/LICENSE/NOTICE | Loader and Hubble |
+| Hubble POMs, assembly descriptor/static files, distribution checker, backend startup properties or frontend build configuration | Hubble |
+| Root/Client/reactor POMs, `.mvn`, `.dockerignore`, shared release docs or unknown shared inputs | Loader and Hubble |
+| Ordinary Java/frontend application source or tests | None; the module behavior tests above still run |
+
+Dockerfile-only changes run the image check without building Server fixtures.
+Hubble's Dockerfile builds Client and Loader before Hubble, so Loader packaging
+changes also select Hubble. Packaged README changes select images; the existing
+nonempty Hubble README content contract still requires fresh validation if the
+file is missing, empty or has an invalid mode. Every selected module and image
+runs in the current workflow; successful results from another run are not reused.
+
+Each image job builds the checkout's actual Dockerfile, records its image ID and
+starts a local run-scoped tag with pulling disabled. The container's image ID must
+match the build. Loader directly executes `./bin/hugegraph-loader.sh --help`,
+including its executable permission and shebang; its default idle
+container alone cannot pass. Hubble must remain running and return a valid `/about`
+JSON response with the application name and version. Its public root page must
+contain the React root and serve nonempty local JavaScript bundles and referenced
+stylesheets with the correct content types, rejecting the HTML fallback for missing
+assets. These checks validate image build, packaging, startup and public entry
+points; Loader data ingestion and Hubble browser workflows remain covered by
+module tests. They do not publish an image.
+
+Selected Client, Loader, Tools and Spark module tests compile and run on Java 11
+and Java 17. Both matrix entries must pass, including Loader's separate HDFS tests.
+Hubble compiles and runs on Java 17, matching its merged Boot 3 runtime requirement,
+and tests against each Server baseline. These runtime checks supplement the
+existing affected-module selection; unrelated modules are not added.
+Client, Loader, Tools, Spark and Go test the released Server 1.7
+fixture on Java 11. Hubble tests both this release and the current Server master
+snapshot on Java 17; the resolved commit stays fixed while the run is queued.
+
+Each Server package is built once and shared, with independent services per job.
+Reuse verifies source, commit, JDK, build inputs and archive checksum. The Server
+JVM is scoped to service startup, so Toolchain compilation and tests keep their
+selected Java 11 or Java 17 runtime. Both Toolchain lanes reuse the same release
+Server fixture; Hubble uses the separate Java 17 current Server fixture.
+Loader's HDFS tests run separately, so other profiles do not wait for Hadoop.
+The Ubuntu HDFS jobs use the ASF Hadoop 3.3.6 image pinned by digest, with separate
+NameNode and DataNode containers on the host network. Tests still run on the
+selected Java 11/17 host JVM and use `localhost:8020`. Startup checks live DataNode
+registration and a real block write/read before testing; containers are removed
+afterward. Image pulling and readiness are bounded. This avoids installing the
+large archive on each runner, but cold runners still need to pull the image.
+The immutable Server packages, `ci-plan` and successful `ci-test-results` artifacts
+are retained for seven days from their upload. Partial reruns need the original
+plan and fixture within that window. After an artifact expires, rerun the whole
+workflow to rebuild its plan and fixtures; extending only the result report cannot
+restore an expired package. Test reports and coverage artifacts keep their
+existing retention settings.
+
+Documentation paths use an explicit allowlist. A PR containing only plain prose
+and static documentation assets has no module tests. Selection uses the cumulative
+PR diff, so a documentation update to a PR that also changes source runs its
+selected consumers again. Source, type definitions, tests and CI configuration
+are never treated as docs. The gate checks the current workflow's actual selected
+job results and fixture producers. Its result report records the run, attempt and
+input metadata for diagnostics; it is not a reusable success receipt.
+The planner determines affected consumers before resolving external Server inputs.
+Go-only changes resolve the release fixture but skip the Hubble master lookup;
+image-only and plain documentation changes need neither baseline. Unverifiable
+selection conservatively resolves both. A failed required baseline lookup still
+fails planning; without a published plan, recovery requires a full workflow rerun.
+
+PR planning records the event's head, base and source before querying live metadata.
+A known mismatch with the checkout merge or current PR fails planning; expanding
+coverage cannot make an outdated checkout current. After all selected jobs and
+fixtures succeed, the gate rechecks the open PR's head, base and source using a
+read-only API request. Changed inputs or unavailable metadata fail the gate.
+Refresh the branch and start a new PR event after head or base movement: GitHub
+reruns retain the original commit and event, so rerunning alone cannot refresh the
+base. An unknown API or selection failure falls back to full coverage; a manual
+full rerun can recover a transient failure while the recorded inputs remain current.
+
+CodeQL follows the module gate and retains its weekly scan. During migration,
+`Analyze (java)` still runs on every PR, after a successful module gate.
+The legacy alias is PR-only; documentation pushes do not force scanning or fail
+because a deliberately unselected scan was skipped. Scan write permissions are
+limited to the security job. After ASF branch protection actually
+requires `check-license-header` and `affected-module-tests`, an administrator can
+set `CI_OPTIMIZED_REQUIRED=true` to skip security scans for documentation-only
+changes. This switch is disabled by default and does not alter branch protection.
+
+A new PR head cancels older first attempts. Reruns use separate concurrency groups,
+so retrying an old commit cannot cancel the current head. Automatic retries run
+failed **push** jobs at most twice, checking the repository branch head and
+unchanged completed-failure run attempt both before and after the 180-second delay.
+Retry code comes from the trusted default branch. PR automatic retries and
+cross-run success reuse are deferred until execution evidence can be verified
+independently; manual workflow reruns remain available.
+
+Validate policy and retry behavior locally:
+
+```bash
+python3 -m unittest discover -s .github/scripts -p 'test_*.py'
+actionlint
+```
diff --git a/hugegraph-client/assembly/travis/install-hugegraph-from-source.sh b/hugegraph-client/assembly/travis/install-hugegraph-from-source.sh
index 2c1bccb7c..4f7b058da 100755
--- a/hugegraph-client/assembly/travis/install-hugegraph-from-source.sh
+++ b/hugegraph-client/assembly/travis/install-hugegraph-from-source.sh
@@ -22,6 +22,13 @@ if [[ $# -lt 1 || $# -gt 2 || ( $# -eq 2 && "$2" != "--build-only" ) ]]; then
exit 1
fi
+if [[ "${FIXTURE_MODE:-standalone}" == start ]]; then
+ FIXTURE_COMMIT=$1
+ ROOT_DIR=$(cd "$(dirname "$0")/../../.." && pwd)
+ source "$ROOT_DIR/.github/actions/setup-hugegraph-server/fixture.sh"
+ ARCHIVE=$(python3 "$ROOT_DIR/.github/actions/setup-hugegraph-server/manifest.py" name)
+ exec bash "$ROOT_DIR/hugegraph-client/assembly/travis/start-hugegraph-servers.sh" "$ARCHIVE"
+fi
SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd)
bash "$SCRIPT_DIR/checkout-server.sh" "$1" hugegraph
# Build the server distribution without replacing published client dependencies
diff --git a/hugegraph-client/assembly/travis/install-server.test.cjs b/hugegraph-client/assembly/travis/install-server.test.cjs
index 5c3f27771..cc3d32ae8 100644
--- a/hugegraph-client/assembly/travis/install-server.test.cjs
+++ b/hugegraph-client/assembly/travis/install-server.test.cjs
@@ -52,6 +52,7 @@ test(`build-only ${installer} preserves source selection and packages without de
const sha = git('-C', source, 'rev-parse', 'HEAD');
const bin = join(root, 'bin');
mkdirSync(bin);
+ writeFileSync(join(bin, 'curl'), '#!/bin/bash\nexit 0\n', { mode: 0o755 });
writeFileSync(join(bin, 'mvn'), '#!/bin/bash\nprintf "%s\\n" "$@" > "$BUILD_ARGS"\nprintf "%s" "$MAVEN_ARGS" > "$BUILD_REPO"\n', { mode: 0o755 });
const env = { ...process.env, SERVER_REPOSITORY: 'example/server', SERVER_FETCH_REF: sha,
PATH: `${bin}:${process.env.PATH}`, BUILD_ARGS: join(root, 'args'), BUILD_REPO: join(root, 'repo'),
@@ -94,9 +95,11 @@ function runStarter(root, entries) {
const archive = join(root, 'server.tar.gz');
// Keep metadata entries as files on macOS too, matching the Linux CI extractor.
const env = { ...process.env, COPYFILE_DISABLE: '1', RUNNER_TEMP: root, JAVA_HOME: '/fixture/java11' };
+ const bin = join(root, 'tar-bin');
+ mkdirSync(bin);
+ writeFileSync(join(bin, 'curl'), '#!/bin/bash\nexit 0\n', { mode: 0o755 });
+ env.PATH = `${bin}:${env.PATH}`;
if (process.platform === 'darwin') {
- const bin = join(root, 'tar-bin');
- mkdirSync(bin);
writeFileSync(join(bin, 'tar'),
'#!/bin/bash\n[[ "$1" == -* ]] || set -- "-$1" "${@:2}"\n' +
'exec /usr/bin/tar --no-mac-metadata "$@"\n', { mode: 0o755 });
@@ -248,6 +251,7 @@ test('Loader retains its source cache while using the selected repository and sh
{ env: { ...process.env, COPYFILE_DISABLE: '1' } });
const bin = join(root, 'bin');
mkdirSync(bin);
+ writeFileSync(join(bin, 'curl'), '#!/bin/bash\nexit 0\n', { mode: 0o755 });
writeFileSync(join(bin, 'mvn'), '#!/bin/bash\nprintf "%s\\n" "$@" > "$BUILD_ARGS"\n' +
'mkdir -p hugegraph-server\ncp "$FIXTURE_ARCHIVE" hugegraph-server/apache-hugegraph-fixture.tar.gz\n' +
'if [[ "$DUPLICATE_ARCHIVE" == true ]]; then\n' +
diff --git a/hugegraph-client/assembly/travis/start-hugegraph-servers.sh b/hugegraph-client/assembly/travis/start-hugegraph-servers.sh
index 310868e5d..9d9ba5c98 100644
--- a/hugegraph-client/assembly/travis/start-hugegraph-servers.sh
+++ b/hugegraph-client/assembly/travis/start-hugegraph-servers.sh
@@ -16,6 +16,7 @@
# under the License.
#
set -euo pipefail
+source "$(cd "$(dirname "$0")/../../.." && pwd)/.github/actions/setup-hugegraph-server/service-wait.sh"
if [[ $# -ne 1 || ! -f "$1" ]]; then
echo "Usage: start-hugegraph-servers.sh " >&2
@@ -60,9 +61,12 @@ start_server() (
)
start_server "${SERVER_DIRS[0]}"
+fixture_wait http://127.0.0.1:8080
HTTPS_DIR="$SERVER_ROOT/hugegraph_https"
sed -i.bak 's?http://127.0.0.1:8080?https://127.0.0.1:8443?g' "$HTTPS_DIR/conf/rest-server.properties"
sed -i.bak 's/rpc.server_port=8091/rpc.server_port=8092/g' "$HTTPS_DIR/conf/rest-server.properties"
sed -i.bak 's/#port: 8182/port: 8282/g' "$HTTPS_DIR/conf/gremlin-server.yaml"
printf '\ngremlinserver.url=http://127.0.0.1:8282\n' >> "$HTTPS_DIR/conf/rest-server.properties"
start_server "$HTTPS_DIR"
+
+fixture_wait https://127.0.0.1:8443
diff --git a/hugegraph-hubble/hubble-dist/assembly/travis/download-hugegraph.sh b/hugegraph-hubble/hubble-dist/assembly/travis/download-hugegraph.sh
index 9aa3f2dde..b69391c49 100755
--- a/hugegraph-hubble/hubble-dist/assembly/travis/download-hugegraph.sh
+++ b/hugegraph-hubble/hubble-dist/assembly/travis/download-hugegraph.sh
@@ -23,32 +23,6 @@ if [[ $# -lt 1 || $# -gt 2 ]]; then
exit 1
fi
-COMMIT_ID=$1
-COMMIT_REF=${2:-}
-SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd)
-REPO_ROOT=$(cd "$SCRIPT_DIR/../../../.." && pwd)
-GIT_DIR=hugegraph
-CACHE_DIR="${HOME}/hugegraph-cache-${COMMIT_ID}"
-
-mkdir -p "${CACHE_DIR}"
-CACHED_TARBALL=$(find "${CACHE_DIR}" -maxdepth 1 \
- -name "apache-hugegraph-*.tar.gz" -print -quit)
-
-if [[ -f "${CACHED_TARBALL}" ]]; then
- echo "Found HugeGraph server tarball cached for commit ${COMMIT_ID}."
- cp "${CACHED_TARBALL}" ./
- exit 0
-fi
-
-# Build the released server without replacing published SDK dependencies.
-SERVER_FETCH_REF="${COMMIT_REF:-$COMMIT_ID}" \
- bash "$REPO_ROOT/hugegraph-client/assembly/travis/checkout-server.sh" "$COMMIT_ID" "$GIT_DIR"
-cd "${GIT_DIR}"
-mvn package -DskipTests -Dmaven.javadoc.skip=true -ntp
-
-cd hugegraph-server
-TAR=$(echo apache-hugegraph-*.tar.gz)
-cp apache-hugegraph-*.tar.gz ../../
-cd ../../
-rm -rf "${GIT_DIR}"
-cp apache-hugegraph-*.tar.gz "${CACHE_DIR}"/
+FIXTURE_COMMIT=$1
+FIXTURE_REF=${2:-$1}
+source "$(cd "$(dirname "$0")/../../../.." && pwd)/.github/actions/setup-hugegraph-server/fixture.sh"
diff --git a/hugegraph-hubble/hubble-dist/assembly/travis/hugegraph-server1/install-hugegraph.sh b/hugegraph-hubble/hubble-dist/assembly/travis/hugegraph-server1/install-hugegraph.sh
index dbc7f8e77..10ca1b925 100755
--- a/hugegraph-hubble/hubble-dist/assembly/travis/hugegraph-server1/install-hugegraph.sh
+++ b/hugegraph-hubble/hubble-dist/assembly/travis/hugegraph-server1/install-hugegraph.sh
@@ -18,6 +18,7 @@
set -ev
SERVER_CONFIG_DIR=$(dirname "$0")
+source "$(cd "$SERVER_CONFIG_DIR/../../../../.." && pwd)/.github/actions/setup-hugegraph-server/service-wait.sh"
SERVER_PARENT_DIR="hugegraph-server1"
mkdir ${SERVER_PARENT_DIR}
@@ -37,3 +38,5 @@ cd "${SERVER_DIR}" && pwd
echo -e "pa" | bin/init-store.sh || exit 1
bin/start-hugegraph.sh || exit 1
+
+fixture_wait http://127.0.0.1:8080
diff --git a/hugegraph-hubble/hubble-dist/assembly/travis/hugegraph-server2/install-hugegraph.sh b/hugegraph-hubble/hubble-dist/assembly/travis/hugegraph-server2/install-hugegraph.sh
index 71ae5c4cf..6ba4bcacd 100755
--- a/hugegraph-hubble/hubble-dist/assembly/travis/hugegraph-server2/install-hugegraph.sh
+++ b/hugegraph-hubble/hubble-dist/assembly/travis/hugegraph-server2/install-hugegraph.sh
@@ -18,6 +18,7 @@
set -ev
SERVER_CONFIG_DIR=$(dirname "$0")
+source "$(cd "$SERVER_CONFIG_DIR/../../../../.." && pwd)/.github/actions/setup-hugegraph-server/service-wait.sh"
SERVER_PARENT_DIR="hugegraph-server2"
mkdir ${SERVER_PARENT_DIR}
@@ -37,3 +38,5 @@ cd "${SERVER_DIR}" && pwd
echo -e "pa" | bin/init-store.sh || exit 1
bin/start-hugegraph.sh || exit 1
+
+fixture_wait http://127.0.0.1:8081
diff --git a/hugegraph-loader/assembly/travis/install-hadoop.sh b/hugegraph-loader/assembly/travis/install-hadoop.sh
index e439411c7..54087f9a7 100755
--- a/hugegraph-loader/assembly/travis/install-hadoop.sh
+++ b/hugegraph-loader/assembly/travis/install-hadoop.sh
@@ -15,29 +15,36 @@
# License for the specific language governing permissions and limitations
# under the License.
#
-set -ev
+set -euo pipefail
-# Upgrade stable version to 3.3.6
HADOOP_VERSION="3.3.6"
HADOOP_TARBALL="hadoop-${HADOOP_VERSION}.tar.gz"
-HADOOP_HOME="/usr/local/hadoop"
+HADOOP_HOME=${HADOOP_HOME:-/usr/local/hadoop}
HADOOP_TARBALL_PATH="${HOME}/${HADOOP_TARBALL}"
-
-if [[ ! -d "${HADOOP_HOME}" ]]; then
- if [[ ! -f "${HADOOP_TARBALL_PATH}" ]]; then
- echo "Downloading Hadoop ${HADOOP_VERSION}..."
- wget -O "${HADOOP_TARBALL_PATH}" "https://archive.apache.org/dist/hadoop/common/hadoop-${HADOOP_VERSION}/${HADOOP_TARBALL}"
- else
- echo "Using cached Hadoop tarball at ${HADOOP_TARBALL_PATH}"
- fi
- echo "Extracting Hadoop to ${HADOOP_HOME}..."
- sudo tar -zxf "${HADOOP_TARBALL_PATH}" -C /usr/local
- cd /usr/local
- sudo mv "hadoop-${HADOOP_VERSION}" hadoop
- sudo chown -R "$(whoami):$(whoami)" "${HADOOP_HOME}"
-else
- echo "Hadoop already installed at ${HADOOP_HOME}, skipping download and extraction."
+# Official archive.apache.org Hadoop 3.3.6 .sha512 release checksum.
+HADOOP_SHA512=de3eaca2e0517e4b569a88b63c89fae19cb8ac6c01ff990f1ff8f0cc0f3128c8e8a23db01577ca562a0e0bb1b4a3889f8c74384e609cd55e537aada3dcaa9f8a
+hadoop_verify() {
+ [[ -f "$HADOOP_TARBALL_PATH" ]] &&
+ [[ "$(shasum -a 512 "$HADOOP_TARBALL_PATH" | cut -d ' ' -f 1)" == "$HADOOP_SHA512" ]]
+}
+if ! hadoop_verify; then
+ rm -f "$HADOOP_TARBALL_PATH"
+ for attempt in 1 2; do
+ echo "Downloading Hadoop ${HADOOP_VERSION} (attempt $attempt)..."
+ if curl --fail --location --connect-timeout 30 --max-time 900 \
+ --speed-time 60 --speed-limit 1024 \
+ --output "$HADOOP_TARBALL_PATH" \
+ "https://archive.apache.org/dist/hadoop/common/hadoop-${HADOOP_VERSION}/${HADOOP_TARBALL}" && hadoop_verify; then
+ break
+ fi
+ rm -f "$HADOOP_TARBALL_PATH"
+ done
fi
+hadoop_verify || { echo 'Hadoop archive download/checksum failed' >&2; exit 1; }
+# Extract a verified archive even on cache hits; no unverified installed directory reuse.
+sudo mkdir -p "$HADOOP_HOME"
+sudo tar -zxf "$HADOOP_TARBALL_PATH" --strip-components=1 -C "$HADOOP_HOME"
+sudo chown -R "$(whoami):$(id -gn)" "$HADOOP_HOME"
cd "${HADOOP_HOME}"
pwd
@@ -59,7 +66,9 @@ if ! grep -qxF "export PATH=\$PATH:${HADOOP_HOME}/bin:${HADOOP_HOME}/sbin" ~/.ba
echo "export PATH=\$PATH:${HADOOP_HOME}/bin:${HADOOP_HOME}/sbin" >> ~/.bashrc
fi
-source ~/.bashrc
+export HADOOP_HOME
+export HADOOP_COMMON_LIB_NATIVE_DIR="${HADOOP_HOME}/lib/native"
+export PATH="${PATH}:${HADOOP_HOME}/bin:${HADOOP_HOME}/sbin"
if [[ ! -f etc/hadoop/core-site.xml ]] || ! grep -q "hdfs://localhost:8020" etc/hadoop/core-site.xml; then
sudo tee etc/hadoop/core-site.xml > /dev/null <0 for b in json.load(sys.stdin)["beans"]))'; then
+ exit 0
+ fi
+ sleep 2
+done
+echo 'Hadoop readiness timed out' >&2
+exit 1
diff --git a/hugegraph-loader/assembly/travis/install-hugegraph-from-source.sh b/hugegraph-loader/assembly/travis/install-hugegraph-from-source.sh
index f7d1cb371..bc8f6bad2 100755
--- a/hugegraph-loader/assembly/travis/install-hugegraph-from-source.sh
+++ b/hugegraph-loader/assembly/travis/install-hugegraph-from-source.sh
@@ -21,6 +21,13 @@ if [[ $# -ne 1 ]]; then
echo "Must input an existing commit id of hugegraph server" && exit 1
fi
+if [[ "${FIXTURE_MODE:-standalone}" == start ]]; then
+ FIXTURE_COMMIT=$1
+ ROOT_DIR=$(cd "$(dirname "$0")/../../.." && pwd)
+ source "$ROOT_DIR/.github/actions/setup-hugegraph-server/fixture.sh"
+ ARCHIVE=$(python3 "$ROOT_DIR/.github/actions/setup-hugegraph-server/manifest.py" name)
+ exec bash "$ROOT_DIR/hugegraph-client/assembly/travis/start-hugegraph-servers.sh" "$ARCHIVE"
+fi
COMMIT_ID=$1
SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd)
SHARED_DIR="$SCRIPT_DIR/../../../hugegraph-client/assembly/travis"
diff --git a/hugegraph-loader/assembly/travis/start-hdfs-ci.sh b/hugegraph-loader/assembly/travis/start-hdfs-ci.sh
new file mode 100755
index 000000000..4df703e2f
--- /dev/null
+++ b/hugegraph-loader/assembly/travis/start-hdfs-ci.sh
@@ -0,0 +1,65 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with this
+# work for additional information regarding copyright ownership. The ASF
+# licenses this file to You under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+#
+set -euo pipefail
+
+# ASF's Hadoop 3.3.6 linux/amd64 image; keep the existing HDFS server version.
+image=apache/hadoop:3.3.6@sha256:62a22627f35a1bbf29c721192c02fe311c93749cc43d51df01792b2ac40ff144
+prefix=${HDFS_CONTAINER_PREFIX:-hugegraph-ci-hdfs}
+timeout 600 docker pull --platform linux/amd64 "$image"
+
+# The tests run on the Ubuntu host and connect to localhost:8020. Host networking
+# also makes the DataNode block address reachable without changing test inputs.
+config=(
+ --env CORE-SITE.XML_fs.defaultFS=hdfs://localhost:8020
+ --env HDFS-SITE.XML_dfs.namenode.rpc-address=localhost:8020
+ --env HDFS-SITE.XML_dfs.replication=1
+ --env HDFS-SITE.XML_dfs.permissions.enabled=false
+ --env HDFS-SITE.XML_dfs.datanode.address=127.0.0.1:9866
+)
+docker run --detach --platform linux/amd64 --network host --name "$prefix-namenode" \
+ "${config[@]}" \
+ --env HDFS-SITE.XML_dfs.namenode.name.dir=file:///tmp/hugegraph-hdfs/name \
+ --env ENSURE_NAMENODE_DIR=/tmp/hugegraph-hdfs/name \
+ "$image" hdfs namenode
+docker run --detach --platform linux/amd64 --network host --name "$prefix-datanode" \
+ "${config[@]}" "$image" hdfs datanode
+
+deadline=$((SECONDS + ${SERVICE_READY_TIMEOUT:-180}))
+while (( SECONDS < deadline )); do
+ for service in namenode datanode; do
+ if [[ $(docker inspect --format '{{.State.Running}}' "$prefix-$service") != true ]]; then
+ docker logs "$prefix-$service" >&2
+ echo "HDFS $service exited before readiness" >&2
+ exit 1
+ fi
+ done
+ if curl --fail --silent --connect-timeout 2 --max-time 5 \
+ 'http://localhost:9870/jmx?qry=Hadoop:service=NameNode,name=FSNamesystemState' |
+ python3 -c 'import json,sys; sys.exit(not any(b.get("NumLiveDataNodes",0)>0 for b in json.load(sys.stdin)["beans"]))'; then
+ # Registration alone does not prove the DataNode accepts block writes.
+ timeout 30 docker exec "$prefix-namenode" hdfs dfs -mkdir -p /files
+ printf 'hugegraph-hdfs-ready\n' |
+ timeout 30 docker exec -i "$prefix-namenode" hdfs dfs -put - /files/.ci-ready
+ [[ $(timeout 30 docker exec "$prefix-namenode" hdfs dfs -cat /files/.ci-ready) == hugegraph-hdfs-ready ]]
+ timeout 30 docker exec "$prefix-namenode" hdfs dfs -rm /files/.ci-ready
+ exit 0
+ fi
+ sleep 2
+done
+echo 'HDFS readiness timed out' >&2
+exit 1