From 56b8f6e7a101c0fcd97151c94a05e9ab33645939 Mon Sep 17 00:00:00 2001 From: imbajin Date: Mon, 12 Oct 2026 05:26:50 +0800 Subject: [PATCH] fix(server): preserve container exit status Own and wait for the foreground launcher. Interrupt and reap readiness and partition checks. Preserve cooperative shutdown across duplicate signals. Cover exit status and pending-probe lifecycle contracts. Document startup deadline and signal behavior. --- docker/README.md | 12 + .../docker/docker-entrypoint-test.sh | 6 + .../docker/docker-entrypoint.sh | 78 +++++- .../test/test-docker-entrypoint-process.sh | 228 ++++++++++++++++++ .../assembly/static/bin/start-hugegraph.sh | 4 +- .../src/assembly/static/bin/wait-partition.sh | 28 ++- 6 files changed, 346 insertions(+), 10 deletions(-) create mode 100644 hugegraph-server/hugegraph-dist/docker/test/test-docker-entrypoint-process.sh diff --git a/docker/README.md b/docker/README.md index e6dc6dedc6..b4f1cdf7ef 100644 --- a/docker/README.md +++ b/docker/README.md @@ -234,6 +234,18 @@ Standalone stores RocksDB data at `/hugegraph-server/rocksdb-data`. The HStore t ### Server logs and crash files +The Server container owns its foreground launcher and returns the JVM's exit status after startup, +including 0 for a normal exit and 137 when the JVM is killed with SIGKILL. TERM and INT request +shutdown through the launcher using TERM, which shuts down and reaps the JVM; the entrypoint returns +143 and 130 respectively. +The startup REST check still uses `HG_SERVER_STARTUP_TIMEOUT_S`, and HStore partition checks still +run after readiness. Shutdown interrupts a stalled readiness probe or pending partition check and +reaps its polling processes. +A startup timeout requests termination and returns 1 without waiting for an +unresponsive JVM beyond the startup deadline; container termination cleans up remaining processes. +Graceful shutdown requires the JVM to respond to TERM; forceful OOM and orchestrator shutdown testing +needs a Linux environment. + This section describes images built from current master. Images from 1.7.0 and earlier do not set `STDOUT_MODE`: `docker logs` shows only the entrypoint's output, and the service logs stay in `logs/` inside the container. Both Server images (`hugegraph/hugegraph` and `hugegraph/server`) write the HugeGraph log to container stdout. WARN and above from Hadoop, ZooKeeper, SOFA, Netty and Commons also reaches stdout. INFO from Hadoop, Netty and Commons, the audit log and the slow-query log stay in files only; ZooKeeper and SOFA log nothing below WARN. Errors that `hugegraph-server.sh` reports before Java starts, such as an unsupported JDK or too little free memory, and fatal bootstrap errors go to stderr and, when `logs/` can be written, also to `hugegraph-server.log`. Earlier entrypoint steps, such as `init-store.sh` in the HStore image, report to stderr only. Either way, when a Server exits during startup, `docker logs` or `kubectl logs --previous` shows why. When the JVM starts, it also prints `Picked up JAVA_TOOL_OPTIONS: ...` with the crash-file defaults below; a start that fails its preflight checks exits before that. diff --git a/hugegraph-server/hugegraph-dist/docker/docker-entrypoint-test.sh b/hugegraph-server/hugegraph-dist/docker/docker-entrypoint-test.sh index 42137dc3b5..e6823e897e 100755 --- a/hugegraph-server/hugegraph-dist/docker/docker-entrypoint-test.sh +++ b/hugegraph-server/hugegraph-dist/docker/docker-entrypoint-test.sh @@ -48,6 +48,11 @@ if IFS= read -r password; then printf '%s' "${password}" > ./docker/init-store-password fi EOF +cat > "${TEST_HOME}/bin/util.sh" <<'EOF' +# Configuration tests isolate readiness; process lifecycle is tested separately. +read_property() { sed -n "s/^$2=//p" "$1"; } +wait_for_startup() { return 0; } +EOF cat > "${TEST_HOME}/bin/enable-auth.sh" <<'EOF' #!/usr/bin/env bash printf 'called\n' >> ./docker/enable-auth-calls @@ -325,3 +330,4 @@ done assert_start_timeout 120 echo "PASS: Docker entrypoint configures HStore discovery and authentication" +bash "${SCRIPT_DIR}/test/test-docker-entrypoint-process.sh" diff --git a/hugegraph-server/hugegraph-dist/docker/docker-entrypoint.sh b/hugegraph-server/hugegraph-dist/docker/docker-entrypoint.sh index 2be225a04e..0e3cc4eab2 100755 --- a/hugegraph-server/hugegraph-dist/docker/docker-entrypoint.sh +++ b/hugegraph-server/hugegraph-dist/docker/docker-entrypoint.sh @@ -252,19 +252,83 @@ else ./bin/init-store.sh fi -./bin/start-hugegraph.sh -j "${JAVA_OPTS:-}" -t "${SERVER_STARTUP_TIMEOUT_S}" +# Own the foreground launcher so wait can recover the JVM's actual status. +# A daemon JVM is not our child; watching its PID with tail loses that status. +source ./bin/util.sh +LAUNCHER_PID="" +READINESS_PID="" +PARTITION_PID="" +stop_server_and_exit() { + local exit_code="$1" + # dumb-init can signal the group before we forward to its children. + # Ignore duplicate signals while those children finish shutting down. + trap '' TERM INT + if [[ -n "$READINESS_PID" ]]; then + kill -TERM "$READINESS_PID" 2>/dev/null || true + fi + if [[ -n "$PARTITION_PID" ]]; then + kill -TERM "$PARTITION_PID" 2>/dev/null || true + fi + if [[ -n "$LAUNCHER_PID" ]]; then + kill -TERM "$LAUNCHER_PID" 2>/dev/null || true + fi + if [[ -n "$PARTITION_PID" ]]; then + wait "$PARTITION_PID" 2>/dev/null || true + fi + if [[ -n "$READINESS_PID" ]]; then + wait "$READINESS_PID" 2>/dev/null || true + fi + if [[ -n "$LAUNCHER_PID" ]]; then + wait "$LAUNCHER_PID" 2>/dev/null || true + fi + exit "$exit_code" +} +trap 'stop_server_and_exit 143' TERM +trap 'stop_server_and_exit 130' INT + +./bin/start-hugegraph.sh -d false -j "${JAVA_OPTS:-}" -t "${SERVER_STARTUP_TIMEOUT_S}" & +LAUNCHER_PID=$! +REST_SERVER_URL=$(read_property "$REST_SERVER_CONF" "restserver.url") +# GNU timeout owns the probe group, including a stalled curl. Background it +# so wait is interruptible; a foreground probe would defer our signal traps. +timeout "${SERVER_STARTUP_TIMEOUT_S}s" bash -c ' + source ./bin/util.sh + wait_for_startup "$@" +' bash "$LAUNCHER_PID" 'HugeGraphServer' "$REST_SERVER_URL/graphs" \ + "$SERVER_STARTUP_TIMEOUT_S" & +READINESS_PID=$! +READY_STATUS=0 +wait "$READINESS_PID" || READY_STATUS=$? +READINESS_PID="" +if [[ "$READY_STATUS" -ne 0 ]]; then + log "ERROR: HugeGraphServer did not become ready; see the container logs" + if kill -0 "$LAUNCHER_PID" 2>/dev/null; then + # Keep the startup deadline even if the JVM ignores TERM. Container + # termination cleans up any remaining processes; don't wait forever. + kill -TERM "$LAUNCHER_PID" 2>/dev/null || true + exit 1 + fi + # Preserve an early failure, but a normal exit before readiness is still + # an unsuccessful start. + STATUS=0 + wait "$LAUNCHER_PID" || STATUS=$? + [[ "$STATUS" -ne 0 ]] || STATUS=1 + exit "$STATUS" +fi # Post-startup cluster stabilization check (hstore only — rocksdb has no partitions) ACTUAL_BACKEND=$(grep -E '^[[:space:]]*backend[[:space:]]*=' "${GRAPH_CONF}" | head -n 1 | sed 's/.*=//' | tr -d '[:space:]' || true) if [[ "${ACTUAL_BACKEND}" == "hstore" ]]; then STORE_REST="${STORE_REST:-store:8520}" export STORE_REST - ./bin/wait-partition.sh || log "WARN: partitions not assigned yet" + ./bin/wait-partition.sh & + PARTITION_PID=$! + wait "$PARTITION_PID" || log "WARN: partitions not assigned yet" + PARTITION_PID="" fi -PID=$(cat ./bin/pid 2>/dev/null || true) -if [[ -n "$PID" ]]; then - trap 'kill -TERM "$PID" 2>/dev/null; while kill -0 "$PID" 2>/dev/null; do sleep 1; done; exit 0' TERM INT - tail --pid="$PID" -f /dev/null - exit 1 +if wait "$LAUNCHER_PID"; then + exit 0 +else + exit $? fi diff --git a/hugegraph-server/hugegraph-dist/docker/test/test-docker-entrypoint-process.sh b/hugegraph-server/hugegraph-dist/docker/test/test-docker-entrypoint-process.sh new file mode 100644 index 0000000000..7dcb7b57c7 --- /dev/null +++ b/hugegraph-server/hugegraph-dist/docker/test/test-docker-entrypoint-process.sh @@ -0,0 +1,228 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +set -euo pipefail + +REPO_ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../.." && pwd) +# Real shell parents and children, with only REST readiness and Java replaced. +# Stalled-probe cases use one temporary localhost socket; no JVM, Docker or DB. +python3 - "$REPO_ROOT" <<'PY' +import os +import pathlib +import shutil +import signal +import socket +import subprocess +import sys +import tempfile +import threading +import time + +repo = pathlib.Path(sys.argv[1]) +static = repo / 'hugegraph-server/hugegraph-dist/src/assembly/static/bin' +entrypoint = repo / 'hugegraph-server/hugegraph-dist/docker/docker-entrypoint.sh' + + +def await_file(path, process): + deadline = time.monotonic() + 5 + while not path.exists(): + assert process.poll() is None, 'entrypoint exited before ' + str(path) + assert time.monotonic() < deadline, 'fixture did not reach ' + str(path) + time.sleep(0.02) + + +with tempfile.TemporaryDirectory(prefix='hugegraph-entrypoint-process-') as directory: + for mode, expected in [('exit0', 0), ('exit42', 42), ('kill137', 137), + ('TERM', 143), ('INT', 130), ('groupTERM', 143), + ('early42', 42), ('early0', 1), ('timeout', 1), + ('timeout-ignore', 1), ('partition42', 42), + ('partitionTERM', 143), ('partitionINT', 130), + ('partitiongroupTERM', 143), ('readinessTERM', 143), + ('readinessINT', 130), ('readinessgroupTERM', 143)]: + work = pathlib.Path(directory) / mode + for name in ['bin', 'tools', 'conf/graphs', 'docker', 'logs']: + (work / name).mkdir(parents=True) + shutil.copy(entrypoint, work / 'entrypoint.sh') + for name in ['start-hugegraph.sh', 'util.sh']: + shutil.copy(static / name, work / 'bin' / name) + (work / 'docker/init_complete').touch() + backend = 'hstore' if mode.startswith('partition') else 'rocksdb' + (work / 'conf/graphs/hugegraph.properties').write_text('backend=' + backend + '\n') + (work / 'conf/rest-server.properties').write_text( + 'restserver.url=http://127.0.0.1:1\ngremlinserver.url=http://127.0.0.1:2\n') + listener = None + accepted, stop_http = threading.Event(), threading.Event() + if mode.startswith('readiness'): + listener = socket.socket() + listener.bind(('127.0.0.1', 0)) + listener.listen(1) + port = listener.getsockname()[1] + (work / 'conf/rest-server.properties').write_text( + 'restserver.url=http://127.0.0.1:%d\ngremlinserver.url=http://127.0.0.1:2\n' % port) + # The fixture HTTP listener replaces the JVM listener. Skip only + # launcher port preflight; readiness still runs the real helper. + with (work / 'bin/util.sh').open('a') as utility: + utility.write('\ncheck_port() { :; }\n') + + def hold_http(): + connection, _ = listener.accept() + connection.recv(8192) + accepted.set() + stop_http.wait(10) + connection.close() + + threading.Thread(target=hold_http, daemon=True).start() + (work / 'bin/init-store.sh').write_text('#!/bin/bash\nexit 0\n') + (work / 'bin/wait-partition.sh').write_text( + '#!/bin/bash\ntouch ./partition-ran\nexit 7\n') + if mode.startswith('partition') and mode != 'partition42': + shutil.copy(static / 'wait-partition.sh', work / 'bin/wait-partition.sh') + (work / 'tools/curl').write_text('''#!/bin/bash +case "$TEST_CHILD_MODE" in + early* | timeout*) printf '000'; exit 28 ;; +esac +if [[ "$TEST_CHILD_MODE" == readiness* ]]; then + ancestor=$$ + for index in 1 2 3 4; do + echo "$ancestor" > "./readiness-owned${index}.pid" + name=$(ps -o comm= -p "$ancestor") + if [[ "${name##*/}" == *timeout ]]; then + echo "$ancestor" > ./readiness-timeout.pid + break + fi + ancestor=$(ps -o ppid= -p "$ancestor" | tr -d ' ') + done + exec "$TEST_REAL_CURL" "$@" +fi +if [[ "$1" != '-I' && "$TEST_CHILD_MODE" == partition* ]]; then + echo "$PPID" > ./partition-shell.pid + timeout_pid=$(ps -o ppid= -p "$PPID" | tr -d ' ') + echo "$timeout_pid" > ./partition-timeout.pid + ps -o ppid= -p "$timeout_pid" | tr -d ' ' > ./partition-wrapper.pid + touch ./partition-ran + printf '{"partitionCount":0}' + exit 0 +fi +touch ./ready-probe +printf '200' +''') + (work / 'bin/hugegraph-server.sh').write_text('''#!/usr/bin/env python3 +import os, pathlib, signal, sys, time +work = pathlib.Path.cwd() +mode = os.environ['TEST_CHILD_MODE'] +def stop(signum, frame): + signal.signal(signal.SIGTERM, signal.SIG_IGN) + (work / 'term-observed').touch() + # Expose a duplicate group signal killing a wrapper before it reaps us. + time.sleep(0.2) + sys.exit(143) +signal.signal(signal.SIGTERM, signal.SIG_IGN if mode == 'timeout-ignore' else stop) +(work / 'child-running').touch() +if mode.startswith('early'): + sys.exit(int(mode[5:])) +while not (work / 'child-exit').exists(): + time.sleep(0.02) +sys.exit(int((work / 'child-exit').read_text())) +''') + for folder in ['bin', 'tools']: + for script in (work / folder).iterdir(): + script.chmod(0o755) + env = {key: value for key, value in os.environ.items() + if not key.startswith('HG_SERVER_') and key not in ['PASSWORD', 'BACKEND', 'PD_PEERS']} + env.update(PATH=str(work / 'tools') + ':' + os.environ['PATH'], + TEST_CHILD_MODE=mode, HG_SERVER_STARTUP_TIMEOUT_S='3', + TEST_REAL_CURL=shutil.which('curl'), NO_PROXY='127.0.0.1', no_proxy='127.0.0.1') + if mode.startswith('readiness'): + env['HG_SERVER_STARTUP_TIMEOUT_S'] = '60' + started = time.monotonic() + with (work / 'output').open('wb') as output: + process = subprocess.Popen(['bash', './entrypoint.sh'], cwd=work, env=env, + stdout=output, stderr=subprocess.STDOUT, + start_new_session=True) + try: + if mode.startswith('timeout'): + await_file(work / 'child-running', process) + if mode.startswith('readiness'): + await_file(work / 'child-running', process) + assert accepted.wait(5), 'real curl did not reach stalled HTTP fixture' + if mode == 'readinessgroupTERM': + os.killpg(process.pid, signal.SIGTERM) + else: + signum = signal.SIGINT if mode.endswith('INT') else signal.SIGTERM + os.kill(process.pid, signum) + elif not mode.startswith(('early', 'timeout')): + await_file(work / 'child-running', process) + await_file(work / 'ready-probe', process) + if mode.startswith('partition'): + await_file(work / 'partition-ran', process) + if mode in ['TERM', 'INT', 'partitionTERM', 'partitionINT']: + signum = signal.SIGINT if mode.endswith('INT') else signal.SIGTERM + os.kill(process.pid, signum) + elif mode in ['groupTERM', 'partitiongroupTERM']: + os.killpg(process.pid, signal.SIGTERM) + elif mode == 'kill137': + os.kill(int((work / 'bin/pid').read_text()), signal.SIGKILL) + else: + status = 42 if mode == 'partition42' else int(mode[4:]) + (work / 'child-exit').write_text(str(status)) + actual = process.wait(timeout=5) + assert actual == expected, (mode, expected, actual, (work / 'output').read_text()) + if mode.startswith('timeout'): + assert time.monotonic() - started < 5, 'startup cleanup exceeded the deadline' + if mode in ['TERM', 'INT', 'groupTERM', 'partitionTERM', 'partitionINT', + 'partitiongroupTERM', 'readinessTERM', 'readinessINT', + 'readinessgroupTERM']: + assert (work / 'term-observed').exists(), 'TERM did not reach the JVM substitute' + if not mode.startswith('timeout'): + owned = ([work / 'bin/pid'] + list(work.glob('partition-*.pid')) + + list(work.glob('readiness-*.pid'))) + for pid_file in owned: + child = int(pid_file.read_text()) + try: + os.kill(child, 0) + except ProcessLookupError: + pass + else: + raise AssertionError(str(pid_file) + ' survived entrypoint exit') + if mode == 'partition42': + assert 'WARN: partitions not assigned yet' in (work / 'output').read_text() + print('PASS: ' + mode + ' -> ' + str(actual)) + finally: + # Timeout intentionally permits an unresponsive process until + # container termination. The fixture owns and kills that group. + try: + os.killpg(process.pid, signal.SIGKILL) + except ProcessLookupError: + pass + # GNU timeout owns a separate process group. Clean it too if + # a regression prevents the wrapper from forwarding shutdown. + for pid_file in [work / 'partition-timeout.pid', work / 'readiness-timeout.pid']: + if pid_file.exists(): + try: + os.killpg(int(pid_file.read_text()), signal.SIGKILL) + except ProcessLookupError: + pass + for pid_file in list(work.glob('partition-*.pid')) + list(work.glob('readiness-*.pid')): + try: + os.kill(int(pid_file.read_text()), signal.SIGKILL) + except ProcessLookupError: + pass + process.wait(timeout=2) + if listener is not None: + stop_http.set() + listener.close() +PY diff --git a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/start-hugegraph.sh b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/start-hugegraph.sh index 8a95059c43..28f36eb427 100644 --- a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/start-hugegraph.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/start-hugegraph.sh @@ -103,7 +103,9 @@ function forward_signal_and_wait() { local signal="$1" local exit_code="$2" - trap - SIGHUP SIGINT SIGQUIT SIGTERM + # A process-group signal and the entrypoint's forwarding can arrive twice. + # Keep the wrapper alive until it has reaped its child in either case. + trap '' SIGHUP SIGINT SIGQUIT SIGTERM if kill -0 "$PID" 2>/dev/null; then kill "-$signal" "$PID" 2>/dev/null || true # The foreground wait is interrupted before the trap runs, so retry diff --git a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/wait-partition.sh b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/wait-partition.sh index d14bc90244..31b2ce0cb0 100755 --- a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/wait-partition.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/wait-partition.sh @@ -19,6 +19,21 @@ set -euo pipefail : "${STORE_REST:?STORE_REST not set}" +CHECK_PID="" +stop_check() { + local exit_code="$1" + trap '' TERM INT + if [[ -n "$CHECK_PID" ]]; then + # GNU timeout forwards TERM to the command group it owns, including + # the polling shell and its current curl/sleep, then reaps the shell. + kill -TERM "$CHECK_PID" 2>/dev/null || true + wait "$CHECK_PID" 2>/dev/null || true + fi + exit "$exit_code" +} +trap 'stop_check 143' TERM +trap 'stop_check 130' INT + timeout "${WAIT_PARTITION_TIMEOUT_S:-120}s" bash -c ' until curl -fsS "http://${STORE_REST}" 2>/dev/null | \ grep -q "\"partitionCount\":[1-9]" @@ -26,9 +41,18 @@ do echo "Waiting for partition assignment..." sleep 5 done -' +' & +CHECK_PID=$! +STATUS=0 +wait "$CHECK_PID" || STATUS=$? +CHECK_PID="" +[[ "$STATUS" -eq 0 ]] || exit "$STATUS" echo "Partitions detected:" URL="http://${STORE_REST}/v1/partitions" echo "$URL" -curl -v "$URL" +curl -v "$URL" & +CHECK_PID=$! +STATUS=0 +wait "$CHECK_PID" || STATUS=$? +exit "$STATUS"