fix(server): preserve container exit status

docker/README.md

@@ -234,6 +234,18 @@ Standalone stores RocksDB data at `/hugegraph-server/rocksdb-data`. The HStore t
 
 ### Server logs and crash files
 
+The Server container owns its foreground launcher and returns the JVM's exit status after startup,
+including 0 for a normal exit and 137 when the JVM is killed with SIGKILL. TERM and INT request
+shutdown through the launcher using TERM, which shuts down and reaps the JVM; the entrypoint returns
+143 and 130 respectively.
+The startup REST check still uses `HG_SERVER_STARTUP_TIMEOUT_S`, and HStore partition checks still
+run after readiness. Shutdown interrupts a stalled readiness probe or pending partition check and
+reaps its polling processes.
+A startup timeout requests termination and returns 1 without waiting for an
+unresponsive JVM beyond the startup deadline; container termination cleans up remaining processes.
+Graceful shutdown requires the JVM to respond to TERM; forceful OOM and orchestrator shutdown testing
+needs a Linux environment.
+
 This section describes images built from current master. Images from 1.7.0 and earlier do not set `STDOUT_MODE`: `docker logs` shows only the entrypoint's output, and the service logs stay in `logs/` inside the container.
 
 Both Server images (`hugegraph/hugegraph` and `hugegraph/server`) write the HugeGraph log to container stdout. WARN and above from Hadoop, ZooKeeper, SOFA, Netty and Commons also reaches stdout. INFO from Hadoop, Netty and Commons, the audit log and the slow-query log stay in files only; ZooKeeper and SOFA log nothing below WARN. Errors that `hugegraph-server.sh` reports before Java starts, such as an unsupported JDK or too little free memory, and fatal bootstrap errors go to stderr and, when `logs/` can be written, also to `hugegraph-server.log`. Earlier entrypoint steps, such as `init-store.sh` in the HStore image, report to stderr only. Either way, when a Server exits during startup, `docker logs` or `kubectl logs --previous` shows why. When the JVM starts, it also prints `Picked up JAVA_TOOL_OPTIONS: ...` with the crash-file defaults below; a start that fails its preflight checks exits before that.

hugegraph-server/hugegraph-dist/docker/docker-entrypoint-test.sh

@@ -48,6 +48,11 @@ if IFS= read -r password; then
     printf '%s' "${password}" > ./docker/init-store-password
 fi
 EOF
+cat > "${TEST_HOME}/bin/util.sh" <<'EOF'
+# Configuration tests isolate readiness; process lifecycle is tested separately.
+read_property() { sed -n "s/^$2=//p" "$1"; }
+wait_for_startup() { return 0; }
+EOF
 cat > "${TEST_HOME}/bin/enable-auth.sh" <<'EOF'
 #!/usr/bin/env bash
 printf 'called\n' >> ./docker/enable-auth-calls
@@ -325,3 +330,4 @@ done
 assert_start_timeout 120
 
 echo "PASS: Docker entrypoint configures HStore discovery and authentication"
+bash "${SCRIPT_DIR}/test/test-docker-entrypoint-process.sh"

hugegraph-server/hugegraph-dist/docker/docker-entrypoint.sh

@@ -252,19 +252,83 @@ else
     ./bin/init-store.sh
 fi
 
-./bin/start-hugegraph.sh -j "${JAVA_OPTS:-}" -t "${SERVER_STARTUP_TIMEOUT_S}"
+# Own the foreground launcher so wait can recover the JVM's actual status.
+# A daemon JVM is not our child; watching its PID with tail loses that status.
+source ./bin/util.sh
+LAUNCHER_PID=""
+READINESS_PID=""
+PARTITION_PID=""
+stop_server_and_exit() {
+    local exit_code="$1"
+    # dumb-init can signal the group before we forward to its children.
+    # Ignore duplicate signals while those children finish shutting down.
+    trap '' TERM INT
+    if [[ -n "$READINESS_PID" ]]; then
+        kill -TERM "$READINESS_PID" 2>/dev/null || true
+    fi
+    if [[ -n "$PARTITION_PID" ]]; then
+        kill -TERM "$PARTITION_PID" 2>/dev/null || true
+    fi
+    if [[ -n "$LAUNCHER_PID" ]]; then
+        kill -TERM "$LAUNCHER_PID" 2>/dev/null || true
+    fi
+    if [[ -n "$PARTITION_PID" ]]; then
+        wait "$PARTITION_PID" 2>/dev/null || true
+    fi
+    if [[ -n "$READINESS_PID" ]]; then
+        wait "$READINESS_PID" 2>/dev/null || true
+    fi
+    if [[ -n "$LAUNCHER_PID" ]]; then
+        wait "$LAUNCHER_PID" 2>/dev/null || true
+    fi
+    exit "$exit_code"
+}
+trap 'stop_server_and_exit 143' TERM
+trap 'stop_server_and_exit 130' INT
+
+./bin/start-hugegraph.sh -d false -j "${JAVA_OPTS:-}" -t "${SERVER_STARTUP_TIMEOUT_S}" &
+LAUNCHER_PID=$!
+REST_SERVER_URL=$(read_property "$REST_SERVER_CONF" "restserver.url")
+# GNU timeout owns the probe group, including a stalled curl. Background it
+# so wait is interruptible; a foreground probe would defer our signal traps.
+timeout "${SERVER_STARTUP_TIMEOUT_S}s" bash -c '
+    source ./bin/util.sh
+    wait_for_startup "$@"
+' bash "$LAUNCHER_PID" 'HugeGraphServer' "$REST_SERVER_URL/graphs" \
+       "$SERVER_STARTUP_TIMEOUT_S" &
+READINESS_PID=$!
+READY_STATUS=0
+wait "$READINESS_PID" || READY_STATUS=$?
+READINESS_PID=""
+if [[ "$READY_STATUS" -ne 0 ]]; then
+    log "ERROR: HugeGraphServer did not become ready; see the container logs"
+    if kill -0 "$LAUNCHER_PID" 2>/dev/null; then
+        # Keep the startup deadline even if the JVM ignores TERM. Container
+        # termination cleans up any remaining processes; don't wait forever.
+        kill -TERM "$LAUNCHER_PID" 2>/dev/null || true
+        exit 1
+    fi
+    # Preserve an early failure, but a normal exit before readiness is still
+    # an unsuccessful start.
+    STATUS=0
+    wait "$LAUNCHER_PID" || STATUS=$?
+    [[ "$STATUS" -ne 0 ]] || STATUS=1
+    exit "$STATUS"
+fi
 
 # Post-startup cluster stabilization check (hstore only — rocksdb has no partitions)
 ACTUAL_BACKEND=$(grep -E '^[[:space:]]*backend[[:space:]]*=' "${GRAPH_CONF}" | head -n 1 | sed 's/.*=//' | tr -d '[:space:]' || true)
 if [[ "${ACTUAL_BACKEND}" == "hstore" ]]; then
     STORE_REST="${STORE_REST:-store:8520}"
     export STORE_REST
-    ./bin/wait-partition.sh || log "WARN: partitions not assigned yet"
+    ./bin/wait-partition.sh &
+    PARTITION_PID=$!
+    wait "$PARTITION_PID" || log "WARN: partitions not assigned yet"
+    PARTITION_PID=""
 fi
 
-PID=$(cat ./bin/pid 2>/dev/null || true)
-if [[ -n "$PID" ]]; then
-    trap 'kill -TERM "$PID" 2>/dev/null; while kill -0 "$PID" 2>/dev/null; do sleep 1; done; exit 0' TERM INT
-    tail --pid="$PID" -f /dev/null
-    exit 1
+if wait "$LAUNCHER_PID"; then
+    exit 0
+else
+    exit $?
 fi

hugegraph-server/hugegraph-dist/docker/test/test-docker-entrypoint-process.sh

@@ -0,0 +1,228 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+set -euo pipefail
+
+REPO_ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../.." && pwd)
+# Real shell parents and children, with only REST readiness and Java replaced.
+# Stalled-probe cases use one temporary localhost socket; no JVM, Docker or DB.
+python3 - "$REPO_ROOT" <<'PY'
+import os
+import pathlib
+import shutil
+import signal
+import socket
+import subprocess
+import sys
+import tempfile
+import threading
+import time
+
+repo = pathlib.Path(sys.argv[1])
+static = repo / 'hugegraph-server/hugegraph-dist/src/assembly/static/bin'
+entrypoint = repo / 'hugegraph-server/hugegraph-dist/docker/docker-entrypoint.sh'
+
+
+def await_file(path, process):
+    deadline = time.monotonic() + 5
+    while not path.exists():
+        assert process.poll() is None, 'entrypoint exited before ' + str(path)
+        assert time.monotonic() < deadline, 'fixture did not reach ' + str(path)
+        time.sleep(0.02)
+
+
+with tempfile.TemporaryDirectory(prefix='hugegraph-entrypoint-process-') as directory:
+    for mode, expected in [('exit0', 0), ('exit42', 42), ('kill137', 137),
+                           ('TERM', 143), ('INT', 130), ('groupTERM', 143),
+                           ('early42', 42), ('early0', 1), ('timeout', 1),
+                           ('timeout-ignore', 1), ('partition42', 42),
+                           ('partitionTERM', 143), ('partitionINT', 130),
+                           ('partitiongroupTERM', 143), ('readinessTERM', 143),
+                           ('readinessINT', 130), ('readinessgroupTERM', 143)]:
+        work = pathlib.Path(directory) / mode
+        for name in ['bin', 'tools', 'conf/graphs', 'docker', 'logs']:
+            (work / name).mkdir(parents=True)
+        shutil.copy(entrypoint, work / 'entrypoint.sh')
+        for name in ['start-hugegraph.sh', 'util.sh']:
+            shutil.copy(static / name, work / 'bin' / name)
+        (work / 'docker/init_complete').touch()
+        backend = 'hstore' if mode.startswith('partition') else 'rocksdb'
+        (work / 'conf/graphs/hugegraph.properties').write_text('backend=' + backend + '\n')
+        (work / 'conf/rest-server.properties').write_text(
+            'restserver.url=http://127.0.0.1:1\ngremlinserver.url=http://127.0.0.1:2\n')
+        listener = None
+        accepted, stop_http = threading.Event(), threading.Event()
+        if mode.startswith('readiness'):
+            listener = socket.socket()
+            listener.bind(('127.0.0.1', 0))
+            listener.listen(1)
+            port = listener.getsockname()[1]
+            (work / 'conf/rest-server.properties').write_text(
+                'restserver.url=http://127.0.0.1:%d\ngremlinserver.url=http://127.0.0.1:2\n' % port)
+            # The fixture HTTP listener replaces the JVM listener. Skip only
+            # launcher port preflight; readiness still runs the real helper.
+            with (work / 'bin/util.sh').open('a') as utility:
+                utility.write('\ncheck_port() { :; }\n')
+
+            def hold_http():
+                connection, _ = listener.accept()
+                connection.recv(8192)
+                accepted.set()
+                stop_http.wait(10)
+                connection.close()
+
+            threading.Thread(target=hold_http, daemon=True).start()
+        (work / 'bin/init-store.sh').write_text('#!/bin/bash\nexit 0\n')
+        (work / 'bin/wait-partition.sh').write_text(
+            '#!/bin/bash\ntouch ./partition-ran\nexit 7\n')
+        if mode.startswith('partition') and mode != 'partition42':
+            shutil.copy(static / 'wait-partition.sh', work / 'bin/wait-partition.sh')
+        (work / 'tools/curl').write_text('''#!/bin/bash
+case "$TEST_CHILD_MODE" in
+    early* | timeout*) printf '000'; exit 28 ;;
+esac
+if [[ "$TEST_CHILD_MODE" == readiness* ]]; then
+    ancestor=$$
+    for index in 1 2 3 4; do
+        echo "$ancestor" > "./readiness-owned${index}.pid"
+        name=$(ps -o comm= -p "$ancestor")
+        if [[ "${name##*/}" == *timeout ]]; then
+            echo "$ancestor" > ./readiness-timeout.pid
+            break
+        fi
+        ancestor=$(ps -o ppid= -p "$ancestor" | tr -d ' ')
+    done
+    exec "$TEST_REAL_CURL" "$@"
+fi
+if [[ "$1" != '-I' && "$TEST_CHILD_MODE" == partition* ]]; then
+    echo "$PPID" > ./partition-shell.pid
+    timeout_pid=$(ps -o ppid= -p "$PPID" | tr -d ' ')
+    echo "$timeout_pid" > ./partition-timeout.pid
+    ps -o ppid= -p "$timeout_pid" | tr -d ' ' > ./partition-wrapper.pid
+    touch ./partition-ran
+    printf '{"partitionCount":0}'
+    exit 0
+fi
+touch ./ready-probe
+printf '200'
+''')
+        (work / 'bin/hugegraph-server.sh').write_text('''#!/usr/bin/env python3
+import os, pathlib, signal, sys, time
+work = pathlib.Path.cwd()
+mode = os.environ['TEST_CHILD_MODE']
+def stop(signum, frame):
+    signal.signal(signal.SIGTERM, signal.SIG_IGN)
+    (work / 'term-observed').touch()
+    # Expose a duplicate group signal killing a wrapper before it reaps us.
+    time.sleep(0.2)
+    sys.exit(143)
+signal.signal(signal.SIGTERM, signal.SIG_IGN if mode == 'timeout-ignore' else stop)
+(work / 'child-running').touch()
+if mode.startswith('early'):
+    sys.exit(int(mode[5:]))
+while not (work / 'child-exit').exists():
+    time.sleep(0.02)
+sys.exit(int((work / 'child-exit').read_text()))
+''')
+        for folder in ['bin', 'tools']:
+            for script in (work / folder).iterdir():
+                script.chmod(0o755)
+        env = {key: value for key, value in os.environ.items()
+               if not key.startswith('HG_SERVER_') and key not in ['PASSWORD', 'BACKEND', 'PD_PEERS']}
+        env.update(PATH=str(work / 'tools') + ':' + os.environ['PATH'],
+                   TEST_CHILD_MODE=mode, HG_SERVER_STARTUP_TIMEOUT_S='3',
+                   TEST_REAL_CURL=shutil.which('curl'), NO_PROXY='127.0.0.1', no_proxy='127.0.0.1')
+        if mode.startswith('readiness'):
+            env['HG_SERVER_STARTUP_TIMEOUT_S'] = '60'
+        started = time.monotonic()
+        with (work / 'output').open('wb') as output:
+            process = subprocess.Popen(['bash', './entrypoint.sh'], cwd=work, env=env,
+                                       stdout=output, stderr=subprocess.STDOUT,
+                                       start_new_session=True)
+            try:
+                if mode.startswith('timeout'):
+                    await_file(work / 'child-running', process)
+                if mode.startswith('readiness'):
+                    await_file(work / 'child-running', process)
+                    assert accepted.wait(5), 'real curl did not reach stalled HTTP fixture'
+                    if mode == 'readinessgroupTERM':
+                        os.killpg(process.pid, signal.SIGTERM)
+                    else:
+                        signum = signal.SIGINT if mode.endswith('INT') else signal.SIGTERM
+                        os.kill(process.pid, signum)
+                elif not mode.startswith(('early', 'timeout')):
+                    await_file(work / 'child-running', process)
+                    await_file(work / 'ready-probe', process)
+                    if mode.startswith('partition'):
+                        await_file(work / 'partition-ran', process)
+                    if mode in ['TERM', 'INT', 'partitionTERM', 'partitionINT']:
+                        signum = signal.SIGINT if mode.endswith('INT') else signal.SIGTERM
+                        os.kill(process.pid, signum)
+                    elif mode in ['groupTERM', 'partitiongroupTERM']:
+                        os.killpg(process.pid, signal.SIGTERM)
+                    elif mode == 'kill137':
+                        os.kill(int((work / 'bin/pid').read_text()), signal.SIGKILL)
+                    else:
+                        status = 42 if mode == 'partition42' else int(mode[4:])
+                        (work / 'child-exit').write_text(str(status))
+                actual = process.wait(timeout=5)
+                assert actual == expected, (mode, expected, actual, (work / 'output').read_text())
+                if mode.startswith('timeout'):
+                    assert time.monotonic() - started < 5, 'startup cleanup exceeded the deadline'
+                if mode in ['TERM', 'INT', 'groupTERM', 'partitionTERM', 'partitionINT',
+                            'partitiongroupTERM', 'readinessTERM', 'readinessINT',
+                            'readinessgroupTERM']:
+                    assert (work / 'term-observed').exists(), 'TERM did not reach the JVM substitute'
+                if not mode.startswith('timeout'):
+                    owned = ([work / 'bin/pid'] + list(work.glob('partition-*.pid')) +
+                             list(work.glob('readiness-*.pid')))
+                    for pid_file in owned:
+                        child = int(pid_file.read_text())
+                        try:
+                            os.kill(child, 0)
+                        except ProcessLookupError:
+                            pass
+                        else:
+                            raise AssertionError(str(pid_file) + ' survived entrypoint exit')
+                if mode == 'partition42':
+                    assert 'WARN: partitions not assigned yet' in (work / 'output').read_text()
+                print('PASS: ' + mode + ' -> ' + str(actual))
+            finally:
+                # Timeout intentionally permits an unresponsive process until
+                # container termination. The fixture owns and kills that group.
+                try:
+                    os.killpg(process.pid, signal.SIGKILL)
+                except ProcessLookupError:
+                    pass
+                # GNU timeout owns a separate process group. Clean it too if
+                # a regression prevents the wrapper from forwarding shutdown.
+                for pid_file in [work / 'partition-timeout.pid', work / 'readiness-timeout.pid']:
+                    if pid_file.exists():
+                        try:
+                            os.killpg(int(pid_file.read_text()), signal.SIGKILL)
+                        except ProcessLookupError:
+                            pass
+                for pid_file in list(work.glob('partition-*.pid')) + list(work.glob('readiness-*.pid')):
+                    try:
+                        os.kill(int(pid_file.read_text()), signal.SIGKILL)
+                    except ProcessLookupError:
+                        pass
+                process.wait(timeout=2)
+                if listener is not None:
+                    stop_http.set()
+                    listener.close()
+PY

hugegraph-server/hugegraph-dist/src/assembly/static/bin/start-hugegraph.sh

@@ -103,7 +103,9 @@ function forward_signal_and_wait() {
     local signal="$1"
     local exit_code="$2"
 
-    trap - SIGHUP SIGINT SIGQUIT SIGTERM
+    # A process-group signal and the entrypoint's forwarding can arrive twice.
+    # Keep the wrapper alive until it has reaped its child in either case.
+    trap '' SIGHUP SIGINT SIGQUIT SIGTERM
     if kill -0 "$PID" 2>/dev/null; then
         kill "-$signal" "$PID" 2>/dev/null || true
         # The foreground wait is interrupted before the trap runs, so retry