diff --git a/firebolt/start b/firebolt/start index 90fe378381..c785658be6 100755 --- a/firebolt/start +++ b/firebolt/start @@ -29,12 +29,21 @@ else # driver runs before every cold query, # and across the agent's pre-snapshot # cycle, or the DB comes back empty). + # data/hits_*.parquet in cwd are (on the playground) symlinks into + # /opt/clickbench/datasets_ro/hits_partitioned/. Inside the + # container the symlink targets need to resolve, so bind-mount + # datasets_ro at the same host path. Without this, the parquet + # files inside /firebolt-core/clickbench dangle and the INSERT + # INTO hits SELECT * FROM hits_external returns 0 rows — the + # entire load "succeeds" in <1 s and every query later returns + # count(*) = 0. sudo docker run -dit --name firebolt-core \ --ulimit memlock=8589934592:8589934592 \ --security-opt seccomp=unconfined \ -p 127.0.0.1:3473:3473 \ -v "$(pwd)/fb-volume:/var/lib/firebolt" \ -v "$(pwd)/data:/firebolt-core/clickbench" \ + -v "/opt/clickbench/datasets_ro:/opt/clickbench/datasets_ro:ro" \ "${FB_IMAGE:-ghcr.io/firebolt-db/engine:dev}" >/dev/null fi diff --git a/oceanbase-row/stop b/oceanbase-row/stop index 6b3332d598..69df6192f2 100755 --- a/oceanbase-row/stop +++ b/oceanbase-row/stop @@ -7,6 +7,30 @@ source "$OB_HOME/bench.env" 2>/dev/null || true # checkpoints, and exits. The pid file is the only handle -- there is no # `observer stop` subcommand. if [ -f "$OB_HOME/run/observer.pid" ]; then - sudo kill -TERM "$(sudo cat "$OB_HOME/run/observer.pid")" 2>/dev/null || true + pid=$(sudo cat "$OB_HOME/run/observer.pid" 2>/dev/null || echo) + if [ -n "$pid" ]; then + sudo kill -TERM "$pid" 2>/dev/null || true + # Wait for the observer to actually exit (checkpoints + slog + # flush + clog fsync). If we return before it's gone, the + # next ./start's observer daemonize will see `pid already + # exists` (the observer's start_daemon reads the pid file + # and kill(0)-checks the pid — still alive → refuses to + # start; exit code 96 → agent /provision fails). Up to 60 s + # is enough for a fresh, mostly-empty cluster; escalate to + # SIGKILL after that. + for _ in $(seq 1 60); do + sudo kill -0 "$pid" 2>/dev/null || break + sleep 1 + done + if sudo kill -0 "$pid" 2>/dev/null; then + sudo kill -KILL "$pid" 2>/dev/null || true + sleep 2 + fi + fi + # Always drop the stale pid file. Even if the observer already + # exited, leaving it there is a footgun: an unrelated pid could + # cycle onto the same number, and the next ./start would see + # `pid already exists`. + sudo rm -f "$OB_HOME/run/observer.pid" fi exit 0 diff --git a/oceanbase/stop b/oceanbase/stop index 6b3332d598..69df6192f2 100755 --- a/oceanbase/stop +++ b/oceanbase/stop @@ -7,6 +7,30 @@ source "$OB_HOME/bench.env" 2>/dev/null || true # checkpoints, and exits. The pid file is the only handle -- there is no # `observer stop` subcommand. if [ -f "$OB_HOME/run/observer.pid" ]; then - sudo kill -TERM "$(sudo cat "$OB_HOME/run/observer.pid")" 2>/dev/null || true + pid=$(sudo cat "$OB_HOME/run/observer.pid" 2>/dev/null || echo) + if [ -n "$pid" ]; then + sudo kill -TERM "$pid" 2>/dev/null || true + # Wait for the observer to actually exit (checkpoints + slog + # flush + clog fsync). If we return before it's gone, the + # next ./start's observer daemonize will see `pid already + # exists` (the observer's start_daemon reads the pid file + # and kill(0)-checks the pid — still alive → refuses to + # start; exit code 96 → agent /provision fails). Up to 60 s + # is enough for a fresh, mostly-empty cluster; escalate to + # SIGKILL after that. + for _ in $(seq 1 60); do + sudo kill -0 "$pid" 2>/dev/null || break + sleep 1 + done + if sudo kill -0 "$pid" 2>/dev/null; then + sudo kill -KILL "$pid" 2>/dev/null || true + sleep 2 + fi + fi + # Always drop the stale pid file. Even if the observer already + # exited, leaving it there is a footgun: an unrelated pid could + # cycle onto the same number, and the next ./start would see + # `pid already exists`. + sudo rm -f "$OB_HOME/run/observer.pid" fi exit 0 diff --git a/playground/agent/clickbench-agent.service b/playground/agent/clickbench-agent.service index 067b1cfdc8..967f550e64 100644 --- a/playground/agent/clickbench-agent.service +++ b/playground/agent/clickbench-agent.service @@ -18,6 +18,17 @@ Environment=HOME=/root # correct. Environment=USER=root Environment=LOGNAME=root +# needrestart's default mode on Ubuntu 24.04 is 'i' (interactive) which +# in non-tty contexts falls back to 'a' (automatically restart every +# service whose libs were upgraded). apt install inside install/ can +# then restart our own clickbench-agent.service — the /provision HTTP +# connection drops mid-flight and the host sees ServerDisconnectedError. +# 'l' = list only: needrestart still scans and reports, restarts +# nothing. Seen on arcticdb and rayforce; will bite any future system +# whose install path upgrades libpython or another lib the agent +# links against. +Environment=NEEDRESTART_MODE=l +Environment=NEEDRESTART_SUSPEND=1 ExecStart=/usr/bin/python3 /opt/clickbench-agent/agent.py Restart=on-failure RestartSec=2 diff --git a/playground/images/build-base-rootfs.sh b/playground/images/build-base-rootfs.sh index 03f3a05669..b5c0c10c73 100755 --- a/playground/images/build-base-rootfs.sh +++ b/playground/images/build-base-rootfs.sh @@ -448,6 +448,24 @@ cat > /etc/hosts < /etc/needrestart/conf.d/99-clickbench.conf <