Fix various provision failures: more cruft for Oceanbase and Firebolt

firebolt/start

@@ -29,12 +29,21 @@ else
     #                            driver runs before every cold query,
     #                            and across the agent's pre-snapshot
     #                            cycle, or the DB comes back empty).
+    # data/hits_*.parquet in cwd are (on the playground) symlinks into
+    # /opt/clickbench/datasets_ro/hits_partitioned/. Inside the
+    # container the symlink targets need to resolve, so bind-mount
+    # datasets_ro at the same host path. Without this, the parquet
+    # files inside /firebolt-core/clickbench dangle and the INSERT
+    # INTO hits SELECT * FROM hits_external returns 0 rows — the
+    # entire load "succeeds" in <1 s and every query later returns
+    # count(*) = 0.
     sudo docker run -dit --name firebolt-core \
         --ulimit memlock=8589934592:8589934592 \
         --security-opt seccomp=unconfined \
         -p 127.0.0.1:3473:3473 \
         -v "$(pwd)/fb-volume:/var/lib/firebolt" \
         -v "$(pwd)/data:/firebolt-core/clickbench" \
+        -v "/opt/clickbench/datasets_ro:/opt/clickbench/datasets_ro:ro" \
         "${FB_IMAGE:-ghcr.io/firebolt-db/engine:dev}" >/dev/null
 fi
 

oceanbase-row/stop

@@ -7,6 +7,30 @@ source "$OB_HOME/bench.env" 2>/dev/null || true
 # checkpoints, and exits. The pid file is the only handle -- there is no
 # `observer stop` subcommand.
 if [ -f "$OB_HOME/run/observer.pid" ]; then
-    sudo kill -TERM "$(sudo cat "$OB_HOME/run/observer.pid")" 2>/dev/null || true
+    pid=$(sudo cat "$OB_HOME/run/observer.pid" 2>/dev/null || echo)
+    if [ -n "$pid" ]; then
+        sudo kill -TERM "$pid" 2>/dev/null || true
+        # Wait for the observer to actually exit (checkpoints + slog
+        # flush + clog fsync). If we return before it's gone, the
+        # next ./start's observer daemonize will see `pid already
+        # exists` (the observer's start_daemon reads the pid file
+        # and kill(0)-checks the pid — still alive → refuses to
+        # start; exit code 96 → agent /provision fails). Up to 60 s
+        # is enough for a fresh, mostly-empty cluster; escalate to
+        # SIGKILL after that.
+        for _ in $(seq 1 60); do
+            sudo kill -0 "$pid" 2>/dev/null || break
+            sleep 1
+        done
+        if sudo kill -0 "$pid" 2>/dev/null; then
+            sudo kill -KILL "$pid" 2>/dev/null || true
+            sleep 2
+        fi
+    fi
+    # Always drop the stale pid file. Even if the observer already
+    # exited, leaving it there is a footgun: an unrelated pid could
+    # cycle onto the same number, and the next ./start would see
+    # `pid already exists`.
+    sudo rm -f "$OB_HOME/run/observer.pid"
 fi
 exit 0

oceanbase/stop

@@ -7,6 +7,30 @@ source "$OB_HOME/bench.env" 2>/dev/null || true
 # checkpoints, and exits. The pid file is the only handle -- there is no
 # `observer stop` subcommand.
 if [ -f "$OB_HOME/run/observer.pid" ]; then
-    sudo kill -TERM "$(sudo cat "$OB_HOME/run/observer.pid")" 2>/dev/null || true
+    pid=$(sudo cat "$OB_HOME/run/observer.pid" 2>/dev/null || echo)
+    if [ -n "$pid" ]; then
+        sudo kill -TERM "$pid" 2>/dev/null || true
+        # Wait for the observer to actually exit (checkpoints + slog
+        # flush + clog fsync). If we return before it's gone, the
+        # next ./start's observer daemonize will see `pid already
+        # exists` (the observer's start_daemon reads the pid file
+        # and kill(0)-checks the pid — still alive → refuses to
+        # start; exit code 96 → agent /provision fails). Up to 60 s
+        # is enough for a fresh, mostly-empty cluster; escalate to
+        # SIGKILL after that.
+        for _ in $(seq 1 60); do
+            sudo kill -0 "$pid" 2>/dev/null || break
+            sleep 1
+        done
+        if sudo kill -0 "$pid" 2>/dev/null; then
+            sudo kill -KILL "$pid" 2>/dev/null || true
+            sleep 2
+        fi
+    fi
+    # Always drop the stale pid file. Even if the observer already
+    # exited, leaving it there is a footgun: an unrelated pid could
+    # cycle onto the same number, and the next ./start would see
+    # `pid already exists`.
+    sudo rm -f "$OB_HOME/run/observer.pid"
 fi
 exit 0

playground/agent/clickbench-agent.service

@@ -18,6 +18,17 @@ Environment=HOME=/root
 # correct.
 Environment=USER=root
 Environment=LOGNAME=root
+# needrestart's default mode on Ubuntu 24.04 is 'i' (interactive) which
+# in non-tty contexts falls back to 'a' (automatically restart every
+# service whose libs were upgraded). apt install inside install/ can
+# then restart our own clickbench-agent.service — the /provision HTTP
+# connection drops mid-flight and the host sees ServerDisconnectedError.
+# 'l' = list only: needrestart still scans and reports, restarts
+# nothing. Seen on arcticdb and rayforce; will bite any future system
+# whose install path upgrades libpython or another lib the agent
+# links against.
+Environment=NEEDRESTART_MODE=l
+Environment=NEEDRESTART_SUSPEND=1
 ExecStart=/usr/bin/python3 /opt/clickbench-agent/agent.py
 Restart=on-failure
 RestartSec=2

playground/images/build-base-rootfs.sh

@@ -448,6 +448,24 @@ cat > /etc/hosts <<EOF
 127.0.0.1   localhost ubuntu
 ::1         localhost ip6-localhost ip6-loopback
 EOF
+
+# Disable needrestart's automatic service restarts. Ubuntu 24.04's
+# needrestart runs after every apt install/upgrade and, if it sees a
+# service whose libs got upgraded, restarts it — including our own
+# clickbench-agent.service (which links to libpython). Any install
+# script that upgrades python3 or its runtime libs (arcticdb pip
+# install → python3-dev, rayforce apt install → build-essential
+# pulling glibc-locale-source, etc.) triggers this. The mid-provision
+# restart drops the /provision HTTP connection → the host sees
+# ServerDisconnectedError and gives up. Set mode=l (list only) so
+# needrestart still runs and reports, but never restarts anything.
+mkdir -p /etc/needrestart/conf.d
+cat > /etc/needrestart/conf.d/99-clickbench.conf <<EOF
+# Managed by clickbench playground base image build.
+\$nrconf{restart} = 'l';
+\$nrconf{kernelhints} = 0;
+\$nrconf{ucodehints} = 0;
+EOF
 CUSTOMIZE
 sudo chmod +x "$MNT/tmp/customize-rest.sh"
 sudo chroot "$MNT" /tmp/customize-rest.sh

playground/server/systems.py

@@ -132,6 +132,20 @@
     # The docker container has no memory.swap.max set, so the guest
     # kernel will swap it the same as any process.
     "umbra",
+    # arcticdb's python server keeps the whole Arctic Library in
+    # process memory during load — 16 GB RSS on the full hits set
+    # triggers the guest's OOM killer, python3 dies mid-load, the
+    # host sees ServerDisconnectedError (empty provision-log).
+    # Guest kernel dmesg:
+    #   Out of memory: Killed process 1952 (python3)
+    #   total-vm:414791488kB anon-rss:16045040kB
+    "arcticdb",
+    # rayforce also OOMed at ~16 GB RSS on load:
+    #   Out of memory: Killed process 2227 (rayforce) anon-rss:15943296kB
+    # Same class as arcticdb / dataframe engines — in-process load
+    # of the full 100M-row hits set doesn't fit the 16 GiB VM cap
+    # without swap.
+    "rayforce",
 })
 
 # Sparse size of the swap.raw block device handed to NEEDS_SWAP systems.