From 51b48f110437324866b550fd10b72a3c6333a78f Mon Sep 17 00:00:00 2001
From: Anton Standrik <astandrik@yandex-team.ru>
Date: Sun, 13 Sep 2026 13:52:16 +0300
Subject: [PATCH 01/10] fix(test): isolate benchmark harness daemon runtimes

scripts/benchmark-index.sh and scripts/benchmark-search-graph.sh ran the
product with no runtime or cache of their own: the benchmark repository
was indexed into the operator's live store and every one-shot joined the
operator's account daemon. Source scripts/test-runtime.sh in both, start
the private daemon before timing, record setup-time.txt and
total-time.txt beside index-time.txt, and index from a repository path
instead of querying a project in the live store.

Add tests/test_benchmark_runtime_isolation_contract.sh, which fails
before this change, and wire it into scripts/test.sh.

Part of #1696.

Signed-off-by: Anton Standrik <astandrik@yandex-team.ru>
---
 scripts/benchmark-index.sh                    | 20 ++++
 scripts/benchmark-search-graph.sh             | 38 +++++++-
 scripts/test.sh                               |  3 +
 ...st_benchmark_runtime_isolation_contract.sh | 96 +++++++++++++++++++
 4 files changed, 152 insertions(+), 5 deletions(-)
 create mode 100755 tests/test_benchmark_runtime_isolation_contract.sh

diff --git a/scripts/benchmark-index.sh b/scripts/benchmark-index.sh
index 756bda06ed..71e694c171 100755
--- a/scripts/benchmark-index.sh
+++ b/scripts/benchmark-index.sh
@@ -9,6 +9,14 @@ LANG="${2:?}"
 REPO="${3:?}"
 RESULTS_DIR="${4:?}"
 
+# The index must run against a daemon rendezvous and cache this run owns: only
+# CBM_RUNTIME_DIR moves the rendezvous, and without a private cache the
+# benchmark repository was indexed into the operator's live store (#1696).
+# shellcheck source=test-runtime.sh
+source "$(dirname "${BASH_SOURCE[0]}")/test-runtime.sh"
+cbm_test_runtime_init
+trap 'cbm_test_runtime_cleanup "$BINARY"' EXIT
+
 # Resolve symlinks
 REPO=$(cd "$REPO" && pwd -P)
 
@@ -33,6 +41,16 @@ LOC=$(find "$REPO" -type f \
 echo "$FILE_COUNT" > "$OUT/file-count.txt"
 echo "$LOC" > "$OUT/loc.txt"
 
+# Start the private daemon before timing so index-time.txt measures the index
+# alone. setup-time.txt keeps the activation cost attributable and
+# total-time.txt is their sum — the figure comparable with earlier runs, which
+# paid activation inside the index timing whenever no daemon was already warm.
+SETUP_START_MS=$(python3 -c "import time; print(int(time.time()*1000))")
+if ! "$BINARY" daemon start >/dev/null 2>&1; then
+  echo "  $LANG: private daemon did not start" >&2
+  exit 1
+fi
+
 # Index via CLI and capture timing
 START_MS=$(python3 -c "import time; print(int(time.time()*1000))")
 
@@ -43,6 +61,8 @@ ELAPSED=$((END_MS - START_MS))
 
 echo "$INDEX_JSON" > "$OUT/00-index.json"
 echo "$ELAPSED" > "$OUT/index-time.txt"
+echo "$((START_MS - SETUP_START_MS))" > "$OUT/setup-time.txt"
+echo "$((END_MS - SETUP_START_MS))" > "$OUT/total-time.txt"
 
 # Extract node/edge counts (CLI wraps in MCP content envelope)
 NODES=$(echo "$INDEX_JSON" | python3 -c "
diff --git a/scripts/benchmark-search-graph.sh b/scripts/benchmark-search-graph.sh
index cc94147ec6..1d67b95818 100755
--- a/scripts/benchmark-search-graph.sh
+++ b/scripts/benchmark-search-graph.sh
@@ -3,18 +3,46 @@
 # codebase-memory-mcp binary to measure the regex / LIKE pre-filter performance.
 #
 # Usage:
-#   scripts/benchmark-search-graph.sh <binary-path> <project-name>
+#   scripts/benchmark-search-graph.sh <binary-path> <repo-path>
 #
 # Example:
-#   scripts/benchmark-search-graph.sh ./build/c/codebase-memory-mcp my-project
+#   scripts/benchmark-search-graph.sh ./build/c/codebase-memory-mcp ~/src/my-project
+#
+# The repository is indexed (untimed) into a private runtime and cache first;
+# the queries then run against that index through a daemon this run keeps warm,
+# so a timing never includes daemon activation and never touches the operator's
+# live store (#1696).
 
 set -euo pipefail
 
-BINARY="${1:?Usage: $0 <binary-path> <project-name>}"
-PROJECT="${2:?Usage: $0 <binary-path> <project-name>}"
+BINARY="${1:?Usage: $0 <binary-path> <repo-path>}"
+REPO="${2:?Usage: $0 <binary-path> <repo-path>}"
+REPO=$(cd "$REPO" && pwd -P)
+
+# shellcheck source=test-runtime.sh
+source "$(dirname "${BASH_SOURCE[0]}")/test-runtime.sh"
+cbm_test_runtime_init
+trap 'cbm_test_runtime_cleanup "$BINARY"' EXIT
+
+if ! "$BINARY" daemon start >/dev/null 2>&1; then
+    echo "private daemon did not start" >&2
+    exit 1
+fi
+INDEX_JSON=$("$BINARY" cli index_repository "{\"repo_path\":\"$REPO\",\"mode\":\"full\"}" 2>/dev/null || echo '{}')
+PROJECT=$(echo "$INDEX_JSON" | python3 -c "
+import json, sys
+d = json.load(sys.stdin)
+if 'content' in d:
+    d = json.loads(d['content'][0]['text'])
+print(d.get('project', ''))
+" 2>/dev/null || echo "")
+if [ -z "$PROJECT" ]; then
+    echo "index of $REPO did not report a project" >&2
+    exit 1
+fi
 
 echo "Binary:  $BINARY"
-echo "Project: $PROJECT"
+echo "Project: $PROJECT (indexed from $REPO)"
 echo ""
 
 run_case() {
diff --git a/scripts/test.sh b/scripts/test.sh
index 7f4fea6e83..f95404712c 100755
--- a/scripts/test.sh
+++ b/scripts/test.sh
@@ -316,6 +316,9 @@ bash "$ROOT/tests/test_parallel_harness_contract.sh"
 echo "=== Step 0i2: suite failure-report contract ==="
 bash "$ROOT/tests/test_failure_report_contract.sh"
 
+echo "=== Step 0i3: benchmark harness runtime isolation contract (#1696) ==="
+bash "$ROOT/tests/test_benchmark_runtime_isolation_contract.sh"
+
 echo "=== Step 0j: venue parity contract (one harness, every venue) ==="
 bash "$ROOT/tests/test_venue_parity_contract.sh"
 
diff --git a/tests/test_benchmark_runtime_isolation_contract.sh b/tests/test_benchmark_runtime_isolation_contract.sh
new file mode 100755
index 0000000000..b944df7683
--- /dev/null
+++ b/tests/test_benchmark_runtime_isolation_contract.sh
@@ -0,0 +1,96 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# Runtime-isolation contract for the benchmark harnesses (#1696, follow-up to
+# #1691).
+#
+# scripts/benchmark-index.sh and scripts/benchmark-search-graph.sh ran the
+# product with no runtime or cache of their own: the index landed in the
+# operator's live store, every one-shot joined the operator's account daemon,
+# and the timings depended on whatever that daemon was doing. Drive both with
+# an environment-probe fixture and require that no product process ever
+# receives the caller's runtime or cache, and that the index benchmark records
+# the setup cost it now pays explicitly.
+
+ROOT="$(cd "$(dirname "$0")/.." && pwd)"
+WORKDIR="$(mktemp -d)"
+trap 'rm -rf "$WORKDIR"' EXIT
+
+fail() {
+    echo "FAIL: $*" >&2
+    exit 1
+}
+
+normalize_path() {
+    local path=${1%$'\r'}
+    if command -v cygpath >/dev/null 2>&1; then
+        cygpath -u "$path" 2>/dev/null && return 0
+    fi
+    printf '%s\n' "${path//\\//}"
+}
+
+ENV_PROBE="$WORKDIR/environment-probe"
+cat > "$ENV_PROBE" <<'EOF'
+#!/usr/bin/env bash
+printf '%s\t%s\n' "${CBM_CACHE_DIR-}" "${CBM_RUNTIME_DIR-}" >> "$CBM_BENCH_ENV_PROBE"
+[[ "${1-} ${2-}" == "daemon status" ]] && exit 1
+exit 0
+EOF
+chmod +x "$ENV_PROBE"
+
+CALLER_CACHE="$WORKDIR/caller-cache"
+CALLER_RUNTIME="$WORKDIR/caller-runtime"
+REPO="$WORKDIR/repo"
+mkdir -p "$CALLER_CACHE" "$CALLER_RUNTIME" "$REPO"
+echo 'def bench(): return 1' > "$REPO/bench.py"
+CALLER_CACHE_NORMALIZED=$(normalize_path "$CALLER_CACHE")
+CALLER_RUNTIME_NORMALIZED=$(normalize_path "$CALLER_RUNTIME")
+
+# The fixture answers nothing, so the search benchmark stops once the index
+# reports no project; only the environment handed to the product is under test.
+assert_isolated() {
+    local harness="$1" env_log="$2" private_root=""
+    [[ -s "$env_log" ]] || fail "$harness did not execute the environment-probe fixture"
+    while IFS=$'\t' read -r child_cache_raw child_runtime_raw; do
+        local child_cache child_runtime
+        child_cache=$(normalize_path "$child_cache_raw")
+        child_runtime=$(normalize_path "$child_runtime_raw")
+        if [[ -z "$child_runtime" || "$child_runtime" == "$CALLER_RUNTIME_NORMALIZED" ]]; then
+            fail "$harness exposed the caller CBM_RUNTIME_DIR to a product process"
+        fi
+        if [[ -z "$child_cache" || "$child_cache" == "$CALLER_CACHE_NORMALIZED" ]]; then
+            fail "$harness exposed the caller CBM_CACHE_DIR to a product process"
+        fi
+        if [[ "${child_runtime%/*}" != "${child_cache%/*}" ||
+              "${child_runtime##*/}" != "runtime" || "${child_cache##*/}" != "cache" ]]; then
+            fail "$harness runtime/cache were not isolated beneath one private root"
+        fi
+        if [[ -n "$private_root" && "$private_root" != "${child_runtime%/*}" ]]; then
+            fail "$harness switched private roots mid-run"
+        fi
+        private_root="${child_runtime%/*}"
+    done < "$env_log"
+    [[ ! -e "$private_root" ]] || fail "$harness left its private root behind: $private_root"
+}
+
+INDEX_LOG="$WORKDIR/index-environment.log"
+CBM_CACHE_DIR="$CALLER_CACHE" \
+CBM_RUNTIME_DIR="$CALLER_RUNTIME" \
+CBM_BENCH_ENV_PROBE="$INDEX_LOG" \
+    "$ROOT/scripts/benchmark-index.sh" "$ENV_PROBE" probe "$REPO" "$WORKDIR/results" \
+    > "$WORKDIR/index.out" 2>&1 || true
+assert_isolated "benchmark-index" "$INDEX_LOG"
+for metric in setup-time total-time index-time; do
+    [[ -s "$WORKDIR/results/probe/$metric.txt" ]] ||
+        fail "benchmark-index did not record $metric.txt"
+done
+
+SEARCH_LOG="$WORKDIR/search-environment.log"
+CBM_CACHE_DIR="$CALLER_CACHE" \
+CBM_RUNTIME_DIR="$CALLER_RUNTIME" \
+CBM_BENCH_ENV_PROBE="$SEARCH_LOG" \
+    "$ROOT/scripts/benchmark-search-graph.sh" "$ENV_PROBE" "$REPO" \
+    > "$WORKDIR/search.out" 2>&1 || true
+assert_isolated "benchmark-search-graph" "$SEARCH_LOG"
+
+echo "PASS: benchmark harnesses isolate their daemon runtime and cache from the caller"

From 27a335b18c23a0a88f5bf3c212d295faf206ecd5 Mon Sep 17 00:00:00 2001
From: Anton Standrik <astandrik@yandex-team.ru>
Date: Mon, 21 Sep 2026 12:00:10 +0300
Subject: [PATCH 02/10] fix(test): escape benchmark paths, keep stderr, use
 monotonic clock

Review follow-up for the benchmark harness isolation.

- Both benchmark scripts build the index request with a JSON-escaped
  repository path (python3 json.dumps, the spelling soak-test.sh already
  uses): a path containing a quote or a backslash produced a payload the
  server could not parse.
- benchmark-index.sh reads its three timestamps from one helper backed by
  time.monotonic_ns(); an NTP step mid-run no longer skews or negates a
  figure whose purpose is comparison across runs.
- benchmark-search-graph.sh keeps the stderr of the index call and of the
  response parse in a run-private temp file (removed by the EXIT trap) and
  prints it, plus the first 500 bytes of the response, under the existing
  "did not report a project" line.
- The contract test now refuses the index from its fixture and requires the
  cause to reach the output, and (POSIX only) drives a repository path with
  a quote and a backslash and requires the recorded request to parse as
  JSON with the resolved path. Both assertions fail against the previous
  head of this branch.

Signed-off-by: Anton Standrik <astandrik@yandex-team.ru>
---
 scripts/benchmark-index.sh                    | 19 +++++--
 scripts/benchmark-search-graph.sh             | 25 ++++++++--
 ...st_benchmark_runtime_isolation_contract.sh | 49 +++++++++++++++++++
 3 files changed, 86 insertions(+), 7 deletions(-)

diff --git a/scripts/benchmark-index.sh b/scripts/benchmark-index.sh
index 71e694c171..0bc24af2f8 100755
--- a/scripts/benchmark-index.sh
+++ b/scripts/benchmark-index.sh
@@ -19,6 +19,17 @@ trap 'cbm_test_runtime_cleanup "$BINARY"' EXIT
 
 # Resolve symlinks
 REPO=$(cd "$REPO" && pwd -P)
+# One pre-escaped spelling of the path for every request below, the way the
+# soak harness builds its own: a repository path may legitimately contain a
+# quote or a backslash, and hand-built JSON turns that into a parse error.
+REPO_JSON=$(python3 -c 'import json,sys; print(json.dumps(sys.argv[1]))' "$REPO")
+
+# Elapsed time is read from a monotonic clock, never the wall clock: an NTP
+# step mid-run would otherwise skew — or negate — a figure whose whole purpose
+# is comparison across runs. Its reference point is fixed per boot on every
+# platform CPython supports here, so the three readings below are comparable
+# even though each comes from its own process.
+bench_now_ms() { python3 -c "import time; print(time.monotonic_ns() // 1000000)"; }
 
 OUT="$RESULTS_DIR/$LANG"
 mkdir -p "$OUT"
@@ -45,18 +56,18 @@ echo "$LOC" > "$OUT/loc.txt"
 # alone. setup-time.txt keeps the activation cost attributable and
 # total-time.txt is their sum — the figure comparable with earlier runs, which
 # paid activation inside the index timing whenever no daemon was already warm.
-SETUP_START_MS=$(python3 -c "import time; print(int(time.time()*1000))")
+SETUP_START_MS=$(bench_now_ms)
 if ! "$BINARY" daemon start >/dev/null 2>&1; then
   echo "  $LANG: private daemon did not start" >&2
   exit 1
 fi
 
 # Index via CLI and capture timing
-START_MS=$(python3 -c "import time; print(int(time.time()*1000))")
+START_MS=$(bench_now_ms)
 
-INDEX_JSON=$("$BINARY" cli index_repository "{\"repo_path\":\"$REPO\",\"mode\":\"full\"}" 2>/dev/null || echo '{"error":"index failed"}')
+INDEX_JSON=$("$BINARY" cli index_repository "{\"repo_path\":$REPO_JSON,\"mode\":\"full\"}" 2>/dev/null || echo '{"error":"index failed"}')
 
-END_MS=$(python3 -c "import time; print(int(time.time()*1000))")
+END_MS=$(bench_now_ms)
 ELAPSED=$((END_MS - START_MS))
 
 echo "$INDEX_JSON" > "$OUT/00-index.json"
diff --git a/scripts/benchmark-search-graph.sh b/scripts/benchmark-search-graph.sh
index 1d67b95818..6296d31fb2 100755
--- a/scripts/benchmark-search-graph.sh
+++ b/scripts/benchmark-search-graph.sh
@@ -22,22 +22,41 @@ REPO=$(cd "$REPO" && pwd -P)
 # shellcheck source=test-runtime.sh
 source "$(dirname "${BASH_SOURCE[0]}")/test-runtime.sh"
 cbm_test_runtime_init
-trap 'cbm_test_runtime_cleanup "$BINARY"' EXIT
+BENCH_TMP=""
+trap 'cbm_test_runtime_cleanup "$BINARY"; [ -z "$BENCH_TMP" ] || rm -rf -- "$BENCH_TMP"' EXIT
+BENCH_TMP=$(mktemp -d)
+INDEX_ERR="$BENCH_TMP/index-stderr.log"
 
 if ! "$BINARY" daemon start >/dev/null 2>&1; then
     echo "private daemon did not start" >&2
     exit 1
 fi
-INDEX_JSON=$("$BINARY" cli index_repository "{\"repo_path\":\"$REPO\",\"mode\":\"full\"}" 2>/dev/null || echo '{}')
+# One pre-escaped spelling of the path, the way the soak harness builds its
+# own: a repository path may legitimately contain a quote or a backslash, and
+# hand-built JSON turns that into a parse error.
+REPO_JSON=$(python3 -c 'import json,sys; print(json.dumps(sys.argv[1]))' "$REPO")
+# Index and parse keep their stderr instead of discarding it: without it the
+# failure below names only its symptom, and the cause — an unreadable
+# repository, a refused daemon, a malformed envelope — is unrecoverable.
+INDEX_JSON=$("$BINARY" cli index_repository "{\"repo_path\":$REPO_JSON,\"mode\":\"full\"}" \
+    2>"$INDEX_ERR" || echo '{}')
 PROJECT=$(echo "$INDEX_JSON" | python3 -c "
 import json, sys
 d = json.load(sys.stdin)
 if 'content' in d:
     d = json.loads(d['content'][0]['text'])
 print(d.get('project', ''))
-" 2>/dev/null || echo "")
+" 2>>"$INDEX_ERR" || echo "")
 if [ -z "$PROJECT" ]; then
     echo "index of $REPO did not report a project" >&2
+    if [ -s "$INDEX_ERR" ]; then
+        echo "--- index/parse stderr ---" >&2
+        cat "$INDEX_ERR" >&2
+    fi
+    if [ -n "$INDEX_JSON" ]; then
+        echo "--- index response (first 500 bytes) ---" >&2
+        printf '%.500s\n' "$INDEX_JSON" >&2
+    fi
     exit 1
 fi
 
diff --git a/tests/test_benchmark_runtime_isolation_contract.sh b/tests/test_benchmark_runtime_isolation_contract.sh
index b944df7683..1dc618ae69 100755
--- a/tests/test_benchmark_runtime_isolation_contract.sh
+++ b/tests/test_benchmark_runtime_isolation_contract.sh
@@ -33,6 +33,11 @@ ENV_PROBE="$WORKDIR/environment-probe"
 cat > "$ENV_PROBE" <<'EOF'
 #!/usr/bin/env bash
 printf '%s\t%s\n' "${CBM_CACHE_DIR-}" "${CBM_RUNTIME_DIR-}" >> "$CBM_BENCH_ENV_PROBE"
+if [[ "${1-} ${2-}" == "cli index_repository" ]]; then
+    [[ -z "${CBM_BENCH_REQUEST_LOG-}" ]] || printf '%s\n' "${3-}" >> "$CBM_BENCH_REQUEST_LOG"
+    echo "probe: index refused" >&2
+    exit 1
+fi
 [[ "${1-} ${2-}" == "daemon status" ]] && exit 1
 exit 0
 EOF
@@ -93,4 +98,48 @@ CBM_BENCH_ENV_PROBE="$SEARCH_LOG" \
     > "$WORKDIR/search.out" 2>&1 || true
 assert_isolated "benchmark-search-graph" "$SEARCH_LOG"
 
+# A refused index must name its cause. The fixture writes one line to stderr and
+# exits non-zero; discarding it leaves "did not report a project" as the only
+# thing an operator sees.
+grep -q -- '--- index/parse stderr ---' "$WORKDIR/search.out" ||
+    fail "benchmark-search-graph hid the index stderr behind its own message"
+grep -q 'probe: index refused' "$WORKDIR/search.out" ||
+    fail "benchmark-search-graph did not surface the cause of the index failure"
+
+# The request carrying the repository path has to be built as JSON. A path may
+# legitimately contain a quote or a backslash — hand-built JSON turns that into
+# a payload the server cannot parse, or one that means something else. NTFS
+# rejects both characters in a path component, so this case is POSIX-only.
+case "$(uname -s)" in
+MINGW* | MSYS* | CYGWIN*) ;;
+*)
+    QUIRKY_REPO="$WORKDIR/re\"po\\dir"
+    mkdir -p "$QUIRKY_REPO"
+    echo 'def bench(): return 1' > "$QUIRKY_REPO/bench.py"
+    QUIRKY_RESOLVED=$(cd "$QUIRKY_REPO" && pwd -P)
+    REQUEST_LOG="$WORKDIR/requests.log"
+    CBM_CACHE_DIR="$CALLER_CACHE" \
+    CBM_RUNTIME_DIR="$CALLER_RUNTIME" \
+    CBM_BENCH_ENV_PROBE="$WORKDIR/quirky-environment.log" \
+    CBM_BENCH_REQUEST_LOG="$REQUEST_LOG" \
+        "$ROOT/scripts/benchmark-search-graph.sh" "$ENV_PROBE" "$QUIRKY_REPO" \
+        > "$WORKDIR/quirky.out" 2>&1 || true
+    python3 - "$REQUEST_LOG" "$QUIRKY_RESOLVED" <<'PY' || fail "benchmark-search-graph built an index request that is not valid JSON for a quoted path"
+import json
+import sys
+
+request_log, expected = sys.argv[1], sys.argv[2]
+try:
+    lines = [line for line in open(request_log).read().splitlines() if line.strip()]
+except OSError:
+    sys.exit("the search benchmark sent no index request")
+if not lines:
+    sys.exit("the search benchmark sent no index request")
+payload = json.loads(lines[0])
+if payload.get("repo_path") != expected:
+    sys.exit(f"repo_path is {payload.get('repo_path')!r}, expected {expected!r}")
+PY
+    ;;
+esac
+
 echo "PASS: benchmark harnesses isolate their daemon runtime and cache from the caller"

From 567ed5d87e807f6463ed90f324137b6794fa23ae Mon Sep 17 00:00:00 2001
From: Anton Standrik <astandrik@yandex-team.ru>
Date: Mon, 21 Sep 2026 12:25:28 +0300
Subject: [PATCH 03/10] fix(test): hand the benchmark index to the caller on
 request
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The evaluation plan (docs/EVALUATION_PLAN.md §7) indexes a language with
benchmark-index.sh and then answers graph questions against that index
from its own MCP session. With a run-private runtime (#1696) the index
was gone before that session could start, so the handoff is now explicit
and opt-in.

With CBM_BENCH_KEEP_RUNTIME set, a SUCCESSFUL run stops its daemon,
leaves the private root in place and records the paths that reach it in
<results>/<lang>/runtime-root.txt (sourceable: CBM_BENCH_RUNTIME_ROOT,
CBM_RUNTIME_DIR, CBM_CACHE_DIR). Ownership of that root, including its
removal, passes to the caller. A failed run cleans up regardless: there
is no index worth keeping and nothing may leak. Nothing changes in the
default path or in the three timing files.

§7's skeleton sets the flag, sources the file before the graph session
and removes the root in step 8 instead of the live-store *.db files.

The contract test drives a kept run and requires the root to survive,
the handoff file to be sourceable, and its paths to be the ones the
product processes actually used; the unflagged run still asserts the
root is gone. The kept-run assertions fail against the previous head.

Signed-off-by: Anton Standrik <astandrik@yandex-team.ru>
---
 docs/EVALUATION_PLAN.md                       |  7 +++++-
 scripts/benchmark-index.sh                    | 23 ++++++++++++++++-
 ...st_benchmark_runtime_isolation_contract.sh | 25 +++++++++++++++++++
 3 files changed, 53 insertions(+), 2 deletions(-)

diff --git a/docs/EVALUATION_PLAN.md b/docs/EVALUATION_PLAN.md
index efd56207db..01398daca5 100644
--- a/docs/EVALUATION_PLAN.md
+++ b/docs/EVALUATION_PLAN.md
@@ -283,8 +283,13 @@ for lang in $ALL_LANGS; do                      # ALL_LANGS = full 159-name list
 
   # --- step 2: cold index in the main channel, TIMED (key metric) ---
   t0=$(now_ms)
+  CBM_BENCH_KEEP_RUNTIME=1 \
   scripts/benchmark-index.sh ~/.local/bin/codebase-memory-mcp "$lang" /tmp/bench/"$lang" /tmp/eval-results
   index_ms=$(( $(now_ms) - t0 ))                 # clone+index wall-clock → manifest + report (§5)
+  # The harness indexes into a run-private root (#1696). CBM_BENCH_KEEP_RUNTIME leaves that
+  # root behind on success and records its paths; the graph session in steps 4-7 must be
+  # started with these CBM_RUNTIME_DIR / CBM_CACHE_DIR, and step 8 removes the root.
+  set -a; . /tmp/eval-results/"$lang"/runtime-root.txt; set +a
 
   # --- step 3: record per-type histograms (zeros back-filled) ---
   #   node-types.json, edge-types.json  (every label + all 32 edge types, zeros kept, §7 below)
@@ -293,7 +298,7 @@ for lang in $ALL_LANGS; do                      # ALL_LANGS = full 159-name list
   #               + per-language report + (deferred, blind) judge ---
 
   # --- step 8: delete THIS language's index so the next is cold, then mark done ---
-  rm -f ~/.cache/codebase-memory-mcp/*.db
+  rm -rf -- "$CBM_BENCH_RUNTIME_ROOT"
   manifest_mark_done "$lang" "$index_ms"
 done
 ```
diff --git a/scripts/benchmark-index.sh b/scripts/benchmark-index.sh
index 0bc24af2f8..11d79ee1fa 100755
--- a/scripts/benchmark-index.sh
+++ b/scripts/benchmark-index.sh
@@ -15,7 +15,28 @@ RESULTS_DIR="${4:?}"
 # shellcheck source=test-runtime.sh
 source "$(dirname "${BASH_SOURCE[0]}")/test-runtime.sh"
 cbm_test_runtime_init
-trap 'cbm_test_runtime_cleanup "$BINARY"' EXIT
+
+# The evaluation plan (docs/EVALUATION_PLAN.md §7) indexes a language here and
+# then answers graph questions against that index from its own MCP session.
+# A run-private root would be gone before that session starts, so the caller
+# may ask for it to be kept: after a SUCCESSFUL run the harness stops its
+# daemon, leaves the root in place, records the paths that reach it in
+# <results>/<lang>/runtime-root.txt (sourceable), and ownership of the root —
+# including its removal — passes to the caller. A failed run cleans up
+# regardless: there is no index worth keeping, and nothing must leak. Never on
+# by default, or an unattended run accumulates one root per language.
+bench_finish() {
+  local rc=$?
+  if [ "$rc" -eq 0 ] && [ -n "${CBM_BENCH_KEEP_RUNTIME:-}" ] && [ -d "${OUT:-}" ]; then
+    "$BINARY" daemon stop >/dev/null 2>&1 || true
+    printf 'CBM_BENCH_RUNTIME_ROOT=%q\nCBM_RUNTIME_DIR=%q\nCBM_CACHE_DIR=%q\n' \
+      "$CBM_TEST_RUNTIME_ROOT" "$CBM_RUNTIME_DIR" "$CBM_CACHE_DIR" > "$OUT/runtime-root.txt"
+    echo "  $LANG: runtime kept at $CBM_TEST_RUNTIME_ROOT; paths in $OUT/runtime-root.txt" >&2
+    return 0
+  fi
+  cbm_test_runtime_cleanup "$BINARY"
+}
+trap bench_finish EXIT
 
 # Resolve symlinks
 REPO=$(cd "$REPO" && pwd -P)
diff --git a/tests/test_benchmark_runtime_isolation_contract.sh b/tests/test_benchmark_runtime_isolation_contract.sh
index 1dc618ae69..73c1758f2b 100755
--- a/tests/test_benchmark_runtime_isolation_contract.sh
+++ b/tests/test_benchmark_runtime_isolation_contract.sh
@@ -90,6 +90,31 @@ for metric in setup-time total-time index-time; do
         fail "benchmark-index did not record $metric.txt"
 done
 
+# The evaluation plan indexes a language and then reads that index from its own
+# MCP session (docs/EVALUATION_PLAN.md §7). Asked to keep the runtime, a
+# successful run must leave its root behind and record, sourceably, the paths
+# that reach it — and they must be the paths the product processes actually
+# used. Unasked, the root is gone (asserted above).
+KEEP_LOG="$WORKDIR/keep-environment.log"
+CBM_CACHE_DIR="$CALLER_CACHE" \
+CBM_RUNTIME_DIR="$CALLER_RUNTIME" \
+CBM_BENCH_ENV_PROBE="$KEEP_LOG" \
+CBM_BENCH_KEEP_RUNTIME=1 \
+    "$ROOT/scripts/benchmark-index.sh" "$ENV_PROBE" keep "$REPO" "$WORKDIR/results" \
+    > "$WORKDIR/keep.out" 2>&1 || true
+HANDOFF="$WORKDIR/results/keep/runtime-root.txt"
+[[ -s "$HANDOFF" ]] || fail "benchmark-index was asked to keep its runtime but recorded no handoff"
+KEPT_ROOT=$(bash -c '. "$1" && printf "%s" "${CBM_BENCH_RUNTIME_ROOT-}"' _ "$HANDOFF")
+KEPT_RUNTIME=$(bash -c '. "$1" && printf "%s" "${CBM_RUNTIME_DIR-}"' _ "$HANDOFF")
+KEPT_CACHE=$(bash -c '. "$1" && printf "%s" "${CBM_CACHE_DIR-}"' _ "$HANDOFF")
+[[ -n "$KEPT_ROOT" && -d "$KEPT_ROOT" && ! -L "$KEPT_ROOT" ]] ||
+    fail "benchmark-index did not keep its runtime root: ${KEPT_ROOT:-<none>}"
+[[ -d "$KEPT_ROOT/cache" && -d "$KEPT_ROOT/runtime" ]] ||
+    fail "the kept root $KEPT_ROOT lost its cache or runtime directory"
+grep -qF -- "${KEPT_CACHE}"$'\t'"${KEPT_RUNTIME}" "$KEEP_LOG" ||
+    fail "runtime-root.txt does not name the runtime and cache the product processes used"
+rm -rf -- "$KEPT_ROOT"
+
 SEARCH_LOG="$WORKDIR/search-environment.log"
 CBM_CACHE_DIR="$CALLER_CACHE" \
 CBM_RUNTIME_DIR="$CALLER_RUNTIME" \

From a2ac0540c1d10cde927707717f8cec2088e245bb Mon Sep 17 00:00:00 2001
From: Anton Standrik <astandrik@yandex-team.ru>
Date: Tue, 22 Sep 2026 13:57:30 +0300
Subject: [PATCH 04/10] fix(test): fail the index benchmark when its index is
 refused
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

bench_finish keeps the private root on a zero exit status, but the index
call swallowed its failure (`|| echo '{"error":"index failed"}'`), so a
refused index exited 0 and, with CBM_BENCH_KEEP_RUNTIME set, kept an
empty root and wrote runtime-root.txt for an index that does not exist.

benchmark-index.sh now records the CLI's exit status, keeps 00-index.json
valid JSON on failure ({"error":"index failed","exit":N}), and exits 1
once every per-run file — the three timing files included — is written,
when the index failed or reported no project (the search-graph twin's own
test). bench_finish then cleans up as its comment always promised, and
the caller's loop notices. The index call keeps its stderr in a
run-private temp file, the twin's shape, and the failure prints it under
`--- index stderr ---` with the first 500 bytes of the response: for a
one-shot CLI, stderr is the only channel a refusal is reported on. A
stale runtime-root.txt from an earlier run in the same results directory
is removed at start, so a failed run leaves no handoff at all.

The contract test's fixture answers `cli index_repository` with a
minimal envelope when CBM_BENCH_PROBE_INDEX_OK is set, so the keep run
covers the successful-index handoff and asserts project.txt; the
unflagged run asserts a non-zero exit and the surfaced cause after a
refused index; a new keep run with a refused index asserts a non-zero
exit, the timing files, no surviving root and no handoff, a pre-seeded
stale one included. Against the previous head the test fails on
"benchmark-index exited 0 after a refused index".

Signed-off-by: Anton Standrik <astandrik@yandex-team.ru>
---
 scripts/benchmark-index.sh                    | 33 +++++++++++-
 ...st_benchmark_runtime_isolation_contract.sh | 54 +++++++++++++++++--
 2 files changed, 81 insertions(+), 6 deletions(-)

diff --git a/scripts/benchmark-index.sh b/scripts/benchmark-index.sh
index 11d79ee1fa..207944a85f 100755
--- a/scripts/benchmark-index.sh
+++ b/scripts/benchmark-index.sh
@@ -27,6 +27,7 @@ cbm_test_runtime_init
 # by default, or an unattended run accumulates one root per language.
 bench_finish() {
   local rc=$?
+  [ -z "${BENCH_TMP:-}" ] || rm -rf -- "$BENCH_TMP" || true
   if [ "$rc" -eq 0 ] && [ -n "${CBM_BENCH_KEEP_RUNTIME:-}" ] && [ -d "${OUT:-}" ]; then
     "$BINARY" daemon stop >/dev/null 2>&1 || true
     printf 'CBM_BENCH_RUNTIME_ROOT=%q\nCBM_RUNTIME_DIR=%q\nCBM_CACHE_DIR=%q\n' \
@@ -36,7 +37,13 @@ bench_finish() {
   fi
   cbm_test_runtime_cleanup "$BINARY"
 }
+BENCH_TMP=""
 trap bench_finish EXIT
+# The index call keeps its stderr, the way the search-graph twin does: for a
+# one-shot CLI that is the only channel a refusal is reported on, and without
+# it the failure below could name only its symptom.
+BENCH_TMP=$(mktemp -d)
+INDEX_ERR="$BENCH_TMP/index-stderr.log"
 
 # Resolve symlinks
 REPO=$(cd "$REPO" && pwd -P)
@@ -54,6 +61,10 @@ bench_now_ms() { python3 -c "import time; print(time.monotonic_ns() // 1000000)"
 
 OUT="$RESULTS_DIR/$LANG"
 mkdir -p "$OUT"
+# A handoff names this run's index or none: the evaluation loop reuses the
+# results directory, and a stale one would point the graph session at a root
+# step 8 has already removed.
+rm -f -- "$OUT/runtime-root.txt"
 
 echo "INDEX: $LANG ($REPO)"
 
@@ -86,7 +97,12 @@ fi
 # Index via CLI and capture timing
 START_MS=$(bench_now_ms)
 
-INDEX_JSON=$("$BINARY" cli index_repository "{\"repo_path\":$REPO_JSON,\"mode\":\"full\"}" 2>/dev/null || echo '{"error":"index failed"}')
+# The CLI's exit status is the index's verdict. It is recorded here and acted
+# on below, once every per-run file is written; an empty response stays valid
+# JSON in 00-index.json so the failure is legible there as well.
+INDEX_RC=0
+INDEX_JSON=$("$BINARY" cli index_repository "{\"repo_path\":$REPO_JSON,\"mode\":\"full\"}" 2>"$INDEX_ERR") ||
+  { INDEX_RC=$?; INDEX_JSON="{\"error\":\"index failed\",\"exit\":$INDEX_RC}"; }
 
 END_MS=$(bench_now_ms)
 ELAPSED=$((END_MS - START_MS))
@@ -130,5 +146,20 @@ echo "$NODES" > "$OUT/nodes.txt"
 echo "$EDGES" > "$OUT/edges.txt"
 echo "$PROJECT" > "$OUT/project.txt"
 
+# An index that failed, or that named no project, fails the run: after the
+# timing files, so the caller keeps its figures, and through the exit status,
+# so the caller's loop notices and bench_finish never keeps a root with no
+# index in it.
+if [ "$INDEX_RC" -ne 0 ] || [ -z "$PROJECT" ]; then
+  echo "  $LANG: index failed (cli exit $INDEX_RC, project '$PROJECT'); response in $OUT/00-index.json" >&2
+  if [ -s "$INDEX_ERR" ]; then
+    echo "--- index stderr ---" >&2
+    cat "$INDEX_ERR" >&2
+  fi
+  echo "--- index response (first 500 bytes) ---" >&2
+  printf '%.500s\n' "$INDEX_JSON" >&2
+  exit 1
+fi
+
 printf "  %s: %s files, %s LOC, %sms, %s nodes, %s edges\n" \
   "$LANG" "$FILE_COUNT" "$LOC" "$ELAPSED" "$NODES" "$EDGES"
diff --git a/tests/test_benchmark_runtime_isolation_contract.sh b/tests/test_benchmark_runtime_isolation_contract.sh
index 73c1758f2b..f2e862d0dd 100755
--- a/tests/test_benchmark_runtime_isolation_contract.sh
+++ b/tests/test_benchmark_runtime_isolation_contract.sh
@@ -9,8 +9,9 @@ set -euo pipefail
 # operator's live store, every one-shot joined the operator's account daemon,
 # and the timings depended on whatever that daemon was doing. Drive both with
 # an environment-probe fixture and require that no product process ever
-# receives the caller's runtime or cache, and that the index benchmark records
-# the setup cost it now pays explicitly.
+# receives the caller's runtime or cache, that the index benchmark records
+# the setup cost it now pays explicitly, and that a refused index fails the
+# run, names its cause, and leaves nothing behind.
 
 ROOT="$(cd "$(dirname "$0")/.." && pwd)"
 WORKDIR="$(mktemp -d)"
@@ -35,6 +36,10 @@ cat > "$ENV_PROBE" <<'EOF'
 printf '%s\t%s\n' "${CBM_CACHE_DIR-}" "${CBM_RUNTIME_DIR-}" >> "$CBM_BENCH_ENV_PROBE"
 if [[ "${1-} ${2-}" == "cli index_repository" ]]; then
     [[ -z "${CBM_BENCH_REQUEST_LOG-}" ]] || printf '%s\n' "${3-}" >> "$CBM_BENCH_REQUEST_LOG"
+    if [[ -n "${CBM_BENCH_PROBE_INDEX_OK-}" ]]; then
+        printf '%s\n' '{"project":"probe"}'
+        exit 0
+    fi
     echo "probe: index refused" >&2
     exit 1
 fi
@@ -78,30 +83,46 @@ assert_isolated() {
     [[ ! -e "$private_root" ]] || fail "$harness left its private root behind: $private_root"
 }
 
+# The fixture refuses the index. That must fail the run — after the timing
+# files are written, so the caller keeps its figures — and name its cause: the
+# fixture's one stderr line has to reach the output.
 INDEX_LOG="$WORKDIR/index-environment.log"
+index_rc=0
 CBM_CACHE_DIR="$CALLER_CACHE" \
 CBM_RUNTIME_DIR="$CALLER_RUNTIME" \
 CBM_BENCH_ENV_PROBE="$INDEX_LOG" \
     "$ROOT/scripts/benchmark-index.sh" "$ENV_PROBE" probe "$REPO" "$WORKDIR/results" \
-    > "$WORKDIR/index.out" 2>&1 || true
+    > "$WORKDIR/index.out" 2>&1 || index_rc=$?
+[[ "$index_rc" -ne 0 ]] || fail "benchmark-index exited 0 after a refused index"
 assert_isolated "benchmark-index" "$INDEX_LOG"
 for metric in setup-time total-time index-time; do
     [[ -s "$WORKDIR/results/probe/$metric.txt" ]] ||
         fail "benchmark-index did not record $metric.txt"
 done
+grep -q -- '--- index stderr ---' "$WORKDIR/index.out" ||
+    fail "benchmark-index hid the index stderr behind its own message"
+grep -q 'probe: index refused' "$WORKDIR/index.out" ||
+    fail "benchmark-index did not surface the cause of the index failure"
 
 # The evaluation plan indexes a language and then reads that index from its own
 # MCP session (docs/EVALUATION_PLAN.md §7). Asked to keep the runtime, a
 # successful run must leave its root behind and record, sourceably, the paths
 # that reach it — and they must be the paths the product processes actually
-# used. Unasked, the root is gone (asserted above).
+# used. Unasked, the root is gone (asserted above). The fixture answers this
+# index with a minimal envelope, so what is kept is an index that succeeded.
 KEEP_LOG="$WORKDIR/keep-environment.log"
+keep_rc=0
 CBM_CACHE_DIR="$CALLER_CACHE" \
 CBM_RUNTIME_DIR="$CALLER_RUNTIME" \
 CBM_BENCH_ENV_PROBE="$KEEP_LOG" \
+CBM_BENCH_PROBE_INDEX_OK=1 \
 CBM_BENCH_KEEP_RUNTIME=1 \
     "$ROOT/scripts/benchmark-index.sh" "$ENV_PROBE" keep "$REPO" "$WORKDIR/results" \
-    > "$WORKDIR/keep.out" 2>&1 || true
+    > "$WORKDIR/keep.out" 2>&1 || keep_rc=$?
+[[ "$keep_rc" -eq 0 ]] || fail "benchmark-index failed a run whose index succeeded (exit $keep_rc)"
+KEPT_PROJECT=$(cat "$WORKDIR/results/keep/project.txt" 2>/dev/null || true)
+[[ "${KEPT_PROJECT%$'\r'}" == "probe" ]] ||
+    fail "benchmark-index did not record the project the index reported: '${KEPT_PROJECT:-<none>}'"
 HANDOFF="$WORKDIR/results/keep/runtime-root.txt"
 [[ -s "$HANDOFF" ]] || fail "benchmark-index was asked to keep its runtime but recorded no handoff"
 KEPT_ROOT=$(bash -c '. "$1" && printf "%s" "${CBM_BENCH_RUNTIME_ROOT-}"' _ "$HANDOFF")
@@ -115,6 +136,29 @@ grep -qF -- "${KEPT_CACHE}"$'\t'"${KEPT_RUNTIME}" "$KEEP_LOG" ||
     fail "runtime-root.txt does not name the runtime and cache the product processes used"
 rm -rf -- "$KEPT_ROOT"
 
+# Asked to keep the runtime and refused the index, the harness has nothing worth
+# keeping: the run fails, no root survives, and no handoff is written — nor left
+# over from an earlier run, since the evaluation loop reuses the results
+# directory.
+REFUSED_LOG="$WORKDIR/keep-refused-environment.log"
+REFUSED_HANDOFF="$WORKDIR/results/keep-refused/runtime-root.txt"
+mkdir -p "${REFUSED_HANDOFF%/*}"
+echo 'CBM_BENCH_RUNTIME_ROOT=/stale/root/from/an/earlier/run' > "$REFUSED_HANDOFF"
+refused_rc=0
+CBM_CACHE_DIR="$CALLER_CACHE" \
+CBM_RUNTIME_DIR="$CALLER_RUNTIME" \
+CBM_BENCH_ENV_PROBE="$REFUSED_LOG" \
+CBM_BENCH_KEEP_RUNTIME=1 \
+    "$ROOT/scripts/benchmark-index.sh" "$ENV_PROBE" keep-refused "$REPO" "$WORKDIR/results" \
+    > "$WORKDIR/keep-refused.out" 2>&1 || refused_rc=$?
+[[ "$refused_rc" -ne 0 ]] || fail "benchmark-index exited 0 after a refused index it was asked to keep"
+for metric in setup-time total-time index-time; do
+    [[ -s "$WORKDIR/results/keep-refused/$metric.txt" ]] ||
+        fail "benchmark-index did not record $metric.txt for a refused index"
+done
+assert_isolated "benchmark-index (keep, refused index)" "$REFUSED_LOG"
+[[ ! -e "$REFUSED_HANDOFF" ]] || fail "benchmark-index left a handoff for an index that failed"
+
 SEARCH_LOG="$WORKDIR/search-environment.log"
 CBM_CACHE_DIR="$CALLER_CACHE" \
 CBM_RUNTIME_DIR="$CALLER_RUNTIME" \

From 54c18648c211ad152cd025af4ee5d76da1616601 Mon Sep 17 00:00:00 2001
From: Anton Standrik <astandrik@yandex-team.ru>
Date: Tue, 22 Sep 2026 13:57:30 +0300
Subject: [PATCH 05/10] docs(eval-plan): stop the kept daemon before its root
 is removed
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

§7 step 8 removed the kept runtime root while the graph session of
steps 4-7 may still have a daemon running under it; stop that daemon
under the sourced CBM_RUNTIME_DIR first. §13 Reproducibility still
cleared ~/.cache/codebase-memory-mcp/*.db and called the harness
unflagged, which no longer describes what the harness does: it indexes
into a run-private root, and only CBM_BENCH_KEEP_RUNTIME=1 hands that
root, via runtime-root.txt, to the graph phase.

Signed-off-by: Anton Standrik <astandrik@yandex-team.ru>
---
 docs/EVALUATION_PLAN.md | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/docs/EVALUATION_PLAN.md b/docs/EVALUATION_PLAN.md
index 01398daca5..39d72848fa 100644
--- a/docs/EVALUATION_PLAN.md
+++ b/docs/EVALUATION_PLAN.md
@@ -298,6 +298,9 @@ for lang in $ALL_LANGS; do                      # ALL_LANGS = full 159-name list
   #               + per-language report + (deferred, blind) judge ---
 
   # --- step 8: delete THIS language's index so the next is cold, then mark done ---
+  # The graph session (steps 4-7) may still have a daemon running under the kept
+  # CBM_RUNTIME_DIR (exported above); stop it before its root goes away.
+  ~/.local/bin/codebase-memory-mcp daemon stop || true
   rm -rf -- "$CBM_BENCH_RUNTIME_ROOT"
   manifest_mark_done "$lang" "$index_ms"
 done
@@ -865,10 +868,13 @@ Deep-Dive section.
 # 1. Clone all 159 repos (shallow; skip existing)
 scripts/clone-bench-repos.sh /tmp/bench
 
-# 2. Cold index all 159 (LSP cohort in full mode)
-rm -f ~/.cache/codebase-memory-mcp/*.db
+# 2. Cold index all 159 (LSP cohort in full mode). Each language is indexed into a run-private
+#    runtime and cache (#1696); CBM_BENCH_KEEP_RUNTIME=1 keeps that root after a successful
+#    index and records its paths in /tmp/eval-results/<lang>/runtime-root.txt. Source it
+#    before the graph phase, then stop the daemon and remove the root, as in §7 step 8.
 mkdir -p /tmp/eval-results
 for lang in $ALL_LANGS; do
+  CBM_BENCH_KEEP_RUNTIME=1 \
   scripts/benchmark-index.sh ~/.local/bin/codebase-memory-mcp "$lang" /tmp/bench/"$lang" /tmp/eval-results
 done
 

From 14b8da00c3a7ee559c5b07ec4e8e220e171c79e7 Mon Sep 17 00:00:00 2001
From: Martin Vogel <martin.vogel.tech@gmail.com>
Date: Mon, 2 Nov 2026 00:06:52 +0100
Subject: [PATCH 06/10] fix(test): drop a stale benchmark handoff before setup
 can fail

benchmark-index.sh removed the runtime-root.txt of an earlier run only
after the repository path, the private runtime and the temp directory had
been set up. A run that failed in one of those steps exited non-zero but left
the previous handoff in the reused results directory, and section 7 of
the evaluation plan sources that file unconditionally, so the graph
session would be pointed at a root that step 8 has already removed.

The results directory is now created and the stale handoff removed right
after argument parsing, before any setup step that can fail.

Signed-off-by: Martin Vogel <martin.vogel.tech@gmail.com>
---
 scripts/benchmark-index.sh | 15 ++++++++-------
 1 file changed, 8 insertions(+), 7 deletions(-)

diff --git a/scripts/benchmark-index.sh b/scripts/benchmark-index.sh
index 207944a85f..5ab4353bb2 100755
--- a/scripts/benchmark-index.sh
+++ b/scripts/benchmark-index.sh
@@ -9,6 +9,14 @@ LANG="${2:?}"
 REPO="${3:?}"
 RESULTS_DIR="${4:?}"
 
+OUT="$RESULTS_DIR/$LANG"
+mkdir -p "$OUT"
+# A handoff names this run's index or none: the evaluation loop reuses the
+# results directory, and a stale one would point the graph session at a root
+# step 8 has already removed. It goes before any setup step that can fail, so
+# a run that never reaches the index cannot leave an earlier one behind.
+rm -f -- "$OUT/runtime-root.txt"
+
 # The index must run against a daemon rendezvous and cache this run owns: only
 # CBM_RUNTIME_DIR moves the rendezvous, and without a private cache the
 # benchmark repository was indexed into the operator's live store (#1696).
@@ -59,13 +67,6 @@ REPO_JSON=$(python3 -c 'import json,sys; print(json.dumps(sys.argv[1]))' "$REPO"
 # even though each comes from its own process.
 bench_now_ms() { python3 -c "import time; print(time.monotonic_ns() // 1000000)"; }
 
-OUT="$RESULTS_DIR/$LANG"
-mkdir -p "$OUT"
-# A handoff names this run's index or none: the evaluation loop reuses the
-# results directory, and a stale one would point the graph session at a root
-# step 8 has already removed.
-rm -f -- "$OUT/runtime-root.txt"
-
 echo "INDEX: $LANG ($REPO)"
 
 # Count source files and LOC (exclude .git, vendor, node_modules, build dirs)

From b9851700a431e5589f050970f275dde140ba7f3c Mon Sep 17 00:00:00 2001
From: Martin Vogel <martin.vogel.tech@gmail.com>
Date: Mon, 2 Nov 2026 00:06:52 +0100
Subject: [PATCH 07/10] fix(test): surface why a benchmark daemon did not start

Both harnesses discarded the output of `daemon start`, so a refused
start was reported only as "private daemon did not start", the symptom
without its cause. The output now goes to a run-private file next to the
index stderr, the shape both scripts already use for the index call, and
is printed under "--- daemon start output ---" when the start fails.

Signed-off-by: Martin Vogel <martin.vogel.tech@gmail.com>
---
 scripts/benchmark-index.sh        | 9 ++++++++-
 scripts/benchmark-search-graph.sh | 9 ++++++++-
 2 files changed, 16 insertions(+), 2 deletions(-)

diff --git a/scripts/benchmark-index.sh b/scripts/benchmark-index.sh
index 5ab4353bb2..a8db36828d 100755
--- a/scripts/benchmark-index.sh
+++ b/scripts/benchmark-index.sh
@@ -52,6 +52,7 @@ trap bench_finish EXIT
 # it the failure below could name only its symptom.
 BENCH_TMP=$(mktemp -d)
 INDEX_ERR="$BENCH_TMP/index-stderr.log"
+DAEMON_START_LOG="$BENCH_TMP/daemon-start.log"
 
 # Resolve symlinks
 REPO=$(cd "$REPO" && pwd -P)
@@ -90,8 +91,14 @@ echo "$LOC" > "$OUT/loc.txt"
 # total-time.txt is their sum — the figure comparable with earlier runs, which
 # paid activation inside the index timing whenever no daemon was already warm.
 SETUP_START_MS=$(bench_now_ms)
-if ! "$BINARY" daemon start >/dev/null 2>&1; then
+# Its output is kept for the failure path: the product names why a daemon did
+# not start, and that is the part an operator needs.
+if ! "$BINARY" daemon start >"$DAEMON_START_LOG" 2>&1; then
   echo "  $LANG: private daemon did not start" >&2
+  if [ -s "$DAEMON_START_LOG" ]; then
+    echo "--- daemon start output ---" >&2
+    cat "$DAEMON_START_LOG" >&2
+  fi
   exit 1
 fi
 
diff --git a/scripts/benchmark-search-graph.sh b/scripts/benchmark-search-graph.sh
index 6296d31fb2..dad4f6ec48 100755
--- a/scripts/benchmark-search-graph.sh
+++ b/scripts/benchmark-search-graph.sh
@@ -26,9 +26,16 @@ BENCH_TMP=""
 trap 'cbm_test_runtime_cleanup "$BINARY"; [ -z "$BENCH_TMP" ] || rm -rf -- "$BENCH_TMP"' EXIT
 BENCH_TMP=$(mktemp -d)
 INDEX_ERR="$BENCH_TMP/index-stderr.log"
+DAEMON_START_LOG="$BENCH_TMP/daemon-start.log"
 
-if ! "$BINARY" daemon start >/dev/null 2>&1; then
+# Its output is kept for the failure path: the product names why a daemon did
+# not start, and that is the part an operator needs.
+if ! "$BINARY" daemon start >"$DAEMON_START_LOG" 2>&1; then
     echo "private daemon did not start" >&2
+    if [ -s "$DAEMON_START_LOG" ]; then
+        echo "--- daemon start output ---" >&2
+        cat "$DAEMON_START_LOG" >&2
+    fi
     exit 1
 fi
 # One pre-escaped spelling of the path, the way the soak harness builds its

From 78bf6a274505aafe662b49955376f2d3f9e9d47e Mon Sep 17 00:00:00 2001
From: Martin Vogel <martin.vogel.tech@gmail.com>
Date: Mon, 2 Nov 2026 00:06:52 +0100
Subject: [PATCH 08/10] fix(test): hand over a kept benchmark runtime only
 after its daemon stopped

With CBM_BENCH_KEEP_RUNTIME=1, bench_finish ran `daemon stop` with its
output and exit status discarded and wrote runtime-root.txt regardless.
`daemon stop` refuses while a committed client still uses the daemon and
fails when the daemon does not answer; the handoff then promised a
stopped daemon that was still running.

A refused stop now prints the explanation of the product, writes no
handoff, keeps the root without cleaning it up (the daemon may still be
using it) and exits non-zero. A stop with nothing to stop still exits 0,
so a clean run is unaffected.

Section 7 step 8 of the evaluation plan follows the same rule: the kept
root is removed only if its daemon stopped, and is kept with a message
otherwise.

Signed-off-by: Martin Vogel <martin.vogel.tech@gmail.com>
---
 docs/EVALUATION_PLAN.md    | 10 +++++++---
 scripts/benchmark-index.sh | 14 ++++++++++++--
 2 files changed, 19 insertions(+), 5 deletions(-)

diff --git a/docs/EVALUATION_PLAN.md b/docs/EVALUATION_PLAN.md
index 39d72848fa..25a8f6aafd 100644
--- a/docs/EVALUATION_PLAN.md
+++ b/docs/EVALUATION_PLAN.md
@@ -299,9 +299,13 @@ for lang in $ALL_LANGS; do                      # ALL_LANGS = full 159-name list
 
   # --- step 8: delete THIS language's index so the next is cold, then mark done ---
   # The graph session (steps 4-7) may still have a daemon running under the kept
-  # CBM_RUNTIME_DIR (exported above); stop it before its root goes away.
-  ~/.local/bin/codebase-memory-mcp daemon stop || true
-  rm -rf -- "$CBM_BENCH_RUNTIME_ROOT"
+  # CBM_RUNTIME_DIR (exported above); stop it before its root goes away. A refused
+  # stop (a session still attached) means the root is still in use: keep it.
+  if ~/.local/bin/codebase-memory-mcp daemon stop; then
+    rm -rf -- "$CBM_BENCH_RUNTIME_ROOT"
+  else
+    echo "$lang: daemon under $CBM_RUNTIME_DIR did not stop; keeping $CBM_BENCH_RUNTIME_ROOT" >&2
+  fi
   manifest_mark_done "$lang" "$index_ms"
 done
 ```
diff --git a/scripts/benchmark-index.sh b/scripts/benchmark-index.sh
index a8db36828d..792ec9febe 100755
--- a/scripts/benchmark-index.sh
+++ b/scripts/benchmark-index.sh
@@ -33,11 +33,21 @@ cbm_test_runtime_init
 # including its removal — passes to the caller. A failed run cleans up
 # regardless: there is no index worth keeping, and nothing must leak. Never on
 # by default, or an unattended run accumulates one root per language.
+#
+# The handoff promises a stopped daemon, so it is written only once
+# `daemon stop` succeeded. A refused stop (a client still committed to the
+# daemon, or no answer) fails the run with the product's explanation, writes
+# no handoff, and keeps the root without cleaning it up: the daemon may still
+# be using it, and the operator decides what happens to it.
 bench_finish() {
-  local rc=$?
+  local rc=$? stop_out=""
   [ -z "${BENCH_TMP:-}" ] || rm -rf -- "$BENCH_TMP" || true
   if [ "$rc" -eq 0 ] && [ -n "${CBM_BENCH_KEEP_RUNTIME:-}" ] && [ -d "${OUT:-}" ]; then
-    "$BINARY" daemon stop >/dev/null 2>&1 || true
+    if ! stop_out=$("$BINARY" daemon stop 2>&1); then
+      echo "  $LANG: private daemon did not stop; runtime kept at $CBM_TEST_RUNTIME_ROOT, no handoff written" >&2
+      [ -z "$stop_out" ] || printf '%s\n' "$stop_out" >&2
+      exit 1
+    fi
     printf 'CBM_BENCH_RUNTIME_ROOT=%q\nCBM_RUNTIME_DIR=%q\nCBM_CACHE_DIR=%q\n' \
       "$CBM_TEST_RUNTIME_ROOT" "$CBM_RUNTIME_DIR" "$CBM_CACHE_DIR" > "$OUT/runtime-root.txt"
     echo "  $LANG: runtime kept at $CBM_TEST_RUNTIME_ROOT; paths in $OUT/runtime-root.txt" >&2

From 32f60a4e8d74eb6fe33de54b22f1a4b7bbf0a8f8 Mon Sep 17 00:00:00 2001
From: Martin Vogel <martin.vogel.tech@gmail.com>
Date: Mon, 2 Nov 2026 00:06:52 +0100
Subject: [PATCH 09/10] docs(eval-plan): stop clearing the live cache before an
 isolated run

Section 7 still began with rm -f ~/.cache/codebase-memory-mcp/*.db.
Every index in the loop now lands in a run-private cache (#1696), so
that line no longer made any run colder; it only deleted the indexes in
the live store of the operator. Dropped, and the comment above it now
names only what the setup still does.

Signed-off-by: Martin Vogel <martin.vogel.tech@gmail.com>
---
 docs/EVALUATION_PLAN.md | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/docs/EVALUATION_PLAN.md b/docs/EVALUATION_PLAN.md
index 25a8f6aafd..17b3701f99 100644
--- a/docs/EVALUATION_PLAN.md
+++ b/docs/EVALUATION_PLAN.md
@@ -274,8 +274,7 @@ then its index is **deleted** before the next language — so **only one project
 every index is genuinely cold. The skeleton:
 
 ```bash
-# Start clean: no leftover DBs, fresh results tree
-rm -f ~/.cache/codebase-memory-mcp/*.db
+# Start clean: fresh results tree
 mkdir -p /tmp/eval-results
 
 for lang in $ALL_LANGS; do                      # ALL_LANGS = full 159-name list (§2 / §8)

From b9a186db479674ff50443546624fe956d4bcf175 Mon Sep 17 00:00:00 2001
From: Martin Vogel <martin.vogel.tech@gmail.com>
Date: Mon, 2 Nov 2026 00:06:52 +0100
Subject: [PATCH 10/10] test(bench): cover setup failure, refused daemon start
 and refused stop

Three more cases in the benchmark runtime isolation contract, each RED
against the harness without its fix:

- a stale runtime-root.txt is gone after a run that fails during setup
  (a repository path that does not exist);
- with the keep flag and a `daemon stop` that refuses, the run fails,
  writes no handoff, keeps its root and prints the refusal; the test
  removes that root itself, before any verdict, so a failing assertion
  leaks nothing;
- a refused `daemon start` fails both harnesses, prints the cause and
  leaves no root behind.

The fixture refuses the start on stderr and the stop on stdout, the
channel the product uses for a busy refusal.

Signed-off-by: Martin Vogel <martin.vogel.tech@gmail.com>
---
 ...st_benchmark_runtime_isolation_contract.sh | 84 ++++++++++++++++++-
 1 file changed, 83 insertions(+), 1 deletion(-)

diff --git a/tests/test_benchmark_runtime_isolation_contract.sh b/tests/test_benchmark_runtime_isolation_contract.sh
index f2e862d0dd..3657910e39 100755
--- a/tests/test_benchmark_runtime_isolation_contract.sh
+++ b/tests/test_benchmark_runtime_isolation_contract.sh
@@ -11,7 +11,9 @@ set -euo pipefail
 # an environment-probe fixture and require that no product process ever
 # receives the caller's runtime or cache, that the index benchmark records
 # the setup cost it now pays explicitly, and that a refused index fails the
-# run, names its cause, and leaves nothing behind.
+# run, names its cause, and leaves nothing behind. A daemon that refuses to
+# start or to stop fails the run the same way, and a kept runtime is handed
+# over only once its daemon stopped.
 
 ROOT="$(cd "$(dirname "$0")/.." && pwd)"
 WORKDIR="$(mktemp -d)"
@@ -43,6 +45,16 @@ if [[ "${1-} ${2-}" == "cli index_repository" ]]; then
     echo "probe: index refused" >&2
     exit 1
 fi
+if [[ "${1-} ${2-}" == "daemon start" && -n "${CBM_BENCH_PROBE_START_REFUSED-}" ]]; then
+    echo "probe: daemon start refused" >&2
+    exit 1
+fi
+# The product reports a busy refusal (a committed client still uses the
+# daemon) on stdout, so that is where the fixture reports its own.
+if [[ "${1-} ${2-}" == "daemon stop" && -n "${CBM_BENCH_PROBE_STOP_REFUSED-}" ]]; then
+    echo "probe: daemon NOT stopped, a committed client still uses it"
+    exit 1
+fi
 [[ "${1-} ${2-}" == "daemon status" ]] && exit 1
 exit 0
 EOF
@@ -136,6 +148,37 @@ grep -qF -- "${KEPT_CACHE}"$'\t'"${KEPT_RUNTIME}" "$KEEP_LOG" ||
     fail "runtime-root.txt does not name the runtime and cache the product processes used"
 rm -rf -- "$KEPT_ROOT"
 
+# The handoff promises a stopped daemon, so a kept runtime is handed over only
+# once `daemon stop` succeeded. A refused stop fails the run, writes no handoff
+# and keeps the root: the daemon may still be using it, so removing it is not
+# the harness's call. The refusal itself has to reach the output.
+STOP_LOG="$WORKDIR/keep-stop-refused-environment.log"
+STOP_HANDOFF="$WORKDIR/results/keep-stop-refused/runtime-root.txt"
+stop_rc=0
+CBM_CACHE_DIR="$CALLER_CACHE" \
+CBM_RUNTIME_DIR="$CALLER_RUNTIME" \
+CBM_BENCH_ENV_PROBE="$STOP_LOG" \
+CBM_BENCH_PROBE_INDEX_OK=1 \
+CBM_BENCH_PROBE_STOP_REFUSED=1 \
+CBM_BENCH_KEEP_RUNTIME=1 \
+    "$ROOT/scripts/benchmark-index.sh" "$ENV_PROBE" keep-stop-refused "$REPO" "$WORKDIR/results" \
+    > "$WORKDIR/keep-stop-refused.out" 2>&1 || stop_rc=$?
+[[ -s "$STOP_LOG" ]] || fail "benchmark-index (keep, refused stop) did not execute the environment-probe fixture"
+STOP_RUNTIME=$(normalize_path "$(tail -n 1 "$STOP_LOG" | cut -f2)")
+STOP_ROOT="${STOP_RUNTIME%/*}"
+stop_root_kept=0
+[[ ! -d "$STOP_ROOT/runtime" ]] || stop_root_kept=1
+# Removed here, before any verdict, so a failing assertion leaks no root.
+case "${STOP_ROOT##*/}" in
+cbmrt.?* | cbmrt-?*) rm -rf -- "$STOP_ROOT" ;;
+*) fail "benchmark-index (keep, refused stop) used an unexpected private root: $STOP_ROOT" ;;
+esac
+[[ "$stop_rc" -ne 0 ]] || fail "benchmark-index exited 0 although its daemon refused to stop"
+[[ ! -e "$STOP_HANDOFF" ]] || fail "benchmark-index handed over a runtime whose daemon refused to stop"
+(( stop_root_kept )) || fail "benchmark-index removed a runtime root whose daemon refused to stop"
+grep -q 'probe: daemon NOT stopped' "$WORKDIR/keep-stop-refused.out" ||
+    fail "benchmark-index did not surface why its daemon refused to stop"
+
 # Asked to keep the runtime and refused the index, the harness has nothing worth
 # keeping: the run fails, no root survives, and no handoff is written — nor left
 # over from an earlier run, since the evaluation loop reuses the results
@@ -159,6 +202,23 @@ done
 assert_isolated "benchmark-index (keep, refused index)" "$REFUSED_LOG"
 [[ ! -e "$REFUSED_HANDOFF" ]] || fail "benchmark-index left a handoff for an index that failed"
 
+# The same holds for a run that never reaches the index: one whose repository
+# does not exist fails during setup, and an earlier run's handoff in the reused
+# results directory must already be gone by then.
+SETUP_HANDOFF="$WORKDIR/results/setup-failed/runtime-root.txt"
+mkdir -p "${SETUP_HANDOFF%/*}"
+echo 'CBM_BENCH_RUNTIME_ROOT=/stale/root/from/an/earlier/run' > "$SETUP_HANDOFF"
+setup_rc=0
+CBM_CACHE_DIR="$CALLER_CACHE" \
+CBM_RUNTIME_DIR="$CALLER_RUNTIME" \
+CBM_BENCH_ENV_PROBE="$WORKDIR/setup-failed-environment.log" \
+CBM_BENCH_KEEP_RUNTIME=1 \
+    "$ROOT/scripts/benchmark-index.sh" "$ENV_PROBE" setup-failed "$WORKDIR/no-such-repo" "$WORKDIR/results" \
+    > "$WORKDIR/setup-failed.out" 2>&1 || setup_rc=$?
+[[ "$setup_rc" -ne 0 ]] || fail "benchmark-index exited 0 for a repository that does not exist"
+[[ ! -e "$SETUP_HANDOFF" ]] ||
+    fail "benchmark-index left an earlier handoff behind after a run that failed during setup"
+
 SEARCH_LOG="$WORKDIR/search-environment.log"
 CBM_CACHE_DIR="$CALLER_CACHE" \
 CBM_RUNTIME_DIR="$CALLER_RUNTIME" \
@@ -175,6 +235,28 @@ grep -q -- '--- index/parse stderr ---' "$WORKDIR/search.out" ||
 grep -q 'probe: index refused' "$WORKDIR/search.out" ||
     fail "benchmark-search-graph did not surface the cause of the index failure"
 
+# A private daemon that does not start fails either harness, and the product's
+# own explanation has to reach the output: "did not start" alone leaves the
+# operator with the symptom only.
+for harness in benchmark-index benchmark-search-graph; do
+    if [[ "$harness" == benchmark-index ]]; then
+        harness_args=("$ENV_PROBE" start-refused "$REPO" "$WORKDIR/results")
+    else
+        harness_args=("$ENV_PROBE" "$REPO")
+    fi
+    start_rc=0
+    CBM_CACHE_DIR="$CALLER_CACHE" \
+    CBM_RUNTIME_DIR="$CALLER_RUNTIME" \
+    CBM_BENCH_ENV_PROBE="$WORKDIR/$harness-start-refused-environment.log" \
+    CBM_BENCH_PROBE_START_REFUSED=1 \
+        "$ROOT/scripts/$harness.sh" "${harness_args[@]}" \
+        > "$WORKDIR/$harness-start-refused.out" 2>&1 || start_rc=$?
+    [[ "$start_rc" -ne 0 ]] || fail "$harness exited 0 although its private daemon did not start"
+    assert_isolated "$harness (daemon start refused)" "$WORKDIR/$harness-start-refused-environment.log"
+    grep -q 'probe: daemon start refused' "$WORKDIR/$harness-start-refused.out" ||
+        fail "$harness did not surface why its private daemon did not start"
+done
+
 # The request carrying the repository path has to be built as JSON. A path may
 # legitimately contain a quote or a backslash — hand-built JSON turns that into
 # a payload the server cannot parse, or one that means something else. NTFS