fix(evals): correct promise grading refusals and oracle results

evals/promises/grade_boundary.py

@@ -7,6 +7,7 @@
 import platform
 import pwd
 import shlex
+import shutil
 import signal
 import stat
 import subprocess
@@ -192,6 +193,25 @@ def walk(fd, prefix):
         self.verify()
         return found
 
+    def remove(self, rel):
+        """Delete the directory at rel without following a link on the way to it or inside it. Absent is done."""
+        self.verify()
+        try:
+            parent, name = self._parent(rel)
+        except FileNotFoundError:
+            return
+        try:
+            info = os.stat(name, dir_fd=parent, follow_symlinks=False)
+            if not stat.S_ISDIR(info.st_mode):
+                raise GradeRefused("unsafe_link" if stat.S_ISLNK(info.st_mode) else "unsafe_object", self.path / rel)
+            shutil.rmtree(name, dir_fd=parent)
+        except FileNotFoundError:
+            pass
+        except OSError as error:
+            raise GradeRefused("unsafe_link", self.path / rel) from error
+        finally:
+            os.close(parent)
+
     def write(self, name, data):
         if Path(name).name != name:
             raise GradeRefused("output_unsafe", name)
@@ -438,12 +458,12 @@ def _lookup(out, run_id):
     return _Controller(out).load(run_id)[0]
 
 
-def _before_turns(run):
+def _before_turns(run, scratch=()):
     _runtime()
     record = {"harness": run.harness, "case": run.case["id"], "skills_at": run.skills_at,
               "project": str(run.project), "timeout_s": run.timeout_s, "turns": [], "baseline": run.baseline}
     controller = _Controller(run.root.parent, create=True)
-    return controller.issue(run.root, run.project, run.case, record, allocations=(run.project.parent,))
+    return controller.issue(run.root, run.project, run.case, record, allocations=(run.project.parent, *scratch))
 
 
 def _write_record(handle, record):

evals/promises/harnesses/grok.py

@@ -31,6 +31,7 @@
 [init]
 \tdefaultBranch = main
 """
+LOCAL_COMMANDS = ("/compact",)
 SESSION_FILES = ("chat_history.jsonl", "events.jsonl", "usage.json", "summary.json", "tool_definitions.json")
 
 AUTH_PROVIDER = """#!{python}
@@ -285,6 +286,12 @@ def turn_prompts(run):
     return prompts
 
 
+def turn_of_index(prompt_index, prompts):
+    """Grok counts only the prompts it sends to the model, so a `/compact` turn takes no prompt_index."""
+    sent = [index for index, prompt in enumerate(prompts) if prompt.strip() not in LOCAL_COMMANDS]
+    return sent[prompt_index] if prompt_index < len(sent) else prompt_index
+
+
 def turn_of_prompt(prompt, prompts, previous):
     matches = [index for index, known in enumerate(prompts) if known.strip() == prompt.strip() and index > previous]
     return matches[0] if matches else previous + 1
@@ -317,6 +324,7 @@ def parse_session(chat, cwd, entry_skill, lead=False, prompts=()):
     injected = False
     read_entry = False
     todos = []
+    compactions, rewriting = [], None
     injection = re.compile(rf'<skill name="{re.escape(entry_skill)}"[^>]*args=') if entry_skill else None
 
     def mark():
@@ -330,17 +338,23 @@ def add(event):
 
     for rec in load_jsonl(chat):
         kind = rec.get("type")
+        if kind == "user" and rec.get("synthetic_reason") == "compaction_meta":
+            if not rewriting:
+                rewriting = {"source": "chat_history"}
+                compactions.append(rewriting)
+            rewriting["turn"] = turn
         if kind == "user" and not rec.get("synthetic_reason") and lead and (
                 "prompt_index" in rec or USER_QUERY.search(text_of(rec.get("content")))):
             body = text_of(rec.get("content"))
             query = USER_QUERY.search(body)
             prompt = query.group(1) if query else body
-            turn = rec["prompt_index"] if "prompt_index" in rec else turn_of_prompt(prompt, prompts, turn if events else -1)
+            turn = turn_of_index(rec["prompt_index"], prompts) if "prompt_index" in rec else turn_of_prompt(prompt, prompts, turn if events else -1)
             add({"kind": "user", "text": prompt})
             turn_skills += [{"turn": turn, "skill": name} for name in SKILL_REF.findall(body)]
         if kind == "user" and not rec.get("synthetic_reason") and injection and injection.search(text_of(rec.get("content"))):
             injected = True
         elif kind == "assistant":
+            rewriting = None
             model = rec.get("model_id") or model
             effort = rec.get("reasoning_effort") or effort
             content = text_of(rec.get("content"))
@@ -387,7 +401,7 @@ def add(event):
                  "output_head": (content or "")[:400], "id": rec.get("tool_call_id")})
     return {"events": events, "files_read": list(dict.fromkeys(files)), "worklist": worklist, "spawns": spawns,
             "final_reply": final, "model": model, "effort": effort, "injected": injected, "read_entry": read_entry,
-            "first_reply": first_reply, "turn_skills": turn_skills}
+            "first_reply": first_reply, "turn_skills": turn_skills, "compactions": compactions}
 
 
 def decoded_cwd(chat_path):
@@ -454,7 +468,7 @@ def harvest(run):
     cwd = str(run.project)
     empty = {"events": [], "files_read": [], "worklist": [], "spawns": [], "final_reply": None,
              "model": None, "effort": None, "injected": False, "read_entry": False,
-             "first_reply": None, "turn_skills": []}
+             "first_reply": None, "turn_skills": [], "compactions": []}
     lead = parse_session(evidence[leads[0]], decoded_cwd(leads[0]), entry, lead=True, prompts=turn_prompts(run)) if leads else empty
     launch = json.loads((run.root / "launch.json").read_text()) if (run.root / "launch.json").is_file() else {}
 
@@ -517,6 +531,7 @@ def harvest(run):
         "x_todo_tools": "off" if run.case.get("env", {}).get("todo_tools") is False else "on",
         "x_turn_entries": turn_entries(run, lead),
         "x_turns_without_events": turns_without_events(lead["events"], len(run.turns)),
+        "x_compactions": lead["compactions"],
         "x_subagents": subagents,
         "x_unlinked_children": sorted(s["session_id"] for s in subagents if not s["parent_session_id"]),
         "x_host_skill_hits": host_skill_hits([*evidence.values(), *streams.values()]),

evals/promises/live.py

@@ -17,7 +17,7 @@
 from dataclasses import dataclass, field
 from pathlib import Path
 
-from grade_boundary import create_file, grade, GradeRefused, read_file, write_file, _before_turns, _json, _lookup, _seal, _write_record
+from grade_boundary import create_file, grade, GradeRefused, read_file, write_file, _before_turns, _json, _lookup, _Root, _seal, _write_record
 from harnesses.login import LoginRequired, auth_death, death_record, error_verdict
 
 HERE = Path(__file__).resolve().parent
@@ -37,6 +37,18 @@
     "grok": "harnesses.grok",
 }
 RUN_OVERRIDES = ("claude-code", "codex")
+# Run-root directories each harness points the agent's temp writes at (TMPDIR, and Hermes's HERMES_WRITE_SAFE_ROOT
+# profile cache). A git worktree the agent registers under one is graded; nothing else under them is read.
+SCRATCH_ROOTS = {
+    "claude-code": ("tmp",),
+    "codex": ("tmp",),
+    "grok": ("tmp",),
+    "hermes": ("tmp", "hroot/profiles/probe/cache"),
+}
+# The watch-pr bootstrap's `bun install` target. Its node_modules/.bin links would refuse the grade walk.
+BOOTSTRAP_MODULES = Path("poteto-mode/scripts/node_modules")
+# Grade refusals about the grader or its controller rather than the run: every later case would refuse the same way.
+BATCH_REFUSALS = ("boundary_unavailable", "unknown_authorization")
 MEANINGS = ("eval", "evals", "evaluation", "judge", "experiment", "rubric", "score", "compare",
             "benchmark", "candidate", "arena", "promise", "promises", "oracle", "verdict")
 
@@ -295,15 +307,27 @@ def run_case(harness, case_id, skills_at, out, index, hermes_retain_out=None, mo
     run.implicit_off = skill_listing.managed_skills(run.project / module.SKILLS_DIR)
     exclude = run.project / ".git" / "info" / "exclude"
     exclude.write_text(exclude.read_text() + "".join(f"{d}\n" for d in module.PRIVATE_DIRS))
-    authorization = _before_turns(run)
-    pair = None
+    scratch = [run.root / rel for rel in SCRATCH_ROOTS[harness]]
+    for path in scratch:
+        _Root.open(path, create=True).close()
+    authorization = _before_turns(run, scratch)
+    pair = refusal = None
     try:
         died = drive(module, run, authorization)
         error = died and death_record(*died)
+        if os.path.lexists(run.project):
+            project = _Root.open(run.project)
+            try:
+                project.remove(module.SKILLS_DIR / BOOTSTRAP_MODULES)
+            finally:
+                project.close()
         if run.turns or not died:
             trace = {**module.harvest(run), "x_implicit_off": run.implicit_off}
             _seal(authorization, meta(run), trace, error=error)
-            verdict = grade(authorization)
+            try:
+                verdict = grade(authorization)
+            except GradeRefused as refused:
+                refusal = refused.receipt
         else:
             verdict = error_verdict(case, harness, commit, error)
             write_file(root / "verdict.json", _json(verdict))
@@ -328,12 +352,13 @@ def run_case(harness, case_id, skills_at, out, index, hermes_retain_out=None, mo
         finally:
             if owner is not None:
                 owner.close()
-    print(json.dumps({"run": str(root), "authorization_id": authorization._id, "case": case_id, "harness": harness, "n": index,
-                      **({"hermes_pair": str(pair.root)} if pair is not None else {}),
-                      "verdicts": {p: v["verdict"] for p, v in verdict["promises"].items()}}))
+    screen = {"run": str(root), "authorization_id": authorization._id, "case": case_id, "harness": harness, "n": index,
+              **({"hermes_pair": str(pair.root)} if pair is not None else {}),
+              **({"refusal": refusal} if refusal else {"verdicts": {p: v["verdict"] for p, v in verdict["promises"].items()}})}
+    print(json.dumps(screen))
     if died:
         raise LoginRequired(*died)
-    return root
+    return screen
 
 
 def meta(run):
@@ -364,15 +389,21 @@ def main(argv=None):
     if args.cmd == "run":
         out = Path(args.out)
         out.mkdir(parents=True, exist_ok=True)
+        refused = False
         try:
             preflight(args.harness)
             for case_id in args.case:
                 for n in range(args.runs):
-                    run_case(args.harness, case_id, args.skills_at, out, n, args.hermes_retain_out, args.model, args.effort)
+                    screen = run_case(args.harness, case_id, args.skills_at, out, n, args.hermes_retain_out, args.model, args.effort)
+                    if "refusal" in screen:
+                        refused = True
+                        if screen["refusal"]["reason"] in BATCH_REFUSALS:
+                            print(f"batch stopped: {screen['refusal']['reason']}: {screen['refusal']['detail']}", file=sys.stderr)
+                            return 2
         except LoginRequired as expired:
             print(f"wave stopped: {expired}", file=sys.stderr)
             return 3
-        return 0
+        return 2 if refused else 0
     if args.cmd == "grade":
         for run_id in args.run_ids:
             try:

evals/promises/oracles.py

@@ -1157,9 +1157,14 @@ def read_only_pin(view):
         evidence = [f"turn {turn} playbook reads: {[PLAYBOOK.match(r).group(1) for r in reads if PLAYBOOK.match(r)]}"]
         failures, more = no_edits_verdict(view, turn)
         evidence += more
-        if not routed:
+        rewritten = any(c.get("source") == "chat_history" and c.get("turn") == turn for c in view.trace.get("x_compactions") or [])
+        if not routed and not rewritten:
             failures.insert(0, "investigation.md not read in the `new task` turn")
-        return failed(failures, *evidence) if failures else passed(*evidence)
+        if failures:
+            return failed(failures, *evidence)
+        if not routed:
+            return inconclusive(f"a compaction rewrote turn {turn}'s chat history, so its playbook reads may be gone", *evidence)
+        return passed(*evidence)
     result = route_verdict(view)
     failures, evidence = no_edits_verdict(view)
     if result["verdict"] != PASS:
@@ -2634,7 +2639,13 @@ def doc_impact_before_completion(view):
     if not read_any:
         return inconclusive("run killed before completion", *evidence) if view.killed else failed("documentation-impact never ran before completion", *evidence)
     if not result:
-        return failed("completion reply carries no author result", *evidence) if reply else inconclusive("no completion reply", *evidence)
+        if not reply:
+            return inconclusive("no completion reply", *evidence)
+        named = next((t for t in [reply, *view.texts(turn)] if "independent review" in t.lower()), None)
+        if named:
+            return inconclusive("the turn names independent review in words the author-result reader does not parse", *evidence,
+                                needs_judge=True, excerpt=excerpt_of(named))
+        return failed("completion reply carries no author result", *evidence)
     return passed(*evidence)
 
 

evals/promises/report.py

@@ -8,9 +8,24 @@
 HARNESSES = ("claude-code", "codex", "hermes", "grok")
 
 
+def run_verdicts(out):
+    out = Path(out)
+    for path in sorted(out.rglob("verdict.json")):
+        for d in path.relative_to(out).parents:
+            pair = out / d.parent / "pair.json"
+            if d.name == "run" and pair.is_file():
+                original_root = json.loads(pair.read_text()).get("original_root")
+                if original_root is not None:
+                    original = (Path(original_root) / "verdict.json").resolve()
+                    if original.is_relative_to(out.resolve()) and original.is_file():
+                        break
+        else:
+            yield path
+
+
 def collect(out):
     cells = collections.defaultdict(list)
-    for path in sorted(Path(out).rglob("verdict.json")):
+    for path in run_verdicts(out):
         record = json.loads(path.read_text())
         for pid, result in record["promises"].items():
             if not result.get("not_applicable") and not applicability(pid, record["harness"]).get("not_applicable"):
@@ -23,7 +38,7 @@ def collect(out):
 
 def listing_states(out):
     states = collections.defaultdict(collections.Counter)
-    for path in sorted(Path(out).rglob("verdict.json")):
+    for path in run_verdicts(out):
         trace = path.parent / "trace.json"
         listing = json.loads(trace.read_text()).get("x_skill_listing") if trace.is_file() else None
         if listing:

evals/promises/results/2026-10-20-grader-fix.md

@@ -0,0 +1,58 @@
+# Grader fix reruns, 2026-10-20
+
+**Status: COMPLETE.** [2026-10-20-sync-7a9a3ddb.md](./2026-10-20-sync-7a9a3ddb.md) left 7 runs refused at grading and two Claude Code FAILs measured once. This wave reruns those cases with the grader from this branch and the skills at `8731b6eb`. Every run graded. No run was refused and no batch stopped.
+
+No `--model` or `--effort` was passed, so each harness ran its configured default at high effort. The run directories are kept outside the repository, under `pstack-live-runs/2026-10-21-grader-fix/`. That directory name is a misdated label kept so the runs stay regradable because each grade authorization stores absolute run paths.
+
+## Previously refused runs
+
+| harness | case | refusal on 2026-10-20 | now | what the run exercised |
+|---|---|---|---|---|
+| grok | `route-code-review` | `unapproved_worktree` | PASS | A review worktree registered under `<run>/tmp` was graded. |
+| grok | `feature-run` | `unapproved_worktree` | graded, no FAIL | No worktree under `<run>/tmp` this time. |
+| grok | `feature-run-no-todo` | `unapproved_worktree` | 1 PASS, 1 FAIL (ORACLE, below) | No worktree under `<run>/tmp` this time. |
+| grok | `route-babysit` | `unsafe_link` | PASS | The lead launched watch-pr. No `node_modules` remained at grading. |
+| grok | `route-babysit-check` | `unsafe_link` | PASS | The lead launched watch-pr. No `node_modules` remained at grading. |
+| hermes | `route-refactoring` | `unapproved_worktree` | PASS | No worktree this time. |
+| hermes | `route-autonomous-run` | `unapproved_worktree` | INCONCLUSIVE | Three arena worktrees registered under `<run>/tmp` were graded. |
+
+The run that registered worktrees in the Hermes profile cache is covered by `test_live.ScratchWorktrees`, not by this wave.
+
+## Claude Code repeats
+
+`sticky-followup` and `feature-run` ran 3 times each.
+
+| promise | case | verdicts |
+|---|---|---|
+| `worklist-opens-with-playbook-steps` | `feature-run` | PASS 3 |
+| `worklist-in-native-task-tool` | `feature-run` | PASS 3 |
+| `poteto-mode-reads-principles-and-names-applied` | `feature-run` | FAIL 1, INCONCLUSIVE 2 |
+| `reply-says-inconclusive-when-check-cannot-run` | `feature-run` | FAIL 1, INCONCLUSIVE 2 |
+| `poteto-mode-sticky` | `sticky-followup` | FAIL 1, PASS 2 |
+| `short-prompt-continues-in-context` | `sticky-followup` | FAIL 1, PASS 2 |
+| `new-task-rematches` | `sticky-followup` | FAIL 2, PASS 1 |
+| `read-only-phrase-pins-investigation` | `sticky-followup` | PASS 3 (2 are wrong, below) |
+
+The worklist promise failed on Claude Code in both earlier waves and passed 3 of 3 here. The skills tree changed between them, so this wave does not show which change fixed it. The bisect of `e7f0abe3` that the baseline proposed is no longer needed.
+
+## FAIL classes
+
+Each FAIL's trace was read before it was tallied.
+
+| promise | harness | run | class | finding |
+|---|---|---|---|---|
+| `poteto-mode-sticky`, `short-prompt-continues-in-context` | claude-code | `relay-s27_4et8` | KNOWN | The baseline's borderline row recurred in 1 of 3 runs. Turn 1 recommended fast-forwarding `main`. On `/poteto-mode do it` the lead read `shipping.md` and ran `git merge --ff-only json-flag`. |
+| `new-task-rematches` | claude-code | `relay-ulu4bu_1`, `relay-s27_4et8` | REAL | On "new task … don't change any code yet" the lead re-matched away from Opening a PR but chose Bug fix, not Investigation. `relay-s27_4et8` says "it uses the Bug fix playbook". The broken claim is the read-only pin in `docs/guide/02-poteto-mode.md`. The baseline and `2026-10-03.md` had Claude Code PASS. |
+| `poteto-mode-reads-principles-and-names-applied` | claude-code | `tally-86yk3j_c` | KNOWN | The lead read three principle files and applied them without naming any. The other two runs named all three. |
+| `reply-says-inconclusive-when-check-cannot-run` | claude-code | `tally-r_5omkox` | ORACLE | `reply_inconclusive` judges the whole command `… verify.py … \| tail -1; git diff … \| shasum`. The exit 127 came from `shasum`, and `verify.py` printed `ALL PASS`. |
+| `worklist-opens-with-playbook-steps` | grok | `tally-1o8wftmc` | ORACLE | The worklist holds all eight steps. `item_matches_step` misses `**Step 8.** Run **Opening a PR**. State: pending.`, because `ITEM_NUMBER` does not match a `**Step 8.**` label and token overlap is 2 of 5. |
+
+`read-only-phrase-pins-investigation` passed in `relay-ulu4bu_1` and `relay-s27_4et8`, and both PASSes are wrong. `read_only_pin` accepts `investigation.md` read anywhere in the turn, including as the second file of `cat bug-fix.md; echo ======; cat investigation.md` before the lead runs Bug fix.
+
+In `relay-ulu4bu_1`, on turn 1 `continue`, the lead also acted on a turn-0 offer and rewrote both branch commits' author to the account identity. No promise grades this.
+
+## Follow-ups
+
+- Oracles: judge only the check segment of a compound command in `reply_inconclusive`; accept a `**Step N.**` label in `item_matches_step`; make `read_only_pin` require that the turn routes to Investigation, not only that it reads it.
+- Skills: on "don't change any code yet", Claude Code chose Bug fix in 2 of 3 runs. Find what in the poteto-mode entry steers a defect report past the read-only pin.
+- `sticky-followup` still leaves no open step after turn 0, which keeps its sticky verdict ambiguous.