From 92508b28f63594c3d98a663d824aa426c26bdce6 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sun, 13 Sep 2026 16:33:20 +0000 Subject: [PATCH 1/8] feat(proof): harvest bounded per-trial harbor logs Guest summarize now fills trials[].agent_log / verifier_log from Harbor trial dirs (trial.log, then trajectory.json, then terminus_2.pane; verifier/test-stdout.txt), redacted and capped like harbor_run_tail. Co-authored-by: Mathis --- .../fixtures/harbor-trials-v1.json | 13 +- crates/proof-results/src/lib.rs | 96 +++++++++++- .../runners/rlm_fc_in_guest_harbor/README.md | 18 ++- .../harness/summarize.py | 57 ++++++- .../job/hello-world__1/agent/trajectory.json | 1 + .../jobs/job/hello-world__1/result.json | 1 + .../job/hello-world__1/verifier/reward.txt | 1 + .../hello-world__1/verifier/test-stdout.txt | 2 + .../jobs/job/no-logs__1/result.json | 1 + .../jobs/job/no-logs__1/verifier/reward.txt | 1 + .../jobs/job/pane-only__1/result.json | 1 + .../jobs/job/pane-only__1/terminus_2.pane | 1 + .../jobs/job/pane-only__1/verifier/reward.txt | 1 + .../job/traj-only__1/agent/trajectory.json | 1 + .../jobs/job/traj-only__1/result.json | 1 + .../jobs/job/traj-only__1/verifier/reward.txt | 1 + .../tests/test_summarize.py | 142 ++++++++++++++++++ .../assert-harbor-runner-results-emit.sh | 4 + docs/external-miner/proof-tbench.md | 3 + docs/external-miner/proof.md | 2 +- docs/runbooks/proof-experiment-vms.md | 4 +- 21 files changed, 339 insertions(+), 13 deletions(-) create mode 100644 deploy/guest/runners/rlm_fc_in_guest_harbor/tests/fixtures/harbor-trial-logs/jobs/job/hello-world__1/agent/trajectory.json create mode 100644 deploy/guest/runners/rlm_fc_in_guest_harbor/tests/fixtures/harbor-trial-logs/jobs/job/hello-world__1/result.json create mode 100644 deploy/guest/runners/rlm_fc_in_guest_harbor/tests/fixtures/harbor-trial-logs/jobs/job/hello-world__1/verifier/reward.txt create mode 100644 deploy/guest/runners/rlm_fc_in_guest_harbor/tests/fixtures/harbor-trial-logs/jobs/job/hello-world__1/verifier/test-stdout.txt create mode 100644 deploy/guest/runners/rlm_fc_in_guest_harbor/tests/fixtures/harbor-trial-logs/jobs/job/no-logs__1/result.json create mode 100644 deploy/guest/runners/rlm_fc_in_guest_harbor/tests/fixtures/harbor-trial-logs/jobs/job/no-logs__1/verifier/reward.txt create mode 100644 deploy/guest/runners/rlm_fc_in_guest_harbor/tests/fixtures/harbor-trial-logs/jobs/job/pane-only__1/result.json create mode 100644 deploy/guest/runners/rlm_fc_in_guest_harbor/tests/fixtures/harbor-trial-logs/jobs/job/pane-only__1/terminus_2.pane create mode 100644 deploy/guest/runners/rlm_fc_in_guest_harbor/tests/fixtures/harbor-trial-logs/jobs/job/pane-only__1/verifier/reward.txt create mode 100644 deploy/guest/runners/rlm_fc_in_guest_harbor/tests/fixtures/harbor-trial-logs/jobs/job/traj-only__1/agent/trajectory.json create mode 100644 deploy/guest/runners/rlm_fc_in_guest_harbor/tests/fixtures/harbor-trial-logs/jobs/job/traj-only__1/result.json create mode 100644 deploy/guest/runners/rlm_fc_in_guest_harbor/tests/fixtures/harbor-trial-logs/jobs/job/traj-only__1/verifier/reward.txt diff --git a/crates/proof-results/fixtures/harbor-trials-v1.json b/crates/proof-results/fixtures/harbor-trials-v1.json index 862a65bfc..52476005c 100644 --- a/crates/proof-results/fixtures/harbor-trials-v1.json +++ b/crates/proof-results/fixtures/harbor-trials-v1.json @@ -17,7 +17,14 @@ "agent_exception_policy": "zero", "harbor_exit": 0, "trials": [ - {"name": "hello-world__1", "reward": 1.0, "outcome": "measured"}, + { + "name": "hello-world__1", + "reward": 1.0, + "outcome": "measured", + "agent_log": "hello-world agent stdout: ran ls\n", + "verifier_log": "verifier: reward=1.0\n", + "log_sources": ["trial.log", "verifier/test-stdout.txt"] + }, {"name": "fix-git-diff__1", "reward": 1.0, "outcome": "measured"}, {"name": "csv-to-parquet__1", "reward": 0.0, "outcome": "measured"}, { @@ -25,7 +32,9 @@ "reward": 0.0, "outcome": "agent_exception", "exception_type": "RuntimeError", - "exception_message": "Command timed out after 120 seconds" + "exception_message": "Command timed out after 120 seconds", + "agent_log": "build-tmux agent raised during harness\n", + "log_sources": ["trial.log"] }, {"name": "nginx-config__1", "reward": 1.0, "outcome": "measured"}, {"name": "sqlite-query__1", "reward": 1.0, "outcome": "measured"}, diff --git a/crates/proof-results/src/lib.rs b/crates/proof-results/src/lib.rs index fd3a5a8cb..df339b10e 100644 --- a/crates/proof-results/src/lib.rs +++ b/crates/proof-results/src/lib.rs @@ -52,7 +52,11 @@ pub const ORCH_RESULTS_ATTACH_HINT: &str = pub const WRITE_RESULTS_EMIT: &str = "write_results_next_to_report"; /// Largest results document accepted (bytes). -pub const MAX_RESULTS_BYTES: u64 = 256 * 1024; +/// +/// Sized for a typical Harbor pack (~10 trials) with optional 8 KiB +/// `agent_log` + `verifier_log` bodies after JSON escaping. A pack that +/// still overflows is fail-closed (`TooLarge`), never truncated here. +pub const MAX_RESULTS_BYTES: u64 = 512 * 1024; /// Signed `constraints.params` key pinning the results contract id. pub const PARAM_RESULTS_CONTRACT: &str = "results_contract"; @@ -470,6 +474,7 @@ fn validate_harbor(obj: &Map, primary: f64) -> Result<(), Results )); } } + optional_trial_log_fields(t)?; rewards.push(reward); } let n_scored = uint_field(obj, "n_scored")?; @@ -513,6 +518,31 @@ fn validate_harbor(obj: &Map, primary: f64) -> Result<(), Results Ok(()) } +/// Optional per-trial Harbor logs. Absent is fine; a wrong type is not. +fn optional_trial_log_fields(t: &Map) -> Result<(), ResultsError> { + if t.get("agent_log").is_some_and(|v| !v.is_string()) { + return Err(ResultsError::Shape( + "trial.agent_log must be a string when present", + )); + } + if t.get("verifier_log").is_some_and(|v| !v.is_string()) { + return Err(ResultsError::Shape( + "trial.verifier_log must be a string when present", + )); + } + if let Some(v) = t.get("log_sources") { + let arr = v.as_array().ok_or(ResultsError::Shape( + "trial.log_sources must be an array of strings when present", + ))?; + if arr.iter().any(|item| !item.is_string()) { + return Err(ResultsError::Shape( + "trial.log_sources must be an array of strings when present", + )); + } + } + Ok(()) +} + fn str_field<'a>(obj: &'a Map, key: &'static str) -> Result<&'a str, ResultsError> { obj.get(key) .and_then(Value::as_str) @@ -621,7 +651,14 @@ mod tests { "agent_exception_policy": "zero", "harbor_exit": 0, "trials": [ - {"name": "task-a__1", "reward": 1.0, "outcome": "measured"}, + { + "name": "task-a__1", + "reward": 1.0, + "outcome": "measured", + "agent_log": "agent: hello\n", + "verifier_log": "verifier: ok\n", + "log_sources": ["trial.log", "verifier/test-stdout.txt"] + }, { "name": "task-b__1", "reward": 0.0, @@ -731,6 +768,36 @@ mod tests { ); } + #[test] + fn harbor_optional_trial_logs_are_typed() { + let b = bind(); + validate(&harbor_ok(&b), &b, None).expect("optional logs allowed"); + let mut omitted = harbor_ok(&b); + let trial0 = omitted["trials"][0].as_object_mut().expect("trial 0"); + trial0.remove("agent_log"); + trial0.remove("verifier_log"); + trial0.remove("log_sources"); + validate(&omitted, &b, None).expect("omitted logs allowed"); + let mut bad_agent = harbor_ok(&b); + bad_agent["trials"][0]["agent_log"] = serde_json::json!(1); + assert!( + validate(&bad_agent, &b, None).is_err(), + "agent_log must be a string when present" + ); + let mut bad_verifier = harbor_ok(&b); + bad_verifier["trials"][0]["verifier_log"] = serde_json::json!(true); + assert!( + validate(&bad_verifier, &b, None).is_err(), + "verifier_log must be a string when present" + ); + let mut bad_sources = harbor_ok(&b); + bad_sources["trials"][0]["log_sources"] = serde_json::json!([1]); + assert!( + validate(&bad_sources, &b, None).is_err(), + "log_sources must be strings when present" + ); + } + #[test] fn generic_requires_displayable_content() { let b = bind(); @@ -792,6 +859,31 @@ mod tests { validate(&value, &b, Some(CONTRACT_TBENCH_HARBOR)).expect("fixture"); assert_eq!(value["n_scored"], 10); assert_eq!(value["trials"].as_array().expect("trials").len(), 10); + let trials = value["trials"].as_array().expect("trials"); + assert_eq!( + trials[0]["agent_log"].as_str().expect("agent_log"), + "hello-world agent stdout: ran ls\n" + ); + assert_eq!( + trials[0]["verifier_log"].as_str().expect("verifier_log"), + "verifier: reward=1.0\n" + ); + assert_eq!( + trials[0]["log_sources"], + serde_json::json!(["trial.log", "verifier/test-stdout.txt"]) + ); + assert!( + trials[2].get("agent_log").is_none(), + "missing Harbor files stay omitted" + ); + assert_eq!( + trials[3]["exception_type"].as_str().expect("exc"), + "RuntimeError" + ); + assert_eq!( + trials[3]["agent_log"].as_str().expect("exc agent_log"), + "build-tmux agent raised during harness\n" + ); } #[test] diff --git a/deploy/guest/runners/rlm_fc_in_guest_harbor/README.md b/deploy/guest/runners/rlm_fc_in_guest_harbor/README.md index 6e9e49c7d..a33b5f595 100644 --- a/deploy/guest/runners/rlm_fc_in_guest_harbor/README.md +++ b/deploy/guest/runners/rlm_fc_in_guest_harbor/README.md @@ -279,7 +279,14 @@ passed the suffix as `model_name`. "claim_holds": true, "evidence": { "trials": [ - {"name": "task-a__1", "reward": 1.0, "outcome": "measured"}, + { + "name": "task-a__1", + "reward": 1.0, + "outcome": "measured", + "agent_log": "…", + "verifier_log": "…", + "log_sources": ["trial.log", "verifier/test-stdout.txt"] + }, {"name": "task-b__1", "reward": 0.0, "outcome": "agent_exception", "exception_type": "RuntimeError", "exception_message": "Command timed out after 120 seconds"} ], @@ -315,7 +322,14 @@ miner-authored results file is deleted with `report.json` before Harbor runs. The guest refuses Done when this file is missing or does not bind the scored report. Frontend consumers read the same object on `GET /v1/submissions/{id}` as `results` and at the artefact zip root as -`results.json`. +`results.json`. Each `trials[]` row may also carry bounded, redacted +`agent_log` / `verifier_log` harvested from Harbor's native trial dir +(`trial.log`, else `agent/trajectory.json`, else `terminus_2.pane`; +verifier `verifier/test-stdout.txt`). Missing files are omitted, never +invented. Job-level `logs.harbor_run_log` / `logs.harbor_run_tail` stay +as today. This harvest lives in the in-guest adaptor — a live pin +needs a guest rebake + RE-LOCK; tipping gateway/challenge alone does not +update `/opt/proof/runners`. `inspect` writes `$PROOF_OUTPUT_DIR/checklist.json` (no Harbor, no keys). diff --git a/deploy/guest/runners/rlm_fc_in_guest_harbor/harness/summarize.py b/deploy/guest/runners/rlm_fc_in_guest_harbor/harness/summarize.py index 6db01e040..2414b5dbc 100755 --- a/deploy/guest/runners/rlm_fc_in_guest_harbor/harness/summarize.py +++ b/deploy/guest/runners/rlm_fc_in_guest_harbor/harness/summarize.py @@ -347,6 +347,7 @@ def collect_trials( jobs_dir: Path, allow: frozenset[str] | None = None, agent_exception_policy: str = POLICY_FAIL, + secrets: list[str] | None = None, ) -> list[dict[str, Any]]: """Load every scored Harbor trial. Do not cap here — the cap is evidence only. @@ -363,12 +364,17 @@ def collect_trials( When ``allow`` is a non-empty set, trials whose name is not a filtered task (or ``task__attempt``) are dropped so a script that ran the unfiltered pack cannot score excluded ids. + + Each scored row also harvests bounded, redacted ``agent_log`` / + ``verifier_log`` from the trial dir when those files exist (FAIL / + incomplete Harbor included). Missing files are omitted. """ if agent_exception_policy not in EXCEPTION_POLICIES: _fail(f"agent_exception_policy must be one of {EXCEPTION_POLICIES}, got {agent_exception_policy!r}") by_dir: dict[str, dict[str, Any]] = {} if not jobs_dir.is_dir(): return [] + redact_secrets = secrets if secrets is not None else load_redact_values() for result_path in sorted(jobs_dir.rglob("result.json")): obj = _load_json(result_path) @@ -378,23 +384,27 @@ def collect_trials( continue reward = trial_complete_reward(trial_dir, obj) if reward is not None: - by_dir[key] = { + row: dict[str, Any] = { "name": _trial_name(obj, trial_dir), "reward": reward, "outcome": "measured", } + attach_trial_logs(row, trial_dir, redact_secrets) + by_dir[key] = row continue if agent_exception_policy != POLICY_ZERO: continue crashed = agent_phase_exception(trial_dir, obj) if crashed is None: continue - by_dir[key] = { + row = { "name": _trial_name(obj, trial_dir), "reward": 0.0, "outcome": "agent_exception", **crashed, } + attach_trial_logs(row, trial_dir, redact_secrets) + by_dir[key] = row rows = [by_dir[k] for k in sorted(by_dir)] if not allow: @@ -483,6 +493,45 @@ def read_tail(path: Path | None, secrets: list[str]) -> str: return redact(text, secrets) +def _trial_pane_path(trial_dir: Path) -> tuple[str, Path]: + """Owner lock is ``terminus_2.pane``; metal retain also has ``agent/terminus_2.pane``.""" + root_pane = trial_dir / "terminus_2.pane" + if root_pane.is_file(): + return "terminus_2.pane", root_pane + return "agent/terminus_2.pane", trial_dir / "agent" / "terminus_2.pane" + + +def attach_trial_logs(row: dict[str, Any], trial_dir: Path, secrets: list[str]) -> None: + """Fill ``agent_log`` / ``verifier_log`` from Harbor's native trial dir. + + Agent sources, first available, each already ≤ ``MAX_TAIL_CHARS`` and + redacted: ``trial.log``, then ``agent/trajectory.json``, then + ``terminus_2.pane`` (optional third). Verifier is only + ``verifier/test-stdout.txt``. Missing files are omitted — never invented. + Runs for measured, agent-exception, FAIL, and incomplete Harbor alike. + """ + sources: list[str] = [] + pane_rel, pane_path = _trial_pane_path(trial_dir) + for rel, path in ( + ("trial.log", trial_dir / "trial.log"), + ("agent/trajectory.json", trial_dir / "agent" / "trajectory.json"), + (pane_rel, pane_path), + ): + text = read_tail(path, secrets) + if not text: + continue + row["agent_log"] = text + sources.append(rel) + break + verifier_rel = "verifier/test-stdout.txt" + verifier_log = read_tail(trial_dir / "verifier" / "test-stdout.txt", secrets) + if verifier_log: + row["verifier_log"] = verifier_log + sources.append(verifier_rel) + if sources: + row["log_sources"] = sources + + def mean_reward(trials: list[dict[str, Any]]) -> float: rewards = [float(t["reward"]) for t in trials] return sum(rewards) / float(len(rewards)) @@ -685,7 +734,8 @@ def main(argv: list[str] | None = None) -> int: f"allow-tasks-dir {args.allow_tasks_dir} has no task directories; " "refusing to invent a primary_value" ) - trials = collect_trials(jobs_dir, allow, policy) + secrets = load_redact_values() + trials = collect_trials(jobs_dir, allow, policy, secrets) if not trials: _fail( f"no measured Harbor trials under {jobs_dir} " @@ -701,7 +751,6 @@ def main(argv: list[str] | None = None) -> int: f"{', '.join(missing)}; agent_exception_policy={policy}); " "refusing a partial primary_value" ) - secrets = load_redact_values() log_path = Path(args.log) if args.log else None report = build_report( trials, diff --git a/deploy/guest/runners/rlm_fc_in_guest_harbor/tests/fixtures/harbor-trial-logs/jobs/job/hello-world__1/agent/trajectory.json b/deploy/guest/runners/rlm_fc_in_guest_harbor/tests/fixtures/harbor-trial-logs/jobs/job/hello-world__1/agent/trajectory.json new file mode 100644 index 000000000..6640439c4 --- /dev/null +++ b/deploy/guest/runners/rlm_fc_in_guest_harbor/tests/fixtures/harbor-trial-logs/jobs/job/hello-world__1/agent/trajectory.json @@ -0,0 +1 @@ +{"ignored": "trajectory must not displace trial.log"} diff --git a/deploy/guest/runners/rlm_fc_in_guest_harbor/tests/fixtures/harbor-trial-logs/jobs/job/hello-world__1/result.json b/deploy/guest/runners/rlm_fc_in_guest_harbor/tests/fixtures/harbor-trial-logs/jobs/job/hello-world__1/result.json new file mode 100644 index 000000000..77f2fc125 --- /dev/null +++ b/deploy/guest/runners/rlm_fc_in_guest_harbor/tests/fixtures/harbor-trial-logs/jobs/job/hello-world__1/result.json @@ -0,0 +1 @@ +{"trial_name": "hello-world__1", "verifier_result": {"rewards": {"reward": 1.0}}} diff --git a/deploy/guest/runners/rlm_fc_in_guest_harbor/tests/fixtures/harbor-trial-logs/jobs/job/hello-world__1/verifier/reward.txt b/deploy/guest/runners/rlm_fc_in_guest_harbor/tests/fixtures/harbor-trial-logs/jobs/job/hello-world__1/verifier/reward.txt new file mode 100644 index 000000000..d3827e75a --- /dev/null +++ b/deploy/guest/runners/rlm_fc_in_guest_harbor/tests/fixtures/harbor-trial-logs/jobs/job/hello-world__1/verifier/reward.txt @@ -0,0 +1 @@ +1.0 diff --git a/deploy/guest/runners/rlm_fc_in_guest_harbor/tests/fixtures/harbor-trial-logs/jobs/job/hello-world__1/verifier/test-stdout.txt b/deploy/guest/runners/rlm_fc_in_guest_harbor/tests/fixtures/harbor-trial-logs/jobs/job/hello-world__1/verifier/test-stdout.txt new file mode 100644 index 000000000..87bd2e901 --- /dev/null +++ b/deploy/guest/runners/rlm_fc_in_guest_harbor/tests/fixtures/harbor-trial-logs/jobs/job/hello-world__1/verifier/test-stdout.txt @@ -0,0 +1,2 @@ +verifier: reward=1.0 +pytest passed diff --git a/deploy/guest/runners/rlm_fc_in_guest_harbor/tests/fixtures/harbor-trial-logs/jobs/job/no-logs__1/result.json b/deploy/guest/runners/rlm_fc_in_guest_harbor/tests/fixtures/harbor-trial-logs/jobs/job/no-logs__1/result.json new file mode 100644 index 000000000..754f2f0f0 --- /dev/null +++ b/deploy/guest/runners/rlm_fc_in_guest_harbor/tests/fixtures/harbor-trial-logs/jobs/job/no-logs__1/result.json @@ -0,0 +1 @@ +{"trial_name": "no-logs__1", "verifier_result": {"rewards": {"reward": 0.0}}} diff --git a/deploy/guest/runners/rlm_fc_in_guest_harbor/tests/fixtures/harbor-trial-logs/jobs/job/no-logs__1/verifier/reward.txt b/deploy/guest/runners/rlm_fc_in_guest_harbor/tests/fixtures/harbor-trial-logs/jobs/job/no-logs__1/verifier/reward.txt new file mode 100644 index 000000000..ba66466c2 --- /dev/null +++ b/deploy/guest/runners/rlm_fc_in_guest_harbor/tests/fixtures/harbor-trial-logs/jobs/job/no-logs__1/verifier/reward.txt @@ -0,0 +1 @@ +0.0 diff --git a/deploy/guest/runners/rlm_fc_in_guest_harbor/tests/fixtures/harbor-trial-logs/jobs/job/pane-only__1/result.json b/deploy/guest/runners/rlm_fc_in_guest_harbor/tests/fixtures/harbor-trial-logs/jobs/job/pane-only__1/result.json new file mode 100644 index 000000000..b68846966 --- /dev/null +++ b/deploy/guest/runners/rlm_fc_in_guest_harbor/tests/fixtures/harbor-trial-logs/jobs/job/pane-only__1/result.json @@ -0,0 +1 @@ +{"trial_name": "pane-only__1", "verifier_result": {"rewards": {"reward": 0.5}}} diff --git a/deploy/guest/runners/rlm_fc_in_guest_harbor/tests/fixtures/harbor-trial-logs/jobs/job/pane-only__1/terminus_2.pane b/deploy/guest/runners/rlm_fc_in_guest_harbor/tests/fixtures/harbor-trial-logs/jobs/job/pane-only__1/terminus_2.pane new file mode 100644 index 000000000..8c25fa624 --- /dev/null +++ b/deploy/guest/runners/rlm_fc_in_guest_harbor/tests/fixtures/harbor-trial-logs/jobs/job/pane-only__1/terminus_2.pane @@ -0,0 +1 @@ +pane: optional third source diff --git a/deploy/guest/runners/rlm_fc_in_guest_harbor/tests/fixtures/harbor-trial-logs/jobs/job/pane-only__1/verifier/reward.txt b/deploy/guest/runners/rlm_fc_in_guest_harbor/tests/fixtures/harbor-trial-logs/jobs/job/pane-only__1/verifier/reward.txt new file mode 100644 index 000000000..2eb3c4fe4 --- /dev/null +++ b/deploy/guest/runners/rlm_fc_in_guest_harbor/tests/fixtures/harbor-trial-logs/jobs/job/pane-only__1/verifier/reward.txt @@ -0,0 +1 @@ +0.5 diff --git a/deploy/guest/runners/rlm_fc_in_guest_harbor/tests/fixtures/harbor-trial-logs/jobs/job/traj-only__1/agent/trajectory.json b/deploy/guest/runners/rlm_fc_in_guest_harbor/tests/fixtures/harbor-trial-logs/jobs/job/traj-only__1/agent/trajectory.json new file mode 100644 index 000000000..35429b283 --- /dev/null +++ b/deploy/guest/runners/rlm_fc_in_guest_harbor/tests/fixtures/harbor-trial-logs/jobs/job/traj-only__1/agent/trajectory.json @@ -0,0 +1 @@ +{"steps": [{"role": "assistant", "content": "traj-only agent step"}]} diff --git a/deploy/guest/runners/rlm_fc_in_guest_harbor/tests/fixtures/harbor-trial-logs/jobs/job/traj-only__1/result.json b/deploy/guest/runners/rlm_fc_in_guest_harbor/tests/fixtures/harbor-trial-logs/jobs/job/traj-only__1/result.json new file mode 100644 index 000000000..11f221c68 --- /dev/null +++ b/deploy/guest/runners/rlm_fc_in_guest_harbor/tests/fixtures/harbor-trial-logs/jobs/job/traj-only__1/result.json @@ -0,0 +1 @@ +{"trial_name": "traj-only__1", "verifier_result": {"rewards": {"reward": 0.0}}} diff --git a/deploy/guest/runners/rlm_fc_in_guest_harbor/tests/fixtures/harbor-trial-logs/jobs/job/traj-only__1/verifier/reward.txt b/deploy/guest/runners/rlm_fc_in_guest_harbor/tests/fixtures/harbor-trial-logs/jobs/job/traj-only__1/verifier/reward.txt new file mode 100644 index 000000000..ba66466c2 --- /dev/null +++ b/deploy/guest/runners/rlm_fc_in_guest_harbor/tests/fixtures/harbor-trial-logs/jobs/job/traj-only__1/verifier/reward.txt @@ -0,0 +1 @@ +0.0 diff --git a/deploy/guest/runners/rlm_fc_in_guest_harbor/tests/test_summarize.py b/deploy/guest/runners/rlm_fc_in_guest_harbor/tests/test_summarize.py index f3a41537f..fbf83e374 100644 --- a/deploy/guest/runners/rlm_fc_in_guest_harbor/tests/test_summarize.py +++ b/deploy/guest/runners/rlm_fc_in_guest_harbor/tests/test_summarize.py @@ -975,5 +975,147 @@ def test_emit_results_from_truncated_report_is_fail_closed(self) -> None: self.assertFalse((root / "results.json").exists()) +class TrialLogHarvestTests(unittest.TestCase): + """Per-trial agent_log / verifier_log from Harbor's native job dir.""" + + FIXTURE_JOBS = HERE / "fixtures" / "harbor-trial-logs" / "jobs" + + def _by_name(self, trials: list[dict]) -> dict[str, dict]: + return {str(t["name"]): t for t in trials} + + def test_fixture_trial_dirs_fill_log_bodies(self) -> None: + trials = summarize.collect_trials(self.FIXTURE_JOBS) + rows = self._by_name(trials) + self.assertEqual( + set(rows), + {"hello-world__1", "traj-only__1", "pane-only__1", "no-logs__1"}, + ) + + hello = rows["hello-world__1"] + self.assertIn("hello-world agent stdout: ran ls", hello["agent_log"]) + self.assertNotIn("trajectory must not displace", hello["agent_log"]) + self.assertIn("verifier: reward=1.0", hello["verifier_log"]) + self.assertEqual( + hello["log_sources"], + ["trial.log", "verifier/test-stdout.txt"], + ) + + traj = rows["traj-only__1"] + self.assertIn("traj-only agent step", traj["agent_log"]) + self.assertEqual(traj["log_sources"], ["agent/trajectory.json"]) + self.assertNotIn("verifier_log", traj) + + pane = rows["pane-only__1"] + self.assertIn("optional third source", pane["agent_log"]) + self.assertEqual(pane["log_sources"], ["terminus_2.pane"]) + + missing = rows["no-logs__1"] + self.assertNotIn("agent_log", missing) + self.assertNotIn("verifier_log", missing) + self.assertNotIn("log_sources", missing) + + def test_results_json_carries_trial_logs_on_nonzero_harbor_exit(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + trial = root / "jobs" / "job" / "t__1" + write_complete_trial(trial, "t__1", 0.6) + (trial / "trial.log").write_text("agent ran under FAIL\n", encoding="utf-8") + (trial / "verifier" / "test-stdout.txt").write_text( + "verifier stdout on incomplete harbor\n", encoding="utf-8" + ) + out = root / "report.json" + rc = summarize.main( + [ + "--jobs-dir", + str(root / "jobs"), + "--output", + str(out), + "--harbor-exit", + "23", + ] + ) + self.assertEqual(rc, 0) + results = json.loads((root / "results.json").read_text(encoding="utf-8")) + self.assertEqual(results["harbor_exit"], 23) + self.assertEqual(results["logs"]["harbor_run_log"], "logs/harbor.run.log") + row = results["trials"][0] + self.assertEqual(row["agent_log"], "agent ran under FAIL\n") + self.assertEqual(row["verifier_log"], "verifier stdout on incomplete harbor\n") + self.assertEqual( + row["log_sources"], + ["trial.log", "verifier/test-stdout.txt"], + ) + + def test_agent_exception_keeps_exception_fields_and_harvests_logs(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + jobs = root / "jobs" + crashed = jobs / "job" / "task-c__1" + write_exception_trial(crashed, "task-c__1") + (crashed / "trial.log").write_text("harness crashed here\n", encoding="utf-8") + trials = summarize.collect_trials(jobs, None, "zero") + self.assertEqual(len(trials), 1) + row = trials[0] + self.assertEqual(row["outcome"], "agent_exception") + self.assertEqual(row["exception_type"], "RuntimeError") + self.assertIn("120 seconds", row["exception_message"]) + self.assertEqual(row["agent_log"], "harness crashed here\n") + self.assertEqual(row["log_sources"], ["trial.log"]) + + def test_nested_agent_pane_is_third_source(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + trial = root / "jobs" / "job" / "nested__1" + write_complete_trial(trial, "nested__1", 1.0) + (trial / "agent").mkdir() + (trial / "agent" / "terminus_2.pane").write_text( + "nested pane body\n", encoding="utf-8" + ) + trials = summarize.collect_trials(root / "jobs") + self.assertEqual(trials[0]["agent_log"], "nested pane body\n") + self.assertEqual(trials[0]["log_sources"], ["agent/terminus_2.pane"]) + + def test_trial_logs_are_redacted_and_capped(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + secrets = root / "secrets" + secrets.mkdir() + (secrets / "inference_key").write_text("sk-secret-owner\n", encoding="utf-8") + trial = root / "jobs" / "job" / "t__1" + write_complete_trial(trial, "t__1", 1.0) + huge = "keep-me\n" + ("x" * (summarize.MAX_TAIL_CHARS + 64)) + "sk-secret-owner tail\n" + (trial / "trial.log").write_text(huge, encoding="utf-8") + (trial / "verifier" / "test-stdout.txt").write_text( + "verifier saw sk-secret-owner\n", encoding="utf-8" + ) + import os + + os.environ["PROOF_SECRETS_DIR"] = str(secrets) + os.environ["PROOF_SECRET_FILES"] = "inference_key" + try: + trials = summarize.collect_trials(root / "jobs") + finally: + os.environ.pop("PROOF_SECRETS_DIR", None) + os.environ.pop("PROOF_SECRET_FILES", None) + row = trials[0] + self.assertNotIn("sk-secret-owner", row["agent_log"]) + self.assertIn("[REDACTED]", row["agent_log"]) + self.assertLessEqual(len(row["agent_log"]), summarize.MAX_TAIL_CHARS) + self.assertNotIn("sk-secret-owner", row["verifier_log"]) + self.assertIn("[REDACTED]", row["verifier_log"]) + + def test_empty_log_files_are_omitted_never_invented(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + trial = root / "jobs" / "job" / "t__1" + write_complete_trial(trial, "t__1", 1.0) + (trial / "trial.log").write_text("", encoding="utf-8") + (trial / "verifier" / "test-stdout.txt").write_text("", encoding="utf-8") + trials = summarize.collect_trials(root / "jobs") + self.assertNotIn("agent_log", trials[0]) + self.assertNotIn("verifier_log", trials[0]) + self.assertNotIn("log_sources", trials[0]) + + if __name__ == "__main__": unittest.main() diff --git a/deploy/scripts/assert-harbor-runner-results-emit.sh b/deploy/scripts/assert-harbor-runner-results-emit.sh index 515a5cef1..8a44f2a65 100755 --- a/deploy/scripts/assert-harbor-runner-results-emit.sh +++ b/deploy/scripts/assert-harbor-runner-results-emit.sh @@ -34,6 +34,10 @@ need "$SUMMARIZE" 'CONTRACT_HARBOR_TRIALS = "harbor-trials-v1"' need "$SUMMARIZE" 'results.json first' need "$SUMMARIZE" 'write_results_next_to_report(out, report, trials, log_tail, secrets)' need "$SUMMARIZE" 'atomic_write(out, dumped + "\n")' +need "$SUMMARIZE" 'def attach_trial_logs' +need "$SUMMARIZE" 'agent_log' +need "$SUMMARIZE" 'verifier/test-stdout.txt' +need "$SUMMARIZE" '"trial.log"' need "$LIB_SH" 'proof_require_harbor_results' need "$RUN_HARBOR" 'proof_require_harbor_results' diff --git a/docs/external-miner/proof-tbench.md b/docs/external-miner/proof-tbench.md index 5a241465f..c0702ac8d 100644 --- a/docs/external-miner/proof-tbench.md +++ b/docs/external-miner/proof-tbench.md @@ -511,6 +511,9 @@ replaces it. A scored evaluate also carries **`results`**: the complete Harbor job summary (`contract` `tbench-harbor-v1` / `harbor-trials-v1`). That is what the Arcade frontend renders — every trial's reward and outcome, +optional bounded `agent_log` / `verifier_log` (UTF-8 from Harbor's +`trial.log` / `agent/trajectory.json` / `terminus_2.pane` and +`verifier/test-stdout.txt`; missing files are omitted, never invented), `n_scored` / `n_measured` / `n_agent_exceptions`, `mean_reward` (equals `primary_value`), agent identity, and `logs.harbor_run_tail` / `logs.harbor_run_log`. It is **obligatory**: a missing or non-conforming diff --git a/docs/external-miner/proof.md b/docs/external-miner/proof.md index 2ad7d0aac..1e6360422 100644 --- a/docs/external-miner/proof.md +++ b/docs/external-miner/proof.md @@ -660,7 +660,7 @@ Known contracts: | `contract` | Extra required fields | |------------|------------------------| | `generic-custom-v1` | `display`: non-empty JSON object (not `primary_value` alone) | -| `harbor-trials-v1` / `tbench-harbor-v1` | Untruncated `trials[]` (`name`, finite `reward`, `outcome` `measured` or `agent_exception`); `n_scored` = `trials.length`; `n_measured` / `n_agent_exceptions` match those outcomes; `mean_reward` = `primary_value` = mean of trial rewards; non-empty `agent`; `logs.harbor_run_tail` and/or `logs.harbor_run_log` | +| `harbor-trials-v1` / `tbench-harbor-v1` | Untruncated `trials[]` (`name`, finite `reward`, `outcome` `measured` or `agent_exception`; optional `agent_log` / `verifier_log` / `log_sources`); `n_scored` = `trials.length`; `n_measured` / `n_agent_exceptions` match those outcomes; `mean_reward` = `primary_value` = mean of trial rewards; non-empty `agent`; `logs.harbor_run_tail` and/or `logs.harbor_run_log` | Harvest `nll` / `throughput` rows have no `results`. A red-checklist reject never ships the file. See [`tbench`](./proof-tbench.md) for the diff --git a/docs/runbooks/proof-experiment-vms.md b/docs/runbooks/proof-experiment-vms.md index 53c2fb51f..663aa45ad 100644 --- a/docs/runbooks/proof-experiment-vms.md +++ b/docs/runbooks/proof-experiment-vms.md @@ -241,8 +241,8 @@ usually lacks several — rebuild the guest kernel with them and re-pin The in-guest adaptor is **the baked image**, not the control-plane tip. `deploy/guest/runners/` on a tipped `proof-challenge` / gateway checkout is **not** what `/opt/proof/runners/` inside the Firecracker guest runs. -After any change to that tree (Harbor summarize emitting `results.json`, -`run-harbor` refuse, inspect rules, …): +After any change to that tree (Harbor summarize emitting `results.json` and +per-trial `agent_log` / `verifier_log`, `run-harbor` refuse, inspect rules, …): 1. Re-bake (`bake-rootfs.sh --runner rlm_fc_in_guest_harbor=deploy/guest/runners/rlm_fc_in_guest_harbor` plus the operator overlay / chroot-hook). 2. Stage the new ext4 on the KVM host; re-pin `PROOF_RLM_VM_IMAGE_DIGEST` (Architecte RE-LOCK — do not invent a digest). From 359f7ef940ad75fe83cc59f286fa0f7bfb1bd6f0 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sun, 13 Sep 2026 16:33:50 +0000 Subject: [PATCH 2/8] test(proof): track harbor fixture trial.log *.log is gitignored; keep the Owner retain filename in the summarize fixture via a negation so CI sees the primary agent_log source. Co-authored-by: Mathis --- .gitignore | 1 + .../harbor-trial-logs/jobs/job/hello-world__1/trial.log | 2 ++ 2 files changed, 3 insertions(+) create mode 100644 deploy/guest/runners/rlm_fc_in_guest_harbor/tests/fixtures/harbor-trial-logs/jobs/job/hello-world__1/trial.log diff --git a/.gitignore b/.gitignore index a168fad4f..d719451df 100644 --- a/.gitignore +++ b/.gitignore @@ -30,6 +30,7 @@ Thumbs.db # Local / ops noise *.log +!deploy/guest/runners/rlm_fc_in_guest_harbor/tests/fixtures/**/trial.log tmp/ .cache/ coverage/ diff --git a/deploy/guest/runners/rlm_fc_in_guest_harbor/tests/fixtures/harbor-trial-logs/jobs/job/hello-world__1/trial.log b/deploy/guest/runners/rlm_fc_in_guest_harbor/tests/fixtures/harbor-trial-logs/jobs/job/hello-world__1/trial.log new file mode 100644 index 000000000..4721bf28d --- /dev/null +++ b/deploy/guest/runners/rlm_fc_in_guest_harbor/tests/fixtures/harbor-trial-logs/jobs/job/hello-world__1/trial.log @@ -0,0 +1,2 @@ +hello-world agent stdout: ran ls +secret-should-stay-if-not-in-redact-set From 46a9ce14252a2c58dc24ddc97468242ec0ef1ef1 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sun, 13 Sep 2026 16:37:05 +0000 Subject: [PATCH 3/8] fix(proof): cap per-trial logs at 2kib under 256kib MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Keep MAX_RESULTS_BYTES at 256 KiB. 8 KiB × 2 × 33 trials would overflow; agent_log and verifier_log are last-2KiB so a full pack still binds. Co-authored-by: Mathis --- crates/proof-results/src/lib.rs | 13 +++--- .../runners/rlm_fc_in_guest_harbor/README.md | 3 +- .../harness/summarize.py | 20 ++++++--- .../tests/test_summarize.py | 45 ++++++++++++++++++- .../assert-harbor-runner-results-emit.sh | 2 +- docs/external-miner/proof-tbench.md | 2 +- 6 files changed, 68 insertions(+), 17 deletions(-) diff --git a/crates/proof-results/src/lib.rs b/crates/proof-results/src/lib.rs index df339b10e..bf2c2e960 100644 --- a/crates/proof-results/src/lib.rs +++ b/crates/proof-results/src/lib.rs @@ -53,10 +53,11 @@ pub const WRITE_RESULTS_EMIT: &str = "write_results_next_to_report"; /// Largest results document accepted (bytes). /// -/// Sized for a typical Harbor pack (~10 trials) with optional 8 KiB -/// `agent_log` + `verifier_log` bodies after JSON escaping. A pack that -/// still overflows is fail-closed (`TooLarge`), never truncated here. -pub const MAX_RESULTS_BYTES: u64 = 512 * 1024; +/// Guest Harbor logs are sized to stay under this cap at 33 scored trials: +/// 2 KiB `agent_log` + 2 KiB `verifier_log` per trial (132 KiB) plus +/// `logs.harbor_run_tail` (8 KiB) and envelope. Overflow is fail-closed +/// (`TooLarge`), never truncated here. 8 KiB per field would not fit. +pub const MAX_RESULTS_BYTES: u64 = 256 * 1024; /// Signed `constraints.params` key pinning the results contract id. pub const PARAM_RESULTS_CONTRACT: &str = "results_contract"; @@ -518,7 +519,9 @@ fn validate_harbor(obj: &Map, primary: f64) -> Result<(), Results Ok(()) } -/// Optional per-trial Harbor logs. Absent is fine; a wrong type is not. +/// Optional per-trial Harbor logs. Not required (`agent_log` / `verifier_log` +/// / `log_sources` may be omitted). A present value of the wrong JSON type +/// is fail-closed so the frontend never treats a non-string as a log body. fn optional_trial_log_fields(t: &Map) -> Result<(), ResultsError> { if t.get("agent_log").is_some_and(|v| !v.is_string()) { return Err(ResultsError::Shape( diff --git a/deploy/guest/runners/rlm_fc_in_guest_harbor/README.md b/deploy/guest/runners/rlm_fc_in_guest_harbor/README.md index a33b5f595..cd0f1b4e4 100644 --- a/deploy/guest/runners/rlm_fc_in_guest_harbor/README.md +++ b/deploy/guest/runners/rlm_fc_in_guest_harbor/README.md @@ -323,7 +323,8 @@ runs. The guest refuses Done when this file is missing or does not bind the scored report. Frontend consumers read the same object on `GET /v1/submissions/{id}` as `results` and at the artefact zip root as `results.json`. Each `trials[]` row may also carry bounded, redacted -`agent_log` / `verifier_log` harvested from Harbor's native trial dir +`agent_log` / `verifier_log` (last 2 KiB each, so 33 trials stay under the +256 KiB `results.json` cap) harvested from Harbor's native trial dir (`trial.log`, else `agent/trajectory.json`, else `terminus_2.pane`; verifier `verifier/test-stdout.txt`). Missing files are omitted, never invented. Job-level `logs.harbor_run_log` / `logs.harbor_run_tail` stay diff --git a/deploy/guest/runners/rlm_fc_in_guest_harbor/harness/summarize.py b/deploy/guest/runners/rlm_fc_in_guest_harbor/harness/summarize.py index 2414b5dbc..7497bb6d2 100755 --- a/deploy/guest/runners/rlm_fc_in_guest_harbor/harness/summarize.py +++ b/deploy/guest/runners/rlm_fc_in_guest_harbor/harness/summarize.py @@ -50,6 +50,10 @@ from typing import Any MAX_TAIL_CHARS = 8 * 1024 +# 33 trials × 2 KiB × 2 fields = 132 KiB, which fits proof-results +# MAX_RESULTS_BYTES (256 KiB) with harbor_run_tail (8 KiB) and envelope. +# 8 KiB per field would be 528 KiB and overflow that cap. +MAX_TRIAL_LOG_CHARS = 2 * 1024 MAX_EVIDENCE_TRIALS = 256 MAX_REWARD_TXT_BYTES = 64 * 1024 MAX_EXCEPTION_CHARS = 400 @@ -480,15 +484,15 @@ def redact(text: str, secrets: list[str]) -> str: return out -def read_tail(path: Path | None, secrets: list[str]) -> str: +def read_tail(path: Path | None, secrets: list[str], max_chars: int = MAX_TAIL_CHARS) -> str: if path is None or not path.is_file(): return "" try: data = path.read_bytes() except OSError: return "" - if len(data) > MAX_TAIL_CHARS: - data = data[-MAX_TAIL_CHARS:] + if len(data) > max_chars: + data = data[-max_chars:] text = data.decode("utf-8", errors="replace") return redact(text, secrets) @@ -504,8 +508,8 @@ def _trial_pane_path(trial_dir: Path) -> tuple[str, Path]: def attach_trial_logs(row: dict[str, Any], trial_dir: Path, secrets: list[str]) -> None: """Fill ``agent_log`` / ``verifier_log`` from Harbor's native trial dir. - Agent sources, first available, each already ≤ ``MAX_TAIL_CHARS`` and - redacted: ``trial.log``, then ``agent/trajectory.json``, then + Agent sources, first available, each already ≤ ``MAX_TRIAL_LOG_CHARS`` + and redacted: ``trial.log``, then ``agent/trajectory.json``, then ``terminus_2.pane`` (optional third). Verifier is only ``verifier/test-stdout.txt``. Missing files are omitted — never invented. Runs for measured, agent-exception, FAIL, and incomplete Harbor alike. @@ -517,14 +521,16 @@ def attach_trial_logs(row: dict[str, Any], trial_dir: Path, secrets: list[str]) ("agent/trajectory.json", trial_dir / "agent" / "trajectory.json"), (pane_rel, pane_path), ): - text = read_tail(path, secrets) + text = read_tail(path, secrets, MAX_TRIAL_LOG_CHARS) if not text: continue row["agent_log"] = text sources.append(rel) break verifier_rel = "verifier/test-stdout.txt" - verifier_log = read_tail(trial_dir / "verifier" / "test-stdout.txt", secrets) + verifier_log = read_tail( + trial_dir / "verifier" / "test-stdout.txt", secrets, MAX_TRIAL_LOG_CHARS + ) if verifier_log: row["verifier_log"] = verifier_log sources.append(verifier_rel) diff --git a/deploy/guest/runners/rlm_fc_in_guest_harbor/tests/test_summarize.py b/deploy/guest/runners/rlm_fc_in_guest_harbor/tests/test_summarize.py index fbf83e374..06765ffdf 100644 --- a/deploy/guest/runners/rlm_fc_in_guest_harbor/tests/test_summarize.py +++ b/deploy/guest/runners/rlm_fc_in_guest_harbor/tests/test_summarize.py @@ -1083,7 +1083,7 @@ def test_trial_logs_are_redacted_and_capped(self) -> None: (secrets / "inference_key").write_text("sk-secret-owner\n", encoding="utf-8") trial = root / "jobs" / "job" / "t__1" write_complete_trial(trial, "t__1", 1.0) - huge = "keep-me\n" + ("x" * (summarize.MAX_TAIL_CHARS + 64)) + "sk-secret-owner tail\n" + huge = "keep-me\n" + ("x" * (summarize.MAX_TRIAL_LOG_CHARS + 64)) + "sk-secret-owner tail\n" (trial / "trial.log").write_text(huge, encoding="utf-8") (trial / "verifier" / "test-stdout.txt").write_text( "verifier saw sk-secret-owner\n", encoding="utf-8" @@ -1100,10 +1100,51 @@ def test_trial_logs_are_redacted_and_capped(self) -> None: row = trials[0] self.assertNotIn("sk-secret-owner", row["agent_log"]) self.assertIn("[REDACTED]", row["agent_log"]) - self.assertLessEqual(len(row["agent_log"]), summarize.MAX_TAIL_CHARS) + self.assertLessEqual(len(row["agent_log"]), summarize.MAX_TRIAL_LOG_CHARS) self.assertNotIn("sk-secret-owner", row["verifier_log"]) self.assertIn("[REDACTED]", row["verifier_log"]) + def test_thirty_three_trials_with_max_logs_fit_results_cap(self) -> None: + """33 scored trials × 2 KiB × 2 fields must stay under 256 KiB results.json.""" + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + jobs = root / "jobs" + body_a = "A" * summarize.MAX_TRIAL_LOG_CHARS + body_v = "V" * summarize.MAX_TRIAL_LOG_CHARS + for i in range(33): + trial = jobs / "job" / f"task-{i:02d}__1" + write_complete_trial(trial, f"task-{i:02d}__1", 0.0) + (trial / "trial.log").write_text(body_a, encoding="utf-8") + (trial / "verifier" / "test-stdout.txt").write_text(body_v, encoding="utf-8") + out = root / "report.json" + rc = summarize.main( + [ + "--jobs-dir", + str(jobs), + "--output", + str(out), + "--harbor-exit", + "23", + ] + ) + self.assertEqual(rc, 0) + results_path = root / "results.json" + size = results_path.stat().st_size + self.assertLessEqual( + size, + 256 * 1024, + f"results.json {size} bytes exceeds proof-results 256 KiB cap", + ) + results = json.loads(results_path.read_text(encoding="utf-8")) + self.assertEqual(len(results["trials"]), 33) + for row in results["trials"]: + self.assertEqual(len(row["agent_log"]), summarize.MAX_TRIAL_LOG_CHARS) + self.assertEqual(len(row["verifier_log"]), summarize.MAX_TRIAL_LOG_CHARS) + self.assertEqual( + row["log_sources"], + ["trial.log", "verifier/test-stdout.txt"], + ) + def test_empty_log_files_are_omitted_never_invented(self) -> None: with tempfile.TemporaryDirectory() as tmp: root = Path(tmp) diff --git a/deploy/scripts/assert-harbor-runner-results-emit.sh b/deploy/scripts/assert-harbor-runner-results-emit.sh index 8a44f2a65..43fc42f96 100755 --- a/deploy/scripts/assert-harbor-runner-results-emit.sh +++ b/deploy/scripts/assert-harbor-runner-results-emit.sh @@ -37,7 +37,7 @@ need "$SUMMARIZE" 'atomic_write(out, dumped + "\n")' need "$SUMMARIZE" 'def attach_trial_logs' need "$SUMMARIZE" 'agent_log' need "$SUMMARIZE" 'verifier/test-stdout.txt' -need "$SUMMARIZE" '"trial.log"' +need "$SUMMARIZE" 'MAX_TRIAL_LOG_CHARS = 2 * 1024' need "$LIB_SH" 'proof_require_harbor_results' need "$RUN_HARBOR" 'proof_require_harbor_results' diff --git a/docs/external-miner/proof-tbench.md b/docs/external-miner/proof-tbench.md index c0702ac8d..da484be73 100644 --- a/docs/external-miner/proof-tbench.md +++ b/docs/external-miner/proof-tbench.md @@ -511,7 +511,7 @@ replaces it. A scored evaluate also carries **`results`**: the complete Harbor job summary (`contract` `tbench-harbor-v1` / `harbor-trials-v1`). That is what the Arcade frontend renders — every trial's reward and outcome, -optional bounded `agent_log` / `verifier_log` (UTF-8 from Harbor's +optional bounded `agent_log` / `verifier_log` (UTF-8 last 2 KiB from Harbor's `trial.log` / `agent/trajectory.json` / `terminus_2.pane` and `verifier/test-stdout.txt`; missing files are omitted, never invented), `n_scored` / `n_measured` / `n_agent_exceptions`, `mean_reward` (equals From 2f62516cf1854b14502a4fabe8bbb3d70036926d Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sun, 13 Sep 2026 16:41:23 +0000 Subject: [PATCH 4/8] fix(proof): budget-guard trial logs under 256kib Harvest prefers 8 KiB agent_log/verifier_log, shrinks to 4 KiB, then omits bodies so a 33-trial pack still scores instead of TooLarge 503. Job-level harbor_run_tail is unchanged. Co-authored-by: Mathis --- crates/proof-results/src/lib.rs | 9 ++- .../runners/rlm_fc_in_guest_harbor/README.md | 16 ++-- .../harness/summarize.py | 75 ++++++++++++++++-- .../tests/test_summarize.py | 79 +++++++++++++++++-- .../assert-harbor-runner-results-emit.sh | 4 +- docs/external-miner/proof-tbench.md | 6 +- 6 files changed, 158 insertions(+), 31 deletions(-) diff --git a/crates/proof-results/src/lib.rs b/crates/proof-results/src/lib.rs index bf2c2e960..bf9f147a3 100644 --- a/crates/proof-results/src/lib.rs +++ b/crates/proof-results/src/lib.rs @@ -53,10 +53,11 @@ pub const WRITE_RESULTS_EMIT: &str = "write_results_next_to_report"; /// Largest results document accepted (bytes). /// -/// Guest Harbor logs are sized to stay under this cap at 33 scored trials: -/// 2 KiB `agent_log` + 2 KiB `verifier_log` per trial (132 KiB) plus -/// `logs.harbor_run_tail` (8 KiB) and envelope. Overflow is fail-closed -/// (`TooLarge`), never truncated here. 8 KiB per field would not fit. +/// Guest summarize prefers 8 KiB `agent_log` / `verifier_log` and shrinks +/// to 4 KiB, then omits those bodies, so a 33-trial pack still binds here +/// instead of 503ing a paid score. Job-level `logs.harbor_run_tail` is +/// unchanged. Overflow of a document that still exceeds this cap is +/// fail-closed (`TooLarge`). pub const MAX_RESULTS_BYTES: u64 = 256 * 1024; /// Signed `constraints.params` key pinning the results contract id. diff --git a/deploy/guest/runners/rlm_fc_in_guest_harbor/README.md b/deploy/guest/runners/rlm_fc_in_guest_harbor/README.md index cd0f1b4e4..8be714ab0 100644 --- a/deploy/guest/runners/rlm_fc_in_guest_harbor/README.md +++ b/deploy/guest/runners/rlm_fc_in_guest_harbor/README.md @@ -323,14 +323,14 @@ runs. The guest refuses Done when this file is missing or does not bind the scored report. Frontend consumers read the same object on `GET /v1/submissions/{id}` as `results` and at the artefact zip root as `results.json`. Each `trials[]` row may also carry bounded, redacted -`agent_log` / `verifier_log` (last 2 KiB each, so 33 trials stay under the -256 KiB `results.json` cap) harvested from Harbor's native trial dir -(`trial.log`, else `agent/trajectory.json`, else `terminus_2.pane`; -verifier `verifier/test-stdout.txt`). Missing files are omitted, never -invented. Job-level `logs.harbor_run_log` / `logs.harbor_run_tail` stay -as today. This harvest lives in the in-guest adaptor — a live pin -needs a guest rebake + RE-LOCK; tipping gateway/challenge alone does not -update `/opt/proof/runners`. +`agent_log` / `verifier_log` (prefer last 8 KiB each, shrink to 4 KiB or +omit if the 256 KiB `results.json` cap would overflow) harvested from Harbor's +native trial dir (`trial.log`, else `agent/trajectory.json`, else +`terminus_2.pane`; verifier `verifier/test-stdout.txt`). Missing files are +omitted, never invented. Job-level `logs.harbor_run_log` / +`logs.harbor_run_tail` stay as today. This harvest lives in the in-guest +adaptor — a live pin needs a guest rebake + RE-LOCK; tipping +gateway/challenge alone does not update `/opt/proof/runners`. `inspect` writes `$PROOF_OUTPUT_DIR/checklist.json` (no Harbor, no keys). diff --git a/deploy/guest/runners/rlm_fc_in_guest_harbor/harness/summarize.py b/deploy/guest/runners/rlm_fc_in_guest_harbor/harness/summarize.py index 7497bb6d2..d5133faef 100755 --- a/deploy/guest/runners/rlm_fc_in_guest_harbor/harness/summarize.py +++ b/deploy/guest/runners/rlm_fc_in_guest_harbor/harness/summarize.py @@ -50,10 +50,12 @@ from typing import Any MAX_TAIL_CHARS = 8 * 1024 -# 33 trials × 2 KiB × 2 fields = 132 KiB, which fits proof-results -# MAX_RESULTS_BYTES (256 KiB) with harbor_run_tail (8 KiB) and envelope. -# 8 KiB per field would be 528 KiB and overflow that cap. -MAX_TRIAL_LOG_CHARS = 2 * 1024 +# Prefer 8 KiB per trial log field. 33 trials × 8 KiB × 2 overflows the +# 256 KiB CustomRunReport.results cap, so write_results_next_to_report +# shrinks to 4 KiB then omits bodies rather than 503 a paid score. +MAX_TRIAL_LOG_CHARS = 8 * 1024 +MAX_TRIAL_LOG_CHARS_STEP = 4 * 1024 +MAX_RESULTS_BYTES = 256 * 1024 MAX_EVIDENCE_TRIALS = 256 MAX_REWARD_TXT_BYTES = 64 * 1024 MAX_EXCEPTION_CHARS = 400 @@ -509,9 +511,11 @@ def attach_trial_logs(row: dict[str, Any], trial_dir: Path, secrets: list[str]) """Fill ``agent_log`` / ``verifier_log`` from Harbor's native trial dir. Agent sources, first available, each already ≤ ``MAX_TRIAL_LOG_CHARS`` - and redacted: ``trial.log``, then ``agent/trajectory.json``, then - ``terminus_2.pane`` (optional third). Verifier is only - ``verifier/test-stdout.txt``. Missing files are omitted — never invented. + (8 KiB) and redacted: ``trial.log``, then ``agent/trajectory.json``, then + ``terminus_2.pane`` (optional third). ``agent/stdout.txt`` only if those + are missing. Verifier is only ``verifier/test-stdout.txt``. Missing + files are omitted — never invented. ``fit_results_under_cap`` may later + shrink to 4 KiB or omit bodies so ``results.json`` stays under 256 KiB. Runs for measured, agent-exception, FAIL, and incomplete Harbor alike. """ sources: list[str] = [] @@ -520,6 +524,7 @@ def attach_trial_logs(row: dict[str, Any], trial_dir: Path, secrets: list[str]) ("trial.log", trial_dir / "trial.log"), ("agent/trajectory.json", trial_dir / "agent" / "trajectory.json"), (pane_rel, pane_path), + ("agent/stdout.txt", trial_dir / "agent" / "stdout.txt"), ): text = read_tail(path, secrets, MAX_TRIAL_LOG_CHARS) if not text: @@ -621,6 +626,60 @@ def build_results( } +def clip_trial_log_bodies( + trials: list[dict[str, Any]], max_chars: int | None +) -> list[dict[str, Any]]: + """Copy trials, shrinking or omitting ``agent_log`` / ``verifier_log``. + + ``max_chars`` is the last-N char cap. ``None`` omits the bodies (and + ``log_sources``) so a large pack still scores under ``MAX_RESULTS_BYTES``. + Job-level ``logs.harbor_run_tail`` is not touched here. + """ + out: list[dict[str, Any]] = [] + for trial in trials: + row = dict(trial) + if max_chars is None: + row.pop("agent_log", None) + row.pop("verifier_log", None) + row.pop("log_sources", None) + else: + for key in ("agent_log", "verifier_log"): + val = row.get(key) + if isinstance(val, str) and len(val) > max_chars: + row[key] = val[-max_chars:] + out.append(row) + return out + + +def results_payload_bytes(results: dict[str, Any], secrets: list[str]) -> int: + dumped = redact(json.dumps(results, indent=2, sort_keys=True), secrets) + return len((dumped + "\n").encode("utf-8")) + + +def fit_results_under_cap( + report: dict[str, Any], + trials: list[dict[str, Any]], + log_tail: str, + contract: str, + secrets: list[str], +) -> dict[str, Any]: + """Prefer 8 KiB trial logs; shrink to 4 KiB, then omit, rather than overflow. + + ``logs.harbor_run_tail`` stays as passed. A pack that still exceeds the + cap after omitting bodies is written as-is (envelope-only overflow is + not a log-body problem). + """ + results = build_results(report, trials, log_tail, contract) + if results_payload_bytes(results, secrets) <= MAX_RESULTS_BYTES: + return results + for cap in (MAX_TRIAL_LOG_CHARS_STEP, None): + fitted = clip_trial_log_bodies(trials, cap) + results = build_results(report, fitted, log_tail, contract) + if results_payload_bytes(results, secrets) <= MAX_RESULTS_BYTES: + return results + return results + + def write_results_next_to_report( report_path: Path, report: dict[str, Any], @@ -643,7 +702,7 @@ def write_results_next_to_report( "evidence.trials is truncated and the full trial table was not supplied; " "refusing a partial results.json" ) - results = build_results(report, trials, log_tail, contract) + results = fit_results_under_cap(report, trials, log_tail, contract, secrets) dumped = redact(json.dumps(results, indent=2, sort_keys=True), secrets) atomic_write(results_path, dumped + "\n") return results_path diff --git a/deploy/guest/runners/rlm_fc_in_guest_harbor/tests/test_summarize.py b/deploy/guest/runners/rlm_fc_in_guest_harbor/tests/test_summarize.py index 06765ffdf..a3a502ee3 100644 --- a/deploy/guest/runners/rlm_fc_in_guest_harbor/tests/test_summarize.py +++ b/deploy/guest/runners/rlm_fc_in_guest_harbor/tests/test_summarize.py @@ -1075,6 +1075,17 @@ def test_nested_agent_pane_is_third_source(self) -> None: self.assertEqual(trials[0]["agent_log"], "nested pane body\n") self.assertEqual(trials[0]["log_sources"], ["agent/terminus_2.pane"]) + def test_stdout_fallback_only_when_primaries_missing(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + trial = root / "jobs" / "job" / "t__1" + write_complete_trial(trial, "t__1", 1.0) + (trial / "agent").mkdir() + (trial / "agent" / "stdout.txt").write_text("fallback stdout\n", encoding="utf-8") + trials = summarize.collect_trials(root / "jobs") + self.assertEqual(trials[0]["agent_log"], "fallback stdout\n") + self.assertEqual(trials[0]["log_sources"], ["agent/stdout.txt"]) + def test_trial_logs_are_redacted_and_capped(self) -> None: with tempfile.TemporaryDirectory() as tmp: root = Path(tmp) @@ -1104,8 +1115,8 @@ def test_trial_logs_are_redacted_and_capped(self) -> None: self.assertNotIn("sk-secret-owner", row["verifier_log"]) self.assertIn("[REDACTED]", row["verifier_log"]) - def test_thirty_three_trials_with_max_logs_fit_results_cap(self) -> None: - """33 scored trials × 2 KiB × 2 fields must stay under 256 KiB results.json.""" + def test_thirty_three_trials_omit_logs_rather_than_overflow(self) -> None: + """33 × 8 KiB × 2 would exceed 256 KiB: omit bodies, still score (no 503).""" with tempfile.TemporaryDirectory() as tmp: root = Path(tmp) jobs = root / "jobs" @@ -1116,11 +1127,15 @@ def test_thirty_three_trials_with_max_logs_fit_results_cap(self) -> None: write_complete_trial(trial, f"task-{i:02d}__1", 0.0) (trial / "trial.log").write_text(body_a, encoding="utf-8") (trial / "verifier" / "test-stdout.txt").write_text(body_v, encoding="utf-8") + harbor_log = root / "harbor.run.log" + harbor_log.write_text("job-level harbor run tail stays\n", encoding="utf-8") out = root / "report.json" rc = summarize.main( [ "--jobs-dir", str(jobs), + "--log", + str(harbor_log), "--output", str(out), "--harbor-exit", @@ -1132,18 +1147,66 @@ def test_thirty_three_trials_with_max_logs_fit_results_cap(self) -> None: size = results_path.stat().st_size self.assertLessEqual( size, - 256 * 1024, - f"results.json {size} bytes exceeds proof-results 256 KiB cap", + summarize.MAX_RESULTS_BYTES, + f"results.json {size} bytes exceeds {summarize.MAX_RESULTS_BYTES}", ) results = json.loads(results_path.read_text(encoding="utf-8")) self.assertEqual(len(results["trials"]), 33) + self.assertEqual(results["n_scored"], 33) + self.assertAlmostEqual(results["primary_value"], 0.0) + self.assertEqual( + results["logs"]["harbor_run_tail"], + "job-level harbor run tail stays\n", + ) + for row in results["trials"]: + self.assertEqual(row["outcome"], "measured") + self.assertIn("name", row) + self.assertNotIn("agent_log", row) + self.assertNotIn("verifier_log", row) + self.assertNotIn("log_sources", row) + + def test_sixteen_trials_shrink_to_four_kib(self) -> None: + """16 × 8 KiB × 2 overflows; 4 KiB step still fits, so bodies are kept.""" + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + jobs = root / "jobs" + body_a = "A" * summarize.MAX_TRIAL_LOG_CHARS + body_v = "V" * summarize.MAX_TRIAL_LOG_CHARS + for i in range(16): + trial = jobs / "job" / f"task-{i:02d}__1" + write_complete_trial(trial, f"task-{i:02d}__1", 0.0) + (trial / "trial.log").write_text(body_a, encoding="utf-8") + (trial / "verifier" / "test-stdout.txt").write_text(body_v, encoding="utf-8") + out = root / "report.json" + rc = summarize.main(["--jobs-dir", str(jobs), "--output", str(out)]) + self.assertEqual(rc, 0) + results = json.loads((root / "results.json").read_text(encoding="utf-8")) + self.assertLessEqual((root / "results.json").stat().st_size, summarize.MAX_RESULTS_BYTES) + self.assertEqual(len(results["trials"]), 16) + for row in results["trials"]: + self.assertEqual(len(row["agent_log"]), summarize.MAX_TRIAL_LOG_CHARS_STEP) + self.assertEqual(len(row["verifier_log"]), summarize.MAX_TRIAL_LOG_CHARS_STEP) + + def test_ten_trials_keep_eight_kib_log_bodies(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + jobs = root / "jobs" + body_a = "A" * summarize.MAX_TRIAL_LOG_CHARS + body_v = "V" * summarize.MAX_TRIAL_LOG_CHARS + for i in range(10): + trial = jobs / "job" / f"task-{i:02d}__1" + write_complete_trial(trial, f"task-{i:02d}__1", 0.0) + (trial / "trial.log").write_text(body_a, encoding="utf-8") + (trial / "verifier" / "test-stdout.txt").write_text(body_v, encoding="utf-8") + out = root / "report.json" + rc = summarize.main(["--jobs-dir", str(jobs), "--output", str(out)]) + self.assertEqual(rc, 0) + results = json.loads((root / "results.json").read_text(encoding="utf-8")) + self.assertLessEqual((root / "results.json").stat().st_size, summarize.MAX_RESULTS_BYTES) + self.assertEqual(len(results["trials"]), 10) for row in results["trials"]: self.assertEqual(len(row["agent_log"]), summarize.MAX_TRIAL_LOG_CHARS) self.assertEqual(len(row["verifier_log"]), summarize.MAX_TRIAL_LOG_CHARS) - self.assertEqual( - row["log_sources"], - ["trial.log", "verifier/test-stdout.txt"], - ) def test_empty_log_files_are_omitted_never_invented(self) -> None: with tempfile.TemporaryDirectory() as tmp: diff --git a/deploy/scripts/assert-harbor-runner-results-emit.sh b/deploy/scripts/assert-harbor-runner-results-emit.sh index 43fc42f96..ce3f37944 100755 --- a/deploy/scripts/assert-harbor-runner-results-emit.sh +++ b/deploy/scripts/assert-harbor-runner-results-emit.sh @@ -37,7 +37,9 @@ need "$SUMMARIZE" 'atomic_write(out, dumped + "\n")' need "$SUMMARIZE" 'def attach_trial_logs' need "$SUMMARIZE" 'agent_log' need "$SUMMARIZE" 'verifier/test-stdout.txt' -need "$SUMMARIZE" 'MAX_TRIAL_LOG_CHARS = 2 * 1024' +need "$SUMMARIZE" '"trial.log"' +need "$SUMMARIZE" 'MAX_TRIAL_LOG_CHARS = 8 * 1024' +need "$SUMMARIZE" 'def fit_results_under_cap' need "$LIB_SH" 'proof_require_harbor_results' need "$RUN_HARBOR" 'proof_require_harbor_results' diff --git a/docs/external-miner/proof-tbench.md b/docs/external-miner/proof-tbench.md index da484be73..451475a5e 100644 --- a/docs/external-miner/proof-tbench.md +++ b/docs/external-miner/proof-tbench.md @@ -511,9 +511,11 @@ replaces it. A scored evaluate also carries **`results`**: the complete Harbor job summary (`contract` `tbench-harbor-v1` / `harbor-trials-v1`). That is what the Arcade frontend renders — every trial's reward and outcome, -optional bounded `agent_log` / `verifier_log` (UTF-8 last 2 KiB from Harbor's +optional bounded `agent_log` / `verifier_log` (UTF-8 last 8 KiB from Harbor's `trial.log` / `agent/trajectory.json` / `terminus_2.pane` and -`verifier/test-stdout.txt`; missing files are omitted, never invented), +`verifier/test-stdout.txt`; a large pack shrinks to 4 KiB or omits bodies +so `results.json` stays under 256 KiB rather than 503; missing files are +omitted, never invented), `n_scored` / `n_measured` / `n_agent_exceptions`, `mean_reward` (equals `primary_value`), agent identity, and `logs.harbor_run_tail` / `logs.harbor_run_log`. It is **obligatory**: a missing or non-conforming From d21fa0c740fe4e44bea3b328e47a0fd0a8c95c26 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sun, 13 Sep 2026 16:47:00 +0000 Subject: [PATCH 5/8] fix(proof): bound results.json by encoded json size MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit fit_results_under_cap now re-measures dumps_results UTF-8 bytes after each 8→4→2→omit step so replacement chars and JSON escapes cannot TooLarge a scored Harbor pack. generic-custom-v1 stays on 256 KiB. Co-authored-by: Mathis --- crates/proof-results/src/lib.rs | 26 +++++++--- .../runners/rlm_fc_in_guest_harbor/README.md | 4 +- .../harness/summarize.py | 42 ++++++++++------ .../tests/test_summarize.py | 48 ++++++++++++++++--- .../assert-harbor-runner-results-emit.sh | 2 + docs/external-miner/proof-tbench.md | 4 +- 6 files changed, 95 insertions(+), 31 deletions(-) diff --git a/crates/proof-results/src/lib.rs b/crates/proof-results/src/lib.rs index bf9f147a3..a166d3451 100644 --- a/crates/proof-results/src/lib.rs +++ b/crates/proof-results/src/lib.rs @@ -51,13 +51,14 @@ pub const ORCH_RESULTS_ATTACH_HINT: &str = /// `report.json`. Absence from the guest runner tree is the skew probe. pub const WRITE_RESULTS_EMIT: &str = "write_results_next_to_report"; -/// Largest results document accepted (bytes). +/// Largest results document accepted (bytes). Applies to **every** contract +/// (`generic-custom-v1` included — Harbor does not get a larger allowance). /// /// Guest summarize prefers 8 KiB `agent_log` / `verifier_log` and shrinks -/// to 4 KiB, then omits those bodies, so a 33-trial pack still binds here -/// instead of 503ing a paid score. Job-level `logs.harbor_run_tail` is -/// unchanged. Overflow of a document that still exceeds this cap is -/// fail-closed (`TooLarge`). +/// 8 → 4 → 2 KiB, then omits those bodies, measuring the encoded JSON it +/// writes so replacement chars / escapes cannot 503 a paid score. +/// Job-level `logs.harbor_run_tail` is unchanged. A document that still +/// exceeds this cap is fail-closed (`TooLarge`). pub const MAX_RESULTS_BYTES: u64 = 256 * 1024; /// Signed `constraints.params` key pinning the results contract id. @@ -275,11 +276,15 @@ pub fn runner_tree_emits_results(runner_dir: &Path) -> bool { EMIT_PROBE.iter().any(|rel| py_emits(&runner_dir.join(rel))) } +/// Max adaptor source file we will scan for [`WRITE_RESULTS_EMIT`]. +/// Not [`MAX_RESULTS_BYTES`] and not a Harbor-only results allowance. +const MAX_EMIT_PROBE_BYTES: u64 = 512 * 1024; + fn py_emits(path: &Path) -> bool { let Ok(meta) = std::fs::symlink_metadata(path) else { return false; }; - if meta.file_type().is_symlink() || !meta.is_file() || meta.len() > 512 * 1024 { + if meta.file_type().is_symlink() || !meta.is_file() || meta.len() > MAX_EMIT_PROBE_BYTES { return false; } std::fs::read_to_string(path).is_ok_and(|body| body.contains(WRITE_RESULTS_EMIT)) @@ -724,6 +729,15 @@ mod tests { )); } + #[test] + fn every_contract_shares_the_256kib_results_cap() { + assert_eq!(MAX_RESULTS_BYTES, 256 * 1024); + assert!( + MAX_EMIT_PROBE_BYTES > MAX_RESULTS_BYTES, + "adaptor source probe is not a results.json allowance" + ); + } + #[test] fn unknown_or_mismatched_contract_fails_closed() { let b = bind(); diff --git a/deploy/guest/runners/rlm_fc_in_guest_harbor/README.md b/deploy/guest/runners/rlm_fc_in_guest_harbor/README.md index 8be714ab0..9a64714f5 100644 --- a/deploy/guest/runners/rlm_fc_in_guest_harbor/README.md +++ b/deploy/guest/runners/rlm_fc_in_guest_harbor/README.md @@ -323,8 +323,8 @@ runs. The guest refuses Done when this file is missing or does not bind the scored report. Frontend consumers read the same object on `GET /v1/submissions/{id}` as `results` and at the artefact zip root as `results.json`. Each `trials[]` row may also carry bounded, redacted -`agent_log` / `verifier_log` (prefer last 8 KiB each, shrink to 4 KiB or -omit if the 256 KiB `results.json` cap would overflow) harvested from Harbor's +`agent_log` / `verifier_log` (prefer last 8 KiB each, then 4 KiB, then 2 KiB, +or omit if the **encoded** 256 KiB `results.json` would overflow) harvested from Harbor's native trial dir (`trial.log`, else `agent/trajectory.json`, else `terminus_2.pane`; verifier `verifier/test-stdout.txt`). Missing files are omitted, never invented. Job-level `logs.harbor_run_log` / diff --git a/deploy/guest/runners/rlm_fc_in_guest_harbor/harness/summarize.py b/deploy/guest/runners/rlm_fc_in_guest_harbor/harness/summarize.py index d5133faef..d2de79dd9 100755 --- a/deploy/guest/runners/rlm_fc_in_guest_harbor/harness/summarize.py +++ b/deploy/guest/runners/rlm_fc_in_guest_harbor/harness/summarize.py @@ -50,11 +50,19 @@ from typing import Any MAX_TAIL_CHARS = 8 * 1024 -# Prefer 8 KiB per trial log field. 33 trials × 8 KiB × 2 overflows the -# 256 KiB CustomRunReport.results cap, so write_results_next_to_report -# shrinks to 4 KiB then omits bodies rather than 503 a paid score. +# Prefer 8 KiB per trial log field. Acceptance is the encoded results.json +# size (json.dumps UTF-8), not the raw field cap: invalid UTF-8 replacement +# chars and JSON escapes can inflate ~8 KiB tails past 256 KiB. Shrink +# 8 → 4 → 2 KiB, then omit bodies, rather than 503 a paid score. MAX_TRIAL_LOG_CHARS = 8 * 1024 MAX_TRIAL_LOG_CHARS_STEP = 4 * 1024 +MAX_TRIAL_LOG_CHARS_STEP_2K = 2 * 1024 +TRIAL_LOG_FIT_STEPS: tuple[int | None, ...] = ( + MAX_TRIAL_LOG_CHARS, + MAX_TRIAL_LOG_CHARS_STEP, + MAX_TRIAL_LOG_CHARS_STEP_2K, + None, +) MAX_RESULTS_BYTES = 256 * 1024 MAX_EVIDENCE_TRIALS = 256 MAX_REWARD_TXT_BYTES = 64 * 1024 @@ -515,7 +523,8 @@ def attach_trial_logs(row: dict[str, Any], trial_dir: Path, secrets: list[str]) ``terminus_2.pane`` (optional third). ``agent/stdout.txt`` only if those are missing. Verifier is only ``verifier/test-stdout.txt``. Missing files are omitted — never invented. ``fit_results_under_cap`` may later - shrink to 4 KiB or omit bodies so ``results.json`` stays under 256 KiB. + shrink 8 → 4 → 2 KiB or omit bodies so the **encoded** ``results.json`` + stays under 256 KiB (JSON escapes / replacement chars included). Runs for measured, agent-exception, FAIL, and incomplete Harbor alike. """ sources: list[str] = [] @@ -651,9 +660,13 @@ def clip_trial_log_bodies( return out +def dumps_results(results: dict[str, Any], secrets: list[str]) -> str: + """Serialize the document that is written to disk (size guard must match).""" + return redact(json.dumps(results, indent=2, sort_keys=True), secrets) + "\n" + + def results_payload_bytes(results: dict[str, Any], secrets: list[str]) -> int: - dumped = redact(json.dumps(results, indent=2, sort_keys=True), secrets) - return len((dumped + "\n").encode("utf-8")) + return len(dumps_results(results, secrets).encode("utf-8")) def fit_results_under_cap( @@ -663,16 +676,16 @@ def fit_results_under_cap( contract: str, secrets: list[str], ) -> dict[str, Any]: - """Prefer 8 KiB trial logs; shrink to 4 KiB, then omit, rather than overflow. + """Keep encoded ``results.json`` ≤ ``MAX_RESULTS_BYTES``. - ``logs.harbor_run_tail`` stays as passed. A pack that still exceeds the - cap after omitting bodies is written as-is (envelope-only overflow is - not a log-body problem). + Field byte caps are only a search: re-measure ``len(json.dumps(…).encode())`` + after each step. Invalid UTF-8 replacement chars and JSON escapes can + inflate an 8 KiB tail well past 256 KiB. Steps: 8 KiB → 4 KiB → 2 KiB + → omit ``agent_log`` / ``verifier_log``. ``logs.harbor_run_tail`` is + unchanged. Envelope-only overflow after omit is not a log-body problem. """ results = build_results(report, trials, log_tail, contract) - if results_payload_bytes(results, secrets) <= MAX_RESULTS_BYTES: - return results - for cap in (MAX_TRIAL_LOG_CHARS_STEP, None): + for cap in TRIAL_LOG_FIT_STEPS: fitted = clip_trial_log_bodies(trials, cap) results = build_results(report, fitted, log_tail, contract) if results_payload_bytes(results, secrets) <= MAX_RESULTS_BYTES: @@ -703,8 +716,7 @@ def write_results_next_to_report( "refusing a partial results.json" ) results = fit_results_under_cap(report, trials, log_tail, contract, secrets) - dumped = redact(json.dumps(results, indent=2, sort_keys=True), secrets) - atomic_write(results_path, dumped + "\n") + atomic_write(results_path, dumps_results(results, secrets)) return results_path diff --git a/deploy/guest/runners/rlm_fc_in_guest_harbor/tests/test_summarize.py b/deploy/guest/runners/rlm_fc_in_guest_harbor/tests/test_summarize.py index a3a502ee3..c9c472cbd 100644 --- a/deploy/guest/runners/rlm_fc_in_guest_harbor/tests/test_summarize.py +++ b/deploy/guest/runners/rlm_fc_in_guest_harbor/tests/test_summarize.py @@ -1115,8 +1115,8 @@ def test_trial_logs_are_redacted_and_capped(self) -> None: self.assertNotIn("sk-secret-owner", row["verifier_log"]) self.assertIn("[REDACTED]", row["verifier_log"]) - def test_thirty_three_trials_omit_logs_rather_than_overflow(self) -> None: - """33 × 8 KiB × 2 would exceed 256 KiB: omit bodies, still score (no 503).""" + def test_thirty_three_trials_shrink_to_two_kib_rather_than_overflow(self) -> None: + """33 × 8 KiB overflows; 2 KiB step keeps encoded results.json under 256 KiB.""" with tempfile.TemporaryDirectory() as tmp: root = Path(tmp) jobs = root / "jobs" @@ -1160,10 +1160,8 @@ def test_thirty_three_trials_omit_logs_rather_than_overflow(self) -> None: ) for row in results["trials"]: self.assertEqual(row["outcome"], "measured") - self.assertIn("name", row) - self.assertNotIn("agent_log", row) - self.assertNotIn("verifier_log", row) - self.assertNotIn("log_sources", row) + self.assertEqual(len(row["agent_log"]), summarize.MAX_TRIAL_LOG_CHARS_STEP_2K) + self.assertEqual(len(row["verifier_log"]), summarize.MAX_TRIAL_LOG_CHARS_STEP_2K) def test_sixteen_trials_shrink_to_four_kib(self) -> None: """16 × 8 KiB × 2 overflows; 4 KiB step still fits, so bodies are kept.""" @@ -1208,6 +1206,44 @@ def test_ten_trials_keep_eight_kib_log_bodies(self) -> None: self.assertEqual(len(row["agent_log"]), summarize.MAX_TRIAL_LOG_CHARS) self.assertEqual(len(row["verifier_log"]), summarize.MAX_TRIAL_LOG_CHARS) + def test_invalid_utf8_and_escapes_use_encoded_json_size(self) -> None: + """0xFF tails JSON-escape to ~986 KiB; encoded-size guard still scores.""" + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + jobs = root / "jobs" + raw = b"\xff" * summarize.MAX_TRIAL_LOG_CHARS + escaped = (b'\\"' * 2048) + (b"\x01" * 2048) + (b"\xff" * 4096) + for i in range(10): + trial = jobs / "job" / f"task-{i:02d}__1" + write_complete_trial(trial, f"task-{i:02d}__1", 0.0) + (trial / "trial.log").write_bytes(raw) + (trial / "verifier" / "test-stdout.txt").write_bytes(escaped) + trials = summarize.collect_trials(jobs) + self.assertEqual(len(trials), 10) + self.assertIn("agent_log", trials[0]) + report = summarize.build_report( + trials, "", 0, "harbor", "", "", summarize.POLICY_FAIL + ) + inflated = summarize.build_results( + report, trials, "", summarize.CONTRACT_TBENCH + ) + self.assertGreater( + summarize.results_payload_bytes(inflated, []), + summarize.MAX_RESULTS_BYTES, + "invalid UTF-8 / escapes must inflate past 256 KiB before the guard", + ) + out = root / "report.json" + rc = summarize.main(["--jobs-dir", str(jobs), "--output", str(out)]) + self.assertEqual(rc, 0) + results_path = root / "results.json" + size = results_path.stat().st_size + self.assertLessEqual(size, summarize.MAX_RESULTS_BYTES, f"encoded {size}") + results = json.loads(results_path.read_text(encoding="utf-8")) + self.assertEqual(results["n_scored"], 10) + self.assertEqual(len(results["trials"]), 10) + self.assertAlmostEqual(results["primary_value"], 0.0) + self.assertEqual(results["logs"]["harbor_run_log"], "logs/harbor.run.log") + def test_empty_log_files_are_omitted_never_invented(self) -> None: with tempfile.TemporaryDirectory() as tmp: root = Path(tmp) diff --git a/deploy/scripts/assert-harbor-runner-results-emit.sh b/deploy/scripts/assert-harbor-runner-results-emit.sh index ce3f37944..292eead36 100755 --- a/deploy/scripts/assert-harbor-runner-results-emit.sh +++ b/deploy/scripts/assert-harbor-runner-results-emit.sh @@ -39,7 +39,9 @@ need "$SUMMARIZE" 'agent_log' need "$SUMMARIZE" 'verifier/test-stdout.txt' need "$SUMMARIZE" '"trial.log"' need "$SUMMARIZE" 'MAX_TRIAL_LOG_CHARS = 8 * 1024' +need "$SUMMARIZE" 'MAX_TRIAL_LOG_CHARS_STEP_2K = 2 * 1024' need "$SUMMARIZE" 'def fit_results_under_cap' +need "$SUMMARIZE" 'def dumps_results' need "$LIB_SH" 'proof_require_harbor_results' need "$RUN_HARBOR" 'proof_require_harbor_results' diff --git a/docs/external-miner/proof-tbench.md b/docs/external-miner/proof-tbench.md index 451475a5e..5ec1264c7 100644 --- a/docs/external-miner/proof-tbench.md +++ b/docs/external-miner/proof-tbench.md @@ -513,8 +513,8 @@ summary (`contract` `tbench-harbor-v1` / `harbor-trials-v1`). That is what the Arcade frontend renders — every trial's reward and outcome, optional bounded `agent_log` / `verifier_log` (UTF-8 last 8 KiB from Harbor's `trial.log` / `agent/trajectory.json` / `terminus_2.pane` and -`verifier/test-stdout.txt`; a large pack shrinks to 4 KiB or omits bodies -so `results.json` stays under 256 KiB rather than 503; missing files are +`verifier/test-stdout.txt`; a large pack shrinks 8 → 4 → 2 KiB or omits bodies +so the encoded `results.json` stays under 256 KiB rather than 503; missing files are omitted, never invented), `n_scored` / `n_measured` / `n_agent_exceptions`, `mean_reward` (equals `primary_value`), agent identity, and `logs.harbor_run_tail` / From 10d4f6db5080cce47c89916250f392794f4b0fa3 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sun, 13 Sep 2026 16:54:14 +0000 Subject: [PATCH 6/8] fix(proof): apply 512kib results cap only to harbor generic-custom-v1 stays at 256 KiB; tbench-harbor-v1 / harbor-trials-v1 get 512 KiB only after contract is identified. Co-authored-by: Mathis --- crates/proof-results/src/lib.rs | 228 ++++++++++++++++++++++++++++++-- 1 file changed, 215 insertions(+), 13 deletions(-) diff --git a/crates/proof-results/src/lib.rs b/crates/proof-results/src/lib.rs index a166d3451..b420fd8fc 100644 --- a/crates/proof-results/src/lib.rs +++ b/crates/proof-results/src/lib.rs @@ -51,16 +51,24 @@ pub const ORCH_RESULTS_ATTACH_HINT: &str = /// `report.json`. Absence from the guest runner tree is the skew probe. pub const WRITE_RESULTS_EMIT: &str = "write_results_next_to_report"; -/// Largest results document accepted (bytes). Applies to **every** contract -/// (`generic-custom-v1` included — Harbor does not get a larger allowance). +/// Default results document cap (bytes). [`CONTRACT_GENERIC`] and any +/// unidentified `contract` stay on this prior limit. /// /// Guest summarize prefers 8 KiB `agent_log` / `verifier_log` and shrinks /// 8 → 4 → 2 KiB, then omits those bodies, measuring the encoded JSON it /// writes so replacement chars / escapes cannot 503 a paid score. /// Job-level `logs.harbor_run_tail` is unchanged. A document that still -/// exceeds this cap is fail-closed (`TooLarge`). +/// exceeds the **selected** contract's cap is fail-closed (`TooLarge`). pub const MAX_RESULTS_BYTES: u64 = 256 * 1024; +/// Harbor family cap (bytes): [`CONTRACT_HARBOR_TRIALS`] / +/// [`CONTRACT_TBENCH_HARBOR`] only, after `contract` is identified. +/// +/// [`load_file`] may *read* up to this ceiling so a Harbor document between +/// [`MAX_RESULTS_BYTES`] and this size can parse; generic-custom-v1 over +/// [`MAX_RESULTS_BYTES`] is still `TooLarge`. +pub const MAX_HARBOR_RESULTS_BYTES: u64 = 512 * 1024; + /// Signed `constraints.params` key pinning the results contract id. pub const PARAM_RESULTS_CONTRACT: &str = "results_contract"; @@ -88,11 +96,13 @@ pub enum ResultsError { /// File missing or unreadable. #[error("results json: {0}")] Io(String), - /// Over the size cap. - #[error("results json is {got} bytes (cap {MAX_RESULTS_BYTES})")] + /// Over the size cap selected for this document's contract. + #[error("results json is {got} bytes (cap {cap})")] TooLarge { /// Observed size. got: u64, + /// Cap that applied (`MAX_RESULTS_BYTES` or Harbor's larger allowance). + cap: u64, }, /// JSON did not parse or was not an object. #[error("results json: {0}")] @@ -183,6 +193,38 @@ pub fn known_contract(id: &str) -> Option { } } +/// Byte cap for a results document **after** its `contract` is known. +/// +/// Harbor ids get [`MAX_HARBOR_RESULTS_BYTES`]. Everything else, including +/// an unknown or missing id, keeps [`MAX_RESULTS_BYTES`]. +#[must_use] +pub fn results_size_cap(contract_id: &str) -> u64 { + match known_contract(contract_id) { + Some(Contract::HarborTrials) => MAX_HARBOR_RESULTS_BYTES, + Some(Contract::Generic) | None => MAX_RESULTS_BYTES, + } +} + +fn results_size_cap_for_value(value: &Value) -> u64 { + value + .get("contract") + .and_then(Value::as_str) + .map_or(MAX_RESULTS_BYTES, results_size_cap) +} + +fn reject_too_large(got: u64, cap: u64) -> Result<(), ResultsError> { + if got > cap { + Err(ResultsError::TooLarge { got, cap }) + } else { + Ok(()) + } +} + +fn encoded_len(value: &Value) -> Result { + let bytes = serde_json::to_vec(value).map_err(|e| ResultsError::Parse(e.to_string()))?; + Ok(u64::try_from(bytes.len()).unwrap_or(u64::MAX)) +} + /// Single path segment, ends with `.json`, conservative charset. #[must_use] pub fn is_results_file_name(name: &str) -> bool { @@ -277,7 +319,7 @@ pub fn runner_tree_emits_results(runner_dir: &Path) -> bool { } /// Max adaptor source file we will scan for [`WRITE_RESULTS_EMIT`]. -/// Not [`MAX_RESULTS_BYTES`] and not a Harbor-only results allowance. +/// Not a results.json allowance (generic stays [`MAX_RESULTS_BYTES`]). const MAX_EMIT_PROBE_BYTES: u64 = 512 * 1024; fn py_emits(path: &Path) -> bool { @@ -325,11 +367,16 @@ pub fn load_file(path: &Path) -> Result { let meta = std::fs::metadata(path).map_err(|e| { ResultsError::Io(format!("{}: {e}", missing_results_detail(name, Some(path)))) })?; - if meta.len() > MAX_RESULTS_BYTES { - return Err(ResultsError::TooLarge { got: meta.len() }); - } + // Hard read ceiling is Harbor's max so a 257–512 KiB Harbor file can + // parse. Acceptance of that size happens only after `contract` is a + // Harbor id — generic-custom-v1 keeps MAX_RESULTS_BYTES. + reject_too_large(meta.len(), MAX_HARBOR_RESULTS_BYTES)?; let body = std::fs::read_to_string(path).map_err(|e| ResultsError::Io(e.to_string()))?; - parse_results(&body) + let got = u64::try_from(body.len()).unwrap_or(u64::MAX); + reject_too_large(got, MAX_HARBOR_RESULTS_BYTES)?; + let value = parse_results(&body)?; + reject_too_large(got, results_size_cap_for_value(&value))?; + Ok(value) } /// Parse results JSON text. @@ -358,6 +405,7 @@ pub fn validate( let contract = str_field(obj, "contract")?; let family = known_contract(contract) .ok_or_else(|| ResultsError::UnknownContract(contract.to_owned()))?; + reject_too_large(encoded_len(value)?, results_size_cap(contract))?; if let Some(pin) = pinned.map(str::trim).filter(|s| !s.is_empty()) { let pin_fam = known_contract(pin).ok_or_else(|| ResultsError::UnknownContract(pin.to_owned()))?; @@ -730,11 +778,165 @@ mod tests { } #[test] - fn every_contract_shares_the_256kib_results_cap() { + fn generic_keeps_256kib_harbor_gets_512kib_after_contract() { assert_eq!(MAX_RESULTS_BYTES, 256 * 1024); + assert_eq!(MAX_HARBOR_RESULTS_BYTES, 512 * 1024); + assert_eq!(results_size_cap(CONTRACT_GENERIC), MAX_RESULTS_BYTES); + assert_eq!( + results_size_cap(CONTRACT_HARBOR_TRIALS), + MAX_HARBOR_RESULTS_BYTES + ); + assert_eq!( + results_size_cap(CONTRACT_TBENCH_HARBOR), + MAX_HARBOR_RESULTS_BYTES + ); + assert_eq!(results_size_cap("not-a-contract"), MAX_RESULTS_BYTES); + // Adaptor source scan; coincidentally 512 KiB, not generic-custom-v1's cap. + assert_eq!(MAX_EMIT_PROBE_BYTES, 512 * 1024); + assert_eq!(MAX_EMIT_PROBE_BYTES, MAX_HARBOR_RESULTS_BYTES); + } + + fn scratch_dir(tag: &str) -> PathBuf { + let dir = std::env::temp_dir().join(format!("proof-results-{tag}-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).expect("dir"); + dir + } + + fn write_contract_sized(dir: &Path, contract: &str, size: usize) -> PathBuf { + let path = dir.join("results.json"); + let prefix = format!("{{\"contract\":\"{contract}\",\"pad\":\""); + let suffix = "\"}"; + assert!(size >= prefix.len() + suffix.len()); + let mut body = String::with_capacity(size); + body.push_str(&prefix); + body.extend(std::iter::repeat_n('x', size - prefix.len() - suffix.len())); + body.push_str(suffix); + assert_eq!(body.len(), size); + std::fs::write(&path, body).expect("write"); + path + } + + fn with_encoded_padding(mut value: Value, min_bytes: usize) -> Value { + let mut n = 1usize; + loop { + value["padding"] = Value::String("x".repeat(n)); + let encoded = serde_json::to_vec(&value).expect("encode"); + if encoded.len() >= min_bytes { + return value; + } + n = n.saturating_add(min_bytes.saturating_sub(encoded.len()).saturating_add(8)); + } + } + + #[test] + fn load_file_generic_rejects_over_256kib_under_harbor_ceiling() { + let dir = scratch_dir("generic-over"); + let over = usize::try_from(MAX_RESULTS_BYTES).expect("cap") + 1; + let path = write_contract_sized(&dir, CONTRACT_GENERIC, over); + let err = load_file(&path).expect_err("generic over 256"); + assert!( + matches!( + err, + ResultsError::TooLarge { + cap: MAX_RESULTS_BYTES, + .. + } + ), + "{err}" + ); + let _ = std::fs::remove_dir_all(&dir); + } + + #[test] + fn load_file_harbor_accepts_between_256kib_and_512kib() { + let dir = scratch_dir("harbor-mid"); + let mid = usize::try_from(MAX_RESULTS_BYTES).expect("cap") + 1; + let path = write_contract_sized(&dir, CONTRACT_HARBOR_TRIALS, mid); + load_file(&path).expect("harbor mid-size parses"); + let alias = scratch_dir("harbor-alias"); + let path = write_contract_sized(&alias, CONTRACT_TBENCH_HARBOR, mid); + load_file(&path).expect("tbench-harbor alias mid-size parses"); + let _ = std::fs::remove_dir_all(&dir); + let _ = std::fs::remove_dir_all(&alias); + } + + #[test] + fn load_file_unknown_contract_keeps_generic_cap() { + let dir = scratch_dir("unknown-over"); + let over = usize::try_from(MAX_RESULTS_BYTES).expect("cap") + 1; + let path = write_contract_sized(&dir, "not-a-contract", over); + let err = load_file(&path).expect_err("unknown is not Harbor"); + assert!( + matches!( + err, + ResultsError::TooLarge { + cap: MAX_RESULTS_BYTES, + .. + } + ), + "{err}" + ); + let _ = std::fs::remove_dir_all(&dir); + } + + #[test] + fn load_file_hard_ceiling_is_harbor_max_before_parse() { + let dir = scratch_dir("ceiling"); + let path = dir.join("results.json"); + let over = usize::try_from(MAX_HARBOR_RESULTS_BYTES).expect("cap") + 1; + std::fs::write(&path, vec![b'x'; over]).expect("write"); + let err = load_file(&path).expect_err("over Harbor ceiling"); assert!( - MAX_EMIT_PROBE_BYTES > MAX_RESULTS_BYTES, - "adaptor source probe is not a results.json allowance" + matches!( + err, + ResultsError::TooLarge { + cap: MAX_HARBOR_RESULTS_BYTES, + .. + } + ), + "{err}" + ); + let _ = std::fs::remove_dir_all(&dir); + } + + #[test] + fn validate_applies_family_cap_after_contract() { + let b = bind(); + let generic_over = with_encoded_padding( + generic_document(&b, &serde_json::json!({"ok": true})), + usize::try_from(MAX_RESULTS_BYTES).expect("cap") + 1, + ); + let err = validate(&generic_over, &b, None).expect_err("generic over"); + assert!( + matches!( + err, + ResultsError::TooLarge { + cap: MAX_RESULTS_BYTES, + .. + } + ), + "{err}" + ); + let harbor_mid = with_encoded_padding( + harbor_ok(&b), + usize::try_from(MAX_RESULTS_BYTES).expect("cap") + 1, + ); + validate(&harbor_mid, &b, None).expect("harbor mid-size still binds"); + let harbor_over = with_encoded_padding( + harbor_ok(&b), + usize::try_from(MAX_HARBOR_RESULTS_BYTES).expect("cap") + 1, + ); + let err = validate(&harbor_over, &b, None).expect_err("harbor over 512"); + assert!( + matches!( + err, + ResultsError::TooLarge { + cap: MAX_HARBOR_RESULTS_BYTES, + .. + } + ), + "{err}" ); } From eaea6aa01fee83689c8d8aad7fc9aca9aba1277d Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sun, 13 Sep 2026 17:04:13 +0000 Subject: [PATCH 7/8] fix(proof): redact harbor logs before tail cut read_tail now redacts an overlap window (tail plus longest secret) before taking the last N chars so a credential that straddles the cut cannot leak into agent_log, verifier_log, or harbor_run_tail. Co-authored-by: Mathis --- .../harness/summarize.py | 38 +++++-- .../tests/test_summarize.py | 101 ++++++++++++++++++ 2 files changed, 130 insertions(+), 9 deletions(-) diff --git a/deploy/guest/runners/rlm_fc_in_guest_harbor/harness/summarize.py b/deploy/guest/runners/rlm_fc_in_guest_harbor/harness/summarize.py index d2de79dd9..2c5509bf5 100755 --- a/deploy/guest/runners/rlm_fc_in_guest_harbor/harness/summarize.py +++ b/deploy/guest/runners/rlm_fc_in_guest_harbor/harness/summarize.py @@ -494,17 +494,36 @@ def redact(text: str, secrets: list[str]) -> str: return out +def _tail_window_bytes(max_chars: int, secrets: list[str]) -> int: + """Bytes to read from EOF: tail plus overlap so a secret can still match.""" + overlap = max((len(s.encode("utf-8")) for s in secrets if s), default=0) + return max_chars + overlap + 3 + + def read_tail(path: Path | None, secrets: list[str], max_chars: int = MAX_TAIL_CHARS) -> str: - if path is None or not path.is_file(): + """Last ``max_chars`` of the log **after** secrets are blanked. + + A credential that starts before the tail cut would otherwise survive as + a suffix that no longer equals the configured value. Read an overlap + window (≥ longest secret, plus 3 bytes for a UTF-8 split), redact + that buffer, then take the last N chars. Trial ``agent_log`` / + ``verifier_log`` and job-level ``harbor_run_tail`` share this helper. + """ + if path is None or not path.is_file() or max_chars <= 0: return "" + window = _tail_window_bytes(max_chars, secrets) try: - data = path.read_bytes() + size = path.stat().st_size + with path.open("rb") as fh: + if size > window: + fh.seek(-window, os.SEEK_END) + data = fh.read() except OSError: return "" - if len(data) > max_chars: - data = data[-max_chars:] - text = data.decode("utf-8", errors="replace") - return redact(text, secrets) + redacted = redact(data.decode("utf-8", errors="replace"), secrets) + if len(redacted) > max_chars: + return redacted[-max_chars:] + return redacted def _trial_pane_path(trial_dir: Path) -> tuple[str, Path]: @@ -640,9 +659,10 @@ def clip_trial_log_bodies( ) -> list[dict[str, Any]]: """Copy trials, shrinking or omitting ``agent_log`` / ``verifier_log``. - ``max_chars`` is the last-N char cap. ``None`` omits the bodies (and - ``log_sources``) so a large pack still scores under ``MAX_RESULTS_BYTES``. - Job-level ``logs.harbor_run_tail`` is not touched here. + ``max_chars`` is the last-N char cap on **already redacted** bodies. + ``None`` omits the bodies (and ``log_sources``) so a large pack still + scores under ``MAX_RESULTS_BYTES``. Job-level ``logs.harbor_run_tail`` + is not touched here. """ out: list[dict[str, Any]] = [] for trial in trials: diff --git a/deploy/guest/runners/rlm_fc_in_guest_harbor/tests/test_summarize.py b/deploy/guest/runners/rlm_fc_in_guest_harbor/tests/test_summarize.py index c9c472cbd..6faffa321 100644 --- a/deploy/guest/runners/rlm_fc_in_guest_harbor/tests/test_summarize.py +++ b/deploy/guest/runners/rlm_fc_in_guest_harbor/tests/test_summarize.py @@ -1115,6 +1115,107 @@ def test_trial_logs_are_redacted_and_capped(self) -> None: self.assertNotIn("sk-secret-owner", row["verifier_log"]) self.assertIn("[REDACTED]", row["verifier_log"]) + def _secret_straddling_cut(self, secret: str, max_chars: int, into: int = 4) -> tuple[str, str]: + """File body where a naive last-``max_chars`` slice keeps only a suffix.""" + into = min(into, len(secret) - 1) + leaked = secret[into:] + suffix = "Z" * (max_chars - len(leaked)) + body = ("X" * (max_chars + 64)) + secret + suffix + naive = body[-max_chars:] + self.assertIn(leaked, naive) + self.assertNotIn(secret, naive) + return body, leaked + + def test_read_tail_redacts_before_cutting(self) -> None: + secret = "sk-secret-owner" + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "trial.log" + body, leaked = self._secret_straddling_cut( + secret, summarize.MAX_TRIAL_LOG_CHARS + ) + path.write_text(body, encoding="utf-8") + out = summarize.read_tail(path, [secret], summarize.MAX_TRIAL_LOG_CHARS) + self.assertNotIn(secret, out) + self.assertNotIn(leaked, out) + self.assertIn(summarize.REDACTED, out) + self.assertLessEqual(len(out), summarize.MAX_TRIAL_LOG_CHARS) + + def test_trial_logs_redact_secret_only_in_last_k_of_larger_file(self) -> None: + secret = "sk-secret-owner" + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + trial = root / "jobs" / "job" / "t__1" + write_complete_trial(trial, "t__1", 1.0) + agent_body, leaked_a = self._secret_straddling_cut( + secret, summarize.MAX_TRIAL_LOG_CHARS + ) + verifier_body, leaked_v = self._secret_straddling_cut( + secret, summarize.MAX_TRIAL_LOG_CHARS, into=6 + ) + (trial / "trial.log").write_text(agent_body, encoding="utf-8") + (trial / "verifier" / "test-stdout.txt").write_text( + verifier_body, encoding="utf-8" + ) + trials = summarize.collect_trials(root / "jobs", secrets=[secret]) + row = trials[0] + for field, leaked in (("agent_log", leaked_a), ("verifier_log", leaked_v)): + text = row[field] + self.assertNotIn(secret, text, field) + self.assertNotIn(leaked, text, field) + self.assertLessEqual(len(text), summarize.MAX_TRIAL_LOG_CHARS, field) + self.assertIn("REDACTED", text, field) + + def test_harbor_run_tail_redacts_secret_straddling_cut(self) -> None: + secret = "sk-secret-owner" + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + jobs = root / "jobs" + write_complete_trial(jobs / "job" / "t__1", "t__1", 1.0) + body, leaked = self._secret_straddling_cut(secret, summarize.MAX_TAIL_CHARS) + log = root / "harbor.log" + log.write_text(body, encoding="utf-8") + out = root / "report.json" + secrets_dir = root / "secrets" + secrets_dir.mkdir() + (secrets_dir / "inference_key").write_text(secret + "\n", encoding="utf-8") + import os + + saved = dict(os.environ) + os.environ["PROOF_SECRETS_DIR"] = str(secrets_dir) + os.environ["PROOF_SECRET_FILES"] = "inference_key" + try: + rc = summarize.main( + [ + "--jobs-dir", + str(jobs), + "--log", + str(log), + "--output", + str(out), + "--harbor-exit", + "0", + "--agent", + "agent.agent:MinerAgent", + "--agent-source", + "artifact_dir/agent", + ] + ) + finally: + os.environ.clear() + os.environ.update(saved) + self.assertEqual(rc, 0) + report = json.loads(out.read_text(encoding="utf-8")) + results = json.loads((root / "results.json").read_text(encoding="utf-8")) + for blob in ( + report["evidence"]["harbor_run_tail"], + results["logs"]["harbor_run_tail"], + out.read_text(encoding="utf-8"), + (root / "results.json").read_text(encoding="utf-8"), + ): + self.assertNotIn(secret, blob) + self.assertNotIn(leaked, blob) + self.assertIn(summarize.REDACTED, results["logs"]["harbor_run_tail"]) + def test_thirty_three_trials_shrink_to_two_kib_rather_than_overflow(self) -> None: """33 × 8 KiB overflows; 2 KiB step keeps encoded results.json under 256 KiB.""" with tempfile.TemporaryDirectory() as tmp: From 9dd2d13e06d974c78d3e4393bb01795c4106a406 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sun, 13 Sep 2026 17:13:16 +0000 Subject: [PATCH 8/8] fix(proof): redact decoded harbor logs then char-tail read_tail decodes up to 1 MiB from EOF, redacts that text, then keeps the last N characters so a UTF-8 continuation split cannot leak a secret suffix into agent_log, verifier_log, or harbor_run_tail. Co-authored-by: Mathis --- .../harness/summarize.py | 31 +++++++------- .../tests/test_summarize.py | 42 +++++++++++++++++++ 2 files changed, 57 insertions(+), 16 deletions(-) diff --git a/deploy/guest/runners/rlm_fc_in_guest_harbor/harness/summarize.py b/deploy/guest/runners/rlm_fc_in_guest_harbor/harness/summarize.py index 2c5509bf5..1aea38881 100755 --- a/deploy/guest/runners/rlm_fc_in_guest_harbor/harness/summarize.py +++ b/deploy/guest/runners/rlm_fc_in_guest_harbor/harness/summarize.py @@ -50,6 +50,11 @@ from typing import Any MAX_TAIL_CHARS = 8 * 1024 +# Hard cap on bytes read from EOF before decode/redact. Last ``max_chars`` +# UTF-8 characters occupy at most 4× that many bytes; 1 MiB is well above +# 8 KiB tails plus any realistic secret, so a char-tail never starts before +# this window. Do not size the read from ``max_chars`` in bytes. +MAX_TAIL_READ_BYTES = 1024 * 1024 # Prefer 8 KiB per trial log field. Acceptance is the encoded results.json # size (json.dumps UTF-8), not the raw field cap: invalid UTF-8 replacement # chars and JSON escapes can inflate ~8 KiB tails past 256 KiB. Shrink @@ -494,29 +499,23 @@ def redact(text: str, secrets: list[str]) -> str: return out -def _tail_window_bytes(max_chars: int, secrets: list[str]) -> int: - """Bytes to read from EOF: tail plus overlap so a secret can still match.""" - overlap = max((len(s.encode("utf-8")) for s in secrets if s), default=0) - return max_chars + overlap + 3 - - def read_tail(path: Path | None, secrets: list[str], max_chars: int = MAX_TAIL_CHARS) -> str: - """Last ``max_chars`` of the log **after** secrets are blanked. - - A credential that starts before the tail cut would otherwise survive as - a suffix that no longer equals the configured value. Read an overlap - window (≥ longest secret, plus 3 bytes for a UTF-8 split), redact - that buffer, then take the last N chars. Trial ``agent_log`` / - ``verifier_log`` and job-level ``harbor_run_tail`` share this helper. + """Last ``max_chars`` **characters** of the log after secrets are blanked. + + Decode first (lossy UTF-8 is OK for display), redact that decoded text, + then take the last N characters. Never pick the tail by a raw byte + offset of the original file: a ``max_chars``-byte window can start in + the middle of a UTF-8 sequence or of a secret, so the decoded suffix + would not match and would leak. Trial ``agent_log`` / ``verifier_log`` + and job-level ``harbor_run_tail`` share this helper. """ if path is None or not path.is_file() or max_chars <= 0: return "" - window = _tail_window_bytes(max_chars, secrets) try: size = path.stat().st_size with path.open("rb") as fh: - if size > window: - fh.seek(-window, os.SEEK_END) + if size > MAX_TAIL_READ_BYTES: + fh.seek(-MAX_TAIL_READ_BYTES, os.SEEK_END) data = fh.read() except OSError: return "" diff --git a/deploy/guest/runners/rlm_fc_in_guest_harbor/tests/test_summarize.py b/deploy/guest/runners/rlm_fc_in_guest_harbor/tests/test_summarize.py index 6faffa321..990b81939 100644 --- a/deploy/guest/runners/rlm_fc_in_guest_harbor/tests/test_summarize.py +++ b/deploy/guest/runners/rlm_fc_in_guest_harbor/tests/test_summarize.py @@ -1140,6 +1140,48 @@ def test_read_tail_redacts_before_cutting(self) -> None: self.assertIn(summarize.REDACTED, out) self.assertLessEqual(len(out), summarize.MAX_TRIAL_LOG_CHARS) + def test_read_tail_redacts_secret_split_on_utf8_continuation(self) -> None: + """A max_chars-byte slice that starts mid-sequence must not leak the secret. + + ``é`` / ``秘`` are 2- and 3-byte UTF-8. A last-N-**byte** window that + begins on a continuation byte decodes to a replacement + suffix that + no longer equals the configured secret. Decode+redact-then-char-tail + must still blank it. + """ + secret = "sk-clé-秘密-owner" + leaked_suffix = "秘密-owner" + sb = secret.encode("utf-8") + split_at = next(i for i, b in enumerate(sb) if b & 0xC0 == 0x80) + max_chars = summarize.MAX_TRIAL_LOG_CHARS + suffix = b"Z" * (max_chars - (len(sb) - split_at)) + raw = ("\N{GRINNING FACE}" * 256).encode("utf-8") + sb + suffix + naive = raw[-max_chars:] + self.assertEqual(naive[0] & 0xC0, 0x80) + naive_text = naive.decode("utf-8", errors="replace") + self.assertNotIn(secret, naive_text) + self.assertIn(leaked_suffix, naive_text) + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "trial.log" + path.write_bytes(raw) + out = summarize.read_tail(path, [secret], max_chars) + self.assertNotIn(secret, out) + self.assertNotIn(leaked_suffix, out) + self.assertIn(summarize.REDACTED, out) + self.assertLessEqual(len(out), max_chars) + + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + trial = root / "jobs" / "job" / "t__1" + write_complete_trial(trial, "t__1", 1.0) + (trial / "trial.log").write_bytes(raw) + (trial / "verifier" / "test-stdout.txt").write_bytes(raw) + trials = summarize.collect_trials(root / "jobs", secrets=[secret]) + row = trials[0] + for field in ("agent_log", "verifier_log"): + text = row[field] + self.assertNotIn(secret, text, field) + self.assertNotIn(leaked_suffix, text, field) + def test_trial_logs_redact_secret_only_in_last_k_of_larger_file(self) -> None: secret = "sk-secret-owner" with tempfile.TemporaryDirectory() as tmp: