From dca13975e3e3ba02ae2c45a4f0568fe93f394796 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Thu, 10 Sep 2026 11:36:52 +0000 Subject: [PATCH] fix(proof): fail closed when inspect skips artefact files Oversized, unreadable, or binary regular files mark the scan incomplete so off-limits rules cannot pass on uninspected contents. Co-authored-by: Mathis --- .../runners/rlm_fc_in_guest_harbor/README.md | 3 +- .../rlm_fc_in_guest_harbor/inspect_scan.py | 15 +- .../tests/test_inspect_scan.py | 133 ++++++++++++++++++ 3 files changed, 146 insertions(+), 5 deletions(-) diff --git a/deploy/guest/runners/rlm_fc_in_guest_harbor/README.md b/deploy/guest/runners/rlm_fc_in_guest_harbor/README.md index c11819f02..13fc6a80a 100644 --- a/deploy/guest/runners/rlm_fc_in_guest_harbor/README.md +++ b/deploy/guest/runners/rlm_fc_in_guest_harbor/README.md @@ -82,7 +82,8 @@ Off-limits in the artefact (inspect fails the named rule): - `no_eval_short_circuit` (and `skip_eval` / `skip_verifier` / `always_pass_eval` / `short_circuit_eval`) - `no_tb4_hardcoding` (and `tb4_answers` / `hardcoded_tb4`) -A file/byte-limit truncation marks the scan incomplete and fails those +A file/byte-limit truncation, or a regular file that is oversized, +unreadable, or binary, marks the scan incomplete and fails those off-limits rules. Unknown rule ids fail closed. Do not quote those markers in miner code or README inside the tar. diff --git a/deploy/guest/runners/rlm_fc_in_guest_harbor/inspect_scan.py b/deploy/guest/runners/rlm_fc_in_guest_harbor/inspect_scan.py index 21d98fccf..151701c84 100755 --- a/deploy/guest/runners/rlm_fc_in_guest_harbor/inspect_scan.py +++ b/deploy/guest/runners/rlm_fc_in_guest_harbor/inspect_scan.py @@ -78,9 +78,10 @@ def load_rules(path: Path) -> list[dict[str, str]]: def collect_artefact_text(root: Path | None) -> tuple[str, int, list[str], bool]: """Return ``(text, n_scanned, names, incomplete)``. - ``incomplete`` is true when a file or byte cap stopped the walk before - every regular file was considered. Callers must not treat a truncated - scan as proof that an off-limits marker is absent. + ``incomplete`` is true when a file or byte cap stopped the walk, or a + regular file was oversized / unreadable so its contents were not + inspected. Callers must not treat that absence as a clean off-limits + pass. """ if root is None or not root.is_dir(): return "", 0, [], False @@ -103,14 +104,20 @@ def collect_artefact_text(root: Path | None) -> tuple[str, int, list[str], bool] try: size = path.stat().st_size except OSError: + incomplete = True + continue + if size == 0: continue - if size == 0 or size > MAX_FILE_BYTES: + if size > MAX_FILE_BYTES: + incomplete = True continue try: data = path.read_bytes() except OSError: + incomplete = True continue if b"\x00" in data[:1024]: + incomplete = True continue total += len(data) blobs.append(data.decode("utf-8", errors="replace")) diff --git a/deploy/guest/runners/rlm_fc_in_guest_harbor/tests/test_inspect_scan.py b/deploy/guest/runners/rlm_fc_in_guest_harbor/tests/test_inspect_scan.py index 8cd49667d..17c5030f8 100644 --- a/deploy/guest/runners/rlm_fc_in_guest_harbor/tests/test_inspect_scan.py +++ b/deploy/guest/runners/rlm_fc_in_guest_harbor/tests/test_inspect_scan.py @@ -8,6 +8,7 @@ import tempfile import unittest from pathlib import Path +from unittest.mock import patch HERE = Path(__file__).resolve().parent sys_path_parent = str(HERE.parent) @@ -156,6 +157,138 @@ def test_unknown_rule_fails_closed_with_artefact(self) -> None: self.assertFalse(items["must_provide_reproducible_benchmark"]["pass"]) self.assertIn("unknown", items["must_provide_reproducible_benchmark"]["evidence"]) + def _run_inspect(self, art: Path, rules: Path, out: Path) -> dict[str, dict]: + os.environ["PROOF_RULES_FILE"] = str(rules) + os.environ["PROOF_OUTPUT_DIR"] = str(out) + os.environ["PROOF_ARTIFACT_DIR"] = str(art) + try: + self.assertEqual(inspect_scan.main([]), 0) + finally: + os.environ.pop("PROOF_RULES_FILE", None) + os.environ.pop("PROOF_OUTPUT_DIR", None) + os.environ.pop("PROOF_ARTIFACT_DIR", None) + return {i["id"]: i for i in json.loads((out / "checklist.json").read_text())} + + def test_oversized_file_fails_off_limits(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + art = root / "artifact" + art.mkdir() + (art / "ok.py").write_text("class MinerAgent:\n pass\n", encoding="utf-8") + oversized = art / "oversized.txt" + payload = ("x" * inspect_scan.MAX_FILE_BYTES) + "\nno_tb4_hardcoding\n" + oversized.write_text(payload, encoding="utf-8") + rules = root / "rules.json" + rules.write_text( + json.dumps( + [ + {"id": "no_eval_short_circuit", "text": "x"}, + {"id": "no_tb4_hardcoding", "text": "x"}, + ] + ), + encoding="utf-8", + ) + out = root / "out" + out.mkdir() + items = self._run_inspect(art, rules, out) + self.assertFalse(items["no_tb4_hardcoding"]["pass"]) + self.assertFalse(items["no_eval_short_circuit"]["pass"]) + self.assertIn("incomplete", items["no_tb4_hardcoding"]["evidence"]) + + def test_unreadable_file_fails_off_limits(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + art = root / "artifact" + art.mkdir() + (art / "ok.py").write_text("class MinerAgent:\n pass\n", encoding="utf-8") + hidden = art / "stat-failed.txt" + hidden.write_text("skip_eval\n", encoding="utf-8") + rules = root / "rules.json" + rules.write_text( + json.dumps( + [ + {"id": "no_eval_short_circuit", "text": "x"}, + {"id": "no_tb4_hardcoding", "text": "x"}, + ] + ), + encoding="utf-8", + ) + out = root / "out" + out.mkdir() + real_stat = Path.stat + size_lookups = {"n": 0} + + def _stat(self: Path, *args: object, **kwargs: object) -> os.stat_result: + result = real_stat(self, *args, **kwargs) + if self.name == "stat-failed.txt": + size_lookups["n"] += 1 + # is_file() must succeed so the walker reaches the size lookup. + if size_lookups["n"] > 1: + raise OSError("simulated stat failure") + return result + + with patch.object(Path, "stat", _stat): + items = self._run_inspect(art, rules, out) + self.assertFalse(items["no_eval_short_circuit"]["pass"]) + self.assertFalse(items["no_tb4_hardcoding"]["pass"]) + self.assertIn("incomplete", items["no_eval_short_circuit"]["evidence"]) + + def test_read_failed_file_fails_off_limits(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + art = root / "artifact" + art.mkdir() + (art / "ok.py").write_text("class MinerAgent:\n pass\n", encoding="utf-8") + hidden = art / "read-failed.txt" + hidden.write_text("tb4_answers\n", encoding="utf-8") + rules = root / "rules.json" + rules.write_text( + json.dumps( + [ + {"id": "no_eval_short_circuit", "text": "x"}, + {"id": "no_tb4_hardcoding", "text": "x"}, + ] + ), + encoding="utf-8", + ) + out = root / "out" + out.mkdir() + real_read = Path.read_bytes + + def _read(self: Path) -> bytes: + if self.name == "read-failed.txt": + raise OSError("simulated read failure") + return real_read(self) + + with patch.object(Path, "read_bytes", _read): + items = self._run_inspect(art, rules, out) + self.assertFalse(items["no_tb4_hardcoding"]["pass"]) + self.assertFalse(items["no_eval_short_circuit"]["pass"]) + self.assertIn("incomplete", items["no_tb4_hardcoding"]["evidence"]) + + def test_binary_file_fails_off_limits(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + art = root / "artifact" + art.mkdir() + (art / "ok.py").write_text("class MinerAgent:\n pass\n", encoding="utf-8") + (art / "blob.bin").write_bytes(b"\x00no_tb4_hardcoding\n") + rules = root / "rules.json" + rules.write_text( + json.dumps( + [ + {"id": "no_eval_short_circuit", "text": "x"}, + {"id": "no_tb4_hardcoding", "text": "x"}, + ] + ), + encoding="utf-8", + ) + out = root / "out" + out.mkdir() + items = self._run_inspect(art, rules, out) + self.assertFalse(items["no_tb4_hardcoding"]["pass"]) + self.assertIn("incomplete", items["no_tb4_hardcoding"]["evidence"]) + if __name__ == "__main__": unittest.main()