From 9fbc35d7d0fbc26b40d87ff0ab2fb06a5d8c6c17 Mon Sep 17 00:00:00 2001 From: mdsmithaustin <15272014+mdsmithaustin@users.noreply.github.com> Date: Tue, 20 Oct 2026 17:03:21 -0600 Subject: [PATCH 01/27] test(skill-recovery): preserve sanitized native response controls --- evals/skill-recovery/fixtures/README.md | 9 +++++++++ evals/skill-recovery/fixtures/entry.md | 1 + evals/skill-recovery/fixtures/mixed-full.jsonl | 5 +++++ evals/skill-recovery/fixtures/repaired-entry.jsonl | 7 +++++++ evals/skill-recovery/fixtures/step.md | 1 + 5 files changed, 23 insertions(+) create mode 100644 evals/skill-recovery/fixtures/README.md create mode 100644 evals/skill-recovery/fixtures/entry.md create mode 100644 evals/skill-recovery/fixtures/mixed-full.jsonl create mode 100644 evals/skill-recovery/fixtures/repaired-entry.jsonl create mode 100644 evals/skill-recovery/fixtures/step.md diff --git a/evals/skill-recovery/fixtures/README.md b/evals/skill-recovery/fixtures/README.md new file mode 100644 index 00000000..b1567921 --- /dev/null +++ b/evals/skill-recovery/fixtures/README.md @@ -0,0 +1,9 @@ +# Sanitized native format controls + +These files are hand-built, minimal projections of retained Codex 0.162.0 output shapes described in [issue 287](https://github.com/mdsmithaustin/pstack/issues/287). They are model-free parser inputs, not new native trials or copies of private recordings. + +`mixed-full.jsonl` preserves the structural failure from a native compaction trial. Two inner command envelopes share one outer response after a `compacted` record. Their complete bodies remain separate blocks. An automatic completion header is a separate block. + +`repaired-entry.jsonl` preserves the relevant shape from the separate Session pickup caller. A supplied context-summary condition precedes a mixed response and a later complete standalone entry response. The supplied condition is not a native compaction marker. The checkpoint covers the entry prerequisite before the local task-state inspection. It does not certify the caller's whole recovery procedure. + +The text in `entry.md` and `step.md`, call identifiers, commands, timings, row positions, and task output are replacements. The fixtures contain no original task text, credentials, host paths, or original session identifiers. The evaluator and reviewed manifests are pending in this first fixture unit. diff --git a/evals/skill-recovery/fixtures/entry.md b/evals/skill-recovery/fixtures/entry.md new file mode 100644 index 00000000..9ffada3a --- /dev/null +++ b/evals/skill-recovery/fixtures/entry.md @@ -0,0 +1 @@ +Recover the entry on its own before the next reviewed action. diff --git a/evals/skill-recovery/fixtures/mixed-full.jsonl b/evals/skill-recovery/fixtures/mixed-full.jsonl new file mode 100644 index 00000000..1fe8dc39 --- /dev/null +++ b/evals/skill-recovery/fixtures/mixed-full.jsonl @@ -0,0 +1,5 @@ +{"type": "compacted", "payload": {"message": "Sanitized compaction marker.", "replacement_history": []}} +{"type": "response_item", "payload": {"type": "custom_tool_call", "call_id": "read-batch", "name": "exec", "input": "text(await tools.exec_command({cmd:\"cat step.md\"})); text(await tools.exec_command({cmd:\"cat entry.md\"}));"}} +{"type": "response_item", "payload": {"type": "custom_tool_call_output", "call_id": "read-batch", "output": [{"type": "input_text", "text": "Script completed\nWall time 0.1 seconds\nOutput:\n"}, {"type": "input_text", "text": "{\"chunk_id\": \"sample\", \"wall_time_seconds\": 0.1, \"exit_code\": 0, \"original_token_count\": 20, \"output\": \"Read the current step requirements before its action.\\n\"}"}, {"type": "input_text", "text": "{\"chunk_id\": \"sample\", \"wall_time_seconds\": 0.1, \"exit_code\": 0, \"original_token_count\": 20, \"output\": \"Recover the entry on its own before the next reviewed action.\\n\"}"}]}} +{"type": "response_item", "payload": {"type": "custom_tool_call", "call_id": "action", "name": "exec", "input": "text(await tools.update_plan({plan:[]}));"}} +{"type": "response_item", "payload": {"type": "custom_tool_call_output", "call_id": "action", "output": [{"type": "input_text", "text": "Script completed\nWall time 0.1 seconds\nOutput:\n"}, {"type": "input_text", "text": "{\"chunk_id\": \"sample\", \"wall_time_seconds\": 0.1, \"exit_code\": 0, \"original_token_count\": 20, \"output\": \"Plan updated.\\n\"}"}]}} diff --git a/evals/skill-recovery/fixtures/repaired-entry.jsonl b/evals/skill-recovery/fixtures/repaired-entry.jsonl new file mode 100644 index 00000000..0a3b4654 --- /dev/null +++ b/evals/skill-recovery/fixtures/repaired-entry.jsonl @@ -0,0 +1,7 @@ +{"type": "response_item", "payload": {"type": "message", "role": "user", "content": [{"type": "input_text", "text": "Resume from a supplied context summary."}]}} +{"type": "response_item", "payload": {"type": "custom_tool_call", "call_id": "read-batch", "name": "exec", "input": "text(await tools.exec_command({cmd:\"cat step.md\"})); text(await tools.exec_command({cmd:\"cat entry.md\"}));"}} +{"type": "response_item", "payload": {"type": "custom_tool_call_output", "call_id": "read-batch", "output": [{"type": "input_text", "text": "Script completed\nWall time 0.1 seconds\nOutput:\n"}, {"type": "input_text", "text": "{\"chunk_id\": \"sample\", \"wall_time_seconds\": 0.1, \"exit_code\": 0, \"original_token_count\": 20, \"output\": \"Read the current step requirements before its action.\\n\"}"}, {"type": "input_text", "text": "{\"chunk_id\": \"sample\", \"wall_time_seconds\": 0.1, \"exit_code\": 0, \"original_token_count\": 20, \"output\": \"Recover the entry on its own before the next reviewed action.\\n\"}"}]}} +{"type": "response_item", "payload": {"type": "custom_tool_call", "call_id": "read-entry", "name": "exec", "input": "text(await tools.exec_command({cmd:\"cat entry.md\"}));"}} +{"type": "response_item", "payload": {"type": "custom_tool_call_output", "call_id": "read-entry", "output": [{"type": "input_text", "text": "Script completed\nWall time 0.1 seconds\nOutput:\n"}, {"type": "input_text", "text": "{\"chunk_id\": \"sample\", \"wall_time_seconds\": 0.1, \"exit_code\": 0, \"original_token_count\": 20, \"output\": \"Recover the entry on its own before the next reviewed action.\\n\"}"}]}} +{"type": "response_item", "payload": {"type": "custom_tool_call", "call_id": "action", "name": "exec", "input": "text(await tools.exec_command({cmd:\"cat TASK.md\"}));"}} +{"type": "response_item", "payload": {"type": "custom_tool_call_output", "call_id": "action", "output": [{"type": "input_text", "text": "Script completed\nWall time 0.1 seconds\nOutput:\n"}, {"type": "input_text", "text": "{\"chunk_id\": \"sample\", \"wall_time_seconds\": 0.1, \"exit_code\": 0, \"original_token_count\": 20, \"output\": \"Sanitized task record.\\n\"}"}]}} diff --git a/evals/skill-recovery/fixtures/step.md b/evals/skill-recovery/fixtures/step.md new file mode 100644 index 00000000..e245ccee --- /dev/null +++ b/evals/skill-recovery/fixtures/step.md @@ -0,0 +1 @@ +Read the current step requirements before its action. From 429cef77501e4f8beaff18004a3f2e819615be15 Mon Sep 17 00:00:00 2001 From: mdsmithaustin <15272014+mdsmithaustin@users.noreply.github.com> Date: Tue, 20 Oct 2026 17:53:42 -0600 Subject: [PATCH 02/27] feat(evals): evaluate native skill recovery checkpoints --- evals/skill-recovery/README.md | 94 +++ evals/skill-recovery/evaluate.py | 590 ++++++++++++++++++ evals/skill-recovery/fixtures/README.md | 10 +- .../fixtures/mixed-full.manifest.json | 88 +++ .../fixtures/pickup-late-entry.jsonl | 7 + .../fixtures/pickup-late-entry.manifest.json | 106 ++++ .../fixtures/repaired-entry.jsonl | 2 +- .../fixtures/repaired-entry.manifest.json | 106 ++++ tools/test_skill_recovery.py | 411 ++++++++++++ 9 files changed, 1409 insertions(+), 5 deletions(-) create mode 100644 evals/skill-recovery/README.md create mode 100644 evals/skill-recovery/evaluate.py create mode 100644 evals/skill-recovery/fixtures/mixed-full.manifest.json create mode 100644 evals/skill-recovery/fixtures/pickup-late-entry.jsonl create mode 100644 evals/skill-recovery/fixtures/pickup-late-entry.manifest.json create mode 100644 evals/skill-recovery/fixtures/repaired-entry.manifest.json create mode 100644 tools/test_skill_recovery.py diff --git a/evals/skill-recovery/README.md b/evals/skill-recovery/README.md new file mode 100644 index 00000000..f7c1ca38 --- /dev/null +++ b/evals/skill-recovery/README.md @@ -0,0 +1,94 @@ +# Skill recovery evaluator + +`evaluate.py` checks reviewed recovery prerequisites in saved native Codex JSONL. It compares returned UTF-8 bytes with frozen guidance, preserves each outer call and response block, and reports content, packaging, transport, and order separately. It uses the Python standard library and calls no model or service. + +This is an offline evaluator. Its manifest supplies the required guidance, recovery boundaries, and action classifications. A reviewer must establish their completeness. A result covers only the named scope and supplied recording. It does not establish runtime interception, model obedience, or compliance outside that scope. + +## CLI reference + +From the repository root, the synthetic repaired control runs with this command. + +```sh +python3 evals/skill-recovery/evaluate.py --raw evals/skill-recovery/fixtures/repaired-entry.jsonl --manifest evals/skill-recovery/fixtures/repaired-entry.manifest.json +``` + +Failure signals are a nonzero exit, a status other than `SATISFIED_WITH_REPAIRS`, or an applicable window count other than one. The earlier mixed response remains in `windows[0].attempts` with `repaired_before_action` set to true. + +The recorded-failure projection runs with this command. + +```sh +python3 evals/skill-recovery/evaluate.py --raw evals/skill-recovery/fixtures/mixed-full.jsonl --manifest evals/skill-recovery/fixtures/mixed-full.manifest.json +``` + +The expected result is exit 1 and status `FAIL`. Any other exit or status fails this negative control. Complete bodies from two files share one outer response and receive `MIXED_FILES` packaging. + +The pickup projection runs with this command. + +```sh +python3 evals/skill-recovery/evaluate.py --raw evals/skill-recovery/fixtures/pickup-late-entry.jsonl --manifest evals/skill-recovery/fixtures/pickup-late-entry.manifest.json +``` + +The expected result is exit 1 and status `FAIL`. Any other exit or status fails this negative control. Its first call both reads guidance and discovers task files. That action freezes recovery before the call returns. The later standalone entry read reports `LATE` and cannot repair the window. + +The CLI writes one JSON report to stdout. Input files remain unchanged. Reports contain hashes and byte counts rather than raw task content. Standard argument errors go to stderr. + +| Exit | Overall result | +| --- | --- | +| 0 | Every applicable window is `SATISFIED` or `SATISFIED_WITH_REPAIRS`. | +| 1 | A supported applicable window fails a required prerequisite. | +| 2 | Invalid, unsupported, incomplete, or unresolved evidence makes the result `INCONCLUSIVE`. | +| 3 | Every window is `NO_LATER_GOVERNED_ACTION`. No recovery checkpoint was checked. | + +`total_windows`, `applicable_windows`, and `non_applicable_windows` expose the denominator. A window without a governed action never counts as recovered. An empty boundary list is inconclusive. Inconclusive results take precedence over failures in the overall result. + +## Manifest reference + +The three fixture manifests are complete examples of schema version 1. Unknown fields and duplicate JSON keys are invalid. Row numbers and block indexes are one-based and zero-based respectively. Byte ranges are half-open offsets into decoded UTF-8 bodies or frozen UTF-8 source bytes. Row hashes include the original line terminator when present. + +| Field | Contract | +| --- | --- | +| `schema_version` | Integer `1`. | +| `raw_sha256` | SHA-256 of every original JSONL byte. | +| `assessment_scope` | `MODE_ENTRY_CHECKPOINT` or `CURRENT_STEP_RECOVERY`. | +| `review` | Nonempty `reviewer` and literal `complete: true`. This records an assertion, not proof of the review. | +| `files` | Unique `id`, manifest-relative `path`, exact `sha256`, and `role` of `mode_entry` or `guidance`. Exactly one file has the mode entry role. Absolute paths and parent traversal are invalid. | +| `boundaries` | Each boundary's `row`, `row_sha256`, `kind`, and complete `requirements`. | +| `calls` | Every outer call after the earliest boundary, with `row`, `row_sha256`, `call_id`, `disposition`, and nonempty `reason`. | +| `reads` | Independent claims binding `call_id`, `return_row`, `return_sha256`, `file`, `block`, `output_range`, and `source_range`. | +| `initial_injections` | Optional pre-boundary user-message claims with `row`, `row_sha256`, `file`, `block`, and `output_range`. | + +Boundary kinds are `native_compacted`, `supplied_context_summary`, and `supplied_truncation`. Every native `compacted` record must appear in the manifest. A supplied context summary must identify a user message. A truncation boundary cannot identify a call start. Supplied boundaries describe reviewed conditions and remain distinct from native compaction. + +Each requirement has `file` and `policy`. The mode entry always uses `whole_standalone`. Other guidance can use `whole_standalone` or `explicit_chunks`. `MODE_ENTRY_CHECKPOINT` allows only the entry requirement. `CURRENT_STEP_RECOVERY` requires the reviewer to establish the complete set for that step. A narrow entry result does not certify the whole procedure. + +Call dispositions are `governed`, `incidental`, and `unresolved`. Read claims do not replace dispositions. A call can both read guidance and perform governed work. Its action occurs at call start, before its returned guidance can count. An unresolved disposition prevents a conclusive window result. + +Exact source spans must match exact returned body spans. Overlapping output claims are invalid. A complete known governing body left unclaimed anywhere in the recording is invalid. This detects an omitted earlier full read attempt. The reviewer still owns the completeness of file inventory, partial-read claims, supplied boundaries, and action semantics. + +## Native input reference + +The parser supports native `function_call` and `custom_tool_call` records with their matching output records. Each `call_id` has exactly one call and one later return of the corresponding kind. Missing, duplicate, or mismatched pairs invalidate the entire input, including duplicates after an apparent success. + +Supported top-level record types are `session_meta`, `turn_context`, `event_msg`, `world_state`, `compacted`, `response_item`, `token_usage_record`, and `inter_agent_communication_metadata`. Message and reasoning records provide context. Names in command arguments or file lists do not prove a read. + +A returned output can be a string or a list of text blocks. Supported body carriers are plain text, a serialized command result with `output` and `exit_code`, and a fulfilled settled-result wrapper with `i`, `result.status`, and `result.value`. The command-result metadata keys are `chunk_id`, `wall_time_seconds`, `original_token_count`, `session_id`, and `truncated`. Unknown carriers remain unsupported. The parser never recursively flattens arbitrary JSON. + +A separate first-block completion header matching `Script completed`, `Wall time ... seconds`, and `Output:` is metadata. Other unclaimed substantive output prevents standalone packaging. Multi-file bodies and blocks remain part of one outer response. Splitting an inner batch cannot turn it into standalone reads. + +Structured truncation flags, transport notices outside matched source spans, command failures, and still-running commands prevent delivery credit. A truncation phrase inside exact source bytes is source text. No whitespace, line ending, or Unicode normalization occurs. + +A requirement is satisfied only by clear, standalone, exact content returned within its recovery window before the first governed call. The entry requires one whole-body claim. Explicit helper chunks can accumulate within one window. Pre-boundary reads, initial injections, replacement history, late returns, and reads crossing a later boundary receive no recovery credit. + +Attempts remain in the report after a repair. A later eligible read can repair an earlier violation before the action. An invalid required-file attempt after the final eligible read remains an unrepaired violation. The evaluator does not erase a bad batch to produce an unqualified clean result. + +## Test reference + +The public CLI suite runs with this command from the repository root. + +```sh +python3 -m unittest discover -s tools -p test_skill_recovery.py -v +``` + +Failure signals are a nonzero exit, zero tests in the `Ran` summary, a missing summary, or a final result other than `OK`. The tests exercise stdout and process exits through the public command. They cover malformed and unsupported evidence, exact hashes and bytes, mixed responses, late repair, multiple windows, explicit chunks, and zero applicable checkpoints. + +Fixture provenance and its limits are documented in [the fixture notes](fixtures/README.md). Private recording replays require separate reviewed manifests and retained derivation receipts. They are not shipped as public fixtures. diff --git a/evals/skill-recovery/evaluate.py b/evals/skill-recovery/evaluate.py new file mode 100644 index 00000000..80d0ab63 --- /dev/null +++ b/evals/skill-recovery/evaluate.py @@ -0,0 +1,590 @@ +#!/usr/bin/env python3 +"""Evaluate reviewed recovery prerequisites in saved native Codex JSONL.""" + +from __future__ import annotations + +import argparse +from dataclasses import dataclass, field +import hashlib +import json +from pathlib import Path +import re + + +class _InputError(Exception): + def __init__(self, code: str, row: int | None = None): + self.code = code + self.row = row + + +def _require(condition, code, row=None): + if not condition: + raise _InputError(code, row) + + +def _json(data, row=None): + def pairs(items): + result = {} + for key, value in items: + _require(key not in result, "duplicate_json_key", row) + result[key] = value + return result + + def constant(_value): + raise _InputError("invalid_json_number", row) + + try: + return json.loads(data, object_pairs_hook=pairs, parse_constant=constant) + except (ValueError, UnicodeError) as error: + raise _InputError("invalid_json_or_utf8", row) from error + + +def _sha(data): + return hashlib.sha256(data).hexdigest() + + +def _text(value, row=None): + _require(isinstance(value, str), "expected_text", row) + try: + return value.encode("utf-8") + except UnicodeError as error: + raise _InputError("invalid_utf8_text", row) from error + + +def _object(value, required, optional=(), row=None): + _require(isinstance(value, dict), "expected_object", row) + _require(set(required) <= value.keys(), "missing_field", row) + _require(value.keys() <= set(required) | set(optional), "unknown_field", row) + return value + + +def _array(value, row=None): + _require(isinstance(value, list), "expected_array", row) + return value + + +def _string(value, row=None): + _require(bool(_text(value, row)), "empty_text", row) + return value + + +def _digest(value, row=None): + _require(isinstance(value, str) and re.fullmatch(r"[0-9a-f]{64}", value), "invalid_sha256", row) + return value + + +def _span(value, limit=None, row=None): + _require(isinstance(value, list) and len(value) == 2, "invalid_byte_range", row) + start, end = value + _require(type(start) is int and type(end) is int and 0 <= start < end and + (limit is None or end <= limit), "invalid_byte_range", row) + return start, end + + +def _flags(*objects): + found = set() + for obj in objects: + if "truncated" in obj: + if obj["truncated"] is True: + found.add("structured_truncation") + elif obj["truncated"] is not False: + found.add("unsupported_truncation_flag") + return tuple(sorted(found)) + + +@dataclass(frozen=True) +class _File: + id: str + role: str + data: bytes + + +@dataclass(frozen=True) +class _Body: + block: int + kind: str + raw_hash: str + raw_size: int + data: bytes | None + flags: tuple[str, ...] + + +@dataclass(frozen=True) +class _Call: + id: str + row: int + return_row: int + bodies: tuple[_Body, ...] + flags: tuple[str, ...] + + +@dataclass(frozen=True) +class _Claim: + call: str + file: str + block: int + output: tuple[int, int] + source: tuple[int, int] + + +@dataclass +class _Window: + row: int + end: int + kind: str + requirements: dict[str, str] + first_action: int | None = None + attempts: list = field(default_factory=list) + coverage: dict = field(default_factory=dict) + unresolved: list = field(default_factory=list) + + +_HEADER = re.compile(r"Script completed\nWall time [0-9.]+ seconds\nOutput:\n") +_NOTICE = re.compile(rb"(?im)^(?:Warning: truncated output\b|\[?output (?:truncated|omitted)\b|\.\.\.\s*\d+ (?:characters|tokens) truncated\b)") +_ENVELOPE_KEYS = {"output", "exit_code", "chunk_id", "wall_time_seconds", "original_token_count", "session_id", "truncated"} + + +def _bodies(output, row): + _require(isinstance(output, (str, list)), "unsupported_outer_output", row) + blocks = output if isinstance(output, list) else [output] + result = [] + for index, block in enumerate(blocks): + flags = () + if isinstance(block, dict): + flags = _flags(block) + if block.get("type") not in ("input_text", "text") or "text" not in block or block.keys() - {"type", "text", "truncated"}: + encoded = json.dumps(block, sort_keys=True).encode() + result.append(_Body(index, "unsupported", _sha(encoded), len(encoded), None, flags)) + continue + text = block["text"] + else: + text = block + data = _text(text, row) + if index == 0 and _HEADER.fullmatch(text): + result.append(_Body(index, "completion_header", _sha(data), len(data), None, flags)) + continue + inner = None + if text.lstrip().startswith("{"): + try: + inner = _json(text, row) + except _InputError: + result.append(_Body(index, "unsupported", _sha(data), len(data), None, flags)) + continue + kind = "text" + if isinstance(inner, dict) and set(inner) == {"i", "result"}: + settled = inner["result"] + if type(inner["i"]) is not int or not isinstance(settled, dict) or set(settled) != {"status", "value"} or settled["status"] != "fulfilled": + result.append(_Body(index, "unsupported", _sha(data), len(data), None, flags + ("settled_result_not_fulfilled",))) + continue + inner = settled["value"] + kind = "settled_command_result" + elif isinstance(inner, dict) and ("output" in inner or "exit_code" in inner): + kind = "command_result" + if kind != "text": + if not isinstance(inner, dict) or inner.keys() - _ENVELOPE_KEYS or not isinstance(inner.get("output"), str): + result.append(_Body(index, "unsupported", _sha(data), len(data), None, flags)) + continue + flags += _flags(inner) + code = inner.get("exit_code") + if type(code) is not int: + flags += ("missing_integer_exit_code",) + code = None + elif code != 0: + flags += ("command_failed",) + if "session_id" in inner and code is None: + flags += ("command_still_running",) + result.append(_Body(index, kind, _sha(data), len(data), _text(inner["output"], row), flags)) + else: + result.append(_Body(index, kind, _sha(data), len(data), data, flags)) + return tuple(result) + + +def _trace(raw): + _require(bool(raw), "empty_raw") + lines = raw.split(b"\n") + if lines[-1] == b"": + lines.pop() + records = [] + line_bytes = [] + starts = {} + returns = {} + allowed = {"session_meta", "turn_context", "event_msg", "world_state", "compacted", + "response_item", "token_usage_record", "inter_agent_communication_metadata"} + for number, line in enumerate(lines, 1): + _require(bool(line.strip()), "empty_jsonl_row", number) + record = _json(line, number) + _require(isinstance(record, dict) and record.get("type") in allowed, "unsupported_record", number) + _require(isinstance(record.get("payload"), dict), "unsupported_payload", number) + records.append(record) + line_bytes.append(line + (b"\n" if number < len(lines) or raw.endswith(b"\n") else b"")) + if record["type"] != "response_item": + continue + payload = record["payload"] + kind = payload.get("type") + if kind in ("message", "reasoning", "agent_message"): + continue + _require(kind in ("function_call", "custom_tool_call", "function_call_output", "custom_tool_call_output"), + "unsupported_response_item", number) + call_id = _string(payload.get("call_id"), number) + target = returns if kind.endswith("_output") else starts + _require(call_id not in target, "duplicate_outer_identity", number) + target[call_id] = (number, record) + _require(starts.keys() == returns.keys(), "unpaired_outer_call") + calls = {} + for call_id, (number, record) in starts.items(): + returned, returned_record = returns[call_id] + payload = record["payload"] + result = returned_record["payload"] + _require(number < returned, "return_precedes_call", returned) + _require(result["type"] == payload["type"] + "_output", "mismatched_outer_kind", returned) + _string(payload.get("name"), number) + _require("output" in result, "missing_outer_output", returned) + calls[call_id] = _Call(call_id, number, returned, _bodies(result["output"], returned), + _flags(record, payload, returned_record, result)) + return records, line_bytes, calls + + +def _read_file(path, code): + try: + return path.read_bytes() + except (OSError, UnicodeError, ValueError) as error: + raise _InputError(code) from error + + +def _bind(raw, manifest, directory): + _object(manifest, {"schema_version", "raw_sha256", "assessment_scope", "review", "files", "boundaries", "calls", "reads"}, {"initial_injections"}) + _require(manifest["schema_version"] == 1 and type(manifest["schema_version"]) is int, "unsupported_schema") + _require(_digest(manifest["raw_sha256"]) == _sha(raw), "raw_hash_mismatch") + scope = manifest["assessment_scope"] + _require(scope in ("MODE_ENTRY_CHECKPOINT", "CURRENT_STEP_RECOVERY"), "invalid_assessment_scope") + review = _object(manifest["review"], {"reviewer", "complete"}) + _string(review["reviewer"]) + _require(review["complete"] is True, "review_not_complete") + records, lines, calls = _trace(raw) + + def row_ref(value, number_key="row", hash_key="row_sha256"): + number = value[number_key] + _require(type(number) is int and 1 <= number <= len(records), "invalid_row_reference") + _require(_digest(value[hash_key], number) == _sha(lines[number - 1]), "row_hash_mismatch", number) + return number, records[number - 1] + + files = {} + for item in _array(manifest["files"]): + _object(item, {"id", "path", "sha256", "role"}) + file_id = _string(item["id"]) + _require(file_id not in files, "duplicate_file") + relative = Path(_string(item["path"])) + _require(not relative.is_absolute() and ".." not in relative.parts, "expected_path_not_relative") + data = _read_file(directory / relative, "expected_file_unreadable") + try: + data.decode("utf-8") + except UnicodeError as error: + raise _InputError("expected_file_not_utf8") from error + _require(bool(data) and _digest(item["sha256"]) == _sha(data), "expected_file_hash_or_bytes_mismatch") + _require(item["role"] in ("mode_entry", "guidance"), "invalid_file_role") + files[file_id] = _File(file_id, item["role"], data) + _require(sum(f.role == "mode_entry" for f in files.values()) == 1, "require_one_mode_entry") + entry = next(f.id for f in files.values() if f.role == "mode_entry") + windows = [] + seen_boundaries = set() + for item in _array(manifest["boundaries"]): + _object(item, {"row", "row_sha256", "kind", "requirements"}) + number, record = row_ref(item) + _require(number not in seen_boundaries, "duplicate_boundary", number) + seen_boundaries.add(number) + kind = item["kind"] + _require(kind in ("native_compacted", "supplied_context_summary", "supplied_truncation"), "invalid_boundary_kind", number) + _require((record["type"] == "compacted") == (kind == "native_compacted"), "boundary_kind_mismatch", number) + if kind == "supplied_context_summary": + payload = record["payload"] + _require(record["type"] == "response_item" and payload.get("type") == "message" and + payload.get("role") == "user", "summary_boundary_not_user_message", number) + if kind == "supplied_truncation": + _require(record["payload"].get("type") not in ("function_call", "custom_tool_call"), + "truncation_boundary_is_call_start", number) + requirements = {} + for required in _array(item["requirements"], number): + _object(required, {"file", "policy"}, row=number) + file_id = required["file"] + _require(isinstance(file_id, str) and file_id in files and file_id not in requirements, "invalid_requirement_file", number) + policy = required["policy"] + _require(policy in ("whole_standalone", "explicit_chunks"), "invalid_read_policy", number) + _require(file_id != entry or policy == "whole_standalone", "mode_entry_cannot_use_chunks", number) + requirements[file_id] = policy + _require(entry in requirements, "missing_mode_requirement", number) + _require(scope != "MODE_ENTRY_CHECKPOINT" or set(requirements) == {entry}, "checkpoint_has_extra_requirements", number) + windows.append(_Window(number, len(records) + 1, kind, requirements)) + _require(bool(windows), "missing_boundary") + windows.sort(key=lambda window: window.row) + _require({i for i, record in enumerate(records, 1) if record["type"] == "compacted"} == + {window.row for window in windows if window.kind == "native_compacted"}, "omitted_native_boundary") + for left, right in zip(windows, windows[1:]): + left.end = right.row + dispositions = {} + for item in _array(manifest["calls"]): + _object(item, {"row", "row_sha256", "call_id", "disposition", "reason"}) + number, _ = row_ref(item) + call_id = item["call_id"] + _require(isinstance(call_id, str) and call_id in calls and calls[call_id].row == number, "call_reference_mismatch", number) + _require(call_id not in dispositions, "duplicate_call_disposition", number) + _require(item["disposition"] in ("governed", "incidental", "unresolved"), "invalid_call_disposition", number) + _string(item["reason"], number) + dispositions[call_id] = item["disposition"] + _require(set(dispositions) == {call.id for call in calls.values() if call.row > windows[0].row}, "incomplete_call_review") + claims = [] + seen_claims = set() + for item in _array(manifest["reads"]): + _object(item, {"call_id", "return_row", "return_sha256", "file", "block", "output_range", "source_range"}) + number, _ = row_ref(item, "return_row", "return_sha256") + call_id = item["call_id"] + _require(isinstance(call_id, str) and call_id in calls and calls[call_id].return_row == number, "read_pair_mismatch", number) + file_id = item["file"] + _require(isinstance(file_id, str) and file_id in files, "unknown_read_file", number) + block = item["block"] + _require(type(block) is int and 0 <= block < len(calls[call_id].bodies), "invalid_block_reference", number) + body = calls[call_id].bodies[block] + _require(body.kind != "completion_header", "metadata_is_not_source", number) + limit = len(body.data) if body.data is not None else None + output_span = _span(item["output_range"], limit, number) + source_span = _span(item["source_range"], len(files[file_id].data), number) + claim = _Claim(call_id, file_id, block, output_span, source_span) + _require(claim not in seen_claims, "duplicate_read_claim", number) + seen_claims.add(claim) + claims.append(claim) + for call_id in calls: + selected = [claim for claim in claims if claim.call == call_id] + for block in {claim.block for claim in selected}: + spans = sorted(claim.output for claim in selected if claim.block == block) + _require(all(left[1] <= right[0] for left, right in zip(spans, spans[1:])), "overlapping_output_claims", calls[call_id].return_row) + for body in calls[call_id].bodies: + if body.data: + spans = [claim.output for claim in selected if claim.block == body.block and + body.data[claim.output[0]:claim.output[1]] == files[claim.file].data[claim.source[0]:claim.source[1]]] + residual = _residual(body.data, spans) + _require(not any(file.data in residual for file in files.values()), + "undeclared_governing_body", calls[call_id].return_row) + injections = [] + for item in _array(manifest.get("initial_injections", [])): + _object(item, {"row", "row_sha256", "file", "block", "output_range"}) + number, record = row_ref(item) + payload = record["payload"] + _require(record["type"] == "response_item" and payload.get("type") == "message" and payload.get("role") == "user", "invalid_initial_injection", number) + _require(number < windows[0].row, "injection_is_not_recovery", number) + file_id = item["file"] + _require(isinstance(file_id, str) and file_id in files, "unknown_injection_file", number) + content = _array(payload.get("content"), number) + block = item["block"] + _require(type(block) is int and 0 <= block < len(content) and isinstance(content[block], dict), "invalid_injection_block", number) + data = _text(content[block].get("text"), number) + start, end = _span(item["output_range"], len(data), number) + injections.append({"row": number, "file": file_id, "exact_content": data[start:end] == files[file_id].data, + "recovery_credit": False}) + return files, windows, calls, dispositions, claims, injections + + +def _complete(spans, size): + end = 0 + for start, stop in sorted(spans): + if start > end: + return False + end = max(end, stop) + return end == size + + +def _residual(data, spans): + cursor = 0 + result = bytearray() + for left, right in sorted(spans): + result.extend(data[cursor:left]) + cursor = right + result.extend(data[cursor:]) + return bytes(result) + + +def _measure(call, claims, files): + exact = [] + flags = set(call.flags) + extra = False + unsupported = False + for body in call.bodies: + flags.update(body.flags) + if body.kind == "completion_header": + continue + if body.data is None: + unsupported = True + flags.add("unsupported_output_carrier") + continue + spans = [] + for claim in claims: + if claim.block != body.block: + continue + left, right = claim.output + start, end = claim.source + if body.data[left:right] == files[claim.file].data[start:end]: + exact.append(claim) + spans.append((left, right)) + residual = _residual(body.data, spans) + if residual: + extra = True + if _NOTICE.search(residual): + flags.add("transport_notice") + file_ids = {claim.file for claim in exact} + if len(file_ids) > 1: + packaging = "MIXED_FILES" + elif unsupported: + packaging = "UNVERIFIED" + elif extra: + packaging = "EXTRA_CONTENT" + elif len(file_ids) == 1: + packaging = "STANDALONE" + else: + packaging = "UNVERIFIED" + return exact, packaging, tuple(sorted(flags)), unsupported + + +def _evaluate(files, windows, calls, dispositions, claims, injections, scope): + claims_by_call = {} + for claim in claims: + claims_by_call.setdefault(claim.call, []).append(claim) + owner = {} + for call in calls.values(): + owner[call.id] = next((window for window in windows if window.row < call.row < window.end), None) + events = [] + for call in calls.values(): + events.append((call.row, "call", call)) + events.append((call.return_row, "return", call)) + outside = [] + for _, kind, call in sorted(events, key=lambda event: event[0]): + window = owner[call.id] + if kind == "call": + if window is not None: + disposition = dispositions[call.id] + if disposition == "unresolved": + window.unresolved.append(call.row) + if disposition == "governed" and window.first_action is None: + window.first_action = call.row + continue + selected = claims_by_call.get(call.id, []) + if not selected: + continue + exact, packaging, flags, unsupported = _measure(call, selected, files) + for file_id in sorted({claim.file for claim in selected}): + spans = [claim.source for claim in exact if claim.file == file_id] + whole = (0, len(files[file_id].data)) in spans + if window is None or call.return_row >= window.end: + ordering = "CROSSES_BOUNDARY" + elif window.first_action is not None: + ordering = "LATE" + else: + ordering = "BEFORE_ACTION" + observed = "COMPLETE" if _complete(spans, len(files[file_id].data)) else "EXACT_CHUNK" if spans else "UNVERIFIED" + attempt = {"call_id": call.id, "call_row": call.row, "return_row": call.return_row, "file": file_id, + "content": observed, "source_ranges": [list(span) for span in spans], "whole_body": whole, + "packaging": packaging, "ordering": ordering, + "transport": "UNVERIFIED" if flags or unsupported else "CLEAR", "reasons": list(flags), + "repaired_before_action": False} + if window is None: + outside.append(attempt) + continue + index = len(window.attempts) + window.attempts.append(attempt) + if ordering == "BEFORE_ACTION" and packaging == "STANDALONE" and not flags and not unsupported: + policy = window.requirements.get(file_id) + if policy == "explicit_chunks" or (policy == "whole_standalone" and whole): + progress = window.coverage.setdefault(file_id, []) + progress.extend((span, index) for span in spans) + reports = [] + for window in windows: + requirement_reports = [] + repaired = False + failed = False + inconclusive = bool(window.unresolved) + for file_id, policy in window.requirements.items(): + progress = window.coverage.get(file_id, []) + satisfied = _complete([span for span, _ in progress], len(files[file_id].data)) + attempts = [(i, attempt) for i, attempt in enumerate(window.attempts) if attempt["file"] == file_id] + witnesses = sorted({index for _, index in progress}) if satisfied else [] + unrepaired = [] + if satisfied: + finish = max(window.attempts[index]["return_row"] for index in witnesses) + for index, attempt in attempts: + invalid = (attempt["packaging"] != "STANDALONE" or attempt["transport"] != "CLEAR" or + attempt["content"] == "UNVERIFIED" or (policy == "whole_standalone" and not attempt["whole_body"])) + if invalid and attempt["ordering"] == "BEFORE_ACTION": + if attempt["return_row"] < finish: + attempt["repaired_before_action"] = True + repaired = True + else: + unrepaired.append(index) + failed = True + else: + failed = True + if any(attempt["transport"] != "CLEAR" or attempt["content"] == "UNVERIFIED" for _, attempt in attempts): + inconclusive = True + requirement_reports.append({"file": file_id, "policy": policy, "satisfied_before_action": satisfied, + "witness_attempts": witnesses, "unrepaired_violation_attempts": unrepaired, + "observed_content_complete": _complete([tuple(span) for _, attempt in attempts for span in attempt["source_ranges"]], len(files[file_id].data))}) + if window.first_action is None: + status = "NO_LATER_GOVERNED_ACTION" if not window.unresolved else "INCONCLUSIVE" + for attempt in window.attempts: + if attempt["ordering"] == "BEFORE_ACTION": + attempt["ordering"] = "NO_ACTION" + elif inconclusive: + status = "INCONCLUSIVE" + elif failed: + status = "FAIL" + else: + status = "SATISFIED_WITH_REPAIRS" if repaired else "SATISFIED" + reports.append({"boundary_row": window.row, "boundary_kind": window.kind, "end_row_exclusive": window.end, + "assessment_scope": scope, "first_action_row": window.first_action, + "status": status, "requirements": requirement_reports, "attempts": window.attempts, + "unresolved_call_rows": window.unresolved}) + statuses = {report["status"] for report in reports} + if "INCONCLUSIVE" in statuses: + code, status = 2, "INCONCLUSIVE" + elif "FAIL" in statuses: + code, status = 1, "FAIL" + elif statuses == {"NO_LATER_GOVERNED_ACTION"}: + code, status = 3, "NO_LATER_GOVERNED_ACTION" + elif "SATISFIED_WITH_REPAIRS" in statuses: + code, status = 0, "SATISFIED_WITH_REPAIRS" + else: + code, status = 0, "SATISFIED" + return code, {"status": status, "assessment_scope": scope, "windows": reports, + "total_windows": len(windows), + "applicable_windows": sum(window.first_action is not None for window in windows), + "non_applicable_windows": sum(report["status"] == "NO_LATER_GOVERNED_ACTION" for report in reports), + "initial_deliveries": injections, "outside_window_attempts": outside} + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--raw", required=True, type=Path) + parser.add_argument("--manifest", required=True, type=Path) + args = parser.parse_args(argv) + report = {"schema_version": 1, "evaluator_sha256": _sha(Path(__file__).read_bytes()), + "semantic_limit": "Action classifications and complete requirements are reviewed inputs, not inferred guarantees."} + try: + raw = _read_file(args.raw, "raw_unreadable") + manifest_bytes = _read_file(args.manifest, "manifest_unreadable") + report.update(raw_sha256=_sha(raw), manifest_sha256=_sha(manifest_bytes)) + manifest = _json(manifest_bytes) + bound = _bind(raw, manifest, args.manifest.parent) + code, result = _evaluate(*bound, manifest["assessment_scope"]) + report.update(result) + files, _, calls, _, _, _ = bound + report["files"] = [{"id": item.id, "role": item.role, "sha256": _sha(item.data), "bytes": len(item.data)} for item in files.values()] + report["outer_responses"] = [{"call_id": call.id, "call_row": call.row, "return_row": call.return_row, + "blocks": [{"index": body.block, "carrier": body.kind, "raw_sha256": body.raw_hash, + "raw_bytes": body.raw_size, "body_sha256": _sha(body.data) if body.data is not None else None, + "body_bytes": len(body.data) if body.data is not None else None} for body in call.bodies]} + for call in calls.values()] + except _InputError as error: + code = 2 + report.update(status="INCONCLUSIVE", errors=[{"code": error.code, "row": error.row}], windows=[]) + print(json.dumps(report, sort_keys=True, indent=2, ensure_ascii=False)) + return code + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/evals/skill-recovery/fixtures/README.md b/evals/skill-recovery/fixtures/README.md index b1567921..4b8b30c8 100644 --- a/evals/skill-recovery/fixtures/README.md +++ b/evals/skill-recovery/fixtures/README.md @@ -1,9 +1,11 @@ # Sanitized native format controls -These files are hand-built, minimal projections of retained Codex 0.162.0 output shapes described in [issue 287](https://github.com/mdsmithaustin/pstack/issues/287). They are model-free parser inputs, not new native trials or copies of private recordings. +These files contain minimal Codex 0.162.0 output shapes described in [issue 287](https://github.com/mdsmithaustin/pstack/issues/287). They are model-free parser inputs. No file is a new native trial or a copy of a private recording. -`mixed-full.jsonl` preserves the structural failure from a native compaction trial. Two inner command envelopes share one outer response after a `compacted` record. Their complete bodies remain separate blocks. An automatic completion header is a separate block. +`mixed-full.jsonl` is a hand-built projection of the recorded native compaction failure. Two inner command envelopes share one outer response after a `compacted` record. Their complete bodies remain separate blocks. An automatic completion header is another block. -`repaired-entry.jsonl` preserves the relevant shape from the separate Session pickup caller. A supplied context-summary condition precedes a mixed response and a later complete standalone entry response. The supplied condition is not a native compaction marker. The checkpoint covers the entry prerequisite before the local task-state inspection. It does not certify the caller's whole recovery procedure. +`pickup-late-entry.jsonl` projects the separate recorded Session pickup caller. The supplied context-summary condition is not native compaction. One outer call reads two governing files and discovers task files. Its settled-result wrappers contain three command results. A later standalone entry response arrives after that task-discovery action. The recovery window must fail even though the later response passes the isolated packaging check. -The text in `entry.md` and `step.md`, call identifiers, commands, timings, row positions, and task output are replacements. The fixtures contain no original task text, credentials, host paths, or original session identifiers. The evaluator and reviewed manifests are pending in this first fixture unit. +`repaired-entry.jsonl` is a synthetic contrast to the failed mixed response. It adds a complete standalone entry read before the task action. It tests a repaired-before-action result, with the earlier packaging violation retained. It cannot establish native model behavior. + +The guidance in `entry.md` and `step.md`, task text, output, call identifiers, commands, timings, and row positions are replacements. The fixtures contain no original task content, credentials, host paths, or original session identifiers. Each JSONL file has a matching reviewed manifest. The public CLI tests verify their expected outcomes. The projections preserve only the native boundaries and carriers needed for these controls. Independent privacy and provenance review belongs to the exact committed fixture revision. diff --git a/evals/skill-recovery/fixtures/mixed-full.manifest.json b/evals/skill-recovery/fixtures/mixed-full.manifest.json new file mode 100644 index 00000000..132a115e --- /dev/null +++ b/evals/skill-recovery/fixtures/mixed-full.manifest.json @@ -0,0 +1,88 @@ +{ + "schema_version": 1, + "raw_sha256": "efe37101bbf054e3cccdc4c3ad046c700cbb563af8f6b70e38b8b6ad64357081", + "assessment_scope": "CURRENT_STEP_RECOVERY", + "review": { + "reviewer": "sanitized control author; independent fixture review is separate", + "complete": true + }, + "files": [ + { + "id": "entry", + "path": "entry.md", + "sha256": "2ca130d0a38406b788362ad5e4c8fea8e56e012b71f0878af454b89117c1e8d9", + "role": "mode_entry" + }, + { + "id": "step", + "path": "step.md", + "sha256": "8f81d54100650a3accbae8a56e5a7d9ab35e4a73803e556c38a0a72f810f4cca", + "role": "guidance" + } + ], + "boundaries": [ + { + "row": 1, + "row_sha256": "d3bea2bc8f126a62908041a9ff4b9eb5952658c3b886ff6a18c6db70902fa8fc", + "kind": "native_compacted", + "requirements": [ + { + "file": "entry", + "policy": "whole_standalone" + }, + { + "file": "step", + "policy": "whole_standalone" + } + ] + } + ], + "calls": [ + { + "row": 2, + "row_sha256": "0854751f2444c8d24a171fbd2dd790f39ab673c3a086ef9bec4011c2f551da4a", + "call_id": "read-batch", + "disposition": "incidental", + "reason": "governing guidance read" + }, + { + "row": 4, + "row_sha256": "501e05cea45df9df58bfb70cd4d7e48ef33698c158798a0183af12be7e7b6849", + "call_id": "action", + "disposition": "governed", + "reason": "task action" + } + ], + "reads": [ + { + "call_id": "read-batch", + "return_row": 3, + "return_sha256": "4dec3a7afac8a565367d6a840929118f338ad4fdc0345ac4e3b66d41924ef072", + "file": "step", + "block": 1, + "output_range": [ + 0, + 54 + ], + "source_range": [ + 0, + 54 + ] + }, + { + "call_id": "read-batch", + "return_row": 3, + "return_sha256": "4dec3a7afac8a565367d6a840929118f338ad4fdc0345ac4e3b66d41924ef072", + "file": "entry", + "block": 2, + "output_range": [ + 0, + 62 + ], + "source_range": [ + 0, + 62 + ] + } + ] +} diff --git a/evals/skill-recovery/fixtures/pickup-late-entry.jsonl b/evals/skill-recovery/fixtures/pickup-late-entry.jsonl new file mode 100644 index 00000000..86c0eb60 --- /dev/null +++ b/evals/skill-recovery/fixtures/pickup-late-entry.jsonl @@ -0,0 +1,7 @@ +{"type": "response_item", "payload": {"type": "message", "role": "user", "content": [{"type": "input_text", "text": "Resume from a supplied context summary."}]}} +{"type": "response_item", "payload": {"type": "custom_tool_call", "call_id": "read-batch", "name": "exec", "input": "const results=await Promise.allSettled([tools.exec_command({cmd:\"cat step.md\"}),tools.exec_command({cmd:\"cat entry.md\"}),tools.exec_command({cmd:\"pwd; rg --files -g TASK.md -g delivery.json\"})]); for(let i=0;i first: + manifest["calls"].append({"row": n, "row_sha256": digest(lines[n - 1]), "call_id": call_id, + "disposition": "governed" if call_id in governed else "incidental", + "reason": "task action" if call_id in governed else "guidance read"}) + for spec in reads: + n = returns[spec["call_id"]] + source = spec["source"] or [0, len(self.files[spec["file"]])] + output = spec["output"] or [0, source[1] - source[0]] + manifest["reads"].append({"call_id": spec["call_id"], "return_row": n, + "return_sha256": digest(lines[n - 1]), "file": spec["file"], + "block": spec["block"], "source_range": source, "output_range": output}) + self.manifest = manifest + self.write_manifest() + return self + + def write_raw(self): + self.raw.write_bytes(b"".join(json.dumps(row, ensure_ascii=True).encode() + b"\n" for row in self.rows)) + + def write_manifest(self): + self.manifest_path.write_text(json.dumps(self.manifest)) + + def rebind(self): + self.write_raw() + lines = self.raw.read_bytes().splitlines(keepends=True) + self.manifest["raw_sha256"] = digest(self.raw.read_bytes()) + for item in self.manifest["boundaries"] + self.manifest["calls"]: + item["row_sha256"] = digest(lines[item["row"] - 1]) + for item in self.manifest["reads"]: + item["return_sha256"] = digest(lines[item["return_row"] - 1]) + self.write_manifest() + + def run_cli(self, expected): + result = subprocess.run([sys.executable, str(CLI), "--raw", str(self.raw), "--manifest", str(self.manifest_path)], + capture_output=True, text=True, timeout=15) + self.assertEqual(result.returncode, expected, (result.stdout, result.stderr)) + self.assertEqual(result.stderr, "") + report = json.loads(result.stdout) + self.assertEqual(report["raw_sha256"], digest(self.raw.read_bytes())) + return report + + def basic(self): + return self.prepare([boundary(), call("read"), returned("read", [ENTRY]), call("action"), returned("action", [b"Done."])], [read("read")]) + + def test_standalone_full_entry(self): + report = self.basic().run_cli(0) + self.assertEqual(report["status"], "SATISFIED") + self.assertEqual(report["applicable_windows"], 1) + attempt = report["windows"][0]["attempts"][0] + self.assertEqual((attempt["content"], attempt["packaging"], attempt["ordering"]), ("COMPLETE", "STANDALONE", "BEFORE_ACTION")) + + def test_committed_controls(self): + cases = [("mixed-full", 1, "FAIL"), ("pickup-late-entry", 1, "FAIL"), ("repaired-entry", 0, "SATISFIED_WITH_REPAIRS")] + for name, code, status in cases: + with self.subTest(name=name): + result = subprocess.run([sys.executable, str(CLI), "--raw", str(FIXTURES / (name + ".jsonl")), + "--manifest", str(FIXTURES / (name + ".manifest.json"))], capture_output=True, text=True, timeout=15) + self.assertEqual(result.returncode, code, result.stdout + result.stderr) + report = json.loads(result.stdout) + self.assertEqual(report["status"], status) + if name == "repaired-entry": + self.assertTrue(report["windows"][0]["attempts"][0]["repaired_before_action"]) + if name == "pickup-late-entry": + self.assertEqual(report["windows"][0]["first_action_row"], 2) + self.assertEqual(report["windows"][0]["attempts"][-1]["packaging"], "STANDALONE") + self.assertEqual(report["windows"][0]["attempts"][-1]["ordering"], "LATE") + + def test_separate_full_blocks_are_one_mixed_outer_response(self): + self.prepare([boundary(), call("read"), returned("read", [ENTRY, STEP]), call("action"), returned("action", [b"Done."])], + [read("read"), read("read", "step", 2)]) + report = self.run_cli(1) + self.assertEqual(len(report["windows"][0]["attempts"]), 2) + self.assertTrue(all(a["content"] == "COMPLETE" and a["packaging"] == "MIXED_FILES" for a in report["windows"][0]["attempts"])) + self.assertEqual(len(report["outer_responses"][0]["blocks"]), 3) + + def test_one_command_multi_file_output_is_mixed(self): + self.prepare([boundary(), call("read"), returned("read", [ENTRY + STEP]), call("action"), returned("action", [b"Done."])], + [read("read"), read("read", "step", 1, output=[len(ENTRY), len(ENTRY) + len(STEP)])]) + report = self.run_cli(1) + self.assertTrue(all(a["packaging"] == "MIXED_FILES" for a in report["windows"][0]["attempts"])) + + def test_extra_output_block_prevents_standalone(self): + self.prepare([boundary(), call("read"), returned("read", [ENTRY, b"unrelated output"]), call("action"), returned("action", [b"Done."])], [read("read")]) + report = self.run_cli(1) + self.assertEqual(report["windows"][0]["attempts"][0]["packaging"], "EXTRA_CONTENT") + self.assertEqual(report["windows"][0]["attempts"][0]["content"], "COMPLETE") + + def test_extra_output_in_same_body_prevents_standalone(self): + self.prepare([boundary(), call("read"), returned("read", [ENTRY + b"extra"]), call("action"), returned("action", [b"Done."])], [read("read")]) + self.assertEqual(self.run_cli(1)["windows"][0]["attempts"][0]["packaging"], "EXTRA_CONTENT") + + def test_same_call_read_and_action_is_late(self): + self.prepare([boundary(), call("read-action"), returned("read-action", [ENTRY])], [read("read-action")], governed=("read-action",)) + report = self.run_cli(1) + self.assertEqual(report["windows"][0]["first_action_row"], 2) + attempt = report["windows"][0]["attempts"][0] + self.assertEqual((attempt["content"], attempt["packaging"], attempt["ordering"]), ("COMPLETE", "STANDALONE", "LATE")) + + def test_later_standalone_does_not_repair_first_action(self): + self.prepare([boundary(), call("first"), returned("first", [ENTRY, STEP]), call("action"), returned("action", [b"Done."]), call("late"), returned("late", [ENTRY])], + [read("first"), read("first", "step", 2), read("late")]) + attempts = self.run_cli(1)["windows"][0]["attempts"] + self.assertEqual(attempts[-1]["ordering"], "LATE") + self.assertFalse(any(a["repaired_before_action"] for a in attempts)) + + def test_invalid_attempt_after_valid_read_is_not_clean(self): + self.prepare([boundary(), call("first"), returned("first", [ENTRY]), call("mixed"), returned("mixed", [ENTRY, STEP]), + call("action"), returned("action", [b"Done."])], + [read("first"), read("mixed"), read("mixed", "step", 2)]) + report = self.run_cli(1) + self.assertEqual(report["windows"][0]["requirements"][0]["unrepaired_violation_attempts"], [1]) + + def test_flagged_attempt_can_be_repaired_before_action(self): + self.prepare([boundary(), call("flagged"), returned("flagged", [ENTRY], truncated=True), call("repair"), returned("repair", [ENTRY]), + call("action"), returned("action", [b"Done."])], [read("flagged"), read("repair")]) + report = self.run_cli(0) + self.assertEqual(report["status"], "SATISFIED_WITH_REPAIRS") + self.assertTrue(report["windows"][0]["attempts"][0]["repaired_before_action"]) + + def test_source_notice_words_are_exact_content(self): + self.files["entry"] = b"Read after compaction, truncation, or an output truncated example.\n" + self.prepare([boundary(), call("read"), returned("read", [self.files["entry"]]), call("action"), returned("action", [b"Done."])], [read("read")]) + self.assertEqual(self.run_cli(0)["windows"][0]["attempts"][0]["transport"], "CLEAR") + + def test_literal_notice_source_is_not_transport(self): + self.files["entry"] = b"output truncated\n" + self.prepare([boundary(), call("read"), returned("read", [self.files["entry"]]), call("action"), returned("action", [b"Done."])], [read("read")]) + self.run_cli(0) + + def test_structured_truncation_with_exact_bytes_is_inconclusive(self): + self.basic() + self.rows[2]["truncated"] = True + self.rebind() + attempt = self.run_cli(2)["windows"][0]["attempts"][0] + self.assertEqual(attempt["content"], "COMPLETE") + self.assertIn("structured_truncation", attempt["reasons"]) + + def test_notice_outside_source_stays_visible(self): + self.prepare([boundary(), call("read"), returned("read", [ENTRY + b"Warning: truncated output\n"]), call("action"), returned("action", [b"Done."])], [read("read")]) + self.assertIn("transport_notice", self.run_cli(2)["windows"][0]["attempts"][0]["reasons"]) + + def test_nonzero_command_does_not_prove_delivery(self): + self.basic() + body = json.loads(self.rows[2]["payload"]["output"][1]["text"]) + body["exit_code"] = 7 + self.rows[2]["payload"]["output"][1]["text"] = json.dumps(body) + self.rebind() + self.assertIn("command_failed", self.run_cli(2)["windows"][0]["attempts"][0]["reasons"]) + + def test_prefix_cannot_satisfy_whole_entry(self): + self.prepare([boundary(), call("read"), returned("read", [ENTRY[:12]]), call("action"), returned("action", [b"Done."])], + [read("read", source=[0, 12])]) + self.assertFalse(self.run_cli(1)["windows"][0]["requirements"][0]["satisfied_before_action"]) + + def test_wrong_bytes_are_inconclusive(self): + self.prepare([boundary(), call("read"), returned("read", [b"x" * len(ENTRY)]), call("action"), returned("action", [b"Done."])], [read("read")]) + self.run_cli(2) + + def test_missing_pair_is_inconclusive(self): + self.basic() + self.rows.pop() + self.rebind() + self.assertEqual(self.run_cli(2)["errors"][0]["code"], "unpaired_outer_call") + + def test_duplicate_call_identity_is_inconclusive(self): + self.basic() + self.rows.append(copy.deepcopy(self.rows[1])) + self.rebind() + self.assertEqual(self.run_cli(2)["errors"][0]["code"], "duplicate_outer_identity") + + def test_later_duplicate_return_invalidates_earlier_apparent_success(self): + self.basic() + self.rows.append(copy.deepcopy(self.rows[2])) + self.rebind() + self.assertEqual(self.run_cli(2)["errors"][0]["code"], "duplicate_outer_identity") + + def test_omitted_call_review_cannot_hide_action(self): + self.basic() + self.manifest["calls"].pop() + self.write_manifest() + self.assertEqual(self.run_cli(2)["errors"][0]["code"], "incomplete_call_review") + + def test_omitted_native_boundary_is_inconclusive(self): + self.basic() + self.rows.append(boundary()) + self.rebind() + self.assertEqual(self.run_cli(2)["errors"][0]["code"], "omitted_native_boundary") + + def test_unclaimed_exact_governing_body_is_inconclusive(self): + self.basic() + self.manifest["reads"] = [] + self.write_manifest() + self.assertEqual(self.run_cli(2)["errors"][0]["code"], "undeclared_governing_body") + + def test_raw_hash_binding(self): + self.basic() + self.raw.write_bytes(self.raw.read_bytes() + b"\n") + self.assertEqual(self.run_cli(2)["errors"][0]["code"], "raw_hash_mismatch") + + def test_row_hash_binding(self): + self.basic() + self.manifest["reads"][0]["return_sha256"] = "0" * 64 + self.write_manifest() + self.assertEqual(self.run_cli(2)["errors"][0]["code"], "row_hash_mismatch") + + def test_expected_hash_binding(self): + self.basic() + (self.directory / "entry.md").write_bytes(b"other") + self.assertEqual(self.run_cli(2)["errors"][0]["code"], "expected_file_hash_or_bytes_mismatch") + + def test_malformed_and_non_utf8_raw(self): + for data in (b"{\n", b"\xff\n", b""): + with self.subTest(data=data): + self.basic() + self.raw.write_bytes(data) + self.manifest["raw_sha256"] = digest(data) + self.write_manifest() + self.run_cli(2) + + def test_duplicate_json_keys(self): + self.basic() + self.raw.write_bytes(self.raw.read_bytes().replace(b'"type": "compacted"', b'"type": "compacted", "type": "compacted"', 1)) + self.manifest["raw_sha256"] = digest(self.raw.read_bytes()) + self.write_manifest() + self.assertEqual(self.run_cli(2)["errors"][0]["code"], "duplicate_json_key") + + def test_unknown_record_is_inconclusive(self): + self.basic() + self.rows[0]["type"] = "unknown_native_action" + self.rebind() + self.assertEqual(self.run_cli(2)["errors"][0]["code"], "unsupported_record") + + def test_unresolved_call_prevents_pass(self): + self.basic() + self.manifest["calls"][0]["disposition"] = "unresolved" + self.write_manifest() + self.assertEqual(self.run_cli(2)["windows"][0]["unresolved_call_rows"], [2]) + + def test_mode_chunks_are_rejected_by_policy(self): + self.basic() + self.manifest["boundaries"][0]["requirements"][0]["policy"] = "explicit_chunks" + self.write_manifest() + self.assertEqual(self.run_cli(2)["errors"][0]["code"], "mode_entry_cannot_use_chunks") + + def test_guidance_chunks_cover_exact_bytes(self): + split = 17 + self.prepare([boundary(), call("entry"), returned("entry", [ENTRY]), call("one"), returned("one", [STEP[:split]]), + call("two"), returned("two", [STEP[split:]]), call("action"), returned("action", [b"Done."])], + [read("entry"), read("one", "step", source=[0, split]), read("two", "step", source=[split, len(STEP)])], + requirements=[{"file": "entry", "policy": "whole_standalone"}, {"file": "step", "policy": "explicit_chunks"}], + scope="CURRENT_STEP_RECOVERY") + self.assertTrue(all(r["satisfied_before_action"] for r in self.run_cli(0)["windows"][0]["requirements"])) + + def test_chunks_do_not_cross_windows(self): + split = 17 + self.prepare([boundary(), call("entry"), returned("entry", [ENTRY]), call("one"), returned("one", [STEP[:split]]), + boundary(), call("two"), returned("two", [STEP[split:]]), call("action"), returned("action", [b"Done."])], + [read("entry"), read("one", "step", source=[0, split]), read("two", "step", source=[split, len(STEP)])], + requirements=[{"file": "entry", "policy": "whole_standalone"}, {"file": "step", "policy": "explicit_chunks"}], scope="CURRENT_STEP_RECOVERY") + report = self.run_cli(1) + self.assertEqual([w["status"] for w in report["windows"]], ["NO_LATER_GOVERNED_ACTION", "FAIL"]) + self.assertEqual((report["applicable_windows"], report["non_applicable_windows"]), (1, 1)) + + def test_multiple_applicable_windows(self): + self.prepare([boundary(), call("one"), returned("one", [ENTRY]), call("action"), returned("action", [b"Done."]), + boundary(), call("action-two"), returned("action-two", [b"Done."])], [read("one")], governed=("action", "action-two")) + report = self.run_cli(1) + self.assertEqual([w["status"] for w in report["windows"]], ["SATISFIED", "FAIL"]) + self.assertEqual(report["applicable_windows"], 2) + + def test_no_later_action_is_not_green(self): + self.prepare([boundary(), call("read"), returned("read", [ENTRY])], [read("read")]) + report = self.run_cli(3) + self.assertEqual(report["status"], "NO_LATER_GOVERNED_ACTION") + self.assertEqual(report["applicable_windows"], 0) + self.assertEqual(report["non_applicable_windows"], 1) + + def test_empty_boundary_list_is_not_green(self): + self.basic() + self.manifest["boundaries"] = [] + self.write_manifest() + self.assertEqual(self.run_cli(2)["errors"][0]["code"], "missing_boundary") + + def test_initial_injection_does_not_satisfy_recovery(self): + message = {"type": "response_item", "payload": {"type": "message", "role": "user", "content": [{"type": "input_text", "text": ENTRY.decode()}]}} + self.prepare([message, boundary(), call("action"), returned("action", [b"Done."])]) + self.manifest["initial_injections"] = [{"row": 1, "row_sha256": digest(self.raw.read_bytes().splitlines(keepends=True)[0]), + "file": "entry", "block": 0, "output_range": [0, len(ENTRY)]}] + self.write_manifest() + report = self.run_cli(1) + self.assertEqual(report["initial_deliveries"], [{"row": 1, "file": "entry", "exact_content": True, "recovery_credit": False}]) + + def test_replacement_history_does_not_create_read(self): + marker = boundary() + marker["payload"]["replacement_history"] = [call("nested"), returned("nested", [ENTRY])] + self.prepare([marker, call("action"), returned("action", [b"Done."])]) + self.assertEqual(self.run_cli(1)["windows"][0]["attempts"], []) + + def test_preboundary_read_does_not_satisfy_recovery(self): + self.prepare([call("before"), returned("before", [ENTRY]), boundary(), call("action"), returned("action", [b"Done."])], [read("before")]) + report = self.run_cli(1) + self.assertEqual(len(report["outside_window_attempts"]), 1) + self.assertEqual(report["windows"][0]["attempts"], []) + + def test_call_spanning_boundary_cannot_satisfy_new_window(self): + self.prepare([boundary(), call("read"), boundary(), returned("read", [ENTRY]), call("action"), returned("action", [b"Done."])], [read("read")]) + report = self.run_cli(1) + self.assertEqual(report["windows"][0]["attempts"][0]["ordering"], "CROSSES_BOUNDARY") + self.assertFalse(report["windows"][1]["requirements"][0]["satisfied_before_action"]) + + def test_utf8_byte_ranges_and_unchanged_inputs(self): + self.files["entry"] = "Recover café 🥔.\n".encode() + self.prepare([boundary(), call("read"), returned("read", [self.files["entry"]]), call("action"), returned("action", [b"Done."])], [read("read")]) + before = {p.name: p.read_bytes() for p in self.directory.iterdir()} + first = self.run_cli(0) + second = self.run_cli(0) + self.assertEqual(first, second) + self.assertEqual(before, {p.name: p.read_bytes() for p in self.directory.iterdir()}) + + def test_unsupported_output_cannot_pass(self): + self.basic() + self.rows[2]["payload"]["output"][1] = {"type": "image", "data": "sample"} + self.rebind() + self.assertEqual(self.run_cli(2)["windows"][0]["attempts"][0]["transport"], "UNVERIFIED") + + def test_malformed_inner_envelope_is_retained_as_unverified(self): + self.basic() + self.rows[2]["payload"]["output"][1]["text"] = '{"exit_code":0,"output":' + self.rebind() + self.run_cli(2) + + def test_supplied_boundary_cannot_hide_call_start(self): + self.basic() + self.manifest["boundaries"][0].update(row=2, row_sha256=self.manifest["calls"][0]["row_sha256"], kind="supplied_context_summary") + self.rows[0]["type"] = "event_msg" + self.rebind() + self.assertEqual(self.run_cli(2)["errors"][0]["code"], "summary_boundary_not_user_message") + + +if __name__ == "__main__": + unittest.main() From 9107c50d541dfcc4d98a09d770c190f1e0a201c1 Mon Sep 17 00:00:00 2001 From: mdsmithaustin <15272014+mdsmithaustin@users.noreply.github.com> Date: Tue, 20 Oct 2026 17:59:14 -0600 Subject: [PATCH 03/27] no-mistakes(review): Fix recovery coverage, frozen verdicts, and malformed record handling --- evals/skill-recovery/README.md | 4 +- evals/skill-recovery/evaluate.py | 14 +++-- tools/test_skill_recovery.py | 92 ++++++++++++++++++++++++++++++++ 3 files changed, 103 insertions(+), 7 deletions(-) diff --git a/evals/skill-recovery/README.md b/evals/skill-recovery/README.md index f7c1ca38..07a75653 100644 --- a/evals/skill-recovery/README.md +++ b/evals/skill-recovery/README.md @@ -61,7 +61,7 @@ Boundary kinds are `native_compacted`, `supplied_context_summary`, and `supplied Each requirement has `file` and `policy`. The mode entry always uses `whole_standalone`. Other guidance can use `whole_standalone` or `explicit_chunks`. `MODE_ENTRY_CHECKPOINT` allows only the entry requirement. `CURRENT_STEP_RECOVERY` requires the reviewer to establish the complete set for that step. A narrow entry result does not certify the whole procedure. -Call dispositions are `governed`, `incidental`, and `unresolved`. Read claims do not replace dispositions. A call can both read guidance and perform governed work. Its action occurs at call start, before its returned guidance can count. An unresolved disposition prevents a conclusive window result. +Call dispositions are `governed`, `incidental`, and `unresolved`. Read claims do not replace dispositions. A call can both read guidance and perform governed work. Its action occurs at call start, before its returned guidance can count. An unresolved disposition before the first governed call, or in a window without one, prevents a conclusive window result. Later unresolved calls and late or boundary-crossing unverified reads remain visible without changing the verdict frozen at the first governed call. Invalid raw input still invalidates the entire assessment. Exact source spans must match exact returned body spans. Overlapping output claims are invalid. A complete known governing body left unclaimed anywhere in the recording is invalid. This detects an omitted earlier full read attempt. The reviewer still owns the completeness of file inventory, partial-read claims, supplied boundaries, and action semantics. @@ -79,7 +79,7 @@ Structured truncation flags, transport notices outside matched source spans, com A requirement is satisfied only by clear, standalone, exact content returned within its recovery window before the first governed call. The entry requires one whole-body claim. Explicit helper chunks can accumulate within one window. Pre-boundary reads, initial injections, replacement history, late returns, and reads crossing a later boundary receive no recovery credit. -Attempts remain in the report after a repair. A later eligible read can repair an earlier violation before the action. An invalid required-file attempt after the final eligible read remains an unrepaired violation. The evaluator does not erase a bad batch to produce an unqualified clean result. +Attempts remain in the report after a repair. Later eligible reads must cover the entire required file after each violation and before the action to repair it. Earlier chunks cannot supply missing repair coverage. An invalid required-file attempt after the final eligible read remains an unrepaired violation. The evaluator does not erase a bad batch to produce an unqualified clean result. ## Test reference diff --git a/evals/skill-recovery/evaluate.py b/evals/skill-recovery/evaluate.py index 80d0ab63..fd288b59 100644 --- a/evals/skill-recovery/evaluate.py +++ b/evals/skill-recovery/evaluate.py @@ -213,7 +213,8 @@ def _trace(raw): for number, line in enumerate(lines, 1): _require(bool(line.strip()), "empty_jsonl_row", number) record = _json(line, number) - _require(isinstance(record, dict) and record.get("type") in allowed, "unsupported_record", number) + _require(isinstance(record, dict) and isinstance(record.get("type"), str) and + record["type"] in allowed, "unsupported_record", number) _require(isinstance(record.get("payload"), dict), "unsupported_payload", number) records.append(record) line_bytes.append(line + (b"\n" if number < len(lines) or raw.endswith(b"\n") else b"")) @@ -498,7 +499,7 @@ def _evaluate(files, windows, calls, dispositions, claims, injections, scope): requirement_reports = [] repaired = False failed = False - inconclusive = bool(window.unresolved) + inconclusive = any(window.first_action is None or row < window.first_action for row in window.unresolved) for file_id, policy in window.requirements.items(): progress = window.coverage.get(file_id, []) satisfied = _complete([span for span, _ in progress], len(files[file_id].data)) @@ -506,12 +507,13 @@ def _evaluate(files, windows, calls, dispositions, claims, injections, scope): witnesses = sorted({index for _, index in progress}) if satisfied else [] unrepaired = [] if satisfied: - finish = max(window.attempts[index]["return_row"] for index in witnesses) for index, attempt in attempts: invalid = (attempt["packaging"] != "STANDALONE" or attempt["transport"] != "CLEAR" or attempt["content"] == "UNVERIFIED" or (policy == "whole_standalone" and not attempt["whole_body"])) if invalid and attempt["ordering"] == "BEFORE_ACTION": - if attempt["return_row"] < finish: + if _complete([span for span, witness in progress + if window.attempts[witness]["return_row"] > attempt["return_row"]], + len(files[file_id].data)): attempt["repaired_before_action"] = True repaired = True else: @@ -519,7 +521,9 @@ def _evaluate(files, windows, calls, dispositions, claims, injections, scope): failed = True else: failed = True - if any(attempt["transport"] != "CLEAR" or attempt["content"] == "UNVERIFIED" for _, attempt in attempts): + if any(attempt["ordering"] == "BEFORE_ACTION" and + (attempt["transport"] != "CLEAR" or attempt["content"] == "UNVERIFIED") + for _, attempt in attempts): inconclusive = True requirement_reports.append({"file": file_id, "policy": policy, "satisfied_before_action": satisfied, "witness_attempts": witnesses, "unrepaired_violation_attempts": unrepaired, diff --git a/tools/test_skill_recovery.py b/tools/test_skill_recovery.py index 0e0732b4..077fe19d 100644 --- a/tools/test_skill_recovery.py +++ b/tools/test_skill_recovery.py @@ -300,6 +300,65 @@ def test_unknown_record_is_inconclusive(self): self.rebind() self.assertEqual(self.run_cli(2)["errors"][0]["code"], "unsupported_record") + def test_non_text_record_tags_are_inconclusive(self): + for tag in ([], {}, None, 7, True): + with self.subTest(tag=tag): + self.basic() + self.rows[0]["type"] = tag + self.rebind() + report = self.run_cli(2) + self.assertEqual(report["status"], "INCONCLUSIVE") + self.assertEqual(report["errors"], [{"code": "unsupported_record", "row": 1}]) + + def test_post_action_unresolved_calls_preserve_frozen_verdict(self): + for satisfied in (False, True): + with self.subTest(satisfied=satisfied): + rows = [boundary()] + if satisfied: + rows += [call("entry"), returned("entry", [ENTRY])] + rows += [call("action"), returned("action", [b"Done."]), + call("unresolved"), returned("unresolved", [b"Later observation."])] + self.prepare(rows, [read("entry")] if satisfied else []) + self.manifest["calls"][-1]["disposition"] = "unresolved" + self.write_manifest() + report = self.run_cli(0 if satisfied else 1) + self.assertEqual(report["status"], "SATISFIED" if satisfied else "FAIL") + self.assertEqual(report["windows"][0]["unresolved_call_rows"], [len(rows) - 1]) + + def test_late_unverified_reads_preserve_frozen_verdict(self): + for satisfied in (False, True): + for observation in ("truncated", "command_failed", "unsupported", "wrong_bytes"): + with self.subTest(satisfied=satisfied, observation=observation): + rows = [boundary()] + reads = [] + if satisfied: + rows += [call("entry"), returned("entry", [ENTRY])] + reads.append(read("entry")) + late = returned("late", [ENTRY]) + if observation == "truncated": + late["truncated"] = True + elif observation == "command_failed": + late["payload"]["output"][1]["text"] = json.dumps({"exit_code": 7, "output": ENTRY.decode()}) + elif observation == "unsupported": + late["payload"]["output"][1] = {"type": "image", "data": "sample"} + else: + late = returned("late", [b"x" * len(ENTRY)]) + rows += [call("action"), returned("action", [b"Done."]), call("late"), late] + self.prepare(rows, reads + [read("late")]) + report = self.run_cli(0 if satisfied else 1) + self.assertEqual(report["status"], "SATISFIED" if satisfied else "FAIL") + attempt = report["windows"][0]["attempts"][-1] + self.assertEqual(attempt["ordering"], "LATE") + self.assertFalse(attempt["repaired_before_action"]) + + def test_flagged_return_crossing_boundary_preserves_failure(self): + self.prepare([boundary(), call("read"), call("action"), returned("action", [b"Done."]), boundary(), + returned("read", [ENTRY], truncated=True), call("next-action"), returned("next-action", [b"Done."])], + [read("read")], governed=("action", "next-action")) + report = self.run_cli(1) + self.assertEqual([window["status"] for window in report["windows"]], ["FAIL", "FAIL"]) + self.assertEqual(report["windows"][0]["attempts"][0]["ordering"], "CROSSES_BOUNDARY") + def test_unresolved_call_prevents_pass(self): self.basic() self.manifest["calls"][0]["disposition"] = "unresolved" @@ -321,6 +380,39 @@ def test_guidance_chunks_cover_exact_bytes(self): scope="CURRENT_STEP_RECOVERY") self.assertTrue(all(r["satisfied_before_action"] for r in self.run_cli(0)["windows"][0]["requirements"])) + def test_chunk_repair_requires_complete_post_violation_coverage(self): + split = 17 + for repair in ("prefix", "suffix", "complete"): + with self.subTest(repair=repair): + rows = [boundary(), call("one"), returned("one", [STEP[:split]]), + call("two"), returned("two", [STEP[split:]]), + call("mixed"), returned("mixed", [ENTRY, STEP])] + reads = [read("one", "step", source=[0, split]), + read("two", "step", source=[split, len(STEP)]), + read("mixed"), read("mixed", "step", 2)] + if repair in ("prefix", "complete"): + rows += [call("prefix"), returned("prefix", [STEP[:split]])] + reads.append(read("prefix", "step", source=[0, split])) + if repair in ("suffix", "complete"): + rows += [call("suffix"), returned("suffix", [STEP[split:]])] + reads.append(read("suffix", "step", source=[split, len(STEP)])) + rows += [call("entry"), returned("entry", [ENTRY]), call("action"), returned("action", [b"Done."])] + reads.append(read("entry")) + self.prepare(rows, reads, + requirements=[{"file": "entry", "policy": "whole_standalone"}, + {"file": "step", "policy": "explicit_chunks"}], scope="CURRENT_STEP_RECOVERY") + complete = repair == "complete" + report = self.run_cli(0 if complete else 1) + self.assertEqual(report["status"], "SATISFIED_WITH_REPAIRS" if complete else "FAIL") + window = report["windows"][0] + step = window["requirements"][1] + self.assertTrue(step["satisfied_before_action"]) + self.assertEqual(step["unrepaired_violation_attempts"], [] if complete else [3]) + self.assertEqual(window["attempts"][2]["packaging"], "MIXED_FILES") + self.assertTrue(window["attempts"][2]["repaired_before_action"]) + self.assertEqual(window["attempts"][3]["packaging"], "MIXED_FILES") + self.assertEqual(window["attempts"][3]["repaired_before_action"], complete) + def test_chunks_do_not_cross_windows(self): split = 17 self.prepare([boundary(), call("entry"), returned("entry", [ENTRY]), call("one"), returned("one", [STEP[:split]]), From 47d40c668badf9806d4a3910e1ab05eb78b76e38 Mon Sep 17 00:00:00 2001 From: mdsmithaustin <15272014+mdsmithaustin@users.noreply.github.com> Date: Tue, 20 Oct 2026 18:33:54 -0600 Subject: [PATCH 04/27] no-mistakes(review): Prevent fragment claims from hiding complete governing bodies --- evals/skill-recovery/evaluate.py | 18 +++++++--- tools/test_skill_recovery.py | 61 ++++++++++++++++++++++++++++++++ 2 files changed, 74 insertions(+), 5 deletions(-) diff --git a/evals/skill-recovery/evaluate.py b/evals/skill-recovery/evaluate.py index fd288b59..32a6d621 100644 --- a/evals/skill-recovery/evaluate.py +++ b/evals/skill-recovery/evaluate.py @@ -357,13 +357,21 @@ def row_ref(value, number_key="row", hash_key="row_sha256"): for block in {claim.block for claim in selected}: spans = sorted(claim.output for claim in selected if claim.block == block) _require(all(left[1] <= right[0] for left, right in zip(spans, spans[1:])), "overlapping_output_claims", calls[call_id].return_row) + data = b"" + offsets = {} for body in calls[call_id].bodies: - if body.data: - spans = [claim.output for claim in selected if claim.block == body.block and - body.data[claim.output[0]:claim.output[1]] == files[claim.file].data[claim.source[0]:claim.source[1]]] - residual = _residual(body.data, spans) - _require(not any(file.data in residual for file in files.values()), + if body.data is not None: + offsets[body.block] = len(data) + data += body.data + for file in files.values(): + start = data.find(file.data) + while start != -1: + spans = [claim.source for claim in selected if claim.file == file.id and claim.block in offsets and + offsets[claim.block] + claim.output[0] == start + claim.source[0] and + offsets[claim.block] + claim.output[1] == start + claim.source[1]] + _require(_complete(spans, len(file.data)), "undeclared_governing_body", calls[call_id].return_row) + start = data.find(file.data, start + 1) injections = [] for item in _array(manifest.get("initial_injections", [])): _object(item, {"row", "row_sha256", "file", "block", "output_range"}) diff --git a/tools/test_skill_recovery.py b/tools/test_skill_recovery.py index 077fe19d..90cd23e8 100644 --- a/tools/test_skill_recovery.py +++ b/tools/test_skill_recovery.py @@ -262,6 +262,67 @@ def test_unclaimed_exact_governing_body_is_inconclusive(self): self.write_manifest() self.assertEqual(self.run_cli(2)["errors"][0]["code"], "undeclared_governing_body") + def test_foreign_fragments_cannot_hide_complete_entry(self): + for embedded in (False, True): + for fragments in (1, 3): + for repaired in (False, True): + with self.subTest(embedded=embedded, fragments=fragments, repaired=repaired): + prefix = b"unrelated prefix\n" if embedded else b"" + body = prefix + ENTRY + (b"unrelated suffix\n" if embedded else b"") + masks = [read("mixed", "step", source=[STEP.index(bytes([byte])), STEP.index(bytes([byte])) + 1], + output=[len(prefix) + offset, len(prefix) + offset + 1]) + for offset, byte in enumerate(ENTRY[:fragments])] + rows = [boundary()] + if not repaired: + rows += [call("valid"), returned("valid", [ENTRY])] + rows += [call("mixed"), returned("mixed", [body, STEP])] + if repaired: + rows += [call("valid"), returned("valid", [ENTRY])] + rows += [call("action"), returned("action", [b"Done."])] + self.prepare(rows, masks + [read("mixed", "step", 2), read("valid")]) + report = self.run_cli(2) + self.assertEqual(report["status"], "INCONCLUSIVE") + self.assertEqual(report["errors"][0]["code"], "undeclared_governing_body") + + def test_every_repeated_or_overlapping_occurrence_needs_own_claim(self): + for entry, body, source, output in ( + (ENTRY, ENTRY + ENTRY, [0, len(ENTRY)], [0, len(ENTRY)]), + (b"aaa", b"aaaa", [0, 3], [0, 3]), + (ENTRY, ENTRY, [0, 12], [0, 12]), + (ENTRY, ENTRY + ENTRY[:12], [0, 12], [len(ENTRY), len(ENTRY) + 12]), + ): + with self.subTest(entry=entry, output=output): + self.files["entry"] = entry + self.prepare([boundary(), call("read"), returned("read", [body]), + call("valid"), returned("valid", [entry]), call("action"), returned("action", [b"Done."])], + [read("read", source=source, output=output), read("valid")]) + self.assertEqual(self.run_cli(2)["errors"][0]["code"], "undeclared_governing_body") + + def test_declared_repeated_occurrences_are_accounted_for(self): + self.prepare([boundary(), call("read"), returned("read", [ENTRY + ENTRY]), + call("action"), returned("action", [b"Done."])], + [read("read"), read("read", output=[len(ENTRY), 2 * len(ENTRY)])]) + self.assertEqual(self.run_cli(0)["status"], "SATISFIED") + + def test_split_blocks_cannot_hide_complete_entry(self): + for split in (1, 17, len(ENTRY) - 1): + for foreign_mask in (False, True): + with self.subTest(split=split, foreign_mask=foreign_mask): + reads = [read("mixed", "step", 3), read("valid")] + if foreign_mask: + reads.append(read("mixed", "step", 1, source=[STEP.index(b"R"), STEP.index(b"R") + 1], output=[0, 1])) + self.prepare([boundary(), call("mixed"), returned("mixed", [ENTRY[:split], ENTRY[split:], STEP]), + call("valid"), returned("valid", [ENTRY]), call("action"), returned("action", [b"Done."])], reads) + self.assertEqual(self.run_cli(2)["errors"][0]["code"], "undeclared_governing_body") + + def test_declared_split_blocks_preserve_whole_entry_policy(self): + self.prepare([boundary(), call("split"), returned("split", [ENTRY[:17], ENTRY[17:]]), + call("valid"), returned("valid", [ENTRY]), call("action"), returned("action", [b"Done."])], + [read("split", source=[0, 17]), read("split", block=2, source=[17, len(ENTRY)]), read("valid")]) + report = self.run_cli(0) + self.assertEqual(report["status"], "SATISFIED_WITH_REPAIRS") + self.assertFalse(report["windows"][0]["attempts"][0]["whole_body"]) + def test_raw_hash_binding(self): self.basic() self.raw.write_bytes(self.raw.read_bytes() + b"\n") From 8797e17d83fb0a50e1139dde0fccf8c54cc7721e Mon Sep 17 00:00:00 2001 From: mdsmithaustin <15272014+mdsmithaustin@users.noreply.github.com> Date: Tue, 20 Oct 2026 19:35:19 -0600 Subject: [PATCH 05/27] test(evals): show unsupported carriers hiding a complete mixed batch --- tools/test_skill_recovery.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/tools/test_skill_recovery.py b/tools/test_skill_recovery.py index 90cd23e8..763f8b69 100644 --- a/tools/test_skill_recovery.py +++ b/tools/test_skill_recovery.py @@ -284,6 +284,30 @@ def test_foreign_fragments_cannot_hide_complete_entry(self): self.assertEqual(report["status"], "INCONCLUSIVE") self.assertEqual(report["errors"][0]["code"], "undeclared_governing_body") + def test_unsupported_carriers_cannot_hide_complete_entry(self): + def envelope(body): + return {"chunk_id": "c", "wall_time_seconds": 0.1, "exit_code": 0, "original_token_count": 1, "output": body.decode()} + + shapes = { + "status_value": lambda body: {"type": "input_text", "text": json.dumps({"status": "fulfilled", "value": envelope(body)})}, + "i_status_value": lambda body: {"type": "input_text", "text": json.dumps({"i": 0, "status": "fulfilled", "value": envelope(body)})}, + "envelope_extra_key": lambda body: {"type": "input_text", "text": json.dumps(dict(envelope(body), stderr=""))}, + "output_metadata": lambda body: {"type": "input_text", "text": json.dumps({"output": body.decode(), "metadata": {"exit_code": 0}})}, + "block_extra_key": lambda body: {"type": "input_text", "text": json.dumps(envelope(body)), "annotations": []}, + } + for name, shape in shapes.items(): + for hidden in (True, False): + with self.subTest(shape=name, hidden=hidden): + mixed = returned("mixed", []) + mixed["payload"]["output"] += [shape(ENTRY if hidden else b"Done.\n"), shape(STEP if hidden else b"Done.\n")] + self.prepare([boundary(), call("valid"), returned("valid", [ENTRY]), call("mixed"), mixed, + call("action"), returned("action", [b"Done."])], [read("valid")]) + report = self.run_cli(2 if hidden else 0) + if hidden: + self.assertEqual(report["errors"][0]["code"], "undeclared_governing_body") + else: + self.assertEqual(report["status"], "SATISFIED") + def test_every_repeated_or_overlapping_occurrence_needs_own_claim(self): for entry, body, source, output in ( (ENTRY, ENTRY + ENTRY, [0, len(ENTRY)], [0, len(ENTRY)]), From 0e3f5bde180ff4b118c4f12eac95ff75222c181e Mon Sep 17 00:00:00 2001 From: mdsmithaustin <15272014+mdsmithaustin@users.noreply.github.com> Date: Tue, 20 Oct 2026 19:36:18 -0600 Subject: [PATCH 06/27] fix(evals): scan unsupported carriers for escaped governing bodies --- evals/skill-recovery/evaluate.py | 37 ++++++++++++++++++++++---------- 1 file changed, 26 insertions(+), 11 deletions(-) diff --git a/evals/skill-recovery/evaluate.py b/evals/skill-recovery/evaluate.py index 32a6d621..dde0c52c 100644 --- a/evals/skill-recovery/evaluate.py +++ b/evals/skill-recovery/evaluate.py @@ -103,8 +103,7 @@ class _File: class _Body: block: int kind: str - raw_hash: str - raw_size: int + raw: bytes data: bytes | None flags: tuple[str, ...] @@ -154,27 +153,27 @@ def _bodies(output, row): flags = _flags(block) if block.get("type") not in ("input_text", "text") or "text" not in block or block.keys() - {"type", "text", "truncated"}: encoded = json.dumps(block, sort_keys=True).encode() - result.append(_Body(index, "unsupported", _sha(encoded), len(encoded), None, flags)) + result.append(_Body(index, "unsupported", encoded, None, flags)) continue text = block["text"] else: text = block data = _text(text, row) if index == 0 and _HEADER.fullmatch(text): - result.append(_Body(index, "completion_header", _sha(data), len(data), None, flags)) + result.append(_Body(index, "completion_header", data, None, flags)) continue inner = None if text.lstrip().startswith("{"): try: inner = _json(text, row) except _InputError: - result.append(_Body(index, "unsupported", _sha(data), len(data), None, flags)) + result.append(_Body(index, "unsupported", data, None, flags)) continue kind = "text" if isinstance(inner, dict) and set(inner) == {"i", "result"}: settled = inner["result"] if type(inner["i"]) is not int or not isinstance(settled, dict) or set(settled) != {"status", "value"} or settled["status"] != "fulfilled": - result.append(_Body(index, "unsupported", _sha(data), len(data), None, flags + ("settled_result_not_fulfilled",))) + result.append(_Body(index, "unsupported", data, None, flags + ("settled_result_not_fulfilled",))) continue inner = settled["value"] kind = "settled_command_result" @@ -182,7 +181,7 @@ def _bodies(output, row): kind = "command_result" if kind != "text": if not isinstance(inner, dict) or inner.keys() - _ENVELOPE_KEYS or not isinstance(inner.get("output"), str): - result.append(_Body(index, "unsupported", _sha(data), len(data), None, flags)) + result.append(_Body(index, "unsupported", data, None, flags)) continue flags += _flags(inner) code = inner.get("exit_code") @@ -193,9 +192,11 @@ def _bodies(output, row): flags += ("command_failed",) if "session_id" in inner and code is None: flags += ("command_still_running",) - result.append(_Body(index, kind, _sha(data), len(data), _text(inner["output"], row), flags)) + result.append(_Body(index, kind, data, _text(inner["output"], row), flags)) + elif inner is not None: + result.append(_Body(index, "unsupported", data, None, flags)) else: - result.append(_Body(index, kind, _sha(data), len(data), data, flags)) + result.append(_Body(index, kind, data, data, flags)) return tuple(result) @@ -352,6 +353,7 @@ def row_ref(value, number_key="row", hash_key="row_sha256"): _require(claim not in seen_claims, "duplicate_read_claim", number) seen_claims.add(claim) claims.append(claim) + encodings = {file.id: _encodings(file.data) for file in files.values()} for call_id in calls: selected = [claim for claim in claims if claim.call == call_id] for block in {claim.block for claim in selected}: @@ -372,6 +374,12 @@ def row_ref(value, number_key="row", hash_key="row_sha256"): _require(_complete(spans, len(file.data)), "undeclared_governing_body", calls[call_id].return_row) start = data.find(file.data, start + 1) + for body in calls[call_id].bodies: + if body.kind == "unsupported": + for file in files.values(): + _require(not any(form in body.raw for form in encodings[file.id]) or + any(claim.file == file.id and claim.block == body.block for claim in selected), + "undeclared_governing_body", calls[call_id].return_row) injections = [] for item in _array(manifest.get("initial_injections", [])): _object(item, {"row", "row_sha256", "file", "block", "output_range"}) @@ -391,6 +399,13 @@ def row_ref(value, number_key="row", hash_key="row_sha256"): return files, windows, calls, dispositions, claims, injections +def _encodings(data): + forms = {data} + for _ in range(2): + forms |= {json.dumps(form.decode(), ensure_ascii=ascii_only)[1:-1].encode() for form in forms for ascii_only in (False, True)} + return forms + + def _complete(spans, size): end = 0 for start, stop in sorted(spans): @@ -587,8 +602,8 @@ def main(argv=None): files, _, calls, _, _, _ = bound report["files"] = [{"id": item.id, "role": item.role, "sha256": _sha(item.data), "bytes": len(item.data)} for item in files.values()] report["outer_responses"] = [{"call_id": call.id, "call_row": call.row, "return_row": call.return_row, - "blocks": [{"index": body.block, "carrier": body.kind, "raw_sha256": body.raw_hash, - "raw_bytes": body.raw_size, "body_sha256": _sha(body.data) if body.data is not None else None, + "blocks": [{"index": body.block, "carrier": body.kind, "raw_sha256": _sha(body.raw), + "raw_bytes": len(body.raw), "body_sha256": _sha(body.data) if body.data is not None else None, "body_bytes": len(body.data) if body.data is not None else None} for body in call.bodies]} for call in calls.values()] except _InputError as error: From 60db6a3ee6734e123caa39416c62d3184234dca3 Mon Sep 17 00:00:00 2001 From: mdsmithaustin <15272014+mdsmithaustin@users.noreply.github.com> Date: Tue, 20 Oct 2026 19:36:41 -0600 Subject: [PATCH 07/27] test(evals): show unverified evidence after a valid read failing --- tools/test_skill_recovery.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tools/test_skill_recovery.py b/tools/test_skill_recovery.py index 763f8b69..8ba03dc3 100644 --- a/tools/test_skill_recovery.py +++ b/tools/test_skill_recovery.py @@ -180,6 +180,20 @@ def test_invalid_attempt_after_valid_read_is_not_clean(self): report = self.run_cli(1) self.assertEqual(report["windows"][0]["requirements"][0]["unrepaired_violation_attempts"], [1]) + def test_unverified_attempt_after_valid_read_is_inconclusive(self): + for observation in ("wrong_bytes", "truncated", "unsupported_carrier"): + with self.subTest(observation=observation): + later = returned("later", [ENTRY]) + if observation == "wrong_bytes": + later = returned("later", [b"x" * len(ENTRY)]) + elif observation == "truncated": + later["truncated"] = True + else: + later["payload"]["output"][1]["text"] = json.dumps({"status": "fulfilled", "value": {"exit_code": 0, "output": ENTRY.decode()}}) + self.prepare([boundary(), call("valid"), returned("valid", [ENTRY]), call("later"), later, + call("action"), returned("action", [b"Done."])], [read("valid"), read("later")]) + self.assertEqual(self.run_cli(2)["status"], "INCONCLUSIVE") + def test_flagged_attempt_can_be_repaired_before_action(self): self.prepare([boundary(), call("flagged"), returned("flagged", [ENTRY], truncated=True), call("repair"), returned("repair", [ENTRY]), call("action"), returned("action", [b"Done."])], [read("flagged"), read("repair")]) From c15e074b13748ca889c692004ee2b9d61b36fc9a Mon Sep 17 00:00:00 2001 From: mdsmithaustin <15272014+mdsmithaustin@users.noreply.github.com> Date: Tue, 20 Oct 2026 19:37:00 -0600 Subject: [PATCH 08/27] fix(evals): keep unverified attempts inconclusive after a valid read --- evals/skill-recovery/evaluate.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/evals/skill-recovery/evaluate.py b/evals/skill-recovery/evaluate.py index dde0c52c..caa5cae0 100644 --- a/evals/skill-recovery/evaluate.py +++ b/evals/skill-recovery/evaluate.py @@ -531,14 +531,17 @@ def _evaluate(files, windows, calls, dispositions, claims, injections, scope): unrepaired = [] if satisfied: for index, attempt in attempts: - invalid = (attempt["packaging"] != "STANDALONE" or attempt["transport"] != "CLEAR" or - attempt["content"] == "UNVERIFIED" or (policy == "whole_standalone" and not attempt["whole_body"])) + unverified = attempt["transport"] != "CLEAR" or attempt["content"] == "UNVERIFIED" + invalid = (unverified or attempt["packaging"] != "STANDALONE" or + (policy == "whole_standalone" and not attempt["whole_body"])) if invalid and attempt["ordering"] == "BEFORE_ACTION": if _complete([span for span, witness in progress if window.attempts[witness]["return_row"] > attempt["return_row"]], len(files[file_id].data)): attempt["repaired_before_action"] = True repaired = True + elif unverified: + inconclusive = True else: unrepaired.append(index) failed = True From dcd93f8dca2b39a64cb914c87a554965dca3d3a4 Mon Sep 17 00:00:00 2001 From: mdsmithaustin <15272014+mdsmithaustin@users.noreply.github.com> Date: Tue, 20 Oct 2026 19:37:16 -0600 Subject: [PATCH 09/27] test(evals): show the native Unicode truncation marker read as clear --- tools/test_skill_recovery.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tools/test_skill_recovery.py b/tools/test_skill_recovery.py index 8ba03dc3..18d15c0c 100644 --- a/tools/test_skill_recovery.py +++ b/tools/test_skill_recovery.py @@ -223,6 +223,15 @@ def test_notice_outside_source_stays_visible(self): self.prepare([boundary(), call("read"), returned("read", [ENTRY + b"Warning: truncated output\n"]), call("action"), returned("action", [b"Done."])], [read("read")]) self.assertIn("transport_notice", self.run_cli(2)["windows"][0]["attempts"][0]["reasons"]) + def test_native_unicode_truncation_marker_is_transport(self): + for marker in ("…5 tokens truncated…", "…40 chars truncated…"): + with self.subTest(marker=marker): + head, tail, notice = ENTRY[:20], ENTRY[20:], marker.encode() + self.prepare([boundary(), call("read"), returned("read", [head + notice + tail]), call("action"), returned("action", [b"Done."])], + [read("read", source=[0, 20], output=[0, 20]), + read("read", source=[20, len(ENTRY)], output=[20 + len(notice), len(ENTRY) + len(notice)])]) + self.assertIn("transport_notice", self.run_cli(2)["windows"][0]["attempts"][0]["reasons"]) + def test_nonzero_command_does_not_prove_delivery(self): self.basic() body = json.loads(self.rows[2]["payload"]["output"][1]["text"]) From 3cb79d55be4599d2ba45f6db1614a29e939f66b0 Mon Sep 17 00:00:00 2001 From: mdsmithaustin <15272014+mdsmithaustin@users.noreply.github.com> Date: Tue, 20 Oct 2026 19:37:32 -0600 Subject: [PATCH 10/27] fix(evals): recognize the native Unicode truncation marker --- evals/skill-recovery/evaluate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/evals/skill-recovery/evaluate.py b/evals/skill-recovery/evaluate.py index caa5cae0..26012371 100644 --- a/evals/skill-recovery/evaluate.py +++ b/evals/skill-recovery/evaluate.py @@ -139,7 +139,7 @@ class _Window: _HEADER = re.compile(r"Script completed\nWall time [0-9.]+ seconds\nOutput:\n") -_NOTICE = re.compile(rb"(?im)^(?:Warning: truncated output\b|\[?output (?:truncated|omitted)\b|\.\.\.\s*\d+ (?:characters|tokens) truncated\b)") +_NOTICE = re.compile(rb"(?im)^(?:Warning: truncated output\b|\[?output (?:truncated|omitted)\b|\.\.\.\s*\d+ (?:characters|tokens) truncated\b)|\xe2\x80\xa6\d+ (?:tokens|chars) truncated\xe2\x80\xa6") _ENVELOPE_KEYS = {"output", "exit_code", "chunk_id", "wall_time_seconds", "original_token_count", "session_id", "truncated"} From 35c45adac72750428c4565af60b9488b17771416 Mon Sep 17 00:00:00 2001 From: mdsmithaustin <15272014+mdsmithaustin@users.noreply.github.com> Date: Tue, 20 Oct 2026 19:37:57 -0600 Subject: [PATCH 11/27] test(evals): show native script and function_call headers failing a valid read --- tools/test_skill_recovery.py | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/tools/test_skill_recovery.py b/tools/test_skill_recovery.py index 18d15c0c..d622ada5 100644 --- a/tools/test_skill_recovery.py +++ b/tools/test_skill_recovery.py @@ -52,9 +52,9 @@ def prepare(self, rows, reads=(), governed=("action",), requirements=None, scope self.write_raw() lines = self.raw.read_bytes().splitlines(keepends=True) calls = {r["payload"]["call_id"]: n for n, r in enumerate(rows, 1) - if r["type"] == "response_item" and r["payload"].get("type") == "custom_tool_call"} + if r["type"] == "response_item" and r["payload"].get("type") in ("custom_tool_call", "function_call")} returns = {r["payload"]["call_id"]: n for n, r in enumerate(rows, 1) - if r["type"] == "response_item" and r["payload"].get("type") == "custom_tool_call_output"} + if r["type"] == "response_item" and r["payload"].get("type") in ("custom_tool_call_output", "function_call_output")} boundaries = [(n, "native_compacted") for n, r in enumerate(rows, 1) if r["type"] == "compacted"] boundaries += [(n, "supplied_context_summary") for n in supplied] required = requirements or [{"file": "entry", "policy": "whole_standalone"}] @@ -232,6 +232,23 @@ def test_native_unicode_truncation_marker_is_transport(self): read("read", source=[20, len(ENTRY)], output=[20 + len(notice), len(ENTRY) + len(notice)])]) self.assertIn("transport_notice", self.run_cli(2)["windows"][0]["attempts"][0]["reasons"]) + def test_failed_or_running_script_header_is_inconclusive(self): + for header in ("Script failed\nWall time 0.1 seconds\nOutput:\n", "Script running with cell ID 7\nWall time 0.1 seconds\nOutput:\n", + "Script failed\nError: interrupted\n"): + with self.subTest(header=header): + self.basic() + self.rows[2]["payload"]["output"][0]["text"] = header + self.rebind() + self.assertEqual(self.run_cli(2)["status"], "INCONCLUSIVE") + + def test_native_function_call_text_header_is_inconclusive(self): + header = b"Chunk ID: c1\nWall time: 0.0100 seconds\nProcess exited with code 0\nOriginal token count: 20\nOutput:\n" + native = {"type": "response_item", "payload": {"type": "function_call", "name": "exec_command", "call_id": "read", "arguments": "{}"}} + output = {"type": "response_item", "payload": {"type": "function_call_output", "call_id": "read", "output": (header + ENTRY).decode()}} + self.prepare([boundary(), native, output, call("action"), returned("action", [b"Done."])], + [read("read", block=0, output=[len(header), len(header) + len(ENTRY)])]) + self.assertEqual(self.run_cli(2)["status"], "INCONCLUSIVE") + def test_nonzero_command_does_not_prove_delivery(self): self.basic() body = json.loads(self.rows[2]["payload"]["output"][1]["text"]) From d35bd0a5aed9fe23d1263796a766845dc6c88709 Mon Sep 17 00:00:00 2001 From: mdsmithaustin <15272014+mdsmithaustin@users.noreply.github.com> Date: Tue, 20 Oct 2026 19:38:33 -0600 Subject: [PATCH 12/27] fix(evals): flag failed and running script headers and leave native text headers unsupported --- evals/skill-recovery/evaluate.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/evals/skill-recovery/evaluate.py b/evals/skill-recovery/evaluate.py index 26012371..fafb4f72 100644 --- a/evals/skill-recovery/evaluate.py +++ b/evals/skill-recovery/evaluate.py @@ -138,7 +138,9 @@ class _Window: unresolved: list = field(default_factory=list) -_HEADER = re.compile(r"Script completed\nWall time [0-9.]+ seconds\nOutput:\n") +_HEADER = re.compile(r"Script (completed|failed|running with cell ID [^\n]+)\nWall time [0-9.]+ seconds\nOutput:\n") +_HEADER_FLAGS = {"completed": (), "failed": ("command_failed",), "running": ("command_still_running",)} +_NATIVE_HEADER = re.compile(r"Script (?:completed|failed|running)\b|Chunk ID: |Wall time: |Exit code: |Process exited with code |Original token count: ") _NOTICE = re.compile(rb"(?im)^(?:Warning: truncated output\b|\[?output (?:truncated|omitted)\b|\.\.\.\s*\d+ (?:characters|tokens) truncated\b)|\xe2\x80\xa6\d+ (?:tokens|chars) truncated\xe2\x80\xa6") _ENVELOPE_KEYS = {"output", "exit_code", "chunk_id", "wall_time_seconds", "original_token_count", "session_id", "truncated"} @@ -159,9 +161,14 @@ def _bodies(output, row): else: text = block data = _text(text, row) - if index == 0 and _HEADER.fullmatch(text): + header = _HEADER.fullmatch(text) if index == 0 else None + if header: + flags += _HEADER_FLAGS[header[1].split()[0]] result.append(_Body(index, "completion_header", data, None, flags)) continue + if _NATIVE_HEADER.match(text): + result.append(_Body(index, "unsupported", data, None, flags)) + continue inner = None if text.lstrip().startswith("{"): try: From 4507d1e410d8ee13b938c014677e7fb6a9961e3b Mon Sep 17 00:00:00 2001 From: mdsmithaustin <15272014+mdsmithaustin@users.noreply.github.com> Date: Tue, 20 Oct 2026 19:39:09 -0600 Subject: [PATCH 13/27] test(evals): show deeply nested JSON crashing with the FAIL exit code --- tools/test_skill_recovery.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tools/test_skill_recovery.py b/tools/test_skill_recovery.py index d622ada5..62a41b9d 100644 --- a/tools/test_skill_recovery.py +++ b/tools/test_skill_recovery.py @@ -412,6 +412,19 @@ def test_malformed_and_non_utf8_raw(self): self.write_manifest() self.run_cli(2) + def test_deeply_nested_json_is_reported(self): + depth = 300000 + self.basic() + self.raw.write_bytes(self.raw.read_bytes() + b'{"type": "event_msg", "payload": ' + b"[" * depth + b"]" * depth + b"}\n") + self.manifest["raw_sha256"] = digest(self.raw.read_bytes()) + self.write_manifest() + self.assertEqual(self.run_cli(2)["errors"], [{"code": "json_nesting_too_deep", "row": 6}]) + self.prepare([boundary(), call("read"), returned("read", [ENTRY]), call("deep"), returned("deep", []), + call("action"), returned("action", [b"Done."])], [read("read")]) + self.rows[4]["payload"]["output"].append({"type": "input_text", "text": '{"a":' * depth + "1" + "}" * depth}) + self.rebind() + self.assertEqual(self.run_cli(0)["status"], "SATISFIED") + def test_duplicate_json_keys(self): self.basic() self.raw.write_bytes(self.raw.read_bytes().replace(b'"type": "compacted"', b'"type": "compacted", "type": "compacted"', 1)) From 785bd55f56083c0e341184f976b553ab45d5e706 Mon Sep 17 00:00:00 2001 From: mdsmithaustin <15272014+mdsmithaustin@users.noreply.github.com> Date: Tue, 20 Oct 2026 19:39:28 -0600 Subject: [PATCH 14/27] fix(evals): report deeply nested JSON as inconclusive input --- evals/skill-recovery/evaluate.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/evals/skill-recovery/evaluate.py b/evals/skill-recovery/evaluate.py index fafb4f72..b16c04a7 100644 --- a/evals/skill-recovery/evaluate.py +++ b/evals/skill-recovery/evaluate.py @@ -37,6 +37,8 @@ def constant(_value): return json.loads(data, object_pairs_hook=pairs, parse_constant=constant) except (ValueError, UnicodeError) as error: raise _InputError("invalid_json_or_utf8", row) from error + except RecursionError as error: + raise _InputError("json_nesting_too_deep", row) from error def _sha(data): From ab16d2ad9bb15c5afdfe22a5cf609dda6170445c Mon Sep 17 00:00:00 2001 From: mdsmithaustin <15272014+mdsmithaustin@users.noreply.github.com> Date: Tue, 20 Oct 2026 19:40:15 -0600 Subject: [PATCH 15/27] docs(evals): state skill-recovery scan scope, carriers, and maintained scope --- evals/README.md | 5 ++++- evals/skill-recovery/README.md | 14 +++++++------- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/evals/README.md b/evals/README.md index 5db83745..26bff435 100644 --- a/evals/README.md +++ b/evals/README.md @@ -18,7 +18,9 @@ Oracle and login regression fixtures under `promises/testdata/` are an exception They are reduced, sanitized traces, CLI error records, and recorded verdicts used by model-free tests. `promises/reduce_trace.py` reduces raw traces for oracle fixtures. `promises/results/` is a second exception. It holds aggregated verdict matrices with no model -answers. +answers. `skill-recovery/fixtures/` is a third exception. It holds hand-built, +sanitized projections of native Codex records with replacement text, used by +model-free evaluator tests. The [port-value comparison](port-value/README.md) documents the separate native, upstream, and current-port application study, including its published @@ -34,6 +36,7 @@ Raw execution evidence stays outside Git. | `runtime-probes/`, `spec-probes/`, `unslop/` | Small development corpora. Findings, scope restraint, and preservation matter more than requested vocabulary or loading alone. | | `canon/` | Native application capture and supported outcome-grader controls. Historical screens and process census are in the [archive index](archive/README.md). | | `harness-routing/` | Targeted read/routing instrumentation with separate unresolved validity axes. | +| `skill-recovery/` | Offline evaluator for recovery reads in saved native Codex JSONL. A result covers only its reviewed manifest and recording, not model obedience. | | `port-value/` and pipeline-exit | Completed comparisons and reproducibility records. Exposed ceiling cases can protect regressions, but repeated results are not fresh efficacy evidence. | Passing instrument tests validates the evaluator, not the skill. Count tasks, diff --git a/evals/skill-recovery/README.md b/evals/skill-recovery/README.md index 07a75653..383402d5 100644 --- a/evals/skill-recovery/README.md +++ b/evals/skill-recovery/README.md @@ -12,7 +12,7 @@ From the repository root, the synthetic repaired control runs with this command. python3 evals/skill-recovery/evaluate.py --raw evals/skill-recovery/fixtures/repaired-entry.jsonl --manifest evals/skill-recovery/fixtures/repaired-entry.manifest.json ``` -Failure signals are a nonzero exit, a status other than `SATISFIED_WITH_REPAIRS`, or an applicable window count other than one. The earlier mixed response remains in `windows[0].attempts` with `repaired_before_action` set to true. +Failure signals are a nonzero exit, a status other than `SATISFIED_WITH_REPAIRS`, or an applicable window count other than one. The earlier mixed response remains in `windows[0].attempts`. Its entry attempt, `attempts[0]`, has `repaired_before_action` set to true. Its step attempt stays false because `MODE_ENTRY_CHECKPOINT` has no step requirement to repair. The recorded-failure projection runs with this command. @@ -63,23 +63,23 @@ Each requirement has `file` and `policy`. The mode entry always uses `whole_stan Call dispositions are `governed`, `incidental`, and `unresolved`. Read claims do not replace dispositions. A call can both read guidance and perform governed work. Its action occurs at call start, before its returned guidance can count. An unresolved disposition before the first governed call, or in a window without one, prevents a conclusive window result. Later unresolved calls and late or boundary-crossing unverified reads remain visible without changing the verdict frozen at the first governed call. Invalid raw input still invalidates the entire assessment. -Exact source spans must match exact returned body spans. Overlapping output claims are invalid. A complete known governing body left unclaimed anywhere in the recording is invalid. This detects an omitted earlier full read attempt. The reviewer still owns the completeness of file inventory, partial-read claims, supplied boundaries, and action semantics. +Exact source spans must match exact returned body spans. Overlapping output claims are invalid. A complete known governing body left unclaimed in any outer-call output is invalid. This detects an omitted earlier full read attempt. The scan covers decoded bodies of supported carriers, including a body split across blocks of one response. It also covers the raw text of unsupported blocks, where a body can appear unescaped or JSON-escaped up to two levels deep. A claim on an unsupported block records an unverified attempt. The scan does not cover message records or `replacement_history`, and those deliveries receive no recovery credit. The reviewer still owns the completeness of file inventory, partial-read claims, supplied boundaries, and action semantics. ## Native input reference -The parser supports native `function_call` and `custom_tool_call` records with their matching output records. Each `call_id` has exactly one call and one later return of the corresponding kind. Missing, duplicate, or mismatched pairs invalidate the entire input, including duplicates after an apparent success. +The parser pairs native `function_call` and `custom_tool_call` records with their matching output records. Both kinds use the body carriers below. A block that starts with a native exec text header, such as `Chunk ID: `, `Wall time: `, `Exit code: `, `Process exited with code `, or `Original token count: `, is unsupported. A read inside such a block is inconclusive because no committed fixture fixes that header layout. Each `call_id` has exactly one call and one later return of the corresponding kind. Missing, duplicate, or mismatched pairs invalidate the entire input, including duplicates after an apparent success. Supported top-level record types are `session_meta`, `turn_context`, `event_msg`, `world_state`, `compacted`, `response_item`, `token_usage_record`, and `inter_agent_communication_metadata`. Message and reasoning records provide context. Names in command arguments or file lists do not prove a read. -A returned output can be a string or a list of text blocks. Supported body carriers are plain text, a serialized command result with `output` and `exit_code`, and a fulfilled settled-result wrapper with `i`, `result.status`, and `result.value`. The command-result metadata keys are `chunk_id`, `wall_time_seconds`, `original_token_count`, `session_id`, and `truncated`. Unknown carriers remain unsupported. The parser never recursively flattens arbitrary JSON. +A returned output can be a string or a list of text blocks. Supported body carriers are plain text, a serialized command result with `output` and `exit_code`, and a fulfilled settled-result wrapper with `i`, `result.status`, and `result.value`. The command-result metadata keys are `chunk_id`, `wall_time_seconds`, `original_token_count`, `session_id`, and `truncated`. Unknown carriers remain unsupported. A JSON object block that matches no supported carrier, such as a `status` and `value` result, is unsupported. The parser never recursively flattens arbitrary JSON. JSON nested too deeply for the parser invalidates a raw row and makes a block unsupported. -A separate first-block completion header matching `Script completed`, `Wall time ... seconds`, and `Output:` is metadata. Other unclaimed substantive output prevents standalone packaging. Multi-file bodies and blocks remain part of one outer response. Splitting an inner batch cannot turn it into standalone reads. +A separate first-block header line of `Script completed`, `Script failed`, or `Script running with cell ID `, followed by `Wall time ... seconds` and `Output:`, is metadata. A failed header adds `command_failed`, and a running header adds `command_still_running`. Any other block that starts with one of those script phrases is unsupported. Other unclaimed substantive output prevents standalone packaging. Multi-file bodies and blocks remain part of one outer response. Splitting an inner batch cannot turn it into standalone reads. -Structured truncation flags, transport notices outside matched source spans, command failures, and still-running commands prevent delivery credit. A truncation phrase inside exact source bytes is source text. No whitespace, line ending, or Unicode normalization occurs. +Structured truncation flags, transport notices outside matched source spans, command failures, and still-running commands prevent delivery credit. Transport notices include line-start ASCII forms and the native `…N tokens truncated…` and `…N chars truncated…` markers anywhere in unclaimed output. A truncation phrase inside exact source bytes is source text. No whitespace, line ending, or Unicode normalization occurs. A requirement is satisfied only by clear, standalone, exact content returned within its recovery window before the first governed call. The entry requires one whole-body claim. Explicit helper chunks can accumulate within one window. Pre-boundary reads, initial injections, replacement history, late returns, and reads crossing a later boundary receive no recovery credit. -Attempts remain in the report after a repair. Later eligible reads must cover the entire required file after each violation and before the action to repair it. Earlier chunks cannot supply missing repair coverage. An invalid required-file attempt after the final eligible read remains an unrepaired violation. The evaluator does not erase a bad batch to produce an unqualified clean result. +Attempts remain in the report after a repair. Later eligible reads must cover the entire required file after each violation and before the action to repair it. Earlier chunks cannot supply missing repair coverage. An invalid required-file attempt after the final eligible read remains unrepaired. It is a violation when exact claims show the packaging failure. It makes the window `INCONCLUSIVE` when its content or transport is unverified. The evaluator does not erase a bad batch to produce an unqualified clean result. ## Test reference From 24436f4f0b242e2dd97faf2d1f209cd8442d1214 Mon Sep 17 00:00:00 2001 From: mdsmithaustin <15272014+mdsmithaustin@users.noreply.github.com> Date: Tue, 20 Oct 2026 19:45:20 -0600 Subject: [PATCH 16/27] no-mistakes(review): Prioritize exact packaging violations over transport uncertainty --- evals/skill-recovery/README.md | 2 +- evals/skill-recovery/evaluate.py | 13 ++++++------ tools/test_skill_recovery.py | 34 ++++++++++++++++++++++++++++++++ 3 files changed, 42 insertions(+), 7 deletions(-) diff --git a/evals/skill-recovery/README.md b/evals/skill-recovery/README.md index 383402d5..a5b15dc8 100644 --- a/evals/skill-recovery/README.md +++ b/evals/skill-recovery/README.md @@ -79,7 +79,7 @@ Structured truncation flags, transport notices outside matched source spans, com A requirement is satisfied only by clear, standalone, exact content returned within its recovery window before the first governed call. The entry requires one whole-body claim. Explicit helper chunks can accumulate within one window. Pre-boundary reads, initial injections, replacement history, late returns, and reads crossing a later boundary receive no recovery credit. -Attempts remain in the report after a repair. Later eligible reads must cover the entire required file after each violation and before the action to repair it. Earlier chunks cannot supply missing repair coverage. An invalid required-file attempt after the final eligible read remains unrepaired. It is a violation when exact claims show the packaging failure. It makes the window `INCONCLUSIVE` when its content or transport is unverified. The evaluator does not erase a bad batch to produce an unqualified clean result. +Attempts remain in the report after a repair. Later eligible reads must cover the entire required file after each violation and before the action to repair it. Earlier chunks cannot supply missing repair coverage. An invalid required-file attempt after the final eligible read remains unrepaired. After a requirement is satisfied, an unrepaired pre-action attempt fails when exact content proves a packaging or whole-body policy violation, even if transport is unverified. Otherwise, unverified content or transport makes that attempt inconclusive. The evaluator does not erase a bad batch to produce an unqualified clean result. ## Test reference diff --git a/evals/skill-recovery/evaluate.py b/evals/skill-recovery/evaluate.py index b16c04a7..de05fd10 100644 --- a/evals/skill-recovery/evaluate.py +++ b/evals/skill-recovery/evaluate.py @@ -541,19 +541,20 @@ def _evaluate(files, windows, calls, dispositions, claims, injections, scope): if satisfied: for index, attempt in attempts: unverified = attempt["transport"] != "CLEAR" or attempt["content"] == "UNVERIFIED" - invalid = (unverified or attempt["packaging"] != "STANDALONE" or - (policy == "whole_standalone" and not attempt["whole_body"])) - if invalid and attempt["ordering"] == "BEFORE_ACTION": + proven_violation = (attempt["content"] != "UNVERIFIED" and + (attempt["packaging"] != "STANDALONE" or + (policy == "whole_standalone" and not attempt["whole_body"]))) + if (unverified or proven_violation) and attempt["ordering"] == "BEFORE_ACTION": if _complete([span for span, witness in progress if window.attempts[witness]["return_row"] > attempt["return_row"]], len(files[file_id].data)): attempt["repaired_before_action"] = True repaired = True - elif unverified: - inconclusive = True - else: + elif proven_violation: unrepaired.append(index) failed = True + else: + inconclusive = True else: failed = True if any(attempt["ordering"] == "BEFORE_ACTION" and diff --git a/tools/test_skill_recovery.py b/tools/test_skill_recovery.py index 62a41b9d..963e23ba 100644 --- a/tools/test_skill_recovery.py +++ b/tools/test_skill_recovery.py @@ -180,6 +180,40 @@ def test_invalid_attempt_after_valid_read_is_not_clean(self): report = self.run_cli(1) self.assertEqual(report["windows"][0]["requirements"][0]["unrepaired_violation_attempts"], [1]) + def test_exact_mixed_batch_with_failed_command_after_valid_read_fails(self): + mixed = returned("mixed", [ENTRY, STEP, b"ls failed\n"]) + mixed["payload"]["output"][3]["text"] = json.dumps({"exit_code": 1, "output": "ls failed\n"}) + self.prepare([boundary(), call("valid"), returned("valid", [ENTRY]), call("mixed"), mixed, + call("action"), returned("action", [b"Done."])], + [read("valid"), read("mixed"), read("mixed", "step", 2)]) + report = self.run_cli(1) + self.assertEqual(report["status"], "FAIL") + window = report["windows"][0] + self.assertTrue(window["requirements"][0]["satisfied_before_action"]) + self.assertEqual(window["requirements"][0]["unrepaired_violation_attempts"], [1]) + attempt = window["attempts"][1] + self.assertEqual((attempt["content"], attempt["packaging"], attempt["transport"]), + ("COMPLETE", "MIXED_FILES", "UNVERIFIED")) + self.assertIn("command_failed", attempt["reasons"]) + self.assertFalse(attempt["repaired_before_action"]) + + def test_exact_packaging_breaches_with_unclear_transport_fail(self): + for bodies, reads, packaging in ( + ([ENTRY, b"extra"], [read("later")], "EXTRA_CONTENT"), + ([ENTRY[:17], ENTRY[17:]], + [read("later", source=[0, 17]), read("later", block=2, source=[17, len(ENTRY)])], "STANDALONE"), + ([ENTRY[:17]], [read("later", source=[0, 17])], "STANDALONE")): + with self.subTest(bodies=bodies): + self.prepare([boundary(), call("valid"), returned("valid", [ENTRY]), + call("later"), returned("later", bodies, truncated=True), + call("action"), returned("action", [b"Done."])], [read("valid"), *reads]) + report = self.run_cli(1) + self.assertEqual(report["status"], "FAIL") + window = report["windows"][0] + self.assertEqual(window["requirements"][0]["unrepaired_violation_attempts"], [1]) + self.assertEqual(window["attempts"][1]["packaging"], packaging) + self.assertEqual(window["attempts"][1]["transport"], "UNVERIFIED") + def test_unverified_attempt_after_valid_read_is_inconclusive(self): for observation in ("wrong_bytes", "truncated", "unsupported_carrier"): with self.subTest(observation=observation): From ad9bc44a0a506214f5d027bf0b01189d6c9b4c7a Mon Sep 17 00:00:00 2001 From: mdsmithaustin <15272014+mdsmithaustin@users.noreply.github.com> Date: Tue, 20 Oct 2026 19:47:45 -0600 Subject: [PATCH 17/27] no-mistakes(review): Exclude unverified packaging from proven violations --- evals/skill-recovery/README.md | 2 +- evals/skill-recovery/evaluate.py | 5 +++-- tools/test_skill_recovery.py | 33 ++++++++++++++++++++++++++++---- 3 files changed, 33 insertions(+), 7 deletions(-) diff --git a/evals/skill-recovery/README.md b/evals/skill-recovery/README.md index a5b15dc8..148a931b 100644 --- a/evals/skill-recovery/README.md +++ b/evals/skill-recovery/README.md @@ -79,7 +79,7 @@ Structured truncation flags, transport notices outside matched source spans, com A requirement is satisfied only by clear, standalone, exact content returned within its recovery window before the first governed call. The entry requires one whole-body claim. Explicit helper chunks can accumulate within one window. Pre-boundary reads, initial injections, replacement history, late returns, and reads crossing a later boundary receive no recovery credit. -Attempts remain in the report after a repair. Later eligible reads must cover the entire required file after each violation and before the action to repair it. Earlier chunks cannot supply missing repair coverage. An invalid required-file attempt after the final eligible read remains unrepaired. After a requirement is satisfied, an unrepaired pre-action attempt fails when exact content proves a packaging or whole-body policy violation, even if transport is unverified. Otherwise, unverified content or transport makes that attempt inconclusive. The evaluator does not erase a bad batch to produce an unqualified clean result. +Attempts remain in the report after a repair. Later eligible reads must cover the entire required file after each violation and before the action to repair it. Earlier chunks cannot supply missing repair coverage. An invalid required-file attempt after the final eligible read remains unrepaired. After a requirement is satisfied, an unrepaired pre-action attempt fails when exact content proves mixed files or extra content, even if transport is unverified. Complete exact content split across claims also proves a whole-body policy violation when packaging is known. Unverified packaging never proves a violation. Otherwise, unverified content or transport makes that attempt inconclusive. The evaluator does not erase a bad batch to produce an unqualified clean result. ## Test reference diff --git a/evals/skill-recovery/evaluate.py b/evals/skill-recovery/evaluate.py index de05fd10..a8b365cf 100644 --- a/evals/skill-recovery/evaluate.py +++ b/evals/skill-recovery/evaluate.py @@ -542,8 +542,9 @@ def _evaluate(files, windows, calls, dispositions, claims, injections, scope): for index, attempt in attempts: unverified = attempt["transport"] != "CLEAR" or attempt["content"] == "UNVERIFIED" proven_violation = (attempt["content"] != "UNVERIFIED" and - (attempt["packaging"] != "STANDALONE" or - (policy == "whole_standalone" and not attempt["whole_body"]))) + (attempt["packaging"] in ("MIXED_FILES", "EXTRA_CONTENT") or + (policy == "whole_standalone" and attempt["content"] == "COMPLETE" and + attempt["packaging"] == "STANDALONE" and not attempt["whole_body"]))) if (unverified or proven_violation) and attempt["ordering"] == "BEFORE_ACTION": if _complete([span for span, witness in progress if window.attempts[witness]["return_row"] > attempt["return_row"]], diff --git a/tools/test_skill_recovery.py b/tools/test_skill_recovery.py index 963e23ba..91fc1abc 100644 --- a/tools/test_skill_recovery.py +++ b/tools/test_skill_recovery.py @@ -201,8 +201,7 @@ def test_exact_packaging_breaches_with_unclear_transport_fail(self): for bodies, reads, packaging in ( ([ENTRY, b"extra"], [read("later")], "EXTRA_CONTENT"), ([ENTRY[:17], ENTRY[17:]], - [read("later", source=[0, 17]), read("later", block=2, source=[17, len(ENTRY)])], "STANDALONE"), - ([ENTRY[:17]], [read("later", source=[0, 17])], "STANDALONE")): + [read("later", source=[0, 17]), read("later", block=2, source=[17, len(ENTRY)])], "STANDALONE")): with self.subTest(bodies=bodies): self.prepare([boundary(), call("valid"), returned("valid", [ENTRY]), call("later"), returned("later", bodies, truncated=True), @@ -214,18 +213,44 @@ def test_exact_packaging_breaches_with_unclear_transport_fail(self): self.assertEqual(window["attempts"][1]["packaging"], packaging) self.assertEqual(window["attempts"][1]["transport"], "UNVERIFIED") + def test_unsupported_sibling_after_valid_read_is_inconclusive(self): + for bodies, reads, content in ( + ([ENTRY], [read("later")], "COMPLETE"), + ([ENTRY[:17], ENTRY[17:]], + [read("later", source=[0, 17]), read("later", block=2, source=[17, len(ENTRY)])], "COMPLETE"), + ([ENTRY[:17]], [read("later", source=[0, 17])], "EXACT_CHUNK")): + with self.subTest(content=content, blocks=len(bodies)): + later = returned("later", []) + later["payload"]["output"].extend({"type": "input_text", "text": body.decode()} for body in bodies) + later["payload"]["output"].append({"type": "input_text", "text": "Chunk ID: unsupported\n"}) + self.prepare([boundary(), call("valid"), returned("valid", [ENTRY]), call("later"), later, + call("action"), returned("action", [b"Done."])], [read("valid"), *reads]) + report = self.run_cli(2) + self.assertEqual(report["status"], "INCONCLUSIVE") + window = report["windows"][0] + self.assertTrue(window["requirements"][0]["satisfied_before_action"]) + self.assertEqual(window["requirements"][0]["unrepaired_violation_attempts"], []) + attempt = window["attempts"][1] + self.assertEqual((attempt["content"], attempt["packaging"], attempt["transport"]), + (content, "UNVERIFIED", "UNVERIFIED")) + self.assertIn("unsupported_output_carrier", attempt["reasons"]) + self.assertFalse(attempt["repaired_before_action"]) + def test_unverified_attempt_after_valid_read_is_inconclusive(self): - for observation in ("wrong_bytes", "truncated", "unsupported_carrier"): + for observation in ("wrong_bytes", "truncated", "partial_truncated", "unsupported_carrier"): with self.subTest(observation=observation): later = returned("later", [ENTRY]) if observation == "wrong_bytes": later = returned("later", [b"x" * len(ENTRY)]) elif observation == "truncated": later["truncated"] = True + elif observation == "partial_truncated": + later = returned("later", [ENTRY[:17]], truncated=True) else: later["payload"]["output"][1]["text"] = json.dumps({"status": "fulfilled", "value": {"exit_code": 0, "output": ENTRY.decode()}}) self.prepare([boundary(), call("valid"), returned("valid", [ENTRY]), call("later"), later, - call("action"), returned("action", [b"Done."])], [read("valid"), read("later")]) + call("action"), returned("action", [b"Done."])], + [read("valid"), read("later", source=[0, 17] if observation == "partial_truncated" else None)]) self.assertEqual(self.run_cli(2)["status"], "INCONCLUSIVE") def test_flagged_attempt_can_be_repaired_before_action(self): From d47b97ac836967442e6ed1e3c7e359d0fd9cbdbc Mon Sep 17 00:00:00 2001 From: mdsmithaustin <15272014+mdsmithaustin@users.noreply.github.com> Date: Tue, 20 Oct 2026 19:50:48 -0600 Subject: [PATCH 18/27] no-mistakes(review): Unify attempt classification and preserve partial-read violations --- evals/skill-recovery/README.md | 2 +- evals/skill-recovery/evaluate.py | 39 +++++++++++++++------------ tools/test_skill_recovery.py | 46 +++++++++++++++++++++++++++----- 3 files changed, 62 insertions(+), 25 deletions(-) diff --git a/evals/skill-recovery/README.md b/evals/skill-recovery/README.md index 148a931b..98562fdf 100644 --- a/evals/skill-recovery/README.md +++ b/evals/skill-recovery/README.md @@ -79,7 +79,7 @@ Structured truncation flags, transport notices outside matched source spans, com A requirement is satisfied only by clear, standalone, exact content returned within its recovery window before the first governed call. The entry requires one whole-body claim. Explicit helper chunks can accumulate within one window. Pre-boundary reads, initial injections, replacement history, late returns, and reads crossing a later boundary receive no recovery credit. -Attempts remain in the report after a repair. Later eligible reads must cover the entire required file after each violation and before the action to repair it. Earlier chunks cannot supply missing repair coverage. An invalid required-file attempt after the final eligible read remains unrepaired. After a requirement is satisfied, an unrepaired pre-action attempt fails when exact content proves mixed files or extra content, even if transport is unverified. Complete exact content split across claims also proves a whole-body policy violation when packaging is known. Unverified packaging never proves a violation. Otherwise, unverified content or transport makes that attempt inconclusive. The evaluator does not erase a bad batch to produce an unqualified clean result. +Attempts remain in the report after a repair. Later eligible reads must cover the entire required file after each violation and before the action to repair it. Earlier chunks cannot supply missing repair coverage. After a requirement is satisfied, each unrepaired pre-action attempt follows this precedence: exact-byte proof of mixed files or extra content fails, as does a clear exact partial or non-whole read under the whole-body policy. Otherwise, unclear transport, unverified content, or unverified packaging makes the attempt inconclusive. All other attempts are valid. Later complete eligible coverage repairs both proven violations and inconclusive attempts. The evaluator does not erase a bad batch to produce an unqualified clean result. ## Test reference diff --git a/evals/skill-recovery/evaluate.py b/evals/skill-recovery/evaluate.py index a8b365cf..7eccf804 100644 --- a/evals/skill-recovery/evaluate.py +++ b/evals/skill-recovery/evaluate.py @@ -466,7 +466,7 @@ def _measure(call, claims, files): packaging = "MIXED_FILES" elif unsupported: packaging = "UNVERIFIED" - elif extra: + elif extra and file_ids: packaging = "EXTRA_CONTENT" elif len(file_ids) == 1: packaging = "STANDALONE" @@ -540,22 +540,27 @@ def _evaluate(files, windows, calls, dispositions, claims, injections, scope): unrepaired = [] if satisfied: for index, attempt in attempts: - unverified = attempt["transport"] != "CLEAR" or attempt["content"] == "UNVERIFIED" - proven_violation = (attempt["content"] != "UNVERIFIED" and - (attempt["packaging"] in ("MIXED_FILES", "EXTRA_CONTENT") or - (policy == "whole_standalone" and attempt["content"] == "COMPLETE" and - attempt["packaging"] == "STANDALONE" and not attempt["whole_body"]))) - if (unverified or proven_violation) and attempt["ordering"] == "BEFORE_ACTION": - if _complete([span for span, witness in progress - if window.attempts[witness]["return_row"] > attempt["return_row"]], - len(files[file_id].data)): - attempt["repaired_before_action"] = True - repaired = True - elif proven_violation: - unrepaired.append(index) - failed = True - else: - inconclusive = True + if attempt["ordering"] != "BEFORE_ACTION": + continue + if (attempt["packaging"] in ("MIXED_FILES", "EXTRA_CONTENT") or + (policy == "whole_standalone" and not attempt["whole_body"] and + attempt["transport"] == "CLEAR" and attempt["content"] != "UNVERIFIED")): + outcome = "FAIL" + elif (attempt["transport"] != "CLEAR" or attempt["content"] == "UNVERIFIED" or + attempt["packaging"] == "UNVERIFIED"): + outcome = "INCONCLUSIVE" + else: + continue + if _complete([span for span, witness in progress + if window.attempts[witness]["return_row"] > attempt["return_row"]], + len(files[file_id].data)): + attempt["repaired_before_action"] = True + repaired = True + elif outcome == "FAIL": + unrepaired.append(index) + failed = True + else: + inconclusive = True else: failed = True if any(attempt["ordering"] == "BEFORE_ACTION" and diff --git a/tools/test_skill_recovery.py b/tools/test_skill_recovery.py index 91fc1abc..6a9a2a8f 100644 --- a/tools/test_skill_recovery.py +++ b/tools/test_skill_recovery.py @@ -180,6 +180,37 @@ def test_invalid_attempt_after_valid_read_is_not_clean(self): report = self.run_cli(1) self.assertEqual(report["windows"][0]["requirements"][0]["unrepaired_violation_attempts"], [1]) + def test_satisfied_requirement_attempt_precedence(self): + mixed = returned("attempt", [ENTRY, STEP, b"ls failed\n"]) + mixed["payload"]["output"][3]["text"] = json.dumps({"exit_code": 1, "output": "ls failed\n"}) + unsupported = returned("attempt", []) + unsupported["payload"]["output"].extend([ + {"type": "input_text", "text": ENTRY.decode()}, + {"type": "input_text", "text": "Chunk ID: unsupported\n"}]) + for name, output, reads, before_valid, code, status in ( + ("mixed_failed_ls", mixed, [read("attempt"), read("attempt", "step", 2)], False, 1, "FAIL"), + ("unsupported_sibling", unsupported, [read("attempt")], False, 2, "INCONCLUSIVE"), + ("clear_partial_after", returned("attempt", [ENTRY[:17]]), + [read("attempt", source=[0, 17])], False, 1, "FAIL"), + ("clear_partial_before", returned("attempt", [ENTRY[:17]]), + [read("attempt", source=[0, 17])], True, 0, "SATISFIED_WITH_REPAIRS"), + ("truncated_partial", returned("attempt", [ENTRY[:17]], truncated=True), + [read("attempt", source=[0, 17])], False, 2, "INCONCLUSIVE")): + with self.subTest(case=name): + valid_rows = [call("valid"), returned("valid", [ENTRY])] + attempt_rows = [call("attempt"), output] + ordered = attempt_rows + valid_rows if before_valid else valid_rows + attempt_rows + self.prepare([boundary(), *ordered, call("action"), returned("action", [b"Done."])], + [read("valid"), *reads]) + report = self.run_cli(code) + self.assertEqual(report["status"], status) + window = report["windows"][0] + requirement = window["requirements"][0] + self.assertTrue(requirement["satisfied_before_action"]) + self.assertEqual(requirement["unrepaired_violation_attempts"], [1] if code == 1 else []) + attempt = next(a for a in window["attempts"] if a["call_id"] == "attempt" and a["file"] == "entry") + self.assertEqual(attempt["repaired_before_action"], before_valid) + def test_exact_mixed_batch_with_failed_command_after_valid_read_fails(self): mixed = returned("mixed", [ENTRY, STEP, b"ls failed\n"]) mixed["payload"]["output"][3]["text"] = json.dumps({"exit_code": 1, "output": "ls failed\n"}) @@ -197,19 +228,20 @@ def test_exact_mixed_batch_with_failed_command_after_valid_read_fails(self): self.assertIn("command_failed", attempt["reasons"]) self.assertFalse(attempt["repaired_before_action"]) - def test_exact_packaging_breaches_with_unclear_transport_fail(self): - for bodies, reads, packaging in ( - ([ENTRY, b"extra"], [read("later")], "EXTRA_CONTENT"), + def test_packaging_and_split_entry_with_unclear_transport(self): + for bodies, reads, packaging, code, status in ( + ([ENTRY, b"extra"], [read("later")], "EXTRA_CONTENT", 1, "FAIL"), ([ENTRY[:17], ENTRY[17:]], - [read("later", source=[0, 17]), read("later", block=2, source=[17, len(ENTRY)])], "STANDALONE")): + [read("later", source=[0, 17]), read("later", block=2, source=[17, len(ENTRY)])], + "STANDALONE", 2, "INCONCLUSIVE")): with self.subTest(bodies=bodies): self.prepare([boundary(), call("valid"), returned("valid", [ENTRY]), call("later"), returned("later", bodies, truncated=True), call("action"), returned("action", [b"Done."])], [read("valid"), *reads]) - report = self.run_cli(1) - self.assertEqual(report["status"], "FAIL") + report = self.run_cli(code) + self.assertEqual(report["status"], status) window = report["windows"][0] - self.assertEqual(window["requirements"][0]["unrepaired_violation_attempts"], [1]) + self.assertEqual(window["requirements"][0]["unrepaired_violation_attempts"], [1] if code == 1 else []) self.assertEqual(window["attempts"][1]["packaging"], packaging) self.assertEqual(window["attempts"][1]["transport"], "UNVERIFIED") From 07362e22d6c987b4e6a30c957ace58de7f1a281b Mon Sep 17 00:00:00 2001 From: mdsmithaustin <15272014+mdsmithaustin@users.noreply.github.com> Date: Tue, 20 Oct 2026 20:26:06 -0600 Subject: [PATCH 19/27] test(evals): show transport markers after a valid read failing as extra content --- tools/test_skill_recovery.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/tools/test_skill_recovery.py b/tools/test_skill_recovery.py index 6a9a2a8f..c0dff06c 100644 --- a/tools/test_skill_recovery.py +++ b/tools/test_skill_recovery.py @@ -323,6 +323,32 @@ def test_native_unicode_truncation_marker_is_transport(self): read("read", source=[20, len(ENTRY)], output=[20 + len(notice), len(ENTRY) + len(notice)])]) self.assertIn("transport_notice", self.run_cli(2)["windows"][0]["attempts"][0]["reasons"]) + def test_transport_markers_after_valid_read_are_not_extra_content(self): + head, tail = ENTRY[:20], ENTRY[20:] + native = b"Warning: truncated output (original token count: 90)\nTotal output lines: 3\n\n" + for name, prefix, marker, suffix, code in ( + ("unicode_marker", b"", "…5 tokens truncated…".encode(), b"", 2), + ("ascii_marker_line", b"", b"\n... 5 tokens truncated\n", b"", 2), + ("warning_suffix", b"", b"", b"Warning: truncated output\n", 2), + ("native_prefix", native, "…5 tokens truncated…".encode(), b"", 2), + ("warning_and_content", b"", b"", b"Warning: truncated output\nunrelated output\n", 1)): + with self.subTest(case=name): + start = len(prefix) + if marker: + reads = [read("again", source=[0, 20], output=[start, start + 20]), + read("again", source=[20, len(ENTRY)], output=[start + 20 + len(marker), start + len(marker) + len(ENTRY)])] + else: + reads = [read("again", output=[start, start + len(ENTRY)])] + self.prepare([boundary(), call("valid"), returned("valid", [ENTRY]), + call("again"), returned("again", [prefix + head + marker + tail + suffix]), + call("action"), returned("action", [b"Done."])], [read("valid"), *reads]) + window = self.run_cli(code)["windows"][0] + attempt = window["attempts"][1] + self.assertTrue(window["requirements"][0]["satisfied_before_action"]) + self.assertEqual(window["requirements"][0]["unrepaired_violation_attempts"], [1] if code == 1 else []) + self.assertEqual((attempt["packaging"], attempt["transport"]), ("EXTRA_CONTENT" if code == 1 else "STANDALONE", "UNVERIFIED")) + self.assertIn("transport_notice", attempt["reasons"]) + def test_failed_or_running_script_header_is_inconclusive(self): for header in ("Script failed\nWall time 0.1 seconds\nOutput:\n", "Script running with cell ID 7\nWall time 0.1 seconds\nOutput:\n", "Script failed\nError: interrupted\n"): From 42e307d4a020608293c7d5c46f199b3ce23b34c0 Mon Sep 17 00:00:00 2001 From: mdsmithaustin <15272014+mdsmithaustin@users.noreply.github.com> Date: Tue, 20 Oct 2026 20:26:23 -0600 Subject: [PATCH 20/27] test(evals): show escaped bodies in supported carriers hiding a mixed batch --- tools/test_skill_recovery.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/tools/test_skill_recovery.py b/tools/test_skill_recovery.py index c0dff06c..a05bed31 100644 --- a/tools/test_skill_recovery.py +++ b/tools/test_skill_recovery.py @@ -465,6 +465,29 @@ def envelope(body): else: self.assertEqual(report["status"], "SATISFIED") + def test_escaped_bodies_in_supported_carriers_cannot_hide_mixed_batch(self): + header = {"type": "input_text", "text": "Script completed\nWall time 0.1 seconds\nOutput:\n"} + batch = json.dumps([{"status": "fulfilled", "value": {"exit_code": 0, "output": body.decode()}} for body in (ENTRY, STEP)]) + both = (ENTRY + STEP).decode() + shapes = { + "array_text": [header, {"type": "input_text", "text": batch}], + "prefixed_json_text": [header, {"type": "input_text", "text": "results: " + batch}], + "envelope_output_is_json": [header, {"type": "input_text", "text": json.dumps({"exit_code": 0, "output": json.dumps([ENTRY.decode(), STEP.decode()])})}], + "string_text": [header, {"type": "input_text", "text": json.dumps(both)}], + "envelope_metadata": [header, {"type": "input_text", "text": json.dumps({"exit_code": 0, "chunk_id": both, "output": "Done.\n"})}], + "function_call_string": batch, + } + for name, output in shapes.items(): + for batch_first in (False, True): + with self.subTest(shape=name, batch_first=batch_first): + kind = "function_call" if isinstance(output, str) else "custom_tool_call" + mixed = [{"type": "response_item", "payload": {"type": kind, "name": "exec", "call_id": "mixed", "input": "reviewed input"}}, + {"type": "response_item", "payload": {"type": kind + "_output", "call_id": "mixed", "output": output}}] + valid = [call("valid"), returned("valid", [ENTRY])] + self.prepare([boundary(), *(mixed + valid if batch_first else valid + mixed), call("action"), returned("action", [b"Done."])], + [read("valid")]) + self.assertEqual(self.run_cli(2)["errors"][0]["code"], "undeclared_governing_body") + def test_every_repeated_or_overlapping_occurrence_needs_own_claim(self): for entry, body, source, output in ( (ENTRY, ENTRY + ENTRY, [0, len(ENTRY)], [0, len(ENTRY)]), From 028c7d38b6d4eb13afa79e81878f47de308bcc14 Mon Sep 17 00:00:00 2001 From: mdsmithaustin <15272014+mdsmithaustin@users.noreply.github.com> Date: Tue, 20 Oct 2026 20:26:44 -0600 Subject: [PATCH 21/27] test(evals): show a mismatched claim beside an exact claim failing the model --- tools/test_skill_recovery.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tools/test_skill_recovery.py b/tools/test_skill_recovery.py index a05bed31..070ef66b 100644 --- a/tools/test_skill_recovery.py +++ b/tools/test_skill_recovery.py @@ -527,6 +527,17 @@ def test_declared_split_blocks_preserve_whole_entry_policy(self): self.assertEqual(report["status"], "SATISFIED_WITH_REPAIRS") self.assertFalse(report["windows"][0]["attempts"][0]["whole_body"]) + def test_mismatched_claim_beside_exact_claim_is_manifest_error(self): + part = ENTRY[:30] + for name, bodies, reads in ( + ("separate_blocks", [ENTRY, part], [read("again"), read("again", block=2, source=[5, 35], output=[0, 30])]), + ("same_block", [ENTRY + part], [read("again"), read("again", source=[5, 35], output=[len(ENTRY), len(ENTRY) + 30])]), + ("other_file", [ENTRY, part], [read("again"), read("again", "step", 2, source=[0, 30])])): + with self.subTest(case=name): + self.prepare([boundary(), call("valid"), returned("valid", [ENTRY]), call("again"), returned("again", bodies), + call("action"), returned("action", [b"Done."])], [read("valid"), *reads]) + self.assertEqual(self.run_cli(2)["errors"], [{"code": "inconsistent_read_claims", "row": 5}]) + def test_raw_hash_binding(self): self.basic() self.raw.write_bytes(self.raw.read_bytes() + b"\n") From 7dd611e4364f19dc0fe161019c8aeb6581323c7c Mon Sep 17 00:00:00 2001 From: mdsmithaustin <15272014+mdsmithaustin@users.noreply.github.com> Date: Tue, 20 Oct 2026 20:26:45 -0600 Subject: [PATCH 22/27] test(evals): show a whole read claimed as adjacent halves failing the model --- tools/test_skill_recovery.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tools/test_skill_recovery.py b/tools/test_skill_recovery.py index 070ef66b..90ec4fc5 100644 --- a/tools/test_skill_recovery.py +++ b/tools/test_skill_recovery.py @@ -538,6 +538,15 @@ def test_mismatched_claim_beside_exact_claim_is_manifest_error(self): call("action"), returned("action", [b"Done."])], [read("valid"), *reads]) self.assertEqual(self.run_cli(2)["errors"], [{"code": "inconsistent_read_claims", "row": 5}]) + def test_adjacent_claims_of_one_read_are_manifest_error(self): + halves = [read("again", source=[0, 17]), read("again", source=[17, len(ENTRY)], output=[17, len(ENTRY)])] + for valid_first in (False, True): + with self.subTest(valid_first=valid_first): + valid = [call("valid"), returned("valid", [ENTRY])] if valid_first else [] + self.prepare([boundary(), *valid, call("again"), returned("again", [ENTRY]), call("action"), returned("action", [b"Done."])], + halves + ([read("valid")] if valid_first else [])) + self.assertEqual(self.run_cli(2)["errors"][0]["code"], "split_read_claim") + def test_raw_hash_binding(self): self.basic() self.raw.write_bytes(self.raw.read_bytes() + b"\n") From c78d87e8f78a295e4340216f3c1e800092f5a99f Mon Sep 17 00:00:00 2001 From: mdsmithaustin <15272014+mdsmithaustin@users.noreply.github.com> Date: Tue, 20 Oct 2026 20:27:21 -0600 Subject: [PATCH 23/27] fix(evals): scan every output block for plain and escaped governing bodies --- evals/skill-recovery/evaluate.py | 36 +++++++++++++++++++++++++------- 1 file changed, 28 insertions(+), 8 deletions(-) diff --git a/evals/skill-recovery/evaluate.py b/evals/skill-recovery/evaluate.py index 7eccf804..59dad5c6 100644 --- a/evals/skill-recovery/evaluate.py +++ b/evals/skill-recovery/evaluate.py @@ -375,19 +375,18 @@ def row_ref(value, number_key="row", hash_key="row_sha256"): offsets[body.block] = len(data) data += body.data for file in files.values(): - start = data.find(file.data) - while start != -1: + for form, start in _hits(data, encodings[file.id]): spans = [claim.source for claim in selected if claim.file == file.id and claim.block in offsets and offsets[claim.block] + claim.output[0] == start + claim.source[0] and - offsets[claim.block] + claim.output[1] == start + claim.source[1]] + offsets[claim.block] + claim.output[1] == start + claim.source[1]] if form == file.data else [] _require(_complete(spans, len(file.data)), "undeclared_governing_body", calls[call_id].return_row) - start = data.find(file.data, start + 1) for body in calls[call_id].bodies: - if body.kind == "unsupported": - for file in files.values(): - _require(not any(form in body.raw for form in encodings[file.id]) or - any(claim.file == file.id and claim.block == body.block for claim in selected), + left, right = _region(body) + for file in files.values(): + claimed = body.data is None and any(claim.file == file.id and claim.block == body.block for claim in selected) + for form, start in _hits(body.raw, encodings[file.id]): + _require(claimed or left <= start and start + len(form) <= right, "undeclared_governing_body", calls[call_id].return_row) injections = [] for item in _array(manifest.get("initial_injections", [])): @@ -415,6 +414,27 @@ def _encodings(data): return forms +def _hits(data, forms): + for form in forms: + start = data.find(form) + while start != -1: + yield form, start + start = data.find(form, start + 1) + + +def _region(body): + if body.data is None: + return 0, 0 + if body.raw == body.data: + return 0, len(body.raw) + for ascii_only in (False, True): + encoded = json.dumps(body.data.decode(), ensure_ascii=ascii_only)[1:-1].encode() + start = body.raw.find(encoded) + if start != -1: + return start, start + len(encoded) + return 0, 0 + + def _complete(spans, size): end = 0 for start, stop in sorted(spans): From 9321172421607ebf324be6d2ddbac57b7e8f2572 Mon Sep 17 00:00:00 2001 From: mdsmithaustin <15272014+mdsmithaustin@users.noreply.github.com> Date: Tue, 20 Oct 2026 20:27:47 -0600 Subject: [PATCH 24/27] fix(evals): keep transport markers out of extra content --- evals/skill-recovery/evaluate.py | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/evals/skill-recovery/evaluate.py b/evals/skill-recovery/evaluate.py index 59dad5c6..842cb426 100644 --- a/evals/skill-recovery/evaluate.py +++ b/evals/skill-recovery/evaluate.py @@ -143,7 +143,7 @@ class _Window: _HEADER = re.compile(r"Script (completed|failed|running with cell ID [^\n]+)\nWall time [0-9.]+ seconds\nOutput:\n") _HEADER_FLAGS = {"completed": (), "failed": ("command_failed",), "running": ("command_still_running",)} _NATIVE_HEADER = re.compile(r"Script (?:completed|failed|running)\b|Chunk ID: |Wall time: |Exit code: |Process exited with code |Original token count: ") -_NOTICE = re.compile(rb"(?im)^(?:Warning: truncated output\b|\[?output (?:truncated|omitted)\b|\.\.\.\s*\d+ (?:characters|tokens) truncated\b)|\xe2\x80\xa6\d+ (?:tokens|chars) truncated\xe2\x80\xa6") +_NOTICE = re.compile(rb"(?im)^(?:Warning: truncated output\b|\[?output (?:truncated|omitted)\b|\.\.\.\s*\d+ (?:characters|tokens) truncated\b|Total output lines: \d+$)[^\n]*|\xe2\x80\xa6\d+ (?:tokens|chars) truncated\xe2\x80\xa6") _ENVELOPE_KEYS = {"output", "exit_code", "chunk_id", "wall_time_seconds", "original_token_count", "session_id", "truncated"} @@ -444,14 +444,12 @@ def _complete(spans, size): return end == size -def _residual(data, spans): +def _gaps(data, spans): cursor = 0 - result = bytearray() for left, right in sorted(spans): - result.extend(data[cursor:left]) + yield data[cursor:left] cursor = right - result.extend(data[cursor:]) - return bytes(result) + yield data[cursor:] def _measure(call, claims, files): @@ -476,11 +474,12 @@ def _measure(call, claims, files): if body.data[left:right] == files[claim.file].data[start:end]: exact.append(claim) spans.append((left, right)) - residual = _residual(body.data, spans) - if residual: - extra = True - if _NOTICE.search(residual): + for gap in _gaps(body.data, spans): + content = _NOTICE.sub(b"", gap) + if content != gap: flags.add("transport_notice") + content = content.strip() + extra = extra or bool(content) file_ids = {claim.file for claim in exact} if len(file_ids) > 1: packaging = "MIXED_FILES" From df8acfbba4e0b97d617913a37418c19327aa10f1 Mon Sep 17 00:00:00 2001 From: mdsmithaustin <15272014+mdsmithaustin@users.noreply.github.com> Date: Tue, 20 Oct 2026 20:28:15 -0600 Subject: [PATCH 25/27] fix(evals): report a mismatched claim beside an exact claim as a manifest error --- evals/skill-recovery/evaluate.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/evals/skill-recovery/evaluate.py b/evals/skill-recovery/evaluate.py index 842cb426..cffbac84 100644 --- a/evals/skill-recovery/evaluate.py +++ b/evals/skill-recovery/evaluate.py @@ -388,6 +388,10 @@ def row_ref(value, number_key="row", hash_key="row_sha256"): for form, start in _hits(body.raw, encodings[file.id]): _require(claimed or left <= start and start + len(form) <= right, "undeclared_governing_body", calls[call_id].return_row) + bodies = calls[call_id].bodies + matches = {bodies[claim.block].data[claim.output[0]:claim.output[1]] == files[claim.file].data[claim.source[0]:claim.source[1]] + for claim in selected if bodies[claim.block].data is not None} + _require(len(matches) < 2, "inconsistent_read_claims", calls[call_id].return_row) injections = [] for item in _array(manifest.get("initial_injections", [])): _object(item, {"row", "row_sha256", "file", "block", "output_range"}) From e6c53d0912b1aac2ebe20fee091e7126ea0432f9 Mon Sep 17 00:00:00 2001 From: mdsmithaustin <15272014+mdsmithaustin@users.noreply.github.com> Date: Tue, 20 Oct 2026 20:28:33 -0600 Subject: [PATCH 26/27] fix(evals): report a contiguous read split into adjacent claims as a manifest error --- evals/skill-recovery/evaluate.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/evals/skill-recovery/evaluate.py b/evals/skill-recovery/evaluate.py index cffbac84..21df1612 100644 --- a/evals/skill-recovery/evaluate.py +++ b/evals/skill-recovery/evaluate.py @@ -392,6 +392,10 @@ def row_ref(value, number_key="row", hash_key="row_sha256"): matches = {bodies[claim.block].data[claim.output[0]:claim.output[1]] == files[claim.file].data[claim.source[0]:claim.source[1]] for claim in selected if bodies[claim.block].data is not None} _require(len(matches) < 2, "inconsistent_read_claims", calls[call_id].return_row) + ordered = sorted(selected, key=lambda claim: (claim.block, claim.output)) + _require(not any(left.block == right.block and left.file == right.file and left.output[1] == right.output[0] and + left.source[1] == right.source[0] for left, right in zip(ordered, ordered[1:])), + "split_read_claim", calls[call_id].return_row) injections = [] for item in _array(manifest.get("initial_injections", [])): _object(item, {"row", "row_sha256", "file", "block", "output_range"}) From 23c1d9f3ed9b0feeb76b3734fd4785dfb3306a12 Mon Sep 17 00:00:00 2001 From: mdsmithaustin <15272014+mdsmithaustin@users.noreply.github.com> Date: Tue, 20 Oct 2026 20:30:00 -0600 Subject: [PATCH 27/27] docs(evals): state carrier-independent scan, transport notices, and claim consistency rules --- evals/skill-recovery/README.md | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/evals/skill-recovery/README.md b/evals/skill-recovery/README.md index 98562fdf..2a010050 100644 --- a/evals/skill-recovery/README.md +++ b/evals/skill-recovery/README.md @@ -30,13 +30,13 @@ python3 evals/skill-recovery/evaluate.py --raw evals/skill-recovery/fixtures/pic The expected result is exit 1 and status `FAIL`. Any other exit or status fails this negative control. Its first call both reads guidance and discovers task files. That action freezes recovery before the call returns. The later standalone entry read reports `LATE` and cannot repair the window. -The CLI writes one JSON report to stdout. Input files remain unchanged. Reports contain hashes and byte counts rather than raw task content. Standard argument errors go to stderr. +The CLI writes one JSON report to stdout. Input files remain unchanged. Reports contain hashes and byte counts rather than raw task content. A usage error, such as a missing `--raw`, writes the argparse message to stderr, prints no JSON report, and exits 2. | Exit | Overall result | | --- | --- | | 0 | Every applicable window is `SATISFIED` or `SATISFIED_WITH_REPAIRS`. | | 1 | A supported applicable window fails a required prerequisite. | -| 2 | Invalid, unsupported, incomplete, or unresolved evidence makes the result `INCONCLUSIVE`. | +| 2 | Invalid, unsupported, incomplete, or unresolved evidence makes the result `INCONCLUSIVE`. A usage error also exits 2, without a report. | | 3 | Every window is `NO_LATER_GOVERNED_ACTION`. No recovery checkpoint was checked. | `total_windows`, `applicable_windows`, and `non_applicable_windows` expose the denominator. A window without a governed action never counts as recovered. An empty boundary list is inconclusive. Inconclusive results take precedence over failures in the overall result. @@ -63,23 +63,25 @@ Each requirement has `file` and `policy`. The mode entry always uses `whole_stan Call dispositions are `governed`, `incidental`, and `unresolved`. Read claims do not replace dispositions. A call can both read guidance and perform governed work. Its action occurs at call start, before its returned guidance can count. An unresolved disposition before the first governed call, or in a window without one, prevents a conclusive window result. Later unresolved calls and late or boundary-crossing unverified reads remain visible without changing the verdict frozen at the first governed call. Invalid raw input still invalidates the entire assessment. -Exact source spans must match exact returned body spans. Overlapping output claims are invalid. A complete known governing body left unclaimed in any outer-call output is invalid. This detects an omitted earlier full read attempt. The scan covers decoded bodies of supported carriers, including a body split across blocks of one response. It also covers the raw text of unsupported blocks, where a body can appear unescaped or JSON-escaped up to two levels deep. A claim on an unsupported block records an unverified attempt. The scan does not cover message records or `replacement_history`, and those deliveries receive no recovery credit. The reviewer still owns the completeness of file inventory, partial-read claims, supplied boundaries, and action semantics. +A claim is exact when its output bytes equal its source bytes. A claim whose bytes differ records unverified content. An exact claim and a mismatched claim in one outer response are invalid input (`inconsistent_read_claims`), because the evaluator cannot tell a manifest error from extra content. Overlapping output claims are invalid. Two claims of one file in one block whose output and source ranges both continue each other are invalid (`split_read_claim`). A contiguous read takes one claim. + +A complete known governing body left unclaimed in any outer-call output is invalid (`undeclared_governing_body`). This detects an omitted earlier full read attempt. The scan does not depend on carrier classification. It searches the raw text and the decoded body of every block for each governing file, unescaped or JSON-escaped up to two levels deep. An unescaped body in the decoded bodies of one response needs exact claims that cover it, including a body split across blocks. A JSON-escaped body cannot be claimed exactly, so it is always undeclared. In raw text, a body is undeclared unless it lies inside the encoding of the block's decoded body. A body in result metadata is one example. An unsupported block has no decoded body, and a claim of the file on that block records an unverified attempt instead. The scan does not cover message records or `replacement_history`, and those deliveries receive no recovery credit. The reviewer still owns the completeness of file inventory, partial-read claims, supplied boundaries, and action semantics. ## Native input reference The parser pairs native `function_call` and `custom_tool_call` records with their matching output records. Both kinds use the body carriers below. A block that starts with a native exec text header, such as `Chunk ID: `, `Wall time: `, `Exit code: `, `Process exited with code `, or `Original token count: `, is unsupported. A read inside such a block is inconclusive because no committed fixture fixes that header layout. Each `call_id` has exactly one call and one later return of the corresponding kind. Missing, duplicate, or mismatched pairs invalidate the entire input, including duplicates after an apparent success. -Supported top-level record types are `session_meta`, `turn_context`, `event_msg`, `world_state`, `compacted`, `response_item`, `token_usage_record`, and `inter_agent_communication_metadata`. Message and reasoning records provide context. Names in command arguments or file lists do not prove a read. +Supported top-level record types are `session_meta`, `turn_context`, `event_msg`, `world_state`, `compacted`, `response_item`, `token_usage_record`, and `inter_agent_communication_metadata`. The `message`, `reasoning`, and `agent_message` response items provide context, and call pairing skips them. Names in command arguments or file lists do not prove a read. A returned output can be a string or a list of text blocks. Supported body carriers are plain text, a serialized command result with `output` and `exit_code`, and a fulfilled settled-result wrapper with `i`, `result.status`, and `result.value`. The command-result metadata keys are `chunk_id`, `wall_time_seconds`, `original_token_count`, `session_id`, and `truncated`. Unknown carriers remain unsupported. A JSON object block that matches no supported carrier, such as a `status` and `value` result, is unsupported. The parser never recursively flattens arbitrary JSON. JSON nested too deeply for the parser invalidates a raw row and makes a block unsupported. -A separate first-block header line of `Script completed`, `Script failed`, or `Script running with cell ID `, followed by `Wall time ... seconds` and `Output:`, is metadata. A failed header adds `command_failed`, and a running header adds `command_still_running`. Any other block that starts with one of those script phrases is unsupported. Other unclaimed substantive output prevents standalone packaging. Multi-file bodies and blocks remain part of one outer response. Splitting an inner batch cannot turn it into standalone reads. +A separate first-block header line of `Script completed`, `Script failed`, or `Script running with cell ID `, followed by `Wall time ... seconds` and `Output:`, is metadata. A failed header adds `command_failed`, and a running header adds `command_still_running`. Any other block that starts with one of those script phrases is unsupported. Unclaimed output prevents standalone packaging unless it is a transport notice. The evaluator judges each unclaimed stretch between claims on its own. In a stretch that holds a notice, the notice and the stretch's whitespace do not count, and any other byte is extra content. An attempt whose only unclaimed bytes are notices gets the packaging it would have without them, and its transport is unverified. Multi-file bodies and blocks remain part of one outer response. Splitting an inner batch cannot turn it into standalone reads. -Structured truncation flags, transport notices outside matched source spans, command failures, and still-running commands prevent delivery credit. Transport notices include line-start ASCII forms and the native `…N tokens truncated…` and `…N chars truncated…` markers anywhere in unclaimed output. A truncation phrase inside exact source bytes is source text. No whitespace, line ending, or Unicode normalization occurs. +Structured truncation flags, transport notices outside matched source spans, command failures, and still-running commands prevent delivery credit. Transport notices are whole lines that start with `Warning: truncated output`, `output truncated`, `output omitted`, `... N tokens truncated`, or `... N characters truncated`, a line `Total output lines: N`, and the native `…N tokens truncated…` and `…N chars truncated…` markers anywhere in unclaimed output. A truncation phrase inside exact source bytes is source text. Byte comparison applies no whitespace, line-ending, or Unicode normalization. -A requirement is satisfied only by clear, standalone, exact content returned within its recovery window before the first governed call. The entry requires one whole-body claim. Explicit helper chunks can accumulate within one window. Pre-boundary reads, initial injections, replacement history, late returns, and reads crossing a later boundary receive no recovery credit. +A requirement is satisfied only by clear, standalone, exact content returned within its recovery window before the first governed call. Every `whole_standalone` requirement, the entry included, requires one whole-body claim in one block. Explicit helper chunks can accumulate within one window. Pre-boundary reads, initial injections, replacement history, late returns, and reads crossing a later boundary receive no recovery credit. -Attempts remain in the report after a repair. Later eligible reads must cover the entire required file after each violation and before the action to repair it. Earlier chunks cannot supply missing repair coverage. After a requirement is satisfied, each unrepaired pre-action attempt follows this precedence: exact-byte proof of mixed files or extra content fails, as does a clear exact partial or non-whole read under the whole-body policy. Otherwise, unclear transport, unverified content, or unverified packaging makes the attempt inconclusive. All other attempts are valid. Later complete eligible coverage repairs both proven violations and inconclusive attempts. The evaluator does not erase a bad batch to produce an unqualified clean result. +Attempts remain in the report after a repair. Later eligible reads must cover the entire required file after each violation and before the action to repair it. Earlier chunks cannot supply missing repair coverage. After a requirement is satisfied, each unrepaired pre-action attempt follows this precedence: exact-byte proof of mixed files or of extra content other than transport notices fails, as does a clear exact partial or non-whole read under the whole-body policy. Otherwise, unclear transport, unverified content, or unverified packaging makes the attempt inconclusive. All other attempts are valid. Later complete eligible coverage repairs both proven violations and inconclusive attempts. The evaluator does not erase a bad batch to produce an unqualified clean result. ## Test reference