diff --git a/.gitignore b/.gitignore index 0ac48980c4..d54d4f078c 100644 --- a/.gitignore +++ b/.gitignore @@ -11,6 +11,7 @@ __pycache__/ build/ dist/ .pytest_cache/ +.hypothesis/ .ruff_cache/ .mypy_cache/ .coverage diff --git a/docs/development/turn-interruption-recovery.md b/docs/development/turn-interruption-recovery.md new file mode 100644 index 0000000000..bec1db4ce3 --- /dev/null +++ b/docs/development/turn-interruption-recovery.md @@ -0,0 +1,339 @@ +# Bounded Turn interruption recovery + +This implementation addresses an interrupted `turn run-once` on the same +original Turn. The export failure used in tests is synthetic. Its accepted +requirement is to preserve input row order, including equal keys. It is not a +historical incident. No controller/executor split or scheduler is introduced. + +## Source and scope + +Initial baseline: `47a525dcc8ca943c7e4ab39834c92e905892839c`, read 2026-10-27 04:56 UTC. +Implementation rebased on `218aa087b66bb99038b1ce7d782110741a5e42ae` at 05:23 UTC. +Design: [RFC #7416](https://github.com/loopx-project/loopx/pull/7416), +head `9d2d022d14732b3b2dac309013fe9538b1d9ba5d`, +base `99921829f2c5fa2ad35a024c72abb88717003ef3`, +merge `b3ce2cfe77dc99136411279c7bdb2f4db87f37f4`. +Its English and Chinese recovery, continuity, stop and settlement clauses are +requirements. They do not certify an implementation. + +#7420 and #7426 are ancestors of this baseline through their respective merges +`d5d72cad4b4b8b6ecbb89682fd9ab161efcfb7b3` and +`0bd508b9445cb60544f5c9f69a07e0a1b72a0d23`. +#7430 (initial `411707c067e70493c08ce23079af6e2eedb03e96`; refreshed +`89d5f0717bfcb1c88231847a8b737cbc0365b096`, then +`275e59c25686f9e353e3c5966b4d06c289a6ed0b`) is +unmerged; none of its complete-contract claims are inherited here. +The existing #3412 proposal concerns prepared settlement effects. This change +concerns an earlier Host dispatch intent, before a typed result exists, and the +current-lifetime preflight on original-journal recovery. #6838 shares the existing +legacy launch helper but does not add this executor recovery callsite. No new +permission engine or admission schema is added. The executing CLI now passes +the existing legacy lifetime admission to the driver as well as the Host. The +driver reuses `launch_lifetime` for both source instances and legacy registrations +before recovery, without a new quota admission. Preview retains its no-capture +behavior. + +The revoke-after-result counterexample found that the CLI discarded legacy +admission before calling the driver, and `require_current()` deliberately +returns without checking legacy registrations. Consequently, a cached result could +write back and spend after its Goal was retired during the interruption. The +new recovery callsite checks its current registration/activation before journal +continuation. Source-session recovery retains its existing exact lifetime check. +This is a recovery preflight, not continuous fencing of legacy in-flight tools. + +| Requirement | Responsible code | Durable record | Check | Gap / boundary | +| --- | --- | --- | --- | --- | +| Same original work and settlement | `turn_driver/driver.py`, `turn_journal.ts` | Saved plan, Turn key, settlement identity | Identity drift and replay checks | A new task/revision is not an original-Turn retry | +| Complete original context | `decision_capture.py`, `quota/context_delivery.ts` | Immutable capture and session-bound delivery | Hash/readback and full/delta tests | A reference alone is not delivery | +| One dispatch after restart | `executor.py`, `turn_journal.ts` | Attempt count before Host call, result checkpoint | Kill/restart and independent downstream log | Added refusal when the outcome is unknown | +| Current authority | `first_party_host_admission.py`, `first_party_host_runtime.ts` | Registry lifetime/witness and current admission | Recreated/revoked Goal checks | Restoring capture cannot grant authority | +| No duplicate settlement | `settlement.py`, `effect_program.ts` | Prepared effect reference, provider receipt, journal | Provider readback and original-identity replay | Existing mechanism; external tools are separate | +| Physical stop | `host_process.ts`, `host_process_transport.py` | Optional process record and OS group observation | Child termination and stop readback | Not a sandbox or external service fence | + +## State machine and commit boundaries + +State names remain the existing journal vocabulary. Unknown is a recovery +decision (`blocked`, reason `host_outcome_unknown`), not a successful result. + +| Trigger / owner | Durable state after atomic checkpoint | Crash observation / next action | +| --- | --- | --- | +| Executor creates journal | `in_progress`, attempt 0, no phases | No dispatch was authorized; continue the original attempt | +| Executor saves dispatch intent | `in_progress`, attempt > 0, no typed result | Delivery/execution is unknown; refuse automatic redispatch | +| Host returns; executor saves result | `in_progress`, `host_execute, typed_result`, saved result | Validate saved result; do not call Host | +| Validator accepts | Append `validation` | Continue original settlement | +| Settlement prepares a provider effect | Existing `effect_attempts` with original effect reference | Query provider; committed means reuse; absent means retry same reference; unknown means stop | +| Provider checkpoint commits | Append `durable_writeback` / `quota_spend` | Resume only remaining phases | +| Scheduler closeout commits | `committed` or `stopped`, original receipt | With a current Goal lifetime, return retained receipt without another debit | + +The journal owner checks transitions and writes a temporary file then replaces +the journal under mutation locks. Provider commits and the journal checkpoint +are separate transactions. Their gap is resolved by provider readback, not by +pretending the transactions are atomic together. Process-kill tests establish +restart behavior; they do not establish power-loss durability or directory-sync +ordering. + +## Identities, invariants and continuation + +Work is the Goal and selected Todo/obligation plus its accepted source revision. +The original settlement identity is Goal, Agent, binding and Turn instance; +its `effect_id` is independent of the host attempt counter. A native session +identifies conversation continuity, not permission. Context epoch/profile binds +retention. Goal instance and delegated lease epochs fence their existing owned +boundaries; this patch does not invent a universal execution generation. + +Executable assertions must establish: + +1. An unresolved dispatch intent never starts a replacement Host, even with + `--retry-failed-turn`. Unknown/invalid attempt evidence cannot prove absence. +2. A saved typed result resumes validation/settlement without another Host. + Replaying a terminal receipt does not produce another settlement. +3. Recovery preserves the original requirement and immutable capture; current + authority is checked separately. Missing/corrupt context uses existing full + recovery or refuses dependent work. + +The lane and journal locks serialize cooperative writers on one local runtime. +They are not an external fencing token. This slice has no replacement executor +while an original dispatch outcome is unknown. It does not claim that an +arbitrary detached process, remote tool, or another newly admitted Turn cannot +write. A generic Host with unqueryable, non-idempotent tools cannot be safely +replayed just because its process has exited. + +Under available storage and valid current authority, proven non-dispatch +continues; a durable result finishes its remaining settlement; a terminal Turn +returns its receipt. An unknown Host outcome returns a finite refusal requiring +operator/provider reconciliation. There is no polling loop or retry budget +that eventually guesses absence. This patch adds no generic reconciliation +API for arbitrary Host tools. Do not clear the journal or choose another Turn +identity to bypass that refusal. + +## Compatibility and rollback + +No new schema or ID is introduced. New initial journals explicitly persist the +existing attempt counter as zero. Historical incomplete journals without that +counter fail closed because non-dispatch cannot be proven. Terminal journals +and journals with a saved typed result retain their recovery route while the +Goal remains active. Retired Goals have no executing-recovery compatibility +path, including terminal receipt replay. Refusal preserves the original journal, +provider receipts and committed debit; `inspect-journal` remains read-only. +No retirement fallback, alternate identity or registration migration is added. +Explicitly recorded failed-Host retry policy is unchanged and is not qualified +as safe replay of arbitrary external effects by this change. + +Old versions can parse these records but lack the new refusal. Therefore a +downgrade must first reconcile or quarantine unresolved Host intents; do not +silently resume them with the old binary. Captures/receipts are never rewritten +to simulate continuity. At-most-once settlement depends on provider atomic +idempotency/readback and retained history. Exactly-once arbitrary external +effects is not promised. + + +## Fault evidence and qualification limits + +The original measurements in this section and `evidence.json` describe head +`335eae25f2544456c9bb644ea5db311210f747e0`. They remain historical evidence; +their source hashes do not certify later integration repairs. Fresh integration +validation is recorded separately below and in the PR description. + +All experiments use disposable workspaces. `test_turn_interruption_recovery.py` +pauses the real CLI at a journal boundary, sends SIGKILL, and starts another +process with the original `--resume-turn-key`. The Codex executable is an +injected deterministic task worker. It repairs a deliberately broken exporter, +reads the original requirement, and calls a separate downstream process. That +service commits every operation to its own SQLite database without deduplication. +Two normal controls use the installed CLI and the same inputs, requirement and +independent CSV oracle as the fourteen interruption cases. +The independent oracle reads that database and the CSV directly. File and SQLite +refer to LoopX authority backends; the Turn journal itself remains a JSON file. + +| Boundary / race | Observation required after restart | Evidence type | +| --- | --- | --- | +| Intent durable, delivery unknown | No Host redispatch or debit, unchanged journal | Real CLI SIGKILL, both providers | +| Export committed, Host result not saved | One downstream operation, unknown refusal, old Host drained | Real CLI SIGKILL, both providers | +| Typed result durable, before validation | Original artifact/result reused, one settlement | Real CLI SIGKILL, both providers | +| Writeback committed, checkpoint absent | Original provider readback, no repeated Host/export | Real CLI SIGKILL, both providers | +| Quota committed, checkpoint absent | Read original debit, one spend record | Real CLI SIGKILL, both providers | +| Terminal commit, response lost | Replay original receipt, one spend record | Real CLI SIGKILL, both providers | +| Authority retired during interruption | No post-restart writeback/debit, evidence preserved | Real CLI SIGKILL, both providers | +| Terminal commit, then Goal retired | Refuse executing replay; preserve journal, one original debit and downstream operation | Real CLI SIGKILL, both providers | +| Old Goal generation returns a result | Reject result before journal/writeback | Existing source-lifetime driver tests and owner mutation | +| Context baseline lost/corrupt/reordered | Same-capture full, current control facts retained | Existing TS codec checks; native qualification below failed | +| Tool still runs when owner disappears | Observe real process termination separately from effect records | Existing process-supervisor tests and downstream readback | + +The Hypothesis `RuleBasedStateMachine` generates dispatch, intent/result/spend +faults, recovery, repeated responses and Goal generation invalidation. Its +expected counts follow chosen actions, not the implementation's result fields. +It uses real journals and a separate SQLite receipt ledger. It runs 30 deterministic +examples of up to 12 actions with Hypothesis 6.142.4. These process-local faults +complement the SIGKILL matrix; they are not substitutes for it. The initial +shortened failure was a fixture `journal_path_mismatch`, corrected by using the +source registry's canonical runtime root, not by weakening a production check. + +`check_turn_recovery_mutations.py` creates temporary sibling copies, removes one +protection, and requires assertion failures. It checks unknown dispatch, old Goal +generation, current stop authority, settlement deduplication and a dropped original +requirement. Parse/import errors do not count as detected mutations. The requirement +mutation probes the independent oracle; the other four mutate production owners. + +Native Codex 0.164.0 with deterministic loopback inference **did not qualify**. +On baseline `218aa087b66bb99038b1ce7d782110741a5e42ae` and earlier candidate +`9d9b1183489164246c4ae770a06a8d43b28b00ef`, two compaction/process-resume +cases failed with missing `candidate`, and four history-change cases failed the +complete model-visible JSON assertion. No paid model ran. Initial missing-binary, +missing installed CLI and sandbox socket refusals are setup failures, not product +passes. The same six later semantic failures are retained. The subsequent legacy +recovery repair does not inherit native qualification from those earlier runs. +The final source hashes qualify the bounded tests listed below, not a final +native-profile pass. This patch does not +repair or qualify the unified writer proposed in #7430. Native installation, +all-host context continuity and model outcomes remain outside the demonstrated +claim, even though the injected-worker CLI path passes. + +No independent maintainer or third party has yet signed off. Automatic Worker +routing was unavailable in this session; the same lead wrote the patch and the +separately specified oracle. Oracle code independence is not personnel independence. +A maintainer must review journal/provider atomicity, historical records, source +lifetime fencing and unknown effects before adoption. + +## Reproduce from a clean checkout + +Use the implementation PR's exact head commit, Python 3.11, Node 26.12.1 and Java +17.0.17. The qualification dependency file pins every Python package used by this +bounded selection; `package-lock.json` pins the existing TypeScript tools. + +```sh +python3 -m venv .venv +.venv/bin/python -m pip install -r tests/models/turn_recovery/requirements.txt +.venv/bin/python -m pip install --no-deps -e . +.venv/bin/python -m pip check +.venv/bin/python -m pytest -q tests/test_stage2c_dependency_lock.py tests/test_loopx_turn_executor.py tests/control_plane/test_turn_host_decision_capture.py tests/control_plane/test_turn_interruption_recovery.py tests/control_plane/test_turn_recovery_state_machine.py tests/control_plane/test_host_process.py tests/control_plane/test_first_party_host_runtime.py tests/control_plane/test_effect_program_fault_replay_matrix.py --basetemp=./recovery-results +node --experimental-strip-types --test --test-isolation=none tests/control_plane_ts/turn_journal.test.ts tests/control_plane_ts/first_party_host_runtime.test.ts tests/control_plane_ts/effect_program.test.ts tests/control_plane_ts/turn_settlement.test.ts tests/control_plane_ts/context_delivery.test.ts +.venv/bin/python tests/control_plane/check_turn_recovery_mutations.py mutation-results.json +npm ci +npm run typecheck:control-plane +``` + +Run mutations serially after the ordinary checks; they create and remove temporary +sibling modules. Each process-fault case saves `before.json` and `downstream-before.json` before +termination. It also writes raw evidence into its pytest `tmp_path/evidence`: original journal, recovery response, downstream records, +artifact and settlement count. `--basetemp` must name a disposable empty directory; +pytest replaces an existing directory at that path. A failing run is not a pass. +SIGSTOP/SIGKILL qualification is POSIX-specific; Windows is not qualified here. + +To reproduce the native **failures**, set `LOOPX_NATIVE_CHECKPOINT_BIN` to an explicit +Codex 0.164.0 binary and run the two named test functions in +`tests/control_plane/test_codex_context_capture.py`: `test_native_compact_recovery_survives_process_resume` +and `test_native_worker_resume_history_change_reads_current_full`. The fixture +uses loopback inference, an isolated Codex home and an installed source-checkout +CLI. Do not point it at an existing user session or a paid provider. + +The bounded model and its action-to-code mapping are in +[`tests/models/turn_recovery/README.md`](../../tests/models/turn_recovery/README.md). +For the external-effect distinction, see the primary +[AWS idempotency guidance](https://aws.amazon.com/builders-library/making-retries-safe-with-idempotent-APIs/). +For the stronger storage test category that this work does not perform, see +[SQLite crash testing](https://www.sqlite.org/testing.html#crash_testing). + + +## Claims and remaining work + +The checked snapshot is [evidence.json](../../tests/models/turn_recovery/evidence.json). +It contains the source file hashes, all sixteen downstream/settlement observations, +model result, retained failures, and the complete set of 38 open PR heads from +the final overlap check (706 paginated changed-file records). The implementation +baseline is fixed above. The later observed target `bafbafc258de72029b54ea016103c0a7bd2409c9` +adds two commits in runtime projection routing and its tests; the submission +refresh observed `71057d44f0d50a3636fcd8796f7c98717ce41742`. These later target commits are not silently +included in this qualification snapshot. The PR targets `main`; integration CI +and maintainer review remain separate. + +| Claim | Evidence | Applies to | Not established | +| --- | --- | --- | --- | +| Unknown Host outcome cannot automatically relaunch | Intent/tool SIGKILL cases, unchanged journal, independent downstream log, guard mutation | Resume of the original Turn under the patched driver | A fresh unrelated Turn or an old binary cannot be fenced by this patch | +| Retired Goal cannot resume cached-result writes | Revoked File/SQLite cases retain one export and zero spends | Current lifetime preflight at this CLI recovery boundary | Continuous fencing of legacy in-flight tools | +| Saved result and original receipt can finish without another debit | Result/writeback/spend/receipt cases and duplicate replay, exact original effect refs | Existing supported provider readback and retained journal | Exactly-once arbitrary external tools or power loss | +| Original sorting requirement remains available and satisfied | Original capture equality, actual delivered prompt, material-read file, CSV and independent service records | Synthetic export repair, normal versus interrupted execution | Real-model comprehension or a separate requirement-edit experiment | + +The final process matrix passed 16 cases. The broader clean-environment selection +passed 173 cases and initially failed two new normal controls because their test +command used unsupported `python -m loopx`; the installed CLI correction then +passed both controls and the entire final matrix. Do not relabel the earlier +failed command as a pass. TypeScript owners passed 138 checks, including context +full fallback and prepared-effect outcomes. The separate delegated-lease process selection passed 16 cases, including renewal +faults, deadline/identity retention and control-pipe cleanup. Reproduce it with +`.venv/bin/python -m pytest -q tests/control_plane/test_leased_host_process.py`. +The fixed dependencies were installed +into a new environment without system site packages and passed `pip check`. + +An isolated maintainer should now rerun the listed commands at the PR head and +review the original-journal authority binding. Full native continuity still has +six reproduced baseline/candidate failures. A real model, independent personnel +review, third-party execution, Windows process faults, power-loss testing and +continuous external fencing remain unqualified. The module's bounded proposal +is reviewable; the complete all-host RFC acceptance is not claimed. + + +CI subsequently caught two dependency-integration errors at `4472325f287a`: +the existing Stage 2C hash lock did not yet include Hypothesis, and this +qualification profile had pinned Pygments 2.19.2 from the original environment. +[GHSA-5239-wwwm-4pmq](https://github.com/advisories/GHSA-5239-wwwm-4pmq) +affects versions below 2.20.0. The profile now uses the repository's existing +2.21.0 pin. The Stage 2C lock adds only Hypothesis, sortedcontainers and their +published distribution hashes, without unlocking other packages or adding a +security exemption. After correction, a fresh environment passed the full +176-case selection (including lock coverage), the seven kernel invariant checks +and the complete kernel ruff scope. Earlier results remain historical; the JSON +contains the corrected dependency-file hashes and fresh sixteen-case records. +The separate 16-case lease run predates this formatter-only dependency correction +and is not relabeled as a run under the corrected environment. + + +The next dependency-review run also rejected the verifier environment's explicit +Setuptools 83.0.0 pin under +[GHSA-xmj3-9gf7-h52w](https://github.com/advisories/GHSA-xmj3-9gf7-h52w). +The verifier environment now pins the patched 84.1.0. Project installation uses +normal PEP 517 build isolation, which retains this baseline's existing +`pyproject.toml` build-system pin. This does not upgrade the repository's default +build tool or duplicate #7205/#7206; those PRs own that separate change. Windows +legacy wheel installation is not qualified here. The new default-build browser +failure is a Goal drawer geometry assertion in unchanged UI files; its cause is +unverified, and this recovery patch does not modify UI padding or test thresholds. + +The isolated verifier was recreated with Setuptools 84.1.0 and Pygments 2.21.0, +then the complete 176-case selection passed again. The sixteen raw subprocess +records and dependency-file hash in `evidence.json` now refer to that run. +The UI build failure matches the existing `typed-actions.mjs` change in #7427 +(`dc16fce9a8b34a073cc664643c7afd20201b2592` at inspection); it remains a separate +integration hold, not a recovery-module pass or an authored UI repair. + +## Main integration and lifecycle support + +The integration repair merges main +`15471f8e832d7fb5893539fa053170ea233de5a7` through +`c3c647d1409e2abe997304f9fcf7bbe2cb247763`. It retains main's +`prepare_turn_host_request` extraction and the recovery admission capture. +Recovery then delegates both source-instance and legacy-registration checks to +the existing `launch_lifetime` owner, instead of branching again in the executor. +It adds no lifecycle owner, migration or retired-Goal replay exception. + +Using the same AST metric, original head `335eae25f2544456c9bb644ea5db311210f747e0` +and the integration baseline both have 92 statements and 56 decision points in +`run_loopx_turn_once`. Runtime repair +`fc9b182a5ce0094c3f698a9ef590677a1a62054a` has 90 statements and 55 decision +points. The original failure is retained; the unchanged 90/60 limits now pass. +The full maintainability ratchet passes without a budget or exception change. + +The process matrix retains active-Goal terminal replay and adds retired-terminal +refusal for File and SQLite. It requires the original journal and single committed +debit to remain unchanged, alongside the independent downstream-operation check. +The refusal is the supported lifecycle boundary; it is not a compatibility gap +to fill with another branch. + +Fresh bounded checks use Python 3.14.8 and Node 24.21.0: 139 TypeScript tests, +control-plane typechecking and all five mutation probes pass. An extra two-file +mypy invocation recursively includes the dependency graph and fails with 5,417 +errors in 617 files on both the immutable integration baseline and repair. +Comparing the complete diagnostics with source line coordinates removed finds +no additions or removals. This is not a passing whole-repository Python typecheck. +The PR description records the final Python and premerge results. The model and +six historical native failures above are not rerun or newly qualified by this +bounded integration repair. diff --git a/docs/reference/protocols/loopx-turn-v0.md b/docs/reference/protocols/loopx-turn-v0.md index df61718ff8..8c29f31e68 100644 --- a/docs/reference/protocols/loopx-turn-v0.md +++ b/docs/reference/protocols/loopx-turn-v0.md @@ -579,6 +579,27 @@ A dangling prepared effect remains owned by the existing provider readback protocol; the recovery decision only records that the readback is the next required check and does not claim general exactly-once execution. +An `in_progress` Journal without a saved typed result may invoke Host only when +its durable `host_attempt_count` is exactly zero. A positive, missing or invalid +counter means `host_outcome_unknown`: recovery is blocked even with +`--retry-failed-turn`. Reconcile the original execution and downstream effects; +do not clear its journal or create a replacement settlement identity. New +journals record zero before dispatch intent. Recovery also rechecks the current +Goal lifetime for legacy registrations, without rerunning quota admission. +Retired Goals are unsupported for executing recovery, including terminal receipt +replay. Refusal preserves the original journal and any committed debit; +read-only `inspect-journal` remains available. Active Goals retain original-receipt +replay without another Host invocation or debit. This check does not provide +continuous external-tool fencing. + +中文:`in_progress` 且未保存 typed result 时,只有持久化的 attempt 计数明确为零, +才可调用 Host。正数、缺失或无效计数返回 `host_outcome_unknown`;显式重试也不能 +绕过。先核对原执行与下游结果,不清空 journal,也不另造结算身份。新 journal 在 +派发意图前保存零值。恢复旧格式 Goal 时也检查当前生命周期,但不重新准入额度。 +退役 Goal 不支持执行恢复,包括终态回执重放。拒绝时保留原 journal 与已提交扣费; +仍可通过只读 `inspect-journal` 查看历史。有效 Goal 继续复用原回执,不重新调用 Host +或扣费。此检查不提供外部工具的持续围栏。 + When an existing Journal is continued, `run-once` persists one bounded `loopx_turn_recovery_audit_v0` record. Its `planned` value is the adopted shared decision; its `actual` value distinguishes `started` from `finished` and records diff --git a/loopx/cli_commands/turn_run_once.py b/loopx/cli_commands/turn_run_once.py index 7f0dd781fd..58d97c8a6c 100644 --- a/loopx/cli_commands/turn_run_once.py +++ b/loopx/cli_commands/turn_run_once.py @@ -80,6 +80,12 @@ def execute_turn_run_once( """Adapt one prepared CLI Turn plan to the bounded Turn driver.""" execution_started = False try: + if args.execute and goal_admission is None: + # Cached-result recovery still requires the current Goal lifetime. + goal_admission = FirstPartyHostGoalAdmission.for_plan( + registry_path=registry_path, goal_id=args.goal_id, + planned_goal_ref=goal_ref, + ) project, raw_argv, task_validator, envelope = prepare_turn_host_request( args, payload, native_host_runner=native_host_runner, native_session_binding_resolver=native_session_binding_resolver, diff --git a/loopx/control_plane/turn_driver/executor.py b/loopx/control_plane/turn_driver/executor.py index bc827590b5..0514370f88 100644 --- a/loopx/control_plane/turn_driver/executor.py +++ b/loopx/control_plane/turn_driver/executor.py @@ -1553,7 +1553,12 @@ def persist_source_journal(source_admission: Mapping[str, Any]) -> None: recovery_decision: dict[str, Any] | None = None if journal is not None: if goal_admission is not None: - goal_admission.require_current() + # One lifetime owner checks source instances and legacy registrations; + # recovery does not reserve another quota interval. + with goal_admission.launch_lifetime( + operation="turn_recovery_lifetime", + ): + pass envelope = ( plan.get("turn_envelope") if isinstance(plan.get("turn_envelope"), Mapping) @@ -1683,6 +1688,7 @@ def persist_source_journal(source_admission: Mapping[str, Any]) -> None: "status": "in_progress", "host": host_projection, "completed_phases": [], + "host_attempt_count": 0, "plan": dict(plan), } persist_journal(journal) diff --git a/loopx/control_plane/turn_driver/recovery.py b/loopx/control_plane/turn_driver/recovery.py index 0408d3f9f2..a934d350f5 100644 --- a/loopx/control_plane/turn_driver/recovery.py +++ b/loopx/control_plane/turn_driver/recovery.py @@ -95,6 +95,11 @@ def require_turn_recovery_continuation( if assessment.error is not None else "LoopX Turn journal recovery is blocked" ) + if assessment.decision.get("reason") == "host_outcome_unknown": + message = ( + "Original Host attempt outcome is unknown; reconcile its execution and " + "downstream effects before retrying. Preserve the original Turn journal." + ) raise TurnRecoveryBlockedError(assessment.decision, message) diff --git a/loopx/control_plane/turn_driver/turn_journal.ts b/loopx/control_plane/turn_driver/turn_journal.ts index 79b49f72bd..3ab05a7c27 100644 --- a/loopx/control_plane/turn_driver/turn_journal.ts +++ b/loopx/control_plane/turn_driver/turn_journal.ts @@ -453,6 +453,18 @@ function recoveryDecision( } const resumeFrom = recoveryResumePhase(journal, completedPhases, journalStatus); + // A durable dispatch intent without its result cannot prove non-execution. + if (journalStatus === "in_progress" && completedPhases.length < 2 + && journal.host_attempt_count !== 0) { + return { + ...base, + action: "blocked", + can_continue: false, + resume_from: resumeFrom, + reinvoke_host: false, + reason: "host_outcome_unknown", + }; + } if (journalStatus === "failed" && !retryFailed) { return { ...base, diff --git a/loopx/control_plane/turn_driver/turn_journal_effect_readback.ts b/loopx/control_plane/turn_driver/turn_journal_effect_readback.ts index b3867f9754..4a1667ecff 100644 --- a/loopx/control_plane/turn_driver/turn_journal_effect_readback.ts +++ b/loopx/control_plane/turn_driver/turn_journal_effect_readback.ts @@ -40,8 +40,7 @@ export function recordedTurnEffects( // An attempt is persisted BEFORE confirmation/host launch. It proves neither // launch nor non-launch until the host checkpoint is durable. const hostCount = journal.host_attempt_count; - const hostUncertain = hostCount !== undefined - && (!Number.isInteger(hostCount) || Number(hostCount) !== 0); + const hostUncertain = hostCount !== 0; return { host_invoked: completed.has("host_execute") ? true : hostUncertain ? null : false, state_written: completed.has("durable_writeback") ? true diff --git a/pyproject.toml b/pyproject.toml index 9f99527e47..10eaf4049f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -47,6 +47,7 @@ test-core = [ "pytest>=9.0.3,<10", ] test = [ + "hypothesis==6.142.4", "tiktoken==0.12.0", "distlib>=0.3.9,<1; sys_platform == 'win32'", "docker==7.2.0", diff --git a/tests/control_plane/check_turn_recovery_mutations.py b/tests/control_plane/check_turn_recovery_mutations.py new file mode 100644 index 0000000000..0dca39e848 --- /dev/null +++ b/tests/control_plane/check_turn_recovery_mutations.py @@ -0,0 +1,133 @@ +"""Check that executable assertions reject removed recovery protections. + +Creates disposable sibling modules to retain real relative imports. It never +edits the production originals. Run serially; sibling modules are removed. +""" + +from __future__ import annotations + +import json +from pathlib import Path +import subprocess +import sys +import tempfile + +ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(ROOT)) + + +def check_mutation(name, module_name, test_name, old, new): + module = ROOT / module_name + test = ROOT / test_name + source = module.read_text() + assert source.count(old) == 1, f"{name}: mutation anchor drifted" + candidate = module.with_name("_recovery_mutant_" + module.name) + probe = test.with_name("_recovery_mutant_" + test.name) + assert not candidate.exists() and not probe.exists() + original = source.encode() + try: + candidate.write_text(source.replace(old, new)) + old_import = "../../" + module_name + new_import = "../../" + str(candidate.relative_to(ROOT)) + assert old_import in test.read_text() + probe.write_text(test.read_text().replace(old_import, new_import)) + run = subprocess.run( + [ + "node", + "--experimental-strip-types", + "--test", + "--test-isolation=none", + str(probe), + ], + cwd=ROOT, + capture_output=True, + text=True, + timeout=30, + ) + # A syntax/import failure is invalid mutation evidence. + assert run.returncode == 1 and "AssertionError" in run.stdout + run.stderr, ( + run.stdout + run.stderr + ) + return { + "mutation": name, + "rejected": True, + "exit_code": run.returncode, + "output": run.stdout + run.stderr, + } + finally: + candidate.unlink(missing_ok=True) + probe.unlink(missing_ok=True) + assert module.read_bytes() == original + + +def lost_requirement(): + from tests.control_plane.turn_interruption_oracle import ( + INPUT_CSV, + REQUIREMENT, + assert_export, + ) + import sqlite3 + + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + (root / "docs").mkdir() + (root / "docs/export.csv").write_text(INPUT_CSV) + (root / "requirements-read.txt").write_text(REQUIREMENT) + (root / "received.txt").write_text("Repair export.") # Lost original clause. + db = root / "service.sqlite" + with sqlite3.connect(db) as c: + c.execute( + "CREATE TABLE operations(sequence INTEGER PRIMARY KEY, body TEXT)" + ) + c.execute("INSERT INTO operations(body) VALUES(?)", (INPUT_CSV,)) + try: + assert_export(root, db, 1) + except AssertionError: + return {"mutation": "drop_original_requirement", "rejected": True} + raise AssertionError("requirement loss was not detected") + + +def main(): + cases = [ + ( + "unknown_dispatch_guard", + "loopx/control_plane/turn_driver/turn_journal.ts", + "tests/control_plane_ts/turn_journal.test.ts", + "&& journal.host_attempt_count !== 0", + "&& false", + ), + ( + "stale_generation_guard", + "loopx/control_plane/goals/first_party_host_runtime.ts", + "tests/control_plane_ts/first_party_host_runtime.test.ts", + "if (!sameGoalRef(planned.value, current.goalRef)) {", + "if (false) {", + ), + ( + "current_authority_guard", + "loopx/control_plane/goals/first_party_host_runtime.ts", + "tests/control_plane_ts/first_party_host_runtime.test.ts", + 'if (current.activationState === "stopped") {', + "if (false) {", + ), + ( + "settlement_dedup", + "loopx/control_plane/effect_program.ts", + "tests/control_plane_ts/effect_program.test.ts", + '(step) => step !== "validation" && !completed.has(step)', + '(step) => step !== "validation"', + ), + ] + report = [check_mutation(*case) for case in cases] + [lost_requirement()] + if len(sys.argv) == 2: + Path(sys.argv[1]).write_text(json.dumps(report, indent=2)) + print( + json.dumps( + [{k: v for k, v in row.items() if k != "output"} for row in report], + indent=2, + ) + ) + + +if __name__ == "__main__": + main() diff --git a/tests/control_plane/test_turn_interruption_recovery.py b/tests/control_plane/test_turn_interruption_recovery.py new file mode 100644 index 0000000000..f767c4142b --- /dev/null +++ b/tests/control_plane/test_turn_interruption_recovery.py @@ -0,0 +1,412 @@ +"""Synthetic export repair through the real CLI and providers, with SIGKILL/restart. + +The Codex executable is an injected deterministic recorder, not a real model. +""" + +from __future__ import annotations + +import importlib.util +import json +import os +from pathlib import Path +import signal +import sqlite3 +import hashlib +import platform +import subprocess +import sys +import time + +import pytest + +from tests.control_plane.canonical_authority_fixture import ( + initialize_canonical_authority, + isolate_sqlite_runtime, +) +from tests.control_plane.turn_interruption_oracle import ( + INPUT_CSV, + REQUIREMENT, + assert_export, + assert_host_stopped, + spend_records, +) +from loopx.control_plane.coordination.runtime_shadow import ( + build_todo_runtime_shadow_projection, +) +from loopx.control_plane.todos.active_state_todo_parser import parse_active_state_todos + +ROOT = Path(__file__).resolve().parents[2] +BOUNDARIES = ( + "intent", "tool", "result", "writeback", "spend", "receipt", "revoked", + "retired_receipt", +) + +# Independent downstream: append EVERY received operation, deliberately no dedup. +SERVICE = """import json,os,sqlite3,sys +from pathlib import Path +db,fifo,ready=sys.argv[1:] +with sqlite3.connect(db) as c: c.execute("CREATE TABLE operations(sequence INTEGER PRIMARY KEY, body TEXT)") +os.mkfifo(fifo) +stream=os.fdopen(os.open(fifo,os.O_RDWR),'r') +Path(ready).touch() +for line in stream: + request=json.loads(line) + with sqlite3.connect(db) as c: c.execute("INSERT INTO operations(body) VALUES(?)",(request['body'],)) + Path(request['ack']).touch() +""" + +# Pauses the real CLI at an exact journal boundary. No production decision is replaced. +DRIVER = """import os,signal,sys +from pathlib import Path +from loopx.control_plane.turn_driver import executor +from loopx.cli import main +original=executor._write_journal +boundary=os.environ.get('FAULT_BOUNDARY','') +root=Path(os.environ['FAULT_ROOT']) +def checkpoint(path,journal,**kwargs): + phases=journal.get('completed_phases',[]) + before=((boundary=='writeback' and phases==['host_execute','typed_result','validation','durable_writeback']) + or (boundary=='spend' and phases==['host_execute','typed_result','validation','durable_writeback','quota_spend'])) + if not before: original(path,journal,**kwargs) + match=(before or (boundary=='intent' and journal.get('host_attempt_count')==1 and not phases) + or (boundary=='result' and phases==['host_execute','typed_result']) + or (boundary=='receipt' and journal.get('status')=='committed')) + if match: + (root/'paused').write_text(boundary) + os.kill(os.getpid(),signal.SIGSTOP) + if before: original(path,journal,**kwargs) +executor._write_journal=checkpoint +raise SystemExit(main(sys.argv[1:])) +""" + + +def wait_for(path: Path, process: subprocess.Popen, seconds: float = 30) -> None: + deadline = time.monotonic() + seconds + while not path.exists(): + assert process.poll() is None, ( + f"process exited before {path.name}: {process.returncode}" + ) + assert time.monotonic() < deadline, f"no {path.name} within {seconds}s" + time.sleep(0.02) + + +def fixture_module(): + path = ROOT / "examples/loopx-turn-codex-cli-e2e-smoke.py" + spec = importlib.util.spec_from_file_location("interruption_fixture", path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def setup_export(tmp_path, monkeypatch, provider, fifo): + isolate_sqlite_runtime(tmp_path, monkeypatch) + f = fixture_module() + project, runtime, workspace, registry = f._write_fixture(tmp_path, turn_count=2) + state = project / ".codex/goals" / f.GOAL_ID / "ACTIVE_GOAL_STATE.md" + state.write_text( + state.read_text().replace( + "Name the completed and next", REQUIREMENT + " Name the completed and next" + ) + ) + summary = parse_active_state_todos( + state.read_text(), state_path=state, item_limit=None + ) + initialize_canonical_authority( + runtime, + f.GOAL_ID, + build_todo_runtime_shadow_projection( + goal_id=f.GOAL_ID, + handoff_mode="soft_claim", + todos=summary["agent_todos"]["items"], + ), + state_path=state, + provider=provider, + ) + (workspace / "docs/input.csv").write_text(INPUT_CSV) + (workspace / "docs/requirements.txt").write_text(REQUIREMENT) + # The injected Host repairs this deliberate test defect before invoking it. + (workspace / "docs/export.py").write_text( + "import csv,io,json,time,uuid\nfrom pathlib import Path\n" + "rows=list(csv.reader(Path('docs/input.csv').open()))\n" + "out=io.StringIO(); writer=csv.writer(out,lineterminator='\\n')\n" + "for row in sorted(rows): writer.writerow(row)\n" + "body=out.getvalue(); Path('docs/export.csv').write_text(body)\n" + f"fifo=Path({str(fifo)!r}); ack=fifo.parent / (uuid.uuid4().hex + '.ack')\n" + "with fifo.open('w') as service: service.write(json.dumps({'body':body,'ack':str(ack)})+'\\n')\n" + "deadline=time.monotonic()+5\nwhile not ack.exists():\n assert time.monotonic() None: + expected = list(csv.reader(io.StringIO(INPUT_CSV))) + with sqlite3.connect(downstream) as db: + observed = db.execute( + "SELECT body FROM operations ORDER BY sequence" + ).fetchall() + assert len(observed) == expected_calls, observed + for (body,) in observed: + assert list(csv.reader(io.StringIO(body))) == expected + if expected_calls: + with (workspace / "docs/export.csv").open() as stream: + assert list(csv.reader(stream)) == expected + assert (workspace / "requirements-read.txt").read_text() == REQUIREMENT + assert REQUIREMENT in (workspace / "received.txt").read_text() + + +def spend_records(runtime: Path, goal: str) -> list[dict]: + path = runtime / "goals" / goal / "runs/index.jsonl" + return ( + [ + row + for line in path.read_text().splitlines() + if (row := json.loads(line)).get("classification") == "quota_slot_spent" + ] + if path.exists() + else [] + ) + + +def assert_host_stopped(workspace: Path) -> None: + """Observe the injected local process independently of LoopX status fields.""" + import os + import time + + pid_file = workspace / "host.pid" + if not pid_file.exists(): + return + pid = int(pid_file.read_text()) + deadline = time.monotonic() + 10 + while True: + try: + os.kill(pid, 0) + except ProcessLookupError: + return + proc_stat = Path(f"/proc/{pid}/stat") + if proc_stat.exists() and proc_stat.read_text().split(") ", 1)[1].startswith( + "Z " + ): + return # A zombie cannot execute tools. + assert time.monotonic() < deadline, f"old Host {pid} still executes" + time.sleep(0.02) diff --git a/tests/control_plane_ts/turn_journal.test.ts b/tests/control_plane_ts/turn_journal.test.ts index c7de588698..6582b60d82 100644 --- a/tests/control_plane_ts/turn_journal.test.ts +++ b/tests/control_plane_ts/turn_journal.test.ts @@ -198,11 +198,47 @@ test("recorded effects distinguish checkpoint, prepared uncertainty and no attem input.journal.completed_phases = []; delete input.journal.effect_attempts; + input.journal.host_attempt_count = 0; assert.equal(interpretTurnJournal(input).recorded_effects.host_invoked, false); input.journal.host_attempt_count = 1; assert.equal(interpretTurnJournal(input).recorded_effects.host_invoked, null); }); +test("an interrupted Host intent is unknown and cannot authorize another Host", () => { + for (const attempt of [1, 2, "1", null, -1, undefined]) { + for (const retry of [false, true]) { + const input = request("in_progress"); + input.journal.completed_phases = []; + delete input.journal.host_result; + delete input.journal.receipt; + if (attempt !== undefined) input.journal.host_attempt_count = attempt; + input.retry_failed = retry; + const before = structuredClone(input); + const decision = interpretTurnJournal(input).recovery_decision; + assert.equal(decision.action, "blocked", `attempt=${attempt}, retry=${retry}`); + assert.equal(decision.reason, "host_outcome_unknown"); + assert.equal(decision.reinvoke_host, false); + assert.deepEqual(input, before); + } + } +}); + +test("durable non-dispatch and a saved result have distinct recovery paths", () => { + const input = request("in_progress"); + input.journal.completed_phases = []; + delete input.journal.host_result; + delete input.journal.receipt; + input.journal.host_attempt_count = 0; + assert.equal(interpretTurnJournal(input).recovery_decision.reinvoke_host, true); + input.journal.host_attempt_count = 1; + input.journal.completed_phases = ["host_execute", "typed_result"]; + input.journal.host_result = {turn_key: turnKey}; + const decision = interpretTurnJournal(input).recovery_decision; + assert.equal(decision.action, "continue"); + assert.equal(decision.resume_from, "validation"); + assert.equal(decision.reinvoke_host, false); +}); + test("pending spend and foreign lineage cannot certify a quota charge", () => { const input = request("in_progress"); input.journal.completed_phases = ["host_execute", "typed_result", "validation", "durable_writeback"]; diff --git a/tests/models/turn_recovery/README.md b/tests/models/turn_recovery/README.md new file mode 100644 index 0000000000..602cac0ab2 --- /dev/null +++ b/tests/models/turn_recovery/README.md @@ -0,0 +1,76 @@ +# Recovery model and reproducible checks + +The implementation note is [turn interruption recovery](../../../docs/development/turn-interruption-recovery.md). +`requirements.txt` pins the Python 3.11 dependencies for its bounded verification +selection. `package-lock.json` at the repository root pins the TypeScript tools. + +## Model scope + +One work item, two competing actors (`1`, `2`), at most two process failures, +one original settlement identity, and a finite set of durable phases. `live` +can retain an old actor after a crash; `admitted` represents current logical +execution authority. The model does not equate physical liveness and permission. + +`OneAuthority` checks the admitted set and its owner. `NoReplacement` checks +that a saved, unresolved dispatch cannot admit a replacement attempt. +`NoDuplicateEffect` and `AtMostOneSettlement` check the corresponding abstract +counters. `NoUnauthorizedAcceptance` checks acceptance under the current +abstract authorization. `Terminates` requires eventual settlement, an explicit +unknown outcome, or revocation. + +Weak fairness applies to `Advance` and `Recover`: continuously enabled progress +and recovery eventually run. Storage is available and each modeled checkpoint +is atomic and retained. Failures stop after two occurrences. Authorization may +be revoked, which ends this bounded work. Without these assumptions, the model +does not establish eventual completion. + +## Actions and code boundaries + +| Model action | Production boundary | Assumption / omitted detail | +| --- | --- | --- | +| `Admit` | `lane_fence.py::turn_lane_singleflight`, journal lock in `run_loopx_turn_once` | Cooperative clients share functioning local locks; not distributed consensus | +| `Dispatch` | `_host_result_stage` persists attempt count before invoking Host | Host launch never precedes this checkpoint | +| `Tool` | Owned Host executes the synthetic exporter | No universal external fencing or rollback is modeled | +| `SaveResult` | `_host_result_stage` writes `host_result` and the first two completed phases | Atomic journal snapshot, no torn write | +| `Crash` | Process termination releases its kernel locks | Durable facts remain; an old physical actor may still exist | +| `Recover` | `turn_journal.ts::recoveryDecision` and `require_turn_recovery_continuation` | Saved result resumes settlement; unknown intent refuses another Host | +| `Settle` | Existing prepared-effect/readback/checkpoint owners and original receipt replay | Abstracts an atomic provider debit per retained identity; the separate provider/journal crash windows are checked by pytest | +| `Revoke` | Existing first-party Goal lifetime checks | Boolean abstraction of current authority; not a lease, scope or version engine | + +The model omits context encoding, schema migration, corruption, power loss, +remote-service failure, arbitrary tool APIs and a universal generation fence. +It cannot establish those properties for code. In particular, its settlement +abstraction does not make the provider and journal one transaction. The +subprocess suite tests their distinct checkpoints. Native context qualification +has a separate retained failure, documented in the implementation note. + +## TLC commands + +Use Java 17.0.17. The tool fetched for this run reports +`TLC2 Version 2026.10.26.231022 (rev: fad899c)`. +The download path is a release asset, but its observed SHA-256 is the pin: + +```sh +curl -fL https://github.com/tlaplus/tlaplus/releases/download/v1.8.0/tla2tools.jar -o tla2tools.jar +printf '%s\n' '78f7e2c6bf9a8b65eda665a23e94431255587e0b43a834a4e2af1723562d9b4a tla2tools.jar' | sha256sum -c - +cd tests/models/turn_recovery +java -Xmx256m -XX:+UseParallelGC -cp ../../../tla2tools.jar tlc2.TLC -seed 7416 -fp 0 -workers 1 -metadir ../../../recovery-model-safe -config TurnRecovery.cfg TurnRecovery.tla +java -Xmx256m -XX:+UseParallelGC -cp ../../../tla2tools.jar tlc2.TLC -seed 7416 -fp 0 -workers 1 -metadir ../../../recovery-model-unsafe -config Unsafe.cfg TurnRecovery.tla +``` + +The safe run explored 80 distinct states (160 generated), depth 10, and completed +its invariant and temporal checks. This supports the stated finite model only. +`Unsafe.cfg` removes the unknown-outcome guard. Its expected failure is exit 12 +at `NoReplacement`, with this seven-state counterexample: + +```text +new -> Admit(1) -> Dispatch(attempt 1) -> Crash + -> Recover(new) -> Admit(1) -> Dispatch(attempt 2) +``` + +The unsafe run generated 84 states and found 55 distinct states before stopping. +The production mutation check independently removes the corresponding guard +and requires real assertion failures. Its other mutations test stale Goal +instances, current stop authority, settlement deduplication and the original +export requirement. Neither a model counterexample nor an assertion failure +is a human maintainer's approval. diff --git a/tests/models/turn_recovery/TurnRecovery.cfg b/tests/models/turn_recovery/TurnRecovery.cfg new file mode 100644 index 0000000000..65c263f129 --- /dev/null +++ b/tests/models/turn_recovery/TurnRecovery.cfg @@ -0,0 +1,5 @@ +SPECIFICATION Spec +CONSTANT GuardUnknown = TRUE +INVARIANT OneAuthority NoReplacement NoDuplicateEffect AtMostOneSettlement NoUnauthorizedAcceptance +PROPERTY Terminates +CHECK_DEADLOCK FALSE diff --git a/tests/models/turn_recovery/TurnRecovery.tla b/tests/models/turn_recovery/TurnRecovery.tla new file mode 100644 index 0000000000..42abcffac8 --- /dev/null +++ b/tests/models/turn_recovery/TurnRecovery.tla @@ -0,0 +1,47 @@ +------------------------ MODULE TurnRecovery ------------------------ +EXTENDS Naturals, FiniteSets, TLC +CONSTANT GuardUnknown +VARIABLES phase, owner, admitted, live, attempts, effects, settlements, failures, authorized +vars == <> +Init == /\ phase = "new" /\ owner = 0 /\ admitted = {} /\ live = {} + /\ attempts = 0 /\ effects = 0 /\ settlements = 0 + /\ failures = 0 /\ authorized = TRUE +Admit(e) == /\ phase = "new" /\ owner = 0 /\ admitted = {} /\ authorized + /\ owner' = e /\ admitted' = admitted \cup {e} /\ live' = live \cup {e} + /\ UNCHANGED <> +Dispatch == /\ phase = "new" /\ owner # 0 /\ authorized + /\ phase' = "intent" /\ attempts' = attempts + 1 + /\ UNCHANGED <> +Tool == /\ phase = "intent" /\ owner # 0 /\ authorized /\ effects < attempts + /\ effects' = effects + 1 + /\ UNCHANGED <> +SaveResult == /\ phase = "intent" /\ owner # 0 /\ authorized /\ effects = attempts + /\ phase' = "result" + /\ UNCHANGED <> +Crash == /\ owner # 0 /\ phase \in {"new", "intent", "result"} /\ failures < 2 + /\ owner' = 0 /\ admitted' = {} /\ failures' = failures + 1 + /\ UNCHANGED <> +Recover == /\ owner = 0 /\ phase \in {"intent", "result"} + /\ phase' = IF phase = "result" THEN "result" + ELSE IF GuardUnknown THEN "unknown" ELSE "new" + /\ live' = {} /\ owner' = IF phase = "result" THEN 1 ELSE 0 + /\ admitted' = IF phase = "result" THEN {1} ELSE {} + /\ UNCHANGED <> +Settle == /\ phase = "result" /\ owner # 0 /\ authorized + /\ phase' = "settled" /\ settlements' = 1 /\ owner' = 0 /\ admitted' = {} /\ live' = {} + /\ UNCHANGED <> +Revoke == /\ authorized /\ phase \notin {"settled", "unknown", "revoked"} + /\ authorized' = FALSE /\ phase' = "revoked" /\ owner' = 0 /\ admitted' = {} /\ live' = {} + /\ UNCHANGED <> +Advance == (\E e \in {1,2}: Admit(e)) \/ Dispatch \/ Tool \/ SaveResult \/ Settle +Next == Advance \/ Crash \/ Recover \/ Revoke +Spec == Init /\ [][Next]_vars /\ WF_vars(Advance) /\ WF_vars(Recover) +OneAuthority == /\ Cardinality(admitted) <= 1 + /\ (owner = 0 <=> admitted = {}) + /\ (owner # 0 => admitted = {owner}) +NoReplacement == attempts <= 1 +NoDuplicateEffect == effects <= 1 +AtMostOneSettlement == settlements <= 1 +NoUnauthorizedAcceptance == ~authorized => settlements = 0 +Terminates == <> (phase \in {"settled", "unknown", "revoked"}) +============================================================================= diff --git a/tests/models/turn_recovery/Unsafe.cfg b/tests/models/turn_recovery/Unsafe.cfg new file mode 100644 index 0000000000..abbfa138b8 --- /dev/null +++ b/tests/models/turn_recovery/Unsafe.cfg @@ -0,0 +1,4 @@ +SPECIFICATION Spec +CONSTANT GuardUnknown = FALSE +INVARIANT OneAuthority NoReplacement NoDuplicateEffect AtMostOneSettlement NoUnauthorizedAcceptance +CHECK_DEADLOCK FALSE diff --git a/tests/models/turn_recovery/evidence.json b/tests/models/turn_recovery/evidence.json new file mode 100644 index 0000000000..7bd1481719 --- /dev/null +++ b/tests/models/turn_recovery/evidence.json @@ -0,0 +1,906 @@ +{ + "synthetic": true, + "observed_at": "2026-10-27T05:54:48.992688+00:00", + "implementation_baseline": "218aa087b66bb99038b1ce7d782110741a5e42ae", + "prepublish_target": "bafbafc258de72029b54ea016103c0a7bd2409c9", + "source_sha256": { + "loopx/cli_commands/turn_run_once.py": "550160370adb28ad46dce9ee34e9531cd48ed68829a19dea3f6e5996e3c8ad3f", + "loopx/control_plane/turn_driver/executor.py": "596dcf1e5052fcec17aef22f44f55d7b7a0286db691355ae7c127d238c87bf03", + "loopx/control_plane/turn_driver/recovery.py": "1307b0eee7ff96fe4393932c90a525b5cfbe2eb6863e4e755f4832f95f8671a3", + "loopx/control_plane/turn_driver/turn_journal.ts": "acb7f24c1408dc459e407d214a9a728b936199995209d5321dc839f68018dd0c", + "loopx/control_plane/turn_driver/turn_journal_effect_readback.ts": "a047c804d79c3d0be3ab2e416744460d5745423f374699a2d9acc3fd21258d3b", + "pyproject.toml": "5ee122852c42eb0679d680abba87e6177e332a00c2ab57ad5eaf895b91206ea0", + "tests/control_plane/test_turn_interruption_recovery.py": "d3e0a932595d61561d02a1c660b63b5d2a9e9c85efd5c6a08d5b8be7fc46a2e2", + "tests/control_plane/turn_interruption_oracle.py": "830de844dbeea4862cc6feca932c959e5942a63594abc7b07c93718d4ae21e4f", + "tests/control_plane/test_turn_recovery_state_machine.py": "f785de49a7ef3143920a2f0b156acc1917060d4e6b2ffc013e96dff0371c9099", + "tests/control_plane/check_turn_recovery_mutations.py": "cd49c479fc6ea9ab5de1bb56c7a3d9233a187ea67b6bc3d4a685882e8b4e76fb", + "tests/control_plane_ts/turn_journal.test.ts": "3cb34482ed73bb1de38ee1e1088ecbea21cb8bc94892b0443d7de94d05f156e7", + "tests/test_loopx_turn_executor.py": "b51bc6ff032b375fb7fee4751d7433547f58121aea3cfedd3716b881ed283fb3", + "tests/models/turn_recovery/TurnRecovery.tla": "3e362eb09bb43e0e9c4636bee2c657608808998dcab2170e9d1f958ead0a1677", + "tests/models/turn_recovery/TurnRecovery.cfg": "4c1227941d232389407df78d44d32b02ac554edb3b17c0bfb3cd49872f878cf5", + "tests/models/turn_recovery/Unsafe.cfg": "2fabe115ffeb2b1fd2b5937058217541f73250221b5afb77d26cbf18f84f288b", + "tests/models/turn_recovery/requirements.txt": "e819a8b3a3b2bc35a2b15132233e73aa713fe9f79e8d4366712b5545e634a27d", + "tests/requirements-stage2c-linux-py311.txt": "eace18917359f05ad4b402362550c53a9c49c742638900b48981fdb8fbe25b65" + }, + "scope": "One original Turn, local cooperative owners, durable result reuse; unknown Host outcomes refuse redispatch.", + "scenarios": [ + { + "provider": "file", + "boundary": "intent", + "signal": "SIGKILL", + "before": { + "goal_id": "loopx-turn-real-cli-e2e", + "status": "in_progress", + "completed_phases": [], + "host_attempt_count": 1, + "turn_key": "sha256:739af8ca99ce328242060ef0ddfdcaefda95dd221321134685796305c68b17df" + }, + "original_effect_id": "loopx-turn-real-cli-e2e:codex-turn-e2e:todo_turnreale2e01:synthetic-export-original", + "downstream_records": [], + "artifact": null, + "recovery_status": null, + "recovery_reason": "host_outcome_unknown", + "recorded_effects": { + "host_invoked": null, + "state_written": false, + "quota_spent": false, + "scheduler_acknowledged": false + }, + "downstream_calls": 0, + "settlements": 0, + "capture_sha256": "d0f17d287727307305dd955071390b3366b81268d1f83445a62695e6623f0b81", + "stop_observation": "host_not_started" + }, + { + "provider": "sqlite", + "boundary": "intent", + "signal": "SIGKILL", + "before": { + "goal_id": "loopx-turn-real-cli-e2e", + "status": "in_progress", + "completed_phases": [], + "host_attempt_count": 1, + "turn_key": "sha256:375b2e8d9c4ea7ca0135ea6d26a268390032cd1a2e651f1180521a79e352186c" + }, + "original_effect_id": "loopx-turn-real-cli-e2e:codex-turn-e2e:todo_turnreale2e01:synthetic-export-original", + "downstream_records": [], + "artifact": null, + "recovery_status": null, + "recovery_reason": "host_outcome_unknown", + "recorded_effects": { + "host_invoked": null, + "state_written": false, + "quota_spent": false, + "scheduler_acknowledged": false + }, + "downstream_calls": 0, + "settlements": 0, + "capture_sha256": "e35f621778bdc3114ab8a8aec7f6b33b2c99688a0860bd6226563dabdaa74cb1", + "stop_observation": "host_not_started" + }, + { + "provider": "file", + "boundary": "receipt", + "signal": "SIGKILL", + "before": { + "goal_id": "loopx-turn-real-cli-e2e", + "status": "committed", + "completed_phases": [ + "host_execute", + "typed_result", + "validation", + "durable_writeback", + "quota_spend", + "scheduler_apply", + "scheduler_ack" + ], + "host_attempt_count": 1, + "turn_key": "sha256:af24ce582eef1847d6f4403e64d0e1ba5bc14b247ee1872d42253a0069c58820" + }, + "original_effect_id": "loopx-turn-real-cli-e2e:codex-turn-e2e:todo_turnreale2e01:synthetic-export-original", + "downstream_records": [ + [ + 1, + "name,score\nZ,3\nA,1\nM,1\n" + ] + ], + "artifact": "name,score\nZ,3\nA,1\nM,1\n", + "recovery_status": "committed", + "recovery_reason": null, + "recorded_effects": null, + "downstream_calls": 1, + "settlements": 1, + "capture_sha256": "e928af0ad759c9ca81163137322e68b2edbf65002d8a655cf0298b6419674f7c", + "stop_observation": "host_stopped" + }, + { + "provider": "sqlite", + "boundary": "receipt", + "signal": "SIGKILL", + "before": { + "goal_id": "loopx-turn-real-cli-e2e", + "status": "committed", + "completed_phases": [ + "host_execute", + "typed_result", + "validation", + "durable_writeback", + "quota_spend", + "scheduler_apply", + "scheduler_ack" + ], + "host_attempt_count": 1, + "turn_key": "sha256:d91afea5c4f507579777fe8226418ac6edfd2e5936dcd0854c00e5ac5b63cfb3" + }, + "original_effect_id": "loopx-turn-real-cli-e2e:codex-turn-e2e:todo_turnreale2e01:synthetic-export-original", + "downstream_records": [ + [ + 1, + "name,score\nZ,3\nA,1\nM,1\n" + ] + ], + "artifact": "name,score\nZ,3\nA,1\nM,1\n", + "recovery_status": "committed", + "recovery_reason": null, + "recorded_effects": null, + "downstream_calls": 1, + "settlements": 1, + "capture_sha256": "9177a5957e8aeb22469699f4ce92cb17f9f46e6f2ee53f9019f8a58911e0761f", + "stop_observation": "host_stopped" + }, + { + "provider": "file", + "boundary": "revoked", + "signal": "SIGKILL", + "before": { + "goal_id": "loopx-turn-real-cli-e2e", + "status": "in_progress", + "completed_phases": [ + "host_execute", + "typed_result" + ], + "host_attempt_count": 1, + "turn_key": "sha256:5e8a10c59d1409120e0340009ab5c2ee512f60179fb5eab2b673defcfab0d309" + }, + "original_effect_id": "loopx-turn-real-cli-e2e:codex-turn-e2e:todo_turnreale2e01:synthetic-export-original", + "downstream_records": [ + [ + 1, + "name,score\nZ,3\nA,1\nM,1\n" + ] + ], + "artifact": "name,score\nZ,3\nA,1\nM,1\n", + "recovery_status": null, + "recovery_reason": "goal_not_registered", + "recorded_effects": { + "host_invoked": true, + "state_written": false, + "quota_spent": false, + "scheduler_acknowledged": false + }, + "downstream_calls": 1, + "settlements": 0, + "capture_sha256": "c4c594bfd2be69f7ac2bac2c10b2c436c33f4acff304b1a96d290f71b076e834", + "stop_observation": "host_stopped" + }, + { + "provider": "sqlite", + "boundary": "revoked", + "signal": "SIGKILL", + "before": { + "goal_id": "loopx-turn-real-cli-e2e", + "status": "in_progress", + "completed_phases": [ + "host_execute", + "typed_result" + ], + "host_attempt_count": 1, + "turn_key": "sha256:d374142b0ef3a21d58dc97ede994dd8e6d35b321f824868182f462e553f69235" + }, + "original_effect_id": "loopx-turn-real-cli-e2e:codex-turn-e2e:todo_turnreale2e01:synthetic-export-original", + "downstream_records": [ + [ + 1, + "name,score\nZ,3\nA,1\nM,1\n" + ] + ], + "artifact": "name,score\nZ,3\nA,1\nM,1\n", + "recovery_status": null, + "recovery_reason": "goal_not_registered", + "recorded_effects": { + "host_invoked": true, + "state_written": false, + "quota_spent": false, + "scheduler_acknowledged": false + }, + "downstream_calls": 1, + "settlements": 0, + "capture_sha256": "56273701d31f5763cae8f8d212abcc5c5886971732f2bb5bd315ca3f235e50eb", + "stop_observation": "host_stopped" + }, + { + "provider": "file", + "boundary": "tool", + "signal": "SIGKILL", + "before": { + "goal_id": "loopx-turn-real-cli-e2e", + "status": "in_progress", + "completed_phases": [], + "host_attempt_count": 1, + "turn_key": "sha256:5c32ef00a7520160e6c2b1f63071b4145882e3fb0dd0c47b86f0c8b839c636cb" + }, + "original_effect_id": "loopx-turn-real-cli-e2e:codex-turn-e2e:todo_turnreale2e01:synthetic-export-original", + "downstream_records": [ + [ + 1, + "name,score\nZ,3\nA,1\nM,1\n" + ] + ], + "artifact": "name,score\nZ,3\nA,1\nM,1\n", + "recovery_status": null, + "recovery_reason": "host_outcome_unknown", + "recorded_effects": { + "host_invoked": null, + "state_written": false, + "quota_spent": false, + "scheduler_acknowledged": false + }, + "downstream_calls": 1, + "settlements": 0, + "capture_sha256": "124caa6e91818c97e3f99976672a682ee7c191b4dc2a31d3c5ac30e1d020d989", + "stop_observation": "host_stopped" + }, + { + "provider": "sqlite", + "boundary": "tool", + "signal": "SIGKILL", + "before": { + "goal_id": "loopx-turn-real-cli-e2e", + "status": "in_progress", + "completed_phases": [], + "host_attempt_count": 1, + "turn_key": "sha256:e58c173a436bda7c5d8e9dd93783bd9d5770950b2275fb4b8c8d6c8324910034" + }, + "original_effect_id": "loopx-turn-real-cli-e2e:codex-turn-e2e:todo_turnreale2e01:synthetic-export-original", + "downstream_records": [ + [ + 1, + "name,score\nZ,3\nA,1\nM,1\n" + ] + ], + "artifact": "name,score\nZ,3\nA,1\nM,1\n", + "recovery_status": null, + "recovery_reason": "host_outcome_unknown", + "recorded_effects": { + "host_invoked": null, + "state_written": false, + "quota_spent": false, + "scheduler_acknowledged": false + }, + "downstream_calls": 1, + "settlements": 0, + "capture_sha256": "ca7253837b56988c3d59d68a518095a6a3a5a2d28dd6e2f1390d0ef8656e8209", + "stop_observation": "host_stopped" + }, + { + "provider": "file", + "boundary": "result", + "signal": "SIGKILL", + "before": { + "goal_id": "loopx-turn-real-cli-e2e", + "status": "in_progress", + "completed_phases": [ + "host_execute", + "typed_result" + ], + "host_attempt_count": 1, + "turn_key": "sha256:8dbb0ba0ae12752c0c75fd92f31e9ed8ea1f031403709c81a4383961d59d00de" + }, + "original_effect_id": "loopx-turn-real-cli-e2e:codex-turn-e2e:todo_turnreale2e01:synthetic-export-original", + "downstream_records": [ + [ + 1, + "name,score\nZ,3\nA,1\nM,1\n" + ] + ], + "artifact": "name,score\nZ,3\nA,1\nM,1\n", + "recovery_status": "committed", + "recovery_reason": null, + "recorded_effects": null, + "downstream_calls": 1, + "settlements": 1, + "capture_sha256": "d089c3674521a76652c010285a6bfb9b98ad08ed28c38f57785cce2b051b9e7a", + "stop_observation": "host_stopped" + }, + { + "provider": "sqlite", + "boundary": "result", + "signal": "SIGKILL", + "before": { + "goal_id": "loopx-turn-real-cli-e2e", + "status": "in_progress", + "completed_phases": [ + "host_execute", + "typed_result" + ], + "host_attempt_count": 1, + "turn_key": "sha256:95662538d142728c911463561768a91af1c8d7a31e58195a0233b2f1c58bea64" + }, + "original_effect_id": "loopx-turn-real-cli-e2e:codex-turn-e2e:todo_turnreale2e01:synthetic-export-original", + "downstream_records": [ + [ + 1, + "name,score\nZ,3\nA,1\nM,1\n" + ] + ], + "artifact": "name,score\nZ,3\nA,1\nM,1\n", + "recovery_status": "committed", + "recovery_reason": null, + "recorded_effects": null, + "downstream_calls": 1, + "settlements": 1, + "capture_sha256": "1a442c4fb938283887cf201d0eeaf3537d7e0f9f689307df9c243c89a126cbd3", + "stop_observation": "host_stopped" + }, + { + "provider": "file", + "boundary": "writeback", + "signal": "SIGKILL", + "before": { + "goal_id": "loopx-turn-real-cli-e2e", + "status": "in_progress", + "completed_phases": [ + "host_execute", + "typed_result", + "validation" + ], + "host_attempt_count": 1, + "turn_key": "sha256:6130efe5d571cc52ec5046f28f0a2dd3d6f47bcc6c79fc81edd2fbe9d948bc09" + }, + "original_effect_id": "loopx-turn-real-cli-e2e:codex-turn-e2e:todo_turnreale2e01:synthetic-export-original", + "downstream_records": [ + [ + 1, + "name,score\nZ,3\nA,1\nM,1\n" + ] + ], + "artifact": "name,score\nZ,3\nA,1\nM,1\n", + "recovery_status": "committed", + "recovery_reason": null, + "recorded_effects": null, + "downstream_calls": 1, + "settlements": 1, + "capture_sha256": "70d7f044c3c2e98429ab32627077bd6a81db243e63bfc95689e4387dc82406bf", + "stop_observation": "host_stopped" + }, + { + "provider": "sqlite", + "boundary": "writeback", + "signal": "SIGKILL", + "before": { + "goal_id": "loopx-turn-real-cli-e2e", + "status": "in_progress", + "completed_phases": [ + "host_execute", + "typed_result", + "validation" + ], + "host_attempt_count": 1, + "turn_key": "sha256:463dbed359b9aedf28e61cb004975794e54d3f823513444100b3d715a2be52fc" + }, + "original_effect_id": "loopx-turn-real-cli-e2e:codex-turn-e2e:todo_turnreale2e01:synthetic-export-original", + "downstream_records": [ + [ + 1, + "name,score\nZ,3\nA,1\nM,1\n" + ] + ], + "artifact": "name,score\nZ,3\nA,1\nM,1\n", + "recovery_status": "committed", + "recovery_reason": null, + "recorded_effects": null, + "downstream_calls": 1, + "settlements": 1, + "capture_sha256": "5a7f03a9b2bbc4f03d53243b609d899733dd4ea7f9e2566b9038f998e0e3d291", + "stop_observation": "host_stopped" + }, + { + "provider": "file", + "boundary": "spend", + "signal": "SIGKILL", + "before": { + "goal_id": "loopx-turn-real-cli-e2e", + "status": "in_progress", + "completed_phases": [ + "host_execute", + "typed_result", + "validation", + "durable_writeback" + ], + "host_attempt_count": 1, + "turn_key": "sha256:9559bef4274019dd35fb1fa1cfb9d73e5caee53b547a4ef0a6b1a933f1efb6b9" + }, + "original_effect_id": "loopx-turn-real-cli-e2e:codex-turn-e2e:todo_turnreale2e01:synthetic-export-original", + "downstream_records": [ + [ + 1, + "name,score\nZ,3\nA,1\nM,1\n" + ] + ], + "artifact": "name,score\nZ,3\nA,1\nM,1\n", + "recovery_status": "committed", + "recovery_reason": null, + "recorded_effects": null, + "downstream_calls": 1, + "settlements": 1, + "capture_sha256": "f1e8e176df8f20dbbebe8c298f8f2a48a489b0c45b1b1d1c6a076d9868ce240b", + "stop_observation": "host_stopped" + }, + { + "provider": "sqlite", + "boundary": "spend", + "signal": "SIGKILL", + "before": { + "goal_id": "loopx-turn-real-cli-e2e", + "status": "in_progress", + "completed_phases": [ + "host_execute", + "typed_result", + "validation", + "durable_writeback" + ], + "host_attempt_count": 1, + "turn_key": "sha256:d604cd52cc7b1182e52f16b8de2781983244c6528fa0fff0f1c898bcc716c017" + }, + "original_effect_id": "loopx-turn-real-cli-e2e:codex-turn-e2e:todo_turnreale2e01:synthetic-export-original", + "downstream_records": [ + [ + 1, + "name,score\nZ,3\nA,1\nM,1\n" + ] + ], + "artifact": "name,score\nZ,3\nA,1\nM,1\n", + "recovery_status": "committed", + "recovery_reason": null, + "recorded_effects": null, + "downstream_calls": 1, + "settlements": 1, + "capture_sha256": "d2cf819367dd9f2ec7657466ca3ce047e32fbfcaf185e7542e722651f18f172f", + "stop_observation": "host_stopped" + }, + { + "provider": "file", + "boundary": "normal", + "signal": null, + "before": null, + "original_effect_id": null, + "downstream_records": [ + [ + 1, + "name,score\nZ,3\nA,1\nM,1\n" + ] + ], + "artifact": "name,score\nZ,3\nA,1\nM,1\n", + "recovery_status": "committed", + "recovery_reason": null, + "recorded_effects": null, + "downstream_calls": 1, + "settlements": 1, + "capture_sha256": null, + "stop_observation": null + }, + { + "provider": "sqlite", + "boundary": "normal", + "signal": null, + "before": null, + "original_effect_id": null, + "downstream_records": [ + [ + 1, + "name,score\nZ,3\nA,1\nM,1\n" + ] + ], + "artifact": "name,score\nZ,3\nA,1\nM,1\n", + "recovery_status": "committed", + "recovery_reason": null, + "recorded_effects": null, + "downstream_calls": 1, + "settlements": 1, + "capture_sha256": null, + "stop_observation": null + } + ], + "model": { + "tool": "TLC2 2026.10.26.231022 fad899c", + "jar_sha256": "78f7e2c6bf9a8b65eda665a23e94431255587e0b43a834a4e2af1723562d9b4a", + "safe": { + "distinct_states": 80, + "generated_states": 160, + "depth": 10, + "invariants_and_temporal_properties": "passed" + }, + "guard_removed": { + "exit_code": 12, + "invariant": "NoReplacement", + "distinct_states": 55, + "generated_states": 84, + "depth": 7 + } + }, + "mutations": [ + { + "mutation": "unknown_dispatch_guard", + "rejected": true, + "exit_code": 1 + }, + { + "mutation": "stale_generation_guard", + "rejected": true, + "exit_code": 1 + }, + { + "mutation": "current_authority_guard", + "rejected": true, + "exit_code": 1 + }, + { + "mutation": "settlement_dedup", + "rejected": true, + "exit_code": 1 + }, + { + "mutation": "drop_original_requirement", + "rejected": true + } + ], + "pr_dedup": { + "open_prs": 38, + "paginated_file_records": 706, + "heads": [ + { + "number": 7455, + "head": "3854bcda12163263eb7da7580d7cc8b0a27583bc", + "draft": false, + "files_count": 1 + }, + { + "number": 7454, + "head": "7362bdc3f3ae90775037560739d66f4916d0033a", + "draft": false, + "files_count": 2 + }, + { + "number": 7453, + "head": "2f422f98fc5a2c782b3db3795a88ccc4b881503d", + "draft": false, + "files_count": 1 + }, + { + "number": 7451, + "head": "b9ca4d1acd85cf7adf429f127bd610513f1ccca8", + "draft": false, + "files_count": 5 + }, + { + "number": 7442, + "head": "80918d50e9a57eac3c67886fffd865fde10bc9a8", + "draft": false, + "files_count": 2 + }, + { + "number": 7440, + "head": "4bc97062eeec25fef3650e1525f8b45e15f8160e", + "draft": false, + "files_count": 8 + }, + { + "number": 7430, + "head": "275e59c25686f9e353e3c5966b4d06c289a6ed0b", + "draft": true, + "files_count": 42 + }, + { + "number": 7427, + "head": "dc16fce9a8b34a073cc664643c7afd20201b2592", + "draft": false, + "files_count": 7 + }, + { + "number": 7411, + "head": "f60d86ab68b5d7a9458d5314a78671d5f3ed2878", + "draft": false, + "files_count": 20 + }, + { + "number": 7398, + "head": "50ed43b9ac9688cb25a3679744159b40ebe4a265", + "draft": false, + "files_count": 4 + }, + { + "number": 7392, + "head": "714b4bb7fcd3d50f8eec35d858914a7101201dcb", + "draft": false, + "files_count": 45 + }, + { + "number": 7321, + "head": "576af8b44def7095aaec839c5e159ed49829d88d", + "draft": false, + "files_count": 4 + }, + { + "number": 7298, + "head": "d8daa76e604f856efccdaf7b7887edc97ec8178d", + "draft": false, + "files_count": 7 + }, + { + "number": 7252, + "head": "d9fdf9906a262ded01a7852ce5b738c0f0496ec2", + "draft": false, + "files_count": 3 + }, + { + "number": 7222, + "head": "48c24a21d2093cb7d969a28250dca68aa0dafb0c", + "draft": false, + "files_count": 7 + }, + { + "number": 7206, + "head": "16ee3e49b09efe0254226c189fff5706893b8c89", + "draft": false, + "files_count": 4 + }, + { + "number": 7205, + "head": "0c0128e309603a23f64b734d4e739248d396b911", + "draft": false, + "files_count": 4 + }, + { + "number": 7082, + "head": "81d84aaa67385730992c1262cabef93934883236", + "draft": true, + "files_count": 5 + }, + { + "number": 6902, + "head": "0567e08ab72410f8fb9487910e22be5935eb88c3", + "draft": true, + "files_count": 10 + }, + { + "number": 6900, + "head": "26778194f41752717637da79089a4e242914f80f", + "draft": true, + "files_count": 8 + }, + { + "number": 6875, + "head": "132b18f7f6ede65ae2b7d6e717c7c01f94bd5928", + "draft": false, + "files_count": 2 + }, + { + "number": 6838, + "head": "eac81014fd3a8842ddd420d7908657af107c6787", + "draft": true, + "files_count": 176 + }, + { + "number": 6764, + "head": "fd90ba87bc3cb439f4badb7debf443f492b39724", + "draft": true, + "files_count": 3 + }, + { + "number": 5969, + "head": "d83590b21bfae9d309b64a4429d9f24748d97fcb", + "draft": true, + "files_count": 10 + }, + { + "number": 5958, + "head": "e6fbd896a2e1d77ac3339afb498383aafeabd464", + "draft": true, + "files_count": 4 + }, + { + "number": 5677, + "head": "6e03113c7b5e1330947b6bff4d60b58d22e3007f", + "draft": false, + "files_count": 53 + }, + { + "number": 5598, + "head": "a0611332053b9c23c5a5fa628c0b55646f26cd5e", + "draft": false, + "files_count": 5 + }, + { + "number": 5385, + "head": "ad478f4753b5cfeaa5ee5248fceb8a06c8c28c2b", + "draft": false, + "files_count": 9 + }, + { + "number": 5291, + "head": "b78744a33904d75c0d6bdcf44e8c6f7000a9dcf0", + "draft": false, + "files_count": 13 + }, + { + "number": 5248, + "head": "a8dac55dffc8d2e428c5d0206587d839765bd872", + "draft": false, + "files_count": 51 + }, + { + "number": 4439, + "head": "7ae8e609834da025c8faa88025b2f446e97f010e", + "draft": false, + "files_count": 12 + }, + { + "number": 4368, + "head": "f640844a54fdb80e7ca489d723b5a9dfcf020de0", + "draft": false, + "files_count": 4 + }, + { + "number": 4140, + "head": "1d526c908b3ed5c87ede8784d95132adcd680ad1", + "draft": true, + "files_count": 7 + }, + { + "number": 4023, + "head": "451fd72407ce53b82898653c1370eb80fad1f842", + "draft": false, + "files_count": 68 + }, + { + "number": 3834, + "head": "c714bb740fbe309f6cc85d89fe98db7e189ad172", + "draft": false, + "files_count": 8 + }, + { + "number": 3412, + "head": "af5c5a75b2953de288c3c9f003fb9fc82512a3ca", + "draft": false, + "files_count": 10 + }, + { + "number": 3313, + "head": "5c6e6a7267caa1d9a3d3673588e3ee4dd9131b40", + "draft": false, + "files_count": 15 + }, + { + "number": 3200, + "head": "f6549a9e9f1f21cb1600de2765b461840c26a196", + "draft": false, + "files_count": 67 + } + ], + "semantic_overlap": { + "7430": "complete contract composition/capture and Host input; does not gate unresolved Host attempts or legacy cached-result recovery lifetime", + "3412": "prepared settlement effect readback; does not cover Host dispatch uncertainty", + "6838": "tracking changes share existing launch authority; not the executor recovery callsite", + "7454": "Windows tzdata dependency and CI; independent of test-only Hypothesis pin", + "7205/7206": "Same Stage 2C file, but those PRs only upgrade setuptools to 84.1.0; this change adds Hypothesis/sortedcontainers and retains all existing pins" + } + }, + "retained_failures": [ + { + "kind": "native_continuity", + "baseline_failed": 6, + "candidate_failed": 6, + "binary": "Codex 0.164.0", + "paid_model": false, + "signatures": { + "candidate_missing": 2, + "incomplete_model_visible_json": 4 + }, + "baseline_revision": "218aa087b66bb99038b1ce7d782110741a5e42ae", + "candidate_revision": "9d9b1183489164246c4ae770a06a8d43b28b00ef", + "final_native_qualification": "not_established" + }, + { + "kind": "revocation_regression_before_fix", + "cases": 2, + "observed": "retired Goal still wrote back and spent during cached-result recovery", + "fixed_by": "pass legacy lifetime admission into driver and check it before original-journal continuation" + }, + { + "kind": "normal_control_setup", + "cases": 2, + "observed": "unsupported python -m loopx test invocation", + "corrected": "use installed loopx console entrypoint; final 16-case matrix passed" + }, + { + "kind": "ci_dependency_integration", + "revision": "4472325f287a4f4d7ffb7a889c311511e8e98a0e", + "failures": [ + "Hypothesis missing from existing Stage 2C hash lock", + "Pygments 2.19.2 rejected by GHSA-5239-wwwm-4pmq" + ], + "correction": "add Hypothesis and sortedcontainers pinned distribution hashes; use existing repository Pygments 2.21.0; no security exemption", + "local_revalidation": "176 passed; kernel invariants 7 passed; whole kernel ruff scope passed" + }, + { + "kind": "followup_ci", + "revision": "e9719aa665046173f863218da72be9318540cc55", + "failures": [ + "verifier setuptools 83.0.0 rejected by GHSA-xmj3-9gf7-h52w", + "browser Goal drawer header geometry assertion" + ], + "resolution": [ + "verifier environment pins 84.1.0; standard project build isolation; fresh full 176 selection passed", + "UI assertion is already addressed by PR 7427; no duplicate UI change made; CI not claimed green" + ] + } + ], + "unverified": [ + "independent personnel review", + "third-party rerun", + "paid-model outcome", + "native continuity (six retained failures)", + "Windows SIGKILL qualification", + "power-loss durability", + "continuous fencing of arbitrary legacy/external tools", + "requirement edits during interruption as a separate revision-change experiment" + ], + "qualification_runs": { + "core_selection": { + "passed": 173, + "failed": 2, + "failure_kind": "invalid normal-control CLI test invocation; corrected in final matrix" + }, + "final_process_matrix": { + "passed": 16, + "failed": 0 + }, + "typescript_selection": { + "passed": 138, + "failed": 0 + }, + "delegated_lease_processes": { + "passed": 16, + "failed": 0, + "profile": "before formatter-only dependency correction; same production source bytes" + }, + "typecheck_control_plane": "passed", + "ruff_changed_paths": "passed", + "pip_check_clean_environment": "passed", + "hypothesis": { + "examples": 30, + "max_actions": 12, + "status": "passed", + "version": "6.142.4" + }, + "secure_dependency_selection": { + "passed": 176, + "failed": 0, + "includes": "core recovery selection, 16-case process matrix, state machine, source-lifetime and host tests, Stage 2C lock coverage" + }, + "kernel_static_invariants": { + "passed": 7, + "failed": 0 + }, + "kernel_ruff_scope": "passed", + "pygments": "2.21.0", + "verifier_environment_setuptools": "84.1.0", + "project_build_isolation": "uses unchanged pyproject.toml setuptools==83.0.0", + "patched_verifier_selection": { + "passed": 176, + "failed": 0, + "fresh_environment": true + } + }, + "submission_target": "71057d44f0d50a3636fcd8796f7c98717ce41742" +} diff --git a/tests/models/turn_recovery/requirements.txt b/tests/models/turn_recovery/requirements.txt new file mode 100644 index 0000000000..3b93611e8e --- /dev/null +++ b/tests/models/turn_recovery/requirements.txt @@ -0,0 +1,16 @@ +# Python 3.11 qualification dependencies; product runtime has no Python dependencies. +PyYAML==6.0.3 +Pygments==2.21.0 +attrs==25.4.0 +hypothesis==6.142.4 +iniconfig==2.3.0 +jsonschema==4.26.0 +jsonschema-specifications==2025.9.1 +packaging==26.0 +pluggy==1.6.0 +pytest==9.0.3 +referencing==0.37.0 +rpds-py==0.30.0 +setuptools==84.1.0 +sortedcontainers==2.4.0 +typing_extensions==4.15.0 diff --git a/tests/requirements-stage2c-linux-py311.txt b/tests/requirements-stage2c-linux-py311.txt index dc8bdea95e..9eb7883bea 100644 --- a/tests/requirements-stage2c-linux-py311.txt +++ b/tests/requirements-stage2c-linux-py311.txt @@ -16,6 +16,7 @@ attrs==26.1.0 \ --hash=sha256:c647aa4a12dfbad9333ca4e71fe62ddc36f4e63b2d260a37a8b83d2f043ac309 \ --hash=sha256:d03ceb89cb322a8fd706d4fb91940737b6642aa36998fe130a9bc96c985eff32 # via + # hypothesis # jsonschema # referencing build==1.6.0 \ @@ -510,6 +511,10 @@ httpx-sse==0.4.3 \ --hash=sha256:0ac1c9fe3c0afad2e0ebb25a934a59f4c7823b60792691f779fad2c5568830fc \ --hash=sha256:9b1ed0127459a66014aec3c56bebd93da3c1bc8bb6618c8082039a44889a755d # via mcp +hypothesis==6.142.4 \ + --hash=sha256:25eecc73fadecd8b491aed822204cfe4be9c98ff5c1e8e038d181136ffc54b5b \ + --hash=sha256:b3e71a84708994aa910ea47f1483ad892a7c390839959d689b2a2b07ebfd160e + # via loopx (pyproject.toml) idna==3.19 \ --hash=sha256:5e0811a4383b21dc5838069f801c4fb62113b7447663d2530d2bd6e77b49bf15 \ --hash=sha256:815e7be7a7806d54abb586dc943addc79e8b2ee16915059658cbeff4b1b43bf4 @@ -1425,6 +1430,10 @@ setuptools==83.0.0 \ --hash=sha256:025bccbbf0fa05b6192bc64ae1e7b16e001fd6d6d4d5de03c97b1c1ade523bef \ --hash=sha256:29b23c360f22f414dc7336bb39178cc7bcbf6021ed2733cde173f09dba19abb3 # via -r tests/requirements-stage2c-tools.in +sortedcontainers==2.4.0 \ + --hash=sha256:25caa5a06cc30b6b83d11423433f65d1f9d76c4c6a0c90e3379eaa43b9bfdb88 \ + --hash=sha256:a163dcaede0f1c021485e957a39245190e74249897e2ae4b2aa38595db237ee0 + # via hypothesis sse-starlette==3.4.11 \ --hash=sha256:1bae716c02f3e6f294be41ff333220692dae7c3cbab077c900f159676719dade \ --hash=sha256:c7b2244bdff016fe7f64e10075e89a3e6bbf899649cc89b0fe884b5545042453 diff --git a/tests/test_loopx_turn_executor.py b/tests/test_loopx_turn_executor.py index c6fd88ce84..ecb7edba57 100644 --- a/tests/test_loopx_turn_executor.py +++ b/tests/test_loopx_turn_executor.py @@ -1406,7 +1406,7 @@ def test_reserved_managed_start_recovers_after_death_before_the_attempt_record( journal_writes = turn_executor._write_journal def die_before_attempt_record(path: Path, journal: dict[str, object]) -> None: - if "host_attempt_count" in journal: + if journal.get("host_attempt_count", 0) > 0: raise RuntimeError("simulated process death before the attempt record") journal_writes(path, journal) @@ -1418,7 +1418,7 @@ def die_before_attempt_record(path: Path, journal: dict[str, object]) -> None: monkeypatch.setattr(turn_executor, "_write_journal", journal_writes) journal = _journal(runtime_root) - assert "host_attempt_count" not in journal + assert journal["host_attempt_count"] == 0 assert journal["admission"]["reserved"] is True assert [ (row["state"], row["request_id"]) for row in _cadence_starts(runtime_root)