From 121b481c598314e9a7e05070003f60efe566d301 Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Fri, 25 Sep 2026 08:10:24 +0000 Subject: [PATCH 1/5] test(durability): require every SQL crash boundary and stale-owner takeover to pass --- tests/durability/sql-run-context.test.ts | 70 +++++++++++-------- tests/helpers/durability/conformance-graph.ts | 4 +- tests/helpers/durability/sql-child.ts | 2 +- .../durability/sql-run-context-child.ts | 56 ++++++++++++--- 4 files changed, 89 insertions(+), 43 deletions(-) diff --git a/tests/durability/sql-run-context.test.ts b/tests/durability/sql-run-context.test.ts index af302e9e0..f6fe42ba8 100644 --- a/tests/durability/sql-run-context.test.ts +++ b/tests/durability/sql-run-context.test.ts @@ -7,7 +7,7 @@ import { fileURLToPath } from 'node:url' import { afterEach, describe, expect, it } from 'vitest' import { contentAddress } from '../../src/durable/content-address' import { createFencedSqlRunContext } from '../../src/runtime/supervise/sql-run-context' -import { EXPECTED_OUT } from '../helpers/durability/conformance-graph' +import { EXPECTED_OUT, RUN_ID, WORKER_NODES } from '../helpers/durability/conformance-graph' import { openSql } from '../helpers/durability/sql-adapter' const roots: string[] = [] @@ -108,13 +108,14 @@ function start(shared: string, cwd: string, mode = 'run', checkpoint = '') { rootMaterializations: number uniqueRootAttempts: boolean uniqueCursorSequences: boolean + workerRecords: Array<{ label: string; spawns: number; settlements: number }> } }, } } function assertComplete(report: Awaited['report']>>) { - expect(report.kind).toBe('winner') + expect(report.kind, JSON.stringify(report)).toBe('winner') expect(report.out).toEqual(EXPECTED_OUT) expect(report.roots).toBe(1) expect(report.rootMaterializations).toBe(1) @@ -124,6 +125,9 @@ function assertComplete(report: Awaited['rep expect(report.completed).toBe(3) expect(report.inDoubt).toBe(0) expect(report.escalations).toEqual([]) + expect(report.workerRecords).toEqual( + WORKER_NODES.map((label) => ({ label, spawns: 1, settlements: 1 })), + ) expect(report.effects).toEqual(['artifact:conductor:final']) expect(report.creations).toBe(3) expect(report.dispatches).toBe(3) @@ -196,35 +200,6 @@ describe('SQL run context', () => { expect(await readdir(host)).toEqual([]) }, 45000) - // The never-dispatched window (spawned+input committed, no admission, no dispatch) is a KNOWN - // GAP against current main: the retained executor's recovery classifies a never-dispatched - // session `pending: unobservable` and the workers settle down (the branch this suite was - // written against, #1368-era main, started them fresh). Every checkpoint before the grouped - // spawn record and from provider create/dispatch onward — the lost-ack windows that motivated - // the salvage — passes below. - it.fails.each([ - 'sql:spawned-child:after', - 'sql:execution-input:after', - 'sql:execution-admitted-intent:before', - 'sql:execution-admitted-intent:after', - 'sql:execution-admitted-environment:before', - 'sql:execution-admitted-environment:after', - ])( - 'KNOWN GAP (never-dispatched window): SIGKILL at %s', - async (checkpoint) => { - const shared = await directory() - const firstHost = await directory() - const secondHost = await directory() - const first = start(shared, firstHost, 'kill', checkpoint) - await first.message('checkpoint') - expect(await first.exit).toEqual({ code: null, signal: 'SIGKILL' }) - await rm(firstHost, { recursive: true, force: true }) - assertComplete(await start(shared, secondHost).report()) - expect(await readdir(secondHost)).toEqual([]) - }, - 60000, - ) - // Before/after are on opposite sides of the actual SQLite statement commit. In the after // cases the adapter NEVER acknowledges the write to Runtime: the process dies first. it.each([ @@ -234,6 +209,12 @@ describe('SQL run context', () => { 'sql:spawned-root:after', 'sql:spawned-child:before', 'sql:execution-input:before', + 'sql:spawned-child:after', + 'sql:execution-input:after', + 'sql:execution-admitted-intent:before', + 'sql:execution-admitted-intent:after', + 'sql:execution-admitted-environment:before', + 'sql:execution-admitted-environment:after', 'provider:create:before', 'provider:create:after', 'provider:dispatch:before', @@ -281,4 +262,31 @@ describe('SQL run context', () => { contender.child.send('retry') assertComplete(await contender.report()) }, 60000) + it('fences a SIGSTOP/SIGCONT owner without releasing its live successor', async () => { + const shared = await directory() + const owner = start(shared, await directory(), 'lease') + await owner.message('owned') + owner.child.kill('SIGSTOP') + const successor = start(shared, await directory(), 'lease') + await successor.message('owned') + owner.child.kill('SIGCONT') + owner.child.send('write') + expect(String((await owner.message('fenced')).error)).toMatch(/lease|ownership/i) + expect(await owner.exit).toEqual({ code: 0, signal: null }) + + const db = openSql(join(shared, 'run-context.sqlite')) + try { + const observer = await createFencedSqlRunContext(db.adapter, RUN_ID, leaseOptions) + await expect(observer.acquire()).rejects.toThrow(/owned|lease|busy/i) + successor.child.send('write') + await successor.message('written') + expect(await successor.exit).toEqual({ code: 0, signal: null }) + expect(await observer.blobs.get(contentAddress({ pid: owner.child.pid }))).toBeUndefined() + expect(await observer.blobs.get(contentAddress({ pid: successor.child.pid }))).toEqual({ + pid: successor.child.pid, + }) + } finally { + db.database.close() + } + }, 60000) }) diff --git a/tests/helpers/durability/conformance-graph.ts b/tests/helpers/durability/conformance-graph.ts index e533c7910..ee0d08eaf 100644 --- a/tests/helpers/durability/conformance-graph.ts +++ b/tests/helpers/durability/conformance-graph.ts @@ -230,7 +230,7 @@ export class ConductorPlanner { private drainedIdle = false turns = 0 - constructor() { + constructor(private readonly options: { forbidInDoubt?: boolean } = {}) { this.nodes = WORKER_NODES.map((node) => ({ node, key: workerKey(node), status: 'pending' })) } @@ -282,6 +282,8 @@ export class ConductorPlanner { const error = typeof r.error === 'string' ? r.error : undefined if (error !== undefined) { if (error.includes('in-doubt')) { + if (this.options.forbidInDoubt) + throw new Error(`conformance forbids replacing in-doubt spawn ${node} (${state.key})`) if (state.escalated) { this.fatal ??= `spawn of ${node} refused in-doubt twice (${state.key})` return diff --git a/tests/helpers/durability/sql-child.ts b/tests/helpers/durability/sql-child.ts index 8b33ef14b..d4480864b 100644 --- a/tests/helpers/durability/sql-child.ts +++ b/tests/helpers/durability/sql-child.ts @@ -48,7 +48,7 @@ const kill = armKillSwitch({ killedFile: `${dir}/killed.log`, }) const site = openSideEffectSite(dir) -const planner = new ConductorPlanner() +const planner = new ConductorPlanner({ forbidInDoubt: true }) let brainCalls = 0 const brain = async (messages: ReadonlyArray>) => { brainCalls += 1 diff --git a/tests/helpers/durability/sql-run-context-child.ts b/tests/helpers/durability/sql-run-context-child.ts index 9a4a720a5..65ab1061f 100644 --- a/tests/helpers/durability/sql-run-context-child.ts +++ b/tests/helpers/durability/sql-run-context-child.ts @@ -5,10 +5,11 @@ import type { AgentEnvironment, AgentEnvironmentProvider, } from '@tangle-network/agent-interface/environment-provider' +import { contentAddress } from '../../../src/durable/content-address' import { runGraph } from '../../../src/runtime/supervise/graph' import { createFencedSqlRunContext } from '../../../src/runtime/supervise/sql-run-context' import { durableRetainedProvider } from '../durable-retained-provider' -import { ConductorPlanner, conformanceGraph, RUN_ID } from './conformance-graph' +import { ConductorPlanner, conformanceGraph, RUN_ID, WORKER_NODES } from './conformance-graph' import { openSideEffectSite, SIDE_EFFECT_TOOL_NAME, sideEffectToolSpec } from './side-effect' import { openSql, type SqlBoundary } from './sql-adapter' @@ -130,7 +131,7 @@ async function phase() { leaseMs: 400, heartbeatMs: 60, }) - const planner = new ConductorPlanner() + const planner = new ConductorPlanner({ forbidInDoubt: true }) const site = openSideEffectSite(shared!) const result = await runGraph(conformanceGraph(), { runId: RUN_ID, @@ -162,7 +163,9 @@ async function phase() { const attempts = events.flatMap((event) => event.kind === 'execution-bound' && event.id === RUN_ID ? [event.binding.attemptId] : [], ) - const cursors = events.filter((event) => event.kind === 'settled').map((event) => event.seq) + const cursors = events + .filter((event) => event.kind === 'settled' || event.kind === 'cancelled') + .map((event) => event.seq) const report = { kind: result.result.kind, out: result.result.kind === 'winner' ? result.result.out : result.result, @@ -175,6 +178,19 @@ async function phase() { workers: workers.length, completed: workers.filter((event) => completed.has(event.id)).length, inDoubt: workers.filter((event) => !completed.has(event.id)).length, + workerRecords: WORKER_NODES.map((label) => { + const spawns = workers.filter((event) => event.label === label) + return { + label, + spawns: spawns.length, + settlements: events.filter( + (event) => + event.kind === 'settled' && + event.status === 'done' && + spawns.some((spawn) => spawn.id === event.id), + ).length, + } + }), planner: planner.report(), escalations: planner.report().escalations, effects: site.committedKeys(), @@ -187,17 +203,37 @@ async function phase() { } try { - if (mode === 'contend') { + if (mode === 'lease') { + const context = await createFencedSqlRunContext(store.adapter, RUN_ID, { + leaseMs: 400, + heartbeatMs: 60, + }) + const lease = await context.acquire() + const write = new Promise((resolve) => process.once('message', () => resolve())) + process.send!({ type: 'owned' }) + await write try { - await phase() - throw new Error('contender unexpectedly acquired the live run') + const value = { pid: process.pid } + await lease.context.blobs.put(contentAddress(value), value) + process.send!({ type: 'written' }) } catch (error) { - if (!/owned|lease|busy/i.test(String(error))) throw error - process.send!({ type: 'denied', error: String(error) }) + process.send!({ type: 'fenced', error: String(error) }) + } finally { + await lease.release() } - await new Promise((resolve) => process.once('message', () => resolve())) + } else { + if (mode === 'contend') { + try { + await phase() + throw new Error('contender unexpectedly acquired the live run') + } catch (error) { + if (!/owned|lease|busy/i.test(String(error))) throw error + process.send!({ type: 'denied', error: String(error) }) + } + await new Promise((resolve) => process.once('message', () => resolve())) + } + await phase() } - await phase() store.database.close() physical.close() process.disconnect?.() From 6556ebb027fe91b740bb7fddcce0381bbaf3a2d9 Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Fri, 25 Sep 2026 08:26:11 +0000 Subject: [PATCH 2/5] fix(durability): replay never-admitted SQL workers under their original keys --- CHANGELOG.md | 11 + conformance/durability/STATUS.md | 56 +-- conformance/durability/results.json | 361 +++++++++--------- docs/agent-managed-compute/README.md | 13 +- docs/agent-managed-compute/reliability.md | 8 +- docs/api/primitive-catalog.md | 2 +- docs/canonical-api.md | 2 +- package.json | 2 +- scripts/run-durability-conformance.mjs | 2 + src/runtime/environment-provider.ts | 8 +- .../fixtures/agent-improvement-proposal.json | 10 +- .../agent-profile-improvement-proposal.json | 6 +- 12 files changed, 260 insertions(+), 221 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bd5c5eedc..4df775c17 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,16 @@ # Changelog +## 0.272.1 + +SQL run-context synthesis preserves the incumbent stores and file path from #1381 and the +fenced ownership/coordination context from #1391. Retained recovery now starts a journaled +worker with no admission using its original execution keys, while any existing admission +still requires its original validated intent. The six formerly expected-failure crash cases +are ordinary passing tests. The SQL fixtures refuse in-doubt replacement keys and assert one +original spawn and one done settlement per worker. A real SIGSTOP/SIGCONT test proves that +a stale owner cannot publish or release its live successor. The conformance command now exits +nonzero when its test report fails, while still writing the evidence. + ## 0.272.0 A Tangle box's harness-transcript capture now carries each harness subagent session its sidecar names. diff --git a/conformance/durability/STATUS.md b/conformance/durability/STATUS.md index 89d5cde06..b80e3f675 100644 --- a/conformance/durability/STATUS.md +++ b/conformance/durability/STATUS.md @@ -1,8 +1,8 @@ # Durability conformance STATUS Question: **is agent-runtime durable?** — kill-and-resume conformance for `runGraph` and the -shipped journal backends, measured 2026-09-24 on `fix/interrupted-keyed-spawns` -(origin/main `12ffa98e`, agent-runtime 0.267.0 + the fixes in this PR). +shipped journal backends, measured 2026-09-25 on `pro/sql-run-context/synthesis` +(base main `66d2caa4`, agent-runtime 0.272.1). Regenerate the evidence: `pnpm run conformance:durability` (latest machine-readable results: [`results.json`](./results.json)). @@ -17,15 +17,20 @@ Regenerate the evidence: `pnpm run conformance:durability` | `runConversation` (6 turns, 2 participants, keyed per-turn effect) | `FileConversationJournal` | 18 kill points (turn start / backend-done-before-commit / turn-committed) + reference + halt-replay | **PASS** | | `runConversation` | `SqlConversationJournal` over real sqlite (`node:sqlite`) | same 18 + reference | **PASS** | | `runGraph` | `FileConversationJournal` / `SqlConversationJournal` | — | **N/A — capability gap**: `runGraph`'s durable layer is the `SpawnJournal` family; `ConversationJournal` is a different interface on a different subsystem. A graph run cannot take these backends. | -| `runGraph` | any SQL store | — | **journal + blobs: CLOSED** (`SqlSpawnJournal`/`SqlResultBlobStore`, full 23-point matrix green). Still open on SQL: the coordination side-log (file-based behind `runDir`) and machine-visible cross-run ownership | +| `runGraph` | `createFencedSqlRunContext` over real SQLite plus an independently retained provider | 26 kill boundaries, reference, SIGKILL takeover, SIGSTOP/SIGCONT fencing, and store invariants | **PASS** — SQL-only orchestration persistence, one live owner, same contender takes over, stale writes/releases rejected, original keys only, one physical create/dispatch per worker and one keyed effect | -Totals from the run: **158/158 green** — the inline matrix (24), the session re-attach matrix (24), -the SQL matrix (24), the conversation matrices (39), the runDir regression locks (2), the known-defect cases (5), the inotify invariant (1), and the -fenced-SQL salvage from #1373 (39: the fenced store + context unit suites, 19 lost-ack kill -points across TWO hosts, lease takeover/contention, plus 6 expected-fail cases that document -the never-dispatched recovery gap against current retained machinery). The inline matrix was also verified green against 0.255.0-era main before the 0.262–0.265 series -landed; the only matrix-visible effect of that series here is the new `open-work` submission gate, -which the suite's driver now drains correctly. +Totals from the synthesis run: **159/159 green, zero expected failures**. The inline matrix (24), +session re-attach matrix (24), incumbent SQL matrix (24), conversation matrices (39), runDir +regression locks (2), known-defect cases (5), and inotify invariant (1) remain intact. Fenced SQL +adds 40 cases: 32 context/real-process cases, 5 store cases, and 3 projection/coordination cases. + +The six never-dispatched cases fail on `2fe7f063` when made ordinary tests, then pass with the +two retained-replay guards restored from p1. SQL fixtures reject the in-doubt escape hatch, +assert no replacement keys, and require one original spawn and one done settlement per worker. +The unchanged store overhead test publishes 100 records using 200 writes and zero history reads. +These are real separate OS processes with independent empty host directories, sharing SQLite and +a durable fake provider. They do not claim a deployed multi-machine database or live-provider test. +Exactly-once external effects require retained keyed create/dispatch and an idempotent effect site. ### Baseline: the same suite against the commit before the first fix @@ -41,7 +46,7 @@ measures what the durability work actually changed: - **24 cases un-runnable**: the session re-attach matrix needs `recoverExecutor` (#1375), which that base refuses as an unknown option — the capability did not exist. -Current main: **94/94 green**. That delta — 52 cases' worth of surface from red-or-absent to +Historical post-#1375 baseline: **94/94 green**. That delta — 52 cases' worth of surface from red-or-absent to proven — is the measured result of the durability work (#1368 + #1375). ## Defect found and fixed by this suite @@ -95,7 +100,9 @@ re-attach. Both arms of the correct behavior are now implemented and pinned by t across the kill. Previously this channel existed only for backend-derived recursive managers; a caller-owned `makeLeafAgent`/`makeWorkerAgent` had no way to recover anything. -## Decision read-out (harden vs. adopt Temporal/Restate/Cloudflare Workflows) +## Historical decision read-out (before #1381 and #1391) + +This section records the original file-only comparison, not the current capability verdict above. What the runtime has today, and what this suite proves: **single-process-kill durability with exactly-once keyed effects and a clean resume contract works on the shipped backends** — for the @@ -177,24 +184,21 @@ The superseded #1373 branch's unique work — ported onto main and proven here: - **`createFencedSqlRunContext`**: the cross-machine run context over it — SQL journal + blobs + COORDINATION SIDE-LOG (the other open item), read-only until a lease is acquired; `supervise`/`runGraph` accept `runContext` and hold ownership for the whole run. -- **Evidence (39 cases)**: the fenced store suite (5), the context stores suite (3), and the - two-host conformance suite — 19 lost-ack kill points where the process dies between a SQLite - statement's commit and its acknowledgement, on either side of every publication, and resumes - ON ANOTHER HOST with an empty working directory (no replacement keys, exactly-once provider - effects), plus lease takeover (a rejected contender, then takeover after the owner's SIGKILL) - and publish-contention tests. -- **Known gap, documented as expected-fail (6)**: kills in the never-dispatched window - (spawned+input committed, no admission, no dispatch) — current retained machinery classifies - the recovered never-dispatched session `pending: unobservable` where #1368-era main started it - fresh. Follow-up named in the suite header; every checkpoint before the grouped spawn record - and from provider create/dispatch onward passes. - -## Verdict (FINAL): keep and harden the own journal layer +- **Synthesis evidence (40 cases)**: all 26 kill boundaries are ordinary passing tests, including + publication-before/after, provider create/dispatch-before/after, and tool effect-before/after. + One live contender is refused and the same process takes over after SIGKILL. A stopped owner + that later resumes cannot publish or release its successor. The existing 5 fenced-store and + 3 context-store invariants remain unchanged. +- **Never-dispatched gap CLOSED**: a journaled execution with no admission may start with its + original keys. Any existing admission still requires the original validated intent. No + replacement-key or in-doubt fallback is allowed by either SQL fixture. + +## Historical pre-#1391 verdict: keep and harden the own journal layer **Recommendation: keep the shipped journal layer and keep hardening it. Do not adopt Temporal/Restate/Cloudflare Workflows for the current product shape.** -What is proven, as of this writing — **119/119 green** (`pnpm run conformance:durability`; +What was proven before the fenced context landed — **119/119 green** (`pnpm run conformance:durability`; machine record: `results.json`): - **Kill-and-resume works on every shipped durable backend, file AND SQL**: the file run context diff --git a/conformance/durability/results.json b/conformance/durability/results.json index 44c11ca17..89d684ac1 100644 --- a/conformance/durability/results.json +++ b/conformance/durability/results.json @@ -1,6 +1,6 @@ { "schema": "agent-runtime/durability-conformance-results/v1", - "generatedAt": "2026-09-25T06:28:46.243Z", + "generatedAt": "2026-09-25T08:57:53.619Z", "files": [ "tests/durability/graph-rundir-journal.test.ts", "tests/durability/graph-kill-resume.test.ts", @@ -14,8 +14,8 @@ "tests/durability/inotify-invariant.test.ts" ], "totals": { - "cases": 158, - "passed": 158, + "cases": 159, + "passed": 159, "failed": 0, "expectedFailed": 0 }, @@ -25,1106 +25,1113 @@ "suite": "runConversation kill-and-resume conformance (File + Sql journals)", "title": "reference run (file backend) completes with the expected transcript", "status": "passed", - "durationMs": 2.5527450000001863 + "durationMs": 3.621135999999751 }, { "file": "tests/durability/conversation-kill-resume.test.ts", "suite": "runConversation kill-and-resume conformance (File + Sql journals)", "title": "reference run (sqlite backend) completes with the expected transcript", "status": "passed", - "durationMs": 0.5556129999999939 + "durationMs": 1.0204030000004423 }, { "file": "tests/durability/conversation-kill-resume.test.ts", "suite": "runConversation kill-and-resume conformance (File + Sql journals)", "title": "file journal: SIGKILL at turn:0:before → resume completes, nothing lost or repeated", "status": "passed", - "durationMs": 877.2138590000002 + "durationMs": 1493.779301 }, { "file": "tests/durability/conversation-kill-resume.test.ts", "suite": "runConversation kill-and-resume conformance (File + Sql journals)", "title": "file journal: SIGKILL at turn:0:after-backend → resume completes, nothing lost or repeated", "status": "passed", - "durationMs": 854.8215799999998 + "durationMs": 1773.7731870000002 }, { "file": "tests/durability/conversation-kill-resume.test.ts", "suite": "runConversation kill-and-resume conformance (File + Sql journals)", "title": "file journal: SIGKILL at turn:0:committed → resume completes, nothing lost or repeated", "status": "passed", - "durationMs": 892.6718089999999 + "durationMs": 1528.5773630000003 }, { "file": "tests/durability/conversation-kill-resume.test.ts", "suite": "runConversation kill-and-resume conformance (File + Sql journals)", "title": "file journal: SIGKILL at turn:1:before → resume completes, nothing lost or repeated", "status": "passed", - "durationMs": 830.981162 + "durationMs": 1708.6014749999995 }, { "file": "tests/durability/conversation-kill-resume.test.ts", "suite": "runConversation kill-and-resume conformance (File + Sql journals)", "title": "file journal: SIGKILL at turn:1:after-backend → resume completes, nothing lost or repeated", "status": "passed", - "durationMs": 890.6666229999992 + "durationMs": 1643.0739599999997 }, { "file": "tests/durability/conversation-kill-resume.test.ts", "suite": "runConversation kill-and-resume conformance (File + Sql journals)", "title": "file journal: SIGKILL at turn:1:committed → resume completes, nothing lost or repeated", "status": "passed", - "durationMs": 895.3060270000005 + "durationMs": 1591.0655719999995 }, { "file": "tests/durability/conversation-kill-resume.test.ts", "suite": "runConversation kill-and-resume conformance (File + Sql journals)", "title": "file journal: SIGKILL at turn:2:before → resume completes, nothing lost or repeated", "status": "passed", - "durationMs": 925.3653089999998 + "durationMs": 1655.0336960000004 }, { "file": "tests/durability/conversation-kill-resume.test.ts", "suite": "runConversation kill-and-resume conformance (File + Sql journals)", "title": "file journal: SIGKILL at turn:2:after-backend → resume completes, nothing lost or repeated", "status": "passed", - "durationMs": 968.0956930000002 + "durationMs": 1674.884845999999 }, { "file": "tests/durability/conversation-kill-resume.test.ts", "suite": "runConversation kill-and-resume conformance (File + Sql journals)", "title": "file journal: SIGKILL at turn:2:committed → resume completes, nothing lost or repeated", "status": "passed", - "durationMs": 940.3454789999996 + "durationMs": 1686.3670139999995 }, { "file": "tests/durability/conversation-kill-resume.test.ts", "suite": "runConversation kill-and-resume conformance (File + Sql journals)", "title": "file journal: SIGKILL at turn:3:before → resume completes, nothing lost or repeated", "status": "passed", - "durationMs": 849.0068200000005 + "durationMs": 1695.9485539999987 }, { "file": "tests/durability/conversation-kill-resume.test.ts", "suite": "runConversation kill-and-resume conformance (File + Sql journals)", "title": "file journal: SIGKILL at turn:3:after-backend → resume completes, nothing lost or repeated", "status": "passed", - "durationMs": 857.4690279999995 + "durationMs": 1590.241119000002 }, { "file": "tests/durability/conversation-kill-resume.test.ts", "suite": "runConversation kill-and-resume conformance (File + Sql journals)", "title": "file journal: SIGKILL at turn:3:committed → resume completes, nothing lost or repeated", "status": "passed", - "durationMs": 854.9235779999999 + "durationMs": 1673.7445650000009 }, { "file": "tests/durability/conversation-kill-resume.test.ts", "suite": "runConversation kill-and-resume conformance (File + Sql journals)", "title": "file journal: SIGKILL at turn:4:before → resume completes, nothing lost or repeated", "status": "passed", - "durationMs": 845.3971959999999 + "durationMs": 1611.6675020000002 }, { "file": "tests/durability/conversation-kill-resume.test.ts", "suite": "runConversation kill-and-resume conformance (File + Sql journals)", "title": "file journal: SIGKILL at turn:4:after-backend → resume completes, nothing lost or repeated", "status": "passed", - "durationMs": 844.935888 + "durationMs": 1721.3841109999994 }, { "file": "tests/durability/conversation-kill-resume.test.ts", "suite": "runConversation kill-and-resume conformance (File + Sql journals)", "title": "file journal: SIGKILL at turn:4:committed → resume completes, nothing lost or repeated", "status": "passed", - "durationMs": 858.9795050000012 + "durationMs": 1557.433723000002 }, { "file": "tests/durability/conversation-kill-resume.test.ts", "suite": "runConversation kill-and-resume conformance (File + Sql journals)", "title": "file journal: SIGKILL at turn:5:before → resume completes, nothing lost or repeated", "status": "passed", - "durationMs": 865.5256509999999 + "durationMs": 1741.518451 }, { "file": "tests/durability/conversation-kill-resume.test.ts", "suite": "runConversation kill-and-resume conformance (File + Sql journals)", "title": "file journal: SIGKILL at turn:5:after-backend → resume completes, nothing lost or repeated", "status": "passed", - "durationMs": 888.2923179999998 + "durationMs": 1455.1751299999996 }, { "file": "tests/durability/conversation-kill-resume.test.ts", "suite": "runConversation kill-and-resume conformance (File + Sql journals)", "title": "file journal: SIGKILL at turn:5:committed → resume completes, nothing lost or repeated", "status": "passed", - "durationMs": 890.2758240000003 + "durationMs": 1626.6998139999996 }, { "file": "tests/durability/conversation-kill-resume.test.ts", "suite": "runConversation kill-and-resume conformance (File + Sql journals)", "title": "sqlite journal: SIGKILL at turn:0:before → resume completes, nothing lost or repeated", "status": "passed", - "durationMs": 1150.3397210000003 + "durationMs": 1579.998865999998 }, { "file": "tests/durability/conversation-kill-resume.test.ts", "suite": "runConversation kill-and-resume conformance (File + Sql journals)", "title": "sqlite journal: SIGKILL at turn:0:after-backend → resume completes, nothing lost or repeated", "status": "passed", - "durationMs": 1147.2300400000022 + "durationMs": 1624.8155409999963 }, { "file": "tests/durability/conversation-kill-resume.test.ts", "suite": "runConversation kill-and-resume conformance (File + Sql journals)", "title": "sqlite journal: SIGKILL at turn:0:committed → resume completes, nothing lost or repeated", "status": "passed", - "durationMs": 1078.8859159999993 + "durationMs": 1475.1594160000022 }, { "file": "tests/durability/conversation-kill-resume.test.ts", "suite": "runConversation kill-and-resume conformance (File + Sql journals)", "title": "sqlite journal: SIGKILL at turn:1:before → resume completes, nothing lost or repeated", "status": "passed", - "durationMs": 1327.1785039999995 + "durationMs": 1562.2064949999985 }, { "file": "tests/durability/conversation-kill-resume.test.ts", "suite": "runConversation kill-and-resume conformance (File + Sql journals)", "title": "sqlite journal: SIGKILL at turn:1:after-backend → resume completes, nothing lost or repeated", "status": "passed", - "durationMs": 1100.6322069999987 + "durationMs": 1549.213638000001 }, { "file": "tests/durability/conversation-kill-resume.test.ts", "suite": "runConversation kill-and-resume conformance (File + Sql journals)", "title": "sqlite journal: SIGKILL at turn:1:committed → resume completes, nothing lost or repeated", "status": "passed", - "durationMs": 1037.4012770000008 + "durationMs": 1578.5015839999978 }, { "file": "tests/durability/conversation-kill-resume.test.ts", "suite": "runConversation kill-and-resume conformance (File + Sql journals)", "title": "sqlite journal: SIGKILL at turn:2:before → resume completes, nothing lost or repeated", "status": "passed", - "durationMs": 1048.5581200000015 + "durationMs": 1522.9602149999992 }, { "file": "tests/durability/conversation-kill-resume.test.ts", "suite": "runConversation kill-and-resume conformance (File + Sql journals)", "title": "sqlite journal: SIGKILL at turn:2:after-backend → resume completes, nothing lost or repeated", "status": "passed", - "durationMs": 1018.384012999999 + "durationMs": 1509.474049000004 }, { "file": "tests/durability/conversation-kill-resume.test.ts", "suite": "runConversation kill-and-resume conformance (File + Sql journals)", "title": "sqlite journal: SIGKILL at turn:2:committed → resume completes, nothing lost or repeated", "status": "passed", - "durationMs": 1045.467869 + "durationMs": 1571.1267389999994 }, { "file": "tests/durability/conversation-kill-resume.test.ts", "suite": "runConversation kill-and-resume conformance (File + Sql journals)", "title": "sqlite journal: SIGKILL at turn:3:before → resume completes, nothing lost or repeated", "status": "passed", - "durationMs": 1164.5261079999982 + "durationMs": 1530.581588000001 }, { "file": "tests/durability/conversation-kill-resume.test.ts", "suite": "runConversation kill-and-resume conformance (File + Sql journals)", "title": "sqlite journal: SIGKILL at turn:3:after-backend → resume completes, nothing lost or repeated", "status": "passed", - "durationMs": 1190.187374000001 + "durationMs": 1614.362341 }, { "file": "tests/durability/conversation-kill-resume.test.ts", "suite": "runConversation kill-and-resume conformance (File + Sql journals)", "title": "sqlite journal: SIGKILL at turn:3:committed → resume completes, nothing lost or repeated", "status": "passed", - "durationMs": 1048.0039849999994 + "durationMs": 1540.8402510000014 }, { "file": "tests/durability/conversation-kill-resume.test.ts", "suite": "runConversation kill-and-resume conformance (File + Sql journals)", "title": "sqlite journal: SIGKILL at turn:4:before → resume completes, nothing lost or repeated", "status": "passed", - "durationMs": 1085.9364660000028 + "durationMs": 1638.2012130000003 }, { "file": "tests/durability/conversation-kill-resume.test.ts", "suite": "runConversation kill-and-resume conformance (File + Sql journals)", "title": "sqlite journal: SIGKILL at turn:4:after-backend → resume completes, nothing lost or repeated", "status": "passed", - "durationMs": 1183.0530040000012 + "durationMs": 1525.303168999999 }, { "file": "tests/durability/conversation-kill-resume.test.ts", "suite": "runConversation kill-and-resume conformance (File + Sql journals)", "title": "sqlite journal: SIGKILL at turn:4:committed → resume completes, nothing lost or repeated", "status": "passed", - "durationMs": 1178.0126630000013 + "durationMs": 1693.5401039999997 }, { "file": "tests/durability/conversation-kill-resume.test.ts", "suite": "runConversation kill-and-resume conformance (File + Sql journals)", "title": "sqlite journal: SIGKILL at turn:5:before → resume completes, nothing lost or repeated", "status": "passed", - "durationMs": 1115.490979000002 + "durationMs": 1502.8769259999972 }, { "file": "tests/durability/conversation-kill-resume.test.ts", "suite": "runConversation kill-and-resume conformance (File + Sql journals)", "title": "sqlite journal: SIGKILL at turn:5:after-backend → resume completes, nothing lost or repeated", "status": "passed", - "durationMs": 1084.664948000005 + "durationMs": 1598.8018520000041 }, { "file": "tests/durability/conversation-kill-resume.test.ts", "suite": "runConversation kill-and-resume conformance (File + Sql journals)", "title": "sqlite journal: SIGKILL at turn:5:committed → resume completes, nothing lost or repeated", "status": "passed", - "durationMs": 1232.0937649999978 + "durationMs": 1487.8922690000036 }, { "file": "tests/durability/conversation-kill-resume.test.ts", "suite": "runConversation kill-and-resume conformance (File + Sql journals)", "title": "a journal that already recorded a halt replays its final state instead of re-running", "status": "passed", - "durationMs": 927.1061310000005 + "durationMs": 1610.3364910000018 }, { "file": "tests/durability/graph-kill-resume.test.ts", "suite": "runGraph kill-and-resume conformance (file run context)", "title": "the reference run completes with the expected outcome and covers boundaries and mid-step", "status": "passed", - "durationMs": 2.2538669999999 + "durationMs": 6.910539999999855 }, { "file": "tests/durability/graph-kill-resume.test.ts", "suite": "runGraph kill-and-resume conformance (file run context)", "title": "SIGKILL at driver:turn:1:before → resume completes, loses nothing, repeats no committed step", "status": "passed", - "durationMs": 1699.4300369999996 + "durationMs": 2804.5962719999998 }, { "file": "tests/durability/graph-kill-resume.test.ts", "suite": "runGraph kill-and-resume conformance (file run context)", "title": "SIGKILL at driver:turn:1:after → resume completes, loses nothing, repeats no committed step", "status": "passed", - "durationMs": 1699.927409 + "durationMs": 2694.1040729999995 }, { "file": "tests/durability/graph-kill-resume.test.ts", "suite": "runGraph kill-and-resume conformance (file run context)", "title": "SIGKILL at driver:turn:2:before → resume completes, loses nothing, repeats no committed step", "status": "passed", - "durationMs": 1764.730356 + "durationMs": 2700.388185 }, { "file": "tests/durability/graph-kill-resume.test.ts", "suite": "runGraph kill-and-resume conformance (file run context)", "title": "SIGKILL at driver:turn:2:after → resume completes, loses nothing, repeats no committed step", "status": "passed", - "durationMs": 1798.0084269999998 + "durationMs": 2699.413106 }, { "file": "tests/durability/graph-kill-resume.test.ts", "suite": "runGraph kill-and-resume conformance (file run context)", "title": "SIGKILL at worker:surveyor:before → resume completes, loses nothing, repeats no committed step", "status": "passed", - "durationMs": 1887.5632179999993 + "durationMs": 2843.584708000002 }, { "file": "tests/durability/graph-kill-resume.test.ts", "suite": "runGraph kill-and-resume conformance (file run context)", "title": "SIGKILL at worker:builder:before → resume completes, loses nothing, repeats no committed step", "status": "passed", - "durationMs": 1851.6879410000001 + "durationMs": 2861.0747379999993 }, { "file": "tests/durability/graph-kill-resume.test.ts", "suite": "runGraph kill-and-resume conformance (file run context)", "title": "SIGKILL at worker:verifier:before → resume completes, loses nothing, repeats no committed step", "status": "passed", - "durationMs": 1724.723575 + "durationMs": 2803.3201960000006 }, { "file": "tests/durability/graph-kill-resume.test.ts", "suite": "runGraph kill-and-resume conformance (file run context)", "title": "SIGKILL at worker:surveyor:mid → resume completes, loses nothing, repeats no committed step", "status": "passed", - "durationMs": 1988.5608520000005 + "durationMs": 2837.2459420000014 }, { "file": "tests/durability/graph-kill-resume.test.ts", "suite": "runGraph kill-and-resume conformance (file run context)", "title": "SIGKILL at worker:builder:mid → resume completes, loses nothing, repeats no committed step", "status": "passed", - "durationMs": 2142.985192 + "durationMs": 2736.2014740000013 }, { "file": "tests/durability/graph-kill-resume.test.ts", "suite": "runGraph kill-and-resume conformance (file run context)", "title": "SIGKILL at worker:verifier:mid → resume completes, loses nothing, repeats no committed step", "status": "passed", - "durationMs": 2077.3067560000018 + "durationMs": 2756.1994559999985 }, { "file": "tests/durability/graph-kill-resume.test.ts", "suite": "runGraph kill-and-resume conformance (file run context)", "title": "SIGKILL at worker:surveyor:after → resume completes, loses nothing, repeats no committed step", "status": "passed", - "durationMs": 1925.7682050000003 + "durationMs": 2853.0135900000023 }, { "file": "tests/durability/graph-kill-resume.test.ts", "suite": "runGraph kill-and-resume conformance (file run context)", "title": "SIGKILL at worker:builder:after → resume completes, loses nothing, repeats no committed step", "status": "passed", - "durationMs": 1929.3499699999993 + "durationMs": 2905.188227999999 }, { "file": "tests/durability/graph-kill-resume.test.ts", "suite": "runGraph kill-and-resume conformance (file run context)", "title": "SIGKILL at worker:verifier:after → resume completes, loses nothing, repeats no committed step", "status": "passed", - "durationMs": 1778.9381350000003 + "durationMs": 2748.6994049999994 }, { "file": "tests/durability/graph-kill-resume.test.ts", "suite": "runGraph kill-and-resume conformance (file run context)", "title": "SIGKILL at driver:turn:3:before → resume completes, loses nothing, repeats no committed step", "status": "passed", - "durationMs": 2192.1925059999994 + "durationMs": 2697.072106000007 }, { "file": "tests/durability/graph-kill-resume.test.ts", "suite": "runGraph kill-and-resume conformance (file run context)", "title": "SIGKILL at driver:turn:3:after → resume completes, loses nothing, repeats no committed step", "status": "passed", - "durationMs": 1869.8160759999992 + "durationMs": 2787.623823999995 }, { "file": "tests/durability/graph-kill-resume.test.ts", "suite": "runGraph kill-and-resume conformance (file run context)", "title": "SIGKILL at driver:turn:4:before → resume completes, loses nothing, repeats no committed step", "status": "passed", - "durationMs": 1972.6202820000035 + "durationMs": 2714.2471300000034 }, { "file": "tests/durability/graph-kill-resume.test.ts", "suite": "runGraph kill-and-resume conformance (file run context)", "title": "SIGKILL at driver:turn:4:after → resume completes, loses nothing, repeats no committed step", "status": "passed", - "durationMs": 1965.4206730000005 + "durationMs": 2598.443944999999 }, { "file": "tests/durability/graph-kill-resume.test.ts", "suite": "runGraph kill-and-resume conformance (file run context)", "title": "SIGKILL at driver:turn:5:before → resume completes, loses nothing, repeats no committed step", "status": "passed", - "durationMs": 1828.1210589999973 + "durationMs": 2665.6174730000057 }, { "file": "tests/durability/graph-kill-resume.test.ts", "suite": "runGraph kill-and-resume conformance (file run context)", "title": "SIGKILL at driver:turn:5:after → resume completes, loses nothing, repeats no committed step", "status": "passed", - "durationMs": 1983.5734230000016 + "durationMs": 2577.9718060000014 }, { "file": "tests/durability/graph-kill-resume.test.ts", "suite": "runGraph kill-and-resume conformance (file run context)", "title": "SIGKILL at tool:commit:before → resume completes, loses nothing, repeats no committed step", "status": "passed", - "durationMs": 1789.9936499999967 + "durationMs": 2646.6200160000008 }, { "file": "tests/durability/graph-kill-resume.test.ts", "suite": "runGraph kill-and-resume conformance (file run context)", "title": "SIGKILL at tool:commit:after-effect → resume completes, loses nothing, repeats no committed step", "status": "passed", - "durationMs": 1574.146922 + "durationMs": 2781.9505470000004 }, { "file": "tests/durability/graph-kill-resume.test.ts", "suite": "runGraph kill-and-resume conformance (file run context)", "title": "SIGKILL at driver:turn:6:before → resume completes, loses nothing, repeats no committed step", "status": "passed", - "durationMs": 1782.5725059999968 + "durationMs": 2653.772711999998 }, { "file": "tests/durability/graph-kill-resume.test.ts", "suite": "runGraph kill-and-resume conformance (file run context)", "title": "SIGKILL at driver:turn:6:after → resume completes, loses nothing, repeats no committed step", "status": "passed", - "durationMs": 1751.8499510000038 + "durationMs": 2526.5732519999947 }, { "file": "tests/durability/graph-rundir-journal.test.ts", "suite": "runGraph durable run context", "title": "runDir journals to the file stores supervise resumes from", "status": "passed", - "durationMs": 58.99278699999991 + "durationMs": 79.58805200000006 }, { "file": "tests/durability/graph-rundir-journal.test.ts", "suite": "runGraph durable run context", "title": "a graph without runDir stays entirely in memory (no writes to disk)", "status": "passed", - "durationMs": 4.836080000000038 + "durationMs": 12.057657000000063 }, { "file": "tests/durability/inotify-invariant.test.ts", "suite": "inotify budget invariant (one run = at most one instance)", "title": "a max-control-surface durable run peaks at <= 1 inotify instance", "status": "passed", - "durationMs": 89.33508500000016 + "durationMs": 78.75077699999997 }, { "file": "tests/durability/known-defects.test.ts", "suite": "known defect: 2026-08-11 resume appended the root record twice", "title": "kill at driver:turn:5:before → resume still wins with one clean root record", "status": "passed", - "durationMs": 2210.5245910000003 + "durationMs": 2623.215436 }, { "file": "tests/durability/known-defects.test.ts", "suite": "known defect: 2026-08-11 resume appended the root record twice", "title": "kill at tool:commit:after-effect → resume still wins with one clean root record", "status": "passed", - "durationMs": 1805.7219339999997 + "durationMs": 2543.2899399999997 }, { "file": "tests/durability/known-defects.test.ts", "suite": "known defect: 2026-08-11 resume appended the root record twice", "title": "kill at driver:turn:6:before → resume still wins with one clean root record", "status": "passed", - "durationMs": 1810.549489 + "durationMs": 2476.001752000001 }, { "file": "tests/durability/known-defects.test.ts", "suite": "known defect: 2026-08-11 resume appended the root record twice", "title": "kill at driver:turn:6:after → resume still wins with one clean root record", "status": "passed", - "durationMs": 1901.1329749999995 + "durationMs": 2526.5729659999997 }, { "file": "tests/durability/known-defects.test.ts", "suite": "known defect: 2026-09-16 re-entry lost the director state (non-retained path)", "title": "an unmet-contract re-entry carries the original task and run state, not a fragment", "status": "passed", - "durationMs": 26.175697000000582 + "durationMs": 77.60279099999934 }, { "file": "tests/durability/session-reattach.test.ts", "suite": "runGraph kill-and-resume conformance (session-backed workers, re-attach)", "title": "the reference run completes, running every session step exactly once", "status": "passed", - "durationMs": 2.7249339999998483 + "durationMs": 3.8973590000000513 }, { "file": "tests/durability/session-reattach.test.ts", "suite": "runGraph kill-and-resume conformance (session-backed workers, re-attach)", "title": "SIGKILL at driver:turn:1:before → resume re-attaches the sessions, nothing lost or repeated", "status": "passed", - "durationMs": 1760.3161420000001 + "durationMs": 2642.523365 }, { "file": "tests/durability/session-reattach.test.ts", "suite": "runGraph kill-and-resume conformance (session-backed workers, re-attach)", "title": "SIGKILL at driver:turn:1:after → resume re-attaches the sessions, nothing lost or repeated", "status": "passed", - "durationMs": 1757.4301029999997 + "durationMs": 2631.8689369999993 }, { "file": "tests/durability/session-reattach.test.ts", "suite": "runGraph kill-and-resume conformance (session-backed workers, re-attach)", "title": "SIGKILL at driver:turn:2:before → resume re-attaches the sessions, nothing lost or repeated", "status": "passed", - "durationMs": 1880.9557259999992 + "durationMs": 2537.3754500000005 }, { "file": "tests/durability/session-reattach.test.ts", "suite": "runGraph kill-and-resume conformance (session-backed workers, re-attach)", "title": "SIGKILL at driver:turn:2:after → resume re-attaches the sessions, nothing lost or repeated", "status": "passed", - "durationMs": 1825.9243199999983 + "durationMs": 2615.412993 }, { "file": "tests/durability/session-reattach.test.ts", "suite": "runGraph kill-and-resume conformance (session-backed workers, re-attach)", "title": "SIGKILL at session:surveyor:step-1 → resume re-attaches the sessions, nothing lost or repeated", "status": "passed", - "durationMs": 1772.5828000000001 + "durationMs": 2616.9387080000015 }, { "file": "tests/durability/session-reattach.test.ts", "suite": "runGraph kill-and-resume conformance (session-backed workers, re-attach)", "title": "SIGKILL at session:builder:step-1 → resume re-attaches the sessions, nothing lost or repeated", "status": "passed", - "durationMs": 1775.747553000001 + "durationMs": 2677.2199310000015 }, { "file": "tests/durability/session-reattach.test.ts", "suite": "runGraph kill-and-resume conformance (session-backed workers, re-attach)", - "title": "SIGKILL at session:surveyor:step-2 → resume re-attaches the sessions, nothing lost or repeated", + "title": "SIGKILL at session:verifier:step-1 → resume re-attaches the sessions, nothing lost or repeated", "status": "passed", - "durationMs": 1741.776813999999 + "durationMs": 2744.8296899999987 }, { "file": "tests/durability/session-reattach.test.ts", "suite": "runGraph kill-and-resume conformance (session-backed workers, re-attach)", - "title": "SIGKILL at session:verifier:step-1 → resume re-attaches the sessions, nothing lost or repeated", + "title": "SIGKILL at session:surveyor:step-2 → resume re-attaches the sessions, nothing lost or repeated", "status": "passed", - "durationMs": 1805.4265820000019 + "durationMs": 2713.449174999998 }, { "file": "tests/durability/session-reattach.test.ts", "suite": "runGraph kill-and-resume conformance (session-backed workers, re-attach)", "title": "SIGKILL at session:builder:step-2 → resume re-attaches the sessions, nothing lost or repeated", "status": "passed", - "durationMs": 1978.537734999998 + "durationMs": 2558.0278670000007 }, { "file": "tests/durability/session-reattach.test.ts", "suite": "runGraph kill-and-resume conformance (session-backed workers, re-attach)", - "title": "SIGKILL at session:surveyor:step-3 → resume re-attaches the sessions, nothing lost or repeated", + "title": "SIGKILL at session:verifier:step-2 → resume re-attaches the sessions, nothing lost or repeated", "status": "passed", - "durationMs": 1919.2524239999984 + "durationMs": 2463.363075000001 }, { "file": "tests/durability/session-reattach.test.ts", "suite": "runGraph kill-and-resume conformance (session-backed workers, re-attach)", - "title": "SIGKILL at session:verifier:step-2 → resume re-attaches the sessions, nothing lost or repeated", + "title": "SIGKILL at session:surveyor:step-3 → resume re-attaches the sessions, nothing lost or repeated", "status": "passed", - "durationMs": 1864.8955769999993 + "durationMs": 2593.7856250000004 }, { "file": "tests/durability/session-reattach.test.ts", "suite": "runGraph kill-and-resume conformance (session-backed workers, re-attach)", "title": "SIGKILL at session:builder:step-3 → resume re-attaches the sessions, nothing lost or repeated", "status": "passed", - "durationMs": 1721.8426579999978 + "durationMs": 2299.9176519999965 }, { "file": "tests/durability/session-reattach.test.ts", "suite": "runGraph kill-and-resume conformance (session-backed workers, re-attach)", "title": "SIGKILL at session:verifier:step-3 → resume re-attaches the sessions, nothing lost or repeated", "status": "passed", - "durationMs": 1746.4670650000007 + "durationMs": 2059.133047999996 }, { "file": "tests/durability/session-reattach.test.ts", "suite": "runGraph kill-and-resume conformance (session-backed workers, re-attach)", "title": "SIGKILL at driver:turn:3:before → resume re-attaches the sessions, nothing lost or repeated", "status": "passed", - "durationMs": 1755.6257460000015 + "durationMs": 2133.9928809999983 }, { "file": "tests/durability/session-reattach.test.ts", "suite": "runGraph kill-and-resume conformance (session-backed workers, re-attach)", "title": "SIGKILL at driver:turn:3:after → resume re-attaches the sessions, nothing lost or repeated", "status": "passed", - "durationMs": 1949.3098000000027 + "durationMs": 2191.526988000005 }, { "file": "tests/durability/session-reattach.test.ts", "suite": "runGraph kill-and-resume conformance (session-backed workers, re-attach)", "title": "SIGKILL at driver:turn:4:before → resume re-attaches the sessions, nothing lost or repeated", "status": "passed", - "durationMs": 1866.3333519999978 + "durationMs": 2129.46471 }, { "file": "tests/durability/session-reattach.test.ts", "suite": "runGraph kill-and-resume conformance (session-backed workers, re-attach)", "title": "SIGKILL at driver:turn:4:after → resume re-attaches the sessions, nothing lost or repeated", "status": "passed", - "durationMs": 1979.0534570000018 + "durationMs": 2203.188495999995 }, { "file": "tests/durability/session-reattach.test.ts", "suite": "runGraph kill-and-resume conformance (session-backed workers, re-attach)", "title": "SIGKILL at driver:turn:5:before → resume re-attaches the sessions, nothing lost or repeated", "status": "passed", - "durationMs": 1963.4175109999996 + "durationMs": 2183.446646000004 }, { "file": "tests/durability/session-reattach.test.ts", "suite": "runGraph kill-and-resume conformance (session-backed workers, re-attach)", "title": "SIGKILL at driver:turn:5:after → resume re-attaches the sessions, nothing lost or repeated", "status": "passed", - "durationMs": 2026.8951009999946 + "durationMs": 2136.6334819999975 }, { "file": "tests/durability/session-reattach.test.ts", "suite": "runGraph kill-and-resume conformance (session-backed workers, re-attach)", "title": "SIGKILL at tool:commit:before → resume re-attaches the sessions, nothing lost or repeated", "status": "passed", - "durationMs": 1913.236754999998 + "durationMs": 2174.692713999997 }, { "file": "tests/durability/session-reattach.test.ts", "suite": "runGraph kill-and-resume conformance (session-backed workers, re-attach)", "title": "SIGKILL at tool:commit:after-effect → resume re-attaches the sessions, nothing lost or repeated", "status": "passed", - "durationMs": 1785.6120759999976 + "durationMs": 2193.9456379999974 }, { "file": "tests/durability/session-reattach.test.ts", "suite": "runGraph kill-and-resume conformance (session-backed workers, re-attach)", "title": "SIGKILL at driver:turn:6:before → resume re-attaches the sessions, nothing lost or repeated", "status": "passed", - "durationMs": 1746.3193699999974 + "durationMs": 2151.850155 }, { "file": "tests/durability/session-reattach.test.ts", "suite": "runGraph kill-and-resume conformance (session-backed workers, re-attach)", "title": "SIGKILL at driver:turn:6:after → resume re-attaches the sessions, nothing lost or repeated", "status": "passed", - "durationMs": 1735.144825000003 + "durationMs": 2107.243177999997 }, { "file": "tests/durability/sql-context-stores.test.ts", "suite": "SQL context store invariants", "title": "rejects an invalid atomic group without exposing its valid prefix, and aborts its owner", "status": "passed", - "durationMs": 101.08553599999982 + "durationMs": 30.880900999999994 }, { "file": "tests/durability/sql-context-stores.test.ts", "suite": "SQL context store invariants", "title": "replays owner-scoped coordination with original bus stamps and no invented delivery outcome", "status": "passed", - "durationMs": 86.29605300000003 + "durationMs": 16.596160000000054 }, { "file": "tests/durability/sql-context-stores.test.ts", "suite": "SQL context store invariants", "title": "refuses a non-retained custom executor before running a graph", "status": "passed", - "durationMs": 52.26740799999993 + "durationMs": 21.99162000000001 }, { "file": "tests/durability/sql-kill-resume.test.ts", "suite": "runGraph kill-and-resume conformance (SQL run context, sqlite)", "title": "the reference run completes with the expected outcome", "status": "passed", - "durationMs": 2.2285069999998086 + "durationMs": 3.6470800000001873 }, { "file": "tests/durability/sql-kill-resume.test.ts", "suite": "runGraph kill-and-resume conformance (SQL run context, sqlite)", "title": "SIGKILL at driver:turn:1:before → resume completes, loses nothing, repeats no committed step", "status": "passed", - "durationMs": 2019.0431049999997 + "durationMs": 2436.721689 }, { "file": "tests/durability/sql-kill-resume.test.ts", "suite": "runGraph kill-and-resume conformance (SQL run context, sqlite)", "title": "SIGKILL at driver:turn:1:after → resume completes, loses nothing, repeats no committed step", "status": "passed", - "durationMs": 2134.517409 + "durationMs": 2585.0466450000004 }, { "file": "tests/durability/sql-kill-resume.test.ts", "suite": "runGraph kill-and-resume conformance (SQL run context, sqlite)", "title": "SIGKILL at driver:turn:2:before → resume completes, loses nothing, repeats no committed step", "status": "passed", - "durationMs": 2279.5093100000004 + "durationMs": 2530.174656 }, { "file": "tests/durability/sql-kill-resume.test.ts", "suite": "runGraph kill-and-resume conformance (SQL run context, sqlite)", "title": "SIGKILL at driver:turn:2:after → resume completes, loses nothing, repeats no committed step", "status": "passed", - "durationMs": 2242.568668 + "durationMs": 2680.171936999999 }, { "file": "tests/durability/sql-kill-resume.test.ts", "suite": "runGraph kill-and-resume conformance (SQL run context, sqlite)", "title": "SIGKILL at worker:surveyor:before → resume completes, loses nothing, repeats no committed step", "status": "passed", - "durationMs": 2575.2709080000004 + "durationMs": 2737.882132999999 }, { "file": "tests/durability/sql-kill-resume.test.ts", "suite": "runGraph kill-and-resume conformance (SQL run context, sqlite)", "title": "SIGKILL at worker:builder:before → resume completes, loses nothing, repeats no committed step", "status": "passed", - "durationMs": 2397.3243249999996 + "durationMs": 2714.7604659999997 }, { "file": "tests/durability/sql-kill-resume.test.ts", "suite": "runGraph kill-and-resume conformance (SQL run context, sqlite)", "title": "SIGKILL at worker:verifier:before → resume completes, loses nothing, repeats no committed step", "status": "passed", - "durationMs": 2834.247055 + "durationMs": 2564.487591000001 }, { "file": "tests/durability/sql-kill-resume.test.ts", "suite": "runGraph kill-and-resume conformance (SQL run context, sqlite)", "title": "SIGKILL at worker:surveyor:mid → resume completes, loses nothing, repeats no committed step", "status": "passed", - "durationMs": 2904.142885000001 + "durationMs": 2513.6176239999986 }, { "file": "tests/durability/sql-kill-resume.test.ts", "suite": "runGraph kill-and-resume conformance (SQL run context, sqlite)", "title": "SIGKILL at worker:builder:mid → resume completes, loses nothing, repeats no committed step", "status": "passed", - "durationMs": 2703.3508010000005 + "durationMs": 2693.457741000002 }, { "file": "tests/durability/sql-kill-resume.test.ts", "suite": "runGraph kill-and-resume conformance (SQL run context, sqlite)", "title": "SIGKILL at worker:verifier:mid → resume completes, loses nothing, repeats no committed step", "status": "passed", - "durationMs": 2788.4562080000032 + "durationMs": 2239.8636850000003 }, { "file": "tests/durability/sql-kill-resume.test.ts", "suite": "runGraph kill-and-resume conformance (SQL run context, sqlite)", "title": "SIGKILL at worker:surveyor:after → resume completes, loses nothing, repeats no committed step", "status": "passed", - "durationMs": 2663.724818999999 + "durationMs": 2105.597423000003 }, { "file": "tests/durability/sql-kill-resume.test.ts", "suite": "runGraph kill-and-resume conformance (SQL run context, sqlite)", "title": "SIGKILL at driver:turn:3:before → resume completes, loses nothing, repeats no committed step", "status": "passed", - "durationMs": 2726.741726 + "durationMs": 2169.9794139999976 }, { "file": "tests/durability/sql-kill-resume.test.ts", "suite": "runGraph kill-and-resume conformance (SQL run context, sqlite)", "title": "SIGKILL at driver:turn:3:after → resume completes, loses nothing, repeats no committed step", "status": "passed", - "durationMs": 2778.7122170000002 + "durationMs": 2142.901472999998 }, { "file": "tests/durability/sql-kill-resume.test.ts", "suite": "runGraph kill-and-resume conformance (SQL run context, sqlite)", "title": "SIGKILL at worker:builder:after → resume completes, loses nothing, repeats no committed step", "status": "passed", - "durationMs": 2789.1243500000055 + "durationMs": 2190.3930890000047 }, { "file": "tests/durability/sql-kill-resume.test.ts", "suite": "runGraph kill-and-resume conformance (SQL run context, sqlite)", "title": "SIGKILL at driver:turn:4:before → resume completes, loses nothing, repeats no committed step", "status": "passed", - "durationMs": 2363.853740999999 + "durationMs": 2181.615159000001 }, { "file": "tests/durability/sql-kill-resume.test.ts", "suite": "runGraph kill-and-resume conformance (SQL run context, sqlite)", "title": "SIGKILL at driver:turn:4:after → resume completes, loses nothing, repeats no committed step", "status": "passed", - "durationMs": 2280.7464360000013 + "durationMs": 2159.785183 }, { "file": "tests/durability/sql-kill-resume.test.ts", "suite": "runGraph kill-and-resume conformance (SQL run context, sqlite)", "title": "SIGKILL at worker:verifier:after → resume completes, loses nothing, repeats no committed step", "status": "passed", - "durationMs": 2536.6415380000035 + "durationMs": 2160.6383110000024 }, { "file": "tests/durability/sql-kill-resume.test.ts", "suite": "runGraph kill-and-resume conformance (SQL run context, sqlite)", "title": "SIGKILL at driver:turn:5:before → resume completes, loses nothing, repeats no committed step", "status": "passed", - "durationMs": 2170.885731000002 + "durationMs": 2068.9863130000012 }, { "file": "tests/durability/sql-kill-resume.test.ts", "suite": "runGraph kill-and-resume conformance (SQL run context, sqlite)", "title": "SIGKILL at driver:turn:5:after → resume completes, loses nothing, repeats no committed step", "status": "passed", - "durationMs": 2117.996852000004 + "durationMs": 2152.6635710000046 }, { "file": "tests/durability/sql-kill-resume.test.ts", "suite": "runGraph kill-and-resume conformance (SQL run context, sqlite)", "title": "SIGKILL at tool:commit:before → resume completes, loses nothing, repeats no committed step", "status": "passed", - "durationMs": 2484.4257639999996 + "durationMs": 2101.3153939999975 }, { "file": "tests/durability/sql-kill-resume.test.ts", "suite": "runGraph kill-and-resume conformance (SQL run context, sqlite)", "title": "SIGKILL at tool:commit:after-effect → resume completes, loses nothing, repeats no committed step", "status": "passed", - "durationMs": 2452.1592610000007 + "durationMs": 2087.268974999999 }, { "file": "tests/durability/sql-kill-resume.test.ts", "suite": "runGraph kill-and-resume conformance (SQL run context, sqlite)", "title": "SIGKILL at driver:turn:6:before → resume completes, loses nothing, repeats no committed step", "status": "passed", - "durationMs": 2167.8538989999943 + "durationMs": 1805.1900700000042 }, { "file": "tests/durability/sql-kill-resume.test.ts", "suite": "runGraph kill-and-resume conformance (SQL run context, sqlite)", "title": "SIGKILL at driver:turn:6:after → resume completes, loses nothing, repeats no committed step", "status": "passed", - "durationMs": 2220.4806119999994 + "durationMs": 1813.9998810000034 }, { "file": "tests/durability/sql-run-context.test.ts", "suite": "SQL run context", "title": "retains a begun tree and content-addressed blobs across independent contexts", "status": "passed", - "durationMs": 106.15304100000003 + "durationMs": 37.365548999999646 }, { "file": "tests/durability/sql-run-context.test.ts", "suite": "SQL run context", "title": "rejects a live second owner but does not serialize distinct runs", "status": "passed", - "durationMs": 483.65852800000016 + "durationMs": 427.50232300000016 }, { "file": "tests/durability/sql-run-context.test.ts", "suite": "SQL run context", "title": "rejects unsafe SQL identifiers before executing any SQL", "status": "passed", - "durationMs": 0.70666099999994 + "durationMs": 0.7633730000002288 }, { "file": "tests/durability/sql-run-context.test.ts", "suite": "SQL run context", "title": "runs the graph with SQL as its only orchestration persistence", "status": "passed", - "durationMs": 1997.2385499999998 + "durationMs": 1581.114826 }, { "file": "tests/durability/sql-run-context.test.ts", "suite": "SQL run context", - "title": "KNOWN GAP (never-dispatched window): SIGKILL at sql:spawned-child:after", + "title": "SIGKILL at sql:begin:before resumes on another host without a replacement key", "status": "passed", - "durationMs": 2221.6113 + "durationMs": 3183.0504499999997 }, { "file": "tests/durability/sql-run-context.test.ts", "suite": "SQL run context", - "title": "KNOWN GAP (never-dispatched window): SIGKILL at sql:execution-input:after", + "title": "SIGKILL at sql:begin:after resumes on another host without a replacement key", "status": "passed", - "durationMs": 2246.2564790000006 + "durationMs": 3325.3854309999997 }, { "file": "tests/durability/sql-run-context.test.ts", "suite": "SQL run context", - "title": "KNOWN GAP (never-dispatched window): SIGKILL at sql:execution-admitted-intent:before", + "title": "SIGKILL at sql:spawned-root:before resumes on another host without a replacement key", "status": "passed", - "durationMs": 2224.6287190000003 + "durationMs": 3204.3900950000007 }, { "file": "tests/durability/sql-run-context.test.ts", "suite": "SQL run context", - "title": "KNOWN GAP (never-dispatched window): SIGKILL at sql:execution-admitted-intent:after", + "title": "SIGKILL at sql:spawned-root:after resumes on another host without a replacement key", "status": "passed", - "durationMs": 2397.4902919999986 + "durationMs": 3176.954113 }, { "file": "tests/durability/sql-run-context.test.ts", "suite": "SQL run context", - "title": "KNOWN GAP (never-dispatched window): SIGKILL at sql:execution-admitted-environment:before", + "title": "SIGKILL at sql:spawned-child:before resumes on another host without a replacement key", "status": "passed", - "durationMs": 2199.6018030000014 + "durationMs": 3206.9439009999987 }, { "file": "tests/durability/sql-run-context.test.ts", "suite": "SQL run context", - "title": "KNOWN GAP (never-dispatched window): SIGKILL at sql:execution-admitted-environment:after", + "title": "SIGKILL at sql:execution-input:before resumes on another host without a replacement key", "status": "passed", - "durationMs": 2322.591504000002 + "durationMs": 3243.942270000003 }, { "file": "tests/durability/sql-run-context.test.ts", "suite": "SQL run context", - "title": "SIGKILL at sql:begin:before resumes on another host without a replacement key", + "title": "SIGKILL at sql:spawned-child:after resumes on another host without a replacement key", "status": "passed", - "durationMs": 2754.8020780000006 + "durationMs": 3198.6545899999983 }, { "file": "tests/durability/sql-run-context.test.ts", "suite": "SQL run context", - "title": "SIGKILL at sql:begin:after resumes on another host without a replacement key", + "title": "SIGKILL at sql:execution-input:after resumes on another host without a replacement key", "status": "passed", - "durationMs": 2846.599700999999 + "durationMs": 3164.1412500000006 }, { "file": "tests/durability/sql-run-context.test.ts", "suite": "SQL run context", - "title": "SIGKILL at sql:spawned-root:before resumes on another host without a replacement key", + "title": "SIGKILL at sql:execution-admitted-intent:before resumes on another host without a replacement key", "status": "passed", - "durationMs": 2515.6086069999983 + "durationMs": 3266.773097999998 }, { "file": "tests/durability/sql-run-context.test.ts", "suite": "SQL run context", - "title": "SIGKILL at sql:spawned-root:after resumes on another host without a replacement key", + "title": "SIGKILL at sql:execution-admitted-intent:after resumes on another host without a replacement key", "status": "passed", - "durationMs": 2583.609013000001 + "durationMs": 3240.217976 }, { "file": "tests/durability/sql-run-context.test.ts", "suite": "SQL run context", - "title": "SIGKILL at sql:spawned-child:before resumes on another host without a replacement key", + "title": "SIGKILL at sql:execution-admitted-environment:before resumes on another host without a replacement key", "status": "passed", - "durationMs": 2872.4368109999996 + "durationMs": 3295.2823290000015 }, { "file": "tests/durability/sql-run-context.test.ts", "suite": "SQL run context", - "title": "SIGKILL at sql:execution-input:before resumes on another host without a replacement key", + "title": "SIGKILL at sql:execution-admitted-environment:after resumes on another host without a replacement key", "status": "passed", - "durationMs": 2798.4966959999983 + "durationMs": 3218.775879999994 }, { "file": "tests/durability/sql-run-context.test.ts", "suite": "SQL run context", "title": "SIGKILL at provider:create:before resumes on another host without a replacement key", "status": "passed", - "durationMs": 3052.0177059999987 + "durationMs": 3139.5591150000037 }, { "file": "tests/durability/sql-run-context.test.ts", "suite": "SQL run context", "title": "SIGKILL at provider:create:after resumes on another host without a replacement key", "status": "passed", - "durationMs": 2866.241117999998 + "durationMs": 3255.731584999994 }, { "file": "tests/durability/sql-run-context.test.ts", "suite": "SQL run context", "title": "SIGKILL at provider:dispatch:before resumes on another host without a replacement key", "status": "passed", - "durationMs": 2532.644490999999 + "durationMs": 3242.8269319999963 }, { "file": "tests/durability/sql-run-context.test.ts", "suite": "SQL run context", "title": "SIGKILL at provider:dispatch:after resumes on another host without a replacement key", "status": "passed", - "durationMs": 2617.004630999996 + "durationMs": 3203.447129 }, { "file": "tests/durability/sql-run-context.test.ts", "suite": "SQL run context", "title": "SIGKILL at sql:execution-admitted-dispatched:before resumes on another host without a replacement key", "status": "passed", - "durationMs": 2471.1962090000015 + "durationMs": 3174.751044000004 }, { "file": "tests/durability/sql-run-context.test.ts", "suite": "SQL run context", "title": "SIGKILL at sql:execution-admitted-dispatched:after resumes on another host without a replacement key", "status": "passed", - "durationMs": 2415.326522000003 + "durationMs": 3174.706745000003 }, { "file": "tests/durability/sql-run-context.test.ts", "suite": "SQL run context", "title": "SIGKILL at sql:blob:before resumes on another host without a replacement key", "status": "passed", - "durationMs": 2606.7106950000016 + "durationMs": 3211.948884999998 }, { "file": "tests/durability/sql-run-context.test.ts", "suite": "SQL run context", "title": "SIGKILL at sql:blob:after resumes on another host without a replacement key", "status": "passed", - "durationMs": 2691.1226989999996 + "durationMs": 3102.8881120000005 }, { "file": "tests/durability/sql-run-context.test.ts", "suite": "SQL run context", "title": "SIGKILL at sql:execution-result:before resumes on another host without a replacement key", "status": "passed", - "durationMs": 2504.722077999999 + "durationMs": 3341.8620260000025 }, { "file": "tests/durability/sql-run-context.test.ts", "suite": "SQL run context", "title": "SIGKILL at sql:execution-result:after resumes on another host without a replacement key", "status": "passed", - "durationMs": 2483.8647839999976 + "durationMs": 3246.2291740000073 }, { "file": "tests/durability/sql-run-context.test.ts", "suite": "SQL run context", "title": "SIGKILL at sql:settled:before resumes on another host without a replacement key", "status": "passed", - "durationMs": 2227.5623359999954 + "durationMs": 3345.3027620000066 }, { "file": "tests/durability/sql-run-context.test.ts", "suite": "SQL run context", "title": "SIGKILL at sql:settled:after resumes on another host without a replacement key", "status": "passed", - "durationMs": 2211.3671990000003 + "durationMs": 3148.2372690000047 }, { "file": "tests/durability/sql-run-context.test.ts", "suite": "SQL run context", "title": "SIGKILL at tool:commit:before resumes on another host without a replacement key", "status": "passed", - "durationMs": 2274.2025280000016 + "durationMs": 3178.782582999993 }, { "file": "tests/durability/sql-run-context.test.ts", "suite": "SQL run context", "title": "SIGKILL at tool:commit:after resumes on another host without a replacement key", "status": "passed", - "durationMs": 5510.123783999996 + "durationMs": 3240.430932000003 }, { "file": "tests/durability/sql-run-context.test.ts", "suite": "SQL run context", "title": "two live orchestrators: rejects the contender, then the SAME process takes over after SIGKILL", "status": "passed", - "durationMs": 4461.246522999994 + "durationMs": 3821.1803139999975 + }, + { + "file": "tests/durability/sql-run-context.test.ts", + "suite": "SQL run context", + "title": "fences a SIGSTOP/SIGCONT owner without releasing its live successor", + "status": "passed", + "durationMs": 3353.350219 }, { "file": "tests/durability/sql-run-store.test.ts", "suite": "SQL publication and fencing", "title": "recovers a committed publication whose acknowledgement was lost, exactly once", "status": "passed", - "durationMs": 73.027444 + "durationMs": 10.973707999999988 }, { "file": "tests/durability/sql-run-store.test.ts", "suite": "SQL publication and fencing", "title": "never replays a staged record that was not published", "status": "passed", - "durationMs": 98.93034299999997 + "durationMs": 8.954714999999993 }, { "file": "tests/durability/sql-run-store.test.ts", "suite": "SQL publication and fencing", "title": "fences a stale publisher on the SAME row that changed ownership, and cannot release the successor", "status": "passed", - "durationMs": 89.149811 + "durationMs": 8.088649000000004 }, { "file": "tests/durability/sql-run-store.test.ts", "suite": "SQL publication and fencing", "title": "does not change the shared lease policy or namespace on another host", "status": "passed", - "durationMs": 88.04328299999997 + "durationMs": 6.045191000000017 }, { "file": "tests/durability/sql-run-store.test.ts", "suite": "SQL publication and fencing", "title": "serializes concurrent appends with constant SQL work and no history reads", "status": "passed", - "durationMs": 789.043836 + "durationMs": 128.34971699999997 } ] } diff --git a/docs/agent-managed-compute/README.md b/docs/agent-managed-compute/README.md index e0dfe34dc..023022301 100644 --- a/docs/agent-managed-compute/README.md +++ b/docs/agent-managed-compute/README.md @@ -4,7 +4,8 @@ This directory is the canonical plan for agents that allocate, steer, and recove ## Verdict -The repository is on the right architectural track, but it is not yet a distributed coordination system. +The runtime supports single-owner SQL failover for retained provider runs. +It is not a general distributed workflow engine. The core execution model is sound: @@ -28,7 +29,11 @@ Providers must advertise runtime MCP attachment support before receiving coordin One-shot providers remain nonsteerable. The file run lock coordinates local ownership. -Distributed generation fencing and live multi-provider recovery validation remain open. +`createFencedSqlRunContext` adds a SQL ownership lease, generation-checked publications, and a durable coordination log. +A second live process cannot acquire the run; after owner loss, a contender can take over without local files. +This requires a shared durable SQL service and a retained provider with idempotent create/dispatch. +External keyed effects must deduplicate at the effect site. Arbitrary unkeyed effects are not made exactly-once by SQL. +Live multi-provider recovery and generation-fenced remote steering remain separate validation work. See [reliability.md](./reliability.md) for the supported recovery boundary and remaining distributed requirements. ## Product Definition @@ -101,9 +106,9 @@ This table tracks the existing documents that materially overlap this plan. | One-shot delegation restart recovery | Partially implemented | `src/mcp/task-queue.ts` | | Conversation turn restart recovery | Implemented for one writer | `src/conversation/run-conversation.ts` | | Supervised tree restart recovery | Committed replay, retained children, and reconstructed nested managers with original reservations and finalizers | `src/runtime/supervise/supervisor.ts` | -| Durable cross-process coordination messages | Not implemented | `src/runtime/supervise/event-bus.ts` | +| Durable cross-process coordination messages | Owner-scoped SQL side-log, replayed with original bus stamps | `tests/durability/sql-context-stores.test.ts` | | Authenticated remote coordination MCP | Implemented; requires caller-provided reachable endpoint | `src/runtime/supervise/coordination-mcp.ts` | -| Concurrent coordinator failover | Not implemented | Current file stores have no compare-and-set or ownership claim. | +| Concurrent coordinator failover | Fenced SQL claim, live-owner exclusion, SIGKILL takeover, and stale-owner rejection | `tests/durability/sql-run-context.test.ts`, `tests/durability/sql-run-store.test.ts` | | One simple multi-round public API | Not implemented | `runConversation`, `runPersonified`, `runAgentic`, and `runAgentRounds` overlap. | | Acyclic runtime and knowledge packages | Implemented | `agent-knowledge` imports no runtime code; `agent-runtime` owns the optional composition in `src/knowledge/`; direct release lines align, while transitive packages may retain internal copies. | diff --git a/docs/agent-managed-compute/reliability.md b/docs/agent-managed-compute/reliability.md index caaf7b718..68e7a961a 100644 --- a/docs/agent-managed-compute/reliability.md +++ b/docs/agent-managed-compute/reliability.md @@ -2,7 +2,7 @@ ## Current recovery boundary -A file-backed run persists original profile and invocation inputs before provider admission. +File-backed and fenced SQL runs persist original profile and invocation inputs before provider admission. The journal records intent, environment, dispatch, and accepted result in order. Child recovery uses the configured executor factory and validates the original request before reconnecting or replaying its idempotent admission. Scope restores each live child's original identity, deadline, reservation, and shared worker slot before its manager acts. @@ -48,6 +48,12 @@ Numeric continuation caps retain their meaning, and zero still disables continua The file run lock protects one local coordinator. It does not fence provider mutations from a partitioned coordinator on another machine. +The additive `createFencedSqlRunContext` holds a SQL lease for the entire `runGraph`/`supervise` call. +Ownership takeover and publication compare-and-set the same head row; stale generations cannot publish or release a successor. +SQL stores retain the journal, content-addressed blobs, and owner-scoped coordination records without a local run directory. +A recorded invocation with no admission starts with its original keys because the intent is committed before any provider call. +Existing admissions still require original request validation and retained provider idempotency. +The SQL conformance fixtures forbid replacement-key escalation and test real SIGKILL and SIGSTOP/SIGCONT processes. No deployed or live multi-provider recovery proof is claimed here. An authenticated coordination listener bound to a wildcard accepts its actual local socket address and port. diff --git a/docs/api/primitive-catalog.md b/docs/api/primitive-catalog.md index d8d6ef490..9b8690515 100644 --- a/docs/api/primitive-catalog.md +++ b/docs/api/primitive-catalog.md @@ -7,7 +7,7 @@ # Primitive catalog — the never-stale anti-reinvention inventory -> **GENERATED** from `@tangle-network/agent-runtime@0.272.0` and `@tangle-network/agent-eval@0.187.0` by `scripts/gen-primitive-catalog.mjs`. Do NOT hand-edit — run `pnpm run docs:api`. This is the mechanical companion to the JUDGMENT in `canonical-api.md` (§2 decision table + §1.5 AgentProfile law): that doc says WHICH primitive to reach for and what NOT to build; this catalog proves WHAT exists. Per-symbol signatures + `file:line` live in the per-module pages under `docs/api/`. +> **GENERATED** from `@tangle-network/agent-runtime@0.272.1` and `@tangle-network/agent-eval@0.187.0` by `scripts/gen-primitive-catalog.mjs`. Do NOT hand-edit — run `pnpm run docs:api`. This is the mechanical companion to the JUDGMENT in `canonical-api.md` (§2 decision table + §1.5 AgentProfile law): that doc says WHICH primitive to reach for and what NOT to build; this catalog proves WHAT exists. Per-symbol signatures + `file:line` live in the per-module pages under `docs/api/`. ## 1. agent-runtime — own public surface diff --git a/docs/canonical-api.md b/docs/canonical-api.md index ca09c6c26..8823a9deb 100644 --- a/docs/canonical-api.md +++ b/docs/canonical-api.md @@ -4,7 +4,7 @@ Generated signatures and the complete export list live in docs/api/. Run pnpm docs:freshness after editing this file. --> -> **Version 0.272.0.** +> **Version 0.272.1.** > [`docs/api/primitive-catalog.md`](./api/primitive-catalog.md) lists every export and import path. > `agent-eval` must satisfy `>=0.185.0 <0.188.0`. > `sandbox` must satisfy `>=0.36.4 <0.48.0 || ^0.49.0-0 || ^0.50.0 || ^0.51.0 || ^0.52.0`. diff --git a/package.json b/package.json index e58c71807..f34661a5e 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@tangle-network/agent-runtime", - "version": "0.272.0", + "version": "0.272.1", "description": "Shared task-lifecycle skeleton for agents: a recursive loop kernel for chat turns, one-shot tasks, and multi-attempt loops, with trace capture and eval-gated self-improvement. Domain behavior lives in adapters; scoring and ship-gates in @tangle-network/agent-eval.", "homepage": "https://github.com/tangle-network/agent-runtime#readme", "repository": { diff --git a/scripts/run-durability-conformance.mjs b/scripts/run-durability-conformance.mjs index 1d8003516..40ff5a4e3 100644 --- a/scripts/run-durability-conformance.mjs +++ b/scripts/run-durability-conformance.mjs @@ -46,6 +46,8 @@ try { if (!error.status) throw error } const report = JSON.parse(readFileSync(outputFile, 'utf8')) + // Still persist the report, but a failing matrix must fail the conformance command. + if (!report.success) process.exitCode = 1 const cases = [] for (const suite of report.testResults ?? []) { for (const assertion of suite.assertionResults ?? []) { diff --git a/src/runtime/environment-provider.ts b/src/runtime/environment-provider.ts index 4196e44cd..19811770d 100644 --- a/src/runtime/environment-provider.ts +++ b/src/runtime/environment-provider.ts @@ -1849,7 +1849,10 @@ async function providerExecutionSource( .find((admission) => admission.phase === 'dispatched') const intent = admissions.find((admission) => admission.phase === 'intent') const environmentAdmission = admissions.find((admission) => admission.phase === 'environment') - if (args.recovering) { + // A journaled execution input can precede the first provider admission. In that case + // no provider call happened: onAdmission(intent) is awaited before create/dispatch. + // Start with the original execution keys; once any admission exists, validate its intent. + if (args.recovering && admissions.length > 0) { if (!intent) throw new Error('retained provider execution has no original intent') assertRetainedRunReplayMaterial(args.provider, material, intent) } @@ -1908,7 +1911,8 @@ async function providerExecutionSource( throw new Error(`retained provider execution is ${recovered.outcome}`) handle = recovered.handle } else { - if (args.recovering) throw new Error('retained provider execution has no durable admission') + if (args.recovering && admissions.length > 0) + throw new Error('retained provider execution has no durable admission') handle = await startRetainedRun({ provider: args.provider, ...material, onAdmission }) } args.onRetained(handle) diff --git a/src/testing/fixtures/agent-improvement-proposal.json b/src/testing/fixtures/agent-improvement-proposal.json index 17606f1b2..09c24d0df 100644 --- a/src/testing/fixtures/agent-improvement-proposal.json +++ b/src/testing/fixtures/agent-improvement-proposal.json @@ -1,6 +1,6 @@ { "changedSurfaces": ["prompt"], - "digest": "sha256:f720344815d0004177b582f63c2accf12445190b3a6d7b1aceffce2b68f518e0", + "digest": "sha256:e18877601b812bc67c10bc3ab28f9561a6c19915ddc44bb244cd06b2b1009b91", "evaluation": { "decision": { "contributingChecks": [ @@ -4882,7 +4882,7 @@ ], "metadata": { "fixture": "agent-improvement-proposal", - "runtimeVersion": "0.272.0" + "runtimeVersion": "0.272.1" }, "objectives": [ { @@ -4993,8 +4993,8 @@ "baselineContentHash": "sha256:5c21ee53e513fc604cb09754e21c392b24a424da0ef37dbf8f1ee4a8a0b08f09", "candidateContentHash": "sha256:60fcbb1c728194bd51d7d19cb732d1c3f1881dce7e0a6266b41c8b98cfd65693", "kind": "agent-eval-loop", - "recordDigest": "sha256:29801c6b5081b6b1fc96ed8ac17f36a37e09a6cbb3a1a9626b768235915ab62f", - "runId": "agent-runtime-0.272.0-proposal-fixture", + "recordDigest": "sha256:78adac097ca84ed7dc4ea53e1600e03f1c84d3e9a05add89559a34554060313a", + "runId": "agent-runtime-0.272.1-proposal-fixture", "schema": "agent-candidate-experiment" } }, @@ -5021,5 +5021,5 @@ ], "kind": "agent-improvement-proposal", "proposedAt": "2026-07-10T01:00:00.000Z", - "runId": "agent-runtime-0.272.0-proposal-fixture" + "runId": "agent-runtime-0.272.1-proposal-fixture" } diff --git a/src/testing/fixtures/agent-profile-improvement-proposal.json b/src/testing/fixtures/agent-profile-improvement-proposal.json index d807dc1ab..420b4e175 100644 --- a/src/testing/fixtures/agent-profile-improvement-proposal.json +++ b/src/testing/fixtures/agent-profile-improvement-proposal.json @@ -1,6 +1,6 @@ { "changedSurfaces": ["prompt", "skills"], - "digest": "sha256:d027fd862aafe8f614c4ba9ed3daa214bbdd0a6258ad1eee4a500df2cc2e9b08", + "digest": "sha256:7c0bc343ead3fdb08d2048a499650639bbc2c513c155e51956bd66f79641fd3a", "evaluation": { "decision": { "contributingChecks": [ @@ -1715,7 +1715,7 @@ ], "metadata": { "fixture": "agent-profile-improvement-proposal", - "runtimeVersion": "0.272.0" + "runtimeVersion": "0.272.1" }, "objectives": [ { @@ -1826,7 +1826,7 @@ "baselineContentHash": "sha256:21c495a37c418c10bde64fbaa188beddeed31f1f051ea60a6a6582a9ee0db704", "candidateContentHash": "sha256:103f77bc8481601eef1ad5fe6ba84a40dffabc3a44f421f8c8559121edab84e9", "kind": "agent-eval-loop", - "recordDigest": "sha256:b52e9a2906575f4bffde758c080499b95658aa2817ba22c315f9bda5ed484a4f", + "recordDigest": "sha256:09ef92db031120d58f74cbde7c0aeef07aa4c9303448760e20c69646ce146094", "runId": "profile-improvement-1", "schema": "agent-profile-improvement-experiment" } From 8d97f9d8e6e11aec9b38adf8ff4e0fdeafdc118b Mon Sep 17 00:00:00 2001 From: drewstone Date: Fri, 25 Sep 2026 02:02:14 -0700 Subject: [PATCH 3/5] ci(durability): run independent synthesis red/green evidence --- .github/workflows/sql-synthesis-evidence.yml | 46 ++++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 .github/workflows/sql-synthesis-evidence.yml diff --git a/.github/workflows/sql-synthesis-evidence.yml b/.github/workflows/sql-synthesis-evidence.yml new file mode 100644 index 000000000..ff9d1c71a --- /dev/null +++ b/.github/workflows/sql-synthesis-evidence.yml @@ -0,0 +1,46 @@ +name: SQL synthesis evidence + +on: + push: + branches: [pro/sql-run-context/synthesis] + +permissions: + contents: read + +jobs: + evidence: + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + persist-credentials: false + fetch-depth: 0 + - uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 + with: + node-version: 22 + cache: pnpm + - name: Install + run: pnpm install --frozen-lockfile + - name: Strict tests against main implementation (expected red) + shell: bash + run: | + git show 66d2caa4224c65e637da474a2a2867d785e0e319:src/runtime/environment-provider.ts > /tmp/synthesis-main-provider.ts + cp src/runtime/environment-provider.ts /tmp/synthesis-candidate-provider.ts + trap 'cp /tmp/synthesis-candidate-provider.ts src/runtime/environment-provider.ts' EXIT + cp /tmp/synthesis-main-provider.ts src/runtime/environment-provider.ts + set +e + npx vitest run tests/durability/sql-run-context.test.ts + status=$? + set -e + echo "STRICT_MAIN_EXIT=$status" + test "$status" -eq 1 + - name: SQL kill/resume and context suites + if: ${{ !cancelled() }} + run: npx vitest run tests/durability/sql-kill-resume.test.ts tests/durability/sql-run-context.test.ts tests/durability/sql-run-store.test.ts tests/durability/sql-context-stores.test.ts + - name: Durability conformance + if: ${{ !cancelled() }} + run: | + node scripts/run-durability-conformance.mjs + node -e 'const r = require("./conformance/durability/results.json"); console.log("SYNTHESIS_TOTALS=" + JSON.stringify(r.totals)); if (r.totals.failed !== 0) process.exitCode = 1' From ce8085289099869d29ae4af6e56e8e3c41041add Mon Sep 17 00:00:00 2001 From: drewstone Date: Fri, 25 Sep 2026 02:10:40 -0700 Subject: [PATCH 4/5] chore(durability): remove one-off synthesis evidence workflow after green run --- .github/workflows/sql-synthesis-evidence.yml | 46 -------------------- 1 file changed, 46 deletions(-) delete mode 100644 .github/workflows/sql-synthesis-evidence.yml diff --git a/.github/workflows/sql-synthesis-evidence.yml b/.github/workflows/sql-synthesis-evidence.yml deleted file mode 100644 index ff9d1c71a..000000000 --- a/.github/workflows/sql-synthesis-evidence.yml +++ /dev/null @@ -1,46 +0,0 @@ -name: SQL synthesis evidence - -on: - push: - branches: [pro/sql-run-context/synthesis] - -permissions: - contents: read - -jobs: - evidence: - runs-on: ubuntu-latest - timeout-minutes: 20 - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 - with: - persist-credentials: false - fetch-depth: 0 - - uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 - - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 - with: - node-version: 22 - cache: pnpm - - name: Install - run: pnpm install --frozen-lockfile - - name: Strict tests against main implementation (expected red) - shell: bash - run: | - git show 66d2caa4224c65e637da474a2a2867d785e0e319:src/runtime/environment-provider.ts > /tmp/synthesis-main-provider.ts - cp src/runtime/environment-provider.ts /tmp/synthesis-candidate-provider.ts - trap 'cp /tmp/synthesis-candidate-provider.ts src/runtime/environment-provider.ts' EXIT - cp /tmp/synthesis-main-provider.ts src/runtime/environment-provider.ts - set +e - npx vitest run tests/durability/sql-run-context.test.ts - status=$? - set -e - echo "STRICT_MAIN_EXIT=$status" - test "$status" -eq 1 - - name: SQL kill/resume and context suites - if: ${{ !cancelled() }} - run: npx vitest run tests/durability/sql-kill-resume.test.ts tests/durability/sql-run-context.test.ts tests/durability/sql-run-store.test.ts tests/durability/sql-context-stores.test.ts - - name: Durability conformance - if: ${{ !cancelled() }} - run: | - node scripts/run-durability-conformance.mjs - node -e 'const r = require("./conformance/durability/results.json"); console.log("SYNTHESIS_TOTALS=" + JSON.stringify(r.totals)); if (r.totals.failed !== 0) process.exitCode = 1' From 4dba2752f23c23d649950e3f6214b9f80489303f Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Fri, 25 Sep 2026 03:37:58 -0600 Subject: [PATCH 5/5] test(durability): every spawned worker settles exactly once, done MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add the last strict assertion from tournament entrant p2's zero-replacement-key contract: each spawned worker has exactly one settled event and it is done — completeness alone could hide a duplicated settlement behind a Set. Verified with the synthesis suite: 56/56 including the former known-gap windows. --- tests/durability/sql-run-context.test.ts | 2 ++ tests/helpers/durability/sql-run-context-child.ts | 10 ++++++++++ 2 files changed, 12 insertions(+) diff --git a/tests/durability/sql-run-context.test.ts b/tests/durability/sql-run-context.test.ts index f6fe42ba8..67e4248a7 100644 --- a/tests/durability/sql-run-context.test.ts +++ b/tests/durability/sql-run-context.test.ts @@ -108,6 +108,7 @@ function start(shared: string, cwd: string, mode = 'run', checkpoint = '') { rootMaterializations: number uniqueRootAttempts: boolean uniqueCursorSequences: boolean + settledPerWorker: boolean workerRecords: Array<{ label: string; spawns: number; settlements: number }> } }, @@ -121,6 +122,7 @@ function assertComplete(report: Awaited['rep expect(report.rootMaterializations).toBe(1) expect(report.uniqueRootAttempts).toBe(true) expect(report.uniqueCursorSequences).toBe(true) + expect(report.settledPerWorker).toBe(true) expect(report.workers).toBe(3) expect(report.completed).toBe(3) expect(report.inDoubt).toBe(0) diff --git a/tests/helpers/durability/sql-run-context-child.ts b/tests/helpers/durability/sql-run-context-child.ts index 65ab1061f..137f60d9a 100644 --- a/tests/helpers/durability/sql-run-context-child.ts +++ b/tests/helpers/durability/sql-run-context-child.ts @@ -166,6 +166,15 @@ async function phase() { const cursors = events .filter((event) => event.kind === 'settled' || event.kind === 'cancelled') .map((event) => event.seq) + const workerIds = new Set(workers.map((event) => event.id)) + const settledCounts = new Map() + for (const event of events) { + if (event.kind === 'settled' && workerIds.has(event.id)) + settledCounts.set(event.id, (settledCounts.get(event.id) ?? 0) + 1) + } + const settledPerWorker = workers.every( + (event) => settledCounts.get(event.id) === 1 && completed.has(event.id), + ) const report = { kind: result.result.kind, out: result.result.kind === 'winner' ? result.result.out : result.result, @@ -174,6 +183,7 @@ async function phase() { ).length, uniqueRootAttempts: attempts.length === new Set(attempts).size, uniqueCursorSequences: cursors.length === new Set(cursors).size, + settledPerWorker, roots: events.filter((event) => event.kind === 'spawned' && event.parent === undefined).length, workers: workers.length, completed: workers.filter((event) => completed.has(event.id)).length,