diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 01d1a27d5..6bd59879b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -303,6 +303,14 @@ jobs: pnpm clean:daemon pnpm test:integration:node + - name: Run seeded concurrency torture lane (fast PR sweep) + # #1416's nightly torture lane lives under test/integration/nightly/, out + # of the test:integration:node glob, so this is a *deliberate* fast PR + # sweep (TORTURE_RUNS default 128 seeds, ~sub-second) — not an accidental + # glob inclusion. The Concurrency Torture Nightly workflow sweeps a much + # larger seed range on schedule. + run: pnpm test:concurrency-torture + - name: Run provider-backed integration tests run: pnpm test:integration:provider diff --git a/.github/workflows/concurrency-torture-nightly.yml b/.github/workflows/concurrency-torture-nightly.yml new file mode 100644 index 000000000..a55f00008 --- /dev/null +++ b/.github/workflows/concurrency-torture-nightly.yml @@ -0,0 +1,65 @@ +name: Concurrency Torture Nightly + +# Seeded concurrency torture lane for session/lease/lock invariants (#1416, +# umbrella #1412 Track A). N concurrent clients drive randomized-but-seeded +# interleavings of open/mutate/close/takeover/kill against the real SessionStore +# + LeaseRegistry, with all concurrency routed through a deterministic scheduler +# so a seed fully determines execution order. Deterministic and offline — no +# devices, simulators, or wall-clock stress (those are out of scope for #1416). +# +# Scheduled + manual only: the PR gate already runs a fast default sweep via the +# Node integration lane; this nightly sweeps a much larger seed range to keep +# mining for ordering bugs. A failure prints the seed and the exact +# `TORTURE_SEED= pnpm test:concurrency-torture` replay command. + +on: + schedule: + - cron: '0 5 * * *' + workflow_dispatch: + inputs: + runs: + description: 'Number of seeded interleavings to sweep' + required: false + default: '5000' + seed-start: + description: 'First seed in the sweep' + required: false + default: '0' + +permissions: + contents: read + +concurrency: + group: ci-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + torture: + name: Session/lease/lock torture sweep + runs-on: ubuntu-latest + timeout-minutes: 20 + env: + TORTURE_RUNS: ${{ github.event.inputs.runs || '5000' }} + TORTURE_SEED_START: ${{ github.event.inputs.seed-start || '0' }} + # Standard scheduled-lane artifact envelope (#1430) built via the shared + # scripts/lib/lane-envelope.ts (commit, tool/configHash, seed range, result); + # the seed sweep rides in the typed data payload. Emitted by the test and + # uploaded below so the freshness watcher can detect a lane going dark. + TORTURE_ENVELOPE: ${{ github.workspace }}/torture-results/envelope.json + steps: + - name: Checkout + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - name: Setup toolchain + uses: ./.github/actions/setup-node-pnpm + + - name: Run concurrency torture sweep + run: pnpm test:concurrency-torture + + - name: Upload torture lane envelope + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: concurrency-torture-envelope + path: torture-results/ + if-no-files-found: warn diff --git a/docs/agents/testing.md b/docs/agents/testing.md index 0ee8cb823..7f5487f88 100644 --- a/docs/agents/testing.md +++ b/docs/agents/testing.md @@ -189,6 +189,69 @@ AGENT_DEVICE_WEB_E2E=1 pnpm test:smoke:web The test is skipped unless `AGENT_DEVICE_WEB_E2E=1` is set. The test runs `agent-device web setup` and `agent-device web doctor` with an isolated state directory before opening the fixture URL, so it verifies the public managed-backend setup path instead of relying on a global `agent-browser`. CI runs the lane on Node 24 because the managed backend requires Node >= 24. Failure artifacts, daemon state, and browser config are written under `test/artifacts/web/`. +## Concurrency torture lane + +`test/integration/nightly/concurrency-torture.test.ts` (#1416, umbrella #1412 Track A) runs N concurrent +clients through randomized-but-**seeded** interleavings of open/mutate/close/takeover/kill against +the real `SessionStore` + `LeaseRegistry` (plus an in-memory device-claim model). After every run it +asserts: no leaked leases or claims, no cross-session state bleed, every lock released after owner +death, the session store stays consistent, and same-device critical sections never overlap (this +pins the router's same-device open serialization under 100+ interleavings). + +A seed alone cannot reproduce Promise/event-loop interleavings, so **all** concurrency is routed +through a deterministic scheduler (`nightly/concurrency-torture/deterministic-scheduler.ts`) — an +instrumented dispatcher that is the sole source of ordering (which fiber steps next, and which waiter +wins a contended lock). A seed therefore fully determines execution order. + +Each operation's lock plan is **not** hand-written: it is built exactly as the daemon builds it in +`createRequestExecutionScope` — gate on the production decision `shouldLockSessionExecution(command)` +(`src/daemon/daemon-command-registry.ts`), and only then resolve keys via the production router +primitive `resolveRequestExecutionLockKeys` (`src/daemon/request-binding.ts`), driven with a fake +device inventory through the production `withDeviceInventoryProvider` seam +(`nightly/concurrency-torture/bindings.ts`). Only the mutex *grant* is modeled by the scheduler, because +`withKeyedLock`'s native microtask hand-off cannot be reproduced from a seed. Consequently reverting +*either* production decision — exempting a command from execution locking, or dropping the `device:` +key — changes the derived plan and trips the overlap invariant, so the lane is genuinely coupled to +production lock resolution, not a duplicate of it. **Real:** +`SessionStore` and `LeaseRegistry`. **Modeled:** the advisory device claim (`InMemoryClaimRegistry`) +and process "kill" — the production claim is a filesystem/OS lock and real process death, both out of +scope for this scheduling lane and covered by their own unit tests. The full real-vs-modeled boundary +is documented at the top of `nightly/concurrency-torture/harness.ts`. + +Because the seeded sweep *models* the mutex grant, a separate **real-scope guard** +(`nightly/concurrency-torture/real-scope-serialization.ts`) drives concurrent same-device opens through the +actual `createRequestExecutionScope().runLocked()` → `withRequestExecutionLocks` → `withKeyedLock` +and asserts the critical sections never overlap. This is intentionally not seeded (it exercises real +event-loop scheduling); its job is to fail if the production lock *application* path regresses, which +the modeled sweep alone could not catch. + +```bash +pnpm test:concurrency-torture # default sweep (TORTURE_RUNS=128 seeds from 0) +TORTURE_SEED=1234 pnpm test:concurrency-torture # replay ONE seed's exact interleaving (seed-replay flag) +TORTURE_RUNS=5000 TORTURE_SEED_START=0 pnpm test:concurrency-torture # widen the sweep +``` + +Replay is exact: a given seed reproduces the whole scheduler trace (`traceSignature`), the terminal +invariant outcome, and the contention profile — equality on all three is asserted not just under +`TORTURE_SEED` but for **every seed in the normal sweep** (each seed is re-run and compared), so +non-determinism is caught on the ordinary CI/nightly path. The sweep also asserts real same-device +lock *contention* occurred (two clients parked on one `device:` lock), and a dedicated forced +two-client same-device test pins both clients to one device via `pinnedDevice` so they cannot land on +different devices, driving that contention deterministically. + +Every failure prints the offending seed and the exact `TORTURE_SEED= pnpm test:concurrency-torture` +replay command. The lane lives under `test/integration/nightly/`, deliberately **out** of the +`test:integration:node` glob so it is not an accidental PR-time run: the PR gate runs a fast default +sweep via an explicit `Run seeded concurrency torture lane` step in the Integration job, and the +`Concurrency Torture Nightly` workflow sweeps a much larger seed range on schedule. The nightly run +emits the shared scheduled-lane envelope (`scripts/lib/lane-envelope.ts`, #1430 — commit, +tool/`configHash` from the lane source hash, `seed` range, duration, result, with the seed sweep in +the typed `data` payload) via `TORTURE_ENVELOPE=`, uploaded as the `concurrency-torture-envelope` +artifact. The +envelope is written once, after **all** lane tests settle, and reports `fail` if any of them (sweep, +replay self-check, or forced-contention guardrail) failed — a later-failing guardrail can never be +published as a passing envelope. Optional knobs: `TORTURE_CLIENTS`, `TORTURE_OPS`. + ## Speed rules (experiment-backed, 2026-07-04) Measured on the full unit suite (340 files, 3,210 tests, 48s wall at ~7x parallelism): diff --git a/package.json b/package.json index beeaa8388..1e1547a44 100644 --- a/package.json +++ b/package.json @@ -161,6 +161,7 @@ "test:smoke": "node --test test/integration/smoke-*.test.ts", "test:integration:node": "node --test test/integration/*.test.ts", "test:integration": "pnpm test:integration:node && pnpm test:integration:provider", + "test:concurrency-torture": "node --test test/integration/nightly/concurrency-torture.test.ts", "test:replay:ios": "node --experimental-strip-types src/bin.ts test test/integration/replays/ios/simulator", "test:replay:ios-device": "node --experimental-strip-types src/bin.ts test test/integration/replays/ios/device", "test:replay:android": "node --experimental-strip-types src/bin.ts test test/integration/replays/android", diff --git a/test/integration/nightly/concurrency-torture.test.ts b/test/integration/nightly/concurrency-torture.test.ts new file mode 100644 index 000000000..f9265ef95 --- /dev/null +++ b/test/integration/nightly/concurrency-torture.test.ts @@ -0,0 +1,242 @@ +// Seeded concurrency torture lane for session / lease / lock invariants (#1416, +// umbrella #1412 Track A). +// +// N concurrent clients drive randomized-but-SEEDED interleavings of +// open / mutate / close / takeover / kill against fake providers (the real +// `SessionStore` + `LeaseRegistry`, an in-memory device-claim model), with ALL +// concurrency routed through a deterministic scheduler so a seed fully +// determines execution order. Each operation's lock plan is derived from the +// production router primitive `resolveRequestExecutionLockKeys`, so reverting +// the router's same-device serialization trips the overlap invariant. After +// every run the harness asserts: +// - no leaked leases or claims, +// - no cross-session state bleed, +// - every lock released after owner death, +// - the session store stays consistent, +// - same-device critical sections never overlap. +// +// Seed replay (documented in docs/agents/testing.md): +// TORTURE_SEED=1234 pnpm test:concurrency-torture +// replays that exact interleaving deterministically — the whole scheduler trace, +// not just its length, is asserted equal. Otherwise the lane sweeps TORTURE_RUNS +// seeds (default 128, ≥100 to satisfy the acceptance bar) starting at +// TORTURE_SEED_START (default 0). Any failure prints the seed and replay command. + +import test, { after } from 'node:test'; +import assert from 'node:assert/strict'; + +import { runTorture, type ClientOp, type TortureRunResult } from './concurrency-torture/harness.ts'; +import { DEVICE_POOL } from './concurrency-torture/bindings.ts'; +import { measureRealScopeMaxOverlap } from './concurrency-torture/real-scope-serialization.ts'; +import { buildEnvelope, writeEnvelopeIfRequested } from './concurrency-torture/envelope.ts'; + +// The #1430 envelope must report the outcome of the WHOLE lane, not just the +// sweep test. Every test records its result here and the envelope is written +// once, after all tests settle, so a later failing guardrail can never be +// published as a passing envelope. +type SweepRange = { seedStart: number; runs: number }; +const laneStartedMs = Date.now(); +let laneFailed = false; +let sweepRange: SweepRange | undefined; + +async function guard(body: () => Promise): Promise { + try { + await body(); + } catch (error) { + laneFailed = true; + throw error; + } +} + +after(() => { + if (!sweepRange) return; // only the sweep lane (nightly) publishes an envelope + // Duration spans the WHOLE lane (sweep + replay + both serialization guards), + // measured to this hook, so it can't understate work by excluding later tests. + const envelope = buildEnvelope({ + ...sweepRange, + startedAtMs: laneStartedMs, + result: laneFailed ? 'fail' : 'pass', + }); + const written = writeEnvelopeIfRequested(envelope); + if (written) console.log(`torture envelope → ${written}\n${JSON.stringify(envelope)}`); +}); + +/** + * Assert a seed replays bit-for-bit: same ordered scheduler trace, same terminal + * invariant outcome, same contention profile. Run for EVERY seed in the sweep so + * determinism is proven on the normal CI/nightly path, not only under TORTURE_SEED. + */ +function assertDeterministicReplay(result: TortureRunResult, replay: TortureRunResult): void { + assert.equal( + replay.traceSignature, + result.traceSignature, + `seed ${result.seed} produced a different scheduler trace on replay — non-determinism`, + ); + assert.deepEqual( + replay.failures, + result.failures, + `seed ${result.seed} produced a different invariant outcome on replay — non-determinism`, + ); + assert.equal( + replay.deviceContention, + result.deviceContention, + `seed ${result.seed} produced different device contention on replay — non-determinism`, + ); +} + +function intFromEnv(name: string, fallback: number): number { + const raw = process.env[name]?.trim(); + if (!raw) return fallback; + const parsed = Number(raw); + if (!Number.isInteger(parsed) || parsed < 0) { + throw new Error(`${name} must be a non-negative integer, got ${raw}`); + } + return parsed; +} + +function optionalIntFromEnv(name: string): number | undefined { + const raw = process.env[name]?.trim(); + if (!raw) return undefined; + const parsed = Number(raw); + if (!Number.isInteger(parsed) || parsed <= 0) { + throw new Error(`${name} must be a positive integer, got ${raw}`); + } + return parsed; +} + +// Seeds are non-negative (0 is a real, replayable seed): the sweeps start at +// seed 0 and print `TORTURE_SEED=0` on failure, so the replay parser MUST accept +// 0 — unlike client/op COUNTS, which must be strictly positive. +function optionalNonNegativeIntFromEnv(name: string): number | undefined { + const raw = process.env[name]?.trim(); + if (!raw) return undefined; + const parsed = Number(raw); + if (!Number.isInteger(parsed) || parsed < 0) { + throw new Error(`${name} must be a non-negative integer, got ${raw}`); + } + return parsed; +} + +function replayHint(seed: number): string { + return `Replay this exact interleaving with: TORTURE_SEED=${seed} pnpm test:concurrency-torture`; +} + +function assertClean(result: TortureRunResult): void { + if (result.failures.length === 0) return; + const lines = result.failures.map((f) => ` - [${f.invariant}] ${f.detail}`); + assert.fail( + `Concurrency invariants violated on seed ${result.seed} ` + + `(${result.clients} clients, ${result.ops} ops, ${result.scheduleLength} scheduled steps):\n` + + `${lines.join('\n')}\n${replayHint(result.seed)}`, + ); +} + +const explicitSeed = optionalNonNegativeIntFromEnv('TORTURE_SEED'); +const clients = optionalIntFromEnv('TORTURE_CLIENTS'); +const opsPerClient = optionalIntFromEnv('TORTURE_OPS'); + +// Guard the PRODUCTION lock APPLICATION path, not just the modeled sweep: drive +// concurrent same-device opens through the real `createRequestExecutionScope() +// .runLocked()` (→ `withRequestExecutionLocks` → `withKeyedLock`) and assert +// they never overlap. Regressing production serialization trips this even +// though the seeded sweep models the grant. Runs in every mode. +test('concurrency torture — real-scope same-device serialization', () => + guard(async () => { + const maxOverlap = await measureRealScopeMaxOverlap(4); + assert.equal( + maxOverlap, + 1, + `real createRequestExecutionScope().runLocked() let ${maxOverlap} same-device critical ` + + 'sections overlap — production execution locking is not serializing', + ); + })); + +// Regression: seed 0 is inside the default/forced sweeps and failures print +// `TORTURE_SEED=0`, so that replay command must be runnable. The seed parser +// must accept 0 (a positive-only parser silently broke this), and seed 0 must +// itself replay bit-for-bit. +test('concurrency torture — seed 0 is a replayable seed', () => + guard(async () => { + const probe = '__TORTURE_SEED_ZERO_PROBE__'; + process.env[probe] = '0'; + try { + assert.equal( + optionalNonNegativeIntFromEnv(probe), + 0, + 'TORTURE_SEED=0 must parse to seed 0 so the printed replay command works', + ); + } finally { + delete process.env[probe]; + } + const result = await runTorture({ seed: 0 }); + assertDeterministicReplay(result, await runTorture({ seed: 0 })); + assertClean(result); + })); + +if (explicitSeed !== undefined) { + test(`concurrency torture — replay seed ${explicitSeed}`, () => + guard(async () => { + const result = await runTorture({ seed: explicitSeed, clients, opsPerClient }); + const replay = await runTorture({ seed: explicitSeed, clients, opsPerClient }); + assertDeterministicReplay(result, replay); + assertClean(result); + })); +} else { + const runs = intFromEnv('TORTURE_RUNS', 128); + const seedStart = intFromEnv('TORTURE_SEED_START', 0); + + test(`concurrency torture — ${runs} seeded interleavings from ${seedStart}`, () => + guard(async () => { + sweepRange = { seedStart, runs }; + let totalContention = 0; + for (let i = 0; i < runs; i += 1) { + const seed = seedStart + i; + const result = await runTorture({ seed, clients, opsPerClient }); + totalContention += result.deviceContention; + // Prove exact replay on the NORMAL sweep path (not just TORTURE_SEED): + // re-run the seed and assert the whole trace + outcome + contention match. + assertDeterministicReplay(result, await runTorture({ seed, clients, opsPerClient })); + assertClean(result); + } + // The sweep must actually PRODUCE same-device contention — two clients + // parked on one device lock — or the lane is not exercising serialization + // and a broken lock could pass unnoticed. + assert.ok( + totalContention > 0, + 'no same-device lock contention occurred across the sweep — the lane is not testing serialization', + ); + })); + + // Forced same-device contention: two clients repeatedly open THE SAME pinned + // device (via `pinnedDevice`, so `doOpen` cannot randomly pick a different one). + // This deterministically drives both onto one `device:` lock so the overlap + // invariant is exercised, not just present. If the router stopped serializing + // same-device opens, `deviceContention` here would collapse and the overlap + // invariant would fire. + test('concurrency torture — forced two-client same-device contention', () => + guard(async () => { + const pinnedDevice = DEVICE_POOL[0]; + const program: ClientOp[][] = [ + ['open', 'mutate', 'close', 'open', 'mutate'], + ['open', 'mutate', 'close', 'open', 'mutate'], + ]; + let contendedSeeds = 0; + for (let seed = 0; seed < 40; seed += 1) { + const result = await runTorture({ seed, program, pinnedDevice }); + assertClean(result); + const perDeviceMax = Object.values(result.perDeviceMaxConcurrency); + assert.ok( + perDeviceMax.every((max) => max <= 1), + `seed ${seed}: same-device critical sections overlapped — ${JSON.stringify( + result.perDeviceMaxConcurrency, + )}`, + ); + if (result.deviceContention > 0) contendedSeeds += 1; + } + assert.ok( + contendedSeeds > 0, + 'two clients never actually contended for the pinned device across 40 seeds — ' + + 'the forced-contention scenario is not exercising the device lock', + ); + })); +} diff --git a/test/integration/nightly/concurrency-torture/bindings.ts b/test/integration/nightly/concurrency-torture/bindings.ts new file mode 100644 index 000000000..0105cedee --- /dev/null +++ b/test/integration/nightly/concurrency-torture/bindings.ts @@ -0,0 +1,169 @@ +// Production lock-plan + device bindings for the concurrency torture lane (#1416). +// +// The lane does NOT hand-write which locks an operation takes. It derives the +// plan from the REAL router primitive `resolveRequestExecutionLockKeys` +// (src/daemon/request-binding.ts), driven with a fake in-memory device +// inventory via the production `withDeviceInventoryProvider` seam. This is what +// makes the same-device serialization invariant revert-sensitive: if production +// stops returning a `device:` key (e.g. the router's same-device open +// serialization is reverted), the derived plan changes and the lane's overlap +// invariant fires. +// +// The plan is built the SAME way the daemon builds it in +// `createRequestExecutionScope`: gate on the production decision +// `shouldLockSessionExecution(command)`, and only then resolve keys via +// `resolveRequestExecutionLockKeys`. So the lane is revert-sensitive to BOTH +// production decision points — exempting a command from execution locking, or +// dropping the device key — even though the mutex GRANT itself stays modeled +// (by the deterministic scheduler) because `withKeyedLock`'s native microtask +// hand-off cannot be reproduced from a seed. + +import type { DeviceInfo } from '../../../../src/kernel/device.ts'; +import type { CommandFlags } from '../../../../src/core/dispatch-context.ts'; +import type { DaemonRequest } from '../../../../src/daemon/types.ts'; +import type { SessionStore } from '../../../../src/daemon/session-store.ts'; +import { resolveRequestExecutionLockKeys } from '../../../../src/daemon/request-binding.ts'; +import { shouldLockSessionExecution } from '../../../../src/daemon/daemon-command-registry.ts'; +import { PUBLIC_COMMANDS } from '../../../../src/command-catalog.ts'; +import { withDeviceInventoryProvider } from '../../../../src/core/dispatch-resolve.ts'; + +import type { LockKey } from './deterministic-scheduler.ts'; + +// Production command each modeled op routes as, so the lane consumes the real +// `shouldLockSessionExecution` gate rather than assuming every op locks. +const OPEN_COMMAND = PUBLIC_COMMANDS.open; +export const MUTATE_COMMAND = PUBLIC_COMMANDS.click; +export const CLOSE_COMMAND = PUBLIC_COMMANDS.close; + +export const DEVICE_POOL: readonly DeviceInfo[] = [ + { + platform: 'apple', + id: 'sim-a', + name: 'iPhone A', + kind: 'simulator', + appleOs: 'ios', + booted: true, + }, + { + platform: 'apple', + id: 'sim-b', + name: 'iPhone B', + kind: 'simulator', + appleOs: 'ios', + booted: true, + }, + { platform: 'android', id: 'emu-c', name: 'Pixel C', kind: 'emulator', booted: true }, +]; + +/** The lease scope for one opened session, kept so the harness can release it. */ +export type LeaseScope = { + leaseId: string; + tenantId: string; + runId: string; + leaseBackend: 'ios-simulator'; + deviceKey: string; + clientId: string; +}; + +/** + * Shadow record of one session instance the harness opened. `instanceId` is + * unique across the whole run even when a session NAME is reused, so a stale + * reference can never be mistaken for a live one. + */ +export type SessionInstance = { + instanceId: number; + name: string; + deviceId: string; + deviceKey: string; + lease: LeaseScope; + claim: { deviceKey: string; ownerToken: string; session: string }; + ownerClient: number; + dead: boolean; + reaped: boolean; + mutations: number; +}; + +/** The advisory-claim device key the daemon derives for a resolved device. */ +export function deviceClaimKey(device: DeviceInfo): string { + return `local:${device.platform}:${device.appleOs ?? 'none'}:${device.id}`; +} + +/** An explicit device selector that resolves to exactly `device` in the pool. */ +function selectorFlagsForDevice(device: DeviceInfo): CommandFlags { + return device.platform === 'android' + ? ({ platform: 'android', serial: device.id } as CommandFlags) + : ({ platform: 'ios', udid: device.id } as CommandFlags); +} + +function openRequest(device: DeviceInfo, sessionName: string): DaemonRequest { + return { + command: OPEN_COMMAND, + positionals: [], + token: 'torture', + session: sessionName, + flags: selectorFlagsForDevice(device), + } as DaemonRequest; +} + +function existingSessionRequest(command: string, sessionName: string): DaemonRequest { + return { + command, + positionals: [], + token: 'torture', + session: sessionName, + flags: {} as CommandFlags, + } as DaemonRequest; +} + +async function withPool(task: () => Promise): Promise { + return await withDeviceInventoryProvider(async () => [...DEVICE_POOL], task); +} + +/** + * Resolve a request's execution lock plan EXACTLY as `createRequestExecutionScope` + * does: skip locking entirely when the command is execution-lock-exempt, else + * resolve the keys through the production router. Reverting either production + * decision changes what this returns. + */ +async function resolveExecutionLockPlan( + req: DaemonRequest, + sessionName: string, + sessionStore: SessionStore, +): Promise { + if (!shouldLockSessionExecution(req.command)) return []; + return (await withPool(() => + resolveRequestExecutionLockKeys({ req, sessionName, sessionStore }), + )) as LockKey[]; +} + +/** + * Lock plan for a fresh open of `device` under `sessionName`, computed by the + * production router (`[session:, device:]` when the device resolves). + */ +export async function resolveOpenLockPlan( + sessionStore: SessionStore, + sessionName: string, + device: DeviceInfo, +): Promise { + return await resolveExecutionLockPlan( + openRequest(device, sessionName), + sessionName, + sessionStore, + ); +} + +/** + * Lock plan for `command` on an already-open session, computed by the production + * router (`[device:]`, read from the stored session's device). + */ +export async function resolveExistingLockPlan( + sessionStore: SessionStore, + sessionName: string, + command: string, +): Promise { + return await resolveExecutionLockPlan( + existingSessionRequest(command, sessionName), + sessionName, + sessionStore, + ); +} diff --git a/test/integration/nightly/concurrency-torture/claim-registry.ts b/test/integration/nightly/concurrency-torture/claim-registry.ts new file mode 100644 index 000000000..8976c017a --- /dev/null +++ b/test/integration/nightly/concurrency-torture/claim-registry.ts @@ -0,0 +1,73 @@ +// In-memory model of the advisory host-global device claim (`src/daemon/device-claims.ts`). +// +// The production claim is a filesystem lock file guarded by a process lock, keyed +// by `canonicalLocalDeviceKey`. That is real I/O and a real OS lock — neither is +// seed-reproducible, and the torture lane is explicitly not wall-clock/I/O stress +// (#1416 out-of-scope). This model preserves the *invariants* the harness asserts: +// - at most one live claim per device key; +// - a claim is released only by its owner (ownerToken match), mirroring +// `clearAdvisoryDeviceClaim`'s token/identity guard; +// - a dead owner's claim must be reclaimable (owner-death reap). +// Mutual exclusion of the acquire/clear critical sections is provided by the +// scheduler mutex in the harness, not here, so this stays a plain synchronous map. + +import crypto from 'node:crypto'; + +export type AdvisoryClaimOwnership = { + deviceKey: string; + ownerToken: string; + session: string; +}; + +type ClaimRecord = { + deviceKey: string; + ownerToken: string; + session: string; +}; + +export type ClaimAcquireResult = + | { ownership: AdvisoryClaimOwnership; conflict?: undefined } + | { ownership?: undefined; conflict: { session: string } }; + +export class InMemoryClaimRegistry { + private readonly claims = new Map(); + + /** + * Mirrors `acquireAdvisoryDeviceClaim`: re-acquiring your own claim is + * idempotent, a foreign live claim is reported as a conflict, and a free key + * is claimed with a fresh owner token. + */ + acquire(deviceKey: string, session: string): ClaimAcquireResult { + const existing = this.claims.get(deviceKey); + if (existing) { + if (existing.session === session) { + return { ownership: { deviceKey, ownerToken: existing.ownerToken, session } }; + } + return { conflict: { session: existing.session } }; + } + const record: ClaimRecord = { deviceKey, ownerToken: crypto.randomUUID(), session }; + this.claims.set(deviceKey, record); + return { ownership: { deviceKey, ownerToken: record.ownerToken, session } }; + } + + /** Mirrors `clearAdvisoryDeviceClaim`: only the token owner may clear. */ + clear(ownership: AdvisoryClaimOwnership | undefined): void { + if (!ownership) return; + const existing = this.claims.get(ownership.deviceKey); + if (!existing || existing.ownerToken !== ownership.ownerToken) return; + this.claims.delete(ownership.deviceKey); + } + + /** Owner of `deviceKey`, or undefined when free. */ + ownerSession(deviceKey: string): string | undefined { + return this.claims.get(deviceKey)?.session; + } + + /** Every live claim, for invariant checks. */ + snapshot(): { deviceKey: string; session: string }[] { + return [...this.claims.values()].map((claim) => ({ + deviceKey: claim.deviceKey, + session: claim.session, + })); + } +} diff --git a/test/integration/nightly/concurrency-torture/deterministic-scheduler.ts b/test/integration/nightly/concurrency-torture/deterministic-scheduler.ts new file mode 100644 index 000000000..f54e1b0a4 --- /dev/null +++ b/test/integration/nightly/concurrency-torture/deterministic-scheduler.ts @@ -0,0 +1,286 @@ +// Deterministic, seed-driven cooperative scheduler for the concurrency torture +// lane (#1416). +// +// Why this exists (issue review amendment, 2026-07-27): a seed alone does NOT +// reproduce Promise/event-loop interleavings. This is the "instrumented task +// queue / controlled dispatcher" the amendment requires — the single source of +// concurrency ordering in the harness. Every client runs as a *fiber* that only +// makes progress when the scheduler resumes it, and mutual exclusion between +// fibers is granted by the scheduler too (an integrated mutex). A given seed +// therefore fully determines execution order: which fiber steps next, and which +// waiter wins a contended lock. +// +// Design contract that makes this deterministic: +// - Fibers yield ONLY through scheduler-owned promises (`step`, `acquire`). +// - Everything a fiber does between yields is synchronous (the harness drives +// the real in-memory `SessionStore` / `LeaseRegistry`, which never await), +// so exactly one park-or-completion happens per resumed turn. +// - No wall clock, no timers, no real I/O: the microtask queue only ever holds +// continuations the scheduler itself resolved. +// +// See docs/agents/testing.md ("Concurrency torture lane") for the seed-replay +// flag and how failures are reproduced. + +type Deferred = { + readonly promise: Promise; + resolve(value: T): void; + reject(error: unknown): void; +}; + +function deferred(): Deferred { + let resolve!: (value: T) => void; + let reject!: (error: unknown) => void; + const promise = new Promise((res, rej) => { + resolve = res; + reject = rej; + }); + return { promise, resolve, reject }; +} + +/** A lock key the harness serializes on, mirroring `RequestExecutionLockKey`. */ +export type LockKey = `session:${string}` | `device:${string}`; + +/** The controls a fiber uses to cooperate with the scheduler. */ +export type Fiber = { + readonly id: number; + readonly name: string; + /** Yield a scheduling point; resolves when the scheduler next steps this fiber. */ + step(label?: string): Promise; + /** Acquire the mutex for `key`, parking (seed-ordered) until it is granted. */ + acquire(key: LockKey): Promise; + /** Release a mutex this fiber holds. */ + release(key: LockKey): void; + /** Run `task` while holding `key`, releasing on completion or throw. */ + withLock(key: LockKey, task: () => Promise): Promise; +}; + +type Mutex = { + holder: number | null; + readonly waiters: Set; +}; + +/** One scheduler decision, recorded so a failing interleaving can be printed. */ +export type SchedulerChoice = + | { kind: 'step'; fiber: number; label?: string } + | { kind: 'grant'; fiber: number; key: LockKey }; + +/** Serialize a decision trace to a canonical string for exact replay comparison. */ +export function serializeTrace(trace: readonly SchedulerChoice[]): string { + return trace + .map((choice) => + choice.kind === 'step' + ? `s:${choice.fiber}:${choice.label ?? ''}` + : `g:${choice.fiber}:${choice.key}`, + ) + .join('|'); +} + +export class SchedulerDeadlockError extends Error { + readonly liveFibers: readonly number[]; + readonly heldLocks: readonly { key: LockKey; holder: number; waiters: number[] }[]; + + constructor( + message: string, + liveFibers: readonly number[], + heldLocks: readonly { key: LockKey; holder: number; waiters: number[] }[], + ) { + super(message); + this.name = 'SchedulerDeadlockError'; + this.liveFibers = liveFibers; + this.heldLocks = heldLocks; + } +} + +export class DeterministicScheduler { + private readonly pickIndex: (bound: number) => number; + private readonly mutexes = new Map(); + private readonly wake = new Map void>(); + private readonly readySteppers = new Set(); + private readonly stepLabels = new Map(); + private readonly liveFibers = new Set(); + private readonly fiberNames = new Map(); + private turn: Deferred | null = null; + private firstError: unknown; + private readonly choiceLog: SchedulerChoice[] = []; + // How many times acquiring a key had to PARK because it was already held — + // i.e. genuine contention actually occurred (not just that a lock was taken). + private readonly contention = new Map(); + + constructor(pickIndex: (bound: number) => number) { + this.pickIndex = pickIndex; + } + + /** The full ordered decision trace — handy in failure output for replay. */ + get trace(): readonly SchedulerChoice[] { + return this.choiceLog; + } + + /** Per-key count of acquisitions that parked on a held lock (observed contention). */ + get contentionByKey(): ReadonlyMap { + return this.contention; + } + + /** + * Run every fiber to completion under seeded interleaving. Resolves once all + * fibers finish; rejects with the first fiber error, or a + * {@link SchedulerDeadlockError} if the fibers wedge (no runnable candidate + * while some are still alive). + */ + async run( + fibers: readonly { name: string; body: (fiber: Fiber) => Promise }[], + ): Promise { + const running = fibers.map((spec, id) => { + this.liveFibers.add(id); + this.fiberNames.set(id, spec.name); + const fiber = this.makeFiber(id, spec.name); + return spec + .body(fiber) + .catch((error: unknown) => { + this.firstError ??= error; + }) + .finally(() => { + this.liveFibers.delete(id); + this.resolveTurn(); + }); + }); + + // Let every fiber run up to its first yield before the first decision. + await Promise.resolve(); + + for (;;) { + const candidates = this.collectCandidates(); + if (candidates.length === 0) { + if (this.liveFibers.size === 0) break; + throw this.deadlock(); + } + const choice = candidates[this.pickIndex(candidates.length)] as SchedulerChoice; + this.choiceLog.push(choice); + this.turn = deferred(); + this.applyChoice(choice); + await this.turn.promise; + this.turn = null; + } + + await Promise.all(running); + if (this.firstError !== undefined) throw this.firstError; + } + + /** Assert the run left no lock held and no waiter parked (invariant: locks released). */ + assertQuiescent(): void { + const stuck = [...this.mutexes.entries()].filter( + ([, mutex]) => mutex.holder !== null || mutex.waiters.size > 0, + ); + if (stuck.length > 0) { + const detail = stuck + .map( + ([key, mutex]) => `${key}(holder=${String(mutex.holder)}, waiters=${mutex.waiters.size})`, + ) + .join(', '); + throw new Error(`scheduler not quiescent — locks still held/awaited: ${detail}`); + } + } + + private makeFiber(id: number, name: string): Fiber { + const step = (label?: string): Promise => { + const gate = deferred(); + this.wake.set(id, gate.resolve); + this.stepLabels.set(id, label); + this.readySteppers.add(id); + this.resolveTurn(); + return gate.promise; + }; + const acquire = (key: LockKey): Promise => { + const mutex = this.mutexFor(key); + if (mutex.holder === null && mutex.waiters.size === 0) { + // Uncontended: take it immediately and keep running this turn. Which + // fiber reaches a free lock first is already a seeded decision (the + // scheduler chose to step it), so no extra decision is needed here. + mutex.holder = id; + return Promise.resolve(); + } + const gate = deferred(); + this.wake.set(id, gate.resolve); + mutex.waiters.add(id); + this.contention.set(key, (this.contention.get(key) ?? 0) + 1); + this.resolveTurn(); + return gate.promise; + }; + const release = (key: LockKey): void => { + const mutex = this.mutexFor(key); + if (mutex.holder !== id) { + throw new Error( + `fiber ${name} released ${key} it does not hold (holder=${String(mutex.holder)})`, + ); + } + mutex.holder = null; + }; + const withLock = async (key: LockKey, task: () => Promise): Promise => { + await acquire(key); + try { + return await task(); + } finally { + release(key); + } + }; + return { id, name, step, acquire, release, withLock }; + } + + private collectCandidates(): SchedulerChoice[] { + const candidates: SchedulerChoice[] = []; + for (const fiber of this.readySteppers) { + candidates.push({ kind: 'step', fiber, label: this.stepLabels.get(fiber) }); + } + for (const [key, mutex] of this.mutexes) { + if (mutex.holder === null && mutex.waiters.size > 0) { + for (const fiber of mutex.waiters) { + candidates.push({ kind: 'grant', fiber, key }); + } + } + } + return candidates; + } + + private applyChoice(choice: SchedulerChoice): void { + if (choice.kind === 'step') { + this.readySteppers.delete(choice.fiber); + this.stepLabels.delete(choice.fiber); + } else { + const mutex = this.mutexFor(choice.key); + mutex.waiters.delete(choice.fiber); + mutex.holder = choice.fiber; + } + const wake = this.wake.get(choice.fiber); + this.wake.delete(choice.fiber); + wake?.(); + } + + private mutexFor(key: LockKey): Mutex { + let mutex = this.mutexes.get(key); + if (!mutex) { + mutex = { holder: null, waiters: new Set() }; + this.mutexes.set(key, mutex); + } + return mutex; + } + + private resolveTurn(): void { + this.turn?.resolve(); + } + + private deadlock(): SchedulerDeadlockError { + const heldLocks = [...this.mutexes.entries()] + .filter(([, mutex]) => mutex.holder !== null || mutex.waiters.size > 0) + .map(([key, mutex]) => ({ + key, + holder: mutex.holder ?? -1, + waiters: [...mutex.waiters], + })); + const live = [...this.liveFibers]; + const named = live.map((id) => `${id}:${this.fiberNames.get(id) ?? '?'}`).join(', '); + return new SchedulerDeadlockError( + `scheduler wedged with live fibers [${named}] and no runnable candidate — probable lock-order deadlock`, + live, + heldLocks, + ); + } +} diff --git a/test/integration/nightly/concurrency-torture/envelope.ts b/test/integration/nightly/concurrency-torture/envelope.ts new file mode 100644 index 000000000..dec5abceb --- /dev/null +++ b/test/integration/nightly/concurrency-torture/envelope.ts @@ -0,0 +1,77 @@ +// Standard scheduled-lane artifact envelope (#1430) for the concurrency torture +// lane (#1416). Builds the shared `LaneEnvelope` from scripts/lib/lane-envelope.ts +// so the #1430 health watcher parses one dialect, not a lane-local one; the +// lane's seed range / run count ride in the typed `data` payload, and the lane +// source hash maps onto the generic `configHash`. Written when TORTURE_ENVELOPE +// names an output path (set by the nightly workflow, which uploads it as an +// artifact); a no-op otherwise. + +import crypto from 'node:crypto'; +import fs from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { + laneEnvelope, + type LaneEnvelope, + type LaneResult, +} from '../../../../scripts/lib/lane-envelope.ts'; + +const LANE_ID = 'concurrency-torture'; + +export type TortureEnvelopeData = { + issue: number; + seedRange: { start: number; end: number }; + runs: number; +}; + +export type TortureEnvelope = LaneEnvelope; + +/** Content hash of the lane's own source, so config/tool drift is visible. */ +function laneSourceHash(): string { + const dir = path.dirname(fileURLToPath(import.meta.url)); + const files = [ + ...fs + .readdirSync(dir) + .filter((name) => name.endsWith('.ts')) + .map((name) => path.join(dir, name)), + path.join(dir, '..', 'concurrency-torture.test.ts'), + ].sort(); + const hash = crypto.createHash('sha256'); + for (const file of files) { + hash.update(path.basename(file)); + hash.update(fs.readFileSync(file)); + } + return hash.digest('hex'); +} + +export function buildEnvelope(params: { + seedStart: number; + runs: number; + startedAtMs: number; + result: LaneResult; +}): TortureEnvelope { + const end = params.seedStart + params.runs; + return laneEnvelope({ + lane: LANE_ID, + commit: process.env.GITHUB_SHA?.trim() || '', + tool: { node: process.version }, + configHash: laneSourceHash(), + seed: `${params.seedStart}-${end - 1}`, + startedAtMs: params.startedAtMs, + result: params.result, + data: { + issue: 1416, + seedRange: { start: params.seedStart, end }, + runs: params.runs, + }, + }); +} + +/** Write the envelope to TORTURE_ENVELOPE when set; returns the path or null. */ +export function writeEnvelopeIfRequested(envelope: TortureEnvelope): string | null { + const target = process.env.TORTURE_ENVELOPE?.trim(); + if (!target) return null; + fs.mkdirSync(path.dirname(target), { recursive: true }); + fs.writeFileSync(target, `${JSON.stringify(envelope, null, 2)}\n`); + return target; +} diff --git a/test/integration/nightly/concurrency-torture/harness.ts b/test/integration/nightly/concurrency-torture/harness.ts new file mode 100644 index 000000000..0a1af6f3e --- /dev/null +++ b/test/integration/nightly/concurrency-torture/harness.ts @@ -0,0 +1,499 @@ +// Seeded concurrency torture harness for session / lease / lock invariants (#1416). +// +// N logical clients drive randomized-but-seeded programs of +// open / mutate / close / takeover / kill against the REAL invariant-bearing +// daemon modules — `SessionStore` and `LeaseRegistry` — plus an in-memory model +// of the advisory device claim. All concurrency is routed through the +// `DeterministicScheduler`, so a seed fully determines the interleaving and any +// failure replays exactly (see docs/agents/testing.md). +// +// What is real vs modeled (issue review amendment "say which and why"): +// - REAL: `SessionStore` (session map/consistency) and `LeaseRegistry` (lease +// allocation, per-device exclusivity, release, scope checks). +// - REAL lock PLAN: every operation's lock keys come from the production +// router primitive `resolveRequestExecutionLockKeys` (see bindings.ts), +// driven with a fake device inventory. Only the mutex GRANT is modeled by +// the scheduler, because `withKeyedLock`'s native microtask hand-off cannot +// be reproduced from a seed. Reverting the router's same-device +// serialization therefore changes the derived plan and trips the overlap +// invariant. `withKeyedLock` reentrancy/cleanup stay covered by unit tests. +// - MODELED: the advisory device claim (`InMemoryClaimRegistry`) and process +// "kill"; the production claim is a filesystem/OS lock and real process +// death, both out of scope for this scheduling-torture lane. + +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; + +import type { DeviceInfo } from '../../../../src/kernel/device.ts'; +import { LeaseRegistry } from '../../../../src/daemon/lease-registry.ts'; +import { SessionStore } from '../../../../src/daemon/session-store.ts'; +import type { SessionState } from '../../../../src/daemon/types.ts'; +import { AppError } from '../../../../src/kernel/errors.ts'; + +import { makePrng, type Prng } from './prng.ts'; +import { + DeterministicScheduler, + SchedulerDeadlockError, + serializeTrace, + type Fiber, + type LockKey, +} from './deterministic-scheduler.ts'; +import { InMemoryClaimRegistry } from './claim-registry.ts'; +import { + CLOSE_COMMAND, + DEVICE_POOL, + MUTATE_COMMAND, + deviceClaimKey, + resolveExistingLockPlan, + resolveOpenLockPlan, + type LeaseScope, + type SessionInstance, +} from './bindings.ts'; +import { checkInvariants, type InvariantFailure } from './invariants.ts'; + +export type ClientOp = 'open' | 'mutate' | 'close' | 'takeover' | 'kill'; + +const ALL_OPS: readonly ClientOp[] = ['open', 'mutate', 'close', 'takeover', 'kill']; + +export type TortureConfig = { + seed: number; + clients?: number; + opsPerClient?: number; + // Deterministic overrides used by the forced same-device contention test. + program?: ClientOp[][]; + pinnedDevice?: DeviceInfo; +}; + +export type TortureRunResult = { + seed: number; + clients: number; + ops: number; + scheduleLength: number; + perDeviceMaxConcurrency: Record; + // Number of device-lock acquisitions that had to park on a held lock, i.e. + // genuine same-device contention that actually occurred this run. + deviceContention: number; + // Canonical serialization of the full scheduler decision trace, for exact + // replay comparison (equal seed ⇒ equal signature). + traceSignature: string; + failures: InvariantFailure[]; +}; + +export async function runTorture(config: TortureConfig): Promise { + const world = new TortureWorld(config); + return await world.run(); +} + +class TortureWorld { + private readonly prng: Prng; + private readonly scheduler: DeterministicScheduler; + private readonly sessionStore: SessionStore; + private readonly leaseRegistry: LeaseRegistry; + private readonly claims = new InMemoryClaimRegistry(); + private readonly stateRoot: string; + + private readonly clientCount: number; + private readonly opsPerClient: number; + + // Shadow bookkeeping. + private readonly instances = new Map(); + private readonly deviceOwner = new Map(); + private readonly clientInstance = new Map(); + private nextInstanceId = 0; + // Every open gets a UNIQUE session name. Production keys a session by name, so + // reusing a name across opens (e.g. kill then re-open on another device) would + // let the store's device for that name diverge from the shadow instance's, + // making the router derive a lock plan for the wrong device. Unique names keep + // store and shadow in lockstep — exactly one device per session name. + private nextSessionSeq = 0; + private activeClients: number; + + // Serialization instrumentation: concurrent critical sections per device. + private readonly deviceActive = new Map(); + private readonly deviceMaxActive = new Map(); + + // Invariant violations detected mid-run (vs. the post-run structural checks). + private readonly runtimeFailures: InvariantFailure[] = []; + + private readonly config: TortureConfig; + + constructor(config: TortureConfig) { + this.config = config; + this.prng = makePrng(config.seed); + this.scheduler = new DeterministicScheduler((bound) => this.prng.int(bound)); + this.clientCount = config.program?.length ?? config.clients ?? 2 + this.prng.int(4); // 2..5 + this.opsPerClient = config.opsPerClient ?? 6 + this.prng.int(9); // 6..14 + this.activeClients = this.clientCount; + this.stateRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'agent-device-torture-')); + this.sessionStore = new SessionStore(path.join(this.stateRoot, 'sessions')); + this.leaseRegistry = new LeaseRegistry({ maxLeaseTtlMs: 10 * 60_000 }); + } + + async run(): Promise { + const programs = this.buildPrograms(); + const fibers = programs.map((ops, client) => ({ + name: `client-${client}`, + body: (fiber: Fiber) => this.runClient(fiber, client, ops), + })); + fibers.push({ name: 'reaper', body: (fiber: Fiber) => this.runReaper(fiber) }); + + const failures: InvariantFailure[] = []; + try { + await this.scheduler.run(fibers); + } catch (error) { + if (error instanceof SchedulerDeadlockError) { + failures.push({ invariant: 'no deadlock', detail: error.message }); + } else { + throw error; + } + } + + failures.push(...this.runtimeFailures); + failures.push( + ...checkInvariants({ + sessionStore: this.sessionStore, + leaseRegistry: this.leaseRegistry, + claimSnapshot: this.claims.snapshot(), + claimOwner: (deviceKey) => this.claims.ownerSession(deviceKey), + instances: [...this.instances.values()], + isInstanceLive: (instance) => this.isInstanceLive(instance), + deviceActive: this.deviceActive, + deviceMaxActive: this.deviceMaxActive, + assertQuiescent: () => this.scheduler.assertQuiescent(), + }), + ); + this.cleanup(); + + return { + seed: this.config.seed, + clients: this.clientCount, + ops: this.opsPerClient * this.clientCount, + scheduleLength: this.scheduler.trace.length, + perDeviceMaxConcurrency: Object.fromEntries(this.deviceMaxActive), + deviceContention: this.deviceContentionTotal(), + traceSignature: serializeTrace(this.scheduler.trace), + failures, + }; + } + + private deviceContentionTotal(): number { + let total = 0; + for (const [key, count] of this.scheduler.contentionByKey) { + if (key.startsWith('device:')) total += count; + } + return total; + } + + // --- program generation ------------------------------------------------- + + private buildPrograms(): ClientOp[][] { + if (this.config.program) return this.config.program.map((ops) => [...ops]); + return Array.from({ length: this.clientCount }, () => { + const ops: ClientOp[] = []; + for (let i = 0; i < this.opsPerClient; i += 1) { + ops.push(this.prng.pick(ALL_OPS)); + } + return ops; + }); + } + + // --- client fiber ------------------------------------------------------- + + private async runClient(fiber: Fiber, client: number, ops: ClientOp[]): Promise { + this.clientInstance.set(client, undefined); + try { + for (const op of ops) { + await fiber.step(`client-${client}:${op}`); + await this.dispatch(fiber, client, op); + } + } finally { + this.activeClients -= 1; + } + } + + private async dispatch(fiber: Fiber, client: number, op: ClientOp): Promise { + switch (op) { + case 'open': + return await this.doOpen(fiber, client, false); + case 'takeover': + return await this.doOpen(fiber, client, true); + case 'mutate': + return await this.doMutate(fiber, client); + case 'close': + return await this.doClose(fiber, client); + case 'kill': + return await this.doKill(fiber, client); + } + } + + /** + * Acquire a production-derived lock plan in order, yielding a scheduling point + * after each grant, then run `body` inside the innermost critical section. + */ + private async withLockPlan( + fiber: Fiber, + plan: readonly LockKey[], + label: string, + body: () => Promise, + ): Promise { + const acquireFrom = async (index: number): Promise => { + if (index >= plan.length) return await body(); + const key = plan[index] as LockKey; + await fiber.withLock(key, async () => { + await fiber.step(`${label}-locked:${key}`); + await acquireFrom(index + 1); + }); + }; + await acquireFrom(0); + } + + private async doOpen(fiber: Fiber, client: number, takeover: boolean): Promise { + if (this.liveInstanceOf(client)) return; // already managing a session + const device = this.config.pinnedDevice ?? this.prng.pick(DEVICE_POOL); + const name = `s${client}-${this.nextSessionSeq++}`; + const plan = await resolveOpenLockPlan(this.sessionStore, name, device); + await this.withLockPlan(fiber, plan, `open-${client}`, async () => { + await this.enterDeviceCritical(fiber, device.id); + try { + const current = this.deviceOwner.get(device.id); + if (current !== undefined) { + const owner = this.instances.get(current); + if (owner && !owner.reaped) { + if (takeover || owner.dead) { + this.reapInstance(owner); + } else { + return; // device busy with a live owner; open is a no-op + } + } + } + this.openSession(client, name, device); + } finally { + this.exitDeviceCritical(device.id); + } + }); + } + + private async doMutate(fiber: Fiber, client: number): Promise { + const instance = this.liveInstanceOf(client); + if (!instance) return; + const plan = await resolveExistingLockPlan(this.sessionStore, instance.name, MUTATE_COMMAND); + await this.withLockPlan(fiber, plan, `mutate-${client}`, async () => { + await this.enterDeviceCritical(fiber, instance.deviceId); + try { + if (!this.isInstanceLive(instance)) return; // evicted/reaped meanwhile + // Heartbeat through the real registry: identity must be preserved and + // must not resolve to a different session's lease (cross-session bleed). + const refreshed = this.leaseRegistry.heartbeatLease({ + leaseId: instance.lease.leaseId, + tenantId: instance.lease.tenantId, + runId: instance.lease.runId, + leaseBackend: instance.lease.leaseBackend, + deviceKey: instance.lease.deviceKey, + clientId: instance.lease.clientId, + }); + if (refreshed.leaseId !== instance.lease.leaseId) { + this.fail( + 'no cross-session bleed', + `heartbeat returned foreign lease ${refreshed.leaseId}`, + ); + } + instance.mutations += 1; + } finally { + this.exitDeviceCritical(instance.deviceId); + } + }); + } + + private fail(invariant: string, detail: string): void { + this.runtimeFailures.push({ invariant, detail }); + } + + private async doClose(fiber: Fiber, client: number): Promise { + const instance = this.liveInstanceOf(client); + if (!instance) return; + const plan = await resolveExistingLockPlan(this.sessionStore, instance.name, CLOSE_COMMAND); + await this.withLockPlan(fiber, plan, `close-${client}`, async () => { + await this.enterDeviceCritical(fiber, instance.deviceId); + try { + if (this.isInstanceLive(instance)) this.reapInstance(instance); + this.clientInstance.set(client, undefined); + } finally { + this.exitDeviceCritical(instance.deviceId); + } + }); + } + + private async doKill(fiber: Fiber, client: number): Promise { + const instance = this.liveInstanceOf(client); + if (!instance) return; + // Owner death: the client forgets its session WITHOUT releasing anything. + // The lease/claim/store entry become an orphan that only the reaper (or a + // takeover) may reclaim — the "every lock released after owner death" case. + await fiber.step(`kill:${client}`); + instance.dead = true; + this.clientInstance.set(client, undefined); + } + + // --- reaper fiber ------------------------------------------------------- + + private async runReaper(fiber: Fiber): Promise { + for (;;) { + await fiber.step('reaper'); + const orphan = this.findOrphan(); + if (!orphan) { + if (this.activeClients <= 0 && !this.findOrphan()) return; + continue; + } + const plan = await resolveExistingLockPlan(this.sessionStore, orphan.name, CLOSE_COMMAND); + await this.withLockPlan(fiber, plan, 'reaper', async () => { + await this.enterDeviceCritical(fiber, orphan.deviceId); + try { + if (this.isInstanceLive(orphan) && orphan.dead) this.reapInstance(orphan); + } finally { + this.exitDeviceCritical(orphan.deviceId); + } + }); + } + } + + private findOrphan(): SessionInstance | undefined { + for (const instance of this.instances.values()) { + if (instance.dead && !instance.reaped) return instance; + } + return undefined; + } + + // --- state transitions (all under the appropriate locks) ---------------- + + private openSession(client: number, name: string, device: DeviceInfo): void { + const deviceKey = deviceClaimKey(device); + const lease = this.leaseRegistry.allocateLease({ + tenantId: `t${client}`, + runId: `r${client}`, + leaseBackend: 'ios-simulator', + deviceKey, + clientId: `c${client}`, + }); + const claimResult = this.claims.acquire(deviceKey, name); + if (!claimResult.ownership) { + // A live claim under a held device lock means the store/claim disagreed. + this.fail('session store consistent', `claim conflict opening ${name} on ${device.id}`); + this.releaseLease(lease.leaseId, client, deviceKey); + return; + } + const leaseScope: LeaseScope = { + leaseId: lease.leaseId, + tenantId: lease.tenantId, + runId: lease.runId, + leaseBackend: 'ios-simulator', + deviceKey, + clientId: `c${client}`, + }; + const state: SessionState = { + name, + device, + createdAt: Date.now(), + actions: [], + lease: { ...leaseScope, expiresAt: lease.expiresAt }, + deviceClaim: { + deviceKey, + ownerToken: claimResult.ownership.ownerToken, + ownerPid: process.pid, + ownerStartTime: null, + }, + }; + this.sessionStore.set(name, state); + const instanceId = this.nextInstanceId++; + const instance: SessionInstance = { + instanceId, + name, + deviceId: device.id, + deviceKey, + lease: leaseScope, + claim: claimResult.ownership, + ownerClient: client, + dead: false, + reaped: false, + mutations: 0, + }; + this.instances.set(instanceId, instance); + this.deviceOwner.set(device.id, instanceId); + this.clientInstance.set(client, instanceId); + } + + private releaseLease(leaseId: string, client: number, deviceKey: string): void { + this.leaseRegistry.releaseLease({ + leaseId, + tenantId: `t${client}`, + runId: `r${client}`, + leaseBackend: 'ios-simulator', + deviceKey, + clientId: `c${client}`, + }); + } + + private reapInstance(instance: SessionInstance): void { + if (instance.reaped) return; + try { + this.leaseRegistry.releaseLease({ + leaseId: instance.lease.leaseId, + tenantId: instance.lease.tenantId, + runId: instance.lease.runId, + leaseBackend: instance.lease.leaseBackend, + deviceKey: instance.lease.deviceKey, + clientId: instance.lease.clientId, + }); + } catch (error) { + if (!(error instanceof AppError)) throw error; + this.fail('no leaked leases', `release threw for ${instance.name}: ${error.message}`); + } + this.claims.clear(instance.claim); + const stored = this.sessionStore.get(instance.name); + if (stored?.lease?.leaseId === instance.lease.leaseId) { + this.sessionStore.delete(instance.name); + } + instance.reaped = true; + if (this.deviceOwner.get(instance.deviceId) === instance.instanceId) { + this.deviceOwner.delete(instance.deviceId); + } + if (this.clientInstance.get(instance.ownerClient) === instance.instanceId) { + this.clientInstance.set(instance.ownerClient, undefined); + } + } + + private isInstanceLive(instance: SessionInstance): boolean { + return !instance.reaped && this.deviceOwner.get(instance.deviceId) === instance.instanceId; + } + + private liveInstanceOf(client: number): SessionInstance | undefined { + const id = this.clientInstance.get(client); + if (id === undefined) return undefined; + const instance = this.instances.get(id); + return instance && this.isInstanceLive(instance) ? instance : undefined; + } + + // --- serialization instrumentation -------------------------------------- + + private async enterDeviceCritical(fiber: Fiber, deviceId: string): Promise { + const active = (this.deviceActive.get(deviceId) ?? 0) + 1; + this.deviceActive.set(deviceId, active); + this.deviceMaxActive.set(deviceId, Math.max(active, this.deviceMaxActive.get(deviceId) ?? 0)); + // Yield WHILE inside the device critical section: if same-device + // serialization were broken, another fiber would interleave here and push + // the active count above 1. + await fiber.step(`device-critical:${deviceId}`); + } + + private exitDeviceCritical(deviceId: string): void { + this.deviceActive.set(deviceId, (this.deviceActive.get(deviceId) ?? 1) - 1); + } + + private cleanup(): void { + try { + fs.rmSync(this.stateRoot, { recursive: true, force: true }); + } catch { + // Best effort; a leftover temp dir never fails the invariant check. + } + } +} diff --git a/test/integration/nightly/concurrency-torture/invariants.ts b/test/integration/nightly/concurrency-torture/invariants.ts new file mode 100644 index 000000000..06afe30a0 --- /dev/null +++ b/test/integration/nightly/concurrency-torture/invariants.ts @@ -0,0 +1,226 @@ +// Invariant checks for the concurrency torture lane (#1416), asserted after +// every seeded run. Kept separate from the world/operation logic so the +// real/model boundary each invariant relies on stays auditable in one read. +// +// - same-device serialization: critical sections for one device never overlap +// - every lock released after owner death: scheduler quiescent (no held/parked) +// - session store consistent: one session per device, shadow/store parity +// - no leaked leases: active leases ⇔ live sessions, one per device key +// - no leaked claims: claims ⇔ live sessions, one per device key, all held +// - no cross-session bleed: each stored session carries exactly its own +// lease/claim/device, never a concurrent session's + +import type { LeaseRegistry } from '../../../../src/daemon/lease-registry.ts'; +import type { SessionStore } from '../../../../src/daemon/session-store.ts'; + +import type { SessionInstance } from './bindings.ts'; + +export type ClaimSnapshot = { deviceKey: string; session: string }; + +export type InvariantFailure = { + invariant: string; + detail: string; +}; + +/** Read-only view of the world the invariant checks inspect. */ +export type WorldView = { + sessionStore: SessionStore; + leaseRegistry: LeaseRegistry; + claimSnapshot: readonly ClaimSnapshot[]; + claimOwner(deviceKey: string): string | undefined; + instances: readonly SessionInstance[]; + isInstanceLive(instance: SessionInstance): boolean; + deviceActive: ReadonlyMap; + deviceMaxActive: ReadonlyMap; + assertQuiescent(): void; +}; + +export function checkInvariants(view: WorldView): InvariantFailure[] { + const checker = new InvariantChecker(view); + return checker.run(); +} + +class InvariantChecker { + private readonly failures: InvariantFailure[] = []; + private readonly view: WorldView; + + constructor(view: WorldView) { + this.view = view; + } + + run(): InvariantFailure[] { + this.checkSerialization(); + this.checkLocksReleased(); + this.checkStoreConsistency(); + this.checkNoLeakedLeases(); + this.checkNoLeakedClaims(); + this.checkNoCrossSessionBleed(); + return this.failures; + } + + private fail(invariant: string, detail: string): void { + this.failures.push({ invariant, detail }); + } + + private liveInstances(): SessionInstance[] { + return this.view.instances.filter((instance) => this.view.isInstanceLive(instance)); + } + + /** Flags any device key that appears more than once across `deviceKeys`. */ + private assertUniquePerDevice( + deviceKeys: readonly string[], + invariant: string, + noun: string, + ): void { + const byDevice = new Map(); + for (const key of deviceKeys) byDevice.set(key, (byDevice.get(key) ?? 0) + 1); + for (const [deviceKey, count] of byDevice) { + if (count > 1) this.fail(invariant, `device ${deviceKey} has ${count} ${noun}`); + } + } + + private checkSerialization(): void { + for (const [deviceId, max] of this.view.deviceMaxActive) { + if (max > 1) { + this.fail( + 'same-device serialization', + `device ${deviceId} had ${max} overlapping critical sections`, + ); + } + } + for (const [deviceId, active] of this.view.deviceActive) { + if (active !== 0) { + this.fail( + 'same-device serialization', + `device ${deviceId} left ${active} critical sections open`, + ); + } + } + } + + private checkLocksReleased(): void { + try { + this.view.assertQuiescent(); + } catch (error) { + this.fail('every lock released after owner death', (error as Error).message); + } + } + + private checkStoreConsistency(): void { + this.checkStoredSessionsUnique(); + this.checkShadowStoreParity(); + } + + private checkStoredSessionsUnique(): void { + const seenDevices = new Map(); + for (const session of this.view.sessionStore.values()) { + if (session.name !== this.view.sessionStore.get(session.name)?.name) { + this.fail('session store consistent', `session ${session.name} key/name mismatch`); + } + const priorName = seenDevices.get(session.device.id); + if (priorName) { + this.fail( + 'session store consistent', + `device ${session.device.id} bound to both ${priorName} and ${session.name}`, + ); + } + seenDevices.set(session.device.id, session.name); + } + } + + private checkShadowStoreParity(): void { + const liveInstances = this.liveInstances(); + for (const instance of liveInstances) { + const stored = this.view.sessionStore.get(instance.name); + if (!stored) { + this.fail('session store consistent', `live instance ${instance.name} missing from store`); + } else if (stored.lease?.leaseId !== instance.lease.leaseId) { + this.fail('session store consistent', `stored ${instance.name} lease != shadow lease`); + } + } + const storeCount = this.view.sessionStore.toArray().length; + if (storeCount !== liveInstances.length) { + this.fail( + 'session store consistent', + `store has ${storeCount} sessions, shadow expects ${liveInstances.length}`, + ); + } + } + + private checkNoLeakedLeases(): void { + const active = this.view.leaseRegistry.listActiveLeases(); + const liveLeaseIds = new Set(this.liveInstances().map((instance) => instance.lease.leaseId)); + for (const lease of active) { + if (!liveLeaseIds.has(lease.leaseId)) { + this.fail( + 'no leaked leases', + `lease ${lease.leaseId} (device ${lease.deviceKey}) has no live session`, + ); + } + } + for (const leaseId of liveLeaseIds) { + if (!active.some((lease) => lease.leaseId === leaseId)) { + this.fail('no leaked leases', `live session lease ${leaseId} missing from registry`); + } + } + // Device exclusivity: at most one active lease per device key. + const deviceKeys = active + .map((lease) => lease.deviceKey) + .filter((key): key is string => Boolean(key)); + this.assertUniquePerDevice(deviceKeys, 'no leaked leases', 'active leases'); + } + + private checkNoLeakedClaims(): void { + const claims = this.view.claimSnapshot; + for (const claim of claims) { + if (!this.view.sessionStore.get(claim.session)) { + this.fail( + 'no leaked claims', + `claim on ${claim.deviceKey} owned by dead session ${claim.session}`, + ); + } + } + this.assertUniquePerDevice( + claims.map((claim) => claim.deviceKey), + 'no leaked claims', + 'claims', + ); + this.checkLiveClaimsHeld(); + } + + private checkLiveClaimsHeld(): void { + for (const session of this.view.sessionStore.values()) { + if (!session.deviceClaim) continue; + const owner = this.view.claimOwner(session.deviceClaim.deviceKey); + if (owner !== session.name) { + this.fail( + 'no leaked claims', + `session ${session.name} claim not held (owner=${String(owner)})`, + ); + } + } + } + + private checkNoCrossSessionBleed(): void { + for (const instance of this.liveInstances()) { + if (!this.view.isInstanceLive(instance)) continue; + const stored = this.view.sessionStore.get(instance.name); + if (!stored) continue; + if (stored.device.id !== instance.deviceId) { + this.fail( + 'no cross-session bleed', + `${instance.name} device ${stored.device.id} != ${instance.deviceId}`, + ); + } + if (stored.deviceClaim?.deviceKey !== instance.deviceKey) { + this.fail('no cross-session bleed', `${instance.name} claim key mismatch`); + } + if (stored.lease?.clientId !== `c${instance.ownerClient}`) { + this.fail( + 'no cross-session bleed', + `${instance.name} lease clientId != owner c${instance.ownerClient}`, + ); + } + } + } +} diff --git a/test/integration/nightly/concurrency-torture/prng.ts b/test/integration/nightly/concurrency-torture/prng.ts new file mode 100644 index 000000000..196b5e3ac --- /dev/null +++ b/test/integration/nightly/concurrency-torture/prng.ts @@ -0,0 +1,71 @@ +// Seeded, deterministic PRNG for the concurrency torture lane (#1416). +// +// A seed alone cannot reproduce Node's Promise/event-loop interleavings, so the +// torture harness never reads `Math.random` or the wall clock: every random +// choice (client count, op programs, and — critically — which fiber the +// scheduler steps next) is drawn from this generator. Replaying a run is then +// exactly "construct the same seed and draw in the same order", which the +// scheduler guarantees by being the sole source of concurrency ordering. +// +// splitmix32: a small, well-distributed 32-bit generator. Chosen over a LCG so +// low-entropy seeds (0, 1, 2, …) still produce well-mixed streams — the lane +// enumerates seeds `0..runs`, so seed 0 must not degenerate. + +export type Prng = { + /** Next uint32 in the stream. */ + uint32(): number; + /** Uniform integer in `[0, bound)`. `bound` must be a positive integer. */ + int(bound: number): number; + /** Uniform element of `items`. Throws on an empty array. */ + pick(items: readonly T[]): T; + /** True with probability `p` (default 0.5). */ + bool(p?: number): boolean; + /** In-place Fisher–Yates shuffle, returning the same array. */ + shuffle(items: T[]): T[]; +}; + +export function makePrng(seed: number): Prng { + let state = seed >>> 0; + + const uint32 = (): number => { + state = (state + 0x9e3779b9) | 0; + let t = state ^ (state >>> 16); + t = Math.imul(t, 0x21f0aaad); + t = t ^ (t >>> 15); + t = Math.imul(t, 0x735a2d97); + t = t ^ (t >>> 15); + return t >>> 0; + }; + + const int = (bound: number): number => { + if (!Number.isInteger(bound) || bound <= 0) { + throw new Error(`prng.int bound must be a positive integer, got ${String(bound)}`); + } + // Rejection sampling keeps the distribution uniform even when `bound` does + // not divide 2^32; the stream stays deterministic because rejected draws + // still advance the generator identically on replay. + const limit = Math.floor(0x1_0000_0000 / bound) * bound; + let value = uint32(); + while (value >= limit) value = uint32(); + return value % bound; + }; + + const pick = (items: readonly T[]): T => { + if (items.length === 0) throw new Error('prng.pick requires a non-empty array'); + return items[int(items.length)] as T; + }; + + const bool = (p = 0.5): boolean => uint32() / 0x1_0000_0000 < p; + + const shuffle = (items: T[]): T[] => { + for (let i = items.length - 1; i > 0; i -= 1) { + const j = int(i + 1); + const tmp = items[i] as T; + items[i] = items[j] as T; + items[j] = tmp; + } + return items; + }; + + return { uint32, int, pick, bool, shuffle }; +} diff --git a/test/integration/nightly/concurrency-torture/real-scope-serialization.ts b/test/integration/nightly/concurrency-torture/real-scope-serialization.ts new file mode 100644 index 000000000..32bdcd555 --- /dev/null +++ b/test/integration/nightly/concurrency-torture/real-scope-serialization.ts @@ -0,0 +1,82 @@ +// Real-scope same-device serialization guard for the concurrency torture lane +// (#1416). The seeded sweep models the mutex GRANT (a seed cannot reproduce +// `withKeyedLock`'s native microtask hand-off), so on its own it could stay +// green if the PRODUCTION lock APPLICATION path regressed. This guard closes +// that gap by driving two concurrent requests for the SAME device through the +// real `createRequestExecutionScope().runLocked()` — i.e. the actual +// `withRequestExecutionLocks` → `withKeyedLock` machinery, not a model — and +// asserting their critical sections never overlap. It is intentionally NOT +// seeded: it exercises real event-loop scheduling. Disabling production locking +// (`shouldLockSessionExecution` → false, or dropping the `device:` key) makes +// two sections overlap and trips this assertion. + +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; + +import { createRequestExecutionScope } from '../../../../src/daemon/request-execution-scope.ts'; +import { LeaseRegistry } from '../../../../src/daemon/lease-registry.ts'; +import { SessionStore } from '../../../../src/daemon/session-store.ts'; +import { withDeviceInventoryProvider } from '../../../../src/core/dispatch-resolve.ts'; +import type { CommandFlags } from '../../../../src/core/dispatch-context.ts'; +import type { DaemonRequest } from '../../../../src/daemon/types.ts'; + +import { DEVICE_POOL } from './bindings.ts'; + +function openRequest(sessionName: string, deviceId: string): DaemonRequest { + return { + command: 'open', + positionals: [], + token: 'torture', + session: sessionName, + flags: { platform: 'ios', udid: deviceId } as CommandFlags, + } as DaemonRequest; +} + +/** + * Drive `concurrency` fresh opens of ONE device concurrently through the real + * request execution scope and return the maximum number of critical sections + * that were ever active at once. Production serialization ⇒ exactly 1. + */ +export async function measureRealScopeMaxOverlap(concurrency: number): Promise { + const device = DEVICE_POOL[0]; + assert.ok(device, 'DEVICE_POOL must not be empty'); + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'torture-real-scope-')); + const sessionStore = new SessionStore(dir); + const leaseRegistry = new LeaseRegistry(); + + let active = 0; + let maxActive = 0; + const criticalSection = async (): Promise => { + active += 1; + maxActive = Math.max(maxActive, active); + // Yield across several event-loop turns so a NON-serializing lock would let + // a second section observe `active > 1` here. + for (let i = 0; i < 5; i += 1) await Promise.resolve(); + await new Promise((resolve) => setImmediate(resolve)); + active -= 1; + }; + + try { + await withDeviceInventoryProvider( + async () => [device], + async () => { + const scopes = await Promise.all( + Array.from({ length: concurrency }, (_, i) => + createRequestExecutionScope({ + req: openRequest(`real-${i}`, device.id), + sessionStore, + leaseRegistry, + }), + ), + ); + await Promise.all(scopes.map((scope) => scope.runLocked(criticalSection))); + }, + ); + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } + + return maxActive; +}