From a4efad510b4df439f397765769ce2f860c1571e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sat, 3 Oct 2026 05:17:29 +0200 Subject: [PATCH 01/20] refactor: resolve session artifact paths without a store --- packages/host-kit/src/session-paths.test.ts | 19 +++++++++ .../__tests__/session-artifact-paths.test.ts | 29 ++++++++++++++ src/daemon/__tests__/session-store.test.ts | 40 ------------------- .../device/device-claim-owner-recovery.ts | 13 +++--- src/daemon/handlers/session-app-deployment.ts | 7 ++-- src/daemon/handlers/trace-runtime.ts | 9 ++--- src/daemon/screenshot-runtime.ts | 4 +- src/daemon/session-artifact-paths.ts | 27 ++++++++++++- .../internal/session-perf-runtime.ts | 11 ++--- src/daemon/session-store.ts | 32 ++++----------- 10 files changed, 106 insertions(+), 85 deletions(-) create mode 100644 packages/host-kit/src/session-paths.test.ts create mode 100644 src/daemon/__tests__/session-artifact-paths.test.ts diff --git a/packages/host-kit/src/session-paths.test.ts b/packages/host-kit/src/session-paths.test.ts new file mode 100644 index 0000000000..be0c8d3116 --- /dev/null +++ b/packages/host-kit/src/session-paths.test.ts @@ -0,0 +1,19 @@ +import { test } from 'vitest'; +import assert from 'node:assert/strict'; +// oxlint-disable-next-line no-restricted-imports -- asserts a path under os.homedir +import os from 'node:os'; +import path from 'node:path'; +import { expandSessionPath } from './session-paths.ts'; + +test('expandSessionPath resolves tilde, relative-with-cwd, and absolute paths', () => { + const homePath = expandSessionPath('~/flows/replay.ad'); + assert.equal(homePath.startsWith(os.homedir()), true); + assert.equal(homePath.endsWith(path.join('flows', 'replay.ad')), true); + + const relativePath = expandSessionPath('workflows/replay.ad', '/tmp/agent-device-cwd'); + assert.equal(relativePath, path.resolve('/tmp/agent-device-cwd', 'workflows/replay.ad')); + + const absoluteInput = path.resolve('/tmp', 'agent-device-absolute.ad'); + const absolutePath = expandSessionPath(absoluteInput, '/tmp/ignored-cwd'); + assert.equal(absolutePath, absoluteInput); +}); diff --git a/src/daemon/__tests__/session-artifact-paths.test.ts b/src/daemon/__tests__/session-artifact-paths.test.ts new file mode 100644 index 0000000000..c76caa9ffb --- /dev/null +++ b/src/daemon/__tests__/session-artifact-paths.test.ts @@ -0,0 +1,29 @@ +import { test } from 'vitest'; +import assert from 'node:assert/strict'; +import path from 'node:path'; +import { AppError } from '@agent-device/kernel/errors'; +import { resolveSessionDir } from '../session-artifact-paths.ts'; +import { mkdtempForTestSync } from '../../__tests__/test-utils/tmp-dir.ts'; + +test('resolveSessionDir keeps every session dir beneath the sessions dir', () => { + const sessionsDir = path.join( + mkdtempForTestSync('agent-device-tests'), + 'agent-device-tests', + 'sessions', + ); + assert.equal(resolveSessionDir(sessionsDir, 'a/b:c d'), path.join(sessionsDir, 'a_b_c_d')); + // `.` and `..` survive `safeSessionName` unchanged, so without an explicit + // refusal `path.join` resolves them to the sessions dir itself and its parent + // (the daemon state dir): a remote caller's `--session ..` would then land + // app.log / runner.log / requests/*.ndjson outside the sessions tree. + for (const name of ['.', '..', '']) { + assert.throws( + () => resolveSessionDir(sessionsDir, name), + (error: unknown) => + error instanceof AppError && + error.code === 'INVALID_ARGS' && + /session name/i.test(error.message), + `expected resolveSessionDir(${JSON.stringify(name)}) to reject`, + ); + } +}); diff --git a/src/daemon/__tests__/session-store.test.ts b/src/daemon/__tests__/session-store.test.ts index 05fe386324..5eddff721d 100644 --- a/src/daemon/__tests__/session-store.test.ts +++ b/src/daemon/__tests__/session-store.test.ts @@ -1,10 +1,7 @@ import { test } from 'vitest'; import assert from 'node:assert/strict'; import fs from 'node:fs'; -// oxlint-disable-next-line no-restricted-imports -- asserts a path under os.homedir -import os from 'node:os'; import path from 'node:path'; -import { AppError } from '@agent-device/kernel/errors'; import { SessionStore } from '../session-store.ts'; import type { SessionState } from '../session-state.ts'; import { buildRequestFinishedEvent } from '@agent-device/session-journal/session-event-log'; @@ -96,19 +93,6 @@ function assertScriptMatches(script: string, patterns: RegExp[]): void { } } -test('expandHome resolves tilde, relative-with-cwd, and absolute paths', () => { - const homePath = SessionStore.expandHome('~/flows/replay.ad'); - assert.equal(homePath.startsWith(os.homedir()), true); - assert.equal(homePath.endsWith(path.join('flows', 'replay.ad')), true); - - const relativePath = SessionStore.expandHome('workflows/replay.ad', '/tmp/agent-device-cwd'); - assert.equal(relativePath, path.resolve('/tmp/agent-device-cwd', 'workflows/replay.ad')); - - const absoluteInput = path.resolve('/tmp', 'agent-device-absolute.ad'); - const absolutePath = SessionStore.expandHome(absoluteInput, '/tmp/ignored-cwd'); - assert.equal(absolutePath, absoluteInput); -}); - test('defaultTracePath sanitizes session name', () => { const store = new SessionStore( path.join(mkdtempForTestSync('agent-device-tests'), 'agent-device-tests'), @@ -119,30 +103,6 @@ test('defaultTracePath sanitizes session name', () => { assert.match(tracePath, /\.trace\.log$/); }); -test('resolveSessionDir keeps every session dir beneath the sessions dir', () => { - const sessionsDir = path.join( - mkdtempForTestSync('agent-device-tests'), - 'agent-device-tests', - 'sessions', - ); - const store = new SessionStore(sessionsDir); - assert.equal(store.resolveSessionDir('a/b:c d'), path.join(sessionsDir, 'a_b_c_d')); - // `.` and `..` survive `safeSessionName` unchanged, so without an explicit - // refusal `path.join` resolves them to the sessions dir itself and its parent - // (the daemon state dir): a remote caller's `--session ..` would then land - // app.log / runner.log / requests/*.ndjson outside the sessions tree. - for (const name of ['.', '..', '']) { - assert.throws( - () => store.resolveSessionDir(name), - (error: unknown) => - error instanceof AppError && - error.code === 'INVALID_ARGS' && - /session name/i.test(error.message), - `expected resolveSessionDir(${JSON.stringify(name)}) to reject`, - ); - } -}); - test('session lease metadata round-trips through the store', () => { const { store, session } = makeFixture('agent-device-session-lease-'); session.lease = { diff --git a/src/daemon/device/device-claim-owner-recovery.ts b/src/daemon/device/device-claim-owner-recovery.ts index c8b60069ee..f57a6c641f 100644 --- a/src/daemon/device/device-claim-owner-recovery.ts +++ b/src/daemon/device/device-claim-owner-recovery.ts @@ -4,7 +4,11 @@ import { createPlatformRuntimeGateway } from '../../platform-runtime.ts'; import { resolveDaemonPaths } from '../../daemon-resolution.ts'; import { createDeviceClaimReconciler } from './device-claim-reconciliation.ts'; import type { DeviceClaimReconciler } from './device-claims.ts'; -import { SessionStore } from '../session-store.ts'; +import { + resolveSessionDir, + resolveSessionAppLogPath, + resolveSessionAppLogPidPath, +} from '../session-artifact-paths.ts'; export type OwnerScopedClaimRecovery = { reconcile: DeviceClaimReconciler; @@ -49,17 +53,16 @@ function composeOwnerScopedClaimRecovery( scope: PlatformRequestScope, ): OwnerScopedClaimRecovery { const daemonPaths = resolveDaemonPaths(stateDir); - const sessionStore = new SessionStore(daemonPaths.sessionsDir); const gateway = createPlatformRuntimeGateway({ sessionsDir: daemonPaths.sessionsDir, ownedProcesses: createOwnedProcessRecordStore({ stateDir: daemonPaths.baseDir, sessionsDir: daemonPaths.sessionsDir, - resolveSessionDir: (sessionId) => sessionStore.resolveSessionDir(sessionId), + resolveSessionDir: (sessionId) => resolveSessionDir(daemonPaths.sessionsDir, sessionId), }), resolveSessionArtifacts: (sessionId) => ({ - outputPath: sessionStore.resolveAppLogPath(sessionId), - pidPath: sessionStore.resolveAppLogPidPath(sessionId), + outputPath: resolveSessionAppLogPath(daemonPaths.sessionsDir, sessionId), + pidPath: resolveSessionAppLogPidPath(daemonPaths.sessionsDir, sessionId), }), }); return { diff --git a/src/daemon/handlers/session-app-deployment.ts b/src/daemon/handlers/session-app-deployment.ts index edf468e306..79768d7158 100644 --- a/src/daemon/handlers/session-app-deployment.ts +++ b/src/daemon/handlers/session-app-deployment.ts @@ -1,3 +1,4 @@ +import { expandSessionPath } from '@agent-device/host-kit/session-paths'; import fs from 'node:fs'; import type { AppDeploymentResult } from '@agent-device/contracts/app-deployment-runtime'; import { @@ -9,7 +10,7 @@ import { readNotificationPayload } from '../dispatch-payload.ts'; import { cleanupUploadedArtifact, prepareUploadedArtifact } from '../artifact-tracking.ts'; import { expireRefFrame } from '../ref-frame.ts'; import type { BindDeviceRuntime, InspectDeviceRuntimeFacts } from '../request-runtime-binding.ts'; -import { SessionStore } from '../session-store.ts'; +import type { SessionStore } from '../session-store.ts'; import type { DaemonRequest, DaemonResponse } from '../daemon-request.ts'; import type { SessionState } from '../session-state.ts'; import { resolvePayloadInput } from '../payload-input.ts'; @@ -62,7 +63,7 @@ export async function handleAppDeploymentCommand(params: { try { const appPath = uploadedArtifactId ? prepareUploadedArtifact(uploadedArtifactId, req.meta?.tenantId) - : SessionStore.expandHome(target.appPathInput); + : expandSessionPath(target.appPathInput); if (!fs.existsSync(appPath)) { return errorResponse('INVALID_ARGS', `App binary not found: ${appPath}`); } @@ -242,7 +243,7 @@ function resolvePushPayload(payloadArg: string, cwd?: string): string { const resolved = resolvePayloadInput(payloadArg, { subject: 'Push payload', cwd, - expandPath: (value, currentCwd) => SessionStore.expandHome(value, currentCwd), + expandPath: (value, currentCwd) => expandSessionPath(value, currentCwd), }); return resolved.kind === 'file' ? resolved.path : resolved.text; } diff --git a/src/daemon/handlers/trace-runtime.ts b/src/daemon/handlers/trace-runtime.ts index 43b8ee86c6..64f804f0b5 100644 --- a/src/daemon/handlers/trace-runtime.ts +++ b/src/daemon/handlers/trace-runtime.ts @@ -1,7 +1,8 @@ +import { expandSessionPath } from '@agent-device/host-kit/session-paths'; import fs from 'node:fs'; import path from 'node:path'; import type { TraceCommandResult } from '@agent-device/contracts/recording'; -import { SessionStore } from '../session-store.ts'; +import type { SessionStore } from '../session-store.ts'; import type { DaemonRequest, DaemonResponse } from '../daemon-request.ts'; import type { SessionState } from '../session-state.ts'; import { recordSessionAction } from '../session-action-recorder.ts'; @@ -29,9 +30,7 @@ function startTrace( session: SessionState, ): DaemonResponse { if (session.trace) return errorResponse('INVALID_ARGS', 'trace already in progress'); - const outPath = SessionStore.expandHome( - req.positionals?.[1] ?? sessionStore.defaultTracePath(session), - ); + const outPath = expandSessionPath(req.positionals?.[1] ?? sessionStore.defaultTracePath(session)); fs.mkdirSync(path.dirname(outPath), { recursive: true }); fs.appendFileSync(outPath, ''); session.trace = { outPath, startedAt: Date.now() }; @@ -72,7 +71,7 @@ function stopTrace( function relocateTraceOutput(currentPath: string, requestedPath: string | undefined): string { if (!requestedPath) return currentPath; - const resolved = SessionStore.expandHome(requestedPath); + const resolved = expandSessionPath(requestedPath); fs.mkdirSync(path.dirname(resolved), { recursive: true }); if (fs.existsSync(currentPath)) fs.renameSync(currentPath, resolved); else fs.appendFileSync(resolved, ''); diff --git a/src/daemon/screenshot-runtime.ts b/src/daemon/screenshot-runtime.ts index f0e4c76477..f10f44ab42 100644 --- a/src/daemon/screenshot-runtime.ts +++ b/src/daemon/screenshot-runtime.ts @@ -1,3 +1,4 @@ +import { expandSessionPath } from '@agent-device/host-kit/session-paths'; import type { CommandFlags } from '@agent-device/contracts/command'; import { retiredScreenshotMaxSizeFlagError, @@ -38,7 +39,6 @@ import { type ScreenshotRuntimeBindings, } from './screenshot-runtime-binding.ts'; import { setSessionSnapshot } from './session-snapshot.ts'; -import { SessionStore } from './session-store.ts'; import type { DaemonRequest } from './daemon-request.ts'; import type { SessionState } from './session-state.ts'; @@ -321,7 +321,7 @@ function readScreenshotRequest( const positionals = req.positionals ?? []; const flags = req.flags ?? {}; const expand = (value: string | undefined) => - value === undefined ? undefined : SessionStore.expandHome(value, req.meta?.cwd); + value === undefined ? undefined : expandSessionPath(value, req.meta?.cwd); const positionalPath = expand(positionals[0]); const outFlag = expand(flags.out); return { diff --git a/src/daemon/session-artifact-paths.ts b/src/daemon/session-artifact-paths.ts index d456d4d199..b445ed1ec1 100644 --- a/src/daemon/session-artifact-paths.ts +++ b/src/daemon/session-artifact-paths.ts @@ -1,6 +1,7 @@ import path from 'node:path'; import type { DiagnosticsRecordRef } from '@agent-device/kernel/errors'; -import { safeSessionName } from '@agent-device/host-kit/session-paths'; +import { AppError } from '@agent-device/kernel/errors'; +import { isSafeSessionSegment, safeSessionName } from '@agent-device/host-kit/session-paths'; /** Path to session-scoped platform subprocess output, such as Apple runner xcodebuild logs. */ export function resolveSessionRunnerLogPath(sessionDir: string): string { @@ -49,3 +50,27 @@ export function resolveRemoteRequestDiagnosticsPath( ref.requestId, ); } + +/** + * The one place a session name becomes a directory, so the invariant that every + * session dir lies beneath `sessionsDir` is enforced here rather than by each + * caller: `.` and `..` survive `safeSessionName` and would resolve to the + * sessions dir itself or the daemon state dir above it. + */ +export function resolveSessionDir(sessionsDir: string, sessionName: string): string { + if (!isSafeSessionSegment(sessionName)) { + throw new AppError( + 'INVALID_ARGS', + `Invalid session name ${JSON.stringify(sessionName)}: a session name cannot be empty, ".", or "..".`, + ); + } + return path.join(sessionsDir, safeSessionName(sessionName)); +} + +export function resolveSessionAppLogPath(sessionsDir: string, address: string): string { + return path.join(resolveSessionDir(sessionsDir, address), 'app.log'); +} + +export function resolveSessionAppLogPidPath(sessionsDir: string, address: string): string { + return path.join(resolveSessionDir(sessionsDir, address), 'app-log.pid'); +} diff --git a/src/daemon/session-observability/internal/session-perf-runtime.ts b/src/daemon/session-observability/internal/session-perf-runtime.ts index 210528eea6..cb6acdb0a7 100644 --- a/src/daemon/session-observability/internal/session-perf-runtime.ts +++ b/src/daemon/session-observability/internal/session-perf-runtime.ts @@ -1,3 +1,4 @@ +import { expandSessionPath } from '@agent-device/host-kit/session-paths'; import path from 'node:path'; import { type PerfCaptureAdmissionLedger } from '@agent-device/capture-kit/perf-capture-admission-ledger'; import { @@ -28,7 +29,7 @@ import type { BindDeviceRuntime, InspectDeviceRuntimeFacts, } from '../../request-runtime-binding.ts'; -import { SessionStore } from '../../session-store.ts'; +import type { SessionStore } from '../../session-store.ts'; import type { DaemonRequest, DaemonResponse } from '../../daemon-request.ts'; import type { SessionState } from '../../session-state.ts'; import { recordSessionAction } from '../../session-action-recorder.ts'; @@ -138,7 +139,7 @@ async function executeAdmittedPerfPlan( appId: session.appBundleId, kind: plan.request.kind, outPath: plan.request.outPath - ? SessionStore.expandHome(plan.request.outPath, params.req.meta?.cwd) + ? expandSessionPath(plan.request.outPath, params.req.meta?.cwd) : undefined, artifactsDir: path.join( params.sessionStore.ensureSessionDir(params.sessionName), @@ -171,7 +172,7 @@ async function executeAdmittedPerfPlan( const data = await runtime.operations.perfProfileReport({ appId: session.appBundleId, kind: plan.request.kind, - tracePath: SessionStore.expandHome(tracePath, params.req.meta?.cwd), + tracePath: expandSessionPath(tracePath, params.req.meta?.cwd), outPath, template: plan.request.template ?? (last?.kind === 'xctrace' ? last.template : undefined), profile: last, @@ -257,7 +258,7 @@ async function stopPerfCapture( const mismatchMessage = perfCaptureStopMismatch(snapshot, request); if (mismatchMessage) return errorResponse('INVALID_ARGS', mismatchMessage); if (request.outPath) { - capture.handle.setOutputPath(SessionStore.expandHome(request.outPath, params.req.meta?.cwd)); + capture.handle.setOutputPath(expandSessionPath(request.outPath, params.req.meta?.cwd)); } const completion = await finishLivePerfCapture({ intent: 'capture', @@ -395,7 +396,7 @@ function resolveNativeOutPath( requestedPath: string | undefined, fallbackFileName: string, ): string { - if (requestedPath) return SessionStore.expandHome(requestedPath, params.req.meta?.cwd); + if (requestedPath) return expandSessionPath(requestedPath, params.req.meta?.cwd); return path.join( params.sessionStore.ensureSessionDir(params.sessionName), `${timestampToken()}-${fallbackFileName}`, diff --git a/src/daemon/session-store.ts b/src/daemon/session-store.ts index 1598dd9414..e31c66403a 100644 --- a/src/daemon/session-store.ts +++ b/src/daemon/session-store.ts @@ -1,14 +1,14 @@ import path from 'node:path'; import fs from 'node:fs'; -import { AppError } from '@agent-device/kernel/errors'; import { emitDiagnostic } from '@agent-device/host-kit/diagnostics'; import type { SessionRef, SessionRuntimeHints, SessionState } from './session-state.ts'; import { recordActionEntry, type RecordActionEntry } from './session-action-recorder.ts'; +import { isSafeSessionSegment, safeSessionName } from '@agent-device/host-kit/session-paths'; import { - expandSessionPath, - isSafeSessionSegment, - safeSessionName, -} from '@agent-device/host-kit/session-paths'; + resolveSessionDir, + resolveSessionAppLogPath, + resolveSessionAppLogPidPath, +} from './session-artifact-paths.ts'; import { readRepairTombstoneFile, resolveRepairTombstonePath, @@ -380,20 +380,8 @@ export class SessionStore { return path.join(this.sessionsDir, `${safeName}-${timestamp}.trace.log`); } - /** - * The one place a session name becomes a directory, so the invariant that every - * session dir lies beneath `sessionsDir` is enforced here rather than by each - * caller: `.` and `..` survive `safeSessionName` and would resolve to the - * sessions dir itself or the daemon state dir above it. - */ resolveSessionDir(sessionName: string): string { - if (!isSafeSessionSegment(sessionName)) { - throw new AppError( - 'INVALID_ARGS', - `Invalid session name ${JSON.stringify(sessionName)}: a session name cannot be empty, ".", or "..".`, - ); - } - return path.join(this.sessionsDir, safeSessionName(sessionName)); + return resolveSessionDir(this.sessionsDir, sessionName); } // Daemon state dir (parent of the `sessions/` dir), matching daemonPaths.baseDir. Called via @@ -411,21 +399,17 @@ export class SessionStore { /** Path to session-scoped app log file. Agent can grep this for token-efficient debugging. */ resolveAppLogPath(sessionName: string): string { - return path.join(this.resolveSessionDir(sessionName), 'app.log'); + return resolveSessionAppLogPath(this.sessionsDir, sessionName); } resolveAppLogPidPath(sessionName: string): string { - return path.join(this.resolveSessionDir(sessionName), 'app-log.pid'); + return resolveSessionAppLogPidPath(this.sessionsDir, sessionName); } resolveEventLogPath(sessionName: string): string { return resolveSessionEventLogPath(this.resolveSessionDir(sessionName)); } - static expandHome(filePath: string, cwd?: string): string { - return expandSessionPath(filePath, cwd); - } - /** * Resolve the map key for a live session object. SessionState.name is the * public session name, while the map key may include cwd/tenant isolation. From 3b309f0dda0b3b880374cb8f91938dee46b08c3c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sat, 3 Oct 2026 00:26:52 +0200 Subject: [PATCH 02/20] fix: bind replay directory cleanup to joined daemon startups --- .../daemon-registration-owner.test.ts | 156 +++++++++++- .../test-utils/registered-daemon-fixture.ts | 84 +++++++ .../__tests__/daemon-client-lifecycle.test.ts | 62 ++--- src/daemon-client/daemon-client-lifecycle.ts | 137 +++++------ src/daemon-registration-owner.ts | 227 ++++++++++++++++-- src/session-repair-tombstone.ts | 56 ++++- 6 files changed, 586 insertions(+), 136 deletions(-) create mode 100644 src/__tests__/test-utils/registered-daemon-fixture.ts diff --git a/src/__tests__/daemon-registration-owner.test.ts b/src/__tests__/daemon-registration-owner.test.ts index edb42abced..f5f67ac699 100644 --- a/src/__tests__/daemon-registration-owner.test.ts +++ b/src/__tests__/daemon-registration-owner.test.ts @@ -1,16 +1,22 @@ import assert from 'node:assert/strict'; import fs from 'node:fs'; import { afterEach, test, vi } from 'vitest'; -import { readCurrentOwnerIdentity } from '@agent-device/host-kit/process'; +import { readCurrentOwnerIdentity, isProcessAlive } from '@agent-device/host-kit/process'; import { tryAcquireDaemonRegistration, stopAndRetireDaemon, recoverAbandonedDaemonRegistration, + createOwnedReplayStateDir, + launchDaemonProcess, + type OwnedReplayStateDir, } from '../daemon-registration-owner.ts'; import { resolveDaemonPaths, type DaemonPaths } from '../daemon-resolution.ts'; import { readRegisteredDaemonOwnership } from '../daemon-registration.ts'; import { readDaemonShutdownReport } from '../daemon-shutdown-report.ts'; import { mkdtempForTestSync } from './test-utils/tmp-dir.ts'; +import { registeredDaemonFixtureArgs } from './test-utils/registered-daemon-fixture.ts'; +import { sleep } from '@agent-device/host-kit/retry'; +import { stopDaemonProcess } from '../daemon-process.ts'; const fields = { socketPort: 4210, @@ -305,3 +311,151 @@ test.skipIf(process.getuid?.() === 0)( } }, ); + +test('a forged private-directory capability cannot authorize even matching dead metadata removal', async () => { + const paths = resolveDaemonPaths(mkdtempForTestSync('agent-device-private-forgery-')); + replaceInfo(paths, deadIdentity.pid, deadIdentity.startTime); + const before = fs.readFileSync(paths.infoPath, 'utf8'); + const result = await stopAndRetireDaemon({ + paths, + observed: deadIdentity, + mode: 'force', + ownedStateDir: Object.freeze({ paths }) as OwnedReplayStateDir, + }); + assert.equal(result.status, 'retained'); + assert.equal(fs.readFileSync(paths.infoPath, 'utf8'), before); +}); + +test('private retirement closes startup admission, joins the actual child and never recreates a removed directory', async () => { + const ownedStateDir = createOwnedReplayStateDir(); + const paths = ownedStateDir.paths; + const args = registeredDaemonFixtureArgs(paths, fields); + const launch = launchDaemonProcess({ paths, args, serverMode: 'socket', ownedStateDir }); + try { + assert.ok(launch.startTime); + await waitForFixtureFile(paths.infoPath); + const input = { + paths, + observed: { pid: launch.pid, startTime: launch.startTime }, + mode: 'graceful' as const, + ownedStateDir, + }; + const pending = stopAndRetireDaemon(input); + assert.throws( + () => launchDaemonProcess({ paths, args, serverMode: 'socket', ownedStateDir }), + (error: { details?: { reason?: string } }) => + error.details?.reason === 'daemon_startup_admission_closed', + ); + const result = await pending; + assert.equal(result.status, 'retired', JSON.stringify(result)); + if (result.status !== 'retired') assert.fail('retirement not confirmed'); + assert.equal(result.removedStateDir, true); + assert.equal((await launch.exited).exitCode, 0); + assert.equal(fs.existsSync(paths.baseDir), false); + assert.deepEqual(await stopAndRetireDaemon(input), result); + assert.equal(fs.existsSync(paths.baseDir), false); + } finally { + await finishPrivateTestDaemons(paths, launch); + } +}); + +test('private retirement retains the directory while an earlier actual startup child is still paused', async () => { + const ownedStateDir = createOwnedReplayStateDir(); + const paths = ownedStateDir.paths; + const args = registeredDaemonFixtureArgs(paths, fields); + const entry = args[1]!; + const ready = `${paths.baseDir}/paused-startup.ready`; + fs.writeFileSync( + entry, + `import fs from 'node:fs'; fs.writeFileSync(${JSON.stringify(ready)}, 'ready'); setInterval(() => {}, 1000);`, + ); + const first = launchDaemonProcess({ paths, args, serverMode: 'socket', ownedStateDir }); + let second: ReturnType | undefined; + try { + await waitForFixtureFile(ready); + second = launchDaemonProcess({ + paths, + args: registeredDaemonFixtureArgs(paths, fields), + serverMode: 'socket', + ownedStateDir, + }); + await waitForFixtureFile(paths.infoPath); + const result = await stopAndRetireDaemon({ + paths, + observed: { pid: second.pid, startTime: second.startTime ?? null }, + mode: 'graceful', + ownedStateDir, + startupJoinTimeoutMs: 0, + }); + assert.equal(result.status, 'retained', JSON.stringify(result)); + if (result.status !== 'retained') assert.fail('private state unexpectedly retired'); + assert.equal(result.reason, 'startup-unconfirmed'); + assert.equal(result.termination?.status, 'exited'); + assert.equal(fs.existsSync(paths.baseDir), true); + assert.equal(isProcessAlive(first.pid), true); + assert.equal((await second.exited).exitCode, 0); + } finally { + await finishPrivateTestDaemons(paths, first, second); + } +}); + +for (const contents of [ + '{broken', + ...[false, null, 0, {}].map((commitFailure) => + JSON.stringify({ owner: 'default', expiresAt: Date.now() + 60_000, commitFailure }), + ), +]) { + test(`malformed repair evidence (${contents}) retains private state after the actual child has exited`, async () => { + const ownedStateDir = createOwnedReplayStateDir(); + const paths = ownedStateDir.paths; + const sessionDir = `${paths.sessionsDir}/default`; + fs.mkdirSync(sessionDir, { recursive: true }); + const evidencePath = `${sessionDir}/repair-tombstone.json`; + fs.writeFileSync(evidencePath, contents); + const launch = launchDaemonProcess({ + paths, + args: registeredDaemonFixtureArgs(paths, fields), + serverMode: 'socket', + ownedStateDir, + }); + try { + await waitForFixtureFile(paths.infoPath); + const result = await stopAndRetireDaemon({ + paths, + observed: { pid: launch.pid, startTime: launch.startTime ?? null }, + mode: 'graceful', + ownedStateDir, + }); + assert.equal(result.status, 'retained', JSON.stringify(result)); + if (result.status !== 'retained') assert.fail('repair evidence unexpectedly discarded'); + assert.equal(result.termination?.status, 'exited'); + assert.equal(result.error?.details?.reason, 'repair_evidence_invalid'); + assert.equal(fs.readFileSync(evidencePath, 'utf8'), contents); + await launch.exited; + } finally { + await finishPrivateTestDaemons(paths, launch); + } + }); +} + +async function waitForFixtureFile(filePath: string): Promise { + const deadline = Date.now() + 2_000; + while (!fs.existsSync(filePath) && Date.now() < deadline) await sleep(20); + assert.equal(fs.existsSync(filePath), true); +} + +async function finishPrivateTestDaemons( + paths: DaemonPaths, + ...launches: (ReturnType | undefined)[] +): Promise { + for (const launch of launches) { + if (!launch) continue; + const termination = await stopDaemonProcess( + { pid: launch.pid, startTime: launch.startTime ?? null }, + { mode: 'force', termTimeoutMs: 0, killTimeoutMs: 2_000 }, + ); + assert.notEqual(termination.status, 'retained', JSON.stringify(termination)); + await launch.exited; + } + fs.rmSync(paths.baseDir, { recursive: true, force: true }); +} diff --git a/src/__tests__/test-utils/registered-daemon-fixture.ts b/src/__tests__/test-utils/registered-daemon-fixture.ts new file mode 100644 index 0000000000..63ee16a7bc --- /dev/null +++ b/src/__tests__/test-utils/registered-daemon-fixture.ts @@ -0,0 +1,84 @@ +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import path from 'node:path'; +import { vi } from 'vitest'; +import type { runCmdDetachedMonitored } from '@agent-device/host-kit/command'; +import { readProcessStartTime } from '@agent-device/host-kit/process'; +import { stopDaemonProcess } from '../../daemon-process.ts'; +import type { DaemonPaths } from '../../daemon-resolution.ts'; +import type { DaemonRegistrationFields } from '../../daemon-registration-owner.ts'; + +const actualCommand = await vi.importActual( + '@agent-device/host-kit/command', +); +const children = new Map< + string, + { launch: ReturnType; startTime: string | null } +>(); + +/** A real registration owner, advertising the caller's HTTP fixture and joining before deletion. */ +export function registeredDaemonFixtureArgs( + paths: DaemonPaths, + fields: DaemonRegistrationFields, +): string[] { + const entry = path.join(paths.baseDir, 'dist', 'src', 'internal', 'daemon.js'); + fs.mkdirSync(path.dirname(entry), { recursive: true }); + fs.writeFileSync(path.join(paths.baseDir, 'package.json'), '{"type":"module"}'); + const registrationUrl = new URL('../../daemon-registration-owner.ts', import.meta.url).href; + fs.writeFileSync( + entry, + `import fs from 'node:fs'; +import path from 'node:path'; +import { tryAcquireDaemonRegistration } from ${JSON.stringify(registrationUrl)}; +const paths = ${JSON.stringify(paths)}; +const acquired = await tryAcquireDaemonRegistration(paths); +if (acquired.status !== 'acquired') process.exit(75); +process.on('SIGTERM', async () => { + const deferred = path.join(paths.baseDir, 'repair-on-shutdown.json'); + if (fs.existsSync(deferred)) { + const dir = path.join(paths.sessionsDir, 'default'); + fs.mkdirSync(dir, { recursive: true }); + fs.copyFileSync(deferred, path.join(dir, 'repair-tombstone.json')); + } + await acquired.owner.finish(); + process.exit(0); +}); +acquired.owner.publish(${JSON.stringify(fields)}); +setInterval(() => {}, 1000); +`, + ); + return ['--experimental-strip-types', entry]; +} + +export function spawnRegisteredDaemonFixture( + paths: DaemonPaths, + fields: DaemonRegistrationFields, + options: Parameters[2], +): ReturnType { + const child = actualCommand.runCmdDetachedMonitored( + process.execPath, + registeredDaemonFixtureArgs(paths, fields), + options, + ); + children.set(paths.baseDir, { launch: child, startTime: readProcessStartTime(child.pid) }); + return child; +} + +export async function finishRegisteredDaemonFixture(stateDir: string): Promise { + const owned = children.get(stateDir); + if (owned) { + const child = owned.launch; + const termination = await stopDaemonProcess( + { pid: child.pid, startTime: owned.startTime }, + { mode: 'force', termTimeoutMs: 0, killTimeoutMs: 2_000 }, + ); + assert.notEqual(termination.status, 'retained', JSON.stringify(termination)); + await child.exited; + children.delete(stateDir); + } + fs.rmSync(stateDir, { recursive: true, force: true }); +} + +export async function finishRegisteredDaemonFixtures(): Promise { + for (const stateDir of children.keys()) await finishRegisteredDaemonFixture(stateDir); +} diff --git a/src/daemon-client/__tests__/daemon-client-lifecycle.test.ts b/src/daemon-client/__tests__/daemon-client-lifecycle.test.ts index 91f4da210a..b27673099b 100644 --- a/src/daemon-client/__tests__/daemon-client-lifecycle.test.ts +++ b/src/daemon-client/__tests__/daemon-client-lifecycle.test.ts @@ -6,6 +6,11 @@ import net from 'node:net'; import path from 'node:path'; import { afterEach, test, vi } from 'vitest'; import { mkdtempForTestSync } from '../../__tests__/test-utils/tmp-dir.ts'; +import { + spawnRegisteredDaemonFixture, + finishRegisteredDaemonFixture, + finishRegisteredDaemonFixtures, +} from '../../__tests__/test-utils/registered-daemon-fixture.ts'; vi.mock('@agent-device/host-kit/command', async (importOriginal) => ({ ...(await importOriginal()), @@ -51,14 +56,19 @@ type DaemonInfoFixture = { processStartTime?: string; }; +const actualRetry = await vi.importActual( + '@agent-device/host-kit/retry', +); const mockRunCmdDetached = vi.mocked(runCmdDetachedMonitored); const mockRunCmdSync = vi.mocked(runCmdSync); const mockSleep = vi.mocked(sleep); -afterEach(() => { +afterEach(async () => { + await finishRegisteredDaemonFixtures(); mockRunCmdDetached.mockReset(); mockRunCmdSync.mockClear(); - mockSleep.mockClear(); + mockSleep.mockReset(); + mockSleep.mockImplementation(async () => {}); vi.unstubAllEnvs(); }); @@ -147,16 +157,22 @@ function installSpawnedHttpDaemonAtOwnedStateDir( httpPort: number, onStateDir: (stateDir: string) => void, ): void { + mockSleep.mockImplementation(actualRetry.sleep); mockRunCmdDetached.mockImplementation((_command, _args, options) => { const ownedStateDir = String(options?.env?.AGENT_DEVICE_STATE_DIR); onStateDir(ownedStateDir); const ownedPaths = resolveDaemonPaths(ownedStateDir); - writeDaemonInfo(ownedPaths, { httpPort, transport: 'http' }); - writeDaemonLock(ownedPaths, { - pid: process.pid, - processStartTime: readProcessStartTime(process.pid) ?? undefined, - }); - return { pid: process.pid, exited: new Promise(() => {}) }; + return spawnRegisteredDaemonFixture( + ownedPaths, + { + httpPort, + token: 'local-secret', + version: readVersion(), + codeOrigin: 'checkout', + codeSignature: currentDaemonCodeSignature(), + }, + options, + ); }); } @@ -813,7 +829,7 @@ test('sendToDaemon keeps an owned ephemeral daemon alive and hints its --state-d assert.equal(fs.existsSync(ownedStateDir), true); } finally { await closeLoopbackServer(daemon.server); - if (ownedStateDir) fs.rmSync(ownedStateDir, { recursive: true, force: true }); + if (ownedStateDir) await finishRegisteredDaemonFixture(ownedStateDir); } }); @@ -854,7 +870,7 @@ test('C1: keep-alive keys on repairSessionHeld, NOT resume.allowed — a HELD di assert.equal(fs.existsSync(ownedStateDir), true); } finally { await closeLoopbackServer(daemon.server); - if (ownedStateDir) fs.rmSync(ownedStateDir, { recursive: true, force: true }); + if (ownedStateDir) await finishRegisteredDaemonFixture(ownedStateDir); } }); @@ -888,7 +904,7 @@ test('sendToDaemon tears down an owned ephemeral daemon on an UNHELD divergence assert.equal(fs.existsSync(ownedStateDir), false); } finally { await closeLoopbackServer(daemon.server); - if (ownedStateDir) fs.rmSync(ownedStateDir, { recursive: true, force: true }); + if (ownedStateDir) await finishRegisteredDaemonFixture(ownedStateDir); } }); @@ -916,16 +932,8 @@ test('BLOCKER 2 (third follow-up): a shutdown-time repair commit failure is surf let ownedStateDir = ''; installSpawnedHttpDaemonAtOwnedStateDir(daemon.port, (dir) => { ownedStateDir = dir; - // Simulate the daemon's OWN shutdown handler (`finalizeRepairTeardown`) - // having already run and left a commit-failure tombstone before this - // fake process "exits" — the real ordering `stopDaemonProcessForTakeover` - // depends on (it waits for the process to exit, and the real daemon only - // exits after teardown finishes writing this file). - const ownedPaths = resolveDaemonPaths(dir); - const sessionDir = path.join(ownedPaths.sessionsDir, 'default'); - fs.mkdirSync(sessionDir, { recursive: true }); fs.writeFileSync( - path.join(sessionDir, 'repair-tombstone.json'), + path.join(dir, 'repair-on-shutdown.json'), `${JSON.stringify({ owner: 'default', reapedAt: Date.now(), @@ -968,7 +976,7 @@ test('BLOCKER 2 (third follow-up): a shutdown-time repair commit failure is surf ); } finally { await closeLoopbackServer(daemon.server); - if (ownedStateDir) fs.rmSync(ownedStateDir, { recursive: true, force: true }); + if (ownedStateDir) await finishRegisteredDaemonFixture(ownedStateDir); } }); @@ -1003,7 +1011,7 @@ test('continuation: sendToDaemon keeps the daemon alive on a held divergence eve assert.equal(fs.existsSync(ownedStateDir), true); } finally { await closeLoopbackServer(daemon.server); - if (ownedStateDir) fs.rmSync(ownedStateDir, { recursive: true, force: true }); + if (ownedStateDir) await finishRegisteredDaemonFixture(ownedStateDir); } }); @@ -1131,7 +1139,7 @@ test('sendToDaemon keeps an owned ephemeral daemon alive and hints its --state-d assert.equal(fs.existsSync(ownedStateDir), true); } finally { await closeLoopbackServer(daemon.server); - if (ownedStateDir) fs.rmSync(ownedStateDir, { recursive: true, force: true }); + if (ownedStateDir) await finishRegisteredDaemonFixture(ownedStateDir); } }); @@ -1170,7 +1178,7 @@ test('closes the loop: a follow-up sendToDaemon using the hinted --state-dir/--s assert.equal(daemon.rpcRequests[1]?.params?.command, 'press'); } finally { await closeLoopbackServer(daemon.server); - if (ownedStateDir) fs.rmSync(ownedStateDir, { recursive: true, force: true }); + if (ownedStateDir) await finishRegisteredDaemonFixture(ownedStateDir); } }); @@ -1202,7 +1210,7 @@ test('sendToDaemon tears down an owned ephemeral daemon when replay reports the assert.equal(fs.existsSync(ownedStateDir), false); } finally { await closeLoopbackServer(daemon.server); - if (ownedStateDir) fs.rmSync(ownedStateDir, { recursive: true, force: true }); + if (ownedStateDir) await finishRegisteredDaemonFixture(ownedStateDir); } }); @@ -1248,7 +1256,7 @@ test('ADR 0012 R7 x ADR 0016: a completed --save-script repair also keeps its ow assert.equal(fs.existsSync(ownedStateDir), true); } finally { await closeLoopbackServer(daemon.server); - if (ownedStateDir) fs.rmSync(ownedStateDir, { recursive: true, force: true }); + if (ownedStateDir) await finishRegisteredDaemonFixture(ownedStateDir); } }); @@ -1328,6 +1336,6 @@ test('sendToDaemon still tears down a `test` command owned ephemeral daemon even assert.equal(fs.existsSync(ownedStateDir), false); } finally { await closeLoopbackServer(daemon.server); - if (ownedStateDir) fs.rmSync(ownedStateDir, { recursive: true, force: true }); + if (ownedStateDir) await finishRegisteredDaemonFixture(ownedStateDir); } }); diff --git a/src/daemon-client/daemon-client-lifecycle.ts b/src/daemon-client/daemon-client-lifecycle.ts index 43a6556781..03cfa395f5 100644 --- a/src/daemon-client/daemon-client-lifecycle.ts +++ b/src/daemon-client/daemon-client-lifecycle.ts @@ -1,17 +1,22 @@ import fs from 'node:fs'; import net from 'node:net'; -import os from 'node:os'; -import path from 'node:path'; import { AppError, normalizeError } from '@agent-device/kernel/errors'; import { readReplayDivergenceResume } from '@agent-device/ad-replay/divergence'; import type { DaemonRequest, DaemonResponse } from '../daemon/daemon-request.ts'; -import { runCmdDetachedMonitored, type ExecDetachedExit } from '@agent-device/host-kit/command'; +import { type ExecDetachedExit } from '@agent-device/host-kit/command'; import { shellQuoteIfNeeded } from '@agent-device/kernel/device-shell'; import { emitDiagnostic } from '@agent-device/host-kit/diagnostics'; -import { isProcessAlive, readProcessStartTime } from '@agent-device/host-kit/process'; +import { isProcessAlive } from '@agent-device/host-kit/process'; import { sleep } from '@agent-device/host-kit/retry'; -import { findUnrecoveredRepairCommitFailure } from '../session-repair-tombstone.ts'; +import type { findUnrecoveredRepairCommitFailure } from '../session-repair-tombstone.ts'; +import { + createOwnedReplayStateDir, + launchDaemonProcess, + stopAndRetireDaemon, + type OwnedReplayStateDir, + type DaemonStartupLaunch, +} from '../daemon-registration-owner.ts'; import { resolveDaemonPaths, resolveDaemonServerMode, @@ -36,7 +41,6 @@ import { readDaemonInfo, recoverDaemonLockHolder, removeDaemonInfo, - removeDaemonLock, resolveDaemonStartupHint, stopDaemonProcessForTakeover, type DaemonInfo, @@ -52,7 +56,7 @@ export type DaemonClientSettings = { paths: DaemonPaths; transportPreference: DaemonTransportPreference; serverMode: DaemonServerMode; - ownedStateDir?: boolean; + ownedStateDir?: OwnedReplayStateDir; remoteBaseUrl?: string; remoteAuthToken?: string; }; @@ -62,13 +66,6 @@ export type EnsuredDaemon = { startedByClient: boolean; }; -type DaemonStartupLaunch = { - pid: number; - /** The launched process's start time, so a reused pid is never taken for it. */ - startTime?: string; - exited: Promise; -}; - type DaemonStartupWaitResult = | { kind: 'ready'; daemon: EnsuredDaemon } | { kind: 'early_exit'; exit: ExecDetachedExit } @@ -91,10 +88,11 @@ export function resolveClientSettings( const explicitStateDir = resolveExplicitStateDir(req); const remote = resolveRemoteClientSettings(req, suppliedAuthToken); const transport = resolveTransportClientSettings(req, remote.remoteBaseUrl); - const ownedStateDir = shouldUseOwnedReplayStateDir(req, explicitStateDir, remote.rawBaseUrl); - const stateDir = ownedStateDir ? createOwnedReplayStateDir() : explicitStateDir; + const ownedStateDir = shouldUseOwnedReplayStateDir(req, explicitStateDir, remote.rawBaseUrl) + ? createOwnedReplayStateDir() + : undefined; return { - paths: resolveDaemonPaths(stateDir), + paths: ownedStateDir?.paths ?? resolveDaemonPaths(explicitStateDir), transportPreference: transport.preference, serverMode: transport.serverMode, ownedStateDir, @@ -153,10 +151,6 @@ function shouldUseOwnedReplayStateDir( return isOneShotReplayCommand(req.command) && !explicitStateDir && !rawRemoteBaseUrl; } -function createOwnedReplayStateDir(): string { - return fs.mkdtempSync(path.join(os.tmpdir(), 'agent-device-replay-daemon-')); -} - export async function ensureDaemon(settings: DaemonClientSettings): Promise { if (settings.remoteBaseUrl) { return await ensureRemoteDaemon(settings); @@ -435,49 +429,40 @@ export async function cleanupDaemonAfterRequest( return response; } - const result = { - pid: daemon.info.pid, - removedInfo: false, - removedLock: false, - removedStateDir: false, - error: undefined as string | undefined, - }; - let surfacedResponse = response; - - try { - await stopDaemonProcessForTakeover(daemon.info); - } catch (error) { - result.error = error instanceof Error ? error.message : String(error); - } finally { - const infoExists = fs.existsSync(settings.paths.infoPath); - removeDaemonInfo(settings.paths.infoPath); - result.removedInfo = infoExists && !fs.existsSync(settings.paths.infoPath); - - const lockExists = fs.existsSync(settings.paths.lockPath); - removeDaemonLock(settings.paths.lockPath); - result.removedLock = lockExists && !fs.existsSync(settings.paths.lockPath); - - if (settings.ownedStateDir) { - // `stopDaemonProcessForTakeover` above waits for the (real) daemon - // process to actually exit, which only happens AFTER its shutdown - // handler finishes `finalizeRepairTeardown` for every session — so by - // now any commit-failure tombstone it would leave is already on disk. - const unrecovered = findUnrecoveredRepairCommitFailure(settings.paths.sessionsDir); - if (unrecovered) { - surfacedResponse = surfaceUnrecoveredRepairCommitFailure(response, unrecovered); - } else { - fs.rmSync(settings.paths.baseDir, { recursive: true, force: true }); - result.removedStateDir = !fs.existsSync(settings.paths.baseDir); - } - } - } - + const result = await stopAndRetireDaemon({ + paths: settings.paths, + observed: { pid: daemon.info.pid, startTime: daemon.info.processStartTime ?? null }, + mode: 'graceful', + ownedStateDir: settings.ownedStateDir, + }); emitDiagnostic({ - level: result.error ? 'warn' : 'info', + level: result.status === 'retained' ? 'warn' : 'info', phase: 'daemon_replay_cleanup', - data: result, + data: { pid: daemon.info.pid, ...result }, }); - return surfacedResponse; + if (result.status === 'retired' && result.repairCommitFailure) { + return surfaceUnrecoveredRepairCommitFailure(response, result.repairCommitFailure); + } + if (result.status === 'retained' && response?.ok) { + return { + ok: false, + error: normalizeError( + new AppError( + 'COMMAND_FAILED', + 'Replay completed, but daemon cleanup could not be confirmed.', + { + reason: 'daemon_retirement_unconfirmed', + retirement: result, + stateDir: settings.paths.baseDir, + hint: + result.error?.hint ?? + `State and diagnostics were retained at ${settings.paths.baseDir}. Resolve the reported cleanup failure before retrying.`, + }, + ), + ), + }; + } + return response; } /** @@ -669,28 +654,14 @@ function isLaunchedDaemon(info: DaemonInfo, launch: DaemonStartupLaunch): boolea function startDaemon(settings: DaemonClientSettings): DaemonStartupLaunch { const launchSpec = resolveDaemonLaunchSpec(); - const args = launchSpec.useSrc - ? ['--experimental-strip-types', launchSpec.srcPath] - : [launchSpec.distPath]; - const env = { - ...process.env, - AGENT_DEVICE_STATE_DIR: settings.paths.baseDir, - AGENT_DEVICE_DAEMON_SERVER_MODE: settings.serverMode, - }; - - fs.mkdirSync(settings.paths.baseDir, { recursive: true }); - const stdoutFd = fs.openSync(settings.paths.logPath, 'a'); - const stderrFd = fs.openSync(settings.paths.logPath, 'a'); - try { - const launched = runCmdDetachedMonitored(process.execPath, args, { - env, - stdio: ['ignore', stdoutFd, stderrFd], - }); - return { ...launched, startTime: readProcessStartTime(launched.pid) ?? undefined }; - } finally { - fs.closeSync(stdoutFd); - fs.closeSync(stderrFd); - } + return launchDaemonProcess({ + paths: settings.paths, + serverMode: settings.serverMode, + ownedStateDir: settings.ownedStateDir, + args: launchSpec.useSrc + ? ['--experimental-strip-types', launchSpec.srcPath] + : [launchSpec.distPath], + }); } function describeDaemonEarlyExit(exit: ExecDetachedExit): string { diff --git a/src/daemon-registration-owner.ts b/src/daemon-registration-owner.ts index 0092f087af..7b95a0c52c 100644 --- a/src/daemon-registration-owner.ts +++ b/src/daemon-registration-owner.ts @@ -1,11 +1,18 @@ import fs from 'node:fs'; -import { normalizeError, type NormalizedError } from '@agent-device/kernel/errors'; +import os from 'node:os'; +import path from 'node:path'; +import { runCmdDetachedMonitored, type ExecDetachedExit } from '@agent-device/host-kit/command'; +import { AppError, normalizeError, type NormalizedError } from '@agent-device/kernel/errors'; import { stopDaemonProcess, waitForDaemonExit, type DaemonTerminationResult, } from './daemon-process.ts'; -import { readCurrentOwnerIdentity, type OwnerIdentity } from '@agent-device/host-kit/process'; +import { + readCurrentOwnerIdentity, + readProcessStartTime, + type OwnerIdentity, +} from '@agent-device/host-kit/process'; import { publishFileSync, tryAcquireProcessLock, @@ -15,7 +22,12 @@ import { } from '@agent-device/host-kit/file'; import { emitDiagnostic, withDiagnosticsScope } from '@agent-device/host-kit/diagnostics'; import type { DaemonCodeOrigin } from '@agent-device/host-kit/code-signature'; -import type { DaemonPaths } from './daemon-resolution.ts'; +import { + resolveDaemonPaths, + type DaemonPaths, + type DaemonServerMode, +} from './daemon-resolution.ts'; +import { findUnrecoveredRepairCommitFailure } from './session-repair-tombstone.ts'; import { readRegisteredDaemonOwnership, type RegisteredDaemonOwnership, @@ -161,6 +173,95 @@ function truncateDaemonLog(logPath: string): void { } } +declare const privateReplayState: unique symbol; +export type OwnedReplayStateDir = Readonly<{ + paths: Readonly; + [privateReplayState]: true; +}>; +export type DaemonStartupLaunch = Readonly<{ + pid: number; + startTime?: string; + exited: Promise; +}>; +type OwnedStartup = { launch: DaemonStartupLaunch; joined: boolean }; +type PrivateReplayState = { + paths: Readonly; + startups: OwnedStartup[]; + sealed: boolean; + retirement?: Promise; +}; +const privateReplayStates = new WeakMap(); + +/** Creates deletion authority only for a fresh private replay directory. */ +export function createOwnedReplayStateDir(): OwnedReplayStateDir { + const paths = Object.freeze( + resolveDaemonPaths(fs.mkdtempSync(path.join(os.tmpdir(), 'agent-device-replay-daemon-'))), + ); + const owned = Object.freeze({ paths }) as OwnedReplayStateDir; + privateReplayStates.set(owned, { paths, startups: [], sealed: false }); + return owned; +} + +/** Launches and monitors the actual child before recording its private-directory authority. */ +export function launchDaemonProcess( + input: Readonly<{ + paths: DaemonPaths; + args: string[]; + serverMode: DaemonServerMode; + ownedStateDir?: OwnedReplayStateDir; + }>, +): DaemonStartupLaunch { + const owned = input.ownedStateDir && requirePrivateReplayState(input.ownedStateDir, input.paths); + if (owned?.sealed) + throw new AppError('COMMAND_FAILED', 'Replay daemon startup admission is closed.', { + reason: 'daemon_startup_admission_closed', + }); + fs.mkdirSync(input.paths.baseDir, { recursive: true }); + const logFd = fs.openSync(input.paths.logPath, 'a'); + try { + const monitored = runCmdDetachedMonitored(process.execPath, input.args, { + env: { + ...process.env, + AGENT_DEVICE_STATE_DIR: input.paths.baseDir, + AGENT_DEVICE_DAEMON_SERVER_MODE: input.serverMode, + }, + stdio: ['ignore', logFd, logFd], + }); + const startup: OwnedStartup = { launch: Object.freeze({ ...monitored }), joined: false }; + if (owned) { + owned.startups.push(startup); + void monitored.exited.then(() => { + startup.joined = true; + }); + } + startup.launch = Object.freeze({ + ...startup.launch, + startTime: readProcessStartTime(monitored.pid) ?? undefined, + }); + return startup.launch; + } finally { + fs.closeSync(logFd); + } +} + +function requirePrivateReplayState( + capability: OwnedReplayStateDir, + paths: DaemonPaths, +): PrivateReplayState { + const owned = privateReplayStates.get(capability); + if ( + !owned || + Object.entries(owned.paths).some(([key, value]) => paths[key as keyof DaemonPaths] !== value) + ) { + throw new AppError( + 'COMMAND_FAILED', + 'Private replay directory ownership could not be verified.', + { reason: 'daemon_private_state_unowned' }, + ); + } + return owned; +} + export type DaemonRetirementInput = Readonly<{ paths: DaemonPaths; observed: OwnerIdentity | null; @@ -171,14 +272,22 @@ export type DaemonRetirementInput = Readonly<{ type ConfirmedDaemonTermination = Extract; export type DaemonRetirementResult = - | Readonly<{ status: 'retired'; termination: ConfirmedDaemonTermination; removedInfo: boolean }> + | Readonly<{ + status: 'retired'; + termination: ConfirmedDaemonTermination; + removedInfo: boolean; + removedStateDir?: boolean; + repairCommitFailure?: NonNullable>; + }> | Readonly<{ status: 'absent'; removedInfo: false }> | Readonly<{ status: 'retained'; termination?: DaemonTerminationResult; removedInfo: boolean; + removedStateDir?: boolean; reason: | 'ownership-unproven' + | 'startup-unconfirmed' | 'exit-unconfirmed' | 'stop-failed' | 'lock-busy' @@ -190,15 +299,55 @@ export type DaemonRetirementResult = /** Stops only the captured daemon lifetime, then retires its registration under the startup lock. */ export async function stopAndRetireDaemon( - input: DaemonRetirementInput & Readonly<{ mode: 'graceful' | 'force' }>, + input: DaemonRetirementInput & + Readonly<{ + mode: 'graceful' | 'force'; + ownedStateDir?: OwnedReplayStateDir; + startupJoinTimeoutMs?: number; + }>, ): Promise { - return await retireObservedDaemon(input, (identity) => - stopDaemonProcess(identity, { - mode: input.mode, - termTimeoutMs: input.termTimeoutMs ?? 3_000, - killTimeoutMs: input.killTimeoutMs ?? 1_000, - }), + let owned: PrivateReplayState | undefined; + try { + if (input.ownedStateDir) { + owned = requirePrivateReplayState(input.ownedStateDir, input.paths); + owned.sealed = true; + const launch = owned.startups.at(-1)?.launch; + if ( + !launch?.startTime || + launch.pid !== input.observed?.pid || + launch.startTime !== input.observed.startTime + ) + throw new AppError( + 'COMMAND_FAILED', + 'The observed daemon is not an owned startup lifetime.', + { reason: 'daemon_private_startup_unowned' }, + ); + if (owned.retirement) return await owned.retirement; + } + } catch (error) { + return { + status: 'retained', + reason: 'ownership-unproven', + removedInfo: false, + error: normalizeError(error), + }; + } + const retirement = retireObservedDaemon( + input, + (identity) => + stopDaemonProcess(identity, { + mode: input.mode, + termTimeoutMs: input.termTimeoutMs ?? 3_000, + killTimeoutMs: input.killTimeoutMs ?? 1_000, + }), + owned, + input.startupJoinTimeoutMs ?? 1_000, ); + if (owned) owned.retirement = retirement; + const result = await retirement; + if (owned && (result.status === 'absent' || !result.removedStateDir)) + owned.retirement = undefined; + return result; } /** Recovers a confirmed abandoned registration without signaling a live process. */ @@ -217,6 +366,8 @@ export async function recoverAbandonedDaemonRegistration( async function retireObservedDaemon( input: DaemonRetirementInput, terminate: (identity: OwnerIdentity) => Promise, + owned?: PrivateReplayState, + startupJoinTimeoutMs = 1_000, ): Promise { const paths = { ...input.paths }; const observed = input.observed && { ...input.observed }; @@ -241,12 +392,16 @@ async function retireObservedDaemon( }; } } - return await retireDaemonRegistration({ ...input, paths }, termination); + if (owned && !(await joinOwnedStartups(owned, startupJoinTimeoutMs))) { + return { status: 'retained', reason: 'startup-unconfirmed', termination, removedInfo: false }; + } + return await retireDaemonRegistration({ ...input, paths }, termination, owned); } async function retireDaemonRegistration( input: DaemonRetirementInput, termination: ConfirmedDaemonTermination | undefined, + owned?: PrivateReplayState, ): Promise { const paths = input.paths; let acquisition: ProcessLockAcquisition; @@ -268,7 +423,11 @@ async function retireDaemonRegistration( error: failure, }; } - let result: DaemonRetirementResult; + let result: DaemonRetirementResult = { + status: 'retained', + reason: 'retirement-unconfirmed', + removedInfo: false, + }; try { const removal = removeRegistrationUnderLock( paths.infoPath, @@ -276,12 +435,13 @@ async function retireDaemonRegistration( acquisition, ); result = retirementAfterRemoval(removal, termination); + result = retirePrivateStateIfEligible(owned, acquisition, result); } catch (error) { result = { status: 'retained', reason: 'retirement-unconfirmed', termination, - removedInfo: false, + removedInfo: result.removedInfo, error: normalizeError(error), }; } @@ -339,3 +499,42 @@ async function recordRegistrationWarning( emitDiagnostic({ level: 'warn', phase, data: { error: normalizeError(error) } }); }); } + +async function joinOwnedStartups(owned: PrivateReplayState, timeoutMs: number): Promise { + if (owned.startups.every((startup) => startup.joined)) return true; + let timer: ReturnType | undefined; + try { + return await Promise.race([ + Promise.all(owned.startups.map(({ launch }) => launch.exited)).then(() => true), + new Promise((resolve) => { + timer = setTimeout(() => resolve(false), timeoutMs); + }), + ]); + } finally { + if (timer) clearTimeout(timer); + } +} + +function retirePrivateStateIfEligible( + owned: PrivateReplayState | undefined, + acquisition: ProcessLockAcquisition, + result: DaemonRetirementResult, +): DaemonRetirementResult { + if (!owned || result.status !== 'retired') return result; + try { + acquisition.assertHeld(); + const failure = findUnrecoveredRepairCommitFailure(owned.paths.sessionsDir); + if (failure) return { ...result, removedStateDir: false, repairCommitFailure: failure }; + acquisition.assertHeld(); + fs.rmSync(owned.paths.baseDir, { recursive: true, force: true }); + return { ...result, removedStateDir: true }; + } catch (error) { + return { + ...result, + status: 'retained', + reason: 'retirement-unconfirmed', + removedStateDir: false, + error: normalizeError(error), + }; + } +} diff --git a/src/session-repair-tombstone.ts b/src/session-repair-tombstone.ts index 59c2b60a9f..293b1ada4d 100644 --- a/src/session-repair-tombstone.ts +++ b/src/session-repair-tombstone.ts @@ -1,5 +1,6 @@ import path from 'node:path'; import fs from 'node:fs'; +import { AppError } from '@agent-device/kernel/errors'; /** * ADR 0012 decision 6, R7 (C5a): a reaped repair session leaves this bounded @@ -31,20 +32,51 @@ export function resolveRepairTombstonePath(sessionDir: string): string { /** Parses/validates a tombstone file at `tombstonePath`; `undefined` if missing, malformed, or expired. */ export function readRepairTombstoneFile(tombstonePath: string): RepairSessionTombstone | undefined { - let raw: string; try { - raw = fs.readFileSync(tombstonePath, 'utf8'); + return readRepairTombstoneForCleanup(tombstonePath); } catch { return undefined; } - let parsed: RepairSessionTombstone; +} + +function readRepairTombstoneForCleanup(tombstonePath: string): RepairSessionTombstone | undefined { + let raw: string; try { - parsed = JSON.parse(raw) as RepairSessionTombstone; - } catch { - return undefined; + raw = fs.readFileSync(tombstonePath, 'utf8'); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ENOENT') return undefined; + throw error; } - if (typeof parsed?.expiresAt !== 'number' || parsed.expiresAt <= Date.now()) return undefined; - return parsed; + const parsed = parseRepairTombstone(raw, tombstonePath); + return parsed.expiresAt > Date.now() ? parsed : undefined; +} + +function parseRepairTombstone(raw: string, tombstonePath: string): RepairSessionTombstone { + try { + const parsed = JSON.parse(raw) as RepairSessionTombstone; + if ( + typeof parsed?.expiresAt !== 'number' || + typeof parsed?.owner !== 'string' || + !validRepairCommitFailure(parsed.commitFailure) + ) + throw new Error('Invalid repair tombstone fields'); + return parsed; + } catch (error) { + throw new AppError( + 'COMMAND_FAILED', + 'Repair evidence could not be inspected.', + { reason: 'repair_evidence_invalid', path: tombstonePath }, + error instanceof Error ? error : undefined, + ); + } +} + +function validRepairCommitFailure(value: unknown): boolean { + const failure = value as RepairSessionTombstone['commitFailure'] | null; + return ( + value === undefined || + (typeof failure?.code === 'string' && typeof failure?.message === 'string') + ); } /** @@ -54,6 +86,7 @@ export function readRepairTombstoneFile(tombstonePath: string): RepairSessionTom * CLIENT side of the daemon boundary (`cleanupDaemonAfterRequest` in * `daemon-client-lifecycle.ts`), which has no live `SessionStore`/session name * to key off of, only the filesystem path an owned ephemeral daemon was given. + * Unreadable or malformed evidence throws so cleanup retains the directory. * An owned ephemeral state dir services exactly one repair transaction at a * time, so the first match found is returned. * @@ -71,12 +104,13 @@ export function findUnrecoveredRepairCommitFailure(sessionsDir: string): let entries: fs.Dirent[]; try { entries = fs.readdirSync(sessionsDir, { withFileTypes: true }); - } catch { - return undefined; + } catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ENOENT') return undefined; + throw error; } for (const entry of entries) { if (!entry.isDirectory()) continue; - const tombstone = readRepairTombstoneFile( + const tombstone = readRepairTombstoneForCleanup( resolveRepairTombstonePath(path.join(sessionsDir, entry.name)), ); if (tombstone?.commitFailure) { From a0ea35d211f31221ed41b3bbc0947e0a3a1ad672 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sat, 3 Oct 2026 01:08:46 +0200 Subject: [PATCH 03/20] fix: preserve repair outcomes through partial replay retirement --- .../daemon-registration-owner.test.ts | 8 +- .../__tests__/daemon-client-lifecycle.test.ts | 119 ++++++++---------- src/daemon-client/daemon-client-lifecycle.ts | 22 +++- src/daemon-registration-owner.ts | 15 +-- src/session-repair-tombstone.ts | 2 +- 5 files changed, 89 insertions(+), 77 deletions(-) diff --git a/src/__tests__/daemon-registration-owner.test.ts b/src/__tests__/daemon-registration-owner.test.ts index f5f67ac699..7f4795f6d8 100644 --- a/src/__tests__/daemon-registration-owner.test.ts +++ b/src/__tests__/daemon-registration-owner.test.ts @@ -7,6 +7,7 @@ import { stopAndRetireDaemon, recoverAbandonedDaemonRegistration, createOwnedReplayStateDir, + DAEMON_STARTUP_EXIT_CODES, launchDaemonProcess, type OwnedReplayStateDir, } from '../daemon-registration-owner.ts'; @@ -331,9 +332,12 @@ test('private retirement closes startup admission, joins the actual child and ne const paths = ownedStateDir.paths; const args = registeredDaemonFixtureArgs(paths, fields); const launch = launchDaemonProcess({ paths, args, serverMode: 'socket', ownedStateDir }); + let contender: ReturnType | undefined; try { assert.ok(launch.startTime); await waitForFixtureFile(paths.infoPath); + contender = launchDaemonProcess({ paths, args, serverMode: 'socket', ownedStateDir }); + assert.equal((await contender.exited).exitCode, DAEMON_STARTUP_EXIT_CODES.busy); const input = { paths, observed: { pid: launch.pid, startTime: launch.startTime }, @@ -355,7 +359,7 @@ test('private retirement closes startup admission, joins the actual child and ne assert.deepEqual(await stopAndRetireDaemon(input), result); assert.equal(fs.existsSync(paths.baseDir), false); } finally { - await finishPrivateTestDaemons(paths, launch); + await finishPrivateTestDaemons(paths, launch, contender); } }); @@ -401,6 +405,8 @@ test('private retirement retains the directory while an earlier actual startup c for (const contents of [ '{broken', + '{"owner":"default","expiresAt":1e400}', + '{"owner":"default","expiresAt":-1e400}', ...[false, null, 0, {}].map((commitFailure) => JSON.stringify({ owner: 'default', expiresAt: Date.now() + 60_000, commitFailure }), ), diff --git a/src/daemon-client/__tests__/daemon-client-lifecycle.test.ts b/src/daemon-client/__tests__/daemon-client-lifecycle.test.ts index b27673099b..1ba1ac1f72 100644 --- a/src/daemon-client/__tests__/daemon-client-lifecycle.test.ts +++ b/src/daemon-client/__tests__/daemon-client-lifecycle.test.ts @@ -532,7 +532,6 @@ test('sendToDaemon prints a takeover notice before replacing an unreachable daem const stateDir = makeTempStateDir('agent-device-daemon-unreachable-takeover-'); const paths = resolveDaemonPaths(stateDir); - // Bind fresh BEFORE freeing the port below: a later bind can reclaim it and skip the takeover. const freshDaemon = await startHttpDaemonFixture({ via: 'fresh-daemon' }); const unreachable = await startHttpDaemonFixture({ via: 'unused' }); await closeLoopbackServer(unreachable.server); @@ -754,9 +753,6 @@ test('sendToDaemon does not replay over HTTP after the socket request is written } }); -// --- ADR 0012 decision 6, R7 (Fix 1, C1): a repair-armed `replay --save-script` -// that comes back as a HELD divergence (the daemon's `resume.repairSessionHeld` -// signal) must keep its owning (owned/ephemeral) daemon alive and addressable. // The keep-alive keys on that signal — the REPAIR-ARMED condition — NOT on // `resume.allowed`, which reports only plan-resumability. --- @@ -908,75 +904,70 @@ test('sendToDaemon tears down an owned ephemeral daemon on an UNHELD divergence } }); -// --- ADR 0012 decision 6 (BLOCKER 2, third follow-up): a one-shot -// `replay --save-script` that completes with no divergence returns SUCCESS -// immediately — the actual healed-script commit is deferred to daemon -// teardown. If that deferred commit then fails, the daemon leaves a -// REPAIR_COMMIT_FAILED tombstone in the owned state dir before exiting. The -// client cleanup must discover it (after waiting for the daemon to actually -// exit) BEFORE deleting the owned state dir, and must surface it in the -// response the caller receives — never silently delete the only evidence of -// the failure while reporting the success already computed for the replay -// itself. --- - test('BLOCKER 2 (third follow-up): a shutdown-time repair commit failure is surfaced and the owned state dir survives', async (t) => { if (!(await supportsLoopbackBind())) { t.skip('loopback listeners are not permitted in this environment'); return; } - // The daemon's RPC response for the replay itself is a plain SUCCESS (the - // plan completed with no divergence) — exactly what a real daemon would - // return before its deferred, teardown-time commit has even attempted. - const daemon = await startHttpDaemonFixture({ session: 'default' }); - let ownedStateDir = ''; - installSpawnedHttpDaemonAtOwnedStateDir(daemon.port, (dir) => { - ownedStateDir = dir; - fs.writeFileSync( - path.join(dir, 'repair-on-shutdown.json'), - `${JSON.stringify({ - owner: 'default', - reapedAt: Date.now(), - expiresAt: Date.now() + 60_000, - sourcePath: '/tmp/flow.ad', - commitFailure: { - code: 'COMMAND_FAILED', - message: 'a prior healed script already exists at /tmp/flow.healed.ad', - }, - })}\n`, - ); - }); - - try { - const response = await sendToDaemon({ - session: 'default', - command: 'replay', - positionals: ['flow.ad'], - flags: { saveScript: true, daemonTransport: 'http' }, - meta: { requestId: 'req-repair-commit-fail-teardown' }, + for (const failRelease of [false, true]) { + const daemon = await startHttpDaemonFixture({ session: 'default' }); + let ownedStateDir = ''; + installSpawnedHttpDaemonAtOwnedStateDir(daemon.port, (dir) => { + ownedStateDir = dir; + fs.writeFileSync( + path.join(dir, 'repair-on-shutdown.json'), + `${JSON.stringify({ + owner: 'default', + reapedAt: Date.now(), + expiresAt: Date.now() + 60_000, + sourcePath: '/tmp/flow.ad', + commitFailure: { + code: 'COMMAND_FAILED', + message: 'a prior healed script already exists at /tmp/flow.healed.ad', + }, + })}\n`, + ); }); - // The client-visible response must surface the deferred commit failure — - // never the raw success the daemon returned for the replay itself, and - // never silently swallowed by cleanup. - assert.equal(response.ok, false); - if (response.ok) return; - assert.equal(response.error.code, 'REPAIR_COMMIT_FAILED'); - assert.match(response.error.message, /a prior healed script already exists/); - assert.ok(response.error.message.includes('replay /tmp/flow.ad --save-script')); + const originalRmdir = fs.rmdirSync; + const releaseSpy = vi.spyOn(fs, 'rmdirSync').mockImplementation((target, options) => { + if (failRelease && target === resolveDaemonPaths(ownedStateDir).lockPath) + throw Object.assign(new Error('release failed'), { code: 'EBUSY' }); + return originalRmdir(target, options); + }); + try { + const response = await sendToDaemon({ + session: 'default', + command: 'replay', + positionals: ['flow.ad'], + flags: { saveScript: true, daemonTransport: 'http' }, + meta: { requestId: 'req-repair-commit-fail-teardown' }, + }); - // The owned state dir — and the tombstone evidence inside it — must - // survive: never rmSync'd while an unrecovered commit failure is on record. - assert.ok(ownedStateDir.length > 0); - assert.equal(fs.existsSync(ownedStateDir), true); - const ownedPaths = resolveDaemonPaths(ownedStateDir); - assert.equal( - fs.existsSync(path.join(ownedPaths.sessionsDir, 'default', 'repair-tombstone.json')), - true, - ); - } finally { - await closeLoopbackServer(daemon.server); - if (ownedStateDir) await finishRegisteredDaemonFixture(ownedStateDir); + assert.equal(response.ok, false); + if (response.ok) return; + assert.equal(response.error.code, 'REPAIR_COMMIT_FAILED'); + const secondary = response.error.details?.cleanupFailure as + | { details?: { ownerReleaseUnverified?: boolean }; hint?: string } + | undefined; + assert.equal(secondary?.details?.ownerReleaseUnverified, failRelease ? true : undefined); + if (failRelease) assert.match(secondary?.hint ?? '', /Restore process inspection/); + assert.match(response.error.message, /a prior healed script already exists/); + assert.ok(response.error.message.includes('replay /tmp/flow.ad --save-script')); + + assert.ok(ownedStateDir.length > 0); + assert.equal(fs.existsSync(ownedStateDir), true); + const ownedPaths = resolveDaemonPaths(ownedStateDir); + assert.equal( + fs.existsSync(path.join(ownedPaths.sessionsDir, 'default', 'repair-tombstone.json')), + true, + ); + } finally { + releaseSpy.mockRestore(); + await closeLoopbackServer(daemon.server); + if (ownedStateDir) await finishRegisteredDaemonFixture(ownedStateDir); + } } }); diff --git a/src/daemon-client/daemon-client-lifecycle.ts b/src/daemon-client/daemon-client-lifecycle.ts index 03cfa395f5..c3d0508745 100644 --- a/src/daemon-client/daemon-client-lifecycle.ts +++ b/src/daemon-client/daemon-client-lifecycle.ts @@ -1,6 +1,6 @@ import fs from 'node:fs'; import net from 'node:net'; -import { AppError, normalizeError } from '@agent-device/kernel/errors'; +import { AppError, normalizeError, type NormalizedError } from '@agent-device/kernel/errors'; import { readReplayDivergenceResume } from '@agent-device/ad-replay/divergence'; import type { DaemonRequest, DaemonResponse } from '../daemon/daemon-request.ts'; import { type ExecDetachedExit } from '@agent-device/host-kit/command'; @@ -440,8 +440,12 @@ export async function cleanupDaemonAfterRequest( phase: 'daemon_replay_cleanup', data: { pid: daemon.info.pid, ...result }, }); - if (result.status === 'retired' && result.repairCommitFailure) { - return surfaceUnrecoveredRepairCommitFailure(response, result.repairCommitFailure); + if (result.status !== 'absent' && result.repairCommitFailure) { + return surfaceUnrecoveredRepairCommitFailure( + response, + result.repairCommitFailure, + result.status === 'retained' ? result.error : undefined, + ); } if (result.status === 'retained' && response?.ok) { return { @@ -483,6 +487,7 @@ export async function cleanupDaemonAfterRequest( function surfaceUnrecoveredRepairCommitFailure( response: DaemonResponse | undefined, unrecovered: NonNullable>, + cleanupFailure?: NormalizedError, ): DaemonResponse { if (response && !response.ok) return response; const { sessionName, tombstone } = unrecovered; @@ -492,7 +497,16 @@ function surfaceUnrecoveredRepairCommitFailure( const message = `The repair transaction for session "${sessionName}" completed, but committing its ` + `healed script failed at teardown: ${tombstone.commitFailure.message}. ${reRun}.`; - return { ok: false, error: normalizeError(new AppError('REPAIR_COMMIT_FAILED', message)) }; + return { + ok: false, + error: normalizeError( + new AppError( + 'REPAIR_COMMIT_FAILED', + message, + cleanupFailure ? { cleanupFailure } : undefined, + ), + ), + }; } /** diff --git a/src/daemon-registration-owner.ts b/src/daemon-registration-owner.ts index 7b95a0c52c..d2afde67bb 100644 --- a/src/daemon-registration-owner.ts +++ b/src/daemon-registration-owner.ts @@ -270,6 +270,7 @@ export type DaemonRetirementInput = Readonly<{ lockTimeoutMs?: number; }>; +type RepairCommitFailure = NonNullable>; type ConfirmedDaemonTermination = Extract; export type DaemonRetirementResult = | Readonly<{ @@ -277,7 +278,7 @@ export type DaemonRetirementResult = termination: ConfirmedDaemonTermination; removedInfo: boolean; removedStateDir?: boolean; - repairCommitFailure?: NonNullable>; + repairCommitFailure?: RepairCommitFailure; }> | Readonly<{ status: 'absent'; removedInfo: false }> | Readonly<{ @@ -295,6 +296,7 @@ export type DaemonRetirementResult = | 'metadata-unreadable' | 'retirement-unconfirmed'; error?: NormalizedError; + repairCommitFailure?: RepairCommitFailure; }>; /** Stops only the captured daemon lifetime, then retires its registration under the startup lock. */ @@ -311,12 +313,11 @@ export async function stopAndRetireDaemon( if (input.ownedStateDir) { owned = requirePrivateReplayState(input.ownedStateDir, input.paths); owned.sealed = true; - const launch = owned.startups.at(-1)?.launch; - if ( - !launch?.startTime || - launch.pid !== input.observed?.pid || - launch.startTime !== input.observed.startTime - ) + const launch = owned.startups.find( + ({ launch }) => + launch.pid === input.observed?.pid && launch.startTime === input.observed?.startTime, + )?.launch; + if (!launch?.startTime) throw new AppError( 'COMMAND_FAILED', 'The observed daemon is not an owned startup lifetime.', diff --git a/src/session-repair-tombstone.ts b/src/session-repair-tombstone.ts index 293b1ada4d..a0fae0461b 100644 --- a/src/session-repair-tombstone.ts +++ b/src/session-repair-tombstone.ts @@ -55,7 +55,7 @@ function parseRepairTombstone(raw: string, tombstonePath: string): RepairSession try { const parsed = JSON.parse(raw) as RepairSessionTombstone; if ( - typeof parsed?.expiresAt !== 'number' || + !Number.isFinite(parsed?.expiresAt) || typeof parsed?.owner !== 'string' || !validRepairCommitFailure(parsed.commitFailure) ) From ecb5472b90891457c9edc3ede03655a8960c2547 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sat, 3 Oct 2026 01:14:21 +0200 Subject: [PATCH 04/20] test: simplify paired repair cleanup assertions --- .../__tests__/daemon-client-lifecycle.test.ts | 16 +++------------- 1 file changed, 3 insertions(+), 13 deletions(-) diff --git a/src/daemon-client/__tests__/daemon-client-lifecycle.test.ts b/src/daemon-client/__tests__/daemon-client-lifecycle.test.ts index 1ba1ac1f72..89c63cb186 100644 --- a/src/daemon-client/__tests__/daemon-client-lifecycle.test.ts +++ b/src/daemon-client/__tests__/daemon-client-lifecycle.test.ts @@ -753,9 +753,6 @@ test('sendToDaemon does not replay over HTTP after the socket request is written } }); -// The keep-alive keys on that signal — the REPAIR-ARMED condition — NOT on -// `resume.allowed`, which reports only plan-resumability. --- - function heldDivergenceError( resume: Record = { allowed: true, from: 3, planDigest: 'digest-abc' }, ): Record { @@ -817,8 +814,6 @@ test('sendToDaemon keeps an owned ephemeral daemon alive and hints its --state-d assert.match(String(response.error.hint), /--state-dir/); assert.ok(String(response.error.hint).includes(ownedStateDir)); - // The daemon was NOT torn down: metadata and the owned state dir itself - // are still on disk, addressable by a follow-up command's --state-dir. const ownedPaths = resolveDaemonPaths(ownedStateDir); assert.equal(fs.existsSync(ownedPaths.infoPath), true); assert.equal(fs.existsSync(ownedPaths.lockPath), true); @@ -835,8 +830,6 @@ test('C1: keep-alive keys on repairSessionHeld, NOT resume.allowed — a HELD di return; } - // resume.allowed:false (plan not resumable), but the daemon still HELD the - // repair session — the agent must be able to reach it to close/inspect. const daemon = await startHttpDaemonErrorFixture( heldDivergenceError({ allowed: false, @@ -895,8 +888,6 @@ test('sendToDaemon tears down an owned ephemeral daemon on an UNHELD divergence if (response.ok) return; assert.equal(response.error.hint, undefined); assert.ok(ownedStateDir.length > 0); - // No held signal (`resume.allowed:true` alone is not the keep-alive key) — - // ordinary one-shot teardown still applies. assert.equal(fs.existsSync(ownedStateDir), false); } finally { await closeLoopbackServer(daemon.server); @@ -945,14 +936,13 @@ test('BLOCKER 2 (third follow-up): a shutdown-time repair commit failure is surf meta: { requestId: 'req-repair-commit-fail-teardown' }, }); - assert.equal(response.ok, false); - if (response.ok) return; + assert.ok(!response.ok); assert.equal(response.error.code, 'REPAIR_COMMIT_FAILED'); const secondary = response.error.details?.cleanupFailure as | { details?: { ownerReleaseUnverified?: boolean }; hint?: string } | undefined; - assert.equal(secondary?.details?.ownerReleaseUnverified, failRelease ? true : undefined); - if (failRelease) assert.match(secondary?.hint ?? '', /Restore process inspection/); + assert.equal(Boolean(secondary?.details?.ownerReleaseUnverified), failRelease); + assert.equal(Boolean(secondary?.hint?.startsWith('Restore process inspection')), failRelease); assert.match(response.error.message, /a prior healed script already exists/); assert.ok(response.error.message.includes('replay /tmp/flow.ad --save-script')); From 2a359db702738041ae8da20fa4de81ae45769b5f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sat, 3 Oct 2026 02:56:47 +0200 Subject: [PATCH 05/20] fix: keep daemon lifecycle outside the CLI import closure --- src/daemon-client/daemon-client-lifecycle.ts | 24 +++++++- src/daemon-client/daemon-client.ts | 63 ++------------------ 2 files changed, 26 insertions(+), 61 deletions(-) diff --git a/src/daemon-client/daemon-client-lifecycle.ts b/src/daemon-client/daemon-client-lifecycle.ts index c3d0508745..62495a82a8 100644 --- a/src/daemon-client/daemon-client-lifecycle.ts +++ b/src/daemon-client/daemon-client-lifecycle.ts @@ -523,7 +523,7 @@ function surfaceUnrecoveredRepairCommitFailure( * `resume.allowed` (plan-resumability): a held divergence with `allowed: false` * still holds the session so the agent can inspect and `close` cleanly. */ -export function isHeldRepairDivergence(response: DaemonResponse | undefined): boolean { +function isHeldRepairDivergence(response: DaemonResponse | undefined): boolean { if (!response || response.ok) return false; if (response.error.code !== 'REPLAY_DIVERGENCE') return false; const resume = readReplayDivergenceResume(response.error.details?.divergence); @@ -538,7 +538,7 @@ export function isHeldRepairDivergence(response: DaemonResponse | undefined): bo * selector-miss's own guidance) so the agent's next command knows to target * the SAME daemon instead of resolving to the default one. */ -export function attachRepairSessionAddressHint( +function attachRepairSessionAddressHint( response: Extract, stateDir: string, ): Extract { @@ -570,7 +570,7 @@ function isOneShotReplayCommand(command: string | undefined): boolean { * anyway, but the explicit command check keeps that carve-out a decision * rather than an accident of the response shape. */ -export function isActiveReplaySessionResponse( +function isActiveReplaySessionResponse( req: Omit, response: DaemonResponse | undefined, ): boolean { @@ -756,3 +756,21 @@ function isLoopbackHostname(hostname: string): boolean { if (net.isIPv6(normalized)) return LOOPBACK_BLOCK_LIST.check(normalized, 'ipv6'); return false; } + +export function attachSessionAddressHints( + response: DaemonResponse, + req: Omit, + settings: DaemonClientSettings, +): DaemonResponse { + if (!response.ok) { + return settings.ownedStateDir && isHeldRepairDivergence(response) + ? attachRepairSessionAddressHint(response, settings.paths.baseDir) + : response; + } + return isActiveReplaySessionResponse(req, response) + ? attachActiveSessionAddressHint( + response, + settings.ownedStateDir ? settings.paths.baseDir : undefined, + ) + : response; +} diff --git a/src/daemon-client/daemon-client.ts b/src/daemon-client/daemon-client.ts index 7d777254fb..4d5e97e561 100644 --- a/src/daemon-client/daemon-client.ts +++ b/src/daemon-client/daemon-client.ts @@ -17,17 +17,7 @@ import { prepareRemoteRequestArtifacts, type PreparedRemoteRequest, } from '../remote/daemon-artifacts.ts'; -import { - attachActiveSessionAddressHint, - attachRepairSessionAddressHint, - cleanupDaemonAfterRequest, - ensureDaemon, - isActiveReplaySessionResponse, - isHeldRepairDivergence, - resolveClientSettings, - type DaemonClientSettings, - type EnsuredDaemon, -} from './daemon-client-lifecycle.ts'; +import type { DaemonClientSettings, EnsuredDaemon } from './daemon-client-lifecycle.ts'; import { sendRequest } from './daemon-client-transport.ts'; import { isRemoteDaemon, type DaemonInfo } from './daemon-client-metadata.ts'; import { leaseScopeFromRequest } from '@agent-device/contracts/lease-scope'; @@ -42,6 +32,8 @@ export async function sendToDaemon( req: Omit, options: DaemonTransportOptions = {}, ): Promise { + const { resolveClientSettings, ensureDaemon, attachSessionAddressHints } = + await import('./daemon-client-lifecycle.ts'); const requestId = req.meta?.requestId ?? createRequestId(); const debug = Boolean(req.meta?.debug || req.flags?.verbose); // A few internal callers build DaemonRequest directly instead of using the @@ -107,11 +99,7 @@ export async function sendToDaemon( ), { requestId, command: req.command }, ); - return withActiveSessionAddressHint( - withRepairSessionAddressHintIfOwned(response, settings), - requestWithoutAuthFlag, - settings, - ); + return attachSessionAddressHints(response, requestWithoutAuthFlag, settings); }, ); } @@ -235,6 +223,7 @@ async function performDaemonRequestWithCleanup( requestFailed = true; requestError = error; } + const { cleanupDaemonAfterRequest } = await import('./daemon-client-lifecycle.ts'); const finalResponse = await cleanupDaemonAfterRequest(req, daemon, settings, response); if (requestFailed) throw requestError; if (!finalResponse) { @@ -246,48 +235,6 @@ async function performDaemonRequestWithCleanup( return finalResponse; } -/** - * ADR 0012 decision 6 (Fix 1): the owned ephemeral state dir this daemon was - * started at is otherwise unaddressable by a later invocation — hint it here, - * only when the daemon is actually being kept alive for it - * (`settings.ownedStateDir` means `daemon.startedByClient` is also true). - */ -function withRepairSessionAddressHintIfOwned( - response: DaemonResponse, - settings: DaemonClientSettings, -): DaemonResponse { - if (response.ok || !settings.ownedStateDir || !isHeldRepairDivergence(response)) { - return response; - } - return attachRepairSessionAddressHint(response, settings.paths.baseDir); -} - -/** - * ADR 0016 counterpart to `withRepairSessionAddressHintIfOwned` — but unlike - * that one, NOT gated on `settings.ownedStateDir`. An owned ephemeral state - * dir is unaddressable by a later invocation either way, so it's included - * when owned; an explicit `--state-dir`/`AGENT_DEVICE_STATE_DIR` caller - * already knows their own dir, so it's omitted then. But the session's own - * name is cwd-qualified and, per #1394, `session list` cannot rediscover it - * either — so `--session` is still worth hinting even at an explicit state - * dir, which is why this runs for every active-session response regardless - * of `ownedStateDir` (`attachActiveSessionAddressHint` itself decides what, - * if anything, is worth attaching). - */ -function withActiveSessionAddressHint( - response: DaemonResponse, - req: Omit, - settings: DaemonClientSettings, -): DaemonResponse { - if (!response.ok || !isActiveReplaySessionResponse(req, response)) { - return response; - } - return attachActiveSessionAddressHint( - response, - settings.ownedStateDir ? settings.paths.baseDir : undefined, - ); -} - function writeInstallInProgressNotice(command: string | undefined): void { if (!isInstallLikeCommand(command) || process.stderr.isTTY !== true || process.env.CI) return; process.stderr.write( From 3e5be9adbce738b24a465ac6f34ac86d6c347bbf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sat, 3 Oct 2026 01:44:41 +0200 Subject: [PATCH 06/20] fix(daemon): retire only the captured failed startup --- .../__tests__/daemon-client-lifecycle.test.ts | 32 ++- .../__tests__/daemon-client-metadata.test.ts | 18 +- .../daemon-client-startup-race.test.ts | 2 +- .../__tests__/daemon-client.test.ts | 183 +----------------- src/daemon-client/daemon-client-lifecycle.ts | 130 +++++++------ src/daemon-client/daemon-client-metadata.ts | 101 ---------- 6 files changed, 93 insertions(+), 373 deletions(-) diff --git a/src/daemon-client/__tests__/daemon-client-lifecycle.test.ts b/src/daemon-client/__tests__/daemon-client-lifecycle.test.ts index 89c63cb186..d4d5ae9880 100644 --- a/src/daemon-client/__tests__/daemon-client-lifecycle.test.ts +++ b/src/daemon-client/__tests__/daemon-client-lifecycle.test.ts @@ -314,7 +314,7 @@ function mockSocketErrorAfterWrite(failingPort: number): { }; } -test('sendToDaemon retries daemon spawn failures and cleans partial metadata on terminal failure', async () => { +test('sendToDaemon retains unknown metadata after a spawn failure', async () => { const stateDir = makeTempStateDir('agent-device-daemon-spawn-retry-'); const paths = resolveDaemonPaths(stateDir); vi.stubEnv('AGENT_DEVICE_STATE_DIR', stateDir); @@ -345,25 +345,17 @@ test('sendToDaemon retries daemon spawn failures and cleans partial metadata on assert.ok(thrown instanceof AppError); assert.equal(thrown.message, 'Failed to start daemon'); - assert.equal(thrown.details?.startError, 'spawn failed 2'); - assert.equal(thrown.details?.startupAttempts, 2); - const cleanupResults = thrown.details?.cleanupResults; - assert.ok(Array.isArray(cleanupResults)); - assert.deepEqual( - cleanupResults.map((result) => ({ - reason: result.reason, - removedInfo: result.removedInfo, - removedLock: result.removedLock, - })), - [ - { reason: 'start_error', removedInfo: true, removedLock: true }, - { reason: 'start_error', removedInfo: true, removedLock: true }, - ], - ); - assert.equal(attempts, 2); - assert.equal(mockSleep.mock.calls[0]?.[0], 150); - assert.equal(fs.existsSync(paths.infoPath), false); - assert.equal(fs.existsSync(paths.lockPath), false); + assert.equal(fs.readFileSync(paths.infoPath, 'utf8'), '{"partial":true}\n'); + assert.equal(fs.readFileSync(paths.lockPath, 'utf8'), 'not-json\n'); + assert.equal(thrown.details?.startError, 'spawn failed 1'); + assert.equal(thrown.details?.startupAttempts, 1); + const results = thrown.details?.cleanupResults as Array<{ + status: string; + removedInfo: boolean; + }>; + assert.equal(results[0]?.status, 'retained'); + assert.equal(results[0]?.removedInfo, false); + assert.equal(attempts, 1); } finally { fs.rmSync(stateDir, { recursive: true, force: true }); } diff --git a/src/daemon-client/__tests__/daemon-client-metadata.test.ts b/src/daemon-client/__tests__/daemon-client-metadata.test.ts index 5b46a91e8d..6bfec46b6e 100644 --- a/src/daemon-client/__tests__/daemon-client-metadata.test.ts +++ b/src/daemon-client/__tests__/daemon-client-metadata.test.ts @@ -5,10 +5,12 @@ import path from 'node:path'; import { afterEach, test, vi } from 'vitest'; import type { DaemonCodeOrigin } from '@agent-device/host-kit/code-signature'; import { mkdtempForTestSync } from '../../__tests__/test-utils/tmp-dir.ts'; -import { tryAcquireDaemonRegistration } from '../../daemon-registration-owner.ts'; +import { + stopAndRetireDaemon, + tryAcquireDaemonRegistration, +} from '../../daemon-registration-owner.ts'; import { readDaemonInfo, - cleanupFailedDaemonStartupMetadata, stopDaemonProcessForTakeover, type DaemonInfo, } from '../daemon-client-metadata.ts'; @@ -81,13 +83,15 @@ for (const artifact of ['daemon.json', 'daemon.lock']) { fs.writeFileSync(file, contents); vi.mocked(isAgentDeviceDaemonProcess).mockReturnValue(true); vi.mocked(stopDaemonProcess).mockResolvedValue({ status: 'retained', reason: 'exit-timeout' }); - const result = await cleanupFailedDaemonStartupMetadata(paths, 'start_error'); + const result = await stopAndRetireDaemon({ + paths, + observed: { pid: 7, startTime: 'start' }, + mode: 'graceful', + }); assert.equal(fs.readFileSync(file, 'utf8'), contents); assert.equal(result.removedInfo, false); - assert.equal(result.removedLock, false); - assert.equal(result.stoppedInfoProcess, false); - assert.equal(result.stoppedLockProcess, false); - assert.match(result.error ?? '', /exit could not be confirmed/); + assert.equal(result.status, 'retained'); + if (result.status === 'retained') assert.equal(result.reason, 'exit-unconfirmed'); }); } diff --git a/src/daemon-client/__tests__/daemon-client-startup-race.test.ts b/src/daemon-client/__tests__/daemon-client-startup-race.test.ts index d52ba0e6c2..fa87f3f65b 100644 --- a/src/daemon-client/__tests__/daemon-client-startup-race.test.ts +++ b/src/daemon-client/__tests__/daemon-client-startup-race.test.ts @@ -200,7 +200,7 @@ test('a start race won by an older daemon replaces it instead of adopting it', a }), ); assert.equal(fixture.rpcRequests.length, 0); - assert.equal(launches, 2); + assert.equal(launches, 1); assert.match( String(stderr.mock.calls.flat().join('')), /Replacing daemon \(pid 43300, v0\.0\.1\)/, diff --git a/src/daemon-client/__tests__/daemon-client.test.ts b/src/daemon-client/__tests__/daemon-client.test.ts index f579a3c4a5..accebf0782 100644 --- a/src/daemon-client/__tests__/daemon-client.test.ts +++ b/src/daemon-client/__tests__/daemon-client.test.ts @@ -1,5 +1,5 @@ import type { RequestProgressEvent } from '@agent-device/contracts/progress'; -import { test, vi } from 'vitest'; +import { test } from 'vitest'; import assert from 'node:assert/strict'; import http from 'node:http'; import net from 'node:net'; @@ -11,57 +11,18 @@ import { listenOnLoopback, supportsLoopbackBind, } from '../../__tests__/test-utils/loopback.ts'; -import { runCmdBackground } from '@agent-device/host-kit/command'; -import { - isProcessAlive, - readProcessCommand, - readProcessStartTime, - waitForProcessExit, -} from '@agent-device/host-kit/process'; +import { readProcessStartTime } from '@agent-device/host-kit/process'; import { sendToDaemon } from '../daemon-client.ts'; import { currentDaemonCodeSignature } from '../../__tests__/test-utils/daemon-http-fixture.ts'; import { computeDaemonCodeSignature } from '@agent-device/host-kit/code-signature'; import { downloadRemoteArtifact } from '../../remote/daemon-artifacts.ts'; -import { - cleanupFailedDaemonStartupMetadata, - resolveDaemonStartupHint, -} from '../daemon-client-metadata.ts'; +import { resolveDaemonStartupHint } from '../daemon-client-metadata.ts'; import { canConnectSocket } from '../daemon-client-transport.ts'; import { DAEMON_RPC_PROTOCOL_VERSION } from '@agent-device/contracts/daemon-http'; import { resolveDaemonPaths } from '../../daemon-resolution.ts'; import { readVersion } from '@agent-device/host-kit/version'; import { mkdtempForTestSync } from '../../__tests__/test-utils/tmp-dir.ts'; -// readProcessStartTime/readProcessCommand shell out to `ps` with a 1s -// timeout (see host-process.ts). isAgentDeviceDaemonProcess re-reads both for -// every liveness check, so a spawned-daemon fixture that is proven live once -// (a real read, right after the process starts) can still be misclassified -// as dead later if a *subsequent* `ps` call happens to miss its deadline -// under full-suite CPU contention. mockReadProcessStartTime/mockReadProcessCommand -// default to `undefined`, which falls through to the real implementation for -// every pid in every test in this file; only the one test below that needs a -// stable answer for its spawned pid configures an override, and clears it -// afterward. -const { mockReadProcessStartTime, mockReadProcessCommand } = vi.hoisted(() => ({ - mockReadProcessStartTime: vi.fn<(pid: number) => string | null | undefined>(), - mockReadProcessCommand: vi.fn<(pid: number) => string | null | undefined>(), -})); - -vi.mock('@agent-device/host-kit/process', async (importOriginal) => { - const actual = await importOriginal(); - return { - ...actual, - readProcessStartTime: (pid: number) => { - const overridden = mockReadProcessStartTime(pid); - return overridden !== undefined ? overridden : actual.readProcessStartTime(pid); - }, - readProcessCommand: (pid: number) => { - const overridden = mockReadProcessCommand(pid); - return overridden !== undefined ? overridden : actual.readProcessCommand(pid); - }, - }; -}); - type MockHttpResponse = EventEmitter & { headers?: Record; statusCode?: number; @@ -219,144 +180,6 @@ test('resolveDaemonStartupHint shell-quotes cleanup paths', () => { ); }); -test('cleanupFailedDaemonStartupMetadata removes partial startup metadata', async () => { - const stateDir = mkdtempForTestSync('agent-device-daemon-cleanup-'); - const paths = resolveDaemonPaths(stateDir); - try { - fs.mkdirSync(paths.baseDir, { recursive: true }); - fs.writeFileSync(paths.infoPath, '{"invalid":true}\n', 'utf8'); - fs.writeFileSync(paths.lockPath, 'not-json\n', 'utf8'); - - const result = await cleanupFailedDaemonStartupMetadata(paths, 'startup_timeout'); - - assert.deepEqual(result, { - reason: 'startup_timeout', - removedInfo: true, - removedLock: true, - stoppedInfoProcess: false, - stoppedLockProcess: false, - }); - assert.equal(fs.existsSync(paths.infoPath), false); - assert.equal(fs.existsSync(paths.lockPath), false); - } finally { - fs.rmSync(stateDir, { recursive: true, force: true }); - } -}); - -test('cleanupFailedDaemonStartupMetadata retains live startup daemon on timeout', async (t) => { - const stateDir = mkdtempForTestSync('agent-device-daemon-live-cleanup-'); - const root = mkdtempForTestSync('agent-device-live-daemon-'); - const daemonDir = path.join(root, 'agent-device', 'dist', 'src', 'internal'); - const daemonScriptPath = path.join(daemonDir, 'daemon.js'); - fs.mkdirSync(daemonDir, { recursive: true }); - fs.writeFileSync(daemonScriptPath, 'setInterval(() => {}, 1000);\n', 'utf8'); - const daemonProcess = runCmdBackground(process.execPath, [daemonScriptPath], { - stdio: 'ignore', - allowFailure: true, - captureOutput: false, - }); - void daemonProcess.wait.catch(() => {}); - const pid = daemonProcess.child.pid; - assert.ok(pid, 'spawned child should have a pid'); - - try { - await new Promise((resolve) => setTimeout(resolve, 50)); - // Read the spawned daemon's real identity once (ground truth: it is - // genuinely alive, with this real start time and command line), then - // pin readProcessStartTime/readProcessCommand to keep returning these - // same proven-real values for this pid. isAgentDeviceDaemonProcess reads - // both again internally on every call inside cleanupFailedDaemonStartupMetadata; - // without pinning, a second real `ps` call could miss its 1s timeout - // under load and misclassify this genuinely-live daemon as dead. - const processStartTime = readProcessStartTime(pid) ?? undefined; - const command = readProcessCommand(pid); - if (command === null || processStartTime === undefined) { - t.skip('process command/start inspection is unavailable in this environment'); - return; - } - mockReadProcessStartTime.mockImplementation((queriedPid: number) => - queriedPid === pid ? processStartTime : undefined, - ); - mockReadProcessCommand.mockImplementation((queriedPid: number) => - queriedPid === pid ? command : undefined, - ); - - const paths = resolveDaemonPaths(stateDir); - fs.mkdirSync(paths.baseDir, { recursive: true }); - fs.writeFileSync( - paths.infoPath, - `${JSON.stringify({ - token: 'startup-secret', - port: 65530, - transport: 'socket', - pid, - processStartTime, - })}\n`, - 'utf8', - ); - fs.writeFileSync( - paths.lockPath, - `${JSON.stringify({ pid, processStartTime, startedAt: Date.now() })}\n`, - 'utf8', - ); - - const result = await cleanupFailedDaemonStartupMetadata(paths, 'startup_timeout', { - stopLiveProcesses: false, - }); - - assert.equal(result.retainedInfoProcess, true); - assert.equal(result.retainedLockProcess, true); - assert.equal(result.removedInfo, false); - assert.equal(result.removedLock, false); - assert.equal(isProcessAlive(pid), true); - assert.equal(fs.existsSync(paths.infoPath), true); - assert.equal(fs.existsSync(paths.lockPath), true); - } finally { - mockReadProcessStartTime.mockReset(); - mockReadProcessCommand.mockReset(); - if (isProcessAlive(pid)) { - process.kill(pid, 'SIGKILL'); - await waitForProcessExit(pid, 1_500); - } - fs.rmSync(stateDir, { recursive: true, force: true }); - fs.rmSync(root, { recursive: true, force: true }); - } -}); - -test('cleanupFailedDaemonStartupMetadata removes stale daemon metadata on timeout', async () => { - const stateDir = mkdtempForTestSync('agent-device-daemon-stale-cleanup-'); - const paths = resolveDaemonPaths(stateDir); - try { - fs.mkdirSync(paths.baseDir, { recursive: true }); - fs.writeFileSync( - paths.infoPath, - `${JSON.stringify({ - token: 'startup-secret', - port: 65530, - transport: 'socket', - pid: 999_999, - })}\n`, - 'utf8', - ); - fs.writeFileSync( - paths.lockPath, - `${JSON.stringify({ pid: 999_999, startedAt: Date.now() })}\n`, - 'utf8', - ); - - const result = await cleanupFailedDaemonStartupMetadata(paths, 'startup_timeout', { - stopLiveProcesses: false, - }); - - assert.equal(result.removedInfo, true); - assert.equal(result.removedLock, true); - assert.equal(fs.existsSync(paths.infoPath), false); - assert.equal(fs.existsSync(paths.lockPath), false); - } finally { - fs.rmSync(stateDir, { recursive: true, force: true }); - } -}); - test('canConnectSocket times out stalled local daemon probes', async () => { const originalCreateConnection = net.createConnection; let timeoutMs: number | undefined; diff --git a/src/daemon-client/daemon-client-lifecycle.ts b/src/daemon-client/daemon-client-lifecycle.ts index 62495a82a8..7e75749e61 100644 --- a/src/daemon-client/daemon-client-lifecycle.ts +++ b/src/daemon-client/daemon-client-lifecycle.ts @@ -8,10 +8,13 @@ import { shellQuoteIfNeeded } from '@agent-device/kernel/device-shell'; import { emitDiagnostic } from '@agent-device/host-kit/diagnostics'; import { isProcessAlive } from '@agent-device/host-kit/process'; import { sleep } from '@agent-device/host-kit/retry'; +import { inspectProcessLock } from '@agent-device/host-kit/file'; import type { findUnrecoveredRepairCommitFailure } from '../session-repair-tombstone.ts'; import { createOwnedReplayStateDir, + recoverAbandonedDaemonRegistration, + type DaemonRetirementResult, launchDaemonProcess, stopAndRetireDaemon, type OwnedReplayStateDir, @@ -33,18 +36,15 @@ import { import { PUBLIC_COMMANDS } from '@agent-device/command-registry/catalog'; import { - cleanupFailedDaemonStartupMetadata, cleanupStaleDaemonLockIfSafe, getDaemonMetadataState, isDaemonLockHeldByAnotherDaemon, isRemoteDaemon, readDaemonInfo, - recoverDaemonLockHolder, removeDaemonInfo, resolveDaemonStartupHint, stopDaemonProcessForTakeover, type DaemonInfo, - type DaemonStartupCleanupResult, } from './daemon-client-metadata.ts'; import { canConnect, @@ -297,65 +297,27 @@ function emitDaemonTakeoverNotice(info: DaemonInfo, reason: string, stateDir: st } } -async function startLocalDaemon(settings: DaemonClientSettings): Promise { - let lockRecoveryCount = 0; - const cleanupResults: DaemonStartupCleanupResult[] = []; - let startError: string | undefined; - let daemonProcess: ExecDetachedExit | { pid: number } | undefined; - for (let attempt = 1; attempt <= DAEMON_STARTUP_ATTEMPTS; attempt += 1) { - let launch: DaemonStartupLaunch; - try { - launch = startDaemon(settings); - daemonProcess = { pid: launch.pid }; - } catch (error) { - startError = error instanceof Error ? error.message : String(error); - cleanupResults.push(await cleanupFailedDaemonStartupMetadata(settings.paths, 'start_error')); - if (attempt < DAEMON_STARTUP_ATTEMPTS) { - await sleep(150); - continue; - } - break; - } - - const startup = await waitForDaemonStartup(DAEMON_STARTUP_TIMEOUT_MS, settings, launch); - if (startup.kind === 'ready') return startup.daemon; - if (startup.kind === 'early_exit') { - daemonProcess = startup.exit; - startError = describeDaemonEarlyExit(startup.exit); - cleanupResults.push(await cleanupFailedDaemonStartupMetadata(settings.paths, 'start_error')); - if (attempt < DAEMON_STARTUP_ATTEMPTS) { - await sleep(150); - continue; - } - break; - } - - if (await recoverDaemonLockHolder(settings.paths)) { - lockRecoveryCount += 1; - continue; - } - - const metadataState = getDaemonMetadataState(settings.paths); - const hasAnotherAttempt = attempt < DAEMON_STARTUP_ATTEMPTS; - const cleanup = await cleanupFailedDaemonStartupMetadata(settings.paths, 'startup_timeout', { - stopLiveProcesses: false, - }); - cleanupResults.push(cleanup); - if (cleanup.retainedInfoProcess || cleanup.retainedLockProcess) { - const extended = await waitForDaemonStartup(DAEMON_STARTUP_TIMEOUT_MS, settings, launch); - if (extended.kind === 'ready') return extended.daemon; - if (extended.kind === 'early_exit') { - daemonProcess = extended.exit; - startError = describeDaemonEarlyExit(extended.exit); - } - break; - } - if (!hasAnotherAttempt) break; +type FailedDaemonStartup = { + cleanup: DaemonRetirementResult; + startError?: string; + daemonProcess?: ExecDetachedExit | { pid: number }; + retry: boolean; +}; - // Detached daemon startup can race on busy CI hosts; retry when no metadata exists yet. - if (!metadataState.hasInfo && !metadataState.hasLock) await sleep(150); +async function startLocalDaemon(settings: DaemonClientSettings): Promise { + const deadline = Date.now() + DAEMON_STARTUP_TIMEOUT_MS; + const cleanupResults: DaemonRetirementResult[] = []; + let failure: FailedDaemonStartup | undefined; + let attempts = 0; + while (attempts < DAEMON_STARTUP_ATTEMPTS && Date.now() < deadline) { + attempts += 1; + const result = await attemptLocalDaemonStartup(settings, deadline); + if ('daemon' in result) return result.daemon; + failure = result; + cleanupResults.push(result.cleanup); + if (!result.retry) break; + await sleep(Math.min(150, Math.max(0, deadline - Date.now()))); } - const state = getDaemonMetadataState(settings.paths); const daemonLogTail = readRecentLogTail(settings.paths.logPath); throw new AppError('COMMAND_FAILED', 'Failed to start daemon', { @@ -365,17 +327,57 @@ async function startLocalDaemon(settings: DaemonClientSettings): Promise { + let launch: DaemonStartupLaunch; + try { + launch = startDaemon(settings); + } catch (error) { + const cleanup = await recoverAbandonedDaemonRegistration({ + paths: settings.paths, + observed: null, + lockTimeoutMs: 0, + }); + return { + cleanup, + startError: normalizeError(error).message, + retry: cleanup.status !== 'retained', + }; + } + const startup = await waitForDaemonStartup(Math.max(0, deadline - Date.now()), settings, launch); + if (startup.kind === 'ready') return { daemon: startup.daemon }; + const cleanup = await stopAndRetireDaemon({ + paths: settings.paths, + observed: { pid: launch.pid, startTime: launch.startTime ?? null }, + mode: 'graceful', + lockTimeoutMs: 0, + }); + const inspection = inspectProcessLock(settings.paths.lockPath); + const available = + inspection.state === 'absent' || + (inspection.state === 'held' && + (inspection.liveness === 'owner-process-dead' || + inspection.liveness === 'owner-process-reused')); + return { + cleanup, + retry: startup.kind === 'early_exit' && available, + startError: startup.kind === 'early_exit' ? describeDaemonEarlyExit(startup.exit) : undefined, + daemonProcess: startup.kind === 'early_exit' ? startup.exit : { pid: launch.pid }, + }; +} + /** * ADR 0012 decision 6 (BLOCKER 2, third follow-up): a one-shot repair * (`replay --save-script`) that COMPLETES without diverging returns SUCCESS diff --git a/src/daemon-client/daemon-client-metadata.ts b/src/daemon-client/daemon-client-metadata.ts index 3a448b099f..bf886cef5c 100644 --- a/src/daemon-client/daemon-client-metadata.ts +++ b/src/daemon-client/daemon-client-metadata.ts @@ -1,7 +1,6 @@ import fs from 'node:fs'; import { AppError } from '@agent-device/kernel/errors'; import { shellQuote } from '@agent-device/kernel/device-shell'; -import { emitDiagnostic } from '@agent-device/host-kit/diagnostics'; import { isAgentDeviceDaemonProcess, stopDaemonProcess, @@ -44,19 +43,6 @@ export type DaemonMetadataState = { hasLock: boolean; }; -type DaemonStartupCleanupReason = 'start_error' | 'startup_timeout'; - -export type DaemonStartupCleanupResult = { - reason: DaemonStartupCleanupReason; - removedInfo: boolean; - removedLock: boolean; - stoppedInfoProcess: boolean; - stoppedLockProcess: boolean; - retainedInfoProcess?: boolean; - retainedLockProcess?: boolean; - error?: string; -}; - const DAEMON_TAKEOVER_TERM_TIMEOUT_MS = 3000; const DAEMON_TAKEOVER_KILL_TIMEOUT_MS = 1000; @@ -161,78 +147,6 @@ export function cleanupStaleDaemonLockIfSafe(paths: DaemonPaths): void { removeDaemonLock(paths.lockPath); } -export async function cleanupFailedDaemonStartupMetadata( - paths: DaemonPaths, - reason: DaemonStartupCleanupReason, - options: { stopLiveProcesses?: boolean } = {}, -): Promise { - const stopLiveProcesses = options.stopLiveProcesses ?? true; - const result: DaemonStartupCleanupResult = { - reason, - removedInfo: false, - removedLock: false, - stoppedInfoProcess: false, - stoppedLockProcess: false, - }; - - try { - const infoExists = fs.existsSync(paths.infoPath); - const info = readDaemonInfo(paths.infoPath); - if (info) { - const liveInfoProcess = isAgentDeviceDaemonProcess(info.pid, info.processStartTime); - if (liveInfoProcess && !stopLiveProcesses) { - result.retainedInfoProcess = true; - } else { - if (liveInfoProcess) { - await stopDaemonProcessForTakeover(info); - result.stoppedInfoProcess = true; - } - removeDaemonInfo(paths.infoPath); - result.removedInfo = true; - } - } else if (infoExists) { - removeDaemonInfo(paths.infoPath); - result.removedInfo = true; - } - - const lockExists = fs.existsSync(paths.lockPath); - const lockInfo = readDaemonLockInfo(paths.lockPath); - if (lockInfo) { - const liveLockProcess = isAgentDeviceDaemonProcess(lockInfo.pid, lockInfo.processStartTime); - if (liveLockProcess && !stopLiveProcesses) { - result.retainedLockProcess = true; - } else { - if (liveLockProcess) { - const termination = await stopDaemonProcess( - { pid: lockInfo.pid, startTime: lockInfo.processStartTime ?? null }, - { - mode: 'graceful', - termTimeoutMs: DAEMON_TAKEOVER_TERM_TIMEOUT_MS, - killTimeoutMs: DAEMON_TAKEOVER_KILL_TIMEOUT_MS, - }, - ); - requireDaemonExit(termination); - result.stoppedLockProcess = true; - } - removeDaemonLock(paths.lockPath); - result.removedLock = true; - } - } else if (lockExists) { - removeDaemonLock(paths.lockPath); - result.removedLock = true; - } - } catch (error) { - result.error = error instanceof Error ? error.message : String(error); - } - - emitDiagnostic({ - level: result.error ? 'warn' : 'info', - phase: 'daemon_startup_metadata_cleanup', - data: result, - }); - return result; -} - export function getDaemonMetadataState(paths: DaemonPaths): DaemonMetadataState { return { hasInfo: fs.existsSync(paths.infoPath), @@ -240,21 +154,6 @@ export function getDaemonMetadataState(paths: DaemonPaths): DaemonMetadataState }; } -export async function recoverDaemonLockHolder(paths: DaemonPaths): Promise { - const state = getDaemonMetadataState(paths); - if (!state.hasLock || state.hasInfo) return false; - const lockInfo = readDaemonLockInfo(paths.lockPath); - if (!lockInfo) { - removeDaemonLock(paths.lockPath); - return true; - } - if (!isAgentDeviceDaemonProcess(lockInfo.pid, lockInfo.processStartTime)) { - removeDaemonLock(paths.lockPath); - return true; - } - return false; -} - export async function stopDaemonProcessForTakeover( info: DaemonInfo, ): Promise { From 5ec91d73ff0de87a99ebca8633fda15f1e04c26b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sat, 3 Oct 2026 02:44:53 +0200 Subject: [PATCH 07/20] fix: join contending daemon startups within one deadline --- .../test-utils/registered-daemon-fixture.ts | 17 +- .../__tests__/daemon-client-lifecycle.test.ts | 55 ++- .../daemon-client-startup-race.test.ts | 463 +++++++++++------- .../__tests__/daemon-client.test.ts | 50 +- src/daemon-client/daemon-client-lifecycle.ts | 225 +++++++-- src/daemon-client/daemon-client-metadata.ts | 78 +-- src/daemon-client/daemon-client-transport.ts | 24 +- website/docs/docs/installation.md | 19 +- 8 files changed, 560 insertions(+), 371 deletions(-) diff --git a/src/__tests__/test-utils/registered-daemon-fixture.ts b/src/__tests__/test-utils/registered-daemon-fixture.ts index 63ee16a7bc..12af382c10 100644 --- a/src/__tests__/test-utils/registered-daemon-fixture.ts +++ b/src/__tests__/test-utils/registered-daemon-fixture.ts @@ -13,7 +13,7 @@ const actualCommand = await vi.importActual; startTime: string | null } + Array<{ launch: ReturnType; startTime: string | null }> >(); /** A real registration owner, advertising the caller's HTTP fixture and joining before deletion. */ @@ -29,10 +29,10 @@ export function registeredDaemonFixtureArgs( entry, `import fs from 'node:fs'; import path from 'node:path'; -import { tryAcquireDaemonRegistration } from ${JSON.stringify(registrationUrl)}; +import { DAEMON_STARTUP_EXIT_CODES, tryAcquireDaemonRegistration } from ${JSON.stringify(registrationUrl)}; const paths = ${JSON.stringify(paths)}; const acquired = await tryAcquireDaemonRegistration(paths); -if (acquired.status !== 'acquired') process.exit(75); +if (acquired.status !== 'acquired') process.exit(DAEMON_STARTUP_EXIT_CODES[acquired.status]); process.on('SIGTERM', async () => { const deferred = path.join(paths.baseDir, 'repair-on-shutdown.json'); if (fs.existsSync(deferred)) { @@ -43,6 +43,8 @@ process.on('SIGTERM', async () => { await acquired.owner.finish(); process.exit(0); }); +fs.writeFileSync(path.join(paths.baseDir, 'registration-held'), 'ready'); +while (fs.existsSync(path.join(paths.baseDir, 'defer-publication'))) await new Promise(resolve => setTimeout(resolve, 10)); acquired.owner.publish(${JSON.stringify(fields)}); setInterval(() => {}, 1000); `, @@ -60,13 +62,14 @@ export function spawnRegisteredDaemonFixture( registeredDaemonFixtureArgs(paths, fields), options, ); - children.set(paths.baseDir, { launch: child, startTime: readProcessStartTime(child.pid) }); + const owned = children.get(paths.baseDir) ?? []; + owned.push({ launch: child, startTime: readProcessStartTime(child.pid) }); + children.set(paths.baseDir, owned); return child; } export async function finishRegisteredDaemonFixture(stateDir: string): Promise { - const owned = children.get(stateDir); - if (owned) { + for (const owned of children.get(stateDir) ?? []) { const child = owned.launch; const termination = await stopDaemonProcess( { pid: child.pid, startTime: owned.startTime }, @@ -74,8 +77,8 @@ export async function finishRegisteredDaemonFixture(stateDir: string): Promise { } function installSpawnedHttpDaemon(paths: DaemonPaths, httpPort: number): void { + mockSleep.mockImplementation(actualRetry.sleep); mockRunCmdDetached.mockImplementation((_command, _args, options) => { assert.equal(options?.env?.AGENT_DEVICE_STATE_DIR, paths.baseDir); - writeDaemonInfo(paths, { httpPort, transport: 'http' }); - writeDaemonLock(paths, { - pid: process.pid, - processStartTime: readProcessStartTime(process.pid) ?? undefined, - }); - return { pid: process.pid, exited: new Promise(() => {}) }; + return spawnRegisteredDaemonFixture( + paths, + { + httpPort, + token: 'local-secret', + version: readVersion(), + codeOrigin: 'checkout', + codeSignature: currentDaemonCodeSignature(), + }, + options, + ); }); } @@ -357,7 +364,7 @@ test('sendToDaemon retains unknown metadata after a spawn failure', async () => assert.equal(results[0]?.removedInfo, false); assert.equal(attempts, 1); } finally { - fs.rmSync(stateDir, { recursive: true, force: true }); + await finishRegisteredDaemonFixture(stateDir); } }); @@ -402,11 +409,11 @@ test('sendToDaemon reports early daemon exit with log tail and startup paths', a assert.match(String(thrown.details?.daemonLogTail), /early daemon failure 2/); assert.equal(attempts, 2); } finally { - fs.rmSync(stateDir, { recursive: true, force: true }); + await finishRegisteredDaemonFixture(stateDir); } }); -test('sendToDaemon removes stale daemon lock before spawning a fresh daemon', async (t) => { +test('daemon acquisition reclaims a proven reused owner before publication', async (t) => { if (!(await supportsLoopbackBind())) { t.skip('loopback listeners are not permitted in this environment'); return; @@ -416,10 +423,11 @@ test('sendToDaemon removes stale daemon lock before spawning a fresh daemon', as const paths = resolveDaemonPaths(stateDir); const daemon = await startHttpDaemonFixture({ via: 'fresh-daemon' }); vi.stubEnv('AGENT_DEVICE_STATE_DIR', stateDir); - writeDaemonLock(paths, { - pid: process.pid, - processStartTime: 'stale-start-time', + const stale = tryAcquireProcessLock({ + lockDirPath: paths.lockPath, + owner: { pid: process.pid, startTime: 'stale-start-time', acquiredAtMs: Date.now() }, }); + assert.equal(stale.status, 'acquired'); installSpawnedHttpDaemon(paths, daemon.port); try { @@ -431,18 +439,15 @@ test('sendToDaemon removes stale daemon lock before spawning a fresh daemon', as meta: { requestId: 'req-stale-lock' }, }); - const freshLock = JSON.parse(fs.readFileSync(paths.lockPath, 'utf8')) as { - pid?: number; - processStartTime?: string; - }; + const freshLock = inspectProcessLock(paths.lockPath); assert.deepEqual(response, { ok: true, data: { via: 'fresh-daemon' } }); assert.equal(mockRunCmdDetached.mock.calls.length, 1); - assert.equal(freshLock.pid, process.pid); - assert.notEqual(freshLock.processStartTime, 'stale-start-time'); + assert.equal(freshLock.state, 'held'); + if (freshLock.state === 'held') assert.notEqual(freshLock.owner.startTime, 'stale-start-time'); assert.deepEqual(daemon.seenPaths, ['GET /health', 'POST /rpc']); } finally { await closeLoopbackServer(daemon.server); - fs.rmSync(stateDir, { recursive: true, force: true }); + await finishRegisteredDaemonFixture(stateDir); } }); @@ -510,7 +515,7 @@ test('sendToDaemon does not reuse reachable daemon metadata with mismatched vers stderrCapture.restore(); await closeLoopbackServer(staleDaemon.server); await closeLoopbackServer(freshDaemon.server); - fs.rmSync(stateDir, { recursive: true, force: true }); + await finishRegisteredDaemonFixture(stateDir); vi.unstubAllEnvs(); } } @@ -553,7 +558,7 @@ test('sendToDaemon prints a takeover notice before replacing an unreachable daem } finally { stderrCapture.restore(); await closeLoopbackServer(freshDaemon.server); - fs.rmSync(stateDir, { recursive: true, force: true }); + await finishRegisteredDaemonFixture(stateDir); } }); @@ -594,7 +599,7 @@ test('sendToDaemon replaces socket-only daemon metadata when HTTP transport is r } finally { stderrCapture.restore(); await closeLoopbackServer(freshDaemon.server); - fs.rmSync(stateDir, { recursive: true, force: true }); + await finishRegisteredDaemonFixture(stateDir); } }); @@ -697,7 +702,7 @@ test('sendToDaemon falls back from failed socket transport to HTTP using daemon } finally { socketFailures.restore(); await closeLoopbackServer(daemon.server); - fs.rmSync(stateDir, { recursive: true, force: true }); + await finishRegisteredDaemonFixture(stateDir); } }); @@ -741,7 +746,7 @@ test('sendToDaemon does not replay over HTTP after the socket request is written } finally { socket.restore(); await closeLoopbackServer(daemon.server); - fs.rmSync(stateDir, { recursive: true, force: true }); + await finishRegisteredDaemonFixture(stateDir); } }); @@ -1275,7 +1280,7 @@ test('issue #1384: sendToDaemon does not stop a client-started daemon at an expl assert.equal(fs.existsSync(paths.lockPath), true); } finally { await closeLoopbackServer(daemon.server); - fs.rmSync(stateDir, { recursive: true, force: true }); + await finishRegisteredDaemonFixture(stateDir); } }); diff --git a/src/daemon-client/__tests__/daemon-client-startup-race.test.ts b/src/daemon-client/__tests__/daemon-client-startup-race.test.ts index fa87f3f65b..303a8b1faa 100644 --- a/src/daemon-client/__tests__/daemon-client-startup-race.test.ts +++ b/src/daemon-client/__tests__/daemon-client-startup-race.test.ts @@ -1,7 +1,24 @@ import assert from 'node:assert/strict'; import fs from 'node:fs'; -import { afterEach, beforeAll, test, vi } from 'vitest'; +import path from 'node:path'; +import { afterEach, test, vi } from 'vitest'; +import { AppError } from '@agent-device/kernel/errors'; +import { tryAcquireProcessLock, inspectProcessLock } from '@agent-device/host-kit/file'; +import { readCurrentOwnerIdentity, isProcessAlive } from '@agent-device/host-kit/process'; +import { readVersion } from '@agent-device/host-kit/version'; import { mkdtempForTestSync } from '../../__tests__/test-utils/tmp-dir.ts'; +import { + spawnRegisteredDaemonFixture, + finishRegisteredDaemonFixtures, +} from '../../__tests__/test-utils/registered-daemon-fixture.ts'; +import { + startHttpDaemonFixture, + currentDaemonCodeSignature, +} from '../../__tests__/test-utils/daemon-http-fixture.ts'; +import { closeLoopbackServer, supportsLoopbackBind } from '../../__tests__/test-utils/loopback.ts'; +import { resolveDaemonPaths, type DaemonPaths } from '../../daemon-resolution.ts'; +import { sendToDaemon } from '../daemon-client.ts'; +import { DAEMON_STARTUP_EXIT_CODES } from '../../daemon-registration-owner.ts'; vi.mock('@agent-device/host-kit/command', async (importOriginal) => ({ ...(await importOriginal()), @@ -9,205 +26,313 @@ vi.mock('@agent-device/host-kit/command', async (importOriginal) => ({ })); vi.mock('@agent-device/host-kit/retry', async (importOriginal) => ({ ...(await importOriginal()), - sleep: vi.fn(async () => {}), + sleep: vi.fn(), })); -const winner = vi.hoisted(() => ({ pid: 43_300, alive: true })); -vi.mock('../../daemon-process.ts', async (importOriginal) => { - const actual = await importOriginal(); - return { - ...actual, - isAgentDeviceDaemonProcess: vi.fn((pid: number, startTime: string | undefined) => - pid === winner.pid ? winner.alive : actual.isAgentDeviceDaemonProcess(pid, startTime), - ), - stopDaemonProcess: vi.fn( - async ( - identity: Parameters[0], - options: Parameters[1], - ) => { - if (identity.pid !== winner.pid) return await actual.stopDaemonProcess(identity, options); - winner.alive = false; - return { - status: 'exited' as const, - identity: { pid: identity.pid, startTime: identity.startTime! }, - mode: 'graceful' as const, - }; - }, - ), - }; -}); - -import { resolveDaemonPaths, type DaemonPaths } from '../../daemon-resolution.ts'; -import { sendToDaemon } from '../daemon-client.ts'; import { runCmdDetachedMonitored, type ExecDetachedExit } from '@agent-device/host-kit/command'; import { sleep } from '@agent-device/host-kit/retry'; -import { readVersion } from '@agent-device/host-kit/version'; -import { resolveLocalDaemonCodeIdentity } from '../daemon-launch-spec.ts'; -import { - startHttpDaemonFixture, - type HttpDaemonFixture, -} from '../../__tests__/test-utils/daemon-http-fixture.ts'; -import { closeLoopbackServer, supportsLoopbackBind } from '../../__tests__/test-utils/loopback.ts'; - -// Two clients that find no daemon both launch one; the daemon that loses the startup lock exits -// cleanly. These pin that the losing client adopts the winner instead of tearing it down. - -const WINNER_PID = winner.pid; -const LOSER_PID = 43_301; - -const mockRunCmdDetached = vi.mocked(runCmdDetachedMonitored); -const mockSleep = vi.mocked(sleep); - -afterEach(() => { - winner.alive = true; - mockRunCmdDetached.mockReset(); - mockSleep.mockReset(); - mockSleep.mockImplementation(async () => {}); - vi.unstubAllEnvs(); +const actualRetry = await vi.importActual( + '@agent-device/host-kit/retry', +); +const spawn = vi.mocked(runCmdDetachedMonitored); +const pause = vi.mocked(sleep); +afterEach(async () => { + vi.restoreAllMocks(); + await finishRegisteredDaemonFixtures(); + spawn.mockReset(); + pause.mockReset(); }); -/** The code signature this client stamps on, and expects of, a daemon it may reuse. */ -let codeSignature: string | undefined; - -beforeAll(async () => { - const identity = await resolveLocalDaemonCodeIdentity(); - codeSignature = identity.origin === 'installed' ? undefined : identity.codeSignature; -}); +function request(paths: DaemonPaths, command = 'devices') { + return { + session: 'default', + command, + positionals: [], + flags: { stateDir: paths.baseDir, daemonTransport: 'http' as const }, + }; +} +function fields(httpPort: number, version = readVersion()) { + return { + httpPort, + token: 'secret', + version, + codeOrigin: 'checkout' as const, + codeSignature: currentDaemonCodeSignature(), + }; +} +async function awaitFile(file: string) { + const deadline = Date.now() + 2_000; + while (!fs.existsSync(file)) { + assert.ok(Date.now() < deadline, `fixture did not publish ${file}`); + await actualRetry.sleep(10); + } +} -/** Records the winning daemon the way it would: the startup lock, then its reachable metadata. */ -function writeWinner( - paths: DaemonPaths, - fixture: HttpDaemonFixture, - parts: 'lock' | 'all', - version = readVersion(), -): void { - fs.mkdirSync(paths.baseDir, { recursive: true }); - fs.writeFileSync( - paths.lockPath, - JSON.stringify({ pid: WINNER_PID, processStartTime: 'winner', startedAt: Date.now() }), - ); - if (parts === 'lock') return; - fs.writeFileSync( - paths.infoPath, - JSON.stringify({ - token: 'winner-secret', - pid: WINNER_PID, - version, - codeSignature, - processStartTime: 'winner', - httpPort: fixture.port, - transport: 'http', - }), - ); +for (const command of ['devices', 'test']) { + test(`a joined busy contender adopts a real winner for ${command}`, async (t) => { + if (!(await supportsLoopbackBind())) return t.skip('loopback unavailable'); + const paths = resolveDaemonPaths(mkdtempForTestSync('daemon-start-winner-')); + const http = await startHttpDaemonFixture({ devices: [] }); + const deferred = path.join(paths.baseDir, 'defer-publication'); + fs.writeFileSync(deferred, 'wait'); + const winner = spawnRegisteredDaemonFixture(paths, fields(http.port), { stdio: 'ignore' }); + await awaitFile(path.join(paths.baseDir, 'registration-held')); + let joined = false; + let genuineExit: ExecDetachedExit | undefined; + let contender: ReturnType | undefined; + let releaseJoin: (exit: ExecDetachedExit) => void = () => {}; + let pauses = 0; + spawn.mockImplementation((_command, _args, options) => { + contender = spawnRegisteredDaemonFixture(paths, fields(http.port), options); + void contender.exited.then((exit) => { + assert.equal(exit.exitCode, DAEMON_STARTUP_EXIT_CODES.busy); + genuineExit = exit; + }); + return { + ...contender, + exited: new Promise((resolve) => { + releaseJoin = resolve; + }), + }; + }); + pause.mockImplementation(async (ms) => { + pauses += 1; + fs.rmSync(deferred, { force: true }); + await awaitFile(paths.infoPath); + await actualRetry.sleep(ms); + if (pauses >= 2 && genuineExit) { + joined = true; + releaseJoin(genuineExit); + } + }); + try { + const response = await sendToDaemon(request(paths, command)); + assert.equal(response.ok, true); + assert.equal(joined, true); + assert.equal(spawn.mock.calls.length, 1); + assert.equal(http.rpcRequests.length, 1); + assert.equal(isProcessAlive(winner.pid), true); + const claim = inspectProcessLock(paths.lockPath); + assert.equal(claim.state, 'held'); + if (claim.state === 'held') assert.equal(claim.owner.pid, winner.pid); + } finally { + if (contender) releaseJoin(await contender.exited); + await closeLoopbackServer(http.server); + } + }); } -test('a client whose daemon lost the startup lock uses the daemon that won it', async (t) => { - if (!(await supportsLoopbackBind())) { - t.skip('loopback listeners are not permitted in this environment'); - return; - } - const stateDir = mkdtempForTestSync('agent-device-daemon-start-race-'); - const paths = resolveDaemonPaths(stateDir); - vi.stubEnv('AGENT_DEVICE_STATE_DIR', stateDir); - const fixture = await startHttpDaemonFixture({ devices: [] }); - let launches = 0; - mockRunCmdDetached.mockImplementation(() => { - launches += 1; - writeWinner(paths, fixture, 'lock'); - const exit: ExecDetachedExit = { pid: LOSER_PID, exitCode: 0 }; - return { pid: LOSER_PID, exited: Promise.resolve(exit) }; +test('a client-held claim is waited out before a fresh daemon attempt', async (t) => { + if (!(await supportsLoopbackBind())) return t.skip('loopback unavailable'); + const paths = resolveDaemonPaths(mkdtempForTestSync('daemon-start-client-holder-')); + const http = await startHttpDaemonFixture({ devices: [] }); + const claim = tryAcquireProcessLock({ + lockDirPath: paths.lockPath, + owner: { ...readCurrentOwnerIdentity(), acquiredAtMs: Date.now() }, }); - mockSleep.mockImplementation(async () => { - if (!fs.existsSync(paths.infoPath)) writeWinner(paths, fixture, 'all'); + assert.equal(claim.status, 'acquired'); + if (claim.status !== 'acquired') throw new Error('fixture claim refused'); + let loserJoined = false; + let released = false; + spawn.mockImplementation((_command, _args, options) => { + const child = spawnRegisteredDaemonFixture(paths, fields(http.port), options); + if (spawn.mock.calls.length === 1) + void child.exited.then((exit) => { + assert.equal(exit.exitCode, DAEMON_STARTUP_EXIT_CODES.busy); + loserJoined = true; + }); + else assert.equal(loserJoined, true); + return child; + }); + pause.mockImplementation(async (ms) => { + if (loserJoined && !released) { + await claim.acquisition.release(); + released = true; + } + await actualRetry.sleep(ms); }); - try { - const response = await sendToDaemon({ - session: 'default', - command: 'devices', - positionals: [], - flags: { stateDir }, - meta: { requestId: 'req-start-race' }, - }); - - assert.equal(response.ok, true); - assert.equal(launches, 1); - assert.equal(fixture.rpcRequests.length, 1); - assert.equal(fs.existsSync(paths.infoPath), true); - assert.equal(fs.existsSync(paths.lockPath), true); + assert.equal((await sendToDaemon(request(paths))).ok, true); + assert.equal(spawn.mock.calls.length, 2); + assert.equal(http.rpcRequests.length, 1); } finally { - await closeLoopbackServer(fixture.server); - fs.rmSync(stateDir, { recursive: true, force: true }); + if (!released) await claim.acquisition.release(); + await closeLoopbackServer(http.server); } }); -test('a one-shot test run leaves a daemon another client started running', async (t) => { - if (!(await supportsLoopbackBind())) { - t.skip('loopback listeners are not permitted in this environment'); - return; - } - const stateDir = mkdtempForTestSync('agent-device-daemon-start-race-owner-'); - const paths = resolveDaemonPaths(stateDir); - vi.stubEnv('AGENT_DEVICE_STATE_DIR', stateDir); - const fixture = await startHttpDaemonFixture({ passed: 1, failed: 0 }); - mockRunCmdDetached.mockImplementation(() => { - writeWinner(paths, fixture, 'all'); - return { pid: LOSER_PID, exited: new Promise(() => {}) }; +for (const exit of [ + { exitCode: 0 }, + { exitCode: 1 }, + { exitCode: DAEMON_STARTUP_EXIT_CODES.unproven }, + { exitCode: DAEMON_STARTUP_EXIT_CODES.busy, error: 'spawn refused' }, + { exitCode: DAEMON_STARTUP_EXIT_CODES.busy, signal: 'SIGTERM' as const }, +]) { + test(`generic exit ${exit.error ?? exit.signal ?? exit.exitCode} cannot adopt or stop a foreign winner`, async (t) => { + if (!(await supportsLoopbackBind())) return t.skip('loopback unavailable'); + const paths = resolveDaemonPaths(mkdtempForTestSync('daemon-start-generic-exit-')); + const http = await startHttpDaemonFixture({ devices: [] }); + const deferred = path.join(paths.baseDir, 'defer-publication'); + fs.writeFileSync(deferred, 'wait'); + const winner = spawnRegisteredDaemonFixture(paths, fields(http.port), { stdio: 'ignore' }); + await awaitFile(path.join(paths.baseDir, 'registration-held')); + spawn.mockImplementation(() => ({ + pid: 999_999, + exited: Promise.resolve({ pid: 999_999, ...exit }), + })); + pause.mockImplementation(async (ms) => { + fs.rmSync(deferred, { force: true }); + await actualRetry.sleep(ms); + }); + try { + await assert.rejects( + sendToDaemon(request(paths)), + (error: unknown) => + error instanceof AppError && error.details?.kind === 'daemon_startup_failed', + ); + assert.equal(spawn.mock.calls.length, 1); + assert.equal(http.rpcRequests.length, 0); + assert.equal(isProcessAlive(winner.pid), true); + assert.equal(inspectProcessLock(paths.lockPath).state, 'held'); + } finally { + await closeLoopbackServer(http.server); + } }); +} - try { - const response = await sendToDaemon({ - session: 'default', - command: 'test', - positionals: [], - flags: { stateDir }, - meta: { requestId: 'req-start-race-test' }, +for (const held of [true, false]) { + test(`startup uses one deadline when the claim is ${held ? 'held' : 'released for relaunch'}`, async () => { + const paths = resolveDaemonPaths(mkdtempForTestSync('daemon-start-budget-')); + const claim = tryAcquireProcessLock({ + lockDirPath: paths.lockPath, + owner: { ...readCurrentOwnerIdentity(), acquiredAtMs: Date.now() }, + }); + assert.equal(claim.status, 'acquired'); + if (claim.status !== 'acquired') throw new Error('fixture claim refused'); + let now = Date.now(); + const started = now; + let released = false; + let finishPending: () => void = () => {}; + const nativeTimeout = globalThis.setTimeout; + vi.spyOn(globalThis, 'setTimeout').mockImplementation((handler, ms, ...args) => + nativeTimeout(handler, ms === 1_000 ? 0 : ms, ...args), + ); + vi.spyOn(Date, 'now').mockImplementation(() => now); + spawn.mockImplementation(() => ({ + pid: 999_999, + exited: + spawn.mock.calls.length === 1 + ? Promise.resolve({ pid: 999_999, exitCode: DAEMON_STARTUP_EXIT_CODES.busy }) + : new Promise((resolve) => { + finishPending = () => resolve({ pid: 999_999, exitCode: 1 }); + }), + })); + pause.mockImplementation(async (ms) => { + if (!held && !released) { + await claim.acquisition.release(); + released = true; + now += 14_750; + } else now += ms; }); + try { + await assert.rejects(sendToDaemon(request(paths)), (error: unknown) => { + assert.ok(error instanceof AppError); + assert.equal(error.details?.startupAttempts, held ? 1 : 2); + assert.equal(error.details?.startupTimeoutMs, 15_000); + return true; + }); + assert.equal(now - started, 15_000); + assert.equal(inspectProcessLock(paths.lockPath).state, held ? 'held' : 'absent'); + } finally { + finishPending(); + vi.restoreAllMocks(); + if (!released) await claim.acquisition.release(); + } + }); +} - assert.equal(response.ok, true); - assert.equal(fs.existsSync(paths.infoPath), true); +test('a joined busy contender retires an older winner before relaunching', async (t) => { + if (!(await supportsLoopbackBind())) return t.skip('loopback unavailable'); + const paths = resolveDaemonPaths(mkdtempForTestSync('daemon-start-older-winner-')); + const http = await startHttpDaemonFixture({ devices: [] }); + const deferred = path.join(paths.baseDir, 'defer-publication'); + fs.writeFileSync(deferred, 'wait'); + const winner = spawnRegisteredDaemonFixture(paths, fields(http.port, '0.0.1'), { + stdio: 'ignore', + }); + await awaitFile(path.join(paths.baseDir, 'registration-held')); + let joined = false; + spawn.mockImplementation((_command, _args, options) => { + if (spawn.mock.calls.length > 1) assert.equal(joined, true); + const child = spawnRegisteredDaemonFixture(paths, fields(http.port), options); + if (spawn.mock.calls.length === 1) + void child.exited.then((exit) => { + assert.equal(exit.exitCode, DAEMON_STARTUP_EXIT_CODES.busy); + joined = true; + }); + return child; + }); + pause.mockImplementation(async (ms) => { + if (joined) fs.rmSync(deferred, { force: true }); + await actualRetry.sleep(ms); + }); + const notice = vi.spyOn(process.stderr, 'write').mockImplementation(() => true); + try { + assert.equal((await sendToDaemon(request(paths))).ok, true); + await winner.exited; + assert.equal(isProcessAlive(winner.pid), false); + assert.equal(spawn.mock.calls.length, 2); + assert.equal(http.rpcRequests.length, 1); + assert.ok( + notice.mock.calls.flat().join('').includes(`Replacing daemon (pid ${winner.pid}, v0.0.1)`), + ); } finally { - await closeLoopbackServer(fixture.server); - fs.rmSync(stateDir, { recursive: true, force: true }); + notice.mockRestore(); + await closeLoopbackServer(http.server); } }); -test('a start race won by an older daemon replaces it instead of adopting it', async (t) => { - if (!(await supportsLoopbackBind())) { - t.skip('loopback listeners are not permitted in this environment'); - return; - } - const stateDir = mkdtempForTestSync('agent-device-daemon-start-race-older-'); - const paths = resolveDaemonPaths(stateDir); - vi.stubEnv('AGENT_DEVICE_STATE_DIR', stateDir); - const fixture = await startHttpDaemonFixture({ devices: [] }); - let launches = 0; - mockRunCmdDetached.mockImplementation(() => { - launches += 1; - if (launches === 1) writeWinner(paths, fixture, 'all', '0.0.1'); - const exit: ExecDetachedExit = { pid: LOSER_PID, exitCode: 0 }; - return { pid: LOSER_PID, exited: Promise.resolve(exit) }; +test('a failed own transport probe retires and joins the private startup before rejecting', async (t) => { + if (!(await supportsLoopbackBind())) return t.skip('loopback unavailable'); + const http = await startHttpDaemonFixture({ devices: [] }); + let paths: DaemonPaths | undefined; + let child: ReturnType | undefined; + let failure: AppError | undefined; + spawn.mockImplementation((_command, _args, options) => { + paths = resolveDaemonPaths(String(options?.env?.AGENT_DEVICE_STATE_DIR)); + child = spawnRegisteredDaemonFixture(paths, fields(http.port), options); + return child; }); - const stderr = vi.spyOn(process.stderr, 'write').mockImplementation(() => true); - + pause.mockImplementation(actualRetry.sleep); try { await assert.rejects( sendToDaemon({ session: 'default', - command: 'devices', + command: 'test', positionals: [], - flags: { stateDir }, - meta: { requestId: 'req-start-race-older' }, + flags: { daemonTransport: 'socket', daemonServerMode: 'http' }, }), + (error: unknown) => { + assert.ok(error instanceof AppError); + assert.equal(error.message, 'Daemon socket endpoint is unavailable'); + assert.equal(error.details?.reason, 'daemon_endpoint_unavailable'); + failure = error; + return true; + }, ); - assert.equal(fixture.rpcRequests.length, 0); - assert.equal(launches, 1); - assert.match( - String(stderr.mock.calls.flat().join('')), - /Replacing daemon \(pid 43300, v0\.0\.1\)/, - ); + assert.ok(paths && child && failure); + assert.equal(isProcessAlive(child.pid), false); + assert.equal(fs.existsSync(paths.baseDir), false); + await child.exited; + assert.equal(failure.details?.startupJoined, true); + assert.equal(failure.details?.stateDir, paths.baseDir); + const results = failure.details?.cleanupResults as Array<{ + status: string; + removedStateDir?: boolean; + }>; + assert.equal(results[0]?.status, 'retired'); + assert.equal(results[0]?.removedStateDir, true); + assert.equal(http.rpcRequests.length, 0); } finally { - stderr.mockRestore(); - await closeLoopbackServer(fixture.server); - fs.rmSync(stateDir, { recursive: true, force: true }); + await closeLoopbackServer(http.server); } }); diff --git a/src/daemon-client/__tests__/daemon-client.test.ts b/src/daemon-client/__tests__/daemon-client.test.ts index accebf0782..cca31e8a26 100644 --- a/src/daemon-client/__tests__/daemon-client.test.ts +++ b/src/daemon-client/__tests__/daemon-client.test.ts @@ -140,44 +140,20 @@ function writeCurrentDaemonInfo( ); } -test('resolveDaemonStartupHint prefers stale lock guidance when lock exists without info', () => { - const hint = resolveDaemonStartupHint({ hasInfo: false, hasLock: true }); - assert.match(hint, /daemon\.lock/i); - assert.match(hint, /automatically/i); - assert.match(hint, /rm -f '.+daemon\.json' '.+daemon\.lock'/); -}); - -test('resolveDaemonStartupHint covers stale info+lock pair', () => { - const hint = resolveDaemonStartupHint({ hasInfo: true, hasLock: true }); - assert.match(hint, /daemon\.json/i); - assert.match(hint, /daemon\.lock/i); - assert.match(hint, /rm -f '.+daemon\.json' '.+daemon\.lock'/); -}); - -test('resolveDaemonStartupHint falls back to daemon.json guidance', () => { - const hint = resolveDaemonStartupHint({ hasInfo: true, hasLock: false }); - assert.match(hint, /daemon\.json/i); - assert.match(hint, /rm -f '.+daemon\.json' '.+daemon\.lock'/); -}); - -test('resolveDaemonStartupHint includes configured state directory paths', () => { - const paths = resolveDaemonPaths('/tmp/ad-custom-state'); - const hint = resolveDaemonStartupHint({ hasInfo: false, hasLock: true }, paths); - assert.match(hint, /\/tmp\/ad-custom-state\/daemon\.lock/); - assert.match(hint, /\/tmp\/ad-custom-state\/daemon\.json/); - assert.match( - hint, - /rm -f '\/tmp\/ad-custom-state\/daemon\.json' '\/tmp\/ad-custom-state\/daemon\.lock'/, - ); -}); - -test('resolveDaemonStartupHint shell-quotes cleanup paths', () => { +test('startup recovery guidance retains configured paths and requires stopping every user', () => { const paths = resolveDaemonPaths("/tmp/ad custom's state"); - const hint = resolveDaemonStartupHint({ hasInfo: true, hasLock: true }, paths); - assert.match( - hint, - /rm -f '\/tmp\/ad custom'\\''s state\/daemon\.json' '\/tmp\/ad custom'\\''s state\/daemon\.lock'/, - ); + for (const state of [ + { hasInfo: false, hasLock: true }, + { hasInfo: true, hasLock: true }, + { hasInfo: true, hasLock: false }, + { hasInfo: false, hasLock: false }, + ]) { + const hint = resolveDaemonStartupHint(state, paths); + if (state.hasInfo) assert.ok(hint.includes(paths.infoPath)); + if (state.hasLock) assert.ok(hint.includes(paths.lockPath)); + assert.match(hint, /stop all older clients and daemons/); + assert.doesNotMatch(hint, /rm -f/); + } }); test('canConnectSocket times out stalled local daemon probes', async () => { diff --git a/src/daemon-client/daemon-client-lifecycle.ts b/src/daemon-client/daemon-client-lifecycle.ts index 7e75749e61..77d46502a5 100644 --- a/src/daemon-client/daemon-client-lifecycle.ts +++ b/src/daemon-client/daemon-client-lifecycle.ts @@ -8,10 +8,11 @@ import { shellQuoteIfNeeded } from '@agent-device/kernel/device-shell'; import { emitDiagnostic } from '@agent-device/host-kit/diagnostics'; import { isProcessAlive } from '@agent-device/host-kit/process'; import { sleep } from '@agent-device/host-kit/retry'; -import { inspectProcessLock } from '@agent-device/host-kit/file'; +import { inspectProcessLock, type ProcessLockInspection } from '@agent-device/host-kit/file'; import type { findUnrecoveredRepairCommitFailure } from '../session-repair-tombstone.ts'; import { + DAEMON_STARTUP_EXIT_CODES, createOwnedReplayStateDir, recoverAbandonedDaemonRegistration, type DaemonRetirementResult, @@ -36,14 +37,10 @@ import { import { PUBLIC_COMMANDS } from '@agent-device/command-registry/catalog'; import { - cleanupStaleDaemonLockIfSafe, getDaemonMetadataState, - isDaemonLockHeldByAnotherDaemon, isRemoteDaemon, readDaemonInfo, - removeDaemonInfo, resolveDaemonStartupHint, - stopDaemonProcessForTakeover, type DaemonInfo, } from './daemon-client-metadata.ts'; import { @@ -69,7 +66,7 @@ export type EnsuredDaemon = { type DaemonStartupWaitResult = | { kind: 'ready'; daemon: EnsuredDaemon } | { kind: 'early_exit'; exit: ExecDetachedExit } - | { kind: 'timeout' }; + | { kind: 'retry' | 'unproven' | 'timeout' }; const DAEMON_STARTUP_TIMEOUT_MS = 15_000; const LIVE_DAEMON_PROBE_RETRIES = 3; @@ -166,7 +163,6 @@ async function ensureLocalDaemon(settings: DaemonClientSettings): Promise { +async function readReusableLocalDaemon( + settings: DaemonClientSettings, + deadline?: number, +): Promise { + const inspection = inspectProcessLock(settings.paths.lockPath); + if (inspection.state === 'unproven') { + throw new AppError('COMMAND_FAILED', 'Daemon registration ownership could not be verified.', { + reason: 'daemon_registration_unproven', + inspection, + stateDir: settings.paths.baseDir, + hint: resolveDaemonStartupHint(getDaemonMetadataState(settings.paths), settings.paths), + }); + } const existing = readDaemonInfo(settings.paths.infoPath); if (!existing) return null; const decision = await resolveDaemonTakeover(existing, { - onClientTransport: () => canReachReusableDaemon(existing, settings.transportPreference), - onAnyAdvertisedTransport: () => canReachReusableDaemon(existing, 'auto'), + onClientTransport: () => + canReachReusableDaemon(existing, settings.transportPreference, deadline), + onAnyAdvertisedTransport: () => canReachReusableDaemon(existing, 'auto', deadline), }); if (decision.kind === 'reuse') return existing; if (decision.kind === 'refuseNewer') { throw newerDaemonRefusedError(existing, decision, settings.paths.baseDir); } + if (deadline !== undefined && Date.now() >= deadline) return null; emitDaemonTakeoverNotice(existing, decision.reason, settings.paths.baseDir); - await stopDaemonProcessForTakeover(existing); - removeDaemonInfo(settings.paths.infoPath); + await retireDaemonForTakeover(existing, settings.paths); return null; } +async function retireDaemonForTakeover(existing: DaemonInfo, paths: DaemonPaths): Promise { + const retirement = await stopAndRetireDaemon({ + paths: paths, + observed: { pid: existing.pid, startTime: existing.processStartTime ?? null }, + mode: 'graceful', + }); + if (retirement.status === 'retained') { + throw new AppError('COMMAND_FAILED', 'Daemon replacement could not be confirmed.', { + reason: 'daemon_retirement_unconfirmed', + retirement, + hint: + retirement.error?.hint ?? resolveDaemonStartupHint(getDaemonMetadataState(paths), paths), + }); + } +} + /** * A daemon whose pid is still alive is probed again before it can be judged unreachable. A probe's * budget is wall-clock time on this client's event loop, so a client that stalls past it (a large @@ -220,12 +245,14 @@ async function readReusableLocalDaemon(settings: DaemonClientSettings): Promise< async function canReachReusableDaemon( info: DaemonInfo, preference: DaemonTransportPreference, + deadline?: number, ): Promise { - if (await canConnectReusableDaemon(info, preference)) return true; + if (await canConnectReusableDaemon(info, preference, deadline)) return true; for (let retry = 1; retry <= LIVE_DAEMON_PROBE_RETRIES; retry += 1) { - if (!isProcessAlive(info.pid)) return false; - await sleep(LIVE_DAEMON_PROBE_RETRY_DELAY_MS); - if (await canConnectReusableDaemon(info, preference)) { + if (!isProcessAlive(info.pid) || (deadline !== undefined && Date.now() >= deadline)) + return false; + await sleep(Math.min(LIVE_DAEMON_PROBE_RETRY_DELAY_MS, remainingStartupBudget(deadline))); + if (await canConnectReusableDaemon(info, preference, deadline)) { emitDiagnostic({ level: 'warn', phase: 'daemon_probe_recovered', @@ -261,9 +288,10 @@ async function assertDaemonPolicyMatches(existing: DaemonInfo, stateDir: string) async function canConnectReusableDaemon( info: DaemonInfo, preference: DaemonTransportPreference, + deadline?: number, ): Promise { try { - return await canConnect(info, preference); + return await canConnect(info, preference, remainingStartupBudget(deadline)); } catch (error) { if (isDaemonTransportUnavailableError(error)) return false; throw error; @@ -298,7 +326,7 @@ function emitDaemonTakeoverNotice(info: DaemonInfo, reason: string, stateDir: st } type FailedDaemonStartup = { - cleanup: DaemonRetirementResult; + cleanup?: DaemonRetirementResult; startError?: string; daemonProcess?: ExecDetachedExit | { pid: number }; retry: boolean; @@ -314,7 +342,7 @@ async function startLocalDaemon(settings: DaemonClientSettings): Promise { const cleanup = await stopAndRetireDaemon({ paths: settings.paths, observed: { pid: launch.pid, startTime: launch.startTime ?? null }, mode: 'graceful', + ownedStateDir, + termTimeoutMs: Math.min(3_000, remainingStartupBudget(deadline)), + killTimeoutMs: 1_000, lockTimeoutMs: 0, }); - const inspection = inspectProcessLock(settings.paths.lockPath); - const available = + const joined = await joinStartup(launch); + return { cleanup, joined }; +} + +async function joinStartup(launch: DaemonStartupLaunch): Promise { + let timer: ReturnType | undefined; + try { + return await Promise.race([ + launch.exited.then(() => true), + new Promise((resolve) => { + timer = setTimeout(() => resolve(false), 1_000); + }), + ]); + } finally { + clearTimeout(timer); + } +} + +function remainingStartupBudget(deadline?: number): number { + return deadline === undefined ? Number.POSITIVE_INFINITY : Math.max(0, deadline - Date.now()); +} + +function isRegistrationAvailable(inspection: ProcessLockInspection): boolean { + return ( inspection.state === 'absent' || (inspection.state === 'held' && (inspection.liveness === 'owner-process-dead' || - inspection.liveness === 'owner-process-reused')); - return { - cleanup, - retry: startup.kind === 'early_exit' && available, - startError: startup.kind === 'early_exit' ? describeDaemonEarlyExit(startup.exit) : undefined, - daemonProcess: startup.kind === 'early_exit' ? startup.exit : { pid: launch.pid }, - }; + inspection.liveness === 'owner-process-reused')) + ); } /** @@ -383,7 +455,7 @@ async function attemptLocalDaemonStartup( * (`replay --save-script`) that COMPLETES without diverging returns SUCCESS * here — the actual healed-script COMMIT is deferred to daemon teardown * (`finalizeRepairTeardown`, run inside the daemon process's own shutdown - * handler, triggered by `stopDaemonProcessForTakeover` below). If that + * handler, triggered by `stopAndRetireDaemon`). If that * deferred commit then FAILS, the daemon leaves a `REPAIR_COMMIT_FAILED` * tombstone in this owned state dir — the only surviving record of the * failure, since the daemon process (and its in-memory session) is gone by @@ -629,36 +701,93 @@ export function attachActiveSessionAddressHint( } async function waitForDaemonStartup( - timeoutMs: number, + deadline: number, settings: DaemonClientSettings, launch: DaemonStartupLaunch, ): Promise { - const start = Date.now(); let earlyExit: ExecDetachedExit | undefined; void launch.exited.then((exit) => { earlyExit = exit; }); - - while (Date.now() - start < timeoutMs) { - const info = readDaemonInfo(settings.paths.infoPath); - if (info && (await canConnect(info, settings.transportPreference))) { - if (isLaunchedDaemon(info, launch)) { - return { kind: 'ready', daemon: { info, startedByClient: true } }; - } - // Another client's daemon won the start: adopt it only as a reusable daemon would be. An - // incompatible one is replaced, and this wait then sees its own daemon's early exit. - const winner = await readReusableLocalDaemon(settings); - if (winner) return { kind: 'ready', daemon: { info: winner, startedByClient: false } }; + while (Date.now() < deadline) { + if (earlyExit) { + const kind = classifyDaemonStartupExit(earlyExit); + if (kind === 'unproven') return { kind: 'unproven' }; + if (kind === 'failed') return { kind: 'early_exit', exit: earlyExit }; + const contender = await observeContendingDaemon(settings, deadline); + if (contender) return contender; + } else { + const info = await readReadyLaunchedDaemon(settings, launch, deadline); + if (info && !earlyExit) return { kind: 'ready', daemon: { info, startedByClient: true } }; } - // A daemon that lost the startup lock exits cleanly; the daemon that won it is still starting. - if (earlyExit && !isDaemonLockHeldByAnotherDaemon(settings.paths, earlyExit.pid)) { - return { kind: 'early_exit', exit: earlyExit }; - } - await sleep(100); + await sleep(Math.min(100, remainingStartupBudget(deadline))); } return { kind: 'timeout' }; } +function classifyDaemonStartupExit(exit: ExecDetachedExit): 'busy' | 'unproven' | 'failed' { + if (exit.error || exit.signal) return 'failed'; + switch (exit.exitCode) { + case DAEMON_STARTUP_EXIT_CODES.busy: + return 'busy'; + case DAEMON_STARTUP_EXIT_CODES.unproven: + return 'unproven'; + default: + return 'failed'; + } +} + +async function observeContendingDaemon( + settings: DaemonClientSettings, + deadline: number, +): Promise { + const winner = await readReusableLocalDaemon(settings, deadline); + if (Date.now() >= deadline) return null; + if (winner) return { kind: 'ready', daemon: { info: winner, startedByClient: false } }; + const inspection = inspectProcessLock(settings.paths.lockPath); + if (inspection.state === 'unproven') return { kind: 'unproven' }; + return isRegistrationAvailable(inspection) ? { kind: 'retry' } : null; +} + +async function readReadyLaunchedDaemon( + settings: DaemonClientSettings, + launch: DaemonStartupLaunch, + deadline: number, +): Promise { + const info = readDaemonInfo(settings.paths.infoPath); + if (!info || !isLaunchedDaemon(info, launch)) return null; + try { + return (await canConnect( + info, + settings.transportPreference, + remainingStartupBudget(deadline), + )) && Date.now() < deadline + ? info + : null; + } catch (error) { + const { cleanup, joined } = await retireStartupAttempt( + settings, + launch, + deadline, + settings.ownedStateDir, + ); + emitDiagnostic({ + level: 'warn', + phase: 'daemon_startup_observation_failed', + data: { stateDir: settings.paths.baseDir, cleanup, joined, error: normalizeError(error) }, + }); + if (error instanceof AppError) { + error.details = { + ...error.details, + stateDir: settings.paths.baseDir, + cleanupResults: [cleanup], + startupJoined: joined, + }; + } + throw error; + } +} + /** Whether `info` names the daemon process this client launched: same pid and start time. */ function isLaunchedDaemon(info: DaemonInfo, launch: DaemonStartupLaunch): boolean { return ( diff --git a/src/daemon-client/daemon-client-metadata.ts b/src/daemon-client/daemon-client-metadata.ts index bf886cef5c..1be88db68b 100644 --- a/src/daemon-client/daemon-client-metadata.ts +++ b/src/daemon-client/daemon-client-metadata.ts @@ -1,11 +1,7 @@ import fs from 'node:fs'; +import { inspectProcessLock, type ProcessLockInspection } from '@agent-device/host-kit/file'; import { AppError } from '@agent-device/kernel/errors'; -import { shellQuote } from '@agent-device/kernel/device-shell'; -import { - isAgentDeviceDaemonProcess, - stopDaemonProcess, - type DaemonTerminationResult, -} from '../daemon-process.ts'; +import { stopDaemonProcess, type DaemonTerminationResult } from '../daemon-process.ts'; import type { DaemonCodeOrigin } from '@agent-device/host-kit/code-signature'; @@ -32,15 +28,10 @@ export type DaemonInfo = { remoteUpstreamInstanceId?: string; }; -type DaemonLockInfo = { - pid: number; - processStartTime?: string; - startedAt?: number; -}; - export type DaemonMetadataState = { hasInfo: boolean; hasLock: boolean; + registration: ProcessLockInspection; }; const DAEMON_TAKEOVER_TERM_TIMEOUT_MS = 3000; @@ -96,35 +87,6 @@ function readPositiveInteger(value: unknown): number | undefined { return Number.isInteger(value) && Number(value) > 0 ? Number(value) : undefined; } -function readDaemonLockInfo(lockPath: string): DaemonLockInfo | null { - const data = readJsonFile(lockPath); - if (!data || typeof data !== 'object') return null; - const parsed = data as Partial; - const hasPid = Number.isInteger(parsed.pid) && Number(parsed.pid) > 0; - if (!hasPid) { - return null; - } - return { - pid: Number(parsed.pid), - processStartTime: - typeof parsed.processStartTime === 'string' ? parsed.processStartTime : undefined, - startedAt: typeof parsed.startedAt === 'number' ? parsed.startedAt : undefined, - }; -} - -/** - * Whether a live daemon other than `pid` holds the startup lock: another client's daemon won the - * start, and the daemon at `pid` exited because it lost the lock. - */ -export function isDaemonLockHeldByAnotherDaemon(paths: DaemonPaths, pid: number): boolean { - const lockInfo = readDaemonLockInfo(paths.lockPath); - return ( - lockInfo !== null && - lockInfo.pid !== pid && - isAgentDeviceDaemonProcess(lockInfo.pid, lockInfo.processStartTime) - ); -} - export function removeDaemonInfo(infoPath: string): void { removeFileIfExists(infoPath); } @@ -133,24 +95,11 @@ export function removeDaemonLock(lockPath: string): void { removeFileIfExists(lockPath); } -export function cleanupStaleDaemonLockIfSafe(paths: DaemonPaths): void { - const state = getDaemonMetadataState(paths); - if (!state.hasLock || state.hasInfo) return; - const lockInfo = readDaemonLockInfo(paths.lockPath); - if (!lockInfo) { - removeDaemonLock(paths.lockPath); - return; - } - if (isAgentDeviceDaemonProcess(lockInfo.pid, lockInfo.processStartTime)) { - return; - } - removeDaemonLock(paths.lockPath); -} - export function getDaemonMetadataState(paths: DaemonPaths): DaemonMetadataState { return { hasInfo: fs.existsSync(paths.infoPath), hasLock: fs.existsSync(paths.lockPath), + registration: inspectProcessLock(paths.lockPath), }; } @@ -187,21 +136,10 @@ export function resolveDaemonStartupHint( process.env.AGENT_DEVICE_STATE_DIR, ), ): string { - const cleanupCommand = buildDaemonMetadataCleanupCommand(paths); - if (state.hasLock && !state.hasInfo) { - return `agent-device attempted to clean stale daemon metadata automatically, but ${paths.lockPath} still exists without ${paths.infoPath}. Retry with --debug; if this persists after confirming no agent-device daemon process is running, run: ${cleanupCommand}`; - } - if (state.hasLock && state.hasInfo) { - return `agent-device attempted to clean stale daemon metadata automatically, but ${paths.infoPath} and ${paths.lockPath} still remain. Retry with --debug; if this persists after confirming no agent-device daemon process is running, run: ${cleanupCommand}`; - } - if (state.hasInfo) { - return `agent-device did not observe reachable daemon metadata after retrying, and ${paths.infoPath} still remains. Stale metadata was cleaned automatically when safe; retry with --debug. If this persists after confirming no agent-device daemon process is running, run: ${cleanupCommand}`; - } - return `agent-device did not observe reachable daemon metadata after retrying. Stale metadata was cleaned automatically when safe; retry with --debug and check daemon diagnostics logs. If stale metadata returns after confirming no agent-device daemon process is running, run: ${cleanupCommand}`; -} - -function buildDaemonMetadataCleanupCommand(paths: Pick) { - return `rm -f ${shellQuote(paths.infoPath)} ${shellQuote(paths.lockPath)}`; + const artifacts = [state.hasInfo ? paths.infoPath : null, state.hasLock ? paths.lockPath : null] + .filter(Boolean) + .join(' and '); + return `Daemon startup did not establish a reachable owner. ${artifacts ? `State was retained at ${artifacts}. ` : ''}Retry with --debug and inspect daemon diagnostics. Before upgrading, stop all older clients and daemons with their original CLI and prevent them from returning to this state directory. Unverified lock state requires confirming every user stopped before manual recovery; deleting metadata alone is not a safe reset.`; } function readJsonFile(filePath: string): unknown | null { diff --git a/src/daemon-client/daemon-client-transport.ts b/src/daemon-client/daemon-client-transport.ts index d09d5af386..aa0f93820d 100644 --- a/src/daemon-client/daemon-client-transport.ts +++ b/src/daemon-client/daemon-client-transport.ts @@ -72,23 +72,31 @@ type RemoteDaemonHealthLink = Pick< export async function canConnect( info: DaemonInfo, preference: DaemonTransportPreference, + probeTimeoutMs?: number, ): Promise { + const deadline = Date.now() + (probeTimeoutMs ?? Number.POSITIVE_INFINITY); const transport = chooseTransport(info, preference); - if (await canConnectWithTransport(info, transport)) return true; + if (await canConnectWithTransport(info, transport, deadline - Date.now())) return true; const fallback = chooseAutoFallbackTransport(info, preference, transport); - return fallback ? await canConnectWithTransport(info, fallback) : false; + return fallback ? await canConnectWithTransport(info, fallback, deadline - Date.now()) : false; } async function canConnectWithTransport( info: DaemonInfo, transport: ResolvedDaemonTransport, + timeoutMs: number, ): Promise { - return transport === 'http' ? await canConnectHttp(info) : await canConnectSocket(info.port); + return transport === 'http' + ? await canConnectHttp(info, timeoutMs) + : await canConnectSocket(info.port, timeoutMs); } -export function canConnectSocket(port: number | undefined): Promise { - if (!port) return Promise.resolve(false); +export function canConnectSocket( + port: number | undefined, + timeoutMs = LOCAL_DAEMON_HEALTHCHECK_TIMEOUT_MS, +): Promise { + if (!port || timeoutMs <= 0) return Promise.resolve(false); return new Promise((resolve) => { let settled = false; const socket = net.createConnection({ host: '127.0.0.1', port }, () => { @@ -100,7 +108,7 @@ export function canConnectSocket(port: number | undefined): Promise { socket.destroy(); resolve(reachable); }; - socket.setTimeout(LOCAL_DAEMON_HEALTHCHECK_TIMEOUT_MS); + socket.setTimeout(Math.min(LOCAL_DAEMON_HEALTHCHECK_TIMEOUT_MS, Math.ceil(timeoutMs))); socket.on('timeout', () => { finish(false); }); @@ -110,8 +118,8 @@ export function canConnectSocket(port: number | undefined): Promise { }); } -function canConnectHttp(info: DaemonInfo): Promise { - return readDaemonHttpHealth(info).then((health) => health.reachable); +function canConnectHttp(info: DaemonInfo, timeoutMs: number): Promise { + return readDaemonHttpHealth(info, timeoutMs).then((health) => health.reachable); } export async function readRemoteDaemonHealth( diff --git a/website/docs/docs/installation.md b/website/docs/docs/installation.md index eca4ff398d..02886095b4 100644 --- a/website/docs/docs/installation.md +++ b/website/docs/docs/installation.md @@ -115,10 +115,15 @@ vega device list - A runner startup failure is typed, not prose: `error.details.reason` is one of `signing_no_development_team`, `signing_provisioning_profile_missing`, `bundle_identifier_already_registered`, `signing_unspecified`, `devtools_security_developer_mode_disabled` (the Mac's `DevToolsSecurity` setting, which says nothing about the device's Developer Mode toggle), `device_developer_mode_disabled`, `device_developer_disk_image_unavailable`, or `build_failed_unclassified` when nothing proved a cause. Branch on `details.reason` and follow `hint`; the code stays `COMMAND_FAILED` for every reason. - The two `device_*` reasons come from the iPhone itself, read over `xcrun devicectl device info details` before the runner builds: `developerModeStatus` for the Settings toggle and `ddiServicesAvailable` for the developer disk image. They are reported apart on purpose. A phone with Developer Mode off cannot serve its disk image either, so it gets the toggle reason; a phone with the toggle on and only the image down gets the disk-image reason, which is a device-support install that has not finished rather than a setting anyone turned off. - If device setup is slow, keep the device connected and inspect daemon diagnostics after retrying. -- If daemon startup reports stale metadata, remove stale files and retry: - - `/daemon.json` - - `/daemon.lock` - - default state dir is `~/.agent-device` for packaged installs; source checkouts default to a worktree-scoped dir under `~/.agent-device/dev/` unless `AGENT_DEVICE_STATE_DIR` or `--state-dir` is set - - `agent-device session state-dir` prints the resolved state dir without starting the daemon - - after pulling the worktree-scoped daemon change in a source checkout, stop any legacy default daemon once with `AGENT_DEVICE_STATE_DIR=~/.agent-device pnpm clean:daemon` - - worktree-scoped state dirs outlive deleted worktrees; `pnpm clean:daemon --prune-dev` removes dirs under `~/.agent-device/dev/` with no live daemon and no activity for 14 days (one line printed per removed dir) + +## Daemon startup and upgrades + +If daemon startup fails, retry with `--debug` and inspect the retained state and diagnostics. `agent-device session state-dir` prints the resolved directory without starting a daemon. + +Before upgrading across the daemon lock change, stop every older client and daemon using that directory with their original CLI. Prevent older versions from returning while the upgraded version runs. Use a single deployed version or separate environments for concurrent installations. + +Startup refuses legacy lock files and unverified ownership. Confirm every user of the state directory stopped before manual recovery; removing `daemon.json` or `daemon.lock` alone is not a safe reset. + +Packaged installs default to `~/.agent-device`; source checkouts use a worktree directory under `~/.agent-device/dev/`. `AGENT_DEVICE_STATE_DIR` or `--state-dir` overrides either default. + +For source checkouts, `pnpm clean:daemon --prune-dev` selects development directories with no activity for 14 days, using their newest mtime. It retires only confirmed abandoned registration and retains shared directories, session artifacts and logs. From 604939b7d66cbb3c3447efa73ad3a8160aced591 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sat, 3 Oct 2026 03:19:37 +0200 Subject: [PATCH 08/20] fix: observe only the current daemon registration owner --- .../__tests__/daemon-client-lifecycle.test.ts | 1 + .../daemon-client-startup-race.test.ts | 38 ++++++++--- src/daemon-client/daemon-client-lifecycle.ts | 66 ++++++++++++++----- src/daemon-client/daemon-client-metadata.ts | 3 - website/docs/docs/installation.md | 2 +- 5 files changed, 81 insertions(+), 29 deletions(-) diff --git a/src/daemon-client/__tests__/daemon-client-lifecycle.test.ts b/src/daemon-client/__tests__/daemon-client-lifecycle.test.ts index 281965b3e9..88d57bbc14 100644 --- a/src/daemon-client/__tests__/daemon-client-lifecycle.test.ts +++ b/src/daemon-client/__tests__/daemon-client-lifecycle.test.ts @@ -446,6 +446,7 @@ test('daemon acquisition reclaims a proven reused owner before publication', asy if (freshLock.state === 'held') assert.notEqual(freshLock.owner.startTime, 'stale-start-time'); assert.deepEqual(daemon.seenPaths, ['GET /health', 'POST /rpc']); } finally { + if (stale.status === 'acquired') await stale.acquisition.release(); await closeLoopbackServer(daemon.server); await finishRegisteredDaemonFixture(stateDir); } diff --git a/src/daemon-client/__tests__/daemon-client-startup-race.test.ts b/src/daemon-client/__tests__/daemon-client-startup-race.test.ts index 303a8b1faa..d5a2b7ede4 100644 --- a/src/daemon-client/__tests__/daemon-client-startup-race.test.ts +++ b/src/daemon-client/__tests__/daemon-client-startup-race.test.ts @@ -76,6 +76,10 @@ for (const command of ['devices', 'test']) { fs.writeFileSync(deferred, 'wait'); const winner = spawnRegisteredDaemonFixture(paths, fields(http.port), { stdio: 'ignore' }); await awaitFile(path.join(paths.baseDir, 'registration-held')); + fs.writeFileSync( + paths.infoPath, + JSON.stringify({ ...fields(http.port, '0.0.1'), pid: 999_999_999, processStartTime: 'old' }), + ); let joined = false; let genuineExit: ExecDetachedExit | undefined; let contender: ReturnType | undefined; @@ -250,7 +254,10 @@ for (const held of [true, false]) { }); } -test('a joined busy contender retires an older winner before relaunching', async (t) => { +test.for([ + { budget: 'ample', offset: 0, launches: 2, alive: false, rpcs: 1 }, + { budget: 'near deadline', offset: 11_000, launches: 1, alive: true, rpcs: 0 }, +])('an older winner is replaced only with enough startup time ($budget)', async (expected, t) => { if (!(await supportsLoopbackBind())) return t.skip('loopback unavailable'); const paths = resolveDaemonPaths(mkdtempForTestSync('daemon-start-older-winner-')); const http = await startHttpDaemonFixture({ devices: [] }); @@ -260,6 +267,9 @@ test('a joined busy contender retires an older winner before relaunching', async stdio: 'ignore', }); await awaitFile(path.join(paths.baseDir, 'registration-held')); + const wallTime = Date.now; + let offset = 0; + const clock = vi.spyOn(Date, 'now').mockImplementation(() => wallTime() + offset); let joined = false; spawn.mockImplementation((_command, _args, options) => { if (spawn.mock.calls.length > 1) assert.equal(joined, true); @@ -272,20 +282,30 @@ test('a joined busy contender retires an older winner before relaunching', async return child; }); pause.mockImplementation(async (ms) => { - if (joined) fs.rmSync(deferred, { force: true }); - await actualRetry.sleep(ms); + if (joined) { + fs.rmSync(deferred, { force: true }); + if (expected.offset) offset = offset ? offset + ms : expected.offset; + } + await actualRetry.sleep(10); }); const notice = vi.spyOn(process.stderr, 'write').mockImplementation(() => true); try { - assert.equal((await sendToDaemon(request(paths))).ok, true); - await winner.exited; - assert.equal(isProcessAlive(winner.pid), false); - assert.equal(spawn.mock.calls.length, 2); - assert.equal(http.rpcRequests.length, 1); - assert.ok( + const pending = sendToDaemon(request(paths)); + if (expected.alive) await assert.rejects(pending); + else { + assert.equal((await pending).ok, true); + await winner.exited; + } + assert.equal(joined, true); + assert.equal(isProcessAlive(winner.pid), expected.alive); + assert.equal(spawn.mock.calls.length, expected.launches); + assert.equal(http.rpcRequests.length, expected.rpcs); + assert.equal( notice.mock.calls.flat().join('').includes(`Replacing daemon (pid ${winner.pid}, v0.0.1)`), + !expected.alive, ); } finally { + clock.mockRestore(); notice.mockRestore(); await closeLoopbackServer(http.server); } diff --git a/src/daemon-client/daemon-client-lifecycle.ts b/src/daemon-client/daemon-client-lifecycle.ts index 77d46502a5..7b25dd630c 100644 --- a/src/daemon-client/daemon-client-lifecycle.ts +++ b/src/daemon-client/daemon-client-lifecycle.ts @@ -66,9 +66,11 @@ export type EnsuredDaemon = { type DaemonStartupWaitResult = | { kind: 'ready'; daemon: EnsuredDaemon } | { kind: 'early_exit'; exit: ExecDetachedExit } - | { kind: 'retry' | 'unproven' | 'timeout' }; + | { kind: 'unproven'; error: AppError } + | { kind: 'retry' | 'timeout' }; const DAEMON_STARTUP_TIMEOUT_MS = 15_000; +const MINIMUM_DAEMON_TAKEOVER_BUDGET_MS = 5_000; const LIVE_DAEMON_PROBE_RETRIES = 3; const LIVE_DAEMON_PROBE_RETRY_DELAY_MS = 200; const DAEMON_STARTUP_ATTEMPTS = 2; @@ -191,16 +193,10 @@ async function readReusableLocalDaemon( deadline?: number, ): Promise { const inspection = inspectProcessLock(settings.paths.lockPath); - if (inspection.state === 'unproven') { - throw new AppError('COMMAND_FAILED', 'Daemon registration ownership could not be verified.', { - reason: 'daemon_registration_unproven', - inspection, - stateDir: settings.paths.baseDir, - hint: resolveDaemonStartupHint(getDaemonMetadataState(settings.paths), settings.paths), - }); - } + if (inspection.state === 'unproven') throw daemonRegistrationUnprovenError(settings, inspection); const existing = readDaemonInfo(settings.paths.infoPath); if (!existing) return null; + if (!registrationAllowsDaemonObservation(inspection, existing)) return null; const decision = await resolveDaemonTakeover(existing, { onClientTransport: () => @@ -212,12 +208,36 @@ async function readReusableLocalDaemon( throw newerDaemonRefusedError(existing, decision, settings.paths.baseDir); } - if (deadline !== undefined && Date.now() >= deadline) return null; + if (remainingStartupBudget(deadline) < MINIMUM_DAEMON_TAKEOVER_BUDGET_MS) return null; emitDaemonTakeoverNotice(existing, decision.reason, settings.paths.baseDir); await retireDaemonForTakeover(existing, settings.paths); return null; } +function registrationAllowsDaemonObservation( + inspection: ProcessLockInspection, + info: DaemonInfo, +): boolean { + return ( + inspection.state === 'absent' || + (inspection.state === 'held' && + inspection.owner.pid === info.pid && + inspection.owner.startTime === (info.processStartTime ?? null)) + ); +} + +function daemonRegistrationUnprovenError( + settings: DaemonClientSettings, + inspection?: ProcessLockInspection, +): AppError { + return new AppError('COMMAND_FAILED', 'Daemon registration ownership could not be verified.', { + reason: 'daemon_registration_unproven', + inspection, + stateDir: settings.paths.baseDir, + hint: resolveDaemonStartupHint(getDaemonMetadataState(settings.paths), settings.paths), + }); +} + async function retireDaemonForTakeover(existing: DaemonInfo, paths: DaemonPaths): Promise { const retirement = await stopAndRetireDaemon({ paths: paths, @@ -328,6 +348,7 @@ function emitDaemonTakeoverNotice(info: DaemonInfo, reason: string, stateDir: st type FailedDaemonStartup = { cleanup?: DaemonRetirementResult; startError?: string; + startupError?: NormalizedError; daemonProcess?: ExecDetachedExit | { pid: number }; retry: boolean; }; @@ -346,6 +367,7 @@ async function startLocalDaemon(settings: DaemonClientSettings): Promise = failure ?? {}; const state = getDaemonMetadataState(settings.paths); const daemonLogTail = readRecentLogTail(settings.paths.logPath); throw new AppError('COMMAND_FAILED', 'Failed to start daemon', { @@ -357,8 +379,9 @@ async function startLocalDaemon(settings: DaemonClientSettings): Promise { - const winner = await readReusableLocalDaemon(settings, deadline); + let winner: DaemonInfo | null; + try { + winner = await readReusableLocalDaemon(settings, deadline); + } catch (error) { + if (error instanceof AppError && error.details?.reason === 'daemon_registration_unproven') + return { kind: 'unproven', error }; + throw error; + } if (Date.now() >= deadline) return null; if (winner) return { kind: 'ready', daemon: { info: winner, startedByClient: false } }; const inspection = inspectProcessLock(settings.paths.lockPath); - if (inspection.state === 'unproven') return { kind: 'unproven' }; + if (inspection.state === 'unproven') + return { kind: 'unproven', error: daemonRegistrationUnprovenError(settings, inspection) }; return isRegistrationAvailable(inspection) ? { kind: 'retry' } : null; } diff --git a/src/daemon-client/daemon-client-metadata.ts b/src/daemon-client/daemon-client-metadata.ts index 1be88db68b..5e9d060daa 100644 --- a/src/daemon-client/daemon-client-metadata.ts +++ b/src/daemon-client/daemon-client-metadata.ts @@ -1,5 +1,4 @@ import fs from 'node:fs'; -import { inspectProcessLock, type ProcessLockInspection } from '@agent-device/host-kit/file'; import { AppError } from '@agent-device/kernel/errors'; import { stopDaemonProcess, type DaemonTerminationResult } from '../daemon-process.ts'; @@ -31,7 +30,6 @@ export type DaemonInfo = { export type DaemonMetadataState = { hasInfo: boolean; hasLock: boolean; - registration: ProcessLockInspection; }; const DAEMON_TAKEOVER_TERM_TIMEOUT_MS = 3000; @@ -99,7 +97,6 @@ export function getDaemonMetadataState(paths: DaemonPaths): DaemonMetadataState return { hasInfo: fs.existsSync(paths.infoPath), hasLock: fs.existsSync(paths.lockPath), - registration: inspectProcessLock(paths.lockPath), }; } diff --git a/website/docs/docs/installation.md b/website/docs/docs/installation.md index 02886095b4..cef44e2760 100644 --- a/website/docs/docs/installation.md +++ b/website/docs/docs/installation.md @@ -126,4 +126,4 @@ Startup refuses legacy lock files and unverified ownership. Confirm every user o Packaged installs default to `~/.agent-device`; source checkouts use a worktree directory under `~/.agent-device/dev/`. `AGENT_DEVICE_STATE_DIR` or `--state-dir` overrides either default. -For source checkouts, `pnpm clean:daemon --prune-dev` selects development directories with no activity for 14 days, using their newest mtime. It retires only confirmed abandoned registration and retains shared directories, session artifacts and logs. +For source checkouts, `pnpm clean:daemon --prune-dev` selects development directories with no activity for 14 days, using the newest mtime of the directory and its immediate children. It retires only registration it can confirm abandoned. Directories, session artifacts and logs remain. From d2c7a19c0644bbab5e8251c0d2a2120448b10e7a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sat, 3 Oct 2026 21:16:14 +0200 Subject: [PATCH 09/20] test: bound held-claim startup deadline simulation --- .../__tests__/daemon-client-startup-race.test.ts | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/daemon-client/__tests__/daemon-client-startup-race.test.ts b/src/daemon-client/__tests__/daemon-client-startup-race.test.ts index d5a2b7ede4..7991d75c32 100644 --- a/src/daemon-client/__tests__/daemon-client-startup-race.test.ts +++ b/src/daemon-client/__tests__/daemon-client-startup-race.test.ts @@ -215,6 +215,7 @@ for (const held of [true, false]) { let now = Date.now(); const started = now; let released = false; + let advanced = false; let finishPending: () => void = () => {}; const nativeTimeout = globalThis.setTimeout; vi.spyOn(globalThis, 'setTimeout').mockImplementation((handler, ms, ...args) => @@ -231,9 +232,12 @@ for (const held of [true, false]) { }), })); pause.mockImplementation(async (ms) => { - if (!held && !released) { - await claim.acquisition.release(); - released = true; + if (!advanced) { + if (!held) { + await claim.acquisition.release(); + released = true; + } + advanced = true; now += 14_750; } else now += ms; }); From 3f2199b86142fb4fcdd9242f5fb9a4a5542844b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sat, 3 Oct 2026 03:50:19 +0200 Subject: [PATCH 10/20] fix: await owned daemon retirement on request timeout --- .../__tests__/daemon-client-lifecycle.test.ts | 46 +++-- .../__tests__/daemon-client-metadata.test.ts | 60 +----- .../daemon-client-timeout-route.test.ts | 193 +++++++++++++++--- .../__tests__/daemon-client-transport.test.ts | 61 ++++++ src/daemon-client/daemon-client-metadata.ts | 44 ---- src/daemon-client/daemon-client-timeout.ts | 78 +++---- src/daemon-client/daemon-client-transport.ts | 128 +++++++----- 7 files changed, 359 insertions(+), 251 deletions(-) diff --git a/src/daemon-client/__tests__/daemon-client-lifecycle.test.ts b/src/daemon-client/__tests__/daemon-client-lifecycle.test.ts index 88d57bbc14..487df065d2 100644 --- a/src/daemon-client/__tests__/daemon-client-lifecycle.test.ts +++ b/src/daemon-client/__tests__/daemon-client-lifecycle.test.ts @@ -27,6 +27,8 @@ import { resolveDaemonPaths, type DaemonPaths } from '../../daemon-resolution.ts import { sendToDaemon, type DaemonRequest, type DaemonResponse } from '../daemon-client.ts'; import { attachActiveSessionAddressHint } from '../daemon-client-lifecycle.ts'; import { sendRequest } from '../daemon-client-transport.ts'; +import { readDaemonInfo } from '../daemon-client-metadata.ts'; +import type { DaemonRetirementResult } from '../../daemon-registration-owner.ts'; import { closeLoopbackServer, listenOnLoopback, @@ -615,12 +617,18 @@ test('sendRequest timeout cleanup uses resolved daemon paths instead of request const daemonPaths = resolveDaemonPaths(daemonStateDir); const requestFlagPaths = resolveDaemonPaths(requestFlagStateDir); const daemon = await startHangingHttpDaemonFixture(); - writeDaemonInfo(daemonPaths, { - httpPort: daemon.port, - transport: 'http', - pid: 999_999, - }); - writeDaemonLock(daemonPaths, { pid: 999_999 }); + mockSleep.mockImplementation(actualRetry.sleep); + const child = spawnRegisteredDaemonFixture( + daemonPaths, + { + httpPort: daemon.port, + token: 'local-secret', + version: readVersion(), + codeOrigin: 'checkout', + codeSignature: currentDaemonCodeSignature(), + }, + undefined, + ); writeDaemonInfo(requestFlagPaths, { httpPort: daemon.port, transport: 'http', @@ -638,26 +646,26 @@ test('sendRequest timeout cleanup uses resolved daemon paths instead of request }; try { + let info = readDaemonInfo(daemonPaths.infoPath); + for (let attempt = 0; !info && attempt < 200; attempt += 1) { + await actualRetry.sleep(10); + info = readDaemonInfo(daemonPaths.infoPath); + } + assert.ok(info); let thrown: unknown; try { - await sendRequest( - { - token: 'local-secret', - pid: 999_999, - httpPort: daemon.port, - transport: 'http', - }, - request, - 'http', - daemonPaths, - 50, - ); + await sendRequest(info, request, 'http', daemonPaths, 50); } catch (error) { thrown = error; } assert.ok(thrown instanceof AppError); assert.equal(thrown.message, 'Daemon request timed out'); + assert.equal( + (thrown.details?.retirement as DaemonRetirementResult | undefined)?.status, + 'retired', + ); + await child.exited; assert.deepEqual(daemon.seenPaths, ['POST /rpc']); assert.equal(fs.existsSync(daemonPaths.infoPath), false); assert.equal(fs.existsSync(daemonPaths.lockPath), false); @@ -665,7 +673,7 @@ test('sendRequest timeout cleanup uses resolved daemon paths instead of request assert.equal(fs.existsSync(requestFlagPaths.lockPath), true); } finally { await closeLoopbackServer(daemon.server); - fs.rmSync(daemonStateDir, { recursive: true, force: true }); + await finishRegisteredDaemonFixture(daemonStateDir); fs.rmSync(requestFlagStateDir, { recursive: true, force: true }); } }); diff --git a/src/daemon-client/__tests__/daemon-client-metadata.test.ts b/src/daemon-client/__tests__/daemon-client-metadata.test.ts index 6bfec46b6e..bae4084103 100644 --- a/src/daemon-client/__tests__/daemon-client-metadata.test.ts +++ b/src/daemon-client/__tests__/daemon-client-metadata.test.ts @@ -1,29 +1,13 @@ import assert from 'node:assert/strict'; -import { AppError, normalizeError } from '@agent-device/kernel/errors'; import fs from 'node:fs'; import path from 'node:path'; -import { afterEach, test, vi } from 'vitest'; +import { test } from 'vitest'; import type { DaemonCodeOrigin } from '@agent-device/host-kit/code-signature'; import { mkdtempForTestSync } from '../../__tests__/test-utils/tmp-dir.ts'; -import { - stopAndRetireDaemon, - tryAcquireDaemonRegistration, -} from '../../daemon-registration-owner.ts'; -import { - readDaemonInfo, - stopDaemonProcessForTakeover, - type DaemonInfo, -} from '../daemon-client-metadata.ts'; -import { isAgentDeviceDaemonProcess, stopDaemonProcess } from '../../daemon-process.ts'; +import { tryAcquireDaemonRegistration } from '../../daemon-registration-owner.ts'; +import { readDaemonInfo, type DaemonInfo } from '../daemon-client-metadata.ts'; import { resolveDaemonPaths } from '../../daemon-resolution.ts'; -vi.mock('../../daemon-process.ts', async (importOriginal) => ({ - ...(await importOriginal()), - isAgentDeviceDaemonProcess: vi.fn(), - stopDaemonProcess: vi.fn(), -})); -afterEach(() => vi.resetAllMocks()); - // The reuse decision is only as good as the identity that survives the round trip // through `daemon.json`: a client cannot compare what the file lost (#2458). @@ -68,41 +52,3 @@ test('a registration this version did not write reads back unreported', () => { assert.equal(readDaemonInfo(infoPath)?.codeOrigin, undefined); } }); - -for (const artifact of ['daemon.json', 'daemon.lock']) { - test(`unconfirmed startup stop retains ${artifact} without claiming cleanup`, async () => { - const [stateDir] = scratchStateDir(); - const paths = resolveDaemonPaths(stateDir); - const file = path.join(stateDir, artifact); - const contents = JSON.stringify({ - pid: 7, - processStartTime: 'start', - port: 1234, - token: 'secret', - }); - fs.writeFileSync(file, contents); - vi.mocked(isAgentDeviceDaemonProcess).mockReturnValue(true); - vi.mocked(stopDaemonProcess).mockResolvedValue({ status: 'retained', reason: 'exit-timeout' }); - const result = await stopAndRetireDaemon({ - paths, - observed: { pid: 7, startTime: 'start' }, - mode: 'graceful', - }); - assert.equal(fs.readFileSync(file, 'utf8'), contents); - assert.equal(result.removedInfo, false); - assert.equal(result.status, 'retained'); - if (result.status === 'retained') assert.equal(result.reason, 'exit-unconfirmed'); - }); -} - -test('a retained takeover keeps its reason at the normalized error boundary', async () => { - vi.mocked(stopDaemonProcess).mockResolvedValue({ status: 'retained', reason: 'exit-timeout' }); - await assert.rejects( - stopDaemonProcessForTakeover({ pid: 7, token: 'secret', processStartTime: 'start' }), - (error: unknown) => { - assert.ok(error instanceof AppError); - assert.equal(normalizeError(error).details?.reason, 'daemon_exit_unconfirmed'); - return true; - }, - ); -}); diff --git a/src/daemon-client/__tests__/daemon-client-timeout-route.test.ts b/src/daemon-client/__tests__/daemon-client-timeout-route.test.ts index f02582c40e..04d4154403 100644 --- a/src/daemon-client/__tests__/daemon-client-timeout-route.test.ts +++ b/src/daemon-client/__tests__/daemon-client-timeout-route.test.ts @@ -26,19 +26,11 @@ import net from 'node:net'; import http from 'node:http'; import path from 'node:path'; +import fs from 'node:fs'; import assert from 'node:assert/strict'; import { beforeEach, afterEach, test, vi } from 'vitest'; -const { mockRunCmdSync, mockIsDaemon, mockStop } = vi.hoisted(() => ({ - mockRunCmdSync: vi.fn(), - mockIsDaemon: vi.fn(), - mockStop: vi.fn(), -})); -vi.mock('../../daemon-process.ts', async (importOriginal) => ({ - ...(await importOriginal()), - isAgentDeviceDaemonProcess: mockIsDaemon, - stopDaemonProcess: mockStop, -})); +const { mockRunCmdSync } = vi.hoisted(() => ({ mockRunCmdSync: vi.fn() })); vi.mock('@agent-device/host-kit/command', async () => { const actual = await vi.importActual( @@ -47,18 +39,29 @@ vi.mock('@agent-device/host-kit/command', async () => { return { ...actual, runCmdSync: mockRunCmdSync }; }); -import { AppError } from '@agent-device/kernel/errors'; +import { AppError, normalizeError } from '@agent-device/kernel/errors'; +import { sleep } from '@agent-device/host-kit/retry'; import { sendRequest } from '../daemon-client-transport.ts'; import type { DaemonRequest } from '../../daemon/daemon-request.ts'; -import type { DaemonInfo } from '../daemon-client-metadata.ts'; -import type { DaemonPaths } from '../../daemon-resolution.ts'; +import { readDaemonInfo, type DaemonInfo } from '../daemon-client-metadata.ts'; +import { resolveDaemonPaths, type DaemonPaths } from '../../daemon-resolution.ts'; +import type { DaemonRetirementResult } from '../../daemon-registration-owner.ts'; import { mkdtempForTestSync } from '../../__tests__/test-utils/tmp-dir.ts'; +import { + spawnRegisteredDaemonFixture, + finishRegisteredDaemonFixture, + finishRegisteredDaemonFixtures, +} from '../../__tests__/test-utils/registered-daemon-fixture.ts'; +import { + closeLoopbackServer, + skipWhenLoopbackUnavailable, +} from '../../__tests__/test-utils/loopback.ts'; const TIMEOUT_MS = 120; // `snapshot`'s timeout policy preserves the daemon (onTimeout !== -// 'reset-daemon'), so `handleRequestTimeout` never reaches -// `resetDaemonAfterTimeout` (`process.kill`) here — keeping this suite +// 'reset-daemon'), so `handleRequestTimeout` never signals the daemon +// in the hint controls — keeping them // side-effect-free outside the mocked pkill sweep. const SNAPSHOT_COMMAND = 'snapshot'; @@ -94,6 +97,7 @@ function startHangingSocketServer(): Promise<{ server: net.Server; port: number // Accept the connection but never write a response — forces the // client's own request-timeout envelope to fire. socket.on('error', () => {}); + socket.resume(); }); server.on('error', reject); server.listen(0, '127.0.0.1', () => { @@ -129,10 +133,11 @@ function startHangingHttpServer(): Promise<{ server: http.Server; port: number } beforeEach(() => { mockRunCmdSync.mockReset(); - mockIsDaemon.mockReset(); - mockStop.mockReset(); }); -afterEach(() => vi.restoreAllMocks()); +afterEach(async () => { + vi.restoreAllMocks(); + await finishRegisteredDaemonFixtures(); +}); test('socket timeout: pkill cleanup still runs for a declared non-Apple platform that actually terminates a runner (rebound-session case), and the hint claims Apple on that evidence', async () => { // Simulates --session-lock strip silently rebinding this request onto an @@ -279,32 +284,156 @@ test('remote HTTP timeout never runs the Apple pkill cleanup and uses the remote assert.equal(mockRunCmdSync.mock.calls.length, 0); }); -test('a refused timeout fallback preserves the timeout without an unhandled rejection', async () => { - mockRunCmdSync.mockReturnValue({ exitCode: 1, stdout: '', stderr: '' }); - mockIsDaemon.mockReturnValue(true); - mockStop.mockResolvedValue({ status: 'retained', reason: 'exit-timeout' }); - vi.spyOn(process, 'kill').mockImplementation(() => { - throw Object.assign(new Error('refused'), { code: 'EPERM' }); +async function publishedInfo(paths: DaemonPaths): Promise { + for (let attempt = 0; attempt < 200; attempt += 1) { + const info = readDaemonInfo(paths.infoPath); + if (info) return info; + await sleep(10); + } + throw new Error('registered child did not publish'); +} + +for (const transport of ['socket', 'http'] as const) { + test(`${transport} timeout waits for force retirement before reporting reset`, async (t) => { + if (await skipWhenLoopbackUnavailable(t)) return; + mockRunCmdSync.mockReturnValue({ exitCode: 1, stdout: '', stderr: '' }); + const endpoint = await (transport === 'socket' + ? startHangingSocketServer() + : startHangingHttpServer()); + const paths = resolveDaemonPaths(mkdtempForTestSync('agent-device-timeout-owner-')); + const child = spawnRegisteredDaemonFixture( + paths, + { + ...(transport === 'socket' ? { socketPort: endpoint.port } : { httpPort: endpoint.port }), + token: 'test-token', + version: 'test', + codeOrigin: 'checkout', + codeSignature: 'test', + }, + undefined, + ); + let exited = false; + void child.exited.then(() => { + exited = true; + }); + let killRequested!: () => void; + const requested = new Promise((resolve) => { + killRequested = resolve; + }); + const actualKill = process.kill.bind(process); + let settled = false; + let outcome: Promise | undefined; + const kill = vi.spyOn(process, 'kill').mockImplementation((pid, signal) => { + if (pid === child.pid && signal === 'SIGKILL') { + killRequested(); + return true; + } + return actualKill(pid, signal); + }); + try { + const info = await publishedInfo(paths); + outcome = sendRequest( + info, + { ...buildRequest(undefined), command: 'open' }, + transport, + paths, + TIMEOUT_MS, + ).then( + () => assert.fail('hanging request unexpectedly succeeded'), + (error: unknown) => { + settled = true; + return error; + }, + ); + await Promise.race([ + requested, + sleep(1_500).then(() => assert.fail('force stop was not requested')), + ]); + await sleep(30); + assert.equal(actualKill(child.pid, 0), true); + assert.equal(settled, false, 'request must remain pending while the daemon is alive'); + kill.mockRestore(); + actualKill(child.pid, 'SIGKILL'); + await child.exited; + const error = await outcome; + assert.ok(error instanceof AppError); + assert.equal(normalizeError(error).details?.reason, 'daemon_transport_timeout'); + const retirement = error.details?.retirement as DaemonRetirementResult | undefined; + assert.ok(retirement?.status === 'retired'); + assert.equal(retirement.termination.mode, 'forced'); + assert.equal(fs.existsSync(paths.infoPath), false); + assert.equal(fs.existsSync(paths.lockPath), false); + assert.equal(fs.existsSync(paths.baseDir), true); + assert.equal(mockRunCmdSync.mock.calls.filter(([cmd]) => cmd === 'pkill').length, 3); + } finally { + kill.mockRestore(); + if (!exited) actualKill(child.pid, 'SIGKILL'); + await child.exited; + await outcome; + await finishRegisteredDaemonFixture(paths.baseDir); + await closeLoopbackServer(endpoint.server); + } }); - const { server, port } = await startHangingSocketServer(); +} + +test('timeout retains a live registration without captured birth proof and reports that outcome', async (t) => { + if (await skipWhenLoopbackUnavailable(t)) return; + mockRunCmdSync.mockReturnValue({ exitCode: 1, stdout: '', stderr: '' }); + const endpoint = await startHangingHttpServer(); + const paths = resolveDaemonPaths(mkdtempForTestSync('agent-device-timeout-retained-')); + const child = spawnRegisteredDaemonFixture( + paths, + { + httpPort: endpoint.port, + token: 'test-token', + version: 'test', + codeOrigin: 'checkout', + codeSignature: 'test', + }, + undefined, + ); + const kill = vi.spyOn(process, 'kill'); try { + const info = await publishedInfo(paths); + const before = fs.readFileSync(paths.infoPath, 'utf8'); + const lockBefore = fs + .readdirSync(paths.lockPath) + .map((name) => [name, fs.readFileSync(path.join(paths.lockPath, name), 'utf8')]); await assert.rejects( sendRequest( - { port, pid: 7, token: 'test-token', processStartTime: 'start' }, + { ...info, processStartTime: undefined }, { ...buildRequest(undefined), command: 'open' }, - 'socket', - dummyStatePaths(), + 'http', + paths, TIMEOUT_MS, ), (error: unknown) => { assert.ok(error instanceof AppError); - assert.equal(error.details?.reason, 'daemon_transport_timeout'); + assert.equal(normalizeError(error).details?.reason, 'daemon_transport_timeout'); + const retirement = error.details?.retirement as DaemonRetirementResult | undefined; + assert.ok(retirement?.status === 'retained'); + assert.ok(retirement.termination?.status === 'retained'); + assert.equal(retirement.termination.reason, 'missing-start-time'); + assert.match(normalizeError(error).hint ?? '', /State was retained/); + assert.doesNotMatch(normalizeError(error).hint ?? '', /daemon was reset/); return true; }, ); - await new Promise((resolve) => setImmediate(resolve)); - assert.equal(mockStop.mock.calls.length, 1); + assert.equal(process.kill(child.pid, 0), true); + assert.equal(fs.readFileSync(paths.infoPath, 'utf8'), before); + assert.deepEqual( + fs + .readdirSync(paths.lockPath) + .map((name) => [name, fs.readFileSync(path.join(paths.lockPath, name), 'utf8')]), + lockBefore, + ); + assert.equal( + kill.mock.calls.some(([pid, signal]) => pid === child.pid && signal !== 0), + false, + ); } finally { - server.close(); + kill.mockRestore(); + await finishRegisteredDaemonFixture(paths.baseDir); + await closeLoopbackServer(endpoint.server); } }); diff --git a/src/daemon-client/__tests__/daemon-client-transport.test.ts b/src/daemon-client/__tests__/daemon-client-transport.test.ts index d9b60a8bd8..d9860c241a 100644 --- a/src/daemon-client/__tests__/daemon-client-transport.test.ts +++ b/src/daemon-client/__tests__/daemon-client-transport.test.ts @@ -1,6 +1,9 @@ import assert from 'node:assert/strict'; import http from 'node:http'; +import net from 'node:net'; import { test, vi } from 'vitest'; +import * as hostTransport from '@agent-device/host-kit/transport'; +import { sleep } from '@agent-device/host-kit/retry'; import { AppError } from '@agent-device/kernel/errors'; import { DAEMON_HTTP_INSTANCE_HEADER, @@ -36,6 +39,64 @@ function sendWithStaleInstance(port: number, timeoutMs: number) { ); } +test('auto health probing reserves time for a healthy fallback when HTTP hangs', async (t) => { + if (await skipWhenLoopbackUnavailable(t)) return; + const httpServer = http.createServer(() => {}); + const socketServer = net.createServer((socket) => socket.on('error', () => {})); + try { + const httpPort = await listenOnLoopback(httpServer); + const port = await listenOnLoopback(socketServer); + assert.equal( + await canConnect({ token: 'secret', pid: 1, transport: 'http', httpPort, port }, 'auto', 120), + true, + ); + } finally { + await closeLoopbackServer(httpServer); + await closeLoopbackServer(socketServer); + } +}); + +test('the health deadline includes requester loading and forbids a late request', async (t) => { + if (await skipWhenLoopbackUnavailable(t)) return; + let requests = 0; + const server = http.createServer((_req, res) => { + requests += 1; + res.end('{}'); + }); + let release!: () => void; + const blocked = new Promise((resolve) => { + release = resolve; + }); + const actualLoad = hostTransport.loadNodeHttpRequester; + const load = vi + .spyOn(hostTransport, 'loadNodeHttpRequester') + .mockImplementation(async (protocol) => { + await blocked; + return actualLoad(protocol); + }); + let probing: Promise | undefined; + try { + const httpPort = await listenOnLoopback(server); + let settled = false; + probing = canConnect({ token: 'secret', pid: 1, httpPort }, 'http', 40).then((reachable) => { + settled = true; + return reachable; + }); + await sleep(90); + assert.equal(settled, true, 'loading must not extend the probe deadline'); + assert.equal(await probing, false); + release(); + await blocked; + await sleep(10); + assert.equal(requests, 0, 'a timed-out loader must not open a request later'); + } finally { + release(); + await probing; + load.mockRestore(); + await closeLoopbackServer(server); + } +}); + test('persistent remote client caches health and retries a refused stale instance before dispatch', async (t) => { if (await skipWhenLoopbackUnavailable(t)) return; const paths: string[] = []; diff --git a/src/daemon-client/daemon-client-metadata.ts b/src/daemon-client/daemon-client-metadata.ts index 5e9d060daa..d517da18e4 100644 --- a/src/daemon-client/daemon-client-metadata.ts +++ b/src/daemon-client/daemon-client-metadata.ts @@ -1,6 +1,4 @@ import fs from 'node:fs'; -import { AppError } from '@agent-device/kernel/errors'; -import { stopDaemonProcess, type DaemonTerminationResult } from '../daemon-process.ts'; import type { DaemonCodeOrigin } from '@agent-device/host-kit/code-signature'; @@ -32,9 +30,6 @@ export type DaemonMetadataState = { hasLock: boolean; }; -const DAEMON_TAKEOVER_TERM_TIMEOUT_MS = 3000; -const DAEMON_TAKEOVER_KILL_TIMEOUT_MS = 1000; - export function readDaemonInfo(infoPath: string): DaemonInfo | null { const data = readJsonFile(infoPath); if (!data || typeof data !== 'object') return null; @@ -85,14 +80,6 @@ function readPositiveInteger(value: unknown): number | undefined { return Number.isInteger(value) && Number(value) > 0 ? Number(value) : undefined; } -export function removeDaemonInfo(infoPath: string): void { - removeFileIfExists(infoPath); -} - -export function removeDaemonLock(lockPath: string): void { - removeFileIfExists(lockPath); -} - export function getDaemonMetadataState(paths: DaemonPaths): DaemonMetadataState { return { hasInfo: fs.existsSync(paths.infoPath), @@ -100,29 +87,6 @@ export function getDaemonMetadataState(paths: DaemonPaths): DaemonMetadataState }; } -export async function stopDaemonProcessForTakeover( - info: DaemonInfo, -): Promise { - const termination = await stopDaemonProcess( - { pid: info.pid, startTime: info.processStartTime ?? null }, - { - mode: 'graceful', - termTimeoutMs: DAEMON_TAKEOVER_TERM_TIMEOUT_MS, - killTimeoutMs: DAEMON_TAKEOVER_KILL_TIMEOUT_MS, - }, - ); - requireDaemonExit(termination); - return termination; -} - -function requireDaemonExit(termination: DaemonTerminationResult): void { - if (termination.status !== 'retained') return; - throw new AppError('COMMAND_FAILED', 'Daemon exit could not be confirmed.', { - reason: 'daemon_exit_unconfirmed', - termination, - }); -} - export function isRemoteDaemon(info: DaemonInfo): boolean { return typeof info.baseUrl === 'string' && info.baseUrl.length > 0; } @@ -147,11 +111,3 @@ function readJsonFile(filePath: string): unknown | null { return null; } } - -function removeFileIfExists(filePath: string): void { - try { - if (fs.existsSync(filePath)) fs.unlinkSync(filePath); - } catch { - // Best-effort cleanup only. - } -} diff --git a/src/daemon-client/daemon-client-timeout.ts b/src/daemon-client/daemon-client-timeout.ts index d338f74d02..c27c95b1b7 100644 --- a/src/daemon-client/daemon-client-timeout.ts +++ b/src/daemon-client/daemon-client-timeout.ts @@ -1,18 +1,13 @@ -import { AppError, normalizeError } from '@agent-device/kernel/errors'; +import { AppError } from '@agent-device/kernel/errors'; import { runCmdSync } from '@agent-device/host-kit/command'; import { emitDiagnostic } from '@agent-device/host-kit/diagnostics'; -import { isAgentDeviceDaemonProcess } from '../daemon-process.ts'; +import type { DaemonRetirementResult } from '../daemon-registration-owner.ts'; import { PUBLIC_COMMANDS } from '@agent-device/command-registry/catalog'; import { resolveCommandTimeoutPolicy } from '@agent-device/command-registry/registry'; import type { DaemonPaths } from '../daemon-resolution.ts'; import type { PlatformSelector } from '@agent-device/kernel/device'; -import { - removeDaemonInfo, - removeDaemonLock, - stopDaemonProcessForTakeover, - type DaemonInfo, -} from './daemon-client-metadata.ts'; +import type { DaemonInfo } from './daemon-client-metadata.ts'; const IOS_RUNNER_XCODEBUILD_KILL_PATTERNS = [ 'xcodebuild .*AgentDeviceRunnerUITests/RunnerTests/testCommand', @@ -40,7 +35,7 @@ function isAffirmativelyApplePlatform(platform: PlatformSelector | undefined): b return platform !== undefined && AFFIRMATIVE_APPLE_PLATFORM_SELECTORS.has(platform); } -export function handleRequestTimeout( +export async function handleRequestTimeout( params: Readonly<{ info: DaemonInfo; statePaths: DaemonPaths; @@ -53,7 +48,7 @@ export function handleRequestTimeout( session?: string; action?: string; }>, -): AppError { +): Promise { const { info, statePaths, remote, timeoutMs, requestId, command, platform, session, action } = params; // Cleanup eligibility stays UNCONDITIONAL for every local (non-remote) @@ -69,9 +64,19 @@ export function handleRequestTimeout( // a wrong skip. const cleanup = remote ? { terminated: 0 } : cleanupTimedOutIosRunnerBuilds(); const resetDaemon = !remote && shouldResetDaemonAfterRequestTimeout(command); - const daemonReset = resetDaemon - ? resetDaemonAfterTimeout(info, statePaths) - : { forcedKill: false }; + let retirement: DaemonRetirementResult | undefined; + if (resetDaemon) { + const { stopAndRetireDaemon } = await import('../daemon-registration-owner.ts'); + retirement = await stopAndRetireDaemon({ + paths: statePaths, + observed: { pid: info.pid, startTime: info.processStartTime ?? null }, + mode: 'force', + }); + } + const forcedKill = + retirement?.status !== 'absent' && + retirement?.termination?.status === 'exited' && + retirement.termination.mode === 'forced'; // The HINT, unlike cleanup, may only name Apple-runner involvement on // evidence this call site actually has: an explicitly declared Apple // platform selector, or the cleanup itself having terminated a matching @@ -89,8 +94,9 @@ export function handleRequestTimeout( command, timedOutRunnerPidsTerminated: cleanup.terminated, timedOutRunnerCleanupError: cleanup.error, - daemonPidReset: resetDaemon ? info.pid : undefined, - daemonPidForceKilled: resetDaemon ? daemonReset.forcedKill : undefined, + daemonPidReset: retirement?.status === 'retired' ? info.pid : undefined, + daemonPidForceKilled: resetDaemon ? forcedKill : undefined, + daemonRetirement: retirement, daemonPreservedAfterTimeout: !remote && !resetDaemon, daemonBaseUrl: info.baseUrl, }, @@ -99,14 +105,18 @@ export function handleRequestTimeout( timeoutMs, requestId, reason: 'daemon_transport_timeout', - hint: resolveRequestTimeoutHint({ - remote, - resetDaemon, - command, - appleCleanupEvidence, - session, - action, - }), + ...(retirement ? { retirement, stateDir: statePaths.baseDir } : {}), + hint: + retirement?.status === 'retained' + ? `The daemon could not be safely retired. State was retained at ${statePaths.baseDir}. ${retirement.error?.hint ?? 'Retry with --debug and inspect daemon diagnostics before retrying.'}` + : resolveRequestTimeoutHint({ + remote, + resetDaemon, + command, + appleCleanupEvidence, + session, + action, + }), }); } @@ -177,25 +187,3 @@ function cleanupTimedOutIosRunnerBuilds(): { terminated: number; error?: string }; } } - -function resetDaemonAfterTimeout(info: DaemonInfo, paths: DaemonPaths): { forcedKill: boolean } { - let forcedKill = false; - try { - if (isAgentDeviceDaemonProcess(info.pid, info.processStartTime)) { - process.kill(info.pid, 'SIGKILL'); - forcedKill = true; - } - } catch { - void stopDaemonProcessForTakeover(info).catch((error: unknown) => { - emitDiagnostic({ - level: 'warn', - phase: 'daemon_timeout_stop_failed', - data: { error: normalizeError(error) }, - }); - }); - } finally { - removeDaemonInfo(paths.infoPath); - removeDaemonLock(paths.lockPath); - } - return { forcedKill }; -} diff --git a/src/daemon-client/daemon-client-transport.ts b/src/daemon-client/daemon-client-transport.ts index aa0f93820d..66d545b0a3 100644 --- a/src/daemon-client/daemon-client-transport.ts +++ b/src/daemon-client/daemon-client-transport.ts @@ -76,10 +76,13 @@ export async function canConnect( ): Promise { const deadline = Date.now() + (probeTimeoutMs ?? Number.POSITIVE_INFINITY); const transport = chooseTransport(info, preference); - if (await canConnectWithTransport(info, transport, deadline - Date.now())) return true; - const fallback = chooseAutoFallbackTransport(info, preference, transport); - return fallback ? await canConnectWithTransport(info, fallback, deadline - Date.now()) : false; + const firstBudget = (deadline - Date.now()) / (fallback ? 2 : 1); + if (await canConnectWithTransport(info, transport, firstBudget)) return Date.now() < deadline; + return fallback + ? (await canConnectWithTransport(info, fallback, deadline - Date.now())) && + Date.now() < deadline + : false; } async function canConnectWithTransport( @@ -160,17 +163,27 @@ async function readDaemonHttpHealth( : null; if (!endpoint) return { reachable: false }; const url = new URL(endpoint); - const transport = await loadNodeHttpRequester(url.protocol); const timeoutMs = Math.min( info.baseUrl ? REMOTE_DAEMON_HEALTHCHECK_TIMEOUT_MS : LOCAL_DAEMON_HEALTHCHECK_TIMEOUT_MS, probeTimeoutMs ?? Number.POSITIVE_INFINITY, ); if (timeoutMs <= 0) return { reachable: false, timedOut: true }; + const deadline = performance.now() + timeoutMs; const signal = AbortSignal.timeout(Math.ceil(timeoutMs)); + const transport = await Promise.race([ + loadNodeHttpRequester(url.protocol), + new Promise((resolve) => { + signal.addEventListener('abort', () => resolve(null), { once: true }); + }), + ]); + if (!transport || healthProbeExpired(signal, deadline)) + return { reachable: false, timedOut: true }; return await new Promise((resolve) => { const headers = info.baseUrl ? buildDaemonHttpAuthHeaders(info.token) : {}; const unreachable = (): RemoteDaemonHealth => - signal.aborted ? { reachable: false, timedOut: true } : { reachable: false }; + healthProbeExpired(signal, deadline) + ? { reachable: false, timedOut: true } + : { reachable: false }; const req = transport.request( { protocol: url.protocol, @@ -190,11 +203,11 @@ async function readDaemonHttpHealth( }); res.on('end', () => { const statusCode = res.statusCode ?? 500; - resolve({ - reachable: statusCode < 500, - statusCode, - ...readHealthPayload(body), - }); + resolve( + healthProbeExpired(signal, deadline) + ? { reachable: false, timedOut: true } + : { reachable: statusCode < 500, statusCode, ...readHealthPayload(body) }, + ); }); res.on('error', () => resolve(unreachable())); res.on('aborted', () => resolve(unreachable())); @@ -211,6 +224,10 @@ async function readDaemonHttpHealth( }); } +function healthProbeExpired(signal: AbortSignal, deadline: number): boolean { + return signal.aborted || performance.now() >= deadline; +} + function readHealthPayload(body: string): Omit { try { const parsed = JSON.parse(body) as { upstream?: unknown }; @@ -245,6 +262,29 @@ export async function sendRequest( statePaths: DaemonPaths, timeoutMs: number | undefined, options: SendRequestOptions = {}, +): Promise { + try { + return await sendRequestWithFallback(info, req, preference, statePaths, timeoutMs, options); + } catch (error) { + if (!(error instanceof AppError) || error.details?.reason !== 'daemon_transport_timeout') + throw error; + const expiredBudget = error.details.timeoutMs; + if (typeof expiredBudget !== 'number') throw error; + throw await handleRequestTimeout({ + info, + statePaths, + ...timeoutRequestContext(req, isRemoteDaemon(info), expiredBudget), + }); + } +} + +async function sendRequestWithFallback( + info: DaemonInfo, + req: DaemonRequest, + preference: DaemonTransportPreference, + statePaths: DaemonPaths, + timeoutMs: number | undefined, + options: SendRequestOptions = {}, ): Promise { const transport = chooseTransport(info, preference); const deadline = typeof timeoutMs === 'number' ? performance.now() + timeoutMs : undefined; @@ -279,30 +319,14 @@ async function retryAfterRemoteInstanceMismatch( options: SendRequestOptions, ): Promise { invalidateRemoteDaemonHealth(info); - const probeTimeoutMs = remainingRemoteRequestTimeoutMs( - info, + const probeTimeoutMs = remainingRemoteRequestTimeoutMs(req, timeoutMs, deadline); + const health = await readRemoteDaemonHealth(info, probeTimeoutMs); + const remainingMs = remainingRemoteRequestTimeoutMs( req, - statePaths, timeoutMs, deadline, + health.timedOut ? probeTimeoutMs : undefined, ); - const health = await readRemoteDaemonHealth(info, probeTimeoutMs); - // The probe's timer starts from the event loop's cached clock, so it can expire while - // performance.now() is still short of the deadline: a probe the RPC deadline capped that ran out - // of time is the RPC timing out. - if ( - health.timedOut && - timeoutMs !== undefined && - probeTimeoutMs !== undefined && - probeTimeoutMs <= REMOTE_DAEMON_HEALTHCHECK_TIMEOUT_MS - ) { - throw handleRequestTimeout({ - info, - statePaths, - ...timeoutRequestContext(req, true, timeoutMs), - }); - } - const remainingMs = remainingRemoteRequestTimeoutMs(info, req, statePaths, timeoutMs, deadline); if (!health.reachable) { throw new AppError('COMMAND_FAILED', 'Remote daemon is unavailable', { daemonBaseUrl: info.baseUrl, @@ -320,20 +344,20 @@ async function retryAfterRemoteInstanceMismatch( } function remainingRemoteRequestTimeoutMs( - info: DaemonInfo, req: DaemonRequest, - statePaths: DaemonPaths, timeoutMs: number | undefined, deadline: number | undefined, + timedOutProbeMs?: number, ): number | undefined { if (deadline === undefined || timeoutMs === undefined) return undefined; const remainingMs = deadline - performance.now(); - if (remainingMs > 0) return remainingMs; - throw handleRequestTimeout({ - info, - statePaths, - ...timeoutRequestContext(req, true, timeoutMs), - }); + // A probe capped by the request can expire before the monotonic clock catches up to its timer. + if ( + remainingMs > 0 && + (timedOutProbeMs === undefined || timedOutProbeMs > REMOTE_DAEMON_HEALTHCHECK_TIMEOUT_MS) + ) + return remainingMs; + throw requestTimeoutError(timeoutMs, req.meta?.requestId); } function isRemoteInstanceMismatch(error: unknown): boolean { @@ -359,7 +383,7 @@ async function sendRequestWithTransport( ): Promise { return transport === 'http' ? await sendHttpRequest(info, req, statePaths, timeoutMs, options) - : await sendSocketRequest(info, req, statePaths, timeoutMs, options); + : await sendSocketRequest(info, req, timeoutMs, options); } function chooseTransport( @@ -454,7 +478,6 @@ function handleTransportError( async function sendSocketRequest( info: DaemonInfo, req: DaemonRequest, - statePaths: DaemonPaths, timeoutMs: number | undefined, options: SendRequestOptions, ): Promise { @@ -470,15 +493,10 @@ async function sendSocketRequest( const timeoutHandle = typeof timeoutMs === 'number' ? setTimeout(() => { + if (settled) return; settled = true; + reject(requestTimeoutError(timeoutMs, req.meta?.requestId)); socket.destroy(); - reject( - handleRequestTimeout({ - info, - statePaths, - ...timeoutRequestContext(req, false, timeoutMs), - }), - ); }, timeoutMs) : undefined; @@ -512,6 +530,14 @@ async function sendSocketRequest( }); } +function requestTimeoutError(timeoutMs: number, requestId: string | undefined): AppError { + return new AppError('COMMAND_FAILED', 'Daemon request timed out', { + reason: 'daemon_transport_timeout', + timeoutMs, + requestId, + }); +} + // The fields a timed-out request is described by, read once so a socket and an HTTP timeout cannot // describe the same request differently. type TimeoutRequestFields = Omit[0], 'info' | 'statePaths'>; @@ -644,14 +670,8 @@ async function sendHttpRequest( const timeoutHandle = typeof timeoutMs === 'number' ? setTimeout(() => { + reject(requestTimeoutError(timeoutMs, req.meta?.requestId)); request.destroy(); - reject( - handleRequestTimeout({ - info, - statePaths, - ...timeoutRequestContext(req, remote, timeoutMs), - }), - ); }, timeoutMs) : undefined; From 50eafe5c434ff946f70a50b04840058679d0afbc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sat, 3 Oct 2026 03:56:11 +0200 Subject: [PATCH 11/20] chore(gates): acknowledge unchanged health payload under probe deadlines --- test/wire-compat/ledger.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/wire-compat/ledger.json b/test/wire-compat/ledger.json index 80e13dc3e9..20fc434604 100644 --- a/test/wire-compat/ledger.json +++ b/test/wire-compat/ledger.json @@ -67,7 +67,7 @@ "src/daemon-client/daemon-client-rpc.ts#toDaemonHttpRpcError": "sha256:888246763c48670e7da893054d025744654f8715c3b4906312617a2b5028316b", "src/daemon-client/daemon-client-transport.ts#RemoteDaemonHealth": "sha256:3bac36fa97090b273afe1128103e1bf476d05441fd9de41317f1b78775b88304", "src/daemon-client/daemon-client-transport.ts#RemoteDaemonHealthLink": "sha256:7702598468b4c82b4ffa93064cd6bdfec938c1c527f7e6007c0584f3884a6eea", - "src/daemon-client/daemon-client-transport.ts#readDaemonHttpHealth": "sha256:eef0e153eb6f0bc02175e464dd6d98559b500b65ea8c74ba2203cfef09ec09a0", + "src/daemon-client/daemon-client-transport.ts#readDaemonHttpHealth": "sha256:f40cc2f5bfb8add78f99b11db7842bc244735786aa390d10a38ef025e16dc53a", "src/daemon-client/daemon-client-transport.ts#readHealthLink": "sha256:f56404b94d73da57de7248719c9136b760d2be8b4a53cde5315a2311b088425b", "src/daemon-client/daemon-client-transport.ts#readHealthPayload": "sha256:4e85ffc3e35e02379c393e9312344757e003cf1f0ad9eb8d1f77d90c81c861f1", "src/daemon-client/daemon-client-transport.ts#readRemoteDaemonHealth": "sha256:bcefa89fbb7fcbd6fee1b5ecb217955b9eee8d6fd2ad175fb653011edbd199d9", @@ -431,8 +431,8 @@ }, { "declaration": "src/daemon-client/daemon-client-transport.ts#readDaemonHttpHealth", - "digest": "sha256:eef0e153eb6f0bc02175e464dd6d98559b500b65ea8c74ba2203cfef09ec09a0", - "rationale": "A restart retry must tell a probe that ran out of time from one that failed, so the client can report the RPC deadline instead of 'Remote daemon is unavailable'. `timedOut` is client-local: the prober sets it on its own timeout and abort paths and never reads it from a /health payload. The health request and the accepted payload fields are unchanged, so a released daemon or proxy is probed and parsed exactly as before." + "digest": "sha256:f40cc2f5bfb8add78f99b11db7842bc244735786aa390d10a38ef025e16dc53a", + "rationale": "#3116 includes requester loading and response completion in the existing health-probe budget. `timedOut` remains client-local; it is never read from a /health payload. Request routes, authentication, accepted fields and protocol-version admission are unchanged, so released daemons and proxies still send payloads this client parses correctly. This is an implementation-only timing change under ADR 0006." } ] } From 740111b0f59761505e30cdc01d6f1c1a8df9de14 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sat, 3 Oct 2026 04:24:26 +0200 Subject: [PATCH 12/20] fix: share manual daemon retirement and verify lock cutover --- docs/adr/0030-process-lock-exclusion.md | 32 +++- .../daemon-registration-owner.test.ts | 159 +++++++++++++++++- .../test-utils/legacy-daemon-fixture.ts | 92 ++++++++++ .../test-utils/registered-daemon-fixture.ts | 10 +- src/daemon/__tests__/daemon-stop.test.ts | 129 ++++++++------ src/daemon/daemon-stop.ts | 56 +++--- 6 files changed, 400 insertions(+), 78 deletions(-) create mode 100644 src/__tests__/test-utils/legacy-daemon-fixture.ts diff --git a/docs/adr/0030-process-lock-exclusion.md b/docs/adr/0030-process-lock-exclusion.md index 5aac07b2df..0e1458f936 100644 --- a/docs/adr/0030-process-lock-exclusion.md +++ b/docs/adr/0030-process-lock-exclusion.md @@ -40,6 +40,32 @@ guard cannot constrain legacy code after its final check. The support boundary t requires deployment control; host-kit does not claim to detect or evict every legacy user. Daemon registration has a separate cutover boundary: keep the same `daemon.lock` path and -refuse legacy files rather than automatically reclaiming them. Its startup tests must cover -an already-running older daemon and concurrent old/new startup. This does not expand the -host-kit mixed-protocol support contract. +refuse legacy files rather than automatically reclaiming them. A legacy daemon creates an +exclusive file; a hardened daemon creates a directory at that same path. The old acquisition +cannot unlink a directory, and the new acquisition retains an existing file. + +The [registration tests](../../src/__tests__/daemon-registration-owner.test.ts) exercise real +children using the c237027737 legacy acquisition and the current owner. They cover an older +daemon already running, a hardened owner waiting to publish metadata, and concurrent startup. +The client refuses an older registration before signaling or changing it. This proves the daemon +cutover; it does not expand the host-kit mixed-protocol support contract. Older clients still +require the deployment controls above. + +## Registration operations + +[Shared retirement](../../src/daemon-registration-owner.ts) owns verified termination, protected +metadata inspection and removal, and release. Takeover, failed startup, replay cleanup, timeout +reset and manual stop await its result. Abandoned recovery uses the same protected retirement +sequence without signaling a live process. Daemon publication and shutdown use functions bound +to their acquired claim. + +```mermaid +flowchart LR + C[Client lifecycle and timeout] --> R[Shared retirement] + M[Manual stop] --> R + P[Abandoned recovery and pruning] --> R + R --> L[Acquired registration claim] + D[Daemon startup and shutdown] --> O[Functions bound to own claim] + O --> L + L --> F[Protected metadata and reports] +``` diff --git a/src/__tests__/daemon-registration-owner.test.ts b/src/__tests__/daemon-registration-owner.test.ts index 7f4795f6d8..47710d6a62 100644 --- a/src/__tests__/daemon-registration-owner.test.ts +++ b/src/__tests__/daemon-registration-owner.test.ts @@ -1,5 +1,6 @@ import assert from 'node:assert/strict'; import fs from 'node:fs'; +import path from 'node:path'; import { afterEach, test, vi } from 'vitest'; import { readCurrentOwnerIdentity, isProcessAlive } from '@agent-device/host-kit/process'; import { @@ -15,9 +16,18 @@ import { resolveDaemonPaths, type DaemonPaths } from '../daemon-resolution.ts'; import { readRegisteredDaemonOwnership } from '../daemon-registration.ts'; import { readDaemonShutdownReport } from '../daemon-shutdown-report.ts'; import { mkdtempForTestSync } from './test-utils/tmp-dir.ts'; -import { registeredDaemonFixtureArgs } from './test-utils/registered-daemon-fixture.ts'; +import { + registeredDaemonFixtureArgs, + spawnRegisteredDaemonFixture, + finishRegisteredDaemonFixture, +} from './test-utils/registered-daemon-fixture.ts'; +import { spawnLegacyDaemonFixture } from './test-utils/legacy-daemon-fixture.ts'; +import { ensureDaemon, resolveClientSettings } from '../daemon-client/daemon-client-lifecycle.ts'; +import { inspectProcessLock } from '@agent-device/host-kit/file'; +import { AppError } from '@agent-device/kernel/errors'; import { sleep } from '@agent-device/host-kit/retry'; import { stopDaemonProcess } from '../daemon-process.ts'; +import { stopDaemon } from '../daemon/daemon-stop.ts'; const fields = { socketPort: 4210, @@ -465,3 +475,150 @@ async function finishPrivateTestDaemons( } fs.rmSync(paths.baseDir, { recursive: true, force: true }); } + +async function waitForCutoverFixture(ready: () => boolean): Promise { + for (let attempt = 0; attempt < 200; attempt += 1) { + if (ready()) return; + await sleep(10); + } + assert.fail('cutover fixture did not reach its barrier'); +} + +function legacyDisposition(paths: DaemonPaths): boolean { + return ( + JSON.parse(fs.readFileSync(path.join(paths.baseDir, 'legacy-disposition.json'), 'utf8')) as { + acquired: boolean; + } + ).acquired; +} + +test('cutover refuses an already-running legacy daemon before signaling or changing registration', async () => { + const paths = resolveDaemonPaths(mkdtempForTestSync('agent-device-old-daemon-')); + const legacy = spawnLegacyDaemonFixture(paths); + try { + await waitForCutoverFixture(() => fs.existsSync(paths.infoPath)); + const metadata = fs.readFileSync(paths.infoPath, 'utf8'); + const lock = fs.readFileSync(paths.lockPath, 'utf8'); + const contender = spawnRegisteredDaemonFixture(paths, fields, undefined); + let disposition: Awaited | undefined; + void contender.exited.then((result) => { + disposition = result; + }); + await waitForCutoverFixture( + () => Boolean(disposition) || fs.existsSync(path.join(paths.baseDir, 'registration-held')), + ); + assert.ok(disposition, 'a new daemon must refuse an occupied legacy file'); + assert.equal(disposition.exitCode, DAEMON_STARTUP_EXIT_CODES.unproven); + await assert.rejects( + ensureDaemon( + resolveClientSettings({ + session: 'default', + command: 'devices', + positionals: [], + flags: { stateDir: paths.baseDir }, + }), + ), + (error: unknown) => { + assert.ok(error instanceof AppError); + assert.equal(error.details?.reason, 'daemon_registration_unproven'); + return true; + }, + ); + assert.equal(process.kill(legacy.pid, 0), true); + assert.equal(fs.readFileSync(paths.infoPath, 'utf8'), metadata); + assert.equal(fs.readFileSync(paths.lockPath, 'utf8'), lock); + } finally { + await legacy.stop(); + await finishRegisteredDaemonFixture(paths.baseDir); + } +}); + +test('a legacy contender cannot unlink a hardened owner while it delays metadata publication', async () => { + const paths = resolveDaemonPaths(mkdtempForTestSync('agent-device-new-daemon-')); + const deferred = path.join(paths.baseDir, 'defer-publication'); + fs.writeFileSync(deferred, 'wait'); + const current = spawnRegisteredDaemonFixture(paths, fields, undefined); + let legacy: ReturnType | undefined; + try { + await waitForCutoverFixture(() => fs.existsSync(path.join(paths.baseDir, 'registration-held'))); + legacy = spawnLegacyDaemonFixture(paths); + await legacy.exited; + assert.equal(legacyDisposition(paths), false); + const claim = inspectProcessLock(paths.lockPath); + assert.equal(claim.state, 'held'); + if (claim.state !== 'held') throw new Error('current owner lost its claim'); + assert.equal(claim.owner.pid, current.pid); + assert.equal(process.kill(current.pid, 0), true); + assert.equal(fs.existsSync(paths.infoPath), false); + fs.unlinkSync(deferred); + await waitForCutoverFixture(() => fs.existsSync(paths.infoPath)); + assert.equal(JSON.parse(fs.readFileSync(paths.infoPath, 'utf8')).pid, current.pid); + } finally { + await legacy?.stop(); + await finishRegisteredDaemonFixture(paths.baseDir); + } +}); + +test('concurrent old and new daemon startup has one owner at the shared lock path', async () => { + const paths = resolveDaemonPaths(mkdtempForTestSync('agent-device-cutover-race-')); + const barrier = path.join(paths.baseDir, 'start'); + const legacy = spawnLegacyDaemonFixture(paths, barrier); + const current = spawnRegisteredDaemonFixture(paths, fields, undefined, barrier); + let currentExited = false; + void current.exited.then(() => { + currentExited = true; + }); + try { + await waitForCutoverFixture( + () => + fs.existsSync(`${barrier}.ready-${legacy.pid}`) && + fs.existsSync(`${barrier}.ready-${current.pid}`), + ); + fs.writeFileSync(barrier, 'start'); + await waitForCutoverFixture( + () => + fs.existsSync(path.join(paths.baseDir, 'legacy-disposition.json')) && + (currentExited || fs.existsSync(path.join(paths.baseDir, 'registration-held'))), + ); + const oldAcquired = legacyDisposition(paths); + const claim = inspectProcessLock(paths.lockPath); + const newAcquired = fs.existsSync(path.join(paths.baseDir, 'registration-held')); + assert.equal(Number(oldAcquired) + Number(newAcquired), 1); + if (newAcquired) { + assert.ok(claim.state === 'held'); + assert.equal(claim.owner.pid, current.pid); + } + if (oldAcquired) + assert.equal((await current.exited).exitCode, DAEMON_STARTUP_EXIT_CODES.unproven); + else await legacy.exited; + await waitForCutoverFixture(() => fs.existsSync(paths.infoPath)); + assert.equal( + JSON.parse(fs.readFileSync(paths.infoPath, 'utf8')).pid, + newAcquired ? current.pid : legacy.pid, + ); + } finally { + await legacy.stop(); + await finishRegisteredDaemonFixture(paths.baseDir); + } +}); + +for (const mode of ['graceful', 'forced'] as const) { + test(`manual ${mode} stop awaits actual child exit and protected registration retirement`, async () => { + const paths = resolveDaemonPaths(mkdtempForTestSync('agent-device-manual-stop-')); + if (mode === 'forced') fs.writeFileSync(path.join(paths.baseDir, 'ignore-sigterm'), 'hold'); + const child = spawnRegisteredDaemonFixture(paths, fields, undefined); + try { + await waitForFixtureFile(paths.infoPath); + const result = await stopDaemon({ paths, graceTimeoutMs: 30, killTimeoutMs: 1_000 }); + assert.equal(result.stopped, true); + assert.equal(result.mode, mode); + assert.equal(result.cleanupConfidence, mode === 'forced' ? 'unknown' : 'known'); + await child.exited; + assert.equal(fs.existsSync(paths.infoPath), false); + assert.equal(fs.existsSync(paths.lockPath), false); + assert.equal(fs.existsSync(paths.baseDir), true); + } finally { + await finishRegisteredDaemonFixture(paths.baseDir); + } + }); +} diff --git a/src/__tests__/test-utils/legacy-daemon-fixture.ts b/src/__tests__/test-utils/legacy-daemon-fixture.ts new file mode 100644 index 0000000000..b5b40ff06a --- /dev/null +++ b/src/__tests__/test-utils/legacy-daemon-fixture.ts @@ -0,0 +1,92 @@ +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import path from 'node:path'; +import { runCmdDetachedMonitored } from '@agent-device/host-kit/command'; +import { readProcessStartTime } from '@agent-device/host-kit/process'; +import { stopDaemonProcess } from '../../daemon-process.ts'; +import type { DaemonPaths } from '../../daemon-resolution.ts'; + +// c237027737: server-lifecycle.ts readLockInfo/acquireDaemonLock/releaseDaemonLock. +const legacyLockProtocol = ` +function readLockInfo(lockPath) { + if (!fs.existsSync(lockPath)) return null; + try { + const parsed = JSON.parse(fs.readFileSync(lockPath, 'utf8')); + if (!Number.isInteger(parsed.pid) || parsed.pid <= 0) return null; + return parsed; + } catch { + return null; + } +} +function acquireDaemonLock(baseDir, lockPath, lockData) { + if (!fs.existsSync(baseDir)) fs.mkdirSync(baseDir, { recursive: true }); + const payload = JSON.stringify(lockData, null, 2); + const tryWriteLock = () => { + try { + fs.writeFileSync(lockPath, payload, { flag: 'wx', mode: 0o600 }); + return true; + } catch (error) { + if (error.code === 'EEXIST') return false; + throw error; + } + }; + if (tryWriteLock()) return true; + const existing = readLockInfo(lockPath); + if (existing?.pid && existing.pid !== process.pid && + isAgentDeviceDaemonProcess(existing.pid, existing.processStartTime)) return false; + try { fs.unlinkSync(lockPath); } catch {} + return tryWriteLock(); +} +function releaseDaemonLock(lockPath) { + const existing = readLockInfo(lockPath); + if (existing && existing.pid !== process.pid) return; + try { if (fs.existsSync(lockPath)) fs.unlinkSync(lockPath); } catch {} +} +`; + +export function spawnLegacyDaemonFixture(paths: DaemonPaths, acquisitionBarrier?: string) { + const codeDir = path.join(paths.baseDir, 'legacy'); + const entry = path.join(codeDir, 'dist', 'src', 'internal', 'daemon.js'); + fs.mkdirSync(path.dirname(entry), { recursive: true }); + fs.writeFileSync(path.join(codeDir, 'package.json'), '{"type":"module"}'); + const processUrl = new URL('../../daemon-process.ts', import.meta.url).href; + const hostProcessUrl = new URL('../../../packages/host-kit/src/process.ts', import.meta.url).href; + fs.writeFileSync( + entry, + ` +import fs from 'node:fs'; +import { isAgentDeviceDaemonProcess } from ${JSON.stringify(processUrl)}; +import { readProcessStartTime } from ${JSON.stringify(hostProcessUrl)}; +${legacyLockProtocol} +const paths = ${JSON.stringify(paths)}; +const barrier = ${JSON.stringify(acquisitionBarrier)}; +if (barrier) { + fs.writeFileSync(barrier + '.ready-' + process.pid, 'ready'); + while (!fs.existsSync(barrier)) await new Promise(resolve => setTimeout(resolve, 10)); +} +const identity = { pid: process.pid, processStartTime: readProcessStartTime(process.pid) }; +const acquired = acquireDaemonLock(paths.baseDir, paths.lockPath, { + ...identity, version: '0.21.20', startedAt: Date.now(), +}); +fs.writeFileSync(paths.baseDir + '/legacy-disposition.json', JSON.stringify({ acquired })); +if (!acquired) process.exit(0); +fs.writeFileSync(paths.infoPath, JSON.stringify({ ...identity, port: 4210, token: 'legacy-token', version: '0.21.20' })); +process.on('SIGTERM', () => { releaseDaemonLock(paths.lockPath); process.exit(0); }); +setInterval(() => {}, 1000); +`, + ); + const child = runCmdDetachedMonitored(process.execPath, ['--experimental-strip-types', entry]); + const startTime = readProcessStartTime(child.pid); + return { + pid: child.pid, + exited: child.exited, + async stop() { + const result = await stopDaemonProcess( + { pid: child.pid, startTime }, + { mode: 'force', termTimeoutMs: 0, killTimeoutMs: 2_000 }, + ); + assert.notEqual(result.status, 'retained', JSON.stringify(result)); + await child.exited; + }, + }; +} diff --git a/src/__tests__/test-utils/registered-daemon-fixture.ts b/src/__tests__/test-utils/registered-daemon-fixture.ts index 12af382c10..ea80ae08b6 100644 --- a/src/__tests__/test-utils/registered-daemon-fixture.ts +++ b/src/__tests__/test-utils/registered-daemon-fixture.ts @@ -20,6 +20,7 @@ const children = new Map< export function registeredDaemonFixtureArgs( paths: DaemonPaths, fields: DaemonRegistrationFields, + acquisitionBarrier?: string, ): string[] { const entry = path.join(paths.baseDir, 'dist', 'src', 'internal', 'daemon.js'); fs.mkdirSync(path.dirname(entry), { recursive: true }); @@ -31,9 +32,15 @@ export function registeredDaemonFixtureArgs( import path from 'node:path'; import { DAEMON_STARTUP_EXIT_CODES, tryAcquireDaemonRegistration } from ${JSON.stringify(registrationUrl)}; const paths = ${JSON.stringify(paths)}; +const barrier = ${JSON.stringify(acquisitionBarrier)}; +if (barrier) { + fs.writeFileSync(barrier + '.ready-' + process.pid, 'ready'); + while (!fs.existsSync(barrier)) await new Promise(resolve => setTimeout(resolve, 10)); +} const acquired = await tryAcquireDaemonRegistration(paths); if (acquired.status !== 'acquired') process.exit(DAEMON_STARTUP_EXIT_CODES[acquired.status]); process.on('SIGTERM', async () => { + if (fs.existsSync(path.join(paths.baseDir, 'ignore-sigterm'))) return; const deferred = path.join(paths.baseDir, 'repair-on-shutdown.json'); if (fs.existsSync(deferred)) { const dir = path.join(paths.sessionsDir, 'default'); @@ -56,10 +63,11 @@ export function spawnRegisteredDaemonFixture( paths: DaemonPaths, fields: DaemonRegistrationFields, options: Parameters[2], + acquisitionBarrier?: string, ): ReturnType { const child = actualCommand.runCmdDetachedMonitored( process.execPath, - registeredDaemonFixtureArgs(paths, fields), + registeredDaemonFixtureArgs(paths, fields, acquisitionBarrier), options, ); const owned = children.get(paths.baseDir) ?? []; diff --git a/src/daemon/__tests__/daemon-stop.test.ts b/src/daemon/__tests__/daemon-stop.test.ts index 8a294910b6..13ac6851d4 100644 --- a/src/daemon/__tests__/daemon-stop.test.ts +++ b/src/daemon/__tests__/daemon-stop.test.ts @@ -2,15 +2,9 @@ import fs from 'node:fs'; import { afterEach, expect, test, vi } from 'vitest'; import { mkdtempForTestSync } from '../../__tests__/test-utils/tmp-dir.ts'; -const mocks = vi.hoisted(() => ({ - stopDaemonProcess: vi.fn(), - sleep: vi.fn(async () => undefined), -})); - -vi.mock('../../daemon-process.ts', () => ({ stopDaemonProcess: mocks.stopDaemonProcess })); -vi.mock('@agent-device/host-kit/retry', async (importOriginal) => ({ - ...(await importOriginal()), - sleep: mocks.sleep, +const mocks = vi.hoisted(() => ({ stopAndRetireDaemon: vi.fn() })); +vi.mock('../../daemon-registration-owner.ts', () => ({ + stopAndRetireDaemon: mocks.stopAndRetireDaemon, })); import { resolveDaemonPaths } from '../../daemon-resolution.ts'; @@ -45,40 +39,34 @@ test('reports not-running when daemon metadata is absent', async () => { test('retained identity verification is reported as failure without known cleanup', async () => { const paths = createDaemonPaths(); - mocks.stopDaemonProcess.mockResolvedValue({ status: 'retained', reason: 'identity-unverified' }); + mocks.stopAndRetireDaemon.mockResolvedValue(retainedExit('identity-unverified')); await expect(stopDaemon({ paths })).rejects.toMatchObject({ code: 'COMMAND_FAILED', details: { reason: 'daemon_exit_unconfirmed', terminationReason: 'identity-unverified' }, }); - expect(mocks.stopDaemonProcess).toHaveBeenCalledWith( - { pid: 123, startTime: 'start-time' }, - { - mode: 'graceful', - termTimeoutMs: 10_000, - killTimeoutMs: 2_000, - }, - ); + expect(mocks.stopAndRetireDaemon).toHaveBeenCalledWith({ + paths, + observed: { pid: 123, startTime: 'start-time' }, + mode: 'graceful', + termTimeoutMs: 10_000, + killTimeoutMs: 2_000, + }); }); test('missing start-time identity is passed to the owning termination operation', async () => { const paths = createDaemonPaths(); fs.writeFileSync(paths.infoPath, JSON.stringify({ pid: 123, processStartTime: ' ' })); - mocks.stopDaemonProcess.mockResolvedValue({ status: 'retained', reason: 'missing-start-time' }); + mocks.stopAndRetireDaemon.mockResolvedValue(retainedExit('missing-start-time')); await expect(stopDaemon({ paths })).rejects.toMatchObject({ details: { terminationReason: 'missing-start-time' }, }); - expect(mocks.stopDaemonProcess).toHaveBeenCalledWith( - { pid: 123, startTime: null }, - expect.anything(), + expect(mocks.stopAndRetireDaemon).toHaveBeenCalledWith( + expect.objectContaining({ paths, observed: { pid: 123, startTime: null } }), ); }); test('a previously exited verified lifetime is reported as not-running', async () => { - mocks.stopDaemonProcess.mockResolvedValue({ - status: 'exited', - mode: 'already-exited', - identity: { pid: 123, startTime: 'start-time' }, - }); + mocks.stopAndRetireDaemon.mockResolvedValue(retired('already-exited')); expect(await stopDaemon({ paths: createDaemonPaths() })).toMatchObject({ stopped: false, mode: 'not-running', @@ -86,8 +74,15 @@ test('a previously exited verified lifetime is reported as not-running', async ( }); test('an already released pid without start time remains not-running without cleanup proof', async () => { - mocks.stopDaemonProcess.mockResolvedValue({ status: 'not-running' }); - expect(await stopDaemon({ paths: createDaemonPaths() })).toMatchObject({ + const paths = createDaemonPaths(); + fs.writeFileSync(paths.infoPath, JSON.stringify({ pid: 123 })); + mocks.stopAndRetireDaemon.mockResolvedValue({ + status: 'retained', + reason: 'exit-unconfirmed', + removedInfo: false, + termination: { status: 'not-running' }, + }); + expect(await stopDaemon({ paths })).toMatchObject({ stopped: false, mode: 'not-running', }); @@ -95,32 +90,24 @@ test('an already released pid without start time remains not-running without cle test('confirmed TERM exit preserves graceful report behavior and configured budgets', async () => { const paths = createDaemonPaths(); - mocks.stopDaemonProcess.mockImplementation(async () => { - fs.rmSync(paths.infoPath, { force: true }); - return { status: 'exited', mode: 'graceful', identity: { pid: 123, startTime: 'start-time' } }; - }); + mocks.stopAndRetireDaemon.mockResolvedValue(retired('graceful')); expect(await stopDaemon({ paths, graceTimeoutMs: 11, killTimeoutMs: 7 })).toMatchObject({ stopped: true, mode: 'graceful', cleanupConfidence: 'known', providerReleases: { pending: [] }, }); - expect(mocks.stopDaemonProcess).toHaveBeenCalledWith( - { pid: 123, startTime: 'start-time' }, - { - mode: 'graceful', - termTimeoutMs: 11, - killTimeoutMs: 7, - }, - ); + expect(mocks.stopAndRetireDaemon).toHaveBeenCalledWith({ + paths, + observed: { pid: 123, startTime: 'start-time' }, + mode: 'graceful', + termTimeoutMs: 11, + killTimeoutMs: 7, + }); }); test('confirmed KILL exit preserves unknown provider cleanup', async () => { - mocks.stopDaemonProcess.mockResolvedValue({ - status: 'exited', - mode: 'forced', - identity: { pid: 123, startTime: 'start-time' }, - }); + mocks.stopAndRetireDaemon.mockResolvedValue(retired('forced')); expect(await stopDaemon({ paths: createDaemonPaths() })).toMatchObject({ stopped: true, mode: 'forced', @@ -133,10 +120,58 @@ test('confirmed KILL exit preserves unknown provider cleanup', async () => { test.each(['signal-failed', 'exit-timeout'])( '%s cannot become a successful stop', async (reason) => { - mocks.stopDaemonProcess.mockResolvedValue({ status: 'retained', reason }); + mocks.stopAndRetireDaemon.mockResolvedValue(retainedExit(reason)); await expect(stopDaemon({ paths: createDaemonPaths() })).rejects.toMatchObject({ code: 'COMMAND_FAILED', details: { reason: 'daemon_exit_unconfirmed', terminationReason: reason }, }); }, ); + +function retainedExit(reason: string) { + return { + status: 'retained', + reason: 'exit-unconfirmed', + removedInfo: false, + termination: { status: 'retained', reason }, + }; +} + +function retired(mode: string) { + return { + status: 'retired', + removedInfo: true, + termination: { status: 'exited', mode, identity: { pid: 123, startTime: 'start-time' } }, + }; +} + +test.each(['registration-replaced', 'retirement-unconfirmed'])( + '%s after confirmed exit cannot report a completed retirement', + async (reason) => { + const paths = createDaemonPaths(); + mocks.stopAndRetireDaemon.mockResolvedValue({ + ...retired('forced'), + status: 'retained', + reason, + removedInfo: false, + error: { + code: 'UNKNOWN', + message: 'retained', + hint: 'Inspect retained state.', + diagnosticId: 'diag-retire', + logPath: '/retained/daemon.log', + }, + }); + await expect(stopDaemon({ paths })).rejects.toMatchObject({ + code: 'COMMAND_FAILED', + details: { + reason: 'daemon_retirement_unconfirmed', + retirement: { status: 'retained', reason }, + hint: 'Inspect retained state.', + diagnosticId: 'diag-retire', + logPath: '/retained/daemon.log', + }, + }); + expect(fs.existsSync(paths.infoPath)).toBe(true); + }, +); diff --git a/src/daemon/daemon-stop.ts b/src/daemon/daemon-stop.ts index bf5be1697b..0f1ae4e1d0 100644 --- a/src/daemon/daemon-stop.ts +++ b/src/daemon/daemon-stop.ts @@ -1,7 +1,6 @@ -import fs from 'node:fs'; import { AppError } from '@agent-device/kernel/errors'; -import { stopDaemonProcess } from '../daemon-process.ts'; -import { sleep } from '@agent-device/host-kit/retry'; +import { stopAndRetireDaemon, type DaemonRetirementResult } from '../daemon-registration-owner.ts'; +import type { OwnerIdentity } from '@agent-device/host-kit/process'; import type { DaemonPaths } from '../daemon-resolution.ts'; import { readRegisteredDaemonIdentity } from '../daemon-registration.ts'; @@ -9,7 +8,6 @@ import type { DeviceClaimRecord, ProviderReleaseRecord } from '../daemon-shutdow const DAEMON_STOP_GRACE_TIMEOUT_MS = 10_000; const DAEMON_STOP_KILL_TIMEOUT_MS = 2_000; -const DAEMON_STOP_METADATA_WAIT_MS = 1_000; export type DaemonStopResult = { stopped: boolean; @@ -45,25 +43,18 @@ export async function stopDaemon(params: { }): Promise { const info = readRegisteredDaemonIdentity(params.paths.infoPath); if (!info) return notRunningResult(); - const termination = await stopDaemonProcess(info, { + const retirement = await stopAndRetireDaemon({ + paths: params.paths, + observed: info, mode: 'graceful', termTimeoutMs: params.graceTimeoutMs ?? DAEMON_STOP_GRACE_TIMEOUT_MS, killTimeoutMs: params.killTimeoutMs ?? DAEMON_STOP_KILL_TIMEOUT_MS, }); - if (termination.status === 'retained') { - throw new AppError('COMMAND_FAILED', 'Daemon termination could not be confirmed.', { - pid: info.pid, - processStartTime: info.startTime, - reason: 'daemon_exit_unconfirmed', - terminationReason: termination.reason, - signal: termination.signal, - }); - } - if (termination.status === 'not-running' || termination.mode === 'already-exited') { + if (retirement.status === 'retained' && retirement.termination?.status === 'not-running') return notRunningResult(); - } - if (termination.mode === 'graceful') { - await waitForDaemonMetadataRemoval(params.paths, DAEMON_STOP_METADATA_WAIT_MS); + if (retirement.status !== 'retired') throw daemonRetirementError(info, retirement); + if (retirement.termination.mode === 'already-exited') return notRunningResult(); + if (retirement.termination.mode === 'graceful') { return { stopped: true, mode: 'graceful', @@ -92,6 +83,27 @@ export async function stopDaemon(params: { }; } +function daemonRetirementError( + info: OwnerIdentity, + retirement: Exclude, +): AppError { + const termination = retirement.status === 'retained' ? retirement.termination : undefined; + const failure = termination?.status === 'retained' ? termination : undefined; + const error = retirement.status === 'retained' ? retirement.error : undefined; + const { hint, diagnosticId, logPath } = error ?? {}; + return new AppError('COMMAND_FAILED', 'Daemon retirement could not be confirmed.', { + pid: info.pid, + processStartTime: info.startTime, + reason: failure ? 'daemon_exit_unconfirmed' : 'daemon_retirement_unconfirmed', + terminationReason: failure?.reason, + signal: failure?.signal, + retirement, + hint, + diagnosticId, + logPath, + }); +} + export function readDaemonStopIdentity( infoPath: string, ): { pid: number; processStartTime: string } | null { @@ -100,14 +112,6 @@ export function readDaemonStopIdentity( return { pid: info.pid, processStartTime: info.startTime }; } -async function waitForDaemonMetadataRemoval(paths: DaemonPaths, timeoutMs: number): Promise { - const startedAt = Date.now(); - while (Date.now() - startedAt < timeoutMs) { - if (!fs.existsSync(paths.infoPath) && !fs.existsSync(paths.lockPath)) return; - await sleep(25); - } -} - function notRunningResult(): DaemonStopResult { return { stopped: false, From 12781ae98f49f01d909e6ea0e74d678d9744a80d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sat, 3 Oct 2026 04:27:25 +0200 Subject: [PATCH 13/20] test: bound legacy contender admission observation --- src/__tests__/daemon-registration-owner.test.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/__tests__/daemon-registration-owner.test.ts b/src/__tests__/daemon-registration-owner.test.ts index 47710d6a62..5426a07764 100644 --- a/src/__tests__/daemon-registration-owner.test.ts +++ b/src/__tests__/daemon-registration-owner.test.ts @@ -542,8 +542,11 @@ test('a legacy contender cannot unlink a hardened owner while it delays metadata try { await waitForCutoverFixture(() => fs.existsSync(path.join(paths.baseDir, 'registration-held'))); legacy = spawnLegacyDaemonFixture(paths); - await legacy.exited; + await waitForCutoverFixture(() => + fs.existsSync(path.join(paths.baseDir, 'legacy-disposition.json')), + ); assert.equal(legacyDisposition(paths), false); + await legacy.exited; const claim = inspectProcessLock(paths.lockPath); assert.equal(claim.state, 'held'); if (claim.state !== 'held') throw new Error('current owner lost its claim'); From 464176df9497e2edd515d7b0a14c485b47b8502a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sat, 3 Oct 2026 05:08:22 +0200 Subject: [PATCH 14/20] fix: preserve daemon ownership across startup and cleanup --- .../internal/owner-identity-liveness.test.ts | 8 ++ .../host-kit/src/internal/owner-identity.ts | 11 ++- .../host-kit/src/internal/process-lock.ts | 3 +- src/__tests__/daemon-exit-wait.test.ts | 10 +++ src/__tests__/daemon-process-takeover.test.ts | 13 ++++ .../test-utils/daemon-http-fixture.ts | 3 +- .../test-utils/registered-daemon-fixture.ts | 21 ++++- .../__tests__/daemon-client-lifecycle.test.ts | 9 +-- .../daemon-client-startup-race.test.ts | 49 +++++++++++- .../daemon-client-timeout-route.test.ts | 75 ++++++++++++------ src/daemon-client/daemon-client-lifecycle.ts | 11 ++- src/daemon-client/daemon-client-metadata.ts | 3 +- src/daemon-client/daemon-client-timeout.ts | 15 ++-- src/daemon-process.ts | 3 + src/daemon-registration.ts | 3 +- src/daemon/daemon-stop.ts | 3 +- .../__tests__/session-device-claims.test.ts | 3 - .../support/daemon-test-cleanup.test.ts | 77 +++++++++++++++++++ .../support/daemon-test-cleanup.ts | 47 ++++++----- 19 files changed, 298 insertions(+), 69 deletions(-) create mode 100644 test/integration/support/daemon-test-cleanup.test.ts diff --git a/packages/host-kit/src/internal/owner-identity-liveness.test.ts b/packages/host-kit/src/internal/owner-identity-liveness.test.ts index fd5278afa8..772d29cac5 100644 --- a/packages/host-kit/src/internal/owner-identity-liveness.test.ts +++ b/packages/host-kit/src/internal/owner-identity-liveness.test.ts @@ -59,3 +59,11 @@ test('a missing entry in a completed process snapshot stays fail-closed without assert.equal(mockIsProcessZombie.mock.calls.length, 0); assert.equal(mockReadProcessStartTime.mock.calls.length, 0); }); + +test('a pid outside the native range is unknown without a liveness probe', () => { + assert.equal( + classifyOwnerLiveness({ owner: { pid: 2_147_483_648, startTime: 'start-a' } }), + 'unknown', + ); + assert.equal(mockIsProcessAlive.mock.calls.length, 0); +}); diff --git a/packages/host-kit/src/internal/owner-identity.ts b/packages/host-kit/src/internal/owner-identity.ts index a7b1352b35..552a030ea7 100644 --- a/packages/host-kit/src/internal/owner-identity.ts +++ b/packages/host-kit/src/internal/owner-identity.ts @@ -11,6 +11,11 @@ export type OwnerIdentity = { startTime: string | null; }; +/** A process id accepted by Node's native signal API. */ +export function isProcessPid(value: unknown): value is number { + return typeof value === 'number' && Number.isInteger(value) && value > 0 && value <= 0x7fff_ffff; +} + export type OwnerLiveness = | 'live' | 'owner-process-dead' @@ -75,6 +80,7 @@ export function classifyOwnerLivenessFromObservation( observation?: HostProcessIdentityObservation | null, ): OwnerLiveness { const { owner, stateDir } = params; + if (!isProcessPid(owner.pid)) return 'unknown'; if (!isProcessAlive(owner.pid)) return 'owner-process-dead'; if (observation !== undefined ? observation?.state.startsWith('Z') : isProcessZombie(owner.pid)) { return 'owner-process-dead'; @@ -88,7 +94,10 @@ export function classifyOwnerLivenessFromObservation( return 'owner-process-reused'; } } - if (!stateDir) return 'live'; + return stateDir ? classifyOwnerStateDirectory(stateDir) : 'live'; +} + +function classifyOwnerStateDirectory(stateDir: string): OwnerLiveness { try { fs.statSync(stateDir); return 'live'; diff --git a/packages/host-kit/src/internal/process-lock.ts b/packages/host-kit/src/internal/process-lock.ts index 989b28a78d..3108c208e1 100644 --- a/packages/host-kit/src/internal/process-lock.ts +++ b/packages/host-kit/src/internal/process-lock.ts @@ -6,6 +6,7 @@ import { publishFileSync } from './atomic-file.ts'; import { emitDiagnostic } from './diagnostics.ts'; import { classifyOwnerLiveness, + isProcessPid, ownerIdentityMatches, type OwnerLiveness, } from './owner-identity.ts'; @@ -553,7 +554,7 @@ function parseProcessLockOwner(contents: string): ProcessLockOwnerRecord | null const PROCESS_LOCK_OWNER_FIELD_SHAPES: Record boolean> = { - pid: (value) => typeof value === 'number' && Number.isInteger(value) && value > 0, + pid: isProcessPid, acquiredAtMs: (value) => typeof value === 'number' && Number.isFinite(value), startTime: (value) => value === undefined || value === null || typeof value === 'string', }; diff --git a/src/__tests__/daemon-exit-wait.test.ts b/src/__tests__/daemon-exit-wait.test.ts index 2111bc2f3b..f2840761a1 100644 --- a/src/__tests__/daemon-exit-wait.test.ts +++ b/src/__tests__/daemon-exit-wait.test.ts @@ -57,6 +57,16 @@ afterEach(() => { vi.restoreAllMocks(); }); +test.each([0, -1, 1.5, 2_147_483_648, Number.MAX_SAFE_INTEGER])( + 'an invalid native pid %s cannot prove exit during recovery', + async (pid) => { + expect(await waitForDaemonExit({ pid, startTime: OURS }, { timeoutMs: 0 })).toEqual({ + exited: false, + elapsedMs: 0, + }); + }, +); + test('waitForDaemonExit reports a pid recycled mid-wait as exited, without burning the deadline', async () => { setTimeout(() => state.starts.set(PID, RECYCLED), 20); const wait = await waitForDaemonExit( diff --git a/src/__tests__/daemon-process-takeover.test.ts b/src/__tests__/daemon-process-takeover.test.ts index 78156ffc64..99292988f6 100644 --- a/src/__tests__/daemon-process-takeover.test.ts +++ b/src/__tests__/daemon-process-takeover.test.ts @@ -12,6 +12,19 @@ const TAKEOVER_TIMEOUTS = { termTimeoutMs: 5_000, killTimeoutMs: 2_000 }; const spawnedChildren: { child: ChildProcess; exited: Promise }[] = []; const spawnedRoots: string[] = []; +test.each([0, -1, 1.5, Number.NaN, '123', 2_147_483_648, Number.MAX_SAFE_INTEGER])( + 'an invalid daemon pid %s cannot prove exit', + async (pid) => { + assert.deepEqual( + await stopDaemonProcess( + { pid: pid as number, startTime: 'captured-birth' }, + { mode: 'force', termTimeoutMs: 0, killTimeoutMs: 0 }, + ), + { status: 'retained', reason: 'identity-unverified' }, + ); + }, +); + afterEach(async () => { for (const { child, exited } of spawnedChildren.splice(0)) { if (child.exitCode === null && child.signalCode === null) child.kill('SIGKILL'); diff --git a/src/__tests__/test-utils/daemon-http-fixture.ts b/src/__tests__/test-utils/daemon-http-fixture.ts index 4712a049c6..485bba4e7e 100644 --- a/src/__tests__/test-utils/daemon-http-fixture.ts +++ b/src/__tests__/test-utils/daemon-http-fixture.ts @@ -16,6 +16,7 @@ export type HttpDaemonFixture = { export async function startHttpDaemonFixture( responseData: Record, + options: { ready?: () => boolean } = {}, ): Promise { const seenPaths: string[] = []; const rpcRequests: Record[] = []; @@ -24,7 +25,7 @@ export async function startHttpDaemonFixture( seenPaths.push(`${req.method ?? 'GET'} ${url.pathname}`); if (req.method === 'GET' && url.pathname === '/health') { - res.writeHead(200); + res.writeHead(options.ready?.() === false ? 503 : 200); res.end('ok'); return; } diff --git a/src/__tests__/test-utils/registered-daemon-fixture.ts b/src/__tests__/test-utils/registered-daemon-fixture.ts index ea80ae08b6..6c99455a4e 100644 --- a/src/__tests__/test-utils/registered-daemon-fixture.ts +++ b/src/__tests__/test-utils/registered-daemon-fixture.ts @@ -2,7 +2,8 @@ import assert from 'node:assert/strict'; import fs from 'node:fs'; import path from 'node:path'; import { vi } from 'vitest'; -import type { runCmdDetachedMonitored } from '@agent-device/host-kit/command'; +import type { runCmdDetachedMonitored, ExecDetachedExit } from '@agent-device/host-kit/command'; +import { readDaemonInfo, type DaemonInfo } from '../../daemon-client/daemon-client-metadata.ts'; import { readProcessStartTime } from '@agent-device/host-kit/process'; import { stopDaemonProcess } from '../../daemon-process.ts'; import type { DaemonPaths } from '../../daemon-resolution.ts'; @@ -76,6 +77,24 @@ export function spawnRegisteredDaemonFixture( return child; } +export async function waitForRegisteredDaemonFixture( + paths: DaemonPaths, + child: ReturnType, +): Promise { + let exit: ExecDetachedExit | undefined; + void child.exited.then((result) => { + exit = result; + }); + for (let attempt = 0; attempt < 400; attempt += 1) { + if (exit) + throw new Error(`Registered child exited before publication: ${JSON.stringify(exit)}`); + const info = readDaemonInfo(paths.infoPath); + if (info?.pid === child.pid) return info; + await new Promise((resolve) => setTimeout(resolve, 10)); + } + throw new Error(`Registered child ${child.pid} did not publish ${paths.infoPath} within 4s`); +} + export async function finishRegisteredDaemonFixture(stateDir: string): Promise { for (const owned of children.get(stateDir) ?? []) { const child = owned.launch; diff --git a/src/daemon-client/__tests__/daemon-client-lifecycle.test.ts b/src/daemon-client/__tests__/daemon-client-lifecycle.test.ts index 487df065d2..9c67e88a31 100644 --- a/src/daemon-client/__tests__/daemon-client-lifecycle.test.ts +++ b/src/daemon-client/__tests__/daemon-client-lifecycle.test.ts @@ -8,6 +8,7 @@ import { afterEach, test, vi } from 'vitest'; import { mkdtempForTestSync } from '../../__tests__/test-utils/tmp-dir.ts'; import { spawnRegisteredDaemonFixture, + waitForRegisteredDaemonFixture, finishRegisteredDaemonFixture, finishRegisteredDaemonFixtures, } from '../../__tests__/test-utils/registered-daemon-fixture.ts'; @@ -27,7 +28,6 @@ import { resolveDaemonPaths, type DaemonPaths } from '../../daemon-resolution.ts import { sendToDaemon, type DaemonRequest, type DaemonResponse } from '../daemon-client.ts'; import { attachActiveSessionAddressHint } from '../daemon-client-lifecycle.ts'; import { sendRequest } from '../daemon-client-transport.ts'; -import { readDaemonInfo } from '../daemon-client-metadata.ts'; import type { DaemonRetirementResult } from '../../daemon-registration-owner.ts'; import { closeLoopbackServer, @@ -646,12 +646,7 @@ test('sendRequest timeout cleanup uses resolved daemon paths instead of request }; try { - let info = readDaemonInfo(daemonPaths.infoPath); - for (let attempt = 0; !info && attempt < 200; attempt += 1) { - await actualRetry.sleep(10); - info = readDaemonInfo(daemonPaths.infoPath); - } - assert.ok(info); + const info = await waitForRegisteredDaemonFixture(daemonPaths, child); let thrown: unknown; try { await sendRequest(info, request, 'http', daemonPaths, 50); diff --git a/src/daemon-client/__tests__/daemon-client-startup-race.test.ts b/src/daemon-client/__tests__/daemon-client-startup-race.test.ts index 7991d75c32..8788a8c887 100644 --- a/src/daemon-client/__tests__/daemon-client-startup-race.test.ts +++ b/src/daemon-client/__tests__/daemon-client-startup-race.test.ts @@ -203,6 +203,46 @@ for (const exit of [ }); } +test('a joined busy contender waits for a published winner to become ready without signaling it', async (t) => { + if (!(await supportsLoopbackBind())) return t.skip('loopback unavailable'); + const paths = resolveDaemonPaths(mkdtempForTestSync('daemon-start-delayed-ready-')); + let probes = 0; + const http = await startHttpDaemonFixture({ devices: [] }, { ready: () => ++probes >= 12 }); + const deferred = path.join(paths.baseDir, 'defer-publication'); + fs.writeFileSync(deferred, 'wait'); + const winner = spawnRegisteredDaemonFixture(paths, fields(http.port), { stdio: 'ignore' }); + await awaitFile(path.join(paths.baseDir, 'registration-held')); + let joined = false; + spawn.mockImplementation((_command, _args, options) => { + const child = spawnRegisteredDaemonFixture(paths, fields(http.port), options); + void child.exited.then((exit) => { + assert.equal(exit.exitCode, DAEMON_STARTUP_EXIT_CODES.busy); + joined = true; + }); + return child; + }); + pause.mockImplementation(async () => { + if (joined) fs.rmSync(deferred, { force: true }); + await actualRetry.sleep(10); + }); + const signal = vi.spyOn(process, 'kill'); + try { + assert.equal((await sendToDaemon(request(paths))).ok, true); + assert.equal(joined, true); + assert.ok(probes >= 12); + assert.equal(spawn.mock.calls.length, 1); + assert.equal(http.rpcRequests.length, 1); + assert.equal(isProcessAlive(winner.pid), true); + assert.equal( + signal.mock.calls.some(([pid, kind]) => pid === winner.pid && kind !== 0), + false, + ); + } finally { + signal.mockRestore(); + await closeLoopbackServer(http.server); + } +}); + for (const held of [true, false]) { test(`startup uses one deadline when the claim is ${held ? 'held' : 'released for relaunch'}`, async () => { const paths = resolveDaemonPaths(mkdtempForTestSync('daemon-start-budget-')); @@ -295,7 +335,14 @@ test.for([ const notice = vi.spyOn(process.stderr, 'write').mockImplementation(() => true); try { const pending = sendToDaemon(request(paths)); - if (expected.alive) await assert.rejects(pending); + if (expected.alive) + await assert.rejects(pending, (error: unknown) => { + assert.ok(error instanceof AppError); + assert.equal(error.details?.kind, 'daemon_startup_failed'); + assert.equal(error.details?.startupAttempts, 1); + assert.equal(error.details?.startupTimeoutMs, 15_000); + return true; + }); else { assert.equal((await pending).ok, true); await winner.exited; diff --git a/src/daemon-client/__tests__/daemon-client-timeout-route.test.ts b/src/daemon-client/__tests__/daemon-client-timeout-route.test.ts index 04d4154403..c28d397d1d 100644 --- a/src/daemon-client/__tests__/daemon-client-timeout-route.test.ts +++ b/src/daemon-client/__tests__/daemon-client-timeout-route.test.ts @@ -41,14 +41,16 @@ vi.mock('@agent-device/host-kit/command', async () => { import { AppError, normalizeError } from '@agent-device/kernel/errors'; import { sleep } from '@agent-device/host-kit/retry'; +import { withDiagnosticsScope } from '@agent-device/host-kit/diagnostics'; import { sendRequest } from '../daemon-client-transport.ts'; import type { DaemonRequest } from '../../daemon/daemon-request.ts'; -import { readDaemonInfo, type DaemonInfo } from '../daemon-client-metadata.ts'; +import type { DaemonInfo } from '../daemon-client-metadata.ts'; import { resolveDaemonPaths, type DaemonPaths } from '../../daemon-resolution.ts'; import type { DaemonRetirementResult } from '../../daemon-registration-owner.ts'; import { mkdtempForTestSync } from '../../__tests__/test-utils/tmp-dir.ts'; import { spawnRegisteredDaemonFixture, + waitForRegisteredDaemonFixture, finishRegisteredDaemonFixture, finishRegisteredDaemonFixtures, } from '../../__tests__/test-utils/registered-daemon-fixture.ts'; @@ -284,13 +286,29 @@ test('remote HTTP timeout never runs the Apple pkill cleanup and uses the remote assert.equal(mockRunCmdSync.mock.calls.length, 0); }); -async function publishedInfo(paths: DaemonPaths): Promise { - for (let attempt = 0; attempt < 200; attempt += 1) { - const info = readDaemonInfo(paths.infoPath); - if (info) return info; - await sleep(10); +async function waitForForceStop(requested: Promise): Promise { + let timer: ReturnType | undefined; + try { + await Promise.race([ + requested, + new Promise((_resolve, reject) => { + timer = setTimeout(() => reject(new Error('force stop was not requested')), 1_500); + }), + ]); + } finally { + clearTimeout(timer); } - throw new Error('registered child did not publish'); +} + +function timeoutDiagnostic(paths: DaemonPaths): Record { + const events = fs + .readFileSync(path.join(paths.baseDir, 'timeout-diagnostics.ndjson'), 'utf8') + .trim() + .split('\n') + .map((line) => JSON.parse(line)); + const event = events.find((entry) => entry.phase === 'daemon_request_timeout'); + assert.ok(event); + return event.data; } for (const transport of ['socket', 'http'] as const) { @@ -331,13 +349,17 @@ for (const transport of ['socket', 'http'] as const) { return actualKill(pid, signal); }); try { - const info = await publishedInfo(paths); - outcome = sendRequest( - info, - { ...buildRequest(undefined), command: 'open' }, - transport, - paths, - TIMEOUT_MS, + const info = await waitForRegisteredDaemonFixture(paths, child); + outcome = withDiagnosticsScope( + { debug: true, logPath: path.join(paths.baseDir, 'timeout-diagnostics.ndjson') }, + () => + sendRequest( + info, + { ...buildRequest(undefined), command: 'open' }, + transport, + paths, + TIMEOUT_MS, + ), ).then( () => assert.fail('hanging request unexpectedly succeeded'), (error: unknown) => { @@ -345,10 +367,7 @@ for (const transport of ['socket', 'http'] as const) { return error; }, ); - await Promise.race([ - requested, - sleep(1_500).then(() => assert.fail('force stop was not requested')), - ]); + await waitForForceStop(requested); await sleep(30); assert.equal(actualKill(child.pid, 0), true); assert.equal(settled, false, 'request must remain pending while the daemon is alive'); @@ -364,6 +383,7 @@ for (const transport of ['socket', 'http'] as const) { assert.equal(fs.existsSync(paths.infoPath), false); assert.equal(fs.existsSync(paths.lockPath), false); assert.equal(fs.existsSync(paths.baseDir), true); + assert.equal(timeoutDiagnostic(paths).daemonPreservedAfterTimeout, false); assert.equal(mockRunCmdSync.mock.calls.filter(([cmd]) => cmd === 'pkill').length, 3); } finally { kill.mockRestore(); @@ -394,18 +414,22 @@ test('timeout retains a live registration without captured birth proof and repor ); const kill = vi.spyOn(process, 'kill'); try { - const info = await publishedInfo(paths); + const info = await waitForRegisteredDaemonFixture(paths, child); const before = fs.readFileSync(paths.infoPath, 'utf8'); const lockBefore = fs .readdirSync(paths.lockPath) .map((name) => [name, fs.readFileSync(path.join(paths.lockPath, name), 'utf8')]); await assert.rejects( - sendRequest( - { ...info, processStartTime: undefined }, - { ...buildRequest(undefined), command: 'open' }, - 'http', - paths, - TIMEOUT_MS, + withDiagnosticsScope( + { debug: true, logPath: path.join(paths.baseDir, 'timeout-diagnostics.ndjson') }, + () => + sendRequest( + { ...info, processStartTime: undefined }, + { ...buildRequest(undefined), command: 'open' }, + 'http', + paths, + TIMEOUT_MS, + ), ), (error: unknown) => { assert.ok(error instanceof AppError); @@ -419,6 +443,7 @@ test('timeout retains a live registration without captured birth proof and repor return true; }, ); + assert.equal(timeoutDiagnostic(paths).daemonPreservedAfterTimeout, true); assert.equal(process.kill(child.pid, 0), true); assert.equal(fs.readFileSync(paths.infoPath, 'utf8'), before); assert.deepEqual( diff --git a/src/daemon-client/daemon-client-lifecycle.ts b/src/daemon-client/daemon-client-lifecycle.ts index 7b25dd630c..1d76d43ecf 100644 --- a/src/daemon-client/daemon-client-lifecycle.ts +++ b/src/daemon-client/daemon-client-lifecycle.ts @@ -190,13 +190,20 @@ async function ensureRemoteDaemon(settings: DaemonClientSettings): Promise { + const { deadline } = options; const inspection = inspectProcessLock(settings.paths.lockPath); if (inspection.state === 'unproven') throw daemonRegistrationUnprovenError(settings, inspection); const existing = readDaemonInfo(settings.paths.infoPath); if (!existing) return null; if (!registrationAllowsDaemonObservation(inspection, existing)) return null; + if ( + options.waitForLiveStartup && + isProcessAlive(existing.pid) && + !(await canConnect(existing, 'auto', remainingStartupBudget(deadline))) + ) + return null; const decision = await resolveDaemonTakeover(existing, { onClientTransport: () => @@ -769,7 +776,7 @@ async function observeContendingDaemon( ): Promise { let winner: DaemonInfo | null; try { - winner = await readReusableLocalDaemon(settings, deadline); + winner = await readReusableLocalDaemon(settings, { deadline, waitForLiveStartup: true }); } catch (error) { if (error instanceof AppError && error.details?.reason === 'daemon_registration_unproven') return { kind: 'unproven', error }; diff --git a/src/daemon-client/daemon-client-metadata.ts b/src/daemon-client/daemon-client-metadata.ts index d517da18e4..718ed7b1cc 100644 --- a/src/daemon-client/daemon-client-metadata.ts +++ b/src/daemon-client/daemon-client-metadata.ts @@ -1,4 +1,5 @@ import fs from 'node:fs'; +import { isProcessPid } from '@agent-device/host-kit/process'; import type { DaemonCodeOrigin } from '@agent-device/host-kit/code-signature'; @@ -42,7 +43,7 @@ export function readDaemonInfo(infoPath: string): DaemonInfo | null { token, ...ports, transport: readDaemonInfoTransport(parsed.transport), - pid: readPositiveInteger(parsed.pid) ?? 0, + pid: isProcessPid(parsed.pid) ? parsed.pid : 0, version: readOptionalString(parsed.version), codeOrigin: readDaemonInfoCodeOrigin(parsed.codeOrigin), codeSignature: readOptionalString(parsed.codeSignature), diff --git a/src/daemon-client/daemon-client-timeout.ts b/src/daemon-client/daemon-client-timeout.ts index c27c95b1b7..f7132c4f87 100644 --- a/src/daemon-client/daemon-client-timeout.ts +++ b/src/daemon-client/daemon-client-timeout.ts @@ -73,10 +73,7 @@ export async function handleRequestTimeout( mode: 'force', }); } - const forcedKill = - retirement?.status !== 'absent' && - retirement?.termination?.status === 'exited' && - retirement.termination.mode === 'forced'; + const retained = retirement?.status === 'retained'; // The HINT, unlike cleanup, may only name Apple-runner involvement on // evidence this call site actually has: an explicitly declared Apple // platform selector, or the cleanup itself having terminated a matching @@ -95,9 +92,9 @@ export async function handleRequestTimeout( timedOutRunnerPidsTerminated: cleanup.terminated, timedOutRunnerCleanupError: cleanup.error, daemonPidReset: retirement?.status === 'retired' ? info.pid : undefined, - daemonPidForceKilled: resetDaemon ? forcedKill : undefined, + daemonPidForceKilled: resetDaemon ? daemonWasForceKilled(retirement) : undefined, daemonRetirement: retirement, - daemonPreservedAfterTimeout: !remote && !resetDaemon, + daemonPreservedAfterTimeout: retained || (!remote && !resetDaemon), daemonBaseUrl: info.baseUrl, }, }); @@ -120,6 +117,12 @@ export async function handleRequestTimeout( }); } +function daemonWasForceKilled(retirement: DaemonRetirementResult | undefined): boolean { + if (!retirement || retirement.status === 'absent') return false; + const termination = retirement.termination; + return termination?.status === 'exited' && termination.mode === 'forced'; +} + // Whether a timed-out request tears down the local daemon is declared on the // command's descriptor (ADR 0008, `timeoutPolicy.onTimeout`): read-only // capture/polling commands preserve the daemon so sessions survive and evidence diff --git a/src/daemon-process.ts b/src/daemon-process.ts index fee88d3622..28869a01b5 100644 --- a/src/daemon-process.ts +++ b/src/daemon-process.ts @@ -1,5 +1,6 @@ import { isProcessAlive, + isProcessPid, readHostProcessIdentityObservations, readProcessCommand, readProcessStartTime, @@ -72,6 +73,7 @@ export async function waitForDaemonExit( identity: DaemonProcessIdentity, options: { timeoutMs: number; pollMs?: number }, ): Promise { + if (!isProcessPid(identity.pid)) return { exited: false, elapsedMs: 0 }; const startedAt = Date.now(); const deadline = startedAt + options.timeoutMs; const pollMs = options.pollMs ?? DAEMON_EXIT_POLL_MS; @@ -114,6 +116,7 @@ export async function stopDaemonProcess( killTimeoutMs: number; }, ): Promise { + if (!isProcessPid(observed.pid)) return { status: 'retained', reason: 'identity-unverified' }; if (!observed.startTime?.trim()) { if (!isProcessAlive(observed.pid)) return { status: 'not-running' }; return { status: 'retained', reason: 'missing-start-time' }; diff --git a/src/daemon-registration.ts b/src/daemon-registration.ts index ad598afc3c..22f892971b 100644 --- a/src/daemon-registration.ts +++ b/src/daemon-registration.ts @@ -1,6 +1,7 @@ import fs from 'node:fs'; import { ownerIdentityDiffers, + isProcessPid, ownerIdentityMatches, type OwnerIdentity, } from '@agent-device/host-kit/process'; @@ -53,7 +54,7 @@ function parseRegistration(parsed: { processStartTime?: unknown; }): ParsedRegistration { const pid = parsed.pid; - if (typeof pid !== 'number' || !Number.isInteger(pid) || pid <= 0) { + if (!isProcessPid(pid)) { return { pid: null, startTime: null }; } return { pid, startTime: readableStartTime(parsed.processStartTime) }; diff --git a/src/daemon/daemon-stop.ts b/src/daemon/daemon-stop.ts index 0f1ae4e1d0..c5e53eefcf 100644 --- a/src/daemon/daemon-stop.ts +++ b/src/daemon/daemon-stop.ts @@ -1,5 +1,5 @@ import { AppError } from '@agent-device/kernel/errors'; -import { stopAndRetireDaemon, type DaemonRetirementResult } from '../daemon-registration-owner.ts'; +import type { DaemonRetirementResult } from '../daemon-registration-owner.ts'; import type { OwnerIdentity } from '@agent-device/host-kit/process'; import type { DaemonPaths } from '../daemon-resolution.ts'; @@ -43,6 +43,7 @@ export async function stopDaemon(params: { }): Promise { const info = readRegisteredDaemonIdentity(params.paths.infoPath); if (!info) return notRunningResult(); + const { stopAndRetireDaemon } = await import('../daemon-registration-owner.ts'); const retirement = await stopAndRetireDaemon({ paths: params.paths, observed: info, diff --git a/src/daemon/handlers/__tests__/session-device-claims.test.ts b/src/daemon/handlers/__tests__/session-device-claims.test.ts index d45941b2c1..bb60b6ec2f 100644 --- a/src/daemon/handlers/__tests__/session-device-claims.test.ts +++ b/src/daemon/handlers/__tests__/session-device-claims.test.ts @@ -75,7 +75,6 @@ const mockResolveTargetDevice = vi.mocked(resolveTargetDevice); const mockEnsureDeviceReady = vi.mocked(ensureDeviceReady); const mockApplyRuntimeHints = vi.mocked(applyRuntimeHintValues); const mockResolveAndroidPackage = vi.mocked(resolveAndroidPackageForOpen); -const roots: string[] = []; const reconcileOrphanedDeviceClaim = async () => ({ status: 'retained' as const, reason: 'test-no-recovery', @@ -115,14 +114,12 @@ afterEach(() => { mockApplyRuntimeHints.mockResolvedValue(undefined); mockResolveAndroidPackage.mockResolvedValue(undefined); delete process.env.AGENT_DEVICE_CLAIMS_DIR; - for (const root of roots.splice(0)) fs.rmSync(root, { recursive: true, force: true }); }); function setup(): { store: SessionStore; stateDir: string } { const stateDir = mkdtempForTestSync('agent-device-session-device-claim-'); const claimsDir = path.join(stateDir, 'claims'); process.env.AGENT_DEVICE_CLAIMS_DIR = claimsDir; - roots.push(stateDir); return { store: new SessionStore(path.join(stateDir, 'sessions')), stateDir }; } diff --git a/test/integration/support/daemon-test-cleanup.test.ts b/test/integration/support/daemon-test-cleanup.test.ts new file mode 100644 index 0000000000..a83780c9d5 --- /dev/null +++ b/test/integration/support/daemon-test-cleanup.test.ts @@ -0,0 +1,77 @@ +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import { test, vi } from 'vitest'; +import { + isProcessAlive, + readHostProcessIdentityObservations, +} from '@agent-device/host-kit/process'; +import { cleanupDaemonTestState } from './daemon-test-cleanup.ts'; +import { resolveDaemonPaths } from '../../../src/daemon-resolution.ts'; +import { stopDaemonProcess } from '../../../src/daemon-process.ts'; +import { mkdtempForTestSync } from '../../../src/__tests__/test-utils/tmp-dir.ts'; +import { + spawnRegisteredDaemonFixture, + waitForRegisteredDaemonFixture, + finishRegisteredDaemonFixture, +} from '../../../src/__tests__/test-utils/registered-daemon-fixture.ts'; + +const fields = { + httpPort: 4210, + token: 'fixture', + version: 'test', + codeOrigin: 'checkout' as const, + codeSignature: 'fixture', +}; + +test.each(['string', 'out-of-range'])( + 'malformed %s registration retains the directory and live daemon', + async (kind) => { + const paths = resolveDaemonPaths(mkdtempForTestSync('daemon-test-invalid-registration-')); + const child = spawnRegisteredDaemonFixture(paths, fields, undefined); + const warnings = vi.spyOn(console, 'warn').mockImplementation(() => {}); + try { + const info = await waitForRegisteredDaemonFixture(paths, child); + const malformed = JSON.stringify({ + ...info, + pid: kind === 'string' ? String(info.pid) : 2_147_483_648, + }); + fs.writeFileSync(paths.infoPath, malformed); + await cleanupDaemonTestState(paths.baseDir, null); + assert.equal(fs.readFileSync(paths.infoPath, 'utf8'), malformed); + assert.equal(isProcessAlive(child.pid), true); + assert.equal(warnings.mock.calls.length, 1); + } finally { + warnings.mockRestore(); + await finishRegisteredDaemonFixture(paths.baseDir); + } + }, +); + +test('cleanup stops the registered replacement when its observation still names the exited original', async () => { + const paths = resolveDaemonPaths(mkdtempForTestSync('daemon-test-replaced-registration-')); + const original = spawnRegisteredDaemonFixture(paths, fields, undefined); + try { + const observed = await waitForRegisteredDaemonFixture(paths, original); + const stopped = await stopDaemonProcess( + { pid: original.pid, startTime: observed.processStartTime ?? null }, + { mode: 'force', termTimeoutMs: 0, killTimeoutMs: 1_000 }, + ); + assert.equal(stopped.status, 'exited'); + await original.exited; + const replacement = spawnRegisteredDaemonFixture(paths, fields, undefined); + await waitForRegisteredDaemonFixture(paths, replacement); + await cleanupDaemonTestState(paths.baseDir, observed); + assert.ok( + !isProcessAlive(replacement.pid) || + readHostProcessIdentityObservations([replacement.pid]) + .get(replacement.pid) + ?.state.startsWith('Z'), + 'the registered replacement must be dead before cleanup returns', + ); + await replacement.exited; + assert.equal(isProcessAlive(replacement.pid), false); + assert.equal(fs.existsSync(paths.baseDir), false); + } finally { + await finishRegisteredDaemonFixture(paths.baseDir); + } +}); diff --git a/test/integration/support/daemon-test-cleanup.ts b/test/integration/support/daemon-test-cleanup.ts index 371159c521..848a90d450 100644 --- a/test/integration/support/daemon-test-cleanup.ts +++ b/test/integration/support/daemon-test-cleanup.ts @@ -2,6 +2,11 @@ import fs from 'node:fs'; import path from 'node:path'; import { normalizeError } from '@agent-device/kernel/errors'; import { stopDaemonProcess } from '../../../src/daemon-process.ts'; +import { + readRegisteredDaemonIdentity, + readRegisteredDaemonOwnership, +} from '../../../src/daemon-registration.ts'; +import { ownerIdentityMatches, type OwnerIdentity } from '@agent-device/host-kit/process'; type TestDaemonIdentity = { pid: number; processStartTime?: string }; @@ -11,29 +16,35 @@ export async function cleanupDaemonTestState( observed: TestDaemonIdentity | null, ): Promise { try { - const identity = observed ?? readIdentity(stateDir); - if (!identity) throw new Error('No daemon lifetime was observed'); - const termination = await stopDaemonProcess( - { pid: identity.pid, startTime: identity.processStartTime ?? null }, - { mode: 'graceful', termTimeoutMs: 1_500, killTimeoutMs: 1_500 }, - ); - if (termination.status !== 'exited') { - console.warn('Daemon test cleanup retained state:', stateDir, termination); - return; + const identities = [ + observed ? { pid: observed.pid, startTime: observed.processStartTime ?? null } : null, + readIdentity(stateDir), + ].filter((identity): identity is OwnerIdentity => identity !== null); + if (identities.length === 0) throw new Error('No daemon lifetime was observed'); + for (const identity of identities) { + const termination = await stopDaemonProcess(identity, { + mode: 'graceful', + termTimeoutMs: 1_500, + killTimeoutMs: 1_500, + }); + if (termination.status !== 'exited') { + console.warn('Daemon test cleanup retained state:', stateDir, termination); + return; + } } + const current = readIdentity(stateDir); + if (current && !identities.some((identity) => ownerIdentityMatches(identity, current))) + throw new Error('Daemon registration changed during test cleanup'); fs.rmSync(stateDir, { recursive: true, force: true }); } catch (error) { console.warn('Daemon test cleanup retained state:', stateDir, normalizeError(error)); } } -function readIdentity(stateDir: string): TestDaemonIdentity | null { - try { - return JSON.parse( - fs.readFileSync(path.join(stateDir, 'daemon.json'), 'utf8'), - ) as TestDaemonIdentity; - } catch (error) { - if ((error as NodeJS.ErrnoException).code === 'ENOENT') return null; - throw error; - } +function readIdentity(stateDir: string): OwnerIdentity | null { + const infoPath = path.join(stateDir, 'daemon.json'); + if (readRegisteredDaemonOwnership(infoPath, null).state === 'absent') return null; + const identity = readRegisteredDaemonIdentity(infoPath); + if (!identity) throw new Error('Daemon registration identity is invalid or unreadable'); + return identity; } From e4a21ee8b4c3294e00989a373197d74aab556236 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sat, 3 Oct 2026 05:08:22 +0200 Subject: [PATCH 15/20] chore(gates): expose native PID validation and discover cleanup controls --- packages/host-kit/src/process.ts | 1 + vitest.config.ts | 1 + 2 files changed, 2 insertions(+) diff --git a/packages/host-kit/src/process.ts b/packages/host-kit/src/process.ts index 2ed4c2c06d..6c061dd11f 100644 --- a/packages/host-kit/src/process.ts +++ b/packages/host-kit/src/process.ts @@ -36,6 +36,7 @@ export { export { classifyOwnerLiveness, classifyOwnerLivenessFromObservation, + isProcessPid, type OwnerIdentity, ownerIdentityDiffers, ownerIdentityMatches, diff --git a/vitest.config.ts b/vitest.config.ts index 1b4390fc19..bb9a0a0046 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -178,6 +178,7 @@ export default defineConfig({ // decisions over fixture state-dir listings, so they need no daemon, // device, or subprocess. 'test/integration/support/daemon-leak-model.test.ts', + 'test/integration/support/daemon-test-cleanup.test.ts', // The Android failed-step evidence reader: it replays adb output through the probe // seam, so the crash/process/activity selectors need no emulator to be pinned. 'test/integration/android-emulator-e2e/device-evidence.test.ts', From 4ef85e9d075e2198a09c82269bb3459152064b73 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sat, 3 Oct 2026 05:31:36 +0200 Subject: [PATCH 16/20] refactor: bind session references to stable lifetime entries --- src/__tests__/test-utils/store-factory.ts | 5 + .../request-lock-identity-policy.test.ts | 3 +- .../__tests__/request-lock-policy.test.ts | 3 +- src/daemon/__tests__/session-selector.test.ts | 3 +- .../__tests__/session-store-lifetime.test.ts | 118 ++++++++++++++++++ src/daemon/server/daemon-runtime.ts | 1 + .../session-open-device-in-use.test.ts | 17 +-- .../session-close-lifecycle-teardown.ts | 10 +- .../internal/session-close.ts | 20 ++- src/daemon/session-recovery-hints.test.ts | 19 ++- src/daemon/session-state.ts | 5 +- src/daemon/session-store.ts | 96 +++++++++++--- 12 files changed, 243 insertions(+), 57 deletions(-) create mode 100644 src/daemon/__tests__/session-store-lifetime.test.ts diff --git a/src/__tests__/test-utils/store-factory.ts b/src/__tests__/test-utils/store-factory.ts index 6fccc173af..0c9d222497 100644 --- a/src/__tests__/test-utils/store-factory.ts +++ b/src/__tests__/test-utils/store-factory.ts @@ -1,8 +1,13 @@ import path from 'node:path'; import { SessionStore } from '../../daemon/session-store.ts'; import { mkdtempForTestSync } from './tmp-dir.ts'; +import type { SessionRef, SessionState } from '../../daemon/session-state.ts'; export function makeSessionStore(prefix = 'agent-device-test-'): SessionStore { const tempRoot = mkdtempForTestSync(prefix); return new SessionStore(path.join(tempRoot, 'sessions')); } + +export function makeStoredSessionRef(session: SessionState, address = session.name): SessionRef { + return makeSessionStore().publish(address, session); +} diff --git a/src/daemon/__tests__/request-lock-identity-policy.test.ts b/src/daemon/__tests__/request-lock-identity-policy.test.ts index 803f608f1c..e84906715f 100644 --- a/src/daemon/__tests__/request-lock-identity-policy.test.ts +++ b/src/daemon/__tests__/request-lock-identity-policy.test.ts @@ -1,3 +1,4 @@ +import { makeStoredSessionRef } from '../../__tests__/test-utils/store-factory.ts'; import { test } from 'vitest'; import assert from 'node:assert/strict'; import { AppError } from '@agent-device/kernel/errors'; @@ -175,7 +176,7 @@ const ROWS: Row[] = [ /** Every row's session is explicitly named, so it is stored under — and addressed by — its name. */ function ref(session: SessionState | undefined): SessionRef | undefined { - return session ? { address: session.name, session } : undefined; + return session ? makeStoredSessionRef(session) : undefined; } for (const row of ROWS) { diff --git a/src/daemon/__tests__/request-lock-policy.test.ts b/src/daemon/__tests__/request-lock-policy.test.ts index a56b5fdb6d..2db8ab0be1 100644 --- a/src/daemon/__tests__/request-lock-policy.test.ts +++ b/src/daemon/__tests__/request-lock-policy.test.ts @@ -1,3 +1,4 @@ +import { makeStoredSessionRef } from '../../__tests__/test-utils/store-factory.ts'; import { test } from 'vitest'; import assert from 'node:assert/strict'; import { applyRequestLockPolicy } from '../request-lock-policy.ts'; @@ -34,7 +35,7 @@ const ANDROID_SESSION: SessionState = { /** Both fixtures are explicitly named, so each is stored under — and addressed by — its name. */ function ref(session: SessionState): SessionRef { - return { address: session.name, session }; + return makeStoredSessionRef(session); } test('allows compatible fresh-session selectors under request lock policy', () => { diff --git a/src/daemon/__tests__/session-selector.test.ts b/src/daemon/__tests__/session-selector.test.ts index a224a7a059..54e978d994 100644 --- a/src/daemon/__tests__/session-selector.test.ts +++ b/src/daemon/__tests__/session-selector.test.ts @@ -1,3 +1,4 @@ +import { makeStoredSessionRef } from '../../__tests__/test-utils/store-factory.ts'; import { test } from 'vitest'; import assert from 'node:assert/strict'; import { assertSessionSelectorMatches } from '../session-selector.ts'; @@ -24,7 +25,7 @@ function makeSession(overrides?: Partial): SessionState { /** These sessions are explicitly named, so each is stored under — and addressed by — its name. */ function ref(session: SessionState): SessionRef { - return { address: session.name, session }; + return makeStoredSessionRef(session); } test('accepts matching platform and serial selectors', () => { diff --git a/src/daemon/__tests__/session-store-lifetime.test.ts b/src/daemon/__tests__/session-store-lifetime.test.ts new file mode 100644 index 0000000000..d3b7ae7e65 --- /dev/null +++ b/src/daemon/__tests__/session-store-lifetime.test.ts @@ -0,0 +1,118 @@ +import assert from 'node:assert/strict'; +import { test } from 'vitest'; +import { AppError } from '@agent-device/kernel/errors'; +import { makeSession } from '../../__tests__/test-utils/session-factories.ts'; +import { makeSessionStore } from '../../__tests__/test-utils/store-factory.ts'; + +const ADDRESS = 'cwd:worktree:default'; + +function ended(error: unknown): boolean { + return error instanceof AppError && error.details?.reason === 'session_lifetime_ended'; +} + +test('refs capture records while resolving rebuilds from the same lifetime', () => { + const store = makeSessionStore(); + const session = makeSession('default'); + const initial = store.publish(ADDRESS, session); + const lookup = store.lookup(ADDRESS)!; + const listed = store.listRefs()[0]!; + const byDevice = store.findByDevice(session.device.id)!; + for (const ref of [lookup, listed, byDevice]) { + assert.notEqual(ref, initial); + assert.equal(ref.lifetime, initial.lifetime); + assert.equal(ref.session, session); + assert.equal(Object.isFrozen(ref), true); + } + const rebuilt = store.update(initial, { appName: 'Reopened' }); + assert.equal(initial.session, session); + assert.equal(initial.session.appName, undefined); + for (const ref of [initial, lookup, listed, byDevice]) { + assert.equal(store.resolveCurrent(ref), rebuilt); + } + assert.equal(store.lookup(ADDRESS)?.session, rebuilt); + assert.equal(store.get('default'), undefined); +}); + +test('updates derive from the latest matching record and preserve intervening fields', () => { + const store = makeSessionStore(); + const ref = store.publish(ADDRESS, makeSession('default', { createdAt: 10 })); + store.update(ref, { appBundleId: 'com.example.updated' }); + store.update(ref, (current) => ({ + appName: current.appBundleId, + createdAt: current.createdAt + 1, + })); + assert.equal(store.get(ADDRESS)?.appBundleId, 'com.example.updated'); + assert.equal(store.get(ADDRESS)?.appName, 'com.example.updated'); + assert.equal(store.get(ADDRESS)?.createdAt, 11); + assert.equal(store.get(ADDRESS)?.actions, ref.session.actions); +}); + +test('address reuse with the same record still starts a different lifetime', () => { + const store = makeSessionStore(); + const session = makeSession('default'); + const old = store.publish(ADDRESS, session); + assert.equal(store.retire(old), true); + const successor = store.publish(ADDRESS, session); + store.setRuntimeHints(ADDRESS, { metroPort: 8082 }); + assert.notEqual(successor.lifetime, old.lifetime); + assert.equal(store.resolveCurrent(old), undefined); + assert.throws(() => store.requireCurrent(old), ended); + assert.throws(() => store.update(old, { appName: 'Stale' }), ended); + assert.equal(store.retire(old), false); + assert.equal(store.get(ADDRESS), session); + assert.equal(store.getRuntimeHints(ADDRESS)?.metroPort, 8082); + assert.equal(store.retire(successor), true); + assert.equal(store.getRuntimeHints(ADDRESS), undefined); +}); + +test('retired updates cannot run their derivation or resurrect a record', () => { + const store = makeSessionStore(); + const ref = store.publish(ADDRESS, makeSession('default')); + store.retire(ref); + let ran = false; + assert.throws( + () => + store.update(ref, () => { + ran = true; + return { appName: 'Late' }; + }), + ended, + ); + assert.equal(ran, false); + assert.equal(store.lookup(ADDRESS), undefined); +}); + +test('an occupied address cannot be published again', () => { + const store = makeSessionStore(); + const ref = store.publish(ADDRESS, makeSession('default')); + assert.throws( + () => store.publish(ADDRESS, makeSession('default')), + (error) => error instanceof AppError && error.details?.reason === 'session_address_occupied', + ); + assert.equal(store.requireCurrent(ref), ref.session); +}); + +test('shutdown closes draft admission while allowing the current lifetime to settle', () => { + const store = makeSessionStore(); + const ref = store.publish(ADDRESS, makeSession('default')); + store.closeAdmission(); + assert.throws( + () => store.publish('late-draft', makeSession('late-draft')), + (error) => error instanceof AppError && error.details?.reason === 'daemon_shutting_down', + ); + store.update(ref, { appName: 'Settled' }); + assert.equal(store.requireCurrent(ref).appName, 'Settled'); + assert.equal(store.retire(ref), true); + assert.equal(store.lookup('late-draft'), undefined); +}); + +test('a ref from another store has no authority over the same address', () => { + const source = makeSessionStore(); + const target = makeSessionStore(); + const foreign = source.publish(ADDRESS, makeSession('default')); + const local = target.publish(ADDRESS, foreign.session); + assert.equal(target.resolveCurrent(foreign), undefined); + assert.throws(() => target.update(foreign, { appName: 'Foreign' }), ended); + assert.equal(target.retire(foreign), false); + assert.equal(target.requireCurrent(local), foreign.session); +}); diff --git a/src/daemon/server/daemon-runtime.ts b/src/daemon/server/daemon-runtime.ts index e561d56290..fe73e03097 100644 --- a/src/daemon/server/daemon-runtime.ts +++ b/src/daemon/server/daemon-runtime.ts @@ -754,6 +754,7 @@ export async function startDaemonRuntime( sessionIdleExpiry.cancel(); if (shuttingDown) return; shuttingDown = true; + sessionStore.closeAdmission(); stopMetadataLossWatch(); if (shutdownOptions.cause) { await emitFatalDiagnostic(shutdownOptions.cause); diff --git a/src/daemon/session-lifecycle/internal/__tests__/session-open-device-in-use.test.ts b/src/daemon/session-lifecycle/internal/__tests__/session-open-device-in-use.test.ts index db7ff7ad8d..f9aad9365e 100644 --- a/src/daemon/session-lifecycle/internal/__tests__/session-open-device-in-use.test.ts +++ b/src/daemon/session-lifecycle/internal/__tests__/session-open-device-in-use.test.ts @@ -1,3 +1,4 @@ +import { makeStoredSessionRef } from '../../../../__tests__/test-utils/store-factory.ts'; import { test, expect } from 'vitest'; import { buildDeviceInUseBySessionError, @@ -13,16 +14,16 @@ import { IOS_SIMULATOR } from '../../../../__tests__/test-utils/device-fixtures. const SCOPED_ADDRESS = 'cwd:8bea844ab16aa9b3:default'; -const scopedRef: SessionRef = { - address: SCOPED_ADDRESS, - session: { +const scopedRef: SessionRef = makeStoredSessionRef( + { name: 'default', sessionScope: { kind: 'cwd', id: '8bea844ab16aa9b3' }, device: IOS_SIMULATOR, createdAt: 0, actions: [], }, -}; + SCOPED_ADDRESS, +); test('the by-session conflict reports the address, in the message, details and hint', () => { const response = buildDeviceInUseBySessionError(scopedRef, IOS_SIMULATOR); @@ -41,16 +42,16 @@ test('the by-session conflict reports the address, in the message, details and h // --session, nor close. test('the foreign-workspace conflict names the owning session address', () => { const foreignAddress = 'cwd:1d9b7c2f4a6e8b03:default'; - const foreignRef: SessionRef = { - address: foreignAddress, - session: { + const foreignRef: SessionRef = makeStoredSessionRef( + { name: 'default', sessionScope: { kind: 'cwd', id: '1d9b7c2f4a6e8b03' }, device: IOS_SIMULATOR, createdAt: 0, actions: [], }, - }; + foreignAddress, + ); const response = buildForeignWorkspaceSessionConflict(foreignRef, IOS_SIMULATOR); diff --git a/src/daemon/session-lifecycle/internal/session-close-lifecycle-teardown.ts b/src/daemon/session-lifecycle/internal/session-close-lifecycle-teardown.ts index 1ce108ec07..4dd4fe6c77 100644 --- a/src/daemon/session-lifecycle/internal/session-close-lifecycle-teardown.ts +++ b/src/daemon/session-lifecycle/internal/session-close-lifecycle-teardown.ts @@ -37,8 +37,7 @@ export type SessionCloseTeardownResult = Readonly<{ /** Runs owned resources, native close, native hint cleanup, and final lifecycle disposal. */ export async function runSessionCloseTeardown(params: { req: DaemonRequest; - session: SessionState; - sessionName: string; + ref: SessionRef; logPath: string; sessionStore: SessionStore; lifecycle: CloseRuntime | CloseRuntimeWithRuntimeHintClear; @@ -56,8 +55,7 @@ export async function runSessionCloseTeardown(params: { }): Promise { const { req, - session, - sessionName, + ref, logPath, sessionStore, lifecycle, @@ -67,6 +65,8 @@ export async function runSessionCloseTeardown(params: { dispatchTargetedPlatformClose, finalizeOrdinaryCloseScript, } = params; + const { address: sessionName } = ref; + const session = sessionStore.requireCurrent(ref); const attemptCleanup = async ( step: string, run: () => Promise, @@ -86,7 +86,7 @@ export async function runSessionCloseTeardown(params: { }); const configuredRuntimeHints = sessionStore.getRuntimeHints(sessionName); await stopBestEffortSessionResources( - { address: sessionName, session }, + ref, sessionStore, attemptCleanup, params.platformResourceCleanup, diff --git a/src/daemon/session-lifecycle/internal/session-close.ts b/src/daemon/session-lifecycle/internal/session-close.ts index 889cc5016b..9d2f95dd9b 100644 --- a/src/daemon/session-lifecycle/internal/session-close.ts +++ b/src/daemon/session-lifecycle/internal/session-close.ts @@ -2,7 +2,7 @@ import { emitDiagnostic } from '@agent-device/host-kit/diagnostics'; import { AppError, normalizeError } from '@agent-device/kernel/errors'; import type { LeaseLifecycleProvider, TargetShutdownResult } from '@agent-device/contracts/device'; import type { DaemonRequest, DaemonResponse } from '../../daemon-request.ts'; -import type { SessionState } from '../../session-state.ts'; +import type { SessionRef, SessionState } from '../../session-state.ts'; import { SessionStore } from '../../session-store.ts'; import { successText, withSuccessText } from '@agent-device/kernel/success-text'; import { resolveCommandDevice } from '../../session-device-resolution.ts'; @@ -193,8 +193,8 @@ export async function handleSessionCloseCommands( params: SessionCloseCommandInput, ): Promise { const { req, sessionName, logPath, sessionStore, leaseRegistry, leaseLifecycleProvider } = params; - const session = sessionStore.get(sessionName); - if (!session) { + const ref = sessionStore.lookup(sessionName); + if (!ref) { return await closeWithoutSession({ req, logPath, @@ -202,6 +202,7 @@ export async function handleSessionCloseCommands( bindDevice: params.bindDevice, }); } + const session = ref.session; assertTerminalRecordingCloseAllowed(req, session); const app = req.positionals?.[0]; if (req.internal?.closeAppOnly === true && !app) { @@ -246,8 +247,7 @@ export async function handleSessionCloseCommands( if ('response' in repair) return repair.response; const closed = await runCloseTeardownAndRelease({ req, - session, - sessionName, + ref, logPath, sessionStore, leaseRegistry, @@ -281,8 +281,7 @@ type SessionCloseFinalization = // lease release keeps the session retryable instead (`{kind:'response'}`). async function runCloseTeardownAndRelease(params: { req: DaemonRequest; - session: SessionState; - sessionName: string; + ref: SessionRef; logPath: string; sessionStore: SessionStore; leaseRegistry: LeaseRegistry; @@ -294,8 +293,7 @@ async function runCloseTeardownAndRelease(params: { }): Promise { const { req, - session, - sessionName, + ref, logPath, sessionStore, leaseRegistry, @@ -303,11 +301,11 @@ async function runCloseTeardownAndRelease(params: { lifecycle, clearRuntimeHints, } = params; + const { address: sessionName, session } = ref; const cleanupFailures: SessionCleanupFailure[] = []; const { platformCloseError, saveScriptError, shutdownResult } = await runSessionCloseTeardown({ req, - session, - sessionName, + ref, logPath, sessionStore, lifecycle, diff --git a/src/daemon/session-recovery-hints.test.ts b/src/daemon/session-recovery-hints.test.ts index c1f0930fa4..15aceaa94d 100644 --- a/src/daemon/session-recovery-hints.test.ts +++ b/src/daemon/session-recovery-hints.test.ts @@ -1,3 +1,4 @@ +import { makeStoredSessionRef } from '../__tests__/test-utils/store-factory.ts'; import { test, expect } from 'vitest'; import { buildSessionRecoveryHint } from './session-recovery-hints.ts'; import type { SessionRef, SessionState } from './session-state.ts'; @@ -12,9 +13,8 @@ import { IOS_SIMULATOR } from '../__tests__/test-utils/device-fixtures.ts'; const SCOPED_ADDRESS = 'cwd:8bea844ab16aa9b3:default'; function scopedRef(overrides: Partial = {}): SessionRef { - return { - address: SCOPED_ADDRESS, - session: { + return makeStoredSessionRef( + { name: 'default', sessionScope: { kind: 'cwd', id: '8bea844ab16aa9b3' }, device: IOS_SIMULATOR, @@ -22,7 +22,8 @@ function scopedRef(overrides: Partial = {}): SessionRef { actions: [], ...overrides, }, - }; + SCOPED_ADDRESS, + ); } test('device-in-use recovery names the address --session accepts, not the public name', () => { @@ -55,10 +56,7 @@ test('a recording session recovery uses the address for both close and record st test('an explicitly named session addresses itself unchanged', () => { const hint = buildSessionRecoveryHint( - { - address: 'checkout', - session: { ...scopedRef().session, name: 'checkout', sessionScope: undefined }, - }, + makeStoredSessionRef({ ...scopedRef().session, name: 'checkout', sessionScope: undefined }), 'device-in-use', ); @@ -81,10 +79,7 @@ test('a device or target conflict does not offer a platform session it cannot an test('selector-conflict recovery offers no platform session to a hand-named session', () => { const hint = buildSessionRecoveryHint( - { - address: 'checkout', - session: { ...scopedRef().session, name: 'checkout', sessionScope: undefined }, - }, + makeStoredSessionRef({ ...scopedRef().session, name: 'checkout', sessionScope: undefined }), 'selector-conflict', ); diff --git a/src/daemon/session-state.ts b/src/daemon/session-state.ts index 8e3350ff04..bf5ee40bb8 100644 --- a/src/daemon/session-state.ts +++ b/src/daemon/session-state.ts @@ -108,10 +108,11 @@ export type PostGestureStabilization = { * target takes this pair rather than a bare record, so it cannot be handed a session whose address * was never resolved. */ -export type SessionRef = { +export type SessionRef = Readonly<{ address: string; session: SessionState; -}; + lifetime: object; +}>; export type SessionState = { name: string; diff --git a/src/daemon/session-store.ts b/src/daemon/session-store.ts index e31c66403a..9e7b1c2d8d 100644 --- a/src/daemon/session-store.ts +++ b/src/daemon/session-store.ts @@ -1,5 +1,6 @@ import path from 'node:path'; import fs from 'node:fs'; +import { AppError } from '@agent-device/kernel/errors'; import { emitDiagnostic } from '@agent-device/host-kit/diagnostics'; import type { SessionRef, SessionRuntimeHints, SessionState } from './session-state.ts'; import { recordActionEntry, type RecordActionEntry } from './session-action-recorder.ts'; @@ -42,9 +43,12 @@ import { } from '@agent-device/session-journal/session-event-log'; const REPAIR_TOMBSTONE_TTL_MS = 60 * 60_000; +type SessionEntry = { current: SessionState }; +type SessionPatch = Partial | ((current: SessionState) => Partial); export class SessionStore { - private readonly sessions = new Map(); + private readonly sessions = new Map(); + private acceptingSessions = true; private readonly runtimeHints = new Map(); private readonly sessionsDir: string; private readonly scriptWriter: SessionScriptWriter; @@ -62,7 +66,67 @@ export class SessionStore { * nothing here can check the invariant a given field carries. */ get(name: string): SessionState | undefined { - return this.sessions.get(name); + return this.sessions.get(name)?.current; + } + + closeAdmission(): void { + this.acceptingSessions = false; + } + + publish(address: string, session: SessionState): SessionRef { + if (!this.acceptingSessions) { + throw new AppError('COMMAND_FAILED', 'Daemon is shutting down', { + reason: 'daemon_shutting_down', + session: address, + }); + } + if (this.sessions.has(address)) { + throw new AppError('COMMAND_FAILED', 'Session address is already occupied', { + reason: 'session_address_occupied', + session: address, + }); + } + const entry = { current: session }; + this.sessions.set(address, entry); + this.clearIdleExpiryTombstone(address); + return this.captureRef(address, entry); + } + + resolveCurrent(ref: SessionRef): SessionState | undefined { + const entry = this.sessions.get(ref.address); + return entry === ref.lifetime ? entry.current : undefined; + } + + requireCurrent(ref: SessionRef): SessionState { + const session = this.resolveCurrent(ref); + if (!session) { + throw new AppError('COMMAND_FAILED', 'Session lifetime has ended', { + reason: 'session_lifetime_ended', + session: ref.address, + hint: 'Open a new session before retrying the command.', + }); + } + return session; + } + + /** Patch callbacks are synchronous and must not call back into the store. */ + update(ref: SessionRef, patch: SessionPatch): SessionState { + const current = this.requireCurrent(ref); + const entry = this.sessions.get(ref.address)!; + const changes = typeof patch === 'function' ? patch(current) : patch; + const next = { ...current, ...changes }; + entry.current = next; + return next; + } + + retire(ref: SessionRef): boolean { + if (!this.resolveCurrent(ref)) return false; + this.runtimeHints.delete(ref.address); + return this.sessions.delete(ref.address); + } + + private captureRef(address: string, entry: SessionEntry): SessionRef { + return Object.freeze({ address, session: entry.current, lifetime: entry }); } /** @@ -76,9 +140,9 @@ export class SessionStore { // every way a record arrives — `open`'s provisional record, a record-only `record` session — and // cannot be forgotten by a future insertion path. A replacing `open` on a live session takes the // other branch and keeps whatever marker that session will earn for itself. - const occupying = this.sessions.has(name); - this.sessions.set(name, session); - if (!occupying) this.clearIdleExpiryTombstone(name); + const entry = this.sessions.get(name); + if (entry) entry.current = session; + else this.publish(name, session); } delete(name: string): boolean { @@ -86,12 +150,12 @@ export class SessionStore { return this.sessions.delete(name); } - values(): IterableIterator { - return this.sessions.values(); + *values(): IterableIterator { + for (const entry of this.sessions.values()) yield entry.current; } toArray(): SessionState[] { - return Array.from(this.sessions.values()); + return Array.from(this.values()); } /** @@ -100,21 +164,21 @@ export class SessionStore { * falls back to `SessionState.name` (#2031/#1394). */ lookup(address: string): SessionRef | undefined { - const session = this.sessions.get(address); - return session ? { address, session } : undefined; + const entry = this.sessions.get(address); + return entry ? this.captureRef(address, entry) : undefined; } /** The session currently bound to `deviceId`, with its address, or `undefined` if none is. */ findByDevice(deviceId: string): SessionRef | undefined { - for (const [address, session] of this.sessions) { - if (session.device.id === deviceId) return { address, session }; + for (const [address, entry] of this.sessions) { + if (entry.current.device.id === deviceId) return this.captureRef(address, entry); } return undefined; } /** Every live session with its address, for surfaces that must report what `--session` accepts. */ listRefs(): SessionRef[] { - return Array.from(this.sessions, ([address, session]) => ({ address, session })); + return Array.from(this.sessions, ([address, entry]) => this.captureRef(address, entry)); } getRuntimeHints(name: string): SessionRuntimeHints | undefined { @@ -298,7 +362,7 @@ export class SessionStore { * never built, and its own `createdAt` already starts that session's deadline clock. */ noteSessionActivity(address: string, atMs: number = Date.now()): void { - const session = this.sessions.get(address); + const session = this.get(address); if (!session) return; session.lastActivityAtMs = atMs; } @@ -415,8 +479,8 @@ export class SessionStore { * public session name, while the map key may include cwd/tenant isolation. */ resolveStoredSessionName(session: SessionState): string { - for (const [name, value] of this.sessions) { - if (value === session) return name; + for (const [name, entry] of this.sessions) { + if (entry.current === session) return name; } return session.name; } From 530c6e976404bb5006576c7b73073b68b4fcc334 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sat, 3 Oct 2026 06:33:56 +0200 Subject: [PATCH 17/20] fix: bind capture adoption to session lifetimes --- .../audio-probe-session-resource.test.ts | 9 +- .../durable-capture-resource.fixtures.ts | 5 ++ .../screen-recording-boundary-faults.test.ts | 6 ++ .../screen-recording-session-resource.test.ts | 32 ++++--- .../__tests__/session-store.fixtures.ts | 12 +-- .../audio-probe-session-resource.ts | 9 +- .../durable-capture-resource.ts | 10 +-- .../perf-capture-session-resource.ts | 9 +- .../screen-recording-session-resource.ts | 5 +- .../src/durable-capture/adoption.ts | 85 ++++++++++--------- .../src/durable-capture/definition.ts | 33 ++++--- .../durable-capture.fixtures.ts | 12 ++- .../session-binding.fixtures.ts | 45 ++++++++++ .../src/recording/failed-finish.test.ts | 31 +++++-- .../app-log-session-resource.test.ts | 74 +++++++++++++++- .../perf-capture-session-resource.test.ts | 5 +- .../screen-recording-session-binding.test.ts | 60 +++++++++++++ .../__tests__/session-capture-binding.test.ts | 72 ++++++++++++++++ src/daemon/app-log-session-resource.ts | 35 +++++--- src/daemon/audio-probe-session-binding.ts | 12 +++ src/daemon/handlers/record-runtime.ts | 35 +++++--- src/daemon/perf-capture-session-binding.ts | 12 +++ .../screen-recording-session-binding.ts | 42 +++++++++ src/daemon/session-capture-binding.ts | 53 ++++++++++++ .../internal/__tests__/session-logs.test.ts | 2 +- .../internal/session-audio.ts | 32 +++---- .../internal/session-observability.ts | 21 +++-- .../internal/session-perf-runtime.ts | 19 +++-- src/daemon/session-store.ts | 10 ++- 29 files changed, 618 insertions(+), 169 deletions(-) create mode 100644 packages/capture-kit/src/durable-capture/session-binding.fixtures.ts create mode 100644 src/daemon/__tests__/screen-recording-session-binding.test.ts create mode 100644 src/daemon/__tests__/session-capture-binding.test.ts create mode 100644 src/daemon/audio-probe-session-binding.ts create mode 100644 src/daemon/perf-capture-session-binding.ts create mode 100644 src/daemon/screen-recording-session-binding.ts create mode 100644 src/daemon/session-capture-binding.ts diff --git a/packages/capture-kit/src/capture-admission/__tests__/audio-probe-session-resource.test.ts b/packages/capture-kit/src/capture-admission/__tests__/audio-probe-session-resource.test.ts index 073c33419b..359fb5f545 100644 --- a/packages/capture-kit/src/capture-admission/__tests__/audio-probe-session-resource.test.ts +++ b/packages/capture-kit/src/capture-admission/__tests__/audio-probe-session-resource.test.ts @@ -1,3 +1,4 @@ +import { makeCaptureSessionBinding } from '../../durable-capture/session-binding.fixtures.ts'; import fs from 'node:fs/promises'; import path from 'node:path'; import { expect, test, vi } from 'vitest'; @@ -44,6 +45,10 @@ test('audio-probe disposes on a failed finish because terminating the helper is ); const session: DurableCaptureSessionState = {}; sessionStore.set(sessionName, session); + const binding = makeCaptureSessionBinding(sessionStore, sessionName, { + read: (session) => session.audioProbe, + replace: (session, audioProbe) => ({ ...session, audioProbe }), + }); const statusPath = path.join(sessionStore.resolveSessionDir(sessionName), 'audio-probe.json'); const terminate = vi.fn(async () => {}); let resolveExit!: (result: HostCommandResult) => void; @@ -85,9 +90,7 @@ test('audio-probe disposes on a failed finish because terminating the helper is }); await adoptStartedAudioProbe({ admissionLedger: createAudioProbeAdmissionLedger(), - session, - sessionName, - sessionStore, + binding, device, owner: localRuntimeOwner('apple'), fence, diff --git a/packages/capture-kit/src/capture-admission/__tests__/durable-capture-resource.fixtures.ts b/packages/capture-kit/src/capture-admission/__tests__/durable-capture-resource.fixtures.ts index 0cef989b60..e022419770 100644 --- a/packages/capture-kit/src/capture-admission/__tests__/durable-capture-resource.fixtures.ts +++ b/packages/capture-kit/src/capture-admission/__tests__/durable-capture-resource.fixtures.ts @@ -1,3 +1,4 @@ +import { makeCaptureSessionBinding } from '../../durable-capture/session-binding.fixtures.ts'; import { vi } from 'vitest'; import type { AppLogCompletion, AppLogLiveHandle } from '@agent-device/contracts/app-log-runtime'; import type { CleanupOutcome, FinishOutcome } from '@agent-device/contracts/durable-resource'; @@ -72,6 +73,10 @@ export function makeDurableCaptureContext( const session: TestCaptureSession = {}; sessionStore.set(sessionName, session); return { + binding: makeCaptureSessionBinding(sessionStore, sessionName, { + read: (session) => session.appLog, + replace: (session, appLog) => ({ ...session, appLog, appLogFailure: undefined }), + }), admissionLedger: createDurableCaptureAdmissionLedger({ displayName: 'test capture' }), session, sessionName, diff --git a/packages/capture-kit/src/capture-admission/__tests__/screen-recording-boundary-faults.test.ts b/packages/capture-kit/src/capture-admission/__tests__/screen-recording-boundary-faults.test.ts index 775f5a3fae..dfab3f92e8 100644 --- a/packages/capture-kit/src/capture-admission/__tests__/screen-recording-boundary-faults.test.ts +++ b/packages/capture-kit/src/capture-admission/__tests__/screen-recording-boundary-faults.test.ts @@ -1,3 +1,4 @@ +import { makeCaptureSessionBinding } from '../../durable-capture/session-binding.fixtures.ts'; import path from 'node:path'; import { expect, test, vi } from 'vitest'; import { createDurableResourceEnvelope } from '../../durable-resource-envelope.ts'; @@ -217,9 +218,14 @@ function makeContext(resource: ReturnType session.screenRecording, + replace: (session, screenRecording) => ({ ...session, screenRecording }), + }); return { admissionLedger: createDurableCaptureAdmissionLedger({ displayName: 'screen recording' }), session, + binding, sessionName, sessionStore, device, diff --git a/packages/capture-kit/src/capture-admission/__tests__/screen-recording-session-resource.test.ts b/packages/capture-kit/src/capture-admission/__tests__/screen-recording-session-resource.test.ts index b048db0b3c..a64497c19d 100644 --- a/packages/capture-kit/src/capture-admission/__tests__/screen-recording-session-resource.test.ts +++ b/packages/capture-kit/src/capture-admission/__tests__/screen-recording-session-resource.test.ts @@ -1,3 +1,4 @@ +import { makeCaptureSessionBinding } from '../../durable-capture/session-binding.fixtures.ts'; import { expect, test, vi } from 'vitest'; import { PendingTransferGuard } from '@agent-device/contracts/async-lifecycle'; import { localRuntimeOwner } from '@agent-device/contracts/platform-runtime'; @@ -35,6 +36,10 @@ test('screen recording persists durable truth before adopting only handle and en const sessionName = 'recording'; const session: TestRecordingSession = { name: sessionName, device }; sessionStore.set(sessionName, session); + const binding = makeCaptureSessionBinding(sessionStore, sessionName, { + read: (session) => session.screenRecording, + replace: (session, screenRecording) => ({ ...session, screenRecording }), + }); const owner = localRuntimeOwner('android'); const fence = { token: 'recording-fence', generation: 1 } as const; const finish = vi.fn(async () => ({ @@ -79,9 +84,7 @@ test('screen recording persists durable truth before adopting only handle and en await adoptStartedScreenRecording({ admissionLedger: createScreenRecordingAdmissionLedger(), - session, - sessionName, - sessionStore, + binding, device: session.device, owner, fence, @@ -103,7 +106,10 @@ test('screen recording persists durable truth before adopting only handle and en if (!active) throw new Error('Expected screen-recording session'); await expect( finishLiveScreenRecording({ intent: 'capture', session: active, sessionName, sessionStore }), - ).resolves.toMatchObject({ backend: 'android', outPath: '/tmp/recording.mp4' }); + ).resolves.toMatchObject({ + backend: 'android', + outPath: '/tmp/recording.mp4', + }); expect(finish).toHaveBeenCalledOnce(); expect(sessionStore.get(sessionName)?.screenRecording).toBeUndefined(); }); @@ -115,6 +121,10 @@ test('a failed recording finish keeps the record open and never disposes the rec const sessionName = 'recording'; const session: TestRecordingSession = { name: sessionName, device }; sessionStore.set(sessionName, session); + const binding = makeCaptureSessionBinding(sessionStore, sessionName, { + read: (session) => session.screenRecording, + replace: (session, screenRecording) => ({ ...session, screenRecording }), + }); const owner = localRuntimeOwner('android'); const fence = { token: 'recording-fence', generation: 1 } as const; const finishError = new Error('failed to retrieve playable Android recording'); @@ -150,9 +160,7 @@ test('a failed recording finish keeps the record open and never disposes the rec }); await adoptStartedScreenRecording({ admissionLedger: createScreenRecordingAdmissionLedger(), - session, - sessionName, - sessionStore, + binding, device: session.device, owner, fence, @@ -182,6 +190,10 @@ test('a record stop that fails after collecting resumes through the fence withou const sessionName = 'recording'; const session: TestRecordingSession = { name: sessionName, device }; sessionStore.set(sessionName, session); + const binding = makeCaptureSessionBinding(sessionStore, sessionName, { + read: (session) => session.screenRecording, + replace: (session, screenRecording) => ({ ...session, screenRecording }), + }); const owner = localRuntimeOwner('android'); const fence = { token: 'recording-fence', generation: 1 } as const; const signals = vi.fn(async () => ({ observation: { recorder: 'confirmed' as const } })); @@ -216,9 +228,7 @@ test('a record stop that fails after collecting resumes through the fence withou ); await adoptStartedScreenRecording({ admissionLedger: createScreenRecordingAdmissionLedger(), - session, - sessionName, - sessionStore, + binding, device: session.device, owner, fence, @@ -239,7 +249,7 @@ test('a record stop that fails after collecting resumes through the fence withou if (!active) throw new Error('Expected screen-recording session'); return finishLiveScreenRecording({ intent: 'capture', - session: active, + session: sessionStore.get(sessionName) ?? session, sessionName, sessionStore, }); diff --git a/packages/capture-kit/src/capture-admission/__tests__/session-store.fixtures.ts b/packages/capture-kit/src/capture-admission/__tests__/session-store.fixtures.ts index dee8763861..ab0ccd73e0 100644 --- a/packages/capture-kit/src/capture-admission/__tests__/session-store.fixtures.ts +++ b/packages/capture-kit/src/capture-admission/__tests__/session-store.fixtures.ts @@ -1,13 +1,13 @@ import path from 'node:path'; import { safeSessionName } from '@agent-device/host-kit/session-paths'; -import type { DurableCaptureSessionStore } from '../../durable-capture/index.ts'; import { mkdtempForTestSync } from '../../tmp-dir.fixtures.ts'; -export type CaptureAdmissionSessionStore = DurableCaptureSessionStore & - Readonly<{ - get(name: string): S | undefined; - sessionsDir: string; - }>; +export type CaptureAdmissionSessionStore = Readonly<{ + set(name: string, session: S): void; + resolveSessionDir(name: string): string; + get(name: string): S | undefined; + sessionsDir: string; +}>; /** * The whole of the daemon `SessionStore` these admission modules ever address — `set`, diff --git a/packages/capture-kit/src/capture-admission/audio-probe-session-resource.ts b/packages/capture-kit/src/capture-admission/audio-probe-session-resource.ts index 83863d3336..ddcd8037b0 100644 --- a/packages/capture-kit/src/capture-admission/audio-probe-session-resource.ts +++ b/packages/capture-kit/src/capture-admission/audio-probe-session-resource.ts @@ -9,7 +9,10 @@ import type { RuntimeOwnerRef, } from '@agent-device/contracts/platform-runtime'; import type { DeviceInfo } from '@agent-device/kernel/device'; -import type { DurableCaptureSessionStore } from '../durable-capture/index.ts'; +import type { + DurableCaptureSessionBinding, + DurableCaptureSessionStore, +} from '../durable-capture/index.ts'; import { createDurableCaptureResource } from './durable-capture-resource.ts'; import type { DurableCaptureFinishIntent } from './durable-capture-resource.ts'; import type { AudioProbeAdmissionLedger } from './audio-probe-admission-ledger.ts'; @@ -48,9 +51,7 @@ export const audioProbeDurableResource = createDurableCaptureResource< export function adoptStartedAudioProbe(params: { admissionLedger: AudioProbeAdmissionLedger; - session: DurableCaptureSessionState; - sessionName: string; - sessionStore: DurableCaptureSessionStore; + binding: DurableCaptureSessionBinding<'audio-probe', AudioProbeLiveHandle>; device: DeviceInfo; owner: RuntimeOwnerRef; fence: ResourceOwnershipFence; diff --git a/packages/capture-kit/src/capture-admission/durable-capture-resource.ts b/packages/capture-kit/src/capture-admission/durable-capture-resource.ts index a03751be14..4dd4636244 100644 --- a/packages/capture-kit/src/capture-admission/durable-capture-resource.ts +++ b/packages/capture-kit/src/capture-admission/durable-capture-resource.ts @@ -23,8 +23,8 @@ import type { DurableSessionResourceKind } from './durable-session-resource-kind export type { DurableCaptureFinishIntent, DurableSessionResourceKind }; -type AdoptStartedSessionCaptureParams = Omit< - AdoptStartedDurableCaptureParams, +type AdoptStartedSessionCaptureParams = Omit< + AdoptStartedDurableCaptureParams, 'reportUndurableCleanup' > & Readonly<{ admissionLedger: DurableCaptureAdmissionLedger }>; @@ -48,7 +48,7 @@ export function createDurableCaptureResource< S, >(definition: DurableCaptureResourceDefinition) { const sessionResourcePath = ( - sessionStore: DurableCaptureSessionStore, + sessionStore: Readonly<{ resolveSessionDir(name: string): string }>, sessionName: string, ): string => definition.store.resolvePath(sessionStore.resolveSessionDir(sessionName)); const recoveryParams = ( @@ -70,7 +70,7 @@ export function createDurableCaptureResource< }): ResourceOwnershipFence { return createNextDurableCaptureFence(definition, params); }, - adoptStarted(params: AdoptStartedSessionCaptureParams): Promise { + adoptStarted(params: AdoptStartedSessionCaptureParams): Promise { return adoptStartedDurableCapture( definition, { @@ -80,7 +80,7 @@ export function createDurableCaptureResource< else params.admissionLedger.blockUndurableCleanup(device, outcome.reason); }, }, - sessionResourcePath(params.sessionStore, params.sessionName), + definition.store.resolvePath(params.binding.sessionDir), ); }, finishLive(params: { diff --git a/packages/capture-kit/src/capture-admission/perf-capture-session-resource.ts b/packages/capture-kit/src/capture-admission/perf-capture-session-resource.ts index 6e6e14104b..41546e034f 100644 --- a/packages/capture-kit/src/capture-admission/perf-capture-session-resource.ts +++ b/packages/capture-kit/src/capture-admission/perf-capture-session-resource.ts @@ -9,7 +9,10 @@ import type { RuntimeOwnerRef, } from '@agent-device/contracts/platform-runtime'; import type { DeviceInfo } from '@agent-device/kernel/device'; -import type { DurableCaptureSessionStore } from '../durable-capture/index.ts'; +import type { + DurableCaptureSessionBinding, + DurableCaptureSessionStore, +} from '../durable-capture/index.ts'; import { createDurableCaptureResource } from './durable-capture-resource.ts'; import type { DurableCaptureFinishIntent } from './durable-capture-resource.ts'; import type { PerfCaptureAdmissionLedger } from './perf-capture-admission-ledger.ts'; @@ -46,9 +49,7 @@ export const perfCaptureDurableResource = createDurableCaptureResource< export function adoptStartedPerfCapture(params: { admissionLedger: PerfCaptureAdmissionLedger; - session: DurableCaptureSessionState; - sessionName: string; - sessionStore: DurableCaptureSessionStore; + binding: DurableCaptureSessionBinding<'perf-capture', PerfNativeCaptureLiveHandle>; device: DeviceInfo; owner: RuntimeOwnerRef; fence: ResourceOwnershipFence; diff --git a/packages/capture-kit/src/capture-admission/screen-recording-session-resource.ts b/packages/capture-kit/src/capture-admission/screen-recording-session-resource.ts index 66b54f4bcc..2286e814e9 100644 --- a/packages/capture-kit/src/capture-admission/screen-recording-session-resource.ts +++ b/packages/capture-kit/src/capture-admission/screen-recording-session-resource.ts @@ -16,6 +16,7 @@ import type { StopObservation } from '@agent-device/contracts/recording-stop-obs import type { DeviceInfo } from '@agent-device/kernel/device'; import type { DurableCaptureRecoveryControl, + DurableCaptureSessionBinding, DurableCaptureSessionStore, } from '../durable-capture/index.ts'; import { createDurableCaptureResource } from './durable-capture-resource.ts'; @@ -50,9 +51,7 @@ export const screenRecordingDurableResource = createDurableCaptureResource< export function adoptStartedScreenRecording(params: { admissionLedger: ScreenRecordingAdmissionLedger; - session: DurableCaptureSessionState; - sessionName: string; - sessionStore: DurableCaptureSessionStore; + binding: DurableCaptureSessionBinding<'screen-recording', ScreenRecordingLiveHandle>; device: DeviceInfo; owner: RuntimeOwnerRef; fence: ResourceOwnershipFence; diff --git a/packages/capture-kit/src/durable-capture/adoption.ts b/packages/capture-kit/src/durable-capture/adoption.ts index 2390539458..eba37b023e 100644 --- a/packages/capture-kit/src/durable-capture/adoption.ts +++ b/packages/capture-kit/src/durable-capture/adoption.ts @@ -31,43 +31,41 @@ export async function adoptStartedDurableCapture< S, >( definition: DurableCaptureResourceDefinition, - params: AdoptStartedDurableCaptureParams, + params: AdoptStartedDurableCaptureParams, resourcePath: string, ): Promise { let state: AdoptionState = { kind: 'pending' }; try { + params.binding.assertAdoptable(); const envelope = withPhase(validateStartedEnvelope(definition, params), 'active'); definition.store.write(resourcePath, envelope); state = { kind: 'persisted' }; params.throwIfCanceled(); const handle = params.pendingHandle.transfer(); state = { kind: 'transferred', handle }; - params.sessionStore.set( - params.sessionName, - definition.sessionSlot.replace(params.session, { handle, envelope }), - ); + params.binding.adopt({ handle, envelope }); } catch (error) { await recoverFailedAdoption(definition, params, resourcePath, state, error); throw error; } } -async function recoverFailedAdoption, C, S>( - definition: DurableCaptureResourceDefinition, - params: AdoptStartedDurableCaptureParams, +async function recoverFailedAdoption, C>( + definition: DurableCaptureRecordDefinition, + params: AdoptStartedDurableCaptureParams, resourcePath: string, state: AdoptionState, primaryError: unknown, ): Promise { + const mayPersist = params.binding.canPersist(); const persisted = - state.kind === 'pending' ? persistRecoveryTombstone(definition, params, resourcePath) : true; + state.kind === 'pending' + ? mayPersist && persistRecoveryTombstone(definition, params, resourcePath) + : true; const initialCleanupError = await disposeFailedAdoption(params, state); - const transition = confirmFailedAdoptionTransition( - definition, - params, - resourcePath, - initialCleanupError, - ); + const transition = !params.binding.canPersist() + ? { confirmed: false, cleanupError: initialCleanupError } + : confirmFailedAdoptionTransition(definition, params, resourcePath, initialCleanupError); params.reportUndurableCleanup( params.device, (!persisted && transition.cleanupError === undefined) || transition.confirmed @@ -84,9 +82,9 @@ async function recoverFailedAdoption, C, S>( +function confirmFailedAdoptionTransition, C>( definition: DurableCaptureRecordDefinition, - params: Pick, 'sessionName' | 'fence'>, + params: Pick, 'binding' | 'fence'>, resourcePath: string, cleanupError: unknown | undefined, ): { confirmed: boolean; cleanupError: unknown | undefined } { @@ -100,7 +98,7 @@ function confirmFailedAdoptionTransition( - params: AdoptStartedDurableCaptureParams, +async function disposeFailedAdoption( + params: AdoptStartedDurableCaptureParams, state: AdoptionState, ): Promise { try { @@ -122,16 +120,13 @@ async function disposeFailedAdoption, C, S>( +function persistRecoveryTombstone, C>( definition: DurableCaptureRecordDefinition, - params: AdoptStartedDurableCaptureParams, + params: AdoptStartedDurableCaptureParams, resourcePath: string, ): boolean { try { - definition.store.write( - resourcePath, - createExpectedEnvelope(definition, params, params.envelope.descriptor), - ); + definition.store.write(resourcePath, createRecoveryEnvelope(definition, params)); return true; } catch (descriptorError) { try { @@ -148,7 +143,7 @@ function persistRecoveryTombstone, C, S>( +function createRecoveryEnvelope, C>( definition: DurableCaptureRecordDefinition, - params: Pick< - AdoptStartedDurableCaptureParams, - 'sessionName' | 'device' | 'owner' | 'fence' - >, + params: AdoptStartedDurableCaptureParams, +): DurableResourceEnvelope { + try { + return withPhase(validateStartedEnvelope(definition, params), 'active'); + } catch { + return createExpectedEnvelope(definition, params, params.envelope.descriptor); + } +} + +function createExpectedEnvelope, C>( + definition: DurableCaptureRecordDefinition, + params: Pick, 'binding' | 'device' | 'owner' | 'fence'>, descriptor: DurableResourceEnvelope['descriptor'], ): DurableResourceEnvelope { return createDurableResourceEnvelope({ resourceKind: definition.resourceKind, - sessionId: params.sessionName, + sessionId: params.binding.address, device: deviceIdentity(params.device), owner: params.owner, fence: params.fence, @@ -180,11 +183,11 @@ function createExpectedEnvelope, C, S>( +function validateStartedEnvelope, C>( definition: DurableCaptureRecordDefinition, params: Pick< - AdoptStartedDurableCaptureParams, - 'sessionName' | 'device' | 'owner' | 'fence' | 'envelope' + AdoptStartedDurableCaptureParams, + 'binding' | 'device' | 'owner' | 'fence' | 'envelope' >, ): DurableResourceEnvelope { const decoded = decodeDurableResourceEnvelope(params.envelope); @@ -207,7 +210,7 @@ function validateStartedEnvelope( return true; } -function emitCleanupDiagnostic( +function emitCleanupDiagnostic( definition: DurableCaptureRecordDefinition, - params: Pick, 'sessionName'>, + params: Pick, 'binding'>, primaryError: unknown, cleanupError: unknown, ): void { @@ -260,7 +263,7 @@ function emitCleanupDiagnostic = Readonly<{ - set(name: string, session: S): void; - resolveSessionDir(name: string): string; -}>; - export type DurableCaptureSessionResource = Readonly<{ handle: H; envelope: DurableResourceEnvelope; }>; +export type DurableCaptureSlotClearResult = 'cleared' | 'retired' | 'resource-changed'; + +export type DurableCaptureSessionBinding = Readonly<{ + address: string; + sessionDir: string; + read(): DurableCaptureSessionResource | undefined; + assertAdoptable(): void; + canPersist(): boolean; + adopt(resource: DurableCaptureSessionResource): void; + clear(expected: DurableCaptureSessionResource): DurableCaptureSlotClearResult; +}>; + +export type DurableCaptureSessionStore = Readonly<{ + set(name: string, session: S): void; + resolveSessionDir(name: string): string; +}>; + export type DurableCaptureSessionSlot = Readonly<{ read(session: S): DurableCaptureSessionResource | undefined; replace(session: S, resource: DurableCaptureSessionResource | undefined): S; @@ -86,11 +93,9 @@ export type DurableCaptureCleanupOutcome = | { confirmed: true } | { confirmed: false; reason: string }; -export type AdoptStartedDurableCaptureParams = { +export type AdoptStartedDurableCaptureParams = { reportUndurableCleanup(device: DeviceInfo, outcome: DurableCaptureCleanupOutcome): void; - session: S; - sessionName: string; - sessionStore: DurableCaptureSessionStore; + binding: DurableCaptureSessionBinding; device: DeviceInfo; owner: RuntimeOwnerRef; fence: ResourceOwnershipFence; diff --git a/packages/capture-kit/src/durable-capture/durable-capture.fixtures.ts b/packages/capture-kit/src/durable-capture/durable-capture.fixtures.ts index 61bdf5d280..fcb88e857c 100644 --- a/packages/capture-kit/src/durable-capture/durable-capture.fixtures.ts +++ b/packages/capture-kit/src/durable-capture/durable-capture.fixtures.ts @@ -1,3 +1,4 @@ +import { makeCaptureSessionBinding } from './session-binding.fixtures.ts'; import path from 'node:path'; import { vi, type Mock } from 'vitest'; import { @@ -15,7 +16,6 @@ import type { DurableCaptureFailedFinishPolicy, DurableCaptureResourceDefinition, DurableCaptureSessionResource, - DurableCaptureSessionStore, } from './definition.ts'; import { createDurableCaptureResourceStore, type DurableCaptureResourceStore } from './store.ts'; @@ -91,8 +91,9 @@ export function makeDurableCaptureContext( const session: TestCaptureSession = { name: sessionName }; sessions.set(sessionName, session); const resolveSessionDir = (name: string): string => path.join(sessionsDir, name); - const sessionStore: DurableCaptureSessionStore = { - set: (name, next) => void sessions.set(name, next), + const sessionStore = { + get: (name: string) => sessions.get(name), + set: (name: string, next: TestCaptureSession) => void sessions.set(name, next), resolveSessionDir, }; const reportUndurableCleanup: Mock< @@ -100,6 +101,11 @@ export function makeDurableCaptureContext( > = vi.fn(); return { reportUndurableCleanup, + binding: makeCaptureSessionBinding( + sessionStore, + sessionName, + testCaptureDefinition.sessionSlot, + ), sessions, sessionsDir, resolveSessionDir, diff --git a/packages/capture-kit/src/durable-capture/session-binding.fixtures.ts b/packages/capture-kit/src/durable-capture/session-binding.fixtures.ts new file mode 100644 index 0000000000..611d9b9108 --- /dev/null +++ b/packages/capture-kit/src/durable-capture/session-binding.fixtures.ts @@ -0,0 +1,45 @@ +import { AppError } from '@agent-device/kernel/errors'; +import type { DurableCaptureSessionBinding, DurableCaptureSessionSlot } from './definition.ts'; + +export function makeCaptureSessionBinding( + store: Readonly<{ + get(name: string): S | undefined; + set(name: string, session: S): void; + resolveSessionDir(name: string): string; + }>, + address: string, + slot: DurableCaptureSessionSlot, +): DurableCaptureSessionBinding { + const requireSession = (): S => { + const session = store.get(address); + if (!session) throw new AppError('COMMAND_FAILED', 'Test session retired'); + return session; + }; + const assertAdoptable = (): void => { + if (slot.read(requireSession())) throw new AppError('COMMAND_FAILED', 'Test resource changed'); + }; + return Object.freeze({ + address, + sessionDir: store.resolveSessionDir(address), + read: () => { + const session = store.get(address); + return session === undefined ? undefined : slot.read(session); + }, + assertAdoptable, + canPersist: () => { + const session = store.get(address); + return session !== undefined && slot.read(session) === undefined; + }, + adopt: (resource) => { + assertAdoptable(); + store.set(address, slot.replace(requireSession(), resource)); + }, + clear: (expected) => { + const current = store.get(address); + if (!current) return 'retired'; + if (slot.read(current)?.handle !== expected.handle) return 'resource-changed'; + store.set(address, slot.replace(current, undefined)); + return 'cleared'; + }, + }); +} diff --git a/packages/platform-android/src/recording/failed-finish.test.ts b/packages/platform-android/src/recording/failed-finish.test.ts index 4c4984020c..d3f7afb0aa 100644 --- a/packages/platform-android/src/recording/failed-finish.test.ts +++ b/packages/platform-android/src/recording/failed-finish.test.ts @@ -13,7 +13,6 @@ import { createDurableCaptureResourceStore, finishLiveDurableCapture, type DurableCaptureResourceDefinition, - type DurableCaptureSessionStore, } from '@agent-device/capture-kit/durable-capture'; import { mkdtempForTestSync } from '../__tests__/test-utils/tmp-dir.ts'; import { androidRecordingDevice, recordingHost, recordingInput } from './fixtures.ts'; @@ -150,20 +149,36 @@ async function adoptAndroidRecording(params: { }; const sessionsDir = mkdtempForTestSync('agent-device-android-failed-finish-session-'); let session: AndroidRecordingSession = {}; - const sessionStore: DurableCaptureSessionStore = { - set: (_name, next) => { + const sessionStore = { + get: () => session, + set: (_name: string, next: AndroidRecordingSession) => { session = next; }, - resolveSessionDir: (name) => path.join(sessionsDir, name), + resolveSessionDir: (name: string) => path.join(sessionsDir, name), }; - const resourcePath = store.resolvePath(sessionStore.resolveSessionDir(params.sessionName)); + const binding = { + address: params.sessionName, + sessionDir: sessionStore.resolveSessionDir(params.sessionName), + read: () => session.recording, + assertAdoptable: () => { + if (session.recording) throw new Error('Already recording'); + }, + canPersist: () => !session.recording, + adopt: (recording: AndroidRecordingSession['recording']) => { + session = { ...session, recording }; + }, + clear: (expected: NonNullable) => { + if (session.recording?.handle !== expected.handle) return 'resource-changed' as const; + session = { ...session, recording: undefined }; + return 'cleared' as const; + }, + }; + const resourcePath = store.resolvePath(binding.sessionDir); await adoptStartedDurableCapture( definition, { reportUndurableCleanup: () => {}, - session, - sessionName: params.sessionName, - sessionStore, + binding, device: androidRecordingDevice, owner: params.owner, fence: params.envelope.fence, diff --git a/src/daemon/__tests__/app-log-session-resource.test.ts b/src/daemon/__tests__/app-log-session-resource.test.ts index 84a3add4f2..eb9402d3ba 100644 --- a/src/daemon/__tests__/app-log-session-resource.test.ts +++ b/src/daemon/__tests__/app-log-session-resource.test.ts @@ -88,7 +88,7 @@ test('SessionStore failure after transfer disposes the transferred handle and pr const context = makeContext(); const runtime = makeStartResult(context); const primary = new Error('store adoption failed'); - vi.spyOn(context.sessionStore, 'set').mockImplementationOnce(() => { + vi.spyOn(context.sessionStore, 'update').mockImplementationOnce(() => { throw primary; }); await expect( @@ -315,6 +315,77 @@ test('app-log disposes on a failed finish because its retry is that same finish ).not.toThrow(); }); +test('shutdown admission rejects a late start while its existing session still occupies the address', async () => { + const context = makeContext(); + const runtime = makeStartResult(context); + context.sessionStore.closeAdmission(); + await expect( + adoptStartedSessionAppLog({ ...context, ...runtime.result, throwIfCanceled: () => {} }), + ).rejects.toMatchObject({ details: { reason: 'daemon_shutting_down' } }); + expect(runtime.forceCleanup).toHaveBeenCalledOnce(); + expect(context.sessionStore.requireCurrent(context.ref).appLog).toBeUndefined(); + expect(appLogResourceStore.read(context.resourcePath)).toMatchObject({ + status: 'decoded', + envelope: { lifecycle: 'completed' }, + }); +}); + +test('failed adoption cannot terminalize successor evidence after its cleanup yields', async () => { + const context = makeContext(); + const runtime = makeStartResult(context); + let release!: () => void; + let entered!: () => void; + const held = new Promise((resolve) => { + release = resolve; + }); + const cleaning = new Promise((resolve) => { + entered = resolve; + }); + runtime.forceCleanup.mockImplementationOnce(async () => { + entered(); + await held; + return { status: 'cleaned' }; + }); + const canceled = new AppError('CANCELED', 'canceled'); + const adoption = adoptStartedSessionAppLog({ + ...context, + ...runtime.result, + throwIfCanceled: () => { + throw canceled; + }, + }); + const rejected = expect(adoption).rejects.toBe(canceled); + await cleaning; + context.sessionStore.retire(context.ref); + const successor = context.sessionStore.publish(context.sessionName, { ...context.session }); + appLogResourceStore.write(context.resourcePath, runtime.result.envelope); + release(); + await rejected; + expect(context.sessionStore.requireCurrent(successor).appLog).toBeUndefined(); + expect(appLogResourceStore.read(context.resourcePath)).toMatchObject({ + status: 'decoded', + envelope: { lifecycle: 'open' }, + }); +}); + +test('late adoption disposes its pending handle without overwriting a successor manifest', async () => { + const context = makeContext(); + const runtime = makeStartResult(context); + context.sessionStore.retire(context.ref); + const successor = context.sessionStore.publish(context.sessionName, { ...context.session }); + const envelope = { ...runtime.result.envelope, fence: { token: 'successor', generation: 2 } }; + appLogResourceStore.write(context.resourcePath, envelope); + await expect( + adoptStartedSessionAppLog({ ...context, ...runtime.result, throwIfCanceled: () => {} }), + ).rejects.toMatchObject({ details: { reason: 'session_lifetime_ended' } }); + expect(runtime.forceCleanup).toHaveBeenCalledOnce(); + expect(context.sessionStore.requireCurrent(successor).appLog).toBeUndefined(); + expect(appLogResourceStore.read(context.resourcePath)).toMatchObject({ + status: 'decoded', + envelope, + }); +}); + function makeContext( device: DeviceInfo = { platform: 'android', @@ -335,6 +406,7 @@ function makeContext( const resourcePath = appLogResourceStore.resolvePath(sessionStore.resolveSessionDir(sessionName)); return { admissionLedger: createAppLogAdmissionLedger(), + ref: sessionStore.lookup(sessionName)!, session, sessionName, sessionStore, diff --git a/src/daemon/__tests__/perf-capture-session-resource.test.ts b/src/daemon/__tests__/perf-capture-session-resource.test.ts index cd4010c381..6d7d4e8841 100644 --- a/src/daemon/__tests__/perf-capture-session-resource.test.ts +++ b/src/daemon/__tests__/perf-capture-session-resource.test.ts @@ -1,3 +1,4 @@ +import { bindSessionPerfCapture } from '../perf-capture-session-binding.ts'; import { beforeEach, expect, test, vi } from 'vitest'; import { localRuntimeOwner } from '@agent-device/contracts/platform-runtime'; import { AppError } from '@agent-device/kernel/errors'; @@ -112,9 +113,7 @@ test('a perf stop whose pull failed re-collects the device-side trace the first }); await adoptStartedPerfCapture({ admissionLedger: createPerfCaptureAdmissionLedger(), - session, - sessionName, - sessionStore, + binding: bindSessionPerfCapture(sessionStore, sessionStore.lookup(sessionName)!), device, owner: localRuntimeOwner('android'), fence, diff --git a/src/daemon/__tests__/screen-recording-session-binding.test.ts b/src/daemon/__tests__/screen-recording-session-binding.test.ts new file mode 100644 index 0000000000..520b6b3452 --- /dev/null +++ b/src/daemon/__tests__/screen-recording-session-binding.test.ts @@ -0,0 +1,60 @@ +import { expect, test, vi } from 'vitest'; +import { createScreenRecordingLiveHandle } from '@agent-device/capture-kit'; +import { PendingTransferGuard } from '@agent-device/contracts/async-lifecycle'; +import { makeSessionStore } from '../../__tests__/test-utils/store-factory.ts'; +import { makeRecordingSession } from './session-teardown.fixtures.ts'; +import { bindRecordOnlyScreenRecording } from '../screen-recording-session-binding.ts'; +import { createScreenRecordingAdmissionLedger } from '@agent-device/capture-kit/screen-recording-admission-ledger'; +import { + adoptStartedScreenRecording, + screenRecordingDurableResource, +} from '@agent-device/capture-kit/screen-recording-session-resource'; + +test('shutdown refuses draft publication while retaining unconfirmed recording cleanup evidence', async () => { + const store = makeSessionStore(); + const session = makeRecordingSession({ + name: 'draft', + sessionStore: store, + finish: async () => ({ status: 'cleanup-pending', reason: 'cleanup-unconfirmed' }), + }); + const { handle: initial, envelope } = session.screenRecording!; + const cleanup = vi.fn( + async () => ({ status: 'cleanup-pending', reason: 'cleanup-unconfirmed' }) as const, + ); + const handle = createScreenRecordingLiveHandle(initial.inspect(), { + finish: async () => ({ status: 'cleanup-pending', reason: 'cleanup-unconfirmed' }), + forceCleanup: cleanup, + }); + const draft = bindRecordOnlyScreenRecording(store, 'draft', { + ...session, + screenRecording: undefined, + }); + store.closeAdmission(); + await expect( + adoptStartedScreenRecording({ + binding: draft.binding, + admissionLedger: createScreenRecordingAdmissionLedger(), + device: session.device, + owner: envelope.owner, + fence: envelope.fence, + pendingHandle: new PendingTransferGuard(handle), + envelope, + throwIfCanceled: () => {}, + }), + ).rejects.toMatchObject({ details: { reason: 'daemon_shutting_down' } }); + expect(cleanup).toHaveBeenCalledOnce(); + expect(store.lookup('draft')).toBeUndefined(); + const record = screenRecordingDurableResource.store.read( + screenRecordingDurableResource.store.resolvePath(draft.binding.sessionDir), + ); + expect(record).toMatchObject({ + status: 'decoded', + envelope: { + lifecycle: 'open', + descriptor: envelope.descriptor, + metadata: { phase: 'cleanup-pending' }, + }, + }); + if (record.status !== 'decoded') throw new Error('Expected recovery evidence'); + expect(record.envelope.metadata?.runtimeContractInvalid).toBeUndefined(); +}); diff --git a/src/daemon/__tests__/session-capture-binding.test.ts b/src/daemon/__tests__/session-capture-binding.test.ts new file mode 100644 index 0000000000..41f5322d5b --- /dev/null +++ b/src/daemon/__tests__/session-capture-binding.test.ts @@ -0,0 +1,72 @@ +import { expect, test } from 'vitest'; +import { makeSessionStore } from '../../__tests__/test-utils/store-factory.ts'; +import { makeRecordingSession } from './session-teardown.fixtures.ts'; +import { bindSessionScreenRecording } from '../screen-recording-session-binding.ts'; + +test('clearing a capture refreshes a rebuilt record without losing its other changes', () => { + const store = makeSessionStore(); + const session = makeRecordingSession({ + name: 'capture', + sessionStore: store, + finish: async () => ({ status: 'cleanup-pending', reason: 'cleanup-unconfirmed' }), + }); + const ref = store.publish('capture', session); + const binding = bindSessionScreenRecording(store, ref); + const active = binding.read()!; + store.update(ref, { appName: 'updated', screenRecording: { ...active } }); + expect(binding.clear(active)).toBe('cleared'); + expect(store.requireCurrent(ref)).toMatchObject({ + appName: 'updated', + screenRecording: undefined, + }); +}); + +test('clearing an older handle or fence leaves a replacement capture intact', () => { + const store = makeSessionStore(); + const session = makeRecordingSession({ + name: 'capture', + sessionStore: store, + finish: async () => ({ status: 'cleanup-pending', reason: 'cleanup-unconfirmed' }), + }); + const ref = store.publish('capture', session); + const binding = bindSessionScreenRecording(store, ref); + const active = binding.read()!; + const replacement = makeRecordingSession({ + name: 'other', + sessionStore: store, + finish: async () => ({ status: 'cleanup-pending', reason: 'cleanup-unconfirmed' }), + }).screenRecording!; + store.update(ref, { screenRecording: replacement }); + expect(binding.clear(active)).toBe('resource-changed'); + expect(binding.read()).toBe(replacement); + const newerFence = { + ...active, + envelope: { ...active.envelope, fence: { token: 'next', generation: 2 } }, + }; + store.update(ref, { screenRecording: newerFence }); + expect(binding.clear(active)).toBe('resource-changed'); + expect(binding.read()).toBe(newerFence); +}); + +test('a retired binding retains its old resource but cannot write into the next lifetime', () => { + const store = makeSessionStore(); + const session = makeRecordingSession({ + name: 'capture', + sessionStore: store, + finish: async () => ({ status: 'cleanup-pending', reason: 'cleanup-unconfirmed' }), + }); + const ref = store.publish('capture', session); + const binding = bindSessionScreenRecording(store, ref); + const active = binding.read()!; + store.retire(ref); + const successor = store.publish('capture', session); + expect(binding.read()).toBe(active); + expect(binding.clear(active)).toBe('retired'); + expect(binding.canPersist()).toBe(false); + expect(() => binding.adopt(active)).toThrow( + expect.objectContaining({ + details: expect.objectContaining({ reason: 'session_lifetime_ended' }), + }), + ); + expect(store.requireCurrent(successor).screenRecording).toBe(active); +}); diff --git a/src/daemon/app-log-session-resource.ts b/src/daemon/app-log-session-resource.ts index e86d50f165..5c3993e19b 100644 --- a/src/daemon/app-log-session-resource.ts +++ b/src/daemon/app-log-session-resource.ts @@ -15,7 +15,8 @@ import { } from '@agent-device/capture-kit/durable-capture-resource'; import { appLogResourceStore } from './app-log-resource-store.ts'; import type { SessionStore } from './session-store.ts'; -import type { SessionState } from './session-state.ts'; +import type { SessionRef, SessionState } from './session-state.ts'; +import { bindSessionCapture } from './session-capture-binding.ts'; export type AppLogSessionSnapshot = Readonly<{ active: boolean; @@ -76,10 +77,8 @@ export function inspectSessionAppLog(session: SessionState): AppLogSessionSnapsh export function adoptStartedSessionAppLog(params: { admissionLedger: AppLogAdmissionLedger; - session: SessionState; - sessionName: string; + ref: SessionRef; sessionStore: SessionStore; - resourcePath: string; device: DeviceInfo; owner: RuntimeOwnerRef; fence: ResourceOwnershipFence; @@ -87,7 +86,10 @@ export function adoptStartedSessionAppLog(params: { envelope: DurableResourceEnvelope<'app-log'>; throwIfCanceled(): void; }): Promise { - return appLogDurableResource.adoptStarted(params); + return appLogDurableResource.adoptStarted({ + ...params, + binding: bindSessionAppLog(params.sessionStore, params.ref), + }); } export function finishSessionAppLog(params: { @@ -110,15 +112,15 @@ export function forceCleanupSessionAppLog(params: { } export function recordSessionAppLogFailure(params: { - session: SessionState; - sessionName: string; + ref: SessionRef; sessionStore: SessionStore; error: unknown; backend?: LogBackend; }): ReturnType { const normalized = normalizeError(params.error); - params.sessionStore.set(params.sessionName, { - ...params.session, + const current = params.sessionStore.resolveCurrent(params.ref); + if (!current || current.appLog) return normalized; + params.sessionStore.update(params.ref, { appLog: undefined, appLogFailure: { backend: params.backend, @@ -131,12 +133,19 @@ export function recordSessionAppLogFailure(params: { } export function clearSessionAppLogFailure(params: { - session: SessionState; - sessionName: string; + ref: SessionRef; sessionStore: SessionStore; }): void { - params.sessionStore.set(params.sessionName, { - ...params.session, + params.sessionStore.update(params.ref, { appLogFailure: undefined, }); } + +function bindSessionAppLog(sessionStore: SessionStore, ref: SessionRef) { + return bindSessionCapture(sessionStore, ref, { + read: (session) => session.appLog, + write: (appLog) => { + sessionStore.update(ref, { appLog, appLogFailure: undefined }); + }, + }); +} diff --git a/src/daemon/audio-probe-session-binding.ts b/src/daemon/audio-probe-session-binding.ts new file mode 100644 index 0000000000..b4eecb43c7 --- /dev/null +++ b/src/daemon/audio-probe-session-binding.ts @@ -0,0 +1,12 @@ +import { bindSessionCapture } from './session-capture-binding.ts'; +import type { SessionRef } from './session-state.ts'; +import type { SessionStore } from './session-store.ts'; + +export function bindSessionAudioProbe(sessionStore: SessionStore, ref: SessionRef) { + return bindSessionCapture(sessionStore, ref, { + read: (session) => session.audioProbe, + write: (audioProbe) => { + sessionStore.update(ref, { audioProbe }); + }, + }); +} diff --git a/src/daemon/handlers/record-runtime.ts b/src/daemon/handlers/record-runtime.ts index 758bd95bc4..1e63e684cb 100644 --- a/src/daemon/handlers/record-runtime.ts +++ b/src/daemon/handlers/record-runtime.ts @@ -30,7 +30,11 @@ import { resolveSessionScope } from '../session-routing.ts'; import type { SessionStore } from '../session-store.ts'; import type { BindDeviceRuntime, BindExactDeviceRuntime } from '../request-runtime-binding.ts'; import type { DaemonRequest, DaemonResponse } from '../daemon-request.ts'; -import type { SessionState } from '../session-state.ts'; +import type { SessionRef, SessionState } from '../session-state.ts'; +import { + bindRecordOnlyScreenRecording, + bindSessionScreenRecording, +} from '../screen-recording-session-binding.ts'; import { recordSessionAction } from '../session-action-recorder.ts'; import { missingAppSessionResponse, @@ -80,17 +84,19 @@ async function handleRecordCommandUnsafe( params: RecordRuntimeHandlerParams, ): Promise { const { req, sessionName, sessionStore } = params; - const existingSession = sessionStore.get(sessionName); + const existingRef = sessionStore.lookup(sessionName); + const existingSession = existingRef?.session; const { plan, scope } = resolveRecordPlan(req, existingSession); if (plan.kind === 'start' && !isWholeScreenRecordingScope(scope) && !existingSession) { return missingAppSessionResponse(req); } - const resolvedSession = await resolveRecordingSession(params, existingSession); + const resolvedSession = await resolveRecordingSession(params, existingRef); const { session } = resolvedSession; if (plan.kind === 'start') { return await startRecording( params, session, + resolvedSession.ref, prepareRecordingRequest(req), plan.use, resolvedSession.needsReadiness, @@ -113,17 +119,18 @@ function resolveRecordPlan(req: DaemonRequest, session: SessionState | undefined async function resolveRecordingSession( params: RecordRuntimeHandlerParams, - existing: SessionState | undefined, -): Promise> { - const device = existing?.device ?? (await resolveTargetDevice(params.req.flags ?? {})); + ref: SessionRef | undefined, +): Promise> { + const device = ref?.session.device ?? (await resolveTargetDevice(params.req.flags ?? {})); await params.retainDeviceExecutionLock(device.id); - if (existing) return { session: existing, needsReadiness: false }; + if (ref) return { session: params.sessionStore.requireCurrent(ref), ref, needsReadiness: false }; return { session: createRecordOnlySession(params, device), needsReadiness: true }; } async function startRecording( params: RecordRuntimeHandlerParams, session: SessionState, + ref: SessionRef | undefined, prepared: ReturnType, use: typeof screenRecordingStartUse, needsReadiness: boolean, @@ -131,6 +138,11 @@ async function startRecording( if (session.screenRecording) { return { ok: false, error: { code: 'INVALID_ARGS', message: 'recording already in progress' } }; } + const draft = ref + ? undefined + : bindRecordOnlyScreenRecording(params.sessionStore, params.sessionName, session); + const binding = ref ? bindSessionScreenRecording(params.sessionStore, ref) : draft!.binding; + binding.assertAdoptable(); const admission = await params.bindDevice(session.device, screenRecordingAdmissionUse); if (needsReadiness) await ensureBoundDeviceReady(admission); const startFact = admission.facts.screenRecordingStart; @@ -142,21 +154,20 @@ async function startRecording( ); await adoptStartedScreenRecording({ admissionLedger: params.admissionLedger, - session, - sessionName: params.sessionName, - sessionStore: params.sessionStore, + binding, device: session.device, owner: runtime.owner, fence, ...started, throwIfCanceled: params.throwIfCanceled, }); - const adopted = params.sessionStore.get(params.sessionName)?.screenRecording; + const adoptedRef = ref ?? draft!.requireRef(); + const adopted = binding.read(); if (!adopted) throw new TypeError('Screen recording adoption did not publish a live handle'); const snapshot = adopted.handle.inspect(); recordSessionAction( params.sessionStore, - session, + params.sessionStore.requireCurrent(adoptedRef), params.req, params.req.command, buildRecordingStartedAction(snapshot), diff --git a/src/daemon/perf-capture-session-binding.ts b/src/daemon/perf-capture-session-binding.ts new file mode 100644 index 0000000000..53463cd8ab --- /dev/null +++ b/src/daemon/perf-capture-session-binding.ts @@ -0,0 +1,12 @@ +import { bindSessionCapture } from './session-capture-binding.ts'; +import type { SessionRef } from './session-state.ts'; +import type { SessionStore } from './session-store.ts'; + +export function bindSessionPerfCapture(sessionStore: SessionStore, ref: SessionRef) { + return bindSessionCapture(sessionStore, ref, { + read: (session) => session.perfCapture, + write: (perfCapture) => { + sessionStore.update(ref, { perfCapture }); + }, + }); +} diff --git a/src/daemon/screen-recording-session-binding.ts b/src/daemon/screen-recording-session-binding.ts new file mode 100644 index 0000000000..d032f819ba --- /dev/null +++ b/src/daemon/screen-recording-session-binding.ts @@ -0,0 +1,42 @@ +import { bindSessionCapture } from './session-capture-binding.ts'; +import type { SessionRef } from './session-state.ts'; +import type { SessionStore } from './session-store.ts'; + +export function bindSessionScreenRecording(sessionStore: SessionStore, ref: SessionRef) { + return bindSessionCapture(sessionStore, ref, { + read: (session) => session.screenRecording, + write: (screenRecording) => { + sessionStore.update(ref, { screenRecording }); + }, + }); +} + +export function bindRecordOnlyScreenRecording( + sessionStore: SessionStore, + address: string, + draft: SessionRef['session'], +) { + let published: SessionRef | undefined; + const binding: ReturnType = Object.freeze({ + address, + sessionDir: sessionStore.resolveSessionDir(address), + read: () => + published ? bindSessionScreenRecording(sessionStore, published).read() : undefined, + assertAdoptable: () => sessionStore.assertPublishable(address), + canPersist: () => !published && sessionStore.lookup(address) === undefined, + adopt: (screenRecording) => { + sessionStore.assertPublishable(address); + draft.screenRecording = screenRecording; + published = sessionStore.publish(address, draft); + }, + clear: (expected) => + published ? bindSessionScreenRecording(sessionStore, published).clear(expected) : 'retired', + }); + return Object.freeze({ + binding, + requireRef: (): SessionRef => { + if (!published) throw new TypeError('Screen recording did not publish its session'); + return published; + }, + }); +} diff --git a/src/daemon/session-capture-binding.ts b/src/daemon/session-capture-binding.ts new file mode 100644 index 0000000000..c3f809cf37 --- /dev/null +++ b/src/daemon/session-capture-binding.ts @@ -0,0 +1,53 @@ +import type { + DurableCaptureSessionBinding, + DurableCaptureSessionResource, +} from '@agent-device/capture-kit/durable-capture'; +import { AppError } from '@agent-device/kernel/errors'; +import type { SessionRef, SessionState } from './session-state.ts'; +import type { SessionStore } from './session-store.ts'; + +export function bindSessionCapture( + sessionStore: SessionStore, + ref: SessionRef, + slot: Readonly<{ + read(session: SessionState): DurableCaptureSessionResource | undefined; + write(resource: DurableCaptureSessionResource | undefined): void; + }>, +): DurableCaptureSessionBinding { + const assertAdoptable = (): void => { + sessionStore.assertAdmissionOpen(ref.address); + if (slot.read(sessionStore.requireCurrent(ref))) { + throw new AppError('COMMAND_FAILED', 'Session capture resource has changed', { + reason: 'session_resource_changed', + session: ref.address, + }); + } + }; + return Object.freeze({ + address: ref.address, + sessionDir: sessionStore.resolveSessionDir(ref.address), + read: () => slot.read(sessionStore.resolveCurrent(ref) ?? ref.session), + assertAdoptable, + canPersist: () => { + const current = sessionStore.resolveCurrent(ref); + return current !== undefined && slot.read(current) === undefined; + }, + adopt: (resource) => { + assertAdoptable(); + slot.write(resource); + }, + clear: (expected) => { + const current = sessionStore.resolveCurrent(ref); + if (!current) return 'retired'; + const active = slot.read(current); + if ( + active?.handle !== expected.handle || + active.envelope.fence.token !== expected.envelope.fence.token || + active.envelope.fence.generation !== expected.envelope.fence.generation + ) + return 'resource-changed'; + slot.write(undefined); + return 'cleared'; + }, + }); +} diff --git a/src/daemon/session-observability/internal/__tests__/session-logs.test.ts b/src/daemon/session-observability/internal/__tests__/session-logs.test.ts index fb40412fe7..f356a1c7a2 100644 --- a/src/daemon/session-observability/internal/__tests__/session-logs.test.ts +++ b/src/daemon/session-observability/internal/__tests__/session-logs.test.ts @@ -269,7 +269,7 @@ test('rejected pending cleanup retains cleanup-pending record and blocks replace test('post-transfer SessionStore failure disposes the transferred handle and preserves primary error', async () => { const { sessionStore, sessionName } = openSession(); const primary = new Error('store adoption failed'); - vi.spyOn(sessionStore, 'set').mockImplementationOnce(() => { + vi.spyOn(sessionStore, 'update').mockImplementationOnce(() => { throw primary; }); const response = await runLogs(sessionStore, sessionName, ['start'], {}, runtime.bindDevice); diff --git a/src/daemon/session-observability/internal/session-audio.ts b/src/daemon/session-observability/internal/session-audio.ts index a7e60b98c7..6d439e65b5 100644 --- a/src/daemon/session-observability/internal/session-audio.ts +++ b/src/daemon/session-observability/internal/session-audio.ts @@ -20,7 +20,8 @@ import type { } from '../../request-runtime-binding.ts'; import type { SessionStore } from '../../session-store.ts'; import type { DaemonRequest, DaemonResponse } from '../../daemon-request.ts'; -import type { SessionState } from '../../session-state.ts'; +import type { SessionRef, SessionState } from '../../session-state.ts'; +import { bindSessionAudioProbe } from '../../audio-probe-session-binding.ts'; import { type DaemonFailureResponse, errorResponse } from '@agent-device/kernel/contracts'; type AudioParams = { @@ -44,7 +45,8 @@ export async function handleAudioCommand(params: AudioParams): Promise { const sessionResult = resolveAudioSession(params); if (!sessionResult.ok) return sessionResult; - const session = sessionResult.session; + const ref = sessionResult.ref; + const session = params.sessionStore.requireCurrent(ref); const request = parseAudioProbeRequest(params.req.positionals); // Facts, not a capability bucket, decide which of the two owner paths this device has — // side-effect-free per ADR 0019 §9; the one bind below uses the plan's own use. @@ -67,7 +69,7 @@ async function handleAudioCommandUnsafe(params: AudioParams): Promise { + let session = params.sessionStore.requireCurrent(ref); + const binding = bindSessionAudioProbe(params.sessionStore, ref); // Start restarts an already-running probe (legacy parity), completing it through the durable // coordinator so the previous envelope terminalizes before a new fence is minted. Nobody reads // that completion, so the previous probe is being handed back rather than captured. @@ -110,12 +114,10 @@ async function startAudioProbe( await finishLiveAudioProbe({ intent: 'disposal', session, - sessionName: params.sessionName, + sessionName: ref.address, sessionStore: params.sessionStore, }); - const refreshed = params.sessionStore.get(params.sessionName); - if (!refreshed) return errorResponse('SESSION_NOT_FOUND', 'audio requires an active session'); - session = refreshed; + session = params.sessionStore.requireCurrent(ref); } const runtime = await params.bindDevice(session.device, use); const resourcePath = audioProbeDurableResource.store.resolvePath( @@ -139,16 +141,14 @@ async function startAudioProbe( }); await adoptStartedAudioProbe({ admissionLedger: params.audioProbeAdmissionLedger, - session, - sessionName: params.sessionName, - sessionStore: params.sessionStore, + binding, device: session.device, owner: runtime.owner, fence, ...started, throwIfCanceled: params.throwIfCanceled, }); - const adopted = params.sessionStore.get(params.sessionName)?.audioProbe; + const adopted = binding.read(); if (!adopted) throw new TypeError('Audio probe adoption did not publish a live handle'); return { ok: true, data: await adopted.handle.status() }; } diff --git a/src/daemon/session-observability/internal/session-observability.ts b/src/daemon/session-observability/internal/session-observability.ts index 2171842c39..2e98bd25de 100644 --- a/src/daemon/session-observability/internal/session-observability.ts +++ b/src/daemon/session-observability/internal/session-observability.ts @@ -31,7 +31,7 @@ import type { } from '../../request-runtime-binding.ts'; import type { SessionStore } from '../../session-store.ts'; import type { DaemonRequest, DaemonResponse } from '../../daemon-request.ts'; -import type { SessionState } from '../../session-state.ts'; +import type { SessionRef, SessionState } from '../../session-state.ts'; import { handleAudioCommand } from './session-audio.ts'; import { handlePerfRuntimeCommand } from './session-perf-runtime.ts'; import { handleNetworkCommand } from './session-network.ts'; @@ -51,6 +51,7 @@ export type SessionObservabilityCommandInput = { type ObservabilityInput = SessionObservabilityCommandInput; type LogsHandlerParams = Omit & { session: SessionState; + ref: SessionRef; bindDevice: BindDeviceRuntime; appLogAdmissionLedger: AppLogAdmissionLedger; }; @@ -143,12 +144,13 @@ async function handleEventsCommand(params: ObservabilityInput): Promise { const { req, sessionName, sessionStore } = params; - const session = sessionStore.get(sessionName); - if (!session) { + const ref = sessionStore.lookup(sessionName); + if (!ref) { return errorResponse('SESSION_NOT_FOUND', 'logs requires an active session'); } try { - const logsParams = requireLogsHandlerParams({ ...params, session }); + const session = sessionStore.requireCurrent(ref); + const logsParams = requireLogsHandlerParams({ ...params, session, ref }); const admission = await logsParams.bindDevice(session.device, appLogAdmissionUse); const inspectFact = admission.facts.appLogInspect; if (!inspectFact.available) { @@ -305,7 +307,7 @@ function handleLogsClear(params: LogsHandlerParams): DaemonResponse { } const logPath = sessionStore.resolveAppLogPath(sessionName); const cleared = clearAppLogFiles(logPath); - clearSessionAppLogFailure({ session, sessionName, sessionStore }); + clearSessionAppLogFailure({ ref: params.ref, sessionStore }); return { ok: true, data: cleared }; } @@ -386,10 +388,8 @@ async function startSessionAppLog( }); await adoptStartedSessionAppLog({ admissionLedger: params.appLogAdmissionLedger, - session, - sessionName, + ref: params.ref, sessionStore, - resourcePath, device: session.device, owner, fence, @@ -400,8 +400,7 @@ async function startSessionAppLog( return { ok: true, data: { path: outputPath, started: true } }; } catch (error) { const normalized = recordSessionAppLogFailure({ - session, - sessionName, + ref: params.ref, sessionStore, error, }); @@ -431,7 +430,7 @@ function requireAudioSeams(params: ObservabilityInput): Parameters { - const session = params.sessionStore.get(params.sessionName); - if (!session) { + const ref = params.sessionStore.lookup(params.sessionName); + if (!ref) { return errorResponse('SESSION_NOT_FOUND', 'perf requires an active session. Run open first.'); } + const session = params.sessionStore.requireCurrent(ref); + const bound = { ...params, ref }; try { if (isRemovedAggregatePerfToken(params.req.positionals?.[0])) { throw new AppError('INVALID_ARGS', PERF_AGGREGATE_REMOVED_ERROR_MESSAGE); @@ -89,7 +92,7 @@ export async function handlePerfRuntimeCommand( } return recordSuccessfulPerfResponse( params, - await executeAdmittedPerfPlan(params, session, admitted), + await executeAdmittedPerfPlan(bound, session, admitted), ); } catch (error) { return { ok: false, error: normalizeError(error) }; @@ -116,7 +119,7 @@ function recordSuccessfulPerfResponse( // the admission/runtime join this handler is meant to keep singular. // fallow-ignore-next-line complexity async function executeAdmittedPerfPlan( - params: PerfRuntimeHandlerParams, + params: PerfRuntimeHandlerParams & { ref: SessionRef }, session: SessionState, admission: AdmittedRuntimePlan>, ): Promise { @@ -183,7 +186,7 @@ async function executeAdmittedPerfPlan( } async function startPerfCapture( - params: PerfRuntimeHandlerParams, + params: PerfRuntimeHandlerParams & { ref: SessionRef }, session: SessionState, runtime: Readonly<{ owner: Parameters[0]['owner']; @@ -225,9 +228,7 @@ async function startPerfCapture( }); await adoptStartedPerfCapture({ admissionLedger: requirePerfCaptureAdmissionLedger(params), - session, - sessionName: params.sessionName, - sessionStore: params.sessionStore, + binding: bindSessionPerfCapture(params.sessionStore, params.ref), device: session.device, owner: runtime.owner, fence, diff --git a/src/daemon/session-store.ts b/src/daemon/session-store.ts index 9e7b1c2d8d..97e2d6dc4b 100644 --- a/src/daemon/session-store.ts +++ b/src/daemon/session-store.ts @@ -73,19 +73,27 @@ export class SessionStore { this.acceptingSessions = false; } - publish(address: string, session: SessionState): SessionRef { + assertAdmissionOpen(address: string): void { if (!this.acceptingSessions) { throw new AppError('COMMAND_FAILED', 'Daemon is shutting down', { reason: 'daemon_shutting_down', session: address, }); } + } + + assertPublishable(address: string): void { + this.assertAdmissionOpen(address); if (this.sessions.has(address)) { throw new AppError('COMMAND_FAILED', 'Session address is already occupied', { reason: 'session_address_occupied', session: address, }); } + } + + publish(address: string, session: SessionState): SessionRef { + this.assertPublishable(address); const entry = { current: session }; this.sessions.set(address, entry); this.clearIdleExpiryTombstone(address); From bfffb65f017ce0a8289150bc5e00127ccde3cbf6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sat, 3 Oct 2026 06:33:56 +0200 Subject: [PATCH 18/20] chore(gates): declare capture adoption field owners --- packages/capture-kit/src/durable-capture/index.ts | 1 + scripts/layering/session-resource-ownership.test.ts | 2 ++ scripts/layering/session-resource-ownership.ts | 7 +++++++ 3 files changed, 10 insertions(+) diff --git a/packages/capture-kit/src/durable-capture/index.ts b/packages/capture-kit/src/durable-capture/index.ts index 09013e180a..dc4349e202 100644 --- a/packages/capture-kit/src/durable-capture/index.ts +++ b/packages/capture-kit/src/durable-capture/index.ts @@ -14,6 +14,7 @@ export type { DurableCaptureRecordDefinition, DurableCaptureResourceDefinition, DurableCaptureSessionResource, + DurableCaptureSessionBinding, DurableCaptureSessionStore, } from './definition.ts'; export type { FinishRecoveredDurableCaptureParams } from './finish-recovered.ts'; diff --git a/scripts/layering/session-resource-ownership.test.ts b/scripts/layering/session-resource-ownership.test.ts index ffac560315..4b2e7fc2b8 100644 --- a/scripts/layering/session-resource-ownership.test.ts +++ b/scripts/layering/session-resource-ownership.test.ts @@ -19,6 +19,7 @@ test('session resources are constructed only by their durable domain owners', () appLogFailure: failure, audioProbe: audio, perfCapture: perf, + screenRecording: recording, });`, ], [ @@ -39,6 +40,7 @@ test('session resources are constructed only by their durable domain owners', () 'src/daemon/handlers/planted.ts: session appLogFailure record constructed outside its owner', 'src/daemon/handlers/planted.ts: session audioProbe record constructed outside its owner', 'src/daemon/handlers/planted.ts: session perfCapture record constructed outside its owner', + 'src/daemon/handlers/planted.ts: session screenRecording record constructed outside its owner', ], ); }); diff --git a/scripts/layering/session-resource-ownership.ts b/scripts/layering/session-resource-ownership.ts index 4990fdef9e..579dd3d637 100644 --- a/scripts/layering/session-resource-ownership.ts +++ b/scripts/layering/session-resource-ownership.ts @@ -29,10 +29,17 @@ const RESOURCE_OWNERS: Readonly>> = { appLogFailure: new Set(['src/daemon/app-log-session-resource.ts', 'src/daemon/session-state.ts']), audioProbe: new Set([ 'packages/capture-kit/src/capture-admission/audio-probe-session-resource.ts', + 'src/daemon/audio-probe-session-binding.ts', + 'src/daemon/session-state.ts', + ]), + screenRecording: new Set([ + 'packages/capture-kit/src/capture-admission/screen-recording-session-resource.ts', + 'src/daemon/screen-recording-session-binding.ts', 'src/daemon/session-state.ts', ]), perfCapture: new Set([ 'packages/capture-kit/src/capture-admission/perf-capture-session-resource.ts', + 'src/daemon/perf-capture-session-binding.ts', 'src/daemon/session-state.ts', ]), }; From ebecbda42605231121388f364ebff68d35140116 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sat, 3 Oct 2026 16:46:15 +0200 Subject: [PATCH 19/20] fix: name the capture clear outcome inline so session bindings emit declarations Daemon session bindings inferred a binding type whose clear result was an alias the durable-capture entry never exported, so declaration emit failed with TS2883. --- packages/capture-kit/src/durable-capture/definition.ts | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/packages/capture-kit/src/durable-capture/definition.ts b/packages/capture-kit/src/durable-capture/definition.ts index adf33b18e2..4b0aba2fdb 100644 --- a/packages/capture-kit/src/durable-capture/definition.ts +++ b/packages/capture-kit/src/durable-capture/definition.ts @@ -14,8 +14,6 @@ export type DurableCaptureSessionResource; }>; -export type DurableCaptureSlotClearResult = 'cleared' | 'retired' | 'resource-changed'; - export type DurableCaptureSessionBinding = Readonly<{ address: string; sessionDir: string; @@ -23,7 +21,7 @@ export type DurableCaptureSessionBinding): void; - clear(expected: DurableCaptureSessionResource): DurableCaptureSlotClearResult; + clear(expected: DurableCaptureSessionResource): 'cleared' | 'retired' | 'resource-changed'; }>; export type DurableCaptureSessionStore = Readonly<{ From cbf0a6e7d5665b5081200e8337cae72af7e514a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sat, 3 Oct 2026 18:45:46 +0200 Subject: [PATCH 20/20] perf: load app-log cleanup on demand in session teardown Binding app-log capture through the session capture binding pulled that module into session-teardown's eager closure. Teardown now imports the app-log resource only when a session has an app log to stop. --- src/daemon/session-teardown.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/daemon/session-teardown.ts b/src/daemon/session-teardown.ts index 377be3fb2b..2f051b2057 100644 --- a/src/daemon/session-teardown.ts +++ b/src/daemon/session-teardown.ts @@ -3,7 +3,6 @@ import { emitDiagnostic } from '@agent-device/host-kit/diagnostics'; import { cleanupRetainedMaterializedPathsForSession } from './materialized-path-registry.ts'; import type { SessionState } from './session-state.ts'; import type { SessionStore } from './session-store.ts'; -import { forceCleanupSessionAppLog } from './app-log-session-resource.ts'; import { appLogResourceStore } from './app-log-resource-store.ts'; import { finishLiveAudioProbe } from '@agent-device/capture-kit/audio-probe-session-resource'; import { finishLivePerfCapture } from '@agent-device/capture-kit/perf-capture-session-resource'; @@ -18,6 +17,7 @@ export async function stopSessionAppLog(params: { }): Promise { const { session, sessionName, sessionStore } = params; if (!session.appLog) return; + const { forceCleanupSessionAppLog } = await import('./app-log-session-resource.ts'); await forceCleanupSessionAppLog({ session, sessionName,