From 24d2e44d7a9598ff8288706678479db7cc5e272f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Fri, 2 Oct 2026 23:06:27 +0200 Subject: [PATCH 1/5] fix: bind daemon publication and retirement to its acquired registration lock --- .../daemon-registration-owner.test.ts | 164 ++++++++++++++ src/__tests__/daemon-registration.test.ts | 10 +- src/__tests__/daemon-shutdown-report.test.ts | 14 +- .../__tests__/daemon-client-metadata.test.ts | 20 +- src/daemon-registration-owner.ts | 161 +++++++++++++ src/daemon-shutdown-report.ts | 50 ++--- .../__tests__/daemon-runtime-app-log.test.ts | 6 +- .../filesystem-boundary-faults.test.ts | 13 +- .../daemon-runtime-metadata-ownership.test.ts | 25 ++- src/daemon/server/daemon-runtime.ts | 83 ++++--- src/daemon/server/server-lifecycle.test.ts | 211 ------------------ src/daemon/server/server-lifecycle.ts | 154 ------------- .../daemon-lifecycle.test.ts | 81 ++----- 13 files changed, 452 insertions(+), 540 deletions(-) create mode 100644 src/__tests__/daemon-registration-owner.test.ts create mode 100644 src/daemon-registration-owner.ts delete mode 100644 src/daemon/server/server-lifecycle.test.ts diff --git a/src/__tests__/daemon-registration-owner.test.ts b/src/__tests__/daemon-registration-owner.test.ts new file mode 100644 index 0000000000..9371681f89 --- /dev/null +++ b/src/__tests__/daemon-registration-owner.test.ts @@ -0,0 +1,164 @@ +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import { afterEach, test, vi } from 'vitest'; +import { readCurrentOwnerIdentity } from '@agent-device/host-kit/process'; +import { tryAcquireDaemonRegistration } from '../daemon-registration-owner.ts'; +import { resolveDaemonPaths, type DaemonPaths } from '../daemon-resolution.ts'; +import { readRegisteredDaemonOwnership } from '../daemon-registration.ts'; +import { readDaemonShutdownReport } from '../daemon-shutdown-report.ts'; +import { mkdtempForTestSync } from './test-utils/tmp-dir.ts'; + +const fields = { + socketPort: 4210, + token: 'token', + version: '0.0.0-test', + codeOrigin: 'checkout' as const, + codeSignature: 'signature', +}; +const ownIdentity = readCurrentOwnerIdentity(); +const report = { + providerReleases: { released: [], pending: [] }, + claims: { released: [], orphaned: [], superseded: [], unattributable: [] }, +}; + +afterEach(() => vi.restoreAllMocks()); + +async function acquire( + paths = resolveDaemonPaths(mkdtempForTestSync('agent-device-registration-owner-')), +) { + const attempt = await tryAcquireDaemonRegistration(paths); + assert.equal(attempt.status, 'acquired'); + if (attempt.status !== 'acquired') throw new Error('registration refused'); + return { paths, owner: attempt.owner }; +} + +function replaceInfo(paths: DaemonPaths, pid: unknown, startTime?: string | null) { + fs.writeFileSync( + paths.infoPath, + JSON.stringify({ pid, processStartTime: startTime, token: 'token', port: 4210 }), + ); +} + +test('publication is atomic, binds identity and paths, and excludes a second acquisition', async () => { + const { paths, owner } = await acquire(); + owner.publish(fields); + const before = fs.statSync(paths.infoPath).ino; + owner.publish({ ...fields, httpPort: 4310 }); + assert.notEqual(fs.statSync(paths.infoPath).ino, before); + assert.equal(readRegisteredDaemonOwnership(paths.infoPath, ownIdentity).state, 'match'); + assert.equal(JSON.parse(fs.readFileSync(paths.infoPath, 'utf8')).transport, 'dual'); + assert.equal((await tryAcquireDaemonRegistration(paths)).status, 'busy'); + assert.deepEqual(await owner.finish(report), { state: 'removed' }); + assert.equal(fs.existsSync(paths.infoPath), false); + assert.equal(fs.existsSync(paths.lockPath), false); + assert.deepEqual(readDaemonShutdownReport(paths.baseDir), report); +}); + +for (const [pid, startTime, reason] of [ + [999_999_999, 'successor-start', 'replaced'], + [process.pid, 'recycled-start', 'replaced'], + [process.pid, undefined, 'unproven'], + [0, undefined, 'ownerless'], + [-3, undefined, 'ownerless'], + [1.5, undefined, 'ownerless'], + ['7', undefined, 'ownerless'], + [null, undefined, 'ownerless'], +] as const) { + test(`finish retains ${reason} registration (${String(pid)}, ${String(startTime)})`, async () => { + const { paths, owner } = await acquire(); + owner.publish(fields); + replaceInfo(paths, pid, startTime); + const before = fs.readFileSync(paths.infoPath, 'utf8'); + assert.deepEqual(await owner.finish(), { + state: reason, + ...(reason === 'replaced' ? { identity: { pid, startTime } } : {}), + }); + assert.equal(fs.readFileSync(paths.infoPath, 'utf8'), before); + assert.equal(fs.existsSync(paths.lockPath), false); + }); +} + +test('absent and corrupt registrations are distinct and corruption is retained', async () => { + const first = await acquire(); + assert.deepEqual(await first.owner.finish(), { state: 'absent' }); + const second = await acquire(); + fs.writeFileSync(second.paths.infoPath, '{not json'); + assert.deepEqual(await second.owner.finish(), { state: 'ownerless' }); + assert.equal(fs.readFileSync(second.paths.infoPath, 'utf8'), '{not json'); +}); + +test.skipIf(process.getuid?.() === 0)('unreadable metadata is retained', async () => { + const { paths, owner } = await acquire(); + owner.publish(fields); + fs.chmodSync(paths.infoPath, 0o000); + try { + assert.deepEqual(await owner.finish(), { state: 'unreadable' }); + assert.equal(fs.existsSync(paths.infoPath), true); + } finally { + fs.chmodSync(paths.infoPath, 0o600); + } +}); + +test('legacy lock files are refused and retained', async () => { + const paths = resolveDaemonPaths(mkdtempForTestSync('agent-device-registration-legacy-')); + const contents = JSON.stringify({ pid: process.pid, processStartTime: ownIdentity.startTime }); + fs.writeFileSync(paths.lockPath, contents); + const result = await tryAcquireDaemonRegistration(paths); + assert.equal(result.status, 'unproven'); + assert.equal(fs.readFileSync(paths.lockPath, 'utf8'), contents); +}); + +test('spent acquisitions cannot publish, remove metadata or write a report', async () => { + const { paths, owner } = await acquire(); + owner.publish(fields); + await owner.finish(); + const successor = await acquire(paths); + successor.owner.publish({ ...fields, token: 'successor-token' }); + fs.writeFileSync(`${paths.baseDir}/daemon-shutdown.json`, JSON.stringify(report)); + const beforeInfo = fs.readFileSync(paths.infoPath, 'utf8'); + const beforeReport = fs.readFileSync(`${paths.baseDir}/daemon-shutdown.json`, 'utf8'); + assert.throws(() => owner.publish(fields)); + await assert.rejects(owner.finish(report)); + assert.equal(fs.readFileSync(paths.infoPath, 'utf8'), beforeInfo); + assert.equal(fs.readFileSync(`${paths.baseDir}/daemon-shutdown.json`, 'utf8'), beforeReport); + await successor.owner.finish(); +}); + +test('startup clears a prior report while held; a failed clear releases the acquisition', async () => { + const paths = resolveDaemonPaths(mkdtempForTestSync('agent-device-registration-clear-')); + const reportPath = `${paths.baseDir}/daemon-shutdown.json`; + fs.writeFileSync(reportPath, 'old report'); + const first = await acquire(paths); + assert.equal(fs.existsSync(reportPath), false); + await first.owner.finish(); + const primary = Object.assign(new Error('clear failure'), { code: 'EACCES' }); + const original = fs.rmSync; + vi.spyOn(fs, 'rmSync').mockImplementation((target, options) => { + if (target === reportPath) throw primary; + return original(target, options); + }); + await assert.rejects(tryAcquireDaemonRegistration(paths), (error) => error === primary); + assert.equal(fs.existsSync(paths.lockPath), false); +}); + +test('unlink disappearance is settled, other failures remain primary even when release fails', async () => { + const first = await acquire(); + first.owner.publish(fields); + const original = fs.unlinkSync; + vi.spyOn(fs, 'unlinkSync').mockImplementation((target) => { + if (target === first.paths.infoPath) + throw Object.assign(new Error('ENOENT'), { code: 'ENOENT' }); + return original(target); + }); + assert.deepEqual(await first.owner.finish(), { state: 'removed' }); + vi.restoreAllMocks(); + const second = await acquire(); + second.owner.publish(fields); + const primary = new Error('metadata failure'); + vi.spyOn(fs, 'unlinkSync').mockImplementation((target) => { + if (target === second.paths.infoPath) throw primary; + throw new Error('release failure'); + }); + await assert.rejects(second.owner.finish(), (error) => error === primary); + assert.equal(fs.existsSync(second.paths.infoPath), true); +}); diff --git a/src/__tests__/daemon-registration.test.ts b/src/__tests__/daemon-registration.test.ts index 6cff9d1d29..2df32f3cee 100644 --- a/src/__tests__/daemon-registration.test.ts +++ b/src/__tests__/daemon-registration.test.ts @@ -7,7 +7,6 @@ import { readRegisteredDaemonIdentity, readRegisteredDaemonOwnership, } from '../daemon-registration.ts'; -import { writeInfo } from '../daemon/server/server-lifecycle.ts'; import { publishDaemonRegistration } from './test-utils/device-claim-store.ts'; import { mkdtempForTestSync } from './test-utils/tmp-dir.ts'; @@ -29,14 +28,7 @@ function infoPathOf(stateDir: string): string { test('reads back the identity a running daemon publishes for its state dir', () => { const stateDir = useStateDir(); - writeInfo(stateDir, infoPathOf(stateDir), path.join(stateDir, 'daemon.log'), { - socketPort: 1234, - token: 'token', - codeOrigin: 'checkout', - version: '0.0.0-test', - codeSignature: 'signature', - processStartTime: 'published-start', - }); + publishDaemonRegistration(stateDir, { pid: process.pid, startTime: 'published-start' }); assert.deepEqual(readRegisteredDaemonIdentity(infoPathOf(stateDir)), { pid: process.pid, diff --git a/src/__tests__/daemon-shutdown-report.test.ts b/src/__tests__/daemon-shutdown-report.test.ts index 54e5812361..0e6e555561 100644 --- a/src/__tests__/daemon-shutdown-report.test.ts +++ b/src/__tests__/daemon-shutdown-report.test.ts @@ -1,11 +1,7 @@ import fs from 'node:fs'; import path from 'node:path'; import { expect, test } from 'vitest'; -import { - clearDaemonShutdownReport, - readDaemonShutdownReport, - writeDaemonShutdownReport, -} from '../daemon-shutdown-report.ts'; +import { readDaemonShutdownReport, buildDaemonShutdownReport } from '../daemon-shutdown-report.ts'; import { LeaseRegistry } from '../daemon/lease-registry.ts'; import { mkdtempForTestSync } from './test-utils/tmp-dir.ts'; @@ -25,11 +21,12 @@ test('round-trips provider release and device claim records without lease creden }); try { - writeDaemonShutdownReport(stateDir, { + const report = buildDaemonShutdownReport({ providerReleases: { released: [lease], pending: [lease] }, claims: { released: [claim], orphaned: [], superseded: [claim], unattributable: [] }, }); + fs.writeFileSync(path.join(stateDir, 'daemon-shutdown.json'), JSON.stringify(report)); expect(readDaemonShutdownReport(stateDir)).toEqual({ providerReleases: { released: [{ leaseId: lease.leaseId, provider: 'limrun' }], @@ -61,7 +58,7 @@ test('a report written before claim reporting still reads its provider releases' } }); -test('ignores malformed shutdown reports and clear removes a prior report', () => { +test('ignores malformed shutdown reports', () => { const stateDir = mkdtempForTestSync('agent-device-shutdown-report-'); const reportPath = path.join(stateDir, 'daemon-shutdown.json'); @@ -75,9 +72,6 @@ test('ignores malformed shutdown reports and clear removes a prior report', () = JSON.stringify({ providerReleases: { released: [{}], pending: [] } }), ); expect(readDaemonShutdownReport(stateDir)).toBeNull(); - - clearDaemonShutdownReport(stateDir); - expect(fs.existsSync(reportPath)).toBe(false); } finally { fs.rmSync(stateDir, { recursive: true, force: true }); } diff --git a/src/daemon-client/__tests__/daemon-client-metadata.test.ts b/src/daemon-client/__tests__/daemon-client-metadata.test.ts index 2ecfce7cae..3aad1ab51d 100644 --- a/src/daemon-client/__tests__/daemon-client-metadata.test.ts +++ b/src/daemon-client/__tests__/daemon-client-metadata.test.ts @@ -5,11 +5,12 @@ import path from 'node:path'; import { afterEach, test, vi } from 'vitest'; import type { DaemonCodeOrigin } from '@agent-device/host-kit/code-signature'; import { mkdtempForTestSync } from '../../__tests__/test-utils/tmp-dir.ts'; -import { writeInfo } from '../../daemon/server/server-lifecycle.ts'; +import { tryAcquireDaemonRegistration } from '../../daemon-registration-owner.ts'; import { readDaemonInfo, cleanupFailedDaemonStartupMetadata, stopDaemonProcessForTakeover, + type DaemonInfo, } from '../daemon-client-metadata.ts'; import { isAgentDeviceDaemonProcess, stopDaemonProcess } from '../../daemon-process.ts'; import { resolveDaemonPaths } from '../../daemon-resolution.ts'; @@ -21,6 +22,7 @@ vi.mock('../../daemon-process.ts', async (importOriginal) => ({ })); afterEach(() => vi.resetAllMocks()); + // The reuse decision is only as good as the identity that survives the round trip // through `daemon.json`: a client cannot compare what the file lost (#2458). @@ -29,22 +31,26 @@ function scratchStateDir(): [stateDir: string, infoPath: string] { return [stateDir, path.join(stateDir, 'daemon.json')]; } -function publishInfo(codeOrigin: DaemonCodeOrigin): string { +async function publishInfo(codeOrigin: DaemonCodeOrigin): Promise { const [stateDir, infoPath] = scratchStateDir(); - writeInfo(stateDir, infoPath, path.join(stateDir, 'daemon.log'), { + const registration = await tryAcquireDaemonRegistration(resolveDaemonPaths(stateDir)); + assert.equal(registration.status, 'acquired'); + if (registration.status !== 'acquired') throw new Error('registration refused'); + registration.owner.publish({ httpPort: 41_234, token: 'local-secret', version: '0.0.0-test', codeOrigin, codeSignature: 'graph:1:abc', - processStartTime: 'start', }); - return infoPath; + const published = readDaemonInfo(infoPath); + await registration.owner.finish(); + return published; } -test('a daemon publishes the code origin its client reads back', () => { +test('a daemon publishes the code origin its client reads back', async () => { for (const codeOrigin of ['installed', 'checkout'] as const) { - assert.equal(readDaemonInfo(publishInfo(codeOrigin))?.codeOrigin, codeOrigin); + assert.equal((await publishInfo(codeOrigin))?.codeOrigin, codeOrigin); } }); diff --git a/src/daemon-registration-owner.ts b/src/daemon-registration-owner.ts new file mode 100644 index 0000000000..119a2d96a5 --- /dev/null +++ b/src/daemon-registration-owner.ts @@ -0,0 +1,161 @@ +import fs from 'node:fs'; +import { readCurrentOwnerIdentity, type OwnerIdentity } from '@agent-device/host-kit/process'; +import { + publishFileSync, + tryAcquireProcessLock, + type ProcessLockAttempt, + type ProcessLockAcquisition, +} from '@agent-device/host-kit/file'; +import { emitDiagnostic } from '@agent-device/host-kit/diagnostics'; +import type { DaemonCodeOrigin } from '@agent-device/host-kit/code-signature'; +import type { DaemonPaths } from './daemon-resolution.ts'; +import { + readRegisteredDaemonOwnership, + type RegisteredDaemonOwnership, +} from './daemon-registration.ts'; +import { + buildDaemonShutdownReport, + resolveDaemonShutdownReportPath, + type DaemonShutdownOutcome, +} from './daemon-shutdown-report.ts'; + +export const DAEMON_STARTUP_EXIT_CODES = Object.freeze({ busy: 75, unproven: 78 }); + +export type DaemonRegistrationFields = Readonly<{ + socketPort?: number; + httpPort?: number; + token: string; + version: string; + codeOrigin: DaemonCodeOrigin; + codeSignature: string; + policyDigest?: string; +}>; + +type DaemonRegistrationRemoval = + | Readonly<{ state: 'removed' }> + | Exclude; + +export type DaemonRegistrationOwner = Readonly<{ + publish(fields: DaemonRegistrationFields): void; + finish(outcome?: DaemonShutdownOutcome): Promise; +}>; + +/** Makes one acquisition attempt and binds every daemon write to that acquisition. */ +export async function tryAcquireDaemonRegistration( + paths: DaemonPaths, +): Promise< + | Readonly<{ status: 'acquired'; owner: DaemonRegistrationOwner }> + | Exclude +> { + const boundPaths = { ...paths }; + const identity = readCurrentOwnerIdentity(); + const attempt = tryAcquireProcessLock({ + lockDirPath: boundPaths.lockPath, + owner: { ...identity, acquiredAtMs: Date.now() }, + description: 'daemon registration', + }); + if (attempt.status !== 'acquired') return attempt; + const { acquisition } = attempt; + try { + acquisition.assertHeld(); + fs.rmSync(resolveDaemonShutdownReportPath(boundPaths.baseDir), { force: true }); + } catch (error) { + await releaseRegistrationAfterFailure(acquisition, error); + } + return { + status: 'acquired', + owner: Object.freeze({ + publish(fields: DaemonRegistrationFields) { + acquisition.assertHeld(); + publishFileSync({ destination: boundPaths.logPath, contents: '', mode: 0o600 }); + const transport = + fields.socketPort && fields.httpPort ? 'dual' : fields.httpPort ? 'http' : 'socket'; + acquisition.assertHeld(); + publishFileSync({ + destination: boundPaths.infoPath, + contents: JSON.stringify( + { + port: fields.socketPort, + httpPort: fields.httpPort, + transport, + token: fields.token, + pid: identity.pid, + version: fields.version, + codeOrigin: fields.codeOrigin, + codeSignature: fields.codeSignature, + processStartTime: identity.startTime ?? undefined, + policyDigest: fields.policyDigest, + stateDir: boundPaths.baseDir, + }, + null, + 2, + ), + mode: 0o600, + }); + }, + async finish(outcome?: DaemonShutdownOutcome) { + let removal: DaemonRegistrationRemoval; + try { + if (outcome) writeShutdownReport(boundPaths.baseDir, outcome, acquisition); + removal = removeRegistrationUnderLock(boundPaths.infoPath, identity, acquisition); + } catch (error) { + return await releaseRegistrationAfterFailure(acquisition, error); + } + await acquisition.release(); + return removal; + }, + }), + }; +} + +function removeRegistrationUnderLock( + infoPath: string, + identity: OwnerIdentity, + acquisition: ProcessLockAcquisition, +): DaemonRegistrationRemoval { + acquisition.assertHeld(); + const ownership = readRegisteredDaemonOwnership(infoPath, identity); + if (ownership.state !== 'match') return ownership; + acquisition.assertHeld(); + try { + fs.unlinkSync(infoPath); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error; + } + return { state: 'removed' }; +} + +async function releaseRegistrationAfterFailure( + acquisition: ProcessLockAcquisition, + error: unknown, +): Promise { + try { + await acquisition.release(); + } catch (releaseError) { + emitDiagnostic({ + level: 'warn', + phase: 'daemon_registration_release_failed', + data: { error: String(releaseError) }, + }); + } + throw error; +} + +function writeShutdownReport( + stateDir: string, + outcome: DaemonShutdownOutcome, + acquisition: ProcessLockAcquisition, +): void { + const filePath = resolveDaemonShutdownReportPath(stateDir); + const contents = `${JSON.stringify(buildDaemonShutdownReport(outcome))}\n`; + acquisition.assertHeld(); + try { + publishFileSync({ destination: filePath, contents, mode: 0o600 }); + } catch { + return; + } + acquisition.assertHeld(); + try { + fs.chmodSync(filePath, 0o600); + } catch {} +} diff --git a/src/daemon-shutdown-report.ts b/src/daemon-shutdown-report.ts index 65b389ed1b..5208ad11ba 100644 --- a/src/daemon-shutdown-report.ts +++ b/src/daemon-shutdown-report.ts @@ -1,7 +1,6 @@ import fs from 'node:fs'; import path from 'node:path'; import type { DeviceLease } from '@agent-device/contracts/device'; -import { publishFileSync } from '@agent-device/host-kit/file'; const SHUTDOWN_REPORT_FILE = 'daemon-shutdown.json'; @@ -40,19 +39,18 @@ export type DaemonShutdownReport = { }; }; -export function writeDaemonShutdownReport( - stateDir: string, - outcome: { - providerReleases: { released: readonly DeviceLease[]; pending: readonly DeviceLease[] }; - claims: { - released: readonly DeviceClaimRecord[]; - orphaned: readonly DeviceClaimRecord[]; - superseded: readonly DeviceClaimRecord[]; - unattributable: readonly DeviceClaimRecord[]; - }; - }, -): void { - const report: DaemonShutdownReport = { +export type DaemonShutdownOutcome = { + providerReleases: { released: readonly DeviceLease[]; pending: readonly DeviceLease[] }; + claims: { + released: readonly DeviceClaimRecord[]; + orphaned: readonly DeviceClaimRecord[]; + superseded: readonly DeviceClaimRecord[]; + unattributable: readonly DeviceClaimRecord[]; + }; +}; + +export function buildDaemonShutdownReport(outcome: DaemonShutdownOutcome): DaemonShutdownReport { + return { providerReleases: { released: outcome.providerReleases.released.map(toProviderReleaseRecord), pending: outcome.providerReleases.pending.map(toProviderReleaseRecord), @@ -64,23 +62,13 @@ export function writeDaemonShutdownReport( unattributable: [...outcome.claims.unattributable], }, }; - const filePath = shutdownReportPath(stateDir); - try { - publishFileSync({ - destination: filePath, - contents: `${JSON.stringify(report)}\n`, - mode: 0o600, - }); - fs.chmodSync(filePath, 0o600); - } catch { - // Shutdown reporting is best effort; the atomic publisher has already - // preserved the primary filesystem failure and cleaned its temp sibling. - } } export function readDaemonShutdownReport(stateDir: string): DaemonShutdownReport | null { try { - const parsed = JSON.parse(fs.readFileSync(shutdownReportPath(stateDir), 'utf8')) as unknown; + const parsed = JSON.parse( + fs.readFileSync(resolveDaemonShutdownReportPath(stateDir), 'utf8'), + ) as unknown; if (!isProviderReleaseReport(parsed)) return null; // A report left behind by a daemon that predates claim reporting still // describes its provider releases honestly; it just knows nothing of claims. @@ -90,13 +78,7 @@ export function readDaemonShutdownReport(stateDir: string): DaemonShutdownReport } } -export function clearDaemonShutdownReport(stateDir: string): void { - try { - fs.rmSync(shutdownReportPath(stateDir), { force: true }); - } catch {} -} - -function shutdownReportPath(stateDir: string): string { +export function resolveDaemonShutdownReportPath(stateDir: string): string { return path.join(stateDir, SHUTDOWN_REPORT_FILE); } diff --git a/src/daemon/__tests__/daemon-runtime-app-log.test.ts b/src/daemon/__tests__/daemon-runtime-app-log.test.ts index 0057a59e90..baad8d7a7a 100644 --- a/src/daemon/__tests__/daemon-runtime-app-log.test.ts +++ b/src/daemon/__tests__/daemon-runtime-app-log.test.ts @@ -18,7 +18,7 @@ import { unavailableDeviceRuntimeGateway } from './test-device-runtime-gateway.t test('daemon startup awaits app-log recovery after acquiring the lock and before opening servers', () => { const source = fs.readFileSync(new URL('../server/daemon-runtime.ts', import.meta.url), 'utf8'); - const acquiredLock = source.indexOf('if (!acquireDaemonLock('); + const acquiredLock = source.indexOf("if (acquisition.status !== 'acquired')"); const legacyRecovery = source.indexOf( 'await platformDaemonLifecycleOwners.recoverLegacyAppLogMarkers(', ); @@ -36,11 +36,11 @@ test('daemon startup configures the Apple runner owner after acquiring the lock, // publish only once this process actually holds the daemon lock, so a losing process never // configures a global platform owner it does not own. const source = fs.readFileSync(new URL('../server/daemon-runtime.ts', import.meta.url), 'utf8'); - const acquiredLock = source.indexOf('if (!acquireDaemonLock('); + const acquiredLock = source.indexOf("if (acquisition.status !== 'acquired')"); const runnerOwnerConfigured = source.indexOf( 'await platformDaemonLifecycleOwners.configureForDaemonLock(', ); - const lockFailureExit = source.indexOf("stderr.write('Daemon lock is held by another process"); + const lockFailureExit = source.indexOf('exit(', acquiredLock); expect(acquiredLock).toBeGreaterThanOrEqual(0); expect(lockFailureExit).toBeGreaterThan(acquiredLock); diff --git a/src/daemon/__tests__/filesystem-boundary-faults.test.ts b/src/daemon/__tests__/filesystem-boundary-faults.test.ts index f20ed62a51..da432598cc 100644 --- a/src/daemon/__tests__/filesystem-boundary-faults.test.ts +++ b/src/daemon/__tests__/filesystem-boundary-faults.test.ts @@ -8,7 +8,8 @@ import type { DeviceInfo } from '@agent-device/kernel/device'; import { acquireDeviceClaim } from '../device/device-claims.ts'; import { canonicalLocalDeviceKey } from '../device/device-claim-paths.ts'; import { createDurableCaptureResourceStore } from '@agent-device/capture-kit/durable-capture'; -import { writeDaemonShutdownReport } from '../../daemon-shutdown-report.ts'; +import { tryAcquireDaemonRegistration } from '../../daemon-registration-owner.ts'; +import { resolveDaemonPaths } from '../../daemon-resolution.ts'; import { SessionScriptWriter, type SessionScriptWriteResult } from '../session-script-writer.ts'; import { SessionStore } from '../session-store.ts'; import { @@ -135,11 +136,15 @@ function createSessionScriptFixture(root: string): FilesystemBoundaryFixture { function createShutdownReportFixture(root: string): FilesystemBoundaryFixture { return { targetPath: path.join(root, 'daemon-shutdown.json'), - run: async () => - writeDaemonShutdownReport(root, { + run: async () => { + const attempt = await tryAcquireDaemonRegistration(resolveDaemonPaths(root)); + assert.equal(attempt.status, 'acquired'); + if (attempt.status !== 'acquired') throw new Error('registration refused'); + await attempt.owner.finish({ providerReleases: { released: [], pending: [] }, claims: { released: [], orphaned: [], superseded: [], unattributable: [] }, - }), + }); + }, expected: 'return', }; } diff --git a/src/daemon/server/daemon-runtime-metadata-ownership.test.ts b/src/daemon/server/daemon-runtime-metadata-ownership.test.ts index 3d02a6b4fd..b9105675b6 100644 --- a/src/daemon/server/daemon-runtime-metadata-ownership.test.ts +++ b/src/daemon/server/daemon-runtime-metadata-ownership.test.ts @@ -92,6 +92,7 @@ function publishSuccessor(paths: DaemonPaths): void { afterEach(() => { startupFailure.active = false; lifecycleEvents.length = 0; + vi.restoreAllMocks(); }); test('a shutdown whose daemon.json names a successor keeps the file and logs the decline', async () => { @@ -124,19 +125,39 @@ test('a shutdown whose daemon.json names a successor keeps the file and logs the } }); -test('a shutdown that still owns its daemon.json removes it without a decline', async () => { +test('a shutdown removes its own metadata and reports an unverified release', async () => { const stateDir = mkdtempForTestSync('agent-device-daemon-info-owned-shutdown-'); const paths = resolveDaemonPaths(stateDir); try { const runtime = await startRuntime(stateDir, () => {}); expect(runtime).not.toBeNull(); + const originalRmdir = fs.rmdirSync; + vi.spyOn(fs, 'rmdirSync').mockImplementation((target, options) => { + if (target === paths.lockPath) throw Object.assign(new Error('busy'), { code: 'EBUSY' }); + return originalRmdir(target, options); + }); await runtime?.shutdown(); expect(fs.existsSync(paths.infoPath)).toBe(false); expect(logEvents(stateDir).map((event) => event.phase)).not.toContain( 'daemon_info_removal_declined', ); + expect(logEvents(stateDir)).toContainEqual( + expect.objectContaining({ + phase: 'daemon_registration_finish_failed', + data: expect.objectContaining({ + error: expect.objectContaining({ + message: 'Cannot verify ownership of daemon registration', + details: expect.objectContaining({ + lockDirPath: paths.lockPath, + ownerReleaseUnverified: true, + }), + hint: expect.stringContaining('confirming all users'), + }), + }), + }), + ); } finally { fs.rmSync(stateDir, { recursive: true, force: true }); } @@ -226,7 +247,7 @@ test('both exits tear the watch down before they touch daemon.json', () => { // arming order above is. const source = fs.readFileSync(new URL('./daemon-runtime.ts', import.meta.url), 'utf8'); const stopped = source.indexOf('stopMetadataLossWatch();'); - const removal = source.indexOf('await removeOwnDaemonInfo('); + const removal = source.indexOf('await finishDaemonRegistration('); expect(stopped).toBeGreaterThanOrEqual(0); expect(removal).toBeGreaterThan(stopped); diff --git a/src/daemon/server/daemon-runtime.ts b/src/daemon/server/daemon-runtime.ts index 70efadb6da..e561d56290 100644 --- a/src/daemon/server/daemon-runtime.ts +++ b/src/daemon/server/daemon-runtime.ts @@ -1,5 +1,5 @@ import crypto from 'node:crypto'; -import { asAppError, AppError } from '@agent-device/kernel/errors'; +import { asAppError, AppError, normalizeError } from '@agent-device/kernel/errors'; import { SessionStore } from '../session-store.ts'; import { resolveSessionRequestLogPath } from '../session-artifact-paths.ts'; import { resolveDaemonPaths, resolveDaemonServerMode } from '../../daemon-resolution.ts'; @@ -27,10 +27,6 @@ import { } from '../../provider-device-runtimes.ts'; import { LeaseRegistry } from '../lease-registry.ts'; import { createExpiredProviderLeaseReleaser } from '../provider-lease-expiry.ts'; -import { - clearDaemonShutdownReport, - writeDaemonShutdownReport, -} from '../../daemon-shutdown-report.ts'; import { createRequestHandler } from '../request-router.ts'; import { getLeaseRegistryExecutionLocks } from '../request-execution-scope.ts'; import { stopSessionAppLog, teardownSessionResources } from '../session-teardown.ts'; @@ -70,15 +66,16 @@ import { import { isEnvTruthy, sleep } from '@agent-device/host-kit/retry'; import { - acquireDaemonLock, parseIntegerEnv, readVersion, - releaseDaemonLock, - removeInfoOwnedBy, resolveDaemonCodeOrigin, resolveDaemonCodeSignature, - writeInfo, } from './server-lifecycle.ts'; +import { + tryAcquireDaemonRegistration, + DAEMON_STARTUP_EXIT_CODES, + type DaemonRegistrationOwner, +} from '../../daemon-registration-owner.ts'; import { watchDaemonMetadataLoss, type DaemonMetadataLoss } from './daemon-metadata-loss.ts'; import { createSocketServer, @@ -262,23 +259,27 @@ async function emitDaemonDiagnostic( ); } -/** - * Removes this daemon's `daemon.json` at exit, and only while the record still names it: a shutdown - * that unlinked whatever file was present took the metadata of the daemon now serving clients (#3087). - * An absent record is not a decline worth logging, because client cleanup removes it routinely and a - * startup that failed before publication must not leave a `daemon.log` behind. - */ -async function removeOwnDaemonInfo(params: { +async function finishDaemonRegistration(params: { + registration: DaemonRegistrationOwner; infoPath: string; logPath: string; - owner: OwnerIdentity; + outcome?: Parameters[0]; }): Promise { - const removal = removeInfoOwnedBy(params.infoPath, params.owner); - if (removal.removed || removal.reason === 'absent') return; - await emitDaemonDiagnostic(params.logPath, 'daemon_info_removal_declined', { - infoPath: params.infoPath, - ...removal, - }); + try { + const removal = await params.registration.finish(params.outcome); + if (removal.state !== 'removed' && removal.state !== 'absent') { + await emitDaemonDiagnostic(params.logPath, 'daemon_info_removal_declined', { + infoPath: params.infoPath, + removed: false, + reason: removal.state, + ...(removal.state === 'replaced' ? { registeredPid: removal.identity.pid } : {}), + }); + } + } catch (error) { + await emitDaemonDiagnostic(params.logPath, 'daemon_registration_finish_failed', { + error: normalizeError(error), + }); + } } async function noteDaemonMetadataLoss(params: { @@ -319,7 +320,7 @@ export async function startDaemonRuntime( const stderr = options.stderr ?? process.stderr; const exit = options.exit ?? ((code: number) => process.exit(code)); const daemonPaths = resolveDaemonPaths(env.AGENT_DEVICE_STATE_DIR); - const { baseDir, infoPath, lockPath, logPath, sessionsDir } = daemonPaths; + const { baseDir, infoPath, logPath, sessionsDir } = daemonPaths; const daemonServerMode = resolveDaemonServerMode(env.AGENT_DEVICE_DAEMON_SERVER_MODE); const retainArtifacts = isEnvTruthy(env.AGENT_DEVICE_RETAIN_ARTIFACTS); // ADR 0029: a policy that cannot be read or validated stops startup; the daemon never runs @@ -347,7 +348,6 @@ export async function startDaemonRuntime( const version = readVersion(); const token = crypto.randomBytes(24).toString('hex'); const daemonIdentity = readCurrentOwnerIdentity(); - const daemonProcessStartTime = daemonIdentity.startTime ?? undefined; const daemonCodeOrigin = resolveDaemonCodeOrigin(); const daemonCodeSignature = resolveDaemonCodeSignature(); const providerComposition = await createDaemonProviderRuntimeComposition(env); @@ -623,14 +623,13 @@ export async function startDaemonRuntime( }; const publishDaemonInfo = (socketPort: number | undefined, httpPort: number | undefined) => { - writeInfo(baseDir, infoPath, logPath, { + registration.publish({ socketPort, httpPort, token, version, codeOrigin: daemonCodeOrigin, codeSignature: daemonCodeSignature, - processStartTime: daemonProcessStartTime, policyDigest: daemonPolicy?.digest, }); if (socketPort) stdout.write(`AGENT_DEVICE_DAEMON_PORT=${socketPort}\n`); @@ -645,21 +644,16 @@ export async function startDaemonRuntime( } }; - const lockData = { - pid: process.pid, - version, - startedAt: Date.now(), - processStartTime: daemonProcessStartTime, - }; - if (!acquireDaemonLock(baseDir, lockPath, lockData)) { + const acquisition = await tryAcquireDaemonRegistration(daemonPaths); + if (acquisition.status !== 'acquired') { await Promise.allSettled( providerDeviceRuntimes.map(async (runtime) => await runtime.shutdown()), ); - stderr.write('Daemon lock is held by another process; exiting.\n'); - exit(0); + stderr.write(`Daemon registration ${acquisition.status}; exiting.\n`); + exit(DAEMON_STARTUP_EXIT_CODES[acquisition.status]); return null; } - clearDaemonShutdownReport(baseDir); + const registration = acquisition.owner; let servers: DaemonServer[] = []; let socketPort: number | undefined; @@ -740,11 +734,10 @@ export async function startDaemonRuntime( stderr.write(`Daemon error: ${appErr.message}\n`); closeServersBestEffort(servers); stopMetadataLossWatch(); - await removeOwnDaemonInfo({ infoPath, logPath, owner: daemonIdentity }); await Promise.allSettled( providerDeviceRuntimes.map(async (runtime) => await runtime.shutdown()), ); - releaseDaemonLock(lockPath); + await finishDaemonRegistration({ registration, infoPath, logPath }); await platformDaemonLifecycleOwners.clearDaemonLockConfiguration(); exit(1); return null; @@ -796,10 +789,6 @@ export async function startDaemonRuntime( const providerReleaseDrain = await expiredProviderLeaseReleaser.drain( DAEMON_PROVIDER_RELEASE_DRAIN_TIMEOUT_MS, ); - writeDaemonShutdownReport(baseDir, { - providerReleases: providerReleaseDrain, - claims: shutdownClaimLedger.claims, - }); emitDiagnostic({ level: providerReleaseDrain.pending.length === 0 ? 'info' : 'warn', phase: 'daemon_shutdown_provider_release_drain', @@ -821,8 +810,12 @@ export async function startDaemonRuntime( terminatePngWorker().catch(() => {}), sleep(DAEMON_PNG_WORKER_TERMINATE_TIMEOUT_MS), ]); - await removeOwnDaemonInfo({ infoPath, logPath, owner: daemonIdentity }); - releaseDaemonLock(lockPath); + await finishDaemonRegistration({ + registration, + infoPath, + logPath, + outcome: { providerReleases: providerReleaseDrain, claims: shutdownClaimLedger.claims }, + }); await platformDaemonLifecycleOwners.clearDaemonLockConfiguration(); exit(shutdownOptions.exitCode ?? 0); }; diff --git a/src/daemon/server/server-lifecycle.test.ts b/src/daemon/server/server-lifecycle.test.ts deleted file mode 100644 index 209595b1d3..0000000000 --- a/src/daemon/server/server-lifecycle.test.ts +++ /dev/null @@ -1,211 +0,0 @@ -import assert from 'node:assert/strict'; -import fs from 'node:fs'; -import path from 'node:path'; -import { test, vi } from 'vitest'; -import { mkdtempForTestSync } from '../../__tests__/test-utils/tmp-dir.ts'; -import type { OwnerIdentity } from '@agent-device/host-kit/process'; -import { removeInfoOwnedBy, writeInfo } from './server-lifecycle.ts'; -import { readRegisteredDaemonOwnership } from '../../daemon-registration.ts'; - -const OWN_PID = process.pid; -const SUCCESSOR_PID = 999_999_999; -const START_TIME = 'own-start-time'; -const SUCCESSOR_START_TIME = 'successor-start-time'; - -const OWN_IDENTITY: OwnerIdentity = { pid: OWN_PID, startTime: START_TIME }; - -function scratchInfoPath(name = 'daemon.json'): [stateDir: string, infoPath: string] { - const stateDir = mkdtempForTestSync('agent-device-server-lifecycle-'); - return [stateDir, path.join(stateDir, name)]; -} - -function writeRegistration( - pid: number, - startTime: string | null, - name = 'daemon.json', -): [string, string] { - const [stateDir, infoPath] = scratchInfoPath(name); - fs.writeFileSync( - infoPath, - JSON.stringify({ - pid, - ...(startTime === null ? {} : { processStartTime: startTime }), - token: 'token', - port: 4210, - }), - ); - return [stateDir, infoPath]; -} - -test('the daemon named by daemon.json removes its own registration', () => { - const [, infoPath] = writeRegistration(OWN_PID, START_TIME); - - assert.deepEqual(removeInfoOwnedBy(infoPath, OWN_IDENTITY), { removed: true }); - assert.equal(fs.existsSync(infoPath), false); -}); - -test('a successor published over the metadata keeps its record through the predecessor shutdown', () => { - const [, infoPath] = writeRegistration(SUCCESSOR_PID, SUCCESSOR_START_TIME); - - assert.deepEqual(removeInfoOwnedBy(infoPath, OWN_IDENTITY), { - removed: false, - reason: 'replaced', - registeredPid: SUCCESSOR_PID, - }); - assert.equal(fs.existsSync(infoPath), true); - assert.equal( - (JSON.parse(fs.readFileSync(infoPath, 'utf8')) as { pid: number }).pid, - SUCCESSOR_PID, - ); -}); - -test('a record recycling our pid after our start time is not ours to remove', () => { - // The pid is not an identity. Removing the successor's record here is the same #3087 failure the - // issue describes, just reached through PID reuse instead of a second daemon. - const [, infoPath] = writeRegistration(OWN_PID, SUCCESSOR_START_TIME); - - assert.deepEqual(removeInfoOwnedBy(infoPath, OWN_IDENTITY), { - removed: false, - reason: 'replaced', - registeredPid: OWN_PID, - }); - assert.equal(fs.existsSync(infoPath), true); -}); - -test('a record agreeing on pid alone is refused, not removed on the strength of the pid', () => { - // One side cannot read its own start time, so nothing proves this record is ours. Removing it would - // be the pid-only rule the fence exists to replace; keeping it costs a client one liveness probe. - const [, infoPath] = writeRegistration(OWN_PID, null); - - assert.deepEqual(removeInfoOwnedBy(infoPath, OWN_IDENTITY), { - removed: false, - reason: 'unproven', - }); - assert.equal(fs.existsSync(infoPath), true); -}); - -test('an absent registration is nothing to remove', () => { - const [, infoPath] = scratchInfoPath(); - - assert.deepEqual(removeInfoOwnedBy(infoPath, OWN_IDENTITY), { removed: false, reason: 'absent' }); -}); - -test('a corrupt or pid-less registration names no owner, so it is refused and kept', () => { - const [stateDir, corrupt] = scratchInfoPath('corrupt.json'); - fs.writeFileSync(corrupt, '{not json'); - assert.deepEqual(removeInfoOwnedBy(corrupt, OWN_IDENTITY), { - removed: false, - reason: 'ownerless', - }); - assert.equal( - fs.existsSync(corrupt), - true, - 'a record that names no owner is not this pid to delete', - ); - - for (const pid of [0, -3, 1.5, '7', null]) { - const pidLess = path.join(stateDir, `pid-less-${String(pid)}.json`); - fs.writeFileSync(pidLess, JSON.stringify({ pid, token: 'token' })); - assert.deepEqual(removeInfoOwnedBy(pidLess, OWN_IDENTITY), { - removed: false, - reason: 'ownerless', - }); - assert.equal(fs.existsSync(pidLess), true); - } -}); - -test.skipIf(process.getuid?.() === 0)( - 'a registration this process cannot read is kept, not treated as removed', - () => { - // EACCES is not evidence that the record is gone, and a root-owned CI host reads mode 000 - // anyway. Treating it as gone would repeat #3087 from the other side: a live daemon loses its - // metadata to a shutdown whose read failed. - const [, infoPath] = writeRegistration(OWN_PID, START_TIME, 'unreadable.json'); - fs.chmodSync(infoPath, 0o000); - try { - assert.deepEqual(removeInfoOwnedBy(infoPath, OWN_IDENTITY), { - removed: false, - reason: 'unreadable', - }); - assert.equal(fs.existsSync(infoPath), true); - } finally { - fs.chmodSync(infoPath, 0o600); - } - }, -); - -test('the record is read at removal time, not remembered from publication', () => { - const [stateDir, infoPath] = scratchInfoPath(); - const publish = (port: number) => - writeInfo(stateDir, infoPath, path.join(stateDir, 'daemon.log'), { - socketPort: port, - token: 'token', - version: '0.0.0-test', - codeOrigin: 'checkout', - codeSignature: 'signature', - processStartTime: START_TIME, - }); - - publish(4210); - assert.deepEqual(removeInfoOwnedBy(infoPath, OWN_IDENTITY), { removed: true }); - - publish(4211); - fs.writeFileSync( - infoPath, - JSON.stringify({ pid: SUCCESSOR_PID, processStartTime: SUCCESSOR_START_TIME, port: 4211 }), - ); - assert.deepEqual(removeInfoOwnedBy(infoPath, OWN_IDENTITY), { - removed: false, - reason: 'replaced', - registeredPid: SUCCESSOR_PID, - }); -}); - -test('publication replaces the record by rename, never by an in-place write', () => { - // A torn `daemon.json` decodes to "names no owner", which every reader refuses to remove and the - // loss watch could misread. An in-place write exposes exactly such a window; a rename does not, - // and the inode swap is what distinguishes the two from the outside. - const [stateDir, infoPath] = scratchInfoPath(); - const publish = () => - writeInfo(stateDir, infoPath, path.join(stateDir, 'daemon.log'), { - socketPort: 4210, - token: 'token', - version: '0.0.0-test', - codeOrigin: 'checkout', - codeSignature: 'signature', - processStartTime: START_TIME, - }); - - publish(); - const before = fs.statSync(infoPath).ino; - publish(); - - assert.notEqual(fs.statSync(infoPath).ino, before, 'republication must not reuse the inode'); - assert.equal(readRegisteredDaemonOwnership(infoPath, OWN_IDENTITY).state, 'match'); -}); - -test('a registration already gone by the time of the unlink is not an error', () => { - const [, infoPath] = writeRegistration(OWN_PID, START_TIME); - const unlinkSync = vi.spyOn(fs, 'unlinkSync').mockImplementation(() => { - const error = new Error('ENOENT') as NodeJS.ErrnoException; - error.code = 'ENOENT'; - throw error; - }); - try { - assert.deepEqual(removeInfoOwnedBy(infoPath, OWN_IDENTITY), { removed: true }); - } finally { - unlinkSync.mockRestore(); - } -}); - -test('a failing unlink other than ENOENT surfaces instead of reporting a removal', () => { - const [, infoPath] = writeRegistration(OWN_PID, START_TIME); - const unlinkSync = vi.spyOn(fs, 'unlinkSync').mockImplementation(() => { - throw new Error('EBUSY'); - }); - try { - assert.throws(() => removeInfoOwnedBy(infoPath, OWN_IDENTITY), /EBUSY/); - } finally { - unlinkSync.mockRestore(); - } -}); diff --git a/src/daemon/server/server-lifecycle.ts b/src/daemon/server/server-lifecycle.ts index 49c1e02796..55e99cd1c1 100644 --- a/src/daemon/server/server-lifecycle.ts +++ b/src/daemon/server/server-lifecycle.ts @@ -1,163 +1,9 @@ -import fs from 'node:fs'; -import type { OwnerIdentity } from '@agent-device/host-kit/process'; -import { publishFileSync } from '@agent-device/host-kit/file'; -import type { DaemonCodeOrigin } from '@agent-device/host-kit/code-signature'; -import { isAgentDeviceDaemonProcess } from '../../daemon-process.ts'; -import { readRegisteredDaemonOwnership } from '../../daemon-registration.ts'; - export { readVersion } from '@agent-device/host-kit/version'; export { - type DaemonCodeOrigin, resolveDaemonCodeOrigin, resolveDaemonCodeSignature, } from '@agent-device/host-kit/code-signature'; -export type DaemonLockInfo = { - pid: number; - version: string; - startedAt: number; - processStartTime?: string; -}; - -export function writeInfo( - baseDir: string, - infoPath: string, - logPath: string, - opts: { - socketPort?: number; - httpPort?: number; - token: string; - version: string; - codeOrigin: DaemonCodeOrigin; - codeSignature: string; - processStartTime: string | undefined; - policyDigest?: string; - }, -): void { - if (!fs.existsSync(baseDir)) fs.mkdirSync(baseDir, { recursive: true }); - fs.writeFileSync(logPath, ''); - const transport = opts.socketPort && opts.httpPort ? 'dual' : opts.httpPort ? 'http' : 'socket'; - // Published through a same-directory temp sibling: a client that reads `daemon.json` while this - // daemon is starting either sees the previous record or this one, never a half-written file it - // would have to decode as naming no owner. - publishFileSync({ - destination: infoPath, - contents: JSON.stringify( - { - port: opts.socketPort, - httpPort: opts.httpPort, - transport, - token: opts.token, - pid: process.pid, - version: opts.version, - codeOrigin: opts.codeOrigin, - codeSignature: opts.codeSignature, - processStartTime: opts.processStartTime, - policyDigest: opts.policyDigest, - stateDir: baseDir, - }, - null, - 2, - ), - mode: 0o600, - }); -} - -export type InfoRemoval = - | Readonly<{ removed: true }> - | Readonly<{ - removed: false; - reason: 'replaced' | 'unproven' | 'absent' | 'unreadable' | 'ownerless'; - registeredPid?: number; - }>; - -/** - * Removes `daemon.json` only while it still names `owner`, which is the rule {@link - * releaseDaemonLock} already applies to the lock next to it: a shutdown that unlinks whatever file is - * present takes the metadata of the daemon now serving clients (#3087). The record is re-read here - * rather than trusted from publication, because a successor that took this state dir makes anything - * this process remembered about the file stale. - * - * Every refusal is fail-closed, including `unproven` for a record that agrees on pid but on no start - * time. Keeping such a record leaves a client to find a dead pid, which its own liveness check - * already recovers from; removing one that belongs to a successor is the unrecoverable direction. - * - * The read and the unlink are not one atomic step, and there is no check-and-delete primitive to make - * them one. What closes the gap is the daemon lock the caller still holds: a successor can only reach - * `writeInfo` after {@link acquireDaemonLock} steals it, and stealing it requires - * `isAgentDeviceDaemonProcess` to prove the holder dead — which this process running this line is not. - * The premise is therefore that nothing removed the lock out from under us, and #3105 records the one - * path that currently breaks it by clearing the lock without proving ownership. - */ -export function removeInfoOwnedBy(infoPath: string, owner: OwnerIdentity): InfoRemoval { - const ownership = readRegisteredDaemonOwnership(infoPath, owner); - if (ownership.state !== 'match') { - return { - removed: false, - reason: ownership.state, - ...(ownership.state === 'replaced' ? { registeredPid: ownership.identity.pid } : {}), - }; - } - try { - fs.unlinkSync(infoPath); - } catch (error) { - if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error; - } - return { removed: true }; -} - -function readLockInfo(lockPath: string): DaemonLockInfo | null { - if (!fs.existsSync(lockPath)) return null; - try { - const parsed = JSON.parse(fs.readFileSync(lockPath, 'utf8')) as DaemonLockInfo; - if (!Number.isInteger(parsed.pid) || parsed.pid <= 0) return null; - return parsed; - } catch { - return null; - } -} - -export function acquireDaemonLock( - baseDir: string, - lockPath: string, - lockData: DaemonLockInfo, -): boolean { - if (!fs.existsSync(baseDir)) fs.mkdirSync(baseDir, { recursive: true }); - const payload = JSON.stringify(lockData, null, 2); - - const tryWriteLock = (): boolean => { - try { - fs.writeFileSync(lockPath, payload, { flag: 'wx', mode: 0o600 }); - return true; - } catch (error) { - if ((error as NodeJS.ErrnoException).code === 'EEXIST') return false; - throw error; - } - }; - - if (tryWriteLock()) return true; - const existing = readLockInfo(lockPath); - if ( - existing?.pid && - existing.pid !== process.pid && - isAgentDeviceDaemonProcess(existing.pid, existing.processStartTime) - ) { - return false; - } - try { - fs.unlinkSync(lockPath); - } catch {} - return tryWriteLock(); -} - -export function releaseDaemonLock(lockPath: string): void { - const existing = readLockInfo(lockPath); - if (existing && existing.pid !== process.pid) return; - try { - if (fs.existsSync(lockPath)) fs.unlinkSync(lockPath); - } catch {} -} - export function parseIntegerEnv(raw: string | undefined): number | undefined { if (raw === undefined) return undefined; const value = Number(raw); diff --git a/test/integration/provider-scenarios/daemon-lifecycle.test.ts b/test/integration/provider-scenarios/daemon-lifecycle.test.ts index c352315552..b85bc32a36 100644 --- a/test/integration/provider-scenarios/daemon-lifecycle.test.ts +++ b/test/integration/provider-scenarios/daemon-lifecycle.test.ts @@ -1,31 +1,30 @@ import assert from 'node:assert/strict'; import fs from 'node:fs'; -import os from 'node:os'; import path from 'node:path'; import { test } from 'vitest'; -import { - acquireDaemonLock, - parseIntegerEnv, - releaseDaemonLock, - removeInfoOwnedBy, - writeInfo, -} from '../../../src/daemon/server/server-lifecycle.ts'; +import { parseIntegerEnv } from '../../../src/daemon/server/server-lifecycle.ts'; +import { tryAcquireDaemonRegistration } from '../../../src/daemon-registration-owner.ts'; +import { resolveDaemonPaths } from '../../../src/daemon-resolution.ts'; +import { mkdtempForTestSync } from '../../../src/__tests__/test-utils/tmp-dir.ts'; -test('Provider-backed integration daemon lifecycle writes metadata and protects process-owned locks', () => { - const root = fs.mkdtempSync(path.join(os.tmpdir(), 'agent-device-daemon-lifecycle-')); +test('Provider-backed integration daemon lifecycle writes metadata and protects acquisitions', async () => { + const root = mkdtempForTestSync('agent-device-daemon-lifecycle-'); const infoPath = path.join(root, 'daemon.json'); - const lockPath = path.join(root, 'daemon.lock'); const logPath = path.join(root, 'daemon.log'); + const paths = resolveDaemonPaths(root); + const attempt = await tryAcquireDaemonRegistration(paths); + assert.equal(attempt.status, 'acquired'); + if (attempt.status !== 'acquired') throw new Error('registration refused'); + const { owner } = attempt; try { - writeInfo(root, infoPath, logPath, { + owner.publish({ socketPort: 4210, httpPort: 4310, token: 'provider-scenario-token', version: '0.0.0-provider-scenario', codeOrigin: 'checkout', codeSignature: 'graph:1:abc', - processStartTime: 'start-time', }); assert.equal(fs.existsSync(logPath), true); @@ -36,78 +35,38 @@ test('Provider-backed integration daemon lifecycle writes metadata and protects assert.equal(info.token, 'provider-scenario-token'); assert.equal(info.stateDir, root); - const httpOnlyInfoPath = path.join(root, 'daemon-http.json'); - writeInfo(root, httpOnlyInfoPath, path.join(root, 'daemon-http.log'), { + owner.publish({ httpPort: 4311, token: 'http-only-token', version: '0.0.0-provider-scenario', codeOrigin: 'checkout', codeSignature: 'graph:1:http', - processStartTime: undefined, }); - const httpOnlyInfo = JSON.parse(fs.readFileSync(httpOnlyInfoPath, 'utf8')); + const httpOnlyInfo = JSON.parse(fs.readFileSync(infoPath, 'utf8')); assert.equal(httpOnlyInfo.transport, 'http'); assert.equal(httpOnlyInfo.port, undefined); assert.equal(httpOnlyInfo.httpPort, 4311); - const socketOnlyInfoPath = path.join(root, 'daemon-socket.json'); - writeInfo(root, socketOnlyInfoPath, path.join(root, 'daemon-socket.log'), { + owner.publish({ socketPort: 4211, token: 'socket-only-token', version: '0.0.0-provider-scenario', codeOrigin: 'checkout', codeSignature: 'graph:1:socket', - processStartTime: undefined, }); - const socketOnlyInfo = JSON.parse(fs.readFileSync(socketOnlyInfoPath, 'utf8')); + const socketOnlyInfo = JSON.parse(fs.readFileSync(infoPath, 'utf8')); assert.equal(socketOnlyInfo.transport, 'socket'); assert.equal(socketOnlyInfo.port, 4211); assert.equal(socketOnlyInfo.httpPort, undefined); - - assert.equal( - acquireDaemonLock(root, lockPath, { - pid: process.pid, - version: '0.0.0-provider-scenario', - startedAt: 1, - }), - true, - ); - assert.equal( - acquireDaemonLock(root, lockPath, { - pid: process.pid, - version: '0.0.0-provider-scenario', - startedAt: 2, - }), - true, - ); - releaseDaemonLock(lockPath); - assert.equal(fs.existsSync(lockPath), false); + assert.equal((await tryAcquireDaemonRegistration(paths)).status, 'busy'); assert.equal(parseIntegerEnv('10'), 10); assert.equal(parseIntegerEnv('1.5'), undefined); assert.equal(parseIntegerEnv(undefined), undefined); - - const owner = { pid: process.pid, startTime: 'start-time' }; - fs.writeFileSync( - infoPath, - JSON.stringify({ pid: process.pid, processStartTime: 'start-time', token: 't', port: 1 }), - ); - assert.deepEqual(removeInfoOwnedBy(infoPath, owner), { removed: true }); - assert.equal(fs.existsSync(infoPath), false); - - // A successor's record survives this process's shutdown, which is what #3087 is about. - const foreignPath = path.join(root, 'daemon-foreign.json'); - fs.writeFileSync( - foreignPath, - JSON.stringify({ pid: 999_999_999, processStartTime: 'other', token: 't', port: 1 }), - ); - assert.deepEqual(removeInfoOwnedBy(foreignPath, owner), { - removed: false, - reason: 'replaced', - registeredPid: 999_999_999, - }); - assert.equal(fs.existsSync(foreignPath), true); } finally { + assert.deepEqual(await owner.finish(), { state: 'removed' }); + assert.equal(fs.existsSync(infoPath), false); + assert.equal(fs.existsSync(paths.lockPath), false); fs.rmSync(root, { recursive: true, force: true }); } }); From 7f5a9abd2ff8d3e08545008c37387a472567f9b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Fri, 2 Oct 2026 23:06:27 +0200 Subject: [PATCH 2/5] chore(gates): keep shutdown publication tracked at the acquisition-bound owner --- src/daemon/__tests__/atomic-publish-ownership.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/daemon/__tests__/atomic-publish-ownership.test.ts b/src/daemon/__tests__/atomic-publish-ownership.test.ts index c0c8c43db3..0075dc1c87 100644 --- a/src/daemon/__tests__/atomic-publish-ownership.test.ts +++ b/src/daemon/__tests__/atomic-publish-ownership.test.ts @@ -6,7 +6,7 @@ const SIMPLE_PUBLISHERS = [ // device-claims.ts delegates every write to device-claim-store.ts's writeDeviceClaim, the // single writer shared by the process-owned and allocator-held claim kinds. new URL('../device/device-claim-store.ts', import.meta.url), - new URL('../../daemon-shutdown-report.ts', import.meta.url), + new URL('../../daemon-registration-owner.ts', import.meta.url), new URL('../provider-lease-expiry.ts', import.meta.url), new URL('../session-script-writer.ts', import.meta.url), new URL('../../../packages/platform-apple/src/runner/runner-lease.ts', import.meta.url), From c02ff87de80b0c8f57777963376a5c95340f612a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Fri, 2 Oct 2026 23:43:50 +0200 Subject: [PATCH 3/5] fix: record registration release failures outside request scopes --- .../daemon-registration-owner.test.ts | 5 +- .../__tests__/daemon-client-metadata.test.ts | 1 - src/daemon-registration-owner.ts | 46 ++++++++----------- .../daemon-runtime-metadata-ownership.test.ts | 13 +++--- 4 files changed, 28 insertions(+), 37 deletions(-) diff --git a/src/__tests__/daemon-registration-owner.test.ts b/src/__tests__/daemon-registration-owner.test.ts index 9371681f89..0484dcd285 100644 --- a/src/__tests__/daemon-registration-owner.test.ts +++ b/src/__tests__/daemon-registration-owner.test.ts @@ -58,10 +58,6 @@ for (const [pid, startTime, reason] of [ [999_999_999, 'successor-start', 'replaced'], [process.pid, 'recycled-start', 'replaced'], [process.pid, undefined, 'unproven'], - [0, undefined, 'ownerless'], - [-3, undefined, 'ownerless'], - [1.5, undefined, 'ownerless'], - ['7', undefined, 'ownerless'], [null, undefined, 'ownerless'], ] as const) { test(`finish retains ${reason} registration (${String(pid)}, ${String(startTime)})`, async () => { @@ -161,4 +157,5 @@ test('unlink disappearance is settled, other failures remain primary even when r }); await assert.rejects(second.owner.finish(), (error) => error === primary); assert.equal(fs.existsSync(second.paths.infoPath), true); + assert.match(fs.readFileSync(second.paths.logPath, 'utf8'), /daemon_registration_release_failed/); }); diff --git a/src/daemon-client/__tests__/daemon-client-metadata.test.ts b/src/daemon-client/__tests__/daemon-client-metadata.test.ts index 3aad1ab51d..5b46a91e8d 100644 --- a/src/daemon-client/__tests__/daemon-client-metadata.test.ts +++ b/src/daemon-client/__tests__/daemon-client-metadata.test.ts @@ -22,7 +22,6 @@ vi.mock('../../daemon-process.ts', async (importOriginal) => ({ })); afterEach(() => vi.resetAllMocks()); - // The reuse decision is only as good as the identity that survives the round trip // through `daemon.json`: a client cannot compare what the file lost (#2458). diff --git a/src/daemon-registration-owner.ts b/src/daemon-registration-owner.ts index 119a2d96a5..ad58757f0e 100644 --- a/src/daemon-registration-owner.ts +++ b/src/daemon-registration-owner.ts @@ -1,4 +1,5 @@ import fs from 'node:fs'; +import { normalizeError } from '@agent-device/kernel/errors'; import { readCurrentOwnerIdentity, type OwnerIdentity } from '@agent-device/host-kit/process'; import { publishFileSync, @@ -6,7 +7,7 @@ import { type ProcessLockAttempt, type ProcessLockAcquisition, } from '@agent-device/host-kit/file'; -import { emitDiagnostic } from '@agent-device/host-kit/diagnostics'; +import { emitDiagnostic, withDiagnosticsScope } from '@agent-device/host-kit/diagnostics'; import type { DaemonCodeOrigin } from '@agent-device/host-kit/code-signature'; import type { DaemonPaths } from './daemon-resolution.ts'; import { @@ -20,7 +21,6 @@ import { } from './daemon-shutdown-report.ts'; export const DAEMON_STARTUP_EXIT_CODES = Object.freeze({ busy: 75, unproven: 78 }); - export type DaemonRegistrationFields = Readonly<{ socketPort?: number; httpPort?: number; @@ -30,17 +30,14 @@ export type DaemonRegistrationFields = Readonly<{ codeSignature: string; policyDigest?: string; }>; - type DaemonRegistrationRemoval = | Readonly<{ state: 'removed' }> | Exclude; - export type DaemonRegistrationOwner = Readonly<{ publish(fields: DaemonRegistrationFields): void; finish(outcome?: DaemonShutdownOutcome): Promise; }>; -/** Makes one acquisition attempt and binds every daemon write to that acquisition. */ export async function tryAcquireDaemonRegistration( paths: DaemonPaths, ): Promise< @@ -60,31 +57,26 @@ export async function tryAcquireDaemonRegistration( acquisition.assertHeld(); fs.rmSync(resolveDaemonShutdownReportPath(boundPaths.baseDir), { force: true }); } catch (error) { - await releaseRegistrationAfterFailure(acquisition, error); + await releaseRegistrationAfterFailure(acquisition, error, boundPaths.logPath); } return { status: 'acquired', owner: Object.freeze({ - publish(fields: DaemonRegistrationFields) { + publish({ socketPort, httpPort, ...fields }: DaemonRegistrationFields) { acquisition.assertHeld(); publishFileSync({ destination: boundPaths.logPath, contents: '', mode: 0o600 }); - const transport = - fields.socketPort && fields.httpPort ? 'dual' : fields.httpPort ? 'http' : 'socket'; + const transport = socketPort && httpPort ? 'dual' : httpPort ? 'http' : 'socket'; acquisition.assertHeld(); publishFileSync({ destination: boundPaths.infoPath, contents: JSON.stringify( { - port: fields.socketPort, - httpPort: fields.httpPort, + ...fields, + port: socketPort, + httpPort, transport, - token: fields.token, pid: identity.pid, - version: fields.version, - codeOrigin: fields.codeOrigin, - codeSignature: fields.codeSignature, processStartTime: identity.startTime ?? undefined, - policyDigest: fields.policyDigest, stateDir: boundPaths.baseDir, }, null, @@ -99,7 +91,7 @@ export async function tryAcquireDaemonRegistration( if (outcome) writeShutdownReport(boundPaths.baseDir, outcome, acquisition); removal = removeRegistrationUnderLock(boundPaths.infoPath, identity, acquisition); } catch (error) { - return await releaseRegistrationAfterFailure(acquisition, error); + return await releaseRegistrationAfterFailure(acquisition, error, boundPaths.logPath); } await acquisition.release(); return removal; @@ -128,15 +120,21 @@ function removeRegistrationUnderLock( async function releaseRegistrationAfterFailure( acquisition: ProcessLockAcquisition, error: unknown, + logPath: string, ): Promise { try { await acquisition.release(); } catch (releaseError) { - emitDiagnostic({ - level: 'warn', - phase: 'daemon_registration_release_failed', - data: { error: String(releaseError) }, - }); + await withDiagnosticsScope( + { command: 'daemon', session: 'daemon', logPath, debug: true }, + () => { + emitDiagnostic({ + level: 'warn', + phase: 'daemon_registration_release_failed', + data: { error: normalizeError(releaseError) }, + }); + }, + ); } throw error; } @@ -154,8 +152,4 @@ function writeShutdownReport( } catch { return; } - acquisition.assertHeld(); - try { - fs.chmodSync(filePath, 0o600); - } catch {} } diff --git a/src/daemon/server/daemon-runtime-metadata-ownership.test.ts b/src/daemon/server/daemon-runtime-metadata-ownership.test.ts index b9105675b6..9683b928e4 100644 --- a/src/daemon/server/daemon-runtime-metadata-ownership.test.ts +++ b/src/daemon/server/daemon-runtime-metadata-ownership.test.ts @@ -246,10 +246,11 @@ test('both exits tear the watch down before they touch daemon.json', () => { // publication needs a real toolchain — so the invariant is read off the source, the same way the // arming order above is. const source = fs.readFileSync(new URL('./daemon-runtime.ts', import.meta.url), 'utf8'); - const stopped = source.indexOf('stopMetadataLossWatch();'); - const removal = source.indexOf('await finishDaemonRegistration('); - - expect(stopped).toBeGreaterThanOrEqual(0); - expect(removal).toBeGreaterThan(stopped); - expect(source.match(/stopMetadataLossWatch\(\);/g)).toHaveLength(2); + const stops = [...source.matchAll(/stopMetadataLossWatch\(\);/g)].map((m) => m.index); + const finishes = [...source.matchAll(/await finishDaemonRegistration\(/g)].map((m) => m.index); + expect(stops).toHaveLength(2); + expect(finishes).toHaveLength(2); + expect(finishes[0]).toBeGreaterThan(stops[0]); + expect(finishes[0]).toBeLessThan(stops[1]); + expect(finishes[1]).toBeGreaterThan(stops[1]); }); From 7bca05f202be6336b0f84b69dc8dce4ff2bfa28e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Fri, 2 Oct 2026 23:45:11 +0200 Subject: [PATCH 4/5] test: narrow verified shutdown watch positions --- src/daemon/server/daemon-runtime-metadata-ownership.test.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/daemon/server/daemon-runtime-metadata-ownership.test.ts b/src/daemon/server/daemon-runtime-metadata-ownership.test.ts index 9683b928e4..fb428af30f 100644 --- a/src/daemon/server/daemon-runtime-metadata-ownership.test.ts +++ b/src/daemon/server/daemon-runtime-metadata-ownership.test.ts @@ -250,7 +250,7 @@ test('both exits tear the watch down before they touch daemon.json', () => { const finishes = [...source.matchAll(/await finishDaemonRegistration\(/g)].map((m) => m.index); expect(stops).toHaveLength(2); expect(finishes).toHaveLength(2); - expect(finishes[0]).toBeGreaterThan(stops[0]); - expect(finishes[0]).toBeLessThan(stops[1]); - expect(finishes[1]).toBeGreaterThan(stops[1]); + expect(finishes[0]).toBeGreaterThan(stops[0]!); + expect(finishes[0]).toBeLessThan(stops[1]!); + expect(finishes[1]).toBeGreaterThan(stops[1]!); }); From b5e49b371958be8c7988d2c0f42717d7b8248afb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sat, 3 Oct 2026 16:33:39 +0200 Subject: [PATCH 5/5] fix: empty the daemon log in place when registration publishes The daemon's stdout and stderr hold append descriptors on daemon.log, so an atomic rename gave the log a new inode and later output went to the unlinked file. Publication now truncates the existing inode. The held-lock provider shutdown test now loses to a real registration instead of a removed helper. --- src/__tests__/daemon-registration-owner.test.ts | 14 ++++++++++++++ src/daemon-registration-owner.ts | 12 +++++++++++- .../daemon-runtime-interactor-composition.test.ts | 14 +++++++++++--- 3 files changed, 36 insertions(+), 4 deletions(-) diff --git a/src/__tests__/daemon-registration-owner.test.ts b/src/__tests__/daemon-registration-owner.test.ts index 0484dcd285..ead76afb78 100644 --- a/src/__tests__/daemon-registration-owner.test.ts +++ b/src/__tests__/daemon-registration-owner.test.ts @@ -54,6 +54,20 @@ test('publication is atomic, binds identity and paths, and excludes a second acq assert.deepEqual(readDaemonShutdownReport(paths.baseDir), report); }); +test('publication truncates the log the daemon is already appending to', async () => { + const { paths, owner } = await acquire(); + const daemonOutput = fs.openSync(paths.logPath, 'a'); + try { + fs.writeSync(daemonOutput, 'previous run\n'); + owner.publish(fields); + fs.writeSync(daemonOutput, 'listening\n'); + } finally { + fs.closeSync(daemonOutput); + } + assert.equal(fs.readFileSync(paths.logPath, 'utf8'), 'listening\n'); + await owner.finish(report); +}); + for (const [pid, startTime, reason] of [ [999_999_999, 'successor-start', 'replaced'], [process.pid, 'recycled-start', 'replaced'], diff --git a/src/daemon-registration-owner.ts b/src/daemon-registration-owner.ts index ad58757f0e..92ec9f9ae0 100644 --- a/src/daemon-registration-owner.ts +++ b/src/daemon-registration-owner.ts @@ -64,7 +64,7 @@ export async function tryAcquireDaemonRegistration( owner: Object.freeze({ publish({ socketPort, httpPort, ...fields }: DaemonRegistrationFields) { acquisition.assertHeld(); - publishFileSync({ destination: boundPaths.logPath, contents: '', mode: 0o600 }); + truncateDaemonLog(boundPaths.logPath); const transport = socketPort && httpPort ? 'dual' : httpPort ? 'http' : 'socket'; acquisition.assertHeld(); publishFileSync({ @@ -153,3 +153,13 @@ function writeShutdownReport( return; } } + +/** The daemon's stdout and stderr append to this inode, so it is emptied in place, never replaced. */ +function truncateDaemonLog(logPath: string): void { + const descriptor = fs.openSync(logPath, 'a', 0o600); + try { + fs.ftruncateSync(descriptor, 0); + } finally { + fs.closeSync(descriptor); + } +} diff --git a/src/daemon/server/daemon-runtime-interactor-composition.test.ts b/src/daemon/server/daemon-runtime-interactor-composition.test.ts index e867a0755f..c191012ef0 100644 --- a/src/daemon/server/daemon-runtime-interactor-composition.test.ts +++ b/src/daemon/server/daemon-runtime-interactor-composition.test.ts @@ -6,6 +6,11 @@ import { setActiveProviderDeviceRuntimes } from '../../provider-device-runtime.t import { IOS_SIMULATOR } from '../../__tests__/test-utils/device-fixtures.ts'; import { mkdtempForTestSync } from '../../__tests__/test-utils/tmp-dir.ts'; import { createDaemonProviderRuntimeComposition } from '../../provider-device-runtimes.ts'; +import { + DAEMON_STARTUP_EXIT_CODES, + tryAcquireDaemonRegistration, +} from '../../daemon-registration-owner.ts'; +import { resolveDaemonPaths } from '../../daemon-resolution.ts'; import { interactorResolution } from '../interactor-resolution.ts'; vi.mock('../../platform-runtime.ts', () => ({ @@ -85,7 +90,9 @@ test('daemon startup composes the interactor resolution the daemon resolves thro }); test('a daemon attempt losing the lock shuts down every constructed provider', async () => { - vi.spyOn(await import('./server-lifecycle.ts'), 'acquireDaemonLock').mockReturnValueOnce(false); + const stateDir = mkdtempForTestSync('daemon-held-lock-'); + const held = await tryAcquireDaemonRegistration(resolveDaemonPaths(stateDir)); + if (held.status !== 'acquired') throw new Error('registration fixture refused'); const shutdown = vi.fn(() => { throw new Error('cleanup failed'); }); @@ -103,7 +110,7 @@ test('a daemon attempt losing the lock shuts down every constructed provider', a }); const exit = vi.fn(); const runtime = await startDaemonRuntime({ - env: { AGENT_DEVICE_STATE_DIR: mkdtempForTestSync('daemon-held-lock-') }, + env: { AGENT_DEVICE_STATE_DIR: stateDir }, exit, registerProcessHandlers: false, stderr: { write: () => {} }, @@ -112,5 +119,6 @@ test('a daemon attempt losing the lock shuts down every constructed provider', a expect(runtime).toBeNull(); expect(shutdown).toHaveBeenCalledOnce(); expect(otherShutdown).toHaveBeenCalledOnce(); - expect(exit).toHaveBeenCalledWith(0); + expect(exit).toHaveBeenCalledWith(DAEMON_STARTUP_EXIT_CODES.busy); + await held.owner.finish(); });