fix: centralize confirmed daemon retirement

CONTRIBUTING.md

@@ -37,6 +37,11 @@ pnpm test-app:typecheck
 `pnpm build` compiles the TypeScript CLI and library. If a running development daemon must pick up
 that build, use `pnpm rebuild:cli`; it builds and then stops the worktree-scoped daemon.
 
+`pnpm clean:daemon` retains state when it cannot confirm the recorded daemon exited. Restore
+process inspection or stop the verified owner before retrying. Its `--prune-dev` option considers
+dev state directories whose newest observed modification is at least 14 days old. It retires
+confirmed abandoned registrations and keeps session artifacts and state directories for inspection.
+
 Build only the Apple runner target you changed:
 
 ```bash

scripts/clean-daemon.ts

@@ -3,81 +3,60 @@ import os from 'node:os';
 import path from 'node:path';
 import { AppError } from '@agent-device/kernel/errors';
 import { resolveDaemonPaths } from '../src/daemon-resolution.ts';
-import { isAgentDeviceDaemonProcess, stopDaemonProcess } from '../src/daemon-process.ts';
+import { readRegisteredDaemonIdentity } from '../src/daemon-registration.ts';
+import {
+  stopAndRetireDaemon,
+  recoverAbandonedDaemonRegistration,
+} from '../src/daemon-registration-owner.ts';
 
 const DAEMON_TERM_TIMEOUT_MS = 15_000;
 const DAEMON_KILL_TIMEOUT_MS = 2_000;
 const PRUNE_DEV_FLAG = '--prune-dev';
 const PRUNE_DEV_MAX_AGE_MS = 14 * 24 * 60 * 60 * 1000;
 
-type DaemonInfo = {
-  pid?: number;
-  processStartTime?: string;
-};
-
 const paths = resolveDaemonPaths(process.env.AGENT_DEVICE_STATE_DIR);
-const info = readDaemonInfo(paths.infoPath);
-const daemonPid = readPositivePid(info?.pid);
-
-if (daemonPid !== null) {
-  const termination = await stopDaemonProcess(
-    { pid: daemonPid, startTime: info?.processStartTime ?? null },
+const retirement = await stopAndRetireDaemon({
+  paths,
+  observed: readRegisteredDaemonIdentity(paths.infoPath),
+  mode: 'graceful',
+  termTimeoutMs: DAEMON_TERM_TIMEOUT_MS,
+  killTimeoutMs: DAEMON_KILL_TIMEOUT_MS,
+});
+if (retirement.status === 'retained') {
+  throw new AppError(
+    'COMMAND_FAILED',
+    'Daemon cleanup retained state because retirement could not be confirmed.',
     {
-      mode: 'graceful',
-      termTimeoutMs: DAEMON_TERM_TIMEOUT_MS,
-      killTimeoutMs: DAEMON_KILL_TIMEOUT_MS,
+      reason: 'daemon_retirement_unconfirmed',
+      retirement,
+      hint: retirement.error?.hint,
     },
   );
-  if (termination.status !== 'exited') {
-    throw new AppError(
-      'COMMAND_FAILED',
-      'Daemon cleanup retained state because exit could not be confirmed.',
-      {
-        reason: 'daemon_exit_unconfirmed',
-        termination,
-      },
-    );
-  }
+}
+if (retirement.status === 'retired') {
   const { cleanupRunnerLeasesForOwner } =
     await import('@agent-device/platform-apple/runner/operations');
-  await cleanupRunnerLeasesForOwner({ pid: daemonPid, startTime: info?.processStartTime });
+  await cleanupRunnerLeasesForOwner(retirement.termination.identity);
 }
 
-removeIfPresent(paths.infoPath);
-removeIfPresent(paths.lockPath);
-
 if (process.argv.includes(PRUNE_DEV_FLAG)) {
-  pruneStaleDevStateDirs();
-}
-
-function readDaemonInfo(infoPath: string): DaemonInfo | null {
-  try {
-    return JSON.parse(fs.readFileSync(infoPath, 'utf8')) as DaemonInfo;
-  } catch {
-    return null;
-  }
-}
-
-function removeIfPresent(filePath: string): void {
-  try {
-    fs.unlinkSync(filePath);
-  } catch (error) {
-    if ((error as NodeJS.ErrnoException).code !== 'ENOENT') {
-      throw error;
-    }
-  }
+  await pruneStaleDevStateDirs();
 }
 
-// Removes worktree-scoped state dirs under ~/.agent-device/dev/ that no live daemon
-// owns and that have been idle past the retention threshold. Never touches the
-// global ~/.agent-device root contents.
-function pruneStaleDevStateDirs(): void {
+async function pruneStaleDevStateDirs(): Promise<void> {
   const devRoot = path.join(os.homedir(), '.agent-device', 'dev');
   const cutoffMs = Date.now() - PRUNE_DEV_MAX_AGE_MS;
   for (const dirPath of listDevStateDirs(devRoot)) {
-    if (hasLiveDaemon(dirPath) || newestMtimeMs(dirPath) > cutoffMs) continue;
-    fs.rmSync(dirPath, { recursive: true, force: true });
-    process.stdout.write(`Removed stale daemon state dir: ${dirPath}\n`);
+    if (newestMtimeMs(dirPath) > cutoffMs) continue;
+    const paths = resolveDaemonPaths(dirPath);
+    const result = await recoverAbandonedDaemonRegistration({
+      paths,
+      observed: readRegisteredDaemonIdentity(paths.infoPath),
+    });
+    if (result.status === 'retired')
+      process.stdout.write(
+        `Retired stale daemon registration: ${dirPath} (session artifacts retained)\n`,
+      );
   }
 }
 
@@ -93,17 +72,6 @@ function listDevStateDirs(devRoot: string): string[] {
     .map((entry) => path.join(devRoot, entry.name));
 }
 
-function hasLiveDaemon(stateDir: string): boolean {
-  const dirInfo = readDaemonInfo(path.join(stateDir, 'daemon.json'));
-  const pid = readPositivePid(dirInfo?.pid);
-  return pid !== null && isAgentDeviceDaemonProcess(pid, dirInfo?.processStartTime);
-}
-
-function readPositivePid(pid: number | undefined): number | null {
-  if (typeof pid !== 'number') return null;
-  return Number.isInteger(pid) && pid > 0 ? pid : null;
-}
-
 function newestMtimeMs(dirPath: string): number {
   let newest = statMtimeMs(dirPath);
   let children: fs.Dirent[];

src/__tests__/daemon-registration-owner.test.ts

@@ -2,7 +2,11 @@ import assert from 'node:assert/strict';
 import fs from 'node:fs';
 import { afterEach, test, vi } from 'vitest';
 import { readCurrentOwnerIdentity } from '@agent-device/host-kit/process';
-import { tryAcquireDaemonRegistration } from '../daemon-registration-owner.ts';
+import {
+  tryAcquireDaemonRegistration,
+  stopAndRetireDaemon,
+  recoverAbandonedDaemonRegistration,
+} from '../daemon-registration-owner.ts';
 import { resolveDaemonPaths, type DaemonPaths } from '../daemon-resolution.ts';
 import { readRegisteredDaemonOwnership } from '../daemon-registration.ts';
 import { readDaemonShutdownReport } from '../daemon-shutdown-report.ts';
@@ -173,3 +177,131 @@ test('unlink disappearance is settled, other failures remain primary even when r
   assert.equal(fs.existsSync(second.paths.infoPath), true);
   assert.match(fs.readFileSync(second.paths.logPath, 'utf8'), /daemon_registration_release_failed/);
 });
+
+const deadIdentity = { pid: 999_999_999, startTime: 'dead-start' };
+for (const scenario of [
+  {
+    name: 'missing-start',
+    observed: { pid: process.pid, startTime: null },
+    published: ownIdentity,
+    retired: false,
+  },
+  { name: 'live-unverified', observed: ownIdentity, published: ownIdentity, retired: false },
+  { name: 'no-observation', observed: null, published: deadIdentity, retired: false },
+  { name: 'replaced', observed: deadIdentity, published: ownIdentity, retired: false },
+  { name: 'dead', observed: deadIdentity, published: deadIdentity, retired: true },
+] as const) {
+  test(`retirement respects the captured lifetime (${scenario.name})`, async () => {
+    const paths = resolveDaemonPaths(mkdtempForTestSync('agent-device-retirement-'));
+    replaceInfo(paths, scenario.published.pid, scenario.published.startTime);
+    const before = fs.readFileSync(paths.infoPath, 'utf8');
+    const result = await stopAndRetireDaemon({
+      paths,
+      observed: scenario.observed,
+      mode: 'graceful',
+      lockTimeoutMs: 0,
+    });
+    if (scenario.retired) {
+      assert.equal(result.status, 'retired');
+      assert.equal(result.removedInfo, true);
+      assert.equal(fs.existsSync(paths.infoPath), false);
+    } else {
+      assert.equal(result.status, 'retained');
+      assert.equal(fs.readFileSync(paths.infoPath, 'utf8'), before);
+    }
+  });
+}
+
+test('abandoned recovery never signals a live owner and absence is inspected under the lock', async () => {
+  const { paths, owner } = await acquire();
+  owner.publish(fields);
+  const before = fs.readFileSync(paths.infoPath, 'utf8');
+  const result = await recoverAbandonedDaemonRegistration({ paths, observed: ownIdentity });
+  assert.equal(result.status, 'retained');
+  assert.equal(fs.readFileSync(paths.infoPath, 'utf8'), before);
+  await owner.finish();
+  assert.deepEqual(await recoverAbandonedDaemonRegistration({ paths, observed: null }), {
+    status: 'absent',
+    removedInfo: false,
+  });
+});
+
+test('retirement preserves a winner holding the startup lock instead of stopping or unlinking it', async () => {
+  const { paths, owner } = await acquire();
+  owner.publish({ ...fields, token: 'winning-token' });
+  const before = fs.readFileSync(paths.infoPath, 'utf8');
+  const result = await stopAndRetireDaemon({
+    paths,
+    observed: { pid: 999_999_999, startTime: 'dead-start' },
+    mode: 'force',
+    lockTimeoutMs: 0,
+  });
+  assert.equal(result.status, 'retained');
+  if (result.status !== 'retained') throw new Error('retirement unexpectedly completed');
+  assert.equal(result.reason, 'lock-busy');
+  assert.equal(result.error?.details?.reason, 'process_lock_timeout');
+  assert.match(result.error?.hint ?? '', /Restore process inspection/);
+  assert.equal(fs.readFileSync(paths.infoPath, 'utf8'), before);
+  assert.equal((await tryAcquireDaemonRegistration(paths)).status, 'busy');
+  await owner.finish();
+});
+
+test('metadata failure remains primary and normalized through a secondary release failure', async () => {
+  const paths = resolveDaemonPaths(mkdtempForTestSync('agent-device-retirement-failures-'));
+  const observed = { pid: 999_999_999, startTime: 'dead-start' };
+  replaceInfo(paths, observed.pid, observed.startTime);
+  const primary = Object.assign(new Error('metadata failure'), { code: 'EIO' });
+  const originalUnlink = fs.unlinkSync;
+  vi.spyOn(fs, 'unlinkSync').mockImplementation((target) => {
+    if (target === paths.infoPath) throw primary;
+    return originalUnlink(target);
+  });
+  const originalRmdir = fs.rmdirSync;
+  vi.spyOn(fs, 'rmdirSync').mockImplementation((target, options) => {
+    if (target === paths.lockPath) throw Object.assign(new Error('busy'), { code: 'EBUSY' });
+    return originalRmdir(target, options);
+  });
+  const result = await stopAndRetireDaemon({ paths, observed, mode: 'force' });
+  assert.equal(result.status, 'retained');
+  if (result.status !== 'retained') throw new Error('retirement unexpectedly completed');
+  assert.equal(result.error?.message, 'metadata failure');
+  assert.equal(result.error?.code, 'UNKNOWN');
+  assert.equal(result.error?.cause?.code, 'EIO');
+  assert.equal(fs.existsSync(paths.infoPath), true);
+});
+
+test('release failure reports partial retirement rather than success', async () => {
+  const paths = resolveDaemonPaths(mkdtempForTestSync('agent-device-retirement-partial-'));
+  const observed = { pid: 999_999_999, startTime: 'dead-start' };
+  replaceInfo(paths, observed.pid, observed.startTime);
+  const original = fs.rmdirSync;
+  vi.spyOn(fs, 'rmdirSync').mockImplementation((target, options) => {
+    if (target === paths.lockPath) throw Object.assign(new Error('busy'), { code: 'EBUSY' });
+    return original(target, options);
+  });
+  const result = await stopAndRetireDaemon({ paths, observed, mode: 'force' });
+  assert.equal(result.status, 'retained');
+  assert.equal(result.removedInfo, true);
+  assert.equal(fs.existsSync(paths.infoPath), false);
+  assert.equal(fs.existsSync(paths.lockPath), true);
+});
+
+test.skipIf(process.getuid?.() === 0)(
+  'an acquisition I/O failure retains metadata without reporting contention',
+  async () => {
+    const paths = resolveDaemonPaths(mkdtempForTestSync('agent-device-retirement-permission-'));
+    replaceInfo(paths, deadIdentity.pid, deadIdentity.startTime);
+    const before = fs.readFileSync(paths.infoPath, 'utf8');
+    fs.chmodSync(paths.baseDir, 0o500);
+    try {
+      const result = await recoverAbandonedDaemonRegistration({ paths, observed: deadIdentity });
+      assert.equal(result.status, 'retained');
+      if (result.status !== 'retained') throw new Error('retirement unexpectedly completed');
+      assert.equal(result.reason, 'retirement-unconfirmed');
+      assert.equal(fs.readFileSync(paths.infoPath, 'utf8'), before);
+      assert.equal(fs.existsSync(paths.lockPath), false);
+    } finally {
+      fs.chmodSync(paths.baseDir, 0o700);
+    }
+  },
+);