diff --git a/packages/daemon/docs/remote-device-lease-contracts.md b/packages/daemon/docs/remote-device-lease-contracts.md index 1172abe50a..e53612521e 100644 --- a/packages/daemon/docs/remote-device-lease-contracts.md +++ b/packages/daemon/docs/remote-device-lease-contracts.md @@ -75,6 +75,20 @@ A request on that session, including `close`, `artifacts`, and other admission-e Lease heartbeat and release name the lease id too. If a request omits the lease id, the daemon refuses it with `UNAUTHORIZED`, reason `LEASE_SESSION_MISMATCH`, and `field: 'leaseId'`. The daemon never fills a missing lease id from the session. +The daemon checks the lease id before the request reads, changes, or ends the session. +A request writes request events to the session journal only after that check. Thus, a request that is refused for the session lease id writes none, and it does not stamp session activity. +If a command writes request events and the request has access to an existing session, the daemon records its start before it runs the command. +For a lock-exempt `doctor`, `capabilities`, or `artifacts` command, the daemon records the start after the handler checks the session lease. +A request that sends the lease id of the session and is refused later, for example a `batch` after the lease deadline, can write its events. +The daemon checks the lease again when a lock-exempt command or `apps` reads the session. +If the lock-exempt command has access to the session and can take the session lock immediately, it can refresh the recording. +A `test` suite reads, records, and closes an attempt session only with the lease id that the suite sent. +After the lease deadline, a request that holds the session lock and sends the lease id of the session ends that session. +A request that sends a different active lease, and that lease admission accepts for a new session, also ends that expired session under its locks. It then runs as the new owner. Its reply and its diagnostics do not name the expired lease. +The successor starts admitted work before it ends the expired session. +A `batch`, a `replay`, and a bounded step do not end the session, and the lease sweep ends it. With the expired lease id, they are refused with `LEASE_NOT_FOUND`. With a different active lease, they are refused with `LEASE_SESSION_MISMATCH`. +A request without an active lease is refused and ends nothing. The lease sweep ends the session. +A command that holds no session lock ends no session. It leaves the expired session to the lease-end teardown. Tenant, run id, lease backend, client id, device key, and lease provider are labels. The daemon refuses a label that differs from the session lease. A missing label is not a refusal and grants nothing. The daemon compares each value that a request sends for a label: the value in `meta`, the value in `flags`, and each alias of a param. @@ -84,6 +98,7 @@ When an auth hook attests the tenant, the attested tenant routes the request. Th An allocate that repeats the labels of a live lease without that lease's id is refused as contention, with `DEVICE_IN_USE` and reason `DEVICE_LEASE_BUSY`. It returns no lease id. If the live lease has no device key, a label that contradicts it stays a scope mismatch, with `UNAUTHORIZED` and reason `LEASE_SCOPE_MISMATCH`. A repeat of a live lease with a device key is `DEVICE_LEASE_BUSY` whatever its client id says. Only an allocate that sends the id of its live lease gets that lease back. A caller-chosen lease id on allocate must have at least 32 hex digits (128 bits). A shorter id fails with reason `LEASE_ID_TOO_SHORT`. Commands that ignore the session lease, such as `devices`, `session_list`, and the lease RPCs, keep their own scope. +On a leased session, these commands do not bind the session, refresh its recording, deliver its script, renew its lease, end it, or write request events to it. ## Admitted request work diff --git a/packages/daemon/src/__tests__/support/human-control-router-fixture.ts b/packages/daemon/src/__tests__/support/human-control-router-fixture.ts index 7f9f72bbbe..d0b97534c5 100644 --- a/packages/daemon/src/__tests__/support/human-control-router-fixture.ts +++ b/packages/daemon/src/__tests__/support/human-control-router-fixture.ts @@ -42,6 +42,7 @@ export function createHumanControlHarness() { deviceRuntimeGateway: lifecycleDeviceRuntimeGateway, androidObservation: clearAndroidObservationFixture, platformResourceCleanup: platformResourceCleanupFixture(), + settleExpiredSession: async () => {}, trackDownloadableArtifact: () => 'artifact-1', }); return { registry, lease, sessionStore, sessionName, handleRequest }; diff --git a/packages/daemon/src/__tests__/support/test-device-runtime-gateway.ts b/packages/daemon/src/__tests__/support/test-device-runtime-gateway.ts index 2872462034..28e8eedb15 100644 --- a/packages/daemon/src/__tests__/support/test-device-runtime-gateway.ts +++ b/packages/daemon/src/__tests__/support/test-device-runtime-gateway.ts @@ -20,6 +20,7 @@ import { } from '../../server/request-router.ts'; import type { BindDeviceRuntime } from '../../core/request-runtime-binding.ts'; import type { DeviceInfo } from '@agent-device/kernel/device'; +import type { SessionRef } from '../../core/session-state.ts'; import { applicationLifecycleRuntimeFixture, type LifecycleOperationsLoader, @@ -33,6 +34,7 @@ import { daemonPlatformServicesFixture, } from './platform-services-fixture.ts'; import { platformResourceCleanupFixture } from './platform-resource-cleanup-fixture.ts'; +import { teardownSessionResources } from '../../sessions/session-teardown.ts'; import { providerCredentialsFixture } from './provider-credentials-fixture.ts'; import { unavailableBinding, @@ -218,6 +220,7 @@ export function createRequestHandler( | 'claimRecoveryGateway' | 'androidObservation' | 'platformResourceCleanup' + | 'settleExpiredSession' > & Partial< Pick< @@ -228,6 +231,7 @@ export function createRequestHandler( | 'claimRecoveryGateway' | 'androidObservation' | 'platformResourceCleanup' + | 'settleExpiredSession' > >, ) { @@ -238,10 +242,19 @@ export function createRequestHandler( claimRecoveryGateway = daemonClaimRecoveryGatewayFixture(), androidObservation = clearAndroidObservationFixture, platformResourceCleanup = platformResourceCleanupFixture(), + // The resource half of the daemon-owned settle, over this handler's cleanup fixture. + settleExpiredSession = async (ref: SessionRef) => + await teardownSessionResources({ + appLog: 'run', + ref, + sessionStore: deps.sessionStore, + platformCleanup: platformResourceCleanup, + }), ...rest } = deps; const handle = createProductionRequestHandler({ ...rest, + settleExpiredSession, androidObservation, platformResourceCleanup, platformServices, diff --git a/packages/daemon/src/core/lease-context.ts b/packages/daemon/src/core/lease-context.ts index f5454391b9..5b28d21f03 100644 --- a/packages/daemon/src/core/lease-context.ts +++ b/packages/daemon/src/core/lease-context.ts @@ -58,6 +58,8 @@ export type SessionLease = Readonly< type LeaseClaimField = keyof LeaseClaims; +type LeaseClaimRefusal = { kind: 'missing-id' } | { kind: 'mismatch'; field: LeaseClaimField }; + type LeaseClaimRequest = Pick; /** The flags that carry each lease field. `meta` carries each field under the field's own name. */ @@ -237,21 +239,36 @@ export function assertRequestHoldsSessionLease( * sends, and so does an empty label or a value that is not a string. */ export function assertRequestPresentsLease(req: LeaseClaimRequest, lease: SessionLease): void { - const claims = readLeaseClaims(req); - if (claims.leaseId.every((claim) => typeof claim === 'string' && !claim.trim())) { + const refusal = compareRequestLeaseClaims(req, lease); + if (!refusal) return; + if (refusal.kind === 'missing-id') { throw new AppError('UNAUTHORIZED', 'This session holds a device lease. Send its lease id.', { reason: LEASE_ADMISSION_REASONS.sessionMismatch.reason, field: 'leaseId', hint: 'Send the leaseId that the allocation returned (meta.leaseId or the leaseId param). From the CLI, open the session through connect proxy or connect --remote-config and keep the same --session; raw --tenant and --session flags do not send it.', }); } + throw new AppError('UNAUTHORIZED', `Lease does not match session owner (${refusal.field})`, { + reason: LEASE_ADMISSION_REASONS.sessionMismatch.reason, + field: refusal.field, + }); +} + +/** Reports the first refusal without constructing an error. */ +export function compareRequestLeaseClaims( + req: LeaseClaimRequest, + lease: SessionLease, +): LeaseClaimRefusal | undefined { + const claims = readLeaseClaims(req); + if (claims.leaseId.every((claim) => typeof claim === 'string' && !claim.trim())) { + return { kind: 'missing-id' }; + } for (const field of LEASE_CLAIM_FIELDS) { - if (claims[field].every((claim) => leaseClaimNamesLease(field, claim, lease))) continue; - throw new AppError('UNAUTHORIZED', `Lease does not match session owner (${field})`, { - reason: LEASE_ADMISSION_REASONS.sessionMismatch.reason, - field, - }); + if (!claims[field].every((claim) => leaseClaimNamesLease(field, claim, lease))) { + return { kind: 'mismatch', field }; + } } + return undefined; } function readLeaseClaims(req: LeaseClaimRequest): Record { diff --git a/packages/daemon/src/resources/lease-in-flight-work.ts b/packages/daemon/src/resources/lease-in-flight-work.ts index 0b943505aa..49e9a6afd0 100644 --- a/packages/daemon/src/resources/lease-in-flight-work.ts +++ b/packages/daemon/src/resources/lease-in-flight-work.ts @@ -14,6 +14,8 @@ export type LeaseWorkWanted = () => boolean; /** A claim on one leased device for the duration of one admitted request's work. */ export type LeaseWorkPass = Readonly<{ leaseId: string; + /** Whether this pass is still registered for its original lease. */ + isRetained(): boolean; /** * Ends the pass. Reports whether the work was still wanted when it ended, which * is what entitles it to renew the lease it just worked on. @@ -39,7 +41,11 @@ export class LeaseInFlightWorkRegistry { const entries = this.entriesByLeaseId.get(leaseId) ?? new Set(); entries.add(entry); this.entriesByLeaseId.set(leaseId, entries); - return { leaseId, release: () => this.releasePass(leaseId, entries, entry) }; + return { + leaseId, + isRetained: () => !entry.released && entries.has(entry), + release: () => this.releasePass(leaseId, entries, entry), + }; } /** diff --git a/packages/daemon/src/resources/lease-registry.ts b/packages/daemon/src/resources/lease-registry.ts index 50027b7c32..ceb1fcc318 100644 --- a/packages/daemon/src/resources/lease-registry.ts +++ b/packages/daemon/src/resources/lease-registry.ts @@ -52,6 +52,18 @@ const LEASE_REGISTRY_REASONS = { type OwnedHumanControlHold = { hold: HumanControlHold; ownerLeaseId?: string }; +export type RetainedLeaseWork = Pick & { + /** Refuses a released pass, even if the host reused its lease id. */ + assertRetained(): void; +}; + +function requireActiveLease(lease: DeviceLease | undefined): DeviceLease { + if (lease) return lease; + throw new AppError('UNAUTHORIZED', 'Lease is not active', { + reason: LEASE_REASONS.notFound.reason, + }); +} + export class LeaseRegistry { private readonly holdsByDevice = new Map>(); private readonly mutations = new DeviceMutationDrain(); @@ -230,11 +242,15 @@ export class LeaseRegistry { const leaseId = normalizeRequiredLeaseId(request.leaseId); this.cleanupExpiredLeases(); const lease = this.getActiveLease(leaseId); + return this.refreshLease(lease, this.resolveHeartbeatTtlMs(lease, request)); + } + + private resolveHeartbeatTtlMs(lease: DeviceLease, request: HeartbeatLeaseRequest): number { assertLeaseOwnerScope(lease, request); assertLeaseScopeMatch(lease, request); - const requestedTtlMs = - request.ttlMs === undefined ? leaseOwnTtlMs(lease) : this.resolveLeaseTtlMs(request.ttlMs); - return this.refreshLease(lease, requestedTtlMs); + return request.ttlMs === undefined + ? leaseOwnTtlMs(lease) + : this.resolveLeaseTtlMs(request.ttlMs); } /** @@ -246,10 +262,13 @@ export class LeaseRegistry { * human-control hold does; a pass whose request was cancelled protects nothing. * `wanted` is how the caller reports that its client is still waiting. */ - retainLeaseWork(lease: Pick, wanted: LeaseWorkWanted): LeaseWorkPass { + retainLeaseWork(lease: Pick, wanted: LeaseWorkWanted): RetainedLeaseWork { const pass = this.inFlightWork.retain(lease.leaseId, wanted); return { leaseId: pass.leaseId, + assertRetained: () => { + requireActiveLease(pass.isRetained() ? this.leases.get(pass.leaseId) : undefined); + }, release: () => { const renewed = pass.release(); if (renewed) this.refreshProtectedLease(lease.leaseId, this.now()); @@ -311,6 +330,27 @@ export class LeaseRegistry { assertLeaseScopeMatch(this.getActiveLease(scope.leaseId), scope); } + /** Checks request admission and renewal without sweeping or refreshing any lease. */ + inspectLeaseAdmission(request: HeartbeatLeaseRequest): DeviceLease { + return this.checkLeaseAdmission(request).lease; + } + + /** Renews a successor before its request tears down the incumbent session. */ + renewInspectedLeaseAdmission(request: HeartbeatLeaseRequest): DeviceLease { + const { lease, ttlMs } = this.checkLeaseAdmission(request); + return this.refreshLease(lease, ttlMs); + } + + private checkLeaseAdmission(request: HeartbeatLeaseRequest): { + lease: DeviceLease; + ttlMs: number; + } { + const scope = normalizeLeaseAdmissionRequest(request); + const lease = requireActiveLease(this.inspectActiveLease(scope.leaseId)); + assertLeaseScopeMatch(lease, scope); + return { lease, ttlMs: this.resolveHeartbeatTtlMs(lease, request) }; + } + /** Whether this daemon ever allocated `tenantId` a `macos-app` lease, released or not. */ hasHeldMacOsAppLease(tenantId: string): boolean { return this.macOsAppTenantIds.has(tenantId); @@ -666,11 +706,7 @@ export class LeaseRegistry { } private getActiveLease(leaseId: string): DeviceLease { - const lease = this.leases.get(leaseId); - if (lease) return lease; - throw new AppError('UNAUTHORIZED', 'Lease is not active', { - reason: LEASE_REASONS.notFound.reason, - }); + return requireActiveLease(this.leases.get(leaseId)); } /** Every renewal route ends here, so a host-pinned lease never outlives the host's window. */ diff --git a/packages/daemon/src/server/__tests__/request-execution-scope-lease-expiry.test.ts b/packages/daemon/src/server/__tests__/request-execution-scope-lease-expiry.test.ts index 7ba59f6cc3..79ba432b93 100644 --- a/packages/daemon/src/server/__tests__/request-execution-scope-lease-expiry.test.ts +++ b/packages/daemon/src/server/__tests__/request-execution-scope-lease-expiry.test.ts @@ -12,6 +12,7 @@ import { import { makeSessionStore } from '../../__tests__/support/store-factory.ts'; import { LeaseRegistry } from '../../resources/lease-registry.ts'; import { createRequestExecutionScope } from '../request-execution-scope.ts'; +import { teardownSessionResources } from '../../sessions/session-teardown.ts'; import type { DaemonRequest } from '../../core/daemon-request.ts'; import { platformRequestScopeFixture, @@ -63,7 +64,7 @@ test('expired leases remove owned sessions before the next command and free capa claimRecoveryGateway: daemonClaimRecoveryGatewayFixture(), deviceRuntimeGateway: unavailableDeviceRuntimeGateway, platformRequestScope: platformRequestScopeFixture(), - req: makeRequest({ command: 'snapshot' }), + req: makeRequest({ command: 'snapshot', meta: { leaseId: lease.leaseId } }), sessionStore, leaseRegistry, }); @@ -121,15 +122,25 @@ test.each(['rebuild', 'retire'] as const)( claimRecoveryGateway: daemonClaimRecoveryGatewayFixture(), deviceRuntimeGateway: unavailableDeviceRuntimeGateway, platformRequestScope: platformRequestScopeFixture(), - req: makeRequest({ session: address, flags: { session: address } }), + req: makeRequest({ + session: address, + flags: { session: address }, + meta: { leaseId: lease.leaseId }, + }), sessionStore: store, leaseRegistry: leases, - platformResourceCleanup: { - stopSnapshotHelper, - closeManagedBrowser: async () => {}, - cleanupSessionlessExecutionHost: async () => {}, - retainExecutionHostAfterClose: () => false, - }, + settleExpiredSession: async (ref) => + await teardownSessionResources({ + appLog: 'run', + ref, + sessionStore: store, + platformCleanup: { + stopSnapshotHelper, + closeManagedBrowser: async () => {}, + cleanupSessionlessExecutionHost: async () => {}, + retainExecutionHostAfterClose: () => false, + }, + }), }); expect(scope.sessionName).toBe(address); const running = scope.runLocked(async () => 'ran'); diff --git a/packages/daemon/src/server/application-lifecycle-recovery.ts b/packages/daemon/src/server/application-lifecycle-recovery.ts index ff5910371d..4ebc4a8f1c 100644 --- a/packages/daemon/src/server/application-lifecycle-recovery.ts +++ b/packages/daemon/src/server/application-lifecycle-recovery.ts @@ -31,10 +31,6 @@ type SessionFinalization = Readonly<{ daemonShutdown?: boolean; }>; -/** - * Both finalization entry points admit the same facts and run the same effects; they differ only - * in how the selected owner is bound and disposed, which each entry point supplies as `bind`. - */ async function finalizeSessionApplicationLifecycle(params: SessionFinalization): Promise { const { inspectFacts, bind, session, stateDir, runtimeHints } = params; const input = { @@ -57,16 +53,6 @@ async function finalizeSessionApplicationLifecycle(params: SessionFinalization): }); } -export async function finalizeBoundSessionApplicationLifecycle(params: { - inspectFacts: InspectDeviceRuntimeFacts; - bindDevice: BindDeviceRuntime; - session: SessionState; - stateDir: string; - runtimeHints: RuntimeHintValues; -}): Promise { - await finalizeSessionApplicationLifecycle({ ...params, bind: params.bindDevice }); -} - /** * Finish a persisted session's lifecycle-owned resources when the daemon, not a request, owns the * teardown. The selected owner is admitted from eager facts, then bound through the provider-first diff --git a/packages/daemon/src/server/http/daemon-runtime.ts b/packages/daemon/src/server/http/daemon-runtime.ts index 2726450ec2..4e8659c16d 100644 --- a/packages/daemon/src/server/http/daemon-runtime.ts +++ b/packages/daemon/src/server/http/daemon-runtime.ts @@ -34,7 +34,7 @@ import { getLeaseRegistryExecutionLocks, } from '../request-execution-scope.ts'; import type { LeaseEndReason } from '../../sessions/lease-lifecycle.ts'; -import { teardownSessionResources } from '../../sessions/session-teardown.ts'; +import { settleExpiredSession } from '../../sessions/session-teardown.ts'; import { resolveDaemonSessionTeardownTimeoutMs } from '../../sessions/session-teardown-budget.ts'; import { finalizeDaemonSessionApplicationLifecycle } from '../application-lifecycle-recovery.ts'; import { runtimeHintValues } from '../../sessions/session-runtime.ts'; @@ -328,6 +328,9 @@ async function startDaemonRuntimeOnHost( leaseRegistry, endSessionsForLease: async (leaseId, reason) => await endSessionsForLease(leaseId, reason ?? 'LEASE_RELEASED'), + // A request that ends an expired session retires it, so the application is always finalized. + settleExpiredSession: async (ref: SessionRef) => + await settleIdleExpiredSession(ref, 'finalize'), appLogAdmissionLedger, audioProbeAdmissionLedger, perfCaptureAdmissionLedger, @@ -402,26 +405,31 @@ async function startDaemonRuntimeOnHost( // survive so the next pass can retry rather than leave a claim owned by a process that no longer // knows what it holds. So: resources, then the platform finalization that stops the execution host // and releases its lease, then the claim, cleared last, by the reaper. - const settleIdleExpiredSession = async (ref: SessionRef): Promise => { + const settleIdleExpiredSession = async ( + ref: SessionRef, + afterResourceFailure: 'stop' | 'finalize' = 'stop', + ): Promise => { const session = sessionStore.resolveCurrent(ref) ?? ref.session; const sessionName = ref.address; const runtimeHints = runtimeHintValues(sessionStore.getRuntimeHints(sessionName)); - await teardownSessionResources({ + await settleExpiredSession({ appLog: 'run', ref, sessionStore, platformCleanup: deps.platformResourceCleanup, - }); - await finalizeDaemonSessionApplicationLifecycle({ - gateway: deviceRuntimeGateway, - scope: createRecoveryPlatformScope(), - session, - stateDir: baseDir, - runtimeHints, - // The one caller that must say so: this daemon is staying alive, so there is no shutdown - // phase a healthy runner could be deferred to. Taking the ordinary-close path stops the - // runner and releases its lease instead of parking it until process exit. - daemonLeaving: false, + afterResourceFailure, + finalizeApplication: async () => + await finalizeDaemonSessionApplicationLifecycle({ + gateway: deviceRuntimeGateway, + scope: createRecoveryPlatformScope(), + session, + stateDir: baseDir, + runtimeHints, + // The one caller that must say so: this daemon is staying alive, so there is no shutdown + // phase a healthy runner could be deferred to. Taking the ordinary-close path stops the + // runner and releases its lease instead of parking it until process exit. + daemonLeaving: false, + }), }); // ADR 0012 R7 binds this teardown too — the healed `.ad` is committed by // `SessionStore.finalizeRepairTeardown` — but NOT from in here. That call publishes the script diff --git a/packages/daemon/src/server/http/http-server-session-lease-touch.test.ts b/packages/daemon/src/server/http/http-server-session-lease-touch.test.ts new file mode 100644 index 0000000000..282c0003db --- /dev/null +++ b/packages/daemon/src/server/http/http-server-session-lease-touch.test.ts @@ -0,0 +1,797 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import { afterEach, beforeEach, expect, test, vi } from 'vitest'; +import * as hostCommand from '@agent-device/host-kit/command'; +import * as hostProcess from '@agent-device/host-kit/process'; +import { + IOS_SIMULATOR, + closeLoopbackServer, + createTestHostEnvironment, + listenOnLoopback, + mkdtempForTestSync, + skipWhenLoopbackUnavailable, +} from '@agent-device/test-kit'; +import { createTestDeviceInventoryGateways } from '@agent-device/device-selection/inventory-gateways-fixtures'; +import { createDaemonHttpServer } from './http-server.ts'; +import { + createLifecycleDeviceRuntimeGatewaySpies, + createRequestHandler, +} from '../../__tests__/support/test-device-runtime-gateway.ts'; +import { + authoringPublication, + makeIosAppSession, +} from '../../__tests__/support/session-factories.ts'; +import { makeSessionStore } from '../../__tests__/support/store-factory.ts'; +import { fakeCreateCommandSurface } from '../../__tests__/support/command-surface-fake.ts'; +import { daemonPlatformServicesFixture } from '../../__tests__/support/platform-services-fixture.ts'; +import { hostDiagnosticsFixture } from '../../__tests__/support/host-diagnostics-fixture.ts'; +import { platformResourceCleanupFixture } from '../../__tests__/support/platform-resource-cleanup-fixture.ts'; +import { makeTestScreenRecordingResource } from '../../__tests__/support/screen-recording-live-handle.ts'; +import { tenantScopedSessionName } from '../../core/session-tenant-scope.ts'; +import type { SessionRef } from '../../core/session-state.ts'; +import { LeaseRegistry } from '../../resources/lease-registry.ts'; +import { + createLeaseSessionEnd, + getLeaseRegistryExecutionLocks, + type LeaseSessionEnd, +} from '../request-execution-scope.ts'; + +const TENANT = 'e2b'; +const SESSION = 'agentA'; +const LEASE_TTL_MS = 60_000; +const OWNER_SCOPE = { + tenantId: TENANT, + runId: 'run-a', + leaseProvider: 'proxy', + clientId: 'client-a', + deviceKey: 'ios:mobile:SIM-001', +} as const; + +type RpcBody = { + result?: { ok?: boolean; data?: Record }; + error?: { + message?: string; + data?: { code?: string; details?: { reason?: string; field?: string } }; + }; +}; + +let world: Awaited> | undefined; + +beforeEach(async (context) => { + if (await skipWhenLoopbackUnavailable(context)) return; + world = await startDaemon(); +}); + +afterEach(async () => { + vi.restoreAllMocks(); + await world?.close(); + world = undefined; +}); + +async function startDaemon() { + let now = 1_000; + const sessionStore = makeSessionStore('agent-device-session-lease-touch-'); + const expiryTeardown = vi.fn(async (_ref: SessionRef) => {}); + const expiryEnds: Promise[] = []; + const leaseRegistry = new LeaseRegistry({ + now: () => now, + // The expiry owner of a real daemon: it ends the sessions of an expired lease under their locks. + onLeaseExpired: (lease) => { + expiryEnds.push(leaseSessionEnd.end(lease.leaseId, 'LEASE_EXPIRED')); + }, + }); + const leaseSessionEnd: LeaseSessionEnd = createLeaseSessionEnd({ + env: {}, + sessionStore, + executionLocks: getLeaseRegistryExecutionLocks(leaseRegistry), + teardownSession: expiryTeardown, + cleanupSessionlessExecutionHost: async () => {}, + }); + const lease = leaseRegistry.allocateLease({ + ...OWNER_SCOPE, + leaseBackend: 'ios-instance', + ttlMs: LEASE_TTL_MS, + }); + const sessionName = tenantScopedSessionName(TENANT, SESSION); + const publishLeasedSession = (address: string): SessionRef => { + const session = makeIosAppSession(address, { + sessionScope: { kind: 'tenant', id: TENANT }, + lease: { + leaseId: lease.leaseId, + tenantId: TENANT, + runId: OWNER_SCOPE.runId, + backend: lease.backend, + leaseProvider: OWNER_SCOPE.leaseProvider, + clientId: OWNER_SCOPE.clientId, + deviceKey: OWNER_SCOPE.deviceKey, + }, + }); + return sessionStore.publish(address, session); + }; + const observeRunnerSession = vi.fn(async () => ({ alive: true, sessionId: 'runner-1' })); + const hostDiagnostics = hostDiagnosticsFixture(); + // A request that ends an expired session itself stops the snapshot helper of that session. + const requestTeardown = platformResourceCleanupFixture(); + const runtime = createLifecycleDeviceRuntimeGatewaySpies(); + const hooks: { + beforeDispatch?: () => void | Promise; + duringDeviceSelection?: () => void; + } = {}; + const handleRequest = createRequestHandler({ + host: createTestHostEnvironment({ + env: { ...process.env, AGENT_DEVICE_MACOS_APP_BACKEND: 'native' }, + }), + createCommandSurface: fakeCreateCommandSurface, + logPath: path.join(mkdtempForTestSync('agent-device-session-lease-touch-log-'), 'daemon.log'), + token: 'daemon-token', + sessionStore, + leaseRegistry, + deviceInventoryGateways: createTestDeviceInventoryGateways({ + local: async () => { + hooks.duringDeviceSelection?.(); + return [IOS_SIMULATOR]; + }, + }), + deviceRuntimeGateway: runtime.gateway, + trackDownloadableArtifact: () => 'artifact-id', + hostDiagnostics, + platformResourceCleanup: requestTeardown, + platformServices: daemonPlatformServicesFixture({ + appleSessionObservation: { + observeRunnerSession, + resolveSoleForegroundApp: async () => undefined, + }, + }), + // Runs after admission and before the handler, where an owner's open can land for a command + // that holds no session lock. + providerDeviceRuntimeScope: async (task) => { + await hooks.beforeDispatch?.(); + return await task(); + }, + }); + const daemon = await createDaemonHttpServer({ + token: 'daemon-token', + handleRequest, + env: {}, + cwd: process.cwd(), + }); + const daemonPort = await listenOnLoopback(daemon); + const send = async ( + command: string, + positionals: string[], + meta: Record, + request: { session?: string; flags?: Record } = {}, + ) => { + const response = await fetch(`http://127.0.0.1:${daemonPort}/rpc`, { + method: 'POST', + headers: { authorization: 'Bearer daemon-token', 'content-type': 'application/json' }, + body: JSON.stringify({ + jsonrpc: '2.0', + id: command, + method: 'agent_device.command', + params: { + session: request.session ?? SESSION, + command, + positionals, + flags: request.flags ?? {}, + meta: { tenantId: TENANT, sessionIsolation: 'tenant', sessionExplicit: true, ...meta }, + }, + }), + }); + return (await response.json()) as RpcBody; + }; + return { + owner: { ...OWNER_SCOPE, leaseBackend: lease.backend, leaseId: lease.leaseId }, + /** Another client of the same proxy token and tenant, with a live lease of its own. */ + allocateOtherClient: (deviceKey: string = OWNER_SCOPE.deviceKey, ttlMs = 10 * LEASE_TTL_MS) => { + const scope = { + ...OWNER_SCOPE, + runId: 'run-b', + clientId: 'client-b', + deviceKey, + leaseBackend: 'ios-instance', + } as const; + const { leaseId } = leaseRegistry.allocateLease({ ...scope, ttlMs }); + return { ...scope, leaseId }; + }, + leaseId: lease.leaseId, + sessionName, + sessionStore, + leaseRegistry, + publishLeasedSession, + expiryTeardown, + observeRunnerSession, + hostDiagnostics, + requestTeardown, + runtime, + hooks, + send, + passLeaseDeadline: () => { + now += LEASE_TTL_MS + 1; + }, + expiryOwnerSettled: async () => { + await Promise.all(expiryEnds); + }, + requestEvents: async (address: string, command?: string) => { + await sessionStore.flushEvents(address); + return sessionStore + .readEvents(address) + .events.filter( + (event) => + event.kind.startsWith('request.') && + (command === undefined || event.command === command), + ) + .map((event) => event.kind); + }, + close: async () => { + await closeLoopbackServer(daemon); + }, + }; +} + +function expectLeaseIdRefusal(body: RpcBody): void { + expect(body.error?.data, JSON.stringify(body)).toMatchObject({ + code: 'UNAUTHORIZED', + details: { reason: 'LEASE_SESSION_MISMATCH', field: 'leaseId' }, + }); +} + +test('a refused request writes no event into the leased session, and the owner request does', async () => { + if (!world) return; + world.publishLeasedSession(world.sessionName); + + expectLeaseIdRefusal(await world.send('snapshot', [], {})); + expect(await world.requestEvents(world.sessionName)).toEqual([]); + + const snapshot = await world.send('snapshot', [], world.owner); + expect(snapshot.result?.ok, JSON.stringify(snapshot)).toBe(true); + expect(await world.requestEvents(world.sessionName)).toEqual([ + 'request.started', + 'request.finished', + ]); +}); + +test.each([ + ['no lease id', {}], + ['a lease id that is not active', { leaseId: 'f'.repeat(32) }], +])('a close with %s does not end an expired leased session', async (_name, scope) => { + if (!world) return; + const ref = world.publishLeasedSession(world.sessionName); + world.passLeaseDeadline(); + + const closed = await world.send('close', [], scope); + + expect(closed.error?.data, JSON.stringify(closed)).toMatchObject({ + code: 'UNAUTHORIZED', + details: { reason: 'LEASE_SESSION_MISMATCH' }, + }); + expect(JSON.stringify(closed)).not.toContain(world.leaseId); + expect(JSON.stringify(closed)).not.toContain('endedLeases'); + expect(world.sessionStore.lookup(world.sessionName)?.lifetime).toBe(ref.lifetime); + expect(world.expiryTeardown).not.toHaveBeenCalled(); + expect(world.requestTeardown.stopSnapshotHelper).not.toHaveBeenCalled(); + expect(world.runtime.inspectFacts).not.toHaveBeenCalled(); + expect(await world.requestEvents(world.sessionName)).toEqual([]); +}); + +test('a new owner opens the session right after the lease deadline, before the sweep', async () => { + if (!world) return; + world.publishLeasedSession(world.sessionName); + // Allocated before the deadline, so no registry call has noticed the expiry yet. + const successor = world.allocateOtherClient('ios:mobile:SIM-002'); + world.passLeaseDeadline(); + + const opened = await world.send('open', ['com.example.app'], successor); + + expect(opened.result?.ok, JSON.stringify(opened)).toBe(true); + expect(world.sessionStore.get(world.sessionName)?.lease?.leaseId).toBe(successor.leaseId); + expect(JSON.stringify(opened)).not.toContain(world.leaseId); + expect(JSON.stringify(opened)).not.toContain('endedLeases'); + expect(await world.requestEvents(world.sessionName)).toEqual([ + 'request.started', + 'request.finished', + ]); + // The open ended the expired session under its own locks, so the lease-end owner found none. + expect(world.requestTeardown.stopSnapshotHelper).toHaveBeenCalledTimes(1); + await world.expiryOwnerSettled(); + expect(world.expiryTeardown).not.toHaveBeenCalled(); +}); + +test('a successor open keeps its lease while the expired session teardown is pending', async () => { + const daemon = world; + if (!daemon) return; + daemon.publishLeasedSession(daemon.sessionName); + const successor = daemon.allocateOtherClient('ios:mobile:SIM-002', LEASE_TTL_MS + 2); + daemon.passLeaseDeadline(); + const originalExpiry = daemon.leaseRegistry.inspectActiveLease(successor.leaseId)?.expiresAt; + const teardownEntered = deferred(); + const releaseTeardown = deferred(); + daemon.requestTeardown.stopSnapshotHelper.mockImplementationOnce(async () => { + teardownEntered.resolve(); + await releaseTeardown.promise; + }); + + const open = daemon.send('open', ['com.example.app'], successor); + let protectedExpiry: number | undefined; + try { + await Promise.race([ + teardownEntered.promise, + open.then((response) => { + throw new Error(`Open finished before teardown: ${JSON.stringify(response)}`); + }), + ]); + daemon.passLeaseDeadline(); + daemon.passLeaseDeadline(); + protectedExpiry = daemon.leaseRegistry.inspectActiveLease(successor.leaseId)?.expiresAt; + } finally { + releaseTeardown.resolve(); + await open; + } + + const opened = await open; + expect(opened.result?.ok, JSON.stringify(opened)).toBe(true); + expect(protectedExpiry).toBeGreaterThan(originalExpiry ?? Infinity); + expect(daemon.sessionStore.get(daemon.sessionName)?.lease?.leaseId).toBe(successor.leaseId); + await daemon.expiryOwnerSettled(); + expect(daemon.expiryTeardown).not.toHaveBeenCalled(); +}); + +test('a successor open cannot use a lease id reissued during the expired session teardown', async () => { + const daemon = world; + if (!daemon) return; + daemon.publishLeasedSession(daemon.sessionName); + const successor = daemon.allocateOtherClient('ios:mobile:SIM-002'); + const replacement = { ...successor, runId: 'run-c', clientId: 'client-c' }; + daemon.passLeaseDeadline(); + let released = false; + daemon.requestTeardown.stopSnapshotHelper.mockImplementationOnce(async () => { + await Promise.resolve(); + released = daemon.leaseRegistry.releaseLease(successor).released; + daemon.leaseRegistry.putHostLease(successor.leaseId, replacement); + }); + + const opened = await daemon.send('open', ['com.example.app'], successor); + + expect(released).toBe(true); + expect(opened.error?.data, JSON.stringify(opened)).toMatchObject({ + code: 'UNAUTHORIZED', + details: { reason: 'LEASE_NOT_FOUND' }, + }); + expect(daemon.sessionStore.get(daemon.sessionName)).toBeUndefined(); + expect(daemon.leaseRegistry.inspectActiveLease(successor.leaseId)?.runId).toBe(replacement.runId); + await daemon.expiryOwnerSettled(); +}); + +test('a new owner that lease admission would refuse does not end the expired session', async () => { + if (!world) return; + const ref = world.publishLeasedSession(world.sessionName); + const { clientId: _omitted, ...successor } = world.allocateOtherClient('ios:mobile:SIM-002'); + world.passLeaseDeadline(); + + // A proxy open of a new session needs the client id. + const opened = await world.send('open', ['com.example.app'], successor); + + expect(opened.error?.data?.code, JSON.stringify(opened)).toBe('INVALID_ARGS'); + expect(world.sessionStore.lookup(world.sessionName)?.lifetime).toBe(ref.lifetime); + expect(world.requestTeardown.stopSnapshotHelper).not.toHaveBeenCalled(); +}); + +test('an unscoped open cannot replace an expired leased session', async () => { + if (!world) return; + const ref = world.publishLeasedSession(SESSION); + const { leaseId } = world.allocateOtherClient('ios:mobile:SIM-002'); + world.passLeaseDeadline(); + + const opened = await world.send('open', ['com.example.app'], { + leaseId, + tenantId: undefined, + sessionIsolation: 'none', + }); + + expect(world.sessionStore.lookup(SESSION)?.lifetime).toBe(ref.lifetime); + expect(world.requestTeardown.stopSnapshotHelper).not.toHaveBeenCalled(); + expectLeaseIdRefusal(opened); +}); + +test.each([ + { + command: 'snapshot', + positionals: [], + meta: { clientId: undefined }, + error: { code: 'UNAUTHORIZED', details: { reason: 'LEASE_SCOPE_REQUIRED' } }, + }, + { + command: 'open', + positionals: ['com.example.app'], + meta: { leaseTtlMs: 1 }, + error: { code: 'INVALID_ARGS' }, + }, +])( + 'a successor that cannot heartbeat on $command leaves the expired session intact', + async ({ command, positionals, meta, error }) => { + if (!world) return; + const ref = world.publishLeasedSession(world.sessionName); + const successor = world.allocateOtherClient('ios:mobile:SIM-002'); + world.passLeaseDeadline(); + + const response = await world.send(command, positionals, { ...successor, ...meta }); + + expect(world.sessionStore.lookup(world.sessionName)?.lifetime).toBe(ref.lifetime); + expect(world.requestTeardown.stopSnapshotHelper).not.toHaveBeenCalled(); + expect(response.error?.data, JSON.stringify(response)).toMatchObject(error); + }, +); + +test.each([ + { + failure: 'process has stopped', + processAlive: false, + details: { reason: 'MACOS_APP_LEASE_DENIED', rule: 'process' }, + }, + { + failure: 'lease expires during PID lookup', + processAlive: true, + details: { reason: 'LEASE_SESSION_MISMATCH', field: 'leaseId' }, + }, +])( + 'a macOS app successor leaves the expired session intact when the $failure', + async ({ processAlive, details }) => { + const daemon = world; + if (!daemon) return; + const ref = daemon.publishLeasedSession(daemon.sessionName); + const bundleId = 'com.example.app'; + const pid = 12345; + const leaseId = 'b'.repeat(32); + const successor = { + leaseId, + tenantId: TENANT, + runId: 'run-b', + leaseBackend: 'macos-app', + deviceKey: `${bundleId}@${pid}`, + } as const; + daemon.leaseRegistry.putHostLease(leaseId, { ...successor, ttlMs: 2 * LEASE_TTL_MS }); + const { isProcessAlive } = hostProcess; + vi.spyOn(hostProcess, 'isProcessAlive').mockImplementation((observedPid) => + observedPid === pid ? processAlive : isProcessAlive(observedPid), + ); + if (processAlive) { + const { runCmd } = hostCommand; + vi.spyOn(hostCommand, 'runCmd').mockImplementation(async (command, args, options) => { + if (command !== '/usr/bin/lsappinfo') return await runCmd(command, args, options); + await Promise.resolve(); + daemon.passLeaseDeadline(); + return { exitCode: 0, stderr: '', stdout: `bundleID="${bundleId}"` }; + }); + } + daemon.passLeaseDeadline(); + + const opened = await daemon.send('open', [bundleId], successor, { + flags: { platform: 'macos' }, + }); + + expect(daemon.sessionStore.lookup(daemon.sessionName)?.lifetime).toBe(ref.lifetime); + expect(daemon.requestTeardown.stopSnapshotHelper).not.toHaveBeenCalled(); + expect(opened.error?.data, JSON.stringify(opened)).toMatchObject({ + code: 'UNAUTHORIZED', + details, + }); + }, +); + +test('doctor with the lease id after the deadline leaves the session to the lease-end owner', async () => { + if (!world) return; + const ref = world.publishLeasedSession(world.sessionName); + world.passLeaseDeadline(); + + await world.send('doctor', [], world.owner); + + expect(world.requestTeardown.stopSnapshotHelper).not.toHaveBeenCalled(); + await world.expiryOwnerSettled(); + expect(world.expiryTeardown).toHaveBeenCalledTimes(1); + expect(world.expiryTeardown.mock.calls[0]?.[0].lifetime).toBe(ref.lifetime); +}); + +test('the owner request after the deadline ends its own expired session', async () => { + if (!world) return; + world.publishLeasedSession(world.sessionName); + world.passLeaseDeadline(); + + const snapshot = await world.send('snapshot', [], world.owner); + + expect(snapshot.error?.data?.code, JSON.stringify(snapshot)).toBe('UNAUTHORIZED'); + expect(snapshot.error?.data?.details?.reason).not.toBe('LEASE_SESSION_MISMATCH'); + expect(world.sessionStore.get(world.sessionName)).toBeUndefined(); +}); + +test('devices still checks the recording of a session that holds no lease', async () => { + if (!world) return; + const session = makeIosAppSession(world.sessionName, { + sessionScope: { kind: 'tenant', id: TENANT }, + }); + session.screenRecording = makeTestScreenRecordingResource(session, { + backend: 'runner AVAssetWriter', + runnerSessionId: 'runner-0', + }); + world.sessionStore.publish(world.sessionName, session); + + const devices = await world.send('devices', [], {}); + + expect(world.observeRunnerSession).toHaveBeenCalledTimes(1); + expect(devices.error?.message, JSON.stringify(devices)).toBe( + 'iOS runner session restarted during recording', + ); +}); + +test('devices with no lease id stays detached from a leased session and its recording', async () => { + if (!world) return; + const { session } = world.publishLeasedSession(world.sessionName); + session.screenRecording = makeTestScreenRecordingResource(session, { + backend: 'runner AVAssetWriter', + runnerSessionId: 'runner-0', + }); + + const devices = await world.send('devices', [], {}); + + expect(devices.result?.ok, JSON.stringify(devices)).toBe(true); + expect(world.observeRunnerSession).not.toHaveBeenCalled(); + expect(session.screenRecording.handle.inspect().invalidatedReason).toBeUndefined(); + expect(await world.requestEvents(world.sessionName)).toEqual([]); + + // The owner's command still checks the recording: the runner restarted, so it is refused. + const snapshot = await world.send('snapshot', [], world.owner); + expect(world.observeRunnerSession).toHaveBeenCalledTimes(1); + expect(snapshot.error?.message, JSON.stringify(snapshot)).toBe( + 'iOS runner session restarted during recording', + ); +}); + +function deferred() { + let resolve!: () => void; + const promise = new Promise((done) => { + resolve = done; + }); + return { promise, resolve }; +} + +test.each([ + { scenario: 'leased', leased: true, openBeforeDispatch: false }, + { scenario: 'unleased', leased: false, openBeforeDispatch: false }, + { scenario: 'opened leased before dispatch', leased: true, openBeforeDispatch: true }, + { scenario: 'opened unleased before dispatch', leased: false, openBeforeDispatch: true }, +])( + 'events shows the doctor start for a session that is $scenario while the toolchain check is pending', + async ({ leased, openBeforeDispatch }) => { + const daemon = world; + if (!daemon) return; + const { owner, sessionStore, hooks } = daemon; + const sessionName = leased ? daemon.sessionName : SESSION; + const scope = leased ? owner : { sessionIsolation: 'none', tenantId: undefined }; + if (openBeforeDispatch) { + hooks.beforeDispatch = async () => { + hooks.beforeDispatch = undefined; + const opened = await daemon.send('open', ['com.example.app'], scope); + expect(opened.result?.ok, JSON.stringify(opened)).toBe(true); + expect(sessionStore.get(sessionName)?.lease?.leaseId).toBe( + leased ? owner.leaseId : undefined, + ); + }; + } else if (leased) { + daemon.publishLeasedSession(sessionName); + } else { + sessionStore.publish(sessionName, makeIosAppSession(sessionName)); + } + const enteredToolchain = deferred(); + const releaseToolchain = deferred(); + vi.mocked(daemon.hostDiagnostics.toolchainCheck).mockImplementation(async () => { + enteredToolchain.resolve(); + await releaseToolchain.promise; + return undefined; + }); + + const doctor = daemon.send('doctor', [], scope); + try { + await Promise.race([ + enteredToolchain.promise, + doctor.then((response) => { + throw new Error( + `Doctor finished before its toolchain check: ${JSON.stringify(response)}`, + ); + }), + ]); + + const events = await daemon.send('events', [], scope); + + expect(events.result?.ok).toBe(true); + expect(events.result?.data?.events).toContainEqual( + expect.objectContaining({ command: 'doctor', kind: 'request.started' }), + ); + } finally { + releaseToolchain.resolve(); + await doctor; + } + expect((await doctor).result?.ok).toBe(true); + expect(await daemon.requestEvents(sessionName, 'doctor')).toEqual([ + 'request.started', + 'request.finished', + ]); + }, +); + +// `apps` needs a lease of its own to reach dispatch, so it sends the lease of another client. +test.each([ + ['doctor', false], + ['events', false], + ['capabilities', false], + ['apps', true], +] as const)( + '%s without the session lease id returns nothing from a leased session opened after its admission', + async (command, sendsOtherLease) => { + if (!world) return; + const { hooks, publishLeasedSession, sessionName } = world; + const scope = sendsOtherLease ? world.allocateOtherClient('ios:mobile:SIM-002') : {}; + hooks.beforeDispatch = () => { + publishLeasedSession(sessionName); + }; + + expectLeaseIdRefusal(await world.send(command, [], scope)); + + expect(world.hostDiagnostics.deviceChecks).not.toHaveBeenCalled(); + expect(world.runtime.inspectFacts).not.toHaveBeenCalled(); + // The request arrived at an empty address, and still leaves no finished event for the new session. + expect(await world.requestEvents(sessionName)).not.toContain('request.finished'); + if (!sendsOtherLease) expect(await world.requestEvents(sessionName)).toEqual([]); + }, +); + +test.each(['doctor', 'capabilities', 'artifacts'] as const)( + '%s leaves no start when open leases the existing session before the handler read', + async (command) => { + const daemon = world; + if (!daemon) return; + const { owner, sessionName, sessionStore, hooks } = daemon; + const ref = sessionStore.publish( + sessionName, + makeIosAppSession(sessionName, { sessionScope: { kind: 'tenant', id: TENANT } }), + ); + hooks.beforeDispatch = async () => { + hooks.beforeDispatch = undefined; + const opened = await daemon.send('open', ['com.example.app'], owner); + expect(opened.result?.ok, JSON.stringify(opened)).toBe(true); + const current = sessionStore.lookup(sessionName); + expect(current?.lifetime).toBe(ref.lifetime); + expect(current?.session.lease?.leaseId).toBe(owner.leaseId); + }; + + expectLeaseIdRefusal(await daemon.send(command, [], {})); + + const events = await daemon.send('events', [], owner); + expect(events.result?.ok, JSON.stringify(events)).toBe(true); + expect(events.result?.data?.events).not.toContainEqual( + expect.objectContaining({ command, kind: 'request.started' }), + ); + }, +); + +test('devices with no lease id is still refused at the address of a macos-app leased session', async () => { + if (!world) return; + const { session } = world.publishLeasedSession(world.sessionName); + session.lease = { ...session.lease!, backend: 'macos-app' }; + + const devices = await world.send('devices', [], {}); + + // The app lease admits only the commands that opt into it, and `devices` does not. + expect(devices.result, JSON.stringify(devices)).toBeUndefined(); + expect(devices.error?.data?.code).toBe('INVALID_ARGS'); + expect(session.lease.backend).toBe('macos-app'); +}); + +test('devices with no lease id delivers no script of a leased session that records one', async () => { + if (!world) return; + const { session } = world.publishLeasedSession(world.sessionName); + session.scriptPublication = authoringPublication('armed', { remote: true }); + // The owner records an action while `devices` runs. + world.hooks.beforeDispatch = () => { + session.actions.push({ ts: Date.now(), command: 'home', positionals: [], flags: {} }); + }; + + const devices = await world.send('devices', [], {}); + + expect(devices.result?.ok, JSON.stringify(devices)).toBe(true); + expect(JSON.stringify(devices)).not.toContain('publication-0001'); + expect(session.scriptPublication).toEqual(authoringPublication('armed', { remote: true })); +}); + +test('capabilities with no lease id reads nothing from a leased session opened during its device selection', async () => { + if (!world) return; + const { hooks, publishLeasedSession, sessionName } = world; + hooks.duringDeviceSelection = () => { + publishLeasedSession(sessionName); + }; + // The runtime cannot read the foreground app here, so only a session app would list `appstate`. + const inspectFacts = world.runtime.inspectFacts.getMockImplementation()!; + world.runtime.inspectFacts.mockImplementation(async (device) => { + const facts = await inspectFacts(device); + const unavailable = { available: false, reason: 'unsupported-device-kind' } as const; + return { + ...facts, + operations: { ...facts.operations, ensureReady: { available: true }, appState: unavailable }, + }; + }); + + const capabilities = await world.send('capabilities', [], {}, { flags: { platform: 'ios' } }); + + expect(capabilities.result?.ok, JSON.stringify(capabilities)).toBe(true); + expect(world.sessionStore.get(sessionName)).toBeDefined(); + // `appstate` is available only through the app of the session. + expect(JSON.stringify(capabilities.result?.data)).not.toContain('appstate'); +}); + +test('doctor with the lease id reports its leased session', async () => { + if (!world) return; + world.publishLeasedSession(world.sessionName); + + const doctor = await world.send('doctor', [], world.owner); + + expect(doctor.result?.ok, JSON.stringify(doctor)).toBe(true); + expect(JSON.stringify(doctor.result?.data)).toContain(world.sessionName); + expect(world.hostDiagnostics.deviceChecks).toHaveBeenCalledTimes(1); + expect(await world.requestEvents(world.sessionName)).toEqual([ + 'request.started', + 'request.finished', + ]); +}); + +function writeSuite(): { flags: Record; positionals: string[] } { + const root = mkdtempForTestSync('agent-device-session-lease-touch-suite-'); + const scriptPath = path.join(root, 'flow.ad'); + const script = 'context platform=ios\nwait 0\n'; + fs.writeFileSync(scriptPath, script); + return { + positionals: [scriptPath], + flags: { + recordVideo: true, + replayScriptSources: [{ entry: scriptPath, files: { [scriptPath]: script } }], + }, + }; +} + +// A direct RPC caller chooses the suite session and request id, and so the attempt address. +const SUITE_SESSION = 'suite'; +const SUITE_REQUEST_ID = 'chosen'; +const attemptAddress = (): string => + `${tenantScopedSessionName(TENANT, SUITE_SESSION)}:test:${SUITE_REQUEST_ID}:1-flow:attempt-1`; + +test('a test suite of another client neither records nor closes a leased session at its attempt address', async () => { + if (!world) return; + const ref = world.publishLeasedSession(attemptAddress()); + const suite = writeSuite(); + + const response = await world.send( + 'test', + suite.positionals, + { ...world.allocateOtherClient('ios:mobile:SIM-002'), requestId: SUITE_REQUEST_ID }, + { session: SUITE_SESSION, flags: suite.flags }, + ); + + expect(JSON.stringify(response)).toContain('LEASE_SESSION_MISMATCH'); + const attempt = world.sessionStore.lookup(attemptAddress()); + expect(attempt?.lifetime).toBe(ref.lifetime); + expect(attempt?.session.screenRecording).toBeUndefined(); + expect(world.runtime.bind).not.toHaveBeenCalled(); +}); + +test('a test suite with the lease id runs on and closes its leased attempt session', async () => { + if (!world) return; + world.publishLeasedSession(attemptAddress()); + const suite = writeSuite(); + + const response = await world.send( + 'test', + suite.positionals, + { ...world.owner, requestId: SUITE_REQUEST_ID }, + { session: SUITE_SESSION, flags: { ...suite.flags, recordVideo: false } }, + ); + + expect(response.result?.data, JSON.stringify(response)).toMatchObject({ passed: 1, failed: 0 }); + expect(world.sessionStore.get(attemptAddress())).toBeUndefined(); +}); diff --git a/packages/daemon/src/server/request-binding.ts b/packages/daemon/src/server/request-binding.ts index 2a724476d5..2e4d856162 100644 --- a/packages/daemon/src/server/request-binding.ts +++ b/packages/daemon/src/server/request-binding.ts @@ -156,7 +156,7 @@ function resolveExplicitDeviceLock(req: DaemonRequest, env: EnvMap) { : undefined; } -function sessionExecutionLockKey(sessionName: string): RequestExecutionLockKey { +export function sessionExecutionLockKey(sessionName: string): RequestExecutionLockKey { return `session:${sessionName}`; } diff --git a/packages/daemon/src/server/request-execution-scope.ts b/packages/daemon/src/server/request-execution-scope.ts index 35e96bda36..ce74d7e14b 100644 --- a/packages/daemon/src/server/request-execution-scope.ts +++ b/packages/daemon/src/server/request-execution-scope.ts @@ -31,7 +31,10 @@ import { assertProviderLeaseRequestNamesSession, assertRequestLeaseAdmissionPreflight, assertRequestTenantAdmission, + requestMayRecordSessionEvents, resolveRequestSessionName, + resolveRequestSessionRef, + type ResolveRequestSessionRef, scopeRequestSession, withTenantAdmissionOnRoutingError, } from '../sessions/request-admission.ts'; @@ -40,6 +43,8 @@ import { admitRequestLeaseForLockedScope, cleanupExpiredLeasedSession, endLeasedSession, + type ExpiredSessionTeardown, + resolveLockedRequestSessionRef, inspectSessionProviderReleaseOnClose, type LeaseEndReason, type SessionTeardown, @@ -48,6 +53,7 @@ import { existingSessionExecutionLockKeys, prepareLockedRequestBinding, resolveRequestExecutionLockPlan, + sessionExecutionLockKey, type RequestExecutionLockPlan, } from './request-binding.ts'; import { @@ -63,9 +69,10 @@ import { isRequestCanceled, throwIfRequestCanceled } from '@agent-device/host-ki import { finalizeDaemonResponse } from './request-finalization.ts'; import { remoteScriptVersionDelivery } from '../core/session-script-publication-capability.ts'; import { refreshRecordingHealth } from '../sessions/request-recording-health.ts'; -import { runAdmittedLeaseWork } from './request-lease-work.ts'; +import { retainSuccessorLeaseWork, runAdmittedLeaseWork } from './request-lease-work.ts'; import { assertMacOsAppLeaseProcess } from '../resources/macos-app-lease.ts'; import { + ignoresSessionLease, resolveCommandAndroidAvdSelection, shouldBlockForInvalidRecording, shouldLockSessionExecution, @@ -76,13 +83,10 @@ import { buildRequestStartedEvent, shouldRecordEventForRequest, } from '@agent-device/session-journal/session-event-log'; -import type { LeaseRegistry } from '../resources/lease-registry.ts'; +import type { LeaseRegistry, RetainedLeaseWork } from '../resources/lease-registry.ts'; import { type SessionStore } from '../core/session-store.ts'; import type { DaemonRequest } from '../core/daemon-request.ts'; import type { SessionRef, SessionState } from '../core/session-state.ts'; -import { teardownSessionResources } from '../sessions/session-teardown.ts'; -import { finalizeBoundSessionApplicationLifecycle } from './application-lifecycle-recovery.ts'; -import { runtimeHintValues } from '../sessions/session-runtime.ts'; import type { DeviceRuntimeGateway } from '@agent-device/contracts/platform-runtime'; import type { PlatformRuntimeOperations } from '@agent-device/contracts/platform-runtime-operations'; import type { PlatformRequestScope } from '@agent-device/contracts/platform-runtime-host'; @@ -163,6 +167,22 @@ async function resolveBoundedClaimDevice( return await resolveCommandDevice(params); } +const expiredSessionSettleNotComposed: SessionTeardown = async () => { + throw new AppError( + 'INTERNAL_ERROR', + 'Expired-session teardown was not supplied by root runtime composition', + ); +}; + +function expiredSessionTeardown( + command: string, + holdsSessionLock: boolean, + boundedStep: boolean, +): ExpiredSessionTeardown { + if (!holdsSessionLock) return 'lease-end-owner'; + return command === 'batch' || command === 'replay' || boundedStep ? 'refuse' : 'inline'; +} + // Production daemon wiring owns one LeaseRegistry per process; scoping locks by registry keeps // test and embedded routers isolated without changing process-level serialization there. const leaseRegistryExecutionLocks = new WeakMap>>(); @@ -170,6 +190,7 @@ const requestScopeFinalizers = new WeakMap< RequestExecutionScope, (response: DaemonResponse) => DaemonResponse >(); +const HANDLER_JOURNAL_START_COMMANDS = new Set(['doctor', 'capabilities', 'artifacts']); export type RequestExecutionScope = AsyncDisposable & { /** The environment and working directory of the daemon serving this request. */ @@ -179,9 +200,18 @@ export type RequestExecutionScope = AsyncDisposable & { sessionName: string; requestLogPath: string; runnerLogPath: string; - startedAtMs: number; runAdmitted(task: () => Promise): Promise; runLocked(task: () => Promise): Promise; + /** For a request without the session lock: runs `task` under it, unless a request holds it. */ + tryRunUnderSessionLock(task: () => Promise): Promise; + /** Resolves an authorized session and starts the journal for a lock-exempt request. */ + resolveSessionRef( + req: DaemonRequest, + ref: SessionRef | undefined, + phase?: 'handler', + ): SessionRef | undefined; + /** Writes the request-finished event, unless the request may not write to its session. */ + recordFinished(req: DaemonRequest, response: DaemonResponse): void; retainDeviceExecutionLock(deviceId: string): Promise; bindDevice: BindDeviceRuntime; /** Root-composed platform services for this request's asks that bind no device runtime. */ @@ -198,6 +228,7 @@ export type LockedRequestScope = { sessionName: string; logPath: string; existingSession: SessionState | undefined; + resolveSessionRef: ResolveRequestSessionRef; retainDeviceExecutionLock(deviceId: string): Promise; bindDevice: BindDeviceRuntime; platformServices: DaemonPlatformServices; @@ -231,7 +262,8 @@ export async function createRequestExecutionScope(params: { platformServices: DaemonPlatformServices; claimRecoveryGateway: ClaimRecoveryGatewayFactory; platformRequestScope: PlatformRequestScope; - platformResourceCleanup?: PlatformResourceCleanup; + /** The daemon-owned teardown of a session whose lease expired (root runtime composition). */ + settleExpiredSession?: SessionTeardown; providerAppCatalog?: ProviderAppCatalog; daemonPolicy?: DaemonPolicy; }): Promise { @@ -286,15 +318,30 @@ export async function createRequestExecutionScope(params: { runnerLogPath, }, }); - const shouldRecordRequestEvents = shouldRecordEventForRequest(scopedReq); - if (shouldRecordRequestEvents) { + // These handlers recheck the current session after the scope's provisional reads. + const startsAtHandlerRead = !attachesToSession && HANDLER_JOURNAL_START_COMMANDS.has(command); + let recordStartAtFinish = !startsAtHandlerRead; + let recordsRequestEvents = false; + const recordStarted = (ts?: string): void => { + if (recordsRequestEvents || !shouldRecordEventForRequest(scopedReq)) return; + if (!requestMayRecordSessionEvents(scopedReq, sessionStore.lookup(sessionName))) return; + recordsRequestEvents = true; + const started = buildRequestStartedEvent({ req: scopedReq }); + sessionStore.recordEvent(sessionName, ts ? { ...started, ts } : started); + }; + const recordFinished = (req: DaemonRequest, response: DaemonResponse): void => { + if (recordStartAtFinish) recordStarted(new Date(startedAtMs).toISOString()); + if (!recordsRequestEvents) return; + if (!requestMayRecordSessionEvents(req, sessionStore.lookup(sessionName))) return; sessionStore.recordEvent( sessionName, - buildRequestStartedEvent({ - req: scopedReq, + buildRequestFinishedEvent({ + req, + response, + durationMs: Math.max(0, Date.now() - startedAtMs), }), ); - } + }; try { const devicePin = scopedReq.meta?.devicePin; const pinPolicy = devicePin @@ -378,25 +425,61 @@ export async function createRequestExecutionScope(params: { sessionName, requestLogPath, runnerLogPath, - startedAtMs, retainDeviceExecutionLock: async (deviceId) => await requestExecutionLocks.retainDevice(deviceId), + tryRunUnderSessionLock: async (task) => { + const key = sessionExecutionLockKey(sessionName); + if (!executionLocks.has(key)) + await withRequestExecutionLockKeys(executionLocks, [key], task); + }, bindDevice: runtimeBindings.bindDevice, inspectFacts: runtimeBindings.inspectFacts, bindExactDevice: runtimeBindings.bindExactDevice, platformServices: { ...params.platformServices }, dispatchLedger, throwIfCanceled: () => throwIfRequestCanceled(scopedReq.meta?.requestId), + resolveSessionRef(req, ref, phase) { + const resolved = resolveRequestSessionRef(req, ref); + if (startsAtHandlerRead && phase !== 'handler') return resolved; + if (startsAtHandlerRead || resolved) recordStartAtFinish = !resolved; + if (resolved && !attachesToSession) recordStarted(); + return resolved; + }, + recordFinished, runAdmitted: async (task) => { throwIfRequestCanceled(scopedReq.meta?.requestId); - const existingRef = sessionStore.lookup(sessionName); - const session = existingRef?.session; - assertRequestTenantAdmission(unscopedReq, session); const boundedStep = command !== 'batch' && command !== 'replay' && isBoundedCompositeRequest(scopedReq); + // The session under the lock governs. A refusal here comes before the request inspects a + // resource, ends an expired session or stamps session activity. + const addressed = sessionStore.get(sessionName); + assertRequestTenantAdmission(unscopedReq, addressed); + assertProviderLeaseRequestNamesSession(scopedReq, addressed); + const expiry = { + env: params.host.env, + sessionStore, + leaseRegistry, + teardown: expiredSessionTeardown(command, attachesToSession, boundedStep), + teardownSession: params.settleExpiredSession ?? expiredSessionSettleNotComposed, + }; + let successorWork: RetainedLeaseWork | undefined; let admissionCompleted = false; try { - assertProviderLeaseRequestNamesSession(scopedReq, session); + const existingRef = await resolveLockedRequestSessionRef({ + ...expiry, + req: scopedReq, + sessionName, + holdDeviceLock: scope.retainDeviceExecutionLock, + admission: params, + resolveSessionRef: scope.resolveSessionRef, + onSuccessorAdmitted: (lease) => { + const retained = retainSuccessorLeaseWork(leaseRegistry, scopedReq, lease); + scopedReq = retained.req; + successorWork = retained.work; + }, + }); + if (attachesToSession) recordStarted(); + const session = existingRef?.session; const boundedResources = boundedStep ? await import('../core/composite-resource-admission.ts') : undefined; @@ -460,42 +543,33 @@ export async function createRequestExecutionScope(params: { }); } throwIfRequestCanceled(scopedReq.meta?.requestId); - await cleanupExpiredLeasedSession({ - env: params.host.env, - sessionName, - sessionStore, - leaseRegistry, - refuseTeardown: command === 'batch' || command === 'replay' || boundedStep, - teardownSession: async (ref) => - await teardownExpiredSession({ - ref, - sessionStore, - inspectFacts: scope.inspectFacts, - bindDevice: scope.bindDevice, - platformCleanup: requirePlatformCleanup(params.platformResourceCleanup), - }), - }); - scopedReq = admitRequestLeaseForLockedScope({ - env: params.host.env, - req: scopedReq, - sessionName, - sessionStore, - leaseRegistry, - providerAppCatalog: params.providerAppCatalog, - daemonPolicy: params.daemonPolicy, - }); + await cleanupExpiredLeasedSession({ ...expiry, ref: existingRef }); + if (!successorWork) { + scopedReq = admitRequestLeaseForLockedScope({ + env: params.host.env, + req: scopedReq, + sessionName, + sessionStore, + leaseRegistry, + providerAppCatalog: params.providerAppCatalog, + daemonPolicy: params.daemonPolicy, + resolveSessionRef: scope.resolveSessionRef, + }); + } scope.req = scopedReq; const admittedLease = scopedReq.internal?.admittedLease; if (admittedLease) await assertMacOsAppLeaseProcess(admittedLease); return await runAdmittedLeaseWork({ leaseRegistry, req: scopedReq, + retainedWork: successorWork, task: async () => { admissionCompleted = true; return await task(); }, }); } finally { + successorWork?.release(); // The #2833 inactivity deadline is measured from the end of the last admitted command // that attached to this session, stamped here under the session's execution lock. The lock // makes this one stamp enough: an expiry has to acquire it too, so it can never catch a @@ -532,36 +606,18 @@ export async function createRequestExecutionScope(params: { [Symbol.asyncDispose]: disposeBindings, }; requestScopeFinalizers.set(scope, (response) => { - if (shouldRecordRequestEvents) { - sessionStore.recordEvent( - sessionName, - buildRequestFinishedEvent({ - req: scopedReq, - response, - durationMs: Math.max(0, Date.now() - startedAtMs), - }), - ); - } + recordFinished(scopedReq, response); return response; }); return scope; } catch (error) { - if (shouldRecordRequestEvents) { - sessionStore.recordEvent( - sessionName, - buildRequestFinishedEvent({ - req: scopedReq, - response: { - ok: false, - error: normalizeError(error, { - diagnosticId: getDiagnosticsMeta().diagnosticId, - logPath: requestLogPath, - }), - }, - durationMs: Math.max(0, Date.now() - startedAtMs), - }), - ); - } + recordFinished(scopedReq, { + ok: false, + error: normalizeError(error, { + diagnosticId: getDiagnosticsMeta().diagnosticId, + logPath: requestLogPath, + }), + }); throw error; } } @@ -641,66 +697,6 @@ function createRequestDeviceAccess(params: { }; } -async function teardownExpiredSession(params: { - ref: SessionRef; - sessionStore: SessionStore; - inspectFacts: InspectDeviceRuntimeFacts; - bindDevice: BindDeviceRuntime; - platformCleanup: PlatformResourceCleanup; -}): Promise { - const { ref, sessionStore, inspectFacts, bindDevice, platformCleanup } = params; - const session = sessionStore.resolveCurrent(ref) ?? ref.session; - const sessionName = ref.address; - const runtimeHints = runtimeHintValues(sessionStore.getRuntimeHints(ref.address)); - let primaryError: unknown; - try { - await teardownSessionResources({ - appLog: 'run', - ref, - sessionStore, - platformCleanup, - }); - } catch (error) { - primaryError = error; - } - try { - await finalizeBoundSessionApplicationLifecycle({ - inspectFacts, - bindDevice, - session, - stateDir: sessionStore.resolveDaemonStateDir(), - runtimeHints, - }); - } catch (cleanupError) { - if (primaryError !== undefined) { - emitDiagnostic({ - level: 'error', - phase: 'expired_session_lifecycle_cleanup_failed', - data: { - session: sessionName, - error: cleanupError instanceof Error ? cleanupError.message : String(cleanupError), - primaryError: primaryError instanceof Error ? primaryError.message : String(primaryError), - }, - }); - } else { - throw cleanupError; - } - } - if (primaryError !== undefined) throw primaryError; -} - -function requirePlatformCleanup( - cleanup: PlatformResourceCleanup | undefined, -): PlatformResourceCleanup { - if (!cleanup) { - throw new AppError( - 'INTERNAL_ERROR', - 'Platform resource cleanup was not supplied by root runtime composition', - ); - } - return cleanup; -} - function applyRequestCommandDefaults(req: DaemonRequest): DaemonRequest { const flags = { ...(req.flags ?? {}) }; const changed = applyCommandDefaults(req.command, flags); @@ -719,12 +715,24 @@ export async function prepareLockedRequestScope(params: { const { scope, sessionStore, trackDownloadableArtifact } = params; const logPath = scope.runnerLogPath; scope.throwIfCanceled(); - const seededRef = sessionStore.lookup(scope.sessionName); - if (seededRef) { + const readSessionRef = (): SessionRef | undefined => + scope.resolveSessionRef(scope.req, sessionStore.lookup(scope.sessionName)); + // A request that holds the session lock keeps the lifetime that it captures here. + let seededRef = readSessionRef(); + if (!shouldLockSessionExecution(scope.command)) { + // Without the session lock, an open can attach a lease to this session while the runner is + // observed. Thus the refresh takes the lock and reads the session again. It yields to a + // request that holds the lock, which checks the recording itself. + await scope.tryRunUnderSessionLock(async () => { + const ref = readSessionRef(); + if (ref) await refreshRecordingHealth(sessionStore, ref, scope.platformServices); + }); + seededRef = readSessionRef(); + } else if (seededRef && scope.command !== 'batch' && scope.command !== 'replay') { // A composite's resource health is checked by its admitted steps; the outer read can invalidate a durable recording. - if (scope.command !== 'batch' && scope.command !== 'replay') { - await refreshRecordingHealth(sessionStore, seededRef, scope.platformServices); - } + await refreshRecordingHealth(sessionStore, seededRef, scope.platformServices); + } + if (seededRef) { scope.throwIfCanceled(); sessionStore.requireCurrent(seededRef); } @@ -737,21 +745,15 @@ export async function prepareLockedRequestScope(params: { // recovery producer below must name — `existingRef.session.name` is only the public name. const existingRef = binding.existingRef; const existingSession = existingRef?.session; - const deliverRemoteScript = remoteScriptVersionDelivery(sessionStore, scope.sessionName); + // A command that ignores the session lease delivers no script of the session at its address. + const deliverRemoteScript = ignoresSessionLease(scope.command) + ? (response: DaemonResponse) => response + : remoteScriptVersionDelivery(sessionStore, scope.sessionName); updateDiagnosticsScope({ traceLogPath: existingSession?.trace?.outPath }); const finalize = (handled: DaemonResponse): DaemonResponse => { const response = deliverRemoteScript(handled); const finalized = finalizeDaemonResponse(lockedReq, response, trackDownloadableArtifact); - if (shouldRecordEventForRequest(lockedReq)) { - sessionStore.recordEvent( - scope.sessionName, - buildRequestFinishedEvent({ - req: lockedReq, - response: finalized, - durationMs: Math.max(0, Date.now() - scope.startedAtMs), - }), - ); - } + scope.recordFinished(lockedReq, finalized); return finalized; }; requestScopeFinalizers.set(scope, finalize); @@ -789,6 +791,7 @@ export async function prepareLockedRequestScope(params: { sessionName: scope.sessionName, logPath, existingSession, + resolveSessionRef: (req, ref) => scope.resolveSessionRef(req, ref, 'handler'), retainDeviceExecutionLock: scope.retainDeviceExecutionLock, bindDevice: scope.bindDevice, platformServices: scope.platformServices, @@ -896,6 +899,7 @@ export function createLeaseSessionEnd(params: { teardownSession, reason, params.env, + true, ); }, ), diff --git a/packages/daemon/src/server/request-lease-work.ts b/packages/daemon/src/server/request-lease-work.ts index e3a9453e78..4a3445a0e3 100644 --- a/packages/daemon/src/server/request-lease-work.ts +++ b/packages/daemon/src/server/request-lease-work.ts @@ -1,7 +1,27 @@ -import { isRequestCanceled } from '@agent-device/host-kit/request'; +import { isRequestCanceled, throwIfRequestCanceled } from '@agent-device/host-kit/request'; import { isHumanControlMutation } from '../core/daemon-command-registry.ts'; -import type { LeaseRegistry } from '../resources/lease-registry.ts'; +import type { LeaseRegistry, RetainedLeaseWork } from '../resources/lease-registry.ts'; import type { DaemonRequest } from '../core/daemon-request.ts'; +import type { DeviceLease } from '@agent-device/contracts/device'; + +function retainRequestLeaseWork( + leaseRegistry: LeaseRegistry, + lease: Pick, + requestId: string | undefined, +): RetainedLeaseWork { + return leaseRegistry.retainLeaseWork(lease, () => !isRequestCanceled(requestId)); +} + +export function retainSuccessorLeaseWork( + leaseRegistry: LeaseRegistry, + req: DaemonRequest, + lease: DeviceLease, +): { req: DaemonRequest; work: RetainedLeaseWork } { + return { + req: { ...req, internal: { ...req.internal, admittedLease: lease } }, + work: retainRequestLeaseWork(leaseRegistry, lease, req.meta?.requestId), + }; +} /** * Runs one admitted request's work under the protection of its lease. @@ -18,19 +38,24 @@ export async function runAdmittedLeaseWork( params: Readonly<{ leaseRegistry: LeaseRegistry; req: DaemonRequest; + /** A work pass already held and released by the caller. */ + retainedWork?: RetainedLeaseWork; task: () => Promise; }>, ): Promise { - const { leaseRegistry, req, task } = params; + const { leaseRegistry, req, retainedWork, task } = params; const lease = req.internal?.admittedLease; if (!lease) return await task(); - const requestId = req.meta?.requestId; - const work = leaseRegistry.retainLeaseWork(lease, () => !isRequestCanceled(requestId)); + if (retainedWork) { + throwIfRequestCanceled(req.meta?.requestId); + retainedWork.assertRetained(); + } + const work = retainedWork ?? retainRequestLeaseWork(leaseRegistry, lease, req.meta?.requestId); try { return await (isHumanControlMutation(req) ? leaseRegistry.runDeviceMutation(lease, task) : task()); } finally { - work.release(); + if (!retainedWork) work.release(); } } diff --git a/packages/daemon/src/server/request-router.ts b/packages/daemon/src/server/request-router.ts index 31e9566d09..33ecb9d4d3 100644 --- a/packages/daemon/src/server/request-router.ts +++ b/packages/daemon/src/server/request-router.ts @@ -1,4 +1,4 @@ -import type { LeaseEndReason } from '../sessions/lease-lifecycle.ts'; +import type { LeaseEndReason, SessionTeardown } from '../sessions/lease-lifecycle.ts'; import type { PluginInventoryEntry } from '@agent-device/contracts/plugin-inventory'; import { createOwnerScopedDeviceClaimReconciler, @@ -166,6 +166,8 @@ export type RequestRouterDeps = { /** Builds the per-request command surface the device-facing runtimes execute through. */ createCommandSurface: CreateDaemonCommandSurface; platformResourceCleanup: PlatformResourceCleanup; + /** Ends the resources of a session whose lease expired, for a request that holds its locks. */ + settleExpiredSession: SessionTeardown; providerDeviceRuntimeScope?: (task: () => Promise) => Promise; /** ADR 0029: the daemon policy every admitted request, including nested steps, obeys. */ daemonPolicy?: DaemonPolicy; @@ -196,6 +198,7 @@ export function createRequestHandler(deps: RequestRouterDeps): DaemonInvokeFn { androidObservation, createCommandSurface, platformResourceCleanup, + settleExpiredSession, providerDeviceRuntimeScope, trackDownloadableArtifact, daemonPolicy, @@ -318,7 +321,7 @@ export function createRequestHandler(deps: RequestRouterDeps): DaemonInvokeFn { platformServices, claimRecoveryGateway, platformRequestScope, - platformResourceCleanup, + settleExpiredSession, providerAppCatalog, daemonPolicy, }); @@ -468,6 +471,7 @@ export function createRequestHandler(deps: RequestRouterDeps): DaemonInvokeFn { perfCaptureAdmissionLedger, screenRecordingAdmissionLedger, hostDiagnostics, + resolveSessionRef: lockedScope.resolveSessionRef, requestScope, retainDeviceExecutionLock: lockedScope.retainDeviceExecutionLock, throwIfCanceled: lockedScope.throwIfCanceled, @@ -509,7 +513,7 @@ export function createRequestHandler(deps: RequestRouterDeps): DaemonInvokeFn { platformServices, claimRecoveryGateway, platformRequestScope: createPlatformRequestScope(scopedReq), - platformResourceCleanup, + settleExpiredSession, providerAppCatalog, daemonPolicy, }); diff --git a/packages/daemon/src/sessions/__tests__/lease-lifecycle.test.ts b/packages/daemon/src/sessions/__tests__/lease-lifecycle.test.ts index 917e3d0319..845bce3760 100644 --- a/packages/daemon/src/sessions/__tests__/lease-lifecycle.test.ts +++ b/packages/daemon/src/sessions/__tests__/lease-lifecycle.test.ts @@ -9,6 +9,7 @@ import { cleanupExpiredLeasedSession, releaseExpiredProviderLease, releaseSessionLease, + resolveLockedRequestSessionRef, } from '../lease-lifecycle.ts'; import type { DaemonRequest } from '../../core/daemon-request.ts'; @@ -61,6 +62,81 @@ test('admitRequestLeaseForLockedScope heartbeats and stores admitted lease on th expect(leaseRegistry.listActiveLeases()[0]?.expiresAt).toBe(62_000); }); +test('admitRequestLeaseForLockedScope renews no session for a command that ignores the session lease', () => { + const sessionStore = makeSessionStore('agent-device-lease-lifecycle-'); + const leaseRegistry = new LeaseRegistry(); + const own = leaseRegistry.allocateLease({ tenantId: 'tenant-a', runId: 'run-b' }); + const sessionLease = { + leaseId: 'a'.repeat(32), + tenantId: 'tenant-a', + runId: 'run-a', + backend: 'macos-app', + } as const; + sessionStore.publish('default', makeIosSession('default', { lease: sessionLease })); + + const req = admitRequestLeaseForLockedScope({ + env: process.env, + req: makeRequest({ + command: 'devices', + meta: { leaseId: own.leaseId, tenantId: 'tenant-a', runId: 'run-b' }, + }), + sessionName: 'default', + sessionStore, + leaseRegistry, + }); + + // The addressed app lease still makes `devices` need a lease; it is admitted under its own. + expect(req.internal?.admittedLease?.leaseId).toBe(own.leaseId); + expect(sessionStore.get('default')?.lease).toEqual(sessionLease); +}); + +test('resolveLockedRequestSessionRef ends nothing when the successor lease expires while it waits for the device lock', async () => { + let now = 1_000; + const sessionStore = makeSessionStore('agent-device-lease-lifecycle-'); + const leaseRegistry = new LeaseRegistry({ minLeaseTtlMs: 1, now: () => now }); + const expired = leaseRegistry.allocateLease({ tenantId: 'tenant-a', runId: 'run-a', ttlMs: 10 }); + const successor = leaseRegistry.allocateLease({ + tenantId: 'tenant-a', + runId: 'run-b', + ttlMs: 50, + }); + const ref = sessionStore.publish( + 'default', + makeIosSession('default', { + lease: { + leaseId: expired.leaseId, + tenantId: expired.tenantId, + runId: expired.runId, + backend: expired.backend, + }, + }), + ); + now = 1_020; + const teardownSession = vi.fn(async () => {}); + const onSuccessorAdmitted = vi.fn(); + + await expect( + resolveLockedRequestSessionRef({ + env: process.env, + req: makeRequest({ command: 'open', meta: { leaseId: successor.leaseId } }), + sessionName: 'default', + sessionStore, + leaseRegistry, + teardown: 'inline', + holdDeviceLock: async () => { + now = 1_060; + }, + teardownSession, + admission: {}, + onSuccessorAdmitted, + }), + ).rejects.toMatchObject({ details: { reason: 'LEASE_SESSION_MISMATCH' } }); + + expect(teardownSession).not.toHaveBeenCalled(); + expect(onSuccessorAdmitted).not.toHaveBeenCalled(); + expect(sessionStore.lookup('default')?.lifetime).toBe(ref.lifetime); +}); + test('cleanupExpiredLeasedSession consumes expired lease and deletes the session after teardown', async () => { let now = 1_000; const sessionStore = makeSessionStore('agent-device-lease-lifecycle-'); @@ -84,10 +160,11 @@ test('cleanupExpiredLeasedSession consumes expired lease and deletes the session const cleaned = await cleanupExpiredLeasedSession({ env: process.env, - sessionName: 'default', + ref: sessionStore.lookup('default'), sessionStore, leaseRegistry, teardownSession, + teardown: 'inline', }); expect(cleaned).toBe(true); diff --git a/packages/daemon/src/sessions/__tests__/replay-suite/session-test-suite-command-video.test.ts b/packages/daemon/src/sessions/__tests__/replay-suite/session-test-suite-command-video.test.ts index c2a79121ec..9b009810aa 100644 --- a/packages/daemon/src/sessions/__tests__/replay-suite/session-test-suite-command-video.test.ts +++ b/packages/daemon/src/sessions/__tests__/replay-suite/session-test-suite-command-video.test.ts @@ -272,6 +272,7 @@ test('test finalizes replay video exactly once when cancellation arrives after s const video = createReplayTestVideoOwner({ platformServices: daemonPlatformServicesFixture(), sessionStore, + assertSuiteMayActOn: () => {}, bindDevice: unavailableBindDevice, bindExactDevice: unavailableBindExactDevice, screenRecordingAdmissionLedger, diff --git a/packages/daemon/src/sessions/__tests__/session-idle-activity.test.ts b/packages/daemon/src/sessions/__tests__/session-idle-activity.test.ts index ae1f91b03d..27e6b82ba5 100644 --- a/packages/daemon/src/sessions/__tests__/session-idle-activity.test.ts +++ b/packages/daemon/src/sessions/__tests__/session-idle-activity.test.ts @@ -79,38 +79,46 @@ test('a command that completes under the session lock stamps the session it ran expect(sessionStore.get('default')?.lastActivityAtMs).toBeGreaterThan(0); }); -test('a standalone request refused at lease admission reports no activity', async () => { - const sessionStore = storeWithSession(); - const leaseRegistry = new LeaseRegistry(); - const lease = leaseRegistry.allocateLease({ tenantId: 'tenant-a', runId: 'run-a' }); - const ref = sessionStore.lookup('default'); - if (!ref) throw new Error('missing test session'); - sessionStore.update(ref, { - lease: { - leaseId: lease.leaseId, - tenantId: lease.tenantId, - runId: lease.runId, - backend: lease.backend, - }, - }); - const scope = await withDiagnosticsScope({ command: 'home', logPath: LOG_PATH }, () => - createRequestExecutionScope({ - host: createTestHostEnvironment(), - platformServices: daemonPlatformServicesFixture(), - claimRecoveryGateway: daemonClaimRecoveryGatewayFixture(), - deviceRuntimeGateway: unavailableDeviceRuntimeGateway, - platformRequestScope: platformRequestScopeFixture(), - req: makeRequest({ command: 'home', meta: { leaseId: 'foreign-lease' } }), - sessionStore, - leaseRegistry, - }), - ); +test.each([false, true])( + 'a request refused at lease admission reports no activity (leased=%s)', + async (leased) => { + const sessionStore = storeWithSession(); + const leaseRegistry = new LeaseRegistry(); + const lease = leaseRegistry.allocateLease({ tenantId: 'tenant-a', runId: 'run-a' }); + const ref = sessionStore.lookup('default'); + if (!ref) throw new Error('missing test session'); + if (leased) { + sessionStore.update(ref, { + lease: { + leaseId: lease.leaseId, + tenantId: lease.tenantId, + runId: lease.runId, + backend: lease.backend, + }, + }); + } + const scope = await withDiagnosticsScope({ command: 'home', logPath: LOG_PATH }, () => + createRequestExecutionScope({ + host: createTestHostEnvironment(), + platformServices: daemonPlatformServicesFixture(), + claimRecoveryGateway: daemonClaimRecoveryGatewayFixture(), + deviceRuntimeGateway: unavailableDeviceRuntimeGateway, + platformRequestScope: platformRequestScopeFixture(), + req: makeRequest({ + command: 'home', + meta: { leaseId: 'f'.repeat(32), tenantId: 'tenant-a', runId: 'run-a' }, + }), + sessionStore, + leaseRegistry, + }), + ); - await expect(scope.runLocked(async () => 'unreachable')).rejects.toMatchObject({ - code: 'UNAUTHORIZED', - }); - expect(sessionStore.get('default')?.lastActivityAtMs).toBeUndefined(); -}); + await expect(scope.runLocked(async () => 'unreachable')).rejects.toMatchObject({ + code: 'UNAUTHORIZED', + }); + expect(sessionStore.get('default')?.lastActivityAtMs).toBeUndefined(); + }, +); test('a lock-exempt inventory command reports no activity, even against a claim-holding session', async () => { const sessionStore = storeWithSession(); diff --git a/packages/daemon/src/sessions/__tests__/session-teardown-expired-settle.test.ts b/packages/daemon/src/sessions/__tests__/session-teardown-expired-settle.test.ts new file mode 100644 index 0000000000..33fa671f18 --- /dev/null +++ b/packages/daemon/src/sessions/__tests__/session-teardown-expired-settle.test.ts @@ -0,0 +1,41 @@ +import { bindTestCommandHost } from '@agent-device/test-kit'; +import { expect, test, vi } from 'vitest'; +import { platformResourceCleanupFixture } from '../../__tests__/support/platform-resource-cleanup-fixture.ts'; +import { makeIosSession } from '../../__tests__/support/session-factories.ts'; +import { makeSessionStore } from '../../__tests__/support/store-factory.ts'; +import { settleExpiredSession } from '../session-teardown.ts'; + +bindTestCommandHost(); + +function settleWithFailingResources(afterResourceFailure: 'stop' | 'finalize') { + const sessionStore = makeSessionStore('agent-device-expired-settle-'); + const ref = sessionStore.publish('default', makeIosSession('default')); + const finalizeApplication = vi.fn(async () => {}); + const settled = settleExpiredSession({ + appLog: 'run', + ref, + sessionStore, + platformCleanup: platformResourceCleanupFixture({ + stopSnapshotHelper: async () => { + throw new Error('snapshot helper stop failed'); + }, + }), + afterResourceFailure, + finalizeApplication, + }); + return { settled, finalizeApplication }; +} + +test('a lease expiry still finalizes the application after a resource failure, which stays the error', async () => { + const { settled, finalizeApplication } = settleWithFailingResources('finalize'); + + await expect(settled).rejects.toThrow(/snapshot helper stop failed/); + expect(finalizeApplication).toHaveBeenCalledTimes(1); +}); + +test('an idle expiry stops at a resource failure, so the next pass can retry', async () => { + const { settled, finalizeApplication } = settleWithFailingResources('stop'); + + await expect(settled).rejects.toThrow(/snapshot helper stop failed/); + expect(finalizeApplication).not.toHaveBeenCalled(); +}); diff --git a/packages/daemon/src/sessions/handlers/__tests__/session-replay-command-suite-lease.test.ts b/packages/daemon/src/sessions/handlers/__tests__/session-replay-command-suite-lease.test.ts index 2a9cb8c424..bff8e92419 100644 --- a/packages/daemon/src/sessions/handlers/__tests__/session-replay-command-suite-lease.test.ts +++ b/packages/daemon/src/sessions/handlers/__tests__/session-replay-command-suite-lease.test.ts @@ -26,7 +26,12 @@ test('suite-owned test-session close releases a non-retained provider lease thro const release = vi.fn(async (_lease: DeviceLease) => ({})); const root = mkdtempForTestSync('agent-device-test-suite-provider-lease-'); fs.writeFileSync(path.join(root, '01-open.ad'), 'context platform=android\nopen "Settings"\n'); - let leaseId: string | undefined; + const lease = leaseRegistry.allocateLease({ + tenantId: 'tenant-a', + runId: 'run-1', + leaseBackend: 'android-instance', + leaseProvider: 'proxy', + }); const response = await handleSessionCommands({ host: createTestHostEnvironment(), @@ -37,7 +42,7 @@ test('suite-owned test-session close releases a non-retained provider lease thro session: 'default', command: 'test', positionals: [root], - meta: { cwd: root, requestId: 'suite-provider-lease' }, + meta: { cwd: root, requestId: 'suite-provider-lease', leaseId: lease.leaseId }, flags: {}, }, sessionName: 'default', @@ -46,13 +51,6 @@ test('suite-owned test-session close releases a non-retained provider lease thro leaseRegistry, leaseLifecycleProvider: { release }, invoke: async (req) => { - const lease = leaseRegistry.allocateLease({ - tenantId: 'tenant-a', - runId: 'run-1', - leaseBackend: 'android-instance', - leaseProvider: 'proxy', - }); - leaseId = lease.leaseId; storeSessionForTest( sessionStore, makeAndroidSession(req.session, { @@ -72,5 +70,5 @@ test('suite-owned test-session close releases a non-retained provider lease thro expect(expectOkData(response).passed).toBe(1); expect(release).toHaveBeenCalledOnce(); - expect(release.mock.calls[0]?.[0]).toMatchObject({ leaseId }); + expect(release.mock.calls[0]?.[0]).toMatchObject({ leaseId: lease.leaseId }); }); diff --git a/packages/daemon/src/sessions/handlers/__tests__/session-replay-command.test.ts b/packages/daemon/src/sessions/handlers/__tests__/session-replay-command.test.ts index 221b393de9..966cbe675d 100644 --- a/packages/daemon/src/sessions/handlers/__tests__/session-replay-command.test.ts +++ b/packages/daemon/src/sessions/handlers/__tests__/session-replay-command.test.ts @@ -87,7 +87,7 @@ test.each([false, true])( command: 'test', positionals: [scriptPath], flags: {}, - meta: { cwd: root, requestId: 'replay-handler-ended-lease' }, + meta: { cwd: root, requestId: 'replay-handler-ended-lease', leaseId: lease.leaseId }, }); const params: SessionCommandParams = { host, diff --git a/packages/daemon/src/sessions/handlers/lease.ts b/packages/daemon/src/sessions/handlers/lease.ts index 64bdc55937..411fa7a634 100644 --- a/packages/daemon/src/sessions/handlers/lease.ts +++ b/packages/daemon/src/sessions/handlers/lease.ts @@ -5,6 +5,7 @@ import { } from '../../resources/lease-release.ts'; import { emitDiagnostic } from '@agent-device/host-kit/diagnostics'; import type { LeaseEndReason } from '../lease-lifecycle.ts'; +import { resolveRequestSessionRef, type ResolveRequestSessionRef } from '../request-admission.ts'; import type { EnvMap, HostEnvironment } from '@agent-device/kernel/source-value'; import type { PluginInventoryEntry } from '@agent-device/contracts/plugin-inventory'; import type { DaemonResponse } from '@agent-device/kernel/contracts'; @@ -59,6 +60,7 @@ type LeaseHandlerArgs = { req: DaemonRequest; sessionName: string; sessionStore: SessionStore; + resolveSessionRef?: ResolveRequestSessionRef; leaseRegistry: LeaseRegistry; providerRuntimeIds?: readonly string[]; failedPlugins?: readonly PluginInventoryEntry[]; @@ -84,7 +86,9 @@ export async function handleLeaseCommands(args: LeaseHandlerArgs): Promise Promise; throwIfCanceled?: () => void; diff --git a/packages/daemon/src/sessions/handlers/session-doctor.ts b/packages/daemon/src/sessions/handlers/session-doctor.ts index c2b2fc5cc4..f2e14b0f58 100644 --- a/packages/daemon/src/sessions/handlers/session-doctor.ts +++ b/packages/daemon/src/sessions/handlers/session-doctor.ts @@ -20,6 +20,7 @@ import { type DoctorDeviceInventory, resolveDoctorDeviceForAppCheck, } from './session-doctor-device.ts'; +import { resolveRequestSessionRef, type ResolveRequestSessionRef } from '../request-admission.ts'; import { probeMetro } from './session-doctor-metro.ts'; import { readDoctorOptions, @@ -57,14 +58,24 @@ export async function handleDoctorCommand(params: { /** Opaque provider-scope transport override; the android family narrows it back. */ androidAdbExecutor?: unknown; hostDiagnostics?: HostDiagnostics; + resolveSessionRef?: ResolveRequestSessionRef; inspectFacts?: InspectDeviceRuntimeFacts; bindDevice?: BindDeviceRuntime; }): Promise { - const { req, sessionName, sessionStore, androidAdbExecutor, inspectFacts, bindDevice } = params; + const { + req, + sessionName, + sessionStore, + androidAdbExecutor, + inspectFacts, + bindDevice, + resolveSessionRef = resolveRequestSessionRef, + } = params; if (req.command !== PUBLIC_COMMANDS.doctor) return null; const hostDiagnostics = requireHostDiagnostics(params.hostDiagnostics); - const session = sessionStore.get(sessionName); + // Doctor holds no session lock, so a leased session can open here after admission. + const session = resolveSessionRef(req, sessionStore.lookup(sessionName))?.session; const options = readDoctorOptions(req, session); const stateDir = resolveDoctorStateDir(sessionStore, sessionName); const checks: DoctorCheck[] = []; diff --git a/packages/daemon/src/sessions/handlers/session-replay-command.ts b/packages/daemon/src/sessions/handlers/session-replay-command.ts index a14db88ace..4dd73779be 100644 --- a/packages/daemon/src/sessions/handlers/session-replay-command.ts +++ b/packages/daemon/src/sessions/handlers/session-replay-command.ts @@ -4,6 +4,7 @@ import type { HostEnvironment } from '@agent-device/kernel/source-value'; import type { DaemonPlatformServices } from '../../core/platform-services.ts'; import { AppError } from '@agent-device/kernel/errors'; import type { DaemonRequest } from '../../core/daemon-request.ts'; +import { assertRequestHoldsSessionLease } from '../../core/lease-context.ts'; import type { SessionRef, SessionState } from '../../core/session-state.ts'; import type { SessionStore } from '../../core/session-store.ts'; import { bindInternalObservationAuthority } from '../../execution/internal-observation.ts'; @@ -109,8 +110,17 @@ export const handleReplayTestCommand: SessionCommandHandler = async ({ 'Platform resource cleanup was not supplied by root runtime composition', ); } + // The suite reaches an attempt session through the handlers below, not through admission. An + // attempt address can already hold a leased session, so every read here checks the lease id that + // the suite sent. + const lookupAttemptSession = (attemptSessionName: string): SessionRef | undefined => { + const ref = sessionStore.lookup(attemptSessionName); + assertRequestHoldsSessionLease(req, ref?.session); + return ref; + }; const video = createReplayTestVideoOwner({ sessionStore, + assertSuiteMayActOn: lookupAttemptSession, bindDevice, platformServices, bindExactDevice, @@ -141,7 +151,9 @@ export const handleReplayTestCommand: SessionCommandHandler = async ({ host, session: createReplaySession(sessionName, logPath, sessionStore, capture), createSession: (testSessionName, testLogPath) => - createReplaySession(testSessionName, testLogPath, sessionStore, capture), + createReplaySession(testSessionName, testLogPath, sessionStore, capture, () => + lookupAttemptSession(testSessionName), + ), invoke: replayInvokeOverDispatch(invokeReplayAction ?? invoke, req), dependencies: replayDaemonDependencies, cleanupSession: async (testSessionName) => @@ -149,6 +161,7 @@ export const handleReplayTestCommand: SessionCommandHandler = async ({ host, req, sessionName: testSessionName, + lookupSession: lookupAttemptSession, logPath, sessionStore, leaseRegistry, @@ -173,19 +186,24 @@ export function createReplaySession( logPath: string, store: SessionStore, capture?: ReplayCaptureAdmission, + lookup: () => SessionRef | undefined = () => store.lookup(name), ): ReplaySession { const updateSession = (mutate: (session: SessionState) => void): boolean => { - const session = store.get(name); + const session = lookup()?.session; if (!session) return false; mutate(session); return true; }; // One read set for both views: the narrowed one the port binds over, and the full-record one the // repair coordinator writes through. + const getRuntimeHints = (): ReturnType => { + lookup(); + return store.getRuntimeHints(name); + }; const reads = { - get: () => store.get(name), - lookup: () => store.lookup(name), - getRuntimeHints: () => store.getRuntimeHints(name), + get: () => lookup()?.session, + lookup, + getRuntimeHints, ensureSessionDir: () => store.ensureSessionDir(name), }; let captureCell: @@ -250,7 +268,7 @@ export function createReplaySession( hostCwd: store.host.cwd, }), assertSelectorMatches: (flags) => { - const ref = store.lookup(name); + const ref = lookup(); if (ref) assertSessionSelectorMatches(ref, flags); }, resolveOpenRuntimeHints: ({ request, device, platform }) => @@ -258,14 +276,14 @@ export function createReplaySession( req: request, sessionStore: { getRuntimeHints: (requestedSessionName) => - requestedSessionName === name ? store.getRuntimeHints(name) : undefined, + requestedSessionName === name ? getRuntimeHints() : undefined, }, sessionName: name, device, platform, }), bindAuthority: (signal) => { - const ref = store.lookup(name); + const ref = lookup(); return { ...bindInternalObservationAuthority({ sessionStore: store, ref, signal }), capture: async ({ flags, logPath: captureLogPath }) => { @@ -306,6 +324,7 @@ type ReplayTestSessionCleanupParams = Readonly<{ host: HostEnvironment; req: DaemonRequest; sessionName: string; + lookupSession: (sessionName: string) => SessionRef | undefined; logPath: string; sessionStore: SessionStore; leaseRegistry: LeaseRegistry; @@ -321,7 +340,7 @@ async function closeReplayTestSession( params: ReplayTestSessionCleanupParams, ): Promise { const { req, sessionName, logPath, sessionStore } = params; - if (!sessionStore.get(sessionName)) return []; + if (!params.lookupSession(sessionName)) return []; const closeResponse = await handleSessionCloseCommands({ host: params.host, req: onDelegatingLedger( diff --git a/packages/daemon/src/sessions/handlers/session-replay-video-owner.ts b/packages/daemon/src/sessions/handlers/session-replay-video-owner.ts index 7ce5724df0..57b10c6cd7 100644 --- a/packages/daemon/src/sessions/handlers/session-replay-video-owner.ts +++ b/packages/daemon/src/sessions/handlers/session-replay-video-owner.ts @@ -14,6 +14,8 @@ type ReplayRecordVideoRequest = Parameters[0]; export type ReplayTestVideoOwnerParams = Readonly<{ sessionStore: SessionStore; + /** Refuses an attempt session that holds a lease the suite did not send. */ + assertSuiteMayActOn: (sessionName: string) => unknown; bindDevice?: BindDeviceRuntime; platformServices: DaemonPlatformServices; bindExactDevice?: BindExactDeviceRuntime; @@ -28,6 +30,7 @@ export function createReplayTestVideoOwner( ): ReplayTestVideoOwner | undefined { const { sessionStore, + assertSuiteMayActOn, bindDevice, platformServices, bindExactDevice, @@ -46,8 +49,9 @@ export function createReplayTestVideoOwner( ) { return undefined; } - const record: ReplayTestVideoOwner['record'] = async (request) => - await handleRecordCommand({ + const record: ReplayTestVideoOwner['record'] = async (request) => { + assertSuiteMayActOn(request.sessionName); + return await handleRecordCommand({ req: recordRequest(request), sessionName: request.sessionName, sessionStore, @@ -59,6 +63,7 @@ export function createReplayTestVideoOwner( retainDeviceExecutionLock, throwIfCanceled, }); + }; return { throwIfCanceled, record, diff --git a/packages/daemon/src/sessions/handlers/session.ts b/packages/daemon/src/sessions/handlers/session.ts index 89a688f76a..5e5cf28a8f 100644 --- a/packages/daemon/src/sessions/handlers/session.ts +++ b/packages/daemon/src/sessions/handlers/session.ts @@ -78,6 +78,7 @@ const handleSessionObservabilityCommandGroup: SessionCommandHandler = async ({ audioProbeAdmissionLedger, perfCaptureAdmissionLedger, throwIfCanceled, + resolveSessionRef, }) => await handleSessionObservabilityCommands({ host, @@ -91,6 +92,7 @@ const handleSessionObservabilityCommandGroup: SessionCommandHandler = async ({ audioProbeAdmissionLedger, perfCaptureAdmissionLedger, throwIfCanceled, + resolveSessionRef, } satisfies SessionObservabilityCommandInput); /** @@ -115,6 +117,7 @@ const SESSION_COMMAND_HANDLER_IMPLS = { sessionStore, androidAdbExecutor, hostDiagnostics, + resolveSessionRef, inspectFacts, bindDevice, }) => @@ -125,6 +128,7 @@ const SESSION_COMMAND_HANDLER_IMPLS = { sessionStore, androidAdbExecutor, hostDiagnostics, + resolveSessionRef, inspectFacts, bindDevice, }), @@ -232,6 +236,7 @@ export async function handleSessionCommands( perfCaptureAdmissionLedger, screenRecordingAdmissionLedger, hostDiagnostics, + resolveSessionRef, requestScope, retainDeviceExecutionLock, throwIfCanceled, @@ -265,6 +270,7 @@ export async function handleSessionCommands( perfCaptureAdmissionLedger, screenRecordingAdmissionLedger, hostDiagnostics, + resolveSessionRef, requestScope, retainDeviceExecutionLock, throwIfCanceled, diff --git a/packages/daemon/src/sessions/lease-lifecycle.ts b/packages/daemon/src/sessions/lease-lifecycle.ts index 74f7687d65..ada42d5a77 100644 --- a/packages/daemon/src/sessions/lease-lifecycle.ts +++ b/packages/daemon/src/sessions/lease-lifecycle.ts @@ -1,15 +1,22 @@ import type { HostEnvironment } from '@agent-device/kernel/source-value'; import { emitDiagnostic } from '@agent-device/host-kit/diagnostics'; -import { LEASE_REASONS } from '@agent-device/contracts/lease-scope'; +import { throwIfRequestCanceled } from '@agent-device/host-kit/request'; +import { LEASE_REASONS, leaseScopeToHeartbeatRequest } from '@agent-device/contracts/lease-scope'; import { clearDeviceClaim } from '@agent-device/daemon-contracts/device-claims'; import type { LeaseRegistry } from '../resources/lease-registry.ts'; import { leaseReleaseRequestFor } from '../resources/lease-registry-scope.ts'; +import { assertMacOsAppLeaseProcess } from '../resources/macos-app-lease.ts'; import type { DeviceLease, LeaseLifecycleProvider, ProviderAppCatalog, } from '@agent-device/contracts/device'; -import { assertRequestLeaseAdmission } from './request-admission.ts'; +import { resolveLeaseScope } from '../core/lease-context.ts'; +import { + assertRequestLeaseAdmission, + resolveRequestSessionRef, + type ResolveRequestSessionRef, +} from './request-admission.ts'; import { AppError, errorMessage } from '@agent-device/kernel/errors'; import type { SessionStore } from '../core/session-store.ts'; import type { DaemonRequest } from '../core/daemon-request.ts'; @@ -53,26 +60,34 @@ export async function releaseExpiredProviderLease( } } +/** + * How a request treats a session whose lease expired. A request that holds the session lock ends + * it (`inline`). A composite refuses the expired lease rather than tear its session down mid-run. + * A request without the session lock leaves the session to the lease-end owner and its locks. + */ +export type ExpiredSessionTeardown = 'inline' | 'refuse' | 'lease-end-owner'; + export async function cleanupExpiredLeasedSession(params: { env: HostEnvironment['env']; - sessionName: string; + /** The session the request holds the lease of; a request that holds none ends no session. */ + ref: SessionRef | undefined; sessionStore: SessionStore; leaseRegistry: LeaseRegistry; teardownSession: SessionTeardown; - /** A composite request refuses an expired lease rather than tear its session down mid-run. */ - refuseTeardown?: boolean; + teardown: ExpiredSessionTeardown; }): Promise { - const ref = params.sessionStore.lookup(params.sessionName); + const { ref } = params; const lease = ref?.session.lease; if (!ref || !lease) return false; if ( !params.leaseRegistry.consumeExpiredLease(lease.leaseId, () => { - if (!params.refuseTeardown) return; + if (params.teardown !== 'refuse') return; throw new AppError('UNAUTHORIZED', 'Lease is not active', { reason: LEASE_REASONS.notFound.reason, dispatched: 'no', }); - }) + }) || + params.teardown !== 'inline' ) return false; await endLeasedSession( @@ -95,9 +110,11 @@ export async function endLeasedSession( teardownSession: SessionTeardown, reason: LeaseEndReason, env: HostEnvironment['env'], + /** Only the daemon log names the lease id. A request record can reach another client. */ + daemonScope = false, ): Promise { const session = ref.session; - const leaseId = session.lease?.leaseId; + const leaseId = daemonScope ? session.lease?.leaseId : undefined; emitDiagnostic({ level: 'info', phase: 'leased_session_ended', @@ -125,6 +142,87 @@ export async function endLeasedSession( sessionStore.retire(ref); } +type LockedSessionParams = { + env: HostEnvironment['env']; + req: DaemonRequest; + sessionName: string; + sessionStore: SessionStore; + leaseRegistry: LeaseRegistry; + teardown: ExpiredSessionTeardown; + holdDeviceLock: (deviceId: string) => Promise; + teardownSession: SessionTeardown; + admission: Readonly<{ providerAppCatalog?: ProviderAppCatalog; daemonPolicy?: DaemonPolicy }>; + onSuccessorAdmitted(lease: DeviceLease): void; + resolveSessionRef?: ResolveRequestSessionRef; +}; + +/** + * The session that a request acts through, resolved under the session lock. A request that + * presents another active lease first ends a session whose lease is over, and then finds none. + */ +export async function resolveLockedRequestSessionRef( + params: LockedSessionParams, +): Promise { + const { req, sessionName, sessionStore, resolveSessionRef = resolveRequestSessionRef } = params; + const ref = sessionStore.lookup(sessionName); + const ended = + ref !== undefined && + params.teardown === 'inline' && + (await endExpiredSessionForSuccessor(params, ref)); + return resolveSessionRef(req, ended ? sessionStore.lookup(sessionName) : ref); +} + +/** + * Ends the session of a lease that is over, for a request that presents another active lease. A + * new owner can then open the address before the lease sweep runs. Any other request ends nothing. + */ +async function endExpiredSessionForSuccessor( + params: LockedSessionParams, + ref: SessionRef, +): Promise { + const { leaseRegistry } = params; + const expiredLeaseId = ref.session.lease?.leaseId; + const successorScope = resolveLeaseScope(params.req); + const successorLeaseId = successorScope.leaseId; + if (!expiredLeaseId || !successorLeaseId || successorLeaseId === expiredLeaseId) return false; + const canEndSession = () => + params.sessionStore.resolveCurrent(ref)?.lease?.leaseId === expiredLeaseId && + !leaseRegistry.inspectActiveLease(expiredLeaseId) && + leaseRegistry.inspectActiveLease(successorLeaseId) !== undefined; + if (!canEndSession()) return false; + await params.holdDeviceLock(ref.session.device.id); + throwIfRequestCanceled(params.req.meta?.requestId); + const admitSuccessor = () => + assertRequestLeaseAdmission(params.req, leaseRegistry, undefined, { + ...params.admission, + env: params.env, + inspectOnly: true, + }); + if (!canEndSession()) return false; + const successor = admitSuccessor(); + if (!successor) return false; + if (successor.backend === 'macos-app') { + await assertMacOsAppLeaseProcess(successor); + throwIfRequestCanceled(params.req.meta?.requestId); + if (!canEndSession() || !admitSuccessor()) return false; + } + const admittedSuccessor = leaseRegistry.renewInspectedLeaseAdmission( + leaseScopeToHeartbeatRequest(successorScope), + ); + params.onSuccessorAdmitted(admittedSuccessor); + await endLeasedSession( + ref, + params.sessionStore, + params.teardownSession, + 'LEASE_EXPIRED', + params.env, + ); + // An inactive lease never becomes active again. The registry notes the expiry only now, so the + // lease-end owner that it starts finds no session to end a second time. + leaseRegistry.consumeExpiredLease(expiredLeaseId); + return true; +} + export function admitRequestLeaseForLockedScope(params: { env: HostEnvironment['env']; req: DaemonRequest; @@ -133,18 +231,22 @@ export function admitRequestLeaseForLockedScope(params: { leaseRegistry: LeaseRegistry; providerAppCatalog?: ProviderAppCatalog; daemonPolicy?: DaemonPolicy; + resolveSessionRef?: ResolveRequestSessionRef; }): DaemonRequest { - const { sessionName, sessionStore, leaseRegistry } = params; - const activeLease = assertRequestLeaseAdmission( - params.req, + const { + sessionName, + sessionStore, leaseRegistry, - sessionStore.lookup(sessionName)?.session, - { - env: params.env, - providerAppCatalog: params.providerAppCatalog, - daemonPolicy: params.daemonPolicy, - }, - ); + resolveSessionRef = resolveRequestSessionRef, + } = params; + // The addressed session still decides the app-lease veto. + const addressed = sessionStore.lookup(sessionName); + resolveSessionRef(params.req, addressed); + const activeLease = assertRequestLeaseAdmission(params.req, leaseRegistry, addressed?.session, { + env: params.env, + providerAppCatalog: params.providerAppCatalog, + daemonPolicy: params.daemonPolicy, + }); if (!activeLease) return params.req; return { ...params.req, internal: { ...params.req.internal, admittedLease: activeLease } }; } diff --git a/packages/daemon/src/sessions/request-admission.ts b/packages/daemon/src/sessions/request-admission.ts index a43f138fbf..c87f5c0ea3 100644 --- a/packages/daemon/src/sessions/request-admission.ts +++ b/packages/daemon/src/sessions/request-admission.ts @@ -14,6 +14,7 @@ import { import type { DeviceLease, ProviderAppCatalog } from '@agent-device/contracts/device'; import { assertRequestHoldsSessionLease, + compareRequestLeaseClaims, findMissingProxyLeaseFields, resolveLeaseRefusalScope, resolveLeaseScope, @@ -26,7 +27,7 @@ import { } from '@agent-device/contracts/lease-scope'; import type { LeaseRegistry } from '../resources/lease-registry.ts'; import type { DaemonRequest } from '../core/daemon-request.ts'; -import type { SessionState } from '../core/session-state.ts'; +import type { SessionRef, SessionState } from '../core/session-state.ts'; import type { SessionStore } from '../core/session-store.ts'; import { assertMacOsAppLeaseAdmitsRequest } from '../resources/macos-app-lease.ts'; import { assertDaemonPolicyAdmitsLease } from '../core/daemon-policy.ts'; @@ -132,6 +133,31 @@ export function resolveRequestSessionName(params: { return sessionName; } +/** + * The session that a request acts through at its address. A request reaches a session that holds + * a lease only when it sends the id of that lease. A command that ignores the session lease gets + * no session there, so it stays detached from it. + */ +export function resolveRequestSessionRef( + req: DaemonRequest, + ref: SessionRef | undefined, +): SessionRef | undefined { + if (ref?.session.lease && ignoresSessionLease(req.command)) return undefined; + assertRequestHoldsSessionLease(req, ref?.session); + return ref; +} + +export type ResolveRequestSessionRef = typeof resolveRequestSessionRef; + +/** Whether a request may write to the journal of its address: only a session it acts through. */ +export function requestMayRecordSessionEvents( + req: DaemonRequest, + ref: SessionRef | undefined, +): boolean { + if (!ref?.session.lease) return true; + return !ignoresSessionLease(req.command) && !compareRequestLeaseClaims(req, ref.session.lease); +} + export function assertRequestLeaseAdmission( req: DaemonRequest, leaseRegistry: LeaseRegistry, @@ -140,6 +166,8 @@ export function assertRequestLeaseAdmission( env: HostEnvironment['env']; providerAppCatalog?: ProviderAppCatalog; daemonPolicy?: DaemonPolicy; + /** Runs the same rules and changes no lease: no registry sweep and no renewal. */ + inspectOnly?: boolean; }>, ): DeviceLease | undefined { // An admission-exempt command still acts on its session, so it must hold the session lease too. @@ -152,7 +180,7 @@ export function assertRequestLeaseAdmission( if (!sessionLease) assertProxyOpenLeaseMetadata(req, requestLeaseScope); if (options.daemonPolicy?.requiredLeaseBackend) { const leased = Boolean(resolveSessionLeaseScope(req, session).leaseId); - const lease = leased ? admitLease(req, leaseRegistry, session, options.env) : undefined; + const lease = leased ? admitLease(req, leaseRegistry, session, options) : undefined; assertDaemonPolicyAdmitsLease(options.daemonPolicy, lease); return lease; } @@ -164,7 +192,7 @@ export function assertRequestLeaseAdmission( return undefined; } if (runsWithoutLease(req, sessionLease, requestLeaseScope)) return undefined; - return admitLease(req, leaseRegistry, session, options.env); + return admitLease(req, leaseRegistry, session, options); } /** @@ -191,17 +219,26 @@ function admitLease( req: DaemonRequest, leaseRegistry: LeaseRegistry, session: SessionState | undefined, - env: HostEnvironment['env'], + options: Readonly<{ env: HostEnvironment['env']; inspectOnly?: boolean }>, ): DeviceLease { - const leaseScope = resolveSessionLeaseScope(req, session); - leaseRegistry.assertLeaseAdmission(leaseScopeToHeartbeatRequest(leaseScope)); + const admission = leaseScopeToHeartbeatRequest(resolveSessionLeaseScope(req, session)); + const lease = options.inspectOnly + ? leaseRegistry.inspectLeaseAdmission(admission) + : renewAdmittedLease(leaseRegistry, admission); + assertMacOsAppLeaseAdmitsRequest(lease, req, session, options.env); + if (isHumanControlMutation(req)) leaseRegistry.assertHumanControlAdmission(lease); + return lease; +} + +function renewAdmittedLease( + leaseRegistry: LeaseRegistry, + admission: ReturnType, +): DeviceLease { + leaseRegistry.assertLeaseAdmission(admission); // Admission renews for the window the lease already carries, or the window this request named. // Naming a proxy-specific default here used to shorten every lease allocated above it — a client // that rented a device for longer than the default lost it on the next admitted command (#2946). - const lease = leaseRegistry.heartbeatLease(leaseScopeToHeartbeatRequest(leaseScope)); - assertMacOsAppLeaseAdmitsRequest(lease, req, session, env); - if (isHumanControlMutation(req)) leaseRegistry.assertHumanControlAdmission(lease); - return lease; + return leaseRegistry.heartbeatLease(admission); } /** diff --git a/packages/daemon/src/sessions/request-handler-chain.ts b/packages/daemon/src/sessions/request-handler-chain.ts index ca9343ab7d..5ab93e5178 100644 --- a/packages/daemon/src/sessions/request-handler-chain.ts +++ b/packages/daemon/src/sessions/request-handler-chain.ts @@ -1,4 +1,5 @@ import type { LeaseEndReason } from './lease-lifecycle.ts'; +import type { ResolveRequestSessionRef } from './request-admission.ts'; import type { PluginInventoryEntry } from '@agent-device/contracts/plugin-inventory'; import type { DaemonResponse } from '@agent-device/kernel/contracts'; import type { DaemonPlatformServices } from '../core/platform-services.ts'; @@ -70,6 +71,7 @@ type RequestHandlerChainParams = { perfCaptureAdmissionLedger: PerfCaptureAdmissionLedger; screenRecordingAdmissionLedger: ScreenRecordingAdmissionLedger; hostDiagnostics?: HostDiagnostics; + resolveSessionRef?: ResolveRequestSessionRef; requestScope: PlatformRequestScope; retainDeviceExecutionLock(deviceId: string): Promise; throwIfCanceled(): void; @@ -169,6 +171,7 @@ async function runLeaseHandler( leaseLifecycleProvider: params.leaseLifecycleProvider, cloudArtifactProvider: params.cloudArtifactProvider, endSessionsForLease: params.endSessionsForLease, + resolveSessionRef: params.resolveSessionRef, }), ); } @@ -203,6 +206,7 @@ async function runSessionHandler( perfCaptureAdmissionLedger: params.perfCaptureAdmissionLedger, screenRecordingAdmissionLedger: params.screenRecordingAdmissionLedger, hostDiagnostics: params.hostDiagnostics, + resolveSessionRef: params.resolveSessionRef, requestScope: params.requestScope, retainDeviceExecutionLock: params.retainDeviceExecutionLock, throwIfCanceled: params.throwIfCanceled, diff --git a/packages/daemon/src/sessions/session-lifecycle/internal/inventory.ts b/packages/daemon/src/sessions/session-lifecycle/internal/inventory.ts index 39b67faef6..16825d4062 100644 --- a/packages/daemon/src/sessions/session-lifecycle/internal/inventory.ts +++ b/packages/daemon/src/sessions/session-lifecycle/internal/inventory.ts @@ -67,6 +67,10 @@ import type { InspectDeviceRuntimeFacts, } from '../../../core/request-runtime-binding.ts'; import { resolveLeaseScope } from '../../../core/lease-context.ts'; +import { + resolveRequestSessionRef, + type ResolveRequestSessionRef, +} from '../../request-admission.ts'; import { leaseProviderRequiringRuntime } from '@agent-device/contracts/lease-scope'; import { getRequestSignal } from '@agent-device/host-kit/request'; @@ -75,6 +79,7 @@ export type SessionInventoryCommandInput = Readonly<{ req: DaemonRequest; sessionName: string; sessionStore: SessionStore; + resolveSessionRef?: ResolveRequestSessionRef; inspectFacts?: InspectDeviceRuntimeFacts; platformServices: DaemonPlatformServices; bindDevice?: BindDeviceRuntime; @@ -91,19 +96,14 @@ export async function handleSessionInventoryCommands( case 'devices': return await devicesInventoryResponse(req, params.host.env, sessionStore); case 'capabilities': - return await capabilitiesInventoryResponse({ - req, - sessionName, - sessionStore, - inspectFacts: params.inspectFacts, - platformServices: params.platformServices, - }); + return await capabilitiesInventoryResponse(params); case 'apps': return await handleAppsInventory({ env: params.host.env, req, sessionName, sessionStore, + resolveSessionRef: params.resolveSessionRef, bindDevice: params.bindDevice, platformServices: params.platformServices, inspectFacts: params.inspectFacts, @@ -280,15 +280,16 @@ async function capabilitiesInventoryResponse(params: { req: DaemonRequest; sessionName: string; sessionStore: SessionStore; + resolveSessionRef?: ResolveRequestSessionRef; inspectFacts?: InspectDeviceRuntimeFacts; platformServices: DaemonPlatformServices; }): Promise { const resolution = await resolveInventoryCommandDevice(params); if ('response' in resolution) return resolution.response; - const { device } = resolution; + const { device, session } = resolution; const sessionOwnedAppStateAvailable = isSessionOwnedAppStateAvailable( device, - params.sessionStore.get(params.sessionName), + session, params.req.flags, ); const facts = await inspectCapabilityFacts(device, params.inspectFacts); @@ -359,23 +360,19 @@ async function handleAppsInventory(params: { req: DaemonRequest; sessionName: string; sessionStore: SessionStore; + resolveSessionRef?: ResolveRequestSessionRef; bindDevice?: BindDeviceRuntime; platformServices: DaemonPlatformServices; providerAppCatalog?: ProviderAppCatalog; inspectFacts?: InspectDeviceRuntimeFacts; }): Promise { - const { req, sessionName, sessionStore, bindDevice, inspectFacts, platformServices } = params; + const { req, bindDevice, inspectFacts, platformServices } = params; const providerCatalogResponse = await resolveProviderAppCatalogResponse( req, params.providerAppCatalog, ); if (providerCatalogResponse) return providerCatalogResponse; - const resolution = await resolveInventoryCommandDevice({ - req, - sessionName, - sessionStore, - platformServices, - }); + const resolution = await resolveInventoryCommandDevice(params); if ('response' in resolution) return resolution.response; const { device } = resolution; const appsFilter = assertResolvedAppsFilter(req.flags?.appsFilter); @@ -466,15 +463,27 @@ async function resolveInventoryCommandDevice(params: { req: DaemonRequest; sessionName: string; sessionStore: SessionStore; + resolveSessionRef?: ResolveRequestSessionRef; platformServices: DaemonPlatformServices; -}): Promise<{ device: DeviceInfo } | { response: DaemonResponse }> { - const { req, sessionName, sessionStore, platformServices } = params; - const session = sessionStore.get(sessionName); +}): Promise< + { device: DeviceInfo; session: ReturnType } | { response: DaemonResponse } +> { + const { + req, + sessionName, + sessionStore, + platformServices, + resolveSessionRef = resolveRequestSessionRef, + } = params; + // The one lease-checked session read of `apps` and `capabilities`. A leased session can open + // here after the admission of a command that holds no session lock. + const session = resolveSessionRef(req, sessionStore.lookup(sessionName))?.session; const flags = req.flags ?? {}; const response = requireSessionOrExplicitSelector(req.command, session, flags); if (response) return { response }; return { + session, device: await resolveCommandDevice({ session, req, diff --git a/packages/daemon/src/sessions/session-observability/__tests__/application.test.ts b/packages/daemon/src/sessions/session-observability/__tests__/application.test.ts index 2ba1b59678..07514d9067 100644 --- a/packages/daemon/src/sessions/session-observability/__tests__/application.test.ts +++ b/packages/daemon/src/sessions/session-observability/__tests__/application.test.ts @@ -19,6 +19,7 @@ import { handleSessionCommands } from '../../handlers/session.ts'; import { handleSessionObservabilityCommands } from '../internal/session-observability.ts'; const mockHandleSessionObservabilityCommands = vi.mocked(handleSessionObservabilityCommands); +const resolveSessionRef = () => undefined; beforeEach(() => { mockHandleSessionObservabilityCommands.mockClear(); @@ -44,6 +45,7 @@ async function run(command: DaemonRequest['command']): Promise ({ ok: true, data: {} }), reconcileOrphanedDeviceClaim: async () => ({ status: 'retained' as const, @@ -69,6 +71,7 @@ test('observability commands retain their route response and narrow facade input 'perfCaptureAdmissionLedger', 'platformServices', 'req', + 'resolveSessionRef', 'sessionName', 'sessionStore', 'throwIfCanceled', @@ -76,6 +79,7 @@ test('observability commands retain their route response and narrow facade input ); expect(forwarded).toMatchObject({ req: expect.objectContaining({ command }), + resolveSessionRef, sessionName: 'default', }); } diff --git a/packages/daemon/src/sessions/session-observability/internal/session-observability.ts b/packages/daemon/src/sessions/session-observability/internal/session-observability.ts index d1ec5e460e..8138b32625 100644 --- a/packages/daemon/src/sessions/session-observability/internal/session-observability.ts +++ b/packages/daemon/src/sessions/session-observability/internal/session-observability.ts @@ -32,6 +32,10 @@ import type { BindDeviceRuntime, InspectDeviceRuntimeFacts, } from '../../../core/request-runtime-binding.ts'; +import { + resolveRequestSessionRef, + type ResolveRequestSessionRef, +} from '../../request-admission.ts'; import type { SessionStore } from '../../../core/session-store.ts'; import { unsupportedOperationResponse } from '../../../core/runtime-admission.ts'; import type { DaemonRequest } from '../../../core/daemon-request.ts'; @@ -50,6 +54,7 @@ export type SessionObservabilityCommandInput = { req: DaemonRequest; sessionName: string; sessionStore: SessionStore; + resolveSessionRef?: ResolveRequestSessionRef; bindDevice?: BindDeviceRuntime; platformServices: DaemonPlatformServices; inspectFacts?: InspectDeviceRuntimeFacts; @@ -134,9 +139,11 @@ export async function handleSessionObservabilityCommands( } async function handleEventsCommand(params: ObservabilityInput): Promise { - const { req, sessionName, sessionStore } = params; + const { req, sessionName, sessionStore, resolveSessionRef = resolveRequestSessionRef } = params; try { await sessionStore.flushEvents(sessionName); + // `events` holds no session lock, so a leased session can open here after admission. + resolveSessionRef(req, sessionStore.lookup(sessionName)); return { ok: true, data: sessionStore.readEvents(sessionName, { diff --git a/packages/daemon/src/sessions/session-teardown.ts b/packages/daemon/src/sessions/session-teardown.ts index 912cd24e91..9111d4b72f 100644 --- a/packages/daemon/src/sessions/session-teardown.ts +++ b/packages/daemon/src/sessions/session-teardown.ts @@ -210,6 +210,34 @@ export async function teardownSessionResources( if (aggregate) throw aggregate; } +/** + * Releases the resources of an expired session, then finalizes its application. An idle expiry + * stops at a resource failure and retries later. A lease expiry retires the session whatever + * happens, so it still finalizes the application, and the resource failure stays the error. + */ +export async function settleExpiredSession( + request: SessionResourceTeardownRequest & { + finalizeApplication: () => Promise; + afterResourceFailure: 'stop' | 'finalize'; + }, +): Promise { + try { + await teardownSessionResources(request); + } catch (error) { + if (request.afterResourceFailure === 'finalize') { + await request.finalizeApplication().catch((finalizeError) => { + emitDiagnostic({ + level: 'error', + phase: 'expired_session_lifecycle_cleanup_failed', + data: { session: request.ref.address, error: String(finalizeError) }, + }); + }); + } + throw error; + } + await request.finalizeApplication(); +} + async function finishSessionScreenRecording(params: { ref: SessionRef; sessionStore: SessionStore;