From 779c4cfb8fbabc7b654e533395642685103270b5 Mon Sep 17 00:00:00 2001 From: Samanyu Das Date: Sun, 22 Nov 2026 18:40:20 +1030 Subject: [PATCH] refactor(snapshot): one shared model for occluding system surfaces SnapshotState carried an occluding system surface on two parallel channels: `systemSurfaceOnly` for Android and `iosSystemSurfaceBundleId` for iOS. Every consumer branched on both, and a third platform would have added a third field. Replace both with `systemSurface?: SnapshotSystemSurface`, a union of the Android overlay (`{ kind: 'android-system-overlay' }`) and the iOS registered host provenance (`IosSystemSurfaceProvenance`). The capture path stores the iOS provenance as the runner reported it, so bundle id and kind cannot disagree. - `systemSurfaceDisclosure(surface)` in the renamed `@agent-device/contracts/system-surface-disclosure` derives the one agent-facing sentence for both platforms. - `systemSurfaceBundleId(surface)` gives the post-action baseline scope. An Android overlay scopes nothing, as before. - `AndroidSnapshotBackendMetadata.systemSurfaceOnly` stays: it is the Android capture backend's own evidence, which the daemon maps to the shared model at its one seam. Disclosure sentences, comparison scoping, and wire payloads are unchanged. Closes #2489 Co-Authored-By: Claude Opus 5.5 --- .../capture-kit/src/snapshot-baseline.test.ts | 4 +- packages/capture-kit/src/snapshot-baseline.ts | 23 ++++++--- packages/capture-kit/src/snapshot-state.ts | 2 +- .../commands/capture/runtime/snapshot.test.ts | 50 ++++++++++++------- .../src/commands/capture/runtime/snapshot.ts | 16 +++--- .../interaction/runtime/interactions.test.ts | 4 +- .../runtime/moving-target-gate.test.ts | 16 +++--- .../runtime/post-action-surface.test.ts | 9 ++-- .../runtime/settle-generic-surface.test.ts | 7 +-- packages/contracts/package.json | 8 +-- .../src/android-system-surface-disclosure.ts | 16 ------ packages/contracts/src/facades/snapshot.ts | 1 + packages/contracts/src/snapshot-model.ts | 25 ++++++---- .../src/system-surface-disclosure.ts | 27 ++++++++++ ...pture-disclosure-target-activation.test.ts | 2 +- .../__tests__/moving-target-baseline.test.ts | 12 ++--- .../post-gesture-stabilization.test.ts | 4 +- .../daemon/src/core/capture-disclosure.ts | 9 ++-- .../src/core/deferred-interaction-outcome.ts | 18 +++++-- packages/daemon/src/core/snapshot-capture.ts | 2 +- .../index-capture-disclosure.test.ts | 16 +++--- .../interaction-ios-tap-outcome.test.ts | 2 +- .../snapshot-runtime-disclosure.test.ts | 2 +- .../layering/contracts-exports.snapshot.json | 2 +- 24 files changed, 159 insertions(+), 118 deletions(-) delete mode 100644 packages/contracts/src/android-system-surface-disclosure.ts create mode 100644 packages/contracts/src/system-surface-disclosure.ts diff --git a/packages/capture-kit/src/snapshot-baseline.test.ts b/packages/capture-kit/src/snapshot-baseline.test.ts index dbb529e0cc..6a5cf46554 100644 --- a/packages/capture-kit/src/snapshot-baseline.test.ts +++ b/packages/capture-kit/src/snapshot-baseline.test.ts @@ -26,7 +26,9 @@ test('the mint carries no presented surface a capture never read', () => { test('the mint carries the system surface the nodes describe', () => { expect( - surfaceScopedNodes(snapshot({ iosSystemSurfaceBundleId: 'com.apple.SafariViewService' })), + surfaceScopedNodes( + snapshot({ systemSurface: { bundleId: 'com.apple.SafariViewService', kind: 'web-auth' } }), + ), ).toEqual({ nodes, surfaceBundleId: 'com.apple.SafariViewService' }); }); diff --git a/packages/capture-kit/src/snapshot-baseline.ts b/packages/capture-kit/src/snapshot-baseline.ts index 50ce2a05bd..5b5b2141a1 100644 --- a/packages/capture-kit/src/snapshot-baseline.ts +++ b/packages/capture-kit/src/snapshot-baseline.ts @@ -1,7 +1,18 @@ -import type { SnapshotState } from '@agent-device/contracts/snapshot-model'; +import type { SnapshotState, SnapshotSystemSurface } from '@agent-device/contracts/snapshot-model'; import type { SurfaceScopedNodes } from '@agent-device/contracts/interaction'; import type { SnapshotPresentation } from '@agent-device/kernel/snapshot'; +function systemSurfaceBundleId(surface: SnapshotSystemSurface | undefined): string | undefined { + if (surface === undefined) return undefined; + switch (surface.kind) { + case 'android-system-overlay': + return undefined; + case 'web-auth': + case 'payment': + return surface.bundleId; + } +} + /** Reproduce a populated baseline's view; without one, capture the interactive view. */ export function postActionPresentation( baseline: SurfaceScopedNodes | undefined, @@ -25,17 +36,13 @@ export function postActionPresentation( */ export function surfaceScopedNodes( snapshot: - | Pick< - SnapshotState, - 'nodes' | 'iosSystemSurfaceBundleId' | 'presentedSurface' | 'presentation' - > + | Pick | undefined, ): SurfaceScopedNodes { + const surfaceBundleId = systemSurfaceBundleId(snapshot?.systemSurface); return { nodes: snapshot?.nodes ?? [], - ...(snapshot?.iosSystemSurfaceBundleId - ? { surfaceBundleId: snapshot.iosSystemSurfaceBundleId } - : {}), + ...(surfaceBundleId ? { surfaceBundleId } : {}), ...(snapshot?.presentedSurface ? { presentedSurface: snapshot.presentedSurface } : {}), ...(snapshot?.presentation ? { presentation: snapshot.presentation } : {}), }; diff --git a/packages/capture-kit/src/snapshot-state.ts b/packages/capture-kit/src/snapshot-state.ts index 4094d2aa9d..053bbf2105 100644 --- a/packages/capture-kit/src/snapshot-state.ts +++ b/packages/capture-kit/src/snapshot-state.ts @@ -132,7 +132,7 @@ export function buildSnapshotState( ...(data.comparisonIdentity ? { comparisonKey: iosSnapshotComparisonIdentityKey(data.comparisonIdentity) } : {}), - ...(data.systemSurface ? { iosSystemSurfaceBundleId: data.systemSurface.bundleId } : {}), + ...(data.systemSurface ? { systemSurface: data.systemSurface } : {}), ...(data.presentedSurface ? { presentedSurface: data.presentedSurface } : {}), ...(data.targetActivation ? { targetActivation: data.targetActivation } : {}), ...(data.keyboard ? { keyboard: data.keyboard } : {}), diff --git a/packages/command-runtime/src/commands/capture/runtime/snapshot.test.ts b/packages/command-runtime/src/commands/capture/runtime/snapshot.test.ts index 67b862f38f..7eefca54dc 100644 --- a/packages/command-runtime/src/commands/capture/runtime/snapshot.test.ts +++ b/packages/command-runtime/src/commands/capture/runtime/snapshot.test.ts @@ -261,26 +261,38 @@ test('runtime snapshot emits filtered Android guidance from backend analysis', a ]); }); -test('runtime snapshot renders the system-surface disclosure from Android annotations', async () => { - const device = createSnapshotOnlyDevice({ - snapshot: makeSnapshotState( - [ - { index: 0, depth: 0, type: 'FrameLayout', label: 'Quick settings' }, - { index: 1, depth: 1, parentIndex: 0, type: 'Switch', label: 'Internet' }, - ], - { truncated: false, backend: 'android' }, - ), - androidSnapshot: { - backend: 'android-helper', - systemSurfaceOnly: true, +test('runtime snapshot renders the system-surface disclosure from Android annotations or SnapshotState.systemSurface', async () => { + const cases = [ + { + snapshot: makeSnapshotState([{ index: 0, depth: 0, type: 'FrameLayout', label: 'Shade' }], { + truncated: false, + backend: 'android', + }), + androidSnapshot: { backend: 'android-helper' as const, systemSurfaceOnly: true }, + match: /system surface \(notification shade, quick settings[\s\S]*press back or swipe up/, }, - }); - - const result = await device.capture.snapshot({ session: 'default' }); - - assert.equal(result.warnings?.length, 1); - assert.match(String(result.warnings?.[0]), /system surface \(notification shade, quick settings/); - assert.match(String(result.warnings?.[0]), /press back or swipe up/); + { + snapshot: makeSnapshotState([{ index: 0, depth: 0, type: 'FrameLayout', label: 'Shade' }], { + truncated: false, + backend: 'android', + systemSurface: { kind: 'android-system-overlay' as const }, + }), + match: /system surface \(notification shade, quick settings/, + }, + { + snapshot: makeSnapshotState([{ index: 0, depth: 0, type: 'Application', label: 'Sheet' }], { + truncated: false, + backend: 'xctest', + systemSurface: { bundleId: 'com.apple.SafariViewService', kind: 'web-auth' as const }, + }), + match: /system web sign-in sheet/, + }, + ]; + for (const { match, ...input } of cases) { + const result = await createSnapshotOnlyDevice(input).capture.snapshot({ session: 'default' }); + assert.equal(result.warnings?.length, 1); + assert.match(String(result.warnings?.[0]), match); + } }); test('runtime snapshot warns when iOS interactive output is root-only', async () => { diff --git a/packages/command-runtime/src/commands/capture/runtime/snapshot.ts b/packages/command-runtime/src/commands/capture/runtime/snapshot.ts index 12e003e61b..0cb9d20c1c 100644 --- a/packages/command-runtime/src/commands/capture/runtime/snapshot.ts +++ b/packages/command-runtime/src/commands/capture/runtime/snapshot.ts @@ -37,7 +37,7 @@ import { import { isLikelyStaleSnapshotDrop } from '@agent-device/capture-kit/snapshot-freshness'; import { formatPostGestureOutcomeWarning } from '@agent-device/capture-kit/post-gesture-stability'; import { buildSnapshotVisibility } from '@agent-device/capture-kit/snapshot-visibility'; -import { ANDROID_SYSTEM_SURFACE_DISCLOSURE } from '@agent-device/contracts/android-system-surface-disclosure'; +import { systemSurfaceDisclosure } from '@agent-device/contracts/system-surface-disclosure'; import { formatReactNativeOverlayWarning } from '@agent-device/capture-kit/react-native-overlay'; import { reactNativeOverlaySurfaceFromCapture } from '@agent-device/contracts/react-native-overlay'; import { now } from '../../runtime-common.ts'; @@ -350,7 +350,12 @@ function buildSnapshotWarnings(params: { ); if (reactNativeOverlayWarning) warnings.push(reactNativeOverlayWarning); - const systemSurfaceWarning = formatAndroidSystemSurfaceWarning(params.annotations); + const systemSurfaceWarning = systemSurfaceDisclosure( + params.snapshot.systemSurface ?? + (params.annotations.androidSnapshot?.systemSurfaceOnly === true + ? { kind: 'android-system-overlay' } + : undefined), + ); if (systemSurfaceWarning) warnings.push(systemSurfaceWarning); const recentDropWarning = formatRecentSnapshotDropWarning(params); @@ -360,13 +365,6 @@ function buildSnapshotWarnings(params: { return Array.from(new Set(warnings)); } -function formatAndroidSystemSurfaceWarning( - annotations: SnapshotCaptureAnnotations, -): string | undefined { - if (annotations.androidSnapshot?.systemSurfaceOnly !== true) return undefined; - return ANDROID_SYSTEM_SURFACE_DISCLOSURE; -} - function buildSparseIosInteractiveWarnings(params: { snapshot: SnapshotState; options: SnapshotCommandOptions; diff --git a/packages/command-runtime/src/commands/interaction/runtime/interactions.test.ts b/packages/command-runtime/src/commands/interaction/runtime/interactions.test.ts index 6324caeb28..ed9fead954 100644 --- a/packages/command-runtime/src/commands/interaction/runtime/interactions.test.ts +++ b/packages/command-runtime/src/commands/interaction/runtime/interactions.test.ts @@ -556,7 +556,7 @@ function webSignInSheetSnapshot(labels: string[]): SnapshotState { hittable: true, })), ), - iosSystemSurfaceBundleId: WEB_SIGN_IN_SHEET_BUNDLE_ID, + systemSurface: { bundleId: WEB_SIGN_IN_SHEET_BUNDLE_ID, kind: 'web-auth' }, }; } @@ -595,7 +595,7 @@ test('runtime press with verify discloses the surface change when a sign-in shee test('runtime press with verify reports the sheet leaving even when the two digests coincide', async () => { const sheet: SnapshotState = { ...selectorSnapshot(), - iosSystemSurfaceBundleId: WEB_SIGN_IN_SHEET_BUNDLE_ID, + systemSurface: { bundleId: WEB_SIGN_IN_SHEET_BUNDLE_ID, kind: 'web-auth' }, }; const app = selectorSnapshot(); // The premise this test exists for: the sheet tree and the app tree that replaces it digest diff --git a/packages/command-runtime/src/commands/interaction/runtime/moving-target-gate.test.ts b/packages/command-runtime/src/commands/interaction/runtime/moving-target-gate.test.ts index 859d0f19a8..50d73c9180 100644 --- a/packages/command-runtime/src/commands/interaction/runtime/moving-target-gate.test.ts +++ b/packages/command-runtime/src/commands/interaction/runtime/moving-target-gate.test.ts @@ -6,6 +6,7 @@ import type { PreresolvedInteractionTarget } from '@agent-device/contracts/inter import type { MovingTargetBaseline } from '@agent-device/contracts/runtime-contract'; import type { Point } from '@agent-device/kernel/snapshot'; import type { SnapshotState } from '@agent-device/contracts/snapshot-model'; +import { surfaceScopedNodes } from '@agent-device/capture-kit/snapshot-baseline'; import { readNodeLocalIdentity, readNodeStructuralDenotation } from '@agent-device/ad-script'; import { makeSnapshotState } from '@agent-device/selectors/snapshot-geometry-fixtures'; import { INTERACTION_ERROR_REASONS } from '@agent-device/selectors/interaction-error'; @@ -53,7 +54,9 @@ function slideTree(x: number, over?: { label?: string; surfaceBundleId?: string hittable: true, }, ], - over?.surfaceBundleId ? { iosSystemSurfaceBundleId: over.surfaceBundleId } : undefined, + over?.surfaceBundleId + ? { systemSurface: { bundleId: over.surfaceBundleId, kind: 'web-auth' } } + : undefined, ); } @@ -93,12 +96,7 @@ function createGuardedDevice(params: { finishedAt: clock.now() - params.baselineAgeMs, // The baseline carries its surface the way a real one does: `surfaceScopedNodes` reads it // off the capture, so a fixture that wants a system-surface baseline asks for one. - tree: { - nodes: baselineTree.nodes, - ...(baselineTree.iosSystemSurfaceBundleId - ? { surfaceBundleId: baselineTree.iosSystemSurfaceBundleId } - : {}), - }, + tree: surfaceScopedNodes(baselineTree), }; const device = createAgentDevice({ backend: { @@ -300,7 +298,7 @@ test('a surface that arrives between the two looks is not rest', async () => { // surfaces for exactly the #2438 reason — a system sheet can carry the same control at the same // rect — and the two looks that certify rest are the same comparison. Without the surface check, a // sheet presented during the sample gap would let a press land on it while reporting agreement. - const sheet = 'com.apple.springboard'; + const sheet = 'com.apple.SafariViewService'; const device = createGuardedDevice({ stored: slideTree(MID_FLIGHT_X), trees: [ @@ -358,7 +356,7 @@ test('a target newly arrived on a different surface is re-observed, not certifie // sheet arriving over the app can carry the same-identity node at the same rect, and that is the // positional coincidence ADR 0014 refuses rather than evidence of rest. The press pays the // re-observations instead of tapping on the strength of the other surface's tree. - const sheet = 'com.apple.springboard'; + const sheet = 'com.apple.SafariViewService'; const device = createGuardedDevice({ stored: slideTree(MID_FLIGHT_X, { surfaceBundleId: sheet }), trees: [ diff --git a/packages/command-runtime/src/commands/interaction/runtime/post-action-surface.test.ts b/packages/command-runtime/src/commands/interaction/runtime/post-action-surface.test.ts index ab868bb842..23d78c727f 100644 --- a/packages/command-runtime/src/commands/interaction/runtime/post-action-surface.test.ts +++ b/packages/command-runtime/src/commands/interaction/runtime/post-action-surface.test.ts @@ -32,7 +32,7 @@ function webSignInSheetSnapshot(labels: string[]): SnapshotState { hittable: true, })), ), - iosSystemSurfaceBundleId: WEB_SIGN_IN_SHEET_BUNDLE_ID, + systemSurface: { bundleId: WEB_SIGN_IN_SHEET_BUNDLE_ID, kind: 'web-auth' }, }; } @@ -81,7 +81,10 @@ test('press --settle attaches no diff across an app-to-sheet surface change and // Disclosed, not hidden: the settled sheet still becomes the stored observation // a follow-up snapshot reads. const stored = (await device.sessions.get('default')) as { snapshot?: SnapshotState }; - assert.equal(stored.snapshot?.iosSystemSurfaceBundleId, WEB_SIGN_IN_SHEET_BUNDLE_ID); + assert.deepEqual(stored.snapshot?.systemSurface, { + bundleId: WEB_SIGN_IN_SHEET_BUNDLE_ID, + kind: 'web-auth', + }); }); test('press --settle attaches no diff across a sheet-to-app surface change and discloses it', async () => { @@ -151,7 +154,7 @@ test('press --settle --verify reports a surface change even when the selector an test('press --settle --verify reports a sheet-to-app surface change when the digests coincide', async () => { const sheet: SnapshotState = { ...welcomeSnapshot(), - iosSystemSurfaceBundleId: WEB_SIGN_IN_SHEET_BUNDLE_ID, + systemSurface: { bundleId: WEB_SIGN_IN_SHEET_BUNDLE_ID, kind: 'web-auth' }, }; const app = welcomeSnapshot(); // The premise this test exists for: the sheet tree and the app tree that replaces diff --git a/packages/command-runtime/src/commands/interaction/runtime/settle-generic-surface.test.ts b/packages/command-runtime/src/commands/interaction/runtime/settle-generic-surface.test.ts index 1512573233..7448f69f3d 100644 --- a/packages/command-runtime/src/commands/interaction/runtime/settle-generic-surface.test.ts +++ b/packages/command-runtime/src/commands/interaction/runtime/settle-generic-surface.test.ts @@ -24,7 +24,7 @@ function signInSheetSnapshot(): SnapshotState { hittable: true, })), ), - iosSystemSurfaceBundleId: SHEET_BUNDLE_ID, + systemSurface: { bundleId: SHEET_BUNDLE_ID, kind: 'web-auth' }, }; } @@ -62,10 +62,7 @@ for (const route of ['scroll', 'back'] as const) { assert.match(settle.hint ?? '', /different surfaces/); assert.match(settle.hint ?? '', /take a snapshot/i); const stored = (await device.sessions.get('default')) as { snapshot?: SnapshotState }; - assert.equal( - stored.snapshot?.iosSystemSurfaceBundleId, - transition.after.iosSystemSurfaceBundleId, - ); + assert.deepEqual(stored.snapshot?.systemSurface, transition.after.systemSurface); }); } } diff --git a/packages/contracts/package.json b/packages/contracts/package.json index 5fd4a71025..49d7c8cc45 100644 --- a/packages/contracts/package.json +++ b/packages/contracts/package.json @@ -36,10 +36,6 @@ "types": "./src/android-system-chrome.ts", "default": "./src/android-system-chrome.ts" }, - "./android-system-surface-disclosure": { - "types": "./src/android-system-surface-disclosure.ts", - "default": "./src/android-system-surface-disclosure.ts" - }, "./app-deployment-runtime": { "types": "./src/app-deployment-runtime.ts", "default": "./src/app-deployment-runtime.ts" @@ -524,6 +520,10 @@ "types": "./src/system-button-runtime.ts", "default": "./src/system-button-runtime.ts" }, + "./system-surface-disclosure": { + "types": "./src/system-surface-disclosure.ts", + "default": "./src/system-surface-disclosure.ts" + }, "./tap-keyboard-occlusion": { "types": "./src/tap-keyboard-occlusion.ts", "default": "./src/tap-keyboard-occlusion.ts" diff --git a/packages/contracts/src/android-system-surface-disclosure.ts b/packages/contracts/src/android-system-surface-disclosure.ts deleted file mode 100644 index 60e9521e57..0000000000 --- a/packages/contracts/src/android-system-surface-disclosure.ts +++ /dev/null @@ -1,16 +0,0 @@ -import type { SnapshotState } from '@agent-device/contracts/snapshot-model'; - -/** - * The one agent-facing explanation for an Android capture that faithfully shows an occluding - * system surface (notification shade, quick settings) instead of app content. Shared by the - * direct snapshot warning and every selector-backed consumer (find/wait/get/is), so the - * disclosure cannot silently drop on one route while surviving on another. - */ -export const ANDROID_SYSTEM_SURFACE_DISCLOSURE = - 'A system surface (notification shade, quick settings, or another system overlay) covers the app, so this snapshot shows that surface. Interact with it or dismiss it (press back or swipe up) to reach app content.'; - -export function systemSurfaceDisclosure( - snapshot: Pick | undefined, -): string | undefined { - return snapshot?.systemSurfaceOnly === true ? ANDROID_SYSTEM_SURFACE_DISCLOSURE : undefined; -} diff --git a/packages/contracts/src/facades/snapshot.ts b/packages/contracts/src/facades/snapshot.ts index 990c33b6cc..66ed7b5e70 100644 --- a/packages/contracts/src/facades/snapshot.ts +++ b/packages/contracts/src/facades/snapshot.ts @@ -27,6 +27,7 @@ export { type SnapshotProducerRegistry, type SnapshotStateProvenance, type SnapshotCaptureProvenance, + type SnapshotSystemSurface, type SnapshotState, type PostGestureAction, type PostGestureOutcome, diff --git a/packages/contracts/src/snapshot-model.ts b/packages/contracts/src/snapshot-model.ts index c4877c0ef2..6a4f5c6f48 100644 --- a/packages/contracts/src/snapshot-model.ts +++ b/packages/contracts/src/snapshot-model.ts @@ -18,6 +18,7 @@ import { type SnapshotProducerRegistry, type SnapshotProvenance, } from './snapshot-producers.ts'; +import type { IosSystemSurfaceProvenance } from './ios-system-surface.ts'; export { isSnapshotBackend, @@ -88,6 +89,15 @@ export function usesMobileSnapshotPresentation(backend: SnapshotBackend | undefi ); } +/** + * An occluding system surface a capture shows instead of app content, one shape for every platform. + * Android reports an overlay (notification shade, quick settings) without a host identity. iOS + * reports the registered in-place host it serves (#2438). + */ +export type SnapshotSystemSurface = + | Readonly<{ kind: 'android-system-overlay' }> + | IosSystemSurfaceProvenance; + export type SnapshotState = { nodes: SnapshotNode[]; createdAt: number; @@ -101,22 +111,17 @@ export type SnapshotState = { /** Opaque equality key for iOS acquisition and presentation lineage. */ comparisonKey?: string; /** - * Android: the capture is an occluding system surface (notification shade, quick settings) - * rather than app content. Consumers that surface this tree to the agent must disclose the - * occlusion (see `@agent-device/contracts/android-system-surface-disclosure`). + * The occluding system surface this capture describes instead of app content. Consumers that + * surface this tree to the agent disclose it through + * `@agent-device/contracts/system-surface-disclosure`. Two captures that name different iOS + * hosts describe different surfaces and are never compared as the same presentation. */ - systemSurfaceOnly?: boolean; + systemSurface?: SnapshotSystemSurface; /** * Android: the producer proved this capture a complete read of the current hierarchy, taken after * it cleared the platform node cache. Absent proves nothing. */ freshTraversal?: true; - /** - * iOS: the bundle id of the in-place system surface this capture describes (a web sign-in sheet - * presented over the app, #2438). Two captures that disagree here describe different surfaces and - * must never be compared as the same presentation; consumers that surface the tree disclose it. - */ - iosSystemSurfaceBundleId?: string; /** * iOS: whether this capture read a modal presentation over the app's own content (#3565), decided * on the acquired tree before interactive filtering, compaction, depth, or scope, so every view of diff --git a/packages/contracts/src/system-surface-disclosure.ts b/packages/contracts/src/system-surface-disclosure.ts new file mode 100644 index 0000000000..a8ce1b79d6 --- /dev/null +++ b/packages/contracts/src/system-surface-disclosure.ts @@ -0,0 +1,27 @@ +import type { SnapshotSystemSurface } from '@agent-device/contracts/snapshot-model'; +import { iosSystemSurfaceDisclosure } from '@agent-device/contracts/ios-system-surface'; + +/** + * The agent-facing explanation for an Android capture that faithfully shows an occluding system + * surface (notification shade, quick settings) instead of app content. + */ +export const ANDROID_SYSTEM_SURFACE_DISCLOSURE = + 'A system surface (notification shade, quick settings, or another system overlay) covers the app, so this snapshot shows that surface. Interact with it or dismiss it (press back or swipe up) to reach app content.'; + +/** + * The one agent-facing explanation for a capture that shows an occluding system surface, on every + * platform. Every selector-backed consumer (find/wait/get/is) reads it from the stored capture, so + * the disclosure cannot silently drop on one route while surviving on another. + */ +export function systemSurfaceDisclosure( + surface: SnapshotSystemSurface | undefined, +): string | undefined { + if (surface === undefined) return undefined; + switch (surface.kind) { + case 'android-system-overlay': + return ANDROID_SYSTEM_SURFACE_DISCLOSURE; + case 'web-auth': + case 'payment': + return iosSystemSurfaceDisclosure(surface.bundleId); + } +} diff --git a/packages/daemon/src/core/__tests__/capture-disclosure-target-activation.test.ts b/packages/daemon/src/core/__tests__/capture-disclosure-target-activation.test.ts index affb5a84dd..d62e708893 100644 --- a/packages/daemon/src/core/__tests__/capture-disclosure-target-activation.test.ts +++ b/packages/daemon/src/core/__tests__/capture-disclosure-target-activation.test.ts @@ -92,7 +92,7 @@ test('surface and foreground disclosures ride one response together', () => { const response = withCaptureDisclosures({ response: okResponse({ nodes: [] }), consumedTree: { - iosSystemSurfaceBundleId: 'com.apple.SafariViewService', + systemSurface: { bundleId: 'com.apple.SafariViewService', kind: 'web-auth' }, targetActivation: FACT, }, captureProof: { targetActivation: FACT }, diff --git a/packages/daemon/src/core/__tests__/moving-target-baseline.test.ts b/packages/daemon/src/core/__tests__/moving-target-baseline.test.ts index a89f319b71..5963545bd7 100644 --- a/packages/daemon/src/core/__tests__/moving-target-baseline.test.ts +++ b/packages/daemon/src/core/__tests__/moving-target-baseline.test.ts @@ -117,15 +117,15 @@ test('a baseline keeps the surface its tree came from', () => { const session = makeSession('ios'); const sheet = makeSnapshotState( [{ index: 0, type: 'Button', label: 'Allow', rect: { x: 10, y: 20, width: 80, height: 40 } }], - { iosSystemSurfaceBundleId: 'com.apple.springboard' }, + { systemSurface: { bundleId: 'com.apple.SafariViewService', kind: 'web-auth' } }, ); session.snapshot = sheet; // The scope rides through as-is: the capture side narrows its tree with the same shape // (`surfaceScopedNodes`), so an iOS system-sheet tree must never be screened against app content. - markPress(session, { nodes: sheet.nodes, surfaceBundleId: 'com.apple.springboard' }); + markPress(session, { nodes: sheet.nodes, surfaceBundleId: 'com.apple.SafariViewService' }); - assert.equal(movingTargetBaseline(session)?.tree.surfaceBundleId, 'com.apple.springboard'); + assert.equal(movingTargetBaseline(session)?.tree.surfaceBundleId, 'com.apple.SafariViewService'); }); test('a device side effect the session did not resolve a target through retires the baseline', () => { @@ -245,11 +245,11 @@ test('a settle that stored its settled tree records that tree, not the tree the test('a stored settled tree keeps the surface and presentation its capture read', () => { // #3565: the rest-proof baseline is minted like every other one, so a settle that came to rest on - // a sheet over SpringBoard hands the gate the same facts the capture published. + // a system sheet hands the gate the same facts the capture published. const session = makeSession('ios'); session.snapshot = { ...appTree('Settled'), - iosSystemSurfaceBundleId: 'com.apple.springboard', + systemSurface: { bundleId: 'com.apple.SafariViewService', kind: 'web-auth' }, presentedSurface: 'presented', }; @@ -263,7 +263,7 @@ test('a stored settled tree keeps the surface and presentation its capture read' }); const tree = movingTargetBaseline(session)?.tree; - assert.equal(tree?.surfaceBundleId, 'com.apple.springboard'); + assert.equal(tree?.surfaceBundleId, 'com.apple.SafariViewService'); assert.equal(tree?.presentedSurface, 'presented'); }); diff --git a/packages/daemon/src/core/__tests__/post-gesture-stabilization.test.ts b/packages/daemon/src/core/__tests__/post-gesture-stabilization.test.ts index 727bdfec75..18a1135773 100644 --- a/packages/daemon/src/core/__tests__/post-gesture-stabilization.test.ts +++ b/packages/daemon/src/core/__tests__/post-gesture-stabilization.test.ts @@ -757,7 +757,7 @@ test('capturePostGestureStabilizedResult re-baselines when an in-place system su makeSnapshotState(pickupSnapshot(500).nodes, { snapshotQuality: { state: 'healthy', backend: 'tree' }, comparisonKey: systemSurfaceCaptureComparisonKey(session.device.id), - iosSystemSurfaceBundleId: 'com.apple.SafariViewService', + systemSurface: { bundleId: 'com.apple.SafariViewService', kind: 'web-auth' }, }), ); @@ -791,7 +791,7 @@ test('capturePostGestureStabilizedResult re-baselines when an in-place system su session.snapshot = makeSnapshotState(pickupSnapshot(500).nodes, { snapshotQuality: { state: 'healthy', backend: 'tree' }, comparisonKey: systemSurfaceCaptureComparisonKey(session.device.id), - iosSystemSurfaceBundleId: 'com.apple.SafariViewService', + systemSurface: { bundleId: 'com.apple.SafariViewService', kind: 'web-auth' }, }); markPostGestureStabilization(session, 'scroll'); diff --git a/packages/daemon/src/core/capture-disclosure.ts b/packages/daemon/src/core/capture-disclosure.ts index 730d9f043a..57c70c32b7 100644 --- a/packages/daemon/src/core/capture-disclosure.ts +++ b/packages/daemon/src/core/capture-disclosure.ts @@ -1,14 +1,13 @@ import type { DaemonResponse } from '@agent-device/kernel/contracts'; import type { SnapshotState } from '@agent-device/contracts/snapshot-model'; -import { systemSurfaceDisclosure } from '@agent-device/contracts/android-system-surface-disclosure'; -import { iosSystemSurfaceDisclosure } from '@agent-device/contracts/ios-system-surface'; +import { systemSurfaceDisclosure } from '@agent-device/contracts/system-surface-disclosure'; import { iosTargetActivationDisclosure } from '@agent-device/contracts/ios-target-activation'; import { formatPostGestureOutcomeWarning } from '@agent-device/capture-kit/post-gesture-stability'; /** The capture provenance a response must be disclosed against (#2438, #2682, gesture outcomes). */ export type CaptureProvenance = Pick< SnapshotState, - 'systemSurfaceOnly' | 'iosSystemSurfaceBundleId' | 'targetActivation' | 'postGestureOutcome' + 'systemSurface' | 'targetActivation' | 'postGestureOutcome' >; /** @@ -53,9 +52,7 @@ export function withSystemSurfaceDisclosure( response: DaemonResponse, snapshot: CaptureProvenance | undefined, ): DaemonResponse { - const disclosure = snapshot?.iosSystemSurfaceBundleId - ? iosSystemSurfaceDisclosure(snapshot.iosSystemSurfaceBundleId) - : systemSurfaceDisclosure(snapshot); + const disclosure = systemSurfaceDisclosure(snapshot?.systemSurface); return disclosure ? appendDisclosure(response, disclosure, 'warning') : response; } diff --git a/packages/daemon/src/core/deferred-interaction-outcome.ts b/packages/daemon/src/core/deferred-interaction-outcome.ts index 1ef7baa1ca..c690e661ef 100644 --- a/packages/daemon/src/core/deferred-interaction-outcome.ts +++ b/packages/daemon/src/core/deferred-interaction-outcome.ts @@ -4,7 +4,7 @@ import type { PostOpenObservation } from '@agent-device/contracts/application-li import type { SnapshotCaptureAnnotations } from '@agent-device/contracts/capture'; import { isApplePlatform, isMobilePlatform } from '@agent-device/kernel/device'; import { formatRole, type SnapshotNode } from '@agent-device/kernel/snapshot'; -import type { SnapshotState } from '@agent-device/contracts/snapshot-model'; +import type { SnapshotState, SnapshotSystemSurface } from '@agent-device/contracts/snapshot-model'; import type { MovingTargetBaseline } from '@agent-device/contracts/runtime-contract'; import { captureAndroidFreshnessRecoveredAttempt, @@ -119,15 +119,25 @@ export function markDeferredInteractionOutcome( */ function restTree(snapshot: SnapshotState | undefined): SurfaceScopedNodes | undefined { if (!snapshot) return undefined; + const surfaceBundleId = systemSurfaceBundleId(snapshot.systemSurface); return { nodes: snapshot.nodes, - ...(snapshot.iosSystemSurfaceBundleId - ? { surfaceBundleId: snapshot.iosSystemSurfaceBundleId } - : {}), + ...(surfaceBundleId ? { surfaceBundleId } : {}), ...(snapshot.presentedSurface ? { presentedSurface: snapshot.presentedSurface } : {}), }; } +function systemSurfaceBundleId(surface: SnapshotSystemSurface | undefined): string | undefined { + if (surface === undefined) return undefined; + switch (surface.kind) { + case 'android-system-overlay': + return undefined; + case 'web-auth': + case 'payment': + return surface.bundleId; + } +} + /** Whether this mutation left an unsettled surface for the capture side to steady. */ function markSurfaceStabilization( session: SessionState, diff --git a/packages/daemon/src/core/snapshot-capture.ts b/packages/daemon/src/core/snapshot-capture.ts index 66684b9cfd..eefb0c3ef4 100644 --- a/packages/daemon/src/core/snapshot-capture.ts +++ b/packages/daemon/src/core/snapshot-capture.ts @@ -153,7 +153,7 @@ async function captureSnapshotAttempt(params: CaptureSnapshotParams): Promise { const response = withSystemSurfaceDisclosure( { ok: true, data: { found: true, warning: 'prior warning text' } }, - { systemSurfaceOnly: true }, + { systemSurface: { kind: 'android-system-overlay' } }, ); expect(response.ok).toBe(true); if (!response.ok) return; @@ -232,7 +232,7 @@ test('disclosure appends after an existing failure hint instead of replacing it' ok: false, error: { code: 'NOT_FOUND', message: 'no match', details: { hint: 'prior hint text' } }, }, - { systemSurfaceOnly: true }, + { systemSurface: { kind: 'android-system-overlay' } }, ); expect(response.ok).toBe(false); if (response.ok) return; @@ -300,8 +300,8 @@ test('sessionless wait timeout still discloses the occluding system surface', as // --- #2438: an in-place iOS system surface (web sign-in sheet) discloses on the same shared seam --- -// The runner stamps `systemSurface` on a capture it served from the sheet; buildSnapshotState turns -// that into `iosSystemSurfaceBundleId`, which selector routes must carry and disclose. +// The runner stamps `systemSurface` on a capture it served from the sheet; buildSnapshotState carries +// it onto the SnapshotState, which selector routes must carry and disclose. const WEB_SIGN_IN_SHEET_BUNDLE_ID = 'com.apple.SafariViewService'; const WEB_SIGN_IN_DISCLOSURE = iosSystemSurfaceDisclosure(WEB_SIGN_IN_SHEET_BUNDLE_ID); const SHEET_SNAPSHOT_DATA = { @@ -392,7 +392,7 @@ test('mutating find that misses on an in-place system surface still discloses it test('the shared disclosure helper reports an iOS system surface on both outcomes', () => { const ok = withSystemSurfaceDisclosure( { ok: true, data: { found: true } }, - { iosSystemSurfaceBundleId: WEB_SIGN_IN_SHEET_BUNDLE_ID }, + { systemSurface: { bundleId: WEB_SIGN_IN_SHEET_BUNDLE_ID, kind: 'web-auth' } }, ); expect(ok.ok).toBe(true); if (!ok.ok) return; @@ -400,7 +400,7 @@ test('the shared disclosure helper reports an iOS system surface on both outcome const failed = withSystemSurfaceDisclosure( { ok: false, error: { code: 'NOT_FOUND', message: 'no match' } }, - { iosSystemSurfaceBundleId: WEB_SIGN_IN_SHEET_BUNDLE_ID }, + { systemSurface: { bundleId: WEB_SIGN_IN_SHEET_BUNDLE_ID, kind: 'web-auth' } }, ); expect(failed.ok).toBe(false); if (failed.ok) return; diff --git a/packages/daemon/src/execution/interaction/internal/interaction-ios-tap-outcome.test.ts b/packages/daemon/src/execution/interaction/internal/interaction-ios-tap-outcome.test.ts index 190da96aaa..4ff69b5d95 100644 --- a/packages/daemon/src/execution/interaction/internal/interaction-ios-tap-outcome.test.ts +++ b/packages/daemon/src/execution/interaction/internal/interaction-ios-tap-outcome.test.ts @@ -443,7 +443,7 @@ test('a capture of a system surface cannot corroborate a tap taken against the a sessionStore.publish(sessionName, session); const after = snapshot(imageViewerNodes); after.comparisonKey = systemSurfaceCaptureComparisonKey(IOS_SIMULATOR.id); - after.iosSystemSurfaceBundleId = 'com.apple.SafariViewService'; + after.systemSurface = { bundleId: 'com.apple.SafariViewService', kind: 'web-auth' }; await expect( corroborateIosTapFailure({ diff --git a/packages/daemon/src/sessions/handlers/snapshot-runtime-disclosure.test.ts b/packages/daemon/src/sessions/handlers/snapshot-runtime-disclosure.test.ts index 0a23ee983f..4310eb5a78 100644 --- a/packages/daemon/src/sessions/handlers/snapshot-runtime-disclosure.test.ts +++ b/packages/daemon/src/sessions/handlers/snapshot-runtime-disclosure.test.ts @@ -160,7 +160,7 @@ function scenario(params: { storedRepair?: boolean; android?: boolean }) { createdAt: Date.now(), nodes: [{ ref: 'e1', index: 0, type: 'Button', label: 'Earlier' }], targetActivation: REPAIR, - iosSystemSurfaceBundleId: 'com.apple.SafariViewService', + systemSurface: { bundleId: 'com.apple.SafariViewService', kind: 'web-auth' }, }; } sessionStore.publish(sessionName, session); diff --git a/scripts/layering/contracts-exports.snapshot.json b/scripts/layering/contracts-exports.snapshot.json index 4c1cd7ccc0..b51f52ae76 100644 --- a/scripts/layering/contracts-exports.snapshot.json +++ b/scripts/layering/contracts-exports.snapshot.json @@ -6,7 +6,6 @@ "@agent-device/contracts/android-observation", "@agent-device/contracts/android-snapshot-quality", "@agent-device/contracts/android-system-chrome", - "@agent-device/contracts/android-system-surface-disclosure", "@agent-device/contracts/app-deployment-runtime", "@agent-device/contracts/app-deployment-runtime-plan", "@agent-device/contracts/app-event-runtime", @@ -127,6 +126,7 @@ "@agent-device/contracts/snapshot-scope", "@agent-device/contracts/startup-recovery-fence", "@agent-device/contracts/system-button-runtime", + "@agent-device/contracts/system-surface-disclosure", "@agent-device/contracts/tap-keyboard-occlusion", "@agent-device/contracts/touch-runtime", "@agent-device/contracts/tv-remote",