Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion packages/capture-kit/src/snapshot-baseline.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,9 @@ test('the mint carries no presented surface a capture never read', () => {

test('the mint carries the system surface the nodes describe', () => {
expect(
surfaceScopedNodes(snapshot({ iosSystemSurfaceBundleId: 'com.apple.SafariViewService' })),
surfaceScopedNodes(
snapshot({ systemSurface: { bundleId: 'com.apple.SafariViewService', kind: 'web-auth' } }),
),
).toEqual({ nodes, surfaceBundleId: 'com.apple.SafariViewService' });
});

Expand Down
23 changes: 15 additions & 8 deletions packages/capture-kit/src/snapshot-baseline.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,18 @@
import type { SnapshotState } from '@agent-device/contracts/snapshot-model';
import type { SnapshotState, SnapshotSystemSurface } from '@agent-device/contracts/snapshot-model';
import type { SurfaceScopedNodes } from '@agent-device/contracts/interaction';
import type { SnapshotPresentation } from '@agent-device/kernel/snapshot';

function systemSurfaceBundleId(surface: SnapshotSystemSurface | undefined): string | undefined {
if (surface === undefined) return undefined;
switch (surface.kind) {
case 'android-system-overlay':
return undefined;
case 'web-auth':
case 'payment':
return surface.bundleId;
}
}

/** Reproduce a populated baseline's view; without one, capture the interactive view. */
export function postActionPresentation(
baseline: SurfaceScopedNodes | undefined,
Expand All @@ -25,17 +36,13 @@ export function postActionPresentation(
*/
export function surfaceScopedNodes(
snapshot:
| Pick<
SnapshotState,
'nodes' | 'iosSystemSurfaceBundleId' | 'presentedSurface' | 'presentation'
>
| Pick<SnapshotState, 'nodes' | 'systemSurface' | 'presentedSurface' | 'presentation'>
| undefined,
): SurfaceScopedNodes {
const surfaceBundleId = systemSurfaceBundleId(snapshot?.systemSurface);
return {
nodes: snapshot?.nodes ?? [],
...(snapshot?.iosSystemSurfaceBundleId
? { surfaceBundleId: snapshot.iosSystemSurfaceBundleId }
: {}),
...(surfaceBundleId ? { surfaceBundleId } : {}),
...(snapshot?.presentedSurface ? { presentedSurface: snapshot.presentedSurface } : {}),
...(snapshot?.presentation ? { presentation: snapshot.presentation } : {}),
};
Expand Down
2 changes: 1 addition & 1 deletion packages/capture-kit/src/snapshot-state.ts
Original file line number Diff line number Diff line change
Expand Up @@ -132,7 +132,7 @@ export function buildSnapshotState(
...(data.comparisonIdentity
? { comparisonKey: iosSnapshotComparisonIdentityKey(data.comparisonIdentity) }
: {}),
...(data.systemSurface ? { iosSystemSurfaceBundleId: data.systemSurface.bundleId } : {}),
...(data.systemSurface ? { systemSurface: data.systemSurface } : {}),
...(data.presentedSurface ? { presentedSurface: data.presentedSurface } : {}),
...(data.targetActivation ? { targetActivation: data.targetActivation } : {}),
...(data.keyboard ? { keyboard: data.keyboard } : {}),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -261,26 +261,38 @@ test('runtime snapshot emits filtered Android guidance from backend analysis', a
]);
});

test('runtime snapshot renders the system-surface disclosure from Android annotations', async () => {
const device = createSnapshotOnlyDevice({
snapshot: makeSnapshotState(
[
{ index: 0, depth: 0, type: 'FrameLayout', label: 'Quick settings' },
{ index: 1, depth: 1, parentIndex: 0, type: 'Switch', label: 'Internet' },
],
{ truncated: false, backend: 'android' },
),
androidSnapshot: {
backend: 'android-helper',
systemSurfaceOnly: true,
test('runtime snapshot renders the system-surface disclosure from Android annotations or SnapshotState.systemSurface', async () => {
const cases = [
{
snapshot: makeSnapshotState([{ index: 0, depth: 0, type: 'FrameLayout', label: 'Shade' }], {
truncated: false,
backend: 'android',
}),
androidSnapshot: { backend: 'android-helper' as const, systemSurfaceOnly: true },
match: /system surface \(notification shade, quick settings[\s\S]*press back or swipe up/,
},
});

const result = await device.capture.snapshot({ session: 'default' });

assert.equal(result.warnings?.length, 1);
assert.match(String(result.warnings?.[0]), /system surface \(notification shade, quick settings/);
assert.match(String(result.warnings?.[0]), /press back or swipe up/);
{
snapshot: makeSnapshotState([{ index: 0, depth: 0, type: 'FrameLayout', label: 'Shade' }], {
truncated: false,
backend: 'android',
systemSurface: { kind: 'android-system-overlay' as const },
}),
match: /system surface \(notification shade, quick settings/,
},
{
snapshot: makeSnapshotState([{ index: 0, depth: 0, type: 'Application', label: 'Sheet' }], {
truncated: false,
backend: 'xctest',
systemSurface: { bundleId: 'com.apple.SafariViewService', kind: 'web-auth' as const },
}),
match: /system web sign-in sheet/,
},
];
for (const { match, ...input } of cases) {
const result = await createSnapshotOnlyDevice(input).capture.snapshot({ session: 'default' });
assert.equal(result.warnings?.length, 1);
assert.match(String(result.warnings?.[0]), match);
}
});

test('runtime snapshot warns when iOS interactive output is root-only', async () => {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ import {
import { isLikelyStaleSnapshotDrop } from '@agent-device/capture-kit/snapshot-freshness';
import { formatPostGestureOutcomeWarning } from '@agent-device/capture-kit/post-gesture-stability';
import { buildSnapshotVisibility } from '@agent-device/capture-kit/snapshot-visibility';
import { ANDROID_SYSTEM_SURFACE_DISCLOSURE } from '@agent-device/contracts/android-system-surface-disclosure';
import { systemSurfaceDisclosure } from '@agent-device/contracts/system-surface-disclosure';
import { formatReactNativeOverlayWarning } from '@agent-device/capture-kit/react-native-overlay';
import { reactNativeOverlaySurfaceFromCapture } from '@agent-device/contracts/react-native-overlay';
import { now } from '../../runtime-common.ts';
Expand Down Expand Up @@ -350,7 +350,12 @@ function buildSnapshotWarnings(params: {
);
if (reactNativeOverlayWarning) warnings.push(reactNativeOverlayWarning);

const systemSurfaceWarning = formatAndroidSystemSurfaceWarning(params.annotations);
const systemSurfaceWarning = systemSurfaceDisclosure(
params.snapshot.systemSurface ??
(params.annotations.androidSnapshot?.systemSurfaceOnly === true
? { kind: 'android-system-overlay' }
: undefined),
);
if (systemSurfaceWarning) warnings.push(systemSurfaceWarning);

const recentDropWarning = formatRecentSnapshotDropWarning(params);
Expand All @@ -360,13 +365,6 @@ function buildSnapshotWarnings(params: {
return Array.from(new Set(warnings));
}

function formatAndroidSystemSurfaceWarning(
annotations: SnapshotCaptureAnnotations,
): string | undefined {
if (annotations.androidSnapshot?.systemSurfaceOnly !== true) return undefined;
return ANDROID_SYSTEM_SURFACE_DISCLOSURE;
}

function buildSparseIosInteractiveWarnings(params: {
snapshot: SnapshotState;
options: SnapshotCommandOptions;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -556,7 +556,7 @@ function webSignInSheetSnapshot(labels: string[]): SnapshotState {
hittable: true,
})),
),
iosSystemSurfaceBundleId: WEB_SIGN_IN_SHEET_BUNDLE_ID,
systemSurface: { bundleId: WEB_SIGN_IN_SHEET_BUNDLE_ID, kind: 'web-auth' },
};
}

Expand Down Expand Up @@ -595,7 +595,7 @@ test('runtime press with verify discloses the surface change when a sign-in shee
test('runtime press with verify reports the sheet leaving even when the two digests coincide', async () => {
const sheet: SnapshotState = {
...selectorSnapshot(),
iosSystemSurfaceBundleId: WEB_SIGN_IN_SHEET_BUNDLE_ID,
systemSurface: { bundleId: WEB_SIGN_IN_SHEET_BUNDLE_ID, kind: 'web-auth' },
};
const app = selectorSnapshot();
// The premise this test exists for: the sheet tree and the app tree that replaces it digest
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import type { PreresolvedInteractionTarget } from '@agent-device/contracts/inter
import type { MovingTargetBaseline } from '@agent-device/contracts/runtime-contract';
import type { Point } from '@agent-device/kernel/snapshot';
import type { SnapshotState } from '@agent-device/contracts/snapshot-model';
import { surfaceScopedNodes } from '@agent-device/capture-kit/snapshot-baseline';
import { readNodeLocalIdentity, readNodeStructuralDenotation } from '@agent-device/ad-script';
import { makeSnapshotState } from '@agent-device/selectors/snapshot-geometry-fixtures';
import { INTERACTION_ERROR_REASONS } from '@agent-device/selectors/interaction-error';
Expand Down Expand Up @@ -53,7 +54,9 @@ function slideTree(x: number, over?: { label?: string; surfaceBundleId?: string
hittable: true,
},
],
over?.surfaceBundleId ? { iosSystemSurfaceBundleId: over.surfaceBundleId } : undefined,
over?.surfaceBundleId
? { systemSurface: { bundleId: over.surfaceBundleId, kind: 'web-auth' } }
: undefined,
);
}

Expand Down Expand Up @@ -93,12 +96,7 @@ function createGuardedDevice(params: {
finishedAt: clock.now() - params.baselineAgeMs,
// The baseline carries its surface the way a real one does: `surfaceScopedNodes` reads it
// off the capture, so a fixture that wants a system-surface baseline asks for one.
tree: {
nodes: baselineTree.nodes,
...(baselineTree.iosSystemSurfaceBundleId
? { surfaceBundleId: baselineTree.iosSystemSurfaceBundleId }
: {}),
},
tree: surfaceScopedNodes(baselineTree),
};
const device = createAgentDevice({
backend: {
Expand Down Expand Up @@ -300,7 +298,7 @@ test('a surface that arrives between the two looks is not rest', async () => {
// surfaces for exactly the #2438 reason — a system sheet can carry the same control at the same
// rect — and the two looks that certify rest are the same comparison. Without the surface check, a
// sheet presented during the sample gap would let a press land on it while reporting agreement.
const sheet = 'com.apple.springboard';
const sheet = 'com.apple.SafariViewService';
const device = createGuardedDevice({
stored: slideTree(MID_FLIGHT_X),
trees: [
Expand Down Expand Up @@ -358,7 +356,7 @@ test('a target newly arrived on a different surface is re-observed, not certifie
// sheet arriving over the app can carry the same-identity node at the same rect, and that is the
// positional coincidence ADR 0014 refuses rather than evidence of rest. The press pays the
// re-observations instead of tapping on the strength of the other surface's tree.
const sheet = 'com.apple.springboard';
const sheet = 'com.apple.SafariViewService';
const device = createGuardedDevice({
stored: slideTree(MID_FLIGHT_X, { surfaceBundleId: sheet }),
trees: [
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ function webSignInSheetSnapshot(labels: string[]): SnapshotState {
hittable: true,
})),
),
iosSystemSurfaceBundleId: WEB_SIGN_IN_SHEET_BUNDLE_ID,
systemSurface: { bundleId: WEB_SIGN_IN_SHEET_BUNDLE_ID, kind: 'web-auth' },
};
}

Expand Down Expand Up @@ -81,7 +81,10 @@ test('press --settle attaches no diff across an app-to-sheet surface change and
// Disclosed, not hidden: the settled sheet still becomes the stored observation
// a follow-up snapshot reads.
const stored = (await device.sessions.get('default')) as { snapshot?: SnapshotState };
assert.equal(stored.snapshot?.iosSystemSurfaceBundleId, WEB_SIGN_IN_SHEET_BUNDLE_ID);
assert.deepEqual(stored.snapshot?.systemSurface, {
bundleId: WEB_SIGN_IN_SHEET_BUNDLE_ID,
kind: 'web-auth',
});
});

test('press --settle attaches no diff across a sheet-to-app surface change and discloses it', async () => {
Expand Down Expand Up @@ -151,7 +154,7 @@ test('press --settle --verify reports a surface change even when the selector an
test('press --settle --verify reports a sheet-to-app surface change when the digests coincide', async () => {
const sheet: SnapshotState = {
...welcomeSnapshot(),
iosSystemSurfaceBundleId: WEB_SIGN_IN_SHEET_BUNDLE_ID,
systemSurface: { bundleId: WEB_SIGN_IN_SHEET_BUNDLE_ID, kind: 'web-auth' },
};
const app = welcomeSnapshot();
// The premise this test exists for: the sheet tree and the app tree that replaces
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ function signInSheetSnapshot(): SnapshotState {
hittable: true,
})),
),
iosSystemSurfaceBundleId: SHEET_BUNDLE_ID,
systemSurface: { bundleId: SHEET_BUNDLE_ID, kind: 'web-auth' },
};
}

Expand Down Expand Up @@ -62,10 +62,7 @@ for (const route of ['scroll', 'back'] as const) {
assert.match(settle.hint ?? '', /different surfaces/);
assert.match(settle.hint ?? '', /take a snapshot/i);
const stored = (await device.sessions.get('default')) as { snapshot?: SnapshotState };
assert.equal(
stored.snapshot?.iosSystemSurfaceBundleId,
transition.after.iosSystemSurfaceBundleId,
);
assert.deepEqual(stored.snapshot?.systemSurface, transition.after.systemSurface);
});
}
}
Expand Down
8 changes: 4 additions & 4 deletions packages/contracts/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -36,10 +36,6 @@
"types": "./src/android-system-chrome.ts",
"default": "./src/android-system-chrome.ts"
},
"./android-system-surface-disclosure": {
"types": "./src/android-system-surface-disclosure.ts",
"default": "./src/android-system-surface-disclosure.ts"
},
"./app-deployment-runtime": {
"types": "./src/app-deployment-runtime.ts",
"default": "./src/app-deployment-runtime.ts"
Expand Down Expand Up @@ -524,6 +520,10 @@
"types": "./src/system-button-runtime.ts",
"default": "./src/system-button-runtime.ts"
},
"./system-surface-disclosure": {
"types": "./src/system-surface-disclosure.ts",
"default": "./src/system-surface-disclosure.ts"
},
"./tap-keyboard-occlusion": {
"types": "./src/tap-keyboard-occlusion.ts",
"default": "./src/tap-keyboard-occlusion.ts"
Expand Down
16 changes: 0 additions & 16 deletions packages/contracts/src/android-system-surface-disclosure.ts

This file was deleted.

1 change: 1 addition & 0 deletions packages/contracts/src/facades/snapshot.ts
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ export {
type SnapshotProducerRegistry,
type SnapshotStateProvenance,
type SnapshotCaptureProvenance,
type SnapshotSystemSurface,
type SnapshotState,
type PostGestureAction,
type PostGestureOutcome,
Expand Down
25 changes: 15 additions & 10 deletions packages/contracts/src/snapshot-model.ts
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ import {
type SnapshotProducerRegistry,
type SnapshotProvenance,
} from './snapshot-producers.ts';
import type { IosSystemSurfaceProvenance } from './ios-system-surface.ts';

export {
isSnapshotBackend,
Expand Down Expand Up @@ -88,6 +89,15 @@ export function usesMobileSnapshotPresentation(backend: SnapshotBackend | undefi
);
}

/**
* An occluding system surface a capture shows instead of app content, one shape for every platform.
* Android reports an overlay (notification shade, quick settings) without a host identity. iOS
* reports the registered in-place host it serves (#2438).
*/
export type SnapshotSystemSurface =
| Readonly<{ kind: 'android-system-overlay' }>
| IosSystemSurfaceProvenance;

export type SnapshotState = {
nodes: SnapshotNode[];
createdAt: number;
Expand All @@ -101,22 +111,17 @@ export type SnapshotState = {
/** Opaque equality key for iOS acquisition and presentation lineage. */
comparisonKey?: string;
/**
* Android: the capture is an occluding system surface (notification shade, quick settings)
* rather than app content. Consumers that surface this tree to the agent must disclose the
* occlusion (see `@agent-device/contracts/android-system-surface-disclosure`).
* The occluding system surface this capture describes instead of app content. Consumers that
* surface this tree to the agent disclose it through
* `@agent-device/contracts/system-surface-disclosure`. Two captures that name different iOS
* hosts describe different surfaces and are never compared as the same presentation.
*/
systemSurfaceOnly?: boolean;
systemSurface?: SnapshotSystemSurface;
/**
* Android: the producer proved this capture a complete read of the current hierarchy, taken after
* it cleared the platform node cache. Absent proves nothing.
*/
freshTraversal?: true;
/**
* iOS: the bundle id of the in-place system surface this capture describes (a web sign-in sheet
* presented over the app, #2438). Two captures that disagree here describe different surfaces and
* must never be compared as the same presentation; consumers that surface the tree disclose it.
*/
iosSystemSurfaceBundleId?: string;
/**
* iOS: whether this capture read a modal presentation over the app's own content (#3565), decided
* on the acquired tree before interactive filtering, compaction, depth, or scope, so every view of
Expand Down
Loading
Loading