Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions packages/capture-kit/src/snapshot-baseline.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@ import type { SnapshotState, SnapshotSystemSurface } from '@agent-device/contrac
import type { SurfaceScopedNodes } from '@agent-device/contracts/interaction';
import type { SnapshotPresentation } from '@agent-device/kernel/snapshot';

// Intentionally duplicates `@agent-device/contracts/snapshot-model`'s `systemSurfaceBundleId`
// so this module keeps a type-only import and stays within ADR 0027's eager-closure budget.
function systemSurfaceBundleId(surface: SnapshotSystemSurface | undefined): string | undefined {
if (surface === undefined) return undefined;
switch (surface.kind) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ import {
} from '@agent-device/contracts/capture';
import { AppError } from '@agent-device/kernel/errors';
import { isSameSnapshotPresentation } from '@agent-device/kernel/snapshot';
import { normalizeType } from '@agent-device/contracts/snapshot';
import { androidSnapshotSystemSurface, normalizeType } from '@agent-device/contracts/snapshot';
import { snapshotProducerFacts } from '@agent-device/contracts/snapshot-producers';
import type {
PresentedSurface,
Expand Down Expand Up @@ -352,9 +352,7 @@ function buildSnapshotWarnings(params: {

const systemSurfaceWarning = systemSurfaceDisclosure(
params.snapshot.systemSurface ??
(params.annotations.androidSnapshot?.systemSurfaceOnly === true
? { kind: 'android-system-overlay' }
: undefined),
androidSnapshotSystemSurface(params.annotations.androidSnapshot),
);
if (systemSurfaceWarning) warnings.push(systemSurfaceWarning);

Expand Down
2 changes: 2 additions & 0 deletions packages/contracts/src/facades/snapshot.ts
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,8 @@ export {
} from '../snapshot-scope.ts';
export {
snapshotStateProvenance,
systemSurfaceBundleId,
androidSnapshotSystemSurface,
isSnapshotBackend,
usesMobileSnapshotPresentation,
inheritPostGestureOutcome,
Expand Down
22 changes: 22 additions & 0 deletions packages/contracts/src/snapshot-model.test.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,9 @@
import { expect, test } from 'vitest';
import {
androidSnapshotSystemSurface,
readPostGestureOutcome,
snapshotStateProvenance,
systemSurfaceBundleId,
type SnapshotProvenance,
type SnapshotStateProvenance,
} from './snapshot-model.ts';
Expand Down Expand Up @@ -45,3 +47,23 @@ test('readPostGestureOutcome reads every outcome kind off the wire and drops mal
).toBeUndefined();
expect(readPostGestureOutcome('loading')).toBeUndefined();
});

test('systemSurfaceBundleId projects the host identity only for host-bearing surfaces', () => {
expect(systemSurfaceBundleId(undefined)).toBeUndefined();
expect(systemSurfaceBundleId({ kind: 'android-system-overlay' })).toBeUndefined();
expect(systemSurfaceBundleId({ kind: 'web-auth', bundleId: 'com.apple.SafariViewService' })).toBe(
'com.apple.SafariViewService',
);
expect(systemSurfaceBundleId({ kind: 'payment', bundleId: 'com.apple.PassbookUIService' })).toBe(
'com.apple.PassbookUIService',
);
});

test('androidSnapshotSystemSurface maps systemSurfaceOnly to the shared Android overlay arm', () => {
expect(androidSnapshotSystemSurface(undefined)).toBeUndefined();
expect(androidSnapshotSystemSurface({})).toBeUndefined();
expect(androidSnapshotSystemSurface({ systemSurfaceOnly: false })).toBeUndefined();
expect(androidSnapshotSystemSurface({ systemSurfaceOnly: true })).toEqual({
kind: 'android-system-overlay',
});
});
24 changes: 24 additions & 0 deletions packages/contracts/src/snapshot-model.ts
Original file line number Diff line number Diff line change
Expand Up @@ -98,6 +98,30 @@ export type SnapshotSystemSurface =
| Readonly<{ kind: 'android-system-overlay' }>
| IosSystemSurfaceProvenance;

/**
* The surface host bundle id a capture scopes comparisons by, or undefined when the surface carries
* no host identity (ordinary app content or an Android system overlay).
*/
export function systemSurfaceBundleId(
surface: SnapshotSystemSurface | undefined,
): string | undefined {
if (surface === undefined) return undefined;
switch (surface.kind) {
case 'android-system-overlay':
return undefined;
case 'web-auth':
case 'payment':
return surface.bundleId;
}
}

/** Maps Android capture metadata to the shared occluding-surface model when it reports one. */
export function androidSnapshotSystemSurface(
metadata: Readonly<{ systemSurfaceOnly?: boolean }> | undefined,
): SnapshotSystemSurface | undefined {
return metadata?.systemSurfaceOnly === true ? { kind: 'android-system-overlay' } : undefined;
}

export type SnapshotState = {
nodes: SnapshotNode[];
createdAt: number;
Expand Down
15 changes: 2 additions & 13 deletions packages/daemon/src/core/deferred-interaction-outcome.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ import type { PostOpenObservation } from '@agent-device/contracts/application-li
import type { SnapshotCaptureAnnotations } from '@agent-device/contracts/capture';
import { isApplePlatform, isMobilePlatform } from '@agent-device/kernel/device';
import { formatRole, type SnapshotNode } from '@agent-device/kernel/snapshot';
import type { SnapshotState, SnapshotSystemSurface } from '@agent-device/contracts/snapshot-model';
import type { SnapshotState } from '@agent-device/contracts/snapshot-model';
import type { MovingTargetBaseline } from '@agent-device/contracts/runtime-contract';
import {
captureAndroidFreshnessRecoveredAttempt,
Expand All @@ -28,7 +28,7 @@ import {
runPostGestureStabilityLoop,
type PostGestureStabilityOutcome,
} from '@agent-device/capture-kit/post-gesture-stability';
import { isViewportRootNode } from '@agent-device/contracts/snapshot';
import { isViewportRootNode, systemSurfaceBundleId } from '@agent-device/contracts/snapshot';
import type { DeferredSurface, SessionState } from './session-state.ts';

/**
Expand Down Expand Up @@ -127,17 +127,6 @@ function restTree(snapshot: SnapshotState | undefined): SurfaceScopedNodes | und
};
}

function systemSurfaceBundleId(surface: SnapshotSystemSurface | undefined): string | undefined {
if (surface === undefined) return undefined;
switch (surface.kind) {
case 'android-system-overlay':
return undefined;
case 'web-auth':
case 'payment':
return surface.bundleId;
}
}

/** Whether this mutation left an unsettled surface for the capture side to steady. */
function markSurfaceStabilization(
session: SessionState,
Expand Down
6 changes: 3 additions & 3 deletions packages/daemon/src/core/snapshot-capture.ts
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ import {
type SnapshotViewportSize,
} from '@agent-device/kernel/snapshot';
import {
androidSnapshotSystemSurface,
type SnapshotCaptureProvenance,
type SnapshotState,
} from '@agent-device/contracts/snapshot-model';
Expand Down Expand Up @@ -152,9 +153,8 @@ async function captureSnapshotAttempt(params: CaptureSnapshotParams): Promise<Sn
// The one seam where snapshot state and capture annotations meet: consumers that only keep the
// SnapshotState (selector-backed find/wait, session-stored snapshots) must still learn that the
// capture is an occluding system surface, so the disclosure is not lost with the annotations.
if (annotations.androidSnapshot?.systemSurfaceOnly === true) {
snapshot.systemSurface = { kind: 'android-system-overlay' };
}
const androidSurface = androidSnapshotSystemSurface(annotations.androidSnapshot);
if (androidSurface) snapshot.systemSurface = androidSurface;
if (provesFreshAndroidTraversal(annotations.androidSnapshot)) snapshot.freshTraversal = true;
return { data, snapshot, annotations };
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -337,6 +337,22 @@ test('a message-less runner capture leaves its disclosures to the verdict', asyn
assert.deepEqual(result.quality?.collapsedLeafIndexes, [1]);
});

test('a runner system-surface capture carries provenance without duplicating the disclosure warning', async () => {
const systemSurface = { bundleId: 'com.apple.SafariViewService', kind: 'web-auth' } as const;
const interactor = createAppleInteractor(IOS_SIMULATOR, {}, createLiveTestHostEnvironment(), {
hasLiveSession: () => true,
runCommand: async () => ({
...runnerResultFor({ command: 'snapshot' }),
systemSurface,
}),
});

const result = presentedSnapshot(await interactor.snapshot());

assert.deepEqual(result.systemSurface, systemSurface);
assert.equal('warnings' in result, false);
});

test('macOS app snapshots preserve runner nodes outside the iOS presentation engine', async () => {
const nodes = [{ index: 0, type: 'Other', value: 'checkbox, unchecked' }];
const interactor = createAppleInteractor(MACOS_DEVICE, {}, createLiveTestHostEnvironment(), {
Expand Down
8 changes: 2 additions & 6 deletions packages/platform-apple/src/interactor.ts
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,6 @@ import {
readAppleSnapshotResult,
type AppleRunnerSnapshotResult,
} from './snapshot-presentation.ts';
import { iosSystemSurfaceDisclosure } from '@agent-device/contracts/ios-system-surface';
import { iosTargetActivationDisclosure } from '@agent-device/contracts/ios-target-activation';

const SNAPSHOT_RESPONSE_REASONS = {
Expand Down Expand Up @@ -409,9 +408,8 @@ function assertReportedRunnerSnapshotNodes(

/**
* Agent-facing warnings for a runner capture: a legacy runner's message text when it carried no
* quality verdict, the shared disclosure when the capture describes an in-place system surface
* (e.g. the web sign-in or Apple Pay sheet) rather than the app itself (#2438), and the disclosure
* when this capture's own command had to bring the session app back to the foreground (#2682).
* quality verdict, and the disclosure when this capture's own command had to bring the session app
* back to the foreground (#2682).
*/
function runnerSnapshotWarnings(result: AppleRunnerSnapshotResult): string[] {
const warnings: string[] = [];
Expand All @@ -421,8 +419,6 @@ function runnerSnapshotWarnings(result: AppleRunnerSnapshotResult): string[] {
);
}
if (!result.quality && result.message) warnings.push(result.message);
if (result.systemSurface)
warnings.push(iosSystemSurfaceDisclosure(result.systemSurface.bundleId));
if (result.targetActivation)
warnings.push(iosTargetActivationDisclosure(result.targetActivation));
return warnings;
Expand Down
Loading