diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerAppFixtures.m b/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerAppFixtures.m index b2c46d0a0e..1062cc07ad 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerAppFixtures.m +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerAppFixtures.m @@ -221,6 +221,35 @@ - (void)viewDidLoad { } @end + +// `--agent-device-custom-drawn-canvas` adds a plain drawn view the accessibility tree never +// exposes: a tap toggles its fill, and the point over it names no element but the window. The +// screenshot-diff verification needs exactly this shape — custom-drawn content whose only answer +// is pixels, at a point no hittable or semantic node covers. +@interface AgentDeviceCustomDrawnCanvas : UIView +@end + +@implementation AgentDeviceCustomDrawnCanvas +- (instancetype)initWithFrame:(CGRect)frame { + self = [super initWithFrame:frame]; + if (self) { + self.isAccessibilityElement = NO; + self.backgroundColor = UIColor.systemGray5Color; + UITapGestureRecognizer *tap = + [[UITapGestureRecognizer alloc] initWithTarget:self action:@selector(toggled:)]; + [self addGestureRecognizer:tap]; + } + return self; +} + +- (void)toggled:(UITapGestureRecognizer *)recognizer { + (void)recognizer; + self.backgroundColor = [self.backgroundColor isEqual:UIColor.systemRedColor] + ? UIColor.systemGray5Color + : UIColor.systemRedColor; +} +@end + @interface AgentDeviceRunnerFixtureViewController () // Held for the app's lifetime: a window nobody retained would be gone before the test queried it. @property(nonatomic, strong) UIWindow *secondWindow; @@ -1466,6 +1495,18 @@ - (void)viewDidLoad { [self.view addSubview:popupTitle]; } + if ([NSProcessInfo.processInfo.arguments containsObject:@"--agent-device-custom-drawn-canvas"]) { + UIView *canvas = [[AgentDeviceCustomDrawnCanvas alloc] initWithFrame:CGRectMake(120, 300, 160, 160)]; + canvas.translatesAutoresizingMaskIntoConstraints = NO; + [self.view addSubview:canvas]; + [NSLayoutConstraint activateConstraints:@[ + [canvas.centerXAnchor constraintEqualToAnchor:self.view.centerXAnchor], + [canvas.topAnchor constraintEqualToAnchor:label.bottomAnchor constant:120], + [canvas.widthAnchor constraintEqualToConstant:160], + [canvas.heightAnchor constraintEqualToConstant:160], + ]]; + } + if ([NSProcessInfo.processInfo.arguments containsObject:@"--agent-device-crowded-screen"]) { for (NSUInteger row = 0; row < 500; row++) { UILabel *rowLabel = [[UILabel alloc] initWithFrame:CGRectMake(0, 60 + (row % 30) * 24, 400, 16)]; diff --git a/docs/evidence/android-verify-screenshot-fallback-2026-11-20.md b/docs/evidence/android-verify-screenshot-fallback-2026-11-20.md new file mode 100644 index 0000000000..4dbc76ddbb --- /dev/null +++ b/docs/evidence/android-verify-screenshot-fallback-2026-11-20.md @@ -0,0 +1,115 @@ +# Android `--verify` screenshot fallback through the full-screen-container exemption, 2026-11-20 + +Evidence for #4297 at the gate-exemption head `28c408b94`. Device: Redmi +K60 (`2312DRAABC`, MIUI Android 14, 1080x2400), Edge (`com.microsoft.emmx`) +serving the local rig page over `adb reverse tcp:8642`, backend +`android-helper` (`snapshotQuality.backend: "android-helper"`). The battery +level was forced to 37 with `adb shell dumpsys battery set level 37` about +400 ms into the press and reset right after, so the only candidate change +besides the page itself was the status-bar glyph. + +## The tree that used to close the gate, and the exemption that opens it + +Interactive snapshot at the rig page (23 nodes; header rows trimmed): + +```json +{"ref": "e1", "type": "android.webkit.WebView", "hittable": true, + "rect": {"x": 0, "y": 237, "width": 1080, "height": 1976}}, +{"ref": "e3", "type": "android.widget.Button", "hittable": true, + "rect": {"x": 385, "y": 253, "width": 310, "height": 143}, "label": "TAB A"}, +{"ref": "e4", "type": "android.widget.TextView", "hittable": null, + "rect": {"x": 0, "y": 237, "width": 1080, "height": 654}, "label": "static body"} +``` + +No window or application node exists (Android emits none), so the point's +resolved viewport comes from `resolveViewportRect`'s largest-containing-rect +fallback. At the dead point `(540, 1500)` the only covering hittable node +is the WebView container, and its rect IS the resolved viewport — the +option-(a) exemption admits the pair. Under the pre-exemption gate this +same tree and point produced tree evidence with no `screenshotDiff` (the +gate closed on every real point). + +## Run 1 — dead tap with a forced status-bar change + +`press 540 1500 --verify --json` on the rig's dead area, battery forced +mid-dispatch: + +```json +{ + "success": true, + "data": { + "x": 540, + "y": 1500, + "message": "Tapped (540, 1500)", + "targetKind": "point", + "evidence": { + "digest": "ax1:f017ea1bdbba74a8", + "nodeCount": 23, + "interactiveNodeCount": 23, + "foregroundApp": "com.microsoft.emmx", + "changedFromBefore": false, + "screenshotDiff": { + "changedFromBefore": false, + "changedPixelRatio": 0 + } + } + } +} +``` + +`changedFromBefore: false`, ratio 0 — the forced change landed between the +frames (the battery read 37 during the window and 100 again after the +reset), the 4% portrait band excluded it, and the outside-the-status-bar +comparison saw an identical frame. A post-run screenshot shows the status +bar back at 100% and the page unchanged. + +A second dead run on the MIUI launcher (`com.miui.home`, `press 540 700 +--verify --json`, same forced change) read the same shape: +`changedFromBefore: false`, `screenshotDiff {false, 0}`, tree digest +`ax1:36d6f2391b33939c`. + +## Run 2 — a tap that redraws custom-drawn content + +The rig page's second version carries a 280x180 canvas (a plain drawn +surface the tree reports as a hittable-less `android.widget.Image` at +`154,391 772x495` with no semantics). `press 540 638 --verify --json` +taps it; its click handler repaints the canvas gray→red and rewrites the +state row: + +```json +{ + "success": true, + "data": { + "x": 540, + "y": 638, + "message": "Tapped (540, 638)", + "targetKind": "point", + "evidence": { + "digest": "ax1:8832c6a9aa185a47", + "nodeCount": 23, + "interactiveNodeCount": 23, + "foregroundApp": "com.microsoft.emmx", + "changedFromBefore": true, + "screenshotDiff": { + "changedFromBefore": true, + "changedPixelRatio": 0.15493023405349796 + } + } + } +} +``` + +Aggregate true with `screenshotDiff {changedFromBefore: true, +changedPixelRatio: 0.155}`. A post-run screenshot shows the canvas red with +`STATE=RED`, matching the pixel answer. + +## Notes + +- Backend: `android-helper` (snapshot helper 0.21.42-dev, built from this + tree); screenshots go through the helper's capture with + `rotationFacts(device, ...)` declaring the band. +- The tap at `(540, 638)` covers one non-blocking candidate (`e3` shape on + the first rig: the WebView only). On the second rig the canvas node is + not hittable and names no touch role, so the gate stays open there too. +- Battery reset and status bar restored after the runs; the rig server was + an unrelated loopback process, left running for reproduction. diff --git a/docs/evidence/ios-verify-screenshot-fallback-2026-11-20.md b/docs/evidence/ios-verify-screenshot-fallback-2026-11-20.md new file mode 100644 index 0000000000..cd794411e0 --- /dev/null +++ b/docs/evidence/ios-verify-screenshot-fallback-2026-11-20.md @@ -0,0 +1,115 @@ +# iOS `--verify` screenshot fallback on the custom-drawn canvas fixture, 2026-11-20 + +Evidence for #4297 at the gate-exemption head `28c408b94` (the only later +commit is this docs change). The runner fixture is the source-built +unit-test runner (`AGENT_DEVICE_XCUITEST_INCLUDE_UNIT_TESTS=1`), installed +on the iPhone 17 Pro Max simulator (`F8658A47-112D-48F1-8D18-226A74F39689`, +iOS 26.5, Xcode 27.2) and launched with `--agent-device-custom-drawn-canvas` +through `open --launch-args`. The fixture adds a 160x160 plain UIView the +accessibility tree never exposes: a tap toggles its fill between systemGray5 +and systemRed. + +## Pre-action tree: no hittable or semantic node covers the canvas point + +Full snapshot at the session app: + +```json +[ + {"index": 0, "type": "Application", "role": "UIApplication", "label": "AgentDeviceRunner", + "rect": {"x": 0, "y": 0, "width": 440, "height": 956}, "enabled": true, "depth": 0, + "pid": 92141, "hittable": false, "ref": "e1", "kind": "application"}, + {"index": 1, "parentIndex": 0, "type": "StaticText", "role": "UILabel", + "label": "Agent Device Runner", + "rect": {"x": 120.0, "y": 465, "width": 200.3, "height": 26.3}, "enabled": true, + "depth": 1, "pid": 92141, "hittable": true, "ref": "e2", "kind": "text"} +] +``` + +The canvas (roughly 140..300 x 611..771 pt, centered under the label) names +no element: the point `(220, 720)` is covered only by the Application node +itself (the viewport root, not hittable). The StaticText does not cover it. +The tree-coverage gate has no covering touch target to block on, so the +pixel pair is admitted and the screenshot fallback runs. + +## Run 1 — status-bar-only change between the frames + +`press 220 720 --verify --json` is a canvas tap; to make the only other +candidate change a status-bar glyph, the simulator's battery level was +forced to 37 about 500 ms into the press (`xcrun simctl status_bar +override --batteryLevel 37`) and cleared right after. The band (6% of the +portrait frame, declared by the apple capture owner) excludes that region, +so the forced change must not surface: + +```json +{ + "success": true, + "data": { + "x": 220, + "y": 720, + "message": "Tapped (220, 720)", + "runnerMainThreadBusy": false, + "targetKind": "point", + "evidence": { + "digest": "ax1:aaf591869124d41c", + "nodeCount": 3, + "interactiveNodeCount": 2, + "changedFromBefore": false, + "screenshotDiff": { + "changedFromBefore": false, + "changedPixelRatio": 0 + } + } + } +} +``` + +Aggregate `changedFromBefore` is false (no comparison verification saw a +change) and the pixel-only answer is `screenshotDiff {changedFromBefore: +false, changedPixelRatio: 0}` — the forced status-bar redraw landed inside +the excluded band and the outside-the-status-bar comparison saw an +identical frame. + +## Run 2 — a tap that redraws the canvas + +`press 220 720 --verify --json` again, this time with no status-bar +override. The tap lands on the custom-drawn view and toggles its fill: + +```json +{ + "success": true, + "data": { + "x": 220, + "y": 720, + "message": "Tapped (220, 720)", + "runnerMainThreadBusy": false, + "targetKind": "point", + "evidence": { + "digest": "ax1:aaf591869124d41c", + "nodeCount": 3, + "interactiveNodeCount": 2, + "changedFromBefore": true, + "screenshotDiff": { + "changedFromBefore": true, + "changedPixelRatio": 0.06512286378804732 + } + } + } +} +``` + +Aggregate true with `screenshotDiff {changedFromBefore: true, +changedPixelRatio: 0.0651}` — the canvas (160/440 x 160/956 ≈ 6.1% of the +frame) turned red, and the ratio matches that share. + +## Notes + +- Backend: XCTest runner (source build with the unit-test fixture define), + session app `com.callstack.agentdevice.runner` (pid 92141 at snapshot). +- The capture binding is the production one: the daemon's + `createVerifyScreenshotCapture` binds the screenshot lane for the session + (`captureScreenshotFromRequest` with the session surface, fullscreen for + a portrait iOS target) — no flag route, no test-injected capture. +- The iOS dead-region run at `(60, 820)` (left margin, no node under it) + with the same forced battery change read `changedFromBefore: false` with + `screenshotDiff {false, 0}` as well. +- Status bar cleared after the runs; the simulator was left booted. diff --git a/packages/capture-kit/package.json b/packages/capture-kit/package.json index 72e1aba544..f2f6c5ffc4 100644 --- a/packages/capture-kit/package.json +++ b/packages/capture-kit/package.json @@ -155,6 +155,10 @@ "types": "./src/png-rgb-difference.ts", "default": "./src/png-rgb-difference.ts" }, + "./png-changed-pixel-ratio": { + "types": "./src/png-changed-pixel-ratio.ts", + "default": "./src/png-changed-pixel-ratio.ts" + }, "./png-size": { "types": "./src/png-size.ts", "default": "./src/png-size.ts" diff --git a/packages/capture-kit/src/png-changed-pixel-ratio.test.ts b/packages/capture-kit/src/png-changed-pixel-ratio.test.ts index 24eb9b979e..1541b2f045 100644 --- a/packages/capture-kit/src/png-changed-pixel-ratio.test.ts +++ b/packages/capture-kit/src/png-changed-pixel-ratio.test.ts @@ -45,3 +45,24 @@ describe('computePngChangedPixelRatio', () => { ).toEqual({ status: 'dimension_mismatch' }); }); }); + +test('computePngChangedPixelRatio ignores the top band share when asked', () => { + const first = { width: 4, height: 4, data: new Uint8Array(4 * 4 * 4).fill(10) }; + const second = { width: 4, height: 4, data: new Uint8Array(4 * 4 * 4).fill(10) }; + // Second frame: top half (rows 0-1) jumps to 200, bottom half (rows 2-3) to 30. + for (let y = 0; y < 2; y += 1) { + for (let x = 0; x < 4; x += 1) second.data[(y * 4 + x) * 4] = 200; + } + for (let y = 2; y < 4; y += 1) { + for (let x = 0; x < 4; x += 1) second.data[(y * 4 + x) * 4] = 30; + } + + const full = computePngChangedPixelRatio(first, second); + expect(full).toEqual({ status: 'compared', changedPixelRatio: 1 }); + + // With the top half excluded, the ignored rows drop out of the comparison: + // the ratio counts the bottom half's changed pixels over that compared + // region, so every pixel the pair was compared over changed. + const half = computePngChangedPixelRatio(first, second, { ignoreTopHeightShare: 0.5 }); + expect(half).toEqual({ status: 'compared', changedPixelRatio: 1 }); +}); diff --git a/packages/capture-kit/src/png-changed-pixel-ratio.ts b/packages/capture-kit/src/png-changed-pixel-ratio.ts index f7b0fa0ebd..8ae92a004d 100644 --- a/packages/capture-kit/src/png-changed-pixel-ratio.ts +++ b/packages/capture-kit/src/png-changed-pixel-ratio.ts @@ -17,10 +17,18 @@ export type PngChangedPixelRatioResult = * * Dimensions must match. Callers comparing frames of different sizes are asking a different * question, and returning a made-up ratio would hide it. + * + * `ignoreTopHeightShare` cuts the top share of the frame out of the comparison entirely, for + * bands whose pixels move without the dispatch causing it — the device's own status bar ticks + * between the bracket's frames, and the readback must answer for what the dispatch changed, not + * for the wall clock. The ratio runs over the compared region: both the changed count and the + * denominator count the rows below the band, so the share is the share of the region the pair + * was actually compared over. */ export function computePngChangedPixelRatio( first: PngRgbImage, second: PngRgbImage, + options: Readonly<{ ignoreTopHeightShare?: number }> = {}, ): PngChangedPixelRatioResult { if (first.width !== second.width || first.height !== second.height) { return { status: 'dimension_mismatch' }; @@ -32,8 +40,29 @@ export function computePngChangedPixelRatio( return { status: 'dimension_mismatch' }; } + // Rows at or above the cut are outside the compared region on both sides. + const ignoredRows = + options.ignoreTopHeightShare && options.ignoreTopHeightShare > 0 + ? Math.min(first.height, Math.floor(first.height * options.ignoreTopHeightShare)) + : 0; + const comparedPixels = totalPixels - ignoredRows * first.width; + if (comparedPixels <= 0) return { status: 'compared', changedPixelRatio: 0 }; + + return { + status: 'compared', + changedPixelRatio: countChangedPixels(first, second, ignoredRows) / comparedPixels, + }; +} + +/** Counts pixels whose RGB differs outside the ignored top rows. */ +function countChangedPixels(first: PngRgbImage, second: PngRgbImage, ignoredRows: number): number { + const declaredBytes = first.width * first.height * 4; let changedPixels = 0; - for (let offset = 0; offset + 3 < first.data.length; offset += 4) { + for ( + let offset = ignoredRows * first.width * 4; + offset + 3 < declaredBytes && offset + 3 < first.data.length; + offset += 4 + ) { if ( first.data[offset] !== second.data[offset] || first.data[offset + 1] !== second.data[offset + 1] || @@ -42,6 +71,5 @@ export function computePngChangedPixelRatio( changedPixels += 1; } } - - return { status: 'compared', changedPixelRatio: changedPixels / totalPixels }; + return changedPixels; } diff --git a/packages/command-runtime/src/commands/interaction/runtime/__tests__/test-utils/index.ts b/packages/command-runtime/src/commands/interaction/runtime/__tests__/test-utils/index.ts index b965fe6f37..1bde55eae9 100644 --- a/packages/command-runtime/src/commands/interaction/runtime/__tests__/test-utils/index.ts +++ b/packages/command-runtime/src/commands/interaction/runtime/__tests__/test-utils/index.ts @@ -409,6 +409,7 @@ export function createInteractionDevice( overrides: Partial< Pick< AgentDeviceBackend, + | 'captureVerifyScreenshot' | 'captureSnapshot' | 'resolveGestureViewport' | 'tap' @@ -430,6 +431,7 @@ export function createInteractionDevice( ) { return createAgentDevice({ backend: { + captureVerifyScreenshot: overrides.captureVerifyScreenshot, platform: overrides.platform ?? 'ios', captureSnapshot: async (...args) => overrides.captureSnapshot ? await overrides.captureSnapshot(...args) : { snapshot }, diff --git a/packages/command-runtime/src/commands/interaction/runtime/interactions.test.ts b/packages/command-runtime/src/commands/interaction/runtime/interactions.test.ts index 6324caeb28..9a44a1ee25 100644 --- a/packages/command-runtime/src/commands/interaction/runtime/interactions.test.ts +++ b/packages/command-runtime/src/commands/interaction/runtime/interactions.test.ts @@ -910,6 +910,25 @@ test('runtime press plan refuses a non-ref target', async () => { ); }); +test('runtime press plan refuses --verify instead of dropping the pixel pair', async () => { + const device = createInteractionDevice(pinPadSnapshot(), { + tap: async () => ({ ok: true }), + }); + + await assert.rejects( + device.interactions.press(ref('@e1'), { + session: 'default', + refs: ['e1', 'e2'], + verify: true, + }), + (error: AppError) => { + assert.equal(error.code, 'INVALID_ARGS'); + assert.equal(error.details?.reason, 'verify_unsupported_on_ref_plan'); + return true; + }, + ); +}); + test('runtime press plan names the refused key and the pending keys when a member fails', async () => { const taps: Point[] = []; const device = createInteractionDevice(pinPadSnapshot(), { diff --git a/packages/command-runtime/src/commands/interaction/runtime/interactions.ts b/packages/command-runtime/src/commands/interaction/runtime/interactions.ts index c5c8da8a60..b04b6d1856 100644 --- a/packages/command-runtime/src/commands/interaction/runtime/interactions.ts +++ b/packages/command-runtime/src/commands/interaction/runtime/interactions.ts @@ -6,12 +6,20 @@ import type { RepeatedInput, ResolvedInteractionTarget, ResolvedTarget, + ScreenshotVerification, } from '@agent-device/contracts/interaction'; +import { isSemanticTouchTarget } from '@agent-device/selectors/touch-semantics'; +import { INTERACTION_ERROR_REASONS } from '@agent-device/selectors/interaction-error'; +import { containsPoint, rectContains } from '@agent-device/kernel/rect'; import { AppError } from '@agent-device/kernel/errors'; import { SELECTOR_PIPELINE_POLICIES } from '@agent-device/selectors/selector-pipeline-policy'; -import type { Point } from '@agent-device/kernel/snapshot'; +import type { Point, SnapshotNode } from '@agent-device/kernel/snapshot'; import type { AgentDeviceRuntime, CommandContext } from '@agent-device/contracts/runtime-contract'; -import { isFillableType } from '@agent-device/contracts/snapshot'; +import { + isFillableType, + isViewportRootNode, + resolveViewportRect, +} from '@agent-device/contracts/snapshot'; import { attachResolvedInteractionTarget } from '@agent-device/host-kit/interaction-outcome'; import { sleep, toBackendContext } from '../../runtime-common.ts'; import { toBackendResult, type RuntimeCommand } from '../../runtime-types.ts'; @@ -176,9 +184,12 @@ async function tapCommand( const atRest = await admitResolvedTarget(runtime, options, resolutionRequest, resolved); const point = requireResolvedPoint(atRest); const pointObservation = pointObservationOf(atRest); - let backendResult; - try { - backendResult = await runtime.backend.tap(toBackendContext(runtime, options), point, { + const dispatchTap = runtime.backend.tap; + const dispatch = () => + // The backend method keeps its receiver: an in-process backend method + // that reads `this` breaks under a bare call, and the adapter's closure + // exposes none. + dispatchTap.call(runtime.backend, toBackendContext(runtime, options), point, { button: options.button, count: options.count, intervalMs: options.intervalMs, @@ -187,6 +198,26 @@ async function tapCommand( doubleTap: options.doubleTap, ...(pointObservation ? { pointObservation } : {}), }); + let backendResult; + let screenshot: ScreenshotVerification | undefined; + try { + const baseline = atRest.preAction; + const useScreenshot = + observation.verify && + atRest.kind === 'point' && + baseline !== undefined && + runtime.backend.captureVerifyScreenshot !== undefined && + !baseline.nodes.some((node) => blocksPixelPairAtPoint(baseline.nodes, node, point)); + if (useScreenshot) { + const { withScreenshotVerification } = await import('./verify-screenshot.ts'); + ({ result: backendResult, screenshot } = await withScreenshotVerification( + runtime, + options, + dispatch, + )); + } else { + backendResult = await dispatch(); + } } catch (error) { // Resolution is complete before the backend call. Preserve it out of // band so a daemon-level failure corroboration can still record the same @@ -204,6 +235,7 @@ async function tapCommand( ...(formattedBackendResult ? { backendResult: formattedBackendResult } : {}), }, observation, + screenshot, ); } @@ -276,6 +308,15 @@ async function tapRefPlanCommand( // response identity, so the plan iterates `refs` alone. const plannedRefs = (options.refs ?? []).map((ref) => (ref.startsWith('@') ? ref : `@${ref}`)); const observation = planPostActionObservation(options); + // A plan member is a tree-visible ref, and the pixel fallback behind + // --verify answers one point the pre-action tree cannot see. Refuse the + // flag instead of silently dropping the pair it promises. + if (observation.verify) { + throw new AppError('INVALID_ARGS', 'A multi-ref press plan cannot carry --verify', { + reason: INTERACTION_ERROR_REASONS.verifyUnsupportedOnRefPlan, + hint: 'Run --verify on a single press; the plan already dispatches tree-visible refs.', + }); + } const members = await resolveRefPlanMembers( runtime, options, @@ -422,3 +463,31 @@ function formatTargetForWarning(result: { if (result.target?.kind === 'selector') return result.target.selector; return 'point'; } + +/** + * Whether a pre-action tree node keeps the coordinate pixel pair off at `point` + * (the `--verify` screenshot fallback's tree-coverage gate). + * + * A node blocks when it covers the point and is a semantic touch target, or is + * hittable and smaller than the point's resolved viewport. The exemption is + * the ruling on #4297 (option a): a hittable node whose rect CONTAINS the + * point's resolved viewport — `resolveViewportRect`, including its + * largest-containing-rect fallback, which is what answers on Android where no + * window nodes exist — is the viewport-filling container every real tree + * carries (Android content `FrameLayout`, iOS full-screen `CollectionView`), + * not a touch target; excluding it keeps the gate reachable without a + * per-platform type list. A full-screen marked button still blocks: semantic + * targets never take the exemption. + */ +function blocksPixelPairAtPoint( + baselineNodes: SnapshotNode[], + node: SnapshotNode, + point: Point, +): boolean { + if (!node.rect || !containsPoint(node.rect, point.x, point.y)) return false; + if (isViewportRootNode(node)) return false; + if (isSemanticTouchTarget(node)) return true; + if (node.hittable !== true) return false; + const viewport = resolveViewportRect(baselineNodes, node.rect); + return !rectContains(node.rect, viewport ?? node.rect); +} diff --git a/packages/command-runtime/src/commands/interaction/runtime/post-action-observation.ts b/packages/command-runtime/src/commands/interaction/runtime/post-action-observation.ts index 396ff5d52e..4dfaca063e 100644 --- a/packages/command-runtime/src/commands/interaction/runtime/post-action-observation.ts +++ b/packages/command-runtime/src/commands/interaction/runtime/post-action-observation.ts @@ -1,6 +1,7 @@ import type { AgentDeviceRuntime, CommandContext } from '@agent-device/contracts/runtime-contract'; import type { InteractionEvidence, + ScreenshotVerification, ResolvedInteractionTarget, SettleObservation, SettleParams, @@ -65,9 +66,28 @@ export async function applyPostActionObservation( resolved: ResolvedInteractionTarget, result: T, params: PostActionObservationPlan, + screenshot?: ScreenshotVerification, ): Promise> { const observed = await observeAfterInteraction(runtime, options, resolved, params); - return reconcileNonHittableHintWithEvidence({ ...result, ...observed }); + const evidence = + screenshot && observed.evidence + ? { + ...observed.evidence, + // Any comparison that verification made saw a change: the tree + // digest, the settle surface, or the pixel pair. The pixel-only + // answer stays in screenshotDiff beside it. + changedFromBefore: + observed.evidence.surfaceChange !== undefined || + observed.evidence.changedFromBefore === true || + screenshot.changedFromBefore, + screenshotDiff: screenshot, + } + : observed.evidence; + return reconcileNonHittableHintWithEvidence({ + ...result, + ...observed, + ...(evidence ? { evidence } : {}), + }); } /** diff --git a/packages/command-runtime/src/commands/interaction/runtime/verify-screenshot.test.ts b/packages/command-runtime/src/commands/interaction/runtime/verify-screenshot.test.ts new file mode 100644 index 0000000000..1b377a1d75 --- /dev/null +++ b/packages/command-runtime/src/commands/interaction/runtime/verify-screenshot.test.ts @@ -0,0 +1,310 @@ +import assert from 'node:assert/strict'; +import { test } from 'vitest'; +import { PNG } from '@agent-device/capture-kit/png'; +import type { RawSnapshotNode } from '@agent-device/kernel/snapshot'; +import { createLocalArtifactAdapter } from '@agent-device/host-kit/local-artifact-adapter'; +import { localCommandPolicy } from '@agent-device/contracts/command-policy'; +import { createMemorySessionStore } from '../../../command-runtime/runtime-factory.ts'; +import { createAgentDevice } from '../../../runtime.ts'; +import type { AgentDeviceBackend } from '@agent-device/contracts/backend'; +import { makeSnapshotState } from '@agent-device/selectors/snapshot-geometry-fixtures'; +import { createInteractionDevice, selectorSnapshot } from './__tests__/test-utils/index.ts'; + +test('coordinate verification keeps a changed tree digest when the frames match', async () => { + const { writeFile } = await import('node:fs/promises'); + let tapped = false; + const device = createInteractionDevice(selectorSnapshot(), { + // The tree digest moves (a navigation replaced the content) while the + // pixels at the settle instant still match. + captureSnapshot: async () => { + if (!tapped) return { snapshot: selectorSnapshot() }; + return { + snapshot: makeSnapshotState([ + { + index: 0, + depth: 0, + type: 'Application', + rect: { x: 0, y: 0, width: 400, height: 800 }, + hittable: true, + }, + ]), + }; + }, + tap: async () => { + tapped = true; + }, + clock: { now: () => 0, sleep: async () => {} }, + captureVerifyScreenshot: async (outPath) => { + const png = new PNG({ width: 4, height: 4 }); + png.data.fill(255); + await writeFile(outPath, PNG.sync.write(png)); + return { ignoreTopHeightShare: 0.25 }; + }, + }); + + const result = await device.interactions.press( + { kind: 'point', x: 200, y: 25 }, + { verify: true }, + ); + + // Any comparison verification made saw a change: the digest answer survives + // identical frames, and the pixel-only answer stays in screenshotDiff. + assert.equal(result.evidence?.changedFromBefore, true); + assert.equal(result.evidence?.screenshotDiff?.changedFromBefore, false); +}); + +test.each([ + { x: 15, verify: true, frames: [], changed: false }, + { x: 200, verify: false, frames: [], changed: undefined }, + { x: 200, verify: true, frames: ['before', 'after'], changed: false }, + { x: 200, verify: true, frames: ['before', 'after'], changed: true }, +])( + 'coordinate verification brackets dispatch only away from a touch target ($x, $verify, $changed)', + async ({ x, verify, frames, changed }) => { + const { writeFile } = await import('node:fs/promises'); + const captures: string[] = []; + let tapped = false; + const device = createInteractionDevice(selectorSnapshot(), { + tap: async () => { + tapped = true; + }, + clock: { now: () => 0, sleep: async () => {} }, + captureVerifyScreenshot: async (outPath) => { + captures.push(tapped ? 'after' : 'before'); + const png = new PNG({ width: 4, height: 4 }); + png.data.fill(255); + if (tapped) png.data[0] = 0; // Status-bar-only change is ignored. + if (tapped && changed) png.data[16] = 0; + await writeFile(outPath, PNG.sync.write(png)); + return { ignoreTopHeightShare: 0.25 }; + }, + }); + const result = await device.interactions.press({ kind: 'point', x, y: 25 }, { verify }); + assert.deepEqual(captures, frames); + assert.equal(result.evidence?.changedFromBefore, changed); + assert.equal( + result.evidence?.screenshotDiff?.changedFromBefore, + frames.length ? changed : undefined, + ); + }, +); + +test('the coordinate dispatch keeps the backend receiver for methods that read this', async () => { + // An in-process backend whose tap is a real method: the dispatch must invoke + // it with the backend as its receiver, the way a bare call would not. + const taps: Array<{ receiver: unknown; point: { x: number; y: number } }> = []; + const backend: AgentDeviceBackend = { + platform: 'ios', + captureSnapshot: async () => ({ snapshot: selectorSnapshot() }), + async tap(context, point) { + taps.push({ receiver: this, point }); + void context; + return { ok: true }; + }, + } satisfies AgentDeviceBackend; + const device = createAgentDevice({ + backend, + artifacts: createLocalArtifactAdapter(), + sessions: createMemorySessionStore([{ name: 'default', snapshot: selectorSnapshot() }]), + policy: localCommandPolicy(), + }); + + const result = await device.interactions.press( + { kind: 'point', x: 200, y: 25 }, + { verify: false }, + ); + + assert.equal(result.kind, 'point'); + assert.equal(taps.length, 1); + assert.equal(taps[0]?.receiver, backend); + assert.deepEqual(taps[0]?.point, { x: 200, y: 25 }); +}); + +// The tree-coverage gate admits the pixel pair only when no covering node is a +// semantic touch target, or hittable and smaller than the point's resolved +// viewport. The ruling on #4297 (option a): the viewport-filling hittable +// container every real tree carries is exempt, so the gate stays reachable — +// the four cases below are the maintainer's list. +function node(overrides: { + index: number; + type: string; + rect: { x: number; y: number; width: number; height: number }; + hittable?: boolean; + parentIndex?: number; + label?: string; +}): RawSnapshotNode { + return { ...overrides }; +} + +async function pressWithGateTree( + nodes: RawSnapshotNode[], + point: { x: number; y: number }, +): Promise<{ pairRan: boolean }> { + const { writeFile } = await import('node:fs/promises'); + let pairRan = false; + let tapped = false; + const device = createInteractionDevice(makeSnapshotState(nodes), { + tap: async () => { + tapped = true; + }, + clock: { now: () => 0, sleep: async () => {} }, + captureVerifyScreenshot: async (outPath) => { + pairRan = true; + const png = new PNG({ width: 4, height: 4 }); + png.data.fill(255); + if (tapped) png.data[16] = 0; + await writeFile(outPath, PNG.sync.write(png)); + return { ignoreTopHeightShare: 0.25 }; + }, + }); + const result = await device.interactions.press({ kind: 'point', ...point }, { verify: true }); + assert.ok(result.evidence); + return { pairRan }; +} + +test('gate admits the pair past a full-screen hittable container that IS the viewport', async () => { + // Android content FrameLayout shape: no window node, so the resolved + // viewport is the largest-containing-rect fallback — the container itself. + const { pairRan } = await pressWithGateTree( + [ + node({ + index: 0, + type: 'FrameLayout', + rect: { x: 0, y: 94, width: 1080, height: 2306 }, + hittable: true, + }), + node({ index: 1, type: 'Text', rect: { x: 40, y: 300, width: 400, height: 40 } }), + ], + { x: 540, y: 1200 }, + ); + assert.equal(pairRan, true); +}); + +test('gate admits the iOS full-screen CollectionView and still blocks its covering cells', async () => { + // iOS Settings shape: the full-screen CollectionView is hittable and IS the + // viewport (exempt), so a point only it covers opens the pair; the Cell and + // Button under the same point are smaller than the viewport, so they keep + // blocking and the pair never runs. + const collectionView = node({ + index: 0, + type: 'CollectionView', + rect: { x: 0, y: 0, width: 440, height: 956 }, + hittable: true, + }); + const windowNode = node({ + index: 2, + type: 'Window', + rect: { x: 0, y: 0, width: 440, height: 956 }, + }); + const opened = await pressWithGateTree([collectionView, windowNode], { x: 200, y: 700 }); + assert.equal(opened.pairRan, true); + + const cellUnderThePoint = node({ + index: 1, + type: 'Cell', + rect: { x: 0, y: 660, width: 440, height: 80 }, + hittable: true, + parentIndex: 0, + label: 'General', + }); + const blocked = await pressWithGateTree([collectionView, cellUnderThePoint, windowNode], { + x: 200, + y: 700, + }); + assert.equal(blocked.pairRan, false); +}); + +test('gate keeps a full-screen semantic touch target blocking', async () => { + const { pairRan } = await pressWithGateTree( + [ + node({ + index: 0, + type: 'Button', + rect: { x: 0, y: 0, width: 1080, height: 2400 }, + hittable: true, + label: 'Continue', + }), + ], + { x: 540, y: 1200 }, + ); + assert.equal(pairRan, false); +}); + +test('gate exempts a hittable scrim whose rect contains the whole viewport', async () => { + // The ruling's literal shape: the exempting direction is "the node's rect + // contains the resolved viewport", so a full-screen scrim drawn beyond the + // window bounds (oversized during a scale animation) still opens the pair. + const { pairRan } = await pressWithGateTree( + [ + node({ index: 0, type: 'Window', rect: { x: 0, y: 0, width: 440, height: 956 } }), + node({ + index: 1, + type: 'View', + rect: { x: -100, y: -100, width: 640, height: 1156 }, + hittable: true, + }), + ], + { x: 200, y: 700 }, + ); + assert.equal(pairRan, true); +}); + +test('gate keeps a smaller hittable node under the point inside the container blocking', async () => { + const { pairRan } = await pressWithGateTree( + [ + node({ + index: 0, + type: 'FrameLayout', + rect: { x: 0, y: 94, width: 1080, height: 2306 }, + hittable: true, + }), + node({ + index: 1, + type: 'View', + rect: { x: 400, y: 1100, width: 280, height: 200 }, + hittable: true, + parentIndex: 0, + }), + ], + { x: 540, y: 1200 }, + ); + assert.equal(pairRan, false); +}); + +test('gate admits the pair on the Redmi K60 Edge tree through the largest-containing-rect fallback', async () => { + // The captured Edge tree (android-helper, #4297 2026-11-19): no window + // nodes and no root node (Android emits neither), Java class names + // isViewportRootNode never matches, and the content-root FrameLayout + // 0,94 1080x2306 is hittable — the largest containing rect, so it IS the + // resolved viewport. Under the old gate every point on this tree closed + // the pixel pair. + const edgeTreeNodes: RawSnapshotNode[] = [ + node({ + index: 0, + type: 'android.widget.FrameLayout', + rect: { x: 0, y: 94, width: 1080, height: 2306 }, + hittable: true, + }), + node({ + index: 1, + type: 'android.view.View', + rect: { x: 0, y: 94, width: 1080, height: 120 }, + parentIndex: 0, + }), + node({ + index: 2, + type: 'android.widget.EditText', + rect: { x: 40, y: 140, width: 600, height: 60 }, + parentIndex: 0, + label: 'Search or type URL', + }), + node({ + index: 3, + type: 'android.view.View', + rect: { x: 0, y: 260, width: 1080, height: 2140 }, + parentIndex: 0, + }), + ]; + const { pairRan } = await pressWithGateTree(edgeTreeNodes, { x: 540, y: 1200 }); + assert.equal(pairRan, true); +}); diff --git a/packages/command-runtime/src/commands/interaction/runtime/verify-screenshot.ts b/packages/command-runtime/src/commands/interaction/runtime/verify-screenshot.ts new file mode 100644 index 0000000000..f5b90caef0 --- /dev/null +++ b/packages/command-runtime/src/commands/interaction/runtime/verify-screenshot.ts @@ -0,0 +1,78 @@ +import { readFile } from 'node:fs/promises'; +import type { TemporaryFile } from '@agent-device/contracts/artifact-adapter'; +import type { ScreenshotVerification } from '@agent-device/contracts/interaction'; +import type { AgentDeviceRuntime, CommandContext } from '@agent-device/contracts/runtime-contract'; +import { createCommandTempFile } from '../../io-policy.ts'; +import { sleep } from '../../runtime-common.ts'; + +export async function withScreenshotVerification( + runtime: AgentDeviceRuntime, + options: CommandContext, + dispatch: () => Promise, +): Promise<{ result: T; screenshot?: ScreenshotVerification }> { + const signal = options.signal ?? runtime.signal; + const before = await captureFrame(runtime, signal); + try { + signal?.throwIfAborted(); + const result = await dispatch(); + if (!before) return { result }; + // Give the tap's first visual update time to reach the display. + await sleep(runtime, VERIFY_AFTER_FRAME_SETTLE_MS, signal).catch(() => {}); + const after = await captureFrame(runtime, signal); + if (!after) return { result }; + try { + const screenshot = await compareFrames(before, after); + return signal?.aborted ? { result } : { result, screenshot }; + } catch { + return { result }; + } finally { + await after.file.cleanup().catch(() => {}); + } + } finally { + await before?.file.cleanup().catch(() => {}); + } +} + +async function captureFrame(runtime: AgentDeviceRuntime, signal: AbortSignal | undefined) { + if (signal?.aborted || !runtime.backend.captureVerifyScreenshot) return; + let file: TemporaryFile | undefined; + try { + file = await createCommandTempFile(runtime, { prefix: 'agent-device-verify', ext: '.png' }); + if (!signal?.aborted) { + const facts = await runtime.backend.captureVerifyScreenshot(file.path); + if (facts && !signal?.aborted) return { file, ...facts }; + } + } catch { + // An unavailable frame leaves tree verification in charge. + } + await file?.cleanup().catch(() => {}); +} + +/** The after frame waits this long past the dispatch reply before it is captured. */ +const VERIFY_AFTER_FRAME_SETTLE_MS = 150; + +async function compareFrames( + before: { file: TemporaryFile; ignoreTopHeightShare: number }, + after: { file: TemporaryFile; ignoreTopHeightShare: number }, +): Promise { + const [{ computePngChangedPixelRatio }, { decodePngAsync }] = await Promise.all([ + import('@agent-device/capture-kit/png-changed-pixel-ratio'), + import('@agent-device/capture-kit/png-worker-client'), + ]); + const [first, second] = await Promise.all([ + decodePngAsync(await readFile(before.file.path), 'verification before frame'), + decodePngAsync(await readFile(after.file.path), 'verification after frame'), + ]); + const comparison = computePngChangedPixelRatio(first, second, { + ignoreTopHeightShare: + first.height >= first.width + ? Math.min(before.ignoreTopHeightShare, after.ignoreTopHeightShare) + : 0, + }); + return comparison.status === 'dimension_mismatch' + ? { changedFromBefore: true } + : { + changedFromBefore: comparison.changedPixelRatio > 0, + changedPixelRatio: comparison.changedPixelRatio, + }; +} diff --git a/packages/command-surface/src/interaction/index.ts b/packages/command-surface/src/interaction/index.ts index e3d01be4ba..45e9a94c0f 100644 --- a/packages/command-surface/src/interaction/index.ts +++ b/packages/command-surface/src/interaction/index.ts @@ -75,9 +75,20 @@ const interactionEvidenceSchema: JsonSchema = objectSchema( interactiveNodeCount: numberSchema('Subset of nodeCount the platform reports as hittable.'), digest: stringSchema('Order-independent digest of the post-action node multiset.'), changedFromBefore: booleanSchema( - 'Whether the post-action digest differs from the pre-action capture digest. false is evidence, not failure. Absent when no pre-action baseline was captured, so no comparison is claimed. With surfaceChange present, no digest comparison is made: it reports that surface transition.', + 'Whether verification observed a change. false is evidence, not failure. Absent when no pre-action baseline was captured, so no comparison is claimed. With surfaceChange present, no digest comparison is made: it reports that surface transition.', ), surfaceChange: postActionSurfaceChangeSchema, + screenshotDiff: objectSchema( + { + changedFromBefore: booleanSchema( + 'Whether the frame dimensions differed or the compared pixels differed; changedPixelRatio is absent on a dimension change.', + ), + changedPixelRatio: numberSchema( + 'Share of compared pixels that changed, when frame dimensions match.', + ), + }, + ['changedFromBefore'], + ), }, ['nodeCount', 'interactiveNodeCount', 'digest'], ); @@ -498,7 +509,7 @@ const pressCommandFacet = defineCommandFacet({ summary: 'Short-press a UI target', searchHint: 'click touch button', cliDetail: - 'The hold duration is positional on longpress, not press --hold-ms. A comma-separated ref plan (press @e13,@e14,@e15,@e16; 2 to 16 refs, repeats allowed) resolves and admits every key against the current frame, then dispatches the admitted points: key 1 expires the frame, later keys dispatch where the snapshot showed them; a later refusal carries details.dispatchedSteps, details.refPlanKey, and details.refPlanPending, and sequence.points holds the dispatched points. --interval-ms sleeps between keys; --count, --hold-ms, --jitter-px, and --double-tap apply to every key; the response ref is the first key. Recording refuses the plan.', + 'The hold duration is positional on longpress, not press --hold-ms. A comma-separated ref plan (press @e13,@e14,@e15,@e16; 2 to 16 refs, repeats allowed) resolves and admits every key against the current frame, then dispatches the admitted points: key 1 expires the frame, later keys dispatch where the snapshot showed them; a later refusal carries details.dispatchedSteps, details.refPlanKey, and details.refPlanPending, and sequence.points holds the dispatched points. --interval-ms sleeps between keys; --count, --hold-ms, --jitter-px, and --double-tap apply to every key; the response ref is the first key. --verify refuses a ref plan; run it on a single press. Recording refuses the plan.', }, metadata: metadata('press'), run: (client, input) => client.interactions.press(toPressOptions(input)), diff --git a/packages/command-surface/src/interaction/metadata.ts b/packages/command-surface/src/interaction/metadata.ts index 68c04f406d..3979d7c5ee 100644 --- a/packages/command-surface/src/interaction/metadata.ts +++ b/packages/command-surface/src/interaction/metadata.ts @@ -65,9 +65,9 @@ const FIND_ACTION_VALUES = [ const interactionCommandDescriptions = { click: - "Activate a UI target by snapshot ref, selector, or coordinates. This can change app state. Success reports dispatch only; settle returns the resulting UI diff and verify returns cheap post-action evidence. A ref or selector target is re-checked for motion before dispatch when the session's previous action just animated the screen, and is dispatched at its resting position; one that has not stopped refuses with target_still_moving rather than acting where it was. With settle, continue from the returned diff. Copy refs exactly, including any ~sN suffix.", + 'Activate a UI target by ref, selector, or coordinates. This can change app state. Success reports dispatch only; settle returns the resulting UI diff; verify compares screenshots outside accessible controls. A ref or selector target re-checked for motion before dispatch after the last action animated the screen dispatches at its resting position; one still moving refuses with target_still_moving rather than acting where it was. With settle, continue from the returned diff. Copy refs exactly, including any ~sN suffix.', press: - "Short-press a UI target by snapshot ref, selector, or coordinates. Success reports dispatch only; settle returns the resulting UI diff and verify returns cheap post-action evidence. A ref or selector target is re-checked for motion before dispatch when the session's previous action just animated the screen, and is dispatched at its resting position; one that has not stopped refuses with target_still_moving rather than acting where it was. A comma-separated ref list dispatches 2-16 keys in order as one command, admitted before the first press.", + 'Short-press a UI target by ref, selector, or coordinates. Success reports dispatch only; settle returns the resulting UI diff; verify compares screenshots outside accessible controls. A ref or selector target re-checked for motion before dispatch after the last action animated the screen dispatches at its resting position; one still moving refuses with target_still_moving rather than acting where it was. A comma-separated ref list dispatches 2-16 keys as one command, admitted before the first press. With settle, continue from the returned diff.', fill: 'Replace text in a UI input selected by snapshot ref, selector, or coordinates. An empty text clears the field. Unconfirmed evidence does not establish the resulting value or screen. The recordAs option keeps sensitive text out of a recorded replay while sending it to the live app. To append instead of replacing, use type.', longpress: 'Hold a UI target by snapshot ref, selector, or coordinates to open a context menu or perform another hold gesture. The durationMs input overrides the default hold duration.', diff --git a/packages/contracts/src/backend.ts b/packages/contracts/src/backend.ts index d57fff756e..e0008d3deb 100644 --- a/packages/contracts/src/backend.ts +++ b/packages/contracts/src/backend.ts @@ -405,6 +405,8 @@ export type AgentDeviceBackend = { outPath: string, options?: BackendScreenshotOptions, ): Promise; + /** Request-bound frame for coordinate verification, with the comparison's status-bar exclusion. */ + captureVerifyScreenshot?(outPath: string): Promise<{ ignoreTopHeightShare: number } | undefined>; readText?(context: BackendCommandContext, node: SnapshotNode): Promise; /** Present only when the bound runtime advertised the conditional `findText` operation. */ findText?(context: BackendCommandContext, text: string): Promise; diff --git a/packages/contracts/src/interaction-guarantees.ts b/packages/contracts/src/interaction-guarantees.ts index 586afe579c..ea54c7878d 100644 --- a/packages/contracts/src/interaction-guarantees.ts +++ b/packages/contracts/src/interaction-guarantees.ts @@ -526,7 +526,8 @@ export const INTERACTION_DISPATCH_PATHS: Record { @@ -213,7 +215,10 @@ function resolveBoundCapture( function createInteractionAgentDevice(params: InteractionRuntimeInput) { return params.createCommandSurface({ backend: createInteractionBackend(params), - ...createDaemonRuntimePolicy('interaction commands', { plural: true }), + ...createDaemonRuntimePolicy('interaction commands', { + plural: true, + allowTempFiles: params.readbackCapture !== undefined, + }), sessions: params.runtimeSessions, signal: getRequestSignal(params.requestId), }); @@ -225,6 +230,7 @@ function createInteractionBackend(params: InteractionRuntimeInput): AgentDeviceB params.contextFromFlags(flags, session.appBundleId, session.trace?.outPath); return { platform: publicPlatformString(session.device), + ...(params.readbackCapture ? { captureVerifyScreenshot: params.readbackCapture } : {}), captureSnapshot: async (context, options): Promise => ({ snapshot: await params.captureSnapshot(flags, { interactiveOnly: options?.interactiveOnly === true, diff --git a/packages/daemon/src/execution/interaction/internal/interaction-touch-press.ts b/packages/daemon/src/execution/interaction/internal/interaction-touch-press.ts index 0e6372f1ac..3ae3ee46e5 100644 --- a/packages/daemon/src/execution/interaction/internal/interaction-touch-press.ts +++ b/packages/daemon/src/execution/interaction/internal/interaction-touch-press.ts @@ -10,6 +10,11 @@ import type { resolveClickButton } from '@agent-device/contracts/click-button'; import type { ReplayTargetGuardDenotation } from '@agent-device/contracts/replay'; import { assertAndroidPressStayedInApp } from './interaction-android-escape.ts'; import { createInteractionRuntimeForRoute } from './interaction-runtime.ts'; +import { resolveBoundScreenshotRuntime } from '../../screenshot-runtime-binding.ts'; +import { captureScreenshotFromRequest } from '../../screenshot-runtime.ts'; +import type { AgentDeviceBackend } from '@agent-device/contracts/backend'; +import type { DeviceRotation } from '@agent-device/contracts/device'; +import type { DeviceInfo } from '@agent-device/kernel/device'; import { readSettleRequest } from './interaction-flags.ts'; import { dispatchDirectIosSelectorTap } from './interaction-touch-direct-ios.ts'; import { readDirectIosSelectorTapTarget } from '../../../core/interaction/internal/interaction-touch-direct-ios-eligibility.ts'; @@ -86,9 +91,55 @@ export async function dispatchTargetedTouchViaRuntime( ...buildTargetedRuntimeOptions(params, command, boundAdmitted), touchExecutor, settleObservationCapture: prepared.settleObservationCapture, + ...(admitted.target.kind === 'point' && + (command === 'press' || command === 'click') && + params.req.flags?.verify === true + ? { readbackCapture: createVerifyScreenshotCapture(params, admitted.session) } + : {}), }); } +function createVerifyScreenshotCapture( + params: TargetedTouchParams, + session: AdmittedTargetedTouch['session'], +): NonNullable { + // Bind lazily. A verified tap with a node at the point never needs screenshot authority. + let binding: ReturnType | undefined; + return async (outPath) => { + const capture = await (binding ??= resolveBoundScreenshotRuntime({ + ...params, + device: session.device, + overlayRefs: false, + })); + if (!capture.ok) return undefined; + const dispatchContext = params.contextFromFlags( + params.req.flags, + session.appBundleId, + session.trace?.outPath, + ); + const facts = await captureScreenshotFromRequest( + { session, dispatchContext, captureScreenshot: capture.runtime.captureScreenshot }, + outPath, + { + surface: session.surface, + fullscreen: verifyStatusBarBand(session.device, undefined) > 0 ? true : undefined, + normalizeStatusBar: false, + }, + ); + return { ignoreTopHeightShare: verifyStatusBarBand(session.device, facts.displayRotation) }; + }; +} + +function verifyStatusBarBand(device: DeviceInfo, rotation: DeviceRotation | undefined): number { + if (rotation?.startsWith('landscape') || device.target === 'tv' || device.target === 'desktop') { + return 0; + } + if (device.appleOs === 'ipados') return 0.02; + if (device.appleOs === 'ios') return 0.06; + if (device.platform === 'android' || device.platform === 'harmonyos') return 0.04; + return 0; +} + function buildTargetedRuntimeOptions( params: TargetedTouchParams, command: TargetedTouchCommand, diff --git a/packages/daemon/src/execution/interaction/internal/interaction-touch-runtime.ts b/packages/daemon/src/execution/interaction/internal/interaction-touch-runtime.ts index d2ffe838ec..03284e0a15 100644 --- a/packages/daemon/src/execution/interaction/internal/interaction-touch-runtime.ts +++ b/packages/daemon/src/execution/interaction/internal/interaction-touch-runtime.ts @@ -1,3 +1,4 @@ +import type { AgentDeviceBackend } from '@agent-device/contracts/backend'; import type { DaemonResponse } from '@agent-device/kernel/contracts'; import { bindInteractionSession } from './interaction-session.ts'; import type { @@ -54,6 +55,7 @@ export async function dispatchRuntimeInteraction< touchExecutor: BoundTouchExecutor; /** The request's own observation capture when its action use carries none (#3510). */ settleObservationCapture?: BoundSnapshotCapture; + readbackCapture?: AgentDeviceBackend['captureVerifyScreenshot']; androidFreshnessBaseline?: SessionState['snapshot']; /** * Present when the action targets a `@ref`: if Android dialog recovery @@ -82,6 +84,7 @@ export async function dispatchRuntimeInteraction< ...params, touchExecutor: options.touchExecutor, settleObservationCapture: options.settleObservationCapture, + readbackCapture: options.readbackCapture, consumedCapture, }); const actionStartedAt = Date.now(); diff --git a/packages/daemon/src/execution/runtime-artifacts.ts b/packages/daemon/src/execution/runtime-artifacts.ts index dc154f67f9..8faf9d92d0 100644 --- a/packages/daemon/src/execution/runtime-artifacts.ts +++ b/packages/daemon/src/execution/runtime-artifacts.ts @@ -1,6 +1,29 @@ -import type { ArtifactAdapter } from '@agent-device/contracts/artifact-adapter'; +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import type { ArtifactAdapter, TemporaryFile } from '@agent-device/contracts/artifact-adapter'; import { AppError } from '@agent-device/kernel/errors'; +/** + * The one temp-file implementation every daemon route shares: an `agent-device-*` root under the + * process tmpdir, one file inside, removed with the cleanup. Routes whose policy refuses temp + * files keep the refusing adapter; routes that opt in (runtime policy's `allowTempFiles`, the + * screenshot artifact adapter) get this implementation. + */ +export async function createDaemonTempFile(options: { + prefix: string; + ext: string; +}): Promise { + const root = await fs.mkdtemp(path.join(os.tmpdir(), `${options.prefix}-`)); + return { + path: path.join(root, `file${options.ext}`), + visibility: 'internal', + cleanup: async () => { + await fs.rm(root, { recursive: true, force: true }); + }, + }; +} + export function createUnsupportedArtifactAdapter( label: string, options: { plural?: boolean } = {}, diff --git a/packages/daemon/src/execution/runtime-policy.ts b/packages/daemon/src/execution/runtime-policy.ts index 0d85d3ac11..312fce86b5 100644 --- a/packages/daemon/src/execution/runtime-policy.ts +++ b/packages/daemon/src/execution/runtime-policy.ts @@ -1,13 +1,18 @@ import type { AgentDeviceRuntimeConfig } from '@agent-device/contracts/runtime-contract'; import { localCommandPolicy } from '@agent-device/contracts/command-policy'; -import { createUnsupportedArtifactAdapter } from './runtime-artifacts.ts'; +import { createDaemonTempFile, createUnsupportedArtifactAdapter } from './runtime-artifacts.ts'; export function createDaemonRuntimePolicy( unsupportedArtifactLabel: string, - options: { plural?: boolean } = {}, + options: { plural?: boolean; allowTempFiles?: boolean } = {}, ): Pick { return { - artifacts: createUnsupportedArtifactAdapter(unsupportedArtifactLabel, options), + artifacts: options.allowTempFiles + ? { + ...createUnsupportedArtifactAdapter(unsupportedArtifactLabel, options), + createTempFile: createDaemonTempFile, + } + : createUnsupportedArtifactAdapter(unsupportedArtifactLabel, options), policy: localCommandPolicy(), }; } diff --git a/packages/daemon/src/execution/screenshot-artifact-adapter.ts b/packages/daemon/src/execution/screenshot-artifact-adapter.ts index 43086e8d61..765fc1ebc7 100644 --- a/packages/daemon/src/execution/screenshot-artifact-adapter.ts +++ b/packages/daemon/src/execution/screenshot-artifact-adapter.ts @@ -2,6 +2,7 @@ import { publishFileSync } from '@agent-device/host-kit/file'; import { emitDiagnostic } from '@agent-device/host-kit/diagnostics'; import type { ArtifactAdapter, ReservedOutputFile } from '@agent-device/contracts/artifact-adapter'; import { AppError, errorMessage } from '@agent-device/kernel/errors'; +import { createDaemonTempFile } from './runtime-artifacts.ts'; import { promises as fs } from 'node:fs'; import os from 'node:os'; import path from 'node:path'; @@ -51,16 +52,7 @@ export function createDaemonScreenshotArtifactAdapter( ); } }, - createTempFile: async (options) => { - const root = await fs.mkdtemp(path.join(os.tmpdir(), `${options.prefix}-`)); - return { - path: path.join(root, `file${options.ext}`), - visibility: 'internal', - cleanup: async () => { - await fs.rm(root, { recursive: true, force: true }); - }, - }; - }, + createTempFile: createDaemonTempFile, }; } diff --git a/packages/daemon/src/execution/screenshot-runtime.ts b/packages/daemon/src/execution/screenshot-runtime.ts index ab094426c2..3984f28976 100644 --- a/packages/daemon/src/execution/screenshot-runtime.ts +++ b/packages/daemon/src/execution/screenshot-runtime.ts @@ -8,11 +8,14 @@ import { screenshotOptionsFromFlags, } from '@agent-device/contracts/capture'; import type { DeviceRotation } from '@agent-device/contracts/device'; -import type { ScreenshotRuntimeExecution } from '@agent-device/contracts/screenshot-runtime'; +import type { + ScreenshotCaptureFacts, + ScreenshotRuntimeExecution, +} from '@agent-device/contracts/screenshot-runtime'; import { isIosFamily, publicPlatformString } from '@agent-device/kernel/device'; import { AppError } from '@agent-device/kernel/errors'; import type { ScreenshotOverlayRef } from '@agent-device/kernel/snapshot'; -import type { AgentDeviceBackend } from '@agent-device/contracts/backend'; +import type { AgentDeviceBackend, BackendScreenshotOptions } from '@agent-device/contracts/backend'; import { localCommandPolicy } from '@agent-device/contracts/command-policy'; import type { CreateDaemonCommandSurface } from '@agent-device/contracts/daemon-command-surface'; import { @@ -369,34 +372,11 @@ function createBoundScreenshotBackend( crop?: ScreenshotCropBinding; }>, ): AgentDeviceBackend { - const { session, dispatchContext, captureScreenshot, crop } = params; + const { session, dispatchContext, crop } = params; return { platform: publicPlatformString(session.device), captureScreenshot: async (_context, outPath, options) => { - const resolved = screenshotOptionsFromFlags({ - ...dispatchContext, - ...screenshotFlagsFromOptions(options), - }); - const facts = await captureScreenshot({ - outPath, - options: { - appBundleId: dispatchContext.appBundleId, - pixelDensity: resolved.pixelDensity, - screenMask: resolved.screenMask, - fullscreen: resolved.fullscreen, - normalizeStatusBar: resolved.normalizeStatusBar, - stabilize: resolved.stabilize, - surface: options?.surface, - // An open session already proved the simulator is booted, so the per-capture recheck is - // the daemon's own knowledge rather than something the caller has to assert. - skipIosSimulatorBootCheck: - dispatchContext.skipIosSimulatorBootCheck ?? - (isIosFamily(session.device) && session.device.kind === 'simulator'), - captureBackend: resolved.captureBackend, - preferredCaptureBackend: dispatchContext.preferredScreenshotCaptureBackend, - }, - execution: screenshotExecutionFromContext(dispatchContext), - }); + const facts = await captureScreenshotFromRequest(params, outPath, options); if (crop) { assertScreenshotCropCapture(session.device, facts); const outcome = await cropScreenshotToSelector({ @@ -415,3 +395,45 @@ function createBoundScreenshotBackend( }, }; } + +/** + * The one builder for a capture backed by this request: the screenshot backend and the press + * readback both capture through it, so their frames come from exactly the same input — the + * session's app identity, the resolved screenshot flags, the surface, and the runner execution + * from the dispatch context. + */ +export async function captureScreenshotFromRequest( + params: Readonly<{ + session: SessionState; + dispatchContext: DaemonCommandContext; + captureScreenshot: BoundScreenshotRuntime['captureScreenshot']; + }>, + outPath: string, + options?: BackendScreenshotOptions, +): Promise { + const { session, dispatchContext, captureScreenshot } = params; + const resolved = screenshotOptionsFromFlags({ + ...dispatchContext, + ...screenshotFlagsFromOptions(options), + }); + return await captureScreenshot({ + outPath, + options: { + appBundleId: dispatchContext.appBundleId, + pixelDensity: resolved.pixelDensity, + screenMask: resolved.screenMask, + fullscreen: resolved.fullscreen, + normalizeStatusBar: resolved.normalizeStatusBar, + stabilize: resolved.stabilize, + surface: options?.surface, + // An open session already proved the simulator is booted, so the per-capture recheck is + // the daemon's own knowledge rather than something the caller has to assert. + skipIosSimulatorBootCheck: + dispatchContext.skipIosSimulatorBootCheck ?? + (isIosFamily(session.device) && session.device.kind === 'simulator'), + captureBackend: resolved.captureBackend, + preferredCaptureBackend: dispatchContext.preferredScreenshotCaptureBackend, + }, + execution: screenshotExecutionFromContext(dispatchContext), + }); +} diff --git a/packages/selectors/src/interaction-error.ts b/packages/selectors/src/interaction-error.ts index a49f3e8f54..b618b1bf61 100644 --- a/packages/selectors/src/interaction-error.ts +++ b/packages/selectors/src/interaction-error.ts @@ -50,6 +50,14 @@ export const INTERACTION_ERROR_REASONS = { * hint that names that recovery. */ targetStillMoving: 'target_still_moving', + /** + * A comma-separated ref plan cannot carry `--verify`: the pixel fallback behind `--verify` + * answers one point the pre-action tree cannot see, and the plan's members are tree-visible + * refs the plan already admits against that tree. The refusal is pre-dispatch, so nothing + * crossed to the device. The owner is + * `packages/command-runtime/src/commands/interaction/runtime/interactions.ts` (`tapRefPlanCommand`). + */ + verifyUnsupportedOnRefPlan: 'verify_unsupported_on_ref_plan', } as const; /** The part of a covered node a refusal reports. */ diff --git a/scripts/layering/package-boundaries.test.ts b/scripts/layering/package-boundaries.test.ts index d99af3c487..75e001b152 100644 --- a/scripts/layering/package-boundaries.test.ts +++ b/scripts/layering/package-boundaries.test.ts @@ -504,6 +504,7 @@ test('the real tree parses, declares, and passes R11', () => { '@agent-device/capture-kit/perf-capture-session-resource', '@agent-device/capture-kit/perf-runtime-plan', '@agent-device/capture-kit/png', + '@agent-device/capture-kit/png-changed-pixel-ratio', '@agent-device/capture-kit/png-crop', '@agent-device/capture-kit/png-resize', '@agent-device/capture-kit/png-rgb-difference', diff --git a/test/integration/provider-scenarios/provider-device-runtime.fixtures.ts b/test/integration/provider-scenarios/provider-device-runtime.fixtures.ts index 9e8442b66e..f589d1aa8c 100644 --- a/test/integration/provider-scenarios/provider-device-runtime.fixtures.ts +++ b/test/integration/provider-scenarios/provider-device-runtime.fixtures.ts @@ -1,3 +1,7 @@ +import { + bindProviderScreenshotInteractor, + defaultCaptureOnly, +} from '@agent-device/contracts/screenshot-runtime'; import type { DaemonWireRequest } from '@agent-device/contracts/command'; import { createProviderDeviceRuntimeRequestProviders } from '../../../src/provider-device-runtime.ts'; import type { @@ -71,6 +75,7 @@ export type FakeProviderCall = { | 'close' | 'tap' | 'snapshot' + | 'screenshot' | 'portReverse.ensure'; [key: string]: unknown; }; @@ -80,8 +85,11 @@ type FakeProviderSession = { interactor: Interactor; }; -export async function createFakeProviderWorld(platform: 'android' | 'ios' = 'android') { - const runtime = new FakeProviderDeviceRuntime(platform); +export async function createFakeProviderWorld( + platform: 'android' | 'ios' = 'android', + screenshot?: Interactor['screenshot'], +) { + const runtime = new FakeProviderDeviceRuntime(platform, screenshot); const providerRuntimeProviders = createProviderDeviceRuntimeRequestProviders([runtime]); const owner = providerRuntimeOwner(FAKE_PROVIDER, platform); const providerModule = createProviderScenarioLifecycleModule(runtime, owner); @@ -161,6 +169,14 @@ async function bindProviderScenarioPlatformRuntime( providerScenarioApplicationLifecycle(runtime, request.device, request.scope.signal), facts.operations, ), + ...(facts.operations.captureScreenshot.available + ? bindProviderScreenshotInteractor({ + device: request.device, + signal: request.scope.signal, + resolveInteractor: (runner) => runtime.getInteractor(request.device, runner), + captureCapabilities: defaultCaptureOnly('Fake provider uses one screenshot backend'), + }) + : {}), ...bindProviderSnapshotInteractor({ device: request.device, signal: request.scope.signal, @@ -246,6 +262,10 @@ function providerScenarioRuntimeFacts( listApps: fakeProviderUnavailable, ...unavailableDeploymentSnapshotAndShutdownOperationFacts, captureSnapshot: fakeProviderAvailable, + captureScreenshot: + runtime instanceof FakeProviderDeviceRuntime && runtime.screenshot + ? fakeProviderAvailable + : fakeProviderUnavailable, // The press rides the same interactor transport as every other catalog operation, but the // hardware exists on the iPhone/iPad leaf only, so the fixture reads the same kernel rule the // Apple owner's fact reads instead of restating it (#2699). @@ -306,18 +326,20 @@ async function unavailableProviderScenarioOperation(): Promise { export class FakeProviderDeviceRuntime implements ProviderDeviceRuntime { readonly provider = FAKE_PROVIDER; readonly calls: FakeProviderCall[] = []; + readonly screenshot?: Interactor['screenshot']; private readonly sessionsByLeaseId = new Map(); private readonly platform: 'android' | 'ios'; - constructor(platform: 'android' | 'ios' = 'android') { + constructor(platform: 'android' | 'ios' = 'android', screenshot?: Interactor['screenshot']) { this.platform = platform; + this.screenshot = screenshot; } readonly leaseLifecycle: LeaseLifecycleProvider = { allocate: async (lease) => { if (lease.leaseProvider !== this.provider) return undefined; const device = this.createDevice(lease); - const interactor = createFakeProviderInteractor(device, this.calls); + const interactor = createFakeProviderInteractor(device, this.calls, this.screenshot); this.sessionsByLeaseId.set(lease.leaseId, { device, interactor }); this.calls.push({ type: 'lease.allocate', @@ -424,7 +446,11 @@ export class FakeProviderDeviceRuntime implements ProviderDeviceRuntime { } } -function createFakeProviderInteractor(device: DeviceInfo, calls: FakeProviderCall[]): Interactor { +function createFakeProviderInteractor( + device: DeviceInfo, + calls: FakeProviderCall[], + screenshot: Interactor['screenshot'] | undefined, +): Interactor { return new Proxy>( { open: async (app, options) => { @@ -437,6 +463,14 @@ function createFakeProviderInteractor(device: DeviceInfo, calls: FakeProviderCal calls.push({ type: 'tap', deviceId: device.id, x, y }); return { backend: 'fake-provider', x, y }; }, + ...(screenshot + ? { + screenshot: async (outPath, options) => { + calls.push({ type: 'screenshot', deviceId: device.id, options }); + return await screenshot(outPath, options); + }, + } + : {}), snapshot: async (options): Promise => { calls.push({ type: 'snapshot', diff --git a/test/integration/provider-scenarios/provider-device-runtime.test.ts b/test/integration/provider-scenarios/provider-device-runtime.test.ts index 98443f6abd..757db18374 100644 --- a/test/integration/provider-scenarios/provider-device-runtime.test.ts +++ b/test/integration/provider-scenarios/provider-device-runtime.test.ts @@ -2,6 +2,7 @@ import assert from 'node:assert/strict'; import fs from 'node:fs'; import path from 'node:path'; import { test } from 'vitest'; +import { PNG } from '@agent-device/capture-kit/png'; import type { DeviceLease } from '@agent-device/contracts/device'; import { assertRpcError, assertRpcOk } from './assertions.ts'; import { @@ -57,6 +58,61 @@ test('provider session retains the allocated device and lease for implicit comma }); }, 15_000); +test('coordinate verify uses the open provider device and excludes its portrait status bar', async () => { + const app = 'com.example.demo'; + let frameCount = 0; + const framePaths: string[] = []; + await withProviderScenarioResource( + async () => + await createFakeProviderWorld('android', async (outPath) => { + framePaths.push(outPath); + const frame = new PNG({ width: 25, height: 100 }); + frame.data.fill(255); + if (frameCount++ === 1) frame.data[25 * 4 * 3] = 0; + await fs.promises.writeFile(outPath, PNG.sync.write(frame)); + return { displayRotation: 'portrait' }; + }), + async ({ daemon, runtime }) => { + const lease = await allocateFakeProviderLease(daemon); + assertRpcOk( + await daemon.callCommand('open', [app], leaseFlags(lease.leaseId), { + meta: leaseMeta(lease.leaseId), + }), + ); + const start = runtime.calls.length; + const data = assertRpcOk(await daemon.callCommand('press', ['10', '80'], { verify: true })); + assert.deepEqual((data.evidence as Record)?.screenshotDiff, { + changedFromBefore: false, + changedPixelRatio: 0, + }); + + const calls = runtime.calls + .slice(start) + .filter(({ type }) => type === 'tap' || type === 'screenshot'); + const deviceId = runtime.deviceIdForLease(lease.leaseId); + assert.deepEqual( + calls.map(({ type, deviceId }) => [type, deviceId]), + [ + ['screenshot', deviceId], + ['tap', deviceId], + ['screenshot', deviceId], + ], + ); + for (const call of calls.filter(({ type }) => type === 'screenshot')) { + assert.partialDeepStrictEqual(call.options, { + appBundleId: app, + fullscreen: true, + normalizeStatusBar: false, + surface: 'app', + }); + } + assert.equal(framePaths.length, 2); + for (const outPath of framePaths) assert.equal(fs.existsSync(outPath), false); + assertRpcOk(await daemon.callCommand('close')); + }, + ); +}); + test('provider-owned iOS simulators relaunch without local CoreSimulator refresh', async () => { await withProviderScenarioResource( async () => await createFakeProviderWorld('ios'), diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index f752010765..6361de9df2 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -552,7 +552,7 @@ A successful `click` or `press` means the action was sent to where the command f `--settle` compares trees in the view of the refs you last read. After a full `snapshot`, it reads the full tree and issues full-tree refs; after `snapshot -i`, it reads the interactive view. It also preserves `-d`, `-s`, `--raw` and `--actions`. This holds for `press`, `click`, `fill`, `longpress`, `scroll` and `back`, including ref, selector and coordinate targets. -Standalone `--verify` captures in the pre-action tree's view, which can differ from the last ref frame when a selector resolves a fresh tree. With `--settle --verify`, verification reuses the settled capture and reports `changedFromBefore` only when the two views match or a known surface transition occurred. Without a populated baseline, post-action captures use the interactive view. +Standalone `--verify` captures in the pre-action tree's view, which can differ from the last ref frame when a selector resolves a fresh tree. With `--settle --verify`, tree verification reuses the settled capture and its `changedFromBefore` follows the tree only when the two views match or a known surface transition occurred. Without a populated baseline, post-action captures use the interactive view. When a coordinate `press` or `click` has no accessibility touch target at that point, `--verify` can include `evidence.screenshotDiff` with the post-action tree summary: the pixel pair is another source for `changedFromBefore`, and the answer is true when any comparison verification made saw a change. The comparison excludes the top status-bar region in portrait on supported mobile devices. The change can come from any visible update between the two captures. --verify refuses a ref list; verify a single press. A `click` or `press` whose target the daemon resolves to a point (`@ref` or selector) re-checks that target's position before dispatching when the session's previous action animated the screen, and dispatches where it came to rest. When that check cannot confirm a stable position — the target was still moving, or a re-check could not measure it — the command refuses with `target_still_moving` and `dispatched: no`: nothing was dispatched, the point the command resolved is in `error.details.rect`, and where the target was last seen is in `error.details.movedToRect` when that last look measured one. Run `wait ` on the target and retry the same command, or add `--settle` so the action waits for the UI to go quiet. A refused `@ref` press needs a fresh `snapshot -i` and a selector, because the wait retires the ref. Two dispatch paths hand off before the check: a press at raw coordinates has no resolved target to re-check, and a dispatch the backend performs on the element itself — `click @ref` on a backend with element dispatch, or a Maestro non-hittable coordinate fallback — goes to that backend rather than through a resolved point. On iOS, a runtime-resolved `click` or `press` whose target is a control nested inside a larger touch control — a swipe action beside its row — also asks the runner whether the platform reports that control as hittable. When it does not, the command refuses with `target_not_hittable` and `dispatched: no`, because a tap at the control's centre would land on another element; `error.details.rect` is where the command resolved it. Maestro replay is the exception: a click that carries `allowNonHittableCoordinateFallback` skips this read, taps resolved bounds, and on the direct-iOS selector route sends the selector straight to the runner, which may use the non-hittable coordinate fallback. Reveal the control (for example swipe its row) and take a fresh `snapshot -i`. An answer the runner cannot tie to the target — no selector, an ambiguous match, another frame — lets the press proceed as before. Other presses pay no extra read. The re-check stops when the request is canceled or reaches its deadline, and nothing is dispatched (`dispatched: no`): a canceled request fails with `details.reason: request_canceled`, and a request out of time with `details.reason: daemon_request_timeout`.