From c134b3468b6708c4e96bf8cf99e4a6fc00145a1f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Wed, 18 Nov 2026 14:32:03 +0100 Subject: [PATCH 01/12] feat(interaction): verify coordinate taps with screenshot fallback MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: 赵桂雄 --- packages/capture-kit/package.json | 4 + .../src/png-changed-pixel-ratio.test.ts | 21 +++++ .../src/png-changed-pixel-ratio.ts | 34 ++++++++- .../runtime/__tests__/test-utils/index.ts | 2 + .../interaction/runtime/interactions.test.ts | 37 +++++++++ .../interaction/runtime/interactions.ts | 38 +++++++++- .../runtime/post-action-observation.ts | 16 +++- .../interaction/runtime/verify-screenshot.ts | 75 ++++++++++++++++++ .../command-surface/src/interaction/index.ts | 11 ++- .../src/interaction/metadata.ts | 4 +- packages/contracts/src/backend.ts | 2 + .../contracts/src/interaction-guarantees.ts | 3 +- packages/contracts/src/interaction.ts | 27 +++---- .../src/core/interaction/internal/types.ts | 2 + .../internal/interaction-runtime.ts | 8 +- .../internal/interaction-touch-press.ts | 51 +++++++++++++ .../internal/interaction-touch-runtime.ts | 3 + .../daemon/src/execution/runtime-artifacts.ts | 25 +++++- .../daemon/src/execution/runtime-policy.ts | 11 ++- .../execution/screenshot-artifact-adapter.ts | 12 +-- .../src/execution/screenshot-runtime.ts | 76 ++++++++++++------- scripts/layering/package-boundaries.test.ts | 1 + .../provider-device-runtime.fixtures.ts | 44 +++++++++-- .../provider-device-runtime.test.ts | 56 ++++++++++++++ website/docs/docs/commands.md | 2 +- 25 files changed, 488 insertions(+), 77 deletions(-) create mode 100644 packages/command-runtime/src/commands/interaction/runtime/verify-screenshot.ts diff --git a/packages/capture-kit/package.json b/packages/capture-kit/package.json index 72e1aba544..f2f6c5ffc4 100644 --- a/packages/capture-kit/package.json +++ b/packages/capture-kit/package.json @@ -155,6 +155,10 @@ "types": "./src/png-rgb-difference.ts", "default": "./src/png-rgb-difference.ts" }, + "./png-changed-pixel-ratio": { + "types": "./src/png-changed-pixel-ratio.ts", + "default": "./src/png-changed-pixel-ratio.ts" + }, "./png-size": { "types": "./src/png-size.ts", "default": "./src/png-size.ts" diff --git a/packages/capture-kit/src/png-changed-pixel-ratio.test.ts b/packages/capture-kit/src/png-changed-pixel-ratio.test.ts index 24eb9b979e..1541b2f045 100644 --- a/packages/capture-kit/src/png-changed-pixel-ratio.test.ts +++ b/packages/capture-kit/src/png-changed-pixel-ratio.test.ts @@ -45,3 +45,24 @@ describe('computePngChangedPixelRatio', () => { ).toEqual({ status: 'dimension_mismatch' }); }); }); + +test('computePngChangedPixelRatio ignores the top band share when asked', () => { + const first = { width: 4, height: 4, data: new Uint8Array(4 * 4 * 4).fill(10) }; + const second = { width: 4, height: 4, data: new Uint8Array(4 * 4 * 4).fill(10) }; + // Second frame: top half (rows 0-1) jumps to 200, bottom half (rows 2-3) to 30. + for (let y = 0; y < 2; y += 1) { + for (let x = 0; x < 4; x += 1) second.data[(y * 4 + x) * 4] = 200; + } + for (let y = 2; y < 4; y += 1) { + for (let x = 0; x < 4; x += 1) second.data[(y * 4 + x) * 4] = 30; + } + + const full = computePngChangedPixelRatio(first, second); + expect(full).toEqual({ status: 'compared', changedPixelRatio: 1 }); + + // With the top half excluded, the ignored rows drop out of the comparison: + // the ratio counts the bottom half's changed pixels over that compared + // region, so every pixel the pair was compared over changed. + const half = computePngChangedPixelRatio(first, second, { ignoreTopHeightShare: 0.5 }); + expect(half).toEqual({ status: 'compared', changedPixelRatio: 1 }); +}); diff --git a/packages/capture-kit/src/png-changed-pixel-ratio.ts b/packages/capture-kit/src/png-changed-pixel-ratio.ts index f7b0fa0ebd..8ae92a004d 100644 --- a/packages/capture-kit/src/png-changed-pixel-ratio.ts +++ b/packages/capture-kit/src/png-changed-pixel-ratio.ts @@ -17,10 +17,18 @@ export type PngChangedPixelRatioResult = * * Dimensions must match. Callers comparing frames of different sizes are asking a different * question, and returning a made-up ratio would hide it. + * + * `ignoreTopHeightShare` cuts the top share of the frame out of the comparison entirely, for + * bands whose pixels move without the dispatch causing it — the device's own status bar ticks + * between the bracket's frames, and the readback must answer for what the dispatch changed, not + * for the wall clock. The ratio runs over the compared region: both the changed count and the + * denominator count the rows below the band, so the share is the share of the region the pair + * was actually compared over. */ export function computePngChangedPixelRatio( first: PngRgbImage, second: PngRgbImage, + options: Readonly<{ ignoreTopHeightShare?: number }> = {}, ): PngChangedPixelRatioResult { if (first.width !== second.width || first.height !== second.height) { return { status: 'dimension_mismatch' }; @@ -32,8 +40,29 @@ export function computePngChangedPixelRatio( return { status: 'dimension_mismatch' }; } + // Rows at or above the cut are outside the compared region on both sides. + const ignoredRows = + options.ignoreTopHeightShare && options.ignoreTopHeightShare > 0 + ? Math.min(first.height, Math.floor(first.height * options.ignoreTopHeightShare)) + : 0; + const comparedPixels = totalPixels - ignoredRows * first.width; + if (comparedPixels <= 0) return { status: 'compared', changedPixelRatio: 0 }; + + return { + status: 'compared', + changedPixelRatio: countChangedPixels(first, second, ignoredRows) / comparedPixels, + }; +} + +/** Counts pixels whose RGB differs outside the ignored top rows. */ +function countChangedPixels(first: PngRgbImage, second: PngRgbImage, ignoredRows: number): number { + const declaredBytes = first.width * first.height * 4; let changedPixels = 0; - for (let offset = 0; offset + 3 < first.data.length; offset += 4) { + for ( + let offset = ignoredRows * first.width * 4; + offset + 3 < declaredBytes && offset + 3 < first.data.length; + offset += 4 + ) { if ( first.data[offset] !== second.data[offset] || first.data[offset + 1] !== second.data[offset + 1] || @@ -42,6 +71,5 @@ export function computePngChangedPixelRatio( changedPixels += 1; } } - - return { status: 'compared', changedPixelRatio: changedPixels / totalPixels }; + return changedPixels; } diff --git a/packages/command-runtime/src/commands/interaction/runtime/__tests__/test-utils/index.ts b/packages/command-runtime/src/commands/interaction/runtime/__tests__/test-utils/index.ts index b965fe6f37..1bde55eae9 100644 --- a/packages/command-runtime/src/commands/interaction/runtime/__tests__/test-utils/index.ts +++ b/packages/command-runtime/src/commands/interaction/runtime/__tests__/test-utils/index.ts @@ -409,6 +409,7 @@ export function createInteractionDevice( overrides: Partial< Pick< AgentDeviceBackend, + | 'captureVerifyScreenshot' | 'captureSnapshot' | 'resolveGestureViewport' | 'tap' @@ -430,6 +431,7 @@ export function createInteractionDevice( ) { return createAgentDevice({ backend: { + captureVerifyScreenshot: overrides.captureVerifyScreenshot, platform: overrides.platform ?? 'ios', captureSnapshot: async (...args) => overrides.captureSnapshot ? await overrides.captureSnapshot(...args) : { snapshot }, diff --git a/packages/command-runtime/src/commands/interaction/runtime/interactions.test.ts b/packages/command-runtime/src/commands/interaction/runtime/interactions.test.ts index 6324caeb28..1c66bad399 100644 --- a/packages/command-runtime/src/commands/interaction/runtime/interactions.test.ts +++ b/packages/command-runtime/src/commands/interaction/runtime/interactions.test.ts @@ -933,3 +933,40 @@ test('runtime press plan names the refused key and the pending keys when a membe ); assert.equal(taps.length, 2); }); + +test.each([ + { x: 15, verify: true, frames: [], changed: false }, + { x: 200, verify: false, frames: [], changed: undefined }, + { x: 200, verify: true, frames: ['before', 'after'], changed: false }, + { x: 200, verify: true, frames: ['before', 'after'], changed: true }, +])( + 'coordinate verification brackets dispatch only away from a touch target ($x, $verify, $changed)', + async ({ x, verify, frames, changed }) => { + const { PNG } = await import('pngjs'); + const { writeFile } = await import('node:fs/promises'); + const captures: string[] = []; + let tapped = false; + const device = createInteractionDevice(selectorSnapshot(), { + tap: async () => { + tapped = true; + }, + clock: { now: () => 0, sleep: async () => {} }, + captureVerifyScreenshot: async (outPath) => { + captures.push(tapped ? 'after' : 'before'); + const png = new PNG({ width: 4, height: 4 }); + png.data.fill(255); + if (tapped) png.data[0] = 0; // Status-bar-only change is ignored. + if (tapped && changed) png.data[16] = 0; + await writeFile(outPath, PNG.sync.write(png)); + return { ignoreTopHeightShare: 0.25 }; + }, + }); + const result = await device.interactions.press({ kind: 'point', x, y: 25 }, { verify }); + assert.deepEqual(captures, frames); + assert.equal(result.evidence?.changedFromBefore, changed); + assert.equal( + result.evidence?.screenshotDiff?.changedFromBefore, + frames.length ? changed : undefined, + ); + }, +); diff --git a/packages/command-runtime/src/commands/interaction/runtime/interactions.ts b/packages/command-runtime/src/commands/interaction/runtime/interactions.ts index c5c8da8a60..52a1ac27a3 100644 --- a/packages/command-runtime/src/commands/interaction/runtime/interactions.ts +++ b/packages/command-runtime/src/commands/interaction/runtime/interactions.ts @@ -6,12 +6,15 @@ import type { RepeatedInput, ResolvedInteractionTarget, ResolvedTarget, + ScreenshotVerification, } from '@agent-device/contracts/interaction'; +import { isSemanticTouchTarget } from '@agent-device/selectors/touch-semantics'; +import { containsPoint } from '@agent-device/kernel/rect'; import { AppError } from '@agent-device/kernel/errors'; import { SELECTOR_PIPELINE_POLICIES } from '@agent-device/selectors/selector-pipeline-policy'; import type { Point } from '@agent-device/kernel/snapshot'; import type { AgentDeviceRuntime, CommandContext } from '@agent-device/contracts/runtime-contract'; -import { isFillableType } from '@agent-device/contracts/snapshot'; +import { isFillableType, isViewportRootNode } from '@agent-device/contracts/snapshot'; import { attachResolvedInteractionTarget } from '@agent-device/host-kit/interaction-outcome'; import { sleep, toBackendContext } from '../../runtime-common.ts'; import { toBackendResult, type RuntimeCommand } from '../../runtime-types.ts'; @@ -176,9 +179,9 @@ async function tapCommand( const atRest = await admitResolvedTarget(runtime, options, resolutionRequest, resolved); const point = requireResolvedPoint(atRest); const pointObservation = pointObservationOf(atRest); - let backendResult; - try { - backendResult = await runtime.backend.tap(toBackendContext(runtime, options), point, { + const tap = runtime.backend.tap; + const dispatch = () => + tap.call(runtime.backend, toBackendContext(runtime, options), point, { button: options.button, count: options.count, intervalMs: options.intervalMs, @@ -187,6 +190,32 @@ async function tapCommand( doubleTap: options.doubleTap, ...(pointObservation ? { pointObservation } : {}), }); + let backendResult; + let screenshot: ScreenshotVerification | undefined; + try { + const baseline = atRest.preAction; + const useScreenshot = + observation.verify && + atRest.kind === 'point' && + baseline !== undefined && + runtime.backend.captureVerifyScreenshot !== undefined && + !baseline.nodes.some( + (node) => + node.rect && + !isViewportRootNode(node) && + (node.hittable === true || isSemanticTouchTarget(node)) && + containsPoint(node.rect, point.x, point.y), + ); + if (useScreenshot) { + const { withScreenshotVerification } = await import('./verify-screenshot.ts'); + ({ result: backendResult, screenshot } = await withScreenshotVerification( + runtime, + options, + dispatch, + )); + } else { + backendResult = await dispatch(); + } } catch (error) { // Resolution is complete before the backend call. Preserve it out of // band so a daemon-level failure corroboration can still record the same @@ -204,6 +233,7 @@ async function tapCommand( ...(formattedBackendResult ? { backendResult: formattedBackendResult } : {}), }, observation, + screenshot, ); } diff --git a/packages/command-runtime/src/commands/interaction/runtime/post-action-observation.ts b/packages/command-runtime/src/commands/interaction/runtime/post-action-observation.ts index 396ff5d52e..9718a6ed25 100644 --- a/packages/command-runtime/src/commands/interaction/runtime/post-action-observation.ts +++ b/packages/command-runtime/src/commands/interaction/runtime/post-action-observation.ts @@ -1,6 +1,7 @@ import type { AgentDeviceRuntime, CommandContext } from '@agent-device/contracts/runtime-contract'; import type { InteractionEvidence, + ScreenshotVerification, ResolvedInteractionTarget, SettleObservation, SettleParams, @@ -65,9 +66,22 @@ export async function applyPostActionObservation( resolved: ResolvedInteractionTarget, result: T, params: PostActionObservationPlan, + screenshot?: ScreenshotVerification, ): Promise> { const observed = await observeAfterInteraction(runtime, options, resolved, params); - return reconcileNonHittableHintWithEvidence({ ...result, ...observed }); + const evidence = + screenshot && observed.evidence + ? { + ...observed.evidence, + changedFromBefore: observed.evidence.surfaceChange ? true : screenshot.changedFromBefore, + screenshotDiff: screenshot, + } + : observed.evidence; + return reconcileNonHittableHintWithEvidence({ + ...result, + ...observed, + ...(evidence ? { evidence } : {}), + }); } /** diff --git a/packages/command-runtime/src/commands/interaction/runtime/verify-screenshot.ts b/packages/command-runtime/src/commands/interaction/runtime/verify-screenshot.ts new file mode 100644 index 0000000000..90d3238ad9 --- /dev/null +++ b/packages/command-runtime/src/commands/interaction/runtime/verify-screenshot.ts @@ -0,0 +1,75 @@ +import { readFile } from 'node:fs/promises'; +import type { TemporaryFile } from '@agent-device/contracts/artifact-adapter'; +import type { ScreenshotVerification } from '@agent-device/contracts/interaction'; +import type { AgentDeviceRuntime, CommandContext } from '@agent-device/contracts/runtime-contract'; +import { createCommandTempFile } from '../../io-policy.ts'; +import { sleep } from '../../runtime-common.ts'; + +export async function withScreenshotVerification( + runtime: AgentDeviceRuntime, + options: CommandContext, + dispatch: () => Promise, +): Promise<{ result: T; screenshot?: ScreenshotVerification }> { + const signal = options.signal ?? runtime.signal; + const before = await captureFrame(runtime, signal); + try { + signal?.throwIfAborted(); + const result = await dispatch(); + if (!before) return { result }; + // Give the tap's first visual update time to reach the display. + await sleep(runtime, 150, signal).catch(() => {}); + const after = await captureFrame(runtime, signal); + if (!after) return { result }; + try { + const screenshot = await compareFrames(before, after); + return signal?.aborted ? { result } : { result, screenshot }; + } catch { + return { result }; + } finally { + await after.file.cleanup().catch(() => {}); + } + } finally { + await before?.file.cleanup().catch(() => {}); + } +} + +async function captureFrame(runtime: AgentDeviceRuntime, signal: AbortSignal | undefined) { + if (signal?.aborted || !runtime.backend.captureVerifyScreenshot) return; + let file: TemporaryFile | undefined; + try { + file = await createCommandTempFile(runtime, { prefix: 'agent-device-verify', ext: '.png' }); + if (!signal?.aborted) { + const facts = await runtime.backend.captureVerifyScreenshot(file.path); + if (facts && !signal?.aborted) return { file, ...facts }; + } + } catch { + // An unavailable frame leaves tree verification in charge. + } + await file?.cleanup().catch(() => {}); +} + +async function compareFrames( + before: { file: TemporaryFile; ignoreTopHeightShare: number }, + after: { file: TemporaryFile; ignoreTopHeightShare: number }, +): Promise { + const [{ computePngChangedPixelRatio }, { decodePngAsync }] = await Promise.all([ + import('@agent-device/capture-kit/png-changed-pixel-ratio'), + import('@agent-device/capture-kit/png-worker-client'), + ]); + const [first, second] = await Promise.all([ + decodePngAsync(await readFile(before.file.path), 'verification before frame'), + decodePngAsync(await readFile(after.file.path), 'verification after frame'), + ]); + const comparison = computePngChangedPixelRatio(first, second, { + ignoreTopHeightShare: + first.height >= first.width + ? Math.min(before.ignoreTopHeightShare, after.ignoreTopHeightShare) + : 0, + }); + return comparison.status === 'dimension_mismatch' + ? { changedFromBefore: true } + : { + changedFromBefore: comparison.changedPixelRatio > 0, + changedPixelRatio: comparison.changedPixelRatio, + }; +} diff --git a/packages/command-surface/src/interaction/index.ts b/packages/command-surface/src/interaction/index.ts index e3d01be4ba..fbec94aedd 100644 --- a/packages/command-surface/src/interaction/index.ts +++ b/packages/command-surface/src/interaction/index.ts @@ -75,9 +75,18 @@ const interactionEvidenceSchema: JsonSchema = objectSchema( interactiveNodeCount: numberSchema('Subset of nodeCount the platform reports as hittable.'), digest: stringSchema('Order-independent digest of the post-action node multiset.'), changedFromBefore: booleanSchema( - 'Whether the post-action digest differs from the pre-action capture digest. false is evidence, not failure. Absent when no pre-action baseline was captured, so no comparison is claimed. With surfaceChange present, no digest comparison is made: it reports that surface transition.', + 'Whether verification observed a change. false is evidence, not failure. Absent when no pre-action baseline was captured, so no comparison is claimed. With surfaceChange present, no digest comparison is made: it reports that surface transition.', ), surfaceChange: postActionSurfaceChangeSchema, + screenshotDiff: objectSchema( + { + changedFromBefore: booleanSchema('Whether the compared screen pixels changed.'), + changedPixelRatio: numberSchema( + 'Share of compared pixels that changed, when frame dimensions match.', + ), + }, + ['changedFromBefore'], + ), }, ['nodeCount', 'interactiveNodeCount', 'digest'], ); diff --git a/packages/command-surface/src/interaction/metadata.ts b/packages/command-surface/src/interaction/metadata.ts index 68c04f406d..c6c34c655f 100644 --- a/packages/command-surface/src/interaction/metadata.ts +++ b/packages/command-surface/src/interaction/metadata.ts @@ -65,9 +65,9 @@ const FIND_ACTION_VALUES = [ const interactionCommandDescriptions = { click: - "Activate a UI target by snapshot ref, selector, or coordinates. This can change app state. Success reports dispatch only; settle returns the resulting UI diff and verify returns cheap post-action evidence. A ref or selector target is re-checked for motion before dispatch when the session's previous action just animated the screen, and is dispatched at its resting position; one that has not stopped refuses with target_still_moving rather than acting where it was. With settle, continue from the returned diff. Copy refs exactly, including any ~sN suffix.", + "Activate a UI target by snapshot ref, selector, or coordinates. This can change app state. Success reports dispatch only; settle returns the resulting UI diff and verify reads the result and can compare screenshots outside accessible controls. A ref or selector target is re-checked for motion before dispatch when the session's previous action just animated the screen, and is dispatched at its resting position; one that has not stopped refuses with target_still_moving rather than acting where it was. With settle, continue from the returned diff. Copy refs exactly, including any ~sN suffix.", press: - "Short-press a UI target by snapshot ref, selector, or coordinates. Success reports dispatch only; settle returns the resulting UI diff and verify returns cheap post-action evidence. A ref or selector target is re-checked for motion before dispatch when the session's previous action just animated the screen, and is dispatched at its resting position; one that has not stopped refuses with target_still_moving rather than acting where it was. A comma-separated ref list dispatches 2-16 keys in order as one command, admitted before the first press.", + "Short-press a UI target by snapshot ref, selector, or coordinates. Success reports dispatch only; settle returns the resulting UI diff and verify reads the result and can compare screenshots outside accessible controls. A ref or selector target is re-checked for motion before dispatch when the session's previous action just animated the screen, and is dispatched at its resting position; one that has not stopped refuses with target_still_moving rather than acting where it was. A comma-separated ref list dispatches 2-16 keys in order as one command, admitted before the first press. With settle, continue from the returned diff.", fill: 'Replace text in a UI input selected by snapshot ref, selector, or coordinates. An empty text clears the field. Unconfirmed evidence does not establish the resulting value or screen. The recordAs option keeps sensitive text out of a recorded replay while sending it to the live app. To append instead of replacing, use type.', longpress: 'Hold a UI target by snapshot ref, selector, or coordinates to open a context menu or perform another hold gesture. The durationMs input overrides the default hold duration.', diff --git a/packages/contracts/src/backend.ts b/packages/contracts/src/backend.ts index d57fff756e..e0008d3deb 100644 --- a/packages/contracts/src/backend.ts +++ b/packages/contracts/src/backend.ts @@ -405,6 +405,8 @@ export type AgentDeviceBackend = { outPath: string, options?: BackendScreenshotOptions, ): Promise; + /** Request-bound frame for coordinate verification, with the comparison's status-bar exclusion. */ + captureVerifyScreenshot?(outPath: string): Promise<{ ignoreTopHeightShare: number } | undefined>; readText?(context: BackendCommandContext, node: SnapshotNode): Promise; /** Present only when the bound runtime advertised the conditional `findText` operation. */ findText?(context: BackendCommandContext, text: string): Promise; diff --git a/packages/contracts/src/interaction-guarantees.ts b/packages/contracts/src/interaction-guarantees.ts index 586afe579c..ea54c7878d 100644 --- a/packages/contracts/src/interaction-guarantees.ts +++ b/packages/contracts/src/interaction-guarantees.ts @@ -526,7 +526,8 @@ export const INTERACTION_DISPATCH_PATHS: Record { @@ -213,7 +215,10 @@ function resolveBoundCapture( function createInteractionAgentDevice(params: InteractionRuntimeInput) { return params.createCommandSurface({ backend: createInteractionBackend(params), - ...createDaemonRuntimePolicy('interaction commands', { plural: true }), + ...createDaemonRuntimePolicy('interaction commands', { + plural: true, + allowTempFiles: params.readbackCapture !== undefined, + }), sessions: params.runtimeSessions, signal: getRequestSignal(params.requestId), }); @@ -225,6 +230,7 @@ function createInteractionBackend(params: InteractionRuntimeInput): AgentDeviceB params.contextFromFlags(flags, session.appBundleId, session.trace?.outPath); return { platform: publicPlatformString(session.device), + ...(params.readbackCapture ? { captureVerifyScreenshot: params.readbackCapture } : {}), captureSnapshot: async (context, options): Promise => ({ snapshot: await params.captureSnapshot(flags, { interactiveOnly: options?.interactiveOnly === true, diff --git a/packages/daemon/src/execution/interaction/internal/interaction-touch-press.ts b/packages/daemon/src/execution/interaction/internal/interaction-touch-press.ts index 0e6372f1ac..3ae3ee46e5 100644 --- a/packages/daemon/src/execution/interaction/internal/interaction-touch-press.ts +++ b/packages/daemon/src/execution/interaction/internal/interaction-touch-press.ts @@ -10,6 +10,11 @@ import type { resolveClickButton } from '@agent-device/contracts/click-button'; import type { ReplayTargetGuardDenotation } from '@agent-device/contracts/replay'; import { assertAndroidPressStayedInApp } from './interaction-android-escape.ts'; import { createInteractionRuntimeForRoute } from './interaction-runtime.ts'; +import { resolveBoundScreenshotRuntime } from '../../screenshot-runtime-binding.ts'; +import { captureScreenshotFromRequest } from '../../screenshot-runtime.ts'; +import type { AgentDeviceBackend } from '@agent-device/contracts/backend'; +import type { DeviceRotation } from '@agent-device/contracts/device'; +import type { DeviceInfo } from '@agent-device/kernel/device'; import { readSettleRequest } from './interaction-flags.ts'; import { dispatchDirectIosSelectorTap } from './interaction-touch-direct-ios.ts'; import { readDirectIosSelectorTapTarget } from '../../../core/interaction/internal/interaction-touch-direct-ios-eligibility.ts'; @@ -86,9 +91,55 @@ export async function dispatchTargetedTouchViaRuntime( ...buildTargetedRuntimeOptions(params, command, boundAdmitted), touchExecutor, settleObservationCapture: prepared.settleObservationCapture, + ...(admitted.target.kind === 'point' && + (command === 'press' || command === 'click') && + params.req.flags?.verify === true + ? { readbackCapture: createVerifyScreenshotCapture(params, admitted.session) } + : {}), }); } +function createVerifyScreenshotCapture( + params: TargetedTouchParams, + session: AdmittedTargetedTouch['session'], +): NonNullable { + // Bind lazily. A verified tap with a node at the point never needs screenshot authority. + let binding: ReturnType | undefined; + return async (outPath) => { + const capture = await (binding ??= resolveBoundScreenshotRuntime({ + ...params, + device: session.device, + overlayRefs: false, + })); + if (!capture.ok) return undefined; + const dispatchContext = params.contextFromFlags( + params.req.flags, + session.appBundleId, + session.trace?.outPath, + ); + const facts = await captureScreenshotFromRequest( + { session, dispatchContext, captureScreenshot: capture.runtime.captureScreenshot }, + outPath, + { + surface: session.surface, + fullscreen: verifyStatusBarBand(session.device, undefined) > 0 ? true : undefined, + normalizeStatusBar: false, + }, + ); + return { ignoreTopHeightShare: verifyStatusBarBand(session.device, facts.displayRotation) }; + }; +} + +function verifyStatusBarBand(device: DeviceInfo, rotation: DeviceRotation | undefined): number { + if (rotation?.startsWith('landscape') || device.target === 'tv' || device.target === 'desktop') { + return 0; + } + if (device.appleOs === 'ipados') return 0.02; + if (device.appleOs === 'ios') return 0.06; + if (device.platform === 'android' || device.platform === 'harmonyos') return 0.04; + return 0; +} + function buildTargetedRuntimeOptions( params: TargetedTouchParams, command: TargetedTouchCommand, diff --git a/packages/daemon/src/execution/interaction/internal/interaction-touch-runtime.ts b/packages/daemon/src/execution/interaction/internal/interaction-touch-runtime.ts index d2ffe838ec..03284e0a15 100644 --- a/packages/daemon/src/execution/interaction/internal/interaction-touch-runtime.ts +++ b/packages/daemon/src/execution/interaction/internal/interaction-touch-runtime.ts @@ -1,3 +1,4 @@ +import type { AgentDeviceBackend } from '@agent-device/contracts/backend'; import type { DaemonResponse } from '@agent-device/kernel/contracts'; import { bindInteractionSession } from './interaction-session.ts'; import type { @@ -54,6 +55,7 @@ export async function dispatchRuntimeInteraction< touchExecutor: BoundTouchExecutor; /** The request's own observation capture when its action use carries none (#3510). */ settleObservationCapture?: BoundSnapshotCapture; + readbackCapture?: AgentDeviceBackend['captureVerifyScreenshot']; androidFreshnessBaseline?: SessionState['snapshot']; /** * Present when the action targets a `@ref`: if Android dialog recovery @@ -82,6 +84,7 @@ export async function dispatchRuntimeInteraction< ...params, touchExecutor: options.touchExecutor, settleObservationCapture: options.settleObservationCapture, + readbackCapture: options.readbackCapture, consumedCapture, }); const actionStartedAt = Date.now(); diff --git a/packages/daemon/src/execution/runtime-artifacts.ts b/packages/daemon/src/execution/runtime-artifacts.ts index dc154f67f9..8faf9d92d0 100644 --- a/packages/daemon/src/execution/runtime-artifacts.ts +++ b/packages/daemon/src/execution/runtime-artifacts.ts @@ -1,6 +1,29 @@ -import type { ArtifactAdapter } from '@agent-device/contracts/artifact-adapter'; +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import type { ArtifactAdapter, TemporaryFile } from '@agent-device/contracts/artifact-adapter'; import { AppError } from '@agent-device/kernel/errors'; +/** + * The one temp-file implementation every daemon route shares: an `agent-device-*` root under the + * process tmpdir, one file inside, removed with the cleanup. Routes whose policy refuses temp + * files keep the refusing adapter; routes that opt in (runtime policy's `allowTempFiles`, the + * screenshot artifact adapter) get this implementation. + */ +export async function createDaemonTempFile(options: { + prefix: string; + ext: string; +}): Promise { + const root = await fs.mkdtemp(path.join(os.tmpdir(), `${options.prefix}-`)); + return { + path: path.join(root, `file${options.ext}`), + visibility: 'internal', + cleanup: async () => { + await fs.rm(root, { recursive: true, force: true }); + }, + }; +} + export function createUnsupportedArtifactAdapter( label: string, options: { plural?: boolean } = {}, diff --git a/packages/daemon/src/execution/runtime-policy.ts b/packages/daemon/src/execution/runtime-policy.ts index 0d85d3ac11..312fce86b5 100644 --- a/packages/daemon/src/execution/runtime-policy.ts +++ b/packages/daemon/src/execution/runtime-policy.ts @@ -1,13 +1,18 @@ import type { AgentDeviceRuntimeConfig } from '@agent-device/contracts/runtime-contract'; import { localCommandPolicy } from '@agent-device/contracts/command-policy'; -import { createUnsupportedArtifactAdapter } from './runtime-artifacts.ts'; +import { createDaemonTempFile, createUnsupportedArtifactAdapter } from './runtime-artifacts.ts'; export function createDaemonRuntimePolicy( unsupportedArtifactLabel: string, - options: { plural?: boolean } = {}, + options: { plural?: boolean; allowTempFiles?: boolean } = {}, ): Pick { return { - artifacts: createUnsupportedArtifactAdapter(unsupportedArtifactLabel, options), + artifacts: options.allowTempFiles + ? { + ...createUnsupportedArtifactAdapter(unsupportedArtifactLabel, options), + createTempFile: createDaemonTempFile, + } + : createUnsupportedArtifactAdapter(unsupportedArtifactLabel, options), policy: localCommandPolicy(), }; } diff --git a/packages/daemon/src/execution/screenshot-artifact-adapter.ts b/packages/daemon/src/execution/screenshot-artifact-adapter.ts index 43086e8d61..765fc1ebc7 100644 --- a/packages/daemon/src/execution/screenshot-artifact-adapter.ts +++ b/packages/daemon/src/execution/screenshot-artifact-adapter.ts @@ -2,6 +2,7 @@ import { publishFileSync } from '@agent-device/host-kit/file'; import { emitDiagnostic } from '@agent-device/host-kit/diagnostics'; import type { ArtifactAdapter, ReservedOutputFile } from '@agent-device/contracts/artifact-adapter'; import { AppError, errorMessage } from '@agent-device/kernel/errors'; +import { createDaemonTempFile } from './runtime-artifacts.ts'; import { promises as fs } from 'node:fs'; import os from 'node:os'; import path from 'node:path'; @@ -51,16 +52,7 @@ export function createDaemonScreenshotArtifactAdapter( ); } }, - createTempFile: async (options) => { - const root = await fs.mkdtemp(path.join(os.tmpdir(), `${options.prefix}-`)); - return { - path: path.join(root, `file${options.ext}`), - visibility: 'internal', - cleanup: async () => { - await fs.rm(root, { recursive: true, force: true }); - }, - }; - }, + createTempFile: createDaemonTempFile, }; } diff --git a/packages/daemon/src/execution/screenshot-runtime.ts b/packages/daemon/src/execution/screenshot-runtime.ts index ab094426c2..3984f28976 100644 --- a/packages/daemon/src/execution/screenshot-runtime.ts +++ b/packages/daemon/src/execution/screenshot-runtime.ts @@ -8,11 +8,14 @@ import { screenshotOptionsFromFlags, } from '@agent-device/contracts/capture'; import type { DeviceRotation } from '@agent-device/contracts/device'; -import type { ScreenshotRuntimeExecution } from '@agent-device/contracts/screenshot-runtime'; +import type { + ScreenshotCaptureFacts, + ScreenshotRuntimeExecution, +} from '@agent-device/contracts/screenshot-runtime'; import { isIosFamily, publicPlatformString } from '@agent-device/kernel/device'; import { AppError } from '@agent-device/kernel/errors'; import type { ScreenshotOverlayRef } from '@agent-device/kernel/snapshot'; -import type { AgentDeviceBackend } from '@agent-device/contracts/backend'; +import type { AgentDeviceBackend, BackendScreenshotOptions } from '@agent-device/contracts/backend'; import { localCommandPolicy } from '@agent-device/contracts/command-policy'; import type { CreateDaemonCommandSurface } from '@agent-device/contracts/daemon-command-surface'; import { @@ -369,34 +372,11 @@ function createBoundScreenshotBackend( crop?: ScreenshotCropBinding; }>, ): AgentDeviceBackend { - const { session, dispatchContext, captureScreenshot, crop } = params; + const { session, dispatchContext, crop } = params; return { platform: publicPlatformString(session.device), captureScreenshot: async (_context, outPath, options) => { - const resolved = screenshotOptionsFromFlags({ - ...dispatchContext, - ...screenshotFlagsFromOptions(options), - }); - const facts = await captureScreenshot({ - outPath, - options: { - appBundleId: dispatchContext.appBundleId, - pixelDensity: resolved.pixelDensity, - screenMask: resolved.screenMask, - fullscreen: resolved.fullscreen, - normalizeStatusBar: resolved.normalizeStatusBar, - stabilize: resolved.stabilize, - surface: options?.surface, - // An open session already proved the simulator is booted, so the per-capture recheck is - // the daemon's own knowledge rather than something the caller has to assert. - skipIosSimulatorBootCheck: - dispatchContext.skipIosSimulatorBootCheck ?? - (isIosFamily(session.device) && session.device.kind === 'simulator'), - captureBackend: resolved.captureBackend, - preferredCaptureBackend: dispatchContext.preferredScreenshotCaptureBackend, - }, - execution: screenshotExecutionFromContext(dispatchContext), - }); + const facts = await captureScreenshotFromRequest(params, outPath, options); if (crop) { assertScreenshotCropCapture(session.device, facts); const outcome = await cropScreenshotToSelector({ @@ -415,3 +395,45 @@ function createBoundScreenshotBackend( }, }; } + +/** + * The one builder for a capture backed by this request: the screenshot backend and the press + * readback both capture through it, so their frames come from exactly the same input — the + * session's app identity, the resolved screenshot flags, the surface, and the runner execution + * from the dispatch context. + */ +export async function captureScreenshotFromRequest( + params: Readonly<{ + session: SessionState; + dispatchContext: DaemonCommandContext; + captureScreenshot: BoundScreenshotRuntime['captureScreenshot']; + }>, + outPath: string, + options?: BackendScreenshotOptions, +): Promise { + const { session, dispatchContext, captureScreenshot } = params; + const resolved = screenshotOptionsFromFlags({ + ...dispatchContext, + ...screenshotFlagsFromOptions(options), + }); + return await captureScreenshot({ + outPath, + options: { + appBundleId: dispatchContext.appBundleId, + pixelDensity: resolved.pixelDensity, + screenMask: resolved.screenMask, + fullscreen: resolved.fullscreen, + normalizeStatusBar: resolved.normalizeStatusBar, + stabilize: resolved.stabilize, + surface: options?.surface, + // An open session already proved the simulator is booted, so the per-capture recheck is + // the daemon's own knowledge rather than something the caller has to assert. + skipIosSimulatorBootCheck: + dispatchContext.skipIosSimulatorBootCheck ?? + (isIosFamily(session.device) && session.device.kind === 'simulator'), + captureBackend: resolved.captureBackend, + preferredCaptureBackend: dispatchContext.preferredScreenshotCaptureBackend, + }, + execution: screenshotExecutionFromContext(dispatchContext), + }); +} diff --git a/scripts/layering/package-boundaries.test.ts b/scripts/layering/package-boundaries.test.ts index d99af3c487..75e001b152 100644 --- a/scripts/layering/package-boundaries.test.ts +++ b/scripts/layering/package-boundaries.test.ts @@ -504,6 +504,7 @@ test('the real tree parses, declares, and passes R11', () => { '@agent-device/capture-kit/perf-capture-session-resource', '@agent-device/capture-kit/perf-runtime-plan', '@agent-device/capture-kit/png', + '@agent-device/capture-kit/png-changed-pixel-ratio', '@agent-device/capture-kit/png-crop', '@agent-device/capture-kit/png-resize', '@agent-device/capture-kit/png-rgb-difference', diff --git a/test/integration/provider-scenarios/provider-device-runtime.fixtures.ts b/test/integration/provider-scenarios/provider-device-runtime.fixtures.ts index 9e8442b66e..f589d1aa8c 100644 --- a/test/integration/provider-scenarios/provider-device-runtime.fixtures.ts +++ b/test/integration/provider-scenarios/provider-device-runtime.fixtures.ts @@ -1,3 +1,7 @@ +import { + bindProviderScreenshotInteractor, + defaultCaptureOnly, +} from '@agent-device/contracts/screenshot-runtime'; import type { DaemonWireRequest } from '@agent-device/contracts/command'; import { createProviderDeviceRuntimeRequestProviders } from '../../../src/provider-device-runtime.ts'; import type { @@ -71,6 +75,7 @@ export type FakeProviderCall = { | 'close' | 'tap' | 'snapshot' + | 'screenshot' | 'portReverse.ensure'; [key: string]: unknown; }; @@ -80,8 +85,11 @@ type FakeProviderSession = { interactor: Interactor; }; -export async function createFakeProviderWorld(platform: 'android' | 'ios' = 'android') { - const runtime = new FakeProviderDeviceRuntime(platform); +export async function createFakeProviderWorld( + platform: 'android' | 'ios' = 'android', + screenshot?: Interactor['screenshot'], +) { + const runtime = new FakeProviderDeviceRuntime(platform, screenshot); const providerRuntimeProviders = createProviderDeviceRuntimeRequestProviders([runtime]); const owner = providerRuntimeOwner(FAKE_PROVIDER, platform); const providerModule = createProviderScenarioLifecycleModule(runtime, owner); @@ -161,6 +169,14 @@ async function bindProviderScenarioPlatformRuntime( providerScenarioApplicationLifecycle(runtime, request.device, request.scope.signal), facts.operations, ), + ...(facts.operations.captureScreenshot.available + ? bindProviderScreenshotInteractor({ + device: request.device, + signal: request.scope.signal, + resolveInteractor: (runner) => runtime.getInteractor(request.device, runner), + captureCapabilities: defaultCaptureOnly('Fake provider uses one screenshot backend'), + }) + : {}), ...bindProviderSnapshotInteractor({ device: request.device, signal: request.scope.signal, @@ -246,6 +262,10 @@ function providerScenarioRuntimeFacts( listApps: fakeProviderUnavailable, ...unavailableDeploymentSnapshotAndShutdownOperationFacts, captureSnapshot: fakeProviderAvailable, + captureScreenshot: + runtime instanceof FakeProviderDeviceRuntime && runtime.screenshot + ? fakeProviderAvailable + : fakeProviderUnavailable, // The press rides the same interactor transport as every other catalog operation, but the // hardware exists on the iPhone/iPad leaf only, so the fixture reads the same kernel rule the // Apple owner's fact reads instead of restating it (#2699). @@ -306,18 +326,20 @@ async function unavailableProviderScenarioOperation(): Promise { export class FakeProviderDeviceRuntime implements ProviderDeviceRuntime { readonly provider = FAKE_PROVIDER; readonly calls: FakeProviderCall[] = []; + readonly screenshot?: Interactor['screenshot']; private readonly sessionsByLeaseId = new Map(); private readonly platform: 'android' | 'ios'; - constructor(platform: 'android' | 'ios' = 'android') { + constructor(platform: 'android' | 'ios' = 'android', screenshot?: Interactor['screenshot']) { this.platform = platform; + this.screenshot = screenshot; } readonly leaseLifecycle: LeaseLifecycleProvider = { allocate: async (lease) => { if (lease.leaseProvider !== this.provider) return undefined; const device = this.createDevice(lease); - const interactor = createFakeProviderInteractor(device, this.calls); + const interactor = createFakeProviderInteractor(device, this.calls, this.screenshot); this.sessionsByLeaseId.set(lease.leaseId, { device, interactor }); this.calls.push({ type: 'lease.allocate', @@ -424,7 +446,11 @@ export class FakeProviderDeviceRuntime implements ProviderDeviceRuntime { } } -function createFakeProviderInteractor(device: DeviceInfo, calls: FakeProviderCall[]): Interactor { +function createFakeProviderInteractor( + device: DeviceInfo, + calls: FakeProviderCall[], + screenshot: Interactor['screenshot'] | undefined, +): Interactor { return new Proxy>( { open: async (app, options) => { @@ -437,6 +463,14 @@ function createFakeProviderInteractor(device: DeviceInfo, calls: FakeProviderCal calls.push({ type: 'tap', deviceId: device.id, x, y }); return { backend: 'fake-provider', x, y }; }, + ...(screenshot + ? { + screenshot: async (outPath, options) => { + calls.push({ type: 'screenshot', deviceId: device.id, options }); + return await screenshot(outPath, options); + }, + } + : {}), snapshot: async (options): Promise => { calls.push({ type: 'snapshot', diff --git a/test/integration/provider-scenarios/provider-device-runtime.test.ts b/test/integration/provider-scenarios/provider-device-runtime.test.ts index 98443f6abd..757db18374 100644 --- a/test/integration/provider-scenarios/provider-device-runtime.test.ts +++ b/test/integration/provider-scenarios/provider-device-runtime.test.ts @@ -2,6 +2,7 @@ import assert from 'node:assert/strict'; import fs from 'node:fs'; import path from 'node:path'; import { test } from 'vitest'; +import { PNG } from '@agent-device/capture-kit/png'; import type { DeviceLease } from '@agent-device/contracts/device'; import { assertRpcError, assertRpcOk } from './assertions.ts'; import { @@ -57,6 +58,61 @@ test('provider session retains the allocated device and lease for implicit comma }); }, 15_000); +test('coordinate verify uses the open provider device and excludes its portrait status bar', async () => { + const app = 'com.example.demo'; + let frameCount = 0; + const framePaths: string[] = []; + await withProviderScenarioResource( + async () => + await createFakeProviderWorld('android', async (outPath) => { + framePaths.push(outPath); + const frame = new PNG({ width: 25, height: 100 }); + frame.data.fill(255); + if (frameCount++ === 1) frame.data[25 * 4 * 3] = 0; + await fs.promises.writeFile(outPath, PNG.sync.write(frame)); + return { displayRotation: 'portrait' }; + }), + async ({ daemon, runtime }) => { + const lease = await allocateFakeProviderLease(daemon); + assertRpcOk( + await daemon.callCommand('open', [app], leaseFlags(lease.leaseId), { + meta: leaseMeta(lease.leaseId), + }), + ); + const start = runtime.calls.length; + const data = assertRpcOk(await daemon.callCommand('press', ['10', '80'], { verify: true })); + assert.deepEqual((data.evidence as Record)?.screenshotDiff, { + changedFromBefore: false, + changedPixelRatio: 0, + }); + + const calls = runtime.calls + .slice(start) + .filter(({ type }) => type === 'tap' || type === 'screenshot'); + const deviceId = runtime.deviceIdForLease(lease.leaseId); + assert.deepEqual( + calls.map(({ type, deviceId }) => [type, deviceId]), + [ + ['screenshot', deviceId], + ['tap', deviceId], + ['screenshot', deviceId], + ], + ); + for (const call of calls.filter(({ type }) => type === 'screenshot')) { + assert.partialDeepStrictEqual(call.options, { + appBundleId: app, + fullscreen: true, + normalizeStatusBar: false, + surface: 'app', + }); + } + assert.equal(framePaths.length, 2); + for (const outPath of framePaths) assert.equal(fs.existsSync(outPath), false); + assertRpcOk(await daemon.callCommand('close')); + }, + ); +}); + test('provider-owned iOS simulators relaunch without local CoreSimulator refresh', async () => { await withProviderScenarioResource( async () => await createFakeProviderWorld('ios'), diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index f752010765..f1111a3a04 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -552,7 +552,7 @@ A successful `click` or `press` means the action was sent to where the command f `--settle` compares trees in the view of the refs you last read. After a full `snapshot`, it reads the full tree and issues full-tree refs; after `snapshot -i`, it reads the interactive view. It also preserves `-d`, `-s`, `--raw` and `--actions`. This holds for `press`, `click`, `fill`, `longpress`, `scroll` and `back`, including ref, selector and coordinate targets. -Standalone `--verify` captures in the pre-action tree's view, which can differ from the last ref frame when a selector resolves a fresh tree. With `--settle --verify`, verification reuses the settled capture and reports `changedFromBefore` only when the two views match or a known surface transition occurred. Without a populated baseline, post-action captures use the interactive view. +Standalone `--verify` captures in the pre-action tree's view, which can differ from the last ref frame when a selector resolves a fresh tree. With `--settle --verify`, verification reuses the settled capture and reports `changedFromBefore` only when the two views match or a known surface transition occurred. Without a populated baseline, post-action captures use the interactive view. When a coordinate `press` or `click` has no accessibility touch target at that point, `--verify` can include `evidence.screenshotDiff` with the post-action tree summary. The comparison excludes the top status-bar region in portrait on supported mobile devices. The change can come from any visible update between the two captures. A ref list uses tree verification for the entire sequence. A `click` or `press` whose target the daemon resolves to a point (`@ref` or selector) re-checks that target's position before dispatching when the session's previous action animated the screen, and dispatches where it came to rest. When that check cannot confirm a stable position — the target was still moving, or a re-check could not measure it — the command refuses with `target_still_moving` and `dispatched: no`: nothing was dispatched, the point the command resolved is in `error.details.rect`, and where the target was last seen is in `error.details.movedToRect` when that last look measured one. Run `wait ` on the target and retry the same command, or add `--settle` so the action waits for the UI to go quiet. A refused `@ref` press needs a fresh `snapshot -i` and a selector, because the wait retires the ref. Two dispatch paths hand off before the check: a press at raw coordinates has no resolved target to re-check, and a dispatch the backend performs on the element itself — `click @ref` on a backend with element dispatch, or a Maestro non-hittable coordinate fallback — goes to that backend rather than through a resolved point. On iOS, a runtime-resolved `click` or `press` whose target is a control nested inside a larger touch control — a swipe action beside its row — also asks the runner whether the platform reports that control as hittable. When it does not, the command refuses with `target_not_hittable` and `dispatched: no`, because a tap at the control's centre would land on another element; `error.details.rect` is where the command resolved it. Maestro replay is the exception: a click that carries `allowNonHittableCoordinateFallback` skips this read, taps resolved bounds, and on the direct-iOS selector route sends the selector straight to the runner, which may use the non-hittable coordinate fallback. Reveal the control (for example swipe its row) and take a fresh `snapshot -i`. An answer the runner cannot tie to the target — no selector, an ambiguous match, another frame — lets the press proceed as before. Other presses pay no extra read. The re-check stops when the request is canceled or reaches its deadline, and nothing is dispatched (`dispatched: no`): a canceled request fails with `details.reason: request_canceled`, and a request out of time with `details.reason: daemon_request_timeout`. From 34f22bc754c717df38307be9070366ef2733dc48 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=B5=B5=E6=A1=82=E9=9B=84?= Date: Thu, 19 Nov 2026 00:57:56 +0800 Subject: [PATCH 02/12] fix(interaction): keep any observed change in changedFromBefore A tree digest that moved survives identical frames: the composed answer is true when any comparison verification made saw a change, and the pixel-only answer stays in screenshotDiff beside it. The tap dispatch becomes a plain closure and the after-frame settle delay gets its name. Co-Authored-By: Claude Code --- .../interaction/runtime/interactions.test.ts | 37 -------- .../interaction/runtime/interactions.ts | 4 +- .../runtime/post-action-observation.ts | 8 +- .../runtime/verify-screenshot.test.ts | 85 +++++++++++++++++++ .../interaction/runtime/verify-screenshot.ts | 5 +- 5 files changed, 98 insertions(+), 41 deletions(-) create mode 100644 packages/command-runtime/src/commands/interaction/runtime/verify-screenshot.test.ts diff --git a/packages/command-runtime/src/commands/interaction/runtime/interactions.test.ts b/packages/command-runtime/src/commands/interaction/runtime/interactions.test.ts index 1c66bad399..6324caeb28 100644 --- a/packages/command-runtime/src/commands/interaction/runtime/interactions.test.ts +++ b/packages/command-runtime/src/commands/interaction/runtime/interactions.test.ts @@ -933,40 +933,3 @@ test('runtime press plan names the refused key and the pending keys when a membe ); assert.equal(taps.length, 2); }); - -test.each([ - { x: 15, verify: true, frames: [], changed: false }, - { x: 200, verify: false, frames: [], changed: undefined }, - { x: 200, verify: true, frames: ['before', 'after'], changed: false }, - { x: 200, verify: true, frames: ['before', 'after'], changed: true }, -])( - 'coordinate verification brackets dispatch only away from a touch target ($x, $verify, $changed)', - async ({ x, verify, frames, changed }) => { - const { PNG } = await import('pngjs'); - const { writeFile } = await import('node:fs/promises'); - const captures: string[] = []; - let tapped = false; - const device = createInteractionDevice(selectorSnapshot(), { - tap: async () => { - tapped = true; - }, - clock: { now: () => 0, sleep: async () => {} }, - captureVerifyScreenshot: async (outPath) => { - captures.push(tapped ? 'after' : 'before'); - const png = new PNG({ width: 4, height: 4 }); - png.data.fill(255); - if (tapped) png.data[0] = 0; // Status-bar-only change is ignored. - if (tapped && changed) png.data[16] = 0; - await writeFile(outPath, PNG.sync.write(png)); - return { ignoreTopHeightShare: 0.25 }; - }, - }); - const result = await device.interactions.press({ kind: 'point', x, y: 25 }, { verify }); - assert.deepEqual(captures, frames); - assert.equal(result.evidence?.changedFromBefore, changed); - assert.equal( - result.evidence?.screenshotDiff?.changedFromBefore, - frames.length ? changed : undefined, - ); - }, -); diff --git a/packages/command-runtime/src/commands/interaction/runtime/interactions.ts b/packages/command-runtime/src/commands/interaction/runtime/interactions.ts index 52a1ac27a3..2f341ce0db 100644 --- a/packages/command-runtime/src/commands/interaction/runtime/interactions.ts +++ b/packages/command-runtime/src/commands/interaction/runtime/interactions.ts @@ -179,9 +179,9 @@ async function tapCommand( const atRest = await admitResolvedTarget(runtime, options, resolutionRequest, resolved); const point = requireResolvedPoint(atRest); const pointObservation = pointObservationOf(atRest); - const tap = runtime.backend.tap; + const dispatchTap = runtime.backend.tap; const dispatch = () => - tap.call(runtime.backend, toBackendContext(runtime, options), point, { + dispatchTap(toBackendContext(runtime, options), point, { button: options.button, count: options.count, intervalMs: options.intervalMs, diff --git a/packages/command-runtime/src/commands/interaction/runtime/post-action-observation.ts b/packages/command-runtime/src/commands/interaction/runtime/post-action-observation.ts index 9718a6ed25..4dfaca063e 100644 --- a/packages/command-runtime/src/commands/interaction/runtime/post-action-observation.ts +++ b/packages/command-runtime/src/commands/interaction/runtime/post-action-observation.ts @@ -73,7 +73,13 @@ export async function applyPostActionObservation( screenshot && observed.evidence ? { ...observed.evidence, - changedFromBefore: observed.evidence.surfaceChange ? true : screenshot.changedFromBefore, + // Any comparison that verification made saw a change: the tree + // digest, the settle surface, or the pixel pair. The pixel-only + // answer stays in screenshotDiff beside it. + changedFromBefore: + observed.evidence.surfaceChange !== undefined || + observed.evidence.changedFromBefore === true || + screenshot.changedFromBefore, screenshotDiff: screenshot, } : observed.evidence; diff --git a/packages/command-runtime/src/commands/interaction/runtime/verify-screenshot.test.ts b/packages/command-runtime/src/commands/interaction/runtime/verify-screenshot.test.ts new file mode 100644 index 0000000000..06761f26e9 --- /dev/null +++ b/packages/command-runtime/src/commands/interaction/runtime/verify-screenshot.test.ts @@ -0,0 +1,85 @@ +import assert from 'node:assert/strict'; +import { test } from 'vitest'; +import { makeSnapshotState } from '@agent-device/selectors/snapshot-geometry-fixtures'; +import { createInteractionDevice, selectorSnapshot } from './__tests__/test-utils/index.ts'; + +test('coordinate verification keeps a changed tree digest when the frames match', async () => { + const { PNG } = await import('pngjs'); + const { writeFile } = await import('node:fs/promises'); + let tapped = false; + const device = createInteractionDevice(selectorSnapshot(), { + // The tree digest moves (a navigation replaced the content) while the + // pixels at the settle instant still match. + captureSnapshot: async () => { + if (!tapped) return { snapshot: selectorSnapshot() }; + return { + snapshot: makeSnapshotState([ + { + index: 0, + depth: 0, + type: 'Application', + rect: { x: 0, y: 0, width: 400, height: 800 }, + hittable: true, + }, + ]), + }; + }, + tap: async () => { + tapped = true; + }, + clock: { now: () => 0, sleep: async () => {} }, + captureVerifyScreenshot: async (outPath) => { + const png = new PNG({ width: 4, height: 4 }); + png.data.fill(255); + await writeFile(outPath, PNG.sync.write(png)); + return { ignoreTopHeightShare: 0.25 }; + }, + }); + + const result = await device.interactions.press( + { kind: 'point', x: 200, y: 25 }, + { verify: true }, + ); + + // Any comparison verification made saw a change: the digest answer survives + // identical frames, and the pixel-only answer stays in screenshotDiff. + assert.equal(result.evidence?.changedFromBefore, true); + assert.equal(result.evidence?.screenshotDiff?.changedFromBefore, false); +}); + +test.each([ + { x: 15, verify: true, frames: [], changed: false }, + { x: 200, verify: false, frames: [], changed: undefined }, + { x: 200, verify: true, frames: ['before', 'after'], changed: false }, + { x: 200, verify: true, frames: ['before', 'after'], changed: true }, +])( + 'coordinate verification brackets dispatch only away from a touch target ($x, $verify, $changed)', + async ({ x, verify, frames, changed }) => { + const { PNG } = await import('pngjs'); + const { writeFile } = await import('node:fs/promises'); + const captures: string[] = []; + let tapped = false; + const device = createInteractionDevice(selectorSnapshot(), { + tap: async () => { + tapped = true; + }, + clock: { now: () => 0, sleep: async () => {} }, + captureVerifyScreenshot: async (outPath) => { + captures.push(tapped ? 'after' : 'before'); + const png = new PNG({ width: 4, height: 4 }); + png.data.fill(255); + if (tapped) png.data[0] = 0; // Status-bar-only change is ignored. + if (tapped && changed) png.data[16] = 0; + await writeFile(outPath, PNG.sync.write(png)); + return { ignoreTopHeightShare: 0.25 }; + }, + }); + const result = await device.interactions.press({ kind: 'point', x, y: 25 }, { verify }); + assert.deepEqual(captures, frames); + assert.equal(result.evidence?.changedFromBefore, changed); + assert.equal( + result.evidence?.screenshotDiff?.changedFromBefore, + frames.length ? changed : undefined, + ); + }, +); diff --git a/packages/command-runtime/src/commands/interaction/runtime/verify-screenshot.ts b/packages/command-runtime/src/commands/interaction/runtime/verify-screenshot.ts index 90d3238ad9..f5b90caef0 100644 --- a/packages/command-runtime/src/commands/interaction/runtime/verify-screenshot.ts +++ b/packages/command-runtime/src/commands/interaction/runtime/verify-screenshot.ts @@ -17,7 +17,7 @@ export async function withScreenshotVerification( const result = await dispatch(); if (!before) return { result }; // Give the tap's first visual update time to reach the display. - await sleep(runtime, 150, signal).catch(() => {}); + await sleep(runtime, VERIFY_AFTER_FRAME_SETTLE_MS, signal).catch(() => {}); const after = await captureFrame(runtime, signal); if (!after) return { result }; try { @@ -48,6 +48,9 @@ async function captureFrame(runtime: AgentDeviceRuntime, signal: AbortSignal | u await file?.cleanup().catch(() => {}); } +/** The after frame waits this long past the dispatch reply before it is captured. */ +const VERIFY_AFTER_FRAME_SETTLE_MS = 150; + async function compareFrames( before: { file: TemporaryFile; ignoreTopHeightShare: number }, after: { file: TemporaryFile; ignoreTopHeightShare: number }, From 2eaace5716479836aa57161b459f4e872f0d1507 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=B5=B5=E6=A1=82=E9=9B=84?= Date: Thu, 19 Nov 2026 01:28:14 +0800 Subject: [PATCH 03/12] test(interaction): draw the verification frames with the capture-kit png Co-Authored-By: Claude Code --- .../src/commands/interaction/runtime/verify-screenshot.test.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/command-runtime/src/commands/interaction/runtime/verify-screenshot.test.ts b/packages/command-runtime/src/commands/interaction/runtime/verify-screenshot.test.ts index 06761f26e9..eb82eac7df 100644 --- a/packages/command-runtime/src/commands/interaction/runtime/verify-screenshot.test.ts +++ b/packages/command-runtime/src/commands/interaction/runtime/verify-screenshot.test.ts @@ -1,10 +1,10 @@ import assert from 'node:assert/strict'; import { test } from 'vitest'; +import { PNG } from '@agent-device/capture-kit/png'; import { makeSnapshotState } from '@agent-device/selectors/snapshot-geometry-fixtures'; import { createInteractionDevice, selectorSnapshot } from './__tests__/test-utils/index.ts'; test('coordinate verification keeps a changed tree digest when the frames match', async () => { - const { PNG } = await import('pngjs'); const { writeFile } = await import('node:fs/promises'); let tapped = false; const device = createInteractionDevice(selectorSnapshot(), { @@ -55,7 +55,6 @@ test.each([ ])( 'coordinate verification brackets dispatch only away from a touch target ($x, $verify, $changed)', async ({ x, verify, frames, changed }) => { - const { PNG } = await import('pngjs'); const { writeFile } = await import('node:fs/promises'); const captures: string[] = []; let tapped = false; From 0c2915cfd09fd2221743fb50cb1f7b3d3e12ede2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=B5=B5=E6=A1=82=E9=9B=84?= Date: Thu, 19 Nov 2026 13:05:46 +0800 Subject: [PATCH 04/12] fix(interaction): keep the backend receiver, name the compared-frames answer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The coordinate dispatch invokes the extracted tap with the backend as its receiver, so an in-process method reading this keeps working; a direct runtime test pins the receiver. The screenshotDiff answer names what its true means — a frame-dimension change or a difference in compared pixels — and the commands.md paragraph limits the view-match rule to tree evidence with the pixel pair as another source. Co-Authored-By: Claude Code --- .../AgentDeviceRunnerAppFixtures.m | 41 +++++++++++++++++++ .../interaction/runtime/interactions.ts | 5 ++- .../runtime/verify-screenshot.test.ts | 36 ++++++++++++++++ website/docs/docs/commands.md | 2 +- 4 files changed, 82 insertions(+), 2 deletions(-) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerAppFixtures.m b/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerAppFixtures.m index b2c46d0a0e..1062cc07ad 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerAppFixtures.m +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerAppFixtures.m @@ -221,6 +221,35 @@ - (void)viewDidLoad { } @end + +// `--agent-device-custom-drawn-canvas` adds a plain drawn view the accessibility tree never +// exposes: a tap toggles its fill, and the point over it names no element but the window. The +// screenshot-diff verification needs exactly this shape — custom-drawn content whose only answer +// is pixels, at a point no hittable or semantic node covers. +@interface AgentDeviceCustomDrawnCanvas : UIView +@end + +@implementation AgentDeviceCustomDrawnCanvas +- (instancetype)initWithFrame:(CGRect)frame { + self = [super initWithFrame:frame]; + if (self) { + self.isAccessibilityElement = NO; + self.backgroundColor = UIColor.systemGray5Color; + UITapGestureRecognizer *tap = + [[UITapGestureRecognizer alloc] initWithTarget:self action:@selector(toggled:)]; + [self addGestureRecognizer:tap]; + } + return self; +} + +- (void)toggled:(UITapGestureRecognizer *)recognizer { + (void)recognizer; + self.backgroundColor = [self.backgroundColor isEqual:UIColor.systemRedColor] + ? UIColor.systemGray5Color + : UIColor.systemRedColor; +} +@end + @interface AgentDeviceRunnerFixtureViewController () // Held for the app's lifetime: a window nobody retained would be gone before the test queried it. @property(nonatomic, strong) UIWindow *secondWindow; @@ -1466,6 +1495,18 @@ - (void)viewDidLoad { [self.view addSubview:popupTitle]; } + if ([NSProcessInfo.processInfo.arguments containsObject:@"--agent-device-custom-drawn-canvas"]) { + UIView *canvas = [[AgentDeviceCustomDrawnCanvas alloc] initWithFrame:CGRectMake(120, 300, 160, 160)]; + canvas.translatesAutoresizingMaskIntoConstraints = NO; + [self.view addSubview:canvas]; + [NSLayoutConstraint activateConstraints:@[ + [canvas.centerXAnchor constraintEqualToAnchor:self.view.centerXAnchor], + [canvas.topAnchor constraintEqualToAnchor:label.bottomAnchor constant:120], + [canvas.widthAnchor constraintEqualToConstant:160], + [canvas.heightAnchor constraintEqualToConstant:160], + ]]; + } + if ([NSProcessInfo.processInfo.arguments containsObject:@"--agent-device-crowded-screen"]) { for (NSUInteger row = 0; row < 500; row++) { UILabel *rowLabel = [[UILabel alloc] initWithFrame:CGRectMake(0, 60 + (row % 30) * 24, 400, 16)]; diff --git a/packages/command-runtime/src/commands/interaction/runtime/interactions.ts b/packages/command-runtime/src/commands/interaction/runtime/interactions.ts index 2f341ce0db..24f7313c36 100644 --- a/packages/command-runtime/src/commands/interaction/runtime/interactions.ts +++ b/packages/command-runtime/src/commands/interaction/runtime/interactions.ts @@ -181,7 +181,10 @@ async function tapCommand( const pointObservation = pointObservationOf(atRest); const dispatchTap = runtime.backend.tap; const dispatch = () => - dispatchTap(toBackendContext(runtime, options), point, { + // The backend method keeps its receiver: an in-process backend method + // that reads `this` breaks under a bare call, and the adapter's closure + // exposes none. + dispatchTap.call(runtime.backend, toBackendContext(runtime, options), point, { button: options.button, count: options.count, intervalMs: options.intervalMs, diff --git a/packages/command-runtime/src/commands/interaction/runtime/verify-screenshot.test.ts b/packages/command-runtime/src/commands/interaction/runtime/verify-screenshot.test.ts index eb82eac7df..c00204bbb7 100644 --- a/packages/command-runtime/src/commands/interaction/runtime/verify-screenshot.test.ts +++ b/packages/command-runtime/src/commands/interaction/runtime/verify-screenshot.test.ts @@ -1,6 +1,11 @@ import assert from 'node:assert/strict'; import { test } from 'vitest'; import { PNG } from '@agent-device/capture-kit/png'; +import { createLocalArtifactAdapter } from '@agent-device/host-kit/local-artifact-adapter'; +import { localCommandPolicy } from '@agent-device/contracts/command-policy'; +import { createMemorySessionStore } from '../../../command-runtime/runtime-factory.ts'; +import { createAgentDevice } from '../../../runtime.ts'; +import type { AgentDeviceBackend } from '@agent-device/contracts/backend'; import { makeSnapshotState } from '@agent-device/selectors/snapshot-geometry-fixtures'; import { createInteractionDevice, selectorSnapshot } from './__tests__/test-utils/index.ts'; @@ -82,3 +87,34 @@ test.each([ ); }, ); + +test('the coordinate dispatch keeps the backend receiver for methods that read this', async () => { + // An in-process backend whose tap is a real method: the dispatch must invoke + // it with the backend as its receiver, the way a bare call would not. + const taps: Array<{ receiver: unknown; point: { x: number; y: number } }> = []; + const backend: AgentDeviceBackend = { + platform: 'ios', + captureSnapshot: async () => ({ snapshot: selectorSnapshot() }), + async tap(context, point) { + taps.push({ receiver: this, point }); + void context; + return { ok: true }; + }, + } satisfies AgentDeviceBackend; + const device = createAgentDevice({ + backend, + artifacts: createLocalArtifactAdapter(), + sessions: createMemorySessionStore([{ name: 'default', snapshot: selectorSnapshot() }]), + policy: localCommandPolicy(), + }); + + const result = await device.interactions.press( + { kind: 'point', x: 200, y: 25 }, + { verify: false }, + ); + + assert.equal(result.kind, 'point'); + assert.equal(taps.length, 1); + assert.equal(taps[0]?.receiver, backend); + assert.deepEqual(taps[0]?.point, { x: 200, y: 25 }); +}); diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index f1111a3a04..cd662d0f9d 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -552,7 +552,7 @@ A successful `click` or `press` means the action was sent to where the command f `--settle` compares trees in the view of the refs you last read. After a full `snapshot`, it reads the full tree and issues full-tree refs; after `snapshot -i`, it reads the interactive view. It also preserves `-d`, `-s`, `--raw` and `--actions`. This holds for `press`, `click`, `fill`, `longpress`, `scroll` and `back`, including ref, selector and coordinate targets. -Standalone `--verify` captures in the pre-action tree's view, which can differ from the last ref frame when a selector resolves a fresh tree. With `--settle --verify`, verification reuses the settled capture and reports `changedFromBefore` only when the two views match or a known surface transition occurred. Without a populated baseline, post-action captures use the interactive view. When a coordinate `press` or `click` has no accessibility touch target at that point, `--verify` can include `evidence.screenshotDiff` with the post-action tree summary. The comparison excludes the top status-bar region in portrait on supported mobile devices. The change can come from any visible update between the two captures. A ref list uses tree verification for the entire sequence. +Standalone `--verify` captures in the pre-action tree's view, which can differ from the last ref frame when a selector resolves a fresh tree. With `--settle --verify`, tree verification reuses the settled capture and its `changedFromBefore` follows the tree only when the two views match or a known surface transition occurred. Without a populated baseline, post-action captures use the interactive view. When a coordinate `press` or `click` has no accessibility touch target at that point, `--verify` can include `evidence.screenshotDiff` with the post-action tree summary: the pixel pair is another source for `changedFromBefore`, and the answer is true when any comparison verification made saw a change. The comparison excludes the top status-bar region in portrait on supported mobile devices. The change can come from any visible update between the two captures. A ref list uses tree verification for the entire sequence. A `click` or `press` whose target the daemon resolves to a point (`@ref` or selector) re-checks that target's position before dispatching when the session's previous action animated the screen, and dispatches where it came to rest. When that check cannot confirm a stable position — the target was still moving, or a re-check could not measure it — the command refuses with `target_still_moving` and `dispatched: no`: nothing was dispatched, the point the command resolved is in `error.details.rect`, and where the target was last seen is in `error.details.movedToRect` when that last look measured one. Run `wait ` on the target and retry the same command, or add `--settle` so the action waits for the UI to go quiet. A refused `@ref` press needs a fresh `snapshot -i` and a selector, because the wait retires the ref. Two dispatch paths hand off before the check: a press at raw coordinates has no resolved target to re-check, and a dispatch the backend performs on the element itself — `click @ref` on a backend with element dispatch, or a Maestro non-hittable coordinate fallback — goes to that backend rather than through a resolved point. On iOS, a runtime-resolved `click` or `press` whose target is a control nested inside a larger touch control — a swipe action beside its row — also asks the runner whether the platform reports that control as hittable. When it does not, the command refuses with `target_not_hittable` and `dispatched: no`, because a tap at the control's centre would land on another element; `error.details.rect` is where the command resolved it. Maestro replay is the exception: a click that carries `allowNonHittableCoordinateFallback` skips this read, taps resolved bounds, and on the direct-iOS selector route sends the selector straight to the runner, which may use the non-hittable coordinate fallback. Reveal the control (for example swipe its row) and take a fresh `snapshot -i`. An answer the runner cannot tie to the target — no selector, an ambiguous match, another frame — lets the press proceed as before. Other presses pay no extra read. The re-check stops when the request is canceled or reaches its deadline, and nothing is dispatched (`dispatched: no`): a canceled request fails with `details.reason: request_canceled`, and a request out of time with `details.reason: daemon_request_timeout`. From 836b521107f6371b5623ac2ff5dd759bcf580709 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=B5=B5=E6=A1=82=E9=9B=84?= Date: Thu, 19 Nov 2026 21:37:26 +0800 Subject: [PATCH 05/12] fix(interaction): refuse --verify on a multi-ref press plan The pixel fallback behind --verify answers one point the pre-action tree cannot see, and every plan member is a tree-visible ref. The refusal carries a typed reason and the single-press alternative, and the press help names the refusal. Co-Authored-By: Claude Code --- .../interaction/runtime/interactions.test.ts | 19 +++++++++++++++++++ .../interaction/runtime/interactions.ts | 9 +++++++++ .../command-surface/src/interaction/index.ts | 2 +- 3 files changed, 29 insertions(+), 1 deletion(-) diff --git a/packages/command-runtime/src/commands/interaction/runtime/interactions.test.ts b/packages/command-runtime/src/commands/interaction/runtime/interactions.test.ts index 6324caeb28..4dcc69aeeb 100644 --- a/packages/command-runtime/src/commands/interaction/runtime/interactions.test.ts +++ b/packages/command-runtime/src/commands/interaction/runtime/interactions.test.ts @@ -910,6 +910,25 @@ test('runtime press plan refuses a non-ref target', async () => { ); }); +test('runtime press plan refuses --verify instead of dropping the pixel pair', async () => { + const device = createInteractionDevice(pinPadSnapshot(), { + tap: async () => ({ ok: true }), + }); + + await assert.rejects( + device.interactions.press(ref('@e1'), { + session: 'default', + refs: ['e1', 'e2'], + verify: true, + }), + (error: AppError) => { + assert.equal(error.code, 'INVALID_ARGS'); + assert.equal(error.details?.reason, 'verify-unsupported-on-ref-plan'); + return true; + }, + ); +}); + test('runtime press plan names the refused key and the pending keys when a member fails', async () => { const taps: Point[] = []; const device = createInteractionDevice(pinPadSnapshot(), { diff --git a/packages/command-runtime/src/commands/interaction/runtime/interactions.ts b/packages/command-runtime/src/commands/interaction/runtime/interactions.ts index 24f7313c36..95b128413d 100644 --- a/packages/command-runtime/src/commands/interaction/runtime/interactions.ts +++ b/packages/command-runtime/src/commands/interaction/runtime/interactions.ts @@ -309,6 +309,15 @@ async function tapRefPlanCommand( // response identity, so the plan iterates `refs` alone. const plannedRefs = (options.refs ?? []).map((ref) => (ref.startsWith('@') ? ref : `@${ref}`)); const observation = planPostActionObservation(options); + // A plan member is a tree-visible ref, and the pixel fallback behind + // --verify answers one point the pre-action tree cannot see. Refuse the + // flag instead of silently dropping the pair it promises. + if (observation.verify) { + throw new AppError('INVALID_ARGS', 'A multi-ref press plan cannot carry --verify', { + reason: 'verify-unsupported-on-ref-plan', + hint: 'Run --verify on a single press; the plan already dispatches tree-visible refs.', + }); + } const members = await resolveRefPlanMembers( runtime, options, diff --git a/packages/command-surface/src/interaction/index.ts b/packages/command-surface/src/interaction/index.ts index fbec94aedd..86c71d6bdc 100644 --- a/packages/command-surface/src/interaction/index.ts +++ b/packages/command-surface/src/interaction/index.ts @@ -507,7 +507,7 @@ const pressCommandFacet = defineCommandFacet({ summary: 'Short-press a UI target', searchHint: 'click touch button', cliDetail: - 'The hold duration is positional on longpress, not press --hold-ms. A comma-separated ref plan (press @e13,@e14,@e15,@e16; 2 to 16 refs, repeats allowed) resolves and admits every key against the current frame, then dispatches the admitted points: key 1 expires the frame, later keys dispatch where the snapshot showed them; a later refusal carries details.dispatchedSteps, details.refPlanKey, and details.refPlanPending, and sequence.points holds the dispatched points. --interval-ms sleeps between keys; --count, --hold-ms, --jitter-px, and --double-tap apply to every key; the response ref is the first key. Recording refuses the plan.', + 'The hold duration is positional on longpress, not press --hold-ms. A comma-separated ref plan (press @e13,@e14,@e15,@e16; 2 to 16 refs, repeats allowed) resolves and admits every key against the current frame, then dispatches the admitted points: key 1 expires the frame, later keys dispatch where the snapshot showed them; a later refusal carries details.dispatchedSteps, details.refPlanKey, and details.refPlanPending, and sequence.points holds the dispatched points. --interval-ms sleeps between keys; --count, --hold-ms, --jitter-px, and --double-tap apply to every key; the response ref is the first key. --verify refuses a ref plan; run it on a single press. Recording refuses the plan.', }, metadata: metadata('press'), run: (client, input) => client.interactions.press(toPressOptions(input)), From 28c408b947135118c34ca701a465251e9860db75 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=B5=B5=E6=A1=82=E9=9B=84?= Date: Fri, 20 Nov 2026 22:50:10 +0800 Subject: [PATCH 06/12] feat(interaction): exempt the viewport-filling container from the pixel-pair gate The --verify tree-coverage gate blocked every real point: each pre-action tree carries a full-screen hittable container (Android content FrameLayout, iOS full-screen CollectionView) that isViewportRootNode never matches, so coordinate press/click --verify never produced evidence.screenshotDiff on a device. Per the maintainer ruling (option a), a covering node is exempt when its rect contains the point's resolved viewport (resolveViewportRect, including its largest-containing-rect fallback); semantic touch targets and hittable nodes smaller than that viewport keep blocking. The screenshotDiff.changedFromBefore schema text now names the dimension_mismatch branch compareFrames can return: true can report a frame-dimension change alone, with changedPixelRatio absent, so agents do not read a rotation as a pixel difference. commands.md now states that --verify refuses a ref list (the refusal landed in 625af0aa9) instead of claiming tree verification for the whole sequence. Gate unit tests cover the maintainer's four cases, including the Redmi K60 Edge tree through the largest-containing-rect fallback, plus the literal ruling shape (a scrim whose rect contains the whole viewport). --- .../interaction/runtime/interactions.ts | 42 +++-- .../runtime/verify-screenshot.test.ts | 144 ++++++++++++++++++ .../command-surface/src/interaction/index.ts | 4 +- website/docs/docs/commands.md | 2 +- 4 files changed, 180 insertions(+), 12 deletions(-) diff --git a/packages/command-runtime/src/commands/interaction/runtime/interactions.ts b/packages/command-runtime/src/commands/interaction/runtime/interactions.ts index 95b128413d..14147d8ff0 100644 --- a/packages/command-runtime/src/commands/interaction/runtime/interactions.ts +++ b/packages/command-runtime/src/commands/interaction/runtime/interactions.ts @@ -9,12 +9,16 @@ import type { ScreenshotVerification, } from '@agent-device/contracts/interaction'; import { isSemanticTouchTarget } from '@agent-device/selectors/touch-semantics'; -import { containsPoint } from '@agent-device/kernel/rect'; +import { containsPoint, rectContains } from '@agent-device/kernel/rect'; import { AppError } from '@agent-device/kernel/errors'; import { SELECTOR_PIPELINE_POLICIES } from '@agent-device/selectors/selector-pipeline-policy'; -import type { Point } from '@agent-device/kernel/snapshot'; +import type { Point, SnapshotNode } from '@agent-device/kernel/snapshot'; import type { AgentDeviceRuntime, CommandContext } from '@agent-device/contracts/runtime-contract'; -import { isFillableType, isViewportRootNode } from '@agent-device/contracts/snapshot'; +import { + isFillableType, + isViewportRootNode, + resolveViewportRect, +} from '@agent-device/contracts/snapshot'; import { attachResolvedInteractionTarget } from '@agent-device/host-kit/interaction-outcome'; import { sleep, toBackendContext } from '../../runtime-common.ts'; import { toBackendResult, type RuntimeCommand } from '../../runtime-types.ts'; @@ -202,13 +206,7 @@ async function tapCommand( atRest.kind === 'point' && baseline !== undefined && runtime.backend.captureVerifyScreenshot !== undefined && - !baseline.nodes.some( - (node) => - node.rect && - !isViewportRootNode(node) && - (node.hittable === true || isSemanticTouchTarget(node)) && - containsPoint(node.rect, point.x, point.y), - ); + !baseline.nodes.some((node) => blocksPixelPairAtPoint(baseline.nodes, node, point)); if (useScreenshot) { const { withScreenshotVerification } = await import('./verify-screenshot.ts'); ({ result: backendResult, screenshot } = await withScreenshotVerification( @@ -464,3 +462,27 @@ function formatTargetForWarning(result: { if (result.target?.kind === 'selector') return result.target.selector; return 'point'; } + +/** + * Whether a pre-action tree node keeps the coordinate pixel pair off at `point` + * (the `--verify` screenshot fallback's tree-coverage gate). + * + * A node blocks when it covers the point and is a semantic touch target, or is + * hittable and smaller than the point's resolved viewport. The exemption is + * the ruling on #4297 (option a): a hittable node whose rect CONTAINS the + * point's resolved viewport — `resolveViewportRect`, including its + * largest-containing-rect fallback, which is what answers on Android where no + * window nodes exist — is the viewport-filling container every real tree + * carries (Android content `FrameLayout`, iOS full-screen `CollectionView`), + * not a touch target; excluding it keeps the gate reachable without a + * per-platform type list. A full-screen marked button still blocks: semantic + * targets never take the exemption. + */ +function blocksPixelPairAtPoint(baselineNodes: SnapshotNode[], node: SnapshotNode, point: Point): boolean { + if (!node.rect || !containsPoint(node.rect, point.x, point.y)) return false; + if (isViewportRootNode(node)) return false; + if (isSemanticTouchTarget(node)) return true; + if (node.hittable !== true) return false; + const viewport = resolveViewportRect(baselineNodes, node.rect); + return !rectContains(node.rect, viewport ?? node.rect); +} diff --git a/packages/command-runtime/src/commands/interaction/runtime/verify-screenshot.test.ts b/packages/command-runtime/src/commands/interaction/runtime/verify-screenshot.test.ts index c00204bbb7..b99c1a7b27 100644 --- a/packages/command-runtime/src/commands/interaction/runtime/verify-screenshot.test.ts +++ b/packages/command-runtime/src/commands/interaction/runtime/verify-screenshot.test.ts @@ -1,6 +1,7 @@ import assert from 'node:assert/strict'; import { test } from 'vitest'; import { PNG } from '@agent-device/capture-kit/png'; +import type { RawSnapshotNode } from '@agent-device/kernel/snapshot'; import { createLocalArtifactAdapter } from '@agent-device/host-kit/local-artifact-adapter'; import { localCommandPolicy } from '@agent-device/contracts/command-policy'; import { createMemorySessionStore } from '../../../command-runtime/runtime-factory.ts'; @@ -118,3 +119,146 @@ test('the coordinate dispatch keeps the backend receiver for methods that read t assert.equal(taps[0]?.receiver, backend); assert.deepEqual(taps[0]?.point, { x: 200, y: 25 }); }); + +// The tree-coverage gate admits the pixel pair only when no covering node is a +// semantic touch target, or hittable and smaller than the point's resolved +// viewport. The ruling on #4297 (option a): the viewport-filling hittable +// container every real tree carries is exempt, so the gate stays reachable — +// the four cases below are the maintainer's list. +function node(overrides: { + index: number; + type: string; + rect: { x: number; y: number; width: number; height: number }; + hittable?: boolean; + parentIndex?: number; + label?: string; +}): RawSnapshotNode { + return { ...overrides }; +} + +async function pressWithGateTree( + nodes: RawSnapshotNode[], + point: { x: number; y: number }, +): Promise<{ pairRan: boolean }> { + const { writeFile } = await import('node:fs/promises'); + let pairRan = false; + let tapped = false; + const device = createInteractionDevice(makeSnapshotState(nodes), { + tap: async () => { + tapped = true; + }, + clock: { now: () => 0, sleep: async () => {} }, + captureVerifyScreenshot: async (outPath) => { + pairRan = true; + const png = new PNG({ width: 4, height: 4 }); + png.data.fill(255); + if (tapped) png.data[16] = 0; + await writeFile(outPath, PNG.sync.write(png)); + return { ignoreTopHeightShare: 0.25 }; + }, + }); + const result = await device.interactions.press({ kind: 'point', ...point }, { verify: true }); + assert.ok(result.evidence); + return { pairRan }; +} + +test('gate admits the pair past a full-screen hittable container that IS the viewport', async () => { + // Android content FrameLayout shape: no window node, so the resolved + // viewport is the largest-containing-rect fallback — the container itself. + const { pairRan } = await pressWithGateTree( + [ + node({ index: 0, type: 'FrameLayout', rect: { x: 0, y: 94, width: 1080, height: 2306 }, hittable: true }), + node({ index: 1, type: 'Text', rect: { x: 40, y: 300, width: 400, height: 40 } }), + ], + { x: 540, y: 1200 }, + ); + assert.equal(pairRan, true); +}); + +test('gate admits the iOS full-screen CollectionView and still blocks its covering cells', async () => { + // iOS Settings shape: the full-screen CollectionView is hittable and IS the + // viewport (exempt), so a point only it covers opens the pair; the Cell and + // Button under the same point are smaller than the viewport, so they keep + // blocking and the pair never runs. + const collectionView = node({ + index: 0, + type: 'CollectionView', + rect: { x: 0, y: 0, width: 440, height: 956 }, + hittable: true, + }); + const windowNode = node({ index: 2, type: 'Window', rect: { x: 0, y: 0, width: 440, height: 956 } }); + const opened = await pressWithGateTree([collectionView, windowNode], { x: 200, y: 700 }); + assert.equal(opened.pairRan, true); + + const cellUnderThePoint = node({ + index: 1, + type: 'Cell', + rect: { x: 0, y: 660, width: 440, height: 80 }, + hittable: true, + parentIndex: 0, + label: 'General', + }); + const blocked = await pressWithGateTree( + [collectionView, cellUnderThePoint, windowNode], + { x: 200, y: 700 }, + ); + assert.equal(blocked.pairRan, false); +}); + +test('gate keeps a full-screen semantic touch target blocking', async () => { + const { pairRan } = await pressWithGateTree( + [ + node({ index: 0, type: 'Button', rect: { x: 0, y: 0, width: 1080, height: 2400 }, hittable: true, label: 'Continue' }), + ], + { x: 540, y: 1200 }, + ); + assert.equal(pairRan, false); +}); + +test('gate exempts a hittable scrim whose rect contains the whole viewport', async () => { + // The ruling's literal shape: the exempting direction is "the node's rect + // contains the resolved viewport", so a full-screen scrim drawn beyond the + // window bounds (oversized during a scale animation) still opens the pair. + const { pairRan } = await pressWithGateTree( + [ + node({ index: 0, type: 'Window', rect: { x: 0, y: 0, width: 440, height: 956 } }), + node({ index: 1, type: 'View', rect: { x: -100, y: -100, width: 640, height: 1156 }, hittable: true }), + ], + { x: 200, y: 700 }, + ); + assert.equal(pairRan, true); +}); + +test('gate keeps a smaller hittable node under the point inside the container blocking', async () => { + const { pairRan } = await pressWithGateTree( + [ + node({ index: 0, type: 'FrameLayout', rect: { x: 0, y: 94, width: 1080, height: 2306 }, hittable: true }), + node({ + index: 1, + type: 'View', + rect: { x: 400, y: 1100, width: 280, height: 200 }, + hittable: true, + parentIndex: 0, + }), + ], + { x: 540, y: 1200 }, + ); + assert.equal(pairRan, false); +}); + +test('gate admits the pair on the Redmi K60 Edge tree through the largest-containing-rect fallback', async () => { + // The captured Edge tree (android-helper, #4297 2026-11-19): no window + // nodes and no root node (Android emits neither), Java class names + // isViewportRootNode never matches, and the content-root FrameLayout + // 0,94 1080x2306 is hittable — the largest containing rect, so it IS the + // resolved viewport. Under the old gate every point on this tree closed + // the pixel pair. + const edgeTreeNodes: RawSnapshotNode[] = [ + node({ index: 0, type: 'android.widget.FrameLayout', rect: { x: 0, y: 94, width: 1080, height: 2306 }, hittable: true }), + node({ index: 1, type: 'android.view.View', rect: { x: 0, y: 94, width: 1080, height: 120 }, parentIndex: 0 }), + node({ index: 2, type: 'android.widget.EditText', rect: { x: 40, y: 140, width: 600, height: 60 }, parentIndex: 0, label: 'Search or type URL' }), + node({ index: 3, type: 'android.view.View', rect: { x: 0, y: 260, width: 1080, height: 2140 }, parentIndex: 0 }), + ]; + const { pairRan } = await pressWithGateTree(edgeTreeNodes, { x: 540, y: 1200 }); + assert.equal(pairRan, true); +}); diff --git a/packages/command-surface/src/interaction/index.ts b/packages/command-surface/src/interaction/index.ts index 86c71d6bdc..0832d09429 100644 --- a/packages/command-surface/src/interaction/index.ts +++ b/packages/command-surface/src/interaction/index.ts @@ -80,7 +80,9 @@ const interactionEvidenceSchema: JsonSchema = objectSchema( surfaceChange: postActionSurfaceChangeSchema, screenshotDiff: objectSchema( { - changedFromBefore: booleanSchema('Whether the compared screen pixels changed.'), + changedFromBefore: booleanSchema( + 'Whether the frame dimensions differed or the compared pixels differed. true can report a dimension change alone: a rotation, for example. changedPixelRatio is absent then, because no pixels were compared.', + ), changedPixelRatio: numberSchema( 'Share of compared pixels that changed, when frame dimensions match.', ), diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index cd662d0f9d..6361de9df2 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -552,7 +552,7 @@ A successful `click` or `press` means the action was sent to where the command f `--settle` compares trees in the view of the refs you last read. After a full `snapshot`, it reads the full tree and issues full-tree refs; after `snapshot -i`, it reads the interactive view. It also preserves `-d`, `-s`, `--raw` and `--actions`. This holds for `press`, `click`, `fill`, `longpress`, `scroll` and `back`, including ref, selector and coordinate targets. -Standalone `--verify` captures in the pre-action tree's view, which can differ from the last ref frame when a selector resolves a fresh tree. With `--settle --verify`, tree verification reuses the settled capture and its `changedFromBefore` follows the tree only when the two views match or a known surface transition occurred. Without a populated baseline, post-action captures use the interactive view. When a coordinate `press` or `click` has no accessibility touch target at that point, `--verify` can include `evidence.screenshotDiff` with the post-action tree summary: the pixel pair is another source for `changedFromBefore`, and the answer is true when any comparison verification made saw a change. The comparison excludes the top status-bar region in portrait on supported mobile devices. The change can come from any visible update between the two captures. A ref list uses tree verification for the entire sequence. +Standalone `--verify` captures in the pre-action tree's view, which can differ from the last ref frame when a selector resolves a fresh tree. With `--settle --verify`, tree verification reuses the settled capture and its `changedFromBefore` follows the tree only when the two views match or a known surface transition occurred. Without a populated baseline, post-action captures use the interactive view. When a coordinate `press` or `click` has no accessibility touch target at that point, `--verify` can include `evidence.screenshotDiff` with the post-action tree summary: the pixel pair is another source for `changedFromBefore`, and the answer is true when any comparison verification made saw a change. The comparison excludes the top status-bar region in portrait on supported mobile devices. The change can come from any visible update between the two captures. --verify refuses a ref list; verify a single press. A `click` or `press` whose target the daemon resolves to a point (`@ref` or selector) re-checks that target's position before dispatching when the session's previous action animated the screen, and dispatches where it came to rest. When that check cannot confirm a stable position — the target was still moving, or a re-check could not measure it — the command refuses with `target_still_moving` and `dispatched: no`: nothing was dispatched, the point the command resolved is in `error.details.rect`, and where the target was last seen is in `error.details.movedToRect` when that last look measured one. Run `wait ` on the target and retry the same command, or add `--settle` so the action waits for the UI to go quiet. A refused `@ref` press needs a fresh `snapshot -i` and a selector, because the wait retires the ref. Two dispatch paths hand off before the check: a press at raw coordinates has no resolved target to re-check, and a dispatch the backend performs on the element itself — `click @ref` on a backend with element dispatch, or a Maestro non-hittable coordinate fallback — goes to that backend rather than through a resolved point. On iOS, a runtime-resolved `click` or `press` whose target is a control nested inside a larger touch control — a swipe action beside its row — also asks the runner whether the platform reports that control as hittable. When it does not, the command refuses with `target_not_hittable` and `dispatched: no`, because a tap at the control's centre would land on another element; `error.details.rect` is where the command resolved it. Maestro replay is the exception: a click that carries `allowNonHittableCoordinateFallback` skips this read, taps resolved bounds, and on the direct-iOS selector route sends the selector straight to the runner, which may use the non-hittable coordinate fallback. Reveal the control (for example swipe its row) and take a fresh `snapshot -i`. An answer the runner cannot tie to the target — no selector, an ambiguous match, another frame — lets the press proceed as before. Other presses pay no extra read. The re-check stops when the request is canceled or reaches its deadline, and nothing is dispatched (`dispatched: no`): a canceled request fails with `details.reason: request_canceled`, and a request out of time with `details.reason: daemon_request_timeout`. From 71bfaf56b4168165c6e88ae27308a1f668cc4810 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=B5=B5=E6=A1=82=E9=9B=84?= Date: Fri, 20 Nov 2026 23:00:56 +0800 Subject: [PATCH 07/12] docs(evidence): re-record the verify fallback pairs at the gate-exemption head iOS: the custom-drawn canvas fixture on an iPhone 17 Pro Max simulator (source runner with the unit-test fixture define, launched with --agent-device-custom-drawn-canvas). A forced status-bar battery change mid-dispatch stays excluded (aggregate false, ratio 0); a canvas redraw reads true at 0.065, matching the canvas's 6.1% frame share. Android: Edge on the Redmi K60 serving the rig page over adb reverse. The WebView container is the covering hittable node and IS the resolved viewport, so the exemption opens the pair the pre-exemption gate closed; a forced battery change reads false with ratio 0, and a canvas redraw reads true at 0.155. --- ...d-verify-screenshot-fallback-2026-11-20.md | 115 ++++++++++++++++++ ...s-verify-screenshot-fallback-2026-11-20.md | 115 ++++++++++++++++++ 2 files changed, 230 insertions(+) create mode 100644 docs/evidence/android-verify-screenshot-fallback-2026-11-20.md create mode 100644 docs/evidence/ios-verify-screenshot-fallback-2026-11-20.md diff --git a/docs/evidence/android-verify-screenshot-fallback-2026-11-20.md b/docs/evidence/android-verify-screenshot-fallback-2026-11-20.md new file mode 100644 index 0000000000..4dbc76ddbb --- /dev/null +++ b/docs/evidence/android-verify-screenshot-fallback-2026-11-20.md @@ -0,0 +1,115 @@ +# Android `--verify` screenshot fallback through the full-screen-container exemption, 2026-11-20 + +Evidence for #4297 at the gate-exemption head `28c408b94`. Device: Redmi +K60 (`2312DRAABC`, MIUI Android 14, 1080x2400), Edge (`com.microsoft.emmx`) +serving the local rig page over `adb reverse tcp:8642`, backend +`android-helper` (`snapshotQuality.backend: "android-helper"`). The battery +level was forced to 37 with `adb shell dumpsys battery set level 37` about +400 ms into the press and reset right after, so the only candidate change +besides the page itself was the status-bar glyph. + +## The tree that used to close the gate, and the exemption that opens it + +Interactive snapshot at the rig page (23 nodes; header rows trimmed): + +```json +{"ref": "e1", "type": "android.webkit.WebView", "hittable": true, + "rect": {"x": 0, "y": 237, "width": 1080, "height": 1976}}, +{"ref": "e3", "type": "android.widget.Button", "hittable": true, + "rect": {"x": 385, "y": 253, "width": 310, "height": 143}, "label": "TAB A"}, +{"ref": "e4", "type": "android.widget.TextView", "hittable": null, + "rect": {"x": 0, "y": 237, "width": 1080, "height": 654}, "label": "static body"} +``` + +No window or application node exists (Android emits none), so the point's +resolved viewport comes from `resolveViewportRect`'s largest-containing-rect +fallback. At the dead point `(540, 1500)` the only covering hittable node +is the WebView container, and its rect IS the resolved viewport — the +option-(a) exemption admits the pair. Under the pre-exemption gate this +same tree and point produced tree evidence with no `screenshotDiff` (the +gate closed on every real point). + +## Run 1 — dead tap with a forced status-bar change + +`press 540 1500 --verify --json` on the rig's dead area, battery forced +mid-dispatch: + +```json +{ + "success": true, + "data": { + "x": 540, + "y": 1500, + "message": "Tapped (540, 1500)", + "targetKind": "point", + "evidence": { + "digest": "ax1:f017ea1bdbba74a8", + "nodeCount": 23, + "interactiveNodeCount": 23, + "foregroundApp": "com.microsoft.emmx", + "changedFromBefore": false, + "screenshotDiff": { + "changedFromBefore": false, + "changedPixelRatio": 0 + } + } + } +} +``` + +`changedFromBefore: false`, ratio 0 — the forced change landed between the +frames (the battery read 37 during the window and 100 again after the +reset), the 4% portrait band excluded it, and the outside-the-status-bar +comparison saw an identical frame. A post-run screenshot shows the status +bar back at 100% and the page unchanged. + +A second dead run on the MIUI launcher (`com.miui.home`, `press 540 700 +--verify --json`, same forced change) read the same shape: +`changedFromBefore: false`, `screenshotDiff {false, 0}`, tree digest +`ax1:36d6f2391b33939c`. + +## Run 2 — a tap that redraws custom-drawn content + +The rig page's second version carries a 280x180 canvas (a plain drawn +surface the tree reports as a hittable-less `android.widget.Image` at +`154,391 772x495` with no semantics). `press 540 638 --verify --json` +taps it; its click handler repaints the canvas gray→red and rewrites the +state row: + +```json +{ + "success": true, + "data": { + "x": 540, + "y": 638, + "message": "Tapped (540, 638)", + "targetKind": "point", + "evidence": { + "digest": "ax1:8832c6a9aa185a47", + "nodeCount": 23, + "interactiveNodeCount": 23, + "foregroundApp": "com.microsoft.emmx", + "changedFromBefore": true, + "screenshotDiff": { + "changedFromBefore": true, + "changedPixelRatio": 0.15493023405349796 + } + } + } +} +``` + +Aggregate true with `screenshotDiff {changedFromBefore: true, +changedPixelRatio: 0.155}`. A post-run screenshot shows the canvas red with +`STATE=RED`, matching the pixel answer. + +## Notes + +- Backend: `android-helper` (snapshot helper 0.21.42-dev, built from this + tree); screenshots go through the helper's capture with + `rotationFacts(device, ...)` declaring the band. +- The tap at `(540, 638)` covers one non-blocking candidate (`e3` shape on + the first rig: the WebView only). On the second rig the canvas node is + not hittable and names no touch role, so the gate stays open there too. +- Battery reset and status bar restored after the runs; the rig server was + an unrelated loopback process, left running for reproduction. diff --git a/docs/evidence/ios-verify-screenshot-fallback-2026-11-20.md b/docs/evidence/ios-verify-screenshot-fallback-2026-11-20.md new file mode 100644 index 0000000000..cd794411e0 --- /dev/null +++ b/docs/evidence/ios-verify-screenshot-fallback-2026-11-20.md @@ -0,0 +1,115 @@ +# iOS `--verify` screenshot fallback on the custom-drawn canvas fixture, 2026-11-20 + +Evidence for #4297 at the gate-exemption head `28c408b94` (the only later +commit is this docs change). The runner fixture is the source-built +unit-test runner (`AGENT_DEVICE_XCUITEST_INCLUDE_UNIT_TESTS=1`), installed +on the iPhone 17 Pro Max simulator (`F8658A47-112D-48F1-8D18-226A74F39689`, +iOS 26.5, Xcode 27.2) and launched with `--agent-device-custom-drawn-canvas` +through `open --launch-args`. The fixture adds a 160x160 plain UIView the +accessibility tree never exposes: a tap toggles its fill between systemGray5 +and systemRed. + +## Pre-action tree: no hittable or semantic node covers the canvas point + +Full snapshot at the session app: + +```json +[ + {"index": 0, "type": "Application", "role": "UIApplication", "label": "AgentDeviceRunner", + "rect": {"x": 0, "y": 0, "width": 440, "height": 956}, "enabled": true, "depth": 0, + "pid": 92141, "hittable": false, "ref": "e1", "kind": "application"}, + {"index": 1, "parentIndex": 0, "type": "StaticText", "role": "UILabel", + "label": "Agent Device Runner", + "rect": {"x": 120.0, "y": 465, "width": 200.3, "height": 26.3}, "enabled": true, + "depth": 1, "pid": 92141, "hittable": true, "ref": "e2", "kind": "text"} +] +``` + +The canvas (roughly 140..300 x 611..771 pt, centered under the label) names +no element: the point `(220, 720)` is covered only by the Application node +itself (the viewport root, not hittable). The StaticText does not cover it. +The tree-coverage gate has no covering touch target to block on, so the +pixel pair is admitted and the screenshot fallback runs. + +## Run 1 — status-bar-only change between the frames + +`press 220 720 --verify --json` is a canvas tap; to make the only other +candidate change a status-bar glyph, the simulator's battery level was +forced to 37 about 500 ms into the press (`xcrun simctl status_bar +override --batteryLevel 37`) and cleared right after. The band (6% of the +portrait frame, declared by the apple capture owner) excludes that region, +so the forced change must not surface: + +```json +{ + "success": true, + "data": { + "x": 220, + "y": 720, + "message": "Tapped (220, 720)", + "runnerMainThreadBusy": false, + "targetKind": "point", + "evidence": { + "digest": "ax1:aaf591869124d41c", + "nodeCount": 3, + "interactiveNodeCount": 2, + "changedFromBefore": false, + "screenshotDiff": { + "changedFromBefore": false, + "changedPixelRatio": 0 + } + } + } +} +``` + +Aggregate `changedFromBefore` is false (no comparison verification saw a +change) and the pixel-only answer is `screenshotDiff {changedFromBefore: +false, changedPixelRatio: 0}` — the forced status-bar redraw landed inside +the excluded band and the outside-the-status-bar comparison saw an +identical frame. + +## Run 2 — a tap that redraws the canvas + +`press 220 720 --verify --json` again, this time with no status-bar +override. The tap lands on the custom-drawn view and toggles its fill: + +```json +{ + "success": true, + "data": { + "x": 220, + "y": 720, + "message": "Tapped (220, 720)", + "runnerMainThreadBusy": false, + "targetKind": "point", + "evidence": { + "digest": "ax1:aaf591869124d41c", + "nodeCount": 3, + "interactiveNodeCount": 2, + "changedFromBefore": true, + "screenshotDiff": { + "changedFromBefore": true, + "changedPixelRatio": 0.06512286378804732 + } + } + } +} +``` + +Aggregate true with `screenshotDiff {changedFromBefore: true, +changedPixelRatio: 0.0651}` — the canvas (160/440 x 160/956 ≈ 6.1% of the +frame) turned red, and the ratio matches that share. + +## Notes + +- Backend: XCTest runner (source build with the unit-test fixture define), + session app `com.callstack.agentdevice.runner` (pid 92141 at snapshot). +- The capture binding is the production one: the daemon's + `createVerifyScreenshotCapture` binds the screenshot lane for the session + (`captureScreenshotFromRequest` with the session surface, fullscreen for + a portrait iOS target) — no flag route, no test-injected capture. +- The iOS dead-region run at `(60, 820)` (left margin, no node under it) + with the same forced battery change read `changedFromBefore: false` with + `screenshotDiff {false, 0}` as well. +- Status bar cleared after the runs; the simulator was left booted. From 9c8f0c8741615325ce938c9467fef204fce39704 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=B5=B5=E6=A1=82=E9=9B=84?= Date: Fri, 20 Nov 2026 23:06:45 +0800 Subject: [PATCH 08/12] style: format the gate and its tests --- .../interaction/runtime/interactions.ts | 6 +- .../runtime/verify-screenshot.test.ts | 72 +++++++++++++++---- 2 files changed, 64 insertions(+), 14 deletions(-) diff --git a/packages/command-runtime/src/commands/interaction/runtime/interactions.ts b/packages/command-runtime/src/commands/interaction/runtime/interactions.ts index 14147d8ff0..cbbab309be 100644 --- a/packages/command-runtime/src/commands/interaction/runtime/interactions.ts +++ b/packages/command-runtime/src/commands/interaction/runtime/interactions.ts @@ -478,7 +478,11 @@ function formatTargetForWarning(result: { * per-platform type list. A full-screen marked button still blocks: semantic * targets never take the exemption. */ -function blocksPixelPairAtPoint(baselineNodes: SnapshotNode[], node: SnapshotNode, point: Point): boolean { +function blocksPixelPairAtPoint( + baselineNodes: SnapshotNode[], + node: SnapshotNode, + point: Point, +): boolean { if (!node.rect || !containsPoint(node.rect, point.x, point.y)) return false; if (isViewportRootNode(node)) return false; if (isSemanticTouchTarget(node)) return true; diff --git a/packages/command-runtime/src/commands/interaction/runtime/verify-screenshot.test.ts b/packages/command-runtime/src/commands/interaction/runtime/verify-screenshot.test.ts index b99c1a7b27..1b377a1d75 100644 --- a/packages/command-runtime/src/commands/interaction/runtime/verify-screenshot.test.ts +++ b/packages/command-runtime/src/commands/interaction/runtime/verify-screenshot.test.ts @@ -167,7 +167,12 @@ test('gate admits the pair past a full-screen hittable container that IS the vie // viewport is the largest-containing-rect fallback — the container itself. const { pairRan } = await pressWithGateTree( [ - node({ index: 0, type: 'FrameLayout', rect: { x: 0, y: 94, width: 1080, height: 2306 }, hittable: true }), + node({ + index: 0, + type: 'FrameLayout', + rect: { x: 0, y: 94, width: 1080, height: 2306 }, + hittable: true, + }), node({ index: 1, type: 'Text', rect: { x: 40, y: 300, width: 400, height: 40 } }), ], { x: 540, y: 1200 }, @@ -186,7 +191,11 @@ test('gate admits the iOS full-screen CollectionView and still blocks its coveri rect: { x: 0, y: 0, width: 440, height: 956 }, hittable: true, }); - const windowNode = node({ index: 2, type: 'Window', rect: { x: 0, y: 0, width: 440, height: 956 } }); + const windowNode = node({ + index: 2, + type: 'Window', + rect: { x: 0, y: 0, width: 440, height: 956 }, + }); const opened = await pressWithGateTree([collectionView, windowNode], { x: 200, y: 700 }); assert.equal(opened.pairRan, true); @@ -198,17 +207,23 @@ test('gate admits the iOS full-screen CollectionView and still blocks its coveri parentIndex: 0, label: 'General', }); - const blocked = await pressWithGateTree( - [collectionView, cellUnderThePoint, windowNode], - { x: 200, y: 700 }, - ); + const blocked = await pressWithGateTree([collectionView, cellUnderThePoint, windowNode], { + x: 200, + y: 700, + }); assert.equal(blocked.pairRan, false); }); test('gate keeps a full-screen semantic touch target blocking', async () => { const { pairRan } = await pressWithGateTree( [ - node({ index: 0, type: 'Button', rect: { x: 0, y: 0, width: 1080, height: 2400 }, hittable: true, label: 'Continue' }), + node({ + index: 0, + type: 'Button', + rect: { x: 0, y: 0, width: 1080, height: 2400 }, + hittable: true, + label: 'Continue', + }), ], { x: 540, y: 1200 }, ); @@ -222,7 +237,12 @@ test('gate exempts a hittable scrim whose rect contains the whole viewport', asy const { pairRan } = await pressWithGateTree( [ node({ index: 0, type: 'Window', rect: { x: 0, y: 0, width: 440, height: 956 } }), - node({ index: 1, type: 'View', rect: { x: -100, y: -100, width: 640, height: 1156 }, hittable: true }), + node({ + index: 1, + type: 'View', + rect: { x: -100, y: -100, width: 640, height: 1156 }, + hittable: true, + }), ], { x: 200, y: 700 }, ); @@ -232,7 +252,12 @@ test('gate exempts a hittable scrim whose rect contains the whole viewport', asy test('gate keeps a smaller hittable node under the point inside the container blocking', async () => { const { pairRan } = await pressWithGateTree( [ - node({ index: 0, type: 'FrameLayout', rect: { x: 0, y: 94, width: 1080, height: 2306 }, hittable: true }), + node({ + index: 0, + type: 'FrameLayout', + rect: { x: 0, y: 94, width: 1080, height: 2306 }, + hittable: true, + }), node({ index: 1, type: 'View', @@ -254,10 +279,31 @@ test('gate admits the pair on the Redmi K60 Edge tree through the largest-contai // resolved viewport. Under the old gate every point on this tree closed // the pixel pair. const edgeTreeNodes: RawSnapshotNode[] = [ - node({ index: 0, type: 'android.widget.FrameLayout', rect: { x: 0, y: 94, width: 1080, height: 2306 }, hittable: true }), - node({ index: 1, type: 'android.view.View', rect: { x: 0, y: 94, width: 1080, height: 120 }, parentIndex: 0 }), - node({ index: 2, type: 'android.widget.EditText', rect: { x: 40, y: 140, width: 600, height: 60 }, parentIndex: 0, label: 'Search or type URL' }), - node({ index: 3, type: 'android.view.View', rect: { x: 0, y: 260, width: 1080, height: 2140 }, parentIndex: 0 }), + node({ + index: 0, + type: 'android.widget.FrameLayout', + rect: { x: 0, y: 94, width: 1080, height: 2306 }, + hittable: true, + }), + node({ + index: 1, + type: 'android.view.View', + rect: { x: 0, y: 94, width: 1080, height: 120 }, + parentIndex: 0, + }), + node({ + index: 2, + type: 'android.widget.EditText', + rect: { x: 40, y: 140, width: 600, height: 60 }, + parentIndex: 0, + label: 'Search or type URL', + }), + node({ + index: 3, + type: 'android.view.View', + rect: { x: 0, y: 260, width: 1080, height: 2140 }, + parentIndex: 0, + }), ]; const { pairRan } = await pressWithGateTree(edgeTreeNodes, { x: 540, y: 1200 }); assert.equal(pairRan, true); From b7b7dfd5fa445a97337c8ea185d882e4874fcc6b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=B5=B5=E6=A1=82=E9=9B=84?= Date: Fri, 20 Nov 2026 23:26:39 +0800 Subject: [PATCH 09/12] fix(interaction): publish the ref-plan verify refusal from the typed reason table R89 (upstream #4779) ratchets wire-published reason literals by value per file against the merge-base. The verify-unsupported-on-ref-plan refusal typed its reason at the call site; move it into the shared INTERACTION_ERROR_REASONS table and read it through the member expression, and compress the screenshotDiff.changedFromBefore schema text to hold the always-loaded byte budget that the longer text broke on CI. --- .../src/commands/interaction/runtime/interactions.ts | 3 ++- packages/command-surface/src/interaction/index.ts | 2 +- packages/selectors/src/interaction-error.ts | 8 ++++++++ 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/packages/command-runtime/src/commands/interaction/runtime/interactions.ts b/packages/command-runtime/src/commands/interaction/runtime/interactions.ts index cbbab309be..b04b6d1856 100644 --- a/packages/command-runtime/src/commands/interaction/runtime/interactions.ts +++ b/packages/command-runtime/src/commands/interaction/runtime/interactions.ts @@ -9,6 +9,7 @@ import type { ScreenshotVerification, } from '@agent-device/contracts/interaction'; import { isSemanticTouchTarget } from '@agent-device/selectors/touch-semantics'; +import { INTERACTION_ERROR_REASONS } from '@agent-device/selectors/interaction-error'; import { containsPoint, rectContains } from '@agent-device/kernel/rect'; import { AppError } from '@agent-device/kernel/errors'; import { SELECTOR_PIPELINE_POLICIES } from '@agent-device/selectors/selector-pipeline-policy'; @@ -312,7 +313,7 @@ async function tapRefPlanCommand( // flag instead of silently dropping the pair it promises. if (observation.verify) { throw new AppError('INVALID_ARGS', 'A multi-ref press plan cannot carry --verify', { - reason: 'verify-unsupported-on-ref-plan', + reason: INTERACTION_ERROR_REASONS.verifyUnsupportedOnRefPlan, hint: 'Run --verify on a single press; the plan already dispatches tree-visible refs.', }); } diff --git a/packages/command-surface/src/interaction/index.ts b/packages/command-surface/src/interaction/index.ts index 0832d09429..45e9a94c0f 100644 --- a/packages/command-surface/src/interaction/index.ts +++ b/packages/command-surface/src/interaction/index.ts @@ -81,7 +81,7 @@ const interactionEvidenceSchema: JsonSchema = objectSchema( screenshotDiff: objectSchema( { changedFromBefore: booleanSchema( - 'Whether the frame dimensions differed or the compared pixels differed. true can report a dimension change alone: a rotation, for example. changedPixelRatio is absent then, because no pixels were compared.', + 'Whether the frame dimensions differed or the compared pixels differed; changedPixelRatio is absent on a dimension change.', ), changedPixelRatio: numberSchema( 'Share of compared pixels that changed, when frame dimensions match.', diff --git a/packages/selectors/src/interaction-error.ts b/packages/selectors/src/interaction-error.ts index a49f3e8f54..04a5034033 100644 --- a/packages/selectors/src/interaction-error.ts +++ b/packages/selectors/src/interaction-error.ts @@ -49,6 +49,14 @@ export const INTERACTION_ERROR_REASONS = { * `packages/command-runtime/src/commands/interaction/runtime/moving-target-gate.ts`, which raises the reason beside the * hint that names that recovery. */ + /** + * A comma-separated ref plan cannot carry `--verify`: the pixel fallback behind `--verify` + * answers one point the pre-action tree cannot see, and the plan's members are tree-visible + * refs the plan already admits against that tree. The refusal is pre-dispatch, so nothing + * crossed to the device. The owner is + * `packages/command-runtime/src/commands/interaction/runtime/interactions.ts` (`tapRefPlanCommand`). + */ + verifyUnsupportedOnRefPlan: 'verify-unsupported-on-ref-plan', targetStillMoving: 'target_still_moving', } as const; From bfaacd17227e3459a674bd2808c962ba5c494a5f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=B5=B5=E6=A1=82=E9=9B=84?= Date: Fri, 20 Nov 2026 23:45:31 +0800 Subject: [PATCH 10/12] fix(command-surface): hold the always-loaded byte budget with compact press and click text The always-loaded tools sat 9 bytes under the 22 KiB budget on main; the verify wording's +84 bytes crossed it. Compress the press and click metadata descriptions (and the verify sentence they share) without dropping a semantic claim: verify still names the screenshot comparison outside accessible controls, the target_still_moving refusal, the plan contract, and the settle continuation. --- packages/command-surface/src/interaction/metadata.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/command-surface/src/interaction/metadata.ts b/packages/command-surface/src/interaction/metadata.ts index c6c34c655f..dbc4e5c3b3 100644 --- a/packages/command-surface/src/interaction/metadata.ts +++ b/packages/command-surface/src/interaction/metadata.ts @@ -65,9 +65,9 @@ const FIND_ACTION_VALUES = [ const interactionCommandDescriptions = { click: - "Activate a UI target by snapshot ref, selector, or coordinates. This can change app state. Success reports dispatch only; settle returns the resulting UI diff and verify reads the result and can compare screenshots outside accessible controls. A ref or selector target is re-checked for motion before dispatch when the session's previous action just animated the screen, and is dispatched at its resting position; one that has not stopped refuses with target_still_moving rather than acting where it was. With settle, continue from the returned diff. Copy refs exactly, including any ~sN suffix.", + "Activate a UI target by ref, selector, or coordinates. This can change app state. Success reports dispatch only; settle returns the resulting UI diff; verify compares screenshots outside accessible controls. A ref or selector target re-checked for motion before dispatch after the last action animated the screen dispatches at its resting position; one still moving refuses with target_still_moving rather than acting where it was. With settle, continue from the returned diff. Copy refs exactly, including any ~sN suffix.", press: - "Short-press a UI target by snapshot ref, selector, or coordinates. Success reports dispatch only; settle returns the resulting UI diff and verify reads the result and can compare screenshots outside accessible controls. A ref or selector target is re-checked for motion before dispatch when the session's previous action just animated the screen, and is dispatched at its resting position; one that has not stopped refuses with target_still_moving rather than acting where it was. A comma-separated ref list dispatches 2-16 keys in order as one command, admitted before the first press. With settle, continue from the returned diff.", + "Short-press a UI target by ref, selector, or coordinates. Success reports dispatch only; settle returns the resulting UI diff; verify compares screenshots outside accessible controls. A ref or selector target re-checked for motion before dispatch after the last action animated the screen dispatches at its resting position; one still moving refuses with target_still_moving rather than acting where it was. A comma-separated ref list dispatches 2-16 keys as one command, admitted before the first press. With settle, continue from the returned diff.", fill: 'Replace text in a UI input selected by snapshot ref, selector, or coordinates. An empty text clears the field. Unconfirmed evidence does not establish the resulting value or screen. The recordAs option keeps sensitive text out of a recorded replay while sending it to the live app. To append instead of replacing, use type.', longpress: 'Hold a UI target by snapshot ref, selector, or coordinates to open a context menu or perform another hold gesture. The durationMs input overrides the default hold duration.', From b483eb7c8ebe509621b021c25f16a28673b5f9d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=B5=B5=E6=A1=82=E9=9B=84?= Date: Fri, 20 Nov 2026 23:51:40 +0800 Subject: [PATCH 11/12] style: format metadata.ts --- packages/command-surface/src/interaction/metadata.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/command-surface/src/interaction/metadata.ts b/packages/command-surface/src/interaction/metadata.ts index dbc4e5c3b3..3979d7c5ee 100644 --- a/packages/command-surface/src/interaction/metadata.ts +++ b/packages/command-surface/src/interaction/metadata.ts @@ -65,9 +65,9 @@ const FIND_ACTION_VALUES = [ const interactionCommandDescriptions = { click: - "Activate a UI target by ref, selector, or coordinates. This can change app state. Success reports dispatch only; settle returns the resulting UI diff; verify compares screenshots outside accessible controls. A ref or selector target re-checked for motion before dispatch after the last action animated the screen dispatches at its resting position; one still moving refuses with target_still_moving rather than acting where it was. With settle, continue from the returned diff. Copy refs exactly, including any ~sN suffix.", + 'Activate a UI target by ref, selector, or coordinates. This can change app state. Success reports dispatch only; settle returns the resulting UI diff; verify compares screenshots outside accessible controls. A ref or selector target re-checked for motion before dispatch after the last action animated the screen dispatches at its resting position; one still moving refuses with target_still_moving rather than acting where it was. With settle, continue from the returned diff. Copy refs exactly, including any ~sN suffix.', press: - "Short-press a UI target by ref, selector, or coordinates. Success reports dispatch only; settle returns the resulting UI diff; verify compares screenshots outside accessible controls. A ref or selector target re-checked for motion before dispatch after the last action animated the screen dispatches at its resting position; one still moving refuses with target_still_moving rather than acting where it was. A comma-separated ref list dispatches 2-16 keys as one command, admitted before the first press. With settle, continue from the returned diff.", + 'Short-press a UI target by ref, selector, or coordinates. Success reports dispatch only; settle returns the resulting UI diff; verify compares screenshots outside accessible controls. A ref or selector target re-checked for motion before dispatch after the last action animated the screen dispatches at its resting position; one still moving refuses with target_still_moving rather than acting where it was. A comma-separated ref list dispatches 2-16 keys as one command, admitted before the first press. With settle, continue from the returned diff.', fill: 'Replace text in a UI input selected by snapshot ref, selector, or coordinates. An empty text clears the field. Unconfirmed evidence does not establish the resulting value or screen. The recordAs option keeps sensitive text out of a recorded replay while sending it to the live app. To append instead of replacing, use type.', longpress: 'Hold a UI target by snapshot ref, selector, or coordinates to open a context menu or perform another hold gesture. The durationMs input overrides the default hold duration.', From 060d80d4d020d27d3add3bd111168b70e1ba1efd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=B5=B5=E6=A1=82=E9=9B=84?= Date: Sat, 21 Nov 2026 10:43:15 +0800 Subject: [PATCH 12/12] fix(interaction): rename the ref-plan verify reason to snake_case The reason became published vocabulary when the refusal moved into the shared INTERACTION_ERROR_REASONS table. Every other member is snake_case, so the dashed literal reads as a typo on the wire surface: rename it to verify_unsupported_on_ref_plan and move the entry below targetStillMoving, whose JSDoc it was splitting from its member. --- .../src/commands/interaction/runtime/interactions.test.ts | 2 +- packages/selectors/src/interaction-error.ts | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/command-runtime/src/commands/interaction/runtime/interactions.test.ts b/packages/command-runtime/src/commands/interaction/runtime/interactions.test.ts index 4dcc69aeeb..9a44a1ee25 100644 --- a/packages/command-runtime/src/commands/interaction/runtime/interactions.test.ts +++ b/packages/command-runtime/src/commands/interaction/runtime/interactions.test.ts @@ -923,7 +923,7 @@ test('runtime press plan refuses --verify instead of dropping the pixel pair', a }), (error: AppError) => { assert.equal(error.code, 'INVALID_ARGS'); - assert.equal(error.details?.reason, 'verify-unsupported-on-ref-plan'); + assert.equal(error.details?.reason, 'verify_unsupported_on_ref_plan'); return true; }, ); diff --git a/packages/selectors/src/interaction-error.ts b/packages/selectors/src/interaction-error.ts index 04a5034033..b618b1bf61 100644 --- a/packages/selectors/src/interaction-error.ts +++ b/packages/selectors/src/interaction-error.ts @@ -49,6 +49,7 @@ export const INTERACTION_ERROR_REASONS = { * `packages/command-runtime/src/commands/interaction/runtime/moving-target-gate.ts`, which raises the reason beside the * hint that names that recovery. */ + targetStillMoving: 'target_still_moving', /** * A comma-separated ref plan cannot carry `--verify`: the pixel fallback behind `--verify` * answers one point the pre-action tree cannot see, and the plan's members are tree-visible @@ -56,8 +57,7 @@ export const INTERACTION_ERROR_REASONS = { * crossed to the device. The owner is * `packages/command-runtime/src/commands/interaction/runtime/interactions.ts` (`tapRefPlanCommand`). */ - verifyUnsupportedOnRefPlan: 'verify-unsupported-on-ref-plan', - targetStillMoving: 'target_still_moving', + verifyUnsupportedOnRefPlan: 'verify_unsupported_on_ref_plan', } as const; /** The part of a covered node a refusal reports. */