diff --git a/packages/command-output/src/cli-output.test.ts b/packages/command-output/src/cli-output.test.ts index 34aa123f26..3ad12792de 100644 --- a/packages/command-output/src/cli-output.test.ts +++ b/packages/command-output/src/cli-output.test.ts @@ -238,6 +238,18 @@ describe('the two stdout contracts in one run', () => { expect(output.stderr).toContain(`Warning: ${REPAIR_WARNING}`); }); + test('find focus/type routes a non-hittable target hint through warnings without polluting stdout', async () => { + const hint = + 'The resolved element reports hittable: false, so this focus may have had no visible effect.'; + const output = await render( + 'find', + { message: 'Focused (60, 40)', x: 60, y: 40, targetHittable: false, hint }, + { query: 'Continue', action: 'focus' }, + ); + expect(output.text).toBe('Focused (60, 40)'); + expect(output.stderr).toBe(`Warning: ${hint}\n`); + }); + test('is prints the disclosure after its verdict on stdout', async () => { const output = await render( 'is', diff --git a/packages/command-output/src/output-common.ts b/packages/command-output/src/output-common.ts index 7e928ed695..ad12673437 100644 --- a/packages/command-output/src/output-common.ts +++ b/packages/command-output/src/output-common.ts @@ -76,12 +76,18 @@ export function messageCliOutput(result: Record): CliOutput { * one renders on a single line. */ function collectResponseWarnings(result: Record): string[] { - return [ + const raw = [ ...(typeof result.warning === 'string' && result.warning.trim() !== '' ? [result.warning] : []), ...readResponseWarnings(result), + ...(result.targetHittable === false && + typeof result.hint === 'string' && + result.hint.trim() !== '' + ? [result.hint] + : []), ] .map((warning) => collapseWarningText(warning)) .filter((warning) => warning.length > 0); + return Array.from(new Set(raw)); } /** diff --git a/packages/command-runtime/src/commands/interaction/runtime/native-ref-interaction.ts b/packages/command-runtime/src/commands/interaction/runtime/native-ref-interaction.ts index 9858268497..68bf80b5a0 100644 --- a/packages/command-runtime/src/commands/interaction/runtime/native-ref-interaction.ts +++ b/packages/command-runtime/src/commands/interaction/runtime/native-ref-interaction.ts @@ -19,10 +19,12 @@ import { toBackendContext } from '../../runtime-common.ts'; import { toBackendResult } from '../../runtime-types.ts'; import type { InteractionAction } from '@agent-device/selectors/interaction-action'; import { tryResolveRefNode } from './ref-target-resolution.ts'; -import { EXACT_REF_RESOLUTION, describeNonHittableTarget } from './resolution-disclosure.ts'; +import { EXACT_REF_RESOLUTION } from './resolution-disclosure.ts'; +import { describeNonHittableTarget } from '@agent-device/selectors/interaction-targeting'; +import { offscreenRefFailure } from '@agent-device/selectors/interaction-offscreen'; import { - assertVisibleRefTarget, runInteractionPipelineStages, + throwIfOffscreenInteractionTarget, } from './target-visibility-stages.ts'; /** @@ -33,7 +35,7 @@ import { * run the SAME shared guards the runtime path uses against it before the * backend call — occlusion (`isSnapshotNodeInteractionBlocked` via * `assertInteractionNotBlocked`) and offscreen (the snapshot visibility resolver via - * `assertVisibleRefTarget`) ERROR with the runtime path's exact shapes, and + * `throwIfOffscreenInteractionTarget`) ERROR with the runtime path's exact shapes, and * the non-hittable annotation is returned for the fast-path result. * * Zero extra round trips by construction on the accept path: no session, no @@ -84,10 +86,13 @@ export async function preflightNativeRefInteraction( label: `Ref ${target.ref}`, hooks: { offscreen: async (node, tree) => - await assertVisibleRefTarget(runtime, options, node, tree, target.ref, { - action, - pipeline, - }), + await throwIfOffscreenInteractionTarget( + runtime, + options, + node, + tree, + offscreenRefFailure(target.ref, action), + ), }, resolveTapPoint: (node) => resolveRectCenter(node.rect), }); diff --git a/packages/command-runtime/src/commands/interaction/runtime/ref-target-resolution.ts b/packages/command-runtime/src/commands/interaction/runtime/ref-target-resolution.ts index 2d7e582898..e75852b876 100644 --- a/packages/command-runtime/src/commands/interaction/runtime/ref-target-resolution.ts +++ b/packages/command-runtime/src/commands/interaction/runtime/ref-target-resolution.ts @@ -34,9 +34,10 @@ import { describeResolvedInteractionNode, type ResolvedRefNode, } from './resolution-disclosure.ts'; +import { offscreenRefFailure } from '@agent-device/selectors/interaction-offscreen'; import { - assertVisibleRefTarget, runInteractionPipelineStages, + throwIfOffscreenInteractionTarget, } from './target-visibility-stages.ts'; /** The node a ref target acts on, plus the tree the shared guards read it against. */ @@ -122,7 +123,13 @@ export async function resolveRefInteractionTarget( hooks: { onResolved: (node, tree) => assertReplayTargetResolution(node, tree, params), offscreen: async (node, tree) => - await assertVisibleRefTarget(runtime, options, node, tree, target.ref, params), + await throwIfOffscreenInteractionTarget( + runtime, + options, + node, + tree, + offscreenRefFailure(target.ref, params.action), + ), }, resolveTapPoint: (node) => resolveNodeTouchPoint(node, nodes, { diff --git a/packages/command-runtime/src/commands/interaction/runtime/resolution-disclosure.ts b/packages/command-runtime/src/commands/interaction/runtime/resolution-disclosure.ts index 9b00a97b06..2b55f5910b 100644 --- a/packages/command-runtime/src/commands/interaction/runtime/resolution-disclosure.ts +++ b/packages/command-runtime/src/commands/interaction/runtime/resolution-disclosure.ts @@ -3,6 +3,7 @@ import type { SnapshotState } from '@agent-device/contracts/snapshot-model'; import type { AgentDeviceRuntime } from '@agent-device/contracts/runtime-contract'; import { buildSelectorChainForNode } from '@agent-device/selectors'; import { resolvePressRecordingTarget } from '@agent-device/selectors/press-retarget'; +import { describeNonHittableTarget } from '@agent-device/selectors/interaction-targeting'; import { resolveRefLabel } from '@agent-device/capture-kit/snapshot-node-lookup'; import { normalizeType } from '@agent-device/contracts/snapshot'; import { truncateUtf8 } from './truncate-utf8.ts'; @@ -157,22 +158,3 @@ function pressRecordingTargetOverride( }, }; } - -/** - * iOS AX `hittable` flags are unreliable on deep React Native trees (see #1037: - * a map-pin annotation exact-matched a longer recents row label and reported tap - * success while doing nothing visible). We deliberately do NOT fail or filter on - * this signal — that would break selectors that only ever resolve to nodes the - * platform marks non-hittable. Instead, surface it so the caller can notice a - * likely no-op tap and re-target with a ref or a more specific selector/longer text. - */ -export function describeNonHittableTarget( - node: SnapshotNode, - action: InteractionAction, -): { targetHittable?: boolean; hint?: string } { - if (node.hittable !== false) return {}; - return { - targetHittable: false, - hint: `The resolved element reports hittable: false, so this ${action} may have had no visible effect. Verify with a snapshot, or prefer a @ref or a longer/more specific selector to target the intended element.`, - }; -} diff --git a/packages/command-runtime/src/commands/interaction/runtime/resolution.ts b/packages/command-runtime/src/commands/interaction/runtime/resolution.ts index e7d7878b97..9ed1dd1e14 100644 --- a/packages/command-runtime/src/commands/interaction/runtime/resolution.ts +++ b/packages/command-runtime/src/commands/interaction/runtime/resolution.ts @@ -25,9 +25,10 @@ import { buildSelectorResolutionDisclosure, describeResolvedInteractionNode, } from './resolution-disclosure.ts'; +import { offscreenSelectorFailure } from '@agent-device/selectors/interaction-offscreen'; import { - assertVisibleSelectorTarget, runInteractionPipelineStages, + throwIfOffscreenInteractionTarget, } from './target-visibility-stages.ts'; export type { InteractionRequestTarget, ResolvedInteractionTarget }; @@ -156,7 +157,13 @@ async function resolveSelectorInteractionTarget( hooks: { onResolved: (node, tree) => assertReplayTargetResolution(node, tree, params), offscreen: async (node, tree) => - await assertVisibleSelectorTarget(runtime, options, node, tree, selected.selector, params), + await throwIfOffscreenInteractionTarget( + runtime, + options, + node, + tree, + offscreenSelectorFailure(selected.selector, params.action), + ), }, resolveTapPoint: (node) => resolveNodeTouchPoint(node, capture.snapshot.nodes, { diff --git a/packages/command-runtime/src/commands/interaction/runtime/target-visibility-stages.ts b/packages/command-runtime/src/commands/interaction/runtime/target-visibility-stages.ts index a66a95a2fc..c660c37410 100644 --- a/packages/command-runtime/src/commands/interaction/runtime/target-visibility-stages.ts +++ b/packages/command-runtime/src/commands/interaction/runtime/target-visibility-stages.ts @@ -1,4 +1,3 @@ -import { shellQuote } from '@agent-device/kernel/device-shell'; import { AppError, discloseDispatch } from '@agent-device/kernel/errors'; import { assertSnapshotScreenOrigin } from '@agent-device/contracts/capture'; import type { @@ -8,7 +7,6 @@ import type { SnapshotNode, } from '@agent-device/kernel/snapshot'; import type { SnapshotState } from '@agent-device/contracts/snapshot-model'; -import { normalizeRef } from '@agent-device/kernel/snapshot'; import type { AgentDeviceRuntime, CommandContext } from '@agent-device/contracts/runtime-contract'; import type { ResolvedInteractionTarget, @@ -19,12 +17,8 @@ import { type SelectorPipelineHooks, } from '@agent-device/selectors/selector-pipeline'; import type { ActingPipelinePolicy } from '@agent-device/selectors/selector-pipeline-policy'; -import { createSnapshotVisibility } from '@agent-device/contracts/snapshot'; import { coveredInteractionDetails } from '@agent-device/selectors/interaction-error'; -import { - classifyOffscreenScrollDirection, - type OffscreenScrollDirection, -} from '@agent-device/capture-kit/mobile-snapshot-semantics'; +import { throwIfOffscreenTarget } from '@agent-device/selectors/interaction-offscreen'; import { toBackendContext } from '../../runtime-common.ts'; import { assertTapTargetClearOfVisibleKeyboard } from '@agent-device/selectors/keyboard-occlusion'; import { @@ -108,91 +102,10 @@ export async function runInteractionPipelineStages( return { node: target.node, tapPoint }; } -/** - * The off-screen stage's refusal shape. Reached only through the pipeline - * owner, and only for rows whose off-screen stage refuses — the row's decision - * is made there, so this builds the message and never re-decides. - */ -type OffscreenStageParams = { action: InteractionAction; pipeline: ActingPipelinePolicy }; - -// Selector parity for the @ref off-screen guard: without it, a selector -// resolving to a closed drawer/carousel item "succeeds" by tapping coordinates -// outside the viewport (observed as `Tapped (-161, 265)` against Bluesky's -// closed drawer) while the same node via @ref is refused. -export async function assertVisibleSelectorTarget( - runtime: AgentDeviceRuntime, - options: CommandContext, - node: SnapshotNode, - nodes: SnapshotState['nodes'], - selector: string, - { action }: OffscreenStageParams, -): Promise { - return await throwIfOffscreenInteractionTarget(runtime, options, node, nodes, { - message: `Selector ${selector} resolved to an off-screen element and is not safe to ${action}`, - details: { reason: 'offscreen_selector', selector }, - // A selector re-resolves against a fresh snapshot on every attempt, so the - // recovery is: move the named direction, then retry THIS selector — no - // separate snapshot step, and no @ref (a scroll expires the ref frame, - // #1366). `--until` is that whole loop as one command: it checks the same - // selector between passes, which is also what keeps a large step from - // overshooting, so the hint no longer has to trade distance for accuracy. - hint: (direction) => - `${scrollRevealClause(direction, selector)} then retry ${action} with the same selector. --until checks the selector between passes, so it stops on the target rather than sailing past it. If it is inside a closed drawer or another tab, open that container first.`, - }); -} - -export async function assertVisibleRefTarget( - runtime: AgentDeviceRuntime, - options: CommandContext, - node: SnapshotNode, - nodes: SnapshotState['nodes'], - refInput: string, - { action }: OffscreenStageParams, -): Promise { - return await throwIfOffscreenInteractionTarget(runtime, options, node, nodes, { - message: `Ref ${refInput} is off-screen and not safe to ${action}`, - details: { reason: 'offscreen_ref', ref: normalizeRef(refInput) }, - // The scroll that reveals the target expires the ref frame (#1366, ADR - // 0014), so retrying this @ref would be rejected next. Steer to a selector, - // which re-resolves against a fresh snapshot and bypasses the ref-frame guard - // — and which `--until` can then check between passes. - hint: (direction) => - `${scrollRevealClause(direction, null)} then retry ${action} with a selector (e.g. text=/id=) rather than this @ref — the scroll expires the ref frame, so re-run snapshot -i before reusing any @ref.`, - }); -} - -/** - * Shared lead-in for both off-screen hints: the one command that reveals the target. - * - * When the geometry names a direction AND the caller has a selector to check, this is a complete - * `scroll --until ` — one request that stops on the target instead of the - * scroll-then-look-again loop the hint used to prescribe. Without a selector to check (an @ref - * refusal) or without a single reveal direction (off more than one edge), it degrades to naming - * the move and leaves the stop condition to the caller's own next step. - */ -function scrollRevealClause( - direction: OffscreenScrollDirection | null, - selector: string | null, -): string { - if (!direction) return 'Scroll toward it,'; - if (!selector) return `Scroll ${direction} toward it,`; - return `Run scroll ${direction} --until ${shellQuote(selector)} to bring it on screen,`; -} - -// Full on-screen visibility (not only the effective-viewport form): items inside an -// off-screen scrollable container (closed drawer) must also count as -// off-screen, not just items scrolled out of an on-screen container. -// -// #1542: once the bulk tree says off-screen, the guard gives iOS one chance -// to rescue a FALSE refusal via the optional backend.confirmOffscreenTargetVisible -// hook — a stale/corrupted bulk tree can say off-screen while the app is -// visually fine (zero cost on the accept path; runs only here). A confirmed -// rescue returns the node PATCHED WITH THE LIVE RECT: the caller must act on -// that returned node, never the original, because in the frozen-bulk-tree -// manifestation the original rect can be stale even when the rescue verdict -// is correct — tapping it would silently land at the wrong coordinate. The -// hook fails closed (null) on anything short of a positive confirmation, so -// a genuine refusal, or any backend without the hook, is unchanged. +// The runtime paths' end-to-end off-screen enforcement point: the shared rule +// (`throwIfOffscreenTarget`) with the backend's optional +// `confirmOffscreenTargetVisible` hook as its #1542 rescue read. A backend that +// omits the hook refuses on the bulk tree's verdict unchanged. // // Exported (not just for callers here) for ADR 0011 registry honesty: // interaction-guarantees.ts's `offscreen` cells point their `via` at this @@ -203,37 +116,22 @@ export async function throwIfOffscreenInteractionTarget( options: CommandContext, node: SnapshotNode, nodes: SnapshotState['nodes'], - failure: { - message: string; - details: Record; - hint: (direction: OffscreenScrollDirection | null) => string; - }, + failure: Parameters[2], ): Promise { - const visibility = createSnapshotVisibility(nodes); - const viewport = node.rect ? visibility.resolveEffectiveViewport(node) : null; - if (!node.rect || !viewport || visibility.isVisibleOnScreen(node)) return node; - const rootViewport = visibility.resolveViewport(node.rect); - const liveRect = await runtime.backend.confirmOffscreenTargetVisible?.( - toBackendContext(runtime, options), + const confirm = runtime.backend.confirmOffscreenTargetVisible; + return await throwIfOffscreenTarget( node, - rootViewport, - ); - if (liveRect) return { ...node, rect: liveRect }; - // The direction that scrolls this off-screen target into view. Named in the - // hint (and surfaced as a machine-readable detail) so the recovery is a single - // deterministic move instead of a guess (#1366). Derived from the same - // boundary the rejection above used, so partial clips and off-screen - // containers get a direction too, not just fully-scrolled-out items. - const scrollDirection = classifyOffscreenScrollDirection(node, visibility); - throw discloseDispatch( - new AppError('COMMAND_FAILED', failure.message, { - ...failure.details, - rect: node.rect, - viewport, - ...(scrollDirection ? { scrollDirection } : {}), - hint: failure.hint(scrollDirection), - }), - 'no', + nodes, + failure, + confirm + ? async (target, rootViewport) => + await confirm.call( + runtime.backend, + toBackendContext(runtime, options), + target, + rootViewport, + ) + : undefined, ); } diff --git a/packages/command-surface/src/interaction/index.ts b/packages/command-surface/src/interaction/index.ts index 10cee88931..3cd1b2dc2e 100644 --- a/packages/command-surface/src/interaction/index.ts +++ b/packages/command-surface/src/interaction/index.ts @@ -345,6 +345,10 @@ export const INTERACTION_COMMAND_OUTPUT_SCHEMAS = { x: numberSchema('Resolved x coordinate for mutating find actions.'), y: numberSchema('Resolved y coordinate for mutating find actions.'), message: stringSchema('Diagnostic message for mutating find actions.'), + targetHittable: booleanSchema( + 'false when the element find focus or find type focused reports hittable: false.', + ), + hint: stringSchema('Recovery hint that accompanies targetHittable: false.'), cost: responseCostSchema, }, [], diff --git a/packages/contracts/src/interaction-guarantees.ts b/packages/contracts/src/interaction-guarantees.ts index 586afe579c..c6098bacce 100644 --- a/packages/contracts/src/interaction-guarantees.ts +++ b/packages/contracts/src/interaction-guarantees.ts @@ -154,7 +154,7 @@ export type InteractionPathContract = { const GAPS_UMBRELLA_ISSUE = 'https://github.com/callstack/agent-device/issues/1081'; const TAP_OUTCOME_NOT_OBSERVED_GAP_ISSUE = 'https://github.com/callstack/agent-device/issues/3335'; -// The four guarantees this route's own seam leaves open are owned by #3734. +// The guarantee this route's own seam leaves open is owned by #3734. const FIND_FOCUS_DISPATCH_GAPS_ISSUE = 'https://github.com/callstack/agent-device/issues/3734'; /** @@ -572,11 +572,12 @@ export const INTERACTION_DISPATCH_PATHS: Record ` reaches), and `type`'s - // second dispatch is `executeBoundTypeText`'s. The seam hand-rolls neither. + // second dispatch is `executeBoundTypeText`'s. The seam hand-rolls neither; it only adds the + // `nonHittable` annotation above. responseConstruction: { kind: 'runtime', via: 'packages/daemon/src/execution/focus-runtime.ts#executeFocusPoint', @@ -637,7 +638,7 @@ export const INTERACTION_DISPATCH_PATHS: Record Promise; typeText?: (input: TypeTextInput) => Promise; + querySelector?: BoundSelectorQuery; }>; }>, ): Readonly<{ focusPoint?: (input: FocusPointInput) => Promise; typeText?: (input: TypeTextInput) => Promise; + querySelector?: BoundSelectorQuery; }> { - const { focusPoint, typeText } = runtime.operations; + const { focusPoint, typeText, querySelector } = runtime.operations; return Object.freeze({ ...(focusPoint ? { focusPoint } : {}), ...(typeText ? { typeText } : {}), + ...(querySelector ? { querySelector } : {}), }); } diff --git a/packages/daemon/src/core/snapshot-runtime-binding.ts b/packages/daemon/src/core/snapshot-runtime-binding.ts index dac4e3bb27..ac385adacc 100644 --- a/packages/daemon/src/core/snapshot-runtime-binding.ts +++ b/packages/daemon/src/core/snapshot-runtime-binding.ts @@ -44,6 +44,7 @@ import { selectWaitObservationOperations, type BoundElementRead, type BoundNativeTextRead, + type BoundSelectorQuery, } from './selector-operation-binding.ts'; import { errorResponse } from '@agent-device/kernel/contracts'; import { AppError } from '@agent-device/kernel/errors'; @@ -101,6 +102,8 @@ export type AdmittedSnapshotCapture = typeText?: ( input: import('@agent-device/contracts/type-text-runtime').TypeTextInput, ) => Promise; + /** The find-focus / find-type plans' conditional live selector read (#1542 rescue). */ + querySelector?: BoundSelectorQuery; }> | Readonly<{ ok: false; response: DaemonResponse }>; @@ -151,6 +154,7 @@ export async function admitAndBindSnapshotCapture( ...(bound.findText ? { findText: bound.findText } : {}), ...(bound.focusPoint ? { focusPoint: bound.focusPoint } : {}), ...(bound.typeText ? { typeText: bound.typeText } : {}), + ...(bound.querySelector ? { querySelector: bound.querySelector } : {}), }); } @@ -278,6 +282,7 @@ async function bindSnapshotCaptureRuntime( typeText?: ( input: import('@agent-device/contracts/type-text-runtime').TypeTextInput, ) => Promise; + querySelector?: BoundSelectorQuery; }> > { const bind = requireRuntimeBinding(bindDevice); diff --git a/packages/daemon/src/execution/__tests__/offscreen-target-probe.test.ts b/packages/daemon/src/execution/__tests__/offscreen-target-probe.test.ts index b51b9d353b..2c7dff69cb 100644 --- a/packages/daemon/src/execution/__tests__/offscreen-target-probe.test.ts +++ b/packages/daemon/src/execution/__tests__/offscreen-target-probe.test.ts @@ -127,3 +127,42 @@ test('confirmIosOffscreenTargetVisible: null when the owner answers found withou assert.equal(rect, null); }); + +test('confirmIosOffscreenTargetVisible: exactSelector probes that exact key with hittability probe and rejects duplicates', async () => { + const uniqueQuery = vi.fn(async () => ({ + found: true, + hittability: { + matchCount: 1, + hittable: true, + rect: { x: 20, y: 240, width: 300, height: 50 }, + }, + })); + const rescued = await confirmIosOffscreenTargetVisible({ + session: makeSession(), + node: { identifier: 'drawer-action', label: 'Explore' }, + rootViewport: ROOT_VIEWPORT, + querySelector: uniqueQuery, + execution: {}, + exactSelector: { key: 'label', value: 'Explore' }, + }); + assert.deepEqual(rescued, { x: 20, y: 240, width: 300, height: 50 }); + assert.deepEqual(uniqueQuery.mock.calls[0]?.[0]?.selector, { key: 'label', value: 'Explore' }); + assert.equal(uniqueQuery.mock.calls[0]?.[0]?.probe, 'hittability'); + + const duplicate = await confirmIosOffscreenTargetVisible({ + session: makeSession(), + node: { label: 'Explore' }, + rootViewport: ROOT_VIEWPORT, + querySelector: async () => ({ + found: true, + hittability: { + matchCount: 2, + hittable: true, + rect: { x: 20, y: 240, width: 300, height: 50 }, + }, + }), + execution: {}, + exactSelector: { key: 'label', value: 'Explore' }, + }); + assert.equal(duplicate, null); +}); diff --git a/packages/daemon/src/execution/interaction/internal/find.ts b/packages/daemon/src/execution/interaction/internal/find.ts index c8ba15678f..ed3d89f7c9 100644 --- a/packages/daemon/src/execution/interaction/internal/find.ts +++ b/packages/daemon/src/execution/interaction/internal/find.ts @@ -1,4 +1,4 @@ -import type { DaemonResponse } from '@agent-device/kernel/contracts'; +import type { DaemonFailureResponse, DaemonResponse } from '@agent-device/kernel/contracts'; import type { PreresolvedInteractionTarget, SurfaceScopedNodes, @@ -9,6 +9,7 @@ import { isReadOnlyFindAction, checkFindArgs, parseFindSelectorExpression, + readSimpleSelectorTarget, type FindLocator, } from '@agent-device/selectors'; import { runNodePipelineStages } from '@agent-device/selectors/selector-pipeline'; @@ -21,6 +22,11 @@ import { resolveFocusTarget, resolveNodeTouchPoint, } from '@agent-device/selectors/node-touch-point'; +import { + offscreenRefFailure, + throwIfOffscreenTarget, +} from '@agent-device/selectors/interaction-offscreen'; +import { describeNonHittableTarget } from '@agent-device/selectors/interaction-targeting'; import { isSnapshotNodeInteractionBlocked } from '@agent-device/capture-kit/snapshot-occlusion'; import { buildNestedTextFieldsError } from '../../selector-match-errors.ts'; import { assertTapTargetClearOfVisibleKeyboard } from '@agent-device/selectors/keyboard-occlusion'; @@ -48,9 +54,12 @@ import { recordSessionAction } from '../../../core/session-action-recorder.ts'; import { stripInternalInteractionFlags } from '../../../core/deferred-interaction-outcome.ts'; import { resolveFindMatch } from './find-match-resolution.ts'; import { executeFocusPoint } from '../../focus-runtime.ts'; +import { confirmIosOffscreenTargetVisible } from '../../offscreen-target-probe.ts'; import { executeBoundTypeText, parseTypeTextRequest } from '../../type-text-runtime.ts'; import { dispatchFindReadOnlyViaRuntime } from '../../selector-runtime.ts'; import { admitAndBindSnapshotCapture } from '../../../core/snapshot-runtime-binding.ts'; +import type { BoundSelectorQuery } from '../../../core/selector-operation-binding.ts'; +import { runtimeExecutionFromContext } from '../../../core/snapshot-runtime-capture-input.ts'; import type { FocusPointInput } from '@agent-device/contracts/focus-runtime'; import { findRuntimeIntent, @@ -77,6 +86,8 @@ type FindContext = { /** The one action-selected bind's directly-executed operations (ADR 0019 §9). */ boundFocusPoint?: (input: FocusPointInput) => Promise; boundTypeText?: TypeTextRuntimeOperations['typeText']; + /** The same bind's conditional live selector read, present only when its owner advertised it. */ + boundQuerySelector?: BoundSelectorQuery; }; type ResolvedMatch = { @@ -184,6 +195,7 @@ export async function handleFindCommands(params: FindRouteInput): Promise { - const response = await dispatchFocusForFindMatch(ctx, match); - if (!response.ok) return response; + const focused = await dispatchFocusForFindMatch(ctx, match, 'focus'); + if (!focused.ok) return focused; recordFindAction(ctx, match, 'focus'); - return response; + return { ok: true, data: { ...focused.data, ...focused.targetAnnotation } }; } async function handleFindType( @@ -386,8 +398,8 @@ async function handleFindType( }; // The value is always text here, so a leading `@e12` is typed, not refused as a target ref. const request = parseTypeTextRequest([value], context(), { rejectRefTarget: false }); - const focusResponse = await dispatchFocusForFindMatch(ctx, match); - if (!focusResponse.ok) return focusResponse; + const focused = await dispatchFocusForFindMatch(ctx, match, 'type'); + if (!focused.ok) return focused; const session = ctx.sessionStore.requireCurrent(ctx.sessionRef); // The focus above already crossed the seam; expiry is idempotent, but keep it // explicit at the type dispatch so it does not rely on the focus-first order. @@ -400,13 +412,26 @@ async function handleFindType( } const response = await executeBoundTypeText({ operations: { typeText } }, request, context()); recordFindAction(ctx, match, 'type'); - return { ok: true, data: response }; + return { ok: true, data: { ...response, ...focused.targetAnnotation } }; } +/** + * The focus leg's dispatch result, with the non-hittable annotation of the node it focused kept apart + * so `type` can answer with it beside its own text payload. + */ +type FindFocusDispatch = + | DaemonFailureResponse + | { + ok: true; + data: Record; + targetAnnotation: ReturnType; + }; + async function dispatchFocusForFindMatch( ctx: FindContext, match: ResolvedMatch, -): Promise { + action: 'focus' | 'type', +): Promise { const { req, logPath } = ctx; const coveredResponse = rejectCoveredFindMatch(match, 'focus'); if (coveredResponse) return coveredResponse; @@ -430,14 +455,27 @@ async function dispatchFocusForFindMatch( }, ); } - const target = focusTarget.node; - if (isSnapshotNodeInteractionBlocked(target)) return coveredFindResponse(target, 'focus'); - const targetRef = `@${target.ref}`; + const located = focusTarget.node; + if (isSnapshotNodeInteractionBlocked(located)) return coveredFindResponse(located, 'focus'); + const targetRef = `@${located.ref}`; + let target: SnapshotAddressableNode; let coords: Point; - // The same touch-point and keyboard decisions the acting paths run (#2589/#2622/#3734): this seam - // dispatches the point itself instead of re-entering the interaction leaf, so they have to run here. + // The same off-screen, touch-point and keyboard decisions the acting paths run + // (#1542/#2589/#2622/#3734): this seam dispatches the point itself instead of re-entering the + // interaction leaf, so they have to run here. find's ranking already preferred an on-screen match. try { await assertSnapshotScreenOrigin(match.tree.nodes, `Ref ${targetRef}`); + const refFailure = offscreenRefFailure(targetRef, 'focus'); + target = await throwIfOffscreenTarget( + located, + match.tree.nodes, + { + ...refFailure, + hint: (direction) => + `${direction ? `Scroll ${direction} toward it` : 'Scroll toward it'} or open its container, then re-run find to ${action}.`, + }, + offscreenRescue(ctx), + ); coords = resolveNodeTouchPoint(target, match.tree.nodes, { invalidMessage: `Ref ${targetRef} has no usable bounds`, blockedTargetLabel: `Ref ${targetRef}`, @@ -470,20 +508,60 @@ async function dispatchFocusForFindMatch( coords, contextFromFlags(logPath, req.flags, session.appBundleId, session.trace?.outPath), ); - return { ok: true, data: response }; + return { ok: true, data: response, targetAnnotation: describeNonHittableTarget(target, 'focus') }; +} + +function directIdentityFindSelector( + ctx: FindContext, +): Readonly<{ key: 'id' | 'label'; value: string }> | null { + if (ctx.locator === 'id' || ctx.locator === 'label') { + const value = ctx.query.trim(); + return value ? { key: ctx.locator, value } : null; + } + if (ctx.locator !== 'any') return null; + const target = readSimpleSelectorTarget(ctx.query); + if (!target || (target.key !== 'id' && target.key !== 'label')) return null; + const value = target.value.trim(); + return value ? { key: target.key, value } : null; +} + +/** + * The #1542 rescue read over the bind's conditional live selector query, or `undefined` when the + * owner did not advertise one or the find locator cannot be verified as an exact `id`/`label` + * match on the focused node, so the off-screen rule fails closed on the capture's verdict. + */ +function offscreenRescue(ctx: FindContext): Parameters[3] { + const querySelector = ctx.boundQuerySelector; + const exactSelector = directIdentityFindSelector(ctx); + if (!querySelector || !exactSelector) return undefined; + const session = ctx.sessionStore.requireCurrent(ctx.sessionRef); + return async (node, rootViewport) => { + const nodeValue = exactSelector.key === 'id' ? node.identifier?.trim() : node.label?.trim(); + if (nodeValue !== exactSelector.value) return null; + return await confirmIosOffscreenTargetVisible({ + session, + node, + rootViewport, + querySelector, + execution: runtimeExecutionFromContext( + contextFromFlags(ctx.logPath, ctx.req.flags, session.appBundleId, session.trace?.outPath), + ), + exactSelector, + }); + }; } function rejectCoveredFindMatch( match: ResolvedMatch, action: InteractionAction, -): DaemonResponse | null { +): DaemonFailureResponse | null { return match.occludedNode ? coveredFindResponse(match.occludedNode, action) : null; } function coveredFindResponse( blockedNode: SnapshotAddressableNode, action: InteractionAction, -): DaemonResponse { +): DaemonFailureResponse { return errorResponse( 'COMMAND_FAILED', `Matched element @${blockedNode.ref} is covered by another visible element and cannot ${interactionVerb(action)} safely`, diff --git a/packages/daemon/src/execution/offscreen-target-probe.ts b/packages/daemon/src/execution/offscreen-target-probe.ts index b28e560b51..805f31336d 100644 --- a/packages/daemon/src/execution/offscreen-target-probe.ts +++ b/packages/daemon/src/execution/offscreen-target-probe.ts @@ -19,24 +19,36 @@ export async function confirmIosOffscreenTargetVisible(params: { rootViewport: Rect | null; querySelector: QuerySelectorRuntimeOperations['querySelector']; execution: SnapshotRuntimeExecution; + /** + * When set, probes this exact requested `id`/`label` selector with the lean hittability probe + * (no hittable preference among duplicates) and requires `matchCount === 1` before rescuing. + */ + exactSelector?: Readonly<{ key: 'id' | 'label'; value: string }>; }): Promise { - const { session, node, rootViewport, querySelector, execution } = params; - const selector = deriveDirectIosNodeSelector(node); + const { session, node, rootViewport, querySelector, execution, exactSelector } = params; + const selector = exactSelector ?? deriveDirectIosNodeSelector(node); if (!selector) return null; - let found: boolean; - let nodeReading: Awaited>['node']; + let result: Awaited>; try { - ({ found, node: nodeReading } = await querySelector({ + result = await querySelector({ selector, + ...(exactSelector ? { probe: 'hittability' as const } : {}), ...(session.appBundleId === undefined ? {} : { options: { appBundleId: session.appBundleId } }), execution, - })); + }); } catch { return null; } - if (!found || !nodeReading?.rect) return null; - const probe = { rect: nodeReading.rect, hittable: nodeReading.hittable === true }; + if (exactSelector) { + const reading = result.hittability; + if (reading?.matchCount !== 1 || reading.hittable !== true || !reading.rect) return null; + return isConfirmedOnScreenProbe({ rect: reading.rect, hittable: true }, rootViewport) + ? reading.rect + : null; + } + if (!result.found || !result.node?.rect) return null; + const probe = { rect: result.node.rect, hittable: result.node.hittable === true }; return isConfirmedOnScreenProbe(probe, rootViewport) ? probe.rect : null; } diff --git a/packages/daemon/src/execution/selector-match-errors.ts b/packages/daemon/src/execution/selector-match-errors.ts index cb6df8c1da..d34d38a750 100644 --- a/packages/daemon/src/execution/selector-match-errors.ts +++ b/packages/daemon/src/execution/selector-match-errors.ts @@ -1,4 +1,4 @@ -import type { DaemonResponse } from '@agent-device/kernel/contracts'; +import type { DaemonFailureResponse, DaemonResponse } from '@agent-device/kernel/contracts'; import type { FindLocator } from '@agent-device/selectors'; import type { SnapshotState } from '@agent-device/contracts/snapshot-model'; import { elementMatchCandidateDetails } from '@agent-device/capture-kit/snapshot-lines'; @@ -35,7 +35,7 @@ export function buildAmbiguousMatchError( export function buildNestedTextFieldsError( ref: string, fields: SnapshotState['nodes'], -): DaemonResponse { +): DaemonFailureResponse { return errorResponse( 'AMBIGUOUS_MATCH', `find matched ${ref}, which contains ${fields.length} text fields. Use a more specific locator or selector.`, diff --git a/packages/selectors/package.json b/packages/selectors/package.json index 4a3d25ceff..07f98c16a5 100644 --- a/packages/selectors/package.json +++ b/packages/selectors/package.json @@ -48,6 +48,10 @@ "types": "./src/interaction-error.ts", "default": "./src/interaction-error.ts" }, + "./interaction-offscreen": { + "types": "./src/interaction-offscreen.ts", + "default": "./src/interaction-offscreen.ts" + }, "./interaction-positionals": { "types": "./src/interaction-positionals.ts", "default": "./src/interaction-positionals.ts" diff --git a/packages/selectors/src/interaction-offscreen.ts b/packages/selectors/src/interaction-offscreen.ts new file mode 100644 index 0000000000..1d56279e80 --- /dev/null +++ b/packages/selectors/src/interaction-offscreen.ts @@ -0,0 +1,119 @@ +import { shellQuote } from '@agent-device/kernel/device-shell'; +import { AppError, discloseDispatch } from '@agent-device/kernel/errors'; +import { normalizeRef, type Rect, type SnapshotNode } from '@agent-device/kernel/snapshot'; +import { createSnapshotVisibility } from '@agent-device/contracts/snapshot'; +import { + classifyOffscreenScrollDirection, + type OffscreenScrollDirection, +} from '@agent-device/capture-kit/mobile-snapshot-semantics'; +import type { InteractionAction } from './interaction-action.ts'; + +/** + * The one off-screen rule every element-targeted dispatch applies: the acting runtime paths through + * `throwIfOffscreenInteractionTarget`, and `find focus|type`, which dispatches its point + * itself. + * + * Full on-screen visibility (not only the effective-viewport form): items inside an off-screen + * scrollable container (closed drawer) must also count as off-screen, not just items scrolled out + * of an on-screen container. + * + * #1542: once the bulk tree says off-screen, the caller's live read gets one chance to rescue a + * FALSE refusal - a stale/corrupted bulk tree can say off-screen while the app is visually fine + * (zero cost on the accept path; runs only here). A confirmed rescue returns the node PATCHED WITH + * THE LIVE RECT: the caller must act on that returned node, never the original, because in the + * frozen-bulk-tree manifestation the original rect can be stale even when the rescue verdict is + * correct. Without a live read, or when it does not confirm, the refusal is unchanged. + */ +export async function throwIfOffscreenTarget( + node: TNode, + nodes: SnapshotNode[], + failure: { + message: string; + details: Record; + hint: (direction: OffscreenScrollDirection | null) => string; + }, + /** + * The optional live, tree-independent read that may overturn the verdict (#1542). It answers the + * target's live rect only on a positive confirmation, and `null` on anything else. + */ + confirmVisible?: (node: SnapshotNode, rootViewport: Rect | null) => Promise, +): Promise { + const visibility = createSnapshotVisibility(nodes); + const viewport = node.rect ? visibility.resolveEffectiveViewport(node) : null; + if (!node.rect || !viewport || visibility.isVisibleOnScreen(node)) return node; + const rootViewport = visibility.resolveViewport(node.rect); + const liveRect = await confirmVisible?.(node, rootViewport); + if (liveRect) return { ...node, rect: liveRect, hittable: true }; + // The direction that scrolls this off-screen target into view. Named in the + // hint (and surfaced as a machine-readable detail) so the recovery is a single + // deterministic move instead of a guess (#1366). Derived from the same + // boundary the rejection above used, so partial clips and off-screen + // containers get a direction too, not just fully-scrolled-out items. + const scrollDirection = classifyOffscreenScrollDirection(node, visibility); + throw discloseDispatch( + new AppError('COMMAND_FAILED', failure.message, { + ...failure.details, + rect: node.rect, + viewport, + ...(scrollDirection ? { scrollDirection } : {}), + hint: failure.hint(scrollDirection), + }), + 'no', + ); +} + +// Selector parity for the @ref off-screen guard: without it, a selector +// resolving to a closed drawer/carousel item "succeeds" by tapping coordinates +// outside the viewport (observed as `Tapped (-161, 265)` against Bluesky's +// closed drawer) while the same node via @ref is refused. +export function offscreenSelectorFailure( + selector: string, + action: InteractionAction, +): Parameters[2] { + return { + message: `Selector ${selector} resolved to an off-screen element and is not safe to ${action}`, + details: { reason: 'offscreen_selector', selector }, + // A selector re-resolves against a fresh snapshot on every attempt, so the + // recovery is: move the named direction, then retry THIS selector - no + // separate snapshot step, and no @ref (a scroll expires the ref frame, + // #1366). `--until` is that whole loop as one command: it checks the same + // selector between passes, which is also what keeps a large step from + // overshooting, so the hint no longer has to trade distance for accuracy. + hint: (direction) => + `${scrollRevealClause(direction, selector)} then retry ${action} with the same selector. --until checks the selector between passes, so it stops on the target rather than sailing past it. If it is inside a closed drawer or another tab, open that container first.`, + }; +} + +export function offscreenRefFailure( + refInput: string, + action: InteractionAction, +): Parameters[2] { + return { + message: `Ref ${refInput} is off-screen and not safe to ${action}`, + details: { reason: 'offscreen_ref', ref: normalizeRef(refInput) }, + // The scroll that reveals the target expires the ref frame (#1366, ADR + // 0014), so retrying this @ref would be rejected next. Steer to a selector, + // which re-resolves against a fresh snapshot and bypasses the ref-frame guard + // - and which `--until` can then check between passes. + hint: (direction) => + `${scrollRevealClause(direction, null)} then retry ${action} with a selector (e.g. text=/id=) rather than this @ref — the scroll expires the ref frame, so re-run snapshot -i before reusing any @ref.`, + }; +} + +/** + * Shared lead-in for both off-screen hints: the one command that reveals the target. + * + * When the geometry names a direction AND the caller has a selector to check, this is a complete + * `scroll --until ` - one request that stops on the target instead of the + * scroll-then-look-again loop the hint used to prescribe. Without a selector to check (an @ref + * refusal) or without a single reveal direction (off more than one edge), it degrades to naming + * the move and leaves the stop condition to the caller's own next step. + */ +function scrollRevealClause( + direction: OffscreenScrollDirection | null, + selector: string | null, +): string { + if (!direction) return 'Scroll toward it,'; + if (!selector) return `Scroll ${direction} toward it,`; + return `Run scroll ${direction} --until ${shellQuote(selector)} to bring it on screen,`; +} diff --git a/packages/selectors/src/interaction-targeting.ts b/packages/selectors/src/interaction-targeting.ts index 66f79b872c..6cd3ae5d95 100644 --- a/packages/selectors/src/interaction-targeting.ts +++ b/packages/selectors/src/interaction-targeting.ts @@ -14,6 +14,7 @@ import { } from '@agent-device/kernel/rect-center'; import { intersectArea } from '@agent-device/kernel/screenshot-geometry'; import { isSemanticTouchTarget } from './touch-semantics.ts'; +import type { InteractionAction } from './interaction-action.ts'; type ActionableTouchResolutionReason = | 'same-rect-descendant' @@ -450,3 +451,25 @@ export function isNestedTouchControl(nodes: readonly SnapshotNode[], node: Snaps return encloses && larger; }); } + +/** + * iOS AX `hittable` flags are unreliable on deep React Native trees (see #1037: + * a map-pin annotation exact-matched a longer recents row label and reported tap + * success while doing nothing visible). We deliberately do NOT fail or filter on + * this signal - that would break selectors that only ever resolve to nodes the + * platform marks non-hittable. Instead, surface it so the caller can notice a + * likely no-op tap and re-target with a ref or a more specific selector/longer text. + * + * The annotation half of the `nonHittable` guarantee, shared by the acting runtime paths and + * `find focus|type`, which dispatches its point itself. + */ +export function describeNonHittableTarget( + node: SnapshotNode, + action: InteractionAction, +): { targetHittable?: boolean; hint?: string } { + if (node.hittable !== false) return {}; + return { + targetHittable: false, + hint: `The resolved element reports hittable: false, so this ${action} may have had no visible effect. Verify with a snapshot, or prefer a @ref or a longer/more specific selector to target the intended element.`, + }; +} diff --git a/scripts/layering/package-boundaries.test.ts b/scripts/layering/package-boundaries.test.ts index 14678f530e..cb6a890b37 100644 --- a/scripts/layering/package-boundaries.test.ts +++ b/scripts/layering/package-boundaries.test.ts @@ -784,6 +784,7 @@ test('the real tree parses, declares, and passes R11', () => { '@agent-device/selectors/engine', '@agent-device/selectors/interaction-action', '@agent-device/selectors/interaction-error', + '@agent-device/selectors/interaction-offscreen', '@agent-device/selectors/interaction-positionals', '@agent-device/selectors/interaction-targeting', '@agent-device/selectors/interaction-touch-point', diff --git a/src/commands/schema/cli-help.ts b/src/commands/schema/cli-help.ts index 5a3bf3ca59..301e01559e 100644 --- a/src/commands/schema/cli-help.ts +++ b/src/commands/schema/cli-help.ts @@ -99,7 +99,7 @@ Snapshots and refs: Selectors: id="field-email", label="Allow", role=button label="Search" -- not bare role keys (button="Search"); no CSS selectors/--selector/--text/raw x-y when refs/selectors exist. Mutating selector ambiguity: press/click/fill/longpress collapse duplicate accessibility wrappers only when every match is one ancestor-descendant chain resolving to the same actionable node. Matches in distinct subtrees fail with AMBIGUOUS_MATCH and a bounded candidate list; geometry never chooses a winner. Retry one printed candidate ref (pinned to refsGeneration) or narrow the selector with role/id/longer text. Uniqueness-based reads (is except exists/absent, get attrs) share that failure shape. - hittable: false on a resolved element does not block dispatch (iOS AX flags are unreliable on deep RN trees); press/fill/click return targetHittable: false plus a hint -- verify or re-target, not a failure. + hittable: false on a resolved element does not block dispatch (iOS AX flags are unreliable on deep RN trees); press/fill/click and find focus/type return targetHittable: false plus a hint -- verify or re-target, not a failure. Text entry: fill replaces; type appends to an already-focused field: fill 'id="field-email"' "qa@example.com"; type "Handle with care" --delay-ms 80 diff --git a/test/integration/interaction-contract/daemon-harness.ts b/test/integration/interaction-contract/daemon-harness.ts index b3793e0c02..14d5b8b3dd 100644 --- a/test/integration/interaction-contract/daemon-harness.ts +++ b/test/integration/interaction-contract/daemon-harness.ts @@ -124,6 +124,17 @@ export function runnerTypeEntry( }; } +export function runnerQuerySelectorEntry( + result: Record, +): ProviderScenarioProviderEntry { + return { + command: 'ios.runner.querySelector', + deviceId: CONTRACT_DEVICE_ID, + platform: 'apple', + result, + }; +} + export function runnerLongPressEntry( result: Record, request?: Record, diff --git a/test/integration/interaction-contract/find-focus-dispatch.contract.test.ts b/test/integration/interaction-contract/find-focus-dispatch.contract.test.ts index ce35576526..779f8bc3c7 100644 --- a/test/integration/interaction-contract/find-focus-dispatch.contract.test.ts +++ b/test/integration/interaction-contract/find-focus-dispatch.contract.test.ts @@ -6,13 +6,30 @@ import { assertRpcError, assertRpcOk } from '../provider-scenarios/assertions.ts import { PARALLEL_PROVIDER_SCENARIO_TIMEOUT_MS } from '../provider-scenarios/test-timeouts.ts'; import { scenarioName, scenarioNames } from './coverage-manifest.ts'; import { FIND_FOCUS_DISPATCH_COVERAGE } from './find-focus-dispatch.coverage.ts'; -import { fullyTiledParentSnapshot } from './fixtures.ts'; import { + closedDrawerSnapshot, + fullyTiledParentSnapshot, + nonHittableButtonSnapshot, +} from './fixtures.ts'; +import { + runnerQuerySelectorEntry, runnerSnapshotEntry, runnerTapEntry, runnerTypeEntry, withIosContractDaemon, } from './daemon-harness.ts'; +import { + APPLICATION_NODE, + coveredFormTree, + fieldFillingCellTree, + labelBesideFieldTree, + labelledWrapperTree, + namedFieldBesideSiblingTree, + nonHittableRowTree, + partlyTiledCardTree, + twinFormTree, + twoFieldsTree, +} from './find-focus-dispatch.fixtures.ts'; // ADR 0011 Layer 3, find-focus-dispatch path: mutating `find focus` and the focus leg of // `find type`. These run over the daemon harness because the seam lives in the daemon's @@ -148,13 +165,11 @@ test( ); test( - 'find-focus-dispatch promotion: a non-hittable label retargets the dispatch to the hittable ancestor row', + scenario('nonHittable'), async () => { // Discriminating, not decorative: the label's own center is (140,195), the row's is (200,200). // Retargeting `findAct.promotion` away from `hittable-ancestor-below-root` dispatches at the - // label and this fails. Title deliberately outside the coverage manifest: the promotion half runs - // while the annotation half stays the `nonHittable` waiver's debt, and the gate forbids a - // manifest entry on a waived cell. + // label and this fails. The promoted row is hittable, so the response carries no annotation. await withIosContractDaemon( [runnerSnapshotEntry(nonHittableRowTree()), runnerTapEntry({})], async (daemon) => { @@ -162,12 +177,172 @@ test( await daemon.callCommand('find', ['label', 'Notifications', 'focus']), ); assert.deepEqual({ x: data.x, y: data.y }, { x: 200, y: 200 }); + assert.equal(data.targetHittable, undefined); + }, + ); + }, + PARALLEL_PROVIDER_SCENARIO_TIMEOUT_MS, +); + +test( + scenarioNames(FIND_FOCUS_DISPATCH_COVERAGE, 'nonHittable')[1]!, + async () => { + // The root itself: nothing to promote to, so the focused node keeps `hittable: false`. + const tree = nonHittableButtonSnapshot().nodes; + const entries = [runnerSnapshotEntry(tree), runnerTapEntry({})]; + await withIosContractDaemon([...entries, ...entries, runnerTypeEntry({})], async (daemon) => { + const focused = assertRpcOk(await daemon.callCommand('find', ['label', 'Continue', 'focus'])); + const typed = assertRpcOk( + await daemon.callCommand('find', ['label', 'Continue', 'type', 'hello']), + ); + assert.deepEqual({ x: focused.x, y: focused.y }, { x: 60, y: 40 }); + assert.equal(typed.text, 'hello'); + for (const data of [focused, typed]) { + assert.equal(data.targetHittable, false); + assert.match(String(data.hint), /reports hittable: false, so this focus may have had no/); + } + }); + }, + PARALLEL_PROVIDER_SCENARIO_TIMEOUT_MS, +); + +// A closed drawer item sits wholly left of the root viewport. A regular iOS capture never carries +// such a node, so these scenarios read with --raw, as a caller reaching one does. +const RAW = { snapshotRaw: true }; +const DRAWER_TREE = closedDrawerSnapshot().nodes; + +test( + scenario('offscreen'), + async () => { + // The live read finds nothing, so the rescue cannot overturn the verdict, and the transcript + // holds no tap or type entry. + const entries = [ + runnerSnapshotEntry(DRAWER_TREE), + runnerQuerySelectorEntry({ found: false, matchCount: 0 }), + ]; + await withIosContractDaemon([...entries, ...entries], async (daemon) => { + for (const action of [['focus'], ['type', 'hello']]) { + const error = assertRpcError( + await daemon.callCommand('find', ['label', 'Explore', ...action], RAW), + 500, + 'COMMAND_FAILED', + /Ref @e2 is off-screen and not safe to focus/, + ); + const details = readErrorDetails(error); + assert.equal(details.reason, 'offscreen_ref'); + assert.equal(details.ref, 'e2'); + assert.deepEqual(details.rect, DRAWER_TREE[1]!.rect); + assert.equal(details.dispatched, 'no'); + assert.match(String(error.hint), new RegExp(`re-run find to ${action[0]}`)); + } + }); + }, + PARALLEL_PROVIDER_SCENARIO_TIMEOUT_MS, +); + +test( + scenarioNames(FIND_FOCUS_DISPATCH_COVERAGE, 'offscreen')[1]!, + async () => { + // The #1542 rescue: a stale bulk tree says off-screen and non-hittable while the live probe + // confirms a single match on screen and hittable, so focus lands at the live rect without a + // stale non-hittable warning. + const staleDrawerTree = DRAWER_TREE.map((node) => + node.index === 1 ? { ...node, hittable: false } : node, + ); + const liveRect = { x: 20, y: 240, width: 300, height: 50 }; + await withIosContractDaemon( + [ + runnerSnapshotEntry(staleDrawerTree), + runnerQuerySelectorEntry({ + found: true, + matchCount: 1, + hittable: true, + rect: liveRect, + }), + runnerTapEntry({}), + ], + async (daemon, transcript) => { + const data = assertRpcOk( + await daemon.callCommand('find', ['label', 'Explore', 'focus'], RAW), + ); + assert.deepEqual({ x: data.x, y: data.y }, { x: 170, y: 265 }); + assert.equal(data.targetHittable, undefined); + assert.equal(data.hint, undefined); + const queryCall = transcript.calls.find( + (call) => call.command === 'ios.runner.querySelector', + ); + const queryReq = queryCall?.request as Record | undefined; + assert.equal(queryReq?.hittabilityProbe, true); + assert.equal(queryReq?.selectorKey, 'label'); + assert.equal(queryReq?.selectorValue, 'Explore'); + }, + ); + }, + PARALLEL_PROVIDER_SCENARIO_TIMEOUT_MS, +); + +test( + scenarioNames(FIND_FOCUS_DISPATCH_COVERAGE, 'offscreen')[2]!, + async () => { + // The ranking still runs before the refusal: an on-screen twin after the drawer item wins + // without any live read. + const twin = { ...DRAWER_TREE[1]!, index: 2, rect: { x: 20, y: 400, width: 300, height: 50 } }; + await withIosContractDaemon( + [runnerSnapshotEntry([...DRAWER_TREE, twin]), runnerTapEntry({})], + async (daemon) => { + const data = assertRpcOk( + await daemon.callCommand('find', ['label', 'Explore', 'focus'], RAW), + ); + assert.deepEqual({ x: data.x, y: data.y }, { x: 170, y: 425 }); + }, + ); + }, + PARALLEL_PROVIDER_SCENARIO_TIMEOUT_MS, +); + +test( + scenarioNames(FIND_FOCUS_DISPATCH_COVERAGE, 'offscreen')[3]!, + async () => { + // A duplicate match on the derived label/id must fail closed so find type cannot rescue onto + // a different control sharing the same label. + await withIosContractDaemon( + [runnerSnapshotEntry(DRAWER_TREE), runnerQuerySelectorEntry({ found: true, matchCount: 2 })], + async (daemon) => { + const error = assertRpcError( + await daemon.callCommand('find', ['label', 'Explore', 'type', 'hello'], RAW), + 500, + 'COMMAND_FAILED', + /Ref @e2 is off-screen and not safe to focus/, + ); + const details = readErrorDetails(error); + assert.equal(details.reason, 'offscreen_ref'); + assert.equal(details.dispatched, 'no'); }, ); }, PARALLEL_PROVIDER_SCENARIO_TIMEOUT_MS, ); +test( + scenarioNames(FIND_FOCUS_DISPATCH_COVERAGE, 'offscreen')[4]!, + async () => { + // A role-qualified query cannot be re-verified by an id/label-only probe; transcript has no + // querySelector, tap, or type entry so any rescue attempt fails assertComplete. + await withIosContractDaemon([runnerSnapshotEntry(DRAWER_TREE)], async (daemon) => { + const error = assertRpcError( + await daemon.callCommand('find', ['role=button label="Explore"', 'type', 'example'], RAW), + 500, + 'COMMAND_FAILED', + /Ref @e2 is off-screen and not safe to focus/, + ); + const details = readErrorDetails(error); + assert.equal(details.reason, 'offscreen_ref'); + assert.equal(details.dispatched, 'no'); + }); + }, + PARALLEL_PROVIDER_SCENARIO_TIMEOUT_MS, +); + test( scenarioNames(FIND_FOCUS_DISPATCH_COVERAGE, 'parentOwnedTouchPoint')[0]!, async () => { @@ -658,225 +833,3 @@ function readErrorDetails(error: Record): Record; } - -const APPLICATION_NODE = { - index: 0, - type: 'Application', - rect: { x: 0, y: 0, width: 400, height: 800 }, -}; - -function labelledWrapperTree(): unknown[] { - return [ - APPLICATION_NODE, - { - index: 1, - parentIndex: 0, - type: 'Other', - label: 'Email', - rect: { x: 20, y: 100, width: 360, height: 80 }, - hittable: true, - }, - { - index: 2, - parentIndex: 1, - type: 'TextField', - rect: { x: 32, y: 130, width: 336, height: 40 }, - hittable: true, - }, - ]; -} - -function fieldFillingCellTree(): unknown[] { - return [ - APPLICATION_NODE, - { - index: 1, - parentIndex: 0, - type: 'Cell', - label: 'Email', - rect: { x: 0, y: 100, width: 400, height: 44 }, - hittable: true, - }, - { - index: 2, - parentIndex: 1, - type: 'TextField', - rect: { x: 16, y: 100, width: 368, height: 44 }, - hittable: true, - }, - ]; -} - -function labelBesideFieldTree(): unknown[] { - return [ - APPLICATION_NODE, - { - index: 1, - parentIndex: 0, - type: 'Other', - rect: { x: 20, y: 100, width: 360, height: 80 }, - hittable: true, - }, - { - index: 2, - parentIndex: 1, - type: 'StaticText', - label: 'Email', - rect: { x: 32, y: 100, width: 336, height: 24 }, - hittable: false, - }, - { - index: 3, - parentIndex: 1, - type: 'TextField', - rect: { x: 32, y: 130, width: 336, height: 40 }, - hittable: true, - }, - ]; -} - -function namedFieldBesideSiblingTree({ enabled }: { enabled: boolean }): unknown[] { - return [ - APPLICATION_NODE, - { - index: 1, - parentIndex: 0, - type: 'Other', - rect: { x: 20, y: 100, width: 360, height: 160 }, - hittable: true, - }, - { - index: 2, - parentIndex: 1, - type: 'TextView', - label: 'Notes', - enabled, - rect: { x: 32, y: 130, width: 336, height: 40 }, - hittable: false, - }, - { - index: 3, - parentIndex: 1, - type: 'TextField', - label: 'Title', - rect: { x: 32, y: 200, width: 336, height: 40 }, - hittable: true, - }, - ]; -} - -function twoFieldsTree(): unknown[] { - return [ - APPLICATION_NODE, - { - index: 1, - parentIndex: 0, - type: 'Other', - label: 'Address', - rect: { x: 20, y: 100, width: 360, height: 160 }, - hittable: true, - }, - { - index: 2, - parentIndex: 1, - type: 'TextField', - label: 'Street', - rect: { x: 32, y: 130, width: 336, height: 40 }, - hittable: true, - }, - { - index: 3, - parentIndex: 1, - type: 'TextField', - label: 'City', - rect: { x: 32, y: 200, width: 336, height: 40 }, - hittable: true, - }, - ]; -} - -function partlyTiledCardTree(): unknown[] { - return [ - { index: 0, type: 'Application', rect: { x: 0, y: 0, width: 400, height: 800 } }, - { - index: 1, - parentIndex: 0, - type: 'Link', - label: 'Card', - rect: { x: 20, y: 100, width: 360, height: 200 }, - hittable: true, - }, - { - index: 2, - parentIndex: 1, - type: 'Button', - label: 'Action', - rect: { x: 20, y: 180, width: 360, height: 40 }, - hittable: true, - }, - ]; -} - -function nonHittableRowTree(): unknown[] { - return [ - { index: 0, type: 'Application', rect: { x: 0, y: 0, width: 400, height: 800 } }, - { - index: 1, - parentIndex: 0, - type: 'Cell', - rect: { x: 40, y: 160, width: 320, height: 80 }, - hittable: true, - }, - { - index: 2, - parentIndex: 1, - type: 'StaticText', - label: 'Notifications', - rect: { x: 60, y: 180, width: 160, height: 30 }, - hittable: false, - }, - ]; -} - -function coveredFormTree(): unknown[] { - return [ - { index: 0, type: 'Application', rect: { x: 0, y: 0, width: 390, height: 844 } }, - { - index: 1, - parentIndex: 0, - type: 'Button', - label: 'Form', - rect: { x: 16, y: 790, width: 140, height: 44 }, - hittable: true, - }, - { - index: 2, - parentIndex: 0, - type: 'TabBar', - rect: { x: 0, y: 760, width: 390, height: 84 }, - hittable: true, - }, - ]; -} - -function twinFormTree(): unknown[] { - return [ - { index: 0, type: 'Application', rect: { x: 0, y: 0, width: 400, height: 800 } }, - { - index: 1, - parentIndex: 0, - type: 'Button', - label: 'Form', - rect: { x: 40, y: 200, width: 120, height: 44 }, - hittable: true, - }, - { - index: 2, - parentIndex: 0, - type: 'Button', - label: 'Form', - rect: { x: 240, y: 200, width: 120, height: 44 }, - hittable: true, - }, - ]; -} diff --git a/test/integration/interaction-contract/find-focus-dispatch.coverage.ts b/test/integration/interaction-contract/find-focus-dispatch.coverage.ts index d3c8eac165..5e0d05f312 100644 --- a/test/integration/interaction-contract/find-focus-dispatch.coverage.ts +++ b/test/integration/interaction-contract/find-focus-dispatch.coverage.ts @@ -26,6 +26,17 @@ export const FIND_FOCUS_DISPATCH_COVERAGE = definePathCoverage('find-focus-dispa 'find-focus-dispatch parentOwnedTouchPoint: an off-screen field neither takes the focus nor counts', 'find-focus-dispatch parentOwnedTouchPoint: a label inside a disabled field refuses instead of writing into its sibling', ], + offscreen: [ + 'find-focus-dispatch offscreen: a closed-drawer match refuses with offscreen_ref before dispatch', + 'find-focus-dispatch offscreen: a live read that confirms the match on screen focuses at the live rect', + 'find-focus-dispatch offscreen: an on-screen twin still wins over an off-screen one', + 'find-focus-dispatch offscreen: a live hittability probe with duplicate matches refuses instead of rescuing', + 'find-focus-dispatch offscreen: a role-qualified find refuses without running an id/label rescue probe', + ], + nonHittable: [ + 'find-focus-dispatch nonHittable: a non-hittable label retargets the dispatch to the hittable ancestor row', + 'find-focus-dispatch nonHittable: a target still non-hittable after promotion answers with targetHittable false and a hint', + ], responseConstruction: 'find-focus-dispatch responseConstruction: find focus returns the shared point-focus payload', errorTaxonomy: diff --git a/test/integration/interaction-contract/find-focus-dispatch.fixtures.ts b/test/integration/interaction-contract/find-focus-dispatch.fixtures.ts new file mode 100644 index 0000000000..e606464eb9 --- /dev/null +++ b/test/integration/interaction-contract/find-focus-dispatch.fixtures.ts @@ -0,0 +1,221 @@ +export const APPLICATION_NODE = { + index: 0, + type: 'Application', + rect: { x: 0, y: 0, width: 400, height: 800 }, +}; + +export function labelledWrapperTree(): unknown[] { + return [ + APPLICATION_NODE, + { + index: 1, + parentIndex: 0, + type: 'Other', + label: 'Email', + rect: { x: 20, y: 100, width: 360, height: 80 }, + hittable: true, + }, + { + index: 2, + parentIndex: 1, + type: 'TextField', + rect: { x: 32, y: 130, width: 336, height: 40 }, + hittable: true, + }, + ]; +} + +export function fieldFillingCellTree(): unknown[] { + return [ + APPLICATION_NODE, + { + index: 1, + parentIndex: 0, + type: 'Cell', + label: 'Email', + rect: { x: 0, y: 100, width: 400, height: 44 }, + hittable: true, + }, + { + index: 2, + parentIndex: 1, + type: 'TextField', + rect: { x: 16, y: 100, width: 368, height: 44 }, + hittable: true, + }, + ]; +} + +export function labelBesideFieldTree(): unknown[] { + return [ + APPLICATION_NODE, + { + index: 1, + parentIndex: 0, + type: 'Other', + rect: { x: 20, y: 100, width: 360, height: 80 }, + hittable: true, + }, + { + index: 2, + parentIndex: 1, + type: 'StaticText', + label: 'Email', + rect: { x: 32, y: 100, width: 336, height: 24 }, + hittable: false, + }, + { + index: 3, + parentIndex: 1, + type: 'TextField', + rect: { x: 32, y: 130, width: 336, height: 40 }, + hittable: true, + }, + ]; +} + +export function namedFieldBesideSiblingTree({ enabled }: { enabled: boolean }): unknown[] { + return [ + APPLICATION_NODE, + { + index: 1, + parentIndex: 0, + type: 'Other', + rect: { x: 20, y: 100, width: 360, height: 160 }, + hittable: true, + }, + { + index: 2, + parentIndex: 1, + type: 'TextView', + label: 'Notes', + enabled, + rect: { x: 32, y: 130, width: 336, height: 40 }, + hittable: false, + }, + { + index: 3, + parentIndex: 1, + type: 'TextField', + label: 'Title', + rect: { x: 32, y: 200, width: 336, height: 40 }, + hittable: true, + }, + ]; +} + +export function twoFieldsTree(): unknown[] { + return [ + APPLICATION_NODE, + { + index: 1, + parentIndex: 0, + type: 'Other', + label: 'Address', + rect: { x: 20, y: 100, width: 360, height: 160 }, + hittable: true, + }, + { + index: 2, + parentIndex: 1, + type: 'TextField', + label: 'Street', + rect: { x: 32, y: 130, width: 336, height: 40 }, + hittable: true, + }, + { + index: 3, + parentIndex: 1, + type: 'TextField', + label: 'City', + rect: { x: 32, y: 200, width: 336, height: 40 }, + hittable: true, + }, + ]; +} + +export function partlyTiledCardTree(): unknown[] { + return [ + APPLICATION_NODE, + { + index: 1, + parentIndex: 0, + type: 'Link', + label: 'Card', + rect: { x: 20, y: 100, width: 360, height: 200 }, + hittable: true, + }, + { + index: 2, + parentIndex: 1, + type: 'Button', + label: 'Action', + rect: { x: 20, y: 180, width: 360, height: 40 }, + hittable: true, + }, + ]; +} + +export function nonHittableRowTree(): unknown[] { + return [ + APPLICATION_NODE, + { + index: 1, + parentIndex: 0, + type: 'Cell', + rect: { x: 40, y: 160, width: 320, height: 80 }, + hittable: true, + }, + { + index: 2, + parentIndex: 1, + type: 'StaticText', + label: 'Notifications', + rect: { x: 60, y: 180, width: 160, height: 30 }, + hittable: false, + }, + ]; +} + +export function coveredFormTree(): unknown[] { + return [ + { index: 0, type: 'Application', rect: { x: 0, y: 0, width: 390, height: 844 } }, + { + index: 1, + parentIndex: 0, + type: 'Button', + label: 'Form', + rect: { x: 16, y: 790, width: 140, height: 44 }, + hittable: true, + }, + { + index: 2, + parentIndex: 0, + type: 'TabBar', + rect: { x: 0, y: 760, width: 390, height: 84 }, + hittable: true, + }, + ]; +} + +export function twinFormTree(): unknown[] { + return [ + APPLICATION_NODE, + { + index: 1, + parentIndex: 0, + type: 'Button', + label: 'Form', + rect: { x: 40, y: 200, width: 120, height: 44 }, + hittable: true, + }, + { + index: 2, + parentIndex: 0, + type: 'Button', + label: 'Form', + rect: { x: 240, y: 200, width: 120, height: 44 }, + hittable: true, + }, + ]; +} diff --git a/test/repo-guards/interaction-guarantees.test.ts b/test/repo-guards/interaction-guarantees.test.ts index e2782a207c..0889da5004 100644 --- a/test/repo-guards/interaction-guarantees.test.ts +++ b/test/repo-guards/interaction-guarantees.test.ts @@ -164,8 +164,6 @@ test('acknowledged gaps are visible and bounded', () => { // (umbrella: https://github.com/callstack/agent-device/issues/1081). assert.deepEqual(gaps.sort(), [ 'coordinate/outcomeObservation', - 'find-focus-dispatch/nonHittable', - 'find-focus-dispatch/offscreen', 'find-focus-dispatch/outcomeObservation', 'find-focus-dispatch/targetStability', 'maestro-direct-selector/errorTaxonomy', diff --git a/website/docs/docs/selectors.md b/website/docs/docs/selectors.md index 02e557bb47..0ac141541b 100644 --- a/website/docs/docs/selectors.md +++ b/website/docs/docs/selectors.md @@ -64,6 +64,8 @@ Tips: - Several enabled on-screen fields inside the match refuse with `AMBIGUOUS_MATCH` and the field candidates. - A disabled target refuses with `COMMAND_FAILED` and reason `target_disabled`. This includes a match whose only on-screen fields are disabled. - A match or field covered by another visible element refuses with `COMMAND_FAILED` and reason `target_covered`. + - A target outside the screen refuses with `COMMAND_FAILED` and reason `offscreen_ref`, as `click @ref` does. `find` prefers an on-screen match, so this happens only when the chosen target is off screen. + - A target that still reports `hittable: false` is focused, and the response adds `targetHittable: false` and a `hint`, as `press`, `click`, and `fill` do. - Web text boxes are not recognized as text fields, so a web match keeps its own aim. - Use `fill` to find a field and replace its text in one step. To append to the text in the focused field, use `click` or `press`, then `type`.