Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 12 additions & 0 deletions packages/command-output/src/cli-output.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -238,6 +238,18 @@ describe('the two stdout contracts in one run', () => {
expect(output.stderr).toContain(`Warning: ${REPAIR_WARNING}`);
});

test('find focus/type routes a non-hittable target hint through warnings without polluting stdout', async () => {
const hint =
'The resolved element reports hittable: false, so this focus may have had no visible effect.';
const output = await render(
'find',
{ message: 'Focused (60, 40)', x: 60, y: 40, targetHittable: false, hint },
{ query: 'Continue', action: 'focus' },
);
expect(output.text).toBe('Focused (60, 40)');
expect(output.stderr).toBe(`Warning: ${hint}\n`);
});

test('is prints the disclosure after its verdict on stdout', async () => {
const output = await render(
'is',
Expand Down
8 changes: 7 additions & 1 deletion packages/command-output/src/output-common.ts
Original file line number Diff line number Diff line change
Expand Up @@ -76,12 +76,18 @@ export function messageCliOutput(result: Record<string, unknown>): CliOutput {
* one renders on a single line.
*/
function collectResponseWarnings(result: Record<string, unknown>): string[] {
return [
const raw = [
...(typeof result.warning === 'string' && result.warning.trim() !== '' ? [result.warning] : []),
...readResponseWarnings(result),
...(result.targetHittable === false &&
typeof result.hint === 'string' &&
result.hint.trim() !== ''
? [result.hint]
: []),
]
.map((warning) => collapseWarningText(warning))
.filter((warning) => warning.length > 0);
return Array.from(new Set(raw));
}

/**
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,10 +19,12 @@ import { toBackendContext } from '../../runtime-common.ts';
import { toBackendResult } from '../../runtime-types.ts';
import type { InteractionAction } from '@agent-device/selectors/interaction-action';
import { tryResolveRefNode } from './ref-target-resolution.ts';
import { EXACT_REF_RESOLUTION, describeNonHittableTarget } from './resolution-disclosure.ts';
import { EXACT_REF_RESOLUTION } from './resolution-disclosure.ts';
import { describeNonHittableTarget } from '@agent-device/selectors/interaction-targeting';
import { offscreenRefFailure } from '@agent-device/selectors/interaction-offscreen';
import {
assertVisibleRefTarget,
runInteractionPipelineStages,
throwIfOffscreenInteractionTarget,
} from './target-visibility-stages.ts';

/**
Expand All @@ -33,7 +35,7 @@ import {
* run the SAME shared guards the runtime path uses against it before the
* backend call — occlusion (`isSnapshotNodeInteractionBlocked` via
* `assertInteractionNotBlocked`) and offscreen (the snapshot visibility resolver via
* `assertVisibleRefTarget`) ERROR with the runtime path's exact shapes, and
* `throwIfOffscreenInteractionTarget`) ERROR with the runtime path's exact shapes, and
* the non-hittable annotation is returned for the fast-path result.
*
* Zero extra round trips by construction on the accept path: no session, no
Expand Down Expand Up @@ -84,10 +86,13 @@ export async function preflightNativeRefInteraction(
label: `Ref ${target.ref}`,
hooks: {
offscreen: async (node, tree) =>
await assertVisibleRefTarget(runtime, options, node, tree, target.ref, {
action,
pipeline,
}),
await throwIfOffscreenInteractionTarget(
runtime,
options,
node,
tree,
offscreenRefFailure(target.ref, action),
),
},
resolveTapPoint: (node) => resolveRectCenter(node.rect),
});
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -34,9 +34,10 @@ import {
describeResolvedInteractionNode,
type ResolvedRefNode,
} from './resolution-disclosure.ts';
import { offscreenRefFailure } from '@agent-device/selectors/interaction-offscreen';
import {
assertVisibleRefTarget,
runInteractionPipelineStages,
throwIfOffscreenInteractionTarget,
} from './target-visibility-stages.ts';

/** The node a ref target acts on, plus the tree the shared guards read it against. */
Expand Down Expand Up @@ -122,7 +123,13 @@ export async function resolveRefInteractionTarget(
hooks: {
onResolved: (node, tree) => assertReplayTargetResolution(node, tree, params),
offscreen: async (node, tree) =>
await assertVisibleRefTarget(runtime, options, node, tree, target.ref, params),
await throwIfOffscreenInteractionTarget(
runtime,
options,
node,
tree,
offscreenRefFailure(target.ref, params.action),
),
},
resolveTapPoint: (node) =>
resolveNodeTouchPoint(node, nodes, {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ import type { SnapshotState } from '@agent-device/contracts/snapshot-model';
import type { AgentDeviceRuntime } from '@agent-device/contracts/runtime-contract';
import { buildSelectorChainForNode } from '@agent-device/selectors';
import { resolvePressRecordingTarget } from '@agent-device/selectors/press-retarget';
import { describeNonHittableTarget } from '@agent-device/selectors/interaction-targeting';
import { resolveRefLabel } from '@agent-device/capture-kit/snapshot-node-lookup';
import { normalizeType } from '@agent-device/contracts/snapshot';
import { truncateUtf8 } from './truncate-utf8.ts';
Expand Down Expand Up @@ -157,22 +158,3 @@ function pressRecordingTargetOverride(
},
};
}

/**
* iOS AX `hittable` flags are unreliable on deep React Native trees (see #1037:
* a map-pin annotation exact-matched a longer recents row label and reported tap
* success while doing nothing visible). We deliberately do NOT fail or filter on
* this signal — that would break selectors that only ever resolve to nodes the
* platform marks non-hittable. Instead, surface it so the caller can notice a
* likely no-op tap and re-target with a ref or a more specific selector/longer text.
*/
export function describeNonHittableTarget(
node: SnapshotNode,
action: InteractionAction,
): { targetHittable?: boolean; hint?: string } {
if (node.hittable !== false) return {};
return {
targetHittable: false,
hint: `The resolved element reports hittable: false, so this ${action} may have had no visible effect. Verify with a snapshot, or prefer a @ref or a longer/more specific selector to target the intended element.`,
};
}
Original file line number Diff line number Diff line change
Expand Up @@ -25,9 +25,10 @@ import {
buildSelectorResolutionDisclosure,
describeResolvedInteractionNode,
} from './resolution-disclosure.ts';
import { offscreenSelectorFailure } from '@agent-device/selectors/interaction-offscreen';
import {
assertVisibleSelectorTarget,
runInteractionPipelineStages,
throwIfOffscreenInteractionTarget,
} from './target-visibility-stages.ts';

export type { InteractionRequestTarget, ResolvedInteractionTarget };
Expand Down Expand Up @@ -156,7 +157,13 @@ async function resolveSelectorInteractionTarget(
hooks: {
onResolved: (node, tree) => assertReplayTargetResolution(node, tree, params),
offscreen: async (node, tree) =>
await assertVisibleSelectorTarget(runtime, options, node, tree, selected.selector, params),
await throwIfOffscreenInteractionTarget(
runtime,
options,
node,
tree,
offscreenSelectorFailure(selected.selector, params.action),
),
},
resolveTapPoint: (node) =>
resolveNodeTouchPoint(node, capture.snapshot.nodes, {
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@
import { shellQuote } from '@agent-device/kernel/device-shell';
import { AppError, discloseDispatch } from '@agent-device/kernel/errors';
import { assertSnapshotScreenOrigin } from '@agent-device/contracts/capture';
import type {
Expand All @@ -8,7 +7,6 @@ import type {
SnapshotNode,
} from '@agent-device/kernel/snapshot';
import type { SnapshotState } from '@agent-device/contracts/snapshot-model';
import { normalizeRef } from '@agent-device/kernel/snapshot';
import type { AgentDeviceRuntime, CommandContext } from '@agent-device/contracts/runtime-contract';
import type {
ResolvedInteractionTarget,
Expand All @@ -19,12 +17,8 @@ import {
type SelectorPipelineHooks,
} from '@agent-device/selectors/selector-pipeline';
import type { ActingPipelinePolicy } from '@agent-device/selectors/selector-pipeline-policy';
import { createSnapshotVisibility } from '@agent-device/contracts/snapshot';
import { coveredInteractionDetails } from '@agent-device/selectors/interaction-error';
import {
classifyOffscreenScrollDirection,
type OffscreenScrollDirection,
} from '@agent-device/capture-kit/mobile-snapshot-semantics';
import { throwIfOffscreenTarget } from '@agent-device/selectors/interaction-offscreen';
import { toBackendContext } from '../../runtime-common.ts';
import { assertTapTargetClearOfVisibleKeyboard } from '@agent-device/selectors/keyboard-occlusion';
import {
Expand Down Expand Up @@ -108,91 +102,10 @@ export async function runInteractionPipelineStages<TPoint extends Point | null>(
return { node: target.node, tapPoint };
}

/**
* The off-screen stage's refusal shape. Reached only through the pipeline
* owner, and only for rows whose off-screen stage refuses — the row's decision
* is made there, so this builds the message and never re-decides.
*/
type OffscreenStageParams = { action: InteractionAction; pipeline: ActingPipelinePolicy };

// Selector parity for the @ref off-screen guard: without it, a selector
// resolving to a closed drawer/carousel item "succeeds" by tapping coordinates
// outside the viewport (observed as `Tapped (-161, 265)` against Bluesky's
// closed drawer) while the same node via @ref is refused.
export async function assertVisibleSelectorTarget(
runtime: AgentDeviceRuntime,
options: CommandContext,
node: SnapshotNode,
nodes: SnapshotState['nodes'],
selector: string,
{ action }: OffscreenStageParams,
): Promise<SnapshotNode> {
return await throwIfOffscreenInteractionTarget(runtime, options, node, nodes, {
message: `Selector ${selector} resolved to an off-screen element and is not safe to ${action}`,
details: { reason: 'offscreen_selector', selector },
// A selector re-resolves against a fresh snapshot on every attempt, so the
// recovery is: move the named direction, then retry THIS selector — no
// separate snapshot step, and no @ref (a scroll expires the ref frame,
// #1366). `--until` is that whole loop as one command: it checks the same
// selector between passes, which is also what keeps a large step from
// overshooting, so the hint no longer has to trade distance for accuracy.
hint: (direction) =>
`${scrollRevealClause(direction, selector)} then retry ${action} with the same selector. --until checks the selector between passes, so it stops on the target rather than sailing past it. If it is inside a closed drawer or another tab, open that container first.`,
});
}

export async function assertVisibleRefTarget(
runtime: AgentDeviceRuntime,
options: CommandContext,
node: SnapshotNode,
nodes: SnapshotState['nodes'],
refInput: string,
{ action }: OffscreenStageParams,
): Promise<SnapshotNode> {
return await throwIfOffscreenInteractionTarget(runtime, options, node, nodes, {
message: `Ref ${refInput} is off-screen and not safe to ${action}`,
details: { reason: 'offscreen_ref', ref: normalizeRef(refInput) },
// The scroll that reveals the target expires the ref frame (#1366, ADR
// 0014), so retrying this @ref would be rejected next. Steer to a selector,
// which re-resolves against a fresh snapshot and bypasses the ref-frame guard
// — and which `--until` can then check between passes.
hint: (direction) =>
`${scrollRevealClause(direction, null)} then retry ${action} with a selector (e.g. text=/id=) rather than this @ref — the scroll expires the ref frame, so re-run snapshot -i before reusing any @ref.`,
});
}

/**
* Shared lead-in for both off-screen hints: the one command that reveals the target.
*
* When the geometry names a direction AND the caller has a selector to check, this is a complete
* `scroll <dir> --until <selector>` — one request that stops on the target instead of the
* scroll-then-look-again loop the hint used to prescribe. Without a selector to check (an @ref
* refusal) or without a single reveal direction (off more than one edge), it degrades to naming
* the move and leaves the stop condition to the caller's own next step.
*/
function scrollRevealClause(
direction: OffscreenScrollDirection | null,
selector: string | null,
): string {
if (!direction) return 'Scroll toward it,';
if (!selector) return `Scroll ${direction} toward it,`;
return `Run scroll ${direction} --until ${shellQuote(selector)} to bring it on screen,`;
}

// Full on-screen visibility (not only the effective-viewport form): items inside an
// off-screen scrollable container (closed drawer) must also count as
// off-screen, not just items scrolled out of an on-screen container.
//
// #1542: once the bulk tree says off-screen, the guard gives iOS one chance
// to rescue a FALSE refusal via the optional backend.confirmOffscreenTargetVisible
// hook — a stale/corrupted bulk tree can say off-screen while the app is
// visually fine (zero cost on the accept path; runs only here). A confirmed
// rescue returns the node PATCHED WITH THE LIVE RECT: the caller must act on
// that returned node, never the original, because in the frozen-bulk-tree
// manifestation the original rect can be stale even when the rescue verdict
// is correct — tapping it would silently land at the wrong coordinate. The
// hook fails closed (null) on anything short of a positive confirmation, so
// a genuine refusal, or any backend without the hook, is unchanged.
// The runtime paths' end-to-end off-screen enforcement point: the shared rule
// (`throwIfOffscreenTarget`) with the backend's optional
// `confirmOffscreenTargetVisible` hook as its #1542 rescue read. A backend that
// omits the hook refuses on the bulk tree's verdict unchanged.
//
// Exported (not just for callers here) for ADR 0011 registry honesty:
// interaction-guarantees.ts's `offscreen` cells point their `via` at this
Expand All @@ -203,37 +116,22 @@ export async function throwIfOffscreenInteractionTarget(
options: CommandContext,
node: SnapshotNode,
nodes: SnapshotState['nodes'],
failure: {
message: string;
details: Record<string, unknown>;
hint: (direction: OffscreenScrollDirection | null) => string;
},
failure: Parameters<typeof throwIfOffscreenTarget>[2],
): Promise<SnapshotNode> {
const visibility = createSnapshotVisibility(nodes);
const viewport = node.rect ? visibility.resolveEffectiveViewport(node) : null;
if (!node.rect || !viewport || visibility.isVisibleOnScreen(node)) return node;
const rootViewport = visibility.resolveViewport(node.rect);
const liveRect = await runtime.backend.confirmOffscreenTargetVisible?.(
toBackendContext(runtime, options),
const confirm = runtime.backend.confirmOffscreenTargetVisible;
return await throwIfOffscreenTarget(
node,
rootViewport,
);
if (liveRect) return { ...node, rect: liveRect };
// The direction that scrolls this off-screen target into view. Named in the
// hint (and surfaced as a machine-readable detail) so the recovery is a single
// deterministic move instead of a guess (#1366). Derived from the same
// boundary the rejection above used, so partial clips and off-screen
// containers get a direction too, not just fully-scrolled-out items.
const scrollDirection = classifyOffscreenScrollDirection(node, visibility);
throw discloseDispatch(
new AppError('COMMAND_FAILED', failure.message, {
...failure.details,
rect: node.rect,
viewport,
...(scrollDirection ? { scrollDirection } : {}),
hint: failure.hint(scrollDirection),
}),
'no',
nodes,
failure,
confirm
? async (target, rootViewport) =>
await confirm.call(
runtime.backend,
toBackendContext(runtime, options),
target,
rootViewport,
)
: undefined,
);
}

Expand Down
4 changes: 4 additions & 0 deletions packages/command-surface/src/interaction/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -345,6 +345,10 @@ export const INTERACTION_COMMAND_OUTPUT_SCHEMAS = {
x: numberSchema('Resolved x coordinate for mutating find actions.'),
y: numberSchema('Resolved y coordinate for mutating find actions.'),
message: stringSchema('Diagnostic message for mutating find actions.'),
targetHittable: booleanSchema(
'false when the element find focus or find type focused reports hittable: false.',
),
hint: stringSchema('Recovery hint that accompanies targetHittable: false.'),
cost: responseCostSchema,
},
[],
Expand Down
Loading
Loading