Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -221,6 +221,35 @@ - (void)viewDidLoad {
}
@end


// `--agent-device-custom-drawn-canvas` adds a plain drawn view the accessibility tree never
// exposes: a tap toggles its fill, and the point over it names no element but the window. The
// screenshot-diff verification needs exactly this shape — custom-drawn content whose only answer
// is pixels, at a point no hittable or semantic node covers.
@interface AgentDeviceCustomDrawnCanvas : UIView
@end

@implementation AgentDeviceCustomDrawnCanvas
- (instancetype)initWithFrame:(CGRect)frame {
self = [super initWithFrame:frame];
if (self) {
self.isAccessibilityElement = NO;
self.backgroundColor = UIColor.systemGray5Color;
UITapGestureRecognizer *tap =
[[UITapGestureRecognizer alloc] initWithTarget:self action:@selector(toggled:)];
[self addGestureRecognizer:tap];
}
return self;
}

- (void)toggled:(UITapGestureRecognizer *)recognizer {
(void)recognizer;
self.backgroundColor = [self.backgroundColor isEqual:UIColor.systemRedColor]
? UIColor.systemGray5Color
: UIColor.systemRedColor;
}
@end

@interface AgentDeviceRunnerFixtureViewController () <UNUserNotificationCenterDelegate, UITextFieldDelegate>
// Held for the app's lifetime: a window nobody retained would be gone before the test queried it.
@property(nonatomic, strong) UIWindow *secondWindow;
Expand Down Expand Up @@ -1466,6 +1495,18 @@ - (void)viewDidLoad {
[self.view addSubview:popupTitle];
}

if ([NSProcessInfo.processInfo.arguments containsObject:@"--agent-device-custom-drawn-canvas"]) {
UIView *canvas = [[AgentDeviceCustomDrawnCanvas alloc] initWithFrame:CGRectMake(120, 300, 160, 160)];
canvas.translatesAutoresizingMaskIntoConstraints = NO;
[self.view addSubview:canvas];
[NSLayoutConstraint activateConstraints:@[
[canvas.centerXAnchor constraintEqualToAnchor:self.view.centerXAnchor],
[canvas.topAnchor constraintEqualToAnchor:label.bottomAnchor constant:120],
[canvas.widthAnchor constraintEqualToConstant:160],
[canvas.heightAnchor constraintEqualToConstant:160],
]];
}

if ([NSProcessInfo.processInfo.arguments containsObject:@"--agent-device-crowded-screen"]) {
for (NSUInteger row = 0; row < 500; row++) {
UILabel *rowLabel = [[UILabel alloc] initWithFrame:CGRectMake(0, 60 + (row % 30) * 24, 400, 16)];
Expand Down
115 changes: 115 additions & 0 deletions docs/evidence/android-verify-screenshot-fallback-2026-11-20.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,115 @@
# Android `--verify` screenshot fallback through the full-screen-container exemption, 2026-11-20

Evidence for #4297 at the gate-exemption head `28c408b94`. Device: Redmi
K60 (`2312DRAABC`, MIUI Android 14, 1080x2400), Edge (`com.microsoft.emmx`)
serving the local rig page over `adb reverse tcp:8642`, backend
`android-helper` (`snapshotQuality.backend: "android-helper"`). The battery
level was forced to 37 with `adb shell dumpsys battery set level 37` about
400 ms into the press and reset right after, so the only candidate change
besides the page itself was the status-bar glyph.

## The tree that used to close the gate, and the exemption that opens it

Interactive snapshot at the rig page (23 nodes; header rows trimmed):

```json
{"ref": "e1", "type": "android.webkit.WebView", "hittable": true,
"rect": {"x": 0, "y": 237, "width": 1080, "height": 1976}},
{"ref": "e3", "type": "android.widget.Button", "hittable": true,
"rect": {"x": 385, "y": 253, "width": 310, "height": 143}, "label": "TAB A"},
{"ref": "e4", "type": "android.widget.TextView", "hittable": null,
"rect": {"x": 0, "y": 237, "width": 1080, "height": 654}, "label": "static body"}
```

No window or application node exists (Android emits none), so the point's
resolved viewport comes from `resolveViewportRect`'s largest-containing-rect
fallback. At the dead point `(540, 1500)` the only covering hittable node
is the WebView container, and its rect IS the resolved viewport — the
option-(a) exemption admits the pair. Under the pre-exemption gate this
same tree and point produced tree evidence with no `screenshotDiff` (the
gate closed on every real point).

## Run 1 — dead tap with a forced status-bar change

`press 540 1500 --verify --json` on the rig's dead area, battery forced
mid-dispatch:

```json
{
"success": true,
"data": {
"x": 540,
"y": 1500,
"message": "Tapped (540, 1500)",
"targetKind": "point",
"evidence": {
"digest": "ax1:f017ea1bdbba74a8",
"nodeCount": 23,
"interactiveNodeCount": 23,
"foregroundApp": "com.microsoft.emmx",
"changedFromBefore": false,
"screenshotDiff": {
"changedFromBefore": false,
"changedPixelRatio": 0
}
}
}
}
```

`changedFromBefore: false`, ratio 0 — the forced change landed between the
frames (the battery read 37 during the window and 100 again after the
reset), the 4% portrait band excluded it, and the outside-the-status-bar
comparison saw an identical frame. A post-run screenshot shows the status
bar back at 100% and the page unchanged.

A second dead run on the MIUI launcher (`com.miui.home`, `press 540 700
--verify --json`, same forced change) read the same shape:
`changedFromBefore: false`, `screenshotDiff {false, 0}`, tree digest
`ax1:36d6f2391b33939c`.

## Run 2 — a tap that redraws custom-drawn content

The rig page's second version carries a 280x180 canvas (a plain drawn
surface the tree reports as a hittable-less `android.widget.Image` at
`154,391 772x495` with no semantics). `press 540 638 --verify --json`
taps it; its click handler repaints the canvas gray→red and rewrites the
state row:

```json
{
"success": true,
"data": {
"x": 540,
"y": 638,
"message": "Tapped (540, 638)",
"targetKind": "point",
"evidence": {
"digest": "ax1:8832c6a9aa185a47",
"nodeCount": 23,
"interactiveNodeCount": 23,
"foregroundApp": "com.microsoft.emmx",
"changedFromBefore": true,
"screenshotDiff": {
"changedFromBefore": true,
"changedPixelRatio": 0.15493023405349796
}
}
}
}
```

Aggregate true with `screenshotDiff {changedFromBefore: true,
changedPixelRatio: 0.155}`. A post-run screenshot shows the canvas red with
`STATE=RED`, matching the pixel answer.

## Notes

- Backend: `android-helper` (snapshot helper 0.21.42-dev, built from this
tree); screenshots go through the helper's capture with
`rotationFacts(device, ...)` declaring the band.
- The tap at `(540, 638)` covers one non-blocking candidate (`e3` shape on
the first rig: the WebView only). On the second rig the canvas node is
not hittable and names no touch role, so the gate stays open there too.
- Battery reset and status bar restored after the runs; the rig server was
an unrelated loopback process, left running for reproduction.
115 changes: 115 additions & 0 deletions docs/evidence/ios-verify-screenshot-fallback-2026-11-20.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,115 @@
# iOS `--verify` screenshot fallback on the custom-drawn canvas fixture, 2026-11-20

Evidence for #4297 at the gate-exemption head `28c408b94` (the only later
commit is this docs change). The runner fixture is the source-built
unit-test runner (`AGENT_DEVICE_XCUITEST_INCLUDE_UNIT_TESTS=1`), installed
on the iPhone 17 Pro Max simulator (`F8658A47-112D-48F1-8D18-226A74F39689`,
iOS 26.5, Xcode 27.2) and launched with `--agent-device-custom-drawn-canvas`
through `open --launch-args`. The fixture adds a 160x160 plain UIView the
accessibility tree never exposes: a tap toggles its fill between systemGray5
and systemRed.

## Pre-action tree: no hittable or semantic node covers the canvas point

Full snapshot at the session app:

```json
[
{"index": 0, "type": "Application", "role": "UIApplication", "label": "AgentDeviceRunner",
"rect": {"x": 0, "y": 0, "width": 440, "height": 956}, "enabled": true, "depth": 0,
"pid": 92141, "hittable": false, "ref": "e1", "kind": "application"},
{"index": 1, "parentIndex": 0, "type": "StaticText", "role": "UILabel",
"label": "Agent Device Runner",
"rect": {"x": 120.0, "y": 465, "width": 200.3, "height": 26.3}, "enabled": true,
"depth": 1, "pid": 92141, "hittable": true, "ref": "e2", "kind": "text"}
]
```

The canvas (roughly 140..300 x 611..771 pt, centered under the label) names
no element: the point `(220, 720)` is covered only by the Application node
itself (the viewport root, not hittable). The StaticText does not cover it.
The tree-coverage gate has no covering touch target to block on, so the
pixel pair is admitted and the screenshot fallback runs.

## Run 1 — status-bar-only change between the frames

`press 220 720 --verify --json` is a canvas tap; to make the only other
candidate change a status-bar glyph, the simulator's battery level was
forced to 37 about 500 ms into the press (`xcrun simctl status_bar
override --batteryLevel 37`) and cleared right after. The band (6% of the
portrait frame, declared by the apple capture owner) excludes that region,
so the forced change must not surface:

```json
{
"success": true,
"data": {
"x": 220,
"y": 720,
"message": "Tapped (220, 720)",
"runnerMainThreadBusy": false,
"targetKind": "point",
"evidence": {
"digest": "ax1:aaf591869124d41c",
"nodeCount": 3,
"interactiveNodeCount": 2,
"changedFromBefore": false,
"screenshotDiff": {
"changedFromBefore": false,
"changedPixelRatio": 0
}
}
}
}
```

Aggregate `changedFromBefore` is false (no comparison verification saw a
change) and the pixel-only answer is `screenshotDiff {changedFromBefore:
false, changedPixelRatio: 0}` — the forced status-bar redraw landed inside
the excluded band and the outside-the-status-bar comparison saw an
identical frame.

## Run 2 — a tap that redraws the canvas

`press 220 720 --verify --json` again, this time with no status-bar
override. The tap lands on the custom-drawn view and toggles its fill:

```json
{
"success": true,
"data": {
"x": 220,
"y": 720,
"message": "Tapped (220, 720)",
"runnerMainThreadBusy": false,
"targetKind": "point",
"evidence": {
"digest": "ax1:aaf591869124d41c",
"nodeCount": 3,
"interactiveNodeCount": 2,
"changedFromBefore": true,
"screenshotDiff": {
"changedFromBefore": true,
"changedPixelRatio": 0.06512286378804732
}
}
}
}
```

Aggregate true with `screenshotDiff {changedFromBefore: true,
changedPixelRatio: 0.0651}` — the canvas (160/440 x 160/956 ≈ 6.1% of the
frame) turned red, and the ratio matches that share.

## Notes

- Backend: XCTest runner (source build with the unit-test fixture define),
session app `com.callstack.agentdevice.runner` (pid 92141 at snapshot).
- The capture binding is the production one: the daemon's
`createVerifyScreenshotCapture` binds the screenshot lane for the session
(`captureScreenshotFromRequest` with the session surface, fullscreen for
a portrait iOS target) — no flag route, no test-injected capture.
- The iOS dead-region run at `(60, 820)` (left margin, no node under it)
with the same forced battery change read `changedFromBefore: false` with
`screenshotDiff {false, 0}` as well.
- Status bar cleared after the runs; the simulator was left booted.
4 changes: 4 additions & 0 deletions packages/capture-kit/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -155,6 +155,10 @@
"types": "./src/png-rgb-difference.ts",
"default": "./src/png-rgb-difference.ts"
},
"./png-changed-pixel-ratio": {
"types": "./src/png-changed-pixel-ratio.ts",
"default": "./src/png-changed-pixel-ratio.ts"
},
"./png-size": {
"types": "./src/png-size.ts",
"default": "./src/png-size.ts"
Expand Down
21 changes: 21 additions & 0 deletions packages/capture-kit/src/png-changed-pixel-ratio.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -45,3 +45,24 @@ describe('computePngChangedPixelRatio', () => {
).toEqual({ status: 'dimension_mismatch' });
});
});

test('computePngChangedPixelRatio ignores the top band share when asked', () => {
const first = { width: 4, height: 4, data: new Uint8Array(4 * 4 * 4).fill(10) };
const second = { width: 4, height: 4, data: new Uint8Array(4 * 4 * 4).fill(10) };
// Second frame: top half (rows 0-1) jumps to 200, bottom half (rows 2-3) to 30.
for (let y = 0; y < 2; y += 1) {
for (let x = 0; x < 4; x += 1) second.data[(y * 4 + x) * 4] = 200;
}
for (let y = 2; y < 4; y += 1) {
for (let x = 0; x < 4; x += 1) second.data[(y * 4 + x) * 4] = 30;
}

const full = computePngChangedPixelRatio(first, second);
expect(full).toEqual({ status: 'compared', changedPixelRatio: 1 });

// With the top half excluded, the ignored rows drop out of the comparison:
// the ratio counts the bottom half's changed pixels over that compared
// region, so every pixel the pair was compared over changed.
const half = computePngChangedPixelRatio(first, second, { ignoreTopHeightShare: 0.5 });
expect(half).toEqual({ status: 'compared', changedPixelRatio: 1 });
});
34 changes: 31 additions & 3 deletions packages/capture-kit/src/png-changed-pixel-ratio.ts
Original file line number Diff line number Diff line change
Expand Up @@ -17,10 +17,18 @@ export type PngChangedPixelRatioResult =
*
* Dimensions must match. Callers comparing frames of different sizes are asking a different
* question, and returning a made-up ratio would hide it.
*
* `ignoreTopHeightShare` cuts the top share of the frame out of the comparison entirely, for
* bands whose pixels move without the dispatch causing it — the device's own status bar ticks
* between the bracket's frames, and the readback must answer for what the dispatch changed, not
* for the wall clock. The ratio runs over the compared region: both the changed count and the
* denominator count the rows below the band, so the share is the share of the region the pair
* was actually compared over.
*/
export function computePngChangedPixelRatio(
first: PngRgbImage,
second: PngRgbImage,
options: Readonly<{ ignoreTopHeightShare?: number }> = {},
): PngChangedPixelRatioResult {
if (first.width !== second.width || first.height !== second.height) {
return { status: 'dimension_mismatch' };
Expand All @@ -32,8 +40,29 @@ export function computePngChangedPixelRatio(
return { status: 'dimension_mismatch' };
}

// Rows at or above the cut are outside the compared region on both sides.
const ignoredRows =
options.ignoreTopHeightShare && options.ignoreTopHeightShare > 0
? Math.min(first.height, Math.floor(first.height * options.ignoreTopHeightShare))
: 0;
const comparedPixels = totalPixels - ignoredRows * first.width;
if (comparedPixels <= 0) return { status: 'compared', changedPixelRatio: 0 };

return {
status: 'compared',
changedPixelRatio: countChangedPixels(first, second, ignoredRows) / comparedPixels,
};
}

/** Counts pixels whose RGB differs outside the ignored top rows. */
function countChangedPixels(first: PngRgbImage, second: PngRgbImage, ignoredRows: number): number {
const declaredBytes = first.width * first.height * 4;
let changedPixels = 0;
for (let offset = 0; offset + 3 < first.data.length; offset += 4) {
for (
let offset = ignoredRows * first.width * 4;
offset + 3 < declaredBytes && offset + 3 < first.data.length;
offset += 4
) {
if (
first.data[offset] !== second.data[offset] ||
first.data[offset + 1] !== second.data[offset + 1] ||
Expand All @@ -42,6 +71,5 @@ export function computePngChangedPixelRatio(
changedPixels += 1;
}
}

return { status: 'compared', changedPixelRatio: changedPixels / totalPixels };
return changedPixels;
}
Original file line number Diff line number Diff line change
Expand Up @@ -409,6 +409,7 @@ export function createInteractionDevice(
overrides: Partial<
Pick<
AgentDeviceBackend,
| 'captureVerifyScreenshot'
| 'captureSnapshot'
| 'resolveGestureViewport'
| 'tap'
Expand All @@ -430,6 +431,7 @@ export function createInteractionDevice(
) {
return createAgentDevice({
backend: {
captureVerifyScreenshot: overrides.captureVerifyScreenshot,
platform: overrides.platform ?? 'ios',
captureSnapshot: async (...args) =>
overrides.captureSnapshot ? await overrides.captureSnapshot(...args) : { snapshot },
Expand Down
Loading
Loading