feat(snapshot): publish the viewport a snapshot's rects are measured in

android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/DisplayExtent.java

@@ -0,0 +1,87 @@
+package com.callstack.agentdevice.snapshothelper;
+
+import android.content.Context;
+import android.graphics.Rect;
+import android.os.Build;
+import android.util.DisplayMetrics;
+import android.view.Display;
+import android.view.WindowManager;
+
+/**
+ * The physical display the captured bounds were measured on, in physical pixels, in the current
+ * rotation (#3182).
+ *
+ * Node bounds reach the host through {@code AccessibilityNodeInfo.getBoundsInScreen()}, which is
+ * absolute screen space across every interactive window, SystemUI's navigation bar included. The
+ * extent therefore has to be the real display extent rather than {@code Resources.getSystem()}'s
+ * app display size: the latter excludes a persistent navigation bar, so a nav-bar node's rect would
+ * land outside the published box. On API 30+ that extent is
+ * {@code WindowManager.getMaximumWindowMetrics().getBounds()}; below it, {@code
+ * Display.getRealMetrics}, which likewise carries the full panel and not the app-usable inset.
+ *
+ * The caller reads this twice around the tree dump and publishes only what {@link #whenBothAgree}
+ * returns. A rotation landing inside the dump changes the extent between the reads, and the capture
+ * then says nothing about its box rather than pairing one rotation's bounds with the other's
+ * dimensions. A display this process cannot address answers with null. Either way the host publishes
+ * the absence it reads as unknown, and the host's own guard is what refuses a zero extent.
+ */
+final class DisplayExtent {
+  private DisplayExtent() {}
+
+  /** The extent one read of the display answered with, in the current rotation. */
+  static final class Extent {
+    final int width;
+    final int height;
+
+    Extent(int width, int height) {
+      this.width = width;
+      this.height = height;
+    }
+  }
+
+  /**
+   * The extent of the display {@code context} is attached to, or null when this process could not
+   * address it. A zero extent is passed through as read: refusing an unusable box is the host's job,
+   * and {@code snapshotViewportSizeFrom} already refuses one, so a second refusal here would only
+   * be a second place that decides what a usable display is.
+   */
+  static Extent read(Context context) {
+    try {
+      WindowManager windowManager =
+          (WindowManager) context.getSystemService(Context.WINDOW_SERVICE);
+      if (windowManager == null) {
+        return null;
+      }
+      if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.R) {
+        Rect bounds = windowManager.getMaximumWindowMetrics().getBounds();
+        return new Extent(bounds.width(), bounds.height());
+      }
+      return legacyExtent(windowManager);
+    } catch (Throwable error) {
+      return null;
+    }
+  }
+
+  @SuppressWarnings("deprecation")
+  private static Extent legacyExtent(WindowManager windowManager) {
+    Display display = windowManager.getDefaultDisplay();
+    DisplayMetrics metrics = new DisplayMetrics();
+    display.getRealMetrics(metrics);
+    return new Extent(metrics.widthPixels, metrics.heightPixels);
+  }
+
+  /**
+   * The extent both reads of one capture agreed on, or null when either read failed or the display
+   * changed underneath the tree dump. Omitting the viewport on disagreement is what keeps the
+   * published box a fact about the capture rather than about two different rotations.
+   */
+  static Extent whenBothAgree(Extent before, Extent after) {
+    if (before == null || after == null) {
+      return null;
+    }
+    if (before.width != after.width || before.height != after.height) {
+      return null;
+    }
+    return before;
+  }
+}

android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/SnapshotInstrumentation.java

@@ -5,6 +5,7 @@
 import android.content.res.Resources;
 import android.os.Bundle;
 import android.util.Base64;
+import android.util.DisplayMetrics;
 import java.io.BufferedReader;
 import java.io.File;
 import java.io.FileOutputStream;
@@ -77,14 +78,17 @@ public void onStart() {
         return;
       }
       long startedAtMs = System.currentTimeMillis();
+      DisplayExtent.Extent beforeDump = DisplayExtent.read(getContext());
       AccessibilityTreeCapture.Result capture =
           captureXml(waitForIdleQuietMs, waitForIdleTimeoutMs, timeoutMs, maxDepth, maxNodes);
+      DisplayExtent.Extent display =
+          DisplayExtent.whenBothAgree(beforeDump, DisplayExtent.read(getContext()));
       writeOutputFile(outputPath, capture.xml);
       if (emitChunks) {
         emitChunks(capture.xml);
       }
       result.putString("ok", "true");
-      putCaptureMetadata(result, capture, System.currentTimeMillis() - startedAtMs);
+      putCaptureMetadata(result, capture, display, System.currentTimeMillis() - startedAtMs);
       finishSafely(0, result);
     } catch (Throwable error) {
       result.putString("ok", "false");
@@ -114,7 +118,10 @@ private static void putBaseMetadata(
   }
 
   private static void putCaptureMetadata(
-      Bundle result, AccessibilityTreeCapture.Result capture, long elapsedMs) {
+      Bundle result,
+      AccessibilityTreeCapture.Result capture,
+      DisplayExtent.Extent display,
+      long elapsedMs) {
     result.putString("rootPresent", Boolean.toString(capture.rootPresent));
     result.putString("captureMode", capture.captureMode);
     result.putString("windowCount", Integer.toString(capture.windowCount));
@@ -124,8 +131,17 @@ private static void putCaptureMetadata(
     // Physical pixels per dp of the display the bounds above were measured on, from the same
     // configuration the framework lays out with (a `wm density` override included), so a host that
     // works in dp has the factor beside the pixels instead of a second adb round trip.
-    result.putString(
-        "pixelDensity", Float.toString(Resources.getSystem().getDisplayMetrics().density));
+    DisplayMetrics metrics = Resources.getSystem().getDisplayMetrics();
+    result.putString("pixelDensity", Float.toString(metrics.density));
+    // The real extent of the display those bounds are measured in, read around the tree dump and
+    // published only when both reads agreed, so the host can publish the box the bounds live in
+    // without reading the tree for it. Published by absence: a display this helper could not
+    // address, or one that rotated mid-dump, adds no keys, which the host reads as unknown rather
+    // than as a screen of no size or of another rotation's size (#3182).
+    if (display != null) {
+      result.putString("displayWidth", Integer.toString(display.width));
+      result.putString("displayHeight", Integer.toString(display.height));
+    }
   }
 
   private void runOneShotViewport(Bundle result) {
@@ -232,10 +248,13 @@ private void writeSessionSnapshot(
     result.putString("requestId", requestId);
     try {
       long startedAtMs = System.currentTimeMillis();
+      DisplayExtent.Extent beforeDump = DisplayExtent.read(getContext());
       AccessibilityTreeCapture.Result capture =
           captureXml(waitForIdleQuietMs, waitForIdleTimeoutMs, timeoutMs, maxDepth, maxNodes);
+      DisplayExtent.Extent display =
+          DisplayExtent.whenBothAgree(beforeDump, DisplayExtent.read(getContext()));
       result.putString("ok", "true");
-      putCaptureMetadata(result, capture, System.currentTimeMillis() - startedAtMs);
+      putCaptureMetadata(result, capture, display, System.currentTimeMillis() - startedAtMs);
       result.putString("byteLength", Integer.toString(capture.xml.getBytes(StandardCharsets.UTF_8).length));
       SessionResponseWriter.writeSessionResponse(output, result, capture.xml);
     } catch (Throwable error) {

android/snapshot-helper/src/test/java/com/callstack/agentdevice/snapshothelper/DisplayExtentTest.java

@@ -0,0 +1,66 @@
+package com.callstack.agentdevice.snapshothelper;
+
+public final class DisplayExtentTest {
+  private DisplayExtentTest() {}
+
+  private static final DisplayExtent.Extent PORTRAIT = new DisplayExtent.Extent(1080, 2400);
+  private static final DisplayExtent.Extent LANDSCAPE = new DisplayExtent.Extent(2400, 1080);
+
+  static void run() {
+    assertTwoReadsOfTheSameDisplayPublishItsExtent();
+    assertADisplayThatMovedUnderTheDumpPublishesNothing();
+    assertAReadThatFailedPublishesNothing();
+  }
+
+  private static void assertTwoReadsOfTheSameDisplayPublishItsExtent() {
+    // The rotation matters: a landscape capture publishes the box its bounds are actually measured
+    // in, which is why the extent is read per capture rather than kept across a helper session.
+    assertExtent(
+        DisplayExtent.whenBothAgree(PORTRAIT, new DisplayExtent.Extent(1080, 2400)),
+        1080,
+        2400,
+        "both reads of one portrait display");
+    assertExtent(
+        DisplayExtent.whenBothAgree(LANDSCAPE, LANDSCAPE),
+        2400,
+        1080,
+        "both reads of one landscape display");
+  }
+
+  private static void assertADisplayThatMovedUnderTheDumpPublishesNothing() {
+    // A rotation landing inside the tree dump would pair that rotation's node bounds with the other
+    // rotation's dimensions. The capture then names no box at all rather than a wrong one (#3182).
+    assertNull(
+        DisplayExtent.whenBothAgree(PORTRAIT, LANDSCAPE), "display rotated mid-dump");
+    assertNull(
+        DisplayExtent.whenBothAgree(
+            PORTRAIT, new DisplayExtent.Extent(1076, 2400)),
+        "display resized mid-dump");
+  }
+
+  private static void assertAReadThatFailedPublishesNothing() {
+    // A read that failed on either side of the dump leaves the box unknown: the surviving read says
+    // nothing about what the tree between the two was measured against.
+    assertNull(DisplayExtent.whenBothAgree(null, PORTRAIT), "read before the dump failed");
+    assertNull(DisplayExtent.whenBothAgree(PORTRAIT, null), "read after the dump failed");
+    assertNull(DisplayExtent.whenBothAgree(null, null), "both reads failed");
+  }
+
+  private static void assertExtent(
+      DisplayExtent.Extent extent, int width, int height, String label) {
+    if (extent == null) {
+      throw new AssertionError("expected an extent for " + label);
+    }
+    if (extent.width != width || extent.height != height) {
+      throw new AssertionError(
+          "wrong extent for " + label + ": got " + extent.width + "x" + extent.height);
+    }
+  }
+
+  private static void assertNull(DisplayExtent.Extent extent, String label) {
+    if (extent != null) {
+      throw new AssertionError(
+          "expected no extent for " + label + " but got " + extent.width + "x" + extent.height);
+    }
+  }
+}

android/snapshot-helper/src/test/java/com/callstack/agentdevice/snapshothelper/SnapshotHelperTestSuite.java

@@ -8,5 +8,6 @@ public static void main(String[] args) throws Exception {
     AccessibilityCaptureStabilizerTest.run();
     BoundedUiAutomationConnectionTest.run();
     GestureViewportReaderTest.run();
+    DisplayExtentTest.run();
   }
 }

packages/capture-kit/src/ios-snapshot-engine/engine.ts

@@ -46,6 +46,9 @@ export function publishIosSnapshot(
     comparisonIdentity: buildIosSnapshotComparisonIdentity(input, request),
     residue:
       input.stage === 'acquired' ? [...input.acquisition.residue] : [...input.validation.residue],
+    ...(presentation.validatedViewport
+      ? { validatedViewport: presentation.validatedViewport }
+      : {}),
   };
 }
 
@@ -143,6 +146,7 @@ function presentAcquiredSnapshot(
         }).nodes;
   return {
     nodes: compacted.nodes,
+    validatedViewport: viewport,
     ...(qualityNodes ? { qualityNodes } : {}),
     presentedIndexesBySourceIndex: remapPresentedIndexes(
       acquisition.nodes,

packages/capture-kit/src/ios-snapshot-engine/runner-presentation.ts

@@ -34,6 +34,7 @@ export function presentIosRunnerSnapshot(
   );
   return {
     nodes: compacted.nodes,
+    ...(viewport ? { validatedViewport: viewport } : {}),
     ...(input.presentation.qualityPayload
       ? { qualityNodes: [...input.presentation.qualityPayload.nodes] }
       : {}),

packages/capture-kit/src/ios-snapshot-engine/types.ts

@@ -26,6 +26,13 @@ export type IosSnapshotPresentationStats = Readonly<{
 
 export type IosSnapshotEnginePresentation = Readonly<{
   nodes: RawSnapshotNode[];
+  /**
+   * The box the regular projection validated this tree against (#3182). The fold already resolves it
+   * (`resolveIosViewport` / `resolveViewportEvidence`), so the engine hands it over instead of making
+   * every caller re-derive "regular validates a box, raw does not". A raw projection validates no box
+   * and carries none.
+   */
+  validatedViewport?: Rect;
   qualityNodes?: RawSnapshotNode[];
   presentedIndexesBySourceIndex: ReadonlyMap<number, readonly number[]>;
   stats: IosSnapshotPresentationStats;

packages/capture-kit/src/snapshot-state.ts

@@ -15,6 +15,7 @@ import {
   type SnapshotKeyboardBandFact,
   snapshotStateProvenance,
   type SnapshotState,
+  type SnapshotViewportSize,
 } from '@agent-device/kernel/snapshot';
 import {
   annotateCoveredSnapshotNodes,
@@ -51,6 +52,8 @@ export function buildSnapshotState(
     targetActivation?: IosTargetActivation;
     /** The keyboard band the producer measured, carried to the state the tap guards read (#2660). */
     keyboard?: SnapshotKeyboardBandFact;
+    /** The box the producer measured the rects in (#3182), carried to the response that reads this state. */
+    viewport?: SnapshotViewportSize;
   } & SnapshotCaptureProvenance,
   flags:
     | (Pick<CommandFlags, 'snapshotDepth' | 'snapshotInteractiveOnly' | 'snapshotRaw'> &
@@ -89,6 +92,7 @@ export function buildSnapshotState(
     ...(data.systemSurface ? { iosSystemSurfaceBundleId: data.systemSurface.bundleId } : {}),
     ...(data.targetActivation ? { targetActivation: data.targetActivation } : {}),
     ...(data.keyboard ? { keyboard: data.keyboard } : {}),
+    ...(data.viewport ? { viewport: data.viewport } : {}),
     presentationKey: buildSnapshotPresentationKey(snapshotPresentationOptionsFromFlags(flags)),
     // Only broad Android snapshots become freshness baselines. If the user asked for a scoped
     // or filtered view, preserve that output contract but avoid pretending it is safe for

packages/capture-kit/src/snapshot/__tests__/ios-snapshot-runtime-viewport.test.ts

@@ -0,0 +1,84 @@
+import assert from 'node:assert/strict';
+import { test } from 'vitest';
+import { createIosSnapshotAcquisition } from '@agent-device/capture-kit/ios-snapshot-acquisition';
+import { presentIosSnapshotAcquisition } from '../ios-snapshot-runtime.ts';
+import type { IosViewportEvidence } from '@agent-device/contracts/ios-snapshot';
+import { AppError } from '@agent-device/kernel/errors';
+import type { RawSnapshotNode } from '@agent-device/kernel/snapshot';
+
+// #3182: the provider acquisitions — the simulator AX bridge, an Appium page source, a Limrun tree —
+// already resolve the box their nodes are measured in, and this presenter is the one seam that turns
+// an acquisition into a snapshot result. Publishing nothing here left the default local-iOS producer
+// without a viewport while it held the evidence, which is the issue's first `Done when` line.
+
+const SCREEN = { x: 0, y: 0, width: 390, height: 844 };
+
+const NODES: readonly RawSnapshotNode[] = [
+  { index: 0, type: 'Application', rect: SCREEN },
+  { index: 1, parentIndex: 0, type: 'Button', label: 'Open', rect: { ...SCREEN, width: 80 } },
+];
+
+function acquire(viewport: IosViewportEvidence) {
+  return createIosSnapshotAcquisition({
+    producer: 'appium-source',
+    nodes: NODES,
+    viewport,
+    lineage: { targetId: 'sim-1:com.example.app' },
+  });
+}
+
+test('an acquisition publishes the viewport box the regular fold measured against (#3182)', () => {
+  const presented = presentIosSnapshotAcquisition(acquire({ kind: 'reported', rect: SCREEN }));
+
+  assert.deepEqual(presented.viewport, { width: 390, height: 844 });
+});
+
+// The regular fold refuses a capture with no viewport, so absence is only reachable on a raw
+// projection — which is also the projection that validates no box at all.
+test('a raw acquisition publishes no viewport, since nothing checked it (#3182)', () => {
+  const presented = presentIosSnapshotAcquisition(acquire({ kind: 'reported', rect: SCREEN }), {
+    raw: true,
+  });
+
+  assert.equal('viewport' in presented, false);
+});
+
+test('a raw acquisition carrying the failed-read box publishes no viewport (#3182)', () => {
+  const presented = presentIosSnapshotAcquisition(
+    acquire({
+      kind: 'reported',
+      rect: {
+        x: -Number.MAX_VALUE / 2,
+        y: -Number.MAX_VALUE / 2,
+        width: Number.MAX_VALUE,
+        height: Number.MAX_VALUE,
+      },
+    }),
+    { raw: true },
+  );
+
+  assert.equal('viewport' in presented, false);
+});
+
+test('a regular acquisition with no viewport evidence still refuses rather than minting one', () => {
+  assert.throws(
+    () => presentIosSnapshotAcquisition(acquire({ kind: 'missing', reason: 'not-provided' })),
+    (error: unknown) => error instanceof AppError && error.code === 'COMMAND_FAILED',
+  );
+});
+
+// The issue's second Done-when line: an empty screen still reports the box. The box is a property of
+// the surface the producer read, so a tree that names nothing on it still answers the question.
+test('an empty iOS acquisition still publishes the viewport it read (#3182)', () => {
+  const empty = createIosSnapshotAcquisition({
+    producer: 'appium-source',
+    nodes: [],
+    viewport: { kind: 'reported', rect: SCREEN },
+    lineage: { targetId: 'sim-1:com.example.app' },
+  });
+
+  const presented = presentIosSnapshotAcquisition(empty);
+
+  assert.deepEqual(presented.nodes, []);
+  assert.deepEqual(presented.viewport, { width: 390, height: 844 });
+});