From a0f99a69b6398b727f5295b246df80659d1a9577 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Tue, 17 Nov 2026 16:40:04 +0100 Subject: [PATCH 1/2] test(android): retain snapshot helper session diagnostics --- .../snapshot-helper-session-lifecycle.test.ts | 35 ++++++++++ .../__tests__/snapshot-helper-session.test.ts | 10 ++- .../src/snapshot-helper-session-lifecycle.ts | 29 +++++++- .../live-automation-scenario.test.ts | 43 ++++++++++++ .../live-automation-scenario.ts | 17 +++-- .../android-emulator-e2e/live-harness.ts | 19 +++++ .../android-emulator-e2e/scenario-start.ts | 7 +- .../android-emulator-e2e/snapshot-evidence.ts | 69 +++++++++++++++++++ 8 files changed, 217 insertions(+), 12 deletions(-) create mode 100644 test/integration/android-emulator-e2e/snapshot-evidence.ts diff --git a/packages/platform-android/src/__tests__/snapshot-helper-session-lifecycle.test.ts b/packages/platform-android/src/__tests__/snapshot-helper-session-lifecycle.test.ts index a082f7a19a..666b38eb2b 100644 --- a/packages/platform-android/src/__tests__/snapshot-helper-session-lifecycle.test.ts +++ b/packages/platform-android/src/__tests__/snapshot-helper-session-lifecycle.test.ts @@ -594,3 +594,38 @@ function countForceStops(options: SessionProviderOptions): number { function isHelperRuntimeForceStop(args: readonly string[]): boolean { return args.join(' ') === 'shell am force-stop com.callstack.agentdevice.snapshothelper'; } + +test('records a declined acquisition in its request diagnostics', async () => { + const dir = await mkdtempForTest('android-session-decline-'); + const logPath = path.join(dir, 'request.ndjson'); + try { + process.env.AGENT_DEVICE_ANDROID_SNAPSHOT_HELPER_SESSION = '0'; + const provider = createSessionProvider({ calls: [] }); + await withDiagnosticsScope( + { debug: true, logPath, session: 'fixture', requestId: 'req-decline', command: 'snapshot' }, + async () => { + assert.equal( + await captureAndroidSnapshotWithHelperSession({ + adb: provider.exec, + adbProvider: provider, + }), + undefined, + ); + }, + ); + const events = (await fs.readFile(logPath, 'utf8')) + .trim() + .split('\n') + .map((line) => JSON.parse(line)); + assert.ok( + events.some( + (event) => + event.phase === 'android_snapshot_helper_session_acquisition' && + event.data.outcome === 'disabled' && + event.requestId === 'req-decline', + ), + ); + } finally { + await fs.rm(dir, { recursive: true, force: true }); + } +}); diff --git a/packages/platform-android/src/__tests__/snapshot-helper-session.test.ts b/packages/platform-android/src/__tests__/snapshot-helper-session.test.ts index e835a98e9b..7db1f32cf3 100644 --- a/packages/platform-android/src/__tests__/snapshot-helper-session.test.ts +++ b/packages/platform-android/src/__tests__/snapshot-helper-session.test.ts @@ -11,7 +11,10 @@ import { import path from 'node:path'; import { promises as fs } from 'node:fs'; import { mkdtempForTest } from './test-utils/tmp-dir.ts'; -import { withDiagnosticsScope } from '@agent-device/host-kit/diagnostics'; +import { + withDiagnosticsScope, + registerDiagnosticSensitiveValue, +} from '@agent-device/host-kit/diagnostics'; beforeEach(async () => { await resetAndroidSnapshotHelperSessions(); @@ -285,16 +288,19 @@ test('the session fallback diagnostic reads the typed timeout reason', async () await withDiagnosticsScope( { debug: true, logPath, session: 'android-test', requestId: 'req-1', command: 'snapshot' }, async () => { + registerDiagnosticSensitiveValue('sensitive-device-value'); const output = await captureAndroidSnapshotWithHelperSession({ adb: provider.exec, adbProvider: provider, - deviceKey: 'android:emulator-5554', + deviceKey: 'android:sensitive-device-value', }); assert.equal(output, undefined); }, ); const log = await fs.readFile(logPath, 'utf8'); + assert.ok(!log.includes('sensitive-device-value')); + assert.ok(log.includes('req-1')); const fallback = log .split('\n') .filter(Boolean) diff --git a/packages/platform-android/src/snapshot-helper-session-lifecycle.ts b/packages/platform-android/src/snapshot-helper-session-lifecycle.ts index 712a66854c..4d0bf46c00 100644 --- a/packages/platform-android/src/snapshot-helper-session-lifecycle.ts +++ b/packages/platform-android/src/snapshot-helper-session-lifecycle.ts @@ -134,7 +134,18 @@ export async function acquireAndroidSnapshotHelperSession( adb: options.adb, signal: options.signal, }); - if (!isAndroidSnapshotHelperSessionEnabled() || !options.adbProvider?.spawn) { + if (!isAndroidSnapshotHelperSessionEnabled()) { + emitDiagnostic({ + phase: 'android_snapshot_helper_session_acquisition', + data: { outcome: 'disabled' }, + }); + return undefined; + } + if (!options.adbProvider?.spawn) { + emitDiagnostic({ + phase: 'android_snapshot_helper_session_acquisition', + data: { outcome: 'spawn-unavailable' }, + }); return undefined; } const callerResolved = resolveAndroidSnapshotHelperCaptureOptions(options); @@ -172,9 +183,21 @@ async function resolveAndroidSnapshotHelperSession(params: { resolved: AndroidSnapshotHelperResolvedCaptureOptions; startBudgetMs: number; }): Promise { - if (isAndroidSnapshotHelperSessionBackedOff(params.identity)) return undefined; + if (isAndroidSnapshotHelperSessionBackedOff(params.identity)) { + emitDiagnostic({ + phase: 'android_snapshot_helper_session_acquisition', + data: { outcome: 'backed-off' }, + }); + return undefined; + } await retireUnusableAndroidSnapshotHelperSession(params.deviceKey, params.identity); - return sessions().get(params.deviceKey) ?? (await tryStartAndroidSnapshotHelperSession(params)); + const existing = sessions().get(params.deviceKey); + const session = existing ?? (await tryStartAndroidSnapshotHelperSession(params)); + emitDiagnostic({ + phase: 'android_snapshot_helper_session_acquisition', + data: { outcome: existing ? 'reused' : session ? 'created' : 'start-failed' }, + }); + return session; } /** Drops a cached session this command cannot write to, so only its forward is left behind. */ diff --git a/test/integration/android-emulator-e2e/live-automation-scenario.test.ts b/test/integration/android-emulator-e2e/live-automation-scenario.test.ts index cc0ed9dc89..03445c601b 100644 --- a/test/integration/android-emulator-e2e/live-automation-scenario.test.ts +++ b/test/integration/android-emulator-e2e/live-automation-scenario.test.ts @@ -1,4 +1,8 @@ import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { writeSnapshotEvidence } from './snapshot-evidence.ts'; import { test, vi } from 'vitest'; import { parseWaitPositionals } from '@agent-device/command-registry/wait-positionals'; import { createCommandSurfaceAgentDevice } from '@agent-device/command-runtime'; @@ -10,6 +14,7 @@ import type { LiveContext } from './live-harness.ts'; const harness = vi.hoisted(() => ({ runStep: vi.fn(), + captureDiagnosticSnapshot: vi.fn(), verifyCommand: vi.fn(), verifyBehavior: vi.fn(), })); @@ -94,5 +99,43 @@ test('catalog readiness waits until the automation screen has changed', async () if (args[0] === 'click' && args[1]?.startsWith('@')) throw done; return result({}); }); + harness.captureDiagnosticSnapshot.mockImplementation((context, _name, step) => + harness.runStep(context, step, ['snapshot', '-i', '--debug']), + ); await assert.rejects(assertAutomationSystem({} as LiveContext), (error) => error === done); }); + +test('retains bounded original helper evidence with request correlation', () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'android-snapshot-evidence-')); + try { + writeSnapshotEvidence(dir, 'catalog', 'fixture', 100, 200, { + status: 0, + stdout: '', + stderr: '', + json: { + data: { + androidSnapshot: { + backend: 'android-helper', + helperTransport: 'instrumentation', + nodeCount: 268, + foregroundWindowPackage: 'token=secret', + unexpected: 'private', + }, + warnings: Array(100).fill('token=secret ' + 'x'.repeat(1000)), + nodes: [{ label: 'private tree' }], + }, + }, + }); + const content = fs.readFileSync(path.join(dir, 'snapshot-catalog.json'), 'utf8'); + const evidence = JSON.parse(content); + assert.equal(evidence.session, 'fixture'); + assert.deepEqual([evidence.startedAtMs, evidence.finishedAtMs], [100, 200]); + assert.equal(evidence.androidSnapshot.helperTransport, 'instrumentation'); + assert.equal(evidence.androidSnapshot.nodeCount, 268); + assert.equal(evidence.warnings.length, 20); + assert.ok(!content.includes('secret') && !content.includes('private')); + assert.ok(content.length < 10_000); + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } +}); diff --git a/test/integration/android-emulator-e2e/live-automation-scenario.ts b/test/integration/android-emulator-e2e/live-automation-scenario.ts index c9992c8e76..9d972093c2 100644 --- a/test/integration/android-emulator-e2e/live-automation-scenario.ts +++ b/test/integration/android-emulator-e2e/live-automation-scenario.ts @@ -14,7 +14,13 @@ import { requireAndroidResourceId, scrollToVisibleSelector, } from './live-assertions.ts'; -import { type LiveContext, runStep, verifyBehavior, verifyCommand } from './live-harness.ts'; +import { + type LiveContext, + runStep, + verifyBehavior, + verifyCommand, + captureDiagnosticSnapshot, +} from './live-harness.ts'; import { assertHumanSnapshotCompaction } from './live-snapshot-compaction.ts'; const C = PUBLIC_COMMANDS; @@ -38,10 +44,11 @@ export async function assertAutomationSystem(context: LiveContext): Promise = {}; + if (source && typeof source === 'object') { + for (const field of fields) { + const value: unknown = source[field]; + if (typeof value === 'string' || typeof value === 'number' || typeof value === 'boolean') + androidSnapshot[field] = value; + } + } + if (Array.isArray(source?.missingRootWindowTypes)) { + const types = source.missingRootWindowTypes + .filter((value: unknown): value is number => typeof value === 'number') + .slice(0, 20); + Object.assign(androidSnapshot, { missingRootWindowTypes: types }); + } + const warnings = Array.isArray(data?.warnings) + ? data.warnings + .filter((value: unknown): value is string => typeof value === 'string') + .slice(0, 20) + : []; + const evidence = redactDiagnosticData({ + session, + startedAtMs, + finishedAtMs, + status: result.status, + androidSnapshot, + warnings, + }); + fs.writeFileSync(path.join(dir, `snapshot-${name}.json`), JSON.stringify(evidence, null, 2)); +} From b517ddc2d4a96a58d0e24508a93be92d5e04a255 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Tue, 17 Nov 2026 16:52:39 +0100 Subject: [PATCH 2/2] test(android): preserve verdict when recording snapshot evidence --- .../android-emulator-e2e/live-harness.ts | 21 ++-- .../live-device-e2e-runtime.test.ts | 108 ++++++++++++++++++ test/integration/live-device-e2e/runtime.ts | 5 + 3 files changed, 124 insertions(+), 10 deletions(-) diff --git a/test/integration/android-emulator-e2e/live-harness.ts b/test/integration/android-emulator-e2e/live-harness.ts index d24cc4be35..1a2eb7088e 100644 --- a/test/integration/android-emulator-e2e/live-harness.ts +++ b/test/integration/android-emulator-e2e/live-harness.ts @@ -113,14 +113,15 @@ export async function captureDiagnosticSnapshot( step: string, ) { const startedAtMs = Date.now(); - const result = await runStep(context, step, ['snapshot', '-i', '--debug']); - writeSnapshotEvidence( - context.artifactDir, - name, - context.session, - startedAtMs, - Date.now(), - result, - ); - return result; + return await runStep(context, step, ['snapshot', '-i', '--debug'], { + observeResult: (result) => + writeSnapshotEvidence( + context.artifactDir, + name, + context.session, + startedAtMs, + Date.now(), + result, + ), + }); } diff --git a/test/integration/live-device-e2e-runtime.test.ts b/test/integration/live-device-e2e-runtime.test.ts index adcff789c4..00281597b0 100644 --- a/test/integration/live-device-e2e-runtime.test.ts +++ b/test/integration/live-device-e2e-runtime.test.ts @@ -5,6 +5,7 @@ import test from 'node:test'; import { mkdtempForTestSync } from '@agent-device/test-kit'; import type { CliJsonResult } from './cli-json.ts'; +import { writeSnapshotEvidence } from './android-emulator-e2e/snapshot-evidence.ts'; import { createLiveDeviceContext, createLiveDeviceHarness } from './live-device-e2e/runtime.ts'; function fixture(options: { captureThrows?: boolean; deviceEvidence?: string } = {}) { @@ -318,3 +319,110 @@ test('unwritable artifact output cannot replace the scenario failure', async () ); assert.equal(reports(), 1); }); + +test('snapshot evidence write failure preserves the successful issued result', async () => { + const context = createLiveDeviceContext({ + artifactRoot: mkdtempForTestSync('snapshot-observer-success-'), + session: 'fixture-session', + }); + const result: CliJsonResult = { + status: 0, + stdout: '', + stderr: '', + json: { + data: { + androidSnapshot: { backend: 'android-helper', helperTransport: 'persistent-session' }, + }, + }, + }; + let observations = 0; + let issues = 0; + const harness = createLiveDeviceHarness({ + behaviorsForScenario: () => [], + commandsForScenario: () => [], + commonFlags: (_context, args) => [...args], + runCli: async () => { + issues++; + return result; + }, + writeCoverageReport: () => {}, + }); + const returned = await harness.runStep(context, 'original', ['snapshot', '-i', '--debug'], { + observeResult: (observed: CliJsonResult) => { + observations++; + writeSnapshotEvidence( + path.join(context.artifactDir, 'missing'), + 'catalog', + context.session, + 10, + 20, + observed, + ); + }, + }); + assert.equal(returned, result); + assert.equal(returned.status, 0); + assert.equal(observations, 1); + assert.equal(issues, 1); + assert.equal(context.stepHistory[0]?.accepted, true); +}); + +test('failed original snapshot evidence is written before failure collection and assertion', async () => { + const context = createLiveDeviceContext({ + artifactRoot: mkdtempForTestSync('snapshot-observer-failure-'), + session: 'fixture-session', + }); + const original: CliJsonResult = { + status: 17, + stdout: '', + stderr: '', + json: { + data: { + androidSnapshot: { + backend: 'android-helper', + helperTransport: 'instrumentation', + nodeCount: 268, + }, + warnings: ['original warning'], + }, + }, + }; + const followup: CliJsonResult = { + status: 0, + stdout: '', + stderr: '', + json: { data: { androidSnapshot: { nodeCount: 1 } } }, + }; + const calls: string[] = []; + const evidencePath = path.join(context.artifactDir, 'snapshot-catalog.json'); + let presentBeforeCollection = false; + const harness = createLiveDeviceHarness({ + behaviorsForScenario: () => [], + commandsForScenario: () => [], + commonFlags: (_context, args) => [...args], + runCli: async (args) => { + calls.push(args[0]!); + if (calls.length === 1) return original; + presentBeforeCollection = fs.existsSync(evidencePath); + return followup; + }, + writeCoverageReport: () => {}, + }); + await assert.rejects( + harness.runStep(context, 'original', ['snapshot', '-i', '--debug'], { + observeResult: (result: CliJsonResult) => + writeSnapshotEvidence(context.artifactDir, 'catalog', context.session, 10, 20, result), + }), + (error: unknown) => error instanceof assert.AssertionError, + ); + assert.deepEqual(calls, ['snapshot', 'screenshot', 'snapshot']); + assert.equal(presentBeforeCollection, true); + const evidence = JSON.parse(fs.readFileSync(evidencePath, 'utf8')); + assert.equal(evidence.status, 17); + assert.equal(evidence.session, 'fixture-session'); + assert.deepEqual([evidence.startedAtMs, evidence.finishedAtMs], [10, 20]); + assert.equal(evidence.androidSnapshot.nodeCount, 268); + assert.deepEqual(evidence.warnings, ['original warning']); + assert.equal(context.stepHistory.length, 1); + assert.equal(context.stepHistory[0]?.accepted, false); +}); diff --git a/test/integration/live-device-e2e/runtime.ts b/test/integration/live-device-e2e/runtime.ts index c1c387befe..98ddbac8db 100644 --- a/test/integration/live-device-e2e/runtime.ts +++ b/test/integration/live-device-e2e/runtime.ts @@ -62,6 +62,8 @@ type HarnessOptions = { }; type RunStepOptions = { + /** Observes each issued result without changing its verdict; observer failures are ignored. */ + observeResult?: (result: CliJsonResult) => void; allowFailure?: boolean; commonFlags?: boolean; expectFailure?: boolean; @@ -251,6 +253,9 @@ export function createLiveDeviceHarness< step, }); updateSessionState(context, args[0], result.status); + try { + stepOptions.observeResult?.(result); + } catch {} return { result, fullArgs }; }