Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -594,3 +594,38 @@ function countForceStops(options: SessionProviderOptions): number {
function isHelperRuntimeForceStop(args: readonly string[]): boolean {
return args.join(' ') === 'shell am force-stop com.callstack.agentdevice.snapshothelper';
}

test('records a declined acquisition in its request diagnostics', async () => {
const dir = await mkdtempForTest('android-session-decline-');
const logPath = path.join(dir, 'request.ndjson');
try {
process.env.AGENT_DEVICE_ANDROID_SNAPSHOT_HELPER_SESSION = '0';
const provider = createSessionProvider({ calls: [] });
await withDiagnosticsScope(
{ debug: true, logPath, session: 'fixture', requestId: 'req-decline', command: 'snapshot' },
async () => {
assert.equal(
await captureAndroidSnapshotWithHelperSession({
adb: provider.exec,
adbProvider: provider,
}),
undefined,
);
},
);
const events = (await fs.readFile(logPath, 'utf8'))
.trim()
.split('\n')
.map((line) => JSON.parse(line));
assert.ok(
events.some(
(event) =>
event.phase === 'android_snapshot_helper_session_acquisition' &&
event.data.outcome === 'disabled' &&
event.requestId === 'req-decline',
),
);
} finally {
await fs.rm(dir, { recursive: true, force: true });
}
});
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,10 @@ import {
import path from 'node:path';
import { promises as fs } from 'node:fs';
import { mkdtempForTest } from './test-utils/tmp-dir.ts';
import { withDiagnosticsScope } from '@agent-device/host-kit/diagnostics';
import {
withDiagnosticsScope,
registerDiagnosticSensitiveValue,
} from '@agent-device/host-kit/diagnostics';

beforeEach(async () => {
await resetAndroidSnapshotHelperSessions();
Expand Down Expand Up @@ -285,16 +288,19 @@ test('the session fallback diagnostic reads the typed timeout reason', async ()
await withDiagnosticsScope(
{ debug: true, logPath, session: 'android-test', requestId: 'req-1', command: 'snapshot' },
async () => {
registerDiagnosticSensitiveValue('sensitive-device-value');
const output = await captureAndroidSnapshotWithHelperSession({
adb: provider.exec,
adbProvider: provider,
deviceKey: 'android:emulator-5554',
deviceKey: 'android:sensitive-device-value',
});
assert.equal(output, undefined);
},
);

const log = await fs.readFile(logPath, 'utf8');
assert.ok(!log.includes('sensitive-device-value'));
assert.ok(log.includes('req-1'));
const fallback = log
.split('\n')
.filter(Boolean)
Expand Down
29 changes: 26 additions & 3 deletions packages/platform-android/src/snapshot-helper-session-lifecycle.ts
Original file line number Diff line number Diff line change
Expand Up @@ -134,7 +134,18 @@ export async function acquireAndroidSnapshotHelperSession(
adb: options.adb,
signal: options.signal,
});
if (!isAndroidSnapshotHelperSessionEnabled() || !options.adbProvider?.spawn) {
if (!isAndroidSnapshotHelperSessionEnabled()) {
emitDiagnostic({
phase: 'android_snapshot_helper_session_acquisition',
data: { outcome: 'disabled' },
});
return undefined;
}
if (!options.adbProvider?.spawn) {
emitDiagnostic({
phase: 'android_snapshot_helper_session_acquisition',
data: { outcome: 'spawn-unavailable' },
});
return undefined;
}
const callerResolved = resolveAndroidSnapshotHelperCaptureOptions(options);
Expand Down Expand Up @@ -172,9 +183,21 @@ async function resolveAndroidSnapshotHelperSession(params: {
resolved: AndroidSnapshotHelperResolvedCaptureOptions;
startBudgetMs: number;
}): Promise<AndroidSnapshotHelperSession | undefined> {
if (isAndroidSnapshotHelperSessionBackedOff(params.identity)) return undefined;
if (isAndroidSnapshotHelperSessionBackedOff(params.identity)) {
emitDiagnostic({
phase: 'android_snapshot_helper_session_acquisition',
data: { outcome: 'backed-off' },
});
return undefined;
}
await retireUnusableAndroidSnapshotHelperSession(params.deviceKey, params.identity);
return sessions().get(params.deviceKey) ?? (await tryStartAndroidSnapshotHelperSession(params));
const existing = sessions().get(params.deviceKey);
const session = existing ?? (await tryStartAndroidSnapshotHelperSession(params));
emitDiagnostic({
phase: 'android_snapshot_helper_session_acquisition',
data: { outcome: existing ? 'reused' : session ? 'created' : 'start-failed' },
});
return session;
}

/** Drops a cached session this command cannot write to, so only its forward is left behind. */
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,8 @@
import assert from 'node:assert/strict';
import fs from 'node:fs';
import os from 'node:os';
import path from 'node:path';
import { writeSnapshotEvidence } from './snapshot-evidence.ts';
import { test, vi } from 'vitest';
import { parseWaitPositionals } from '@agent-device/command-registry/wait-positionals';
import { createCommandSurfaceAgentDevice } from '@agent-device/command-runtime';
Expand All @@ -10,6 +14,7 @@ import type { LiveContext } from './live-harness.ts';

const harness = vi.hoisted(() => ({
runStep: vi.fn(),
captureDiagnosticSnapshot: vi.fn(),
verifyCommand: vi.fn(),
verifyBehavior: vi.fn(),
}));
Expand Down Expand Up @@ -94,5 +99,43 @@ test('catalog readiness waits until the automation screen has changed', async ()
if (args[0] === 'click' && args[1]?.startsWith('@')) throw done;
return result({});
});
harness.captureDiagnosticSnapshot.mockImplementation((context, _name, step) =>
harness.runStep(context, step, ['snapshot', '-i', '--debug']),
);
await assert.rejects(assertAutomationSystem({} as LiveContext), (error) => error === done);
});

test('retains bounded original helper evidence with request correlation', () => {
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'android-snapshot-evidence-'));
try {
writeSnapshotEvidence(dir, 'catalog', 'fixture', 100, 200, {
status: 0,
stdout: '',
stderr: '',
json: {
data: {
androidSnapshot: {
backend: 'android-helper',
helperTransport: 'instrumentation',
nodeCount: 268,
foregroundWindowPackage: 'token=secret',
unexpected: 'private',
},
warnings: Array(100).fill('token=secret ' + 'x'.repeat(1000)),
nodes: [{ label: 'private tree' }],
},
},
});
const content = fs.readFileSync(path.join(dir, 'snapshot-catalog.json'), 'utf8');
const evidence = JSON.parse(content);
assert.equal(evidence.session, 'fixture');
assert.deepEqual([evidence.startedAtMs, evidence.finishedAtMs], [100, 200]);
assert.equal(evidence.androidSnapshot.helperTransport, 'instrumentation');
assert.equal(evidence.androidSnapshot.nodeCount, 268);
assert.equal(evidence.warnings.length, 20);
assert.ok(!content.includes('secret') && !content.includes('private'));
assert.ok(content.length < 10_000);
} finally {
fs.rmSync(dir, { recursive: true, force: true });
}
});
17 changes: 12 additions & 5 deletions test/integration/android-emulator-e2e/live-automation-scenario.ts
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,13 @@ import {
requireAndroidResourceId,
scrollToVisibleSelector,
} from './live-assertions.ts';
import { type LiveContext, runStep, verifyBehavior, verifyCommand } from './live-harness.ts';
import {
type LiveContext,
runStep,
verifyBehavior,
verifyCommand,
captureDiagnosticSnapshot,
} from './live-harness.ts';
import { assertHumanSnapshotCompaction } from './live-snapshot-compaction.ts';

const C = PUBLIC_COMMANDS;
Expand All @@ -38,10 +44,11 @@ export async function assertAutomationSystem(context: LiveContext): Promise<void
'id="automation-continue-catalog"',
]);
await assertWaitSelector(context, 'id="catalog-title"');
const catalogSnapshot = await runStep(context, 'capture Android catalog navigation', [
'snapshot',
'-i',
]);
const catalogSnapshot = await captureDiagnosticSnapshot(
context,
'catalog',
'capture Android catalog navigation',
);
assertPersistentAndroidHelper(catalogSnapshot, { reused: true });
const settingsTab = (
catalogSnapshot.json?.data?.nodes as { label?: unknown; ref?: unknown }[] | undefined
Expand Down
20 changes: 20 additions & 0 deletions test/integration/android-emulator-e2e/live-harness.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import fs from 'node:fs';
import { writeSnapshotEvidence } from './snapshot-evidence.ts';
import path from 'node:path';

import {
Expand Down Expand Up @@ -105,3 +106,22 @@ function assertTier(value: string): asserts value is Tier {
if (value === 'smoke' || value === 'full') return;
throw new Error(`unsupported Android E2E tier: ${value}`);
}

export async function captureDiagnosticSnapshot(
context: LiveContext,
name: 'preflight' | 'catalog',
step: string,
) {
const startedAtMs = Date.now();
return await runStep(context, step, ['snapshot', '-i', '--debug'], {
observeResult: (result) =>
writeSnapshotEvidence(
context.artifactDir,
name,
context.session,
startedAtMs,
Date.now(),
result,
),
});
}
7 changes: 5 additions & 2 deletions test/integration/android-emulator-e2e/scenario-start.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import assert from 'node:assert/strict';
import { assertWaitText } from './live-assertions.ts';
import { type LiveContext, runStep } from './live-harness.ts';
import { type LiveContext, runStep, captureDiagnosticSnapshot } from './live-harness.ts';

export type AndroidEmulatorScenarioStart = {
ime: 'system' | 'test';
Expand All @@ -24,7 +24,10 @@ export async function prepareAndroidEmulatorScenario(
...(start.ime === 'system' ? ['--no-test-ime'] : []),
start.url,
]);
const preflight = await runStep(context, 'preflight Android snapshot helper', ['snapshot', '-i']);
const preflight =
context.currentScenario === 'smoke:automation-system'
? await captureDiagnosticSnapshot(context, 'preflight', 'preflight Android snapshot helper')
: await runStep(context, 'preflight Android snapshot helper', ['snapshot', '-i']);
assert.equal(preflight.json?.data?.androidSnapshot?.backend, 'android-helper');
await assertWaitText(context, start.landmark);
}
69 changes: 69 additions & 0 deletions test/integration/android-emulator-e2e/snapshot-evidence.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,69 @@
import fs from 'node:fs';
import path from 'node:path';
import { redactDiagnosticData } from '@agent-device/kernel/redaction';
import type { CliJsonResult } from '../cli-json.ts';

const fields = [
'backend',
'pixelDensity',
'helperVersion',
'helperApiVersion',
'helperTransport',
'helperSessionReused',
'installReason',
'waitForIdleTimeoutMs',
'waitForIdleQuietMs',
'timeoutMs',
'maxDepth',
'maxNodes',
'rootPresent',
'cacheInvalidated',
'captureMode',
'systemSurfaceOnly',
'foregroundWindowPackage',
'windowCount',
'nodeCount',
'helperTruncated',
'elapsedMs',
'occlusionScanUnavailable',
] as const;

export function writeSnapshotEvidence(
dir: string,
name: 'preflight' | 'catalog',
session: string,
startedAtMs: number,
finishedAtMs: number,
result: CliJsonResult,
): void {
const data = result.json?.data;
const source = data?.androidSnapshot;
const androidSnapshot: Record<string, string | number | boolean> = {};
if (source && typeof source === 'object') {
for (const field of fields) {
const value: unknown = source[field];
if (typeof value === 'string' || typeof value === 'number' || typeof value === 'boolean')
androidSnapshot[field] = value;
}
}
if (Array.isArray(source?.missingRootWindowTypes)) {
const types = source.missingRootWindowTypes
.filter((value: unknown): value is number => typeof value === 'number')
.slice(0, 20);
Object.assign(androidSnapshot, { missingRootWindowTypes: types });
}
const warnings = Array.isArray(data?.warnings)
? data.warnings
.filter((value: unknown): value is string => typeof value === 'string')
.slice(0, 20)
: [];
const evidence = redactDiagnosticData({
session,
startedAtMs,
finishedAtMs,
status: result.status,
androidSnapshot,
warnings,
});
fs.writeFileSync(path.join(dir, `snapshot-${name}.json`), JSON.stringify(evidence, null, 2));
}
Loading
Loading