From f1bd3733223c33d6d41abf005d9933ace16cf7a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Mon, 27 Jul 2026 11:00:34 +0200 Subject: [PATCH 1/2] feat(bench): renderer-pinned samples, topic-coverage gate, error quizzes; trim skillgym to agentic checks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The help conformance bench's quoted CLI output is now sourced from scripts/help-conformance-sample-outputs.mjs, and every sample is rebuilt through the real production renderers (settle output formatters, printHumanError, formatSnapshotText, refMutationAdmissionResponse) by scripts/__tests__/help-conformance-sample-outputs.test.ts — a rendering or message change fails deterministically instead of leaving the bench grading against output the CLI no longer prints. This retires the fabricated recoverable-failure envelope (production never throws a textual settle timeout; that case is replaced by a real DEVICE_IN_USE recovery quiz). Bench cases move to scripts/help-conformance-cases.mjs and are enumerated against the help-topic registry: helpTopicIds() is exported from cli-help, and scripts/__tests__/help-conformance-topic-coverage.test.ts fails when a help topic has neither a bench case nor an explicit waiver. New case families: error-envelope recovery quizzes (device-in-use, stale pinned ref, ambiguous find match, app-not-installed) pinned to real error text, topic coverage for tv/web/react-native/debugging/workflow, and a metamorphic twin of the settled-diff quiz. The skillgym smoke suite shrinks from 119 cases to the 5 that measure what only an agentic runner can show: skill routing plus output interpretation with a proven local CLI help probe (local-cli-help-policy). Its embedded samples now import the same pinned constants, replacing hand-transcribed output that had already drifted from the renderer. Knowledge checks belong to the bench; live fixture behavior belongs to the iOS simulator e2e suite. --- .fallowrc.json | 11 +- docs/agents/testing.md | 5 +- package.json | 4 +- .../__tests__/help-conformance-bench.test.ts | 6 +- .../help-conformance-sample-outputs.test.ts | 202 ++ .../help-conformance-topic-coverage.test.ts | 67 + scripts/help-conformance-bench.mjs | 236 +- scripts/help-conformance-cases.mjs | 426 +++ scripts/help-conformance-sample-outputs.d.mts | 13 + scripts/help-conformance-sample-outputs.mjs | 102 + src/cli/parser/cli-help.ts | 9 + test/skillgym/README.md | 73 +- .../suites/agent-device-smoke-suite.ts | 2488 +---------------- vitest.config.ts | 2 + 14 files changed, 963 insertions(+), 2681 deletions(-) create mode 100644 scripts/__tests__/help-conformance-sample-outputs.test.ts create mode 100644 scripts/__tests__/help-conformance-topic-coverage.test.ts create mode 100644 scripts/help-conformance-cases.mjs create mode 100644 scripts/help-conformance-sample-outputs.d.mts create mode 100644 scripts/help-conformance-sample-outputs.mjs diff --git a/.fallowrc.json b/.fallowrc.json index c1961e772..80ce2306c 100644 --- a/.fallowrc.json +++ b/.fallowrc.json @@ -30,7 +30,8 @@ "scripts/layering/**", "scripts/maestro-conformance/**", "apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests.xctestplan", - "scripts/write-xcuitest-cache-metadata.mjs" + "scripts/write-xcuitest-cache-metadata.mjs", + "scripts/help-conformance-sample-outputs.d.mts" ], "ignoreDependencies": [ "@theme" @@ -79,6 +80,14 @@ "PlatformDescriptorsAreTotal", "GatedKeysAreResolverKeys" ] + }, + { + "comment": "Help-benchmark conformance seams: helpTopicIds feeds the topic-coverage gate and PRIVATE_AX_RECOVERY_SAMPLE feeds the skillgym suite; both consumers are test-tree files outside --production analysis.", + "file": "{src/cli/parser/cli-help.ts,scripts/help-conformance-sample-outputs.mjs}", + "exports": [ + "helpTopicIds", + "PRIVATE_AX_RECOVERY_SAMPLE" + ] } ], "usedClassMembers": [ diff --git a/docs/agents/testing.md b/docs/agents/testing.md index b36969283..c51becd9f 100644 --- a/docs/agents/testing.md +++ b/docs/agents/testing.md @@ -16,8 +16,9 @@ The mapping it encodes, for when you need to run a gate directly or reason about | Platform/device response — anything emitting `platform`/`appleOs` on the wire, or shaping a daemon response | `pnpm test:integration:provider` **and** `pnpm test:coverage` | | Cross-platform behavior | `pnpm test:integration` | | iOS runner / Swift | `pnpm build:xcuitest` | -| CLI help/guidance (`src/cli/parser/cli-help.ts`, `src/cli-schema/`) | `pnpm exec vitest run src/cli/parser/__tests__ src/cli-schema/command-schema-guards.test.ts` | -| SkillGym prompts/assertions | `pnpm test:skillgym:case ` (broad: `pnpm test:skillgym`, filter with `-- --tag fixture-smoke` or `-- --tag skill-guidance`) | +| CLI help/guidance (`src/cli/parser/cli-help.ts`, `src/cli-schema/`) | `pnpm exec vitest run src/cli/parser/__tests__ src/cli-schema/command-schema-guards.test.ts scripts/__tests__` — the `scripts/__tests__` gates enforce help-topic benchmark coverage and pin the bench's quoted CLI samples to the real renderers | +| Help benchmark cases (`scripts/help-conformance-*.mjs`) | `pnpm exec vitest run scripts/__tests__` (deterministic gates); model-backed: `pnpm bench:help-conformance` (paid LLM calls, local only) | +| SkillGym prompts/assertions | `pnpm test:skillgym:case ` (broad: `pnpm test:skillgym`, filter with `-- --tag fixture-smoke` or `-- --tag skill-guidance`) — agentic routing + local-help-consumption proof only; command-planning knowledge checks belong in the help bench | | Anything in `src/`, `test/`, `skills/` | `pnpm format` | Two traps worth naming: diff --git a/package.json b/package.json index 000b9626f..4d31fef14 100644 --- a/package.json +++ b/package.json @@ -109,8 +109,8 @@ "perf:ios": "node --experimental-strip-types scripts/perf/run.ts --platform ios", "perf:android": "node --experimental-strip-types scripts/perf/run.ts --platform android", "lint": "oxlint . --deny-warnings", - "format": "node ./node_modules/oxfmt/bin/oxfmt --write src test skills scripts/help-conformance-bench.mjs scripts/help-conformance-case-checks.mjs scripts/help-conformance-command-validator.ts scripts/help-conformance-expectations.mjs scripts/help-conformance-plan-validator.mjs scripts/help-conformance-runner-output.mjs scripts/help-conformance-summary.mjs scripts/__tests__/help-conformance-bench.test.ts package.json tsconfig.json tsconfig.lib.json tsdown.config.ts vitest.config.ts .github/actions/setup-node-pnpm/action.yml .oxlintrc.json .oxfmtrc.json '!test/skillgym/.skillgym-results/**'", - "format:check": "node ./node_modules/oxfmt/bin/oxfmt --check src test skills scripts/help-conformance-bench.mjs scripts/help-conformance-case-checks.mjs scripts/help-conformance-command-validator.ts scripts/help-conformance-expectations.mjs scripts/help-conformance-plan-validator.mjs scripts/help-conformance-runner-output.mjs scripts/help-conformance-summary.mjs scripts/__tests__/help-conformance-bench.test.ts package.json tsconfig.json tsconfig.lib.json tsdown.config.ts vitest.config.ts .github/actions/setup-node-pnpm/action.yml .oxlintrc.json .oxfmtrc.json '!test/skillgym/.skillgym-results/**'", + "format": "node ./node_modules/oxfmt/bin/oxfmt --write src test skills scripts/help-conformance-bench.mjs scripts/help-conformance-case-checks.mjs scripts/help-conformance-command-validator.ts scripts/help-conformance-expectations.mjs scripts/help-conformance-plan-validator.mjs scripts/help-conformance-runner-output.mjs scripts/help-conformance-summary.mjs scripts/help-conformance-cases.mjs scripts/help-conformance-sample-outputs.mjs scripts/__tests__/help-conformance-bench.test.ts scripts/__tests__/help-conformance-sample-outputs.test.ts scripts/__tests__/help-conformance-topic-coverage.test.ts package.json tsconfig.json tsconfig.lib.json tsdown.config.ts vitest.config.ts .github/actions/setup-node-pnpm/action.yml .oxlintrc.json .oxfmtrc.json '!test/skillgym/.skillgym-results/**'", + "format:check": "node ./node_modules/oxfmt/bin/oxfmt --check src test skills scripts/help-conformance-bench.mjs scripts/help-conformance-case-checks.mjs scripts/help-conformance-command-validator.ts scripts/help-conformance-expectations.mjs scripts/help-conformance-plan-validator.mjs scripts/help-conformance-runner-output.mjs scripts/help-conformance-summary.mjs scripts/help-conformance-cases.mjs scripts/help-conformance-sample-outputs.mjs scripts/__tests__/help-conformance-bench.test.ts scripts/__tests__/help-conformance-sample-outputs.test.ts scripts/__tests__/help-conformance-topic-coverage.test.ts package.json tsconfig.json tsconfig.lib.json tsdown.config.ts vitest.config.ts .github/actions/setup-node-pnpm/action.yml .oxlintrc.json .oxfmtrc.json '!test/skillgym/.skillgym-results/**'", "fallow": "fallow audit --base origin/main", "fallow:all": "fallow --summary", "fallow:baseline": "(fallow dead-code --save-baseline fallow-baselines/dead-code.json --summary || true) && (fallow health --save-baseline fallow-baselines/health.json --summary || true)", diff --git a/scripts/__tests__/help-conformance-bench.test.ts b/scripts/__tests__/help-conformance-bench.test.ts index 20e4088ce..3bccd5fd8 100644 --- a/scripts/__tests__/help-conformance-bench.test.ts +++ b/scripts/__tests__/help-conformance-bench.test.ts @@ -214,8 +214,12 @@ test('runtime-output cases do not coach the command that their output should imp [ 'settle-diff-is-observation', 'sample-output-settled-diff-next-target', + 'metamorphic-settled-diff-next-target-notes', 'sample-output-not-settled-needs-observe', - 'sample-output-recoverable-failure-retries-in-session', + 'sample-output-device-in-use-reuses-session', + 'sample-output-stale-ref-resnapshots', + 'sample-output-ambiguous-match-reobserves', + 'sample-output-app-not-installed-discovers-first', ].join(','), '--runner', 'claude:test-model', diff --git a/scripts/__tests__/help-conformance-sample-outputs.test.ts b/scripts/__tests__/help-conformance-sample-outputs.test.ts new file mode 100644 index 000000000..1e151e8f6 --- /dev/null +++ b/scripts/__tests__/help-conformance-sample-outputs.test.ts @@ -0,0 +1,202 @@ +import assert from 'node:assert/strict'; +import { test } from 'vitest'; +import { + AMBIGUOUS_MATCH_SAMPLE, + APP_NOT_INSTALLED_SAMPLE, + DEVICE_IN_USE_SAMPLE, + NOT_SETTLED_SAMPLE, + PRIVATE_AX_RECOVERY_SAMPLE, + SETTLE_DIFF_SAMPLE, + SETTLE_DIFF_SAMPLE_NOTES, + SETTLE_TAIL_SAMPLE, + STALE_REF_SAMPLE, +} from '../help-conformance-sample-outputs.mjs'; +import { interactionCliOutputFormatters } from '../../src/commands/interaction/output.ts'; +import { NEVER_SETTLED_HINT } from '../../src/commands/interaction/runtime/settle.ts'; +import { refMutationAdmissionResponse } from '../../src/daemon/handlers/interaction-ref-policy.ts'; +import { buildSessionRecoveryHint } from '../../src/daemon/session-recovery-hints.ts'; +import { resolveRefStalenessWarning } from '../../src/daemon/session-snapshot.ts'; +import type { SessionState } from '../../src/daemon/types.ts'; +import { AppError, normalizeError } from '../../src/kernel/errors.ts'; +import type { SnapshotQualityVerdict } from '../../src/snapshot/snapshot-quality.ts'; +import { renderSnapshotQualityWarnings } from '../../src/snapshot/snapshot-quality.ts'; +import { formatSnapshotText, printHumanError } from '../../src/utils/output.ts'; + +// Every quiz case in scripts/help-conformance-cases.mjs quotes "captured" +// agent-device output. These tests rebuild each quoted sample through the real +// production renderer, so a rendering or message change fails HERE instead of +// leaving the benchmark grading models against output the CLI no longer +// prints — the drift class that made hand-transcribed samples untrustworthy. + +const formatPress = (result: Record) => + interactionCliOutputFormatters.press({ input: {}, result }); + +const formatFill = (result: Record) => + interactionCliOutputFormatters.fill({ input: {}, result }); + +function renderHumanError(error: AppError): string { + const lines: string[] = []; + const originalWrite = process.stderr.write; + process.stderr.write = ((chunk: string | Uint8Array) => { + lines.push(String(chunk)); + return true; + }) as typeof process.stderr.write; + try { + printHumanError(normalizeError(error)); + } finally { + process.stderr.write = originalWrite; + } + return lines.join('').trimEnd(); +} + +test('settle tail sample matches the press renderer, including ADR 0014 pinning', () => { + const output = formatPress({ + ref: 'e37', + x: 203, + y: 88, + settle: { + settled: true, + waitedMs: 540, + refsGeneration: 5, + diff: { + summary: { additions: 0, removals: 1, unchanged: 15 }, + lines: [{ kind: 'removed', text: '@e50 [text] "Suggested for you"' }], + }, + tail: [ + { ref: 'e64', role: 'text-field', label: 'Search' }, + { ref: 'e65', role: 'text', label: 'Recent searches' }, + { ref: 'e12', role: 'tab', label: 'Home' }, + { ref: 'e40', role: 'tab', label: 'Profile' }, + ], + }, + }); + assert.equal(output.text, SETTLE_TAIL_SAMPLE.output); +}); + +test('settled diff sample matches the fill renderer', () => { + const output = formatFill({ + text: 'callstack', + message: 'Filled 9 chars', + settle: { + settled: true, + waitedMs: 610, + refsGeneration: 6, + diff: { + summary: { additions: 2, removals: 0, unchanged: 18 }, + lines: [ + { kind: 'added', text: '@e64 [button] "@callstack.com"' }, + { kind: 'added', text: '@e65 [text] "Callstack"' }, + ], + }, + }, + }); + assert.equal(output.text, SETTLE_DIFF_SAMPLE.output); +}); + +test('metamorphic settled diff sample matches the fill renderer', () => { + const output = formatFill({ + text: 'groceries', + message: 'Filled 9 chars', + settle: { + settled: true, + waitedMs: 480, + refsGeneration: 3, + diff: { + summary: { additions: 2, removals: 0, unchanged: 11 }, + lines: [ + { kind: 'added', text: '@e21 [button] "Groceries list"' }, + { kind: 'added', text: '@e22 [text] "3 items"' }, + ], + }, + }, + }); + assert.equal(output.text, SETTLE_DIFF_SAMPLE_NOTES.output); +}); + +test('not-settled sample matches the press renderer and the production hint', () => { + const output = formatPress({ + ref: 'e12', + x: 166, + y: 240, + settle: { settled: false, waitedMs: 10_000, hint: NEVER_SETTLED_HINT }, + }); + assert.equal(output.text, NOT_SETTLED_SAMPLE.output); +}); + +test('private-ax recovery sample matches the snapshot renderer and quality warning', () => { + const nodes = [ + { + index: 1, + ref: 'e5', + type: 'Button', + label: 'Search', + interactive: true, + rect: { x: 20, y: 120, width: 200, height: 44 }, + }, + { + index: 2, + ref: 'e8', + type: 'Tab', + label: 'Home', + selected: true, + interactive: true, + rect: { x: 0, y: 780, width: 100, height: 60 }, + }, + ]; + const verdict = { state: 'recovered', backend: 'private-ax' } as SnapshotQualityVerdict; + const text = formatSnapshotText({ + nodes, + backend: 'private-ax', + snapshotQuality: verdict, + warnings: renderSnapshotQualityWarnings(verdict, nodes), + interactiveOnly: true, + }); + assert.equal(text.trimEnd(), PRIVATE_AX_RECOVERY_SAMPLE.output); +}); + +test('device-in-use sample matches the human error rendering and recovery hint', () => { + const owningSession = { name: 'checkout' } as SessionState; + const rendered = renderHumanError( + new AppError('DEVICE_IN_USE', 'Device is already in use by session "checkout".', { + session: 'checkout', + hint: buildSessionRecoveryHint(owningSession, 'device-in-use'), + }), + ); + assert.equal(rendered, DEVICE_IN_USE_SAMPLE.output); +}); + +test('stale-ref sample matches the real admission rejection and staleness hint', () => { + const session = { refFrameGeneration: 7 } as SessionState; + const response = refMutationAdmissionResponse({ + session, + ref: '@e12', + mintedGeneration: 5, + staleRefsWarning: resolveRefStalenessWarning({ session, ref: '@e12', mintedGeneration: 5 }), + }); + assert.ok(response && !response.ok, 'a superseded pin must be rejected'); + const rendered = renderHumanError( + new AppError( + response.error.code as ConstructorParameters[0], + response.error.message, + response.error.details, + ), + ); + assert.equal(rendered, STALE_REF_SAMPLE.output); +}); + +test('ambiguous-match sample matches the human error rendering and default hint', () => { + const rendered = renderHumanError( + new AppError( + 'AMBIGUOUS_MATCH', + 'find matched 3 elements for text "Follow". Use a more specific locator or selector.', + ), + ); + assert.equal(rendered, AMBIGUOUS_MATCH_SAMPLE.output); +}); + +test('app-not-installed sample matches the human error rendering and default hint', () => { + const rendered = renderHumanError( + new AppError('APP_NOT_INSTALLED', 'No app found matching "Shoply"'), + ); + assert.equal(rendered, APP_NOT_INSTALLED_SAMPLE.output); +}); diff --git a/scripts/__tests__/help-conformance-topic-coverage.test.ts b/scripts/__tests__/help-conformance-topic-coverage.test.ts new file mode 100644 index 000000000..b9931ebd5 --- /dev/null +++ b/scripts/__tests__/help-conformance-topic-coverage.test.ts @@ -0,0 +1,67 @@ +import assert from 'node:assert/strict'; +import { test } from 'vitest'; +import { CASES } from '../help-conformance-cases.mjs'; +import { helpTopicIds } from '../../src/cli/parser/cli-help.ts'; + +// "What enumerates N": benchmark cases are keyed to help topics, and this gate +// keys the case list to the topic registry itself. A new help topic must gain +// a benchmark case (docs: [..., '']) or an explicit waiver here — the +// waiver names why the topic's guidance is not yet benchmarked, so uncovered +// topics are a visible decision instead of silent drift. +const WAIVED_TOPICS: Record = { + cdp: 'JS-heap forensics niche; add cases when heap-guidance regressions show up in practice.', + macos: 'macOS surface guidance is thin and stable; no observed planning regressions yet.', + maestro: 'Compatibility reference, not a planning loop; conformance is oracle-tested instead.', + 'physical-device': 'Needs device-specific setup guidance; no portable planning task defined yet.', + 'react-devtools': + 'Profiling-window guidance; add cases when render-diagnosis planning regresses.', + remote: 'Remote/cloud lease setup; niche until remote workflows are benchmarked end to end.', +}; + +const FIRST_SCREEN_DOC = '--help:first30'; + +test('every case doc id is the first screen or a real help topic', () => { + const topics = new Set(helpTopicIds()); + for (const testCase of CASES) { + for (const doc of testCase.docs) { + assert.ok( + doc === FIRST_SCREEN_DOC || topics.has(doc), + `case "${testCase.id}" references unknown help doc "${doc}"`, + ); + } + } +}); + +test('every help topic has a benchmark case or an explicit waiver', () => { + const covered = new Set(CASES.flatMap((testCase) => testCase.docs)); + const uncovered = helpTopicIds().filter( + (topic) => !covered.has(topic) && !(topic in WAIVED_TOPICS), + ); + assert.deepEqual( + uncovered, + [], + 'new help topics need a benchmark case in scripts/help-conformance-cases.mjs or a waiver above', + ); +}); + +test('waivers only name real, uncovered topics', () => { + const topics = new Set(helpTopicIds()); + const covered = new Set(CASES.flatMap((testCase) => testCase.docs)); + for (const [topic, reason] of Object.entries(WAIVED_TOPICS)) { + assert.ok(topics.has(topic), `waived topic "${topic}" no longer exists — remove the waiver`); + assert.ok( + !covered.has(topic), + `waived topic "${topic}" is now covered by a case — remove the waiver`, + ); + assert.ok(reason.trim().length > 0, `waiver for "${topic}" needs a reason`); + } +}); + +test('the first help screen is exercised by every case family', () => { + for (const testCase of CASES) { + assert.ok( + testCase.docs.includes(FIRST_SCREEN_DOC), + `case "${testCase.id}" must include the first-screen doc — it is the only text every runner sees before choosing a topic`, + ); + } +}); diff --git a/scripts/help-conformance-bench.mjs b/scripts/help-conformance-bench.mjs index 45e3ab0b2..0d89ad57c 100644 --- a/scripts/help-conformance-bench.mjs +++ b/scripts/help-conformance-bench.mjs @@ -9,6 +9,7 @@ import { countChecks, scoreExpectations, } from './help-conformance-case-checks.mjs'; +import { CASES } from './help-conformance-cases.mjs'; import { validatePlanCommands } from './help-conformance-plan-validator.mjs'; import { detectRunnerError, extractCommands } from './help-conformance-runner-output.mjs'; import { summarizeResults } from './help-conformance-summary.mjs'; @@ -83,241 +84,6 @@ const OPTION_APPLIERS = { }, }; -// Raw-coordinate fallback the ported skillgym quiz cases forbid: a -// click/fill/press targeting bare numbers instead of a ref or selector. -const RAW_COORDINATE_TARGET = - /(?:^|\n)(?:agent-device\s+)?(?:click|fill|press)\s+-?\d+(?:\.\d+)?\s+-?\d+(?:\.\d+)?/i; - -const CASES = [ - { - id: 'raw-first-screen-bluesky', - docs: ['--help:first30'], - task: 'Plan commands to open an already installed Bluesky app, search "callstack", open the @callstack.com account, press Follow or Following, and close.', - expectations: [ - 'validPlanCommands', - 'fullPrefix', - 'usesSnapshotI', - 'usesSettleOnMutations', - 'noWaitStable', - ], - }, - { - id: 'metamorphic-community-search', - docs: ['--help:first30'], - task: 'Plan commands to open the already installed app com.example.community, open the visible Discover destination, fill the People search field with "react native", open the @react.dev account, press Connect or Connected, and close.', - expectations: [ - 'validPlanCommands', - 'fullPrefix', - 'usesSnapshotI', - 'usesSettleOnMutations', - 'noWaitStable', - 'opensAndCloses', - ], - matchers: [ - { - id: 'opensKnownCommunityApp', - pattern: /\bagent-device\s+open\s+com\.example\.community\b/i, - }, - { - id: 'fillsExpectedSearch', - pattern: /\bagent-device\s+fill\b[^\n]*(?:"react native"|'react native')[^\n]*--settle\b/i, - }, - { - id: 'usesLiteralHandleSelector', - pattern: - /\bagent-device\s+(?:press|click|tap)\b[^\n]*(?:label|text)=@react\.dev\b[^\n]*--settle\b/i, - }, - ], - forbidden: [ - { - id: 'noBlueskyLeakage', - pattern: /(?:bluesky|callstack|@e64|@callstack\.com)/i, - }, - { id: 'noRawCoordinateTarget', pattern: RAW_COORDINATE_TARGET }, - ], - }, - { - id: 'manual-qa-bluesky-script', - docs: ['--help:first30', 'manual-qa'], - task: 'You are following a manual QA script: on Bluesky, open Search, search "callstack", open @callstack.com, press Follow or Following, verify the button state changed, then close. Plan commands only.', - expectations: [ - 'validPlanCommands', - 'fullPrefix', - 'usesSnapshotI', - 'usesSettleOnMutations', - 'verifiesNamedExpectation', - 'noWaitStable', - ], - }, - { - id: 'dogfood-mode', - docs: ['--help:first30', 'dogfood'], - task: 'Plan a short dogfood pass for the logged-in iOS shop app com.example.shop. Exercise the visible Home, Search, and Cart destinations and capture reproducible evidence for any issue found.', - allowedExternalCommands: ['mkdir'], - expectations: [ - 'validPlanCommands', - 'fullPrefix', - 'usesSnapshotI', - 'usesSettleOnMutations', - 'usesDogfoodEvidence', - 'opensAndCloses', - ], - matchers: [ - { id: 'opensKnownDogfoodApp', pattern: /\bagent-device\s+open\s+com\.example\.shop\b/i }, - { - id: 'capturesStrongIssueEvidence', - pattern: /\b(?:screenshot\b[^\n]*--overlay-refs|record\s+start\b|logs\s+mark\b)/i, - }, - ], - }, - { - id: 'engineering-validate-mode', - docs: ['--help:first30', 'validate'], - task: 'Plan commands to validate a TypeScript-only CLI/runtime change to settled press output against the already installed iOS Settings app. Use the known General control, prove current built output is running, and clean up. Swift runner code did not change.', - allowedExternalCommands: ['pnpm'], - expectations: [ - 'validPlanCommands', - 'fullPrefix', - 'usesSnapshotI', - 'usesSettleOnMutations', - 'usesValidationPrep', - 'opensAndCloses', - ], - matchers: [ - { - id: 'opensSettings', - pattern: /\bagent-device\s+open\s+(?:settings|com\.apple\.Preferences)\b/i, - }, - ], - forbidden: [ - { - id: 'avoidsUnrelatedPlatformBuild', - pattern: /\bpnpm\s+(?:run\s+)?build:(?:android|xcuitest)\b/i, - }, - ], - }, - // The first three cases below are ported from - // test/skillgym/suites/agent-device-smoke-suite.ts (settle-diff-is-observation, - // sample-output-settled-diff-next-target, sample-output-not-settled-needs-observe). - // The fourth comes from test/output-economy/routine-workflow.ts. They are - // self-contained "next-command quiz" cases: a captured agent-device - // output plus a task, scored by regex instead of the named expectation - // scorers above. Output text mirrors the CURRENT settle rendering in - // src/commands/interaction/output.ts, including the "unchanged interactive - // (N):" tail added by #1167/#1172 for diffs with no meaningful added ref. - { - id: 'settle-diff-is-observation', - docs: ['--help:first30'], - task: `You already ran this command and observed its settled output: - -agent-device press @e37 --settle -Tapped @e37 (203, 88) -settled after 540ms: +0 -1 (~15 unchanged) -- @e50 [text] "Suggested for you" -unchanged interactive (4): -= @e64 [text-field] "Search" -= @e65 [text] "Recent searches" -= @e12 [tab] "Home" -= @e40 [tab] "Profile" - -Use the output already shown to determine whether the feed-search UI is present, then close the session. What command should run next?`, - expectations: ['validPlanCommands', 'fullPrefix'], - matchers: [{ id: 'plansClose', pattern: /(?:^|\n)(?:agent-device\s+)?close\b/i }], - forbidden: [ - { id: 'noSnapshot', pattern: /\bsnapshot\b/i }, - { id: 'noWait', pattern: /\bwait\b/i }, - { id: 'noFind', pattern: /\bfind\b/i }, - { id: 'noGet', pattern: /\bget\b/i }, - { id: 'noIs', pattern: /\bis\b/i }, - { id: 'noPressOrClick', pattern: /\b(?:press|click)\b/i }, - ], - }, - { - id: 'sample-output-settled-diff-next-target', - docs: ['--help:first30'], - task: `Read this previous agent-device output, then plan the next command: - -agent-device fill 'id="account-search"' "callstack" --settle -Filled 9 chars -settled after 610ms: +2 -0 (~18 unchanged) -+ @e64 [button] "@callstack.com" -+ @e65 [text] "Callstack" - -The task is to open the matching account result. What command should run next?`, - expectations: ['validPlanCommands', 'fullPrefix'], - matchers: [ - { id: 'pressOrClickOrTap', pattern: /\b(?:press|click|tap)\b/i }, - { id: 'usesE64RefOrLabel', pattern: /@e64\b|label=(?:["']?@callstack\.com["']?)/i }, - { id: 'usesSettleFlag', pattern: /--settle\b/i }, - ], - forbidden: [ - { id: 'noSnapshot', pattern: /\bsnapshot\b/i }, - { id: 'noWaitStable', pattern: /wait\s+stable/i }, - { id: 'noFill', pattern: /\bfill\b/i }, - { id: 'noRawCoordinateTarget', pattern: RAW_COORDINATE_TARGET }, - ], - }, - { - id: 'sample-output-not-settled-needs-observe', - docs: ['--help:first30'], - task: `Read this previous agent-device output, then plan the next command: - -agent-device press @e12 --settle -Tapped @e12 (166, 240) -not settled after 10000ms -hint: The UI kept changing for the whole settle budget (animation, carousel, or ticker?), so no settled diff is shown. Raise --timeout, wait for specific content, or take a fresh snapshot. - -The next target is not known yet. What command should run next?`, - expectations: ['validPlanCommands', 'fullPrefix'], - matchers: [ - { - id: 'observesBeforeActing', - pattern: /(?:^|\n)(?:agent-device\s+)?(?:wait\b|snapshot\b[^\n]*-i\b)/i, - }, - ], - forbidden: [ - { - id: 'noBareRefMutation', - pattern: /(?:^|\n)(?:agent-device\s+)?(?:press|click|fill|longpress)\s+@e\d+/i, - }, - { id: 'noRawCoordinateTarget', pattern: RAW_COORDINATE_TARGET }, - ], - }, - // The next-step decision measured by the routine-workflow oracle - // (test/output-economy/routine-workflow.ts): an actionable failure that - // preserved the session, the failing ref, its snapshot generation, a retry - // signal, and retry guidance should be recovered IN SESSION with a targeted - // retry — not by reopening the session or re-observing evidence the error - // already kept valid. - { - id: 'sample-output-recoverable-failure-retries-in-session', - docs: ['--help:first30'], - task: `Read this previous agent-device output, then plan the next command: - -agent-device press @e6 --settle -error COMMAND_FAILED: Tap on @e6 did not settle within 10000ms -retriable: true -hint: The tap did not settle in time. Retry press @e6 --settle with a higher --timeout; refs from this session are still valid. -details: { reason: "timeout", timeoutMs: 10000, ref: "@e6", session: "checkout", refsGeneration: 22 } - -What command should run next?`, - expectations: ['validPlanCommands', 'fullPrefix'], - matchers: [ - { id: 'retriesSameRef', pattern: /(?:^|\n)(?:agent-device\s+)?press\s+@e6\b/i }, - { id: 'keepsSettle', pattern: /--settle\b/i }, - { id: 'raisesTimeout', pattern: /--timeout\s+\d+/i }, - ], - forbidden: [ - { id: 'noReopen', pattern: /(?:^|\n)(?:agent-device\s+)?open\b/i }, - { id: 'noSnapshot', pattern: /\bsnapshot\b/i }, - { id: 'noFind', pattern: /\bfind\b/i }, - { id: 'noGet', pattern: /\bget\b/i }, - { id: 'noWait', pattern: /\bwait\b/i }, - { id: 'noRawCoordinateTarget', pattern: RAW_COORDINATE_TARGET }, - ], - }, -]; - function parseArgs(argv) { const args = { runners: undefined, cases: undefined, dryRun: false }; readArgs(args, argv, 0); diff --git a/scripts/help-conformance-cases.mjs b/scripts/help-conformance-cases.mjs new file mode 100644 index 000000000..942eab346 --- /dev/null +++ b/scripts/help-conformance-cases.mjs @@ -0,0 +1,426 @@ +import { + AMBIGUOUS_MATCH_SAMPLE, + APP_NOT_INSTALLED_SAMPLE, + DEVICE_IN_USE_SAMPLE, + NOT_SETTLED_SAMPLE, + SETTLE_DIFF_SAMPLE, + SETTLE_DIFF_SAMPLE_NOTES, + SETTLE_TAIL_SAMPLE, + STALE_REF_SAMPLE, + sampleText, +} from './help-conformance-sample-outputs.mjs'; + +// Raw-coordinate fallback the quiz cases forbid: a click/fill/press targeting +// bare numbers instead of a ref or selector. +const RAW_COORDINATE_TARGET = + /(?:^|\n)(?:agent-device\s+)?(?:click|fill|press)\s+-?\d+(?:\.\d+)?\s+-?\d+(?:\.\d+)?/i; + +function quiz(sample, question) { + return `Read this previous agent-device output, then plan the next command: + +${sampleText(sample)} + +${question}`; +} + +// Case docs reference help topic ids from src/cli/parser/cli-help.ts plus the +// synthetic '--help:first30' first-screen slice. Topic coverage is enforced by +// scripts/__tests__/help-conformance-topic-coverage.test.ts: a new help topic +// needs a case here or an explicit waiver there. +export const CASES = [ + { + id: 'raw-first-screen-bluesky', + docs: ['--help:first30'], + task: 'Plan commands to open an already installed Bluesky app, search "callstack", open the @callstack.com account, press Follow or Following, and close.', + expectations: [ + 'validPlanCommands', + 'fullPrefix', + 'usesSnapshotI', + 'usesSettleOnMutations', + 'noWaitStable', + ], + }, + { + id: 'metamorphic-community-search', + docs: ['--help:first30'], + task: 'Plan commands to open the already installed app com.example.community, open the visible Discover destination, fill the People search field with "react native", open the @react.dev account, press Connect or Connected, and close.', + expectations: [ + 'validPlanCommands', + 'fullPrefix', + 'usesSnapshotI', + 'usesSettleOnMutations', + 'noWaitStable', + 'opensAndCloses', + ], + matchers: [ + { + id: 'opensKnownCommunityApp', + pattern: /\bagent-device\s+open\s+com\.example\.community\b/i, + }, + { + id: 'fillsExpectedSearch', + pattern: /\bagent-device\s+fill\b[^\n]*(?:"react native"|'react native')[^\n]*--settle\b/i, + }, + { + id: 'usesLiteralHandleSelector', + pattern: + /\bagent-device\s+(?:press|click|tap)\b[^\n]*(?:label|text)=@react\.dev\b[^\n]*--settle\b/i, + }, + ], + forbidden: [ + { + id: 'noBlueskyLeakage', + pattern: /(?:bluesky|callstack|@e64|@callstack\.com)/i, + }, + { id: 'noRawCoordinateTarget', pattern: RAW_COORDINATE_TARGET }, + ], + }, + { + id: 'manual-qa-bluesky-script', + docs: ['--help:first30', 'manual-qa'], + task: 'You are following a manual QA script: on Bluesky, open Search, search "callstack", open @callstack.com, press Follow or Following, verify the button state changed, then close. Plan commands only.', + expectations: [ + 'validPlanCommands', + 'fullPrefix', + 'usesSnapshotI', + 'usesSettleOnMutations', + 'verifiesNamedExpectation', + 'noWaitStable', + ], + }, + { + id: 'dogfood-mode', + docs: ['--help:first30', 'dogfood'], + task: 'Plan a short dogfood pass for the logged-in iOS shop app com.example.shop. Exercise the visible Home, Search, and Cart destinations and capture reproducible evidence for any issue found.', + allowedExternalCommands: ['mkdir'], + expectations: [ + 'validPlanCommands', + 'fullPrefix', + 'usesSnapshotI', + 'usesSettleOnMutations', + 'usesDogfoodEvidence', + 'opensAndCloses', + ], + matchers: [ + { id: 'opensKnownDogfoodApp', pattern: /\bagent-device\s+open\s+com\.example\.shop\b/i }, + { + id: 'capturesStrongIssueEvidence', + pattern: /\b(?:screenshot\b[^\n]*--overlay-refs|record\s+start\b|logs\s+mark\b)/i, + }, + ], + }, + { + id: 'engineering-validate-mode', + docs: ['--help:first30', 'validate'], + task: 'Plan commands to validate a TypeScript-only CLI/runtime change to settled press output against the already installed iOS Settings app. Use the known General control, prove current built output is running, and clean up. Swift runner code did not change.', + allowedExternalCommands: ['pnpm'], + expectations: [ + 'validPlanCommands', + 'fullPrefix', + 'usesSnapshotI', + 'usesSettleOnMutations', + 'usesValidationPrep', + 'opensAndCloses', + ], + matchers: [ + { + id: 'opensSettings', + pattern: /\bagent-device\s+open\s+(?:settings|com\.apple\.Preferences)\b/i, + }, + ], + forbidden: [ + { + id: 'avoidsUnrelatedPlatformBuild', + pattern: /\bpnpm\s+(?:run\s+)?build:(?:android|xcuitest)\b/i, + }, + ], + }, + { + id: 'tv-focus-first-remote', + docs: ['--help:first30', 'tv'], + task: 'On an Android TV emulator, open the installed app com.example.tvhub, move focus to the "Continue watching" tile two positions to the right of the initially focused tile, activate it, verify the player screen appeared, and close. Plan commands only.', + expectations: ['validPlanCommands', 'fullPrefix', 'usesSnapshotI', 'opensAndCloses'], + matchers: [ + { id: 'movesFocusWithRemote', pattern: /\btv-remote\s+press\s+right\b/i }, + { id: 'activatesWithSelect', pattern: /\btv-remote\s+press\s+select\b/i }, + { id: 'verifiesOutcome', pattern: /\b(?:is\s+focused|wait\b|find\b)/i }, + ], + forbidden: [ + // Focus-first surface: activation goes through tv-remote select, not a + // coordinate/element tap (help tv "Do not assume press/click @ref works"). + { + id: 'noDirectTapActivation', + pattern: /(?:^|\n)agent-device\s+(?:press|click|tap)\s/i, + }, + { id: 'noRawAdbKeyevent', pattern: /\badb\s+shell\s+input\b/i }, + ], + }, + { + id: 'web-managed-backend-loop', + docs: ['--help:first30', 'web'], + task: 'On a fresh machine that has never run web automation, plan commands to set up and verify the managed web backend, open https://shop.example/login, fill the Email field with "qa@example.com", press the "Sign in" button, verify the "Welcome back" text appears, capture a screenshot to ./artifacts/web-login.png, and close. Plan commands only.', + expectations: ['validPlanCommands', 'fullPrefix', 'usesSnapshotI', 'opensAndCloses'], + matchers: [ + { + id: 'setsUpBackendBeforeOpen', + pattern: /\bagent-device\s+web\s+setup\b[\s\S]*\n[^\n]*\bopen\s+https:\/\//i, + }, + { id: 'verifiesBackendWithDoctor', pattern: /\bagent-device\s+web\s+doctor\b/i }, + { id: 'usesWebPlatform', pattern: /--platform\s+web\b/i }, + { id: 'verifiesWelcomeText', pattern: /\b(?:wait|is|find)\b[^\n]*welcome/i }, + ], + forbidden: [ + // help web: native mobile/desktop setup commands are out of scope for + // --platform web sessions. + { + id: 'noNativeSetupCommands', + pattern: /(?:^|\n)agent-device\s+(?:boot|apps|install|alert|keyboard|perf|logs)\b/i, + }, + { id: 'noStandaloneAgentBrowser', pattern: /(?:^|\n)agent-browser\b/i }, + ], + }, + { + id: 'react-native-overlay-before-tap', + docs: ['--help:first30', 'react-native'], + task: 'An Expo dev-client app on the iOS simulator shows a React Native warning overlay in the latest snapshot. Plan the commands that safely get past it and then press the control with id "submit-order". Plan commands only.', + expectations: ['validPlanCommands', 'fullPrefix', 'usesSettleOnMutations'], + matchers: [ + { + id: 'usesDismissOverlayCommand', + pattern: /(?:^|\n)agent-device\s+react-native\s+dismiss-overlay\b/i, + }, + { + id: 'refreshesRefsAfterDismiss', + pattern: /dismiss-overlay\b[\s\S]*\n[^\n]*\bsnapshot\s+-i\b/i, + }, + { id: 'pressesSubmitTarget', pattern: /(?:^|\n)agent-device\s+press\s+[^\n]*submit-order/i }, + ], + forbidden: [ + // help react-native: never press warning/error overlay text manually; + // the dismiss-overlay command owns LogBox/RedBox targeting. + { + id: 'noManualOverlayPress', + pattern: /(?:^|\n)agent-device\s+(?:press|click)\s+[^\n]*(?:warning|error|logbox|redbox)/i, + }, + { id: 'noPlainReloadCommand', pattern: /(?:^|\n)agent-device\s+reload\b/i }, + ], + }, + { + id: 'debugging-small-log-window', + docs: ['--help:first30', 'debugging'], + task: 'The "Load diagnostics" control (id "load-diagnostics") in the already-open iOS app intermittently fails. Plan commands to capture a small fresh log window plus request/response metadata around one reproduction. Plan commands only.', + expectations: ['validPlanCommands', 'fullPrefix'], + matchers: [ + { + id: 'clearsAndRestartsLogs', + pattern: /(?:^|\n)agent-device\s+logs\s+clear\s+--restart\b/i, + }, + { id: 'marksBeforeRepro', pattern: /\blogs\s+mark\b/i }, + { + id: 'reproducesTargetPress', + pattern: /(?:^|\n)agent-device\s+press\s+[^\n]*load-diagnostics/i, + }, + { id: 'readsLogPath', pattern: /\blogs\s+path\b/i }, + { id: 'dumpsNetworkMetadata', pattern: /\bnetwork\s+dump\b/i }, + ], + forbidden: [ + { id: 'noSessionReopen', pattern: /(?:^|\n)agent-device\s+open\b/i }, + { id: 'noSplitLogRestart', pattern: /\blogs\s+stop\b/i }, + ], + }, + { + id: 'workflow-install-artifact-before-open', + docs: ['--help:first30', 'workflow'], + task: 'A local Android build artifact ./dist/app-release.apk contains the app com.example.orders, which is not yet on the emulator. Plan commands to get it running with fresh state and confirm its first screen shows "Orders". Plan commands only.', + expectations: ['validPlanCommands', 'fullPrefix', 'usesSnapshotI'], + matchers: [ + { + id: 'installsIdThenArtifact', + pattern: /(?:^|\n)agent-device\s+install\s+com\.example\.orders\s+\S*app-release\.apk/i, + }, + { + id: 'opensFreshAfterInstall', + pattern: /\binstall\b[\s\S]*\n[^\n]*\bopen\s+com\.example\.orders\b[^\n]*--relaunch\b/i, + }, + { id: 'verifiesFirstScreen', pattern: /\b(?:wait|find|is|get)\b[^\n]*orders/i }, + ], + forbidden: [ + // help workflow: install for a first install; reinstall only when + // explicitly requested; never open an artifact path. + { id: 'noReinstall', pattern: /(?:^|\n)agent-device\s+reinstall\b/i }, + { id: 'noOpenArtifactPath', pattern: /(?:^|\n)agent-device\s+open\s+[^\n]*\.apk\b/i }, + ], + }, + // Next-command quiz cases: captured output (pinned to the real renderer by + // scripts/__tests__/help-conformance-sample-outputs.test.ts) plus a task, + // scored by regex instead of the named expectation scorers above. + { + id: 'settle-diff-is-observation', + docs: ['--help:first30'], + task: `You already ran this command and observed its settled output: + +${sampleText(SETTLE_TAIL_SAMPLE)} + +Use the output already shown to determine whether the feed-search UI is present, then close the session. What command should run next?`, + expectations: ['validPlanCommands', 'fullPrefix'], + matchers: [{ id: 'plansClose', pattern: /(?:^|\n)(?:agent-device\s+)?close\b/i }], + forbidden: [ + { id: 'noSnapshot', pattern: /\bsnapshot\b/i }, + { id: 'noWait', pattern: /\bwait\b/i }, + { id: 'noFind', pattern: /\bfind\b/i }, + { id: 'noGet', pattern: /\bget\b/i }, + { id: 'noIs', pattern: /\bis\b/i }, + { id: 'noPressOrClick', pattern: /\b(?:press|click)\b/i }, + ], + }, + { + id: 'sample-output-settled-diff-next-target', + docs: ['--help:first30'], + task: quiz( + SETTLE_DIFF_SAMPLE, + 'The task is to open the matching account result. What command should run next?', + ), + expectations: ['validPlanCommands', 'fullPrefix'], + matchers: [ + { id: 'pressOrClickOrTap', pattern: /\b(?:press|click|tap)\b/i }, + { id: 'usesE64RefOrLabel', pattern: /@e64\b|label=(?:["']?@callstack\.com["']?)/i }, + { id: 'usesSettleFlag', pattern: /--settle\b/i }, + ], + forbidden: [ + { id: 'noSnapshot', pattern: /\bsnapshot\b/i }, + { id: 'noWaitStable', pattern: /wait\s+stable/i }, + { id: 'noFill', pattern: /\bfill\b/i }, + { id: 'noRawCoordinateTarget', pattern: RAW_COORDINATE_TARGET }, + ], + }, + { + id: 'metamorphic-settled-diff-next-target-notes', + docs: ['--help:first30'], + task: quiz( + SETTLE_DIFF_SAMPLE_NOTES, + 'The task is to open the matching list result. What command should run next?', + ), + expectations: ['validPlanCommands', 'fullPrefix'], + matchers: [ + { id: 'pressOrClickOrTap', pattern: /\b(?:press|click|tap)\b/i }, + { id: 'usesE21RefOrLabel', pattern: /@e21\b|label=(?:["']?groceries list["']?)/i }, + { id: 'usesSettleFlag', pattern: /--settle\b/i }, + ], + forbidden: [ + { id: 'noSnapshot', pattern: /\bsnapshot\b/i }, + { id: 'noWaitStable', pattern: /wait\s+stable/i }, + { id: 'noFill', pattern: /\bfill\b/i }, + { id: 'noCallstackLeakage', pattern: /(?:callstack|@e64)/i }, + { id: 'noRawCoordinateTarget', pattern: RAW_COORDINATE_TARGET }, + ], + }, + { + id: 'sample-output-not-settled-needs-observe', + docs: ['--help:first30'], + task: quiz( + NOT_SETTLED_SAMPLE, + 'The next target is not known yet. What command should run next?', + ), + expectations: ['validPlanCommands', 'fullPrefix'], + matchers: [ + { + id: 'observesBeforeActing', + pattern: /(?:^|\n)(?:agent-device\s+)?(?:wait\b|snapshot\b[^\n]*-i\b)/i, + }, + ], + forbidden: [ + { + id: 'noBareRefMutation', + pattern: /(?:^|\n)(?:agent-device\s+)?(?:press|click|fill|longpress)\s+@e\d+/i, + }, + { id: 'noRawCoordinateTarget', pattern: RAW_COORDINATE_TARGET }, + ], + }, + { + id: 'sample-output-device-in-use-reuses-session', + docs: ['--help:first30'], + task: quiz( + DEVICE_IN_USE_SAMPLE, + 'You are continuing the checkout flow that the "checkout" session was already running on this device. What command should run next?', + ), + expectations: ['validPlanCommands', 'fullPrefix'], + matchers: [ + { + id: 'retriesWithOwningSession', + pattern: /(?:^|\n)agent-device\s+press\b[^\n]*--session\s+checkout\b/i, + }, + { id: 'keepsSettle', pattern: /--settle\b/i }, + ], + forbidden: [ + { id: 'noClose', pattern: /(?:^|\n)agent-device\s+close\b/i }, + { id: 'noReopen', pattern: /(?:^|\n)agent-device\s+open\b/i }, + { id: 'noRawCoordinateTarget', pattern: RAW_COORDINATE_TARGET }, + ], + }, + { + id: 'sample-output-stale-ref-resnapshots', + docs: ['--help:first30'], + task: quiz( + STALE_REF_SAMPLE, + 'The Continue control this ref pointed at may have moved. What command should run next?', + ), + expectations: ['validPlanCommands', 'fullPrefix'], + matchers: [ + { id: 'refreshesInteractiveRefs', pattern: /(?:^|\n)agent-device\s+snapshot\s+-i\b/i }, + ], + forbidden: [ + { + id: 'noBareRefRetry', + pattern: /(?:^|\n)agent-device\s+(?:press|click|fill|longpress)\s+@e\d/i, + }, + { id: 'noReopen', pattern: /(?:^|\n)agent-device\s+open\b/i }, + { id: 'noRawCoordinateTarget', pattern: RAW_COORDINATE_TARGET }, + ], + }, + { + id: 'sample-output-ambiguous-match-reobserves', + docs: ['--help:first30'], + task: quiz( + AMBIGUOUS_MATCH_SAMPLE, + 'The intent is to follow the @callstack.com account row. The candidate refs were not shown. What command should run next?', + ), + expectations: ['validPlanCommands', 'fullPrefix'], + matchers: [ + { + id: 'reobservesOrNarrows', + pattern: + /(?:^|\n)agent-device\s+(?:snapshot\s+-i\b|(?:find|press|click)\s+[^\n]*(?:role=|id=|label="?@callstack\.com))/i, + }, + ], + forbidden: [ + // The candidates live in error details the human output never printed, + // so a ref-targeting command here would be a guess. + { id: 'noGuessedRef', pattern: /(?:^|\n)agent-device\s+(?:press|click)\s+@e\d/i }, + { + id: 'noVerbatimRetry', + pattern: /(?:^|\n)agent-device\s+find\s+text\s+"?follow"?\s+press\b/i, + }, + { id: 'noRawCoordinateTarget', pattern: RAW_COORDINATE_TARGET }, + ], + }, + { + id: 'sample-output-app-not-installed-discovers-first', + docs: ['--help:first30'], + task: quiz( + APP_NOT_INSTALLED_SAMPLE, + 'The goal is still to open the shop app on this simulator; no build artifact was provided. What command should run next?', + ), + expectations: ['validPlanCommands', 'fullPrefix'], + matchers: [{ id: 'discoversInstalledApps', pattern: /(?:^|\n)agent-device\s+apps\b/i }], + forbidden: [ + { id: 'noBlindReopen', pattern: /(?:^|\n)agent-device\s+open\s+"?shoply\b/i }, + // No artifact exists to install; inventing one is the failure mode help + // workflow forbids ("Do not open artifact paths or invent package ids"). + { + id: 'noInventedInstall', + pattern: /(?:^|\n)agent-device\s+(?:install|install-from-source)\b/i, + }, + { id: 'noRawCoordinateTarget', pattern: RAW_COORDINATE_TARGET }, + ], + }, +]; diff --git a/scripts/help-conformance-sample-outputs.d.mts b/scripts/help-conformance-sample-outputs.d.mts new file mode 100644 index 000000000..461b07b5d --- /dev/null +++ b/scripts/help-conformance-sample-outputs.d.mts @@ -0,0 +1,13 @@ +export type CapturedSample = { command: string; output: string }; + +export declare function sampleText(sample: CapturedSample): string; + +export declare const SETTLE_TAIL_SAMPLE: CapturedSample; +export declare const SETTLE_DIFF_SAMPLE: CapturedSample; +export declare const SETTLE_DIFF_SAMPLE_NOTES: CapturedSample; +export declare const NOT_SETTLED_SAMPLE: CapturedSample; +export declare const PRIVATE_AX_RECOVERY_SAMPLE: CapturedSample; +export declare const DEVICE_IN_USE_SAMPLE: CapturedSample; +export declare const STALE_REF_SAMPLE: CapturedSample; +export declare const AMBIGUOUS_MATCH_SAMPLE: CapturedSample; +export declare const APP_NOT_INSTALLED_SAMPLE: CapturedSample; diff --git a/scripts/help-conformance-sample-outputs.mjs b/scripts/help-conformance-sample-outputs.mjs new file mode 100644 index 000000000..09e7616b0 --- /dev/null +++ b/scripts/help-conformance-sample-outputs.mjs @@ -0,0 +1,102 @@ +// Captured agent-device output embedded in benchmark quiz cases. Each sample +// is { command, output }: the command line the prompt claims was run, and the +// exact text the CLI would print for it. Every `output` is pinned to the real +// renderer by scripts/__tests__/help-conformance-sample-outputs.test.ts, which +// rebuilds it through src/commands/interaction/output.ts or +// src/utils/output.ts printHumanError — a rendering change fails that test +// instead of silently leaving the benchmark grading against stale output. + +export function sampleText(sample) { + return `${sample.command}\n${sample.output}`; +} + +// Settled press whose diff carried no added ref, so the response appended the +// unchanged-interactive tail (#1167/#1172). The response carried a diff plus +// refsGeneration, so the tail refs render pinned (ADR 0014). +export const SETTLE_TAIL_SAMPLE = { + command: 'agent-device press @e37 --settle', + output: `Tapped @e37 (203, 88) +settled after 540ms: +0 -1 (~15 unchanged) +- @e50 [text] "Suggested for you" +unchanged interactive (4): += @e64~s5 [text-field] "Search" += @e65~s5 [text] "Recent searches" += @e12~s5 [tab] "Home" += @e40~s5 [tab] "Profile"`, +}; + +// Settled fill whose diff exposes the next target directly. +export const SETTLE_DIFF_SAMPLE = { + command: `agent-device fill 'id="account-search"' "callstack" --settle`, + output: `Filled 9 chars +settled after 610ms: +2 -0 (~18 unchanged) ++ @e64 [button] "@callstack.com" ++ @e65 [text] "Callstack"`, +}; + +// Metamorphic twin of SETTLE_DIFF_SAMPLE on a different app and ref range, so +// a pass cannot come from memorizing the callstack sample. +export const SETTLE_DIFF_SAMPLE_NOTES = { + command: `agent-device fill 'id="notes-search"' "groceries" --settle`, + output: `Filled 9 chars +settled after 480ms: +2 -0 (~11 unchanged) ++ @e21 [button] "Groceries list" ++ @e22 [text] "3 items"`, +}; + +// Never-settled press: success response, no diff, NEVER_SETTLED_HINT attached. +export const NOT_SETTLED_SAMPLE = { + command: 'agent-device press @e12 --settle', + output: `Tapped @e12 (166, 240) +not settled after 10000ms +hint: The UI kept changing for the whole settle budget (animation, carousel, or ticker?), so no settled diff is shown. Raise --timeout, wait for specific content, or take a fresh snapshot.`, +}; + +// Recovered snapshot: the private-ax fallback fired but still exposed +// actionable refs. Warning wording is renderSnapshotQualityWarnings +// (src/snapshot/snapshot-quality.ts); lines are the structured snapshot +// renderer (src/utils/output.ts formatSnapshotText). +export const PRIVATE_AX_RECOVERY_SAMPLE = { + command: 'agent-device snapshot -i', + output: `Snapshot: 2 nodes +Detected an overly complex or slow accessibility tree. Fell back to the private-ax snapshot backend. It is OK to continue; use --json to inspect snapshotQuality.reason if you need recovery details. +@e5 [button] "Search" +@e8 [tab] "Home" [selected]`, +}; + +// DEVICE_IN_USE from src/daemon/handlers/session-open.ts: another session owns +// the device; the hint is buildSessionRecoveryHint(session, 'device-in-use'). +export const DEVICE_IN_USE_SAMPLE = { + command: `agent-device press 'label="Place order"' --settle`, + output: `Error (DEVICE_IN_USE): Device is already in use by session "checkout". +Hint: Run agent-device session list to inspect active sessions. To reuse this device, rerun the command with --session checkout. To open a new session on this device, first run agent-device close --session checkout.`, +}; + +// ADR 0014 mutation rejection from +// src/daemon/handlers/interaction-ref-policy.ts: a pinned ref minted from a +// superseded generation is rejected before dispatch. The daemon strips the +// `~s5` pin at the boundary (interaction-touch-targets.ts), so the message +// names the plain ref; the hint is the precise resolveRefStalenessWarning. +export const STALE_REF_SAMPLE = { + command: 'agent-device press @e12~s5 --settle', + output: `Error (COMMAND_FAILED): Ref @e12 was minted from a superseded snapshot generation +Hint: Ref @e12 was minted from snapshot s5 but the session's ref frame is now s7 — re-run snapshot -i.`, +}; + +// AMBIGUOUS_MATCH from src/daemon/handlers/find.ts buildAmbiguousMatchError; +// the by-design rejection instead of silent disambiguation. Candidate refs +// live in details, which the human rendering does not print — the agent must +// re-observe or narrow, not guess a ref it never saw. +export const AMBIGUOUS_MATCH_SAMPLE = { + command: 'agent-device find text "Follow" press', + output: `Error (AMBIGUOUS_MATCH): find matched 3 elements for text "Follow". Use a more specific locator or selector. +Hint: Multiple candidates matched. Narrow the query or pass an exact identifier.`, +}; + +// APP_NOT_INSTALLED from src/platforms/apple/core/app-resolution.ts; the hint +// is defaultHintForCode('APP_NOT_INSTALLED'). +export const APP_NOT_INSTALLED_SAMPLE = { + command: 'agent-device open Shoply --platform ios', + output: `Error (APP_NOT_INSTALLED): No app found matching "Shoply" +Hint: Run apps to discover the exact installed package or bundle id, or install the app before open.`, +}; diff --git a/src/cli/parser/cli-help.ts b/src/cli/parser/cli-help.ts index 9b19cc883..dee24591b 100644 --- a/src/cli/parser/cli-help.ts +++ b/src/cli/parser/cli-help.ts @@ -1282,6 +1282,15 @@ Usage: `; } +/** + * Topic-id registry view for conformance tooling: the help benchmark's topic + * coverage gate enumerates this instead of a hand-maintained list, so adding a + * topic without benchmark coverage (or an explicit waiver) fails a test. + */ +export function helpTopicIds(): string[] { + return Object.keys(HELP_TOPICS); +} + function buildHelpTopicUsageText(topicName: string): string | null { const topic = HELP_TOPICS[topicName as keyof typeof HELP_TOPICS]; if (!topic) return null; diff --git a/test/skillgym/README.md b/test/skillgym/README.md index 2315fc05a..4f7539583 100644 --- a/test/skillgym/README.md +++ b/test/skillgym/README.md @@ -1,58 +1,29 @@ # Skillgym For agent-device -This folder is a starter `skillgym` setup for benchmarking the `agent-device` skill with a controlled Expo target app. +This folder benchmarks the one thing only an agentic runner can prove: given the `agent-device` skill, does a real agent route to it, consult local CLI help before answering, and read captured `agent-device` output correctly in that agentic setting? -## Why `skillgym` fits here +Everything else this suite used to cover moved to cheaper, stronger owners: -`skillgym` is useful for `agent-device` in three layers: +- Command-planning knowledge checks (can a model choose the right next command from a help slice, a captured output, or an error?) live in the help conformance bench (`scripts/help-conformance-bench.mjs`). Its plans are validated by the production CLI parser, its case list is enumerated against the help-topic registry (`scripts/__tests__/help-conformance-topic-coverage.test.ts`), and its quoted samples are pinned to the real renderers (`scripts/__tests__/help-conformance-sample-outputs.test.ts`). +- Live fixture-app behavior is owned by the deterministic iOS simulator e2e suite (`test/integration/ios-simulator-e2e/`) and its coverage manifest. -1. Skill-routing checks: verify that the runner loads `skills/agent-device/SKILL.md` and its required references before it answers. -2. Workflow-planning checks: verify that the agent describes the right `agent-device` loop for a known fixture app. -3. Optional live-device smoke runs: locally, you can extend prompts so the agent actually drives `agent-device` against a simulator or device. +## What stays here -The included suite focuses on the first two layers so it stays stable and CI-safe. -The suite uses SkillGym v0.8 case tags: +The suite uses two case tags: -- `fixture-smoke`: fixture-specific app surface coverage -- `skill-guidance`: command-planning guidance regressions +- `fixture-smoke`: one routing smoke (`open-and-snapshot`) — a sane fixture-app plan from the skill plus local help, without reading project source. +- `skill-guidance`: four output-interpretation cases that require a REAL observed local help probe before scoring (`requireLocalCliHelp` + `suites/local-cli-help-policy.ts` over observed command events): `settle-diff-is-observation`, `sample-output-settled-diff-next-target`, `sample-output-not-settled-needs-observe`, `sample-output-private-ax-recovery-continues`. Allowing help without verifying that the runner actually read it can misclassify a model-prior failure as a help-guidance failure. + +The captured output quoted in the skill-guidance cases is imported from `scripts/help-conformance-sample-outputs.mjs` — the same constants the bench embeds — so a renderer change fails the pinning test instead of silently grading agents against output the CLI no longer prints. ## Included files -- `../../examples/test-app/`: minimal Expo SDK 56 development-build fixture app for broad UI coverage +- `../../examples/test-app/`: minimal Expo SDK 56 development-build fixture app - `skillgym.config.ts`: starter config that runs Codex and Claude Haiku against this repo -- `suites/agent-device-smoke-suite.ts`: planning suite for skill routing, fixture-aware flows, and skill-guidance regressions - -## Current coverage - -The suite keeps the app small while separating coverage into two non-overlapping groups. - -Fixture smoke cases cover concrete app surfaces: - -- open/snapshot/close defaults with the installed Expo development build -- banners, alerts, toggles, and quick actions on Home -- search debounce, filters, long-list scroll, favorites, and cart updates in Catalog -- detail navigation, quantity edits, note append, and save-to-cart on Product -- form validation, success submit, iOS keyboard-dismiss fallback, and reset on Checkout form -- diagnostics load/error/retry plus reset alert handling in Settings -- accessibility audit via screenshot + snapshot - -Skill-guidance regression cases cover distinct command-planning habits: +- `suites/agent-device-smoke-suite.ts`: the routing + output-interpretation suite +- `suites/local-cli-help-policy.ts`: the observed-command policy that proves a real local help command ran -- read-only inspection versus mutation -- fresh `@ref` targeting, durable selectors, raw-rect fallbacks, and off-screen scroll recovery -- interpreting representative `agent-device` output, including settled diffs, not-settled hints, and private-AX recovery warnings -- text replacement, append semantics, supported field clearing, keyboard status, and keyboard fallback -- install/open setup, Expo Go/dev-client launch paths, app discovery, session scoping, and app-owned navigation fallbacks -- Metro reload, logs, network dump, alert fallback, and screenshot evidence -- performance metrics, React DevTools profiling, gestures, settings, and trace capture -- Android TV and Vega OS remote planning, including focus moves, exact holds, and Vega VVD lifecycle -- remote config, macOS menu bar surfaces, replay update, same-session mutation ordering, and batch schema/recording - -Use SkillGym for stable behavior regressions: can a runner choose the right next command from help, app-contract facts, or representative CLI output? For rapid help-layout A/B testing, prefer the lighter help conformance bench (`scripts/help-conformance-bench.mjs`) because it can feed only the top-level first screen or one help topic without letting the runner read the full help page, it runs runner x case pairs concurrently (`HELP_BENCH_CONCURRENCY`, default 4), and it can grade a draft help rewrite with zero rebuild via `--override-doc =` (repeatable; last occurrence per topic wins), which loads that file's contents in place of shelling out to `node bin/agent-device.mjs help ` for that one topic while still applying the same post-processing as the live source (the `--help:first30` doc id stays capped to its first 30 lines), so the A/B grade compares like with like; a topic id no selected case uses fails fast instead of silently grading the real doc. Returned plans are checked against the production CLI parser and interaction grammar, plus a narrow executable allowlist and shell-projection guard, so invented flags, malformed selectors, placeholder refs, redirection, and unsupported helper commands fail even when keyword checks pass. Use `--repeat ` to measure variance: every runner x case pair gets numbered independent trials plus an aggregate pass-rate and failure-taxonomy report. The `metamorphic-community-search` case renames the app, controls, query, and account from the original search flow and explicitly forbids original-answer leakage, providing a cheap generalization check. The bench also includes four deliberately uncoached "next-command quiz" cases (`settle-diff-is-observation`, `sample-output-settled-diff-next-target`, `sample-output-not-settled-needs-observe`, and `sample-output-recoverable-failure-retries-in-session`) ported from this suite's skill-guidance regressions and the routine-workflow oracle (`test/output-economy/routine-workflow.ts`), for scoring whether a runner can infer the next action from representative captured `agent-device` output — including the `--settle` "unchanged interactive (N):" tail and an actionable settle timeout that must be recovered in-session rather than by reopening or re-observing — instead of being told the answer by the task. Filter with `--cases`/`--case` and `--runners`/`--runner` (both repeatable/CSV) the same way as this suite. - -Parser-backed validation proves command syntax, positional bounds, and interaction grammar; it does not prove runtime semantics for intentionally free-form values such as `open `. A selector-looking app string can therefore be syntactically valid even when it would not resolve to an installed app. - -The SkillGym output-interpretation cases require an observed local `--help` lookup before scoring. Allowing help without verifying that the runner actually read it can misclassify a model-prior failure as a help-guidance failure. +For help-layout A/B testing and command-planning regressions, use the bench instead of adding cases here: it feeds only the top-level first screen or one help topic, runs runner x case pairs concurrently (`HELP_BENCH_CONCURRENCY`, default 4), grades draft help rewrites with zero rebuild via `--override-doc =`, measures variance with `--repeat ` plus an aggregate pass-rate and failure-taxonomy report, and filters with `--cases`/`--case` and `--runners`/`--runner` (both repeatable/CSV) the same way as this suite. `assertAgentDeviceEvidence` is intentionally soft when a runner does not expose skill-detection telemetry. When telemetry exists, the suite asserts that `agent-device` was loaded; when it is absent, the cases still judge command-planning output instead of failing on missing runner metadata. @@ -62,15 +33,9 @@ SkillGym v0.8 command assertions are for observed command events. This suite pri The source-read guardrails use `assert.soft.*` plus deferred explain questions so one failing run can report multiple routing mistakes and can later be inspected with `skillgym explain`. Suite types use the v0.8 root export name `Case`; older `TestCase` imports no longer typecheck. -## Suggested workflow - -1. Start with the included smoke suite to benchmark routing and default guidance. -2. Extend the suite with app-specific prompts that cover a new command-planning category rather than duplicating an existing one. -3. Add local-only cases that expect real `agent-device` shell commands once you are ready to involve a running simulator. - ## Running the suite -`skillgym` is installed as a repo dev dependency, so run the starter suite from the project root: +`skillgym` is installed as a repo dev dependency, so run the suite from the project root: ```bash cd /absolute/path/to/agent-device @@ -89,7 +54,7 @@ pnpm test:skillgym:case open-and-snapshot Useful v0.8 filters, reporters, and recovery options: ```bash -pnpm test:skillgym -- --tag fixture-smoke +pnpm test:skillgym -- --tag skill-guidance pnpm test:skillgym -- --reporter json pnpm test:skillgym -- --repeat 3 --repeat-failure 1 ``` @@ -133,6 +98,6 @@ The configured runners call external Codex and Claude model backends. In Codex s ## Where to extend next -- Add suite cases that ask for selector-based plans against `Agent Device Tester`. -- Add local-only prompts that expect `agent-device open`, `snapshot`, `snapshot -i`, `get`, and `wait`. -- Add regression snapshots once the prompt set stabilizes. +- Add a case here ONLY when it measures agentic behavior: skill routing, local-help consultation, or multi-turn discovery a single non-agentic call cannot show. +- Add command-planning knowledge checks to `scripts/help-conformance-cases.mjs` instead; the topic-coverage gate tells you which help topics are unbenchmarked. +- Add local-only cases that drive a live simulator only when the deterministic e2e suite cannot own the behavior. diff --git a/test/skillgym/suites/agent-device-smoke-suite.ts b/test/skillgym/suites/agent-device-smoke-suite.ts index 72332a43f..c75ac72e8 100644 --- a/test/skillgym/suites/agent-device-smoke-suite.ts +++ b/test/skillgym/suites/agent-device-smoke-suite.ts @@ -1,24 +1,37 @@ import { assert, commandMatcher, type Case, type CommandMatcher } from 'skillgym'; +import { + NOT_SETTLED_SAMPLE, + PRIVATE_AX_RECOVERY_SAMPLE, + SETTLE_DIFF_SAMPLE, + SETTLE_TAIL_SAMPLE, + sampleText, +} from '../../../scripts/help-conformance-sample-outputs.mjs'; import { extractCommandEvents, isActualLocalCliHelpProbe, isAllowedLocalCliHelpCommand, } from './local-cli-help-policy.ts'; +// This suite is deliberately small. It keeps only what an agentic runner can +// prove and a single non-agentic LLM call cannot: that a runner with the +// agent-device skill routes to it, consults LOCAL CLI help before answering +// (isActualLocalCliHelpProbe over observed command events), and reads +// captured agent-device output correctly in that agentic setting. Knowledge +// checks about command planning live in the cheaper, parser-validated help +// conformance bench (scripts/help-conformance-bench.mjs); live fixture +// behavior is owned by the iOS simulator e2e suite. The quoted samples are +// shared with the bench and pinned to the real renderers by +// scripts/__tests__/help-conformance-sample-outputs.test.ts. + type SessionReport = Parameters[0]; type AssertionContext = Parameters[1]; -type OutputMatcher = string | RegExp | PlannedCommandMatcher | TopLevelPlannedCommandMatcher; +type OutputMatcher = string | RegExp | PlannedCommandMatcher; interface PlannedCommandMatcher { kind: 'planned-command'; matchers: CommandMatcher[]; } -interface TopLevelPlannedCommandMatcher { - kind: 'top-level-planned-command'; - commands: string[]; -} - interface OutputAbsenceContext { output: string; finalOutput: string; @@ -61,19 +74,12 @@ function escapeRegExp(value: string) { return value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); } -function buildPrompt(options: { - contract: string[]; - task: string; - finalOutputInstructions?: string; - requireLocalCliHelp?: boolean; -}) { +function buildPrompt(options: { contract: string[]; task: string; requireLocalCliHelp?: boolean }) { const contractLines = options.contract.map((line) => `- ${line}`).join('\n'); - const finalOutputInstructions = - options.finalOutputInstructions ?? DEFAULT_FINAL_OUTPUT_INSTRUCTIONS; const helpRequirement = options.requireLocalCliHelp ? '\nBefore planning, run node bin/agent-device.mjs --help and use that output as the command contract.' : ''; - return `${BASE_INSTRUCTIONS}${helpRequirement}\n${finalOutputInstructions}\n\nApp contract:\n${contractLines}\n\nTask:\n${options.task}`; + return `${BASE_INSTRUCTIONS}${helpRequirement}\n${DEFAULT_FINAL_OUTPUT_INSTRUCTIONS}\n\nApp contract:\n${contractLines}\n\nTask:\n${options.task}`; } function assertAgentDeviceEvidence(report: SessionReport) { @@ -126,22 +132,6 @@ function plannedCommandAlternatives(commands: string[]): PlannedCommandMatcher { }; } -function topLevelPlannedCommand(command: string): TopLevelPlannedCommandMatcher { - return topLevelPlannedCommandAlternatives([command]); -} - -function topLevelPlannedCommandAlternatives(commands: string[]): TopLevelPlannedCommandMatcher { - return { - kind: 'top-level-planned-command', - commands: commands.map((command) => { - const [executable, ...args] = commandParts(command); - assert.ok(executable, 'top-level planned command must not be empty'); - assert.equal(args.length, 0, 'top-level planned command matches only one command token'); - return executable; - }), - }; -} - function assertOutputs(finalOutput: string, matchers: OutputMatcher[]) { const output = normalizedFinalOutput(finalOutput); const plannedReport = plannedCommandReport(output); @@ -151,11 +141,6 @@ function assertOutputs(finalOutput: string, matchers: OutputMatcher[]) { continue; } - if (isTopLevelPlannedCommandMatcher(matcher)) { - assertTopLevelPlannedCommandIncludes(output, matcher); - continue; - } - assert.output.includes(normalizedOutputReport(output), matcher); } } @@ -176,11 +161,6 @@ function assertOutputAbsent(context: OutputAbsenceContext, matcher: OutputMatche return; } - if (isTopLevelPlannedCommandMatcher(matcher)) { - assertTopLevelPlannedCommandNotIncludes(context.output, matcher); - return; - } - if (typeof matcher === 'string') { assertStringOutputAbsent(context, matcher); return; @@ -204,16 +184,6 @@ function isPlannedCommandMatcher(matcher: OutputMatcher): matcher is PlannedComm ); } -function isTopLevelPlannedCommandMatcher( - matcher: OutputMatcher, -): matcher is TopLevelPlannedCommandMatcher { - return ( - typeof matcher === 'object' && - !(matcher instanceof RegExp) && - matcher.kind === 'top-level-planned-command' - ); -} - function assertPlannedCommandIncludes(report: SessionReport, matcher: PlannedCommandMatcher) { if (matcher.matchers.length === 1) { assert.commands.includes(report, matcher.matchers[0]!); @@ -239,58 +209,6 @@ function assertPlannedCommandNotIncludes(report: SessionReport, matcher: Planned } } -function assertTopLevelPlannedCommandIncludes( - output: string, - matcher: TopLevelPlannedCommandMatcher, -) { - const observed = topLevelPlannedCommands(output); - assert.ok( - observed.some((command) => matcher.commands.includes(command)), - `Expected final output to include top-level command ${matcher.commands - .map((command) => JSON.stringify(command)) - .join(' or ')}. Observed top-level commands: ${observed - .map((command) => JSON.stringify(command)) - .join(', ')}`, - ); -} - -function assertTopLevelPlannedCommandNotIncludes( - output: string, - matcher: TopLevelPlannedCommandMatcher, -) { - const observed = topLevelPlannedCommands(output); - const forbidden = observed.filter((command) => matcher.commands.includes(command)); - assert.deepEqual( - forbidden, - [], - `Expected final output not to include top-level command ${matcher.commands - .map((command) => JSON.stringify(command)) - .join(' or ')}. Observed top-level commands: ${observed - .map((command) => JSON.stringify(command)) - .join(', ')}`, - ); -} - -function topLevelPlannedCommands(output: string): string[] { - return normalizedFinalOutput(output) - .split('\n') - .map(topLevelPlannedCommandFromLine) - .filter((command): command is string => command !== undefined); -} - -function topLevelPlannedCommandFromLine(line: string): string | undefined { - const [executable, firstArg] = commandParts(line.trim()); - if (executable === undefined) { - return undefined; - } - - return topLevelCommandToken(executable, firstArg); -} - -function topLevelCommandToken(executable: string, firstArg: string | undefined): string { - return executable === 'agent-device' && firstArg !== undefined ? firstArg : executable; -} - function normalizedFinalOutput(output: string): string { return output .replace(/```[a-z]*\n?/gi, '') @@ -330,29 +248,6 @@ function assertExpectedOutput( assertOutputs(ctx.finalOutput(), matchers); } -function assertFinalOutputAgentDeviceCommandsOnly(finalOutput: string) { - const output = normalizedFinalOutput(finalOutput); - const lines = output - .split('\n') - .map((line) => line.trim()) - .filter(Boolean); - - assert.ok(lines.length > 0, 'Expected final output to contain agent-device commands.'); - - for (const line of lines) { - assert.match( - line, - /^agent-device\s+\S+/, - `Expected final output line to be one agent-device command with no prose: ${JSON.stringify(line)}`, - ); - assert.doesNotMatch( - line, - /\s(?:&&|\|\||\|)\s|;/, - `Expected one command per line without shell chaining: ${JSON.stringify(line)}`, - ); - } -} - function assertOnlyLocalCliHelpCommands(report: SessionReport) { const commandEvents = extractCommandEvents(report); const forbiddenCommands = commandEvents.filter( @@ -378,21 +273,7 @@ function assertLocalCliHelpUsed(report: SessionReport) { const RAW_COORDINATE_TARGET = /(?:^|\n)(?:agent-device\s+)?(?:click|fill|press)\s+-?\d+(?:\.\d+)?\s+-?\d+(?:\.\d+)?/i; -const PSEUDO_ASSERTION_COMMAND = /(?:^|\n)\s*(?:assert|assertVisible|waitFor|waitForText)\b/i; -const RAW_RECT_SNAPSHOT = /snapshot\b(?=[^\n]*-i\b)(?=[^\n]*(?:--json|--raw))/i; const SHELL_OUTPUT_PROJECTION = /(?:2>\s*\/dev\/null|\|\s*(?:jq|grep|head|tail)\b)/i; -const UNSUPPORTED_SELECTOR_KEY = /(?:^|[\s'"])(?:button|placeholder|index|key)\s*=/i; -const BOUNDED_PROFILE_SLOW = /react-devtools\s+profile\s+slow\b[^\n]*--limit\s+(?:5|10)\b/i; -const BOUNDED_PROFILE_RERENDERS = - /react-devtools\s+profile\s+rerenders\b[^\n]*--limit\s+(?:5|10)\b/i; -const BOUNDED_PROFILE_TIMELINE = - /react-devtools\s+profile\s+timeline\b[^\n]*--limit\s+(?:10|20)\b/i; -const BROAD_PROFILE_SLOW_LIMIT = - /react-devtools\s+profile\s+slow\b[^\n]*--limit\s+(?:[5-9]\d|[1-9]\d{2,})\b/i; -const CDP_MEMORY_USAGE_SAMPLE = /cdp\s+memory\s+usage\s+sample\b/i; -const CDP_MEMORY_SNAPSHOT_CAPTURE = /cdp\s+memory\s+snapshot\s+capture\b/i; -const IOS_EXPO_GO_OPEN = - /(?:^|\n)(?:agent-device\s+)?open\s+["']Expo Go["']\s+["']?exp:\/\/127\.0\.0\.1:8081["']?/i; const IOS_TEST_APP_DEV_BUILD_OPEN = new RegExp( String.raw`(?:^|\n)(?:agent-device\s+)?open\s+` + String.raw`(?:(?:"Agent Device Tester")|(?:'Agent Device Tester')|com\.callstack\.agentdevicelab)\b`, @@ -406,10 +287,8 @@ function makeCase(options: { tags?: string[]; outputs?: OutputMatcher[]; forbiddenOutputs?: OutputMatcher[]; - strictFinalOutput?: boolean; allowOnlyLocalCliHelpCommands?: boolean; requireLocalCliHelp?: boolean; - finalOutputInstructions?: string; }): Case { return { id: options.id, @@ -417,7 +296,6 @@ function makeCase(options: { prompt: buildPrompt({ contract: options.contract, task: options.task, - finalOutputInstructions: options.finalOutputInstructions, requireLocalCliHelp: options.requireLocalCliHelp, }), assert(report, ctx) { @@ -428,25 +306,12 @@ function makeCase(options: { }); assertExpectedOutput(report, ctx, options.outputs); assertNoOutputs(ctx.finalOutput(), options.forbiddenOutputs ?? []); - assertOptionalCasePolicies(report, ctx, options); + if (options.allowOnlyLocalCliHelpCommands) assertOnlyLocalCliHelpCommands(report); + if (options.requireLocalCliHelp) assertLocalCliHelpUsed(report); }, }; } -function assertOptionalCasePolicies( - report: SessionReport, - ctx: AssertionContext, - options: { - strictFinalOutput?: boolean; - allowOnlyLocalCliHelpCommands?: boolean; - requireLocalCliHelp?: boolean; - }, -) { - if (options.strictFinalOutput) assertFinalOutputAgentDeviceCommandsOnly(ctx.finalOutput()); - if (options.allowOnlyLocalCliHelpCommands) assertOnlyLocalCliHelpCommands(report); - if (options.requireLocalCliHelp) assertLocalCliHelpUsed(report); -} - function withTags(tags: string[], cases: Case[]): Case[] { return cases.map((testCase) => ({ ...testCase, @@ -454,6 +319,16 @@ function withTags(tags: string[], cases: Case[]): Case[] { })); } +function quiz(sample: { command: string; output: string }, question: string): string { + return `Read this previous agent-device output, then plan the next command: + +${sampleText(sample)} + +${question}`; +} + +// Routing smoke: proves the runner reaches a sane plan for the fixture app +// from the skill plus local help, without reading project source. const FIXTURE_SMOKE_CASES: Case[] = [ makeCase({ id: 'open-and-snapshot', @@ -471,2268 +346,109 @@ const FIXTURE_SMOKE_CASES: Case[] = [ /exp:\/\/127\.0\.0\.1:8081/i, ], }), +]; + +// Output-interpretation cases: each run must consult real local CLI help +// (requireLocalCliHelp + the local-cli-help-policy events check) and then read +// captured agent-device output correctly. The bench carries the same cases as +// single-call knowledge checks; here they measure the agentic path. +const SKILL_GUIDANCE_CASES: Case[] = [ makeCase({ - id: 'home-dismiss-notice', - contract: [ - 'App name: Agent Device Tester', - 'Current screen: Home tab', - 'testID=dismiss-notice', - 'visible text: Release notice', - ], - task: 'Assume Agent Device Tester is already open on the Home tab. Plan the commands to dismiss the Release notice using the dismiss-notice testID, verify it is gone with diff snapshot -i, then close.', - outputs: [ - /dismiss-notice/i, - /(?:diff snapshot -i|snapshot\b.*(?:-i\b.*--diff|--diff\b.*-i\b))/i, - plannedCommand('close'), - ], - }), - makeCase({ - id: 'home-confirm-alert', - contract: [ - 'App name: Agent Device Tester', - 'Current screen: Home tab', - 'testID=home-open-modal', - 'Opening it shows a native confirmation alert', - 'Home tab selector: label="Home"', - ], - task: 'Assume Agent Device Tester is already open on the Home tab. Plan the commands to open the confirmation alert, dismiss it using alert wait + alert dismiss, then verify the app is still on Home.', - outputs: [ - /home-open-modal/i, - plannedCommand('alert wait'), - plannedCommand('alert dismiss'), - /label=(?:["']Home["']|Home)/i, - ], - }), - makeCase({ - id: 'home-refresh-metrics', - contract: [ - 'App name: Agent Device Tester', - 'Current screen: Home tab', - 'testID=refresh-metrics', - 'visible loading text: Refreshing metrics...', - ], - task: 'Assume Agent Device Tester is already open on Home. Plan the commands to tap Refresh metrics, wait for "Refreshing metrics..." to appear, then verify the loading state is gone.', - outputs: [/refresh-metrics/i, plannedCommand('wait'), /Refreshing metrics/i], - }), - makeCase({ - id: 'home-toggle-online', - contract: [ - 'App name: Agent Device Tester', - 'Current screen: Home tab', - 'testID=toggle-online', - 'visible badge text after disabling: Offline', - ], - task: 'Assume Agent Device Tester is open on Home. Plan the commands to toggle Lab online off and verify the Offline badge is visible.', - outputs: [/toggle-online/i, /Offline/i], - }), - makeCase({ - id: 'catalog-search-debounce', - contract: [ - 'App name: Agent Device Tester', - 'Current screen: Catalog tab', - 'testID=catalog-search', - 'Search should respect debounce timing', - ], - task: 'Assume Agent Device Tester is on the Catalog tab. Plan the commands to fill the search field with "tart" using --delay-ms to respect the debounce, then wait for results to update.', - outputs: [/catalog-search/i, /--delay-ms/i, plannedCommand('wait')], - }), - makeCase({ - id: 'catalog-filter-bakery', - contract: [ - 'App name: Agent Device Tester', - 'Current screen: Catalog tab', - 'category chip: category-bakery', - 'visible product after filtering: Berry Tart', - ], - task: 'Assume Agent Device Tester is on the Catalog tab. Plan the commands to select the Bakery category and verify Berry Tart is visible.', - outputs: [/(?:category-bakery|Bakery)/i, /Berry Tart/i], - }), - makeCase({ - id: 'catalog-favorite-toggle', - contract: [ - 'App name: Agent Device Tester', - 'Current screen: Catalog tab', - 'testID=favorite-citrus-kit', - 'label after toggling favorite: Saved', - ], - task: 'Assume Agent Device Tester is on the Catalog tab. Plan the commands to toggle favorite for Citrus Starter Kit and verify the label changes to Saved.', - outputs: [/favorite-citrus-kit/i, /Saved/i], - }), - makeCase({ - id: 'catalog-add-to-cart', - contract: [ - 'App name: Agent Device Tester', - 'Current screen: Catalog tab', - 'testID=add-pepper-mix', - 'visible text after add: In cart: 1', - ], - task: 'Assume Agent Device Tester is on the Catalog tab. Plan the commands to add Pepper Mix to the cart and verify the card shows In cart: 1.', - outputs: [/add-pepper-mix/i, /In cart: 1/i], - }), - makeCase({ - id: 'catalog-scroll-footer', - contract: [ - 'App name: Agent Device Tester', - 'Current screen: Catalog tab', - 'testID=catalog-footer', - 'footer visible text: Seasonal footer target', - ], - task: 'Assume Agent Device Tester is on the Catalog tab. Plan the commands to scroll to the Seasonal footer target card using the scroll command.', - outputs: [plannedCommand('scroll'), /(?:catalog-footer|Seasonal footer|down)/i], - forbiddenOutputs: [/scrollintoview/i], - }), - makeCase({ - id: 'product-open-details', - contract: [ - 'App name: Agent Device Tester', - 'Current screen: Catalog tab', - 'testID=details-citrus-kit', - 'Product detail screen has testID=product-title', - ], - task: 'Assume Agent Device Tester is on the Catalog tab. Plan the commands to open Citrus Starter Kit details and verify the product title is visible.', - outputs: [/details-citrus-kit/i, /product-title/i], - }), - makeCase({ - id: 'product-quantity', - contract: [ - 'App name: Agent Device Tester', - 'Current screen: product detail', - 'testID=quantity-increase', - 'testID=quantity-decrease', - 'testID=quantity-value', - ], - task: 'Assume Agent Device Tester is already on a product detail screen. Plan the commands to increase quantity once, decrease it once, and get the quantity value through the durable quantity-value id rather than ambiguous visible number text.', - outputs: [ - /quantity-increase/i, - /quantity-decrease/i, - plannedCommand('get text'), - /id=(?:["']quantity-value["']|quantity-value)/i, - ], - forbiddenOutputs: [/get text ['"]?2['"]?/i, /wait text ['"]?2['"]?/i, /label=["']2["']/i], - }), - makeCase({ - id: 'product-note-append', - contract: [ - 'App name: Agent Device Tester', - 'Current screen: product detail', - 'testID=product-note', - 'Use append semantics rather than replacement', - ], - task: 'Assume Agent Device Tester is already on a product detail screen. Plan the commands to append "Handle with care" to the product note using press + type (not fill).', - outputs: [/product-note/i, plannedCommand('press'), plannedCommand('type')], - forbiddenOutputs: [plannedCommand('fill'), /(?:^|\n)(?:agent-device\s+)?type\s+@/i], - }), - makeCase({ - id: 'product-save-to-cart', - contract: [ - 'App name: Agent Device Tester', - 'Current screen: product detail', - 'testID=product-save', - 'toast text after saving: Cart updated', - ], - task: 'Assume Agent Device Tester is already on a product detail screen. Plan the commands to press Save to cart and verify the Cart updated toast appears.', - outputs: [/product-save/i, /Cart updated/i], - }), - makeCase({ - id: 'form-validation-errors', - contract: [ - 'App name: Agent Device Tester', - 'Current screen: Checkout form tab', - 'testID=submit-order', - 'validation errors card uses testID=form-errors', - ], - task: 'Assume Agent Device Tester is on the Checkout form tab. Plan the commands to submit with empty fields and verify the validation errors card is visible.', - outputs: [/submit-order/i, /form-errors/i], - }), - makeCase({ - id: 'form-success-submit', - contract: [ - 'App name: Agent Device Tester', - 'Current screen: Checkout form tab', - 'testID=field-name', - 'testID=field-email', - 'testID=checkbox-agree', - 'success card uses testID=form-success', - ], - task: 'Assume Agent Device Tester is on the Checkout form tab. Plan the commands to fill name and email, check order confirmation, submit, and verify the Order summary card is visible.', - outputs: [/field-name/i, /field-email/i, /checkbox-agree/i, /form-success/i], - }), - makeCase({ - id: 'form-keyboard-dismiss-ios-fallback', + id: 'settle-diff-is-observation', contract: [ 'App name: Agent Device Tester', - 'Platform: iOS simulator', - 'Current screen: Checkout form tab', - 'testID=field-name', - 'keyboard dismiss already returned UNSUPPORTED_OPERATION', - 'Visible text "Order summary" belongs to a tappable accordion', - 'testID=submit-order is partly occluded, and pressing it would submit the form', - 'No app-provided keyboard close control is available', + 'Previous command output is from agent-device, not a task description', + 'The task was to confirm the feed-search UI and close the session', ], - task: 'Decide whether there is a safe app-agnostic fallback that can hide the keyboard.', - finalOutputInstructions: - 'Final output: one concise prose sentence stating the safe outcome. Do not output commands or Markdown.', - outputs: [/(?:keyboard dismissal is unavailable|cannot safely dismiss the keyboard)/i], + task: quiz( + SETTLE_TAIL_SAMPLE, + 'Use the output already shown to determine whether the feed-search UI is present, then close the session. What command should run next?', + ), + outputs: [plannedCommand('close')], forbiddenOutputs: [ - plannedCommand('keyboard dismiss'), - plannedCommand('back'), + plannedCommand('snapshot'), + plannedCommand('wait'), + plannedCommand('find'), + plannedCommand('get'), + plannedCommand('is'), plannedCommandAlternatives(['press', 'click']), - /agent-device/i, - /(?:press|click)[^\n]*(?:Order summary|submit-order)/i, ], + allowOnlyLocalCliHelpCommands: true, + requireLocalCliHelp: true, }), makeCase({ - id: 'form-keyboard-dismiss-ios-app-control-fallback', + id: 'sample-output-settled-diff-next-target', contract: [ 'App name: Agent Device Tester', - 'Platform: iOS simulator', - 'Current screen: Checkout form tab with the keyboard visible', - 'keyboard dismiss already returned UNSUPPORTED_OPERATION', - 'The focused field accessory exposes a Done control', - 'Visible text "Order summary" belongs to a tappable accordion', - 'testID=submit-order is occluded while the keyboard is visible', + 'Previous command output is from agent-device, not a task description', + 'Need to continue from the settled diff without taking another snapshot', ], - task: 'Plan the safe fallback commands to hide the keyboard and prove that the previously occluded Submit order target became visible.', + task: quiz( + SETTLE_DIFF_SAMPLE, + 'The task is to open the matching account result. What command should run next?', + ), outputs: [ - plannedCommandAlternatives(['press', 'click']), - plannedCommand('is visible'), - /(?:press|click)[^\n]*Done[\s\S]*is visible[^\n]*submit-order/i, + plannedCommandAlternatives(['press', 'click', 'tap']), + /@e64(?:~s\d+)?\b|(?:label|text)=(?:["']@callstack\.com["']|@callstack\.com)/i, + /--settle\b/i, ], forbiddenOutputs: [ - plannedCommand('keyboard dismiss'), - plannedCommand('back'), - /(?:press|click)[^\n]*Order summary/i, + plannedCommand('snapshot'), + plannedCommand('wait stable'), + plannedCommand('fill'), + RAW_COORDINATE_TARGET, ], + allowOnlyLocalCliHelpCommands: true, + requireLocalCliHelp: true, }), makeCase({ - id: 'android-keyboard-dismiss-back-when-allowed', + id: 'sample-output-not-settled-needs-observe', contract: [ 'App name: Agent Device Tester', - 'Platform: Android', - 'Current screen: Checkout form tab with the keyboard visible', - 'testID=field-name', - 'keyboard dismiss already returned UNSUPPORTED_OPERATION because the current IME requires navigation to close', - 'Moving one level toward the previous UI state is acceptable for this task', + 'Previous command output is from agent-device, not a task description', + 'The next target is unknown because no settled tree was printed', + 'Old refs may be stale after the mutation', ], - task: 'Plan the safe fallback commands to hide the keyboard and verify the Full name field remains visible.', - outputs: [ - plannedCommand('back'), - plannedCommand('is visible'), - /back[\s\S]*is visible[\s\S]*field-name/i, + task: quiz( + NOT_SETTLED_SAMPLE, + 'The next target is not known yet. What command should run next?', + ), + outputs: [/(?:^|\n)(?:agent-device\s+)?(?:wait\b|snapshot\b[^\n]*-i\b)/i], + forbiddenOutputs: [ + /(?:^|\n)(?:agent-device\s+)?(?:press|click|fill|longpress)\s+@e\d+/i, + RAW_COORDINATE_TARGET, + SHELL_OUTPUT_PROJECTION, ], - forbiddenOutputs: [plannedCommand('keyboard dismiss'), RAW_COORDINATE_TARGET], + allowOnlyLocalCliHelpCommands: true, + requireLocalCliHelp: true, }), makeCase({ - id: 'form-keyboard-dismiss-ios-done-control', + id: 'sample-output-private-ax-recovery-continues', contract: [ 'Platform: iOS', - 'App name: Agent Device Tester', - 'Current screen: Checkout form tab', - 'testID=field-name', - 'The focused field shows an iOS keyboard toolbar with a visible Done control', - ], - task: 'Plan the commands to focus the Full name field and dismiss the iOS keyboard without manually pressing Done.', - outputs: [/field-name/i, /keyboard dismiss/i], - forbiddenOutputs: [plannedCommand('back'), /press\s+.*Done/i, /click\s+.*Done/i], - }), - makeCase({ - id: 'form-reset', - contract: [ - 'App name: Agent Device Tester', - 'Current screen: Checkout form tab', - 'testID=reset-form', - 'Full name field can be checked through id="field-name" or id="full-name"', - 'Validation errors can be checked through id="form-errors", id="full-name-error", or visible text "Required"', - 'toast text after reset: Form cleared', - ], - task: 'Assume Agent Device Tester is on the Checkout form tab after validation errors. Plan the commands to press Reset form, verify the Form cleared toast appears, verify validation errors are hidden, and verify the Full name field state is cleared.', - outputs: [ - /reset-form/i, - /Form cleared/i, - /(?:form-errors|full-name-error|Required)/i, - /(?:field-name|full-name)/i, - ], - }), - makeCase({ - id: 'settings-toggle-preferences', - contract: [ - 'App name: Agent Device Tester', - 'Current screen: Settings tab', - 'testID=toggle-notifications', - 'testID=toggle-reduced-motion', - ], - task: 'Assume Agent Device Tester is on the Settings tab. Plan the commands to toggle Push notifications and Reduced motion.', - outputs: [/toggle-notifications/i, /toggle-reduced-motion/i], - }), - makeCase({ - id: 'settings-diagnostics-error', - contract: [ - 'App name: Agent Device Tester', - 'Current screen: Settings tab', - 'testID=load-diagnostics', - 'error panel uses testID=diagnostics-error', - ], - task: 'Assume Agent Device Tester is on the Settings tab. Plan the commands to load diagnostics, wait for the error state, and verify the diagnostics error panel is visible.', - outputs: [/load-diagnostics/i, /diagnostics-error/i], - }), - makeCase({ - id: 'settings-diagnostics-retry', - contract: [ - 'App name: Agent Device Tester', - 'Current screen: Settings tab', - 'testID=load-diagnostics', - 'testID=retry-diagnostics', - 'ready state uses testID=diagnostics-ready', - ], - task: 'Assume Agent Device Tester is on the Settings tab. Plan the commands to load diagnostics, wait for the error state, retry diagnostics, then verify the Ready badge is visible.', - outputs: [/load-diagnostics/i, /retry-diagnostics/i, /diagnostics-ready/i], - }), - makeCase({ - id: 'settings-reset-alert', - contract: [ - 'App name: Agent Device Tester', - 'Current screen: Settings tab', - 'testID=reset-lab', - 'native alert title: Reset Agent Device Tester?', - ], - task: 'Assume Agent Device Tester is on the Settings tab. Plan the commands to trigger Reset lab state, then accept the native alert using alert wait + alert accept.', - outputs: [/reset-lab/i, plannedCommand('alert wait'), plannedCommand('alert accept')], - }), - makeCase({ - id: 'home-accessibility-audit', - contract: [ - 'App name: Agent Device Tester', - 'Current screen: Home tab', - 'Compare visible UI with the accessibility tree', - ], - task: 'Assume Agent Device Tester is on Home. Plan the commands to capture a screenshot and a snapshot to compare visible UI vs accessibility tree.', - outputs: [/screenshot/i, /snapshot/i], - }), -]; - -const SKILL_GUIDANCE_CASES: Case[] = [ - makeCase({ - id: 'web-first-use-runs-managed-setup', - contract: [ - 'Platform: Web', - 'URL to verify: https://example.com', - 'No web backend has been set up in this state directory yet', - 'Web automation uses the managed backend through agent-device, not direct backend commands', - ], - task: 'Plan the first-run commands to set up web automation, open the URL, inspect interactive refs, and close the session.', - outputs: [ - /(?:^|\n)(?:agent-device\s+)?web\s+setup\b[\s\S]*(?:^|\n)(?:agent-device\s+)?open\s+["']?https:\/\/example\.com["']?\s+--platform\s+web/i, - /(?:^|\n)(?:agent-device\s+)?snapshot\s+-i\s+--platform\s+web\b/i, - /(?:^|\n)(?:agent-device\s+)?close\s+--platform\s+web\b/i, - ], - forbiddenOutputs: [/agent-browser/i, /(?:^|\n)\s*(?:npm|pnpm|npx)\b/i], - strictFinalOutput: true, - }), - makeCase({ - id: 'web-minimal-browser-loop', - contract: [ - 'Platform: web', - 'Target URL: https://example.com/login', - 'agent-device web setup already passed', - 'Fresh interactive snapshot will expose @e12 as the email field and @e13 as the Continue button', - 'Expected success text after submit: Welcome', - 'Visual evidence path: ./artifacts/web-login.png', + 'Previous command output is from agent-device, not a task description', + 'The fallback snapshot still exposed an actionable Search button', + 'Need to open Search and observe the resulting UI', ], - task: 'Plan the minimal agent-device web commands to open the page, inspect interactive refs, fill the email field, click Continue, wait for the success text, capture a screenshot, and close.', + task: quiz( + PRIVATE_AX_RECOVERY_SAMPLE, + 'Treat the recovery warning as a warning, not a fatal error. Use the exposed Search button. What command should run next?', + ), outputs: [ - /(?:^|\n)(?:agent-device\s+)?open\s+https:\/\/example\.com\/login\b[^\n]*--platform\s+web/i, - /(?:^|\n)(?:agent-device\s+)?snapshot\b[^\n]*-i\b[^\n]*--platform\s+web/i, - /(?:^|\n)(?:agent-device\s+)?fill\s+@e12\s+["']?[^"'\n]+@[^"'\n]+["']?[^\n]*--platform\s+web/i, - /(?:^|\n)(?:agent-device\s+)?(?:click|press)\s+@e13\b[^\n]*--platform\s+web/i, - /(?:^|\n)(?:agent-device\s+)?wait\s+text\s+["']Welcome["'][^\n]*--platform\s+web/i, - /(?:^|\n)(?:agent-device\s+)?screenshot\s+\.\/artifacts\/web-login\.png\b[^\n]*--platform\s+web/i, - /(?:^|\n)(?:agent-device\s+)?close\b[^\n]*--platform\s+web/i, + plannedCommandAlternatives(['press', 'click', 'tap']), + /@e5\b|label=(?:["']Search["']|Search)/i, + /(?:--settle\b|(?:^|\n)(?:agent-device\s+)?snapshot\b[^\n]*-i\b)/i, ], forbiddenOutputs: [ - /agent-browser/i, - plannedCommand('boot'), - plannedCommand('apps'), - plannedCommand('install'), - plannedCommand('alert'), - plannedCommand('keyboard'), - plannedCommand('react-devtools'), - ], - strictFinalOutput: true, - allowOnlyLocalCliHelpCommands: true, - }), - makeCase({ - id: 'web-network-dump', - contract: [ - 'Platform: web', - 'Target URL: https://example.com/account', - 'agent-device web setup already passed', - 'Need recent browser network requests with headers after loading the page', - ], - task: 'Plan the agent-device commands to open the web page, inspect recent browser network requests with headers, and close.', - outputs: [ - /(?:^|\n)(?:agent-device\s+)?open\s+https:\/\/example\.com\/account\b[^\n]*--platform\s+web/i, - /(?:^|\n)(?:agent-device\s+)?network\s+dump\b[^\n]*(?:--include\s+headers|\bheaders\b)[^\n]*--platform\s+web/i, - /(?:^|\n)(?:agent-device\s+)?close\b[^\n]*--platform\s+web/i, - ], - forbiddenOutputs: [/agent-browser/i, plannedCommand('logs')], - strictFinalOutput: true, - allowOnlyLocalCliHelpCommands: true, - }), - makeCase({ - id: 'web-fixed-layout-viewport-and-fullshot', - contract: [ - 'Platform: web', - 'Target URL: https://example.com/app', - 'agent-device web setup already passed', - 'The app uses a fixed 100vh layout, so a taller viewport is needed before taking evidence screenshots', - 'Visual evidence path: ./artifacts/web-app.png', - ], - task: 'Plan the agent-device commands to open the web app, resize the viewport to 1280x900, capture a full-page screenshot, and close.', - outputs: [ - /(?:^|\n)(?:agent-device\s+)?open\s+https:\/\/example\.com\/app\b[^\n]*--platform\s+web/i, - /(?:^|\n)(?:agent-device\s+)?viewport\s+1280\s+900\b[^\n]*--platform\s+web/i, - /(?:^|\n)(?:agent-device\s+)?screenshot\s+\.\/artifacts\/web-app\.png\b[^\n]*(?:--fullscreen|--full|-f)[^\n]*--platform\s+web/i, - /(?:^|\n)(?:agent-device\s+)?close\b[^\n]*--platform\s+web/i, - ], - forbiddenOutputs: [/agent-browser/i, /--full-page\b/i], - strictFinalOutput: true, - allowOnlyLocalCliHelpCommands: true, - }), - makeCase({ - id: 'inspect-visible-text-readonly', - contract: [ - 'App name: Agent Device Tester', - 'Current screen: Home tab', - 'visible status badge text: Online', - 'No interaction is needed to answer this task', - ], - task: 'Plan the minimal read-only command to verify whether the Online badge is visible. Do not request interactive refs or mutate the UI.', - outputs: [/(?:^|\n)(?:agent-device\s+)?(?:snapshot|is|find)(?:\s|$)/i, /Online/i], - forbiddenOutputs: [ - /snapshot -i/i, - plannedCommand('click'), - plannedCommand('fill'), - plannedCommand('press'), - ], - }), - makeCase({ - id: 'target-ref-after-interactive-snapshot', - contract: [ - 'App name: Agent Device Tester', - 'Current screen: Home tab', - 'The target control has no stable selector in the task context', - 'Fresh interactive snapshot will expose the target as @e12', - ], - task: 'Plan the commands to capture fresh interactive refs, press the target control with its @e12 ref, then verify the nearby change with diff snapshot -i.', - outputs: [ - /snapshot -i/i, - /(?:^|\n)(?:agent-device\s+)?(?:click|press)\s+@e12\b/i, - /(?:diff snapshot -i|snapshot\b.*(?:-i\b.*--diff|--diff\b.*-i\b))/i, - ], - forbiddenOutputs: [RAW_COORDINATE_TARGET, /\btestID=/i], - }), - makeCase({ - id: 'post-type-refresh-interactive-refs', - contract: [ - 'App name: Agent Device Tester', - 'Current screen: Chat composer', - 'The previous command typed into a focused text input', - 'The keyboard appeared, so previous @e refs are stale', - 'The Send control has no stable selector in the task context', - 'Fresh interactive snapshot will expose Send as @e9', - 'Submitted text to verify: sent with agent-device v0.15', - 'Use wait/find for that exact text or diff snapshot -i; do not use a plain full snapshot for verification', - ], - task: 'Plan the next commands to refresh only current interactive refs, press Send, then verify the message was submitted.', - outputs: [ - /snapshot -i/i, - /(?:^|\n)(?:agent-device\s+)?(?:press|click)\s+@e9\b/i, - /(?:diff snapshot -i|wait\b.*sent with agent-device v0\.15|find\b.*sent with agent-device v0\.15|snapshot\b.*-i)/i, - ], - forbiddenOutputs: [ - /(?:^|\n)(?:agent-device\s+)?snapshot(?![^\n]*\s-i\b)/i, - plannedCommand('screenshot'), - RAW_COORDINATE_TARGET, - /(?:^|\n)(?:agent-device\s+)?(?:press|click)\s+@e(?!9\b)\d+\b/i, - ], - strictFinalOutput: true, - allowOnlyLocalCliHelpCommands: true, - }), - makeCase({ - id: 'ios-disabled-row-raw-rect-fallback', - contract: [ - 'Platform: iOS simulator', - 'Current screen: Orders list', - 'Fresh interactive snapshot shows @e12 [disabled] hittable:false label "Order #1042"', - 'press @e12 already returned success, but diff snapshot showed no navigation', - 'Raw JSON rect center for @e12 is x=196 y=318', - ], - task: 'Plan the fallback commands to inspect raw snapshot rects, press the row center, then verify the nearby change.', - outputs: [ - RAW_RECT_SNAPSHOT, - /(?:^|\n)(?:agent-device\s+)?(?:press|click)\s+196\s+318\b/i, - /(?:diff snapshot|snapshot\b.*--diff)/i, - ], - forbiddenOutputs: [/(?:^|\n)(?:agent-device\s+)?(?:press|click)\s+@e12\b/i, /scrollintoview/i], - }), - makeCase({ - id: 'truncated-text-input-scope-ref', - contract: [ - 'App name: Agent Device Tester', - 'Current screen: Checkout form tab', - 'Fresh interactive snapshot already shows @e7 [textinput] "Delivery instructions" [preview:"Leave at side gate..." truncated]', - 'Need the full text value of that truncated input before deciding whether to edit it', - ], - task: 'Plan the command to expand the truncated Delivery instructions text input using the current @e7 ref.', - outputs: [ - plannedCommand('snapshot'), - /(?:^|\n)(?:agent-device\s+)?snapshot\b.*(?:-s|--scope)\s+@e7\b/i, - ], - forbiddenOutputs: [ - /snapshot --raw/i, - plannedCommand('get'), - plannedCommand('fill'), - plannedCommand('type'), - plannedCommand('press'), - RAW_COORDINATE_TARGET, - ], - }), - makeCase({ - id: 'target-selector-for-durable-field', - contract: [ - 'App name: Agent Device Tester', - 'Current screen: Catalog tab', - 'Durable selector: id="catalog-search"', - 'Search should respect debounce timing', - ], - task: 'Plan the commands to fill the catalog search field through the durable id selector with "tart" using --delay-ms, then wait for results.', - outputs: [ - plannedCommand('fill'), - /id=(?:["']catalog-search["']|catalog-search)/i, - /--delay-ms/i, - plannedCommand('wait'), - ], - forbiddenOutputs: [ - RAW_COORDINATE_TARGET, - /(?:^|\n)(?:agent-device\s+)?type\s+@/i, - /--selector\b/i, - /--text\b/i, - ], - }), - makeCase({ - id: 'network-search-settle-then-wait', - contract: [ - 'App name: Agent Device Tester', - 'Current screen: Catalog search tab', - 'Search field selector: id="catalog-search"', - 'Search results are network-backed and may arrive after the local UI has gone quiet', - 'Expected result text: Alpenglow Trail Mug', - ], - task: 'Plan commands to search for "alpenglow" with the durable field selector, use --settle for the fill action, then wait for the network-backed result text. Do not verify this by taking repeated snapshots.', - outputs: [ - plannedCommand('fill'), - /id=(?:["']catalog-search["']|catalog-search)/i, - /alpenglow/i, - /--settle\b/i, - plannedCommand('wait text'), - /Alpenglow Trail Mug/i, - ], - forbiddenOutputs: [ - SHELL_OUTPUT_PROJECTION, - /(?:^|\n)(?:agent-device\s+)?snapshot\b[\s\S]*(?:^|\n)(?:agent-device\s+)?snapshot\b/i, - RAW_COORDINATE_TARGET, - ], - }), - makeCase({ - id: 'raw-output-before-shell-projection', - contract: [ - 'App name: Agent Device Tester', - 'Current screen: Home tab', - 'The target control has no stable selector in the task context', - 'Fresh interactive snapshot will expose the target as @e12', - 'agent-device output includes refs, warnings, hints, and diagnostics needed for the next step', - ], - task: 'Plan robust commands to inspect interactive refs, press the discovered @e12 target, then verify the nearby change. Do not pipe, grep, jq, or hide command output while exploring.', - outputs: [ - /snapshot -i/i, - /(?:^|\n)(?:agent-device\s+)?(?:press|click)\s+@e12\b/i, - /(?:diff snapshot -i|snapshot\b.*(?:-i\b.*--diff|--diff\b.*-i\b)|wait\b|find\b)/i, - ], - forbiddenOutputs: [SHELL_OUTPUT_PROJECTION, RAW_COORDINATE_TARGET], - strictFinalOutput: true, - allowOnlyLocalCliHelpCommands: true, - }), - makeCase({ - id: 'advisory-device-status-no-daemon', - contract: [ - 'Another worktree may have an advisory claim on Android emulator-5554', - 'Need to inspect local ownership without starting or contacting any daemon', - 'Claims are observational only: device release is unavailable; device status --stale only inspects proven-stale records and does not clean them', - ], - task: 'Plan the one command that inspects this Android device claim directly. Do not use ps, daemon commands, or an unavailable release command.', - outputs: [plannedCommand('device status --platform android --serial emulator-5554')], - forbiddenOutputs: [/\bdaemon\b/i, /\bdevice\s+release\b/i, /\bps\b/i], - strictFinalOutput: true, - }), - makeCase({ - id: 'selector-role-filter-not-role-key', - contract: [ - 'App name: Agent Device Tester', - 'Current screen: Feed directory', - 'Fresh interactive snapshot shows @e37 [button] "Search for more feeds"', - 'If a role filter is needed, the supported selector shape is role=button label="Search for more feeds"', - 'button=, placeholder=, index=, and key= are not valid selector keys', - 'Need the resulting UI after pressing this control', - ], - task: 'Plan the command to press Search for more feeds with settle. Prefer the visible @e37 ref or a valid selector; do not invent a role-name selector key.', - outputs: [ - plannedCommandAlternatives(['press', 'click']), - /(?:@e37|label=(?:["']Search for more feeds["']|Search for more feeds)|role=button)/i, - /--settle\b/i, - ], - forbiddenOutputs: [UNSUPPORTED_SELECTOR_KEY, RAW_COORDINATE_TARGET], - strictFinalOutput: true, - allowOnlyLocalCliHelpCommands: true, - }), - makeCase({ - id: 'settle-diff-is-observation', - contract: [ - 'App name: Agent Device Tester', - 'Previous command: press @e37 --settle', - 'The settled diff already showed + @e64 [text-field] "Search"', - 'The settled diff already showed + @e65 [text] "Recent searches"', - 'The task was to confirm the feed-search UI and close the session', - 'No more refs or visible evidence are needed', - ], - task: 'Use the previous command output to determine whether the task is complete, then plan the next command.', - outputs: [plannedCommand('close')], - forbiddenOutputs: [ - plannedCommand('snapshot'), - plannedCommand('wait'), - plannedCommand('find'), - plannedCommand('get'), - plannedCommand('is'), - plannedCommandAlternatives(['press', 'click']), - ], - allowOnlyLocalCliHelpCommands: true, - requireLocalCliHelp: true, - }), - makeCase({ - id: 'sample-output-settled-diff-next-target', - contract: [ - 'App name: Agent Device Tester', - 'Previous command output is from agent-device, not a task description', - 'Need to continue from the settled diff without taking another snapshot', - 'Need to open the matching account result', - 'Need the resulting account UI from the same command, without a separate observation', - ], - task: `Read this previous agent-device output, then plan the next command: - -agent-device fill 'id="account-search"' "callstack" --settle -Filled id="account-search" with "callstack" -settled:true refsGeneration: 12 -Changed: -+ @e64 [button] "@callstack.com" -+ @e65 [text] "Callstack" - -The task is to open the matching account result and return its resulting UI state in the same command. What command should run next?`, - outputs: [ - plannedCommandAlternatives(['press', 'click', 'tap']), - /@e64(?:~s12)?\b|(?:label|text)=(?:["']@callstack\.com["']|@callstack\.com)/i, - /--settle\b/i, - ], - forbiddenOutputs: [ - plannedCommand('snapshot'), - plannedCommand('wait stable'), - plannedCommand('fill'), - RAW_COORDINATE_TARGET, - ], - allowOnlyLocalCliHelpCommands: true, - requireLocalCliHelp: true, - }), - makeCase({ - id: 'sample-output-not-settled-needs-observe', - contract: [ - 'App name: Agent Device Tester', - 'Previous command output is from agent-device, not a task description', - 'The next target is unknown because no settled tree was printed', - 'Old refs may be stale after the mutation', - ], - task: `Read this previous agent-device output, then plan the next command: - -agent-device press @e12 --settle -Pressed @e12 -not settled after 10000ms -Hint: UI kept changing. Run agent-device wait stable or agent-device snapshot -i before the next ref-based action. - -What command should run next?`, - outputs: [/(?:^|\n)(?:agent-device\s+)?(?:wait\s+stable|snapshot\b[^\n]*-i\b)/i], - forbiddenOutputs: [ - /(?:^|\n)(?:agent-device\s+)?(?:press|click|fill|longpress)\s+@e\d+/i, - RAW_COORDINATE_TARGET, - SHELL_OUTPUT_PROJECTION, - ], - allowOnlyLocalCliHelpCommands: true, - requireLocalCliHelp: true, - }), - makeCase({ - id: 'sample-output-private-ax-recovery-continues', - contract: [ - 'Platform: iOS', - 'Previous command output is from agent-device, not a task description', - 'The fallback snapshot still exposed an actionable Search button', - 'Need to open Search and observe the resulting UI', - ], - task: `Read this previous agent-device output, then plan the next command: - -agent-device snapshot -i -Recovered this snapshot with the private-ax accessibility backend. -Detected overly complex accessibility tree. Falling back to another snapshot backend. -It's OK to continue. For more information, rerun with --debug. - -@e5 [button] "Search" -@e8 [tab] "Home" selected - -Treat the recovery message as a warning, not a fatal error. Use the exposed Search button.`, - outputs: [ - plannedCommandAlternatives(['press', 'click', 'tap']), - /@e5\b|label=(?:["']Search["']|Search)/i, - /(?:--settle\b|(?:^|\n)(?:agent-device\s+)?snapshot\b[^\n]*-i\b)/i, - ], - forbiddenOutputs: [ - /--debug|--verbose/i, - plannedCommand('screenshot'), - plannedCommand('close'), - plannedCommand('help'), - RAW_COORDINATE_TARGET, + /--debug|--verbose/i, + plannedCommand('screenshot'), + plannedCommand('close'), + plannedCommand('help'), + RAW_COORDINATE_TARGET, ], allowOnlyLocalCliHelpCommands: true, requireLocalCliHelp: true, }), - makeCase({ - id: 'text-replace-uses-fill', - contract: [ - 'App name: Agent Device Tester', - 'Current screen: Checkout form tab', - 'Field selector: id="field-email"', - 'Existing field value must be replaced', - ], - task: 'Plan the command to replace the Email field value with "qa@example.com".', - outputs: [ - plannedCommand('fill'), - /id=(?:["']field-email["']|field-email)/i, - /qa@example\.com/i, - ], - forbiddenOutputs: [plannedCommand('type'), /(?:^|\n)(?:agent-device\s+)?fill\s+\d+\s+\d+/i], - }), - makeCase({ - id: 'empty-fill-not-clear-field', - contract: [ - 'App name: Agent Device Tester', - 'Current screen: Catalog tab', - 'Search field selector: id="catalog-search"', - 'Visible clear control selector: id="clear-search"', - 'Need to clear the existing search text', - 'fill "" is not supported', - ], - task: 'Plan the supported command to clear the search field without using empty fill replacement.', - outputs: [ - /id=(?:["']clear-search["']|clear-search)/i, - plannedCommandAlternatives(['press', 'click']), - ], - forbiddenOutputs: [ - /fill\b[^\n]*(?:id=["']catalog-search["']|catalog-search)[^\n]*(?:""|''|\s$)/i, - plannedCommand('type'), - /\bclear\s+field\b/i, - ], - }), - makeCase({ - id: 'ios-allow-paste-prefill-only', - contract: [ - 'App name: Agent Device Tester', - 'Platform: iOS simulator', - 'App reads UIPasteboard.general when opened', - 'iOS Allow Paste system prompt is suppressed under XCUITest automation', - 'Need to test app behavior when pasteboard contains: some text', - ], - task: 'Plan commands to prefill the simulator pasteboard and open the app for paste-driven behavior. Do not try to automate the Allow Paste system dialog.', - outputs: [plannedCommand('clipboard'), /write/i, /some text/i, plannedCommand('open')], - forbiddenOutputs: [ - /Allow Paste/i, - /alert (?:wait|accept|dismiss)/i, - /\bxcrun\b/i, - /\bsimctl\b/i, - ], - }), - makeCase({ - id: 'offscreen-target-scroll-resnapshot', - contract: [ - 'App name: Agent Device Tester', - 'Current screen: Catalog tab', - 'Visible-first snapshot says [off-screen below] "Seasonal footer target"', - 'Off-screen refs are discovery hints, not actionable refs', - 'The target is below the viewport, not necessarily at the absolute bottom of the list', - 'Do not use scroll bottom for this task', - ], - task: 'Plan the commands to reach the Seasonal footer target from the off-screen summary, then refresh interactive refs before acting or verifying.', - outputs: [plannedCommand('scroll'), /\bdown\b/i, /snapshot -i/i], - forbiddenOutputs: [ - /scrollintoview/i, - /\bscroll\s+bottom\b/i, - /(?:^|\n)(?:agent-device\s+)?(?:click|press)\s+@\S+/i, - ], - strictFinalOutput: true, - allowOnlyLocalCliHelpCommands: true, - }), - makeCase({ - id: 'android-tv-focus-uses-tv-remote', - contract: [ - 'Platform: Android TV emulator', - 'Target selector is not currently focused', - 'TV apps are focus-first: move focus with D-pad/remote buttons before selecting', - 'Visual focus evidence is required before selecting', - 'Raw ADB keyevent is the old workaround; use agent-device command surface instead', - 'If you inspect CLI help, run it raw; do not pipe it through head, grep, jq, or tail', - 'Final answer must be agent-device command lines only, with no prose or introduction', - ], - task: 'Plan commands to move focus down twice, capture overlay-ref screenshot evidence, and select the focused Android TV control.', - outputs: [ - plannedCommand('tv-remote'), - /\bdown\b/i, - /screenshot\b[^\n]*--overlay-refs/i, - /\bselect\b/i, - ], - forbiddenOutputs: [ - /\badb\b/i, - /\bkeyevent\b/i, - /(?:^|\n)(?:agent-device\s+)?(?:press|click)\s+@(?:e\d+|ref)\b/i, - ], - strictFinalOutput: true, - allowOnlyLocalCliHelpCommands: true, - }), - makeCase({ - id: 'android-tv-remote-longpress-preset', - contract: [ - 'Platform: Android TV emulator', - 'The focused control is a TV remote target', - 'Use the agent-device TV remote command surface, not raw ADB keyevents', - 'For a held TV remote button, use the tv-remote longpress preset', - 'Final answer must be agent-device command lines only, with no prose or introduction', - ], - task: 'Plan the command to hold the focused Android TV select button with the default TV remote longpress preset.', - outputs: [plannedCommand('tv-remote'), /\blongpress\b/i, /\bselect\b/i], - forbiddenOutputs: [/\badb\b/i, /\bkeyevent\b/i, /--duration-ms/i, plannedCommand('longpress')], - strictFinalOutput: true, - allowOnlyLocalCliHelpCommands: true, - }), - makeCase({ - id: 'vega-tv-vvd-remote-lifecycle', - contract: [ - 'Platform: Amazon Vega OS TV', - 'Vega component ID: com.example.vega.main', - 'Vega device serial: VirtualDevice', - 'The Vega Virtual Device is not running yet', - 'Move focus right, then hold select for exactly 725 ms', - 'Vega capture, selectors, app inventory, and ADB are unsupported', - 'Initial Vega support is VVD-only; physical Fire TV is unsupported', - 'The VVD must be stopped after the agent-device session is closed', - 'If you inspect CLI help, run it raw; do not pipe it through head, grep, jq, or tail', - ], - task: 'Plan the complete local VVD lifecycle and Vega TV remote flow, including explicit device selection and cleanup.', - outputs: [ - plannedCommand('vega virtual-device start'), - /devices\b[^\n]*--platform\s+vega\b[^\n]*--target\s+tv/i, - /open\s+com\.example\.vega\.main\b/i, - /--serial\s+VirtualDevice\b/i, - /tv-remote\s+press\s+right\b/i, - /tv-remote\s+press\s+select\b[^\n]*--duration-ms\s+725\b/i, - /close\s+com\.example\.vega\.main\b/i, - plannedCommand('vega virtual-device stop'), - ], - forbiddenOutputs: [ - /\badb\b/i, - /\b(?:snapshot|screenshot|apps)\b/i, - /(?:^|\n)(?:agent-device\s+)?(?:press|click)\s+@/i, - /tv-remote\s+longpress/i, - /\bphysical\b/i, - ], - allowOnlyLocalCliHelpCommands: true, - finalOutputInstructions: ` -Final output: only commands, one per line, with no prose or Markdown. -Every final output line must start with agent-device or vega. -Do not combine final commands with shell operators such as &&, ||, pipes, or semicolons. -`.trim(), - }), - makeCase({ - id: 'ios-composite-horizontal-tabs-coordinate-fallback', - contract: [ - 'Platform: iOS simulator', - 'Current screen: Catalog filters', - 'Horizontal filter tabs are collapsed into one [seekbar] in snapshot -i', - 'The individual Bakery tab has no @ref or selector on iOS', - 'Raw JSON plus visual inspection gives Bakery center x=84 y=220', - ], - task: 'Plan commands to handle the missing child refs by inspecting raw rects, tapping the Bakery center, and verifying the selected filter changed.', - outputs: [ - RAW_RECT_SNAPSHOT, - /(?:^|\n)(?:agent-device\s+(?:--platform\s+ios\s+)?)?(?:press|click)\s+84\s+220\b/i, - /(?:diff snapshot -i|snapshot\b.*(?:-i\b.*--diff|--diff\b.*-i\b)|snapshot\b.*-i|Berry Tart|Bakery)/i, - ], - forbiddenOutputs: [ - /(?:^|\n)(?:agent-device\s+)?(?:press|click)\s+@(?:e\d+|ref)\b/i, - /scrollintoview/i, - ], - }), - makeCase({ - id: 'list-text-presence-prefers-wait-text', - contract: [ - 'App name: Agent Device Tester', - 'Current screen: Catalog tab', - 'List content loads asynchronously', - 'Expected list item text: Trip ideas', - 'No interaction is needed; this is a presence check for visible text in a list', - ], - task: 'Plan the robust read-only command to wait for the Trip ideas list text to appear.', - outputs: [plannedCommand('wait'), /Trip ideas/i], - forbiddenOutputs: [plannedCommand('is visible'), /snapshot -i/i, plannedCommand('press')], - }), - makeCase({ - id: 'navigation-back-in-app', - contract: [ - 'App name: Agent Device Tester', - 'Current screen: product detail', - 'Goal: return to the Catalog tab through normal app navigation', - 'No visible Catalog tab selector or nav-back selector is currently exposed', - ], - task: 'Plan the command to go back to Catalog using the app-owned back command.', - outputs: [plannedCommand('back')], - forbiddenOutputs: [/back\s+--system/i], - }), - makeCase({ - id: 'navigation-back-ambiguous-use-visible-nav', - contract: [ - 'App name: Agent Device Tester', - 'Current screen: nested product detail', - 'Previous back command navigated to an unexpected screen', - 'Visible app nav button selector: id="nav-back"', - 'Goal: return one level inside the app, not trigger system back', - ], - task: 'Plan the next navigation command using the visible app-owned control instead of retrying back.', - outputs: [/nav-back/i, plannedCommandAlternatives(['press', 'click'])], - forbiddenOutputs: [plannedCommand('back'), /back\s+--system/i], - }), - makeCase({ - id: 'setup-unknown-app-discover-first', - contract: [ - 'Platform: Android', - 'Target app display name is known: Agent Device Tester', - 'Package id is unknown', - 'No app session is open yet', - ], - task: 'Plan the bootstrap commands to discover the correct Android device and app identifier, then open the discovered app.', - outputs: [plannedCommand('devices'), plannedCommand('apps'), plannedCommand('open')], - forbiddenOutputs: [/com\.agent\.device\.tester/i, /com\.example/i], - }), - makeCase({ - id: 'install-artifact-before-open', - contract: [ - 'Platform: Android', - 'Known artifact path: ./dist/agent-device-tester.apk', - 'Known package after install: com.callstack.agentdevicetester', - 'The task requires installing the artifact', - ], - task: 'Plan the commands to install the APK artifact, then open the installed package in a fresh runtime state.', - outputs: [ - plannedCommand('install'), - /\.\/dist\/agent-device-tester\.apk/i, - plannedCommand('open'), - /--relaunch/i, - ], - forbiddenOutputs: [/open\s+\.\/dist\/agent-device-tester\.apk/i], - }), - makeCase({ - id: 'install-from-github-artifact-before-open', - contract: [ - 'Platform: Android', - 'Install source: GitHub Actions artifact callstack/agent-device:agent-device-tester-apk', - 'Known package after install: com.callstack.agentdevicetester', - 'Remote daemon can resolve the artifact server-side', - ], - task: 'Plan commands to install from the GitHub Actions artifact, then open the installed package in fresh runtime state.', - outputs: [ - plannedCommand('install-from-source'), - /--github-actions-artifact\s+callstack\/agent-device:agent-device-tester-apk/i, - plannedCommand('open'), - /com\.callstack\.agentdevicetester/i, - /--relaunch/i, - ], - forbiddenOutputs: [ - /curl\b/i, - /gh\s+(?:run|artifact|download)/i, - /open\s+.*agent-device-tester-apk/i, - ], - }), - makeCase({ - id: 'hidden-info-do-not-force-ui', - contract: [ - 'App name: Agent Device Tester', - 'Current screen: Home tab', - 'Question: what is the hidden promo code?', - 'The current screen does not expose any promo code text or selector', - 'No interaction was requested', - ], - task: 'Plan the minimal read-only command to inspect exposed UI without typing, navigating, or mutating the app to reveal hidden information.', - outputs: [plannedCommand('snapshot')], - forbiddenOutputs: [ - plannedCommand('press'), - plannedCommand('click'), - plannedCommand('fill'), - plannedCommand('type'), - plannedCommand('open'), - ], - }), - makeCase({ - id: 'metro-reload-dev-loop', - contract: [ - 'App name: Agent Device Tester', - 'React Native dev build is already open and connected to Metro', - 'Only JavaScript changed', - ], - task: 'Plan the commands to reload the running app after the JS change, then verify the Home screen is visible.', - outputs: [ - /(?:^|\n)(?:agent-device\s+)?metro\s+reload(?:\s|$)/i, - /(?:^|\n)(?:agent-device\s+)?(?:snapshot\b|find\b[^\n]*Home|is\b[^\n]*Home|wait\b[^\n]*Home)/i, - ], - forbiddenOutputs: [/open\b.*--relaunch/i, /(?:^|\n)(?:agent-device\s+)?screenshot\b/i], - }), - makeCase({ - id: 'ios-rn-two-worktrees-one-native-build', - contract: [ - 'App name: React Navigation Example', - 'Platform: iOS simulators', - 'The app is already installed on both simulators', - 'Worktree A Metro is already running on 127.0.0.1:8081', - 'Worktree B Metro is already running on 127.0.0.1:8082', - 'Use iPhone 17 for worktree A and iPhone 17 Pro for worktree B', - 'Use separate sessions rn-a and rn-b', - 'Do not rebuild, reinstall, or run package manager commands', - ], - task: 'Plan the agent-device commands to launch the same installed React Native iOS app against each worktree Metro instance and verify both sessions with interactive snapshots.', - outputs: [ - /open\s+["']?React Navigation Example["']?(?=[^\n]*--platform ios)(?=[^\n]*--device ["']?iPhone 17["']?)(?=[^\n]*--session rn-a)(?=[^\n]*--metro-host 127\.0\.0\.1)(?=[^\n]*--metro-port 8081)(?=[^\n]*--relaunch)/i, - /open\s+["']?React Navigation Example["']?(?=[^\n]*--platform ios)(?=[^\n]*--device ["']?iPhone 17 Pro["']?)(?=[^\n]*--session rn-b)(?=[^\n]*--metro-host 127\.0\.0\.1)(?=[^\n]*--metro-port 8082)(?=[^\n]*--relaunch)/i, - /snapshot -i(?=[^\n]*--platform ios)(?=[^\n]*--session rn-a)/i, - /snapshot -i(?=[^\n]*--platform ios)(?=[^\n]*--session rn-b)/i, - ], - forbiddenOutputs: [ - /run-ios/i, - /reinstall/i, - /install\b/i, - /yarn|pnpm|npm|npx/i, - /adb\s+reverse/i, - /--device "iPhone 17"[^\n]*--session rn-b/i, - /--device "iPhone 17 Pro"[^\n]*--session rn-a/i, - ], - strictFinalOutput: true, - allowOnlyLocalCliHelpCommands: true, - }), - makeCase({ - id: 'rn-warning-overlay-dismiss-before-tap', - contract: [ - 'App name: Agent Device Tester', - 'Current screen after opening will trigger console.warn', - 'Fresh interactive snapshot should show the React Native warning overlay', - 'Fresh interactive snapshot shows @e4 label="×" for the small close icon', - 'The warning overlay can obscure UI and intercept taps', - 'Target selector after dismissing overlay: id="submit-order"', - ], - task: 'Plan commands to identify the warning overlay in snapshot -i, dismiss it with the React Native overlay command, verify the overlay is gone with diff snapshot -i or a fresh snapshot -i, then press the submit target.', - outputs: [ - /snapshot -i[\s\S]*react-native\s+dismiss-overlay[\s\S]*(?:diff snapshot -i|snapshot\b.*-i)/i, - /submit-order/i, - ], - forbiddenOutputs: [ - plannedCommand('screenshot'), - RAW_COORDINATE_TARGET, - /(?:^|\n)(?:agent-device\s+)?(?:press|click)\b[^\n]*submit-order[\s\S]*(?:Dismiss|Close)/i, - /(?:^|\n)(?:agent-device\s+)?(?:press|click)\s+@e4\b/i, - /(?:press|click)\b[^\n]*(?:warning|LogBox)/i, - /alert accept/i, - ], - strictFinalOutput: true, - allowOnlyLocalCliHelpCommands: true, - }), - makeCase({ - id: 'rn-collapsed-warning-banner-expand-dismiss', - contract: [ - 'App name: Agent Device Tester', - 'Current screen has a collapsed React Native warning banner', - 'Fresh interactive snapshot shows @e12 label="!, Open debugger to view warnings."', - 'Dismiss is not visible until the warning banner is pressed', - 'The post-expansion Dismiss/Close ref is unknown in this prompt; use find or a label selector instead of inventing a new @e ref', - 'Target selector after dismissing overlay: id="submit-order"', - ], - task: 'Plan commands to dismiss the collapsed warning banner with the React Native overlay command, re-snapshot, then press id="submit-order". Do not stop after dismissing the overlay.', - outputs: [ - /(?:snapshot -i[\s\S]*)?react-native\s+dismiss-overlay[\s\S]*snapshot -i[\s\S]*submit-order/i, - ], - forbiddenOutputs: [ - plannedCommand('screenshot'), - RAW_COORDINATE_TARGET, - /(?:^|\n)(?:agent-device\s+)?(?:press|click)\b[^\n]*submit-order[\s\S]*(?:@e12|Dismiss|Close)/i, - /(?:^|\n)(?:agent-device\s+)?(?:press|click)\s+@e12\b/i, - /alert accept/i, - ], - strictFinalOutput: true, - allowOnlyLocalCliHelpCommands: true, - }), - makeCase({ - id: 'rn-error-overlay-human-flag', - contract: [ - 'App name: Agent Device Tester', - 'Platform: Android', - 'Fresh interactive snapshot shows a React Native error overlay', - 'Overlay controls include Dismiss and Reload JS', - 'React DevTools is connected', - 'The overlay is unrelated to the requested checkout task but should be reported', - 'Checkout target selector after dismissing overlay: id="submit-order"', - 'Need overlay screenshot evidence with refs before dismissing it', - ], - task: 'Plan commands to capture the error overlay using screenshot --overlay-refs, inspect React DevTools errors, dismiss only the unrelated overlay with the React Native overlay command, re-snapshot, then continue to id="submit-order".', - outputs: [ - /snapshot(?:\s+-i)?/i, - /screenshot\b[^\n]*--overlay-refs/i, - plannedCommand('react-devtools errors'), - plannedCommand('react-native dismiss-overlay'), - /(?:diff snapshot -i|snapshot(?:\s+-i)?)/i, - /submit-order/i, - ], - forbiddenOutputs: [ - RAW_COORDINATE_TARGET, - /(?:press|click)\b[^\n]*(?:Dismiss|Close|warning|LogBox|RedBox)/i, - /alert accept/i, - /ignore/i, - ], - }), - makeCase({ - id: 'rn-redbox-stack-dismiss-before-continuing', - contract: [ - 'App name: Agent Device Tester', - 'Platform: Android', - 'Fresh interactive snapshot shows a full-screen React Native RedBox stack trace', - 'Stack includes useOnyx.ts:80:43 and LHNOptionsList.tsx:77', - 'Overlay controls include Dismiss and Minimize', - 'The RedBox may be caused by an infinite render loop', - 'Target selector after dismissing overlay: id="submit-order"', - ], - task: 'Plan commands to recognize the RedBox stack trace, run the React Native overlay command so it dismisses the overlay, re-snapshot, then continue to id="submit-order" while reporting the RedBox later.', - outputs: [ - /snapshot -i[\s\S]*react-native\s+dismiss-overlay[\s\S]*snapshot -i[\s\S]*submit-order/i, - ], - forbiddenOutputs: [ - RAW_COORDINATE_TARGET, - /(?:press|click)\b[^\n]*(?:Dismiss|Minimize)/i, - /alert accept/i, - /failed nav|navigation failed/i, - ], - strictFinalOutput: true, - allowOnlyLocalCliHelpCommands: true, - }), - makeCase({ - id: 'expo-go-ios-project-url', - contract: [ - 'Platform: iOS simulator', - 'Launch context: Expo Go', - 'Project URL: exp://127.0.0.1:8081', - 'The native bundle id for the project is not installed separately', - 'The final command must include --platform ios', - ], - task: 'Plan the command to launch the Expo project in Expo Go on iOS without inventing a native bundle id.', - outputs: [IOS_EXPO_GO_OPEN, /--platform ios/i], - forbiddenOutputs: [ - /open\s+Agent Device Tester/i, - /host\.exp\.Exponent/i, - /com\.(?:callstack|example|agent)/i, - ], - }), - makeCase({ - id: 'expo-go-ios-runner-splash-retry-host-shell', - contract: [ - 'Platform: iOS simulator', - 'Launch context: Expo Go', - 'Project URL: exp://127.0.0.1:8081', - 'Previous command open exp://127.0.0.1:8081 returned Opened', - 'Fresh snapshot -i showed only the Agent Device Runner splash', - 'Expo Go is available as a host shell', - ], - task: 'Plan the next commands to recover by opening the project through Expo Go and verifying the app UI loaded.', - outputs: [IOS_EXPO_GO_OPEN, /snapshot -i/i], - forbiddenOutputs: [ - /open\s+Agent Device Runner/i, - /open\s+Agent Device Tester/i, - /com\.(?:callstack|example|agent)/i, - /host\.exp\.Exponent/i, - ], - }), - makeCase({ - id: 'expo-go-ios-after-app-id-miss', - contract: [ - 'Platform: iOS simulator', - 'Target app display name: Agent Device Tester', - 'Previous apps lookup did not list Agent Device Tester', - 'Previous apps lookup did list Expo Go', - 'Project URL: exp://127.0.0.1:8081', - ], - task: 'Plan the next command to launch the project after the app-id lookup miss without inventing a native bundle id.', - outputs: [IOS_EXPO_GO_OPEN], - forbiddenOutputs: [ - /open-url/i, - /open\s+Agent Device Tester/i, - /com\.(?:callstack|example|agent)/i, - /host\.exp\.Exponent/i, - ], - }), - makeCase({ - id: 'expo-go-android-url-only', - contract: [ - 'Platform: Android', - 'Launch context: Expo Go', - 'Project URL: exp://10.0.2.2:8081', - 'Android Expo URLs can be opened directly when no specific app package must be forced', - ], - task: 'Plan the command to launch the Expo project on Android using the project URL.', - outputs: [plannedCommand('open'), /exp:\/\/10\.0\.2\.2:8081/i, /--platform android/i], - forbiddenOutputs: [ - /open\s+(?:"Expo Go"|Expo\s+Go)\s+exp:\/\//i, - /--activity/i, - /host\.exp\.exponent/i, - ], - }), - makeCase({ - id: 'android-local-metro-reverse-before-url-open', - contract: [ - 'Platform: Android', - 'Launch context: Expo Go because the user provided an exp:// project URL', - 'Local Metro port: 8082', - 'Project URL: exp://127.0.0.1:8082', - 'Direct Android localhost URL opens auto-configure host reachability', - 'On Android, open the URL target directly; do not use the iOS host-plus-URL form with "Expo Go"', - 'Every agent-device command must target Android explicitly with --platform android', - 'Do not assume every React Native app is Expo; this one is Expo only because an exp:// URL was provided', - ], - task: 'Plan the explicit Android direct-URL commands to open the Expo project URL on local Metro port 8082 and verify the app UI with an interactive snapshot.', - outputs: [ - plannedCommand('open'), - /exp:\/\/127\.0\.0\.1:8082/i, - /--platform android/i, - /snapshot -i/i, - ], - forbiddenOutputs: [ - /adb\s+reverse/i, - /exp:\/\/10\.0\.2\.2:8082/i, - /open\s+(?:"Expo Go"|Expo\s+Go)\s+exp:\/\//i, - /com\.(?:expensify|agent|example)/i, - /--activity/i, - ], - }), - makeCase({ - id: 'debug-logs-short-window', - contract: [ - 'App name: Agent Device Tester', - 'Current screen: Settings tab', - 'Repro button selector: id="load-diagnostics"', - 'Need app logs only for the retry failure window', - ], - task: 'Plan the commands to clear and restart logs, mark the repro window, trigger diagnostics, and inspect the log path without dumping a whole stale log into context.', - outputs: [/logs clear --restart/i, /logs mark/i, /load-diagnostics/i, /logs path/i], - forbiddenOutputs: [/cat .*log/i, /tail -n \+1/i], - }), - makeCase({ - id: 'ios-simulator-open-launch-console', - contract: [ - 'App name: Agent Device Tester', - 'Platform: iOS simulator', - 'Need initial launch stdout/stderr in artifacts/launch-console.log', - 'After launch, verify the app UI loaded with an interactive snapshot', - ], - task: 'Plan commands for launching the iOS simulator app while capturing the initial launch console output, then verify the UI loaded.', - outputs: [ - /open\b[^\n]*Agent Device Tester[^\n]*--launch-console artifacts\/launch-console\.log/i, - /snapshot -i/i, - ], - forbiddenOutputs: [/simctl\b/i, /--console-pty/i, /\bsleep\s+\d+/i], - }), - makeCase({ - id: 'ios-simulator-open-device-hub', - contract: [ - 'App name: Agent Device Tester', - 'Platform: iOS simulator', - 'User explicitly wants Xcode Device Hub when surfacing the simulator', - 'After launch, verify the app UI loaded with an interactive snapshot', - ], - task: 'Plan commands for launching the iOS simulator app through Xcode Device Hub, then verify the UI loaded.', - outputs: [/open\b[^\n]*Agent Device Tester[^\n]*--device-hub/i, /snapshot -i/i], - forbiddenOutputs: [/--no-device-hub/i, /simctl\b/i, /\bsleep\s+\d+/i], - }), - makeCase({ - id: 'debug-network-session-dump', - contract: [ - 'App name: Agent Device Tester', - 'Current screen: Settings tab', - 'Diagnostics load triggers HTTP traffic logged by the app', - 'Repro button selector: id="load-diagnostics"', - 'Need request and response headers', - ], - task: 'Plan the commands to reproduce the diagnostics request and inspect recent session network traffic with headers.', - outputs: [ - /load-diagnostics/i, - plannedCommand('network'), - /(?:dump|log)/i, - /(?:--include\s+headers|\bheaders\b|network dump\b[^\n]*\ball\b)/i, - ], - forbiddenOutputs: [/logs path/i, /cat .*log/i], - }), - makeCase({ - id: 'debug-symbols-apple-crash', - contract: [ - 'Artifact: artifacts/crash.ips', - 'Build products directory: ./build', - 'Need a symbolicated crash artifact at artifacts/crash-symbolicated.ips', - 'Do not inspect or paste the full crash into context', - ], - task: 'Plan the command to symbolicate this Apple crash report with local debug symbols.', - outputs: [ - plannedCommand('debug'), - /symbols/i, - /--artifact\s+artifacts\/crash\.ips/i, - /--search-path\s+\.\/build/i, - /--out\s+artifacts\/crash-symbolicated\.ips/i, - ], - forbiddenOutputs: [ - plannedCommand('perf'), - plannedCommand('logs'), - /cat\s+artifacts\/crash\.ips/i, - /react-devtools/i, - ], - }), - makeCase({ - id: 'android-open-verify-ui', - contract: [ - 'App name: Agent Device Tester', - 'Platform: Android', - 'Package id: com.agentdevice.tester', - 'Expected loaded text: Agent Device Tester', - 'Need to verify the UI loaded after relaunch', - ], - task: 'Plan commands to open the Android package with a relaunch and verify the app UI loaded. Do not rely on screenshot-only verification.', - outputs: [ - plannedCommand('open'), - /com\.agentdevice\.tester/i, - /--relaunch/i, - /(?:wait(?:\s+text)?|is visible|find)\b[^\n]*Agent Device Tester/i, - ], - forbiddenOutputs: [plannedCommand('screenshot')], - }), - makeCase({ - id: 'android-action-sheet-document-scan-wait', - contract: [ - 'App name: Document Fixture', - 'Platform: Android', - 'Current screen: chat composer', - 'Action sheet trigger selector: id="composer-actions"', - 'Scan option text: Scan document', - 'Expected result text after upload: Document uploaded', - 'Android camera permission may appear as a runtime permission dialog', - ], - task: 'Plan commands to open the composer action sheet, choose Scan document, handle any visible permission prompt, and wait for the upload result.', - outputs: [ - /composer-actions/i, - /Scan document/i, - /alert (?:wait|accept)|Allow|snapshot -i/i, - /wait\b[^\n]*Document uploaded/i, - ], - forbiddenOutputs: [RAW_COORDINATE_TARGET, PSEUDO_ASSERTION_COMMAND, /settings permission/i], - }), - makeCase({ - id: 'evidence-screenshot-overlay-refs', - contract: [ - 'App name: Agent Device Tester', - 'Current screen: Catalog tab', - 'The bug report needs visual proof and tappable-region context for icon-only controls', - ], - task: 'Plan the command to capture screenshot evidence with current interactive ref overlays.', - outputs: [plannedCommand('screenshot'), /--overlay-refs/i], - forbiddenOutputs: [/snapshot --raw/i], - }), - makeCase({ - id: 'ios-ax-unavailable-screenshot-coordinate-recovery', - contract: [ - 'App name: Agent Device Tester', - 'Platform: iOS simulator', - 'Current screen returned a sparse snapshot: Snapshot: 1 node (truncated)', - 'The hint says snapshot state is invalid or unavailable for this screen', - 'If screenshot shows the Home Screen or another app, the hint says to open the app again first', - 'The visible screenshot shows the next tab target centered near x=124 y=817', - 'Accessibility may work again after leaving this screen', - ], - task: 'Plan fallback commands to recover from the AX-unavailable snapshot state: capture visual truth, navigate out using the visible coordinate, then try AX again on the next screen.', - outputs: [plannedCommand('screenshot'), /(?:click|press)\s+124\s+817/i, /snapshot -i/i], - forbiddenOutputs: [/--overlay-refs/i, /@e\d+/i, /(?:find|wait|is|get)\b/i, /snapshot --raw/i], - strictFinalOutput: true, - allowOnlyLocalCliHelpCommands: true, - }), - makeCase({ - id: 'perf-session-metrics', - contract: [ - 'App name: Agent Device Tester', - 'Platform: iOS simulator', - 'No startup sample exists until the app is opened', - 'Need session startup, memory, and CPU data as JSON', - ], - task: 'Plan the commands to open the app first if needed, then collect session performance metrics as JSON.', - outputs: [ - plannedCommand('open'), - plannedCommandAlternatives(['perf metrics', 'metrics']), - /--json/i, - ], - forbiddenOutputs: [plannedCommand('network')], - }), - makeCase({ - id: 'perf-session-frames', - contract: [ - 'App name: Agent Device Tester', - 'Platform: Android emulator', - 'The app is already open', - 'Need focused frame and jank health as JSON', - ], - task: 'Plan the command to collect focused frame and jank health as JSON without collecting React component render profiling.', - outputs: [plannedCommand('perf frames'), /--json/i], - forbiddenOutputs: [plannedCommand('react-devtools'), plannedCommand('network')], - }), - makeCase({ - id: 'perf-memory-diagnostics', - contract: [ - 'App name: Agent Device Tester', - 'Platform: Android emulator', - 'The app is already open', - 'Symptom: memory grows after repeatedly opening the Settings diagnostics screen', - 'Need a compact first-pass memory sample, then a Java heap artifact only if the sample suggests escalation', - ], - task: 'Plan the commands for memory diagnostics without using React DevTools or debug symbols.', - outputs: [ - plannedCommand('perf memory sample'), - /--json/i, - plannedCommand('perf memory snapshot'), - /--kind\s+android-hprof/i, - /--out\s+\S+\.hprof/i, - ], - forbiddenOutputs: [ - plannedCommand('react-devtools'), - plannedCommand('debug'), - plannedCommand('perf frames'), - ], - }), - makeCase({ - id: 'react-native-js-heap-leak-cdp-triplet', - contract: [ - 'App name: Agent Device Tester', - 'Platform: iOS simulator', - 'Metro is running at http://127.0.0.1:8081', - 'The app exposes a React Native CDP target through Metro', - 'Symptom: JavaScript heap grows after opening and closing the Cart screen', - 'Need proof that retained JS objects survive cleanup, plus shortest useful retaining paths', - 'This is not a native/process memory investigation', - ], - task: 'Plan a bounded React Native JS heap leak workflow using cdp: select the Metro CDP target, sample heap usage, capture baseline/action/cleanup snapshots, diff them, run leak-triplet, and inspect retainers for a leaked node.', - outputs: [ - plannedCommand('cdp target list'), - /--url\s+http:\/\/127\.0\.0\.1:8081/i, - plannedCommand('cdp target select'), - CDP_MEMORY_SNAPSHOT_CAPTURE, - /--name\s+baseline/i, - /--name\s+(?:after-action|action)/i, - /--name\s+cleanup/i, - plannedCommand('cdp memory snapshot diff'), - plannedCommand('cdp memory snapshot leak-triplet'), - plannedCommand('cdp memory snapshot retainers'), - ], - forbiddenOutputs: [ - plannedCommand('perf memory sample'), - plannedCommand('perf memory snapshot'), - plannedCommand('react-devtools'), - plannedCommand('cdp profile cpu'), - plannedCommand('cdp trace'), - plannedCommand('cdp network'), - plannedCommand('cdp console'), - ], - }), - makeCase({ - id: 'react-native-js-heap-quick-signal-cdp', - contract: [ - 'App name: Agent Device Tester', - 'Platform: Android emulator', - 'Metro is running at http://127.0.0.1:8081', - 'The app exposes a React Native CDP target through Metro', - 'Symptom: JavaScript heap may grow after filtering the product list', - 'Need only a compact first-pass JS heap growth signal before deciding whether to capture heap snapshots', - 'This is not a native/process memory investigation', - ], - task: 'Plan the CDP commands to select the Metro target and collect compact before/after JavaScript heap usage samples with GC, then diff the usage samples.', - outputs: [ - plannedCommand('cdp target list'), - /--url\s+http:\/\/127\.0\.0\.1:8081/i, - plannedCommand('cdp target select'), - CDP_MEMORY_USAGE_SAMPLE, - /--label\s+baseline/i, - /--label\s+after-action/i, - plannedCommand('cdp memory usage diff'), - ], - forbiddenOutputs: [ - plannedCommand('perf memory sample'), - plannedCommand('perf memory snapshot'), - CDP_MEMORY_SNAPSHOT_CAPTURE, - plannedCommand('react-devtools'), - ], - }), - makeCase({ - id: 'react-native-native-memory-uses-perf-not-cdp', - contract: [ - 'App name: Agent Device Tester', - 'Platform: Android emulator', - 'The app is already open', - 'Symptom: total process RSS/PSS grows while scrolling a native image gallery', - 'Need native/process memory evidence and an Android heap artifact if escalation is needed', - 'This is not a JavaScript heap or retained JS object investigation', - ], - task: 'Plan the memory diagnostics commands for this native/process memory issue without using CDP heap snapshots.', - outputs: [ - plannedCommand('perf memory sample'), - /--json/i, - plannedCommand('perf memory snapshot'), - /--kind\s+android-hprof/i, - /--out\s+\S+\.hprof/i, - ], - forbiddenOutputs: [ - plannedCommand('cdp'), - CDP_MEMORY_USAGE_SAMPLE, - CDP_MEMORY_SNAPSHOT_CAPTURE, - plannedCommand('react-devtools'), - ], - }), - makeCase({ - id: 'perf-apple-xctrace-profile', - contract: [ - 'App name: Agent Device Tester', - 'Platform: iOS simulator', - 'The app is already open', - 'Need Apple native CPU profiling evidence as a .trace artifact and compact JSON report', - 'Do not use debug or React DevTools for this native profile', - ], - task: 'Plan commands to record an Apple xctrace Time Profiler CPU profile under perf, stop it, then generate the compact report.', - outputs: [ - plannedCommand('perf cpu profile start'), - /--kind\s+xctrace/i, - /--template\s+"?Time Profiler"?/i, - /--out\s+\S+\.trace/i, - plannedCommand('perf cpu profile stop'), - plannedCommand('perf cpu profile report'), - /--out\s+\S+\.json/i, - ], - forbiddenOutputs: [plannedCommand('debug'), plannedCommand('react-devtools')], - }), - makeCase({ - id: 'perf-android-native-profiling', - contract: [ - 'App package: com.example.app', - 'Platform: Android emulator', - 'Need native CPU profile and system trace artifacts for PR evidence', - 'Collect lightweight perf metrics and focused frame health first', - 'Do not claim iOS native Simpleperf or Perfetto support', - ], - task: 'Plan commands to open the Android app, collect perf metrics and frames, then capture Android native Simpleperf CPU and Perfetto trace artifacts.', - outputs: [ - plannedCommand('open'), - plannedCommand('perf metrics'), - plannedCommand('perf frames'), - plannedCommand('perf cpu profile start'), - /--kind\s+simpleperf/i, - plannedCommand('perf cpu profile stop'), - plannedCommand('perf cpu profile report'), - plannedCommand('perf trace start'), - /--kind\s+perfetto/i, - plannedCommand('perf trace stop'), - /--out/i, - ], - forbiddenOutputs: [/ios/i, plannedCommand('debug'), plannedCommand('react-devtools')], - }), - makeCase({ - id: 'react-devtools-profile-search', - contract: [ - 'App name: Agent Device Tester', - 'React Native DevTools can connect to the running app', - 'Interaction to profile: type in the Catalog search field', - 'Search field selector: id="catalog-search"', - 'Need slow components and rerender counts', - ], - task: 'Plan the commands to verify React DevTools is connected, profile the Catalog search interaction, then list slow components and rerenders.', - outputs: [ - plannedCommandAlternatives(['react-devtools status', 'react-devtools start']), - plannedCommand('react-devtools wait'), - plannedCommand('react-devtools profile start'), - /catalog-search/i, - plannedCommand('react-devtools profile stop'), - BOUNDED_PROFILE_SLOW, - BOUNDED_PROFILE_RERENDERS, - ], - forbiddenOutputs: [ - plannedCommand('snapshot'), - plannedCommand('perf'), - BROAD_PROFILE_SLOW_LIMIT, - ], - }), - makeCase({ - id: 'react-devtools-bounded-profile-survey', - contract: [ - 'App name: Agent Device Tester', - 'React Native DevTools is connected', - 'Interaction to profile: expand the checkout totals panel', - 'Totals panel selector: id="checkout-totals"', - 'Need a compact performance report with initial summary, slow components, rerender counts, and commit timing', - 'Do not over-fetch broad profile tables while looking for offenders', - ], - task: 'Plan a bounded first-pass React DevTools profile workflow for the checkout totals interaction.', - outputs: [ - plannedCommand('react-devtools wait'), - plannedCommand('react-devtools profile start'), - /checkout-totals/i, - plannedCommand('react-devtools profile stop'), - BOUNDED_PROFILE_SLOW, - BOUNDED_PROFILE_RERENDERS, - BOUNDED_PROFILE_TIMELINE, - ], - forbiddenOutputs: [BROAD_PROFILE_SLOW_LIMIT], - }), - makeCase({ - id: 'react-devtools-exact-component-inspect', - contract: [ - 'App name: Agent Device Tester', - 'React Native DevTools is connected', - 'Need current props, state, and hooks for component SearchScreen', - 'Fuzzy component search returns noisy matches unless exact matching is used', - ], - task: 'Plan bounded React DevTools commands to find the exact SearchScreen component and inspect it.', - outputs: [ - plannedCommand('react-devtools find'), - /SearchScreen/i, - /--exact/i, - plannedCommand('react-devtools get component'), - ], - forbiddenOutputs: [plannedCommand('snapshot'), plannedCommand('perf')], - }), - makeCase({ - id: 'react-devtools-render-cause-report', - contract: [ - 'App name: Agent Device Tester', - 'React Native profile has already been stopped', - 'Rerender suspect from profile output: @c12', - 'Need render causes and changed props/state/hooks for that component', - ], - task: 'Plan the React DevTools command to inspect render causes for @c12.', - outputs: [plannedCommand('react-devtools profile report'), /@c12/i], - forbiddenOutputs: [ - plannedCommand('snapshot'), - plannedCommand('perf'), - plannedCommand('react-devtools profile slow'), - plannedCommand('react-devtools profile rerenders'), - ], - }), - makeCase({ - id: 'react-native-diagnostics-flow', - contract: [ - 'App name: Agent Device Tester', - 'Platform: Android', - 'Current screen: Settings tab', - 'Slow interaction: load diagnostics from the Settings screen', - 'Diagnostics trigger selector: id="load-diagnostics"', - 'Expected async result selector: id="diagnostics-error"', - 'The diagnostics request can take 5-10 seconds', - 'Need React component offenders and network evidence', - 'Open Agent Device Tester on Android and take snapshot -i before interacting', - ], - task: 'Plan commands for a focused React Native performance run around the Settings diagnostics load flow, including debug markers, async verification, slow/rerender output, and network headers.', - outputs: [ - plannedCommand('open'), - /snapshot -i/i, - /logs (?:clear --restart|start)/i, - /logs mark\b[^\n]*(?:before|start|begin)/i, - plannedCommand('react-devtools status'), - plannedCommand('react-devtools wait'), - plannedCommand('react-devtools profile start'), - /load-diagnostics/i, - /wait\b[^\n]*(?:diagnostics-error|Diagnostics error)/i, - /(?:5000|10000|12000|15000|20000)/i, - /logs mark\b[^\n]*(?:after|end|verified|diagnostics|loaded)/i, - plannedCommand('react-devtools profile stop'), - BOUNDED_PROFILE_SLOW, - BOUNDED_PROFILE_RERENDERS, - /network dump\b[^\n]*(?:--include headers|\bheaders\b|\ball\b)/i, - ], - forbiddenOutputs: [ - RAW_COORDINATE_TARGET, - /cat .*log/i, - /alert wait/i, - /agent-devtools/i, - BROAD_PROFILE_SLOW_LIMIT, - ], - }), - makeCase({ - id: 'gesture-swipe-carousel', - contract: [ - 'Platform: iOS simulator', - 'Current screen: onboarding carousel', - 'Need to advance and return across pages repeatedly', - 'Gesture should use a swipe series, not scroll', - 'Use one direct swipe command with --count and --pattern; do not use batch', - 'Swipe series are bounded to 200 repetitions, 10000ms pauses, and 60000ms scheduled time', - ], - task: 'Plan one direct gesture command to swipe horizontally across the carousel eight times with a 30ms pause and ping-pong pattern.', - outputs: [ - plannedCommand('swipe'), - /--count\s+8/i, - /--pause-ms\s+30/i, - /--pattern\s+ping-pong/i, - ], - forbiddenOutputs: [plannedCommand('scroll'), plannedCommand('batch'), RAW_COORDINATE_TARGET], - }), - makeCase({ - id: 'gesture-longpress-context-menu', - contract: [ - 'Platform: Android', - 'Current screen: Catalog tab', - 'Target center is x=300 y=500', - 'Need to open a native context menu with an 800ms long press', - ], - task: 'Plan the gesture command to long-press the target center for 800ms.', - outputs: [plannedCommand('longpress'), /300\s+500\s+800/i], - forbiddenOutputs: [/--duration-ms/i, /--hold-ms/i, plannedCommand('click')], - }), - makeCase({ - id: 'gesture-longpress-ref-context-menu', - contract: [ - 'Platform: iOS simulator', - 'Current screen: chat thread', - 'Target message current ref: @e42', - 'Need to open the reaction menu with an 800ms long press', - ], - task: 'Plan the gesture command to long-press the current message ref for 800ms.', - outputs: [plannedCommand('longpress'), /@e42/i, /\b800\b/i], - forbiddenOutputs: [ - /--duration-ms/i, - /--hold-ms/i, - plannedCommand('click'), - RAW_COORDINATE_TARGET, - ], - }), - makeCase({ - id: 'gesture-pinch-zoom', - contract: [ - 'Platform: iOS simulator', - 'Current screen: image preview', - 'Pinch is supported on Apple simulators', - 'Need to zoom out around x=200 y=400', - 'Zoom-out scale: 0.5', - ], - task: 'Plan the gesture command to pinch zoom out at the specified center.', - outputs: [plannedCommand('gesture pinch'), /0\.5/i, /200\s+400/i], - forbiddenOutputs: [ - /(?:^|\s)--scale(?!\w)/i, - /(?:^|\s)--x(?!\w)/i, - /(?:^|\s)--y(?!\w)/i, - plannedCommand('scroll'), - plannedCommand('swipe'), - ], - }), - makeCase({ - id: 'gesture-pan-fling-rotate', - contract: [ - 'Platform: iOS simulator', - 'Current screen: gesture lab', - 'Target center is x=200 y=420', - 'The target point is app-owned content away from screen edges, tab bars, navigation bars, and home indicators', - 'Need to test a slow upward pan, a right fling, and app-content rotation', - 'Pan delta is dx=0 dy=-80 over 500ms', - 'Fling distance is 180px', - 'Rotation is 35 degrees', - ], - task: 'Plan direct agent-device gesture commands for the pan, fling, and rotate gesture.', - outputs: [ - plannedCommand('gesture pan'), - /200\s+420\s+0\s+-80\s+500/i, - plannedCommand('gesture fling'), - /right\s+200\s+420\s+180/i, - plannedCommand('gesture rotate'), - /35\s+200\s+420/i, - ], - forbiddenOutputs: [ - plannedCommand('swipe'), - topLevelPlannedCommand('pan'), - topLevelPlannedCommand('fling'), - topLevelPlannedCommand('rotate'), - topLevelPlannedCommand('rotate-gesture'), - /--duration-ms/i, - /--pointer-count/i, - ], - }), - makeCase({ - id: 'gesture-two-finger-pan', - contract: [ - 'Platform: Android', - 'Current screen: map tilt canary', - 'Target center is x=200 y=420', - 'The recognizer requires exactly two fingers moving in parallel', - 'Pan delta is dx=80 dy=-40 over 700ms', - 'Do not add scale or rotation', - ], - task: 'Plan the direct agent-device gesture command for the two-finger pan.', - outputs: [plannedCommand('gesture pan'), /200\s+420\s+80\s+-40\s+700/i, /--pointer-count\s+2/i], - forbiddenOutputs: [ - plannedCommand('gesture transform'), - plannedCommand('gesture pinch'), - plannedCommand('gesture rotate'), - /(?:^|\s)--count(?:\s|=)/i, - ], - }), - makeCase({ - id: 'android-gesture-transform', - contract: [ - 'Platform: Android', - 'Current screen: gesture lab', - 'Target center is x=200 y=420', - 'Need the direct transform command rather than separate gesture commands', - 'Pan delta is dx=80 dy=-40', - 'Zoom scale is 2', - 'Rotation is 35 degrees', - 'Duration is 700ms', - 'After the command, verify Android changed qualitatively instead of asserting exact x, y, scale, or rotate values', - ], - task: 'Plan the direct agent-device command for the combined pan, zoom, and rotate gesture, then verify qualitative state.', - outputs: [ - plannedCommand('gesture transform'), - /200\s+420\s+80\s+-40\s+2\s+35\s+700/i, - plannedCommand('wait'), - /pan changed yes/i, - /pinch changed yes/i, - /rotate changed yes/i, - ], - forbiddenOutputs: [ - plannedCommand('gesture pan'), - plannedCommand('gesture pinch'), - plannedCommand('gesture rotate'), - plannedCommand('compose-gestures'), - /wait\s+["']?x\s/i, - /wait\s+["']?scale\s/i, - /wait\s+["']?rotate\s+\d/i, - ], - }), - makeCase({ - id: 'ios-simulator-gesture-transform', - contract: [ - 'Platform: iOS simulator', - 'Current screen: gesture lab', - 'Target center is x=200 y=420', - 'Need one continuous two-finger gesture without lifting fingers', - 'Pan delta is dx=80 dy=-40', - 'Zoom scale is 2', - 'Rotation is 35 degrees', - 'Duration is 700ms', - ], - task: 'Plan the direct agent-device command for the combined pan, zoom, and rotate gesture.', - outputs: [plannedCommand('gesture transform'), /200\s+420\s+80\s+-40\s+2\s+35\s+700/i], - forbiddenOutputs: [ - plannedCommand('gesture pan'), - plannedCommand('gesture pinch'), - plannedCommand('gesture rotate'), - plannedCommand('rotate-gesture'), - plannedCommand('swipe'), - ], - }), - makeCase({ - id: 'settings-animation-stabilizer', - contract: [ - 'Platform: Android', - 'App name: Agent Device Tester', - 'Animations make this flow flaky', - 'Animations should be restored after the check', - ], - task: 'Plan the commands to disable platform animations before the app check, run a snapshot, then restore animations.', - outputs: [/settings animations off/i, plannedCommand('snapshot'), /settings animations on/i], - forbiddenOutputs: [ - /--platform macos/i, - /settings appearance/i, - /animations disable/i, - /animations restore/i, - ], - }), - makeCase({ - id: 'trace-capture-session', - contract: [ - 'App name: Agent Device Tester', - 'An app session is already open', - 'Repro button selector: id="load-diagnostics"', - 'Need low-level session diagnostics for one diagnostics-button repro', - 'Trace artifact path: ./traces/diagnostics.trace', - ], - task: 'Plan the commands to start trace capture, trigger diagnostics, then stop the trace into the requested artifact path.', - outputs: [ - /trace start \.\/traces\/diagnostics\.trace/i, - /load-diagnostics/i, - /trace stop \.\/traces\/diagnostics\.trace/i, - ], - forbiddenOutputs: [ - plannedCommand('record'), - /logs clear --restart/i, - /trace (?:start|stop) --path/i, - ], - }), - makeCase({ - id: 'alert-visible-ui-fallback', - contract: [ - 'App name: Agent Device Tester', - 'Current screen: Home tab', - 'A visible permission sheet contains the button text "Allow"', - 'alert accept already returned no alert found', - ], - task: 'Plan the fallback commands to handle the visible sheet as normal tappable UI instead of looping on alert accept.', - outputs: [ - /(?:^|\n)(?:agent-device\s+)?(?:find\b.*\bpress\b|(?:press|click)\b.*Allow|snapshot -i)/is, - /Allow/i, - ], - forbiddenOutputs: [/alert accept.*\n.*alert accept/is, RAW_COORDINATE_TARGET], - }), - makeCase({ - id: 'android-keyboard-readonly-status', - contract: [ - 'Platform: Android', - 'App name: Agent Device Tester', - 'Current screen: Checkout form tab', - 'Question: is the keyboard visible and what input type is active?', - ], - task: 'Plan the read-only command to inspect Android keyboard visibility and input type.', - outputs: [/(?:^|\n)(?:agent-device\s+)?keyboard(?:\s+(?:status|get))?(?:\s|$)/i], - forbiddenOutputs: [plannedCommand('fill'), plannedCommand('type'), /keyboard dismiss/i], - }), - makeCase({ - id: 'android-fill-ime-capture-stop-retry-loop', - contract: [ - 'Platform: Android', - 'App name: Agent Device Tester', - 'Current screen: Checkout form tab', - 'The Android IME capture fixture is visible: id="android-ime-capture-fixture"', - 'The app field is still visible as targetInput id="field-ime-capture-target"', - 'The last fill failed: Android fill input was captured by the active keyboard instead of the app field', - 'Diagnostic details show actualInput packageName="com.google.android.inputmethod.latin" inputMethodOwned=true', - ], - task: 'Plan the next diagnostic step. Do not retry fill or type until the IME handwriting/stylus state has been corrected.', - outputs: [ - /(?:^|\n)(?:agent-device\s+)?keyboard(?:\s+(?:status|get))?(?:\s|$)/i, - /(?:IME|keyboard|Gboard|handwriting|stylus)/i, - ], - forbiddenOutputs: [ - plannedCommand('fill'), - plannedCommand('type'), - RAW_COORDINATE_TARGET, - /adb shell/i, - ], - }), - makeCase({ - id: 'direct-proxy-remote-simulator-flow', - contract: [ - 'The agent is running in a cloud Linux environment', - 'The user says a remote iOS simulator is available through agent-device proxy', - 'Proxy daemon base URL: https://example.trycloudflare.com/agent-device', - 'Proxy daemon auth token: proxy-secret', - 'Platform: iOS', - 'Device: iPhone 17 Pro', - 'App: Maps', - 'This is direct proxy mode, not cloud/profile mode', - 'Use one explicit session for the whole flow: maps', - ], - task: 'Plan commands to open Maps on the remote simulator through the direct proxy, capture interactive refs, and close the session.', - outputs: [ - /open\s+Maps\b/i, - /snapshot\b[^\n]*-i\b/i, - /close\b/i, - /--session maps/i, - /--platform ios/i, - /--device ["']iPhone 17 Pro["']/i, - /--daemon-base-url https:\/\/example\.trycloudflare\.com\/agent-device/i, - /--daemon-auth-token proxy-secret/i, - ], - forbiddenOutputs: [ - plannedCommand('connect'), - plannedCommand('disconnect'), - /--remote-config/i, - /--tenant/i, - /--run-id/i, - /--lease-id/i, - ], - }), - makeCase({ - id: 'remote-cloud-connect-flow', - contract: [ - 'Cloud control plane owns the connection profile', - 'No local remote config path was provided', - 'App package: com.callstack.agentdevicetester', - 'The cloud profile owns tenant, run, lease, and Metro hints', - ], - task: 'Plan a remote flow that discovers the cloud connection profile, opens the app, captures a snapshot, and disconnects cleanly.', - outputs: [ - plannedCommand('connect'), - plannedCommand('open'), - plannedCommand('snapshot'), - plannedCommand('disconnect'), - ], - forbiddenOutputs: [ - /--remote-config/i, - /--daemon-base-url/i, - /--tenant/i, - /--run-id/i, - plannedCommand('screenshot'), - ], - }), - makeCase({ - id: 'remote-config-connect-flow', - contract: [ - 'Remote config path: ./remote-config.json', - 'App package: com.callstack.agentdevicetester', - 'The remote profile owns tenant, run, lease, and Metro hints', - ], - task: 'Plan a remote flow that connects through the remote config, opens the app, captures a snapshot, and disconnects cleanly.', - outputs: [ - /connect --remote-config \.\/remote-config\.json/i, - plannedCommand('open'), - plannedCommand('snapshot'), - plannedCommand('disconnect'), - ], - forbiddenOutputs: [ - /--daemon-base-url/i, - /--tenant/i, - /--run-id/i, - plannedCommand('screenshot'), - ], - }), - makeCase({ - id: 'remote-config-script-flow', - contract: [ - 'Remote config path: ./remote-config.json', - 'App package: com.callstack.agentdevicetester', - 'This is a self-contained script where every command must be explicit', - 'The remote profile owns tenant, run, lease, and Metro hints', - ], - task: 'Plan a self-contained remote script that opens the app, captures a snapshot, and disconnects using the remote config on every command.', - outputs: [ - /(?:--remote-config\s+\.\/remote-config\.json[^\n]*open|open\b[^\n]*--remote-config\s+\.\/remote-config\.json)/i, - /(?:--remote-config\s+\.\/remote-config\.json[^\n]*snapshot|snapshot\b[^\n]*--remote-config\s+\.\/remote-config\.json)/i, - /(?:--remote-config\s+\.\/remote-config\.json[^\n]*disconnect|disconnect\b[^\n]*--remote-config\s+\.\/remote-config\.json)/i, - ], - forbiddenOutputs: [/--daemon-base-url/i, /--tenant/i, /--run-id/i], - }), - makeCase({ - id: 'remote-ios-runner-lease-retry-snapshot', - contract: [ - 'Direct proxy flow to a remote Mac is already configured', - 'Platform: iOS', - 'Device: iPhone 17 Pro', - 'The remote agent already opened Maps successfully', - 'The first interactive snapshot failed: iOS runner is already owned by another agent-device daemon', - 'The proxy daemon can reclaim stale same-state runner leases on retry', - ], - task: 'Plan the next remote client command. Do not run prepare ios-runner or prescribe host process cleanup; retry the original interactive snapshot.', - outputs: [/snapshot\b[^\n]*-i\b/i, /--platform ios/i, /--device ["']iPhone 17 Pro["']/i], - forbiddenOutputs: [ - plannedCommand('prepare ios-runner'), - /prepare\s+ios-runner/i, - plannedCommand('open'), - /\bkill\b/i, - /clean:daemon/i, - ], - }), - makeCase({ - id: 'macos-menubar-surface', - contract: [ - 'Platform: macOS', - 'App name: Agent Device Tester Menu', - 'The app lives entirely as a menu bar extra', - 'Normal app snapshots can be sparse or empty', - 'Required flags: --platform macos --surface menubar', - ], - task: 'Plan the commands to inspect the menu bar app surface with --platform macos --surface menubar and capture interactive refs with snapshot -i.', - outputs: [/--platform macos/i, /--surface menubar/i, /snapshot\b.*(?:-i\b|\s-i\b)/i], - forbiddenOutputs: [/--surface app/i, /snapshot --raw/i], - }), - makeCase({ - id: 'macos-context-menu-secondary-click', - contract: [ - 'Platform: macOS', - 'Current surface: app', - 'Target row current ref: @e66', - 'Need to open its native context menu and inspect menu item refs', - 'Required platform flag: --platform macos', - ], - task: 'Plan commands with --platform macos to open the context menu for @e66 and then refresh interactive refs for the menu items.', - outputs: [ - plannedCommand('click'), - /@e66/i, - /--button\s+secondary/i, - /--platform\s+macos/i, - /snapshot\b.*-i/i, - ], - forbiddenOutputs: [plannedCommand('longpress'), RAW_COORDINATE_TARGET, /--surface menubar/i], - }), - makeCase({ - id: 'replay-maintenance-update', - contract: [ - 'Replay path: ./replays/catalog-checkout.ad', - 'Selectors drifted after a UI label change', - '--update is accepted but retired: it never rewrites the replay file and reports healed: 0', - 'The divergence reports resume.from=4 and resume.planDigest=0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef', - 'Workflow A: after accepting a suggestion and editing the script, the old digest is stale; run a fresh full replay with no resume flags', - 'Workflow B: when the script and includes stay unchanged, repair app state so retrying the failed action is safe, then resume with the reported values', - 'Output exactly the two alternative replay commands in A-then-B order; manual edits and app-state repairs happen out of band', - ], - task: 'Plan both valid replay-maintenance commands after selector drift.', - outputs: [ - /(?:^|\n)agent-device\s+replay\s+\.\/replays\/catalog-checkout\.ad\s*(?:\n|$)/i, - /(?:^|\n)agent-device\s+replay\s+\.\/replays\/catalog-checkout\.ad(?=[^\n]*--from\s+4\b)(?=[^\n]*--plan-digest\s+0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef\b)[^\n]*(?:\n|$)/i, - ], - forbiddenOutputs: [ - /(?:^|\n)agent-device\s+replay\s+\.\/replays\/catalog-checkout\.ad[^\n]*(?:-u\b|--update\b)/i, - /(?:^|\n)agent-device\s+replay\s+\.\/replays\/catalog-checkout\.ad(?=[^\n]*--from\b)(?![^\n]*--plan-digest\b)[^\n]*/i, - /(?:^|\n)agent-device\s+replay\s+\.\/replays\/catalog-checkout\.ad(?=[^\n]*--plan-digest\b)(?![^\n]*--from\b)[^\n]*/i, - ], - }), - makeCase({ - id: 'record-and-heal-diagnostics-no-record', - contract: [ - 'Replay path: ./replays/checkout-form.ad', - 'The replay was armed with replay ./replays/checkout-form.ad --save-script before step 1', - 'A divergence occurred; repairHint: record-and-heal', - 'resume.repairSessionHeld is true; the repair session is held live', - "The divergence screen.refs lists two candidates sharing the recorded 'Submit' label: @e5 and @e9", - 'The divergence report says: "While armed, read-only inspection (snapshot -i, get attrs, find, is) is recorded too — use --no-record on those commands or they will land in the healed script."', - 'get attrs @e5 reports disabled: true; get attrs @e9 reports disabled: false', - '@e9 is therefore the corrected control to press; pressing it is the one step that SHOULD be recorded, so do not use --no-record on it', - 'The divergence reports resume.from=6 and resume.planDigest=abcdef0123456789abcdef0123456789abcdef0123456789abcdef01234567', - 'Resume by running replay on the same path again with --from 6 --plan-digest abcdef0123456789abcdef0123456789abcdef0123456789abcdef01234567', - ], - task: 'Plan the commands to inspect both candidate refs and find which one is the corrected control, press the corrected control, then resume the armed repair with the reported values. Follow the record-and-heal repair rules for which of these commands should be recorded and which should not.', - outputs: [ - /(?:^|\n)agent-device\s+get\s+attrs\s+@e5\b(?=[^\n]*--no-record\b)[^\n]*(?:\n|$)/i, - /(?:^|\n)agent-device\s+get\s+attrs\s+@e9\b(?=[^\n]*--no-record\b)[^\n]*(?:\n|$)/i, - /(?:^|\n)agent-device\s+(?:press|click)\s+@e9\b(?:(?!\n)(?!--no-record))*(?:\n|$)/i, - /(?:^|\n)agent-device\s+replay\s+\.\/replays\/checkout-form\.ad(?=[^\n]*--from\s+6\b)(?=[^\n]*--plan-digest\s+abcdef0123456789abcdef0123456789abcdef0123456789abcdef01234567\b)[^\n]*(?:\n|$)/i, - ], - forbiddenOutputs: [ - // Diagnostic reads used only to locate the target must not be recorded - // into the healed script — a bare `get attrs` line (no --no-record) - // is exactly the #1271 foot-gun the interim guidance stops. - /(?:^|\n)agent-device\s+get\s+attrs\s+@e5\b(?:(?!\n)(?!--no-record))*(?:\n|$)/i, - /(?:^|\n)agent-device\s+get\s+attrs\s+@e9\b(?:(?!\n)(?!--no-record))*(?:\n|$)/i, - // The corrective press is the one step that SHOULD land in the healed - // script, so it must not be suppressed with --no-record. - /(?:^|\n)agent-device\s+(?:press|click)\s+@e9\b(?=[^\n]*--no-record\b)[^\n]*(?:\n|$)/i, - // Never press the disabled decoy candidate. - /(?:^|\n)agent-device\s+(?:press|click)\s+@e5\b/i, - ], - }), - makeCase({ - id: 'replay-maestro-compatibility-flow', - contract: [ - 'Flow path: ./flows/checkout-form.yaml', - 'The flow is a Maestro YAML compatibility flow', - 'Need to run it through Agent Device replay, not the Maestro CLI', - 'Target platform: iOS', - ], - task: 'Plan the command to replay the Maestro YAML flow through Agent Device on iOS.', - outputs: [ - plannedCommand('replay'), - /--maestro/i, - /\.\/flows\/checkout-form\.yaml/i, - /--platform\s+ios/i, - ], - forbiddenOutputs: [/maestro\s+test/i, /maestro\s+cloud/i, plannedCommand('test')], - }), - makeCase({ - id: 'test-maestro-shard-all-devices', - contract: [ - 'Suite path: ./e2e/maestro', - 'The suite contains Maestro YAML compatibility flows', - 'Connected device ids: udid1, emulator-5554', - 'Need local cross-device validation by running the full suite on each device', - ], - task: 'Plan the Agent Device test command that runs the Maestro suite on both connected devices without calling the Maestro CLI directly.', - outputs: [ - plannedCommand('test'), - /--maestro/i, - /--device\s+["']?udid1,emulator-5554["']?/i, - /--shard-all\s+2/i, - /\.\/e2e\/maestro/i, - ], - forbiddenOutputs: [/maestro\s+test/i, /--shard-split/i, /--platform\s+(?:ios|android)/i], - }), - makeCase({ - id: 'batch-known-stable-flow', - contract: [ - 'App name: Agent Device Tester', - 'The full checkout flow is already known and stable', - 'Known batch steps file: ./checkout-steps.json', - 'Need fewer round trips while recording evidence', - ], - task: 'Plan the commands to start a recording, execute the known checkout steps from the provided steps file as one batch, and stop the recording.', - outputs: [ - /(?:^|\n)(?:agent-device\s+)?record\s+start/i, - plannedCommand('batch'), - /(?:^|\n)(?:agent-device\s+)?record\s+stop/i, - ], - forbiddenOutputs: [PSEUDO_ASSERTION_COMMAND, /workflow batch/i, plannedCommand('trace')], - }), - makeCase({ - id: 'same-session-mutations-serial', - contract: [ - 'Session: dogfood-test-app', - 'Current screen: Checkout form tab', - 'Name field selector: id="field-name"', - 'Email field selector: id="field-email"', - 'Submit button selector: id="submit-order"', - 'Need to fill name, fill email, and press submit as three separate commands', - 'All commands mutate the same active device session', - 'Parallel same-session mutations can pollute focus and field state', - 'Do not use batch for this case; demonstrate serial command ordering', - ], - task: 'Plan the three separate serial commands for this same-session form flow using the durable selectors.', - outputs: [/--session dogfood-test-app/i, /field-name/i, /field-email/i, /submit-order/i], - forbiddenOutputs: [ - /Based on my/i, - /Let me/i, - /Promise\.all/i, - /(?:^|\n).*(?:fill|press).*(?:&|&&).*(?:fill|press)/i, - /parallel/i, - plannedCommand('batch'), - ], - }), - makeCase({ - id: 'batch-inline-step-schema-input', - contract: [ - 'Need one inline batch command', - 'Step 1: open settings', - 'Step 2: wait 100 ms', - 'Batch step schema supports command, input, and runtime', - 'The args field is invalid and must not be used', - ], - task: 'Plan the batch command with inline JSON steps using the supported structured input field.', - outputs: [plannedCommand('batch'), /--steps/i, /"input"\s*:/i, /"open"/i, /"wait"/i], - forbiddenOutputs: [/"args"\s*:/i, /"positionals"\s*:/i, /workflow batch/i], - }), ]; const suite: Case[] = [ diff --git a/vitest.config.ts b/vitest.config.ts index 6c2642473..14db5736b 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -38,6 +38,8 @@ export default defineConfig({ include: [ 'src/**/*.test.ts', 'scripts/__tests__/help-conformance-bench.test.ts', + 'scripts/__tests__/help-conformance-sample-outputs.test.ts', + 'scripts/__tests__/help-conformance-topic-coverage.test.ts', 'test/skillgym/suites/local-cli-help-policy.test.ts', // The Maestro conformance oracle runs via `node --test` in its own CI // job (scripts/maestro-conformance), like the layering guard. From ec31963a9d6bb4d641f0095cef5c3333d9e678f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Mon, 27 Jul 2026 13:42:04 +0200 Subject: [PATCH 2/2] review: drive error samples through the real producers; enforce local-help on the routing smoke MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The DEVICE_IN_USE, AMBIGUOUS_MATCH, and APP_NOT_INSTALLED parity tests no longer hand-author the producer message before rendering: each drives the actual producer — buildDeviceInUseBySessionError (extracted in session-open.ts and called by the handler), buildAmbiguousMatchError (now exported from find.ts), and buildAppNotInstalledError (extracted in app-resolution.ts and thrown by the resolver). Because each factory is exported from its producer file and called by the production path, dropping the production call would make it test-only and fail check:production-exports — the wiring is gate-enforced, not conventional. open-and-snapshot now sets requireLocalCliHelp and allowOnlyLocalCliHelpCommands, so the 'skill plus local help' claim is observed rather than assumed; without them the case can pass on model prior alone. --- .../help-conformance-sample-outputs.test.ts | 40 +++++++++++++------ scripts/help-conformance-sample-outputs.mjs | 19 +++++---- src/daemon/handlers/find.ts | 5 ++- src/daemon/handlers/session-open.ts | 11 +++++ src/platforms/apple/core/app-resolution.ts | 9 ++++- .../suites/agent-device-smoke-suite.ts | 6 +++ 6 files changed, 67 insertions(+), 23 deletions(-) diff --git a/scripts/__tests__/help-conformance-sample-outputs.test.ts b/scripts/__tests__/help-conformance-sample-outputs.test.ts index 1e151e8f6..07652a03f 100644 --- a/scripts/__tests__/help-conformance-sample-outputs.test.ts +++ b/scripts/__tests__/help-conformance-sample-outputs.test.ts @@ -13,10 +13,12 @@ import { } from '../help-conformance-sample-outputs.mjs'; import { interactionCliOutputFormatters } from '../../src/commands/interaction/output.ts'; import { NEVER_SETTLED_HINT } from '../../src/commands/interaction/runtime/settle.ts'; +import { buildAmbiguousMatchError } from '../../src/daemon/handlers/find.ts'; import { refMutationAdmissionResponse } from '../../src/daemon/handlers/interaction-ref-policy.ts'; -import { buildSessionRecoveryHint } from '../../src/daemon/session-recovery-hints.ts'; +import { buildDeviceInUseBySessionError } from '../../src/daemon/handlers/session-open.ts'; import { resolveRefStalenessWarning } from '../../src/daemon/session-snapshot.ts'; import type { SessionState } from '../../src/daemon/types.ts'; +import { buildAppNotInstalledError } from '../../src/platforms/apple/core/app-resolution.ts'; import { AppError, normalizeError } from '../../src/kernel/errors.ts'; import type { SnapshotQualityVerdict } from '../../src/snapshot/snapshot-quality.ts'; import { renderSnapshotQualityWarnings } from '../../src/snapshot/snapshot-quality.ts'; @@ -154,13 +156,19 @@ test('private-ax recovery sample matches the snapshot renderer and quality warni assert.equal(text.trimEnd(), PRIVATE_AX_RECOVERY_SAMPLE.output); }); -test('device-in-use sample matches the human error rendering and recovery hint', () => { +test('device-in-use sample matches the real session-open producer', () => { const owningSession = { name: 'checkout' } as SessionState; + const device = { id: 'SIM-001', name: 'iPhone 17 Pro' } as Parameters< + typeof buildDeviceInUseBySessionError + >[1]; + const response = buildDeviceInUseBySessionError(owningSession, device); + assert.ok(!response.ok, 'the by-session conflict must be an error response'); const rendered = renderHumanError( - new AppError('DEVICE_IN_USE', 'Device is already in use by session "checkout".', { - session: 'checkout', - hint: buildSessionRecoveryHint(owningSession, 'device-in-use'), - }), + new AppError( + response.error.code as ConstructorParameters[0], + response.error.message, + response.error.details, + ), ); assert.equal(rendered, DEVICE_IN_USE_SAMPLE.output); }); @@ -184,19 +192,25 @@ test('stale-ref sample matches the real admission rejection and staleness hint', assert.equal(rendered, STALE_REF_SAMPLE.output); }); -test('ambiguous-match sample matches the human error rendering and default hint', () => { +test('ambiguous-match sample matches the real find producer and default hint', () => { + const matches = [ + { ref: 'e2', label: 'Follow' }, + { ref: 'e5', label: 'Follow' }, + { ref: 'e9', label: 'Follow' }, + ] as Parameters[0]; + const response = buildAmbiguousMatchError(matches, 'text', 'Follow'); + assert.ok(!response.ok, 'an ambiguous find must be an error response'); const rendered = renderHumanError( new AppError( - 'AMBIGUOUS_MATCH', - 'find matched 3 elements for text "Follow". Use a more specific locator or selector.', + response.error.code as ConstructorParameters[0], + response.error.message, + response.error.details, ), ); assert.equal(rendered, AMBIGUOUS_MATCH_SAMPLE.output); }); -test('app-not-installed sample matches the human error rendering and default hint', () => { - const rendered = renderHumanError( - new AppError('APP_NOT_INSTALLED', 'No app found matching "Shoply"'), - ); +test('app-not-installed sample matches the real app-resolution producer and default hint', () => { + const rendered = renderHumanError(buildAppNotInstalledError('Shoply')); assert.equal(rendered, APP_NOT_INSTALLED_SAMPLE.output); }); diff --git a/scripts/help-conformance-sample-outputs.mjs b/scripts/help-conformance-sample-outputs.mjs index 09e7616b0..e083a722b 100644 --- a/scripts/help-conformance-sample-outputs.mjs +++ b/scripts/help-conformance-sample-outputs.mjs @@ -64,8 +64,9 @@ Detected an overly complex or slow accessibility tree. Fell back to the private- @e8 [tab] "Home" [selected]`, }; -// DEVICE_IN_USE from src/daemon/handlers/session-open.ts: another session owns -// the device; the hint is buildSessionRecoveryHint(session, 'device-in-use'). +// DEVICE_IN_USE from buildDeviceInUseBySessionError +// (src/daemon/handlers/session-open.ts) — the parity test drives that exact +// producer. export const DEVICE_IN_USE_SAMPLE = { command: `agent-device press 'label="Place order"' --settle`, output: `Error (DEVICE_IN_USE): Device is already in use by session "checkout". @@ -83,18 +84,20 @@ export const STALE_REF_SAMPLE = { Hint: Ref @e12 was minted from snapshot s5 but the session's ref frame is now s7 — re-run snapshot -i.`, }; -// AMBIGUOUS_MATCH from src/daemon/handlers/find.ts buildAmbiguousMatchError; -// the by-design rejection instead of silent disambiguation. Candidate refs -// live in details, which the human rendering does not print — the agent must -// re-observe or narrow, not guess a ref it never saw. +// AMBIGUOUS_MATCH from buildAmbiguousMatchError (src/daemon/handlers/find.ts) +// — the parity test drives that exact producer. The by-design rejection +// instead of silent disambiguation: candidate refs live in details, which the +// human rendering does not print, so the agent must re-observe or narrow, not +// guess a ref it never saw. export const AMBIGUOUS_MATCH_SAMPLE = { command: 'agent-device find text "Follow" press', output: `Error (AMBIGUOUS_MATCH): find matched 3 elements for text "Follow". Use a more specific locator or selector. Hint: Multiple candidates matched. Narrow the query or pass an exact identifier.`, }; -// APP_NOT_INSTALLED from src/platforms/apple/core/app-resolution.ts; the hint -// is defaultHintForCode('APP_NOT_INSTALLED'). +// APP_NOT_INSTALLED from buildAppNotInstalledError +// (src/platforms/apple/core/app-resolution.ts) — the parity test drives that +// exact producer; the hint is defaultHintForCode('APP_NOT_INSTALLED'). export const APP_NOT_INSTALLED_SAMPLE = { command: 'agent-device open Shoply --platform ios', output: `Error (APP_NOT_INSTALLED): No app found matching "Shoply" diff --git a/src/daemon/handlers/find.ts b/src/daemon/handlers/find.ts index 36a826076..485f9b109 100644 --- a/src/daemon/handlers/find.ts +++ b/src/daemon/handlers/find.ts @@ -542,7 +542,10 @@ function publicFindFlags(flags: DaemonRequest['flags']): Record return { ...(stripInternalInteractionFlags(flags) ?? {}) }; } -function buildAmbiguousMatchError( +// Exported as the single AMBIGUOUS_MATCH producer so the help-benchmark +// sample parity test renders the exact error this handler returns; a message +// change here fails that gate instead of drifting past it. +export function buildAmbiguousMatchError( matches: SnapshotState['nodes'], locator: FindLocator, query: string, diff --git a/src/daemon/handlers/session-open.ts b/src/daemon/handlers/session-open.ts index 1bc7e85d2..f28f83e22 100644 --- a/src/daemon/handlers/session-open.ts +++ b/src/daemon/handlers/session-open.ts @@ -559,6 +559,17 @@ function findNewSessionDeviceConflict(params: { }, ); } + return buildDeviceInUseBySessionError(inUse, device); +} + +// Exported as the single by-session DEVICE_IN_USE producer so the +// help-benchmark sample parity test renders the exact error this handler +// returns; a message or hint change here fails that gate instead of drifting +// past it. +export function buildDeviceInUseBySessionError( + inUse: SessionState, + device: DeviceInfo, +): DaemonResponse { return errorResponse('DEVICE_IN_USE', `Device is already in use by session "${inUse.name}".`, { session: inUse.name, deviceId: device.id, diff --git a/src/platforms/apple/core/app-resolution.ts b/src/platforms/apple/core/app-resolution.ts index 55853c9eb..9313f542a 100644 --- a/src/platforms/apple/core/app-resolution.ts +++ b/src/platforms/apple/core/app-resolution.ts @@ -59,7 +59,14 @@ export async function resolveIosApp(device: DeviceInfo, app: string): Promise