feat(bench): renderer-pinned samples, topic-coverage gate, error-recovery quizzes; trim skillgym to agentic checks

.fallowrc.json

@@ -30,7 +30,8 @@
     "scripts/layering/**",
     "scripts/maestro-conformance/**",
     "apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests.xctestplan",
-    "scripts/write-xcuitest-cache-metadata.mjs"
+    "scripts/write-xcuitest-cache-metadata.mjs",
+    "scripts/help-conformance-sample-outputs.d.mts"
   ],
   "ignoreDependencies": [
     "@theme"
@@ -79,6 +80,14 @@
         "PlatformDescriptorsAreTotal",
         "GatedKeysAreResolverKeys"
       ]
+    },
+    {
+      "comment": "Help-benchmark conformance seams: helpTopicIds feeds the topic-coverage gate and PRIVATE_AX_RECOVERY_SAMPLE feeds the skillgym suite; both consumers are test-tree files outside --production analysis.",
+      "file": "{src/cli/parser/cli-help.ts,scripts/help-conformance-sample-outputs.mjs}",
+      "exports": [
+        "helpTopicIds",
+        "PRIVATE_AX_RECOVERY_SAMPLE"
+      ]
     }
   ],
   "usedClassMembers": [

docs/agents/testing.md

@@ -16,8 +16,9 @@ The mapping it encodes, for when you need to run a gate directly or reason about
 | Platform/device response — anything emitting `platform`/`appleOs` on the wire, or shaping a daemon response | `pnpm test:integration:provider` **and** `pnpm test:coverage` |
 | Cross-platform behavior | `pnpm test:integration` |
 | iOS runner / Swift | `pnpm build:xcuitest` |
-| CLI help/guidance (`src/cli/parser/cli-help.ts`, `src/cli-schema/`) | `pnpm exec vitest run src/cli/parser/__tests__ src/cli-schema/command-schema-guards.test.ts` |
-| SkillGym prompts/assertions | `pnpm test:skillgym:case <case-id>` (broad: `pnpm test:skillgym`, filter with `-- --tag fixture-smoke` or `-- --tag skill-guidance`) |
+| CLI help/guidance (`src/cli/parser/cli-help.ts`, `src/cli-schema/`) | `pnpm exec vitest run src/cli/parser/__tests__ src/cli-schema/command-schema-guards.test.ts scripts/__tests__` — the `scripts/__tests__` gates enforce help-topic benchmark coverage and pin the bench's quoted CLI samples to the real renderers |
+| Help benchmark cases (`scripts/help-conformance-*.mjs`) | `pnpm exec vitest run scripts/__tests__` (deterministic gates); model-backed: `pnpm bench:help-conformance` (paid LLM calls, local only) |
+| SkillGym prompts/assertions | `pnpm test:skillgym:case <case-id>` (broad: `pnpm test:skillgym`, filter with `-- --tag fixture-smoke` or `-- --tag skill-guidance`) — agentic routing + local-help-consumption proof only; command-planning knowledge checks belong in the help bench |
 | Anything in `src/`, `test/`, `skills/` | `pnpm format` |
 
 Two traps worth naming:

package.json

@@ -109,8 +109,8 @@
     "perf:ios": "node --experimental-strip-types scripts/perf/run.ts --platform ios",
     "perf:android": "node --experimental-strip-types scripts/perf/run.ts --platform android",
     "lint": "oxlint . --deny-warnings",
-    "format": "node ./node_modules/oxfmt/bin/oxfmt --write src test skills scripts/help-conformance-bench.mjs scripts/help-conformance-case-checks.mjs scripts/help-conformance-command-validator.ts scripts/help-conformance-expectations.mjs scripts/help-conformance-plan-validator.mjs scripts/help-conformance-runner-output.mjs scripts/help-conformance-summary.mjs scripts/__tests__/help-conformance-bench.test.ts package.json tsconfig.json tsconfig.lib.json tsdown.config.ts vitest.config.ts .github/actions/setup-node-pnpm/action.yml .oxlintrc.json .oxfmtrc.json '!test/skillgym/.skillgym-results/**'",
-    "format:check": "node ./node_modules/oxfmt/bin/oxfmt --check src test skills scripts/help-conformance-bench.mjs scripts/help-conformance-case-checks.mjs scripts/help-conformance-command-validator.ts scripts/help-conformance-expectations.mjs scripts/help-conformance-plan-validator.mjs scripts/help-conformance-runner-output.mjs scripts/help-conformance-summary.mjs scripts/__tests__/help-conformance-bench.test.ts package.json tsconfig.json tsconfig.lib.json tsdown.config.ts vitest.config.ts .github/actions/setup-node-pnpm/action.yml .oxlintrc.json .oxfmtrc.json '!test/skillgym/.skillgym-results/**'",
+    "format": "node ./node_modules/oxfmt/bin/oxfmt --write src test skills scripts/help-conformance-bench.mjs scripts/help-conformance-case-checks.mjs scripts/help-conformance-command-validator.ts scripts/help-conformance-expectations.mjs scripts/help-conformance-plan-validator.mjs scripts/help-conformance-runner-output.mjs scripts/help-conformance-summary.mjs scripts/help-conformance-cases.mjs scripts/help-conformance-sample-outputs.mjs scripts/__tests__/help-conformance-bench.test.ts scripts/__tests__/help-conformance-sample-outputs.test.ts scripts/__tests__/help-conformance-topic-coverage.test.ts package.json tsconfig.json tsconfig.lib.json tsdown.config.ts vitest.config.ts .github/actions/setup-node-pnpm/action.yml .oxlintrc.json .oxfmtrc.json '!test/skillgym/.skillgym-results/**'",
+    "format:check": "node ./node_modules/oxfmt/bin/oxfmt --check src test skills scripts/help-conformance-bench.mjs scripts/help-conformance-case-checks.mjs scripts/help-conformance-command-validator.ts scripts/help-conformance-expectations.mjs scripts/help-conformance-plan-validator.mjs scripts/help-conformance-runner-output.mjs scripts/help-conformance-summary.mjs scripts/help-conformance-cases.mjs scripts/help-conformance-sample-outputs.mjs scripts/__tests__/help-conformance-bench.test.ts scripts/__tests__/help-conformance-sample-outputs.test.ts scripts/__tests__/help-conformance-topic-coverage.test.ts package.json tsconfig.json tsconfig.lib.json tsdown.config.ts vitest.config.ts .github/actions/setup-node-pnpm/action.yml .oxlintrc.json .oxfmtrc.json '!test/skillgym/.skillgym-results/**'",
     "fallow": "fallow audit --base origin/main",
     "fallow:all": "fallow --summary",
     "fallow:baseline": "(fallow dead-code --save-baseline fallow-baselines/dead-code.json --summary || true) && (fallow health --save-baseline fallow-baselines/health.json --summary || true)",

scripts/__tests__/help-conformance-bench.test.ts

@@ -214,8 +214,12 @@ test('runtime-output cases do not coach the command that their output should imp
     [
       'settle-diff-is-observation',
       'sample-output-settled-diff-next-target',
+      'metamorphic-settled-diff-next-target-notes',
       'sample-output-not-settled-needs-observe',
-      'sample-output-recoverable-failure-retries-in-session',
+      'sample-output-device-in-use-reuses-session',
+      'sample-output-stale-ref-resnapshots',
+      'sample-output-ambiguous-match-reobserves',
+      'sample-output-app-not-installed-discovers-first',
     ].join(','),
     '--runner',
     'claude:test-model',

scripts/__tests__/help-conformance-sample-outputs.test.ts

@@ -0,0 +1,216 @@
+import assert from 'node:assert/strict';
+import { test } from 'vitest';
+import {
+  AMBIGUOUS_MATCH_SAMPLE,
+  APP_NOT_INSTALLED_SAMPLE,
+  DEVICE_IN_USE_SAMPLE,
+  NOT_SETTLED_SAMPLE,
+  PRIVATE_AX_RECOVERY_SAMPLE,
+  SETTLE_DIFF_SAMPLE,
+  SETTLE_DIFF_SAMPLE_NOTES,
+  SETTLE_TAIL_SAMPLE,
+  STALE_REF_SAMPLE,
+} from '../help-conformance-sample-outputs.mjs';
+import { interactionCliOutputFormatters } from '../../src/commands/interaction/output.ts';
+import { NEVER_SETTLED_HINT } from '../../src/commands/interaction/runtime/settle.ts';
+import { buildAmbiguousMatchError } from '../../src/daemon/handlers/find.ts';
+import { refMutationAdmissionResponse } from '../../src/daemon/handlers/interaction-ref-policy.ts';
+import { buildDeviceInUseBySessionError } from '../../src/daemon/handlers/session-open.ts';
+import { resolveRefStalenessWarning } from '../../src/daemon/session-snapshot.ts';
+import type { SessionState } from '../../src/daemon/types.ts';
+import { buildAppNotInstalledError } from '../../src/platforms/apple/core/app-resolution.ts';
+import { AppError, normalizeError } from '../../src/kernel/errors.ts';
+import type { SnapshotQualityVerdict } from '../../src/snapshot/snapshot-quality.ts';
+import { renderSnapshotQualityWarnings } from '../../src/snapshot/snapshot-quality.ts';
+import { formatSnapshotText, printHumanError } from '../../src/utils/output.ts';
+
+// Every quiz case in scripts/help-conformance-cases.mjs quotes "captured"
+// agent-device output. These tests rebuild each quoted sample through the real
+// production renderer, so a rendering or message change fails HERE instead of
+// leaving the benchmark grading models against output the CLI no longer
+// prints — the drift class that made hand-transcribed samples untrustworthy.
+
+const formatPress = (result: Record<string, unknown>) =>
+  interactionCliOutputFormatters.press({ input: {}, result });
+
+const formatFill = (result: Record<string, unknown>) =>
+  interactionCliOutputFormatters.fill({ input: {}, result });
+
+function renderHumanError(error: AppError): string {
+  const lines: string[] = [];
+  const originalWrite = process.stderr.write;
+  process.stderr.write = ((chunk: string | Uint8Array) => {
+    lines.push(String(chunk));
+    return true;
+  }) as typeof process.stderr.write;
+  try {
+    printHumanError(normalizeError(error));
+  } finally {
+    process.stderr.write = originalWrite;
+  }
+  return lines.join('').trimEnd();
+}
+
+test('settle tail sample matches the press renderer, including ADR 0014 pinning', () => {
+  const output = formatPress({
+    ref: 'e37',
+    x: 203,
+    y: 88,
+    settle: {
+      settled: true,
+      waitedMs: 540,
+      refsGeneration: 5,
+      diff: {
+        summary: { additions: 0, removals: 1, unchanged: 15 },
+        lines: [{ kind: 'removed', text: '@e50 [text] "Suggested for you"' }],
+      },
+      tail: [
+        { ref: 'e64', role: 'text-field', label: 'Search' },
+        { ref: 'e65', role: 'text', label: 'Recent searches' },
+        { ref: 'e12', role: 'tab', label: 'Home' },
+        { ref: 'e40', role: 'tab', label: 'Profile' },
+      ],
+    },
+  });
+  assert.equal(output.text, SETTLE_TAIL_SAMPLE.output);
+});
+
+test('settled diff sample matches the fill renderer', () => {
+  const output = formatFill({
+    text: 'callstack',
+    message: 'Filled 9 chars',
+    settle: {
+      settled: true,
+      waitedMs: 610,
+      refsGeneration: 6,
+      diff: {
+        summary: { additions: 2, removals: 0, unchanged: 18 },
+        lines: [
+          { kind: 'added', text: '@e64 [button] "@callstack.com"' },
+          { kind: 'added', text: '@e65 [text] "Callstack"' },
+        ],
+      },
+    },
+  });
+  assert.equal(output.text, SETTLE_DIFF_SAMPLE.output);
+});
+
+test('metamorphic settled diff sample matches the fill renderer', () => {
+  const output = formatFill({
+    text: 'groceries',
+    message: 'Filled 9 chars',
+    settle: {
+      settled: true,
+      waitedMs: 480,
+      refsGeneration: 3,
+      diff: {
+        summary: { additions: 2, removals: 0, unchanged: 11 },
+        lines: [
+          { kind: 'added', text: '@e21 [button] "Groceries list"' },
+          { kind: 'added', text: '@e22 [text] "3 items"' },
+        ],
+      },
+    },
+  });
+  assert.equal(output.text, SETTLE_DIFF_SAMPLE_NOTES.output);
+});
+
+test('not-settled sample matches the press renderer and the production hint', () => {
+  const output = formatPress({
+    ref: 'e12',
+    x: 166,
+    y: 240,
+    settle: { settled: false, waitedMs: 10_000, hint: NEVER_SETTLED_HINT },
+  });
+  assert.equal(output.text, NOT_SETTLED_SAMPLE.output);
+});
+
+test('private-ax recovery sample matches the snapshot renderer and quality warning', () => {
+  const nodes = [
+    {
+      index: 1,
+      ref: 'e5',
+      type: 'Button',
+      label: 'Search',
+      interactive: true,
+      rect: { x: 20, y: 120, width: 200, height: 44 },
+    },
+    {
+      index: 2,
+      ref: 'e8',
+      type: 'Tab',
+      label: 'Home',
+      selected: true,
+      interactive: true,
+      rect: { x: 0, y: 780, width: 100, height: 60 },
+    },
+  ];
+  const verdict = { state: 'recovered', backend: 'private-ax' } as SnapshotQualityVerdict;
+  const text = formatSnapshotText({
+    nodes,
+    backend: 'private-ax',
+    snapshotQuality: verdict,
+    warnings: renderSnapshotQualityWarnings(verdict, nodes),
+    interactiveOnly: true,
+  });
+  assert.equal(text.trimEnd(), PRIVATE_AX_RECOVERY_SAMPLE.output);
+});
+
+test('device-in-use sample matches the real session-open producer', () => {
+  const owningSession = { name: 'checkout' } as SessionState;
+  const device = { id: 'SIM-001', name: 'iPhone 17 Pro' } as Parameters<
+    typeof buildDeviceInUseBySessionError
+  >[1];
+  const response = buildDeviceInUseBySessionError(owningSession, device);
+  assert.ok(!response.ok, 'the by-session conflict must be an error response');
+  const rendered = renderHumanError(
+    new AppError(
+      response.error.code as ConstructorParameters<typeof AppError>[0],
+      response.error.message,
+      response.error.details,
+    ),
+  );
+  assert.equal(rendered, DEVICE_IN_USE_SAMPLE.output);
+});
+
+test('stale-ref sample matches the real admission rejection and staleness hint', () => {
+  const session = { refFrameGeneration: 7 } as SessionState;
+  const response = refMutationAdmissionResponse({
+    session,
+    ref: '@e12',
+    mintedGeneration: 5,
+    staleRefsWarning: resolveRefStalenessWarning({ session, ref: '@e12', mintedGeneration: 5 }),
+  });
+  assert.ok(response && !response.ok, 'a superseded pin must be rejected');
+  const rendered = renderHumanError(
+    new AppError(
+      response.error.code as ConstructorParameters<typeof AppError>[0],
+      response.error.message,
+      response.error.details,
+    ),
+  );
+  assert.equal(rendered, STALE_REF_SAMPLE.output);
+});
+
+test('ambiguous-match sample matches the real find producer and default hint', () => {
+  const matches = [
+    { ref: 'e2', label: 'Follow' },
+    { ref: 'e5', label: 'Follow' },
+    { ref: 'e9', label: 'Follow' },
+  ] as Parameters<typeof buildAmbiguousMatchError>[0];
+  const response = buildAmbiguousMatchError(matches, 'text', 'Follow');
+  assert.ok(!response.ok, 'an ambiguous find must be an error response');
+  const rendered = renderHumanError(
+    new AppError(
+      response.error.code as ConstructorParameters<typeof AppError>[0],
+      response.error.message,
+      response.error.details,
+    ),
+  );
+  assert.equal(rendered, AMBIGUOUS_MATCH_SAMPLE.output);
+});
+
+test('app-not-installed sample matches the real app-resolution producer and default hint', () => {
+  const rendered = renderHumanError(buildAppNotInstalledError('Shoply'));
+  assert.equal(rendered, APP_NOT_INSTALLED_SAMPLE.output);
+});

scripts/__tests__/help-conformance-topic-coverage.test.ts

@@ -0,0 +1,67 @@
+import assert from 'node:assert/strict';
+import { test } from 'vitest';
+import { CASES } from '../help-conformance-cases.mjs';
+import { helpTopicIds } from '../../src/cli/parser/cli-help.ts';
+
+// "What enumerates N": benchmark cases are keyed to help topics, and this gate
+// keys the case list to the topic registry itself. A new help topic must gain
+// a benchmark case (docs: [..., '<topic>']) or an explicit waiver here — the
+// waiver names why the topic's guidance is not yet benchmarked, so uncovered
+// topics are a visible decision instead of silent drift.
+const WAIVED_TOPICS: Record<string, string> = {
+  cdp: 'JS-heap forensics niche; add cases when heap-guidance regressions show up in practice.',
+  macos: 'macOS surface guidance is thin and stable; no observed planning regressions yet.',
+  maestro: 'Compatibility reference, not a planning loop; conformance is oracle-tested instead.',
+  'physical-device': 'Needs device-specific setup guidance; no portable planning task defined yet.',
+  'react-devtools':
+    'Profiling-window guidance; add cases when render-diagnosis planning regresses.',
+  remote: 'Remote/cloud lease setup; niche until remote workflows are benchmarked end to end.',
+};
+
+const FIRST_SCREEN_DOC = '--help:first30';
+
+test('every case doc id is the first screen or a real help topic', () => {
+  const topics = new Set(helpTopicIds());
+  for (const testCase of CASES) {
+    for (const doc of testCase.docs) {
+      assert.ok(
+        doc === FIRST_SCREEN_DOC || topics.has(doc),
+        `case "${testCase.id}" references unknown help doc "${doc}"`,
+      );
+    }
+  }
+});
+
+test('every help topic has a benchmark case or an explicit waiver', () => {
+  const covered = new Set(CASES.flatMap((testCase) => testCase.docs));
+  const uncovered = helpTopicIds().filter(
+    (topic) => !covered.has(topic) && !(topic in WAIVED_TOPICS),
+  );
+  assert.deepEqual(
+    uncovered,
+    [],
+    'new help topics need a benchmark case in scripts/help-conformance-cases.mjs or a waiver above',
+  );
+});
+
+test('waivers only name real, uncovered topics', () => {
+  const topics = new Set(helpTopicIds());
+  const covered = new Set(CASES.flatMap((testCase) => testCase.docs));
+  for (const [topic, reason] of Object.entries(WAIVED_TOPICS)) {
+    assert.ok(topics.has(topic), `waived topic "${topic}" no longer exists — remove the waiver`);
+    assert.ok(
+      !covered.has(topic),
+      `waived topic "${topic}" is now covered by a case — remove the waiver`,
+    );
+    assert.ok(reason.trim().length > 0, `waiver for "${topic}" needs a reason`);
+  }
+});
+
+test('the first help screen is exercised by every case family', () => {
+  for (const testCase of CASES) {
+    assert.ok(
+      testCase.docs.includes(FIRST_SCREEN_DOC),
+      `case "${testCase.id}" must include the first-screen doc — it is the only text every runner sees before choosing a topic`,
+    );
+  }
+});