test: frozen replay-compat corpus with expected verdicts (#1417)

.github/workflows/ci.yml

@@ -216,6 +216,26 @@ jobs:
       - name: Check for production-unused exports
         run: pnpm check:production-exports
 
+  replay-compat-provenance:
+    # The frozen replay-compat corpus (#1417) claims each entry was published by
+    # a released tag. Only a full-history checkout can re-derive that claim, so
+    # this job exists separately from the shallow-clone-safe unit lane.
+    name: Replay-Compat Provenance
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    steps:
+      - name: Checkout
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          fetch-depth: 0
+          fetch-tags: true
+
+      - name: Setup toolchain
+        uses: ./.github/actions/setup-node-pnpm
+
+      - name: Verify corpus entries against their released blobs
+        run: pnpm check:replay-compat
+
   coverage:
     # Runs the full unit + provider-integration suites under coverage with
     # thresholds, so a separate unit-tests job would rerun the same tests.

docs/agents/testing.md

@@ -19,6 +19,7 @@ The mapping it encodes, for when you need to run a gate directly or reason about
 | CLI help/guidance (`src/cli/parser/cli-help.ts`, `src/cli-schema/`) | `pnpm exec vitest run src/cli/parser/__tests__ src/cli-schema/command-schema-guards.test.ts scripts/__tests__` — the `scripts/__tests__` gates enforce help-topic benchmark coverage and pin the bench's quoted CLI samples to the real renderers |
 | Help benchmark cases (`scripts/help-conformance-*.mjs`) | `pnpm exec vitest run scripts/__tests__` (deterministic gates); model-backed: `pnpm bench:help-conformance` (paid LLM calls, local only) |
 | SkillGym prompts/assertions | `pnpm test:skillgym:case <case-id>` (broad: `pnpm test:skillgym`, filter with `-- --tag fixture-smoke` or `-- --tag skill-guidance`) — agentic routing + local-help-consumption proof only; command-planning knowledge checks belong in the help bench |
+| `.ad` grammar (`src/replay/script.ts`, gesture arity, replay vars) | `pnpm exec vitest run --project unit-core test/replay-compat` — the frozen replay-compat corpus asserts which released script surfaces still parse; a flipped verdict is edited in `test/replay-compat/manifest.ts`, never in the script. Adding or re-pinning a corpus entry also runs `pnpm check:replay-compat`, which re-derives each entry from its release tag in git history |
 | Anything in `src/`, `test/`, `skills/` | `pnpm format` |
 
 Two traps worth naming:

package.json

@@ -122,6 +122,7 @@
     "depgraph:test": "node --experimental-strip-types --test scripts/depgraph/model.test.ts",
     "check:production-exports": "fallow dead-code --config fallow-production-exports.json --production --unused-exports --fail-on-issues",
     "check:bundle-owner-files": "node --experimental-strip-types scripts/check-bundle-owner-files.ts",
+    "check:replay-compat": "node --experimental-strip-types scripts/check-replay-compat-provenance.ts",
     "check:freerange": "fr",
     "check:quick": "pnpm lint && pnpm typecheck",
     "sync:mcp-metadata": "node scripts/sync-mcp-metadata.mjs",

scripts/check-affected/checks.ts

@@ -152,6 +152,15 @@ export const CHECK_CATALOG: readonly CheckSpec[] = [
     ciJobs: [],
     localRunnable: true,
   },
+  {
+    id: 'replay-compat',
+    label: 'Replay-compat corpus provenance (released blobs)',
+    kind: { type: 'script', script: 'check:replay-compat' },
+    // Needs full history and tags, so it runs in its own fetch-depth: 0 job
+    // rather than inside the shallow-clone-safe unit lane.
+    ciJobs: ['Replay-Compat Provenance'],
+    localRunnable: true,
+  },
 ];
 
 export function getCheckSpec(id: CheckId): CheckSpec {

scripts/check-affected/model.test.ts

@@ -114,6 +114,19 @@ test('a non-.ts fixture under an owned root fails open (format alone is not owne
   assert.equal(result.failOpenReasons[0]?.rule, 'ambiguous-path');
 });
 
+test('a frozen replay-compat corpus script selects the unit lane and the provenance verifier', () => {
+  const result = plan(['test/replay-compat/scripts/examples/gesture-lab.v0.16.8.ad']);
+  assert.equal(result.failOpen, false);
+  assert.ok(result.checks.includes('unit'));
+  assert.ok(result.checks.includes('replay-compat'));
+});
+
+test('a replay-compat manifest edit selects the provenance verifier', () => {
+  const result = plan(['test/replay-compat/manifest.ts']);
+  assert.equal(result.failOpen, false);
+  assert.ok(result.checks.includes('replay-compat'));
+});
+
 test('skills guidance change selects format + skillgym, not docs-only', () => {
   const result = plan(['skills/agent-device/SKILL.md']);
   assert.equal(result.failOpen, false);
@@ -175,6 +188,7 @@ test('every catalog command resolves against package scripts', () => {
     'build:macos-helper': 'x',
     'test:smoke:web': 'x',
     'test:skillgym': 'x',
+    'check:replay-compat': 'x',
   };
   for (const spec of CHECK_CATALOG) {
     const command = resolveCommand(spec, scripts, 'origin/main');

scripts/check-affected/model.ts

@@ -40,7 +40,8 @@ export type CheckId =
   | 'android-helpers'
   | 'macos-helper'
   | 'web-smoke'
-  | 'skillgym';
+  | 'skillgym'
+  | 'replay-compat';
 
 // The complete local check universe. A fail-open plan selects all of these;
 // keep it in sync with the catalog in checks.ts (asserted by the self-test).
@@ -63,6 +64,7 @@ export const ALL_CHECKS: readonly CheckId[] = [
   'macos-helper',
   'web-smoke',
   'skillgym',
+  'replay-compat',
 ];
 
 export type SelectionReason = {
@@ -228,6 +230,34 @@ const nodeIntegrationOwnership: OwnershipRule = ({ file }) =>
     ? [reason('integration-node', file, 'node-integration', 'node --test integration smoke')]
     : [];
 
+// The frozen replay-compat corpus (#1417). `.ad` fixture data would otherwise
+// fail open on its extension: its only consumer is the unit-lane corpus test.
+// Any corpus change — script or manifest — also runs the history-backed
+// provenance verifier, which is the only gate that can prove an entry's blob
+// really came from the release tag it names.
+const replayCompatOwnership: OwnershipRule = ({ file }) => {
+  if (!file.startsWith('test/replay-compat/')) return [];
+  const selections = [
+    reason(
+      'replay-compat',
+      file,
+      'own:replay-compat-provenance',
+      'corpus provenance is re-derived from released git blobs',
+    ),
+  ];
+  if (file.endsWith('.ad')) {
+    selections.push(
+      reason(
+        'unit',
+        file,
+        'own:replay-compat',
+        'frozen replay-compat corpus is asserted by the unit-lane corpus test',
+      ),
+    );
+  }
+  return selections;
+};
+
 // SkillGym validates skill guidance (`skills/`) and owns its harness
 // (`test/skillgym/`); the Testing Matrix in docs/agents/testing.md routes
 // skill-prompt/assertion changes here.
@@ -294,6 +324,7 @@ const OWNERSHIP_RULES: readonly OwnershipRule[] = [
   srcProdGate,
   vitestRelatedOwnership,
   nodeIntegrationOwnership,
+  replayCompatOwnership,
   skillgymOwnership,
   buildOwnership,
 ];

scripts/check-affected/run.test.ts

@@ -101,6 +101,7 @@ const ALL_SCRIPTS: Record<string, string> = {
   'test:integration:node': 'x',
   'test:integration:progress:check': 'x',
   'test:skillgym': 'x',
+  'check:replay-compat': 'x',
 };
 
 const ARGS = { base: 'origin/main', head: 'HEAD', json: false, run: true };

scripts/check-replay-compat-provenance.ts

@@ -0,0 +1,84 @@
+/**
+ * Re-derives the frozen replay-compat corpus (#1417) from git history: every
+ * mined entry must still be the exact blob a RELEASED tag published at the path
+ * it claims, and every tag the corpus cites must be a tag that was actually cut.
+ *
+ * The unit gate (`test/replay-compat/corpus.test.ts`) hashes the checked-in
+ * bytes against the same object ids, which catches a rewritten script; only this
+ * check can say those object ids came from a release rather than from whoever
+ * wrote the manifest line.
+ */
+
+import fs from 'node:fs';
+import path from 'node:path';
+import {
+  REPLAY_COMPAT_CORPUS,
+  REPLAY_COMPAT_RELEASED_TAGS,
+} from '../test/replay-compat/manifest.ts';
+import { findProvenanceKindViolations } from '../test/replay-compat/provenance-rules.ts';
+import { runCmdSync } from '../src/utils/exec.ts';
+
+const repoRoot = path.resolve(import.meta.dirname, '..');
+const corpusRoot = path.join(repoRoot, 'test', 'replay-compat');
+
+function git(args: string[]): string {
+  return runCmdSync('git', args, { cwd: repoRoot, allowFailure: true }).stdout.trim();
+}
+
+if (git(['rev-parse', '--is-shallow-repository']) === 'true') {
+  throw new Error(
+    'Corpus provenance needs full history and tags. Run `git fetch --unshallow --tags` first.',
+  );
+}
+
+// An entry that escaped `mined` would also escape the history check below, so
+// the area rule is enforced here as well as in the unit lane.
+const failures: string[] = [...findProvenanceKindViolations(REPLAY_COMPAT_CORPUS)];
+
+const existingTags = new Set(git(['tag', '--list']).split('\n').filter(Boolean));
+if (existingTags.size === 0) {
+  throw new Error('No release tags found. Run `git fetch --tags` first.');
+}
+for (const tag of REPLAY_COMPAT_RELEASED_TAGS) {
+  if (!existingTags.has(tag)) {
+    failures.push(`${tag} is cited as a released version but is not a tag in this repository`);
+  }
+}
+
+for (const entry of REPLAY_COMPAT_CORPUS) {
+  if (!REPLAY_COMPAT_RELEASED_TAGS.includes(entry.recordedBy)) {
+    failures.push(`${entry.id}: recordedBy ${entry.recordedBy} is not in REPLAY_COMPAT_RELEASED_TAGS`);
+    continue;
+  }
+  if (entry.provenance.kind !== 'mined') continue;
+
+  const { path: sourcePath, blob } = entry.provenance;
+  const historical = git(['rev-parse', `${entry.recordedBy}:${sourcePath}`]);
+  if (historical !== blob) {
+    failures.push(
+      `${entry.id}: ${entry.recordedBy}:${sourcePath} is ${historical || '<missing>'}, manifest pins ${blob}`,
+    );
+    continue;
+  }
+  const frozen = fs.readFileSync(path.join(corpusRoot, entry.file));
+  const released = runCmdSync('git', ['cat-file', 'blob', blob], {
+    cwd: repoRoot,
+    binaryStdout: true,
+  }).stdoutBuffer;
+  if (!released?.equals(frozen)) {
+    failures.push(`${entry.id}: ${entry.file} no longer matches the released blob ${blob}`);
+  }
+}
+
+if (failures.length > 0) {
+  throw new Error(
+    `Replay-compat corpus provenance is broken (see test/replay-compat/README.md):\n${failures
+      .map((failure) => `- ${failure}`)
+      .join('\n')}`,
+  );
+}
+
+const mined = REPLAY_COMPAT_CORPUS.filter((entry) => entry.provenance.kind === 'mined').length;
+process.stdout.write(
+  `Verified ${mined} mined replay-compat scripts against their released blobs across ${REPLAY_COMPAT_RELEASED_TAGS.length} tags (${REPLAY_COMPAT_CORPUS.length - mined} derived entries are digest-pinned).\n`,
+);

test/replay-compat/README.md

@@ -0,0 +1,100 @@
+# Replay-compat corpus
+
+`.ad` scripts as **released** versions of agent-device wrote them, each paired with the verdict
+today's parser owes it: `parses`, or `fails` with an error code and the migration hint substring.
+
+`corpus.test.ts` runs in `unit-core` and asserts every verdict through `parseReplayInput` — the same
+composition `replay`/`test` use — so a grammar change cannot quietly alter which historical script
+surface still loads. #1417.
+
+## The freeze rule
+
+**Never edit a script under `scripts/` to make a parser change pass.**
+
+A corpus script is a recording that already exists in the wild; editing it deletes the evidence the
+gate is here to keep, and the pinned identities below make an edited script fail rather than pass.
+When a grammar change flips a verdict:
+
+1. Leave the script byte-identical.
+2. Update its `verdict` in `manifest.ts` in the same PR.
+3. Say in the PR description which historical surface broke, and why the new refusal is the intended
+   migration rather than a regression.
+
+A verdict that changes from `parses` to `fails` is a compatibility break by definition. That is
+allowed — the corpus only insists it be deliberate, reviewed, and paired with a migration hint good
+enough to put in front of whoever recorded the script.
+
+## What is in the corpus
+
+- `scripts/integration/` and `scripts/examples/` — mined from the git history of
+  `test/integration/replays` and `examples/test-app/replays`.
+- `scripts/docs/` — surfaces the released grammar wrote that those two suites never exercised
+  (`${VAR}` parameterization, reserved/ordering `env` refusals, `wait stable` and landmark waits,
+  `target-v1` annotations, retired `gesture rotate` velocity and `gesture swipe` durationMs).
+
+## What earns an entry
+
+This is a **parser**-compatibility corpus, not a device matrix. An entry has to be the only witness
+of something:
+
+- a shipped syntactic form no other entry's bytes contain, or
+- a distinct migration refusal (error code + hint) no other entry already provokes.
+
+So the same flow recorded on another platform, or re-recorded at an adjacent release with only
+coordinates and labels changed, does **not** get an entry: it costs maintenance and adds no parser
+leverage. Each entry's `note` says which form or refusal it is the sole witness of; if a new entry
+cannot claim one, it does not belong here. Keep the corpus in the tens of entries, not a mirror of
+the replay-fixture tree.
+
+## How "frozen" and "released" are enforced, not asserted
+
+Provenance lives in `manifest.ts`. `recordedBy` is the released tag whose grammar produced the
+surface, and `provenance` pins the bytes:
+
+- `mined` entries carry the **git object id of the historical blob**. `corpus.test.ts` hashes the
+  checked-in bytes and must reproduce that id, so a rewritten script cannot be made green by editing
+  the manifest to match — the id is only obtainable from the released content.
+- `derived` entries (`scripts/docs/`) have no historical blob to point at, so their bytes are pinned
+  by SHA-256. Only the bytes are machine-checked: the `from` citation naming the released grammar or
+  doc that emitted the surface is **reviewer-verified**, so a derived entry has to be reviewed against
+  its cited source at that tag (`git show <tag>:<path>`) before it lands.
+
+**Two sources, never invention.** The _form_ comes from the cited release — the script bytes are what
+that tag's grammar or docs emitted. The _verdict_ comes from today's parser, because a migration
+refusal for a retired form did not exist at the tag that emitted it (v0.16.8 wrote
+`gesture rotate <deg> <x> <y> <velocity>`; the hint telling you to drop the velocity is current
+output). So copy the form from the release and paste the code/hint from an observed run — never
+paraphrase either.
+
+The kind is not a free choice: `provenance-rules.ts` fixes it by corpus area (`scripts/integration/`
+and `scripts/examples/` must stay `mined`, `scripts/docs/` is `derived`), so an edited script cannot
+be relabelled `derived`, re-pinned by digest, and thereby skipped by the history check. Both the unit
+test and the verifier enforce that rule, and a new area must declare its kind before entries can live
+under it.
+
+`pnpm check:replay-compat` closes the other half: it re-derives every mined id from git history
+(`git rev-parse <recordedBy>:<path>`) and checks every cited tag against `git tag --list`, so an
+entry cannot claim a version that was never cut. It needs full history and tags, so it runs in its
+own `Replay-Compat Provenance` CI job (`fetch-depth: 0`) rather than inside the shallow-clone-safe
+unit lane, and any change under `test/replay-compat/` selects it in `pnpm check:affected`.
+
+**Released surfaces only.** A grammar state that only ever existed between commits is not compat
+surface — `git tag --contains <commit>` decides, and unreleased shapes stay out (AGENTS.md,
+"Unreleased API surface dies free"). The corpus is baselined against tags up to v0.20.0.
+
+## Adding an entry
+
+1. Find the surface at a release tag (`git show <tag>:<path>`, or the released grammar/docs that
+   emitted it) and copy it verbatim into `scripts/<area>/<name>.<tag>.ad`.
+2. Add a `manifest.ts` entry with `recordedBy`, `provenance`, `covers`, and a `note` naming the form
+   or refusal it is the sole witness of. For a mined entry the blob id is `git rev-parse <tag>:<path>`;
+   for a derived one it is `shasum -a 256 <file>`. Add the tag to `REPLAY_COMPAT_RELEASED_TAGS` if it
+   is new.
+3. Take the `verdict` from what the current parser does with those bytes: start from
+   `verdict: { kind: 'parses' }`, run the suite, and if it refuses, paste the reported code and hint
+   substring verbatim. Do not transcribe a hint from a release, an issue, or memory — the refusal is
+   produced by today's parser, not by the tag that emitted the form.
+4. Run `pnpm exec vitest run --project unit-core test/replay-compat` and `pnpm check:replay-compat`.
+
+Add an entry when a change retires, renames, or narrows a `.ad` form — the corpus is the record of
+what that costs someone with a saved recording.