test: nightly parser fuzz lane — parser input fails as typed AppErrors, never hangs (#1414)

.github/workflows/replays-nightly.yml

@@ -4,6 +4,14 @@ on:
   schedule:
     - cron: '0 3 * * *'
   workflow_dispatch:
+    inputs:
+      fuzz-iterations:
+        description: 'Parser fuzz cases per target'
+        required: false
+        default: '50000'
+      fuzz-seed:
+        description: 'Parser fuzz PRNG seed (defaults to the run number)'
+        required: false
 
 permissions:
   contents: read
@@ -13,6 +21,73 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
+  # Parser fuzz lane (#1414): hostile input into parseArgs, selector parsing, .ad replay
+  # scripts, batch --steps JSON, and the Maestro compat parser. One invariant — every
+  # rejection is a typed AppError with a non-empty hint, and no case hangs. Device-free, so
+  # it rides this nightly rather than owning a workflow; the seed varies per run so the lane
+  # keeps exploring, and anything it catches is appended to the checked-in corpus that the
+  # unit lane replays (scripts/fuzz/corpus/regressions.json).
+  nightly-parser-fuzz:
+    name: Parser Fuzz Lane
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    steps:
+      - name: Checkout
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+
+      - name: Setup toolchain
+        uses: ./.github/actions/setup-node-pnpm
+
+      # A lane whose classifier or watchdog regressed would pass forever. This runs the
+      # broken-on-purpose targets first and fails unless each violation is caught.
+      - name: Self-check the harness
+        run: pnpm fuzz:parsers --self-check --artifact-dir .tmp/fuzz/self-check
+
+      # Runs even when the self-check fails, so the lane always produces a fuzz envelope too
+      # (a step that never runs writes no envelope, and monitoring cannot tell that apart from
+      # a lane that went dark). The job still fails because both steps report their status.
+      - name: Fuzz parsers
+        if: always()
+        env:
+          FUZZ_ITERATIONS: ${{ github.event.inputs.fuzz-iterations || '50000' }}
+          FUZZ_SEED: ${{ github.event.inputs.fuzz-seed || github.run_number }}
+        run: |
+          pnpm fuzz:parsers \
+            --iterations "$FUZZ_ITERATIONS" \
+            --seed "$FUZZ_SEED" \
+            --artifact-dir .tmp/fuzz/run
+
+      # Uploaded on pass as well as failure: each subdirectory of .tmp/fuzz holds a
+      # run-envelope.json on #1430's shared lane contract (scripts/lib/lane-envelope.ts) —
+      # commit/ref/run provenance, tool + config hash, seed, result — which is what freshness
+      # monitoring reads.
+      - name: Upload run envelopes and failing cases
+        if: always()
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+        with:
+          name: parser-fuzz-run-${{ github.run_id }}-${{ github.run_attempt }}
+          path: .tmp/fuzz
+          if-no-files-found: warn
+
+      # Never fails the job on its own: a missing envelope is reported as such rather than
+      # masking the real failure with a `cat` error.
+      - name: Summarize
+        if: always()
+        run: |
+          {
+            echo '### Parser fuzz lane'
+            found=0
+            for envelope in .tmp/fuzz/*/run-envelope.json; do
+              [ -f "$envelope" ] || continue
+              found=1
+              echo "#### $envelope"
+              echo '```json'
+              cat "$envelope"
+              echo '```'
+            done
+            [ "$found" = 1 ] || echo 'No run envelope was produced — the lane failed before it could run.'
+          } >> "$GITHUB_STEP_SUMMARY"
+
   nightly-android:
     name: Android Replay Suite
     runs-on: ubuntu-latest

docs/agents/testing.md

@@ -179,6 +179,48 @@ pnpm mutation:baseline                  # full sweep, then record it (reviewed c
   written on every exit path, including a crash before any mutant runs: an absent envelope would be
   indistinguishable from a lane that never ran.
 
+## Parser fuzz lane
+
+`pnpm fuzz:parsers` feeds generated hostile input to `parseArgs`, selector parsing,
+`parseReplayScriptDetailed`, `batch --steps` JSON, and the Maestro compat parser, and enforces one
+invariant: every rejection is a typed `AppError` whose normalized `hint` is non-empty, and no case
+hangs (a worker-thread watchdog attributes a stall to the exact input).
+
+```sh
+pnpm fuzz:parsers                                  # all targets, 2,000 cases each, seed 1
+pnpm fuzz:parsers --target selector --iterations 50000 --seed 7
+pnpm fuzz:parsers --input-file .tmp/fuzz/<case>.json                  # repro a saved case
+pnpm fuzz:parsers --input-file .tmp/fuzz/<case>.json --append-corpus  # …and pin it
+pnpm fuzz:parsers --self-check                     # require the harness to still fail
+```
+
+The generating run is nightly (`Parser Fuzz Lane` in `.github/workflows/replays-nightly.yml`, seeded
+by the run number). Every terminal path — pass, fail, `--self-check`, or a crash in the harness
+itself — writes `<artifact-dir>/run-envelope.json` on the shared lane contract
+(`scripts/lib/lane-envelope.ts`, #1430), with the lane's own facts under `data`: mode, per-target
+cases/failures/durations, failures, repro commands, and `stage` (`error` marks a run that could not
+complete itself, since the shared `result` is only `pass`/`fail`). `configHash` hashes the modules
+that decide a case set, so "the same seed means different inputs now" is distinguishable from "the
+parsers changed". The self-check and fuzz steps write to separate artifact subdirectories and both
+run unconditionally; the step summary prints each envelope it finds and never fails on a missing
+file.
+
+Cases come from fast-check arbitraries (`scripts/fuzz/arbitraries.ts`) built on the hazard vocabulary
+shared with `src/__tests__/test-utils/property-arbitraries.ts`, so a hazard added for the property
+suite reaches the fuzz lane too — and a counterexample is reported **shrunk**, with fast-check's seed
+and replay path printed alongside the saved artifact.
+
+A nightly discovery reaches the unit lane by promotion, not hand-editing: the printed
+`promote:` command re-runs the downloaded artifact and appends it to
+`scripts/fuzz/corpus/regressions.json`, which `scripts/fuzz/corpus-replay.test.ts` replays on every
+PR — through the same worker watchdog, so a promoted hang case fails against its per-case budget
+instead of wedging the unit job. `scripts/fuzz/harness.test.ts` covers the harness itself — an
+untyped throw, an empty hint, and a wedged worker must each be reported, startup time is never charged against the per-case budget, and
+every mode writes an envelope — using the broken-on-purpose targets in
+`scripts/fuzz/self-check-targets.ts` (also what `--self-check` runs in CI), so a regressed classifier
+or watchdog cannot pass silently. Adding a parser to the lane means adding a target to
+`scripts/fuzz/targets.ts` — nothing else.
+
 ## Live web smoke
 
 The live web platform smoke runs the public built CLI against a local fixture page through the managed web backend:

package.json

@@ -114,8 +114,9 @@
     "mutation:affected": "node --experimental-strip-types scripts/mutation/run.ts --affected",
     "mutation:test": "node --experimental-strip-types --test scripts/mutation/*.test.ts",
     "lint": "oxlint . --deny-warnings",
-    "format": "node ./node_modules/oxfmt/bin/oxfmt --write src test skills scripts/mutation scripts/lib scripts/help-conformance-bench.mjs scripts/help-conformance-case-checks.mjs scripts/help-conformance-command-validator.ts scripts/help-conformance-expectations.mjs scripts/help-conformance-plan-validator.mjs scripts/help-conformance-runner-output.mjs scripts/help-conformance-summary.mjs scripts/help-conformance-cases.mjs scripts/help-conformance-sample-outputs.mjs scripts/__tests__/help-conformance-bench.test.ts scripts/__tests__/help-conformance-sample-outputs.test.ts scripts/__tests__/help-conformance-topic-coverage.test.ts package.json tsconfig.json tsconfig.lib.json tsdown.config.ts vitest.config.ts vitest.mutation.config.ts .github/actions/setup-node-pnpm/action.yml .oxlintrc.json .oxfmtrc.json '!test/skillgym/.skillgym-results/**'",
-    "format:check": "node ./node_modules/oxfmt/bin/oxfmt --check src test skills scripts/mutation scripts/lib scripts/help-conformance-bench.mjs scripts/help-conformance-case-checks.mjs scripts/help-conformance-command-validator.ts scripts/help-conformance-expectations.mjs scripts/help-conformance-plan-validator.mjs scripts/help-conformance-runner-output.mjs scripts/help-conformance-summary.mjs scripts/help-conformance-cases.mjs scripts/help-conformance-sample-outputs.mjs scripts/__tests__/help-conformance-bench.test.ts scripts/__tests__/help-conformance-sample-outputs.test.ts scripts/__tests__/help-conformance-topic-coverage.test.ts package.json tsconfig.json tsconfig.lib.json tsdown.config.ts vitest.config.ts vitest.mutation.config.ts .github/actions/setup-node-pnpm/action.yml .oxlintrc.json .oxfmtrc.json '!test/skillgym/.skillgym-results/**'",
+    "format": "node ./node_modules/oxfmt/bin/oxfmt --write src test skills scripts/fuzz scripts/mutation scripts/lib scripts/help-conformance-bench.mjs scripts/help-conformance-case-checks.mjs scripts/help-conformance-command-validator.ts scripts/help-conformance-expectations.mjs scripts/help-conformance-plan-validator.mjs scripts/help-conformance-runner-output.mjs scripts/help-conformance-summary.mjs scripts/help-conformance-cases.mjs scripts/help-conformance-sample-outputs.mjs scripts/__tests__/help-conformance-bench.test.ts scripts/__tests__/help-conformance-sample-outputs.test.ts scripts/__tests__/help-conformance-topic-coverage.test.ts package.json tsconfig.json tsconfig.lib.json tsdown.config.ts vitest.config.ts vitest.mutation.config.ts .github/actions/setup-node-pnpm/action.yml .oxlintrc.json .oxfmtrc.json '!test/skillgym/.skillgym-results/**'",
+    "format:check": "node ./node_modules/oxfmt/bin/oxfmt --check src test skills scripts/fuzz scripts/mutation scripts/lib scripts/help-conformance-bench.mjs scripts/help-conformance-case-checks.mjs scripts/help-conformance-command-validator.ts scripts/help-conformance-expectations.mjs scripts/help-conformance-plan-validator.mjs scripts/help-conformance-runner-output.mjs scripts/help-conformance-summary.mjs scripts/help-conformance-cases.mjs scripts/help-conformance-sample-outputs.mjs scripts/__tests__/help-conformance-bench.test.ts scripts/__tests__/help-conformance-sample-outputs.test.ts scripts/__tests__/help-conformance-topic-coverage.test.ts package.json tsconfig.json tsconfig.lib.json tsdown.config.ts vitest.config.ts vitest.mutation.config.ts .github/actions/setup-node-pnpm/action.yml .oxlintrc.json .oxfmtrc.json '!test/skillgym/.skillgym-results/**'",
+    "fuzz:parsers": "node --experimental-strip-types scripts/fuzz/run.ts",
     "fallow": "fallow audit --base origin/main",
     "fallow:all": "fallow --summary",
     "fallow:baseline": "(fallow dead-code --save-baseline fallow-baselines/dead-code.json --summary || true) && (fallow health --save-baseline fallow-baselines/health.json --summary || true)",

scripts/fuzz/arbitraries.ts

@@ -0,0 +1,114 @@
+// Case generators for the parser fuzz lane (#1414).
+//
+// fast-check rather than a bespoke PRNG, for two reasons the lane depends on: a reported
+// counterexample is the SHRUNK input (a hand-rolled mutator reports the 20k-character random one),
+// and the hazard vocabulary is the one #1437's property suite already curates — a hazard added
+// there for a round-trip property reaches the fuzzer without being retyped here.
+//
+// Cases stay near the grammar's edge on purpose: a well-formed base with hostile chunks spliced in
+// reaches deep parser branches that uniformly random noise never gets past the first token of.
+
+import fc from 'fast-check';
+import {
+  replayScriptArb,
+  SELECTOR_VALUE_HAZARDS,
+  selectorChainArb,
+} from '../../src/__tests__/test-utils/property-arbitraries.ts';
+import type { FuzzTarget, FuzzTargetName } from './target-types.ts';
+
+/**
+ * Hazards a valid-input property cannot use — they exist to be *rejected*: structural JSON/YAML
+ * punctuation, delimiter lookalikes, prototype-pollution keys, numeric edges, bidi/zero-width
+ * controls. The shared list above carries the ones valid inputs must also survive.
+ */
+const REJECTION_HAZARDS = [
+  '`',
+  '==',
+  '&&',
+  '--',
+  '---',
+  '#',
+  ':',
+  ',',
+  '{',
+  '}',
+  '[',
+  ']',
+  '(',
+  ')',
+  '${',
+  '${}',
+  '@',
+  '~=',
+  '*',
+  '\r\n',
+  '\u0000',
+  '\u200b',
+  '\u202e',
+  '\ufeff',
+  '-0',
+  'NaN',
+  'Infinity',
+  '1e999',
+  '9007199254740993',
+  'null',
+  'undefined',
+  '__proto__',
+  'constructor',
+] as const;
+
+const hazardArb: fc.Arbitrary<string> = fc.constantFrom(
+  ...SELECTOR_VALUE_HAZARDS,
+  ...REJECTION_HAZARDS,
+);
+
+/** A base string with 1–4 hazards spliced in at shrinkable positions. */
+function corrupted(base: fc.Arbitrary<string>): fc.Arbitrary<string> {
+  return fc
+    .tuple(
+      base,
+      fc.array(fc.tuple(hazardArb, fc.nat({ max: 4096 })), { minLength: 1, maxLength: 4 }),
+    )
+    .map(([text, edits]) =>
+      edits.reduce((current, [chunk, at]) => {
+        const index = at % (current.length + 1);
+        return current.slice(0, index) + chunk + current.slice(index);
+      }, text),
+    );
+}
+
+/** A long run of one hazard — regex-backtracking and quadratic-scan bait. */
+const repeatedHazardArb: fc.Arbitrary<string> = fc
+  .tuple(hazardArb, fc.integer({ min: 50, max: 400 }))
+  .map(([chunk, times]) => chunk.repeat(times));
+
+/** Input that is not trying to look like the grammar at all. */
+const noiseArb: fc.Arbitrary<string> = fc.oneof(
+  fc.string({ maxLength: 40 }),
+  fc.string({ unit: 'binary', maxLength: 40 }),
+  fc.array(hazardArb, { maxLength: 8 }).map((parts) => parts.join('')),
+  repeatedHazardArb,
+);
+
+/**
+ * Bases borrowed from the property suite, which generates *valid* inputs: corrupting a
+ * grammar-correct script or selector chain is how the fuzzer reaches branches past the first
+ * rejection. Targets without one generate from their own seed list.
+ */
+const STRUCTURED_BASES: Partial<Record<FuzzTargetName, fc.Arbitrary<string>>> = {
+  selector: selectorChainArb.map((chain) => chain.expression),
+  'replay-script': replayScriptArb,
+  'batch-steps': fc.json({ maxDepth: 3 }),
+};
+
+/** The case distribution for one target: mostly near-miss, some valid, some pure noise. */
+export function arbitraryForTarget(target: FuzzTarget): fc.Arbitrary<string> {
+  const seeded = fc.constantFrom(...target.seeds);
+  const structured = STRUCTURED_BASES[target.name];
+  const base = structured === undefined ? seeded : fc.oneof(seeded, structured);
+  return fc.oneof(
+    { weight: 6, arbitrary: corrupted(base) },
+    { weight: 2, arbitrary: base },
+    { weight: 2, arbitrary: noiseArb },
+  );
+}

scripts/fuzz/corpus-replay.test.ts

@@ -0,0 +1,83 @@
+// Unit-lane replay of the parser fuzz regression corpus (#1414).
+//
+// The nightly lane finds cases; this replays every case it ever found so a regression fails in
+// seconds on a PR instead of a night later. Cases go through the same worker-backed watchdog the
+// nightly lane uses: a promoted hang case must fail this test against its per-case budget, not
+// wedge the unit job until the CI timeout.
+
+import fc from 'fast-check';
+import { describe, expect, it } from 'vitest';
+import { arbitraryForTarget } from './arbitraries.ts';
+import { readCorpus } from './corpus.ts';
+import { runCases } from './execute.ts';
+import { describeFailure } from './invariant.ts';
+import { getFuzzTarget } from './registry.ts';
+import { FUZZ_TARGETS } from './targets.ts';
+
+/**
+ * Generous enough that a loaded CI runner never reports a healthy parser as hung, small enough
+ * that a genuinely wedged case fails the file in seconds.
+ */
+const CASE_TIMEOUT_MS = 5_000;
+
+/**
+ * Vitest must outlast the watchdog for every case a test replays, or a wedged parser is reported as
+ * a bare `Test timed out` instead of the named `hang:` failure that says which input wedged.
+ */
+const timeoutFor = (cases: number) => cases * CASE_TIMEOUT_MS + 10_000;
+
+describe('parser fuzz regression corpus', () => {
+  const corpus = readCorpus();
+
+  it('is non-empty and free of duplicates', () => {
+    expect(corpus.length).toBeGreaterThan(0);
+    const keys = corpus.map((entry) => `${entry.target}\u0000${entry.input}`);
+    expect(new Set(keys).size).toBe(keys.length);
+  });
+
+  it('names only real parser targets and explains every entry', () => {
+    const parserTargets = new Set<string>(FUZZ_TARGETS.map((target) => target.name));
+    for (const entry of corpus) {
+      expect(parserTargets).toContain(entry.target);
+      expect(entry.note.trim()).not.toBe('');
+    }
+  });
+
+  // One worker per target rather than per case: startup is the only real cost here, and the
+  // watchdog budget is per case either way.
+  it.each(FUZZ_TARGETS.map((target) => [target.name, target] as const))(
+    '%s corpus cases and seeds hold the invariant, under the watchdog',
+    async (name, target) => {
+      const cases = [
+        ...target.seeds,
+        ...corpus.filter((entry) => entry.target === name).map((entry) => entry.input),
+      ];
+      const failures = await runCases(target, cases, CASE_TIMEOUT_MS);
+      expect(failures.map(describeFailure)).toEqual([]);
+    },
+    timeoutFor(corpus.length + Math.max(...FUZZ_TARGETS.map((t) => t.seeds.length))),
+  );
+});
+
+describe('fuzz case generation', () => {
+  const target = getFuzzTarget('selector');
+
+  it('is deterministic for a seed, so a reported counterexample replays', () => {
+    const sample = (seed: number) => fc.sample(arbitraryForTarget(target), { numRuns: 32, seed });
+    expect(sample(7)).toEqual(sample(7));
+    expect(sample(7)).not.toEqual(sample(8));
+  });
+
+  const GENERATED_CASES = 64;
+
+  it(
+    'generates strings the target can be fed directly',
+    async () => {
+      const inputs = fc.sample(arbitraryForTarget(target), { numRuns: GENERATED_CASES, seed: 1 });
+      expect(inputs.every((input) => typeof input === 'string')).toBe(true);
+      const failures = await runCases(target, inputs, CASE_TIMEOUT_MS);
+      expect(failures.map(describeFailure)).toEqual([]);
+    },
+    timeoutFor(GENERATED_CASES),
+  );
+});