diff --git a/.github/workflows/replays-nightly.yml b/.github/workflows/replays-nightly.yml index 1aad1ba45..1776deb4d 100644 --- a/.github/workflows/replays-nightly.yml +++ b/.github/workflows/replays-nightly.yml @@ -4,6 +4,14 @@ on: schedule: - cron: '0 3 * * *' workflow_dispatch: + inputs: + fuzz-iterations: + description: 'Parser fuzz cases per target' + required: false + default: '50000' + fuzz-seed: + description: 'Parser fuzz PRNG seed (defaults to the run number)' + required: false permissions: contents: read @@ -13,6 +21,73 @@ concurrency: cancel-in-progress: true jobs: + # Parser fuzz lane (#1414): hostile input into parseArgs, selector parsing, .ad replay + # scripts, batch --steps JSON, and the Maestro compat parser. One invariant — every + # rejection is a typed AppError with a non-empty hint, and no case hangs. Device-free, so + # it rides this nightly rather than owning a workflow; the seed varies per run so the lane + # keeps exploring, and anything it catches is appended to the checked-in corpus that the + # unit lane replays (scripts/fuzz/corpus/regressions.json). + nightly-parser-fuzz: + name: Parser Fuzz Lane + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - name: Checkout + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - name: Setup toolchain + uses: ./.github/actions/setup-node-pnpm + + # A lane whose classifier or watchdog regressed would pass forever. This runs the + # broken-on-purpose targets first and fails unless each violation is caught. + - name: Self-check the harness + run: pnpm fuzz:parsers --self-check --artifact-dir .tmp/fuzz/self-check + + # Runs even when the self-check fails, so the lane always produces a fuzz envelope too + # (a step that never runs writes no envelope, and monitoring cannot tell that apart from + # a lane that went dark). The job still fails because both steps report their status. + - name: Fuzz parsers + if: always() + env: + FUZZ_ITERATIONS: ${{ github.event.inputs.fuzz-iterations || '50000' }} + FUZZ_SEED: ${{ github.event.inputs.fuzz-seed || github.run_number }} + run: | + pnpm fuzz:parsers \ + --iterations "$FUZZ_ITERATIONS" \ + --seed "$FUZZ_SEED" \ + --artifact-dir .tmp/fuzz/run + + # Uploaded on pass as well as failure: each subdirectory of .tmp/fuzz holds a + # run-envelope.json on #1430's shared lane contract (scripts/lib/lane-envelope.ts) — + # commit/ref/run provenance, tool + config hash, seed, result — which is what freshness + # monitoring reads. + - name: Upload run envelopes and failing cases + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: parser-fuzz-run-${{ github.run_id }}-${{ github.run_attempt }} + path: .tmp/fuzz + if-no-files-found: warn + + # Never fails the job on its own: a missing envelope is reported as such rather than + # masking the real failure with a `cat` error. + - name: Summarize + if: always() + run: | + { + echo '### Parser fuzz lane' + found=0 + for envelope in .tmp/fuzz/*/run-envelope.json; do + [ -f "$envelope" ] || continue + found=1 + echo "#### $envelope" + echo '```json' + cat "$envelope" + echo '```' + done + [ "$found" = 1 ] || echo 'No run envelope was produced — the lane failed before it could run.' + } >> "$GITHUB_STEP_SUMMARY" + nightly-android: name: Android Replay Suite runs-on: ubuntu-latest diff --git a/docs/agents/testing.md b/docs/agents/testing.md index 0ee8cb823..dbe4a3b19 100644 --- a/docs/agents/testing.md +++ b/docs/agents/testing.md @@ -179,6 +179,48 @@ pnpm mutation:baseline # full sweep, then record it (reviewed c written on every exit path, including a crash before any mutant runs: an absent envelope would be indistinguishable from a lane that never ran. +## Parser fuzz lane + +`pnpm fuzz:parsers` feeds generated hostile input to `parseArgs`, selector parsing, +`parseReplayScriptDetailed`, `batch --steps` JSON, and the Maestro compat parser, and enforces one +invariant: every rejection is a typed `AppError` whose normalized `hint` is non-empty, and no case +hangs (a worker-thread watchdog attributes a stall to the exact input). + +```sh +pnpm fuzz:parsers # all targets, 2,000 cases each, seed 1 +pnpm fuzz:parsers --target selector --iterations 50000 --seed 7 +pnpm fuzz:parsers --input-file .tmp/fuzz/.json # repro a saved case +pnpm fuzz:parsers --input-file .tmp/fuzz/.json --append-corpus # …and pin it +pnpm fuzz:parsers --self-check # require the harness to still fail +``` + +The generating run is nightly (`Parser Fuzz Lane` in `.github/workflows/replays-nightly.yml`, seeded +by the run number). Every terminal path — pass, fail, `--self-check`, or a crash in the harness +itself — writes `/run-envelope.json` on the shared lane contract +(`scripts/lib/lane-envelope.ts`, #1430), with the lane's own facts under `data`: mode, per-target +cases/failures/durations, failures, repro commands, and `stage` (`error` marks a run that could not +complete itself, since the shared `result` is only `pass`/`fail`). `configHash` hashes the modules +that decide a case set, so "the same seed means different inputs now" is distinguishable from "the +parsers changed". The self-check and fuzz steps write to separate artifact subdirectories and both +run unconditionally; the step summary prints each envelope it finds and never fails on a missing +file. + +Cases come from fast-check arbitraries (`scripts/fuzz/arbitraries.ts`) built on the hazard vocabulary +shared with `src/__tests__/test-utils/property-arbitraries.ts`, so a hazard added for the property +suite reaches the fuzz lane too — and a counterexample is reported **shrunk**, with fast-check's seed +and replay path printed alongside the saved artifact. + +A nightly discovery reaches the unit lane by promotion, not hand-editing: the printed +`promote:` command re-runs the downloaded artifact and appends it to +`scripts/fuzz/corpus/regressions.json`, which `scripts/fuzz/corpus-replay.test.ts` replays on every +PR — through the same worker watchdog, so a promoted hang case fails against its per-case budget +instead of wedging the unit job. `scripts/fuzz/harness.test.ts` covers the harness itself — an +untyped throw, an empty hint, and a wedged worker must each be reported, startup time is never charged against the per-case budget, and +every mode writes an envelope — using the broken-on-purpose targets in +`scripts/fuzz/self-check-targets.ts` (also what `--self-check` runs in CI), so a regressed classifier +or watchdog cannot pass silently. Adding a parser to the lane means adding a target to +`scripts/fuzz/targets.ts` — nothing else. + ## Live web smoke The live web platform smoke runs the public built CLI against a local fixture page through the managed web backend: diff --git a/package.json b/package.json index beeaa8388..15597c917 100644 --- a/package.json +++ b/package.json @@ -114,8 +114,9 @@ "mutation:affected": "node --experimental-strip-types scripts/mutation/run.ts --affected", "mutation:test": "node --experimental-strip-types --test scripts/mutation/*.test.ts", "lint": "oxlint . --deny-warnings", - "format": "node ./node_modules/oxfmt/bin/oxfmt --write src test skills scripts/mutation scripts/lib scripts/help-conformance-bench.mjs scripts/help-conformance-case-checks.mjs scripts/help-conformance-command-validator.ts scripts/help-conformance-expectations.mjs scripts/help-conformance-plan-validator.mjs scripts/help-conformance-runner-output.mjs scripts/help-conformance-summary.mjs scripts/help-conformance-cases.mjs scripts/help-conformance-sample-outputs.mjs scripts/__tests__/help-conformance-bench.test.ts scripts/__tests__/help-conformance-sample-outputs.test.ts scripts/__tests__/help-conformance-topic-coverage.test.ts package.json tsconfig.json tsconfig.lib.json tsdown.config.ts vitest.config.ts vitest.mutation.config.ts .github/actions/setup-node-pnpm/action.yml .oxlintrc.json .oxfmtrc.json '!test/skillgym/.skillgym-results/**'", - "format:check": "node ./node_modules/oxfmt/bin/oxfmt --check src test skills scripts/mutation scripts/lib scripts/help-conformance-bench.mjs scripts/help-conformance-case-checks.mjs scripts/help-conformance-command-validator.ts scripts/help-conformance-expectations.mjs scripts/help-conformance-plan-validator.mjs scripts/help-conformance-runner-output.mjs scripts/help-conformance-summary.mjs scripts/help-conformance-cases.mjs scripts/help-conformance-sample-outputs.mjs scripts/__tests__/help-conformance-bench.test.ts scripts/__tests__/help-conformance-sample-outputs.test.ts scripts/__tests__/help-conformance-topic-coverage.test.ts package.json tsconfig.json tsconfig.lib.json tsdown.config.ts vitest.config.ts vitest.mutation.config.ts .github/actions/setup-node-pnpm/action.yml .oxlintrc.json .oxfmtrc.json '!test/skillgym/.skillgym-results/**'", + "format": "node ./node_modules/oxfmt/bin/oxfmt --write src test skills scripts/fuzz scripts/mutation scripts/lib scripts/help-conformance-bench.mjs scripts/help-conformance-case-checks.mjs scripts/help-conformance-command-validator.ts scripts/help-conformance-expectations.mjs scripts/help-conformance-plan-validator.mjs scripts/help-conformance-runner-output.mjs scripts/help-conformance-summary.mjs scripts/help-conformance-cases.mjs scripts/help-conformance-sample-outputs.mjs scripts/__tests__/help-conformance-bench.test.ts scripts/__tests__/help-conformance-sample-outputs.test.ts scripts/__tests__/help-conformance-topic-coverage.test.ts package.json tsconfig.json tsconfig.lib.json tsdown.config.ts vitest.config.ts vitest.mutation.config.ts .github/actions/setup-node-pnpm/action.yml .oxlintrc.json .oxfmtrc.json '!test/skillgym/.skillgym-results/**'", + "format:check": "node ./node_modules/oxfmt/bin/oxfmt --check src test skills scripts/fuzz scripts/mutation scripts/lib scripts/help-conformance-bench.mjs scripts/help-conformance-case-checks.mjs scripts/help-conformance-command-validator.ts scripts/help-conformance-expectations.mjs scripts/help-conformance-plan-validator.mjs scripts/help-conformance-runner-output.mjs scripts/help-conformance-summary.mjs scripts/help-conformance-cases.mjs scripts/help-conformance-sample-outputs.mjs scripts/__tests__/help-conformance-bench.test.ts scripts/__tests__/help-conformance-sample-outputs.test.ts scripts/__tests__/help-conformance-topic-coverage.test.ts package.json tsconfig.json tsconfig.lib.json tsdown.config.ts vitest.config.ts vitest.mutation.config.ts .github/actions/setup-node-pnpm/action.yml .oxlintrc.json .oxfmtrc.json '!test/skillgym/.skillgym-results/**'", + "fuzz:parsers": "node --experimental-strip-types scripts/fuzz/run.ts", "fallow": "fallow audit --base origin/main", "fallow:all": "fallow --summary", "fallow:baseline": "(fallow dead-code --save-baseline fallow-baselines/dead-code.json --summary || true) && (fallow health --save-baseline fallow-baselines/health.json --summary || true)", diff --git a/scripts/fuzz/arbitraries.ts b/scripts/fuzz/arbitraries.ts new file mode 100644 index 000000000..b884564c0 --- /dev/null +++ b/scripts/fuzz/arbitraries.ts @@ -0,0 +1,114 @@ +// Case generators for the parser fuzz lane (#1414). +// +// fast-check rather than a bespoke PRNG, for two reasons the lane depends on: a reported +// counterexample is the SHRUNK input (a hand-rolled mutator reports the 20k-character random one), +// and the hazard vocabulary is the one #1437's property suite already curates — a hazard added +// there for a round-trip property reaches the fuzzer without being retyped here. +// +// Cases stay near the grammar's edge on purpose: a well-formed base with hostile chunks spliced in +// reaches deep parser branches that uniformly random noise never gets past the first token of. + +import fc from 'fast-check'; +import { + replayScriptArb, + SELECTOR_VALUE_HAZARDS, + selectorChainArb, +} from '../../src/__tests__/test-utils/property-arbitraries.ts'; +import type { FuzzTarget, FuzzTargetName } from './target-types.ts'; + +/** + * Hazards a valid-input property cannot use — they exist to be *rejected*: structural JSON/YAML + * punctuation, delimiter lookalikes, prototype-pollution keys, numeric edges, bidi/zero-width + * controls. The shared list above carries the ones valid inputs must also survive. + */ +const REJECTION_HAZARDS = [ + '`', + '==', + '&&', + '--', + '---', + '#', + ':', + ',', + '{', + '}', + '[', + ']', + '(', + ')', + '${', + '${}', + '@', + '~=', + '*', + '\r\n', + '\u0000', + '\u200b', + '\u202e', + '\ufeff', + '-0', + 'NaN', + 'Infinity', + '1e999', + '9007199254740993', + 'null', + 'undefined', + '__proto__', + 'constructor', +] as const; + +const hazardArb: fc.Arbitrary = fc.constantFrom( + ...SELECTOR_VALUE_HAZARDS, + ...REJECTION_HAZARDS, +); + +/** A base string with 1–4 hazards spliced in at shrinkable positions. */ +function corrupted(base: fc.Arbitrary): fc.Arbitrary { + return fc + .tuple( + base, + fc.array(fc.tuple(hazardArb, fc.nat({ max: 4096 })), { minLength: 1, maxLength: 4 }), + ) + .map(([text, edits]) => + edits.reduce((current, [chunk, at]) => { + const index = at % (current.length + 1); + return current.slice(0, index) + chunk + current.slice(index); + }, text), + ); +} + +/** A long run of one hazard — regex-backtracking and quadratic-scan bait. */ +const repeatedHazardArb: fc.Arbitrary = fc + .tuple(hazardArb, fc.integer({ min: 50, max: 400 })) + .map(([chunk, times]) => chunk.repeat(times)); + +/** Input that is not trying to look like the grammar at all. */ +const noiseArb: fc.Arbitrary = fc.oneof( + fc.string({ maxLength: 40 }), + fc.string({ unit: 'binary', maxLength: 40 }), + fc.array(hazardArb, { maxLength: 8 }).map((parts) => parts.join('')), + repeatedHazardArb, +); + +/** + * Bases borrowed from the property suite, which generates *valid* inputs: corrupting a + * grammar-correct script or selector chain is how the fuzzer reaches branches past the first + * rejection. Targets without one generate from their own seed list. + */ +const STRUCTURED_BASES: Partial>> = { + selector: selectorChainArb.map((chain) => chain.expression), + 'replay-script': replayScriptArb, + 'batch-steps': fc.json({ maxDepth: 3 }), +}; + +/** The case distribution for one target: mostly near-miss, some valid, some pure noise. */ +export function arbitraryForTarget(target: FuzzTarget): fc.Arbitrary { + const seeded = fc.constantFrom(...target.seeds); + const structured = STRUCTURED_BASES[target.name]; + const base = structured === undefined ? seeded : fc.oneof(seeded, structured); + return fc.oneof( + { weight: 6, arbitrary: corrupted(base) }, + { weight: 2, arbitrary: base }, + { weight: 2, arbitrary: noiseArb }, + ); +} diff --git a/scripts/fuzz/corpus-replay.test.ts b/scripts/fuzz/corpus-replay.test.ts new file mode 100644 index 000000000..101ed1de8 --- /dev/null +++ b/scripts/fuzz/corpus-replay.test.ts @@ -0,0 +1,83 @@ +// Unit-lane replay of the parser fuzz regression corpus (#1414). +// +// The nightly lane finds cases; this replays every case it ever found so a regression fails in +// seconds on a PR instead of a night later. Cases go through the same worker-backed watchdog the +// nightly lane uses: a promoted hang case must fail this test against its per-case budget, not +// wedge the unit job until the CI timeout. + +import fc from 'fast-check'; +import { describe, expect, it } from 'vitest'; +import { arbitraryForTarget } from './arbitraries.ts'; +import { readCorpus } from './corpus.ts'; +import { runCases } from './execute.ts'; +import { describeFailure } from './invariant.ts'; +import { getFuzzTarget } from './registry.ts'; +import { FUZZ_TARGETS } from './targets.ts'; + +/** + * Generous enough that a loaded CI runner never reports a healthy parser as hung, small enough + * that a genuinely wedged case fails the file in seconds. + */ +const CASE_TIMEOUT_MS = 5_000; + +/** + * Vitest must outlast the watchdog for every case a test replays, or a wedged parser is reported as + * a bare `Test timed out` instead of the named `hang:` failure that says which input wedged. + */ +const timeoutFor = (cases: number) => cases * CASE_TIMEOUT_MS + 10_000; + +describe('parser fuzz regression corpus', () => { + const corpus = readCorpus(); + + it('is non-empty and free of duplicates', () => { + expect(corpus.length).toBeGreaterThan(0); + const keys = corpus.map((entry) => `${entry.target}\u0000${entry.input}`); + expect(new Set(keys).size).toBe(keys.length); + }); + + it('names only real parser targets and explains every entry', () => { + const parserTargets = new Set(FUZZ_TARGETS.map((target) => target.name)); + for (const entry of corpus) { + expect(parserTargets).toContain(entry.target); + expect(entry.note.trim()).not.toBe(''); + } + }); + + // One worker per target rather than per case: startup is the only real cost here, and the + // watchdog budget is per case either way. + it.each(FUZZ_TARGETS.map((target) => [target.name, target] as const))( + '%s corpus cases and seeds hold the invariant, under the watchdog', + async (name, target) => { + const cases = [ + ...target.seeds, + ...corpus.filter((entry) => entry.target === name).map((entry) => entry.input), + ]; + const failures = await runCases(target, cases, CASE_TIMEOUT_MS); + expect(failures.map(describeFailure)).toEqual([]); + }, + timeoutFor(corpus.length + Math.max(...FUZZ_TARGETS.map((t) => t.seeds.length))), + ); +}); + +describe('fuzz case generation', () => { + const target = getFuzzTarget('selector'); + + it('is deterministic for a seed, so a reported counterexample replays', () => { + const sample = (seed: number) => fc.sample(arbitraryForTarget(target), { numRuns: 32, seed }); + expect(sample(7)).toEqual(sample(7)); + expect(sample(7)).not.toEqual(sample(8)); + }); + + const GENERATED_CASES = 64; + + it( + 'generates strings the target can be fed directly', + async () => { + const inputs = fc.sample(arbitraryForTarget(target), { numRuns: GENERATED_CASES, seed: 1 }); + expect(inputs.every((input) => typeof input === 'string')).toBe(true); + const failures = await runCases(target, inputs, CASE_TIMEOUT_MS); + expect(failures.map(describeFailure)).toEqual([]); + }, + timeoutFor(GENERATED_CASES), + ); +}); diff --git a/scripts/fuzz/corpus.ts b/scripts/fuzz/corpus.ts new file mode 100644 index 000000000..ceac4a120 --- /dev/null +++ b/scripts/fuzz/corpus.ts @@ -0,0 +1,66 @@ +// The checked-in regression corpus for the parser fuzz lane (#1414). +// +// Every input the fuzzer ever catches is appended here and replayed by the unit lane +// (scripts/fuzz/corpus-replay.test.ts), so a fixed parser stays fixed without waiting for +// the nightly to rediscover the case. Entries are sorted and deduplicated on write, which +// keeps the diff of an append small and the replay order deterministic. + +import fs from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import type { FuzzTargetName } from './target-types.ts'; + +export type CorpusEntry = { + target: FuzzTargetName; + /** The failing input, verbatim. */ + input: string; + /** Why it was added — the invariant it broke, or where it came from. */ + note: string; +}; + +// AGENT_DEVICE_FUZZ_CORPUS retargets the corpus file so the harness's own tests can exercise +// the real promotion path against a scratch file instead of mutating the checked-in one. +const CORPUS_PATH = + process.env.AGENT_DEVICE_FUZZ_CORPUS ?? + path.join(path.dirname(fileURLToPath(import.meta.url)), 'corpus', 'regressions.json'); + +export function readCorpus(corpusPath = CORPUS_PATH): CorpusEntry[] { + const raw = fs.readFileSync(corpusPath, 'utf8'); + const parsed: unknown = JSON.parse(raw); + if (!Array.isArray(parsed)) throw new Error(`${corpusPath} must contain a JSON array.`); + return parsed.map((entry, index) => readEntry(entry, index, corpusPath)); +} + +function readEntry(entry: unknown, index: number, corpusPath: string): CorpusEntry { + if (entry === null || typeof entry !== 'object') { + throw new Error(`${corpusPath}[${index}] must be an object.`); + } + const record = entry as Record; + const { target, input, note } = record; + if (typeof target !== 'string' || typeof input !== 'string' || typeof note !== 'string') { + throw new Error(`${corpusPath}[${index}] needs string target, input, and note fields.`); + } + return { target: target as FuzzTargetName, input, note }; +} + +function entryKey(entry: CorpusEntry): string { + return `${entry.target}\u0000${entry.input}`; +} + +/** Merges `additions` into the corpus file. Returns the entries actually added. */ +export function appendToCorpus( + additions: readonly CorpusEntry[], + corpusPath = CORPUS_PATH, +): CorpusEntry[] { + const existing = readCorpus(corpusPath); + const seen = new Set(existing.map(entryKey)); + const added = additions.filter((entry) => { + if (seen.has(entryKey(entry))) return false; + seen.add(entryKey(entry)); + return true; + }); + if (added.length === 0) return []; + const merged = [...existing, ...added].sort((a, b) => entryKey(a).localeCompare(entryKey(b))); + fs.writeFileSync(corpusPath, `${JSON.stringify(merged, null, 2)}\n`); + return added; +} diff --git a/scripts/fuzz/corpus/regressions.json b/scripts/fuzz/corpus/regressions.json new file mode 100644 index 000000000..719c85338 --- /dev/null +++ b/scripts/fuzz/corpus/regressions.json @@ -0,0 +1,42 @@ +[ + { + "target": "batch-steps", + "input": "[{\"command\":\"snapshot\",\"input\":{\"__proto__\":{\"polluted\":true}}}]", + "note": "seed: prototype-pollution shaped batch step must reject or ignore, never throw untyped" + }, + { + "target": "cli-args", + "input": "click --selector", + "note": "seed: flag with a missing value must reject as INVALID_ARGS with a hint" + }, + { + "target": "maestro", + "input": "appId: com.example.app\n---\n- tapOn: \"unterminated\n", + "note": "seed: unterminated YAML scalar must surface as a typed parse error" + }, + { + "target": "replay-script", + "input": "# target-v1 role=button\n\nclick @e1\n", + "note": "seed: unbound target-v1 annotation must reject with a typed error" + }, + { + "target": "replay-script", + "input": "fill @e1 --text \"hello\tworld\"\na", + "note": "untyped-throw: raw tab inside a quoted value leaked a SyntaxError (fixed in readQuotedReplayToken)" + }, + { + "target": "replay-script", + "input": "fill @e1 --text \"hello wor\\ld\"\nassert text=Welcome\n", + "note": "untyped-throw: quoted value with an invalid JSON escape leaked a SyntaxError (fixed in readQuotedReplayToken)" + }, + { + "target": "replay-script", + "input": "fill @e1 --text \"hel\u0000lo world\"\n", + "note": "untyped-throw: raw control character inside a quoted value leaked a SyntaxError (fixed in readQuotedReplayToken)" + }, + { + "target": "selector", + "input": "label=\"unterminated", + "note": "seed: unterminated quoted selector value must reject with a typed error" + } +] diff --git a/scripts/fuzz/envelope.ts b/scripts/fuzz/envelope.ts new file mode 100644 index 000000000..0d6011328 --- /dev/null +++ b/scripts/fuzz/envelope.ts @@ -0,0 +1,112 @@ +// Run envelope for the parser fuzz lane (#1414), on #1430's shared contract. +// +// A scheduled lane goes dark quietly: it can stop running, or fail for weeks, while PR CI stays +// green. Freshness monitoring therefore needs one machine-readable record per run — green runs +// included. This module only maps the lane's own facts onto `scripts/lib/lane-envelope.ts`; the +// envelope shape itself is cross-lane and lives there. +// +// `error` (a crash, or config the harness could not parse) is reported as `result: 'fail'` with +// `data.stage: 'error'`: the shared contract deliberately has two results, and a lane that could +// not complete itself is not a passing lane. + +import crypto from 'node:crypto'; +import fs from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { laneEnvelope } from '../lib/lane-envelope.ts'; +import { runCmdSync } from '../../src/utils/exec.ts'; +import type { FuzzFailure } from './invariant.ts'; + +const FILENAME = 'run-envelope.json'; +/** Shared with the property suite (#1437): its hazard list feeds the fuzz arbitraries. */ +const PROPERTY_ARBITRARIES = '../../src/__tests__/test-utils/property-arbitraries.ts'; +const LANE = 'parser-fuzz'; +const TOOL = 'scripts/fuzz/run.ts'; + +export type FuzzTargetRun = { + target: string; + cases: number; + failures: number; + durationMs: number; +}; + +export type FuzzRunMode = 'generate' | 'replay-corpus' | 'replay-artifact' | 'self-check'; + +export type FuzzEnvelopeDetails = { + mode: FuzzRunMode; + corpusEntries: number; + targetRuns: FuzzTargetRun[]; + failures: (FuzzFailure & { artifact?: string })[]; + reproCommands: string[]; +}; + +/** `stage` separates "the lane ran and found violations" from "the lane could not run". */ +export type FuzzEnvelopeData = FuzzEnvelopeDetails & { + stage: 'complete' | 'error'; + config: Record; +}; + +/** Writes the envelope for one fuzz run into `artifactDir`; returns its path. */ +export function writeFuzzEnvelope(input: { + artifactDir: string; + startedAt: number; + finishedAt: number; + result: 'pass' | 'fail' | 'error'; + config: Record; + details: FuzzEnvelopeDetails; +}): string { + const seed = input.config.seed; + const envelope = laneEnvelope({ + lane: LANE, + commit: runCmdSync('git', ['rev-parse', 'HEAD'], { allowFailure: true }).stdout.trim(), + // fast-check is a case-generation input, not just a dependency: an upgrade can change what a + // seed produces, so its version belongs in provenance next to Node's. + tool: { node: process.version, 'fast-check': fastCheckVersion(), harness: TOOL }, + configHash: harnessHash(), + seed: typeof seed === 'number' ? String(seed) : null, + startedAtMs: input.startedAt, + now: input.finishedAt, + result: input.result === 'pass' ? 'pass' : 'fail', + data: { + stage: input.result === 'error' ? 'error' : 'complete', + config: { mode: input.details.mode, ...input.config }, + ...input.details, + }, + }); + fs.mkdirSync(input.artifactDir, { recursive: true }); + const file = path.join(input.artifactDir, FILENAME); + fs.writeFileSync(file, `${JSON.stringify(envelope, null, 2)}\n`); + return file; +} + +/** + * Every module that decides which inputs a seed produces, or what counts as a violation: the + * arbitraries, the targets they are built for, the generation loop (numRuns, property, shrinking), + * and the invariant itself. Hashing a subset would let a changed case set look like an unchanged + * lane, which is exactly the drift this field exists to catch. + */ +const CASE_GENERATION_INPUTS = [ + 'arbitraries.ts', + 'generate.ts', + 'targets.ts', + 'invariant.ts', +] as const; + +/** Content hash of `CASE_GENERATION_INPUTS`, alongside their shared source of hazards. */ +function harnessHash(): string { + const here = path.dirname(new URL(import.meta.url).pathname); + const digest = crypto.createHash('sha256'); + for (const name of CASE_GENERATION_INPUTS) { + digest.update(fs.readFileSync(path.join(here, name))); + } + digest.update(fs.readFileSync(path.join(here, PROPERTY_ARBITRARIES))); + return `sha256:${digest.digest('hex').slice(0, 16)}`; +} + +/** The generators read from the installed package, so its version is read from there too. */ +function fastCheckVersion(): string { + const manifest = fileURLToPath(import.meta.resolve('fast-check/package.json')); + const parsed: unknown = JSON.parse(fs.readFileSync(manifest, 'utf8')); + const version = (parsed as { version?: unknown }).version; + return typeof version === 'string' ? version : 'unknown'; +} diff --git a/scripts/fuzz/execute.ts b/scripts/fuzz/execute.ts new file mode 100644 index 000000000..212651529 --- /dev/null +++ b/scripts/fuzz/execute.ts @@ -0,0 +1,148 @@ +// Case execution with hang detection for the parser fuzz lane (#1414). +// +// Cases run one at a time in a worker thread: a synchronous parser that never returns cannot be +// timed out from inside its own tick, so the budget is enforced from *another* thread, which can +// terminate the wedged one and attribute the stall to the exact input. Cases go over the wire +// individually (rather than as one batch) because fast-check drives the loop — it decides the next +// input, including the shrink candidates it derives from a failing one. + +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { Worker } from 'node:worker_threads'; +import type { FuzzFailure } from './invariant.ts'; +import type { FuzzTarget } from './target-types.ts'; +import type { FuzzWorkerData, FuzzWorkerMessage } from './worker.ts'; + +const WORKER_PATH = path.join(path.dirname(fileURLToPath(import.meta.url)), 'worker.ts'); + +/** Bound on worker startup only; a per-case budget must never be charged for it. */ +const STARTUP_BUDGET_MS = 60_000; + +export type CaseRunner = { + /** The failure this input violates the invariant with, or `null` when it holds. */ + run: (input: string) => Promise; + close: () => Promise; +}; + +type Session = { worker: Worker; ready: Promise }; + +/** A worker plus the promise that settles when it has finished importing the parsers. */ +function startSession(targetName: string): Session { + const workerData: FuzzWorkerData = { targetName }; + // Type stripping is requested explicitly rather than inherited: under Vitest the parent's + // execArgv carries no such flag, and the worker is a plain `.ts` file Node must strip itself. + // The warning is silenced because a restarted worker re-emits it — one per hang buries the report. + const worker = new Worker(WORKER_PATH, { + workerData, + execArgv: ['--experimental-strip-types', '--disable-warning=ExperimentalWarning'], + }); + const ready = new Promise((resolve, reject) => { + const timer = setTimeout( + () => reject(new Error(`fuzz worker did not start within ${STARTUP_BUDGET_MS}ms`)), + STARTUP_BUDGET_MS, + ); + worker.once('message', (message: FuzzWorkerMessage) => { + clearTimeout(timer); + if (message.kind === 'ready') resolve(); + else reject(new Error(`unexpected first worker message: ${message.kind}`)); + }); + worker.once('error', (error) => { + clearTimeout(timer); + reject(error); + }); + }); + return { worker, ready }; +} + +/** + * Runs one case on a started worker. Startup is already awaited by the caller, so the budget below + * covers parser time only — the reason a slow import can never be misreported as a parser hang. + */ +function runOnSession( + session: Session, + target: FuzzTarget, + input: string, + caseTimeoutMs: number, +): Promise<{ failure: FuzzFailure | null; hung: boolean }> { + return new Promise((resolve, reject) => { + const settle = (failure: FuzzFailure | null, hung: boolean) => { + clearTimeout(timer); + session.worker.off('message', onMessage); + session.worker.off('error', onError); + resolve({ failure, hung }); + }; + const onMessage = (message: FuzzWorkerMessage) => { + if (message.kind === 'result') settle(message.failure, false); + }; + const onError = (error: Error) => { + clearTimeout(timer); + session.worker.off('message', onMessage); + reject(error); + }; + const timer = setTimeout(() => { + settle( + { + target: target.name, + input, + kind: 'hang', + detail: `case did not finish within ${caseTimeoutMs}ms`, + }, + true, + ); + }, caseTimeoutMs); + session.worker.on('message', onMessage); + session.worker.once('error', onError); + session.worker.postMessage({ kind: 'case', input }); + }); +} + +/** + * A worker-backed runner for one target. A hang leaves the thread wedged in its own loop, so the + * runner terminates it and starts a fresh one for the next case — otherwise a single hang would + * silently turn every later case into a hang too. + */ +export async function createCaseRunner( + target: FuzzTarget, + caseTimeoutMs: number, +): Promise { + let session = startSession(target.name); + await session.ready; + let closed = false; + + return { + run: async (input) => { + if (closed) throw new Error('fuzz case runner is closed'); + await session.ready; + const { failure, hung } = await runOnSession(session, target, input, caseTimeoutMs); + if (hung) { + await session.worker.terminate(); + session = startSession(target.name); + await session.ready; + } + return failure; + }, + close: async () => { + closed = true; + await session.worker.terminate(); + }, + }; +} + +/** Runs a fixed list of cases (corpus replay, artifact replay, self-check) on one runner. */ +export async function runCases( + target: FuzzTarget, + cases: readonly string[], + caseTimeoutMs: number, +): Promise { + const runner = await createCaseRunner(target, caseTimeoutMs); + try { + const failures: FuzzFailure[] = []; + for (const input of cases) { + const failure = await runner.run(input); + if (failure) failures.push(failure); + } + return failures; + } finally { + await runner.close(); + } +} diff --git a/scripts/fuzz/generate.ts b/scripts/fuzz/generate.ts new file mode 100644 index 000000000..6ddd2737f --- /dev/null +++ b/scripts/fuzz/generate.ts @@ -0,0 +1,103 @@ +// The generating fuzz run for one target (#1414). +// +// fast-check drives the loop so a violation is reported SHRUNK: the first failing input is +// typically a long mutated string, while the minimized one names the actual parser branch. The +// run stays reproducible — `--seed` is fast-check's seed, and the reported `path` replays the exact +// case, shrink steps included. + +import fc from 'fast-check'; +import { arbitraryForTarget } from './arbitraries.ts'; +import { type CaseRunner, createCaseRunner } from './execute.ts'; +import type { FuzzFailure } from './invariant.ts'; +import type { FuzzTarget } from './target-types.ts'; + +export type GeneratedRun = { + /** Cases actually executed, seeds and shrink candidates included. */ + cases: number; + failure: FuzzFailure | null; + /** fast-check's replay coordinates for the counterexample, when there is one. */ + replay?: { seed: number; path: string }; +}; + +export type GenerateOptions = { iterations: number; seed: number; caseTimeoutMs: number }; + +/** The first seed that already violates the invariant, or `null` when they all hold. */ +async function checkSeeds(target: FuzzTarget, runner: CaseRunner): Promise { + for (const seed of target.seeds) { + const failure = await runner.run(seed); + if (failure) return failure; + } + return null; +} + +/** + * Re-runs the shrunk counterexample, so the reported failure — the one written as an artifact and + * promoted into the corpus — describes the minimized input rather than the original random one. + */ +async function describeCounterexample( + target: FuzzTarget, + runner: CaseRunner, + input: string, +): Promise { + const failure = await runner.run(input); + return ( + failure ?? { + target: target.name, + input, + kind: 'hang', + detail: 'counterexample no longer reproduces outside the shrink run', + } + ); +} + +/** + * Fuzzes one target. Seeds run verbatim first: they are the known-good shapes, and a lane that + * only ever ran generated cases could pass while the plain grammar is broken. + */ +export async function generateAndCheck( + target: FuzzTarget, + options: GenerateOptions, +): Promise { + const runner = await createCaseRunner(target, options.caseTimeoutMs); + try { + const seedFailure = await checkSeeds(target, runner); + if (seedFailure) return { cases: target.seeds.length, failure: seedFailure }; + return await checkGenerated(target, runner, options); + } finally { + await runner.close(); + } +} + +/** The generated half of a run: fast-check picks the inputs and shrinks any counterexample. */ +async function checkGenerated( + target: FuzzTarget, + runner: CaseRunner, + options: GenerateOptions, +): Promise { + let cases = target.seeds.length; + const details = await fc.check( + fc.asyncProperty(arbitraryForTarget(target), async (input) => { + cases += 1; + return (await runner.run(input)) === null; + }), + { numRuns: Math.max(options.iterations - target.seeds.length, 1), seed: options.seed }, + ); + if (!details.failed) return { cases, failure: null }; + return { + cases, + failure: await describeCounterexample(target, runner, counterexampleOf(details)), + replay: replayOf(details), + }; +} + +type CheckDetails = { counterexample: [string] | null; counterexamplePath: string | null }; + +/** The shrunk input fast-check settled on; `''` when it reported a failure without one. */ +function counterexampleOf(details: CheckDetails): string { + return details.counterexample?.[0] ?? ''; +} + +/** Coordinates that replay the counterexample, shrink steps included. */ +function replayOf(details: CheckDetails & { seed: number }): { seed: number; path: string } { + return { seed: details.seed, path: details.counterexamplePath ?? '' }; +} diff --git a/scripts/fuzz/harness.test.ts b/scripts/fuzz/harness.test.ts new file mode 100644 index 000000000..f58464ad5 --- /dev/null +++ b/scripts/fuzz/harness.test.ts @@ -0,0 +1,208 @@ +// Tests of the fuzz harness itself (#1414). +// +// The corpus replay only proves clean inputs pass, so a regressed classifier or watchdog would +// leave every test green. These run the harness against the broken-on-purpose targets through +// the real CLI — same worker, watchdog, artifact, and promotion path a nightly failure takes — +// and require each failure kind to be reported. + +import { execFileSync } from 'node:child_process'; +import crypto from 'node:crypto'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { describe, expect, it } from 'vitest'; +import { LANE_ENVELOPE_SCHEMA_VERSION } from '../lib/lane-envelope.ts'; +import { checkCase } from './invariant.ts'; +import { SELF_CHECK_TARGETS } from './self-check-targets.ts'; + +const FUZZ_DIR = path.dirname(fileURLToPath(import.meta.url)); +const RUN = path.join(FUZZ_DIR, 'run.ts'); + +function runHarness( + args: readonly string[], + env: Record = {}, +): { status: number; stdout: string } { + try { + const stdout = execFileSync(process.execPath, ['--experimental-strip-types', RUN, ...args], { + encoding: 'utf8', + env: { ...process.env, ...env }, + }); + return { status: 0, stdout }; + } catch (error) { + const failure = error as { status?: number; stdout?: string }; + return { status: failure.status ?? 1, stdout: failure.stdout ?? '' }; + } +} + +function targetNamed(name: string) { + const target = SELF_CHECK_TARGETS.find((candidate) => candidate.name === name); + if (!target) throw new Error(`missing self-check target ${name}`); + return target; +} + +describe('fuzz invariant classifier', () => { + it('reports a bare Error as untyped-throw', () => { + const failure = checkCase(targetNamed('self-check-untyped-throw'), 'case'); + expect(failure?.kind).toBe('untyped-throw'); + expect(failure?.detail).toContain('Error: self-check untyped throw'); + }); + + it('reports an AppError with a blank hint as empty-hint', () => { + const failure = checkCase(targetNamed('self-check-empty-hint'), 'case'); + expect(failure?.kind).toBe('empty-hint'); + }); +}); + +describe('fuzz harness self-check', () => { + it('catches an untyped throw, an empty hint, and a wedged worker', () => { + const { status, stdout } = runHarness(['--self-check', '--case-timeout-ms', '750']); + expect(stdout).toContain('ok self-check-untyped-throw: expected untyped-throw'); + expect(stdout).toContain('ok self-check-empty-hint: expected empty-hint'); + expect(stdout).toContain('ok self-check-hang: expected hang, got hang'); + expect(status).toBe(0); + }); +}); + +describe('worker startup budget', () => { + // The watchdog used to start before the worker reported ready, so thread spawn plus type + // stripping plus parser imports — hundreds of milliseconds — was charged to the first case and + // reported as a hung parser. A budget far below real startup time is the regression: this only + // passes because the budget starts at the worker's `ready` handshake. + it('does not report a hang when startup outlasts the per-case budget', () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'fuzz-startup-')); + const { stdout } = runHarness([ + '--target', + 'self-check-untyped-throw', + '--iterations', + '1', + '--case-timeout-ms', + '50', + '--artifact-dir', + dir, + ]); + expect(stdout).toContain('untyped-throw'); + expect(stdout).not.toContain('hang'); + fs.rmSync(dir, { recursive: true, force: true }); + }); +}); + +/** + * Recomputes `configHash` over every case-generation input except `skip`. If the real hash equals + * one of these, that file is not covered and a change to it would look like an unchanged lane. + */ +function hashWithout(skip: string): string { + const digest = crypto.createHash('sha256'); + for (const name of ['arbitraries.ts', 'generate.ts', 'targets.ts', 'invariant.ts']) { + if (name !== skip) digest.update(fs.readFileSync(path.join(FUZZ_DIR, name))); + } + digest.update( + fs.readFileSync(path.join(FUZZ_DIR, '../../src/__tests__/test-utils/property-arbitraries.ts')), + ); + return `sha256:${digest.digest('hex').slice(0, 16)}`; +} + +describe('run envelope', () => { + function envelopeFrom(args: readonly string[]) { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'fuzz-envelope-')); + const { status } = runHarness([...args, '--artifact-dir', dir]); + const file = path.join(dir, 'run-envelope.json'); + const envelope = JSON.parse(fs.readFileSync(file, 'utf8')); + fs.rmSync(dir, { recursive: true, force: true }); + return { envelope, status }; + } + + it('is written for a passing generate run', () => { + const { envelope, status } = envelopeFrom(['--target', 'selector', '--iterations', '20']); + expect(status).toBe(0); + expect(envelope.lane).toBe('parser-fuzz'); + // The shape is #1430's shared contract, not this lane's invention. + expect(envelope.schemaVersion).toBe(LANE_ENVELOPE_SCHEMA_VERSION); + // Drift provenance: fast-check's version and the case-generation sources both decide what a + // seed produces, so an upgrade or a generator edit must be visible without reading logs. + expect(envelope.tool['fast-check']).toMatch(/^\d+\.\d+\.\d+/); + expect(envelope.configHash).toMatch(/^sha256:/); + expect(envelope.configHash).not.toBe(hashWithout('generate.ts')); + expect(envelope.result).toBe('pass'); + expect(envelope.data.mode).toBe('generate'); + expect(envelope.data.targetRuns[0].target).toBe('selector'); + }); + + // A self-check failure must not leave the lane without an envelope: monitoring reads it and + // the workflow summary prints it on every terminal path. + it('is written for a failing run too', () => { + const { envelope, status } = envelopeFrom([ + '--target', + 'self-check-untyped-throw', + '--iterations', + '1', + '--case-timeout-ms', + '2000', + ]); + expect(status).toBe(1); + expect(envelope.result).toBe('fail'); + expect(envelope.data.failures[0].kind).toBe('untyped-throw'); + expect(envelope.data.reproCommands[0]).toContain('--input-file'); + }); + + // A malformed workflow-dispatch input used to throw out of option parsing before anything + // could write an envelope, which reads to monitoring exactly like a lane that went dark. + it('is written when the options themselves are malformed', () => { + const { envelope, status } = envelopeFrom(['--iterations', 'lots']); + expect(status).toBe(1); + expect(envelope.result).toBe('fail'); + expect(envelope.data.stage).toBe('error'); + expect(envelope.data.targetRuns).toEqual([]); + }); + + it('is written for an unknown flag', () => { + const { envelope, status } = envelopeFrom(['--not-a-flag']); + expect(status).toBe(1); + expect(envelope.result).toBe('fail'); + expect(envelope.data.stage).toBe('error'); + }); + + it('is written for a self-check run', () => { + const { envelope, status } = envelopeFrom(['--self-check', '--case-timeout-ms', '750']); + expect(status).toBe(0); + expect(envelope.result).toBe('pass'); + expect(envelope.data.mode).toBe('self-check'); + expect(envelope.data.targetRuns).toHaveLength(3); + }); +}); + +describe('artifact promotion', () => { + it('promotes a saved failing case into the corpus, once', () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'fuzz-promote-')); + const corpus = path.join(dir, 'regressions.json'); + fs.writeFileSync(corpus, '[]\n'); + const artifact = path.join(dir, 'case.json'); + fs.writeFileSync( + artifact, + JSON.stringify({ + target: 'self-check-untyped-throw', + input: 'promote me', + kind: 'untyped-throw', + detail: 'seeded', + }), + ); + const env = { AGENT_DEVICE_FUZZ_CORPUS: corpus }; + + const first = runHarness(['--input-file', artifact, '--append-corpus'], env); + expect(first.stdout).toContain('Case still violates the invariant'); + expect(first.stdout).toContain('Appended 1 case(s)'); + expect(first.status).toBe(1); + expect(JSON.parse(fs.readFileSync(corpus, 'utf8'))).toEqual([ + { + target: 'self-check-untyped-throw', + input: 'promote me', + note: expect.stringContaining('untyped-throw'), + }, + ]); + + const second = runHarness(['--input-file', artifact, '--append-corpus'], env); + expect(second.stdout).toContain('already contains every failing case'); + expect(JSON.parse(fs.readFileSync(corpus, 'utf8'))).toHaveLength(1); + fs.rmSync(dir, { recursive: true, force: true }); + }); +}); diff --git a/scripts/fuzz/invariant.ts b/scripts/fuzz/invariant.ts new file mode 100644 index 000000000..7935b764b --- /dev/null +++ b/scripts/fuzz/invariant.ts @@ -0,0 +1,63 @@ +// The single invariant the parser fuzz lane enforces (#1414). +// +// Parsers are the front door for agent-authored input, so the contract is not "parses +// correctly" (nobody can say what a mutated string should mean) but "fails well": +// +// 1. a rejection is an `AppError` — never a bare Error, TypeError, string, or undefined; +// 2. the normalized error carries a non-empty `hint`, so the caller is told what to do; +// 3. the case terminates — enforced by the harness watchdog, not by this module, because +// synchronous parsers cannot be interrupted from inside their own tick. + +import { AppError, normalizeError } from '../../src/kernel/errors.ts'; +import type { FuzzTarget, FuzzTargetName } from './target-types.ts'; + +export type FuzzFailureKind = 'untyped-throw' | 'empty-hint' | 'hang'; + +export type FuzzFailure = { + target: FuzzTargetName; + input: string; + kind: FuzzFailureKind; + detail: string; +}; + +/** + * Runs one case and returns the invariant violation it produced, or `null`. + * Accepting the parse is a pass: the lane judges rejections, not results. + */ +export function checkCase(target: FuzzTarget, input: string): FuzzFailure | null { + try { + target.run(input); + return null; + } catch (error) { + if (!(error instanceof AppError)) { + return { + target: target.name, + input, + kind: 'untyped-throw', + detail: describeThrown(error), + }; + } + const hint = normalizeError(error).hint; + if (typeof hint !== 'string' || hint.trim().length === 0) { + return { + target: target.name, + input, + kind: 'empty-hint', + detail: `AppError ${error.code} has no hint: ${error.message}`, + }; + } + return null; + } +} + +function describeThrown(error: unknown): string { + if (error instanceof Error) { + const stackLine = error.stack?.split('\n')[1]?.trim(); + return `${error.name}: ${error.message}${stackLine ? ` (at ${stackLine})` : ''}`; + } + return `non-Error throw: ${typeof error} ${String(error)}`; +} + +export function describeFailure(failure: FuzzFailure): string { + return `[${failure.target}] ${failure.kind}: ${failure.detail}`; +} diff --git a/scripts/fuzz/options.ts b/scripts/fuzz/options.ts new file mode 100644 index 000000000..bd17008fd --- /dev/null +++ b/scripts/fuzz/options.ts @@ -0,0 +1,99 @@ +// Command-line surface for `pnpm fuzz:parsers` (#1414). + +import { parseArgs } from 'node:util'; + +export type FuzzOptions = { + target?: string; + iterations: number; + seed: number; + caseTimeoutMs: number; + artifactDir: string; + inputFile?: string; + appendCorpus: boolean; + replayCorpus: boolean; + selfCheck: boolean; +}; + +export const FUZZ_USAGE = `Usage: pnpm fuzz:parsers [options] + + --target Fuzz one target only (default: all) + --iterations Cases per target (default: 2000) + --seed fast-check seed (default: 1). Same seed = same cases. + --case-timeout-ms Per-case watchdog budget (default: 2000) + --artifact-dir Where failing cases are written (default: .tmp/fuzz) + --input-file Replay a single saved failing case (JSON artifact) and exit + --append-corpus Promote failures (incl. an --input-file artifact) into the corpus + --replay-corpus Replay the checked-in corpus instead of generating cases + --self-check Run the broken-on-purpose targets and require each to be caught +`; + +const DEFAULTS = { + iterations: '2000', + seed: '1', + caseTimeoutMs: '2000', + artifactDir: '.tmp/fuzz', +} as const; + +function positiveInt(raw: string | undefined, name: string): number { + const value = Number(raw); + if (!Number.isInteger(value) || value <= 0) { + throw new Error(`--${name} must be a positive integer (got ${String(raw)}).`); + } + return value; +} + +/** `{ key: value }`, or nothing when the flag was not passed (exactOptionalPropertyTypes). */ +function optional(key: K, value: string | undefined): Record | object { + return value === undefined ? {} : ({ [key]: value } as Record); +} + +/** + * Options to fall back on when argv itself is unusable, so a malformed dispatch input still + * produces an envelope. `--artifact-dir` is recovered positionally: the value that decides *where* + * monitoring looks must survive a rejected flag elsewhere in argv. + */ +export function fallbackFuzzOptions(argv: readonly string[]): FuzzOptions { + const flag = argv.indexOf('--artifact-dir'); + const dir = flag === -1 ? undefined : argv[flag + 1]; + return { + iterations: Number(DEFAULTS.iterations), + seed: Number(DEFAULTS.seed), + caseTimeoutMs: Number(DEFAULTS.caseTimeoutMs), + artifactDir: dir !== undefined && !dir.startsWith('--') ? dir : DEFAULTS.artifactDir, + appendCorpus: false, + replayCorpus: false, + selfCheck: argv.includes('--self-check'), + }; +} + +/** Parses argv into options; `null` means usage was requested and nothing should run. */ +export function readFuzzOptions(argv: readonly string[]): FuzzOptions | null { + const { values } = parseArgs({ + args: [...argv], + options: { + target: { type: 'string' }, + iterations: { type: 'string', default: DEFAULTS.iterations }, + seed: { type: 'string', default: DEFAULTS.seed }, + 'case-timeout-ms': { type: 'string', default: DEFAULTS.caseTimeoutMs }, + 'artifact-dir': { type: 'string', default: DEFAULTS.artifactDir }, + 'input-file': { type: 'string' }, + 'append-corpus': { type: 'boolean', default: false }, + 'replay-corpus': { type: 'boolean', default: false }, + 'self-check': { type: 'boolean', default: false }, + help: { type: 'boolean', short: 'h', default: false }, + }, + allowPositionals: false, + }); + if (values.help === true) return null; + return { + ...optional('target', values.target), + ...optional('inputFile', values['input-file']), + iterations: positiveInt(values.iterations, 'iterations'), + seed: positiveInt(values.seed, 'seed'), + caseTimeoutMs: positiveInt(values['case-timeout-ms'], 'case-timeout-ms'), + artifactDir: String(values['artifact-dir']), + appendCorpus: values['append-corpus'] === true, + replayCorpus: values['replay-corpus'] === true, + selfCheck: values['self-check'] === true, + }; +} diff --git a/scripts/fuzz/registry.ts b/scripts/fuzz/registry.ts new file mode 100644 index 000000000..5be54a1f1 --- /dev/null +++ b/scripts/fuzz/registry.ts @@ -0,0 +1,21 @@ +// Name → target lookup for the parser fuzz lane (#1414). +// +// Resolves the real parser targets plus the broken-on-purpose self-check targets, which is the +// only place the two lists meet. The worker resolves targets by name (a target cannot cross a +// worker boundary as a value), so this lookup is what lets self-check cases run in the same +// worker/watchdog path as real ones. + +import { SELF_CHECK_TARGETS } from './self-check-targets.ts'; +import type { FuzzTarget } from './target-types.ts'; +import { FUZZ_TARGETS } from './targets.ts'; + +export function getFuzzTarget(name: string): FuzzTarget { + const target = [...FUZZ_TARGETS, ...SELF_CHECK_TARGETS].find( + (candidate) => candidate.name === name, + ); + if (!target) { + const known = FUZZ_TARGETS.map((candidate) => candidate.name).join(', '); + throw new Error(`Unknown fuzz target "${name}". Known targets: ${known}`); + } + return target; +} diff --git a/scripts/fuzz/report.ts b/scripts/fuzz/report.ts new file mode 100644 index 000000000..667dbd121 --- /dev/null +++ b/scripts/fuzz/report.ts @@ -0,0 +1,61 @@ +// Failure reporting for the parser fuzz lane (#1414). +// +// A nightly failure has to be actionable from the run log alone, so every failing case is +// written as an artifact and printed with two copy-pasteable commands: one that reproduces it +// and one that promotes it into the checked-in corpus the unit lane replays. + +import fs from 'node:fs'; +import path from 'node:path'; +import { appendToCorpus, type CorpusEntry } from './corpus.ts'; +import { describeFailure, type FuzzFailure } from './invariant.ts'; + +export type ReportedFailure = FuzzFailure & { artifact?: string }; + +function writeArtifact(artifactDir: string, failure: FuzzFailure, ordinal: number): string { + fs.mkdirSync(artifactDir, { recursive: true }); + const file = path.join(artifactDir, `${failure.target}-${failure.kind}-${ordinal}.json`); + fs.writeFileSync(file, `${JSON.stringify(failure, null, 2)}\n`); + return file; +} + +function corpusEntryFor(failure: FuzzFailure): CorpusEntry { + return { + target: failure.target, + input: failure.input, + note: `${failure.kind}: ${failure.detail}`, + }; +} + +/** Writes an artifact per failure and prints repro + promote commands. */ +export function reportFailures( + failures: readonly FuzzFailure[], + artifactDir: string, +): { reported: ReportedFailure[]; reproCommands: string[] } { + if (failures.length === 0) return { reported: [], reproCommands: [] }; + process.stdout.write(`\n${failures.length} invariant violation(s):\n`); + const reported: ReportedFailure[] = []; + const reproCommands: string[] = []; + failures.forEach((failure, ordinal) => { + const artifact = writeArtifact(artifactDir, failure, ordinal); + const repro = `pnpm fuzz:parsers --input-file ${artifact}`; + reproCommands.push(repro); + reported.push({ ...failure, artifact }); + process.stdout.write(`\n${describeFailure(failure)}\n`); + process.stdout.write(` input: ${JSON.stringify(failure.input)}\n`); + process.stdout.write(` repro: ${repro}\n`); + process.stdout.write(` promote: ${repro} --append-corpus\n`); + }); + return { reported, reproCommands }; +} + +/** Appends failing cases to the checked-in corpus, printing what was added. */ +export function promoteFailures(failures: readonly FuzzFailure[]): CorpusEntry[] { + if (failures.length === 0) return []; + const added = appendToCorpus(failures.map(corpusEntryFor)); + process.stdout.write( + added.length === 0 + ? '\nRegression corpus already contains every failing case.\n' + : `\nAppended ${added.length} case(s) to the regression corpus.\n`, + ); + return added; +} diff --git a/scripts/fuzz/run.ts b/scripts/fuzz/run.ts new file mode 100644 index 000000000..e675d5887 --- /dev/null +++ b/scripts/fuzz/run.ts @@ -0,0 +1,219 @@ +// Entry point for `pnpm fuzz:parsers` — the nightly parser fuzz lane (#1414). +// +// Feeds mutated hostile input to the CLI/selector/replay/batch/Maestro parsers and enforces +// one invariant: a rejection is a typed AppError with a non-empty hint, and no case hangs. +// Every terminal path — pass, fail, self-check, or a crash in the harness itself — writes one +// scheduled-lane envelope, so monitoring never reads a missing file. Failing cases are written +// as artifacts and printed with repro and corpus-promotion commands. + +import { readCorpus } from './corpus.ts'; +import { + type FuzzEnvelopeDetails, + type FuzzRunMode, + type FuzzTargetRun, + writeFuzzEnvelope, +} from './envelope.ts'; +import { runCases } from './execute.ts'; +import { generateAndCheck } from './generate.ts'; +import { describeFailure, type FuzzFailure } from './invariant.ts'; +import { fallbackFuzzOptions, FUZZ_USAGE, readFuzzOptions, type FuzzOptions } from './options.ts'; +import { getFuzzTarget } from './registry.ts'; +import { promoteFailures, reportFailures } from './report.ts'; +import { readSavedCase } from './saved-case.ts'; +import { runSelfCheck } from './self-check.ts'; +import type { FuzzTarget } from './target-types.ts'; +import { FUZZ_TARGETS } from './targets.ts'; + +function selectTargets(name: string | undefined): FuzzTarget[] { + return name === undefined ? [...FUZZ_TARGETS] : [getFuzzTarget(name)]; +} + +function corpusCasesFor(target: FuzzTarget): string[] { + return readCorpus() + .filter((entry) => entry.target === target.name) + .map((entry) => entry.input); +} + +/** One target's run: generated (fast-check, shrinking) or a replay of the checked-in corpus. */ +async function runOneTarget( + target: FuzzTarget, + options: FuzzOptions, +): Promise<{ cases: number; failures: FuzzFailure[]; replayHint?: string }> { + if (options.replayCorpus) { + const cases = corpusCasesFor(target); + return { cases: cases.length, failures: await runCases(target, cases, options.caseTimeoutMs) }; + } + const run = await generateAndCheck(target, options); + return { + cases: run.cases, + failures: run.failure ? [run.failure] : [], + ...(run.replay + ? { replayHint: `fast-check replay: --seed ${run.replay.seed} (path ${run.replay.path})` } + : {}), + }; +} + +/** Fuzzes every selected target, printing a per-target line as each finishes. */ +async function fuzzTargets(targets: readonly FuzzTarget[], options: FuzzOptions) { + const failures: FuzzFailure[] = []; + const targetRuns: FuzzTargetRun[] = []; + for (const target of targets) { + const started = Date.now(); + const { cases, failures: found, replayHint } = await runOneTarget(target, options); + const durationMs = Date.now() - started; + targetRuns.push({ target: target.name, cases, failures: found.length, durationMs }); + process.stdout.write( + `${target.name}: ${cases} cases, ${found.length} failures (${durationMs}ms)\n`, + ); + if (replayHint !== undefined) process.stdout.write(` ${replayHint}\n`); + failures.push(...found); + } + return { failures, targetRuns }; +} + +function verdictLine(targetName: string, failed: boolean): string { + return failed + ? `Case still violates the invariant (${targetName}).\n` + : `Case passes the invariant now (${targetName}).\n`; +} + +type ModeOutcome = { + mode: FuzzRunMode; + failed: boolean; + targetRuns: FuzzTargetRun[]; + details?: Partial; + summary: (envelope: string) => string; +}; + +/** Replays a saved artifact, optionally promoting it into the checked-in corpus. */ +async function replaySavedCase(options: FuzzOptions): Promise { + const started = Date.now(); + const { target, input } = readSavedCase(options.inputFile!); + const failures = await runCases(target, [input], options.caseTimeoutMs); + for (const failure of failures) process.stdout.write(`${describeFailure(failure)}\n`); + process.stdout.write(verdictLine(target.name, failures.length > 0)); + if (options.appendCorpus) promoteFailures(failures); + return { + mode: 'replay-artifact', + failed: failures.length > 0, + targetRuns: [ + { + target: target.name, + cases: 1, + failures: failures.length, + durationMs: Date.now() - started, + }, + ], + details: { failures }, + summary: (envelope) => `Envelope: ${envelope}\n`, + }; +} + +async function selfCheckMode(options: FuzzOptions): Promise { + const { ok, targetRuns } = await runSelfCheck(options); + return { + mode: 'self-check', + failed: !ok, + targetRuns, + summary: (envelope) => `Envelope: ${envelope}\n`, + }; +} + +async function fuzzMode(options: FuzzOptions): Promise { + const targets = selectTargets(options.target); + const { failures, targetRuns } = await fuzzTargets(targets, options); + const { reported, reproCommands } = reportFailures(failures, options.artifactDir); + if (options.appendCorpus) promoteFailures(failures); + const failed = failures.length > 0; + return { + mode: options.replayCorpus ? 'replay-corpus' : 'generate', + failed, + targetRuns, + details: { failures: reported, reproCommands }, + summary: (envelope) => + failed + ? `\nEnvelope: ${envelope}\n` + : `Invariant held across ${targets.length} target(s), seed ${options.seed}. Envelope: ${envelope}\n`, + }; +} + +function runMode(options: FuzzOptions): Promise { + if (options.selfCheck) return selfCheckMode(options); + if (options.inputFile !== undefined) return replaySavedCase(options); + return fuzzMode(options); +} + +function configFor(options: FuzzOptions): Record { + return { + seed: options.seed, + iterations: options.iterations, + caseTimeoutMs: options.caseTimeoutMs, + targets: options.target === undefined ? 'all' : [options.target], + }; +} + +/** Writes the envelope every terminal path owes the scheduled-lane contract. */ +function writeOutcomeEnvelope( + options: FuzzOptions, + startedAt: number, + outcome: ModeOutcome, + result: 'pass' | 'fail' | 'error', +): string { + return writeFuzzEnvelope({ + artifactDir: options.artifactDir, + startedAt, + finishedAt: Date.now(), + result, + config: configFor(options), + details: { + mode: outcome.mode, + corpusEntries: readCorpus().length, + targetRuns: outcome.targetRuns, + failures: [], + reproCommands: [], + ...outcome.details, + }, + }); +} + +/** A harness crash still owes an envelope, otherwise monitoring sees nothing at all. */ +function reportCrash(options: FuzzOptions, startedAt: number, error: unknown): number { + const mode: FuzzRunMode = options.selfCheck ? 'self-check' : 'generate'; + const outcome: ModeOutcome = { mode, failed: true, targetRuns: [], summary: () => '' }; + const envelope = writeOutcomeEnvelope(options, startedAt, outcome, 'error'); + process.stderr.write(`${String(error)}\nEnvelope: ${envelope}\n`); + return 1; +} + +async function run(options: FuzzOptions, startedAt: number): Promise { + try { + const outcome = await runMode(options); + const result = outcome.failed ? 'fail' : 'pass'; + process.stdout.write( + outcome.summary(writeOutcomeEnvelope(options, startedAt, outcome, result)), + ); + return outcome.failed ? 1 : 0; + } catch (error) { + return reportCrash(options, startedAt, error); + } +} + +async function main(): Promise { + const startedAt = Date.now(); + const argv = process.argv.slice(2); + // Option parsing is inside the guarded path on purpose: a malformed workflow-dispatch input is + // exactly the kind of terminal failure monitoring must still see an envelope for. + let options: FuzzOptions | null; + try { + options = readFuzzOptions(argv); + } catch (error) { + return reportCrash(fallbackFuzzOptions(argv), startedAt, error); + } + if (!options) { + process.stdout.write(FUZZ_USAGE); + return 0; + } + return await run(options, startedAt); +} + +process.exitCode = await main(); diff --git a/scripts/fuzz/saved-case.ts b/scripts/fuzz/saved-case.ts new file mode 100644 index 000000000..d01494245 --- /dev/null +++ b/scripts/fuzz/saved-case.ts @@ -0,0 +1,17 @@ +// Reading a saved failing case back (#1414). +// +// The artifact a nightly failure uploads is the unit of reproduction: `--input-file` re-runs it +// and `--append-corpus` promotes it, so its shape is validated in one place. + +import fs from 'node:fs'; +import { getFuzzTarget } from './registry.ts'; +import type { FuzzTarget } from './target-types.ts'; + +export function readSavedCase(inputFile: string): { target: FuzzTarget; input: string } { + const saved: unknown = JSON.parse(fs.readFileSync(inputFile, 'utf8')); + const { target, input } = Object(saved) as Record; + if (typeof target !== 'string' || typeof input !== 'string') { + throw new Error(`${inputFile} needs string target and input fields.`); + } + return { target: getFuzzTarget(target), input }; +} diff --git a/scripts/fuzz/self-check-targets.ts b/scripts/fuzz/self-check-targets.ts new file mode 100644 index 000000000..3aa51ab7f --- /dev/null +++ b/scripts/fuzz/self-check-targets.ts @@ -0,0 +1,48 @@ +// Broken-on-purpose targets that prove the harness can still fail (#1414). +// +// A fuzz lane whose classifier or watchdog regresses goes green forever, which is worse than +// no lane at all. These targets each violate the invariant one way, so `--self-check` (and the +// unit test in harness.test.ts) can assert the harness reports every failure kind. They are +// deliberately kept out of FUZZ_TARGETS: only the registry resolves them by name, so a normal +// run never sees them and no production module gains a test-only seam. + +import { AppError } from '../../src/kernel/errors.ts'; +import type { FuzzTarget, SelfCheckTargetName } from './target-types.ts'; + +const SEEDS = ['self-check']; + +export const SELF_CHECK_TARGETS: readonly FuzzTarget[] = [ + { + name: 'self-check-untyped-throw', + description: 'always throws a bare Error (expects an untyped-throw failure)', + run: () => { + throw new Error('self-check untyped throw'); + }, + seeds: SEEDS, + }, + { + name: 'self-check-empty-hint', + description: 'always throws an AppError without a hint (expects an empty-hint failure)', + run: () => { + throw new AppError('INVALID_ARGS', 'self-check missing hint', { hint: ' ' }); + }, + seeds: SEEDS, + }, + { + name: 'self-check-hang', + description: 'never returns (expects a hang failure)', + run: () => { + for (;;) { + // Spin forever: only the out-of-thread watchdog can end this case. + } + }, + seeds: SEEDS, + }, +]; + +/** The failure kind each self-check target must produce, keyed by target name. */ +export const SELF_CHECK_EXPECTATIONS = { + 'self-check-untyped-throw': 'untyped-throw', + 'self-check-empty-hint': 'empty-hint', + 'self-check-hang': 'hang', +} as const satisfies Record; diff --git a/scripts/fuzz/self-check.ts b/scripts/fuzz/self-check.ts new file mode 100644 index 000000000..67a588e11 --- /dev/null +++ b/scripts/fuzz/self-check.ts @@ -0,0 +1,62 @@ +// `pnpm fuzz:parsers --self-check` — proves the harness can still fail (#1414). +// +// Runs each broken-on-purpose target through the same worker/watchdog path a real case takes +// and asserts the expected failure kind comes back. Inverted expectations are the point: this +// mode fails when the harness reports nothing. + +import type { FuzzTargetRun } from './envelope.ts'; +import { runCases } from './execute.ts'; +import { describeFailure } from './invariant.ts'; +import type { FuzzOptions } from './options.ts'; +import { SELF_CHECK_EXPECTATIONS, SELF_CHECK_TARGETS } from './self-check-targets.ts'; +import type { SelfCheckTargetName } from './target-types.ts'; + +type SelfCheckResult = { + target: string; + expected: string; + actual: string; + ok: boolean; + durationMs: number; +}; + +/** Runs every self-check target once; `caseTimeoutMs` also bounds the hang target. */ +async function selfCheckResults(caseTimeoutMs: number): Promise { + const results: SelfCheckResult[] = []; + for (const target of SELF_CHECK_TARGETS) { + const expected = SELF_CHECK_EXPECTATIONS[target.name as SelfCheckTargetName]; + const started = Date.now(); + const failures = await runCases(target, [...target.seeds], caseTimeoutMs); + const actual = failures[0] ? failures[0].kind : 'none'; + results.push({ + target: target.name, + expected, + actual, + ok: actual === expected, + durationMs: Date.now() - started, + }); + if (failures[0]) process.stdout.write(` ${describeFailure(failures[0])}\n`); + } + return results; +} + +/** Runs the self-check and reports per-target rows for the run envelope. */ +export async function runSelfCheck( + options: FuzzOptions, +): Promise<{ ok: boolean; targetRuns: FuzzTargetRun[] }> { + process.stdout.write('Self-check: the harness must catch each seeded violation.\n'); + const results = await selfCheckResults(options.caseTimeoutMs); + for (const result of results) { + process.stdout.write( + `${result.ok ? 'ok ' : 'FAIL'} ${result.target}: expected ${result.expected}, got ${result.actual}\n`, + ); + } + return { + ok: results.every((result) => result.ok), + targetRuns: results.map((result) => ({ + target: result.target, + cases: 1, + failures: result.ok ? 1 : 0, + durationMs: result.durationMs, + })), + }; +} diff --git a/scripts/fuzz/target-types.ts b/scripts/fuzz/target-types.ts new file mode 100644 index 000000000..773b7373f --- /dev/null +++ b/scripts/fuzz/target-types.ts @@ -0,0 +1,31 @@ +// Shape of a parser fuzz target (#1414). +// +// Lives apart from the target lists so the real parser targets (targets.ts) and the +// deliberately-broken self-check targets (self-check-targets.ts) can share it without an +// import cycle through the registry that unifies them. + +/** Names of the real parser targets the lane fuzzes. */ +export type ParserTargetName = + | 'cli-args' + | 'selector' + | 'replay-script' + | 'batch-steps' + | 'maestro'; + +/** Names of the broken-on-purpose targets that prove the harness can still fail. */ +export type SelfCheckTargetName = + | 'self-check-untyped-throw' + | 'self-check-empty-hint' + | 'self-check-hang'; + +export type FuzzTargetName = ParserTargetName | SelfCheckTargetName; + +export type FuzzTarget = { + name: FuzzTargetName; + /** Human-readable description used in failure reports. */ + description: string; + /** Runs the parser on one case; may throw. */ + run: (input: string) => void; + /** Valid-ish inputs the mutator derives cases from. */ + seeds: string[]; +}; diff --git a/scripts/fuzz/targets.ts b/scripts/fuzz/targets.ts new file mode 100644 index 000000000..16b88bba0 --- /dev/null +++ b/scripts/fuzz/targets.ts @@ -0,0 +1,110 @@ +// Parser fuzz targets (#1414). +// +// Every target takes one string case and calls a real parser. The harness owns the +// invariant (typed AppError with a non-empty hint, no hang); a target only says how a +// case string reaches its parser, and supplies the seed inputs the mutator chews on. +// +// Keep targets pure and synchronous: the hang watchdog (scripts/fuzz/worker.ts) can only +// attribute a stall to a case if the case runs to completion in one tick. + +import { parseArgs } from '../../src/cli/parser/args.ts'; +import { parseSelectorChain } from '../../src/selectors/parse.ts'; +import { parseReplayScriptDetailed } from '../../src/replay/script.ts'; +import { readCliBatchStepsJson } from '../../src/cli/batch-steps.ts'; +import { parseMaestroProgram } from '../../src/compat/maestro/program-ir-parser.ts'; +import type { FuzzTarget } from './target-types.ts'; + +// argv is carried as one string so a case (and its corpus entry, artifact, and repro +// command) stays a single copy-pasteable value. Splitting on spaces is deliberate: the +// fuzzer wants odd tokens, not a faithful shell grammar. +function toArgv(input: string): string[] { + return input.split(' ').filter((token) => token.length > 0); +} + +export const FUZZ_TARGETS: readonly FuzzTarget[] = [ + { + name: 'cli-args', + description: 'parseArgs (strict flags)', + run: (input) => void parseArgs(toArgv(input), { strictFlags: true }), + seeds: [ + 'click --selector text=Login', + 'open com.example.app --platform ios --json', + 'snapshot --depth 3 --format text', + 'fill @e1 --text hello --submit', + 'batch --steps [] --timeout 1000', + 'devices --platform android --json --debug', + 'wait --selector role=button --timeout-ms 500', + 'test replays -- --raw --passthrough', + 'click --selector', + '--json', + '', + ], + }, + { + name: 'selector', + description: 'parseSelectorChain', + run: (input) => void parseSelectorChain(input), + seeds: [ + 'text=Login', + 'label="Sign in" && role=button', + 'text=Save || label=Done', + 'id=com.example:id/button[2]', + 'role=button and is=enabled', + 'text~=partial', + 'label="quoted \\"inner\\" value"', + '@e1', + 'text=🚀', + 'text=', + '', + ], + }, + { + name: 'replay-script', + description: 'parseReplayScriptDetailed (.ad scripts)', + run: (input) => void parseReplayScriptDetailed(input), + seeds: [ + 'open com.example.app\nclick text=Login\nclose\n', + '# context platform=ios target=mobile\nopen com.example.app\n', + '# context timeoutMs=1000 retries=2\nsnapshot\n', + '# target-v1 role=button label=Login\nclick @e1\n', + 'fill @e1 --text "hello world"\nassert text=Welcome\n', + 'swipe 10 20 30 40\nwait 250\n', + 'env FOO=bar\nclick text=${FOO}\n', + 'screenshot --quality low\n', + '# target-v1 role=button\n\nclick @e1\n', + '', + ], + }, + { + name: 'batch-steps', + description: 'readCliBatchStepsJson (batch --steps)', + run: (input) => void readCliBatchStepsJson(input), + seeds: [ + '[{"command":"snapshot","input":{}}]', + '[{"command":"click","input":{"selector":"text=Login"}}]', + '[{"command":"open","positionals":["com.example.app"],"flags":{"json":true}}]', + '[{"command":"snapshot","input":{}},{"command":"close","input":{}}]', + '[{"command":"wait","input":{"timeoutMs":250}}]', + '[]', + '{}', + 'not json', + '', + ], + }, + { + name: 'maestro', + description: 'parseMaestroProgram (Maestro compat)', + run: (input) => void parseMaestroProgram(input, { sourcePath: 'fuzz.yaml' }), + seeds: [ + 'appId: com.example.app\n---\n- launchApp\n- tapOn: "Login"\n', + 'appId: com.example.app\n---\n- tapOn:\n id: "login"\n', + 'appId: com.example.app\n---\n- inputText: "hello"\n- assertVisible: "Welcome"\n', + 'appId: com.example.app\n---\n- swipe:\n direction: UP\n', + 'appId: com.example.app\n---\n- runFlow: other.yaml\n', + 'appId: com.example.app\n---\n- repeat:\n times: 2\n commands:\n - back\n', + '- launchApp\n', + 'appId: com.example.app\n---\n', + '', + ], + }, +]; diff --git a/scripts/fuzz/worker.ts b/scripts/fuzz/worker.ts new file mode 100644 index 000000000..3997b14b0 --- /dev/null +++ b/scripts/fuzz/worker.ts @@ -0,0 +1,35 @@ +// Case-executing worker for the parser fuzz lane (#1414). +// +// The cases run off the main thread for one reason: a synchronous parser that never returns +// cannot be timed out from inside its own tick. The worker answers one case at a time; the +// runner (scripts/fuzz/execute.ts) budgets the answer from the main thread and terminates this +// thread when a case stops answering, which is how a hang is attributed to an exact input. + +import { parentPort, workerData } from 'node:worker_threads'; +import { checkCase, type FuzzFailure } from './invariant.ts'; +import { getFuzzTarget } from './registry.ts'; + +export type FuzzWorkerData = { targetName: string }; + +export type FuzzWorkerRequest = { kind: 'case'; input: string }; + +export type FuzzWorkerMessage = { kind: 'ready' } | { kind: 'result'; failure: FuzzFailure | null }; + +const port = parentPort; +if (!port) throw new Error('scripts/fuzz/worker.ts must be run as a worker thread.'); + +const { targetName } = workerData as FuzzWorkerData; +const target = getFuzzTarget(targetName); + +// The batch-steps parser warns on deprecated step shapes; a fuzz run would emit thousands of +// those lines and bury the failure report. +process.stderr.write = (() => true) as typeof process.stderr.write; + +port.on('message', (request: FuzzWorkerRequest) => { + const failure = checkCase(target, request.input); + port.postMessage({ kind: 'result', failure } satisfies FuzzWorkerMessage); +}); + +// Module loading (type stripping, parser imports) can outlast a per-case budget, so the runner +// only starts a case budget after this handshake. +port.postMessage({ kind: 'ready' } satisfies FuzzWorkerMessage); diff --git a/src/__tests__/test-utils/property-arbitraries.ts b/src/__tests__/test-utils/property-arbitraries.ts index b01ae4306..765f746b6 100644 --- a/src/__tests__/test-utils/property-arbitraries.ts +++ b/src/__tests__/test-utils/property-arbitraries.ts @@ -73,8 +73,12 @@ function selectorKeysOfKind(kind: 'text' | 'boolean'): SelectorKey[] { * The shapes that broke hand-written selector examples: both quote characters, * backslash runs before a quote, the `||` fallback separator and `=` inside a * value, whitespace-only values, and non-BMP text. + * + * Exported because the nightly parser fuzz lane (#1414) generates from this same + * list: a hazard added here for a round-trip property reaches the fuzzer too, + * instead of the two suites drifting into two vocabularies. */ -const SELECTOR_VALUE_HAZARDS = [ +export const SELECTOR_VALUE_HAZARDS = [ '', ' ', '"', diff --git a/src/replay/__tests__/script.test.ts b/src/replay/__tests__/script.test.ts index d1d185ff2..3a7ea288f 100644 --- a/src/replay/__tests__/script.test.ts +++ b/src/replay/__tests__/script.test.ts @@ -473,6 +473,23 @@ test('a malformed target-v1 payload is rejected as INVALID_ARGS, not silently dr ); }); +// Found by the nightly parser fuzz lane (#1414): the closing-quote scan accepted these +// literals and the JSON decode behind it leaked a raw SyntaxError. +test.each([ + ['invalid escape', 'fill @e1 --text "hello wor\\ld"'], + ['raw control character', 'fill @e1 --text "hel\u0000lo"'], + ['raw tab', 'fill @e1 --text "hello\tworld"'], +])('a quoted value with an %s is rejected as INVALID_ARGS with a hint', (_case, script) => { + assert.throws( + () => parseReplayScriptDetailed(script), + (error: unknown) => + error instanceof AppError && + error.code === 'INVALID_ARGS' && + typeof error.details?.hint === 'string' && + error.details.hint.length > 0, + ); +}); + test('an unknown future target-vN comment is an ordinary comment: no binding requirement, no evidence attached', () => { const script = ['# agent-device:target-v2 {"whatever":true}', '', 'click @e12 "Save"'].join('\n'); const { actions } = parseReplayScriptDetailed(script); diff --git a/src/replay/script.ts b/src/replay/script.ts index 13ad3da92..fb58399d7 100644 --- a/src/replay/script.ts +++ b/src/replay/script.ts @@ -503,10 +503,22 @@ function readQuotedReplayToken( if (end >= line.length) { throw new AppError('INVALID_ARGS', `Invalid replay script line: ${line}`); } - return { - value: JSON.parse(line.slice(cursor, end + 1)) as string, - nextCursor: end + 1, - }; + // The scan above only locates the closing quote; the literal between the quotes can still + // carry an invalid escape or a raw control character that JSON.parse refuses. + const literal = line.slice(cursor, end + 1); + let value: unknown; + try { + value = JSON.parse(literal); + } catch { + throw new AppError( + 'INVALID_ARGS', + `Invalid quoted value ${literal} in replay script line: ${line}`, + { + hint: 'Quoted replay values are JSON strings: escape backslashes, quotes, tabs, and newlines (\\\\, \\", \\t, \\n).', + }, + ); + } + return { value: value as string, nextCursor: end + 1 }; } function readBareReplayToken(line: string, cursor: number): { value: string; nextCursor: number } { diff --git a/vitest.config.ts b/vitest.config.ts index 0a69ec351..969645513 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -16,6 +16,12 @@ export const SUBPROCESS_STUB_TESTS = [ 'src/platforms/apple/core/__tests__/index.test.ts', // Stubs npx + the package managers on PATH and spawns a real Metro dev server per case. 'src/__tests__/client-metro.test.ts', + // Proves the parser fuzz harness still fails (#1414): every case spawns a node subprocess or a + // worker thread and one target is a deliberate hang, so it waits real watchdog time. + 'scripts/fuzz/harness.test.ts', + // Replays the fuzz regression corpus (#1414) through that same worker watchdog, so a promoted + // hang case fails against its per-case budget instead of wedging the unit job. + 'scripts/fuzz/corpus-replay.test.ts', ]; export default defineConfig({