diff --git a/.changeset/gap-cli-runtime.md b/.changeset/gap-cli-runtime.md index 8e550ad..b0b47ed 100644 --- a/.changeset/gap-cli-runtime.md +++ b/.changeset/gap-cli-runtime.md @@ -2,9 +2,9 @@ '@tanstack/oracles': minor --- -The `oracle-gaps` CLI bounds a mutant's confirming runs, reports long runs, stops the processes it started, and writes nothing beside the tests. +The `oracle-gaps` CLI bounds a mutant's copies by counts, not by wall-clock limits, reports long runs, stops the processes it started, and writes nothing beside the tests. -- Each mutant has a budget for the runs that confirm its cut-off copies: the larger of the new `--mutant-limit` (30 s by default) and ten times its owner files' unmutated time. The confirmations run in the owner files' order, each with at most what the mutant's earlier confirmations left, so the outcome depends only on the recorded runs, not on their order or on reuse through the daemon. A mutant whose confirmations pass the budget is a `timeout` gap. With the default limits a single owner's confirmation always fits; the budget binds when several owners need long confirmations, or when `--hang-limit` is above it, which the run notes. To give one copy more time with `--hang-limit`, raise `--mutant-limit` too. +- No wall-clock limit decides an outcome. A copy that spins past its call budget, makes no progress, or whose work passes the mutant's work budget (the larger of 100,000 and ten times its owner files' unmutated work, counted in site evaluations, not in time) is killed by the built-in law `bounded work` or `no progress` on its test's route. A copy's call budget is a hundred times the most site evaluations that any unmutated test of its file made, and at least 100,000. A spin or a work-budget overrun kills only when the counts that decide it cannot follow the clock: a test that reads the clock, makes a timer, or does I/O of its own anywhere in its run is charged nothing, and its spin is a `timeout` named `(cause: spin)`. A copy makes no progress when none of its test's own timers is due before its backstop and none of its test's own I/O is pending; timers and I/O the runner keeps for itself, such as Vitest's task-update timer or a pending write that forwards the test's console output on Windows, do not keep it busy. Only the backstop (`--hang-limit`, 60 s by default, plus 3 s per second of the file's time) is a gap: the mutant is a `timeout`, named `(cause: backstop)` on its gap line, and no later copy runs. `--mutant-limit` and `--first-limit` are retired: they are checked and noted, and change nothing. - While one copy of a test file runs for 30 s or more, the CLI prints `busy: in for s` every 30 s (`baseline` for a run with no mutant). Through `--daemon` the heartbeats arrive when the check ends. - The check runs in a process group of its own, with every Vitest worker and every process a test starts, except one started detached in a session of its own. On SIGINT or SIGTERM the CLI signals the group, kills what is left 2 s later, and exits 130 or 143. Before, a SIGTERM left `forks` workers running. On Windows the run is not supervised. - The runner's copies of the test files exist only in memory. Vitest's read of a copy for its per-file pragmas (`@vitest-environment`, `@module-tag`) is served its text, so a copy runs in its file's environment. The capture setup and Vitest configs are in a session directory of its own while it runs, which the run removes when it ends; one that a killed run left is removed by the next run. Before, the copies and configs sat beside the tests and the package's config, where a Vitest run started during a check found them and then failed when they were removed. diff --git a/README.md b/README.md index 4899460..e4bb081 100644 --- a/README.md +++ b/README.md @@ -35,15 +35,25 @@ agent. While one copy of a test file runs for 30 s or more, the CLI prints `oracle-gaps: busy in for s` every 30 s. A run of many fast copies prints none, and a check through `--daemon` prints its heartbeats - only when the check ends. The runs that confirm a mutant's cut-off copies - share one budget: the larger of `--mutant-limit` (30 s by default) and ten - times the owner files' unmutated time. They run in the owner files' - order, each with at most what its mutant's earlier confirmations left. A - mutant whose confirmations pass the budget is a `timeout` gap. With the - default limits one owner's confirmation always fits; the budget binds - when several owners need long confirmations, or when `--hang-limit` is - above it, which the run notes. To give one copy more time with - `--hang-limit`, raise `--mutant-limit` too. + only when the check ends. No wall-clock limit decides an outcome. A copy + is cut in one of three ways: by its call count, when it runs past its + call budget (`spin`); by its state, when it makes no progress (`no progress`: + none of its test's own timers is due before its backstop, and none of its + test's own I/O is pending, while timers and I/O the runner keeps for itself, + such as Vitest's task-update timer or a write that forwards the test's + console output, do not count); or by the backstop, when it runs past `--hang-limit` (60 s by default) plus + 3 s per second of its file's unmutated time (`backstop`). A spin and no + progress are kills: the built-in law `bounded work` or `no progress` fails + on the route of the test that was running, so on a reproducible route the + mutant is `killed`, and on another route the next owner's copy still runs. + The copies of one mutant share a work budget, counted in site evaluations + and not in time: the larger of 100,000 and ten times their owner files' + unmutated work. The copy whose work passes it fails `bounded work` on the + routes of its tests that did that work, and no later copy runs. Only the + backstop is a gap: the mutant is a `timeout`, with `(cause: backstop)` on + its gap line and `detail` in `results.json`, and no later copy runs. A kill + in an earlier owner still decides. `--first-limit` and `--mutant-limit` are retired: + they are checked and noted, and change nothing. The report, on stdout and in `summary.md`, leads with the baseline: each test file's result with its test counts (ordinary files too), and for a @@ -108,7 +118,7 @@ agent. | `baseline` | `oracle-gaps: baseline pass tests/a.test.ts: 3 tests`, `oracle-gaps: baseline fail tests/b.test.ts: 1 of 2 tests failed` | | `baseline` (decisive, before any mutant) | `oracle-gaps: baseline all 4 oracle files pass`, or `oracle-gaps: baseline 1 of 4 oracle files failed (provisional)` | | `blocked` | `oracle-gaps: blocked fails with no mutant active at , so the mutants it owns wait for it: …` | - | `gap` | `oracle-gaps: gap L ` | + | `gap` | `oracle-gaps: gap L `, and for a `timeout` ` (cause: )` | | `progress` | `oracle-gaps: progress judged 25 of 120 mutants: 7 gaps so far, 18 not gaps` | | `busy` | `oracle-gaps: busy in for s` | | `note` | `oracle-gaps: note …`: the `--hang-limit` note, Vite's own messages (`note vite: …`), ordinary files left out, the provisional count, unowned counts, the results' path, and where the tests' console output went (`note test output in `) | @@ -199,6 +209,19 @@ agent. watchdog has polled twice. The directory holds a `.begun` marker per batch and a `.reached` or `.unreached` marker per hold (`.unreached` after 60 s with no two polls). + - `ORACLE_GAPS_TEST_SAMPLES` names a directory where each worker's idle + sampler writes, to `-`, how many times it has judged the + running test. A witness holds its own I/O open until that count rises, so + the hold is a count and not a time. + - `ORACLE_GAPS_TEST_RUNNER_WRITE` names a directory. Through each test, the + runner itself holds one write pending on a local pipe whose reader never + reads, as a write that forwards a test's output can stay pending on + win32. A copy that goes idle while that write is pending writes an + `idle--` marker there. + - `ORACLE_GAPS_TEST_RUNNER_TIMER` names a directory. Through each test, the + runner itself keeps one 100 ms interval running, made through the global + `setInterval` as Vitest's task-update timer is. A copy that goes idle while + it runs writes an `idle--` marker there. A few endings print no done line, or one more line: a SIGKILL of the CLI, or a signal that comes while Node is still starting the CLI, before its diff --git a/packages/oracles/oracle-gaps/args.mjs b/packages/oracles/oracle-gaps/args.mjs index 010c8c5..67a9328 100644 --- a/packages/oracles/oracle-gaps/args.mjs +++ b/packages/oracles/oracle-gaps/args.mjs @@ -39,10 +39,18 @@ export const usage = `Usage: oracle-gaps --package --source [--env-prefix ] [--daemon] [--ordinary] [--seed ] [--mutants ] [--out ] [--only ] [--diff-base ] [--test-timeout ] [--sample ] [--contexts ] [--pool threads|forks] [--config ] - [--first-limit ] [--hang-limit ] [--mutant-limit ] + [--hang-limit ] [--first-limit ] [--mutant-limit ] oracle-gaps --stop-daemon --package oracle-gaps --help +Defaults: --env-prefix ORACLE; --test-timeout 65000 (Vitest's per-test limit, +in ms); --hang-limit 60000 (the base of each copy's backstop, in ms, plus 3 s +per second of its file's unmutated time); --contexts the machine's parallelism +minus 2, at most 16; --pool threads (forks when a package has a native +addon); --sample and --only off; --out a directory under the package's state. +Retired: --first-limit and --mutant-limit are checked and print a note, and +change nothing. A copy has no first budget, and a mutant's work budget bounds it. + The paths in --source and --config are relative to --package. Without --config, the run loads the Vitest config that Vitest discovers for the package: in its directory, or else in the nearest parent directory that @@ -223,17 +231,20 @@ function optionsOf(args, cwd) { sample: args.sample ? Number(args.sample) : undefined, // A mutant can make a test hang. Oracles that run in milliseconds do not // need Vitest's default limits, which would make each hang cost seconds. - testTimeout: Number(args[`test-timeout`] ?? 15000), - // The base of each mutant run's time limit, in milliseconds; the limit - // adds three times the file's unmutated duration (see limitFor). - hangLimit: Number(args[`hang-limit`] ?? 10000), - // The base of a mutant run's first, short budget; a copy past it runs - // again alone with the full limit before it can be a timeout. - firstLimit: Number(args[`first-limit`] ?? 2000), - // The floor of a mutant's budget, which the runs that confirm its - // cut-off copies share (check.mjs, `budgetOf`): 30 s by default, room - // for three confirmations at the default hang limit. - mutantLimit: Number(args[`mutant-limit`] ?? 30000), + // Vitest's per-test limit, above the copies' backstop (`hangLimit` plus + // five seconds, so a copy that waits for its backstop is cut by it first). + testTimeout: Number(args[`test-timeout`] ?? 65000), + // The base of each mutant run's backstop time limit, in milliseconds; the + // limit adds three times the file's unmutated duration (see limitFor). A + // backstop reports a gap and never decides a kill (deterministic cuts, + // option A, 2026-11-14). + hangLimit: Number(args[`hang-limit`] ?? 60000), + // The `--first-limit` option is retired (option A): it is validated and + // noted, and changes nothing. + retiredFirstLimit: args[`first-limit`] !== undefined, + // `--mutant-limit` is retired too (deterministic cuts, option A): the work + // budget bounds a mutant, so it is validated and noted, and changes nothing. + retiredMutantLimit: args[`mutant-limit`] !== undefined, // Vitest's worker pool. Threads start much faster than child processes; // a package with a native addon in its tree runs with `forks`. pool: args.pool ?? defaultPool(pkg), diff --git a/packages/oracles/oracle-gaps/capture-setup.mjs b/packages/oracles/oracle-gaps/capture-setup.mjs index 314d984..c3c6e08 100644 --- a/packages/oracles/oracle-gaps/capture-setup.mjs +++ b/packages/oracles/oracle-gaps/capture-setup.mjs @@ -23,11 +23,16 @@ // // In a mutant run, fast-check stops a property at its first failure instead // of shrinking it: a kill needs one failing history, not the smallest one. +import { createHook } from 'node:async_hooks' import { createHash } from 'node:crypto' +import { once } from 'node:events' import fs, { appendFileSync, readFileSync } from 'node:fs' import { createRequire, syncBuiltinESMExports } from 'node:module' +import { connect, createServer } from 'node:net' +import { tmpdir } from 'node:os' import { dirname, isAbsolute, join, resolve as resolvePath } from 'node:path' import { fileURLToPath } from 'node:url' +import { threadId } from 'node:worker_threads' import { afterAll, beforeEach, chai, expect, inject, vi } from 'vitest' import { posix, writeWhole } from './paths.mjs' @@ -50,7 +55,6 @@ const run = const real = (globalThis.__oracleGapsReal ??= { setInterval: globalThis.setInterval, now: Date.now.bind(Date), - use: performance.eventLoopUtilization.bind(performance), keys: Object.keys, getPrototypeOf: Object.getPrototypeOf, getOwnPropertyDescriptor: Object.getOwnPropertyDescriptor, @@ -64,7 +68,6 @@ const real = (globalThis.__oracleGapsReal ??= { }) const realInterval = real.setInterval const realNow = real.now -const realUse = real.use // A wrapper keeps every own property of the function it wraps, such as // `util.promisify.custom`, so `util.promisify(setTimeout)` still resolves @@ -77,48 +80,12 @@ function keepProperties(from, to) { return to } -// The timers that code in this worker has pending, counted where they are -// made, each with when it is next due: Node's resource list does not -// reliably show one entry per timer. -// The wrappers are installed once per worker, before any test file loads, so -// a timer a module makes when it loads is counted too. Vitest's own timers -// are not: it took the real functions before this file ran. Fake timers -// install over the wrappers and give them back. -const pending = (globalThis.__oracleGapsPendingTimers ??= (() => { - const handles = new Map() - const wrap = (make, clear, once, timed) => { - globalThis[make.name] = keepProperties(make, function (callback, ...rest) { - if (typeof callback !== `function`) return make(callback, ...rest) - // An immediate is due at once; a timer or interval after its delay. - const delay = make === globalThis.setImmediate ? 0 : Number(rest[0]) || 0 - const handle = make( - function (...args) { - if (once) handles.delete(handle) - else handles.set(handle, real.now() + delay) - return callback.apply(this, args) - }, - ...rest, - ) - handles.set(handle, real.now() + delay) - // The copy now running hears of a timer or an interval: a test that - // made one can do work that follows the clock, as a polling or a - // deadline loop does (classify.mjs). An immediate waits a turn, not a - // time, as the framework's `withinTurns` does. - if (timed) globalThis.__oracleGapsClock?.() - return handle - }) - globalThis[clear.name] = function (handle) { - handles.delete(handle) - return clear(handle) - } - } - wrap(globalThis.setTimeout, globalThis.clearTimeout, true, true) - wrap(globalThis.setInterval, globalThis.clearInterval, false, true) - wrap(globalThis.setImmediate, globalThis.clearImmediate, true, false) - return handles -})()) -// The other paths to the clock, wrapped once per worker as the timers are, -// each telling the copy now running (`__oracleGapsClock`): a read of +// The paths to the clock, wrapped once per worker, before any test file +// loads, each telling the copy now running (`__oracleGapsClock`): a timer or +// an interval of the global functions (a test that made one can do work that +// follows the clock, as a polling or a deadline loop does, classify.mjs; an +// immediate waits a turn, not a time, as the framework's `withinTurns` does, +// so it is not told); a read of // `Date.now`, `new Date()` or `Date()` with no argument, `performance.now`, // `performance.eventLoopUtilization`, `process.hrtime`, `process.uptime`, // `os.uptime`, and `Temporal.Now`; a timer of `node:timers` or @@ -135,6 +102,8 @@ if (!globalThis.__oracleGapsClocks) { globalThis.__oracleGapsClock?.() return read.apply(this, args) }) + globalThis.setTimeout = told(globalThis.setTimeout) + globalThis.setInterval = told(globalThis.setInterval) const RealDate = Date RealDate.now = told(RealDate.now) const TrackedDate = new Proxy(RealDate, { @@ -459,7 +428,8 @@ const active = state.mutant // copy's budget (`callBudget`, from check.mjs: a hundred times the most calls // any unmutated test of the file made, and at least 100,000), the copy throws // from every helper call and writes a spin row, and the batch reports it as a -// timeout. A count, unlike a clock, does not depend on load. It is also the +// spin, which fails bounded work when its count cannot follow the clock +// (check.mjs `countOnly`). A count, unlike a clock, does not depend on load. It is also the // test's work, which a work bound judges (classify.mjs): each site // evaluation counts once, so a call statement, a call whose value is used, a // comparison, a logical operator, an `if`, and each pass of a loop's @@ -476,12 +446,24 @@ const hit = (id) => { if (inTest && !probeOnly.has(id)) calls += 1 if (spun || (state.callBudget && calls > state.callBudget)) spin() } +// The test a cut ends, the route it registered, and whether its count could +// have followed the clock by then: a spin or no progress is a verdict of a +// built-in law on that route (M, 2026-11-16, "Spin, no progress and the work +// budget are kills"), and a spin only when its count could not (M, 2026-11-16, +// "Count-only kills"). +function cutTest() { + return { + key: currentTask && testKey(currentTask), + route, + ...(clocked || ioDone.value ? { clocked: true } : {}), + } +} function spin() { if (!spun) { spun = true appendFileSync( state.capture, - `${JSON.stringify({ file: thisFile, spin: true })}\n`, + `${JSON.stringify({ file: thisFile, spin: true, ...cutTest() })}\n`, ) } throw new Error( @@ -923,23 +905,23 @@ globalThis.__oracleWorkBound = (ratio) => { // Whether the current test's count could follow the clock (maintainer // decision, 2026-11-12, under P3: a kill must replay on an idle machine): // it made a real timer or interval by any path, or read the clock (the -// wrappers above), or held I/O open (`sawIo`). A read counts when any frame +// wrappers above), or its own I/O completed while it ran (`ioDone`, by state: +// the async_hooks hook below, never a sample). A read counts when any frame // of its stack is the test's or production's: a read whose every frame is -// the test runner's, fast-check's, the oracle framework's (its shrink +// the test runner's (Vitest's, or Vite's module runner, which evaluates this +// file and the test files), fast-check's, the oracle framework's (its shrink // budget), Node's, or this file's does not, while a user's read through a // spy (tinyspy's frame first) still does. let clocked = false // A backslash separates only on win32 (on POSIX it is a name's character). const SEP = process.platform === `win32` ? `[\\\\/]` : `/` const machinery = new RegExp( - `${SEP}node_modules${SEP}(?:vitest|@vitest|fast-check|@fast-check|tinypool|tinyspy|chai|@tanstack${SEP}oracles)${SEP}|${SEP}packages${SEP}oracles${SEP}src${SEP}`, + `${SEP}node_modules${SEP}(?:vitest|@vitest|vite|fast-check|@fast-check|tinypool|tinyspy|chai|@tanstack${SEP}oracles)${SEP}|${SEP}packages${SEP}oracles${SEP}src${SEP}`, ) -function fromTest() { - const limit = Error.stackTraceLimit - Error.stackTraceLimit = 50 - const stack = String(new Error().stack) - Error.stackTraceLimit = limit - return stack +// Whether a stack names the test's or production's code: a frame outside +// Node, this file, and the machinery. +const namesTest = (stack) => + String(stack) .split(`\n`) .slice(1) .some((frame) => { @@ -953,6 +935,12 @@ function fromTest() { !machinery.test(file) ) }) +function fromTest() { + const limit = Error.stackTraceLimit + Error.stackTraceLimit = 50 + const stack = new Error().stack + Error.stackTraceLimit = limit + return namesTest(stack) } globalThis.__oracleGapsClock = () => { if (inTest && !clocked && fromTest()) clocked = true @@ -1042,8 +1030,7 @@ function finishTest(context) { calls, infected: [...infected], operands: [...operands], - ...(sawIo ? { io: true } : {}), - ...(clocked || sawIo ? { clocked: true } : {}), + ...(clocked || ioDone.value ? { clocked: true } : {}), outcomes: [...outcomes], verdicts: [...verdicts.values()], route, @@ -1072,64 +1059,175 @@ function finishTest(context) { const thisFile = globalThis.__vitest_worker__?.filepath -// I/O and idleness (maintainer decision, 2026-11-07). Each test's row says -// whether the test ever had an I/O handle open: anything but the runner's -// own channel, stdio, and timers. A copy given `idleMs` (a file whose -// unmutated tests did no I/O) is idle when no counted timer is pending, no -// more timers are listed than when its test began (which covers timers made -// outside the wrappers, such as `node:timers/promises`), nothing but the -// channel and stdio is open, and its event loop is under 5% busy. After -// `idleMs` of that it writes an idle row, and the batch's watchdog cuts it -// off as a hang. A late sample means the thread was busy. -const quiet = new Set([ - `MessagePort`, - `PipeWrap`, - `TTYWrap`, - `Timeout`, - `Immediate`, -]) -const timersListed = (kinds) => - kinds.filter((kind) => kind === `Timeout` || kind === `Immediate`).length -let sawIo = false +// I/O and idleness (maintainer decision, 2026-11-07; state only, 2026-11-15; +// only the test's I/O, 2026-11-15). A test's I/O is a handle or request that +// the test's or production's code started, never one the runner made for +// itself (its channel, stdio, a write that forwards the test's console +// output); a test that had some active is clocked (its count could follow +// the time). A copy given `idleMs` (every mutant copy, per copy, v21.2 R51) is +// idle, by state alone, when none of its test's timers is due within its +// horizon and none of its test's I/O is active (C, timers by ownership, +// 2026-11-16): a timer the +// runner keeps (Vitest's task-update or test-limit timer) never counts, and +// one the test made counts by any path (the globals, `node:timers`, +// `node:timers/promises`), ref'd or not. A +// busy synchronous or microtask loop blocks this sampler, so it cannot look +// idle; it ends by its call budget or the backstop. After `idleMs` of that +// state it writes an idle row, and the batch's watchdog cuts it off as no +// progress. +// +// Who started each timer, handle and request is recorded once per worker, when Node +// makes it: the test, when the test's or production's code is on the stack +// then (`namesTest`), or when the resource that triggered it is the test's. +// Node triggers a write by the stream it writes to, so the runner's write on +// its own channel is the runner's even when a test's console.log led to it. +// The stack is read only when the sampler meets the resource, or when the +// resource triggers another. Promises are not recorded. +// Each live timer and immediate, with its record. +const liveTimers = (globalThis.__oracleGapsTimers ??= new Map()) +// Whether an I/O handle or request of the running test's ran its callback +// (a read completed, data arrived) since the test began: by state, so a +// short read that no sample sees still counts (C, 2026-11-17). +const ioDone = (globalThis.__oracleGapsIoDone ??= { value: false }) +const startedBy = (globalThis.__oracleGapsStartedBy ??= (() => { + const records = new WeakMap() + const live = new Map() + createHook({ + init(id, type, trigger, resource) { + if (type === `PROMISE`) return + const parent = live.get(trigger) + const record = {} + if (parent !== undefined && owned(parent)) record.test = true + else { + const limit = Error.stackTraceLimit + Error.stackTraceLimit = 50 + Error.captureStackTrace(record) + Error.stackTraceLimit = limit + } + live.set(id, record) + records.set(resource, record) + if (type === `Timeout` || type === `Immediate`) + liveTimers.set(id, [resource, record]) + else record.io = true + }, + before(id) { + const record = live.get(id) + if (record?.io && !ioDone.value && owned(record)) ioDone.value = true + }, + destroy(id) { + live.delete(id) + liveTimers.delete(id) + }, + trackPromises: false, + }).enable() + return records +})()) +function owned(record) { + return (record.test ??= namesTest(record.stack)) +} +// Node lists a handle by its owner (a socket, a child process), and records +// the handle itself. +const testsIo = () => + [...process._getActiveRequests(), ...process._getActiveHandles()].some( + (resource) => { + const record = startedBy.get(resource) ?? startedBy.get(resource?._handle) + return record !== undefined && owned(record) + }, + ) +// Whether a timer of the test's is due within `horizon` ms of `now`, both in +// the timers' own clock: an immediate is due at once, and a timer at its +// start plus its delay (its next run, for an interval). +const testsTimerDue = (now, horizon) => + [...liveTimers.values()].some( + ([timer, record]) => + owned(record) && + (timer._idleStart === undefined || + timer._idleStart + timer._idleTimeout - now <= horizon), + ) +// Test-only hooks for the CLI's own oracle, not for users. With +// ORACLE_GAPS_TEST_SAMPLES (a directory), the sampler writes how many times +// it has judged the running test to `-` there, so a witness +// can hold its own I/O open for a count of samples rather than for a time. +// With ORACLE_GAPS_TEST_RUNNER_WRITE (a directory), the runner itself holds +// one write pending through each test, on a local pipe whose reader never +// reads, as a write that forwards output can stay pending on win32; a copy +// that goes idle while it is pending marks the directory. With +// ORACLE_GAPS_TEST_RUNNER_TIMER (a directory), the runner itself keeps one +// 100 ms interval through each test, made through the global setInterval as +// Vitest's task-update timer is; a copy that goes idle while it runs marks the +// directory. +const samplesAt = process.env.ORACLE_GAPS_TEST_SAMPLES +const runnerWrites = process.env.ORACLE_GAPS_TEST_RUNNER_WRITE +const runnerTimers = process.env.ORACLE_GAPS_TEST_RUNNER_TIMER +let held +let heldTimer if (state.capture) { - let baseTimers + let testing = false let since - let last - let use = realUse() let reported = false + let samples = 0 beforeEach(() => { if (continuing) return - baseTimers = timersListed(process.getActiveResourcesInfo()) + testing = true since = undefined - sawIo = false + ioDone.value = false + samples = 0 }) beforeEach((context) => { if (continuing) return context.onTestFinished(() => { - baseTimers = undefined + testing = false }) }) - realInterval(() => { + if (runnerWrites) + beforeEach(async (context) => { + if (continuing) return + const name = `oracle-gaps-${process.pid}-${threadId}-${(globalThis.__oracleGapsHolds = (globalThis.__oracleGapsHolds ?? 0) + 1)}` + const server = createServer({ pauseOnConnect: true }).listen( + process.platform === `win32` + ? `\\\\.\\pipe\\${name}` + : join(tmpdir(), `${name}.sock`), + ) + await once(server, `listening`) + const accepted = once(server, `connection`) + const client = connect(server.address()) + await once(client, `connect`) + const [peer] = await accepted + client.write(Buffer.alloc(1 << 22)) + held = client + context.onTestFinished(() => { + held = undefined + client.destroy() + peer.destroy() + server.close() + }) + }) + if (runnerTimers) + beforeEach((context) => { + if (continuing) return + const timer = setInterval(() => {}, 100) + heldTimer = timer + context.onTestFinished(() => { + heldTimer = undefined + clearInterval(timer) + }) + }) + realInterval(function () { const now = realNow() - const late = last !== undefined && now - last > 150 - last = now - const used = realUse(use) - use = realUse() - if (baseTimers === undefined || reported) return - const kinds = process.getActiveResourcesInfo() - const io = kinds.some((kind) => !quiet.has(kind)) - sawIo ||= io + if (!testing || reported) return + if (samplesAt) + writeWhole(join(samplesAt, `${process.pid}-${threadId}`), `${++samples}`) + const io = testsIo() // A timer due after the copy's full time limit cannot end its wait in // time, so it does not keep the copy busy: a test waiting on it would be // a timeout at the limit anyway (a collection's five-minute GC timer). - const horizon = now + (state.idleHorizonMs ?? Infinity) - const timerDue = [...pending.values()].some((due) => due <= horizon) + // The timers' clock now is this interval's own due time. if ( - late || io || - used.utilization > 0.05 || - timerDue || - timersListed(kinds) > baseTimers + testsTimerDue( + this._idleStart + this._idleTimeout, + state.idleHorizonMs ?? Infinity, + ) ) { since = undefined return @@ -1137,9 +1235,13 @@ if (state.capture) { since ??= now if (!state.idleMs || now - since < state.idleMs) return reported = true + if (held?.writableLength) + writeWhole(join(runnerWrites, `idle-${process.pid}-${threadId}`), ``) + if (heldTimer) + writeWhole(join(runnerTimers, `idle-${process.pid}-${threadId}`), ``) appendFileSync( state.capture, - `${JSON.stringify({ file: thisFile, idle: true })}\n`, + `${JSON.stringify({ file: thisFile, idle: true, ...cutTest() })}\n`, ) }, 50).unref() } diff --git a/packages/oracles/oracle-gaps/check.mjs b/packages/oracles/oracle-gaps/check.mjs index be801ba..448f2d4 100644 --- a/packages/oracles/oracle-gaps/check.mjs +++ b/packages/oracles/oracle-gaps/check.mjs @@ -63,6 +63,7 @@ import { testNamesOf, } from './stream.mjs' import { analyze, isRunnerCopy } from './analysis.mjs' +import { instrument } from './schemata.mjs' import { Refusal } from './ending.mjs' import { baseOf, @@ -91,14 +92,33 @@ import { const here = dirname(fileURLToPath(import.meta.url)) const sha1 = (text) => createHash(`sha1`).update(text).digest(`hex`) -// How long a copy of a file with no I/O may sit idle before it is cut off as -// a hang (capture-setup.mjs). The longest idle stretch measured in correct +// How long a copy may sit idle, by state, before it is cut off as no progress +// (capture-setup.mjs). The longest idle stretch measured in correct // tests was 51 ms (79,180 test runs, 2026-11-07), so this is ten times that. const IDLE_MS = 500 +// The test a spin or no progress ended, from its cut row (capture-setup.mjs), +// with the route it registered, under its original file's key (`map`): the +// built-in law (bounded work or no progress) judges it on that route (M, +// 2026-11-16, "Spin, no progress and the work budget are kills"). The row +// also says whether the test had read the clock by then, which the check's +// count-only rule reads for a spin (`countOnly`). +const cutVerdicts = (row, map = (key) => key) => [ + { + key: row?.key && map(row.key), + test: row?.key && map(row.key).split(` > `).slice(1).join(` > `), + route: row?.route, + ...(row?.clocked ? { clocked: true } : {}), + }, +] // The least call budget a copy gets (capture-setup.mjs): mutation-helper calls // in one test before it counts as a loop that never ends. A file's budget is // a hundred times the most calls any of its unmutated tests made, if more. const CALL_FLOOR = 100_000 +// The least work (site evaluations) a mutant's copies may do together: the +// larger of this and ten times their owner files' unmutated work (`workBudgetOf`). +// It is the call floor: two copies that each stay inside their own call budget +// can pass it together. +const WORK_FLOOR = CALL_FLOOR const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)) /** The host that a session directory names, if it names one. */ function sessionHostOf(dir) { @@ -810,14 +830,25 @@ export default async (env) => { } return map } + // A context's close, up to the backstop (60 s). A close still pending past it + // is a loud backstop gap (register L8), never a silent one: the caller never + // warms a replacement while the old context is still closing. + async function closeWithin(context, label) { + const closed = await Promise.race([ + closeContext(context).then( + () => true, + () => true, + ), + sleep(60_000).then(() => false), + ]) + if (!closed) + throw new Error( + `oracle-gaps: gap backstop: a context close (${label}) is still pending after 60 s; no replacement is warmed while it closes`, + ) + } async function closeContexts() { await Promise.all( - contexts.map((context) => - Promise.race([ - closeContext(context).catch(() => undefined), - sleep(10_000), - ]), - ), + contexts.map((context) => closeWithin(context, `end of run`)), ) // The closed contexts leave the registry with the array: a context closed for a config change is no longer // reachable through indexOf or contextSpecs (the daemon's edit sequence left them there before). @@ -939,8 +970,11 @@ export default async (env) => { // A run past its call budget (capture-setup.mjs) is cut off once it has // had IDLE_MS to end on its own, as in a batch. const spinRow = () => - existsSync(capture) && - readFileSync(capture, `utf8`).includes(`"spin":true`) + existsSync(capture) + ? readFileSync(capture, `utf8`) + .split(`\n`) + .find((line) => line.includes(`"spin":true`)) + : undefined let spunAt const beat = heartbeat() // A run with no time limit (an unmutated one) only beats. @@ -962,7 +996,8 @@ export default async (env) => { clearTimeout(timer) clearTimeout(poller) const seconds = (Date.now() - started) / 1000 - // A run that spun is a timeout as it stands, even one that then ended. + // A run that spun fails bounded work as it stands, even one that then + // ended. const spun = spinRow() const profiled = { file: baseOf(file), @@ -979,7 +1014,13 @@ export default async (env) => { forceCancel(context) span(`run`, started, { ...profiled, timedOut: true }) rmSync(capture, { force: true }) - return { file, timedOut: true, seconds, ...(spun ? { spin: true } : {}) } + return { + file, + timedOut: true, + seconds, + cause: spun ? `spin` : `backstop`, + ...(spun ? { builtIn: cutVerdicts(JSON.parse(spun)) } : {}), + } } // The result also lists test modules from earlier runs, with their old // results. Only this run's file counts. @@ -1008,12 +1049,20 @@ export default async (env) => { .filter(Boolean)) { const row = JSON.parse(line) // A closing row marks the end of a file for a batch's watchdog. - if (!row.closed && !row.idle) rows.push([row.key, row]) + if (!row.closed && !row.idle && !row.spin) rows.push([row.key, row]) if (row.closed && row.file === file) closing = row } rmSync(capture) } - if (spun) return spunRun(file, seconds, modules, closing) + // A run that spun ended, so its closing row names what it read (F-D). + if (spun) + return spunRun( + file, + seconds, + modules, + closing, + cutVerdicts(JSON.parse(spun)), + ) const tests = modules.flatMap((module) => [...module.children.allTests()].map((testCase) => { const testResult = testCase.result() @@ -1065,11 +1114,12 @@ export default async (env) => { * a timeout. It ended, so its closing row names what it read, which it is * stored under (F-D). */ - function spunRun(file, seconds, modules, closing = {}) { + function spunRun(file, seconds, modules, closing = {}, builtIn) { return { file, timedOut: true, - spin: true, + cause: `spin`, + builtIn, seconds, inputs: importsOf(file, modules, closing), readAt: closing.readAt, @@ -1196,6 +1246,8 @@ export default async (env) => { // at once (capture-setup.mjs). const idle = new Set() const spun = new Set() + // Each cut copy's spin or idle row: the test it ended and its route. + const cutRows = new Map() // When each copy's spin row was first seen. The budget's throw usually // ends the loop and the copy finishes on its own; a copy still running // IDLE_MS after it spun swallowed the throw and is cut off. @@ -1208,11 +1260,16 @@ export default async (env) => { if (line.includes(`"started":true`)) at = Math.max(at, copies.indexOf(JSON.parse(line).file)) if (line.includes(`"closed":true`)) closed.add(JSON.parse(line).file) - if (line.includes(`"idle":true`)) idle.add(JSON.parse(line).file) + if (line.includes(`"idle":true`)) { + const row = JSON.parse(line) + idle.add(row.file) + cutRows.set(row.file, row) + } if (line.includes(`"spin":true`)) { - const file = JSON.parse(line).file - spun.add(file) - if (!spunAt.has(file)) spunAt.set(file, Date.now()) + const row = JSON.parse(line) + spun.add(row.file) + cutRows.set(row.file, row) + if (!spunAt.has(row.file)) spunAt.set(row.file, Date.now()) } } if (closed.has(copies[at])) at += 1 @@ -1221,6 +1278,10 @@ export default async (env) => { let current = 0 let copyStarted = Date.now() let timer + // The cut that ended the running copy, if the watchdog cut it off, and how + // long the copy had run then. + let cutCause + let cutMs const beat = heartbeat() // Each poll counts itself, after it has read the capture (the hold in // capture-setup.mjs waits for two of these). @@ -1239,12 +1300,18 @@ export default async (env) => { items[current].file, copyStarted, ) - if ( - idle.has(copies[current]) || - Date.now() - (spunAt.get(copies[current]) ?? Infinity) > IDLE_MS || - Date.now() - copyStarted > items[current].limitMs - ) + // The cut that ends the copy, in this order: a spin past the call + // budget (a count), no progress (a state), then the backstop (a clock). + const copy = copies[current] + if (Date.now() - (spunAt.get(copy) ?? Infinity) > IDLE_MS) + cutCause = `spin` + else if (idle.has(copy)) cutCause = `no progress` + else if (Date.now() - copyStarted > items[current].limitMs) + cutCause = `backstop` + if (cutCause !== undefined) { + cutMs = Date.now() - copyStarted return resolve(undefined) + } timer = setTimeout(poll, 50) } timer = setTimeout(poll, 50) @@ -1277,7 +1344,10 @@ export default async (env) => { ) if (copy && row.skipped) skipped.add(copy) if (copy && row.closed) closings.set(copy, row) - if (copy && row.spin) spun.add(copy) + if (copy && row.spin) { + spun.add(copy) + cutRows.set(copy, row) + } if (copy && !row.closed && !row.idle && !row.spin && !row.started) rows.get(copy).push(row) } @@ -1305,10 +1375,20 @@ export default async (env) => { if (!closed.has(copies[i])) return undefined if (skipped.has(copies[i])) return skippedUnit(item) // A copy that ran past its call budget, even one whose throw - // ended the loop and let it finish, is a timeout as it stands (a - // count, not a clock: no confirming run). + // ended the loop and let it finish, fails bounded work as it + // stands, when its count cannot follow the clock (a count, not a + // clock: no confirming run). if (spun.has(copies[i])) - return spunRun(item.file, seconds, [], closings.get(copies[i])) + return spunRun( + item.file, + seconds, + [], + closings.get(copies[i]), + cutVerdicts( + cutRows.get(copies[i]), + keyFor(copies[i], item.file), + ), + ) const map = keyFor(copies[i], item.file) const own = rows .get(copies[i]) @@ -1338,8 +1418,16 @@ export default async (env) => { } }), timedOutAt: current, - idle: idle.has(copies[current]), - spin: spun.has(copies[current]), + cause: cutCause, + ...(cutCause === `backstop` + ? {} + : { + builtIn: cutVerdicts( + cutRows.get(copies[current]), + keyFor(copies[current], items[current].file), + ), + }), + cutMs, } } // Errors outside any test cannot be attributed to one copy, so the @@ -1353,7 +1441,13 @@ export default async (env) => { const copy = copies[i] if (skipped.has(copy)) return skippedUnit(item) if (spun.has(copy)) - return spunRun(item.file, seconds, [], closings.get(copy)) + return spunRun( + item.file, + seconds, + [], + closings.get(copy), + cutVerdicts(cutRows.get(copy), keyFor(copy, item.file)), + ) const map = keyFor(copy, item.file) const modules = result.testModules.filter((m) => m.moduleId === copy) return { @@ -1528,9 +1622,13 @@ export default async (env) => { stopIfGone() const authored = options.mutants ? loadAuthored(options.mutants) : [] seedVariable = `${options.envPrefix}_SEED` - if (options.hangLimit > options.mutantLimit) + if (options.retiredFirstLimit) log( - `oracle-gaps: note --hang-limit ${options.hangLimit} ms is above --mutant-limit ${options.mutantLimit} ms; a mutant's confirmations together get at most the larger of --mutant-limit and ten times its owner files' unmutated time`, + `oracle-gaps: note --first-limit is retired: a copy has no first budget, so it changes nothing (deterministic cuts, option A)`, + ) + if (options.retiredMutantLimit) + log( + `oracle-gaps: note --mutant-limit is retired: a mutant's work budget bounds it, so it changes nothing (deterministic cuts, option A)`, ) pulse = { log, @@ -1655,7 +1753,6 @@ export default async (env) => { settings, // A run cut off under one limit may end under a longer one. options.hangLimit, - options.mutantLimit, ]), ) // How the evidence was produced: units reused and computed, runs cut off @@ -1663,8 +1760,8 @@ export default async (env) => { // after a cut-off batch, and mutants that changed no value and so needed // no run. Under the retention test hook only (README "Test hooks"), also // the computed units that ended without a cut (`ran`), for the e2e - // oracle: `cut` counts cut events (a cut copy confirmed alone is cut in - // its batch and again if its confirmation is cut), so `computed - cut` + // oracle: `cut` counts copies cut off, while a copy that spun past its + // call budget and then ended is timed out with no cut, so `computed - cut` // is no count of runs. A normal run's report has no `ran`. const evidence = { reused: 0, @@ -1675,7 +1772,7 @@ export default async (env) => { spin: 0, rerun: 0, unchanged: 0, - // Confirmations that a mutant's budget cut short or left no time. + // Copies past their mutant's work budget (each a timeout). budget: 0, } // Each file's imports in its unmutated run. A run that timed out reports @@ -1771,8 +1868,7 @@ export default async (env) => { ) return { ...batch, inputs } } - // A unit that depends on more than its inputs (a confirmation that its - // mutant's budget cut short) is not kept (`keep`). + // A unit that depends on more than its inputs is not kept (`keep`). async function unit(context, file, spec, keep, runOptions = {}) { const index = indexOf.get(context) const id = idOf(file, spec, runOptions.select, runOptions.ignore) @@ -1816,13 +1912,30 @@ export default async (env) => { return result } // Units for many requests at once. Each is looked up first; the rest run - // as one batch (runBatch). A batch cut off by a hang queues that item's - // confirming run, which any free slot takes, and reruns the rest in a - // fresh context; a batch with + // as one batch (runBatch). A batch cut off (a copy's backstop, idleness, + // or a spin) settles that copy as cut, with its cause and the built-in + // verdict of a cut by count or state, and reruns the rest in a fresh + // context; a batch with // errors outside any test runs its items alone. `needed` says, from the // units found so far, whether a request must still run; a request that // need not run, gets no unit; a copy that skipped its tests gets a // skipped marker, which its caller runs again (runOwners). + // A profile row for one copy (ORACLE_GAPS_PROFILE): its mutant, its file's + // measured time, its limit, the cause that ended it or `completed`, and its + // duration. + const copyRow = (request, { cause, durationMs }) => + appendFileSync( + profileFile, + `${JSON.stringify({ + kind: `copy`, + mutant: request.label, + file: baseOf(request.file), + cause, + fileMs: Math.round((fileSeconds.get(request.file) ?? 0) * 1000), + limitMs: request.limitMs, + durationMs, + })}\n`, + ) async function unitsBatch(slot, requests, needed = () => true) { const index = indexOf.get(slot.context) const got = new Array(requests.length) @@ -1860,33 +1973,47 @@ export default async (env) => { // A skipped copy keeps its marker, so the caller can run it again. got[rest[k].i] = u if (u.skipped) return - if (u.spin) evidence.spin += 1 + if (u.cause === `spin`) evidence.spin += 1 remember(rest[k].id, u, batch.inputs[k]) evidence.computed += 1 // A copy that spun past its call budget is timed out, as `unit` // counts it: it did not end without a cut. if (hook && !u.timedOut) evidence.ran += 1 }) + // Per-copy rows for ORACLE_GAPS_PROFILE: each copy's cut or completion, + // with its file's measured time and its limit, to see what a cut was. + if (profileFile) + batch.units.forEach((u, k) => { + if (u === undefined || u.skipped) return + copyRow(rest[k], { + cause: u.cause ?? `completed`, + durationMs: Math.round(u.seconds * 1000), + }) + }) if (batch.timedOutAt === undefined) break - // A copy past its first budget may only be slow: a loaded machine - // slows a run that ends. It runs again alone, in a fresh context, - // with the full limit, and only a run cut off there too is a timeout. + // A copy cut off by its backstop, for idleness, or for spinning is a + // cut as it stands: a count or a state decides its built-in verdict, and + // no clock decides a kill (deterministic cuts). const hung = rest[batch.timedOutAt] - const full = limitFor(hung.file) + if (profileFile) + copyRow(hung, { cause: batch.cause, durationMs: batch.cutMs }) evidence.cut += 1 - if (batch.idle) evidence.idle += 1 - if (batch.spin) evidence.spin += 1 + if (batch.cause === `no progress`) evidence.idle += 1 + if (batch.cause === `spin`) evidence.spin += 1 await replace(slot) - // A copy that already had the full limit, or that sat idle, is - // final; any other comes back marked `cut`, for its mutant's own - // confirmation (`runOwners`). - if (hung.limitMs >= full || batch.idle || batch.spin) - settle( - hung, - { file: hung.file, timedOut: true, seconds: full / 1000 }, - batch.inputs[batch.timedOutAt], - ) - else got[hung.i] = { file: hung.file, cut: true } + // A count or state cut (a spin, or no progress) is a built-in kill for its + // copy alone; only a wall-clock cut ends the mutant's copies (cutByTime). + settle( + hung, + { + file: hung.file, + timedOut: true, + seconds: hung.limitMs / 1000, + cause: batch.cause, + ...(batch.builtIn ? { builtIn: batch.builtIn } : {}), + }, + batch.inputs[batch.timedOutAt], + ) rest = [ ...rest.slice(0, batch.timedOutAt).filter((_, k) => !batch.units[k]), ...rest.slice(batch.timedOutAt + 1), @@ -1907,58 +2034,10 @@ export default async (env) => { limitMs: r.limitMs, callBudget: r.callBudget, }) - if (u.spin) evidence.spin += 1 + if (u.cause === `spin`) evidence.spin += 1 if (u.timedOut) await replace(slot) return u } - // A hung copy's confirming run: alone, in a clean context, with the full - // limit, or with what is left of its mutant's budget when that is less - // (`left`, in milliseconds). Only a run cut off there too is a timeout. - // An idle copy in a file with no I/O is final at once (see unitsBatch): - // it waits on nothing that can come. The unit is marked `confirmed`, so - // its seconds count against the budget whenever it is used, reused ones - // too. One that the budget cut short, or left no time, depends on the - // mutant's other units, so it is not kept. - async function confirmAlone(slot, request, left) { - // The confirmation is kept under the request's own identity, so a - // later check reuses it in place of the request's first run. - const hung = { - ...request, - id: idOf(request.file, request.spec, request.select, request.ignore), - } - const full = limitFor(hung.file) - const limit = Math.min(full, left) - const confirmed = (u) => ({ ...u, confirmed: true }) - if (limit <= 0) { - evidence.budget += 1 - return confirmed({ file: hung.file, timedOut: true, seconds: 0 }) - } - const alone = await batchIn(slot, [{ ...hung, limitMs: limit }]) - if (alone.unattributed) - return confirmed( - await runAlone(slot, { ...hung, limitMs: limit }, limit >= full), - ) - const [inputs] = alone.inputs - if (alone.timedOutAt === 0) { - evidence.cut += 1 - if (alone.idle) evidence.idle += 1 - if (limit < full) evidence.budget += 1 - const u = confirmed({ - file: hung.file, - timedOut: true, - seconds: limit / 1000, - }) - if (limit >= full) remember(hung.id, u, inputs) - await replace(slot) - return u - } - const u = confirmed(alone.units[0]) - remember(hung.id, u, inputs) - evidence.computed += 1 - if (hook && !u.timedOut) evidence.ran += 1 - return u - } - const vitest = contexts[0] const fileSpecs = new Map(contextSpecs.get(vitest)) const relative = (file) => relativeWithin(pkg, file) ?? posix(file) @@ -1976,49 +2055,46 @@ export default async (env) => { // detect anything, and it may read source text that instrumentation // changes. Each file's unmutated duration sets a mutant run's time limit. const fileSeconds = new Map() - // A mutant can slow a run that still ends, for example by lengthening a - // retry backoff, and a loaded machine slows every run. A mutant run - // first gets a short budget (`--first-limit`); a copy cut off there runs - // again alone, with the full limit (`--hang-limit`, 10 s by default: - // a 5 s base once turned a slow kill into a timeout), and only a run - // cut off there too is a timeout (see unitsBatch). + // A copy's time limit is a backstop: the full limit, which reports a gap and + // never decides a kill (deterministic cuts, option A, 2026-11-14). There is + // no first budget, so a copy runs once, to its end. const limitFor = (file) => options.hangLimit + 3000 * (fileSeconds.get(file) ?? 10) - const firstLimitFor = (file) => - Math.min( - limitFor(file), - options.firstLimit + 3000 * (fileSeconds.get(file) ?? 10), - ) - // Each mutant's budget (maintainer disposition G1, 2026-11-11), for the - // runs that confirm its cut-off copies: the larger of `--mutant-limit` - // (30 s by default) and ten times its owner files' unmutated time. Its - // confirmations run in its owner files' order, and each gets at most the - // smaller of its full limit and what the budget has left after the - // seconds of the mutant's earlier confirmed units, reused ones included - // (`runOwners`), so the outcome is a function of the units, whatever ran - // first or was reused. With the default limits the budget cuts a - // confirmation below its full limit (10 s plus three times its file's - // time t) only when the mutant's confirmations together pass it: for - // one owner file that is never, since 10t >= 10 + 3t once t >= 1.43 s - // and the 30 s floor covers a smaller t; it binds when `--hang-limit` is - // raised past the budget, or when several owner files need long - // confirmations. It does not bound a mutant's first runs (each has its - // first budget) or the context replacements after cuts (each up to a - // 10 s close and a warm start). - const budgetOf = (files) => + // A mutant's work budget (maintainer disposition G1, option A): the larger of + // WORK_FLOOR and ten times its owner files' unmutated work, in site + // evaluations, each file counted by its most-working unmutated test. + const workBudgetOf = (files) => Math.max( - options.mutantLimit, - 10_000 * - files.reduce((sum, file) => sum + (fileSeconds.get(file) ?? 0), 0), + WORK_FLOOR, + 10 * files.reduce((sum, file) => sum + (mostCalls.get(file) ?? 0), 0), ) + // A copy cut by a clock or by its call count: its own run's timeout, or + // Vitest's own limit ("Test timed out"), which the copy outlived. Such a + // copy is never charged, and it ends the mutant's copies (the outcome rule, + // register 2026-11-14, `Deterministic cuts, outcome rule`). + const cutByTime = (unit) => + unit.cause === `backstop` || + (unit.tests ?? []).some( + (test) => + test.state === `failed` && /Test timed out/.test(test.message), + ) + // A copy's work: the calls of its deterministic, unclocked tests + // (`baseline.work`, classify.mjs), so a row that a clock could move counts + // nothing. + const workOf = (unit) => + cutByTime(unit) + ? 0 + : (unit.rows ?? []).reduce( + (sum, [key, row]) => + sum + + (baseline.work.has(key) && !row.clocked ? (row.calls ?? 0) : 0), + 0, + ) // A run cut off is still running, so its context is replaced; closing // it stops the worker. const replace = async (slot) => { const started = Date.now() - await Promise.race([ - closeContext(slot.context).catch(() => undefined), - sleep(10_000), - ]) + await closeWithin(slot.context, `slot ${slot.index}`) const replaced = slot.context // The daemon's test-only hold before the replacement warms: a slow // config load mid-phase, released by the test, never by time. @@ -2267,15 +2343,12 @@ export default async (env) => { valueChanges.get(file).add(number) } const unmutatedUnits = new Map(first.units.map((u) => [u.file, u])) - // Files whose unmutated tests had an I/O handle open (a socket, a child - // process, file I/O): a test there may wait on something outside the - // worker, so no copy of it is cut for idleness (capture-setup.mjs). - const ioFiles = new Set() - for (const rows of [first.rows, second.rows]) - for (const [key, row] of rows) if (row.io) ioFiles.add(fileOf(key)) // Each file's call budget: a hundred times the most mutation-helper calls - // any of its unmutated tests made, and at least CALL_FLOOR. A copy past it - // is spinning (capture-setup.mjs) and is a timeout as it stands. + // any of its unmutated tests made, in either run, and at least CALL_FLOOR + // (G6). Every test counts, a clocked one too: a budget that left one out + // would cut its copies at the floor. A copy past it is spinning + // (capture-setup.mjs), which fails bounded work only by a count that + // cannot follow the clock (`countOnly`). const mostCalls = new Map() for (const rows of [first.rows, second.rows]) for (const [key, row] of rows) @@ -2286,6 +2359,17 @@ export default async (env) => { ) const callBudgetFor = (file) => Math.max(CALL_FLOOR, 100 * (mostCalls.get(file) ?? 0)) + // Count-only kills (M, 2026-11-16; v21.1 R19a): a spin kills only when the + // count that decided it could not follow the clock at all, that is, when + // its test's unmutated runs agreed, read no clock and made equal counts + // (`baseline.work`), and the copy had read none when it spun. A test that + // reads the clock anywhere in its run, after its work too, has no such + // count. Otherwise the spin names no test, and the mutant is a gap. + const countOnly = (u) => + u?.cause === `spin` && + !u.builtIn?.every((v) => !v.clocked && baseline.work.has(v.key)) + ? { ...u, builtIn: undefined } + : u const changesNothing = (mutant, targets) => targets.oracleTargets.every( (file) => @@ -2337,12 +2421,16 @@ export default async (env) => { // hang), so a result never depends on how its evidence was produced. The // ordinary tier, when asked for, runs the same way afterwards. const GROUP = 4 - // A copy that a batch skipped, or cut off for its confirmation, has no + // A copy that a batch skipped, or cut off by a count or a state, has no // verdicts of its own yet. const kills = (u) => !u.skipped && - !u.cut && - outcomeOf([u], baseline.nondeterministic).outcome === `killed` + outcomeOf([countOnly(u)], baseline.nondeterministic).outcome === `killed` + // A mutant's copies run in their owner files' order and are summed in that + // order (the work budget, below). The order is the report's, so it does not + // depend on the timing of the copies. A later owner's kill does not count + // once the budget is passed; that is deterministic because the work is a + // count, and it is the rule the design note (option A) states. const runOwners = async (slot, states, filesOf, keysOf) => { const active = states.filter((st) => !st.done) const requests = [] @@ -2359,9 +2447,9 @@ export default async (env) => { spec: `mutant:${st.mutant.number}`, mutant: st.mutant.number, select: await selectionFor(slot, file, keysOf(st)), - limitMs: firstLimitFor(file), - // Idle detection only where no unmutated test of the file did I/O. - idleMs: ioFiles.has(file) ? null : IDLE_MS, + limitMs: limitFor(file), + // Idle by state, per copy: the test's own I/O keeps it busy. + idleMs: IDLE_MS, // A loop that spins on instrumented code is cut by count. callBudget: callBudgetFor(file), // Timers due after the full limit do not keep a copy busy. @@ -2385,43 +2473,62 @@ export default async (env) => { (request, found) => !killedBefore(request, found), ) for (const st of active) { - // The mutant's budget, spent in its files' order by the seconds of - // each confirmed unit, reused or run now, so its outcome is a - // function of its units, whatever ran first or was reused. - let left = budgetOf(filesOf(st)) + const files = filesOf(st) + // The mutant's copies in their files' order, judged by count: each copy's + // work is added, and once the sum passes the work budget, that copy's + // own verdicts do not count and no later copy runs: that copy fails + // bounded work instead (below). + const budget = workBudgetOf(files) + let work = 0 for (const [i, request] of requests.entries()) { if (request.st !== st) continue - // A copy its batch skipped (an earlier copy killed the mutant) - // runs again through unitsBatch, so it is confirmed like any copy - // while no kill is in hand. + // A copy its batch skipped (an earlier copy killed the mutant) runs + // again through unitsBatch, while no kill is in hand. if (got[i]?.skipped) { if (st.units.some(kills)) break got[i] = (await unitsBatch(slot, [request], () => true))[0] } - // A kill earlier in the mutant's files leaves a file without a - // unit. Any other missing unit stops the run. + // A kill earlier in the mutant's files leaves a file without a unit. + // Any other missing unit stops the run. if (got[i] === undefined || got[i].skipped) { if (st.units.some(kills)) break throw new Error( `${st.mutant.id}: ${relative(request.file)} has no unit and no earlier kill`, ) } - // A cut copy is confirmed only while no later owner's unit already - // killed the mutant: a kill decides, so its confirmation would - // spend time for nothing. - if ( - got[i].cut && - requests.some( - (other, j) => - other.st === st && - j > i && - got[j] !== undefined && - kills(got[j]), - ) - ) - continue - if (got[i].cut) got[i] = await confirmAlone(slot, request, left) - if (got[i].confirmed) left -= got[i].seconds * 1000 + // A spin judges only by a count that cannot follow the clock. + got[i] = countOnly(got[i]) + // A copy cut by the backstop is a timeout, uncharged, and no + // later copy runs (the outcome rule). + if (cutByTime(got[i])) { + st.units.push(got[i]) + break + } + work += workOf(got[i]) + // The copy whose work passes the budget fails bounded work on the + // routes of its tests that did counted work; its own verdicts do not + // count, and no later copy runs. + if (work > budget) { + evidence.budget += 1 + st.units.push({ + file: request.file, + timedOut: true, + seconds: 0, + cause: `work budget`, + builtIn: (got[i].rows ?? []) + .filter( + ([key, row]) => + baseline.work.has(key) && + !row.clocked && + (row.calls ?? 0) > 0, + ) + .map(([key, row]) => ({ + test: key.split(` > `).slice(1).join(` > `), + route: row.route, + })), + }) + break + } st.units.push(got[i]) if (kills(got[i])) break } @@ -2523,11 +2630,15 @@ export default async (env) => { endPhase(`operator mutants`, { mutants: results.length }) // 6. Authored mutants, spread across the contexts. Each serves a patched - // copy of its source only to the context that runs it. The copy is not - // instrumented, so an authored mutant runs against every owner of the - // function its edit lands in, whole files, still stopping at its first - // kill. + // copy of its source only to the context that runs it. An authored mutant + // runs against every owner of the function its edit lands in, whole + // files, still stopping at its first kill. The patched copy is + // instrumented with no site active, under site numbers past every + // operator mutant's, only so that it counts calls: a copy that spins past + // its call budget is cut by count, as an operator copy is (v21.2 R50, G6; + // issue #167), and its probe sites count nothing (R53). const authoredJobs = [] + let authoredSites = mutants.length for (const mutant of authored) { const id = `authored:${mutant.id}` if (options.only && !options.only.has(id)) continue @@ -2558,10 +2669,22 @@ export default async (env) => { result.outcome = `unowned` continue } + const counted = instrument( + ts, + path, + patched.text, + mutant.source, + authoredSites, + ) + authoredSites += counted.mutants.length + probeSites.push( + ...counted.mutants.filter((m) => m.probeGuard).map((m) => m.number), + ) authoredJobs.push({ id, path, patched, + counted: counted.code, result, owners, targets: [...owners].filter((file) => fileSpecs.has(file)), @@ -2584,7 +2707,7 @@ export default async (env) => { item = authoredItems.shift() ) { const { job, file } = item - serve[slot.index] = new Map([[job.path, job.patched.text]]) + serve[slot.index] = new Map([[job.path, job.counted]]) invalidateChanged([job.path], slot.context) const [got] = await unitsBatch(slot, [ { @@ -2594,6 +2717,11 @@ export default async (env) => { mutant: -1, select: null, limitMs: limitFor(file), + // An authored copy makes no progress, and spins, as an operator + // copy does (v21.2 R50): it is idle by state and cut by count. + idleMs: IDLE_MS, + callBudget: callBudgetFor(file), + idleHorizonMs: limitFor(file), stopOnKill: true, ignore: ignoredIn(file), }, @@ -2601,7 +2729,8 @@ export default async (env) => { // In the order the contexts finish: the classification lists // the tests and laws in report order (classify.mjs // `inReportOrder`), so they are listed alike in every run (law 9). - job.units.push(got) + // A spin judges only by a count that cannot follow the clock. + job.units.push(countOnly(got)) serve[slot.index] = undefined invalidateChanged([job.path], slot.context) } diff --git a/packages/oracles/oracle-gaps/classify.mjs b/packages/oracles/oracle-gaps/classify.mjs index 72da617..35175d0 100644 --- a/packages/oracles/oracle-gaps/classify.mjs +++ b/packages/oracles/oracle-gaps/classify.mjs @@ -33,6 +33,17 @@ function routeOf(registered) { return `unknown` } +// The built-in law each cut by count or by state fails (M, 2026-11-16, "Spin, +// no progress and the work budget are kills", restoring v21.1 K1 and P3): a +// spin past the call budget and a copy past the mutant's work budget fail +// bounded work, and a copy idle by state fails no progress. Only the backstop, +// a wall-clock cut, is a gap. +const BUILT_IN = { + spin: `bounded work`, + 'work budget': `bounded work`, + 'no progress': `no progress`, +} + /** * A mutant's outcome from its units. A test whose observations varied * between the two unmutated runs is nondeterministic: its failures judge @@ -47,16 +58,43 @@ export function outcomeOf( assertions = false, pastBound = () => false, ) { - // A run that timed out recorded nothing. A kill in another run still - // decides; failing that, the mutant is `timeout`. - const timeout = units.some((unit) => unit.timedOut) - ? { outcome: `timeout`, detail: `run time limit` } + // A run that the backstop cut recorded nothing, and a test that Vitest's own + // limit ended (`Test timed out`) is the same kind of backstop (deterministic + // cuts, 2026-11-14): a gap that never keeps a kill from deciding. A kill in + // another run still decides; failing that, the mutant is `timeout`. + const timedOutTest = (test) => + test.state === `failed` && /Test timed out/.test(test.message) + // The timeout's detail is the cause of the first gap cut in report order: + // the backstop, or a spin whose count could follow the clock (`spin`). + const cut = [...units] + .sort((a, b) => byPath(a.file, b.file)) + .find( + (unit) => + (unit.timedOut && !(BUILT_IN[unit.cause] && unit.builtIn?.length)) || + (unit.tests ?? []).some(timedOutTest), + ) + const timeout = cut + ? { outcome: `timeout`, detail: cut.timedOut ? cut.cause : `backstop` } : undefined const failures = [] // By file path (report order, `inReportOrder`), in whatever order the // runner gathered the units. for (const unit of [...units].sort((a, b) => byPath(a.file, b.file))) { - if (unit.timedOut) continue + // A copy cut by its call count or by no progress, and the copy whose + // work passed the mutant's work budget, record nothing of their own: each + // test the cut names fails the built-in law on its route. A spin whose + // count could follow the clock names none, and is a timeout (a gap). + if (unit.timedOut) { + for (const v of BUILT_IN[unit.cause] ? (unit.builtIn ?? []) : []) { + const route = routeOf(v.route) + failures.push({ + test: v.test ?? `(unnamed test)`, + route, + verdicts: [{ law: BUILT_IN[unit.cause], route }], + }) + } + continue + } const rows = new Map(unit.rows) for (const message of unit.setupErrors) failures.push({ setup: true, message }) @@ -87,7 +125,6 @@ export function outcomeOf( // fast-check's wrapper text ("Property failed …") is not an // assertion; the wrapped error, which its message quotes, decides. /AssertionError|expected .+ to /.test(test.message), - timeout: /Test timed out/.test(test.message), // fast-check quotes the error its predicate threw after its own // header; that error is the cause the report shows. message: ( @@ -153,9 +190,7 @@ export function outcomeOf( laws: lawsOf(unregistered), } const tests = failures.filter((f) => f.test) - if (tests.length > 0 && tests.every((f) => f.timeout)) - return { outcome: `timeout`, by: tests.map((f) => f.test) } - const crashed = tests.filter((f) => f.route !== `framework` && !f.timeout) + const crashed = tests.filter((f) => f.route !== `framework`) if (crashed.length > 0) return { outcome: `crashed`, @@ -278,7 +313,9 @@ export function targetsOf(mutant, baseline, owners, oracleFiles) { return { oracleKeys, testKeys, - oracleTargets: [...new Set(oracleKeys.map(fileOf))], + // Report order (byPath), so a mutant's owners are summed in one order + // however its copies ran. + oracleTargets: [...new Set(oracleKeys.map(fileOf))].sort(byPath), testTargets: [...new Set(testKeys.map(fileOf))], } } @@ -522,8 +559,9 @@ export function judgeAuthored(units, baseline) { ), baseline, ) - // An authored mutant is not instrumented, so only its changed observations - // are measured: the rest is `not-observed`, the patch changed the code. + // An authored mutant's copy counts calls only (check.mjs), so only its + // changed observations are measured: the rest is `not-observed`, the patch + // changed the code. result.class = result.changedTests.length > 0 ? `not-judged` : `not-observed` return result } diff --git a/packages/oracles/oracle-gaps/gap-report-classify-oracle.property.test.mjs b/packages/oracles/oracle-gaps/gap-report-classify-oracle.property.test.mjs index 40343d4..a1f5fb9 100644 --- a/packages/oracles/oracle-gaps/gap-report-classify-oracle.property.test.mjs +++ b/packages/oracles/oracle-gaps/gap-report-classify-oracle.property.test.mjs @@ -36,8 +36,8 @@ * so a kill names a non-empty part of them. An operator mutant's owner * files run one after another in report order (law 7), and the runner * stops at the first that kills, so its kill names laws of that file - * alone. Failing that, a run that - * timed out makes it `timeout`. Failing that, reproducible verdicts under + * alone. Failing that, a run that timed out, or a test that Vitest's limit + * ended, makes it `timeout`. Failing that, reproducible verdicts under * reach agreement alone make it an `agreement-only-kill`, which names * them all. Failing that, a verdict on the random campaign * makes it a `random-only-kill`, and one on an unknown or unregistered @@ -62,7 +62,10 @@ * and the design decided with it that day: a speed shortcut is owed a * cost law, not an exit, so a regression that deletes it must be * catchable, by a deterministic count and never by wall-clock time. - * 4. Failing that, if every failing test timed out, it is `timeout`; a + * 4. Failing that, a test that Vitest's own limit ended (its message is + * `Test timed out`) is a backstop, with a run that timed out: the mutant + * is `timeout`, a gap that never keeps a kill from deciding (deterministic + * cuts, option A, 2026-11-14). Otherwise a * failing test outside the framework's own checks makes it `crashed`; a * failing framework check alone (the `calibration` route, which covers * calibrations, probe calibrations, and the determinism test) makes it a @@ -296,7 +299,8 @@ const SLOTS = MUTANTS + AUTHORED // operands a comparison there saw, and what it did under it: nothing // different, a changed asserted or observed value, a law verdict (on the // route then active, or with none active), an assertion failure, a crash, a -// timeout message, a timed-out run, a setup error, or an unhandled error. +// timeout message, a timed-out run, a run cut by its call count or by no +// progress while it ran, a setup error, or an unhandled error. // Each test also has a name and enclosing suites, whose names hold regular // expression and shell syntax, and each file a baseline: it passes, a test // of it fails with no mutant active, or it does not load. A law verdict @@ -445,6 +449,10 @@ const effect = fc.record({ `runTimeout`, `setupError`, `unhandled`, + // A copy cut by its call count or by no progress, where this test ran: + // drawn, under M "Randomness wins" (2026-11-16), appended last. + `runSpin`, + `runIdle`, ), law: fc.constantFrom(`L`, `M`, `reach agreement`), // The route active when a verdict fired, when it differs from the test's. @@ -712,6 +720,32 @@ function boundBelow(before, after) { return (hundredths / 100).toFixed(2) } +// A copy cut by its call count (`runSpin`) or by no progress (`runIdle`) +// fails a built-in law on the route of the test that was running, and records +// nothing else (M, 2026-11-16, "Spin, no progress and the work budget are +// kills", restoring v21.1 K1 and P3). +const CUT_LAW = { runSpin: `bounded work`, runIdle: `no progress` } +// The kinds that end a file's run, so that it ran none of its tests. +const ENDS_RUN = [`setupError`, `runTimeout`, ...Object.keys(CUT_LAW)] +// What cut file `fi`'s run under the mutant in `slot`: the backstop (a run +// that timed out, or Vitest's own limit in a file that loaded and that no cut +// by count or state ended first), the built-in law of its first test cut by +// count or by state, or a spin whose count could follow the clock (a test +// whose row is clocked), which kills nothing and is a gap (M, 2026-11-16, +// "Count-only kills"). +function cutOf(h, fi, slot) { + const kinds = h.files[fi].tests.map((t) => t.under[slot].kind) + if (kinds.includes(`runTimeout`)) return { gap: `backstop` } + const first = h.files[fi].tests.find((t) => CUT_LAW[t.under[slot].kind]) + if (first) + return first.under[slot].kind === `runSpin` && first.clocked !== `none` + ? { gap: `spin` } + : { law: CUT_LAW[first.under[slot].kind], t: first } + if (kinds.includes(`timeoutMessage`) && !kinds.includes(`setupError`)) + return { gap: `backstop` } + return {} +} + // Laws 2 to 4 over the tests that judge a mutant, read from each test's // effect in the mutant's slot: the failure's outcome, if a failure decides, // and the tests that ran (a file that failed to load, or whose run timed @@ -720,14 +754,17 @@ function boundBelow(before, after) { // at a kill reads no work. function failureOf(h, slot, tests, work = false) { const under = ({ t }) => t.under[slot].kind - const timedOut = tests.some((x) => under(x) === `runTimeout`) + // The first gap cut in report order names a timeout's cause. + const files = [...new Set(tests.map((x) => x.fi))] + const timedOut = files.map((fi) => cutOf(h, fi, slot).gap).find(Boolean) const setup = tests.some((x) => under(x) === `setupError`) const ran = tests.filter( (x) => - !h.files[x.fi].tests.some((t) => - [`setupError`, `runTimeout`].includes(t.under[slot].kind), - ), + !h.files[x.fi].tests.some((t) => ENDS_RUN.includes(t.under[slot].kind)), ) + // The test whose copy a cut by count or by state ended, one per file (the + // first), in a file whose run no backstop ended, when it kills. + const cut = tests.filter((x) => cutOf(h, x.fi, slot).t === x.t) const failing = ran.filter( (x) => !x.t.flaky && @@ -736,15 +773,18 @@ function failureOf(h, slot, tests, work = false) { // A test past its work bound reports a verdict under `work bound` on its // own route; the tests stay in report order. const overBound = (x) => work && pastBound(h, slot, x) - const lawOf = (x) => (overBound(x) ? `work bound` : x.t.under[slot].law) - const verdictsIn = (kind) => - ran.filter( + const lawOf = (x) => + CUT_LAW[under(x)] ?? (overBound(x) ? `work bound` : x.t.under[slot].law) + const verdictsIn = (kind) => [ + ...ran.filter( (x) => (failing.includes(x) && under(x) === `verdict` && kindOf(x.t.under[slot].firedAt ?? x.t.route) === kind) || (overBound(x) && kindOf(x.t.route) === kind), - ) + ), + ...cut.filter((x) => kindOf(x.t.route) === kind), + ] const kills = verdictsIn(`reproducible`).map(lawOf) const behavioral = kills.filter((law) => law !== `reach agreement`) const decided = (failure) => ({ failure, ran }) @@ -762,7 +802,8 @@ function failureOf(h, slot, tests, work = false) { laws: [...new Set(behavioral)].sort(), lawsMayBePart: true, }) - if (timedOut) return decided({ outcome: `timeout` }) + // A gap cut: the timeout names the first (classify.mjs). + if (timedOut) return decided({ outcome: `timeout`, detail: timedOut }) if (kills.length > 0) return naming(`agreement-only-kill`, verdictsIn(`reproducible`), { laws: [`reach agreement`], @@ -771,13 +812,12 @@ function failureOf(h, slot, tests, work = false) { return naming(`random-only-kill`, verdictsIn(`random`)) if (verdictsIn(`unknown`).length > 0) return naming(`unregistered-kill`, verdictsIn(`unknown`)) - if (failing.length > 0 && failing.every((x) => under(x) === `timeoutMessage`)) - return decided({ outcome: `timeout` }) - const crashed = failing.filter( - (x) => under(x) !== `timeoutMessage` && kindOf(x.t.route) !== `framework`, + const crashed = [...failing, ...cut].filter( + (x) => kindOf(x.t.route) !== `framework`, ) if (crashed.length > 0) return naming(`crashed`, crashed) - if (failing.length > 0) return naming(`framework-error`, failing) + if (failing.length + cut.length > 0) + return naming(`framework-error`, [...failing, ...cut]) const unhandled = ran.some((x) => under(x) === `unhandled`) if (setup || unhandled) return decided({ outcome: `setup-error` }) return decided(undefined) @@ -1102,42 +1142,43 @@ function operandsSeen(h, mi, owners) { function ordinaryOutcome(h, mi, files) { const tests = files.flatMap((fi) => h.files[fi].tests.map((t) => ({ t, fi }))) const under = ({ t }) => t.under[mi].kind - const timedOut = tests.some((x) => under(x) === `runTimeout`) + // A gap cut in any of the files (the backstop, or a spin that could follow + // the clock). + const timedOut = files.some((fi) => cutOf(h, fi, mi).gap) const ran = tests.filter( (x) => - !h.files[x.fi].tests.some((t) => - [`setupError`, `runTimeout`].includes(t.under[mi].kind), - ), - ) - const failing = ran.filter((x) => - [`verdict`, `assertion`, `crash`, `timeoutMessage`].includes(under(x)), + !h.files[x.fi].tests.some((t) => ENDS_RUN.includes(t.under[mi].kind)), ) + const cut = tests.filter((x) => cutOf(h, x.fi, mi).t === x.t) + const failing = [ + ...ran.filter((x) => + [`verdict`, `assertion`, `crash`, `timeoutMessage`].includes(under(x)), + ), + ...cut, + ] const judgedOn = (kind) => failing.filter( (x) => (under(x) === `verdict` && kindOf(x.t.under[mi].firedAt ?? x.t.route) === kind) || + (CUT_LAW[under(x)] && kindOf(x.t.route) === kind) || (under(x) === `assertion` && kindOf(x.t.route) === kind && kind !== `framework`), ) const kills = judgedOn(`reproducible`) const behavioral = kills.filter( - (x) => under(x) === `assertion` || x.t.under[mi].law !== `reach agreement`, + (x) => + under(x) === `assertion` || + CUT_LAW[under(x)] || + x.t.under[mi].law !== `reach agreement`, ) if (behavioral.length > 0) return `killed` if (timedOut) return `timeout` if (kills.length > 0) return `agreement-only-kill` if (judgedOn(`random`).length > 0) return `random-only-kill` if (judgedOn(`unknown`).length > 0) return `unregistered-kill` - if (failing.length > 0 && failing.every((x) => under(x) === `timeoutMessage`)) - return `timeout` - if ( - failing.some( - (x) => under(x) !== `timeoutMessage` && kindOf(x.t.route) !== `framework`, - ) - ) - return `crashed` + if (failing.some((x) => kindOf(x.t.route) !== `framework`)) return `crashed` if (failing.length > 0) return `framework-error` if (tests.some((x) => under(x) === `setupError` || under(x) === `unhandled`)) return `setup-error` @@ -1487,7 +1528,7 @@ const messages = { verdict: `Error: violation`, assertion: `AssertionError: expected [ 'a' ] to deeply equal [ 'b' ]`, crash: `TypeError: x.y is not a function`, - timeoutMessage: `Error: Test timed out in 15000ms.`, + timeoutMessage: `Error: Test timed out in 65000ms.`, } // The capture's comparison and the swap a comparison mutant states, as the // instrumented code evaluates them (capture-setup.mjs `cmp`). @@ -1633,7 +1674,30 @@ function unitOf(h, fi, slot) { const f = h.files[fi] const under = (t) => t.under[slot] if (f.tests.some((t) => under(t).kind === `runTimeout`)) - return { file: fileName(h, fi), timedOut: true, seconds: 1 } + return { + file: fileName(h, fi), + timedOut: true, + cause: `backstop`, + seconds: 1, + } + // A copy cut by its call count or by no progress: the test that was running + // and its registered route, as the capture's cut row names them. + const cutAt = f.tests.findIndex((t) => CUT_LAW[under(t).kind]) + if (cutAt >= 0) + return { + file: fileName(h, fi), + timedOut: true, + cause: under(f.tests[cutAt]).kind === `runSpin` ? `spin` : `no progress`, + seconds: 1, + // A spin whose test's row is clocked names no test (check.mjs). + ...(cutOf(h, fi, slot).gap + ? {} + : { + builtIn: [ + { test: nameOf(h, fi, cutAt), route: f.tests[cutAt].route }, + ], + }), + } const loadFailed = f.tests.some((t) => under(t).kind === `setupError`) const tests = loadFailed ? [] : f.tests return { @@ -1924,12 +1988,7 @@ async function driver(h, observe) { // reproducible route, in a run that neither timed out nor failed to load. const lawsIn = (fi) => { const tests = h.files[fi].tests - if ( - tests.some((t) => - [`runTimeout`, `setupError`].includes(t.under[mi].kind), - ) - ) - return `` + if (tests.some((t) => ENDS_RUN.includes(t.under[mi].kind))) return `` const laws = tests.flatMap((t) => !t.flaky && t.under[mi].kind === `verdict` && @@ -1945,8 +2004,27 @@ async function driver(h, observe) { if (new Set(killing).size > 1) states.add(`two owner files kill under different laws`) } - if (result.outcome === `killed` && units.some((u) => u.timedOut)) + if ( + result.outcome === `killed` && + units.some((u) => u.timedOut && u.cause === `backstop`) + ) states.add(`killed beside a timeout`) + // A kill by a built-in law, from a copy cut by its call count or by no + // progress, and the same cut on the random campaign. + for (const law of [`bounded work`, `no progress`]) + if (result.outcome === `killed` && result.laws?.includes(law)) + states.add(`killed by ${law}`) + if ( + result.outcome === `random-only-kill` && + units.some((u) => u.builtIn !== undefined) + ) + states.add(`a random-only kill by a built-in law`) + // A spin whose count could follow the clock: a gap, not a kill. + if ( + result.outcome === `timeout` && + units.some((u) => u.cause === `spin` && u.builtIn === undefined) + ) + states.add(`a spin whose count could follow the clock timed out`) // The work law's states (laws 3 and 5): a kill by a work bound, alone // or beside a changed value that would otherwise be not-judged, a work // bound on the random campaign, and survivors whose work rose under a @@ -2120,6 +2198,8 @@ async function driver(h, observe) { class: r.class, laws: r.laws && [...r.laws].sort(), testOutcome: r.testOutcome, + // A timeout's detail is its cut (`outcomeOf`). + detail: r.outcome === `timeout` ? r.detail : undefined, })), summary: readSummary(summary), messages: messagesOf(summary), @@ -2259,8 +2339,14 @@ function check(h, observations, expected) { return violation(`classified`, `nothing was classified`, `completion`) // A kill names a non-empty part of its behavioral laws (the runner may // stop at the first); an agreement-only kill names reach agreement. + // A timeout names the cut that ended it (its `detail`, the cause). const view = (o) => - JSON.stringify([o.outcome, o.class, o.testOutcome ?? null]) + JSON.stringify([ + o.outcome, + o.class, + o.testOutcome ?? null, + o.outcome === `timeout` ? o.detail : null, + ]) for (const [mi, owed] of expected.outcomes.entries()) { const actual = got.outcomes[mi] const lawsFit = @@ -2599,11 +2685,18 @@ function check(h, observations, expected) { ) // Law 8: each gap line is one line, and shows its change shortened: an // operator mutant's original and replacement as two code spans, an - // authored mutant's intent as a code span and its author. The results keep the full text. + // authored mutant's intent as a code span and its author. A timeout's line + // ends with its cause, `(cause: )`, after the change. The results + // keep the full text. for (const owed of gapsOfModel) { const head = `oracle-gaps: gap ${owed.class ?? owed.outcome} L${owed.line} ${owed.id} ` const line = streamed.find((l) => l.startsWith(head)) - const rest = line?.slice(head.length) + const cause = owed.outcome === `timeout` ? ` (cause: ${owed.detail})` : `` + const tail = line?.slice(head.length) + const rest = + tail !== undefined && tail.endsWith(cause) + ? tail.slice(0, tail.length - cause.length) + : undefined let shown if (owed.authored) { // The intent is the author's text, so it sits in a code span. @@ -3167,6 +3260,10 @@ const required = [ `timeout`, `killed`, `killed beside a timeout`, + `killed by bounded work`, + `killed by no progress`, + `a random-only kill by a built-in law`, + `a spin whose count could follow the clock timed out`, `owners declared against file order name another first changed test`, `baseline records against file order name another first changed test`, `a mutant run's records against the baseline's name another first changed test`, @@ -3376,6 +3473,30 @@ const constructed = [ single(`a verdict on a fixed seed`, [`killed`], { effect: { kind: `verdict` }, }), + single( + `a copy cut by its call count on a fixed seed`, + [`killed by bounded work`], + { + effect: { kind: `runSpin` }, + }, + ), + single( + `a copy cut by no progress on a fixed seed`, + [`killed by no progress`], + { + effect: { kind: `runIdle` }, + }, + ), + single( + `a clocked copy cut by its call count`, + [`a spin whose count could follow the clock timed out`], + { test: { clocked: `timer` }, effect: { kind: `runSpin` } }, + ), + single( + `a copy cut by no progress in the random campaign`, + [`a random-only kill by a built-in law`], + { test: { route: `random seed` }, effect: { kind: `runIdle` } }, + ), single(`reach agreement alone`, [`agreement-only-kill`], { effect: { kind: `verdict`, law: `reach agreement` }, }), diff --git a/packages/oracles/oracle-gaps/gap-report-model-oracle.mjs b/packages/oracles/oracle-gaps/gap-report-model-oracle.mjs index bc29077..bf9b870 100644 --- a/packages/oracles/oracle-gaps/gap-report-model-oracle.mjs +++ b/packages/oracles/oracle-gaps/gap-report-model-oracle.mjs @@ -129,30 +129,53 @@ const TOP_LOOP = `@..gen` // Whether this filesystem ignores letter case, so that `src/Sut.ts` names // `src/sut.ts`; the `cased` spelling applies only where it does. const caseless = existsSync(fileURLToPath(import.meta.url).toUpperCase()) -// The full limit of a cut-off copy's confirmation that the driver passes -// (`--hang-limit`), and, under one budget, the first budget too. +// The backstop base that the driver passes (`--hang-limit`): a copy that runs +// past it, plus three seconds per second of its file's time, is cut. const FULL_LIMIT_MS = 20000 -// How long a slow test waits, in real milliseconds: past the 1 s first -// budget that the driver passes (`--first-limit`), within the full limit. +// How long a slow test waits, in real milliseconds, where its output differs. const SLOW_MS = 3000 -// How long each slow test of `w_slowpair` waits: past its file's first -// budget (the 1 s `--first-limit` plus three times its one-test file's -// unmutated time t), so its copy is always cut and confirmed; past half the -// mutant's budget, so two such confirmations never fit it; and under the -// full limit, so each would fit alone. The window holds while t < 3 s and -// ten times the two files' time stays under 2 × 10 s (their sum under 2 s). -const PAIR_SLOW_MS = 10000 -// A mutant's budget for the runs that confirm its cut-off copies, which the -// driver passes (`--mutant-limit`): one slow owner's confirmation fits it -// (SLOW_MS), and two of `w_slowpair`'s do not. Ten times the fixture's -// one-test files' time stays under it, so the floor is the budget. -const MUTANT_LIMIT_MS = 15000 +// The slow waits a drawn test holds: SLOW_MS and the 16 s wait. +const SLOW_WAITS = [SLOW_MS, 16_000] +// The `pendingIo: 'test'` and `'short'` witnesses' child (run with `node -e`, +// given the sampler's count file and a count): it writes to its stdout once +// the count has risen by that many samples, and gives up after 60 s, a +// backstop that leaves the read pending, so the witness then fails loudly. +const SAMPLE_HOLD = `const read = () => { try { return Number(require('fs').readFileSync(process.argv[1], 'utf8')) } catch { return 0 } }; const from = read(); setInterval(() => { if (read() >= from + Number(process.argv[2])) process.stdout.write('go', () => process.exit(0)) }, 10); setTimeout(() => process.exit(1), 60000)` +// How many of the sampler's samples each pipe read holds: the `test` witness's +// read waits past the 500 ms idle window, and a `short` read only two samples, +// so the sampler sees it once or twice. +const SAMPLES_HELD = { test: 20, short: 2 } +// The calls a heavy test (`heavy`) makes before each campaign's own call, with +// the same input, in every run: its unmutated work is then that many calls' +// more. The `until` function evaluates about eleven sites a call, so a heavy +// test of it does about 110,000, past CALL_FLOOR: its copies' call budget is +// a hundred times that (G6), and a copy that does a third more is inside it. +const HEAVY_CALLS = 10_000 +// A wait long past any copy's horizon (the full limit and 3 s per second of +// the file's time). +const PAST_HORIZON_MS = 600_000 +// The calls that each copy of `w_slowpair` makes beyond its first: its test +// calls the function this many more times where its output differs. A call +// evaluates two or three instrumented sites, so a copy makes 60,000 to 90,000 +// site evaluations: inside its call budget (CALL_FLOOR in check.mjs, 100,000), +// and the two copies together past the mutant's work budget (WORK_FLOOR). +// Work is counted, not timed, so the outcome is the same on any machine. +const PAIR_LOOPS = 30_000 +// A mutant's work budget (maintainer disposition G1, under the deterministic +// cuts decision of 2026-11-14): the larger of this floor and ten times its +// owner files' unmutated work, in site evaluations. The floor is the copies' +// call budget, so two copies that each stay inside theirs can pass it. +const WORK_FLOOR = 100_000 +// The least call budget a copy gets (check.mjs CALL_FLOOR): site evaluations in +// one copy's runs before it counts as a loop that never ends. A file's budget +// is a hundred times the most its unmutated tests did, if more. Past it, the +// copy is cut by count (a spin) and charged nothing (the outcome rule, item 4). +const CALL_FLOOR = 100_000 // How often the CLI says which mutant a busy copy runs (its heartbeat): the // driver shortens it from 30 s (`ORACLE_GAPS_HEARTBEAT_MS`), so that a slow // test's wait of SLOW_MS passes it. A copy beats once its own run has lasted -// an interval, so the interval is twice the 1 s first budget: a copy cut at -// its first budget never lasts it, and whether a run beats is a function of -// the history, not of the machine's load, as the determinism tests require. +// an interval, and whether a run beats is a function of the history, not of +// the machine's load, as the determinism tests require. const HEARTBEAT_MS = 2000 // The slow kill's hold (#84; v21.2 R57, the register's C entry "Determinism // under load covers witnesses too"). A wait of SLOW_MS spans the heartbeat @@ -448,6 +471,32 @@ const testCase = fc.record({ // Only a witness waits on a timer, sleeps, reads the clock, or waits a // turn: its work could follow the clock. timed: fc.constant(false), + // The deterministic-cuts fields, drawn (M, 2026-11-16, "Randomness wins"), + // appended last: a slow test's wait (only a witness is slow), the extra + // calls a test makes where its output differs, a timer it waits on there + // (rare, since each wait costs real time on every mutant that changes its + // output), and a repeating timer kept due while a hang waits (rare, since + // such a hang runs to the backstop). Each set holds the values the + // constructed histories use. A test that loops declares no work bound + // (`resolve`), since the model judges no work in a generated test. + slowMs: fc.constantFrom(...SLOW_WAITS), + loops: fc.constantFrom(0, 0, 0, 0, 0, 0, 0, 30_000), + timerWait: fc.oneof( + { arbitrary: fc.constant(null), weight: 15 }, + { + arbitrary: fc.record({ + waitMs: fc.constantFrom(600, 3000), + unref: fc.boolean(), + }), + weight: 1, + }, + ), + pulse: fc.constantFrom(false, false, false, false, false, false, false, true), + // What else is pending while a test waits (`pendingIo`: its own I/O, the + // runner's write, or the runner's timer). Each value switches a test-only + // hook, so it stays a constant that only constructed histories set (the + // hook exception of "Randomness wins"). + pendingIo: fc.constant(null), }) // Law 9's edits. Each content edit changes what some evidence depends on: // a test file, a source statement, the source's line numbers, an owns list, @@ -596,9 +645,10 @@ const drawn = fc.record({ tests: fc.array(testCase, { minLength: 1, maxLength: 5 }), // The witness module: all of it, or only the functions whose outcomes // need real test runs (see `captureWitness`). - // The budget witness alone (`budget`) costs two 10 s confirmations per - // mutant in every CLI run of a history, so generated histories rarely - // draw it; its constructed history (the daemon recheck) always does. + // The budget witness alone (`budget`) makes two copies of about 90,000 + // site evaluations each in every CLI run of a history, so generated + // histories rarely draw it; its constructed history (the daemon recheck) + // always does. witness: fc.oneof( { weight: 30, arbitrary: fc.constantFrom(false, true, `capture`) }, { weight: 1, arbitrary: fc.constant(`budget`) }, @@ -718,9 +768,9 @@ const drawn = fc.record({ // module loads, with 0. A mutant under which it throws then fails every // file that imports the module, before any test runs. boot: fc.constantFrom(false, false, false, true), - // The run's time budgets: a short first budget confirmed by a run alone - // with the full limit, or one budget (`--first-limit` equal to - // `--hang-limit`), where a run past it is a timeout at once. + // The run's budget mode: `one` passes the retired `--first-limit`, which + // the CLI checks and notes and which changes nothing; `two` passes no + // retired option. budget: fc.constantFrom(`two`, `two`, `two`, `one`), // Runs that the driver stops by a signal while the stop witness's test // spins (`w_stop`), each in a pool: Vitest's workers as child processes or @@ -1102,6 +1152,36 @@ const drawn = fc.record({ }), { maxLength: 8 }, ), + // The history's backstop (`--hang-limit`), drawn (M, 2026-11-16, "Randomness + // wins"): null is the full limit, and a longer one moves no copy into the + // backstop band. The constructed backstop of 2 s is taken back as null. + hangMs: fc.constantFrom(null, null, null, 40_000), + // Whether each of the history's own tests calls its function HEAVY_CALLS + // more times before each campaign's call (`heavy`; `resolve` gives it to + // each test of `tests`, never to the witness module's), so that its + // unmutated work passes CALL_FLOOR and its copies' call budget is its own + // hundred times (G6, under M "Count-only kills", 2026-11-16). Drawn, rarely + // (M, 2026-11-16, "Randomness wins"), and appended last (C-6), so every + // earlier field keeps its draws. A constructed history whose work budget + // decides its state sets it to 0. + heavy: fc.constantFrom( + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + HEAVY_CALLS, + ), }) // The edit sequence over the witness module, the one list of steps that a @@ -1110,7 +1190,7 @@ const drawn = fc.record({ // only file b, so the daemon reuses file a's timed-out runs; the revert // returns to a state that the daemon has seen. The budget witness alone // (`budget`) takes its own one step instead: a second daemon check with -// nothing changed, which reuses the confirmations that the first check ran. +// nothing changed, which reuses the results that the first check ran. const witnessSteps = [`test`, `helper`, `revert`] const budgetSteps = [`recheck`] // The stop witness alone takes no steps, so it counts only where the history @@ -1135,8 +1215,8 @@ const keepsSteps = (history) => { // witness module" takes `witnessSteps`, so every campaign still judges law // 9 over the witness module's outcome classes, along one short sequence // (maintainer decision, 2026-11-11, option M of the CI plan; a stated -// limit of the property). The budget witness's confirmations cost about -// 15 s a mutant, so its histories follow the same rule, and its constructed +// limit of the property). The budget witness's two copies make about 90,000 +// site evaluations each, so its histories follow the same rule, and its constructed // history "a second daemon check of the budget witness" takes // `budgetSteps` (the review of the gap CLI's runtime fixes, 2026-11-12, // P1-a: the budget witness only in its own histories). The map is its own @@ -1158,6 +1238,25 @@ const keepsSteps = (history) => { // (`constructedNeutral`, as `logLost` does): a shrink switches no hook on. // The drawn fields (`warmReplaced`, `testRoot`, `loadPlugin`, `spinData`, // `siblingDir`, `spinThrows`) come back as they are. +// A constructed history may set `slowTwin`, which no generated history +// draws, and which is taken back without it. Its tests may set `pendingIo`, a hook switch the drawn test +// record holds at null, a clock path (`timed`, drawn as false), and a timer +// wait written without `unref`, which is taken back as a ref'd one. +const takenBack = (t) => ({ + ...t, + // A constructed test's clock path (the clocked spin) is not drawn. + timed: false, + slow: false, + // A constructed test may leave a drawn field out, or set a wait the draw + // does not hold (#131's 10 s wait): it holds the neutral value. + slowMs: SLOW_WAITS.includes(t.slowMs) ? t.slowMs : SLOW_MS, + loops: t.loops ?? 0, + pulse: t.pulse ?? false, + timerWait: t.timerWait + ? { waitMs: t.timerWait.waitMs, unref: t.timerWait.unref === true } + : null, + pendingIo: null, +}) // The neutral value of each constructed-only field: a drawn record holds it, // and the take-back maps a constructed history back to it. const constructedNeutral = { @@ -1224,29 +1323,39 @@ const histories = drawn.map( ) if (!keepsSteps(history)) throw new Error(`a history with the witness module takes no steps`) + // A constructed backstop outside the drawn set (`hangMs`) comes back as null. + const { slowTwin, hangMs, ...drawnValue } = + /** @type {Record} */ (value) + const tests = /** @type {Record[]} */ (drawnValue.tests) + const plain = + /** @type {{ steps: unknown[] | null, stopRuns: Array<{ when: string }> } & Record} */ ({ + ...drawnValue, + hangMs: hangMs === 40_000 ? hangMs : null, + tests: tests.map(takenBack), + }) + // Take-back into the drawn language: a disarmed pause (pauseEngages false, + // only the constructed history sets it) comes back as engaged, and a + // rejected batch (rejectBatch, likewise) as none, so the arbitrary can + // shrink it. Neither is therefore kept by a shrink. const reset = Object.entries(constructedNeutral).every( - ([field, neutral]) => history[field] === neutral, + ([field, neutral]) => plain[field] === neutral, ) - ? value - : { ...history, ...constructedNeutral } - const stopRuns = plainStops(history.stopRuns ?? NO_STOPS) - const steps = plainSteps(history.steps) - // A constructed slow test (a wait where the output differs, seconds of - // real time) is read as the drawn one, which never waits. - const tests = history.tests.some((test) => test.slow) - ? history.tests.map(({ slowMs, ...test }) => ({ ...test, slow: false })) - : history.tests - // A half of the witness module (captureHalves, constructed only) comes - // back as the whole module, which the arbitrary draws. - const whole = Object.hasOwn(captureHalves, history.witness) - ? { ...reset, witness: `capture` } - : reset - const taken = - stopRuns === history.stopRuns && - steps === history.steps && - tests === history.tests - ? whole - : { ...whole, stopRuns, steps, tests } + ? plain + : { ...plain, ...constructedNeutral } + const stopRuns = plainStops(plain.stopRuns ?? NO_STOPS) + const steps = plainSteps(plain.steps) + // A constructed test's fields come back as drawn ones (`takenBack`): a + // constructed slow test (a wait where the output differs, seconds of real + // time) is read as the drawn one, which never waits. A half of the + // witness module (captureHalves, constructed only) comes back as the + // whole module, which the arbitrary draws. + const taken = { + ...(Object.hasOwn(captureHalves, plain.witness) + ? { ...reset, witness: `capture` } + : reset), + stopRuns, + steps, + } const later = (steps ?? []).filter((step) => laterEdits.includes(step)) if (later.length === 0) return taken const earlier = steps.filter((step) => !laterEdits.includes(step)) @@ -1978,9 +2087,10 @@ const witness = [ judging(`a`, `fixed seed`, 7, { judgesShared: true }), ], }, - // Hangs. A test that hangs where it would judge cuts its file's run off - // at the time limit. Alone, the mutant is a timeout; a kill in another - // owner still decides, and a survivor in another owner does not. + // Hangs. A test that hangs where it would judge makes no progress: its copy + // is idle by state, and the built-in law no progress kills the mutant on the + // test's route (M, 2026-11-16). Owner a's copy runs first, so a's hang + // decides beside another owner's kill or survivor alike. // A module imported through a query is loaded by Vite as written: the // owner's test reads it with `?raw` and must see the module's own text, // which the instrumented copy does not change. The judging test kills the @@ -2037,33 +2147,24 @@ const witness = [ body: [kill], tests: [judging(`a`, `random seed`, 7), judging(`b`, `fixed seed`, 7)], }, - // A slow kill: the run ends after the first budget, within the full - // limit, so the confirming rerun kills it (a run is a timeout only when - // it passes the full limit alone). + // A slow kill: its owner test waits 3 s where its output differs, then + // judges, so its copy runs to its end and the kill decides. { id: `w_slowkill`, owners: [`a`], body: [kill], tests: [judging(`a`, `fixed seed`, 7, { slow: true })], }, - // Slow in both owners, and judged by neither: each owner's copy is cut at - // its first budget, and the two confirmations together need more than the - // mutant's budget, so the mutant is a timeout, not a survivor. + // Slow in both owners, and judged by neither: each owner's copy makes + // 30,000 more calls, and the two copies together pass the work budget, so + // the built-in law bounded work kills the mutant (M, 2026-11-16). { id: `w_slowpair`, owners: PAIR_FILES, body: [kill], tests: [ - judging(`p`, `fixed seed`, 7, { - judges: false, - slow: true, - slowMs: PAIR_SLOW_MS, - }), - judging(`q`, `fixed seed`, 7, { - judges: false, - slow: true, - slowMs: PAIR_SLOW_MS, - }), + judging(`p`, `fixed seed`, 7, { judges: false, loops: PAIR_LOOPS }), + judging(`q`, `fixed seed`, 7, { judges: false, loops: PAIR_LOOPS }), ], }, // A kill as usual. In a run that the driver stops (`GAP_STOP_DIR` set), @@ -2156,8 +2257,8 @@ const witness = [ tests: [judging(`a`, `fixed seed`, 1)], }, // A test that waits on a child process where the output differs: an I/O - // handle is open, so its copy is busy, never cut for idleness, and the - // confirming run kills it. + // handle is open, so its copy is busy, never cut for idleness, and its + // kill decides. { id: `w_iowait`, owners: [`a`], @@ -3074,8 +3175,8 @@ function resolveHistory(history) { ? witness .filter( (w) => - // The budget witness costs two 10 s confirmations per mutant, - // so only histories of the `budget` mode carry it. + // The budget witness's two copies make about 90,000 site + // evaluations each, so only histories of the `budget` mode carry it. (history.witness === true && w.id !== `w_slowpair`) || (history.witness === `capture` && captureWitness.has(w.id)) || captureHalves[history.witness]?.has(w.id) || @@ -3341,7 +3442,9 @@ function resolveHistory(history) { // its input is a multiple of 3: derived from the plan, not drawn, so // the grammar's generated histories, and their cost, stay those of // the grammar without the work law. - work: t.work ?? (t.input % 3 === 0 ? 1 : null), + work: t.work ?? (t.input % 3 === 0 && !t.loops ? 1 : null), + // The history's heavy calls (`heavy`), for its own tests only. + heavy: history.heavy ?? 0, })), ...functions.flatMap((fn) => fn.generated ? [] : fn.tests.map((t) => ({ ...t, fn: t.fn ?? fn.id })), @@ -4608,6 +4711,15 @@ function runsOf(plan, tests, sites, active) { return tests .filter((t) => !t.flaky) .map((t) => { + // Whether the test's count could follow the clock when its run was cut + // (`clocked`): it reads the clock, makes a timer, or does I/O of its own + // anywhere in its run (`readsClock`, before production or after it), or + // it waited where an earlier campaign's output changed. Its count could + // then depend on time, so a spin there is no kill (M, 2026-11-16, + // "Count-only kills"): the call budget and the spin judge the test's + // count as a whole, and a clock read after the spin point still makes + // the unmutated count one that the clock could move (v21.1 R19a). + let waited = false let changedAny = false let slowed = false const found = [] @@ -4616,61 +4728,175 @@ function runsOf(plan, tests, sites, active) { // calls no production code, so it does none. let before = 0 let after = 0 + // A heavy test (`heavy`) calls its function HEAVY_CALLS more times before + // each campaign's own call, in every run. + const extra = heavyIn(t) ? t.heavy : 0 for (const part of partsOf(t)) { const base = interpret(plan, t.fn, part.input, sites) const mutated = interpret(plan, t.fn, part.input, sites, active) + const route = routeOf(t, part) + const changed = + !mutated.threw && mutated.trace.join() !== base.trace.join() + // Where the output differs, a `loops` witness calls the function that + // many more times, so its work is the call's work once per call. if (!t.judgesShared) { - before += base.work - after += mutated.work + before += base.work * (1 + extra) + after += mutated.work * (1 + extra + (changed ? (t.loops ?? 0) : 0)) } - const route = routeOf(t, part) + // A wait past the history's backstop (its `hangMs`, else the full limit) + // and a margin of 6 s for three seconds per second of a one-test file + // (the file's time is under 2 s) is past the copy's horizon: its timer is + // not due inside it, so the copy is idle and cut as no progress, by state + // (M entry "idle is decided by state only", 2026-11-15). Its outcome does + // not depend on load. + const backstopMs = plan.history.hangMs ?? FULL_LIMIT_MS + if (changed && t.slow && (t.slowMs ?? SLOW_MS) > backstopMs + 6000) + return { + clocked: readsClock(t) || waited, + rowClocked: + readsClock(t) || waited || (changed && waitsIn(plan, t)), + t, + route, + hang: true, + cut: 'state', + verdicts: found, + changed: true, + work: after, + before, + } + // A test that writes to the console and then waits on a timer due past + // its horizon, while the runner's own write is pending (`pendingIo: + // 'runner'`) or its own interval runs (`'runnerTimer'`), cannot make + // progress of its own: the runner's I/O and timers are not the test's, + // and the test's timer is not due inside its horizon, so the copy is + // idle and cut as no progress, by state (M entry "Idle counts only the + // test's I/O", 2026-11-15; C, timers by ownership, 2026-11-16). + if ( + changed && + (t.pendingIo === `runner` || t.pendingIo === `runnerTimer`) + ) + return { + clocked: readsClock(t) || waited, + rowClocked: + readsClock(t) || waited || (changed && waitsIn(plan, t)), + t, + route, + hang: true, + cut: 'state', + verdicts: found, + changed: true, + work: after, + before, + } // A loop that never ends cuts the file's run off, as a hang does. if (mutated.spun) - return { t, route, hang: true, verdicts: found, changed: true } - const changed = - !mutated.threw && mutated.trace.join() !== base.trace.join() + return { + clocked: readsClock(t) || waited, + rowClocked: + readsClock(t) || waited || (changed && waitsIn(plan, t)), + t, + route, + hang: true, + cut: 'count', + verdicts: found, + changed: true, + work: after, + before, + } changedAny ||= changed slowed ||= changed && (t.slow || t.ioWait) if (mutated.threw) return { + clocked: readsClock(t) || waited, + rowClocked: + readsClock(t) || waited || (changed && waitsIn(plan, t)), t, route, crash: true, verdicts: found, changed: changedAny, slowed, + work: after, + before, } - // A test that hangs where another would judge never ends. + // A test that hangs where another would judge never ends. Without a + // `pulse` it goes idle (no progress); with one, its timer keeps it busy, + // so only the backstop cuts it (a wall-clock cut). if (changed && t.hangs) - return { t, route, hang: true, verdicts: found, changed: true } + return { + clocked: readsClock(t) || waited, + rowClocked: + readsClock(t) || waited || (changed && waitsIn(plan, t)), + t, + route, + hang: true, + cut: t.pulse ? 'wall' : 'state', + verdicts: found, + changed: true, + work: after, + before, + } // A verdict test of file b that reads the data spins there instead, // while the data is open, which it is before any edit (`spinData`): - // a loop that never ends, as above. + // a loop that never ends, cut by its call count, as above. A copy + // that first schedules its error (`spinThrows`) has made a timer, and + // one whose data read is async has had its own I/O complete first + // (C, 2026-11-17), so its count could follow the clock (v21.1 R19a): + // its spin is a gap. if (changed && plan.spinData && fsGated(plan, t.file, t)) - return { t, route, hang: true, verdicts: found, changed: true } + return { + clocked: + readsClock(t) || waited || plan.spinThrows || waitsIn(plan, t), + rowClocked: + readsClock(t) || waited || plan.spinThrows || waitsIn(plan, t), + t, + route, + hang: true, + cut: 'count', + verdicts: found, + changed: true, + work: after, + before, + } if (changed && t.judges) { found.push({ route, law: t.judgeBy === `verdict` ? t.law : `(unnamed)`, }) if (t.judgeBy !== `soft`) - return { t, route, verdicts: found, changed: true, slowed } + return { + clocked: readsClock(t) || waited, + rowClocked: + readsClock(t) || waited || (changed && waitsIn(plan, t)), + t, + route, + verdicts: found, + changed: true, + slowed, + work: after, + before, + } } + if (changed && waitsIn(plan, t)) waited = true } // A test that passed: its route is the last one it registered, and - // its work is judged against the bound it declared. An authored - // mutant's copy is not instrumented, so it counts no work, and the - // work law judges only operator mutants. + // its work is judged against the bound it declared. The work law + // judges only operator mutants: an authored copy counts calls only for + // its call budget. const route = routeOf(t, partsOf(t).at(-1)) // Only a route that oracleProperty registers for production can carry - // a bound, and a test that made a timer can do work that follows the - // clock, so neither judges work. + // a bound, and a test whose row is clocked (it made a timer, read the + // clock, or its own I/O completed, here or where it waited) can do work + // that follows the clock, so neither judges work (the CLI's `!clocked`). const judgesWork = found.length === 0 && active?.site !== undefined && [`reproducible`, `random`].includes(route) && - (!t.timed || t.timed === `turn`) + !readsClock(t) && + !waited return { + clocked: readsClock(t) || waited, + rowClocked: readsClock(t) || waited, t, route, verdicts: @@ -4683,10 +4909,34 @@ function runsOf(plan, tests, sites, active) { changed: changedAny, slowed, rose: judgesWork && after > before, + work: after, + before, } }) } +// Whether a test's count could follow the clock in every run: it reads the +// clock or makes a timer (`timed`, except a turn, which waits no time), +// before it runs production or after it (`after`), or its own I/O completes +// during it (`timed: 'read'`, a short async read, and `pendingIo: 'short'`, +// a pipe read), whatever a sample saw (C, 2026-11-17: clocked by state). And whether it waits +// on a timer or on I/O where its output changed. +const readsClock = (t) => + (Boolean(t.timed) && t.timed !== `turn`) || t.pendingIo === `short` +// A verdict test of file b that reads the data with an async read (`fsRead` +// promise or callback) where its output differs has I/O of its own that +// completes there (C, 2026-11-17: clocked by state). +const waitsIn = (plan, t) => + Boolean(t.slow || t.timerWait || t.ioWait || t.pendingIo === `test`) || + (fsGated(plan, t.file, t) && + (plan.fsRead === `promise` || plan.fsRead === `callback`)) +// Whether a test makes its heavy calls (`heavy`): a test that calls +// production on its own input, once per run (a test.prop runs its body many +// times, and the shared-state witness calls it only after another test, so +// neither is heavy). +const heavyIn = (t) => + Boolean(t.heavy) && !t.judgesShared && !t.readsShared && !t.asProp + // A kill: a behavioral verdict on a reproducible route. const killsIn = (r) => r.verdicts.some( @@ -4714,48 +4964,130 @@ function filesRun(runs) { return ran } +// The built-in law that each cut by count or by state fails (M, 2026-11-16, +// "Spin, no progress and the work budget are kills", restoring v21.1 K1 and +// P3): a spin past the call budget, and a copy whose work passes the +// mutant's work budget, fail bounded work; a copy idle by state fails no +// progress. Only a wall-clock cut (the backstop) is a gap, a `timeout`. +const BUILT_IN = [`bounded work`, `no progress`] + /** - * A mutant's kind from its owner tests' runs. An operator mutant's - * confirming runs share its budget (`budgeted`); an authored mutant's runs - * are each final at the full limit, so it has no confirmations to budget. + * A mutant's kind from its owner tests' runs. An operator mutant's copies + * share its work budget (`budgeted`); an authored mutant's copies are judged + * alone, with no budget. `detail` receives a timeout's cause and, for a kill, + * the built-in law that decided it (`builtIn`, null for a behavioral law). */ -function judge(plan, tests, sites, active, { budgeted = true } = {}) { +function judge(plan, tests, sites, active, { budgeted = true, detail } = {}) { const runs = runsOf(plan, tests, sites, active) // A file's tests run in order, and a kill (a behavioral verdict on a - // reproducible route) stops the file. A hang before that cuts the file's - // run off, and a run that was cut off records nothing, not even its - // earlier verdicts. A kill in another file still decides (a kill outranks - // a timeout); failing that, the mutant is a timeout. + // reproducible route) stops the file. A cut copy records nothing, not even + // its earlier verdicts. A cut by count or by state is a verdict of a + // built-in law on the route of the test that was running, so it kills on a + // reproducible route and decides the mutant; on another route the next + // owner's copy still runs. A wall-clock cut (the backstop) ends the owner + // files after it, and the mutant is a timeout unless a copy before it + // killed. const hung = new Set() - const slowed = new Map() - for (const file of new Set(runs.map((r) => r.t.file))) + const builtIns = [] + let wallCut = false + // The cause of a timeout, the first gap cut in owner order: the backstop, + // or a spin whose count could depend on time (M, 2026-11-16, "Count-only + // kills"), which ends its copy as a gap and lets the next owner run. + let gapCause + // The mutant's work budget (G1): its owner files' copies are counted in + // their order, each copy's work being the site evaluations of its runs up + // to a cut or a kill. The copy whose work passes the budget fails bounded + // work on the routes of its tests that did work, its own verdicts do not + // count, and no later copy runs. The budget is the larger of WORK_FLOOR and + // ten times the owner files' unmutated work, where a file's unmutated work + // is its most-working test's. + // Copies run in sorted owner order (register 2026-11-14, "Deterministic cuts, + // outcome rule"; G44), the order production takes by path, whatever order the + // history lists its tests in. + const files = [...new Set(runs.map((r) => r.t.file))].sort((a, b) => + a < b ? -1 : a > b ? 1 : 0, + ) + // A file's most-working unmutated test, in site evaluations. + const mostBefore = (file) => + Math.max(0, ...runs.filter((r) => r.t.file === file).map((r) => r.before)) + const unmutated = files.reduce((sum, file) => sum + mostBefore(file), 0) + const workBudget = Math.max(WORK_FLOOR, 10 * unmutated) + const counted = new Set() + let spent = 0 + for (const file of files) { + let work = 0 + const callBudget = Math.max(CALL_FLOOR, 100 * mostBefore(file)) + let builtIn + let countGap = false for (const r of runs.filter((x) => x.t.file === file)) { - if (r.slowed) slowed.set(file, r.t.slowMs ?? SLOW_MS) - if (r.hang) hung.add(file) - if (r.hang || killsIn(r)) break + // A test whose calls pass its call budget is cut by count, where the + // run goes past it (the CLI's spin row; the budget is per test). An + // authored mutant's copy counts calls too (issue #167), so it spins as + // an operator copy does. + const spun = !r.hang && r.work > callBudget + const byCount = spun || (r.hang && r.cut === `count`) + if (r.hang && r.cut === `wall`) wallCut = true + // A count that could depend on time kills nothing: a gap. + else if (byCount && r.clocked) countGap = true + else if (r.hang || spun) + builtIn = { + route: r.route, + law: byCount ? `bounded work` : `no progress`, + } + if (r.hang || spun) { + hung.add(file) + break + } + // Only a count that does not depend on time is charged to the budget. + work += r.rowClocked ? 0 : r.work + if (killsIn(r)) break + } + if (wallCut) { + gapCause ??= `backstop` + break + } + if (countGap) { + gapCause ??= `spin` + continue } - const live = runs.filter((r) => !hung.has(r.t.file)) + if (builtIn) { + builtIns.push(builtIn) + if (builtIn.route === `reproducible`) break + continue + } + if (budgeted && spent + work > workBudget) { + for (const r of runs.filter( + (x) => x.t.file === file && x.work > 0 && !x.rowClocked, + )) + builtIns.push({ route: r.route, law: `bounded work` }) + break + } + spent += work + counted.add(file) + } + const live = [ + ...runs.filter((r) => counted.has(r.t.file) && !hung.has(r.t.file)), + ...builtIns.map((v) => ({ route: v.route, verdicts: [v], changed: true })), + ] const verdicts = (route) => live.flatMap((r) => r.verdicts.filter((v) => v.route === route).map((v) => v.law), ) const reproducible = verdicts(`reproducible`) - if (reproducible.some((law) => law !== `reach agreement`)) return `killed` - if (hung.size > 0) return `timeout` - // The runs that confirm a mutant's cut-off copies share its budget - // (`--mutant-limit`, MUTANT_LIMIT_MS here; maintainer disposition G1, - // 2026-11-11). Under two budgets, a slow owner file is cut at its first - // budget and confirmed alone, and its confirmation takes longer than the - // time its slow test waits. When the confirmations of every slow owner - // file do not fit the budget together, one is cut, and the mutant is a - // timeout; a kill still decides first. Under one budget no copy is - // confirmed, so the budget never binds. - if ( - budgeted && - plan.budget === `two` && - [...slowed.values()].reduce((sum, ms) => sum + ms, 0) >= MUTANT_LIMIT_MS - ) + if (reproducible.some((law) => law !== `reach agreement`)) { + if (detail) + detail.builtIn = + reproducible.find((law) => BUILT_IN.includes(law)) ?? null + return `killed` + } + if (gapCause) { + if (detail) detail.cause = gapCause return `timeout` + } + // No wall-clock budget decides here (maintainer decision, deterministic + // cuts, option A, 2026-11-14): a slow copy runs to its end, and the mutant + // budget is a work count (above). The wall-clock backstops report a gap no + // history here reaches, so a slow owner's wait never makes a timeout. if (reproducible.length > 0) return `agreement-only-kill` if (verdicts(`random`).length > 0) return `random-only-kill` if (verdicts(`unknown`).length > 0) return `unregistered-kill` @@ -5118,6 +5450,13 @@ function model(plan) { `indeterminate`, `unreached`, ]) + // A timeout's cause (the backstop), and the built-in law of a kill that one + // decided (`judge`). + const detailOf = (tests, active, options) => { + const detail = {} + judge(plan, tests, sites, active, { ...options, detail }) + return detail + } const mutants = sites.map((site) => { const kind = site.probe ? undefined @@ -5134,6 +5473,14 @@ function model(plan) { ordinal: site.ordinal, probe: site.probe, kind, + cause: + kind === `timeout` + ? detailOf(ownerTests(site.fnId), { site: site.id }).cause + : undefined, + builtIn: + kind === `killed` + ? detailOf(ownerTests(site.fnId), { site: site.id }).builtIn + : undefined, ordinaryOutcome: plan.ordinary && survivorKinds.has(kind) && ordinaryReaches(site), } @@ -5143,8 +5490,9 @@ function model(plan) { const authoredAll = plan.authoredList.map((a) => ({ id: a.id, key: `${SOURCE}#authored:${a.id}`, - // An authored mutant is not instrumented, so a survivor is not-judged - // (a changed observation) or not-observed (the patch changed the code). + // An authored mutant's copy counts calls only, with no value changes + // recorded, so a survivor is not-judged (a changed observation) or + // not-observed (the patch changed the code). kind: !a.applies ? `does-not-apply` : owners(a.fn).size === 0 @@ -5160,6 +5508,11 @@ function model(plan) { ), ), line: a.applies ? a.line : undefined, + cause: + a.applies && owners(a.fn).size > 0 + ? detailOf(ownerTests(a.fn), { authored: a.label }, { budgeted: false }) + .cause + : undefined, })) const authored = authoredAll[0]?.kind const isGap = (kind) => @@ -5320,9 +5673,25 @@ function oracleTestFile(plan, file) { ...(plan.tests.some((t) => t.file === file && t.shrinks) ? [`import fc from 'fast-check'`] : []), - ...(plan.tests.some((t) => t.file === file && (t.ioWait || t.stops)) + ...(plan.tests.some( + (t) => + t.file === file && + (t.ioWait || + t.stops || + t.pendingIo === `test` || + t.pendingIo === `short`), + ) ? [`import { spawn } from 'node:child_process'`] : []), + ...(plan.tests.some( + (t) => + t.file === file && (t.pendingIo === `test` || t.pendingIo === `short`), + ) + ? [ + `import { join as samplesPath } from 'node:path'`, + `import { threadId } from 'node:worker_threads'`, + ] + : []), ...(plan.tests.some((t) => t.file === file && t.timed === `sleep`) ? [`import { setTimeout as sleep } from 'node:timers/promises'`] : []), @@ -5332,6 +5701,7 @@ function oracleTestFile(plan, file) { `import { setTimeout as nodeSetTimeout } from 'node:timers'`, `import { uptime as osUptime } from 'node:os'`, `import { monitorEventLoopDelay } from 'node:perf_hooks'`, + `import { readFile as readFileAsync } from 'node:fs/promises'`, ] : []), ...(plan.tests.some((t) => t.file === file && t.stops) @@ -5408,8 +5778,9 @@ function oracleTestFile(plan, file) { ...(t.registered && t.work !== null ? [` g.__oracleWorkBound?.(${t.work})`] : []), - // A test that waits on a timer before it runs production. - ...(t.timed + // A test that waits on a timer before it runs production (a test that + // reads the clock `after` its work does so below). + ...(t.timed && t.timed !== `after` ? [ { timer: ` if ((await promisify(setTimeout)(0, 'v')) !== 'v') throw new Error('promisify(setTimeout) lost its value')`, @@ -5423,10 +5794,27 @@ function oracleTestFile(plan, file) { waitasync: ` if (Atomics.waitAsync(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 0).value !== 'timed-out') throw new Error('waitAsync did not time out')`, wait: ` if (Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 0) !== 'timed-out') throw new Error('wait did not time out')`, loopdelay: ` { const delay = monitorEventLoopDelay(); delay.enable(); delay.disable() }`, + // A short async read of its own (a millisecond, which the + // sampler rarely sees): I/O that completes during the test. + read: ` if ((await readFileAsync(new URL('../package.json', import.meta.url), 'utf8')).length === 0) throw new Error('the package file is empty')`, turn: ` await new Promise((r) => setImmediate(r))`, }[t.timed], ] : []), + // A short read of a pipe the test opened, before it runs production, + // in every run: held for two of the sampler's samples (a count), so + // the test's unmutated rows record I/O of its own. + ...(t.pendingIo === `short` + ? [ + ` await new Promise((r) => { const child = spawn(process.execPath, ['-e', ${JSON.stringify(SAMPLE_HOLD)}, samplesPath(process.env.ORACLE_GAPS_TEST_SAMPLES ?? '', \`\${process.pid}-\${threadId}\`), '${SAMPLES_HELD.short}'], { stdio: ['ignore', 'pipe', 'ignore'] }); child.unref(); child.stdout.once('data', r) })`, + ] + : []), + // A heavy test's calls before its own (`heavy`), in every run. + ...(heavyIn(t) + ? [ + ` for (let h${i} = 0; h${i} < ${t.heavy}; h${i} += 1) ${callFor(t.fn)}(${part.input}, [])`, + ] + : []), ` const ${out}: Array = []`, ...(t.setsShared ? [` shared.value = 5`] : []), t.readsShared @@ -5476,8 +5864,7 @@ function oracleTestFile(plan, file) { ` if (reads${i} !== 0) throw new Error('the payload was read')`, ] : []), - // A slow test waits past the first budget, within the full limit, - // where the output differs; it then judges as usual. The slow kill + // A slow test waits where the output differs; it then judges as usual. The slow kill // (`w_slowkill`, no `slowMs`) waits on the CLI instead of the clock // where the driver names the run's stream log (`beatHeld`). ...(t.slow @@ -5485,6 +5872,60 @@ function oracleTestFile(plan, file) { ` if (${when} && JSON.stringify(${out}) !== ${JSON.stringify(expected)}) await ${t.slowMs === undefined ? `beatHeld()` : `new Promise((r) => setTimeout(r, ${t.slowMs}))`}`, ] : []), + // Where its output differs, a test that calls the function `loops` + // more times: each call is work the copy does, and the work budget + // counts it. + ...(t.loops + ? [ + ` if (${when} && JSON.stringify(${out}) !== ${JSON.stringify(expected)}) for (let k${i} = 0; k${i} < ${t.loops}; k${i} += 1) ${callFor(t.fn)}(${part.input}, [])`, + ] + : []), + // A test that reads the clock only after its work (`after`), in every + // run it reaches here: its unmutated rows are clocked, while a copy + // that spins before this line never reads it. + ...(t.timed === `after` + ? [ + ` if (Date.now() < 0) throw new Error('the clock reads before 1970')`, + ] + : []), + // Where its output differs, a test that waits on one timer due well + // inside its horizon. Nothing else keeps it busy: no load is in the test + // (a loaded variant would be a CPU hog outside the worker), so the timer + // check alone stands between this copy and an idle cut. + ...(t.timerWait + ? [ + ` if (${when} && JSON.stringify(${out}) !== ${JSON.stringify(expected)}) await new Promise((r) => setTimeout(r, ${t.timerWait.waitMs})${t.timerWait.unref ? `.unref()` : ``})`, + ] + : []), + // Where its output differs, a test that reads a pipe it opened: the + // stdout of a child it unrefs, so that only the read is active. The + // child writes once the copy's sampler has judged the test 20 more + // times (ORACLE_GAPS_TEST_SAMPLES), a count of samples at least 50 ms + // apart, past the 500 ms idle window, so the state is held by a count + // and not by a time. A copy that wrongly goes idle stops its sampler's + // count, so the read never ends and the copy is cut. The test then + // judges as usual. + ...(t.pendingIo === `test` + ? [ + ` if (${when} && JSON.stringify(${out}) !== ${JSON.stringify(expected)}) await new Promise((r) => { const child = spawn(process.execPath, ['-e', ${JSON.stringify(SAMPLE_HOLD)}, samplesPath(process.env.ORACLE_GAPS_TEST_SAMPLES ?? '', \`\${process.pid}-\${threadId}\`), '${SAMPLES_HELD.test}'], { stdio: ['ignore', 'pipe', 'ignore'] }); child.unref(); child.stdout.once('data', r) })`, + ] + : []), + // Where its output differs, a test that writes to the console and then + // waits on an unref'd timer due long past its horizon, while the + // runner holds its own write pending (ORACLE_GAPS_TEST_RUNNER_WRITE). + ...(t.pendingIo === `runner` + ? [ + ` if (${when} && JSON.stringify(${out}) !== ${JSON.stringify(expected)}) { console.log('waiting past the horizon'); await new Promise((r) => setTimeout(r, ${PAST_HORIZON_MS}).unref()) }`, + ] + : []), + // Where its output differs, a test that waits on a ref'd timer due long + // past its horizon, while the runner keeps its own interval running + // (ORACLE_GAPS_TEST_RUNNER_TIMER), as Vitest's task-update timer can be. + ...(t.pendingIo === `runnerTimer` + ? [ + ` if (${when} && JSON.stringify(${out}) !== ${JSON.stringify(expected)}) await new Promise((r) => setTimeout(r, ${PAST_HORIZON_MS}))`, + ] + : []), // A test that waits on I/O (a child process, about 3 s) where the // output differs, then judges as usual. ...(t.ioWait @@ -5513,10 +5954,14 @@ function oracleTestFile(plan, file) { ` }`, ] : []), - // A flaky test never hangs: the model leaves flaky tests out. + // A flaky test never hangs: the model leaves flaky tests out. A + // `pulse` hang keeps a repeating timer due inside its horizon, so the + // copy is never idle, and it never finishes: only the backstop cuts it. ...(t.hangs && !t.flaky ? [ - ` if (${when} && JSON.stringify(${out}) !== ${JSON.stringify(expected)}) await new Promise(() => {})`, + t.pulse + ? ` if (${when} && JSON.stringify(${out}) !== ${JSON.stringify(expected)}) { setInterval(() => {}, 100); await new Promise(() => {}) }` + : ` if (${when} && JSON.stringify(${out}) !== ${JSON.stringify(expected)}) await new Promise(() => {})`, ] : []), // Where the output differs and the data is open, a verdict test of @@ -5563,7 +6008,7 @@ function oracleTestFile(plan, file) { : []), t.asProp ? ` fcTest.prop([fcv.integer({ min: 0, max: 10 })], { numRuns: 20, seed: 1 })('${testName(t, index)}', async (n) => {` - : ` test('${testName(t, index)}', ${(t.hangs && !t.flaky) || t.slow || t.ioWait || t.timed || lazyGated(plan, file, t) || fsGated(plan, file, t) ? `async ` : ``}() => {`, + : ` test('${testName(t, index)}', ${(t.hangs && !t.flaky) || t.slow || t.ioWait || t.timerWait || t.pendingIo || t.timed || lazyGated(plan, file, t) || fsGated(plan, file, t) ? `async ` : ``}() => {`, ...partsOf(t).flatMap(campaign), ` })`, `})`, @@ -6653,6 +7098,7 @@ export { kill, push, judging, + HEAVY_CALLS, witness, probeText, guardText, @@ -6675,7 +7121,6 @@ export { ownersOf, interpret, judge, - MUTANT_LIMIT_MS, HEARTBEAT_MS, FULL_LIMIT_MS, refusal, diff --git a/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs b/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs index d105f89..7c8541c 100644 --- a/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs +++ b/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs @@ -140,28 +140,24 @@ * decision of 2026-11-06); a mutant that only nondeterministic owner tests * could judge is `indeterminate`. A mutant under which the module throws * while it loads fails every file that imports it before any test runs: - * an error outside any test, a `setup-error`. A run is cut off when it - * passes its first, short budget, and only the copy that ran past it is - * charged with that time (a copy whose file fails to load ends its turn - * there); it then runs again alone with the full limit - * (the short budget with confirmation, maintainer decision of - * 2026-11-06): a run that ends within the full limit is judged as if it - * had never been cut off, so a slow kill is a kill. An owner test that - * does not end within the full limit cuts its file's run off, and the - * clock-cut run records nothing, not even its earlier verdicts; a kill in - * another owner still decides (a kill outranks a timeout, v19), and failing - * one, the mutant is a `timeout`, whatever any other owner reported. The - * confirming runs of one mutant share its budget: the larger of - * `--mutant-limit` (30 s by default) and ten times its owner files' - * unmutated time (maintainer disposition G1, 2026-11-11). Its - * confirmations run in its owner files' order, and each gets at most what - * the budget has left after the seconds of its earlier confirmed runs, - * reused ones included, so the outcome is a function of the runs, not of - * their order or of reuse (law 9 checks a second daemon check of the same - * state); one that the budget cuts, or that gets nothing, is a timeout; so - * a mutant that is slow in two owners, whose confirmations together pass - * the budget, is a `timeout`, and a kill in an earlier owner still - * decides. A copy + * an error outside any test, a `setup-error`. No wall-clock limit decides + * an outcome (deterministic cuts, option A, 2026-11-14): a copy runs to its + * end, or is cut by its call count (a spin past its call budget), by its + * state (no progress: idle, with no timer due in its horizon), or by the + * backstop (`--hang-limit`, 60 s by default, plus 3 s per second of its + * file's unmutated time), which reports a gap. A spin or no-progress cut + * ends only its own copy, charged nothing, and the next owner's copy still + * runs; a backstop cut ends the mutant's later copies. A kill in another + * owner still decides (a kill outranks a timeout), and failing that the + * mutant is a `timeout`, whatever any other owner reported. The copies of + * one mutant share a work budget, counted in site evaluations and not in + * time: the larger of 100,000 and ten times their owner files' unmutated + * work (maintainer disposition G1, 2026-11-11, as a work count). Those + * copies are summed in owner order, and once the sum passes the budget, + * that copy and every later one is a timeout, so a kill among them does not + * count. Every timeout names its cause: `backstop`, `spin`, `no progress`, + * or `work budget`, the first cut in owner order (the CLI's `detail`). A + * copy * cut off for spinning (its call budget ran out, not its clock) is a * timeout as it stands: its rows are not judged, so a verdict it recorded * before the cut does not kill. (The framework's `bounded work` law, which @@ -454,8 +450,8 @@ * * Limits. Sources are one file of exported functions and classes over one * number. A hang is generated only as an owner test that waits on a promise - * that never settles, under a 1 s first budget and a 20 s full limit - * (`--first-limit`, `--hang-limit`, and a 15 s `--mutant-limit`); a slow run + * that never settles, under a 20 s full limit + * (`--hang-limit`); a slow run * that ends is only a witness, a test that waits 3 s where its output * changed and then judges, or 10 s in two one-test owners without judging. Not generated: a busy * loop, a slow run past the full limit that would still end, a hang in an @@ -517,9 +513,8 @@ * after its wait) and a stop that waits for the close are not generated. * Cut-off * charging is judged only where it changes an outcome: under one budget - * (`--first-limit` equal to `--hang-limit`), where a misplaced cut is a - * timeout at once; under two budgets a misplaced cut only costs a - * confirming run, which no law here measures. A copy that failed to load in + * (retired), where a misplaced cut is a timeout at once. A copy that failed + * to load in * a batch cut off later is run again rather than judged from its rows, so * its setup error keeps Vitest's message. Native addons only as a * `binding.gyp` beside the workspace dependency or a prebuilt `.node` file @@ -584,6 +579,7 @@ import { doneLine, push, judging, + HEAVY_CALLS, witness, resolveHistory, probeHelpers, @@ -610,7 +606,6 @@ import { sourceArgument, outsideRepository, ENV_PREFIX, - MUTANT_LIMIT_MS, HEARTBEAT_MS, FULL_LIMIT_MS, NO_LATER_STEPS, @@ -674,6 +669,7 @@ function cliArgs( ordinary = false, sources = `${SOURCE},${LIB}`, budget = `two`, + hangMs: hangMsOrNull = null, outName = daemon ? `out-daemon` : `out`, // `default` leaves `--out` out, so the CLI picks its own output // directory (args.mjs): the run's first stdout line names it. @@ -690,6 +686,8 @@ function cliArgs( // relative to there, as an author types them, and the daemon must read // them as the CLI's caller meant them. const path = (absolute) => (daemon ? relative(repo, absolute) : absolute) + // A generated history's backstop is the drawn null: the full limit. + const hangMs = hangMsOrNull ?? FULL_LIMIT_MS const args = [ join(cli, `index.mjs`), ...(daemon ? [`--daemon`] : []), @@ -706,18 +704,13 @@ function cliArgs( // in a fixed order; results do not depend on the count. `--contexts`, String(contexts), - // A fixture's tests run in milliseconds. A 1 s first budget cuts off - // the slow witnesses, and the full limit lets them end; a hang is cut - // for idleness. With one budget, a run past the full limit is a timeout - // at once. - `--first-limit`, - budget === `one` ? String(FULL_LIMIT_MS) : `1000`, + // A fixture's tests run in milliseconds. A run's copies have no first + // budget (option A): the full limit is the only time limit, and a hang + // is cut for idleness. `--first-limit` is retired; the budget `one` + // history passes it, and the CLI notes that it changes nothing. + ...(budget === `one` ? [`--first-limit`, `1000`] : []), `--hang-limit`, - String(FULL_LIMIT_MS), - // The budget that a mutant's confirming runs share, the full limit of - // one copy: one slow owner's confirmation fits it, two do not (law 4). - `--mutant-limit`, - String(MUTANT_LIMIT_MS), + String(hangMs), ] if (out !== undefined) args.push(`--out`, path(out)) if (only !== undefined) args.push(`--only`, only.join(`,`)) @@ -2387,6 +2380,16 @@ function readRunIn(dir, out, status, stderr, cli, stdout, derived) { outcome: r.outcome, class: r.class, laws: r.laws, + // The cut that ended a timeout (classify.mjs `outcomeOf`). + cause: r.outcome === `timeout` ? r.detail : undefined, + // The built-in law of a kill that a cut by count or state decided + // (bounded work or no progress), null for a behavioral law. + builtIn: + r.outcome === `killed` + ? ((r.laws ?? []).find((law) => + [`bounded work`, `no progress`].includes(law), + ) ?? null) + : undefined, probe: r.probeGuard === true, ordinaryOutcome: r.testOutcome !== undefined, })), @@ -2569,6 +2572,7 @@ async function incremental(dir, plan, cli, copy, observe, states) { ordinary: plan.ordinary, sources: sourceArgument(plan, `${dir}.link`), budget: plan.budget, + hangMs: plan.history.hangMs, ...configArgs(plan), // After the `config` edit, `--config` names the second config. ...(state.altConfig ? { config: altConfigPath(plan) } : {}), @@ -2729,13 +2733,16 @@ async function incremental(dir, plan, cli, copy, observe, states) { } if (twice && step === `start`) states.add(`two checks started the daemon at once`) - // A warm daemon's check under the short bound that cut a slow copy and - // ran its 10 s confirmation (`shortWait`), and did not fall back. + // A warm daemon's check under the short bound that ran a slow copy's 10 s + // wait (`shortWait`), and did not fall back. Under deterministic cuts no + // first budget cuts the copy: it waits in its batch, so the check that + // computed file a's runs again worked past the bound (#99, M-17; the + // state is kept, its precondition follows the runtime, v21.2 R7). if ( plan.shortWait && step !== `start` && daemon.reported && - (daemon.evidence?.cut ?? 0) > 0 && + (daemon.evidence?.computed ?? 0) > 0 && !daemon.stream.notes.some((note) => coldFallbackNote.test(note)) && plan.tests.some((t) => t.slow) ) @@ -3421,25 +3428,51 @@ async function driver(plan, observe) { // and holds each batch's count and its witness markers. const tornPolls = `${dir}.torn-polls` if (plan.tornRows) mkdirSync(tornPolls, { recursive: true }) + // A test that holds its own I/O open for a count of the sampler's samples + // (`pendingIo: 'test'`, or `'short'` before production in every run), and + // the runner's own pending write beside a test + // that can make no progress (`pendingIo: 'runner'`): each hook's directory + // sits beside the package, and the runner's marks each copy that went idle + // while its write was pending. Every run of the history sets the hooks (its + // partial run, twin and stability rerun too), since the test needs them. + const pendingIo = (owner) => + plan.history.tests.some((t) => t.pendingIo === owner) + const samples = `${dir}.samples` + const runnerWrites = `${dir}.runner-writes` + const runnerTimers = `${dir}.runner-timers` + const sampled = pendingIo(`test`) || pendingIo(`short`) + if (sampled) mkdirSync(samples, { recursive: true }) + if (pendingIo(`runner`)) mkdirSync(runnerWrites, { recursive: true }) + if (pendingIo(`runnerTimer`)) mkdirSync(runnerTimers, { recursive: true }) const how = { ordinary: plan.ordinary, sources: sourceArgument(plan, `${dir}.link`), budget: plan.budget, + hangMs: plan.history.hangMs, outMode: plan.outMode, logBlocked: plan.logBlocked, unsupervised: plan.unsupervised, psMissing: plan.psMissing, cli: plan.cliPath === `spaced` ? copy : here, ...configArgs(plan), + env: { + ...(plan.rejectBatch > 0 + ? { ORACLE_GAPS_TEST_REJECT_BATCH: String(plan.rejectBatch) } + : {}), + ...(sampled ? { ORACLE_GAPS_TEST_SAMPLES: samples } : {}), + ...(pendingIo(`runner`) + ? { ORACLE_GAPS_TEST_RUNNER_WRITE: runnerWrites } + : {}), + ...(pendingIo(`runnerTimer`) + ? { ORACLE_GAPS_TEST_RUNNER_TIMER: runnerTimers } + : {}), + }, // A rejected batch ends through the internal error's ending, the one ending a history reaches that // names its log, so its output directory's name holds a line break where the platform allows one // (not on win32): its error line and done line must escape it (the register's C entry "The // internal-error line"). ...(plan.rejectBatch > 0 - ? { - env: { ORACLE_GAPS_TEST_REJECT_BATCH: String(plan.rejectBatch) }, - outBreak: process.platform !== `win32`, - } + ? { outBreak: process.platform !== `win32` } : {}), foreign: plan.foreign, } @@ -3461,6 +3494,12 @@ async function driver(plan, observe) { : how, ) if (overlapping) observe(`overlap`, overlapping.overlap) + const idleBesideRunnerWrite = pendingIo(`runner`) + ? readdirSync(runnerWrites).length + : 0 + const idleBesideRunnerTimer = pendingIo(`runnerTimer`) + ? readdirSync(runnerTimers).length + : 0 // The watchdog read a closing row while the worker wrote it: a hold // marks `reached` once two polls came after its first half. A hold that // waited 60 s with no poll marks `unreached`, and the witness fails @@ -3528,11 +3567,199 @@ async function driver(plan, observe) { rejectBatch: plan.rejectBatch, }), ) + // The slow twin (`slowTwin`): the same package, where file a's first + // test waits 3 s wherever the output differs, and + // then judges as before. Its verdicts are the same, so its report must + // be too ("Determinism under load", 2026-11-13: an outcome does not + // depend on how long a run took). + let twinSame = false + if (plan.history.slowTwin) { + const twin = resolveHistory({ + ...plan.history, + slowTwin: undefined, + tests: plan.history.tests.map((t, i) => + i === 0 ? { ...t, slow: true } : t, + ), + }) + const slow = runCli(dir, fixtureFiles(twin), how) + twinSame = + slow.exit === cli.exit && + JSON.stringify(slow.report?.results) === + JSON.stringify(cli.report?.results) + observe(`twin`, { + fast: cli.report?.results, + slow: slow.report?.results, + exit: slow.exit, + }) + } // A failing owner's replay, run as the author would run it (law 3). if (plan.fault === `brokenOwner`) observe(`replay`, { command: replay, ran: replayed(dir, replay) }) // Reach comes from what the CLI reported, mapped onto the plan. const states = new Set() + if (twinSame) + states.add(`a slow owner's kill reported as when it did not wait`) + // A kill after a 16 s wait, and one whose copy waits on a timer, each + // killed as the report says. + const killed = cli.report?.results?.some((r) => r.outcome === `killed`) + if (killed && plan.history.tests.some((t) => t.slowMs === 16_000)) + states.add(`a slow kill after a 16 s wait killed`) + if (killed && plan.history.tests.some((t) => t.timerWait)) + states.add(`a copy waiting on a timer killed`) + if (killed && plan.history.tests.some((t) => t.timerWait?.unref)) + states.add(`a copy waiting on an unref'd timer killed`) + // A copy that read a pipe it opened, past the idle window by count, was + // kept busy by that read alone, and its kill decided. + if (killed && pendingIo(`test`)) + states.add(`a copy waiting on I/O it opened killed`) + // A copy went idle while the runner's own write was pending (its mark), + // and the mutant is killed by no progress, not a backstop gap. + if ( + idleBesideRunnerWrite > 0 && + cli.report?.results?.some( + (r) => r.outcome === `killed` && r.laws?.includes(`no progress`), + ) + ) + states.add(`a copy beside the runner's pending write cut as no progress`) + // A copy went idle while the runner's own interval ran (its mark), its + // test waiting on a ref'd timer past its horizon: no progress. + if ( + idleBesideRunnerTimer > 0 && + cli.report?.results?.some( + (r) => r.outcome === `killed` && r.laws?.includes(`no progress`), + ) + ) + states.add(`a copy beside the runner's pending timer cut as no progress`) + // An authored mutant killed by no progress (its owner hung). + if ( + plan.history.authored && + plan.history.tests.some((t) => t.hangs) && + cli.report?.results?.some( + (r) => + r.authored && + r.outcome === `killed` && + r.laws?.includes(`no progress`), + ) + ) + states.add(`an authored mutant killed by no progress`) + // A spin whose test read the clock first is a timeout by its spin. + if ( + plan.history.tests.some((t) => t.timed === `clock`) && + cli.report?.results?.some( + (r) => r.outcome === `timeout` && r.detail === `spin`, + ) + ) + states.add(`a spin whose count could follow the clock timed out`) + // Count-only kills (M, 2026-11-16) judge a test's count as a whole: a heavy + // test that reads the clock only after its work has clocked unmutated rows, + // so its copy's spin, which comes before that read, is a gap. + if ( + plan.history.tests.some((t) => t.timed === `after`) && + cli.report?.results?.some( + (r) => r.outcome === `timeout` && r.detail === `spin`, + ) + ) + states.add( + `a spin of a test that reads the clock after its work timed out`, + ) + // A spin after the test's own short async read: the read completed during + // the test, so its count could follow the clock, on every run (C, + // 2026-11-17: clocked by state, not by what a sample saw). + if ( + plan.history.tests.some((t) => t.timed === `read`) && + cli.report?.results?.some( + (r) => r.outcome === `timeout` && r.detail === `spin`, + ) + ) + states.add(`a spin after a short async read timed out`) + // An authored mutant whose owner test spins on the function it changed: + // its copy counts calls as an operator copy does, so its spin is cut by + // its call budget and kills by count (v21.2 R50; M "Count-only kills"), + // never by the backstop. + if ( + plan.history.authored && + cli.report?.results?.some( + (r) => + r.authored && + r.outcome === `killed` && + r.laws?.includes(`bounded work`), + ) + ) + states.add(`an authored mutant killed by its spin`) + // A heavy test that reads the clock first: its copies' call budget is a + // hundred times its unmutated work (G6), so a copy past CALL_FLOOR and + // inside that budget runs to its end and is killed by its own law. + if ( + plan.history.heavy > 0 && + plan.history.tests.some((t) => t.timed === `clock`) && + cli.report?.results?.some( + (r) => r.outcome === `killed` && !r.laws?.includes(`bounded work`), + ) + ) + states.add( + `a heavy clocked test's copy past the call floor killed by its law`, + ) + // Two owners that each make 30,000 extra calls, the first of them clocked: + // only the second's count is charged, which stays inside the work budget, + // so the mutant is judged by its owners' laws, and neither judges. + if ( + plan.history.tests.length === 2 && + plan.history.tests.every((t) => t.loops === 30_000) && + plan.history.tests.some((t) => t.timed === `clock`) && + cli.report?.results?.some((r) => r.class === `not-judged`) + ) + states.add(`a clocked owner's work left out of the work budget`) + // Owner a's witness copy of w_hangkill hangs and is cut as no progress, and + // the mutant is killed by that cut before owner b's copy runs. Owner b's + // verdict test spins on the function where its file is read (`spinData`, + // a read that is not `none`), so that spin is the later owner's, which the + // sorted-owner order (G44) runs after a. The detector reads the history's + // fields and the CLI's result, not the position of the tests in the listing. + if ( + plan.history.spinData && + plan.history.fsRead !== `none` && + cli.report?.results?.some( + (r) => + r.function === `w_hangkill` && + r.outcome === `killed` && + r.laws?.includes(`no progress`), + ) + ) + states.add(`a copy cut as no progress before a later owner's spin`) + // A file whose test does a short read of its own before production, in + // every run: its copy that hangs is still idle by state, and is killed by + // no progress (v21.2 R50, R51). + if ( + pendingIo(`short`) && + cli.report?.results?.some( + (r) => r.outcome === `killed` && r.laws?.includes(`no progress`), + ) + ) + states.add( + `a hang in a file whose test did a short read killed by no progress`, + ) + // A copy that waits past the history's backstop is cut off, and no later + // owner runs, so the mutant is a timeout, not the later owner's kill. + if ( + plan.history.hangMs && + cli.report?.results?.some((r) => r.outcome === `timeout`) + ) + states.add(`a backstop cut ends the copies ahead of a later kill`) + // Two owners that each make 30,000 extra calls: the second owner's copy + // passes the work budget, so its own kill does not count, and it fails + // bounded work instead (M, 2026-11-16). + if ( + plan.history.tests.length === 2 && + plan.history.tests.every((t) => t.loops === 30_000) && + cli.report?.results?.some( + (r) => + r.outcome === `killed` && + JSON.stringify(r.laws) === JSON.stringify([`bounded work`]), + ) + ) + states.add( + `a kill past the work budget in a later owner is a bounded-work kill`, + ) if ( plan.rejectBatch > 0 && retention.some( @@ -3566,6 +3793,14 @@ async function driver(plan, observe) { await unwritableLogRuns(dir, files, how, plan.logLost), ) states.add(lostStateOf[plan.logLost]) + // The unsupervised run writes its own stopped line (stopNow) into a log + // that can no longer take it: every win32 run, and the constructed + // unsupervised history on POSIX. + if ( + plan.logLost === `stop` && + (plan.unsupervised || process.platform === `win32`) + ) + states.add(`an unsupervised run stopped after its stream log was lost`) } if (plan.outMode === `default`) states.add(`a run with the default output directory`) @@ -4124,15 +4359,19 @@ async function driver(plan, observe) { if (m.function === `w_iowait` && m.outcome === `killed`) states.add(`a copy waiting on I/O killed, not cut for idleness`) if (m.function === `w_slowkill` && m.outcome === `killed`) - states.add(`a slow kill past the first time limit killed`) - if (m.function === `w_hang` && m.outcome === `timeout`) - states.add(`a hang alone timed out`) + states.add(`a slow kill killed`) + if (m.function === `w_hang` && m.laws?.includes(`no progress`)) + states.add(`a hang alone killed by no progress`) if (m.function === `w_hangkill` && m.outcome === `killed`) states.add(`a hang in one owner and a kill in another killed`) - if (m.function === `w_hangsurvive` && m.outcome === `timeout`) - states.add(`a hang in one owner and a survivor in another timed out`) - if (m.function === `w_slowpair` && m.outcome === `timeout`) - states.add(`confirmations past a mutant's budget timed out`) + if (m.function === `w_hangsurvive` && m.laws?.includes(`no progress`)) + states.add( + `a hang in one owner and a survivor in another killed by no progress`, + ) + if (m.function === `w_slowpair` && m.laws?.includes(`bounded work`)) + states.add( + `a mutant's copies past its work budget killed by bounded work`, + ) const named = { killed: `killed`, 'random-only-kill': `random-only kill`, @@ -4282,11 +4521,14 @@ async function driver(plan, observe) { } if (cli.mutants.some((m) => m.outcome === `setup-error`)) { states.add(`a setup error`) + // Beside a hang in the same batch, which is a kill by no progress now + // (M, 2026-11-16), so a watchdog that charged the hang to the copy that + // failed to load would still show. if ( plan.budget === `one` && - cli.mutants.some((m) => m.outcome === `timeout`) + cli.mutants.some((m) => m.laws?.includes(`no progress`)) ) - states.add(`a setup error beside a timeout under one budget`) + states.add(`a setup error beside a hang under one budget`) } if (plan.ordinary && cli.mutants.some((m) => m.ordinaryOutcome)) states.add(`ordinary tests run when asked`) @@ -4327,6 +4569,9 @@ async function driver(plan, observe) { for (const state of states) observe(`reach`, state) } finally { if (copy) rmSync(copy, { recursive: true, force: true }) + rmSync(`${dir}.samples`, { recursive: true, force: true }) + rmSync(`${dir}.runner-writes`, { recursive: true, force: true }) + rmSync(`${dir}.runner-timers`, { recursive: true, force: true }) removePath(`${dir}.link`) written.delete(dir) dropHook(dir) @@ -4521,6 +4766,23 @@ function daemonCheck(observations) { function check(plan, observations, expected) { const cli = observations.find((o) => o.checkpoint === `cli`)?.value if (!cli) return violation(`cli`, `no CLI run was observed`, `completion`) + // A run's outcome does not depend on how long its runs took: the slow + // twin reports each mutant as the package does (determinism under load). + const twin = observations.find((o) => o.checkpoint === `twin`)?.value + if (twin) { + const at = (twin.fast ?? []).findIndex( + (r, i) => JSON.stringify(r) !== JSON.stringify(twin.slow?.[i]), + ) + if (twin.exit !== cli.exit || at >= 0) + return violation( + `twin`, + `the slow twin reports ${at >= 0 ? twin.fast[at].key : `exit ${twin.exit}`} otherwise than the package whose test does not wait`, + `determinism under load`, + at >= 0 + ? { actual: twin.slow?.[at], expected: twin.fast[at] } + : undefined, + ) + } // An internal error comes first: it also leaves the session directory, so // the namespace law below would name only its symptom (issue #83). The // message carries the run's error.log (readRun). @@ -4867,6 +5129,23 @@ function check(plan, observations, expected) { : `classification`, { actual: got, expected: owed }, ) + // A timeout names the cut that ended it (its `detail`). + if (kind === `timeout` && got.cause !== owed.cause) + return violation( + `cli`, + `${site(got)}: its timeout is cut by ${got.cause}, owed ${owed.cause}`, + `classification`, + { actual: got, expected: owed }, + ) + // A kill names the built-in law that decided it, or none for a behavioral + // law (M, 2026-11-16, "Spin, no progress and the work budget are kills"). + if (kind === `killed` && got.builtIn !== owed.builtIn) + return violation( + `cli`, + `${site(got)}: killed by ${got.builtIn ?? `a behavioral law`}, owed ${owed.builtIn ?? `a behavioral law`}`, + `classification`, + { actual: got, expected: owed }, + ) } // Each gap is streamed once as its mutant is judged (stream.mjs), and the // streamed gaps are the report's gaps, each once. @@ -4913,6 +5192,13 @@ function check(plan, observations, expected) { `authored mutants`, { actual: kind, expected: owed.kind }, ) + if (kind === `timeout` && got.cause !== owed.cause) + return violation( + `cli`, + `the authored mutant ${owed.id} is cut by ${got.cause}, owed ${owed.cause}`, + `authored mutants`, + { actual: got.cause, expected: owed.cause }, + ) if (owed.line !== undefined && got.line !== owed.line) return violation( `cli`, @@ -5193,7 +5479,9 @@ const logNoteOf = (run) => { * `, and it shows the gap's kind (its class, or its outcome), its * line (`?` when the mutant never applied), and its change: two code spans * around ` → `, the code and its replacement, or the intent's span and - * ` (authored by )`. A span shows its code on one line, with each + * ` (authored by )`. A timeout's line then names its cause, the cut + * that ended it, as ` (cause: )`, which is results.json's `detail`, + * and no other line names one. A span shows its code on one line, with each * run of whitespace as one space, whole or cut and ended with `...`; the * cut's length is the printer's choice. The comparison is by id, so a run * that prints one gap twice and drops another breaks it, though its count @@ -5228,6 +5516,7 @@ function stdoutGapBreach(stdoutGaps, results) { const owed = { kind: gap.class ?? gap.outcome, line: gap.line === undefined ? `?` : String(gap.line), + cause: gap.outcome === `timeout` ? gap.detail : undefined, } const shows = gap.authored ? read.intent !== undefined && @@ -5236,9 +5525,14 @@ function stdoutGapBreach(stdoutGaps, results) { : read.original !== undefined && showsCode(read.original, gap.original) && showsCode(read.replacement, gap.replacement) - if (read.kind !== owed.kind || read.line !== owed.line || !shows) + if ( + read.kind !== owed.kind || + read.line !== owed.line || + read.cause !== owed.cause || + !shows + ) return { - difference: `the stdout's gap line for ${read.id} is ${JSON.stringify(line)}, and results.json reports it as ${owed.kind} at line ${owed.line}, changing ${JSON.stringify(gap.authored ? gap.intent : `${gap.original} → ${gap.replacement}`)}`, + difference: `the stdout's gap line for ${read.id} is ${JSON.stringify(line)}, and results.json reports it as ${owed.kind}${owed.cause === undefined ? `` : ` (cause: ${owed.cause})`} at line ${owed.line}, changing ${JSON.stringify(gap.authored ? gap.intent : `${gap.original} → ${gap.replacement}`)}`, extra: { actual: line, expected: { ...owed, id: read.id } }, } } @@ -5253,7 +5547,8 @@ function stdoutGapBreach(stdoutGaps, results) { /** * A gap line's fields, read from its text as an agent reads it: its kind, - * line, id, and change. Undefined when the line does not read so. + * line, id, change, and a timeout's cause (` (cause: )` after the + * change). Undefined when the line does not read so. */ function readGapLine(line) { const head = line.match(/^oracle-gaps: gap ([\w-]+) L(\d+|\?) (\S+) (.*)$/) @@ -5261,18 +5556,31 @@ function readGapLine(line) { const [, kind, at, id, change] = head const first = readCodeSpan(change) if (!first) return undefined - const authored = first.after.match(/^ \(authored by (.*)\)$/) + // What follows the change's last part: nothing, or the cause of a timeout. + const ending = (after) => after.match(/^(.*?)(?: \(cause: ([^()]+)\))?$/) + const [, afterIntent, intentCause] = ending(first.after) + const authored = afterIntent.match(/^ \(authored by (.*)\)$/) if (authored) - return { kind, line: at, id, intent: first.code, author: authored[1] } + return { + kind, + line: at, + id, + intent: first.code, + author: authored[1], + cause: intentCause, + } if (!first.after.startsWith(` → `)) return undefined const second = readCodeSpan(first.after.slice(` → `.length)) - if (!second || second.after !== ``) return undefined + if (!second) return undefined + const [, rest, cause] = ending(second.after) + if (rest !== ``) return undefined return { kind, line: at, id, original: first.code, replacement: second.code, + cause, } } @@ -5952,6 +6260,35 @@ calibrations.push( return { ...cli, stdoutGaps: [renamed, ...rest] } }), }, + { + name: `a stdout gap line names another cause`, + law: `greppable stream`, + // A timeout is now only a backstop or a spin that could follow the clock, + // which no generated history is likely to print, so the calibration runs + // the clocked spin's history, whose timeout line names `spin`. + history: { constructed: `a clocked copy that spins` }, + // A timeout's line whose cause is another cut than results.json's + // `detail`: the line still reads, so only the cause comparison rejects it. + corrupt: corruptCli((cli) => { + // A run that did not complete (a rejected batch) records no gap lines. + if (cli.exit !== 0 || cli.stdoutGaps === undefined) return undefined + const at = cli.stdoutGaps.findIndex((line) => + / \(cause: [^()]+\)$/.test(line), + ) + if (at < 0) return undefined + const recaused = cli.stdoutGaps[at].replace( + / \(cause: ([^()]+)\)$/, + (_, cause) => + cause === `backstop` ? ` (cause: no progress)` : ` (cause: backstop)`, + ) + return { + ...cli, + stdoutGaps: cli.stdoutGaps.map((line, i) => + i === at ? recaused : line, + ), + } + }), + }, { name: `a stdout gap line does not read as a gap`, law: `greppable stream`, @@ -6314,6 +6651,7 @@ const required = [ `a daemon check with the default output directory`, `a start refuses a dangling claim`, `an unsupervised run stopped after its note`, + `an unsupervised run stopped after its stream log was lost`, ...posixStarts, ...posixPs, `killed by a second owner`, @@ -6371,7 +6709,7 @@ const required = [ `timeout`, `a kill after a nondeterministic verdict in its file`, `a kill in one owner after a gap in another`, - `a slow kill past the first time limit killed`, + `a slow kill killed`, `a mutant that changes a value only at load killed`, `a mutant that changes no value judged without a run`, `a boolean flip killed`, @@ -6387,10 +6725,28 @@ const required = [ `a shrinking calibration passes under a mutant`, `a workspace dependency's state judged fresh`, `each owner's value change killed its own mutant`, - `a hang alone timed out`, + `a hang alone killed by no progress`, `a hang in one owner and a kill in another killed`, - `a hang in one owner and a survivor in another timed out`, - `confirmations past a mutant's budget timed out`, + `a slow owner's kill reported as when it did not wait`, + `a slow kill after a 16 s wait killed`, + `a copy waiting on a timer killed`, + `a copy waiting on an unref'd timer killed`, + `a kill past the work budget in a later owner is a bounded-work kill`, + `a backstop cut ends the copies ahead of a later kill`, + `a copy waiting on I/O it opened killed`, + `a copy beside the runner's pending write cut as no progress`, + `a copy beside the runner's pending timer cut as no progress`, + `a spin whose count could follow the clock timed out`, + `an authored mutant killed by no progress`, + `a spin of a test that reads the clock after its work timed out`, + `a heavy clocked test's copy past the call floor killed by its law`, + `a clocked owner's work left out of the work budget`, + `a hang in a file whose test did a short read killed by no progress`, + `a spin after a short async read timed out`, + `an authored mutant killed by its spin`, + `a copy cut as no progress before a later owner's spin`, + `a hang in one owner and a survivor in another killed by no progress`, + `a mutant's copies past its work budget killed by bounded work`, `a heartbeat named a busy mutant`, `a run stopped by SIGINT while a worker spun`, `a stop signal after the completed line`, @@ -6427,7 +6783,7 @@ const required = [ `an uncalled function in an owned function is a gap`, `the CLI ran from a path with a space and #`, `a setup error`, - `a setup error beside a timeout under one budget`, + `a setup error beside a hang under one budget`, `refused projects`, `refused outsideSource`, `a refused run whose package lies inside the CLI's directory`, @@ -6507,6 +6863,7 @@ const faults = [ // history: they are not witnessed by "every outcome class" (a named history // of its own must enter them, or registration fails). const ownHistoryStates = [ + `a copy cut as no progress before a later owner's spin`, `a run after a killed CLI that left its own session`, `two starts race for a stale claim`, `a start refuses a stale claim`, @@ -6523,6 +6880,7 @@ const ownHistoryStates = [ `a daemon check with the default output directory`, `a start refuses a dangling claim`, `an unsupervised run stopped after its note`, + `an unsupervised run stopped after its stream log was lost`, ...posixStarts, ...posixPs, ] @@ -6534,7 +6892,7 @@ const separate = [ `two authored mutants judged apart`, `a package with a prebuilt addon deep in its tree ran in child processes`, `a setup error`, - `a setup error beside a timeout under one budget`, + `a setup error beside a hang under one budget`, `stale authored mutant`, `ambiguous authored mutant`, `a config refusal through the daemon`, @@ -6555,6 +6913,29 @@ const separate = [ `a session failure larger than a socket buffer`, `a config fault in a run that owns nothing`, `a refused run whose package lies inside the CLI's directory`, + // The deterministic-cuts states have constructed histories of their own, so + // no history that names every state (`every outcome class`) must enter them. + // A timeout is now only a backstop cut (M, 2026-11-16: spin, no progress and + // the work budget are kills), which the witness module has no copy for; the + // backstop history enters it. + `timeout`, + `a slow owner's kill reported as when it did not wait`, + `a slow kill after a 16 s wait killed`, + `a copy waiting on a timer killed`, + `a copy waiting on an unref'd timer killed`, + `a kill past the work budget in a later owner is a bounded-work kill`, + `a backstop cut ends the copies ahead of a later kill`, + `a copy waiting on I/O it opened killed`, + `a copy beside the runner's pending write cut as no progress`, + `a copy beside the runner's pending timer cut as no progress`, + `a spin whose count could follow the clock timed out`, + `an authored mutant killed by no progress`, + `a spin of a test that reads the clock after its work timed out`, + `a heavy clocked test's copy past the call floor killed by its law`, + `a clocked owner's work left out of the work budget`, + `a hang in a file whose test did a short read killed by no progress`, + `a spin after a short async read timed out`, + `an authored mutant killed by its spin`, // The config and contexts states have constructed histories of their own. `a config that --config names ran`, ...Object.values(testRootStates), @@ -6566,7 +6947,7 @@ const separate = [ `an awaiting config in a package with no module type ran`, `a .cjs or .cts config that calls require ran`, `a .js or .ts config in a package with no module type that calls require ran`, - `confirmations past a mutant's budget timed out`, + `a mutant's copies past its work budget killed by bounded work`, `a second daemon check of the budget witness's state`, `a run with eleven contexts`, `a failing owner replayed under a config that --config names`, @@ -6774,12 +7155,46 @@ const witnessEditSequence = (witness) => (generated) => ({ steps: witnessSteps, }) +// A constructed history whose only owned function is a kill, owned by file a +// alone, with the tests given (the rest of the generated base is left as the +// grammar draws it, as the histories above do). +const soloKill = (generated, tests) => ({ + ...generated, + functions: [[{ kind: `if`, op: `>`, k: 4, not: false, quote: `'` }]], + containers: [], + overloads: [], + owners: [`a`], + library: [], + libraryOwners: [], + runes: [], + derived: null, + dotGen: false, + dataOwners: [], + exportOwners: [], + importer: `none`, + wholeFile: `none`, + template: false, + tests, + witness: false, + authored: null, + secondAuthored: false, + fault: `none`, + stability: `none`, + ordinary: false, + native: `none`, + boot: false, + budget: `two`, + cliPath: `plain`, + concurrentStart: false, + contexts: 1, + steps: null, +}) oracleProperty({ id: `oracle-gaps.report`, law: { - statement: `The gap report enumerates the promised operator mutants, leaves out exactly the probe-only code, discovers every oracle that owns a function's call tree and refuses broken ownership, judges each mutant only by its owners' tests on their registered routes and counts only law verdicts on reproducible routes as kills, which outrank a run that never ends, anchors each mutant by its function and normalized statement, lists every gap and unowned mutant once with consistent counts, keeps keys stable under unrelated edits, reports through one daemon per package what a cold run reports at every state of a sequence of edits, and bounds the runs that confirm a mutant's cut-off copies by one budget per mutant and names in each heartbeat a mutant that is running, in a file that runs it, leaves nothing in the package and writes nothing where Vitest's include could find it while it runs, leaves no process running once it is stopped, runs the package as Vitest does from any install path and any --source spelling: one instance of each workspace dependency, no refusal over an unbuildable devDependency, child processes for a native addon anywhere in the tree, a capture that a test's builtin spies cannot see, and a refusal of test.projects; leads its report with every test file's baseline, blocks only the mutants a failing oracle file owns and judges the rest, prints every line of its stream with the prefix oracle-gaps: and its kind, one decisive baseline line before any mutant, and a done line last, and every run that does not complete ends, cold or through the daemon, with a refusal that exits 2, prints a refused: block and one fix: directive with no stack trace, and writes the refusal to the run's output directory.`, + statement: `The gap report enumerates the promised operator mutants, leaves out exactly the probe-only code, discovers every oracle that owns a function's call tree and refuses broken ownership, judges each mutant only by its owners' tests on their registered routes and counts only law verdicts on reproducible routes as kills, which outrank a run that never ends, anchors each mutant by its function and normalized statement, lists every gap and unowned mutant once with consistent counts, keeps keys stable under unrelated edits, reports through one daemon per package what a cold run reports at every state of a sequence of edits, and bounds a mutant's copies by one work budget per mutant (site evaluations, never time) and names in each heartbeat a mutant that is running, in a file that runs it, leaves nothing in the package and writes nothing where Vitest's include could find it while it runs, leaves no process running once it is stopped, runs the package as Vitest does from any install path and any --source spelling: one instance of each workspace dependency, no refusal over an unbuildable devDependency, child processes for a native addon anywhere in the tree, a capture that a test's builtin spies cannot see, and a refusal of test.projects; leads its report with every test file's baseline, blocks only the mutants a failing oracle file owns and judges the rest, prints every line of its stream with the prefix oracle-gaps: and its kind, one decisive baseline line before any mutant, and a done line last, and every run that does not complete ends, cold or through the daemon, with a refusal that exits 2, prints a refused: block and one fix: directive with no stack trace, and writes the refusal to the run's output directory.`, authority: `The maintainer's decisions in design runs v11, v14, and v19 (2026-11-04 and 2026-11-05), not in this package's records, including that a kill by reach agreement alone is a gap and that an incremental run through the daemon equals a cold one (v19.3); the project's oracle guide on quiet exclusions; the oracle rubric's outcome kinds (ORC-006); the CLI's documented operator list in schemata.mjs; the maintainer's dispositions G1, G2, G4, G5, G6, and G13 of the framework papercut ledger (2026-11-11) and the v20 freeze's namespacing, one-loop, and provisional-report decisions (2026-11-10); the maintainer's request for a greppable stream (2026-11-12, relayed by the coordinator with its decisions of that day: the same prefixes on stdout and stderr, a done line for a stopped run, and no --max-gaps).`, - limits: `Law 12 judges refusals. The one internal error a history reaches is the constructed rejected-batch history (rejectBatch, a test-only fault that rejects a batch's run after its copies are in Vitest's and Vite's state). Its run is judged in this order: the output namespace, process warnings, and config lines, as for any run; then bounded retention, which owes the rejected batch's own line with its state after retirement; then the stream law (law 15) on both streams, as for any run, so its error line and its done line carry the prefix, and no other line does not, which the state \`a run ended in an internal error\` declares; then exit 3 with an internal error's done line last on both streams, no stack frame on stderr, and an error.log in its output directory. What the error.log's stack says is not judged. An internal error that no fault constructs is a completion failure, reported before the stream law; an internal error through the daemon is reached by no history, and its ending is the same function's (endRun), by inspection. The internal error's lines follow the register's C entry "The internal-error line" (2026-11-16): the general law (endsBreach) owes every run that ends through endRun one error line on stderr naming the done line's log, the output directory's error.log, and none on stdout, and owes the supervisor's own exit 3 none. The message's escape is witnessed by the fault's message, which holds a line break and a forged done line after it; the log path's escape only on POSIX, where the rejected batch's output directory holds a line break (a name on win32 cannot). Paths in a completed run's lines (the results note, the done line's report, the test-output note) are escaped the same way, by inspection: no completed history names such a directory. P10's exempt lines (v21.2 P10 and I3) are read by their own shapes (exemptBlocks), never by indentation alone: a baseline failure's indented details, a refusal's block, Node's warning with its \`--trace-warnings\` frames and its hint, V8's fatal block, esbuild's message block, and Vite's own \`[vite]\` line. The V8, traced-warning, and esbuild blocks are calibrated beside our own line with no prefix (the stream calibrations put them before the rejected-batch run's done line, and the law must name our line alone); no generated history prints any of them or a Vite line, and a shape of those tools that exemptBlocks does not name is read as a stray. The stdout summary's exemption is unused and not exercised: the CLI prefixes every summary line (its summary, legend, and function kinds). \`--help\` prints its usage text with no prefix and no done line, exempt by the register's M entry "\`--help\` is exempt" (2026-11-16), and the static oracle's Help law owes exactly that text; \`--stop-daemon\` prints nothing and exits 0, read the same way as a command outside the run protocol (C, pending ratification), and this oracle runs it without observing its streams. A syntax error in a module that the config imports lets the esbuild that Vite bundles configs with print its block before the refusal (trace, Vite 7.3.7, esbuild 0.28.2: \`✘ [ERROR] Unexpected end of file\`, an empty line, \` helper.ts:2:0:\`, the code frame, then the refused block): its lines are exempt by the register's M entry "Other tools' output is exempt" (2026-11-16), the refusal and the done line around them keep the prefix, and no history generates it. Bounded retention reads the hook's lines (retention-hook.mjs). Its count of one line per batch (runner law L2) rests on the index the hook takes when a batch starts, so a CLI that skipped a batch's index and its line together, or that omitted only the line of a check's last batch, keeps the indices 0..n-1 and is not seen; a check that ended before its report (the rejected batch, a stopped run) and a daemon the driver killed are judged for at most one line per index, since the batches they were running at their end owe none. A line's process is told by check.pid, which the check writes, and the driver itself knows only the daemon it killed. The cold store's law (held) counts the entries given to the store's put that a full collection leaves reachable, so a store that kept only a part of an entry (its unit without the entry) is not seen; staleOpen reads Vitest's own close state (closingPromise, set when close starts), so a close that starts and never resolves is the runner oracle's (L8); and a context replaced after a check's last batch is observed only by a later check's first batch through the daemon. The run whose process aborts is declared on every platform. Its win32 witness (process.abort through ORACLE_GAPS_TEST_ABORT_AT, under the win32 supervisor) lands in #79; until then the Windows end-to-end job is non-blocking (C entry (c)). On win32 Node cannot deliver SIGTERM (its kill is TerminateProcess, which ends the process with no stopped line), so the stop states that name SIGTERM are required on POSIX only, and a stop there goes through the CLI's test stop hook and is observed as SIGINT, which owes its done line as any stop does (no history sends a win32 child.kill, the SIGKILL-equivalent that owes none); every SIGINT state stays required on win32. A stop as the CLI starts lands only in one of two holds: the driver's own preload before the CLI's first statement (outcome a, nothing written; POSIX only, since its witness is a signal's default action), or the CLI's hold after its started marker, which the stop the CLI keeps releases (outcome b, judged only when the supervisor started no child); a stop during Node's own start between the two is not generated, since its outcome follows timing. The drawn spawn delay is a wait inside a hold, so it cannot move a stop out of it. Stop runs that land while the config warms (ORACLE_GAPS_TEST_WARM_PROBE) are still drawn, and on win32 every drawn stop goes through ORACLE_GAPS_TEST_STOP: each is a test-only hook that a drawn field switches, main's pattern, which the review of #57 (A3) brought to the maintainer; only the stop after the completed line (ORACLE_GAPS_TEST_HOLD_UNTIL) was made constructed-only. A missing \`ps\` is generated only as a PATH that names no command, on POSIX, where the run is supervised. The unsupervised path (stopNow, every win32 stop) is reached on POSIX by one constructed history whose stop is held at the run's note, before the check starts a process: an unsupervised POSIX run has no group to end, so a stop while the witness's test spins leaves that test's own child (measured; C 2026-11-13, Windows (e)), which win32 ends with the job object libuv puts its children in. Argument refusals are the static oracle's; the check's rejection of a stack trace, a missing fix, and a wrong exit code is calibrated in the static oracle, through the same refusalBreach, and here only a refused run reported as complete is calibrated, because each calibration here runs real CLI histories. Law 15's stdout gap lines are judged against each completed run's own results.json, by id: the cold run, and each check through the daemon (every step's check and the check that a mid-run edit overlaps); a stopped run and a refusal list no gaps and are not judged so. A line's change is read loosely: its code on one line, whole or cut after a kept part and ended with \`...\`; where the printer cuts (60 code points, kept 57) is the classify oracle's law 8 (\`shortened\`), in-process. A line with any text after its change does not read and fails loudly: the \` (cause: )\` suffix that #99 adds to timeout lines is not read yet, so whichever of #99 and #115 lands second extends readGapLine. Hangs only as an owner test that awaits a promise that never settles, under a 1 s first budget and a 20 s full limit, and a slow run only as two witnesses whose tests wait where the output changed, 3 s in one that then judges and 10 s in two one-test owners of one that does not; busy loops only on instrumented code (cut by the call budget), no slow runs past the full limit that would end, hangs in ordinary or nondeterministic tests, setup errors other than a throw at module load, loops, multi-line conditions, two functions on one line, or await, new, and void statements; nested functions, object-literal methods, accessors, and class-field arrows only where no test calls them (their ownership is the static oracle's); two source files of one package, with no re-exports, namespace imports, or calls into another package; --only only in a stopped run and a partial run of one function's mutants, and no --sample or --diff-base; the baseline's failures only as one failing owner test that throws an error and a setup file that throws, so a baseline law violation's law and counterexample lines are the classify oracle's; a derived class's file is instrumented and its sites are judged as unowned, but no test imports or constructs the class, so its no-mutant and super-region behavior is the static oracle's (in-process), not this oracle's; package configs only as vite.config or vitest.config with one of the extensions .ts, .mts, .cts, .js, .mjs, and .cjs (not every pair), as config/gap-vitest with the same extension through --config, or none, as an object or a sync or async function, with no plugins of their own except one enforce: 'pre' transform that appends a comment to each module of the package (prePlugin) and a plugin whose load reads each test file with a sync read (loadPlugin), drawn for a config in ES module syntax and reached by its own constructed history too; a plugin whose output depends on anything but the module's text (a sidecar file it reads) and an environment variable that changes between checks are not generated, since the runner tracks neither yet and the daemon's environment is fixed when it starts (F-A1: the maintainer decided on 2026-11-13 that the daemon tracks both and restarts when they change, in the next pull request); and test.projects only as a refusal, and only under the package's own package.json; the supervisor-death state is required and entered only off win32: win32 has no supervisor until #79, so there the state is not reached and its history is not judged (platform-scoped reach; the win32 run is unsupervised, which register C entry (e) of #58 records as not guaranteeing that no process outlives the CLI); a config's test.root only in four shapes, drawn for an ES module config that Vitest discovers in the package (the randomness-wins decision, 2026-11-16) and each reached by its own constructed history of one owner and one context, where the test files' globs and the setup file are relative to the root (issue #101): the directory that holds the tests, the directory above the package, a directory of the package that holds no test file, and the package's tests as a symbolic link to the directory that holds them (required on every platform, since Windows creates the link as the driver's other links); not under --config (nor in a history whose config edit names a second config), a CommonJS config, a parent's config, test.projects, several contexts with different roots, or a root that is the filesystem's root; the runner serves each copy of a test file by its absolute path, through its pre plugin's load (which answers before the config's own plugins) and through its read of the copy's path for Vitest's pragma read (serveCopies), and histories run copies only in Vite's server environments (node and the pragma's ssr environment): jsdom and happy-dom, Vite's client environment, are not generated (the #103 merge-gate review ran jsdom by hand under all five root shapes, and the copies loaded in each); a top-level await only in an ES module config and require only in a CommonJS one, as Vitest itself loads no other; a --config outside the package is refused by design, through the rule for paths outside the package (args.mjs), and no history generates one; contexts only one, four, or eleven, and process warnings judged only as MaxListenersExceededWarning on the CLI's own stderr (a daemon's process warnings go to its log); a context that fails to open gives back its listener allowance, which no observation shows (a kept allowance only raises the limit, so no warning follows), so that accounting is fixed by inspection; no probe-bound states, so reach agreement's killing of probe mutants is not checked; no gap answers or gate; law 9 judges the witness module's outcome classes along one constructed edit sequence only (a test edit, a helper edit, and a revert), which runs as two constructed histories, one for each half of the module (captureA and captureB, whose union is the module and which share no function; a function stays with the functions its tests or statements read), since the whole module along it took one CI job (#106, #145; maintainer decision "A one-history job over budget is cut by cost", 2026-11-16), so an interaction between functions of different halves, beyond the ones the split keeps together, is not judged along the sequence (the whole module runs together in \`every outcome class\` and in generated histories, which take no steps); and the budget witness's outcome classes along one constructed second check with nothing changed, because a generated history with the witness module or the budget witness takes no steps (maintainer decision, 2026-11-11, and the runtime review's P1-a, 2026-11-12; this oracle owns the gap); the state "a timed-out run reused after an edit" counts only runs cut by their call budget (the spin, in the first half), a count the CLI records, by the maintainer's decision "#104 state narrowed, #99 restores" (2026-11-17), which narrows "#90 job stays blocking" for that state only: the reuse of a run cut by the clock (a hang) is judged by law 9 but not required, because on main a hang's timeout is stored only when the sampler saw the copy idle, which load decides, and otherwise its confirmation is cut short by its mutant's budget and not kept (#104); #99 makes a hang a state-based no-progress kill and restores the reach as "a run killed for no progress, reused after an edit" (#156); law 9's edits come between checks, except one, midrun, which lands during a check through the daemon: a test of file a, whose context has loaded file b's helper, rewrites the helper in the baseline's first run of file a, before file b's runs read it, in one context or several; or a test of file b rewrites its own helper, or the data file that its verdict read through a sync fs read, after file b's judged tests read them, in the run that judges with the old text (review of #55, F1); that check must report, for each mutant, what a cold run of the state before the edit or of the state after it reports, and exit as one of them does (third review of #55, F-C), compared per mutant result rather than per file; when file b's own test lands the edit, b's baseline reads the old text and b's mutant runs read the new one, so b's own mutants in that check are not judged, since their results legitimately mix the two texts (maintainer decision, 2026-11-13), while every other mutant there is judged against both cold runs, with the check's exit code, its mutants' keys, and its stream; the stronger follow-up compares each run's raw outcome with the cold outcome of the state that run read; and the next check, with nothing changed, must equal a cold run of the state after the edit, compared for equality of results only, not for what the daemon reused or computed (the runner's rule: a run whose input changed after the run read it is invalidated and not stored, settleIn); a check through the daemon of a state that nothing changed must compute no run that was not cut off, judged where no test reads the clock (F-A, F-A2), and a test's fs read of a file and the module that the context serves from it are two inputs, each under its own hash (F-A2); an edit during a mutant run is not generated, and a file is edited mid-run only by the run's own tests, not by another process at an arbitrary time; a file that a test reads through fs is edited only as one JSON file that file b's verdict reads by a sync, a promise, or a callback read, or by a sync read after it stats its own directory and tests that it exists (statDir: the directory is an input as a directory, not by its content, and is not itself edited), each read first checking that its fs function keeps its own name and length (the runner stores a run under the files in the package that its copy reads with readFileSync, readFile, existsSync, or statSync, or the fs/promises readFile or stat, each with its hash at the copy's first read; a read outside the package or under node_modules, from a worker thread or a child process that a test starts, or through any other fs API, which includes readdir, opendir, glob, globSync, createReadStream, realpath, lstat, access, open, a read through a file descriptor (read, readSync, readv, fstat, fstatSync, and readFileSync or readFile given a file descriptor), copyFile and copyFileSync, cp and cpSync, openAsBlob, statfs, readlink, the callback stat, exists, watch, watchFile, and the fs/promises access, open, and glob, is no input, and none is generated: the maintainer decided on 2026-11-13 that every fs read except watch and watchFile is an input, which a follow-up of #55 implements, and until then this oracle owns the gap); a run cut off by its clock writes no closing row, so it is stored under its file's known inputs only, while a run that spun ends and keeps the inputs it named (F-D), in a batch as alone; a spun copy that reads a file that its file's unmutated run never reads is generated (spinData: file b's verdict tests that read the data spin on the function they test while it is open), and the data's edit is judged by law 9 for copies that spin in a batch; a copy that spins alone (runAlone, after a batch with an error outside any test) is generated too (spinThrows: each spun copy schedules an error that fires after its test), and its data's edit is judged the same way; an error outside any test only from a copy that spins, so the alone path's other units (a slow copy's confirmation) stay unwitnessed; a test that rewrites a file it read with the same text makes the next check compute that run again (1 run in the self-edit histories), since the edit changes the file's modification time: the daemon is conservative there, nothing goes stale, and content keys (D10) remove it, with a selfWrite dimension in that pull request; the mid-run marker sits in the daemon's session directory, .oracle-gaps-, whose location predates #55; the runner's hash memo knows a file by its size and modification time, so an edit that keeps both is not seen, and no history makes one (D10, planned in phase 2, A5); law 9's edits leave out other environment variables, new files that change how an unchanged import resolves, Vite config and installed-package changes, and concurrent checks other than two that start the daemon; cut-off charging judged only under one budget; native addons only as a binding.gyp or a prebuilt .node file, not as optional platform packages; a scratch package lies in the CLI's directory (.scratch-*), in a directory beside it whose name extends the CLI directory's name (siblingDir, drawn), or in the OS temp directory, so the driver's inside-the-CLI label is judged against a prefix that is not a parent; a CLI path spelled with another drive-letter case or separator than the package's is win32-only and not generated; daemon races (two CLIs reading one error file, a CLI hearing closing from the failing daemon it started, a reset connection on macOS) fixed by inspection; a daemon that stops answering only at four holds, each released by a count, never by time: a fresh daemon's first check held before it runs until the daemon accepts the driver's next request (ORACLE_GAPS_TEST_HOLD_OVERLAP), which stands in for a session slower to open than the bound while the CLI carries the warm-up pause; and, after a warm check with the pause disarmed (ORACLE_GAPS_TEST_PAUSE_ARM), the daemon's own pauses (pauseAt): before a check runs (daemon-check), before a check that reuses every run warms its contexts (daemon-warm), and before a context warms in place of one that a cut stopped, after the test edit (daemon-replace), each released by the driver's file once the CLI has ended; the CLI's bound is the socket's idle timeout on a daemon that sends nothing (60 s; these histories shorten it to 3 s), a wall-clock bound that changes the run's path and never its outcome; its cold check must equal a cold run (law 9's cold fallback, R39), and it never holds, since only the daemon gives its session the pauses (G43: a hook changes no outcome); the daemon must still run after the fallback, must leave the run's --out as the CLI left it once released (every file's text, with a poison the driver writes into its results.json and error.log), and must serve the next check (its served mark); the daemon drops a check whose CLI has gone before it runs, at each phase's end, and after each context that warms in place of a cut one, which is the one stretch inside a phase with no running copy to beat, and a check ends only once every context's share of a phase has; not witnessed: a gone CLI found while two contexts are both warming in place of cut ones (the wait for every context's share is by inspection), and a copy that starts in the moment between the CLI's bound and the daemon's reading of the closed connection, which runs to its end and writes into the run's --out (the race is the bound's own, a wall clock's, and no history generates it); the held warm-up and the held re-warm rely on the warm daemon's check reaching its hold within the 3 s bound, as the shortWait history relies on its first copy starting within it, and the driver's wait for the hold fails loudly otherwise; the one beat left, a running copy's poll, is witnessed by a warm daemon's check under the 3 s bound while a copy waits 10 s (shortWait); a CLI whose daemon is closing and hung waits 60 s and fails with an internal error rather than checking cold, and --stop-daemon against a hung daemon has no bound (merge gate F3; owed: a constructed history that closes a held daemon, issue #135); a fallback whose cold check refuses is not generated (law 9's refusal-notes comparison would see the fallback note); the runner's copies of the test files exist only in memory, and its harness and Vitest configs sit in the package's .oracle-gaps-, which the next run removes once a killed run has left it (judged after two stopped runs, not after a SIGKILL); a default --out is judged here too: the default-directory histories (a stop, a daemon check, and the unsupervised daemon check) run without --out, and their directory is named in the trace; the namespace is checked after each run and, in a stopped run, while the run runs; a slow kill in a later owner, after earlier owners' confirmations spent the budget, becomes a timeout (owner order; not generated); the Vitest 5 path of the copies' pragma read and of the runner is verified from Vitest 5's source only, since the Vitest 5 CI job runs no gap oracle; a batch whose run reports an error outside any test reruns each copy alone (unitsBatch, runAlone) with its first budget, and a copy cut there is kept as a timeout with no confirmation, a defect older than this oracle's budget work that no history reaches (no history has an error outside any test beside a slow copy), recorded here with that trace for the maintainer; stale-session removal is scoped to this host's session directories (a host file), and is not judged across pid namespaces; a generated history stops only while the witness spins, or aborts, and on win32 not at all, since every other stop switches a test-only hook (the hold after the completed line, the warm probe, and on win32 the test stop file; randomness wins, 2026-11-16), so only the halves of every outcome class stop with one; the stop law covers SIGTERM and SIGINT sent to the CLI; a SIGKILL of a supervisor is witnessed on POSIX by the constructed history "a supervised child exits when its supervisor is killed" (its child exits within the watchdog tick), and a signal sent to the daemon is not witnessed here, and its witness is one test that spins synchronously and starts one child process; the mutant budget is witnessed as two slow owners whose confirmations pass the --mutant-limit floor, not through its multiple of the owners' unmutated time, and authored mutants have no confirmations to budget; the witness w_slowpair is a wall-clock window, since the CLI's budget has no controlled clock and a fixture cannot see the time the CLI measures for its file: with one-test owner files p and q of measured unmutated time t each (a file first in its context pays the instrumented transform too), a wait W of 10 s, a 20 s --hang-limit and a 15 s --mutant-limit, it tells a budget from none only while (a) its first attempt is cut, W > 1 s + 3t, (b) its two confirmations do not fit the budget, 2W > max(15 s, 10 x 2t), and (c) each would fit its own full limit, W < 20 s + 3t; (b) is the binding one, so it holds while the two files' time stays under about 2 s together (about 1 s each, some 20 times the authoring machine's idle and 5 to 10 times its loaded measurements), and under one budget a run's overhead must stay under 10 s + 3t; CI runners measure 1.5 to 2.3 times the idle machine; beyond that w_slowpair would read not-judged; heartbeats are checked for shape only (maintainer decision "Heartbeats are progress, not outcome", 2026-11-14): each names a mutant of the run, or the baseline, in a file that runs it, and a beat after its mutant's own gap line is a breach, while their set and count, which depend on timing, are not compared and no busy mutant's beat is owed; the shape law reads "was running" only against a mutant's own streamed gap line, so a beat that names a killed mutant (which streams no line) after it finished, or a mutant that has not started, is not a breach; the state "a heartbeat named a busy mutant" is required (maintainer decision "#90 job stays blocking", 2026-11-16) and is entered only through a deterministic hold (#84): the slow kill's test, where its output differs, holds until the run's stream log has one more heartbeat that names a mutant in its file, released by the CLI's own line and never by the clock, so a cold run that holds it has such a beat on any machine; the state is entered only by a cold run that has the slow kill and named its stream log to the hold (\`every outcome class\`, which pins its own --out, and drawn histories with the slow kill and an --out), and a beat anywhere else enters nothing: a check through the daemon (a daemon keeps the environment of the call that started it), a stop run, a run with the default output directory, and the first half of the witness sequence (no slow kill; its I/O wait spans the interval by timing) run unheld, and their beats are progress only; a held copy that no beat releases is cut by the CLI's budget and is judged \`timeout, owed killed\` (the bound of 2,400 polls on the hold is a backstop that the CLI's 15 s budget and 20 s limit always reach first); a config only in the package's parent directory is generated one level up, in a temporary directory with no package.json (so its .js or .ts name is CommonJS), as a plain config, one that throws, one with no default export, or one that defines test.projects; a config two or more levels up, or a parent config beside a package config, is not generated, and the order in one directory follows Vitest's configFiles list by inspection; a package with no config lives in a temporary directory, where no parent has a config; a counterexample shrinks through at most six histories, and within half the unit's timeout, so it can be larger than the smallest that fails; in CI's random units it does not shrink (GAP_E2E_SHRINKS=0, the maintainer's decision "Shrink briefly in CI", 2026-11-17), and its history is filed whole; module-level data only as four \`const\` declarations of the main source (an object literal, an array that holds a function, a boolean, and an object of two methods), read in place, with no data imported from another file, destructured, reassigned, or declared as an enum or a default export; the work law only through shortcut witnesses (an early return before a counting loop with a bound twin of ratio 2, and a push beside one that calls String with a bound twin of ratio 1), the early return in tests that reach the clock by each watched path and in tests that wait a turn, a generated test's bound of 1 derived from its plan; probe code is never work (maintainer decision, 2026-11-16), so the model and the capture count no probe-only site, and only a constructed history (a probe branch under a generated test's work bound) reaches a probe branch beside a bound: the fixed seeds' generated histories reach neither that state nor \`killed by a work bound in a generated function\`, which stays measured; work counts are observed only through a bound's ratio and a rise, never as numbers (v21.2 O1), so a counter that skips every site on a probe's line, which drops only the condition sites of an elseIf that run once on both sides, survives (mutant M6, equivalent under these observations), while one that skips every site of a function that holds probe code is killed (M7), and one warm-cache witness whose selection reproduces observations but not work; work is the capture's count of site evaluations per test, so code with no site (a plain assignment, a for...of pass, a builtin's own work) counts nothing and a test's histories are summed, and a test whose work varies between the unmutated runs while its observations agree is not generated. The skip law (law 4: a mutant is judged without a run only where no owner's unmutated run reached its site with a value that the mutant changes) is judged through the kills, crashes, and setup errors of mutants whose value changed, and through the survivors of those whose value did not: for \`?.\` on an \`undefined\` and a \`null\` object (the drawn \`absentObjects\`), on an absent object two or three links deep, and past a link whose object is absent, where JavaScript short-circuits the rest of the chain so a plain read there changes nothing (the drawn \`optionalChains\`), and for a returned value made \`undefined\`, in the constructed history \`a value change at a ?. read and at a return\`; for a \`?.\` whose object is absent only while the module loads (a booted first function), in \`a ?. read of an object absent only at load\`, whose mutant throws as the module loads and is a setup error; and for a comparison swap, a dropped \`!\`, a skipped call, a boolean flip, a forced \`if\`, a loop that never runs, a dropped field, a value-position call, and an arithmetic swap, by the witness module's kills under \`every outcome class\`. A logical operator's swap (\`&&\`, \`||\`, \`??\`) has no killable site in the grammar (its only sites are probe guards and the old probe form), so that its switch infects the site on every reach is fixed by inspection. The switch's extra infection (it infects on every reach, including a \`return-undefined\` of a value that is already \`undefined\`) costs a run and changes no outcome only because its active form computes exactly the mutated expression; the \`?.\` helper's active form throws wherever it infects, so it infects exactly where the link's own object is absent and no earlier link of the chain short-circuited, and the outer links' helpers are told of an earlier link's short-circuit. An optional element read (\`a?.[k]\`) is not generated: it goes through the same capture helper as \`a?.b\`, so under its mutant an absent object throws before the key expression runs, where the plain read \`a[k]\` evaluates the key first; a key with an observed effect or a throw of its own is judged by that order, which differs from the real mutant's, and no history reaches it. An optional call (\`f?.()\`) is no site; where it feeds a later \`?.\` link, it passes its short-circuit on to that link, so the link's mutant is skipped where the call's method is absent, and runs where the call ran; this is judged for a method that returns an object or nothing, absent itself or behind an absent object (the drawn \`call\` and \`returns\` of \`optionalChains\`), and for a plain read of \`v\` after the call. The generated calls take no arguments; the spread that marks a call as run comes before any argument, so a call's arguments run only where it does, as before. The pass-through keeps, per call site, a stack of the open evaluations' records, so a record pairs with its evaluation only where evaluations of the same expression nest and each one ends. Four cases break the pairing, and no history generates any of them: an evaluation of the same expression nested in the receiver, or in the method lookup, that throws and is caught there after its call ran leaves a run record open, so an outer evaluation whose method is absent reads it as run (a false infection, and a false throw under the next link's mutant, P3); one nested in the call's arguments or in the method's body that throws before its call runs leaves a not-run record open, so an outer evaluation whose call ran reads it as skipped (a false skip: the next link's mutant is judged without a run, the class of issue #109, P5); and two async evaluations of the same expression interleaved at an \`await\` close each other's records (a false skip or a false throw). Each is filed for an owner as issue #162. Open Limits, owned by the in-process runner oracle (L1, state \"torn tail at a cut-off read\"; RED: captureLines without the guard): (a) the batch's cut-off read (runBatch's rowsOf after the cut-off) drops a partial last row, and no witness reaches that drop yet, because a copy held half written across the cut-off did not observe its release in the harness; (b) a short fragment on the strict path, which the no-newline mutant does not reach. The strict read after a finished run (a fragment is an internal error) is witnessed by the no-newline mutant. Windows idle gap (recorded, not witnessed): the state \`a hang cut off for idleness\` is required on every platform, and win32 reaches it (run 43683794816, job 132719872988), so it is not platform-scoped. The sampler (capture-setup.mjs) counts a copy idle only after 500 ms with no late tick, no non-quiet handle, utilization at most 5%, and no timer due, so \`every outcome class\` reaches the state only when the sampler sees that window. On win32 the check missed it once (run 43683794816, job 132719872940); the cause is not confirmed (a late tick on a shared runner, or a stdio handle outside \`quiet\` for this file's console output, are the candidates). Its witness, a test-only hold at the hang until the window is seen, is owed, and that is a timing dependency the determinism-under-load rule forbids; the maintainer decides whether it lands before the Windows job becomes blocking. A run killed before it cleans up may leave its own directory: the CLI killed with SIGKILL (\`kill\`, constructed only), the unsupervised abort, and every abort on win32. The constructed history \`a run after a killed CLI\` kills the CLI and its check's group while the witness spins, and the run after it must remove the directory the killed CLI wrote, through the \`host\` file that CLI wrote. Every other stop run that opens a session also finds a killed run's directory that the driver wrote, under a pid that no process can have (above Linux's pid_max and macOS's 99999, and not a multiple of 4 as every win32 pid is). The abort's killed directory is never followed by a run. Whether a run is killed is read from the stop and the platform, not observed, so a win32 supervisor (#79) must narrow it to the unsupervised abort. A run stopped before its first statement can remove nothing, so a killed run followed by such a stop would keep the killed run's directory whatever the CLI does; no history has that order (a stop before the CLI starts is the last run of its history, and the killed runs are not followed by one). On POSIX a supervised run's directory is removed by its supervisor once the child ends, whatever ended it; an unsupervised run removes its own when its process exits, and that path is judged on win32 alone (its hookStops histories), since POSIX's one unsupervised history is stopped at its note, before a session opens. No history makes a removal fail or reuses a pid, so the catch and the live-pid guard of removeSession are not judged; their witnesses are owed (#168).`, + limits: `Law 12 judges refusals. The one internal error a history reaches is the constructed rejected-batch history (rejectBatch, a test-only fault that rejects a batch's run after its copies are in Vitest's and Vite's state). Its run is judged in this order: the output namespace, process warnings, and config lines, as for any run; then bounded retention, which owes the rejected batch's own line with its state after retirement; then the stream law (law 15) on both streams, as for any run, so its error line and its done line carry the prefix, and no other line does not, which the state \`a run ended in an internal error\` declares; then exit 3 with an internal error's done line last on both streams, no stack frame on stderr, and an error.log in its output directory. What the error.log's stack says is not judged. An internal error that no fault constructs is a completion failure, reported before the stream law; an internal error through the daemon is reached by no history, and its ending is the same function's (endRun), by inspection. The internal error's lines follow the register's C entry "The internal-error line" (2026-11-16): the general law (endsBreach) owes every run that ends through endRun one error line on stderr naming the done line's log, the output directory's error.log, and none on stdout, and owes the supervisor's own exit 3 none. The message's escape is witnessed by the fault's message, which holds a line break and a forged done line after it; the log path's escape only on POSIX, where the rejected batch's output directory holds a line break (a name on win32 cannot). Paths in a completed run's lines (the results note, the done line's report, the test-output note) are escaped the same way, by inspection: no completed history names such a directory. P10's exempt lines (v21.2 P10 and I3) are read by their own shapes (exemptBlocks), never by indentation alone: a baseline failure's indented details, a refusal's block, Node's warning with its \`--trace-warnings\` frames and its hint, V8's fatal block, esbuild's message block, and Vite's own \`[vite]\` line. The V8, traced-warning, and esbuild blocks are calibrated beside our own line with no prefix (the stream calibrations put them before the rejected-batch run's done line, and the law must name our line alone); no generated history prints any of them or a Vite line, and a shape of those tools that exemptBlocks does not name is read as a stray. The stdout summary's exemption is unused and not exercised: the CLI prefixes every summary line (its summary, legend, and function kinds). \`--help\` prints its usage text with no prefix and no done line, exempt by the register's M entry "\`--help\` is exempt" (2026-11-16), and the static oracle's Help law owes exactly that text; \`--stop-daemon\` prints nothing and exits 0, read the same way as a command outside the run protocol (C, pending ratification), and this oracle runs it without observing its streams. A syntax error in a module that the config imports lets the esbuild that Vite bundles configs with print its block before the refusal (trace, Vite 7.3.7, esbuild 0.28.2: \`✘ [ERROR] Unexpected end of file\`, an empty line, \` helper.ts:2:0:\`, the code frame, then the refused block): its lines are exempt by the register's M entry "Other tools' output is exempt" (2026-11-16), the refusal and the done line around them keep the prefix, and no history generates it. Bounded retention reads the hook's lines (retention-hook.mjs). Its count of one line per batch (runner law L2) rests on the index the hook takes when a batch starts, so a CLI that skipped a batch's index and its line together, or that omitted only the line of a check's last batch, keeps the indices 0..n-1 and is not seen; a check that ended before its report (the rejected batch, a stopped run) and a daemon the driver killed are judged for at most one line per index, since the batches they were running at their end owe none. A line's process is told by check.pid, which the check writes, and the driver itself knows only the daemon it killed. The cold store's law (held) counts the entries given to the store's put that a full collection leaves reachable, so a store that kept only a part of an entry (its unit without the entry) is not seen; staleOpen reads Vitest's own close state (closingPromise, set when close starts), so a close that starts and never resolves is the runner oracle's (L8); and a context replaced after a check's last batch is observed only by a later check's first batch through the daemon. The run whose process aborts is declared on every platform. Its win32 witness (process.abort through ORACLE_GAPS_TEST_ABORT_AT, under the win32 supervisor) lands in #79; until then the Windows end-to-end job is non-blocking (C entry (c)). On win32 Node cannot deliver SIGTERM (its kill is TerminateProcess, which ends the process with no stopped line), so the stop states that name SIGTERM are required on POSIX only, and a stop there goes through the CLI's test stop hook and is observed as SIGINT, which owes its done line as any stop does (no history sends a win32 child.kill, the SIGKILL-equivalent that owes none); every SIGINT state stays required on win32. A stop as the CLI starts lands only in one of two holds: the driver's own preload before the CLI's first statement (outcome a, nothing written; POSIX only, since its witness is a signal's default action), or the CLI's hold after its started marker, which the stop the CLI keeps releases (outcome b, judged only when the supervisor started no child); a stop during Node's own start between the two is not generated, since its outcome follows timing. The drawn spawn delay is a wait inside a hold, so it cannot move a stop out of it. Stop runs that land while the config warms (ORACLE_GAPS_TEST_WARM_PROBE) are still drawn, and on win32 every drawn stop goes through ORACLE_GAPS_TEST_STOP: each is a test-only hook that a drawn field switches, main's pattern, which the review of #57 (A3) brought to the maintainer; only the stop after the completed line (ORACLE_GAPS_TEST_HOLD_UNTIL) was made constructed-only. A missing \`ps\` is generated only as a PATH that names no command, on POSIX, where the run is supervised. The unsupervised path (stopNow, every win32 stop) is reached on POSIX by one constructed history whose stop is held at the run's note, before the check starts a process, and by one whose stream log is lost after the note, where the stop is the run's one ending whatever the log (a check error after the stop's line and before its streams flush is not reached, so a stop that only swallowed the log's error survives): an unsupervised POSIX run has no group to end, so a stop while the witness's test spins leaves that test's own child (measured; C 2026-11-13, Windows (e)), which win32 ends with the job object libuv puts its children in. Argument refusals are the static oracle's; the check's rejection of a stack trace, a missing fix, and a wrong exit code is calibrated in the static oracle, through the same refusalBreach, and here only a refused run reported as complete is calibrated, because each calibration here runs real CLI histories. Law 15's stdout gap lines are judged against each completed run's own results.json, by id: the cold run, and each check through the daemon (every step's check and the check that a mid-run edit overlaps); a stopped run and a refusal list no gaps and are not judged so. A line's change is read loosely: its code on one line, whole or cut after a kept part and ended with \`...\`; where the printer cuts (60 code points, kept 57) is the classify oracle's law 8 (\`shortened\`), in-process. A timeout's line names its cause as \` (cause: )\` after its change, judged equal to results.json's \`detail\`; a line with any other text after its change does not read and fails loudly. Hangs only as an owner test that awaits a promise that never settles, under a 20 s full limit, and a slow run only as two witnesses whose tests wait where the output changed, 3 s in one that then judges and 10 s in two one-test owners of one that does not; busy loops only on instrumented code (cut by the call budget), heavy tests (\`heavy\`, drawn for 1 history in 16, for its own tests) only as 10,000 more calls before each campaign's call, never in a test.prop or the shared-state witness; a test that reads the clock only after its work makes its spin a gap although its count up to the spin did not follow the clock, a conservative reading accepted under M "Count-only kills" (coordinator, 2026-11-17), since the CLI sees only that its rows are clocked; a row is clocked by state (C, 2026-11-17): the test read the clock, made a timer, or its own I/O ran a callback while it ran (async_hooks), so a sync read, which makes no resource, leaves it unclocked, and a short async read is witnessed by \`a spin after a short async read\`, no slow runs past the full limit that would end, hangs in ordinary or nondeterministic tests (so a nondeterministic test's no-progress kill, which its ordinary failures would not give, is not seen: issue #149), setup errors other than a throw at module load, loops, multi-line conditions, two functions on one line, or await, new, and void statements; nested functions, object-literal methods, accessors, and class-field arrows only where no test calls them (their ownership is the static oracle's); two source files of one package, with no re-exports, namespace imports, or calls into another package; --only only in a stopped run and a partial run of one function's mutants, and no --sample or --diff-base; the baseline's failures only as one failing owner test that throws an error and a setup file that throws, so a baseline law violation's law and counterexample lines are the classify oracle's; a derived class's file is instrumented and its sites are judged as unowned, but no test imports or constructs the class, so its no-mutant and super-region behavior is the static oracle's (in-process), not this oracle's; package configs only as vite.config or vitest.config with one of the extensions .ts, .mts, .cts, .js, .mjs, and .cjs (not every pair), as config/gap-vitest with the same extension through --config, or none, as an object or a sync or async function, with no plugins of their own except one enforce: 'pre' transform that appends a comment to each module of the package (prePlugin) and a plugin whose load reads each test file with a sync read (loadPlugin), drawn for a config in ES module syntax and reached by its own constructed history too; a plugin whose output depends on anything but the module's text (a sidecar file it reads) and an environment variable that changes between checks are not generated, since the runner tracks neither yet and the daemon's environment is fixed when it starts (F-A1: the maintainer decided on 2026-11-13 that the daemon tracks both and restarts when they change, in the next pull request); and test.projects only as a refusal, and only under the package's own package.json; the supervisor-death state is required and entered only off win32: win32 has no supervisor until #79, so there the state is not reached and its history is not judged (platform-scoped reach; the win32 run is unsupervised, which register C entry (e) of #58 records as not guaranteeing that no process outlives the CLI); a config's test.root only in four shapes, drawn for an ES module config that Vitest discovers in the package (the randomness-wins decision, 2026-11-16) and each reached by its own constructed history of one owner and one context, where the test files' globs and the setup file are relative to the root (issue #101): the directory that holds the tests, the directory above the package, a directory of the package that holds no test file, and the package's tests as a symbolic link to the directory that holds them (required on every platform, since Windows creates the link as the driver's other links); not under --config (nor in a history whose config edit names a second config), a CommonJS config, a parent's config, test.projects, several contexts with different roots, or a root that is the filesystem's root; the runner serves each copy of a test file by its absolute path, through its pre plugin's load (which answers before the config's own plugins) and through its read of the copy's path for Vitest's pragma read (serveCopies), and histories run copies only in Vite's server environments (node and the pragma's ssr environment): jsdom and happy-dom, Vite's client environment, are not generated (the #103 merge-gate review ran jsdom by hand under all five root shapes, and the copies loaded in each); a top-level await only in an ES module config and require only in a CommonJS one, as Vitest itself loads no other; a --config outside the package is refused by design, through the rule for paths outside the package (args.mjs), and no history generates one; contexts only one, four, or eleven, and process warnings judged only as MaxListenersExceededWarning on the CLI's own stderr (a daemon's process warnings go to its log); a context that fails to open gives back its listener allowance, which no observation shows (a kept allowance only raises the limit, so no warning follows), so that accounting is fixed by inspection; no probe-bound states, so reach agreement's killing of probe mutants is not checked; no gap answers or gate; law 9 judges the witness module's outcome classes along one constructed edit sequence only (a test edit, a helper edit, and a revert), which runs as two constructed histories, one for each half of the module (captureA and captureB, whose union is the module and which share no function; a function stays with the functions its tests or statements read), since the whole module along it took one CI job (#106, #145; maintainer decision "A one-history job over budget is cut by cost", 2026-11-16), so an interaction between functions of different halves, beyond the ones the split keeps together, is not judged along the sequence (the whole module runs together in \`every outcome class\` and in generated histories, which take no steps); and the budget witness's outcome classes along one constructed second check with nothing changed, because a generated history with the witness module or the budget witness takes no steps (maintainer decision, 2026-11-11, and the runtime review's P1-a, 2026-11-12; this oracle owns the gap); the state "a timed-out run reused after an edit" counts only runs cut by their call budget (the spin, in the first half), a count the CLI records, by the maintainer's decision "#104 state narrowed, #99 restores" (2026-11-17), which narrows "#90 job stays blocking" for that state only: the reuse of a run cut by the clock (a hang) is judged by law 9 but not required, because on main a hang's timeout is stored only when the sampler saw the copy idle, which load decides, and otherwise its confirmation is cut short by its mutant's budget and not kept (#104); #99 makes a hang a state-based no-progress kill and restores the reach as "a run killed for no progress, reused after an edit" (#156); law 9's edits come between checks, except one, midrun, which lands during a check through the daemon: a test of file a, whose context has loaded file b's helper, rewrites the helper in the baseline's first run of file a, before file b's runs read it, in one context or several; or a test of file b rewrites its own helper, or the data file that its verdict read through a sync fs read, after file b's judged tests read them, in the run that judges with the old text (review of #55, F1); that check must report, for each mutant, what a cold run of the state before the edit or of the state after it reports, and exit as one of them does (third review of #55, F-C), compared per mutant result rather than per file; when file b's own test lands the edit, b's baseline reads the old text and b's mutant runs read the new one, so b's own mutants in that check are not judged, since their results legitimately mix the two texts (maintainer decision, 2026-11-13), while every other mutant there is judged against both cold runs, with the check's exit code, its mutants' keys, and its stream; the stronger follow-up compares each run's raw outcome with the cold outcome of the state that run read; and the next check, with nothing changed, must equal a cold run of the state after the edit, compared for equality of results only, not for what the daemon reused or computed (the runner's rule: a run whose input changed after the run read it is invalidated and not stored, settleIn); a check through the daemon of a state that nothing changed must compute no run that was not cut off, judged where no test reads the clock (F-A, F-A2), and a test's fs read of a file and the module that the context serves from it are two inputs, each under its own hash (F-A2); an edit during a mutant run is not generated, and a file is edited mid-run only by the run's own tests, not by another process at an arbitrary time; a file that a test reads through fs is edited only as one JSON file that file b's verdict reads by a sync, a promise, or a callback read, or by a sync read after it stats its own directory and tests that it exists (statDir: the directory is an input as a directory, not by its content, and is not itself edited), each read first checking that its fs function keeps its own name and length (the runner stores a run under the files in the package that its copy reads with readFileSync, readFile, existsSync, or statSync, or the fs/promises readFile or stat, each with its hash at the copy's first read; a read outside the package or under node_modules, from a worker thread or a child process that a test starts, or through any other fs API, which includes readdir, opendir, glob, globSync, createReadStream, realpath, lstat, access, open, a read through a file descriptor (read, readSync, readv, fstat, fstatSync, and readFileSync or readFile given a file descriptor), copyFile and copyFileSync, cp and cpSync, openAsBlob, statfs, readlink, the callback stat, exists, watch, watchFile, and the fs/promises access, open, and glob, is no input, and none is generated: the maintainer decided on 2026-11-13 that every fs read except watch and watchFile is an input, which a follow-up of #55 implements, and until then this oracle owns the gap); a run cut off by its clock writes no closing row, so it is stored under its file's known inputs only, while a run that spun ends and keeps the inputs it named (F-D), in a batch as alone; a spun copy that reads a file that its file's unmutated run never reads is generated (spinData: file b's verdict tests that read the data spin on the function they test while it is open), and the data's edit is judged by law 9 for copies that spin in a batch; a copy that spins alone (runAlone, after a batch with an error outside any test) is generated too (spinThrows: each spun copy schedules an error that fires after its test), and its data's edit is judged the same way; an error outside any test only from a copy that spins, so the alone path's other units (a slow copy's confirmation) stay unwitnessed; a test that rewrites a file it read with the same text makes the next check compute that run again (1 run in the self-edit histories), since the edit changes the file's modification time: the daemon is conservative there, nothing goes stale, and content keys (D10) remove it, with a selfWrite dimension in that pull request; the mid-run marker sits in the daemon's session directory, .oracle-gaps-, whose location predates #55; the runner's hash memo knows a file by its size and modification time, so an edit that keeps both is not seen, and no history makes one (D10, planned in phase 2, A5); law 9's edits leave out other environment variables, new files that change how an unchanged import resolves, Vite config and installed-package changes, and concurrent checks other than two that start the daemon; native addons only as a binding.gyp or a prebuilt .node file, not as optional platform packages; a scratch package lies in the CLI's directory (.scratch-*), in a directory beside it whose name extends the CLI directory's name (siblingDir, drawn), or in the OS temp directory, so the driver's inside-the-CLI label is judged against a prefix that is not a parent; a CLI path spelled with another drive-letter case or separator than the package's is win32-only and not generated; daemon races (two CLIs reading one error file, a CLI hearing closing from the failing daemon it started, a reset connection on macOS) fixed by inspection; a daemon that stops answering only at four holds, each released by a count, never by time: a fresh daemon's first check held before it runs until the daemon accepts the driver's next request (ORACLE_GAPS_TEST_HOLD_OVERLAP), which stands in for a session slower to open than the bound while the CLI carries the warm-up pause; and, after a warm check with the pause disarmed (ORACLE_GAPS_TEST_PAUSE_ARM), the daemon's own pauses (pauseAt): before a check runs (daemon-check), before a check that reuses every run warms its contexts (daemon-warm), and before a context warms in place of one that a cut stopped, after the test edit (daemon-replace), each released by the driver's file once the CLI has ended; the CLI's bound is the socket's idle timeout on a daemon that sends nothing (60 s; these histories shorten it to 3 s), a wall-clock bound that changes the run's path and never its outcome; its cold check must equal a cold run (law 9's cold fallback, R39), and it never holds, since only the daemon gives its session the pauses (G43: a hook changes no outcome); the daemon must still run after the fallback, must leave the run's --out as the CLI left it once released (every file's text, with a poison the driver writes into its results.json and error.log), and must serve the next check (its served mark); the daemon drops a check whose CLI has gone before it runs, at each phase's end, and after each context that warms in place of a cut one, which is the one stretch inside a phase with no running copy to beat, and a check ends only once every context's share of a phase has; not witnessed: a gone CLI found while two contexts are both warming in place of cut ones (the wait for every context's share is by inspection), and a copy that starts in the moment between the CLI's bound and the daemon's reading of the closed connection, which runs to its end and writes into the run's --out (the race is the bound's own, a wall clock's, and no history generates it); the held warm-up and the held re-warm rely on the warm daemon's check reaching its hold within the 3 s bound, as the shortWait history relies on its first copy starting within it, and the driver's wait for the hold fails loudly otherwise; the one beat left, a running copy's poll, is witnessed by a warm daemon's check under the 3 s bound while a copy waits 10 s (shortWait); a CLI whose daemon is closing and hung waits 60 s and fails with an internal error rather than checking cold, and --stop-daemon against a hung daemon has no bound (merge gate F3; owed: a constructed history that closes a held daemon, issue #135); a fallback whose cold check refuses is not generated (law 9's refusal-notes comparison would see the fallback note); the runner's copies of the test files exist only in memory, and its harness and Vitest configs sit in the package's .oracle-gaps-, which the next run removes once a killed run has left it (judged after two stopped runs, not after a SIGKILL); a default --out is judged here too: the default-directory histories (a stop, a daemon check, and the unsupervised daemon check) run without --out, and their directory is named in the trace; the namespace is checked after each run and, in a stopped run, while the run runs; a kill in a later owner, after earlier owners' copies spent the work budget, is not counted (owner order; the count is deterministic; generated histories never reach the budget; not generated); the Vitest 5 path of the copies' pragma read and of the runner is verified from Vitest 5's source only, since the Vitest 5 CI job runs no gap oracle; a batch whose run reports an error outside any test reruns each copy alone (unitsBatch, runAlone) with its backstop, since a copy has no first budget (option A); a copy run alone has no idle detection, so a hang there runs to its backstop, and its spin row is read without the whole-row guard (issue #149; no history reaches the path with a hang); stale-session removal is scoped to this host's session directories (a host file), and is not judged across pid namespaces; a generated history stops only while the witness spins, or aborts, and on win32 not at all, since every other stop switches a test-only hook (the hold after the completed line, the warm probe, and on win32 the test stop file; randomness wins, 2026-11-16), so only the halves of every outcome class stop with one; the stop law covers SIGTERM and SIGINT sent to the CLI; a SIGKILL of a supervisor is witnessed on POSIX by the constructed history "a supervised child exits when its supervisor is killed" (its child exits within the watchdog tick), and a signal sent to the daemon is not witnessed here, and its witness is one test that spins synchronously and starts one child process; a config only in the package's parent directory is generated one level up, in a temporary directory with no package.json (so its .js or .ts name is CommonJS), as a plain config, one that throws, one with no default export, or one that defines test.projects; a config two or more levels up, or a parent config beside a package config, is not generated, and the order in one directory follows Vitest's configFiles list by inspection; a package with no config lives in a temporary directory, where no parent has a config; a counterexample shrinks through at most six histories, and within half the unit's timeout, so it can be larger than the smallest that fails; in CI's random units it does not shrink (GAP_E2E_SHRINKS=0, the maintainer's decision "Shrink briefly in CI", 2026-11-17), and its history is filed whole; module-level data only as four \`const\` declarations of the main source (an object literal, an array that holds a function, a boolean, and an object of two methods), read in place, with no data imported from another file, destructured, reassigned, or declared as an enum or a default export; the work law only through shortcut witnesses (an early return before a counting loop with a bound twin of ratio 2, and a push beside one that calls String with a bound twin of ratio 1), the early return in tests that reach the clock by each watched path and in tests that wait a turn, a generated test's bound of 1 derived from its plan; probe code is never work (maintainer decision, 2026-11-16), so the model and the capture count no probe-only site, and only a constructed history (a probe branch under a generated test's work bound) reaches a probe branch beside a bound: the fixed seeds' generated histories reach neither that state nor \`killed by a work bound in a generated function\`, which stays measured; work counts are observed only through a bound's ratio and a rise, never as numbers (v21.2 O1), so a counter that skips every site on a probe's line, which drops only the condition sites of an elseIf that run once on both sides, survives (mutant M6, equivalent under these observations), while one that skips every site of a function that holds probe code is killed (M7), and one warm-cache witness whose selection reproduces observations but not work; work is the capture's count of site evaluations per test, so code with no site (a plain assignment, a for...of pass, a builtin's own work) counts nothing and a test's histories are summed, and a test whose work varies between the unmutated runs while its observations agree is not generated. The skip law (law 4: a mutant is judged without a run only where no owner's unmutated run reached its site with a value that the mutant changes) is judged through the kills, crashes, and setup errors of mutants whose value changed, and through the survivors of those whose value did not: for \`?.\` on an \`undefined\` and a \`null\` object (the drawn \`absentObjects\`), on an absent object two or three links deep, and past a link whose object is absent, where JavaScript short-circuits the rest of the chain so a plain read there changes nothing (the drawn \`optionalChains\`), and for a returned value made \`undefined\`, in the constructed history \`a value change at a ?. read and at a return\`; for a \`?.\` whose object is absent only while the module loads (a booted first function), in \`a ?. read of an object absent only at load\`, whose mutant throws as the module loads and is a setup error; and for a comparison swap, a dropped \`!\`, a skipped call, a boolean flip, a forced \`if\`, a loop that never runs, a dropped field, a value-position call, and an arithmetic swap, by the witness module's kills under \`every outcome class\`. A logical operator's swap (\`&&\`, \`||\`, \`??\`) has no killable site in the grammar (its only sites are probe guards and the old probe form), so that its switch infects the site on every reach is fixed by inspection. The switch's extra infection (it infects on every reach, including a \`return-undefined\` of a value that is already \`undefined\`) costs a run and changes no outcome only because its active form computes exactly the mutated expression; the \`?.\` helper's active form throws wherever it infects, so it infects exactly where the link's own object is absent and no earlier link of the chain short-circuited, and the outer links' helpers are told of an earlier link's short-circuit. An optional element read (\`a?.[k]\`) is not generated: it goes through the same capture helper as \`a?.b\`, so under its mutant an absent object throws before the key expression runs, where the plain read \`a[k]\` evaluates the key first; a key with an observed effect or a throw of its own is judged by that order, which differs from the real mutant's, and no history reaches it. An optional call (\`f?.()\`) is no site; where it feeds a later \`?.\` link, it passes its short-circuit on to that link, so the link's mutant is skipped where the call's method is absent, and runs where the call ran; this is judged for a method that returns an object or nothing, absent itself or behind an absent object (the drawn \`call\` and \`returns\` of \`optionalChains\`), and for a plain read of \`v\` after the call. The generated calls take no arguments; the spread that marks a call as run comes before any argument, so a call's arguments run only where it does, as before. The pass-through keeps, per call site, a stack of the open evaluations' records, so a record pairs with its evaluation only where evaluations of the same expression nest and each one ends. Four cases break the pairing, and no history generates any of them: an evaluation of the same expression nested in the receiver, or in the method lookup, that throws and is caught there after its call ran leaves a run record open, so an outer evaluation whose method is absent reads it as run (a false infection, and a false throw under the next link's mutant, P3); one nested in the call's arguments or in the method's body that throws before its call runs leaves a not-run record open, so an outer evaluation whose call ran reads it as skipped (a false skip: the next link's mutant is judged without a run, the class of issue #109, P5); and two async evaluations of the same expression interleaved at an \`await\` close each other's records (a false skip or a false throw). Each is filed for an owner as issue #162. Open Limits, owned by the in-process runner oracle (L1, state \"torn tail at a cut-off read\"; RED: captureLines without the guard): (a) the batch's cut-off read (runBatch's rowsOf after the cut-off) drops a partial last row, and no witness reaches that drop yet, because a copy held half written across the cut-off did not observe its release in the harness; (b) a short fragment on the strict path, which the no-newline mutant does not reach. The strict read after a finished run (a fragment is an internal error) is witnessed by the no-newline mutant. Idle and the runner's I/O (M entries "idle is decided by state only" and "Idle counts only the test's I/O", 2026-11-15): the sampler (capture-setup.mjs) counts a copy idle after 500 ms with none of its test's own timers due inside its horizon and none of its test's own I/O active (C, timers by ownership, 2026-11-16), where a timer, handle or request is the test's when the test's or production's code was on the stack when Node made it, or when the resource that triggered it is the test's. No progress is decided per copy, never per file: \`a hang in a file whose test does a short read\` holds the test's own pipe read for two samples before production in every run, and its copy that hangs is killed by no progress. Three constructed histories hold the other states by count and by state: \`a copy beside the runner's pending timer\` (a 100 ms interval the runner keeps through the test, beside the test's ref'd timer past its horizon), \`a copy waiting on a pipe it opened\` (the test's own pipe read, released after 20 samples) \`a copy beside the runner's pending write\` (a write the runner holds pending through the test, as Vitest's forwarded output can stay pending on win32; Windows shard 7, run 43778323034). Not witnessed: a handle whose owner keeps its wrap elsewhere than \`_handle\` (a dgram socket) is not found as the test's, so a test that waits only on it looks idle; a test that awaits only a reply over the runner's own channel (a file snapshot) looks idle on every platform, and since idle is a kill (M-24) a test that waits longer than the 500 ms idle window for such a reply (a file snapshot, a dynamic import(), vi.importActual, a mocked module's importOriginal, each a module fetch over the runner's RPC) is killed by no progress, which on a loaded machine follows load (issue #148 owns it); a resource whose stack frames all lie in node_modules machinery (vitest, Vite's module runner, fast-check, chai, this package) is the runner's even when a test's call led to it, so in a package under test that uses Vite as a library (a plugin test that calls createServer or build), or in this package's own sources when the CLI runs on them, a production timer or I/O request is the runner's unless its trigger chain is the test's, a wait on it alone is a no-progress kill, and a production clock read there leaves the row unclocked (no history generates either); a timer's due time is read from Node's own timer fields (\`_idleStart\`, \`_idleTimeout\`) against the sampler's interval, so a late sample sees a timer as due later, which matters only within a sample's lateness of the horizon, the backstop band. A run killed before it cleans up may leave its own directory: the CLI killed with SIGKILL (\`kill\`, constructed only), the unsupervised abort, and every abort on win32. The constructed history \`a run after a killed CLI\` kills the CLI and its check's group while the witness spins, and the run after it must remove the directory the killed CLI wrote, through the \`host\` file that CLI wrote. Every other stop run that opens a session also finds a killed run's directory that the driver wrote, under a pid that no process can have (above Linux's pid_max and macOS's 99999, and not a multiple of 4 as every win32 pid is). The abort's killed directory is never followed by a run. Whether a run is killed is read from the stop and the platform, not observed, so a win32 supervisor (#79) must narrow it to the unsupervised abort. A run stopped before its first statement can remove nothing, so a killed run followed by such a stop would keep the killed run's directory whatever the CLI does; no history has that order (a stop before the CLI starts is the last run of its history, and the killed runs are not followed by one). On POSIX a supervised run's directory is removed by its supervisor once the child ends, whatever ended it; an unsupervised run removes its own when its process exits, and that path is judged on win32 alone (its hookStops histories), since POSIX's one unsupervised history is stopped at its note, before a session opens. No history makes a removal fail or reuses a pid, so the catch and the live-pid guard of removeSession are not judged; their witnesses are owed (#168).`, }, histories: { // A failing history shrinks through at most six smaller ones. Each runs @@ -6879,6 +7294,12 @@ oracleProperty({ then: { route: 'case', input: 4 }, work: null, timed: false, + // #99's test fields, at their neutral values. + slowMs: 3000, + loops: 0, + timerWait: null, + pulse: false, + pendingIo: null, }, { file: 'b', @@ -6904,6 +7325,12 @@ oracleProperty({ then: { route: 'random seed', input: 7 }, work: null, timed: false, + // #99's test fields, at their neutral values. + slowMs: 3000, + loops: 0, + timerWait: null, + pulse: false, + pendingIo: null, }, { file: 'b', @@ -6929,6 +7356,12 @@ oracleProperty({ then: { route: 'enumerated', input: 0 }, work: null, timed: false, + // #99's test fields, at their neutral values. + slowMs: 3000, + loops: 0, + timerWait: null, + pulse: false, + pendingIo: null, }, ], witness: false, @@ -6997,6 +7430,9 @@ oracleProperty({ shortWait: false, absentObjects: [], optionalChains: [], + // #99's fields, at their neutral values. + hangMs: null, + heavy: 0, stopRuns: NO_STOPS, laterSteps: NO_LATER_STEPS, }, @@ -7057,13 +7493,13 @@ oracleProperty({ }, { // The budget witness checked through the daemon twice with nothing - // changed between: the second check reuses the confirmations the - // first ran, and must report what a cold run reports (law 9, with - // the budget of law 4). + // changed between: the second check reuses the work-budget results + // the first ran, and must report what a cold run reports (law 9, with + // the work budget of law 4). name: `a second daemon check of the budget witness`, enters: [ `a second daemon check of the budget witness's state`, - `confirmations past a mutant's budget timed out`, + `a mutant's copies past its work budget killed by bounded work`, ], build: (generated) => ({ ...generated, @@ -7090,11 +7526,12 @@ oracleProperty({ // mutants and so the same batch, a mutant whose owner test in file // b then hangs. Under one budget a run past it is final, so a // watchdog that charges b's hang to the copy that failed to load - // reports that mutant as a timeout instead of a setup error. + // reports that mutant as killed by no progress instead of a setup + // error. name: `a load failure beside a hang`, enters: [ `a setup error`, - `a setup error beside a timeout under one budget`, + `a setup error beside a hang under one budget`, `a completed run whose tests wrote nothing`, ], build: (generated) => ({ @@ -9162,8 +9599,8 @@ oracleProperty({ }, { // A check through a warm daemon that works past its CLI's bound (3 s, - // `shortWait`): after the test edit, a slow copy is cut at its first - // budget, and its confirming run waits 10 s where the output differs. The daemon's beats while the copy runs keep the CLI + // `shortWait`): after the test edit, a slow copy waits 10 s where the + // output differs, in its batch. The daemon's beats while the copy runs keep the CLI // waiting: it must not fall back (the beat's witness, delta 3). name: `a check through the daemon that works past the bound`, enters: [`a check through the daemon worked past its CLI's bound`], @@ -9179,7 +9616,7 @@ oracleProperty({ }, ], // The test edit recomputes file a's runs, the slow one with them; - // a first budget cuts the slow copy, which then waits 10 s alone. + // the slow copy waits 10 s in its batch (no first budget, #99). steps: [`test`], laterSteps: NO_LATER_STEPS, budget: `two`, @@ -9407,6 +9844,538 @@ oracleProperty({ ], }), }, + { + // A determinism check (the determinism under load law, 2026-11-13): + // file a's random-route test and file b's fixed-seed test both kill + // one mutant (as w_crosskill does). In the slow twin, a's test waits + // 3 s before it judges, so its copy runs to its end, and b's kill + // decides the mutant first. The report must still be the one the + // package without the wait gives, its `randomAlso` included. It + // passes on its parent too, so it guards the law and has no RED of + // its own; the outcome rule's RED is the classify oracle's. + name: `a slow owner's kill and another owner's kill`, + enters: [`a slow owner's kill reported as when it did not wait`], + build: (generated) => ({ + ...generated, + functions: [[{ kind: `if`, op: `>`, k: 4, not: false, quote: `'` }]], + containers: [], + overloads: [], + owners: [`both`], + library: [], + libraryOwners: [], + runes: [], + derived: null, + dotGen: false, + dataOwners: [], + exportOwners: [], + importer: `none`, + wholeFile: `none`, + template: false, + tests: [ + { ...judging(`a`, `random seed`, 7), fn: 0 }, + { ...judging(`b`, `fixed seed`, 7), fn: 0 }, + ], + witness: false, + authored: null, + secondAuthored: false, + fault: `none`, + stability: `none`, + ordinary: false, + native: `none`, + boot: false, + budget: `two`, + cliPath: `plain`, + concurrentStart: false, + contexts: 1, + steps: null, + slowTwin: true, + }), + }, + // Constructed histories added by the deterministic-cuts change follow. + // Owner a's copy never finishes and never goes idle: it awaits a promise + // that never resolves, while a repeating timer stays due inside its + // horizon (`pulse`). So it is cut only by the 2 s backstop, on every + // machine, and owner b would kill. The cut ends the copies, so b does + // not run: the mutant is a backstop gap (a timeout), never a kill. + { + name: `a backstop cut ends the copies ahead of a later kill`, + enters: [ + `a backstop cut ends the copies ahead of a later kill`, + `timeout`, + ], + build: (generated) => ({ + ...generated, + functions: [[{ kind: `if`, op: `>`, k: 4, not: false, quote: `'` }]], + containers: [], + overloads: [], + owners: [`both`], + library: [], + libraryOwners: [], + runes: [], + derived: null, + dotGen: false, + dataOwners: [], + exportOwners: [], + importer: `none`, + wholeFile: `none`, + template: false, + tests: [ + { + ...judging(`a`, `fixed seed`, 7), + fn: 0, + hangs: true, + pulse: true, + }, + { ...judging(`b`, `fixed seed`, 7), fn: 0 }, + ], + witness: false, + authored: null, + secondAuthored: false, + fault: `none`, + stability: `none`, + ordinary: false, + native: `none`, + boot: false, + budget: `two`, + cliPath: `plain`, + concurrentStart: false, + contexts: 1, + steps: null, + hangMs: 2_000, + }), + }, + // Owner a's copy makes 30,000 extra calls (under its budget), and so does + // owner b's, which would kill. Together they pass the work budget, so b's + // kill does not count: b's copy fails bounded work instead, a kill by + // count. + { + name: `a kill past the work budget in a later owner`, + enters: [ + `a kill past the work budget in a later owner is a bounded-work kill`, + ], + build: (generated) => ({ + ...generated, + functions: [[{ kind: `if`, op: `>`, k: 4, not: false, quote: `'` }]], + containers: [], + overloads: [], + owners: [`both`], + library: [], + libraryOwners: [], + runes: [], + derived: null, + dotGen: false, + dataOwners: [], + exportOwners: [], + importer: `none`, + wholeFile: `none`, + template: false, + tests: [ + { + ...judging(`a`, `fixed seed`, 7), + fn: 0, + judges: false, + loops: 30_000, + }, + { ...judging(`b`, `fixed seed`, 7), fn: 0, loops: 30_000 }, + ], + witness: false, + authored: null, + secondAuthored: false, + fault: `none`, + stability: `none`, + ordinary: false, + native: `none`, + boot: false, + budget: `two`, + cliPath: `plain`, + concurrentStart: false, + contexts: 1, + steps: null, + // Its work budget decides its state, so its tests make no heavy calls. + heavy: 0, + }), + }, + // A slow kill whose test waits 16 s where its output differs: no + // wall-clock limit decides an outcome, so the mutant is still killed. + { + name: `a slow kill after a 16 s wait`, + enters: [`a slow kill after a 16 s wait killed`], + build: (generated) => + soloKill(generated, [ + { + ...judging(`a`, `fixed seed`, 7), + fn: 0, + slow: true, + slowMs: 16000, + }, + ]), + }, + // A copy whose test starts a timer due inside its horizon, then waits + // for it: the copy is never idle, and the mutant is killed. + { + name: `a copy waiting on a timer`, + enters: [`a copy waiting on a timer killed`], + build: (generated) => + soloKill(generated, [ + { + ...judging(`a`, `fixed seed`, 7), + fn: 0, + timerWait: { waitMs: 3000 }, + }, + ]), + }, + // The same wait on an unref'd timer: the timer is not listed among the + // process's active resources, so only the copy's own pending timers + // keep it busy (the held state, deterministic cuts, 2026-11-15). + { + name: `a copy waiting on an unref'd timer`, + enters: [`a copy waiting on an unref'd timer killed`], + build: (generated) => + soloKill(generated, [ + { + ...judging(`a`, `fixed seed`, 7), + fn: 0, + timerWait: { waitMs: 3000, unref: true }, + }, + ]), + }, + // Idle counts only the test's I/O (M entry, 2026-11-15). A copy whose + // test reads a pipe it opened, the stdout of a child it unrefs, is busy + // while the read is pending, and nothing else keeps it busy: the read is + // released by a count of the sampler's samples past the idle window, not + // by a time. The mutant is killed. + { + name: `a copy waiting on a pipe it opened`, + enters: [`a copy waiting on I/O it opened killed`], + build: (generated) => + soloKill(generated, [ + { + ...judging(`a`, `fixed seed`, 7), + fn: 0, + pendingIo: `test`, + }, + ]), + }, + // A copy whose test writes to the console and then waits on an unref'd + // timer due long past its horizon, while the runner holds its own write + // pending through the test, as Vitest's forwarded output can be on + // win32: the runner's write is not the test's I/O, so the copy is idle + // and the mutant is killed by no progress, not a backstop gap. The + // package has a native addon, so its tests run in child processes, whose + // output Vitest forwards over their IPC channel, as on Windows shard 7. + { + name: `a copy beside the runner's pending write`, + enters: [`a copy beside the runner's pending write cut as no progress`], + build: (generated) => ({ + ...soloKill(generated, [ + { + ...judging(`a`, `fixed seed`, 7), + fn: 0, + judges: false, + pendingIo: `runner`, + }, + ]), + native: `gyp`, + }), + }, + // Timers by ownership (C, 2026-11-16, under both idle M entries): a copy + // whose test waits on a ref'd timer due long past its horizon, while the + // runner keeps its own 100 ms interval running through the test, as + // Vitest's task-update timer can be pending. Neither timer can let the + // test progress inside the horizon: the test's is due past it, and the + // runner's is not the test's. The mutant is killed by no progress. + { + name: `a copy beside the runner's pending timer`, + enters: [`a copy beside the runner's pending timer cut as no progress`], + build: (generated) => + soloKill(generated, [ + { + ...judging(`a`, `fixed seed`, 7), + fn: 0, + judges: false, + pendingIo: `runnerTimer`, + }, + ]), + }, + // Count-only kills (M, 2026-11-16): a copy whose test reads the clock + // before it runs production, and then spins past its call budget in + // instrumented code (the until loop without its push), could have a + // count that follows the clock, so the spin kills nothing: the mutant is + // a timeout whose cause is the spin, a gap. + { + name: `a clocked copy that spins`, + enters: [`a spin whose count could follow the clock timed out`], + build: (generated) => ({ + ...soloKill(generated, [ + { ...judging(`a`, `fixed seed`, 1), fn: 0, timed: `clock` }, + ]), + functions: [[{ kind: `until`, quote: `'` }]], + }), + }, + // An authored mutant whose owner test hangs where its output changed: its + // copy makes no progress, as an operator mutant's does, so it is killed + // by no progress on the test's route (v21.2 R50, which names no + // exception for an authored mutant), not cut by the backstop. + { + name: `an authored mutant whose owner hangs`, + enters: [`an authored mutant killed by no progress`], + build: (generated) => ({ + ...soloKill(generated, [ + { ...judging(`a`, `fixed seed`, 7), fn: 0, hangs: true }, + ]), + authored: { stmt: 0, kind: `apply` }, + }), + }, + // The unsupervised stop after a lost stream log (the path every win32 run + // takes; ORACLE_GAPS_TEST_UNSUPERVISED on POSIX). The run's own process + // writes its stopped line (stopNow) to stdout, to stderr, and to a stream + // log that is now a directory. Once that stop has begun, nothing else ends + // the run (P10, v21.2 R10 and I7): the failed log write is no second + // ending, so each stream holds one done line, the stopped line. + { + name: `a stream log lost after its note, then stopped, unsupervised`, + enters: [`an unsupervised run stopped after its stream log was lost`], + build: (generated) => ({ + ...generated, + functions: [[{ kind: `push`, quote: `'` }]], + containers: [], + overloads: [], + owners: [`a`], + library: [], + libraryOwners: [], + runes: [], + wholeFile: `none`, + witness: false, + tests: [{ ...judging(`a`, `fixed seed`, 1), fn: 0 }], + authored: null, + fault: `none`, + stability: `none`, + steps: null, + ordinary: false, + budget: `two`, + outMode: `explicit`, + logBlocked: false, + overlap: false, + logLost: `stop`, + unsupervised: true, + }), + }, + // Count-only kills (M, 2026-11-16) judge a test's count as a whole (v21.1 + // R19a): a test that reads the clock anywhere in its run has a count that + // could follow the clock. This heavy test (HEAVY_CALLS more calls of the + // until loop, about 110,000 site evaluations) reads it only after its + // work. Its copies' call budget is still a hundred times its unmutated + // work (G6), counted over every unmutated test; a copy that spins does + // so before the clock read, and it is a gap, not a bounded-work kill. + // The test judges nothing, so a copy that does a third more work (the + // loop's `<=`) is not judged, and no copy is killed by count. + { + name: `a heavy test that reads the clock after its work`, + enters: [ + `a spin of a test that reads the clock after its work timed out`, + ], + build: (generated) => ({ + ...soloKill(generated, [ + { + ...judging(`a`, `fixed seed`, 1), + fn: 0, + judges: false, + timed: `after`, + }, + ]), + functions: [[{ kind: `until`, quote: `'` }]], + heavy: HEAVY_CALLS, + }), + }, + // The same heavy test, reading the clock before its work and judging. + // Its copies' call budget is a hundred times its unmutated work, not + // CALL_FLOOR: a copy that does a third more work (past 100,000 site + // evaluations) runs to its end and is killed by the test's law, and only + // the copy that spins forever is a gap. + { + name: `a heavy test that reads the clock first`, + enters: [ + `a heavy clocked test's copy past the call floor killed by its law`, + ], + build: (generated) => ({ + ...soloKill(generated, [ + { + ...judging(`a`, `fixed seed`, 1), + fn: 0, + timed: `clock`, + }, + ]), + functions: [[{ kind: `until`, quote: `'` }]], + heavy: HEAVY_CALLS, + }), + }, + // No progress is decided per copy, by state and the test's own I/O (v21.2 + // R50, R51; M entries "Idle is decided by state only" and "Idle counts + // only the test's I/O"), never per file. The test reads a pipe it opened + // before it runs production, in every run, for two of the sampler's + // samples (a count), so its unmutated rows record I/O of its own. Its + // copy that hangs where the output changed has no I/O open then, so it + // is idle and killed by no progress, not cut by the backstop. + { + name: `a hang in a file whose test does a short read`, + enters: [ + `a hang in a file whose test did a short read killed by no progress`, + ], + build: (generated) => + soloKill(generated, [ + { + ...judging(`a`, `fixed seed`, 7), + fn: 0, + hangs: true, + pendingIo: `short`, + }, + ]), + }, + // The work budget charges only counts that cannot follow the clock (M, + // 2026-11-16, "Count-only kills"; v21.2 G6). Owner a's copy reads the + // clock first and makes 30,000 extra calls, and owner b's makes 30,000 + // too: the two together would pass the work budget, but a's count is + // charged nothing, so b's copy stays inside it, and the mutant that + // forces the if false is judged by the owners' laws: neither judges, so + // it is not judged. (The mutant that deletes the push makes each copy + // pass its call budget alone: a's spin is clocked, a gap, and b's kills.) + { + name: `a clocked owner beside a counted one under the work budget`, + enters: [`a clocked owner's work left out of the work budget`], + build: (generated) => ({ + ...generated, + functions: [[{ kind: `if`, op: `>`, k: 4, not: false, quote: `'` }]], + containers: [], + overloads: [], + owners: [`both`], + library: [], + libraryOwners: [], + runes: [], + derived: null, + dotGen: false, + dataOwners: [], + exportOwners: [], + importer: `none`, + wholeFile: `none`, + template: false, + tests: [ + { + ...judging(`a`, `fixed seed`, 7), + fn: 0, + judges: false, + timed: `clock`, + loops: 30_000, + }, + { + ...judging(`b`, `fixed seed`, 7), + fn: 0, + judges: false, + loops: 30_000, + }, + ], + witness: false, + authored: null, + secondAuthored: false, + fault: `none`, + stability: `none`, + ordinary: false, + native: `none`, + boot: false, + budget: `two`, + cliPath: `plain`, + concurrentStart: false, + contexts: 1, + steps: null, + // Its work budget decides its state, so its tests make no heavy calls. + heavy: 0, + }), + }, + // Clocked by state (C, 2026-11-17, under M "Count-only kills" and I15): + // a test whose own I/O completes during it has a count that could + // follow the clock, whatever the sampler saw. This test reads the + // package file asynchronously before production (a millisecond, which a + // 50 ms sample rarely sees), and its copy that spins (the until loop + // without its push) is a gap on every run, never a kill on one and a + // gap on another. + { + name: `a spin after a short async read`, + enters: [`a spin after a short async read timed out`], + build: (generated) => ({ + ...soloKill(generated, [ + { ...judging(`a`, `fixed seed`, 1), fn: 0, timed: `read` }, + ]), + functions: [[{ kind: `until`, quote: `'` }]], + }), + }, + // An authored mutant whose owner spins (issue #167, a random seed of #99): + // file b's verdict test reads its data (a sync read, so its count cannot + // follow the clock) and, while the data is open, spins on the function + // whose output the authored patch changed (`spinData`). Its copy is + // owed a kill by bounded work, by count (v21.2 R50, G6; M "Count-only + // kills"); the backstop must never decide it. + { + name: `an authored mutant whose owner spins`, + enters: [`an authored mutant killed by its spin`], + build: (generated) => ({ + ...generated, + functions: [[{ kind: `push`, quote: `'` }]], + containers: [], + overloads: [], + owners: [`b`], + library: [], + libraryOwners: [], + runes: [], + derived: null, + dotGen: false, + dataOwners: [], + exportOwners: [], + importer: `none`, + wholeFile: `none`, + template: false, + tests: [{ ...judging(`b`, `fixed seed`, 1), fn: 0 }], + witness: false, + authored: { stmt: 0, kind: `apply` }, + secondAuthored: false, + fault: `none`, + stability: `none`, + ordinary: false, + native: `none`, + boot: false, + budget: `two`, + cliPath: `plain`, + concurrentStart: false, + contexts: 1, + steps: null, + laterSteps: NO_LATER_STEPS, + fsRead: `sync`, + spinData: true, + spinThrows: false, + heavy: 0, + }), + }, + // A later owner's test listed first beside an earlier owner's hang (issue + // #182, the random seed of #99 that the capture drew). The history lists + // file b's test before the witness's tests, so a copy order taken from the + // listing would run b first. Owner a's copy of w_hangkill hangs where it + // would judge and is cut as no progress by state. Copies run in sorted owner + // order (register 2026-11-14, "Deterministic cuts, outcome rule"; G44), so + // a decides the mutant before b's copy spins on the function (`spinData`), + // which would be a cut by count. + { + name: `a later owner's test listed first beside an earlier owner's hang`, + enters: [`a copy cut as no progress before a later owner's spin`], + build: (generated) => ({ + ...soloKill(generated, [ + { ...judging(`b`, `fixed seed`, 1), fn: 0, judges: false }, + ]), + witness: `capture`, + spinData: true, + spinThrows: false, + fsRead: `sync`, + hangMs: null, + heavy: 0, + }), + }, ]), }, resolve: resolveHistory, diff --git a/packages/oracles/oracle-gaps/gap-report-static-oracle.property.test.mjs b/packages/oracles/oracle-gaps/gap-report-static-oracle.property.test.mjs index b370f24..a47880a 100644 --- a/packages/oracles/oracle-gaps/gap-report-static-oracle.property.test.mjs +++ b/packages/oracles/oracle-gaps/gap-report-static-oracle.property.test.mjs @@ -1849,5 +1849,10 @@ oracleProperty({ driver, check, calibrations, - timeout: 120_000, + // Each unit's wall-clock backstop: a test that runs past it fails loudly, + // and it decides no outcome of the CLI's (it is not a run budget). The + // generated units sat near 120 s on the CI runner (#144, #133, and #99's + // drawn `heavy`), so the backstop is 240 s (C ruling, 2026-11-17); cutting + // the generated unit's cost is #163. + timeout: 240_000, }) diff --git a/packages/oracles/oracle-gaps/history-fingerprint.json b/packages/oracles/oracle-gaps/history-fingerprint.json index 0dbfd65..c256339 100644 --- a/packages/oracles/oracle-gaps/history-fingerprint.json +++ b/packages/oracles/oracle-gaps/history-fingerprint.json @@ -13,74 +13,77 @@ ["ownsSpelling", "ee558b4907ec0610"], ["wholeFile", "0859b46b3c4572ac"], ["template", "1e253eba700355bc"], - ["tests", "e9029433a23d1f10"], - ["witness", "6f211b08cd814713"], - ["authored", "4b74f07a6bfdc5b0"], - ["declaration", "2ee70d6c3838fd00"], - ["ownerless", "540c73dc876d0786"], - ["fault", "ca08551d3690b334"], - ["stability", "bea8a50ec49b866a"], - ["ordinary", "9ad680df8aee1908"], - ["lazyGate", "b007abcf09d74433"], - ["secondAuthored", "347688cfe2dfc5ed"], - ["consoleOutput", "9740dbd24d665949"], - ["config", "85c9c9e20cdd9abd"], - ["configEnv", "29b6d3a0c09b9f38"], - ["setupGlobal", "36f832869df2229d"], - ["native", "aae1372e72126f9f"], - ["depSubpath", "5f11abd6f6990d18"], - ["unbuiltDep", "ece5968476f7042d"], - ["boot", "6fc1fdf2bd90d59e"], - ["budget", "d8481d5ff2e964b9"], - ["stopRuns", "e7a9ea343f07a902"], - ["onlyRun", "439d74b1930e4623"], - ["pragma", "731664592bd605c0"], - ["cliPath", "cba4d240a707ec0b"], - ["concurrentStart", "d55cbe3bd58d3fec"], - ["configFile", "9c1bbcd20191f1ae"], - ["configPlace", "894ac9262e6aeea6"], - ["packageType", "c45ce0c59c52806b"], - ["configAwait", "cda5d28fd16345ff"], - ["configRequire", "79654da1ed474074"], - ["contexts", "16320f05c7e90361"], - ["steps", "d2fc0a77ed3512fd"], - ["dataOwners", "dddde2f870a331e0"], - ["exportOwners", "b63c5a16d33b7d92"], - ["importer", "8cf68b28f1e26fa2"], - ["midrunBy", "f9286c40098474b3"], - ["midrunTarget", "00b0112258dd6b75"], - ["prePlugin", "7093bcf61828dea9"], - ["fsRead", "265eb24ce88fdbe2"], - ["laterSteps", "fb28f2575c00f223"], + ["tests", "02cea0c59a5c4124"], + ["witness", "0ee637313ae43f72"], + ["authored", "4065618afba83c7e"], + ["declaration", "0cdb1bb36f5758a4"], + ["ownerless", "8b008533250fe2df"], + ["fault", "f8a6c5e1031073a5"], + ["stability", "a9fdb0a14723da9c"], + ["ordinary", "6affcaa05d314dbd"], + ["lazyGate", "ed78e3cd8a4d484c"], + ["secondAuthored", "b044ac0d553a2346"], + ["consoleOutput", "570c3c461f1cdf68"], + ["config", "d37c3fcbce6d88ee"], + ["configEnv", "7ea7f6d3fb8934b5"], + ["setupGlobal", "e3c5a6eb134ed5dd"], + ["native", "a5485a0680fd7190"], + ["depSubpath", "94b7798f20d6ea5d"], + ["unbuiltDep", "012926f7259fc12b"], + ["boot", "77499b08354c31d4"], + ["budget", "a230a2b1135ae4e4"], + ["stopRuns", "aadc847695aedebf"], + ["onlyRun", "ea8aececdc69e59f"], + ["pragma", "8508183eaa171990"], + ["cliPath", "9d862ed7d15cd0c1"], + ["concurrentStart", "5c876b58eb50ce71"], + ["configFile", "3abc5f3c79733dee"], + ["configPlace", "4e95cd5e063067bb"], + ["packageType", "bb2e5c99af477ba8"], + ["configAwait", "b667d7659e71eaab"], + ["configRequire", "f811f9e78a974862"], + ["contexts", "d98701c946096a37"], + ["steps", "780917bc47ba4d78"], + ["dataOwners", "a57cca74a7d005fa"], + ["exportOwners", "854aa90d28d5de6b"], + ["importer", "dd2746fe9dac0584"], + ["midrunBy", "27f2bcbfbca5aa99"], + ["midrunTarget", "029c38dbaee03722"], + ["prePlugin", "eb9e3cc8b0fabb62"], + ["fsRead", "b73b40525abee45e"], + ["laterSteps", "186196e956323c41"], ["tornRows", "ed6866229ecc4969"], ["pauseEngages", "c93e2e40cd0c4b9f"], - ["warmReplaced", "84f62a790b9d0777"], - ["testRoot", "8f4dfb34f4e51863"], - ["loadPlugin", "1b19291d29a6e2b2"], + ["warmReplaced", "7e7206ee129451f2"], + ["testRoot", "7da5040bf592d7ac"], + ["loadPlugin", "ae7762aa7dc995e4"], ["rejectBatch", "e5386ea9402dd268"], - ["outMode", "6b85258d5ef18840"], - ["logBlocked", "88af484a06a50fc2"], + ["outMode", "1bbc6e2eac61af33"], + ["logBlocked", "47cc43be2da5e9c5"], ["overlap", "ed6866229ecc4969"], - ["stopWitness", "da3f0557cc2765bb"], + ["stopWitness", "8eb5a2ee804ce325"], ["spawnStop", "ed6866229ecc4969"], - ["spawnSignal", "137092f74f7b633b"], - ["spawnDelay", "b0781ec226c1e8fc"], + ["spawnSignal", "c0b64be12d1a3cc4"], + ["spawnDelay", "787b206a41863977"], ["spawnOnStart", "ed6866229ecc4969"], - ["claimScenario", "5ee5622f507f1bf8"], + ["claimScenario", "8c6c878c0dbc2346"], ["logLost", "ed6866229ecc4969"], ["supervisorDeath", "ed6866229ecc4969"], ["unsupervised", "ed6866229ecc4969"], - ["psMissing", "46766ca617b50e52"], + ["psMissing", "4c56fb8f46df9fd5"], ["holdOverlap", "ed6866229ecc4969"], ["replyHeld", "ed6866229ecc4969"], - ["spinData", "4352e9616b999e5d"], - ["siblingDir", "b30dea3f1825fc7c"], + ["spinData", "eafbb6b695fcb459"], + ["siblingDir", "fe167dd0870ece0c"], ["hookStops", "ed6866229ecc4969"], - ["spinThrows", "fbc4c4e88a554e03"], + ["spinThrows", "0c0ede6e889793cd"], ["slowStart", "ed6866229ecc4969"], ["shortWait", "ed6866229ecc4969"], - ["absentObjects", "b9e3dd12df07d597"], - ["optionalChains", "92f0ef0b4dde6c60"] + ["absentObjects", "819b4a9d10c98c0b"], + ["optionalChains", "d383282128c31644"], + ["hangMs", "0d53e8bb13182e66"], + ["heavy", "803af343d7fb884e"], + ["tests.pendingIo", "02f66613e9490d15"] ], "histories": [ ["functions", "66c0e9c56951c377"], @@ -96,73 +99,76 @@ ["ownsSpelling", "ee558b4907ec0610"], ["wholeFile", "0859b46b3c4572ac"], ["template", "1e253eba700355bc"], - ["tests", "e9029433a23d1f10"], - ["witness", "6f211b08cd814713"], - ["authored", "4b74f07a6bfdc5b0"], - ["declaration", "2ee70d6c3838fd00"], - ["ownerless", "540c73dc876d0786"], - ["fault", "ca08551d3690b334"], - ["stability", "bea8a50ec49b866a"], - ["ordinary", "9ad680df8aee1908"], - ["lazyGate", "b007abcf09d74433"], - ["secondAuthored", "347688cfe2dfc5ed"], - ["consoleOutput", "9740dbd24d665949"], - ["config", "85c9c9e20cdd9abd"], - ["configEnv", "29b6d3a0c09b9f38"], - ["setupGlobal", "36f832869df2229d"], - ["native", "aae1372e72126f9f"], - ["depSubpath", "5f11abd6f6990d18"], - ["unbuiltDep", "ece5968476f7042d"], - ["boot", "6fc1fdf2bd90d59e"], - ["budget", "d8481d5ff2e964b9"], - ["stopRuns", "1053061b3e4b0d5d"], - ["onlyRun", "439d74b1930e4623"], - ["pragma", "731664592bd605c0"], - ["cliPath", "cba4d240a707ec0b"], - ["concurrentStart", "d55cbe3bd58d3fec"], - ["configFile", "9c1bbcd20191f1ae"], - ["configPlace", "894ac9262e6aeea6"], - ["packageType", "c45ce0c59c52806b"], - ["configAwait", "cda5d28fd16345ff"], - ["configRequire", "79654da1ed474074"], - ["contexts", "16320f05c7e90361"], - ["steps", "cad4109ec56e04cd"], - ["dataOwners", "dddde2f870a331e0"], - ["exportOwners", "b63c5a16d33b7d92"], - ["importer", "8cf68b28f1e26fa2"], - ["midrunBy", "f9286c40098474b3"], - ["midrunTarget", "00b0112258dd6b75"], - ["prePlugin", "7093bcf61828dea9"], - ["fsRead", "265eb24ce88fdbe2"], - ["laterSteps", "fb28f2575c00f223"], + ["tests", "02cea0c59a5c4124"], + ["witness", "0ee637313ae43f72"], + ["authored", "4065618afba83c7e"], + ["declaration", "0cdb1bb36f5758a4"], + ["ownerless", "8b008533250fe2df"], + ["fault", "f8a6c5e1031073a5"], + ["stability", "a9fdb0a14723da9c"], + ["ordinary", "6affcaa05d314dbd"], + ["lazyGate", "ed78e3cd8a4d484c"], + ["secondAuthored", "b044ac0d553a2346"], + ["consoleOutput", "570c3c461f1cdf68"], + ["config", "d37c3fcbce6d88ee"], + ["configEnv", "7ea7f6d3fb8934b5"], + ["setupGlobal", "e3c5a6eb134ed5dd"], + ["native", "a5485a0680fd7190"], + ["depSubpath", "94b7798f20d6ea5d"], + ["unbuiltDep", "012926f7259fc12b"], + ["boot", "77499b08354c31d4"], + ["budget", "a230a2b1135ae4e4"], + ["stopRuns", "3b13d5c7b3af43bd"], + ["onlyRun", "ea8aececdc69e59f"], + ["pragma", "8508183eaa171990"], + ["cliPath", "9d862ed7d15cd0c1"], + ["concurrentStart", "5c876b58eb50ce71"], + ["configFile", "3abc5f3c79733dee"], + ["configPlace", "4e95cd5e063067bb"], + ["packageType", "bb2e5c99af477ba8"], + ["configAwait", "b667d7659e71eaab"], + ["configRequire", "f811f9e78a974862"], + ["contexts", "d98701c946096a37"], + ["steps", "7c64770273d56db6"], + ["dataOwners", "a57cca74a7d005fa"], + ["exportOwners", "854aa90d28d5de6b"], + ["importer", "dd2746fe9dac0584"], + ["midrunBy", "27f2bcbfbca5aa99"], + ["midrunTarget", "029c38dbaee03722"], + ["prePlugin", "eb9e3cc8b0fabb62"], + ["fsRead", "b73b40525abee45e"], + ["laterSteps", "186196e956323c41"], ["tornRows", "ed6866229ecc4969"], ["pauseEngages", "c93e2e40cd0c4b9f"], - ["warmReplaced", "84f62a790b9d0777"], - ["testRoot", "8f4dfb34f4e51863"], - ["loadPlugin", "1b19291d29a6e2b2"], + ["warmReplaced", "7e7206ee129451f2"], + ["testRoot", "7da5040bf592d7ac"], + ["loadPlugin", "ae7762aa7dc995e4"], ["rejectBatch", "e5386ea9402dd268"], - ["outMode", "6b85258d5ef18840"], - ["logBlocked", "88af484a06a50fc2"], + ["outMode", "1bbc6e2eac61af33"], + ["logBlocked", "47cc43be2da5e9c5"], ["overlap", "ed6866229ecc4969"], - ["stopWitness", "da3f0557cc2765bb"], + ["stopWitness", "8eb5a2ee804ce325"], ["spawnStop", "ed6866229ecc4969"], - ["spawnSignal", "137092f74f7b633b"], - ["spawnDelay", "b0781ec226c1e8fc"], + ["spawnSignal", "c0b64be12d1a3cc4"], + ["spawnDelay", "787b206a41863977"], ["spawnOnStart", "ed6866229ecc4969"], - ["claimScenario", "5ee5622f507f1bf8"], + ["claimScenario", "8c6c878c0dbc2346"], ["logLost", "ed6866229ecc4969"], ["supervisorDeath", "ed6866229ecc4969"], ["unsupervised", "ed6866229ecc4969"], - ["psMissing", "46766ca617b50e52"], + ["psMissing", "4c56fb8f46df9fd5"], ["holdOverlap", "ed6866229ecc4969"], ["replyHeld", "ed6866229ecc4969"], - ["spinData", "4352e9616b999e5d"], - ["siblingDir", "b30dea3f1825fc7c"], + ["spinData", "eafbb6b695fcb459"], + ["siblingDir", "fe167dd0870ece0c"], ["hookStops", "ed6866229ecc4969"], - ["spinThrows", "fbc4c4e88a554e03"], + ["spinThrows", "0c0ede6e889793cd"], ["slowStart", "ed6866229ecc4969"], ["shortWait", "ed6866229ecc4969"], - ["absentObjects", "b9e3dd12df07d597"], - ["optionalChains", "92f0ef0b4dde6c60"] + ["absentObjects", "819b4a9d10c98c0b"], + ["optionalChains", "d383282128c31644"], + ["hangMs", "0d53e8bb13182e66"], + ["heavy", "803af343d7fb884e"], + ["tests.pendingIo", "02f66613e9490d15"] ] } diff --git a/packages/oracles/oracle-gaps/index.mjs b/packages/oracles/oracle-gaps/index.mjs index 9284d98..b9d7f54 100755 --- a/packages/oracles/oracle-gaps/index.mjs +++ b/packages/oracles/oracle-gaps/index.mjs @@ -84,15 +84,13 @@ // [--ordinary] (also run ordinary test files, for information) // [--seed ] [--mutants ] [--out ] [--only ] // [--diff-base ] -// [--first-limit ] (a copy's first budget; default 2000) -// [--hang-limit ] (the full limit a cut copy gets again; default 10000) -// [--mutant-limit ] (the floor of each mutant's budget, which the -// runs that confirm its cut copies share; the -// budget is the larger of this and ten times -// its owner files' unmutated time; default 30000) -// [--test-timeout ] (default 15000; also bounds hooks and teardown. -// Each mutant run also has a wall-clock limit of -// 10 s plus three times its file's unmutated time.) +// [--hang-limit ] (the base of each copy's backstop, which adds +// 3 s per second of its file's unmutated time; +// default 60000) +// [--first-limit ] (retired: checked and noted, no effect) +// [--mutant-limit ] (retired: checked and noted, no effect) +// [--test-timeout ] (Vitest's per-test limit, also bounding hooks +// and teardown; default 65000) // [--sample ] (a fixed-seed sample of n mutants per source; // the report marks the run as partial) // [--contexts ] (warm contexts, one worker each; default @@ -638,17 +636,20 @@ async function checkThroughDaemon(options) { return reply.stdout } -// The run's ending, once: its refusal or error is written to its output -// directory and stdout and stderr, and the process exits with its code. +// The run's ending, once: its refusal, error or stop is written to stdout and +// stderr (and a refusal or error to its output directory), and the process +// exits with its code. Once an ending has begun, nothing ends the run again +// (P10, v21.2 I7): not a stop, and not the error of a log that can no longer +// be written. let ended = false const holdStops = () => { writingEnding = true process.send?.(`ending`) } -function endWith(error, out) { +function end(endingOf) { if (ended) return ended = true - const ending = endRun(error, out) + const ending = endingOf() holdStops() try { writeStream(ending.stderr) @@ -658,6 +659,7 @@ function endWith(error, out) { process.stdout.write(`${ending.done}\n`) exitWhenFlushed(ending.exitCode) } +const endWith = (error, out) => end(() => endRun(error, out)) // A file URL escapes a space, `#`, and the like, so the script's own URL is // compared with the URL of the path it was started as. @@ -675,10 +677,12 @@ const supervisor = process.env.ORACLE_GAPS_INTERNAL_SUPERVISED let runStarted = false /** Ends an unsupervised run with its stopped line on both streams (#58). */ function stopNow(signal) { - const line = `oracle-gaps: done stopped (${signal})\n` - process.stdout.write(line) - writeStream(line) - exitWhenFlushed(128 + constants.signals[signal]) + const done = `oracle-gaps: done stopped (${signal})` + end(() => ({ + stderr: `${done}\n`, + done, + exitCode: 128 + constants.signals[signal], + })) } async function run() { diff --git a/packages/oracles/oracle-gaps/stream.mjs b/packages/oracles/oracle-gaps/stream.mjs index 280bc7f..1d5de6b 100644 --- a/packages/oracles/oracle-gaps/stream.mjs +++ b/packages/oracles/oracle-gaps/stream.mjs @@ -54,9 +54,11 @@ export function change(result) { * and again in the summary. */ export function gapLine(result) { + // A timeout names the cut that ended it (classify.mjs `outcomeOf`). + const cut = result.outcome === `timeout` ? ` (cause: ${result.detail})` : `` return streamLine( `gap`, - `${kindOf(result)} L${result.line ?? `?`} ${result.id} ${change(result)}`, + `${kindOf(result)} L${result.line ?? `?`} ${result.id} ${change(result)}${cut}`, ) } diff --git a/packages/oracles/scripts/e2e-shards.mjs b/packages/oracles/scripts/e2e-shards.mjs index b236849..cfbbdb4 100644 --- a/packages/oracles/scripts/e2e-shards.mjs +++ b/packages/oracles/scripts/e2e-shards.mjs @@ -329,6 +329,11 @@ export const shards = { // with the daemon fix of fix/daemon-current-inputs. `fixed seed 20261105: an edit sequence over the witness module, the first half`, `fixed seed 20261105: an edit sequence over the witness module, the second half`, + // #182's constructed history, a later owner's test listed first beside an + // earlier owner's hang: 57 s here with GAP_E2E_SHRINKS=0. + `fixed seed 20261105: a later owner's test listed first beside an earlier owner's hang`, + `random seed: a later owner's test listed first beside an earlier owner's hang`, + `determinism: constructed history: a later owner's test listed first beside an earlier owner's hang`, ], // 522 s in the measured run, about 221 s idle. 'e2e-5': [ @@ -406,6 +411,62 @@ export const shards = { // with the daemon fix of fix/daemon-current-inputs. `random seed: an edit sequence over the witness module, the first half`, `random seed: an edit sequence over the witness module, the second half`, + // #99's deterministic-cuts and merge-gate histories. + `fixed seed 20261105: a slow owner's kill and another owner's kill`, + `fixed seed 20261105: a backstop cut ends the copies ahead of a later kill`, + `fixed seed 20261105: a kill past the work budget in a later owner`, + `fixed seed 20261105: a slow kill after a 16 s wait`, + `fixed seed 20261105: a copy waiting on a timer`, + `fixed seed 20261105: a copy waiting on an unref'd timer`, + `fixed seed 20261105: a copy waiting on a pipe it opened`, + `fixed seed 20261105: a copy beside the runner's pending write`, + `fixed seed 20261105: a copy beside the runner's pending timer`, + `fixed seed 20261105: a clocked copy that spins`, + `fixed seed 20261105: an authored mutant whose owner hangs`, + `fixed seed 20261105: a stream log lost after its note, then stopped, unsupervised`, + `fixed seed 20261105: a heavy test that reads the clock after its work`, + `fixed seed 20261105: a heavy test that reads the clock first`, + `fixed seed 20261105: a hang in a file whose test does a short read`, + `fixed seed 20261105: a clocked owner beside a counted one under the work budget`, + `fixed seed 20261105: a spin after a short async read`, + `random seed: a slow owner's kill and another owner's kill`, + `random seed: a backstop cut ends the copies ahead of a later kill`, + `random seed: a kill past the work budget in a later owner`, + `random seed: a slow kill after a 16 s wait`, + `random seed: a copy waiting on a timer`, + `random seed: a copy waiting on an unref'd timer`, + `random seed: a copy waiting on a pipe it opened`, + `random seed: a copy beside the runner's pending write`, + `random seed: a copy beside the runner's pending timer`, + `random seed: a clocked copy that spins`, + `random seed: an authored mutant whose owner hangs`, + `random seed: a stream log lost after its note, then stopped, unsupervised`, + `random seed: a heavy test that reads the clock after its work`, + `random seed: a heavy test that reads the clock first`, + `random seed: a hang in a file whose test does a short read`, + `random seed: a clocked owner beside a counted one under the work budget`, + `random seed: a spin after a short async read`, + `calibration: a stdout gap line names another cause`, + `determinism: constructed history: a slow owner's kill and another owner's kill`, + `determinism: constructed history: a backstop cut ends the copies ahead of a later kill`, + `determinism: constructed history: a kill past the work budget in a later owner`, + `determinism: constructed history: a slow kill after a 16 s wait`, + `determinism: constructed history: a copy waiting on a timer`, + `determinism: constructed history: a copy waiting on an unref'd timer`, + `determinism: constructed history: a copy waiting on a pipe it opened`, + `determinism: constructed history: a copy beside the runner's pending write`, + `determinism: constructed history: a copy beside the runner's pending timer`, + `determinism: constructed history: a clocked copy that spins`, + `determinism: constructed history: an authored mutant whose owner hangs`, + `determinism: constructed history: a stream log lost after its note, then stopped, unsupervised`, + `determinism: constructed history: a heavy test that reads the clock after its work`, + `determinism: constructed history: a heavy test that reads the clock first`, + `determinism: constructed history: a hang in a file whose test does a short read`, + `determinism: constructed history: a clocked owner beside a counted one under the work budget`, + `determinism: constructed history: a spin after a short async read`, + `fixed seed 20261105: an authored mutant whose owner spins`, + `random seed: an authored mutant whose owner spins`, + `determinism: constructed history: an authored mutant whose owner spins`, ], } diff --git a/packages/oracles/scripts/pr-shards.mjs b/packages/oracles/scripts/pr-shards.mjs index 4966ff5..19db3e6 100644 --- a/packages/oracles/scripts/pr-shards.mjs +++ b/packages/oracles/scripts/pr-shards.mjs @@ -467,6 +467,8 @@ export const partitions = { `a declaration fault: configNull`, `a declaration fault: missingSource`, `a config fault in a run that owns nothing`, + // Its slow copies each wait 10 s under #99 (86 s on the pool at 83a0b3a8, where this shard took 124 s and the rest shard 491 s). + 'a check through the daemon that works past the bound', ]), // 560 s of weight. 3: anyOf([ @@ -491,6 +493,8 @@ export const partitions = { `a declaration fault: configNoDefault`, `a declaration fault: projects`, `test.projects in a parent's config`, + // #99's 16 s wait (67 s on the pool; this shard took 147 s). + 'a slow kill after a 16 s wait', ]), // The rest: 560 s of weight, and any unit added later. 4: rest, @@ -532,6 +536,21 @@ export const partitions = { `constructed history: a start refuses a stale claim`, `constructed history: a declaration fault: configSyntax`, `constructed history: a declaration fault: nonLiteral`, + // #99's deterministic-cuts histories, the heavier (141 s on the pool at 83a0b3a8; this shard took 32 s). + 'constructed history: a slow kill after a 16 s wait', + 'constructed history: a backstop cut ends the copies ahead of a later kill', + "constructed history: a slow owner's kill and another owner's kill", + 'constructed history: a copy waiting on a timer', + "constructed history: a copy waiting on an unref'd timer", + // #164's two histories and the filed base, out of the rest shard (7), + // which took 12:03 on CI at cb1de4ff (P12). + 'constructed history: a run after a killed CLI', + 'constructed history: a stop before the CLI starts, after a stopped check', + 'fixed history: a filed base whose owner reads an absent object through ?.', + // Still 10:51 with only those three moved (run 44552467396), so two more + // of the rest shard's units (about 89 s on the pool) come here too. + 'constructed history: a daemon pause that cannot engage', + 'constructed history: every outcome class, the abort stop', ]), // 471 s of weight. 2: anyOf([ @@ -613,6 +632,20 @@ export const partitions = { `constructed history: a start refuses a dangling claim`, `constructed history: a declaration fault: setupThrows`, `constructed history: a refused run beside the CLI's directory, under a name that extends it`, + // #99's lighter histories (74 s on the pool; this shard took 205 s). + 'constructed history: a hang in a file whose test does a short read', + "constructed history: a copy beside the runner's pending timer", + 'constructed history: a copy waiting on a pipe it opened', + 'constructed history: an authored mutant whose owner hangs', + "constructed history: a copy beside the runner's pending write", + 'constructed history: a stream log lost after its note, then stopped, unsupervised', + 'constructed history: a kill past the work budget in a later owner', + 'constructed history: a clocked owner beside a counted one under the work budget', + 'constructed history: a heavy test that reads the clock after its work', + 'constructed history: a heavy test that reads the clock first', + 'constructed history: a spin after a short async read', + 'constructed history: an authored mutant whose owner spins', + 'constructed history: a clocked copy that spins', ]), // The rest: 471 s of weight, and any unit added later. 7: rest, @@ -835,7 +868,11 @@ export const partitions = { 'an edit sequence over the witness module, the second half', ]), 2: anyOf(['every outcome class']), - 3: anyOf(['a declaration fault: noTypescript']), + // #182's constructed history (one unit here, 57 s on the pool). + 3: anyOf([ + 'a declaration fault: noTypescript', + "a later owner's test listed first beside an earlier owner's hang", + ]), 4: anyOf(['every outcome class, the warm stop']), 5: anyOf([ 'an edit sequence through the daemon', @@ -858,6 +895,11 @@ export const partitions = { 'two runs on one output directory', 'a stream log that cannot be written', 'a start refuses an empty claim', + // #99's merge-gate witnesses: per-copy no progress, the clocked work-budget rows, clocked by state. + 'a hang in a file whose test does a short read', + 'a clocked owner beside a counted one under the work budget', + 'a spin after a short async read', + 'an authored mutant whose owner spins', ]), 7: anyOf([ 'a daemon pause that cannot engage', @@ -894,6 +936,9 @@ export const partitions = { 'a stop with the default output directory', 'a daemon check with the default output directory', 'a daemon check with the default output directory, unsupervised', + // #99's heavy-test witnesses. + 'a heavy test that reads the clock after its work', + 'a heavy test that reads the clock first', ]), 9: anyOf([ 'a lazy module loaded before a hang in the same batch', @@ -914,6 +959,8 @@ export const partitions = { 'a start refuses a dangling claim', 'a stop before the CLI starts, after a stopped check', 'a run after a killed CLI', + // #99's unsupervised stop after a lost stream log. + 'a stream log lost after its note, then stopped, unsupervised', ]), 10: anyOf([ 'an edit that changed the cold result after a hang replaced the only context', @@ -934,6 +981,18 @@ export const partitions = { 'a supervised check with no ps on its PATH', 'a value change at a ?. read and at a return', 'a ?. read of an object absent only at load', + // #99's deterministic-cuts histories (the 16 s wait is the longest). + "a slow owner's kill and another owner's kill", + 'a backstop cut ends the copies ahead of a later kill', + 'a kill past the work budget in a later owner', + 'a slow kill after a 16 s wait', + 'a copy waiting on a timer', + "a copy waiting on an unref'd timer", + 'a copy waiting on a pipe it opened', + "a copy beside the runner's pending write", + "a copy beside the runner's pending timer", + 'a clocked copy that spins', + 'an authored mutant whose owner hangs', ]), }, }, diff --git a/packages/oracles/tests/gap-report-history-fingerprint.test.ts b/packages/oracles/tests/gap-report-history-fingerprint.test.ts index 559f04d..0c71d33 100644 --- a/packages/oracles/tests/gap-report-history-fingerprint.test.ts +++ b/packages/oracles/tests/gap-report-history-fingerprint.test.ts @@ -28,9 +28,28 @@ import { drawn, histories } from '../oracle-gaps/gap-report-model-oracle.mjs' // resolved plans: a new field may still change what a plan does (the plan // takes `laterSteps` after a history's steps). The default fixed-seed // campaign draws 2 histories; since `warmReplaced`, `testRoot` and -// `loadPlugin` are drawn, those resolve with these fields' draws. +// `loadPlugin` are drawn, those resolve with these fields' draws. Under M +// "Randomness wins" (2026-11-16) a new field that changes what runs is drawn, +// so the fields after it shift: the PR that adds it records the lists anew +// and reports the fixed seed's reach states lost and gained (#99 drew the +// deterministic-cuts fields and appended `heavy`). const FIXED_SEED = 20261105 const DRAWS = 500 +// The constructed-only test fields: constants in every drawn test record, +// which no generated history sets. Since M "Randomness wins" (2026-11-16) +// only a field that switches a test-only hook stays one (`pendingIo`); the +// deterministic-cuts fields `slowMs`, `loops`, `timerWait` and `pulse` are +// drawn, and their draws shifted the later fields' lines (recorded anew). A +// test's record hashes without the constructed fields, and each one is +// recorded as its own line after the others, with its neutral value checked. +const CONSTRUCTED_TEST_FIELDS = [`pendingIo`] +const NEUTRAL_TEST_VALUES: Record = { + pendingIo: null, +} +type Sample = Record + +const hashOf = (value: unknown): string => + createHash(`sha1`).update(JSON.stringify(value)).digest(`hex`).slice(0, 16) const fingerprints = ( arbitrary: fc.Arbitrary, @@ -39,14 +58,37 @@ const fingerprints = ( const samples = fc.sample(arbitrary, { seed: FIXED_SEED, numRuns: DRAWS, - }) as Array> - return Object.keys(samples[0] ?? {}).map((key) => [ - key, - createHash(`sha1`) - .update(JSON.stringify(samples.map((history) => history[key]))) - .digest(`hex`) - .slice(0, 16), - ]) + }) as Array + const testsOf = (history: Sample) => history.tests as Array + const withoutConstructed = (record: Sample) => + Object.fromEntries( + Object.entries(record).filter( + ([key]) => !CONSTRUCTED_TEST_FIELDS.includes(key), + ), + ) + const lines: Array<[string, string]> = Object.keys(samples[0] ?? {}).map( + (key) => [ + key, + hashOf( + samples.map((history) => + key === `tests` + ? testsOf(history).map(withoutConstructed) + : history[key], + ), + ), + ], + ) + for (const field of CONSTRUCTED_TEST_FIELDS) { + const values = samples.flatMap((history) => + testsOf(history).map((record) => record[field]), + ) + // Generated tests hold the neutral value: only a constructed history sets it. + expect(values.every((value) => value === NEUTRAL_TEST_VALUES[field])).toBe( + true, + ) + lines.push([`tests.${field}`, hashOf(values)]) + } + return lines } describe(`the gap oracle's history grammar`, () => { @@ -174,16 +216,27 @@ describe(`the gap oracle's history grammar`, () => { } const hookSteps = [`dieMidCheck`, `resetBeforeRead`] const hookStops = [`after`, `warm`] + // A test's own hook switch (#99): what is pending while it waits, which + // switches ORACLE_GAPS_TEST_SAMPLES, ORACLE_GAPS_TEST_RUNNER_WRITE or + // ORACLE_GAPS_TEST_RUNNER_TIMER. + const neutralTest: Record = { pendingIo: null } for (const seed of [FIXED_SEED, 1, 2]) { const samples = fc.sample(histories, { seed, numRuns: DRAWS }) as Array< Record & { steps: Array | null stopRuns: Array<{ when: string }> + tests: Array> } > for (const history of samples) { for (const [field, value] of Object.entries(neutral)) expect([field, history[field]]).toEqual([field, value]) + for (const t of history.tests) + for (const [field, value] of Object.entries(neutralTest)) + expect([`tests.${field}`, t[field]]).toEqual([ + `tests.${field}`, + value, + ]) for (const step of history.steps ?? []) expect(hookSteps).not.toContain(step) for (const stop of history.stopRuns)