From ecbae849884ec8a3736cc42c4c1649d6ada26000 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Mon, 16 Nov 2026 11:57:17 +0000 Subject: [PATCH 01/29] test(oracles): law constructed independence in the framework self-oracle An appended pair, two synthetic properties alike except that the second's constructed list is the first's with one history appended, must run the same generated histories, in the same order, at every fixed seed and at the seeded run's random seed, and the same constructed histories of the shorter list. RED on main: the campaign's oneof weights the constructed histories by their count, so appending one moves the generated draws. Co-authored-by: Isaac --- .../tests/oracle-property-self-oracle.test.ts | 197 ++++++++++++++++++ 1 file changed, 197 insertions(+) diff --git a/packages/oracles/tests/oracle-property-self-oracle.test.ts b/packages/oracles/tests/oracle-property-self-oracle.test.ts index aa4e6046..55599fda 100644 --- a/packages/oracles/tests/oracle-property-self-oracle.test.ts +++ b/packages/oracles/tests/oracle-property-self-oracle.test.ts @@ -38,6 +38,16 @@ * fast-check itself, never through the framework. A unit that fails ran that stream up to its * first failure, then its shrinks. The random units of a property run the one seed it logged: the * seed of the replay environment in a seeded run, and a drawn seed in a seedless run. + * - `constructed independence` (maintainer decision, 2026-11-16, review of #121, F2): at a fixed + * seed, the generated histories a campaign runs do not depend on its constructed list, its + * length or its contents, and each constructed history runs as declared, independently of the + * generated draw. So appending a constructed history keeps every fixed seed's histories, which + * the decisions register's generator rules assume. Judged on an appended pair: two properties + * alike except that the second's constructed list is the first's with one history appended. At + * each fixed seed, and at the seeded run's random seed, the `generated` unit and the unit of each + * constructed history of the shorter list run the same histories in the same order in both, up + * to the shorter list's first failure, and the appended history's own unit runs. The law + * compares the two properties' own logs, so it holds without the model's streams. * - `verdict`: a test fails with a violation of law L exactly when its first * failing history violates L, whatever values the history holds (a bigint, * a Map, NaN, or -0 too). Within one history a driver error decides @@ -2860,8 +2870,12 @@ const timeoutReceivingKinds = [ const timeoutReceivingStates = timeoutReceivingKinds.map( (kind) => `timeout-input-${kind}` as const, ) +// Law `constructed independence`: the required states of an appended pair. +const appendedStates = [`appended-to-empty-list`, `appended-to-listed`] as const const frameworkStates = [ ...timeoutReceivingStates, + ...appendedStates, + `appended-pair-whole-stream`, `timeout-presentation-equivalence`, `timeout-continuation-cut`, // A slow spec that fails to load, in a plan that presents timeouts: the @@ -3905,6 +3919,41 @@ function frameworkReach( ) ) reached.add(`timeout-continuation-cut`) + // Law `constructed independence` compared a pair: both specs loaded, the + // shorter list's `generated` unit ran at its first fixed seed, and the + // appended history's unit ran. The shorter list is empty or holds some + // constructed histories; a `generated` unit that ran its whole stream + // without a failure compares every history it drew. + for (const pair of appendedPairs(plan, expected)) { + const [seed] = fixedSeedsOf(pair.listed) + const ran = (spec: SyntheticSpec, unit: string) => + (run?.log ?? []) + .filter((entry) => entry.test === `${spec.id} > ${unit}`) + .map((entry) => entry.history) + const generated = ran(pair.listed, fixedUnit(seed!, `generated`)) + if ( + !run?.files[pair.listed.id]?.loaded || + !run.files[pair.appended.id]?.loaded || + generated.length === 0 || + ran( + pair.appended, + fixedUnit(seed!, pair.appended.constructed.at(-1)!.name), + ).length === 0 + ) + continue + reached.add( + pair.listed.constructed.length === 0 + ? `appended-to-empty-list` + : `appended-to-listed`, + ) + if ( + generated.every( + (history) => + !failsUnit(pair.listed, fixedUnit(seed!, `generated`), history), + ) + ) + reached.add(`appended-pair-whole-stream`) + } return reached } @@ -6005,6 +6054,84 @@ const comparedRun = ( } } +// Law `constructed independence`. Two specs of one plan are an appended pair +// when the second is the first with one constructed history appended to its +// list: the same seed, runs, and every other part, whatever the appended +// history holds and whichever states it adds to the declaration. The relation +// is structural, so it holds for any suite that draws such a pair, not only +// for the constructed histories that build one. A slow spec is left out, as +// law `units` leaves it out: its driver runs on after its test timed out. +type AppendedPair = { listed: SyntheticSpec; appended: SyntheticSpec } +const withoutList = ({ + id: _id, + constructed: _constructed, + states: _states, + ...rest +}: SyntheticSpec) => JSON.stringify(rest) +function appendedPairs( + plan: Plan, + expected: ReturnType, +): Array { + const runs = plan.specs.filter( + (spec) => + spec.source.kind === `generated` && + !spec.slow && + expected.get(spec.id)!.loads, + ) + return runs.flatMap((listed) => + runs + .filter( + (appended) => + appended.constructed.length === listed.constructed.length + 1 && + JSON.stringify(appended.constructed.slice(0, -1)) === + JSON.stringify(listed.constructed) && + withoutList(appended) === withoutList(listed), + ) + .map((appended) => ({ listed, appended })), + ) +} +// What an appended pair ran, judged without the model's streams. At each +// fixed seed, and at the seeded child run's random seed, the `generated` unit +// and the unit of each constructed history of the shorter list ran at least +// one history in both specs, and the same histories in the same order, up to +// the shorter list's first failure (shrinks are not compared). The appended +// history's own unit ran too. Returns the first difference, or undefined. +function appendedDifference( + { listed, appended }: AppendedPair, + log: ReadonlyArray, +): { unit: string; listed: unknown; appended: unknown } | undefined { + const ran = (spec: SyntheticSpec, unit: string) => + log + .filter((entry) => entry.test === `${spec.id} > ${unit}`) + .map((entry) => entry.history) + const names = unitNames(listed) + const shared = [ + ...fixedSeedsOf(listed).flatMap((seed) => + names.map((name) => fixedUnit(seed, name)), + ), + ...names.map(randomUnit), + ] + for (const unit of shared) { + const before = ran(listed, unit) + const first = before.findIndex((history) => + failsUnit(listed, unit, history), + ) + const cut = first < 0 ? before.length : first + 1 + const after = ran(appended, unit) + const compared = first < 0 ? after : after.slice(0, cut) + if ( + before.length === 0 || + JSON.stringify(before.slice(0, cut)) !== JSON.stringify(compared) + ) + return { unit, listed: before.slice(0, cut), appended: compared } + } + const own = appended.constructed.at(-1)!.name + for (const seed of fixedSeedsOf(appended)) + if (ran(appended, fixedUnit(seed, own)).length === 0) + return { unit: fixedUnit(seed, own), listed: null, appended: [] } + return undefined +} + // Model law `replay-config`: the first error of the drawn replay // environment, in the order the law states, or none. function environmentError(plan: Plan): string | undefined { @@ -6576,6 +6703,19 @@ function checkSuite( `host RNG scope`, ) } + // Law `constructed independence`: appending a constructed history changes + // no history that the shorter list's campaigns ran. The comparison is + // between the two specs' own logs, so it needs no model stream. + for (const pair of appendedPairs(plan, expected)) { + const difference = appendedDifference(pair, run.log) + if (difference) + return violation( + `run`, + `${pair.appended.id} "${difference.unit}" ran ${JSON.stringify(difference.appended)} with one constructed history appended to the list of ${pair.listed.id}, which ran ${JSON.stringify(difference.listed)}`, + `constructed independence`, + { actual: difference.appended, expected: difference.listed }, + ) + } for (const spec of plan.specs) { const owed = expected.get(spec.id)! const seen = run.files[spec.id]! @@ -8143,6 +8283,54 @@ const unitWitnessSuite = ( environment: null, ...options, }) +// Law `constructed independence`: a pair of drafts, the second the first with +// one constructed history appended. The first draft's seed, runs, list, and +// naming come from the generated suite, so random seeds vary them; a drawn +// list is kept to the ordinary items, so each of its histories enters the +// state of its first item. The appended history's prefix comes from the +// suite's first fixed history, and it enters a state in some suites only, so +// the pair's declared states differ in those. An enumerated first draft +// gives the pair a generated source of its own. +const ordinaryPrefix = (history: SyntheticHistory | undefined, or: number) => { + const items = (history ?? []).slice(0, 2).map((entry) => Math.abs(entry) % 3) + return items.length > 0 ? items : [or] +} +const appendedPairSuite = (generated: Suite, listed: boolean): Suite => { + const base = generated.drafts[0]! + const tail = base.clock ? 1 : 2 + const constructed = listed + ? (base.constructed.length > 0 + ? base.constructed + : [{ prefix: [tail], enters: [] }] + ).map(({ prefix }) => { + const items = ordinaryPrefix(prefix, tail) + return { prefix: items, enters: [`saw-${items[0]}`] } + }) + : [] + const first = generatedDraft({ + source: + base.source.kind === `generated` + ? base.source + : { kind: `generated`, runs: 3, seed: 20 + tail }, + constructed, + constructedNames: + base.constructedNames === `prefixed` ? `prefixed` : `indexed`, + secondSeed: base.secondSeed, + isolatedPlans: base.isolatedPlans, + reachVia: base.reachVia, + }) + const prefix = ordinaryPrefix(base.fixed[0]?.history, tail) + return unitWitnessSuite([ + first, + { + ...first, + constructed: [ + ...constructed, + { prefix, enters: base.isolatedPlans ? [`saw-${prefix[0]}`] : [] }, + ], + }, + ]) +} const mainUnitRequiredConstructed: Array<{ name: string enters: ReadonlyArray<(typeof mainUnitRequiredStates)[number]> @@ -8334,9 +8522,11 @@ oracleProperty({ `bound-calibration`, `bound-calibration-registration-error`, `bound-constructed-driver`, + ...appendedStates, ], measured: frameworkStates.filter( (state) => + !appendedStates.some((required) => required === state) && state !== `timeout-presentation-equivalence` && state !== `timeout-continuation-cut` && !timeoutReceivingStates.some((required) => required === state) && @@ -9092,6 +9282,13 @@ oracleProperty({ enters: [`bound-selector-prototype-${kind}` as const], build: (generated: Suite) => selectorPrototypeSuite(generated, kind), })), + // Law `constructed independence`, after an empty list and after a + // drawn one (maintainer decision, 2026-11-16). + ...([false, true] as const).map((listed) => ({ + name: `appended constructed history after ${listed ? `a drawn list` : `none`}`, + enters: [listed ? `appended-to-listed` : `appended-to-empty-list`], + build: (generated: Suite) => appendedPairSuite(generated, listed), + })), ], fixed: [ { From 2b3b4842298cbba047511012b51251f75f23a2be Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Mon, 16 Nov 2026 13:28:18 +0000 Subject: [PATCH 02/29] fix(oracles): draw a campaign's generated histories apart from its constructed ones The generated unit ran fc.oneof over the arbitrary and each constructed history, weighted by the constructed count, so appending a constructed history moved every fixed seed's generated histories. The generated unit now draws from the arbitrary alone; each constructed history still runs in its own unit, built from the sample at seed plus its index. The self-oracle model's units rule drops the weighting (maintainer decision, 2026-11-16). Co-authored-by: Isaac --- .changeset/constructed-stream-independent.md | 7 ++++ packages/oracles/docs/oracle-checklist.md | 11 +++--- packages/oracles/src/oracle-property.ts | 29 +++++++-------- .../tests/oracle-property-self-oracle.test.ts | 35 +++++-------------- 4 files changed, 35 insertions(+), 47 deletions(-) create mode 100644 .changeset/constructed-stream-independent.md diff --git a/.changeset/constructed-stream-independent.md b/.changeset/constructed-stream-independent.md new file mode 100644 index 00000000..df12f26f --- /dev/null +++ b/.changeset/constructed-stream-independent.md @@ -0,0 +1,7 @@ +--- +'@tanstack/oracles': minor +--- + +A campaign's generated histories no longer depend on its constructed histories. The `generated` unit draws only from the property's arbitrary, so appending a constructed history, or changing one, keeps every fixed seed's generated histories, as the generator rules assume. Before, half of the generated unit's runs were constructed histories, weighted by how many there were, so appending one moved every run's draw. + +This shifts each fixed seed's `generated` unit once: a fixed seed now runs other generated histories than before, and a replay line printed before this change for a `generated` unit no longer replays. Each constructed history still runs in its own unit, built from the arbitrary's sample at the campaign's seed plus its index, unchanged. Constructed histories are no longer also explored over the `generated` unit's runs; a random seed builds each one from a new sample. diff --git a/packages/oracles/docs/oracle-checklist.md b/packages/oracles/docs/oracle-checklist.md index 31c11fd8..1e1df603 100644 --- a/packages/oracles/docs/oracle-checklist.md +++ b/packages/oracles/docs/oracle-checklist.md @@ -275,10 +275,13 @@ No requirement here can be met by declaring something out of reach. on, and a history MUST be constructed to enter each one. - **Framework:** Enforced. Every state in `states` must be named by a constructed history. Every campaign runs each constructed history, and one - that does not enter its named state fails. Half of the generated histories - are also constructed, so the histories after each state are explored. A - state that random histories reach only by chance goes in `measured`, which - is reported but not required. + that does not enter its named state fails. The generated histories are + drawn apart from the constructed ones, so at a fixed seed they do not + depend on the constructed list: appending a constructed history keeps + every fixed seed's generated histories. Each random seed builds the + constructed histories from new samples. A state that random histories + reach only by chance goes in `measured`, which is reported but not + required. - **Author:** Reach is a predicate over the plan or trace, or a production probe. A literal example is not reach. A `fixed` history is a regression, not reach. A law that needs a state entered at least N times in each diff --git a/packages/oracles/src/oracle-property.ts b/packages/oracles/src/oracle-property.ts index 010fb8a8..f5cd58b7 100644 --- a/packages/oracles/src/oracle-property.ts +++ b/packages/oracles/src/oracle-property.ts @@ -170,9 +170,15 @@ * constructed histories make reach required: a state that random histories * reach is measured, never gated. So every state in `states` must be named * by a constructed history, and a state the property only measures is - * declared in `measured` instead (ORC-015). Half of a campaign's generated - * histories are also constructed, so the histories after each required - * state are explored, not visited once. + * declared in `measured` instead (ORC-015). The constructed histories and + * the generated ones are independent streams: the `generated` unit draws + * only from the arbitrary, and constructed history k builds from the + * arbitrary's sample at the campaign's seed plus k. So at a fixed seed the + * generated histories do not depend on the constructed list, its length or + * its contents, and appending a constructed history changes no history that + * a fixed seed ran before (maintainer decision, 2026-11-16). A random seed + * builds each constructed history from a new sample, so the histories after + * a required state vary from run to run. * - A **fixed history** is a named history, such as a fixed regression. It may * name the states it `enters`, which its run must reach. Fixed and * constructed histories must be histories the arbitrary can take back @@ -2391,18 +2397,6 @@ function generatedTests( throw new Error( `${id}: no constructed history enters ${unconstructed.join(`, `)}. A required state needs a constructed history; a state that is only counted belongs in \`measured\`.`, ) - // Generated histories, half of them constructed. A constructed history - // shrinks through its generated input, as a mapped arbitrary does. - const arbitrary = - constructions.length === 0 - ? source.arbitrary - : fc.oneof( - { arbitrary: source.arbitrary, weight: constructions.length }, - ...constructions.map((construction) => ({ - arbitrary: source.arbitrary.map(construction.build), - weight: 1, - })), - ) for (const construction of constructions) { const undeclared = construction.enters.filter( (state) => declared !== undefined && !required.includes(state), @@ -2570,9 +2564,12 @@ function generatedTests( : timeout > 0 ? started + timeout / 2 : Infinity + // Every unit runs the property's own arbitrary: the `generated` unit + // draws from it alone, so the constructed list moves none of its + // histories, and a constructed unit's one history shrinks through it. const details = await fc.check( fc.asyncProperty( - arbitrary, + source.arbitrary, predicateFor(constructionOf, counts, shrinkUntil, rerun, violations), ), run, diff --git a/packages/oracles/tests/oracle-property-self-oracle.test.ts b/packages/oracles/tests/oracle-property-self-oracle.test.ts index 55599fda..866ba42d 100644 --- a/packages/oracles/tests/oracle-property-self-oracle.test.ts +++ b/packages/oracles/tests/oracle-property-self-oracle.test.ts @@ -33,8 +33,9 @@ * - `units` (maintainer decision, 2026-11-11, option D): a campaign's units run exactly the * histories that the campaign ran, each in its own test. The constructed unit of history k, for * seed S, runs one history: the construction built from the arbitrary's sample at seed S + k. - * The `generated` unit runs the stream that `fc.sample` draws from the property's arbitrary, with - * its constructed weighting, at seed S, for the property's `runs`. The model draws these with + * The `generated` unit runs the stream that `fc.sample` draws from the property's arbitrary alone, + * at seed S, for the property's `runs` (maintainer decision, 2026-11-16, which retired the + * constructed weighting of option D's design). The model draws these with * fast-check itself, never through the framework. A unit that fails ran that stream up to its * first failure, then its shrinks. The random units of a property run the one seed it logged: the * seed of the replay environment in a seeded run, and a drawn seed in a seedless run. @@ -1734,15 +1735,10 @@ const unitOf = (test: string): { seed?: number; name: string } | undefined => { } // The histories a unit draws at `seed`, in order: a constructed unit its one // construction, built from seed + its index; the `generated` unit the stream -// that fast-check samples from the property's arbitrary with its constructed -// weighting (each constructed history weighs 1, and the language weighs their -// count) for `runs` draws. Drawn with fast-check itself, never the framework. -// The weighting restates how production composes its arbitrary, which the -// design of option D authorizes the model to state (design-c-d.md, law -// `units`: "the property's arbitrary (with its constructed weighting)"). It -// is that composition, not a contract promise: the header promises only that -// half of a campaign's generated histories are constructed, and a change to -// the composition must change this model with it. +// that fast-check samples from the property's arbitrary alone, for `runs` +// draws. Drawn with fast-check itself, never the framework. The `generated` +// stream names no constructed history (law `constructed independence`, +// maintainer decision, 2026-11-16). function unitStream( spec: SyntheticSpec, seed: number, @@ -1754,22 +1750,7 @@ function unitStream( ) if (at >= 0) return [constructedAt(spec, spec.constructed[at]!.prefix, seed + at)] - const weighted = - spec.constructed.length === 0 - ? syntheticArbitrary - : fc.oneof( - { arbitrary: syntheticArbitrary, weight: spec.constructed.length }, - ...spec.constructed.map(({ prefix }) => ({ - arbitrary: syntheticArbitrary.map((items) => - [ - ...prefix, - ...items.map((item) => (spec.isolatedPlans ? item % 3 : item)), - ].slice(0, 3), - ), - weight: 1, - })), - ) - return fc.sample(weighted, { seed, numRuns: spec.source.runs }) + return fc.sample(syntheticArbitrary, { seed, numRuns: spec.source.runs }) } // The history each determinism unit runs twice: a generated source's fixed // histories, its constructed histories built from the first fixed seed, and From bdd7917e1bd44a85924b2a071c4bfe39b7ba8d24 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Mon, 16 Nov 2026 13:58:13 +0000 Subject: [PATCH 03/29] test(oracles): keep the self-oracle's split-verdict and appended histories in the language The appended pair kept a drawn list of two, so the appended draft held three constructed histories, which the draft grammar cannot take back: keep the first. The split-verdict witnesses used seed 5, whose stream failed under law four only through the retired constructed weighting; derive the seed from the arbitrary instead. Co-authored-by: Isaac --- .../tests/oracle-property-self-oracle.test.ts | 33 ++++++++++++++++--- 1 file changed, 28 insertions(+), 5 deletions(-) diff --git a/packages/oracles/tests/oracle-property-self-oracle.test.ts b/packages/oracles/tests/oracle-property-self-oracle.test.ts index 866ba42d..d36c516e 100644 --- a/packages/oracles/tests/oracle-property-self-oracle.test.ts +++ b/packages/oracles/tests/oracle-property-self-oracle.test.ts @@ -8267,7 +8267,7 @@ const unitWitnessSuite = ( // Law `constructed independence`: a pair of drafts, the second the first with // one constructed history appended. The first draft's seed, runs, list, and // naming come from the generated suite, so random seeds vary them; a drawn -// list is kept to the ordinary items, so each of its histories enters the +// list keeps its first history only, and only the ordinary items, so each of its histories enters the // state of its first item. The appended history's prefix comes from the // suite's first fixed history, and it enters a state in some suites only, so // the pair's declared states differ in those. An enumerated first draft @@ -8281,7 +8281,9 @@ const appendedPairSuite = (generated: Suite, listed: boolean): Suite => { const tail = base.clock ? 1 : 2 const constructed = listed ? (base.constructed.length > 0 - ? base.constructed + ? // The draft grammar holds at most two constructed histories, and + // the appended draft adds one. + base.constructed.slice(0, 1) : [{ prefix: [tail], enters: [] }] ).map(({ prefix }) => { const items = ordinaryPrefix(prefix, tail) @@ -8312,6 +8314,23 @@ const appendedPairSuite = (generated: Suite, listed: boolean): Suite => { }, ]) } +// The first seed of the grammar's range (1 to 999) whose generated stream of +// `runs` histories first fails under law `four`: its first history that holds +// a 3, a 4, or a 6 holds a 4 first, as `verdictOf` judges a plain spec. The +// seed is derived from the arbitrary, not written down, so a change in how a +// campaign draws its `generated` stream moves the seed and keeps the state it +// reaches (law `constructed independence` moved every stream once). +const seedFailingUnderFour = (runs: number): number => { + const fails = (history: SyntheticHistory) => + history.find((entry) => entry === 3 || entry === 4 || entry === 6) + for (let seed = 1; seed <= 999; seed++) { + const failing = fc + .sample(syntheticArbitrary, { seed, numRuns: runs }) + .find((history) => fails(history) !== undefined) + if (failing && fails(failing) === 4) return seed + } + throw new Error(`no seed from 1 to 999 fails first under law four`) +} const mainUnitRequiredConstructed: Array<{ name: string enters: ReadonlyArray<(typeof mainUnitRequiredStates)[number]> @@ -8325,7 +8344,11 @@ const mainUnitRequiredConstructed: Array<{ return unitWitnessSuite( [ generatedDraft({ - source: { kind: `generated`, runs: 2, seed: 5 }, + source: { + kind: `generated`, + runs: 2, + seed: seedFailingUnderFour(2), + }, secondSeed: true, constructedNames: `prefixed`, constructed: [ @@ -10999,14 +11022,14 @@ oracleProperty({ [`timeout`], generatedDraft({ timing: `slow`, fixed: [at([0])] }), ), - // Units (maintainer decision, 2026-11-11, option D). At seed 5, the + // Units (maintainer decision, 2026-11-11, option D). At the seed, the // generated unit's first failing history holds a 4 first, and the // constructed unit's history starts with a 3. fixedHistory( `each unit decides its own first failure`, [`unit-split-verdict`, `campaign-violation`, `replay-probe`], generatedDraft({ - source: { kind: `generated`, runs: 2, seed: 5 }, + source: { kind: `generated`, runs: 2, seed: seedFailingUnderFour(2) }, constructed: [{ prefix: [3], enters: [] }], }), ), From d01a3464067ceffd9d176b489d3a9ad9587afb59 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Mon, 16 Nov 2026 16:32:36 +0000 Subject: [PATCH 04/29] test(oracles): each constructed history runs its N bases from its own stream Extends law constructed independence and the units and campaign budget model rules (register M, 2026-11-16, "N bases per constructed history"): constructed history k at seed S runs the first N draws of the arbitrary's sample at S + k, N being the property's bases (default 5), and a bases that is not a whole number of at least 1 fails the load. The draft grammar draws bases and up to three constructed histories, and the appended pair keeps two before its append. RED on the one-base framework. Co-authored-by: Isaac --- .../oracle-property-self-oracle-fixture.ts | 7 + .../tests/oracle-property-self-oracle.test.ts | 203 ++++++++++++++---- 2 files changed, 174 insertions(+), 36 deletions(-) diff --git a/packages/oracles/tests/oracle-property-self-oracle-fixture.ts b/packages/oracles/tests/oracle-property-self-oracle-fixture.ts index 7bad497f..b38c217c 100644 --- a/packages/oracles/tests/oracle-property-self-oracle-fixture.ts +++ b/packages/oracles/tests/oracle-property-self-oracle-fixture.ts @@ -259,6 +259,12 @@ export type SyntheticSpec = { | { kind: `enumerated`; histories: ReadonlyArray } /** A second fixed seed, ten times the first, whose text the first's extends. */ secondSeed?: boolean + /** + * The bases each constructed history runs (the property's `bases`); absent + * leaves the framework's default. 0 and 1.5 are not whole numbers of at + * least 1, so they fail the load. + */ + bases?: number /** Required states; each must be entered by a constructed history. */ states: ReadonlyArray constructed: ReadonlyArray<{ @@ -1481,6 +1487,7 @@ export function syntheticProperty(spec: SyntheticSpec): void { .filter((state) => !spec.states.includes(state)) .map(declare), runs: spec.source.runs, + ...(spec.bases === undefined ? {} : { bases: spec.bases }), fixedSeeds: [ spec.source.seed, ...(spec.secondSeed ? [spec.source.seed * 10] : []), diff --git a/packages/oracles/tests/oracle-property-self-oracle.test.ts b/packages/oracles/tests/oracle-property-self-oracle.test.ts index d36c516e..eaf9c723 100644 --- a/packages/oracles/tests/oracle-property-self-oracle.test.ts +++ b/packages/oracles/tests/oracle-property-self-oracle.test.ts @@ -29,21 +29,27 @@ * wasted work. A generated source also fails to load when two of its tests would share a name: a * constructed history named `generated`, two constructed histories of one name, or two fixed * histories of one name (maintainer decision, 2026-11-11, option D). These come after its state - * checks and before law `membership`, in that order (analyst-chosen, 2026-11-11). + * checks and before law `membership`, in that order (analyst-chosen, 2026-11-11). A generated + * source whose `bases` is not a whole number of at least 1 fails to load, naming `bases`, after + * the pairing of `predict` with a probe-bound state and before its state checks (analyst-chosen, + * 2026-11-16). * - `units` (maintainer decision, 2026-11-11, option D): a campaign's units run exactly the * histories that the campaign ran, each in its own test. The constructed unit of history k, for - * seed S, runs one history: the construction built from the arbitrary's sample at seed S + k. + * seed S, runs N histories: the construction built from each of the first N draws of the + * arbitrary's sample at seed S + k, in order, where N is the property's `bases`, 5 by default + * (register M, 2026-11-16, "N bases per constructed history"). * The `generated` unit runs the stream that `fc.sample` draws from the property's arbitrary alone, - * at seed S, for the property's `runs` (maintainer decision, 2026-11-16, which retired the - * constructed weighting of option D's design). The model draws these with + * at seed S, for the property's `runs` (register M, 2026-11-16, "N bases per constructed + * history", which retired the constructed weighting of option D's design). The model draws these with * fast-check itself, never through the framework. A unit that fails ran that stream up to its * first failure, then its shrinks. The random units of a property run the one seed it logged: the * seed of the replay environment in a seeded run, and a drawn seed in a seedless run. - * - `constructed independence` (maintainer decision, 2026-11-16, review of #121, F2): at a fixed - * seed, the generated histories a campaign runs do not depend on its constructed list, its - * length or its contents, and each constructed history runs as declared, independently of the - * generated draw. So appending a constructed history keeps every fixed seed's histories, which - * the decisions register's generator rules assume. Judged on an appended pair: two properties + * - `constructed independence` (register M, 2026-11-16, "N bases per constructed history"; + * found by the review of #121, F2): at a fixed seed, the generated histories a campaign runs do + * not depend on its constructed list, its length or its contents, and each constructed history + * runs its N bases from a stream of its own, which neither the generated draws nor the rest of + * the list touch. So appending a constructed history keeps the fixed seed's generated histories + * and every other constructed history's bases. Judged on an appended pair: two properties * alike except that the second's constructed list is the first's with one history appended. At * each fixed seed, and at the seeded run's random seed, the `generated` unit and the unit of each * constructed history of the shorter list run the same histories in the same order in both, up @@ -149,9 +155,10 @@ * has its own determinism unit, which runs it twice and fails if its trace changes, * compared by their data encoding: `-0` is not `0`, and key order and * Map and Set order count, which a check that iterates them can see. - * - `campaign budget`: each passing constructed unit runs one example, and + * - `campaign budget`: each passing constructed unit runs one example per base (its property's + * `bases`, 5 by default), and * each passing generated unit runs its authored sample count. A wholly - * passing lane retains their sum: runs plus constructions, independently + * passing lane retains their sum: runs plus bases times constructions, independently * in ordinary/seedless and fixed/random lanes. Children discard parent replay * settings under both prefixes, so their multiplier is one. Path replay runs * one history; failures, shrinking, and timeouts owe no fixed call count. @@ -266,7 +273,10 @@ * unmapper, at the whole history or at each item; `.chain` and a filter are * not drawn, and a constructed history whose build leaves the language only * for some seeds is not drawn, so the run-time check of a later campaign's - * constructed histories has no witness here. The model learns which histories a test ran from the + * constructed histories has no witness here. A synthetic property holds at most three constructed + * histories, and an appended pair at most two before its append, so a design that changes only for + * longer lists has no witness; `bases` is drawn as the default, 1, 2, or 3, or as 0 or 1.5, which + * fail the load. The model learns which histories a test ran from the * synthetic driver's log, and relies on fast-check running a campaign's examples * (its constructed histories) first, in order. A replay or rerun probe covers * every failing campaign but only the first violating fixed or enumerated history of each @@ -495,6 +505,12 @@ type Draft = { enumeratedShape: `ok` | `extra key` | `probe state` nested: boolean strictModel: boolean + /** + * A generated source's `bases`: absent (the framework's default), a whole + * number of bases, or a value that is not a whole number of at least 1, + * which fails the load. + */ + bases: number | null } // `seedless`: one more child run, without `ORACLE_SEED`, records the routes // of each random campaign as an ordinary run draws its seed. @@ -656,9 +672,11 @@ const draft: fc.Arbitrary = fc.record({ weight: 1, }, ), + // Up to three, so a list long enough for a design that mixes only "when + // there are many" is drawn (review of #122, F5). constructed: fc.array( fc.record({ prefix: insideOrOutside(6), enters: fc.subarray(allStates) }), - { maxLength: 2 }, + { maxLength: 3 }, ), fixed: fc.array( fc.record( @@ -973,6 +991,13 @@ const draft: fc.Arbitrary = fc.record({ { weight: 12, arbitrary: fc.constant(false) }, { weight: 1, arbitrary: fc.constant(true) }, ), + // Most drafts keep the default number of bases; some declare one to three, + // and a few declare a value that fails the load. + bases: fc.oneof( + { weight: 6, arbitrary: fc.constant(null) }, + { weight: 3, arbitrary: fc.constantFrom(1, 2, 3) }, + { weight: 1, arbitrary: fc.constantFrom(0, 1.5) }, + ), }) const replayEnvironment: fc.Arbitrary = fc.record({ retired: fc.oneof( @@ -1192,6 +1217,7 @@ function resolveSuite({ id: syntheticIds[index]!, source: d.source, ...(generated && d.secondSeed ? { secondSeed: true } : {}), + ...(generated && d.bases !== null ? { bases: d.bases } : {}), states, constructed, fixed: generated @@ -1699,19 +1725,28 @@ const calibrationCandidates = ( ] } // Model law `units`. A synthetic constructed history is its prefix followed by -// a generated history, cut to three items; the generated history is the -// arbitrary's sample at the seed it is built from. -const constructedAt = ( +// a base, a generated history, cut to three items. +const builtFrom = ( spec: SyntheticSpec, prefix: SyntheticHistory, - seed: number, + base: SyntheticHistory, ) => [ ...prefix, - ...fc - .sample(syntheticArbitrary, { seed, numRuns: 1 })[0]! - .map((item) => (spec.isolatedPlans ? item % 3 : item)), + ...base.map((item) => (spec.isolatedPlans ? item % 3 : item)), ].slice(0, 3) +// A construction's first base: the arbitrary's first draw at `seed`, which +// registration, the determinism test, and the calibrations build from. +const constructedAt = ( + spec: SyntheticSpec, + prefix: SyntheticHistory, + seed: number, +) => + builtFrom( + spec, + prefix, + fc.sample(syntheticArbitrary, { seed, numRuns: 1 })[0]!, + ) // The seed every seeded child run gives the random units (`${prefix}_SEED`). const childSeed = 20251103 const fixedSeedsOf = (spec: SyntheticSpec): Array => @@ -1733,12 +1768,18 @@ const unitOf = (test: string): { seed?: number; name: string } | undefined => { const random = /^random seed: (.*)$/.exec(test) return random ? { name: random[1]! } : undefined } -// The histories a unit draws at `seed`, in order: a constructed unit its one -// construction, built from seed + its index; the `generated` unit the stream -// that fast-check samples from the property's arbitrary alone, for `runs` -// draws. Drawn with fast-check itself, never the framework. The `generated` -// stream names no constructed history (law `constructed independence`, -// maintainer decision, 2026-11-16). +// The number of bases each constructed history runs: the property's `bases`, +// or the framework's default of 5 (maintainer decision, register M, 2026-11-16, +// "N bases per constructed history"). +const defaultBases = 5 +const basesOf = (spec: SyntheticSpec) => spec.bases ?? defaultBases +// The histories a unit draws at `seed`, in order. Constructed history k's unit +// runs its construction over the first N draws of the arbitrary's sample at +// seed + k, N being its bases: a stream of its own, which neither the +// generated draws nor the rest of the list touch. The `generated` unit runs +// the stream that fast-check samples from the property's arbitrary alone, for +// `runs` draws. Drawn with fast-check itself, never the framework (law +// `units`, and law `constructed independence`). function unitStream( spec: SyntheticSpec, seed: number, @@ -1749,7 +1790,9 @@ function unitStream( (construction) => construction.name === name, ) if (at >= 0) - return [constructedAt(spec, spec.constructed[at]!.prefix, seed + at)] + return fc + .sample(syntheticArbitrary, { seed: seed + at, numRuns: basesOf(spec) }) + .map((base) => builtFrom(spec, spec.constructed[at]!.prefix, base)) return fc.sample(syntheticArbitrary, { seed, numRuns: spec.source.runs }) } // The history each determinism unit runs twice: a generated source's fixed @@ -2228,6 +2271,16 @@ function predictSpec( ) return { loads: false } if (generated) { + // A number of bases that is not a whole number of at least 1 would run a + // constructed history no times, or a part of one. + if ( + spec.bases !== undefined && + !(Number.isInteger(spec.bases) && spec.bases >= 1) + ) + return { + loads: false, + loadError: `${spec.id}: \`bases\` must be a whole number of at least 1`, + } const declared = new Set(spec.states) const entered = new Set( spec.constructed.flatMap((construction) => construction.enters), @@ -2851,12 +2904,20 @@ const timeoutReceivingKinds = [ const timeoutReceivingStates = timeoutReceivingKinds.map( (kind) => `timeout-input-${kind}` as const, ) -// Law `constructed independence`: the required states of an appended pair. -const appendedStates = [`appended-to-empty-list`, `appended-to-listed`] as const +// Law `constructed independence`: the required states of an appended pair, +// and of constructed units that run their bases (laws `units` and `campaign +// budget`): the default number, and a number the property declares. +const appendedStates = [ + `appended-to-empty-list`, + `appended-to-listed`, + `constructed-default-bases`, + `constructed-declared-bases`, +] as const const frameworkStates = [ ...timeoutReceivingStates, ...appendedStates, `appended-pair-whole-stream`, + `bases-load-error`, `timeout-presentation-equivalence`, `timeout-continuation-cut`, // A slow spec that fails to load, in a plan that presents timeouts: the @@ -3155,7 +3216,7 @@ function frameworkReach( ? spec.source.kind === `generated` ? spec.source.runs : 0 - : 1 + : basesOf(spec) if (test?.state !== `passed` || calls.length !== count) return false total += calls.length reached.add( @@ -3163,7 +3224,10 @@ function frameworkReach( ) return true }) - if (complete && total === spec.source.runs + spec.constructed.length) { + if ( + complete && + total === spec.source.runs + basesOf(spec) * spec.constructed.length + ) { reached.add(`passing-budget-${child.context.kind}-${lane}`) if (spec.constructed.length > 0) reached.add(`passing-budget-constructed`) @@ -3935,6 +3999,35 @@ function frameworkReach( ) reached.add(`appended-pair-whole-stream`) } + // A constructed unit that passed after running its property's bases, each + // base a history of its own; a property without `bases` runs the default. + for (const spec of plan.specs) { + const seen = run?.files[spec.id] + if (spec.source.kind !== `generated` || spec.slow) continue + if ( + !expected.get(spec.id)!.loads && + seen?.loadErrors.some((message) => message.includes(`\`bases\``)) + ) + reached.add(`bases-load-error`) + const [seed] = fixedSeedsOf(spec) + if (!seen?.loaded || seed === undefined || spec.constructed.length === 0) + continue + const ranBases = spec.constructed.every(({ name }) => { + const unit = fixedUnit(seed, name) + return ( + seen.tests.find((test) => test.name === unit)?.state === `passed` && + (run?.log ?? []).filter( + (entry) => entry.test === `${spec.id} > ${unit}`, + ).length === basesOf(spec) + ) + }) + if (ranBases) + reached.add( + spec.bases === undefined + ? `constructed-default-bases` + : `constructed-declared-bases`, + ) + } return reached } @@ -6364,7 +6457,7 @@ function childExpectation( ? 1 : unitOf(test.name)!.name === `generated` ? spec.source.runs - : 1 + : basesOf(spec) if (actualCalls !== owedCalls) return { failure: violation( @@ -7843,6 +7936,7 @@ const generatedDraft = (overrides: Partial): Draft => ({ labelStyle: `plain`, nested: false, strictModel: false, + bases: null, ...overrides, }) const fixedHistory = ( @@ -8267,7 +8361,7 @@ const unitWitnessSuite = ( // Law `constructed independence`: a pair of drafts, the second the first with // one constructed history appended. The first draft's seed, runs, list, and // naming come from the generated suite, so random seeds vary them; a drawn -// list keeps its first history only, and only the ordinary items, so each of its histories enters the +// list keeps its first two histories, and only the ordinary items, so each of its histories enters the // state of its first item. The appended history's prefix comes from the // suite's first fixed history, and it enters a state in some suites only, so // the pair's declared states differ in those. An enumerated first draft @@ -8281,9 +8375,9 @@ const appendedPairSuite = (generated: Suite, listed: boolean): Suite => { const tail = base.clock ? 1 : 2 const constructed = listed ? (base.constructed.length > 0 - ? // The draft grammar holds at most two constructed histories, and - // the appended draft adds one. - base.constructed.slice(0, 1) + ? // The draft grammar holds at most three constructed histories, + // and the appended draft adds one. + base.constructed.slice(0, 2) : [{ prefix: [tail], enters: [] }] ).map(({ prefix }) => { const items = ordinaryPrefix(prefix, tail) @@ -8331,6 +8425,34 @@ const seedFailingUnderFour = (runs: number): number => { } throw new Error(`no seed from 1 to 999 fails first under law four`) } +// A passing campaign whose constructed histories run their bases: ordinary +// items only (isolated plans keep the bases ordinary too), one or two +// constructed histories, and, when declared, the generated suite's own number +// of bases, or 1 to 3 from its seed when it declares none or an invalid one. +const basesSuite = (generated: Suite, declared: boolean): Suite => { + const base = generated.drafts[0]! + const tail = base.clock ? 1 : 2 + const drawn = + base.bases !== null && Number.isInteger(base.bases) && base.bases >= 1 + ? base.bases + : 1 + ((base.source.kind === `generated` ? base.source.seed : tail) % 3) + return unitWitnessSuite([ + generatedDraft({ + source: + base.source.kind === `generated` + ? base.source + : { kind: `generated`, runs: 2, seed: 30 + tail }, + isolatedPlans: true, + constructed: [ + { prefix: [0], enters: [`saw-0`] }, + ...(base.constructed.length > 1 + ? [{ prefix: [tail], enters: [] }] + : []), + ], + ...(declared ? { bases: drawn } : {}), + }), + ]) +} const mainUnitRequiredConstructed: Array<{ name: string enters: ReadonlyArray<(typeof mainUnitRequiredStates)[number]> @@ -9293,6 +9415,15 @@ oracleProperty({ enters: [listed ? `appended-to-listed` : `appended-to-empty-list`], build: (generated: Suite) => appendedPairSuite(generated, listed), })), + // Laws `units` and `campaign budget`: each constructed history runs its + // bases, the default or a declared number (register M, 2026-11-16). + ...([false, true] as const).map((declared) => ({ + name: `constructed histories run ${declared ? `declared` : `default`} bases`, + enters: [ + declared ? `constructed-declared-bases` : `constructed-default-bases`, + ], + build: (generated: Suite) => basesSuite(generated, declared), + })), ], fixed: [ { From 656442183fd6c7375f1be0658f4fe8606c8cecee Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Mon, 16 Nov 2026 16:34:33 +0000 Subject: [PATCH 05/29] feat(oracles): each constructed history runs N bases from its own stream Constructed history k at seed S runs its construction over the first N draws of the arbitrary's sample at S + k, N being histories.bases (default 5; a value that is not a whole number of at least 1 fails the load). The stream is independent of the generated draws and of the rest of the list, so appending a constructed history keeps every other one's bases, and each constructed history is explored again (register M, 2026-11-16, "N bases per constructed history"). Also corrects the scheduler oracle's prose. Co-authored-by: Isaac --- .changeset/constructed-stream-independent.md | 4 +- packages/oracles/docs/oracle-checklist.md | 11 +-- packages/oracles/src/oracle-property.ts | 79 ++++++++++++++----- ...property-scheduler-oracle.property.test.ts | 7 +- 4 files changed, 70 insertions(+), 31 deletions(-) diff --git a/.changeset/constructed-stream-independent.md b/.changeset/constructed-stream-independent.md index df12f26f..51ec490a 100644 --- a/.changeset/constructed-stream-independent.md +++ b/.changeset/constructed-stream-independent.md @@ -2,6 +2,6 @@ '@tanstack/oracles': minor --- -A campaign's generated histories no longer depend on its constructed histories. The `generated` unit draws only from the property's arbitrary, so appending a constructed history, or changing one, keeps every fixed seed's generated histories, as the generator rules assume. Before, half of the generated unit's runs were constructed histories, weighted by how many there were, so appending one moved every run's draw. +A campaign's generated histories no longer depend on its constructed histories, and each constructed history runs several bases from a stream of its own. The `generated` unit draws only from the property's arbitrary. Constructed history k builds from each of the first `bases` draws of the arbitrary's sample at the campaign's seed plus k, and every base must enter the states it names. The new `histories.bases` option sets that number. It must be a whole number of at least 1 (default 5), or the property fails to load. So appending a constructed history keeps every fixed seed's generated histories and every other constructed history's bases. Before, half of the generated unit's runs were constructed histories, weighted by how many there were, so appending one moved every run's draw, and each constructed unit ran one history. -This shifts each fixed seed's `generated` unit once: a fixed seed now runs other generated histories than before, and a replay line printed before this change for a `generated` unit no longer replays. Each constructed history still runs in its own unit, built from the arbitrary's sample at the campaign's seed plus its index, unchanged. Constructed histories are no longer also explored over the `generated` unit's runs; a random seed builds each one from a new sample. +This shifts each fixed seed once, for properties that have at least one constructed history. Their `generated` unit now runs other histories, and a seed-and-path replay line printed before this change for that unit no longer replays the same history. A constructed unit keeps its first base, and its first replay path index still selects it. Replay lines for properties without constructed histories stay valid: their `generated` unit already drew from the arbitrary alone. Each constructed unit now runs `bases` histories, so a campaign takes longer; set `bases` lower for a property whose constructed histories are slow. diff --git a/packages/oracles/docs/oracle-checklist.md b/packages/oracles/docs/oracle-checklist.md index 1e1df603..dc6cb520 100644 --- a/packages/oracles/docs/oracle-checklist.md +++ b/packages/oracles/docs/oracle-checklist.md @@ -275,11 +275,12 @@ No requirement here can be met by declaring something out of reach. on, and a history MUST be constructed to enter each one. - **Framework:** Enforced. Every state in `states` must be named by a constructed history. Every campaign runs each constructed history, and one - that does not enter its named state fails. The generated histories are - drawn apart from the constructed ones, so at a fixed seed they do not - depend on the constructed list: appending a constructed history keeps - every fixed seed's generated histories. Each random seed builds the - constructed histories from new samples. A state that random histories + that does not enter its named state, on any of its bases, fails. Each + constructed history runs `bases` bases (5 by default) from a stream of its + own, and the generated histories are drawn apart from the constructed ones. + So at a fixed seed, appending a constructed history keeps the generated + histories and every other constructed history's bases, and the histories + after each required state are explored, not visited once. A state that random histories reach only by chance goes in `measured`, which is reported but not required. - **Author:** Reach is a predicate over the plan or trace, or a production diff --git a/packages/oracles/src/oracle-property.ts b/packages/oracles/src/oracle-property.ts index f5cd58b7..42d9bbc5 100644 --- a/packages/oracles/src/oracle-property.ts +++ b/packages/oracles/src/oracle-property.ts @@ -165,28 +165,32 @@ * driver must await the work it starts. * - A **constructed history** is built from a generated history so that it * reaches states that the laws need, and it names those declared states. - * Every campaign runs every constructed history once, in its own unit, and a - * constructed history that does not enter a named state fails. Only + * Every campaign runs every constructed history in its own unit, over a + * number of bases (`bases`, 5 by default), and a constructed history that + * does not enter a named state, on any base, fails. Only * constructed histories make reach required: a state that random histories * reach is measured, never gated. So every state in `states` must be named * by a constructed history, and a state the property only measures is * declared in `measured` instead (ORC-015). The constructed histories and * the generated ones are independent streams: the `generated` unit draws - * only from the arbitrary, and constructed history k builds from the - * arbitrary's sample at the campaign's seed plus k. So at a fixed seed the - * generated histories do not depend on the constructed list, its length or - * its contents, and appending a constructed history changes no history that - * a fixed seed ran before (maintainer decision, 2026-11-16). A random seed - * builds each constructed history from a new sample, so the histories after - * a required state vary from run to run. + * only from the arbitrary, and constructed history k builds from each of + * the first `bases` draws of the arbitrary's sample at the campaign's seed + * plus k. So at a fixed seed the generated histories do not depend on the + * constructed list, its length or its contents, appending a constructed + * history changes no other constructed history's bases, and each + * constructed history is explored over `bases` histories, not visited once + * (register M, 2026-11-16, "N bases per constructed history"). A history + * inserted mid-list moves the bases of the ones after it, so new + * constructed histories go at the end of the list. * - A **fixed history** is a named history, such as a fixed regression. It may * name the states it `enters`, which its run must reach. Fixed and * constructed histories must be histories the arbitrary can take back * (`canShrinkWithoutContext`), so they shrink like generated ones and cannot * cover a dimension that the grammar lacks; such a dimension belongs in the * grammar, or in the property's stated limits. The property checks this when - * it registers, for every fixed history and for the constructed histories - * built from the first fixed seed (or seed 0 when there is none), so a + * it registers, for every fixed history and for the first base of each + * constructed history at the first fixed seed (or seed 0 when there is + * none); a campaign checks every base it builds. So a * constructed history's `build` runs when the test file loads. It fails to * load naming the history, and a `build` that throws there fails to load * naming its constructed history @@ -604,6 +608,13 @@ export interface GeneratedHistories { /** One fixed-seed campaign per seed, beside the random campaign. */ fixedSeeds: ReadonlyArray constructed?: ReadonlyArray> + /** + * How many bases each constructed history runs in each campaign: the first + * `bases` draws of the arbitrary's sample at the campaign's seed plus the + * history's index, each built into one history. A whole number of at least + * 1; 5 when absent. + */ + bases?: number fixed?: ReadonlyArray> } @@ -1777,10 +1788,23 @@ function buildConstruction( seed: number, registering = false, ): THistory { - let history: THistory + return buildBases(id, arbitrary, construction, seed, 1, registering)[0]! +} + +/** A construction built from each of the first `count` draws at `seed`. */ +function buildBases( + id: string, + arbitrary: fc.Arbitrary, + construction: ConstructedHistory, + seed: number, + count: number, + registering = false, +): Array { + let histories: Array try { - const [generated] = fc.sample(arbitrary, { seed, numRuns: 1 }) - history = construction.build(generated!) + histories = fc + .sample(arbitrary, { seed, numRuns: count }) + .map((generated) => construction.build(generated)) } catch (error) { if (!registering) throw error throw new Error( @@ -1788,9 +1812,9 @@ function buildConstruction( { cause: error }, ) } - if (!arbitrary.canShrinkWithoutContext(history)) + if (histories.some((history) => !arbitrary.canShrinkWithoutContext(history))) throw untakeable(id, `constructed history "${construction.name}" built`) - return history + return histories } /** Stable across list insertion and reordering; no process-random replay route. */ @@ -2386,6 +2410,13 @@ function generatedTests( ) const options: BoundOptions = probes.size > 0 ? { ...unbound, probes } : unbound + // Each constructed history runs this many bases in every campaign (register + // M, 2026-11-16, "N bases per constructed history"). + const bases = source.bases ?? 5 + if (!(Number.isInteger(bases) && bases >= 1)) + throw new Error( + `${id}: \`bases\` must be a whole number of at least 1, the number of bases each constructed history runs in a campaign; it is ${reportText(bases)}.`, + ) const required = (source.states ?? []).map(stateName) const constructions = source.constructed ?? [] const declared = declaredOf(source) @@ -2494,6 +2525,11 @@ function generatedTests( registering, ) } + // A constructed unit's histories: its construction over the first `bases` + // draws of its own stream, the sample at seed + index, which neither the + // generated draws nor the other constructed histories touch. + const constructedBases = (seed: number, index: number) => + buildBases(id, source.arbitrary, constructions[index]!, seed + index, bases) // Registration checks every fixed history, then the constructed histories // built from the first fixed seed (or seed 0 when there is none), so a // history that cannot shrink fails the load rather than a full run. @@ -2508,8 +2544,8 @@ function generatedTests( const unitNames = [...constructedNames, `generated`] /** * Runs one unit of a campaign whose parameters are `base`. A constructed - * unit runs its one history, built from the campaign's seed, as an - * example with no generated runs; the `generated` unit runs the + * unit runs its bases, built from the campaign's seed, as examples with + * no generated runs; the `generated` unit runs the * campaign's generated runs, with no examples, so fast-check draws the * same histories that the whole campaign drew. A unit reports its reach * counts after it passes, and its rerun and replay commands when it fails. @@ -2534,12 +2570,13 @@ function generatedTests( const { reporter: _reporter, asyncReporter: _asyncReporter, ...rest } = base let run: typeof rest = rest if (index >= 0) { - const history = constructedAt(seed, index) + const histories = constructedBases(seed, index) // fast-check passes example inputs by reference, which identifies them. - constructionOf.set(history, constructions[index]!) + for (const history of histories) + constructionOf.set(history, constructions[index]!) run = withOracleFixedHistories( run.path === undefined ? { ...run, numRuns: 0 } : run, - [[history]], + histories.map((history) => [history]), ) } oracleRoute(oracleRouteOf(run)) diff --git a/packages/oracles/tests/oracle-property-scheduler-oracle.property.test.ts b/packages/oracles/tests/oracle-property-scheduler-oracle.property.test.ts index a52c38e9..2b1356c2 100644 --- a/packages/oracles/tests/oracle-property-scheduler-oracle.property.test.ts +++ b/packages/oracles/tests/oracle-property-scheduler-oracle.property.test.ts @@ -24,9 +24,10 @@ * task through `scheduleFrom`, waits until idle, and records the release * order and each task's outcome, or that `scheduleFrom` threw. The check * compares them. The declared state `a choice that names no queue` is entered - * by a constructed history that puts one first, so half of each campaign - * reaches it; the generated choices are mostly valid, so the release order is - * checked in most of the rest. + * by a constructed history that puts one first, which each campaign runs + * over its bases (5 by default); the generated choices are mostly valid, so + * they reach it only by chance, and the release order is checked in most of + * them. * * Limits: tasks are scheduled up front; tasks scheduled by released tasks' * continuations are not generated. From b91168ad3023a80be9c32cb290e55875a9e6f110 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Mon, 16 Nov 2026 16:39:28 +0000 Subject: [PATCH 06/29] test(oracle-gaps): a constructed history enters authored agreement-only-kill The classify oracle measured the state, and its fixed seed's generated histories stopped reaching it once constructed histories left the generated unit. It is a G11 failure-gap class, so it is now required and entered by its own constructed history, appended at the end of the list. Co-authored-by: Isaac --- .../gap-report-classify-oracle.property.test.mjs | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/packages/oracles/oracle-gaps/gap-report-classify-oracle.property.test.mjs b/packages/oracles/oracle-gaps/gap-report-classify-oracle.property.test.mjs index 5765025d..40343d47 100644 --- a/packages/oracles/oracle-gaps/gap-report-classify-oracle.property.test.mjs +++ b/packages/oracles/oracle-gaps/gap-report-classify-oracle.property.test.mjs @@ -3230,6 +3230,7 @@ const required = [ `a survivor whose work fell`, `extra-work named past an unregistered rise`, `a rise exactly at the declared bound`, + `authored agreement-only-kill`, ] // One constructed history per outcome class: one mutant judged by one @@ -4000,6 +4001,18 @@ const constructed = [ baseline: `load`, }, ), + // An authored mutant that only the reach agreement rejects: a G11 + // failure-gap class, so a constructed history enters it (review of #122; + // the fixed seed's generated histories stopped reaching it once the + // constructed histories left the generated unit). + single( + `an authored mutant killed by reach agreement alone`, + [`authored agreement-only-kill`], + { + effect: { kind: `verdict`, law: `reach agreement` }, + authored: [{}], + }, + ), ] oracleProperty({ @@ -4016,7 +4029,6 @@ oracleProperty({ measured: [ ...[ `timeout`, - `agreement-only-kill`, `unregistered-kill`, `crashed`, `framework-error`, From 378b5e7435334bd6154d3e2e9f34d8c81c674dab Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Mon, 16 Nov 2026 17:25:07 +0000 Subject: [PATCH 07/29] test(oracles): one base per constructed history in the two end-to-end oracles The gap report's end-to-end oracle and the framework self-oracle run real CLI and Vitest processes in each constructed history. At the default of five bases their fixed-seed shards took 18 to 30 and 27 minutes on an Arclet, past the 10-minute CI limit, so each declares bases: 1. Co-authored-by: Isaac --- .../oracles/oracle-gaps/gap-report-oracle.property.test.mjs | 5 +++++ packages/oracles/tests/oracle-property-self-oracle.test.ts | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs b/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs index 0dcce7d9..d819e6bc 100644 --- a/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs +++ b/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs @@ -4576,6 +4576,11 @@ oracleProperty({ `a check that ran more than one batch`, ], runs: 2, + // One base per constructed history: each runs the CLI end to end, often + // for minutes, and five bases took a fixed-seed shard to 18 to 30 + // minutes on an Arclet, past the 10-minute CI limit (#122). Random seeds + // vary each one's base from run to run. + bases: 1, fixedSeeds: [20261105], constructed: withDeclaration([ { diff --git a/packages/oracles/tests/oracle-property-self-oracle.test.ts b/packages/oracles/tests/oracle-property-self-oracle.test.ts index eaf9c723..3c62f736 100644 --- a/packages/oracles/tests/oracle-property-self-oracle.test.ts +++ b/packages/oracles/tests/oracle-property-self-oracle.test.ts @@ -8666,6 +8666,11 @@ oracleProperty({ state !== `registration-warning-policy`, ), runs: 3, + // One base per constructed history: each spawns real Vitest runs, and + // five bases took the shard of campaign units to 27 minutes on an + // Arclet, past the 10-minute CI limit (#122). Random seeds vary each + // one's base from run to run. + bases: 1, fixedSeeds: [1], constructed: [ { From 774d915f5c95141cd2cac18b2403775ebdf12a4c Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Mon, 16 Nov 2026 17:55:22 +0000 Subject: [PATCH 08/29] test(oracle-gaps): three bases per constructed history in the static oracle Five bases took the static oracle's larger fixed-seed shard from 116 s to 530 s on an Arclet, close to the 10-minute CI limit. Co-authored-by: Isaac --- .../oracle-gaps/gap-report-static-oracle.property.test.mjs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/packages/oracles/oracle-gaps/gap-report-static-oracle.property.test.mjs b/packages/oracles/oracle-gaps/gap-report-static-oracle.property.test.mjs index 5ccab440..72a4fa47 100644 --- a/packages/oracles/oracle-gaps/gap-report-static-oracle.property.test.mjs +++ b/packages/oracles/oracle-gaps/gap-report-static-oracle.property.test.mjs @@ -1450,6 +1450,10 @@ oracleProperty({ arbitrary: argumentHistories, states: required, runs: 40, + // Three bases per constructed history: each runs the CLI on a fixture + // package, and five bases took the larger fixed-seed shard from 116 s to + // 530 s on an Arclet, close to the 10-minute CI limit (#122). + bases: 3, fixedSeeds: [20261105], constructed: withDeclaration([ { From ba8ec15cc43bfd6a0426b9bd19784d1d644f8204 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Mon, 16 Nov 2026 17:58:10 +0000 Subject: [PATCH 09/29] docs(oracles): cite the register's N-bases entry as the self-oracle's authority Co-authored-by: Isaac --- packages/oracles/tests/oracle-property-self-oracle.test.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/oracles/tests/oracle-property-self-oracle.test.ts b/packages/oracles/tests/oracle-property-self-oracle.test.ts index 3c62f736..a9a88461 100644 --- a/packages/oracles/tests/oracle-property-self-oracle.test.ts +++ b/packages/oracles/tests/oracle-property-self-oracle.test.ts @@ -265,7 +265,9 @@ * the e2e speed plan, with Q2: one seed per property per process, Q3: a * `${prefix}_UNIT` variable that requires `${prefix}_PROPERTY`, and Q4: a path * replay of a generated source without `UNIT` fails at registration, and - * every replay line names its unit). + * every replay line names its unit); and the register's M entry of 2026-11-16, "N bases per + * constructed history" (v21.2 K18, with C-6 for appending at the end of the list), for laws + * `constructed independence`, `units`, and `campaign budget`. * * Limits: synthetic properties with histories of one to three integers, held * as an array, a bigint-tagged object, or a Map, one or two per run. An @@ -9414,7 +9416,7 @@ oracleProperty({ build: (generated: Suite) => selectorPrototypeSuite(generated, kind), })), // Law `constructed independence`, after an empty list and after a - // drawn one (maintainer decision, 2026-11-16). + // drawn one (register M, 2026-11-16, "N bases per constructed history"). ...([false, true] as const).map((listed) => ({ name: `appended constructed history after ${listed ? `a drawn list` : `none`}`, enters: [listed ? `appended-to-listed` : `appended-to-empty-list`], From 04cf1b62a5c24ad545fc5917a4778d6542cef796 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Tue, 17 Nov 2026 11:24:42 +0000 Subject: [PATCH 10/29] test(oracles): a costly property's constructed unit runs base 0 plus bases drawn from the run's seed Register M, 2026-11-17, "Random bases at today's cost for expensive oracles": after its fixed bases, a constructed unit runs drawnBases more, draws N to N + D - 1 of the sample at the run's random seed + k, so the random unit runs them at the same indices and a drawn base replays with that seed. The first unit that uses the run's seed logs it. The draft grammar draws drawnBases, and a constructed history enters constructed-drawn-bases. RED on the fixed-bases framework. Co-authored-by: Isaac --- .../oracle-property-self-oracle-fixture.ts | 9 + .../tests/oracle-property-self-oracle.test.ts | 164 ++++++++++++++---- 2 files changed, 140 insertions(+), 33 deletions(-) diff --git a/packages/oracles/tests/oracle-property-self-oracle-fixture.ts b/packages/oracles/tests/oracle-property-self-oracle-fixture.ts index b38c217c..f05231a9 100644 --- a/packages/oracles/tests/oracle-property-self-oracle-fixture.ts +++ b/packages/oracles/tests/oracle-property-self-oracle-fixture.ts @@ -265,6 +265,12 @@ export type SyntheticSpec = { * least 1, so they fail the load. */ bases?: number + /** + * The bases each constructed history draws from the run's random seed, + * after its fixed ones (the property's `drawnBases`); absent leaves none. + * -1 and 0.5 are not whole numbers of at least 0, so they fail the load. + */ + drawnBases?: number /** Required states; each must be entered by a constructed history. */ states: ReadonlyArray constructed: ReadonlyArray<{ @@ -1488,6 +1494,9 @@ export function syntheticProperty(spec: SyntheticSpec): void { .map(declare), runs: spec.source.runs, ...(spec.bases === undefined ? {} : { bases: spec.bases }), + ...(spec.drawnBases === undefined + ? {} + : { drawnBases: spec.drawnBases }), fixedSeeds: [ spec.source.seed, ...(spec.secondSeed ? [spec.source.seed * 10] : []), diff --git a/packages/oracles/tests/oracle-property-self-oracle.test.ts b/packages/oracles/tests/oracle-property-self-oracle.test.ts index a9a88461..f60fd9a9 100644 --- a/packages/oracles/tests/oracle-property-self-oracle.test.ts +++ b/packages/oracles/tests/oracle-property-self-oracle.test.ts @@ -31,13 +31,18 @@ * histories of one name (maintainer decision, 2026-11-11, option D). These come after its state * checks and before law `membership`, in that order (analyst-chosen, 2026-11-11). A generated * source whose `bases` is not a whole number of at least 1 fails to load, naming `bases`, after - * the pairing of `predict` with a probe-bound state and before its state checks (analyst-chosen, - * 2026-11-16). + * the pairing of `predict` with a probe-bound state and before its state checks, and then one + * whose `drawnBases` is not a whole number of at least 0, naming `drawnBases` (analyst-chosen, + * 2026-11-16 and 2026-11-17). * - `units` (maintainer decision, 2026-11-11, option D): a campaign's units run exactly the * histories that the campaign ran, each in its own test. The constructed unit of history k, for * seed S, runs N histories: the construction built from each of the first N draws of the * arbitrary's sample at seed S + k, in order, where N is the property's `bases`, 5 by default - * (register M, 2026-11-16, "N bases per constructed history"). + * (register M, 2026-11-16, "N bases per constructed history"). Then it runs D more, D being the + * property's `drawnBases` (none by default): draws N to N + D - 1 of the sample at R + k, where R + * is the run's random seed, the one the random units run (register M, 2026-11-17, "Random bases + * at today's cost for expensive oracles"). So the random unit of the same name, at R, runs those + * histories at the same indices, and a drawn base's failure replays with seed R. * The `generated` unit runs the stream that `fc.sample` draws from the property's arbitrary alone, * at seed S, for the property's `runs` (register M, 2026-11-16, "N bases per constructed * history", which retired the constructed weighting of option D's design). The model draws these with @@ -156,7 +161,7 @@ * compared by their data encoding: `-0` is not `0`, and key order and * Map and Set order count, which a check that iterates them can see. * - `campaign budget`: each passing constructed unit runs one example per base (its property's - * `bases`, 5 by default), and + * `bases`, 5 by default, plus its `drawnBases`), and * each passing generated unit runs its authored sample count. A wholly * passing lane retains their sum: runs plus bases times constructions, independently * in ordinary/seedless and fixed/random lanes. Children discard parent replay @@ -194,7 +199,8 @@ * above. * - `timeout`: a property's `timeout` limits every test it registers, each unit * on its own. - * - `replay`: a failing campaign unit prints a replay command: the seed, the + * - `replay`: a failing campaign unit prints a replay command: the seed (for a fixed-seed unit + * whose failing example is a drawn base, the run's random seed), the * path, the property, and `${prefix}_UNIT` naming the random unit of the same * name (`random seed: `), under the configured prefix, then that unit's * `-t` pattern (maintainer decisions, 2026-11-11, Q3 and Q4). A failing random @@ -245,10 +251,11 @@ * `calibration`; a property without one registers none. The gap report reads it to judge the * test's work under a mutant; a plain run has no unmodified run to compare with, so there the * bound judges nothing (maintainer decision, 2026-11-12: a speed shortcut is owed a cost law). - * - `seed-log`: a property logs exactly one `ORACLE_SEED` line, in its first - * random unit, and no other test logs one; every random unit runs that seed - * (maintainer decision, 2026-11-11, Q2: one seed per property per process). A - * replay tool reads that line. + * - `seed-log`: a property logs exactly one `ORACLE_SEED` line, in the first unit that uses the + * run's random seed, and no other test logs one; every random unit runs that seed (maintainer + * decision, 2026-11-11, Q2: one seed per property per process). That unit is the first random + * unit, or, in a property that draws bases, its first fixed-seed constructed unit (register M, + * 2026-11-17, "Random bases at today's cost for expensive oracles"). A replay tool reads that line. * * Authority: the contract in the header of `oracle-property.ts`, including * its state probes (user decision of 2026-11-04 adopting differential reach); the @@ -513,6 +520,11 @@ type Draft = { * which fails the load. */ bases: number | null + /** + * A generated source's `drawnBases`: absent (none), one or two, or a value + * that is not a whole number of at least 0, which fails the load. + */ + drawnBases: number | null } // `seedless`: one more child run, without `ORACLE_SEED`, records the routes // of each random campaign as an ordinary run draws its seed. @@ -1000,6 +1012,13 @@ const draft: fc.Arbitrary = fc.record({ { weight: 3, arbitrary: fc.constantFrom(1, 2, 3) }, { weight: 1, arbitrary: fc.constantFrom(0, 1.5) }, ), + // Most drafts draw no base from the run's seed; some draw one or two, and + // a few declare a value that fails the load. + drawnBases: fc.oneof( + { weight: 6, arbitrary: fc.constant(null) }, + { weight: 3, arbitrary: fc.constantFrom(1, 2) }, + { weight: 1, arbitrary: fc.constantFrom(-1, 0.5) }, + ), }) const replayEnvironment: fc.Arbitrary = fc.record({ retired: fc.oneof( @@ -1220,6 +1239,9 @@ function resolveSuite({ source: d.source, ...(generated && d.secondSeed ? { secondSeed: true } : {}), ...(generated && d.bases !== null ? { bases: d.bases } : {}), + ...(generated && d.drawnBases !== null + ? { drawnBases: d.drawnBases } + : {}), states, constructed, fixed: generated @@ -1775,10 +1797,18 @@ const unitOf = (test: string): { seed?: number; name: string } | undefined => { // "N bases per constructed history"). const defaultBases = 5 const basesOf = (spec: SyntheticSpec) => spec.bases ?? defaultBases +// The bases each constructed history draws from the run's random seed, after +// its fixed ones (register M, 2026-11-17, "Random bases at today's cost for +// expensive oracles"); none when the property declares none. +const drawnOf = (spec: SyntheticSpec) => spec.drawnBases ?? 0 +// The histories each constructed unit runs in all. +const constructedRuns = (spec: SyntheticSpec) => basesOf(spec) + drawnOf(spec) // The histories a unit draws at `seed`, in order. Constructed history k's unit // runs its construction over the first N draws of the arbitrary's sample at // seed + k, N being its bases: a stream of its own, which neither the -// generated draws nor the rest of the list touch. The `generated` unit runs +// generated draws nor the rest of the list touch. Then it runs D drawn bases, +// D being its drawn bases: draws N to N + D - 1 of the sample at the run's +// random seed + k. The `generated` unit runs // the stream that fast-check samples from the property's arbitrary alone, for // `runs` draws. Drawn with fast-check itself, never the framework (law // `units`, and law `constructed independence`). @@ -1786,15 +1816,31 @@ function unitStream( spec: SyntheticSpec, seed: number, name: string, + // The run's random seed, which the random units run and the drawn bases + // come from. + runSeed: number, ): Array { if (spec.source.kind !== `generated`) return [] const at = spec.constructed.findIndex( (construction) => construction.name === name, ) - if (at >= 0) - return fc - .sample(syntheticArbitrary, { seed: seed + at, numRuns: basesOf(spec) }) - .map((base) => builtFrom(spec, spec.constructed[at]!.prefix, base)) + if (at >= 0) { + const fixedBases = fc.sample(syntheticArbitrary, { + seed: seed + at, + numRuns: basesOf(spec), + }) + // The drawn bases follow the fixed ones in the run seed's own stream, so + // the random unit, whose seed is the run's, runs them at the same index. + const drawn = fc + .sample(syntheticArbitrary, { + seed: runSeed + at, + numRuns: constructedRuns(spec), + }) + .slice(basesOf(spec)) + return [...fixedBases, ...drawn].map((base) => + builtFrom(spec, spec.constructed[at]!.prefix, base), + ) + } return fc.sample(syntheticArbitrary, { seed, numRuns: spec.source.runs }) } // The history each determinism unit runs twice: a generated source's fixed @@ -2283,6 +2329,14 @@ function predictSpec( loads: false, loadError: `${spec.id}: \`bases\` must be a whole number of at least 1`, } + if ( + spec.drawnBases !== undefined && + !(Number.isInteger(spec.drawnBases) && spec.drawnBases >= 0) + ) + return { + loads: false, + loadError: `${spec.id}: \`drawnBases\` must be a whole number of at least 0`, + } const declared = new Set(spec.states) const entered = new Set( spec.constructed.flatMap((construction) => construction.enters), @@ -2423,13 +2477,19 @@ function predictSpec( ...fixedSeedsOf(spec).flatMap((seed) => unitNames(spec).map( (name) => - [fixedUnit(seed, name), unitStream(spec, seed, name)] as const, + [ + fixedUnit(seed, name), + unitStream(spec, seed, name, randomSeed), + ] as const, ), ), ...(generated ? unitNames(spec).map( (name) => - [randomUnit(name), unitStream(spec, randomSeed, name)] as const, + [ + randomUnit(name), + unitStream(spec, randomSeed, name, randomSeed), + ] as const, ) : []), ]) @@ -2914,6 +2974,7 @@ const appendedStates = [ `appended-to-listed`, `constructed-default-bases`, `constructed-declared-bases`, + `constructed-drawn-bases`, ] as const const frameworkStates = [ ...timeoutReceivingStates, @@ -3218,7 +3279,7 @@ function frameworkReach( ? spec.source.kind === `generated` ? spec.source.runs : 0 - : basesOf(spec) + : constructedRuns(spec) if (test?.state !== `passed` || calls.length !== count) return false total += calls.length reached.add( @@ -3903,7 +3964,12 @@ function frameworkReach( const { ran, owedRun } = comparedRun( spec, unit, - unitStream(spec, seen.seeds[0]!, unitOf(unit)!.name), + unitStream( + spec, + seen.seeds[0]!, + unitOf(unit)!.name, + seen.seeds[0]!, + ), seen.ran[unit] ?? [], ) return ( @@ -4008,7 +4074,7 @@ function frameworkReach( if (spec.source.kind !== `generated` || spec.slow) continue if ( !expected.get(spec.id)!.loads && - seen?.loadErrors.some((message) => message.includes(`\`bases\``)) + seen?.loadErrors.some((message) => message.includes(`ases\` must be`)) ) reached.add(`bases-load-error`) const [seed] = fixedSeedsOf(spec) @@ -4020,15 +4086,18 @@ function frameworkReach( seen.tests.find((test) => test.name === unit)?.state === `passed` && (run?.log ?? []).filter( (entry) => entry.test === `${spec.id} > ${unit}`, - ).length === basesOf(spec) + ).length === constructedRuns(spec) ) }) - if (ranBases) + if (ranBases) { reached.add( spec.bases === undefined ? `constructed-default-bases` : `constructed-declared-bases`, ) + // A fixed-seed unit that also ran bases drawn from the run's seed. + if (drawnOf(spec) > 0) reached.add(`constructed-drawn-bases`) + } } return reached } @@ -6259,7 +6328,7 @@ function environmentSelection(plan: Plan, spec: SyntheticSpec) { } function environmentUnitOutcome(spec: SyntheticSpec, unit: string): Outcome { - const history = unitStream(spec, 5, unitOf(unit)!.name)[0]! + const history = unitStream(spec, 5, unitOf(unit)!.name, 5)[0]! const ordinary = dataOutcome(spec) ?? (spec.slow ? { kind: `timeout` as const } : judged(spec, history)) @@ -6459,7 +6528,7 @@ function childExpectation( ? 1 : unitOf(test.name)!.name === `generated` ? spec.source.runs - : basesOf(spec) + : constructedRuns(spec) if (actualCalls !== owedCalls) return { failure: violation( @@ -7005,11 +7074,16 @@ function checkSuite( `registration`, { actual: seenNames, expected: names }, ) - const firstRandom = - spec.source.kind === `generated` - ? randomUnit(unitNames(spec)[0]!) - : undefined - const loggedSeed = seen.tests.find(({ name }) => name === firstRandom) + // The first unit that uses the run's random seed selects and logs it: the + // first fixed-seed constructed unit when the property draws bases from + // it, and otherwise the first random unit (law `seed-log`). + const seedUnit = + spec.source.kind !== `generated` + ? undefined + : drawnOf(spec) > 0 && spec.constructed.length > 0 + ? fixedUnit(fixedSeedsOf(spec)[0]!, spec.constructed[0]!.name) + : randomUnit(unitNames(spec)[0]!) + const loggedSeed = seen.tests.find(({ name }) => name === seedUnit) ?.seedLogs[0] for (const test of seen.tests) { const outcome = owed.tests.get(test.name)! @@ -7266,9 +7340,9 @@ function checkSuite( `${spec.id} "${test.name}" printed a History line that does not show the history`, `verdict`, ) - // A property selects its seed once, in its first random unit, and - // logs that selection once; every random unit runs it. - const owedSeedLogs = test.name === firstRandom ? 1 : 0 + // A property selects its seed once, in the first unit that uses it, + // and logs that selection once; every random unit runs it. + const owedSeedLogs = test.name === seedUnit ? 1 : 0 if (test.seedLogs.length !== owedSeedLogs) return violation( `run`, @@ -7502,7 +7576,7 @@ function checkSuite( const { ran: logged, owedRun } = comparedRun( spec, unit, - unitStream(spec, seeds[0]!, unitOf(unit)!.name), + unitStream(spec, seeds[0]!, unitOf(unit)!.name, seeds[0]!), ran[unit] ?? [], ) if (JSON.stringify(logged) !== JSON.stringify(owedRun)) @@ -7842,7 +7916,7 @@ function checkEnvironment( `${spec.id} ran ${JSON.stringify(running.map((test) => test.name))} in a path replay that selects ${JSON.stringify(unit ?? null)}, owed ${JSON.stringify(owedRunning)}`, ) if (owedRunning.length === 0) continue - const [history] = unitStream(spec, 5, unitOf(unit!)!.name) + const [history] = unitStream(spec, 5, unitOf(unit!)!.name, 5) const ran = under.log .filter((entry) => entry.test === `${spec.id} > ${unit}`) .map((entry) => entry.history) @@ -7939,6 +8013,7 @@ const generatedDraft = (overrides: Partial): Draft => ({ nested: false, strictModel: false, bases: null, + drawnBases: null, ...overrides, }) const fixedHistory = ( @@ -8455,6 +8530,21 @@ const basesSuite = (generated: Suite, declared: boolean): Suite => { }), ]) } +// The suite of `basesSuite` with one fixed base, as a costly property +// declares, and one or two bases drawn from the run's random seed. +const drawnBasesSuite = (generated: Suite): Suite => { + const withBases = basesSuite(generated, true) + return { + ...withBases, + drafts: [ + { + ...withBases.drafts[0]!, + bases: 1, + drawnBases: generated.drafts[0]!.clock ? 1 : 2, + }, + ], + } +} const mainUnitRequiredConstructed: Array<{ name: string enters: ReadonlyArray<(typeof mainUnitRequiredStates)[number]> @@ -9431,6 +9521,14 @@ oracleProperty({ ], build: (generated: Suite) => basesSuite(generated, declared), })), + // Each costly property's constructed history runs base 0 at the fixed + // seed and bases drawn from the run's random seed (register M, + // 2026-11-17, "Random bases at today's cost for expensive oracles"). + { + name: `constructed histories run drawn bases`, + enters: [`constructed-drawn-bases`], + build: (generated: Suite) => drawnBasesSuite(generated), + }, ], fixed: [ { From 3afff1ee3c1b193cb4253afee22e39639018b828 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Tue, 17 Nov 2026 11:35:42 +0000 Subject: [PATCH 11/29] feat(oracles): drawnBases runs further bases from the run's random seed A costly property keeps few fixed bases and still varies its constructed histories from run to run (register M, 2026-11-17, "Random bases at today's cost for expensive oracles"). After its fixed bases, a constructed unit runs drawnBases more: the following draws of the sample at the run's random seed + k, which the random unit runs at the same indices. The first unit that uses that seed logs it, and a drawn base's failure names it, prints its history, and replays with it. The gap e2e oracle (except on win32) and the self-oracle draw one base each. Co-authored-by: Isaac --- .changeset/constructed-stream-independent.md | 2 + .../gap-report-oracle.property.test.mjs | 13 +++- packages/oracles/src/oracle-property.ts | 77 ++++++++++++++++--- .../tests/oracle-property-self-oracle.test.ts | 10 ++- 4 files changed, 84 insertions(+), 18 deletions(-) diff --git a/.changeset/constructed-stream-independent.md b/.changeset/constructed-stream-independent.md index 51ec490a..251bf4ae 100644 --- a/.changeset/constructed-stream-independent.md +++ b/.changeset/constructed-stream-independent.md @@ -4,4 +4,6 @@ A campaign's generated histories no longer depend on its constructed histories, and each constructed history runs several bases from a stream of its own. The `generated` unit draws only from the property's arbitrary. Constructed history k builds from each of the first `bases` draws of the arbitrary's sample at the campaign's seed plus k, and every base must enter the states it names. The new `histories.bases` option sets that number. It must be a whole number of at least 1 (default 5), or the property fails to load. So appending a constructed history keeps every fixed seed's generated histories and every other constructed history's bases. Before, half of the generated unit's runs were constructed histories, weighted by how many there were, so appending one moved every run's draw, and each constructed unit ran one history. +A costly property can keep few fixed bases and still vary its constructed histories from run to run with the new `histories.drawnBases` option (a whole number of at least 0, default 0). After its fixed bases, each constructed unit runs that many more, drawn from the run's random seed: the seed the random units run, which the `ORACLE_SEED` line logs. That line now comes from the first unit that uses the seed, which may be a fixed-seed constructed unit. A failure on a drawn base says which base it was, prints its full history as the counterexample, and prints a replay line with the run's seed, at which the random unit of the same name runs that base at the same index. + This shifts each fixed seed once, for properties that have at least one constructed history. Their `generated` unit now runs other histories, and a seed-and-path replay line printed before this change for that unit no longer replays the same history. A constructed unit keeps its first base, and its first replay path index still selects it. Replay lines for properties without constructed histories stay valid: their `generated` unit already drew from the arbitrary alone. Each constructed unit now runs `bases` histories, so a campaign takes longer; set `bases` lower for a property whose constructed histories are slow. diff --git a/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs b/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs index d5298c81..537cf1d0 100644 --- a/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs +++ b/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs @@ -4790,11 +4790,16 @@ oracleProperty({ `a check that ran more than one batch`, ], runs: 2, - // One base per constructed history: each runs the CLI end to end, often - // for minutes, and five bases took a fixed-seed shard to 18 to 30 - // minutes on an Arclet, past the 10-minute CI limit (#122). Random seeds - // vary each one's base from run to run. + // One fixed base per constructed history, and one more drawn from the + // run's random seed, so each run explores another base at about today's + // cost (register M, 2026-11-17, "Random bases at today's cost for + // expensive oracles"). Each runs the CLI end to end, often for minutes: + // five fixed bases took a fixed-seed shard to 18 to 30 minutes on an + // Arclet (#122). Windows draws none: one history there already takes 9 to + // 12 minutes of its non-blocking job (#72), and a drawn base would pass + // the job's 20-minute timeout. bases: 1, + drawnBases: process.platform === `win32` ? 0 : 1, fixedSeeds: [20261105], constructed: withDeclaration([ { diff --git a/packages/oracles/src/oracle-property.ts b/packages/oracles/src/oracle-property.ts index 42d9bbc5..0239eb7f 100644 --- a/packages/oracles/src/oracle-property.ts +++ b/packages/oracles/src/oracle-property.ts @@ -91,10 +91,12 @@ * fast-check draws its generated histories independently of its examples, * so the units of seed N run exactly the histories that the campaign ran. * The random campaign registers `random seed: ` and `random seed: - * generated` in the same way. Its units share one seed per registration: - * the first random unit that runs selects it through + * generated` in the same way. Its units share one seed per registration, + * the run's random seed: the first unit that uses it selects it through * `oraclePropertyOptions`, which reads the replay variables and logs the one - * `ORACLE_SEED` line (maintainer decision, 2026-11-11, Q2). Each unit keeps + * `ORACLE_SEED` line (maintainer decision, 2026-11-11, Q2). That is the + * first random unit, or, in a property with `drawnBases`, its first + * fixed-seed constructed unit. Each unit keeps * its campaign's route, decides its own first failure, shrinks on its own * within its own timeout, and prints its own reach line, ` reach * {...}`; a campaign's reach is the sum of its units' lines. A constructed @@ -113,7 +115,9 @@ * ${prefix}_SEED=... ${prefix}_PATH=... ${prefix}_PROPERTY=... ${prefix}_UNIT=... * vitest run -t '...'`, which names the random unit of its own name: a * fixed-seed unit's replay runs that random unit with the fixed seed, which - * draws the same stream. Run with the project's test command in place of + * draws the same stream. A fixed-seed unit whose failing example is a drawn + * base names the run's random seed instead, at which the random unit runs + * that base at the same index, and its report says which base it was. Run with the project's test command in place of * `vitest run`, it runs only that unit and reproduces the counterexample. A * failing random unit's violation prints the same command as its rerun * line, and a failing fixed-seed unit prints `Rerun: -t '...'`, which reruns @@ -175,7 +179,10 @@ * the generated ones are independent streams: the `generated` unit draws * only from the arbitrary, and constructed history k builds from each of * the first `bases` draws of the arbitrary's sample at the campaign's seed - * plus k. So at a fixed seed the generated histories do not depend on the + * plus k, and then from `drawnBases` more draws, the ones after them in + * the sample at the run's random seed plus k (register M, 2026-11-17, + * "Random bases at today's cost for expensive oracles"), which vary from + * run to run. So at a fixed seed the generated histories do not depend on the * constructed list, its length or its contents, appending a constructed * history changes no other constructed history's bases, and each * constructed history is explored over `bases` histories, not visited once @@ -615,6 +622,15 @@ export interface GeneratedHistories { * 1; 5 when absent. */ bases?: number + /** + * How many more bases each constructed history runs in each campaign, + * after its `bases` fixed ones: draws `bases` to `bases + drawnBases - 1` + * of the arbitrary's sample at the run's random seed (the seed the random + * units run and the `ORACLE_SEED` line logs) plus the history's index. So + * a costly property can keep one fixed base and still vary its others from + * run to run. A whole number of at least 0; 0 when absent. + */ + drawnBases?: number fixed?: ReadonlyArray> } @@ -2417,6 +2433,14 @@ function generatedTests( throw new Error( `${id}: \`bases\` must be a whole number of at least 1, the number of bases each constructed history runs in a campaign; it is ${reportText(bases)}.`, ) + // A costly property's constructed histories also draw bases from the run's + // random seed (register M, 2026-11-17, "Random bases at today's cost for + // expensive oracles"). + const drawnBases = source.drawnBases ?? 0 + if (!(Number.isInteger(drawnBases) && drawnBases >= 0)) + throw new Error( + `${id}: \`drawnBases\` must be a whole number of at least 0, the number of bases each constructed history draws from the run's random seed; it is ${reportText(drawnBases)}.`, + ) const required = (source.states ?? []).map(stateName) const constructions = source.constructed ?? [] const declared = declaredOf(source) @@ -2528,8 +2552,19 @@ function generatedTests( // A constructed unit's histories: its construction over the first `bases` // draws of its own stream, the sample at seed + index, which neither the // generated draws nor the other constructed histories touch. - const constructedBases = (seed: number, index: number) => - buildBases(id, source.arbitrary, constructions[index]!, seed + index, bases) + // Then its drawn bases: the following draws of the stream at the run's + // random seed + index, so the random unit, whose seed is the run's, runs + // them at the same indices, and a drawn base replays with that seed. + const constructedBases = (seed: number, index: number, runSeed: number) => { + const build = (at: number, count: number) => + buildBases(id, source.arbitrary, constructions[index]!, at + index, count) + if (drawnBases === 0) return build(seed, bases) + if (seed === runSeed) return build(seed, bases + drawnBases) + return [ + ...build(seed, bases), + ...build(runSeed, bases + drawnBases).slice(bases), + ] + } // Registration checks every fixed history, then the constructed histories // built from the first fixed seed (or seed 0 when there is none), so a // history that cannot shrink fails the load rather than a full run. @@ -2542,6 +2577,12 @@ function generatedTests( // A campaign's units: one per constructed history, then `generated`. const unitNames = [...constructedNames, `generated`] + // The run's random seed, selected and logged once, by the first unit that + // uses it: the first random unit, or a fixed-seed constructed unit that + // draws bases from it. + let randomParameters: ReturnType | undefined + const runSeed = () => + (randomParameters ??= oraclePropertyOptions(source.runs, id)).seed! /** * Runs one unit of a campaign whose parameters are `base`. A constructed * unit runs its bases, built from the campaign's seed, as examples with @@ -2565,12 +2606,15 @@ function generatedTests( const started = performance.now() const index = constructedNames.indexOf(unit) const constructionOf = new Map>() + // The run's random seed, when this fixed-seed unit draws bases from it. + let drawnFrom: number | undefined // fast-check reports a run's details itself; the replay witness is // recorded from them below. const { reporter: _reporter, asyncReporter: _asyncReporter, ...rest } = base let run: typeof rest = rest if (index >= 0) { - const histories = constructedBases(seed, index) + if (!random && drawnBases > 0) drawnFrom = runSeed() + const histories = constructedBases(seed, index, drawnFrom ?? seed) // fast-check passes example inputs by reference, which identifies them. for (const history of histories) constructionOf.set(history, constructions[index]!) @@ -2615,12 +2659,21 @@ function generatedTests( if (details.failed) { const report = await fc.asyncDefaultReportMessage(details) const path = details.counterexamplePath + // A fixed-seed unit's failing example past its fixed bases is a drawn + // base: the random unit of its name runs it at the same index with the + // run's seed, so the replay names that seed, and the report says so. + const at = path === null ? -1 : Number(path.split(`:`)[0]) + const drawn = drawnFrom !== undefined && at >= bases const replay = path === null ? undefined : commandOf( [ - [oracleEnvName(`SEED`), `${details.seed}`, false], + [ + oracleEnvName(`SEED`), + `${drawn ? drawnFrom : details.seed}`, + false, + ], [oracleEnvName(`PATH`), `${path}`, false], [oracleEnvName(`PROPERTY`), id, true], [oracleEnvName(`UNIT`), replayed.name, true], @@ -2639,6 +2692,11 @@ function generatedTests( throw new Error( [ report, + ...(drawn + ? [ + `Drawn base: example ${at} is base ${at} of constructed history "${unit}" drawn from the run's random seed ${drawnFrom} (its ORACLE_SEED line), not from seed ${seed}. The counterexample above is its full history, and the Replay line runs it with the run's seed; a rerun draws other bases.`, + ] + : []), ...commandLines(`Rerun`, rerun), ...commandLines(`Replay`, replay), ].join(`\n`), @@ -2671,7 +2729,6 @@ function generatedTests( // that runs selects and logs it, and a replay reads that log line. A // replay of that seed rebuilds the same constructed histories, which the // replay path counts. - let randomParameters: ReturnType | undefined const randomUnits = unitNames.map((unit): PropertyTest => ({ name: `random seed: ${unit}`, route: `random seed`, diff --git a/packages/oracles/tests/oracle-property-self-oracle.test.ts b/packages/oracles/tests/oracle-property-self-oracle.test.ts index f60fd9a9..55e9cedf 100644 --- a/packages/oracles/tests/oracle-property-self-oracle.test.ts +++ b/packages/oracles/tests/oracle-property-self-oracle.test.ts @@ -8758,11 +8758,13 @@ oracleProperty({ state !== `registration-warning-policy`, ), runs: 3, - // One base per constructed history: each spawns real Vitest runs, and - // five bases took the shard of campaign units to 27 minutes on an - // Arclet, past the 10-minute CI limit (#122). Random seeds vary each - // one's base from run to run. + // One fixed base per constructed history, and one more drawn from the + // run's random seed (register M, 2026-11-17, "Random bases at today's + // cost for expensive oracles"): each spawns real Vitest runs, and five + // fixed bases took the shard of campaign units to 27 minutes on an + // Arclet (#122). bases: 1, + drawnBases: 1, fixedSeeds: [1], constructed: [ { From 5df0fc01e015fa835f5bbf43f13d03ea9cee0e50 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Tue, 17 Nov 2026 15:07:51 +0000 Subject: [PATCH 12/29] test(oracles): a drawn base's failure names the run's seed and replays with it A failing drawn base must report its example index and the run's random seed, and its replay must name that seed and reproduce the failure. A construction whose seed and bases are derived from the arbitrary makes a drawn base fail, and enters drawn-base-failure-replayed. Co-authored-by: Isaac --- .../tests/oracle-property-self-oracle.test.ts | 117 +++++++++++++++++- 1 file changed, 116 insertions(+), 1 deletion(-) diff --git a/packages/oracles/tests/oracle-property-self-oracle.test.ts b/packages/oracles/tests/oracle-property-self-oracle.test.ts index 55e9cedf..b011fa2b 100644 --- a/packages/oracles/tests/oracle-property-self-oracle.test.ts +++ b/packages/oracles/tests/oracle-property-self-oracle.test.ts @@ -2975,6 +2975,7 @@ const appendedStates = [ `constructed-default-bases`, `constructed-declared-bases`, `constructed-drawn-bases`, + `drawn-base-failure-replayed`, ] as const const frameworkStates = [ ...timeoutReceivingStates, @@ -4099,6 +4100,19 @@ function frameworkReach( if (drawnOf(spec) > 0) reached.add(`constructed-drawn-bases`) } } + // A drawn base's failure, reported with the run's seed, whose printed + // replay reproduced it under the same law. + for (const { checkpoint, value } of trace) { + if (checkpoint !== `replay`) continue + const probe = value as ProbeRun + if ( + probe.original.drawnBase !== undefined && + probe.failure?.law !== undefined && + probe.failure.law === probe.original.law && + (probe.ran?.length ?? 0) > 0 + ) + reached.add(`drawn-base-failure-replayed`) + } return reached } @@ -4167,6 +4181,8 @@ type Failure = { violationRerun?: string /** A campaign unit's replay command, as printed. */ replay?: string + /** A drawn base's report: the failing example's index and the run's seed. */ + drawnBase?: { at: number; seed: number } /** Whether a law's failure prints the history's items. */ showsHistory?: boolean /** Whether a model error quotes this machine's paths. */ @@ -4635,6 +4651,10 @@ function classify( ? /^Replay: (.*)$/m.exec(outer)?.[1] : undefined const history = /^History: (.*)$/m.exec(text)?.[1] + const drawnLine = + /^Drawn base: example (\d+) is base \d+ of constructed history ".*" drawn from the run's random seed (-?\d+)/m.exec( + outer, + ) const law = /\(law: ([^)]+)\)/.exec(text)?.[1] const checkpoint = /violated at checkpoint "([^"]+)"/.exec(text)?.[1] const diagnosis = /was never reached/.test(text) @@ -4656,6 +4676,11 @@ function classify( const kind = classifyKind(inner, text, outer) const callbackContext = reportedCallbackContext(text, candidates) return { + ...(drawnLine + ? { + drawnBase: { at: Number(drawnLine[1]), seed: Number(drawnLine[2]) }, + } + : {}), kind, // Context rendering is not a deterministic difference; keep its decoded // actual identity while raw reports/sidecars retain the original text. @@ -7379,6 +7404,28 @@ function checkSuite( `${spec.id} "${test.name}" failed and printed no \`-t\` rerun of itself`, `rerun`, ) + // A fixed-seed unit whose failing example is a drawn base says so, + // with the run's seed, and its replay runs that seed. + const example = Number((failure.path ?? ``).split(`:`)[0]) + const owesDrawn = + unit.seed !== undefined && + unit.name !== `generated` && + drawnOf(spec) > 0 && + example >= basesOf(spec) + const replaySeed = /_SEED=(-?\d+)/.exec(failure.replay)?.[1] + if ( + owesDrawn + ? failure.drawnBase?.at !== example || + failure.drawnBase.seed !== loggedSeed || + replaySeed !== `${loggedSeed}` + : failure.drawnBase !== undefined + ) + return violation( + `run`, + `${spec.id} "${test.name}" failed at example ${example} and reported drawn base ${JSON.stringify(failure.drawnBase ?? null)} with replay seed ${replaySeed}; owed ${owesDrawn ? `drawn base ${example} of the run's seed ${loggedSeed}, replayed with it` : `no drawn base`}`, + `replay`, + { actual: failure.drawnBase ?? null, expected: owesDrawn }, + ) const owedLine = unit.seed === undefined ? failure.replay : failure.rerun if ( @@ -7690,8 +7737,21 @@ function checkSuite( for (const { checkpoint, probe } of probes) { const probed = probe.probed if (checkpoint === `replay`) { + // A fixed-seed unit's failing drawn base replays at the run's seed, + // the one its property logged; any other failure at its own seed. + const spec = plan.specs.find(({ id }) => id === probe.id)! + const target = unitOf(probe.target) + const example = Number((probe.original.path ?? ``).split(`:`)[0]) + const drawnTarget = + target?.seed !== undefined && + target.name !== `generated` && + drawnOf(spec) > 0 && + example >= basesOf(spec) + const runSeed = run.files[spec.id]!.tests.flatMap( + ({ seedLogs }) => seedLogs, + )[0] const owedVariables = { - [envName(`SEED`)]: String(probe.original.seed), + [envName(`SEED`)]: String(drawnTarget ? runSeed : probe.original.seed), [envName(`PATH`)]: probe.original.path, [envName(`PROPERTY`)]: probe.id, [envName(`UNIT`)]: probe.probed, @@ -8545,6 +8605,53 @@ const drawnBasesSuite = (generated: Suite): Suite => { ], } } +// A campaign whose constructed history k passes on its fixed bases and fails, +// under law three or four, on a base it draws from the seeded child run's +// seed (`childSeed`), after histories that pass on every base. Its seed, k, +// and numbers of bases are derived from the arbitrary, as +// `seedFailingUnderFour` derives its seed, so the failing example is a drawn +// base whatever the streams hold. +const drawnFailureDraft = (reachVia: Draft[`reachVia`]): Draft => { + const built = (base: SyntheticHistory) => [0, ...base].slice(0, 3) + const firstStop = (history: SyntheticHistory) => + history.find((entry) => entry === 3 || entry === 4 || entry === 6) + const sample = (seed: number, count: number) => + fc.sample(syntheticArbitrary, { seed, numRuns: count }).map(built) + for (const bases of [1, 2, 3]) + for (const drawnBases of [1, 2]) + for (let failing = 0; failing < 3; failing++) { + // Each history's drawn bases, in order, at the run's seed. + const drawn = (index: number) => + sample(childSeed + index, bases + drawnBases).slice(bases) + const passes = (histories: ReadonlyArray) => + histories.every((history) => firstStop(history) === undefined) + const failingDrawn = drawn(failing) + const stopAt = failingDrawn.findIndex( + (history) => firstStop(history) !== undefined, + ) + const stop = stopAt < 0 ? undefined : firstStop(failingDrawn[stopAt]!) + if (stop !== 3 && stop !== 4) continue + if (!Array.from({ length: failing }, (_, j) => drawn(j)).every(passes)) + continue + for (let seed = 1; seed <= 999; seed++) + if ( + Array.from({ length: failing + 1 }, (_, j) => + sample(seed + j, bases), + ).every(passes) + ) + return generatedDraft({ + source: { kind: `generated`, runs: 2, seed }, + bases, + drawnBases, + reachVia, + constructed: Array.from({ length: failing + 1 }, () => ({ + prefix: [0], + enters: [], + })), + }) + } + throw new Error(`no seed from 1 to 999 fails first on a drawn base`) +} const mainUnitRequiredConstructed: Array<{ name: string enters: ReadonlyArray<(typeof mainUnitRequiredStates)[number]> @@ -9531,6 +9638,14 @@ oracleProperty({ enters: [`constructed-drawn-bases`], build: (generated: Suite) => drawnBasesSuite(generated), }, + // A drawn base's failure names the run's seed and its index, and its + // printed replay, with that seed, reproduces it. + { + name: `a drawn base fails and replays with the run's seed`, + enters: [`drawn-base-failure-replayed`], + build: (generated: Suite) => + unitWitnessSuite([drawnFailureDraft(generated.drafts[0]!.reachVia)]), + }, ], fixed: [ { From c9eac4c890637c09f6d258c8e6a06701a2378b30 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Tue, 17 Nov 2026 15:14:14 +0000 Subject: [PATCH 13/29] test(oracles): a constructed history may declare its own drawnBases A history's own drawnBases replaces its property's for that history, and one that is not a whole number of at least 0 fails the load, naming the history. The draft grammar draws it on each constructed history. RED on a framework that reads only the property's drawnBases. Co-authored-by: Isaac --- .../oracle-property-self-oracle-fixture.ts | 65 +++++++------- .../tests/oracle-property-self-oracle.test.ts | 86 +++++++++++++++---- 2 files changed, 103 insertions(+), 48 deletions(-) diff --git a/packages/oracles/tests/oracle-property-self-oracle-fixture.ts b/packages/oracles/tests/oracle-property-self-oracle-fixture.ts index f05231a9..c77c5a6d 100644 --- a/packages/oracles/tests/oracle-property-self-oracle-fixture.ts +++ b/packages/oracles/tests/oracle-property-self-oracle-fixture.ts @@ -277,6 +277,8 @@ export type SyntheticSpec = { name: string prefix: SyntheticHistory enters: ReadonlyArray + /** This history's own `drawnBases`; absent takes the property's. */ + drawnBases?: number }> fixed: ReadonlyArray<{ name: string @@ -1501,37 +1503,40 @@ export function syntheticProperty(spec: SyntheticSpec): void { spec.source.seed, ...(spec.secondSeed ? [spec.source.seed * 10] : []), ], - constructed: spec.constructed.map(({ name, prefix, enters }, at) => ({ - name, - enters: enters.map(publicState), - // A constructed history extends a generated history, so its histories - // depend on the campaign's seed, as real constructed histories do. - build: (generated: Formed) => { - if (at === 0 && spec.throwingBuild) - throw new Error(`synthetic build failed`) - const tail = spec.isolatedPlans - ? items(generated).map((item) => item % 3) - : items(generated) - const history = markOrigin( - form([...prefix, ...tail].slice(0, 3)), - `constructed:${name}`, - ) - const log = process.env.SELF_ORACLE_LOG - if (log) - appendFileSync( - `${log}.builds`, - `${JSON.stringify({ - id: spec.id, - test: owningTestName() ?? ``, - name, - generated: items(generated), - history: items(history), - })}\n`, + constructed: spec.constructed.map( + ({ name, prefix, enters, drawnBases }, at) => ({ + name, + enters: enters.map(publicState), + ...(drawnBases === undefined ? {} : { drawnBases }), + // A constructed history extends a generated history, so its histories + // depend on the campaign's seed, as real constructed histories do. + build: (generated: Formed) => { + if (at === 0 && spec.throwingBuild) + throw new Error(`synthetic build failed`) + const tail = spec.isolatedPlans + ? items(generated).map((item) => item % 3) + : items(generated) + const history = markOrigin( + form([...prefix, ...tail].slice(0, 3)), + `constructed:${name}`, ) - built.add(history as object) - return history - }, - })), + const log = process.env.SELF_ORACLE_LOG + if (log) + appendFileSync( + `${log}.builds`, + `${JSON.stringify({ + id: spec.id, + test: owningTestName() ?? ``, + name, + generated: items(generated), + history: items(history), + })}\n`, + ) + built.add(history as object) + return history + }, + }), + ), fixed: spec.fixed.map((fixed) => { const declaration = { ...fixed, diff --git a/packages/oracles/tests/oracle-property-self-oracle.test.ts b/packages/oracles/tests/oracle-property-self-oracle.test.ts index b011fa2b..19609143 100644 --- a/packages/oracles/tests/oracle-property-self-oracle.test.ts +++ b/packages/oracles/tests/oracle-property-self-oracle.test.ts @@ -32,14 +32,16 @@ * checks and before law `membership`, in that order (analyst-chosen, 2026-11-11). A generated * source whose `bases` is not a whole number of at least 1 fails to load, naming `bases`, after * the pairing of `predict` with a probe-bound state and before its state checks, and then one - * whose `drawnBases` is not a whole number of at least 0, naming `drawnBases` (analyst-chosen, - * 2026-11-16 and 2026-11-17). + * whose `drawnBases` is not a whole number of at least 0, naming `drawnBases`, and then the first + * constructed history whose own `drawnBases` is not, naming it (analyst-chosen, 2026-11-16 and + * 2026-11-17). * - `units` (maintainer decision, 2026-11-11, option D): a campaign's units run exactly the * histories that the campaign ran, each in its own test. The constructed unit of history k, for * seed S, runs N histories: the construction built from each of the first N draws of the * arbitrary's sample at seed S + k, in order, where N is the property's `bases`, 5 by default * (register M, 2026-11-16, "N bases per constructed history"). Then it runs D more, D being the - * property's `drawnBases` (none by default): draws N to N + D - 1 of the sample at R + k, where R + * property's `drawnBases` (none by default), or the history's own `drawnBases`, which replaces + * it: draws N to N + D - 1 of the sample at R + k, where R * is the run's random seed, the one the random units run (register M, 2026-11-17, "Random bases * at today's cost for expensive oracles"). So the random unit of the same name, at R, runs those * histories at the same indices, and a drawn base's failure replays with seed R. @@ -413,7 +415,12 @@ type Draft = { | { kind: `enumerated`; histories: ReadonlyArray } /** `exact` declares the constructed histories' states; the others break registration. */ statesMode: `exact` | `extra` | `missing` - constructed: Array<{ prefix: SyntheticHistory; enters: Array }> + /** A history's own `drawnBases`, when present, replaces the property's. */ + constructed: Array<{ + prefix: SyntheticHistory + enters: Array + drawnBases?: number + }> fixed: Array<{ history: SyntheticHistory enters: Array | null @@ -688,8 +695,21 @@ const draft: fc.Arbitrary = fc.record({ ), // Up to three, so a list long enough for a design that mixes only "when // there are many" is drawn (review of #122, F5). + // A history's own `drawnBases` is mostly absent (the property's), and + // sometimes 0, 1, or -1, which fails the load. constructed: fc.array( - fc.record({ prefix: insideOrOutside(6), enters: fc.subarray(allStates) }), + fc.record( + { + prefix: insideOrOutside(6), + enters: fc.subarray(allStates), + drawnBases: fc.oneof( + { weight: 8, arbitrary: fc.constant(undefined) }, + { weight: 2, arbitrary: fc.constantFrom(0, 1) }, + { weight: 1, arbitrary: fc.constant(-1) }, + ), + }, + { requiredKeys: [`prefix`, `enters`] }, + ), { maxLength: 3 }, ), fixed: fc.array( @@ -1800,9 +1820,19 @@ const basesOf = (spec: SyntheticSpec) => spec.bases ?? defaultBases // The bases each constructed history draws from the run's random seed, after // its fixed ones (register M, 2026-11-17, "Random bases at today's cost for // expensive oracles"); none when the property declares none. -const drawnOf = (spec: SyntheticSpec) => spec.drawnBases ?? 0 -// The histories each constructed unit runs in all. -const constructedRuns = (spec: SyntheticSpec) => basesOf(spec) + drawnOf(spec) +// A constructed history may declare its own number, which replaces the +// property's for that history. +const drawnOf = (spec: SyntheticSpec, name: string) => + spec.constructed.find((construction) => construction.name === name) + ?.drawnBases ?? + spec.drawnBases ?? + 0 +// Whether any constructed history draws a base. +const drawsAny = (spec: SyntheticSpec) => + spec.constructed.some(({ name }) => drawnOf(spec, name) > 0) +// The histories the unit of constructed history `name` runs in all. +const constructedRuns = (spec: SyntheticSpec, name: string) => + basesOf(spec) + drawnOf(spec, name) // The histories a unit draws at `seed`, in order. Constructed history k's unit // runs its construction over the first N draws of the arbitrary's sample at // seed + k, N being its bases: a stream of its own, which neither the @@ -1834,7 +1864,7 @@ function unitStream( const drawn = fc .sample(syntheticArbitrary, { seed: runSeed + at, - numRuns: constructedRuns(spec), + numRuns: constructedRuns(spec, spec.constructed[at]!.name), }) .slice(basesOf(spec)) return [...fixedBases, ...drawn].map((base) => @@ -2337,6 +2367,16 @@ function predictSpec( loads: false, loadError: `${spec.id}: \`drawnBases\` must be a whole number of at least 0`, } + const badDrawn = spec.constructed.find( + ({ drawnBases }) => + drawnBases !== undefined && + !(Number.isInteger(drawnBases) && drawnBases >= 0), + ) + if (badDrawn) + return { + loads: false, + loadError: `${spec.id}: constructed history "${badDrawn.name}": \`drawnBases\` must be a whole number of at least 0`, + } const declared = new Set(spec.states) const entered = new Set( spec.constructed.flatMap((construction) => construction.enters), @@ -3280,7 +3320,7 @@ function frameworkReach( ? spec.source.kind === `generated` ? spec.source.runs : 0 - : constructedRuns(spec) + : constructedRuns(spec, unitOf(name)!.name) if (test?.state !== `passed` || calls.length !== count) return false total += calls.length reached.add( @@ -3290,7 +3330,13 @@ function frameworkReach( }) if ( complete && - total === spec.source.runs + basesOf(spec) * spec.constructed.length + total === + spec.source.runs + + spec.constructed.reduce( + (sum, { name: constructed }) => + sum + constructedRuns(spec, constructed), + 0, + ) ) { reached.add(`passing-budget-${child.context.kind}-${lane}`) if (spec.constructed.length > 0) @@ -4087,7 +4133,7 @@ function frameworkReach( seen.tests.find((test) => test.name === unit)?.state === `passed` && (run?.log ?? []).filter( (entry) => entry.test === `${spec.id} > ${unit}`, - ).length === constructedRuns(spec) + ).length === constructedRuns(spec, name) ) }) if (ranBases) { @@ -4097,7 +4143,7 @@ function frameworkReach( : `constructed-declared-bases`, ) // A fixed-seed unit that also ran bases drawn from the run's seed. - if (drawnOf(spec) > 0) reached.add(`constructed-drawn-bases`) + if (drawsAny(spec)) reached.add(`constructed-drawn-bases`) } } // A drawn base's failure, reported with the run's seed, whose printed @@ -6553,7 +6599,7 @@ function childExpectation( ? 1 : unitOf(test.name)!.name === `generated` ? spec.source.runs - : constructedRuns(spec) + : constructedRuns(spec, unitOf(test.name)!.name) if (actualCalls !== owedCalls) return { failure: violation( @@ -7105,8 +7151,12 @@ function checkSuite( const seedUnit = spec.source.kind !== `generated` ? undefined - : drawnOf(spec) > 0 && spec.constructed.length > 0 - ? fixedUnit(fixedSeedsOf(spec)[0]!, spec.constructed[0]!.name) + : drawsAny(spec) + ? fixedUnit( + fixedSeedsOf(spec)[0]!, + spec.constructed.find(({ name }) => drawnOf(spec, name) > 0)! + .name, + ) : randomUnit(unitNames(spec)[0]!) const loggedSeed = seen.tests.find(({ name }) => name === seedUnit) ?.seedLogs[0] @@ -7410,7 +7460,7 @@ function checkSuite( const owesDrawn = unit.seed !== undefined && unit.name !== `generated` && - drawnOf(spec) > 0 && + drawnOf(spec, unit.name) > 0 && example >= basesOf(spec) const replaySeed = /_SEED=(-?\d+)/.exec(failure.replay)?.[1] if ( @@ -7745,7 +7795,7 @@ function checkSuite( const drawnTarget = target?.seed !== undefined && target.name !== `generated` && - drawnOf(spec) > 0 && + drawnOf(spec, target.name) > 0 && example >= basesOf(spec) const runSeed = run.files[spec.id]!.tests.flatMap( ({ seedLogs }) => seedLogs, From 63dbc34b4e8744bba94c424d558140a81bba89f9 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Tue, 17 Nov 2026 15:14:43 +0000 Subject: [PATCH 14/29] test(oracles): a constructed history enters constructed-own-drawn-bases Co-authored-by: Isaac --- .../tests/oracle-property-self-oracle.test.ts | 38 +++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/packages/oracles/tests/oracle-property-self-oracle.test.ts b/packages/oracles/tests/oracle-property-self-oracle.test.ts index 19609143..47f04aca 100644 --- a/packages/oracles/tests/oracle-property-self-oracle.test.ts +++ b/packages/oracles/tests/oracle-property-self-oracle.test.ts @@ -3016,6 +3016,7 @@ const appendedStates = [ `constructed-declared-bases`, `constructed-drawn-bases`, `drawn-base-failure-replayed`, + `constructed-own-drawn-bases`, ] as const const frameworkStates = [ ...timeoutReceivingStates, @@ -4144,6 +4145,14 @@ function frameworkReach( ) // A fixed-seed unit that also ran bases drawn from the run's seed. if (drawsAny(spec)) reached.add(`constructed-drawn-bases`) + // A history whose own drawnBases replaced the property's. + if ( + spec.constructed.some( + ({ drawnBases }) => + drawnBases !== undefined && drawnBases !== (spec.drawnBases ?? 0), + ) + ) + reached.add(`constructed-own-drawn-bases`) } } // A drawn base's failure, reported with the run's seed, whose printed @@ -8702,6 +8711,29 @@ const drawnFailureDraft = (reachVia: Draft[`reachVia`]): Draft => { } throw new Error(`no seed from 1 to 999 fails first on a drawn base`) } +// The suite of `drawnBasesSuite` whose first constructed history draws no +// base and whose second draws one, against the property's one or two. +const ownDrawnSuite = (generated: Suite): Suite => { + const withDrawn = drawnBasesSuite(generated) + const draft = withDrawn.drafts[0]! + const tail = generated.drafts[0]!.clock ? 1 : 2 + return { + ...withDrawn, + drafts: [ + { + ...draft, + constructed: [ + { prefix: [0], enters: [`saw-0`], drawnBases: 0 }, + { + prefix: [tail], + enters: [], + drawnBases: draft.drawnBases === 1 ? 0 : 1, + }, + ], + }, + ], + } +} const mainUnitRequiredConstructed: Array<{ name: string enters: ReadonlyArray<(typeof mainUnitRequiredStates)[number]> @@ -9688,6 +9720,12 @@ oracleProperty({ enters: [`constructed-drawn-bases`], build: (generated: Suite) => drawnBasesSuite(generated), }, + // A constructed history's own drawnBases replaces the property's. + { + name: `a constructed history declares its own drawn bases`, + enters: [`constructed-own-drawn-bases`], + build: (generated: Suite) => ownDrawnSuite(generated), + }, // A drawn base's failure names the run's seed and its index, and its // printed replay, with that seed, reproduces it. { From c6b5a3a2add0eeb717a896409ba60d46c3374384 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Tue, 17 Nov 2026 15:18:27 +0000 Subject: [PATCH 15/29] feat(oracles): a constructed history's own drawnBases; reshard the costly suites A constructed history may declare its own drawnBases, which replaces its property's; one that is not a whole number of at least 0 fails the load. The gap e2e oracle's witness-module history declares 0: at one base it takes 245 to 346 s of a CI job, so a drawn base would pass the 10-minute limit in one test (#145 cuts its cost). With one drawn base per constructed history, the e2e fixed-seed units run in six Linux jobs and the self-oracle's campaign units in two. Co-authored-by: Isaac --- .github/workflows/pr.yml | 13 ++-- .../gap-report-oracle.property.test.mjs | 4 + packages/oracles/scripts/pr-shards.mjs | 75 +++++++++++++------ packages/oracles/src/oracle-property.ts | 29 ++++++- .../tests/oracle-property-self-oracle.test.ts | 6 +- 5 files changed, 90 insertions(+), 37 deletions(-) diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index edf59139..86868898 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -60,12 +60,12 @@ jobs: - run: pnpm run build - run: pnpm --filter @tanstack/eslint-plugin-oracles run test test-oracle-framework: - name: Framework self-oracle (${{ matrix.shard }}/8) + name: Framework self-oracle (${{ matrix.shard }}/9) runs-on: ubuntu-latest strategy: fail-fast: false matrix: - shard: [1, 2, 3, 4, 5, 6, 7, 8] + shard: [1, 2, 3, 4, 5, 6, 7, 8, 9] steps: - name: Checkout uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 @@ -74,7 +74,7 @@ jobs: - name: Setup Tools uses: tanstack/config/.github/setup@e4b48f16568324f76f467aa4c2aac2f05db632c3 # The oracleProperty self-oracle spawns real Vitest runs, so the - # default test run leaves it out; it runs here on its own, in eight + # default test run leaves it out; it runs here on its own, in nine # shards. - run: pnpm exec vitest --run --config vitest.oracle-framework.config.mjs -t "$(node scripts/pr-shards.mjs pattern oracle-framework "$SHARD")" working-directory: packages/oracles @@ -106,12 +106,12 @@ jobs: env: SHARD: ${{ matrix.shard }} test-oracle-gaps-end-to-end: - name: Gap report end-to-end oracle (fixed seed, ${{ matrix.shard }}/3) + name: Gap report end-to-end oracle (fixed seed, ${{ matrix.shard }}/6) runs-on: ubuntu-latest strategy: fail-fast: false matrix: - shard: [1, 2, 3] + shard: [1, 2, 3, 4, 5, 6] steps: - name: Checkout uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 @@ -119,7 +119,7 @@ jobs: persist-credentials: false - name: Setup Tools uses: tanstack/config/.github/setup@e4b48f16568324f76f467aa4c2aac2f05db632c3 - # The end-to-end oracle's fixed-seed units, in three shards. Its random + # The end-to-end oracle's fixed-seed units, in six shards. Its random # units, calibrations, and determinism units run locally. - run: pnpm exec vitest --run --config oracle-gaps/vitest.config.mjs gap-report-oracle -t "$(node scripts/pr-shards.mjs pattern oracle-gaps-fixed-seed "$SHARD")" working-directory: packages/oracles @@ -143,6 +143,7 @@ jobs: - oracle-framework 6 - oracle-framework 7 - oracle-framework 8 + - oracle-framework 9 steps: - name: Checkout uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 diff --git a/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs b/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs index 52d287f1..59a42ce4 100644 --- a/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs +++ b/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs @@ -6360,6 +6360,10 @@ oracleProperty({ // runs and four daemon checks of the witness module. name: `an edit sequence over the witness module`, enters: [`a timed-out run reused after an edit`], + // Base 0 only: this one history takes 245 to 346 s of a CI job at one + // base, so a drawn base would pass the 10-minute limit in a single + // test (P12, #122). Cutting its cost is tracked separately. + drawnBases: 0, build: (generated) => ({ ...generated, functions: [ diff --git a/packages/oracles/scripts/pr-shards.mjs b/packages/oracles/scripts/pr-shards.mjs index ca8b9b4f..e2410ddd 100644 --- a/packages/oracles/scripts/pr-shards.mjs +++ b/packages/oracles/scripts/pr-shards.mjs @@ -244,7 +244,8 @@ export const partitions = { // The oracleProperty self-oracle: 843 tests, 2,258 s in all, measured in // CI on main at e2af13a (run 43266105332), where its three shards took // 4:12, 16:12, and 18:09. One file whose tests run in sequence, so a - // shard's time is the sum of its tests. Eight shards of 195 to 330 s: + // shard's time is the sum of its tests. Eight shards of 195 to 330 s, and + // a ninth since the campaign units split in two (#122): // the fixed histories' determinism units in three, the constructed // histories' determinism units, the fixed histories and the calibrations // in three, and the rest (the seeded units, 312 s, and any new kind). @@ -260,7 +261,11 @@ export const partitions = { 5: `(?:${startsWith(`fixed history: `, fixedA)}|${startsWith(`calibration: `, calibration1)})`, 6: `(?:${startsWith(`fixed history: `, fixedB)}|${startsWith(`calibration: `, calibration2)})`, 7: startsWith(`calibration: `, [...calibration3, ...calibration4]), - 8: rest, + // The campaign units: each constructed history runs one fixed base and + // one drawn from the run's seed (#122), which doubles their 5.6 min, so + // the fixed-seed units run apart from the random ones. + 8: `fixed seed 1: .*`, + 9: rest, }, }, // The gap report's static oracle, the slowest of the fast gap suites and @@ -287,8 +292,9 @@ export const partitions = { // the grammar is derived from the plan, not drawn, 2026-11-13). A unit's // cost follows the generated history its constructed history is built // from, and the witness module's histories grow with its witnesses. The - // units are packed longest first into three shards: about 290 s, 277 s, - // and 260 s of CI time. + // units are packed longest first: into three shards of about 290 s, 277 s, + // and 260 s of CI time at first, and into six since each constructed + // history also runs a base drawn from the run's seed (#122). 'oracle-gaps-fixed-seed': { args: [`--config`, `oracle-gaps/vitest.config.mjs`, `gap-report-oracle`], suite: `oracle-gaps.report`, @@ -296,12 +302,31 @@ export const partitions = { // The tests that one job selected before: the shards together must // select exactly these. previously: `fixed seed`, + // Each constructed history runs one fixed base and one drawn from the + // run's seed (#122), which doubles a constructed unit's time. The times + // below are main's CI times at 153539b, at one base, doubled. shards: { - // 261 s, 25 s, and small units: #57's last constructed histories sit - // here, since the rest (shard 3) is near its 10-minute limit. - 1: anyOf([ - 'an edit sequence over the witness module', + // The witness module's edit sequence alone: 346 s at one base, and it + // draws no base (its own `drawnBases: 0`), since two would pass the + // 10-minute limit in one test. + 1: anyOf(['an edit sequence over the witness module']), + // Every part of `every outcome class` (126 s, 9 s, and 3 s) and + // brokenVitest (22 s): about 320 s. The stop parts are named here: + // `every outcome class` alone matches the first half only. + 2: anyOf([ + 'every outcome class', + 'every outcome class, the warm stop', + 'every outcome class, the abort stop', + 'a declaration fault: brokenVitest', + ]), + // noTypescript (84 s), a second daemon check (45 s), the stale authored + // mutant (29 s), a cold-result edit (10 s), and #57's small units: + // about 375 s. + 3: anyOf([ + 'a declaration fault: noTypescript', + 'a second daemon check of the budget witness', 'a stale authored mutant and a reformat', + 'an edit that changed the cold result after a hang replaced the only context', 'a failing owner under a config that --config names', 'a stop before the CLI starts', 'a stop by SIGTERM as the CLI starts', @@ -309,23 +334,25 @@ export const partitions = { 'a start refuses a dangling claim', 'a supervised check with no ps on its PATH', ]), - // Every part of `every outcome class` (the first half took 134 s on the CI - // runner; the warm and abort stops are two small histories over the stop - // witness alone), a second daemon check (101 s), and two small units. The - // stop parts are named here: `every outcome class` alone matches the - // first half only, and the others would fall into the rest (shard 3). - 2: anyOf([ - 'every outcome class', - 'every outcome class, the warm stop', - 'every outcome class, the abort stop', - 'a second daemon check of the budget witness', - 'a declaration fault: brokenVitest', - 'an edit that changed the cold result after a hang replaced the only context', + // The edit sequence through the daemon (94 s) and brokenOwner (69 s): + // about 325 s. + 4: anyOf([ + 'an edit sequence through the daemon', + 'a declaration fault: brokenOwner', + ]), + // A pause that cannot engage (26 s), the lazy module (13 s), two + // mid-check test edits (12 s and 10 s), and a reset check (11 s): + // about 145 s. + 5: anyOf([ + 'a daemon pause that cannot engage', + 'a lazy module loaded before a hang in the same batch', + 'a test edits the helper it read, during a check through the daemon', + 'a test edits the data it read, during a check through the daemon', + 'a reset check after its warm daemon was replaced', ]), - // The rest: noTypescript (174 s on the CI runner), the edit sequence - // through the daemon (94 s), brokenOwner (83 s), generated (23 s), the - // lazy module (12 s), and small units. - 3: rest, + // The rest: generated (10 s, not doubled) and small units, about + // 260 s, and any new unit. + 6: rest, }, }, 'oracle-gaps-static-windows': { diff --git a/packages/oracles/src/oracle-property.ts b/packages/oracles/src/oracle-property.ts index 0239eb7f..0d07aa78 100644 --- a/packages/oracles/src/oracle-property.ts +++ b/packages/oracles/src/oracle-property.ts @@ -576,6 +576,12 @@ export interface ConstructedHistory { name: string enters: ReadonlyArray build: (generated: THistory) => THistory + /** + * This history's own number of bases drawn from the run's random seed, + * which replaces the property's `drawnBases` for it: for a history whose + * one base already takes most of a CI job. A whole number of at least 0. + */ + drawnBases?: number } /** @@ -2441,6 +2447,20 @@ function generatedTests( throw new Error( `${id}: \`drawnBases\` must be a whole number of at least 0, the number of bases each constructed history draws from the run's random seed; it is ${reportText(drawnBases)}.`, ) + for (const construction of source.constructed ?? []) + if ( + construction.drawnBases !== undefined && + !( + Number.isInteger(construction.drawnBases) && + construction.drawnBases >= 0 + ) + ) + throw new Error( + `${id}: constructed history "${construction.name}": \`drawnBases\` must be a whole number of at least 0, the number of bases it draws from the run's random seed; it is ${reportText(construction.drawnBases)}.`, + ) + // The bases a constructed history draws: its own number, or the property's. + const drawnOf = (index: number) => + source.constructed![index]!.drawnBases ?? drawnBases const required = (source.states ?? []).map(stateName) const constructions = source.constructed ?? [] const declared = declaredOf(source) @@ -2558,11 +2578,12 @@ function generatedTests( const constructedBases = (seed: number, index: number, runSeed: number) => { const build = (at: number, count: number) => buildBases(id, source.arbitrary, constructions[index]!, at + index, count) - if (drawnBases === 0) return build(seed, bases) - if (seed === runSeed) return build(seed, bases + drawnBases) + const drawn = drawnOf(index) + if (drawn === 0) return build(seed, bases) + if (seed === runSeed) return build(seed, bases + drawn) return [ ...build(seed, bases), - ...build(runSeed, bases + drawnBases).slice(bases), + ...build(runSeed, bases + drawn).slice(bases), ] } // Registration checks every fixed history, then the constructed histories @@ -2613,7 +2634,7 @@ function generatedTests( const { reporter: _reporter, asyncReporter: _asyncReporter, ...rest } = base let run: typeof rest = rest if (index >= 0) { - if (!random && drawnBases > 0) drawnFrom = runSeed() + if (!random && drawnOf(index) > 0) drawnFrom = runSeed() const histories = constructedBases(seed, index, drawnFrom ?? seed) // fast-check passes example inputs by reference, which identifies them. for (const history of histories) diff --git a/packages/oracles/tests/oracle-property-self-oracle.test.ts b/packages/oracles/tests/oracle-property-self-oracle.test.ts index 47f04aca..4a4722f5 100644 --- a/packages/oracles/tests/oracle-property-self-oracle.test.ts +++ b/packages/oracles/tests/oracle-property-self-oracle.test.ts @@ -8715,19 +8715,19 @@ const drawnFailureDraft = (reachVia: Draft[`reachVia`]): Draft => { // base and whose second draws one, against the property's one or two. const ownDrawnSuite = (generated: Suite): Suite => { const withDrawn = drawnBasesSuite(generated) - const draft = withDrawn.drafts[0]! + const first = withDrawn.drafts[0]! const tail = generated.drafts[0]!.clock ? 1 : 2 return { ...withDrawn, drafts: [ { - ...draft, + ...first, constructed: [ { prefix: [0], enters: [`saw-0`], drawnBases: 0 }, { prefix: [tail], enters: [], - drawnBases: draft.drawnBases === 1 ? 0 : 1, + drawnBases: first.drawnBases === 1 ? 0 : 1, }, ], }, From 7315badaeb6dca1fd6e7c9698a3366905ee2e3c4 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Tue, 17 Nov 2026 16:59:21 +0000 Subject: [PATCH 16/29] test(eslint-plugin-oracles): two bases per constructed history in the two slow oracles At the framework's default of five bases, the registration-only and test-source-recognition oracles took the Windows unit job from about 5 to 11.7 minutes. Co-authored-by: Isaac --- .../tests/registration-only-oracle.property.test.js | 6 ++++++ .../tests/test-source-recognition-oracle.property.test.js | 3 +++ 2 files changed, 9 insertions(+) diff --git a/packages/eslint-plugin-oracles/tests/registration-only-oracle.property.test.js b/packages/eslint-plugin-oracles/tests/registration-only-oracle.property.test.js index f79b191b..4de02c23 100644 --- a/packages/eslint-plugin-oracles/tests/registration-only-oracle.property.test.js +++ b/packages/eslint-plugin-oracles/tests/registration-only-oracle.property.test.js @@ -596,6 +596,9 @@ oracleProperty({ histories: { arbitrary: historyArbitrary, runs: 40, + // Two bases per constructed history: at the framework's default of five + // this file took the Windows unit job past 10 minutes (#122). + bases: 2, fixedSeeds: [2261111], states: STATES, constructed, @@ -1283,6 +1286,9 @@ oracleProperty({ histories: { arbitrary: agreeHistories, runs: 40, + // Two bases per constructed history: at the framework's default of five + // this file took the Windows unit job past 10 minutes (#122). + bases: 2, fixedSeeds: [2261112], states: AGREE_STATES, constructed: agreeConstructed, diff --git a/packages/eslint-plugin-oracles/tests/test-source-recognition-oracle.property.test.js b/packages/eslint-plugin-oracles/tests/test-source-recognition-oracle.property.test.js index 6790ef26..8d885c66 100644 --- a/packages/eslint-plugin-oracles/tests/test-source-recognition-oracle.property.test.js +++ b/packages/eslint-plugin-oracles/tests/test-source-recognition-oracle.property.test.js @@ -2157,6 +2157,9 @@ oracleProperty({ histories: { arbitrary: historyArbitrary, runs: 40, + // Two bases per constructed history: at the framework's default of five + // this file took the Windows unit job past 10 minutes (#122). + bases: 2, fixedSeeds: [2261113], states: STATES, constructed, From 59b5c44924dc2030afb06eb79ce68292082d20d4 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Tue, 17 Nov 2026 23:22:49 +0000 Subject: [PATCH 17/29] test(oracle-gaps): two bases per constructed history in the static oracle At three bases its larger fixed-seed shard took 11m58s on CI, past the 10-minute limit; each base adds about 4.5 minutes there. Co-authored-by: Isaac --- .../gap-report-static-oracle.property.test.mjs | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/packages/oracles/oracle-gaps/gap-report-static-oracle.property.test.mjs b/packages/oracles/oracle-gaps/gap-report-static-oracle.property.test.mjs index 6dd78065..b370f243 100644 --- a/packages/oracles/oracle-gaps/gap-report-static-oracle.property.test.mjs +++ b/packages/oracles/oracle-gaps/gap-report-static-oracle.property.test.mjs @@ -1482,10 +1482,11 @@ oracleProperty({ arbitrary: argumentHistories, states: required, runs: 40, - // Three bases per constructed history: each runs the CLI on a fixture - // package, and five bases took the larger fixed-seed shard from 116 s to - // 530 s on an Arclet, close to the 10-minute CI limit (#122). - bases: 3, + // Two bases per constructed history: each runs the CLI on a fixture + // package, and each base adds about 4.5 minutes to the larger fixed-seed + // shard on CI (3m00s at one base on main, 11m58s at three), so two keep it + // under the 10-minute limit (#122). + bases: 2, fixedSeeds: [20261105], constructed: withDeclaration([ { From 95b2ee6c2d0d47bdbf9be744b8e9cdfcef519d08 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Wed, 18 Nov 2026 00:23:54 +0000 Subject: [PATCH 18/29] test(oracles): drawn bases vary from run to run In each unseeded child run, a fixed-seed unit's drawn bases must follow that run's logged seed, and the first unit that draws from it must log it. A plan whose property draws bases runs a second unseeded child with another host draw, and a constructed history enters seedless-drawn-bases-vary when the two runs draw other bases (review of #122, round 2, G1). RED: unseeded runs that draw from the fixed seed. Co-authored-by: Isaac --- .../tests/oracle-property-self-oracle.test.ts | 150 +++++++++++++++--- 1 file changed, 129 insertions(+), 21 deletions(-) diff --git a/packages/oracles/tests/oracle-property-self-oracle.test.ts b/packages/oracles/tests/oracle-property-self-oracle.test.ts index 4a4722f5..099286d6 100644 --- a/packages/oracles/tests/oracle-property-self-oracle.test.ts +++ b/packages/oracles/tests/oracle-property-self-oracle.test.ts @@ -60,7 +60,9 @@ * alike except that the second's constructed list is the first's with one history appended. At * each fixed seed, and at the seeded run's random seed, the `generated` unit and the unit of each * constructed history of the shorter list run the same histories in the same order in both, up - * to the shorter list's first failure, and the appended history's own unit runs. The law + * to the shorter list's first failure, and the appended history's own unit runs. Drawn bases + * vary from run to run: in each unseeded child run, a fixed-seed unit's drawn bases follow + * that run's logged seed, and two unseeded runs with other host draws draw other bases. The law * compares the two properties' own logs, so it holds without the model's streams. * - `verdict`: a test fails with a violation of law L exactly when its first * failing history violates L, whatever values the history holds (a bigint, @@ -3017,6 +3019,7 @@ const appendedStates = [ `constructed-drawn-bases`, `drawn-base-failure-replayed`, `constructed-own-drawn-bases`, + `seedless-drawn-bases-vary`, ] as const const frameworkStates = [ ...timeoutReceivingStates, @@ -4155,6 +4158,32 @@ function frameworkReach( reached.add(`constructed-own-drawn-bases`) } } + // Two unseeded runs logged other seeds, and a fixed-seed unit's drawn + // bases differ between them. + const unseeded = trace + .filter( + ({ checkpoint }) => + checkpoint === `seedless` || checkpoint === `seedless-second`, + ) + .map(({ value }) => value as SeedlessRun) + if ( + unseeded.length === 2 && + plan.specs.some((spec) => { + const [first, second] = unseeded.map((runs) => runs[spec.id]) + if (!first || !second || first.seeds[0] === second.seeds[0]) return false + return Object.keys(first.fixedRan).some((unit) => { + const name = unitOf(unit)!.name + const drawn = (ran: Array | undefined) => + JSON.stringify((ran ?? []).slice(basesOf(spec))) + return ( + drawnOf(spec, name) > 0 && + (first.fixedRan[unit]?.length ?? 0) > basesOf(spec) && + drawn(first.fixedRan[unit]) !== drawn(second.fixedRan[unit]) + ) + }) + }) + ) + reached.add(`seedless-drawn-bases-vary`) // A drawn base's failure, reported with the run's seed, whose printed // replay reproduced it under the same law. for (const { checkpoint, value } of trace) { @@ -4339,6 +4368,10 @@ type SeedlessRun = Record< routes: Array seeds: Array ran: Record> + /** What each fixed-seed constructed unit ran, including its drawn bases. */ + fixedRan: Record> + /** The tests that logged an ORACLE_SEED line for the property. */ + seedTests: Array } > /** The run under a drawn replay environment. */ @@ -5800,12 +5833,27 @@ function driveSuite( writeFileSync(file, original) } } - if (plan.seedless) { - const seedlessLog = join(directory, `seedless-${runCount}.jsonl`) + // A seedless child run, and, when a property draws bases, a second one + // with another host draw: two unseeded runs, whose drawn bases must + // follow each run's own seed (law `constructed independence`, M + // 2026-11-17, "Random bases at today's cost for expensive oracles"). + const seedlessRuns: Array<[checkpoint: string, draw: number]> = + plan.seedless + ? [ + [`seedless`, plan.seedlessDraw ?? 0.5], + ...(plan.specs.some( + (spec) => spec.source.kind === `generated` && drawsAny(spec), + ) + ? [[`seedless-second`, 0.25] as [string, number]] + : []), + ] + : [] + for (const [checkpoint, draw] of seedlessRuns) { + const seedlessLog = join(directory, `${checkpoint}-${runCount}.jsonl`) const seedless = receive({ kind: `seedless` }, directory, files, { env: { SELF_ORACLE_LOG: seedlessLog, - SELF_ORACLE_RANDOM_DRAW: String(plan.seedlessDraw ?? 0.5), + SELF_ORACLE_RANDOM_DRAW: String(draw), }, seeded: false, prefix, @@ -5813,33 +5861,46 @@ function driveSuite( const logged = readLog(seedlessLog) const settled = new Set(settledLog(plan, logged)) // Per property: the routes its random units ran under, the same - // whatever seed it drew; the seed it logged; and what each random unit - // ran, which that seed decides (none for a slow spec). + // whatever seed it drew; the seed it logged, and the tests that logged + // it; what each random unit ran, which that seed decides (none for a + // slow spec); and what each fixed-seed constructed unit ran. observe( - `seedless`, + checkpoint, Object.fromEntries( plan.specs.map((spec, index) => { + const unitName = (entry: LogEntry) => + entry.test.slice(spec.id.length + 3) const random = logged.filter( (entry) => entry.id === spec.id && - unitOf(entry.test.slice(spec.id.length + 3))?.seed === - undefined && - isCampaign(entry.test.slice(spec.id.length + 3)), + unitOf(unitName(entry))?.seed === undefined && + isCampaign(unitName(entry)), ) const ran: Record> = {} for (const entry of random.filter((kept) => settled.has(kept))) - (ran[entry.test.slice(spec.id.length + 3)] ??= []).push( - entry.history, - ) + (ran[unitName(entry)] ??= []).push(entry.history) + const fixedRan: Record> = {} + for (const entry of logged.filter( + (kept) => + kept.id === spec.id && + settled.has(kept) && + unitOf(unitName(kept))?.seed !== undefined && + unitOf(unitName(kept))?.name !== `generated`, + )) + (fixedRan[unitName(entry)] ??= []).push(entry.history) + const tests = + seedless.files.find((file) => file.file === files[index]) + ?.tests ?? [] return [ spec.id, { routes: [...new Set(random.map((entry) => entry.route))].sort(), - seeds: ( - seedless.files.find((file) => file.file === files[index]) - ?.tests ?? [] - ).flatMap((test) => seedLogsOf(test, spec.id)), + seeds: tests.flatMap((test) => seedLogsOf(test, spec.id)), ran, + fixedRan, + seedTests: tests + .filter((test) => seedLogsOf(test, spec.id).length > 0) + .map((test) => short(spec.id, test.name)), }, ] }), @@ -6749,6 +6810,7 @@ function checkSuite( : [ `binding-reordered`, `seedless`, + `seedless-second`, `replay`, `rerun`, `reach-alone`, @@ -7655,14 +7717,21 @@ function checkSuite( // `random seed`, and a loaded property runs at least one. A slow unit // times out, but its first history starts before it does. The property // logged one seed, and each random unit ran the stream of that seed. - const seedless = trace.find(({ checkpoint }) => checkpoint === `seedless`) - ?.value as SeedlessRun | undefined - if (seedless) + // Each unseeded run is judged on its own seed: the random units, and the + // fixed-seed constructed units that draw bases from it, whose drawn bases + // follow that seed, so two unseeded runs draw other bases. The first unit + // that uses the seed logs it. + for (const seedless of trace + .filter( + ({ checkpoint }) => + checkpoint === `seedless` || checkpoint === `seedless-second`, + ) + .map(({ value }) => value as SeedlessRun)) for (const spec of plan.specs) { const generated = expected.get(spec.id)!.loads && spec.source.kind === `generated` const owedRoutes = generated ? [`random seed`] : [] - const { routes, seeds, ran } = seedless[spec.id]! + const { routes, seeds, ran, fixedRan, seedTests } = seedless[spec.id]! if (JSON.stringify(routes) !== JSON.stringify(owedRoutes)) return violation( `seedless`, @@ -7693,6 +7762,36 @@ function checkSuite( { actual: logged, expected: owedRun }, ) } + const [fixedSeed] = fixedSeedsOf(spec) + const drawing = spec.constructed.filter( + ({ name }) => drawnOf(spec, name) > 0, + ) + if (drawing.length === 0 || fixedSeed === undefined) continue + const owedSeedTest = fixedUnit(fixedSeed, drawing[0]!.name) + if (JSON.stringify(seedTests) !== JSON.stringify([owedSeedTest])) + return violation( + `seedless`, + `${spec.id}: an ordinary run logged its seed in ${JSON.stringify(seedTests)}, owed ${JSON.stringify([owedSeedTest])}, the first unit that draws a base from it`, + `seed-log`, + { actual: seedTests, expected: [owedSeedTest] }, + ) + for (const { name } of drawing) + for (const seed of fixedSeedsOf(spec)) { + const unit = fixedUnit(seed, name) + const { ran: logged, owedRun } = comparedRun( + spec, + unit, + unitStream(spec, seed, name, seeds[0]!), + fixedRan[unit] ?? [], + ) + if (JSON.stringify(logged) !== JSON.stringify(owedRun)) + return violation( + `seedless`, + `${spec.id} "${unit}" ran ${JSON.stringify(logged)} in an ordinary run that logged seed ${seeds[0]}, owed ${JSON.stringify(owedRun)}: its drawn bases follow the run's seed`, + `units`, + { actual: logged, expected: owedRun }, + ) + } } // In a full run, every history's test starts, so the reach test runs // none of them again: it reuses what their tests recorded. @@ -9720,6 +9819,15 @@ oracleProperty({ enters: [`constructed-drawn-bases`], build: (generated: Suite) => drawnBasesSuite(generated), }, + // Two unseeded runs draw other bases (law `constructed independence`). + { + name: `two unseeded runs draw other bases`, + enters: [`seedless-drawn-bases-vary`], + build: (generated: Suite) => ({ + ...drawnBasesSuite(generated), + seedless: true, + }), + }, // A constructed history's own drawnBases replaces the property's. { name: `a constructed history declares its own drawn bases`, From 3a1094f90671d740eb582c0ecf18f9e1b104700e Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Wed, 18 Nov 2026 00:25:56 +0000 Subject: [PATCH 19/29] test(oracles): a drawn base's report directs base replay and filing Law replay: a failing drawn base's report must direct the reader to replay the history on the base commit and to file, not block on, a failure the base shares (R42, phase 1; review of #122, round 2, G3). Co-authored-by: Isaac --- .../tests/oracle-property-self-oracle.test.ts | 23 +++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/packages/oracles/tests/oracle-property-self-oracle.test.ts b/packages/oracles/tests/oracle-property-self-oracle.test.ts index 099286d6..b036a027 100644 --- a/packages/oracles/tests/oracle-property-self-oracle.test.ts +++ b/packages/oracles/tests/oracle-property-self-oracle.test.ts @@ -204,7 +204,9 @@ * - `timeout`: a property's `timeout` limits every test it registers, each unit * on its own. * - `replay`: a failing campaign unit prints a replay command: the seed (for a fixed-seed unit - * whose failing example is a drawn base, the run's random seed), the + * whose failing example is a drawn base, the run's random seed, with a report that names the + * base and directs the reader to replay the history on the base commit and to file, not block + * on, a failure the base shares: R42, phase 1), the * path, the property, and `${prefix}_UNIT` naming the random unit of the same * name (`random seed: `), under the configured prefix, then that unit's * `-t` pattern (maintainer decisions, 2026-11-11, Q3 and Q4). A failing random @@ -4265,8 +4267,12 @@ type Failure = { violationRerun?: string /** A campaign unit's replay command, as printed. */ replay?: string - /** A drawn base's report: the failing example's index and the run's seed. */ - drawnBase?: { at: number; seed: number } + /** + * A drawn base's report: the failing example's index, the run's seed, and + * whether it directs the reader to replay the history on the base commit + * and to file, not block on, a failure the base shares (R42). + */ + drawnBase?: { at: number; seed: number; directsBase: boolean } /** Whether a law's failure prints the history's items. */ showsHistory?: boolean /** Whether a model error quotes this machine's paths. */ @@ -4766,7 +4772,15 @@ function classify( return { ...(drawnLine ? { - drawnBase: { at: Number(drawnLine[1]), seed: Number(drawnLine[2]) }, + drawnBase: { + at: Number(drawnLine[1]), + seed: Number(drawnLine[2]), + directsBase: + /replay this history on the base commit/.test(outer) && + /file it with its full history, or find its issue, and do not block on it/.test( + outer, + ), + }, } : {}), kind, @@ -7538,6 +7552,7 @@ function checkSuite( owesDrawn ? failure.drawnBase?.at !== example || failure.drawnBase.seed !== loggedSeed || + !failure.drawnBase.directsBase || replaySeed !== `${loggedSeed}` : failure.drawnBase !== undefined ) From 2e3752b26f40f69fa2945996cf46e856a143c543 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Wed, 18 Nov 2026 00:27:39 +0000 Subject: [PATCH 20/29] feat(oracles): a drawn base's report directs base replay and filing A rerun draws other bases, so it would hide a drawn base's failure. The report now says to replay the history on the base commit before rerunning, and to file the counterexample, or find its issue, without blocking when the base fails too (R42, phase 1). Co-authored-by: Isaac --- packages/oracles/src/oracle-property.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/oracles/src/oracle-property.ts b/packages/oracles/src/oracle-property.ts index 0d07aa78..c0507455 100644 --- a/packages/oracles/src/oracle-property.ts +++ b/packages/oracles/src/oracle-property.ts @@ -2715,7 +2715,7 @@ function generatedTests( report, ...(drawn ? [ - `Drawn base: example ${at} is base ${at} of constructed history "${unit}" drawn from the run's random seed ${drawnFrom} (its ORACLE_SEED line), not from seed ${seed}. The counterexample above is its full history, and the Replay line runs it with the run's seed; a rerun draws other bases.`, + `Drawn base: example ${at} is base ${at} of constructed history "${unit}" drawn from the run's random seed ${drawnFrom} (its ORACLE_SEED line), not from seed ${seed}. The counterexample above is its full history, and the Replay line runs it with the run's seed; a rerun draws other bases, so it would hide this failure. Before you rerun, replay this history on the base commit: the Replay line, or the counterexample as a fixed history. If the base fails too, the failure is not this change's: file it with its full history, or find its issue, and do not block on it (R42; base replay is not automated).`, ] : []), ...commandLines(`Rerun`, rerun), From 8e29e10bfb6dd2166b2fb3884bbde9290caa6be3 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Wed, 18 Nov 2026 00:31:30 +0000 Subject: [PATCH 21/29] ci: split the two thinnest shards The self-oracle's random-seed campaign units (8m51s on CI) run in two jobs, and the gap e2e oracle's apiLessTypescript unit (146 s) moves from the rest (8m07s) to shard 5 (2m37s), so each job has more than a minute of margin under the 10-minute limit (review of #122, round 2, G4). Co-authored-by: Isaac --- .github/workflows/pr.yml | 7 ++++--- packages/oracles/scripts/pr-shards.mjs | 11 +++++++++-- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 86868898..ce68e99a 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -60,12 +60,12 @@ jobs: - run: pnpm run build - run: pnpm --filter @tanstack/eslint-plugin-oracles run test test-oracle-framework: - name: Framework self-oracle (${{ matrix.shard }}/9) + name: Framework self-oracle (${{ matrix.shard }}/10) runs-on: ubuntu-latest strategy: fail-fast: false matrix: - shard: [1, 2, 3, 4, 5, 6, 7, 8, 9] + shard: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] steps: - name: Checkout uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 @@ -74,7 +74,7 @@ jobs: - name: Setup Tools uses: tanstack/config/.github/setup@e4b48f16568324f76f467aa4c2aac2f05db632c3 # The oracleProperty self-oracle spawns real Vitest runs, so the - # default test run leaves it out; it runs here on its own, in nine + # default test run leaves it out; it runs here on its own, in ten # shards. - run: pnpm exec vitest --run --config vitest.oracle-framework.config.mjs -t "$(node scripts/pr-shards.mjs pattern oracle-framework "$SHARD")" working-directory: packages/oracles @@ -144,6 +144,7 @@ jobs: - oracle-framework 7 - oracle-framework 8 - oracle-framework 9 + - oracle-framework 10 steps: - name: Checkout uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 diff --git a/packages/oracles/scripts/pr-shards.mjs b/packages/oracles/scripts/pr-shards.mjs index e2410ddd..e6a3b84c 100644 --- a/packages/oracles/scripts/pr-shards.mjs +++ b/packages/oracles/scripts/pr-shards.mjs @@ -245,7 +245,7 @@ export const partitions = { // CI on main at e2af13a (run 43266105332), where its three shards took // 4:12, 16:12, and 18:09. One file whose tests run in sequence, so a // shard's time is the sum of its tests. Eight shards of 195 to 330 s, and - // a ninth since the campaign units split in two (#122): + // two more since the campaign units split in three (#122): // the fixed histories' determinism units in three, the constructed // histories' determinism units, the fixed histories and the calibrations // in three, and the rest (the seeded units, 312 s, and any new kind). @@ -265,7 +265,11 @@ export const partitions = { // one drawn from the run's seed (#122), which doubles their 5.6 min, so // the fixed-seed units run apart from the random ones. 8: `fixed seed 1: .*`, - 9: rest, + // The random-seed campaign units, split by name: about 249 s of CI + // time before `round3`, and about 260 s with the rest (59b5c449's 9/9 + // took 8m51s whole). + 9: `random seed: (?:[a-q]|r(?!ound[3-9])).*`, + 10: rest, }, }, // The gap report's static oracle, the slowest of the fast gap suites and @@ -344,6 +348,9 @@ export const partitions = { // mid-check test edits (12 s and 10 s), and a reset check (11 s): // about 145 s. 5: anyOf([ + // apiLessTypescript (146 s at 59b5c449, with its drawn base) moved + // here from the rest, which took 8m07s with it. + 'a declaration fault: apiLessTypescript', 'a daemon pause that cannot engage', 'a lazy module loaded before a hang in the same batch', 'a test edits the helper it read, during a check through the daemon', From f763593638ecab186c18f2209321a2ed7ba33bfb Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Wed, 18 Nov 2026 02:07:09 +0000 Subject: [PATCH 22/29] ci: pack jobs to about 9 minutes Register M, 2026-11-17, "Pack CI jobs to about 9 min": the self-oracle runs in 8 jobs (from 10), the gap report oracles in 2 (from 3), and the Windows static oracle in 2 (from 6). The gap e2e stays at 6: its witness history takes 245 to 346 s alone, and no pair of the others fits under about 9 minutes. Co-authored-by: Isaac --- .github/workflows/pr.yml | 26 +- packages/oracles/scripts/pr-shards.mjs | 395 +++++++++++++------------ 2 files changed, 216 insertions(+), 205 deletions(-) diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index ce68e99a..a433b018 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -60,12 +60,12 @@ jobs: - run: pnpm run build - run: pnpm --filter @tanstack/eslint-plugin-oracles run test test-oracle-framework: - name: Framework self-oracle (${{ matrix.shard }}/10) + name: Framework self-oracle (${{ matrix.shard }}/8) runs-on: ubuntu-latest strategy: fail-fast: false matrix: - shard: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] + shard: [1, 2, 3, 4, 5, 6, 7, 8] steps: - name: Checkout uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 @@ -74,19 +74,19 @@ jobs: - name: Setup Tools uses: tanstack/config/.github/setup@e4b48f16568324f76f467aa4c2aac2f05db632c3 # The oracleProperty self-oracle spawns real Vitest runs, so the - # default test run leaves it out; it runs here on its own, in ten - # shards. + # default test run leaves it out; it runs here on its own, in eight + # shards, packed to about 9 minutes each. - run: pnpm exec vitest --run --config vitest.oracle-framework.config.mjs -t "$(node scripts/pr-shards.mjs pattern oracle-framework "$SHARD")" working-directory: packages/oracles env: SHARD: ${{ matrix.shard }} test-oracle-gaps-fast: - name: Gap report oracles (${{ matrix.shard }}/3) + name: Gap report oracles (${{ matrix.shard }}/2) runs-on: ubuntu-latest strategy: fail-fast: false matrix: - shard: [1, 2, 3] + shard: [1, 2] steps: - name: Checkout uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 @@ -96,10 +96,12 @@ jobs: uses: tanstack/config/.github/setup@e4b48f16568324f76f467aa4c2aac2f05db632c3 # The gap report's own oracles also run real Vitest runs on fixture # packages (test:oracle-gaps:fast): the static oracle in two shards, - # and the classify, probe-form, and mutant-validity oracles in a third. + # the first beside the classify, probe-form, and mutant-validity + # oracles, so each job runs about 8 minutes (register M, 2026-11-17, + # "Pack CI jobs to about 9 min"). - run: | - if [ "$SHARD" = 3 ]; then - exec pnpm exec vitest --run --config oracle-gaps/vitest.config.mjs gap-report-classify gap-report-probe-form gap-report-mutant-validity gap-report-path-spelling + if [ "$SHARD" = 1 ]; then + pnpm exec vitest --run --config oracle-gaps/vitest.config.mjs gap-report-classify gap-report-probe-form gap-report-mutant-validity gap-report-path-spelling fi pnpm exec vitest --run --config oracle-gaps/vitest.config.mjs gap-report-static -t "$(node scripts/pr-shards.mjs pattern oracle-gaps-static "$SHARD")" working-directory: packages/oracles @@ -143,8 +145,6 @@ jobs: - oracle-framework 6 - oracle-framework 7 - oracle-framework 8 - - oracle-framework 9 - - oracle-framework 10 steps: - name: Checkout uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 @@ -224,13 +224,13 @@ jobs: - if: always() run: pnpm --filter @tanstack/eslint-plugin-oracles run test windows-gap-oracles: - name: Windows gap report static oracle (${{ matrix.shard }}/6) + name: Windows gap report static oracle (${{ matrix.shard }}/2) runs-on: windows-latest timeout-minutes: 20 strategy: fail-fast: false matrix: - shard: [1, 2, 3, 4, 5, 6] + shard: [1, 2] steps: - name: Checkout uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 diff --git a/packages/oracles/scripts/pr-shards.mjs b/packages/oracles/scripts/pr-shards.mjs index e6a3b84c..7d21c680 100644 --- a/packages/oracles/scripts/pr-shards.mjs +++ b/packages/oracles/scripts/pr-shards.mjs @@ -253,23 +253,26 @@ export const partitions = { args: [`--config`, `vitest.oracle-framework.config.mjs`], suite: `self-oracle.framework`, select: ``, + // Packed to about 9 minutes a job (register M, 2026-11-17, "Pack CI jobs + // to about 9 min"), from the CI times at 59b5c449: the first two + // determinism shards (4m33s and 5m03s) run together, and so do the + // constructed histories' determinism units and the last calibrations + // (6m02s and 3m19s). No other pair fits under about 9 minutes. shards: { - 1: startsWith(`determinism: fixed history: `, detFixedA), - 2: startsWith(`determinism: fixed history: `, detFixedB), - 3: startsWith(`determinism: fixed history: `, detFixedC), - 4: `determinism: constructed history: .*`, - 5: `(?:${startsWith(`fixed history: `, fixedA)}|${startsWith(`calibration: `, calibration1)})`, - 6: `(?:${startsWith(`fixed history: `, fixedB)}|${startsWith(`calibration: `, calibration2)})`, - 7: startsWith(`calibration: `, [...calibration3, ...calibration4]), + 1: `(?:${startsWith(`determinism: fixed history: `, detFixedA)}|${startsWith(`determinism: fixed history: `, detFixedB)})`, + 2: startsWith(`determinism: fixed history: `, detFixedC), + 3: `(?:determinism: constructed history: .*|${startsWith(`calibration: `, [...calibration3, ...calibration4])})`, + 4: `(?:${startsWith(`fixed history: `, fixedA)}|${startsWith(`calibration: `, calibration1)})`, + 5: `(?:${startsWith(`fixed history: `, fixedB)}|${startsWith(`calibration: `, calibration2)})`, // The campaign units: each constructed history runs one fixed base and // one drawn from the run's seed (#122), which doubles their 5.6 min, so // the fixed-seed units run apart from the random ones. - 8: `fixed seed 1: .*`, + 6: `fixed seed 1: .*`, // The random-seed campaign units, split by name: about 249 s of CI // time before `round3`, and about 260 s with the rest (59b5c449's 9/9 // took 8m51s whole). - 9: `random seed: (?:[a-q]|r(?!ound[3-9])).*`, - 10: rest, + 7: `random seed: (?:[a-q]|r(?!ound[3-9])).*`, + 8: rest, }, }, // The gap report's static oracle, the slowest of the fast gap suites and @@ -366,192 +369,200 @@ export const partitions = { args: [`--config`, `oracle-gaps/vitest.config.mjs`, `gap-report-static`], suite: `oracle-gaps.report-static`, select: ``, - // Windows runs on its own partition: the same tests, in six shards, - // because each Linux shard takes 3 to 6 times as long on windows-latest. + // Windows runs on its own partition: the same tests, in six shards at + // first, because each Linux shard took 3 to 6 times as long on + // windows-latest, and in two since #122. shards: { + // Packed to about 9 minutes a job (register M, 2026-11-17, "Pack CI + // jobs to about 9 min"): the six shards took 1m32s to 3m00s each at + // 59b5c449, so they run as two. 1: anyOf([ - "calibration: a derived class's super call is made conditional", - 'calibration: a refusal names another cause', - 'calibration: production code is marked as probe code', - 'determinism: constructed history: a declaration fault: outsideSource', - 'determinism: constructed history: a source in another letter case', - 'determinism: constructed history: an owns that is not a literal in a expect-aliased assertion refuses nothing', - 'determinism: constructed history: an owns that is not a literal in a node-assert-default assertion refuses nothing', - 'determinism: constructed history: bad arguments: directoryConfig', - 'determinism: constructed history: bad arguments: mutantLacksField', - 'determinism: constructed history: bad arguments: repeatedMutantIds', - 'determinism: constructed history: no oracle file refuses the run', - 'determinism: generated 1', - 'fixed seed 20261105: a declaration fault: outsideSource', - 'fixed seed 20261105: a source in another letter case', - 'fixed seed 20261105: an owns that is not a literal in a expect-aliased assertion refuses nothing', - 'fixed seed 20261105: an owns that is not a literal in a node-assert-default assertion refuses nothing', - 'fixed seed 20261105: bad arguments: directoryConfig', - 'fixed seed 20261105: bad arguments: mutantLacksField', - 'fixed seed 20261105: bad arguments: repeatedMutantIds', - 'fixed seed 20261105: module data owned by readers and by name', - 'random seed: a bad entry in a run with no owner names the bad entry', - 'random seed: a derived class calling super after ||', - 'random seed: a whole-file owner', - 'random seed: an owns that is not a literal in a expect-computed assertion refuses nothing', - 'random seed: an owns that is not a literal in a node-assert-namespace assertion refuses nothing', - 'random seed: bad arguments: envPrefix', - 'random seed: bad arguments: noSource', - 'random seed: bad arguments: unknown', - 'random seed: no owns entry refuses the run', + ...[ + "calibration: a derived class's super call is made conditional", + 'calibration: a refusal names another cause', + 'calibration: production code is marked as probe code', + 'determinism: constructed history: a declaration fault: outsideSource', + 'determinism: constructed history: a source in another letter case', + 'determinism: constructed history: an owns that is not a literal in a expect-aliased assertion refuses nothing', + 'determinism: constructed history: an owns that is not a literal in a node-assert-default assertion refuses nothing', + 'determinism: constructed history: bad arguments: directoryConfig', + 'determinism: constructed history: bad arguments: mutantLacksField', + 'determinism: constructed history: bad arguments: repeatedMutantIds', + 'determinism: constructed history: no oracle file refuses the run', + 'determinism: generated 1', + 'fixed seed 20261105: a declaration fault: outsideSource', + 'fixed seed 20261105: a source in another letter case', + 'fixed seed 20261105: an owns that is not a literal in a expect-aliased assertion refuses nothing', + 'fixed seed 20261105: an owns that is not a literal in a node-assert-default assertion refuses nothing', + 'fixed seed 20261105: bad arguments: directoryConfig', + 'fixed seed 20261105: bad arguments: mutantLacksField', + 'fixed seed 20261105: bad arguments: repeatedMutantIds', + 'fixed seed 20261105: module data owned by readers and by name', + 'random seed: a bad entry in a run with no owner names the bad entry', + 'random seed: a derived class calling super after ||', + 'random seed: a whole-file owner', + 'random seed: an owns that is not a literal in a expect-computed assertion refuses nothing', + 'random seed: an owns that is not a literal in a node-assert-namespace assertion refuses nothing', + 'random seed: bad arguments: envPrefix', + 'random seed: bad arguments: noSource', + 'random seed: bad arguments: unknown', + 'random seed: no owns entry refuses the run', + ], + ...[ + 'calibration: a key drops its statement', + 'calibration: a refusal prints a stack trace, as a crash does', + 'calibration: the instrumented derived class differs with no mutant active', + 'determinism: constructed history: a derived class calling super after &&', + 'determinism: constructed history: a source through a link to the package', + 'determinism: constructed history: an owns that is not a literal in a expect-argument assertion refuses nothing', + 'determinism: constructed history: an owns that is not a literal in a node-assert-named assertion refuses nothing', + 'determinism: constructed history: bad arguments: directorySource', + 'determinism: constructed history: bad arguments: noPackage', + 'determinism: constructed history: bad arguments: seed', + 'determinism: constructed history: no owns entry refuses the run', + 'determinism: generated 2', + 'fixed seed 20261105: a derived class calling super after &&', + 'fixed seed 20261105: a source through a link to the package', + 'fixed seed 20261105: an owns that is not a literal in a expect-argument assertion refuses nothing', + 'fixed seed 20261105: an owns that is not a literal in a node-assert-named assertion refuses nothing', + 'fixed seed 20261105: bad arguments: directorySource', + 'fixed seed 20261105: bad arguments: noPackage', + 'fixed seed 20261105: bad arguments: seed', + 'fixed seed 20261105: no oracle file refuses the run', + 'random seed: a declaration fault: badEntry', + 'random seed: a derived class calling super in an arrow', + 'random seed: an owns that is not a literal in a assert assertion refuses nothing', + 'random seed: an owns that is not a literal in a expect-not assertion refuses nothing', + 'random seed: bad arguments: absentSource', + 'random seed: bad arguments: missingConfig', + 'random seed: bad arguments: noValue', + 'random seed: every static state', + 'random seed: owns read from both declarations', + ], + ...[ + 'calibration: a mutant is dropped', + "calibration: a refusal's fix offers a way around it", + 'determinism: constructed history: a bad entry in a run with no owner names the bad entry', + 'determinism: constructed history: a derived class calling super after ||', + 'determinism: constructed history: a whole-file owner', + 'determinism: constructed history: an owns that is not a literal in a expect-computed assertion refuses nothing', + 'determinism: constructed history: an owns that is not a literal in a node-assert-namespace assertion refuses nothing', + 'determinism: constructed history: bad arguments: envPrefix', + 'determinism: constructed history: bad arguments: noSource', + 'determinism: constructed history: bad arguments: unknown', + 'determinism: constructed history: owns read from both declarations', + 'fixed seed 20261105: a bad entry in a run with no owner names the bad entry', + 'fixed seed 20261105: a derived class calling super after ||', + 'fixed seed 20261105: a whole-file owner', + 'fixed seed 20261105: an owns that is not a literal in a expect-computed assertion refuses nothing', + 'fixed seed 20261105: an owns that is not a literal in a node-assert-namespace assertion refuses nothing', + 'fixed seed 20261105: bad arguments: envPrefix', + 'fixed seed 20261105: bad arguments: noSource', + 'fixed seed 20261105: bad arguments: unknown', + 'fixed seed 20261105: no owns entry refuses the run', + 'random seed: a declaration fault: missingSource', + 'random seed: a literal owns in an assertion makes no oracle file', + 'random seed: an owns that is not a literal in a chai assertion refuses nothing', + 'random seed: an owns that is not a literal in a expect-soft assertion refuses nothing', + 'random seed: bad arguments: badMutantJson', + 'random seed: bad arguments: missingMutants', + 'random seed: bad arguments: outsideSource', + 'random seed: generated', + 'random seed: owns read from oracleProperty', + ], ]), 2: anyOf([ - 'calibration: a key drops its statement', - 'calibration: a refusal prints a stack trace, as a crash does', - 'calibration: the instrumented derived class differs with no mutant active', - 'determinism: constructed history: a derived class calling super after &&', - 'determinism: constructed history: a source through a link to the package', - 'determinism: constructed history: an owns that is not a literal in a expect-argument assertion refuses nothing', - 'determinism: constructed history: an owns that is not a literal in a node-assert-named assertion refuses nothing', - 'determinism: constructed history: bad arguments: directorySource', - 'determinism: constructed history: bad arguments: noPackage', - 'determinism: constructed history: bad arguments: seed', - 'determinism: constructed history: no owns entry refuses the run', - 'determinism: generated 2', - 'fixed seed 20261105: a derived class calling super after &&', - 'fixed seed 20261105: a source through a link to the package', - 'fixed seed 20261105: an owns that is not a literal in a expect-argument assertion refuses nothing', - 'fixed seed 20261105: an owns that is not a literal in a node-assert-named assertion refuses nothing', - 'fixed seed 20261105: bad arguments: directorySource', - 'fixed seed 20261105: bad arguments: noPackage', - 'fixed seed 20261105: bad arguments: seed', - 'fixed seed 20261105: no oracle file refuses the run', - 'random seed: a declaration fault: badEntry', - 'random seed: a derived class calling super in an arrow', - 'random seed: an owns that is not a literal in a assert assertion refuses nothing', - 'random seed: an owns that is not a literal in a expect-not assertion refuses nothing', - 'random seed: bad arguments: absentSource', - 'random seed: bad arguments: missingConfig', - 'random seed: bad arguments: noValue', - 'random seed: every static state', - 'random seed: owns read from both declarations', - ]), - 3: anyOf([ - 'calibration: a mutant is dropped', - "calibration: a refusal's fix offers a way around it", - 'determinism: constructed history: a bad entry in a run with no owner names the bad entry', - 'determinism: constructed history: a derived class calling super after ||', - 'determinism: constructed history: a whole-file owner', - 'determinism: constructed history: an owns that is not a literal in a expect-computed assertion refuses nothing', - 'determinism: constructed history: an owns that is not a literal in a node-assert-namespace assertion refuses nothing', - 'determinism: constructed history: bad arguments: envPrefix', - 'determinism: constructed history: bad arguments: noSource', - 'determinism: constructed history: bad arguments: unknown', - 'determinism: constructed history: owns read from both declarations', - 'fixed seed 20261105: a bad entry in a run with no owner names the bad entry', - 'fixed seed 20261105: a derived class calling super after ||', - 'fixed seed 20261105: a whole-file owner', - 'fixed seed 20261105: an owns that is not a literal in a expect-computed assertion refuses nothing', - 'fixed seed 20261105: an owns that is not a literal in a node-assert-namespace assertion refuses nothing', - 'fixed seed 20261105: bad arguments: envPrefix', - 'fixed seed 20261105: bad arguments: noSource', - 'fixed seed 20261105: bad arguments: unknown', - 'fixed seed 20261105: no owns entry refuses the run', - 'random seed: a declaration fault: missingSource', - 'random seed: a literal owns in an assertion makes no oracle file', - 'random seed: an owns that is not a literal in a chai assertion refuses nothing', - 'random seed: an owns that is not a literal in a expect-soft assertion refuses nothing', - 'random seed: bad arguments: badMutantJson', - 'random seed: bad arguments: missingMutants', - 'random seed: bad arguments: outsideSource', - 'random seed: generated', - 'random seed: owns read from oracleProperty', - ]), - 4: anyOf([ - 'calibration: a refusal ends without its done line', - 'calibration: a rune is moved from its place', - 'determinism: constructed history: a declaration fault: badEntry', - 'determinism: constructed history: a derived class calling super in an arrow', - 'determinism: constructed history: an owns that is not a literal in a assert assertion refuses nothing', - 'determinism: constructed history: an owns that is not a literal in a expect-not assertion refuses nothing', - 'determinism: constructed history: bad arguments: absentSource', - 'determinism: constructed history: bad arguments: missingConfig', - 'determinism: constructed history: bad arguments: noValue', - 'determinism: constructed history: every static state', - 'determinism: constructed history: owns read from oracleProperty', - 'fixed seed 20261105: a declaration fault: badEntry', - 'fixed seed 20261105: a derived class calling super in an arrow', - 'fixed seed 20261105: an owns that is not a literal in a assert assertion refuses nothing', - 'fixed seed 20261105: an owns that is not a literal in a expect-not assertion refuses nothing', - 'fixed seed 20261105: bad arguments: absentSource', - 'fixed seed 20261105: bad arguments: missingConfig', - 'fixed seed 20261105: bad arguments: noValue', - 'fixed seed 20261105: every static state', - 'fixed seed 20261105: owns read from both declarations', - 'random seed: a declaration fault: nonLiteral', - 'random seed: a plain-parameter derived class', - 'random seed: an owns that is not a literal in a expect assertion refuses nothing', - 'random seed: an owns that is not a literal in a expect-type-of assertion refuses nothing', - 'random seed: bad arguments: count', - 'random seed: bad arguments: missingPackage', - 'random seed: bad arguments: pool', - 'random seed: module data named by an entry leaves its methods to their callers', - 'random seed: unnamed exports owned through an import', - ]), - 5: anyOf([ - 'calibration: a refusal exits 1, as a crash does', - 'calibration: an edit moves a key', - 'determinism: constructed history: a declaration fault: missingSource', - 'determinism: constructed history: a literal owns in an assertion makes no oracle file', - 'determinism: constructed history: an owns that is not a literal in a chai assertion refuses nothing', - 'determinism: constructed history: an owns that is not a literal in a expect-soft assertion refuses nothing', - 'determinism: constructed history: bad arguments: badMutantJson', - 'determinism: constructed history: bad arguments: missingMutants', - 'determinism: constructed history: bad arguments: outsideSource', - 'determinism: constructed history: module data named by an entry leaves its methods to their callers', - 'determinism: constructed history: unnamed exports owned through an import', - 'fixed seed 20261105: a declaration fault: missingSource', - 'fixed seed 20261105: a literal owns in an assertion makes no oracle file', - 'fixed seed 20261105: an owns that is not a literal in a chai assertion refuses nothing', - 'fixed seed 20261105: an owns that is not a literal in a expect-soft assertion refuses nothing', - 'fixed seed 20261105: bad arguments: badMutantJson', - 'fixed seed 20261105: bad arguments: missingMutants', - 'fixed seed 20261105: bad arguments: outsideSource', - 'fixed seed 20261105: generated', - 'fixed seed 20261105: owns read from oracleProperty', - 'random seed: a declaration fault: outsideSource', - 'random seed: a source in another letter case', - 'random seed: an owns that is not a literal in a expect-aliased assertion refuses nothing', - 'random seed: an owns that is not a literal in a node-assert-default assertion refuses nothing', - 'random seed: bad arguments: directoryConfig', - 'random seed: bad arguments: mutantLacksField', - 'random seed: bad arguments: repeatedMutantIds', - 'random seed: module data owned by readers and by name', - ]), - 6: anyOf([ - 'calibration: a refusal gives no fix', - 'calibration: an owned mutant is reported unowned', - 'determinism: constructed history: a declaration fault: nonLiteral', - 'determinism: constructed history: a plain-parameter derived class', - 'determinism: constructed history: an owns that is not a literal in a expect assertion refuses nothing', - 'determinism: constructed history: an owns that is not a literal in a expect-type-of assertion refuses nothing', - 'determinism: constructed history: bad arguments: count', - 'determinism: constructed history: bad arguments: missingPackage', - 'determinism: constructed history: bad arguments: pool', - 'determinism: constructed history: module data owned by readers and by name', - 'determinism: generated 0', - 'fixed seed 20261105: a declaration fault: nonLiteral', - 'fixed seed 20261105: a plain-parameter derived class', - 'fixed seed 20261105: an owns that is not a literal in a expect assertion refuses nothing', - 'fixed seed 20261105: an owns that is not a literal in a expect-type-of assertion refuses nothing', - 'fixed seed 20261105: bad arguments: count', - 'fixed seed 20261105: bad arguments: missingPackage', - 'fixed seed 20261105: bad arguments: pool', - 'fixed seed 20261105: module data named by an entry leaves its methods to their callers', - 'fixed seed 20261105: unnamed exports owned through an import', - 'random seed: a derived class calling super after &&', - 'random seed: a source through a link to the package', - 'random seed: an owns that is not a literal in a expect-argument assertion refuses nothing', - 'random seed: an owns that is not a literal in a node-assert-named assertion refuses nothing', - 'random seed: bad arguments: directorySource', - 'random seed: bad arguments: noPackage', - 'random seed: bad arguments: seed', - 'random seed: no oracle file refuses the run', + ...[ + 'calibration: a refusal ends without its done line', + 'calibration: a rune is moved from its place', + 'determinism: constructed history: a declaration fault: badEntry', + 'determinism: constructed history: a derived class calling super in an arrow', + 'determinism: constructed history: an owns that is not a literal in a assert assertion refuses nothing', + 'determinism: constructed history: an owns that is not a literal in a expect-not assertion refuses nothing', + 'determinism: constructed history: bad arguments: absentSource', + 'determinism: constructed history: bad arguments: missingConfig', + 'determinism: constructed history: bad arguments: noValue', + 'determinism: constructed history: every static state', + 'determinism: constructed history: owns read from oracleProperty', + 'fixed seed 20261105: a declaration fault: badEntry', + 'fixed seed 20261105: a derived class calling super in an arrow', + 'fixed seed 20261105: an owns that is not a literal in a assert assertion refuses nothing', + 'fixed seed 20261105: an owns that is not a literal in a expect-not assertion refuses nothing', + 'fixed seed 20261105: bad arguments: absentSource', + 'fixed seed 20261105: bad arguments: missingConfig', + 'fixed seed 20261105: bad arguments: noValue', + 'fixed seed 20261105: every static state', + 'fixed seed 20261105: owns read from both declarations', + 'random seed: a declaration fault: nonLiteral', + 'random seed: a plain-parameter derived class', + 'random seed: an owns that is not a literal in a expect assertion refuses nothing', + 'random seed: an owns that is not a literal in a expect-type-of assertion refuses nothing', + 'random seed: bad arguments: count', + 'random seed: bad arguments: missingPackage', + 'random seed: bad arguments: pool', + 'random seed: module data named by an entry leaves its methods to their callers', + 'random seed: unnamed exports owned through an import', + ], + ...[ + 'calibration: a refusal exits 1, as a crash does', + 'calibration: an edit moves a key', + 'determinism: constructed history: a declaration fault: missingSource', + 'determinism: constructed history: a literal owns in an assertion makes no oracle file', + 'determinism: constructed history: an owns that is not a literal in a chai assertion refuses nothing', + 'determinism: constructed history: an owns that is not a literal in a expect-soft assertion refuses nothing', + 'determinism: constructed history: bad arguments: badMutantJson', + 'determinism: constructed history: bad arguments: missingMutants', + 'determinism: constructed history: bad arguments: outsideSource', + 'determinism: constructed history: module data named by an entry leaves its methods to their callers', + 'determinism: constructed history: unnamed exports owned through an import', + 'fixed seed 20261105: a declaration fault: missingSource', + 'fixed seed 20261105: a literal owns in an assertion makes no oracle file', + 'fixed seed 20261105: an owns that is not a literal in a chai assertion refuses nothing', + 'fixed seed 20261105: an owns that is not a literal in a expect-soft assertion refuses nothing', + 'fixed seed 20261105: bad arguments: badMutantJson', + 'fixed seed 20261105: bad arguments: missingMutants', + 'fixed seed 20261105: bad arguments: outsideSource', + 'fixed seed 20261105: generated', + 'fixed seed 20261105: owns read from oracleProperty', + 'random seed: a declaration fault: outsideSource', + 'random seed: a source in another letter case', + 'random seed: an owns that is not a literal in a expect-aliased assertion refuses nothing', + 'random seed: an owns that is not a literal in a node-assert-default assertion refuses nothing', + 'random seed: bad arguments: directoryConfig', + 'random seed: bad arguments: mutantLacksField', + 'random seed: bad arguments: repeatedMutantIds', + 'random seed: module data owned by readers and by name', + ], + ...[ + 'calibration: a refusal gives no fix', + 'calibration: an owned mutant is reported unowned', + 'determinism: constructed history: a declaration fault: nonLiteral', + 'determinism: constructed history: a plain-parameter derived class', + 'determinism: constructed history: an owns that is not a literal in a expect assertion refuses nothing', + 'determinism: constructed history: an owns that is not a literal in a expect-type-of assertion refuses nothing', + 'determinism: constructed history: bad arguments: count', + 'determinism: constructed history: bad arguments: missingPackage', + 'determinism: constructed history: bad arguments: pool', + 'determinism: constructed history: module data owned by readers and by name', + 'determinism: generated 0', + 'fixed seed 20261105: a declaration fault: nonLiteral', + 'fixed seed 20261105: a plain-parameter derived class', + 'fixed seed 20261105: an owns that is not a literal in a expect assertion refuses nothing', + 'fixed seed 20261105: an owns that is not a literal in a expect-type-of assertion refuses nothing', + 'fixed seed 20261105: bad arguments: count', + 'fixed seed 20261105: bad arguments: missingPackage', + 'fixed seed 20261105: bad arguments: pool', + 'fixed seed 20261105: module data named by an entry leaves its methods to their callers', + 'fixed seed 20261105: unnamed exports owned through an import', + 'random seed: a derived class calling super after &&', + 'random seed: a source through a link to the package', + 'random seed: an owns that is not a literal in a expect-argument assertion refuses nothing', + 'random seed: an owns that is not a literal in a node-assert-named assertion refuses nothing', + 'random seed: bad arguments: directorySource', + 'random seed: bad arguments: noPackage', + 'random seed: bad arguments: seed', + 'random seed: no oracle file refuses the run', + ], ]), }, }, From 3ea7bcc4f59dce12204ca5a306fe9db0e73b2761 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Wed, 18 Nov 2026 03:36:33 +0000 Subject: [PATCH 23/29] ci: seven gap e2e jobs after #131's daemon witnesses At the merged head, shard 3 took 610 s on an Arclet with #131's daemon witnesses and a failing drawn base (#150). The units are repacked into seven jobs of about 9 minutes of CI at most (register M, 2026-11-17, "Pack CI jobs to about 9 min"). Co-authored-by: Isaac --- .github/workflows/pr.yml | 6 +-- packages/oracles/scripts/pr-shards.mjs | 69 +++++++++++++------------- 2 files changed, 38 insertions(+), 37 deletions(-) diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index a433b018..7796a26a 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -108,12 +108,12 @@ jobs: env: SHARD: ${{ matrix.shard }} test-oracle-gaps-end-to-end: - name: Gap report end-to-end oracle (fixed seed, ${{ matrix.shard }}/6) + name: Gap report end-to-end oracle (fixed seed, ${{ matrix.shard }}/7) runs-on: ubuntu-latest strategy: fail-fast: false matrix: - shard: [1, 2, 3, 4, 5, 6] + shard: [1, 2, 3, 4, 5, 6, 7] steps: - name: Checkout uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 @@ -121,7 +121,7 @@ jobs: persist-credentials: false - name: Setup Tools uses: tanstack/config/.github/setup@e4b48f16568324f76f467aa4c2aac2f05db632c3 - # The end-to-end oracle's fixed-seed units, in six shards. Its random + # The end-to-end oracle's fixed-seed units, in seven shards. Its random # units, calibrations, and determinism units run locally. - run: pnpm exec vitest --run --config oracle-gaps/vitest.config.mjs gap-report-oracle -t "$(node scripts/pr-shards.mjs pattern oracle-gaps-fixed-seed "$SHARD")" working-directory: packages/oracles diff --git a/packages/oracles/scripts/pr-shards.mjs b/packages/oracles/scripts/pr-shards.mjs index 42eee85f..aa14eab2 100644 --- a/packages/oracles/scripts/pr-shards.mjs +++ b/packages/oracles/scripts/pr-shards.mjs @@ -313,67 +313,68 @@ export const partitions = { // run's seed (#122), which doubles a constructed unit's time. The times // below are main's CI times at 153539b, at one base, doubled. shards: { - // The witness module's edit sequence alone: 346 s at one base, and it - // draws no base (its own `drawnBases: 0`), since two would pass the - // 10-minute limit in one test. + // Times are Arclet times at #122's merged head b470a883, with one + // fixed base and one drawn (CI takes about 1.8 times as long), packed + // to about 9 minutes of CI a job (register M, 2026-11-17, "Pack CI + // jobs to about 9 min"). + // The witness module's edit sequence alone: 193 s, and it draws no + // base (its own `drawnBases: 0`). 1: anyOf(['an edit sequence over the witness module']), - // Every part of `every outcome class` (126 s, 9 s, and 3 s) and - // brokenVitest (22 s): about 320 s. The stop parts are named here: - // `every outcome class` alone matches the first half only. + // Every part of `every outcome class` and brokenVitest: 285 s. 2: anyOf([ 'every outcome class', 'every outcome class, the warm stop', 'every outcome class, the abort stop', 'a declaration fault: brokenVitest', ]), - // noTypescript (84 s), a second daemon check (45 s), the stale authored - // mutant (29 s), a cold-result edit (10 s), and #57's small units: - // about 375 s. + // #131's check past the bound (190 s), the stale authored mutant + // (38 s), a cold-result edit (20 s), and three small units: about + // 270 s. 3: anyOf([ - 'a declaration fault: noTypescript', - 'a second daemon check of the budget witness', + 'a check through the daemon that works past the bound', 'a stale authored mutant and a reformat', 'an edit that changed the cold result after a hang replaced the only context', - 'a failing owner under a config that --config names', - 'a stop before the CLI starts', - 'a stop by SIGTERM as the CLI starts', - 'an unsupervised run stopped after its note', - 'a start refuses a dangling claim', - 'a supervised check with no ps on its PATH', - // #131's daemon witnesses: the beat's (a 10 s confirmation through - // the daemon), the slow warm-up, the held check, and both F-D paths, - // moved off shard 3, which ran 11.2 min with them. - 'a check through the daemon that works past the bound', - 'a slow but healthy warm-up', - 'a check whose daemon stops answering', 'an edit to data that only a spun copy reads', 'an edit to data that only a copy spun alone reads', + 'a failing owner under a config that --config names', ]), - // The edit sequence through the daemon (94 s) and brokenOwner (69 s): - // about 325 s. + // The edit sequence through the daemon, brokenOwner, and delta 4's + // two daemon witnesses: 283 s. 4: anyOf([ 'an edit sequence through the daemon', 'a declaration fault: brokenOwner', - // Delta 4's two daemon witnesses (11 s and 20 s on Arca, #131). 'a daemon start slower than the bound', 'a check abandoned while a context warms in place of a cut one', ]), - // A pause that cannot engage (26 s), the lazy module (13 s), two - // mid-check test edits (12 s and 10 s), and a reset check (11 s): - // about 145 s. + // apiLessTypescript, noTypescript (55 s), a pause that cannot engage, + // the lazy module, two mid-check test edits, and a reset check: about + // 160 s. 5: anyOf([ - // apiLessTypescript (146 s at 59b5c449, with its drawn base) moved - // here from the rest, which took 8m07s with it. 'a declaration fault: apiLessTypescript', + 'a declaration fault: noTypescript', 'a daemon pause that cannot engage', 'a lazy module loaded before a hang in the same batch', 'a test edits the helper it read, during a check through the daemon', 'a test edits the data it read, during a check through the daemon', 'a reset check after its warm daemon was replaced', ]), - // The rest: generated (10 s, not doubled) and small units, about - // 260 s, and any new unit. - 6: rest, + // A second daemon check (89 s), #131's daemon that stops answering + // (22 s) and slow warm-up (19 s), and #57's stop units: about 155 s, + // and about 285 s when a drawn base of `a stop before the CLI starts` + // fails and shrinks (#150). + 6: anyOf([ + 'a second daemon check of the budget witness', + 'a check whose daemon stops answering', + 'a slow but healthy warm-up', + 'a stop before the CLI starts', + 'a stop by SIGTERM as the CLI starts', + 'an unsupervised run stopped after its note', + 'a start refuses a dangling claim', + 'a supervised check with no ps on its PATH', + ]), + // The rest: generated (not doubled) and small units, 232 s, and any + // new unit. + 7: rest, }, }, 'oracle-gaps-static-windows': { From d7e0b784c519b5ffea30c8cb15d1a4b3f192d222 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Wed, 18 Nov 2026 05:26:41 +0000 Subject: [PATCH 24/29] ci: eight gap e2e jobs, each with a minute of margin Shards 3 and 5 took 280 s and 313 s on an Arclet at 3ea7bcc4, within a minute of the limit at CI's factor (about 1.8); noTypescript, two mid-check edits, and the stale authored mutant run in a job of their own. Co-authored-by: Isaac --- .github/workflows/pr.yml | 6 +++--- packages/oracles/scripts/pr-shards.mjs | 26 ++++++++++++++------------ 2 files changed, 17 insertions(+), 15 deletions(-) diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 7796a26a..061e35d9 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -108,12 +108,12 @@ jobs: env: SHARD: ${{ matrix.shard }} test-oracle-gaps-end-to-end: - name: Gap report end-to-end oracle (fixed seed, ${{ matrix.shard }}/7) + name: Gap report end-to-end oracle (fixed seed, ${{ matrix.shard }}/8) runs-on: ubuntu-latest strategy: fail-fast: false matrix: - shard: [1, 2, 3, 4, 5, 6, 7] + shard: [1, 2, 3, 4, 5, 6, 7, 8] steps: - name: Checkout uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 @@ -121,7 +121,7 @@ jobs: persist-credentials: false - name: Setup Tools uses: tanstack/config/.github/setup@e4b48f16568324f76f467aa4c2aac2f05db632c3 - # The end-to-end oracle's fixed-seed units, in seven shards. Its random + # The end-to-end oracle's fixed-seed units, in eight shards. Its random # units, calibrations, and determinism units run locally. - run: pnpm exec vitest --run --config oracle-gaps/vitest.config.mjs gap-report-oracle -t "$(node scripts/pr-shards.mjs pattern oracle-gaps-fixed-seed "$SHARD")" working-directory: packages/oracles diff --git a/packages/oracles/scripts/pr-shards.mjs b/packages/oracles/scripts/pr-shards.mjs index aa14eab2..4680b9a1 100644 --- a/packages/oracles/scripts/pr-shards.mjs +++ b/packages/oracles/scripts/pr-shards.mjs @@ -327,12 +327,10 @@ export const partitions = { 'every outcome class, the abort stop', 'a declaration fault: brokenVitest', ]), - // #131's check past the bound (190 s), the stale authored mutant - // (38 s), a cold-result edit (20 s), and three small units: about - // 270 s. + // #131's check past the bound (190 s), a cold-result edit (20 s), and + // three small units: about 240 s. 3: anyOf([ 'a check through the daemon that works past the bound', - 'a stale authored mutant and a reformat', 'an edit that changed the cold result after a hang replaced the only context', 'an edit to data that only a spun copy reads', 'an edit to data that only a copy spun alone reads', @@ -346,16 +344,12 @@ export const partitions = { 'a daemon start slower than the bound', 'a check abandoned while a context warms in place of a cut one', ]), - // apiLessTypescript, noTypescript (55 s), a pause that cannot engage, - // the lazy module, two mid-check test edits, and a reset check: about - // 160 s. + // apiLessTypescript, a pause that cannot engage, the lazy module, and + // a reset check. 5: anyOf([ 'a declaration fault: apiLessTypescript', - 'a declaration fault: noTypescript', 'a daemon pause that cannot engage', 'a lazy module loaded before a hang in the same batch', - 'a test edits the helper it read, during a check through the daemon', - 'a test edits the data it read, during a check through the daemon', 'a reset check after its warm daemon was replaced', ]), // A second daemon check (89 s), #131's daemon that stops answering @@ -372,9 +366,17 @@ export const partitions = { 'a start refuses a dangling claim', 'a supervised check with no ps on its PATH', ]), - // The rest: generated (not doubled) and small units, 232 s, and any + // noTypescript, the two mid-check test edits, and the stale authored + // mutant, moved off shards 3 and 5 (280 s and 313 s at 3ea7bcc4). + 7: anyOf([ + 'a declaration fault: noTypescript', + 'a test edits the helper it read, during a check through the daemon', + 'a test edits the data it read, during a check through the daemon', + 'a stale authored mutant and a reformat', + ]), + // The rest: generated (not doubled) and small units, 238 s, and any // new unit. - 7: rest, + 8: rest, }, }, 'oracle-gaps-static-windows': { From 460b7b363f30331183d77b8b9b1963a9ae2ede01 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Wed, 18 Nov 2026 09:04:20 +0000 Subject: [PATCH 25/29] test(oracle-gaps): one fixed base and no drawn base in the e2e oracle Register M, 2026-11-18, "Gap e2e: 1 fixed base plus random-seed jobs": its per-run randomness comes from the random-seed jobs that every pull request runs (#90), so its constructed histories draw no base, and its jobs keep #90's partitions. Co-authored-by: Isaac --- .../gap-report-oracle.property.test.mjs | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs b/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs index 14cdd687..8072cdeb 100644 --- a/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs +++ b/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs @@ -6521,16 +6521,13 @@ oracleProperty({ `a check that ran more than one batch`, ], runs: 2, - // One fixed base per constructed history, and one more drawn from the - // run's random seed, so each run explores another base at about today's - // cost (register M, 2026-11-17, "Random bases at today's cost for - // expensive oracles"). Each runs the CLI end to end, often for minutes: - // five fixed bases took a fixed-seed shard to 18 to 30 minutes on an - // Arclet (#122). Windows draws none: one history there already takes 9 to - // 12 minutes of its non-blocking job (#72), and a drawn base would pass - // the job's 20-minute timeout. + // One fixed base per constructed history and none drawn: the per-run + // randomness comes from the random-seed jobs that every pull request runs + // (#90), which build each constructed history from a fresh sample (register + // M, 2026-11-18, "Gap e2e: 1 fixed base plus random-seed jobs"). Each + // history runs the CLI end to end, often for minutes: five fixed bases + // took a fixed-seed shard to 18 to 30 minutes on an Arclet (#122). bases: 1, - drawnBases: process.platform === `win32` ? 0 : 1, fixedSeeds: [20261105], constructed: withDeclaration([ { From cf2bca810658edc5d5fdd71a35dca3059a4d103c Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Wed, 18 Nov 2026 10:12:49 +0000 Subject: [PATCH 26/29] ci: seven self-oracle jobs, each with a minute of margin At deb5997f, shard 3 (the constructed histories' determinism units and the last calibrations) took 9m49s. The calibrations join the random-seed campaign units, which ran 3m22s and 3m59s as two jobs. Co-authored-by: Isaac --- .github/workflows/pr.yml | 7 +++---- packages/oracles/scripts/pr-shards.mjs | 21 +++++++++++---------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index f7c126d1..534eef87 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -60,12 +60,12 @@ jobs: - run: pnpm run build - run: pnpm --filter @tanstack/eslint-plugin-oracles run test test-oracle-framework: - name: Framework self-oracle (${{ matrix.shard }}/8) + name: Framework self-oracle (${{ matrix.shard }}/7) runs-on: ubuntu-latest strategy: fail-fast: false matrix: - shard: [1, 2, 3, 4, 5, 6, 7, 8] + shard: [1, 2, 3, 4, 5, 6, 7] steps: - name: Checkout uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 @@ -74,7 +74,7 @@ jobs: - name: Setup Tools uses: tanstack/config/.github/setup@e4b48f16568324f76f467aa4c2aac2f05db632c3 # The oracleProperty self-oracle spawns real Vitest runs, so the - # default test run leaves it out; it runs here on its own, in eight + # default test run leaves it out; it runs here on its own, in seven # shards, packed to about 9 minutes each. - run: pnpm exec vitest --run --config vitest.oracle-framework.config.mjs -t "$(node scripts/pr-shards.mjs pattern oracle-framework "$SHARD")" working-directory: packages/oracles @@ -201,7 +201,6 @@ jobs: - oracle-framework 5 - oracle-framework 6 - oracle-framework 7 - - oracle-framework 8 steps: - name: Checkout uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 diff --git a/packages/oracles/scripts/pr-shards.mjs b/packages/oracles/scripts/pr-shards.mjs index fb174aad..89d1c775 100644 --- a/packages/oracles/scripts/pr-shards.mjs +++ b/packages/oracles/scripts/pr-shards.mjs @@ -254,22 +254,23 @@ export const partitions = { suite: `self-oracle.framework`, select: ``, // Packed to about 9 minutes a job (register M, 2026-11-17, "Pack CI jobs - // to about 9 min"), from the CI times at #122's 59b5c449: the first two - // determinism shards (4m33s and 5m03s) run together, and so do the - // constructed histories' determinism units and the last calibrations - // (6m02s and 3m19s). The campaign units, which run one fixed base and one - // drawn from the run's seed (#122), run in three jobs: the fixed-seed - // units, and the random-seed units split by name (about 249 s of CI time - // before `round3`, and about 260 s with the rest). + // to about 9 min"), from the CI times at #122's 59b5c449 and deb5997f: + // the first two determinism shards (4m33s and 5m03s) run together; the + // constructed histories' determinism units alone (9m49s with the last + // calibrations); and the campaign units, which run one fixed base and one + // drawn from the run's seed (#122), as the fixed-seed units and, in the + // rest, the random-seed units with the last calibrations. shards: { 1: `(?:${startsWith(`determinism: fixed history: `, detFixedA)}|${startsWith(`determinism: fixed history: `, detFixedB)})`, 2: startsWith(`determinism: fixed history: `, detFixedC), - 3: `(?:determinism: constructed history: .*|${startsWith(`calibration: `, [...calibration3, ...calibration4])})`, + 3: `determinism: constructed history: .*`, 4: `(?:${startsWith(`fixed history: `, fixedA)}|${startsWith(`calibration: `, calibration1)})`, 5: `(?:${startsWith(`fixed history: `, fixedB)}|${startsWith(`calibration: `, calibration2)})`, 6: `fixed seed 1: .*`, - 7: `random seed: (?:[a-q]|r(?!ound[3-9])).*`, - 8: rest, + // The rest: the random-seed campaign units (3m22s and 3m59s as two jobs + // at deb5997f), the last calibrations (3m19s at 59b5c449), and any new + // kind. + 7: rest, }, }, // The gap report's static oracle, the slowest of the fast gap suites and From e463ea7deffb6c91fcbe8305fea041fa035a6fbc Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Wed, 18 Nov 2026 11:08:05 +0000 Subject: [PATCH 27/29] ci: eight self-oracle jobs; no two determinism shards share one Two determinism shards together ran 8m37s and 9m42s in two CI runs, so each runs alone, the first beside the first half of the random-seed campaign units and the last calibrations beside the second half. Co-authored-by: Isaac --- .github/workflows/pr.yml | 7 +++--- packages/oracles/scripts/pr-shards.mjs | 32 +++++++++++++------------- 2 files changed, 20 insertions(+), 19 deletions(-) diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 534eef87..f7c126d1 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -60,12 +60,12 @@ jobs: - run: pnpm run build - run: pnpm --filter @tanstack/eslint-plugin-oracles run test test-oracle-framework: - name: Framework self-oracle (${{ matrix.shard }}/7) + name: Framework self-oracle (${{ matrix.shard }}/8) runs-on: ubuntu-latest strategy: fail-fast: false matrix: - shard: [1, 2, 3, 4, 5, 6, 7] + shard: [1, 2, 3, 4, 5, 6, 7, 8] steps: - name: Checkout uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 @@ -74,7 +74,7 @@ jobs: - name: Setup Tools uses: tanstack/config/.github/setup@e4b48f16568324f76f467aa4c2aac2f05db632c3 # The oracleProperty self-oracle spawns real Vitest runs, so the - # default test run leaves it out; it runs here on its own, in seven + # default test run leaves it out; it runs here on its own, in eight # shards, packed to about 9 minutes each. - run: pnpm exec vitest --run --config vitest.oracle-framework.config.mjs -t "$(node scripts/pr-shards.mjs pattern oracle-framework "$SHARD")" working-directory: packages/oracles @@ -201,6 +201,7 @@ jobs: - oracle-framework 5 - oracle-framework 6 - oracle-framework 7 + - oracle-framework 8 steps: - name: Checkout uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 diff --git a/packages/oracles/scripts/pr-shards.mjs b/packages/oracles/scripts/pr-shards.mjs index 89d1c775..8c92ac35 100644 --- a/packages/oracles/scripts/pr-shards.mjs +++ b/packages/oracles/scripts/pr-shards.mjs @@ -254,23 +254,23 @@ export const partitions = { suite: `self-oracle.framework`, select: ``, // Packed to about 9 minutes a job (register M, 2026-11-17, "Pack CI jobs - // to about 9 min"), from the CI times at #122's 59b5c449 and deb5997f: - // the first two determinism shards (4m33s and 5m03s) run together; the - // constructed histories' determinism units alone (9m49s with the last - // calibrations); and the campaign units, which run one fixed base and one - // drawn from the run's seed (#122), as the fixed-seed units and, in the - // rest, the random-seed units with the last calibrations. + // to about 9 min"), from CI runs 44198893231 (59b5c449), 44279328397 + // (deb5997f) and 44283777367 (cf2bca81). Two determinism shards together + // ran 8m37s and 9m42s, so each runs alone; the first runs beside the + // first half of the random-seed campaign units (4m33s and 3m22s), and the + // last calibrations beside the second half, in the rest. The campaign + // units run one fixed base and one drawn from the run's seed (#122). shards: { - 1: `(?:${startsWith(`determinism: fixed history: `, detFixedA)}|${startsWith(`determinism: fixed history: `, detFixedB)})`, - 2: startsWith(`determinism: fixed history: `, detFixedC), - 3: `determinism: constructed history: .*`, - 4: `(?:${startsWith(`fixed history: `, fixedA)}|${startsWith(`calibration: `, calibration1)})`, - 5: `(?:${startsWith(`fixed history: `, fixedB)}|${startsWith(`calibration: `, calibration2)})`, - 6: `fixed seed 1: .*`, - // The rest: the random-seed campaign units (3m22s and 3m59s as two jobs - // at deb5997f), the last calibrations (3m19s at 59b5c449), and any new - // kind. - 7: rest, + 1: `(?:${startsWith(`determinism: fixed history: `, detFixedA)}|random seed: (?:[a-q]|r(?!ound[3-9])).*)`, + 2: startsWith(`determinism: fixed history: `, detFixedB), + 3: startsWith(`determinism: fixed history: `, detFixedC), + 4: `determinism: constructed history: .*`, + 5: `(?:${startsWith(`fixed history: `, fixedA)}|${startsWith(`calibration: `, calibration1)})`, + 6: `(?:${startsWith(`fixed history: `, fixedB)}|${startsWith(`calibration: `, calibration2)})`, + 7: `fixed seed 1: .*`, + // The rest: the second half of the random-seed campaign units (3m59s), + // the last calibrations (3m19s), and any new kind. + 8: rest, }, }, // The gap report's static oracle, the slowest of the fast gap suites and From fefd9e3ca39030923445b78e079d83fe84c5d832 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Wed, 18 Nov 2026 11:37:09 +0000 Subject: [PATCH 28/29] ci: the classify group runs apart from the first static shard again Packed together they ran 8m10s and 9m08s on CI, within a minute of the 10-minute limit. Co-authored-by: Isaac --- .github/workflows/pr.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index f7c126d1..bbda3909 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -81,12 +81,12 @@ jobs: env: SHARD: ${{ matrix.shard }} test-oracle-gaps-fast: - name: Gap report oracles (${{ matrix.shard }}/2) + name: Gap report oracles (${{ matrix.shard }}/3) runs-on: ubuntu-latest strategy: fail-fast: false matrix: - shard: [1, 2] + shard: [1, 2, 3] steps: - name: Checkout uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 @@ -96,12 +96,12 @@ jobs: uses: tanstack/config/.github/setup@e4b48f16568324f76f467aa4c2aac2f05db632c3 # The gap report's own oracles also run real Vitest runs on fixture # packages (test:oracle-gaps:fast): the static oracle in two shards, - # the first beside the classify, probe-form, and mutant-validity - # oracles, so each job runs about 8 minutes (register M, 2026-11-17, - # "Pack CI jobs to about 9 min"). + # and the classify, probe-form, and mutant-validity oracles in a third. + # Packed with the first static shard, that job ran 8m10s and 9m08s on + # #122's CI, within a minute of the limit, so it runs apart. - run: | - if [ "$SHARD" = 1 ]; then - pnpm exec vitest --run --config oracle-gaps/vitest.config.mjs gap-report-classify gap-report-probe-form gap-report-mutant-validity gap-report-path-spelling + if [ "$SHARD" = 3 ]; then + exec pnpm exec vitest --run --config oracle-gaps/vitest.config.mjs gap-report-classify gap-report-probe-form gap-report-mutant-validity gap-report-path-spelling fi pnpm exec vitest --run --config oracle-gaps/vitest.config.mjs gap-report-static -t "$(node scripts/pr-shards.mjs pattern oracle-gaps-static "$SHARD")" working-directory: packages/oracles From 616dbcc5a4ecdbb7b373ca6a702b933b3586adf0 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Wed, 18 Nov 2026 20:53:01 +0000 Subject: [PATCH 29/29] ci: the first half of the random-seed campaign units runs alone Beside a determinism shard it ran 7m59s and 9m42s in two CI runs; its bases vary from run to run. The self-oracle runs in nine jobs. Co-authored-by: Isaac --- .github/workflows/pr.yml | 7 ++++--- packages/oracles/scripts/pr-shards.mjs | 14 +++++++++----- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index bbda3909..3fafd6fb 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -60,12 +60,12 @@ jobs: - run: pnpm run build - run: pnpm --filter @tanstack/eslint-plugin-oracles run test test-oracle-framework: - name: Framework self-oracle (${{ matrix.shard }}/8) + name: Framework self-oracle (${{ matrix.shard }}/9) runs-on: ubuntu-latest strategy: fail-fast: false matrix: - shard: [1, 2, 3, 4, 5, 6, 7, 8] + shard: [1, 2, 3, 4, 5, 6, 7, 8, 9] steps: - name: Checkout uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 @@ -74,7 +74,7 @@ jobs: - name: Setup Tools uses: tanstack/config/.github/setup@e4b48f16568324f76f467aa4c2aac2f05db632c3 # The oracleProperty self-oracle spawns real Vitest runs, so the - # default test run leaves it out; it runs here on its own, in eight + # default test run leaves it out; it runs here on its own, in nine # shards, packed to about 9 minutes each. - run: pnpm exec vitest --run --config vitest.oracle-framework.config.mjs -t "$(node scripts/pr-shards.mjs pattern oracle-framework "$SHARD")" working-directory: packages/oracles @@ -202,6 +202,7 @@ jobs: - oracle-framework 6 - oracle-framework 7 - oracle-framework 8 + - oracle-framework 9 steps: - name: Checkout uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 diff --git a/packages/oracles/scripts/pr-shards.mjs b/packages/oracles/scripts/pr-shards.mjs index 8c92ac35..91dc9c29 100644 --- a/packages/oracles/scripts/pr-shards.mjs +++ b/packages/oracles/scripts/pr-shards.mjs @@ -256,21 +256,25 @@ export const partitions = { // Packed to about 9 minutes a job (register M, 2026-11-17, "Pack CI jobs // to about 9 min"), from CI runs 44198893231 (59b5c449), 44279328397 // (deb5997f) and 44283777367 (cf2bca81). Two determinism shards together - // ran 8m37s and 9m42s, so each runs alone; the first runs beside the - // first half of the random-seed campaign units (4m33s and 3m22s), and the - // last calibrations beside the second half, in the rest. The campaign + // ran 8m37s and 9m42s, so each runs alone. The first half of the + // random-seed campaign units runs alone too (beside a determinism shard + // it ran 7m59s and 9m42s, run 44297247671), and the last calibrations + // run beside the second half, in the rest. The campaign // units run one fixed base and one drawn from the run's seed (#122). shards: { - 1: `(?:${startsWith(`determinism: fixed history: `, detFixedA)}|random seed: (?:[a-q]|r(?!ound[3-9])).*)`, + 1: startsWith(`determinism: fixed history: `, detFixedA), 2: startsWith(`determinism: fixed history: `, detFixedB), 3: startsWith(`determinism: fixed history: `, detFixedC), 4: `determinism: constructed history: .*`, 5: `(?:${startsWith(`fixed history: `, fixedA)}|${startsWith(`calibration: `, calibration1)})`, 6: `(?:${startsWith(`fixed history: `, fixedB)}|${startsWith(`calibration: `, calibration2)})`, 7: `fixed seed 1: .*`, + // The first half of the random-seed campaign units: its bases vary from + // run to run, and beside a determinism shard it ran 7m59s and 9m42s. + 8: `random seed: (?:[a-q]|r(?!ound[3-9])).*`, // The rest: the second half of the random-seed campaign units (3m59s), // the last calibrations (3m19s), and any new kind. - 8: rest, + 9: rest, }, }, // The gap report's static oracle, the slowest of the fast gap suites and