From c2f718ee502eac0c0414059df98f21a852b66eaf Mon Sep 17 00:00:00 2001 From: AlexZ005 Date: Sat, 24 Oct 2026 17:13:20 +0300 Subject: [PATCH 01/46] [test] test-health-1 P0: budget targets follow the Quest/phone research MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - perf/budgets.json targets reconciled with cloud plans/core/pending/research/quest-phone-perf.md §1: phone calls 300 -> 150 and shadow draws 300 -> 150 (the MID phone row, 100-150 [3P]); desktop texture memory 512 -> 1024 MB (1-2 GB [U]). The comment records the source. - Quest unchanged: 150 calls is inside <200 [M] / 100-150 [3P]; its 300k triangles / 64 MB are stricter than the research's [U] web estimates (QUESTIONS Q1). Low-phone row = follow-up (Q2). - validateBudgets clean; vitest perfBudget + budgetMeterCore 44/0; svelte-check unchanged (298/39). Co-Authored-By: Claude Opus 5.5 (1M context) --- perf/budgets.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/perf/budgets.json b/perf/budgets.json index ef7734fc2..737962320 100644 --- a/perf/budgets.json +++ b/perf/budgets.json @@ -16,10 +16,10 @@ "textureMB": 64 }, "targets": { - "comment": "46 B1/B2 — the editor's budget chip and Statistics ▸ Budget judge a scene against ONE of these (picked in the chip's menu, LOCAL). Quest's four gated metrics are `defaults` above and are NOT repeated here (perfBudget.cjs refuses a copy, so the chip and the gate cannot disagree); the gate never reads this block. shadowLights / shadowCasters are meter-only rows; shadowCasters counts shadow DRAWS (casters x shadow lights) and is held to the draw-call budget, since every one is a call. Phone and desktop are starting values, not gated: phone = sceneBudget's measured VR/mobile green ceilings, desktop = 26-E's measured desktop green ceilings (owed on device).", + "comment": "46 B1/B2 — the editor's budget chip and Statistics ▸ Budget judge a scene against ONE of these (picked in the chip's menu, LOCAL). Quest's four gated metrics are `defaults` above and are NOT repeated here (perfBudget.cjs refuses a copy, so the chip and the gate cannot disagree); the gate never reads this block. shadowLights / shadowCasters are meter-only rows; shadowCasters counts shadow DRAWS (casters x shadow lights) and is held to the draw-call budget, since every one is a call. Phone and desktop are starting values, not gated: phone = sceneBudget's measured VR/mobile green ceilings, desktop = 26-E's measured desktop green ceilings (owed on device) test-health-1 (2026-10-24) reconciled them with cloud plans/core/pending/research/quest-phone-perf.md §1: phone = its MID phone row (calls 100-150 [3P] -> 150, shadow draws held to that; 500k triangles, 256 MB, one shadow map already inside it), desktop texture memory 512 -> 1024 MB (its 1-2 GB [U]). Quest is unchanged: 150 calls is inside <200 [M] / 100-150 [3P], and its triangles / texture MB are STRICTER than the research's [U] web estimates (500-750k, <=512 MB), which say measure on device.", "quest": { "label": "Quest / VR", "short": "Quest", "shadowLights": 1, "shadowCasters": 150 }, - "phone": { "label": "Phone", "short": "Phone", "calls": 300, "triangles": 500000, "lights": 3, "textureMB": 256, "shadowLights": 1, "shadowCasters": 300 }, - "desktop": { "label": "Desktop", "short": "Desktop", "calls": 2000, "triangles": 4000000, "lights": 8, "textureMB": 512, "shadowLights": 4, "shadowCasters": 2000 } + "phone": { "label": "Phone", "short": "Phone", "calls": 150, "triangles": 500000, "lights": 3, "textureMB": 256, "shadowLights": 1, "shadowCasters": 150 }, + "desktop": { "label": "Desktop", "short": "Desktop", "calls": 2000, "triangles": 4000000, "lights": 8, "textureMB": 1024, "shadowLights": 4, "shadowCasters": 2000 } }, "games": {}, "levels": {}, From 607b3fecb4b90ce4fdb9d69c9fb851283ce13fcf Mon Sep 17 00:00:00 2001 From: AlexZ005 Date: Sat, 24 Oct 2026 17:22:58 +0300 Subject: [PATCH 02/46] [test] test-health-1 P1: e2e quarantine manifest + the battery's non-blocking tier MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - tests/e2e/quarantine.json: suite + optional check (substring of its FAIL label) + reason / owner / since / review — perf/budgets.json "allow" for e2e. Seeded with 25 check-level flakes (lock-ux, ui-fixes-15lmno x3, checkpoints x3, scene-load L3 1.4/6.3/4.4, net-handshake, flow-physics-actions x4, select-41 G4, avatar-feet, aquarium-reef x2, knock-physics, pack-behavior, code-workspace, animation-node, dock-strip-drop, toolbar-slots-42; sources in each reason: 50-int, deps-1, mux-pc-host, 53/57/59/65 STATUS) + the 30 whole-suite carried reds moved from battery.json "optional" (phase 2 removes them one by one). Every label was checked against today's suite text. - scripts/ci/quarantine.cjs: validate / classify / warnings / triage. Non-blocking only when EVERY failed check of the run matches an entry for that suite; a crash or timeout never matches a check. - scripts/ci/battery.cjs: `optional` gone; a red-twice suite the quarantine covers = `quarantined` (own tier in the step summary, not a red); report lists past-review / now-green entries. - tests/unit/quarantine.test.js 10/0 (counterfactuals: a new red beside a quarantined row gates; a crash gates; another suite's identical label gates). depsCi 7/0. svelte-check 298/39 (= base). Co-Authored-By: Claude Opus 5.5 (1M context) --- scripts/ci/battery.cjs | 53 +++++++---- scripts/ci/quarantine.cjs | 173 ++++++++++++++++++++++++++++++++++ tests/e2e/README.md | 16 ++++ tests/e2e/battery.json | 36 +------ tests/e2e/quarantine.json | 60 ++++++++++++ tests/unit/quarantine.test.js | 73 ++++++++++++++ 6 files changed, 359 insertions(+), 52 deletions(-) create mode 100644 scripts/ci/quarantine.cjs create mode 100644 tests/e2e/quarantine.json create mode 100644 tests/unit/quarantine.test.js diff --git a/scripts/ci/battery.cjs b/scripts/ci/battery.cjs index f994cf223..f4c6d6c77 100755 --- a/scripts/ci/battery.cjs +++ b/scripts/ci/battery.cjs @@ -4,9 +4,10 @@ // The battery is the list an integrator runs before a release: tests/e2e/battery.json holds it // (suite → seconds measured on a GitHub runner, or twice the desk time until one has run), the // suites a public runner must skip (`skip`: they need the self-hosted signaling box, a hosts-file -// origin or a built app) and the ones that run but do not gate (`optional`: red or flaky on -// SwiftShader with the same red on the unchanged base — the reason is on the line). Nothing here -// names a suite; the list is data, refreshed from the integrators' battery runs (`refresh`). +// origin or a built app). What runs but does not gate is tests/e2e/quarantine.json (test-health-1: +// it replaced battery.json's suite-wide `optional` map) — per CHECK, with an owner and a review date, +// read through scripts/ci/quarantine.cjs. Nothing here names a suite; the list is data, refreshed from +// the integrators' battery runs (`refresh`). // // node scripts/ci/battery.cjs plan # matrix JSON for GITHUB_OUTPUT // node scripts/ci/battery.cjs run --shard 3 --of 14 # run one shard (serves the app itself) @@ -15,20 +16,22 @@ // // Sharding is longest-first onto the least-loaded shard, by name on ties: every shard computes the // same assignment from the same file, so the plan job only has to say how many shards there are. -// A failed suite runs once more; green on the retry = `flaky` (reported, not a red). +// A failed suite runs once more; green on the retry = `flaky` (reported, not a red). Red twice and +// every failed check quarantined = `quarantined` (reported in its own tier, not a red). const fs = require('fs'); const path = require('path'); const { spawn, spawnSync } = require('child_process'); +const quarantine = require('./quarantine.cjs'); const ROOT = path.resolve(__dirname, '..', '..'); const BATTERY = path.join(ROOT, 'tests', 'e2e', 'battery.json'); /** * @typedef {{ comment?: string, refs: Record, shards: {target_minutes: number, max: number, min: number}, - * pc?: {games: string[], levels: string[]}, skip: Record, optional: Record, setup: Record, + * pc?: {games: string[], levels: string[]}, skip: Record, setup: Record, * suites: Record }} Battery - * @typedef {{ name: string, status: 'pass'|'flaky'|'fail'|'timeout'|'gone', required: boolean, rc: number|null, secs: number, + * @typedef {{ name: string, status: 'pass'|'flaky'|'quarantined'|'fail'|'timeout'|'gone', required: boolean, rc: number|null, secs: number, * retrySecs?: number, pass: number, fail: number, last: string, shard: number }} SuiteResult */ @@ -166,7 +169,7 @@ function runSuite(name, est, out, tag) { fs.closeSync(log); const text = fs.readFileSync(logFile, 'utf8'); if (timedOut) fs.appendFileSync(logFile, `\n[battery] TIMEOUT after ${timeout / 1000}s\n`); - resolve({ rc: timedOut ? null : code, secs: Math.round((Date.now() - t0) / 1000), timedOut, ...counts(text) }); + resolve({ rc: timedOut ? null : code, secs: Math.round((Date.now() - t0) / 1000), timedOut, text, ...counts(text) }); }); }); } @@ -174,6 +177,12 @@ function runSuite(name, est, out, tag) { /** @param {string[]} argv */ async function run(argv) { const b = load(); + const q = quarantine.load(); + const qErrors = quarantine.validate(q); + if (qErrors.length) { + for (const e of qErrors) console.error('quarantine.json: ' + e); + process.exit(2); + } const of = Number(opt(argv, '--of') || shardCount(b)); const shard = Number(opt(argv, '--shard') || 1); const out = path.resolve(opt(argv, '--out') || process.env.BATTERY_OUT || path.join(ROOT, '..', 'battery-out')); @@ -187,7 +196,7 @@ async function run(argv) { const results = []; console.log(`shard ${shard}/${of}: ${mine.length} suites, est ${Math.round(mine.reduce((t, s) => t + (b.suites[s] || 60), 0) / 60)} min`); for (const name of mine) { - const required = !(name in b.optional); + const required = true; if (!fs.existsSync(path.join(ROOT, 'tests', 'e2e', name + '.test.cjs'))) { results.push({ name, status: 'gone', required: false, rc: null, secs: 0, pass: 0, fail: 0, last: 'no such suite (refresh battery.json)', shard }); continue; @@ -214,10 +223,19 @@ async function run(argv) { if (again.rc === 0) status = 'flaky'; else r = again; } - const res = { name, status, required, rc: r.rc, secs: r.secs, retrySecs, pass: r.pass, fail: r.fail, last: r.last, shard }; + /** @type {string[]} */ + let quarantinedChecks = []; + if (status === 'fail' || status === 'timeout') { + const v = quarantine.classify(q, name, r.text, { timedOut: r.timedOut }); + if (v.quarantined) { + status = 'quarantined'; + quarantinedChecks = v.whole ? ['(whole suite)'] : v.failed; + } + } + const res = { name, status, required, rc: r.rc, secs: r.secs, retrySecs, pass: r.pass, fail: r.fail, last: quarantinedChecks.length ? 'quarantined: ' + quarantinedChecks.join(' | ').slice(0, 140) : r.last, shard }; results.push(res); - if (status === 'pass' || status === 'flaky') fs.rmSync(path.join(out, 'evidence', name), { recursive: true, force: true }); - console.log(`${name} ${status}${required ? '' : ' (optional)'} rc=${r.rc} pass~${r.pass} fail~${r.fail} ${r.secs}s${retrySecs ? ` retry ${retrySecs}s` : ''} :: ${r.last}`); + if (status === 'pass' || status === 'flaky' || status === 'quarantined') fs.rmSync(path.join(out, 'evidence', name), { recursive: true, force: true }); + console.log(`${name} ${status} rc=${r.rc} pass~${r.pass} fail~${r.fail} ${r.secs}s${retrySecs ? ` retry ${retrySecs}s` : ''} :: ${r.last}`); } stop(server); fs.writeFileSync(path.join(out, `results-${shard}.json`), JSON.stringify({ shard, of, results }, null, 1)); @@ -232,14 +250,14 @@ async function run(argv) { function table(results, title) { const red = results.filter((r) => r.status === 'fail' || r.status === 'timeout'); const req = red.filter((r) => r.required); - const opt = red.filter((r) => !r.required); + const quar = results.filter((r) => r.status === 'quarantined'); const flaky = results.filter((r) => r.status === 'flaky'); const secs = results.reduce((t, r) => t + r.secs + (r.retrySecs || 0), 0); const row = (/** @type {SuiteResult} */ r) => `| ${r.name} | ${r.status} | ${r.shard} | ${r.pass}/${r.fail} | ${r.secs}s | ${r.last.replace(/\|/g, '\\|')} |`; const head = '| suite | status | shard | ✓/✗ | time | last line |\n|---|---|---|---|---|---|\n'; - let md = `### ${title}: ${results.length} suites, ${req.length} required red, ${opt.length} optional red, ${flaky.length} flaky (${Math.round(secs / 60)} suite-min)\n\n`; + let md = `### ${title}: ${results.length} suites, ${req.length} required red, ${quar.length} quarantined, ${flaky.length} flaky (${Math.round(secs / 60)} suite-min)\n\n`; if (req.length) md += `**Required reds**\n\n${head}${req.map(row).join('\n')}\n\n`; - if (opt.length) md += `
Optional reds (do not gate)\n\n${head}${opt.map(row).join('\n')}\n\n
\n\n`; + if (quar.length) md += `
Quarantined reds (tests/e2e/quarantine.json — do not gate)\n\n${head}${quar.map(row).join('\n')}\n\n
\n\n`; if (flaky.length) md += `
Flaky (green on the retry)\n\n${head}${flaky.map(row).join('\n')}\n\n
\n\n`; const gone = results.filter((r) => r.status === 'gone'); if (gone.length) md += `Listed but gone (refresh tests/e2e/battery.json): ${gone.map((r) => r.name).join(', ')}\n\n`; @@ -262,6 +280,8 @@ function report(argv) { const b = load(); const missing = Array.from({ length: of || shardCount(b) }, (_, i) => i + 1).filter((s) => !seen.has(s)); let md = table(all, 'e2e battery'); + const qWarn = quarantine.warnings(quarantine.load(), all, Object.keys(b.suites)); + if (qWarn.length) md += `
Quarantine to review (${qWarn.length})\n\n${qWarn.map((w) => '- ' + w).join('\n')}\n\n
\n\n`; if (missing.length) md += `**Shards with no results (crashed or cancelled): ${missing.join(', ')}**\n`; if (process.env.GITHUB_STEP_SUMMARY) fs.appendFileSync(process.env.GITHUB_STEP_SUMMARY, md); else console.log(md); @@ -296,7 +316,6 @@ function write(b) { `\t"shards": ${JSON.stringify(b.shards)}`, `\t"pc": ${JSON.stringify(b.pc || { games: [], levels: [] })}`, `\t"skip": ${obj(b.skip, '\t')}`, - `\t"optional": ${obj(b.optional, '\t')}`, `\t"setup": ${obj(b.setup, '\t')}`, `\t"suites": ${obj(b.suites, '\t')}` ]; @@ -329,9 +348,9 @@ function refresh(argv) { for (const [n, s] of Object.entries(JSON.parse(fs.readFileSync(f, 'utf8')))) if (n in b.suites) b.suites[n] = Number(s); } } - for (const k of Object.keys(b.optional)) if (!(k in b.suites)) delete b.optional[k]; write(b); - console.log(`battery.json: ${Object.keys(b.suites).length} suites, ${Object.keys(b.skip).length} skipped, ${Object.keys(b.optional).length} optional, ${shardCount(b)} shards`); + console.log(`battery.json: ${Object.keys(b.suites).length} suites, ${Object.keys(b.skip).length} skipped, ${shardCount(b)} shards`); + for (const w of quarantine.warnings(quarantine.load(), [], Object.keys(b.suites))) console.log('warning: ' + w); } // ---------------------------------------------------------------- plan diff --git a/scripts/ci/quarantine.cjs b/scripts/ci/quarantine.cjs new file mode 100644 index 000000000..2f79f0333 --- /dev/null +++ b/scripts/ci/quarantine.cjs @@ -0,0 +1,173 @@ +#!/usr/bin/env node +// test-health-1 — THE QUARANTINE: known flaky or known-red e2e checks, by name, with an owner and a date. +// +// The battery (scripts/ci/battery.cjs, and the integrators' battery.sh through `triage`) gates on every +// red EXCEPT one listed in tests/e2e/quarantine.json. It is perf/budgets.json's `allow` list one domain +// over: an entry records a known debt, it is not a place to put a red you have not read. Every entry +// names the suite, optionally ONE check (a substring of its `FAIL