diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 496f60fdb..170e924b0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -5,11 +5,11 @@ # enforced by whoever remembered to run it, and a broken main was discovered at release # time. Everything below is what a maintainer already does by hand. # -# WHY e2e-smoke IS NOT REQUIRED YET: it renders WebGL through SwiftShader on a GPU-less -# runner, and this project's own notes measure ~4.5 fps there plus a documented family of -# timing-sensitive suites. Blocking every PR on that before it has been observed green on -# GitHub would train people to ignore a red tick, which is worse than not having it. It -# reports, it does not gate, and the comment below says when to flip it. +# e2e-smoke GATES since test-health-1 (feat/60). It renders WebGL through SwiftShader on a +# GPU-less runner (~4.5 fps), so it was report-only until it had been seen green on GitHub: +# the runner step now really fails (it used to `|| echo` every red away), a failed suite runs +# once more (green on the retry = flaky, reported), and a known flake is entered in +# tests/e2e/quarantine.json with an owner and a review date instead of blocking every PR. # # Two-peer suites stay a MANUAL gate: they meet on the self-hosted signaling box, which a # public runner cannot reach and should not be pointed at. @@ -150,9 +150,7 @@ jobs: e2e-smoke: runs-on: ubuntu-latest - # NOT a gate yet — see the header. Flip this to false once it has been green on a few - # PRs in a row; until then a red here is a signal to read, not a block. - continue-on-error: true + # a gate — see the header (green on GitHub for bc453ae8 and e77ca3f7 before the flip) steps: - uses: actions/checkout@v4 - uses: actions/setup-node@v4 @@ -180,14 +178,20 @@ jobs: # Deliberately NOT added: net-stress and signaling-reconnect are multi-peer and # meet on the self-hosted signaling box, which a public runner cannot reach and # should not be pointed at (see the header) — they stay a manual gate. - for s in net-backoff net-mesh wire-hardening runtime-resilience diagnostics mesh-budget script-guard dispose approval-timeout; do - echo "::group::$s" - npm run e2e -- "$s" || echo "SUITE FAILED: $s" - echo "::endgroup::" - done + # test-health-1: through the battery runner, so a red is a red (the old loop's + # `|| echo SUITE FAILED` could never fail the step), a failed suite runs once more + # (green on the retry = flaky, reported) and tests/e2e/quarantine.json applies. + node scripts/ci/battery.cjs run --only net-backoff,net-mesh,wire-hardening,runtime-resilience,diagnostics,mesh-budget,script-guard,dispose,approval-timeout --port 5173 --out /tmp/smoke - name: dev server log on failure if: failure() run: tail -40 /tmp/dev.log + - name: suite logs + if: always() + uses: actions/upload-artifact@v4 + with: + name: e2e-smoke-logs + path: /tmp/smoke/logs + if-no-files-found: ignore # 53-reader (A4): THE ACCESSIBILITY GATE. axe-core over the main surfaces (editor shell, object # list, Inspector, Settings, Asset Browser on recorded fixtures, phone shell) FAILS on serious and @@ -230,6 +234,13 @@ jobs: - name: dev server log on failure if: failure() run: tail -40 /tmp/dev.log + - name: suite logs + if: always() + uses: actions/upload-artifact@v4 + with: + name: e2e-smoke-logs + path: /tmp/smoke/logs + if-no-files-found: ignore # 38 (NOTES-38 #18): THE TOKEN SCREENSHOT DIFF. /kit renders every redesign primitive in every # state; this job shoots it in the dark, light and custom themes on the PR's BASE ref and on its diff --git a/check-baseline.json b/check-baseline.json index b7dfe7ec8..c512987b8 100644 --- a/check-baseline.json +++ b/check-baseline.json @@ -1,6 +1,6 @@ { "comment": "27-I: the svelte-check floor, read ONLY by scripts/check-ratchet.cjs. It used to be hardcoded in release.yml's shell block, where it went stale (362 while the tree measured 359). Ratchet it DOWN whenever a change legitimately removes errors - that is the project convention, and --update does it in one command.", - "errors": 299, + "errors": 288, "warnings": 39, - "measured": "2026-10-11" + "measured": "2026-10-24" } diff --git a/perf/budgets.json b/perf/budgets.json index ef7734fc2..737962320 100644 --- a/perf/budgets.json +++ b/perf/budgets.json @@ -16,10 +16,10 @@ "textureMB": 64 }, "targets": { - "comment": "46 B1/B2 — the editor's budget chip and Statistics ▸ Budget judge a scene against ONE of these (picked in the chip's menu, LOCAL). Quest's four gated metrics are `defaults` above and are NOT repeated here (perfBudget.cjs refuses a copy, so the chip and the gate cannot disagree); the gate never reads this block. shadowLights / shadowCasters are meter-only rows; shadowCasters counts shadow DRAWS (casters x shadow lights) and is held to the draw-call budget, since every one is a call. Phone and desktop are starting values, not gated: phone = sceneBudget's measured VR/mobile green ceilings, desktop = 26-E's measured desktop green ceilings (owed on device).", + "comment": "46 B1/B2 — the editor's budget chip and Statistics ▸ Budget judge a scene against ONE of these (picked in the chip's menu, LOCAL). Quest's four gated metrics are `defaults` above and are NOT repeated here (perfBudget.cjs refuses a copy, so the chip and the gate cannot disagree); the gate never reads this block. shadowLights / shadowCasters are meter-only rows; shadowCasters counts shadow DRAWS (casters x shadow lights) and is held to the draw-call budget, since every one is a call. Phone and desktop are starting values, not gated: phone = sceneBudget's measured VR/mobile green ceilings, desktop = 26-E's measured desktop green ceilings (owed on device) test-health-1 (2026-10-24) reconciled them with cloud plans/core/pending/research/quest-phone-perf.md §1: phone = its MID phone row (calls 100-150 [3P] -> 150, shadow draws held to that; 500k triangles, 256 MB, one shadow map already inside it), desktop texture memory 512 -> 1024 MB (its 1-2 GB [U]). Quest is unchanged: 150 calls is inside <200 [M] / 100-150 [3P], and its triangles / texture MB are STRICTER than the research's [U] web estimates (500-750k, <=512 MB), which say measure on device.", "quest": { "label": "Quest / VR", "short": "Quest", "shadowLights": 1, "shadowCasters": 150 }, - "phone": { "label": "Phone", "short": "Phone", "calls": 300, "triangles": 500000, "lights": 3, "textureMB": 256, "shadowLights": 1, "shadowCasters": 300 }, - "desktop": { "label": "Desktop", "short": "Desktop", "calls": 2000, "triangles": 4000000, "lights": 8, "textureMB": 512, "shadowLights": 4, "shadowCasters": 2000 } + "phone": { "label": "Phone", "short": "Phone", "calls": 150, "triangles": 500000, "lights": 3, "textureMB": 256, "shadowLights": 1, "shadowCasters": 150 }, + "desktop": { "label": "Desktop", "short": "Desktop", "calls": 2000, "triangles": 4000000, "lights": 8, "textureMB": 1024, "shadowLights": 4, "shadowCasters": 2000 } }, "games": {}, "levels": {}, diff --git a/scripts/ci/battery.cjs b/scripts/ci/battery.cjs index f994cf223..f4c6d6c77 100755 --- a/scripts/ci/battery.cjs +++ b/scripts/ci/battery.cjs @@ -4,9 +4,10 @@ // The battery is the list an integrator runs before a release: tests/e2e/battery.json holds it // (suite → seconds measured on a GitHub runner, or twice the desk time until one has run), the // suites a public runner must skip (`skip`: they need the self-hosted signaling box, a hosts-file -// origin or a built app) and the ones that run but do not gate (`optional`: red or flaky on -// SwiftShader with the same red on the unchanged base — the reason is on the line). Nothing here -// names a suite; the list is data, refreshed from the integrators' battery runs (`refresh`). +// origin or a built app). What runs but does not gate is tests/e2e/quarantine.json (test-health-1: +// it replaced battery.json's suite-wide `optional` map) — per CHECK, with an owner and a review date, +// read through scripts/ci/quarantine.cjs. Nothing here names a suite; the list is data, refreshed from +// the integrators' battery runs (`refresh`). // // node scripts/ci/battery.cjs plan # matrix JSON for GITHUB_OUTPUT // node scripts/ci/battery.cjs run --shard 3 --of 14 # run one shard (serves the app itself) @@ -15,20 +16,22 @@ // // Sharding is longest-first onto the least-loaded shard, by name on ties: every shard computes the // same assignment from the same file, so the plan job only has to say how many shards there are. -// A failed suite runs once more; green on the retry = `flaky` (reported, not a red). +// A failed suite runs once more; green on the retry = `flaky` (reported, not a red). Red twice and +// every failed check quarantined = `quarantined` (reported in its own tier, not a red). const fs = require('fs'); const path = require('path'); const { spawn, spawnSync } = require('child_process'); +const quarantine = require('./quarantine.cjs'); const ROOT = path.resolve(__dirname, '..', '..'); const BATTERY = path.join(ROOT, 'tests', 'e2e', 'battery.json'); /** * @typedef {{ comment?: string, refs: Record, shards: {target_minutes: number, max: number, min: number}, - * pc?: {games: string[], levels: string[]}, skip: Record, optional: Record, setup: Record, + * pc?: {games: string[], levels: string[]}, skip: Record, setup: Record, * suites: Record }} Battery - * @typedef {{ name: string, status: 'pass'|'flaky'|'fail'|'timeout'|'gone', required: boolean, rc: number|null, secs: number, + * @typedef {{ name: string, status: 'pass'|'flaky'|'quarantined'|'fail'|'timeout'|'gone', required: boolean, rc: number|null, secs: number, * retrySecs?: number, pass: number, fail: number, last: string, shard: number }} SuiteResult */ @@ -166,7 +169,7 @@ function runSuite(name, est, out, tag) { fs.closeSync(log); const text = fs.readFileSync(logFile, 'utf8'); if (timedOut) fs.appendFileSync(logFile, `\n[battery] TIMEOUT after ${timeout / 1000}s\n`); - resolve({ rc: timedOut ? null : code, secs: Math.round((Date.now() - t0) / 1000), timedOut, ...counts(text) }); + resolve({ rc: timedOut ? null : code, secs: Math.round((Date.now() - t0) / 1000), timedOut, text, ...counts(text) }); }); }); } @@ -174,6 +177,12 @@ function runSuite(name, est, out, tag) { /** @param {string[]} argv */ async function run(argv) { const b = load(); + const q = quarantine.load(); + const qErrors = quarantine.validate(q); + if (qErrors.length) { + for (const e of qErrors) console.error('quarantine.json: ' + e); + process.exit(2); + } const of = Number(opt(argv, '--of') || shardCount(b)); const shard = Number(opt(argv, '--shard') || 1); const out = path.resolve(opt(argv, '--out') || process.env.BATTERY_OUT || path.join(ROOT, '..', 'battery-out')); @@ -187,7 +196,7 @@ async function run(argv) { const results = []; console.log(`shard ${shard}/${of}: ${mine.length} suites, est ${Math.round(mine.reduce((t, s) => t + (b.suites[s] || 60), 0) / 60)} min`); for (const name of mine) { - const required = !(name in b.optional); + const required = true; if (!fs.existsSync(path.join(ROOT, 'tests', 'e2e', name + '.test.cjs'))) { results.push({ name, status: 'gone', required: false, rc: null, secs: 0, pass: 0, fail: 0, last: 'no such suite (refresh battery.json)', shard }); continue; @@ -214,10 +223,19 @@ async function run(argv) { if (again.rc === 0) status = 'flaky'; else r = again; } - const res = { name, status, required, rc: r.rc, secs: r.secs, retrySecs, pass: r.pass, fail: r.fail, last: r.last, shard }; + /** @type {string[]} */ + let quarantinedChecks = []; + if (status === 'fail' || status === 'timeout') { + const v = quarantine.classify(q, name, r.text, { timedOut: r.timedOut }); + if (v.quarantined) { + status = 'quarantined'; + quarantinedChecks = v.whole ? ['(whole suite)'] : v.failed; + } + } + const res = { name, status, required, rc: r.rc, secs: r.secs, retrySecs, pass: r.pass, fail: r.fail, last: quarantinedChecks.length ? 'quarantined: ' + quarantinedChecks.join(' | ').slice(0, 140) : r.last, shard }; results.push(res); - if (status === 'pass' || status === 'flaky') fs.rmSync(path.join(out, 'evidence', name), { recursive: true, force: true }); - console.log(`${name} ${status}${required ? '' : ' (optional)'} rc=${r.rc} pass~${r.pass} fail~${r.fail} ${r.secs}s${retrySecs ? ` retry ${retrySecs}s` : ''} :: ${r.last}`); + if (status === 'pass' || status === 'flaky' || status === 'quarantined') fs.rmSync(path.join(out, 'evidence', name), { recursive: true, force: true }); + console.log(`${name} ${status} rc=${r.rc} pass~${r.pass} fail~${r.fail} ${r.secs}s${retrySecs ? ` retry ${retrySecs}s` : ''} :: ${r.last}`); } stop(server); fs.writeFileSync(path.join(out, `results-${shard}.json`), JSON.stringify({ shard, of, results }, null, 1)); @@ -232,14 +250,14 @@ async function run(argv) { function table(results, title) { const red = results.filter((r) => r.status === 'fail' || r.status === 'timeout'); const req = red.filter((r) => r.required); - const opt = red.filter((r) => !r.required); + const quar = results.filter((r) => r.status === 'quarantined'); const flaky = results.filter((r) => r.status === 'flaky'); const secs = results.reduce((t, r) => t + r.secs + (r.retrySecs || 0), 0); const row = (/** @type {SuiteResult} */ r) => `| ${r.name} | ${r.status} | ${r.shard} | ${r.pass}/${r.fail} | ${r.secs}s | ${r.last.replace(/\|/g, '\\|')} |`; const head = '| suite | status | shard | ✓/✗ | time | last line |\n|---|---|---|---|---|---|\n'; - let md = `### ${title}: ${results.length} suites, ${req.length} required red, ${opt.length} optional red, ${flaky.length} flaky (${Math.round(secs / 60)} suite-min)\n\n`; + let md = `### ${title}: ${results.length} suites, ${req.length} required red, ${quar.length} quarantined, ${flaky.length} flaky (${Math.round(secs / 60)} suite-min)\n\n`; if (req.length) md += `**Required reds**\n\n${head}${req.map(row).join('\n')}\n\n`; - if (opt.length) md += `
Optional reds (do not gate)\n\n${head}${opt.map(row).join('\n')}\n\n
\n\n`; + if (quar.length) md += `
Quarantined reds (tests/e2e/quarantine.json — do not gate)\n\n${head}${quar.map(row).join('\n')}\n\n
\n\n`; if (flaky.length) md += `
Flaky (green on the retry)\n\n${head}${flaky.map(row).join('\n')}\n\n
\n\n`; const gone = results.filter((r) => r.status === 'gone'); if (gone.length) md += `Listed but gone (refresh tests/e2e/battery.json): ${gone.map((r) => r.name).join(', ')}\n\n`; @@ -262,6 +280,8 @@ function report(argv) { const b = load(); const missing = Array.from({ length: of || shardCount(b) }, (_, i) => i + 1).filter((s) => !seen.has(s)); let md = table(all, 'e2e battery'); + const qWarn = quarantine.warnings(quarantine.load(), all, Object.keys(b.suites)); + if (qWarn.length) md += `
Quarantine to review (${qWarn.length})\n\n${qWarn.map((w) => '- ' + w).join('\n')}\n\n
\n\n`; if (missing.length) md += `**Shards with no results (crashed or cancelled): ${missing.join(', ')}**\n`; if (process.env.GITHUB_STEP_SUMMARY) fs.appendFileSync(process.env.GITHUB_STEP_SUMMARY, md); else console.log(md); @@ -296,7 +316,6 @@ function write(b) { `\t"shards": ${JSON.stringify(b.shards)}`, `\t"pc": ${JSON.stringify(b.pc || { games: [], levels: [] })}`, `\t"skip": ${obj(b.skip, '\t')}`, - `\t"optional": ${obj(b.optional, '\t')}`, `\t"setup": ${obj(b.setup, '\t')}`, `\t"suites": ${obj(b.suites, '\t')}` ]; @@ -329,9 +348,9 @@ function refresh(argv) { for (const [n, s] of Object.entries(JSON.parse(fs.readFileSync(f, 'utf8')))) if (n in b.suites) b.suites[n] = Number(s); } } - for (const k of Object.keys(b.optional)) if (!(k in b.suites)) delete b.optional[k]; write(b); - console.log(`battery.json: ${Object.keys(b.suites).length} suites, ${Object.keys(b.skip).length} skipped, ${Object.keys(b.optional).length} optional, ${shardCount(b)} shards`); + console.log(`battery.json: ${Object.keys(b.suites).length} suites, ${Object.keys(b.skip).length} skipped, ${shardCount(b)} shards`); + for (const w of quarantine.warnings(quarantine.load(), [], Object.keys(b.suites))) console.log('warning: ' + w); } // ---------------------------------------------------------------- plan diff --git a/scripts/ci/quarantine.cjs b/scripts/ci/quarantine.cjs new file mode 100644 index 000000000..2f79f0333 --- /dev/null +++ b/scripts/ci/quarantine.cjs @@ -0,0 +1,173 @@ +#!/usr/bin/env node +// test-health-1 — THE QUARANTINE: known flaky or known-red e2e checks, by name, with an owner and a date. +// +// The battery (scripts/ci/battery.cjs, and the integrators' battery.sh through `triage`) gates on every +// red EXCEPT one listed in tests/e2e/quarantine.json. It is perf/budgets.json's `allow` list one domain +// over: an entry records a known debt, it is not a place to put a red you have not read. Every entry +// names the suite, optionally ONE check (a substring of its `FAIL