diff --git a/.github/workflows/test-quality.yml b/.github/workflows/test-quality.yml index 88605860a..2785585fc 100644 --- a/.github/workflows/test-quality.yml +++ b/.github/workflows/test-quality.yml @@ -16,6 +16,8 @@ concurrency: cancel-in-progress: true env: + BUN_BASELINE_VERSION: "1.3.14" + BUN_CANDIDATE_VERSION: "1.4.0" CODEX_SECURITY_INTEGRATION: "0" CODEX_SECURITY_ALLOW_MACHINE_POLICY_TEST: "false" @@ -30,35 +32,42 @@ jobs: fail-fast: false matrix: os: [ubuntu-latest, windows-latest] - mode: [baseline, isolated, parallel] + mode: [baseline, native-baseline] include: - mode: baseline args: "" - - mode: isolated - args: --isolate - - mode: parallel - args: --parallel=2 + - mode: native-baseline + args: "" + - os: ubuntu-latest + mode: native-shard-1 + args: --shard=1/3 --timings=reports/native-test-timings.json + - os: ubuntu-latest + mode: native-shard-2 + args: --shard=2/3 --timings=reports/native-test-timings.json + - os: ubuntu-latest + mode: native-shard-3 + args: --shard=3/3 --timings=reports/native-test-timings.json - os: windows-latest - mode: shard-1 - args: --shard=1/7 + mode: native-shard-1 + args: --shard=1/7 --timings=reports/native-test-timings.json - os: windows-latest - mode: shard-2 - args: --shard=2/7 + mode: native-shard-2 + args: --shard=2/7 --timings=reports/native-test-timings.json - os: windows-latest - mode: shard-3 - args: --shard=3/7 + mode: native-shard-3 + args: --shard=3/7 --timings=reports/native-test-timings.json - os: windows-latest - mode: shard-4 - args: --shard=4/7 + mode: native-shard-4 + args: --shard=4/7 --timings=reports/native-test-timings.json - os: windows-latest - mode: shard-5 - args: --shard=5/7 + mode: native-shard-5 + args: --shard=5/7 --timings=reports/native-test-timings.json - os: windows-latest - mode: shard-6 - args: --shard=6/7 + mode: native-shard-6 + args: --shard=6/7 --timings=reports/native-test-timings.json - os: windows-latest - mode: shard-7 - args: --shard=7/7 + mode: native-shard-7 + args: --shard=7/7 --timings=reports/native-test-timings.json steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: @@ -76,7 +85,7 @@ jobs: plugins/codex-security/mcp-app/pnpm-lock.yaml - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 with: - bun-version: "1.3.13" + bun-version: ${{ matrix.mode == 'baseline' && env.BUN_BASELINE_VERSION || env.BUN_CANDIDATE_VERSION }} - name: Install dependencies run: | pnpm --dir sdk/typescript install --frozen-lockfile @@ -86,14 +95,27 @@ jobs: id: windows-temp shell: pwsh run: ./sdk/typescript/scripts/prepare-windows-test-root.ps1 + - name: Prepare native timing estimates + shell: bash + working-directory: sdk/typescript + run: | + node --input-type=module <<'NODE' + import { mkdirSync, readFileSync, writeFileSync } from 'node:fs'; + const measured = JSON.parse(readFileSync('scripts/ci-test-durations.json', 'utf8')); + const durations = measured[process.platform === 'win32' ? 'windows' : 'unix']; + mkdirSync('reports', { recursive: true }); + writeFileSync('reports/native-test-timings.json', JSON.stringify({ + version: 1, + files: Object.fromEntries(Object.entries(durations).map(([file, seconds]) => [`tests-ts/${file}`, seconds * 1000])), + })); + NODE - name: Test runner mode env: TEMP: ${{ steps.windows-temp.outputs.path || runner.temp }} TMP: ${{ steps.windows-temp.outputs.path || runner.temp }} TMPDIR: ${{ steps.windows-temp.outputs.path || runner.temp }} run: | - node -e "require('node:fs').mkdirSync('sdk/typescript/reports',{recursive:true})" - pnpm --dir sdk/typescript run test ${{ runner.os == 'Windows' && '--timeout=120000' || '' }} ${{ matrix.args }} --seed=${{ env.CODEX_SECURITY_PROPERTY_SEED }} --reporter=junit --reporter-outfile=reports/runner-${{ matrix.os }}-${{ matrix.mode }}.xml + pnpm --dir sdk/typescript run test ${{ runner.os == 'Windows' && '--timeout=120000' || '' }} ${{ matrix.args }} --path-ignore-patterns='**/windows-machine-policy.test.ts' --seed=${{ env.CODEX_SECURITY_PROPERTY_SEED }} --reporter=junit --reporter-outfile=reports/runner-${{ matrix.os }}-${{ matrix.mode }}.xml - name: Upload runner report if: always() uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 @@ -123,11 +145,9 @@ jobs: run: | comparison_status=0 for os in ubuntu-latest windows-latest; do - for mode in isolated parallel; do - python3 sdk/typescript/scripts/compare-test-reports.py "reports/runner-$os-baseline.xml" "reports/runner-$os-$mode.xml" >> "$GITHUB_STEP_SUMMARY" || comparison_status=1 - done + python3 sdk/typescript/scripts/compare-test-reports.py "reports/runner-$os-baseline.xml" "reports/runner-$os-native-baseline.xml" >> "$GITHUB_STEP_SUMMARY" || comparison_status=1 + python3 sdk/typescript/scripts/compare-test-reports.py "reports/runner-$os-native-baseline.xml" "reports/runner-$os-native-shard-*.xml" >> "$GITHUB_STEP_SUMMARY" || comparison_status=1 done - python3 sdk/typescript/scripts/compare-test-reports.py reports/runner-windows-latest-baseline.xml 'reports/runner-windows-latest-shard-*.xml' >> "$GITHUB_STEP_SUMMARY" || comparison_status=1 exit "$comparison_status" mutation: diff --git a/sdk/typescript/TESTING.md b/sdk/typescript/TESTING.md index 261ce99c6..7fd52af18 100644 --- a/sdk/typescript/TESTING.md +++ b/sdk/typescript/TESTING.md @@ -110,19 +110,24 @@ Failed tests and missing package artifacts block CI; failed diagnostic uploads do not. Use `python -m pytest -n 0` to reproduce Python failures serially. The separate `test-quality` workflow runs weekly, can be dispatched manually, -and runs on pull requests that change its workflow file. It compares Bun's -default runner, `--isolate`, `--parallel=2`, and seven-way Windows sharding. +and runs on pull requests that change its workflow file. Its baseline uses Bun +1.3.14, matching required CI. It compares that run with an unsharded Bun 1.4.0 +candidate, then compares the candidate with native timing-aware shards: three +on Linux and seven on Windows. Native timing files are derived from the same +checked-in CI estimates as the required file-balanced runner, converting +seconds to Bun's milliseconds. New files remain eligible without an estimate. + Every mode uses the same seed for property cases and test ordering: pull requests replay seed 1; scheduled and manual runs use the workflow run number. -The workflow compares test identities and outcomes against the unsharded -default run and records timings, including failed shards, in the job summary. -It is not a required check or part of the release trigger. - -Runner trials use Bun 1.3.13 to avoid the -[async-module initialization bug in 1.3.14](https://github.com/oven-sh/bun/issues/31410) -that breaks the Ink UI tests under isolation. Keep the trial pin until a newer -release passes the full SDK suite in every mode. Required CI and the mutation -trial remain on Bun 1.3.14. +The comparison checks test identities and outcomes and records test durations +from JUnit reports, including failed shards, in the job summary. It is not a +required check or part of the release trigger. Machine-wide policy tests are +excluded from every trial and retain their separate serial CI step. + +The candidate does not enable process parallelism or isolation. Those modes +have separate compatibility concerns and are not needed to evaluate native +sharding. Required CI and the mutation trial stay on Bun 1.3.14 while this +experiment establishes candidate compatibility and shard inventories. Keep the measured file-balanced runner until the native runner has matching inventories and acceptable Windows timings. Before promotion, compare diff --git a/sdk/typescript/tests-ts/cli-publish.test.ts b/sdk/typescript/tests-ts/cli-publish.test.ts index 3b7fdbcc3..2a13178a6 100644 --- a/sdk/typescript/tests-ts/cli-publish.test.ts +++ b/sdk/typescript/tests-ts/cli-publish.test.ts @@ -1086,7 +1086,8 @@ describe("publish scan", () => { deps, ), ).toBe(0); - expect(stdout.text()).toContain(" SEC-400\n"); + // Terminal sanitizers may also discard the C1 control's printable payload. + expect(stdout.text()).toMatch(/\n {2}SEC-400(?: safe)?\n/u); for (const identifier of ["SEC-401", "SEC-402", "SEC-403", "SEC-404"]) { expect(stdout.text()).toContain(` ${identifier}\n`); } diff --git a/sdk/typescript/tests-ts/release-automation.test.ts b/sdk/typescript/tests-ts/release-automation.test.ts index c8f11d8ca..de8e86685 100644 --- a/sdk/typescript/tests-ts/release-automation.test.ts +++ b/sdk/typescript/tests-ts/release-automation.test.ts @@ -789,14 +789,14 @@ describe("monotonic stable release versions", () => { }); test.each([ - { version: "0.1.1", registryError: { error: { code: "E404" } } }, - { version: "0.1.0", registryError: { error: { code: "E403" } } }, - { version: "0.1.0", registryError: { error: { code: "ETIMEDOUT" } } }, - { version: "0.1.0", registryError: { error: null } }, - { version: "0.1.0", registryError: null }, - ])( - "rejects unverified npm history for $version and registry response $registryError", - ({ version, registryError }) => { + ["0.1.1", { error: { code: "E404" } }], + ["0.1.0", { error: { code: "E403" } }], + ["0.1.0", { error: { code: "ETIMEDOUT" } }], + ["0.1.0", { error: null }], + ["0.1.0", null], + ] as const)( + "rejects unverified npm history for %s and registry response %j", + (version, registryError) => { expect(() => initialPublishedVersions(version, registryError)).toThrow( "Unable to verify published npm release history.", ); @@ -4288,7 +4288,7 @@ describe("GitHub release workflow safeguards", () => { "fix: generated title\n\nUnreviewed injected highlight\n", "fix: preserve a trailing line feed\n", "fix: preserve a trailing carriage return\r", - ])("active title gates reject %s", (title) => { + ])("active title gates reject %j", (title) => { for (const [workflow, step] of [ [nodeCiWorkflow, "Require a Conventional Commit pull request title"], [titleWorkflow, "Check conventional title"], @@ -4330,7 +4330,7 @@ describe("GitHub release workflow safeguards", () => { "fix: generated title\n\nUnreviewed injected highlight\n", "fix: preserve a trailing line feed\n", "fix: preserve a trailing carriage return\r", - ])("rejects nonconventional pull request title %s", async (title) => { + ])("rejects nonconventional pull request title %j", async (title) => { const script = workflowStepShell( releaseLabelsWorkflow, "Categorize pull request without checking out its code", diff --git a/sdk/typescript/tests-ts/scan-dashboard.test.ts b/sdk/typescript/tests-ts/scan-dashboard.test.ts index db1ca1a14..8b5ae476d 100644 --- a/sdk/typescript/tests-ts/scan-dashboard.test.ts +++ b/sdk/typescript/tests-ts/scan-dashboard.test.ts @@ -1,6 +1,6 @@ import { EventEmitter } from "node:events"; import { Writable } from "node:stream"; -import { stripVTControlCharacters } from "node:util"; +import { pathToFileURL } from "node:url"; import { describe, expect, test } from "bun:test"; import type { ComponentReceipt } from "../src/component-scan.js"; import { ScanDashboard } from "../src/scan-dashboard.js"; @@ -17,7 +17,7 @@ function fakeClock(now: () => number = () => STARTED_AT) { } function lastFrame(stderr: ReturnType): string { - return stripVTControlCharacters(stderr.text()) + return Bun.stripANSI(stderr.text()) .split("CODEX SECURITY · juice-shop") .at(-1)!; } @@ -72,8 +72,7 @@ describe("live scan dashboard", () => { outputDir: `/synthetic/results/${index + 1}`, }), ); - const frame = () => - stripVTControlCharacters(stderr.text().split("\u001B[H").at(-1)!); + const frame = () => Bun.stripANSI(stderr.text().split("\u001B[H").at(-1)!); dashboard.start(); dashboard.setComponents(receipts); for (const receipt of receipts.slice(0, 2)) @@ -202,8 +201,7 @@ describe("live scan dashboard", () => { outputDir: `/synthetic/${index}`, }), ); - const frame = () => - stripVTControlCharacters(stderr.text().split("\u001B[H").at(-1)!); + const frame = () => Bun.stripANSI(stderr.text().split("\u001B[H").at(-1)!); dashboard.start(); dashboard.setComponents(receipts); dashboard.updateComponent({ @@ -242,7 +240,7 @@ describe("live scan dashboard", () => { dashboard.setStage("Connecting to Linear"); dashboard.start(); input.emit("data", "d"); - let text = stripVTControlCharacters(stderr.text()); + let text = Bun.stripANSI(stderr.text()); expect(text).toContain("CODEX SECURITY · PUBLISH · payments-api"); expect(text).toContain("Waiting for publication activity"); expect(text).toContain("FINDINGS waiting for findings"); @@ -261,7 +259,7 @@ describe("live scan dashboard", () => { description: "Preparing the next Linear issue.", paths: [], }); - text = stripVTControlCharacters(stderr.text()); + text = Bun.stripANSI(stderr.text()); expect(text).toContain("FINDINGS 2 / 5 processed"); expect(text).toContain("Publishing findings · 2/5"); expect(text).toContain("Preparing the next Linear issue."); @@ -467,7 +465,7 @@ describe("live scan dashboard", () => { ); dashboard.stop(); - const text = stripVTControlCharacters(stderr.text()); + const text = Bun.stripANSI(stderr.text()); expect(text).toContain("CODEX SECURITY · juice-shop"); expect(text).not.toContain("ACTIVITY"); expect(text).not.toContain("events · live"); @@ -625,7 +623,7 @@ describe("live scan dashboard", () => { }); dashboard.stop(); - expect(stripVTControlCharacters(stderr.text())).toContain( + expect(Bun.stripANSI(stderr.text())).toContain( '[09:41:00] ◐ worker 2 · rg -n "password" routes/login.ts', ); }); @@ -1109,7 +1107,7 @@ describe("live scan dashboard", () => { test("renders compact clickable local Markdown links without repeating timestamps", () => { const stderr = capture(true); const target = - "/private/tmp/codex security/scans/promptfoo-cloud/artifacts/02_discovery/raw_candidates_02.jsonl"; + "/synthetic/codex security/scans/example/artifacts/02_discovery/raw_candidates_02.jsonl"; const dashboard = new ScanDashboard( { ...stderr.stream, columns: 55, rows: 18 }, { repository: "/code/juice-shop", clock: fakeClock() }, @@ -1127,10 +1125,10 @@ describe("live scan dashboard", () => { const frame = lastFrame(stderr); expect(frame).toContain("raw_candidates_02.jsonl"); - expect(frame).not.toContain("/private/tmp/"); + expect(frame).not.toContain("/synthetic/"); expect(frame.match(/\[09:41:00\]/gu)).toHaveLength(1); expect(stderr.text()).toContain( - `\u001B]8;;file:///private/tmp/codex%20security/scans/promptfoo-cloud/artifacts/02_discovery/raw_candidates_02.jsonl\u0007raw_candidates_02.jsonl\u001B]8;;\u0007`, + `\u001B]8;;${pathToFileURL(target).href}\u0007raw_candidates_02.jsonl\u001B]8;;\u0007`, ); dashboard.stop(); }); @@ -1198,9 +1196,11 @@ describe("live scan dashboard", () => { expect(frame).not.toContain("`db.raw( input )`"); expect(stderr.text()).toContain("\u001B[2mdb.raw( input )\u001B[22m"); expect(stderr.text()).toContain( - "\u001B]8;;file:///tmp/report.md\u0007report\u001B]8;;\u0007", + `\u001B]8;;${pathToFileURL("/tmp/report.md").href}\u0007report\u001B]8;;\u0007`, + ); + expect(stderr.text()).not.toContain( + `\u001B]8;;${pathToFileURL("/tmp/example.md").href}`, ); - expect(stderr.text()).not.toContain("\u001B]8;;file:///tmp/example.md"); dashboard.record({ id: "command-inline-code", @@ -1420,8 +1420,6 @@ describe("live scan dashboard", () => { dashboard.stop(); expect(stderr.text()).not.toContain("\u001B[31m"); - expect(stripVTControlCharacters(stderr.text())).toContain( - "routes/spoofed.ts", - ); + expect(Bun.stripANSI(stderr.text())).toContain("routes/spoofed.ts"); }); }); diff --git a/sdk/typescript/tests-ts/skeleton.test.ts b/sdk/typescript/tests-ts/skeleton.test.ts index 27fda31d6..ee5a3b596 100644 --- a/sdk/typescript/tests-ts/skeleton.test.ts +++ b/sdk/typescript/tests-ts/skeleton.test.ts @@ -288,15 +288,11 @@ describe("TypeScript package skeleton", () => { "false", ); expect(quality.env?.["CODEX_SECURITY_INTEGRATION"]).toBe("0"); - for (let shard = 1; shard <= 7; shard += 1) { - expect( - quality.jobs["runner"]?.strategy?.matrix["include"], - ).toContainEqual({ - os: "windows-latest", - mode: `shard-${shard}`, - args: `--shard=${shard}/7`, - }); - } + expect( + quality.jobs["runner"]?.steps.find( + (step) => step.name === "Test runner mode", + )?.run, + ).toContain("--path-ignore-patterns='**/windows-machine-policy.test.ts'"); }); test("keeps runner modes reproducible and report uploads rerunnable", async () => { @@ -309,18 +305,42 @@ describe("TypeScript package skeleton", () => { expect(runner.env?.["CODEX_SECURITY_PROPERTY_SEED"]).toBe(seed); expect(runner.strategy?.matrix["mode"]).toEqual([ "baseline", - "isolated", - "parallel", + "native-baseline", ]); - for (const [mode, args] of [ - ["baseline", ""], - ["isolated", "--isolate"], - ["parallel", "--parallel=2"], + const variants = runner.strategy?.matrix["include"] as Array<{ + os?: string; + mode: string; + args: string; + }>; + const baseline = variants.find(({ mode }) => mode === "baseline")!; + const candidate = variants.find(({ mode }) => mode === "native-baseline")!; + expect(baseline.args).toBe(""); + expect(candidate.args).toBe(""); + expect(quality.env?.["BUN_CANDIDATE_VERSION"]).not.toBe( + quality.env?.["BUN_BASELINE_VERSION"], + ); + expect( + ci.jobs["test"]?.steps.find((step) => step.name === "Set up Bun")?.with?.[ + "bun-version" + ], + ).toBe(quality.env?.["BUN_BASELINE_VERSION"]); + expect( + runner.steps.find((step) => step.uses?.startsWith("oven-sh/setup-bun@")) + ?.with?.["bun-version"], + ).toBe( + "${{ matrix.mode == 'baseline' && env.BUN_BASELINE_VERSION || env.BUN_CANDIDATE_VERSION }}", + ); + for (const [os, count] of [ + ["ubuntu-latest", 3], + ["windows-latest", 7], ] as const) { - expect(runner.strategy?.matrix["include"]).toContainEqual({ - mode, - args, - }); + for (let shard = 1; shard <= count; shard += 1) { + expect(variants).toContainEqual({ + os, + mode: `native-shard-${shard}`, + args: `--shard=${shard}/${count} --timings=reports/native-test-timings.json`, + }); + } } const command = runner.steps.find( (step) => step.name === "Test runner mode", diff --git a/sdk/typescript/tests-ts/test-reports.test.ts b/sdk/typescript/tests-ts/test-reports.test.ts index d85fba7bf..f8da0ff43 100644 --- a/sdk/typescript/tests-ts/test-reports.test.ts +++ b/sdk/typescript/tests-ts/test-reports.test.ts @@ -80,14 +80,10 @@ describe("JUnit inventory comparison", () => { const script = workflow.jobs.compare.steps.find( (step) => step.name === "Compare inventories and outcomes", )!.run!; - const expected = [ - ...["ubuntu-latest", "windows-latest"].flatMap((os) => - ["isolated", "parallel"].map( - (mode) => `reports/runner-${os}-${mode}.xml`, - ), - ), - "reports/runner-windows-latest-shard-*.xml", - ]; + const expected = ["ubuntu-latest", "windows-latest"].flatMap((os) => [ + `reports/runner-${os}-native-baseline.xml`, + `reports/runner-${os}-native-shard-*.xml`, + ]); const mock = `python3() { printf '%s\\n' "$3" [[ "$3" != "$CODEX_SECURITY_TEST_FAIL_REPORT" ]]