ci: compare native Bun scheduling with the pinned baseline

.github/workflows/test-quality.yml

@@ -16,6 +16,8 @@ concurrency:
   cancel-in-progress: true
 
 env:
+  BUN_BASELINE_VERSION: "1.3.14"
+  BUN_CANDIDATE_VERSION: "1.4.0"
   CODEX_SECURITY_INTEGRATION: "0"
   CODEX_SECURITY_ALLOW_MACHINE_POLICY_TEST: "false"
 
@@ -30,35 +32,42 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-latest, windows-latest]
-        mode: [baseline, isolated, parallel]
+        mode: [baseline, native-baseline]
         include:
           - mode: baseline
             args: ""
-          - mode: isolated
-            args: --isolate
-          - mode: parallel
-            args: --parallel=2
+          - mode: native-baseline
+            args: ""
+          - os: ubuntu-latest
+            mode: native-shard-1
+            args: --shard=1/3 --timings=reports/native-test-timings.json
+          - os: ubuntu-latest
+            mode: native-shard-2
+            args: --shard=2/3 --timings=reports/native-test-timings.json
+          - os: ubuntu-latest
+            mode: native-shard-3
+            args: --shard=3/3 --timings=reports/native-test-timings.json
           - os: windows-latest
-            mode: shard-1
-            args: --shard=1/7
+            mode: native-shard-1
+            args: --shard=1/7 --timings=reports/native-test-timings.json
           - os: windows-latest
-            mode: shard-2
-            args: --shard=2/7
+            mode: native-shard-2
+            args: --shard=2/7 --timings=reports/native-test-timings.json
           - os: windows-latest
-            mode: shard-3
-            args: --shard=3/7
+            mode: native-shard-3
+            args: --shard=3/7 --timings=reports/native-test-timings.json
           - os: windows-latest
-            mode: shard-4
-            args: --shard=4/7
+            mode: native-shard-4
+            args: --shard=4/7 --timings=reports/native-test-timings.json
           - os: windows-latest
-            mode: shard-5
-            args: --shard=5/7
+            mode: native-shard-5
+            args: --shard=5/7 --timings=reports/native-test-timings.json
           - os: windows-latest
-            mode: shard-6
-            args: --shard=6/7
+            mode: native-shard-6
+            args: --shard=6/7 --timings=reports/native-test-timings.json
           - os: windows-latest
-            mode: shard-7
-            args: --shard=7/7
+            mode: native-shard-7
+            args: --shard=7/7 --timings=reports/native-test-timings.json
     steps:
       - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
         with:
@@ -76,7 +85,7 @@ jobs:
             plugins/codex-security/mcp-app/pnpm-lock.yaml
       - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
         with:
-          bun-version: "1.3.13"
+          bun-version: ${{ matrix.mode == 'baseline' && env.BUN_BASELINE_VERSION || env.BUN_CANDIDATE_VERSION }}
       - name: Install dependencies
         run: |
           pnpm --dir sdk/typescript install --frozen-lockfile
@@ -86,14 +95,27 @@ jobs:
         id: windows-temp
         shell: pwsh
         run: ./sdk/typescript/scripts/prepare-windows-test-root.ps1
+      - name: Prepare native timing estimates
+        shell: bash
+        working-directory: sdk/typescript
+        run: |
+          node --input-type=module <<'NODE'
+          import { mkdirSync, readFileSync, writeFileSync } from 'node:fs';
+          const measured = JSON.parse(readFileSync('scripts/ci-test-durations.json', 'utf8'));
+          const durations = measured[process.platform === 'win32' ? 'windows' : 'unix'];
+          mkdirSync('reports', { recursive: true });
+          writeFileSync('reports/native-test-timings.json', JSON.stringify({
+            version: 1,
+            files: Object.fromEntries(Object.entries(durations).map(([file, seconds]) => [`tests-ts/${file}`, seconds * 1000])),
+          }));
+          NODE
       - name: Test runner mode
         env:
           TEMP: ${{ steps.windows-temp.outputs.path || runner.temp }}
           TMP: ${{ steps.windows-temp.outputs.path || runner.temp }}
           TMPDIR: ${{ steps.windows-temp.outputs.path || runner.temp }}
         run: |
-          node -e "require('node:fs').mkdirSync('sdk/typescript/reports',{recursive:true})"
-          pnpm --dir sdk/typescript run test ${{ runner.os == 'Windows' && '--timeout=120000' || '' }} ${{ matrix.args }} --seed=${{ env.CODEX_SECURITY_PROPERTY_SEED }} --reporter=junit --reporter-outfile=reports/runner-${{ matrix.os }}-${{ matrix.mode }}.xml
+          pnpm --dir sdk/typescript run test ${{ runner.os == 'Windows' && '--timeout=120000' || '' }} ${{ matrix.args }} --path-ignore-patterns='**/windows-machine-policy.test.ts' --seed=${{ env.CODEX_SECURITY_PROPERTY_SEED }} --reporter=junit --reporter-outfile=reports/runner-${{ matrix.os }}-${{ matrix.mode }}.xml
       - name: Upload runner report
         if: always()
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
@@ -123,11 +145,9 @@ jobs:
         run: |
           comparison_status=0
           for os in ubuntu-latest windows-latest; do
-            for mode in isolated parallel; do
-              python3 sdk/typescript/scripts/compare-test-reports.py "reports/runner-$os-baseline.xml" "reports/runner-$os-$mode.xml" >> "$GITHUB_STEP_SUMMARY" || comparison_status=1
-            done
+            python3 sdk/typescript/scripts/compare-test-reports.py "reports/runner-$os-baseline.xml" "reports/runner-$os-native-baseline.xml" >> "$GITHUB_STEP_SUMMARY" || comparison_status=1
+            python3 sdk/typescript/scripts/compare-test-reports.py "reports/runner-$os-native-baseline.xml" "reports/runner-$os-native-shard-*.xml" >> "$GITHUB_STEP_SUMMARY" || comparison_status=1
           done
-          python3 sdk/typescript/scripts/compare-test-reports.py reports/runner-windows-latest-baseline.xml 'reports/runner-windows-latest-shard-*.xml' >> "$GITHUB_STEP_SUMMARY" || comparison_status=1
           exit "$comparison_status"
 
   mutation:

sdk/typescript/TESTING.md

@@ -110,19 +110,24 @@ Failed tests and missing package artifacts block CI; failed diagnostic uploads
 do not. Use `python -m pytest -n 0` to reproduce Python failures serially.
 
 The separate `test-quality` workflow runs weekly, can be dispatched manually,
-and runs on pull requests that change its workflow file. It compares Bun's
-default runner, `--isolate`, `--parallel=2`, and seven-way Windows sharding.
+and runs on pull requests that change its workflow file. Its baseline uses Bun
+1.3.14, matching required CI. It compares that run with an unsharded Bun 1.4.0
+candidate, then compares the candidate with native timing-aware shards: three
+on Linux and seven on Windows. Native timing files are derived from the same
+checked-in CI estimates as the required file-balanced runner, converting
+seconds to Bun's milliseconds. New files remain eligible without an estimate.
+
 Every mode uses the same seed for property cases and test ordering: pull
 requests replay seed 1; scheduled and manual runs use the workflow run number.
-The workflow compares test identities and outcomes against the unsharded
-default run and records timings, including failed shards, in the job summary.
-It is not a required check or part of the release trigger.
-
-Runner trials use Bun 1.3.13 to avoid the
-[async-module initialization bug in 1.3.14](https://github.com/oven-sh/bun/issues/31410)
-that breaks the Ink UI tests under isolation. Keep the trial pin until a newer
-release passes the full SDK suite in every mode. Required CI and the mutation
-trial remain on Bun 1.3.14.
+The comparison checks test identities and outcomes and records test durations
+from JUnit reports, including failed shards, in the job summary. It is not a
+required check or part of the release trigger. Machine-wide policy tests are
+excluded from every trial and retain their separate serial CI step.
+
+The candidate does not enable process parallelism or isolation. Those modes
+have separate compatibility concerns and are not needed to evaluate native
+sharding. Required CI and the mutation trial stay on Bun 1.3.14 while this
+experiment establishes candidate compatibility and shard inventories.
 
 Keep the measured file-balanced runner until the native runner has
 matching inventories and acceptable Windows timings. Before promotion, compare

sdk/typescript/tests-ts/cli-publish.test.ts

@@ -1086,7 +1086,8 @@ describe("publish scan", () => {
         deps,
       ),
     ).toBe(0);
-    expect(stdout.text()).toContain("  SEC-400\n");
+    // Terminal sanitizers may also discard the C1 control's printable payload.
+    expect(stdout.text()).toMatch(/\n {2}SEC-400(?: safe)?\n/u);
     for (const identifier of ["SEC-401", "SEC-402", "SEC-403", "SEC-404"]) {
       expect(stdout.text()).toContain(`  ${identifier}\n`);
     }

sdk/typescript/tests-ts/release-automation.test.ts

@@ -789,14 +789,14 @@ describe("monotonic stable release versions", () => {
   });
 
   test.each([
-    { version: "0.1.1", registryError: { error: { code: "E404" } } },
-    { version: "0.1.0", registryError: { error: { code: "E403" } } },
-    { version: "0.1.0", registryError: { error: { code: "ETIMEDOUT" } } },
-    { version: "0.1.0", registryError: { error: null } },
-    { version: "0.1.0", registryError: null },
-  ])(
-    "rejects unverified npm history for $version and registry response $registryError",
-    ({ version, registryError }) => {
+    ["0.1.1", { error: { code: "E404" } }],
+    ["0.1.0", { error: { code: "E403" } }],
+    ["0.1.0", { error: { code: "ETIMEDOUT" } }],
+    ["0.1.0", { error: null }],
+    ["0.1.0", null],
+  ] as const)(
+    "rejects unverified npm history for %s and registry response %j",
+    (version, registryError) => {
       expect(() => initialPublishedVersions(version, registryError)).toThrow(
         "Unable to verify published npm release history.",
       );
@@ -4288,7 +4288,7 @@ describe("GitHub release workflow safeguards", () => {
     "fix: generated title\n<!-- codex-security-release-summary:start -->\nUnreviewed injected highlight\n<!-- codex-security-release-summary:end -->",
     "fix: preserve a trailing line feed\n",
     "fix: preserve a trailing carriage return\r",
-  ])("active title gates reject %s", (title) => {
+  ])("active title gates reject %j", (title) => {
     for (const [workflow, step] of [
       [nodeCiWorkflow, "Require a Conventional Commit pull request title"],
       [titleWorkflow, "Check conventional title"],
@@ -4330,7 +4330,7 @@ describe("GitHub release workflow safeguards", () => {
     "fix: generated title\n<!-- codex-security-release-summary:start -->\nUnreviewed injected highlight\n<!-- codex-security-release-summary:end -->",
     "fix: preserve a trailing line feed\n",
     "fix: preserve a trailing carriage return\r",
-  ])("rejects nonconventional pull request title %s", async (title) => {
+  ])("rejects nonconventional pull request title %j", async (title) => {
     const script = workflowStepShell(
       releaseLabelsWorkflow,
       "Categorize pull request without checking out its code",

sdk/typescript/tests-ts/scan-dashboard.test.ts

@@ -1,6 +1,6 @@
 import { EventEmitter } from "node:events";
 import { Writable } from "node:stream";
-import { stripVTControlCharacters } from "node:util";
+import { pathToFileURL } from "node:url";
 import { describe, expect, test } from "bun:test";
 import type { ComponentReceipt } from "../src/component-scan.js";
 import { ScanDashboard } from "../src/scan-dashboard.js";
@@ -17,7 +17,7 @@ function fakeClock(now: () => number = () => STARTED_AT) {
 }
 
 function lastFrame(stderr: ReturnType<typeof capture>): string {
-  return stripVTControlCharacters(stderr.text())
+  return Bun.stripANSI(stderr.text())
     .split("CODEX SECURITY  ·  juice-shop")
     .at(-1)!;
 }
@@ -72,8 +72,7 @@ describe("live scan dashboard", () => {
         outputDir: `/synthetic/results/${index + 1}`,
       }),
     );
-    const frame = () =>
-      stripVTControlCharacters(stderr.text().split("\u001B[H").at(-1)!);
+    const frame = () => Bun.stripANSI(stderr.text().split("\u001B[H").at(-1)!);
     dashboard.start();
     dashboard.setComponents(receipts);
     for (const receipt of receipts.slice(0, 2))
@@ -202,8 +201,7 @@ describe("live scan dashboard", () => {
         outputDir: `/synthetic/${index}`,
       }),
     );
-    const frame = () =>
-      stripVTControlCharacters(stderr.text().split("\u001B[H").at(-1)!);
+    const frame = () => Bun.stripANSI(stderr.text().split("\u001B[H").at(-1)!);
     dashboard.start();
     dashboard.setComponents(receipts);
     dashboard.updateComponent({
@@ -242,7 +240,7 @@ describe("live scan dashboard", () => {
     dashboard.setStage("Connecting to Linear");
     dashboard.start();
     input.emit("data", "d");
-    let text = stripVTControlCharacters(stderr.text());
+    let text = Bun.stripANSI(stderr.text());
     expect(text).toContain("CODEX SECURITY  ·  PUBLISH  ·  payments-api");
     expect(text).toContain("Waiting for publication activity");
     expect(text).toContain("FINDINGS  waiting for findings");
@@ -261,7 +259,7 @@ describe("live scan dashboard", () => {
       description: "Preparing the next Linear issue.",
       paths: [],
     });
-    text = stripVTControlCharacters(stderr.text());
+    text = Bun.stripANSI(stderr.text());
     expect(text).toContain("FINDINGS  2 / 5 processed");
     expect(text).toContain("Publishing findings · 2/5");
     expect(text).toContain("Preparing the next Linear issue.");
@@ -467,7 +465,7 @@ describe("live scan dashboard", () => {
     );
     dashboard.stop();
 
-    const text = stripVTControlCharacters(stderr.text());
+    const text = Bun.stripANSI(stderr.text());
     expect(text).toContain("CODEX SECURITY  ·  juice-shop");
     expect(text).not.toContain("ACTIVITY");
     expect(text).not.toContain("events · live");
@@ -625,7 +623,7 @@ describe("live scan dashboard", () => {
     });
     dashboard.stop();
 
-    expect(stripVTControlCharacters(stderr.text())).toContain(
+    expect(Bun.stripANSI(stderr.text())).toContain(
       '[09:41:00] ◐ worker 2 · rg -n "password" routes/login.ts',
     );
   });
@@ -1109,7 +1107,7 @@ describe("live scan dashboard", () => {
   test("renders compact clickable local Markdown links without repeating timestamps", () => {
     const stderr = capture(true);
     const target =
-      "/private/tmp/codex security/scans/promptfoo-cloud/artifacts/02_discovery/raw_candidates_02.jsonl";
+      "/synthetic/codex security/scans/example/artifacts/02_discovery/raw_candidates_02.jsonl";
     const dashboard = new ScanDashboard(
       { ...stderr.stream, columns: 55, rows: 18 },
       { repository: "/code/juice-shop", clock: fakeClock() },
@@ -1127,10 +1125,10 @@ describe("live scan dashboard", () => {
 
     const frame = lastFrame(stderr);
     expect(frame).toContain("raw_candidates_02.jsonl");
-    expect(frame).not.toContain("/private/tmp/");
+    expect(frame).not.toContain("/synthetic/");
     expect(frame.match(/\[09:41:00\]/gu)).toHaveLength(1);
     expect(stderr.text()).toContain(
-      `\u001B]8;;file:///private/tmp/codex%20security/scans/promptfoo-cloud/artifacts/02_discovery/raw_candidates_02.jsonl\u0007raw_candidates_02.jsonl\u001B]8;;\u0007`,
+      `\u001B]8;;${pathToFileURL(target).href}\u0007raw_candidates_02.jsonl\u001B]8;;\u0007`,
     );
     dashboard.stop();
   });
@@ -1198,9 +1196,11 @@ describe("live scan dashboard", () => {
     expect(frame).not.toContain("`db.raw(  input  )`");
     expect(stderr.text()).toContain("\u001B[2mdb.raw(  input  )\u001B[22m");
     expect(stderr.text()).toContain(
-      "\u001B]8;;file:///tmp/report.md\u0007report\u001B]8;;\u0007",
+      `\u001B]8;;${pathToFileURL("/tmp/report.md").href}\u0007report\u001B]8;;\u0007`,
+    );
+    expect(stderr.text()).not.toContain(
+      `\u001B]8;;${pathToFileURL("/tmp/example.md").href}`,
     );
-    expect(stderr.text()).not.toContain("\u001B]8;;file:///tmp/example.md");
 
     dashboard.record({
       id: "command-inline-code",
@@ -1420,8 +1420,6 @@ describe("live scan dashboard", () => {
     dashboard.stop();
 
     expect(stderr.text()).not.toContain("\u001B[31m");
-    expect(stripVTControlCharacters(stderr.text())).toContain(
-      "routes/spoofed.ts",
-    );
+    expect(Bun.stripANSI(stderr.text())).toContain("routes/spoofed.ts");
   });
 });