test(design-diff): replay frozen schema 3 benchmark in isolated cloud shards

Author: Bill Leoutsakos

.github/workflows/design-diff-benchmark.yml

@@ -0,0 +1,65 @@
+name: Design diff frozen benchmark
+
+on:
+  push:
+    branches: [codex/design-diff-benchmark-v3]
+
+permissions:
+  contents: read
+
+concurrency:
+  group: design-diff-benchmark-v3
+  cancel-in-progress: true
+
+env:
+  ENGINE_SHA: 777a4aa4b303d3e06aa652818b068ac3374bfa28
+
+jobs:
+  replay:
+    runs-on: ${{ (vars.CI_PROVIDER == '' || vars.CI_PROVIDER == 'blacksmith') && 'blacksmith-2vcpu-ubuntu-2404' || 'ubuntu-latest' }}
+    timeout-minutes: 60
+    strategy:
+      fail-fast: false
+      max-parallel: 8
+      matrix:
+        shard: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19]
+    steps:
+      - name: Checkout immutable engine
+        uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6
+        with:
+          ref: ${{ env.ENGINE_SHA }}
+          path: engine
+          fetch-depth: 0
+          persist-credentials: false
+      - name: Checkout benchmark driver
+        uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6
+        with:
+          ref: ${{ github.sha }}
+          path: driver
+          persist-credentials: false
+      - name: Setup Bun
+        uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
+        with:
+          bun-version: 1.4.1
+      - name: Install trusted dependencies
+        working-directory: engine
+        run: bun install --frozen-lockfile --ignore-scripts
+      - name: Freeze shard and fetch source as data
+        env:
+          SHARD: ${{ matrix.shard }}
+          GH_TOKEN: ${{ github.token }}
+        run: bun --no-env-file driver/scripts/design-diff/cloud-benchmark.ts
+      - name: Replay frozen comparisons
+        working-directory: engine
+        run: |
+          bun --no-env-file scripts/design-diff/benchmark.ts \
+            --engine "$PWD" --sha "$ENGINE_SHA" --workers 1 \
+            --manifest "$RUNNER_TEMP/comparisons.json" --output "$RUNNER_TEMP/results"
+      - name: Retain available results
+        if: always()
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
+        with:
+          name: design-benchmark-${{ env.ENGINE_SHA }}-${{ github.run_id }}-${{ github.run_attempt }}-${{ matrix.shard }}
+          path: ${{ runner.temp }}/results/
+          retention-days: 7
+          if-no-files-found: error

scripts/design-diff/cloud-benchmark.ts

@@ -0,0 +1,48 @@
+import { execFileSync } from 'node:child_process'
+import { readFileSync, writeFileSync } from 'node:fs'
+import path from 'node:path'
+
+/** Prepare a deterministic shard from the pinned engine's frozen manifest. */
+try {
+  const engine = path.resolve('engine')
+  const manifest = JSON.parse(
+    readFileSync(path.join(engine, 'scripts/design-diff/benchmark/comparisons.json'), 'utf8')
+  )
+  const shard = Number(process.env.SHARD)
+  if (!Number.isInteger(shard) || shard < 0 || shard >= 20) throw new Error('Invalid shard')
+  manifest.comparisons = manifest.comparisons.filter(
+    (entry: { sampleOrder: number }) => entry.sampleOrder % 20 === shard
+  )
+  const commits = [
+    ...new Set<string>(
+      manifest.comparisons.flatMap((entry: { base: string; head: string }) => [
+        entry.base,
+        entry.head,
+      ])
+    ),
+  ]
+  if (commits.some((commit) => !/^[a-f0-9]{40}$/.test(commit))) throw new Error('Invalid revision')
+  writeFileSync(
+    path.join(process.env.RUNNER_TEMP!, 'comparisons.json'),
+    `${JSON.stringify(manifest, null, 2)}\n`
+  )
+  const authorization = Buffer.from(`x-access-token:${process.env.GH_TOKEN}`).toString('base64')
+  execFileSync(
+    'git',
+    [
+      '-c',
+      `http.extraheader=AUTHORIZATION: basic ${authorization}`,
+      'fetch',
+      '--no-tags',
+      'origin',
+      ...commits,
+    ],
+    { cwd: engine, stdio: 'ignore' }
+  )
+  process.stdout.write(
+    `Prepared shard ${shard}: ${manifest.comparisons.length} frozen comparisons\n`
+  )
+} catch {
+  process.stderr.write('Benchmark shard preparation failed; no source code was executed.\n')
+  process.exitCode = 1
+}