test(design-diff): replay collection-aware engine in smaller frozen batches

Author: Bill Leoutsakos

.github/workflows/design-diff-benchmark.yml

@@ -12,17 +12,17 @@ concurrency:
   cancel-in-progress: true
 
 env:
-  ENGINE_SHA: 309054cecfaca8c1504cd837b79b2dd70693ef59
+  ENGINE_SHA: e279fddb00dfc22ca181838352c436d7f480c286
 
 jobs:
   replay:
     runs-on: ${{ (vars.CI_PROVIDER == '' || vars.CI_PROVIDER == 'blacksmith') && 'blacksmith-2vcpu-ubuntu-2404' || 'ubuntu-latest' }}
-    timeout-minutes: 60
+    timeout-minutes: 50
     strategy:
       fail-fast: false
-      max-parallel: 8
+      max-parallel: 12
       matrix:
-        shard: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19]
+        shard: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59]
     steps:
       - name: Checkout immutable engine
         uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6

scripts/design-diff/cloud-benchmark.ts

@@ -9,9 +9,9 @@ try {
     readFileSync(path.join(engine, 'scripts/design-diff/benchmark/comparisons.json'), 'utf8')
   )
   const shard = Number(process.env.SHARD)
-  if (!Number.isInteger(shard) || shard < 0 || shard >= 20) throw new Error('Invalid shard')
+  if (!Number.isInteger(shard) || shard < 0 || shard >= 60) throw new Error('Invalid shard')
   manifest.comparisons = manifest.comparisons.filter(
-    (entry: { sampleOrder: number }) => entry.sampleOrder % 20 === shard
+    (entry: { sampleOrder: number }) => entry.sampleOrder % 60 === shard
   )
   const commits = [
     ...new Set<string>(