test: fix some bugs in benchmark tests + add three more

bench/lib/targets.ts

@@ -1,8 +1,10 @@
 import { execFileSync } from 'node:child_process'
-import { cpSync, existsSync, mkdirSync, readdirSync, readFileSync, rmSync, statSync, writeFileSync } from 'node:fs'
+import { existsSync, mkdirSync, readdirSync, readFileSync, rmSync, statSync, writeFileSync } from 'node:fs'
 import { join } from 'node:path'
 import { fileURLToPath } from 'node:url'
 
+import { syncFixture } from '../../capture/lib/fixture.ts'
+
 export const repoRoot = fileURLToPath(new URL('../..', import.meta.url))
 
 export interface Target {
@@ -53,6 +55,10 @@ export function prepareTargets(workdir: string, baselineSpec: string = DEFAULT_B
       private: true,
       dependencies: { '@nuxt/cli': installSpecs[target.id] },
     }, null, 2)}\n`)
+    // Both targets pack to the same filename at the same version, so npm would
+    // otherwise keep the previous run's build.
+    rmSync(join(target.dir, 'node_modules'), { recursive: true, force: true })
+    rmSync(join(target.dir, 'package-lock.json'), { force: true })
     npm(['install', '--no-audit', '--no-fund'], target.dir)
     target.bin = join(target.dir, 'node_modules/@nuxt/cli/bin/nuxi.mjs')
     target.version = JSON.parse(readFileSync(join(target.dir, 'node_modules/@nuxt/cli/package.json'), 'utf8')).version
@@ -90,21 +96,19 @@ export function npm(args: string[], cwd: string): string {
   return execFileSync('npm', args, { cwd, encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'] })
 }
 
+/** What the fixtures do not own: installed dependencies and build caches. */
+const PRESERVED_FIXTURE_ENTRIES = new Set(['node_modules', 'package-lock.json', '.nuxt', '.data', '.output', '.nitro'])
+
 export function prepareFixtures(workdir: string): Fixture[] {
   const fixtures: Fixture[] = [
     { id: 'playground', label: 'repo `playground/` (2 pages, 1 layer, websocket nitro)', dir: join(workdir, 'fixture-playground') },
     { id: 'large', label: 'generated app (60 pages, 40 components, 10 server routes)', dir: join(workdir, 'fixture-large') },
   ]
 
-  if (!existsSync(fixtures[0]!.dir)) {
-    cpSync(join(repoRoot, 'playground'), fixtures[0]!.dir, {
-      recursive: true,
-      filter: src => !/node_modules|\.nuxt|\.data|\.output/.test(src),
-    })
-  }
-  if (!existsSync(fixtures[1]!.dir)) {
-    generateLargeFixture(fixtures[1]!.dir)
-  }
+  // Mirrored rather than copied once, so an edit reaches the next run and an
+  // untouched fixture stays warm.
+  syncFixture(join(repoRoot, 'playground'), fixtures[0]!.dir, PRESERVED_FIXTURE_ENTRIES)
+  syncFixture(stageLargeFixture(workdir), fixtures[1]!.dir, PRESERVED_FIXTURE_ENTRIES)
 
   for (const fixture of fixtures) {
     mkdirSync(join(fixture.dir, 'server/routes'), { recursive: true })
@@ -125,6 +129,14 @@ export function prepareFixtures(workdir: string): Fixture[] {
   return fixtures
 }
 
+/** Write the generated app somewhere the sync can compare it against. */
+function stageLargeFixture(workdir: string): string {
+  const staging = join(workdir, 'fixture-large-staging')
+  rmSync(staging, { recursive: true, force: true })
+  generateLargeFixture(staging)
+  return staging
+}
+
 function installedNuxtSpec(fixtureDir: string): string | undefined {
   try {
     const lock = JSON.parse(readFileSync(join(fixtureDir, 'package-lock.json'), 'utf8'))

bench/run.ts

@@ -13,10 +13,11 @@ import { buildSuite } from './suites/build.ts'
 import { devSuite } from './suites/dev.ts'
 import { footprintSuite } from './suites/footprint.ts'
 import { modulesSuite } from './suites/modules.ts'
+import { panelSuite } from './suites/panel.ts'
 import { restartSuite } from './suites/restart.ts'
 import { startupSuite } from './suites/startup.ts'
 
-const ALL_SUITES = ['startup', 'modules', 'dev', 'restart', 'build', 'footprint'] as const
+const ALL_SUITES = ['startup', 'modules', 'panel', 'dev', 'restart', 'build', 'footprint'] as const
 type SuiteName = typeof ALL_SUITES[number]
 
 const { values } = parseArgs({
@@ -27,6 +28,7 @@ const { values } = parseArgs({
     'workdir': { type: 'string', default: join(homedir(), '.cache', 'nuxt-cli-bench') },
     'out': { type: 'string', default: join(repoRoot, 'bench/results/report.md') },
     'startup-reps': { type: 'string', default: '15' },
+    'panel-reps': { type: 'string', default: '5' },
     'dev-reps': { type: 'string', default: '5' },
     'restart-reps': { type: 'string', default: '5' },
     'build-reps': { type: 'string', default: '3' },
@@ -46,10 +48,10 @@ mkdirSync(workdir, { recursive: true })
 
 console.log(`workdir: ${workdir}`)
 const targets = prepareTargets(workdir, values.baseline!)
-const needsFixtures = suites.has('dev') || suites.has('restart') || suites.has('build')
+const needsFixtures = suites.has('panel') || suites.has('dev') || suites.has('restart') || suites.has('build')
 const fixtures = needsFixtures ? prepareFixtures(workdir).filter(fixture => values.fixture!.includes(fixture.id)) : []
 if (needsFixtures && fixtures.length === 0) {
-  throw new Error(`no fixtures matched ${values.fixture!.join(', ')}; the dev, restart and build suites need at least one`)
+  throw new Error(`no fixtures matched ${values.fixture!.join(', ')}; the panel, dev, restart and build suites need at least one`)
 }
 for (const target of targets) {
   console.log(`target ${target.id}: ${target.spec} -> v${target.version}`)
@@ -77,6 +79,12 @@ if (suites.has('modules')) {
   sections.push(`## Module load cost\n\n${markdown}`)
 }
 
+if (suites.has('panel')) {
+  console.log('running panel suite')
+  const { markdown } = await panelSuite(targets, fixtures, Number(values['panel-reps']))
+  sections.push(`## \`nuxt dev\` panel startup\n\n${markdown}`)
+}
+
 if (suites.has('dev')) {
   console.log('running dev suite')
   const { markdown, results } = await devSuite(targets, fixtures, Number(values['dev-reps']))

bench/suites/dev.ts

@@ -14,11 +14,13 @@ export interface DevSample {
   fixture: string
   mode: 'cold' | 'warm'
   target: string
+  bound: number
   ready: number
   firstResponse: number
 }
 
 interface DevMeasurement {
+  bound: number
   ready: number
   firstResponse: number
 }
@@ -34,9 +36,11 @@ async function measureDevStart(target: Target, fixture: Fixture, cold: boolean,
   })
   const spawnedAt = performance.now()
   try {
-    const ready = await server.waitFor(new RegExp(`localhost:${port}`))
+    // The URL is printed when the socket binds, long before the app can answer.
+    const bound = await server.waitFor(new RegExp(`localhost:${port}`))
+    const ready = await server.waitFor(/ready in/i)
     const firstResponse = await waitForHttp(`http://localhost:${port}/`, spawnedAt)
-    return { ready, firstResponse }
+    return { bound, ready, firstResponse }
   }
   finally {
     await server.stop()
@@ -65,32 +69,27 @@ export async function devSuite(targets: Target[], fixtures: Fixture[], reps: num
         const entries = samples.get(target.id)!
         results.push(...entries)
         return {
+          bound: summarise(entries.map(e => e.bound)),
           ready: summarise(entries.map(e => e.ready)),
           firstResponse: summarise(entries.map(e => e.firstResponse)),
         }
       })
       const [baseline, head] = summaries
-      rows.push([
-        `${fixture.id} / ${mode} / ready`,
-        formatMs(baseline!.ready.median),
-        formatMs(head!.ready.median),
-        formatDelta(baseline!.ready.median, head!.ready.median),
-        `${formatMs(baseline!.ready.min)} / ${formatMs(baseline!.ready.max)}`,
-        `${formatMs(head!.ready.min)} / ${formatMs(head!.ready.max)}`,
-      ])
-      rows.push([
-        `${fixture.id} / ${mode} / first 200 response`,
-        formatMs(baseline!.firstResponse.median),
-        formatMs(head!.firstResponse.median),
-        formatDelta(baseline!.firstResponse.median, head!.firstResponse.median),
-        `${formatMs(baseline!.firstResponse.min)} / ${formatMs(baseline!.firstResponse.max)}`,
-        `${formatMs(head!.firstResponse.min)} / ${formatMs(head!.firstResponse.max)}`,
-      ])
+      for (const [label, key] of [['socket bound', 'bound'], ['ready', 'ready'], ['first 200 response', 'firstResponse']] as const) {
+        rows.push([
+          `${fixture.id} / ${mode} / ${label}`,
+          formatMs(baseline![key].median),
+          formatMs(head![key].median),
+          formatDelta(baseline![key].median, head![key].median),
+          `${formatMs(baseline![key].min)} / ${formatMs(baseline![key].max)}`,
+          `${formatMs(head![key].min)} / ${formatMs(head![key].max)}`,
+        ])
+      }
     }
   }
 
   const markdown = [
-    `Median of ${reps} interleaved runs. "ready" is the first URL printed by the CLI, "first 200 response" is measured from process spawn to a successful \`GET /\`. Cold runs delete \`.nuxt\`, \`.data\`, \`.output\` and \`node_modules/.cache\` first.`,
+    `Median of ${reps} interleaved runs, from process spawn. "Socket bound" is the first URL printed, which happens as soon as the server can accept a connection; "ready" is the line the CLI prints once the app is built; "first 200 response" is a successful \`GET /\`. Cold runs delete \`.nuxt\`, \`.data\`, \`.output\` and \`node_modules/.cache\` first.`,
     '',
     markdownTable(
       ['Fixture / mode / metric', `${shortLabel(targets[0]!)} median`, `${shortLabel(targets[1]!)} median`, 'Delta', `${shortLabel(targets[0]!)} min / max`, `${shortLabel(targets[1]!)} min / max`],

bench/suites/panel.ts

@@ -0,0 +1,132 @@
+import type { Fixture, Target } from '../lib/targets.ts'
+
+import { record } from '../../capture/lib/pty.ts'
+import { formatDelta, formatMs, markdownTable, summarise } from '../lib/stats.ts'
+import { shortLabel } from '../lib/targets.ts'
+import { allocatePort } from './dev.ts'
+
+/** Wide and tall enough that the panel is not refused for want of room. */
+const COLUMNS = 100
+const ROWS = 30
+
+/** How often a keypress is offered until one is answered. */
+const KEY_INTERVAL_MS = 20
+
+export interface PanelSample {
+  fixture: string
+  target: string
+  /** Time to the first thing a user can see, rather than the first byte. */
+  firstPaint: number
+  /** Time until a key pressed at the first frame is answered. */
+  interactive: number
+}
+
+interface PanelMeasurement {
+  firstPaint: number
+  interactive: number
+}
+
+/** Whether a chunk puts something on screen, rather than moving the cursor. */
+function isVisible(text: string): boolean {
+  const printable = text
+    // eslint-disable-next-line no-control-regex
+    .replace(/\u001B\[[0-9;?]*[a-z]/gi, '')
+    // eslint-disable-next-line no-control-regex
+    .replace(/\u001B\][^\u0007]*(?:\u0007|\u001B\\)?/g, '')
+  // eslint-disable-next-line no-control-regex
+  return /[^\s\u0000-\u001F\u007F]/.test(printable)
+}
+
+/**
+ * Time the two moments before a `nuxt dev` serves anything: when the panel
+ * appears, and when it first answers the keyboard.
+ *
+ * `?` is offered until the help view answers, since a key pressed before the
+ * panel takes stdin may be dropped.
+ */
+async function measurePanel(target: Target, fixture: Fixture): Promise<PanelMeasurement> {
+  const port = allocatePort()
+  const session = record(`exec ${process.execPath} ${target.bin} dev --port ${port}`, {
+    cwd: fixture.dir,
+    columns: COLUMNS,
+    rows: ROWS,
+    env: { NUXT_TELEMETRY_DISABLED: '1', NUXT_IGNORE_LOCK: '1' },
+  })
+
+  let keys: NodeJS.Timeout | undefined
+  try {
+    const painted = session.chunks.find(chunk => isVisible(chunk.data))
+      ?? await new Promise<{ at: number }>((resolve, reject) => {
+        const timer = setInterval(() => {
+          const chunk = session.chunks.find(entry => isVisible(entry.data))
+          if (chunk) {
+            clearInterval(timer)
+            resolve(chunk)
+          }
+        }, 5)
+        session.exited.then(() => {
+          clearInterval(timer)
+          reject(new Error(`the session ended before it painted:\n${session.output().slice(-800)}`))
+        }, reject)
+      })
+
+    session.send('?')
+    keys = setInterval(() => session.send('?'), KEY_INTERVAL_MS)
+    await session.waitFor(/keyboard shortcuts/)
+    const answered = session.chunks.find(chunk => chunk.data.includes('keyboard shortcuts'))!
+
+    return { firstPaint: painted.at, interactive: answered.at }
+  }
+  finally {
+    clearInterval(keys)
+    await session.stop()
+  }
+}
+
+export async function panelSuite(targets: Target[], fixtures: Fixture[], reps: number): Promise<{ results: PanelSample[], markdown: string }> {
+  const results: PanelSample[] = []
+  const rows: string[][] = []
+
+  for (const fixture of fixtures) {
+    const samples = new Map<string, PanelSample[]>(targets.map(target => [target.id, []]))
+    for (const target of targets) {
+      await measurePanel(target, fixture)
+    }
+    for (let rep = 0; rep < reps; rep++) {
+      for (const target of targets) {
+        const measurement = await measurePanel(target, fixture)
+        samples.get(target.id)!.push({ fixture: fixture.id, target: target.id, ...measurement })
+      }
+    }
+    const summaries = targets.map((target) => {
+      const entries = samples.get(target.id)!
+      results.push(...entries)
+      return {
+        firstPaint: summarise(entries.map(entry => entry.firstPaint)),
+        interactive: summarise(entries.map(entry => entry.interactive)),
+      }
+    })
+    const [baseline, head] = summaries
+    for (const [label, key] of [['first paint', 'firstPaint'], ['first answered keypress', 'interactive']] as const) {
+      rows.push([
+        `${fixture.id} / ${label}`,
+        formatMs(baseline![key].median),
+        formatMs(head![key].median),
+        formatDelta(baseline![key].median, head![key].median),
+        `${formatMs(baseline![key].min)} / ${formatMs(baseline![key].max)}`,
+        `${formatMs(head![key].min)} / ${formatMs(head![key].max)}`,
+      ])
+    }
+  }
+
+  const markdown = [
+    `Median of ${reps} interleaved runs in a ${COLUMNS}x${ROWS} pty, one warmup discarded. "First paint" is the first chunk carrying printable content, so cursor moves and the room the panel makes for itself do not count. "First answered keypress" offers \`?\` every ${KEY_INTERVAL_MS}ms from the first paint and waits for the help view.`,
+    '',
+    markdownTable(
+      ['Fixture / metric', `${shortLabel(targets[0]!)} median`, `${shortLabel(targets[1]!)} median`, 'Delta', `${shortLabel(targets[0]!)} min / max`, `${shortLabel(targets[1]!)} min / max`],
+      rows,
+    ),
+  ].join('\n')
+
+  return { results, markdown }
+}

capture/lib/fixture.ts

@@ -0,0 +1,49 @@
+import { mkdirSync, readdirSync, readFileSync, rmSync, statSync, writeFileSync } from 'node:fs'
+import { join } from 'node:path'
+
+/**
+ * Mirror a fixture into a work directory, returning the changed paths.
+ *
+ * Only files whose content actually changed are rewritten: a blanket copy
+ * touches every config file's mtime and invalidates Vite's dependency cache,
+ * putting a "Re-optimizing dependencies" line into the next run. Files deleted
+ * from the fixture are removed so stale routes or handlers cannot leak in.
+ *
+ * `preserve` names the top-level entries the fixture does not own, such as
+ * installed dependencies and build caches.
+ */
+export function syncFixture(from: string, to: string, preserve: Set<string>): string[] {
+  const changed: string[] = []
+  mkdirSync(to, { recursive: true })
+  const sourceEntries = readdirSync(from, { recursive: true, encoding: 'utf8' })
+  const sourceFiles = new Set(sourceEntries)
+  for (const entry of sourceEntries) {
+    const source = join(from, entry)
+    if (statSync(source).isDirectory()) {
+      continue
+    }
+    const target = join(to, entry)
+    const content = readFileSync(source)
+    let unchanged = false
+    try {
+      unchanged = readFileSync(target).equals(content)
+    }
+    catch {}
+    if (!unchanged) {
+      mkdirSync(join(target, '..'), { recursive: true })
+      writeFileSync(target, content)
+      changed.push(entry)
+    }
+  }
+  for (const entry of readdirSync(to, { recursive: true, encoding: 'utf8' })) {
+    if (preserve.has(entry.split('/')[0]!)) {
+      continue
+    }
+    const target = join(to, entry)
+    if (!sourceFiles.has(entry) && !statSync(target).isDirectory()) {
+      rmSync(target)
+      changed.push(entry)
+    }
+  }
+  return changed
+}