From 5f93f5be87669b90522fd0199683985d14398d75 Mon Sep 17 00:00:00 2001 From: Daniel Roe Date: Mon, 21 Sep 2026 19:14:46 +0000 Subject: [PATCH 1/4] bench: measure the two moments before a dev server serves anything --- bench/run.ts | 14 ++++- bench/suites/panel.ts | 133 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 144 insertions(+), 3 deletions(-) create mode 100644 bench/suites/panel.ts diff --git a/bench/run.ts b/bench/run.ts index 5ee81bb65..d59060cdc 100644 --- a/bench/run.ts +++ b/bench/run.ts @@ -13,10 +13,11 @@ import { buildSuite } from './suites/build.ts' import { devSuite } from './suites/dev.ts' import { footprintSuite } from './suites/footprint.ts' import { modulesSuite } from './suites/modules.ts' +import { panelSuite } from './suites/panel.ts' import { restartSuite } from './suites/restart.ts' import { startupSuite } from './suites/startup.ts' -const ALL_SUITES = ['startup', 'modules', 'dev', 'restart', 'build', 'footprint'] as const +const ALL_SUITES = ['startup', 'modules', 'panel', 'dev', 'restart', 'build', 'footprint'] as const type SuiteName = typeof ALL_SUITES[number] const { values } = parseArgs({ @@ -27,6 +28,7 @@ const { values } = parseArgs({ 'workdir': { type: 'string', default: join(homedir(), '.cache', 'nuxt-cli-bench') }, 'out': { type: 'string', default: join(repoRoot, 'bench/results/report.md') }, 'startup-reps': { type: 'string', default: '15' }, + 'panel-reps': { type: 'string', default: '5' }, 'dev-reps': { type: 'string', default: '5' }, 'restart-reps': { type: 'string', default: '5' }, 'build-reps': { type: 'string', default: '3' }, @@ -46,10 +48,10 @@ mkdirSync(workdir, { recursive: true }) console.log(`workdir: ${workdir}`) const targets = prepareTargets(workdir, values.baseline!) -const needsFixtures = suites.has('dev') || suites.has('restart') || suites.has('build') +const needsFixtures = suites.has('panel') || suites.has('dev') || suites.has('restart') || suites.has('build') const fixtures = needsFixtures ? prepareFixtures(workdir).filter(fixture => values.fixture!.includes(fixture.id)) : [] if (needsFixtures && fixtures.length === 0) { - throw new Error(`no fixtures matched ${values.fixture!.join(', ')}; the dev, restart and build suites need at least one`) + throw new Error(`no fixtures matched ${values.fixture!.join(', ')}; the panel, dev, restart and build suites need at least one`) } for (const target of targets) { console.log(`target ${target.id}: ${target.spec} -> v${target.version}`) @@ -77,6 +79,12 @@ if (suites.has('modules')) { sections.push(`## Module load cost\n\n${markdown}`) } +if (suites.has('panel')) { + console.log('running panel suite') + const { markdown } = await panelSuite(targets, fixtures, Number(values['panel-reps'])) + sections.push(`## \`nuxt dev\` panel startup\n\n${markdown}`) +} + if (suites.has('dev')) { console.log('running dev suite') const { markdown, results } = await devSuite(targets, fixtures, Number(values['dev-reps'])) diff --git a/bench/suites/panel.ts b/bench/suites/panel.ts new file mode 100644 index 000000000..756211980 --- /dev/null +++ b/bench/suites/panel.ts @@ -0,0 +1,133 @@ +import type { Fixture, Target } from '../lib/targets.ts' + +import { record } from '../../capture/lib/pty.ts' +import { formatDelta, formatMs, markdownTable, summarise } from '../lib/stats.ts' +import { shortLabel } from '../lib/targets.ts' +import { allocatePort } from './dev.ts' + +/** Wide and tall enough that the panel is not refused for want of room. */ +const COLUMNS = 100 +const ROWS = 30 + +/** How often a keypress is offered until one is answered. */ +const KEY_INTERVAL_MS = 20 + +export interface PanelSample { + fixture: string + target: string + /** Time to the first thing a user can see, rather than the first byte. */ + firstPaint: number + /** Time until a key pressed at the first frame is answered. */ + interactive: number +} + +interface PanelMeasurement { + firstPaint: number + interactive: number +} + +/** Whether a chunk puts something on screen, rather than moving the cursor. */ +function isVisible(text: string): boolean { + const printable = text + // eslint-disable-next-line no-control-regex + .replace(/\u001B\[[0-9;?]*[a-z]/gi, '') + // eslint-disable-next-line no-control-regex + .replace(/\u001B\][^\u0007]*(?:\u0007|\u001B\\)?/g, '') + // eslint-disable-next-line no-control-regex + return /[^\s\u0000-\u001F\u007F]/.test(printable) +} + +/** + * Time the two moments a `nuxt dev` session is judged by before it serves + * anything: when the panel appears, and when it first answers the keyboard. + * + * Both need a real terminal, so this runs in the same pty the captures use. + * `?` is offered until the help view answers, since the panel takes stdin + * partway through startup and a key pressed before that may be dropped. + */ +async function measurePanel(target: Target, fixture: Fixture): Promise { + const port = allocatePort() + const session = record(`exec ${process.execPath} ${target.bin} dev --port ${port}`, { + cwd: fixture.dir, + columns: COLUMNS, + rows: ROWS, + env: { NUXT_TELEMETRY_DISABLED: '1', NUXT_IGNORE_LOCK: '1' }, + }) + + let keys: NodeJS.Timeout | undefined + try { + const painted = session.chunks.find(chunk => isVisible(chunk.data)) + ?? await new Promise<{ at: number }>((resolve, reject) => { + const timer = setInterval(() => { + const chunk = session.chunks.find(entry => isVisible(entry.data)) + if (chunk) { + clearInterval(timer) + resolve(chunk) + } + }, 5) + session.exited.then(() => { + clearInterval(timer) + reject(new Error(`the session ended before it painted:\n${session.output().slice(-800)}`)) + }, reject) + }) + + session.send('?') + keys = setInterval(() => session.send('?'), KEY_INTERVAL_MS) + await session.waitFor(/keyboard shortcuts/) + const answered = session.chunks.find(chunk => chunk.data.includes('keyboard shortcuts'))! + + return { firstPaint: painted.at, interactive: answered.at } + } + finally { + clearInterval(keys) + await session.stop() + } +} + +export async function panelSuite(targets: Target[], fixtures: Fixture[], reps: number): Promise<{ results: PanelSample[], markdown: string }> { + const results: PanelSample[] = [] + const rows: string[][] = [] + + for (const fixture of fixtures) { + const samples = new Map(targets.map(target => [target.id, []])) + for (const target of targets) { + await measurePanel(target, fixture) + } + for (let rep = 0; rep < reps; rep++) { + for (const target of targets) { + const measurement = await measurePanel(target, fixture) + samples.get(target.id)!.push({ fixture: fixture.id, target: target.id, ...measurement }) + } + } + const summaries = targets.map((target) => { + const entries = samples.get(target.id)! + results.push(...entries) + return { + firstPaint: summarise(entries.map(entry => entry.firstPaint)), + interactive: summarise(entries.map(entry => entry.interactive)), + } + }) + const [baseline, head] = summaries + for (const [label, key] of [['first paint', 'firstPaint'], ['first answered keypress', 'interactive']] as const) { + rows.push([ + `${fixture.id} / ${label}`, + formatMs(baseline![key].median), + formatMs(head![key].median), + formatDelta(baseline![key].median, head![key].median), + `${formatMs(baseline![key].min)} / ${formatMs(baseline![key].max)}`, + `${formatMs(head![key].min)} / ${formatMs(head![key].max)}`, + ]) + } + } + + const markdown = [ + `Median of ${reps} interleaved runs in a ${COLUMNS}x${ROWS} pty, one warmup discarded. "First paint" is the first chunk carrying printable content, so cursor moves and the room the panel makes for itself do not count. "First answered keypress" offers \`?\` every ${KEY_INTERVAL_MS}ms from the first paint and waits for the help view.`, + '', + markdownTable( + ['Fixture / metric', `${shortLabel(targets[0]!)} median`, `${shortLabel(targets[1]!)} median`, 'Delta', `${shortLabel(targets[0]!)} min / max`, `${shortLabel(targets[1]!)} min / max`], + rows, + ), + ].join('\n') + + return { results, markdown } +} From 280ae81d85b3cc77ca2924f20a9ba5fd680dacd3 Mon Sep 17 00:00:00 2001 From: Daniel Roe Date: Mon, 21 Sep 2026 19:20:06 +0000 Subject: [PATCH 2/4] bench: install the build under test, not the one from last time --- bench/lib/targets.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/bench/lib/targets.ts b/bench/lib/targets.ts index 87d2a2ea7..15624adc0 100644 --- a/bench/lib/targets.ts +++ b/bench/lib/targets.ts @@ -53,6 +53,11 @@ export function prepareTargets(workdir: string, baselineSpec: string = DEFAULT_B private: true, dependencies: { '@nuxt/cli': installSpecs[target.id] }, }, null, 2)}\n`) + // Every run packs to the same filename and the version never changes, so + // npm treats the tree as already satisfied and keeps the build from the + // previous run: without this, a second run silently measures stale code. + rmSync(join(target.dir, 'node_modules'), { recursive: true, force: true }) + rmSync(join(target.dir, 'package-lock.json'), { force: true }) npm(['install', '--no-audit', '--no-fund'], target.dir) target.bin = join(target.dir, 'node_modules/@nuxt/cli/bin/nuxi.mjs') target.version = JSON.parse(readFileSync(join(target.dir, 'node_modules/@nuxt/cli/package.json'), 'utf8')).version From abf1d45842e5863ce93420f3a273a8135cb1d9c0 Mon Sep 17 00:00:00 2001 From: Daniel Roe Date: Mon, 21 Sep 2026 19:27:00 +0000 Subject: [PATCH 3/4] bench: rebuild a fixture when what it was built from changes --- bench/lib/targets.ts | 29 ++++++++++++++++--------- capture/lib/fixture.ts | 49 ++++++++++++++++++++++++++++++++++++++++++ capture/record.ts | 48 +++-------------------------------------- 3 files changed, 71 insertions(+), 55 deletions(-) create mode 100644 capture/lib/fixture.ts diff --git a/bench/lib/targets.ts b/bench/lib/targets.ts index 15624adc0..405f05563 100644 --- a/bench/lib/targets.ts +++ b/bench/lib/targets.ts @@ -1,8 +1,10 @@ import { execFileSync } from 'node:child_process' -import { cpSync, existsSync, mkdirSync, readdirSync, readFileSync, rmSync, statSync, writeFileSync } from 'node:fs' +import { existsSync, mkdirSync, readdirSync, readFileSync, rmSync, statSync, writeFileSync } from 'node:fs' import { join } from 'node:path' import { fileURLToPath } from 'node:url' +import { syncFixture } from '../../capture/lib/fixture.ts' + export const repoRoot = fileURLToPath(new URL('../..', import.meta.url)) export interface Target { @@ -95,21 +97,20 @@ export function npm(args: string[], cwd: string): string { return execFileSync('npm', args, { cwd, encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'] }) } +/** What the fixtures do not own: installed dependencies and build caches. */ +const PRESERVED_FIXTURE_ENTRIES = new Set(['node_modules', 'package-lock.json', '.nuxt', '.data', '.output', '.nitro']) + export function prepareFixtures(workdir: string): Fixture[] { const fixtures: Fixture[] = [ { id: 'playground', label: 'repo `playground/` (2 pages, 1 layer, websocket nitro)', dir: join(workdir, 'fixture-playground') }, { id: 'large', label: 'generated app (60 pages, 40 components, 10 server routes)', dir: join(workdir, 'fixture-large') }, ] - if (!existsSync(fixtures[0]!.dir)) { - cpSync(join(repoRoot, 'playground'), fixtures[0]!.dir, { - recursive: true, - filter: src => !/node_modules|\.nuxt|\.data|\.output/.test(src), - }) - } - if (!existsSync(fixtures[1]!.dir)) { - generateLargeFixture(fixtures[1]!.dir) - } + // Mirrored rather than copied once: editing `playground/` or the generator has + // to reach the next run, and rewriting only what changed keeps a warm fixture + // warm. Both fixtures are staged first so the same sync handles either. + syncFixture(join(repoRoot, 'playground'), fixtures[0]!.dir, PRESERVED_FIXTURE_ENTRIES) + syncFixture(stageLargeFixture(workdir), fixtures[1]!.dir, PRESERVED_FIXTURE_ENTRIES) for (const fixture of fixtures) { mkdirSync(join(fixture.dir, 'server/routes'), { recursive: true }) @@ -130,6 +131,14 @@ export function prepareFixtures(workdir: string): Fixture[] { return fixtures } +/** Write the generated app somewhere the sync can compare it against. */ +function stageLargeFixture(workdir: string): string { + const staging = join(workdir, 'fixture-large-staging') + rmSync(staging, { recursive: true, force: true }) + generateLargeFixture(staging) + return staging +} + function installedNuxtSpec(fixtureDir: string): string | undefined { try { const lock = JSON.parse(readFileSync(join(fixtureDir, 'package-lock.json'), 'utf8')) diff --git a/capture/lib/fixture.ts b/capture/lib/fixture.ts new file mode 100644 index 000000000..143824e1b --- /dev/null +++ b/capture/lib/fixture.ts @@ -0,0 +1,49 @@ +import { mkdirSync, readdirSync, readFileSync, rmSync, statSync, writeFileSync } from 'node:fs' +import { join } from 'node:path' + +/** + * Mirror a fixture into a work directory, returning the changed paths. + * + * Only files whose content actually changed are rewritten: a blanket copy + * touches every config file's mtime and invalidates Vite's dependency cache, + * putting a "Re-optimizing dependencies" line into the next run. Files deleted + * from the fixture are removed so stale routes or handlers cannot leak in. + * + * `preserve` names the top-level entries the fixture does not own, such as + * installed dependencies and build caches. + */ +export function syncFixture(from: string, to: string, preserve: Set): string[] { + const changed: string[] = [] + mkdirSync(to, { recursive: true }) + const sourceEntries = readdirSync(from, { recursive: true, encoding: 'utf8' }) + const sourceFiles = new Set(sourceEntries) + for (const entry of sourceEntries) { + const source = join(from, entry) + if (statSync(source).isDirectory()) { + continue + } + const target = join(to, entry) + const content = readFileSync(source) + let unchanged = false + try { + unchanged = readFileSync(target).equals(content) + } + catch {} + if (!unchanged) { + mkdirSync(join(target, '..'), { recursive: true }) + writeFileSync(target, content) + changed.push(entry) + } + } + for (const entry of readdirSync(to, { recursive: true, encoding: 'utf8' })) { + if (preserve.has(entry.split('/')[0]!)) { + continue + } + const target = join(to, entry) + if (!sourceFiles.has(entry) && !statSync(target).isDirectory()) { + rmSync(target) + changed.push(entry) + } + } + return changed +} diff --git a/capture/record.ts b/capture/record.ts index 63146b6ea..2845f9c1f 100644 --- a/capture/record.ts +++ b/capture/record.ts @@ -1,12 +1,13 @@ import type { Capture } from './captures.config.ts' import { execFileSync } from 'node:child_process' -import { existsSync, mkdirSync, readdirSync, readFileSync, rmSync, statSync, writeFileSync } from 'node:fs' +import { existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs' import { homedir } from 'node:os' import { join } from 'node:path' import process from 'node:process' import { fileURLToPath } from 'node:url' import { parseArgs } from 'node:util' import { captures, NEEDS_DEV_SERVER } from './captures.config.ts' +import { syncFixture } from './lib/fixture.ts' import { buildFingerprint, buildFrames } from './lib/frames.ts' import { record } from './lib/pty.ts' import { describeRules } from './lib/scrub.ts' @@ -40,55 +41,12 @@ mkdirSync(scratchDir, { recursive: true }) /** Caches that live in the work directory but are not fixture-managed. */ const PRESERVED_DIRS = new Set(['node_modules', '.nuxt', '.output', '.data']) -const changedFixtureFiles = syncFixture(join(repoRoot, 'capture/fixture'), appDir) +const changedFixtureFiles = syncFixture(join(repoRoot, 'capture/fixture'), appDir, PRESERVED_DIRS) if (!existsSync(join(appDir, 'node_modules/nuxt')) || changedFixtureFiles.includes('package-lock.json')) { console.log('installing capture fixture dependencies') execFileSync('npm', ['ci', '--no-audit', '--no-fund'], { cwd: appDir, stdio: 'inherit' }) } -/** - * Mirror the fixture into the work directory, returning the changed paths. - * Only files whose content actually changed are rewritten: a blanket copy - * touches every config file's mtime and invalidates Vite's dependency cache, - * putting a "Re-optimizing dependencies" line into the next recording. Files - * deleted from the fixture are removed so stale routes or handlers cannot - * leak into a capture. - */ -function syncFixture(from: string, to: string): string[] { - const changed: string[] = [] - const sourceEntries = readdirSync(from, { recursive: true, encoding: 'utf8' }) - const sourceFiles = new Set(sourceEntries) - for (const entry of sourceEntries) { - const source = join(from, entry) - if (statSync(source).isDirectory()) { - continue - } - const target = join(to, entry) - const content = readFileSync(source) - let unchanged = false - try { - unchanged = readFileSync(target).equals(content) - } - catch {} - if (!unchanged) { - mkdirSync(join(target, '..'), { recursive: true }) - writeFileSync(target, content) - changed.push(entry) - } - } - for (const entry of readdirSync(to, { recursive: true, encoding: 'utf8' })) { - if (PRESERVED_DIRS.has(entry.split('/')[0]!)) { - continue - } - const target = join(to, entry) - if (!sourceFiles.has(entry) && !statSync(target).isDirectory()) { - rmSync(target) - changed.push(entry) - } - } - return changed -} - const selected = values.only!.length ? captures.filter(capture => values.only!.includes(capture.id)) : captures const unknown = values.only!.filter(id => !captures.some(capture => capture.id === id)) if (unknown.length) { From c40bdf4d607404e6d7b66a2328e9f0760925a1f6 Mon Sep 17 00:00:00 2001 From: Daniel Roe Date: Mon, 21 Sep 2026 19:35:21 +0000 Subject: [PATCH 4/4] bench: separate binding a socket from being ready --- bench/lib/targets.ts | 10 ++++------ bench/suites/dev.ts | 37 ++++++++++++++++++------------------- bench/suites/panel.ts | 9 ++++----- 3 files changed, 26 insertions(+), 30 deletions(-) diff --git a/bench/lib/targets.ts b/bench/lib/targets.ts index 405f05563..b27edbc87 100644 --- a/bench/lib/targets.ts +++ b/bench/lib/targets.ts @@ -55,9 +55,8 @@ export function prepareTargets(workdir: string, baselineSpec: string = DEFAULT_B private: true, dependencies: { '@nuxt/cli': installSpecs[target.id] }, }, null, 2)}\n`) - // Every run packs to the same filename and the version never changes, so - // npm treats the tree as already satisfied and keeps the build from the - // previous run: without this, a second run silently measures stale code. + // Both targets pack to the same filename at the same version, so npm would + // otherwise keep the previous run's build. rmSync(join(target.dir, 'node_modules'), { recursive: true, force: true }) rmSync(join(target.dir, 'package-lock.json'), { force: true }) npm(['install', '--no-audit', '--no-fund'], target.dir) @@ -106,9 +105,8 @@ export function prepareFixtures(workdir: string): Fixture[] { { id: 'large', label: 'generated app (60 pages, 40 components, 10 server routes)', dir: join(workdir, 'fixture-large') }, ] - // Mirrored rather than copied once: editing `playground/` or the generator has - // to reach the next run, and rewriting only what changed keeps a warm fixture - // warm. Both fixtures are staged first so the same sync handles either. + // Mirrored rather than copied once, so an edit reaches the next run and an + // untouched fixture stays warm. syncFixture(join(repoRoot, 'playground'), fixtures[0]!.dir, PRESERVED_FIXTURE_ENTRIES) syncFixture(stageLargeFixture(workdir), fixtures[1]!.dir, PRESERVED_FIXTURE_ENTRIES) diff --git a/bench/suites/dev.ts b/bench/suites/dev.ts index 7051bf2a5..39bd2eab3 100644 --- a/bench/suites/dev.ts +++ b/bench/suites/dev.ts @@ -14,11 +14,13 @@ export interface DevSample { fixture: string mode: 'cold' | 'warm' target: string + bound: number ready: number firstResponse: number } interface DevMeasurement { + bound: number ready: number firstResponse: number } @@ -34,9 +36,11 @@ async function measureDevStart(target: Target, fixture: Fixture, cold: boolean, }) const spawnedAt = performance.now() try { - const ready = await server.waitFor(new RegExp(`localhost:${port}`)) + // The URL is printed when the socket binds, long before the app can answer. + const bound = await server.waitFor(new RegExp(`localhost:${port}`)) + const ready = await server.waitFor(/ready in/i) const firstResponse = await waitForHttp(`http://localhost:${port}/`, spawnedAt) - return { ready, firstResponse } + return { bound, ready, firstResponse } } finally { await server.stop() @@ -65,32 +69,27 @@ export async function devSuite(targets: Target[], fixtures: Fixture[], reps: num const entries = samples.get(target.id)! results.push(...entries) return { + bound: summarise(entries.map(e => e.bound)), ready: summarise(entries.map(e => e.ready)), firstResponse: summarise(entries.map(e => e.firstResponse)), } }) const [baseline, head] = summaries - rows.push([ - `${fixture.id} / ${mode} / ready`, - formatMs(baseline!.ready.median), - formatMs(head!.ready.median), - formatDelta(baseline!.ready.median, head!.ready.median), - `${formatMs(baseline!.ready.min)} / ${formatMs(baseline!.ready.max)}`, - `${formatMs(head!.ready.min)} / ${formatMs(head!.ready.max)}`, - ]) - rows.push([ - `${fixture.id} / ${mode} / first 200 response`, - formatMs(baseline!.firstResponse.median), - formatMs(head!.firstResponse.median), - formatDelta(baseline!.firstResponse.median, head!.firstResponse.median), - `${formatMs(baseline!.firstResponse.min)} / ${formatMs(baseline!.firstResponse.max)}`, - `${formatMs(head!.firstResponse.min)} / ${formatMs(head!.firstResponse.max)}`, - ]) + for (const [label, key] of [['socket bound', 'bound'], ['ready', 'ready'], ['first 200 response', 'firstResponse']] as const) { + rows.push([ + `${fixture.id} / ${mode} / ${label}`, + formatMs(baseline![key].median), + formatMs(head![key].median), + formatDelta(baseline![key].median, head![key].median), + `${formatMs(baseline![key].min)} / ${formatMs(baseline![key].max)}`, + `${formatMs(head![key].min)} / ${formatMs(head![key].max)}`, + ]) + } } } const markdown = [ - `Median of ${reps} interleaved runs. "ready" is the first URL printed by the CLI, "first 200 response" is measured from process spawn to a successful \`GET /\`. Cold runs delete \`.nuxt\`, \`.data\`, \`.output\` and \`node_modules/.cache\` first.`, + `Median of ${reps} interleaved runs, from process spawn. "Socket bound" is the first URL printed, which happens as soon as the server can accept a connection; "ready" is the line the CLI prints once the app is built; "first 200 response" is a successful \`GET /\`. Cold runs delete \`.nuxt\`, \`.data\`, \`.output\` and \`node_modules/.cache\` first.`, '', markdownTable( ['Fixture / mode / metric', `${shortLabel(targets[0]!)} median`, `${shortLabel(targets[1]!)} median`, 'Delta', `${shortLabel(targets[0]!)} min / max`, `${shortLabel(targets[1]!)} min / max`], diff --git a/bench/suites/panel.ts b/bench/suites/panel.ts index 756211980..6d757df8f 100644 --- a/bench/suites/panel.ts +++ b/bench/suites/panel.ts @@ -38,12 +38,11 @@ function isVisible(text: string): boolean { } /** - * Time the two moments a `nuxt dev` session is judged by before it serves - * anything: when the panel appears, and when it first answers the keyboard. + * Time the two moments before a `nuxt dev` serves anything: when the panel + * appears, and when it first answers the keyboard. * - * Both need a real terminal, so this runs in the same pty the captures use. - * `?` is offered until the help view answers, since the panel takes stdin - * partway through startup and a key pressed before that may be dropped. + * `?` is offered until the help view answers, since a key pressed before the + * panel takes stdin may be dropped. */ async function measurePanel(target: Target, fixture: Fixture): Promise { const port = allocatePort()