From 9f03128c625954073409e50f8d2f895a3f6947b7 Mon Sep 17 00:00:00 2001 From: Alexander Karan <47707063+AlexanderKaran@users.noreply.github.com> Date: Sun, 9 Aug 2026 13:24:39 +0800 Subject: [PATCH 1/7] Swap out INP test --- .../components/ClientSideRenderedCharts.astro | 2 - .../ClientSideRenderedINPChart.astro | 15 ----- .../ClientSideRenderedStatsTable.astro | 1 - ...FrameworkClientRenderedVersionCharts.astro | 7 -- ...FrameworkServerRenderedVersionCharts.astro | 7 -- .../components/ServerSideRenderedCharts.astro | 2 - .../ServerSideRenderedINPChart.astro | 15 ----- .../ServerSideRenderedStatsTable.astro | 1 - packages/docs/src/content.config.ts | 24 ++++++- packages/docs/src/content/docs/methodology.md | 20 ++++-- packages/docs/src/lib/collections.ts | 30 --------- .../src/clientSideRendered/run-benchmark.ts | 51 ++++++++++++--- .../src/clientSideRendered/types.ts | 14 ++-- .../stats-generator/src/interaction-timing.ts | 64 +++++++++++++++++++ .../src/run-client-side-rendered-benchmark.ts | 6 +- .../src/run-server-side-rendered-benchmark.ts | 6 +- packages/stats-generator/src/schemas.ts | 42 ++++++++++++ .../src/serverSideRendered/run-benchmark.ts | 51 ++++++++++++--- .../src/serverSideRendered/types.ts | 14 ++-- packages/stats-generator/src/types.ts | 20 +++++- .../stats-generator/src/validate-stats.ts | 20 +++++- 21 files changed, 293 insertions(+), 119 deletions(-) delete mode 100644 packages/docs/src/components/ClientSideRenderedINPChart.astro delete mode 100644 packages/docs/src/components/ServerSideRenderedINPChart.astro create mode 100644 packages/stats-generator/src/interaction-timing.ts diff --git a/packages/docs/src/components/ClientSideRenderedCharts.astro b/packages/docs/src/components/ClientSideRenderedCharts.astro index 14767259..d8336819 100644 --- a/packages/docs/src/components/ClientSideRenderedCharts.astro +++ b/packages/docs/src/components/ClientSideRenderedCharts.astro @@ -1,9 +1,7 @@ --- import ClientSideRenderedFPChart from './ClientSideRenderedFPChart.astro' import ClientSideRenderedFCPChart from './ClientSideRenderedFCPChart.astro' -import ClientSideRenderedINPChart from './ClientSideRenderedINPChart.astro' --- - diff --git a/packages/docs/src/components/ClientSideRenderedINPChart.astro b/packages/docs/src/components/ClientSideRenderedINPChart.astro deleted file mode 100644 index 166fb3d0..00000000 --- a/packages/docs/src/components/ClientSideRenderedINPChart.astro +++ /dev/null @@ -1,15 +0,0 @@ ---- -import { chartClientSideRenderedINPData } from '../lib/collections' -import ComparisonBarChart from './ComparisonBarChart.astro' - -const data = [...chartClientSideRenderedINPData].sort( - (a, b) => a.value - b.value || a.name.localeCompare(b.name), -) ---- - - diff --git a/packages/docs/src/components/ClientSideRenderedStatsTable.astro b/packages/docs/src/components/ClientSideRenderedStatsTable.astro index 76b2fa79..705538f1 100644 --- a/packages/docs/src/components/ClientSideRenderedStatsTable.astro +++ b/packages/docs/src/components/ClientSideRenderedStatsTable.astro @@ -15,7 +15,6 @@ const columns = [ }, { key: 'firstPaintMs', header: 'First Paint' }, { key: 'fcpMs', header: 'FCP' }, - { key: 'inpMs', header: 'INP' }, ] const tableData = clientSideRenderedStats diff --git a/packages/docs/src/components/FrameworkClientRenderedVersionCharts.astro b/packages/docs/src/components/FrameworkClientRenderedVersionCharts.astro index 31da749a..4d1991da 100644 --- a/packages/docs/src/components/FrameworkClientRenderedVersionCharts.astro +++ b/packages/docs/src/components/FrameworkClientRenderedVersionCharts.astro @@ -22,10 +22,3 @@ const { versions } = Astro.props valueFormat="ms" yAxisLabel="Milliseconds" /> - diff --git a/packages/docs/src/components/FrameworkServerRenderedVersionCharts.astro b/packages/docs/src/components/FrameworkServerRenderedVersionCharts.astro index ad21cf98..af53c3f6 100644 --- a/packages/docs/src/components/FrameworkServerRenderedVersionCharts.astro +++ b/packages/docs/src/components/FrameworkServerRenderedVersionCharts.astro @@ -22,10 +22,3 @@ const { versions } = Astro.props valueFormat="ms" yAxisLabel="Milliseconds" /> - diff --git a/packages/docs/src/components/ServerSideRenderedCharts.astro b/packages/docs/src/components/ServerSideRenderedCharts.astro index fd3c377d..d7d79f76 100644 --- a/packages/docs/src/components/ServerSideRenderedCharts.astro +++ b/packages/docs/src/components/ServerSideRenderedCharts.astro @@ -1,9 +1,7 @@ --- import ServerSideRenderedFPChart from './ServerSideRenderedFPChart.astro' import ServerSideRenderedFCPChart from './ServerSideRenderedFCPChart.astro' -import ServerSideRenderedINPChart from './ServerSideRenderedINPChart.astro' --- - diff --git a/packages/docs/src/components/ServerSideRenderedINPChart.astro b/packages/docs/src/components/ServerSideRenderedINPChart.astro deleted file mode 100644 index d95f24b5..00000000 --- a/packages/docs/src/components/ServerSideRenderedINPChart.astro +++ /dev/null @@ -1,15 +0,0 @@ ---- -import { chartServerSideRenderedINPData } from '../lib/collections' -import ComparisonBarChart from './ComparisonBarChart.astro' - -const data = [...chartServerSideRenderedINPData].sort( - (a, b) => a.value - b.value || a.name.localeCompare(b.name), -) ---- - - diff --git a/packages/docs/src/components/ServerSideRenderedStatsTable.astro b/packages/docs/src/components/ServerSideRenderedStatsTable.astro index 146d1255..ce4f889f 100644 --- a/packages/docs/src/components/ServerSideRenderedStatsTable.astro +++ b/packages/docs/src/components/ServerSideRenderedStatsTable.astro @@ -15,7 +15,6 @@ const columns = [ }, { key: 'firstPaintMs', header: 'First Paint' }, { key: 'fcpMs', header: 'FCP' }, - { key: 'inpMs', header: 'INP' }, ] const tableData = serverSideRenderedStats diff --git a/packages/docs/src/content.config.ts b/packages/docs/src/content.config.ts index 708c97f0..3fcf534c 100644 --- a/packages/docs/src/content.config.ts +++ b/packages/docs/src/content.config.ts @@ -111,7 +111,17 @@ const runtimeSchema = z.object({ .object({ firstPaintMs: z.number(), fcpMs: z.number(), - inpMs: z.number(), + inpMs: z.number().optional(), + interactionTests: z + .object({ + scenario: z.literal('first-row-detail-navigation'), + source: z.literal('chrome-event-timing'), + interactionLatencyMs: z.number(), + inputDelayMs: z.number(), + processingDurationMs: z.number(), + presentationDelayMs: z.number(), + }) + .optional(), runs: z.number(), }) .optional(), @@ -119,7 +129,17 @@ const runtimeSchema = z.object({ .object({ firstPaintMs: z.number(), fcpMs: z.number(), - inpMs: z.number(), + inpMs: z.number().optional(), + interactionTests: z + .object({ + scenario: z.literal('first-row-detail-navigation'), + source: z.literal('chrome-event-timing'), + interactionLatencyMs: z.number(), + inputDelayMs: z.number(), + processingDurationMs: z.number(), + presentationDelayMs: z.number(), + }) + .optional(), runs: z.number(), }) .optional(), diff --git a/packages/docs/src/content/docs/methodology.md b/packages/docs/src/content/docs/methodology.md index 2e47597f..7b4afc2b 100644 --- a/packages/docs/src/content/docs/methodology.md +++ b/packages/docs/src/content/docs/methodology.md @@ -163,9 +163,14 @@ throughput, and load behavior for comparable production apps. - Metrics are measured with Lighthouse flow in Chromium through Puppeteer. - First Paint and First Contentful Paint are measured on initial navigation to `/client-side-rendered`. -- Interaction to Next Paint is measured by clicking the first row's detail link - and waiting for the detail view. -- Benchmarks run 5 times by default and average successful metrics. +- A controlled interaction clicks the first row's detail link and waits for the + detail view. Chrome's + [Event Timing API](https://www.w3.org/TR/event-timing/) records its interaction + latency, input delay, processing duration, and presentation delay. This is + one controlled interaction, not Google's page-lifetime + [Interaction to Next Paint (INP)](https://web.dev/articles/inp) metric. +- Benchmarks run 5 times by default. Paint and interaction timing values are + averaged across those runs. - These are route-based client-side rendering tests, not full-app SPA mode tests. Each app uses its normal production build and configures the benchmark routes so the measured table and detail content are rendered in the browser. @@ -202,14 +207,17 @@ throughput, and load behavior for comparable production apps. - Metrics are measured with Lighthouse flow in Chromium through Puppeteer. - First Paint and First Contentful Paint are measured on initial navigation to `/server-side-rendered`. -- Interaction to Next Paint is measured by clicking the first row's detail link - and waiting for `/server-side-rendered/:id`. +- A controlled interaction clicks the first row's detail link and waits for + `/server-side-rendered/:id`. Chrome Event Timing records its total latency, + input delay, processing duration, and presentation delay. These interaction + measurements are retained in the stats data but are not currently charted. - Detail links use each framework's default production navigation component or idiomatic anchor behavior. Default meta-framework route prefetching or preloading is allowed when it is part of the framework's default link behavior, but the measured SSR routes are still rendered on demand rather than converted to prerendered static output. -- Benchmarks run 5 times by default and average successful metrics. +- Benchmarks run 5 times by default and require every Chrome paint and + interaction measurement to be present and greater than zero. - Astro keeps the default static output mode, but the measured `/server-side-rendered` route and its detail route use `export const prerender = false` so they are rendered on demand by the diff --git a/packages/docs/src/lib/collections.ts b/packages/docs/src/lib/collections.ts index 3dfac11e..0819c205 100644 --- a/packages/docs/src/lib/collections.ts +++ b/packages/docs/src/lib/collections.ts @@ -162,7 +162,6 @@ export const serverSideRenderedStats = runtimeEntries isFocused: framework.isFocused, firstPaintMs: `${framework.serverSideRenderedTests!.firstPaintMs}ms`, fcpMs: `${framework.serverSideRenderedTests!.fcpMs}ms`, - inpMs: `${framework.serverSideRenderedTests!.inpMs}ms`, })) export const clientSideRenderedStats = runtimeEntries @@ -179,7 +178,6 @@ export const clientSideRenderedStats = runtimeEntries isFocused: framework.isFocused, firstPaintMs: `${framework.clientSideRenderedTests!.firstPaintMs}ms`, fcpMs: `${framework.clientSideRenderedTests!.fcpMs}ms`, - inpMs: `${framework.clientSideRenderedTests!.inpMs}ms`, })) export const depsStats = starterStats.map((f) => ({ @@ -324,20 +322,6 @@ export const chartServerSideRenderedFCPData = runtimeEntries focused: f.isFocused, })) -export const chartServerSideRenderedINPData = runtimeEntries - .map((entry) => entry.data) - .sort((a, b) => a.order - b.order) - .filter( - (f) => - f.serverSideRenderedTests != null && - Number.isFinite(f.serverSideRenderedTests.inpMs), - ) - .map((f) => ({ - name: f.name, - value: f.serverSideRenderedTests!.inpMs, - focused: f.isFocused, - })) - export const chartClientSideRenderedFPData = runtimeEntries .map((entry) => entry.data) .sort((a, b) => a.order - b.order) @@ -366,20 +350,6 @@ export const chartClientSideRenderedFCPData = runtimeEntries focused: f.isFocused, })) -export const chartClientSideRenderedINPData = runtimeEntries - .map((entry) => entry.data) - .sort((a, b) => a.order - b.order) - .filter( - (f) => - f.clientSideRenderedTests != null && - Number.isFinite(f.clientSideRenderedTests.inpMs), - ) - .map((f) => ({ - name: f.name, - value: f.clientSideRenderedTests!.inpMs, - focused: f.isFocused, - })) - export const coreJsTableData = starterStats.map((f) => { const hasCorejs = (f.vendoredCoreJsUnnecessaryModules?.length ?? 0) > 0 return { diff --git a/packages/stats-generator/src/clientSideRendered/run-benchmark.ts b/packages/stats-generator/src/clientSideRendered/run-benchmark.ts index d5f3f68f..79655ecb 100644 --- a/packages/stats-generator/src/clientSideRendered/run-benchmark.ts +++ b/packages/stats-generator/src/clientSideRendered/run-benchmark.ts @@ -2,6 +2,12 @@ import { existsSync, readdirSync } from 'node:fs' import { execFileSync } from 'node:child_process' import puppeteer from 'puppeteer-core' import { startFlow } from 'lighthouse' +import { + getInteractionTimingFromTrace, + INTERACTION_SCENARIO, + INTERACTION_SOURCE, +} from '../interaction-timing.ts' +import type { InteractionTestStats } from '../interaction-timing.ts' import type { ClientSideRenderedBenchmarkResult, ClientSideRenderedRunResult, @@ -73,7 +79,6 @@ async function runOnce( await flow.navigate(`${url}${CLIENT_SIDE_RENDERED_PATH}`) await page.waitForSelector('table tbody tr', { timeout: 15_000 }) - // INP: click the first row's detail link await flow.startTimespan() await page.click('table tbody tr:first-child a') await page.waitForSelector('#detail-id', { timeout: 15_000 }) @@ -89,18 +94,20 @@ async function runOnce( const flowResult = await flow.createFlowResult() const navLhr = flowResult.steps[0].lhr - const timespanLhr = flowResult.steps[1].lhr + const timespanArtifacts = + flow.createArtifactsJson().gatherSteps[1].artifacts const metricsItems = ( navLhr.audits['metrics']?.details as { items?: Record[] } )?.items?.[0] const firstPaintMs = metricsItems?.observedFirstPaint ?? null const fcpMs = navLhr.audits['first-contentful-paint']?.numericValue ?? null - const inpMs = - timespanLhr.audits['interaction-to-next-paint']?.numericValue ?? null + const interaction = getInteractionTimingFromTrace( + timespanArtifacts.Trace.traceEvents, + ) await page.close() - return { firstPaintMs, fcpMs, inpMs } + return { firstPaintMs, fcpMs, interaction } } finally { await browser.close() } @@ -119,6 +126,9 @@ export async function runBenchmark( ): Promise { const chromiumPath = findChromium() const browserVersion = getBrowserVersion(chromiumPath) + if (!browserVersion) { + throw new Error(`Could not read Chrome version from ${chromiumPath}`) + } const results: ClientSideRenderedRunResult[] = [] for (let i = 0; i < runs; i++) { @@ -130,17 +140,40 @@ export async function runBenchmark( .map((r) => r.firstPaintMs) .filter((v): v is number => v !== null) const fcp = results.map((r) => r.fcpMs).filter((v): v is number => v !== null) - const inp = results.map((r) => r.inpMs).filter((v): v is number => v !== null) + const interactions = results + .map((r) => r.interaction) + .filter((value) => value !== null) + if (interactions.length !== results.length) { + throw new Error( + `Interaction timing could not be measured in ${results.length - interactions.length} of ${results.length} runs`, + ) + } + const firstPaintMs = avg(fp) + const fcpMs = avg(fcp) + const interactionTests: InteractionTestStats = { + scenario: INTERACTION_SCENARIO, + source: INTERACTION_SOURCE, + interactionLatencyMs: avg( + interactions.map((value) => value.interactionLatencyMs), + ), + inputDelayMs: avg(interactions.map((value) => value.inputDelayMs)), + processingDurationMs: avg( + interactions.map((value) => value.processingDurationMs), + ), + presentationDelayMs: avg( + interactions.map((value) => value.presentationDelayMs), + ), + } return { name: packageName, displayName, package: packageName, browserVersion, clientSideRenderedTests: { - firstPaintMs: avg(fp), - fcpMs: avg(fcp), - inpMs: avg(inp), + firstPaintMs, + fcpMs, + interactionTests, runs: results.length, }, } diff --git a/packages/stats-generator/src/clientSideRendered/types.ts b/packages/stats-generator/src/clientSideRendered/types.ts index be21c54c..b406ac4a 100644 --- a/packages/stats-generator/src/clientSideRendered/types.ts +++ b/packages/stats-generator/src/clientSideRendered/types.ts @@ -1,18 +1,23 @@ +import type { + InteractionTestStats, + InteractionTiming, +} from '../interaction-timing.ts' + export interface ClientSideRenderedRunResult { firstPaintMs: number | null fcpMs: number | null - inpMs: number | null + interaction: InteractionTiming | null } export interface ClientSideRenderedBenchmarkResult { name: string displayName: string package: string - browserVersion?: string + browserVersion: string clientSideRenderedTests: { firstPaintMs: number fcpMs: number - inpMs: number + interactionTests: InteractionTestStats runs: number } } @@ -25,7 +30,8 @@ export interface ClientSideRenderedStats { clientSideRenderedTests: { firstPaintMs: number fcpMs: number - inpMs: number + inpMs?: number + interactionTests?: InteractionTestStats runs: number } } diff --git a/packages/stats-generator/src/interaction-timing.ts b/packages/stats-generator/src/interaction-timing.ts new file mode 100644 index 00000000..d277cb9a --- /dev/null +++ b/packages/stats-generator/src/interaction-timing.ts @@ -0,0 +1,64 @@ +export const INTERACTION_SCENARIO = 'first-row-detail-navigation' +export const INTERACTION_SOURCE = 'chrome-event-timing' + +export interface InteractionTiming { + interactionLatencyMs: number + inputDelayMs: number + processingDurationMs: number + presentationDelayMs: number +} + +export interface InteractionTestStats extends InteractionTiming { + scenario: typeof INTERACTION_SCENARIO + source: typeof INTERACTION_SOURCE +} + +interface EventTimingData { + duration?: number + interactionId?: number + processingEnd?: number + processingStart?: number + timeStamp?: number +} + +interface TraceEvent { + name?: string + ph?: string + args?: { + data?: EventTimingData + } +} + +export function getInteractionTimingFromTrace( + traceEvents: TraceEvent[], +): InteractionTiming | null { + for (const event of traceEvents) { + if (event.name !== 'EventTiming' || event.ph === 'e') continue + + const data = event.args?.data + const interactionId = data?.interactionId + const duration = data?.duration + const processingEnd = data?.processingEnd + const processingStart = data?.processingStart + const timeStamp = data?.timeStamp + if ( + interactionId == null || + interactionId <= 0 || + duration == null || + processingEnd == null || + processingStart == null || + timeStamp == null + ) { + continue + } + + return { + interactionLatencyMs: duration, + inputDelayMs: processingStart - timeStamp, + processingDurationMs: processingEnd - processingStart, + presentationDelayMs: timeStamp + duration - processingEnd, + } + } + + return null +} diff --git a/packages/stats-generator/src/run-client-side-rendered-benchmark.ts b/packages/stats-generator/src/run-client-side-rendered-benchmark.ts index dd4d4f41..d3a196a0 100644 --- a/packages/stats-generator/src/run-client-side-rendered-benchmark.ts +++ b/packages/stats-generator/src/run-client-side-rendered-benchmark.ts @@ -55,7 +55,11 @@ async function main() { ` First Paint: ${result.clientSideRenderedTests.firstPaintMs}ms`, ) console.info(` FCP: ${result.clientSideRenderedTests.fcpMs}ms`) - console.info(` INP: ${result.clientSideRenderedTests.inpMs}ms`) + const interaction = result.clientSideRenderedTests.interactionTests + console.info(` Interaction: ${interaction.interactionLatencyMs}ms`) + console.info(` Input delay: ${interaction.inputDelayMs}ms`) + console.info(` Processing: ${interaction.processingDurationMs}ms`) + console.info(` Presentation: ${interaction.presentationDelayMs}ms`) } main().catch((error) => { diff --git a/packages/stats-generator/src/run-server-side-rendered-benchmark.ts b/packages/stats-generator/src/run-server-side-rendered-benchmark.ts index 2c49b1ce..ba5c6437 100644 --- a/packages/stats-generator/src/run-server-side-rendered-benchmark.ts +++ b/packages/stats-generator/src/run-server-side-rendered-benchmark.ts @@ -55,7 +55,11 @@ async function main() { ` First Paint: ${result.serverSideRenderedTests.firstPaintMs}ms`, ) console.info(` FCP: ${result.serverSideRenderedTests.fcpMs}ms`) - console.info(` INP: ${result.serverSideRenderedTests.inpMs}ms`) + const interaction = result.serverSideRenderedTests.interactionTests + console.info(` Interaction: ${interaction.interactionLatencyMs}ms`) + console.info(` Input delay: ${interaction.inputDelayMs}ms`) + console.info(` Processing: ${interaction.processingDurationMs}ms`) + console.info(` Presentation: ${interaction.presentationDelayMs}ms`) } main().catch((error) => { diff --git a/packages/stats-generator/src/schemas.ts b/packages/stats-generator/src/schemas.ts index bf8f5777..65719bd3 100644 --- a/packages/stats-generator/src/schemas.ts +++ b/packages/stats-generator/src/schemas.ts @@ -82,6 +82,42 @@ export const SSRLoadStatsSchema = z.object({ browserVersion: z.string().optional(), }) +const InteractionTestsSchema = z.object({ + scenario: z.literal('first-row-detail-navigation'), + source: z.literal('chrome-event-timing'), + interactionLatencyMs: z.number(), + inputDelayMs: z.number(), + processingDurationMs: z.number(), + presentationDelayMs: z.number(), +}) + +const RenderedTestsSchema = z + .object({ + firstPaintMs: z.number().positive(), + fcpMs: z.number().positive(), + /** Historical only; new runs record interactionTests instead. */ + inpMs: z.number().optional(), + interactionTests: InteractionTestsSchema.optional(), + runs: z.number().int().positive(), + }) + .refine( + (stats) => + stats.inpMs !== undefined || stats.interactionTests !== undefined, + { + message: 'Expected historical inpMs or current interactionTests', + }, + ) + +export const ClientSideRenderedStatsSchema = z.object({ + clientSideRenderedTests: RenderedTestsSchema, + browserVersion: z.string().min(1), +}) + +export const ServerSideRenderedStatsSchema = z.object({ + serverSideRenderedTests: RenderedTestsSchema, + browserVersion: z.string().min(1), +}) + export type InstallStats = z.infer export type BuildStats = z.infer export type BrowserBaselineStats = z.infer @@ -89,4 +125,10 @@ export type SSRRequestThroughputStats = z.infer< typeof SSRRequestThroughputStatsSchema > export type SSRLoadStats = z.infer +export type ClientSideRenderedStats = z.infer< + typeof ClientSideRenderedStatsSchema +> +export type ServerSideRenderedStats = z.infer< + typeof ServerSideRenderedStatsSchema +> export type TimeStat = z.infer diff --git a/packages/stats-generator/src/serverSideRendered/run-benchmark.ts b/packages/stats-generator/src/serverSideRendered/run-benchmark.ts index d51e2dd0..f6f334e8 100644 --- a/packages/stats-generator/src/serverSideRendered/run-benchmark.ts +++ b/packages/stats-generator/src/serverSideRendered/run-benchmark.ts @@ -2,6 +2,12 @@ import { existsSync, readdirSync } from 'node:fs' import { execFileSync } from 'node:child_process' import puppeteer from 'puppeteer-core' import { startFlow } from 'lighthouse' +import { + getInteractionTimingFromTrace, + INTERACTION_SCENARIO, + INTERACTION_SOURCE, +} from '../interaction-timing.ts' +import type { InteractionTestStats } from '../interaction-timing.ts' import type { ServerSideRenderedBenchmarkResult, ServerSideRenderedRunResult, @@ -73,7 +79,6 @@ async function runOnce( await flow.navigate(`${url}${SERVER_SIDE_RENDERED_PATH}`) await page.waitForSelector('table tbody tr', { timeout: 15_000 }) - // INP: click the first row's detail link await flow.startTimespan() await page.click('table tbody tr:first-child a') await page.waitForSelector('#detail-id', { timeout: 15_000 }) @@ -91,18 +96,20 @@ async function runOnce( const flowResult = await flow.createFlowResult() const navLhr = flowResult.steps[0].lhr - const timespanLhr = flowResult.steps[1].lhr + const timespanArtifacts = + flow.createArtifactsJson().gatherSteps[1].artifacts const metricsItems = ( navLhr.audits['metrics']?.details as { items?: Record[] } )?.items?.[0] const firstPaintMs = metricsItems?.observedFirstPaint ?? null const fcpMs = navLhr.audits['first-contentful-paint']?.numericValue ?? null - const inpMs = - timespanLhr.audits['interaction-to-next-paint']?.numericValue ?? null + const interaction = getInteractionTimingFromTrace( + timespanArtifacts.Trace.traceEvents, + ) await page.close() - return { firstPaintMs, fcpMs, inpMs } + return { firstPaintMs, fcpMs, interaction } } finally { await browser.close() } @@ -121,6 +128,9 @@ export async function runBenchmark( ): Promise { const chromiumPath = findChromium() const browserVersion = getBrowserVersion(chromiumPath) + if (!browserVersion) { + throw new Error(`Could not read Chrome version from ${chromiumPath}`) + } const results: ServerSideRenderedRunResult[] = [] for (let i = 0; i < runs; i++) { @@ -132,17 +142,40 @@ export async function runBenchmark( .map((r) => r.firstPaintMs) .filter((v): v is number => v !== null) const fcp = results.map((r) => r.fcpMs).filter((v): v is number => v !== null) - const inp = results.map((r) => r.inpMs).filter((v): v is number => v !== null) + const interactions = results + .map((r) => r.interaction) + .filter((value) => value !== null) + if (interactions.length !== results.length) { + throw new Error( + `Interaction timing could not be measured in ${results.length - interactions.length} of ${results.length} runs`, + ) + } + const firstPaintMs = avg(fp) + const fcpMs = avg(fcp) + const interactionTests: InteractionTestStats = { + scenario: INTERACTION_SCENARIO, + source: INTERACTION_SOURCE, + interactionLatencyMs: avg( + interactions.map((value) => value.interactionLatencyMs), + ), + inputDelayMs: avg(interactions.map((value) => value.inputDelayMs)), + processingDurationMs: avg( + interactions.map((value) => value.processingDurationMs), + ), + presentationDelayMs: avg( + interactions.map((value) => value.presentationDelayMs), + ), + } return { name: packageName, displayName, package: packageName, browserVersion, serverSideRenderedTests: { - firstPaintMs: avg(fp), - fcpMs: avg(fcp), - inpMs: avg(inp), + firstPaintMs, + fcpMs, + interactionTests, runs: results.length, }, } diff --git a/packages/stats-generator/src/serverSideRendered/types.ts b/packages/stats-generator/src/serverSideRendered/types.ts index 55bafb91..79b0046b 100644 --- a/packages/stats-generator/src/serverSideRendered/types.ts +++ b/packages/stats-generator/src/serverSideRendered/types.ts @@ -1,18 +1,23 @@ +import type { + InteractionTestStats, + InteractionTiming, +} from '../interaction-timing.ts' + export interface ServerSideRenderedRunResult { firstPaintMs: number | null fcpMs: number | null - inpMs: number | null + interaction: InteractionTiming | null } export interface ServerSideRenderedBenchmarkResult { name: string displayName: string package: string - browserVersion?: string + browserVersion: string serverSideRenderedTests: { firstPaintMs: number fcpMs: number - inpMs: number + interactionTests: InteractionTestStats runs: number } } @@ -25,7 +30,8 @@ export interface ServerSideRenderedStats { serverSideRenderedTests: { firstPaintMs: number fcpMs: number - inpMs: number + inpMs?: number + interactionTests?: InteractionTestStats runs: number } } diff --git a/packages/stats-generator/src/types.ts b/packages/stats-generator/src/types.ts index f4cadb04..90830b10 100644 --- a/packages/stats-generator/src/types.ts +++ b/packages/stats-generator/src/types.ts @@ -83,14 +83,30 @@ export interface CIStats { clientSideRenderedTests?: { firstPaintMs: number fcpMs: number - inpMs: number + inpMs?: number + interactionTests?: { + scenario: 'first-row-detail-navigation' + source: 'chrome-event-timing' + interactionLatencyMs: number + inputDelayMs: number + processingDurationMs: number + presentationDelayMs: number + } runs: number } // Server-side rendered route stats (browser paint + interaction timings) serverSideRenderedTests?: { firstPaintMs: number fcpMs: number - inpMs: number + inpMs?: number + interactionTests?: { + scenario: 'first-row-detail-navigation' + source: 'chrome-event-timing' + interactionLatencyMs: number + inputDelayMs: number + processingDurationMs: number + presentationDelayMs: number + } runs: number } // Core-js vendored polyfill stats diff --git a/packages/stats-generator/src/validate-stats.ts b/packages/stats-generator/src/validate-stats.ts index 09a1b5e9..fc0f2a08 100644 --- a/packages/stats-generator/src/validate-stats.ts +++ b/packages/stats-generator/src/validate-stats.ts @@ -9,10 +9,18 @@ import { BrowserBaselineStatsSchema, SSRRequestThroughputStatsSchema, SSRLoadStatsSchema, + ClientSideRenderedStatsSchema, + ServerSideRenderedStatsSchema, } from './schemas.ts' type BenchmarkType = - 'install' | 'build' | 'browserBaseline' | 'ssrRequestThroughput' | 'ssrLoad' + | 'install' + | 'build' + | 'browserBaseline' + | 'ssrRequestThroughput' + | 'ssrLoad' + | 'clientSideRendered' + | 'serverSideRendered' interface BenchmarkConfig { type: BenchmarkType @@ -31,6 +39,16 @@ const STARTER_BENCHMARKS: BenchmarkConfig[] = [ ] const APP_BENCHMARKS: BenchmarkConfig[] = [ + { + type: 'clientSideRendered', + file: 'ci-stats.json', + schema: ClientSideRenderedStatsSchema, + }, + { + type: 'serverSideRendered', + file: 'ci-stats.json', + schema: ServerSideRenderedStatsSchema, + }, { type: 'ssrRequestThroughput', file: 'ci-stats.json', From 0fdeaca109f39aac16a54dc8b18c183420944b45 Mon Sep 17 00:00:00 2001 From: Alexander Karan <47707063+AlexanderKaran@users.noreply.github.com> Date: Sun, 9 Aug 2026 15:54:17 +0800 Subject: [PATCH 2/7] Moved to Lighthouse new version --- packages/docs/src/content.config.ts | 4 +- packages/docs/src/content/docs/methodology.md | 17 ++-- .../src/clientSideRendered/run-benchmark.ts | 11 +-- .../src/clientSideRendered/types.ts | 5 +- .../stats-generator/src/interaction-timing.ts | 93 ++++++++----------- packages/stats-generator/src/schemas.ts | 2 +- .../src/serverSideRendered/run-benchmark.ts | 11 +-- .../src/serverSideRendered/types.ts | 5 +- packages/stats-generator/src/types.ts | 30 +++--- 9 files changed, 78 insertions(+), 100 deletions(-) diff --git a/packages/docs/src/content.config.ts b/packages/docs/src/content.config.ts index 3fcf534c..36f5c262 100644 --- a/packages/docs/src/content.config.ts +++ b/packages/docs/src/content.config.ts @@ -115,7 +115,7 @@ const runtimeSchema = z.object({ interactionTests: z .object({ scenario: z.literal('first-row-detail-navigation'), - source: z.literal('chrome-event-timing'), + source: z.literal('lighthouse-inp-breakdown'), interactionLatencyMs: z.number(), inputDelayMs: z.number(), processingDurationMs: z.number(), @@ -133,7 +133,7 @@ const runtimeSchema = z.object({ interactionTests: z .object({ scenario: z.literal('first-row-detail-navigation'), - source: z.literal('chrome-event-timing'), + source: z.literal('lighthouse-inp-breakdown'), interactionLatencyMs: z.number(), inputDelayMs: z.number(), processingDurationMs: z.number(), diff --git a/packages/docs/src/content/docs/methodology.md b/packages/docs/src/content/docs/methodology.md index 7b4afc2b..37819e4f 100644 --- a/packages/docs/src/content/docs/methodology.md +++ b/packages/docs/src/content/docs/methodology.md @@ -164,10 +164,12 @@ throughput, and load behavior for comparable production apps. - First Paint and First Contentful Paint are measured on initial navigation to `/client-side-rendered`. - A controlled interaction clicks the first row's detail link and waits for the - detail view. Chrome's - [Event Timing API](https://www.w3.org/TR/event-timing/) records its interaction - latency, input delay, processing duration, and presentation delay. This is - one controlled interaction, not Google's page-lifetime + detail view. Lighthouse's + [INP breakdown insight](https://developer.chrome.com/docs/performance/insights/inp-breakdown) + processes Chrome Event Timing trace data and reports its input delay, + processing duration, and presentation delay. Their sum is recorded as the + interaction latency. This is one controlled interaction, not Google's + page-lifetime [Interaction to Next Paint (INP)](https://web.dev/articles/inp) metric. - Benchmarks run 5 times by default. Paint and interaction timing values are averaged across those runs. @@ -208,9 +210,10 @@ throughput, and load behavior for comparable production apps. - First Paint and First Contentful Paint are measured on initial navigation to `/server-side-rendered`. - A controlled interaction clicks the first row's detail link and waits for - `/server-side-rendered/:id`. Chrome Event Timing records its total latency, - input delay, processing duration, and presentation delay. These interaction - measurements are retained in the stats data but are not currently charted. + `/server-side-rendered/:id`. Lighthouse's INP breakdown insight processes + Chrome Event Timing trace data and reports its input delay, processing + duration, and presentation delay. Their sum is recorded as the interaction + latency. - Detail links use each framework's default production navigation component or idiomatic anchor behavior. Default meta-framework route prefetching or preloading is allowed when it is part of the framework's default link diff --git a/packages/stats-generator/src/clientSideRendered/run-benchmark.ts b/packages/stats-generator/src/clientSideRendered/run-benchmark.ts index 79655ecb..6ca74adc 100644 --- a/packages/stats-generator/src/clientSideRendered/run-benchmark.ts +++ b/packages/stats-generator/src/clientSideRendered/run-benchmark.ts @@ -3,11 +3,11 @@ import { execFileSync } from 'node:child_process' import puppeteer from 'puppeteer-core' import { startFlow } from 'lighthouse' import { - getInteractionTimingFromTrace, + getInteractionTimingFromLighthouse, INTERACTION_SCENARIO, INTERACTION_SOURCE, } from '../interaction-timing.ts' -import type { InteractionTestStats } from '../interaction-timing.ts' +import type { InteractionTestStats } from '../types.ts' import type { ClientSideRenderedBenchmarkResult, ClientSideRenderedRunResult, @@ -94,16 +94,15 @@ async function runOnce( const flowResult = await flow.createFlowResult() const navLhr = flowResult.steps[0].lhr - const timespanArtifacts = - flow.createArtifactsJson().gatherSteps[1].artifacts + const timespanLhr = flowResult.steps[1].lhr const metricsItems = ( navLhr.audits['metrics']?.details as { items?: Record[] } )?.items?.[0] const firstPaintMs = metricsItems?.observedFirstPaint ?? null const fcpMs = navLhr.audits['first-contentful-paint']?.numericValue ?? null - const interaction = getInteractionTimingFromTrace( - timespanArtifacts.Trace.traceEvents, + const interaction = getInteractionTimingFromLighthouse( + timespanLhr.audits['inp-breakdown-insight'], ) await page.close() diff --git a/packages/stats-generator/src/clientSideRendered/types.ts b/packages/stats-generator/src/clientSideRendered/types.ts index b406ac4a..550c97ab 100644 --- a/packages/stats-generator/src/clientSideRendered/types.ts +++ b/packages/stats-generator/src/clientSideRendered/types.ts @@ -1,7 +1,4 @@ -import type { - InteractionTestStats, - InteractionTiming, -} from '../interaction-timing.ts' +import type { InteractionTestStats, InteractionTiming } from '../types.ts' export interface ClientSideRenderedRunResult { firstPaintMs: number | null diff --git a/packages/stats-generator/src/interaction-timing.ts b/packages/stats-generator/src/interaction-timing.ts index d277cb9a..bbe3a461 100644 --- a/packages/stats-generator/src/interaction-timing.ts +++ b/packages/stats-generator/src/interaction-timing.ts @@ -1,64 +1,49 @@ -export const INTERACTION_SCENARIO = 'first-row-detail-navigation' -export const INTERACTION_SOURCE = 'chrome-event-timing' +import type { InteractionTiming } from './types.ts' -export interface InteractionTiming { - interactionLatencyMs: number - inputDelayMs: number - processingDurationMs: number - presentationDelayMs: number -} +export const INTERACTION_SCENARIO = 'first-row-detail-navigation' +export const INTERACTION_SOURCE = 'lighthouse-inp-breakdown' -export interface InteractionTestStats extends InteractionTiming { - scenario: typeof INTERACTION_SCENARIO - source: typeof INTERACTION_SOURCE +interface LighthouseAudit { + details?: unknown } -interface EventTimingData { - duration?: number - interactionId?: number - processingEnd?: number - processingStart?: number - timeStamp?: number +interface INPBreakdownDetails { + items?: Array<{ + type?: string + items?: Array<{ subpart?: string; duration?: number }> + }> } -interface TraceEvent { - name?: string - ph?: string - args?: { - data?: EventTimingData - } -} - -export function getInteractionTimingFromTrace( - traceEvents: TraceEvent[], +export function getInteractionTimingFromLighthouse( + audit: LighthouseAudit | undefined, ): InteractionTiming | null { - for (const event of traceEvents) { - if (event.name !== 'EventTiming' || event.ph === 'e') continue - - const data = event.args?.data - const interactionId = data?.interactionId - const duration = data?.duration - const processingEnd = data?.processingEnd - const processingStart = data?.processingStart - const timeStamp = data?.timeStamp - if ( - interactionId == null || - interactionId <= 0 || - duration == null || - processingEnd == null || - processingStart == null || - timeStamp == null - ) { - continue - } - - return { - interactionLatencyMs: duration, - inputDelayMs: processingStart - timeStamp, - processingDurationMs: processingEnd - processingStart, - presentationDelayMs: timeStamp + duration - processingEnd, - } + const details = audit?.details as INPBreakdownDetails | undefined + const items = details?.items?.find( + (item) => item.type === 'table', + )?.items + const inputDelayMs = items?.find( + (item) => item.subpart === 'inputDelay', + )?.duration + const processingDurationMs = items?.find( + (item) => item.subpart === 'processingDuration', + )?.duration + const presentationDelayMs = items?.find( + (item) => item.subpart === 'presentationDelay', + )?.duration + + if ( + inputDelayMs == null || + processingDurationMs == null || + presentationDelayMs == null + ) { + return null } - return null + return { + interactionLatencyMs: + inputDelayMs + processingDurationMs + presentationDelayMs, + inputDelayMs, + processingDurationMs, + presentationDelayMs, + } } diff --git a/packages/stats-generator/src/schemas.ts b/packages/stats-generator/src/schemas.ts index 65719bd3..71b2100c 100644 --- a/packages/stats-generator/src/schemas.ts +++ b/packages/stats-generator/src/schemas.ts @@ -84,7 +84,7 @@ export const SSRLoadStatsSchema = z.object({ const InteractionTestsSchema = z.object({ scenario: z.literal('first-row-detail-navigation'), - source: z.literal('chrome-event-timing'), + source: z.literal('lighthouse-inp-breakdown'), interactionLatencyMs: z.number(), inputDelayMs: z.number(), processingDurationMs: z.number(), diff --git a/packages/stats-generator/src/serverSideRendered/run-benchmark.ts b/packages/stats-generator/src/serverSideRendered/run-benchmark.ts index f6f334e8..668344ee 100644 --- a/packages/stats-generator/src/serverSideRendered/run-benchmark.ts +++ b/packages/stats-generator/src/serverSideRendered/run-benchmark.ts @@ -3,11 +3,11 @@ import { execFileSync } from 'node:child_process' import puppeteer from 'puppeteer-core' import { startFlow } from 'lighthouse' import { - getInteractionTimingFromTrace, + getInteractionTimingFromLighthouse, INTERACTION_SCENARIO, INTERACTION_SOURCE, } from '../interaction-timing.ts' -import type { InteractionTestStats } from '../interaction-timing.ts' +import type { InteractionTestStats } from '../types.ts' import type { ServerSideRenderedBenchmarkResult, ServerSideRenderedRunResult, @@ -96,16 +96,15 @@ async function runOnce( const flowResult = await flow.createFlowResult() const navLhr = flowResult.steps[0].lhr - const timespanArtifacts = - flow.createArtifactsJson().gatherSteps[1].artifacts + const timespanLhr = flowResult.steps[1].lhr const metricsItems = ( navLhr.audits['metrics']?.details as { items?: Record[] } )?.items?.[0] const firstPaintMs = metricsItems?.observedFirstPaint ?? null const fcpMs = navLhr.audits['first-contentful-paint']?.numericValue ?? null - const interaction = getInteractionTimingFromTrace( - timespanArtifacts.Trace.traceEvents, + const interaction = getInteractionTimingFromLighthouse( + timespanLhr.audits['inp-breakdown-insight'], ) await page.close() diff --git a/packages/stats-generator/src/serverSideRendered/types.ts b/packages/stats-generator/src/serverSideRendered/types.ts index 79b0046b..2460be19 100644 --- a/packages/stats-generator/src/serverSideRendered/types.ts +++ b/packages/stats-generator/src/serverSideRendered/types.ts @@ -1,7 +1,4 @@ -import type { - InteractionTestStats, - InteractionTiming, -} from '../interaction-timing.ts' +import type { InteractionTestStats, InteractionTiming } from '../types.ts' export interface ServerSideRenderedRunResult { firstPaintMs: number | null diff --git a/packages/stats-generator/src/types.ts b/packages/stats-generator/src/types.ts index 90830b10..ab7e11a8 100644 --- a/packages/stats-generator/src/types.ts +++ b/packages/stats-generator/src/types.ts @@ -21,6 +21,18 @@ export interface TestConfig { measurements: MeasurementConfig[] } +export interface InteractionTiming { + interactionLatencyMs: number + inputDelayMs: number + processingDurationMs: number + presentationDelayMs: number +} + +export interface InteractionTestStats extends InteractionTiming { + scenario: 'first-row-detail-navigation' + source: 'lighthouse-inp-breakdown' +} + export interface FrameworkConfig { name: string displayName: string @@ -84,14 +96,7 @@ export interface CIStats { firstPaintMs: number fcpMs: number inpMs?: number - interactionTests?: { - scenario: 'first-row-detail-navigation' - source: 'chrome-event-timing' - interactionLatencyMs: number - inputDelayMs: number - processingDurationMs: number - presentationDelayMs: number - } + interactionTests?: InteractionTestStats runs: number } // Server-side rendered route stats (browser paint + interaction timings) @@ -99,14 +104,7 @@ export interface CIStats { firstPaintMs: number fcpMs: number inpMs?: number - interactionTests?: { - scenario: 'first-row-detail-navigation' - source: 'chrome-event-timing' - interactionLatencyMs: number - inputDelayMs: number - processingDurationMs: number - presentationDelayMs: number - } + interactionTests?: InteractionTestStats runs: number } // Core-js vendored polyfill stats From 7b496882086e799d1a656b398c69a5aa2a747b59 Mon Sep 17 00:00:00 2001 From: Alexander Karan <47707063+AlexanderKaran@users.noreply.github.com> Date: Sun, 9 Aug 2026 15:59:06 +0800 Subject: [PATCH 3/7] Lint --- packages/stats-generator/src/interaction-timing.ts | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/packages/stats-generator/src/interaction-timing.ts b/packages/stats-generator/src/interaction-timing.ts index bbe3a461..5991627e 100644 --- a/packages/stats-generator/src/interaction-timing.ts +++ b/packages/stats-generator/src/interaction-timing.ts @@ -18,9 +18,7 @@ export function getInteractionTimingFromLighthouse( audit: LighthouseAudit | undefined, ): InteractionTiming | null { const details = audit?.details as INPBreakdownDetails | undefined - const items = details?.items?.find( - (item) => item.type === 'table', - )?.items + const items = details?.items?.find((item) => item.type === 'table')?.items const inputDelayMs = items?.find( (item) => item.subpart === 'inputDelay', )?.duration From a2ddcddc084e8bbe7eaf44b8dcb30756142718e1 Mon Sep 17 00:00:00 2001 From: Alexander Karan <47707063+AlexanderKaran@users.noreply.github.com> Date: Sun, 9 Aug 2026 16:48:57 +0800 Subject: [PATCH 4/7] extra checks --- packages/docs/src/content.config.ts | 16 ++++++++-------- .../src/clientSideRendered/run-benchmark.ts | 12 ++++++++++++ .../stats-generator/src/interaction-timing.ts | 12 +++++++++--- packages/stats-generator/src/schemas.ts | 8 ++++---- .../src/serverSideRendered/run-benchmark.ts | 12 ++++++++++++ 5 files changed, 45 insertions(+), 15 deletions(-) diff --git a/packages/docs/src/content.config.ts b/packages/docs/src/content.config.ts index 36f5c262..e9d02062 100644 --- a/packages/docs/src/content.config.ts +++ b/packages/docs/src/content.config.ts @@ -116,10 +116,10 @@ const runtimeSchema = z.object({ .object({ scenario: z.literal('first-row-detail-navigation'), source: z.literal('lighthouse-inp-breakdown'), - interactionLatencyMs: z.number(), - inputDelayMs: z.number(), - processingDurationMs: z.number(), - presentationDelayMs: z.number(), + interactionLatencyMs: z.number().positive(), + inputDelayMs: z.number().positive(), + processingDurationMs: z.number().positive(), + presentationDelayMs: z.number().positive(), }) .optional(), runs: z.number(), @@ -134,10 +134,10 @@ const runtimeSchema = z.object({ .object({ scenario: z.literal('first-row-detail-navigation'), source: z.literal('lighthouse-inp-breakdown'), - interactionLatencyMs: z.number(), - inputDelayMs: z.number(), - processingDurationMs: z.number(), - presentationDelayMs: z.number(), + interactionLatencyMs: z.number().positive(), + inputDelayMs: z.number().positive(), + processingDurationMs: z.number().positive(), + presentationDelayMs: z.number().positive(), }) .optional(), runs: z.number(), diff --git a/packages/stats-generator/src/clientSideRendered/run-benchmark.ts b/packages/stats-generator/src/clientSideRendered/run-benchmark.ts index 6ca74adc..59658e53 100644 --- a/packages/stats-generator/src/clientSideRendered/run-benchmark.ts +++ b/packages/stats-generator/src/clientSideRendered/run-benchmark.ts @@ -139,6 +139,18 @@ export async function runBenchmark( .map((r) => r.firstPaintMs) .filter((v): v is number => v !== null) const fcp = results.map((r) => r.fcpMs).filter((v): v is number => v !== null) + if ( + fp.length !== results.length || + fp.some((value) => !Number.isFinite(value) || value <= 0) + ) { + throw new Error('First Paint was missing, non-finite, or zero') + } + if ( + fcp.length !== results.length || + fcp.some((value) => !Number.isFinite(value) || value <= 0) + ) { + throw new Error('FCP was missing, non-finite, or zero') + } const interactions = results .map((r) => r.interaction) .filter((value) => value !== null) diff --git a/packages/stats-generator/src/interaction-timing.ts b/packages/stats-generator/src/interaction-timing.ts index 5991627e..8916e973 100644 --- a/packages/stats-generator/src/interaction-timing.ts +++ b/packages/stats-generator/src/interaction-timing.ts @@ -30,9 +30,15 @@ export function getInteractionTimingFromLighthouse( )?.duration if ( - inputDelayMs == null || - processingDurationMs == null || - presentationDelayMs == null + typeof inputDelayMs !== 'number' || + !Number.isFinite(inputDelayMs) || + inputDelayMs <= 0 || + typeof processingDurationMs !== 'number' || + !Number.isFinite(processingDurationMs) || + processingDurationMs <= 0 || + typeof presentationDelayMs !== 'number' || + !Number.isFinite(presentationDelayMs) || + presentationDelayMs <= 0 ) { return null } diff --git a/packages/stats-generator/src/schemas.ts b/packages/stats-generator/src/schemas.ts index 71b2100c..ded2e328 100644 --- a/packages/stats-generator/src/schemas.ts +++ b/packages/stats-generator/src/schemas.ts @@ -85,10 +85,10 @@ export const SSRLoadStatsSchema = z.object({ const InteractionTestsSchema = z.object({ scenario: z.literal('first-row-detail-navigation'), source: z.literal('lighthouse-inp-breakdown'), - interactionLatencyMs: z.number(), - inputDelayMs: z.number(), - processingDurationMs: z.number(), - presentationDelayMs: z.number(), + interactionLatencyMs: z.number().positive(), + inputDelayMs: z.number().positive(), + processingDurationMs: z.number().positive(), + presentationDelayMs: z.number().positive(), }) const RenderedTestsSchema = z diff --git a/packages/stats-generator/src/serverSideRendered/run-benchmark.ts b/packages/stats-generator/src/serverSideRendered/run-benchmark.ts index 668344ee..01abea9f 100644 --- a/packages/stats-generator/src/serverSideRendered/run-benchmark.ts +++ b/packages/stats-generator/src/serverSideRendered/run-benchmark.ts @@ -141,6 +141,18 @@ export async function runBenchmark( .map((r) => r.firstPaintMs) .filter((v): v is number => v !== null) const fcp = results.map((r) => r.fcpMs).filter((v): v is number => v !== null) + if ( + fp.length !== results.length || + fp.some((value) => !Number.isFinite(value) || value <= 0) + ) { + throw new Error('First Paint was missing, non-finite, or zero') + } + if ( + fcp.length !== results.length || + fcp.some((value) => !Number.isFinite(value) || value <= 0) + ) { + throw new Error('FCP was missing, non-finite, or zero') + } const interactions = results .map((r) => r.interaction) .filter((value) => value !== null) From dfe2dd2946a5ef6efbed2d4a493b039ccbb1b6d5 Mon Sep 17 00:00:00 2001 From: Alexander Karan <47707063+AlexanderKaran@users.noreply.github.com> Date: Sun, 9 Aug 2026 17:19:22 +0800 Subject: [PATCH 5/7] Fix --- .github/workflows/validate-stats.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/validate-stats.yml b/.github/workflows/validate-stats.yml index 31097f55..15d0771f 100644 --- a/.github/workflows/validate-stats.yml +++ b/.github/workflows/validate-stats.yml @@ -109,7 +109,7 @@ jobs: echo "Building client-side rendered app for $PKG..." (cd packages/$PKG && pnpm build) echo "Running client-side rendered benchmark for $PKG..." - pnpm --filter @framework-tracker/stats-generator run:client-side-rendered $PKG + pnpm --filter @framework-tracker/stats-generator run:client-side-rendered $PKG 1 done echo "" @@ -120,7 +120,7 @@ jobs: echo "Building server-side rendered app for $PKG..." (cd packages/$PKG && pnpm build) echo "Running server-side rendered benchmark for $PKG..." - pnpm --filter @framework-tracker/stats-generator run:server-side-rendered $PKG + pnpm --filter @framework-tracker/stats-generator run:server-side-rendered $PKG 1 done echo "" From f0522503026511219b507db6ce2ed5349fe34e8b Mon Sep 17 00:00:00 2001 From: Alexander Karan <47707063+AlexanderKaran@users.noreply.github.com> Date: Sun, 9 Aug 2026 18:49:34 +0800 Subject: [PATCH 6/7] Sometimes things can be 0 lol --- packages/docs/src/content.config.ts | 12 ++++++------ .../stats-generator/src/interaction-timing.ts | 16 +++++++++++----- packages/stats-generator/src/schemas.ts | 6 +++--- 3 files changed, 20 insertions(+), 14 deletions(-) diff --git a/packages/docs/src/content.config.ts b/packages/docs/src/content.config.ts index e9d02062..9939970a 100644 --- a/packages/docs/src/content.config.ts +++ b/packages/docs/src/content.config.ts @@ -117,9 +117,9 @@ const runtimeSchema = z.object({ scenario: z.literal('first-row-detail-navigation'), source: z.literal('lighthouse-inp-breakdown'), interactionLatencyMs: z.number().positive(), - inputDelayMs: z.number().positive(), - processingDurationMs: z.number().positive(), - presentationDelayMs: z.number().positive(), + inputDelayMs: z.number().nonnegative(), + processingDurationMs: z.number().nonnegative(), + presentationDelayMs: z.number().nonnegative(), }) .optional(), runs: z.number(), @@ -135,9 +135,9 @@ const runtimeSchema = z.object({ scenario: z.literal('first-row-detail-navigation'), source: z.literal('lighthouse-inp-breakdown'), interactionLatencyMs: z.number().positive(), - inputDelayMs: z.number().positive(), - processingDurationMs: z.number().positive(), - presentationDelayMs: z.number().positive(), + inputDelayMs: z.number().nonnegative(), + processingDurationMs: z.number().nonnegative(), + presentationDelayMs: z.number().nonnegative(), }) .optional(), runs: z.number(), diff --git a/packages/stats-generator/src/interaction-timing.ts b/packages/stats-generator/src/interaction-timing.ts index 8916e973..4463ed90 100644 --- a/packages/stats-generator/src/interaction-timing.ts +++ b/packages/stats-generator/src/interaction-timing.ts @@ -32,20 +32,26 @@ export function getInteractionTimingFromLighthouse( if ( typeof inputDelayMs !== 'number' || !Number.isFinite(inputDelayMs) || - inputDelayMs <= 0 || + inputDelayMs < 0 || typeof processingDurationMs !== 'number' || !Number.isFinite(processingDurationMs) || - processingDurationMs <= 0 || + processingDurationMs < 0 || typeof presentationDelayMs !== 'number' || !Number.isFinite(presentationDelayMs) || - presentationDelayMs <= 0 + presentationDelayMs < 0 ) { return null } + const interactionLatencyMs = + inputDelayMs + processingDurationMs + presentationDelayMs + + if (interactionLatencyMs <= 0) { + return null + } + return { - interactionLatencyMs: - inputDelayMs + processingDurationMs + presentationDelayMs, + interactionLatencyMs, inputDelayMs, processingDurationMs, presentationDelayMs, diff --git a/packages/stats-generator/src/schemas.ts b/packages/stats-generator/src/schemas.ts index ded2e328..9961d72f 100644 --- a/packages/stats-generator/src/schemas.ts +++ b/packages/stats-generator/src/schemas.ts @@ -86,9 +86,9 @@ const InteractionTestsSchema = z.object({ scenario: z.literal('first-row-detail-navigation'), source: z.literal('lighthouse-inp-breakdown'), interactionLatencyMs: z.number().positive(), - inputDelayMs: z.number().positive(), - processingDurationMs: z.number().positive(), - presentationDelayMs: z.number().positive(), + inputDelayMs: z.number().nonnegative(), + processingDurationMs: z.number().nonnegative(), + presentationDelayMs: z.number().nonnegative(), }) const RenderedTestsSchema = z From 8725eba1718c0d8b5b8896641d0d3ee35f58fd2f Mon Sep 17 00:00:00 2001 From: Alexander Karan <47707063+AlexanderKaran@users.noreply.github.com> Date: Sun, 9 Aug 2026 19:46:35 +0800 Subject: [PATCH 7/7] Different measurement when hard navigation happens --- packages/app-solid-start/ci-stats.json | 21 ++++++--- .../src/clientSideRendered/index.ts | 9 ++++ .../src/clientSideRendered/run-benchmark.ts | 41 +++++++++++------- .../src/serverSideRendered/index.ts | 10 +++++ .../src/serverSideRendered/run-benchmark.ts | 43 ++++++++++++------- 5 files changed, 86 insertions(+), 38 deletions(-) diff --git a/packages/app-solid-start/ci-stats.json b/packages/app-solid-start/ci-stats.json index a8c5a6c4..fed9eee0 100644 --- a/packages/app-solid-start/ci-stats.json +++ b/packages/app-solid-start/ci-stats.json @@ -1,12 +1,19 @@ { - "timingMeasuredAt": "2026-08-08T23:59:59.029Z", - "runner": "Depot depot-ubuntu-24.04 GitHub Actions runner (2 CPUs, 8 GB RAM, 100 GB disk, 2 GB disk accelerator)", + "timingMeasuredAt": "2026-08-09T11:19:10.269Z", + "runner": "local", "frameworkVersion": "2.0.0", "clientSideRenderedTests": { - "firstPaintMs": 163.8, - "fcpMs": 163.92, - "inpMs": 0, - "runs": 5 + "firstPaintMs": 112, + "fcpMs": 111.68, + "interactionTests": { + "scenario": "first-row-detail-navigation", + "source": "lighthouse-inp-breakdown", + "interactionLatencyMs": 23.04, + "inputDelayMs": 0.5, + "processingDurationMs": 1.9, + "presentationDelayMs": 20.64 + }, + "runs": 1 }, "ssrRequestThroughputTests": { "opsPerSec": 429, @@ -151,5 +158,5 @@ }, "devDependencies": {} }, - "browserVersion": "Google Chrome 151.0.7922.75" + "browserVersion": "Google Chrome 151.0.7922.77" } diff --git a/packages/stats-generator/src/clientSideRendered/index.ts b/packages/stats-generator/src/clientSideRendered/index.ts index 2e8af2b5..16adbc07 100644 --- a/packages/stats-generator/src/clientSideRendered/index.ts +++ b/packages/stats-generator/src/clientSideRendered/index.ts @@ -15,6 +15,7 @@ interface ClientSideRenderedFrameworkConfig { package: string /** Filename of the serve script in src/serve/ */ serveScript: string + fullDocumentNavigation: boolean } const CLIENT_SIDE_RENDERED_FRAMEWORKS: ClientSideRenderedFrameworkConfig[] = [ @@ -23,42 +24,49 @@ const CLIENT_SIDE_RENDERED_FRAMEWORKS: ClientSideRenderedFrameworkConfig[] = [ displayName: 'Astro Client Side Rendered', package: 'app-astro', serveScript: 'astro.ts', + fullDocumentNavigation: true, }, { name: 'next-client-side-rendered', displayName: 'Next.js Client Side Rendered', package: 'app-next-js', serveScript: 'next.ts', + fullDocumentNavigation: false, }, { name: 'nuxt-client-side-rendered', displayName: 'Nuxt Client Side Rendered', package: 'app-nuxt', serveScript: 'nitro.ts', + fullDocumentNavigation: false, }, { name: 'react-router-client-side-rendered', displayName: 'React Router Client Side Rendered', package: 'app-react-router', serveScript: 'react-router.ts', + fullDocumentNavigation: false, }, { name: 'solid-start-client-side-rendered', displayName: 'SolidStart Client Side Rendered', package: 'app-solid-start', serveScript: 'nitro.ts', + fullDocumentNavigation: false, }, { name: 'sveltekit-client-side-rendered', displayName: 'SvelteKit Client Side Rendered', package: 'app-sveltekit', serveScript: 'sveltekit.ts', + fullDocumentNavigation: false, }, { name: 'tanstack-start-client-side-rendered', displayName: 'TanStack Start Client Side Rendered', package: 'app-tanstack-start-react', serveScript: 'tanstack-start.ts', + fullDocumentNavigation: false, }, ] @@ -155,6 +163,7 @@ export async function runClientSideRenderedBenchmark( config.name, config.displayName, runs, + config.fullDocumentNavigation, ) } finally { killServer() diff --git a/packages/stats-generator/src/clientSideRendered/run-benchmark.ts b/packages/stats-generator/src/clientSideRendered/run-benchmark.ts index 59658e53..b6abcd91 100644 --- a/packages/stats-generator/src/clientSideRendered/run-benchmark.ts +++ b/packages/stats-generator/src/clientSideRendered/run-benchmark.ts @@ -56,6 +56,7 @@ function getBrowserVersion(chromiumPath: string): string | undefined { async function runOnce( url: string, chromiumPath: string, + fullDocumentNavigation: boolean, ): Promise { const browser = await puppeteer.launch({ executablePath: chromiumPath, @@ -79,22 +80,31 @@ async function runOnce( await flow.navigate(`${url}${CLIENT_SIDE_RENDERED_PATH}`) await page.waitForSelector('table tbody tr', { timeout: 15_000 }) - await flow.startTimespan() - await page.click('table tbody tr:first-child a') - await page.waitForSelector('#detail-id', { timeout: 15_000 }) - // Double rAF ensures the paint entry is recorded before the timespan ends. - await page.evaluate( - (): Promise => - new Promise((r) => { - // @ts-expect-error — callback runs in browser context, not Node.js - requestAnimationFrame(() => requestAnimationFrame(r)) - }), - ) - await flow.endTimespan() + // Use navigation mode when the interaction triggers a full document load. + // https://github.com/GoogleChrome/lighthouse/blob/main/docs/user-flows.md#triggering-a-navigation-via-user-interactions + if (fullDocumentNavigation) { + await flow.startNavigation() + await page.click('table tbody tr:first-child a') + await flow.endNavigation() + await page.waitForSelector('#detail-id', { timeout: 15_000 }) + } else { + await flow.startTimespan() + await page.click('table tbody tr:first-child a') + await page.waitForSelector('#detail-id', { timeout: 15_000 }) + // Double rAF ensures the paint entry is recorded before the timespan ends. + await page.evaluate( + (): Promise => + new Promise((r) => { + // @ts-expect-error — callback runs in browser context, not Node.js + requestAnimationFrame(() => requestAnimationFrame(r)) + }), + ) + await flow.endTimespan() + } const flowResult = await flow.createFlowResult() const navLhr = flowResult.steps[0].lhr - const timespanLhr = flowResult.steps[1].lhr + const interactionLhr = flowResult.steps[1].lhr const metricsItems = ( navLhr.audits['metrics']?.details as { items?: Record[] } @@ -102,7 +112,7 @@ async function runOnce( const firstPaintMs = metricsItems?.observedFirstPaint ?? null const fcpMs = navLhr.audits['first-contentful-paint']?.numericValue ?? null const interaction = getInteractionTimingFromLighthouse( - timespanLhr.audits['inp-breakdown-insight'], + interactionLhr.audits['inp-breakdown-insight'], ) await page.close() @@ -122,6 +132,7 @@ export async function runBenchmark( packageName: string, displayName: string, runs: number, + fullDocumentNavigation: boolean, ): Promise { const chromiumPath = findChromium() const browserVersion = getBrowserVersion(chromiumPath) @@ -132,7 +143,7 @@ export async function runBenchmark( for (let i = 0; i < runs; i++) { console.log(` Run ${i + 1}/${runs}...`) - results.push(await runOnce(url, chromiumPath)) + results.push(await runOnce(url, chromiumPath, fullDocumentNavigation)) } const fp = results diff --git a/packages/stats-generator/src/serverSideRendered/index.ts b/packages/stats-generator/src/serverSideRendered/index.ts index e7f311d7..e494ed85 100644 --- a/packages/stats-generator/src/serverSideRendered/index.ts +++ b/packages/stats-generator/src/serverSideRendered/index.ts @@ -14,6 +14,7 @@ interface ServerSideRenderedFrameworkConfig { displayName: string package: string serveScript: string + fullDocumentNavigation: boolean } const SERVER_SIDE_RENDERED_FRAMEWORKS: ServerSideRenderedFrameworkConfig[] = [ @@ -22,48 +23,56 @@ const SERVER_SIDE_RENDERED_FRAMEWORKS: ServerSideRenderedFrameworkConfig[] = [ displayName: 'Astro Server Side Rendered', package: 'app-astro', serveScript: 'astro.ts', + fullDocumentNavigation: true, }, { name: 'mastro-server-side-rendered', displayName: 'Mastro Server Side Rendered', package: 'app-mastro', serveScript: 'mastro.ts', + fullDocumentNavigation: true, }, { name: 'next-server-side-rendered', displayName: 'Next.js Server Side Rendered', package: 'app-next-js', serveScript: 'next.ts', + fullDocumentNavigation: false, }, { name: 'nuxt-server-side-rendered', displayName: 'Nuxt Server Side Rendered', package: 'app-nuxt', serveScript: 'nitro.ts', + fullDocumentNavigation: false, }, { name: 'react-router-server-side-rendered', displayName: 'React Router Server Side Rendered', package: 'app-react-router', serveScript: 'react-router.ts', + fullDocumentNavigation: false, }, { name: 'solid-start-server-side-rendered', displayName: 'SolidStart Server Side Rendered', package: 'app-solid-start', serveScript: 'nitro.ts', + fullDocumentNavigation: false, }, { name: 'sveltekit-server-side-rendered', displayName: 'SvelteKit Server Side Rendered', package: 'app-sveltekit', serveScript: 'sveltekit.ts', + fullDocumentNavigation: false, }, { name: 'tanstack-start-server-side-rendered', displayName: 'TanStack Start Server Side Rendered', package: 'app-tanstack-start-react', serveScript: 'tanstack-start.ts', + fullDocumentNavigation: false, }, ] @@ -160,6 +169,7 @@ export async function runServerSideRenderedBenchmark( config.name, config.displayName, runs, + config.fullDocumentNavigation, ) } finally { killServer() diff --git a/packages/stats-generator/src/serverSideRendered/run-benchmark.ts b/packages/stats-generator/src/serverSideRendered/run-benchmark.ts index 01abea9f..2a00b47a 100644 --- a/packages/stats-generator/src/serverSideRendered/run-benchmark.ts +++ b/packages/stats-generator/src/serverSideRendered/run-benchmark.ts @@ -56,6 +56,7 @@ function getBrowserVersion(chromiumPath: string): string | undefined { async function runOnce( url: string, chromiumPath: string, + fullDocumentNavigation: boolean, ): Promise { const browser = await puppeteer.launch({ executablePath: chromiumPath, @@ -79,24 +80,33 @@ async function runOnce( await flow.navigate(`${url}${SERVER_SIDE_RENDERED_PATH}`) await page.waitForSelector('table tbody tr', { timeout: 15_000 }) - await flow.startTimespan() - await page.click('table tbody tr:first-child a') - await page.waitForSelector('#detail-id', { timeout: 15_000 }) - // Double rAF ensures the paint entry is recorded before the timespan ends. - await page.evaluate((): Promise => { - const { requestAnimationFrame: nextFrame } = globalThis as unknown as { - requestAnimationFrame: (callback: () => void) => number - } - - return new Promise((resolve) => { - nextFrame(() => nextFrame(resolve)) + // Use navigation mode when the interaction triggers a full document load. + // https://github.com/GoogleChrome/lighthouse/blob/main/docs/user-flows.md#triggering-a-navigation-via-user-interactions + if (fullDocumentNavigation) { + await flow.startNavigation() + await page.click('table tbody tr:first-child a') + await flow.endNavigation() + await page.waitForSelector('#detail-id', { timeout: 15_000 }) + } else { + await flow.startTimespan() + await page.click('table tbody tr:first-child a') + await page.waitForSelector('#detail-id', { timeout: 15_000 }) + // Double rAF ensures the paint entry is recorded before the timespan ends. + await page.evaluate((): Promise => { + const { requestAnimationFrame: nextFrame } = globalThis as unknown as { + requestAnimationFrame: (callback: () => void) => number + } + + return new Promise((resolve) => { + nextFrame(() => nextFrame(resolve)) + }) }) - }) - await flow.endTimespan() + await flow.endTimespan() + } const flowResult = await flow.createFlowResult() const navLhr = flowResult.steps[0].lhr - const timespanLhr = flowResult.steps[1].lhr + const interactionLhr = flowResult.steps[1].lhr const metricsItems = ( navLhr.audits['metrics']?.details as { items?: Record[] } @@ -104,7 +114,7 @@ async function runOnce( const firstPaintMs = metricsItems?.observedFirstPaint ?? null const fcpMs = navLhr.audits['first-contentful-paint']?.numericValue ?? null const interaction = getInteractionTimingFromLighthouse( - timespanLhr.audits['inp-breakdown-insight'], + interactionLhr.audits['inp-breakdown-insight'], ) await page.close() @@ -124,6 +134,7 @@ export async function runBenchmark( packageName: string, displayName: string, runs: number, + fullDocumentNavigation: boolean, ): Promise { const chromiumPath = findChromium() const browserVersion = getBrowserVersion(chromiumPath) @@ -134,7 +145,7 @@ export async function runBenchmark( for (let i = 0; i < runs; i++) { console.info(` Run ${i + 1}/${runs}...`) - results.push(await runOnce(url, chromiumPath)) + results.push(await runOnce(url, chromiumPath, fullDocumentNavigation)) } const fp = results