diff --git a/.github/workflows/validate-stats.yml b/.github/workflows/validate-stats.yml index 31097f55..15d0771f 100644 --- a/.github/workflows/validate-stats.yml +++ b/.github/workflows/validate-stats.yml @@ -109,7 +109,7 @@ jobs: echo "Building client-side rendered app for $PKG..." (cd packages/$PKG && pnpm build) echo "Running client-side rendered benchmark for $PKG..." - pnpm --filter @framework-tracker/stats-generator run:client-side-rendered $PKG + pnpm --filter @framework-tracker/stats-generator run:client-side-rendered $PKG 1 done echo "" @@ -120,7 +120,7 @@ jobs: echo "Building server-side rendered app for $PKG..." (cd packages/$PKG && pnpm build) echo "Running server-side rendered benchmark for $PKG..." - pnpm --filter @framework-tracker/stats-generator run:server-side-rendered $PKG + pnpm --filter @framework-tracker/stats-generator run:server-side-rendered $PKG 1 done echo "" diff --git a/packages/app-solid-start/ci-stats.json b/packages/app-solid-start/ci-stats.json index a8c5a6c4..fed9eee0 100644 --- a/packages/app-solid-start/ci-stats.json +++ b/packages/app-solid-start/ci-stats.json @@ -1,12 +1,19 @@ { - "timingMeasuredAt": "2026-08-08T23:59:59.029Z", - "runner": "Depot depot-ubuntu-24.04 GitHub Actions runner (2 CPUs, 8 GB RAM, 100 GB disk, 2 GB disk accelerator)", + "timingMeasuredAt": "2026-08-09T11:19:10.269Z", + "runner": "local", "frameworkVersion": "2.0.0", "clientSideRenderedTests": { - "firstPaintMs": 163.8, - "fcpMs": 163.92, - "inpMs": 0, - "runs": 5 + "firstPaintMs": 112, + "fcpMs": 111.68, + "interactionTests": { + "scenario": "first-row-detail-navigation", + "source": "lighthouse-inp-breakdown", + "interactionLatencyMs": 23.04, + "inputDelayMs": 0.5, + "processingDurationMs": 1.9, + "presentationDelayMs": 20.64 + }, + "runs": 1 }, "ssrRequestThroughputTests": { "opsPerSec": 429, @@ -151,5 +158,5 @@ }, "devDependencies": {} }, - "browserVersion": "Google Chrome 151.0.7922.75" + "browserVersion": "Google Chrome 151.0.7922.77" } diff --git a/packages/docs/src/components/ClientSideRenderedCharts.astro b/packages/docs/src/components/ClientSideRenderedCharts.astro index 14767259..d8336819 100644 --- a/packages/docs/src/components/ClientSideRenderedCharts.astro +++ b/packages/docs/src/components/ClientSideRenderedCharts.astro @@ -1,9 +1,7 @@ --- import ClientSideRenderedFPChart from './ClientSideRenderedFPChart.astro' import ClientSideRenderedFCPChart from './ClientSideRenderedFCPChart.astro' -import ClientSideRenderedINPChart from './ClientSideRenderedINPChart.astro' --- - diff --git a/packages/docs/src/components/ClientSideRenderedINPChart.astro b/packages/docs/src/components/ClientSideRenderedINPChart.astro deleted file mode 100644 index 166fb3d0..00000000 --- a/packages/docs/src/components/ClientSideRenderedINPChart.astro +++ /dev/null @@ -1,15 +0,0 @@ ---- -import { chartClientSideRenderedINPData } from '../lib/collections' -import ComparisonBarChart from './ComparisonBarChart.astro' - -const data = [...chartClientSideRenderedINPData].sort( - (a, b) => a.value - b.value || a.name.localeCompare(b.name), -) ---- - - diff --git a/packages/docs/src/components/ClientSideRenderedStatsTable.astro b/packages/docs/src/components/ClientSideRenderedStatsTable.astro index 76b2fa79..705538f1 100644 --- a/packages/docs/src/components/ClientSideRenderedStatsTable.astro +++ b/packages/docs/src/components/ClientSideRenderedStatsTable.astro @@ -15,7 +15,6 @@ const columns = [ }, { key: 'firstPaintMs', header: 'First Paint' }, { key: 'fcpMs', header: 'FCP' }, - { key: 'inpMs', header: 'INP' }, ] const tableData = clientSideRenderedStats diff --git a/packages/docs/src/components/FrameworkClientRenderedVersionCharts.astro b/packages/docs/src/components/FrameworkClientRenderedVersionCharts.astro index 31da749a..4d1991da 100644 --- a/packages/docs/src/components/FrameworkClientRenderedVersionCharts.astro +++ b/packages/docs/src/components/FrameworkClientRenderedVersionCharts.astro @@ -22,10 +22,3 @@ const { versions } = Astro.props valueFormat="ms" yAxisLabel="Milliseconds" /> - diff --git a/packages/docs/src/components/FrameworkServerRenderedVersionCharts.astro b/packages/docs/src/components/FrameworkServerRenderedVersionCharts.astro index ad21cf98..af53c3f6 100644 --- a/packages/docs/src/components/FrameworkServerRenderedVersionCharts.astro +++ b/packages/docs/src/components/FrameworkServerRenderedVersionCharts.astro @@ -22,10 +22,3 @@ const { versions } = Astro.props valueFormat="ms" yAxisLabel="Milliseconds" /> - diff --git a/packages/docs/src/components/ServerSideRenderedCharts.astro b/packages/docs/src/components/ServerSideRenderedCharts.astro index fd3c377d..d7d79f76 100644 --- a/packages/docs/src/components/ServerSideRenderedCharts.astro +++ b/packages/docs/src/components/ServerSideRenderedCharts.astro @@ -1,9 +1,7 @@ --- import ServerSideRenderedFPChart from './ServerSideRenderedFPChart.astro' import ServerSideRenderedFCPChart from './ServerSideRenderedFCPChart.astro' -import ServerSideRenderedINPChart from './ServerSideRenderedINPChart.astro' --- - diff --git a/packages/docs/src/components/ServerSideRenderedINPChart.astro b/packages/docs/src/components/ServerSideRenderedINPChart.astro deleted file mode 100644 index d95f24b5..00000000 --- a/packages/docs/src/components/ServerSideRenderedINPChart.astro +++ /dev/null @@ -1,15 +0,0 @@ ---- -import { chartServerSideRenderedINPData } from '../lib/collections' -import ComparisonBarChart from './ComparisonBarChart.astro' - -const data = [...chartServerSideRenderedINPData].sort( - (a, b) => a.value - b.value || a.name.localeCompare(b.name), -) ---- - - diff --git a/packages/docs/src/components/ServerSideRenderedStatsTable.astro b/packages/docs/src/components/ServerSideRenderedStatsTable.astro index 146d1255..ce4f889f 100644 --- a/packages/docs/src/components/ServerSideRenderedStatsTable.astro +++ b/packages/docs/src/components/ServerSideRenderedStatsTable.astro @@ -15,7 +15,6 @@ const columns = [ }, { key: 'firstPaintMs', header: 'First Paint' }, { key: 'fcpMs', header: 'FCP' }, - { key: 'inpMs', header: 'INP' }, ] const tableData = serverSideRenderedStats diff --git a/packages/docs/src/content.config.ts b/packages/docs/src/content.config.ts index 708c97f0..9939970a 100644 --- a/packages/docs/src/content.config.ts +++ b/packages/docs/src/content.config.ts @@ -111,7 +111,17 @@ const runtimeSchema = z.object({ .object({ firstPaintMs: z.number(), fcpMs: z.number(), - inpMs: z.number(), + inpMs: z.number().optional(), + interactionTests: z + .object({ + scenario: z.literal('first-row-detail-navigation'), + source: z.literal('lighthouse-inp-breakdown'), + interactionLatencyMs: z.number().positive(), + inputDelayMs: z.number().nonnegative(), + processingDurationMs: z.number().nonnegative(), + presentationDelayMs: z.number().nonnegative(), + }) + .optional(), runs: z.number(), }) .optional(), @@ -119,7 +129,17 @@ const runtimeSchema = z.object({ .object({ firstPaintMs: z.number(), fcpMs: z.number(), - inpMs: z.number(), + inpMs: z.number().optional(), + interactionTests: z + .object({ + scenario: z.literal('first-row-detail-navigation'), + source: z.literal('lighthouse-inp-breakdown'), + interactionLatencyMs: z.number().positive(), + inputDelayMs: z.number().nonnegative(), + processingDurationMs: z.number().nonnegative(), + presentationDelayMs: z.number().nonnegative(), + }) + .optional(), runs: z.number(), }) .optional(), diff --git a/packages/docs/src/content/docs/methodology.md b/packages/docs/src/content/docs/methodology.md index 2e47597f..37819e4f 100644 --- a/packages/docs/src/content/docs/methodology.md +++ b/packages/docs/src/content/docs/methodology.md @@ -163,9 +163,16 @@ throughput, and load behavior for comparable production apps. - Metrics are measured with Lighthouse flow in Chromium through Puppeteer. - First Paint and First Contentful Paint are measured on initial navigation to `/client-side-rendered`. -- Interaction to Next Paint is measured by clicking the first row's detail link - and waiting for the detail view. -- Benchmarks run 5 times by default and average successful metrics. +- A controlled interaction clicks the first row's detail link and waits for the + detail view. Lighthouse's + [INP breakdown insight](https://developer.chrome.com/docs/performance/insights/inp-breakdown) + processes Chrome Event Timing trace data and reports its input delay, + processing duration, and presentation delay. Their sum is recorded as the + interaction latency. This is one controlled interaction, not Google's + page-lifetime + [Interaction to Next Paint (INP)](https://web.dev/articles/inp) metric. +- Benchmarks run 5 times by default. Paint and interaction timing values are + averaged across those runs. - These are route-based client-side rendering tests, not full-app SPA mode tests. Each app uses its normal production build and configures the benchmark routes so the measured table and detail content are rendered in the browser. @@ -202,14 +209,18 @@ throughput, and load behavior for comparable production apps. - Metrics are measured with Lighthouse flow in Chromium through Puppeteer. - First Paint and First Contentful Paint are measured on initial navigation to `/server-side-rendered`. -- Interaction to Next Paint is measured by clicking the first row's detail link - and waiting for `/server-side-rendered/:id`. +- A controlled interaction clicks the first row's detail link and waits for + `/server-side-rendered/:id`. Lighthouse's INP breakdown insight processes + Chrome Event Timing trace data and reports its input delay, processing + duration, and presentation delay. Their sum is recorded as the interaction + latency. - Detail links use each framework's default production navigation component or idiomatic anchor behavior. Default meta-framework route prefetching or preloading is allowed when it is part of the framework's default link behavior, but the measured SSR routes are still rendered on demand rather than converted to prerendered static output. -- Benchmarks run 5 times by default and average successful metrics. +- Benchmarks run 5 times by default and require every Chrome paint and + interaction measurement to be present and greater than zero. - Astro keeps the default static output mode, but the measured `/server-side-rendered` route and its detail route use `export const prerender = false` so they are rendered on demand by the diff --git a/packages/docs/src/lib/collections.ts b/packages/docs/src/lib/collections.ts index 3dfac11e..0819c205 100644 --- a/packages/docs/src/lib/collections.ts +++ b/packages/docs/src/lib/collections.ts @@ -162,7 +162,6 @@ export const serverSideRenderedStats = runtimeEntries isFocused: framework.isFocused, firstPaintMs: `${framework.serverSideRenderedTests!.firstPaintMs}ms`, fcpMs: `${framework.serverSideRenderedTests!.fcpMs}ms`, - inpMs: `${framework.serverSideRenderedTests!.inpMs}ms`, })) export const clientSideRenderedStats = runtimeEntries @@ -179,7 +178,6 @@ export const clientSideRenderedStats = runtimeEntries isFocused: framework.isFocused, firstPaintMs: `${framework.clientSideRenderedTests!.firstPaintMs}ms`, fcpMs: `${framework.clientSideRenderedTests!.fcpMs}ms`, - inpMs: `${framework.clientSideRenderedTests!.inpMs}ms`, })) export const depsStats = starterStats.map((f) => ({ @@ -324,20 +322,6 @@ export const chartServerSideRenderedFCPData = runtimeEntries focused: f.isFocused, })) -export const chartServerSideRenderedINPData = runtimeEntries - .map((entry) => entry.data) - .sort((a, b) => a.order - b.order) - .filter( - (f) => - f.serverSideRenderedTests != null && - Number.isFinite(f.serverSideRenderedTests.inpMs), - ) - .map((f) => ({ - name: f.name, - value: f.serverSideRenderedTests!.inpMs, - focused: f.isFocused, - })) - export const chartClientSideRenderedFPData = runtimeEntries .map((entry) => entry.data) .sort((a, b) => a.order - b.order) @@ -366,20 +350,6 @@ export const chartClientSideRenderedFCPData = runtimeEntries focused: f.isFocused, })) -export const chartClientSideRenderedINPData = runtimeEntries - .map((entry) => entry.data) - .sort((a, b) => a.order - b.order) - .filter( - (f) => - f.clientSideRenderedTests != null && - Number.isFinite(f.clientSideRenderedTests.inpMs), - ) - .map((f) => ({ - name: f.name, - value: f.clientSideRenderedTests!.inpMs, - focused: f.isFocused, - })) - export const coreJsTableData = starterStats.map((f) => { const hasCorejs = (f.vendoredCoreJsUnnecessaryModules?.length ?? 0) > 0 return { diff --git a/packages/stats-generator/src/clientSideRendered/index.ts b/packages/stats-generator/src/clientSideRendered/index.ts index 2e8af2b5..16adbc07 100644 --- a/packages/stats-generator/src/clientSideRendered/index.ts +++ b/packages/stats-generator/src/clientSideRendered/index.ts @@ -15,6 +15,7 @@ interface ClientSideRenderedFrameworkConfig { package: string /** Filename of the serve script in src/serve/ */ serveScript: string + fullDocumentNavigation: boolean } const CLIENT_SIDE_RENDERED_FRAMEWORKS: ClientSideRenderedFrameworkConfig[] = [ @@ -23,42 +24,49 @@ const CLIENT_SIDE_RENDERED_FRAMEWORKS: ClientSideRenderedFrameworkConfig[] = [ displayName: 'Astro Client Side Rendered', package: 'app-astro', serveScript: 'astro.ts', + fullDocumentNavigation: true, }, { name: 'next-client-side-rendered', displayName: 'Next.js Client Side Rendered', package: 'app-next-js', serveScript: 'next.ts', + fullDocumentNavigation: false, }, { name: 'nuxt-client-side-rendered', displayName: 'Nuxt Client Side Rendered', package: 'app-nuxt', serveScript: 'nitro.ts', + fullDocumentNavigation: false, }, { name: 'react-router-client-side-rendered', displayName: 'React Router Client Side Rendered', package: 'app-react-router', serveScript: 'react-router.ts', + fullDocumentNavigation: false, }, { name: 'solid-start-client-side-rendered', displayName: 'SolidStart Client Side Rendered', package: 'app-solid-start', serveScript: 'nitro.ts', + fullDocumentNavigation: false, }, { name: 'sveltekit-client-side-rendered', displayName: 'SvelteKit Client Side Rendered', package: 'app-sveltekit', serveScript: 'sveltekit.ts', + fullDocumentNavigation: false, }, { name: 'tanstack-start-client-side-rendered', displayName: 'TanStack Start Client Side Rendered', package: 'app-tanstack-start-react', serveScript: 'tanstack-start.ts', + fullDocumentNavigation: false, }, ] @@ -155,6 +163,7 @@ export async function runClientSideRenderedBenchmark( config.name, config.displayName, runs, + config.fullDocumentNavigation, ) } finally { killServer() diff --git a/packages/stats-generator/src/clientSideRendered/run-benchmark.ts b/packages/stats-generator/src/clientSideRendered/run-benchmark.ts index d5f3f68f..b6abcd91 100644 --- a/packages/stats-generator/src/clientSideRendered/run-benchmark.ts +++ b/packages/stats-generator/src/clientSideRendered/run-benchmark.ts @@ -2,6 +2,12 @@ import { existsSync, readdirSync } from 'node:fs' import { execFileSync } from 'node:child_process' import puppeteer from 'puppeteer-core' import { startFlow } from 'lighthouse' +import { + getInteractionTimingFromLighthouse, + INTERACTION_SCENARIO, + INTERACTION_SOURCE, +} from '../interaction-timing.ts' +import type { InteractionTestStats } from '../types.ts' import type { ClientSideRenderedBenchmarkResult, ClientSideRenderedRunResult, @@ -50,6 +56,7 @@ function getBrowserVersion(chromiumPath: string): string | undefined { async function runOnce( url: string, chromiumPath: string, + fullDocumentNavigation: boolean, ): Promise { const browser = await puppeteer.launch({ executablePath: chromiumPath, @@ -73,34 +80,43 @@ async function runOnce( await flow.navigate(`${url}${CLIENT_SIDE_RENDERED_PATH}`) await page.waitForSelector('table tbody tr', { timeout: 15_000 }) - // INP: click the first row's detail link - await flow.startTimespan() - await page.click('table tbody tr:first-child a') - await page.waitForSelector('#detail-id', { timeout: 15_000 }) - // Double rAF ensures the paint entry is recorded before the timespan ends. - await page.evaluate( - (): Promise => - new Promise((r) => { - // @ts-expect-error — callback runs in browser context, not Node.js - requestAnimationFrame(() => requestAnimationFrame(r)) - }), - ) - await flow.endTimespan() + // Use navigation mode when the interaction triggers a full document load. + // https://github.com/GoogleChrome/lighthouse/blob/main/docs/user-flows.md#triggering-a-navigation-via-user-interactions + if (fullDocumentNavigation) { + await flow.startNavigation() + await page.click('table tbody tr:first-child a') + await flow.endNavigation() + await page.waitForSelector('#detail-id', { timeout: 15_000 }) + } else { + await flow.startTimespan() + await page.click('table tbody tr:first-child a') + await page.waitForSelector('#detail-id', { timeout: 15_000 }) + // Double rAF ensures the paint entry is recorded before the timespan ends. + await page.evaluate( + (): Promise => + new Promise((r) => { + // @ts-expect-error — callback runs in browser context, not Node.js + requestAnimationFrame(() => requestAnimationFrame(r)) + }), + ) + await flow.endTimespan() + } const flowResult = await flow.createFlowResult() const navLhr = flowResult.steps[0].lhr - const timespanLhr = flowResult.steps[1].lhr + const interactionLhr = flowResult.steps[1].lhr const metricsItems = ( navLhr.audits['metrics']?.details as { items?: Record[] } )?.items?.[0] const firstPaintMs = metricsItems?.observedFirstPaint ?? null const fcpMs = navLhr.audits['first-contentful-paint']?.numericValue ?? null - const inpMs = - timespanLhr.audits['interaction-to-next-paint']?.numericValue ?? null + const interaction = getInteractionTimingFromLighthouse( + interactionLhr.audits['inp-breakdown-insight'], + ) await page.close() - return { firstPaintMs, fcpMs, inpMs } + return { firstPaintMs, fcpMs, interaction } } finally { await browser.close() } @@ -116,31 +132,70 @@ export async function runBenchmark( packageName: string, displayName: string, runs: number, + fullDocumentNavigation: boolean, ): Promise { const chromiumPath = findChromium() const browserVersion = getBrowserVersion(chromiumPath) + if (!browserVersion) { + throw new Error(`Could not read Chrome version from ${chromiumPath}`) + } const results: ClientSideRenderedRunResult[] = [] for (let i = 0; i < runs; i++) { console.log(` Run ${i + 1}/${runs}...`) - results.push(await runOnce(url, chromiumPath)) + results.push(await runOnce(url, chromiumPath, fullDocumentNavigation)) } const fp = results .map((r) => r.firstPaintMs) .filter((v): v is number => v !== null) const fcp = results.map((r) => r.fcpMs).filter((v): v is number => v !== null) - const inp = results.map((r) => r.inpMs).filter((v): v is number => v !== null) + if ( + fp.length !== results.length || + fp.some((value) => !Number.isFinite(value) || value <= 0) + ) { + throw new Error('First Paint was missing, non-finite, or zero') + } + if ( + fcp.length !== results.length || + fcp.some((value) => !Number.isFinite(value) || value <= 0) + ) { + throw new Error('FCP was missing, non-finite, or zero') + } + const interactions = results + .map((r) => r.interaction) + .filter((value) => value !== null) + if (interactions.length !== results.length) { + throw new Error( + `Interaction timing could not be measured in ${results.length - interactions.length} of ${results.length} runs`, + ) + } + const firstPaintMs = avg(fp) + const fcpMs = avg(fcp) + const interactionTests: InteractionTestStats = { + scenario: INTERACTION_SCENARIO, + source: INTERACTION_SOURCE, + interactionLatencyMs: avg( + interactions.map((value) => value.interactionLatencyMs), + ), + inputDelayMs: avg(interactions.map((value) => value.inputDelayMs)), + processingDurationMs: avg( + interactions.map((value) => value.processingDurationMs), + ), + presentationDelayMs: avg( + interactions.map((value) => value.presentationDelayMs), + ), + } return { name: packageName, displayName, package: packageName, browserVersion, clientSideRenderedTests: { - firstPaintMs: avg(fp), - fcpMs: avg(fcp), - inpMs: avg(inp), + firstPaintMs, + fcpMs, + interactionTests, runs: results.length, }, } diff --git a/packages/stats-generator/src/clientSideRendered/types.ts b/packages/stats-generator/src/clientSideRendered/types.ts index be21c54c..550c97ab 100644 --- a/packages/stats-generator/src/clientSideRendered/types.ts +++ b/packages/stats-generator/src/clientSideRendered/types.ts @@ -1,18 +1,20 @@ +import type { InteractionTestStats, InteractionTiming } from '../types.ts' + export interface ClientSideRenderedRunResult { firstPaintMs: number | null fcpMs: number | null - inpMs: number | null + interaction: InteractionTiming | null } export interface ClientSideRenderedBenchmarkResult { name: string displayName: string package: string - browserVersion?: string + browserVersion: string clientSideRenderedTests: { firstPaintMs: number fcpMs: number - inpMs: number + interactionTests: InteractionTestStats runs: number } } @@ -25,7 +27,8 @@ export interface ClientSideRenderedStats { clientSideRenderedTests: { firstPaintMs: number fcpMs: number - inpMs: number + inpMs?: number + interactionTests?: InteractionTestStats runs: number } } diff --git a/packages/stats-generator/src/interaction-timing.ts b/packages/stats-generator/src/interaction-timing.ts new file mode 100644 index 00000000..4463ed90 --- /dev/null +++ b/packages/stats-generator/src/interaction-timing.ts @@ -0,0 +1,59 @@ +import type { InteractionTiming } from './types.ts' + +export const INTERACTION_SCENARIO = 'first-row-detail-navigation' +export const INTERACTION_SOURCE = 'lighthouse-inp-breakdown' + +interface LighthouseAudit { + details?: unknown +} + +interface INPBreakdownDetails { + items?: Array<{ + type?: string + items?: Array<{ subpart?: string; duration?: number }> + }> +} + +export function getInteractionTimingFromLighthouse( + audit: LighthouseAudit | undefined, +): InteractionTiming | null { + const details = audit?.details as INPBreakdownDetails | undefined + const items = details?.items?.find((item) => item.type === 'table')?.items + const inputDelayMs = items?.find( + (item) => item.subpart === 'inputDelay', + )?.duration + const processingDurationMs = items?.find( + (item) => item.subpart === 'processingDuration', + )?.duration + const presentationDelayMs = items?.find( + (item) => item.subpart === 'presentationDelay', + )?.duration + + if ( + typeof inputDelayMs !== 'number' || + !Number.isFinite(inputDelayMs) || + inputDelayMs < 0 || + typeof processingDurationMs !== 'number' || + !Number.isFinite(processingDurationMs) || + processingDurationMs < 0 || + typeof presentationDelayMs !== 'number' || + !Number.isFinite(presentationDelayMs) || + presentationDelayMs < 0 + ) { + return null + } + + const interactionLatencyMs = + inputDelayMs + processingDurationMs + presentationDelayMs + + if (interactionLatencyMs <= 0) { + return null + } + + return { + interactionLatencyMs, + inputDelayMs, + processingDurationMs, + presentationDelayMs, + } +} diff --git a/packages/stats-generator/src/run-client-side-rendered-benchmark.ts b/packages/stats-generator/src/run-client-side-rendered-benchmark.ts index dd4d4f41..d3a196a0 100644 --- a/packages/stats-generator/src/run-client-side-rendered-benchmark.ts +++ b/packages/stats-generator/src/run-client-side-rendered-benchmark.ts @@ -55,7 +55,11 @@ async function main() { ` First Paint: ${result.clientSideRenderedTests.firstPaintMs}ms`, ) console.info(` FCP: ${result.clientSideRenderedTests.fcpMs}ms`) - console.info(` INP: ${result.clientSideRenderedTests.inpMs}ms`) + const interaction = result.clientSideRenderedTests.interactionTests + console.info(` Interaction: ${interaction.interactionLatencyMs}ms`) + console.info(` Input delay: ${interaction.inputDelayMs}ms`) + console.info(` Processing: ${interaction.processingDurationMs}ms`) + console.info(` Presentation: ${interaction.presentationDelayMs}ms`) } main().catch((error) => { diff --git a/packages/stats-generator/src/run-server-side-rendered-benchmark.ts b/packages/stats-generator/src/run-server-side-rendered-benchmark.ts index 2c49b1ce..ba5c6437 100644 --- a/packages/stats-generator/src/run-server-side-rendered-benchmark.ts +++ b/packages/stats-generator/src/run-server-side-rendered-benchmark.ts @@ -55,7 +55,11 @@ async function main() { ` First Paint: ${result.serverSideRenderedTests.firstPaintMs}ms`, ) console.info(` FCP: ${result.serverSideRenderedTests.fcpMs}ms`) - console.info(` INP: ${result.serverSideRenderedTests.inpMs}ms`) + const interaction = result.serverSideRenderedTests.interactionTests + console.info(` Interaction: ${interaction.interactionLatencyMs}ms`) + console.info(` Input delay: ${interaction.inputDelayMs}ms`) + console.info(` Processing: ${interaction.processingDurationMs}ms`) + console.info(` Presentation: ${interaction.presentationDelayMs}ms`) } main().catch((error) => { diff --git a/packages/stats-generator/src/schemas.ts b/packages/stats-generator/src/schemas.ts index bf8f5777..9961d72f 100644 --- a/packages/stats-generator/src/schemas.ts +++ b/packages/stats-generator/src/schemas.ts @@ -82,6 +82,42 @@ export const SSRLoadStatsSchema = z.object({ browserVersion: z.string().optional(), }) +const InteractionTestsSchema = z.object({ + scenario: z.literal('first-row-detail-navigation'), + source: z.literal('lighthouse-inp-breakdown'), + interactionLatencyMs: z.number().positive(), + inputDelayMs: z.number().nonnegative(), + processingDurationMs: z.number().nonnegative(), + presentationDelayMs: z.number().nonnegative(), +}) + +const RenderedTestsSchema = z + .object({ + firstPaintMs: z.number().positive(), + fcpMs: z.number().positive(), + /** Historical only; new runs record interactionTests instead. */ + inpMs: z.number().optional(), + interactionTests: InteractionTestsSchema.optional(), + runs: z.number().int().positive(), + }) + .refine( + (stats) => + stats.inpMs !== undefined || stats.interactionTests !== undefined, + { + message: 'Expected historical inpMs or current interactionTests', + }, + ) + +export const ClientSideRenderedStatsSchema = z.object({ + clientSideRenderedTests: RenderedTestsSchema, + browserVersion: z.string().min(1), +}) + +export const ServerSideRenderedStatsSchema = z.object({ + serverSideRenderedTests: RenderedTestsSchema, + browserVersion: z.string().min(1), +}) + export type InstallStats = z.infer export type BuildStats = z.infer export type BrowserBaselineStats = z.infer @@ -89,4 +125,10 @@ export type SSRRequestThroughputStats = z.infer< typeof SSRRequestThroughputStatsSchema > export type SSRLoadStats = z.infer +export type ClientSideRenderedStats = z.infer< + typeof ClientSideRenderedStatsSchema +> +export type ServerSideRenderedStats = z.infer< + typeof ServerSideRenderedStatsSchema +> export type TimeStat = z.infer diff --git a/packages/stats-generator/src/serverSideRendered/index.ts b/packages/stats-generator/src/serverSideRendered/index.ts index e7f311d7..e494ed85 100644 --- a/packages/stats-generator/src/serverSideRendered/index.ts +++ b/packages/stats-generator/src/serverSideRendered/index.ts @@ -14,6 +14,7 @@ interface ServerSideRenderedFrameworkConfig { displayName: string package: string serveScript: string + fullDocumentNavigation: boolean } const SERVER_SIDE_RENDERED_FRAMEWORKS: ServerSideRenderedFrameworkConfig[] = [ @@ -22,48 +23,56 @@ const SERVER_SIDE_RENDERED_FRAMEWORKS: ServerSideRenderedFrameworkConfig[] = [ displayName: 'Astro Server Side Rendered', package: 'app-astro', serveScript: 'astro.ts', + fullDocumentNavigation: true, }, { name: 'mastro-server-side-rendered', displayName: 'Mastro Server Side Rendered', package: 'app-mastro', serveScript: 'mastro.ts', + fullDocumentNavigation: true, }, { name: 'next-server-side-rendered', displayName: 'Next.js Server Side Rendered', package: 'app-next-js', serveScript: 'next.ts', + fullDocumentNavigation: false, }, { name: 'nuxt-server-side-rendered', displayName: 'Nuxt Server Side Rendered', package: 'app-nuxt', serveScript: 'nitro.ts', + fullDocumentNavigation: false, }, { name: 'react-router-server-side-rendered', displayName: 'React Router Server Side Rendered', package: 'app-react-router', serveScript: 'react-router.ts', + fullDocumentNavigation: false, }, { name: 'solid-start-server-side-rendered', displayName: 'SolidStart Server Side Rendered', package: 'app-solid-start', serveScript: 'nitro.ts', + fullDocumentNavigation: false, }, { name: 'sveltekit-server-side-rendered', displayName: 'SvelteKit Server Side Rendered', package: 'app-sveltekit', serveScript: 'sveltekit.ts', + fullDocumentNavigation: false, }, { name: 'tanstack-start-server-side-rendered', displayName: 'TanStack Start Server Side Rendered', package: 'app-tanstack-start-react', serveScript: 'tanstack-start.ts', + fullDocumentNavigation: false, }, ] @@ -160,6 +169,7 @@ export async function runServerSideRenderedBenchmark( config.name, config.displayName, runs, + config.fullDocumentNavigation, ) } finally { killServer() diff --git a/packages/stats-generator/src/serverSideRendered/run-benchmark.ts b/packages/stats-generator/src/serverSideRendered/run-benchmark.ts index d51e2dd0..2a00b47a 100644 --- a/packages/stats-generator/src/serverSideRendered/run-benchmark.ts +++ b/packages/stats-generator/src/serverSideRendered/run-benchmark.ts @@ -2,6 +2,12 @@ import { existsSync, readdirSync } from 'node:fs' import { execFileSync } from 'node:child_process' import puppeteer from 'puppeteer-core' import { startFlow } from 'lighthouse' +import { + getInteractionTimingFromLighthouse, + INTERACTION_SCENARIO, + INTERACTION_SOURCE, +} from '../interaction-timing.ts' +import type { InteractionTestStats } from '../types.ts' import type { ServerSideRenderedBenchmarkResult, ServerSideRenderedRunResult, @@ -50,6 +56,7 @@ function getBrowserVersion(chromiumPath: string): string | undefined { async function runOnce( url: string, chromiumPath: string, + fullDocumentNavigation: boolean, ): Promise { const browser = await puppeteer.launch({ executablePath: chromiumPath, @@ -73,36 +80,45 @@ async function runOnce( await flow.navigate(`${url}${SERVER_SIDE_RENDERED_PATH}`) await page.waitForSelector('table tbody tr', { timeout: 15_000 }) - // INP: click the first row's detail link - await flow.startTimespan() - await page.click('table tbody tr:first-child a') - await page.waitForSelector('#detail-id', { timeout: 15_000 }) - // Double rAF ensures the paint entry is recorded before the timespan ends. - await page.evaluate((): Promise => { - const { requestAnimationFrame: nextFrame } = globalThis as unknown as { - requestAnimationFrame: (callback: () => void) => number - } - - return new Promise((resolve) => { - nextFrame(() => nextFrame(resolve)) + // Use navigation mode when the interaction triggers a full document load. + // https://github.com/GoogleChrome/lighthouse/blob/main/docs/user-flows.md#triggering-a-navigation-via-user-interactions + if (fullDocumentNavigation) { + await flow.startNavigation() + await page.click('table tbody tr:first-child a') + await flow.endNavigation() + await page.waitForSelector('#detail-id', { timeout: 15_000 }) + } else { + await flow.startTimespan() + await page.click('table tbody tr:first-child a') + await page.waitForSelector('#detail-id', { timeout: 15_000 }) + // Double rAF ensures the paint entry is recorded before the timespan ends. + await page.evaluate((): Promise => { + const { requestAnimationFrame: nextFrame } = globalThis as unknown as { + requestAnimationFrame: (callback: () => void) => number + } + + return new Promise((resolve) => { + nextFrame(() => nextFrame(resolve)) + }) }) - }) - await flow.endTimespan() + await flow.endTimespan() + } const flowResult = await flow.createFlowResult() const navLhr = flowResult.steps[0].lhr - const timespanLhr = flowResult.steps[1].lhr + const interactionLhr = flowResult.steps[1].lhr const metricsItems = ( navLhr.audits['metrics']?.details as { items?: Record[] } )?.items?.[0] const firstPaintMs = metricsItems?.observedFirstPaint ?? null const fcpMs = navLhr.audits['first-contentful-paint']?.numericValue ?? null - const inpMs = - timespanLhr.audits['interaction-to-next-paint']?.numericValue ?? null + const interaction = getInteractionTimingFromLighthouse( + interactionLhr.audits['inp-breakdown-insight'], + ) await page.close() - return { firstPaintMs, fcpMs, inpMs } + return { firstPaintMs, fcpMs, interaction } } finally { await browser.close() } @@ -118,31 +134,70 @@ export async function runBenchmark( packageName: string, displayName: string, runs: number, + fullDocumentNavigation: boolean, ): Promise { const chromiumPath = findChromium() const browserVersion = getBrowserVersion(chromiumPath) + if (!browserVersion) { + throw new Error(`Could not read Chrome version from ${chromiumPath}`) + } const results: ServerSideRenderedRunResult[] = [] for (let i = 0; i < runs; i++) { console.info(` Run ${i + 1}/${runs}...`) - results.push(await runOnce(url, chromiumPath)) + results.push(await runOnce(url, chromiumPath, fullDocumentNavigation)) } const fp = results .map((r) => r.firstPaintMs) .filter((v): v is number => v !== null) const fcp = results.map((r) => r.fcpMs).filter((v): v is number => v !== null) - const inp = results.map((r) => r.inpMs).filter((v): v is number => v !== null) + if ( + fp.length !== results.length || + fp.some((value) => !Number.isFinite(value) || value <= 0) + ) { + throw new Error('First Paint was missing, non-finite, or zero') + } + if ( + fcp.length !== results.length || + fcp.some((value) => !Number.isFinite(value) || value <= 0) + ) { + throw new Error('FCP was missing, non-finite, or zero') + } + const interactions = results + .map((r) => r.interaction) + .filter((value) => value !== null) + if (interactions.length !== results.length) { + throw new Error( + `Interaction timing could not be measured in ${results.length - interactions.length} of ${results.length} runs`, + ) + } + const firstPaintMs = avg(fp) + const fcpMs = avg(fcp) + const interactionTests: InteractionTestStats = { + scenario: INTERACTION_SCENARIO, + source: INTERACTION_SOURCE, + interactionLatencyMs: avg( + interactions.map((value) => value.interactionLatencyMs), + ), + inputDelayMs: avg(interactions.map((value) => value.inputDelayMs)), + processingDurationMs: avg( + interactions.map((value) => value.processingDurationMs), + ), + presentationDelayMs: avg( + interactions.map((value) => value.presentationDelayMs), + ), + } return { name: packageName, displayName, package: packageName, browserVersion, serverSideRenderedTests: { - firstPaintMs: avg(fp), - fcpMs: avg(fcp), - inpMs: avg(inp), + firstPaintMs, + fcpMs, + interactionTests, runs: results.length, }, } diff --git a/packages/stats-generator/src/serverSideRendered/types.ts b/packages/stats-generator/src/serverSideRendered/types.ts index 55bafb91..2460be19 100644 --- a/packages/stats-generator/src/serverSideRendered/types.ts +++ b/packages/stats-generator/src/serverSideRendered/types.ts @@ -1,18 +1,20 @@ +import type { InteractionTestStats, InteractionTiming } from '../types.ts' + export interface ServerSideRenderedRunResult { firstPaintMs: number | null fcpMs: number | null - inpMs: number | null + interaction: InteractionTiming | null } export interface ServerSideRenderedBenchmarkResult { name: string displayName: string package: string - browserVersion?: string + browserVersion: string serverSideRenderedTests: { firstPaintMs: number fcpMs: number - inpMs: number + interactionTests: InteractionTestStats runs: number } } @@ -25,7 +27,8 @@ export interface ServerSideRenderedStats { serverSideRenderedTests: { firstPaintMs: number fcpMs: number - inpMs: number + inpMs?: number + interactionTests?: InteractionTestStats runs: number } } diff --git a/packages/stats-generator/src/types.ts b/packages/stats-generator/src/types.ts index f4cadb04..ab7e11a8 100644 --- a/packages/stats-generator/src/types.ts +++ b/packages/stats-generator/src/types.ts @@ -21,6 +21,18 @@ export interface TestConfig { measurements: MeasurementConfig[] } +export interface InteractionTiming { + interactionLatencyMs: number + inputDelayMs: number + processingDurationMs: number + presentationDelayMs: number +} + +export interface InteractionTestStats extends InteractionTiming { + scenario: 'first-row-detail-navigation' + source: 'lighthouse-inp-breakdown' +} + export interface FrameworkConfig { name: string displayName: string @@ -83,14 +95,16 @@ export interface CIStats { clientSideRenderedTests?: { firstPaintMs: number fcpMs: number - inpMs: number + inpMs?: number + interactionTests?: InteractionTestStats runs: number } // Server-side rendered route stats (browser paint + interaction timings) serverSideRenderedTests?: { firstPaintMs: number fcpMs: number - inpMs: number + inpMs?: number + interactionTests?: InteractionTestStats runs: number } // Core-js vendored polyfill stats diff --git a/packages/stats-generator/src/validate-stats.ts b/packages/stats-generator/src/validate-stats.ts index 09a1b5e9..fc0f2a08 100644 --- a/packages/stats-generator/src/validate-stats.ts +++ b/packages/stats-generator/src/validate-stats.ts @@ -9,10 +9,18 @@ import { BrowserBaselineStatsSchema, SSRRequestThroughputStatsSchema, SSRLoadStatsSchema, + ClientSideRenderedStatsSchema, + ServerSideRenderedStatsSchema, } from './schemas.ts' type BenchmarkType = - 'install' | 'build' | 'browserBaseline' | 'ssrRequestThroughput' | 'ssrLoad' + | 'install' + | 'build' + | 'browserBaseline' + | 'ssrRequestThroughput' + | 'ssrLoad' + | 'clientSideRendered' + | 'serverSideRendered' interface BenchmarkConfig { type: BenchmarkType @@ -31,6 +39,16 @@ const STARTER_BENCHMARKS: BenchmarkConfig[] = [ ] const APP_BENCHMARKS: BenchmarkConfig[] = [ + { + type: 'clientSideRendered', + file: 'ci-stats.json', + schema: ClientSideRenderedStatsSchema, + }, + { + type: 'serverSideRendered', + file: 'ci-stats.json', + schema: ServerSideRenderedStatsSchema, + }, { type: 'ssrRequestThroughput', file: 'ci-stats.json',