diff --git a/docs/design/behaviors.md b/docs/design/behaviors.md index 94948f0f..d178ee62 100644 --- a/docs/design/behaviors.md +++ b/docs/design/behaviors.md @@ -259,8 +259,12 @@ Rule of thumb: `app:` styles apply to **installed** (standalone **or** native); installed-only, because the browser tab on the same simulator passed geometry the same day; the e2e spec that followed (`playground/e2e/drawer-keyboard.spec.ts`) shows it on headless Chromium and on Playwright's WebKit at a 390 × 844 viewport too, so the variable is the viewport the - scenario runs in, not the shell. The harness now names the turn on the overlay and carries the - sample series on every `StepResult`, and the series says what the turn is: an **overshoot**. + scenario runs in, not the shell. That spec runs the harness with `?virtual` on Playwright's + paused `page.clock`, stepped 16ms at a time, with every CSS animation held to that clock, so + each step reads ~44 samples whatever the runner's frame rate; a step with fewer than 3 fails as + `undersampled` instead of passing checks it had no series for. The harness now names the turn + on the overlay and carries the sample series on every `StepResult`, and the series says what + the turn is: an **overshoot**. The content edge sits at the screen bottom while the picker collapses, then moves in ONE frame to a point past its rest, and eases back to the rest over the next ~300ms: diff --git a/playground/apps/frontend/src/routing/pages/lab/drawer-keyboard.page.tsx b/playground/apps/frontend/src/routing/pages/lab/drawer-keyboard.page.tsx index 00de5845..5ba3af0c 100644 --- a/playground/apps/frontend/src/routing/pages/lab/drawer-keyboard.page.tsx +++ b/playground/apps/frontend/src/routing/pages/lab/drawer-keyboard.page.tsx @@ -66,6 +66,46 @@ type Trace = { samples: Sample[]; frames: number[] } const SETTLE_MS = 700 const FRAME_TOLERANCE = 2 +const MIN_SAMPLES = 3 + +/* + * `?virtual`: every animation on the page is held and moved by hand to `performance.now()`, once + * per frame and on every DOM change. The engine's timers and its re-aim maths already read that + * clock, so a test that installs Playwright's `page.clock` and steps it 16ms at a time gets one + * sample per 16ms of motion whatever the machine's frame rate, with the CSS transitions on the + * same timeline as the code that starts and re-aims them. Without the flag nothing is touched. + */ +let followPageClock: (() => void) | null = null + +function holdAnimationsToPageClock() { + const startedAt = new WeakMap() + const follow = () => { + const now = performance.now() + for (const animation of document.getAnimations()) { + let start = startedAt.get(animation) + if (start === undefined) { + start = now - (Number(animation.currentTime) || 0) + startedAt.set(animation, start) + animation.pause() + } + const end = animation.effect?.getComputedTiming().endTime + if (typeof end === "number" && now - start >= end) animation.finish() + else animation.currentTime = now - start + } + } + //a transition starts on the style change that caused it; catch it before it moves on its own + new MutationObserver(follow).observe(document, { + attributes: true, + childList: true, + subtree: true, + }) + const frame = () => { + follow() + requestAnimationFrame(frame) + } + requestAnimationFrame(frame) + return follow +} function sampleGeometry(content: HTMLElement, startedAt: number): Sample { const rect = content.getBoundingClientRect() @@ -134,6 +174,7 @@ function recordTransition( perturbed = true mid.apply() } + followPageClock?.() samples.push(sampleGeometry(content, startedAt)) if (elapsed < SETTLE_MS) { raf = requestAnimationFrame(tick) @@ -245,6 +286,16 @@ function checkTransition( * That reads the painted frames, so there is nothing to tune and nothing to fool. */ const checks: Check[] = [ + //A turn needs a sample before it, at it and after it. With fewer, the reversal checks below + //pass on any motion, so the step says it was not judged instead of saying it was fine. + { + name: "sampled enough to judge", + pass: samples.length >= MIN_SAMPLES, + detail: + samples.length >= MIN_SAMPLES + ? `${samples.length} samples` + : `undersampled: ${samples.length} samples (needs >=${MIN_SAMPLES})`, + }, { name: "never above the safe top", pass: minTop >= expected.safeTop - FRAME_TOLERANCE, @@ -342,6 +393,12 @@ function LabDrawerKeyboardPage() { const extraRowsRef = useRef(0) extraRowsRef.current = extraRows + useEffect(() => { + if (new URLSearchParams(window.location.search).has("virtual")) { + followPageClock ??= holdAnimationsToPageClock() + } + }, []) + //the mock has to exist before the drawer's observer mounts useEffect(() => { setMockKeyboard(0) @@ -391,16 +448,32 @@ function LabDrawerKeyboardPage() { const content = document.querySelector( "[data-pwa-drawer] > div", ) - if (!content) { - setRunning(false) - return - } const collected: StepResult[] = [] const publish = () => { ;( window as unknown as { __drawerConformance?: StepResult[] } ).__drawerConformance = collected } + //a run that cannot start is a verdict too: a reader waiting on the window property must not + //wait on a run that never began + if (!content) { + collected.push({ + step: "run aborted", + samples: [], + checks: [ + { + name: "every scenario ran", + pass: false, + detail: + "no sheet content ([data-pwa-drawer] > div) when the run started", + }, + ], + }) + setResults([...collected]) + publish() + setRunning(false) + return + } try { await scenarios(content, only, collected) } catch (error) { diff --git a/playground/e2e/drawer-keyboard.spec.ts b/playground/e2e/drawer-keyboard.spec.ts index 3d6b55c7..4ece7153 100644 --- a/playground/e2e/drawer-keyboard.spec.ts +++ b/playground/e2e/drawer-keyboard.spec.ts @@ -1,3 +1,4 @@ +import type { Page } from "@playwright/test" import { expect, test } from "@playwright/test" import { awaitClientHandover } from "./support/hydrated" @@ -13,9 +14,8 @@ import { awaitClientHandover } from "./support/hydrated" * * The geometry checks are asserted here on both engines. The two TIMING checks are read but not * asserted, by name: "runs at frame rate" and "eased, not snapped" measure the sampler as much - * as the sheet, and a CI runner that starves requestAnimationFrame fails them with the geometry - * intact (docs/design/behaviors.md records the simulator doing the same). They stay in the report - * this spec attaches, so a run that went slow is visible without being red. + * as the sheet, and on the page clock below every frame is 16ms by construction, so they say + * nothing about the machine. They stay in the report this spec attaches. */ type Check = { name: string; pass: boolean; detail: string } @@ -28,6 +28,10 @@ type StepResult = { const TIMING_CHECKS = new Set(["runs at frame rate", "eased, not snapped"]) +//a turn needs a sample before it, at it and after it: with two, a reversal or an overshoot +//cannot show, and every geometry check of the step passes without reading anything +const MIN_SAMPLES = 3 + //a phone held upright: the scenarios size the keyboard as a fraction of the viewport, and the //interesting geometry (a sheet taller than the room left above the keyboard) needs a tall one. //Touch, because the drag step fires TouchEvents and a desktop context has no constructor for them. @@ -36,7 +40,7 @@ test.use({ hasTouch: true, viewport: { width: 390, height: 844 } }) /* * One geometry failure is KNOWN and allowed by name, and only that one: in the scenario where the * picker collapses first and the keyboard lands 48ms later, the content edge overshoots its rest - * and comes back (headless chromium at this viewport: 844 → 506, turned 8px at 129ms, 463 → 471; + * and comes back (both engines at this viewport, on the page clock: 844 → 506, turned 13px at 116ms; * the installed targets in docs/design/behaviors.md show the same turn). The harness names it so * the fix can be aimed; until it lands, this spec fails on ANY other geometry failure and on the * day the known one disappears, so the allowance is deleted the moment it is stale. @@ -50,25 +54,11 @@ test.setTimeout(120_000) test("the conformance run passes every geometry check @tailwind", async ({ page, }, testInfo) => { - await page.goto("/lab/drawer-keyboard?autorun") - //the SSR splash self-unmounts on hydration, and `?autorun` opens the sheet from a client - //effect, so nothing below is meaningful before it is gone - await awaitClientHandover(page) - //a SYNC predicate: an async one returns a Promise, which is truthy, and the wait is a no-op - await page.waitForFunction( - () => - Array.isArray( - (window as unknown as { __drawerConformance?: unknown }) - .__drawerConformance, - ), - null, - { timeout: 90_000 }, - ) - const results = await page.evaluate( - () => - (window as unknown as { __drawerConformance: StepResult[] }) - .__drawerConformance, - ) + //One frame per 16ms of page time, stepped from here, with the sheet's CSS transitions held to + //the same clock (`?virtual`): how many samples a step gets no longer depends on how fast the + //runner paints (a starved webkit runner once gave "release 70%→0" two). + await openOnPausedClock(page, "/lab/drawer-keyboard?virtual") + const results = await runOnClock(page, () => page.clock.runFor(16)) await testInfo.attach("conformance", { body: JSON.stringify( results.map(({ samples, ...rest }) => ({ @@ -105,13 +95,10 @@ test("the conformance run passes every geometry check @tailwind", async ({ expect(known.map((c) => c.detail)).toHaveLength(1) expect(known[0].detail).toMatch(/turned \d+px at \d+ms/) - //every frame-sampled step carries a series to read its checks from. Only that it EXISTS is - //asserted: how many samples a step gets is the runner's frame rate (a starved webkit runner - //yielded 2 for "release 70%→0" with every geometry check passing), the same sampler - //dependence the TIMING_CHECKS above are exempt for. The per-step counts are in the attachment. + //every frame-sampled step carries a series long enough to read its checks from for (const r of results) { if (r.step === "drag with the keyboard up") continue - expect(r.samples.length, r.step).toBeGreaterThan(0) + expect(r.samples.length, r.step).toBeGreaterThanOrEqual(MIN_SAMPLES) } //the overlay says the same thing the data does @@ -121,3 +108,73 @@ test("the conformance run passes every geometry check @tailwind", async ({ failedTotal === 0 ? "PASS" : `FAIL ${failedTotal}/${checks.length}`, ) }) + +/* + * A starved runner, made on purpose: the page clock jumps 400ms at a time, so each 700ms step + * sees two frames. The run must call that out, step by step, instead of passing geometry checks + * it had no series to read. + */ +test("a step sampled fewer than 3 times is reported undersampled @tailwind", async ({ + page, +}) => { + await openOnPausedClock(page, "/lab/drawer-keyboard?only=dismiss") + //Frames at the normal pace until the sheet's content is mounted: a 400ms jump can fire the + //run's 600ms start timer before the sheet has rendered, and the run then has nothing to drive. + //Starved from there on. + const sheet = page.locator("[data-pwa-drawer] > div") + const results = await runOnClock(page, async () => + (await sheet.count()) + ? page.clock.fastForward(400) + : page.clock.runFor(16), + ) + + const sampled = results.map((r) => ({ + step: r.step, + samples: r.samples.length, + check: r.checks.find((c) => c.name === "sampled enough to judge"), + })) + expect(sampled.length).toBeGreaterThan(0) + for (const s of sampled) { + expect(s.samples, s.step).toBeLessThan(MIN_SAMPLES) + expect(s.check?.pass, s.step).toBe(false) + expect(s.check?.detail, s.step).toMatch(/^undersampled: \d samples/) + } +}) + +/** + * The splash leaves on timers, so the clock runs through the handover; it is paused before the + * sheet opens, so the run's first timer is already on the paused clock. + */ +async function openOnPausedClock(page: Page, url: string) { + await page.clock.install() + await page.goto(url) + await awaitClientHandover(page) + //a second ahead: the clock is still running, and pausing in its past throws + await page.clock.pauseAt((await page.evaluate(() => Date.now())) + 1_000) + await page + .getByRole("button", { name: "Open drawer (runs itself)" }) + .click() +} + +/** Moves the page clock with `advance` until the run publishes, and returns what it published. */ +async function runOnClock(page: Page, advance: () => Promise) { + for (let i = 0; i < 5_000; i++) { + await advance() + const done = await page.evaluate(() => + Array.isArray( + (window as unknown as { __drawerConformance?: unknown }) + .__drawerConformance, + ), + ) + if (done) { + return page.evaluate( + () => + (window as unknown as { __drawerConformance: StepResult[] }) + .__drawerConformance, + ) + } + } + throw new Error( + "the conformance run never published window.__drawerConformance (5000 clock steps)", + ) +}