From af29b40ef15eb11e1c347b1feed2a62e6b398ae0 Mon Sep 17 00:00:00 2001 From: Vance Ingalls Date: Tue, 20 Oct 2026 17:39:04 -0700 Subject: [PATCH 1/2] feat(core): capture motion blur pose, discovery and compositor Add the three pure modules capture motion blur is built from: pose (an element's box as an affine in composition space, and the poses between two times), discover (find the elements that move fast enough for a smear to show) and composite (redraw each mover along its path into one frame). Nothing calls them yet; the runtime that wires them up lands in the next change. The pose reads the individual translate, rotate and scale properties in CSS order, adds the offsetParent border back and subtracts ancestor scroll, so it matches getBoundingClientRect for a box centered with translate: -50% -50%. Folded review fixes: R3 (pose half; the scan's 3D skip for individual transforms lands with the mover scan). Co-Authored-By: Claude Sonnet 5.5 --- .../captureMotionBlur/composite.test.ts | 80 +++++ .../runtime/captureMotionBlur/composite.ts | 135 +++++++ .../captureMotionBlur/discover.test.ts | 89 +++++ .../src/runtime/captureMotionBlur/discover.ts | 124 +++++++ .../runtime/captureMotionBlur/pose.test.ts | 230 ++++++++++++ .../src/runtime/captureMotionBlur/pose.ts | 337 ++++++++++++++++++ 6 files changed, 995 insertions(+) create mode 100644 packages/core/src/runtime/captureMotionBlur/composite.test.ts create mode 100644 packages/core/src/runtime/captureMotionBlur/composite.ts create mode 100644 packages/core/src/runtime/captureMotionBlur/discover.test.ts create mode 100644 packages/core/src/runtime/captureMotionBlur/discover.ts create mode 100644 packages/core/src/runtime/captureMotionBlur/pose.test.ts create mode 100644 packages/core/src/runtime/captureMotionBlur/pose.ts diff --git a/packages/core/src/runtime/captureMotionBlur/composite.test.ts b/packages/core/src/runtime/captureMotionBlur/composite.test.ts new file mode 100644 index 00000000000..3fd0033873f --- /dev/null +++ b/packages/core/src/runtime/captureMotionBlur/composite.test.ts @@ -0,0 +1,80 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { createCompositor, type MoverFrame } from "./composite"; +import { IDENTITY, translate } from "./pose"; + +function fakeContext() { + return { + drawImage: vi.fn(), + clearRect: vi.fn(), + setTransform: vi.fn(), + globalAlpha: 1, + globalCompositeOperation: "source-over", + }; +} +let offscreen: Array> = []; + +beforeEach(() => { + offscreen = []; + vi.stubGlobal( + "OffscreenCanvas", + class { + constructor( + public width: number, + public height: number, + ) {} + getContext() { + const ctx = fakeContext(); + offscreen.push(ctx); + return ctx; + } + }, + ); +}); +afterEach(() => vi.unstubAllGlobals()); + +const capture = {} as CanvasImageSource; +const background = {} as CanvasImageSource; +const slide = (from: number, to: number, n = 16) => + Array.from({ length: n }, (_, k) => translate(IDENTITY, from + ((to - from) * (k + 0.5)) / n, 0)); +const mover = (w: number, h: number, taps: ReturnType): MoverFrame => ({ + w, + h, + now: translate(IDENTITY, 50, 0), + taps, +}); + +describe("createCompositor", () => { + it("keeps no crop for a mover that does not travel a pixel, and draws only the background", () => { + const c = createCompositor(100, 100); + const movers = [mover(10, 10, slide(0, 0.5))]; + const kept = c.crop(capture, movers); + expect(kept).toEqual([false]); + const out = fakeContext(); + expect(c.draw(background, out as never, movers, kept)).toEqual({ blurred: 0, taps: 0 }); + expect(out.drawImage).toHaveBeenCalledTimes(1); + expect(out.drawImage.mock.calls[0]?.[0]).toBe(background); + }); + + it("crops a moving mover once and lays one tap per draw over the background", () => { + const c = createCompositor(1000, 100); + const movers = [mover(10, 10, slide(0, 100))]; + const kept = c.crop(capture, movers); + expect(kept).toEqual([true]); + const crop = offscreen.at(-1); + expect(crop?.drawImage).toHaveBeenCalledWith(capture, 0, 0); + const out = fakeContext(); + const stats = c.draw(background, out as never, movers, kept); + const accum = offscreen[0]; + expect(stats.blurred).toBe(1); + expect(accum?.drawImage).toHaveBeenCalledTimes(stats.taps); + expect(accum?.globalAlpha).toBeCloseTo(1 / stats.taps, 9); + // Background first, then the trail over it. + expect(out.drawImage.mock.calls[0]).toEqual([background, 0, 0]); + expect(out.drawImage).toHaveBeenCalledTimes(2); + }); + + it("leaves a mover sharp when its box is too big to copy every frame", () => { + const c = createCompositor(100, 100); + expect(c.crop(capture, [mover(1000, 1000, slide(0, 100))])).toEqual([false]); + }); +}); diff --git a/packages/core/src/runtime/captureMotionBlur/composite.ts b/packages/core/src/runtime/captureMotionBlur/composite.ts new file mode 100644 index 00000000000..81532fad777 --- /dev/null +++ b/packages/core/src/runtime/captureMotionBlur/composite.ts @@ -0,0 +1,135 @@ +/** + * The per-frame draw: a background, then each mover's own pixels redrawn along its path. + * + * Each mover is cropped from a capture of the scene and drawn at every tap pose into a float + * buffer, additive at weight 1/n: the shutter integral. Its coverage fades at the path ends, so + * the trail needs a background without the mover (a second capture, movers hidden); falling back + * to the sharp frame leaves thin lines looking unblurred. + * + * Image-space approximation: a mover's box contents move with it; ink outside it is not redrawn. + */ + +import { type Affine, invert, sweep, thinTaps } from "./pose"; + +export interface MoverFrame { + w: number; + h: number; + /** The pose the capture shows (frame time). */ + now: Affine; + /** Poses across the shutter. */ + taps: Affine[]; +} + +interface CompositeStats { + blurred: number; + taps: number; +} + +type Context2D = CanvasRenderingContext2D | OffscreenCanvasRenderingContext2D; +interface Layer { + canvas: OffscreenCanvas; + ctx: OffscreenCanvasRenderingContext2D; +} + +// Float16 keeps n additive 1/n-weighted taps exact; 8-bit loses up to ~n/2 levels per channel. +function makeLayer(w: number, h: number): Layer { + const canvas = new OffscreenCanvas(w, h); + let ctx: OffscreenCanvasRenderingContext2D | null = null; + try { + ctx = canvas.getContext("2d", { colorType: "float16" } as CanvasRenderingContext2DSettings); + } catch { + ctx = null; + } + ctx = ctx ?? canvas.getContext("2d"); + if (!ctx) throw new Error("capture motion blur: no 2D context for the accumulation buffer"); + return { canvas, ctx }; +} + +/** Whether a mover travels far enough within this shutter to draw a trail at all. */ +function moves(mover: MoverFrame, frameW: number, frameH: number): boolean { + const { travel, rect } = sweep(mover.taps, mover.w, mover.h, frameW, frameH); + return rect !== null && travel >= 1; +} + +export function createCompositor(width: number, height: number) { + const accum = makeLayer(width, height); + const crops: Layer[] = []; + // A mover box this many times the frame is too big to copy every frame; it is left sharp. + const maxCropArea = width * height * 4; + + const cropLayer = (i: number, w: number, h: number): Layer => { + let layer = crops[i]; + if (!layer || layer.canvas.width < w || layer.canvas.height < h) { + layer = makeLayer( + Math.max(w, layer?.canvas.width ?? 1), + Math.max(h, layer?.canvas.height ?? 1), + ); + crops[i] = layer; + } + return layer; + }; + + /** Copy each mover's frame-time pixels out of `src`; false marks a mover to leave sharp. */ + function crop(src: CanvasImageSource, movers: MoverFrame[]): boolean[] { + return movers.map((mover, i) => { + const { w, h } = mover; + const back = invert(mover.now); + if (!back || w * h > maxCropArea || !moves(mover, width, height)) return false; + const c = cropLayer(i, w, h).ctx; + c.setTransform(1, 0, 0, 1, 0, 0); + c.globalCompositeOperation = "source-over"; + c.globalAlpha = 1; + c.clearRect(0, 0, w, h); + c.setTransform(back.a, back.b, back.c, back.d, back.e, back.f); + c.drawImage(src, 0, 0); + return true; + }); + } + + function trail(out: Context2D, mover: MoverFrame, layer: Layer): number { + const { w, h } = mover; + const { travel, rect } = sweep(mover.taps, w, h, width, height); + if (!rect) return 0; + const taps = thinTaps(mover.taps, travel); + const a = accum.ctx; + a.setTransform(1, 0, 0, 1, 0, 0); + a.globalCompositeOperation = "source-over"; + a.globalAlpha = 1; + a.clearRect(rect.x, rect.y, rect.w, rect.h); + a.globalCompositeOperation = "lighter"; + a.globalAlpha = 1 / taps.length; + for (const m of taps) { + a.setTransform(m.a, m.b, m.c, m.d, m.e, m.f); + a.drawImage(layer.canvas, 0, 0, w, h, 0, 0, w, h); + } + out.drawImage(accum.canvas, rect.x, rect.y, rect.w, rect.h, rect.x, rect.y, rect.w, rect.h); + return taps.length; + } + + /** `background` into `out`, then the trail of every mover `crop` kept. */ + function draw( + background: CanvasImageSource, + out: Context2D, + movers: MoverFrame[], + kept: boolean[], + ): CompositeStats { + out.setTransform(1, 0, 0, 1, 0, 0); + out.globalAlpha = 1; + out.globalCompositeOperation = "copy"; + out.drawImage(background, 0, 0); + out.globalCompositeOperation = "source-over"; + const stats: CompositeStats = { blurred: 0, taps: 0 }; + movers.forEach((mover, i) => { + const layer = crops[i]; + if (!kept[i] || !layer) return; + const n = trail(out, mover, layer); + if (n > 0) { + stats.blurred++; + stats.taps += n; + } + }); + return stats; + } + + return { crop, draw }; +} diff --git a/packages/core/src/runtime/captureMotionBlur/discover.test.ts b/packages/core/src/runtime/captureMotionBlur/discover.test.ts new file mode 100644 index 00000000000..0f0aac55a3f --- /dev/null +++ b/packages/core/src/runtime/captureMotionBlur/discover.test.ts @@ -0,0 +1,89 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { discoverMovers } from "./discover"; + +// jsdom lays nothing out: each element's rect follows the x its fake timeline writes. +function element(parent: Element, id: string): HTMLElement { + const el = document.createElement("div"); + el.id = id; + parent.append(el); + el.getBoundingClientRect = () => { + const x = parseFloat(el.style.transform.replace("translateX(", "")) || 0; + return { left: x, top: 0, right: x + 10, bottom: 10, width: 10, height: 10 } as DOMRect; + }; + return el; +} + +function setup() { + const scene = document.createElement("div"); + const fast = element(scene, "fast"); + const fastChild = element(fast, "fast-child"); + const slow = element(scene, "slow"); + const still = element(scene, "still"); + const off = element(scene, "off"); + off.setAttribute("data-motion-blur", "off"); + // 1 s at 30 fps: `fast` and `off` cross 300 px (10 px/frame), `slow` 15 px (0.5 px/frame). + const seek = (t: number): void => { + fast.style.transform = `translateX(${300 * t}px)`; + fastChild.style.transform = `translateX(${200 * t}px)`; + slow.style.transform = `translateX(${15 * t}px)`; + off.style.transform = `translateX(${300 * t}px)`; + }; + seek(0); + return { scene, fast, fastChild, slow, still, off, seek }; +} + +describe("discoverMovers", () => { + afterEach(() => vi.restoreAllMocks()); + + it("keeps fast, outermost movers that are not opted out", () => { + const { scene, fast, seek } = setup(); + const { movers } = discoverMovers({ scene, durationSeconds: 1, fps: 30, seek }); + expect(movers).toEqual([fast]); + }); + + it("marks every changing element dynamic, slow and opted-out ones included", () => { + const { scene, fast, fastChild, slow, still, off, seek } = setup(); + const { dynamic } = discoverMovers({ scene, durationSeconds: 1, fps: 30, seek }); + expect( + dynamic.has(fast) && dynamic.has(fastChild) && dynamic.has(slow) && dynamic.has(off), + ).toBe(true); + expect(dynamic.has(still)).toBe(false); + }); + + it("ignores a CSS animation of offset-distance, which the pose has no term for", () => { + const scene = document.createElement("div"); + const box = document.createElement("div"); + scene.append(box); + let x = 0; // moves with no inline style change: only getAnimations can nominate it + box.getBoundingClientRect = () => + ({ left: x, right: x + 10, top: 0, bottom: 10, width: 10 }) as DOMRect; + const scan = (prop: string) => { + const effect = { target: box, getKeyframes: () => [{ [prop]: "0" }] }; + Object.defineProperty(document, "getAnimations", { + value: () => [{ effect }], + configurable: true, + }); + const seek = (t: number): void => void (x = 300 * t); + return discoverMovers({ scene, durationSeconds: 1, fps: 30, seek }).movers; + }; + expect(scan("offsetDistance")).toEqual([]); + expect(scan("translate")).toEqual([box]); + Reflect.deleteProperty(document, "getAnimations"); + }); + + it("drops a mover in a 3D context, or on an offset-path the pose cannot follow", () => { + const { scene, fast, seek } = setup(); + const real = window.getComputedStyle; + let own: object = {}; + vi.spyOn(window, "getComputedStyle").mockImplementation((el: Element) => + el === fast + ? ({ transform: "none", transformStyle: "flat", perspective: "none", ...own } as never) + : real(el), + ); + for (own of [ + { transform: "matrix3d(1,0,0,0,0,1,0,0,0,0,1,0,0,0,0,1)" }, + { offsetPath: "path('M 0 0')" }, + ]) + expect(discoverMovers({ scene, durationSeconds: 1, fps: 30, seek }).movers).toEqual([]); + }); +}); diff --git a/packages/core/src/runtime/captureMotionBlur/discover.ts b/packages/core/src/runtime/captureMotionBlur/discover.ts new file mode 100644 index 00000000000..18ac0cc84f5 --- /dev/null +++ b/packages/core/src/runtime/captureMotionBlur/discover.ts @@ -0,0 +1,124 @@ +/** + * Which elements to blur: the ones that move fast enough for a smear to show. + * + * One scan: candidates are elements whose inline style changes or that a CSS / WAAPI animation + * moves; they must reach MIN_SPEED px/frame outside any 3D context (the redraw is 2D), and only + * the outermost of nested candidates is kept. + */ + +import { isHtmlElement } from "../domRealm"; + +/** Under this peak speed (px per frame) a smear is invisible, so the element stays sharp. */ +const MIN_SPEED = 2; + +/** Scan resolution: at most this many timeline samples, however long the composition. */ +const SCAN_STEPS = 240; + +const ANIMATED_POSE_PROPS = new Set([ + "transform", + "translate", + "rotate", + "scale", + "left", + "top", + "right", + "bottom", +]); + +export interface DiscoverOptions { + scene: HTMLElement; + durationSeconds: number; + fps: number; + /** Seek the composition (timeline + adapters) to a time. */ + seek: (t: number) => void; +} + +export interface Discovery { + movers: HTMLElement[]; + /** Every element whose pose changes at all; their transforms are never cached. */ + dynamic: Set; +} + +// The whole inline style in one read; opacity or colour tweens only cost a rect check. +const styleKey = (el: HTMLElement): string => el.getAttribute("style") ?? ""; + +function optedOut(el: Element, scene: Element): boolean { + for (let n: Element | null = el; n && n !== scene; n = n.parentElement) { + if (n.getAttribute("data-motion-blur") === "off") return true; + } + return false; +} + +function in3d(el: Element, scene: Element): boolean { + for (let n: Element | null = el; n && n !== scene; n = n.parentElement) { + const cs = getComputedStyle(n); + if ( + cs.transform.startsWith("matrix3d") || + cs.transformStyle === "preserve-3d" || + cs.perspective !== "none" || + (cs.offsetPath ?? "none") !== "none" + ) { + return true; + } + } + return false; +} + +/** Elements a CSS animation or WAAPI animation moves (no inline style to diff). */ +function pageAnimated(scene: Element): Set { + const out = new Set(); + if (typeof document.getAnimations !== "function") return out; + for (const animation of document.getAnimations()) { + const effect = animation.effect as KeyframeEffect | null; + const target = effect?.target; + if (!isHtmlElement(target) || !scene.contains(target)) continue; + const frames = typeof effect?.getKeyframes === "function" ? effect.getKeyframes() : []; + if (frames.some((frame) => Object.keys(frame).some((k) => ANIMATED_POSE_PROPS.has(k)))) + out.add(target); + } + return out; +} + +/** How far a box's farthest-moving edge travelled between two reads; 0 when either is empty. */ +function edgeShift(a: DOMRect | undefined, b: DOMRect | undefined): number { + if (!a || !b || !a.width || !b.width) return 0; + return Math.max( + Math.abs(a.left - b.left), + Math.abs(a.top - b.top), + Math.abs(a.right - b.right), + Math.abs(a.bottom - b.bottom), + ); +} + +export function discoverMovers({ scene, durationSeconds, fps, seek }: DiscoverOptions): Discovery { + const all = [...scene.querySelectorAll("*")].filter(isHtmlElement); + const dynamic = new Set(pageAnimated(scene)); + const first = all.map(styleKey); + const frames = Math.max(1, Math.round(durationSeconds * fps)); + // One pass, about every other frame on a typical composition. A whip shorter than one step + // can be under-measured; ponytail: raise SCAN_STEPS if a fast move goes unblurred. + const steps = Math.min(frames, SCAN_STEPS); + const framesPerStep = frames / steps; + const fast = new Set(); + const threeD = new Set(); + let prev: Array | null = null; + + for (let i = 0; i <= steps; i++) { + seek((i / steps) * durationSeconds); + // Rects only for elements whose style has changed; the first read flushes layout once. + const rects: Array = []; + all.forEach((el, j) => { + if (!dynamic.has(el) && styleKey(el) === first[j]) return; + dynamic.add(el); + rects[j] = el.getBoundingClientRect(); + if (edgeShift(prev?.[j], rects[j]) / framesPerStep >= MIN_SPEED) fast.add(el); + }); + // Style reads up the chain are the costly part; every 10th step is enough to see a 3D turn. + if (i % 10 === 0 || i === steps) for (const el of fast) if (in3d(el, scene)) threeD.add(el); + prev = rects; + } + + const candidates = all.filter((el) => fast.has(el) && !threeD.has(el) && !optedOut(el, scene)); + const movers = candidates.filter((el) => !candidates.some((o) => o !== el && o.contains(el))); + return { movers, dynamic }; +} diff --git a/packages/core/src/runtime/captureMotionBlur/pose.test.ts b/packages/core/src/runtime/captureMotionBlur/pose.test.ts new file mode 100644 index 00000000000..42c8967108d --- /dev/null +++ b/packages/core/src/runtime/captureMotionBlur/pose.test.ts @@ -0,0 +1,230 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { + type Affine, + IDENTITY, + corners, + createPoseReader, + individualIs3d, + individualTransform, + interpolate, + invert, + multiply, + parseTransform, + sweep, + tapsFromReads, + thinTaps, + translate, +} from "./pose"; + +const rot = (deg: number, e = 0, f = 0): Affine => { + const r = (deg * Math.PI) / 180; + return { a: Math.cos(r), b: Math.sin(r), c: -Math.sin(r), d: Math.cos(r), e, f }; +}; +const scaleOf = (m: Affine): number => Math.hypot(m.a, m.b); +const angleOf = (m: Affine): number => (Math.atan2(m.b, m.a) * 180) / Math.PI; + +describe("pose math", () => { + it("interpolates a rotation without shrinking it", () => { + // Entry-wise lerp halfway through a 90° turn gives scale cos(45°) ≈ 0.707. + const mid = interpolate(rot(0), rot(90), 0.5); + expect(scaleOf(mid)).toBeCloseTo(1, 6); + expect(angleOf(mid)).toBeCloseTo(45, 6); + }); + + it("rotates the short way round", () => { + expect(angleOf(interpolate(rot(350), rot(10), 0.5))).toBeCloseTo(0, 6); + }); + + it("interpolates translation and scale linearly", () => { + const p: Affine = { a: 1, b: 0, c: 0, d: 1, e: 0, f: 0 }; + const q: Affine = { a: 3, b: 0, c: 0, d: 2, e: 100, f: -40 }; + expect(interpolate(p, q, 0.25)).toMatchObject({ a: 1.5, d: 1.25, e: 25, f: -10 }); + }); + + it("inverts an affine", () => { + const m = multiply(rot(30, 12, -7), { a: 2, b: 0, c: 0.5, d: 3, e: 4, f: 5 }); + const back = invert(m); + expect(back).not.toBeNull(); + const id = multiply(m, back as Affine); + for (const [k, v] of Object.entries(IDENTITY)) expect(id[k as keyof Affine]).toBeCloseTo(v, 9); + }); + + it("refuses to invert a collapsed box", () => { + expect(invert({ a: 0, b: 0, c: 0, d: 1, e: 0, f: 0 })).toBeNull(); + }); + + it("parses computed 2D and 3D transforms", () => { + const values = (css: string) => Object.values(parseTransform(css) ?? {}); + expect(values("matrix(1, 2, 3, 4, 5, 6)")).toEqual([1, 2, 3, 4, 5, 6]); + expect(values("matrix3d(1,0,0,0, 0,1,0,0, 0,0,1,0, 7,8,9,1)")).toEqual([1, 0, 0, 1, 7, 8]); + expect(parseTransform("none")).toBeNull(); + expect(parseTransform("matrix(1, 2)")).toBeNull(); + }); +}); + +describe("taps", () => { + it("spreads taps across the shutter, centred in each slice", () => { + const reads = [translate(IDENTITY, 0, 0), translate(IDENTITY, 100, 0)]; + const xs = tapsFromReads(reads, 4).map((m) => m.e); + expect(xs.map((x) => Math.round(x * 100) / 100)).toEqual([12.5, 37.5, 62.5, 87.5]); + }); + + it("measures the fastest corner's path and the swept rect", () => { + const taps = [ + translate(IDENTITY, 0, 0), + translate(IDENTITY, 30, 0), + translate(IDENTITY, 60, 0), + ]; + const { travel, rect } = sweep(taps, 10, 10, 1920, 1080); + expect(travel).toBeCloseTo(60, 6); + expect(rect).toEqual({ x: 0, y: 0, w: 72, h: 12 }); + }); + + it("reports an off-frame sweep as no rect", () => { + const taps = [translate(IDENTITY, -500, -500), translate(IDENTITY, -400, -500)]; + expect(sweep(taps, 10, 10, 1920, 1080).rect).toBeNull(); + }); + + it("thins taps to about one per 1.5 px, never below two", () => { + const taps = tapsFromReads([IDENTITY, translate(IDENTITY, 300, 0)], 32); + expect(thinTaps(taps, 3)).toHaveLength(2); + expect(thinTaps(taps, 15)).toHaveLength(10); + expect(thinTaps(taps, 300)).toHaveLength(32); + }); + + it("places a rotated box's corners", () => { + const [, tr] = corners(rot(90), 10, 0); + expect(tr?.[0]).toBeCloseTo(0, 9); + expect(tr?.[1]).toBeCloseTo(10, 9); + }); +}); + +const extra = new Map>(); + +describe("createPoseReader", () => { + afterEach(() => { + vi.restoreAllMocks(); + extra.clear(); + }); + + function place(el: HTMLElement, left: number, top: number, parent: Element | null): void { + Object.defineProperty(el, "offsetLeft", { value: left, configurable: true }); + Object.defineProperty(el, "offsetTop", { value: top, configurable: true }); + Object.defineProperty(el, "offsetParent", { value: parent, configurable: true }); + } + + function tree() { + const scene = document.createElement("div"); + const group = document.createElement("div"); + const mover = document.createElement("div"); + scene.append(group); + group.append(mover); + place(group, 100, 50, scene); + place(mover, 10, 20, group); + const transforms = new Map([ + [group, "matrix(2, 0, 0, 2, 0, 0)"], + [mover, "none"], + ]); + vi.spyOn(window, "getComputedStyle").mockImplementation( + (el: Element) => + ({ + transform: transforms.get(el) ?? "none", + transformOrigin: "0px 0px", + ...extra.get(el), + }) as CSSStyleDeclaration, + ); + return { scene, group, mover, transforms }; + } + + it("composes layout offsets with every transform up the chain", () => { + const { scene, mover } = tree(); + const reader = createPoseReader(scene, new Set([mover])); + // group at (100,50) scaled 2×, mover at (10,20) inside it → (120, 90), scale 2. + expect(reader.pose(mover)).toEqual({ a: 2, b: 0, c: 0, d: 2, e: 120, f: 90 }); + }); + + it("re-reads a dynamic node's transform after reset, but caches a static one", () => { + const { scene, group, mover, transforms } = tree(); + const reader = createPoseReader(scene, new Set([mover])); + reader.pose(mover); + transforms.set(mover, "matrix(1, 0, 0, 1, 5, 0)"); + transforms.set(group, "matrix(1, 0, 0, 1, 0, 0)"); + reader.reset(); + // The mover's new 5 px shift lands (in the still-cached 2× group), the group's change does not. + expect(reader.pose(mover)).toMatchObject({ a: 2, e: 130, f: 90 }); + }); + + it("puts them into a pose, before the transform property and about its origin", () => { + const { scene, group, mover } = tree(); + extra.set(group, { scale: "2" }); + extra.set(mover, { translate: "-50% -50%" }); + Object.defineProperty(mover, "offsetWidth", { value: 20, configurable: true }); + Object.defineProperty(mover, "offsetHeight", { value: 40, configurable: true }); + const reader = createPoseReader(scene, new Set()); + // group: layout (100,50), scale 2 from the origin, then its transform scale 2 → 4×. + // mover: layout (10,20) in the group, shifted (-10,-20): at the group's origin. + expect(reader.pose(mover)).toEqual({ a: 4, b: 0, c: 0, d: 4, e: 100, f: 50 }); + }); + + it.each([ + ["translate", "120px 0px", "matrix(1, 0, 0, 1, 120, 0)"], + ["translate", "180px 0px", "matrix(1, 0, 0, 1, 180, 0)"], + ["rotate", "90deg", "matrix(0, 1, -1, 0, 0, 0)"], + ["scale", "2 3", "matrix(2, 0, 0, 3, 0, 0)"], + ])("poses `%s: %s` like the equivalent matrix transform", (prop, value, matrix) => { + const { scene, mover, transforms } = tree(); + const reader = createPoseReader(scene, new Set([mover])); + transforms.set(mover, matrix); + const control = reader.pose(mover); + transforms.set(mover, "none"); + extra.set(mover, { [prop]: value }); + reader.reset(); + const pose = reader.pose(mover); + for (const k of ["a", "b", "c", "d", "e", "f"] as const) + expect(pose[k]).toBeCloseTo(control[k], 9); + }); + + it("measures from the offsetParent's border and subtracts scroll", () => { + const { scene, group, mover } = tree(); + Object.defineProperty(group, "clientLeft", { value: 5, configurable: true }); + Object.defineProperty(group, "clientTop", { value: 3, configurable: true }); + Object.defineProperty(group, "scrollTop", { value: 7, configurable: true }); + const reader = createPoseReader(scene, new Set()); + // group's 2× transform scales mover's in-group offset (10 + 5, 20 + 3 - 7) → (30, 32). + expect(reader.pose(mover)).toMatchObject({ e: 130, f: 82 }); + }); +}); + +describe("individual transform properties", () => { + it("centres a box with translate percentages of its own size", () => { + expect(individualTransform({ translate: "-50% -50%" }, 100, 60)).toEqual({ + ...IDENTITY, + e: -50, + f: -30, + }); + expect(individualTransform({ translate: "calc(50% + 10px) 5px" }, 100, 60)).toMatchObject({ + e: 60, + f: 5, + }); + }); + + it("applies translate, then rotate, then scale", () => { + const m = individualTransform({ translate: "10px 0px", rotate: "90deg", scale: "2" }, 10, 10); + // A point (1, 0): scaled to (2, 0), rotated to (0, 2), shifted to (10, 2). + expect(m && [m.a + m.e, m.b + m.f]).toEqual([expect.closeTo(10, 9), expect.closeTo(2, 9)]); + }); + + it("is null when none is set", () => { + expect(individualTransform({ translate: "none", rotate: "none", scale: "none" }, 1, 1)).toBe( + null, + ); + expect(individualTransform({}, 1, 1)).toBe(null); + }); + + it("flags a z translate or a rotate axis as 3D", () => { + expect(individualIs3d({ translate: "10px 20px 5px" })).toBe(true); + expect(individualIs3d({ rotate: "x 30deg" })).toBe(true); + expect(individualIs3d({ rotate: "1 1 0 57.2958deg" })).toBe(true); + expect(individualIs3d({ translate: "10px 20px", rotate: "30deg", scale: "2" })).toBe(false); + }); +}); diff --git a/packages/core/src/runtime/captureMotionBlur/pose.ts b/packages/core/src/runtime/captureMotionBlur/pose.ts new file mode 100644 index 00000000000..84956767916 --- /dev/null +++ b/packages/core/src/runtime/captureMotionBlur/pose.ts @@ -0,0 +1,337 @@ +/** + * Pose math for capture motion blur: where an element's box sits in composition space at a + * given timeline time, and the in-between poses the shutter sweeps through. + * + * A pose is the full 2D affine from the element's own border box to the composition root: + * layout offsets plus every CSS transform up the ancestor chain. Reading it costs style and + * layout reads only; no paint. + */ + +export interface Affine { + a: number; + b: number; + c: number; + d: number; + e: number; + f: number; +} + +export const IDENTITY: Affine = { a: 1, b: 0, c: 0, d: 1, e: 0, f: 0 }; + +export function multiply(m: Affine, n: Affine): Affine { + return { + a: m.a * n.a + m.c * n.b, + b: m.b * n.a + m.d * n.b, + c: m.a * n.c + m.c * n.d, + d: m.b * n.c + m.d * n.d, + e: m.a * n.e + m.c * n.f + m.e, + f: m.b * n.e + m.d * n.f + m.f, + }; +} + +export function translate(m: Affine, x: number, y: number): Affine { + return multiply(m, { a: 1, b: 0, c: 0, d: 1, e: x, f: y }); +} + +export function invert(m: Affine): Affine | null { + const det = m.a * m.d - m.b * m.c; + if (!Number.isFinite(det) || Math.abs(det) < 1e-12) return null; + return { + a: m.d / det, + b: -m.b / det, + c: -m.c / det, + d: m.a / det, + e: (m.c * m.f - m.d * m.e) / det, + f: (m.b * m.e - m.a * m.f) / det, + }; +} + +/** The four corners of a w×h box under `m`. */ +export function corners(m: Affine, w: number, h: number): Array<[number, number]> { + return [ + [m.e, m.f], + [m.a * w + m.e, m.b * w + m.f], + [m.c * h + m.e, m.d * h + m.f], + [m.a * w + m.c * h + m.e, m.b * w + m.d * h + m.f], + ]; +} + +/** + * Interpolate two poses by translation, rotation, scale and shear rather than matrix entries: + * entry-wise lerp of a rotating box shrinks it mid-segment (halfway through a 90° turn it is + * scaled by cos 45°). Rotation takes the short way round. + */ +export function interpolate(p: Affine, q: Affine, f: number): Affine { + const dp = decompose(p); + const dq = decompose(q); + let turn = dq.angle - dp.angle; + if (turn > Math.PI) turn -= 2 * Math.PI; + else if (turn < -Math.PI) turn += 2 * Math.PI; + const lerp = (x: number, y: number): number => x + (y - x) * f; + return compose({ + angle: dp.angle + turn * f, + sx: lerp(dp.sx, dq.sx), + sy: lerp(dp.sy, dq.sy), + shear: lerp(dp.shear, dq.shear), + e: lerp(p.e, q.e), + f: lerp(p.f, q.f), + }); +} + +interface Decomposed { + angle: number; + sx: number; + sy: number; + shear: number; + e: number; + f: number; +} + +// QR split of the linear part: first column = sx·(cos, sin), second = shear·(cos, sin) + sy·(−sin, cos). +function decompose(m: Affine): Decomposed { + const sx = Math.hypot(m.a, m.b); + if (sx < 1e-12) return { angle: 0, sx: 0, sy: m.d, shear: m.c, e: m.e, f: m.f }; + return { + angle: Math.atan2(m.b, m.a), + sx, + sy: (m.a * m.d - m.b * m.c) / sx, + shear: (m.a * m.c + m.b * m.d) / sx, + e: m.e, + f: m.f, + }; +} + +function compose(d: Decomposed): Affine { + const cos = Math.cos(d.angle); + const sin = Math.sin(d.angle); + return { + a: d.sx * cos, + b: d.sx * sin, + c: d.shear * cos - d.sy * sin, + d: d.shear * sin + d.sy * cos, + e: d.e, + f: d.f, + }; +} + +/** + * `samples` taps spread evenly across the shutter, interpolated from `reads` poses read at + * evenly spaced times from the start to the end of the shutter (endpoints included). + */ +export function tapsFromReads(reads: Affine[], samples: number): Affine[] { + if (reads.length === 0) return []; + if (reads.length === 1) return Array.from({ length: samples }, () => reads[0] as Affine); + const last = reads.length - 1; + return Array.from({ length: samples }, (_, k) => { + const x = ((k + 0.5) / samples) * last; + const j = Math.min(last - 1, Math.floor(x)); + return interpolate(reads[j] as Affine, reads[j + 1] as Affine, x - j); + }); +} + +export interface Sweep { + /** Path length of the box's fastest corner across the taps, in pixels. */ + travel: number; + /** Pixel rect covering every tap, clipped to the frame; null when off-frame. */ + rect: { x: number; y: number; w: number; h: number } | null; +} + +export function sweep(taps: Affine[], w: number, h: number, frameW: number, frameH: number): Sweep { + let x0 = Infinity; + let y0 = Infinity; + let x1 = -Infinity; + let y1 = -Infinity; + let travel = 0; + let prev: Array<[number, number]> | null = null; + for (const m of taps) { + const cs = corners(m, w, h); + for (const [x, y] of cs) { + x0 = Math.min(x0, x); + y0 = Math.min(y0, y); + x1 = Math.max(x1, x); + y1 = Math.max(y1, y); + } + if (prev) { + let step = 0; + for (let j = 0; j < 4; j++) { + const [px, py] = prev[j] as [number, number]; + const [cx, cy] = cs[j] as [number, number]; + step = Math.max(step, Math.hypot(cx - px, cy - py)); + } + travel += step; + } + prev = cs; + } + // 2 px of slack for bilinear filtering at the edges. + const rx0 = Math.max(0, Math.floor(x0) - 2); + const ry0 = Math.max(0, Math.floor(y0) - 2); + const rx1 = Math.min(frameW, Math.ceil(x1) + 2); + const ry1 = Math.min(frameH, Math.ceil(y1) + 2); + const rect = rx1 > rx0 && ry1 > ry0 ? { x: rx0, y: ry0, w: rx1 - rx0, h: ry1 - ry0 } : null; + return { travel, rect }; +} + +/** About one tap per `spacing` px of travel (no visible steps), at least 2, at most all. */ +export function thinTaps(taps: Affine[], travel: number, spacing = 1.5): Affine[] { + const n = Math.min(taps.length, Math.max(2, Math.ceil(travel / spacing))); + if (n >= taps.length) return taps; + return Array.from({ length: n }, (_, i) => { + const k = Math.round(((i + 0.5) / n) * taps.length - 0.5); + return taps[k] as Affine; + }); +} + +/** + * Reads poses relative to `scene`, memoised for one timeline time: movers share ancestors, so + * each node's matrix is computed once from its parent's. Call `reset()` after every seek. + * + * `dynamic` names the nodes whose transform can change over time; everyone else's own + * transform is read once and cached for the reader's lifetime. + */ +export function createPoseReader(scene: HTMLElement, dynamic: ReadonlySet) { + const staticOwn = new Map(); + let positions = new Map(); + let matrices = new Map(); + + const ownTransform = (n: HTMLElement): Affine | null => { + const cached = !dynamic.has(n) && staticOwn.has(n); + if (cached) return staticOwn.get(n) ?? null; + const cs = getComputedStyle(n); + const t = multiplyAll([ + individualTransform(cs, n.offsetWidth, n.offsetHeight), + parseTransform(cs.transform), + ]); + let own: Affine | null = null; + if (t) { + const [ox = 0, oy = 0] = cs.transformOrigin.split(" ").map((v) => parseFloat(v) || 0); + own = translate(multiply(translate(IDENTITY, ox, oy), t), -ox, -oy); + } + if (!dynamic.has(n)) staticOwn.set(n, own); + return own; + }; + + // Untransformed border-box position relative to the scene, along the offsetParent chain. + // offsetLeft/Top measure from the offsetParent's padding edge, so its border is added back. + const layoutPos = (n: Element | null): [number, number] => { + if (!n || n === scene || !isOffsetElement(n)) return [0, 0]; + let p = positions.get(n); + if (!p) { + const op = n.offsetParent; + const [px, py] = layoutPos(op); + p = [px + n.offsetLeft + (op?.clientLeft ?? 0), py + n.offsetTop + (op?.clientTop ?? 0)]; + positions.set(n, p); + } + return p; + }; + + const matrixOf = (n: Element | null): Affine => { + if (!n || n === scene || !isOffsetElement(n)) return IDENTITY; + let m = matrices.get(n); + if (!m) { + const parent = n.parentElement; + const [x, y] = layoutPos(n); + const [px, py] = layoutPos(parent); + const [sx, sy] = scrollOf(parent); + m = translate(matrixOf(parent), x - px - sx, y - py - sy); + const own = ownTransform(n); + if (own) m = multiply(m, own); + matrices.set(n, m); + } + return m; + }; + + return { + pose: matrixOf as (el: HTMLElement) => Affine, + reset(): void { + positions = new Map(); + matrices = new Map(); + }, + }; +} + +function scrollOf(el: Element | null): [number, number] { + return el ? [el.scrollLeft, el.scrollTop] : [0, 0]; +} + +function multiplyAll(ms: Array): Affine | null { + let out: Affine | null = null; + for (const m of ms) if (m) out = out ? multiply(out, m) : m; + return out; +} + +type IndividualProps = Pick; + +// Splits at spaces outside parentheses (`calc(50% + 10px) 5%` is two tokens). +const tokens = (value: string): string[] => value.trim().split(/\s+(?![^(]*\))/); + +// A computed length or percentage (`calc(50% + 10px)` included) in px against `basis`. +function lengthPx(token: string, basis: number): number { + let sum = 0; + for (const m of token.matchAll(/([+-]?\s*[\d.]+(?:e[+-]?\d+)?)(px|%)?/g)) { + const v = Number(m[1]?.replace(/\s/g, "")); + sum += m[2] === "%" ? (v / 100) * basis : v; + } + return sum; +} + +const ANGLE_UNIT: Record = { + deg: Math.PI / 180, + rad: 1, + grad: Math.PI / 200, + turn: 2 * Math.PI, +}; + +/** + * The `translate`, `rotate` and `scale` properties as one 2D affine, in the CSS order (translate, + * then rotate, then scale; `transform` comes after). Null when none is set. + */ +export function individualTransform( + cs: Partial, + width: number, + height: number, +): Affine | null { + const t = cs.translate ?? "none"; + const r = cs.rotate ?? "none"; + const s = cs.scale ?? "none"; + const parts: Array = []; + if (t !== "none") { + const [x = "0", y = "0"] = tokens(t); + parts.push({ ...IDENTITY, e: lengthPx(x, width), f: lengthPx(y, height) }); + } + const turn = /^(-?[\d.]+)(deg|rad|grad|turn)$/.exec(r); + if (turn) { + const a = Number(turn[1]) * (ANGLE_UNIT[turn[2] ?? "deg"] ?? 0); + parts.push({ a: Math.cos(a), b: Math.sin(a), c: -Math.sin(a), d: Math.cos(a), e: 0, f: 0 }); + } + if (s !== "none") { + const [sx = 1, sy = sx] = tokens(s).map(Number); + parts.push({ ...IDENTITY, a: sx, d: sy }); + } + return multiplyAll(parts); +} + +/** Whether `translate` has a z part or `rotate` an axis: a 3D move the 2D pose cannot track. */ +export function individualIs3d(cs: Partial): boolean { + const t = cs.translate ?? "none"; + const r = cs.rotate ?? "none"; + return (t !== "none" && tokens(t).length > 2) || !/^(none|-?[\d.]+[a-z]+)$/.test(r); +} + +/** + * A computed `transform` (always `none`, `matrix(...)` or `matrix3d(...)` once resolved) as its + * 2D affine part; null for `none` or anything unparsable. + */ +export function parseTransform(value: string): Affine | null { + const match = /^matrix(3d)?\(([^)]*)\)$/.exec(value.trim()); + if (!match) return null; + const v = (match[2] ?? "").split(",").map(Number); + if (v.some((x) => !Number.isFinite(x))) return null; + const pick = match[1] ? [0, 1, 4, 5, 12, 13] : [0, 1, 2, 3, 4, 5]; + if (v.length !== (match[1] ? 16 : 6)) return null; + const [a = 1, b = 0, c = 0, d = 1, e = 0, f = 0] = pick.map((i) => v[i] ?? 0); + return { a, b, c, d, e, f }; +} + +function isOffsetElement(n: Element): n is HTMLElement { + return typeof (n as HTMLElement).offsetLeft === "number"; +} From 254e20a5d84aafbb29dea1e9744694aa1b258326 Mon Sep 17 00:00:00 2001 From: Vance Ingalls Date: Tue, 20 Oct 2026 17:44:10 -0700 Subject: [PATCH 2/2] feat(core): capture motion blur runtime lifecycle Wire the pure modules into the composition runtime. A root that opts in with data-motion-blur wraps itself in a capture canvas and, per frame, redraws its fast movers along their paths into an output canvas. The runtime turns itself off, and says why, when the browser lacks html-in-canvas, when a WebGL or nested capture canvas is present, or when the engine's own sub-frame blur is on. It unwraps when paused so Studio can select inside. A frame whose paint never comes within 2000 ms is rendered as plain DOM (root back in the page, movers revealed), counted in report().failedFrames and never counted toward giving up. The page-side off switch, PAGE_MOTION_BLUR_OFF_GLOBAL, is exported from @hyperframes/core for the engine to set. Co-Authored-By: Claude Sonnet 5.5 --- packages/core/src/index.ts | 1 + packages/core/src/pageMotionBlur.ts | 12 + .../runtime/captureMotionBlur/index.test.ts | 311 +++++++++ .../src/runtime/captureMotionBlur/index.ts | 605 ++++++++++++++++++ packages/core/src/runtime/entry.test.ts | 17 + packages/core/src/runtime/entry.ts | 2 + packages/core/src/runtime/init.ts | 45 +- 7 files changed, 991 insertions(+), 2 deletions(-) create mode 100644 packages/core/src/pageMotionBlur.ts create mode 100644 packages/core/src/runtime/captureMotionBlur/index.test.ts create mode 100644 packages/core/src/runtime/captureMotionBlur/index.ts diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 50c3c5bcdf0..51514f023ed 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -437,3 +437,4 @@ export { type HfVfxParam, type HfVfxParamValues, } from "./vfx.js"; +export { PAGE_MOTION_BLUR_OFF_GLOBAL } from "./pageMotionBlur.js"; diff --git a/packages/core/src/pageMotionBlur.ts b/packages/core/src/pageMotionBlur.ts new file mode 100644 index 00000000000..d540d046cf0 --- /dev/null +++ b/packages/core/src/pageMotionBlur.ts @@ -0,0 +1,12 @@ +// Set by the engine: its own sub-frame blur runs, or the Studio preview could not blur. +export const PAGE_MOTION_BLUR_OFF_GLOBAL = "__HF_PAGE_MOTION_BLUR_OFF__"; + +export function supportsCaptureMotionBlur(win: Window = window): boolean { + const w = win as Window & { + CanvasRenderingContext2D?: { prototype: object }; + HTMLCanvasElement?: { prototype: object }; + }; + const ctx = w.CanvasRenderingContext2D?.prototype; + const canvas = w.HTMLCanvasElement?.prototype; + return !!ctx && !!canvas && "drawElementImage" in ctx && "requestPaint" in canvas; +} diff --git a/packages/core/src/runtime/captureMotionBlur/index.test.ts b/packages/core/src/runtime/captureMotionBlur/index.test.ts new file mode 100644 index 00000000000..ccd7a8c55df --- /dev/null +++ b/packages/core/src/runtime/captureMotionBlur/index.test.ts @@ -0,0 +1,311 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { + type CaptureMotionBlurHost, + initCaptureMotionBlur, + paintCaptureMotionBlur, + releaseCaptureMotionBlurFrame, + resetCaptureMotionBlurForTests, + trackAcceleratedCanvases, +} from "."; + +type Win = Window & { + __HF_PAGE_MOTION_BLUR_OFF__?: unknown; + __hfMotionBlur?: { + report: () => { + state: string; + reason: string | null; + shutterDeg: number; + moverNames: string[]; + }; + }; + __hf_page_composite_pending?: boolean; + __hf_page_composite_resolve?: () => boolean; +}; +const win = window as Win; + +// Every 2D context (page canvas or offscreen) is this recording fake. +function fakeContext() { + return { + reset: vi.fn(), + drawElementImage: vi.fn(), + drawImage: vi.fn(), + clearRect: vi.fn(), + setTransform: vi.fn(), + globalAlpha: 1, + globalCompositeOperation: "source-over", + }; +} +let contexts: Array> = []; + +function installHtmlInCanvas(): void { + vi.spyOn(HTMLCanvasElement.prototype, "getContext").mockImplementation(() => { + const ctx = fakeContext(); + contexts.push(ctx); + return ctx as never; + }); + vi.stubGlobal( + "CanvasRenderingContext2D", + class { + drawElementImage(): void {} + }, + ); + (HTMLCanvasElement.prototype as unknown as Record).requestPaint = () => {}; + vi.stubGlobal( + "OffscreenCanvas", + class { + width: number; + height: number; + constructor(w: number, h: number) { + this.width = w; + this.height = h; + } + getContext() { + return fakeContext(); + } + }, + ); +} + +function makeRoot(attrs: Record = { "data-motion-blur": "on" }): HTMLElement { + const root = document.createElement("div"); + root.setAttribute("data-composition-id", "main"); + root.setAttribute("data-width", "640"); + root.setAttribute("data-height", "360"); + for (const [k, v] of Object.entries(attrs)) root.setAttribute(k, v); + document.body.append(root); + return root; +} + +function host( + root: HTMLElement, + seek = vi.fn(), +): CaptureMotionBlurHost & { seek: ReturnType } { + return { root, fps: 30, durationSeconds: () => 1, timeline: () => "tl", seek }; +} + +const captureCanvas = (root: HTMLElement) => root.parentElement as HTMLCanvasElement; + +beforeEach(() => { + contexts = []; + vi.spyOn(console, "info").mockImplementation(() => {}); + vi.spyOn(console, "warn").mockImplementation(() => {}); +}); + +afterEach(() => { + resetCaptureMotionBlurForTests(); + vi.restoreAllMocks(); + vi.unstubAllGlobals(); + vi.useRealTimers(); + delete (HTMLCanvasElement.prototype as unknown as Record).requestPaint; + delete win.__HF_PAGE_MOTION_BLUR_OFF__; + delete win.__hfMotionBlur; + delete win.__hf_page_composite_pending; + delete win.__hf_page_composite_resolve; + document.body.innerHTML = ""; +}); + +/** A root with one 10×10 element that slides 300 px over the 1 s composition (10 px/frame). */ +function movingHost(attrs?: Record) { + const root = makeRoot(attrs); + const el = document.createElement("div"); + el.id = "slider"; + root.append(el); + for (const [k, v] of Object.entries({ + offsetWidth: 10, + offsetHeight: 10, + offsetLeft: 0, + offsetTop: 0, + offsetParent: root, + })) { + Object.defineProperty(el, k, { value: v, configurable: true }); + } + const x = () => Number(/,\s*([-\d.]+),\s*0\)$/.exec(el.style.transform)?.[1] ?? 0); + el.getBoundingClientRect = () => + ({ left: x(), top: 0, right: x() + 10, bottom: 10, width: 10, height: 10 }) as DOMRect; + const seek = vi.fn((t: number) => { + el.style.transform = `matrix(1, 0, 0, 1, ${300 * t}, 0)`; + }); + return { root, el, h: host(root, seek) }; +} + +const PLAY = { playing: true }; + +describe("opt-in and off switches", () => { + it("does nothing for a root that has not opted in", () => { + installHtmlInCanvas(); + const root = makeRoot({}); + initCaptureMotionBlur(host(root)); + expect(root.parentElement).toBe(document.body); + expect(paintCaptureMotionBlur(0)).toBeNull(); + expect(win.__hfMotionBlur).toBeUndefined(); + }); + + it("stays off without html-in-canvas and says why", () => { + const root = makeRoot(); + initCaptureMotionBlur(host(root)); + expect(root.parentElement).toBe(document.body); + expect(win.__hfMotionBlur?.report()).toMatchObject({ + state: "off", + reason: expect.stringContaining("html-in-canvas"), + }); + }); + + it("stays off while the engine's own sub-frame blur is on (never both)", () => { + installHtmlInCanvas(); + win.__HF_PAGE_MOTION_BLUR_OFF__ = "engine-motion-blur"; + const { root, h } = movingHost(); + initCaptureMotionBlur(h); + expect(paintCaptureMotionBlur(0.5, PLAY)).toBeNull(); + expect(root.parentElement).toBe(document.body); + expect(win.__hfMotionBlur?.report().reason).toContain("engine"); + }); + + it("stays off for a WebGL canvas, which html-in-canvas captures frozen, but not for a 2D one", () => { + installHtmlInCanvas(); + trackAcceleratedCanvases(); + const root = makeRoot(); + const flat = document.createElement("canvas"); + root.append(flat); + flat.getContext("2d"); + initCaptureMotionBlur(host(root)); + expect(win.__hfMotionBlur?.report().state).toBe("on"); + + resetCaptureMotionBlurForTests(); + const gl = document.createElement("canvas"); + root.append(gl); + gl.getContext("webgl"); + initCaptureMotionBlur(host(root)); + expect(win.__hfMotionBlur?.report().reason).toMatch(/WebGL/); + }); + + it("turns off instead of wrapping when a WebGL canvas appears after init", () => { + installHtmlInCanvas(); + trackAcceleratedCanvases(); + const { root, h } = movingHost(); + initCaptureMotionBlur(h); + const late = document.createElement("canvas"); + root.append(late); + late.getContext("webgl2"); + void paintCaptureMotionBlur(0.5, { engineMode: true }); + expect(root.parentElement).toBe(document.body); + expect(win.__hfMotionBlur?.report().reason).toMatch(/WebGL/); + }); +}); + +describe("wrapping", () => { + it("wraps the root itself in a capture canvas, with an output canvas after it", () => { + installHtmlInCanvas(); + const { root, h } = movingHost(); + initCaptureMotionBlur(h); + expect(root.parentElement).toBe(document.body); // not until it plays + void paintCaptureMotionBlur(0.5, PLAY); + const src = captureCanvas(root); + expect(src.tagName).toBe("CANVAS"); + expect(src.hasAttribute("layoutsubtree")).toBe(true); + expect(src.hasAttribute("data-hf-ignore")).toBe(true); + expect([src.width, src.height]).toEqual([640, 360]); + const out = src.nextElementSibling as HTMLElement; + expect(out.getAttribute("data-hf-motion-blur")).toBe("output"); + // The output sits over the composition: clicks must reach what is under it. + expect(out.style.pointerEvents).toBe("none"); + }); + + it("turns off instead of wrapping when a capture canvas appears after init", () => { + installHtmlInCanvas(); + const { root, h } = movingHost(); + initCaptureMotionBlur(h); + const late = document.createElement("canvas"); + late.setAttribute("layoutsubtree", ""); + root.append(late); + void paintCaptureMotionBlur(0.5, PLAY); + expect(root.parentElement).toBe(document.body); + expect(win.__hfMotionBlur?.report().state).toBe("off"); + }); +}); + +describe("paint timeouts", () => { + const HIDE = "data-hf-motion-blur-hide"; + const info = () => vi.mocked(console.info); + const engineFrame = (t: number): Promise | null => { + const done = paintCaptureMotionBlur(t, { engineMode: true }); + releaseCaptureMotionBlurFrame(); + return done; + }; + const plainDom = (root: HTMLElement): void => { + expect(root.parentElement).toBe(document.body); + expect(document.querySelectorAll("canvas")).toHaveLength(0); + }; + + it("renders a released frame whose paint never comes plain, counts it, and leaves nothing for the engine to finish (first frame, output never drawn)", async () => { + vi.useFakeTimers(); + installHtmlInCanvas(); + const { root, el, h } = movingHost(); + initCaptureMotionBlur(h); + const done = engineFrame(0.5); + expect(captureCanvas(root).tagName).toBe("CANVAS"); + await vi.advanceTimersByTimeAsync(2001); + await done; + plainDom(root); + expect(el.hasAttribute(HIDE)).toBe(false); + expect(win.__hfMotionBlur?.report()).toMatchObject({ + state: "on", + failures: 0, + failedFrames: 1, + }); + expect(info()).toHaveBeenCalledWith( + expect.stringMatching(/capture failed \(1 so far\).*paint/), + ); + win.__hf_page_composite_resolve?.(); // the engine's forced composite arrives late + plainDom(root); + expect(contexts[0]?.drawElementImage).not.toHaveBeenCalled(); + expect(win.__hf_page_composite_pending).toBe(false); + void paintCaptureMotionBlur(0.6, { engineMode: true }); // the next frame wraps again + expect(captureCanvas(root).tagName).toBe("CANVAS"); + }); + + it("does the same for a frame after one that painted, whose output already holds an earlier frame", async () => { + vi.useFakeTimers(); + installHtmlInCanvas(); + const { root, el, h } = movingHost(); + initCaptureMotionBlur(h); + const first = engineFrame(0.5); + captureCanvas(root).dispatchEvent(new Event("paint")); + captureCanvas(root).dispatchEvent(new Event("paint")); + await first; + expect(win.__hfMotionBlur?.report()).toMatchObject({ failedFrames: 0, frames: 1 }); + const captures = contexts[0]?.drawElementImage.mock.calls.length; + const second = engineFrame(0.6); + await vi.advanceTimersByTimeAsync(2001); + await second; + plainDom(root); + expect(el.hasAttribute(HIDE)).toBe(false); + expect(win.__hfMotionBlur?.report()).toMatchObject({ failures: 0, failedFrames: 1 }); + win.__hf_page_composite_resolve?.(); + plainDom(root); + expect(contexts[0]?.drawElementImage.mock.calls.length).toBe(captures); + }); + + it("keeps blurring through preview paint timeouts, each frame plain, and resumes when painting returns", async () => { + vi.useFakeTimers(); + installHtmlInCanvas(); + const { root, h } = movingHost(); + initCaptureMotionBlur(h); + for (let i = 0; i < 4; i++) { + void paintCaptureMotionBlur(i / 30, PLAY); + expect(captureCanvas(root).tagName).toBe("CANVAS"); + await vi.advanceTimersByTimeAsync(2000); + plainDom(root); + } + expect(win.__hfMotionBlur?.report()).toMatchObject({ + state: "on", + failures: 0, + failedFrames: 4, + }); + expect(info()).toHaveBeenCalledTimes(1); + const done = paintCaptureMotionBlur(0.5, PLAY); + captureCanvas(root).dispatchEvent(new Event("paint")); + captureCanvas(root).dispatchEvent(new Event("paint")); + await done; + expect(contexts[0]?.drawElementImage).toHaveBeenCalled(); + }); +}); diff --git a/packages/core/src/runtime/captureMotionBlur/index.ts b/packages/core/src/runtime/captureMotionBlur/index.ts new file mode 100644 index 00000000000..9324a8f22c1 --- /dev/null +++ b/packages/core/src/runtime/captureMotionBlur/index.ts @@ -0,0 +1,605 @@ +/** + * Capture motion blur, in preview and render. Root attributes: data-motion-blur="on"|"off", + * -shutter (0–720°, default 180), -samples (2–64, default 16), -events="on" (run timeline + * callbacks at sub-frame samples). PAGE_MOTION_BLUR_OFF_GLOBAL (engine blur, or a preview that + * could not blur) turns all off. + * + * Per frame: read movers' poses across the shutter (seeks, no paint), capture the root and crop + * the movers, hide them and capture the background, then redraw each along its path. + * The root is wrapped, not its children (author selectors keep matching), and only while playing + * or rendering with something moving: paused, it is plain DOM so Studio can select inside it. + * Needs html-in-canvas (`drawElementImage`); without it nothing changes. + */ + +import { PAGE_MOTION_BLUR_OFF_GLOBAL, supportsCaptureMotionBlur } from "../../pageMotionBlur"; +import { swallow } from "../diagnostics"; +import { createCompositor, type MoverFrame } from "./composite"; +import { discoverMovers } from "./discover"; +import { type Affine, createPoseReader, IDENTITY, tapsFromReads } from "./pose"; + +export interface CaptureMotionBlurHost { + root: HTMLElement; + fps: number; + durationSeconds: () => number; + /** The bound root timeline; movers are re-scanned when it changes. */ + timeline: () => unknown; + /** Seek the timeline and adapters (not media) for a pose read. */ + seek: (t: number, suppressEvents: boolean) => void; +} + +interface Config { + shutterDeg: number; + samples: number; + events: boolean; +} + +interface Report { + state: "on" | "off"; + reason: string | null; + movers: number; + /** The blurred elements, as `tag#id.class`, for diagnosing what was (not) picked. */ + moverNames: string[]; + /** One-time cost of finding the movers, in ms. */ + scanMs: number; + shutterDeg: number; + frames: number; + meanMs: number; + failures: number; + failedFrames: number; +} + +type DrawElementContext = CanvasRenderingContext2D & { + drawElementImage: (el: Element, x: number, y: number) => unknown; +}; +type PaintCanvas = HTMLCanvasElement & { requestPaint: () => void }; +type CompositeWindow = Window & { + __hfMotionBlur?: { report: () => Report }; + __hf_page_composite_pending?: boolean; + __hf_page_composite_resolve?: () => boolean; +}; + +/** Marks a mover hidden for the background capture; set only between a frame's two paints. */ +const HIDE_ATTR = "data-hf-motion-blur-hide"; + +interface Job { + seq: number; + els: HTMLElement[]; + frames: MoverFrame[]; + /** Which movers were cropped (null until the first capture of this frame). */ + kept: boolean[] | null; + /** Engine frames wait for a release: video frames are injected after the seek. */ + armed: boolean; + /** The paint wait starts on arming, so slow video injection cannot time a frame out unseen. */ + startTimer: () => void; + ms: number; + settle: () => void; +} + +/** Pose reads per frame; the taps between them are interpolated. A render reads more. */ +const PREVIEW_READS = 4; +const RENDER_READS = 8; +/** Same ceiling as vfx: a real paint is two animation frames. */ +const PAINT_TIMEOUT_MS = 2000; +/** Consecutive capture failures before the blur turns itself off for the session. */ +const MAX_FAILURES = 3; +const LOG_EVERY = 25; + +const clamp = (x: number, lo: number, hi: number): number => Math.min(hi, Math.max(lo, x)); + +function readConfig(root: HTMLElement): Config | null { + if (root.getAttribute("data-motion-blur") !== "on") return null; + const num = (name: string, fallback: number): number => { + const v = Number(root.getAttribute(name)); + return root.hasAttribute(name) && Number.isFinite(v) ? v : fallback; + }; + return { + shutterDeg: clamp(num("data-motion-blur-shutter", 180), 0, 720), + samples: Math.round(clamp(num("data-motion-blur-samples", 16), 2, 64)), + events: root.getAttribute("data-motion-blur-events") === "on", + }; +} + +const ACCELERATED_CONTEXTS = new Set(["webgl", "webgl2", "experimental-webgl", "webgpu"]); +const acceleratedCanvases = new WeakSet(); + +/** + * Remember canvases given a WebGL / WebGPU context. Must run before composition scripts: once + * one has presented, html-in-canvas keeps capturing that frame however often it redraws. + */ +export function trackAcceleratedCanvases(): void { + const proto = HTMLCanvasElement.prototype; + const original: unknown = proto.getContext; + if (typeof original !== "function" || "__hfAccelTracked" in original) return; + const tracked = function (this: HTMLCanvasElement, ...args: unknown[]): unknown { + const ctx: unknown = Reflect.apply(original, this, args); + if (ctx && ACCELERATED_CONTEXTS.has(String(args[0]))) acceleratedCanvases.add(this); + return ctx; + }; + Object.defineProperty(tracked, "__hfAccelTracked", { value: true }); + Object.defineProperty(proto, "getContext", { + value: tracked, + writable: true, + configurable: true, + }); +} + +/** What in the composition the capture cannot take, checked again per frame (scripts add canvases). */ +function captureBlocker(root: HTMLElement): string | null { + // A capture canvas inside another one blocks the renderer's main thread. + if (root.querySelector("canvas[layoutsubtree]")) + return "the composition already uses html-in-canvas"; + for (const canvas of root.querySelectorAll("canvas")) { + if (acceleratedCanvases.has(canvas)) + return "the composition draws a WebGL/WebGPU canvas, which html-in-canvas captures frozen"; + } + return null; +} + +function unsupportedReason(root: HTMLElement): string | null { + const off = (window as unknown as Record)[PAGE_MOTION_BLUR_OFF_GLOBAL]; + if (off) return `the renderer turned it off (${String(off)})`; + if (!supportsCaptureMotionBlur()) { + return "html-in-canvas (drawElementImage) is unavailable in this browser"; + } + return captureBlocker(root); +} + +function describe(el: Element): string { + const cls = + typeof el.className === "string" && el.className + ? `.${el.className.trim().split(/\s+/).join(".")}` + : ""; + return `${el.tagName.toLowerCase()}${el.id ? `#${el.id}` : ""}${cls}`; +} + +function rootSize(root: HTMLElement): { width: number; height: number } { + const w = Number(root.getAttribute("data-width")) || root.offsetWidth; + const h = Number(root.getAttribute("data-height")) || root.offsetHeight; + return { width: Math.max(1, Math.round(w)), height: Math.max(1, Math.round(h)) }; +} + +function makeCanvas(width: number, height: number, role: string): HTMLCanvasElement { + const canvas = document.createElement("canvas"); + canvas.width = width; + canvas.height = height; + canvas.setAttribute("data-hf-ignore", ""); + canvas.setAttribute("data-hf-motion-blur", role); + // ponytail: assumes the root sits at its parent's origin (true for compositions); place the + // canvases at the root's offset if a layout ever puts it elsewhere. + canvas.style.cssText = `position:absolute;left:0;top:0;width:${width}px;height:${height}px;`; + return canvas; +} + +function createInstance(host: CaptureMotionBlurHost, config: Config) { + const { root } = host; + const { width, height } = rootSize(root); + const src = makeCanvas(width, height, "capture") as PaintCanvas; + const out = makeCanvas(width, height, "output"); + out.style.pointerEvents = "none"; + src.setAttribute("layoutsubtree", ""); + + const captureContext = src.getContext("2d") as DrawElementContext | null; + const outputContext = out.getContext("2d"); + if (!captureContext || !outputContext) throw new Error("capture motion blur: no 2D context"); + const sctx: DrawElementContext = captureContext; + const octx: CanvasRenderingContext2D = outputContext; + const compositor = createCompositor(width, height); + // Hides movers for the background capture without touching author or GSAP opacity. + const hideRule = document.createElement("style"); + hideRule.setAttribute("data-hf-ignore", ""); + hideRule.textContent = `[${HIDE_ATTR}] { opacity: 0 !important; }`; + + let movers: HTMLElement[] = []; + let reader: ReturnType | null = null; + let pending: Job | null = null; + // While a frame is in flight (two paints), newer requests collapse into the latest one. + let queued: { t: number; engineMode: boolean; release: () => void } | null = null; + let seq = 0; + let failures = 0; + let failedFrames = 0; + let disabled: string | null = null; + const timings: number[] = []; + let sampleMs = 0; + let scanMs = 0; + let scannedTimeline: unknown = undefined; + let wrapped = false; + // The output still shows the last frame of the previous wrap until this one draws. + let staleOutput = false; + // Once the engine drives frames, paused-style requests come from the transport, not a user. + let rendering = false; + + function wrap(): void { + if (wrapped) return; + root.before(src); + src.append(root); + src.after(out); + document.head.append(hideRule); + wrapped = true; + staleOutput = true; + } + + /** Back to plain DOM: drop any frame in flight, put the root where it was. */ + function unwrap(): void { + queued?.release(); + queued = null; + if (pending) { + reveal(pending); + pending.settle(); + pending = null; + } + if (!wrapped) return; + wrapped = false; + hideRule.remove(); + src.before(root); + src.remove(); + out.remove(); + } + + const seek = (t: number): void => host.seek(t, !config.events); + + function discover(restoreTo: number): void { + scannedTimeline = host.timeline(); + const s0 = performance.now(); + const found = discoverMovers({ + scene: root, + durationSeconds: host.durationSeconds(), + fps: host.fps, + seek, + }); + seek(restoreTo); + movers = found.movers; + reader = createPoseReader(root, found.dynamic); + scanMs = Math.round(performance.now() - s0); + } + + function frames(t: number, reads: number): { els: HTMLElement[]; frames: MoverFrame[] } { + const r = reader; + if (!r) return { els: [], frames: [] }; + const visible = movers.filter( + (el) => + typeof el.checkVisibility !== "function" || + el.checkVisibility({ visibilityProperty: true }), + ); + const shutter = config.shutterDeg / 360 / host.fps; + if (visible.length === 0 || shutter <= 0) return { els: [], frames: [] }; + const start = t - shutter / 2; + const poses = visible.map((): Affine[] => []); + for (let j = 0; j < reads; j++) { + seek(Math.max(0, start + (j / (reads - 1)) * shutter)); + r.reset(); + visible.forEach((el, i) => poses[i]?.push(r.pose(el))); + } + seek(t); + r.reset(); + const result = visible.map((el, i) => ({ + w: el.offsetWidth, + h: el.offsetHeight, + now: r.pose(el), + taps: tapsFromReads(poses[i] ?? [IDENTITY], config.samples), + })); + r.reset(); + return { els: visible, frames: result }; + } + + const capture = (): void => { + sctx.reset(); + sctx.drawElementImage(root, 0, 0); + }; + const reveal = (job: Job): void => { + for (const el of job.els) el.removeAttribute(HIDE_ATTR); + }; + + /** + * One step of the current frame, run on each paint (or by the engine's composite hook): + * first capture the movers and crop their pixels, then hide them and, on the next paint, + * capture the background and draw it with the trails. A frame with nothing moving is done + * after the first capture. `finalStep` forces the frame to finish now (no further paint will + * come), drawing trails over the sharp frame if the background was never captured. + */ + function step(finalStep = false): void { + const job = pending; + if (!job || disabled || !job.armed) return; + const t0 = performance.now(); + let done = true; + try { + done = captureAndDraw(job, finalStep); + failures = 0; + } catch (err) { + failures++; + swallow("runtime.captureMotionBlur.compose", err); + if (failures >= MAX_FAILURES) disable(`capture failed ${failures} times in a row`); + } + job.ms += performance.now() - t0; + if (!done && !disabled) { + src.requestPaint(); + return; + } + timings.push(sampleMs + job.ms); + if (timings.length > 600) timings.shift(); + finishJob(job); + } + + /** This paint's half of the frame; false when the background capture is still to come. */ + function captureAndDraw(job: Job, finalStep: boolean): boolean { + capture(); + try { + if (!job.kept) { + job.kept = compositor.crop(src, job.frames); + if (staleOutput) { + // First paint since wrapping: show this frame sharp until its trails are drawn. + octx.setTransform(1, 0, 0, 1, 0, 0); + octx.globalAlpha = 1; + octx.globalCompositeOperation = "copy"; + octx.drawImage(src, 0, 0); + staleOutput = false; + } + if (job.kept.some(Boolean) && !finalStep) { + job.els.forEach((el, i) => job.kept?.[i] && el.setAttribute(HIDE_ATTR, "")); + return false; + } + } + compositor.draw(src, octx, job.frames, job.kept); + return true; + } finally { + sctx.reset(); // the capture canvas sits under the output: keep its own bitmap empty + } + } + + /** End a frame (drawn or given up on) and start the newest frame requested meanwhile. */ + function finishJob(job: Job): void { + reveal(job); + if (pending === job) pending = null; + job.settle(); + const next = queued; + queued = null; + if (next && !disabled) void start(next.t, next.engineMode).then(next.release); + else next?.release(); + } + const onPaint = (): void => step(); + + function countFailedFrame(why: string): void { + failedFrames++; + if (failedFrames % LOG_EVERY === 1) + console.info(`[motion-blur] capture failed (${failedFrames} so far): ${why}`); + } + + // A tab that stops painting while the clock runs is not a broken capture: this frame is plain + // DOM (root back in the page, movers revealed), the next tries again, and timeouts never count + // toward giving up. + function onPaintTimeout(job: Job): void { + countFailedFrame(`no paint within ${PAINT_TIMEOUT_MS} ms`); + unwrap(); + finishJob(job); + } + + src.addEventListener("paint", onPaint); + + function disable(reason: string): void { + if (disabled) return; + disabled = reason; + console.warn(`[motion-blur] off: ${reason}`); + teardown(); + } + + function teardown(): void { + src.removeEventListener("paint", onPaint); + unwrap(); + } + + /** + * Request one blurred frame at `t`; resolves once it is drawn (or given up on). A frame still + * in flight (between its two paints) is not abandoned: the request waits as the newest pending + * one. An engine frame replaces it, since the engine waits on exactly this frame. + */ + function paint(t: number, engineMode: boolean): Promise { + if (disabled) return Promise.resolve(); + if (engineMode) rendering = true; + if (pending && !engineMode) { + const previous = queued; + return new Promise((resolve) => { + queued = { + t, + engineMode, + release: () => { + previous?.release(); + resolve(); + }, + }; + }); + } + return start(t, engineMode); + } + + function start(t: number, engineMode: boolean): Promise { + try { + if (!reader || host.timeline() !== scannedTimeline) discover(t); + if (movers.length === 0) { + unwrap(); + return Promise.resolve(); + } + // Checked again here, not only at init: a block's script can add its own capture canvas + // (wrapping one inside ours hangs the renderer) or a WebGL canvas later. + const blocked = captureBlocker(root); + if (blocked) { + disable(blocked); + return Promise.resolve(); + } + wrap(); + const s0 = performance.now(); + const next = frames(t, engineMode ? RENDER_READS : PREVIEW_READS); + sampleMs = performance.now() - s0; + if (pending) finishJob(pending); + return new Promise((resolve) => { + const mine = ++seq; + let timer: ReturnType | undefined; + const startTimer = (): void => { + timer = setTimeout(() => { + if (pending?.seq === mine) onPaintTimeout(pending); + else resolve(); + }, PAINT_TIMEOUT_MS); + }; + pending = { + seq: mine, + els: next.els, + frames: next.frames, + kept: null, + armed: !engineMode, + startTimer, + ms: 0, + settle: () => { + clearTimeout(timer); + resolve(); + }, + }; + if (engineMode) return; + startTimer(); + src.requestPaint(); + }); + } catch (err) { + swallow("runtime.captureMotionBlur.paint", err); + disable(err instanceof Error ? err.message : String(err)); + return Promise.resolve(); + } + } + + function report(): Report { + const mean = timings.length ? timings.reduce((s, x) => s + x, 0) / timings.length : 0; + return { + state: disabled ? "off" : "on", + reason: disabled, + movers: movers.length, + moverNames: movers.map(describe), + scanMs, + shutterDeg: config.shutterDeg, + frames: timings.length, + meanMs: Math.round(mean * 10) / 10, + failures, + failedFrames, + }; + } + + return { + root, + paint, + /** Paused in preview: show plain DOM so it can be selected. A render never rests. */ + rest: (): boolean => { + if (rendering) return false; + unwrap(); + return true; + }, + arm: () => { + if (!pending || pending.armed) return; + pending.armed = true; + pending.startTimer(); + src.requestPaint(); + }, + /** Finish the current frame now: the engine's forced paint just ran. */ + finish: () => { + if (pending) pending.armed = true; + step(true); + }, + teardown, + report, + busy: (): boolean => pending !== null, + rediscover: () => { + reader = null; + }, + }; +} + +type Instance = ReturnType; +let instance: Instance | null = null; +let priorResolver: (() => boolean) | null = null; +let ourResolver: (() => boolean) | null = null; + +/** + * Read the root's opt-in and wrap (or unwrap) it. Called wherever the runtime (re)initialises + * scene DOM; a repeat call on the same root only schedules a fresh mover scan. + */ +export function initCaptureMotionBlur(host: CaptureMotionBlurHost): void { + const config = readConfig(host.root); + const w = window as CompositeWindow; + if (instance && (instance.root !== host.root || !config)) { + instance.teardown(); + instance = null; + delete w.__hfMotionBlur; + } + if (!config) return; + if (instance) { + instance.rediscover(); + return; + } + const reason = unsupportedReason(host.root); + if (reason) { + console.info(`[motion-blur] off: ${reason}`); + w.__hfMotionBlur = { + report: () => ({ + state: "off", + reason, + movers: 0, + moverNames: [], + scanMs: 0, + shutterDeg: config.shutterDeg, + frames: 0, + meanMs: 0, + failures: 0, + failedFrames: 0, + }), + }; + return; + } + instance = createInstance(host, config); + const current = instance; + w.__hfMotionBlur = { report: () => current.report() }; +} + +/** + * Draw the blurred frame for `t` after a seek or a playback tick. In preview only playback blurs: + * a paused request (no `playing`) unwraps the composition instead and returns null. In engine + * mode it also claims the engine's + * page-composite step, composing with whoever already holds it, so a render frame is complete + * before the screenshot even if no paint event fired. Returns the frame's completion. + */ +export function paintCaptureMotionBlur( + t: number, + options?: { engineMode?: boolean; playing?: boolean }, +): Promise | null { + if (!instance) return null; + const engineMode = options?.engineMode === true; + if (!engineMode && !options?.playing && instance.rest()) return null; + const done = instance.paint(t, engineMode); + // Nothing in flight (nothing moves, or blur turned off): leave the engine's composite step idle. + if (engineMode && instance.busy()) armPageComposite(); + return done; +} + +/** Start capturing a deferred engine frame: the page (injected video frames included) is final. */ +export function releaseCaptureMotionBlurFrame(): void { + instance?.arm(); +} + +function armPageComposite(): void { + const w = window as CompositeWindow; + const current = w.__hf_page_composite_resolve; + if (current !== ourResolver) { + priorResolver = typeof current === "function" ? current : null; + ourResolver = () => { + if (priorResolver) priorResolver(); + // The engine's forced paint has just run, so the paint records are valid here even if + // no paint event reached us; compose() is a no-op when the event already drew the frame. + instance?.finish(); + w.__hf_page_composite_pending = false; + return true; + }; + w.__hf_page_composite_resolve = ourResolver; + } + w.__hf_page_composite_pending = true; +} + +/** Test-only: drop module state between cases. */ +export function resetCaptureMotionBlurForTests(): void { + instance?.teardown(); + instance = null; + priorResolver = null; + ourResolver = null; +} diff --git a/packages/core/src/runtime/entry.test.ts b/packages/core/src/runtime/entry.test.ts index d54f98505f7..5f64f36305c 100644 --- a/packages/core/src/runtime/entry.test.ts +++ b/packages/core/src/runtime/entry.test.ts @@ -96,6 +96,23 @@ const neverDecodes = (clip: HTMLElement) => { describe("runtime entry", () => { afterEach(resetRuntimeGlobals); + it("starts a held capture-motion-blur frame when capture waits on the seek, after video injection", async () => { + const release = vi.fn(); + vi.doMock("./captureMotionBlur", async (original) => ({ + ...(await original()), + releaseCaptureMotionBlurFrame: release, + })); + try { + mountRoot().appendChild(document.createElement("div")); + await evaluateRuntime(); + expect(release).not.toHaveBeenCalled(); + await window.__hfWaitForSeekCompletion?.(); + expect(release).toHaveBeenCalledTimes(1); + } finally { + vi.doUnmock("./captureMotionBlur"); + } + }); + it("drives an async frame source without a GSAP timeline and waits before capture", async () => { const root = mountRoot(); root.setAttribute("data-duration", "10"); diff --git a/packages/core/src/runtime/entry.ts b/packages/core/src/runtime/entry.ts index 6f34c13baf0..236222cfb75 100644 --- a/packages/core/src/runtime/entry.ts +++ b/packages/core/src/runtime/entry.ts @@ -17,6 +17,7 @@ import { runScriptsAfterFonts } from "./afterFonts"; import { AFTER_FONTS_SCRIPTS } from "../compiler/scriptRuns"; import { hasFrameSources, registerFrameSource } from "./frameSources"; import { createFilmBridge } from "./filmBridge"; +import { trackAcceleratedCanvases } from "./captureMotionBlur"; type HyperframeWindow = Window & { __hyperframeRuntimeBootstrapped?: boolean; @@ -43,6 +44,7 @@ type HyperframeWindow = Window & { installAuthoredOpacityCapture(); installAuthoredMediaCapture(); installFlatGsapTransforms(); +trackAcceleratedCanvases(); window.__hfHasFrameSources = hasFrameSources; hideTimedClipsUntilFirstPass(); diff --git a/packages/core/src/runtime/init.ts b/packages/core/src/runtime/init.ts index fab9328bc38..109e5864706 100644 --- a/packages/core/src/runtime/init.ts +++ b/packages/core/src/runtime/init.ts @@ -125,6 +125,11 @@ import { applyVariableBindings, unproxiedMediaSrc } from "./applyVariableBinding import { createColorGradingRuntime, type RuntimeColorGradingApi } from "./colorGrading"; import { COLOR_GRADING_AUTHORED_OPACITY_ATTR } from "../colorGrading"; import { initVfx, paintVfx } from "./vfx"; +import { + initCaptureMotionBlur, + paintCaptureMotionBlur, + releaseCaptureMotionBlurFrame, +} from "./captureMotionBlur"; import { TransportClock } from "./clock"; import { WebAudioTransport } from "./webAudioTransport"; import { @@ -3659,6 +3664,24 @@ export function initSandboxRuntimeModular(): void { // Passes that must see scene DOM, which arrives after init: when scenes mount, or when one is // swapped. Resolves when caption overrides have landed. + // Capture motion blur (data-motion-blur="on" on the root). Its pose reads seek the timeline + // and adapters only: media and transport bookkeeping stay on the frame's seek. + function initMotionBlur(): void { + const root = resolveRootCompositionElement(); + if (!root) return; + initCaptureMotionBlur({ + root, + fps: state.canonicalFps, + durationSeconds: () => + getSafeTimelineDurationSeconds(state.capturedTimeline, 0) || + Number(root.getAttribute("data-duration")) || + 0, + timeline: () => state.capturedTimeline, + seek: (t, suppressEvents) => { + seekRootChildrenAndAdapters(state.capturedTimeline, t, { suppressEvents }); + }, + }); + } const settleSceneDom = (): Promise => { bindMediaMetadataListeners(); installAssetFailureDiagnostics(); @@ -3667,6 +3690,7 @@ export function initSandboxRuntimeModular(): void { applyVariableBindings(document); // An unregistered vfx chain paints nothing and logs nothing, so re-scan the new DOM. initVfx(document.body, state.canonicalFps); + initMotionBlur(); return captionsApplied; }; if (!externalCompositionsReady) { @@ -4012,6 +4036,7 @@ export function initSandboxRuntimeModular(): void { } }; initVfx(document.body, state.canonicalFps); + initMotionBlur(); applyFetchedCaptionOverrides(captionOverrides, captionHosts, rewindCaptionTimelines); releaseDetachedMedia(); childrenBound = false; @@ -4064,6 +4089,7 @@ export function initSandboxRuntimeModular(): void { }); // Per-pixel effect chains: compile once here, repaint on every seek below. initVfx(document.body, state.canonicalFps); + initMotionBlur(); const applyPlaybackRate = (nextRate: number) => { const parsed = Number(nextRate); @@ -4120,6 +4146,8 @@ export function initSandboxRuntimeModular(): void { } colorGrading.redraw(); paintVfx(state.currentTime); + const blurred = paintCaptureMotionBlur(state.currentTime); + if (blurred) registerSeekCompletion(blurred); postState(true); }; @@ -4172,6 +4200,8 @@ export function initSandboxRuntimeModular(): void { runAdapters("pause"); syncMediaForCurrentState(); colorGrading.redraw(); + // Paused: back to plain DOM so Studio can select inside the composition. + void paintCaptureMotionBlur(state.currentTime); postState(true); }, seek: (timeSeconds, options) => { @@ -4232,6 +4262,8 @@ export function initSandboxRuntimeModular(): void { syncMediaForCurrentState(); colorGrading.redraw(); paintVfx(state.currentTime, { engineMode: true }); + const blurred = paintCaptureMotionBlur(state.currentTime, { engineMode: true }); + if (blurred) registerSeekCompletion(blurred); postState(true); }, getTime: () => heldSeek?.time ?? clock.now(), @@ -4340,9 +4372,15 @@ export function initSandboxRuntimeModular(): void { window.__hfTypegpuTime = t; forceDispatchSeekEvent(t); }; - window.__hfWaitForSeekCompletion = waitForSeekCompletion; + // Engine and CLI capture wait here once the page is final (the engine's injected video frames + // land after renderSeek), so a blurred engine frame is captured now, not at the seek. + const waitForFinalFrame: typeof waitForSeekCompletion = (options) => { + releaseCaptureMotionBlurFrame(); + return waitForSeekCompletion(options); + }; + window.__hfWaitForSeekCompletion = waitForFinalFrame; runtimeCleanupCallbacks.push(() => { - if (window.__hfWaitForSeekCompletion === waitForSeekCompletion) { + if (window.__hfWaitForSeekCompletion === waitForFinalFrame) { delete window.__hfWaitForSeekCompletion; } }); @@ -5151,6 +5189,8 @@ export function initSandboxRuntimeModular(): void { state.capturedTimeline !== lastTransportSeekTimeline ) { seekTimelineAndAdapters(t, { revision: timingRevision }); + // Playback: draw and move on; nothing waits on this frame's blur. Paused, it unwraps. + void paintCaptureMotionBlur(t, { playing: isPlaying }); if (!isPlaying) pausedSeekDeferredByManualGesture = false; } if (isPlaying) { @@ -5173,6 +5213,7 @@ export function initSandboxRuntimeModular(): void { } runAdapters("pause"); syncMediaForCurrentState(timingRevision); + void paintCaptureMotionBlur(state.currentTime); postState(true); return; }