diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 50c3c5bcdf0..51514f023ed 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -437,3 +437,4 @@ export { type HfVfxParam, type HfVfxParamValues, } from "./vfx.js"; +export { PAGE_MOTION_BLUR_OFF_GLOBAL } from "./pageMotionBlur.js"; diff --git a/packages/core/src/pageMotionBlur.ts b/packages/core/src/pageMotionBlur.ts new file mode 100644 index 00000000000..d540d046cf0 --- /dev/null +++ b/packages/core/src/pageMotionBlur.ts @@ -0,0 +1,12 @@ +// Set by the engine: its own sub-frame blur runs, or the Studio preview could not blur. +export const PAGE_MOTION_BLUR_OFF_GLOBAL = "__HF_PAGE_MOTION_BLUR_OFF__"; + +export function supportsCaptureMotionBlur(win: Window = window): boolean { + const w = win as Window & { + CanvasRenderingContext2D?: { prototype: object }; + HTMLCanvasElement?: { prototype: object }; + }; + const ctx = w.CanvasRenderingContext2D?.prototype; + const canvas = w.HTMLCanvasElement?.prototype; + return !!ctx && !!canvas && "drawElementImage" in ctx && "requestPaint" in canvas; +} diff --git a/packages/core/src/runtime/captureMotionBlur/composite.test.ts b/packages/core/src/runtime/captureMotionBlur/composite.test.ts new file mode 100644 index 00000000000..3fd0033873f --- /dev/null +++ b/packages/core/src/runtime/captureMotionBlur/composite.test.ts @@ -0,0 +1,80 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { createCompositor, type MoverFrame } from "./composite"; +import { IDENTITY, translate } from "./pose"; + +function fakeContext() { + return { + drawImage: vi.fn(), + clearRect: vi.fn(), + setTransform: vi.fn(), + globalAlpha: 1, + globalCompositeOperation: "source-over", + }; +} +let offscreen: Array> = []; + +beforeEach(() => { + offscreen = []; + vi.stubGlobal( + "OffscreenCanvas", + class { + constructor( + public width: number, + public height: number, + ) {} + getContext() { + const ctx = fakeContext(); + offscreen.push(ctx); + return ctx; + } + }, + ); +}); +afterEach(() => vi.unstubAllGlobals()); + +const capture = {} as CanvasImageSource; +const background = {} as CanvasImageSource; +const slide = (from: number, to: number, n = 16) => + Array.from({ length: n }, (_, k) => translate(IDENTITY, from + ((to - from) * (k + 0.5)) / n, 0)); +const mover = (w: number, h: number, taps: ReturnType): MoverFrame => ({ + w, + h, + now: translate(IDENTITY, 50, 0), + taps, +}); + +describe("createCompositor", () => { + it("keeps no crop for a mover that does not travel a pixel, and draws only the background", () => { + const c = createCompositor(100, 100); + const movers = [mover(10, 10, slide(0, 0.5))]; + const kept = c.crop(capture, movers); + expect(kept).toEqual([false]); + const out = fakeContext(); + expect(c.draw(background, out as never, movers, kept)).toEqual({ blurred: 0, taps: 0 }); + expect(out.drawImage).toHaveBeenCalledTimes(1); + expect(out.drawImage.mock.calls[0]?.[0]).toBe(background); + }); + + it("crops a moving mover once and lays one tap per draw over the background", () => { + const c = createCompositor(1000, 100); + const movers = [mover(10, 10, slide(0, 100))]; + const kept = c.crop(capture, movers); + expect(kept).toEqual([true]); + const crop = offscreen.at(-1); + expect(crop?.drawImage).toHaveBeenCalledWith(capture, 0, 0); + const out = fakeContext(); + const stats = c.draw(background, out as never, movers, kept); + const accum = offscreen[0]; + expect(stats.blurred).toBe(1); + expect(accum?.drawImage).toHaveBeenCalledTimes(stats.taps); + expect(accum?.globalAlpha).toBeCloseTo(1 / stats.taps, 9); + // Background first, then the trail over it. + expect(out.drawImage.mock.calls[0]).toEqual([background, 0, 0]); + expect(out.drawImage).toHaveBeenCalledTimes(2); + }); + + it("leaves a mover sharp when its box is too big to copy every frame", () => { + const c = createCompositor(100, 100); + expect(c.crop(capture, [mover(1000, 1000, slide(0, 100))])).toEqual([false]); + }); +}); diff --git a/packages/core/src/runtime/captureMotionBlur/composite.ts b/packages/core/src/runtime/captureMotionBlur/composite.ts new file mode 100644 index 00000000000..81532fad777 --- /dev/null +++ b/packages/core/src/runtime/captureMotionBlur/composite.ts @@ -0,0 +1,135 @@ +/** + * The per-frame draw: a background, then each mover's own pixels redrawn along its path. + * + * Each mover is cropped from a capture of the scene and drawn at every tap pose into a float + * buffer, additive at weight 1/n: the shutter integral. Its coverage fades at the path ends, so + * the trail needs a background without the mover (a second capture, movers hidden); falling back + * to the sharp frame leaves thin lines looking unblurred. + * + * Image-space approximation: a mover's box contents move with it; ink outside it is not redrawn. + */ + +import { type Affine, invert, sweep, thinTaps } from "./pose"; + +export interface MoverFrame { + w: number; + h: number; + /** The pose the capture shows (frame time). */ + now: Affine; + /** Poses across the shutter. */ + taps: Affine[]; +} + +interface CompositeStats { + blurred: number; + taps: number; +} + +type Context2D = CanvasRenderingContext2D | OffscreenCanvasRenderingContext2D; +interface Layer { + canvas: OffscreenCanvas; + ctx: OffscreenCanvasRenderingContext2D; +} + +// Float16 keeps n additive 1/n-weighted taps exact; 8-bit loses up to ~n/2 levels per channel. +function makeLayer(w: number, h: number): Layer { + const canvas = new OffscreenCanvas(w, h); + let ctx: OffscreenCanvasRenderingContext2D | null = null; + try { + ctx = canvas.getContext("2d", { colorType: "float16" } as CanvasRenderingContext2DSettings); + } catch { + ctx = null; + } + ctx = ctx ?? canvas.getContext("2d"); + if (!ctx) throw new Error("capture motion blur: no 2D context for the accumulation buffer"); + return { canvas, ctx }; +} + +/** Whether a mover travels far enough within this shutter to draw a trail at all. */ +function moves(mover: MoverFrame, frameW: number, frameH: number): boolean { + const { travel, rect } = sweep(mover.taps, mover.w, mover.h, frameW, frameH); + return rect !== null && travel >= 1; +} + +export function createCompositor(width: number, height: number) { + const accum = makeLayer(width, height); + const crops: Layer[] = []; + // A mover box this many times the frame is too big to copy every frame; it is left sharp. + const maxCropArea = width * height * 4; + + const cropLayer = (i: number, w: number, h: number): Layer => { + let layer = crops[i]; + if (!layer || layer.canvas.width < w || layer.canvas.height < h) { + layer = makeLayer( + Math.max(w, layer?.canvas.width ?? 1), + Math.max(h, layer?.canvas.height ?? 1), + ); + crops[i] = layer; + } + return layer; + }; + + /** Copy each mover's frame-time pixels out of `src`; false marks a mover to leave sharp. */ + function crop(src: CanvasImageSource, movers: MoverFrame[]): boolean[] { + return movers.map((mover, i) => { + const { w, h } = mover; + const back = invert(mover.now); + if (!back || w * h > maxCropArea || !moves(mover, width, height)) return false; + const c = cropLayer(i, w, h).ctx; + c.setTransform(1, 0, 0, 1, 0, 0); + c.globalCompositeOperation = "source-over"; + c.globalAlpha = 1; + c.clearRect(0, 0, w, h); + c.setTransform(back.a, back.b, back.c, back.d, back.e, back.f); + c.drawImage(src, 0, 0); + return true; + }); + } + + function trail(out: Context2D, mover: MoverFrame, layer: Layer): number { + const { w, h } = mover; + const { travel, rect } = sweep(mover.taps, w, h, width, height); + if (!rect) return 0; + const taps = thinTaps(mover.taps, travel); + const a = accum.ctx; + a.setTransform(1, 0, 0, 1, 0, 0); + a.globalCompositeOperation = "source-over"; + a.globalAlpha = 1; + a.clearRect(rect.x, rect.y, rect.w, rect.h); + a.globalCompositeOperation = "lighter"; + a.globalAlpha = 1 / taps.length; + for (const m of taps) { + a.setTransform(m.a, m.b, m.c, m.d, m.e, m.f); + a.drawImage(layer.canvas, 0, 0, w, h, 0, 0, w, h); + } + out.drawImage(accum.canvas, rect.x, rect.y, rect.w, rect.h, rect.x, rect.y, rect.w, rect.h); + return taps.length; + } + + /** `background` into `out`, then the trail of every mover `crop` kept. */ + function draw( + background: CanvasImageSource, + out: Context2D, + movers: MoverFrame[], + kept: boolean[], + ): CompositeStats { + out.setTransform(1, 0, 0, 1, 0, 0); + out.globalAlpha = 1; + out.globalCompositeOperation = "copy"; + out.drawImage(background, 0, 0); + out.globalCompositeOperation = "source-over"; + const stats: CompositeStats = { blurred: 0, taps: 0 }; + movers.forEach((mover, i) => { + const layer = crops[i]; + if (!kept[i] || !layer) return; + const n = trail(out, mover, layer); + if (n > 0) { + stats.blurred++; + stats.taps += n; + } + }); + return stats; + } + + return { crop, draw }; +} diff --git a/packages/core/src/runtime/captureMotionBlur/discover.test.ts b/packages/core/src/runtime/captureMotionBlur/discover.test.ts new file mode 100644 index 00000000000..0f0aac55a3f --- /dev/null +++ b/packages/core/src/runtime/captureMotionBlur/discover.test.ts @@ -0,0 +1,89 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { discoverMovers } from "./discover"; + +// jsdom lays nothing out: each element's rect follows the x its fake timeline writes. +function element(parent: Element, id: string): HTMLElement { + const el = document.createElement("div"); + el.id = id; + parent.append(el); + el.getBoundingClientRect = () => { + const x = parseFloat(el.style.transform.replace("translateX(", "")) || 0; + return { left: x, top: 0, right: x + 10, bottom: 10, width: 10, height: 10 } as DOMRect; + }; + return el; +} + +function setup() { + const scene = document.createElement("div"); + const fast = element(scene, "fast"); + const fastChild = element(fast, "fast-child"); + const slow = element(scene, "slow"); + const still = element(scene, "still"); + const off = element(scene, "off"); + off.setAttribute("data-motion-blur", "off"); + // 1 s at 30 fps: `fast` and `off` cross 300 px (10 px/frame), `slow` 15 px (0.5 px/frame). + const seek = (t: number): void => { + fast.style.transform = `translateX(${300 * t}px)`; + fastChild.style.transform = `translateX(${200 * t}px)`; + slow.style.transform = `translateX(${15 * t}px)`; + off.style.transform = `translateX(${300 * t}px)`; + }; + seek(0); + return { scene, fast, fastChild, slow, still, off, seek }; +} + +describe("discoverMovers", () => { + afterEach(() => vi.restoreAllMocks()); + + it("keeps fast, outermost movers that are not opted out", () => { + const { scene, fast, seek } = setup(); + const { movers } = discoverMovers({ scene, durationSeconds: 1, fps: 30, seek }); + expect(movers).toEqual([fast]); + }); + + it("marks every changing element dynamic, slow and opted-out ones included", () => { + const { scene, fast, fastChild, slow, still, off, seek } = setup(); + const { dynamic } = discoverMovers({ scene, durationSeconds: 1, fps: 30, seek }); + expect( + dynamic.has(fast) && dynamic.has(fastChild) && dynamic.has(slow) && dynamic.has(off), + ).toBe(true); + expect(dynamic.has(still)).toBe(false); + }); + + it("ignores a CSS animation of offset-distance, which the pose has no term for", () => { + const scene = document.createElement("div"); + const box = document.createElement("div"); + scene.append(box); + let x = 0; // moves with no inline style change: only getAnimations can nominate it + box.getBoundingClientRect = () => + ({ left: x, right: x + 10, top: 0, bottom: 10, width: 10 }) as DOMRect; + const scan = (prop: string) => { + const effect = { target: box, getKeyframes: () => [{ [prop]: "0" }] }; + Object.defineProperty(document, "getAnimations", { + value: () => [{ effect }], + configurable: true, + }); + const seek = (t: number): void => void (x = 300 * t); + return discoverMovers({ scene, durationSeconds: 1, fps: 30, seek }).movers; + }; + expect(scan("offsetDistance")).toEqual([]); + expect(scan("translate")).toEqual([box]); + Reflect.deleteProperty(document, "getAnimations"); + }); + + it("drops a mover in a 3D context, or on an offset-path the pose cannot follow", () => { + const { scene, fast, seek } = setup(); + const real = window.getComputedStyle; + let own: object = {}; + vi.spyOn(window, "getComputedStyle").mockImplementation((el: Element) => + el === fast + ? ({ transform: "none", transformStyle: "flat", perspective: "none", ...own } as never) + : real(el), + ); + for (own of [ + { transform: "matrix3d(1,0,0,0,0,1,0,0,0,0,1,0,0,0,0,1)" }, + { offsetPath: "path('M 0 0')" }, + ]) + expect(discoverMovers({ scene, durationSeconds: 1, fps: 30, seek }).movers).toEqual([]); + }); +}); diff --git a/packages/core/src/runtime/captureMotionBlur/discover.ts b/packages/core/src/runtime/captureMotionBlur/discover.ts new file mode 100644 index 00000000000..18ac0cc84f5 --- /dev/null +++ b/packages/core/src/runtime/captureMotionBlur/discover.ts @@ -0,0 +1,124 @@ +/** + * Which elements to blur: the ones that move fast enough for a smear to show. + * + * One scan: candidates are elements whose inline style changes or that a CSS / WAAPI animation + * moves; they must reach MIN_SPEED px/frame outside any 3D context (the redraw is 2D), and only + * the outermost of nested candidates is kept. + */ + +import { isHtmlElement } from "../domRealm"; + +/** Under this peak speed (px per frame) a smear is invisible, so the element stays sharp. */ +const MIN_SPEED = 2; + +/** Scan resolution: at most this many timeline samples, however long the composition. */ +const SCAN_STEPS = 240; + +const ANIMATED_POSE_PROPS = new Set([ + "transform", + "translate", + "rotate", + "scale", + "left", + "top", + "right", + "bottom", +]); + +export interface DiscoverOptions { + scene: HTMLElement; + durationSeconds: number; + fps: number; + /** Seek the composition (timeline + adapters) to a time. */ + seek: (t: number) => void; +} + +export interface Discovery { + movers: HTMLElement[]; + /** Every element whose pose changes at all; their transforms are never cached. */ + dynamic: Set; +} + +// The whole inline style in one read; opacity or colour tweens only cost a rect check. +const styleKey = (el: HTMLElement): string => el.getAttribute("style") ?? ""; + +function optedOut(el: Element, scene: Element): boolean { + for (let n: Element | null = el; n && n !== scene; n = n.parentElement) { + if (n.getAttribute("data-motion-blur") === "off") return true; + } + return false; +} + +function in3d(el: Element, scene: Element): boolean { + for (let n: Element | null = el; n && n !== scene; n = n.parentElement) { + const cs = getComputedStyle(n); + if ( + cs.transform.startsWith("matrix3d") || + cs.transformStyle === "preserve-3d" || + cs.perspective !== "none" || + (cs.offsetPath ?? "none") !== "none" + ) { + return true; + } + } + return false; +} + +/** Elements a CSS animation or WAAPI animation moves (no inline style to diff). */ +function pageAnimated(scene: Element): Set { + const out = new Set(); + if (typeof document.getAnimations !== "function") return out; + for (const animation of document.getAnimations()) { + const effect = animation.effect as KeyframeEffect | null; + const target = effect?.target; + if (!isHtmlElement(target) || !scene.contains(target)) continue; + const frames = typeof effect?.getKeyframes === "function" ? effect.getKeyframes() : []; + if (frames.some((frame) => Object.keys(frame).some((k) => ANIMATED_POSE_PROPS.has(k)))) + out.add(target); + } + return out; +} + +/** How far a box's farthest-moving edge travelled between two reads; 0 when either is empty. */ +function edgeShift(a: DOMRect | undefined, b: DOMRect | undefined): number { + if (!a || !b || !a.width || !b.width) return 0; + return Math.max( + Math.abs(a.left - b.left), + Math.abs(a.top - b.top), + Math.abs(a.right - b.right), + Math.abs(a.bottom - b.bottom), + ); +} + +export function discoverMovers({ scene, durationSeconds, fps, seek }: DiscoverOptions): Discovery { + const all = [...scene.querySelectorAll("*")].filter(isHtmlElement); + const dynamic = new Set(pageAnimated(scene)); + const first = all.map(styleKey); + const frames = Math.max(1, Math.round(durationSeconds * fps)); + // One pass, about every other frame on a typical composition. A whip shorter than one step + // can be under-measured; ponytail: raise SCAN_STEPS if a fast move goes unblurred. + const steps = Math.min(frames, SCAN_STEPS); + const framesPerStep = frames / steps; + const fast = new Set(); + const threeD = new Set(); + let prev: Array | null = null; + + for (let i = 0; i <= steps; i++) { + seek((i / steps) * durationSeconds); + // Rects only for elements whose style has changed; the first read flushes layout once. + const rects: Array = []; + all.forEach((el, j) => { + if (!dynamic.has(el) && styleKey(el) === first[j]) return; + dynamic.add(el); + rects[j] = el.getBoundingClientRect(); + if (edgeShift(prev?.[j], rects[j]) / framesPerStep >= MIN_SPEED) fast.add(el); + }); + // Style reads up the chain are the costly part; every 10th step is enough to see a 3D turn. + if (i % 10 === 0 || i === steps) for (const el of fast) if (in3d(el, scene)) threeD.add(el); + prev = rects; + } + + const candidates = all.filter((el) => fast.has(el) && !threeD.has(el) && !optedOut(el, scene)); + const movers = candidates.filter((el) => !candidates.some((o) => o !== el && o.contains(el))); + return { movers, dynamic }; +} diff --git a/packages/core/src/runtime/captureMotionBlur/index.test.ts b/packages/core/src/runtime/captureMotionBlur/index.test.ts new file mode 100644 index 00000000000..ccd7a8c55df --- /dev/null +++ b/packages/core/src/runtime/captureMotionBlur/index.test.ts @@ -0,0 +1,311 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { + type CaptureMotionBlurHost, + initCaptureMotionBlur, + paintCaptureMotionBlur, + releaseCaptureMotionBlurFrame, + resetCaptureMotionBlurForTests, + trackAcceleratedCanvases, +} from "."; + +type Win = Window & { + __HF_PAGE_MOTION_BLUR_OFF__?: unknown; + __hfMotionBlur?: { + report: () => { + state: string; + reason: string | null; + shutterDeg: number; + moverNames: string[]; + }; + }; + __hf_page_composite_pending?: boolean; + __hf_page_composite_resolve?: () => boolean; +}; +const win = window as Win; + +// Every 2D context (page canvas or offscreen) is this recording fake. +function fakeContext() { + return { + reset: vi.fn(), + drawElementImage: vi.fn(), + drawImage: vi.fn(), + clearRect: vi.fn(), + setTransform: vi.fn(), + globalAlpha: 1, + globalCompositeOperation: "source-over", + }; +} +let contexts: Array> = []; + +function installHtmlInCanvas(): void { + vi.spyOn(HTMLCanvasElement.prototype, "getContext").mockImplementation(() => { + const ctx = fakeContext(); + contexts.push(ctx); + return ctx as never; + }); + vi.stubGlobal( + "CanvasRenderingContext2D", + class { + drawElementImage(): void {} + }, + ); + (HTMLCanvasElement.prototype as unknown as Record).requestPaint = () => {}; + vi.stubGlobal( + "OffscreenCanvas", + class { + width: number; + height: number; + constructor(w: number, h: number) { + this.width = w; + this.height = h; + } + getContext() { + return fakeContext(); + } + }, + ); +} + +function makeRoot(attrs: Record = { "data-motion-blur": "on" }): HTMLElement { + const root = document.createElement("div"); + root.setAttribute("data-composition-id", "main"); + root.setAttribute("data-width", "640"); + root.setAttribute("data-height", "360"); + for (const [k, v] of Object.entries(attrs)) root.setAttribute(k, v); + document.body.append(root); + return root; +} + +function host( + root: HTMLElement, + seek = vi.fn(), +): CaptureMotionBlurHost & { seek: ReturnType } { + return { root, fps: 30, durationSeconds: () => 1, timeline: () => "tl", seek }; +} + +const captureCanvas = (root: HTMLElement) => root.parentElement as HTMLCanvasElement; + +beforeEach(() => { + contexts = []; + vi.spyOn(console, "info").mockImplementation(() => {}); + vi.spyOn(console, "warn").mockImplementation(() => {}); +}); + +afterEach(() => { + resetCaptureMotionBlurForTests(); + vi.restoreAllMocks(); + vi.unstubAllGlobals(); + vi.useRealTimers(); + delete (HTMLCanvasElement.prototype as unknown as Record).requestPaint; + delete win.__HF_PAGE_MOTION_BLUR_OFF__; + delete win.__hfMotionBlur; + delete win.__hf_page_composite_pending; + delete win.__hf_page_composite_resolve; + document.body.innerHTML = ""; +}); + +/** A root with one 10×10 element that slides 300 px over the 1 s composition (10 px/frame). */ +function movingHost(attrs?: Record) { + const root = makeRoot(attrs); + const el = document.createElement("div"); + el.id = "slider"; + root.append(el); + for (const [k, v] of Object.entries({ + offsetWidth: 10, + offsetHeight: 10, + offsetLeft: 0, + offsetTop: 0, + offsetParent: root, + })) { + Object.defineProperty(el, k, { value: v, configurable: true }); + } + const x = () => Number(/,\s*([-\d.]+),\s*0\)$/.exec(el.style.transform)?.[1] ?? 0); + el.getBoundingClientRect = () => + ({ left: x(), top: 0, right: x() + 10, bottom: 10, width: 10, height: 10 }) as DOMRect; + const seek = vi.fn((t: number) => { + el.style.transform = `matrix(1, 0, 0, 1, ${300 * t}, 0)`; + }); + return { root, el, h: host(root, seek) }; +} + +const PLAY = { playing: true }; + +describe("opt-in and off switches", () => { + it("does nothing for a root that has not opted in", () => { + installHtmlInCanvas(); + const root = makeRoot({}); + initCaptureMotionBlur(host(root)); + expect(root.parentElement).toBe(document.body); + expect(paintCaptureMotionBlur(0)).toBeNull(); + expect(win.__hfMotionBlur).toBeUndefined(); + }); + + it("stays off without html-in-canvas and says why", () => { + const root = makeRoot(); + initCaptureMotionBlur(host(root)); + expect(root.parentElement).toBe(document.body); + expect(win.__hfMotionBlur?.report()).toMatchObject({ + state: "off", + reason: expect.stringContaining("html-in-canvas"), + }); + }); + + it("stays off while the engine's own sub-frame blur is on (never both)", () => { + installHtmlInCanvas(); + win.__HF_PAGE_MOTION_BLUR_OFF__ = "engine-motion-blur"; + const { root, h } = movingHost(); + initCaptureMotionBlur(h); + expect(paintCaptureMotionBlur(0.5, PLAY)).toBeNull(); + expect(root.parentElement).toBe(document.body); + expect(win.__hfMotionBlur?.report().reason).toContain("engine"); + }); + + it("stays off for a WebGL canvas, which html-in-canvas captures frozen, but not for a 2D one", () => { + installHtmlInCanvas(); + trackAcceleratedCanvases(); + const root = makeRoot(); + const flat = document.createElement("canvas"); + root.append(flat); + flat.getContext("2d"); + initCaptureMotionBlur(host(root)); + expect(win.__hfMotionBlur?.report().state).toBe("on"); + + resetCaptureMotionBlurForTests(); + const gl = document.createElement("canvas"); + root.append(gl); + gl.getContext("webgl"); + initCaptureMotionBlur(host(root)); + expect(win.__hfMotionBlur?.report().reason).toMatch(/WebGL/); + }); + + it("turns off instead of wrapping when a WebGL canvas appears after init", () => { + installHtmlInCanvas(); + trackAcceleratedCanvases(); + const { root, h } = movingHost(); + initCaptureMotionBlur(h); + const late = document.createElement("canvas"); + root.append(late); + late.getContext("webgl2"); + void paintCaptureMotionBlur(0.5, { engineMode: true }); + expect(root.parentElement).toBe(document.body); + expect(win.__hfMotionBlur?.report().reason).toMatch(/WebGL/); + }); +}); + +describe("wrapping", () => { + it("wraps the root itself in a capture canvas, with an output canvas after it", () => { + installHtmlInCanvas(); + const { root, h } = movingHost(); + initCaptureMotionBlur(h); + expect(root.parentElement).toBe(document.body); // not until it plays + void paintCaptureMotionBlur(0.5, PLAY); + const src = captureCanvas(root); + expect(src.tagName).toBe("CANVAS"); + expect(src.hasAttribute("layoutsubtree")).toBe(true); + expect(src.hasAttribute("data-hf-ignore")).toBe(true); + expect([src.width, src.height]).toEqual([640, 360]); + const out = src.nextElementSibling as HTMLElement; + expect(out.getAttribute("data-hf-motion-blur")).toBe("output"); + // The output sits over the composition: clicks must reach what is under it. + expect(out.style.pointerEvents).toBe("none"); + }); + + it("turns off instead of wrapping when a capture canvas appears after init", () => { + installHtmlInCanvas(); + const { root, h } = movingHost(); + initCaptureMotionBlur(h); + const late = document.createElement("canvas"); + late.setAttribute("layoutsubtree", ""); + root.append(late); + void paintCaptureMotionBlur(0.5, PLAY); + expect(root.parentElement).toBe(document.body); + expect(win.__hfMotionBlur?.report().state).toBe("off"); + }); +}); + +describe("paint timeouts", () => { + const HIDE = "data-hf-motion-blur-hide"; + const info = () => vi.mocked(console.info); + const engineFrame = (t: number): Promise | null => { + const done = paintCaptureMotionBlur(t, { engineMode: true }); + releaseCaptureMotionBlurFrame(); + return done; + }; + const plainDom = (root: HTMLElement): void => { + expect(root.parentElement).toBe(document.body); + expect(document.querySelectorAll("canvas")).toHaveLength(0); + }; + + it("renders a released frame whose paint never comes plain, counts it, and leaves nothing for the engine to finish (first frame, output never drawn)", async () => { + vi.useFakeTimers(); + installHtmlInCanvas(); + const { root, el, h } = movingHost(); + initCaptureMotionBlur(h); + const done = engineFrame(0.5); + expect(captureCanvas(root).tagName).toBe("CANVAS"); + await vi.advanceTimersByTimeAsync(2001); + await done; + plainDom(root); + expect(el.hasAttribute(HIDE)).toBe(false); + expect(win.__hfMotionBlur?.report()).toMatchObject({ + state: "on", + failures: 0, + failedFrames: 1, + }); + expect(info()).toHaveBeenCalledWith( + expect.stringMatching(/capture failed \(1 so far\).*paint/), + ); + win.__hf_page_composite_resolve?.(); // the engine's forced composite arrives late + plainDom(root); + expect(contexts[0]?.drawElementImage).not.toHaveBeenCalled(); + expect(win.__hf_page_composite_pending).toBe(false); + void paintCaptureMotionBlur(0.6, { engineMode: true }); // the next frame wraps again + expect(captureCanvas(root).tagName).toBe("CANVAS"); + }); + + it("does the same for a frame after one that painted, whose output already holds an earlier frame", async () => { + vi.useFakeTimers(); + installHtmlInCanvas(); + const { root, el, h } = movingHost(); + initCaptureMotionBlur(h); + const first = engineFrame(0.5); + captureCanvas(root).dispatchEvent(new Event("paint")); + captureCanvas(root).dispatchEvent(new Event("paint")); + await first; + expect(win.__hfMotionBlur?.report()).toMatchObject({ failedFrames: 0, frames: 1 }); + const captures = contexts[0]?.drawElementImage.mock.calls.length; + const second = engineFrame(0.6); + await vi.advanceTimersByTimeAsync(2001); + await second; + plainDom(root); + expect(el.hasAttribute(HIDE)).toBe(false); + expect(win.__hfMotionBlur?.report()).toMatchObject({ failures: 0, failedFrames: 1 }); + win.__hf_page_composite_resolve?.(); + plainDom(root); + expect(contexts[0]?.drawElementImage.mock.calls.length).toBe(captures); + }); + + it("keeps blurring through preview paint timeouts, each frame plain, and resumes when painting returns", async () => { + vi.useFakeTimers(); + installHtmlInCanvas(); + const { root, h } = movingHost(); + initCaptureMotionBlur(h); + for (let i = 0; i < 4; i++) { + void paintCaptureMotionBlur(i / 30, PLAY); + expect(captureCanvas(root).tagName).toBe("CANVAS"); + await vi.advanceTimersByTimeAsync(2000); + plainDom(root); + } + expect(win.__hfMotionBlur?.report()).toMatchObject({ + state: "on", + failures: 0, + failedFrames: 4, + }); + expect(info()).toHaveBeenCalledTimes(1); + const done = paintCaptureMotionBlur(0.5, PLAY); + captureCanvas(root).dispatchEvent(new Event("paint")); + captureCanvas(root).dispatchEvent(new Event("paint")); + await done; + expect(contexts[0]?.drawElementImage).toHaveBeenCalled(); + }); +}); diff --git a/packages/core/src/runtime/captureMotionBlur/index.ts b/packages/core/src/runtime/captureMotionBlur/index.ts new file mode 100644 index 00000000000..9324a8f22c1 --- /dev/null +++ b/packages/core/src/runtime/captureMotionBlur/index.ts @@ -0,0 +1,605 @@ +/** + * Capture motion blur, in preview and render. Root attributes: data-motion-blur="on"|"off", + * -shutter (0–720°, default 180), -samples (2–64, default 16), -events="on" (run timeline + * callbacks at sub-frame samples). PAGE_MOTION_BLUR_OFF_GLOBAL (engine blur, or a preview that + * could not blur) turns all off. + * + * Per frame: read movers' poses across the shutter (seeks, no paint), capture the root and crop + * the movers, hide them and capture the background, then redraw each along its path. + * The root is wrapped, not its children (author selectors keep matching), and only while playing + * or rendering with something moving: paused, it is plain DOM so Studio can select inside it. + * Needs html-in-canvas (`drawElementImage`); without it nothing changes. + */ + +import { PAGE_MOTION_BLUR_OFF_GLOBAL, supportsCaptureMotionBlur } from "../../pageMotionBlur"; +import { swallow } from "../diagnostics"; +import { createCompositor, type MoverFrame } from "./composite"; +import { discoverMovers } from "./discover"; +import { type Affine, createPoseReader, IDENTITY, tapsFromReads } from "./pose"; + +export interface CaptureMotionBlurHost { + root: HTMLElement; + fps: number; + durationSeconds: () => number; + /** The bound root timeline; movers are re-scanned when it changes. */ + timeline: () => unknown; + /** Seek the timeline and adapters (not media) for a pose read. */ + seek: (t: number, suppressEvents: boolean) => void; +} + +interface Config { + shutterDeg: number; + samples: number; + events: boolean; +} + +interface Report { + state: "on" | "off"; + reason: string | null; + movers: number; + /** The blurred elements, as `tag#id.class`, for diagnosing what was (not) picked. */ + moverNames: string[]; + /** One-time cost of finding the movers, in ms. */ + scanMs: number; + shutterDeg: number; + frames: number; + meanMs: number; + failures: number; + failedFrames: number; +} + +type DrawElementContext = CanvasRenderingContext2D & { + drawElementImage: (el: Element, x: number, y: number) => unknown; +}; +type PaintCanvas = HTMLCanvasElement & { requestPaint: () => void }; +type CompositeWindow = Window & { + __hfMotionBlur?: { report: () => Report }; + __hf_page_composite_pending?: boolean; + __hf_page_composite_resolve?: () => boolean; +}; + +/** Marks a mover hidden for the background capture; set only between a frame's two paints. */ +const HIDE_ATTR = "data-hf-motion-blur-hide"; + +interface Job { + seq: number; + els: HTMLElement[]; + frames: MoverFrame[]; + /** Which movers were cropped (null until the first capture of this frame). */ + kept: boolean[] | null; + /** Engine frames wait for a release: video frames are injected after the seek. */ + armed: boolean; + /** The paint wait starts on arming, so slow video injection cannot time a frame out unseen. */ + startTimer: () => void; + ms: number; + settle: () => void; +} + +/** Pose reads per frame; the taps between them are interpolated. A render reads more. */ +const PREVIEW_READS = 4; +const RENDER_READS = 8; +/** Same ceiling as vfx: a real paint is two animation frames. */ +const PAINT_TIMEOUT_MS = 2000; +/** Consecutive capture failures before the blur turns itself off for the session. */ +const MAX_FAILURES = 3; +const LOG_EVERY = 25; + +const clamp = (x: number, lo: number, hi: number): number => Math.min(hi, Math.max(lo, x)); + +function readConfig(root: HTMLElement): Config | null { + if (root.getAttribute("data-motion-blur") !== "on") return null; + const num = (name: string, fallback: number): number => { + const v = Number(root.getAttribute(name)); + return root.hasAttribute(name) && Number.isFinite(v) ? v : fallback; + }; + return { + shutterDeg: clamp(num("data-motion-blur-shutter", 180), 0, 720), + samples: Math.round(clamp(num("data-motion-blur-samples", 16), 2, 64)), + events: root.getAttribute("data-motion-blur-events") === "on", + }; +} + +const ACCELERATED_CONTEXTS = new Set(["webgl", "webgl2", "experimental-webgl", "webgpu"]); +const acceleratedCanvases = new WeakSet(); + +/** + * Remember canvases given a WebGL / WebGPU context. Must run before composition scripts: once + * one has presented, html-in-canvas keeps capturing that frame however often it redraws. + */ +export function trackAcceleratedCanvases(): void { + const proto = HTMLCanvasElement.prototype; + const original: unknown = proto.getContext; + if (typeof original !== "function" || "__hfAccelTracked" in original) return; + const tracked = function (this: HTMLCanvasElement, ...args: unknown[]): unknown { + const ctx: unknown = Reflect.apply(original, this, args); + if (ctx && ACCELERATED_CONTEXTS.has(String(args[0]))) acceleratedCanvases.add(this); + return ctx; + }; + Object.defineProperty(tracked, "__hfAccelTracked", { value: true }); + Object.defineProperty(proto, "getContext", { + value: tracked, + writable: true, + configurable: true, + }); +} + +/** What in the composition the capture cannot take, checked again per frame (scripts add canvases). */ +function captureBlocker(root: HTMLElement): string | null { + // A capture canvas inside another one blocks the renderer's main thread. + if (root.querySelector("canvas[layoutsubtree]")) + return "the composition already uses html-in-canvas"; + for (const canvas of root.querySelectorAll("canvas")) { + if (acceleratedCanvases.has(canvas)) + return "the composition draws a WebGL/WebGPU canvas, which html-in-canvas captures frozen"; + } + return null; +} + +function unsupportedReason(root: HTMLElement): string | null { + const off = (window as unknown as Record)[PAGE_MOTION_BLUR_OFF_GLOBAL]; + if (off) return `the renderer turned it off (${String(off)})`; + if (!supportsCaptureMotionBlur()) { + return "html-in-canvas (drawElementImage) is unavailable in this browser"; + } + return captureBlocker(root); +} + +function describe(el: Element): string { + const cls = + typeof el.className === "string" && el.className + ? `.${el.className.trim().split(/\s+/).join(".")}` + : ""; + return `${el.tagName.toLowerCase()}${el.id ? `#${el.id}` : ""}${cls}`; +} + +function rootSize(root: HTMLElement): { width: number; height: number } { + const w = Number(root.getAttribute("data-width")) || root.offsetWidth; + const h = Number(root.getAttribute("data-height")) || root.offsetHeight; + return { width: Math.max(1, Math.round(w)), height: Math.max(1, Math.round(h)) }; +} + +function makeCanvas(width: number, height: number, role: string): HTMLCanvasElement { + const canvas = document.createElement("canvas"); + canvas.width = width; + canvas.height = height; + canvas.setAttribute("data-hf-ignore", ""); + canvas.setAttribute("data-hf-motion-blur", role); + // ponytail: assumes the root sits at its parent's origin (true for compositions); place the + // canvases at the root's offset if a layout ever puts it elsewhere. + canvas.style.cssText = `position:absolute;left:0;top:0;width:${width}px;height:${height}px;`; + return canvas; +} + +function createInstance(host: CaptureMotionBlurHost, config: Config) { + const { root } = host; + const { width, height } = rootSize(root); + const src = makeCanvas(width, height, "capture") as PaintCanvas; + const out = makeCanvas(width, height, "output"); + out.style.pointerEvents = "none"; + src.setAttribute("layoutsubtree", ""); + + const captureContext = src.getContext("2d") as DrawElementContext | null; + const outputContext = out.getContext("2d"); + if (!captureContext || !outputContext) throw new Error("capture motion blur: no 2D context"); + const sctx: DrawElementContext = captureContext; + const octx: CanvasRenderingContext2D = outputContext; + const compositor = createCompositor(width, height); + // Hides movers for the background capture without touching author or GSAP opacity. + const hideRule = document.createElement("style"); + hideRule.setAttribute("data-hf-ignore", ""); + hideRule.textContent = `[${HIDE_ATTR}] { opacity: 0 !important; }`; + + let movers: HTMLElement[] = []; + let reader: ReturnType | null = null; + let pending: Job | null = null; + // While a frame is in flight (two paints), newer requests collapse into the latest one. + let queued: { t: number; engineMode: boolean; release: () => void } | null = null; + let seq = 0; + let failures = 0; + let failedFrames = 0; + let disabled: string | null = null; + const timings: number[] = []; + let sampleMs = 0; + let scanMs = 0; + let scannedTimeline: unknown = undefined; + let wrapped = false; + // The output still shows the last frame of the previous wrap until this one draws. + let staleOutput = false; + // Once the engine drives frames, paused-style requests come from the transport, not a user. + let rendering = false; + + function wrap(): void { + if (wrapped) return; + root.before(src); + src.append(root); + src.after(out); + document.head.append(hideRule); + wrapped = true; + staleOutput = true; + } + + /** Back to plain DOM: drop any frame in flight, put the root where it was. */ + function unwrap(): void { + queued?.release(); + queued = null; + if (pending) { + reveal(pending); + pending.settle(); + pending = null; + } + if (!wrapped) return; + wrapped = false; + hideRule.remove(); + src.before(root); + src.remove(); + out.remove(); + } + + const seek = (t: number): void => host.seek(t, !config.events); + + function discover(restoreTo: number): void { + scannedTimeline = host.timeline(); + const s0 = performance.now(); + const found = discoverMovers({ + scene: root, + durationSeconds: host.durationSeconds(), + fps: host.fps, + seek, + }); + seek(restoreTo); + movers = found.movers; + reader = createPoseReader(root, found.dynamic); + scanMs = Math.round(performance.now() - s0); + } + + function frames(t: number, reads: number): { els: HTMLElement[]; frames: MoverFrame[] } { + const r = reader; + if (!r) return { els: [], frames: [] }; + const visible = movers.filter( + (el) => + typeof el.checkVisibility !== "function" || + el.checkVisibility({ visibilityProperty: true }), + ); + const shutter = config.shutterDeg / 360 / host.fps; + if (visible.length === 0 || shutter <= 0) return { els: [], frames: [] }; + const start = t - shutter / 2; + const poses = visible.map((): Affine[] => []); + for (let j = 0; j < reads; j++) { + seek(Math.max(0, start + (j / (reads - 1)) * shutter)); + r.reset(); + visible.forEach((el, i) => poses[i]?.push(r.pose(el))); + } + seek(t); + r.reset(); + const result = visible.map((el, i) => ({ + w: el.offsetWidth, + h: el.offsetHeight, + now: r.pose(el), + taps: tapsFromReads(poses[i] ?? [IDENTITY], config.samples), + })); + r.reset(); + return { els: visible, frames: result }; + } + + const capture = (): void => { + sctx.reset(); + sctx.drawElementImage(root, 0, 0); + }; + const reveal = (job: Job): void => { + for (const el of job.els) el.removeAttribute(HIDE_ATTR); + }; + + /** + * One step of the current frame, run on each paint (or by the engine's composite hook): + * first capture the movers and crop their pixels, then hide them and, on the next paint, + * capture the background and draw it with the trails. A frame with nothing moving is done + * after the first capture. `finalStep` forces the frame to finish now (no further paint will + * come), drawing trails over the sharp frame if the background was never captured. + */ + function step(finalStep = false): void { + const job = pending; + if (!job || disabled || !job.armed) return; + const t0 = performance.now(); + let done = true; + try { + done = captureAndDraw(job, finalStep); + failures = 0; + } catch (err) { + failures++; + swallow("runtime.captureMotionBlur.compose", err); + if (failures >= MAX_FAILURES) disable(`capture failed ${failures} times in a row`); + } + job.ms += performance.now() - t0; + if (!done && !disabled) { + src.requestPaint(); + return; + } + timings.push(sampleMs + job.ms); + if (timings.length > 600) timings.shift(); + finishJob(job); + } + + /** This paint's half of the frame; false when the background capture is still to come. */ + function captureAndDraw(job: Job, finalStep: boolean): boolean { + capture(); + try { + if (!job.kept) { + job.kept = compositor.crop(src, job.frames); + if (staleOutput) { + // First paint since wrapping: show this frame sharp until its trails are drawn. + octx.setTransform(1, 0, 0, 1, 0, 0); + octx.globalAlpha = 1; + octx.globalCompositeOperation = "copy"; + octx.drawImage(src, 0, 0); + staleOutput = false; + } + if (job.kept.some(Boolean) && !finalStep) { + job.els.forEach((el, i) => job.kept?.[i] && el.setAttribute(HIDE_ATTR, "")); + return false; + } + } + compositor.draw(src, octx, job.frames, job.kept); + return true; + } finally { + sctx.reset(); // the capture canvas sits under the output: keep its own bitmap empty + } + } + + /** End a frame (drawn or given up on) and start the newest frame requested meanwhile. */ + function finishJob(job: Job): void { + reveal(job); + if (pending === job) pending = null; + job.settle(); + const next = queued; + queued = null; + if (next && !disabled) void start(next.t, next.engineMode).then(next.release); + else next?.release(); + } + const onPaint = (): void => step(); + + function countFailedFrame(why: string): void { + failedFrames++; + if (failedFrames % LOG_EVERY === 1) + console.info(`[motion-blur] capture failed (${failedFrames} so far): ${why}`); + } + + // A tab that stops painting while the clock runs is not a broken capture: this frame is plain + // DOM (root back in the page, movers revealed), the next tries again, and timeouts never count + // toward giving up. + function onPaintTimeout(job: Job): void { + countFailedFrame(`no paint within ${PAINT_TIMEOUT_MS} ms`); + unwrap(); + finishJob(job); + } + + src.addEventListener("paint", onPaint); + + function disable(reason: string): void { + if (disabled) return; + disabled = reason; + console.warn(`[motion-blur] off: ${reason}`); + teardown(); + } + + function teardown(): void { + src.removeEventListener("paint", onPaint); + unwrap(); + } + + /** + * Request one blurred frame at `t`; resolves once it is drawn (or given up on). A frame still + * in flight (between its two paints) is not abandoned: the request waits as the newest pending + * one. An engine frame replaces it, since the engine waits on exactly this frame. + */ + function paint(t: number, engineMode: boolean): Promise { + if (disabled) return Promise.resolve(); + if (engineMode) rendering = true; + if (pending && !engineMode) { + const previous = queued; + return new Promise((resolve) => { + queued = { + t, + engineMode, + release: () => { + previous?.release(); + resolve(); + }, + }; + }); + } + return start(t, engineMode); + } + + function start(t: number, engineMode: boolean): Promise { + try { + if (!reader || host.timeline() !== scannedTimeline) discover(t); + if (movers.length === 0) { + unwrap(); + return Promise.resolve(); + } + // Checked again here, not only at init: a block's script can add its own capture canvas + // (wrapping one inside ours hangs the renderer) or a WebGL canvas later. + const blocked = captureBlocker(root); + if (blocked) { + disable(blocked); + return Promise.resolve(); + } + wrap(); + const s0 = performance.now(); + const next = frames(t, engineMode ? RENDER_READS : PREVIEW_READS); + sampleMs = performance.now() - s0; + if (pending) finishJob(pending); + return new Promise((resolve) => { + const mine = ++seq; + let timer: ReturnType | undefined; + const startTimer = (): void => { + timer = setTimeout(() => { + if (pending?.seq === mine) onPaintTimeout(pending); + else resolve(); + }, PAINT_TIMEOUT_MS); + }; + pending = { + seq: mine, + els: next.els, + frames: next.frames, + kept: null, + armed: !engineMode, + startTimer, + ms: 0, + settle: () => { + clearTimeout(timer); + resolve(); + }, + }; + if (engineMode) return; + startTimer(); + src.requestPaint(); + }); + } catch (err) { + swallow("runtime.captureMotionBlur.paint", err); + disable(err instanceof Error ? err.message : String(err)); + return Promise.resolve(); + } + } + + function report(): Report { + const mean = timings.length ? timings.reduce((s, x) => s + x, 0) / timings.length : 0; + return { + state: disabled ? "off" : "on", + reason: disabled, + movers: movers.length, + moverNames: movers.map(describe), + scanMs, + shutterDeg: config.shutterDeg, + frames: timings.length, + meanMs: Math.round(mean * 10) / 10, + failures, + failedFrames, + }; + } + + return { + root, + paint, + /** Paused in preview: show plain DOM so it can be selected. A render never rests. */ + rest: (): boolean => { + if (rendering) return false; + unwrap(); + return true; + }, + arm: () => { + if (!pending || pending.armed) return; + pending.armed = true; + pending.startTimer(); + src.requestPaint(); + }, + /** Finish the current frame now: the engine's forced paint just ran. */ + finish: () => { + if (pending) pending.armed = true; + step(true); + }, + teardown, + report, + busy: (): boolean => pending !== null, + rediscover: () => { + reader = null; + }, + }; +} + +type Instance = ReturnType; +let instance: Instance | null = null; +let priorResolver: (() => boolean) | null = null; +let ourResolver: (() => boolean) | null = null; + +/** + * Read the root's opt-in and wrap (or unwrap) it. Called wherever the runtime (re)initialises + * scene DOM; a repeat call on the same root only schedules a fresh mover scan. + */ +export function initCaptureMotionBlur(host: CaptureMotionBlurHost): void { + const config = readConfig(host.root); + const w = window as CompositeWindow; + if (instance && (instance.root !== host.root || !config)) { + instance.teardown(); + instance = null; + delete w.__hfMotionBlur; + } + if (!config) return; + if (instance) { + instance.rediscover(); + return; + } + const reason = unsupportedReason(host.root); + if (reason) { + console.info(`[motion-blur] off: ${reason}`); + w.__hfMotionBlur = { + report: () => ({ + state: "off", + reason, + movers: 0, + moverNames: [], + scanMs: 0, + shutterDeg: config.shutterDeg, + frames: 0, + meanMs: 0, + failures: 0, + failedFrames: 0, + }), + }; + return; + } + instance = createInstance(host, config); + const current = instance; + w.__hfMotionBlur = { report: () => current.report() }; +} + +/** + * Draw the blurred frame for `t` after a seek or a playback tick. In preview only playback blurs: + * a paused request (no `playing`) unwraps the composition instead and returns null. In engine + * mode it also claims the engine's + * page-composite step, composing with whoever already holds it, so a render frame is complete + * before the screenshot even if no paint event fired. Returns the frame's completion. + */ +export function paintCaptureMotionBlur( + t: number, + options?: { engineMode?: boolean; playing?: boolean }, +): Promise | null { + if (!instance) return null; + const engineMode = options?.engineMode === true; + if (!engineMode && !options?.playing && instance.rest()) return null; + const done = instance.paint(t, engineMode); + // Nothing in flight (nothing moves, or blur turned off): leave the engine's composite step idle. + if (engineMode && instance.busy()) armPageComposite(); + return done; +} + +/** Start capturing a deferred engine frame: the page (injected video frames included) is final. */ +export function releaseCaptureMotionBlurFrame(): void { + instance?.arm(); +} + +function armPageComposite(): void { + const w = window as CompositeWindow; + const current = w.__hf_page_composite_resolve; + if (current !== ourResolver) { + priorResolver = typeof current === "function" ? current : null; + ourResolver = () => { + if (priorResolver) priorResolver(); + // The engine's forced paint has just run, so the paint records are valid here even if + // no paint event reached us; compose() is a no-op when the event already drew the frame. + instance?.finish(); + w.__hf_page_composite_pending = false; + return true; + }; + w.__hf_page_composite_resolve = ourResolver; + } + w.__hf_page_composite_pending = true; +} + +/** Test-only: drop module state between cases. */ +export function resetCaptureMotionBlurForTests(): void { + instance?.teardown(); + instance = null; + priorResolver = null; + ourResolver = null; +} diff --git a/packages/core/src/runtime/captureMotionBlur/pose.test.ts b/packages/core/src/runtime/captureMotionBlur/pose.test.ts new file mode 100644 index 00000000000..42c8967108d --- /dev/null +++ b/packages/core/src/runtime/captureMotionBlur/pose.test.ts @@ -0,0 +1,230 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { + type Affine, + IDENTITY, + corners, + createPoseReader, + individualIs3d, + individualTransform, + interpolate, + invert, + multiply, + parseTransform, + sweep, + tapsFromReads, + thinTaps, + translate, +} from "./pose"; + +const rot = (deg: number, e = 0, f = 0): Affine => { + const r = (deg * Math.PI) / 180; + return { a: Math.cos(r), b: Math.sin(r), c: -Math.sin(r), d: Math.cos(r), e, f }; +}; +const scaleOf = (m: Affine): number => Math.hypot(m.a, m.b); +const angleOf = (m: Affine): number => (Math.atan2(m.b, m.a) * 180) / Math.PI; + +describe("pose math", () => { + it("interpolates a rotation without shrinking it", () => { + // Entry-wise lerp halfway through a 90° turn gives scale cos(45°) ≈ 0.707. + const mid = interpolate(rot(0), rot(90), 0.5); + expect(scaleOf(mid)).toBeCloseTo(1, 6); + expect(angleOf(mid)).toBeCloseTo(45, 6); + }); + + it("rotates the short way round", () => { + expect(angleOf(interpolate(rot(350), rot(10), 0.5))).toBeCloseTo(0, 6); + }); + + it("interpolates translation and scale linearly", () => { + const p: Affine = { a: 1, b: 0, c: 0, d: 1, e: 0, f: 0 }; + const q: Affine = { a: 3, b: 0, c: 0, d: 2, e: 100, f: -40 }; + expect(interpolate(p, q, 0.25)).toMatchObject({ a: 1.5, d: 1.25, e: 25, f: -10 }); + }); + + it("inverts an affine", () => { + const m = multiply(rot(30, 12, -7), { a: 2, b: 0, c: 0.5, d: 3, e: 4, f: 5 }); + const back = invert(m); + expect(back).not.toBeNull(); + const id = multiply(m, back as Affine); + for (const [k, v] of Object.entries(IDENTITY)) expect(id[k as keyof Affine]).toBeCloseTo(v, 9); + }); + + it("refuses to invert a collapsed box", () => { + expect(invert({ a: 0, b: 0, c: 0, d: 1, e: 0, f: 0 })).toBeNull(); + }); + + it("parses computed 2D and 3D transforms", () => { + const values = (css: string) => Object.values(parseTransform(css) ?? {}); + expect(values("matrix(1, 2, 3, 4, 5, 6)")).toEqual([1, 2, 3, 4, 5, 6]); + expect(values("matrix3d(1,0,0,0, 0,1,0,0, 0,0,1,0, 7,8,9,1)")).toEqual([1, 0, 0, 1, 7, 8]); + expect(parseTransform("none")).toBeNull(); + expect(parseTransform("matrix(1, 2)")).toBeNull(); + }); +}); + +describe("taps", () => { + it("spreads taps across the shutter, centred in each slice", () => { + const reads = [translate(IDENTITY, 0, 0), translate(IDENTITY, 100, 0)]; + const xs = tapsFromReads(reads, 4).map((m) => m.e); + expect(xs.map((x) => Math.round(x * 100) / 100)).toEqual([12.5, 37.5, 62.5, 87.5]); + }); + + it("measures the fastest corner's path and the swept rect", () => { + const taps = [ + translate(IDENTITY, 0, 0), + translate(IDENTITY, 30, 0), + translate(IDENTITY, 60, 0), + ]; + const { travel, rect } = sweep(taps, 10, 10, 1920, 1080); + expect(travel).toBeCloseTo(60, 6); + expect(rect).toEqual({ x: 0, y: 0, w: 72, h: 12 }); + }); + + it("reports an off-frame sweep as no rect", () => { + const taps = [translate(IDENTITY, -500, -500), translate(IDENTITY, -400, -500)]; + expect(sweep(taps, 10, 10, 1920, 1080).rect).toBeNull(); + }); + + it("thins taps to about one per 1.5 px, never below two", () => { + const taps = tapsFromReads([IDENTITY, translate(IDENTITY, 300, 0)], 32); + expect(thinTaps(taps, 3)).toHaveLength(2); + expect(thinTaps(taps, 15)).toHaveLength(10); + expect(thinTaps(taps, 300)).toHaveLength(32); + }); + + it("places a rotated box's corners", () => { + const [, tr] = corners(rot(90), 10, 0); + expect(tr?.[0]).toBeCloseTo(0, 9); + expect(tr?.[1]).toBeCloseTo(10, 9); + }); +}); + +const extra = new Map>(); + +describe("createPoseReader", () => { + afterEach(() => { + vi.restoreAllMocks(); + extra.clear(); + }); + + function place(el: HTMLElement, left: number, top: number, parent: Element | null): void { + Object.defineProperty(el, "offsetLeft", { value: left, configurable: true }); + Object.defineProperty(el, "offsetTop", { value: top, configurable: true }); + Object.defineProperty(el, "offsetParent", { value: parent, configurable: true }); + } + + function tree() { + const scene = document.createElement("div"); + const group = document.createElement("div"); + const mover = document.createElement("div"); + scene.append(group); + group.append(mover); + place(group, 100, 50, scene); + place(mover, 10, 20, group); + const transforms = new Map([ + [group, "matrix(2, 0, 0, 2, 0, 0)"], + [mover, "none"], + ]); + vi.spyOn(window, "getComputedStyle").mockImplementation( + (el: Element) => + ({ + transform: transforms.get(el) ?? "none", + transformOrigin: "0px 0px", + ...extra.get(el), + }) as CSSStyleDeclaration, + ); + return { scene, group, mover, transforms }; + } + + it("composes layout offsets with every transform up the chain", () => { + const { scene, mover } = tree(); + const reader = createPoseReader(scene, new Set([mover])); + // group at (100,50) scaled 2×, mover at (10,20) inside it → (120, 90), scale 2. + expect(reader.pose(mover)).toEqual({ a: 2, b: 0, c: 0, d: 2, e: 120, f: 90 }); + }); + + it("re-reads a dynamic node's transform after reset, but caches a static one", () => { + const { scene, group, mover, transforms } = tree(); + const reader = createPoseReader(scene, new Set([mover])); + reader.pose(mover); + transforms.set(mover, "matrix(1, 0, 0, 1, 5, 0)"); + transforms.set(group, "matrix(1, 0, 0, 1, 0, 0)"); + reader.reset(); + // The mover's new 5 px shift lands (in the still-cached 2× group), the group's change does not. + expect(reader.pose(mover)).toMatchObject({ a: 2, e: 130, f: 90 }); + }); + + it("puts them into a pose, before the transform property and about its origin", () => { + const { scene, group, mover } = tree(); + extra.set(group, { scale: "2" }); + extra.set(mover, { translate: "-50% -50%" }); + Object.defineProperty(mover, "offsetWidth", { value: 20, configurable: true }); + Object.defineProperty(mover, "offsetHeight", { value: 40, configurable: true }); + const reader = createPoseReader(scene, new Set()); + // group: layout (100,50), scale 2 from the origin, then its transform scale 2 → 4×. + // mover: layout (10,20) in the group, shifted (-10,-20): at the group's origin. + expect(reader.pose(mover)).toEqual({ a: 4, b: 0, c: 0, d: 4, e: 100, f: 50 }); + }); + + it.each([ + ["translate", "120px 0px", "matrix(1, 0, 0, 1, 120, 0)"], + ["translate", "180px 0px", "matrix(1, 0, 0, 1, 180, 0)"], + ["rotate", "90deg", "matrix(0, 1, -1, 0, 0, 0)"], + ["scale", "2 3", "matrix(2, 0, 0, 3, 0, 0)"], + ])("poses `%s: %s` like the equivalent matrix transform", (prop, value, matrix) => { + const { scene, mover, transforms } = tree(); + const reader = createPoseReader(scene, new Set([mover])); + transforms.set(mover, matrix); + const control = reader.pose(mover); + transforms.set(mover, "none"); + extra.set(mover, { [prop]: value }); + reader.reset(); + const pose = reader.pose(mover); + for (const k of ["a", "b", "c", "d", "e", "f"] as const) + expect(pose[k]).toBeCloseTo(control[k], 9); + }); + + it("measures from the offsetParent's border and subtracts scroll", () => { + const { scene, group, mover } = tree(); + Object.defineProperty(group, "clientLeft", { value: 5, configurable: true }); + Object.defineProperty(group, "clientTop", { value: 3, configurable: true }); + Object.defineProperty(group, "scrollTop", { value: 7, configurable: true }); + const reader = createPoseReader(scene, new Set()); + // group's 2× transform scales mover's in-group offset (10 + 5, 20 + 3 - 7) → (30, 32). + expect(reader.pose(mover)).toMatchObject({ e: 130, f: 82 }); + }); +}); + +describe("individual transform properties", () => { + it("centres a box with translate percentages of its own size", () => { + expect(individualTransform({ translate: "-50% -50%" }, 100, 60)).toEqual({ + ...IDENTITY, + e: -50, + f: -30, + }); + expect(individualTransform({ translate: "calc(50% + 10px) 5px" }, 100, 60)).toMatchObject({ + e: 60, + f: 5, + }); + }); + + it("applies translate, then rotate, then scale", () => { + const m = individualTransform({ translate: "10px 0px", rotate: "90deg", scale: "2" }, 10, 10); + // A point (1, 0): scaled to (2, 0), rotated to (0, 2), shifted to (10, 2). + expect(m && [m.a + m.e, m.b + m.f]).toEqual([expect.closeTo(10, 9), expect.closeTo(2, 9)]); + }); + + it("is null when none is set", () => { + expect(individualTransform({ translate: "none", rotate: "none", scale: "none" }, 1, 1)).toBe( + null, + ); + expect(individualTransform({}, 1, 1)).toBe(null); + }); + + it("flags a z translate or a rotate axis as 3D", () => { + expect(individualIs3d({ translate: "10px 20px 5px" })).toBe(true); + expect(individualIs3d({ rotate: "x 30deg" })).toBe(true); + expect(individualIs3d({ rotate: "1 1 0 57.2958deg" })).toBe(true); + expect(individualIs3d({ translate: "10px 20px", rotate: "30deg", scale: "2" })).toBe(false); + }); +}); diff --git a/packages/core/src/runtime/captureMotionBlur/pose.ts b/packages/core/src/runtime/captureMotionBlur/pose.ts new file mode 100644 index 00000000000..84956767916 --- /dev/null +++ b/packages/core/src/runtime/captureMotionBlur/pose.ts @@ -0,0 +1,337 @@ +/** + * Pose math for capture motion blur: where an element's box sits in composition space at a + * given timeline time, and the in-between poses the shutter sweeps through. + * + * A pose is the full 2D affine from the element's own border box to the composition root: + * layout offsets plus every CSS transform up the ancestor chain. Reading it costs style and + * layout reads only; no paint. + */ + +export interface Affine { + a: number; + b: number; + c: number; + d: number; + e: number; + f: number; +} + +export const IDENTITY: Affine = { a: 1, b: 0, c: 0, d: 1, e: 0, f: 0 }; + +export function multiply(m: Affine, n: Affine): Affine { + return { + a: m.a * n.a + m.c * n.b, + b: m.b * n.a + m.d * n.b, + c: m.a * n.c + m.c * n.d, + d: m.b * n.c + m.d * n.d, + e: m.a * n.e + m.c * n.f + m.e, + f: m.b * n.e + m.d * n.f + m.f, + }; +} + +export function translate(m: Affine, x: number, y: number): Affine { + return multiply(m, { a: 1, b: 0, c: 0, d: 1, e: x, f: y }); +} + +export function invert(m: Affine): Affine | null { + const det = m.a * m.d - m.b * m.c; + if (!Number.isFinite(det) || Math.abs(det) < 1e-12) return null; + return { + a: m.d / det, + b: -m.b / det, + c: -m.c / det, + d: m.a / det, + e: (m.c * m.f - m.d * m.e) / det, + f: (m.b * m.e - m.a * m.f) / det, + }; +} + +/** The four corners of a w×h box under `m`. */ +export function corners(m: Affine, w: number, h: number): Array<[number, number]> { + return [ + [m.e, m.f], + [m.a * w + m.e, m.b * w + m.f], + [m.c * h + m.e, m.d * h + m.f], + [m.a * w + m.c * h + m.e, m.b * w + m.d * h + m.f], + ]; +} + +/** + * Interpolate two poses by translation, rotation, scale and shear rather than matrix entries: + * entry-wise lerp of a rotating box shrinks it mid-segment (halfway through a 90° turn it is + * scaled by cos 45°). Rotation takes the short way round. + */ +export function interpolate(p: Affine, q: Affine, f: number): Affine { + const dp = decompose(p); + const dq = decompose(q); + let turn = dq.angle - dp.angle; + if (turn > Math.PI) turn -= 2 * Math.PI; + else if (turn < -Math.PI) turn += 2 * Math.PI; + const lerp = (x: number, y: number): number => x + (y - x) * f; + return compose({ + angle: dp.angle + turn * f, + sx: lerp(dp.sx, dq.sx), + sy: lerp(dp.sy, dq.sy), + shear: lerp(dp.shear, dq.shear), + e: lerp(p.e, q.e), + f: lerp(p.f, q.f), + }); +} + +interface Decomposed { + angle: number; + sx: number; + sy: number; + shear: number; + e: number; + f: number; +} + +// QR split of the linear part: first column = sx·(cos, sin), second = shear·(cos, sin) + sy·(−sin, cos). +function decompose(m: Affine): Decomposed { + const sx = Math.hypot(m.a, m.b); + if (sx < 1e-12) return { angle: 0, sx: 0, sy: m.d, shear: m.c, e: m.e, f: m.f }; + return { + angle: Math.atan2(m.b, m.a), + sx, + sy: (m.a * m.d - m.b * m.c) / sx, + shear: (m.a * m.c + m.b * m.d) / sx, + e: m.e, + f: m.f, + }; +} + +function compose(d: Decomposed): Affine { + const cos = Math.cos(d.angle); + const sin = Math.sin(d.angle); + return { + a: d.sx * cos, + b: d.sx * sin, + c: d.shear * cos - d.sy * sin, + d: d.shear * sin + d.sy * cos, + e: d.e, + f: d.f, + }; +} + +/** + * `samples` taps spread evenly across the shutter, interpolated from `reads` poses read at + * evenly spaced times from the start to the end of the shutter (endpoints included). + */ +export function tapsFromReads(reads: Affine[], samples: number): Affine[] { + if (reads.length === 0) return []; + if (reads.length === 1) return Array.from({ length: samples }, () => reads[0] as Affine); + const last = reads.length - 1; + return Array.from({ length: samples }, (_, k) => { + const x = ((k + 0.5) / samples) * last; + const j = Math.min(last - 1, Math.floor(x)); + return interpolate(reads[j] as Affine, reads[j + 1] as Affine, x - j); + }); +} + +export interface Sweep { + /** Path length of the box's fastest corner across the taps, in pixels. */ + travel: number; + /** Pixel rect covering every tap, clipped to the frame; null when off-frame. */ + rect: { x: number; y: number; w: number; h: number } | null; +} + +export function sweep(taps: Affine[], w: number, h: number, frameW: number, frameH: number): Sweep { + let x0 = Infinity; + let y0 = Infinity; + let x1 = -Infinity; + let y1 = -Infinity; + let travel = 0; + let prev: Array<[number, number]> | null = null; + for (const m of taps) { + const cs = corners(m, w, h); + for (const [x, y] of cs) { + x0 = Math.min(x0, x); + y0 = Math.min(y0, y); + x1 = Math.max(x1, x); + y1 = Math.max(y1, y); + } + if (prev) { + let step = 0; + for (let j = 0; j < 4; j++) { + const [px, py] = prev[j] as [number, number]; + const [cx, cy] = cs[j] as [number, number]; + step = Math.max(step, Math.hypot(cx - px, cy - py)); + } + travel += step; + } + prev = cs; + } + // 2 px of slack for bilinear filtering at the edges. + const rx0 = Math.max(0, Math.floor(x0) - 2); + const ry0 = Math.max(0, Math.floor(y0) - 2); + const rx1 = Math.min(frameW, Math.ceil(x1) + 2); + const ry1 = Math.min(frameH, Math.ceil(y1) + 2); + const rect = rx1 > rx0 && ry1 > ry0 ? { x: rx0, y: ry0, w: rx1 - rx0, h: ry1 - ry0 } : null; + return { travel, rect }; +} + +/** About one tap per `spacing` px of travel (no visible steps), at least 2, at most all. */ +export function thinTaps(taps: Affine[], travel: number, spacing = 1.5): Affine[] { + const n = Math.min(taps.length, Math.max(2, Math.ceil(travel / spacing))); + if (n >= taps.length) return taps; + return Array.from({ length: n }, (_, i) => { + const k = Math.round(((i + 0.5) / n) * taps.length - 0.5); + return taps[k] as Affine; + }); +} + +/** + * Reads poses relative to `scene`, memoised for one timeline time: movers share ancestors, so + * each node's matrix is computed once from its parent's. Call `reset()` after every seek. + * + * `dynamic` names the nodes whose transform can change over time; everyone else's own + * transform is read once and cached for the reader's lifetime. + */ +export function createPoseReader(scene: HTMLElement, dynamic: ReadonlySet) { + const staticOwn = new Map(); + let positions = new Map(); + let matrices = new Map(); + + const ownTransform = (n: HTMLElement): Affine | null => { + const cached = !dynamic.has(n) && staticOwn.has(n); + if (cached) return staticOwn.get(n) ?? null; + const cs = getComputedStyle(n); + const t = multiplyAll([ + individualTransform(cs, n.offsetWidth, n.offsetHeight), + parseTransform(cs.transform), + ]); + let own: Affine | null = null; + if (t) { + const [ox = 0, oy = 0] = cs.transformOrigin.split(" ").map((v) => parseFloat(v) || 0); + own = translate(multiply(translate(IDENTITY, ox, oy), t), -ox, -oy); + } + if (!dynamic.has(n)) staticOwn.set(n, own); + return own; + }; + + // Untransformed border-box position relative to the scene, along the offsetParent chain. + // offsetLeft/Top measure from the offsetParent's padding edge, so its border is added back. + const layoutPos = (n: Element | null): [number, number] => { + if (!n || n === scene || !isOffsetElement(n)) return [0, 0]; + let p = positions.get(n); + if (!p) { + const op = n.offsetParent; + const [px, py] = layoutPos(op); + p = [px + n.offsetLeft + (op?.clientLeft ?? 0), py + n.offsetTop + (op?.clientTop ?? 0)]; + positions.set(n, p); + } + return p; + }; + + const matrixOf = (n: Element | null): Affine => { + if (!n || n === scene || !isOffsetElement(n)) return IDENTITY; + let m = matrices.get(n); + if (!m) { + const parent = n.parentElement; + const [x, y] = layoutPos(n); + const [px, py] = layoutPos(parent); + const [sx, sy] = scrollOf(parent); + m = translate(matrixOf(parent), x - px - sx, y - py - sy); + const own = ownTransform(n); + if (own) m = multiply(m, own); + matrices.set(n, m); + } + return m; + }; + + return { + pose: matrixOf as (el: HTMLElement) => Affine, + reset(): void { + positions = new Map(); + matrices = new Map(); + }, + }; +} + +function scrollOf(el: Element | null): [number, number] { + return el ? [el.scrollLeft, el.scrollTop] : [0, 0]; +} + +function multiplyAll(ms: Array): Affine | null { + let out: Affine | null = null; + for (const m of ms) if (m) out = out ? multiply(out, m) : m; + return out; +} + +type IndividualProps = Pick; + +// Splits at spaces outside parentheses (`calc(50% + 10px) 5%` is two tokens). +const tokens = (value: string): string[] => value.trim().split(/\s+(?![^(]*\))/); + +// A computed length or percentage (`calc(50% + 10px)` included) in px against `basis`. +function lengthPx(token: string, basis: number): number { + let sum = 0; + for (const m of token.matchAll(/([+-]?\s*[\d.]+(?:e[+-]?\d+)?)(px|%)?/g)) { + const v = Number(m[1]?.replace(/\s/g, "")); + sum += m[2] === "%" ? (v / 100) * basis : v; + } + return sum; +} + +const ANGLE_UNIT: Record = { + deg: Math.PI / 180, + rad: 1, + grad: Math.PI / 200, + turn: 2 * Math.PI, +}; + +/** + * The `translate`, `rotate` and `scale` properties as one 2D affine, in the CSS order (translate, + * then rotate, then scale; `transform` comes after). Null when none is set. + */ +export function individualTransform( + cs: Partial, + width: number, + height: number, +): Affine | null { + const t = cs.translate ?? "none"; + const r = cs.rotate ?? "none"; + const s = cs.scale ?? "none"; + const parts: Array = []; + if (t !== "none") { + const [x = "0", y = "0"] = tokens(t); + parts.push({ ...IDENTITY, e: lengthPx(x, width), f: lengthPx(y, height) }); + } + const turn = /^(-?[\d.]+)(deg|rad|grad|turn)$/.exec(r); + if (turn) { + const a = Number(turn[1]) * (ANGLE_UNIT[turn[2] ?? "deg"] ?? 0); + parts.push({ a: Math.cos(a), b: Math.sin(a), c: -Math.sin(a), d: Math.cos(a), e: 0, f: 0 }); + } + if (s !== "none") { + const [sx = 1, sy = sx] = tokens(s).map(Number); + parts.push({ ...IDENTITY, a: sx, d: sy }); + } + return multiplyAll(parts); +} + +/** Whether `translate` has a z part or `rotate` an axis: a 3D move the 2D pose cannot track. */ +export function individualIs3d(cs: Partial): boolean { + const t = cs.translate ?? "none"; + const r = cs.rotate ?? "none"; + return (t !== "none" && tokens(t).length > 2) || !/^(none|-?[\d.]+[a-z]+)$/.test(r); +} + +/** + * A computed `transform` (always `none`, `matrix(...)` or `matrix3d(...)` once resolved) as its + * 2D affine part; null for `none` or anything unparsable. + */ +export function parseTransform(value: string): Affine | null { + const match = /^matrix(3d)?\(([^)]*)\)$/.exec(value.trim()); + if (!match) return null; + const v = (match[2] ?? "").split(",").map(Number); + if (v.some((x) => !Number.isFinite(x))) return null; + const pick = match[1] ? [0, 1, 4, 5, 12, 13] : [0, 1, 2, 3, 4, 5]; + if (v.length !== (match[1] ? 16 : 6)) return null; + const [a = 1, b = 0, c = 0, d = 1, e = 0, f = 0] = pick.map((i) => v[i] ?? 0); + return { a, b, c, d, e, f }; +} + +function isOffsetElement(n: Element): n is HTMLElement { + return typeof (n as HTMLElement).offsetLeft === "number"; +} diff --git a/packages/core/src/runtime/entry.test.ts b/packages/core/src/runtime/entry.test.ts index d54f98505f7..5f64f36305c 100644 --- a/packages/core/src/runtime/entry.test.ts +++ b/packages/core/src/runtime/entry.test.ts @@ -96,6 +96,23 @@ const neverDecodes = (clip: HTMLElement) => { describe("runtime entry", () => { afterEach(resetRuntimeGlobals); + it("starts a held capture-motion-blur frame when capture waits on the seek, after video injection", async () => { + const release = vi.fn(); + vi.doMock("./captureMotionBlur", async (original) => ({ + ...(await original()), + releaseCaptureMotionBlurFrame: release, + })); + try { + mountRoot().appendChild(document.createElement("div")); + await evaluateRuntime(); + expect(release).not.toHaveBeenCalled(); + await window.__hfWaitForSeekCompletion?.(); + expect(release).toHaveBeenCalledTimes(1); + } finally { + vi.doUnmock("./captureMotionBlur"); + } + }); + it("drives an async frame source without a GSAP timeline and waits before capture", async () => { const root = mountRoot(); root.setAttribute("data-duration", "10"); diff --git a/packages/core/src/runtime/entry.ts b/packages/core/src/runtime/entry.ts index 6f34c13baf0..236222cfb75 100644 --- a/packages/core/src/runtime/entry.ts +++ b/packages/core/src/runtime/entry.ts @@ -17,6 +17,7 @@ import { runScriptsAfterFonts } from "./afterFonts"; import { AFTER_FONTS_SCRIPTS } from "../compiler/scriptRuns"; import { hasFrameSources, registerFrameSource } from "./frameSources"; import { createFilmBridge } from "./filmBridge"; +import { trackAcceleratedCanvases } from "./captureMotionBlur"; type HyperframeWindow = Window & { __hyperframeRuntimeBootstrapped?: boolean; @@ -43,6 +44,7 @@ type HyperframeWindow = Window & { installAuthoredOpacityCapture(); installAuthoredMediaCapture(); installFlatGsapTransforms(); +trackAcceleratedCanvases(); window.__hfHasFrameSources = hasFrameSources; hideTimedClipsUntilFirstPass(); diff --git a/packages/core/src/runtime/init.ts b/packages/core/src/runtime/init.ts index fab9328bc38..109e5864706 100644 --- a/packages/core/src/runtime/init.ts +++ b/packages/core/src/runtime/init.ts @@ -125,6 +125,11 @@ import { applyVariableBindings, unproxiedMediaSrc } from "./applyVariableBinding import { createColorGradingRuntime, type RuntimeColorGradingApi } from "./colorGrading"; import { COLOR_GRADING_AUTHORED_OPACITY_ATTR } from "../colorGrading"; import { initVfx, paintVfx } from "./vfx"; +import { + initCaptureMotionBlur, + paintCaptureMotionBlur, + releaseCaptureMotionBlurFrame, +} from "./captureMotionBlur"; import { TransportClock } from "./clock"; import { WebAudioTransport } from "./webAudioTransport"; import { @@ -3659,6 +3664,24 @@ export function initSandboxRuntimeModular(): void { // Passes that must see scene DOM, which arrives after init: when scenes mount, or when one is // swapped. Resolves when caption overrides have landed. + // Capture motion blur (data-motion-blur="on" on the root). Its pose reads seek the timeline + // and adapters only: media and transport bookkeeping stay on the frame's seek. + function initMotionBlur(): void { + const root = resolveRootCompositionElement(); + if (!root) return; + initCaptureMotionBlur({ + root, + fps: state.canonicalFps, + durationSeconds: () => + getSafeTimelineDurationSeconds(state.capturedTimeline, 0) || + Number(root.getAttribute("data-duration")) || + 0, + timeline: () => state.capturedTimeline, + seek: (t, suppressEvents) => { + seekRootChildrenAndAdapters(state.capturedTimeline, t, { suppressEvents }); + }, + }); + } const settleSceneDom = (): Promise => { bindMediaMetadataListeners(); installAssetFailureDiagnostics(); @@ -3667,6 +3690,7 @@ export function initSandboxRuntimeModular(): void { applyVariableBindings(document); // An unregistered vfx chain paints nothing and logs nothing, so re-scan the new DOM. initVfx(document.body, state.canonicalFps); + initMotionBlur(); return captionsApplied; }; if (!externalCompositionsReady) { @@ -4012,6 +4036,7 @@ export function initSandboxRuntimeModular(): void { } }; initVfx(document.body, state.canonicalFps); + initMotionBlur(); applyFetchedCaptionOverrides(captionOverrides, captionHosts, rewindCaptionTimelines); releaseDetachedMedia(); childrenBound = false; @@ -4064,6 +4089,7 @@ export function initSandboxRuntimeModular(): void { }); // Per-pixel effect chains: compile once here, repaint on every seek below. initVfx(document.body, state.canonicalFps); + initMotionBlur(); const applyPlaybackRate = (nextRate: number) => { const parsed = Number(nextRate); @@ -4120,6 +4146,8 @@ export function initSandboxRuntimeModular(): void { } colorGrading.redraw(); paintVfx(state.currentTime); + const blurred = paintCaptureMotionBlur(state.currentTime); + if (blurred) registerSeekCompletion(blurred); postState(true); }; @@ -4172,6 +4200,8 @@ export function initSandboxRuntimeModular(): void { runAdapters("pause"); syncMediaForCurrentState(); colorGrading.redraw(); + // Paused: back to plain DOM so Studio can select inside the composition. + void paintCaptureMotionBlur(state.currentTime); postState(true); }, seek: (timeSeconds, options) => { @@ -4232,6 +4262,8 @@ export function initSandboxRuntimeModular(): void { syncMediaForCurrentState(); colorGrading.redraw(); paintVfx(state.currentTime, { engineMode: true }); + const blurred = paintCaptureMotionBlur(state.currentTime, { engineMode: true }); + if (blurred) registerSeekCompletion(blurred); postState(true); }, getTime: () => heldSeek?.time ?? clock.now(), @@ -4340,9 +4372,15 @@ export function initSandboxRuntimeModular(): void { window.__hfTypegpuTime = t; forceDispatchSeekEvent(t); }; - window.__hfWaitForSeekCompletion = waitForSeekCompletion; + // Engine and CLI capture wait here once the page is final (the engine's injected video frames + // land after renderSeek), so a blurred engine frame is captured now, not at the seek. + const waitForFinalFrame: typeof waitForSeekCompletion = (options) => { + releaseCaptureMotionBlurFrame(); + return waitForSeekCompletion(options); + }; + window.__hfWaitForSeekCompletion = waitForFinalFrame; runtimeCleanupCallbacks.push(() => { - if (window.__hfWaitForSeekCompletion === waitForSeekCompletion) { + if (window.__hfWaitForSeekCompletion === waitForFinalFrame) { delete window.__hfWaitForSeekCompletion; } }); @@ -5151,6 +5189,8 @@ export function initSandboxRuntimeModular(): void { state.capturedTimeline !== lastTransportSeekTimeline ) { seekTimelineAndAdapters(t, { revision: timingRevision }); + // Playback: draw and move on; nothing waits on this frame's blur. Paused, it unwraps. + void paintCaptureMotionBlur(t, { playing: isPlaying }); if (!isPlaying) pausedSeekDeferredByManualGesture = false; } if (isPlaying) { @@ -5173,6 +5213,7 @@ export function initSandboxRuntimeModular(): void { } runAdapters("pause"); syncMediaForCurrentState(timingRevision); + void paintCaptureMotionBlur(state.currentTime); postState(true); return; }