feat(core,engine): track accelerated canvases before any page script runs

packages/core/src/compiler/htmlDocument.test.ts

@@ -8,6 +8,7 @@ import {
   injectScriptsIntoHtml,
   injectTagsAtHeadStart,
   insertBeforeCloseTag,
+  insertRuntimeTag,
   hasCompositionOutsideTemplates,
   isFullHtmlDocument,
   parseHTMLContent,
@@ -454,3 +455,29 @@ describe("hasCompositionOutsideTemplates", () => {
     expect(hasCompositionOutsideTemplates(html)).toBe(expected);
   });
 });
+
+describe("insertRuntimeTag", () => {
+  const runtime = '<script data-hf-runtime src="/rt.js"></script>';
+  const page = (attrs: string) =>
+    `<html><head><script>window.author=1</script></head><body><div id="root" ${attrs}></div></body></html>`;
+
+  it("puts the canvas tracker ahead of author head scripts for a page that opts into blur", () => {
+    const out = insertRuntimeTag(page('data-motion-blur="on"'), runtime);
+    const tracker = out.indexOf("data-hf-accel-tracker");
+    expect(tracker).toBeGreaterThan(-1);
+    expect(tracker).toBeLessThan(out.indexOf("window.author"));
+    expect(out.indexOf("window.author")).toBeLessThan(out.indexOf(runtime));
+  });
+
+  it("adds the tracker once, however many times the runtime is placed", () => {
+    const once = insertRuntimeTag(page('data-motion-blur="on"'), runtime);
+    const twice = insertRuntimeTag(once, runtime);
+    expect(twice.split("data-hf-accel-tracker").length).toBe(2);
+  });
+
+  it("leaves pages that did not opt in alone", () => {
+    for (const attrs of ["", 'data-motion-blur="off"', 'data-motion-blur-shutter="on"']) {
+      expect(insertRuntimeTag(page(attrs), runtime)).not.toContain("data-hf-accel-tracker");
+    }
+  });
+});

packages/core/src/compiler/htmlDocument.ts

@@ -1,3 +1,4 @@
+import { acceleratedCanvasTrackerSource } from "../pageMotionBlur.js";
 import { parseHTML } from "linkedom";
 
 export const RUNTIME_BOOTSTRAP_ATTR = "data-hyperframes-preview-runtime";
@@ -347,7 +348,7 @@ export function insertBeforeCloseTag(
   return insertBeforeDocumentTag(html, `</${name}`, markup);
 }
 
-export function insertRuntimeTag(html: string, tag: string): string {
+function insertTagInHead(html: string, tag: string): string {
   const withHead = insertBeforeCloseTag(html, "head", `${tag}\n`);
   if (withHead !== null) return withHead;
   const htmlOpenMatch = html.match(/<html\b[^>]*>/i);
@@ -363,6 +364,16 @@ export function insertRuntimeTag(html: string, tag: string): string {
   return tag + html;
 }
 
+const ACCEL_TRACKER_ATTR = "data-hf-accel-tracker";
+const OPTS_INTO_MOTION_BLUR = /\sdata-motion-blur\s*=\s*["']?on[\s"'>]/i;
+
+export function insertRuntimeTag(html: string, tag: string): string {
+  const withRuntime = insertTagInHead(html, tag);
+  if (!OPTS_INTO_MOTION_BLUR.test(html) || html.includes(ACCEL_TRACKER_ATTR)) return withRuntime;
+  const tracker = `<script ${ACCEL_TRACKER_ATTR}>${acceleratedCanvasTrackerSource()}</script>`;
+  return injectTagsAtHeadStart(withRuntime, tracker);
+}
+
 /**
  * Insert raw tag markup at the very start of `<head>`, ahead of every author
  * script (inline or external). Falls back to just before `<body>`, then to the

packages/core/src/index.ts

@@ -437,4 +437,8 @@ export {
   type HfVfxParam,
   type HfVfxParamValues,
 } from "./vfx.js";
-export { PAGE_DEVICE_SCALE_GLOBAL, PAGE_MOTION_BLUR_OFF_GLOBAL } from "./pageMotionBlur.js";
+export {
+  PAGE_DEVICE_SCALE_GLOBAL,
+  PAGE_MOTION_BLUR_OFF_GLOBAL,
+  acceleratedCanvasTrackerSource,
+} from "./pageMotionBlur.js";

packages/core/src/pageMotionBlur.test.ts

@@ -0,0 +1,59 @@
+// @vitest-environment jsdom
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+import {
+  acceleratedCanvasTrackerSource,
+  installAcceleratedCanvasTracker,
+  isAcceleratedCanvas,
+} from "./pageMotionBlur";
+
+type Proto = Record<string, unknown>;
+const proto = HTMLCanvasElement.prototype as unknown as Proto;
+const saved: Proto = {};
+
+beforeEach(() => {
+  for (const name of ["getContext", "transferControlToOffscreen"]) saved[name] = proto[name];
+  proto.getContext = vi.fn(() => ({}));
+  proto.transferControlToOffscreen = vi.fn(() => ({}));
+});
+
+afterEach(() => {
+  for (const name of Object.keys(saved)) proto[name] = saved[name];
+  Reflect.deleteProperty(window, "__hfAcceleratedCanvases");
+});
+
+describe("accelerated canvas tracker", () => {
+  it("remembers WebGL and WebGPU canvases, not 2D ones", () => {
+    installAcceleratedCanvasTracker();
+    const [gl, gpu, flat] = [1, 2, 3].map(() => document.createElement("canvas"));
+    gl?.getContext("webgl2");
+    gpu?.getContext("webgpu");
+    flat?.getContext("2d");
+    expect([gl, gpu, flat].map((c) => c !== undefined && isAcceleratedCanvas(c))).toEqual([
+      true,
+      true,
+      false,
+    ]);
+  });
+
+  it("treats a canvas handed to an OffscreenCanvas as accelerated", () => {
+    installAcceleratedCanvasTracker();
+    const canvas = document.createElement("canvas");
+    canvas.transferControlToOffscreen();
+    expect(isAcceleratedCanvas(canvas)).toBe(true);
+  });
+
+  it("installs once however many times it runs", () => {
+    const original = proto.getContext as ReturnType<typeof vi.fn>;
+    installAcceleratedCanvasTracker();
+    installAcceleratedCanvasTracker();
+    document.createElement("canvas").getContext("2d");
+    expect(original).toHaveBeenCalledTimes(1);
+  });
+
+  it("works from its source text, as the engine and the preview inline it", () => {
+    (0, eval)(acceleratedCanvasTrackerSource());
+    const canvas = document.createElement("canvas");
+    canvas.getContext("webgl");
+    expect(isAcceleratedCanvas(canvas)).toBe(true);
+  });
+});

packages/core/src/pageMotionBlur.ts

@@ -12,3 +12,48 @@ export function supportsCaptureMotionBlur(win: Window = window): boolean {
   const canvas = w.HTMLCanvasElement?.prototype;
   return !!ctx && !!canvas && "drawElementImage" in ctx && "requestPaint" in canvas;
 }
+
+const ACCELERATED_CANVASES_GLOBAL = "__hfAcceleratedCanvases";
+
+/**
+ * Remember canvases html-in-canvas cannot capture: WebGL, WebGPU, or handed to an OffscreenCanvas.
+ * Self-contained: the engine and the preview run its source ahead of every author script.
+ */
+export function installAcceleratedCanvasTracker(): void {
+  const key = "__hfAcceleratedCanvases";
+  const known: unknown = Reflect.get(window, key);
+  const seen = known instanceof WeakSet ? known : new WeakSet<object>();
+  if (seen !== known) Object.defineProperty(window, key, { value: seen });
+  const accelerated = ["webgl", "webgl2", "experimental-webgl", "webgpu"];
+  const wrap = (
+    name: string,
+    onCall: (canvas: object, args: unknown[], result: unknown) => void,
+  ) => {
+    const proto = HTMLCanvasElement.prototype;
+    const original: unknown = Reflect.get(proto, name);
+    if (typeof original !== "function" || "__hfAccelTracked" in original) return;
+    const tracked = function (this: object, ...args: unknown[]): unknown {
+      const result: unknown = Reflect.apply(original, this, args);
+      onCall(this, args, result);
+      return result;
+    };
+    Object.defineProperty(tracked, "__hfAccelTracked", { value: true });
+    Object.defineProperty(proto, name, { value: tracked, writable: true, configurable: true });
+  };
+  wrap("getContext", (canvas, args, ctx) => {
+    if (ctx && accelerated.includes(String(args[0]))) seen.add(canvas);
+  });
+  wrap("transferControlToOffscreen", (canvas) => {
+    seen.add(canvas);
+  });
+}
+
+export function isAcceleratedCanvas(canvas: HTMLCanvasElement): boolean {
+  const seen: unknown = Reflect.get(window, ACCELERATED_CANVASES_GLOBAL);
+  return seen instanceof WeakSet && seen.has(canvas);
+}
+
+/** The tracker as a script body, for `evaluateOnNewDocument` and inline `<script>` injection. */
+export function acceleratedCanvasTrackerSource(): string {
+  return `(${installAcceleratedCanvasTracker.toString()})();`;
+}

packages/core/src/runtime/captureMotionBlur/index.test.ts

@@ -17,6 +17,7 @@ type Win = Window & {
       reason: string | null;
       shutterDeg: number;
       moverNames: string[];
+      failedFrames: number;
     };
   };
   __hf_page_composite_pending?: boolean;
@@ -204,6 +205,22 @@ describe("opt-in and off switches", () => {
     expect(win.__hfMotionBlur?.report().reason).toMatch(/WebGL/);
   });
 
+  it("stays off for a canvas handed to an OffscreenCanvas, whose context is out of reach", () => {
+    installHtmlInCanvas();
+    (HTMLCanvasElement.prototype as unknown as Record<string, unknown>).transferControlToOffscreen =
+      () => ({});
+    trackAcceleratedCanvases();
+    const root = makeRoot();
+    const canvas = document.createElement("canvas");
+    root.append(canvas);
+    canvas.transferControlToOffscreen();
+    initCaptureMotionBlur(host(root));
+    expect(win.__hfMotionBlur?.report()).toMatchObject({ state: "off" });
+    expect(win.__hfMotionBlur?.report().reason).toMatch(/WebGL/);
+    delete (HTMLCanvasElement.prototype as unknown as Record<string, unknown>)
+      .transferControlToOffscreen;
+  });
+
   it("turns off instead of wrapping when a WebGL canvas appears after init", () => {
     installHtmlInCanvas();
     trackAcceleratedCanvases();
@@ -672,6 +689,30 @@ describe("painting", () => {
     expect(win.__hfMotionBlur?.report()).toMatchObject({ state: "off" });
   });
 
+  it("counts every frame that fell back to plain DOM, though the failure streak resets", async () => {
+    vi.useFakeTimers();
+    installHtmlInCanvas();
+    const { root, h } = movingHost();
+    initCaptureMotionBlur(h);
+    const frame = async (t: number, fails: boolean): Promise<void> => {
+      captureContext()?.drawElementImage.mockImplementation(() => {
+        if (fails) throw new Error("No cached paint record");
+      });
+      const done = paintCaptureMotionBlur(t, { engineMode: true });
+      releaseCaptureMotionBlurFrame();
+      captureCanvas(root).dispatchEvent(new Event("paint"));
+      captureCanvas(root).dispatchEvent(new Event("paint"));
+      await done;
+    };
+    await frame(0.1, true);
+    await frame(0.2, false);
+    void paintCaptureMotionBlur(0.3, { engineMode: true });
+    releaseCaptureMotionBlurFrame();
+    await vi.advanceTimersByTimeAsync(2000);
+    expect(win.__hfMotionBlur?.report()).toMatchObject({ failures: 0, failedFrames: 2 });
+    expect(Object.keys(win.__hfMotionBlur?.report() ?? {})).toContain("failedFrames");
+  });
+
   it("keeps blurring in a render after repeated capture failures, each failed frame plain DOM", async () => {
     installHtmlInCanvas();
     const { root, h } = movingHost();
@@ -689,15 +730,24 @@ describe("painting", () => {
       await engineFrame(i);
       expect(root.parentElement).toBe(document.body);
     }
-    expect(win.__hfMotionBlur?.report()).toMatchObject({ state: "on", failures: 5 });
+    expect(win.__hfMotionBlur?.report()).toMatchObject({
+      state: "on",
+      failures: 5,
+      failedFrames: 5,
+    });
     captureContext()?.drawElementImage.mockReset();
     const done = paintCaptureMotionBlur(0.5, { engineMode: true });
     expect(captureCanvas(root).tagName).toBe("CANVAS");
     releaseCaptureMotionBlurFrame();
     captureCanvas(root).dispatchEvent(new Event("paint"));
     captureCanvas(root).dispatchEvent(new Event("paint"));
     await done;
-    expect(win.__hfMotionBlur?.report()).toMatchObject({ state: "on", failures: 0, frames: 6 });
+    expect(win.__hfMotionBlur?.report()).toMatchObject({
+      state: "on",
+      failures: 0,
+      failedFrames: 5,
+      frames: 6,
+    });
     expect(contexts[1]?.drawImage.mock.calls.length).toBeGreaterThan(1);
   });
 });

packages/core/src/runtime/captureMotionBlur/index.ts

@@ -14,6 +14,8 @@
 import {
   PAGE_DEVICE_SCALE_GLOBAL,
   PAGE_MOTION_BLUR_OFF_GLOBAL,
+  installAcceleratedCanvasTracker,
+  isAcceleratedCanvas,
   supportsCaptureMotionBlur,
 } from "../../pageMotionBlur";
 import { swallow } from "../diagnostics";
@@ -105,37 +107,16 @@ function readConfig(root: HTMLElement): Config | null {
   };
 }
 
-const ACCELERATED_CONTEXTS = new Set(["webgl", "webgl2", "experimental-webgl", "webgpu"]);
-const acceleratedCanvases = new WeakSet<HTMLCanvasElement>();
-
-/**
- * Remember canvases given a WebGL / WebGPU context. Must run before composition scripts: once
- * one has presented, html-in-canvas keeps capturing that frame however often it redraws.
- */
-export function trackAcceleratedCanvases(): void {
-  const proto = HTMLCanvasElement.prototype;
-  const original: unknown = proto.getContext;
-  if (typeof original !== "function" || "__hfAccelTracked" in original) return;
-  const tracked = function (this: HTMLCanvasElement, ...args: unknown[]): unknown {
-    const ctx: unknown = Reflect.apply(original, this, args);
-    if (ctx && ACCELERATED_CONTEXTS.has(String(args[0]))) acceleratedCanvases.add(this);
-    return ctx;
-  };
-  Object.defineProperty(tracked, "__hfAccelTracked", { value: true });
-  Object.defineProperty(proto, "getContext", {
-    value: tracked,
-    writable: true,
-    configurable: true,
-  });
-}
+/** Must run before composition scripts: see `installAcceleratedCanvasTracker`. */
+export const trackAcceleratedCanvases = installAcceleratedCanvasTracker;
 
 /** What in the composition the capture cannot take, checked again per frame (scripts add canvases). */
 function captureBlocker(root: HTMLElement): string | null {
   // A capture canvas inside another one blocks the renderer's main thread.
   if (root.querySelector("canvas[layoutsubtree]"))
     return "the composition already uses html-in-canvas";
   for (const canvas of root.querySelectorAll("canvas")) {
-    if (acceleratedCanvases.has(canvas))
+    if (isAcceleratedCanvas(canvas))
       return "the composition draws a WebGL/WebGPU canvas, which html-in-canvas captures frozen";
   }
   return null;

packages/engine/src/services/pageMotionBlur.test.ts

@@ -1,5 +1,9 @@
 import { describe, expect, it, vi } from "vitest";
-import { PAGE_DEVICE_SCALE_GLOBAL, PAGE_MOTION_BLUR_OFF_GLOBAL } from "@hyperframes/core";
+import {
+  PAGE_DEVICE_SCALE_GLOBAL,
+  PAGE_MOTION_BLUR_OFF_GLOBAL,
+  acceleratedCanvasTrackerSource,
+} from "@hyperframes/core";
 import { installPageMotionBlur } from "./pageMotionBlur";
 
 const fakePage = () => ({ evaluateOnNewDocument: vi.fn(async () => ({ identifier: "1" })) });
@@ -8,7 +12,7 @@ describe("installPageMotionBlur", () => {
   it("tells the page to stand down when the engine blurs", async () => {
     const page = fakePage();
     await installPageMotionBlur(page, { motionBlur: { samplesPerFrame: 16 } });
-    expect(page.evaluateOnNewDocument).toHaveBeenCalledTimes(1);
+    expect(page.evaluateOnNewDocument).toHaveBeenCalledTimes(2);
     expect(page.evaluateOnNewDocument).toHaveBeenCalledWith(
       expect.any(Function),
       PAGE_MOTION_BLUR_OFF_GLOBAL,
@@ -36,9 +40,9 @@ describe("installPageMotionBlur", () => {
     );
   });
 
-  it("leaves the page alone otherwise", async () => {
+  it("installs the canvas tracker before any page script, and nothing else by default", async () => {
     const page = fakePage();
     await installPageMotionBlur(page, {});
-    expect(page.evaluateOnNewDocument).not.toHaveBeenCalled();
+    expect(page.evaluateOnNewDocument.mock.calls).toEqual([[acceleratedCanvasTrackerSource()]]);
   });
 });

packages/engine/src/services/pageMotionBlur.ts

@@ -4,14 +4,19 @@
  * when the render must match a Studio preview that could not blur (`pageMotionBlur: "off"`).
  */
 
-import { PAGE_DEVICE_SCALE_GLOBAL, PAGE_MOTION_BLUR_OFF_GLOBAL } from "@hyperframes/core";
+import {
+  PAGE_DEVICE_SCALE_GLOBAL,
+  PAGE_MOTION_BLUR_OFF_GLOBAL,
+  acceleratedCanvasTrackerSource,
+} from "@hyperframes/core";
 import type { Page } from "puppeteer-core";
 import type { CaptureOptions } from "../types.js";
 
 export async function installPageMotionBlur(
   page: Pick<Page, "evaluateOnNewDocument">,
   options: Pick<CaptureOptions, "motionBlur" | "pageMotionBlur" | "deviceScaleFactor">,
 ): Promise<void> {
+  await page.evaluateOnNewDocument(acceleratedCanvasTrackerSource());
   const scale = options.deviceScaleFactor ?? 1;
   if (scale > 1) {
     await page.evaluateOnNewDocument(