Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
34 changes: 34 additions & 0 deletions packages/cli/src/commands/snapshot.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1027,3 +1027,37 @@ describe("snapshot composite recapture", () => {
expect(finalCapture).toBeGreaterThan(recapture);
});
});

describe("snapshot media identity", () => {
it("keys two videos that share an author id apart, as the render path does", async () => {
const project = mkdtempSync(join(tmpdir(), "hf-snapshot-dup-ids-"));
const clip = (src: string) =>
`<video id="clip" class="clip" src="${src}" data-start="0" data-duration="4" muted></video>`;
writeFileSync(
join(project, "index.html"),
`<html><body><div data-composition-id="main" data-width="1920" data-height="1080" data-duration="4">` +
`${clip("a.mp4")}${clip("b.mp4")}</div></body></html>`,
);
snapshotState.openSettledPage.mockClear();
const log = vi.spyOn(console, "log").mockImplementation(() => undefined);

try {
await expect(
snapshotCommand.run?.({
args: { dir: project, at: "1", end: false, output: join(project, "out") },
} as never),
).rejects.toBeDefined();
const [html, , options] = snapshotState.openSettledPage.mock.calls[0] as unknown as [
string,
string,
{ beforeNavigate?: unknown },
];
expect(html).toContain('data-hf-render-id="clip"');
expect(html).toContain('data-hf-render-id="clip__hf2"');
expect(typeof options.beforeNavigate).toBe("function");
} finally {
log.mockRestore();
rmSync(project, { recursive: true, force: true });
}
});
});
25 changes: 21 additions & 4 deletions packages/cli/src/commands/snapshot.ts
Original file line number Diff line number Diff line change
Expand Up @@ -36,13 +36,17 @@ import {
type ZoomTarget,
} from "../capture/captureCompositionFrame.js";
import {
assignMediaRenderIds,
isClipVisibleAt,
isInClipWindow,
MEDIA_RENDER_ID_ATTR,
readElementRateSpec,
sourceTimeAt,
timeAtSourceTime,
type RateSpec,
} from "@hyperframes/core";
import { parseHTML } from "linkedom";
import type { Page } from "puppeteer-core";
import { resolveProject } from "../utils/project.js";
import {
definitiveEntryMismatchComposition,
Expand Down Expand Up @@ -491,6 +495,18 @@ export function serveCapturePage(projectDir: string, html: string, proxy: boolea
return serveStaticProjectHtml(projectDir, html, undefined, [], proxy === true);
}

/** Two clips sharing an author id must stay two frames, keyed as the render path keys them. */
function withMediaRenderIds(html: string): string {
const { document } = parseHTML(html);
assignMediaRenderIds(document);
return document.toString();
}

async function installMediaRenderIdBridge(page: Page): Promise<void> {
const engine = await import("@hyperframes/engine").catch(() => null);
await engine?.installMediaRenderIdBridge(page);
}

/**
* Render key frames from a composition as PNG screenshots.
* The agent can Read these to verify its output visually.
Expand Down Expand Up @@ -541,7 +557,7 @@ async function captureSnapshots(

// Localize fonts (embed remote @font-face as data URIs, matching the render
// path) so snapshots render the real font instead of a fallback sans.
const html = await bundleWithLocalizedFonts(projectDir);
const html = withMediaRenderIds(await bundleWithLocalizedFonts(projectDir));
const server = await serveCapturePage(projectDir, html, opts.autoProxy);

const savedPaths: string[] = [];
Expand All @@ -555,6 +571,7 @@ async function captureSnapshots(
renderReadyTimeoutMs: opts.timeout ?? 5000,
renderReadyWarningSuffix: "snapshots may be inaccurate",
browserGpuMode: opts.browserGpuMode,
beforeNavigate: installMediaRenderIdBridge,
});

try {
Expand Down Expand Up @@ -755,7 +772,7 @@ async function captureSnapshots(
if (cameraExpr) await page.evaluate(cameraExpr);

if (injectVideoFramesBatch && syncVideoFrameVisibility) {
const candidates = await page.evaluate(() => {
const candidates = await page.evaluate((renderIdAttr: string) => {
const runtimeWindow = window as Window & {
__hfResolveMediaStartSeconds?: (element: Element) => number;
};
Expand All @@ -769,7 +786,7 @@ async function captureSnapshots(
const rawDuration = parseFloat(v.dataset.duration ?? "");
const srcDur = Number.isFinite(v.duration) && v.duration > 0 ? v.duration : 0;
return {
id: v.id,
id: v.getAttribute(renderIdAttr) || v.id,

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[P2] Keep the final visibility sync in this ID namespace too

candidate.id now contains the render key, but syncVideoFrameVisibility still checks active.has(video.id) (screenshotService.ts:892). Snapshot invokes it after injection (lines 841–847), so it hides the correctly injected image with visibility:hidden !important before recapture/saving whenever the two IDs differ.

This also regresses static input without duplicate authored IDs: put an untimed <img src="assets/still.png"> before a timed <video id="hf-media-0" src="assets/clip.mp4" data-start="0" data-duration="4" muted>. assignMediaRenderIds gives the idless image hf-media-0 and the video hf-media-0__hf2. At t=1 with successful extraction and visible ancestors, BASE injects/syncs using the video's unique raw ID; HEAD injects through the bridge using hf-media-0__hf2, then sync rejects raw hf-media-0 and hides it.

Make visibility use the same canonical reader/fallback and cover stamping → injection → sync. The new identity test stops at mocked page opening, so it cannot catch this final-state mismatch. This is source-level control analysis, not a browser/media reproduction.

Copy link
Copy Markdown
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Confirmed and fixed at b0f6d5a, at the consumer: syncVideoFrameVisibility now matches window.__hfMediaId?.(video) || video.id, the same reader injection uses, so every caller is in the render-id namespace.

  • Reproduced your static case with the real CLI (id-less untimed <img>, then a red <video id="hf-media-0">, which gets render id hf-media-0__hf2). Pixel at t=1: 0.8.185 253 0 0, this branch with the raw-id sync 0 0 0 (hidden), fixed 253 0 0.
  • Render never showed it because it syncs before it injects, and the injection then shows the frame. Snapshot syncs after.
  • New engine test covers stamping, then injection, then sync with the real bridge. It fails without the fix (expected 'hidden' to be 'visible').
  • Rebased onto main. Main retired product-launch-video (feat(skills)!: publish seven knowledge skills aligned with Studio and retire the workflow skills #5651), so the skill-side changes are gone and the PR is now snapshot plus engine only. Title and body updated.

src: v.currentSrc || v.src,
authoredStart,
authoredRate: v.dataset.playbackRate,
Expand All @@ -788,7 +805,7 @@ async function captureSnapshots(
devicePixelRatio: window.devicePixelRatio,
};
});
});
}, MEDIA_RENDER_ID_ATTR);
const compositionDuration = duration;
const active = candidates.flatMap((candidate) => {
const start = resolveSnapshotVideoClipStart(candidate);
Expand Down
1 change: 1 addition & 0 deletions packages/engine/src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -173,6 +173,7 @@ export {
DOM_LAYER_MASK_STYLE_ID,
type BeginFrameResult,
} from "./services/screenshotService.js";
export { installMediaRenderIdBridge } from "./services/mediaRenderIdBridge.js";

// ── Encoding ───────────────────────────────────────────────────────────────────
export {
Expand Down
28 changes: 28 additions & 0 deletions packages/engine/src/services/screenshotService.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ import {
initTransparentBackground,
DOM_LAYER_MASK_STYLE_ID,
} from "./screenshotService.js";
import { installMediaRenderIdBridge } from "./mediaRenderIdBridge.js";

// These DOM tests isolate layout/decode from the CDP transport, exercised in Chrome separately.
vi.mock("./videoFrameTransport.js", () => ({
Expand Down Expand Up @@ -621,6 +622,33 @@ describe("video-frame injection respects ancestor visibility", () => {
expect(seededImg.style.visibility).toBe("hidden");
});

it("keeps a frame injected by render id visible through the sync, as snapshot calls them", async () => {
const { teardown, setup } = withGlobals(
setupHostHiddenScenario({}, { hostAttribute: "data-start" }),
);
setup.video.setAttribute("data-hf-render-id", "pip__hf2");
try {
await installMediaRenderIdBridge({
evaluateOnNewDocument: async (fn: (...args: unknown[]) => unknown, ...args: unknown[]) =>
fn(...args),
} as unknown as Page);
await injectVideoFramesBatch(passthroughPage(), [
{
videoId: "pip__hf2",
dataUri:
"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNkAAIAAAoAAv/lxKUAAAAASUVORK5CYII=",
},
]);
await syncVideoFrameVisibility(passthroughPage(), ["pip__hf2"]);
} finally {
teardown();
}

const img = setup.video.nextElementSibling as HTMLElement | null;
expect(img?.classList.contains("__render_frame__")).toBe(true);
expect(img?.style.visibility).toBe("visible");
});

it("syncVideoFrameVisibility hides the replacement <img> for ancestor-hidden actives", async () => {
const { teardown, setup } = withGlobals(setupHostHiddenScenario({ visibility: "hidden" }));
const seededImg = setup.document.createElement("img");
Expand Down
2 changes: 1 addition & 1 deletion packages/engine/src/services/screenshotService.ts
Original file line number Diff line number Diff line change
Expand Up @@ -889,7 +889,7 @@ export async function syncVideoFrameVisibility(
const img = video.nextElementSibling as HTMLElement | null;
const hasImg = img && img.classList.contains("__render_frame__");
const ancestorHidden = isVisualAncestorHidden(video);
const visible = active.has(video.id) && !ancestorHidden;
const visible = active.has(window.__hfMediaId?.(video) || video.id) && !ancestorHidden;
if (visible) {
// Active video: show injected <img>, hide native <video>.
// Do NOT clobber inline opacity here — GSAP-controlled opacity must
Expand Down
Loading