From 2008aeabdd01ff39e8172212dce4efc1825e3549 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miguel=20=C3=81ngel?= Date: Thu, 22 Oct 2026 11:48:58 -0400 Subject: [PATCH 1/2] feat(aws-lambda): attach the arm64 video image to releases and pin it The release builds the arm64 video function image from the handler ZIP it attaches, attaches it as an OCI layout tarball with its sha256, and pins it in prebuilt/video-image.json, read through prebuiltVideoImage(). A retried release reuses the image already attached. --- .github/workflows/lambda-arm64-image.yml | 11 +++ .github/workflows/publish.yml | 74 ++++++++++++++++++- .../scripts/pin-handler-zip.test.ts | 42 ++++++++++- .../aws-lambda/scripts/pin-handler-zip.ts | 53 ++++++++----- packages/aws-lambda/src/sdk/index.ts | 7 +- .../aws-lambda/src/sdk/prebuiltHandler.ts | 28 +++++-- 6 files changed, 185 insertions(+), 30 deletions(-) diff --git a/.github/workflows/lambda-arm64-image.yml b/.github/workflows/lambda-arm64-image.yml index 1f330694b2a..8ae3b2365b6 100644 --- a/.github/workflows/lambda-arm64-image.yml +++ b/.github/workflows/lambda-arm64-image.yml @@ -14,6 +14,11 @@ on: - "packages/aws-lambda/package.json" - "packages/aws-lambda/scripts/build-zip.ts" - ".github/workflows/lambda-arm64-image.yml" + # On main it also seeds publish.yml's ffmpeg cache. + push: + branches: [main] + paths: + - "packages/aws-lambda/image/**" workflow_dispatch: concurrency: @@ -35,6 +40,12 @@ jobs: docker buildx build --progress=plain --platform linux/arm64 --target artifact --output type=local,dest=out packages/aws-lambda/image - run: cat out/hf-media-check.txt && sha256sum out/*.tar.gz + # Same path and key as publish.yml's lambda-arm64-ffmpeg job reads. + - if: matrix.copy == 'a' && github.ref == 'refs/heads/main' + uses: actions/cache/save@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 + with: + path: out/ + key: hf-media-linux-aarch64-${{ hashFiles('packages/aws-lambda/image/**') }} - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 with: name: hf-media-${{ matrix.copy }} diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index e3dbef36ca0..7376b598862 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -13,10 +13,47 @@ on: branches: [main] jobs: + lambda-arm64-ffmpeg: + name: Lambda arm64 ffmpeg + runs-on: ubuntu-24.04-arm + # A cache miss compiles ffmpeg natively, 15-20 minutes. + timeout-minutes: 60 + permissions: + contents: read + env: + EXPECTED_RELEASE_SHA: >- + ${{ github.event_name == 'pull_request' + && github.event.pull_request.merge_commit_sha + || github.sha }} + if: >- + github.event_name == 'push' || + (github.event.pull_request.merged == true && + startsWith(github.event.pull_request.head.ref, 'release/v')) + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + with: + ref: ${{ env.EXPECTED_RELEASE_SHA }} + # lambda-arm64-image.yml saves this key on main; a release run's own saves reach only its re-runs. + - id: cache + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 + with: + path: out/ + key: hf-media-linux-aarch64-${{ hashFiles('packages/aws-lambda/image/**') }} + - if: steps.cache.outputs.cache-hit != 'true' + run: >- + docker buildx build --progress=plain --platform linux/arm64 --target artifact + --output type=local,dest=out packages/aws-lambda/image + - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: hf-media-linux-aarch64 + path: out/hf-media-linux-aarch64.tar.gz + if-no-files-found: error + publish: name: Publish + needs: lambda-arm64-ffmpeg runs-on: ubuntu-latest - timeout-minutes: 20 + timeout-minutes: 30 environment: npm-publish permissions: contents: write @@ -137,6 +174,33 @@ jobs: fi bun packages/aws-lambda/scripts/pin-handler-zip.ts + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 + with: + name: hf-media-linux-aarch64 + path: dist/hf-media + + # Built from the zip pinned above. The video stage only copies files, so no emulation is needed. + - name: Build Lambda video image + id: lambda_video_image + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + VERSION: ${{ steps.version.outputs.version }} + REPO: ${{ github.repository }} + run: | + # A retried run reuses the image already on the release, so the pin still matches it. + if gh release download "v${VERSION}" --repo "$REPO" --pattern hyperframes-lambda-video-arm64.oci.tar --dir dist; then + echo "attached=true" >> "$GITHUB_OUTPUT" + else + bun packages/aws-lambda/scripts/stage-arm64-image.ts --zip=dist/hyperframes-lambda-handler.zip \ + --ffmpeg=dist/hf-media/hf-media-linux-aarch64.tar.gz --out=dist/arm64-image + # OCI export needs a docker-container builder; no attestations, so index.json lists one image. + docker buildx create --use + docker buildx build --progress=plain --platform linux/arm64 --target video \ + --provenance=false --sbom=false --build-context task=dist/arm64-image/task \ + --output type=oci,dest=dist/hyperframes-lambda-video-arm64.oci.tar packages/aws-lambda/image + fi + bun packages/aws-lambda/scripts/pin-handler-zip.ts --video-image + - run: bun run verify:packed-manifests - name: Create GitHub Release @@ -173,6 +237,14 @@ jobs: REPO: ${{ github.repository }} run: gh release upload "v${VERSION}" dist/hyperframes-lambda-handler.zip dist/hyperframes-lambda-handler.sha256 --repo "$REPO" + - name: Attach Lambda video image to release + if: steps.lambda_video_image.outputs.attached != 'true' + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + VERSION: ${{ steps.version.outputs.version }} + REPO: ${{ github.repository }} + run: gh release upload "v${VERSION}" dist/hyperframes-lambda-video-arm64.oci.tar dist/hyperframes-lambda-video-arm64.sha256 --repo "$REPO" + - name: Publish packages env: NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} diff --git a/packages/aws-lambda/scripts/pin-handler-zip.test.ts b/packages/aws-lambda/scripts/pin-handler-zip.test.ts index a57573d47e2..52c299c5b20 100644 --- a/packages/aws-lambda/scripts/pin-handler-zip.test.ts +++ b/packages/aws-lambda/scripts/pin-handler-zip.test.ts @@ -3,8 +3,14 @@ import { createHash } from "node:crypto"; import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; -import { adapterPackageRoot, prebuiltHandler } from "../src/sdk/prebuiltHandler.js"; -import { pinHandlerZip } from "./pin-handler-zip.js"; +import { + adapterPackageRoot, + prebuiltHandler, + prebuiltVideoImage, + VIDEO_IMAGE_ASSET, + VIDEO_IMAGE_MANIFEST_FILE, +} from "../src/sdk/prebuiltHandler.js"; +import { pinHandlerZip, pinReleaseAsset } from "./pin-handler-zip.js"; const examples = join(import.meta.dir, "..", "..", "..", "examples", "aws-lambda"); const realTemplate = readFileSync(join(examples, "template.yaml"), "utf-8"); @@ -93,6 +99,31 @@ describe("pinHandlerZip", () => { }); }); +describe("pinReleaseAsset", () => { + it("pins the arm64 video image beside the handler zip without touching the zip's pin", () => { + const { root, pkgDir } = fixtureRepo(); + writeFileSync(join(pkgDir, "dist", "handler.zip"), "zip"); + const zipPin = pinHandlerZip(root); + const image = Buffer.from("oci layout tar"); + writeFileSync(join(root, "dist", VIDEO_IMAGE_ASSET), image); + const sha256 = createHash("sha256").update(image).digest("hex"); + + const manifest = pinReleaseAsset(root, VIDEO_IMAGE_ASSET, VIDEO_IMAGE_MANIFEST_FILE); + + expect(manifest).toEqual({ + version: "9.8.7", + url: "https://github.com/heygen-com/hyperframes/releases/download/v9.8.7/hyperframes-lambda-video-arm64.oci.tar", + sha256, + bytes: image.length, + }); + expect(readFileSync(join(root, "dist", "hyperframes-lambda-video-arm64.sha256"), "utf-8")).toBe( + `${sha256} hyperframes-lambda-video-arm64.oci.tar\n`, + ); + expect(prebuiltVideoImage(pkgDir)).toEqual(manifest); + expect(prebuiltHandler(pkgDir)?.sha256).toBe(zipPin.sha256); + }); +}); + describe("prebuiltHandler", () => { it("reads from its own package root", () => { expect(adapterPackageRoot()).toBe(join(import.meta.dir, "..")); @@ -102,4 +133,11 @@ describe("prebuiltHandler", () => { const { pkgDir } = fixtureRepo(); expect(prebuiltHandler(pkgDir)).toBeUndefined(); }); + + it("has no video image when the package pins only the zip", () => { + const { root, pkgDir } = fixtureRepo(); + writeFileSync(join(pkgDir, "dist", "handler.zip"), "zip"); + pinHandlerZip(root); + expect(prebuiltVideoImage(pkgDir)).toBeUndefined(); + }); }); diff --git a/packages/aws-lambda/scripts/pin-handler-zip.ts b/packages/aws-lambda/scripts/pin-handler-zip.ts index 60f8c607e7f..361b8a3e07a 100644 --- a/packages/aws-lambda/scripts/pin-handler-zip.ts +++ b/packages/aws-lambda/scripts/pin-handler-zip.ts @@ -1,6 +1,7 @@ #!/usr/bin/env bun // Release step: writes the handler ZIP's pin (asset URL + sha256) and the SAM template into the -// package. A retried release run reuses the ZIP already on the release. +// package, or with --video-image the arm64 video image's pin. A retried release run reuses the +// asset already on the release. import { createHash } from "node:crypto"; import { existsSync, mkdirSync, readFileSync, renameSync, rmSync, writeFileSync } from "node:fs"; import { dirname, join, resolve } from "node:path"; @@ -8,41 +9,57 @@ import { fileURLToPath } from "node:url"; import { HANDLER_MANIFEST_FILE, HANDLER_ZIP_ASSET, - type HandlerZipManifest, PREBUILT_DIR, + type ReleaseAssetManifest, + VIDEO_IMAGE_ASSET, + VIDEO_IMAGE_MANIFEST_FILE, } from "../src/sdk/prebuiltHandler.js"; import { writePrebuiltTemplate } from "./prebuilt-template.mjs"; -export function pinHandlerZip(repoRoot: string): HandlerZipManifest { +/** Hashes `dist/`, writes its `.sha256` beside it and its pin to `prebuilt/`. */ +export function pinReleaseAsset( + repoRoot: string, + asset: string, + manifestFile: string, +): ReleaseAssetManifest { const pkgDir = join(repoRoot, "packages", "aws-lambda"); const pkg = JSON.parse(readFileSync(join(pkgDir, "package.json"), "utf-8")) as { version: string; repository: { url: string }; }; - const asset = join(repoRoot, "dist", HANDLER_ZIP_ASSET); - const built = join(pkgDir, "dist", "handler.zip"); - if (existsSync(built)) { - mkdirSync(dirname(asset), { recursive: true }); - renameSync(built, asset); - rmSync(`${built}.manifest.json`, { force: true }); - } - const bytes = readFileSync(asset); + const bytes = readFileSync(join(repoRoot, "dist", asset)); const sha256 = createHash("sha256").update(bytes).digest("hex"); - writeFileSync(asset.replace(/\.zip$/, ".sha256"), `${sha256} ${HANDLER_ZIP_ASSET}\n`); + writeFileSync(join(repoRoot, "dist", `${asset.split(".")[0]}.sha256`), `${sha256} ${asset}\n`); - const outDir = join(pkgDir, PREBUILT_DIR); - writePrebuiltTemplate(repoRoot, outDir); - const manifest: HandlerZipManifest = { + const manifest: ReleaseAssetManifest = { version: pkg.version, - url: `${pkg.repository.url}/releases/download/v${pkg.version}/${HANDLER_ZIP_ASSET}`, + url: `${pkg.repository.url}/releases/download/v${pkg.version}/${asset}`, sha256, bytes: bytes.length, }; - writeFileSync(join(outDir, HANDLER_MANIFEST_FILE), `${JSON.stringify(manifest, null, 2)}\n`); + const outDir = join(pkgDir, PREBUILT_DIR); + mkdirSync(outDir, { recursive: true }); + writeFileSync(join(outDir, manifestFile), `${JSON.stringify(manifest, null, 2)}\n`); return manifest; } +export function pinHandlerZip(repoRoot: string): ReleaseAssetManifest { + const pkgDir = join(repoRoot, "packages", "aws-lambda"); + const asset = join(repoRoot, "dist", HANDLER_ZIP_ASSET); + const built = join(pkgDir, "dist", "handler.zip"); + if (existsSync(built)) { + mkdirSync(dirname(asset), { recursive: true }); + renameSync(built, asset); + rmSync(`${built}.manifest.json`, { force: true }); + } + writePrebuiltTemplate(repoRoot, join(pkgDir, PREBUILT_DIR)); + return pinReleaseAsset(repoRoot, HANDLER_ZIP_ASSET, HANDLER_MANIFEST_FILE); +} + if (import.meta.main) { const repoRoot = resolve(dirname(fileURLToPath(import.meta.url)), "../../.."); - console.log(JSON.stringify(pinHandlerZip(repoRoot), null, 2)); + const manifest = process.argv.includes("--video-image") + ? pinReleaseAsset(repoRoot, VIDEO_IMAGE_ASSET, VIDEO_IMAGE_MANIFEST_FILE) + : pinHandlerZip(repoRoot); + console.log(JSON.stringify(manifest, null, 2)); } diff --git a/packages/aws-lambda/src/sdk/index.ts b/packages/aws-lambda/src/sdk/index.ts index c03f7bbfbdd..7fe14606002 100644 --- a/packages/aws-lambda/src/sdk/index.ts +++ b/packages/aws-lambda/src/sdk/index.ts @@ -39,6 +39,11 @@ export { validateStepFunctionsInputSize, validateVariablesPayload, } from "./validateConfig.js"; -export { type PrebuiltHandler, prebuiltHandler } from "./prebuiltHandler.js"; +export { + type PrebuiltHandler, + prebuiltHandler, + prebuiltVideoImage, + type ReleaseAssetManifest, +} from "./prebuiltHandler.js"; export type { SerializableDistributedRenderConfig } from "../events.js"; export type { DistributedFormat } from "../formatExtension.js"; diff --git a/packages/aws-lambda/src/sdk/prebuiltHandler.ts b/packages/aws-lambda/src/sdk/prebuiltHandler.ts index 68085305039..9dddb50d9cd 100644 --- a/packages/aws-lambda/src/sdk/prebuiltHandler.ts +++ b/packages/aws-lambda/src/sdk/prebuiltHandler.ts @@ -1,4 +1,4 @@ -// The handler ZIP pinned to this package version; the release workflow writes `prebuilt/`. +// The release assets pinned to this package version; the release workflow writes `prebuilt/`. import { existsSync, readFileSync } from "node:fs"; import { dirname, join } from "node:path"; @@ -7,15 +7,17 @@ import { fileURLToPath } from "node:url"; export const HANDLER_ZIP_ASSET = "hyperframes-lambda-handler.zip"; export const PREBUILT_DIR = "prebuilt"; export const HANDLER_MANIFEST_FILE = "handler-zip.json"; +export const VIDEO_IMAGE_ASSET = "hyperframes-lambda-video-arm64.oci.tar"; +export const VIDEO_IMAGE_MANIFEST_FILE = "video-image.json"; -export interface HandlerZipManifest { +export interface ReleaseAssetManifest { version: string; url: string; sha256: string; bytes: number; } -export interface PrebuiltHandler extends HandlerZipManifest { +export interface PrebuiltHandler extends ReleaseAssetManifest { /** SAM template whose `CodeUri` is `handler.zip` in the same directory. */ templatePath: string; } @@ -34,11 +36,21 @@ export function adapterPackageRoot(): string { throw new Error(`[aws-lambda] no @hyperframes/aws-lambda package.json above ${from}`); } +function readManifest(root: string, file: string): ReleaseAssetManifest | undefined { + const path = join(root, PREBUILT_DIR, file); + if (!existsSync(path)) return undefined; + return JSON.parse(readFileSync(path, "utf-8")) as ReleaseAssetManifest; +} + /** The pinned handler for this install, or undefined when this build carries none. */ export function prebuiltHandler(root: string = adapterPackageRoot()): PrebuiltHandler | undefined { - const dir = join(root, PREBUILT_DIR); - const manifestPath = join(dir, HANDLER_MANIFEST_FILE); - if (!existsSync(manifestPath)) return undefined; - const manifest = JSON.parse(readFileSync(manifestPath, "utf-8")) as HandlerZipManifest; - return { ...manifest, templatePath: join(dir, "template.yaml") }; + const manifest = readManifest(root, HANDLER_MANIFEST_FILE); + return manifest && { ...manifest, templatePath: join(root, PREBUILT_DIR, "template.yaml") }; +} + +/** The pinned arm64 video function image (an OCI layout tarball), or undefined when none is pinned. */ +export function prebuiltVideoImage( + root: string = adapterPackageRoot(), +): ReleaseAssetManifest | undefined { + return readManifest(root, VIDEO_IMAGE_MANIFEST_FILE); } From e409d84fa007096a857d5906da6d55993e7d59a3 Mon Sep 17 00:00:00 2001 From: Miguel Angel Simon Sierra Date: Thu, 22 Oct 2026 23:15:38 -0400 Subject: [PATCH 2/2] fix(aws-lambda): keep the release pin script out of the published package pin-handler-zip.ts is run from a checkout by publish.yml; nothing in dist or the CLI imports it. Leaving it out keeps aws-lambda under its size budget with the video image reader added. --- packages/aws-lambda/package.json | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/aws-lambda/package.json b/packages/aws-lambda/package.json index 8ac6589d38e..0c6eaf7f4f7 100644 --- a/packages/aws-lambda/package.json +++ b/packages/aws-lambda/package.json @@ -12,6 +12,7 @@ "prebuilt/", "scripts/", "!scripts/*.test.ts", + "!scripts/pin-handler-zip.ts", "README.md" ], "type": "module",