diff --git a/.github/workflows/lambda-arm64-image.yml b/.github/workflows/lambda-arm64-image.yml index 1f330694b2a..8ae3b2365b6 100644 --- a/.github/workflows/lambda-arm64-image.yml +++ b/.github/workflows/lambda-arm64-image.yml @@ -14,6 +14,11 @@ on: - "packages/aws-lambda/package.json" - "packages/aws-lambda/scripts/build-zip.ts" - ".github/workflows/lambda-arm64-image.yml" + # On main it also seeds publish.yml's ffmpeg cache. + push: + branches: [main] + paths: + - "packages/aws-lambda/image/**" workflow_dispatch: concurrency: @@ -35,6 +40,12 @@ jobs: docker buildx build --progress=plain --platform linux/arm64 --target artifact --output type=local,dest=out packages/aws-lambda/image - run: cat out/hf-media-check.txt && sha256sum out/*.tar.gz + # Same path and key as publish.yml's lambda-arm64-ffmpeg job reads. + - if: matrix.copy == 'a' && github.ref == 'refs/heads/main' + uses: actions/cache/save@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 + with: + path: out/ + key: hf-media-linux-aarch64-${{ hashFiles('packages/aws-lambda/image/**') }} - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 with: name: hf-media-${{ matrix.copy }} diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index e3dbef36ca0..7376b598862 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -13,10 +13,47 @@ on: branches: [main] jobs: + lambda-arm64-ffmpeg: + name: Lambda arm64 ffmpeg + runs-on: ubuntu-24.04-arm + # A cache miss compiles ffmpeg natively, 15-20 minutes. + timeout-minutes: 60 + permissions: + contents: read + env: + EXPECTED_RELEASE_SHA: >- + ${{ github.event_name == 'pull_request' + && github.event.pull_request.merge_commit_sha + || github.sha }} + if: >- + github.event_name == 'push' || + (github.event.pull_request.merged == true && + startsWith(github.event.pull_request.head.ref, 'release/v')) + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + with: + ref: ${{ env.EXPECTED_RELEASE_SHA }} + # lambda-arm64-image.yml saves this key on main; a release run's own saves reach only its re-runs. + - id: cache + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 + with: + path: out/ + key: hf-media-linux-aarch64-${{ hashFiles('packages/aws-lambda/image/**') }} + - if: steps.cache.outputs.cache-hit != 'true' + run: >- + docker buildx build --progress=plain --platform linux/arm64 --target artifact + --output type=local,dest=out packages/aws-lambda/image + - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: hf-media-linux-aarch64 + path: out/hf-media-linux-aarch64.tar.gz + if-no-files-found: error + publish: name: Publish + needs: lambda-arm64-ffmpeg runs-on: ubuntu-latest - timeout-minutes: 20 + timeout-minutes: 30 environment: npm-publish permissions: contents: write @@ -137,6 +174,33 @@ jobs: fi bun packages/aws-lambda/scripts/pin-handler-zip.ts + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 + with: + name: hf-media-linux-aarch64 + path: dist/hf-media + + # Built from the zip pinned above. The video stage only copies files, so no emulation is needed. + - name: Build Lambda video image + id: lambda_video_image + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + VERSION: ${{ steps.version.outputs.version }} + REPO: ${{ github.repository }} + run: | + # A retried run reuses the image already on the release, so the pin still matches it. + if gh release download "v${VERSION}" --repo "$REPO" --pattern hyperframes-lambda-video-arm64.oci.tar --dir dist; then + echo "attached=true" >> "$GITHUB_OUTPUT" + else + bun packages/aws-lambda/scripts/stage-arm64-image.ts --zip=dist/hyperframes-lambda-handler.zip \ + --ffmpeg=dist/hf-media/hf-media-linux-aarch64.tar.gz --out=dist/arm64-image + # OCI export needs a docker-container builder; no attestations, so index.json lists one image. + docker buildx create --use + docker buildx build --progress=plain --platform linux/arm64 --target video \ + --provenance=false --sbom=false --build-context task=dist/arm64-image/task \ + --output type=oci,dest=dist/hyperframes-lambda-video-arm64.oci.tar packages/aws-lambda/image + fi + bun packages/aws-lambda/scripts/pin-handler-zip.ts --video-image + - run: bun run verify:packed-manifests - name: Create GitHub Release @@ -173,6 +237,14 @@ jobs: REPO: ${{ github.repository }} run: gh release upload "v${VERSION}" dist/hyperframes-lambda-handler.zip dist/hyperframes-lambda-handler.sha256 --repo "$REPO" + - name: Attach Lambda video image to release + if: steps.lambda_video_image.outputs.attached != 'true' + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + VERSION: ${{ steps.version.outputs.version }} + REPO: ${{ github.repository }} + run: gh release upload "v${VERSION}" dist/hyperframes-lambda-video-arm64.oci.tar dist/hyperframes-lambda-video-arm64.sha256 --repo "$REPO" + - name: Publish packages env: NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} diff --git a/packages/aws-lambda/package.json b/packages/aws-lambda/package.json index 8ac6589d38e..0c6eaf7f4f7 100644 --- a/packages/aws-lambda/package.json +++ b/packages/aws-lambda/package.json @@ -12,6 +12,7 @@ "prebuilt/", "scripts/", "!scripts/*.test.ts", + "!scripts/pin-handler-zip.ts", "README.md" ], "type": "module", diff --git a/packages/aws-lambda/scripts/pin-handler-zip.test.ts b/packages/aws-lambda/scripts/pin-handler-zip.test.ts index a57573d47e2..52c299c5b20 100644 --- a/packages/aws-lambda/scripts/pin-handler-zip.test.ts +++ b/packages/aws-lambda/scripts/pin-handler-zip.test.ts @@ -3,8 +3,14 @@ import { createHash } from "node:crypto"; import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; -import { adapterPackageRoot, prebuiltHandler } from "../src/sdk/prebuiltHandler.js"; -import { pinHandlerZip } from "./pin-handler-zip.js"; +import { + adapterPackageRoot, + prebuiltHandler, + prebuiltVideoImage, + VIDEO_IMAGE_ASSET, + VIDEO_IMAGE_MANIFEST_FILE, +} from "../src/sdk/prebuiltHandler.js"; +import { pinHandlerZip, pinReleaseAsset } from "./pin-handler-zip.js"; const examples = join(import.meta.dir, "..", "..", "..", "examples", "aws-lambda"); const realTemplate = readFileSync(join(examples, "template.yaml"), "utf-8"); @@ -93,6 +99,31 @@ describe("pinHandlerZip", () => { }); }); +describe("pinReleaseAsset", () => { + it("pins the arm64 video image beside the handler zip without touching the zip's pin", () => { + const { root, pkgDir } = fixtureRepo(); + writeFileSync(join(pkgDir, "dist", "handler.zip"), "zip"); + const zipPin = pinHandlerZip(root); + const image = Buffer.from("oci layout tar"); + writeFileSync(join(root, "dist", VIDEO_IMAGE_ASSET), image); + const sha256 = createHash("sha256").update(image).digest("hex"); + + const manifest = pinReleaseAsset(root, VIDEO_IMAGE_ASSET, VIDEO_IMAGE_MANIFEST_FILE); + + expect(manifest).toEqual({ + version: "9.8.7", + url: "https://github.com/heygen-com/hyperframes/releases/download/v9.8.7/hyperframes-lambda-video-arm64.oci.tar", + sha256, + bytes: image.length, + }); + expect(readFileSync(join(root, "dist", "hyperframes-lambda-video-arm64.sha256"), "utf-8")).toBe( + `${sha256} hyperframes-lambda-video-arm64.oci.tar\n`, + ); + expect(prebuiltVideoImage(pkgDir)).toEqual(manifest); + expect(prebuiltHandler(pkgDir)?.sha256).toBe(zipPin.sha256); + }); +}); + describe("prebuiltHandler", () => { it("reads from its own package root", () => { expect(adapterPackageRoot()).toBe(join(import.meta.dir, "..")); @@ -102,4 +133,11 @@ describe("prebuiltHandler", () => { const { pkgDir } = fixtureRepo(); expect(prebuiltHandler(pkgDir)).toBeUndefined(); }); + + it("has no video image when the package pins only the zip", () => { + const { root, pkgDir } = fixtureRepo(); + writeFileSync(join(pkgDir, "dist", "handler.zip"), "zip"); + pinHandlerZip(root); + expect(prebuiltVideoImage(pkgDir)).toBeUndefined(); + }); }); diff --git a/packages/aws-lambda/scripts/pin-handler-zip.ts b/packages/aws-lambda/scripts/pin-handler-zip.ts index 60f8c607e7f..361b8a3e07a 100644 --- a/packages/aws-lambda/scripts/pin-handler-zip.ts +++ b/packages/aws-lambda/scripts/pin-handler-zip.ts @@ -1,6 +1,7 @@ #!/usr/bin/env bun // Release step: writes the handler ZIP's pin (asset URL + sha256) and the SAM template into the -// package. A retried release run reuses the ZIP already on the release. +// package, or with --video-image the arm64 video image's pin. A retried release run reuses the +// asset already on the release. import { createHash } from "node:crypto"; import { existsSync, mkdirSync, readFileSync, renameSync, rmSync, writeFileSync } from "node:fs"; import { dirname, join, resolve } from "node:path"; @@ -8,41 +9,57 @@ import { fileURLToPath } from "node:url"; import { HANDLER_MANIFEST_FILE, HANDLER_ZIP_ASSET, - type HandlerZipManifest, PREBUILT_DIR, + type ReleaseAssetManifest, + VIDEO_IMAGE_ASSET, + VIDEO_IMAGE_MANIFEST_FILE, } from "../src/sdk/prebuiltHandler.js"; import { writePrebuiltTemplate } from "./prebuilt-template.mjs"; -export function pinHandlerZip(repoRoot: string): HandlerZipManifest { +/** Hashes `dist/`, writes its `.sha256` beside it and its pin to `prebuilt/`. */ +export function pinReleaseAsset( + repoRoot: string, + asset: string, + manifestFile: string, +): ReleaseAssetManifest { const pkgDir = join(repoRoot, "packages", "aws-lambda"); const pkg = JSON.parse(readFileSync(join(pkgDir, "package.json"), "utf-8")) as { version: string; repository: { url: string }; }; - const asset = join(repoRoot, "dist", HANDLER_ZIP_ASSET); - const built = join(pkgDir, "dist", "handler.zip"); - if (existsSync(built)) { - mkdirSync(dirname(asset), { recursive: true }); - renameSync(built, asset); - rmSync(`${built}.manifest.json`, { force: true }); - } - const bytes = readFileSync(asset); + const bytes = readFileSync(join(repoRoot, "dist", asset)); const sha256 = createHash("sha256").update(bytes).digest("hex"); - writeFileSync(asset.replace(/\.zip$/, ".sha256"), `${sha256} ${HANDLER_ZIP_ASSET}\n`); + writeFileSync(join(repoRoot, "dist", `${asset.split(".")[0]}.sha256`), `${sha256} ${asset}\n`); - const outDir = join(pkgDir, PREBUILT_DIR); - writePrebuiltTemplate(repoRoot, outDir); - const manifest: HandlerZipManifest = { + const manifest: ReleaseAssetManifest = { version: pkg.version, - url: `${pkg.repository.url}/releases/download/v${pkg.version}/${HANDLER_ZIP_ASSET}`, + url: `${pkg.repository.url}/releases/download/v${pkg.version}/${asset}`, sha256, bytes: bytes.length, }; - writeFileSync(join(outDir, HANDLER_MANIFEST_FILE), `${JSON.stringify(manifest, null, 2)}\n`); + const outDir = join(pkgDir, PREBUILT_DIR); + mkdirSync(outDir, { recursive: true }); + writeFileSync(join(outDir, manifestFile), `${JSON.stringify(manifest, null, 2)}\n`); return manifest; } +export function pinHandlerZip(repoRoot: string): ReleaseAssetManifest { + const pkgDir = join(repoRoot, "packages", "aws-lambda"); + const asset = join(repoRoot, "dist", HANDLER_ZIP_ASSET); + const built = join(pkgDir, "dist", "handler.zip"); + if (existsSync(built)) { + mkdirSync(dirname(asset), { recursive: true }); + renameSync(built, asset); + rmSync(`${built}.manifest.json`, { force: true }); + } + writePrebuiltTemplate(repoRoot, join(pkgDir, PREBUILT_DIR)); + return pinReleaseAsset(repoRoot, HANDLER_ZIP_ASSET, HANDLER_MANIFEST_FILE); +} + if (import.meta.main) { const repoRoot = resolve(dirname(fileURLToPath(import.meta.url)), "../../.."); - console.log(JSON.stringify(pinHandlerZip(repoRoot), null, 2)); + const manifest = process.argv.includes("--video-image") + ? pinReleaseAsset(repoRoot, VIDEO_IMAGE_ASSET, VIDEO_IMAGE_MANIFEST_FILE) + : pinHandlerZip(repoRoot); + console.log(JSON.stringify(manifest, null, 2)); } diff --git a/packages/aws-lambda/src/sdk/index.ts b/packages/aws-lambda/src/sdk/index.ts index c03f7bbfbdd..7fe14606002 100644 --- a/packages/aws-lambda/src/sdk/index.ts +++ b/packages/aws-lambda/src/sdk/index.ts @@ -39,6 +39,11 @@ export { validateStepFunctionsInputSize, validateVariablesPayload, } from "./validateConfig.js"; -export { type PrebuiltHandler, prebuiltHandler } from "./prebuiltHandler.js"; +export { + type PrebuiltHandler, + prebuiltHandler, + prebuiltVideoImage, + type ReleaseAssetManifest, +} from "./prebuiltHandler.js"; export type { SerializableDistributedRenderConfig } from "../events.js"; export type { DistributedFormat } from "../formatExtension.js"; diff --git a/packages/aws-lambda/src/sdk/prebuiltHandler.ts b/packages/aws-lambda/src/sdk/prebuiltHandler.ts index 68085305039..9dddb50d9cd 100644 --- a/packages/aws-lambda/src/sdk/prebuiltHandler.ts +++ b/packages/aws-lambda/src/sdk/prebuiltHandler.ts @@ -1,4 +1,4 @@ -// The handler ZIP pinned to this package version; the release workflow writes `prebuilt/`. +// The release assets pinned to this package version; the release workflow writes `prebuilt/`. import { existsSync, readFileSync } from "node:fs"; import { dirname, join } from "node:path"; @@ -7,15 +7,17 @@ import { fileURLToPath } from "node:url"; export const HANDLER_ZIP_ASSET = "hyperframes-lambda-handler.zip"; export const PREBUILT_DIR = "prebuilt"; export const HANDLER_MANIFEST_FILE = "handler-zip.json"; +export const VIDEO_IMAGE_ASSET = "hyperframes-lambda-video-arm64.oci.tar"; +export const VIDEO_IMAGE_MANIFEST_FILE = "video-image.json"; -export interface HandlerZipManifest { +export interface ReleaseAssetManifest { version: string; url: string; sha256: string; bytes: number; } -export interface PrebuiltHandler extends HandlerZipManifest { +export interface PrebuiltHandler extends ReleaseAssetManifest { /** SAM template whose `CodeUri` is `handler.zip` in the same directory. */ templatePath: string; } @@ -34,11 +36,21 @@ export function adapterPackageRoot(): string { throw new Error(`[aws-lambda] no @hyperframes/aws-lambda package.json above ${from}`); } +function readManifest(root: string, file: string): ReleaseAssetManifest | undefined { + const path = join(root, PREBUILT_DIR, file); + if (!existsSync(path)) return undefined; + return JSON.parse(readFileSync(path, "utf-8")) as ReleaseAssetManifest; +} + /** The pinned handler for this install, or undefined when this build carries none. */ export function prebuiltHandler(root: string = adapterPackageRoot()): PrebuiltHandler | undefined { - const dir = join(root, PREBUILT_DIR); - const manifestPath = join(dir, HANDLER_MANIFEST_FILE); - if (!existsSync(manifestPath)) return undefined; - const manifest = JSON.parse(readFileSync(manifestPath, "utf-8")) as HandlerZipManifest; - return { ...manifest, templatePath: join(dir, "template.yaml") }; + const manifest = readManifest(root, HANDLER_MANIFEST_FILE); + return manifest && { ...manifest, templatePath: join(root, PREBUILT_DIR, "template.yaml") }; +} + +/** The pinned arm64 video function image (an OCI layout tarball), or undefined when none is pinned. */ +export function prebuiltVideoImage( + root: string = adapterPackageRoot(), +): ReleaseAssetManifest | undefined { + return readManifest(root, VIDEO_IMAGE_MANIFEST_FILE); }