build(aws-lambda): pinned native ffmpeg build for an arm64 Lambda image

.github/workflows/lambda-ffmpeg.yml

@@ -0,0 +1,47 @@
+name: Lambda arm64 ffmpeg
+
+permissions:
+  contents: read
+
+# Builds the native ffmpeg for the arm64 Lambda video function on two separate ARM runners.
+# Each build runs the codec check on the bare Lambda base; the tarballs must be byte-identical.
+on:
+  pull_request:
+    paths:
+      - "packages/aws-lambda/image/**"
+      - ".github/workflows/lambda-ffmpeg.yml"
+  workflow_dispatch:
+
+concurrency:
+  group: lambda-ffmpeg-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  build:
+    name: Build ${{ matrix.copy }}
+    runs-on: ubuntu-24.04-arm
+    timeout-minutes: 90
+    strategy:
+      matrix:
+        copy: [a, b]
+    steps:
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
+      - name: Build and check
+        run: >-
+          docker buildx build --progress=plain --platform linux/arm64 --target artifact
+          --output type=local,dest=out packages/aws-lambda/image
+      - run: cat out/hf-media-check.txt && sha256sum out/*.tar.gz
+      - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
+        with:
+          name: hf-media-${{ matrix.copy }}
+          path: out/
+          retention-days: 7
+
+  reproducible:
+    name: Builds match
+    needs: build
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    steps:
+      - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
+      - run: cmp hf-media-a/hf-media-linux-aarch64.tar.gz hf-media-b/hf-media-linux-aarch64.tar.gz

packages/aws-lambda/image/Dockerfile

@@ -0,0 +1,34 @@
+# The arm64 Lambda video function's native ffmpeg, built on the Lambda Node 22 base it runs on:
+#   docker buildx build --platform linux/arm64 --target artifact --output type=local,dest=out packages/aws-lambda/image
+# The artifact stage depends on the check stage, so no tarball comes out of a build whose codecs fail.
+ARG BASE=public.ecr.aws/lambda/nodejs:22@sha256:d138eff445f6ea04c51091d8e6cbb064d5663e850cddfa62396061586f98b126
+
+FROM ${BASE} AS build
+ENV SOURCE_DATE_EPOCH=1722643200
+RUN rpm -qa --qf '%{NAME}\n' | sort >/build-base-rpms.txt && \
+    microdnf install -y --nodocs --setopt=install_weak_deps=0 --releasever=2023.12.20260914 \
+      gcc gcc-c++ make cmake meson ninja-build autoconf automake libtool pkgconf-pkg-config diffutils \
+      git xz bzip2 tar gzip which patchelf gnutls-devel zlib-devel $([ "$(uname -m)" = x86_64 ] && echo nasm) && \
+    microdnf clean all && mkdir /build && mv /build-base-rpms.txt /build/base-rpms.txt
+COPY sources.txt build-ffmpeg.sh /build/
+RUN bash /build/build-ffmpeg.sh
+
+FROM ${BASE} AS certs
+# certtool, not openssl: the base's openssl-libs cannot run the repo's openssl CLI.
+RUN microdnf install -y --nodocs --releasever=2023.12.20260914 gnutls-utils && mkdir /certs && cd /certs && \
+    printf 'cn = check-ca\nca\ncert_signing_key\nexpiration_days = 2\n' >ca.tmpl && \
+    printf 'cn = localhost\ndns_name = localhost\ntls_www_server\nexpiration_days = 2\n' >server.tmpl && \
+    certtool -p --outfile ca.key && certtool -s --load-privkey ca.key --template ca.tmpl --outfile ca.crt && \
+    certtool -p --outfile server.key && \
+    certtool -c --load-privkey server.key --load-ca-certificate ca.crt --load-ca-privkey ca.key \
+      --template server.tmpl --outfile server.crt
+
+FROM ${BASE} AS check
+COPY --from=build /opt/hf-media /opt/hf-media
+COPY --from=certs /certs /certs
+COPY check-ffmpeg.sh /check-ffmpeg.sh
+RUN cp /certs/ca.crt /etc/pki/ca-trust/source/anchors/ && update-ca-trust && bash /check-ffmpeg.sh >/check.txt
+
+FROM scratch AS artifact
+COPY --from=build /out/ /
+COPY --from=check /check.txt /hf-media-check.txt

packages/aws-lambda/image/build-ffmpeg.sh

@@ -0,0 +1,92 @@
+#!/bin/bash
+# Builds ffmpeg + ffprobe into /opt/hf-media and packs /out/hf-media-linux-<arch>.tar.gz.
+# Codec libraries link statically; glibc and the TLS stack stay dynamic (a static glibc build crashed on Lambda).
+set -euo pipefail
+
+PREFIX=/opt/hf-media
+DEPS=/build/deps
+SRC=/build/src
+JOBS=$(nproc)
+export PKG_CONFIG_PATH="$DEPS/lib/pkgconfig:$DEPS/lib64/pkgconfig"
+export CFLAGS="-O2 -fPIC" CXXFLAGS="-O2 -fPIC"
+mkdir -p "$SRC" "$DEPS" "$PREFIX/share/licenses" /out
+
+fetch() {
+  local name=$1 sum url
+  read -r sum url < <(awk -v n="$name" '$1 == n { print $2, $3 }' /build/sources.txt)
+  curl -fsSL --retry 3 -o "$SRC/$name" "$url"
+  echo "$sum  $SRC/$name" | sha256sum -c --quiet
+  tar -xf "$SRC/$name" -C "$SRC"
+}
+
+license() { mkdir -p "$PREFIX/share/licenses/$1" && cp "${@:2}" "$PREFIX/share/licenses/$1/"; }
+
+read -r x264_commit x264_url < <(awk '$1 == "x264" { print $2, $3 }' /build/sources.txt)
+git clone -q "$x264_url" "$SRC/x264"
+git -C "$SRC/x264" checkout -q "$x264_commit"
+[ "$(git -C "$SRC/x264" rev-parse HEAD)" = "$x264_commit" ]
+(cd "$SRC/x264" && ./configure --prefix="$DEPS" --enable-static --enable-pic --disable-cli &&
+  make -j"$JOBS" && make install)
+license x264 "$SRC/x264/COPYING"
+
+fetch x265_4.1.tar.gz
+# 8-bit x265 with the 10-bit library linked in (x265's multilib layout): HDR renders encode yuv420p10le.
+# SVE2 kernels need -march=armv9-a, which the base's gcc 11 does not know (x86 ignores the flag).
+x265=(-S "$SRC/x265_4.1/source" -DCMAKE_BUILD_TYPE=Release -DENABLE_SHARED=OFF -DENABLE_CLI=OFF -DENABLE_SVE2=OFF)
+cmake "${x265[@]}" -B "$SRC/x265-10" -DHIGH_BIT_DEPTH=ON -DEXPORT_C_API=OFF
+cmake --build "$SRC/x265-10" -j"$JOBS"
+cmake "${x265[@]}" -B "$SRC/x265-8" -DCMAKE_INSTALL_PREFIX="$DEPS" -DLINKED_10BIT=ON \
+  -DEXTRA_LIB="$SRC/x265-10/libx265.a"
+cmake --build "$SRC/x265-8" -j"$JOBS"
+cmake --install "$SRC/x265-8"
+printf 'CREATE %s\nADDLIB %s\nADDLIB %s\nSAVE\nEND\n' "$DEPS/lib/libx265.a" \
+  "$SRC/x265-8/libx265.a" "$SRC/x265-10/libx265.a" | ar -M
+license x265 "$SRC/x265_4.1/COPYING"
+
+fetch libvpx-1.15.0.tar.gz
+(cd "$SRC/libvpx-1.15.0" && ./configure --prefix="$DEPS" --enable-static --disable-shared --enable-pic \
+  --enable-vp9-highbitdepth --disable-examples --disable-tools --disable-docs --disable-unit-tests &&
+  make -j"$JOBS" && make install)
+license libvpx "$SRC/libvpx-1.15.0/LICENSE" "$SRC/libvpx-1.15.0/PATENTS"
+
+fetch opus-1.5.2.tar.gz
+(cd "$SRC/opus-1.5.2" && ./configure --prefix="$DEPS" --enable-static --disable-shared --with-pic \
+  --disable-doc --disable-extra-programs && make -j"$JOBS" && make install)
+license opus "$SRC/opus-1.5.2/COPYING"
+
+fetch zimg-3.0.5.tar.gz
+(cd "$SRC/zimg-release-3.0.5" && ./autogen.sh && ./configure --prefix="$DEPS" --enable-static --disable-shared \
+  --with-pic && make -j"$JOBS" && make install)
+license zimg "$SRC/zimg-release-3.0.5/COPYING"
+
+fetch dav1d-1.5.1.tar.xz
+meson setup "$SRC/dav1d-build" "$SRC/dav1d-1.5.1" --prefix="$DEPS" --libdir=lib --buildtype=release \
+  --default-library=static -Denable_tools=false -Denable_tests=false
+ninja -C "$SRC/dav1d-build" install
+license dav1d "$SRC/dav1d-1.5.1/COPYING"
+
+fetch ffmpeg-7.0.2.tar.xz
+# No --static pkg-config: gnutls.pc then names a libgmp.a from its own RPM build tree. The deps above
+# exist only as .a, so they link statically anyway; --extra-libs carries their private libraries.
+(cd "$SRC/ffmpeg-7.0.2" && ./configure --prefix="$PREFIX" \
+  --extra-cflags="-I$DEPS/include" --extra-ldflags="-L$DEPS/lib" --extra-libs="-lstdc++ -lpthread -lm -ldl" \
+  --enable-gpl --enable-version3 --disable-debug --disable-doc --disable-ffplay --disable-autodetect \
+  --enable-gnutls --enable-zlib --enable-libx264 --enable-libx265 --enable-libvpx --enable-libopus \
+  --enable-libzimg --enable-libdav1d && make -j"$JOBS" && make install)
+license ffmpeg "$SRC/ffmpeg-7.0.2/COPYING.GPLv3" "$SRC/ffmpeg-7.0.2/LICENSE.md"
+rm -rf "$PREFIX/include" "$PREFIX/lib" "$PREFIX/share/ffmpeg"
+strip "$PREFIX/bin/ffmpeg" "$PREFIX/bin/ffprobe"
+
+# Bundle every shared library the bare base image lacks (gnutls, nettle); the rpath finds them.
+mkdir -p "$PREFIX/lib"
+ldd "$PREFIX/bin/ffmpeg" "$PREFIX/bin/ffprobe" | awk '$2 == "=>" { print $3 }' | sort -u | while read -r lib; do
+  if ! grep -qx "$(rpm -qf --qf '%{NAME}' "$lib")" /build/base-rpms.txt; then
+    cp -L "$lib" "$PREFIX/lib/" && patchelf --set-rpath '$ORIGIN' "$PREFIX/lib/$(basename "$lib")"
+    echo "$(rpm -qf "$lib") $(basename "$lib")" >>"$PREFIX/share/licenses/bundled-libraries.txt"
+  fi
+done
+patchelf --set-rpath '$ORIGIN/../lib' "$PREFIX/bin/ffmpeg" "$PREFIX/bin/ffprobe"
+cp /build/sources.txt "$PREFIX/share/licenses/sources.txt"
+
+tar --sort=name --mtime="@$SOURCE_DATE_EPOCH" --owner=0 --group=0 --numeric-owner -C /opt -cf - hf-media |
+  gzip -n -9 >"/out/hf-media-linux-$(uname -m).tar.gz"

packages/aws-lambda/image/check-ffmpeg.sh

@@ -0,0 +1,61 @@
+#!/bin/bash
+# Runs on the bare Lambda base with only /opt/hf-media added: every codec the engine calls must work.
+# Prints a deterministic summary to stdout; the artifact stage keeps it.
+set -euo pipefail
+ff() { /opt/hf-media/bin/ffmpeg -hide_banner -loglevel error -nostdin "$@"; }
+probe() { local input=$1; shift; /opt/hf-media/bin/ffprobe -hide_banner -loglevel error "$@" -- "$input"; }
+listed() { # listed <ffmpeg list flag> <awk field> <name...>
+  local names
+  names=$(/opt/hf-media/bin/ffmpeg -hide_banner "$1" | awk -v f="$2" '{ print $f }')
+  for name in "${@:3}"; do grep -qx -- "$name" <<<"$names" || { echo "missing $1 $name" >&2; exit 1; }; done
+  echo "${1#-}: ${*:3}"
+}
+frames() { probe "$1" -count_frames -select_streams v:0 -show_entries stream=nb_read_frames -of csv=p=0 "${@:2}"; }
+
+cd "$(mktemp -d)"
+listed -encoders 2 libx264 libx265 libvpx-vp9 libopus prores_ks aac png mjpeg
+listed -decoders 2 libvpx-vp9 libvpx libdav1d h264 hevc prores aac opus png mjpeg
+listed -filters 2 zscale tonemap setparams ebur128 alphamerge overlay
+listed -protocols 1 https tls file
+
+src=(-f lavfi -i "testsrc2=s=320x240:r=25:d=1")
+ff "${src[@]}" -c:v libx264 -pix_fmt yuv420p -bf 0 h264.mp4
+ff "${src[@]}" -c:v libx265 -pix_fmt yuv420p -tag:v hvc1 -x265-params log-level=error h265.mp4
+ff "${src[@]}" -c:v prores_ks -profile:v 4444 -pix_fmt yuva444p10le prores.mov
+ff -f lavfi -i "sine=d=1" -c:a libopus -b:a 128k opus.webm
+ff -f lavfi -i "sine=d=1" -c:a aac -b:a 192k aac.m4a
+ff "${src[@]}" -frames:v 1 frame.png
+for f in h264.mp4 h265.mp4 prores.mov; do [ "$(frames "$f")" = 25 ] || { echo "decode $f" >&2; exit 1; }; done
+for f in opus.webm aac.m4a; do ff -i "$f" -f null -; done
+[ "$(probe frame.png -show_entries stream=width -of csv=p=0)" = 320 ]
+echo "encode+decode: h264 h265 prores opus aac png"
+
+# VP9 with alpha: left half transparent, decoded through libvpx-vp9 as the engine does.
+ff "${src[@]}" -vf "format=yuva420p,geq=lum='lum(X,Y)':cb='cb(X,Y)':cr='cr(X,Y)':a='if(lt(X,W/2),0,255)'" \
+  -c:v libvpx-vp9 -pix_fmt yuva420p -auto-alt-ref 0 -b:v 0 -crf 30 alpha.webm
+ff -c:v libvpx-vp9 -i alpha.webm -frames:v 1 -vf alphaextract,crop=40:40:20:100 -f rawvideo -pix_fmt gray left.raw
+ff -c:v libvpx-vp9 -i alpha.webm -frames:v 1 -vf alphaextract,crop=40:40:260:100 -f rawvideo -pix_fmt gray right.raw
+node -e '
+  const fs = require("fs"), mean = (f) => fs.readFileSync(f).reduce((a, b) => a + b, 0) / 1600;
+  if (!(mean("left.raw") < 16 && mean("right.raw") > 240)) throw new Error("vp9 alpha lost");'
+echo "vp9 alpha: encode+decode"
+
+# The engine's HDR to SDR chain (core/src/hdrToneMap.ts) on tagged 10-bit HEVC.
+ff "${src[@]}" -c:v libx265 -pix_fmt yuv420p10le -color_primaries bt2020 -color_trc smpte2084 \
+  -colorspace bt2020nc -x265-params log-level=error hdr.mp4
+[ "$(probe hdr.mp4 -select_streams v:0 -show_entries stream=pix_fmt -of csv=p=0)" = yuv420p10le ]
+ff -i hdr.mp4 -vf "zscale=t=linear:npl=100,tonemap=hable:desat=0,zscale=p=bt709:t=bt709:m=bt709:r=tv" \
+  -f null -
+echo "hdr: 10-bit hevc encode, zscale tonemap"
+
+# TLS against the system trust store: a trusted host passes, a wrong host name is refused.
+exec 4< <(node -e '
+  const fs = require("fs");
+  require("https").createServer({ key: fs.readFileSync("/certs/server.key"), cert: fs.readFileSync("/certs/server.crt") },
+    (_, res) => res.end(fs.readFileSync("h264.mp4"))).listen(8443, () => console.log("ready"));')
+server=$!
+trap 'kill $server' EXIT
+read -r -t 30 -u 4 ready && [ "$ready" = ready ]
+[ "$(frames https://localhost:8443/h264.mp4 -tls_verify 1)" = 25 ]
+if frames https://127.0.0.1:8443/h264.mp4 -tls_verify 1 2>/dev/null; then echo "tls accepted a wrong host" >&2; exit 1; fi
+echo "tls: verified, wrong host refused"

packages/aws-lambda/image/sources.txt

@@ -0,0 +1,8 @@
+# name sha256 url  (x264 has no stable tarballs: its line pins a git commit, r3191 as in the x86 ffmpeg-static)
+ffmpeg-7.0.2.tar.xz 8646515b638a3ad303e23af6a3587734447cb8fc0a0c064ecdb8e95c4fd8b389 https://ffmpeg.org/releases/ffmpeg-7.0.2.tar.xz
+x264 4613ac3c15fd75cebc4b9f65b7fb95e70a3acce1 https://code.videolan.org/videolan/x264.git
+x265_4.1.tar.gz a31699c6a89806b74b0151e5e6a7df65de4b49050482fe5ebf8a4379d7af8f29 https://bitbucket.org/multicoreware/x265_git/downloads/x265_4.1.tar.gz
+libvpx-1.15.0.tar.gz e935eded7d81631a538bfae703fd1e293aad1c7fd3407ba00440c95105d2011e https://github.com/webmproject/libvpx/archive/refs/tags/v1.15.0.tar.gz
+opus-1.5.2.tar.gz 65c1d2f78b9f2fb20082c38cbe47c951ad5839345876e46941612ee87f9a7ce1 https://downloads.xiph.org/releases/opus/opus-1.5.2.tar.gz
+zimg-3.0.5.tar.gz a9a0226bf85e0d83c41a8ebe4e3e690e1348682f6a2a7838f1b8cbff1b799bcf https://github.com/sekrit-twc/zimg/archive/refs/tags/release-3.0.5.tar.gz
+dav1d-1.5.1.tar.xz 401813f1f89fa8fd4295805aa5284d9aed9bc7fc1fdbe554af4292f64cbabe21 https://downloads.videolan.org/pub/videolan/dav1d/1.5.1/dav1d-1.5.1.tar.xz