diff --git a/.github/workflows/lambda-ffmpeg.yml b/.github/workflows/lambda-ffmpeg.yml new file mode 100644 index 00000000000..00c0ce4edc0 --- /dev/null +++ b/.github/workflows/lambda-ffmpeg.yml @@ -0,0 +1,47 @@ +name: Lambda arm64 ffmpeg + +permissions: + contents: read + +# Builds the native ffmpeg for the arm64 Lambda video function on two separate ARM runners. +# Each build runs the codec check on the bare Lambda base; the tarballs must be byte-identical. +on: + pull_request: + paths: + - "packages/aws-lambda/image/**" + - ".github/workflows/lambda-ffmpeg.yml" + workflow_dispatch: + +concurrency: + group: lambda-ffmpeg-${{ github.ref }} + cancel-in-progress: true + +jobs: + build: + name: Build ${{ matrix.copy }} + runs-on: ubuntu-24.04-arm + timeout-minutes: 90 + strategy: + matrix: + copy: [a, b] + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + - name: Build and check + run: >- + docker buildx build --progress=plain --platform linux/arm64 --target artifact + --output type=local,dest=out packages/aws-lambda/image + - run: cat out/hf-media-check.txt && sha256sum out/*.tar.gz + - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: hf-media-${{ matrix.copy }} + path: out/ + retention-days: 7 + + reproducible: + name: Builds match + needs: build + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 + - run: cmp hf-media-a/hf-media-linux-aarch64.tar.gz hf-media-b/hf-media-linux-aarch64.tar.gz diff --git a/packages/aws-lambda/image/Dockerfile b/packages/aws-lambda/image/Dockerfile new file mode 100644 index 00000000000..1cc7b124ba4 --- /dev/null +++ b/packages/aws-lambda/image/Dockerfile @@ -0,0 +1,34 @@ +# The arm64 Lambda video function's native ffmpeg, built on the Lambda Node 22 base it runs on: +# docker buildx build --platform linux/arm64 --target artifact --output type=local,dest=out packages/aws-lambda/image +# The artifact stage depends on the check stage, so no tarball comes out of a build whose codecs fail. +ARG BASE=public.ecr.aws/lambda/nodejs:22@sha256:d138eff445f6ea04c51091d8e6cbb064d5663e850cddfa62396061586f98b126 + +FROM ${BASE} AS build +ENV SOURCE_DATE_EPOCH=1722643200 +RUN rpm -qa --qf '%{NAME}\n' | sort >/build-base-rpms.txt && \ + microdnf install -y --nodocs --setopt=install_weak_deps=0 --releasever=2023.12.20260914 \ + gcc gcc-c++ make cmake meson ninja-build autoconf automake libtool pkgconf-pkg-config diffutils \ + git xz bzip2 tar gzip which patchelf gnutls-devel zlib-devel $([ "$(uname -m)" = x86_64 ] && echo nasm) && \ + microdnf clean all && mkdir /build && mv /build-base-rpms.txt /build/base-rpms.txt +COPY sources.txt build-ffmpeg.sh /build/ +RUN bash /build/build-ffmpeg.sh + +FROM ${BASE} AS certs +# certtool, not openssl: the base's openssl-libs cannot run the repo's openssl CLI. +RUN microdnf install -y --nodocs --releasever=2023.12.20260914 gnutls-utils && mkdir /certs && cd /certs && \ + printf 'cn = check-ca\nca\ncert_signing_key\nexpiration_days = 2\n' >ca.tmpl && \ + printf 'cn = localhost\ndns_name = localhost\ntls_www_server\nexpiration_days = 2\n' >server.tmpl && \ + certtool -p --outfile ca.key && certtool -s --load-privkey ca.key --template ca.tmpl --outfile ca.crt && \ + certtool -p --outfile server.key && \ + certtool -c --load-privkey server.key --load-ca-certificate ca.crt --load-ca-privkey ca.key \ + --template server.tmpl --outfile server.crt + +FROM ${BASE} AS check +COPY --from=build /opt/hf-media /opt/hf-media +COPY --from=certs /certs /certs +COPY check-ffmpeg.sh /check-ffmpeg.sh +RUN cp /certs/ca.crt /etc/pki/ca-trust/source/anchors/ && update-ca-trust && bash /check-ffmpeg.sh >/check.txt + +FROM scratch AS artifact +COPY --from=build /out/ / +COPY --from=check /check.txt /hf-media-check.txt diff --git a/packages/aws-lambda/image/build-ffmpeg.sh b/packages/aws-lambda/image/build-ffmpeg.sh new file mode 100755 index 00000000000..65492cfe22c --- /dev/null +++ b/packages/aws-lambda/image/build-ffmpeg.sh @@ -0,0 +1,92 @@ +#!/bin/bash +# Builds ffmpeg + ffprobe into /opt/hf-media and packs /out/hf-media-linux-.tar.gz. +# Codec libraries link statically; glibc and the TLS stack stay dynamic (a static glibc build crashed on Lambda). +set -euo pipefail + +PREFIX=/opt/hf-media +DEPS=/build/deps +SRC=/build/src +JOBS=$(nproc) +export PKG_CONFIG_PATH="$DEPS/lib/pkgconfig:$DEPS/lib64/pkgconfig" +export CFLAGS="-O2 -fPIC" CXXFLAGS="-O2 -fPIC" +mkdir -p "$SRC" "$DEPS" "$PREFIX/share/licenses" /out + +fetch() { + local name=$1 sum url + read -r sum url < <(awk -v n="$name" '$1 == n { print $2, $3 }' /build/sources.txt) + curl -fsSL --retry 3 -o "$SRC/$name" "$url" + echo "$sum $SRC/$name" | sha256sum -c --quiet + tar -xf "$SRC/$name" -C "$SRC" +} + +license() { mkdir -p "$PREFIX/share/licenses/$1" && cp "${@:2}" "$PREFIX/share/licenses/$1/"; } + +read -r x264_commit x264_url < <(awk '$1 == "x264" { print $2, $3 }' /build/sources.txt) +git clone -q "$x264_url" "$SRC/x264" +git -C "$SRC/x264" checkout -q "$x264_commit" +[ "$(git -C "$SRC/x264" rev-parse HEAD)" = "$x264_commit" ] +(cd "$SRC/x264" && ./configure --prefix="$DEPS" --enable-static --enable-pic --disable-cli && + make -j"$JOBS" && make install) +license x264 "$SRC/x264/COPYING" + +fetch x265_4.1.tar.gz +# 8-bit x265 with the 10-bit library linked in (x265's multilib layout): HDR renders encode yuv420p10le. +# SVE2 kernels need -march=armv9-a, which the base's gcc 11 does not know (x86 ignores the flag). +x265=(-S "$SRC/x265_4.1/source" -DCMAKE_BUILD_TYPE=Release -DENABLE_SHARED=OFF -DENABLE_CLI=OFF -DENABLE_SVE2=OFF) +cmake "${x265[@]}" -B "$SRC/x265-10" -DHIGH_BIT_DEPTH=ON -DEXPORT_C_API=OFF +cmake --build "$SRC/x265-10" -j"$JOBS" +cmake "${x265[@]}" -B "$SRC/x265-8" -DCMAKE_INSTALL_PREFIX="$DEPS" -DLINKED_10BIT=ON \ + -DEXTRA_LIB="$SRC/x265-10/libx265.a" +cmake --build "$SRC/x265-8" -j"$JOBS" +cmake --install "$SRC/x265-8" +printf 'CREATE %s\nADDLIB %s\nADDLIB %s\nSAVE\nEND\n' "$DEPS/lib/libx265.a" \ + "$SRC/x265-8/libx265.a" "$SRC/x265-10/libx265.a" | ar -M +license x265 "$SRC/x265_4.1/COPYING" + +fetch libvpx-1.15.0.tar.gz +(cd "$SRC/libvpx-1.15.0" && ./configure --prefix="$DEPS" --enable-static --disable-shared --enable-pic \ + --enable-vp9-highbitdepth --disable-examples --disable-tools --disable-docs --disable-unit-tests && + make -j"$JOBS" && make install) +license libvpx "$SRC/libvpx-1.15.0/LICENSE" "$SRC/libvpx-1.15.0/PATENTS" + +fetch opus-1.5.2.tar.gz +(cd "$SRC/opus-1.5.2" && ./configure --prefix="$DEPS" --enable-static --disable-shared --with-pic \ + --disable-doc --disable-extra-programs && make -j"$JOBS" && make install) +license opus "$SRC/opus-1.5.2/COPYING" + +fetch zimg-3.0.5.tar.gz +(cd "$SRC/zimg-release-3.0.5" && ./autogen.sh && ./configure --prefix="$DEPS" --enable-static --disable-shared \ + --with-pic && make -j"$JOBS" && make install) +license zimg "$SRC/zimg-release-3.0.5/COPYING" + +fetch dav1d-1.5.1.tar.xz +meson setup "$SRC/dav1d-build" "$SRC/dav1d-1.5.1" --prefix="$DEPS" --libdir=lib --buildtype=release \ + --default-library=static -Denable_tools=false -Denable_tests=false +ninja -C "$SRC/dav1d-build" install +license dav1d "$SRC/dav1d-1.5.1/COPYING" + +fetch ffmpeg-7.0.2.tar.xz +# No --static pkg-config: gnutls.pc then names a libgmp.a from its own RPM build tree. The deps above +# exist only as .a, so they link statically anyway; --extra-libs carries their private libraries. +(cd "$SRC/ffmpeg-7.0.2" && ./configure --prefix="$PREFIX" \ + --extra-cflags="-I$DEPS/include" --extra-ldflags="-L$DEPS/lib" --extra-libs="-lstdc++ -lpthread -lm -ldl" \ + --enable-gpl --enable-version3 --disable-debug --disable-doc --disable-ffplay --disable-autodetect \ + --enable-gnutls --enable-zlib --enable-libx264 --enable-libx265 --enable-libvpx --enable-libopus \ + --enable-libzimg --enable-libdav1d && make -j"$JOBS" && make install) +license ffmpeg "$SRC/ffmpeg-7.0.2/COPYING.GPLv3" "$SRC/ffmpeg-7.0.2/LICENSE.md" +rm -rf "$PREFIX/include" "$PREFIX/lib" "$PREFIX/share/ffmpeg" +strip "$PREFIX/bin/ffmpeg" "$PREFIX/bin/ffprobe" + +# Bundle every shared library the bare base image lacks (gnutls, nettle); the rpath finds them. +mkdir -p "$PREFIX/lib" +ldd "$PREFIX/bin/ffmpeg" "$PREFIX/bin/ffprobe" | awk '$2 == "=>" { print $3 }' | sort -u | while read -r lib; do + if ! grep -qx "$(rpm -qf --qf '%{NAME}' "$lib")" /build/base-rpms.txt; then + cp -L "$lib" "$PREFIX/lib/" && patchelf --set-rpath '$ORIGIN' "$PREFIX/lib/$(basename "$lib")" + echo "$(rpm -qf "$lib") $(basename "$lib")" >>"$PREFIX/share/licenses/bundled-libraries.txt" + fi +done +patchelf --set-rpath '$ORIGIN/../lib' "$PREFIX/bin/ffmpeg" "$PREFIX/bin/ffprobe" +cp /build/sources.txt "$PREFIX/share/licenses/sources.txt" + +tar --sort=name --mtime="@$SOURCE_DATE_EPOCH" --owner=0 --group=0 --numeric-owner -C /opt -cf - hf-media | + gzip -n -9 >"/out/hf-media-linux-$(uname -m).tar.gz" diff --git a/packages/aws-lambda/image/check-ffmpeg.sh b/packages/aws-lambda/image/check-ffmpeg.sh new file mode 100755 index 00000000000..3aa601ebbdf --- /dev/null +++ b/packages/aws-lambda/image/check-ffmpeg.sh @@ -0,0 +1,61 @@ +#!/bin/bash +# Runs on the bare Lambda base with only /opt/hf-media added: every codec the engine calls must work. +# Prints a deterministic summary to stdout; the artifact stage keeps it. +set -euo pipefail +ff() { /opt/hf-media/bin/ffmpeg -hide_banner -loglevel error -nostdin "$@"; } +probe() { local input=$1; shift; /opt/hf-media/bin/ffprobe -hide_banner -loglevel error "$@" -- "$input"; } +listed() { # listed + local names + names=$(/opt/hf-media/bin/ffmpeg -hide_banner "$1" | awk -v f="$2" '{ print $f }') + for name in "${@:3}"; do grep -qx -- "$name" <<<"$names" || { echo "missing $1 $name" >&2; exit 1; }; done + echo "${1#-}: ${*:3}" +} +frames() { probe "$1" -count_frames -select_streams v:0 -show_entries stream=nb_read_frames -of csv=p=0 "${@:2}"; } + +cd "$(mktemp -d)" +listed -encoders 2 libx264 libx265 libvpx-vp9 libopus prores_ks aac png mjpeg +listed -decoders 2 libvpx-vp9 libvpx libdav1d h264 hevc prores aac opus png mjpeg +listed -filters 2 zscale tonemap setparams ebur128 alphamerge overlay +listed -protocols 1 https tls file + +src=(-f lavfi -i "testsrc2=s=320x240:r=25:d=1") +ff "${src[@]}" -c:v libx264 -pix_fmt yuv420p -bf 0 h264.mp4 +ff "${src[@]}" -c:v libx265 -pix_fmt yuv420p -tag:v hvc1 -x265-params log-level=error h265.mp4 +ff "${src[@]}" -c:v prores_ks -profile:v 4444 -pix_fmt yuva444p10le prores.mov +ff -f lavfi -i "sine=d=1" -c:a libopus -b:a 128k opus.webm +ff -f lavfi -i "sine=d=1" -c:a aac -b:a 192k aac.m4a +ff "${src[@]}" -frames:v 1 frame.png +for f in h264.mp4 h265.mp4 prores.mov; do [ "$(frames "$f")" = 25 ] || { echo "decode $f" >&2; exit 1; }; done +for f in opus.webm aac.m4a; do ff -i "$f" -f null -; done +[ "$(probe frame.png -show_entries stream=width -of csv=p=0)" = 320 ] +echo "encode+decode: h264 h265 prores opus aac png" + +# VP9 with alpha: left half transparent, decoded through libvpx-vp9 as the engine does. +ff "${src[@]}" -vf "format=yuva420p,geq=lum='lum(X,Y)':cb='cb(X,Y)':cr='cr(X,Y)':a='if(lt(X,W/2),0,255)'" \ + -c:v libvpx-vp9 -pix_fmt yuva420p -auto-alt-ref 0 -b:v 0 -crf 30 alpha.webm +ff -c:v libvpx-vp9 -i alpha.webm -frames:v 1 -vf alphaextract,crop=40:40:20:100 -f rawvideo -pix_fmt gray left.raw +ff -c:v libvpx-vp9 -i alpha.webm -frames:v 1 -vf alphaextract,crop=40:40:260:100 -f rawvideo -pix_fmt gray right.raw +node -e ' + const fs = require("fs"), mean = (f) => fs.readFileSync(f).reduce((a, b) => a + b, 0) / 1600; + if (!(mean("left.raw") < 16 && mean("right.raw") > 240)) throw new Error("vp9 alpha lost");' +echo "vp9 alpha: encode+decode" + +# The engine's HDR to SDR chain (core/src/hdrToneMap.ts) on tagged 10-bit HEVC. +ff "${src[@]}" -c:v libx265 -pix_fmt yuv420p10le -color_primaries bt2020 -color_trc smpte2084 \ + -colorspace bt2020nc -x265-params log-level=error hdr.mp4 +[ "$(probe hdr.mp4 -select_streams v:0 -show_entries stream=pix_fmt -of csv=p=0)" = yuv420p10le ] +ff -i hdr.mp4 -vf "zscale=t=linear:npl=100,tonemap=hable:desat=0,zscale=p=bt709:t=bt709:m=bt709:r=tv" \ + -f null - +echo "hdr: 10-bit hevc encode, zscale tonemap" + +# TLS against the system trust store: a trusted host passes, a wrong host name is refused. +exec 4< <(node -e ' + const fs = require("fs"); + require("https").createServer({ key: fs.readFileSync("/certs/server.key"), cert: fs.readFileSync("/certs/server.crt") }, + (_, res) => res.end(fs.readFileSync("h264.mp4"))).listen(8443, () => console.log("ready"));') +server=$! +trap 'kill $server' EXIT +read -r -t 30 -u 4 ready && [ "$ready" = ready ] +[ "$(frames https://localhost:8443/h264.mp4 -tls_verify 1)" = 25 ] +if frames https://127.0.0.1:8443/h264.mp4 -tls_verify 1 2>/dev/null; then echo "tls accepted a wrong host" >&2; exit 1; fi +echo "tls: verified, wrong host refused" diff --git a/packages/aws-lambda/image/sources.txt b/packages/aws-lambda/image/sources.txt new file mode 100644 index 00000000000..66e2683544e --- /dev/null +++ b/packages/aws-lambda/image/sources.txt @@ -0,0 +1,8 @@ +# name sha256 url (x264 has no stable tarballs: its line pins a git commit, r3191 as in the x86 ffmpeg-static) +ffmpeg-7.0.2.tar.xz 8646515b638a3ad303e23af6a3587734447cb8fc0a0c064ecdb8e95c4fd8b389 https://ffmpeg.org/releases/ffmpeg-7.0.2.tar.xz +x264 4613ac3c15fd75cebc4b9f65b7fb95e70a3acce1 https://code.videolan.org/videolan/x264.git +x265_4.1.tar.gz a31699c6a89806b74b0151e5e6a7df65de4b49050482fe5ebf8a4379d7af8f29 https://bitbucket.org/multicoreware/x265_git/downloads/x265_4.1.tar.gz +libvpx-1.15.0.tar.gz e935eded7d81631a538bfae703fd1e293aad1c7fd3407ba00440c95105d2011e https://github.com/webmproject/libvpx/archive/refs/tags/v1.15.0.tar.gz +opus-1.5.2.tar.gz 65c1d2f78b9f2fb20082c38cbe47c951ad5839345876e46941612ee87f9a7ce1 https://downloads.xiph.org/releases/opus/opus-1.5.2.tar.gz +zimg-3.0.5.tar.gz a9a0226bf85e0d83c41a8ebe4e3e690e1348682f6a2a7838f1b8cbff1b799bcf https://github.com/sekrit-twc/zimg/archive/refs/tags/release-3.0.5.tar.gz +dav1d-1.5.1.tar.xz 401813f1f89fa8fd4295805aa5284d9aed9bc7fc1fdbe554af4292f64cbabe21 https://downloads.videolan.org/pub/videolan/dav1d/1.5.1/dav1d-1.5.1.tar.xz