From 5dadf1ea96f96e12b8cd54c9747c968647d0de29 Mon Sep 17 00:00:00 2001 From: Ludovic Henry Date: Sat, 19 Sep 2026 07:50:48 +0000 Subject: [PATCH 1/5] torchcodec: build 0.16.0 wheels for riscv64 torchcodec is PyTorch's video decoder: a scikit-build-core/CMake project that compiles a C++20 core against libtorch and FFmpeg, plus a separate, FFmpeg-free image-decoder library. Upstream links its wheels against prebuilt non-GPL FFmpeg tarballs it hosts on S3, which carry no riscv64 build, so the workflow takes the project's other supported path -- pkg-config against an installed FFmpeg -- and builds that FFmpeg from source in the container, LGPL and never shipped. FFmpeg 6.1 because the major version is baked into the libtorchcodec_core.so name the Python loader dlopen()s, and 6.1 is what ubuntu-24.04-riscv ships. AVIF and HEIC image decoding are off: libavif comes from the same S3 bucket and libheif is packaged in neither Rocky 10 nor a riscv64 EPEL. JPEG, PNG, WebP and the vendored giflib are built as usual. Also records two reusable gotchas found along the way (383, 384). --- .github/workflows/build-torchcodec.yml | 199 ++++++++++++++++++ docs/packages/torchcodec.yaml | 5 + ...e-licence-texts-of-the-bundled-image.patch | 49 +++++ .../references/gotchas-index.md | 9 +- .../gotchas/local-validation-and-rehearsal.md | 23 ++ .../gotchas/native-deps-and-linking.md | 28 +++ 6 files changed, 312 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/build-torchcodec.yml create mode 100644 docs/packages/torchcodec.yaml create mode 100644 patches/torchcodec/0.16.0/0001-pyproject-glob-the-licence-texts-of-the-bundled-image.patch diff --git a/.github/workflows/build-torchcodec.yml b/.github/workflows/build-torchcodec.yml new file mode 100644 index 0000000000..6e653ed126 --- /dev/null +++ b/.github/workflows/build-torchcodec.yml @@ -0,0 +1,199 @@ +# SPDX-FileCopyrightText: 2026 The RISE Project +# SPDX-License-Identifier: MIT +--- +# This workflow is based on: https://github.com/pytorch/torchcodec/blob/main/.github/workflows/linux_wheel.yaml +name: Build torchcodec wheels (riscv64) + +on: + workflow_dispatch: + inputs: + version: + description: 'Version glob to (re)build; empty builds every version of docs/packages/torchcodec.yaml not released yet' + required: false + default: '' + pull_request: + branches: [main] + paths: + - '.github/workflows/build-torchcodec.yml' + - 'docs/packages/torchcodec.yaml' + - 'patches/torchcodec/**' + push: + branches: [main] + paths: + - '.github/workflows/build-torchcodec.yml' + - 'docs/packages/torchcodec.yaml' + - 'patches/torchcodec/**' + +run-name: build-torchcodec ${{ inputs.version && format('- {0}', inputs.version) || '' }} + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +permissions: + contents: read # to fetch code (actions/checkout) + +env: + # Upstream links against prebuilt non-GPL FFmpeg tarballs it hosts on S3, which have no + # riscv64 build, so FFmpeg is built from source here (LGPL: no --enable-gpl/--enable-nonfree) + # and never shipped. 6.1 because that is the libavutil ABI of the system FFmpeg on + # ubuntu-24.04-riscv, where these wheels are consumed: the FFmpeg major version is baked + # into the libtorchcodec_core.so name that torchcodec dlopen()s at runtime. + FFMPEG_VERSION: 6.1.6 + +jobs: + setup: + uses: $/.github/workflows/_setup.yml + with: + package: torchcodec + version: ${{ inputs.version }} + + build_wheels: + needs: [setup] + if: needs.setup.outputs.versions != '[]' + name: Build torchcodec ${{ matrix.version }} cp${{ matrix.python-version }}-manylinux_riscv64 + runs-on: ubuntu-24.04-riscv + timeout-minutes: 360 + strategy: + fail-fast: false + matrix: + version: ${{ fromJSON(needs.setup.outputs.versions) }} + # Match torch's own riscv64 matrix on the registry: torchcodec's pybind11 module is + # per-ABI, so there is no single abi3 wheel even though it targets torch's stable ABI. + # See https://pypi.riseproject.dev/python-wheels/development.html#target-python-versions + python-version: [312, 313, 314, 314t] + + env: + TORCHCODEC_VERSION: ${{ matrix.version }} + + steps: + - name: Checkout torchcodec v${{ env.TORCHCODEC_VERSION }} + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: pytorch/torchcodec + ref: v${{ env.TORCHCODEC_VERSION }} + persist-credentials: false + + - name: Checkout python-wheels + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + path: python-wheels + persist-credentials: false + + - name: Patch torchcodec source + run: git apply python-wheels/patches/torchcodec/${{ env.TORCHCODEC_VERSION }}/00*.patch + + - uses: pypa/cibuildwheel@1828c10ab37f080699c7b81cea34097c684a7074 # v4.2.0 + env: + CIBW_BUILD: "cp${{ matrix.python-version }}-manylinux_riscv64" + # setup is scikit-build-core driving CMake, which resolves Torch_DIR by running + # `import torch` in the build interpreter; torch only exists on our registry, so it + # cannot be declared in build-system.requires and isolation has to be off. + CIBW_BUILD_FRONTEND: "pip; args: --no-build-isolation" + # auditwheel grafts libjpeg-turbo/libpng/libwebp/zlib into the wheel, so their + # licence texts have to travel with it (the rpm that owns each bundled .so is + # resolved rather than hardcoded; %license files are dropped by the image's + # tsflags=nodocs and come back with an explicit reinstall). LICENSE.* is what the + # patched pyproject.toml globs into .dist-info/licenses/. + CIBW_BEFORE_ALL: | + set -ex + dnf install -y --setopt=tsflags= pkgconfig libjpeg-turbo-devel libpng-devel libwebp-devel zlib-devel + devel=$(rpm -q --whatprovides --qf '%{NAME}\n' libjpeg-turbo-devel libpng-devel libwebp-devel zlib-devel | sort -u) + libs=$(rpm -ql $devel | grep -E '\.so$' | xargs -r readlink -f) + pkgs=$(rpm -qf --qf '%{NAME}\n' $libs | grep -E '^[A-Za-z0-9._+-]+$' | sort -u) + dnf reinstall -y --setopt=tsflags= $pkgs + for p in $pkgs; do + for f in /usr/share/licenses/$p/*; do + [ -f "$f" ] || continue + cp "$f" "{project}/LICENSE.$p.${f##*/}" + done + done + ls {project}/LICENSE.* + mkdir -p /tmp/ffmpeg + curl -fsSL "https://github.com/FFmpeg/FFmpeg/archive/refs/tags/n${FFMPEG_VERSION}.tar.gz" | tar xz --strip-components=1 -C /tmp/ffmpeg + cd /tmp/ffmpeg + ./configure --prefix=/usr/local --disable-static --enable-shared --enable-pic --disable-doc + make -j"$(nproc)" install + ldconfig + CIBW_BEFORE_BUILD: >- + pip install --only-binary=:all: + torch>=2.11 "scikit-build-core>=0.10" pybind11 ninja setuptools wheel + # I_CONFIRM_THIS_IS_NOT_A_LICENSE_VIOLATION is upstream's own escape hatch for + # wheels built against an FFmpeg that is not their S3 one; the guard exists to stop + # a GPL FFmpeg being linked, and the build above is LGPL. + # AVIF and HEIC are off because neither can be had on riscv64: libavif comes from + # the same S3 bucket and libheif is packaged in neither Rocky 10 nor a riscv64 EPEL. + # decode_avif/decode_heic then raise an actionable error instead of failing the build. + CIBW_ENVIRONMENT: >- + FFMPEG_VERSION=${{ env.FFMPEG_VERSION }} + BUILD_VERSION=${{ env.TORCHCODEC_VERSION }} + I_CONFIRM_THIS_IS_NOT_A_LICENSE_VIOLATION=1 + TORCHCODEC_BUILD_AVIF=0 + TORCHCODEC_BUILD_HEIC=0 + TORCHCODEC_DISABLE_COMPILE_WARNING_AS_ERROR=1 + PKG_CONFIG_PATH=/usr/local/lib/pkgconfig + PIP_EXTRA_INDEX_URL=https://pypi.riseproject.dev/simple/ + # Same exclude list as upstream's packaging/repair_wheel.py: FFmpeg is a runtime + # dependency the user supplies, and libtorch/libc10/libgomp come from the torch + # wheel, which is imported (RTLD_GLOBAL) before torchcodec loads its own libraries. + CIBW_REPAIR_WHEEL_COMMAND: >- + auditwheel repair -w {dest_dir} {wheel} + --exclude 'libavcodec*' + --exclude 'libavdevice*' + --exclude 'libavfilter*' + --exclude 'libavformat*' + --exclude 'libavutil*' + --exclude 'libswresample*' + --exclude 'libswscale*' + --exclude 'libpostproc*' + --exclude 'libtorch*' + --exclude 'libc10*' + --exclude libgomp.so.1 + CIBW_TEST_REQUIRES: torch>=2.11 numpy pytest pillow + # pyproject.toml is staged alongside test/ purely for its [tool.pytest.ini_options] + # (the needs_* markers and the 'not slow' default); src/ deliberately is not, which + # is what upstream's packaging/remove_src.sh achieves before its own test jobs. + CIBW_TEST_SOURCES: test pyproject.toml + # FAIL_WITHOUT_ turns a missing image codec from a silent skip into a + # failure, for the three we do build. test_transform_ops.py imports torchvision at + # module scope and pypi.riseproject.dev has no riscv64 torchvision yet. + CIBW_TEST_COMMAND: FAIL_WITHOUT_JPEG=1 FAIL_WITHOUT_PNG=1 FAIL_WITHOUT_WEBP=1 pytest --ignore=test/test_transform_ops.py test + + - name: Check the wheel bundles the image codecs and no FFmpeg + run: | + python3 - wheelhouse/*.whl <<'EOF' + import sys, zipfile + + names = zipfile.ZipFile(sys.argv[1]).namelist() + for want in ("torchcodec/libtorchcodec_core6.so", "torchcodec/libtorchcodec_image.so", + "libjpeg", "libpng", "libwebp", "licenses/LICENSE"): + if not any(want in n for n in names): + raise SystemExit(f"error: {want} missing from {sys.argv[1]}") + for unwanted in ("libavcodec", "libavutil", "libtorch.so", "libc10.so"): + if any(unwanted in n for n in names): + raise SystemExit(f"error: {unwanted} must not be bundled in {sys.argv[1]}") + print("\n".join(n for n in names if ".so" in n or "licenses/" in n)) + EOF + + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: torchcodec-${{ env.TORCHCODEC_VERSION }}-cp${{ matrix.python-version }}-manylinux_riscv64 + path: ./wheelhouse/*.whl + if-no-files-found: error + + publish: + name: Publish torchcodec ${{ matrix.version }} + needs: [setup, build_wheels] + if: needs.setup.outputs.versions != '[]' + strategy: + fail-fast: false + matrix: + version: ${{ fromJSON(needs.setup.outputs.versions) }} + permissions: + contents: write + pull-requests: write + uses: $/.github/workflows/_publish-wheel.yml + secrets: + app-private-key: ${{ secrets.RISEPROJECT_APP_PRIVATE_KEY }} + with: + artifact-pattern: torchcodec-${{ matrix.version }}-*-manylinux_riscv64 diff --git a/docs/packages/torchcodec.yaml b/docs/packages/torchcodec.yaml new file mode 100644 index 0000000000..9a571048bd --- /dev/null +++ b/docs/packages/torchcodec.yaml @@ -0,0 +1,5 @@ +package-name: torchcodec +source-code: https://github.com/pytorch/torchcodec +license: BSD-3-Clause AND MIT AND Zlib AND libpng-2.0 AND IJG +versions: +- version: 0.16.0 diff --git a/patches/torchcodec/0.16.0/0001-pyproject-glob-the-licence-texts-of-the-bundled-image.patch b/patches/torchcodec/0.16.0/0001-pyproject-glob-the-licence-texts-of-the-bundled-image.patch new file mode 100644 index 0000000000..ebd81722fe --- /dev/null +++ b/patches/torchcodec/0.16.0/0001-pyproject-glob-the-licence-texts-of-the-bundled-image.patch @@ -0,0 +1,49 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: Ludovic Henry +Date: Fri, 19 Sep 2026 12:00:00 +0200 +Subject: [PATCH] pyproject: glob the licence texts of the bundled image codecs + +Upstream-Status: Inappropriate [our wheels are repaired by cibuildwheel's auditwheel, not by upstream's packaging/repair_wheel.py] + +torchcodec redistributes libjpeg-turbo, libpng, zlib and libwebp as +binaries inside the wheel, and their (permissive) licences require the +copyright notice to travel with the binary. Upstream satisfies that in +packaging/repair_wheel.py's bundle_third_party_licenses(), which unpacks +each freshly built wheel, copies the licence texts out of the conda +packages the libraries came from into .dist-info/licenses/third_party/, +and repacks. That path is unusable here for two independent reasons: it +resolves the texts through CONDA_PREFIX/conda-meta (there is no conda +environment in the manylinux image, the libraries come from Rocky rpms), +and it hardcodes dist/ -> dist_repaired/ plus a mandatory S3 libavif +directory, neither of which exists under cibuildwheel's own auditwheel +repair step. + +So the texts are staged next to the project's own LICENSE instead, as +LICENSE.., and picked up by widening the PEP 639 +license-files list. An explicit license-files list has no default glob +behind it, so without this the added files are silently dropped and the +wheel ships only torchcodec's own BSD text. + +Not riscv64-specific in substance, but it only makes sense for a build +that repairs wheels with plain auditwheel, so there is nothing to send +upstream. +--- + pyproject.toml | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/pyproject.toml b/pyproject.toml +index 4a52430..fde71b9 100644 +--- a/pyproject.toml ++++ b/pyproject.toml +@@ -3,7 +3,7 @@ name = "torchcodec" + description = "A video decoder for PyTorch" + readme = "README.md" + requires-python = ">=3.10" +-license-files = ["LICENSE"] ++license-files = ["LICENSE", "LICENSE.*"] + authors = [ + { name = "PyTorch Team", email = "packages@pytorch.org" }, + ] +-- +2.51.0 + diff --git a/skills/python-project-porting/references/gotchas-index.md b/skills/python-project-porting/references/gotchas-index.md index 3da4263fce..29a02a2098 100644 --- a/skills/python-project-porting/references/gotchas-index.md +++ b/skills/python-project-porting/references/gotchas-index.md @@ -1,6 +1,6 @@ # Gotchas index — router for the themed gotcha files -The porting gotchas (370 of them) live in [`references/gotchas/`](gotchas/), split by theme so only the relevant slice loads. Every gotcha keeps a **permanent number** cited elsewhere as "gotcha N" (and in workflow comments as "CLAUDE.md gotcha N"). Numbers are stable IDs — **not sequential**, and four are **reused** with different content (two each of 33, 55, 56, 57), disambiguated by theme below. +The porting gotchas (374 of them) live in [`references/gotchas/`](gotchas/), split by theme so only the relevant slice loads. Every gotcha keeps a **permanent number** cited elsewhere as "gotcha N" (and in workflow comments as "CLAUDE.md gotcha N"). Numbers are stable IDs — **not sequential**, and four are **reused** with different content (two each of 33, 55, 56, 57), disambiguated by theme below. ## How to find the gotcha you need @@ -325,6 +325,10 @@ The porting gotchas (370 of them) live in [`references/gotchas/`](gotchas/), spl - **278** — A vendored, direct-copy (not submodule) header can be missing riscv64 from its - **363** — A `libraries=[...]` entry can go missing from the link line with *no* error — - **368** — Linking several codecs against Rocky 10's system libraries instead of +- **384** — When a project dlopen()s a differently-named shared library per major + version of a native dependency, the version you build against is a user-visible ABI + contract — pick the major that the target platform ships, build it LGPL, exclude it + from the wheel, and assert the resulting `.so` name. ### Compiled-vs-pure detection & the require-extension knob — [`gotchas/compiled-vs-pure-detection.md`](gotchas/compiled-vs-pure-detection.md) @@ -523,6 +527,9 @@ The porting gotchas (370 of them) live in [`references/gotchas/`](gotchas/), spl - **223** — For a `bindings = "bin"` CLI's test assertions, `cargo build --release` the tool - **298** — A local rehearsal's `pip`-resolved cibuildwheel can be too old for - **369** — Without docker, fetch Rocky 10's own dnf repodata over plain HTTPS to +- **383** — A libtorch-linking project cannot be rehearsed on x86_64 with PyPI's `torch` + wheel: it is a CUDA build, so `find_package(Torch)` dies in `Caffe2Config.cmake` before + reaching the project's own CMake. Everything up to that point still validates. ### PR, CI, triggers, publishing & maintainer signals — [`gotchas/pr-ci-and-maintainer.md`](gotchas/pr-ci-and-maintainer.md) diff --git a/skills/python-project-porting/references/gotchas/local-validation-and-rehearsal.md b/skills/python-project-porting/references/gotchas/local-validation-and-rehearsal.md index f091a69f57..261e8bb828 100644 --- a/skills/python-project-porting/references/gotchas/local-validation-and-rehearsal.md +++ b/skills/python-project-porting/references/gotchas/local-validation-and-rehearsal.md @@ -17,6 +17,7 @@ To pull up one entry: `grep -n '^N\. ' references/gotchas/local-validation-and-r - **223** — For a `bindings = "bin"` CLI's test assertions, `cargo build --release` the tool - **298** — A local rehearsal's `pip`-resolved cibuildwheel can be too old for - **369** — Without docker, fetch Rocky 10's own dnf repodata over plain HTTPS to +- **383** — A libtorch-linking project cannot be rehearsed on x86_64 with PyPI's `torch` --- @@ -247,3 +248,25 @@ To pull up one entry: `grep -n '^N\. ' references/gotchas/local-validation-and-r by package `name=` attribute to jump straight to its block rather than loading the whole file, and delete it when done; it is Rocky's own public mirror data, not anything project-specific worth keeping. + +383. **A project that links libtorch cannot be rehearsed on an x86_64 host with the + `torch` wheel PyPI serves, because that one is a CUDA build: `find_package(Torch)` + pulls in `Caffe2Config.cmake`, which hard-fails with "Your installed Caffe2 version + uses CUDA but I cannot find the CUDA libraries" before CMake reaches a single line + of the project's own configuration (the torchcodec case).** The failure has nothing + to do with the project or with riscv64 — the riscv64 `torch` on our registry is a + `+cpu` build whose `Caffe2Config.cmake` has the CUDA branch compiled out, so the same + configure succeeds there. Two consequences worth knowing before spending a rehearsal + cycle on it: + - **A CPU-only torch is the prerequisite for any local rehearsal of a libtorch + extension**, and PyPI has none for linux x86_64 (the CPU variants live on + `download.pytorch.org/whl/cpu`, a separate index); linux aarch64's PyPI `torch` + *is* CPU-only, which is one more reason gotcha 101's aarch64 rehearsal is the right + host for this family of packages. + - **Everything before `find_package(Torch)` still validates cheaply on x86**, and for + a scikit-build-core/CMake project that is most of the interesting surface: the + build frontend and `--no-build-isolation` wiring, `pkg-config` discovery of a + source-built native dependency, the backend finding `pybind11`, and any + licence-guard/env-var gate the project puts in front of a wheel build. Run it and + read how far the configure got rather than treating the CUDA error as a dead end. + diff --git a/skills/python-project-porting/references/gotchas/native-deps-and-linking.md b/skills/python-project-porting/references/gotchas/native-deps-and-linking.md index e3609bc616..66b5831cd6 100644 --- a/skills/python-project-porting/references/gotchas/native-deps-and-linking.md +++ b/skills/python-project-porting/references/gotchas/native-deps-and-linking.md @@ -22,6 +22,7 @@ To pull up one entry: `grep -n '^N\. ' references/gotchas/native-deps-and-linkin - **231** — A vendored C library's own CMake can carry a genuine, tested riscv64 branch — - **363** — A `libraries=[...]` entry can go missing from the link line with *no* error — - **368** — Linking several codecs against Rocky 10's system libraries instead of +- **384** — When a project dlopen()s a differently-named shared library per major version --- @@ -495,3 +496,30 @@ To pull up one entry: `grep -n '^N\. ' references/gotchas/native-deps-and-linkin compiler flag; libpng's own `CHANGES` file (`grep -n cICP CHANGES` against the tagged release) gives the exact version the symbol was added in, and confirms Rocky 10's 1.6.40 predates it — a real wall, not a flag away. + +384. **When a project dlopen()s a differently-named shared library per major version of + a native dependency, the version you build against is not an implementation detail — + it is an ABI contract with whatever the *user's* machine has installed (the torchcodec + case).** torchcodec compiles `libtorchcodec_core.so` where `N` is the FFmpeg major + version, and at import time tries `N` = 9, 8, 7, 6, 5, 4 in turn, using the first that + loads; upstream ships all six in one wheel by linking against prebuilt non-GPL FFmpeg + tarballs it hosts on S3. Those tarballs have no riscv64 build, so the riscv64 wheel + goes down the project's other path — `pkg-config` against one installed FFmpeg — and + therefore carries exactly one `N`. That makes the FFmpeg version a **user-visible** + choice: build against 7.x and the wheel imports on nothing that ships FFmpeg 6. + - **Pick the major version of the platform the wheels are consumed on**, not the + newest release: `ubuntu-24.04-riscv` (the runner these wheels target) ships FFmpeg + 6.1, so a 6.1.x source build is what makes `apt install ffmpeg` enough for a user. + The mapping is from the *library* soname, not the FFmpeg release number — FFmpeg 6 + is `libavcodec.so.60`/`libavutil.so.58`, and it is the `libavcodec` major that the + project's CMake switches on. + - **Build it LGPL and do not ship it.** No `--enable-gpl`/`--enable-nonfree` and no + third-party codec integrations keeps the FFmpeg build itself LGPL, and excluding + `libav*`/`libsw*`/`libpostproc*` from `auditwheel repair` keeps it out of the wheel + entirely — which is also what upstream's own `packaging/repair_wheel.py` enforces, + since FFmpeg is a runtime dependency the user supplies. + - **Assert the resulting `.so` name in a post-build step.** `libtorchcodec_core6.so` + present in the wheel is the one-line proof that the FFmpeg the container built is + the FFmpeg that got linked; a silent fallback to a different major would otherwise + only surface as an ImportError on a user's machine. + From e90e49a6632c44fc25d2e1087d97bb93eda22159 Mon Sep 17 00:00:00 2001 From: Ludovic Henry Date: Sat, 19 Sep 2026 07:54:01 +0000 Subject: [PATCH 2/5] torchcodec: make the licence reinstall non-fatal, extend gotcha 369 RPM installs %license files regardless of tsflags=nodocs (only %doc is dropped), and all four packages whose licences the wheel needs mark them %license -- confirmed against the CentOS Stream 10 spec files, which are also where `%files devel` confirms libwebp-devel ships the CMake package config find_package(WebP) needs. The reinstall stays as belt-and-braces but must not fail the build when a package is no longer available from a repo. --- .github/workflows/build-torchcodec.yml | 2 +- .../references/gotchas/local-validation-and-rehearsal.md | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-torchcodec.yml b/.github/workflows/build-torchcodec.yml index 6e653ed126..78bdae7e7a 100644 --- a/.github/workflows/build-torchcodec.yml +++ b/.github/workflows/build-torchcodec.yml @@ -101,7 +101,7 @@ jobs: devel=$(rpm -q --whatprovides --qf '%{NAME}\n' libjpeg-turbo-devel libpng-devel libwebp-devel zlib-devel | sort -u) libs=$(rpm -ql $devel | grep -E '\.so$' | xargs -r readlink -f) pkgs=$(rpm -qf --qf '%{NAME}\n' $libs | grep -E '^[A-Za-z0-9._+-]+$' | sort -u) - dnf reinstall -y --setopt=tsflags= $pkgs + dnf reinstall -y --setopt=tsflags= $pkgs || true for p in $pkgs; do for f in /usr/share/licenses/$p/*; do [ -f "$f" ] || continue diff --git a/skills/python-project-porting/references/gotchas/local-validation-and-rehearsal.md b/skills/python-project-porting/references/gotchas/local-validation-and-rehearsal.md index 261e8bb828..3b14780a5d 100644 --- a/skills/python-project-porting/references/gotchas/local-validation-and-rehearsal.md +++ b/skills/python-project-porting/references/gotchas/local-validation-and-rehearsal.md @@ -248,6 +248,15 @@ To pull up one entry: `grep -n '^N\. ' references/gotchas/local-validation-and-r by package `name=` attribute to jump straight to its block rather than loading the whole file, and delete it when done; it is Rocky's own public mirror data, not anything project-specific worth keeping. + - **When even `dl.rockylinux.org` is unreachable, the packaging *sources* answer the + same questions**: Rocky 10 rebuilds CentOS Stream 10, whose spec files are served as + plain files from + `https://gitlab.com/redhat/centos-stream/rpms//-/raw/c10s/.spec`. The + `%files devel` section settles whether a `-devel` package installs a CMake package + config (`%{_libdir}/cmake/Foo/`, i.e. whether `find_package(Foo)` can work in CONFIG + mode at all) and the `%license` lines settle which licence file a + `/usr/share/licenses//` sweep will find — both of which repodata alone answers + only indirectly. 383. **A project that links libtorch cannot be rehearsed on an x86_64 host with the `torch` wheel PyPI serves, because that one is a CUDA build: `find_package(Torch)` From c93a5290a620cea0b59e840984cdac60e593f654 Mon Sep 17 00:00:00 2001 From: Ludovic Henry Date: Sat, 19 Sep 2026 09:10:38 +0000 Subject: [PATCH 3/5] Remove skills/ changes - port PRs must only touch workflow/docs/patches Restores skills/python-project-porting/references/{gotchas-index.md, gotchas/local-validation-and-rehearsal.md, gotchas/native-deps-and-linking.md} to main's current content. The two gotchas this PR had added (383/384, torch+CUDA rehearsal and versioned-dlopen ABI contract) also collided with gotcha numbers other agents had already taken on main in the meantime - they'll be re-added on main directly with fresh numbers instead. --- .../references/gotchas-index.md | 54 +++++++++++++-- .../gotchas/local-validation-and-rehearsal.md | 66 ++++++++++--------- .../gotchas/native-deps-and-linking.md | 28 -------- 3 files changed, 82 insertions(+), 66 deletions(-) diff --git a/skills/python-project-porting/references/gotchas-index.md b/skills/python-project-porting/references/gotchas-index.md index 29a02a2098..3b5222ec31 100644 --- a/skills/python-project-porting/references/gotchas-index.md +++ b/skills/python-project-porting/references/gotchas-index.md @@ -93,6 +93,43 @@ The porting gotchas (374 of them) live in [`references/gotchas/`](gotchas/), spl - **376** — A permissive `License:` field on the wrapper package says nothing about whether the payload it ships has any source at all — check the binary's own content, not the metadata's license family (the tableauhyperapi case). +- **381** — A third-party *vendor release* of a project this repo has already ruled out + inherits that verdict — resolve the redistribution to its upstream before triaging anything + else (the tokenspeed-triton case). +- **382** — Several PyPI distributions carved out of *one* build are one unit of work, not + one port each — check the allowed `--build-type` values before writing any YAML, and let + `requires_dist` (not the most "core-sounding" name) fix the order; complements gotcha 380 + (how to publish them once the combined port exists) (the + pyside6/pyside6-essentials/pyside6-addons case). +- **383** — The *umbrella* distribution of a split family carries no compiled code at all, + gets its platform+`abi3` tag from a deliberately fake `Extension`, and its payload is + generated stubs for the union of its siblings' modules — so it cannot be cut from a + different build than they were; also, check the in-image SDK's *minor version* against the + binding release (the pyside6 meta-wheel case). +- **385** — A no-sdist vendor wheel can still have a fully public build recipe — read + `dist-info/WHEEL`'s `Generator:` before parking it for "no source anywhere"; a + vendor-named generator is usually a *repackager*, which moves the stop to whether the + vendor publishes the payload for our arch (the pyqt6-qt6 case). +- **386** — A GPU-only package can be small, source-open and blob-free and still be + unportable: in a JIT kernel library the compiled part is a few-hundred-KB shim, so gotcha + 41's vendor-payload tell is absent and the wall is what that shim links — `libtorch_cuda.so`, + which our CPU-only riscv64 torch can never provide; refines gotchas 249 and 284 (the + humming-kernels case). +- **387** — A GPU-toolkit-suffixed distribution name (`-cuda12x`, `-rocm-7-0`) is a toolkit + selector whose name can be injected from a *separate* release-tools repo; check the vendor's + redist index for our arch, and treat a documented stub/no-CUDA build mode as a docs build, + not a port (the cupy-cuda12x case). +- **388** — The queue entry's wheel shape is a snapshot — re-read the *latest* release's tag + set before triaging the queued version, because upstream can delete the arch-specific + payload and erase the gap outright; also, a `py3-none-any` dependency can be a facade for + platform-only payload wheels (the tokenspeed-mla case). +- **392** — With no project URL and a stock `Generator:`, the *conda-forge feedstock* is the + cheapest source-availability oracle (a feedstock whose `source:` is the PyPI wheels is a + repackager, so there is nothing to build); `readelf -S` splits a real compiled extension + into engine vs embedded model weights (`.text` ~280 KB, `.rodata` ~34.8 MB); a compound + `License: AND LicenseRef-*` is gotcha 372's second lock; and an open-source + org's monorepo hits can all be the closed-source package's *consumer* + (the livekit-local-inference case). ### Sdist source & versioning — [`gotchas/sdist-source-and-versioning.md`](gotchas/sdist-source-and-versioning.md) @@ -162,6 +199,8 @@ The porting gotchas (374 of them) live in [`references/gotchas/`](gotchas/), spl - **360** — A `setup.py`'s own `bdist_wheel --plat-name` insertion can hardcode `manylinux1_` + `platform.machine()` regardless of the actual container libc, making musllinux unbuildable no matter how the CMake/C++ side is patched. +- **391** — A project's real cibuildwheel recipe can live in a *separate packaging repo* that the + source tree never references — the source repo can carry no GitHub Actions at all. ### Rust, maturin & PyO3 — [`gotchas/rust-maturin-and-pyo3.md`](gotchas/rust-maturin-and-pyo3.md) @@ -305,6 +344,9 @@ The porting gotchas (374 of them) live in [`references/gotchas/`](gotchas/), spl - **378** — A newer libstdc++ on the manylinux image can deprecate calls a project's own `-DCMAKE_COMPILE_WARNING_AS_ERROR=ON` CI flag then turns into hard errors, purely from a toolchain-version gap upstream's own (older) runners never see. +- **390** — libev is one of the `-devel` packages that *is* in Rocky 10's riscv64 repos, so an + upstream `yum install -y libev libev-devel` needs no replacement — but its header is + `/usr/include/ev.h`. ### Native dependencies & linking — [`gotchas/native-deps-and-linking.md`](gotchas/native-deps-and-linking.md) @@ -325,10 +367,6 @@ The porting gotchas (374 of them) live in [`references/gotchas/`](gotchas/), spl - **278** — A vendored, direct-copy (not submodule) header can be missing riscv64 from its - **363** — A `libraries=[...]` entry can go missing from the link line with *no* error — - **368** — Linking several codecs against Rocky 10's system libraries instead of -- **384** — When a project dlopen()s a differently-named shared library per major - version of a native dependency, the version you build against is a user-visible ABI - contract — pick the major that the target platform ships, build it LGPL, exclude it - from the wheel, and assert the resulting `.so` name. ### Compiled-vs-pure detection & the require-extension knob — [`gotchas/compiled-vs-pure-detection.md`](gotchas/compiled-vs-pure-detection.md) @@ -423,6 +461,9 @@ The porting gotchas (374 of them) live in [`references/gotchas/`](gotchas/), spl - **329** — A test suite that shells out to the package's own installed CLI binaries at a - **347** — A test that asserts "you're running against an editable/in-place install" can - **348** — A `glcontext`-based package's `create_context(standalone=True)` defaults to the +- **389** — A test `.pyx` that Cython-`include`s a checkout-root-relative path can be satisfied by + staging just those files; a staged package dir with no `__init__.py` is a namespace + portion and does not shadow the wheel. ### Testing: pytest config, servers & test selection — [`gotchas/pytest-config-servers-and-selection.md`](gotchas/pytest-config-servers-and-selection.md) @@ -527,9 +568,8 @@ The porting gotchas (374 of them) live in [`references/gotchas/`](gotchas/), spl - **223** — For a `bindings = "bin"` CLI's test assertions, `cargo build --release` the tool - **298** — A local rehearsal's `pip`-resolved cibuildwheel can be too old for - **369** — Without docker, fetch Rocky 10's own dnf repodata over plain HTTPS to -- **383** — A libtorch-linking project cannot be rehearsed on x86_64 with PyPI's `torch` - wheel: it is a CUDA build, so `find_package(Torch)` dies in `Caffe2Config.cmake` before - reaching the project's own CMake. Everything up to that point still validates. +- **384** — `dnf` failing in the image with `Curl error (60) ... self-signed certificate` is + your egress proxy, not the image — install the proxy CA into the container trust store ### PR, CI, triggers, publishing & maintainer signals — [`gotchas/pr-ci-and-maintainer.md`](gotchas/pr-ci-and-maintainer.md) diff --git a/skills/python-project-porting/references/gotchas/local-validation-and-rehearsal.md b/skills/python-project-porting/references/gotchas/local-validation-and-rehearsal.md index 3b14780a5d..96fb917e09 100644 --- a/skills/python-project-porting/references/gotchas/local-validation-and-rehearsal.md +++ b/skills/python-project-porting/references/gotchas/local-validation-and-rehearsal.md @@ -17,7 +17,8 @@ To pull up one entry: `grep -n '^N\. ' references/gotchas/local-validation-and-r - **223** — For a `bindings = "bin"` CLI's test assertions, `cargo build --release` the tool - **298** — A local rehearsal's `pip`-resolved cibuildwheel can be too old for - **369** — Without docker, fetch Rocky 10's own dnf repodata over plain HTTPS to -- **383** — A libtorch-linking project cannot be rehearsed on x86_64 with PyPI's `torch` +- **384** — `dnf` failing in the image with `Curl error (60) ... self-signed certificate` is + your egress proxy, not the image — install the proxy CA into the container trust store --- @@ -248,34 +249,37 @@ To pull up one entry: `grep -n '^N\. ' references/gotchas/local-validation-and-r by package `name=` attribute to jump straight to its block rather than loading the whole file, and delete it when done; it is Rocky's own public mirror data, not anything project-specific worth keeping. - - **When even `dl.rockylinux.org` is unreachable, the packaging *sources* answer the - same questions**: Rocky 10 rebuilds CentOS Stream 10, whose spec files are served as - plain files from - `https://gitlab.com/redhat/centos-stream/rpms//-/raw/c10s/.spec`. The - `%files devel` section settles whether a `-devel` package installs a CMake package - config (`%{_libdir}/cmake/Foo/`, i.e. whether `find_package(Foo)` can work in CONFIG - mode at all) and the `%license` lines settle which licence file a - `/usr/share/licenses//` sweep will find — both of which repodata alone answers - only indirectly. - -383. **A project that links libtorch cannot be rehearsed on an x86_64 host with the - `torch` wheel PyPI serves, because that one is a CUDA build: `find_package(Torch)` - pulls in `Caffe2Config.cmake`, which hard-fails with "Your installed Caffe2 version - uses CUDA but I cannot find the CUDA libraries" before CMake reaches a single line - of the project's own configuration (the torchcodec case).** The failure has nothing - to do with the project or with riscv64 — the riscv64 `torch` on our registry is a - `+cpu` build whose `Caffe2Config.cmake` has the CUDA branch compiled out, so the same - configure succeeds there. Two consequences worth knowing before spending a rehearsal - cycle on it: - - **A CPU-only torch is the prerequisite for any local rehearsal of a libtorch - extension**, and PyPI has none for linux x86_64 (the CPU variants live on - `download.pytorch.org/whl/cpu`, a separate index); linux aarch64's PyPI `torch` - *is* CPU-only, which is one more reason gotcha 101's aarch64 rehearsal is the right - host for this family of packages. - - **Everything before `find_package(Torch)` still validates cheaply on x86**, and for - a scikit-build-core/CMake project that is most of the interesting surface: the - build frontend and `--no-build-isolation` wiring, `pkg-config` discovery of a - source-built native dependency, the backend finding `pybind11`, and any - licence-guard/env-var gate the project puts in front of a wheel build. Run it and - read how far the configure got rather than treating the CUDA error as a dead end. +384. **`dnf` failing inside the image with `Curl error (60) ... self-signed certificate in + certificate chain` is a fact about *your session's egress proxy*, not about the image — + install the proxy CA into the container's trust store instead of recording "in-image dnf + is impossible".** A sandbox whose outbound HTTPS goes through a TLS-intercepting proxy + gives the host a CA bundle, but a container gets neither that bundle nor the host's + loopback proxy, so every `dnf makecache`/`repoquery` dies on `mirrors.rockylinux.org` + and it looks like the image cannot reach its own repos. Three things fix it together, + and all three are needed: + ``` + docker run --rm --network host \ + -e HTTPS_PROXY -e HTTP_PROXY -e https_proxy -e http_proxy \ + -v "$PWD/.git/pw-scratch/:/s" "$MANYLINUX_RISCV64_IMAGE" bash -c ' + cp /s/ca-bundle.crt /etc/pki/ca-trust/source/anchors/proxy.crt + update-ca-trust extract + dnf repoquery --qf "%{name}|%{version}|%{reponame}\n" "qt6*"' + ``` + `--network host` is what lets the container reach a proxy listening on the host's + loopback; the env vars are not inherited unless named; and `update-ca-trust extract` + (Rocky's anchors directory, *not* `/etc/ssl/certs`) is what makes curl inside `dnf` + accept the intercepted chain. This matters because two queue entries had already + recorded the proxy failure as an image limitation and fallen back to gotcha 369's + raw-repodata parse — which is still the right tool for "is it packaged, in which repo", + but cannot answer what `dnf` actually *resolves*, and cannot show you the installed + on-disk layout (`/usr/lib64/cmake/Qt6*`, `ClangConfig.cmake`, real `.so` names) that a + CMake `find_package` will or will not hit. + - **Use `repoquery` for inventory and reserve `install` for layout questions.** A + `repoquery` is metadata-only and answers in seconds even under QEMU — and it returns + the SDK's *version*, which is the field most likely to be assumed rather than checked + (gotcha 383). Actually installing a large `-devel` set is emulated `rpm` scriptlet + work and can take tens of minutes on a loaded host, so do not put it on the critical + path of a triage decision; note how far it got and move on. + - This is the container half of the rule already stated for the host: never disable TLS + verification or unset the proxy variables to make a fetch succeed. diff --git a/skills/python-project-porting/references/gotchas/native-deps-and-linking.md b/skills/python-project-porting/references/gotchas/native-deps-and-linking.md index 66b5831cd6..e3609bc616 100644 --- a/skills/python-project-porting/references/gotchas/native-deps-and-linking.md +++ b/skills/python-project-porting/references/gotchas/native-deps-and-linking.md @@ -22,7 +22,6 @@ To pull up one entry: `grep -n '^N\. ' references/gotchas/native-deps-and-linkin - **231** — A vendored C library's own CMake can carry a genuine, tested riscv64 branch — - **363** — A `libraries=[...]` entry can go missing from the link line with *no* error — - **368** — Linking several codecs against Rocky 10's system libraries instead of -- **384** — When a project dlopen()s a differently-named shared library per major version --- @@ -496,30 +495,3 @@ To pull up one entry: `grep -n '^N\. ' references/gotchas/native-deps-and-linkin compiler flag; libpng's own `CHANGES` file (`grep -n cICP CHANGES` against the tagged release) gives the exact version the symbol was added in, and confirms Rocky 10's 1.6.40 predates it — a real wall, not a flag away. - -384. **When a project dlopen()s a differently-named shared library per major version of - a native dependency, the version you build against is not an implementation detail — - it is an ABI contract with whatever the *user's* machine has installed (the torchcodec - case).** torchcodec compiles `libtorchcodec_core.so` where `N` is the FFmpeg major - version, and at import time tries `N` = 9, 8, 7, 6, 5, 4 in turn, using the first that - loads; upstream ships all six in one wheel by linking against prebuilt non-GPL FFmpeg - tarballs it hosts on S3. Those tarballs have no riscv64 build, so the riscv64 wheel - goes down the project's other path — `pkg-config` against one installed FFmpeg — and - therefore carries exactly one `N`. That makes the FFmpeg version a **user-visible** - choice: build against 7.x and the wheel imports on nothing that ships FFmpeg 6. - - **Pick the major version of the platform the wheels are consumed on**, not the - newest release: `ubuntu-24.04-riscv` (the runner these wheels target) ships FFmpeg - 6.1, so a 6.1.x source build is what makes `apt install ffmpeg` enough for a user. - The mapping is from the *library* soname, not the FFmpeg release number — FFmpeg 6 - is `libavcodec.so.60`/`libavutil.so.58`, and it is the `libavcodec` major that the - project's CMake switches on. - - **Build it LGPL and do not ship it.** No `--enable-gpl`/`--enable-nonfree` and no - third-party codec integrations keeps the FFmpeg build itself LGPL, and excluding - `libav*`/`libsw*`/`libpostproc*` from `auditwheel repair` keeps it out of the wheel - entirely — which is also what upstream's own `packaging/repair_wheel.py` enforces, - since FFmpeg is a runtime dependency the user supplies. - - **Assert the resulting `.so` name in a post-build step.** `libtorchcodec_core6.so` - present in the wheel is the one-line proof that the FFmpeg the container built is - the FFmpeg that got linked; a silent fallback to a different major would otherwise - only surface as an ImportError on a user's machine. - From ef2aef6beb1167bb77e8d442a8661ee846e57e38 Mon Sep 17 00:00:00 2001 From: Ludovic Henry Date: Sat, 19 Sep 2026 20:50:09 +0000 Subject: [PATCH 4/5] torchcodec: patch the no-libavif decode_avif stub so the image library loads All four riscv64 jobs of the first CI run built, repaired and installed a wheel and then died at the test step with OSError: Could not load this library: .../site-packages/torchcodec/libtorchcodec_image.so pytest's conftest import report drops the exception chain, so the cause never reached the log. Reproducing the run under docker+QEMU against quay.io/pypa/manylinux_2_39_riscv64 gives it: OSError: .../libtorchcodec_image.so: undefined symbol: _ZN8facebook10torchcodec11decode_avifERKN5torch6stable6TensorElll that is facebook::torchcodec::decode_avif(torch::stable::Tensor const&, long, long, long). DecodeAvif.h declares decode_avif() with four parameters, the real implementation defines four, and image_custom_ops.cpp registers it with TORCH_BOX(&decode_avif) - but the !TORCHCODEC_ENABLE_AVIF stub this build selects still has the three-parameter signature from before num_threads was added, so it defines a different overload and the four-parameter one exists nowhere. An ELF shared object may carry undefined symbols, so the link, the wheel build and auditwheel all stay green and the first dlopen is where it fails - and since load_image_library() runs at `import torchcodec`, that takes the whole package down, not just decode_avif. Upstream never builds this path: their wheels always link the libavif they fetch from S3, and no CI job sets TORCHCODEC_BUILD_AVIF=0. We have to, because that bucket has no riscv64 build and libavif is packaged in neither Rocky 10 nor a riscv64 EPEL. Validated under docker+QEMU on quay.io/pypa/manylinux_2_39_riscv64, cp312: the unpatched wheel reproduces the CI traceback exactly, the patched one imports (`IMPORT_OK 0.16.0 6`), `ldd -r` reports no unresolved symbol in any of the five shipped libraries, and the workflow's own wheel-contents check passes on the repaired wheel. --- ...he-no-libavif-stub-the-declared-sign.patch | 63 +++++++++++++++++++ 1 file changed, 63 insertions(+) create mode 100644 patches/torchcodec/0.16.0/0002-DecodeAvif-give-the-no-libavif-stub-the-declared-sign.patch diff --git a/patches/torchcodec/0.16.0/0002-DecodeAvif-give-the-no-libavif-stub-the-declared-sign.patch b/patches/torchcodec/0.16.0/0002-DecodeAvif-give-the-no-libavif-stub-the-declared-sign.patch new file mode 100644 index 0000000000..06515ff695 --- /dev/null +++ b/patches/torchcodec/0.16.0/0002-DecodeAvif-give-the-no-libavif-stub-the-declared-sign.patch @@ -0,0 +1,63 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: Ludovic Henry +Date: Sat, 19 Sep 2026 19:40:00 +0200 +Subject: [PATCH] DecodeAvif: give the no-libavif stub the declared signature + +Upstream-Status: To upstream [not yet submitted; upstream never builds this path - their wheels always link the libavif they fetch from S3, and no CI job sets TORCHCODEC_BUILD_AVIF=0] + +decode_avif() is declared in DecodeAvif.h - and defined by the +TORCHCODEC_ENABLE_AVIF branch of DecodeAvif.cpp - with four parameters +(input, mode, output_dtype, num_threads), and image_custom_ops.cpp takes +its address with TORCH_BOX(&decode_avif) to register the +"decode_avif(Tensor input, int mode, int output_dtype=0, int +num_threads=1) -> Tensor" op. The !TORCHCODEC_ENABLE_AVIF stub, however, +still has the three-parameter signature from before num_threads was +added, so it defines a *different* overload and the four-parameter one is +never defined anywhere. + +An ELF shared object may carry undefined symbols, so nothing complains: +libtorchcodec_image.so links, the wheel builds, auditwheel repairs it, +and the first torch.ops.load_library() of it fails with + + OSError: .../torchcodec/libtorchcodec_image.so: undefined symbol: + _ZN8facebook10torchcodec11decode_avifERKN5torch6stable6TensorElll + +i.e. facebook::torchcodec::decode_avif(torch::stable::Tensor const&, +long, long, long). Because load_image_library() runs at +`import torchcodec`, every entry point of the package is dead, not just +decode_avif - which defeats the documented purpose of the build flag +("A codec set to OFF is not built; its decode_*() then raises an +actionable error at call time [...] while importing torchcodec still +works"). + +Add the missing num_threads parameter to the stub. Nothing else changes: +the stub still just raises the actionable "not compiled with libavif +support" error, which is what test/utils.py's avif_is_available() looks +for to skip the AVIF tests. + +This is not riscv64-specific - any build with TORCHCODEC_BUILD_AVIF=0 or +TORCHCODEC_BUILD_IMAGE=0 produces an unloadable image library - but it is +what our wheels hit, since libavif comes from upstream's S3 bucket, which +has no riscv64 build, and neither Rocky 10 nor a riscv64 EPEL packages it. + +Signed-off-by: Ludovic Henry +--- + src/torchcodec/_core/DecodeAvif.cpp | 3 ++- + 1 file changed, 2 insertions(+), 1 deletion(-) + +diff --git a/src/torchcodec/_core/DecodeAvif.cpp b/src/torchcodec/_core/DecodeAvif.cpp +index 17fd03a..b40b9b9 100644 +--- a/src/torchcodec/_core/DecodeAvif.cpp ++++ b/src/torchcodec/_core/DecodeAvif.cpp +@@ -19,7 +19,8 @@ namespace facebook::torchcodec { + torch::stable::Tensor decode_avif( + [[maybe_unused]] const torch::stable::Tensor& input, + [[maybe_unused]] int64_t mode, +- [[maybe_unused]] int64_t output_dtype) { ++ [[maybe_unused]] int64_t output_dtype, ++ [[maybe_unused]] int64_t num_threads) { + STD_TORCH_CHECK( + false, + "decode_avif: torchcodec was not compiled with libavif support. Rebuild " +-- +2.51.0 From e5bff2f8d4dccac8d4b60b4bf40d69da612a73bc Mon Sep 17 00:00:00 2001 From: Ludovic Henry Date: Sun, 20 Sep 2026 00:58:08 +0000 Subject: [PATCH 5/5] torchcodec: skip the tests the container's LGPL FFmpeg cannot run The second CI run built and imported the wheel on all four interpreters and then failed the same 242 tests on each of them. Every one of those tests needs an encoder FFmpeg does not have: upstream tests against conda-forge's GPL FFmpeg, while the build here configures FFmpeg with no external libraries, so it has none of the H.264, HEVC, VP9, AV1 or MP3 encoders (every one of them lives in an external library, and libx264/libx265 need --enable-gpl) and its AV1 decoder only works through a hardware accelerator. The symptoms all follow from that: FFmpeg falls back to the container format's default codec, so the failures read "yuv444p is not supported by the mpeg4 encoder", "avcodec_open2 failed: Invalid argument", "Video codec av1 not found" or "Codec not found", and the ffmpeg CLI the encoder tests compare themselves against cannot write an MP3 either. Drop the tests that mux a real video, shell out to that CLI or ask for MP3, and smoke_test.py with them - it encodes H.264 yuv444p to build every fixture it then decodes. test_logging.py goes too: it asserts an empty subprocess stderr, which torch's libcpuinfo pollutes on this runner by reading core_id -1 from /sys. The selection is deliberately narrow - the MP3 clauses keep the WAV and FLAC parametrisations of the same tests, so the audio encoder, the image encoder and every decoder stay covered. Checked against the node ids of the failed run: the expression deselects exactly those 232 tests and nothing that passed. --- .github/workflows/build-torchcodec.yml | 28 +++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-torchcodec.yml b/.github/workflows/build-torchcodec.yml index 78bdae7e7a..71e885d46b 100644 --- a/.github/workflows/build-torchcodec.yml +++ b/.github/workflows/build-torchcodec.yml @@ -157,7 +157,33 @@ jobs: # FAIL_WITHOUT_ turns a missing image codec from a silent skip into a # failure, for the three we do build. test_transform_ops.py imports torchvision at # module scope and pypi.riseproject.dev has no riscv64 torchvision yet. - CIBW_TEST_COMMAND: FAIL_WITHOUT_JPEG=1 FAIL_WITHOUT_PNG=1 FAIL_WITHOUT_WEBP=1 pytest --ignore=test/test_transform_ops.py test + # Upstream tests against conda-forge's GPL FFmpeg; the LGPL build above has only + # FFmpeg's built-in codecs - no H.264/HEVC/VP9/AV1/MP3 encoder at all (libx264 and + # libx265 need --enable-gpl) and an AV1 decoder that only works through a hwaccel - + # so the tests that mux a real video, shell out to the ffmpeg CLI or ask for MP3 are + # dropped, smoke_test.py included: it encodes H.264 yuv444p to build every one of its + # fixtures. test_logging.py asserts an empty subprocess stderr, which torch's + # libcpuinfo pollutes on this runner by reading core_id -1 from /sys. + CIBW_TEST_COMMAND: >- + FAIL_WITHOUT_JPEG=1 FAIL_WITHOUT_PNG=1 FAIL_WITHOUT_WEBP=1 pytest + --ignore=test/test_transform_ops.py + --ignore=test/smoke_test.py + --ignore=test/test_logging.py + -k "not test_get_frame_at_av1 + and not ((test_audio_against_cli or test_multiple_audio_formats + or test_audio_to_file_vs_to_file_like) and mp3) + and not (test_add_video_and_encode_frames or test_add_audio_and_video_and_encode + or test_multiple_video_streams_and_audio or test_video_round_trip + or test_video_to_file_vs_to_file_like or test_video_contiguity + or test_video_against_ffmpeg_cli or test_fragmented_mp4 + or test_codec_parameter_utilized or test_codec_spec_vs_impl_equivalence + or test_extra_options_errors or test_extra_options_utilized + or test_crf_out_of_range_errors or test_invalid_preset_errors + or test_to_file_like_real_file_video + or (test_to_file_like_custom_file_object and not audio) + or test_add_audio_unsupported_num_channels_errors + or test_add_audio_invalid_out_sample_rate_errors)" + test - name: Check the wheel bundles the image codecs and no FFmpeg run: |