From 616cd8e046debd8775afe4e5d308fc65d261b493 Mon Sep 17 00:00:00 2001 From: Ludovic Henry Date: Sat, 19 Sep 2026 07:53:13 +0000 Subject: [PATCH 1/5] coremltools: add riscv64 wheel build for 9.0 Build the libmilstoragepython/libmodelpackage pybind11 extensions and the wheel the way upstream's scripts/build.sh does (CMake, then the `dist` target), narrowed to the manylinux_2_39_riscv64 container. --- .github/workflows/build-coremltools.yml | 187 ++++++++++++++++++ docs/packages/coremltools.yaml | 5 + ...Linux-wheel-platform-tag-from-the-ta.patch | 43 ++++ 3 files changed, 235 insertions(+) create mode 100644 .github/workflows/build-coremltools.yml create mode 100644 docs/packages/coremltools.yaml create mode 100644 patches/coremltools/9.0/0001-cmake-derive-the-Linux-wheel-platform-tag-from-the-ta.patch diff --git a/.github/workflows/build-coremltools.yml b/.github/workflows/build-coremltools.yml new file mode 100644 index 0000000000..fb11210bd1 --- /dev/null +++ b/.github/workflows/build-coremltools.yml @@ -0,0 +1,187 @@ +# SPDX-FileCopyrightText: 2026 The RISE Project +# SPDX-License-Identifier: MIT +# +# This workflow is based on: https://github.com/apple/coremltools/blob/main/scripts/build.sh +name: Build coremltools wheels (riscv64) + +on: + workflow_dispatch: + inputs: + version: + description: 'Version glob to (re)build; empty builds every version of docs/packages/coremltools.yaml not released yet' + required: false + default: '' + pull_request: + branches: [main] + paths: + - '.github/workflows/build-coremltools.yml' + - 'docs/packages/coremltools.yaml' + push: + branches: [main] + paths: + - '.github/workflows/build-coremltools.yml' + - 'docs/packages/coremltools.yaml' + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +permissions: + contents: read + +env: + MANYLINUX_RISCV64_IMAGE: quay.io/pypa/manylinux_2_39_riscv64 + PIP_EXTRA_INDEX_URL: https://pypi.riseproject.dev/simple/ + +jobs: + setup: + uses: $/.github/workflows/_setup.yml + with: + package: coremltools + version: ${{ inputs.version }} + + build_wheels: + needs: [setup] + if: needs.setup.outputs.versions != '[]' + name: Build coremltools ${{ matrix.version }} ${{ matrix.python }}-manylinux_riscv64 + runs-on: ubuntu-24.04-riscv + timeout-minutes: 720 + strategy: + fail-fast: false + matrix: + version: ${{ fromJSON(needs.setup.outputs.versions) }} + # Upstream stops at 3.13, and the vendored pybind11 2.13.1 predates 3.14. + python: [cp312-cp312, cp313-cp313] + + env: + COREMLTOOLS_VERSION: ${{ matrix.version }} + + steps: + - name: Checkout apple/coremltools ${{ env.COREMLTOOLS_VERSION }} + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: apple/coremltools + ref: ${{ env.COREMLTOOLS_VERSION }} + path: coremltools + persist-credentials: false + + - name: Checkout python-wheels + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + path: python-wheels + persist-credentials: false + + - name: Patch coremltools source + run: git -C coremltools apply ../python-wheels/patches/coremltools/${{ env.COREMLTOOLS_VERSION }}/00*.patch + + - name: Build the C++ libraries and the wheel + run: | + mkdir -p output "$HOME/.cache/coremltools-ccache" + docker run -i --rm \ + -v "$PWD/coremltools:/coremltools" \ + -v "$PWD/output:/output" \ + -v "$HOME/.cache/coremltools-ccache:/ccache" \ + -e CCACHE_DIR=/ccache \ + -e COREMLTOOLS_VERSION \ + "$MANYLINUX_RISCV64_IMAGE" bash -s <<'COREMLTOOLS_BUILD_EOF' + #!/usr/bin/env bash + set -euxo pipefail + + COREMLTOOLS_VERSION="${COREMLTOOLS_VERSION:?must be set, e.g. 9.0}" + PYBIN=/opt/python/${{ matrix.python }}/bin + # CMakeLists.txt builds the vendored kmeans1d with a bare `python3`, so the + # interpreter this wheel is for has to be the one that wins on PATH. + export PATH="$PYBIN:$PATH" + # The vendored protobuf 3.19 declares cmake_minimum_required(VERSION 3.0.0). + export CMAKE_POLICY_VERSION_MINIMUM=3.5 + + dnf install -y --setopt=install_weak_deps=False libuuid-devel + + python3 -m pip install -U setuptools wheel auditwheel + + # Upstream's NOTICE.txt covers only kmeans1d; the rest of deps/ is compiled + # into the shipped .so files, and auditwheel grafts in libuuid. + cp /coremltools/deps/protobuf/LICENSE /coremltools/LICENSE.protobuf + cp /coremltools/deps/pybind11/LICENSE /coremltools/LICENSE.pybind11 + cp /coremltools/deps/nlohmann/LICENSE.MIT /coremltools/LICENSE.nlohmann-json + cp /coremltools/deps/FP16/LICENSE /coremltools/LICENSE.FP16 + uuid_pkg=$(rpm -qf --qf '%{NAME}\n' "$(ldconfig -p | awk '/libuuid\.so\.1 /{print $NF; exit}')") + dnf reinstall -y --setopt=tsflags= "$uuid_pkg" || true + cat /usr/share/licenses/"$uuid_pkg"/* > /coremltools/LICENSE.libuuid 2>/dev/null \ + || rpm -q --qf '%{NAME} %{VERSION}: %{LICENSE}\n' "$uuid_pkg" > /coremltools/LICENSE.libuuid + + CCACHE_ARGS="" + command -v ccache >/dev/null 2>&1 && \ + CCACHE_ARGS="-DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache" + + cmake -S /coremltools -B /tmp/build \ + -DCMAKE_BUILD_TYPE=Release \ + -DPYTHON_EXECUTABLE:FILEPATH="$PYBIN/python3" \ + -DOVERWRITE_PB_SOURCE=0 \ + ${CCACHE_ARGS} + cmake --build /tmp/build -j "$(nproc)" + cmake --build /tmp/build --target dist + + auditwheel repair --strip /tmp/build/dist/coremltools-*.whl -w /output + ls -la /output + COREMLTOOLS_BUILD_EOF + + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: coremltools-${{ env.COREMLTOOLS_VERSION }}-${{ matrix.python }}-manylinux_riscv64 + path: output/*.whl + if-no-files-found: error + + - name: Test coremltools wheel + run: | + docker run -i --rm \ + -v "$PWD/output:/output" \ + -e PIP_EXTRA_INDEX_URL \ + -e PIP_ONLY_BINARY=numpy,pillow \ + "$MANYLINUX_RISCV64_IMAGE" bash -s <<'COREMLTOOLS_TEST_EOF' + #!/usr/bin/env bash + set -euxo pipefail + + PYBIN=/opt/python/${{ matrix.python }}/bin + + "$PYBIN/pip" install -U pytest pytest-timeout parameterized pillow + "$PYBIN/pip" install /output/coremltools-*.whl + + "$PYBIN/python3" -c " + from coremltools import libmilstoragepython, libmodelpackage + assert libmilstoragepython.__file__.endswith('.so'), libmilstoragepython.__file__ + assert libmodelpackage.__file__.endswith('.so'), libmodelpackage.__file__ + " + + # Upstream's scripts/test.sh runs the suite out of the installed wheel with + # --pyargs, from a directory that is not the checkout. torch, tensorflow, + # scikit-learn and xgboost have no riscv64 wheels, so this is the subset of + # upstream's coremltools.test / mil jobs that collects without them; the two + # -k exclusions are tests that import torch inside the test body. + mkdir -p /tmp/testrun && cd /tmp/testrun + "$PYBIN/python3" -m pytest -v -ra --timeout=600 \ + -k "not (TestApiVisibilities and test_optimize) and not test_model_save_no_extension" \ + --pyargs \ + coremltools.test.api \ + coremltools.test.blob \ + coremltools.test.modelpackage.test_modelpackage \ + coremltools.converters.mil.backend \ + coremltools.converters.mil.mil.tests.test_types + COREMLTOOLS_TEST_EOF + + publish: + name: Publish coremltools ${{ matrix.version }} + needs: [setup, build_wheels] + if: needs.setup.outputs.versions != '[]' + strategy: + fail-fast: false + matrix: + version: ${{ fromJSON(needs.setup.outputs.versions) }} + permissions: + contents: write + pull-requests: write + uses: $/.github/workflows/_publish-wheel.yml + secrets: + app-private-key: ${{ secrets.RISEPROJECT_APP_PRIVATE_KEY }} + with: + artifact-pattern: coremltools-${{ matrix.version }}-*-manylinux_riscv64 diff --git a/docs/packages/coremltools.yaml b/docs/packages/coremltools.yaml new file mode 100644 index 0000000000..79440f8ead --- /dev/null +++ b/docs/packages/coremltools.yaml @@ -0,0 +1,5 @@ +package-name: coremltools +source-code: https://github.com/apple/coremltools +license: BSD-3-Clause +versions: +- version: '9.0' diff --git a/patches/coremltools/9.0/0001-cmake-derive-the-Linux-wheel-platform-tag-from-the-ta.patch b/patches/coremltools/9.0/0001-cmake-derive-the-Linux-wheel-platform-tag-from-the-ta.patch new file mode 100644 index 0000000000..06f3e685e7 --- /dev/null +++ b/patches/coremltools/9.0/0001-cmake-derive-the-Linux-wheel-platform-tag-from-the-ta.patch @@ -0,0 +1,43 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: Ludovic Henry +Date: Sat, 19 Sep 2026 07:45:02 +0000 +Subject: [PATCH] cmake: derive the Linux wheel platform tag from the target + architecture + +The `dist` target hardcodes `--plat-name=manylinux1_x86_64` for every +Linux build, so a wheel produced on any other Linux architecture is +mislabelled as x86_64: pip either refuses to install it or installs it +on the wrong machine. `setup.py` declares no `ext_modules` (the +`libmilstoragepython.so`/`libmodelpackage.so` pybind11 modules are built +by this CMakeLists and picked up through `package_data`), so the tag +CMake passes is the only thing that describes the wheel's architecture. + +Keep the existing x86_64 tag exactly as it is and fall back to a plain +`linux_` tag elsewhere, which `auditwheel repair` then turns into +the correct `manylinux_*` tag for the build container. + +Upstream-Status: To upstream [not yet submitted to apple/coremltools] +--- + CMakeLists.txt | 6 +++++- + 1 file changed, 5 insertions(+), 1 deletion(-) + +diff --git a/CMakeLists.txt b/CMakeLists.txt +index 5777bac..16fe61c 100644 +--- a/CMakeLists.txt ++++ b/CMakeLists.txt +@@ -233,7 +233,11 @@ if(APPLE) + endif() + set(PLAT_NAME "macosx_${MIN_MAC_OS}_${HARDWARE_NAME}") + elseif("${CMAKE_SYSTEM_NAME}" MATCHES "Linux") +- set(PLAT_NAME "manylinux1_x86_64") ++ if(${CMAKE_SYSTEM_PROCESSOR} MATCHES "x86_64") ++ set(PLAT_NAME "manylinux1_x86_64") ++ else() ++ set(PLAT_NAME "linux_${CMAKE_SYSTEM_PROCESSOR}") ++ endif() + else() + message(FATAL_ERROR "Unsupported build platform. Supported platforms are Linux and macOS.") + endif() +-- +2.43.0 + From 681b404f78c91fcbb51c1be3a81cef58ffe61ad0 Mon Sep 17 00:00:00 2001 From: Ludovic Henry Date: Sat, 19 Sep 2026 07:57:21 +0000 Subject: [PATCH 2/5] skills: record gotchas 381 and 382 381: a `cpXY-none-` wheel whose setup.py declares no ext_modules at all, with the extension modules and the platform tag both supplied by a sibling CMake build. 382: a CMake build that shells out to a bare `python3` for one vendored sub-extension builds it for the container's default interpreter instead of the one the wheel is for. --- .../references/gotchas-index.md | 6 ++++ .../gotchas/cibuildwheel-matrix-and-abi3.md | 33 +++++++++++++++++++ .../gotchas/native-build-bazel-and-drivers.md | 24 ++++++++++++++ 3 files changed, 63 insertions(+) diff --git a/skills/python-project-porting/references/gotchas-index.md b/skills/python-project-porting/references/gotchas-index.md index 3da4263fce..676630c385 100644 --- a/skills/python-project-porting/references/gotchas-index.md +++ b/skills/python-project-porting/references/gotchas-index.md @@ -162,6 +162,9 @@ The porting gotchas (370 of them) live in [`references/gotchas/`](gotchas/), spl - **360** — A `setup.py`'s own `bdist_wheel --plat-name` insertion can hardcode `manylinux1_` + `platform.machine()` regardless of the actual container libc, making musllinux unbuildable no matter how the CMake/C++ side is patched. +- **381** — A `cpXY-none-` wheel is the third plat-name shape: `setup.py` declares + no `ext_modules` at all, and a sibling CMake build both compiles the extension modules and + hands `bdist_wheel` the tag (the coremltools case). ### Rust, maturin & PyO3 — [`gotchas/rust-maturin-and-pyo3.md`](gotchas/rust-maturin-and-pyo3.md) @@ -257,6 +260,9 @@ The porting gotchas (370 of them) live in [`references/gotchas/`](gotchas/), spl - **202** — A monorepo's "regenerate deps from Bazel" helper may already tolerate a missing - **219** — GDAL's cmake build produces no `gdal-config` script — a second consumer of the - **233** — A package can have no Python build backend at all — the wheel comes from an +- **382** — A CMake build that shells out to a bare `python3` for one vendored sub-extension + silently builds it for the container's default interpreter, not the one the wheel is for + (the coremltools/kmeans1d case). ### The manylinux image & toolchain — [`gotchas/manylinux-image-and-toolchain.md`](gotchas/manylinux-image-and-toolchain.md) diff --git a/skills/python-project-porting/references/gotchas/cibuildwheel-matrix-and-abi3.md b/skills/python-project-porting/references/gotchas/cibuildwheel-matrix-and-abi3.md index 34867ace90..cd39727532 100644 --- a/skills/python-project-porting/references/gotchas/cibuildwheel-matrix-and-abi3.md +++ b/skills/python-project-porting/references/gotchas/cibuildwheel-matrix-and-abi3.md @@ -39,6 +39,9 @@ To pull up one entry: `grep -n '^N\. ' references/gotchas/cibuildwheel-matrix-an - **360** — A `setup.py`'s own `bdist_wheel --plat-name` insertion can hardcode `manylinux1_` + `platform.machine()` regardless of the actual container libc, making musllinux unbuildable no matter how the CMake/C++ side is patched. +- **381** — A `cpXY-none-` wheel is the third plat-name shape: `setup.py` declares + no `ext_modules` at all, and a sibling CMake build both compiles the extension modules and + hands `bdist_wheel` the tag. --- @@ -759,3 +762,33 @@ To pull up one entry: `grep -n '^N\. ' references/gotchas/cibuildwheel-matrix-an keystone-engine never built musllinux either — see gotcha 34's "the default four" logic) is the lower-risk, upstream-faithful choice when this is caught on the first port rather than treated as a regression to fix. + +381. **A `cpXY-none-` wheel is the third plat-name shape: `setup.py` declares no + `ext_modules` at all, and a sibling CMake build both compiles the extension modules and + hands `bdist_wheel` the tag (the coremltools case; see `build-coremltools.yml`).** + Gotcha 27 reads `py3-none-` as "the platform half was forced by hand, nothing + is compiled"; gotcha 81 adds the ctypes-payload exception. Neither covers a wheel tagged + `cpXY-**none**-` — an interpreter tag with no ABI tag — which is what you get + when the compiled modules are real, per-interpreter pybind11 `.so`s, but setuptools never + sees them: coremltools' `setup.py` has zero `ext_modules` and ships + `libmilstoragepython.so`/`libmodelpackage.so` through `package_data` globs, while + `CMakeLists.txt` builds them and then invokes `setup.py bdist_wheel + --plat-name= --python-tag=cp${MAJOR}${MINOR}` itself. Read the interpreter half + before concluding anything from the missing ABI tag: `cpXY-none` means per-interpreter + content that setuptools was not told about, whereas gotcha 27's `py3-none` means no + per-interpreter content at all. + - **The architecture of the whole wheel is then one string in the build system, not a + `platform.machine()` call.** Gotcha 360's keystone-engine case at least interpolated + the real machine; coremltools' CMakeLists is `elseif(... MATCHES "Linux") + set(PLAT_NAME "manylinux1_x86_64")` — a literal, so a riscv64 build silently produces + an x86_64-tagged wheel that installs on the wrong machine. Grep the build system for + `plat-name`/`plat_name`, not just `setup.py`, whenever `setup.py` has no `ext_modules` + but the released wheels carry a platform tag. + - **Patch it to a libc-neutral `linux_` and let `auditwheel repair` retag**, the + same resolution gotcha 360 points at, and keep the existing x86_64 branch byte for + byte so the diff is upstreamable: `if(${CMAKE_SYSTEM_PROCESSOR} MATCHES "x86_64")` → + the old literal, `else()` → `linux_${CMAKE_SYSTEM_PROCESSOR}`. + - **cibuildwheel is the wrong tool for this shape.** It drives a PEP 517 build, which + here would run `setup.py` on its own and emit a `py3-none-any` wheel containing + riscv64 `.so`s — worse than a wrong platform tag. Drive the container yourself + (gotcha 15) and run the project's own `dist` target. diff --git a/skills/python-project-porting/references/gotchas/native-build-bazel-and-drivers.md b/skills/python-project-porting/references/gotchas/native-build-bazel-and-drivers.md index 24849e1958..b206cc9870 100644 --- a/skills/python-project-porting/references/gotchas/native-build-bazel-and-drivers.md +++ b/skills/python-project-porting/references/gotchas/native-build-bazel-and-drivers.md @@ -19,6 +19,8 @@ To pull up one entry: `grep -n '^N\. ' references/gotchas/native-build-bazel-and - **202** — A monorepo's "regenerate deps from Bazel" helper may already tolerate a missing - **219** — GDAL's cmake build produces no `gdal-config` script — a second consumer of the - **233** — A package can have no Python build backend at all — the wheel comes from an +- **382** — A CMake build that shells out to a bare `python3` for one vendored sub-extension + silently builds it for the container's default interpreter, not the one the wheel is for. --- @@ -386,3 +388,25 @@ To pull up one entry: `grep -n '^N\. ' references/gotchas/native-build-bazel-and sibling wheels and `entry_points.txt` shims for other targets; only `write_base_packages` (the base `pip` wheel) matters here, so the heredoc reproduces that function alone and ignores the rest of the tool. + +382. **A CMake build that shells out to a bare `python3` for one vendored sub-extension + silently builds it for the container's default interpreter, not the one the wheel is + for (the coremltools/kmeans1d case).** Driving the container yourself means every + per-interpreter loop iteration passes the interpreter explicitly — `-DPYTHON_EXECUTABLE`, + `$PYBIN/python3`, a venv — and that covers the targets CMake compiles itself. It does not + cover an `execute_process(COMMAND python3 setup.py build_ext --inplace WORKING_DIRECTORY + ${DEPS}/kmeans1d)` buried in the same `CMakeLists.txt`: that resolves `python3` from + `PATH` at *configure* time, so a `-DPYTHON_EXECUTABLE=/opt/python/cp312-cp312/bin/python3` + build happily ships `_core.cpython-311--linux-gnu.so` inside a cp312 wheel. It is + invisible in a green build and a green import — the module is only imported by the + palettization code path, so the whole suite can pass — and `unzip -l | grep '\.so'` + is what catches it. + - **Upstream never sees it** because its own build script activates a conda env first, + making `python3` and `PYTHON_EXECUTABLE` the same binary. Reproducing that is one line + in the build script — `export PATH="$PYBIN:$PATH"` before `cmake` — and is strictly + safer than auditing every `execute_process` for the hardcoded name. + - **Grep for the bare interpreter name, not for `PYTHON_EXECUTABLE`.** `grep -rn + 'COMMAND python' CMakeLists.txt cmake/` finds both this and the `python -m lib2to3` + style post-processing steps that protobuf codegen rules commonly carry; the ones that + use `${PYTHON_EXECUTABLE}` are already correct, and the ones that do not are the list + the PATH export exists to cover. From fb32b6fbdc0e473a9705e290389ba7a76c52e697 Mon Sep 17 00:00:00 2001 From: Ludovic Henry Date: Sat, 19 Sep 2026 08:01:56 +0000 Subject: [PATCH 3/5] coremltools: pin the CMake Python include/library to the target interpreter --- .github/workflows/build-coremltools.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/build-coremltools.yml b/.github/workflows/build-coremltools.yml index fb11210bd1..7fc1242852 100644 --- a/.github/workflows/build-coremltools.yml +++ b/.github/workflows/build-coremltools.yml @@ -114,9 +114,17 @@ jobs: command -v ccache >/dev/null 2>&1 && \ CCACHE_ARGS="-DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache" + # FindPythonLibs searches standard prefixes and would find the image's own + # /usr/local interpreter instead of /opt/python/, so pin it the way + # scripts/env_activate.sh pins it for upstream's conda env. + py_inc=$(python3 -c 'import sysconfig; print(sysconfig.get_path("include"))') + py_lib=$(python3 -c 'import os, sysconfig; print(os.path.join(sysconfig.get_config_var("LIBDIR"), sysconfig.get_config_var("LIBRARY")))') + cmake -S /coremltools -B /tmp/build \ -DCMAKE_BUILD_TYPE=Release \ -DPYTHON_EXECUTABLE:FILEPATH="$PYBIN/python3" \ + -DPYTHON_INCLUDE_DIR="$py_inc" \ + -DPYTHON_LIBRARY="$py_lib" \ -DOVERWRITE_PB_SOURCE=0 \ ${CCACHE_ARGS} cmake --build /tmp/build -j "$(nproc)" From 18af957b6750f83d3d2aa44ec3d59daa07bfeded Mon Sep 17 00:00:00 2001 From: Ludovic Henry Date: Sat, 19 Sep 2026 09:15:27 +0000 Subject: [PATCH 4/5] Remove skills/ changes - port PRs must only touch workflow/docs/patches Restores skills/python-project-porting/references/{gotchas-index.md, gotchas/cibuildwheel-matrix-and-abi3.md, gotchas/native-build-bazel-and-drivers.md} to main's current content. The two gotchas this PR had added (381/382, the cpXY-none plat-name shape and the CMake-shells-out-to-bare-python3 case) also collided with gotcha numbers other agents had already taken on main in the meantime - they'll be re-added on main directly with fresh numbers. --- .../references/gotchas-index.md | 61 ++++++++++++++++--- .../gotchas/cibuildwheel-matrix-and-abi3.md | 57 ++++++++--------- .../gotchas/native-build-bazel-and-drivers.md | 24 -------- 3 files changed, 79 insertions(+), 63 deletions(-) diff --git a/skills/python-project-porting/references/gotchas-index.md b/skills/python-project-porting/references/gotchas-index.md index 676630c385..a64ce7d4ae 100644 --- a/skills/python-project-porting/references/gotchas-index.md +++ b/skills/python-project-porting/references/gotchas-index.md @@ -1,6 +1,6 @@ # Gotchas index — router for the themed gotcha files -The porting gotchas (370 of them) live in [`references/gotchas/`](gotchas/), split by theme so only the relevant slice loads. Every gotcha keeps a **permanent number** cited elsewhere as "gotcha N" (and in workflow comments as "CLAUDE.md gotcha N"). Numbers are stable IDs — **not sequential**, and four are **reused** with different content (two each of 33, 55, 56, 57), disambiguated by theme below. +The porting gotchas (374 of them) live in [`references/gotchas/`](gotchas/), split by theme so only the relevant slice loads. Every gotcha keeps a **permanent number** cited elsewhere as "gotcha N" (and in workflow comments as "CLAUDE.md gotcha N"). Numbers are stable IDs — **not sequential**, and four are **reused** with different content (two each of 33, 55, 56, 57), disambiguated by theme below. ## How to find the gotcha you need @@ -93,6 +93,47 @@ The porting gotchas (370 of them) live in [`references/gotchas/`](gotchas/), spl - **376** — A permissive `License:` field on the wrapper package says nothing about whether the payload it ships has any source at all — check the binary's own content, not the metadata's license family (the tableauhyperapi case). +- **381** — A third-party *vendor release* of a project this repo has already ruled out + inherits that verdict — resolve the redistribution to its upstream before triaging anything + else (the tokenspeed-triton case). +- **382** — Several PyPI distributions carved out of *one* build are one unit of work, not + one port each — check the allowed `--build-type` values before writing any YAML, and let + `requires_dist` (not the most "core-sounding" name) fix the order; complements gotcha 380 + (how to publish them once the combined port exists) (the + pyside6/pyside6-essentials/pyside6-addons case). +- **383** — The *umbrella* distribution of a split family carries no compiled code at all, + gets its platform+`abi3` tag from a deliberately fake `Extension`, and its payload is + generated stubs for the union of its siblings' modules — so it cannot be cut from a + different build than they were; also, check the in-image SDK's *minor version* against the + binding release (the pyside6 meta-wheel case). +- **385** — A no-sdist vendor wheel can still have a fully public build recipe — read + `dist-info/WHEEL`'s `Generator:` before parking it for "no source anywhere"; a + vendor-named generator is usually a *repackager*, which moves the stop to whether the + vendor publishes the payload for our arch (the pyqt6-qt6 case). +- **386** — A GPU-only package can be small, source-open and blob-free and still be + unportable: in a JIT kernel library the compiled part is a few-hundred-KB shim, so gotcha + 41's vendor-payload tell is absent and the wall is what that shim links — `libtorch_cuda.so`, + which our CPU-only riscv64 torch can never provide; refines gotchas 249 and 284 (the + humming-kernels case). +- **387** — A GPU-toolkit-suffixed distribution name (`-cuda12x`, `-rocm-7-0`) is a toolkit + selector whose name can be injected from a *separate* release-tools repo; check the vendor's + redist index for our arch, and treat a documented stub/no-CUDA build mode as a docs build, + not a port (the cupy-cuda12x case). +- **388** — The queue entry's wheel shape is a snapshot — re-read the *latest* release's tag + set before triaging the queued version, because upstream can delete the arch-specific + payload and erase the gap outright; also, a `py3-none-any` dependency can be a facade for + platform-only payload wheels (the tokenspeed-mla case). +- **392** — With no project URL and a stock `Generator:`, the *conda-forge feedstock* is the + cheapest source-availability oracle (a feedstock whose `source:` is the PyPI wheels is a + repackager, so there is nothing to build); `readelf -S` splits a real compiled extension + into engine vs embedded model weights (`.text` ~280 KB, `.rodata` ~34.8 MB); a compound + `License: AND LicenseRef-*` is gotcha 372's second lock; and an open-source + org's monorepo hits can all be the closed-source package's *consumer* + (the livekit-local-inference case). +- **393** — The bindings half of a "bindings wheel + vendored-SDK wheel" pair looks unblocked + from its sdist and is not: the blocking pin is added by the vendor's release step, not by the + sources, and the coupling is a `RUNPATH` into the sibling wheel's directory; a distro-SDK + build is defeated by the sibling's dlopened plugin/QML payload (the pyqt6 case). ### Sdist source & versioning — [`gotchas/sdist-source-and-versioning.md`](gotchas/sdist-source-and-versioning.md) @@ -162,9 +203,8 @@ The porting gotchas (370 of them) live in [`references/gotchas/`](gotchas/), spl - **360** — A `setup.py`'s own `bdist_wheel --plat-name` insertion can hardcode `manylinux1_` + `platform.machine()` regardless of the actual container libc, making musllinux unbuildable no matter how the CMake/C++ side is patched. -- **381** — A `cpXY-none-` wheel is the third plat-name shape: `setup.py` declares - no `ext_modules` at all, and a sibling CMake build both compiles the extension modules and - hands `bdist_wheel` the tag (the coremltools case). +- **391** — A project's real cibuildwheel recipe can live in a *separate packaging repo* that the + source tree never references — the source repo can carry no GitHub Actions at all. ### Rust, maturin & PyO3 — [`gotchas/rust-maturin-and-pyo3.md`](gotchas/rust-maturin-and-pyo3.md) @@ -260,9 +300,6 @@ The porting gotchas (370 of them) live in [`references/gotchas/`](gotchas/), spl - **202** — A monorepo's "regenerate deps from Bazel" helper may already tolerate a missing - **219** — GDAL's cmake build produces no `gdal-config` script — a second consumer of the - **233** — A package can have no Python build backend at all — the wheel comes from an -- **382** — A CMake build that shells out to a bare `python3` for one vendored sub-extension - silently builds it for the container's default interpreter, not the one the wheel is for - (the coremltools/kmeans1d case). ### The manylinux image & toolchain — [`gotchas/manylinux-image-and-toolchain.md`](gotchas/manylinux-image-and-toolchain.md) @@ -311,6 +348,9 @@ The porting gotchas (370 of them) live in [`references/gotchas/`](gotchas/), spl - **378** — A newer libstdc++ on the manylinux image can deprecate calls a project's own `-DCMAKE_COMPILE_WARNING_AS_ERROR=ON` CI flag then turns into hard errors, purely from a toolchain-version gap upstream's own (older) runners never see. +- **390** — libev is one of the `-devel` packages that *is* in Rocky 10's riscv64 repos, so an + upstream `yum install -y libev libev-devel` needs no replacement — but its header is + `/usr/include/ev.h`. ### Native dependencies & linking — [`gotchas/native-deps-and-linking.md`](gotchas/native-deps-and-linking.md) @@ -331,6 +371,7 @@ The porting gotchas (370 of them) live in [`references/gotchas/`](gotchas/), spl - **278** — A vendored, direct-copy (not submodule) header can be missing riscv64 from its - **363** — A `libraries=[...]` entry can go missing from the link line with *no* error — - **368** — Linking several codecs against Rocky 10's system libraries instead of +- **395** — When a project dlopen()s a differently-named shared library per major ### Compiled-vs-pure detection & the require-extension knob — [`gotchas/compiled-vs-pure-detection.md`](gotchas/compiled-vs-pure-detection.md) @@ -425,6 +466,9 @@ The porting gotchas (370 of them) live in [`references/gotchas/`](gotchas/), spl - **329** — A test suite that shells out to the package's own installed CLI binaries at a - **347** — A test that asserts "you're running against an editable/in-place install" can - **348** — A `glcontext`-based package's `create_context(standalone=True)` defaults to the +- **389** — A test `.pyx` that Cython-`include`s a checkout-root-relative path can be satisfied by + staging just those files; a staged package dir with no `__init__.py` is a namespace + portion and does not shadow the wheel. ### Testing: pytest config, servers & test selection — [`gotchas/pytest-config-servers-and-selection.md`](gotchas/pytest-config-servers-and-selection.md) @@ -529,6 +573,9 @@ The porting gotchas (370 of them) live in [`references/gotchas/`](gotchas/), spl - **223** — For a `bindings = "bin"` CLI's test assertions, `cargo build --release` the tool - **298** — A local rehearsal's `pip`-resolved cibuildwheel can be too old for - **369** — Without docker, fetch Rocky 10's own dnf repodata over plain HTTPS to +- **384** — `dnf` failing in the image with `Curl error (60) ... self-signed certificate` is + your egress proxy, not the image — install the proxy CA into the container trust store +- **394** — A libtorch-linking project cannot be rehearsed on x86_64 with PyPI's `torch` ### PR, CI, triggers, publishing & maintainer signals — [`gotchas/pr-ci-and-maintainer.md`](gotchas/pr-ci-and-maintainer.md) diff --git a/skills/python-project-porting/references/gotchas/cibuildwheel-matrix-and-abi3.md b/skills/python-project-porting/references/gotchas/cibuildwheel-matrix-and-abi3.md index cd39727532..b8bf35cc62 100644 --- a/skills/python-project-porting/references/gotchas/cibuildwheel-matrix-and-abi3.md +++ b/skills/python-project-porting/references/gotchas/cibuildwheel-matrix-and-abi3.md @@ -39,9 +39,8 @@ To pull up one entry: `grep -n '^N\. ' references/gotchas/cibuildwheel-matrix-an - **360** — A `setup.py`'s own `bdist_wheel --plat-name` insertion can hardcode `manylinux1_` + `platform.machine()` regardless of the actual container libc, making musllinux unbuildable no matter how the CMake/C++ side is patched. -- **381** — A `cpXY-none-` wheel is the third plat-name shape: `setup.py` declares - no `ext_modules` at all, and a sibling CMake build both compiles the extension modules and - hands `bdist_wheel` the tag. +- **391** — A project's real cibuildwheel recipe can live in a *separate packaging repo* that the + source tree never references — the source repo can carry no GitHub Actions at all. --- @@ -763,32 +762,26 @@ To pull up one entry: `grep -n '^N\. ' references/gotchas/cibuildwheel-matrix-an logic) is the lower-risk, upstream-faithful choice when this is caught on the first port rather than treated as a regression to fix. -381. **A `cpXY-none-` wheel is the third plat-name shape: `setup.py` declares no - `ext_modules` at all, and a sibling CMake build both compiles the extension modules and - hands `bdist_wheel` the tag (the coremltools case; see `build-coremltools.yml`).** - Gotcha 27 reads `py3-none-` as "the platform half was forced by hand, nothing - is compiled"; gotcha 81 adds the ctypes-payload exception. Neither covers a wheel tagged - `cpXY-**none**-` — an interpreter tag with no ABI tag — which is what you get - when the compiled modules are real, per-interpreter pybind11 `.so`s, but setuptools never - sees them: coremltools' `setup.py` has zero `ext_modules` and ships - `libmilstoragepython.so`/`libmodelpackage.so` through `package_data` globs, while - `CMakeLists.txt` builds them and then invokes `setup.py bdist_wheel - --plat-name= --python-tag=cp${MAJOR}${MINOR}` itself. Read the interpreter half - before concluding anything from the missing ABI tag: `cpXY-none` means per-interpreter - content that setuptools was not told about, whereas gotcha 27's `py3-none` means no - per-interpreter content at all. - - **The architecture of the whole wheel is then one string in the build system, not a - `platform.machine()` call.** Gotcha 360's keystone-engine case at least interpolated - the real machine; coremltools' CMakeLists is `elseif(... MATCHES "Linux") - set(PLAT_NAME "manylinux1_x86_64")` — a literal, so a riscv64 build silently produces - an x86_64-tagged wheel that installs on the wrong machine. Grep the build system for - `plat-name`/`plat_name`, not just `setup.py`, whenever `setup.py` has no `ext_modules` - but the released wheels carry a platform tag. - - **Patch it to a libc-neutral `linux_` and let `auditwheel repair` retag**, the - same resolution gotcha 360 points at, and keep the existing x86_64 branch byte for - byte so the diff is upstreamable: `if(${CMAKE_SYSTEM_PROCESSOR} MATCHES "x86_64")` → - the old literal, `else()` → `linux_${CMAKE_SYSTEM_PROCESSOR}`. - - **cibuildwheel is the wrong tool for this shape.** It drives a PEP 517 build, which - here would run `setup.py` on its own and emit a `py3-none-any` wheel containing - riscv64 `.so`s — worse than a wrong platform tag. Drive the container yourself - (gotcha 15) and run the project's own `dist` target. +391. **A project's real cibuildwheel recipe can live in a *separate packaging repo* that + the source tree never references — the source repo can carry no GitHub Actions at all + (the cassandra-driver case).** `pyproject.toml` has no `[tool.cibuildwheel]` table, the + release tag has no `.github/` directory, and the only CI file is a `Jenkinsfile` that + builds no wheel (it is the CCM/DSE integration matrix, installing the driver + `--editable`). Read only the source repo and the port looks like it has no upstream + recipe to mirror, which is how a workflow ends up invented from scratch — exactly the + divergence goal 2 forbids. Here it was `datastax/python-driver-wheels`, a `multibuild` + repo carrying the driver as a git submodule, named once in `README-dev.rst`'s release + checklist ("Update the `python-driver` submodule of `python-driver-wheels` … Trigger + the Github Actions necessary to build wheels"); its `build_wheels_linux.yml` holds the + whole recipe — `CIBW_BUILD`, `CIBW_SKIP`, `CIBW_BEFORE_ALL`, `CIBW_ENVIRONMENT` and the + `CIBW_TEST_*` set. + - **Where to look, in order**: the developer/release doc (`README-dev`, + `CONTRIBUTING`, `RELEASING`) for the name of a wheels/packaging repo; then that + repo's `.github/workflows/`; then its `config.sh`/`build_wheel.sh` if it is a + `multibuild` project — the pre-cibuildwheel hooks (`pre_build`, `run_tests`) often + still hold the *real* wheel-verification script that the cibuildwheel workflow later + replaced with a stub, which is the gotcha 94 answer for that package handed over + ready-made. + - Distinct from gotcha 338: there the packaging fork *is* where the released wheels + come from and it drags in a sibling dependency of its own; here the packaging repo is + upstream's and only holds the recipe, so the port still builds the source tag. diff --git a/skills/python-project-porting/references/gotchas/native-build-bazel-and-drivers.md b/skills/python-project-porting/references/gotchas/native-build-bazel-and-drivers.md index b206cc9870..24849e1958 100644 --- a/skills/python-project-porting/references/gotchas/native-build-bazel-and-drivers.md +++ b/skills/python-project-porting/references/gotchas/native-build-bazel-and-drivers.md @@ -19,8 +19,6 @@ To pull up one entry: `grep -n '^N\. ' references/gotchas/native-build-bazel-and - **202** — A monorepo's "regenerate deps from Bazel" helper may already tolerate a missing - **219** — GDAL's cmake build produces no `gdal-config` script — a second consumer of the - **233** — A package can have no Python build backend at all — the wheel comes from an -- **382** — A CMake build that shells out to a bare `python3` for one vendored sub-extension - silently builds it for the container's default interpreter, not the one the wheel is for. --- @@ -388,25 +386,3 @@ To pull up one entry: `grep -n '^N\. ' references/gotchas/native-build-bazel-and sibling wheels and `entry_points.txt` shims for other targets; only `write_base_packages` (the base `pip` wheel) matters here, so the heredoc reproduces that function alone and ignores the rest of the tool. - -382. **A CMake build that shells out to a bare `python3` for one vendored sub-extension - silently builds it for the container's default interpreter, not the one the wheel is - for (the coremltools/kmeans1d case).** Driving the container yourself means every - per-interpreter loop iteration passes the interpreter explicitly — `-DPYTHON_EXECUTABLE`, - `$PYBIN/python3`, a venv — and that covers the targets CMake compiles itself. It does not - cover an `execute_process(COMMAND python3 setup.py build_ext --inplace WORKING_DIRECTORY - ${DEPS}/kmeans1d)` buried in the same `CMakeLists.txt`: that resolves `python3` from - `PATH` at *configure* time, so a `-DPYTHON_EXECUTABLE=/opt/python/cp312-cp312/bin/python3` - build happily ships `_core.cpython-311--linux-gnu.so` inside a cp312 wheel. It is - invisible in a green build and a green import — the module is only imported by the - palettization code path, so the whole suite can pass — and `unzip -l | grep '\.so'` - is what catches it. - - **Upstream never sees it** because its own build script activates a conda env first, - making `python3` and `PYTHON_EXECUTABLE` the same binary. Reproducing that is one line - in the build script — `export PATH="$PYBIN:$PATH"` before `cmake` — and is strictly - safer than auditing every `execute_process` for the hardcoded name. - - **Grep for the bare interpreter name, not for `PYTHON_EXECUTABLE`.** `grep -rn - 'COMMAND python' CMakeLists.txt cmake/` finds both this and the `python -m lib2to3` - style post-processing steps that protobuf codegen rules commonly carry; the ones that - use `${PYTHON_EXECUTABLE}` are already correct, and the ones that do not are the list - the PATH export exists to cover. From 81bc0ad0e936664e3c87a5d0e99c7921a82771cb Mon Sep 17 00:00:00 2001 From: Ludovic Henry Date: Sat, 19 Sep 2026 18:41:54 +0000 Subject: [PATCH 5/5] coremltools: fix the patch step's path resolution Both riscv64 jobs failed ~30s in with error: can't open patch '../python-wheels/patches/coremltools/9.0/00*.patch': No such file or directory `git -C coremltools apply ../python-wheels/...` changes the directory for git only - the shell still expands the glob relative to the step's own cwd, which is the workspace root. The repo name doubles in the runner's workspace path (/home/runner/work/python-wheels/python-wheels), so `../python-wheels` from there resolves back to the workspace itself, the glob matches nothing and bash hands git the literal `00*.patch`. Use `working-directory: coremltools` with a plain `git apply` so the shell and git share one base, matching what every other subdirectory-checkout workflow here does (build-ray.yml, build-tink.yml, build-torch.yml, ...). Also list patches/coremltools/** in the triggers so a patch-only change rebuilds. --- .github/workflows/build-coremltools.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-coremltools.yml b/.github/workflows/build-coremltools.yml index 7fc1242852..5c691005d0 100644 --- a/.github/workflows/build-coremltools.yml +++ b/.github/workflows/build-coremltools.yml @@ -16,11 +16,13 @@ on: paths: - '.github/workflows/build-coremltools.yml' - 'docs/packages/coremltools.yaml' + - 'patches/coremltools/**' push: branches: [main] paths: - '.github/workflows/build-coremltools.yml' - 'docs/packages/coremltools.yaml' + - 'patches/coremltools/**' concurrency: group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} @@ -72,7 +74,8 @@ jobs: persist-credentials: false - name: Patch coremltools source - run: git -C coremltools apply ../python-wheels/patches/coremltools/${{ env.COREMLTOOLS_VERSION }}/00*.patch + working-directory: coremltools + run: git apply ../python-wheels/patches/coremltools/${{ env.COREMLTOOLS_VERSION }}/00*.patch - name: Build the C++ libraries and the wheel run: |