diff --git a/.cargo/config.toml b/.cargo/config.toml index 8fdd89a22..ba4523219 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -35,9 +35,15 @@ # portable baseline (AVX2): cargo build --config .cargo/config-v3.toml ... # AVX-512 (v4): cargo build --config .cargo/config-avx512.toml ... # -# CI is unaffected: every workflow sets `RUSTFLAGS` with an explicit -# `-C target-cpu=x86-64-v3`, and a RUSTFLAGS env REPLACES every cargo-config -# rustflags entry. Everything else that ships pins x86_64 to v3 without +# CI: build.yml, rust-test.yml and style.yml set `RUSTFLAGS` with an explicit +# `-C target-cpu=x86-64-v3` (a RUSTFLAGS env REPLACES every cargo-config +# rustflags entry). Every other workflow builds with `native`. Cargo +# fingerprints the flag TEXT, not the CPU, so a workflow that caches `target/` +# spells `native` out as the runner's exact feature list first +# (.github/scripts/resolve-native-rustflags.sh, used by dir-sim-persist.yml and +# identity.yml) and keys the cache to those flags; without that, an artifact +# built on a runner with AVX-512 is reused on one without and dies with SIGILL. +# The same script works locally: RUSTFLAGS="$(.github/scripts/resolve-native-rustflags.sh)". Everything else that ships pins x86_64 to v3 without # touching other architectures: the Dockerfiles that copy this tree replace # this file with config-v3.toml (x86_64-only), the thinking-engine images set # CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_RUSTFLAGS, and `make publish` in diff --git a/.github/scripts/resolve-native-rustflags.sh b/.github/scripts/resolve-native-rustflags.sh new file mode 100755 index 000000000..a665ae3c8 --- /dev/null +++ b/.github/scripts/resolve-native-rustflags.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash +# Print RUSTFLAGS whose target features equal `.cargo/config.toml`'s +# `-Ctarget-cpu=native` on this machine, spelled out as an explicit list. +# With --cpu, print the name rustc resolves `native` to instead (for logs). +# +# Why: cargo fingerprints the flag TEXT. `-Ctarget-cpu=native` reads the same +# on every machine, so an artifact built on a CPU with AVX-512 is reused on one +# without and dies with SIGILL. Spelled out, the text names the actual +# features: a different CPU gives different flags, and cargo recompiles. +# +# No `-Ctarget-cpu=`: a named CPU enables its model's default features, +# and a `+feature` list cannot take them back. On a VM that hides AVX-512 from +# a CPU still reporting a Skylake-server model, `-Ctarget-cpu=skylake-avx512` +# would compile AVX-512 back in. The target's baseline CPU plus the exact +# native list has no hidden defaults; the cost is tuning, not correctness. +# Features rustc hides on stable (amx, avx10) are not in the list, so they are +# never enabled; native could have enabled them, this cannot. +# +# RUSTFLAGS replaces every rustflags entry in .cargo/config.toml. That file +# sets only target-cpu, so nothing else is lost; extra flags given as +# arguments are appended. +set -euo pipefail +die() { echo "resolve-native-rustflags: $*" >&2; exit 1; } + +if [ "${1:-}" = "--cpu" ]; then + cpu=$(rustc --print target-cpus | sed -n 's/.*native.*(currently \([^)]*\)).*/\1/p') + [ -n "$cpu" ] || die "rustc did not report the native CPU" + echo "$cpu" + exit 0 +fi + +target_features() { rustc --print cfg "$@" | sed -n 's/^target_feature="\(.*\)"$/\1/p' | sort; } + +native=$(target_features -C target-cpu=native) +[ -n "$native" ] || die "no target features reported" +flags="-Ctarget-feature=$(printf '%s\n' "$native" | sed 's/^/+/' | paste -sd, -)" + +# The point of the script: these flags must enable exactly what native does. +# shellcheck disable=SC2086 +resolved=$(target_features $flags) +[ "$resolved" = "$native" ] || die "flags do not reproduce native: +$(diff <(printf '%s\n' "$native") <(printf '%s\n' "$resolved") || true)" + +echo "$flags${*:+ $*}" diff --git a/.github/workflows/dir-sim-persist.yml b/.github/workflows/dir-sim-persist.yml index 77300aed4..89c682181 100644 --- a/.github/workflows/dir-sim-persist.yml +++ b/.github/workflows/dir-sim-persist.yml @@ -62,8 +62,23 @@ jobs: run: sudo apt-get update && sudo apt-get install -y protobuf-compiler libprotobuf-dev - name: Setup rust toolchain (pinned by rust-toolchain.toml) run: rustup show + # `.cargo/config.toml` builds with target-cpu=native, and cargo's + # fingerprint sees the flag text, not the CPU: an artifact built on a + # runner with AVX-512 would be reused on one without and die with + # SIGILL. Spell `native` out as this runner's exact feature list: + # a different CPU then gives different flags, cargo recompiles by itself, + # and the cache is keyed to the same flags. + - name: Resolve target-cpu=native for this runner + id: cpu + run: | + flags=$(.github/scripts/resolve-native-rustflags.sh) + echo "RUSTFLAGS=$flags" >> "$GITHUB_ENV" + hash=$(printf %s "$flags" | sha256sum | cut -c1-16) + echo "flags=$hash" >> "$GITHUB_OUTPUT" + echo "native is \`$(.github/scripts/resolve-native-rustflags.sh --cpu)\`, features $hash, job \`$GITHUB_JOB\`" >> "$GITHUB_STEP_SUMMARY" - uses: Swatinem/rust-cache@v2 with: + key: cpu-${{ steps.cpu.outputs.flags }} workspaces: lance-graph/crates/lance-graph-dir-sim-lance - name: clippy run: cargo clippy --manifest-path crates/lance-graph-dir-sim-lance/Cargo.toml --all-targets -- -D warnings diff --git a/.github/workflows/identity.yml b/.github/workflows/identity.yml index 93fd412f1..04a9465e7 100644 --- a/.github/workflows/identity.yml +++ b/.github/workflows/identity.yml @@ -73,8 +73,23 @@ jobs: persist-credentials: false - name: Setup rust toolchain (pinned by rust-toolchain.toml) run: rustup show + # `.cargo/config.toml` builds with target-cpu=native, and cargo's + # fingerprint sees the flag text, not the CPU: an artifact built on a + # runner with AVX-512 would be reused on one without and die with + # SIGILL. Spell `native` out as this runner's exact feature list: + # a different CPU then gives different flags, cargo recompiles by itself, + # and the cache is keyed to the same flags. + - name: Resolve target-cpu=native for this runner + id: cpu + run: | + flags=$(.github/scripts/resolve-native-rustflags.sh) + echo "RUSTFLAGS=$flags" >> "$GITHUB_ENV" + hash=$(printf %s "$flags" | sha256sum | cut -c1-16) + echo "flags=$hash" >> "$GITHUB_OUTPUT" + echo "native is \`$(.github/scripts/resolve-native-rustflags.sh --cpu)\`, features $hash, job \`$GITHUB_JOB\`" >> "$GITHUB_STEP_SUMMARY" - uses: Swatinem/rust-cache@v2 with: + key: cpu-${{ steps.cpu.outputs.flags }} workspaces: lance-graph/crates/lance-graph-dir-sim - name: Run the simulator's tests run: cargo test --manifest-path crates/lance-graph-dir-sim/Cargo.toml @@ -213,8 +228,23 @@ jobs: persist-credentials: false - name: Setup rust toolchain (pinned by rust-toolchain.toml) run: rustup show + # `.cargo/config.toml` builds with target-cpu=native, and cargo's + # fingerprint sees the flag text, not the CPU: an artifact built on a + # runner with AVX-512 would be reused on one without and die with + # SIGILL. Spell `native` out as this runner's exact feature list: + # a different CPU then gives different flags, cargo recompiles by itself, + # and the cache is keyed to the same flags. + - name: Resolve target-cpu=native for this runner + id: cpu + run: | + flags=$(.github/scripts/resolve-native-rustflags.sh) + echo "RUSTFLAGS=$flags" >> "$GITHUB_ENV" + hash=$(printf %s "$flags" | sha256sum | cut -c1-16) + echo "flags=$hash" >> "$GITHUB_OUTPUT" + echo "native is \`$(.github/scripts/resolve-native-rustflags.sh --cpu)\`, features $hash, job \`$GITHUB_JOB\`" >> "$GITHUB_STEP_SUMMARY" - uses: Swatinem/rust-cache@v2 with: + key: cpu-${{ steps.cpu.outputs.flags }} workspaces: lance-graph/crates/lance-graph-rbac - name: Run the IAM tests run: cargo test --manifest-path crates/lance-graph-rbac/Cargo.toml