diff --git a/.cargo/config-v2.toml b/.cargo/config-v2.toml new file mode 100644 index 00000000..a21c1c58 --- /dev/null +++ b/.cargo/config-v2.toml @@ -0,0 +1,16 @@ +# x86-64-v2 (SSE2 through SSE4.2, no AVX) — the scalar realization, which LLVM +# vectorizes to SSE. +# +# Usage: +# env -u RUSTFLAGS cargo --config .cargo/config-v2.toml +# bash scripts/masking-parity.sh v2-qemu (runs under qemu -cpu Nehalem) +# +# Selects the arm `all(target_arch = "x86_64", target_feature = "sse4.2", +# not(target_feature = "avx"))`: Nehalem / Westmere, Silvermont-class Atom, +# VMs that hide AVX. `simd.rs` routes it to `simd_scalar.rs`. Plan: +# `.claude/plans/simd-sse-v2-tier-v1.md`. Overlay semantics as in +# `config-v3.toml`: only the target-cpu is set here; the crypto cfgs come from +# `.cargo/config.toml` through cargo's rustflags join. `env -u RUSTFLAGS` is +# load-bearing for the same reason. +[target.'cfg(target_arch = "x86_64")'] +rustflags = ["-Ctarget-cpu=x86-64-v2"] diff --git a/.claude/blackboard.md b/.claude/blackboard.md index 3894c478..32841821 100644 --- a/.claude/blackboard.md +++ b/.claude/blackboard.md @@ -1,3 +1,14 @@ +## 2026-10-10 — x86-64-v2 realization (SSE4.2 without AVX) via the scalar backend + +- **What:** an eighth realization, selected at compile time by `all(target_arch = "x86_64", target_feature = "sse4.2", not(target_feature = "avx"))`. You get it with `target-cpu=native` on a v2-only CPU (Nehalem/Westmere, Silvermont-class Atom, VMs that hide AVX) or with `.cargo/config-v2.toml`. `simd.rs` routes this arm to `simd_scalar.rs`: the AVX2/AVX-512 types wrap 256/512-bit registers and cannot be the v2 realization, so plain arrays that LLVM compiles to SSE are used instead. Baseline x86-64 builds (no SSE4.2) are unchanged: they keep the AVX2 arm and the `cpu_guard` AVX2 floor, which now exempts `sse4.2 && !avx2` (the AVX and v2 arms together). +- **API gap closed in `simd_scalar.rs`:** `F32Mask16::to_bitmask` and `F32x16::gather` (unsafe, same contract as the AVX2 polyfill). sentinel-qa: `gather` CONDITIONAL, satisfied by the SAFETY comment added at the one call site in `hpc/cam_pq.rs`; `to_bitmask` PARITY-OK. +- **Evidence:** + - `scripts/masking-parity.sh v2-qemu` passes all 15 groups under `qemu-x86_64-static -cpu Nehalem`. The objdump witness finds no `%ymm`, `%zmm`, mask register, or VEX op on xmm. Disable run: with the `simd.rs` routing reverted, the binary SIGILLs under Nehalem. + - Per-test sweep of all 2541 v2 lib tests, each as its own process under Nehalem: 28 SIGILLs, all in `simd_avx2`'s own test modules (`tests`, `u8x32_tests`, `f16_precision_tests`), which call AVX2 intrinsics directly. No library path faulted. Those modules are gated off v2 (36 tests). Full v2 suite afterwards: 2474 passed, 0 failed, 31 ignored. + - Clippy `-D warnings` clean on v2, v3, v4, avx and baseline. Codegen witness avx2/avx512 PASS. fmt clean. + - New CI row `realization/v2` runs the parity arm, the objdump witness and the SIMD unit tests under Nehalem. +- **Open:** `crate::simd_avx2` stays public on v2 builds and nothing checks for AVX2 at startup there, so a direct caller SIGILLs. Documented in the `simd.rs` module doc; the safe path is the `crate::simd` re-exports. + ## 2026-10-10 — `tests/splat3d_correctness.rs`: clippy 1.99 clean under `--features splat3d` - Five lints, all in the test file: an unneeded `mut` on a closure, an over-precise f32 literal (`0.28209479177387814` → `0.282_094_8`; same bits, `0x3e906ebb`), and three `chunks_exact(3)` → `as_chunks::<3>().0.iter()`. diff --git a/.claude/plans/simd-sse-v2-tier-v1.md b/.claude/plans/simd-sse-v2-tier-v1.md index c52a14ce..6ffced86 100644 --- a/.claude/plans/simd-sse-v2-tier-v1.md +++ b/.claude/plans/simd-sse-v2-tier-v1.md @@ -30,16 +30,27 @@ goes through `crate::simd`, so the switch is confined to `simd.rs`. ## Work items -- [ ] P0 TESTS FIRST: `v2-qemu` parity arm (`--config .cargo/config-v2.toml`, +- [x] P0 TESTS FIRST: `v2-qemu` parity arm (`--config .cargo/config-v2.toml`, run under `qemu-x86_64-static -cpu Nehalem`) in `scripts/masking-parity.sh` and a `realization/v2` row in `simd-matrix.yaml`, with an objdump witness (no AVX instruction at all: no `%ymm`, no VEX `v`-prefixed op). Expect it to fail before the arm exists. -- [ ] P1 `.cargo/config-v2.toml`; `simd.rs` routing; `cpu_guard` floor gated off +- [x] P1 `.cargo/config-v2.toml`; `simd.rs` routing; `cpu_guard` floor gated off the v2 arm. -- [ ] P2 close the API gap: compile v2, list every missing item, add each to +- [x] P2 close the API gap: compile v2, list every missing item, add each to `simd_scalar.rs` (types `U8x32`, `F32Mask16`, `F64Mask8` and the x86-only methods). Safe Rust only. -- [ ] P3 gates: lib tests on v2 under qemu Nehalem; parity on every arm; clippy +- [x] P3 gates: lib tests on v2 under qemu Nehalem; parity on every arm; clippy v2/v3/v4/avx/baseline; codegen witness avx2/avx512 unchanged. -- [ ] P4 docs: CLAUDE.md realization list, blackboard, `simd.rs` dispatch comment. +- [x] P4 docs: CLAUDE.md realization list, blackboard, `simd.rs` dispatch comment. + +## Outcome (2026-10-10) + +- Per-test sweep: all 2541 v2 lib tests, each run as its own process under + `qemu-x86_64-static -cpu Nehalem`. 28 faulted with SIGILL, every one in + `simd_avx2`'s own test modules (`tests`, `u8x32_tests`, + `f16_precision_tests`), which call AVX2 intrinsics directly. No library + path faulted. Those three modules are now gated off the v2 arm. +- Open: `crate::simd_avx2` stays public on v2 builds and has no startup + check there, so a direct caller faults with SIGILL. Documented in the + `simd.rs` module doc; the safe path is the `crate::simd` re-exports. diff --git a/.github/workflows/simd-matrix.yaml b/.github/workflows/simd-matrix.yaml index b7834310..71d918a5 100644 --- a/.github/workflows/simd-matrix.yaml +++ b/.github/workflows/simd-matrix.yaml @@ -266,6 +266,48 @@ jobs: # the multithreaded harness while a test was panicking. qemu-x86_64-static -cpu SandyBridge "$bin" simd --test-threads=1 + v2: + # x86-64-v2 realization (SSE4.2, no AVX): `simd.rs` routes it to the + # scalar backend, which LLVM vectorizes to SSE. Built for `x86-64-v2` + # (`.cargo/config-v2.toml`) and RUN under `qemu-x86_64-static -cpu + # Nehalem`, so any AVX instruction on the executed path faults. Three + # rungs, as for `avx`: parity, an assembly witness that the parity binary + # has no AVX encoding at all, and the SIMD unit tests under emulation. + runs-on: ubuntu-latest + name: realization/v2 × x86_64 + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + with: + key: v2 + - name: install qemu-user + run: sudo apt-get update && sudo apt-get install -y qemu-user-static + - name: masking parity (x86-64-v2 build, qemu -cpu Nehalem) + run: bash scripts/masking-parity.sh v2-qemu + - name: no AVX instruction in the parity binary + run: | + bin=crates/simd-masking-parity/target/release/simd-masking-parity + # Nehalem has no AVX: no ymm/zmm or mask register, and no VEX-encoded + # (v-prefixed) op on xmm either. The parity binary has no + # runtime-gated paths, so any hit came from an ungated intrinsic. + bad=$(objdump -d --no-show-raw-insn "$bin" | grep -E '%ymm|%zmm|%k[0-7]|\sv[a-z0-9]+\s.*%xmm' || true) + if [ -n "$bad" ]; then echo "$bad" | head -20; echo "::error::AVX instruction in the v2 parity binary"; exit 1; fi + echo "no AVX instruction found" + - name: SIMD unit tests under qemu -cpu Nehalem + env: + CARGO_PROFILE_DEV_DEBUG: "0" + CARGO_PROFILE_TEST_DEBUG: "0" + CARGO_PROFILE_DEV_OPT_LEVEL: "1" + CARGO_PROFILE_TEST_OPT_LEVEL: "1" + run: | + bin=$(env -u RUSTFLAGS cargo --config .cargo/config-v2.toml test --lib --no-run --message-format=json \ + | jq -r 'select(.executable != null and .target.kind == ["lib"] and .target.name == "ndarray") | .executable') + # One thread: see the `avx` job. + qemu-x86_64-static -cpu Nehalem "$bin" simd --test-threads=1 + neon: # aarch64 realization: cross-build on the x86 runner, run under qemu-user. # Three rungs: parity under qemu (bits), the codegen witness (opt-3 NEON diff --git a/CLAUDE.md b/CLAUDE.md index 673f8805..5a8bbc8a 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -196,7 +196,7 @@ src/ ### The parity arms are ALL reachable here — never report one as blocked without apt `scripts/masking-parity.sh` takes `native | nightly | wasm | wasm-scalar | -neon-qemu | avx-qemu`, and the cross arms are an `apt-get` away, not an environment +neon-qemu | avx-qemu | v2-qemu`, and the cross arms are an `apt-get` away, not an environment limit. Measured 2026-09-16: `neon-qemu` failed with a bare `No such file or directory (os error 2)` — which reads as "this target is not available here" and is in fact a **missing linker**, then a missing @@ -239,12 +239,14 @@ env -u RUSTFLAGS cargo --config ../../.cargo/config-v4.toml run --release Read the program's own header line to confirm which arm you actually got (`avx512f=true`, `neon=true`, …) — that line exists precisely because the -config can silently not apply. Six of the seven realizations are reachable -without nightly (AVX2, AVX-512, AVX-without-AVX2, NEON, wasm-simd128, -wasm-scalar); the AVX arm (`src/simd_avx.rs`, `.cargo/config-avx.toml`) runs -under `qemu-x86_64-static -cpu SandyBridge` (`apt-get install -y -qemu-user-static`); only -`nightly-simd` needs a toolchain this repo does not pin. The same lesson the +config can silently not apply. Seven of the eight realizations are reachable +without nightly (AVX2, AVX-512, AVX-without-AVX2, x86-64-v2, NEON, +wasm-simd128, wasm-scalar). The AVX arm (`src/simd_avx.rs`, +`.cargo/config-avx.toml`) runs under `qemu-x86_64-static -cpu SandyBridge`; +the v2 arm (SSE4.2 without AVX, routed to `simd_scalar.rs`, +`.cargo/config-v2.toml`) runs under `-cpu Nehalem` (both need `apt-get install +-y qemu-user-static`). Only `nightly-simd` needs a toolchain this repo does +not pin. The same lesson the sibling `lance-graph-java` records for the JDK: **a stale index or a missing helper binary reporting absence is not evidence of absence.** diff --git a/scripts/masking-parity.sh b/scripts/masking-parity.sh index a8d1418c..ccc9302c 100755 --- a/scripts/masking-parity.sh +++ b/scripts/masking-parity.sh @@ -9,12 +9,13 @@ # scripts/masking-parity.sh wasm-scalar wasm32 WITHOUT simd128 under node = the scalar arm # scripts/masking-parity.sh neon-qemu aarch64 cross-build run under qemu-aarch64-static # scripts/masking-parity.sh avx-qemu AVX-without-AVX2 build (config-avx) under qemu -cpu SandyBridge +# scripts/masking-parity.sh v2-qemu x86-64-v2 build (config-v2, scalar realization) under qemu -cpu Nehalem # # Extra cargo arguments (e.g. `--config .cargo/config-v4.toml` for the v4 # realization on the native row) pass through CARGO_ARGS, see # scripts/codegen-witness.sh for why that form and not the env var. set -euo pipefail -ARM="${1:?native|nightly|wasm|wasm-scalar|neon-qemu|avx-qemu}" +ARM="${1:?native|nightly|wasm|wasm-scalar|neon-qemu|avx-qemu|v2-qemu}" ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" MANIFEST="$ROOT/crates/simd-masking-parity/Cargo.toml" TD="${CARGO_TARGET_DIR:-$ROOT/crates/simd-masking-parity/target}" @@ -70,6 +71,14 @@ case "$ARM" in env -u RUSTFLAGS cargo ${CARGO_ARGS:-} --config "$ROOT/.cargo/config-avx.toml" build --release --manifest-path "$MANIFEST" --bin simd-masking-parity "$QEMU" -cpu SandyBridge "$TD/release/simd-masking-parity" ;; + v2-qemu) + # The x86-64-v2 arm: SSE4.2, no AVX, routed to the scalar realization. + # Built for x86-64-v2 and run on an emulated Nehalem, so an AVX + # instruction anywhere in the binary faults instead of passing silently. + QEMU="${QEMU_X86_64:-qemu-x86_64-static}" + env -u RUSTFLAGS cargo ${CARGO_ARGS:-} --config "$ROOT/.cargo/config-v2.toml" build --release --manifest-path "$MANIFEST" --bin simd-masking-parity + "$QEMU" -cpu Nehalem "$TD/release/simd-masking-parity" + ;; *) echo "unknown arm: $ARM"; exit 2 ;; esac echo "masking parity ($ARM): PASS" diff --git a/src/cpu_guard.rs b/src/cpu_guard.rs index f34fca3b..d96e7e88 100644 --- a/src/cpu_guard.rs +++ b/src/cpu_guard.rs @@ -98,7 +98,8 @@ impl fmt::Display for BuildCpuMismatch { f, " This build uses ndarray's AVX2 SIMD backend. For a CPU with AVX but \ not AVX2, rebuild with `target-cpu=native` on it or with \ - `--config .cargo/config-avx.toml`; a CPU without AVX is not supported." + `--config .cargo/config-avx.toml`; for one with SSE4.2 but not AVX, \ + `--config .cargo/config-v2.toml`. A CPU without SSE4.2 is not supported." ) } else { write!( @@ -274,12 +275,14 @@ const FEATURES: &[FeatureRow] = { /// The floor of `crate::simd`'s AVX2 realization (see the module docs): /// required on every x86_64 build that compiles that realization in, whatever -/// its target features. The one x86_64 build that does not is the -/// AVX-without-AVX2 arm (`simd_avx.rs`, `.cargo/config-avx.toml`), whose -/// `avx` requirement the compiled-feature table already enforces. +/// its target features. Two x86_64 builds do not, and the compiled-feature +/// table already enforces what each needs: the AVX-without-AVX2 arm +/// (`simd_avx.rs`, `.cargo/config-avx.toml`) and the x86-64-v2 arm +/// (SSE4.2 without AVX, the scalar realization, `.cargo/config-v2.toml`). +/// Together they are exactly `sse4.2` without `avx2`. #[cfg(target_arch = "x86_64")] const BACKEND_FLOOR: &[FeatureRow] = - &[("avx2", cfg!(not(all(target_feature = "avx", not(target_feature = "avx2")))), || { + &[("avx2", cfg!(not(all(target_feature = "sse4.2", not(target_feature = "avx2")))), || { cpuid::bit(7, 0, 'b', 5) && cpuid::os_avx() })]; @@ -448,14 +451,15 @@ mod tests { #[test] #[cfg(target_arch = "x86_64")] - fn avx2_is_required_unless_the_avx_arm_is_compiled_in() { - // A baseline build (no `avx2` compiled in) still reaches the AVX2 SIMD - // backend, so the floor applies to it; only the AVX-without-AVX2 arm - // is exempt, because it never selects that backend. + fn avx2_is_required_unless_the_avx_or_v2_arm_is_compiled_in() { + // A baseline build (no `sse4.2` compiled in) still reaches the AVX2 + // SIMD backend, so the floor applies to it; the AVX-without-AVX2 arm + // and the x86-64-v2 arm are exempt, because neither selects it. let avx_arm = cfg!(all(target_feature = "avx", not(target_feature = "avx2"))); + let v2_arm = cfg!(all(target_feature = "sse4.2", not(target_feature = "avx"))); assert!(BACKEND_FLOOR .iter() - .any(|&(n, compiled, _)| n == "avx2" && compiled == !avx_arm)); + .any(|&(n, compiled, _)| n == "avx2" && compiled == !(avx_arm || v2_arm))); } #[test] @@ -463,6 +467,7 @@ mod tests { let floor = BuildCpuMismatch { missing: vec!["avx2"] }.to_string(); assert!(floor.contains("AVX2 SIMD backend"), "{floor}"); assert!(floor.contains("config-avx.toml"), "{floor}"); + assert!(floor.contains("config-v2.toml"), "{floor}"); let other = BuildCpuMismatch { missing: vec!["avx512f"], } diff --git a/src/hpc/cam_pq.rs b/src/hpc/cam_pq.rs index 3684e17b..392af553 100644 --- a/src/hpc/cam_pq.rs +++ b/src/hpc/cam_pq.rs @@ -248,6 +248,11 @@ impl DistanceTables { // Gather distances from precomputed table let base_ptr = self.tables[s].as_ptr(); + // SAFETY: every index is a `u8` code (`CamFingerprint = [u8; 6]`), + // so 0..=255 < NUM_CENTROIDS, and `base_ptr` starts the + // `[f32; NUM_CENTROIDS]` table `self.tables[s]` (`s < + // NUM_SUBSPACES`): each offset lands on an initialised, aligned + // `f32` in that one array. let distances = F32x16::gather(indices, base_ptr); acc = acc + distances; diff --git a/src/simd.rs b/src/simd.rs index 24253c3e..3c618191 100644 --- a/src/simd.rs +++ b/src/simd.rs @@ -4,6 +4,26 @@ //! AVX-512 → AVX2 → Scalar. Consumer writes `crate::simd::F32x16`. Period. //! //! When `std::simd` stabilizes: swap this file. Zero consumer changes. +//! +//! The `Tier` above is runtime detection. The SIMD *types* are chosen at +//! compile time, one realization per build, from the target features: +//! +//! * x86_64 `avx512f`: `simd_avx512.rs`. +//! * x86_64 `avx2`, no `avx512f`: `simd_avx2.rs`. This arm is also what a +//! baseline x86-64 build compiles, and `cpu_guard` refuses to start such a +//! build on a CPU without AVX2. +//! * x86_64 `avx`, no `avx2`: `simd_avx.rs` (`.cargo/config-avx.toml`). +//! * x86_64 `sse4.2`, no `avx` (x86-64-v2, `.cargo/config-v2.toml`): +//! `simd_scalar.rs`. The AVX2/AVX-512 types wrap 256/512-bit registers, so +//! this arm uses plain arrays that LLVM compiles to SSE. +//! * aarch64: NEON; wasm32 with `simd128`: `simd_wasm.rs`; any other target: +//! `simd_scalar.rs`. The `nightly-simd` feature replaces all of these with +//! `core::simd` for validation only. +//! +//! The module `crate::simd_avx2` stays public on every x86_64 build, +//! including v2. Calling it directly on a CPU without AVX2 faults with +//! SIGILL, and on the v2 arm no startup check catches it. Use the types +//! re-exported from `crate::simd`. #[cfg(feature = "std")] use std::sync::LazyLock; @@ -346,6 +366,7 @@ pub use crate::simd_avx512::{BF16x16, BF16x8}; #[cfg(all( target_arch = "x86_64", not(target_feature = "avx512f"), + not(all(target_feature = "sse4.2", not(target_feature = "avx"))), not(feature = "nightly-simd") ))] pub use crate::simd_avx512::{ @@ -356,6 +377,7 @@ pub use crate::simd_avx512::{ #[cfg(all( target_arch = "x86_64", not(target_feature = "avx512f"), + not(all(target_feature = "sse4.2", not(target_feature = "avx"))), not(feature = "nightly-simd") ))] pub use crate::simd_avx2::{ @@ -369,7 +391,11 @@ pub use crate::simd_avx2::{ // AVX2 ops, and on AVX-512 builds it's the half-register companion to // U8x64. Lives in simd_avx2.rs (single source of truth) and is re-exported // from both tier branches. -#[cfg(all(target_arch = "x86_64", not(feature = "nightly-simd")))] +#[cfg(all( + target_arch = "x86_64", + not(all(target_feature = "sse4.2", not(target_feature = "avx"))), + not(feature = "nightly-simd") +))] pub use crate::simd_avx2::{u8x32, U8x32}; // ============================================================================ @@ -381,7 +407,13 @@ pub use crate::simd_avx2::{u8x32, U8x32}; // the existing `pub use scalar::{...}` re-exports below don't need to // change. Extracted from this file in Phase 4 of the integration plan // (1271 LoC of macro expansions out of the dispatcher). -#[cfg(all(not(target_arch = "x86_64"), not(feature = "nightly-simd")))] +#[cfg(all( + any( + not(target_arch = "x86_64"), + all(target_feature = "sse4.2", not(target_feature = "avx")) + ), + not(feature = "nightly-simd") +))] #[path = "simd_scalar.rs"] pub(crate) mod scalar; @@ -444,9 +476,17 @@ pub use scalar::{ // Other non-x86 targets — wasm32 without simd128, riscv, etc.: full scalar // fallback. Excludes the wasm32+simd128 case handled by the native arm above. #[cfg(all( - not(target_arch = "x86_64"), - not(target_arch = "aarch64"), - not(all(target_arch = "wasm32", target_feature = "simd128")), + any( + all( + not(target_arch = "x86_64"), + not(target_arch = "aarch64"), + not(all(target_arch = "wasm32", target_feature = "simd128")) + ), + // x86-64-v2 (SSE4.2, no AVX): `simd_avx2.rs`/`simd_avx512.rs` types wrap + // 256/512-bit registers, so this arm takes the scalar realization, which + // LLVM vectorizes to SSE. Plan: `.claude/plans/simd-sse-v2-tier-v1.md`. + all(target_arch = "x86_64", all(target_feature = "sse4.2", not(target_feature = "avx"))) + ), not(feature = "nightly-simd") ))] pub use scalar::{ @@ -1083,15 +1123,14 @@ mod tests { /// transpose that drops, duplicates or misroutes any word fails, and the /// fixture is asserted not to be symmetric, so an identity "transpose" /// fails too. Transposing twice must return the input. - /// Every `i8` value through `I8x16`/`I8x32::saturating_abs`, on whichever - /// path this build compiled: the SSSE3 intrinsic (`cfg(target_feature = - /// "ssse3")`: v3, v4, native) or the scalar fallback (a baseline build, - /// e.g. CI's `tests/*` jobs, whose RUSTFLAGS drop `target-cpu`). Lives - /// here, not in `simd_avx512.rs`, whose test modules are `avx512f`-only. - #[cfg(all(feature = "std", target_arch = "x86_64"))] + /// Every `i8` value through `I8x16`/`I8x32::saturating_abs` as the facade + /// resolves them on this build: on x86 the SSSE3 intrinsic (v3, v4, + /// native) or its scalar fallback (a baseline build, e.g. CI's `tests/*` + /// jobs, whose RUSTFLAGS drop `target-cpu`); on the x86-64-v2 arm, NEON + /// and wasm, that arm's own type. Lives here, not in `simd_avx512.rs`, + /// whose test modules are `avx512f`-only. #[test] - fn x86_saturating_abs_is_exact_for_all_256_i8_values() { - use crate::simd_avx512::{I8x16, I8x32}; + fn saturating_abs_is_exact_for_all_256_i8_values() { let all: [i8; 256] = core::array::from_fn(|i| i as u8 as i8); for c in all.chunks_exact(16) { let v: [i8; 16] = c.try_into().unwrap(); @@ -1107,7 +1146,11 @@ mod tests { /// each checked lane-by-lane against a scalar reference on asymmetric /// data. They run on the AVX2, AVX-512 and AVX-without-AVX2 realizations; /// a swapped operand, swapped 128-bit halves or a wrong intrinsic fails. - #[cfg(all(feature = "std", target_arch = "x86_64"))] + #[cfg(all( + feature = "std", + target_arch = "x86_64", + not(all(target_feature = "sse4.2", not(target_feature = "avx"))) + ))] #[test] fn x86_256bit_integer_methods_match_scalar() { use crate::simd_avx2::{U16x16, U8x32}; @@ -1160,7 +1203,10 @@ mod tests { /// (`255 * 127 * 2 > i16::MAX` after the `^ 0x80` bias), an unfolded i32 /// accumulator wraps on long inputs, and a shorter `b` was read past its /// end. On the AVX-without-AVX2 arm the same name is `simd_avx::dot_i8`. - #[cfg(target_arch = "x86_64")] + #[cfg(all( + target_arch = "x86_64", + not(all(target_feature = "sse4.2", not(target_feature = "avx"))) + ))] #[test] fn simd_avx2_dot_i8_is_exact() { use crate::simd_avx2::dot_i8 as dot; diff --git a/src/simd_avx2.rs b/src/simd_avx2.rs index 54b6dc24..2387f4a6 100644 --- a/src/simd_avx2.rs +++ b/src/simd_avx2.rs @@ -3248,7 +3248,13 @@ impl Default for U8x32 { // U8x32 tests // ═══════════════════════════════════════════════════════════════════ -#[cfg(all(test, target_arch = "x86_64"))] +// Off the x86-64-v2 arm: these call AVX2 intrinsics directly, and a v2 build +// (SSE4.2, no AVX) routes `crate::simd` to the scalar realization instead. +#[cfg(all( + test, + target_arch = "x86_64", + not(all(target_feature = "sse4.2", not(target_feature = "avx"))) +))] mod u8x32_tests { use super::U8x32; @@ -3446,7 +3452,9 @@ pub type i32x8 = I32x8; #[allow(non_camel_case_types)] pub type i64x4 = I64x4; -#[cfg(test)] +// Off the x86-64-v2 arm: these call AVX2 intrinsics directly, and a v2 build +// (SSE4.2, no AVX) routes `crate::simd` to the scalar realization instead. +#[cfg(all(test, not(all(target_feature = "sse4.2", not(target_feature = "avx")))))] mod tests { use super::*; @@ -4001,7 +4009,9 @@ impl F16Scaler { } } -#[cfg(test)] +// Off the x86-64-v2 arm: these call AVX2 intrinsics directly, and a v2 build +// (SSE4.2, no AVX) routes `crate::simd` to the scalar realization instead. +#[cfg(all(test, not(all(target_feature = "sse4.2", not(target_feature = "avx")))))] mod f16_precision_tests { use super::*; diff --git a/src/simd_scalar.rs b/src/simd_scalar.rs index 4bd377c2..3308ee3e 100644 --- a/src/simd_scalar.rs +++ b/src/simd_scalar.rs @@ -1064,8 +1064,49 @@ impl Neg for I32x16 { } } +// x86-shared API on the mask: `to_bitmask` exists on every x86 arm (and NEON); +// the x86-64-v2 arm takes this realization, so it needs it too. Only on +// `F32Mask16`, as on x86 — `F64Mask8` has no `to_bitmask` there. +impl F32Mask16 { + /// Lane `i` of the comparison is bit `i` of the result. + /// + /// ``` + /// use ndarray::simd::F32x16; + /// let mut a = [1.0f32; 16]; + /// a[0] = -1.0; + /// a[15] = -1.0; + /// let m = F32x16::from_array(a).simd_lt(F32x16::splat(0.0)); + /// assert_eq!(m.to_bitmask(), 0b1000_0000_0000_0001); + /// ``` + #[inline(always)] + pub fn to_bitmask(self) -> u16 { + self.0 + } +} + // Extra for F32x16: to_bits/from_bits/cast_i32 impl F32x16 { + /// Gather 16 `f32` values from `base_ptr` at the signed element offsets in + /// `indices` — the same signature and contract as the AVX-512 backend's + /// `_mm512_i32gather_ps` form and the AVX2 polyfill. Offsets are signed: + /// a negative index reads an element before `base_ptr`. + /// + /// # Safety + /// For every `i in 0..16`, `base_ptr.offset(indices[i] as isize)` must + /// lie inside one allocation together with `base_ptr`, be 4-byte + /// aligned, and point at an initialised, readable `f32`. + #[inline(always)] + pub unsafe fn gather(indices: I32x16, base_ptr: *const f32) -> Self { + let idx = indices.0; + let mut o = [0.0f32; 16]; + for i in 0..16 { + // SAFETY: the caller's contract above — each signed offset stays + // inside `base_ptr`'s allocation and points at a readable `f32`. + o[i] = unsafe { *base_ptr.offset(idx[i] as isize) }; + } + Self(o) + } + #[inline(always)] pub fn to_bits(self) -> U32x16 { let mut out = [0u32; 16];