Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 16 additions & 4 deletions .github/workflows/ci.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -342,9 +342,11 @@ jobs:
- uses: dtolnay/rust-toolchain@nightly
- uses: Swatinem/rust-cache@v2
- name: cargo check (nightly + nightly-simd feature)
run: cargo check -p ndarray --features approx,serde,rayon,nightly-simd
# `+nightly`: rust-toolchain.toml pins stable and takes precedence over
# the toolchain dtolnay installs, so a plain `cargo` here is stable.
run: cargo +nightly check -p ndarray --features approx,serde,rayon,nightly-simd
- name: cargo check (nightly + nightly-simd + hpc-extras)
run: cargo check -p ndarray --features approx,serde,rayon,nightly-simd,hpc-extras
run: cargo +nightly check -p ndarray --features approx,serde,rayon,nightly-simd,hpc-extras

blas-msrv:
runs-on: ubuntu-latest
Expand Down Expand Up @@ -418,11 +420,17 @@ jobs:
steps:
- uses: actions/checkout@v4
- uses: dtolnay/rust-toolchain@nightly
with:
# careful rebuilds std from source; without rust-src it stalls
# (seen locally: no compiler process, no output, for 25 min).
components: rust-src
- uses: Swatinem/rust-cache@v2
- uses: taiki-e/install-action@nextest
- name: Install cargo-careful
run: cargo install cargo-careful
- run: cargo careful nextest run -Zcareful-sanitizer --features="$FEATURES"
# `+nightly`: see nightly-simd-polyfill. On stable, careful fails at
# `rustc --print target-spec-json` ("output is invalid JSON").
- run: cargo +nightly careful nextest run -Zcareful-sanitizer --features="$FEATURES"

docs:
# Nightly needed for --cfg docsrs. Only on merge queue / push.
Expand All @@ -434,7 +442,11 @@ jobs:
steps:
- uses: actions/checkout@v4
- uses: dtolnay/rust-toolchain@nightly
- run: cargo doc --no-deps --all-features
# `+nightly`: see nightly-simd-polyfill. Features match
# `[package.metadata.docs.rs]`: `--all-features` cannot build, because
# `intel-mkl`/`openblas` and `runtime-dispatch`/`nightly-simd` are
# mutually exclusive (`compile_error!`).
- run: cargo +nightly doc --no-deps --features approx,serde,rayon

conclusion:
needs:
Expand Down
8 changes: 7 additions & 1 deletion scripts/miri-tests.sh
Original file line number Diff line number Diff line change
Expand Up @@ -64,9 +64,10 @@ cargo +nightly miri nextest run -v \
test(/^hpc::/) - test(/^hpc::byte_scan/)
) and !test(/^simd::tests::/)
and !test(/^hpc::framebuffer::pyramid_tests::/)
and !test(/^cpu_guard::/)
'
#
# Filter rationale (3-clause AND):
# Filter rationale (4-clause AND):
#
# 1. `!(test(/^hpc::/) - test(/^hpc::byte_scan/))`
# Skip everything in `hpc::*` EXCEPT `hpc::byte_scan` (the scalar-fallback
Expand All @@ -85,3 +86,8 @@ cargo +nightly miri nextest run -v \
# loops over SIMD-shaped data). Not a UB signal — pure runtime cost.
# Re-enable once the test fixtures are sized down or the loops are
# cfg(miri)-shortened.
#
# 4. `!test(/^cpu_guard::/)`
# `cpu_guard` reads the CPU with `cpuid`, which is inline assembly; Miri
# rejects it ("inline assembly is not supported"). The pre-`main` hook is
# `cfg(not(miri))` for the same reason, so test binaries start under Miri.
10 changes: 5 additions & 5 deletions src/backend/kernels_avx512.rs
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ use crate::simd::{F32x16, F64x8};
// BLAS Level 1 — 12 functions (compat types)
// ═══════════════════════════════════════════════════════════════════

/// Dot product: sum(x[i] * y[i]) using 4x-unrolled FMA.
/// Dot product: sum(x\[i\] * y\[i\]) using 4x-unrolled FMA.
/// # Safety
/// Caller must ensure AVX-512F is available (`simd_caps().avx512f`).
#[cfg(target_arch = "x86_64")]
Expand Down Expand Up @@ -167,7 +167,7 @@ pub fn scal_f64(alpha: f64, x: &mut [f64]) {
}
}

/// L1 norm: sum(|x[i]|) (f32, 16-wide).
/// L1 norm: sum(|x\[i\]|) (f32, 16-wide).
/// # Safety
/// Caller must ensure AVX-512F is available (`simd_caps().avx512f`).
#[cfg(target_arch = "x86_64")]
Expand All @@ -185,7 +185,7 @@ pub fn asum_f32(x: &[f32]) -> f32 {
sum
}

/// L1 norm: sum(|x[i]|) (f64, 8-wide).
/// L1 norm: sum(|x\[i\]|) (f64, 8-wide).
/// # Safety
/// Caller must ensure AVX-512F is available (`simd_caps().avx512f`).
#[cfg(target_arch = "x86_64")]
Expand All @@ -203,7 +203,7 @@ pub fn asum_f64(x: &[f64]) -> f64 {
sum
}

/// L2 norm: sqrt(sum(x[i]^2)) (f32, 16-wide FMA).
/// L2 norm: sqrt(sum(x\[i\]^2)) (f32, 16-wide FMA).
/// # Safety
/// Caller must ensure AVX-512F is available (`simd_caps().avx512f`).
#[cfg(target_arch = "x86_64")]
Expand All @@ -225,7 +225,7 @@ pub fn nrm2_f32(x: &[f32]) -> f32 {
sum.sqrt()
}

/// L2 norm: sqrt(sum(x[i]^2)) (f64, 8-wide FMA).
/// L2 norm: sqrt(sum(x\[i\]^2)) (f64, 8-wide FMA).
/// # Safety
/// Caller must ensure AVX-512F is available (`simd_caps().avx512f`).
#[cfg(target_arch = "x86_64")]
Expand Down
16 changes: 8 additions & 8 deletions src/backend/native.rs
Original file line number Diff line number Diff line change
Expand Up @@ -167,10 +167,10 @@ macro_rules! dispatch {
// ─── BLAS-1 dispatch ──────────────────────────────────────────────

dispatch!(
/// Dot product: result = sum(x[i] * y[i]) (f32).
/// Dot product: result = sum(x\[i\] * y\[i\]) (f32).
dot_f32(x: &[f32], y: &[f32]) -> f32);
dispatch!(
/// Dot product: result = sum(x[i] * y[i]) (f64).
/// Dot product: result = sum(x\[i\] * y\[i\]) (f64).
dot_f64(x: &[f64], y: &[f64]) -> f64);
dispatch!(
/// AXPY: y = alpha * x + y (f32).
Expand All @@ -186,16 +186,16 @@ dispatch!(
/// Scale: x = alpha * x (f64).
scal_f64(alpha: f64, x: &mut [f64]));
dispatch!(
/// L2 norm: sqrt(sum(x[i]^2)) (f32).
/// L2 norm: sqrt(sum(x\[i\]^2)) (f32).
nrm2_f32(x: &[f32]) -> f32);
dispatch!(
/// L2 norm: sqrt(sum(x[i]^2)) (f64).
/// L2 norm: sqrt(sum(x\[i\]^2)) (f64).
nrm2_f64(x: &[f64]) -> f64);
dispatch!(
/// L1 norm: sum(|x[i]|) (f32).
/// L1 norm: sum(|x\[i\]|) (f32).
asum_f32(x: &[f32]) -> f32);
dispatch!(
/// L1 norm: sum(|x[i]|) (f64).
/// L1 norm: sum(|x\[i\]|) (f64).
asum_f64(x: &[f64]) -> f64);

// ─── GEMM dispatch ───────────────────────────────────────────────
Expand Down Expand Up @@ -284,7 +284,7 @@ pub fn gemm_f64(
/// GEMV: y = alpha * A * x + beta * y (f32, row-major).
///
/// SIMD tiers compute each row via [`dot_f32`]; the scalar tier uses
/// the byte-stable [`scalar::gemv_f32`] reference.
/// the byte-stable `scalar::gemv_f32` reference.
pub fn gemv_f32(m: usize, n: usize, alpha: f32, a: &[f32], lda: usize, x: &[f32], beta: f32, y: &mut [f32]) {
if m == 0 {
return; // no rows ⇒ no-op; must not slice `x[..n]` (scalar ref returns too)
Expand All @@ -307,7 +307,7 @@ pub fn gemv_f32(m: usize, n: usize, alpha: f32, a: &[f32], lda: usize, x: &[f32]
/// GEMV: y = alpha * A * x + beta * y (f64, row-major).
///
/// SIMD tiers compute each row via [`dot_f64`]; the scalar tier uses
/// the byte-stable [`scalar::gemv_f64`] reference.
/// the byte-stable `scalar::gemv_f64` reference.
pub fn gemv_f64(m: usize, n: usize, alpha: f64, a: &[f64], lda: usize, x: &[f64], beta: f64, y: &mut [f64]) {
if m == 0 {
return; // no rows ⇒ no-op; must not slice `x[..n]` (scalar ref returns too)
Expand Down
2 changes: 1 addition & 1 deletion src/bitwise.rs
Original file line number Diff line number Diff line change
Expand Up @@ -204,7 +204,7 @@ const HAMMING_WITHIN_BLOCK: usize = 256;
///
/// Exact, not approximate: the running total only grows, so once it exceeds
/// `max` the final distance must too, and the remaining bytes are skipped.
/// The check happens after every [`HAMMING_WITHIN_BLOCK`]-byte block, so a
/// The check happens after every `HAMMING_WITHIN_BLOCK`-byte block, so a
/// reject costs at most one block beyond the point where the budget ran out.
/// Like [`hamming_distance_raw`], unequal lengths measure the common prefix.
///
Expand Down
10 changes: 7 additions & 3 deletions src/cpu_guard.rs
Original file line number Diff line number Diff line change
Expand Up @@ -348,6 +348,10 @@ pub fn assert_build_cpu() {

/// The pre-`main` hook. Prints and exits instead of panicking: unwinding out
/// of a loader-invoked `extern "C"` function would abort with no message.
///
/// Not installed under Miri: `cpuid` is inline assembly, which Miri cannot
/// execute, and an interpreter has no CPU to check.
#[cfg(not(miri))]
extern "C" fn guard_before_main() {
if let Err(mismatch) = check_build_cpu() {
use std::io::Write;
Expand All @@ -358,17 +362,17 @@ extern "C" fn guard_before_main() {

// The loader calls every function pointer in these sections before `main`.
// `#[used]` keeps the static even though nothing references it.
#[cfg(any(target_os = "linux", target_os = "android", target_os = "freebsd"))]
#[cfg(all(not(miri), any(target_os = "linux", target_os = "android", target_os = "freebsd")))]
#[used]
#[link_section = ".init_array"]
static GUARD_BEFORE_MAIN: extern "C" fn() = guard_before_main;

#[cfg(any(target_os = "macos", target_os = "ios"))]
#[cfg(all(not(miri), any(target_os = "macos", target_os = "ios")))]
#[used]
#[link_section = "__DATA,__mod_init_func"]
static GUARD_BEFORE_MAIN: extern "C" fn() = guard_before_main;

#[cfg(target_os = "windows")]
#[cfg(all(not(miri), target_os = "windows"))]
#[used]
#[link_section = ".CRT$XCU"]
static GUARD_BEFORE_MAIN: extern "C" fn() = guard_before_main;
Expand Down
4 changes: 2 additions & 2 deletions src/heel_f64x8.rs
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ use crate::simd::{array_chunks, F64x8};
///
/// `distances[i]` = distance for HEEL plane i.
/// `weights[i]` = importance weight for plane i.
/// Returns: Σ(distances[i] × weights[i]).
/// Returns: Σ(distances\[i\] × weights\[i\]).
///
/// One F64x8 multiply + reduce_sum. On AVX-512: single vmulpd + vreducepd.
/// On AVX2: 2× vmulpd + 2× haddpd. Scalar: 8 multiplies + sum.
Expand All @@ -28,7 +28,7 @@ pub fn heel_weighted_distance(distances: &[f64; 8], weights: &[f64; 8]) -> f64 {

/// Compute L1-like distance across 8 HEEL planes.
///
/// For each plane i: distance[i] = popcount(a[i] XOR b[i]) as f64.
/// For each plane i: distance\[i\] = popcount(a\[i\] XOR b\[i\]) as f64.
/// This is Hamming on binary HEEL planes — valid because HEEL planes
/// ARE uniform binary data (unlike bgz17 i16 which must use L1).
pub fn heel_plane_distances(a: &[u64; 8], b: &[u64; 8]) -> [f64; 8] {
Expand Down
2 changes: 1 addition & 1 deletion src/hpc/amx_matmul.rs
Original file line number Diff line number Diff line change
Expand Up @@ -81,7 +81,7 @@ impl TileConfig {
/// register-blocked int8 kernel: tmm0-3 = the four C accumulators, tmm4-5 =
/// two plain A row-blocks (rm/unsigned), tmm6-7 = two VNNI B col-blocks
/// (vvvv/signed). Every tile is 16×64 so one config serves all roles. Same
/// XTILECFG layout as [`Self::for_dpbusd`]: colsb[t] u16 @ 16+2t, rows[t]
/// XTILECFG layout as [`Self::for_dpbusd`]: colsb\[t\] u16 @ 16+2t, rows\[t\]
/// u8 @ 48+t.
///
/// # Examples
Expand Down
36 changes: 18 additions & 18 deletions src/hpc/amx_ops.rs
Original file line number Diff line number Diff line change
Expand Up @@ -34,15 +34,15 @@
//!
//! | tier | ops | CPUID | first silicon |
//! |---|---|---|---|
//! | AMX-TILE | config, zero, load/store, release | 7.0:EDX[24] | Sapphire Rapids |
//! | AMX-INT8 | `tdpb{ss,su,us,uu}d` | 7.0:EDX[25] | Sapphire Rapids |
//! | AMX-BF16 | `tdpbf16ps` | 7.0:EDX[22] | Sapphire Rapids |
//! | AMX-FP16 | `tdpfp16ps` | 7.1:EAX[21] | Granite Rapids |
//! | AMX-COMPLEX | `tcmm{im,rl}fp16ps` | 7.1:EDX[8] | Granite Rapids-D |
//! | AMX-FP8 | `tdp{b,bh,hb,h}f8ps` | 1E.1:EAX[4] | Diamond Rapids |
//! | AMX-TF32 | `tmmultf32ps` | 1E.1:EAX[6] | Diamond Rapids (see note) |
//! | AMX-AVX512 | `tcvtrow*`, `tilemovrow` | 1E.1:EAX[7] | Diamond Rapids |
//! | AMX-MOVRS | `tileloaddrs{,t1}` | 1E.1:EAX[8] | Diamond Rapids |
//! | AMX-TILE | config, zero, load/store, release | 7.0:EDX\[24\] | Sapphire Rapids |
//! | AMX-INT8 | `tdpb{ss,su,us,uu}d` | 7.0:EDX\[25\] | Sapphire Rapids |
//! | AMX-BF16 | `tdpbf16ps` | 7.0:EDX\[22\] | Sapphire Rapids |
//! | AMX-FP16 | `tdpfp16ps` | 7.1:EAX\[21\] | Granite Rapids |
//! | AMX-COMPLEX | `tcmm{im,rl}fp16ps` | 7.1:EDX\[8\] | Granite Rapids-D |
//! | AMX-FP8 | `tdp{b,bh,hb,h}f8ps` | 1E.1:EAX\[4\] | Diamond Rapids |
//! | AMX-TF32 | `tmmultf32ps` | 1E.1:EAX\[6\] | Diamond Rapids (see note) |
//! | AMX-AVX512 | `tcvtrow*`, `tilemovrow` | 1E.1:EAX\[7\] | Diamond Rapids |
//! | AMX-MOVRS | `tileloaddrs{,t1}` | 1E.1:EAX\[8\] | Diamond Rapids |
//!
//! Note on TF32: LLVM `main` has removed `amx-tf32` (and `amx-transpose`)
//! from both the assembler and `Host.cpp`. The 22.1.8 assembler in the
Expand Down Expand Up @@ -655,23 +655,23 @@ tile_row_to_zmm!(
/// and gates the INT8 ops. This struct answers "which ops exist once I may".
#[derive(Copy, Clone, Debug, PartialEq, Eq, Default)]
pub struct AmxFeatures {
/// AMX-TILE (7.0:EDX[24]).
/// AMX-TILE (7.0:EDX\[24\]).
pub tile: bool,
/// AMX-INT8 (7.0:EDX[25]).
/// AMX-INT8 (7.0:EDX\[25\]).
pub int8: bool,
/// AMX-BF16 (7.0:EDX[22]).
/// AMX-BF16 (7.0:EDX\[22\]).
pub bf16: bool,
/// AMX-FP16 (7.1:EAX[21]).
/// AMX-FP16 (7.1:EAX\[21\]).
pub fp16: bool,
/// AMX-COMPLEX (7.1:EDX[8]).
/// AMX-COMPLEX (7.1:EDX\[8\]).
pub complex: bool,
/// AMX-FP8 (1E.1:EAX[4]).
/// AMX-FP8 (1E.1:EAX\[4\]).
pub fp8: bool,
/// AMX-TF32 (1E.1:EAX[6]; the bit LLVM used before dropping the feature).
/// AMX-TF32 (1E.1:EAX\[6\]; the bit LLVM used before dropping the feature).
pub tf32: bool,
/// AMX-AVX512 (1E.1:EAX[7]).
/// AMX-AVX512 (1E.1:EAX\[7\]).
pub avx512: bool,
/// AMX-MOVRS (1E.1:EAX[8]).
/// AMX-MOVRS (1E.1:EAX\[8\]).
pub movrs: bool,
}

Expand Down
2 changes: 1 addition & 1 deletion src/hpc/audio/mdct.rs
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ pub const FRAME_SIZE: usize = 960;
pub const MDCT_SIZE: usize = FRAME_SIZE / 2;

/// Sine window for MDCT (Opus uses a sine window for CELT mode).
/// w[n] = sin(π/N × (n + 0.5))
/// w\[n\] = sin(π/N × (n + 0.5))
pub fn sine_window(n: usize) -> Vec<f32> {
(0..n)
.map(|i| (PI / n as f32 * (i as f32 + 0.5)).sin())
Expand Down
6 changes: 3 additions & 3 deletions src/hpc/audio/modes.rs
Original file line number Diff line number Diff line change
Expand Up @@ -290,9 +290,9 @@ impl OctaveBand {
/// Build from a fundamental frequency.
///
/// The pattern captures the harmonic envelope at that frequency:
/// pattern[0] = fundamental energy weight
/// pattern[1] = 2nd harmonic weight
/// pattern[2] = 3rd harmonic weight
/// pattern\[0\] = fundamental energy weight
/// pattern\[1\] = 2nd harmonic weight
/// pattern\[2\] = 3rd harmonic weight
///
/// The harmonic decay rate determines voice character:
/// steep decay → flute/sine (pure tone)
Expand Down
2 changes: 1 addition & 1 deletion src/hpc/audio/voice.rs
Original file line number Diff line number Diff line change
Expand Up @@ -84,7 +84,7 @@ impl VoiceArchetype {
/// via strided sampling + quantization.
///
/// The stride determines which embedding dimensions map to which channels:
/// dim[0], dim[stride], dim[2*stride], ... → channels 0..15
/// dim\[0\], dim\[stride\], dim\[2*stride\], ... → channels 0..15
pub fn from_embedding(embedding: &[f32], stride: usize) -> Self {
let mut channels = [0i8; N_VOICE_CHANNELS];

Expand Down
6 changes: 3 additions & 3 deletions src/hpc/bgz17_bridge.rs
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
//! Bridge between ndarray Fingerprint<256> (2KB) and Base17 (34 bytes).
//!
//! Converts flat 16384-bit fingerprint planes to i16[17] base patterns
//! Converts flat 16384-bit fingerprint planes to i16\[17\] base patterns
//! using golden-step octave averaging.
//!
//! This is a self-contained port of the bgz17 crate's `base17` module,
Expand Down Expand Up @@ -376,7 +376,7 @@ impl Base17 {
/// Byte size of serialized form.
pub const BYTE_SIZE: usize = BASE_DIM * 2; // 34

/// Encode i8[16384] accumulator into a Base17 pattern.
/// Encode i8\[16384\] accumulator into a Base17 pattern.
///
/// For each of 17 base dimensions, averages the accumulator values at
/// golden-step-selected positions across all octaves, then scales by
Expand Down Expand Up @@ -527,7 +527,7 @@ impl SpoBase17 {
/// Byte size of serialized form.
pub const BYTE_SIZE: usize = Base17::BYTE_SIZE * 3; // 102

/// Encode three i8[16384] accumulator planes.
/// Encode three i8\[16384\] accumulator planes.
pub fn encode(s: &[i8], p: &[i8], o: &[i8]) -> Self {
SpoBase17 {
subject: Base17::encode(s),
Expand Down
8 changes: 4 additions & 4 deletions src/hpc/blas_level1.rs
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ use crate::imp_prelude::*;
/// assert!((x.blas_dot(&y) - 32.0).abs() < 1e-5);
/// ```
pub trait BlasLevel1<A> {
/// Dot product: Σ x[i] * y[i]
/// Dot product: Σ x\[i\] * y\[i\]
fn blas_dot(&self, other: &Self) -> A;

/// AXPY: self = alpha * other + self (modifies self in-place)
Expand All @@ -28,10 +28,10 @@ pub trait BlasLevel1<A> {
/// Scale: self = alpha * self
fn blas_scal(&mut self, alpha: A);

/// L2 norm: sqrt(Σ self[i]²)
/// L2 norm: sqrt(Σ self\[i\]²)
fn blas_nrm2(&self) -> A;

/// L1 norm (absolute sum): Σ |self[i]|
/// L1 norm (absolute sum): Σ |self\[i\]|
fn blas_asum(&self) -> A;

/// Index of maximum absolute value element.
Expand Down Expand Up @@ -118,7 +118,7 @@ where

/// Givens rotation parameters.
///
/// Returned by [`blas_rotg`](GivensRotation::blas_rotg). Contains the cosine
/// Returned by [`blas_rotg`]. Contains the cosine
/// and sine of the rotation that zeroes out the second component.
#[derive(Clone, Copy, Debug)]
pub struct GivensRotation<A> {
Expand Down
Loading
Loading