simd: U64x8::transpose8 — the 8x8 u64 cross-lane transpose, all six backends

out[i] lane j == rows[j] lane i. The one physical routing step a lane-wise
consumer cannot express as a coordinate map: argon2's row pass -> column pass
(a column G reads words that live in eight different lanes).

AVX-512: 8 unpack + 16 vshufi64x2 (24). AVX2: four 4x4 unpack/vperm2i128
blocks; off-diagonal blocks swap by destination choice only. NEON / wasm:
32 vtrn1q/vtrn2q or i64x2.shuffle on 2x2 blocks, block placement is
renaming. Scalar / nightly: the index map.

Tests: facade test on 64 distinct words (fixture asserted non-symmetric so
identity fails; transpose is an involution); NEON (qemu) and wasm (node)
parity arms gain check 0x310.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_019HnekoM1EidTwQLS3oFVFm

Author: claude

crates/neon-simd-parity/src/main.rs

@@ -380,6 +380,16 @@ mod checks {
                 return Err(0x30F);
             }
         }
+        // transpose8: out[i] lane j == rows[j] lane i, on 64 distinct words.
+        let rows: [[u64; 8]; 8] = core::array::from_fn(|r| core::array::from_fn(|c| (r * 100 + c) as u64));
+        let t = U64x8::transpose8(rows.map(U64x8::from_array)).map(|v| v.to_array());
+        for i in 0..8 {
+            for j in 0..8 {
+                if t[i][j] != rows[j][i] {
+                    return Err(0x310);
+                }
+            }
+        }
         Ok(())
     }
 

crates/wasm-simd-parity/src/lib.rs

@@ -361,6 +361,16 @@ fn check_u64x8_algebra() -> Result<(), u32> {
             return Err(0x30F);
         }
     }
+    // transpose8: out[i] lane j == rows[j] lane i, on 64 distinct words.
+    let rows: [[u64; 8]; 8] = core::array::from_fn(|r| core::array::from_fn(|c| (r * 100 + c) as u64));
+    let t = U64x8::transpose8(rows.map(U64x8::from_array)).map(|v| v.to_array());
+    for i in 0..8 {
+        for j in 0..8 {
+            if t[i][j] != rows[j][i] {
+                return Err(0x310);
+            }
+        }
+    }
     Ok(())
 }
 

src/simd.rs

@@ -1044,6 +1044,33 @@ mod tests {
         }
     }
 
+    /// `U64x8::transpose8` against its definition, `out[i][j] == rows[j][i]`.
+    ///
+    /// Every one of the 64 input words is distinct (row·100 + lane), so a
+    /// transpose that drops, duplicates or misroutes any word fails, and the
+    /// fixture is asserted not to be symmetric, so an identity "transpose"
+    /// fails too. Transposing twice must return the input.
+    #[test]
+    fn u64x8_transpose8_matches_the_index_map() {
+        use super::U64x8;
+
+        let rows_arr: [[u64; 8]; 8] = core::array::from_fn(|r| core::array::from_fn(|c| (r * 100 + c) as u64));
+        assert!(
+            (0..8).any(|r| (0..8).any(|c| rows_arr[r][c] != rows_arr[c][r])),
+            "fixture is symmetric, so identity would pass"
+        );
+        let rows = rows_arr.map(U64x8::from_array);
+
+        let out = U64x8::transpose8(rows).map(|v| v.to_array());
+        for i in 0..8 {
+            for j in 0..8 {
+                assert_eq!(out[i][j], rows_arr[j][i], "out[{i}] lane {j}");
+            }
+        }
+        let back = U64x8::transpose8(U64x8::transpose8(rows)).map(|v| v.to_array());
+        assert_eq!(back, rows_arr, "transpose is not an involution");
+    }
+
     /// The BLAKE3 shuffle surface on `U32x16`, checked against the REAL x86
     /// intrinsics it reproduces — applied to each 256-bit half.
     ///

src/simd_avx2.rs

@@ -1688,6 +1688,59 @@ impl U64x8 {
         // values only.
         unsafe { Self::from_avx2_halves(_mm256_mul_epu32(a_lo, b_lo), _mm256_mul_epu32(a_hi, b_hi)) }
     }
+
+    /// 8×8 transpose of `u64` words across eight registers:
+    /// `out[i]` lane `j` == `rows[j]` lane `i`.
+    ///
+    /// This is a *physical* cross-lane move, for the case where a lane-wise
+    /// consumer genuinely needs the other orientation (argon2's row pass →
+    /// column pass: a column `G` reads words that live in eight different
+    /// lanes). Where a consumer can read the other orientation by index
+    /// instead, prefer that; this is the materialization step, not the model.
+    ///
+    /// Four 4×4 `unpack` + `vperm2i128` blocks, 8 shuffles each.
+    #[inline(always)]
+    pub fn transpose8(rows: [Self; 8]) -> [Self; 8] {
+        // Four 4×4 blocks. Block (bi, bj) is input rows 4bi.. , 256-bit half bj;
+        // it lands transposed in output rows 4bj.. , half bi. The off-diagonal
+        // blocks swapping places is only a choice of destination, not a move.
+        let halves = rows.map(|v| v.avx2_halves());
+        // Every (row, half) slot is overwritten below; `halves` only seeds the array.
+        let mut out = halves;
+        for bi in 0..2 {
+            for bj in 0..2 {
+                let h = |k: usize| {
+                    if bj == 0 {
+                        halves[4 * bi + k].0
+                    } else {
+                        halves[4 * bi + k].1
+                    }
+                };
+                // SAFETY: as `avx2_halves` — AVX2 present on any host this arm
+                // runs on; register ops only.
+                let c = unsafe {
+                    let t0 = _mm256_unpacklo_epi64(h(0), h(1));
+                    let t1 = _mm256_unpackhi_epi64(h(0), h(1));
+                    let t2 = _mm256_unpacklo_epi64(h(2), h(3));
+                    let t3 = _mm256_unpackhi_epi64(h(2), h(3));
+                    [
+                        _mm256_permute2x128_si256::<0x20>(t0, t2),
+                        _mm256_permute2x128_si256::<0x20>(t1, t3),
+                        _mm256_permute2x128_si256::<0x31>(t0, t2),
+                        _mm256_permute2x128_si256::<0x31>(t1, t3),
+                    ]
+                };
+                for k in 0..4 {
+                    if bi == 0 {
+                        out[4 * bj + k].0 = c[k];
+                    } else {
+                        out[4 * bj + k].1 = c[k];
+                    }
+                }
+            }
+        }
+        out.map(|(lo, hi)| Self::from_avx2_halves(lo, hi))
+    }
 }
 
 /// Lane-wise variable shifts for the mask family's word ops (the Morton hex

src/simd_avx512.rs

@@ -1805,6 +1805,59 @@ impl U64x8 {
         Self(unsafe { _mm512_mul_epu32(self.0, rhs.0) })
     }
 
+    /// 8×8 transpose of `u64` words across eight registers:
+    /// `out[i]` lane `j` == `rows[j]` lane `i`.
+    ///
+    /// This is a *physical* cross-lane move, for the case where a lane-wise
+    /// consumer genuinely needs the other orientation (argon2's row pass →
+    /// column pass: a column `G` reads words that live in eight different
+    /// lanes). Where a consumer can read the other orientation by index
+    /// instead, prefer that; this is the materialization step, not the model.
+    ///
+    /// 24 shuffles: 8 unpacks, then two rounds of 8 `vshufi64x2`.
+    #[inline(always)]
+    pub fn transpose8(rows: [Self; 8]) -> [Self; 8] {
+        // SAFETY: `Self` is a native `__m512i` and this arm is compiled only
+        // under the avx512f dispatch; every intrinsic below is a register op.
+        unsafe {
+            let r = rows.map(|v| v.0);
+            // Stage 1: interleave lane pairs. t[2p] = (r[2p]_2k, r[2p+1]_2k) in
+            // 128-bit chunk k; t[2p+1] the same for the odd lanes.
+            let t = [
+                _mm512_unpacklo_epi64(r[0], r[1]),
+                _mm512_unpackhi_epi64(r[0], r[1]),
+                _mm512_unpacklo_epi64(r[2], r[3]),
+                _mm512_unpackhi_epi64(r[2], r[3]),
+                _mm512_unpacklo_epi64(r[4], r[5]),
+                _mm512_unpackhi_epi64(r[4], r[5]),
+                _mm512_unpacklo_epi64(r[6], r[7]),
+                _mm512_unpackhi_epi64(r[6], r[7]),
+            ];
+            // Stage 2: gather 128-bit chunks {0,1} and {2,3} of row pairs.
+            let s = [
+                _mm512_shuffle_i64x2::<0x44>(t[0], t[2]),
+                _mm512_shuffle_i64x2::<0xEE>(t[0], t[2]),
+                _mm512_shuffle_i64x2::<0x44>(t[4], t[6]),
+                _mm512_shuffle_i64x2::<0xEE>(t[4], t[6]),
+                _mm512_shuffle_i64x2::<0x44>(t[1], t[3]),
+                _mm512_shuffle_i64x2::<0xEE>(t[1], t[3]),
+                _mm512_shuffle_i64x2::<0x44>(t[5], t[7]),
+                _mm512_shuffle_i64x2::<0xEE>(t[5], t[7]),
+            ];
+            // Stage 3: pick chunk k of all four row pairs -> column 2k / 2k+1.
+            [
+                Self(_mm512_shuffle_i64x2::<0x88>(s[0], s[2])),
+                Self(_mm512_shuffle_i64x2::<0x88>(s[4], s[6])),
+                Self(_mm512_shuffle_i64x2::<0xDD>(s[0], s[2])),
+                Self(_mm512_shuffle_i64x2::<0xDD>(s[4], s[6])),
+                Self(_mm512_shuffle_i64x2::<0x88>(s[1], s[3])),
+                Self(_mm512_shuffle_i64x2::<0x88>(s[5], s[7])),
+                Self(_mm512_shuffle_i64x2::<0xDD>(s[1], s[3])),
+                Self(_mm512_shuffle_i64x2::<0xDD>(s[5], s[7])),
+            ]
+        }
+    }
+
     #[inline(always)]
     pub fn splat(v: u64) -> Self {
         Self(unsafe { _mm512_set1_epi64(v as i64) })

src/simd_neon.rs

@@ -2837,6 +2837,38 @@ impl U64x8 {
         Self(core::array::from_fn(|p| unsafe { U64x2(vmull_u32(vmovn_u64(self.0[p].0), vmovn_u64(rhs.0[p].0))) }))
     }
 
+    /// 8×8 transpose of `u64` words across eight registers:
+    /// `out[i]` lane `j` == `rows[j]` lane `i`.
+    ///
+    /// This is a *physical* cross-lane move, for the case where a lane-wise
+    /// consumer genuinely needs the other orientation (argon2's row pass →
+    /// column pass: a column `G` reads words that live in eight different
+    /// lanes). Where a consumer can read the other orientation by index
+    /// instead, prefer that; this is the materialization step, not the model.
+    ///
+    /// 32 `vtrn1q`/`vtrn2q`, one per output pair.
+    #[inline(always)]
+    pub fn transpose8(rows: [Self; 8]) -> [Self; 8] {
+        // Each register is four `U64x2` pairs, so the 8×8 is a 4×4 grid of
+        // 2×2 blocks. Block (bi, bj) = pair bj of rows 2bi, 2bi+1; transposed
+        // with `vtrn1q`/`vtrn2q`, it becomes pair bi of rows 2bj, 2bj+1. Moving
+        // whole blocks is only a choice of destination.
+        core::array::from_fn(|i| {
+            let (bj, odd) = (i / 2, i % 2 == 1);
+            Self(core::array::from_fn(|bi| {
+                let (x, y) = (rows[2 * bi].0[bj].0, rows[2 * bi + 1].0[bj].0);
+                // SAFETY: NEON baseline; pure register ops on uint64x2_t.
+                U64x2(unsafe {
+                    if odd {
+                        vtrn2q_u64(x, y)
+                    } else {
+                        vtrn1q_u64(x, y)
+                    }
+                })
+            }))
+        })
+    }
+
     /// Lane-wise population count: `vcntq_u8` on the bytes, then the
     /// `vpaddlq_u8 → vpaddlq_u16 → vpaddlq_u32` widening-add ladder back to
     /// one count per u64 lane (0..=64).

src/simd_nightly/u_word_types.rs

@@ -72,6 +72,20 @@ impl U64x8 {
         Self((self.0 & lo) * (rhs.0 & lo))
     }
 
+    /// 8×8 transpose of `u64` words across eight registers:
+    /// `out[i]` lane `j` == `rows[j]` lane `i`.
+    ///
+    /// This is a *physical* cross-lane move, for the case where a lane-wise
+    /// consumer genuinely needs the other orientation (argon2's row pass →
+    /// column pass: a column `G` reads words that live in eight different
+    /// lanes). Where a consumer can read the other orientation by index
+    /// instead, prefer that; this is the materialization step, not the model.
+    #[inline(always)]
+    pub fn transpose8(rows: [Self; 8]) -> [Self; 8] {
+        let a = rows.map(|v| v.to_array());
+        core::array::from_fn(|i| Self::from_array(core::array::from_fn(|j| a[j][i])))
+    }
+
     #[inline(always)]
     pub fn splat(v: u64) -> Self {
         Self(u64x8::splat(v))

src/simd_scalar.rs

@@ -587,6 +587,22 @@ impl U64x8 {
         }
         Self::from_array(o)
     }
+
+    /// 8×8 transpose of `u64` words across eight registers:
+    /// `out[i]` lane `j` == `rows[j]` lane `i`.
+    ///
+    /// This is a *physical* cross-lane move, for the case where a lane-wise
+    /// consumer genuinely needs the other orientation (argon2's row pass →
+    /// column pass: a column `G` reads words that live in eight different
+    /// lanes). Where a consumer can read the other orientation by index
+    /// instead, prefer that; this is the materialization step, not the model.
+    ///
+    /// The index map itself; the reference the native arms are tested against.
+    #[inline(always)]
+    pub fn transpose8(rows: [Self; 8]) -> [Self; 8] {
+        let a = rows.map(|v| v.to_array());
+        core::array::from_fn(|i| Self::from_array(core::array::from_fn(|j| a[j][i])))
+    }
 }
 
 // I8/I16 SIMD types (scalar fallback)

src/simd_wasm.rs

@@ -1875,6 +1875,33 @@ pub mod wasm32_simd {
             }))
         }
 
+        /// 8×8 transpose of `u64` words across eight registers:
+        /// `out[i]` lane `j` == `rows[j]` lane `i`.
+        ///
+        /// This is a *physical* cross-lane move, for the case where a lane-wise
+        /// consumer genuinely needs the other orientation (argon2's row pass →
+        /// column pass: a column `G` reads words that live in eight different
+        /// lanes). Where a consumer can read the other orientation by index
+        /// instead, prefer that; this is the materialization step, not the model.
+        ///
+        /// 32 `i64x2.shuffle`s, one per output pair.
+        #[inline(always)]
+        pub fn transpose8(rows: [Self; 8]) -> [Self; 8] {
+            // Four `U64x2` pairs per register: a 4×4 grid of 2×2 blocks. Block
+            // (bi, bj) = pair bj of rows 2bi, 2bi+1 -> pair bi of rows 2bj, 2bj+1.
+            core::array::from_fn(|i| {
+                let (bj, odd) = (i / 2, i % 2 == 1);
+                Self(core::array::from_fn(|bi| {
+                    let (x, y) = (rows[2 * bi].0[bj].0, rows[2 * bi + 1].0[bj].0);
+                    U64x2(if odd {
+                        i64x2_shuffle::<1, 3>(x, y)
+                    } else {
+                        i64x2_shuffle::<0, 2>(x, y)
+                    })
+                }))
+            })
+        }
+
         /// Lane-wise population count: `i8x16_popcnt`, then the pairwise
         /// widening adds up to 32-bit halves, then the two halves of each u64
         /// summed (`u64x2_shr` 32 + masked add).