Skip to content

Commit ba9326d

Browse files
authored
Merge pull request #356 from AdaWorldAPI/ccr-0455e606-wmtsor
D-KEY16 W1+W2: parity for the stride-16 key match, and a pricing probe
2 parents e6d1280 + 93121cb commit ba9326d

3 files changed

Lines changed: 488 additions & 6 deletions

File tree

‎Cargo.toml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -71,6 +71,10 @@ required-features = ["std"]
7171
name = "ternlog_fold_probe"
7272
required-features = ["std"]
7373

74+
[[example]]
75+
name = "key16_match_probe"
76+
required-features = ["std"]
77+
7478
[[example]]
7579
name = "perm_field_probe"
7680
required-features = ["std"]

‎crates/simd-masking-parity/src/lib.rs‎

Lines changed: 144 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,9 @@
2828
//! index-addressed `masked_group_sum_i32_via` (two-hop zero-fallback); `0xD4x`
2929
//! `eq_u32_via_to_mask` (the same index lane, packed as a predicate rather than
3030
//! folded into a sum); `0xExx` the `F64x8` lane compares (all six relations
31-
//! over every pair of 16 IEEE edge values: NaN, ±0, ±inf, subnormal); `0xFxx` the 16-lane byte vectors `I8x16` / `U8x16` (wrapping
31+
//! over every pair of 16 IEEE edge values: NaN, ±0, ±inf, subnormal); `0x73x` / `0x74x` / `0x750` the strided 16-byte register match (all sixteen
32+
//! bytes; strides 16 / 20 / 32 / 512 from first offsets 0..15, care bits in
33+
//! bytes 12..16); `0xFxx` the 16-lane byte vectors `I8x16` / `U8x16` (wrapping
3234
//! `add`/`sub`, signed/unsigned `min`/`max`, the load/store round trips). `main.rs` (native / qemu) and
3335
//! `selfcheck()` (the wasm cdylib export, driven by `run.mjs`) both call
3436
//! [`run`].
@@ -44,13 +46,13 @@ use ndarray::simd::{
4446
mask_xor_assign, masked_group_sum_i32, masked_group_sum_i32_via, masked_key_run_count_u32, masked_max_i32,
4547
masked_min_i32, masked_strided_group_sum, masked_sum_i32, masked_sum_wrapping_add_i32, ne_i32_to_mask,
4648
ne_i32_to_mask_under, ne_u32_to_mask, ne_u32_to_mask_under, ne_u64_to_mask, ne_u8_to_mask,
47-
ternary_match_strided_to_mask, ternary_match_u32_to_mask, ternary_match_u32_to_mask_under,
48-
ternary_match_u64_to_mask, ternary_match_u64_to_mask_under, ternlog, F64x8, I32x16, I8x16, KeyRunCarry, MortonDir,
49-
U32x16, U64x8, U8x16,
49+
ternary_match_strided16_to_mask, ternary_match_strided_to_mask, ternary_match_u32_to_mask,
50+
ternary_match_u32_to_mask_under, ternary_match_u64_to_mask, ternary_match_u64_to_mask_under, ternlog, F64x8,
51+
I32x16, I8x16, KeyRunCarry, MortonDir, U32x16, U64x8, U8x16,
5052
};
5153

5254
/// Number of check groups [`run`] executes (for the log line only).
53-
pub const CHECKS: usize = 15;
55+
pub const CHECKS: usize = 16;
5456

5557
/// The wasm export: identical to [`run`], `extern "C"` so `run.mjs` can call it.
5658
#[no_mangle]
@@ -64,7 +66,7 @@ pub fn run() -> u32 {
6466
check_ternlog_all_tables, check_u64x8_algebra, check_i32x16_compare, check_predicates_to_mask,
6567
check_mask_algebra, check_care_match, check_masked_reductions, check_blend, check_morton_shift,
6668
check_predicates_under, check_set_range, check_unsigned_compare_to_mask, check_gather_scatter_group,
67-
check_f64x8_compare, check_i8x16_u8x16_lanes,
69+
check_f64x8_compare, check_i8x16_u8x16_lanes, check_care_match16,
6870
];
6971
for g in groups {
7072
if let Err(code) = g() {
@@ -1019,6 +1021,142 @@ fn check_care_match() -> Result<(), u32> {
10191021
Ok(())
10201022
}
10211023

1024+
// ── 0x73x / 0x74x / 0x750: the strided 16-byte register match ───────────────
1025+
//
1026+
// `ternary_match_strided16_to_mask` reads ALL sixteen bytes of each record in
1027+
// place. The 12-byte check above cannot see bytes 12..16, so this group pins
1028+
// them: strides 16 / 20 / 32 / 512, every first offset 0..15 (so every
1029+
// alignment of the record start), the tail lengths 0 / 1 / 3 / 4 / 5 / 15 /
1030+
// 16 / 17 / 63 / 64 / 65 / 130, and care masks that put their bits in bytes
1031+
// 12..16 alone and in the middle of the register. The buffer is sized to the
1032+
// last record's last byte and not one byte more, so any read past the final
1033+
// record faults instead of passing on slack. The reference is bytewise.
1034+
1035+
/// Strides under test: the packed Dn128 lane, an odd non-multiple of 8, a
1036+
/// power of two above 16, and the wide row.
1037+
const STRIDES16: [usize; 4] = [16, 20, 32, 512];
1038+
1039+
/// Row counts under test: the empty lane, every sub-group tail, a whole group,
1040+
/// and the 64-row word boundary either side.
1041+
const TAILS16: [usize; 12] = [0, 1, 3, 4, 5, 15, 16, 17, 63, 64, 65, 130];
1042+
1043+
/// Bytewise definition of the match: record at byte `o` hits iff every byte
1044+
/// `k < 16` satisfies `(reg[k] ^ pattern[k]) & care[k] == 0`.
1045+
fn strided16_hit(bytes: &[u8], o: usize, pattern: &[u8; 16], care: &[u8; 16]) -> bool {
1046+
(0..16).all(|k| (bytes[o + k] ^ pattern[k]) & care[k] == 0)
1047+
}
1048+
1049+
fn check_care_match16() -> Result<(), u32> {
1050+
let mut rng = SplitMix64(0x7000_0000_0016);
1051+
let mut pattern = [0u8; 16];
1052+
for b in pattern.iter_mut() {
1053+
*b = rng.next() as u8;
1054+
}
1055+
let single = |k: usize| {
1056+
let mut c = [0u8; 16];
1057+
c[k] = 0xFF;
1058+
c
1059+
};
1060+
let mut tail4 = [0u8; 16];
1061+
tail4[12..16].fill(0xFF);
1062+
let mut boundary = [0u8; 16];
1063+
boundary[7] = 0x80; // bit 63
1064+
boundary[8] = 0x01; // bit 64
1065+
let mut cares: Vec<[u8; 16]> =
1066+
vec![[0u8; 16], [0xFFu8; 16], single(12), single(13), single(14), single(15), tail4, single(0), boundary];
1067+
for _ in 0..3 {
1068+
let mut c = [0u8; 16];
1069+
for b in c.iter_mut() {
1070+
*b = rng.next() as u8;
1071+
}
1072+
cares.push(c);
1073+
}
1074+
1075+
let (mut hits, mut misses) = (0usize, 0usize);
1076+
for &stride in &STRIDES16 {
1077+
for off in 0..16usize {
1078+
for &n in &TAILS16 {
1079+
let len = if n == 0 { off } else { off + (n - 1) * stride + 16 };
1080+
let out_len = words_for(n) + 1;
1081+
for (ci, care) in cares.iter().enumerate() {
1082+
// Filler everywhere (record gaps included), then the records.
1083+
let mut bytes = vec![0u8; len];
1084+
for b in bytes.iter_mut() {
1085+
*b = rng.next() as u8;
1086+
}
1087+
for i in 0..n {
1088+
let o = off + i * stride;
1089+
for k in 0..16 {
1090+
let noise = !care[k] & (rng.next() as u8);
1091+
bytes[o + k] = match i % 4 {
1092+
// hit: equal on every cared bit, noise elsewhere
1093+
0 | 3 => pattern[k] ^ noise,
1094+
// near-miss: a hit with ONE cared bit flipped,
1095+
// in a byte that walks 12, 13, 14, 15
1096+
1 => {
1097+
let hit = pattern[k] ^ noise;
1098+
if k == 12 + (i / 4) % 4 && care[k] != 0 {
1099+
hit ^ (care[k] & care[k].wrapping_neg())
1100+
} else {
1101+
hit
1102+
}
1103+
}
1104+
_ => rng.next() as u8,
1105+
};
1106+
}
1107+
}
1108+
let mut out = vec![u64::MAX; out_len];
1109+
ternary_match_strided16_to_mask(&bytes, off, stride, n, &pattern, care, &mut out);
1110+
let want = reference_mask(n, out_len, |i| strided16_hit(&bytes, off + i * stride, &pattern, care));
1111+
if out != want {
1112+
return Err(0x730 | ci as u32);
1113+
}
1114+
let h = want.iter().map(|w| w.count_ones() as usize).sum::<usize>();
1115+
hits += h;
1116+
misses += n - h;
1117+
}
1118+
}
1119+
}
1120+
}
1121+
// Anti-vacuity: the sweep must have produced both hits and misses.
1122+
if hits == 0 || misses == 0 {
1123+
return Err(0x750);
1124+
}
1125+
1126+
// Each of bytes 12..16 alone decides the answer: every record equals the
1127+
// pattern except that the chosen rows differ in byte `k` only, and the
1128+
// expected mask is written out here, not derived from the matcher.
1129+
for k in 12..16usize {
1130+
let care = single(k);
1131+
for &stride in &STRIDES16 {
1132+
for off in 0..16usize {
1133+
for &n in TAILS16.iter().filter(|&&n| n > 0) {
1134+
let mut bytes = vec![0u8; off + (n - 1) * stride + 16];
1135+
for b in bytes.iter_mut() {
1136+
*b = rng.next() as u8;
1137+
}
1138+
let flipped = [n - 1, n / 2];
1139+
for i in 0..n {
1140+
let o = off + i * stride;
1141+
bytes[o..o + 16].copy_from_slice(&pattern);
1142+
if flipped.contains(&i) {
1143+
bytes[o + k] ^= 0x40;
1144+
}
1145+
}
1146+
let out_len = words_for(n) + 1;
1147+
let mut out = vec![u64::MAX; out_len];
1148+
ternary_match_strided16_to_mask(&bytes, off, stride, n, &pattern, &care, &mut out);
1149+
let want = reference_mask(n, out_len, |i| !flipped.contains(&i));
1150+
if out != want {
1151+
return Err(0x740 | (k - 12) as u32);
1152+
}
1153+
}
1154+
}
1155+
}
1156+
}
1157+
Ok(())
1158+
}
1159+
10221160
// ── 0x8xx: masked reductions ────────────────────────────────────────────────
10231161

10241162
fn check_masked_reductions() -> Result<(), u32> {

0 commit comments

Comments
 (0)