Merge pull request #346 from AdaWorldAPI/ccr-b2e415d9-4jfvyk-deferred-pillars

Flag deferred pillars; document that Cascade Stroke 1 can drop true hits

Author: AdaWorldAPI

.claude/blackboard.md

@@ -1,3 +1,17 @@
+## 2026-10-08 — `PillarReport::deferred` + `certified()`; `Cascade::query` documents that Stroke 1 can drop true hits
+
+- **Pillars:** Pillar-15/16/17 run no probe but report `passed = true`. A gate on `passed` counted them as certified. `PillarReport` now has `deferred: bool` (true only for those three) and `certified() = passed && !deferred`. `print()` shows `DEFERRED`. `passed` keeps its meaning, so existing callers do not change.
+  - Breaking for code that builds `PillarReport` with a struct literal: add `deferred: false`. All 18 in-tree literals are updated.
+  - Pinned by `deferred_pillars_are_not_certified`: 12–14 certified, 15–17 deferred, and `deferred` agrees with `n_paths == n_hops == 0` on every report.
+- **Cascade:** `Cascade::query`'s Stroke 1 drops any candidate whose scaled prefix estimate exceeds `threshold + 3σ`. Nothing falls back to an exact check, so a hit within the threshold whose differing bits sit in the prefix is lost. This is how a statistical cascade works, so the behaviour stays. The doc now says recall is not guaranteed and that returned hits are exact.
+  - Pinned by `stroke1_drops_a_prefix_heavy_true_hit`: 400 bits in the prefix are dropped, the same 400 at the tail are kept, and 127 fillers keep the warm-up σ small.
+- **Disable runs, both red, both restored:**
+  - `certified()` returning `passed`;
+  - Stroke 1 also admitting `d <= threshold`.
+- **Also:** the `SplitMix64::new` doctest failed under `deny(warnings)` because of an unused `mut`. Fixed.
+- **Not fixed (pre-existing, unrelated):** clippy errors in `tests/splat3d_correctness.rs` under `--tests`.
+- **Source:** lance-graph board `2026-10-08-mexhat-bucket-cascade-probe.md`, which flagged both behaviours.
+
 ## 2026-10-08 — `PowerSums::checked_affine`, `CrossPowerSums::checked_affine`: power sums of an affine image from the sums alone
 
 - **New** (both in `src/simd_masking_ops.rs`, reached through `ndarray::simd`):

src/hpc/cascade.rs

@@ -266,6 +266,16 @@ impl Cascade {
     }
 
     /// Run the full 3-stroke cascade query.
+    ///
+    /// **Approximate: recall is not guaranteed.** Stroke 1 scales the prefix
+    /// distance by `vec_bytes / s1_bytes` and drops any candidate whose
+    /// estimate exceeds `threshold + 3σ`. A candidate whose differing bits sit
+    /// mostly in the prefix is dropped even when its full distance is within
+    /// `threshold`; nothing falls through to an exact check. Strokes 2 and 3
+    /// are exact: every hit returned carries its true distance and is within
+    /// `threshold`. Vectors shorter than 128 bytes skip Stroke 1 and are exact.
+    /// For an exact scan, call [`crate::hpc::bitwise::hamming_distance_within`]
+    /// per candidate.
     pub fn query(&self, query: &[u8], database: &[u8], vec_bytes: usize, num_vectors: usize) -> Vec<RankedHit> {
         assert_eq!(query.len(), vec_bytes);
         assert_eq!(database.len(), vec_bytes * num_vectors);
@@ -790,6 +800,46 @@ mod tests {
             .collect()
     }
 
+    /// Stroke 1 is a statistical prune: a hit whose differences sit in the
+    /// prefix is dropped although its full distance is within the threshold.
+    /// The documented contract is "returned hits are exact", not "all hits
+    /// are returned". A change that makes Stroke 1 exact will flip this test.
+    #[test]
+    fn stroke1_drops_a_prefix_heavy_true_hit() {
+        let vec_bytes = 2048;
+        let threshold = 600;
+        let query: Vec<u8> = (0..vec_bytes).map(|i| (i as u8).wrapping_mul(37)).collect();
+        let mut database = Vec::new();
+        // Candidate 0: 400 differing bits, all in the stroke-1 prefix.
+        let mut v = query.clone();
+        for bit in 0..400 {
+            v[bit / 8] ^= 1 << (bit % 8);
+        }
+        database.extend_from_slice(&v);
+        // Candidates 1..128: the same 400 bits, all at the tail. They fill the
+        // 128-row warmup so the population sigma stays small.
+        let mut w = query.clone();
+        for bit in 0..400 {
+            w[vec_bytes - 1 - bit / 8] ^= 1 << (bit % 8);
+        }
+        let n = 128;
+        for _ in 1..n {
+            database.extend_from_slice(&w);
+        }
+        for c in database.chunks(vec_bytes) {
+            assert_eq!(bitwise::hamming_distance_raw(&query, c), 400);
+        }
+        let cascade = Cascade::from_threshold(threshold, vec_bytes);
+        let mut got: Vec<(usize, u64)> = cascade
+            .query(&query, &database, vec_bytes, n)
+            .iter()
+            .map(|r| (r.index, r.hamming))
+            .collect();
+        got.sort_unstable();
+        let expected: Vec<(usize, u64)> = (1..n).map(|i| (i, 400)).collect();
+        assert_eq!(got, expected, "Stroke 1 should drop candidate 0 and keep the rest");
+    }
+
     #[test]
     fn query_exact_strokes_respect_the_prefix_budget() {
         let (query, database, vec_bytes, expected) = straddling_database();

src/hpc/pillar/btsp_unbiased.rs

@@ -124,6 +124,7 @@ pub fn prove_pillar_16() -> PillarReport {
         psd_rate: 0.0,
         lognorm_concentration: 0.0,
         passed: true,
+        deferred: true,
     }
 }
 

src/hpc/pillar/cov_high_d.rs

@@ -417,6 +417,7 @@ pub fn prove_pillar_9() -> PillarReport {
         psd_rate: clt_rate, // repurposed: stores CLT convergence rate
         lognorm_concentration: clt_rate,
         passed,
+        deferred: false,
     }
 }
 

src/hpc/pillar/ewa_sandwich_2d.rs

@@ -226,6 +226,7 @@ pub fn prove_pillar_6() -> PillarReport {
         psd_rate,
         lognorm_concentration,
         passed,
+        deferred: false,
     }
 }
 

src/hpc/pillar/ewa_sandwich_3d.rs

@@ -192,6 +192,7 @@ pub fn prove_pillar_7() -> PillarReport {
         psd_rate,
         lognorm_concentration,
         passed,
+        deferred: false,
     }
 }
 

src/hpc/pillar/hhtl_contraction.rs

@@ -343,6 +343,7 @@ pub fn prove_pillar_13() -> PillarReport {
         psd_rate,
         lognorm_concentration,
         passed,
+        deferred: false,
     }
 }
 

src/hpc/pillar/koestenberger.rs

@@ -285,6 +285,7 @@ pub fn prove_pillar_7_5() -> PillarReport {
         psd_rate,
         lognorm_concentration: max_err, // repurpose field: tracks max path-parity error
         passed,
+        deferred: false,
     }
 }
 

src/hpc/pillar/lattice_signature.rs

@@ -505,6 +505,7 @@ pub fn prove_pillar_11_lattice() -> PillarReport {
         psd_rate: 1.0 - f64::from(leg.reduced_merged) / f64::from(leg.reduced_checked.max(1)),
         lognorm_concentration: f64::from(leg.false_merge_max_sep_level),
         passed,
+        deferred: false,
     }
 }
 

src/hpc/pillar/mexican_hat.rs

@@ -117,6 +117,7 @@ pub fn prove_pillar_15() -> PillarReport {
         psd_rate: 0.0,
         lognorm_concentration: 0.0,
         passed: true,
+        deferred: true,
     }
 }