diff --git a/.claude/blackboard.md b/.claude/blackboard.md index d3709b09..4aad00ba 100644 --- a/.claude/blackboard.md +++ b/.claude/blackboard.md @@ -1,3 +1,17 @@ +## 2026-10-08 — `PillarReport::deferred` + `certified()`; `Cascade::query` documents that Stroke 1 can drop true hits + +- **Pillars:** Pillar-15/16/17 run no probe but report `passed = true`. A gate on `passed` counted them as certified. `PillarReport` now has `deferred: bool` (true only for those three) and `certified() = passed && !deferred`. `print()` shows `DEFERRED`. `passed` keeps its meaning, so existing callers do not change. + - Breaking for code that builds `PillarReport` with a struct literal: add `deferred: false`. All 18 in-tree literals are updated. + - Pinned by `deferred_pillars_are_not_certified`: 12–14 certified, 15–17 deferred, and `deferred` agrees with `n_paths == n_hops == 0` on every report. +- **Cascade:** `Cascade::query`'s Stroke 1 drops any candidate whose scaled prefix estimate exceeds `threshold + 3σ`. Nothing falls back to an exact check, so a hit within the threshold whose differing bits sit in the prefix is lost. This is how a statistical cascade works, so the behaviour stays. The doc now says recall is not guaranteed and that returned hits are exact. + - Pinned by `stroke1_drops_a_prefix_heavy_true_hit`: 400 bits in the prefix are dropped, the same 400 at the tail are kept, and 127 fillers keep the warm-up σ small. +- **Disable runs, both red, both restored:** + - `certified()` returning `passed`; + - Stroke 1 also admitting `d <= threshold`. +- **Also:** the `SplitMix64::new` doctest failed under `deny(warnings)` because of an unused `mut`. Fixed. +- **Not fixed (pre-existing, unrelated):** clippy errors in `tests/splat3d_correctness.rs` under `--tests`. +- **Source:** lance-graph board `2026-10-08-mexhat-bucket-cascade-probe.md`, which flagged both behaviours. + ## 2026-10-08 — `PowerSums::checked_affine`, `CrossPowerSums::checked_affine`: power sums of an affine image from the sums alone - **New** (both in `src/simd_masking_ops.rs`, reached through `ndarray::simd`): diff --git a/src/hpc/cascade.rs b/src/hpc/cascade.rs index 46b7e341..de2e34fc 100644 --- a/src/hpc/cascade.rs +++ b/src/hpc/cascade.rs @@ -266,6 +266,16 @@ impl Cascade { } /// Run the full 3-stroke cascade query. + /// + /// **Approximate: recall is not guaranteed.** Stroke 1 scales the prefix + /// distance by `vec_bytes / s1_bytes` and drops any candidate whose + /// estimate exceeds `threshold + 3σ`. A candidate whose differing bits sit + /// mostly in the prefix is dropped even when its full distance is within + /// `threshold`; nothing falls through to an exact check. Strokes 2 and 3 + /// are exact: every hit returned carries its true distance and is within + /// `threshold`. Vectors shorter than 128 bytes skip Stroke 1 and are exact. + /// For an exact scan, call [`crate::hpc::bitwise::hamming_distance_within`] + /// per candidate. pub fn query(&self, query: &[u8], database: &[u8], vec_bytes: usize, num_vectors: usize) -> Vec { assert_eq!(query.len(), vec_bytes); assert_eq!(database.len(), vec_bytes * num_vectors); @@ -790,6 +800,46 @@ mod tests { .collect() } + /// Stroke 1 is a statistical prune: a hit whose differences sit in the + /// prefix is dropped although its full distance is within the threshold. + /// The documented contract is "returned hits are exact", not "all hits + /// are returned". A change that makes Stroke 1 exact will flip this test. + #[test] + fn stroke1_drops_a_prefix_heavy_true_hit() { + let vec_bytes = 2048; + let threshold = 600; + let query: Vec = (0..vec_bytes).map(|i| (i as u8).wrapping_mul(37)).collect(); + let mut database = Vec::new(); + // Candidate 0: 400 differing bits, all in the stroke-1 prefix. + let mut v = query.clone(); + for bit in 0..400 { + v[bit / 8] ^= 1 << (bit % 8); + } + database.extend_from_slice(&v); + // Candidates 1..128: the same 400 bits, all at the tail. They fill the + // 128-row warmup so the population sigma stays small. + let mut w = query.clone(); + for bit in 0..400 { + w[vec_bytes - 1 - bit / 8] ^= 1 << (bit % 8); + } + let n = 128; + for _ in 1..n { + database.extend_from_slice(&w); + } + for c in database.chunks(vec_bytes) { + assert_eq!(bitwise::hamming_distance_raw(&query, c), 400); + } + let cascade = Cascade::from_threshold(threshold, vec_bytes); + let mut got: Vec<(usize, u64)> = cascade + .query(&query, &database, vec_bytes, n) + .iter() + .map(|r| (r.index, r.hamming)) + .collect(); + got.sort_unstable(); + let expected: Vec<(usize, u64)> = (1..n).map(|i| (i, 400)).collect(); + assert_eq!(got, expected, "Stroke 1 should drop candidate 0 and keep the rest"); + } + #[test] fn query_exact_strokes_respect_the_prefix_budget() { let (query, database, vec_bytes, expected) = straddling_database(); diff --git a/src/hpc/pillar/btsp_unbiased.rs b/src/hpc/pillar/btsp_unbiased.rs index bcd63405..bd4f6eed 100644 --- a/src/hpc/pillar/btsp_unbiased.rs +++ b/src/hpc/pillar/btsp_unbiased.rs @@ -124,6 +124,7 @@ pub fn prove_pillar_16() -> PillarReport { psd_rate: 0.0, lognorm_concentration: 0.0, passed: true, + deferred: true, } } diff --git a/src/hpc/pillar/cov_high_d.rs b/src/hpc/pillar/cov_high_d.rs index 6d565ba3..dcf48c61 100644 --- a/src/hpc/pillar/cov_high_d.rs +++ b/src/hpc/pillar/cov_high_d.rs @@ -417,6 +417,7 @@ pub fn prove_pillar_9() -> PillarReport { psd_rate: clt_rate, // repurposed: stores CLT convergence rate lognorm_concentration: clt_rate, passed, + deferred: false, } } diff --git a/src/hpc/pillar/ewa_sandwich_2d.rs b/src/hpc/pillar/ewa_sandwich_2d.rs index 58b7317a..71e0ea78 100644 --- a/src/hpc/pillar/ewa_sandwich_2d.rs +++ b/src/hpc/pillar/ewa_sandwich_2d.rs @@ -226,6 +226,7 @@ pub fn prove_pillar_6() -> PillarReport { psd_rate, lognorm_concentration, passed, + deferred: false, } } diff --git a/src/hpc/pillar/ewa_sandwich_3d.rs b/src/hpc/pillar/ewa_sandwich_3d.rs index d6a8605f..512f2e7d 100644 --- a/src/hpc/pillar/ewa_sandwich_3d.rs +++ b/src/hpc/pillar/ewa_sandwich_3d.rs @@ -192,6 +192,7 @@ pub fn prove_pillar_7() -> PillarReport { psd_rate, lognorm_concentration, passed, + deferred: false, } } diff --git a/src/hpc/pillar/hhtl_contraction.rs b/src/hpc/pillar/hhtl_contraction.rs index ee10e57c..1fdd9c74 100644 --- a/src/hpc/pillar/hhtl_contraction.rs +++ b/src/hpc/pillar/hhtl_contraction.rs @@ -343,6 +343,7 @@ pub fn prove_pillar_13() -> PillarReport { psd_rate, lognorm_concentration, passed, + deferred: false, } } diff --git a/src/hpc/pillar/koestenberger.rs b/src/hpc/pillar/koestenberger.rs index 0d338f5b..4f826cec 100644 --- a/src/hpc/pillar/koestenberger.rs +++ b/src/hpc/pillar/koestenberger.rs @@ -285,6 +285,7 @@ pub fn prove_pillar_7_5() -> PillarReport { psd_rate, lognorm_concentration: max_err, // repurpose field: tracks max path-parity error passed, + deferred: false, } } diff --git a/src/hpc/pillar/lattice_signature.rs b/src/hpc/pillar/lattice_signature.rs index df7bb4b7..19521d80 100644 --- a/src/hpc/pillar/lattice_signature.rs +++ b/src/hpc/pillar/lattice_signature.rs @@ -505,6 +505,7 @@ pub fn prove_pillar_11_lattice() -> PillarReport { psd_rate: 1.0 - f64::from(leg.reduced_merged) / f64::from(leg.reduced_checked.max(1)), lognorm_concentration: f64::from(leg.false_merge_max_sep_level), passed, + deferred: false, } } diff --git a/src/hpc/pillar/mexican_hat.rs b/src/hpc/pillar/mexican_hat.rs index 2cf7d803..aa37e74c 100644 --- a/src/hpc/pillar/mexican_hat.rs +++ b/src/hpc/pillar/mexican_hat.rs @@ -117,6 +117,7 @@ pub fn prove_pillar_15() -> PillarReport { psd_rate: 0.0, lognorm_concentration: 0.0, passed: true, + deferred: true, } } diff --git a/src/hpc/pillar/mod.rs b/src/hpc/pillar/mod.rs index 799d7ed1..4ff2bb92 100644 --- a/src/hpc/pillar/mod.rs +++ b/src/hpc/pillar/mod.rs @@ -168,7 +168,8 @@ pub use tree_balance::prove_pillar_17; /// for r in &reports { /// r.print(); /// } -/// assert!(reports.iter().all(|r| r.passed)); +/// // `passed` is also true for a deferred pillar; gate on `certified()`. +/// assert!(reports.iter().filter(|r| !r.deferred).all(|r| r.certified())); /// ``` pub fn run_substrate_tier() -> Vec { vec![ @@ -203,6 +204,28 @@ mod tests { } } + #[test] + fn deferred_pillars_are_not_certified() { + // 12-14 run a probe; 15-17 are placeholders. A gate that reads + // `passed` alone counts all six as certified. + let reports = run_substrate_tier(); + let certified: Vec = reports + .iter() + .filter(|r| r.certified()) + .map(|r| r.pillar_id) + .collect(); + let deferred: Vec = reports + .iter() + .filter(|r| r.deferred) + .map(|r| r.pillar_id) + .collect(); + assert_eq!(certified, vec![12, 13, 14]); + assert_eq!(deferred, vec![15, 16, 17]); + assert!(reports + .iter() + .all(|r| r.deferred == (r.n_paths == 0 && r.n_hops == 0))); + } + #[test] fn substrate_tier_is_deterministic() { // Two consecutive runs must produce identical reports. diff --git a/src/hpc/pillar/ogit_lattice.rs b/src/hpc/pillar/ogit_lattice.rs index 0caa7f14..80f545b4 100644 --- a/src/hpc/pillar/ogit_lattice.rs +++ b/src/hpc/pillar/ogit_lattice.rs @@ -336,6 +336,7 @@ pub fn prove_pillar_14() -> PillarReport { psd_rate, lognorm_concentration, passed, + deferred: false, } } diff --git a/src/hpc/pillar/pflug.rs b/src/hpc/pillar/pflug.rs index 9e552b3d..8e0d5990 100644 --- a/src/hpc/pillar/pflug.rs +++ b/src/hpc/pillar/pflug.rs @@ -178,6 +178,7 @@ pub fn prove_pillar_10() -> PillarReport { psd_rate: if passed { 1.0 } else { 0.0 }, lognorm_concentration, passed, + deferred: false, } } diff --git a/src/hpc/pillar/prove_runner.rs b/src/hpc/pillar/prove_runner.rs index caa75531..e6a1a238 100644 --- a/src/hpc/pillar/prove_runner.rs +++ b/src/hpc/pillar/prove_runner.rs @@ -14,7 +14,7 @@ //! let _v = rng.next_f32(); //! let report = PillarReport { //! pillar_id: 6, seed: SEED, n_paths: 10, n_hops: 5, -//! psd_rate: 1.0, lognorm_concentration: 0.0, passed: true, +//! psd_rate: 1.0, lognorm_concentration: 0.0, passed: true, deferred: false, //! }; //! report.print(); //! assert!(assert_psd_rate(10, 10, 0.999)); @@ -49,7 +49,7 @@ impl SplitMix64 { /// /// ```rust /// use ndarray::hpc::pillar::prove_runner::SplitMix64; - /// let mut rng = SplitMix64::new(0xDA5ADC5ADD); + /// let _rng = SplitMix64::new(0xDA5ADC5ADD); /// ``` #[inline] pub const fn new(seed: u64) -> Self { @@ -140,6 +140,7 @@ impl SplitMix64 { /// psd_rate: 0.9997, /// lognorm_concentration: 0.0023, /// passed: true, +/// deferred: false, /// }; /// r.print(); /// ``` @@ -159,18 +160,39 @@ pub struct PillarReport { /// Log-norm Frobenius concentration metric (pillar-specific semantics). pub lognorm_concentration: f64, /// `true` if all PASS criteria were met. + /// + /// A deferred pillar reports `passed = true` because the deferral is not + /// a failure. That makes `passed` alone unfit as a certification gate: + /// use [`PillarReport::certified`], which also requires that the probe ran. pub passed: bool, + /// `true` if no probe ran: the pillar is a placeholder waiting for the + /// kernel it certifies. Its numeric fields are zero and carry no evidence. + pub deferred: bool, } impl PillarReport { + /// `true` only if the probe ran and every PASS criterion held. + /// + /// This is the predicate a gate should use. `passed` is also `true` for a + /// deferred pillar, which measured nothing. + pub fn certified(&self) -> bool { + self.passed && !self.deferred + } + /// Pretty-print the report to stdout in a deterministic, grep-friendly format. /// /// Output format: /// ```text /// [PILLAR-7] seed=0xEDA5ADC5ADD paths=1000 hops=10 psd_rate=0.9997 lognorm_conc=0.0023 PASS /// ``` + /// + /// The status is `PASS`, `FAIL`, or `DEFERRED` (no probe ran). pub fn print(&self) { - let status = if self.passed { "PASS" } else { "FAIL" }; + let status = match (self.deferred, self.passed) { + (true, _) => "DEFERRED", + (false, true) => "PASS", + (false, false) => "FAIL", + }; // Use plain integer formatting for seed so the output is reproducible // across platforms (no locale-dependent float printing). println!( @@ -431,11 +453,13 @@ mod tests { psd_rate: 0.9997, lognorm_concentration: 0.001, passed: true, + deferred: false, }; assert!(r_pass.passed); let r_fail = PillarReport { passed: false, + deferred: false, ..r_pass.clone() }; assert!(!r_fail.passed); @@ -452,6 +476,7 @@ mod tests { psd_rate: 1.0, lognorm_concentration: 0.0, passed: true, + deferred: false, }; // Two prints of the same report produce the same line (determinism). // We can't easily capture stdout in no_std, so we just assert it diff --git a/src/hpc/pillar/signature.rs b/src/hpc/pillar/signature.rs index 7fff48f2..51b1224c 100644 --- a/src/hpc/pillar/signature.rs +++ b/src/hpc/pillar/signature.rs @@ -412,6 +412,7 @@ pub fn prove_pillar_11() -> PillarReport { psd_rate, lognorm_concentration: concentration, passed, + deferred: false, } } diff --git a/src/hpc/pillar/splat_invariants.rs b/src/hpc/pillar/splat_invariants.rs index c45b1ce8..41a99c12 100644 --- a/src/hpc/pillar/splat_invariants.rs +++ b/src/hpc/pillar/splat_invariants.rs @@ -335,6 +335,7 @@ pub fn prove_pillar_12() -> PillarReport { psd_rate, lognorm_concentration, passed, + deferred: false, } } diff --git a/src/hpc/pillar/temporal_sandwich.rs b/src/hpc/pillar/temporal_sandwich.rs index cdaff36d..a30fb37e 100644 --- a/src/hpc/pillar/temporal_sandwich.rs +++ b/src/hpc/pillar/temporal_sandwich.rs @@ -301,6 +301,7 @@ pub fn prove_pillar_8_band(band: MotionBand) -> PillarReport { psd_rate, lognorm_concentration, passed, + deferred: false, } } diff --git a/src/hpc/pillar/tree_balance.rs b/src/hpc/pillar/tree_balance.rs index 8160c6e9..5e316448 100644 --- a/src/hpc/pillar/tree_balance.rs +++ b/src/hpc/pillar/tree_balance.rs @@ -118,6 +118,7 @@ pub fn prove_pillar_17() -> PillarReport { psd_rate: 0.0, lognorm_concentration: 0.0, passed: true, + deferred: true, } }