From 55681c7d76e809885573c9ed8bb54b142f2f037d Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 12 Oct 2026 20:48:08 +0000 Subject: [PATCH 1/5] mask-risc: GatherChain, chained hops folded into one op MaskOp::GatherChain { lane, chain, under, dst } with Program::chains: dst[i] holds when the key lane[i] leads through every hop of the named chain, each hop's reached row live in its resident foreign plane, passing its HopTests read in place, and the hop's next lane keying the following hop. No mask over any hop table is built and no program feeds another; the op realises through ndarray::simd::mask_gather_chain_u32[_under]. Gather could only test a target through a plane over the target's table, so a depth-2 predicate needed a depth-1 mask built first and a Keep fed into another program's Gather, the shape the ScatterOrU32 survival condition forbids. - validate: chain exists, non-empty, <= MAX_CHAIN_HOPS hops, <= MAX_HOP_TESTS tests, next on every hop but the last, planes and U32 lanes long enough (ChainOutOfRange / ChainShape / LenMismatch), so the facade's shape panics are unreachable. - executor builds the facade chain on the stack (L1 holds; test). - oracle walks the chain from the IR, never through the facade. - tests/chain.rs: executor, oracle and a fixture row loop agree, gated and ungated, across tile boundaries; one bare hop equals Gather; eight refusals identical on both paths. - exhaustive matches in quack and glove-parity extended. Needs AdaWorldAPI/ndarray#358 (mask_gather_chain_u32). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_011BMJnDBAfzcuQh5HeT39Rh --- .../lance-graph-glove-parity/src/awareness.rs | 5 +- .../examples/program_collapse_probe.rs | 3 +- crates/lance-graph-mask-risc/src/exec.rs | 102 ++++- crates/lance-graph-mask-risc/src/ir.rs | 88 +++- crates/lance-graph-mask-risc/src/lib.rs | 7 +- crates/lance-graph-mask-risc/src/reference.rs | 115 +++++- crates/lance-graph-mask-risc/src/value.rs | 8 + crates/lance-graph-mask-risc/tests/chain.rs | 377 ++++++++++++++++++ .../lance-graph-mask-risc/tests/no_alloc.rs | 96 +++++ crates/lance-graph-quack/src/lib.rs | 7 +- 10 files changed, 776 insertions(+), 32 deletions(-) create mode 100644 crates/lance-graph-mask-risc/tests/chain.rs diff --git a/crates/lance-graph-glove-parity/src/awareness.rs b/crates/lance-graph-glove-parity/src/awareness.rs index 3189d130a..8ae58a097 100644 --- a/crates/lance-graph-glove-parity/src/awareness.rs +++ b/crates/lance-graph-glove-parity/src/awareness.rs @@ -352,6 +352,7 @@ pub fn op_name(op: &MaskOp) -> String { MaskOp::Not { .. } => "Not".into(), MaskOp::Ternlog { .. } => "Ternlog".into(), MaskOp::Gather { .. } => "Gather".into(), + MaskOp::GatherChain { .. } => "GatherChain".into(), } } @@ -1202,7 +1203,9 @@ fn gate_planes(p: &Program) -> Vec { }; for op in &p.ops { match *op { - MaskOp::Pred { under, .. } | MaskOp::Gather { under, .. } => { + MaskOp::Pred { under, .. } + | MaskOp::Gather { under, .. } + | MaskOp::GatherChain { under, .. } => { if let Some(u) = under { see(u); } diff --git a/crates/lance-graph-mask-risc/examples/program_collapse_probe.rs b/crates/lance-graph-mask-risc/examples/program_collapse_probe.rs index cc0af61ad..52a01adde 100644 --- a/crates/lance-graph-mask-risc/examples/program_collapse_probe.rs +++ b/crates/lance-graph-mask-risc/examples/program_collapse_probe.rs @@ -123,7 +123,8 @@ fn op_dst(op: &MaskOp) -> u16 { | MaskOp::AndNot { dst, .. } | MaskOp::Not { dst, .. } | MaskOp::Ternlog { dst, .. } - | MaskOp::Gather { dst, .. } => dst, + | MaskOp::Gather { dst, .. } + | MaskOp::GatherChain { dst, .. } => dst, } } diff --git a/crates/lance-graph-mask-risc/src/exec.rs b/crates/lance-graph-mask-risc/src/exec.rs index eacd37876..c5ba63be6 100644 --- a/crates/lance-graph-mask-risc/src/exec.rs +++ b/crates/lance-graph-mask-risc/src/exec.rs @@ -27,26 +27,28 @@ use ndarray::simd::{ eq_u32_to_mask_under, eq_u32_via_to_mask, ge_i32_to_mask, ge_i32_to_mask_under, gt_i32_to_mask, gt_i32_to_mask_under, le_i32_to_mask, le_i32_to_mask_under, lt_i32_to_mask, lt_i32_to_mask_under, mask_all, mask_and, mask_and_assign, mask_andnot, mask_andnot_assign, - mask_any, mask_gather_u32, mask_gather_u32_under, mask_not, mask_not_assign, mask_or, - mask_or_assign, mask_scatter_or_u32, mask_set_range, mask_xor, mask_xor_assign, - masked_group_count_u32, masked_group_count_u32_pair, masked_group_count_u32_via, - masked_group_cross_power_sums_i32, masked_group_cross_power_sums_i32_pair, - masked_group_cross_power_sums_i32_via, masked_group_max_i32, masked_group_max_i32_pair, - masked_group_max_i32_via, masked_group_min_i32, masked_group_min_i32_pair, - masked_group_min_i32_via, masked_group_power_sums_i32, masked_group_power_sums_i32_pair, - masked_group_power_sums_i32_via, masked_group_sum_i32, masked_group_sum_i32_via, - masked_group_sum_sym_i32, masked_group_sum_sym_i32_pair, masked_group_sum_sym_i32_via, - masked_key_run_count_u32, masked_max_i32, masked_min_i32, masked_strided_group_sum, - masked_sum_i32, ne_i32_to_mask, ne_i32_to_mask_under, ne_u32_to_mask, ne_u32_to_mask_under, - popcount_batch_u64, ternary_match_strided16_to_mask, ternary_match_strided_to_mask, - ternary_match_u32_to_mask, ternary_match_u32_to_mask_under, ternary_match_u64_to_mask, - ternary_match_u64_to_mask_under, CrossPowerSums, KeyRunCarry, PowerSums, + mask_any, mask_gather_chain_u32, mask_gather_chain_u32_under, mask_gather_u32, + mask_gather_u32_under, mask_not, mask_not_assign, mask_or, mask_or_assign, mask_scatter_or_u32, + mask_set_range, mask_xor, mask_xor_assign, masked_group_count_u32, masked_group_count_u32_pair, + masked_group_count_u32_via, masked_group_cross_power_sums_i32, + masked_group_cross_power_sums_i32_pair, masked_group_cross_power_sums_i32_via, + masked_group_max_i32, masked_group_max_i32_pair, masked_group_max_i32_via, + masked_group_min_i32, masked_group_min_i32_pair, masked_group_min_i32_via, + masked_group_power_sums_i32, masked_group_power_sums_i32_pair, masked_group_power_sums_i32_via, + masked_group_sum_i32, masked_group_sum_i32_via, masked_group_sum_sym_i32, + masked_group_sum_sym_i32_pair, masked_group_sum_sym_i32_via, masked_key_run_count_u32, + masked_max_i32, masked_min_i32, masked_strided_group_sum, masked_sum_i32, ne_i32_to_mask, + ne_i32_to_mask_under, ne_u32_to_mask, ne_u32_to_mask_under, popcount_batch_u64, + ternary_match_strided16_to_mask, ternary_match_strided_to_mask, ternary_match_u32_to_mask, + ternary_match_u32_to_mask_under, ternary_match_u64_to_mask, ternary_match_u64_to_mask_under, + ChainHop, ChainTest, CrossPowerSums, KeyRunCarry, PowerSums, }; use crate::ir::{ span_words, touched_words, Compiled, Foreign, FusedFold, FusedKeep, FusedTerminal, FusedTern2, - FusedTern3, FusedTernlog, GroupFold, GroupKey, LaneRef, Lowering, MaskOp, Operand, Planes, - Pred, Program, StridedRef, Terminal, Tern2Fold, Tern3In, FUSED_SLOT_CAP, MAX_SCRATCH_SLOTS, + FusedTern3, FusedTernlog, GroupFold, GroupKey, Hop, LaneRef, Lowering, MaskOp, Operand, Planes, + Pred, Program, StridedRef, Terminal, Tern2Fold, Tern3In, FUSED_SLOT_CAP, MAX_CHAIN_HOPS, + MAX_HOP_TESTS, MAX_SCRATCH_SLOTS, }; use crate::reference::{out_shape, validate, OutShape}; use crate::ternlog_dispatch::{ @@ -549,6 +551,55 @@ fn foreign_lane_u32<'a>(foreign: &Foreign<'a>, key: u16) -> &'a [u32] { } } +/// One [`MaskOp::GatherChain`] pass into `out` (one tile of it). The facade +/// chain is built on the stack — `validate` has bounded it by +/// [`MAX_CHAIN_HOPS`] and [`MAX_HOP_TESTS`] — so `execute` still allocates +/// nothing (L1). +fn run_gather_chain( + hops: &[Hop], + foreign: &Foreign<'_>, + index: &[u32], + under: Option<&[u64]>, + out: &mut [u64], +) { + let none = ChainTest { + lane: &[], + value: 0, + negate: false, + }; + let mut tests = [[none; MAX_HOP_TESTS]; MAX_CHAIN_HOPS]; + for (row, hop) in tests.iter_mut().zip(hops) { + for (slot, t) in row.iter_mut().zip(&hop.tests) { + *slot = ChainTest { + lane: foreign_lane_u32(foreign, t.lane), + value: t.value, + negate: t.negate, + }; + } + } + let empty = ChainHop { + live: &[], + rows: 0, + tests: &[], + next: None, + }; + let mut chain = [empty; MAX_CHAIN_HOPS]; + for ((slot, hop), row) in chain.iter_mut().zip(hops).zip(&tests) { + let fp = &foreign.planes[usize::from(hop.live)]; + *slot = ChainHop { + live: fp.words, + rows: fp.rows, + tests: &row[..hop.tests.len()], + next: hop.next.map(|k| foreign_lane_u32(foreign, k)), + }; + } + let chain = &chain[..hops.len()]; + match under { + None => mask_gather_chain_u32(index, chain, out), + Some(g) => mask_gather_chain_u32_under(index, g, chain, out), + } +} + /// One predicate pass into `dst` (one tile of it): the ungated facade member, /// or the `_under` member when a gate is present (cost then follows the /// gate's live words). @@ -1780,6 +1831,23 @@ pub fn execute_compiled( ), } } + // Same tile discipline as `Gather`: the key lane is this + // tile's rows, every hop's plane and lanes are read WHOLE. + MaskOp::GatherChain { + lane, + chain, + under, + dst, + } => { + let (d, rest) = scratch.split(dst); + run_gather_chain( + &program.chains[usize::from(chain)], + foreign, + lane_u32(planes, lane, t), + under.map(|u| read(planes, &rest, u, t)), + &mut d[..t.words], + ); + } } } @@ -2537,6 +2605,7 @@ mod tests { terminal: Terminal::Count { mask: Operand::Plane(0), }, + chains: Vec::new(), scratch_slots: MAX_SCRATCH_SLOTS + 1, }; let mut small = Scratch::new(2, 1); @@ -2574,6 +2643,7 @@ mod tests { terminal: Terminal::Count { mask: Operand::Plane(0), }, + chains: Vec::new(), scratch_slots: MAX_SCRATCH_SLOTS + 1, }; assert_eq!( diff --git a/crates/lance-graph-mask-risc/src/ir.rs b/crates/lance-graph-mask-risc/src/ir.rs index 069c45750..07266f82b 100644 --- a/crates/lance-graph-mask-risc/src/ir.rs +++ b/crates/lance-graph-mask-risc/src/ir.rs @@ -280,6 +280,69 @@ pub enum MaskOp { under: Option, dst: u16, }, + /// The CHAINED semijoin: chained hops are folded hops. `dst[i]` is set + /// when the key `lane[i]` leads through every hop of + /// [`Program::chains`]`[chain]`: at each hop the row reached is in + /// range, live in that hop's resident [`ForeignPlane`], and passes that + /// hop's [`HopTest`]s — read IN PLACE at that row — and the hop's `next` + /// lane names the following hop's row. + /// + /// No mask over any hop's table is built and no program feeds another: + /// each hop is one more aperture in the same pass, and the only + /// materialisation is this op's `dst` over THIS table's rows + /// (`ndarray::simd::mask_gather_chain_u32[_under]`). One hop with no + /// tests is exactly [`MaskOp::Gather`]. + /// + /// The fold is the reason this op exists: [`MaskOp::Gather`] can only + /// test a hop's target through a plane over that target's table, so a + /// predicate at depth 2 would need a mask over the depth-1 table built + /// first, and a program's `Keep` feeding another program's `Gather` — + /// the projection → population → projection shape the + /// [`Terminal::ScatterOrU32`] survival condition forbids. + /// + /// `under` gates it exactly as it gates [`MaskOp::Gather`], and may not + /// be `dst`. + GatherChain { + lane: u16, + chain: u16, + under: Option, + dst: u16, + }, +} + +/// The most hops one [`MaskOp::GatherChain`] may name. The executor builds +/// the chain on the stack (L1: `execute` never allocates), so the bound is +/// a capacity, not a policy; a longer chain is refused at validation. +pub const MAX_CHAIN_HOPS: usize = 8; + +/// The most [`HopTest`]s one [`Hop`] may carry — the same stack capacity. +pub const MAX_HOP_TESTS: usize = 8; + +/// One equality test a [`Hop`] applies to the row its key reached: +/// `foreign.lanes[lane][row] == value`, or `!=` when `negate`. `lane` must +/// be a `U32` lane of [`Foreign::lanes`] at least the hop's plane rows long. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct HopTest { + /// Index into [`Foreign::lanes`]. + pub lane: u16, + /// The value compared against. + pub value: u32, + /// Test `!=` instead of `==`. + pub negate: bool, +} + +/// One hop of a [`MaskOp::GatherChain`]: which foreign table the key lands +/// in, what must hold there, and which key leads on. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Hop { + /// Index into [`Foreign::planes`]: the hop table's resident validity, + /// whose `rows` is the hop table's row count. + pub live: u16, + /// Tests on the reached row, at most [`MAX_HOP_TESTS`]. + pub tests: Vec, + /// Index into [`Foreign::lanes`] (`U32`): the key into the next hop. + /// `Some` on every hop but the last, `None` on the last. + pub next: Option, } /// What the program produces. Exactly one per program; the mask it reads is @@ -711,6 +774,9 @@ pub struct Program { pub ops: Vec, /// The single result. pub terminal: Terminal, + /// The hop chains [`MaskOp::GatherChain`] names, by index. Empty for + /// every program without one. + pub chains: Vec>, /// How many scratch slots the program touches: `max slot + 1` over EVERY /// `Operand::Scratch` the program names — destinations, sources, a /// `Pred`'s gate, and the terminal's operands alike — sizing from @@ -790,7 +856,7 @@ impl Program { // `foreign` is not a `Scratch`/`Plane` operand — it indexes // `Foreign::planes`, a wholly separate address space this // touch-based accounting has no business over. - MaskOp::Gather { under, dst, .. } => { + MaskOp::Gather { under, dst, .. } | MaskOp::GatherChain { under, dst, .. } => { if let Some(u) = under { touch(u); } @@ -820,10 +886,20 @@ impl Program { Self { ops, terminal, + chains: Vec::new(), scratch_slots: slots, } } + /// [`Program::new`] with the hop chains its [`MaskOp::GatherChain`]s + /// name. + pub fn with_chains(ops: Vec, terminal: Terminal, chains: Vec>) -> Self { + Self { + chains, + ..Self::new(ops, terminal) + } + } + /// The fused lowering of this program, if its terminal can fold straight /// from its operands without any derived membership bits being written. /// @@ -1009,7 +1085,9 @@ impl Program { ); (apply_table(*imm, ta, tb, tc), *dst) } - MaskOp::Pred { .. } | MaskOp::Gather { .. } => return None, + MaskOp::Pred { .. } | MaskOp::Gather { .. } | MaskOp::GatherChain { .. } => { + return None + } }; *slots.get_mut(usize::from(dst))? = Some(t); } @@ -1155,7 +1233,9 @@ impl Program { ); (apply_table64(*imm, ta, tb, tc), *dst) } - MaskOp::Pred { .. } | MaskOp::Gather { .. } => return None, + MaskOp::Pred { .. } | MaskOp::Gather { .. } | MaskOp::GatherChain { .. } => { + return None + } }; *slots.get_mut(usize::from(dst))? = Some(t); } @@ -1337,7 +1417,7 @@ impl Program { | MaskOp::AndNot { .. } => h.two_input += 1, MaskOp::Not { .. } => h.not += 1, MaskOp::Ternlog { .. } => h.ternlog += 1, - MaskOp::Gather { .. } => h.gather += 1, + MaskOp::Gather { .. } | MaskOp::GatherChain { .. } => h.gather += 1, } } h diff --git a/crates/lance-graph-mask-risc/src/lib.rs b/crates/lance-graph-mask-risc/src/lib.rs index bf90bb0bf..e2306b311 100644 --- a/crates/lance-graph-mask-risc/src/lib.rs +++ b/crates/lance-graph-mask-risc/src/lib.rs @@ -128,9 +128,10 @@ pub use exec::{ pub use fuse::{fuse, fuse_program, ternlog_imm, BoolExpr, FuseError, Fused}; pub use ir::{ touched_words, Compiled, Foreign, ForeignPlane, FusedFold, FusedKeep, FusedTerminal, - FusedTern2, FusedTern3, FusedTernlog, GroupFold, GroupKey, LaneRef, Lowering, MaskOp, Operand, - Planes, Pred, Program, StridedRef, Terminal, Tern2Fold, Tern3In, Tern3Step, FUSED_SLOT_CAP, - GROUP_SUM_SYM_MAX_ROWS, MASKED_SUM_I32_MAX_ROWS, MAX_SCRATCH_SLOTS, + FusedTern2, FusedTern3, FusedTernlog, GroupFold, GroupKey, Hop, HopTest, LaneRef, Lowering, + MaskOp, Operand, Planes, Pred, Program, StridedRef, Terminal, Tern2Fold, Tern3In, Tern3Step, + FUSED_SLOT_CAP, GROUP_SUM_SYM_MAX_ROWS, MASKED_SUM_I32_MAX_ROWS, MAX_CHAIN_HOPS, MAX_HOP_TESTS, + MAX_SCRATCH_SLOTS, }; pub use reference::{ reference_execute, reference_execute_into, reference_scratch, reference_scratch_with_foreign, diff --git a/crates/lance-graph-mask-risc/src/reference.rs b/crates/lance-graph-mask-risc/src/reference.rs index 37338af0c..bcc2a7d69 100644 --- a/crates/lance-graph-mask-risc/src/reference.rs +++ b/crates/lance-graph-mask-risc/src/reference.rs @@ -17,8 +17,9 @@ //! path a consumer should reach for. use crate::ir::{ - Foreign, GroupFold, GroupKey, LaneRef, MaskOp, Operand, Planes, Pred, Program, Terminal, - GROUP_SUM_SYM_MAX_ROWS, MASKED_SUM_I32_MAX_ROWS, MAX_SCRATCH_SLOTS, + Foreign, GroupFold, GroupKey, Hop, LaneRef, MaskOp, Operand, Planes, Pred, Program, Terminal, + GROUP_SUM_SYM_MAX_ROWS, MASKED_SUM_I32_MAX_ROWS, MAX_CHAIN_HOPS, MAX_HOP_TESTS, + MAX_SCRATCH_SLOTS, }; use crate::value::{CrossPowerSums, PowerSums}; use crate::value::{ExecError, LaneKind, Out, Value}; @@ -286,10 +287,85 @@ fn dst_of(op: &MaskOp) -> u16 { | MaskOp::AndNot { dst, .. } | MaskOp::Not { dst, .. } | MaskOp::Ternlog { dst, .. } - | MaskOp::Gather { dst, .. } => dst, + | MaskOp::Gather { dst, .. } + | MaskOp::GatherChain { dst, .. } => dst, } } +/// Validate `p.chains[chain]` against `foreign`: the shape the facade +/// requires (non-empty, within the stack caps, `next` on every hop but the +/// last) and every plane and lane it names, at the lengths the facade +/// reads. Refusing here means the executor never reaches the facade's own +/// shape panics. +fn check_chain(p: &Program, foreign: &Foreign<'_>, chain: u16) -> Result<(), ExecError> { + let hops = p + .chains + .get(usize::from(chain)) + .ok_or(ExecError::ChainOutOfRange(chain))?; + if hops.is_empty() || hops.len() > MAX_CHAIN_HOPS { + return Err(ExecError::ChainShape { chain, hop: 0 }); + } + let last = hops.len() - 1; + for (h, hop) in hops.iter().enumerate() { + if hop.tests.len() > MAX_HOP_TESTS || hop.next.is_some() != (h < last) { + return Err(ExecError::ChainShape { chain, hop: h }); + } + let fp = foreign + .planes + .get(usize::from(hop.live)) + .ok_or(ExecError::ForeignOutOfRange(hop.live))?; + let want = words_for(fp.rows); + if fp.words.len() < want { + return Err(ExecError::LenMismatch { + what: "foreign", + expected: want, + found: fp.words.len(), + }); + } + for key in hop.tests.iter().map(|t| t.lane).chain(hop.next) { + check_foreign_lane(foreign, key, LaneKind::U32)?; + if let LaneRef::U32(l) = foreign.lanes[usize::from(key)] { + if l.len() < fp.rows { + return Err(ExecError::LenMismatch { + what: "chain lane", + expected: fp.rows, + found: l.len(), + }); + } + } + } + } + Ok(()) +} + +/// Whether the key `k` leads through every hop of `hops` — the row-loop +/// statement of [`MaskOp::GatherChain`], read straight off the IR so the +/// oracle never calls the facade it checks. Assumes [`check_chain`] passed. +fn chain_holds_at(hops: &[Hop], foreign: &Foreign<'_>, k: u32) -> bool { + let u32_lane = |key: u16| match foreign.lanes[usize::from(key)] { + LaneRef::U32(l) => l, + _ => &[][..], + }; + let mut cur = k as usize; + for hop in hops { + let fp = &foreign.planes[usize::from(hop.live)]; + if cur >= fp.rows || (fp.words[cur / 64] >> (cur % 64)) & 1 == 0 { + return false; + } + if hop + .tests + .iter() + .any(|t| (u32_lane(t.lane)[cur] == t.value) == t.negate) + { + return false; + } + if let Some(next) = hop.next { + cur = u32_lane(next)[cur] as usize; + } + } + true +} + /// The validation rules, in the order both the executor and the oracle apply /// them: (1) addressable scratch and plane counts; (2) plane and lane lengths, /// then a dirty plane tail; (3) every op in program order, operands in field @@ -462,6 +538,23 @@ pub(crate) fn validate( } check_operand(p, planes, Operand::Scratch(dst))?; } + MaskOp::GatherChain { + lane, + chain, + under, + dst, + } => { + if let Some(u) = under { + check_operand(p, planes, u)?; + if u == Operand::Scratch(dst) { + return Err(ExecError::GateAliasesDst { dst }); + } + written(u)?; + } + check_lane(planes, lane, LaneKind::U32)?; + check_chain(p, foreign, chain)?; + check_operand(p, planes, Operand::Scratch(dst))?; + } } written_slots.mark(dst_of(op)); } @@ -931,6 +1024,16 @@ fn run(p: &Program, planes: &Planes<'_>, foreign: &Foreign<'_>) -> Rows { && idx < fp.rows && (fp.words[idx / 64] >> (idx % 64)) & 1 == 1 } + MaskOp::GatherChain { + lane, chain, under, .. + } => { + under.is_none_or(|u| rows.bit(planes, u, row)) + && chain_holds_at( + &p.chains[usize::from(chain)], + foreign, + u32_at(planes, lane, row), + ) + } }; let dst = match *op { MaskOp::Pred { dst, .. } @@ -940,7 +1043,8 @@ fn run(p: &Program, planes: &Planes<'_>, foreign: &Foreign<'_>) -> Rows { | MaskOp::AndNot { dst, .. } | MaskOp::Not { dst, .. } | MaskOp::Ternlog { dst, .. } - | MaskOp::Gather { dst, .. } => dst, + | MaskOp::Gather { dst, .. } + | MaskOp::GatherChain { dst, .. } => dst, }; rows.set(dst, row, v); } @@ -1731,6 +1835,7 @@ mod tests { terminal: Terminal::Any { mask: Operand::Scratch(4), }, + chains: Vec::new(), scratch_slots: 1, }; assert_eq!( @@ -1868,6 +1973,7 @@ mod tests { let lying = Program { ops: vec![], terminal: Terminal::Any { mask: P0 }, + chains: Vec::new(), scratch_slots: MAX_SCRATCH_SLOTS + 1, }; // Anti-vacuity: nothing here NAMES a scratch slot, so the only @@ -1907,6 +2013,7 @@ mod tests { let exact = Program { ops: vec![], terminal: Terminal::Any { mask: P0 }, + chains: Vec::new(), scratch_slots: MAX_SCRATCH_SLOTS, }; assert!(reference_execute(&exact, planes, None).is_ok()); diff --git a/crates/lance-graph-mask-risc/src/value.rs b/crates/lance-graph-mask-risc/src/value.rs index fd3833147..d6e90adc1 100644 --- a/crates/lance-graph-mask-risc/src/value.rs +++ b/crates/lance-graph-mask-risc/src/value.rs @@ -182,6 +182,14 @@ pub enum ExecError { /// [`crate::MaskOp::Gather`]'s `foreign` names no entry of the caller's /// [`crate::Foreign::planes`]. ForeignOutOfRange(u16), + /// [`crate::MaskOp::GatherChain`]'s `chain` names no entry of + /// [`crate::Program::chains`]. + ChainOutOfRange(u16), + /// A chain that cannot run: empty, longer than + /// [`crate::MAX_CHAIN_HOPS`], a hop with more than + /// [`crate::MAX_HOP_TESTS`] tests, a missing `next` before the last hop + /// or a `next` on it. `chain` names the chain, `hop` the offending hop. + ChainShape { chain: u16, hop: usize }, /// [`crate::Terminal::GroupSumViaI32`]'s `key` names no entry of the /// caller's [`crate::Foreign::lanes`]. Distinct from [`Self::LaneOutOfRange`] /// — that one bounds a program's OWN `Planes::lanes`, this one bounds diff --git a/crates/lance-graph-mask-risc/tests/chain.rs b/crates/lance-graph-mask-risc/tests/chain.rs new file mode 100644 index 000000000..7fbe63f0b --- /dev/null +++ b/crates/lance-graph-mask-risc/tests/chain.rs @@ -0,0 +1,377 @@ +//! `MaskOp::GatherChain` — chained hops folded into one pass — against the +//! row-at-a-time oracle AND against an independent row loop written here +//! from the fixture's own lanes, so neither crate path grades itself. +//! +//! Shape: `line.partner -> partner.country -> country`, with a test at each +//! depth (`partner.status == 1`, `country.region != 2`). No mask over +//! `partner` or `country` is built anywhere in the executed program. + +use lance_graph_mask_risc::exec::{execute_into, Scratch}; +use lance_graph_mask_risc::reference::reference_execute_into; +use lance_graph_mask_risc::{ + scratch_words_for, words_for, ExecError, Foreign, ForeignPlane, Hop, HopTest, LaneRef, MaskOp, + Operand, Out, Planes, Pred, Program, Terminal, Value, MAX_CHAIN_HOPS, MAX_HOP_TESTS, +}; + +fn lcg(seed: &mut u64) -> u64 { + *seed = seed + .wrapping_mul(6364136223846793005) + .wrapping_add(1442695040888963407); + *seed >> 11 +} + +const S0: Operand = Operand::Scratch(0); +const S1: Operand = Operand::Scratch(1); + +/// Narrow tiles, so fixtures of a few thousand rows span several of them. +const TEST_TILE_WORDS: usize = 8; + +const ROWS: [usize; 6] = [1, 63, 64, 65, 700, 3000]; + +struct Fixture { + /// line → partner key (some out of range). + partner_of: Vec, + /// line's own status, for the gate. + line_status: Vec, + partner_rows: usize, + partner_live: Vec, + partner_status: Vec, + /// partner → country key (some out of range). + country_of: Vec, + country_rows: usize, + country_live: Vec, + country_region: Vec, +} + +fn live_bits(rows: usize, s: &mut u64) -> Vec { + let mut w: Vec = (0..words_for(rows)).map(|_| lcg(s) | lcg(s)).collect(); + if rows % 64 != 0 { + if let Some(last) = w.last_mut() { + *last &= (1u64 << (rows % 64)) - 1; + } + } + w +} + +impl Fixture { + fn new(n: usize, seed: u64) -> Self { + let mut s = seed ^ n as u64; + let (partner_rows, country_rows) = (90usize, 12usize); + let mut partner_of: Vec = (0..n).map(|_| (lcg(&mut s) % 95) as u32).collect(); + // A guaranteed out-of-range key at depth 1. + if let Some(last) = partner_of.last_mut() { + *last = 93; + } + let line_status = (0..n).map(|_| (lcg(&mut s) % 3) as u32).collect(); + let partner_status = (0..partner_rows) + .map(|_| (lcg(&mut s) % 2) as u32) + .collect(); + let mut country_of: Vec = (0..partner_rows) + .map(|_| (lcg(&mut s) % 14) as u32) + .collect(); + // A guaranteed out-of-range key at depth 2. + country_of[0] = 13; + let country_region = (0..country_rows) + .map(|_| (lcg(&mut s) % 3) as u32) + .collect(); + Self { + partner_of, + line_status, + partner_rows, + partner_live: live_bits(partner_rows, &mut s), + partner_status, + country_of, + country_rows, + country_live: live_bits(country_rows, &mut s), + country_region, + } + } + + /// The query, written as a row loop over the fixture's own lanes. + fn holds(&self, i: usize) -> bool { + let bit = |w: &[u64], k: usize| (w[k / 64] >> (k % 64)) & 1 == 1; + let p = self.partner_of[i] as usize; + if p >= self.partner_rows || !bit(&self.partner_live, p) || self.partner_status[p] != 1 { + return false; + } + let c = self.country_of[p] as usize; + c < self.country_rows && bit(&self.country_live, c) && self.country_region[c] != 2 + } +} + +fn chain() -> Vec { + vec![ + Hop { + live: 0, + tests: vec![HopTest { + lane: 0, + value: 1, + negate: false, + }], + next: Some(1), + }, + Hop { + live: 1, + tests: vec![HopTest { + lane: 2, + value: 2, + negate: true, + }], + next: None, + }, + ] +} + +fn run( + p: &Program, + planes: &Planes<'_>, + foreign: &Foreign<'_>, +) -> (Result, Result) { + let words = words_for(planes.n_rows).min(TEST_TILE_WORDS).max(1); + let slots = p.scratch_slots as usize; + let mut buf = vec![0u64; scratch_words_for(words, slots).expect("sized")]; + let mut scratch = Scratch::over(&mut buf, words, slots).expect("carves"); + ( + execute_into(p, planes, foreign, &mut scratch, Out::None), + reference_execute_into(p, planes, foreign, Out::None), + ) +} + +/// FAILS IF the executor or the oracle disagrees with the row loop on the +/// two-hop chain, gated or ungated, on any tile boundary. +/// Non-vacuous: the answer is neither empty nor every row, and both depths +/// reject something. +#[test] +fn a_two_hop_chain_matches_the_oracle_and_the_row_loop() { + let mut kept = 0usize; + let mut total = 0usize; + for &n in &ROWS { + let fx = Fixture::new(n, 0xC4A1); + let lanes = [LaneRef::U32(&fx.partner_of), LaneRef::U32(&fx.line_status)]; + let planes = Planes { + n_rows: n, + masks: &[], + lanes: &lanes, + }; + let fplanes = [ + ForeignPlane { + words: &fx.partner_live, + rows: fx.partner_rows, + }, + ForeignPlane { + words: &fx.country_live, + rows: fx.country_rows, + }, + ]; + let flanes = [ + LaneRef::U32(&fx.partner_status), + LaneRef::U32(&fx.country_of), + LaneRef::U32(&fx.country_region), + ]; + let foreign = Foreign { + planes: &fplanes, + lanes: &flanes, + }; + + let want = (0..n).filter(|&i| fx.holds(i)).count(); + let ungated = Program::with_chains( + vec![MaskOp::GatherChain { + lane: 0, + chain: 0, + under: None, + dst: 0, + }], + Terminal::Count { mask: S0 }, + vec![chain()], + ); + let (got, oracle) = run(&ungated, &planes, &foreign); + assert_eq!(got, Ok(Value::Count(want)), "n={n}: executor, ungated"); + assert_eq!(oracle, Ok(Value::Count(want)), "n={n}: oracle, ungated"); + + // Gate: line_status == 0, then the chain only where it holds. + let want_g = (0..n) + .filter(|&i| fx.line_status[i] == 0 && fx.holds(i)) + .count(); + let gated = Program::with_chains( + vec![ + MaskOp::Pred { + pred: Pred::EqU32 { lane: 1, v: 0 }, + under: None, + dst: 0, + }, + MaskOp::GatherChain { + lane: 0, + chain: 0, + under: Some(S0), + dst: 1, + }, + ], + Terminal::Count { mask: S1 }, + vec![chain()], + ); + let (got, oracle) = run(&gated, &planes, &foreign); + assert_eq!(got, Ok(Value::Count(want_g)), "n={n}: executor, gated"); + assert_eq!(oracle, Ok(Value::Count(want_g)), "n={n}: oracle, gated"); + + kept += want; + total += n; + } + assert!( + kept > 0 && kept * 3 < total, + "vacuous fixture: kept {kept} of {total}" + ); +} + +/// FAILS IF a chain of one hop with no tests is not exactly `Gather`. +#[test] +fn one_bare_hop_is_gather() { + let n = 700; + let fx = Fixture::new(n, 7); + let lanes = [LaneRef::U32(&fx.partner_of)]; + let planes = Planes { + n_rows: n, + masks: &[], + lanes: &lanes, + }; + let fplanes = [ForeignPlane { + words: &fx.partner_live, + rows: fx.partner_rows, + }]; + let foreign = Foreign { + planes: &fplanes, + lanes: &[], + }; + let chained = Program::with_chains( + vec![MaskOp::GatherChain { + lane: 0, + chain: 0, + under: None, + dst: 0, + }], + Terminal::Count { mask: S0 }, + vec![vec![Hop { + live: 0, + tests: vec![], + next: None, + }]], + ); + let plain = Program::new( + vec![MaskOp::Gather { + lane: 0, + foreign: 0, + under: None, + dst: 0, + }], + Terminal::Count { mask: S0 }, + ); + let (a, _) = run(&chained, &planes, &foreign); + let (b, _) = run(&plain, &planes, &foreign); + assert_eq!(a, b); + assert!(matches!(a, Ok(Value::Count(c)) if c > 0)); +} + +/// FAILS IF a malformed chain reaches the facade (whose shape checks panic) +/// instead of being refused, identically, by both paths. +#[test] +fn malformed_chains_are_refused_by_both_paths() { + let n = 65; + let fx = Fixture::new(n, 3); + let lanes = [LaneRef::U32(&fx.partner_of)]; + let planes = Planes { + n_rows: n, + masks: &[], + lanes: &lanes, + }; + let fplanes = [ + ForeignPlane { + words: &fx.partner_live, + rows: fx.partner_rows, + }, + ForeignPlane { + words: &fx.country_live, + rows: fx.country_rows, + }, + ]; + let short = vec![0u32; 3]; + let flanes = [ + LaneRef::U32(&fx.partner_status), + LaneRef::U32(&fx.country_of), + LaneRef::U32(&fx.country_region), + LaneRef::U32(&short), + ]; + let foreign = Foreign { + planes: &fplanes, + lanes: &flanes, + }; + let op = |chain| MaskOp::GatherChain { + lane: 0, + chain, + under: None, + dst: 0, + }; + let refused = |chains: Vec>, chain: u16, want: ExecError| { + let p = Program::with_chains(vec![op(chain)], Terminal::Count { mask: S0 }, chains); + let (got, oracle) = run(&p, &planes, &foreign); + assert_eq!(got, Err(want), "executor"); + assert_eq!(oracle, Err(want), "oracle"); + }; + let hop = |live, next| Hop { + live, + tests: vec![], + next, + }; + + refused(vec![chain()], 1, ExecError::ChainOutOfRange(1)); + refused(vec![vec![]], 0, ExecError::ChainShape { chain: 0, hop: 0 }); + // A hop before the last without its key, and a key after the last. + refused( + vec![vec![hop(0, None), hop(1, None)]], + 0, + ExecError::ChainShape { chain: 0, hop: 0 }, + ); + refused( + vec![vec![hop(0, Some(1)), hop(1, Some(1))]], + 0, + ExecError::ChainShape { chain: 0, hop: 1 }, + ); + refused( + vec![vec![hop(0, None); MAX_CHAIN_HOPS + 1]], + 0, + ExecError::ChainShape { chain: 0, hop: 0 }, + ); + refused( + vec![vec![Hop { + live: 0, + tests: vec![ + HopTest { + lane: 0, + value: 0, + negate: false + }; + MAX_HOP_TESTS + 1 + ], + next: None, + }]], + 0, + ExecError::ChainShape { chain: 0, hop: 0 }, + ); + refused(vec![vec![hop(2, None)]], 0, ExecError::ForeignOutOfRange(2)); + // A test lane shorter than its hop table. + refused( + vec![vec![Hop { + live: 0, + tests: vec![HopTest { + lane: 3, + value: 0, + negate: false, + }], + next: None, + }]], + 0, + ExecError::LenMismatch { + what: "chain lane", + expected: fx.partner_rows, + found: 3, + }, + ); +} diff --git a/crates/lance-graph-mask-risc/tests/no_alloc.rs b/crates/lance-graph-mask-risc/tests/no_alloc.rs index e865d9160..606664d9a 100644 --- a/crates/lance-graph-mask-risc/tests/no_alloc.rs +++ b/crates/lance-graph-mask-risc/tests/no_alloc.rs @@ -228,3 +228,99 @@ fn six_plane_recognition_allocates_nothing() { "six-plane recognition allocated {bytes} bytes over 100 runs" ); } + +/// FAILS IF: a `GatherChain` allocates — the executor builds the facade +/// chain on the stack, bounded by `MAX_CHAIN_HOPS` / `MAX_HOP_TESTS`, so a +/// `Vec` built per tile would show here. Two hops, a test at each, gated. +#[test] +fn a_gather_chain_allocates_nothing() { + use lance_graph_mask_risc::exec::execute_into; + use lance_graph_mask_risc::{Foreign, ForeignPlane, Hop, HopTest, Out}; + let _serial = SERIAL.lock().unwrap_or_else(|e| e.into_inner()); + let n = 65_536; + let mut seed = 11u64; + let key: Vec = (0..n).map(|_| (lcg(&mut seed) % 520) as u32).collect(); + let status: Vec = (0..n).map(|_| (lcg(&mut seed) % 2) as u32).collect(); + let (mid, far) = (500usize, 40usize); + let mid_live: Vec = (0..mid.div_ceil(64)).map(|_| lcg(&mut seed)).collect(); + let far_live: Vec = vec![u64::MAX]; + let mid_test: Vec = (0..mid).map(|_| (lcg(&mut seed) % 2) as u32).collect(); + let mid_next: Vec = (0..mid).map(|_| (lcg(&mut seed) % 45) as u32).collect(); + let far_test: Vec = (0..far).map(|_| (lcg(&mut seed) % 3) as u32).collect(); + let lanes = [LaneRef::U32(&key), LaneRef::U32(&status)]; + let planes = Planes { + n_rows: n, + masks: &[], + lanes: &lanes, + }; + let fplanes = [ + ForeignPlane { + words: &mid_live, + rows: mid, + }, + ForeignPlane { + words: &far_live, + rows: far, + }, + ]; + let flanes = [ + LaneRef::U32(&mid_test), + LaneRef::U32(&mid_next), + LaneRef::U32(&far_test), + ]; + let foreign = Foreign { + planes: &fplanes, + lanes: &flanes, + }; + let t = |lane, value, negate| HopTest { + lane, + value, + negate, + }; + let p = Program::with_chains( + vec![ + MaskOp::Pred { + pred: Pred::EqU32 { lane: 1, v: 1 }, + under: None, + dst: 0, + }, + MaskOp::GatherChain { + lane: 0, + chain: 0, + under: Some(Operand::Scratch(0)), + dst: 1, + }, + ], + Terminal::Count { + mask: Operand::Scratch(1), + }, + vec![vec![ + Hop { + live: 0, + tests: vec![t(0, 1, false)], + next: Some(1), + }, + Hop { + live: 1, + tests: vec![t(2, 2, true)], + next: None, + }, + ]], + ); + let mut scratch = Scratch::for_program(&p, n).expect("addressable"); + let warm = execute_into(&p, &planes, &foreign, &mut scratch, Out::None); + assert!( + matches!(warm, Ok(Value::Count(c)) if c > 0), + "vacuous: {warm:?}" + ); + let bytes = measure(|| { + for _ in 0..1000 { + let v = execute_into(&p, &planes, &foreign, &mut scratch, Out::None); + assert_eq!(v, warm); + } + }); + assert_eq!( + bytes, 0, + "GatherChain allocated {bytes} bytes over 1000 runs" + ); +} diff --git a/crates/lance-graph-quack/src/lib.rs b/crates/lance-graph-quack/src/lib.rs index 6866d3dab..13d6454bb 100644 --- a/crates/lance-graph-quack/src/lib.rs +++ b/crates/lance-graph-quack/src/lib.rs @@ -3143,9 +3143,10 @@ mod tests { MaskOp::Ternlog { a, b, c, .. } => { read_as_leaf |= a == gate || b == gate || c == gate; } - // `lane` and `foreign` are plain indices; the only operand a - // `Gather` reads is its gate, classified exactly as `Pred`'s. - MaskOp::Gather { under, .. } => match under { + // `lane`, `foreign` and `chain` are plain indices; the only + // operand a `Gather`/`GatherChain` reads is its gate, + // classified exactly as `Pred`'s. + MaskOp::Gather { under, .. } | MaskOp::GatherChain { under, .. } => match under { Some(Operand::Scratch(_)) => on_accumulator = true, Some(_) => {} None => all_gated = false, From 8c8154e3480376eec608e836ebaf056c0b5c66be Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 12 Oct 2026 20:51:04 +0000 Subject: [PATCH 2/5] quack: Filter::Chain lowers to one GatherChain MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Filter::Chain { fk, hops: Vec } — the chained semijoin, hops each a RESIDENT plane, in-place tests and the key onward. gate_walk turns it into Node::GatherChain; index_chains moves the hops into the program's chain table; both emitters (in place and fused) emit one MaskOp::GatherChain carrying the gate. NULL fk follows Semijoin (FALSE, not UNKNOWN). Test: two hops, a test at each depth, an out-of-range key and a dead partner row; both lowerings emit exactly one GatherChain with both hops and no Gather, and the count matches a per-row reading of the filter. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_011BMJnDBAfzcuQh5HeT39Rh --- crates/lance-graph-mask-risc/tests/chain.rs | 4 +- crates/lance-graph-quack/src/lib.rs | 285 +++++++++++++++++++- 2 files changed, 281 insertions(+), 8 deletions(-) diff --git a/crates/lance-graph-mask-risc/tests/chain.rs b/crates/lance-graph-mask-risc/tests/chain.rs index 7fbe63f0b..0002ad0b7 100644 --- a/crates/lance-graph-mask-risc/tests/chain.rs +++ b/crates/lance-graph-mask-risc/tests/chain.rs @@ -45,7 +45,7 @@ struct Fixture { fn live_bits(rows: usize, s: &mut u64) -> Vec { let mut w: Vec = (0..words_for(rows)).map(|_| lcg(s) | lcg(s)).collect(); - if rows % 64 != 0 { + if !rows.is_multiple_of(64) { if let Some(last) = w.last_mut() { *last &= (1u64 << (rows % 64)) - 1; } @@ -127,7 +127,7 @@ fn run( planes: &Planes<'_>, foreign: &Foreign<'_>, ) -> (Result, Result) { - let words = words_for(planes.n_rows).min(TEST_TILE_WORDS).max(1); + let words = words_for(planes.n_rows).clamp(1, TEST_TILE_WORDS); let slots = p.scratch_slots as usize; let mut buf = vec![0u64; scratch_words_for(words, slots).expect("sized")]; let mut scratch = Scratch::over(&mut buf, words, slots).expect("carves"); diff --git a/crates/lance-graph-quack/src/lib.rs b/crates/lance-graph-quack/src/lib.rs index 13d6454bb..2522af7c7 100644 --- a/crates/lance-graph-quack/src/lib.rs +++ b/crates/lance-graph-quack/src/lib.rs @@ -173,8 +173,8 @@ use std::cmp::Reverse; use lance_graph_contract::facet::{SemanticAperture, SemanticPrefix}; use lance_graph_contract::ordered_lane::{OrderedLaneWitness, SealedFacetLane}; use lance_graph_mask_risc::{ - fuse, BoolExpr, FuseError, GroupFold, GroupKey, MaskOp, Operand, Pred, Program, Terminal, - MAX_SCRATCH_SLOTS, + fuse, BoolExpr, FuseError, GroupFold, GroupKey, Hop, HopTest, MaskOp, Operand, Pred, Program, + Terminal, MAX_SCRATCH_SLOTS, }; /// A column reference — an index into [`Planes::lanes`](lance_graph_mask_risc::Planes). @@ -207,6 +207,32 @@ pub struct ForeignPlane(pub u16); #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] pub struct ForeignLane(pub u16); +/// One equality test a [`ChainHop`] applies to the row its key reached: +/// `lane[row] == value`, or `!=` when `negate`. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct ChainTest { + /// The hop table's value lane (`U32`). + pub lane: ForeignLane, + /// The value compared against. + pub value: u32, + /// Test `!=` instead of `==`. + pub negate: bool, +} + +/// One hop of a [`Filter::Chain`]: the RESIDENT validity of the table the key +/// lands in, the tests on the row reached, and the key into the next hop +/// (`None` on the last hop only). +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ChainHop { + /// The hop table's resident kept plane — the same precondition as + /// [`Filter::Semijoin`]'s `foreign`: never a produced mask. + pub live: ForeignPlane, + /// Tests on the reached row. + pub tests: Vec, + /// The hop table's key into the next hop (`U32`). + pub next: Option, +} + /// A predicate over one column — the filter's whole vocabulary. /// /// One variant per masking op that produces a mask from a value lane. That is @@ -390,6 +416,22 @@ pub enum Filter { /// Which foreign table's RESIDENT kept plane the fk resolves into. foreign: ForeignPlane, }, + /// The CHAINED semijoin — chained hops are folded hops. Holds when `fk` + /// leads through every hop: at each, the row reached is live in the hop's + /// resident plane and passes its tests, and the hop's `next` key names + /// the following hop's row. Lowered to ONE + /// [`MaskOp::GatherChain`]: no mask over any hop table is built, and the + /// tests at every depth are read in place at the row the key reached. + /// A single hop with no tests is exactly [`Filter::Semijoin`]. + /// + /// NULL `fk`: as [`Filter::Semijoin`] — `EXISTS` over a chain that starts + /// nowhere is FALSE, never UNKNOWN. + Chain { + /// The foreign-key column on THIS table. + fk: Col, + /// The hops, in order. + hops: Vec, + }, /// Every child must hold. And(Vec), /// Some child must hold. @@ -420,6 +462,14 @@ impl Filter { Filter::Semijoin { fk, foreign } } + /// The chained semijoin leaf — see [`Filter::Chain`]. + pub fn chain(fk: Col, hops: impl IntoIterator) -> Self { + Filter::Chain { + fk, + hops: hops.into_iter().collect(), + } + } + /// Conjunction. pub fn and(parts: impl IntoIterator) -> Self { Filter::And(parts.into_iter().collect()) @@ -581,7 +631,7 @@ impl Filter { let valid = leaf.leaf_validity(nullable)?; // `EXISTS` over a NULL fk is FALSE, not UNKNOWN: gate the // leaf, then complement the gated leaf for `F`. - if let Filter::Semijoin { .. } = leaf { + if let Filter::Semijoin { .. } | Filter::Chain { .. } = leaf { let gated = Filter::And(vec![Filter::Plane(valid), leaf.clone()]); return Some(gated.polar(want_true)); } @@ -607,7 +657,9 @@ impl Filter { let col = match *self { Filter::Plane(_) => return None, Filter::Cmp(col, _) => col, - Filter::EqU32Via { fk, .. } | Filter::Semijoin { fk, .. } => fk, + Filter::EqU32Via { fk, .. } + | Filter::Semijoin { fk, .. } + | Filter::Chain { fk, .. } => fk, Filter::And(_) | Filter::Or(_) | Filter::Not(_) => return None, }; nullable.iter().find(|(c, _)| *c == col).map(|&(_, m)| m) @@ -1091,6 +1143,8 @@ pub enum LowerError { /// the executor refuses a zero-length sink — so the plan would be /// unexecutable. Refused here, where the caller can see why. EmptyGroupUniverse, + /// More [`Filter::Chain`] leaves than a `u16` chain index can name. + TooManyChains, } impl core::fmt::Display for LowerError { @@ -1102,6 +1156,9 @@ impl core::fmt::Display for LowerError { LowerError::TooManySlots { needed } => { write!(f, "needs {needed} scratch slots; the address space is u16") } + LowerError::TooManyChains => { + write!(f, "more chained semijoins than a u16 chain index names") + } LowerError::GroupedBlend => { write!(f, "a blend writes the whole output; it cannot be grouped") } @@ -1954,10 +2011,31 @@ fn lower_with( emit: impl FnOnce(&Node, &mut Vec) -> Result, ) -> Result { let filter = fold_ranges(filter); - let (node, _) = gate_walk(&filter, None)?; + let (mut node, _) = gate_walk(&filter, None)?; + let mut chains = Vec::new(); + index_chains(&mut node, &mut chains)?; let mut ops = Vec::new(); let mask = emit(&node, &mut ops)?; - Ok(Program::new(ops, terminal_of(agg, mask))) + Ok(Program::with_chains(ops, terminal_of(agg, mask), chains)) +} + +/// Move every [`Node::GatherChain`]'s hops into the program's chain table +/// and leave the index behind, so both emitters only copy an index. +fn index_chains(n: &mut Node, chains: &mut Vec>) -> Result<(), LowerError> { + match n { + Node::GatherChain { chain, hops, .. } => { + *chain = u16::try_from(chains.len()).map_err(|_| LowerError::TooManyChains)?; + chains.push(core::mem::take(hops)); + } + Node::Not(inner) => index_chains(inner, chains)?, + Node::And(parts) | Node::Or(parts) => { + for p in parts { + index_chains(p, chains)?; + } + } + Node::Pred { .. } | Node::Plane(_) | Node::Gather { .. } => {} + } + Ok(()) } /// The filter after the survivor-skip pass, before any slot is assigned. @@ -1981,6 +2059,15 @@ enum Node { foreign: u16, under: Option, }, + /// The chained semijoin leaf ([`Filter::Chain`]), gated exactly as + /// `Gather` is. `hops` is moved into the program's chain table by + /// [`index_chains`] before emission, leaving `chain`. + GatherChain { + fk: u16, + chain: u16, + hops: Vec, + under: Option, + }, And(Vec), Or(Vec), Not(Box), @@ -2060,6 +2147,32 @@ fn gate_walk(f: &Filter, gate: Option) -> Result<(Node, bool), LowerError> }, gate.is_some(), ), + // A `Semijoin` with more apertures: the gate is baked into + // `MaskOp::GatherChain { under }` the same way. + Filter::Chain { fk, hops } => ( + Node::GatherChain { + fk: fk.0, + chain: 0, + hops: hops + .iter() + .map(|h| Hop { + live: h.live.0, + tests: h + .tests + .iter() + .map(|t| HopTest { + lane: t.lane.0, + value: t.value, + negate: t.negate, + }) + .collect(), + next: h.next.map(|k| k.0), + }) + .collect(), + under: gate, + }, + gate.is_some(), + ), // Gating passes THROUGH a negation (the rewrite is sound for any // remainder while the gate stays a leaf), but `!x` is 1 where the // gate is 0, so a negation never lets the gate be dropped. @@ -2264,6 +2377,18 @@ fn emit_gated( }); Ok(Operand::Scratch(dst)) } + Node::GatherChain { + fk, chain, under, .. + } => { + let gate = under.map(|m| Operand::Plane(m.0)).or(acc_gate); + ops.push(MaskOp::GatherChain { + lane: *fk, + chain: *chain, + under: gate, + dst, + }); + Ok(Operand::Scratch(dst)) + } Node::Not(inner) => { let a = emit_gated(inner, dst, acc_gate, ops)?; ops.push(MaskOp::Not { a, dst }); @@ -2358,6 +2483,20 @@ fn assign_slots(n: &Node, ops: &mut Vec) -> Result }); Ok(BoolExpr::Leaf(Operand::Scratch(dst))) } + Node::GatherChain { + fk, chain, under, .. + } => { + let dst = u16::try_from(ops.len()).map_err(|_| LowerError::TooManySlots { + needed: ops.len() + 1, + })?; + ops.push(MaskOp::GatherChain { + lane: *fk, + chain: *chain, + under: under.map(|m| Operand::Plane(m.0)), + dst, + }); + Ok(BoolExpr::Leaf(Operand::Scratch(dst))) + } Node::Not(inner) => Ok(BoolExpr::Not(Box::new(assign_slots(inner, ops)?))), Node::And(parts) | Node::Or(parts) => { let is_and = matches!(n, Node::And(_)); @@ -2804,6 +2943,27 @@ mod tests { let rows = self.foreign_rows[usize::from(foreign.0)]; idx < rows && (bits[idx / 64] >> (idx % 64)) & 1 == 1 } + Filter::Chain { fk, hops } => { + let mut cur = self.u32_at(*fk, row) as usize; + for h in hops { + let bits = &self.foreign[usize::from(h.live.0)]; + let rows = self.foreign_rows[usize::from(h.live.0)]; + if cur >= rows || (bits[cur / 64] >> (cur % 64)) & 1 == 0 { + return false; + } + let lane = |k: ForeignLane| &self.foreign_lanes[usize::from(k.0)]; + if h.tests + .iter() + .any(|t| (lane(t.lane)[cur] == t.value) == t.negate) + { + return false; + } + if let Some(k) = h.next { + cur = lane(k)[cur] as usize; + } + } + true + } Filter::And(ps) => ps.iter().all(|p| self.oracle(p, row)), Filter::Or(ps) => ps.iter().any(|p| self.oracle(p, row)), Filter::Not(p) => !self.oracle(p, row), @@ -4209,6 +4369,119 @@ mod tests { } } + /// FAILS IF: `Filter::Chain` disagrees with an independent per-row + /// reading in EITHER lowering, or lowers to anything but ONE + /// `GatherChain` carrying both hops — the fold. Shape: `CLASS` keys a + /// 5-row partner table (live, `status == 1`), whose `country` key leads + /// to a 4-row country table (live, `region != 2`), with an out-of-range + /// country key. Anti-vacuity: each depth rejects rows the other keeps, + /// and the chain is selective. + #[test] + fn chain_folds_to_one_gather_chain_and_matches_a_per_row_reading() { + let mut fx = Fx::new(N); + let (p_rows, c_rows) = (5usize, 4usize); + let p_live = plane(p_rows, |r| r != 4); + let c_live = plane(c_rows, |r| r != 1); + let partner = fx.push_foreign(p_live.clone(), p_rows); + let country = fx.push_foreign(c_live.clone(), c_rows); + let status = vec![1, 1, 0, 1, 1]; + let country_of = vec![0, 2, 3, 9, 0]; + let region = vec![0, 0, 2, 1]; + let status_l = fx.push_foreign_lane(status.clone()); + let next_l = fx.push_foreign_lane(country_of.clone()); + let region_l = fx.push_foreign_lane(region.clone()); + let hops = vec![ + ChainHop { + live: partner, + tests: vec![ChainTest { + lane: status_l, + value: 1, + negate: false, + }], + next: Some(next_l), + }, + ChainHop { + live: country, + tests: vec![ChainTest { + lane: region_l, + value: 2, + negate: true, + }], + next: None, + }, + ]; + let bare = Filter::chain(CLASS, hops.clone()); + // CLASS = i % 5: class 0 -> country 0 (kept); 1 -> country 2 + // (region 2, rejected at depth 2); 2 -> status 0 (rejected at + // depth 1); 3 -> country 9 (out of range); 4 -> partner dead. + let kept = fx.rows(&bare).len(); + assert_eq!(kept, N / 5, "only class 0 survives every hop"); + let one_hop = Filter::semijoin(CLASS, partner); + assert!( + fx.rows(&one_hop).len() > kept, + "depth 2 must reject rows depth 1 keeps" + ); + + let f = Filter::and([Filter::cmp(VALS, Cmp::GtI32(0)), bare]); + let expected = fx.rows(&f).len(); + assert!(expected > 0 && expected < kept); + let q = Query { + filter: f, + agg: Agg::Count, + }; + let fplanes = [ + lance_graph_mask_risc::ForeignPlane { + words: &p_live, + rows: p_rows, + }, + lance_graph_mask_risc::ForeignPlane { + words: &c_live, + rows: c_rows, + }, + ]; + let flanes = [ + lance_graph_mask_risc::LaneRef::U32(&status), + lance_graph_mask_risc::LaneRef::U32(&country_of), + lance_graph_mask_risc::LaneRef::U32(®ion), + ]; + let mask_risc_foreign = lance_graph_mask_risc::Foreign { + planes: &fplanes, + lanes: &flanes, + }; + for (label, program) in [ + ("in place", lower(&q).expect("lowers in place")), + ("fused", lower_fused(&q).expect("lowers fused")), + ] { + let chained = program + .ops + .iter() + .filter(|o| matches!(o, MaskOp::GatherChain { .. })) + .count(); + assert_eq!(chained, 1, "{label}: one folded op"); + assert!( + !program + .ops + .iter() + .any(|o| matches!(o, MaskOp::Gather { .. })), + "{label}: no per-hop gather" + ); + assert_eq!(program.chains.len(), 1, "{label}: one chain"); + assert_eq!(program.chains[0].len(), 2, "{label}: both hops in it"); + let mut scratch = Scratch::for_program(&program, fx.n()).expect("carves"); + let value = fx.with_planes(&[], |planes| { + lance_graph_mask_risc::execute_into( + &program, + planes, + &mask_risc_foreign, + &mut scratch, + lance_graph_mask_risc::Out::None, + ) + .expect("runs") + }); + assert_eq!(value, Value::Count(expected), "{label}: value"); + } + } + /// FAILS IF: `Agg::ScatterOrU32` disagrees with an independent per-row /// reading — CLASS is the fk, out_rows = 5 (the same "foreign" row /// space, addressed as a plain OWNED `Out::Mask` this time, not a From 6cbd6affcaa53bdc80d9689b32ae99e87fd7a79b Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 12 Oct 2026 20:51:35 +0000 Subject: [PATCH 3/5] board: D-CMH-G1-7 chained hops folded (entry + status row) Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_011BMJnDBAfzcuQh5HeT39Rh --- .claude/board/STATUS_BOARD.md | 3 +- .../entries/2026-10-12-chained-hops-folded.md | 32 +++++++++++++++++++ .claude/board/entries/README.md | 3 +- 3 files changed, 36 insertions(+), 2 deletions(-) create mode 100644 .claude/board/entries/2026-10-12-chained-hops-folded.md diff --git a/.claude/board/STATUS_BOARD.md b/.claude/board/STATUS_BOARD.md index cd01f268b..49754fd96 100644 --- a/.claude/board/STATUS_BOARD.md +++ b/.claude/board/STATUS_BOARD.md @@ -24,7 +24,8 @@ Plan: `.claude/plans/gremlin-composable-algebra-v1.md`. | D-CMH-G1-3 | Gremlin-shaped runtime declarations | Shipped (#1508) | loads after the build; effects refused at load | | D-CMH-G1-4a | functional relation carrier: `out(rel)`, `out(rel).has`, `count()` | Shipped (#1514; ConflictingUniverse fix follows) | row loop over the text at n ∈ {1,64,65,4097}; 6 disable runs red; R2IL refuses (no `GATHER`) | | D-CMH-G1-4b | edge table as the anchor (`E()`, M:N one hop, `outE/inV`) + `Fold::Distinct` (`out(rel).dedup().count()`) | In PR | bag and set == row loop over the text at n ∈ {1,64,65,4097}; unordered key lane refused (`LaneNotOrdered`); 7 disable runs red | -| D-CMH-G1-4 | rest: a second hop (blocked on the chained-hop decision), `V(parent).in(rel)` spelling, `bothE` | Queued | — | +| D-CMH-G1-7 | chained hops folded: `MaskOp::GatherChain` + Quack `Filter::Chain` (one op, tests at every depth, no hop mask) | In PR (needs ndarray#358) | executor == oracle == row loop, gated/ungated, tiles; 0-byte L1; 7 disable runs red | +| D-CMH-G1-4 | rest: a second hop (now D-CMH-G1-7 + glove lowering), `V(parent).in(rel)` spelling, `bothE` | Queued | — | | D-CMH-G1-5 | FirstOfRun | Queued | insufficiency proof first; Quack + R2IL vs independent oracle | | D-CMH-G1-6 | ordered `I32` conditions + `MIN`/`MAX` folds as method steps; R2IL `MIN`/`MAX` wired | In PR | both frontends == row loop incl. empty selection; 8 disable runs red | diff --git a/.claude/board/entries/2026-10-12-chained-hops-folded.md b/.claude/board/entries/2026-10-12-chained-hops-folded.md new file mode 100644 index 000000000..602482a50 --- /dev/null +++ b/.claude/board/entries/2026-10-12-chained-hops-folded.md @@ -0,0 +1,32 @@ +# 2026-10-12 — D-CMH-G1-7: chained hops are folded hops (mask-risc + Quack) + +**Status:** In PR (needs AdaWorldAPI/ndarray#358). **Plan:** `.claude/plans/gremlin-composable-algebra-v1.md`. + +`MaskOp::GatherChain { lane, chain, under, dst }` with `Program::chains`: +a key leads through every hop of the chain, each hop's reached row live in +its RESIDENT plane and passing its tests read in place, each hop's `next` +key naming the following hop's row. One op, one terminal, no mask over any +hop table, no program feeding another. Realised by +`ndarray::simd::mask_gather_chain_u32[_under]`. Quack's `Filter::Chain` +lowers to exactly one such op in both lowerings. + +WHY a new op: `Gather` tests a target only through a plane over the +target's table, so a depth-2 predicate needed a depth-1 mask built first +and a `Keep` fed into another program's `Gather`, the shape the +`ScatterOrU32` survival condition forbids. (#1521, which built that +per-hop shape, was closed.) + +MEASURED: `mask-risc/tests/chain.rs` (executor, oracle and a fixture row +loop agree gated and ungated across tile boundaries; one bare hop equals +`Gather`; eight refusals identical on both paths); `no_alloc.rs` (1000 +gated two-hop executes, 0 bytes); Quack +`chain_folds_to_one_gather_chain_and_matches_a_per_row_reading`. Seven +disable runs red (gate dropped, tests dropped, oracle negate, chain shape +check, lowering negate, lowering to `Gather`, oracle stopping at depth 1). + +DECISION: chains are stack-bounded (`MAX_CHAIN_HOPS = 8`, +`MAX_HOP_TESTS = 8`) so `execute` keeps L1 (no allocation). REVISIT WHEN: a +real query needs a deeper chain. + +OPEN: the glove/composer `out(r1).out(r2)` lowering (step 3); the M:N +two-edge-table hop (run-level OR) awaits a decision. diff --git a/.claude/board/entries/README.md b/.claude/board/entries/README.md index a7624e066..ce58a0fac 100644 --- a/.claude/board/entries/README.md +++ b/.claude/board/entries/README.md @@ -25,7 +25,7 @@ index row, (3) no duplicate entry id. Checks 1 and 2 are deliberately opposite directions; the stranding this convention prevents shows up in exactly one of them, never both. -319 entries, 2026-08-06 .. 2026-10-12. +320 entries, 2026-08-06 .. 2026-10-12. | date | entry id | finding | file | |---|---|---|---| @@ -38,6 +38,7 @@ exactly one of them, never both. | 2026-10-12 | `D-CMH-G1-4b` | | [2026-10-12-edge-table-anchor-distinct.md](2026-10-12-edge-table-anchor-distinct.md) | | 2026-10-12 | `D-CMH-0` | Class × Schema × Methods activation connects HotPlug, WideFieldMask, Quack and R2IL without a second registry | [2026-10-12-d-cmh-0-class-method-hotplug-activation.md](2026-10-12-d-cmh-0-class-method-hotplug-activation.md) | | 2026-10-12 | `D-CMH-1` | | [2026-10-12-class-method-polyfill-ledger.md](2026-10-12-class-method-polyfill-ledger.md) | +| 2026-10-12 | `D-CMH-G1-7` | | [2026-10-12-chained-hops-folded.md](2026-10-12-chained-hops-folded.md) | | 2026-10-12 | `D-HL-0` | CE64 hypothesis layers: plateau of the design (opaque LE bytes, 64 sparse-delta layers merged on read, Narsese-parallel validation), 8 design-vs-code conflicts, 9 unification hypotheses | [2026-10-12-ce64-hypothesis-layers-plateau.md](2026-10-12-ce64-hypothesis-layers-plateau.md) | | 2026-10-11 | `D-TELEPORT-0` | | [2026-10-11-teleport-probe-source-restriction.md](2026-10-11-teleport-probe-source-restriction.md) | | 2026-10-11 | `synergy-ledger-classid-hotplug` | | [2026-10-11-synergy-ledger-classid-hotplug.md](2026-10-11-synergy-ledger-classid-hotplug.md) | From 3937204cd1bf74b4a4a52e23b66b2d7af132d608 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 12 Oct 2026 20:58:29 +0000 Subject: [PATCH 4/5] glove: out(r1).out(r2) folds into one chained gather Path (1..=MAX_PATH relations, Copy) and Cond::ReachesPath / ViaPath for a path two relations or longer. Preparation walks each relation on the previous one's target glove; every condition on a maximal path and on its prefixes binds into ONE BoundCond::Chain (Filter::Chain, one MaskOp::GatherChain): hop k's live plane, the tests the conditions put on it, and the next key, which must be a proven Code field the call supplies as a foreign lane (NoForeignLane otherwise). One-relation paths no chain extends keep their semijoin / EqU32Via leaves unchanged. Every target universe along the path is checked at the call. R2IL names the missing GATHER. Gremlin: out after out extends the path (SecondHop is gone; PathTooDeep past MAX_PATH); has after it is a ViaPath; dedup, values, min/max after a path of two are refused. The text form and the awareness graph carry the path. tests/two_hop.rs: line -billed_to-> partner -located_in-> region against a row loop over the text at n in {1,64,65,4097}, tests at either hop, negated, with a local condition; the lowering is one chain with the hop-1 test folded in; composer, text form and awareness; refusals. The relation.rs SecondHop pin is re-pinned: a second hop now composes. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_011BMJnDBAfzcuQh5HeT39Rh --- .../lance-graph-glove-parity/src/awareness.rs | 28 + crates/lance-graph-glove-parity/src/decl.rs | 45 +- .../lance-graph-glove-parity/src/gremlin.rs | 64 +- crates/lance-graph-glove-parity/src/method.rs | 409 +++++++++-- .../tests/relation.rs | 13 +- .../lance-graph-glove-parity/tests/two_hop.rs | 657 ++++++++++++++++++ 6 files changed, 1117 insertions(+), 99 deletions(-) create mode 100644 crates/lance-graph-glove-parity/tests/two_hop.rs diff --git a/crates/lance-graph-glove-parity/src/awareness.rs b/crates/lance-graph-glove-parity/src/awareness.rs index 8ae58a097..7675c44e4 100644 --- a/crates/lance-graph-glove-parity/src/awareness.rs +++ b/crates/lance-graph-glove-parity/src/awareness.rs @@ -340,6 +340,15 @@ pub fn field_ident(concept: u16, seal: SchemaSeal, name: &str) -> String { format!("field:{concept:#06x}/{:016x}/{name}", seal.0) } +/// `out(a).out(b)…` for a relation path, as written. +fn outs(path: crate::method::Path<'_>) -> String { + path.rels() + .iter() + .map(|r| format!("out({r})")) + .collect::>() + .join(".") +} + /// The primitive kind of a mask op, by variant. #[must_use] pub fn op_name(op: &MaskOp) -> String { @@ -667,6 +676,25 @@ impl SoftwareGraph { self.link(m, Rel::Filters, n, Tier::Declared, m, prov); continue; } + Cond::ReachesPath { path } => { + let n = self.intern(Kind::Cond, format!("cond:{}@r{r}", outs(path))); + self.link(m, Rel::Filters, n, Tier::Declared, m, prov); + continue; + } + Cond::ViaPath { + path, + field, + literal, + negate, + } => { + let op = if negate { "!=" } else { "=" }; + let n = self.intern( + Kind::Cond, + format!("cond:{}.{field}{op}{literal}@r{r}", outs(path)), + ); + self.link(m, Rel::Filters, n, Tier::Declared, m, prov); + continue; + } }; let f = field(self, name); let n = self.intern(Kind::Cond, format!("cond:{f}{op}{lit}@r{r}")); diff --git a/crates/lance-graph-glove-parity/src/decl.rs b/crates/lance-graph-glove-parity/src/decl.rs index 6c60531b4..5c4f8b803 100644 --- a/crates/lance-graph-glove-parity/src/decl.rs +++ b/crates/lance-graph-glove-parity/src/decl.rs @@ -32,7 +32,7 @@ use lance_graph_ogar::ogar_vocab::canonical_concept_id; use crate::gremlin::{compose, GremlinRefusal, Pred, Step}; use crate::method::{ Cond, EffectClass, Fold, Glove, Law, MethodBody, MethodKey, MethodRef, MethodSpec, MethodTable, - Order, ReadingId, SchemaSeal, + Order, Path, ReadingId, SchemaSeal, }; /// A method declared in text: a [`MethodSpec`] with owned storage. @@ -73,6 +73,21 @@ enum DeclCond { literal: String, negate: bool, }, + /// `out(r1).out(r2)…`, two relations or more. + ReachesPath(Vec), + /// `out(r1).out(r2)….has(field, literal)`. + ViaPath { + path: Vec, + field: String, + literal: String, + negate: bool, + }, +} + +/// A loaded path as a [`Path`]; its length was checked when it was composed. +fn path_of(rels: &[String]) -> Path<'_> { + let rels: Vec<&str> = rels.iter().map(String::as_str).collect(); + Path::try_of(&rels).expect("composed paths are 1..=MAX_PATH long") } #[derive(Debug, Clone, PartialEq, Eq)] @@ -118,6 +133,20 @@ impl MethodDecl { literal, negate: *negate, }, + DeclCond::ReachesPath(path) => Cond::ReachesPath { + path: path_of(path), + }, + DeclCond::ViaPath { + path, + field, + literal, + negate, + } => Cond::ViaPath { + path: path_of(path), + field, + literal, + negate: *negate, + }, }) .collect() } @@ -719,6 +748,20 @@ fn traversal( literal: literal.to_owned(), negate, }, + Cond::ReachesPath { path } => { + DeclCond::ReachesPath(path.rels().iter().map(|r| (*r).to_owned()).collect()) + } + Cond::ViaPath { + path, + field, + literal, + negate, + } => DeclCond::ViaPath { + path: path.rels().iter().map(|r| (*r).to_owned()).collect(), + field: field.to_owned(), + literal: literal.to_owned(), + negate, + }, Cond::Method(MethodRef::Key(_)) => unreachable!("text names methods by label"), }) .collect(); diff --git a/crates/lance-graph-glove-parity/src/gremlin.rs b/crates/lance-graph-glove-parity/src/gremlin.rs index cdd5266a3..7ed55aacd 100644 --- a/crates/lance-graph-glove-parity/src/gremlin.rs +++ b/crates/lance-graph-glove-parity/src/gremlin.rs @@ -37,7 +37,7 @@ //! | `has(f, eq/neq)` | a condition on the target's field, read through the key | //! | `has(f, lt/lte/gt/gte)` | refused (`OrderThroughRelation`): no ordered compare reads through a key | //! | `count()` | `COUNT(*)` of anchor rows that reach a live target | -//! | `out` | refused (`SecondHop`): a key read through a key (#1311 G1) | +//! | `out` | the next hop of the path; the anchor stays. All the path's conditions fold into one chained gather (D-CMH-G1-7); past `MAX_PATH` relations refused (`PathTooDeep`) | //! | `values(f)` | refused (`ValueThroughRelation`): a sum through the key is not in Quack | //! | `dedup().count()` | the number of distinct live targets ([`Fold::Distinct`]); the key lane must be stored in key order | //! | `dedup()` then `has(f, eq/neq)` | as `has` before `dedup`: a target filter commutes with it | @@ -56,7 +56,7 @@ //! stream this layer has no answer type for (`NoTerminal`). Effectful steps //! (`addV`, `property`, `drop`, …) have no [`Step`] at all: a traversal cannot //! express them, so it cannot carry authority. -use crate::method::{Cond, Fold, Law, MethodBody, MethodRef, Order}; +use crate::method::{Cond, Fold, Law, MethodBody, MethodRef, Order, Path}; /// A predicate in `has(field, pred)`: equality over a coded field's /// literal, or an ordered compare of an `I32` field against a number. @@ -129,8 +129,8 @@ pub enum GremlinRefusal { AnchorMismatch { step: u16, concept: u16 }, /// `in` / `both`: a fan-out moves the anchor; not carried. FanOut(String), - /// A hop after `out(rel)`. - SecondHop(String), + /// An `out(rel)` past [`MAX_PATH`](crate::method::MAX_PATH) relations; the relation is named. + PathTooDeep(String), /// `values(f)` after `out(rel)` (the relation's label). ValueThroughRelation(String), /// `dedup()` after `out(rel)` not followed by `count()` (or `has`). @@ -198,8 +198,8 @@ impl<'a> Composed<'a> { enum Cursor<'a> { Vertex, - /// On the target of `out(rel)`; the anchor has not moved. - Reached(&'a str), + /// On the target of `out(r1)…out(rk)`; the anchor has not moved. + Reached(Path<'a>), /// On the distinct targets of `out(rel)` (`dedup()` after the hop). /// Only `has` (which commutes with `dedup`) and `count()` follow. Distinct(&'a str), @@ -231,31 +231,48 @@ pub fn compose<'a>(concept: u16, steps: &[Step<'a>]) -> Result, Gre (Step::In(r) | Step::Both(r), _) => return Err(GremlinRefusal::FanOut(r.to_owned())), (Step::Out(r), Cursor::Vertex) => { filter.push(Cond::Reaches { rel: r }); - Cursor::Reached(r) + Cursor::Reached(Path::of(&[r])) } - (Step::Out(r), Cursor::Reached(_)) => { - return Err(GremlinRefusal::SecondHop(r.to_owned())) + (Step::Out(r), Cursor::Reached(path)) => { + let path = path + .then(r) + .ok_or_else(|| GremlinRefusal::PathTooDeep(r.to_owned()))?; + filter.push(Cond::ReachesPath { path }); + Cursor::Reached(path) } - (Step::Has { field, pred }, Cursor::Reached(rel)) => { + (Step::Has { field, pred }, Cursor::Reached(path)) => { let (literal, negate) = match pred { Pred::Eq(l) => (l, false), Pred::Neq(l) => (l, true), Pred::Lt(_) | Pred::Lte(_) | Pred::Gt(_) | Pred::Gte(_) => { - return Err(GremlinRefusal::OrderThroughRelation(rel.to_owned())) + return Err(GremlinRefusal::OrderThroughRelation(path.last().to_owned())) } }; - filter.push(Cond::Via { - rel, - field, - literal, - negate, + filter.push(match path.rels() { + [rel] => Cond::Via { + rel, + field, + literal, + negate, + }, + _ => Cond::ViaPath { + path, + field, + literal, + negate, + }, }); - Cursor::Reached(rel) + Cursor::Reached(path) } - (Step::Where(_), Cursor::Reached(rel)) => { - return Err(GremlinRefusal::WhereThroughRelation(rel.to_owned())) + (Step::Where(_), Cursor::Reached(path)) => { + return Err(GremlinRefusal::WhereThroughRelation(path.last().to_owned())) } - (Step::Dedup, Cursor::Reached(rel)) => Cursor::Distinct(rel), + // The distinct targets of a chain are not carried: the distinct + // fold counts one relation's key lane. + (Step::Dedup, Cursor::Reached(path)) => match path.rels() { + [rel] => Cursor::Distinct(rel), + _ => return Err(GremlinRefusal::DedupThroughRelation(path.last().to_owned())), + }, (Step::Has { field, pred }, Cursor::Distinct(rel)) => { let (Pred::Eq(literal) | Pred::Neq(literal)) = pred else { return Err(GremlinRefusal::OrderThroughRelation(rel.to_owned())); @@ -275,11 +292,8 @@ pub fn compose<'a>(concept: u16, steps: &[Step<'a>]) -> Result, Gre (_, Cursor::Distinct(rel)) => { return Err(GremlinRefusal::DedupThroughRelation(rel.to_owned())) } - (Step::Values(_), Cursor::Reached(rel)) => { - return Err(GremlinRefusal::ValueThroughRelation(rel.to_owned())) - } - (Step::Min | Step::Max, Cursor::Reached(rel)) => { - return Err(GremlinRefusal::ValueThroughRelation(rel.to_owned())) + (Step::Values(_) | Step::Min | Step::Max, Cursor::Reached(path)) => { + return Err(GremlinRefusal::ValueThroughRelation(path.last().to_owned())) } (Step::Min, Cursor::Value(f)) => { folds.push(Fold::Min(f)); diff --git a/crates/lance-graph-glove-parity/src/method.rs b/crates/lance-graph-glove-parity/src/method.rs index 7c8dc4d9d..f1ac8d99e 100644 --- a/crates/lance-graph-glove-parity/src/method.rs +++ b/crates/lance-graph-glove-parity/src/method.rs @@ -56,7 +56,8 @@ use lance_graph_mask_risc::{ }; use lance_graph_quack::bind::{Binder, BoundField, FieldKind, TableId}; use lance_graph_quack::{ - lower, Agg, Cmp, Col, Filter, ForeignLane, ForeignPlane, LowerError, Mask, Query, + lower, Agg, ChainHop, ChainTest, Cmp, Col, Filter, ForeignLane, ForeignPlane, LowerError, Mask, + Query, }; use lance_graph_r2il::{procs, FoldDialect, FoldError, Val}; @@ -139,6 +140,91 @@ pub enum Order { Ge, } +/// The most relations one [`Path`] may chain: the depth one +/// `MaskOp::GatherChain` carries ([`lance_graph_mask_risc::MAX_CHAIN_HOPS`]). +pub const MAX_PATH: usize = lance_graph_mask_risc::MAX_CHAIN_HOPS; + +/// A chain of functional relations, from the anchor outward: +/// `out(rels[0]).out(rels[1])…`. Each relation is looked up on the previous +/// relation's target glove. At least one, at most [`MAX_PATH`]. +#[derive(Clone, Copy, PartialEq, Eq)] +pub struct Path<'a> { + rels: [&'a str; MAX_PATH], + len: u8, +} + +impl<'a> Path<'a> { + /// The path through `rels`. + /// + /// # Panics + /// + /// When `rels` is empty or longer than [`MAX_PATH`] — a path literal + /// is a programming error, not input. Input goes through + /// [`Path::try_of`]. + #[must_use] + pub const fn of(rels: &[&'a str]) -> Self { + match Self::try_of(rels) { + Some(p) => p, + None => panic!("a path has 1..=MAX_PATH relations"), + } + } + + /// The path through `rels`, or `None` when it is empty or longer than + /// [`MAX_PATH`]. + #[must_use] + pub const fn try_of(rels: &[&'a str]) -> Option { + if rels.is_empty() || rels.len() > MAX_PATH { + return None; + } + let mut out = [""; MAX_PATH]; + let mut i = 0; + while i < rels.len() { + out[i] = rels[i]; + i += 1; + } + Some(Self { + rels: out, + len: rels.len() as u8, + }) + } + + /// The relations, in order. + #[must_use] + pub fn rels(&self) -> &[&'a str] { + &self.rels[..usize::from(self.len)] + } + + /// This path followed by `rel`, or `None` past [`MAX_PATH`]. + #[must_use] + pub fn then(self, rel: &'a str) -> Option { + let n = usize::from(self.len); + if n == MAX_PATH { + return None; + } + let mut p = self; + p.rels[n] = rel; + p.len += 1; + Some(p) + } + + /// The last relation. + #[must_use] + pub fn last(&self) -> &'a str { + self.rels[usize::from(self.len) - 1] + } + + /// Whether `self` is `other` or a prefix of it. + fn is_prefix_of(&self, other: &Path<'_>) -> bool { + self.len <= other.len && self.rels() == &other.rels()[..usize::from(self.len)] + } +} + +impl core::fmt::Debug for Path<'_> { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.debug_list().entries(self.rels()).finish() + } +} + /// One condition of a body's `WHERE`, on a canonical field. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum Cond<'a> { @@ -172,9 +258,35 @@ pub enum Cond<'a> { literal: &'a str, negate: bool, }, + /// `out(r1).out(r2)…`: the key chain along `path` (two relations or + /// more; one is [`Cond::Reaches`]) reaches a live row at every hop. + /// Every relation is functional, so the anchor does not move. Lowered + /// with every other condition on the same path into ONE chained gather: + /// no hop table gets a mask. + ReachesPath { path: Path<'a> }, + /// `out(r1).out(r2)….has(field, = literal)`, or `!=` when `negate`: the + /// path reaches live rows (as [`Cond::ReachesPath`]) and the LAST target's + /// canonical `field` equals the literal, or does not. + ViaPath { + path: Path<'a>, + field: &'a str, + literal: &'a str, + negate: bool, + }, } impl<'a> Cond<'a> { + /// The relation path the condition walks, if any: one relation for + /// [`Cond::Reaches`] / [`Cond::Via`]. + #[must_use] + pub fn path(self) -> Option> { + match self { + Cond::Reaches { rel } | Cond::Via { rel, .. } => Path::try_of(&[rel]), + Cond::ReachesPath { path } | Cond::ViaPath { path, .. } => Some(path), + _ => None, + } + } + /// The canonical field of the method's own concept the condition names. /// A relation's key is known only to the glove, so a relation condition /// names none here; preparation adds the key to `required`. @@ -183,7 +295,11 @@ impl<'a> Cond<'a> { Cond::Eq { field, .. } | Cond::Ne { field, .. } | Cond::Order { field, .. } => { Some(field) } - Cond::Method(_) | Cond::Reaches { .. } | Cond::Via { .. } => None, + Cond::Method(_) + | Cond::Reaches { .. } + | Cond::Via { .. } + | Cond::ReachesPath { .. } + | Cond::ViaPath { .. } => None, } } } @@ -401,6 +517,11 @@ pub enum PrepareError { /// The relation supplies no foreign lane for this target field /// (`relation.field`). NoForeignLane(String), + /// More conditions on one hop of a chain than a chained gather carries + /// (`lance_graph_mask_risc::MAX_HOP_TESTS`); the relation is named. + TooManyHopTests(String), + /// More relation chains in one body than a `u16` index names. + TooManyChains, /// Two relations the method reads reach this target concept in /// different populations. A call names one population per concept. ConflictingUniverse(u16), @@ -498,6 +619,9 @@ enum BoundCond { code: u32, negate: bool, }, + /// The key `fk` leads through the hops of `chains[chain]` + /// ([`Filter::Chain`]): every condition on one relation path, folded. + Chain { fk: Col, chain: u16 }, } #[derive(Debug, Clone, PartialEq, Eq)] @@ -505,6 +629,8 @@ enum Bound { Query { live: Mask, conds: Vec, + /// The hop chains [`BoundCond::Chain`] names. + chains: Vec>, folds: Vec, law: OwnedLaw, /// The Quack programs, lowered once at preparation, one per fold. @@ -551,12 +677,16 @@ impl PreparedMethod { #[must_use] pub fn queries(&self) -> Vec { let Bound::Query { - live, conds, folds, .. + live, + conds, + chains, + folds, + .. } = &self.bound else { return Vec::new(); }; - bound_queries(*live, conds, folds) + bound_queries(*live, conds, chains, folds) } /// The lowered Quack programs a call executes, one per fold. Lowered once, @@ -570,7 +700,12 @@ impl PreparedMethod { } } -fn bound_queries(live: Mask, conds: &[BoundCond], folds: &[BoundFold]) -> Vec { +fn bound_queries( + live: Mask, + conds: &[BoundCond], + chains: &[Vec], + folds: &[BoundFold], +) -> Vec { // A distinct fold tests its relation's liveness, unless the conditions // already do: the same test is never emitted twice. let filter = |fold: &BoundFold| { @@ -603,6 +738,9 @@ fn bound_queries(live: Mask, conds: &[BoundCond], folds: &[BoundFold]) -> Vec Filter::semijoin(fk, plane), + BoundCond::Chain { fk, chain } => { + Filter::chain(fk, chains[usize::from(chain)].clone()) + } BoundCond::Via { fk, lane, @@ -674,6 +812,7 @@ impl PreparedMethod { folds, law, programs, + .. } = &self.bound else { return Err(match self.bound { @@ -743,12 +882,12 @@ impl PreparedMethod { // A relation condition always tests the target row's // liveness, and R2IL's fold band has no call that // reads a foreign plane through a key. - BoundCond::Reaches { .. } | BoundCond::Via { .. } => { - Err(CallError::FrontendGap { - frontend: Frontend::R2il, - missing: "GATHER", - }) - } + BoundCond::Reaches { .. } + | BoundCond::Via { .. } + | BoundCond::Chain { .. } => Err(CallError::FrontendGap { + frontend: Frontend::R2il, + missing: "GATHER", + }), }) .collect::, _>>()?; folds @@ -1184,8 +1323,8 @@ impl Glove<'_> { _ => &[], }; let mut keys = Vec::new(); - for rel in relations_named(filter, folds_of) { - let fk = self.relation(rel)?.fk; + for path in relations_named(filter, folds_of) { + let fk = self.relation(path.rels()[0])?.fk; if !reads.contains(&fk) && !keys.contains(&fk) { keys.push(fk); } @@ -1214,7 +1353,7 @@ impl Glove<'_> { .binder .table(self.table) .ok_or_else(|| PrepareError::UnknownTable(self.table.to_owned()))?; - let conds = self.conds(t, filter)?; + let (conds, chains) = self.conds(t, filter)?; let folds = folds .iter() .map(|f| match *f { @@ -1243,7 +1382,7 @@ impl Glove<'_> { }) .collect::, _>>()?; let live = self.binder.live(t); - let programs = bound_queries(live, &conds, &folds) + let programs = bound_queries(live, &conds, &chains, &folds) .iter() .map(lower) .collect::, _>>() @@ -1251,6 +1390,7 @@ impl Glove<'_> { Bound::Query { live, conds, + chains, folds, law: law.into(), programs, @@ -1270,15 +1410,16 @@ impl Glove<'_> { _ => return Err(PrepareError::EffectMismatch), }; let mut reaches: Vec<(u16, Universe)> = Vec::new(); - for rel in relations_named(filter, folds_of) { - let r = self.relation(rel)?; - let (concept, universe) = (r.target.concept, r.target.universe); - match reaches.iter().find(|(c, _)| *c == concept) { - None => reaches.push((concept, universe)), - Some(&(_, u)) if u != universe => { - return Err(PrepareError::ConflictingUniverse(concept)); + for path in relations_named(filter, folds_of) { + for r in self.walk(path)? { + let (concept, universe) = (r.target.concept, r.target.universe); + match reaches.iter().find(|(c, _)| *c == concept) { + None => reaches.push((concept, universe)), + Some(&(_, u)) if u != universe => { + return Err(PrepareError::ConflictingUniverse(concept)); + } + Some(_) => {} } - Some(_) => {} } } Ok(PreparedMethod { @@ -1298,48 +1439,159 @@ impl Glove<'_> { .ok_or_else(|| PrepareError::UnknownRelation(name.to_owned())) } - /// Bind a filter. Each relation's liveness test is emitted once, where - /// the relation is first named. - fn conds(&self, t: TableId, filter: &[Cond<'_>]) -> Result, PrepareError> { + /// The relations `path` walks, each looked up on the previous one's + /// target glove. + fn walk(&self, path: Path<'_>) -> Result>, PrepareError> { + let mut out: Vec<&Relation<'_>> = Vec::with_capacity(path.rels().len()); + for (k, name) in path.rels().iter().enumerate() { + let r = match out.last() { + None => self.relation(name)?, + Some(prev) => prev.target.relation(name)?, + }; + debug_assert_eq!(out.len(), k); + out.push(r); + } + Ok(out) + } + + /// Bind a filter. Conditions on a one-relation path keep their own + /// leaves (the relation's liveness test emitted once, where it is first + /// named). Conditions on a longer path, and every condition on one of + /// its prefixes, fold into ONE chain per maximal path: the chain is + /// emitted where the first of them stands. + fn conds( + &self, + t: TableId, + filter: &[Cond<'_>], + ) -> Result<(Vec, Vec>), PrepareError> { + let paths: Vec> = filter.iter().filter_map(|c| c.path()).collect(); + // A path two relations or longer that no other path extends. + let mut chained: Vec> = Vec::new(); + for p in &paths { + let extended = paths + .iter() + .any(|q| q.rels().len() > p.rels().len() && p.is_prefix_of(q)); + if p.rels().len() >= 2 && !extended && !chained.contains(p) { + chained.push(*p); + } + } let mut out = Vec::with_capacity(filter.len()); + let mut chains: Vec> = Vec::new(); + let mut emitted = vec![false; chained.len()]; let mut reached: Vec<&str> = Vec::new(); for c in filter { - let rel = match *c { - Cond::Reaches { rel } | Cond::Via { rel, .. } => rel, - _ => { - out.push(self.cond(t, *c)?); - continue; - } + let Some(path) = c.path() else { + out.push(self.cond(t, *c)?); + continue; }; - let r = self.relation(rel)?; - let fk = self.field(t, r.fk)?; - if fk.kind != FieldKind::Code { - return Err(PrepareError::KindMismatch(r.fk.to_owned())); + let owners: Vec = (0..chained.len()) + .filter(|&i| path.is_prefix_of(&chained[i])) + .collect(); + if owners.is_empty() { + // A one-relation path no chain extends. + let rel = path.last(); + let r = self.relation(rel)?; + let fk = self.field(t, r.fk)?; + if fk.kind != FieldKind::Code { + return Err(PrepareError::KindMismatch(r.fk.to_owned())); + } + if !reached.contains(&rel) { + reached.push(rel); + out.push(BoundCond::Reaches { + fk: fk.col, + plane: ForeignPlane(r.plane), + }); + } + if let Cond::Via { + field, + literal, + negate, + .. + } = *c + { + let (lane, code) = r.bind(field, literal)?; + out.push(BoundCond::Via { + fk: fk.col, + lane, + code, + negate, + }); + } + continue; } - if !reached.contains(&rel) { - reached.push(rel); - out.push(BoundCond::Reaches { - fk: fk.col, - plane: ForeignPlane(r.plane), - }); + for i in owners { + if emitted[i] { + continue; + } + emitted[i] = true; + let (fk, hops) = self.chain(t, chained[i], filter)?; + let chain = u16::try_from(chains.len()).map_err(|_| PrepareError::TooManyChains)?; + chains.push(hops); + out.push(BoundCond::Chain { fk, chain }); } - if let Cond::Via { - field, - literal, - negate, - .. - } = *c - { - let (lane, code) = r.bind(field, literal)?; - out.push(BoundCond::Via { - fk: fk.col, + } + Ok((out, chains)) + } + + /// The chain along `path`: each hop's live plane, the tests every + /// condition in `filter` puts on that hop (a condition on the prefix + /// ending there), and the key into the next hop. + fn chain( + &self, + t: TableId, + path: Path<'_>, + filter: &[Cond<'_>], + ) -> Result<(Col, Vec), PrepareError> { + let rels = self.walk(path)?; + let first = rels[0]; + let fk = self.field(t, first.fk)?; + if fk.kind != FieldKind::Code { + return Err(PrepareError::KindMismatch(first.fk.to_owned())); + } + let mut hops = Vec::with_capacity(rels.len()); + for (k, r) in rels.iter().enumerate() { + let mut tests: Vec = Vec::new(); + for c in filter { + let at_hop = c.path().is_some_and(|p| p.rels() == &path.rels()[..=k]); + let (field, literal, negate) = match *c { + Cond::Via { + field, + literal, + negate, + .. + } + | Cond::ViaPath { + field, + literal, + negate, + .. + } if at_hop => (field, literal, negate), + _ => continue, + }; + let (lane, value) = r.bind(field, literal)?; + let test = ChainTest { lane, - code, + value, negate, - }); + }; + if !tests.contains(&test) { + tests.push(test); + } } + if tests.len() > lance_graph_mask_risc::MAX_HOP_TESTS { + return Err(PrepareError::TooManyHopTests(r.name.to_owned())); + } + let next = match rels.get(k + 1) { + Some(n) => Some(r.key_lane(n.fk)?), + None => None, + }; + hops.push(ChainHop { + live: ForeignPlane(r.plane), + tests, + next, + }); } - Ok(out) + Ok((fk.col, hops)) } /// A required field's lane, which must be of `kind`. @@ -1390,7 +1642,10 @@ impl Glove<'_> { let (field, literal, negate) = match c { Cond::Eq { field, literal } => (field, literal, false), Cond::Ne { field, literal } => (field, literal, true), - Cond::Reaches { .. } | Cond::Via { .. } => unreachable!("bound by `conds`"), + Cond::Reaches { .. } + | Cond::Via { .. } + | Cond::ReachesPath { .. } + | Cond::ViaPath { .. } => unreachable!("bound by `conds`"), Cond::Order { field, op, value } => { return Ok(BoundCond::Ordered { col: self.lane(t, field, FieldKind::I32)?, @@ -1426,7 +1681,28 @@ impl Glove<'_> { impl Relation<'_> { /// A target field and literal, as a foreign lane and code. Errors name /// the field `relation.field`. + /// The foreign lane a call supplies for target `field`, which must be a + /// proven `Code` field: the key into the next hop of a chain. Its lane + /// holds row ordinals, so no literal is resolved against it. + fn key_lane(&self, field: &str) -> Result { + self.checked(field).map(|(lane, _, _)| lane) + } + fn bind(&self, field: &str, literal: &str) -> Result<(ForeignLane, u32), PrepareError> { + let (lane, t, b) = self.checked(field)?; + let named = || format!("{}.{field}", self.name); + let code = self.target.binder.code(t, b.col, literal).ok_or_else(|| { + PrepareError::UnknownValue { + field: named(), + literal: literal.to_owned(), + } + })?; + Ok((lane, code)) + } + + /// `field` of the target: in its basis, proven by a claim, supplied as a + /// foreign lane, bound, and a `Code` field. + fn checked(&self, field: &str) -> Result<(ForeignLane, TableId, BoundField), PrepareError> { let named = || format!("{}.{field}", self.name); let g = self.target; if !g.basis.iter().any(|b| b == field) { @@ -1459,27 +1735,18 @@ impl Relation<'_> { if b.kind != FieldKind::Code { return Err(PrepareError::KindMismatch(named())); } - let code = g - .binder - .code(t, b.col, literal) - .ok_or_else(|| PrepareError::UnknownValue { - field: named(), - literal: literal.to_owned(), - })?; - Ok((lane, code)) + Ok((lane, t, b)) } } -/// The relations a body names, in order: its conditions', then its folds'. -fn relations_named<'a>(filter: &[Cond<'a>], folds: &[Fold<'a>]) -> Vec<&'a str> { +/// The relation paths a body names, in order: its conditions', then its +/// folds'. +fn relations_named<'a>(filter: &[Cond<'a>], folds: &[Fold<'a>]) -> Vec> { filter .iter() - .filter_map(|c| match *c { - Cond::Reaches { rel } | Cond::Via { rel, .. } => Some(rel), - _ => None, - }) + .filter_map(|c| c.path()) .chain(folds.iter().filter_map(|f| match *f { - Fold::Distinct { rel } => Some(rel), + Fold::Distinct { rel } => Path::try_of(&[rel]), _ => None, })) .collect() diff --git a/crates/lance-graph-glove-parity/tests/relation.rs b/crates/lance-graph-glove-parity/tests/relation.rs index 065e81e05..15c987a99 100644 --- a/crates/lance-graph-glove-parity/tests/relation.rs +++ b/crates/lance-graph-glove-parity/tests/relation.rs @@ -606,9 +606,18 @@ fn out_composes_and_keeps_the_anchor() { assert_eq!(c.filter(), &[REACHES, NOT_DE]); let refuse = |steps: &[Step<'static>]| compose(LINE, steps).unwrap_err(); let out = Step::Out("billed_to"); + // A second hop composes since D-CMH-G1-7 (it was `SecondHop` before): + // the path folds into one chained gather. Whether `owned_by` exists is + // preparation's to say (tests/two_hop.rs). + let c = compose(LINE, &[v, out, Step::Out("owned_by"), Step::Count]).unwrap(); assert_eq!( - refuse(&[v, out, Step::Out("owned_by"), Step::Count]), - GremlinRefusal::SecondHop("owned_by".into()) + c.filter(), + &[ + REACHES, + Cond::ReachesPath { + path: lance_graph_glove_parity::method::Path::of(&["billed_to", "owned_by"]), + }, + ] ); assert_eq!( refuse(&[v, out, Step::Values("credit"), Step::Sum]), diff --git a/crates/lance-graph-glove-parity/tests/two_hop.rs b/crates/lance-graph-glove-parity/tests/two_hop.rs new file mode 100644 index 000000000..5e41e4fff --- /dev/null +++ b/crates/lance-graph-glove-parity/tests/two_hop.rs @@ -0,0 +1,657 @@ +//! D-CMH-G1-7: chained hops are folded hops. +//! +//! `out(r1)…out(r2)` over two functional relations keeps the anchor (each +//! line still names at most one partner, each partner at most one region), +//! and lowers to ONE `Filter::Chain` — one `MaskOp::GatherChain` — with every +//! condition on a hop read in place at the row the key reached. No mask over +//! `partner` or `region` exists anywhere in the program. +//! +//! FIXTURE: `line -billed_to-> partner -located_in-> region`. Every answer is +//! checked against a row loop over the TEXT of each field. +use lance_graph_glove_parity::awareness::{method_ident, Rel, SoftwareGraph}; +use lance_graph_glove_parity::basis::{FieldMap, Grade}; +use lance_graph_glove_parity::decl::parse_decls; +use lance_graph_glove_parity::gremlin::{compose, GremlinRefusal, Pred, Step}; +use lance_graph_glove_parity::method::{ + Answer, CallError, Cond, EffectClass, Fold, FoldValue, Frontend, Glove, Law, MethodBody, + MethodKey, MethodSpec, MethodTable, Path, PrepareError, Reach, ReadingId, Relation, SchemaSeal, + Universe, MAX_PATH, +}; +use lance_graph_mask_risc::{words_for, Foreign, ForeignPlane, LaneRef, MaskOp, Planes}; +use lance_graph_ogar::ogar_vocab::class_ids::DOCUMENT; +use lance_graph_quack::bind::{Binder, BoundField, FieldKind, TableId}; +use lance_graph_quack::{ + ChainHop, ChainTest, Cmp, Col, Filter, ForeignLane, ForeignPlane as QPlane, Mask, +}; + +/// The anchor is a minted concept so the text form can name it. +const LINE: u16 = DOCUMENT; +const PARTNER: u16 = 0xFFE2; +const REGION: u16 = 0xFFE3; +const READING: ReadingId = ReadingId(1); +const LINE_U: Universe = Universe(7); +const PARTNER_U: Universe = Universe(8); +const REGION_U: Universe = Universe(9); +const PARTNERS: usize = 37; +const REGIONS: usize = 11; +const SIZES: [usize; 4] = [1, 64, 65, 4097]; + +const STATUS: [&str; 3] = ["open", "posted", "void"]; +const COUNTRY: [&str; 3] = ["DE", "FR", "US"]; +const ZONE: [&str; 3] = ["EU", "NA", "APAC"]; + +fn status(i: usize) -> &'static str { + STATUS[(i * 5 + i / 7) % 3] +} +fn amount(i: usize) -> i32 { + ((i * 37) % 201) as i32 - 100 +} +fn line_live(i: usize) -> bool { + i % 13 != 5 +} +/// Some keys dangle at depth 1. +fn partner_of(i: usize) -> usize { + (i * 11 + 3) % (PARTNERS + 4) +} +fn partner_live(p: usize) -> bool { + p % 7 != 2 +} +fn country(p: usize) -> &'static str { + COUNTRY[(p * 3 + p / 4) % 3] +} +/// Some keys dangle at depth 2. +fn region_of(p: usize) -> usize { + (p * 5 + 1) % (REGIONS + 3) +} +fn region_live(r: usize) -> bool { + r % 5 != 3 +} +fn zone(r: usize) -> &'static str { + ZONE[(r * 2 + r / 3) % 3] +} + +/// The partner a line reaches: in range and live. +fn reached(i: usize) -> Option { + let p = partner_of(i); + (p < PARTNERS && partner_live(p)).then_some(p) +} +/// The region a line reaches through its partner. +fn reached2(i: usize) -> Option { + let r = region_of(reached(i)?); + (r < REGIONS && region_live(r)).then_some(r) +} + +struct Erp; + +impl Binder for Erp { + fn table(&self, name: &str) -> Option { + match name { + "line" => Some(TableId(0)), + "partner" => Some(TableId(1)), + "region" => Some(TableId(2)), + _ => None, + } + } + fn live(&self, _: TableId) -> Mask { + Mask(0) + } + fn field(&self, t: TableId, name: &str) -> Option { + let (col, kind) = match (t.0, name) { + (0, "status") => (0, FieldKind::Code), + (0, "amount") => (1, FieldKind::I32), + (0, "partner") => (2, FieldKind::Code), + (1, "country") => (0, FieldKind::Code), + (1, "region") => (1, FieldKind::Code), + (2, "zone") => (0, FieldKind::Code), + _ => return None, + }; + Some(BoundField { + col: Col(col), + kind, + validity: None, + }) + } + fn code(&self, t: TableId, col: Col, literal: &str) -> Option { + let domain: &[&str] = match (t.0, col.0) { + (0, 0) => &STATUS, + (1, 0) => &COUNTRY, + (2, 0) => &ZONE, + _ => return None, + }; + domain.iter().position(|d| *d == literal).map(|p| p as u32) + } +} + +fn exact(fields: &[&'static str]) -> Vec { + fields + .iter() + .map(|f| FieldMap { + native: f, + canonical: f, + grade: Grade::Exact, + note: "fixture", + }) + .collect() +} + +fn names(fields: &[&str]) -> Vec { + fields.iter().map(|f| (*f).to_owned()).collect() +} + +fn bits(n: usize, f: impl Fn(usize) -> bool) -> Vec { + let mut w = vec![0u64; words_for(n)]; + for i in (0..n).filter(|&i| f(i)) { + w[i / 64] |= 1 << (i % 64); + } + w +} + +fn code(d: &[&str], s: &str) -> u32 { + d.iter().position(|x| *x == s).unwrap() as u32 +} + +struct Data { + n: usize, + line_live: Vec, + status: Vec, + amount: Vec, + partner: Vec, + partner_live: Vec, + country: Vec, + region: Vec, + region_live: Vec, + zone: Vec, +} + +fn data(n: usize) -> Data { + Data { + n, + line_live: bits(n, line_live), + status: (0..n).map(|i| code(&STATUS, status(i))).collect(), + amount: (0..n).map(amount).collect(), + partner: (0..n).map(|i| partner_of(i) as u32).collect(), + partner_live: bits(PARTNERS, partner_live), + country: (0..PARTNERS).map(|p| code(&COUNTRY, country(p))).collect(), + region: (0..PARTNERS).map(|p| region_of(p) as u32).collect(), + region_live: bits(REGIONS, region_live), + zone: (0..REGIONS).map(|r| code(&ZONE, zone(r))).collect(), + } +} + +impl Data { + fn run( + &self, + table: &MethodTable, + key: MethodKey, + frontend: Frontend, + ) -> Result { + let masks = [self.line_live.as_slice()]; + let lanes = [ + LaneRef::U32(&self.status), + LaneRef::I32(&self.amount), + LaneRef::U32(&self.partner), + ]; + let planes = Planes { + n_rows: self.n, + masks: &masks, + lanes: &lanes, + }; + let fplanes = [ + ForeignPlane { + words: &self.partner_live, + rows: PARTNERS, + }, + ForeignPlane { + words: &self.region_live, + rows: REGIONS, + }, + ]; + let flanes = [ + LaneRef::U32(&self.country), + LaneRef::U32(&self.region), + LaneRef::U32(&self.zone), + ]; + let reach = Reach { + foreign: Foreign { + planes: &fplanes, + lanes: &flanes, + }, + universes: &[(PARTNER, PARTNER_U), (REGION, REGION_U)], + }; + table.call_with(key, &planes, LINE_U, &reach, frontend) + } +} + +/// Row loop over the text: `(count, sum(amount))` of live rows passing `keep`. +fn oracle(n: usize, keep: impl Fn(usize) -> bool) -> (i64, i64) { + let rows: Vec = (0..n).filter(|&i| line_live(i) && keep(i)).collect(); + ( + rows.len() as i64, + rows.iter().map(|&i| i64::from(amount(i))).sum(), + ) +} + +fn scalars(a: Answer) -> (i64, i64) { + match a { + Answer::Values(v) => match v.as_slice() { + [FoldValue::Scalar(c), FoldValue::Scalar(s)] => (*c, *s), + other => panic!("{other:?}"), + }, + other => panic!("{other:?}"), + } +} + +struct Fx { + line_basis: Vec, + line_claims: Vec, + partner_basis: Vec, + partner_claims: Vec, + region_basis: Vec, + region_claims: Vec, +} + +impl Fx { + fn new() -> Self { + Self { + line_basis: names(&["status", "amount", "partner"]), + line_claims: exact(&["status", "amount", "partner"]), + partner_basis: names(&["country", "region"]), + partner_claims: exact(&["country", "region"]), + region_basis: names(&["zone"]), + region_claims: exact(&["zone"]), + } + } + + fn spec<'b>(&self, key: u8, body: MethodBody<'b>) -> MethodSpec<'b> { + MethodSpec { + key: MethodKey([key; 16]), + concept: LINE, + schema: SchemaSeal::of(&self.line_basis), + reading: READING, + effect: EffectClass::Pure, + body, + } + } + + fn table(&self, specs: &[(&str, MethodSpec<'_>)]) -> MethodTable { + self.table_with(specs, &[("country", 0), ("region", 1)]) + } + + /// `partner_lanes` is what a call supplies for the partner table; the + /// region key must be among them for a second hop to bind. + fn table_with( + &self, + specs: &[(&str, MethodSpec<'_>)], + partner_lanes: &[(&str, u16)], + ) -> MethodTable { + self.build(partner_lanes, |g| MethodTable::prepare(g, specs)) + } + + fn build( + &self, + partner_lanes: &[(&str, u16)], + f: impl FnOnce(&Glove<'_>) -> MethodTable, + ) -> MethodTable { + let region = Glove { + concept: REGION, + basis: &self.region_basis, + claims: &self.region_claims, + binder: &Erp, + table: "region", + reading: READING, + universe: REGION_U, + relations: &[], + }; + let partner_rels = [Relation { + name: "located_in", + fk: "region", + target: ®ion, + plane: 1, + lanes: &[("zone", 2)], + }]; + let partner = Glove { + concept: PARTNER, + basis: &self.partner_basis, + claims: &self.partner_claims, + binder: &Erp, + table: "partner", + reading: READING, + universe: PARTNER_U, + relations: &partner_rels, + }; + let rels = [Relation { + name: "billed_to", + fk: "partner", + target: &partner, + plane: 0, + lanes: partner_lanes, + }]; + let glove = Glove { + concept: LINE, + basis: &self.line_basis, + claims: &self.line_claims, + binder: &Erp, + table: "line", + reading: READING, + universe: LINE_U, + relations: &rels, + }; + f(&glove) + } +} + +fn query(filter: &'static [Cond<'static>], folds: &'static [Fold<'static>]) -> MethodBody<'static> { + MethodBody::Query { + filter, + folds, + law: Law::Values, + } +} + +const COUNT_SUM: &[Fold] = &[Fold::Count, Fold::Sum("amount")]; +const BL: Path = Path::of(&["billed_to", "located_in"]); +const OPEN: Cond = Cond::Eq { + field: "status", + literal: "open", +}; +const TO_DE: Cond = Cond::Via { + rel: "billed_to", + field: "country", + literal: "DE", + negate: false, +}; +const REACHES2: Cond = Cond::ReachesPath { path: BL }; +const IN_EU: Cond = Cond::ViaPath { + path: BL, + field: "zone", + literal: "EU", + negate: false, +}; +const NOT_EU: Cond = Cond::ViaPath { + path: BL, + field: "zone", + literal: "EU", + negate: true, +}; + +/// FAILS IF a two-hop condition disagrees with the row loop at any size — +/// with tests at hop 1, hop 2, both, negated, or with a local condition. +/// Anti-vacuity: each depth rejects rows the other keeps. +#[test] +fn two_hops_match_the_row_loop() { + let fx = Fx::new(); + let specs = [ + ("reaches2", fx.spec(1, query(&[REACHES2], COUNT_SUM))), + ("de_eu", fx.spec(2, query(&[TO_DE, IN_EU], COUNT_SUM))), + ("open_not_eu", fx.spec(3, query(&[OPEN, NOT_EU], COUNT_SUM))), + ]; + let table = fx.table(&specs); + for n in SIZES { + let d = data(n); + type Keep = Box bool>; + let cases: [(u8, Keep); 3] = [ + (1, Box::new(|i| reached2(i).is_some())), + ( + 2, + Box::new(|i| { + reached(i).is_some_and(|p| country(p) == "DE") + && reached2(i).is_some_and(|r| zone(r) == "EU") + }), + ), + ( + 3, + Box::new(|i| status(i) == "open" && reached2(i).is_some_and(|r| zone(r) != "EU")), + ), + ]; + for (k, keep) in cases { + let got = scalars(d.run(&table, MethodKey([k; 16]), Frontend::Quack).unwrap()); + assert_eq!(got, oracle(n, keep), "method {k}, n={n}"); + } + } + let n = 4097; + let one = oracle(n, |i| reached(i).is_some()).0; + let two = oracle(n, |i| reached2(i).is_some()).0; + let eu = oracle(n, |i| reached2(i).is_some_and(|r| zone(r) == "EU")).0; + assert!(0 < eu && eu < two && two < one, "{eu} {two} {one}"); + // `!=` at depth 2 still requires a live region. + let not_eu = oracle(n, |i| reached2(i).is_some_and(|r| zone(r) != "EU")).0; + let not_eu_loose = oracle(n, |i| { + reached(i).is_some_and(|p| { + let r = region_of(p); + !(r < REGIONS && zone(r) == "EU") + }) + }) + .0; + assert!(not_eu < not_eu_loose, "{not_eu} {not_eu_loose}"); +} + +/// FAILS IF the two hops lower to anything but ONE chain over both, with the +/// hop-1 condition folded into hop 1 (no separate semijoin or `EqU32Via`). +#[test] +fn two_hops_lower_to_one_chain() { + let fx = Fx::new(); + let specs = [("de_eu", fx.spec(1, query(&[OPEN, TO_DE, IN_EU], COUNT_SUM)))]; + let table = fx.table(&specs); + let m = table.get(MethodKey([1; 16])).unwrap().as_ref().unwrap(); + let chain = Filter::chain( + Col(2), + [ + ChainHop { + live: QPlane(0), + tests: vec![ChainTest { + lane: ForeignLane(0), + value: 0, + negate: false, + }], + next: Some(ForeignLane(1)), + }, + ChainHop { + live: QPlane(1), + tests: vec![ChainTest { + lane: ForeignLane(2), + value: 0, + negate: false, + }], + next: None, + }, + ], + ); + let filter = Filter::and([ + Filter::plane(Mask(0)), + Filter::cmp(Col(0), Cmp::EqU32(0)), + chain, + ]); + let queries = m.queries(); + assert_eq!(queries.len(), 2); + for q in &queries { + assert_eq!(q.filter, filter); + } + for p in m.programs() { + let gc = p + .ops + .iter() + .filter(|o| matches!(o, MaskOp::GatherChain { .. })) + .count(); + assert_eq!(gc, 1, "{:?}", p.ops); + assert!( + !p.ops.iter().any(|o| matches!(o, MaskOp::Gather { .. })), + "no per-hop gather: {:?}", + p.ops + ); + assert_eq!(p.chains.len(), 1); + assert_eq!(p.chains[0].len(), 2); + } + // Writing `out(billed_to)` explicitly changes nothing. + let specs = [ + ( + "explicit", + fx.spec( + 1, + query( + &[ + OPEN, + Cond::Reaches { rel: "billed_to" }, + TO_DE, + REACHES2, + IN_EU, + ], + COUNT_SUM, + ), + ), + ), + ( + "implied", + fx.spec(2, query(&[OPEN, TO_DE, IN_EU], COUNT_SUM)), + ), + ]; + let t = fx.table(&specs); + let get = |k: u8| t.get(MethodKey([k; 16])).unwrap().as_ref().unwrap(); + assert_eq!(get(1).queries(), get(2).queries()); +} + +/// FAILS IF the composer refuses a second hop, or composes it to other +/// conditions than the written ones. +#[test] +fn gremlin_composes_a_second_hop() { + let v = Step::V(LINE); + let c = compose( + LINE, + &[ + v, + Step::Out("billed_to"), + Step::Has { + field: "country", + pred: Pred::Eq("DE"), + }, + Step::Out("located_in"), + Step::Has { + field: "zone", + pred: Pred::Neq("EU"), + }, + Step::Count, + ], + ) + .unwrap(); + assert_eq!( + c.filter(), + &[Cond::Reaches { rel: "billed_to" }, TO_DE, REACHES2, NOT_EU] + ); + assert_eq!(c.folds(), &[Fold::Count]); + let refuse = |steps: &[Step<'static>]| compose(LINE, steps).unwrap_err(); + let two = [v, Step::Out("billed_to"), Step::Out("located_in")]; + let mut s = two.to_vec(); + s.extend([Step::Dedup, Step::Count]); + assert_eq!( + refuse(&s), + GremlinRefusal::DedupThroughRelation("located_in".into()) + ); + let mut s = two.to_vec(); + s.extend([Step::Values("zone"), Step::Sum]); + assert_eq!( + refuse(&s), + GremlinRefusal::ValueThroughRelation("located_in".into()) + ); + let mut deep = vec![v]; + deep.extend(std::iter::repeat_n(Step::Out("r"), MAX_PATH + 1)); + deep.push(Step::Count); + assert_eq!(refuse(&deep), GremlinRefusal::PathTooDeep("r".into())); +} + +/// FAILS IF a second hop binds without the relation, without the next key +/// supplied, or with the R2IL frontend (which has no gather). +#[test] +fn a_second_hop_is_refused_where_it_cannot_bind() { + let fx = Fx::new(); + const UNKNOWN: Cond = Cond::ReachesPath { + path: Path::of(&["billed_to", "shipped_to"]), + }; + let specs = [("x", fx.spec(1, query(&[UNKNOWN], COUNT_SUM)))]; + let t = fx.table(&specs); + assert_eq!( + t.get(MethodKey([1; 16])).unwrap().as_ref().unwrap_err(), + &PrepareError::UnknownRelation("shipped_to".into()) + ); + let specs = [("y", fx.spec(1, query(&[REACHES2], COUNT_SUM)))]; + let t = fx.table_with(&specs, &[("country", 0)]); + assert_eq!( + t.get(MethodKey([1; 16])).unwrap().as_ref().unwrap_err(), + &PrepareError::NoForeignLane("billed_to.region".into()) + ); + let t = fx.table(&specs); + match data(65).run(&t, MethodKey([1; 16]), Frontend::R2il) { + Err(CallError::FrontendGap { + frontend: Frontend::R2il, + missing: "GATHER", + }) => {} + other => panic!("{other:?}"), + } + // A call that does not supply the region table's universe is refused. + let d = data(65); + let masks = [d.line_live.as_slice()]; + let lanes = [ + LaneRef::U32(&d.status), + LaneRef::I32(&d.amount), + LaneRef::U32(&d.partner), + ]; + let planes = Planes { + n_rows: d.n, + masks: &masks, + lanes: &lanes, + }; + let reach = Reach { + foreign: Foreign { + planes: &[], + lanes: &[], + }, + universes: &[(PARTNER, PARTNER_U)], + }; + match t.call_with(MethodKey([1; 16]), &planes, LINE_U, &reach, Frontend::Quack) { + Err(CallError::ForeignUniverse { concept, .. }) => assert_eq!(concept, REGION), + other => panic!("{other:?}"), + } +} + +/// FAILS IF the text form of a two-hop traversal prepares to other programs +/// than the compiled one, or the awareness graph does not record the path. +#[test] +fn a_two_hop_traversal_loads_from_text_and_projects() { + let fx = Fx::new(); + let line = format!( + "document::t key={} schema={:016x} reading=1 := \ + V(document).out(billed_to).has(country, DE).out(located_in).has(zone, EU).count()\n", + "e7".repeat(16), + SchemaSeal::of(&fx.line_basis).0 + ); + let decls = parse_decls(&line).unwrap(); + let compiled = [( + "c", + fx.spec( + 1, + query( + &[Cond::Reaches { rel: "billed_to" }, TO_DE, REACHES2, IN_EU], + &[Fold::Count], + ), + ), + )]; + let t = fx.build(&[("country", 0), ("region", 1)], |g| { + MethodTable::prepare_with(g, &compiled, &decls) + }); + let programs = |l: &str| { + t.get(t.resolve(l).unwrap()) + .unwrap() + .as_ref() + .unwrap() + .programs() + .to_vec() + }; + assert_eq!(programs("t"), programs("c")); + let mut g = SoftwareGraph::new(); + g.project(&t, &compiled, &fx.line_claims, "two_hop.rs", "worktree"); + let filters = g.out(&method_ident(MethodKey([1; 16])), Rel::Filters); + assert!( + filters + .iter() + .any(|(f, _)| f.starts_with("cond:out(billed_to).out(located_in).zone=EU")), + "{filters:?}" + ); +} From 3eaf15f370074c231a9348374f5700b8689616cc Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 12 Oct 2026 21:00:29 +0000 Subject: [PATCH 5/5] =?UTF-8?q?plan=20+=20board:=20D-CMH-G1-7=20glove=20lo?= =?UTF-8?q?wering=20(=C2=A711)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_011BMJnDBAfzcuQh5HeT39Rh --- .claude/board/STATUS_BOARD.md | 4 +- .../entries/2026-10-12-chained-hops-folded.md | 8 +++- .../plans/gremlin-composable-algebra-v1.md | 45 ++++++++++++++++--- 3 files changed, 48 insertions(+), 9 deletions(-) diff --git a/.claude/board/STATUS_BOARD.md b/.claude/board/STATUS_BOARD.md index 49754fd96..9c7a4552e 100644 --- a/.claude/board/STATUS_BOARD.md +++ b/.claude/board/STATUS_BOARD.md @@ -24,8 +24,8 @@ Plan: `.claude/plans/gremlin-composable-algebra-v1.md`. | D-CMH-G1-3 | Gremlin-shaped runtime declarations | Shipped (#1508) | loads after the build; effects refused at load | | D-CMH-G1-4a | functional relation carrier: `out(rel)`, `out(rel).has`, `count()` | Shipped (#1514; ConflictingUniverse fix follows) | row loop over the text at n ∈ {1,64,65,4097}; 6 disable runs red; R2IL refuses (no `GATHER`) | | D-CMH-G1-4b | edge table as the anchor (`E()`, M:N one hop, `outE/inV`) + `Fold::Distinct` (`out(rel).dedup().count()`) | In PR | bag and set == row loop over the text at n ∈ {1,64,65,4097}; unordered key lane refused (`LaneNotOrdered`); 7 disable runs red | -| D-CMH-G1-7 | chained hops folded: `MaskOp::GatherChain` + Quack `Filter::Chain` (one op, tests at every depth, no hop mask) | In PR (needs ndarray#358) | executor == oracle == row loop, gated/ungated, tiles; 0-byte L1; 7 disable runs red | -| D-CMH-G1-4 | rest: a second hop (now D-CMH-G1-7 + glove lowering), `V(parent).in(rel)` spelling, `bothE` | Queued | — | +| D-CMH-G1-7 | chained hops folded: `MaskOp::GatherChain` + Quack `Filter::Chain` + glove `out(r1).out(r2)…` (one op, tests at every depth, no hop mask) | In PR (#1522; needs ndarray#358) | executor == oracle == row loop, gated/ungated, tiles; 0-byte L1; two-hop methods == row loop over text; 13 disable runs red | +| D-CMH-G1-4 | rest: `V(parent).in(rel)` spelling, `bothE` | Queued | — | | D-CMH-G1-5 | FirstOfRun | Queued | insufficiency proof first; Quack + R2IL vs independent oracle | | D-CMH-G1-6 | ordered `I32` conditions + `MIN`/`MAX` folds as method steps; R2IL `MIN`/`MAX` wired | In PR | both frontends == row loop incl. empty selection; 8 disable runs red | diff --git a/.claude/board/entries/2026-10-12-chained-hops-folded.md b/.claude/board/entries/2026-10-12-chained-hops-folded.md index 602482a50..c8bf96a29 100644 --- a/.claude/board/entries/2026-10-12-chained-hops-folded.md +++ b/.claude/board/entries/2026-10-12-chained-hops-folded.md @@ -28,5 +28,9 @@ DECISION: chains are stack-bounded (`MAX_CHAIN_HOPS = 8`, `MAX_HOP_TESTS = 8`) so `execute` keeps L1 (no allocation). REVISIT WHEN: a real query needs a deeper chain. -OPEN: the glove/composer `out(r1).out(r2)` lowering (step 3); the M:N -two-edge-table hop (run-level OR) awaits a decision. +Glove (plan §11): `Path` + `Cond::ReachesPath` / `ViaPath`; every +condition on a path and its prefixes binds into one chain; Gremlin `out` +after `out` extends the path. `tests/two_hop.rs` against a row loop over the +text; six more disable runs red. + +OPEN: the M:N two-edge-table hop (run-level OR) awaits a decision. diff --git a/.claude/plans/gremlin-composable-algebra-v1.md b/.claude/plans/gremlin-composable-algebra-v1.md index b46faf54f..589a95de6 100644 --- a/.claude/plans/gremlin-composable-algebra-v1.md +++ b/.claude/plans/gremlin-composable-algebra-v1.md @@ -142,7 +142,7 @@ Rows group steps of one mechanism, so percentages are of rows, not of step names | in(rel) reverse fk | traversal | V->V' | re-anchor on child rows | — | pure | DIRECT (G1-4b) | spelled `V(child).out(rel)`, anchored on the edge or child concept (tests/distinct.rs); the `V(parent).in(rel)` spelling is still refused (`FanOut`) | composer re-anchoring of the `in` spelling | | out/in(rel) M:N | traversal | V->V' | re-anchor on edge rows | — | pure | DIRECT (G1-4b) | `V(edge).where(p()).out(end)`: `count()` = bag, `dedup().count()` = set, both ends filtered at the source; anchored on the edge or child concept (tests/distinct.rs) | — | | both(rel) | traversal | V->V' | union of two bags | — | pure | YET | no bag union across anchors | typed bag merge | -| second hop, functional | traversal | V->V'' | keeps | — | pure | YET | #1311 `functional_two_hop_is_refused…` | composed fk read (G1) | +| second hop, functional | traversal | V->V'' | keeps | — | pure | DIRECT (G1-7) | `out(r1)…out(rk)` folds into ONE `MaskOp::GatherChain` (§11): tests/two_hop.rs vs a row loop over the text; R2IL refuses (no `GATHER`) | R2IL `GATHER` | | second fan-out (fan-in carried) | traversal | V->V'' | bag | — | pure | YET | #1311 `many_to_many_two_hop_is_refused…`; #1312 §D | barrier / fan-in operand (G2) | | outE/inE/bothE | traversal | V->E | edge rows | — | pure | DIRECT (G1-4b) | `V(edge).where(p())` with `p` over `out(src)`; anchored on the edge or child concept (tests/distinct.rs); `bothE` needs the union below | — | | outV/inV/otherV/bothV | traversal | E->V | fk read | — | pure | DIRECT (G1-4b) | `out(src)` / `out(dst)` from `V(edge)`: EqU32Via + liveness Semijoin; anchored on the edge or child concept (tests/distinct.rs) | — | @@ -250,7 +250,7 @@ adapter, minus the relations that fan out. test is emitted once per relation, wherever it is first named. - `count()` after `out(rel)` counts anchor rows: a functional relation yields at most one traverser per anchor row, so the anchor does not move. -- Refused by name: a second hop (`SecondHop`), `values(f)` on the target +- Refused by name (at §8; a second hop composes since §11): `values(f)` on the target (`ValueThroughRelation`; no sum through a key in Quack), `dedup()` on the target (`DedupThroughRelation`; a set count), `where(method())` on the target (`WhereThroughRelation`), and `in`/`both` (`FanOut`). @@ -330,8 +330,7 @@ Composer: `dedup()` after `out(rel)` moves to the distinct targets; `has` commutes with it; `count()` folds; ending there, or any other step, stays `DedupThroughRelation`. `V(parent).in(rel)` is still refused (`FanOut`); the same population is spelled `V(child).out(rel)`. R2IL: `FrontendGap { R2il, -"GATHER" }`. A second hop stays refused: it needs the chained-hop decision -(`.claude/board/entries/2026-10-03-coresearch-chained-hop.md`). +"GATHER" }`. A second hop composes since §11. Evidence: `tests/distinct.rs` (6 tests). FIXTURE: synthetic `method`, `field` and `reads` tables, dangling keys and dead rows on each, two @@ -342,6 +341,41 @@ test dropped, the test emitted twice, the fold lowered as `COUNT(*)`, the R2IL refusal dropped, `has` after `dedup` dropped, ending on the distinct targets accepted, the key left out of `required`. +## §11 — chained hops are folded hops (D-CMH-G1-7) + +`out(r1).out(r2)…` over functional relations keeps the anchor and lowers to +ONE chained gather: `Filter::Chain` → `MaskOp::GatherChain` → +`ndarray::simd::mask_gather_chain_u32[_under]`. A condition on any hop is a +test read in place at the row the key reached; no mask over a hop table is +built and no program feeds another (the photolithography pattern: each hop +is one more aperture in the same pass, materialised once at the terminal). + +Method layer: `Path` (1..=`MAX_PATH` = 8 relations) and +`Cond::ReachesPath` / `Cond::ViaPath` for a path of two or more; a one-hop +path is still `Cond::Reaches` / `Cond::Via`. Preparation walks each +relation on the previous one's target glove; every condition on a maximal +path and on its prefixes binds into one `BoundCond::Chain` (hop-1 tests +folded into hop 1, never a separate semijoin). The next key must be a +proven `Code` field of the hop table that the call supplies as a foreign +lane (`NoForeignLane` otherwise). Every target universe along the path is +checked at the call. + +Composer: `out` after `out` extends the path (`PathTooDeep` past +`MAX_PATH`); `has` after it is a `ViaPath`; `dedup`, `values`, `min`/`max` +after a path of two are refused. Text form and awareness graph carry the +path. + +Evidence: `tests/two_hop.rs` (5 tests): `line -billed_to-> partner +-located_in-> region`, dangling keys and dead rows at both depths, against a +row loop over the text at n ∈ {1, 64, 65, 4097}; the lowering is exactly one +chain; composer, text form, awareness, refusals. Six disable runs red: +prefix not absorbed, tests dropped, tests on the wrong hop, next key not +followed, only the first universe checked, the composer refusing the hop. +The executor side (mask-risc, Quack) has its own seven (board entry +`2026-10-12-chained-hops-folded.md`). + +OPEN: the M:N two-edge-table hop (an edge concept per hop, run-level OR). + ## D-ids | D-id | scope | status | @@ -352,6 +386,7 @@ targets accepted, the key left out of `required`. | D-CMH-G1-3 | Gremlin-shaped runtime declarations | Shipped (#1508) | | D-CMH-G1-4a | functional relation carrier: `out(rel)`, `out(rel).has`, `count()` after it (§8) | Shipped (#1514) | | D-CMH-G1-4b | edge table as the anchor (`E()`, M:N, `outE/inV`, `in` spelled from the child), `Fold::Distinct` (§10) | In PR | -| D-CMH-G1-4 | the rest: a second hop (needs the chained-hop decision), the `V(parent).in(rel)` spelling, `bothE` | Queued | +| D-CMH-G1-4 | the rest: the `V(parent).in(rel)` spelling, `bothE` | Queued | +| D-CMH-G1-7 | chained hops folded: `out(r1).out(r2)…` → one `GatherChain` (§11) | In PR | | D-CMH-G1-5 | FirstOfRun (insufficiency proof first; separate PR #3) | Queued | | D-CMH-G1-6 | ordered `I32` conditions, `MIN`/`MAX` folds; R2IL `MIN`/`MAX` wired (§9) | In PR |