From d4703f5bc85cc9c69f5beb08a904d2c40d611840 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 14 Oct 2026 20:56:48 +0000 Subject: [PATCH 01/13] =?UTF-8?q?plan:=20metta-polyfill-v1=20=E2=80=94=20H?= =?UTF-8?q?yperon/MeTTa=20as=20a=20front=20end=20over=20the=20masking-alge?= =?UTF-8?q?bra=20polyfill=20(D-MTP-0)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adoption map for hyperon-experimental: MeTTa is a declaration IR, its Datalog subset lowers through lance-graph-relalg -> R2IL -> mask-risc, and everything else stays in Hyperon's unchanged interpreter. The seam is a named LanceSpace implementing hyperon_space::Space; ModuleSpace splits conjunctions and requires ModuleSpace dependencies, so &self is not the seam, and recursive rules are declared into the named space. Measured on generated Datalog programs: PeTTa (SWI-Prolog) near-linear, Hyperon superlinear (anc_800 0.42 s vs 417 s). Records a second trie panic (expressions with >= 1,024 children). Recommends the crate name lance-graph-metta. D-MTP-1..5 queued, not authorized. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_011BMJnDBAfzcuQh5HeT39Rh --- .claude/board/AGENT_LOG.md | 6 + .claude/board/INTEGRATION_PLANS.md | 8 + .claude/board/STATUS_BOARD.md | 13 ++ .../entries/2026-10-14-metta-polyfill-map.md | 25 +++ .claude/board/entries/README.md | 3 +- .claude/plans/metta-polyfill-v1.md | 186 ++++++++++++++++++ 6 files changed, 240 insertions(+), 1 deletion(-) create mode 100644 .claude/board/entries/2026-10-14-metta-polyfill-map.md create mode 100644 .claude/plans/metta-polyfill-v1.md diff --git a/.claude/board/AGENT_LOG.md b/.claude/board/AGENT_LOG.md index 31ac79589..c934024c3 100644 --- a/.claude/board/AGENT_LOG.md +++ b/.claude/board/AGENT_LOG.md @@ -1,3 +1,9 @@ +## 2026-10-14 — D-MTP-0 harvest: six substrate-mapper runs, one reply each + +- Card `substrate-mapper` (run as a general-purpose agent with the card pasted; the card is not registered in this session), six units: `hyperon-atom`, `hyperon-space`, the minimal-MeTTa interpreter, the grounded stdlib, lance-graph `nal`/`r2il`/`relalg`, `quack`. +- Ledger: spot-checks `Space::query` L175, `complex_query` L340, `GroundingSpace` L17, interpreter dispatch L417/L453, `query` L604, `FoldDialect` L260 — all correct; 0 wrong. 1 missed: no map recorded that `ModuleSpace` splits conjunctions and requires `ModuleSpace` dependencies (`space/module.rs:33-48`); the orchestrator found it by reading the file in full. +- Cost: about 515k–646k subagent tokens per run. No conclusion came from a card. + ## 2026-10-14 — D-HOLO H6b harvest: four substrate-mapper runs, one reply each - Card `substrate-mapper`, four units: `holo_interval_probe.rs` (H6), mask-risc exec, mask-risc IR, ndarray `simd` mask re-exports. diff --git a/.claude/board/INTEGRATION_PLANS.md b/.claude/board/INTEGRATION_PLANS.md index a718e5bb1..86f39c0c8 100644 --- a/.claude/board/INTEGRATION_PLANS.md +++ b/.claude/board/INTEGRATION_PLANS.md @@ -1,3 +1,11 @@ +## 2026-10-14 (2) — metta-polyfill-v1 — Hyperon/MeTTa as a front end over the masking-algebra polyfill → `.claude/plans/metta-polyfill-v1.md` + +**Status:** PROPOSAL; D-MTP-0 (map + measurements) done, D-MTP-1..5 not authorized. + +- MeTTa is a declaration IR, not an execution IR: the Datalog subset lowers through `lance-graph-relalg` → R2IL → mask-risc; everything else stays in Hyperon's interpreter. +- Seam: a named `LanceSpace` implementing `hyperon_space::Space`; `ModuleSpace` splits conjunctions, so `&self` is not the seam. +- Measured PeTTa (SWI-Prolog) near-linear, Hyperon superlinear (anc_800: 0.42 s vs 417 s); new trie panic at ≥ 1,024 children. + ## 2026-10-14 — d-holo-address-indexed-algebra-v1 — falsifiers H1, H5, H6 for the address-indexed algebra → `.claude/plans/d-holo-address-indexed-algebra-v1.md` **Status:** PRE-REGISTERED; nothing measured. diff --git a/.claude/board/STATUS_BOARD.md b/.claude/board/STATUS_BOARD.md index af046bf12..bdb02ca79 100644 --- a/.claude/board/STATUS_BOARD.md +++ b/.claude/board/STATUS_BOARD.md @@ -1,3 +1,16 @@ +## D-MTP — Hyperon/MeTTa over the masking-algebra polyfill (2026-10-14) + +Plan: `.claude/plans/metta-polyfill-v1.md`. Proposed crate `lance-graph-metta` (not created). + +| D-id | scope | status | gate / falsifier | +|---|---|---|---| +| **D-MTP-0** | Adoption map, MeTTa-as-IR verdict, PeTTa vs Hyperon measurements, trie panic ≥ 1,024 children | Done 2026-10-14 (entry `2026-10-14-metta-polyfill-map.md`) | — | +| **D-MTP-1** | `join_n` lowered by hand through relalg/R2IL, n = 10²..10⁶, v4 | Queued (not authorized) | == Hyperon and PeTTa answers (count and bag); cost reported, no threshold before the run | +| **D-MTP-2** | `LanceSpace` (`Space + SpaceMut`) as a named space for the non-recursive Datalog subset; golden corpus rerun | Queued (not authorized) | 32/33 golden files still pass; refusals named; whole-conjunction vs per-conjunct calls recorded | +| **D-MTP-3** | Recursive Datalog rules → relalg μ through the external driver d′ | Queued (not authorized) | `anc_n` answers identical; no fixpoint state in mask-risc | +| **D-MTP-4** | Upstream fix for the trie panic, test first | Queued (needs operator's word) | red before, green after | +| **D-MTP-5** | PeTTa as second oracle in the probes | Queued (not authorized) | PeTTa/Hyperon divergences recorded, never averaged | + ## D-RA — Relational algebra over coordinate populations, a D-CMR prototype (2026-10-13) Plan: `.claude/plans/relalg-v1.md`. Crate: `crates/lance-graph-relalg` (own workspace root, excluded). Prototype for D-CMR-6 rules over residual comparisons and a lowering through R2IL. diff --git a/.claude/board/entries/2026-10-14-metta-polyfill-map.md b/.claude/board/entries/2026-10-14-metta-polyfill-map.md new file mode 100644 index 000000000..48afcabea --- /dev/null +++ b/.claude/board/entries/2026-10-14-metta-polyfill-map.md @@ -0,0 +1,25 @@ +# 2026-10-14 — D-MTP-0 — where Hyperon/MeTTa attaches to the masking-algebra polyfill + +**Status:** MEASURED (engine comparison, v4 host, single whole-run timings) + VERIFIED-IN-CODE +(Hyperon fork `1ffdbd6`). Plan `.claude/plans/metta-polyfill-v1.md`. No crate, no code. + +**Outcome:** +- MeTTa works as a declaration IR (rules, facts, queries as typed S-expressions, names resolved + once to addresses), not as the execution IR. Execution: MeTTa Datalog subset → `lance-graph-relalg` + → R2IL `FoldDialect` → mask-risc. Term construction, bags, grounded ops, types and state stay in + Hyperon's unchanged interpreter. +- Seam: `hyperon_space::Space::query` (`hyperon-space/src/lib.rs:175`). A space queried directly by + `match` gets the whole `(, …)` pattern (`stdlib/core.rs:163`); `ModuleSpace` splits it into single + conjuncts first and panics on a non-`ModuleSpace` dependency (`space/module.rs:33-48`). So the + adapter is a named space, never `&self` or a module dependency. +- Measured, same generated programs: two-hop join PeTTa 0.29 / 0.41 / 1.47 s vs Hyperon 0.11 / 5.9 s / + panic at n = 10² / 10³ / 10⁴; recursive ancestor PeTTa 0.29 / 0.33 / 0.42 s vs Hyperon + 0.28 / 7.4 / 417 s at n = 50 / 200 / 800. Answers correct wherever an engine finished. +- New upstream panic: an expression with ≥ 1,024 children passed to a MeTTa function aborts in + `TrieKey::start_expr` (`hyperon-space/src/index/trie.rs:539`, `TK_MAX_EXPRESSION_SIZE = 1 << 10`); + 1,023 children works. This stopped the n = 10⁴ join. +- Crate name: `lance-graph-metta` (one concern: front end + `Space` adapter); "cog" would mix in + PLN, the miner, attention and Classic AtomSpace (AGPL). + +**Open:** whether `match &self` can reach whole conjunctions without patching Hyperon; D-MTP-1..5 +not authorized; the trie panic is not reported upstream. diff --git a/.claude/board/entries/README.md b/.claude/board/entries/README.md index 9eb7728c6..bf2286c22 100644 --- a/.claude/board/entries/README.md +++ b/.claude/board/entries/README.md @@ -25,11 +25,12 @@ index row, (3) no duplicate entry id. Checks 1 and 2 are deliberately opposite directions; the stranding this convention prevents shows up in exactly one of them, never both. -330 entries, 2026-08-06 .. 2026-10-14. +331 entries, 2026-08-06 .. 2026-10-14. | date | entry id | finding | file | |---|---|---|---| | 2026-10-14 | `revision-layer-invariants` | | [2026-10-14-revision-layer-invariants.md](2026-10-14-revision-layer-invariants.md) | +| 2026-10-14 | `D-MTP-0` | where Hyperon/MeTTa attaches to the masking-algebra polyfill | [2026-10-14-metta-polyfill-map.md](2026-10-14-metta-polyfill-map.md) | | 2026-10-14 | `D-NALP-4` | | [2026-10-14-isa-truth-exact-codes.md](2026-10-14-isa-truth-exact-codes.md) | | 2026-10-14 | `D-HOLO-H6b` | normal forms on the Context × Operation tile (measured) | [2026-10-14-d-holo-h6b.md](2026-10-14-d-holo-h6b.md) | | 2026-10-14 | `D-HOLO` | | [2026-10-14-d-holo-falsifiers.md](2026-10-14-d-holo-falsifiers.md) | diff --git a/.claude/plans/metta-polyfill-v1.md b/.claude/plans/metta-polyfill-v1.md new file mode 100644 index 000000000..a49b995e7 --- /dev/null +++ b/.claude/plans/metta-polyfill-v1.md @@ -0,0 +1,186 @@ +# metta-polyfill-v1 — Hyperon/MeTTa over the masking-algebra polyfill + +**Status:** PROPOSAL (D-MTP-0..5). Nothing beyond D-MTP-0 (this map and its +measurements) is authorized. No crate exists yet. +**Date:** 2026-10-14. +**Repos:** lance-graph (`lance-graph-relalg`, `lance-graph-r2il`, +`lance-graph-mask-risc`, `lance-graph-quack`, `lance-graph-nal`), +`AdaWorldAPI/hyperon-experimental` (fork, MIT), `AdaWorldAPI/petta` (MIT). +**Parents:** `.arxiv/2026-10_hyperon_metta_to_lance_graph_parity_map.md` +(guardrails, HY-Q01..15, probes P0–P7), `nal-polyfill-v1.md` (the template: +text front end → bind → procedures → `FoldDialect` → mask-risc), +`lance-graph-relalg` (#1546, the describing IR). + +## 1. The question + +Can MeTTa serve as an IR for the masking algebra, and where does +hyperon-experimental attach so that the atoms MeTTa uses are served by the +polyfill rather than by Hyperon's own trie? Should the crate be +`lance-graph-cog` or `lance-graph-metta`? + +## 2. What Hyperon is, at the seams (VERIFIED-IN-CODE, fork `1ffdbd6`) + +- **Atoms:** `Symbol`, `Variable`, `Grounded`, `Expression` + (`hyperon-atom/src/lib.rs`), matched by `match_atoms` / `Bindings` + (`matcher.rs`). +- **Interpreter:** minimal MeTTa is one dispatch over 12 instruction symbols + (`lib/src/metta/interpreter.rs:415-458`: `eval`, `evalc`, `chain`, + `function`, `collapse-bind`, `unify`, `decons-atom`, `cons-atom`, + `superpose-bind`, `metta`, `context-space`, `call-native`; `function` is an + internal state that panics if reached). The full language (types, `let`, + `if`, `case`, …) is MeTTa code in `stdlib.metta` plus grounded ops on top. +- **The space seam:** `hyperon_space::Space::query(&self, &Atom) -> + BindingsSet` (`hyperon-space/src/lib.rs:175`) and `SpaceMut` + (`add`/`remove`/`replace`, `:219`). The interpreter reaches data only + through `query` (`interpreter.rs:604`, `eval_impl` → `query`). +- **Conjunctions:** `GroundingSpace::query` (`lib/src/space/grounding/mod.rs:147`) + calls `complex_query` (`hyperon-space/src/lib.rs:340`), which splits + `(, q1 q2 …)` and joins **tuple at a time**: for every binding of the + conjuncts so far it substitutes into the next conjunct and probes the trie + again (`:345-368`). The trie (`index/trie.rs`) is the only index. + +## 3. Measured (2026-10-14, v4 host, whole-run seconds; engine comparison, not a primitive timing) + +Two Datalog-shaped MeTTa programs, generated, identical for both engines: +`join_n` = n facts `(parent nᵢ n₍₇ᵢ₊₃₎ mod n)` and the two-hop conjunctive +query `(match &self (, (parent $x $y) (parent $y $z)) ($x $z))`, counted; +`anc_n` = a chain of n−1 `parent` facts and the recursive ancestor rule, +`(anc n0 $z)` counted. Both engines return the correct counts wherever they +finish (100 / 1,000 / 10,000 pairs; 49 / 199 / 799 ancestors). + +| program | PeTTa (SWI-Prolog 10.0.2), total | Hyperon (fork, release, native), run | +|---|---|---| +| startup only (`!(+ 1 1)`) | 0.32 | 0.000 (init 0.016–0.020) | +| join_100 | 0.29 | 0.11 | +| join_1000 | 0.41 | 5.92 | +| join_10000 | 1.47 | **panic** (`trie.rs:539`, see §3.1) | +| anc_50 | 0.29 | 0.28 | +| anc_200 | 0.33 | 7.37 | +| anc_800 | 0.42 | 416.9 | + +PeTTa compiles MeTTa to Prolog clauses and stays near its startup cost; Hyperon +grows about n^1.7 on the join (×54 for ×10) and n^2.4–2.9 on the recursion +(×26 then ×56 per ×4). Single runs; the ratios are far outside the 40 % noise +band, the absolute seconds are not a primitive claim. + +### 3.1 A second Hyperon trie bug (upstream, sibling of #1107) + +Any expression with 1,024 or more children passed to a MeTTa-defined function +panics: `(let $r (0 … 1022) (size-atom $r))` returns 1023; the same with 1,024 +items aborts with `assertion failed: size < TK_MAX_EXPRESSION_SIZE` +(`hyperon-space/src/index/trie.rs:539`, `TrieKey::start_expr`; +`TK_MAX_EXPRESSION_SIZE = 1 << 10`, `:511`). The interpreter queries the +space for `(= (let $r …) $X)`, and the trie cannot key an expression +that wide. `(size-atom (0 … 1023))` called directly does not panic, because a +grounded op is executed without a space query. This is what stopped join_10000 +(its `collapse` result has 10,000 children). Not fixed here. + +## 4. Can MeTTa work as an IR? + +**As a declaration IR: yes. As the execution IR: no.** + +- MeTTa expressions are typed S-expressions over four atom kinds; rules + (`=`), facts and queries (`match`, `(, …)`) are data. That is what the + stack needs from a front end: text parsed once, names resolved through the + label codebook to addresses (nal-polyfill §4 H), nothing after the binder + holding a string. +- Minimal MeTTa is an interpreter for nondeterministic, tuple-at-a-time + rewriting (`superpose-bind`, `collapse-bind`, `cons-atom`, + `decons-atom`, `call-native`). It constructs new terms and runs grounded + host code. That is the opposite of a folded mask program, and §3 shows what + executing it naively costs. +- So the arrangement is the one NAL and SQL already use: **MeTTa is a + front end; `lance-graph-relalg` describes; R2IL / `FoldDialect` executes; + mask-risc is the only engine.** Three tiers of the language map to three + outcomes: + +| MeTTa fragment | lowers to | outcome | +|---|---|---| +| facts in a space, `match` over one pattern, `(, …)` conjunctions with shared variables, no new terms | `Rel` selections per conjunct, joins on shared variables (`EqU32Via` / `Gather`), one fold, bindings only at the terminal | **polyfill** (one program per query, not one trie probe per binding) | +| recursive `=` rules that are Datalog (no function symbols in heads, no grounded calls) | relalg `μ` driven by the external driver d′ (`2026-10-03-coresearch-chained-hop.md`: recursion stays outside mask-risc), each iteration one fold | **polyfill, gated** (D-MTP-3) | +| term construction, `superpose`/`collapse` bags, grounded ops, dependent types, state | Hyperon's own interpreter, unchanged | **stays in Hyperon** (typed refusal on our side, never approximated) | + +## 5. Where it attaches + +```text +MeTTa text ─► hyperon parser ─► Hyperon interpreter (fork, unchanged, 12 ops) + │ Space::query(&Atom) / SpaceMut + ▼ + lance-graph-metta: LanceSpace (impl Space + SpaceMut) + atoms → addresses (label codebook), facts → rows + query atom → Rel (σ per conjunct, ⋈ on shared vars) + ▼ + lance-graph-relalg (normalize, symbolic ∅ / count) + ▼ + lance-graph-r2il FoldDialect ─► mask-risc ─► ndarray::simd +``` + +- **The seam is `Space`, not the interpreter.** `complex_query` is a helper a + space chooses to call, not part of the trait, so a `LanceSpace` that is + queried directly receives the whole `(, q1 q2 …)` atom and can lower the full + conjunction into one program. +- **But only as a named space, never inside `&self`.** `ModuleSpace` + (`lib/src/space/module.rs:33-48`) runs `complex_query` itself and hands its + main space and every dependency one conjunct at a time; a dependency that is + not a `ModuleSpace` panics (`:44`). So `LanceSpace` is bound as its own token + (`!(bind! &kb (new-lance-space …))`, `(match &kb (, …) …)`), and the module's + `&self` stays an ordinary `GroundingSpace` holding rules, grounded atoms and + everything outside the Datalog subset. `match` hands its pattern unchanged to + the named space's `query` (`lib/src/metta/runner/stdlib/core.rs:163`). + Wrapped in a `ModuleSpace`, it would degrade to one fold per conjunct per + binding, which is the cost §3 measures. Whether `match &self` can reach the + conjunction without patching Hyperon is open (D-MTP-2). +- **Recursive rules live in the named space too.** The interpreter evaluates + `=` rules by querying the context space (`interpreter.rs:604`), i.e. `&self`, + one call per reduction step. A recursive Datalog rule left there never + reaches the polyfill. D-MTP-3 therefore stores the rules as algebra values in + `LanceSpace` and answers a pattern over the derived relation + (`(match &kb (anc n0 $z) $z)`) with the closure computed by d′, not by + `=` rewriting. +- **Atoms are addresses, not strings.** A `Symbol` resolves once to a + `classid × NodeGuid` (or a term id within a class); an `Expression` fact of + fixed shape `(functor a₁ … aₖ)` is a row of a class keyed by the functor and + arity, one lane per argument. `Variable` never reaches a row. Shapes the + class does not declare are refused, never stored as text. +- **Multiplicity is part of the answer.** `BindingsSet` is a bag; the + terminal must emit one `Bindings` per match (the `b4_nondeterm` kill + condition in the parity map). A row mask alone is not an answer. +- **The NAL polyfill is the precedent and stays separate.** PeTTa's + `lib_nars.metta` / `lib_pln.metta` are oracles; NAL operations go through + `lance-graph-nal` (`nal::{Op}`, global core methods, no classid plug), not + through MeTTa rules. + +## 6. `lance-graph-cog` or `lance-graph-metta` + +Recommendation: **`lance-graph-metta`**, workspace-excluded like +`lance-graph-relalg`, depending on the fork's `hyperon-atom` and +`hyperon-space` only (not the interpreter crate). + +- It holds exactly one concern: the MeTTa front end and the `Space` adapter. +- "cog" would pull in OpenCog Classic AtomSpace, PLN, the miner and + attention, which the parity map assigns elsewhere (#1497 owns + scheduling/attention; Classic AtomSpace is AGPL and a different engine). +- An earlier note proposed a separate `metta-rs` repo. That fits a full + transcode of the interpreter; this plan does not transcode it, so a crate + beside relalg is enough. Revisit if the interpreter itself is ever ported. + +## 7. Deliverables + +| D-id | scope | gate | +|---|---|---| +| D-MTP-0 | this map, measurements §3, bug §3.1 | — (done) | +| D-MTP-1 | pre-registered probe: `join_n` lowered by hand through relalg/R2IL against the Hyperon and PeTTa answers, n = 10²..10⁶, v4, ns per row, build and terminal charged | answers identical (count and bag of pairs); cost reported, no threshold claimed before the run | +| D-MTP-2 | `LanceSpace` adapter (`Space + SpaceMut`) for the non-recursive Datalog subset, bound as a named space; Hyperon's golden corpus (`hyperon-probe/golden`, 32 of 33 passing) rerun with facts moved into it | every golden file that passes today still passes; refusals named; the probe records whether the space received whole conjunctions or single conjuncts | +| D-MTP-3 | recursive Datalog rules declared into `LanceSpace` → relalg `μ` through d′, queried by `match` over the derived relation | `anc_n` answers identical; no fixpoint state inside mask-risc | +| D-MTP-4 | upstream: the §3.1 panic as an issue/PR on the fork, test first | red before, green after; upstream only with the operator's word | +| D-MTP-5 | PeTTa as the second oracle in the probes (needs SWI-Prolog ≥ 9.3; 10.0.2 installed from the official PPA) | divergences between PeTTa and Hyperon recorded, never averaged | + +## 8. Fences + +- No second engine or interpreter inside lance-graph; no new opcode; no + MeTTa text past the binder. +- Hyperon's interpreter is not modified for this; the fork's upstream fixes + are separate PRs. +- An unsupported shape is a typed refusal answered by the in-memory space, + never an approximation. +- Licenses: Hyperon and PeTTa MIT; no Classic AtomSpace (AGPL) code. From 68d40b203b69ff44f4a1a082b653ef2fa3d2098b Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 14 Oct 2026 21:00:25 +0000 Subject: [PATCH 02/13] =?UTF-8?q?plan:=20metta-polyfill-v1=20=E2=80=94=20s?= =?UTF-8?q?eparate=20the=20expression-width=20panic=20from=20the=20id-deco?= =?UTF-8?q?ding=20fix=201ffdbd6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_011BMJnDBAfzcuQh5HeT39Rh --- .claude/board/STATUS_BOARD.md | 2 +- .../board/entries/2026-10-14-metta-polyfill-map.md | 5 +++-- .claude/plans/metta-polyfill-v1.md | 12 +++++++++--- 3 files changed, 13 insertions(+), 6 deletions(-) diff --git a/.claude/board/STATUS_BOARD.md b/.claude/board/STATUS_BOARD.md index bdb02ca79..8bdf5d7b4 100644 --- a/.claude/board/STATUS_BOARD.md +++ b/.claude/board/STATUS_BOARD.md @@ -8,7 +8,7 @@ Plan: `.claude/plans/metta-polyfill-v1.md`. Proposed crate `lance-graph-metta` ( | **D-MTP-1** | `join_n` lowered by hand through relalg/R2IL, n = 10²..10⁶, v4 | Queued (not authorized) | == Hyperon and PeTTa answers (count and bag); cost reported, no threshold before the run | | **D-MTP-2** | `LanceSpace` (`Space + SpaceMut`) as a named space for the non-recursive Datalog subset; golden corpus rerun | Queued (not authorized) | 32/33 golden files still pass; refusals named; whole-conjunction vs per-conjunct calls recorded | | **D-MTP-3** | Recursive Datalog rules → relalg μ through the external driver d′ | Queued (not authorized) | `anc_n` answers identical; no fixpoint state in mask-risc | -| **D-MTP-4** | Upstream fix for the trie panic, test first | Queued (needs operator's word) | red before, green after | +| **D-MTP-4** | Fork fix for the expression-width trie panic (≥ 1,024 children), test first; distinct from the id-decoding fix `1ffdbd6` | Queued (needs operator's word) | red before, green after | | **D-MTP-5** | PeTTa as second oracle in the probes | Queued (not authorized) | PeTTa/Hyperon divergences recorded, never averaged | ## D-RA — Relational algebra over coordinate populations, a D-CMR prototype (2026-10-13) diff --git a/.claude/board/entries/2026-10-14-metta-polyfill-map.md b/.claude/board/entries/2026-10-14-metta-polyfill-map.md index 48afcabea..ead3b7ce0 100644 --- a/.claude/board/entries/2026-10-14-metta-polyfill-map.md +++ b/.claude/board/entries/2026-10-14-metta-polyfill-map.md @@ -15,11 +15,12 @@ - Measured, same generated programs: two-hop join PeTTa 0.29 / 0.41 / 1.47 s vs Hyperon 0.11 / 5.9 s / panic at n = 10² / 10³ / 10⁴; recursive ancestor PeTTa 0.29 / 0.33 / 0.42 s vs Hyperon 0.28 / 7.4 / 417 s at n = 50 / 200 / 800. Answers correct wherever an engine finished. -- New upstream panic: an expression with ≥ 1,024 children passed to a MeTTa function aborts in +- New panic, separate from the id-decoding fix already on the fork (`1ffdbd6`, branch + `fix-trie-key-value`, on which these runs were taken): an expression with ≥ 1,024 children passed to a MeTTa function aborts in `TrieKey::start_expr` (`hyperon-space/src/index/trie.rs:539`, `TK_MAX_EXPRESSION_SIZE = 1 << 10`); 1,023 children works. This stopped the n = 10⁴ join. - Crate name: `lance-graph-metta` (one concern: front end + `Space` adapter); "cog" would mix in PLN, the miner, attention and Classic AtomSpace (AGPL). **Open:** whether `match &self` can reach whole conjunctions without patching Hyperon; D-MTP-1..5 -not authorized; the trie panic is not reported upstream. +not authorized; the expression-width panic has no fix yet. diff --git a/.claude/plans/metta-polyfill-v1.md b/.claude/plans/metta-polyfill-v1.md index a49b995e7..5ea785a3f 100644 --- a/.claude/plans/metta-polyfill-v1.md +++ b/.claude/plans/metta-polyfill-v1.md @@ -63,7 +63,13 @@ grows about n^1.7 on the join (×54 for ×10) and n^2.4–2.9 on the recursion (×26 then ×56 per ×4). Single runs; the ratios are far outside the 40 % noise band, the absolute seconds are not a primitive claim. -### 3.1 A second Hyperon trie bug (upstream, sibling of #1107) +### 3.1 A second Hyperon trie bug (distinct from the fix already on the fork) + +The measurements above ran on the fork's `fix-trie-key-value` branch (`1ffdbd6`), +which already carries the earlier fix: `TrieKey::value()` decoded ids ≥ 1,024 +wrongly (`& (MASK - 1024)` instead of `- 1024`), so a space with more than 1,024 +distinct atoms returned wrong atoms or panicked. That fix is in place and is not +what follows. The panic below is a separate limit that shares the constant 1,024. Any expression with 1,024 or more children passed to a MeTTa-defined function panics: `(let $r (0 … 1022) (size-atom $r))` returns 1023; the same with 1,024 @@ -73,7 +79,7 @@ items aborts with `assertion failed: size < TK_MAX_EXPRESSION_SIZE` space for `(= (let $r …) $X)`, and the trie cannot key an expression that wide. `(size-atom (0 … 1023))` called directly does not panic, because a grounded op is executed without a space query. This is what stopped join_10000 -(its `collapse` result has 10,000 children). Not fixed here. +(its `collapse` result has 10,000 children). Not fixed here; reproduced on `1ffdbd6`. ## 4. Can MeTTa work as an IR? @@ -172,7 +178,7 @@ Recommendation: **`lance-graph-metta`**, workspace-excluded like | D-MTP-1 | pre-registered probe: `join_n` lowered by hand through relalg/R2IL against the Hyperon and PeTTa answers, n = 10²..10⁶, v4, ns per row, build and terminal charged | answers identical (count and bag of pairs); cost reported, no threshold claimed before the run | | D-MTP-2 | `LanceSpace` adapter (`Space + SpaceMut`) for the non-recursive Datalog subset, bound as a named space; Hyperon's golden corpus (`hyperon-probe/golden`, 32 of 33 passing) rerun with facts moved into it | every golden file that passes today still passes; refusals named; the probe records whether the space received whole conjunctions or single conjuncts | | D-MTP-3 | recursive Datalog rules declared into `LanceSpace` → relalg `μ` through d′, queried by `match` over the derived relation | `anc_n` answers identical; no fixpoint state inside mask-risc | -| D-MTP-4 | upstream: the §3.1 panic as an issue/PR on the fork, test first | red before, green after; upstream only with the operator's word | +| D-MTP-4 | the §3.1 expression-width panic as a fix on the fork, test first (separate from `1ffdbd6`) | red before, green after; upstream only with the operator's word | | D-MTP-5 | PeTTa as the second oracle in the probes (needs SWI-Prolog ≥ 9.3; 10.0.2 installed from the official PPA) | divergences between PeTTa and Hyperon recorded, never averaged | ## 8. Fences From da76da3a89cfe23d5b8503853043c97077121caf Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 14 Oct 2026 21:30:49 +0000 Subject: [PATCH 03/13] =?UTF-8?q?plan:=20metta-polyfill-v1=20=E2=80=94=20d?= =?UTF-8?q?atafrog=20Rust=20Datalog=20oracle,=20measured=20(D-MTP-6)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_011BMJnDBAfzcuQh5HeT39Rh --- .claude/board/STATUS_BOARD.md | 1 + .../entries/2026-10-14-metta-polyfill-map.md | 3 +++ .claude/plans/metta-polyfill-v1.md | 23 +++++++++++++++++++ 3 files changed, 27 insertions(+) diff --git a/.claude/board/STATUS_BOARD.md b/.claude/board/STATUS_BOARD.md index 8bdf5d7b4..9d4289049 100644 --- a/.claude/board/STATUS_BOARD.md +++ b/.claude/board/STATUS_BOARD.md @@ -9,6 +9,7 @@ Plan: `.claude/plans/metta-polyfill-v1.md`. Proposed crate `lance-graph-metta` ( | **D-MTP-2** | `LanceSpace` (`Space + SpaceMut`) as a named space for the non-recursive Datalog subset; golden corpus rerun | Queued (not authorized) | 32/33 golden files still pass; refusals named; whole-conjunction vs per-conjunct calls recorded | | **D-MTP-3** | Recursive Datalog rules → relalg μ through the external driver d′ | Queued (not authorized) | `anc_n` answers identical; no fixpoint state in mask-risc | | **D-MTP-4** | Fork fix for the expression-width trie panic (≥ 1,024 children), test first; distinct from the id-decoding fix `1ffdbd6` | Queued (needs operator's word) | red before, green after | +| **D-MTP-6** | datafrog as the exact Rust oracle (probe/test only) | Measured 2026-10-14 (v4): join 10⁶ in 65–69 ns/pair, seeded ancestors 10⁶ in 286–298 ns/tuple; counts match; not yet a committed probe | bag-of-pairs equality with Hyperon and PeTTa | | **D-MTP-5** | PeTTa as second oracle in the probes | Queued (not authorized) | PeTTa/Hyperon divergences recorded, never averaged | ## D-RA — Relational algebra over coordinate populations, a D-CMR prototype (2026-10-13) diff --git a/.claude/board/entries/2026-10-14-metta-polyfill-map.md b/.claude/board/entries/2026-10-14-metta-polyfill-map.md index ead3b7ce0..bf6874d7c 100644 --- a/.claude/board/entries/2026-10-14-metta-polyfill-map.md +++ b/.claude/board/entries/2026-10-14-metta-polyfill-map.md @@ -19,6 +19,9 @@ `fix-trie-key-value`, on which these runs were taken): an expression with ≥ 1,024 children passed to a MeTTa function aborts in `TrieKey::start_expr` (`hyperon-space/src/index/trie.rs:539`, `TK_MAX_EXPRESSION_SIZE = 1 << 10`); 1,023 children works. This stopped the n = 10⁴ join. +- Rust Datalog oracle: datafrog 2 (v4, best of 5) runs join 10⁶ in 65–69 ms (65–69 ns/pair) and the + seeded ancestor query 10⁶ in 0.29 s; join_10000 in 0.5 ms against PeTTa 1.47 s and Hyperon > 600 s. + Oracle and comparison bar only, never a production engine (hash/sort joins beside mask-risc). - Crate name: `lance-graph-metta` (one concern: front end + `Space` adapter); "cog" would mix in PLN, the miner, attention and Classic AtomSpace (AGPL). diff --git a/.claude/plans/metta-polyfill-v1.md b/.claude/plans/metta-polyfill-v1.md index 5ea785a3f..0c4d79614 100644 --- a/.claude/plans/metta-polyfill-v1.md +++ b/.claude/plans/metta-polyfill-v1.md @@ -63,6 +63,28 @@ grows about n^1.7 on the join (×54 for ×10) and n^2.4–2.9 on the recursion (×26 then ×56 per ×4). Single runs; the ratios are far outside the 40 % noise band, the absolute seconds are not a primitive claim. +### 3.0 A Rust Datalog oracle (datafrog 2, measured 2026-10-14) + +`datafrog` (the semi-naive engine behind Polonius; no AdaWorldAPI fork exists, +so the crates.io release is used) on the same generated programs, release, +`target-cpu=native`, program-printed tier `avx512f=true`, best of 5, three runs +within 5–15 %. Relation construction (sorting) is charged. Counts asserted +against the known answers, which match Hyperon's and PeTTa's. + +| program | datafrog | per output tuple | +|---|---|---| +| join, n = 10² / 10⁴ / 10⁶ | 3 µs / 0.51–0.62 ms / 65–69 ms | 33 / 51–62 / 65–69 ns | +| ancestors from n0 (the MeTTa query, seeded), n = 800 / 10⁴ / 10⁶ | 0.14 ms / 2.2 ms / 0.29 s | 179–185 / 222 / 286–298 ns | +| full closure on the chain, n = 2,000 (2.0 M tuples) | 0.28 s | 139–143 ns | + +It finishes join_10000 in 0.5 ms where Hyperon does not finish in 600 s and +PeTTa needs 1.47 s (mostly its 0.32 s startup plus compilation). The seeded +ancestor query costs more per tuple than the join because a chain derives one +tuple per semi-naive round, so per-round overhead dominates. These are +per-tuple engine costs on hash-free sorted merges, not mask-risc primitives; +they are the bar a folded lowering is compared against, not a production path. +Probe source: local crate `/home/user/datalog-oracle-probe` (not committed). + ### 3.1 A second Hyperon trie bug (distinct from the fix already on the fork) The measurements above ran on the fork's `fix-trie-key-value` branch (`1ffdbd6`), @@ -179,6 +201,7 @@ Recommendation: **`lance-graph-metta`**, workspace-excluded like | D-MTP-2 | `LanceSpace` adapter (`Space + SpaceMut`) for the non-recursive Datalog subset, bound as a named space; Hyperon's golden corpus (`hyperon-probe/golden`, 32 of 33 passing) rerun with facts moved into it | every golden file that passes today still passes; refusals named; the probe records whether the space received whole conjunctions or single conjuncts | | D-MTP-3 | recursive Datalog rules declared into `LanceSpace` → relalg `μ` through d′, queried by `match` over the derived relation | `anc_n` answers identical; no fixpoint state inside mask-risc | | D-MTP-4 | the §3.1 expression-width panic as a fix on the fork, test first (separate from `1ffdbd6`) | red before, green after; upstream only with the operator's word | +| D-MTP-6 | datafrog as the exact Rust oracle in D-MTP-1/3 probes and the `ruff_metta_spo` lowerable tag (test-only dependency) | answers identical to Hyperon and PeTTa as bags of pairs, not only counts | | D-MTP-5 | PeTTa as the second oracle in the probes (needs SWI-Prolog ≥ 9.3; 10.0.2 installed from the official PPA) | divergences between PeTTa and Hyperon recorded, never averaged | ## 8. Fences From fb43a4ec4a32993d430ad025ebf8e4f5e6d84f72 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 14 Oct 2026 21:32:19 +0000 Subject: [PATCH 04/13] =?UTF-8?q?plan:=20metta-polyfill-v1=20=E2=80=94=20d?= =?UTF-8?q?atafrog=20consumed=20from=20the=20AdaWorldAPI=20fork?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_011BMJnDBAfzcuQh5HeT39Rh --- .claude/board/entries/2026-10-14-metta-polyfill-map.md | 2 +- .claude/plans/metta-polyfill-v1.md | 8 +++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/.claude/board/entries/2026-10-14-metta-polyfill-map.md b/.claude/board/entries/2026-10-14-metta-polyfill-map.md index bf6874d7c..961110664 100644 --- a/.claude/board/entries/2026-10-14-metta-polyfill-map.md +++ b/.claude/board/entries/2026-10-14-metta-polyfill-map.md @@ -19,7 +19,7 @@ `fix-trie-key-value`, on which these runs were taken): an expression with ≥ 1,024 children passed to a MeTTa function aborts in `TrieKey::start_expr` (`hyperon-space/src/index/trie.rs:539`, `TK_MAX_EXPRESSION_SIZE = 1 << 10`); 1,023 children works. This stopped the n = 10⁴ join. -- Rust Datalog oracle: datafrog 2 (v4, best of 5) runs join 10⁶ in 65–69 ms (65–69 ns/pair) and the +- Rust Datalog oracle: datafrog 2.0.1 from the fork `AdaWorldAPI/datafrog` (v4, best of 5) runs join 10⁶ in 65–69 ms (65–69 ns/pair) and the seeded ancestor query 10⁶ in 0.29 s; join_10000 in 0.5 ms against PeTTa 1.47 s and Hyperon > 600 s. Oracle and comparison bar only, never a production engine (hash/sort joins beside mask-risc). - Crate name: `lance-graph-metta` (one concern: front end + `Space` adapter); "cog" would mix in diff --git a/.claude/plans/metta-polyfill-v1.md b/.claude/plans/metta-polyfill-v1.md index 0c4d79614..4d87cb46a 100644 --- a/.claude/plans/metta-polyfill-v1.md +++ b/.claude/plans/metta-polyfill-v1.md @@ -65,8 +65,8 @@ band, the absolute seconds are not a primitive claim. ### 3.0 A Rust Datalog oracle (datafrog 2, measured 2026-10-14) -`datafrog` (the semi-naive engine behind Polonius; no AdaWorldAPI fork exists, -so the crates.io release is used) on the same generated programs, release, +`datafrog` (the semi-naive engine behind Polonius), consumed from the fork +`AdaWorldAPI/datafrog` (`master`, `edd7cfc`, version 2.0.1), on the same generated programs, release, `target-cpu=native`, program-printed tier `avx512f=true`, best of 5, three runs within 5–15 %. Relation construction (sorting) is charged. Counts asserted against the known answers, which match Hyperon's and PeTTa's. @@ -83,7 +83,9 @@ ancestor query costs more per tuple than the join because a chain derives one tuple per semi-naive round, so per-round overhead dominates. These are per-tuple engine costs on hash-free sorted merges, not mask-risc primitives; they are the bar a folded lowering is compared against, not a production path. -Probe source: local crate `/home/user/datalog-oracle-probe` (not committed). +First run against the crates.io 2.0.1 release, re-run against the fork: every count +identical, timings within the noise band (join 10⁶ 65.0 ns/pair; seeded ancestors +10⁶ 0.31 s). Probe source: local crate `/home/user/datalog-oracle-probe` (not committed). ### 3.1 A second Hyperon trie bug (distinct from the fix already on the fork) From eecfe209697419a53abe05a833404848aacb3d2e Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 14 Oct 2026 21:35:58 +0000 Subject: [PATCH 05/13] =?UTF-8?q?plan:=20metta-polyfill-v1=20=E2=80=94=20p?= =?UTF-8?q?re-register=20D-MTP-1=20(join=20via=20Quack=20Semijoin,=20dataf?= =?UTF-8?q?rog=20oracle)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_011BMJnDBAfzcuQh5HeT39Rh --- .claude/plans/metta-polyfill-v1.md | 51 ++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/.claude/plans/metta-polyfill-v1.md b/.claude/plans/metta-polyfill-v1.md index 4d87cb46a..fa1ecea49 100644 --- a/.claude/plans/metta-polyfill-v1.md +++ b/.claude/plans/metta-polyfill-v1.md @@ -194,6 +194,57 @@ Recommendation: **`lance-graph-metta`**, workspace-excluded like transcode of the interpreter; this plan does not transcode it, so a crate beside relalg is enough. Revisit if the interpreter itself is ever ported. +## 6a. D-MTP-1 pre-registration (2026-10-14, before any code) + +**Correction to §4:** `lance-graph-relalg` cannot lower the join: its own +crate docs say "recursion (`μ`) and the hop (preimage) are not built". The +existing join surface is Quack's `Filter::Semijoin { fk, foreign }`, which +lowers to mask-risc `MaskOp::Gather` over a RESIDENT foreign plane. D-MTP-1 +uses that path; relalg gains the hop later or not at all. + +**Layout (ingest, charged separately).** Symbols resolve once to addresses. +The `parent` facts become a table over the address space of their first +argument: row `a` holds the fact whose subject has address `a`, the `dst` lane +holds the object's address (a coordinate, not a key), and a `valid` plane marks +the addresses that have a fact. Functional relations only: one fact per +subject. A second fact for the same subject is refused at ingest, not +approximated (one-to-many needs `ScatterOrU32`/a CSR and is out of scope here). + +**Query.** `(match &kb (, (parent $x $y) (parent $y $z)) ($x $z))` lowers to +`Filter::and([Plane(valid), Semijoin { fk: dst, foreign: valid }])`. Two +terminals: `Agg::Count` (the MeTTa `size-atom` of the collapsed result) and +`Agg::Rows` (`Keep`) followed by binding emission `(x, dst[dst[x]])` over the +kept bits only — the one place pairs are materialized. + +**Fixtures** (generated, identical in all engines): +- F1 `join_n`: `parent(i, (7i+3) mod n)` for every `i` (total, every row + survives; count = n). +- F2 `join_n_partial`: the same, only for `i mod 3 ≠ 0` (the semijoin rejects + rows whose target has no fact; count < n). +- n = 10², 10³, 10⁴, 10⁵, 10⁶. + +**Oracles.** datafrog (fork) at every n: the emitted pairs as a sorted vector +must equal datafrog's, not only the count. Hyperon and PeTTa on F1 and F2 at +n = 10², 10³ (and PeTTa 10⁴): counts must match. + +**Disable runs** (commit first, assert each patch applied once): +1. drop the `Semijoin` leaf → F2 count and pairs must go red (F1 stays green, + which is why F2 exists); +2. emit `(x, dst[x])` instead of `(x, dst[dst[x]])` → pairs red on F1 and F2. + +**Measured, reported, no threshold claimed before the run** (v4 arm, +program-printed tier, best of 5): ingest ns/fact; count program ns/row; +Rows + emission ns/pair; datafrog ns/pair beside it (its relation build +charged, as in §3.0). Expected only as a direction, not a gate: the folded +count costs a small fraction of a nanosecond per row, emission a few ns per +pair, both below datafrog's 65 ns/pair, because ingest has already turned +the join key into a coordinate. If ingest is charged to the query, the +comparison is reported both ways. + +**Home.** First probe of `crates/lance-graph-metta` (own workspace root, +excluded, like `lance-graph-relalg`), so the datafrog git dev-dependency never +enters the main workspace's resolution. + ## 7. Deliverables | D-id | scope | gate | From 78f13cec63d3eb81508ac3169d5d402274d3151d Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 14 Oct 2026 21:37:31 +0000 Subject: [PATCH 06/13] =?UTF-8?q?lance-graph-metta:=20D-MTP-1=20join=20pro?= =?UTF-8?q?be=20=E2=80=94=20MeTTa=20two-hop=20join=20via=20Quack=20Semijoi?= =?UTF-8?q?n,=20datafrog=20oracle?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_011BMJnDBAfzcuQh5HeT39Rh --- Cargo.toml | 3 + crates/lance-graph-metta/Cargo.toml | 24 ++ .../lance-graph-metta/examples/join_probe.rs | 238 ++++++++++++++++++ crates/lance-graph-metta/src/lib.rs | 7 + 4 files changed, 272 insertions(+) create mode 100644 crates/lance-graph-metta/Cargo.toml create mode 100644 crates/lance-graph-metta/examples/join_probe.rs create mode 100644 crates/lance-graph-metta/src/lib.rs diff --git a/Cargo.toml b/Cargo.toml index f7b0de8d7..22d085929 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -115,6 +115,9 @@ exclude = [ # lance-graph-r2il like lance-graph-nal. Own workspace root; verify via # `cargo test --manifest-path crates/lance-graph-relalg/Cargo.toml`. "crates/lance-graph-relalg", + # D-MTP-1: its own workspace root (datafrog oracle is a git dev-dependency). + # `cargo test --manifest-path crates/lance-graph-metta/Cargo.toml`. + "crates/lance-graph-metta", "crates/symbiont", # OGAR (Open Graph of Active Record) activation crate — re-exports OGAR's full # AR surface (ogar-vocab Class/codebook + ogar-class-view impl ClassView + diff --git a/crates/lance-graph-metta/Cargo.toml b/crates/lance-graph-metta/Cargo.toml new file mode 100644 index 000000000..65acf228d --- /dev/null +++ b/crates/lance-graph-metta/Cargo.toml @@ -0,0 +1,24 @@ +[package] +name = "lance-graph-metta" +version = "0.1.0" +edition = "2021" +publish = false +description = "MeTTa (Hyperon) as a declaration front end over the masking-algebra polyfill: the Datalog subset lowers through lance-graph-quack into mask-risc; everything else stays in Hyperon's interpreter. Plan: .claude/plans/metta-polyfill-v1.md." + +# Its OWN workspace root, like `lance-graph-relalg`: the datafrog oracle is a +# git dev-dependency that must not enter the main workspace's resolution. +# Verify via `cargo test --manifest-path crates/lance-graph-metta/Cargo.toml`. +[workspace] + +[dependencies] +# The only bulk engine, reached through Quack's lowering. +lance-graph-mask-risc = { path = "../lance-graph-mask-risc" } +lance-graph-quack = { path = "../lance-graph-quack" } + +[dev-dependencies] +# D-MTP-6: the exact Datalog oracle, from the AdaWorldAPI fork. Probe/test only. +datafrog = { git = "https://github.com/AdaWorldAPI/datafrog", branch = "master" } + +[[example]] +name = "join_probe" +test = true diff --git a/crates/lance-graph-metta/examples/join_probe.rs b/crates/lance-graph-metta/examples/join_probe.rs new file mode 100644 index 000000000..e0b79b342 --- /dev/null +++ b/crates/lance-graph-metta/examples/join_probe.rs @@ -0,0 +1,238 @@ +//! D-MTP-1 — the two-hop MeTTa join lowered through Quack into mask-risc. +//! +//! MeTTa: `(match &kb (, (parent $x $y) (parent $y $z)) ($x $z))`. +//! +//! Layout (ingest): a functional binary relation over the subject's address +//! space. Row `a` is the fact whose subject has address `a`; lane `dst` holds +//! the object's address; plane `valid` marks the addresses that have a fact. +//! A second fact for the same subject is refused. +//! +//! Lowering: `Plane(valid) AND Semijoin { fk: dst, foreign: valid }`, then +//! `Count` (the size of the collapsed result) or `Rows` + binding emission +//! `(x, dst[dst[x]])` over the kept bits — the only place pairs exist. +//! +//! Oracle: datafrog (AdaWorldAPI fork), compared as sorted pair vectors. +//! Pre-registration: plan §6a. +//! +//! ```text +//! CARGO_PROFILE_RELEASE_DEBUG=0 RUSTFLAGS=-Ctarget-cpu=native \ +//! cargo run --release --manifest-path crates/lance-graph-metta/Cargo.toml --example join_probe +//! ``` + +use std::time::Instant; + +use datafrog::Relation; +use lance_graph_mask_risc::exec::{execute_into, Scratch}; +use lance_graph_mask_risc::{words_for, Foreign, ForeignPlane, LaneRef, Out, Planes, Value}; +use lance_graph_quack::{lower, Agg, Col, Filter, ForeignPlane as QForeign, Mask, Query}; + +const DST: Col = Col(0); +const VALID: Mask = Mask(0); +const VALID_FOREIGN: QForeign = QForeign(0); + +/// A functional binary relation laid out over its subjects' address space. +struct Relation2 { + n: usize, + dst: Vec, + valid: Vec, +} + +#[derive(Debug, PartialEq, Eq)] +enum IngestError { + /// A subject already has a fact: not a functional relation. + SecondFact { subject: u32 }, + /// An address outside the declared address space. + OutOfSpace { addr: u32 }, +} + +impl Relation2 { + /// Ingest facts `(subject, object)` over addresses `0..n`. + fn ingest(n: usize, facts: &[(u32, u32)]) -> Result { + let mut dst = vec![0u32; n]; + let mut valid = vec![0u64; words_for(n)]; + for &(s, o) in facts { + for a in [s, o] { + if a as usize >= n { + return Err(IngestError::OutOfSpace { addr: a }); + } + } + let (w, b) = (s as usize / 64, s as usize % 64); + if (valid[w] >> b) & 1 == 1 { + return Err(IngestError::SecondFact { subject: s }); + } + valid[w] |= 1 << b; + dst[s as usize] = o; + } + Ok(Relation2 { n, dst, valid }) + } + + fn query(&self) -> Filter { + Filter::and([Filter::plane(VALID), Filter::semijoin(DST, VALID_FOREIGN)]) + } + + fn run(&self, agg: Agg, kept: &mut [u64]) -> Value { + let keep = matches!(agg, Agg::Rows); + let program = lower(&Query { filter: self.query(), agg }).expect("lowers"); + let lanes = [LaneRef::U32(&self.dst)]; + let masks = [self.valid.as_slice()]; + let planes = Planes { n_rows: self.n, masks: &masks, lanes: &lanes }; + let fplanes = [ForeignPlane { words: &self.valid, rows: self.n }]; + let foreign = Foreign { planes: &fplanes, lanes: &[] }; + let mut scratch = Scratch::for_program(&program, self.n).expect("carves"); + let out = if keep { Out::Mask(kept) } else { Out::None }; + execute_into(&program, &planes, &foreign, &mut scratch, out).expect("runs") + } + + /// `size-atom` of the collapsed result. + fn count(&self) -> usize { + match self.run(Agg::Count, &mut []) { + Value::Count(c) => c, + other => panic!("not a count: {other:?}"), + } + } + + /// The bindings `($x, $z)`, emitted only for kept rows. + fn pairs(&self, kept: &mut [u64], out: &mut Vec<(u32, u32)>) { + match self.run(Agg::Rows, kept) { + Value::Mask(_) => {} + other => panic!("not a kept mask: {other:?}"), + } + out.clear(); + for (w, &word) in kept.iter().enumerate() { + let mut bits = word; + while bits != 0 { + let x = (w * 64 + bits.trailing_zeros() as usize) as u32; + let y = self.dst[x as usize]; + out.push((x, self.dst[y as usize])); + bits &= bits - 1; + } + } + } +} + +/// F1 (`partial = false`): `parent(i, (7i+3) mod n)` for every `i`. +/// F2 (`partial = true`): the same, only for `i mod 3 != 0`. +fn facts(n: u32, partial: bool) -> Vec<(u32, u32)> { + (0..n) + .filter(|i| !partial || i % 3 != 0) + .map(|i| (i, ((7u64 * i as u64 + 3) % n as u64) as u32)) + .collect() +} + +/// The datafrog oracle: `(x, z)` with `parent(x, y)` and `parent(y, z)`. +fn oracle(facts: &[(u32, u32)]) -> Vec<(u32, u32)> { + let parent: Relation<(u32, u32)> = facts.iter().copied().collect(); + let by_dst: Relation<(u32, u32)> = facts.iter().map(|&(x, y)| (y, x)).collect(); + let pairs = Relation::from_join(&by_dst, &parent, |_y, &x, &z| (x, z)); + pairs.elements.clone() +} + +fn best(reps: usize, mut f: impl FnMut() -> T) -> (T, f64) { + let mut best = f64::INFINITY; + let mut out = None; + for _ in 0..reps { + let t = Instant::now(); + let r = f(); + best = best.min(t.elapsed().as_secs_f64()); + out = Some(r); + } + (out.expect("reps > 0"), best) +} + +fn main() { + println!( + "tier: avx512f={} avx2={}", + cfg!(target_feature = "avx512f"), + cfg!(target_feature = "avx2") + ); + println!("fixture n | pairs | ingest ns/fact | count ns/row | rows+emit ns/pair | datafrog ns/pair"); + for partial in [false, true] { + for n in [100u32, 1_000, 10_000, 100_000, 1_000_000] { + let f = facts(n, partial); + let (rel, t_ingest) = best(5, || Relation2::ingest(n as usize, &f).expect("functional")); + let (count, t_count) = best(5, || rel.count()); + let mut kept = vec![0u64; words_for(n as usize)]; + let mut pairs = Vec::with_capacity(n as usize); + let (_, t_pairs) = best(5, || rel.pairs(&mut kept, &mut pairs)); + let (want, t_frog) = best(5, || oracle(&f)); + let mut got = pairs.clone(); + got.sort_unstable(); + assert_eq!(got, want, "pairs differ from datafrog (F{}, n={n})", 1 + partial as u8); + assert_eq!(count, want.len(), "count differs from datafrog"); + let p = want.len().max(1) as f64; + println!( + "F{} {n:>9} | {:>7} | {:>6.2} | {:>6.3} | {:>6.2} | {:>6.1}", + 1 + partial as u8, + want.len(), + t_ingest * 1e9 / f.len() as f64, + t_count * 1e9 / n as f64, + t_pairs * 1e9 / p, + t_frog * 1e9 / p, + ); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn check(n: u32, partial: bool) { + let f = facts(n, partial); + let rel = Relation2::ingest(n as usize, &f).expect("functional"); + let mut kept = vec![0u64; words_for(n as usize)]; + let mut got = Vec::new(); + rel.pairs(&mut kept, &mut got); + got.sort_unstable(); + let want = oracle(&f); + assert_eq!(got, want, "F{} n={n}", 1 + partial as u8); + assert_eq!(rel.count(), want.len(), "F{} n={n}", 1 + partial as u8); + } + + #[test] + fn f1_total_relation_matches_datafrog() { + for n in [1, 2, 63, 64, 65, 100, 1_000] { + check(n, false); + } + } + + #[test] + fn f2_partial_relation_matches_datafrog() { + for n in [3, 63, 64, 65, 100, 1_000] { + check(n, true); + } + } + + /// Anti-vacuity: F2 must reject rows, or it cannot tell a semijoin from + /// no semijoin. + #[test] + fn f2_semijoin_rejects_rows() { + let f = facts(1_000, true); + let want = oracle(&f); + assert!(want.len() < f.len(), "{} of {} facts survive", want.len(), f.len()); + assert!(want.len() * 2 > f.len(), "fixture too sparse to be informative"); + } + + /// The counts Hyperon and PeTTa returned for the same MeTTa programs. + #[test] + fn counts_match_hyperon_and_petta() { + for (n, partial, expected) in [(100, false, 100), (1_000, false, 1_000)] { + let f = facts(n, partial); + let rel = Relation2::ingest(n as usize, &f).expect("functional"); + assert_eq!(rel.count(), expected, "n={n}"); + } + } + + #[test] + fn ingest_refuses_a_second_fact_for_one_subject() { + assert_eq!( + Relation2::ingest(4, &[(1, 2), (1, 3)]).err(), + Some(IngestError::SecondFact { subject: 1 }) + ); + assert_eq!( + Relation2::ingest(4, &[(1, 4)]).err(), + Some(IngestError::OutOfSpace { addr: 4 }) + ); + assert!(Relation2::ingest(4, &[(1, 2), (2, 1)]).is_ok()); + } +} diff --git a/crates/lance-graph-metta/src/lib.rs b/crates/lance-graph-metta/src/lib.rs new file mode 100644 index 000000000..acb39a333 --- /dev/null +++ b/crates/lance-graph-metta/src/lib.rs @@ -0,0 +1,7 @@ +//! MeTTa (Hyperon) as a declaration front end over the masking-algebra +//! polyfill. Nothing is shipped here yet: D-MTP-1 is a probe +//! (`examples/join_probe.rs`) that lowers one MeTTa conjunctive query by hand +//! through `lance-graph-quack` into one mask-risc program and checks it +//! against the datafrog oracle. The `Space` adapter is D-MTP-2. +//! +//! Plan: `.claude/plans/metta-polyfill-v1.md`. From 2682f62dcfd3cc12ab1f59e3b5b85fcaa0199e6d Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 14 Oct 2026 21:37:52 +0000 Subject: [PATCH 07/13] lance-graph-metta: pin the F2 counts Hyperon and PeTTa returned Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_011BMJnDBAfzcuQh5HeT39Rh --- crates/lance-graph-metta/examples/join_probe.rs | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/crates/lance-graph-metta/examples/join_probe.rs b/crates/lance-graph-metta/examples/join_probe.rs index e0b79b342..b26792f26 100644 --- a/crates/lance-graph-metta/examples/join_probe.rs +++ b/crates/lance-graph-metta/examples/join_probe.rs @@ -216,7 +216,15 @@ mod tests { /// The counts Hyperon and PeTTa returned for the same MeTTa programs. #[test] fn counts_match_hyperon_and_petta() { - for (n, partial, expected) in [(100, false, 100), (1_000, false, 1_000)] { + // F1 from the D-MTP-0 runs; F2 measured 2026-10-14: Hyperon 46 / 476, + // PeTTa 46 / 476 / 4,762. + for (n, partial, expected) in [ + (100, false, 100), + (1_000, false, 1_000), + (100, true, 46), + (1_000, true, 476), + (10_000, true, 4_762), + ] { let f = facts(n, partial); let rel = Relation2::ingest(n as usize, &f).expect("functional"); assert_eq!(rel.count(), expected, "n={n}"); From a6c3d7fc83cb8826c41db5d5a68f0a91318b1b13 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 14 Oct 2026 21:39:57 +0000 Subject: [PATCH 08/13] D-MTP-1 results: two-hop join through Quack Semijoin, 7-15x below datafrog per pair at 1e6 Pairs equal datafrog at every n on a total and a partial relation; counts equal Hyperon and PeTTa. Two disable runs red as pre-registered. The folded count is 1.2-1.8 ns/row, not sub-nanosecond as expected: the random gather read sets the cost. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_011BMJnDBAfzcuQh5HeT39Rh --- .claude/board/STATUS_BOARD.md | 2 +- .../board/entries/2026-10-14-d-mtp-1-join.md | 20 ++++++++ .claude/board/entries/README.md | 3 +- .claude/plans/metta-polyfill-v1.md | 39 ++++++++++++++- .../lance-graph-metta/examples/join_probe.rs | 48 +++++++++++++++---- 5 files changed, 100 insertions(+), 12 deletions(-) create mode 100644 .claude/board/entries/2026-10-14-d-mtp-1-join.md diff --git a/.claude/board/STATUS_BOARD.md b/.claude/board/STATUS_BOARD.md index 9d4289049..7815172e5 100644 --- a/.claude/board/STATUS_BOARD.md +++ b/.claude/board/STATUS_BOARD.md @@ -5,7 +5,7 @@ Plan: `.claude/plans/metta-polyfill-v1.md`. Proposed crate `lance-graph-metta` ( | D-id | scope | status | gate / falsifier | |---|---|---|---| | **D-MTP-0** | Adoption map, MeTTa-as-IR verdict, PeTTa vs Hyperon measurements, trie panic ≥ 1,024 children | Done 2026-10-14 (entry `2026-10-14-metta-polyfill-map.md`) | — | -| **D-MTP-1** | `join_n` lowered by hand through relalg/R2IL, n = 10²..10⁶, v4 | Queued (not authorized) | == Hyperon and PeTTa answers (count and bag); cost reported, no threshold before the run | +| **D-MTP-1** | Two-hop join lowered by hand through Quack `Semijoin` (relalg has no hop), n = 10²..10⁶, v4 | Measured 2026-10-14: pairs == datafrog at every n, counts == Hyperon/PeTTa; at 10⁶ emit 4.0–5.7 ns/pair vs datafrog 59–71 (7–9× with ingest charged); count 1.2–1.8 ns/row, the random `Gather` read, not sub-ns as expected | == oracles; two disable runs red as pre-registered | | **D-MTP-2** | `LanceSpace` (`Space + SpaceMut`) as a named space for the non-recursive Datalog subset; golden corpus rerun | Queued (not authorized) | 32/33 golden files still pass; refusals named; whole-conjunction vs per-conjunct calls recorded | | **D-MTP-3** | Recursive Datalog rules → relalg μ through the external driver d′ | Queued (not authorized) | `anc_n` answers identical; no fixpoint state in mask-risc | | **D-MTP-4** | Fork fix for the expression-width trie panic (≥ 1,024 children), test first; distinct from the id-decoding fix `1ffdbd6` | Queued (needs operator's word) | red before, green after | diff --git a/.claude/board/entries/2026-10-14-d-mtp-1-join.md b/.claude/board/entries/2026-10-14-d-mtp-1-join.md new file mode 100644 index 000000000..f10dc8f1f --- /dev/null +++ b/.claude/board/entries/2026-10-14-d-mtp-1-join.md @@ -0,0 +1,20 @@ +# 2026-10-14 — D-MTP-1 — the MeTTa two-hop join through Quack Semijoin (measured) + +**Status:** MEASURED (v4, `avx512f=true`, release, best of 5, three runs). Plan +`.claude/plans/metta-polyfill-v1.md` §6a (pre-registration) and §6b (results). Probe +`crates/lance-graph-metta/examples/join_probe.rs`. + +**Outcome:** +- `(match &kb (, (parent $x $y) (parent $y $z)) ($x $z))` lowers to + `Plane(valid) AND Semijoin { fk: dst, foreign: valid }` → one mask-risc program + (`Gather`); pairs are emitted only over the kept bits. `lance-graph-relalg` has no hop, + so the plan's "through relalg" was corrected to Quack. +- Pairs == datafrog (fork) at n = 10²..10⁶ on a total (F1) and a partial (F2) relation; + counts == Hyperon and PeTTa. Two disable runs red exactly as pre-registered. +- At 10⁶: emission 4.0–5.7 ns/pair against datafrog 59–71 ns/pair; with ingest charged, + 6.5–8 ns. The folded count is 1.2–1.8 ns/row, not the sub-nanosecond the plan + expected: the cost is the dependent random read of the gather, not the mask algebra. + +**Open:** a layout where the hop is a coordinate range (no per-row gather); one-to-many +relations; recursion (D-MTP-3); the `Space` adapter (D-MTP-2); symbol → address +resolution cost. diff --git a/.claude/board/entries/README.md b/.claude/board/entries/README.md index bf2286c22..83ae2971a 100644 --- a/.claude/board/entries/README.md +++ b/.claude/board/entries/README.md @@ -25,13 +25,14 @@ index row, (3) no duplicate entry id. Checks 1 and 2 are deliberately opposite directions; the stranding this convention prevents shows up in exactly one of them, never both. -331 entries, 2026-08-06 .. 2026-10-14. +332 entries, 2026-08-06 .. 2026-10-14. | date | entry id | finding | file | |---|---|---|---| | 2026-10-14 | `revision-layer-invariants` | | [2026-10-14-revision-layer-invariants.md](2026-10-14-revision-layer-invariants.md) | | 2026-10-14 | `D-MTP-0` | where Hyperon/MeTTa attaches to the masking-algebra polyfill | [2026-10-14-metta-polyfill-map.md](2026-10-14-metta-polyfill-map.md) | | 2026-10-14 | `D-NALP-4` | | [2026-10-14-isa-truth-exact-codes.md](2026-10-14-isa-truth-exact-codes.md) | +| 2026-10-14 | `D-MTP-1` | the MeTTa two-hop join through Quack Semijoin (measured) | [2026-10-14-d-mtp-1-join.md](2026-10-14-d-mtp-1-join.md) | | 2026-10-14 | `D-HOLO-H6b` | normal forms on the Context × Operation tile (measured) | [2026-10-14-d-holo-h6b.md](2026-10-14-d-holo-h6b.md) | | 2026-10-14 | `D-HOLO` | | [2026-10-14-d-holo-falsifiers.md](2026-10-14-d-holo-falsifiers.md) | | 2026-10-13 | `D-PROD-V4` | | [2026-10-13-production-targets-v4.md](2026-10-13-production-targets-v4.md) | diff --git a/.claude/plans/metta-polyfill-v1.md b/.claude/plans/metta-polyfill-v1.md index fa1ecea49..8a5380803 100644 --- a/.claude/plans/metta-polyfill-v1.md +++ b/.claude/plans/metta-polyfill-v1.md @@ -245,12 +245,49 @@ comparison is reported both ways. excluded, like `lance-graph-relalg`), so the datafrog git dev-dependency never enters the main workspace's resolution. +## 6b. D-MTP-1 results (2026-10-14) + +Probe `crates/lance-graph-metta/examples/join_probe.rs` (5 tests, run as part +of `cargo test`). v4 host, program-printed `avx512f=true`, release, best of 5, +three runs; ranges below span the three. + +**Correctness.** The emitted pairs equal datafrog's as sorted vectors at every +n on F1 and F2. Counts equal Hyperon (F1 100 / 1,000; F2 46 / 476) and PeTTa +(F2 46 / 476 / 4,762). Disable runs, after committing: dropping the +`Semijoin` leaf fails F2 and the pinned counts while F1 stays green; emitting +`(x, dst[x])` fails the pairs on F1 and F2. Both as pre-registered. + +| fixture, n = 10⁶ | ingest ns/fact | count ns/row | Rows + emit ns/pair | datafrog ns/pair | +|---|---|---|---|---| +| F1 (10⁶ pairs) | 2.5–3.0 | 1.72–1.79 | 3.97–4.06 | 59–62 | +| F2 (476,190 pairs) | 2.3–2.5 | 1.21–1.24 | 5.4–5.7 | 66–71 | + +From 10³ to 10⁶ the count stays at 1.1–2.4 ns/row and emission at 3.0–5.7 +ns/pair; at n = 100 fixed costs (lowering, scratch) dominate (8–9 ns/row, +10–21 ns/pair). + +**Against the pre-registration.** +- Pairs and counts identical: met. +- Emission below datafrog: met, by 11–15× per pair at 10⁶, and by 7–9× with + ingest charged to the query (ingest + emit ≈ 6.5–8 ns against 59–71). +- "The folded count costs a small fraction of a nanosecond per row": NOT met. + It costs 1.2–1.8 ns/row. The semijoin is a `Gather` through `dst`, one + dependent random read per row (`(7i+3) mod n` scatters the targets), and + that read, not the mask algebra, sets the cost. A layout where the hop is a + range (rule: hops are coordinate ranges, chained hops are intersections) is + what would remove it; this fixture has no such locality to exploit. + +**Scope.** Functional relations over a dense address space, integer facts +given to every engine. Symbol → address resolution through the label codebook +is not charged to anyone here; one-to-many relations, recursion and the +`Space` adapter are not covered. + ## 7. Deliverables | D-id | scope | gate | |---|---|---| | D-MTP-0 | this map, measurements §3, bug §3.1 | — (done) | -| D-MTP-1 | pre-registered probe: `join_n` lowered by hand through relalg/R2IL against the Hyperon and PeTTa answers, n = 10²..10⁶, v4, ns per row, build and terminal charged | answers identical (count and bag of pairs); cost reported, no threshold claimed before the run | +| D-MTP-1 | pre-registered probe (§6a): the two-hop join lowered by hand through Quack `Semijoin` (relalg has no hop) against datafrog, Hyperon and PeTTa, n = 10²..10⁶, v4 | answers identical (count and bag of pairs); cost reported, no threshold claimed before the run — **measured, §6b** | | D-MTP-2 | `LanceSpace` adapter (`Space + SpaceMut`) for the non-recursive Datalog subset, bound as a named space; Hyperon's golden corpus (`hyperon-probe/golden`, 32 of 33 passing) rerun with facts moved into it | every golden file that passes today still passes; refusals named; the probe records whether the space received whole conjunctions or single conjuncts | | D-MTP-3 | recursive Datalog rules declared into `LanceSpace` → relalg `μ` through d′, queried by `match` over the derived relation | `anc_n` answers identical; no fixpoint state inside mask-risc | | D-MTP-4 | the §3.1 expression-width panic as a fix on the fork, test first (separate from `1ffdbd6`) | red before, green after; upstream only with the operator's word | diff --git a/crates/lance-graph-metta/examples/join_probe.rs b/crates/lance-graph-metta/examples/join_probe.rs index b26792f26..6ea973de3 100644 --- a/crates/lance-graph-metta/examples/join_probe.rs +++ b/crates/lance-graph-metta/examples/join_probe.rs @@ -72,12 +72,26 @@ impl Relation2 { fn run(&self, agg: Agg, kept: &mut [u64]) -> Value { let keep = matches!(agg, Agg::Rows); - let program = lower(&Query { filter: self.query(), agg }).expect("lowers"); + let program = lower(&Query { + filter: self.query(), + agg, + }) + .expect("lowers"); let lanes = [LaneRef::U32(&self.dst)]; let masks = [self.valid.as_slice()]; - let planes = Planes { n_rows: self.n, masks: &masks, lanes: &lanes }; - let fplanes = [ForeignPlane { words: &self.valid, rows: self.n }]; - let foreign = Foreign { planes: &fplanes, lanes: &[] }; + let planes = Planes { + n_rows: self.n, + masks: &masks, + lanes: &lanes, + }; + let fplanes = [ForeignPlane { + words: &self.valid, + rows: self.n, + }]; + let foreign = Foreign { + planes: &fplanes, + lanes: &[], + }; let mut scratch = Scratch::for_program(&program, self.n).expect("carves"); let out = if keep { Out::Mask(kept) } else { Out::None }; execute_into(&program, &planes, &foreign, &mut scratch, out).expect("runs") @@ -145,11 +159,14 @@ fn main() { cfg!(target_feature = "avx512f"), cfg!(target_feature = "avx2") ); - println!("fixture n | pairs | ingest ns/fact | count ns/row | rows+emit ns/pair | datafrog ns/pair"); + println!( + "fixture n | pairs | ingest ns/fact | count ns/row | rows+emit ns/pair | datafrog ns/pair" + ); for partial in [false, true] { for n in [100u32, 1_000, 10_000, 100_000, 1_000_000] { let f = facts(n, partial); - let (rel, t_ingest) = best(5, || Relation2::ingest(n as usize, &f).expect("functional")); + let (rel, t_ingest) = + best(5, || Relation2::ingest(n as usize, &f).expect("functional")); let (count, t_count) = best(5, || rel.count()); let mut kept = vec![0u64; words_for(n as usize)]; let mut pairs = Vec::with_capacity(n as usize); @@ -157,7 +174,12 @@ fn main() { let (want, t_frog) = best(5, || oracle(&f)); let mut got = pairs.clone(); got.sort_unstable(); - assert_eq!(got, want, "pairs differ from datafrog (F{}, n={n})", 1 + partial as u8); + assert_eq!( + got, + want, + "pairs differ from datafrog (F{}, n={n})", + 1 + partial as u8 + ); assert_eq!(count, want.len(), "count differs from datafrog"); let p = want.len().max(1) as f64; println!( @@ -209,8 +231,16 @@ mod tests { fn f2_semijoin_rejects_rows() { let f = facts(1_000, true); let want = oracle(&f); - assert!(want.len() < f.len(), "{} of {} facts survive", want.len(), f.len()); - assert!(want.len() * 2 > f.len(), "fixture too sparse to be informative"); + assert!( + want.len() < f.len(), + "{} of {} facts survive", + want.len(), + f.len() + ); + assert!( + want.len() * 2 > f.len(), + "fixture too sparse to be informative" + ); } /// The counts Hyperon and PeTTa returned for the same MeTTa programs. From b2e6b65ca666b1a25398de46dc34810114ad31dc Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 14 Oct 2026 21:53:40 +0000 Subject: [PATCH 09/13] D-MTP-1: normalize ingest per emitted pair; integration status reflects D-MTP-1 and D-MTP-6 F2 ingests 666,666 facts for 476,190 pairs, so its ingest cost counts 1.4x per pair: 8.6-9.2 ns/pair with ingest charged, not 6.5-8 (codex review). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_011BMJnDBAfzcuQh5HeT39Rh --- .claude/board/INTEGRATION_PLANS.md | 2 +- .claude/board/STATUS_BOARD.md | 2 +- .claude/board/entries/2026-10-14-d-mtp-1-join.md | 4 ++-- .claude/plans/metta-polyfill-v1.md | 6 ++++-- 4 files changed, 8 insertions(+), 6 deletions(-) diff --git a/.claude/board/INTEGRATION_PLANS.md b/.claude/board/INTEGRATION_PLANS.md index 86f39c0c8..e4d300562 100644 --- a/.claude/board/INTEGRATION_PLANS.md +++ b/.claude/board/INTEGRATION_PLANS.md @@ -1,6 +1,6 @@ ## 2026-10-14 (2) — metta-polyfill-v1 — Hyperon/MeTTa as a front end over the masking-algebra polyfill → `.claude/plans/metta-polyfill-v1.md` -**Status:** PROPOSAL; D-MTP-0 (map + measurements) done, D-MTP-1..5 not authorized. +**Status:** PROPOSAL; D-MTP-0 (map + measurements), D-MTP-1 (join probe, measured) and D-MTP-6 (datafrog oracle) done; D-MTP-2..5 not authorized. - MeTTa is a declaration IR, not an execution IR: the Datalog subset lowers through `lance-graph-relalg` → R2IL → mask-risc; everything else stays in Hyperon's interpreter. - Seam: a named `LanceSpace` implementing `hyperon_space::Space`; `ModuleSpace` splits conjunctions, so `&self` is not the seam. diff --git a/.claude/board/STATUS_BOARD.md b/.claude/board/STATUS_BOARD.md index 7815172e5..c9e1d3a02 100644 --- a/.claude/board/STATUS_BOARD.md +++ b/.claude/board/STATUS_BOARD.md @@ -5,7 +5,7 @@ Plan: `.claude/plans/metta-polyfill-v1.md`. Proposed crate `lance-graph-metta` ( | D-id | scope | status | gate / falsifier | |---|---|---|---| | **D-MTP-0** | Adoption map, MeTTa-as-IR verdict, PeTTa vs Hyperon measurements, trie panic ≥ 1,024 children | Done 2026-10-14 (entry `2026-10-14-metta-polyfill-map.md`) | — | -| **D-MTP-1** | Two-hop join lowered by hand through Quack `Semijoin` (relalg has no hop), n = 10²..10⁶, v4 | Measured 2026-10-14: pairs == datafrog at every n, counts == Hyperon/PeTTa; at 10⁶ emit 4.0–5.7 ns/pair vs datafrog 59–71 (7–9× with ingest charged); count 1.2–1.8 ns/row, the random `Gather` read, not sub-ns as expected | == oracles; two disable runs red as pre-registered | +| **D-MTP-1** | Two-hop join lowered by hand through Quack `Semijoin` (relalg has no hop), n = 10²..10⁶, v4 | Measured 2026-10-14: pairs == datafrog at every n, counts == Hyperon/PeTTa; at 10⁶ emit 4.0–5.7 ns/pair vs datafrog 59–71 (7.2–9.6× with ingest charged per emitted pair); count 1.2–1.8 ns/row, the random `Gather` read, not sub-ns as expected | == oracles; two disable runs red as pre-registered | | **D-MTP-2** | `LanceSpace` (`Space + SpaceMut`) as a named space for the non-recursive Datalog subset; golden corpus rerun | Queued (not authorized) | 32/33 golden files still pass; refusals named; whole-conjunction vs per-conjunct calls recorded | | **D-MTP-3** | Recursive Datalog rules → relalg μ through the external driver d′ | Queued (not authorized) | `anc_n` answers identical; no fixpoint state in mask-risc | | **D-MTP-4** | Fork fix for the expression-width trie panic (≥ 1,024 children), test first; distinct from the id-decoding fix `1ffdbd6` | Queued (needs operator's word) | red before, green after | diff --git a/.claude/board/entries/2026-10-14-d-mtp-1-join.md b/.claude/board/entries/2026-10-14-d-mtp-1-join.md index f10dc8f1f..faaa83d1c 100644 --- a/.claude/board/entries/2026-10-14-d-mtp-1-join.md +++ b/.claude/board/entries/2026-10-14-d-mtp-1-join.md @@ -11,8 +11,8 @@ so the plan's "through relalg" was corrected to Quack. - Pairs == datafrog (fork) at n = 10²..10⁶ on a total (F1) and a partial (F2) relation; counts == Hyperon and PeTTa. Two disable runs red exactly as pre-registered. -- At 10⁶: emission 4.0–5.7 ns/pair against datafrog 59–71 ns/pair; with ingest charged, - 6.5–8 ns. The folded count is 1.2–1.8 ns/row, not the sub-nanosecond the plan +- At 10⁶: emission 4.0–5.7 ns/pair against datafrog 59–71 ns/pair; with ingest charged + per emitted pair, 6.5–7.1 ns (F1) and 8.6–9.2 ns (F2, 666,666 facts for 476,190 pairs). The folded count is 1.2–1.8 ns/row, not the sub-nanosecond the plan expected: the cost is the dependent random read of the gather, not the mask algebra. **Open:** a layout where the hop is a coordinate range (no per-row gather); one-to-many diff --git a/.claude/plans/metta-polyfill-v1.md b/.claude/plans/metta-polyfill-v1.md index 8a5380803..8414bb811 100644 --- a/.claude/plans/metta-polyfill-v1.md +++ b/.claude/plans/metta-polyfill-v1.md @@ -268,8 +268,10 @@ ns/pair; at n = 100 fixed costs (lowering, scratch) dominate (8–9 ns/row, **Against the pre-registration.** - Pairs and counts identical: met. -- Emission below datafrog: met, by 11–15× per pair at 10⁶, and by 7–9× with - ingest charged to the query (ingest + emit ≈ 6.5–8 ns against 59–71). +- Emission below datafrog: met, by 11–15× per pair at 10⁶. With ingest + charged to the query, normalized per emitted pair (F2 ingests 666,666 facts + for 476,190 pairs, so its ingest counts 1.4×): F1 6.5–7.1 ns/pair against + 59–62 (8.4–9.6×), F2 8.6–9.2 ns/pair against 66–71 (7.2–8.2×). - "The folded count costs a small fraction of a nanosecond per row": NOT met. It costs 1.2–1.8 ns/row. The semijoin is a `Gather` through `dst`, one dependent random read per row (`(7i+3) mod n` scatters the targets), and From b0540078fce969fe36011d475f84fff9bc5d7e63 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 14 Oct 2026 22:02:03 +0000 Subject: [PATCH 10/13] =?UTF-8?q?plan:=20metta-polyfill-v1=20=E2=80=94=20p?= =?UTF-8?q?re-register=20D-MTP-1b=20(the=20hop=20as=20a=20range)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_011BMJnDBAfzcuQh5HeT39Rh --- .claude/board/STATUS_BOARD.md | 1 + .claude/plans/metta-polyfill-v1.md | 50 ++++++++++++++++++++++++++++++ 2 files changed, 51 insertions(+) diff --git a/.claude/board/STATUS_BOARD.md b/.claude/board/STATUS_BOARD.md index c9e1d3a02..3d4565cec 100644 --- a/.claude/board/STATUS_BOARD.md +++ b/.claude/board/STATUS_BOARD.md @@ -6,6 +6,7 @@ Plan: `.claude/plans/metta-polyfill-v1.md`. Proposed crate `lance-graph-metta` ( |---|---|---|---| | **D-MTP-0** | Adoption map, MeTTa-as-IR verdict, PeTTa vs Hyperon measurements, trie panic ≥ 1,024 children | Done 2026-10-14 (entry `2026-10-14-metta-polyfill-map.md`) | — | | **D-MTP-1** | Two-hop join lowered by hand through Quack `Semijoin` (relalg has no hop), n = 10²..10⁶, v4 | Measured 2026-10-14: pairs == datafrog at every n, counts == Hyperon/PeTTa; at 10⁶ emit 4.0–5.7 ns/pair vs datafrog 59–71 (7.2–9.6× with ingest charged per emitted pair); count 1.2–1.8 ns/row, the random `Gather` read, not sub-ns as expected | == oracles; two disable runs red as pre-registered | +| **D-MTP-1b** | The hop as a range: preimage of `valid`'s runs under a monotone `dst` vs the gather, clustered and scattered | In progress (pre-registered §6c) | == datafrog; two disable runs red; costs reported | | **D-MTP-2** | `LanceSpace` (`Space + SpaceMut`) as a named space for the non-recursive Datalog subset; golden corpus rerun | Queued (not authorized) | 32/33 golden files still pass; refusals named; whole-conjunction vs per-conjunct calls recorded | | **D-MTP-3** | Recursive Datalog rules → relalg μ through the external driver d′ | Queued (not authorized) | `anc_n` answers identical; no fixpoint state in mask-risc | | **D-MTP-4** | Fork fix for the expression-width trie panic (≥ 1,024 children), test first; distinct from the id-decoding fix `1ffdbd6` | Queued (needs operator's word) | red before, green after | diff --git a/.claude/plans/metta-polyfill-v1.md b/.claude/plans/metta-polyfill-v1.md index 8414bb811..5be6ccd42 100644 --- a/.claude/plans/metta-polyfill-v1.md +++ b/.claude/plans/metta-polyfill-v1.md @@ -284,6 +284,55 @@ given to every engine. Symbol → address resolution through the label codebook is not charged to anyone here; one-to-many relations, recursion and the `Space` adapter are not covered. +## 6c. D-MTP-1b pre-registration: the hop as a range (2026-10-14, before any code) + +**Question.** D-MTP-1's count costs 1.2–1.8 ns/row because the semijoin is a +`Gather`: one dependent random read per row. When the link lane is monotone in +row order, can the hop become interval arithmetic instead (the hop as a +coordinate range, chained hops as intersections)? + +**Mechanism.** If `dst` is non-decreasing over rows, the rows whose target lies +in `[a, b)` are exactly `[pp(dst < a), pp(dst < b))` (`pp` = `partition_point`, +as in `teleport_probe`). So `Semijoin(dst, valid)` restricted to rows in +`valid` is the intersection of two interval sets: `valid`'s runs, and the +preimages of `valid`'s runs. The count is the length of that intersection; +the pairs `(x, dst[dst[x]])` are emitted by walking the intervals. + +**Precondition, checked not assumed.** Monotonicity of `dst` is verified once at +ingest (an O(n) pass, charged to ingest) and recorded as a flag; the range path +refuses a lane without it and the caller falls back to the gather. This is a +probe-local witness, not the contract's `OrderedLaneWitness` (which covers +sealed facet lanes only, `teleport-probe` OPEN item). + +**Fixtures** (generated, n = 10³..10⁶): +- F3 BFS tree, fanout 4: `parent(x, (x-1)/4)` for every `x ≥ 1` (root has no + fact); `dst` monotone; `valid` is one run. +- F4: F3 without the facts of nodes whose depth is a multiple of 3; `valid` is + one run per kept level band (about depth·2/3 runs), so both the source and the + target restriction bite. +- F3s / F4s: the same facts under a random permutation of addresses (the + scattered control; no range path exists there). + +**Arms.** +- G-scattered: D-MTP-1's gather program on F3s/F4s. +- G-clustered: the same gather program on F3/F4 (same algebra, local reads). +- R-scan: `valid`'s runs extracted from the plane by a word scan, then the + interval path. +- R-declared: `valid`'s runs recorded at ingest (no scan), then the interval path. + +**Oracle and tests.** datafrog pairs (sorted) equal every arm's pairs at every +n; counts equal. The range path refuses a non-monotone `dst`. Disable runs +(commit first, assert each patch applied once): (1) intersect with the +preimage runs only, skipping `valid`'s own runs → F4 red, F3 green; (2) take +`valid`'s runs as the answer, skipping the preimage → F4 red. + +**Expected, as a direction only (no threshold before the run).** R count costs +O(runs · log n), far below a nanosecond per row at 10⁶; G-clustered sits +between G-scattered and R; pair emission on the range path is sequential and +should not exceed G-clustered's. Reported per row and per pair, v4, +program-printed tier, best of 5, three runs; ingest (including the +monotonicity pass and, for R-declared, the run recording) reported separately. + ## 7. Deliverables | D-id | scope | gate | @@ -293,6 +342,7 @@ is not charged to anyone here; one-to-many relations, recursion and the | D-MTP-2 | `LanceSpace` adapter (`Space + SpaceMut`) for the non-recursive Datalog subset, bound as a named space; Hyperon's golden corpus (`hyperon-probe/golden`, 32 of 33 passing) rerun with facts moved into it | every golden file that passes today still passes; refusals named; the probe records whether the space received whole conjunctions or single conjuncts | | D-MTP-3 | recursive Datalog rules declared into `LanceSpace` → relalg `μ` through d′, queried by `match` over the derived relation | `anc_n` answers identical; no fixpoint state inside mask-risc | | D-MTP-4 | the §3.1 expression-width panic as a fix on the fork, test first (separate from `1ffdbd6`) | red before, green after; upstream only with the operator's word | +| D-MTP-1b | the hop as a range (§6c): preimage of `valid`'s runs under a monotone `dst`, against the gather on clustered and scattered layouts | pairs == datafrog on F3/F4 and their scattered twins; two disable runs red; costs reported | | D-MTP-6 | datafrog as the exact Rust oracle in D-MTP-1/3 probes and the `ruff_metta_spo` lowerable tag (test-only dependency) | answers identical to Hyperon and PeTTa as bags of pairs, not only counts | | D-MTP-5 | PeTTa as the second oracle in the probes (needs SWI-Prolog ≥ 9.3; 10.0.2 installed from the official PPA) | divergences between PeTTa and Hyperon recorded, never averaged | From 69c5713f7679b2c4e231d4da46ea865475eff50f Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 14 Oct 2026 22:06:02 +0000 Subject: [PATCH 11/13] D-MTP-1b: range-shaped hop probe (preimage of valid's runs under a monotone dst) Arms G-scattered, G-clustered, R-scan, R-declared; datafrog oracle; refusal of a non-monotone lane; fixture guard that both halves of the intersection matter on F4. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_011BMJnDBAfzcuQh5HeT39Rh --- crates/lance-graph-metta/Cargo.toml | 4 + .../lance-graph-metta/examples/common/mod.rs | 250 ++++++++++++++++++ .../examples/join_range_probe.rs | 203 ++++++++++++++ 3 files changed, 457 insertions(+) create mode 100644 crates/lance-graph-metta/examples/common/mod.rs create mode 100644 crates/lance-graph-metta/examples/join_range_probe.rs diff --git a/crates/lance-graph-metta/Cargo.toml b/crates/lance-graph-metta/Cargo.toml index 65acf228d..c2aa62b00 100644 --- a/crates/lance-graph-metta/Cargo.toml +++ b/crates/lance-graph-metta/Cargo.toml @@ -22,3 +22,7 @@ datafrog = { git = "https://github.com/AdaWorldAPI/datafrog", branch = "master" [[example]] name = "join_probe" test = true + +[[example]] +name = "join_range_probe" +test = true diff --git a/crates/lance-graph-metta/examples/common/mod.rs b/crates/lance-graph-metta/examples/common/mod.rs new file mode 100644 index 000000000..f8c77078d --- /dev/null +++ b/crates/lance-graph-metta/examples/common/mod.rs @@ -0,0 +1,250 @@ +//! Shared by the D-MTP probes: the fact layout, the gather path (D-MTP-1), +//! the range path (D-MTP-1b) and the datafrog oracle. Probe code, not a +//! shipped API. +#![allow(dead_code)] + +use std::time::Instant; + +use datafrog::Relation; +use lance_graph_mask_risc::exec::{execute_into, Scratch}; +use lance_graph_mask_risc::{words_for, Foreign, ForeignPlane, LaneRef, Out, Planes, Value}; +use lance_graph_quack::{lower, Agg, Col, Filter, ForeignPlane as QForeign, Mask, Query}; + +const DST: Col = Col(0); +const VALID: Mask = Mask(0); +const VALID_FOREIGN: QForeign = QForeign(0); + +/// A half-open row interval `[lo, hi)`. +pub type Run = (u32, u32); + +/// A functional binary relation laid out over its subjects' address space: +/// row `a` is the fact whose subject has address `a`, lane `dst` holds the +/// object's address, plane `valid` marks the addresses that have a fact. +pub struct Relation2 { + pub n: usize, + pub dst: Vec, + pub valid: Vec, + /// `valid`'s runs, recorded during ingest when the facts arrive in + /// address order (`None` otherwise). + pub declared_runs: Option>, +} + +#[derive(Debug, PartialEq, Eq)] +pub enum IngestError { + /// A subject already has a fact: not a functional relation. + SecondFact { subject: u32 }, + /// An address outside the declared address space. + OutOfSpace { addr: u32 }, +} + +impl Relation2 { + /// Ingest facts `(subject, object)` over addresses `0..n`. A row without a + /// fact gets the previous row's `dst` (it lies outside `valid`, so the + /// value is never read as a fact; it only keeps a monotone lane monotone). + pub fn ingest(n: usize, facts: &[(u32, u32)]) -> Result { + let mut dst = vec![u32::MAX; n]; + let mut valid = vec![0u64; words_for(n)]; + let mut runs: Vec = Vec::new(); + let mut in_order = true; + let mut last: Option = None; + for &(s, o) in facts { + for a in [s, o] { + if a as usize >= n { + return Err(IngestError::OutOfSpace { addr: a }); + } + } + let (w, b) = (s as usize / 64, s as usize % 64); + if (valid[w] >> b) & 1 == 1 { + return Err(IngestError::SecondFact { subject: s }); + } + valid[w] |= 1 << b; + dst[s as usize] = o; + match (in_order, last) { + (true, Some(p)) if s <= p => in_order = false, + (true, _) => match runs.last_mut() { + Some(r) if r.1 == s => r.1 = s + 1, + _ => runs.push((s, s + 1)), + }, + _ => {} + } + last = Some(s); + } + let mut prev = 0u32; + for d in dst.iter_mut() { + if *d == u32::MAX { + *d = prev; + } else { + prev = *d; + } + } + Ok(Relation2 { + n, + dst, + valid, + declared_runs: in_order.then_some(runs), + }) + } + + /// The order witness for the range path: `dst` non-decreasing over rows. + pub fn dst_monotone(&self) -> bool { + self.dst.windows(2).all(|w| w[0] <= w[1]) + } + + // ───────────── gather path (D-MTP-1): Plane(valid) AND Semijoin ───────────── + + fn query(&self) -> Filter { + Filter::and([Filter::plane(VALID), Filter::semijoin(DST, VALID_FOREIGN)]) + } + + fn run(&self, agg: Agg, kept: &mut [u64]) -> Value { + let keep = matches!(agg, Agg::Rows); + let program = lower(&Query { + filter: self.query(), + agg, + }) + .expect("lowers"); + let lanes = [LaneRef::U32(&self.dst)]; + let masks = [self.valid.as_slice()]; + let planes = Planes { + n_rows: self.n, + masks: &masks, + lanes: &lanes, + }; + let fplanes = [ForeignPlane { + words: &self.valid, + rows: self.n, + }]; + let foreign = Foreign { + planes: &fplanes, + lanes: &[], + }; + let mut scratch = Scratch::for_program(&program, self.n).expect("carves"); + let out = if keep { Out::Mask(kept) } else { Out::None }; + execute_into(&program, &planes, &foreign, &mut scratch, out).expect("runs") + } + + /// `size-atom` of the collapsed result, by the gather program. + pub fn count(&self) -> usize { + match self.run(Agg::Count, &mut []) { + Value::Count(c) => c, + other => panic!("not a count: {other:?}"), + } + } + + /// The bindings `($x, $z)`, emitted only for rows the gather program kept. + pub fn pairs(&self, kept: &mut [u64], out: &mut Vec<(u32, u32)>) { + match self.run(Agg::Rows, kept) { + Value::Mask(_) => {} + other => panic!("not a kept mask: {other:?}"), + } + out.clear(); + for (w, &word) in kept.iter().enumerate() { + let mut bits = word; + while bits != 0 { + let x = (w * 64 + bits.trailing_zeros() as usize) as u32; + let y = self.dst[x as usize]; + out.push((x, self.dst[y as usize])); + bits &= bits - 1; + } + } + } + + // ───────────── range path (D-MTP-1b): the hop as interval arithmetic ───────────── + + /// `valid`'s runs, found by a word scan of the plane. + pub fn scanned_runs(&self) -> Vec { + let mut runs: Vec = Vec::new(); + for (w, &word) in self.valid.iter().enumerate() { + let mut bits = word; + let base = (w * 64) as u32; + while bits != 0 { + let start = bits.trailing_zeros(); + let len = (!(bits >> start)).trailing_zeros().min(64 - start); + let (lo, hi) = (base + start, base + start + len); + match runs.last_mut() { + Some(r) if r.1 == lo => r.1 = hi, + _ => runs.push((lo, hi)), + } + bits = if start + len >= 64 { 0 } else { bits & (u64::MAX << (start + len)) }; + } + } + runs + } + + /// The rows whose target lies in one of `runs`: one interval per run, + /// found by binary search on the monotone `dst` lane. + pub fn preimage(&self, runs: &[Run]) -> Vec { + runs.iter() + .filter_map(|&(a, b)| { + let lo = self.dst.partition_point(|&d| d < a) as u32; + let hi = self.dst.partition_point(|&d| d < b) as u32; + (lo < hi).then_some((lo, hi)) + }) + .collect() + } + + /// The kept rows of `Plane(valid) AND Semijoin(dst, valid)` as intervals. + /// `None` when the order witness fails: the caller uses the gather. + pub fn kept_ranges(&self, valid_runs: &[Run]) -> Option> { + if !self.dst_monotone() { + return None; + } + Some(intersect(valid_runs, &self.preimage(valid_runs))) + } + + /// The bindings `($x, $z)` by walking the kept intervals. + pub fn range_pairs(&self, kept: &[Run], out: &mut Vec<(u32, u32)>) { + out.clear(); + for &(lo, hi) in kept { + for x in lo..hi { + let y = self.dst[x as usize]; + out.push((x, self.dst[y as usize])); + } + } + } +} + +/// Intersection of two sorted, disjoint interval lists. +pub fn intersect(a: &[Run], b: &[Run]) -> Vec { + let (mut i, mut j) = (0, 0); + let mut out = Vec::new(); + while i < a.len() && j < b.len() { + let lo = a[i].0.max(b[j].0); + let hi = a[i].1.min(b[j].1); + if lo < hi { + out.push((lo, hi)); + } + if a[i].1 < b[j].1 { + i += 1; + } else { + j += 1; + } + } + out +} + +/// Total length of an interval list. +pub fn total(runs: &[Run]) -> usize { + runs.iter().map(|&(lo, hi)| (hi - lo) as usize).sum() +} + +/// The datafrog oracle: `(x, z)` with `parent(x, y)` and `parent(y, z)`. +pub fn oracle(facts: &[(u32, u32)]) -> Vec<(u32, u32)> { + let parent: Relation<(u32, u32)> = facts.iter().copied().collect(); + let by_dst: Relation<(u32, u32)> = facts.iter().map(|&(x, y)| (y, x)).collect(); + let pairs = Relation::from_join(&by_dst, &parent, |_y, &x, &z| (x, z)); + pairs.elements.clone() +} + +/// Best wall time of `reps` runs, and the last result. +pub fn best(reps: usize, mut f: impl FnMut() -> T) -> (T, f64) { + let mut best = f64::INFINITY; + let mut out = None; + for _ in 0..reps { + let t = Instant::now(); + let r = f(); + best = best.min(t.elapsed().as_secs_f64()); + out = Some(r); + } + (out.expect("reps > 0"), best) +} diff --git a/crates/lance-graph-metta/examples/join_range_probe.rs b/crates/lance-graph-metta/examples/join_range_probe.rs new file mode 100644 index 000000000..be3970867 --- /dev/null +++ b/crates/lance-graph-metta/examples/join_range_probe.rs @@ -0,0 +1,203 @@ +//! D-MTP-1b — the hop as a range (plan §6c). +//! +//! MeTTa: `(match &kb (, (parent $x $y) (parent $y $z)) ($x $z))`, the same +//! two-hop join as D-MTP-1. When the link lane `dst` is non-decreasing over +//! rows, `Semijoin(dst, valid)` restricted to `valid` is an intersection of +//! interval sets: `valid`'s runs and their preimages under `dst` (found by +//! `partition_point`). The count is interval arithmetic; pairs are emitted by +//! walking the kept intervals. A non-monotone lane is refused and the caller +//! keeps the gather program. +//! +//! Arms: G-scattered / G-clustered (the D-MTP-1 gather program), R-scan +//! (`valid`'s runs from a word scan), R-declared (runs recorded at ingest). +//! Oracle: datafrog (AdaWorldAPI fork). +//! +//! ```text +//! CARGO_PROFILE_RELEASE_DEBUG=0 RUSTFLAGS=-Ctarget-cpu=native \ +//! cargo run --release --manifest-path crates/lance-graph-metta/Cargo.toml --example join_range_probe +//! ``` + +mod common; + +use common::{best, oracle, total, Relation2}; + +/// Depth of node `x` in the fanout-4 BFS tree (`parent(x) = (x-1)/4`). +fn depth(mut x: u32) -> u32 { + let mut d = 0; + while x > 0 { + x = (x - 1) / 4; + d += 1; + } + d +} + +/// F3 (`banded = false`) or F4 (`banded = true`: no facts for nodes whose +/// depth is a multiple of 3), in address order. +fn tree(n: u32, banded: bool) -> Vec<(u32, u32)> { + (1..n) + .filter(|&x| !banded || depth(x) % 3 != 0) + .map(|x| (x, (x - 1) / 4)) + .collect() +} + +/// A deterministic random permutation of `0..n` (xorshift + Fisher–Yates). +fn permutation(n: u32) -> Vec { + let mut p: Vec = (0..n).collect(); + let mut s: u64 = 0x9E37_79B9_7F4A_7C15; + for i in (1..n as usize).rev() { + s ^= s << 13; + s ^= s >> 7; + s ^= s << 17; + p.swap(i, (s % (i as u64 + 1)) as usize); + } + p +} + +/// The same facts under a permutation of addresses (the scattered twin). +fn scatter(n: u32, facts: &[(u32, u32)]) -> Vec<(u32, u32)> { + let p = permutation(n); + facts.iter().map(|&(x, y)| (p[x as usize], p[y as usize])).collect() +} + +fn sorted(mut v: Vec<(u32, u32)>) -> Vec<(u32, u32)> { + v.sort_unstable(); + v +} + +#[cfg(test)] +mod tests { + use super::*; + use common::intersect; + + const SIZES: [u32; 3] = [100, 1000, 4097]; + + fn gather_pairs(r: &Relation2) -> Vec<(u32, u32)> { + let mut kept = vec![0u64; r.valid.len()]; + let mut out = Vec::new(); + r.pairs(&mut kept, &mut out); + sorted(out) + } + + fn range_pairs(r: &Relation2, runs: &[(u32, u32)]) -> Vec<(u32, u32)> { + let kept = r.kept_ranges(runs).expect("monotone"); + let mut out = Vec::new(); + r.range_pairs(&kept, &mut out); + sorted(out) + } + + #[test] + fn every_arm_equals_datafrog() { + for n in SIZES { + for banded in [false, true] { + let f = tree(n, banded); + let want = sorted(oracle(&f)); + assert!(!want.is_empty()); + let r = Relation2::ingest(n as usize, &f).unwrap(); + assert!(r.dst_monotone()); + let declared = r.declared_runs.clone().expect("address order"); + let scanned = r.scanned_runs(); + assert_eq!(declared, scanned, "n={n} banded={banded}"); + assert_eq!(gather_pairs(&r), want, "G-clustered n={n}"); + assert_eq!(range_pairs(&r, &scanned), want, "R n={n}"); + assert_eq!(r.count(), want.len()); + let kept = r.kept_ranges(&scanned).unwrap(); + assert_eq!(total(&kept), want.len()); + + let s = scatter(n, &f); + let rs = Relation2::ingest(n as usize, &s).unwrap(); + assert_eq!(gather_pairs(&rs).len(), want.len(), "G-scattered n={n}"); + assert_eq!(sorted(oracle(&s)), gather_pairs(&rs)); + } + } + } + + #[test] + fn a_non_monotone_lane_is_refused() { + let s = scatter(1000, &tree(1000, true)); + let r = Relation2::ingest(1000, &s).unwrap(); + assert!(!r.dst_monotone()); + assert!(r.declared_runs.is_none()); + assert!(r.kept_ranges(&r.scanned_runs()).is_none()); + } + + /// Both halves of the intersection must matter on F4, or the disable runs + /// in plan §6c prove nothing. + #[test] + fn f4_needs_both_halves_of_the_intersection() { + let n = 4097; + let r = Relation2::ingest(n as usize, &tree(n, true)).unwrap(); + let runs = r.scanned_runs(); + // Levels 1-2 and 4-5 merge: [1, 21) and [85, 1365) at n = 4097. + assert_eq!(runs, vec![(1, 21), (85, 1365)], "F4 valid runs"); + let want = total(&r.kept_ranges(&runs).unwrap()); + assert_ne!(total(&r.preimage(&runs)), want, "preimage alone"); + assert_ne!(total(&runs), want, "valid alone"); + // F3: one run, and the preimage alone already equals the answer. + let r3 = Relation2::ingest(n as usize, &tree(n, false)).unwrap(); + let runs3 = r3.scanned_runs(); + assert_eq!(runs3.len(), 1); + assert_eq!( + intersect(&runs3, &r3.preimage(&runs3)), + r3.kept_ranges(&runs3).unwrap() + ); + } +} + +fn main() { + println!( + "tier: avx512f={} avx2={}", + cfg!(target_feature = "avx512f"), + cfg!(target_feature = "avx2") + ); + const REPS: usize = 5; + for n in [1_000u32, 10_000, 100_000, 1_000_000] { + for (name, banded) in [("F3", false), ("F4", true)] { + let f = tree(n, banded); + let s = scatter(n, &f); + let (want, t_oracle) = best(REPS, || oracle(&f)); + let want = sorted(want); + let pairs = want.len(); + + let (r, t_ingest) = best(REPS, || Relation2::ingest(n as usize, &f).unwrap()); + let (rs, t_ingest_s) = best(REPS, || Relation2::ingest(n as usize, &s).unwrap()); + let (_, t_mono) = best(REPS, || r.dst_monotone()); + + let (c, t_gc) = best(REPS, || r.count()); + let (cs, t_gs) = best(REPS, || rs.count()); + let mut kept = vec![0u64; r.valid.len()]; + let mut out = Vec::with_capacity(pairs); + let (_, t_gc_pairs) = best(REPS, || r.pairs(&mut kept, &mut out)); + assert_eq!(sorted(out.clone()), want); + let (_, t_gs_pairs) = best(REPS, || rs.pairs(&mut kept, &mut out)); + assert_eq!(out.len(), pairs); + + let (rc, t_rscan) = best(REPS, || total(&r.kept_ranges(&r.scanned_runs()).unwrap())); + let declared = r.declared_runs.clone().unwrap(); + let (rd, t_rdecl) = best(REPS, || total(&r.kept_ranges(&declared).unwrap())); + let kept_r = r.kept_ranges(&declared).unwrap(); + let (_, t_r_pairs) = best(REPS, || r.range_pairs(&kept_r, &mut out)); + assert_eq!(sorted(out.clone()), want); + assert!(c == pairs && cs == pairs && rc == pairs && rd == pairs); + + let per_row = |t: f64| t * 1e9 / n as f64; + let per_pair = |t: f64| t * 1e9 / pairs.max(1) as f64; + println!( + "{name} n={n:>8} pairs={pairs:>8} runs={:>3} | count ns/row: G-scat {:.3} G-clus {:.3} R-scan {:.4} R-decl {:.5} \ + | emit ns/pair: G-scat {:.2} G-clus {:.2} R {:.2} | ingest ns/row: clus {:.2} scat {:.2} mono {:.3} \ + | datafrog ns/pair {:.1}", + declared.len(), + per_row(t_gs), + per_row(t_gc), + per_row(t_rscan), + per_row(t_rdecl), + per_pair(t_gs_pairs), + per_pair(t_gc_pairs), + per_pair(t_r_pairs), + per_row(t_ingest), + per_row(t_ingest_s), + per_row(t_mono), + per_pair(t_oracle), + ); + } + } +} From d090bf947101ca8353084e60af8efecc352f1103 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 14 Oct 2026 22:06:25 +0000 Subject: [PATCH 12/13] D-MTP-1b probe: name the fixture in the range-arm assertion Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_011BMJnDBAfzcuQh5HeT39Rh --- crates/lance-graph-metta/examples/join_range_probe.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/lance-graph-metta/examples/join_range_probe.rs b/crates/lance-graph-metta/examples/join_range_probe.rs index be3970867..14d907ae2 100644 --- a/crates/lance-graph-metta/examples/join_range_probe.rs +++ b/crates/lance-graph-metta/examples/join_range_probe.rs @@ -98,7 +98,7 @@ mod tests { let scanned = r.scanned_runs(); assert_eq!(declared, scanned, "n={n} banded={banded}"); assert_eq!(gather_pairs(&r), want, "G-clustered n={n}"); - assert_eq!(range_pairs(&r, &scanned), want, "R n={n}"); + assert_eq!(range_pairs(&r, &scanned), want, "R n={n} banded={banded}"); assert_eq!(r.count(), want.len()); let kept = r.kept_ranges(&scanned).unwrap(); assert_eq!(total(&kept), want.len()); From 457a8c411709d6c9a30ec0f8d663786c7c2eb56a Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 14 Oct 2026 22:08:28 +0000 Subject: [PATCH 13/13] D-MTP-1b results: the hop as a range, 3-4 orders of magnitude below the gather count at 1e6 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The order witness is checked once at ingest (a first version recomputed it per query, which made the range count O(n)). Plan §6d, board row and entry. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_011BMJnDBAfzcuQh5HeT39Rh --- .claude/board/INTEGRATION_PLANS.md | 2 +- .claude/board/STATUS_BOARD.md | 2 +- .../entries/2026-10-14-d-mtp-1b-range-hop.md | 19 ++++++++ .claude/board/entries/README.md | 3 +- .claude/plans/metta-polyfill-v1.md | 47 ++++++++++++++++++- .../lance-graph-metta/examples/common/mod.rs | 6 ++- .../examples/join_range_probe.rs | 6 +-- 7 files changed, 77 insertions(+), 8 deletions(-) create mode 100644 .claude/board/entries/2026-10-14-d-mtp-1b-range-hop.md diff --git a/.claude/board/INTEGRATION_PLANS.md b/.claude/board/INTEGRATION_PLANS.md index e4d300562..666580bb7 100644 --- a/.claude/board/INTEGRATION_PLANS.md +++ b/.claude/board/INTEGRATION_PLANS.md @@ -1,6 +1,6 @@ ## 2026-10-14 (2) — metta-polyfill-v1 — Hyperon/MeTTa as a front end over the masking-algebra polyfill → `.claude/plans/metta-polyfill-v1.md` -**Status:** PROPOSAL; D-MTP-0 (map + measurements), D-MTP-1 (join probe, measured) and D-MTP-6 (datafrog oracle) done; D-MTP-2..5 not authorized. +**Status:** PROPOSAL; D-MTP-0 (map + measurements), D-MTP-1 (join probe, measured), D-MTP-1b (range hop, measured) and D-MTP-6 (datafrog oracle) done; D-MTP-2..5 not authorized. - MeTTa is a declaration IR, not an execution IR: the Datalog subset lowers through `lance-graph-relalg` → R2IL → mask-risc; everything else stays in Hyperon's interpreter. - Seam: a named `LanceSpace` implementing `hyperon_space::Space`; `ModuleSpace` splits conjunctions, so `&self` is not the seam. diff --git a/.claude/board/STATUS_BOARD.md b/.claude/board/STATUS_BOARD.md index 3d4565cec..bd0c62540 100644 --- a/.claude/board/STATUS_BOARD.md +++ b/.claude/board/STATUS_BOARD.md @@ -6,7 +6,7 @@ Plan: `.claude/plans/metta-polyfill-v1.md`. Proposed crate `lance-graph-metta` ( |---|---|---|---| | **D-MTP-0** | Adoption map, MeTTa-as-IR verdict, PeTTa vs Hyperon measurements, trie panic ≥ 1,024 children | Done 2026-10-14 (entry `2026-10-14-metta-polyfill-map.md`) | — | | **D-MTP-1** | Two-hop join lowered by hand through Quack `Semijoin` (relalg has no hop), n = 10²..10⁶, v4 | Measured 2026-10-14: pairs == datafrog at every n, counts == Hyperon/PeTTa; at 10⁶ emit 4.0–5.7 ns/pair vs datafrog 59–71 (7.2–9.6× with ingest charged per emitted pair); count 1.2–1.8 ns/row, the random `Gather` read, not sub-ns as expected | == oracles; two disable runs red as pre-registered | -| **D-MTP-1b** | The hop as a range: preimage of `valid`'s runs under a monotone `dst` vs the gather, clustered and scattered | In progress (pre-registered §6c) | == datafrog; two disable runs red; costs reported | +| **D-MTP-1b** | The hop as a range: preimage of `valid`'s runs under a monotone `dst` vs the gather, clustered and scattered | Measured (§6d): R-declared count 0.0001–0.0004 ns/row vs gather 1.3–2.4 at 10⁶ | == datafrog; two disable runs red; costs reported | | **D-MTP-2** | `LanceSpace` (`Space + SpaceMut`) as a named space for the non-recursive Datalog subset; golden corpus rerun | Queued (not authorized) | 32/33 golden files still pass; refusals named; whole-conjunction vs per-conjunct calls recorded | | **D-MTP-3** | Recursive Datalog rules → relalg μ through the external driver d′ | Queued (not authorized) | `anc_n` answers identical; no fixpoint state in mask-risc | | **D-MTP-4** | Fork fix for the expression-width trie panic (≥ 1,024 children), test first; distinct from the id-decoding fix `1ffdbd6` | Queued (needs operator's word) | red before, green after | diff --git a/.claude/board/entries/2026-10-14-d-mtp-1b-range-hop.md b/.claude/board/entries/2026-10-14-d-mtp-1b-range-hop.md new file mode 100644 index 000000000..dfc7f573e --- /dev/null +++ b/.claude/board/entries/2026-10-14-d-mtp-1b-range-hop.md @@ -0,0 +1,19 @@ +# 2026-10-14 — D-MTP-1b — the hop as a range (measured) + +**Status:** MEASURED (v4, `avx512f=true`, release, best of 5, three runs). Plan +`.claude/plans/metta-polyfill-v1.md` §6c (pre-registration) and §6d (results). Probe +`crates/lance-graph-metta/examples/join_range_probe.rs`. + +**Outcome:** +- With `dst` non-decreasing over rows (checked once at ingest), the two-hop join's kept rows + are `valid`'s runs intersected with their preimages under `dst` (`partition_point`). + Pairs == datafrog on a BFS tree (F3), the same tree with every third level removed (F4) + and both under a random address permutation; the range path refuses the permuted lane. +- At 10⁶ the count costs 0.0001–0.0004 ns/row with runs recorded at ingest (O(runs·log n), + 0.1–0.4 µs total) and 0.04–0.05 ns/row when the runs are scanned from the plane, against + 1.3–2.4 ns/row for the gather program on the same data. Pair emission is 1.2–2.0 ns/pair. +- Clustering alone (same gather, local reads) saves only 11–36 %; the win comes from the + hop being a range, not from locality. + +**Open:** the order witness is probe-local (the contract's `OrderedLaneWitness` covers sealed +facet lanes only); a chained range for the second hop; scattered layouts keep the gather. diff --git a/.claude/board/entries/README.md b/.claude/board/entries/README.md index d523e01e3..b575ceba5 100644 --- a/.claude/board/entries/README.md +++ b/.claude/board/entries/README.md @@ -25,7 +25,7 @@ index row, (3) no duplicate entry id. Checks 1 and 2 are deliberately opposite directions; the stranding this convention prevents shows up in exactly one of them, never both. -333 entries, 2026-08-06 .. 2026-10-14. +334 entries, 2026-08-06 .. 2026-10-14. | date | entry id | finding | file | |---|---|---|---| @@ -33,6 +33,7 @@ exactly one of them, never both. | 2026-10-14 | `D-MTP-0` | where Hyperon/MeTTa attaches to the masking-algebra polyfill | [2026-10-14-metta-polyfill-map.md](2026-10-14-metta-polyfill-map.md) | | 2026-10-14 | `ledger-revision-after-1553` | | [2026-10-14-ledger-revision-after-1553.md](2026-10-14-ledger-revision-after-1553.md) | | 2026-10-14 | `D-NALP-4` | | [2026-10-14-isa-truth-exact-codes.md](2026-10-14-isa-truth-exact-codes.md) | +| 2026-10-14 | `D-MTP-1b` | the hop as a range (measured) | [2026-10-14-d-mtp-1b-range-hop.md](2026-10-14-d-mtp-1b-range-hop.md) | | 2026-10-14 | `D-MTP-1` | the MeTTa two-hop join through Quack Semijoin (measured) | [2026-10-14-d-mtp-1-join.md](2026-10-14-d-mtp-1-join.md) | | 2026-10-14 | `D-HOLO-H6b` | normal forms on the Context × Operation tile (measured) | [2026-10-14-d-holo-h6b.md](2026-10-14-d-holo-h6b.md) | | 2026-10-14 | `D-HOLO` | | [2026-10-14-d-holo-falsifiers.md](2026-10-14-d-holo-falsifiers.md) | diff --git a/.claude/plans/metta-polyfill-v1.md b/.claude/plans/metta-polyfill-v1.md index 5be6ccd42..351de5def 100644 --- a/.claude/plans/metta-polyfill-v1.md +++ b/.claude/plans/metta-polyfill-v1.md @@ -333,6 +333,51 @@ should not exceed G-clustered's. Reported per row and per pair, v4, program-printed tier, best of 5, three runs; ingest (including the monotonicity pass and, for R-declared, the run recording) reported separately. +## 6d. D-MTP-1b results (2026-10-14) + +Probe `crates/lance-graph-metta/examples/join_range_probe.rs` (3 tests, shared +fixture code in `examples/common/mod.rs`). v4 host, program-printed +`avx512f=true`, release, best of 5, three runs; ranges span the three. + +**Correctness.** Every arm's pairs equal datafrog's (sorted) on F3, F4 and the +scattered twins at n = 100, 1,000, 4,097; counts equal at every n up to 10⁶. +The range path refuses the scattered (non-monotone) lane. Disable runs, after +committing: (1) taking the preimage of `valid`'s runs without intersecting +with `valid` itself fails F4 (`banded=true`) while F3 stays green; (2) taking +`valid`'s runs without the preimage fails already on F3. Both as +pre-registered. A fixture test pins that both halves of the intersection +change the answer on F4. + +| n = 10⁶ | G-scattered count ns/row | G-clustered count ns/row | R-scan count ns/row | R-declared count ns/row | emit ns/pair G-clustered / R | datafrog ns/pair | +|---|---|---|---|---|---|---| +| F3 (999,995 pairs, 1 run) | 1.96–3.13 | 1.74–1.99 | 0.037–0.055 | 0.00010–0.00011 | 3.4–5.2 / 1.5–1.8 | 16–24 | +| F4 (66,576 pairs, 4 runs) | 1.52–2.43 | 1.27–2.08 | 0.037–0.051 | 0.00026–0.00038 | 21–35 / 1.2–2.0 | 144–196 | + +Ingest is charged separately: 3.4–7.1 ns/row clustered, of which the +monotonicity pass is 0.5–0.9 ns/row (scalar `windows(2)`), and 7.8–10.2 ns/row +scattered. G-clustered's emission per pair on F4 is high because the gather +pass runs over all 10⁶ rows for 66,576 pairs. + +**Against the pre-registration.** +- Pairs identical; two disable runs red: met. +- "R count far below a nanosecond per row": met. R-declared is + O(runs · log n), about 0.1–0.4 µs for the whole count at 10⁶, 3,000–24,000× + below the gather. R-scan pays the word scan of the plane (n/64 words) and + sits at 0.04–0.05 ns/row. +- "G-clustered between G-scattered and R": met only barely. Clustering the + same gather program saves 11–36 %; the gather's per-row work, not its cache + misses, sets most of its cost at this size. +- "R emission not above G-clustered": met (1.2–2.0 ns/pair; the remaining + cost is the dependent read `dst[dst[x]]`). +- The fixture note "about depth·2/3 runs" was wrong: adjacent kept levels + merge, so F4 has one run per band of two levels (2 runs at 4,097, 4 at 10⁶). + +**Scope.** The range path needs the order witness; it is a probe-local flag +set at ingest, not the contract's `OrderedLaneWitness`, which covers sealed +facet lanes only. A scattered layout (F3s/F4s) has no range path and keeps +the gather. Pair emission still reads `dst` twice per pair; a chained range +(the second hop as a range too) is not built here. + ## 7. Deliverables | D-id | scope | gate | @@ -342,7 +387,7 @@ monotonicity pass and, for R-declared, the run recording) reported separately. | D-MTP-2 | `LanceSpace` adapter (`Space + SpaceMut`) for the non-recursive Datalog subset, bound as a named space; Hyperon's golden corpus (`hyperon-probe/golden`, 32 of 33 passing) rerun with facts moved into it | every golden file that passes today still passes; refusals named; the probe records whether the space received whole conjunctions or single conjuncts | | D-MTP-3 | recursive Datalog rules declared into `LanceSpace` → relalg `μ` through d′, queried by `match` over the derived relation | `anc_n` answers identical; no fixpoint state inside mask-risc | | D-MTP-4 | the §3.1 expression-width panic as a fix on the fork, test first (separate from `1ffdbd6`) | red before, green after; upstream only with the operator's word | -| D-MTP-1b | the hop as a range (§6c): preimage of `valid`'s runs under a monotone `dst`, against the gather on clustered and scattered layouts | pairs == datafrog on F3/F4 and their scattered twins; two disable runs red; costs reported | +| D-MTP-1b | the hop as a range (§6c): preimage of `valid`'s runs under a monotone `dst`, against the gather on clustered and scattered layouts | pairs == datafrog on F3/F4 and their scattered twins; two disable runs red; costs reported — **measured, §6d** | | D-MTP-6 | datafrog as the exact Rust oracle in D-MTP-1/3 probes and the `ruff_metta_spo` lowerable tag (test-only dependency) | answers identical to Hyperon and PeTTa as bags of pairs, not only counts | | D-MTP-5 | PeTTa as the second oracle in the probes (needs SWI-Prolog ≥ 9.3; 10.0.2 installed from the official PPA) | divergences between PeTTa and Hyperon recorded, never averaged | diff --git a/crates/lance-graph-metta/examples/common/mod.rs b/crates/lance-graph-metta/examples/common/mod.rs index f8c77078d..8e98abf2f 100644 --- a/crates/lance-graph-metta/examples/common/mod.rs +++ b/crates/lance-graph-metta/examples/common/mod.rs @@ -27,6 +27,8 @@ pub struct Relation2 { /// `valid`'s runs, recorded during ingest when the facts arrive in /// address order (`None` otherwise). pub declared_runs: Option>, + /// The order witness, checked once at ingest: `dst` non-decreasing. + pub monotone: bool, } #[derive(Debug, PartialEq, Eq)] @@ -77,11 +79,13 @@ impl Relation2 { prev = *d; } } + let monotone = dst.windows(2).all(|w| w[0] <= w[1]); Ok(Relation2 { n, dst, valid, declared_runs: in_order.then_some(runs), + monotone, }) } @@ -186,7 +190,7 @@ impl Relation2 { /// The kept rows of `Plane(valid) AND Semijoin(dst, valid)` as intervals. /// `None` when the order witness fails: the caller uses the gather. pub fn kept_ranges(&self, valid_runs: &[Run]) -> Option> { - if !self.dst_monotone() { + if !self.monotone { return None; } Some(intersect(valid_runs, &self.preimage(valid_runs))) diff --git a/crates/lance-graph-metta/examples/join_range_probe.rs b/crates/lance-graph-metta/examples/join_range_probe.rs index 14d907ae2..2c549d30e 100644 --- a/crates/lance-graph-metta/examples/join_range_probe.rs +++ b/crates/lance-graph-metta/examples/join_range_probe.rs @@ -93,7 +93,7 @@ mod tests { let want = sorted(oracle(&f)); assert!(!want.is_empty()); let r = Relation2::ingest(n as usize, &f).unwrap(); - assert!(r.dst_monotone()); + assert!(r.dst_monotone() && r.monotone); let declared = r.declared_runs.clone().expect("address order"); let scanned = r.scanned_runs(); assert_eq!(declared, scanned, "n={n} banded={banded}"); @@ -115,7 +115,7 @@ mod tests { fn a_non_monotone_lane_is_refused() { let s = scatter(1000, &tree(1000, true)); let r = Relation2::ingest(1000, &s).unwrap(); - assert!(!r.dst_monotone()); + assert!(!r.dst_monotone() && !r.monotone); assert!(r.declared_runs.is_none()); assert!(r.kept_ranges(&r.scanned_runs()).is_none()); } @@ -160,7 +160,7 @@ fn main() { let (r, t_ingest) = best(REPS, || Relation2::ingest(n as usize, &f).unwrap()); let (rs, t_ingest_s) = best(REPS, || Relation2::ingest(n as usize, &s).unwrap()); - let (_, t_mono) = best(REPS, || r.dst_monotone()); + let (_, t_mono) = best(REPS, || std::hint::black_box(std::hint::black_box(&r).dst_monotone())); let (c, t_gc) = best(REPS, || r.count()); let (cs, t_gs) = best(REPS, || rs.count());