From b89151812dcfa4ed9c2c2a7f30a33557e81a6786 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 10 Oct 2026 06:41:29 +0000 Subject: [PATCH 1/4] WIP: lance 13 / arrow 58 / datafusion 54 manifest bump (not yet compiled) lance 2.0 -> 13.*, arrow* / parquet / arrow-flight 57 -> 58, datafusion 51 -> 54, sqlparser 0.60 -> 0.62 (datafusion-sql 54's). datafusion is taken with default features minus `compression`: that feature links `lzma` via liblzma, while the n8n vendor crates (datafusion 51) link it via xz2, and cargo allows one `links = "lzma"` per graph. lance 13 does not enable it either. No arrow type crosses the n8n boundary (only n8n_grpc's string/bytes handlers), so arrow 57 and 58 coexist. The graph resolves (cargo metadata); compile + tests follow. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_017YPZhKpSxN7eSxk7VH2vzB --- Cargo.toml | 44 +++++++++++++++++++++++++------------------- 1 file changed, 25 insertions(+), 19 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index e8a6dfd..5c31031 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -73,22 +73,22 @@ ladybug-contract = { path = "crates/ladybug-contract" } # ----------------------------------------------------------------------------- # Storage: LanceDB + Arrow ecosystem # Lance 2.0.0 released on crates.io (Feb 5, 2026). -# NOT bumped to 12 in the 2026-09-25 sweep: lance 12 requires arrow ^58 / -# datafusion ^54, and this crate is on arrow 57 / datafusion 51. The lance -# bump lands together with that migration, never alone. -# Arrow 57.x aligns with DataFusion 52 / Lance 2.x. +# 2026-10-10: lance 13 lands together with arrow 58 / datafusion 54, as the +# 2026-09-25 note required (lance 12+ needs arrow ^58 / datafusion ^54). The +# n8n vendor crates stay on their own arrow line: no arrow type crosses the +# n8n boundary (only n8n_grpc's string/bytes handlers), so cargo carries both. # ----------------------------------------------------------------------------- -lance = { version = "2.0", optional = true, default-features = false } -arrow = { version = "57", features = ["ffi"] } -arrow-array = "57" -arrow-schema = "57" -arrow-buffer = "57" -arrow-data = "57" -arrow-cast = "57" -arrow-select = "57" -parquet = { version = "57", features = ["snap", "lz4", "zstd"] } -arrow-flight = { version = "57", features = ["flight-sql-experimental"], optional = true } -arrow-ipc = "57" +lance = { version = "13.*", optional = true, default-features = false } +arrow = { version = "58", features = ["ffi"] } +arrow-array = "58" +arrow-schema = "58" +arrow-buffer = "58" +arrow-data = "58" +arrow-cast = "58" +arrow-select = "58" +parquet = { version = "58", features = ["snap", "lz4", "zstd"] } +arrow-flight = { version = "58", features = ["flight-sql-experimental"], optional = true } +arrow-ipc = "58" # tonic/prost must match arrow-flight's versions (0.14.1) tonic = { version = "0.14", optional = true } prost = { version = "0.14", optional = true } @@ -96,11 +96,17 @@ bytes = "1.11" # ----------------------------------------------------------------------------- # Query: DataFusion SQL engine -# DataFusion 51.x - upgrade to 52 when liblzma conflict fixed -# To use DF 52: uncomment patch below and set datafusion = "52" +# DataFusion 54 (lance 13 requires ^54). # ----------------------------------------------------------------------------- -datafusion = "51" -sqlparser = "0.60" +# Default features minus `compression`: that feature links `lzma` via +# liblzma, and the n8n vendor crates (datafusion 51) link it via xz2 -- cargo +# allows one `links = "lzma"` per graph. lance 13 itself does not use it. +datafusion = { version = "54", default-features = false, features = [ + "nested_expressions", "crypto_expressions", "datetime_expressions", + "encoding_expressions", "regex_expressions", "string_expressions", + "unicode_expressions", "parquet", "recursive_protection", "sql", +] } +sqlparser = "0.62" # ----------------------------------------------------------------------------- # Graph: Neo4j driver From 924ad537e5d02a8d7c6d5df3a74f2629ffae09c6 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 10 Oct 2026 07:21:47 +0000 Subject: [PATCH 2/4] datafusion 54: drop as_any, properties returns &Arc DataFusion 54 removes `as_any` from ScalarUDFImpl, ExecutionPlan and TableProvider (they take `Any` as a supertrait now) and changes `ExecutionPlan::properties` to return `&Arc`. The plan structs in query/{dn_tree_provider,fingerprint_table,graph_provider, scent_scan}.rs store `Arc`; the 22 `as_any` impls and their now-unused `std::any::Any` imports are removed. The lance 13 storage code (storage/lance*.rs, query/datafusion.rs) compiled unchanged. `cargo +1.99.0 check --all-targets`: the lib and lib tests compile; the one remaining error is pre-existing on main (tests/proof_level_a_gaps.rs imports `ladybug::core::simd`, which does not exist there either). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_017YPZhKpSxN7eSxk7VH2vzB --- src/query/cognitive_udfs.rs | 37 ---------------------------------- src/query/dn_tree_provider.rs | 14 +++---------- src/query/fingerprint_table.rs | 14 +++---------- src/query/graph_provider.rs | 24 ++++++---------------- src/query/scent_scan.rs | 17 +++------------- 5 files changed, 15 insertions(+), 91 deletions(-) diff --git a/src/query/cognitive_udfs.rs b/src/query/cognitive_udfs.rs index b0344b9..68d6aa2 100644 --- a/src/query/cognitive_udfs.rs +++ b/src/query/cognitive_udfs.rs @@ -16,7 +16,6 @@ //! - `nars_abduction(f1, c1, f2, c2)` -> Abduction truth function //! - `nars_revision(f1, c1, f2, c2)` -> Revision truth function -use std::any::Any; use std::sync::Arc; use arrow::array::*; @@ -196,9 +195,6 @@ impl Default for HammingUdf { } impl ScalarUDFImpl for HammingUdf { - fn as_any(&self) -> &dyn Any { - self - } fn name(&self) -> &str { "hamming" @@ -310,9 +306,6 @@ impl Default for SimilarityUdf { } impl ScalarUDFImpl for SimilarityUdf { - fn as_any(&self) -> &dyn Any { - self - } fn name(&self) -> &str { "similarity" @@ -425,9 +418,6 @@ impl Default for PopcountUdf { } impl ScalarUDFImpl for PopcountUdf { - fn as_any(&self) -> &dyn Any { - self - } fn name(&self) -> &str { "popcount" @@ -542,9 +532,6 @@ impl Default for XorBindUdf { } impl ScalarUDFImpl for XorBindUdf { - fn as_any(&self) -> &dyn Any { - self - } fn name(&self) -> &str { "xor_bind" @@ -613,9 +600,6 @@ impl Default for ExtractScentUdf { } impl ScalarUDFImpl for ExtractScentUdf { - fn as_any(&self) -> &dyn Any { - self - } fn name(&self) -> &str { "extract_scent" @@ -705,9 +689,6 @@ impl Default for ScentDistanceUdf { } impl ScalarUDFImpl for ScentDistanceUdf { - fn as_any(&self) -> &dyn Any { - self - } fn name(&self) -> &str { "scent_distance" @@ -809,9 +790,6 @@ impl Default for NarsDeductionUdf { } impl ScalarUDFImpl for NarsDeductionUdf { - fn as_any(&self) -> &dyn Any { - self - } fn name(&self) -> &str { "nars_deduction" @@ -876,9 +854,6 @@ impl Default for NarsInductionUdf { } impl ScalarUDFImpl for NarsInductionUdf { - fn as_any(&self) -> &dyn Any { - self - } fn name(&self) -> &str { "nars_induction" @@ -943,9 +918,6 @@ impl Default for NarsAbductionUdf { } impl ScalarUDFImpl for NarsAbductionUdf { - fn as_any(&self) -> &dyn Any { - self - } fn name(&self) -> &str { "nars_abduction" @@ -1016,9 +988,6 @@ impl Default for NarsRevisionUdf { } impl ScalarUDFImpl for NarsRevisionUdf { - fn as_any(&self) -> &dyn Any { - self - } fn name(&self) -> &str { "nars_revision" @@ -1138,9 +1107,6 @@ impl Default for MembraneEncodeUdf { } impl ScalarUDFImpl for MembraneEncodeUdf { - fn as_any(&self) -> &dyn Any { - self - } fn name(&self) -> &str { "membrane_encode" @@ -1225,9 +1191,6 @@ impl Default for MembraneDecodeUdf { } impl ScalarUDFImpl for MembraneDecodeUdf { - fn as_any(&self) -> &dyn Any { - self - } fn name(&self) -> &str { "membrane_decode" diff --git a/src/query/dn_tree_provider.rs b/src/query/dn_tree_provider.rs index 4794551..c8bf0be 100644 --- a/src/query/dn_tree_provider.rs +++ b/src/query/dn_tree_provider.rs @@ -12,7 +12,6 @@ //! After BindSpace Unification, this provider reads directly from BindSpace //! via the DnIndex — no separate DnSpineCache needed. -use std::any::Any; use std::sync::Arc; use arrow::array::*; @@ -98,9 +97,6 @@ impl DnTreeTableProvider { #[async_trait] impl TableProvider for DnTreeTableProvider { - fn as_any(&self) -> &dyn Any { - self - } fn schema(&self) -> SchemaRef { self.schema.clone() @@ -140,7 +136,7 @@ impl TableProvider for DnTreeTableProvider { projected_schema, bind_space: self.bind_space.clone(), projection: projection.cloned(), - properties, + properties: Arc::new(properties), })) } } @@ -154,7 +150,7 @@ struct DnTreeScan { projected_schema: SchemaRef, bind_space: Arc>, projection: Option>, - properties: PlanProperties, + properties: Arc, } impl std::fmt::Debug for DnTreeScan { @@ -178,15 +174,11 @@ impl ExecutionPlan for DnTreeScan { "DnTreeScan" } - fn as_any(&self) -> &dyn Any { - self - } - fn schema(&self) -> SchemaRef { self.projected_schema.clone() } - fn properties(&self) -> &PlanProperties { + fn properties(&self) -> &Arc { &self.properties } diff --git a/src/query/fingerprint_table.rs b/src/query/fingerprint_table.rs index 10cf5c4..2a4e915 100644 --- a/src/query/fingerprint_table.rs +++ b/src/query/fingerprint_table.rs @@ -18,7 +18,6 @@ //! - access_count: UINT32 //! - zone: UTF8 ('surface', 'fluid', 'node') -use std::any::Any; use std::sync::Arc; use arrow::array::*; @@ -111,9 +110,6 @@ impl std::fmt::Debug for FingerprintTableProvider { #[async_trait] impl TableProvider for FingerprintTableProvider { - fn as_any(&self) -> &dyn Any { - self - } fn schema(&self) -> SchemaRef { self.schema.clone() @@ -148,7 +144,7 @@ pub struct BindSpaceScan { projected_schema: SchemaRef, bind_space: Arc>, projection: Option>, - properties: PlanProperties, + properties: Arc, } impl std::fmt::Debug for BindSpaceScan { @@ -187,7 +183,7 @@ impl BindSpaceScan { projected_schema, bind_space, projection, - properties, + properties: Arc::new(properties), } } } @@ -203,15 +199,11 @@ impl ExecutionPlan for BindSpaceScan { "BindSpaceScan" } - fn as_any(&self) -> &dyn Any { - self - } - fn schema(&self) -> SchemaRef { self.projected_schema.clone() } - fn properties(&self) -> &PlanProperties { + fn properties(&self) -> &Arc { &self.properties } diff --git a/src/query/graph_provider.rs b/src/query/graph_provider.rs index 203025b..3b3caee 100644 --- a/src/query/graph_provider.rs +++ b/src/query/graph_provider.rs @@ -35,7 +35,6 @@ //! - `neighbors`: Function-like traversal (source, hop, target, target_label, via_verb) //! - `dict`: Dictionary view with popcount pre-filter -use std::any::Any; use std::collections::HashSet; use std::fmt; use std::sync::Arc; @@ -142,9 +141,6 @@ impl fmt::Debug for EdgeTableProvider { #[async_trait] impl TableProvider for EdgeTableProvider { - fn as_any(&self) -> &dyn Any { - self - } fn schema(&self) -> SchemaRef { self.schema.clone() @@ -179,7 +175,7 @@ struct EdgeScanExec { projected_schema: SchemaRef, bind_space: Arc>, projection: Option>, - properties: PlanProperties, + properties: Arc, } impl EdgeScanExec { @@ -210,7 +206,7 @@ impl EdgeScanExec { projected_schema, bind_space, projection, - properties, + properties: Arc::new(properties), } } } @@ -234,15 +230,11 @@ impl ExecutionPlan for EdgeScanExec { "EdgeScanExec" } - fn as_any(&self) -> &dyn Any { - self - } - fn schema(&self) -> SchemaRef { self.projected_schema.clone() } - fn properties(&self) -> &PlanProperties { + fn properties(&self) -> &Arc { &self.properties } @@ -396,7 +388,7 @@ pub struct GraphTraversalExec { config: TraversalConfig, dict: Option>>, projection: Option>, - properties: PlanProperties, + properties: Arc, } impl GraphTraversalExec { @@ -430,7 +422,7 @@ impl GraphTraversalExec { config, dict: None, projection, - properties, + properties: Arc::new(properties), } } @@ -468,15 +460,11 @@ impl ExecutionPlan for GraphTraversalExec { "GraphTraversalExec" } - fn as_any(&self) -> &dyn Any { - self - } - fn schema(&self) -> SchemaRef { self.projected_schema.clone() } - fn properties(&self) -> &PlanProperties { + fn properties(&self) -> &Arc { &self.properties } diff --git a/src/query/scent_scan.rs b/src/query/scent_scan.rs index af26f81..925e38e 100644 --- a/src/query/scent_scan.rs +++ b/src/query/scent_scan.rs @@ -34,7 +34,6 @@ //! RecordBatch (filtered results) //! ``` -use std::any::Any; use std::fmt; use std::sync::Arc; @@ -159,7 +158,7 @@ pub struct ScentScanExec { /// Pushed-down predicate predicate: ScentPredicate, /// Plan properties - properties: PlanProperties, + properties: Arc, } impl ScentScanExec { @@ -195,7 +194,7 @@ impl ScentScanExec { bind_space, hdr_index: None, predicate, - properties, + properties: Arc::new(properties), } } @@ -244,15 +243,11 @@ impl ExecutionPlan for ScentScanExec { "ScentScanExec" } - fn as_any(&self) -> &dyn Any { - self - } - fn schema(&self) -> SchemaRef { self.projected_schema.clone() } - fn properties(&self) -> &PlanProperties { + fn properties(&self) -> &Arc { &self.properties } @@ -543,9 +538,6 @@ impl HammingDistanceUdf { } impl ScalarUDFImpl for HammingDistanceUdf { - fn as_any(&self) -> &dyn Any { - self - } fn name(&self) -> &str { "hamming_distance" @@ -656,9 +648,6 @@ impl SimilarityUdf { } impl ScalarUDFImpl for SimilarityUdf { - fn as_any(&self) -> &dyn Any { - self - } fn name(&self) -> &str { "similarity" From ddee1fae030386fcd5c341007f84cada8d9875b7 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 10 Oct 2026 08:49:47 +0000 Subject: [PATCH 3/4] ci: install protoc for lance-encoding 13 Lance 13's lance-encoding compiles .proto files in its build script and needs protoc plus the well-known includes. The Dockerfiles already install protobuf-compiler; the proof and ci-master workflows did not, so Compile Check failed with 'Could not find protoc'. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_017YPZhKpSxN7eSxk7VH2vzB --- .github/workflows/ci-master.yml | 6 ++++++ .github/workflows/proof.yml | 12 ++++++++++++ 2 files changed, 18 insertions(+) diff --git a/.github/workflows/ci-master.yml b/.github/workflows/ci-master.yml index c464e70..0b9b647 100644 --- a/.github/workflows/ci-master.yml +++ b/.github/workflows/ci-master.yml @@ -35,6 +35,8 @@ jobs: git clone --depth 1 https://github.com/AdaWorldAPI/crewai-rust.git ../crewai-rust git clone --depth 1 https://github.com/AdaWorldAPI/n8n-rs.git ../n8n-rs + - name: Install protoc (lance-encoding 13 build script) + run: sudo apt-get update && sudo apt-get install -y protobuf-compiler libprotobuf-dev - uses: dtolnay/rust-toolchain@master with: toolchain: "1.99.0" @@ -64,6 +66,8 @@ jobs: git clone --depth 1 https://github.com/AdaWorldAPI/crewai-rust.git ../crewai-rust git clone --depth 1 https://github.com/AdaWorldAPI/n8n-rs.git ../n8n-rs + - name: Install protoc (lance-encoding 13 build script) + run: sudo apt-get update && sudo apt-get install -y protobuf-compiler libprotobuf-dev - uses: dtolnay/rust-toolchain@master with: toolchain: "1.99.0" @@ -98,6 +102,8 @@ jobs: git clone --depth 1 https://github.com/AdaWorldAPI/crewai-rust.git ../crewai-rust git clone --depth 1 https://github.com/AdaWorldAPI/n8n-rs.git ../n8n-rs + - name: Install protoc (lance-encoding 13 build script) + run: sudo apt-get update && sudo apt-get install -y protobuf-compiler libprotobuf-dev - uses: dtolnay/rust-toolchain@master with: toolchain: "1.99.0" diff --git a/.github/workflows/proof.yml b/.github/workflows/proof.yml index 2256a1b..a0589be 100644 --- a/.github/workflows/proof.yml +++ b/.github/workflows/proof.yml @@ -63,6 +63,8 @@ jobs: git clone --depth 1 https://github.com/AdaWorldAPI/crewai-rust.git ../crewai-rust git clone --depth 1 https://github.com/AdaWorldAPI/n8n-rs.git ../n8n-rs + - name: Install protoc (lance-encoding 13 build script) + run: sudo apt-get update && sudo apt-get install -y protobuf-compiler libprotobuf-dev - uses: dtolnay/rust-toolchain@master with: toolchain: "1.99.0" @@ -87,6 +89,8 @@ jobs: git clone --depth 1 https://github.com/AdaWorldAPI/crewai-rust.git ../crewai-rust git clone --depth 1 https://github.com/AdaWorldAPI/n8n-rs.git ../n8n-rs + - name: Install protoc (lance-encoding 13 build script) + run: sudo apt-get update && sudo apt-get install -y protobuf-compiler libprotobuf-dev - uses: dtolnay/rust-toolchain@master with: toolchain: "1.99.0" @@ -111,6 +115,8 @@ jobs: git clone --depth 1 https://github.com/AdaWorldAPI/crewai-rust.git ../crewai-rust git clone --depth 1 https://github.com/AdaWorldAPI/n8n-rs.git ../n8n-rs + - name: Install protoc (lance-encoding 13 build script) + run: sudo apt-get update && sudo apt-get install -y protobuf-compiler libprotobuf-dev - uses: dtolnay/rust-toolchain@master with: toolchain: "1.99.0" @@ -135,6 +141,8 @@ jobs: git clone --depth 1 https://github.com/AdaWorldAPI/crewai-rust.git ../crewai-rust git clone --depth 1 https://github.com/AdaWorldAPI/n8n-rs.git ../n8n-rs + - name: Install protoc (lance-encoding 13 build script) + run: sudo apt-get update && sudo apt-get install -y protobuf-compiler libprotobuf-dev - uses: dtolnay/rust-toolchain@master with: toolchain: "1.99.0" @@ -159,6 +167,8 @@ jobs: git clone --depth 1 https://github.com/AdaWorldAPI/crewai-rust.git ../crewai-rust git clone --depth 1 https://github.com/AdaWorldAPI/n8n-rs.git ../n8n-rs + - name: Install protoc (lance-encoding 13 build script) + run: sudo apt-get update && sudo apt-get install -y protobuf-compiler libprotobuf-dev - uses: dtolnay/rust-toolchain@master with: toolchain: "1.99.0" @@ -183,6 +193,8 @@ jobs: git clone --depth 1 https://github.com/AdaWorldAPI/crewai-rust.git ../crewai-rust git clone --depth 1 https://github.com/AdaWorldAPI/n8n-rs.git ../n8n-rs + - name: Install protoc (lance-encoding 13 build script) + run: sudo apt-get update && sudo apt-get install -y protobuf-compiler libprotobuf-dev - uses: dtolnay/rust-toolchain@master with: toolchain: "1.99.0" From a6e1b51ec6228e9533fe495855bfffe8ef3ab450 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 10 Oct 2026 09:19:00 +0000 Subject: [PATCH 4/4] test: repoint proof_level_a_gaps at core::hamming_distance core::simd was removed in ced28ba but this proof still imported core::simd::{hamming_distance, hamming_scalar}, so the test target did not compile (on main too). Import the live core::hamming_distance and keep the scalar reference as a local test oracle (popcount of word-wise XOR). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_017YPZhKpSxN7eSxk7VH2vzB --- tests/proof_level_a_gaps.rs | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/tests/proof_level_a_gaps.rs b/tests/proof_level_a_gaps.rs index 12a0003..22f3778 100644 --- a/tests/proof_level_a_gaps.rs +++ b/tests/proof_level_a_gaps.rs @@ -20,7 +20,7 @@ use ladybug::cognitive::{ use ladybug::container::search::cascade_search; use ladybug::container::{CONTAINER_BITS, Container}; use ladybug::core::Fingerprint; -use ladybug::core::simd::{hamming_distance, hamming_scalar}; +use ladybug::core::hamming_distance; use ladybug::nars::TruthValue; use ladybug::storage::{FINGERPRINT_WORDS, WalEntry}; @@ -28,6 +28,19 @@ use ladybug::storage::{FINGERPRINT_WORDS, WalEntry}; // A.1.4: SIMD–Scalar Equivalence // ============================================================================= +/// Scalar reference (test oracle): popcount of the word-wise XOR. +/// +/// `core::simd::hamming_scalar` was removed with `core::simd` (ced28ba); the +/// oracle lives here so the SIMD-dispatched path is still checked against an +/// independent scalar computation. +fn hamming_scalar(a: &Fingerprint, b: &Fingerprint) -> u32 { + a.as_raw() + .iter() + .zip(b.as_raw().iter()) + .map(|(x, y)| (x ^ y).count_ones()) + .sum() +} + /// PROOF A.1.4: hamming_distance() == hamming_scalar() for all inputs /// /// The SIMD-dispatched `hamming_distance` must produce identical results to