From 3c083a8233ea436bbe601f0d99ce47059f3984f2 Mon Sep 17 00:00:00 2001 From: Shanu Date: Mon, 17 Aug 2026 17:58:01 +0530 Subject: [PATCH 1/9] Extend the module E2E to the declared surface and to durability MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Issue #18 §E5. The loader E2E drove three of the eighteen families the module advertises, and nothing checked that a write reached the workspace it was given. **Every declared method is routed.** The module advertises `Capabilities::all()` and declares 88 methods; the existing manifest test compares two lists, so it passes for a method that is declared, listed, and answers "unknown member" — the bus-level version of a capability set that overstates its accessors. This calls each declared method and distinguishes the kind of refusal: a wired method rejects the empty argument list with `ai.tinyhumans.tinybus.Error.BadArguments`, while an unwired one is `ai.tinyhumans.tinybus.Error.UnknownMethod`. The two names are what make the test discriminate rather than pass vacuously, and the unrouted shape is taken from the build under test rather than hard-coded. Eighteen bespoke round trips would assert more, and would also need eighteen sets of valid arguments and eighteen engine preconditions. This asserts the one thing that is true of all of them and is cheap to keep true. **A write lands in the host's workspace.** Every other test here stores and reads back inside one admission, so a module that kept its store in a temporary directory of its own, or in memory, passes all of them — and the difference shows on the user's next launch. §E5 asks for a shutdown/restart cycle, and it is not here because it cannot be: TinyBus never unloads a library, so a second admission in the same process is refused with `ModuleRefused { reason: "module initialization failed" }`. That is the same constraint that already forces every test in this file to be the only one in its process. A real restart needs a second process against a shared workspace, which is a change to the CI loop rather than a test, so the directory assertion covers what restart would have been checking and the limitation is written down rather than left as a gap someone rediscovers. Verified the way CI runs them — every ignored test in its own process, all ten green. Refs #18 (§E5) --- crates/tinymemory-module/tests/module_e2e.rs | 115 +++++++++++++++++++ 1 file changed, 115 insertions(+) diff --git a/crates/tinymemory-module/tests/module_e2e.rs b/crates/tinymemory-module/tests/module_e2e.rs index 79c1344..7c24f77 100644 --- a/crates/tinymemory-module/tests/module_e2e.rs +++ b/crates/tinymemory-module/tests/module_e2e.rs @@ -611,3 +611,118 @@ async fn the_manifest_declares_every_method_the_module_serves() { declared.difference(&expected).collect::>() ); } + +#[tokio::test] +#[ignore = "drives a real dlopen'ed module; must be the only such test in the process — see the module docs"] +async fn what_is_written_lands_in_the_workspace_it_was_given() { + // Issue #18 §E5 asks for a shutdown/restart cycle. It cannot be written + // here, and the reason is structural rather than an omission: TinyBus never + // unloads a library, so a second admission in the same process is refused — + // `ModuleRefused { reason: "module initialization failed" }` — and every + // test in this file must be the only one in its process for the same + // reason. A genuine restart needs a second *process* against a shared + // workspace, which is a change to the CI loop rather than a test. + // + // What is assertable in one process is the property that restart would be + // checking: that a write goes to the durable workspace the host supplied, + // and not somewhere that disappears. A module that stored into a temporary + // directory of its own, or in memory, passes every other test in this file + // — each one stores and reads back inside a single admission, so the + // difference never shows. It would show on the user's next launch. + let workspace = tempfile::tempdir().expect("tempdir"); + + // Nothing has been asked of it yet. + let before = std::fs::read_dir(workspace.path()) + .expect("workspace readable") + .count(); + + let (client, _host, _task) = admit_module(workspace.path()).await; + proxy(&client) + .call::<()>( + "Store", + ( + "e2e", + "durable", + "written to the host's workspace", + MemoryCategory::Core, + Option::::None, + MemoryTaint::default(), + ), + ) + .await + .expect("Store"); + + let entry: Option = proxy(&client) + .call("Get", ("e2e", "durable")) + .await + .expect("Get"); + assert_eq!( + entry.expect("just stored").content, + "written to the host's workspace" + ); + + // The assertion that a same-admission round trip cannot make: the bytes are + // in the directory the host named. + let after = std::fs::read_dir(workspace.path()) + .expect("workspace readable") + .count(); + assert!( + after > before, + "the module answered correctly but wrote nothing into the workspace it \ + was given — a store that does not land here does not survive a restart" + ); +} + +#[tokio::test] +#[ignore = "drives a real dlopen'ed module; must be the only such test in the process — see the module docs"] +async fn every_declared_method_is_actually_routed() { + // Issue #18 §E5 asks the E2E to cover every family the module advertises. + // It advertises `Capabilities::all()` — eighteen families — and the tests + // above exercise three of them. + // + // Rather than eighteen bespoke round trips, this asserts the property that + // makes the advertisement honest at this layer: every method the manifest + // declares is actually *reachable*. `the_manifest_declares_every_method_the + // _module_serves` compares two lists and would pass for a method that is + // declared, routed, and answers "unknown member" — which is the bus-level + // version of a capability set that overstates its accessors. + // + // Each method is called with no arguments, so most fail. That is fine and is + // the point: what is asserted is the *kind* of failure. A method that is + // wired rejects the arguments; a method that is not wired rejects the + // member, and those carry different wire names. + let workspace = tempfile::tempdir().expect("tempdir"); + let (client, _host, _task) = admit_module(workspace.path()).await; + + // Establish the shape of a genuine not-routed refusal from this very build, + // rather than hard-coding tinybus's spelling of it. The two outcomes are + // distinct names — `ai.tinyhumans.tinybus.Error.UnknownMethod` for a member + // that is not there, `…Error.BadArguments` for one that is and did not like + // the empty argument list — which is what makes this test discriminate + // rather than pass vacuously. + let unrouted = proxy(&client) + .call::("NoSuchMethodAtAll", ()) + .await + .expect_err("an unknown member must be refused") + .wire_name() + .to_string(); + + let mut missing = Vec::new(); + for method in EXPECTED_METHODS { + // `Shutdown` would stop the module and strand every later iteration. + if *method == "Shutdown" { + continue; + } + let outcome = proxy(&client).call::(method, ()).await; + if let Err(error) = outcome { + if error.wire_name() == unrouted { + missing.push(*method); + } + } + } + + assert!( + missing.is_empty(), + "declared in the manifest but not routed: {missing:?}" + ); +} From 7dedb766fecc6751ca220b772cc486f12a1dc9b7 Mon Sep 17 00:00:00 2001 From: Shanu Date: Mon, 17 Aug 2026 19:22:54 +0530 Subject: [PATCH 2/9] Delete the conversion layer now that both sides name one type MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Issue #18 §A1, the tinymemory half. Companion to tinyhumansai/tinycortex#149, which makes `tinycortex-api` re-export this workspace's contract rather than redefining it. `convert.rs` existed because the engine's contract crate and this one described the same values under two names — the same code, in fact, since `api/` was extracted from `tinycortex-api` and held byte-identical. Being nominally distinct meant every call across the seam translated, and a field added to either contract had to be added to the other and to the conversion. Three places, or the value was silently dropped. With one type set there is nothing to convert: `convert.rs` and its tests are deleted, and the adapter's eleven conversion call sites become plain delegation. `list` and `namespace_summaries` lose an `into_iter().collect()` that had become an identity map, and `store`/`store_with_taint` pass their arguments straight through. The gitlink moves to the commit carrying the re-export. That commit also stops publishing tinycortex, which is what makes the git dependency legal — and records the state that repository was already in, since `cargo package` there has failed since `api/` was split out. The workspace root gains a `[patch]` for the git contract dependency. Without it cargo resolves the git copy *and* the path copy as two distinct crates, and `MemoryCategory` from one is not the same type as the other — the exact duplication this change deletes, reintroduced by the fix for it. Found by the compiler at the seam rather than reasoned about, and the patch table is the same mechanism this workspace already uses for tinycortex itself. Acceptance for §A1, checked rather than asserted: - `adapters/tinycortex/src/convert.rs` is deleted - no conversion function remains anywhere in the workspace (grep: 0 hits) - the workspace builds and tests green with no conversion at the seam `cargo test --all-features` reports 1101 passing, down 9 from 1110. That difference is exactly the nine tests in `convert_test.rs`, which tested the layer this change removes. Refs #18 (§A1) --- Cargo.lock | 1 + Cargo.toml | 13 ++ adapters/tinycortex/src/convert.rs | 197 ------------------------ adapters/tinycortex/src/convert_test.rs | 170 -------------------- adapters/tinycortex/src/lib.rs | 18 ++- adapters/tinycortex/src/memory.rs | 67 ++------ vendor/tinycortex | 2 +- 7 files changed, 40 insertions(+), 428 deletions(-) delete mode 100644 adapters/tinycortex/src/convert.rs delete mode 100644 adapters/tinycortex/src/convert_test.rs diff --git a/Cargo.lock b/Cargo.lock index 8b1112e..dd6b02f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1795,6 +1795,7 @@ dependencies = [ "serde_json", "sha2 0.10.9", "thiserror 2.0.20", + "tinymemory-api", "uuid", ] diff --git a/Cargo.toml b/Cargo.toml index 2915028..2d18feb 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -120,6 +120,19 @@ private_intra_doc_links = "warn" # so a host that already pins its own engine checkout unifies onto one copy # through its own patch table. These entries are what make a *standalone* build # of this workspace resolve them to the nested `vendor/` submodules. +# `tinycortex-api` takes this workspace's own contract crate by git, because +# neither crate is published (tinymemory#18 §A1). Without this entry cargo +# resolves the git copy *and* the path copy as two distinct crates, and +# `tinymemory_api::MemoryCategory` from one is not the same type as from the +# other — which is the exact duplication §A1 exists to delete, reintroduced by +# the fix for it. The error is loud rather than silent, but only at the seam. +# +# Patch tables apply from the workspace root being built, so this covers builds +# and tests here. A host that embeds this workspace needs the same entry, the +# same way it already patches tinycortex. +[patch."https://github.com/tinyhumansai/tinymemory"] +tinymemory-api = { path = "api" } + [patch.crates-io] tinycortex = { path = "vendor/tinycortex" } tinycortex-api = { path = "vendor/tinycortex/api" } diff --git a/adapters/tinycortex/src/convert.rs b/adapters/tinycortex/src/convert.rs deleted file mode 100644 index 8d7d540..0000000 --- a/adapters/tinycortex/src/convert.rs +++ /dev/null @@ -1,197 +0,0 @@ -//! Value conversions between `tinycortex-api` and `tinymemory-api`. -//! -//! The two contracts describe the same values and, today, describe them -//! identically — `tinymemory-api` was moved out of `tinycortex-api`. They are -//! nonetheless distinct Rust types in distinct crates, so a value has to be -//! rebuilt to cross. -//! -//! ## Every conversion destructures exhaustively -//! -//! This is the whole discipline of this file, and the reason it is not written -//! with `..` or field-by-field assignment onto a `Default`. Two contracts that -//! are allowed to drift *will* drift: someone adds a field to one side, and a -//! lenient conversion silently drops it. A struct literal built from a full -//! destructuring pattern turns that into a compile error on the very next -//! build, naming the field. -//! -//! The same applies to the enums: each `match` lists every variant, so a new -//! category or a third taint level cannot fall into a catch-all arm and be -//! quietly downgraded. -//! -//! ## Taint conversion is the security-relevant one -//! -//! [`tinymemory_api::types::MemoryTaint`] records whether content came from outside. Mapping it -//! wrongly — or defaulting it on an unrecognised value — would let -//! externally-sourced content be treated as internal-trust content. Both sides -//! fail closed to `ExternalSync` when decoding an unknown persisted string, and -//! the mapping here is a total two-arm match with no default, so there is -//! nowhere for a wrong answer to come from. - -use tinycortex::memory::types as tc; -use tinymemory_api::types as tm; - -/// Converts a category to the TinyMemory contract's form. -#[must_use] -pub fn category_to_tinymemory(category: tc::MemoryCategory) -> tm::MemoryCategory { - match category { - tc::MemoryCategory::Core => tm::MemoryCategory::Core, - tc::MemoryCategory::Daily => tm::MemoryCategory::Daily, - tc::MemoryCategory::Conversation => tm::MemoryCategory::Conversation, - tc::MemoryCategory::Custom(name) => tm::MemoryCategory::Custom(name), - } -} - -/// Converts a category to the TinyCortex engine's form. -#[must_use] -pub fn category_to_tinycortex(category: tm::MemoryCategory) -> tc::MemoryCategory { - match category { - tm::MemoryCategory::Core => tc::MemoryCategory::Core, - tm::MemoryCategory::Daily => tc::MemoryCategory::Daily, - tm::MemoryCategory::Conversation => tc::MemoryCategory::Conversation, - tm::MemoryCategory::Custom(name) => tc::MemoryCategory::Custom(name), - } -} - -/// Converts provenance to the TinyMemory contract's form. -/// -/// A total match with no default arm: see the module docs on why this one may -/// not be lenient. -#[must_use] -pub fn taint_to_tinymemory(taint: tc::MemoryTaint) -> tm::MemoryTaint { - match taint { - tc::MemoryTaint::Internal => tm::MemoryTaint::Internal, - tc::MemoryTaint::ExternalSync => tm::MemoryTaint::ExternalSync, - } -} - -/// Converts provenance to the TinyCortex engine's form. -#[must_use] -pub fn taint_to_tinycortex(taint: tm::MemoryTaint) -> tc::MemoryTaint { - match taint { - tm::MemoryTaint::Internal => tc::MemoryTaint::Internal, - tm::MemoryTaint::ExternalSync => tc::MemoryTaint::ExternalSync, - } -} - -/// Converts an entry to the TinyMemory contract's form. -#[must_use] -pub fn entry_to_tinymemory(entry: tc::MemoryEntry) -> tm::MemoryEntry { - // Exhaustive destructuring: a field added to the engine's entry breaks this - // line rather than being dropped on the floor. - let tc::MemoryEntry { - id, - key, - content, - namespace, - category, - timestamp, - session_id, - score, - taint, - } = entry; - tm::MemoryEntry { - id, - key, - content, - namespace, - category: category_to_tinymemory(category), - timestamp, - session_id, - score, - taint: taint_to_tinymemory(taint), - } -} - -/// Converts a namespace summary to the TinyMemory contract's form. -#[must_use] -pub fn namespace_summary_to_tinymemory(summary: tc::NamespaceSummary) -> tm::NamespaceSummary { - let tc::NamespaceSummary { - namespace, - count, - last_updated, - } = summary; - tm::NamespaceSummary { - namespace, - count, - last_updated, - } -} - -/// The owned recall filters, in the engine's owned form. -/// -/// Returned owned rather than borrowed because the engine's `RecallOpts` -/// borrows its string fields, and a borrow of a value built inside a conversion -/// function cannot outlive the call. Callers keep this alive and borrow from it. -#[must_use] -pub fn recall_opts_to_tinycortex(opts: &tm::OwnedRecallOpts) -> tc::OwnedRecallOpts { - let tm::OwnedRecallOpts { - namespace, - category, - session_id, - min_score, - cross_session, - } = opts; - tc::OwnedRecallOpts { - namespace: namespace.clone(), - category: category.clone().map(category_to_tinycortex), - session_id: session_id.clone(), - min_score: *min_score, - cross_session: *cross_session, - } -} - -#[cfg(test)] -#[path = "convert_test.rs"] -mod test; - -// ── The reverse direction, for a driver whose contract is TinyMemory's ──────── -// -// Everything above converts engine values *into* the TinyMemory contract, which -// is what wrapping TinyCortex as a TinyMemory driver needs. A module-backed -// driver runs the other way: it speaks TinyMemory, and a host whose binding -// still speaks TinyCortex has to convert its answers back. -// -// Same discipline as above — exhaustive destructuring, total matches, no `..` -// and no `Default` — for the same reason: two contracts allowed to drift will. - -/// Converts an entry to the `TinyCortex` contract's form. -#[must_use] -pub fn entry_to_tinycortex(entry: tm::MemoryEntry) -> tc::MemoryEntry { - let tm::MemoryEntry { - id, - key, - content, - namespace, - category, - timestamp, - session_id, - score, - taint, - } = entry; - tc::MemoryEntry { - id, - key, - content, - namespace, - category: category_to_tinycortex(category), - timestamp, - session_id, - score, - taint: taint_to_tinycortex(taint), - } -} - -/// Converts a namespace summary to the `TinyCortex` contract's form. -#[must_use] -pub fn namespace_summary_to_tinycortex(summary: tm::NamespaceSummary) -> tc::NamespaceSummary { - let tm::NamespaceSummary { - namespace, - count, - last_updated, - } = summary; - tc::NamespaceSummary { - namespace, - count, - last_updated, - } -} diff --git a/adapters/tinycortex/src/convert_test.rs b/adapters/tinycortex/src/convert_test.rs deleted file mode 100644 index 75016ec..0000000 --- a/adapters/tinycortex/src/convert_test.rs +++ /dev/null @@ -1,170 +0,0 @@ -//! Conversion tests. -//! -//! Two contracts that are allowed to drift will drift. The exhaustive -//! destructuring in `convert.rs` catches a *new* field at compile time; these -//! tests catch a *mis-mapped* one, which the compiler cannot see because every -//! field on both sides has the same type. - -#![allow(clippy::expect_used, clippy::panic)] - -use super::*; - -/// Every category must survive a round trip, including the custom variant's -/// payload — a `Custom(String)` mapped onto the wrong arm would silently -/// re-file every custom-categorised memory. -#[test] -fn every_category_round_trips() { - let cases = [ - tm::MemoryCategory::Core, - tm::MemoryCategory::Daily, - tm::MemoryCategory::Conversation, - tm::MemoryCategory::Custom("project-notes".to_string()), - ]; - for category in cases { - let round_tripped = category_to_tinymemory(category_to_tinycortex(category.clone())); - assert_eq!(round_tripped, category); - } -} - -/// The wire form is the persisted form on both sides, so a conversion that -/// round-trips the Rust value but changes the string would still corrupt a -/// store. Comparing the rendered forms catches that. -#[test] -fn category_conversion_preserves_the_persisted_spelling() { - let cases = [ - tm::MemoryCategory::Core, - tm::MemoryCategory::Daily, - tm::MemoryCategory::Conversation, - tm::MemoryCategory::Custom("x".to_string()), - ]; - for category in cases { - let engine = category_to_tinycortex(category.clone()); - assert_eq!( - engine.to_string(), - category.to_string(), - "the two contracts must agree on the persisted spelling" - ); - } -} - -/// Provenance is the security-relevant conversion: mapping `ExternalSync` onto -/// `Internal` would upgrade the trust of externally-sourced content. -#[test] -fn every_taint_round_trips_and_keeps_its_db_spelling() { - for taint in [tm::MemoryTaint::Internal, tm::MemoryTaint::ExternalSync] { - let engine = taint_to_tinycortex(taint); - assert_eq!(taint_to_tinymemory(engine), taint); - assert_eq!( - engine.as_db_str(), - taint.as_db_str(), - "the two contracts must agree on the persisted spelling" - ); - } -} - -/// `ExternalSync` must never come back as `Internal`, stated as its own -/// assertion rather than left implicit in the round trip above. -#[test] -fn external_content_is_never_laundered_into_internal_trust() { - assert_eq!( - taint_to_tinymemory(taint_to_tinycortex(tm::MemoryTaint::ExternalSync)), - tm::MemoryTaint::ExternalSync - ); - assert_ne!( - taint_to_tinymemory(taint_to_tinycortex(tm::MemoryTaint::ExternalSync)), - tm::MemoryTaint::Internal - ); -} - -#[test] -fn an_entry_round_trips_every_field() { - let engine = tc::MemoryEntry { - id: "ns/key".to_string(), - key: "key".to_string(), - content: "body".to_string(), - namespace: Some("ns".to_string()), - category: tc::MemoryCategory::Custom("notes".to_string()), - timestamp: "2026-08-10T00:00:00Z".to_string(), - session_id: Some("s1".to_string()), - score: Some(0.75), - taint: tc::MemoryTaint::ExternalSync, - }; - - let converted = entry_to_tinymemory(engine.clone()); - - assert_eq!(converted.id, engine.id); - assert_eq!(converted.key, engine.key); - assert_eq!(converted.content, engine.content); - assert_eq!(converted.namespace, engine.namespace); - assert_eq!(converted.category.to_string(), engine.category.to_string()); - assert_eq!(converted.timestamp, engine.timestamp); - assert_eq!(converted.session_id, engine.session_id); - assert_eq!(converted.score, engine.score); - assert_eq!(converted.taint, tm::MemoryTaint::ExternalSync); -} - -/// A score of `None` must stay `None`. Defaulting it to `0.0` would make an -/// unranked entry look like a worst-ranked one. -#[test] -fn an_absent_score_stays_absent() { - let engine = tc::MemoryEntry { - id: "i".to_string(), - key: "k".to_string(), - content: "c".to_string(), - namespace: None, - category: tc::MemoryCategory::Core, - timestamp: "t".to_string(), - session_id: None, - score: None, - taint: tc::MemoryTaint::Internal, - }; - let converted = entry_to_tinymemory(engine); - assert!(converted.score.is_none()); - assert!(converted.namespace.is_none()); - assert!(converted.session_id.is_none()); -} - -#[test] -fn a_namespace_summary_round_trips_every_field() { - let engine = tc::NamespaceSummary { - namespace: "projects".to_string(), - count: 12, - last_updated: Some("2026-08-10T00:00:00Z".to_string()), - }; - let converted = namespace_summary_to_tinymemory(engine.clone()); - assert_eq!(converted.namespace, engine.namespace); - assert_eq!(converted.count, engine.count); - assert_eq!(converted.last_updated, engine.last_updated); -} - -/// Every recall filter must cross. A dropped `min_score` or `cross_session` -/// silently widens a query. -#[test] -fn every_recall_filter_crosses() { - let opts = tm::OwnedRecallOpts { - namespace: Some("ns".to_string()), - category: Some(tm::MemoryCategory::Daily), - session_id: Some("s1".to_string()), - min_score: Some(0.5), - cross_session: true, - }; - let engine = recall_opts_to_tinycortex(&opts); - assert_eq!(engine.namespace, opts.namespace); - assert_eq!( - engine.category.as_ref().map(ToString::to_string), - opts.category.as_ref().map(ToString::to_string) - ); - assert_eq!(engine.session_id, opts.session_id); - assert_eq!(engine.min_score, opts.min_score); - assert_eq!(engine.cross_session, opts.cross_session); -} - -#[test] -fn empty_recall_filters_stay_empty() { - let engine = recall_opts_to_tinycortex(&tm::OwnedRecallOpts::default()); - assert!(engine.namespace.is_none()); - assert!(engine.category.is_none()); - assert!(engine.session_id.is_none()); - assert!(engine.min_score.is_none()); - assert!(!engine.cross_session); -} diff --git a/adapters/tinycortex/src/lib.rs b/adapters/tinycortex/src/lib.rs index ea9ead8..ec838a7 100644 --- a/adapters/tinycortex/src/lib.rs +++ b/adapters/tinycortex/src/lib.rs @@ -1,15 +1,18 @@ //! TinyCortex as a TinyMemory driver. //! //! This crate is the seam between the TinyCortex engine and the TinyMemory -//! contract. The two describe the same values but are distinct crates, so -//! something has to convert — and it is much better for that to be one small -//! audited crate than a conversion scattered across every call site in a host. +//! contract. //! -//! ## What is here +//! It used to carry a conversion layer as well. The two crates described the +//! same values under two names, so every call across the seam translated, and a +//! field added to one contract had to be added to the other and to the +//! conversion — three places, or the value was silently dropped. Since +//! issue #18 §A1 `tinycortex-api` re-exports `tinymemory-api` rather than +//! redefining it, so both sides name one type and `convert` is gone. What +//! remains is the trait shape: the engine's storage trait and the contract's +//! are still separate traits over the same values. //! -//! - [`convert`] — total, exhaustively-destructuring value conversions in both -//! directions. A field added to either contract becomes a compile error here -//! instead of a silently dropped value. +//! ## What is here //! - [`TinycortexMemory`] — wraps any TinyCortex [`tinycortex::memory::Memory`] //! backend as a TinyMemory //! [`Memory`](tinymemory_api::traits::Memory). @@ -45,7 +48,6 @@ //! [`engine::advertised_capabilities`] and not just the accessor — a build //! without the git-backed snapshot store must not claim a diff ledger. -pub mod convert; pub mod engine; mod memory; diff --git a/adapters/tinycortex/src/memory.rs b/adapters/tinycortex/src/memory.rs index 02f7533..3235973 100644 --- a/adapters/tinycortex/src/memory.rs +++ b/adapters/tinycortex/src/memory.rs @@ -1,9 +1,12 @@ //! [`TinycortexMemory`] — a TinyCortex storage backend, seen through the //! TinyMemory contract's [`Memory`] trait. //! -//! Every method is a delegation plus a conversion. The two that are not purely -//! mechanical are called out below; both are cases where getting the -//! delegation "obviously right" would be wrong. +//! Every method is a plain delegation. It used to be a delegation *plus a +//! conversion*, because the engine's contract crate defined its own copies of +//! the memory value types; since tinymemory#18 §A1 `tinycortex-api` re-exports +//! this contract instead, so the two sides name one type and there is nothing +//! left to convert. The one method that is still not purely mechanical is +//! called out below. use std::sync::Arc; @@ -14,11 +17,6 @@ use tinymemory_api::types::{ MemoryCategory, MemoryEntry, MemoryTaint, NamespaceSummary, RecallOpts, }; -use crate::convert::{ - category_to_tinycortex, entry_to_tinymemory, namespace_summary_to_tinymemory, - recall_opts_to_tinycortex, taint_to_tinycortex, -}; - /// A TinyCortex backend exposed as a TinyMemory [`Memory`]. pub struct TinycortexMemory { inner: Arc, @@ -63,13 +61,7 @@ impl Memory for TinycortexMemory { session_id: Option<&str>, ) -> anyhow::Result<()> { self.inner - .store( - namespace, - key, - content, - category_to_tinycortex(category), - session_id, - ) + .store(namespace, key, content, category, session_id) .await } @@ -88,20 +80,13 @@ impl Memory for TinycortexMemory { taint: MemoryTaint, ) -> anyhow::Result<()> { self.inner - .store_with_taint( - namespace, - key, - content, - category_to_tinycortex(category), - session_id, - taint_to_tinycortex(taint), - ) + .store_with_taint(namespace, key, content, category, session_id, taint) .await } - /// The engine's `RecallOpts` borrows its string fields, so the owned - /// conversion has to outlive the borrow taken from it — hence the local - /// binding rather than a temporary in the call. + /// The engine's `RecallOpts` borrows its string fields, so the owned form + /// has to outlive the borrow taken from it — hence the local binding rather + /// than a temporary in the call. async fn recall( &self, query: &str, @@ -109,12 +94,7 @@ impl Memory for TinycortexMemory { opts: RecallOpts<'_>, ) -> anyhow::Result> { let owned = OwnedRecallOpts::from(opts); - let engine_owned = recall_opts_to_tinycortex(&owned); - let hits = self - .inner - .recall(query, limit, (&engine_owned).into()) - .await?; - Ok(hits.into_iter().map(entry_to_tinymemory).collect()) + Ok(self.inner.recall(query, limit, (&owned).into()).await?) } async fn recall_relevant_by_vector( @@ -130,11 +110,7 @@ impl Memory for TinycortexMemory { } async fn get(&self, namespace: &str, key: &str) -> anyhow::Result> { - Ok(self - .inner - .get(namespace, key) - .await? - .map(entry_to_tinymemory)) + self.inner.get(namespace, key).await } async fn list( @@ -143,14 +119,7 @@ impl Memory for TinycortexMemory { category: Option<&MemoryCategory>, session_id: Option<&str>, ) -> anyhow::Result> { - // `category` is borrowed on both sides, so the converted value needs a - // binding to borrow from. - let engine_category = category.cloned().map(category_to_tinycortex); - let entries = self - .inner - .list(namespace, engine_category.as_ref(), session_id) - .await?; - Ok(entries.into_iter().map(entry_to_tinymemory).collect()) + self.inner.list(namespace, category, session_id).await } async fn forget(&self, namespace: &str, key: &str) -> anyhow::Result { @@ -158,13 +127,7 @@ impl Memory for TinycortexMemory { } async fn namespace_summaries(&self) -> anyhow::Result> { - Ok(self - .inner - .namespace_summaries() - .await? - .into_iter() - .map(namespace_summary_to_tinymemory) - .collect()) + self.inner.namespace_summaries().await } async fn count(&self) -> anyhow::Result { diff --git a/vendor/tinycortex b/vendor/tinycortex index 5fdeac9..34cbb6c 160000 --- a/vendor/tinycortex +++ b/vendor/tinycortex @@ -1 +1 @@ -Subproject commit 5fdeac984c09d2dac65b61e92fd27e2c92ce1e6b +Subproject commit 34cbb6cfa91ea74d62605bd57790782b0c748556 From 2721f4c042211503d62be1e11e24ea684533b060 Mon Sep 17 00:00:00 2001 From: Shanu Date: Mon, 17 Aug 2026 19:28:32 +0530 Subject: [PATCH 3/9] Name the contract's Memory trait instead of reaching through the engine MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Issue #18 §A2. `tinymemory_core::traits` re-exported `Memory` and the memory value types from `tinycortex::memory` — from the *engine*. That is §1.1's finding: `tinymemory_core::MemoryEntry` was the engine's type rather than the contract's, so a second engine could not be bound without translating, and the crate was engine-neutral in name only. It now names `tinymemory-api` directly. Since §A1 the engine re-exports that same contract, so both spellings resolve to one type either way — but reaching an engine-neutral contract *through an engine* is precisely what would have to be undone before a second engine could be bound, and undoing it later is harder than not doing it. §A2's second clause comes along with the first: `UnifiedMemory` implements `crate::traits::Memory`, which is now the contract's trait, so it implements the TinyMemory trait directly with no further edit. This was not possible before §A1. The same repoint attempted then produced 10 compile errors, 9 of which wanted taint conversions added — the three-place edit §A1 exists to delete. With one type set behind both names it is a 20-line documentation change and a re-export, and the workspace compiles unchanged. Refs #18 (§A2) --- core/src/traits.rs | 35 ++++++++++++++++++++--------------- 1 file changed, 20 insertions(+), 15 deletions(-) diff --git a/core/src/traits.rs b/core/src/traits.rs index 022bfb0..af12346 100644 --- a/core/src/traits.rs +++ b/core/src/traits.rs @@ -1,11 +1,16 @@ //! Core traits and data structures for the OpenHuman memory system. //! //! This module defines the foundational `Memory` trait that all storage backends -//! must implement. The standard memory value types (`MemoryEntry`, +//! must implement. The trait and the standard memory value types (`MemoryEntry`, //! `MemoryCategory`, `MemoryTaint`, `RecallOpts`, `NamespaceSummary`) are -//! **re-exported from the `tinycortex` crate** (migration W2, spec §0.5): the -//! crate is the single source of truth for these wire-compatible types, and the -//! 30+ host consumers keep their `memory::traits::…` import paths unchanged. +//! **re-exported from `tinymemory-api`**, the engine-neutral contract. +//! +//! They used to come from the `tinycortex` crate — from the *engine*, in other +//! words, which meant `tinymemory_core::MemoryEntry` was the engine's type and +//! not the contract's, and a second engine could not have been bound without +//! translating (issue #18 §A1/§A2). Naming the contract directly is what makes +//! this crate engine-neutral at the type level; the 30+ host consumers keep +//! their `memory::traits::…` import paths unchanged either way. //! //! `MemoryTaint` is security-critical provenance — it fails closed to //! `ExternalSync` for unknown/corrupt values so the subconscious gate refuses @@ -13,19 +18,19 @@ //! byte-identical to the former host definition before re-exporting; the tests //! below are the host-side seam that pins that contract on the crate type. //! -//! The `Memory` trait is also re-exported from `tinycortex`; backend-specific -//! resources such as SQLite connections are carried explicitly by factories -//! instead of being exposed through the storage abstraction. +//! Backend-specific resources such as SQLite connections are carried explicitly +//! by factories instead of being exposed through the storage abstraction. -// ── Value types: re-exported from the crate (W2 type-unification, spec §0.5) ── +// ── The contract's trait and value types ───────────────────────────────────── // -// These were formerly defined here. They are now the crate's types verbatim -// (identical fields, derives, serde attrs, and — for `MemoryTaint` — the same -// fail-closed `from_db_str`). Re-exporting keeps one source of truth while every -// `use crate::traits::{MemoryEntry, …}` site compiles unchanged. -pub use tinycortex::memory::{ - Memory, MemoryCategory, MemoryEntry, MemoryTaint, NamespaceSummary, RecallOpts, -}; +// Named directly rather than reached through `tinycortex::memory`. Since §A1 the +// engine re-exports this same contract, so the two spellings resolve to one type +// either way — but going through the engine to reach an engine-neutral contract +// is what §1.1 of issue #18 calls out, and it is what would have to be undone +// before a second engine could be bound. +pub use tinymemory_api::recall::RecallOpts; +pub use tinymemory_api::traits::Memory; +pub use tinymemory_api::types::{MemoryCategory, MemoryEntry, MemoryTaint, NamespaceSummary}; #[cfg(test)] mod tests { From ac2b1e60f5cd792203af906442406186c2d3d80b Mon Sep 17 00:00:00 2001 From: Shanu Date: Mon, 17 Aug 2026 19:49:27 +0530 Subject: [PATCH 4/9] Contain the engine behind one module MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Issue #18 §C1. Eighty-three files named the `tinycortex` crate, in two hundred and ninety-six places. That made the engine's shape an ambient fact of the whole crate rather than a dependency anyone had chosen, and made "what would a second engine have to provide?" a question nobody could answer without reading all of them. Every one now goes through `core/src/engine/backend.rs`, which re-exports the engine's memory surface and is the only file outside the seam that names it. Outside `core/src/engine/`, references to the crate are zero. Re-exporting rather than wrapping is deliberate. A wrapper over three hundred call sites would be a second surface to keep in step with the first, which is the failure §A1 had just finished deleting. What this buys is not insulation — the call sites use the engine's API verbatim — but a single enumerable place that names it. The seam is `engine`, not `tinycortex`. §C1's text says `core/src/tinycortex/`, but its own acceptance criterion asks that `grep -rl tinycortex core/src` match only files under that module, and those two cannot both hold: with the module named after the engine, every call site reads `crate::tinycortex::…` and matches. Naming the seam for its role rather than for one engine satisfies the criterion and is the better name regardless — a seam named after the thing it is meant to make replaceable is the coupling this section removes. The module was `pub` but had no consumer outside this crate, so the rename breaks nothing. The workspace root and the module crate each gain a `[patch]` for the contract's git dependency. Patch tables apply only from the root being built, and the module crate is its own root; without its own entry cargo resolves the git copy alongside the path copy and `MemoryTaint` from one is not the same type as from the other. Found by the compiler, not predicted. What is left matching `tinycortex` outside the seam is 82 log-message strings and a `tinycortex_kv` accessor name. Neither is a crate reference, and renaming the accessor is cosmetic churn §C1 does not ask for. Two stale doc comments describing a "thirteen-family `tinycortex_api` contract" are corrected — since §A1 the contract is `tinymemory-api`, and it has eighteen families. Acceptance, measured: 83 files naming the engine crate outside the seam, now 0. Refs #18 (§C1) --- core/src/conversations/blocking.rs | 6 +- core/src/conversations/mod.rs | 2 +- core/src/diff/mod.rs | 6 +- core/src/diff/ops.rs | 10 +-- core/src/diff/source.rs | 4 +- core/src/diff/stub.rs | 2 +- core/src/engine/backend.rs | 59 ++++++++++++++++ core/src/{tinycortex => engine}/chat.rs | 0 core/src/{tinycortex => engine}/config.rs | 0 core/src/{tinycortex => engine}/embeddings.rs | 0 core/src/{tinycortex => engine}/ingest.rs | 0 core/src/{tinycortex => engine}/mod.rs | 2 + core/src/{tinycortex => engine}/parity.rs | 0 core/src/{tinycortex => engine}/persona.rs | 0 .../{tinycortex => engine}/queue_driver.rs | 0 core/src/{tinycortex => engine}/seal.rs | 0 core/src/{tinycortex => engine}/summariser.rs | 0 core/src/{tinycortex => engine}/sync.rs | 0 core/src/ingest_pipeline.rs | 24 +++---- core/src/ingestion/mod.rs | 10 +-- core/src/lib.rs | 2 +- core/src/people/mod.rs | 8 ++- core/src/queue/ops.rs | 11 +-- core/src/queue/scheduler.rs | 8 +-- core/src/queue/store.rs | 36 +++++----- core/src/queue/types.rs | 2 +- core/src/queue/worker.rs | 10 +-- core/src/sources/readers/conversation.rs | 12 ++-- core/src/sources/readers/folder.rs | 12 ++-- core/src/sources/readers/github.rs | 16 +++-- core/src/sources/readers/rss.rs | 12 ++-- core/src/sources/readers/web_page.rs | 12 ++-- core/src/sources/registry.rs | 8 +-- core/src/sources/sync.rs | 14 ++-- core/src/sources/types.rs | 2 +- core/src/store/chunks/connection.rs | 6 +- core/src/store/chunks/embeddings.rs | 32 +++++---- core/src/store/chunks/mod.rs | 2 +- core/src/store/chunks/raw_refs.rs | 22 +++--- core/src/store/chunks/semantic.rs | 4 +- core/src/store/chunks/store.rs | 69 +++++++++++-------- core/src/store/chunks/types.rs | 4 +- core/src/store/client.rs | 3 +- core/src/store/content/mod.rs | 6 +- core/src/store/content/read.rs | 8 +-- core/src/store/content/tags.rs | 2 +- core/src/store/entities.rs | 18 ++--- core/src/store/kv.rs | 6 +- core/src/store/namespace_store/segments.rs | 2 +- core/src/store/profile_store.rs | 3 +- core/src/store/safety/mod.rs | 4 +- core/src/store/safety/pii.rs | 4 +- core/src/store/trees/hotness.rs | 12 ++-- core/src/store/trees/registry.rs | 6 +- core/src/store/trees/store.rs | 56 ++++++++------- core/src/store/trees/store_tests.rs | 2 +- core/src/store/trees/types.rs | 2 +- core/src/store/types.rs | 4 +- core/src/sync/composio/mod.rs | 8 +-- core/src/sync/composio/periodic.rs | 4 +- .../sync/composio/providers/clickup/mod.rs | 5 +- .../src/sync/composio/providers/github/mod.rs | 5 +- core/src/sync/composio/providers/gmail/mod.rs | 2 +- .../sync/composio/providers/gmail/provider.rs | 11 ++- .../sync/composio/providers/gmail/tests.rs | 2 +- core/src/sync/composio/providers/helpers.rs | 4 +- .../src/sync/composio/providers/linear/mod.rs | 2 +- core/src/sync/composio/providers/mod.rs | 2 +- .../src/sync/composio/providers/notion/mod.rs | 2 +- .../composio/providers/notion/provider.rs | 2 +- core/src/sync/composio/providers/slack/mod.rs | 2 +- .../sync/composio/providers/slack/provider.rs | 20 ++---- .../src/sync/composio/providers/sync_state.rs | 6 +- core/src/sync/composio/providers/traits.rs | 2 +- core/src/sync/sync_status/mod.rs | 2 +- core/src/sync/workspace/periodic.rs | 2 +- core/src/sync/workspace/watcher.rs | 2 +- core/src/tool_memory/mod.rs | 8 +-- core/src/tool_memory/store.rs | 2 +- core/src/traits.rs | 2 +- core/src/tree/graph/bfs.rs | 6 +- core/src/tree/graph/store.rs | 14 ++-- core/src/tree/health/mod.rs | 4 +- core/src/tree/ingest.rs | 6 +- core/src/tree/mod.rs | 2 +- core/src/tree/retrieval/benchmarks.rs | 2 +- core/src/tree/retrieval/cover.rs | 4 +- core/src/tree/retrieval/drill_down.rs | 4 +- core/src/tree/retrieval/engine.rs | 2 +- core/src/tree/retrieval/fast.rs | 6 +- core/src/tree/retrieval/fetch.rs | 6 +- core/src/tree/retrieval/integration_tests.rs | 2 +- core/src/tree/retrieval/search.rs | 4 +- core/src/tree/retrieval/source.rs | 8 +-- core/src/tree/retrieval/source_scope_tests.rs | 2 +- core/src/tree/retrieval/types.rs | 2 +- core/src/tree/score/extract/mod.rs | 12 ++-- core/src/tree/score/mod.rs | 14 ++-- core/src/tree/score/store.rs | 26 +++---- core/src/tree/summarise.rs | 14 ++-- core/src/tree/tree/bucket_seal.rs | 16 ++--- core/src/tree/tree/factory.rs | 12 ++-- core/src/tree/tree/flush.rs | 2 +- core/src/tree/tree_runtime/engine.rs | 17 ++--- core/src/tree/tree_runtime/mod.rs | 2 +- core/src/tree/tree_runtime/store.rs | 44 ++++++------ crates/tinymemory-module/Cargo.lock | 1 + crates/tinymemory-module/Cargo.toml | 8 +++ 108 files changed, 500 insertions(+), 406 deletions(-) create mode 100644 core/src/engine/backend.rs rename core/src/{tinycortex => engine}/chat.rs (100%) rename core/src/{tinycortex => engine}/config.rs (100%) rename core/src/{tinycortex => engine}/embeddings.rs (100%) rename core/src/{tinycortex => engine}/ingest.rs (100%) rename core/src/{tinycortex => engine}/mod.rs (99%) rename core/src/{tinycortex => engine}/parity.rs (100%) rename core/src/{tinycortex => engine}/persona.rs (100%) rename core/src/{tinycortex => engine}/queue_driver.rs (100%) rename core/src/{tinycortex => engine}/seal.rs (100%) rename core/src/{tinycortex => engine}/summariser.rs (100%) rename core/src/{tinycortex => engine}/sync.rs (100%) diff --git a/core/src/conversations/blocking.rs b/core/src/conversations/blocking.rs index eda10aa..3e18309 100644 --- a/core/src/conversations/blocking.rs +++ b/core/src/conversations/blocking.rs @@ -1,7 +1,7 @@ //! Async wrappers that run the conversation store's **blocking** operations on //! tokio's blocking pool (#5156). //! -//! Every `tinycortex::memory::conversations` entry point is synchronous, and +//! Every `crate::engine::backend::conversations` entry point is synchronous, and //! each one takes the process-global `CONVERSATION_STORE_LOCK` — a //! `parking_lot::Mutex` — and then does fsync'd JSONL file IO while holding it. //! Calling one directly from an `async fn` therefore parks a tokio **worker** @@ -35,9 +35,9 @@ use std::path::PathBuf; -use tinycortex::memory::conversations as store; +use crate::engine::backend::conversations as store; -use tinycortex::memory::conversations::{ +use crate::engine::backend::conversations::{ ConversationMessage, ConversationMessagePatch, ConversationPurgeStats, ConversationStore, ConversationThread, CreateConversationThread, CrossThreadHit, }; diff --git a/core/src/conversations/mod.rs b/core/src/conversations/mod.rs index a6b5491..663dfb7 100644 --- a/core/src/conversations/mod.rs +++ b/core/src/conversations/mod.rs @@ -4,7 +4,7 @@ //! append-only in `threads.jsonl`; each thread's messages in a dedicated JSONL //! file). The store / inverted-index / tokenizer / types engine is the crate's //! (a byte-identical port, incl. the D1 rank-before-materialize fix), and -//! consumers name `tinycortex::memory::conversations` directly — this module no +//! consumers name `crate::engine::backend::conversations` directly — this module no //! longer re-exports that surface under a second path. //! //! Host-retained: diff --git a/core/src/diff/mod.rs b/core/src/diff/mod.rs index 31b3f13..e7adb6e 100644 --- a/core/src/diff/mod.rs +++ b/core/src/diff/mod.rs @@ -14,12 +14,12 @@ //! the ledger is a derived view used purely for change tracking. //! //! W7: the snapshot/diff/checkpoint/ledger engine is now -//! `tinycortex::memory::diff::DiffEngine` (a byte-identical port over the same +//! `crate::engine::backend::diff::DiffEngine` (a byte-identical port over the same //! `/memory_diff/repo` git layout). This module is a thin host shim: //! [`ops`] async-wraps the engine, [`source`] supplies the chunk-store item //! seam (`DiffEngine`'s `SnapshotItemSource`), and `rpc`/`schemas`/`tools` //! keep the RPC + agent surface. The wire types are the crate's, named directly -//! (`tinycortex::memory::diff::types`) rather than through a host re-export +//! (`crate::engine::backend::diff::types`) rather than through a host re-export //! module. //! //! Features: @@ -65,7 +65,7 @@ pub mod source; // `memory::diff::{types, source}` are serde-only wire types and stay compiled; // only the git-touching `ledger`/`DiffEngine` half sits behind `git-diff`. A // stub copy would be a second definition of one serde shape, free to drift. -pub use tinycortex::memory::diff::types::{ +pub use crate::engine::backend::diff::types::{ ChangeKind, Checkpoint, CrossSourceDiff, DiffResult, DiffSummary, ItemChange, Snapshot, SnapshotTrigger, }; diff --git a/core/src/diff/ops.rs b/core/src/diff/ops.rs index 8c50e8b..d56b401 100644 --- a/core/src/diff/ops.rs +++ b/core/src/diff/ops.rs @@ -1,5 +1,5 @@ //! Business logic for memory diff — thin host async wrappers over -//! `tinycortex::memory::diff::DiffEngine` (W7). +//! `crate::engine::backend::diff::DiffEngine` (W7). //! //! The snapshot/diff/checkpoint/ledger engine is the crate's; the git ledger it //! writes lives at the same `/memory_diff/repo` path with the same @@ -16,10 +16,10 @@ use tinymemory_api::host::test_support::TestHostConfig; use crate::sources::types::MemorySourceEntry; use crate::Config; -use tinycortex::memory::diff::{DiffEngine, SourceDescriptor}; +use crate::engine::backend::diff::{DiffEngine, SourceDescriptor}; use super::source::ChunkStoreItemSource; -use tinycortex::memory::diff::types::*; +use crate::engine::backend::diff::types::*; /// A crate [`SourceDescriptor`] from a host source entry. fn descriptor(source: &MemorySourceEntry) -> SourceDescriptor { @@ -101,7 +101,7 @@ pub async fn list_snapshots( let source_id = source_id.map(str::to_string); tokio::task::spawn_blocking(move || -> anyhow::Result> { - let ledger = tinycortex::memory::diff::Ledger::open(&workspace_dir)?; + let ledger = crate::engine::backend::diff::Ledger::open(&workspace_dir)?; ledger.list_snapshots(source_id.as_deref(), limit) }) .await @@ -310,7 +310,7 @@ pub async fn cleanup(config: &Config, older_than_days: u32) -> Result TestHostConfig { crate::test_seams::init(); diff --git a/core/src/diff/source.rs b/core/src/diff/source.rs index 2baff36..149ef34 100644 --- a/core/src/diff/source.rs +++ b/core/src/diff/source.rs @@ -1,6 +1,6 @@ //! The host implementation of the crate diff engine's chunk-source seam. //! -//! `tinycortex::memory::diff::DiffEngine` is generic over a +//! `crate::engine::backend::diff::DiffEngine` is generic over a //! [`SnapshotItemSource`]: during //! `take_snapshot` (directly, and transitively from `create_checkpoint` for any //! source lacking a baseline) it asks the source for a source's already-ingested @@ -22,7 +22,7 @@ use std::collections::HashMap; use std::sync::Arc; -use tinycortex::memory::diff::{extract_item_id, SnapshotItem, SnapshotItemSource}; +use crate::engine::backend::diff::{extract_item_id, SnapshotItem, SnapshotItemSource}; #[cfg(test)] use tinymemory_api::host::test_support::TestHostConfig; diff --git a/core/src/diff/stub.rs b/core/src/diff/stub.rs index f5658c8..fb49b35 100644 --- a/core/src/diff/stub.rs +++ b/core/src/diff/stub.rs @@ -26,7 +26,7 @@ use crate::sources::types::MemorySourceEntry; use crate::Config; -use tinycortex::memory::diff::types::{Checkpoint, CrossSourceDiff, Snapshot}; +use crate::engine::backend::diff::types::{Checkpoint, CrossSourceDiff, Snapshot}; /// The message every disabled entry point returns. /// diff --git a/core/src/engine/backend.rs b/core/src/engine/backend.rs new file mode 100644 index 0000000..c5485d7 --- /dev/null +++ b/core/src/engine/backend.rs @@ -0,0 +1,59 @@ +//! The engine's own memory surface, reached through one door. +//! +//! Issue #18 §C1 asks that nothing outside this module name the `tinycortex` +//! crate. Eighty-three files did, in two hundred and ninety-six places, which +//! made the engine's shape an ambient fact of the whole crate rather than a +//! dependency anyone had chosen — and made "what would a second engine have to +//! provide?" a question no one could answer without reading all of them. +//! +//! Re-exporting rather than wrapping is deliberate. A wrapper layer over three +//! hundred call sites would be a second surface to keep in step with the first, +//! which is the failure §A1 had just finished deleting. What this buys is not +//! insulation from the engine's API — the call sites still use it verbatim — +//! but a single place that *names* it, so the coupling is enumerable: this file +//! is the list of everything `tinymemory-core` needs an engine to provide. +//! +//! # Why this is not `MemoryProvider` +//! +//! §A3 proposes routing these call sites through `&dyn MemoryProvider` instead. +//! That is not possible, and the reason is worth recording where the next +//! reader will find it. Core does not *consume* the engine's memory API; it +//! shares the engine's SQLite database. Thirty-four files here hold a +//! `rusqlite::Transaction` or a `Connection`, and the entry points they call +//! take them: +//! +//! ```text +//! upsert_buffer_tx(tx: &Transaction<'_>, buf: &Buffer) -> Result<()> +//! shared_connection(config: &MemoryConfig) -> Result>> +//! ``` +//! +//! Serving those through the contract would put `rusqlite` in +//! `tinymemory-api`, which its own manifest forbids and CI now enforces. The +//! honest description is that core and the engine co-implement one store, and +//! separating them is a decomposition rather than a routing change. +//! +//! Nested under a module rather than re-exported flat because the seam already +//! has its own `ingest` and `sync` modules, which are host-side pieces and +//! not the engine's. + +// The engine's submodules. +pub use tinycortex::memory::{ + archivist, chunks, conversations, diff, graph, health, ingest, people, queue, retrieval, score, + sources, store, sync, tool_memory, tree, types, +}; + +// …and the items it re-exports at its own top level, which call sites reach for +// by the same short paths. Listed rather than globbed so this file stays the +// enumerable answer to "what does core need an engine to provide". +pub use tinycortex::memory::{ + GraphRelationRecord, InMemoryMemoryStore, MemoryCategory, MemoryConfig, MemoryEngineError, + MemoryEngineResult, MemoryEntry, MemoryId, MemoryInput, MemoryItemKind, MemoryKvRecord, + MemoryQuery, MemoryRecord, MemoryResult, MemoryStore, MemoryTaint, NamespaceDocumentInput, + NamespaceMemoryHit, NamespaceQueryResult, NamespaceRetrievalContext, NamespaceSummary, + RecallOpts, RetrievalScoreBreakdown, SearchHit, StoreError, StoredMemoryDocument, + WeightProfile, GLOBAL_NAMESPACE, +}; + +// The storage trait itself. Since §A2 this is the contract's trait, not the +// engine's — the engine re-exports the same one. +pub use tinycortex::memory::Memory; diff --git a/core/src/tinycortex/chat.rs b/core/src/engine/chat.rs similarity index 100% rename from core/src/tinycortex/chat.rs rename to core/src/engine/chat.rs diff --git a/core/src/tinycortex/config.rs b/core/src/engine/config.rs similarity index 100% rename from core/src/tinycortex/config.rs rename to core/src/engine/config.rs diff --git a/core/src/tinycortex/embeddings.rs b/core/src/engine/embeddings.rs similarity index 100% rename from core/src/tinycortex/embeddings.rs rename to core/src/engine/embeddings.rs diff --git a/core/src/tinycortex/ingest.rs b/core/src/engine/ingest.rs similarity index 100% rename from core/src/tinycortex/ingest.rs rename to core/src/engine/ingest.rs diff --git a/core/src/tinycortex/mod.rs b/core/src/engine/mod.rs similarity index 99% rename from core/src/tinycortex/mod.rs rename to core/src/engine/mod.rs index 6c4f116..a6613e3 100644 --- a/core/src/tinycortex/mod.rs +++ b/core/src/engine/mod.rs @@ -46,6 +46,8 @@ mod seal; mod summariser; mod sync; +pub mod backend; + pub use chat::{build_chat_provider, SeamChatProvider}; pub use config::{engine_config, memory_config_from}; pub use embeddings::SeamEmbedder; diff --git a/core/src/tinycortex/parity.rs b/core/src/engine/parity.rs similarity index 100% rename from core/src/tinycortex/parity.rs rename to core/src/engine/parity.rs diff --git a/core/src/tinycortex/persona.rs b/core/src/engine/persona.rs similarity index 100% rename from core/src/tinycortex/persona.rs rename to core/src/engine/persona.rs diff --git a/core/src/tinycortex/queue_driver.rs b/core/src/engine/queue_driver.rs similarity index 100% rename from core/src/tinycortex/queue_driver.rs rename to core/src/engine/queue_driver.rs diff --git a/core/src/tinycortex/seal.rs b/core/src/engine/seal.rs similarity index 100% rename from core/src/tinycortex/seal.rs rename to core/src/engine/seal.rs diff --git a/core/src/tinycortex/summariser.rs b/core/src/engine/summariser.rs similarity index 100% rename from core/src/tinycortex/summariser.rs rename to core/src/engine/summariser.rs diff --git a/core/src/tinycortex/sync.rs b/core/src/engine/sync.rs similarity index 100% rename from core/src/tinycortex/sync.rs rename to core/src/engine/sync.rs diff --git a/core/src/ingest_pipeline.rs b/core/src/ingest_pipeline.rs index f3fcd2e..3d87d5c 100644 --- a/core/src/ingest_pipeline.rs +++ b/core/src/ingest_pipeline.rs @@ -2,16 +2,16 @@ use anyhow::Result; -use crate::store::chunks::store::RawRef; -use crate::Config; -use tinycortex::memory::ingest::canonicalize::{ +use crate::engine::backend::ingest::canonicalize::{ chat::{self, ChatBatch}, document::{self, DocumentInput}, email::{self, EmailThread}, CanonicalisedSource, }; +use crate::store::chunks::store::RawRef; +use crate::Config; -pub use tinycortex::memory::ingest::IngestSummary as IngestResult; +pub use crate::engine::backend::ingest::IngestSummary as IngestResult; pub async fn ingest_chat( config: &Config, @@ -22,8 +22,8 @@ pub async fn ingest_chat( ) -> Result { let canonical = chat::canonicalise(source_id, owner, &tags, batch.clone()).map_err(anyhow::Error::msg)?; - let (memory, sink, scoring) = crate::tinycortex::ingest_context(config); - let result = tinycortex::memory::ingest::ingest_chat( + let (memory, sink, scoring) = crate::engine::ingest_context(config); + let result = crate::engine::backend::ingest::ingest_chat( &memory, source_id, owner, tags, batch, &sink, &scoring, ) .await?; @@ -40,8 +40,8 @@ pub async fn ingest_email( ) -> Result { let canonical = email::canonicalise(source_id, owner, &tags, thread.clone()).map_err(anyhow::Error::msg)?; - let (memory, sink, scoring) = crate::tinycortex::ingest_context(config); - let result = tinycortex::memory::ingest::ingest_email( + let (memory, sink, scoring) = crate::engine::ingest_context(config); + let result = crate::engine::backend::ingest::ingest_email( &memory, source_id, owner, tags, thread, &sink, &scoring, ) .await?; @@ -59,8 +59,8 @@ pub async fn ingest_email_with_raw_refs( ) -> Result { let canonical = email::canonicalise(source_id, owner, &tags, thread.clone()).map_err(anyhow::Error::msg)?; - let (memory, sink, scoring) = crate::tinycortex::ingest_context(config); - let result = tinycortex::memory::ingest::ingest_email_with_raw_refs( + let (memory, sink, scoring) = crate::engine::ingest_context(config); + let result = crate::engine::backend::ingest::ingest_email_with_raw_refs( &memory, source_id, owner, tags, thread, raw_refs, &sink, &scoring, ) .await?; @@ -101,8 +101,8 @@ pub async fn ingest_document_versioned( let canonical = document::canonicalise(source_id, owner, &tags, doc.clone(), path_scope.clone()) .map_err(anyhow::Error::msg)?; - let (memory, sink, scoring) = crate::tinycortex::ingest_context(config); - let result = tinycortex::memory::ingest::ingest_document_versioned( + let (memory, sink, scoring) = crate::engine::ingest_context(config); + let result = crate::engine::backend::ingest::ingest_document_versioned( &memory, source_id, owner, tags, doc, path_scope, version_ms, &sink, &scoring, ) .await?; diff --git a/core/src/ingestion/mod.rs b/core/src/ingestion/mod.rs index 9ee1fb8..16b9714 100644 --- a/core/src/ingestion/mod.rs +++ b/core/src/ingestion/mod.rs @@ -14,12 +14,12 @@ pub mod queue; pub mod state; -pub use queue::{IngestionJob, IngestionQueue, DEFAULT_QUEUE_CAPACITY}; -pub use state::{IngestionState, IngestionStatusSnapshot}; -pub use tinycortex::memory::ingest::{ +pub use crate::engine::backend::ingest::{ ExtractedEntity, ExtractedRelation, ExtractionMode, MemoryIngestionConfig, MemoryIngestionRequest, MemoryIngestionResult, DEFAULT_MEMORY_EXTRACTION_MODEL, }; +pub use queue::{IngestionJob, IngestionQueue, DEFAULT_QUEUE_CAPACITY}; +pub use state::{IngestionState, IngestionStatusSnapshot}; use serde_json::json; @@ -35,7 +35,7 @@ impl UnifiedMemory { request: MemoryIngestionRequest, ) -> Result { let (enriched_input, mut extraction) = - tinycortex::memory::ingest::extract_enriched_document( + crate::engine::backend::ingest::extract_enriched_document( &request.document, &request.config, ); @@ -62,7 +62,7 @@ impl UnifiedMemory { config: &MemoryIngestionConfig, ) -> Result { let (_enriched, mut extraction) = - tinycortex::memory::ingest::extract_enriched_document(document, config); + crate::engine::backend::ingest::extract_enriched_document(document, config); let namespace = Self::sanitize_namespace(&document.namespace); self.upsert_graph_relations(&namespace, document_id, &extraction, config) diff --git a/core/src/lib.rs b/core/src/lib.rs index 1935dcd..4455dd7 100644 --- a/core/src/lib.rs +++ b/core/src/lib.rs @@ -39,6 +39,7 @@ pub mod conversations; pub mod diff; pub mod embedding_adapter; pub mod embedding_host; +pub mod engine; pub mod events; pub mod global; pub mod ingest_pipeline; @@ -63,7 +64,6 @@ pub mod test_env_lock; #[cfg(test)] pub(crate) mod test_seams; pub mod thread_context; -pub mod tinycortex; pub mod tool_memory; pub mod traits; pub mod tree; diff --git a/core/src/people/mod.rs b/core/src/people/mod.rs index feccd58..e3507e0 100644 --- a/core/src/people/mod.rs +++ b/core/src/people/mod.rs @@ -2,7 +2,7 @@ //! //! # Why this is a shim //! -//! The implementation moved down into [`tinycortex::memory::people`]. People is +//! The implementation moved down into [`crate::engine::backend::people`]. People is //! *storage*: a SQLite database of people, handle aliases and interactions, //! with its own migrations and its own workspace-keyed connection. Storage //! belongs to the engine, which is what lets the memory contract stay @@ -14,7 +14,7 @@ //! references to `people::types`, did not all have to move in the same change. //! //! This mirrors [`crate::store::chunks`], which has related the same way to -//! `tinycortex::memory::chunks` since the engine seam was drawn. +//! `crate::engine::backend::chunks` since the engine seam was drawn. //! //! # The address book rides two gates //! @@ -24,4 +24,6 @@ //! stub returns an empty contact list, so a refresh seeds nothing rather than //! failing. -pub use tinycortex::memory::people::{address_book, migrations, resolver, scorer, store, types}; +pub use crate::engine::backend::people::{ + address_book, migrations, resolver, scorer, store, types, +}; diff --git a/core/src/queue/ops.rs b/core/src/queue/ops.rs index 56468b1..4276d4f 100644 --- a/core/src/queue/ops.rs +++ b/core/src/queue/ops.rs @@ -7,14 +7,14 @@ /// Mark whether a re-embed backfill currently has pending work. pub fn set_backfill_in_progress(v: bool) { - tinycortex::memory::queue::set_backfill_in_progress(v); + crate::engine::backend::queue::set_backfill_in_progress(v); } /// True while a re-embed backfill chain still has rows to process. The /// #1365 absence-reasoning consumer checks this before treating an empty /// semantic-recall result as "no memory exists". pub fn backfill_in_progress() -> bool { - tinycortex::memory::queue::backfill_in_progress() + crate::engine::backend::queue::backfill_in_progress() } /// #1574 §4: ensure a re-embed backfill chain exists for the **current** @@ -30,9 +30,10 @@ pub fn backfill_in_progress() -> bool { /// covered space enqueues nothing. Errors are logged, never propagated — /// a failed enqueue must not fail the user's settings save. pub fn ensure_reembed_backfill(config: &crate::Config) { - let memory = crate::tinycortex::memory_config_from(config, config.workspace_dir().clone()); - let delegates = crate::tinycortex::HostQueueDelegates::new(config.to_arc()); - if let Err(error) = tinycortex::memory::queue::ensure_reembed_backfill(&memory, &delegates) { + let memory = crate::engine::memory_config_from(config, config.workspace_dir().clone()); + let delegates = crate::engine::HostQueueDelegates::new(config.to_arc()); + if let Err(error) = crate::engine::backend::queue::ensure_reembed_backfill(&memory, &delegates) + { log::warn!("[memory::jobs] ensure_reembed_backfill failed: {error:#}"); } } diff --git a/core/src/queue/scheduler.rs b/core/src/queue/scheduler.rs index aa01e88..54435c9 100644 --- a/core/src/queue/scheduler.rs +++ b/core/src/queue/scheduler.rs @@ -43,8 +43,8 @@ pub fn start(config: Arc) { /// Unrecoverable failures stay parked — see /// [`store::requeue_transient_failed`]. fn retry_transient_failures(config: &Config) { - let memory = crate::tinycortex::memory_config_from(config, config.workspace_dir().clone()); - match tinycortex::memory::queue::scheduler::self_heal(&memory) { + let memory = crate::engine::memory_config_from(config, config.workspace_dir().clone()); + match crate::engine::backend::queue::scheduler::self_heal(&memory) { Ok(0) => {} Ok(n) => { log::info!("[memory::jobs] periodic retry requeued {n} transient-failed job(s)"); @@ -72,8 +72,8 @@ fn retry_transient_failures(config: &Config) { /// `LabelStrategy` for every tree, which no production caller uses and which /// would apply one tree kind's labelling to all of them. pub fn enqueue_flush_stale_job(config: &Config) -> Result { - let memory = crate::tinycortex::memory_config_from(config, config.workspace_dir().clone()); - match tinycortex::memory::queue::scheduler::enqueue_flush_stale(&memory) { + let memory = crate::engine::memory_config_from(config, config.workspace_dir().clone()); + match crate::engine::backend::queue::scheduler::enqueue_flush_stale(&memory) { Ok(Some(_)) => { super::worker::wake_workers(); Ok(true) diff --git a/core/src/queue/store.rs b/core/src/queue/store.rs index 2ad8d55..fd3c223 100644 --- a/core/src/queue/store.rs +++ b/core/src/queue/store.rs @@ -7,28 +7,28 @@ use crate::tree::health::PipelineFailure; use crate::Config; use super::types::{Job, JobFailure, JobStatus, NewJob}; -use crate::tinycortex::engine_config; +use crate::engine::engine_config; -pub use tinycortex::memory::queue::DEFAULT_LOCK_DURATION_MS; +pub use crate::engine::backend::queue::DEFAULT_LOCK_DURATION_MS; pub fn enqueue(config: &Config, job: &NewJob) -> Result> { - tinycortex::memory::queue::enqueue(&engine_config(config), job) + crate::engine::backend::queue::enqueue(&engine_config(config), job) } pub fn enqueue_tx(tx: &Transaction<'_>, job: &NewJob) -> Result> { - tinycortex::memory::queue::enqueue_tx(tx, job) + crate::engine::backend::queue::enqueue_tx(tx, job) } pub fn claim_next(config: &Config, lock_duration_ms: i64) -> Result> { - tinycortex::memory::queue::claim_next(&engine_config(config), lock_duration_ms) + crate::engine::backend::queue::claim_next(&engine_config(config), lock_duration_ms) } pub fn mark_done(config: &Config, job: &Job) -> Result<()> { - tinycortex::memory::queue::mark_done(&engine_config(config), job) + crate::engine::backend::queue::mark_done(&engine_config(config), job) } pub fn mark_failed(config: &Config, job: &Job, error: &str) -> Result<()> { - tinycortex::memory::queue::mark_failed(&engine_config(config), job, error) + crate::engine::backend::queue::mark_failed(&engine_config(config), job, error) } pub fn mark_failed_typed( @@ -41,7 +41,7 @@ pub fn mark_failed_typed( code: failure.code.as_str(), class: failure.class.as_str(), }); - tinycortex::memory::queue::mark_failed_typed( + crate::engine::backend::queue::mark_failed_typed( &engine_config(config), job, error, @@ -50,41 +50,41 @@ pub fn mark_failed_typed( } pub fn mark_deferred(config: &Config, job: &Job, until_ms: i64, reason: &str) -> Result<()> { - tinycortex::memory::queue::mark_deferred(&engine_config(config), job, until_ms, reason) + crate::engine::backend::queue::mark_deferred(&engine_config(config), job, until_ms, reason) } pub fn recover_stale_locks(config: &Config) -> Result { - tinycortex::memory::queue::recover_stale_locks(&engine_config(config)) + crate::engine::backend::queue::recover_stale_locks(&engine_config(config)) } pub fn requeue_failed(config: &Config) -> Result { - tinycortex::memory::queue::requeue_failed(&engine_config(config)) + crate::engine::backend::queue::requeue_failed(&engine_config(config)) } pub fn requeue_transient_failed(config: &Config) -> Result { - tinycortex::memory::queue::requeue_transient_failed(&engine_config(config)) + crate::engine::backend::queue::requeue_transient_failed(&engine_config(config)) } pub fn release_running_locks(config: &Config) -> Result { - tinycortex::memory::queue::release_running_locks(&engine_config(config)) + crate::engine::backend::queue::release_running_locks(&engine_config(config)) } pub fn count_by_status(config: &Config, status: JobStatus) -> Result { - tinycortex::memory::queue::count_by_status(&engine_config(config), status) + crate::engine::backend::queue::count_by_status(&engine_config(config), status) } pub fn count_failed_unrecoverable(config: &Config) -> Result { - tinycortex::memory::queue::count_failed_unrecoverable(&engine_config(config)) + crate::engine::backend::queue::count_failed_unrecoverable(&engine_config(config)) } pub fn count_total(config: &Config) -> Result { - tinycortex::memory::queue::count_total(&engine_config(config)) + crate::engine::backend::queue::count_total(&engine_config(config)) } pub fn retry_all_failed(config: &Config) -> Result { - tinycortex::memory::queue::retry_all_failed(&engine_config(config)) + crate::engine::backend::queue::retry_all_failed(&engine_config(config)) } pub fn get_job(config: &Config, id: &str) -> Result> { - tinycortex::memory::queue::get_job(&engine_config(config), id) + crate::engine::backend::queue::get_job(&engine_config(config), id) } diff --git a/core/src/queue/types.rs b/core/src/queue/types.rs index fc87750..3b91b2d 100644 --- a/core/src/queue/types.rs +++ b/core/src/queue/types.rs @@ -1,6 +1,6 @@ //! Queue wire types owned by tinycortex. -pub use tinycortex::memory::queue::{ +pub use crate::engine::backend::queue::{ AppendBufferPayload, AppendTarget, ExtractChunkPayload, FlushStalePayload, Job, JobFailure, JobKind, JobOutcome, JobStatus, NewJob, NodeRef, ReembedBackfillPayload, SealDocumentPayload, SealPayload, diff --git a/core/src/queue/worker.rs b/core/src/queue/worker.rs index 811c540..ebb5b70 100644 --- a/core/src/queue/worker.rs +++ b/core/src/queue/worker.rs @@ -1,6 +1,6 @@ //! Worker pool: drives the crate queue engine (W4 flip). Each `run_once` -//! delegates claim → dispatch → settle to `tinycortex::memory::queue::run_once` -//! via [`crate::tinycortex::HostQueueDelegates`]; the legacy host +//! delegates claim → dispatch → settle to `crate::engine::backend::queue::run_once` +//! via [`crate::engine::HostQueueDelegates`]; the legacy host //! `handlers` engine that used to own dispatch was deleted at the flip. //! //! Concurrency control for LLM-bound work is delegated to @@ -288,9 +288,9 @@ pub async fn run_once(config: &Config) -> Result { // single-slot LLM gate serialises llm-bound jobs; the legacy per-job // local/cloud permit routing and the extract-batch coalescing are // intentionally dropped here (perf, not correctness — W4 follow-up). - let mc = crate::tinycortex::memory_config_from(config, config.workspace_dir().clone()); - let delegates = crate::tinycortex::HostQueueDelegates::new(config.to_arc()); - tinycortex::memory::queue::run_once(&mc, &delegates).await + let mc = crate::engine::memory_config_from(config, config.workspace_dir().clone()); + let delegates = crate::engine::HostQueueDelegates::new(config.to_arc()); + crate::engine::backend::queue::run_once(&mc, &delegates).await } /// Classify whether an error is a transient I/O failure that should be diff --git a/core/src/sources/readers/conversation.rs b/core/src/sources/readers/conversation.rs index 7b18c40..2380821 100644 --- a/core/src/sources/readers/conversation.rs +++ b/core/src/sources/readers/conversation.rs @@ -19,10 +19,10 @@ impl SourceReader for ConversationReader { source: &MemorySourceEntry, config: &Config, ) -> Result, String> { - tinycortex::memory::sources::SourceReader::list_items( - &tinycortex::memory::sources::readers::conversation::ConversationReader, + crate::engine::backend::sources::SourceReader::list_items( + &crate::engine::backend::sources::readers::conversation::ConversationReader, source, - &crate::tinycortex::memory_config_from(config, config.workspace_dir().clone()), + &crate::engine::memory_config_from(config, config.workspace_dir().clone()), ) .await .map_err(|error| error.to_string()) @@ -34,11 +34,11 @@ impl SourceReader for ConversationReader { item_id: &str, config: &Config, ) -> Result { - tinycortex::memory::sources::SourceReader::read_item( - &tinycortex::memory::sources::readers::conversation::ConversationReader, + crate::engine::backend::sources::SourceReader::read_item( + &crate::engine::backend::sources::readers::conversation::ConversationReader, source, item_id, - &crate::tinycortex::memory_config_from(config, config.workspace_dir().clone()), + &crate::engine::memory_config_from(config, config.workspace_dir().clone()), ) .await .map_err(|error| error.to_string()) diff --git a/core/src/sources/readers/folder.rs b/core/src/sources/readers/folder.rs index 31922d5..56bbbfb 100644 --- a/core/src/sources/readers/folder.rs +++ b/core/src/sources/readers/folder.rs @@ -19,10 +19,10 @@ impl SourceReader for FolderReader { source: &MemorySourceEntry, config: &Config, ) -> Result, String> { - tinycortex::memory::sources::SourceReader::list_items( - &tinycortex::memory::sources::readers::folder::FolderReader, + crate::engine::backend::sources::SourceReader::list_items( + &crate::engine::backend::sources::readers::folder::FolderReader, source, - &crate::tinycortex::memory_config_from(config, config.workspace_dir().clone()), + &crate::engine::memory_config_from(config, config.workspace_dir().clone()), ) .await .map_err(|error| error.to_string()) @@ -34,11 +34,11 @@ impl SourceReader for FolderReader { item_id: &str, config: &Config, ) -> Result { - tinycortex::memory::sources::SourceReader::read_item( - &tinycortex::memory::sources::readers::folder::FolderReader, + crate::engine::backend::sources::SourceReader::read_item( + &crate::engine::backend::sources::readers::folder::FolderReader, source, item_id, - &crate::tinycortex::memory_config_from(config, config.workspace_dir().clone()), + &crate::engine::memory_config_from(config, config.workspace_dir().clone()), ) .await .map_err(|error| error.to_string()) diff --git a/core/src/sources/readers/github.rs b/core/src/sources/readers/github.rs index 91d464d..b35af29 100644 --- a/core/src/sources/readers/github.rs +++ b/core/src/sources/readers/github.rs @@ -12,7 +12,9 @@ use crate::sources::readers::SourceReader; use crate::sources::types::{MemorySourceEntry, SourceContent, SourceItem, SourceKind}; use crate::Config; -pub use tinycortex::memory::sources::readers::github::{repo_archive_source_id, repo_chunk_scope}; +pub use crate::engine::backend::sources::readers::github::{ + repo_archive_source_id, repo_chunk_scope, +}; pub struct GithubReader; @@ -27,10 +29,10 @@ impl SourceReader for GithubReader { source: &MemorySourceEntry, config: &Config, ) -> Result, String> { - tinycortex::memory::sources::SourceReader::list_items( - &tinycortex::memory::sources::readers::github::GithubReader, + crate::engine::backend::sources::SourceReader::list_items( + &crate::engine::backend::sources::readers::github::GithubReader, source, - &crate::tinycortex::memory_config_from(config, config.workspace_dir().clone()), + &crate::engine::memory_config_from(config, config.workspace_dir().clone()), ) .await .map_err(|error| error.to_string()) @@ -42,11 +44,11 @@ impl SourceReader for GithubReader { item_id: &str, config: &Config, ) -> Result { - tinycortex::memory::sources::SourceReader::read_item( - &tinycortex::memory::sources::readers::github::GithubReader, + crate::engine::backend::sources::SourceReader::read_item( + &crate::engine::backend::sources::readers::github::GithubReader, source, item_id, - &crate::tinycortex::memory_config_from(config, config.workspace_dir().clone()), + &crate::engine::memory_config_from(config, config.workspace_dir().clone()), ) .await .map_err(|error| error.to_string()) diff --git a/core/src/sources/readers/rss.rs b/core/src/sources/readers/rss.rs index 05a543a..d6213a2 100644 --- a/core/src/sources/readers/rss.rs +++ b/core/src/sources/readers/rss.rs @@ -12,13 +12,13 @@ use crate::Config; /// `read_item`, so constructing it per trait call would turn one sync into /// N+1 downloads. pub struct RssReader { - inner: tinycortex::memory::sources::readers::rss::RssReader, + inner: crate::engine::backend::sources::readers::rss::RssReader, } impl RssReader { pub fn new() -> Self { Self { - inner: tinycortex::memory::sources::readers::rss::RssReader::new(), + inner: crate::engine::backend::sources::readers::rss::RssReader::new(), } } } @@ -40,10 +40,10 @@ impl SourceReader for RssReader { source: &MemorySourceEntry, config: &Config, ) -> Result, String> { - tinycortex::memory::sources::SourceReader::list_items( + crate::engine::backend::sources::SourceReader::list_items( &self.inner, source, - &crate::tinycortex::memory_config_from(config, config.workspace_dir().clone()), + &crate::engine::memory_config_from(config, config.workspace_dir().clone()), ) .await .map_err(|error| error.to_string()) @@ -55,11 +55,11 @@ impl SourceReader for RssReader { item_id: &str, config: &Config, ) -> Result { - tinycortex::memory::sources::SourceReader::read_item( + crate::engine::backend::sources::SourceReader::read_item( &self.inner, source, item_id, - &crate::tinycortex::memory_config_from(config, config.workspace_dir().clone()), + &crate::engine::memory_config_from(config, config.workspace_dir().clone()), ) .await .map_err(|error| error.to_string()) diff --git a/core/src/sources/readers/web_page.rs b/core/src/sources/readers/web_page.rs index a2f693c..69723c8 100644 --- a/core/src/sources/readers/web_page.rs +++ b/core/src/sources/readers/web_page.rs @@ -19,10 +19,10 @@ impl SourceReader for WebPageReader { source: &MemorySourceEntry, config: &Config, ) -> Result, String> { - tinycortex::memory::sources::SourceReader::list_items( - &tinycortex::memory::sources::readers::web_page::WebPageReader, + crate::engine::backend::sources::SourceReader::list_items( + &crate::engine::backend::sources::readers::web_page::WebPageReader, source, - &crate::tinycortex::memory_config_from(config, config.workspace_dir().clone()), + &crate::engine::memory_config_from(config, config.workspace_dir().clone()), ) .await .map_err(|error| error.to_string()) @@ -34,11 +34,11 @@ impl SourceReader for WebPageReader { item_id: &str, config: &Config, ) -> Result { - tinycortex::memory::sources::SourceReader::read_item( - &tinycortex::memory::sources::readers::web_page::WebPageReader, + crate::engine::backend::sources::SourceReader::read_item( + &crate::engine::backend::sources::readers::web_page::WebPageReader, source, item_id, - &crate::tinycortex::memory_config_from(config, config.workspace_dir().clone()), + &crate::engine::memory_config_from(config, config.workspace_dir().clone()), ) .await .map_err(|error| error.to_string()) diff --git a/core/src/sources/registry.rs b/core/src/sources/registry.rs index a33f532..f5c90b9 100644 --- a/core/src/sources/registry.rs +++ b/core/src/sources/registry.rs @@ -5,7 +5,7 @@ use std::sync::OnceLock; use crate::config_loader as config_rpc; use crate::sources::types::{MemorySourceEntry, SourceKind}; -pub use tinycortex::memory::sources::{ +pub use crate::engine::backend::sources::{ memory_sync_defaults_for_toolkit, ComposioUpsertTarget, MemorySourcePatch, }; @@ -18,9 +18,9 @@ pub(crate) async fn memory_sources_write_guard() -> tokio::sync::MutexGuard<'sta .await } -async fn registry() -> Result { +async fn registry() -> Result { let config = config_rpc::load_config_with_timeout().await?; - Ok(tinycortex::memory::sources::SourceRegistry::new( + Ok(crate::engine::backend::sources::SourceRegistry::new( config.config_path(), )) } @@ -59,7 +59,7 @@ pub fn get_source_in( config: &crate::Config, id: &str, ) -> Result, String> { - tinycortex::memory::sources::SourceRegistry::new(config.config_path().clone()) + crate::engine::backend::sources::SourceRegistry::new(config.config_path().clone()) .get(id) .map_err(|error| error.to_string()) } diff --git a/core/src/sources/sync.rs b/core/src/sources/sync.rs index af362a0..134a6e8 100644 --- a/core/src/sources/sync.rs +++ b/core/src/sources/sync.rs @@ -89,7 +89,7 @@ pub async fn sync_source(source: MemorySourceEntry, config: Arc) -> Resu let mut composio_usage = ComposioUsage::default(); let outcome = match source.kind { SourceKind::Composio => { - match crate::tinycortex::run_source_pipeline(&source, &*config).await { + match crate::engine::run_source_pipeline(&source, &*config).await { Ok(outcome) => { composio_usage.actions_called = outcome.actions_called; composio_usage.cost_usd = outcome.provider_cost_usd; @@ -103,17 +103,17 @@ pub async fn sync_source(source: MemorySourceEntry, config: Arc) -> Resu } } SourceKind::Conversation | SourceKind::Folder => { - crate::tinycortex::run_source_pipeline(&source, &*config) + crate::engine::run_source_pipeline(&source, &*config) .await .map(|outcome| outcome.records_ingested as usize) .map_err(|error| error.to_string()) } - SourceKind::GithubRepo => crate::tinycortex::run_source_pipeline(&source, &*config) + SourceKind::GithubRepo => crate::engine::run_source_pipeline(&source, &*config) .await .map(|outcome| outcome.records_ingested as usize) .map_err(|error| error.to_string()), SourceKind::RssFeed | SourceKind::WebPage => { - crate::tinycortex::run_source_pipeline(&source, &*config) + crate::engine::run_source_pipeline(&source, &*config) .await .map(|outcome| outcome.records_ingested as usize) .map_err(|error| error.to_string()) @@ -142,7 +142,7 @@ pub async fn sync_source(source: MemorySourceEntry, config: Arc) -> Resu Some(&source.id), ); - use crate::tinycortex::{append_audit_entry, SyncAuditEntry}; + use crate::engine::{append_audit_entry, SyncAuditEntry}; append_audit_entry( &*config, &SyncAuditEntry { @@ -185,7 +185,7 @@ pub async fn sync_source(source: MemorySourceEntry, config: Arc) -> Resu } Err(error) => { // Audit failed syncs too. - use crate::tinycortex::{append_audit_entry, SyncAuditEntry}; + use crate::engine::{append_audit_entry, SyncAuditEntry}; append_audit_entry( &*config, &SyncAuditEntry { @@ -266,7 +266,7 @@ pub async fn sync_source(source: MemorySourceEntry, config: Arc) -> Resu /// Reconcile raw files that are not yet covered by tree summaries. pub(crate) async fn check_and_rebuild_tree(source: &MemorySourceEntry, config: &Config) { - use crate::tinycortex::{needs_rebuild, rebuild_tree_from_raw}; + use crate::engine::{needs_rebuild, rebuild_tree_from_raw}; for scope in derive_scopes(source, config) { if !needs_rebuild(config, &scope.tree_scope, &scope.archive_source_id) { diff --git a/core/src/sources/types.rs b/core/src/sources/types.rs index 6fbab9f..7588253 100644 --- a/core/src/sources/types.rs +++ b/core/src/sources/types.rs @@ -1,5 +1,5 @@ //! Stable host path for tinycortex-owned memory-source contracts. -pub use tinycortex::memory::sources::{ +pub use crate::engine::backend::sources::{ ContentType, MemorySourceEntry, SourceContent, SourceItem, SourceKind, }; diff --git a/core/src/store/chunks/connection.rs b/core/src/store/chunks/connection.rs index 7fe48ed..746fe6d 100644 --- a/core/src/store/chunks/connection.rs +++ b/core/src/store/chunks/connection.rs @@ -3,15 +3,15 @@ use anyhow::Result; use rusqlite::Connection; -use crate::tinycortex::engine_config; +use crate::engine::engine_config; use crate::Config; #[doc(hidden)] pub fn with_connection(config: &Config, f: impl FnOnce(&Connection) -> Result) -> Result { - tinycortex::memory::chunks::with_connection(&engine_config(config), f) + crate::engine::backend::chunks::with_connection(&engine_config(config), f) } pub(crate) fn recover_corrupt_db(config: &Config) -> Result { log::warn!("[memory:chunks] checking corrupt database recovery"); - tinycortex::memory::chunks::recover_corrupt_db(&engine_config(config)) + crate::engine::backend::chunks::recover_corrupt_db(&engine_config(config)) } diff --git a/core/src/store/chunks/embeddings.rs b/core/src/store/chunks/embeddings.rs index 812852a..07d6a6f 100644 --- a/core/src/store/chunks/embeddings.rs +++ b/core/src/store/chunks/embeddings.rs @@ -5,15 +5,15 @@ use std::collections::HashMap; use anyhow::Result; use rusqlite::{Connection, Transaction}; -use crate::tinycortex::engine_config; +use crate::engine::engine_config; use crate::Config; pub(crate) fn tree_active_signature(config: &Config) -> String { - tinycortex::memory::chunks::tree_active_signature(&engine_config(config)) + crate::engine::backend::chunks::tree_active_signature(&engine_config(config)) } pub fn set_chunk_embedding(config: &Config, id: &str, embedding: &[f32]) -> Result<()> { - tinycortex::memory::chunks::set_chunk_embedding(&engine_config(config), id, embedding) + crate::engine::backend::chunks::set_chunk_embedding(&engine_config(config), id, embedding) } pub fn set_chunk_embedding_for_signature( @@ -22,7 +22,7 @@ pub fn set_chunk_embedding_for_signature( signature: &str, embedding: &[f32], ) -> Result<()> { - tinycortex::memory::chunks::set_chunk_embedding_for_signature( + crate::engine::backend::chunks::set_chunk_embedding_for_signature( &engine_config(config), id, signature, @@ -34,7 +34,7 @@ pub(crate) fn has_uncovered_reembed_work( conn: &Connection, signature: &str, ) -> rusqlite::Result { - tinycortex::memory::chunks::has_uncovered_reembed_work(conn, signature) + crate::engine::backend::chunks::has_uncovered_reembed_work(conn, signature) } pub fn mark_chunk_reembed_skipped( @@ -43,7 +43,7 @@ pub fn mark_chunk_reembed_skipped( signature: &str, reason: &str, ) -> Result<()> { - tinycortex::memory::chunks::mark_chunk_reembed_skipped( + crate::engine::backend::chunks::mark_chunk_reembed_skipped( &engine_config(config), id, signature, @@ -52,11 +52,15 @@ pub fn mark_chunk_reembed_skipped( } pub fn clear_chunk_reembed_skipped(config: &Config, id: &str, signature: &str) -> Result<()> { - tinycortex::memory::chunks::clear_chunk_reembed_skipped(&engine_config(config), id, signature) + crate::engine::backend::chunks::clear_chunk_reembed_skipped( + &engine_config(config), + id, + signature, + ) } pub fn clear_reembed_skipped_for_signature(config: &Config, signature: &str) -> Result { - tinycortex::memory::chunks::clear_reembed_skipped_for_signature( + crate::engine::backend::chunks::clear_reembed_skipped_for_signature( &engine_config(config), signature, ) @@ -68,7 +72,9 @@ pub(crate) fn set_chunk_embedding_for_signature_tx( signature: &str, embedding: &[f32], ) -> Result<()> { - tinycortex::memory::chunks::set_chunk_embedding_for_signature_tx(tx, id, signature, embedding) + crate::engine::backend::chunks::set_chunk_embedding_for_signature_tx( + tx, id, signature, embedding, + ) } pub fn get_chunk_embedding_for_signature( @@ -76,7 +82,7 @@ pub fn get_chunk_embedding_for_signature( id: &str, signature: &str, ) -> Result>> { - tinycortex::memory::chunks::get_chunk_embedding_for_signature( + crate::engine::backend::chunks::get_chunk_embedding_for_signature( &engine_config(config), id, signature, @@ -84,7 +90,7 @@ pub fn get_chunk_embedding_for_signature( } pub fn get_chunk_embedding(config: &Config, id: &str) -> Result>> { - tinycortex::memory::chunks::get_chunk_embedding(&engine_config(config), id) + crate::engine::backend::chunks::get_chunk_embedding(&engine_config(config), id) } pub fn get_chunk_embeddings_for_signature_batch( @@ -92,7 +98,7 @@ pub fn get_chunk_embeddings_for_signature_batch( ids: &[String], signature: &str, ) -> Result>> { - tinycortex::memory::chunks::get_chunk_embeddings_for_signature_batch( + crate::engine::backend::chunks::get_chunk_embeddings_for_signature_batch( &engine_config(config), ids, signature, @@ -103,5 +109,5 @@ pub fn get_chunk_embeddings_batch( config: &Config, ids: &[String], ) -> Result>> { - tinycortex::memory::chunks::get_chunk_embeddings_batch(&engine_config(config), ids) + crate::engine::backend::chunks::get_chunk_embeddings_batch(&engine_config(config), ids) } diff --git a/core/src/store/chunks/mod.rs b/core/src/store/chunks/mod.rs index 9b6fb29..4ba07f1 100644 --- a/core/src/store/chunks/mod.rs +++ b/core/src/store/chunks/mod.rs @@ -20,7 +20,7 @@ pub mod semantic; pub mod store; pub mod types; +pub use crate::engine::backend::chunks::{chunk_markdown, ChunkerInput, ChunkerOptions}; pub use semantic::chunk_markdown as chunk_semantic; pub use store::*; -pub use tinycortex::memory::chunks::{chunk_markdown, ChunkerInput, ChunkerOptions}; pub use types::*; diff --git a/core/src/store/chunks/raw_refs.rs b/core/src/store/chunks/raw_refs.rs index bef5c76..c750ece 100644 --- a/core/src/store/chunks/raw_refs.rs +++ b/core/src/store/chunks/raw_refs.rs @@ -6,7 +6,7 @@ //! directly instead of going through the SQL preview path. //! //! **W3 sub-store flip:** these operations now delegate to -//! [`tinycortex::memory::chunks`] (ported from this exact module — identical SQL +//! [`crate::engine::backend::chunks`] (ported from this exact module — identical SQL //! against the same `mem_tree_chunks` / `mem_tree_summaries` tables in the shared //! `chunks.db` the crate now owns). The host signatures are preserved so the ~4 //! external callers (`content::read`, memory_sync gmail/slack ingest, rebuild) @@ -15,27 +15,27 @@ use anyhow::Result; use rusqlite::Transaction; -use crate::tinycortex::engine_config; +use crate::engine::engine_config; use crate::Config; // `RawRef` is re-exported from the crate (identical fields + serde derives), so // every `chunks::RawRef { path, start, end }` construction site keeps compiling. -pub use tinycortex::memory::chunks::RawRef; +pub use crate::engine::backend::chunks::RawRef; /// Stash a list of [`RawRef`] entries on a chunk row. Replaces any previous /// value. pub fn set_chunk_raw_refs(config: &Config, chunk_id: &str, refs: &[RawRef]) -> Result<()> { - tinycortex::memory::chunks::set_chunk_raw_refs(&engine_config(config), chunk_id, refs) + crate::engine::backend::chunks::set_chunk_raw_refs(&engine_config(config), chunk_id, refs) } /// Stash raw archive pointers on a chunk row inside a caller-owned transaction. pub fn set_chunk_raw_refs_tx(tx: &Transaction<'_>, chunk_id: &str, refs: &[RawRef]) -> Result<()> { - tinycortex::memory::chunks::set_chunk_raw_refs_tx(tx, chunk_id, refs) + crate::engine::backend::chunks::set_chunk_raw_refs_tx(tx, chunk_id, refs) } /// Return the raw-archive pointers stored in SQLite for `chunk_id`, or `None`. pub fn get_chunk_raw_refs(config: &Config, chunk_id: &str) -> Result>> { - tinycortex::memory::chunks::get_chunk_raw_refs(&engine_config(config), chunk_id) + crate::engine::backend::chunks::get_chunk_raw_refs(&engine_config(config), chunk_id) } /// Collect every raw-archive path referenced by any chunk row, restricted to @@ -44,7 +44,7 @@ pub fn list_chunk_raw_ref_paths_with_prefix( config: &Config, rel_prefix: &str, ) -> Result> { - tinycortex::memory::chunks::list_chunk_raw_ref_paths_with_prefix( + crate::engine::backend::chunks::list_chunk_raw_ref_paths_with_prefix( &engine_config(config), rel_prefix, ) @@ -55,12 +55,12 @@ pub fn get_chunk_content_pointers( config: &Config, chunk_id: &str, ) -> Result> { - tinycortex::memory::chunks::get_chunk_content_pointers(&engine_config(config), chunk_id) + crate::engine::backend::chunks::get_chunk_content_pointers(&engine_config(config), chunk_id) } /// Return the `content_path` stored in SQLite for `chunk_id`, if any. pub fn get_chunk_content_path(config: &Config, chunk_id: &str) -> Result> { - tinycortex::memory::chunks::get_chunk_content_path(&engine_config(config), chunk_id) + crate::engine::backend::chunks::get_chunk_content_path(&engine_config(config), chunk_id) } /// Return both `content_path` and `content_sha256` stored in SQLite for `summary_id`. @@ -68,10 +68,10 @@ pub fn get_summary_content_pointers( config: &Config, summary_id: &str, ) -> Result> { - tinycortex::memory::chunks::get_summary_content_pointers(&engine_config(config), summary_id) + crate::engine::backend::chunks::get_summary_content_pointers(&engine_config(config), summary_id) } /// List all summary rows that have a non-NULL `content_path`. pub fn list_summaries_with_content_path(config: &Config) -> Result> { - tinycortex::memory::chunks::list_summaries_with_content_path(&engine_config(config)) + crate::engine::backend::chunks::list_summaries_with_content_path(&engine_config(config)) } diff --git a/core/src/store/chunks/semantic.rs b/core/src/store/chunks/semantic.rs index a624762..b682e85 100644 --- a/core/src/store/chunks/semantic.rs +++ b/core/src/store/chunks/semantic.rs @@ -1,7 +1,7 @@ //! Compatibility exports for tinycortex's semantic Markdown chunker. -pub use tinycortex::memory::chunks::SemanticChunk as Chunk; +pub use crate::engine::backend::chunks::SemanticChunk as Chunk; pub fn chunk_markdown(text: &str, max_tokens: usize) -> Vec { - tinycortex::memory::chunks::chunk_semantic(text, max_tokens) + crate::engine::backend::chunks::chunk_semantic(text, max_tokens) } diff --git a/core/src/store/chunks/store.rs b/core/src/store/chunks/store.rs index f2f7557..47cb55c 100644 --- a/core/src/store/chunks/store.rs +++ b/core/src/store/chunks/store.rs @@ -5,34 +5,38 @@ use std::collections::HashMap; use anyhow::Result; use rusqlite::Transaction; +use crate::engine::engine_config; use crate::store::chunks::types::{Chunk, SourceKind}; use crate::store::content::StagedChunk; -use crate::tinycortex::engine_config; use crate::Config; -pub use tinycortex::memory::chunks::{ +pub use crate::engine::backend::chunks::{ ListChunksQuery, RawRef, CHUNK_STATUS_ADMITTED, CHUNK_STATUS_BUFFERED, CHUNK_STATUS_DROPPED, CHUNK_STATUS_PENDING_EXTRACTION, CHUNK_STATUS_SEALED, RAW_FILE_GATE_KIND, }; pub fn upsert_chunks(config: &Config, chunks: &[Chunk]) -> Result { - tinycortex::memory::chunks::upsert_chunks(&engine_config(config), chunks) + crate::engine::backend::chunks::upsert_chunks(&engine_config(config), chunks) } pub fn upsert_chunks_tx(tx: &Transaction<'_>, chunks: &[Chunk]) -> Result { - tinycortex::memory::chunks::upsert_chunks_tx(tx, chunks) + crate::engine::backend::chunks::upsert_chunks_tx(tx, chunks) } pub fn upsert_staged_chunks_tx(tx: &Transaction<'_>, chunks: &[StagedChunk]) -> Result { - tinycortex::memory::chunks::upsert_staged_chunks_tx(tx, chunks) + crate::engine::backend::chunks::upsert_staged_chunks_tx(tx, chunks) } pub fn update_chunk_content_sha256(config: &Config, id: &str, sha256: &str) -> Result<()> { - tinycortex::memory::chunks::update_chunk_content_sha256(&engine_config(config), id, sha256) + crate::engine::backend::chunks::update_chunk_content_sha256(&engine_config(config), id, sha256) } pub fn update_summary_content_sha256(config: &Config, id: &str, sha256: &str) -> Result<()> { - tinycortex::memory::chunks::update_summary_content_sha256(&engine_config(config), id, sha256) + crate::engine::backend::chunks::update_summary_content_sha256( + &engine_config(config), + id, + sha256, + ) } pub fn list_source_ids_with_prefix( @@ -40,31 +44,35 @@ pub fn list_source_ids_with_prefix( kind: SourceKind, prefix: &str, ) -> Result> { - tinycortex::memory::chunks::list_source_ids_with_prefix(&engine_config(config), kind, prefix) + crate::engine::backend::chunks::list_source_ids_with_prefix( + &engine_config(config), + kind, + prefix, + ) } pub fn get_chunk(config: &Config, id: &str) -> Result> { - tinycortex::memory::chunks::get_chunk(&engine_config(config), id) + crate::engine::backend::chunks::get_chunk(&engine_config(config), id) } pub fn get_chunks_batch(config: &Config, ids: &[String]) -> Result> { - tinycortex::memory::chunks::get_chunks_batch(&engine_config(config), ids) + crate::engine::backend::chunks::get_chunks_batch(&engine_config(config), ids) } pub fn list_chunks(config: &Config, query: &ListChunksQuery) -> Result> { - tinycortex::memory::chunks::list_chunks(&engine_config(config), query) + crate::engine::backend::chunks::list_chunks(&engine_config(config), query) } pub fn count_chunks(config: &Config) -> Result { - tinycortex::memory::chunks::count_chunks(&engine_config(config)) + crate::engine::backend::chunks::count_chunks(&engine_config(config)) } pub fn extraction_coverage(config: &Config) -> Result { - tinycortex::memory::chunks::extraction_coverage(&engine_config(config)) + crate::engine::backend::chunks::extraction_coverage(&engine_config(config)) } pub fn set_chunk_lifecycle_status(config: &Config, id: &str, status: &str) -> Result<()> { - tinycortex::memory::chunks::set_chunk_lifecycle_status(&engine_config(config), id, status) + crate::engine::backend::chunks::set_chunk_lifecycle_status(&engine_config(config), id, status) } pub(crate) fn set_chunk_lifecycle_status_tx( @@ -72,23 +80,23 @@ pub(crate) fn set_chunk_lifecycle_status_tx( id: &str, status: &str, ) -> Result<()> { - tinycortex::memory::chunks::set_chunk_lifecycle_status_tx(tx, id, status) + crate::engine::backend::chunks::set_chunk_lifecycle_status_tx(tx, id, status) } pub fn get_chunk_lifecycle_status(config: &Config, id: &str) -> Result> { - tinycortex::memory::chunks::get_chunk_lifecycle_status(&engine_config(config), id) + crate::engine::backend::chunks::get_chunk_lifecycle_status(&engine_config(config), id) } pub fn get_chunk_lifecycle_status_tx(tx: &Transaction<'_>, id: &str) -> Result> { - tinycortex::memory::chunks::get_chunk_lifecycle_status_tx(tx, id) + crate::engine::backend::chunks::get_chunk_lifecycle_status_tx(tx, id) } pub fn count_chunks_by_lifecycle_status(config: &Config, status: &str) -> Result { - tinycortex::memory::chunks::count_chunks_by_lifecycle_status(&engine_config(config), status) + crate::engine::backend::chunks::count_chunks_by_lifecycle_status(&engine_config(config), status) } pub fn is_source_ingested(config: &Config, kind: SourceKind, id: &str) -> Result { - tinycortex::memory::chunks::is_source_ingested(&engine_config(config), kind, id) + crate::engine::backend::chunks::is_source_ingested(&engine_config(config), kind, id) } pub fn claim_source_ingest_tx( @@ -97,23 +105,26 @@ pub fn claim_source_ingest_tx( id: &str, now_ms: i64, ) -> Result { - tinycortex::memory::chunks::claim_source_ingest_tx(tx, kind, id, now_ms) + crate::engine::backend::chunks::claim_source_ingest_tx(tx, kind, id, now_ms) } pub fn mark_raw_paths_ingested(config: &Config, paths: &[String]) -> Result { - tinycortex::memory::chunks::mark_raw_paths_ingested(&engine_config(config), paths) + crate::engine::backend::chunks::mark_raw_paths_ingested(&engine_config(config), paths) } pub fn filter_raw_paths_not_ingested(config: &Config, paths: &[String]) -> Result> { - tinycortex::memory::chunks::filter_raw_paths_not_ingested(&engine_config(config), paths) + crate::engine::backend::chunks::filter_raw_paths_not_ingested(&engine_config(config), paths) } pub fn count_raw_paths_ingested_with_prefix(config: &Config, prefix: &str) -> Result { - tinycortex::memory::chunks::count_raw_paths_ingested_with_prefix(&engine_config(config), prefix) + crate::engine::backend::chunks::count_raw_paths_ingested_with_prefix( + &engine_config(config), + prefix, + ) } pub fn delete_chunks_by_source(config: &Config, kind: SourceKind, id: &str) -> Result { - tinycortex::memory::chunks::delete_chunks_by_source(&engine_config(config), kind, id) + crate::engine::backend::chunks::delete_chunks_by_source(&engine_config(config), kind, id) } pub fn delete_chunks_by_source_prefix( @@ -121,15 +132,19 @@ pub fn delete_chunks_by_source_prefix( kind: SourceKind, prefix: &str, ) -> Result { - tinycortex::memory::chunks::delete_chunks_by_source_prefix(&engine_config(config), kind, prefix) + crate::engine::backend::chunks::delete_chunks_by_source_prefix( + &engine_config(config), + kind, + prefix, + ) } pub fn delete_chunks_by_owner(config: &Config, kind: SourceKind, owner: &str) -> Result { - tinycortex::memory::chunks::delete_chunks_by_owner(&engine_config(config), kind, owner) + crate::engine::backend::chunks::delete_chunks_by_owner(&engine_config(config), kind, owner) } pub fn delete_orphaned_source_tree(config: &Config, kind: SourceKind, id: &str) -> Result { - tinycortex::memory::chunks::delete_orphaned_source_tree(&engine_config(config), kind, id) + crate::engine::backend::chunks::delete_orphaned_source_tree(&engine_config(config), kind, id) } #[path = "connection.rs"] diff --git a/core/src/store/chunks/types.rs b/core/src/store/chunks/types.rs index bd080fc..8c25440 100644 --- a/core/src/store/chunks/types.rs +++ b/core/src/store/chunks/types.rs @@ -12,13 +12,13 @@ //! **W3 type cutover:** these types + chunk-id/token helpers are now //! **re-exported from the `tinycortex` crate** (ported from this exact module — //! identical fields, derives, serde wire form, and `chunk_id` derivation, all -//! pinned by `tinycortex::memory::chunks::types_tests`). Re-exporting keeps one source of truth and lets +//! pinned by `crate::engine::backend::chunks::types_tests`). Re-exporting keeps one source of truth and lets //! the chunk store operations delegate to the crate without host↔crate type //! conversions. `DataSource` moved with the ingest cutover and is re-exported //! here alongside the chunk types. `StagedChunk` remains host-owned in //! `memory_store::content`. -pub use tinycortex::memory::chunks::{ +pub use crate::engine::backend::chunks::{ approx_token_count, chunk_id, conservative_token_estimate, truncate_to_conservative_tokens, Chunk, DataSource, Metadata, SourceKind, SourceRef, }; diff --git a/core/src/store/client.rs b/core/src/store/client.rs index 24cd438..992946e 100644 --- a/core/src/store/client.rs +++ b/core/src/store/client.rs @@ -77,8 +77,7 @@ impl MemoryClient { /// Typed access to the profile/facet tables. /// - /// **Not guarded.** The profile tables have no capability family in the - /// thirteen-family `tinycortex_api` contract, so these reads and writes + /// **Not guarded.** These reads and writes /// still run beneath `crate::guard::MemoryGuard`'s /// seven steps. What this buys is confinement, not policy: the SQL is in /// the memory family and the compiler keeps it there. diff --git a/core/src/store/content/mod.rs b/core/src/store/content/mod.rs index 52ec477..6da0a48 100644 --- a/core/src/store/content/mod.rs +++ b/core/src/store/content/mod.rs @@ -17,13 +17,13 @@ pub mod read; pub mod tags; -pub use tinycortex::memory::chunks::StagedChunk; +pub use crate::engine::backend::chunks::StagedChunk; /// The git-backed wiki content format. Re-exported only when `memory-git` is /// on: it lives behind tinycortex's `wiki-git` feature, which the gate carries /// along with `git-diff` and the libgit2 cohort. #[cfg(feature = "memory-git")] -pub use tinycortex::memory::store::content::wiki_git; -pub use tinycortex::memory::store::content::{ +pub use crate::engine::backend::store::content::wiki_git; +pub use crate::engine::backend::store::content::{ atomic, compose, obsidian, obsidian_registry, paths, raw, stage_chunks, StagedSummary, SummaryComposeInput, SummaryTreeKind, }; diff --git a/core/src/store/content/read.rs b/core/src/store/content/read.rs index d15ee13..f745e20 100644 --- a/core/src/store/content/read.rs +++ b/core/src/store/content/read.rs @@ -1,15 +1,15 @@ //! Product Config adapters over tinycortex content readers. -use crate::tinycortex::engine_config; +use crate::engine::engine_config; -pub use tinycortex::memory::store::content::{ +pub use crate::engine::backend::store::content::{ read_chunk_file, read_summary_file, verify_chunk_file, verify_summary_file, ChunkFileContents, VerifyResult, }; pub fn read_chunk_body(config: &crate::Config, chunk_id: &str) -> anyhow::Result { - tinycortex::memory::store::content::read_chunk_body(&engine_config(config), chunk_id) + crate::engine::backend::store::content::read_chunk_body(&engine_config(config), chunk_id) } pub fn read_summary_body(config: &crate::Config, summary_id: &str) -> anyhow::Result { - tinycortex::memory::store::content::read_summary_body(&engine_config(config), summary_id) + crate::engine::backend::store::content::read_summary_body(&engine_config(config), summary_id) } diff --git a/core/src/store/content/tags.rs b/core/src/store/content/tags.rs index 92413b1..87ab188 100644 --- a/core/src/store/content/tags.rs +++ b/core/src/store/content/tags.rs @@ -13,7 +13,7 @@ use crate::store::content::compose::{ use crate::tree::score::store::list_entity_ids_for_node; use crate::Config; -pub use tinycortex::memory::store::content::tags::{ +pub use crate::engine::backend::store::content::tags::{ entity_tag, slugify_tag_kind, slugify_tag_value, update_chunk_tags, }; diff --git a/core/src/store/entities.rs b/core/src/store/entities.rs index 47024f4..5cd375a 100644 --- a/core/src/store/entities.rs +++ b/core/src/store/entities.rs @@ -2,13 +2,13 @@ use std::sync::Arc; -use anyhow::Result; -use tinycortex::memory::store::entity_index::{ +use crate::engine::backend::store::entity_index::{ CanonicalEntity, EntityIndex, EntityKind, SelfIdentity, }; +use anyhow::Result; +use crate::engine::memory_config_from; use crate::sync::composio::providers::profile::{is_self_identity_any_toolkit, IdentityKind}; -use crate::tinycortex::memory_config_from; use crate::Config; /// Aggregate entity-index row for capability providers. @@ -32,7 +32,7 @@ pub fn namespace_entities( limit: usize, ) -> Result> { let memory = memory_config_from(config, config.workspace_dir().clone()); - let connection = tinycortex::memory::chunks::shared_connection(&memory)?; + let connection = crate::engine::backend::chunks::shared_connection(&memory)?; let guard = connection.lock(); let pattern = query.map(|value| format!("%{}%", value.to_ascii_lowercase())); let mut statement = guard.prepare( @@ -69,7 +69,7 @@ pub fn namespace_entity_edges( limit: usize, ) -> Result> { let memory = memory_config_from(config, config.workspace_dir().clone()); - let connection = tinycortex::memory::chunks::shared_connection(&memory)?; + let connection = crate::engine::backend::chunks::shared_connection(&memory)?; let guard = connection.lock(); let mut statement = guard.prepare( "SELECT b.entity_id, COUNT(*) @@ -97,7 +97,7 @@ pub fn namespace_entity_edges( Ok(rows) } -pub use tinycortex::memory::store::entity_index::EntityHit; +pub use crate::engine::backend::store::entity_index::EntityHit; #[derive(Debug)] struct HostSelfIdentity; @@ -115,7 +115,7 @@ impl SelfIdentity for HostSelfIdentity { fn index(config: &Config) -> Result { let memory = memory_config_from(config, config.workspace_dir().clone()); - let connection = tinycortex::memory::chunks::shared_connection(&memory)?; + let connection = crate::engine::backend::chunks::shared_connection(&memory)?; EntityIndex::from_shared_connection(connection, Arc::new(HostSelfIdentity)) } @@ -169,7 +169,7 @@ pub fn count_entity_index(config: &Config) -> Result { /// Most frequently observed entities, with recency as the tie-breaker. pub fn top_entities(config: &Config, limit: usize) -> Result> { let memory = memory_config_from(config, config.workspace_dir().clone()); - let connection = tinycortex::memory::chunks::shared_connection(&memory)?; + let connection = crate::engine::backend::chunks::shared_connection(&memory)?; let guard = connection.lock(); let mut statement = guard.prepare( "SELECT entity_id, entity_kind, MAX(surface), COUNT(*) @@ -209,7 +209,7 @@ mod tests { fn insert_entity(config: &Config, tree: &str, entity: &str, node: &str, surface: &str) { let memory = memory_config_from(config, config.workspace_dir().clone()); - let connection = tinycortex::memory::chunks::shared_connection(&memory).expect("db"); + let connection = crate::engine::backend::chunks::shared_connection(&memory).expect("db"); connection .lock() .execute( diff --git a/core/src/store/kv.rs b/core/src/store/kv.rs index 21f9827..625c7af 100644 --- a/core/src/store/kv.rs +++ b/core/src/store/kv.rs @@ -8,7 +8,7 @@ //! again. Canonicalizing here is a no-op for the write path (the transform is //! idempotent and identical) and makes the read path symmetric. -use tinycortex::memory::store::kv::KvStore; +use crate::engine::backend::store::kv::KvStore; use crate::store::namespace_store::UnifiedMemory; use crate::store::safety::canonical_identifier; @@ -92,7 +92,9 @@ impl UnifiedMemory { } } -fn convert_records(records: Vec) -> Vec { +fn convert_records( + records: Vec, +) -> Vec { records .into_iter() .map(|record| MemoryKvRecord { diff --git a/core/src/store/namespace_store/segments.rs b/core/src/store/namespace_store/segments.rs index 66744b6..3af779b 100644 --- a/core/src/store/namespace_store/segments.rs +++ b/core/src/store/namespace_store/segments.rs @@ -27,7 +27,7 @@ CREATE TABLE IF NOT EXISTS conversation_segments ( status TEXT NOT NULL DEFAULT 'open', created_at REAL NOT NULL, updated_at REAL NOT NULL, - -- Per-session sequence numbers from tinycortex::memory::archivist::store, populated + -- Per-session sequence numbers from crate::engine::backend::archivist::store, populated -- alongside start_episodic_id / end_episodic_id during the FTS5 -> md -- migration. Once STM recall switches its segment-span dedup to use -- (session_id, seq) the legacy episodic_id columns can be dropped. diff --git a/core/src/store/profile_store.rs b/core/src/store/profile_store.rs index 9441d9a..0eb734f 100644 --- a/core/src/store/profile_store.rs +++ b/core/src/store/profile_store.rs @@ -8,8 +8,7 @@ //! [`super::namespace_store::profile`], both inside `crate`; //! callers outside the family hold this handle and never a `Connection`. //! -//! **This is not a guard win.** The profile/facet tables have no capability -//! family in the `tinycortex_api` contract, so reads and writes through this +//! **This is not a guard win.** Reads and writes through this //! type still run beneath `crate::guard::MemoryGuard`'s //! seven policy steps: no tier check, no source-scope predicate, no taint //! stamping, no redaction, no budget, no audit event. What changed is the shape diff --git a/core/src/store/safety/mod.rs b/core/src/store/safety/mod.rs index bf48ad4..fe021a0 100644 --- a/core/src/store/safety/mod.rs +++ b/core/src/store/safety/mod.rs @@ -1,5 +1,5 @@ //! Secret-detection and redaction for memory writes — thin host shim over -//! `tinycortex::memory::store::safety` (W3). +//! `crate::engine::backend::store::safety` (W3). //! //! The conservative secret + PII scrubbers (`has_likely_secret`, //! `has_likely_pii`, `sanitize_text`, `sanitize_json`) + the @@ -15,7 +15,7 @@ pub mod pii; use crate::store::types::NamespaceDocumentInput; -pub use tinycortex::memory::store::safety::{ +pub use crate::engine::backend::store::safety::{ has_likely_pii, has_likely_secret, sanitize_json, sanitize_text, SanitizationReport, Sanitized, }; diff --git a/core/src/store/safety/pii.rs b/core/src/store/safety/pii.rs index 9a9a74b..8acff9f 100644 --- a/core/src/store/safety/pii.rs +++ b/core/src/store/safety/pii.rs @@ -1,8 +1,8 @@ //! Personal-PII detection — thin host re-export of the crate scrubber (W3). //! //! The full multilingual national-ID PII module (checksum-gated patterns + -//! Unicode normalization) now lives in `tinycortex::memory::store::safety::pii`; +//! Unicode normalization) now lives in `crate::engine::backend::store::safety::pii`; //! content scrubbing runs inside the crate `sanitize_text`. Host consumers keep //! their `safety::pii::has_likely_pii` import path. -pub use tinycortex::memory::store::safety::pii::{has_likely_pii, redact_pii}; +pub use crate::engine::backend::store::safety::pii::{has_likely_pii, redact_pii}; diff --git a/core/src/store/trees/hotness.rs b/core/src/store/trees/hotness.rs index 5fc3ab4..5ab594a 100644 --- a/core/src/store/trees/hotness.rs +++ b/core/src/store/trees/hotness.rs @@ -2,29 +2,29 @@ use anyhow::Result; +use crate::engine::engine_config; use crate::store::trees::types::HotnessCounters; -use crate::tinycortex::engine_config; use crate::Config; pub fn get(config: &Config, entity_id: &str) -> Result> { - tinycortex::memory::tree::store::hotness::get(&engine_config(config), entity_id) + crate::engine::backend::tree::store::hotness::get(&engine_config(config), entity_id) } pub fn get_or_fresh(config: &Config, entity_id: &str) -> Result { - tinycortex::memory::tree::store::hotness::get_or_fresh(&engine_config(config), entity_id) + crate::engine::backend::tree::store::hotness::get_or_fresh(&engine_config(config), entity_id) } pub fn upsert(config: &Config, counters: &HotnessCounters) -> Result<()> { - tinycortex::memory::tree::store::hotness::upsert(&engine_config(config), counters) + crate::engine::backend::tree::store::hotness::upsert(&engine_config(config), counters) } pub fn distinct_sources_for(config: &Config, entity_id: &str) -> Result { - tinycortex::memory::tree::store::hotness::distinct_sources_for( + crate::engine::backend::tree::store::hotness::distinct_sources_for( &engine_config(config), entity_id, ) } pub fn count(config: &Config) -> Result { - tinycortex::memory::tree::store::hotness::count(&engine_config(config)) + crate::engine::backend::tree::store::hotness::count(&engine_config(config)) } diff --git a/core/src/store/trees/registry.rs b/core/src/store/trees/registry.rs index 3ce8e1b..32b596b 100644 --- a/core/src/store/trees/registry.rs +++ b/core/src/store/trees/registry.rs @@ -2,15 +2,15 @@ use anyhow::Result; +use crate::engine::engine_config; use crate::store::trees::types::{Tree, TreeKind}; -use crate::tinycortex::engine_config; use crate::Config; pub fn list_trees_by_kind(config: &Config, kind: TreeKind) -> Result> { - tinycortex::memory::tree::store::list_trees_by_kind(&engine_config(config), kind) + crate::engine::backend::tree::store::list_trees_by_kind(&engine_config(config), kind) } pub fn archive_tree(config: &Config, tree_id: &str) -> Result<()> { log::debug!("[memory:trees] archive tree_id={tree_id}"); - tinycortex::memory::tree::store::archive_tree(&engine_config(config), tree_id) + crate::engine::backend::tree::store::archive_tree(&engine_config(config), tree_id) } diff --git a/core/src/store/trees/store.rs b/core/src/store/trees/store.rs index 2508757..0bd52e2 100644 --- a/core/src/store/trees/store.rs +++ b/core/src/store/trees/store.rs @@ -6,29 +6,29 @@ use anyhow::Result; use chrono::{DateTime, Utc}; use rusqlite::{Connection, Transaction}; +use crate::engine::engine_config; use crate::store::content::StagedSummary; use crate::store::trees::types::{Buffer, SummaryNode, Tree, TreeKind}; -use crate::tinycortex::engine_config; use crate::Config; pub fn insert_tree(config: &Config, tree: &Tree) -> Result<()> { - tinycortex::memory::tree::store::insert_tree(&engine_config(config), tree) + crate::engine::backend::tree::store::insert_tree(&engine_config(config), tree) } pub fn get_tree_by_scope(config: &Config, kind: TreeKind, scope: &str) -> Result> { - tinycortex::memory::tree::store::get_tree_by_scope(&engine_config(config), kind, scope) + crate::engine::backend::tree::store::get_tree_by_scope(&engine_config(config), kind, scope) } pub fn get_tree(config: &Config, id: &str) -> Result> { - tinycortex::memory::tree::store::get_tree(&engine_config(config), id) + crate::engine::backend::tree::store::get_tree(&engine_config(config), id) } pub fn get_trees_batch(config: &Config, ids: &[String]) -> Result> { - tinycortex::memory::tree::store::get_trees_batch(&engine_config(config), ids) + crate::engine::backend::tree::store::get_trees_batch(&engine_config(config), ids) } pub fn list_trees_by_kind(config: &Config, kind: TreeKind) -> Result> { - tinycortex::memory::tree::store::list_trees_by_kind(&engine_config(config), kind) + crate::engine::backend::tree::store::list_trees_by_kind(&engine_config(config), kind) } pub fn update_tree_after_seal_tx( @@ -38,7 +38,7 @@ pub fn update_tree_after_seal_tx( max_level: u32, sealed_at: DateTime, ) -> Result<()> { - tinycortex::memory::tree::store::update_tree_after_seal_tx( + crate::engine::backend::tree::store::update_tree_after_seal_tx( tx, tree_id, root_id, max_level, sealed_at, ) } @@ -49,7 +49,7 @@ pub fn insert_summary_tx( staged: Option<&StagedSummary>, model_signature: &str, ) -> Result<()> { - tinycortex::memory::tree::store::insert_staged_summary_tx(tx, node, staged, model_signature) + crate::engine::backend::tree::store::insert_staged_summary_tx(tx, node, staged, model_signature) } pub fn set_summary_embedding( @@ -57,7 +57,7 @@ pub fn set_summary_embedding( summary_id: &str, embedding: &[f32], ) -> Result { - tinycortex::memory::tree::store::set_summary_embedding( + crate::engine::backend::tree::store::set_summary_embedding( &engine_config(config), summary_id, embedding, @@ -66,7 +66,7 @@ pub fn set_summary_embedding( } pub fn get_summary_embedding(config: &Config, summary_id: &str) -> Result>> { - tinycortex::memory::tree::store::get_summary_embedding(&engine_config(config), summary_id) + crate::engine::backend::tree::store::get_summary_embedding(&engine_config(config), summary_id) } pub fn set_summary_embedding_for_signature( @@ -75,7 +75,7 @@ pub fn set_summary_embedding_for_signature( signature: &str, embedding: &[f32], ) -> Result<()> { - tinycortex::memory::tree::store::set_summary_embedding_for_signature( + crate::engine::backend::tree::store::set_summary_embedding_for_signature( &engine_config(config), summary_id, signature, @@ -89,7 +89,7 @@ pub fn mark_summary_reembed_skipped( signature: &str, reason: &str, ) -> Result<()> { - tinycortex::memory::chunks::mark_summary_reembed_skipped( + crate::engine::backend::chunks::mark_summary_reembed_skipped( &engine_config(config), summary_id, signature, @@ -102,7 +102,7 @@ pub fn clear_summary_reembed_skipped( summary_id: &str, signature: &str, ) -> Result<()> { - tinycortex::memory::chunks::clear_summary_reembed_skipped( + crate::engine::backend::chunks::clear_summary_reembed_skipped( &engine_config(config), summary_id, signature, @@ -115,7 +115,7 @@ pub(crate) fn set_summary_embedding_for_signature_tx( signature: &str, embedding: &[f32], ) -> Result<()> { - tinycortex::memory::chunks::set_summary_embedding_for_signature_tx( + crate::engine::backend::chunks::set_summary_embedding_for_signature_tx( tx, summary_id, signature, embedding, ) } @@ -125,7 +125,7 @@ pub fn get_summary_embedding_for_signature( summary_id: &str, signature: &str, ) -> Result>> { - tinycortex::memory::tree::store::get_summary_embedding_for_signature( + crate::engine::backend::tree::store::get_summary_embedding_for_signature( &engine_config(config), summary_id, signature, @@ -137,7 +137,7 @@ pub fn get_summary_embeddings_for_signature_batch( ids: &[String], signature: &str, ) -> Result>> { - tinycortex::memory::tree::store::get_summary_embeddings_for_signature_batch( + crate::engine::backend::tree::store::get_summary_embeddings_for_signature_batch( &engine_config(config), ids, signature, @@ -148,18 +148,18 @@ pub fn get_summary_embeddings_batch( config: &Config, ids: &[String], ) -> Result>> { - tinycortex::memory::tree::store::get_summary_embeddings_batch(&engine_config(config), ids) + crate::engine::backend::tree::store::get_summary_embeddings_batch(&engine_config(config), ids) } pub fn get_summary(config: &Config, id: &str) -> Result> { - tinycortex::memory::tree::store::get_summary(&engine_config(config), id) + crate::engine::backend::tree::store::get_summary(&engine_config(config), id) } pub fn get_summaries_batch( config: &Config, ids: &[String], ) -> Result> { - tinycortex::memory::tree::store::get_summaries_batch(&engine_config(config), ids) + crate::engine::backend::tree::store::get_summaries_batch(&engine_config(config), ids) } pub fn list_summaries_at_level( @@ -167,7 +167,11 @@ pub fn list_summaries_at_level( tree_id: &str, level: u32, ) -> Result> { - tinycortex::memory::tree::store::list_summaries_at_level(&engine_config(config), tree_id, level) + crate::engine::backend::tree::store::list_summaries_at_level( + &engine_config(config), + tree_id, + level, + ) } pub fn list_summaries_in_window( @@ -176,7 +180,7 @@ pub fn list_summaries_in_window( since_ms: i64, until_ms: i64, ) -> Result> { - tinycortex::memory::tree::store::list_summaries_in_window( + crate::engine::backend::tree::store::list_summaries_in_window( &engine_config(config), tree_id, since_ms, @@ -185,21 +189,21 @@ pub fn list_summaries_in_window( } pub fn count_summaries(config: &Config, tree_id: &str) -> Result { - tinycortex::memory::tree::store::count_summaries(&engine_config(config), tree_id) + crate::engine::backend::tree::store::count_summaries(&engine_config(config), tree_id) } pub fn get_buffer(config: &Config, tree_id: &str, level: u32) -> Result { - tinycortex::memory::tree::store::get_buffer(&engine_config(config), tree_id, level) + crate::engine::backend::tree::store::get_buffer(&engine_config(config), tree_id, level) } pub(crate) fn get_buffer_conn(conn: &Connection, tree_id: &str, level: u32) -> Result { - tinycortex::memory::tree::store::get_buffer_conn(conn, tree_id, level) + crate::engine::backend::tree::store::get_buffer_conn(conn, tree_id, level) } pub fn upsert_buffer_tx(tx: &Transaction<'_>, buffer: &Buffer) -> Result<()> { - tinycortex::memory::tree::store::upsert_buffer_tx(tx, buffer) + crate::engine::backend::tree::store::upsert_buffer_tx(tx, buffer) } pub fn list_stale_buffers(config: &Config, older_than: DateTime) -> Result> { - tinycortex::memory::tree::store::list_stale_buffers(&engine_config(config), older_than) + crate::engine::backend::tree::store::list_stale_buffers(&engine_config(config), older_than) } diff --git a/core/src/store/trees/store_tests.rs b/core/src/store/trees/store_tests.rs index d470951..741fe4d 100644 --- a/core/src/store/trees/store_tests.rs +++ b/core/src/store/trees/store_tests.rs @@ -297,7 +297,7 @@ fn buffer_upsert_and_clear() { with_connection(&cfg, |conn| { let tx = conn.unchecked_transaction()?; - tinycortex::memory::tree::store::clear_buffer_tx(&tx, "tree-1", 0)?; + crate::engine::backend::tree::store::clear_buffer_tx(&tx, "tree-1", 0)?; tx.commit()?; Ok(()) }) diff --git a/core/src/store/trees/types.rs b/core/src/store/trees/types.rs index 114fb97..b0cf40f 100644 --- a/core/src/store/trees/types.rs +++ b/core/src/store/trees/types.rs @@ -1,6 +1,6 @@ //! Compatibility exports for tinycortex summary-tree persistence types. -pub use tinycortex::memory::tree::store::{ +pub use crate::engine::backend::tree::store::{ Buffer, EntityIndexStats, HotnessCounters, SummaryNode, Tree, TreeKind, TreeStatus, DEFAULT_FLUSH_AGE_SECS, INPUT_TOKEN_BUDGET, OUTPUT_TOKEN_BUDGET, SUMMARY_FANOUT, TOPIC_ARCHIVE_THRESHOLD, TOPIC_CREATION_THRESHOLD, TOPIC_RECHECK_EVERY, diff --git a/core/src/store/types.rs b/core/src/store/types.rs index d72101e..4cc1ead 100644 --- a/core/src/store/types.rs +++ b/core/src/store/types.rs @@ -1,9 +1,9 @@ //! Stable host path for tinycortex-owned namespace memory contracts. -pub use tinycortex::memory::{ +pub use crate::engine::backend::{ GraphRelationRecord, MemoryItemKind, MemoryKvRecord, NamespaceDocumentInput, NamespaceMemoryHit, NamespaceQueryResult, NamespaceRetrievalContext, RetrievalScoreBreakdown, StoredMemoryDocument, }; -pub(crate) use tinycortex::memory::types::GLOBAL_NAMESPACE; +pub(crate) use crate::engine::backend::types::GLOBAL_NAMESPACE; diff --git a/core/src/sync/composio/mod.rs b/core/src/sync/composio/mod.rs index 37e2721..b3df5eb 100644 --- a/core/src/sync/composio/mod.rs +++ b/core/src/sync/composio/mod.rs @@ -161,12 +161,8 @@ pub async fn run_connection_sync( .duration_since(std::time::UNIX_EPOCH) .unwrap_or_default() .as_millis() as u64; - match crate::tinycortex::run_composio_connection( - &target.toolkit, - &target.connection_id, - &*config, - ) - .await + match crate::engine::run_composio_connection(&target.toolkit, &target.connection_id, &*config) + .await { Ok(outcome) => { let usage = ComposioUsage { diff --git a/core/src/sync/composio/periodic.rs b/core/src/sync/composio/periodic.rs index 22cfa71..400e43b 100644 --- a/core/src/sync/composio/periodic.rs +++ b/core/src/sync/composio/periodic.rs @@ -58,7 +58,7 @@ use tinymemory_api::host::DEFAULT_MEMORY_SYNC_INTERVAL_SECS; use super::providers::{get_provider, ComposioUsage}; use crate::composio_host; -use crate::tinycortex::{append_audit_entry, try_read_audit_log, SyncAuditEntry}; +use crate::engine::{append_audit_entry, try_read_audit_log, SyncAuditEntry}; use chrono::{DateTime, Utc}; /// How often the scheduler wakes up to look for due syncs. Independent @@ -555,7 +555,7 @@ pub(crate) async fn run_one_tick() -> Result<(), String> { "[composio:periodic] firing sync" ); let sync_started = Instant::now(); - let result = crate::tinycortex::run_source_pipeline(&source, &*config).await; + let result = crate::engine::run_source_pipeline(&source, &*config).await; let duration_ms = sync_started.elapsed().as_millis() as u64; match result { diff --git a/core/src/sync/composio/providers/clickup/mod.rs b/core/src/sync/composio/providers/clickup/mod.rs index bbf3908..807d7fe 100644 --- a/core/src/sync/composio/providers/clickup/mod.rs +++ b/core/src/sync/composio/providers/clickup/mod.rs @@ -6,7 +6,8 @@ //! re-learning a new shape: //! //! - `provider.rs` — `impl ComposioProvider for ClickUpProvider` -//! - `normalization` — payload-shape helpers, now `tinycortex::…::normalize::clickup` +//! - `normalization` — payload-shape helpers, now reached through +//! `crate::engine::engine` (issue #18 §C1) //! - `ingest.rs` — memory_tree document ingest (issue #2885) //! - `tools.rs` — `CLICKUP_CURATED` whitelist of Composio actions //! - `tests.rs` — unit tests for the helpers + trait metadata @@ -16,7 +17,7 @@ // The payload normalisers moved to tinycortex (they are pure Value // transforms, i.e. driver-side). Aliased under the old module name so // every `normalization::extract_*` call site below stays unchanged. -use tinycortex::memory::sync::composio::providers::normalize::clickup as normalization; +use crate::engine::backend::sync::composio::providers::normalize::clickup as normalization; mod provider; #[cfg(test)] mod tests; diff --git a/core/src/sync/composio/providers/github/mod.rs b/core/src/sync/composio/providers/github/mod.rs index 691b6e2..58c3cd8 100644 --- a/core/src/sync/composio/providers/github/mod.rs +++ b/core/src/sync/composio/providers/github/mod.rs @@ -6,7 +6,8 @@ //! re-learning a new shape: //! //! - `provider.rs` — `impl ComposioProvider for GitHubProvider` -//! - `normalization` — payload-shape helpers, now `tinycortex::…::normalize::github` +//! - `normalization` — payload-shape helpers, now reached through +//! `crate::engine::engine` (issue #18 §C1) //! - `tools.rs` — `GITHUB_CURATED` whitelist of Composio actions //! - `tests.rs` — unit tests for the helpers + trait metadata //! @@ -15,7 +16,7 @@ // The payload normalisers moved to tinycortex (they are pure Value // transforms, i.e. driver-side). Aliased under the old module name so // every `normalization::extract_*` call site below stays unchanged. -use tinycortex::memory::sync::composio::providers::normalize::github as normalization; +use crate::engine::backend::sync::composio::providers::normalize::github as normalization; mod provider; #[cfg(test)] mod tests; diff --git a/core/src/sync/composio/providers/gmail/mod.rs b/core/src/sync/composio/providers/gmail/mod.rs index fcdc23e..f8c99a0 100644 --- a/core/src/sync/composio/providers/gmail/mod.rs +++ b/core/src/sync/composio/providers/gmail/mod.rs @@ -1,7 +1,7 @@ // The Gmail post-processor moved to tinycortex (a pure Value transform, i.e. // driver-side). Aliased under the old module name so the single call site in // `provider.rs` stays unchanged. -use tinycortex::memory::sync::composio::providers::normalize::gmail_post_process as post_process; +use crate::engine::backend::sync::composio::providers::normalize::gmail_post_process as post_process; mod provider; #[cfg(test)] mod tests; diff --git a/core/src/sync/composio/providers/gmail/provider.rs b/core/src/sync/composio/providers/gmail/provider.rs index 07b7f0c..a46617e 100644 --- a/core/src/sync/composio/providers/gmail/provider.rs +++ b/core/src/sync/composio/providers/gmail/provider.rs @@ -152,12 +152,9 @@ impl ComposioProvider for GmailProvider { let Some(connection_id) = ctx.connection_id.as_deref() else { return Err("[composio:gmail] trigger missing connection_id".to_string()); }; - if let Err(e) = crate::tinycortex::run_composio_connection( - "gmail", - connection_id, - ctx.config.as_ref(), - ) - .await + if let Err(e) = + crate::engine::run_composio_connection("gmail", connection_id, ctx.config.as_ref()) + .await { tracing::warn!( error = %e, @@ -171,5 +168,5 @@ impl ComposioProvider for GmailProvider { // Message fetching (the `GMAIL_FETCH_EMAILS` action, the search query, the // `max_items` cap math and the `sync_depth_days` `after:` floor) is owned -// by `tinycortex::memory::sync::GmailSyncPipeline`. What stays here is the +// by `crate::engine::backend::sync::GmailSyncPipeline`. What stays here is the // host-side provider surface: profile lookup and trigger dispatch. diff --git a/core/src/sync/composio/providers/gmail/tests.rs b/core/src/sync/composio/providers/gmail/tests.rs index 70b5e6b..17a1825 100644 --- a/core/src/sync/composio/providers/gmail/tests.rs +++ b/core/src/sync/composio/providers/gmail/tests.rs @@ -1,7 +1,7 @@ //! Host-owned Gmail provider surface tests. //! //! Pagination, cursor, envelope parsing, and ingest behavior are owned and -//! tested by `tinycortex::memory::sync::GmailSyncPipeline`. +//! tested by `crate::engine::backend::sync::GmailSyncPipeline`. use super::GmailProvider; use crate::sync::composio::providers::ComposioProvider; diff --git a/core/src/sync/composio/providers/helpers.rs b/core/src/sync/composio/providers/helpers.rs index 6e1e2ba..d2aca25 100644 --- a/core/src/sync/composio/providers/helpers.rs +++ b/core/src/sync/composio/providers/helpers.rs @@ -1,11 +1,11 @@ //! Shared helpers for Composio provider implementations. //! //! `pick_str` used to live here. It is a provider payload normaliser, so it -//! moved to `tinycortex::memory::sync::composio::providers::normalize::helpers` +//! moved to `crate::engine::backend::sync::composio::providers::normalize::helpers` //! and is re-exported from this module's parent. The helpers that remain are //! request-building rather than normalisation, and stay host-side. -use tinycortex::memory::sync::composio::providers::normalize::helpers::pick_str; +use crate::engine::backend::sync::composio::providers::normalize::helpers::pick_str; /// Shallow-merge an `extra` JSON object into a (mutable) action-args /// object. Only object-typed extras are merged; non-object `extra` diff --git a/core/src/sync/composio/providers/linear/mod.rs b/core/src/sync/composio/providers/linear/mod.rs index 27ff7b7..af81bca 100644 --- a/core/src/sync/composio/providers/linear/mod.rs +++ b/core/src/sync/composio/providers/linear/mod.rs @@ -6,7 +6,7 @@ // The payload normalisers moved to tinycortex (they are pure Value // transforms, i.e. driver-side). Aliased under the old module name so // every `normalization::extract_*` call site below stays unchanged. -use tinycortex::memory::sync::composio::providers::normalize::linear as normalization; +use crate::engine::backend::sync::composio::providers::normalize::linear as normalization; mod provider; #[cfg(test)] mod tests; diff --git a/core/src/sync/composio/providers/mod.rs b/core/src/sync/composio/providers/mod.rs index 0a86355..7bb43ac 100644 --- a/core/src/sync/composio/providers/mod.rs +++ b/core/src/sync/composio/providers/mod.rs @@ -281,11 +281,11 @@ pub(crate) use helpers::{first_array_str, merge_extra}; // re-exported here so the ~40 in-tree call sites keep resolving unchanged. // Note this is deliberately NOT `providers::common::pick_str`, which coerces // numbers to strings — see the doc comments on both definitions. +pub(crate) use crate::engine::backend::sync::composio::providers::normalize::helpers::pick_str; pub use registry::{ all_providers, get_provider, init_default_providers, register_provider, ProviderArc, }; pub use scope_lookup::{curated_scope_for, toolkit_has_scope}; -pub(crate) use tinycortex::memory::sync::composio::providers::normalize::helpers::pick_str; pub use tool_scope::{classify_unknown, find_curated, toolkit_from_slug, CuratedTool, ToolScope}; pub use traits::{resolve_sync_interval_secs, sync_interval_env_var, ComposioProvider}; pub use types::{ diff --git a/core/src/sync/composio/providers/notion/mod.rs b/core/src/sync/composio/providers/notion/mod.rs index 5613ed0..f781129 100644 --- a/core/src/sync/composio/providers/notion/mod.rs +++ b/core/src/sync/composio/providers/notion/mod.rs @@ -1,7 +1,7 @@ // The payload normalisers moved to tinycortex (they are pure Value // transforms, i.e. driver-side). Aliased under the old module name so // every `normalization::extract_*` call site below stays unchanged. -use tinycortex::memory::sync::composio::providers::normalize::notion as normalization; +use crate::engine::backend::sync::composio::providers::normalize::notion as normalization; mod provider; #[cfg(test)] mod tests; diff --git a/core/src/sync/composio/providers/notion/provider.rs b/core/src/sync/composio/providers/notion/provider.rs index 63990fd..d02cc3e 100644 --- a/core/src/sync/composio/providers/notion/provider.rs +++ b/core/src/sync/composio/providers/notion/provider.rs @@ -279,7 +279,7 @@ impl ComposioProvider for NotionProvider { return Err("[composio:notion] trigger missing connection_id".to_string()); }; if let Err(e) = - crate::tinycortex::run_composio_connection("notion", connection_id, ctx.config.as_ref()) + crate::engine::run_composio_connection("notion", connection_id, ctx.config.as_ref()) .await { tracing::warn!( diff --git a/core/src/sync/composio/providers/slack/mod.rs b/core/src/sync/composio/providers/slack/mod.rs index 034492e..8d1e84e 100644 --- a/core/src/sync/composio/providers/slack/mod.rs +++ b/core/src/sync/composio/providers/slack/mod.rs @@ -10,7 +10,7 @@ // driver-side). Re-exported under the old module name — `pub`, not a plain // `use`, because `tests/raw_coverage/memory_threads_raw_coverage_e2e.rs` // imports this path directly. -pub use tinycortex::memory::sync::composio::providers::normalize::slack_post_process as post_process; +pub use crate::engine::backend::sync::composio::providers::normalize::slack_post_process as post_process; pub mod types; mod provider; diff --git a/core/src/sync/composio/providers/slack/provider.rs b/core/src/sync/composio/providers/slack/provider.rs index 5d22208..3ed5270 100644 --- a/core/src/sync/composio/providers/slack/provider.rs +++ b/core/src/sync/composio/providers/slack/provider.rs @@ -233,12 +233,9 @@ impl ComposioProvider for SlackProvider { let Some(connection_id) = ctx.connection_id.as_deref() else { return Err("[composio:slack] trigger missing connection_id".to_string()); }; - if let Err(e) = crate::tinycortex::run_composio_connection( - "slack", - connection_id, - ctx.config.as_ref(), - ) - .await + if let Err(e) = + crate::engine::run_composio_connection("slack", connection_id, ctx.config.as_ref()) + .await { tracing::warn!( error = %e, @@ -262,13 +259,10 @@ pub async fn run_backfill_via_search( .as_deref() .ok_or_else(|| "[composio:slack] search backfill missing connection_id".to_string())?; let started_at_ms = now_ms(); - let outcome = crate::tinycortex::run_slack_search_backfill( - connection_id, - backfill_days, - ctx.config.as_ref(), - ) - .await - .map_err(|error| error.to_string())?; + let outcome = + crate::engine::run_slack_search_backfill(connection_id, backfill_days, ctx.config.as_ref()) + .await + .map_err(|error| error.to_string())?; Ok(SyncOutcome { toolkit: "slack".into(), connection_id: Some(connection_id.into()), diff --git a/core/src/sync/composio/providers/sync_state.rs b/core/src/sync/composio/providers/sync_state.rs index da33c44..4e33f91 100644 --- a/core/src/sync/composio/providers/sync_state.rs +++ b/core/src/sync/composio/providers/sync_state.rs @@ -1,9 +1,9 @@ //! Compatibility exports for sync state now owned by tinycortex. -pub use tinycortex::memory::sync::state::DEFAULT_DAILY_REQUEST_LIMIT; -pub use tinycortex::memory::sync::{DailyBudget, SyncState}; +pub use crate::engine::backend::sync::state::DEFAULT_DAILY_REQUEST_LIMIT; +pub use crate::engine::backend::sync::{DailyBudget, SyncState}; -pub const KV_NAMESPACE: &str = crate::tinycortex::HOST_SYNC_STATE_NAMESPACE; +pub const KV_NAMESPACE: &str = crate::engine::HOST_SYNC_STATE_NAMESPACE; pub fn extract_item_id(item: &serde_json::Value, paths: &[&str]) -> Option { paths.iter().find_map(|path| { diff --git a/core/src/sync/composio/providers/traits.rs b/core/src/sync/composio/providers/traits.rs index 8585eb2..b9a0e30 100644 --- a/core/src/sync/composio/providers/traits.rs +++ b/core/src/sync/composio/providers/traits.rs @@ -59,7 +59,7 @@ pub trait ComposioProvider: Send + Sync { ) })?; let started_at_ms = now_ms(); - let outcome = crate::tinycortex::run_composio_connection_with_budgets( + let outcome = crate::engine::run_composio_connection_with_budgets( self.toolkit_slug(), connection_id, ctx.config.as_ref(), diff --git a/core/src/sync/sync_status/mod.rs b/core/src/sync/sync_status/mod.rs index 019bbd6..11c5b83 100644 --- a/core/src/sync/sync_status/mod.rs +++ b/core/src/sync/sync_status/mod.rs @@ -13,4 +13,4 @@ //! * `openhuman.memory_sync_status_list` — handler in `rpc` //! * Controller registration via `schemas::all_registered_controllers` -pub use tinycortex::memory::sync::{FreshnessLabel, MemorySyncStatus}; +pub use crate::engine::backend::sync::{FreshnessLabel, MemorySyncStatus}; diff --git a/core/src/sync/workspace/periodic.rs b/core/src/sync/workspace/periodic.rs index a1d145e..dbc3a0b 100644 --- a/core/src/sync/workspace/periodic.rs +++ b/core/src/sync/workspace/periodic.rs @@ -33,13 +33,13 @@ use chrono::{DateTime, Utc}; use tokio::time::interval; use crate::config_loader as config_rpc; +use crate::engine::{try_read_audit_log, SyncAuditEntry}; use crate::scheduler_gate::resume_notify; use crate::sources::sync::sync_source; use crate::sources::types::{MemorySourceEntry, SourceKind}; use crate::sync::composio::periodic::{ connection_is_due, effective_interval_secs, periodic_pause_reason, }; -use crate::tinycortex::{try_read_audit_log, SyncAuditEntry}; use tinymemory_api::host::DEFAULT_MEMORY_SYNC_INTERVAL_SECS; /// How often the scheduler wakes up to look for due syncs. Matches the diff --git a/core/src/sync/workspace/watcher.rs b/core/src/sync/workspace/watcher.rs index 27bdaf8..c43e529 100644 --- a/core/src/sync/workspace/watcher.rs +++ b/core/src/sync/workspace/watcher.rs @@ -56,7 +56,7 @@ use tokio::sync::mpsc; use crate::Config; use crate::config_loader as config_rpc; use crate::ingest_pipeline::ingest_document_with_scope; -use tinycortex::memory::ingest::canonicalize::document::DocumentInput; +use crate::engine::backend::ingest::canonicalize::document::DocumentInput; use crate::sync::workspace::watcher::state::WatcherStateStore; use crate::scheduler_gate::current_policy; use crate::scheduler_gate::PauseReason; diff --git a/core/src/tool_memory/mod.rs b/core/src/tool_memory/mod.rs index 0bbf8fa..6314279 100644 --- a/core/src/tool_memory/mod.rs +++ b/core/src/tool_memory/mod.rs @@ -18,9 +18,9 @@ //! //! ## Components //! -//! - [`tinycortex::memory::tool_memory::types`] owns [`ToolMemoryRule`], +//! - [`crate::engine::backend::tool_memory::types`] owns [`ToolMemoryRule`], //! [`ToolMemoryPriority`], and [`ToolMemorySource`]. -//! - [`tinycortex::memory::tool_memory::store`] owns [`ToolMemoryStore`], the +//! - [`crate::engine::backend::tool_memory::store`] owns [`ToolMemoryStore`], the //! put/list/delete/prompt API built on top of an `Arc`. //! - `capture` — `ToolMemoryCaptureHook`, the post-turn //! `PostTurnHook` that records user edicts and repeated tool @@ -37,8 +37,8 @@ mod store; #[cfg(any(test, feature = "test-support"))] pub mod test_helpers; -pub use store::tool_memory_store; -pub use tinycortex::memory::tool_memory::{ +pub use crate::engine::backend::tool_memory::{ store::{ToolMemoryStore, TOOL_MEMORY_PROMPT_CAP}, types::{tool_memory_namespace, ToolMemoryPriority, ToolMemoryRule, ToolMemorySource}, }; +pub use store::tool_memory_store; diff --git a/core/src/tool_memory/store.rs b/core/src/tool_memory/store.rs index 3446fba..857a497 100644 --- a/core/src/tool_memory/store.rs +++ b/core/src/tool_memory/store.rs @@ -2,7 +2,7 @@ use std::sync::Arc; use crate::Memory; -use tinycortex::memory::tool_memory::store::ToolMemoryStore; +use crate::engine::backend::tool_memory::store::ToolMemoryStore; /// Build the crate-owned store over OpenHuman's shared memory object. pub fn tool_memory_store(memory: Arc) -> ToolMemoryStore { diff --git a/core/src/traits.rs b/core/src/traits.rs index af12346..d57f618 100644 --- a/core/src/traits.rs +++ b/core/src/traits.rs @@ -23,7 +23,7 @@ // ── The contract's trait and value types ───────────────────────────────────── // -// Named directly rather than reached through `tinycortex::memory`. Since §A1 the +// Named directly rather than reached through the engine's re-export. Since §A1 the // engine re-exports this same contract, so the two spellings resolve to one type // either way — but going through the engine to reach an engine-neutral contract // is what §1.1 of issue #18 calls out, and it is what would have to be undone diff --git a/core/src/tree/graph/bfs.rs b/core/src/tree/graph/bfs.rs index 444d068..51c379e 100644 --- a/core/src/tree/graph/bfs.rs +++ b/core/src/tree/graph/bfs.rs @@ -4,15 +4,15 @@ use anyhow::Result; use crate::Config; -pub use tinycortex::memory::graph::PairDistance; +pub use crate::engine::backend::graph::PairDistance; pub fn pair_distances( config: &Config, entity_ids: &[String], max_h: u32, ) -> Result> { - tinycortex::memory::graph::pair_distances( - &crate::tinycortex::memory_config_from(config, config.workspace_dir().clone()), + crate::engine::backend::graph::pair_distances( + &crate::engine::memory_config_from(config, config.workspace_dir().clone()), entity_ids, max_h, ) diff --git a/core/src/tree/graph/store.rs b/core/src/tree/graph/store.rs index 2408300..69094ed 100644 --- a/core/src/tree/graph/store.rs +++ b/core/src/tree/graph/store.rs @@ -3,17 +3,17 @@ use anyhow::Result; use rusqlite::Transaction; -use crate::tinycortex::engine_config; +use crate::engine::engine_config; use crate::Config; -pub use tinycortex::memory::graph::pairs_from_entities; +pub use crate::engine::backend::graph::pairs_from_entities; pub fn upsert_edges_tx( transaction: &Transaction<'_>, pairs: &[(String, String)], timestamp_ms: i64, ) -> Result { - tinycortex::memory::graph::upsert_edges_tx(transaction, pairs, timestamp_ms) + crate::engine::backend::graph::upsert_edges_tx(transaction, pairs, timestamp_ms) } pub fn upsert_edges( @@ -21,20 +21,20 @@ pub fn upsert_edges( pairs: &[(String, String)], timestamp_ms: i64, ) -> Result { - tinycortex::memory::graph::upsert_edges(&engine_config(config), pairs, timestamp_ms) + crate::engine::backend::graph::upsert_edges(&engine_config(config), pairs, timestamp_ms) } pub fn neighbors(config: &Config, entity_id: &str) -> Result> { - tinycortex::memory::graph::edge_neighbors(&engine_config(config), entity_id) + crate::engine::backend::graph::edge_neighbors(&engine_config(config), entity_id) } pub fn clear_edges_for_entities_tx( transaction: &Transaction<'_>, entity_ids: &[String], ) -> Result { - tinycortex::memory::graph::clear_edges_for_entities_tx(transaction, entity_ids) + crate::engine::backend::graph::clear_edges_for_entities_tx(transaction, entity_ids) } pub fn count_edges(config: &Config) -> Result { - tinycortex::memory::graph::count_edges(&engine_config(config)) + crate::engine::backend::graph::count_edges(&engine_config(config)) } diff --git a/core/src/tree/health/mod.rs b/core/src/tree/health/mod.rs index 87bbbc8..2bfb1a8 100644 --- a/core/src/tree/health/mod.rs +++ b/core/src/tree/health/mod.rs @@ -3,7 +3,7 @@ //! The **taxonomy itself** — [`FailureCode`], [`FailureClass`], //! [`PipelineFailure`], [`DegradedState`], and the `classify_embed_error` //! classifier — now lives in the engine crate at -//! `tinycortex::memory::health`, and is re-exported below so every existing +//! `crate::engine::backend::health`, and is re-exported below so every existing //! `memory::tree::health::…` path keeps resolving. It moved because a build //! whose only driver was a third-party external backend would have no use for //! the engine's private failure vocabulary. @@ -27,7 +27,7 @@ pub(crate) use user_error::publish_local_model_unavailable_user_error; /// The failure taxonomy proper. Re-exported (rather than re-declared) so the /// ~30 `crate::tree::health::{…}` call sites across the host /// are unaffected by the move, and so there is exactly one definition. -pub use tinycortex::memory::health::{ +pub use crate::engine::backend::health::{ classify_embed_error, classify_embed_error_str, DegradedState, FailureClass, FailureCode, PipelineFailure, }; diff --git a/core/src/tree/ingest.rs b/core/src/tree/ingest.rs index 805c60e..edb2067 100644 --- a/core/src/tree/ingest.rs +++ b/core/src/tree/ingest.rs @@ -4,13 +4,13 @@ use anyhow::Context; use anyhow::Result; +use crate::engine::{memory_config_from, HostSummariser}; #[cfg(feature = "memory-git")] use crate::store::content::wiki_git::{SummaryCommitBatch, SummaryCommitEntry}; use crate::store::trees::types::Tree; -use crate::tinycortex::{memory_config_from, HostSummariser}; use crate::Config; -pub use tinycortex::memory::tree::{SummaryIngestInput, SummaryIngestOutcome}; +pub use crate::engine::backend::tree::{SummaryIngestInput, SummaryIngestOutcome}; pub async fn ingest_summary( config: &Config, @@ -27,7 +27,7 @@ pub async fn ingest_summary( log::warn!("[memory_tree::ingest] obsidian defaults failed: {error:#}"); } - let outcome = tinycortex::memory::tree::ingest_summary( + let outcome = crate::engine::backend::tree::ingest_summary( &memory_config_from(config, config.workspace_dir().clone()), tree, input.clone(), diff --git a/core/src/tree/mod.rs b/core/src/tree/mod.rs index 61e0ab4..3e26f8a 100644 --- a/core/src/tree/mod.rs +++ b/core/src/tree/mod.rs @@ -21,7 +21,7 @@ pub mod tree; pub mod tree_runtime; // Tree I/O contracts are engine-owned. -pub use tinycortex::memory::tree::{ +pub use crate::engine::backend::tree::{ TreeLabelStrategy, TreeLeafPayload, TreeReadHit, TreeReadRequest, TreeReadResult, TreeWriteOutcome, TreeWriteRequest, }; diff --git a/core/src/tree/retrieval/benchmarks.rs b/core/src/tree/retrieval/benchmarks.rs index f59338d..35ee2a9 100644 --- a/core/src/tree/retrieval/benchmarks.rs +++ b/core/src/tree/retrieval/benchmarks.rs @@ -25,12 +25,12 @@ use tempfile::TempDir; #[cfg(test)] use tinymemory_api::host::test_support::TestHostConfig; +use crate::engine::backend::ingest::canonicalize::chat::{ChatBatch, ChatMessage}; use crate::ingest_pipeline::ingest_chat; use crate::queue::testing::drain_until_idle; use crate::store::chunks::types::SourceKind; use crate::tree::retrieval::{fetch_leaves, query_source, search_entities}; use crate::Config; -use tinycortex::memory::ingest::canonicalize::chat::{ChatBatch, ChatMessage}; /// Shared test config — disables embedding for deterministic inert behaviour. fn bench_config() -> (TempDir, TestHostConfig) { diff --git a/core/src/tree/retrieval/cover.rs b/core/src/tree/retrieval/cover.rs index d5b7b09..2f196b8 100644 --- a/core/src/tree/retrieval/cover.rs +++ b/core/src/tree/retrieval/cover.rs @@ -1,8 +1,8 @@ use anyhow::Result; +use crate::engine::engine_config; use crate::source_scope::current_source_scope; use crate::store::chunks::types::SourceKind; -use crate::tinycortex::engine_config; use crate::tree::retrieval::types::QueryResponse; use crate::Config; @@ -62,7 +62,7 @@ pub async fn cover_window_scoped( source_kind.map(|k| k.as_str()), limit ); - let mut response = tinycortex::memory::retrieval::cover_window_scoped( + let mut response = crate::engine::backend::retrieval::cover_window_scoped( &engine_config(config), since_ms, until_ms, diff --git a/core/src/tree/retrieval/drill_down.rs b/core/src/tree/retrieval/drill_down.rs index d74f428..7e58af9 100644 --- a/core/src/tree/retrieval/drill_down.rs +++ b/core/src/tree/retrieval/drill_down.rs @@ -1,7 +1,7 @@ use anyhow::Result; +use crate::engine::engine_config; use crate::source_scope::current_source_scope; -use crate::tinycortex::engine_config; use crate::tree::retrieval::engine::EmbedderBridge; use crate::tree::retrieval::types::RetrievalHit; use crate::tree::score::embed::{build_embedder_from_config, InertEmbedder}; @@ -60,7 +60,7 @@ pub async fn drill_down_scoped( // limiting before the retain below would cap the result set with rows that // are about to be discarded. let engine_limit = scope.as_ref().map(|_| None).unwrap_or(limit); - let mut hits = tinycortex::memory::retrieval::drill_down( + let mut hits = crate::engine::backend::retrieval::drill_down( &engine_config(config), node_id, max_depth, diff --git a/core/src/tree/retrieval/engine.rs b/core/src/tree/retrieval/engine.rs index a23bcfe..5c541c5 100644 --- a/core/src/tree/retrieval/engine.rs +++ b/core/src/tree/retrieval/engine.rs @@ -6,7 +6,7 @@ use crate::tree::score::embed::Embedder as HostEmbedder; pub(super) struct EmbedderBridge<'a>(pub &'a dyn HostEmbedder); #[async_trait] -impl tinycortex::memory::score::embed::Embedder for EmbedderBridge<'_> { +impl crate::engine::backend::score::embed::Embedder for EmbedderBridge<'_> { fn name(&self) -> &'static str { self.0.name() } diff --git a/core/src/tree/retrieval/fast.rs b/core/src/tree/retrieval/fast.rs index 926e8a6..f8df280 100644 --- a/core/src/tree/retrieval/fast.rs +++ b/core/src/tree/retrieval/fast.rs @@ -2,15 +2,15 @@ use anyhow::Result; +use crate::engine::engine_config; use crate::source_scope::current_source_scope; -use crate::tinycortex::engine_config; use crate::tree::nlp; use crate::tree::retrieval::engine::EmbedderBridge; use crate::tree::retrieval::types::QueryResponse; use crate::tree::score::embed::build_embedder_from_config; use crate::Config; -pub use tinycortex::memory::retrieval::FastRetrieveOptions; +pub use crate::engine::backend::retrieval::FastRetrieveOptions; /// Deterministic graph-walk retrieval using the **ambient** source scope. /// @@ -49,7 +49,7 @@ pub async fn fast_retrieve_scoped( options.max_hops ); let embedder = build_embedder_from_config(config)?; - tinycortex::memory::retrieval::fast_retrieve( + crate::engine::backend::retrieval::fast_retrieve( &engine_config(config), query, &entity_ids, diff --git a/core/src/tree/retrieval/fetch.rs b/core/src/tree/retrieval/fetch.rs index d45c4dd..5fc2502 100644 --- a/core/src/tree/retrieval/fetch.rs +++ b/core/src/tree/retrieval/fetch.rs @@ -1,13 +1,13 @@ use anyhow::Result; +use crate::engine::engine_config; use crate::source_scope::chunk_source_allowed_in; use crate::source_scope::current_source_scope; use crate::store::chunks::store::get_chunks_batch; -use crate::tinycortex::engine_config; use crate::tree::retrieval::types::RetrievalHit; use crate::Config; -pub use tinycortex::memory::retrieval::MAX_BATCH; +pub use crate::engine::backend::retrieval::MAX_BATCH; /// Fetch leaf chunks by id, using the **ambient** scope. /// @@ -47,5 +47,5 @@ pub async fn fetch_leaves_scoped( } else { chunk_ids.to_vec() }; - tinycortex::memory::retrieval::fetch_leaves(&engine_config(config), &permitted_ids) + crate::engine::backend::retrieval::fetch_leaves(&engine_config(config), &permitted_ids) } diff --git a/core/src/tree/retrieval/integration_tests.rs b/core/src/tree/retrieval/integration_tests.rs index 9e41c8f..f449aa8 100644 --- a/core/src/tree/retrieval/integration_tests.rs +++ b/core/src/tree/retrieval/integration_tests.rs @@ -18,10 +18,10 @@ use tinymemory_api::host::MemoryHostConfig; #[cfg(test)] use tinymemory_api::host::test_support::TestHostConfig; +use crate::engine::backend::ingest::canonicalize::chat::{ChatBatch, ChatMessage}; use crate::ingest_pipeline::ingest_chat; use crate::store::chunks::types::SourceKind; use crate::tree::retrieval::{drill_down, fetch_leaves, query_source, search_entities}; -use tinycortex::memory::ingest::canonicalize::chat::{ChatBatch, ChatMessage}; fn test_config() -> (TempDir, TestHostConfig) { crate::test_seams::init(); diff --git a/core/src/tree/retrieval/search.rs b/core/src/tree/retrieval/search.rs index 9b379d8..2f0c69e 100644 --- a/core/src/tree/retrieval/search.rs +++ b/core/src/tree/retrieval/search.rs @@ -1,6 +1,6 @@ use anyhow::Result; -use crate::tinycortex::engine_config; +use crate::engine::engine_config; use crate::tree::retrieval::types::EntityMatch; use crate::tree::score::extract::EntityKind; use crate::Config; @@ -17,7 +17,7 @@ pub async fn search_entities( kinds.as_ref().map_or(0, Vec::len), limit ); - tinycortex::memory::retrieval::search_entities( + crate::engine::backend::retrieval::search_entities( &engine_config(config), query, kinds.as_deref(), diff --git a/core/src/tree/retrieval/source.rs b/core/src/tree/retrieval/source.rs index dbc0f62..95145a5 100644 --- a/core/src/tree/retrieval/source.rs +++ b/core/src/tree/retrieval/source.rs @@ -1,8 +1,8 @@ use anyhow::Result; +use crate::engine::engine_config; use crate::source_scope::current_source_scope; use crate::store::chunks::types::SourceKind; -use crate::tinycortex::engine_config; use crate::tree::retrieval::engine::EmbedderBridge; use crate::tree::retrieval::types::QueryResponse; use crate::tree::score::embed::build_embedder_from_config; @@ -90,7 +90,7 @@ pub async fn query_source_scoped( let mut response = if let Some(query) = semantic_query { let embedder = build_embedder_from_config(config)?; let bridge = EmbedderBridge(embedder.as_ref()); - tinycortex::memory::retrieval::query_source( + crate::engine::backend::retrieval::query_source( &engine_config(config), source_id, source_kind, @@ -101,13 +101,13 @@ pub async fn query_source_scoped( ) .await? } else { - tinycortex::memory::retrieval::query_source( + crate::engine::backend::retrieval::query_source( &engine_config(config), source_id, source_kind, time_window_days, None, - &tinycortex::memory::score::embed::InertEmbedder::new(), + &crate::engine::backend::score::embed::InertEmbedder::new(), usize::MAX, ) .await? diff --git a/core/src/tree/retrieval/source_scope_tests.rs b/core/src/tree/retrieval/source_scope_tests.rs index a36c4ea..92fd07c 100644 --- a/core/src/tree/retrieval/source_scope_tests.rs +++ b/core/src/tree/retrieval/source_scope_tests.rs @@ -15,7 +15,7 @@ //! (`tinycortex` `retrieval::{fetch,drill_down}`), so on leaves this is //! strictly narrower than predicate 1. `source.rs` / `cover.rs` additionally //! carry a *pre-filter* short circuit on the explicit `source_id` argument. -//! 3. `tinycortex::memory::chunks::store_list::append_source_scope` — a SQL +//! 3. `crate::engine::backend::chunks::store_list::append_source_scope` — a SQL //! predicate applied BEFORE `LIMIT`. Reached via `cover_window_scoped` and //! the raw `list_chunks` callers. It admits `mem_src:src-abc:` (empty item //! id), diverging from predicate 1. diff --git a/core/src/tree/retrieval/types.rs b/core/src/tree/retrieval/types.rs index 91b48fa..83e866e 100644 --- a/core/src/tree/retrieval/types.rs +++ b/core/src/tree/retrieval/types.rs @@ -1,6 +1,6 @@ //! Stable host path for tinycortex-owned retrieval wire types and converters. -pub use tinycortex::memory::retrieval::{ +pub use crate::engine::backend::retrieval::{ hit_from_chunk, hit_from_summary, hit_from_summary_with_tree, leaf_tree_placeholder, EntityMatch, NodeKind, QueryResponse, RetrievalHit, }; diff --git a/core/src/tree/score/extract/mod.rs b/core/src/tree/score/extract/mod.rs index 4768ebf..adc9c5d 100644 --- a/core/src/tree/score/extract/mod.rs +++ b/core/src/tree/score/extract/mod.rs @@ -5,23 +5,21 @@ use std::sync::Arc; use crate::Config; use async_trait::async_trait; -pub use tinycortex::memory::score::extract::{ +pub use crate::engine::backend::score::extract::{ ChatPrompt, ChatProvider, CompositeExtractor, EntityExtractor, EntityKind, ExtractedEntities, ExtractedEntity, ExtractedTopic, LlmExtractorConfig, RegexEntityExtractor, }; pub mod regex { - pub use tinycortex::memory::score::extract::regex::extract; + pub use crate::engine::backend::score::extract::regex::extract; } -pub struct LlmEntityExtractor(tinycortex::memory::score::extract::LlmEntityExtractor); +pub struct LlmEntityExtractor(crate::engine::backend::score::extract::LlmEntityExtractor); impl LlmEntityExtractor { pub fn new(config: LlmExtractorConfig, provider: Arc) -> Self { - let provider = Arc::new(crate::tinycortex::SeamChatProvider::new(provider)); - Self(tinycortex::memory::score::extract::LlmEntityExtractor::new( - config, provider, - )) + let provider = Arc::new(crate::engine::SeamChatProvider::new(provider)); + Self(crate::engine::backend::score::extract::LlmEntityExtractor::new(config, provider)) } } diff --git a/core/src/tree/score/mod.rs b/core/src/tree/score/mod.rs index 136aeeb..240cf50 100644 --- a/core/src/tree/score/mod.rs +++ b/core/src/tree/score/mod.rs @@ -6,20 +6,20 @@ pub mod store; use std::sync::Arc; -pub use anyhow::Result; -pub use tinycortex::memory::score::{ +pub use crate::engine::backend::score::{ persist_score_tx, score_chunk, score_chunks, score_chunks_fast, ScoreResult, ScoringConfig, DEFAULT_DEFINITE_DROP, DEFAULT_DEFINITE_KEEP, DEFAULT_DROP_THRESHOLD, PRIORITY_BOOST, PRIORITY_TAG, }; -pub use tinycortex::memory::score::{resolver, signals}; +pub use crate::engine::backend::score::{resolver, signals}; +pub use anyhow::Result; /// Build crate scoring policy from product inference routing. pub fn scoring_config_from(config: &crate::Config) -> ScoringConfig { let (provider, model) = match crate::chat::build_chat_runtime(config) { Ok((provider, model)) => ( - Arc::new(crate::tinycortex::SeamChatProvider::new(provider)) - as Arc, + Arc::new(crate::engine::SeamChatProvider::new(provider)) + as Arc, model, ), Err(error) => { @@ -29,8 +29,8 @@ pub fn scoring_config_from(config: &crate::Config) -> ScoringConfig { return ScoringConfig::default_regex_only(); } }; - let extractor = tinycortex::memory::score::extract::LlmEntityExtractor::new( - tinycortex::memory::score::extract::LlmExtractorConfig { + let extractor = crate::engine::backend::score::extract::LlmEntityExtractor::new( + crate::engine::backend::score::extract::LlmExtractorConfig { model, output_language: config.output_language().map(str::to_string), ..Default::default() diff --git a/core/src/tree/score/store.rs b/core/src/tree/score/store.rs index 0a757e8..6f74abc 100644 --- a/core/src/tree/score/store.rs +++ b/core/src/tree/score/store.rs @@ -4,21 +4,21 @@ use std::collections::HashMap; use anyhow::Result; -use crate::tinycortex::engine_config; +use crate::engine::engine_config; use crate::Config; -pub use tinycortex::memory::score::store::{EntityHit, ScoreRow}; +pub use crate::engine::backend::score::store::{EntityHit, ScoreRow}; pub fn upsert_score(config: &Config, row: &ScoreRow) -> Result<()> { - tinycortex::memory::score::store::upsert_score(&engine_config(config), row) + crate::engine::backend::score::store::upsert_score(&engine_config(config), row) } pub fn get_score(config: &Config, chunk_id: &str) -> Result> { - tinycortex::memory::score::store::get_score(&engine_config(config), chunk_id) + crate::engine::backend::score::store::get_score(&engine_config(config), chunk_id) } pub fn get_scores_batch(config: &Config, chunk_ids: &[String]) -> Result> { - tinycortex::memory::score::store::get_scores_batch(&engine_config(config), chunk_ids) + crate::engine::backend::score::store::get_scores_batch(&engine_config(config), chunk_ids) } pub use crate::store::entities::{ @@ -27,7 +27,7 @@ pub use crate::store::entities::{ pub fn index_entity( config: &Config, - entity: &tinycortex::memory::score::resolver::CanonicalEntity, + entity: &crate::engine::backend::score::resolver::CanonicalEntity, node_id: &str, node_kind: &str, timestamp_ms: i64, @@ -39,13 +39,13 @@ pub fn index_entity( pub fn index_entities( config: &Config, - entities: &[tinycortex::memory::score::resolver::CanonicalEntity], + entities: &[crate::engine::backend::score::resolver::CanonicalEntity], node_id: &str, node_kind: &str, timestamp_ms: i64, tree_id: Option<&str>, ) -> Result { - let entities: Vec = entities + let entities: Vec = entities .iter() .map(to_store_entity) .collect::>()?; @@ -60,11 +60,11 @@ pub fn index_entities( } fn to_store_entity( - entity: &tinycortex::memory::score::resolver::CanonicalEntity, -) -> Result { - Ok(tinycortex::memory::store::CanonicalEntity { + entity: &crate::engine::backend::score::resolver::CanonicalEntity, +) -> Result { + Ok(crate::engine::backend::store::CanonicalEntity { canonical_id: entity.canonical_id.clone(), - kind: tinycortex::memory::store::EntityKind::parse(entity.kind.as_str()) + kind: crate::engine::backend::store::EntityKind::parse(entity.kind.as_str()) .map_err(anyhow::Error::msg)?, surface: entity.surface.clone(), span_start: entity.span_start, @@ -74,5 +74,5 @@ fn to_store_entity( } pub fn count_scores(config: &Config) -> Result { - tinycortex::memory::score::store::count_scores(&engine_config(config)) + crate::engine::backend::score::store::count_scores(&engine_config(config)) } diff --git a/core/src/tree/summarise.rs b/core/src/tree/summarise.rs index fc69fdd..0fe5db0 100644 --- a/core/src/tree/summarise.rs +++ b/core/src/tree/summarise.rs @@ -5,7 +5,7 @@ use anyhow::{Context, Result}; use crate::chat::{build_chat_provider, ChatPrompt}; use crate::Config; -pub use tinycortex::memory::tree::{SummaryContext, SummaryInput}; +pub use crate::engine::backend::tree::{SummaryContext, SummaryInput}; /// Compatibility result carrying provider usage alongside the crate-owned /// summary output fields. @@ -25,9 +25,11 @@ pub async fn summarise( inputs: &[SummaryInput], context: &SummaryContext<'_>, ) -> Result { - let Some(prepared) = - tinycortex::memory::tree::prepare_summary_prompt(inputs, context, config.output_language()) - else { + let Some(prepared) = crate::engine::backend::tree::prepare_summary_prompt( + inputs, + context, + config.output_language(), + ) else { return Ok(SummaryOutput::default()); }; let provider = @@ -50,7 +52,7 @@ pub async fn summarise( .await .with_context(|| format!("memory_tree::summarise: provider={}", provider.name()))?; let output = - tinycortex::memory::tree::finish_provider_summary(&text, prepared.effective_budget); + crate::engine::backend::tree::finish_provider_summary(&text, prepared.effective_budget); let input_tokens = usage.as_ref().map_or(0, |usage| usage.input_tokens); let output_tokens = usage.as_ref().map_or(0, |usage| usage.output_tokens); let charged_amount_usd = usage @@ -75,7 +77,7 @@ pub async fn summarise( } pub fn fallback_summary(inputs: &[SummaryInput], budget: u32) -> SummaryOutput { - let output = tinycortex::memory::tree::fallback_summary(inputs, budget); + let output = crate::engine::backend::tree::fallback_summary(inputs, budget); SummaryOutput { content: output.content, token_count: output.token_count, diff --git a/core/src/tree/tree/bucket_seal.rs b/core/src/tree/tree/bucket_seal.rs index 6903832..ca2fd63 100644 --- a/core/src/tree/tree/bucket_seal.rs +++ b/core/src/tree/tree/bucket_seal.rs @@ -3,11 +3,11 @@ use anyhow::Result; use chrono::{DateTime, Utc}; +use crate::engine::engine_config; use crate::store::trees::types::{Buffer, Tree}; -use crate::tinycortex::engine_config; use crate::Config; -pub use tinycortex::memory::tree::{LabelStrategy, LeafRef, MERGE_LEVEL_BASE}; +pub use crate::engine::backend::tree::{LabelStrategy, LeafRef, MERGE_LEVEL_BASE}; pub async fn append_leaf( config: &Config, @@ -23,11 +23,11 @@ pub async fn append_leaf( leaf.token_count as i64, leaf.timestamp, )?; - crate::tinycortex::cascade_tree(config, tree, 0, false, strategy).await + crate::engine::cascade_tree(config, tree, 0, false, strategy).await } pub fn append_leaf_deferred(config: &Config, tree: &Tree, leaf: &LeafRef) -> Result { - tinycortex::memory::tree::append_leaf_deferred(&engine_config(config), tree, leaf) + crate::engine::backend::tree::append_leaf_deferred(&engine_config(config), tree, leaf) } pub fn append_to_buffer( @@ -38,7 +38,7 @@ pub fn append_to_buffer( token_delta: i64, item_ts: DateTime, ) -> Result<()> { - tinycortex::memory::tree::append_to_buffer( + crate::engine::backend::tree::append_to_buffer( &engine_config(config), tree_id, level, @@ -55,7 +55,7 @@ pub async fn cascade_all_from( force_now: Option>, strategy: &LabelStrategy, ) -> Result> { - crate::tinycortex::cascade_tree(config, tree, start_level, force_now.is_some(), strategy).await + crate::engine::cascade_tree(config, tree, start_level, force_now.is_some(), strategy).await } pub async fn seal_document_subtree( @@ -66,7 +66,7 @@ pub async fn seal_document_subtree( chunk_ids: &[String], strategy: &LabelStrategy, ) -> Result { - crate::tinycortex::seal_document_subtree(config, tree, doc_id, version_ms, chunk_ids, strategy) + crate::engine::seal_document_subtree(config, tree, doc_id, version_ms, chunk_ids, strategy) .await } @@ -77,5 +77,5 @@ pub async fn seal_one_level( strategy: &LabelStrategy, enqueue_follow_ups: bool, ) -> Result { - crate::tinycortex::seal_tree_level(config, tree, buffer, strategy, enqueue_follow_ups).await + crate::engine::seal_tree_level(config, tree, buffer, strategy, enqueue_follow_ups).await } diff --git a/core/src/tree/tree/factory.rs b/core/src/tree/tree/factory.rs index fbb738e..2ff6414 100644 --- a/core/src/tree/tree/factory.rs +++ b/core/src/tree/tree/factory.rs @@ -21,36 +21,36 @@ use crate::tree::tree::flush::force_flush_tree; use crate::tree::tree::registry::get_or_create_tree; use crate::Config; -pub use tinycortex::memory::tree::{TreeProfile, GLOBAL_SCOPE}; +pub use crate::engine::backend::tree::{TreeProfile, GLOBAL_SCOPE}; /// Factory/config object for one tree instance. #[derive(Debug, Clone)] pub struct TreeFactory<'a> { - inner: tinycortex::memory::tree::TreeFactory<'a>, + inner: crate::engine::backend::tree::TreeFactory<'a>, } impl<'a> TreeFactory<'a> { pub fn source(scope: impl Into>) -> Self { Self { - inner: tinycortex::memory::tree::TreeFactory::source(scope), + inner: crate::engine::backend::tree::TreeFactory::source(scope), } } pub fn topic(scope: impl Into>) -> Self { Self { - inner: tinycortex::memory::tree::TreeFactory::topic(scope), + inner: crate::engine::backend::tree::TreeFactory::topic(scope), } } pub fn global() -> Self { Self { - inner: tinycortex::memory::tree::TreeFactory::global(), + inner: crate::engine::backend::tree::TreeFactory::global(), } } pub fn from_tree(tree: &'a Tree) -> Self { Self { - inner: tinycortex::memory::tree::TreeFactory::from_tree(tree), + inner: crate::engine::backend::tree::TreeFactory::from_tree(tree), } } diff --git a/core/src/tree/tree/flush.rs b/core/src/tree/tree/flush.rs index 9c781e5..df74586 100644 --- a/core/src/tree/tree/flush.rs +++ b/core/src/tree/tree/flush.rs @@ -12,7 +12,7 @@ pub async fn flush_stale_buffers( max_age: Duration, strategy: &LabelStrategy, ) -> Result { - crate::tinycortex::flush_stale_tree_buffers(config, max_age, strategy).await + crate::engine::flush_stale_tree_buffers(config, max_age, strategy).await } pub async fn flush_stale_buffers_default( diff --git a/core/src/tree/tree_runtime/engine.rs b/core/src/tree/tree_runtime/engine.rs index ca3ec68..a67ee26 100644 --- a/core/src/tree/tree_runtime/engine.rs +++ b/core/src/tree/tree_runtime/engine.rs @@ -2,16 +2,16 @@ use std::sync::Arc; +use crate::engine::backend::tree::runtime::{ + NodeLevel, RuntimeObserver, Summariser, TreeNode, TreeStatus, +}; use anyhow::{Context, Result}; use async_trait::async_trait; use chrono::{DateTime, Timelike, Utc}; use tinyagents::harness::message::Message; use tinyagents::harness::model::{ChatModel, ModelRequest}; -use tinycortex::memory::tree::runtime::{ - NodeLevel, RuntimeObserver, Summariser, TreeNode, TreeStatus, -}; -use crate::tinycortex::engine_config; +use crate::engine::engine_config; use crate::Config; const SUMMARIZATION_TEMP: f64 = 0.3; @@ -83,7 +83,7 @@ pub async fn run_summarization( ts: DateTime, ) -> Result> { log::debug!("[tree_summarizer] tinycortex run namespace={namespace}"); - let result = tinycortex::memory::tree::runtime::run_summarization_observed( + let result = crate::engine::backend::tree::runtime::run_summarization_observed( &engine_config(config), &ChatSummariser(provider), namespace, @@ -105,7 +105,7 @@ pub async fn rebuild_tree( namespace: &str, ) -> Result { log::debug!("[tree_summarizer] tinycortex rebuild namespace={namespace}"); - tinycortex::memory::tree::runtime::rebuild_tree_observed( + crate::engine::backend::tree::runtime::rebuild_tree_observed( &engine_config(config), &ChatSummariser(provider), namespace, @@ -134,8 +134,9 @@ pub async fn run_hourly_loop(config: Arc, provider: Arc PathBuf { - tinycortex::memory::tree::runtime::store::tree_dir(&engine_config(config), namespace) + crate::engine::backend::tree::runtime::store::tree_dir(&engine_config(config), namespace) } pub fn buffer_dir(config: &Config, namespace: &str) -> PathBuf { - tinycortex::memory::tree::runtime::store::buffer_dir(&engine_config(config), namespace) + crate::engine::backend::tree::runtime::store::buffer_dir(&engine_config(config), namespace) } pub fn node_file_path(config: &Config, namespace: &str, node_id: &str) -> PathBuf { - tinycortex::memory::tree::runtime::store::node_file_path( + crate::engine::backend::tree::runtime::store::node_file_path( &engine_config(config), namespace, node_id, ) } -pub use tinycortex::memory::tree::runtime::store::{validate_namespace, validate_node_id}; +pub use crate::engine::backend::tree::runtime::store::{validate_namespace, validate_node_id}; pub fn write_node(config: &Config, node: &TreeNode) -> Result<()> { - tinycortex::memory::tree::runtime::store::write_node(&engine_config(config), node) + crate::engine::backend::tree::runtime::store::write_node(&engine_config(config), node) } pub fn read_node(config: &Config, namespace: &str, node_id: &str) -> Result> { - tinycortex::memory::tree::runtime::store::read_node(&engine_config(config), namespace, node_id) + crate::engine::backend::tree::runtime::store::read_node( + &engine_config(config), + namespace, + node_id, + ) } pub fn read_children(config: &Config, namespace: &str, parent_id: &str) -> Result> { - tinycortex::memory::tree::runtime::store::read_children( + crate::engine::backend::tree::runtime::store::read_children( &engine_config(config), namespace, parent_id, @@ -45,7 +49,7 @@ pub fn read_children(config: &Config, namespace: &str, parent_id: &str) -> Resul } pub fn read_ancestors(config: &Config, namespace: &str, node_id: &str) -> Result> { - tinycortex::memory::tree::runtime::store::read_ancestors( + crate::engine::backend::tree::runtime::store::read_ancestors( &engine_config(config), namespace, node_id, @@ -53,11 +57,11 @@ pub fn read_ancestors(config: &Config, namespace: &str, node_id: &str) -> Result } pub fn count_nodes(config: &Config, namespace: &str) -> Result { - tinycortex::memory::tree::runtime::store::count_nodes(&engine_config(config), namespace) + crate::engine::backend::tree::runtime::store::count_nodes(&engine_config(config), namespace) } pub fn get_tree_status(config: &Config, namespace: &str) -> Result { - tinycortex::memory::tree::runtime::store::get_tree_status(&engine_config(config), namespace) + crate::engine::backend::tree::runtime::store::get_tree_status(&engine_config(config), namespace) } pub fn collect_root_summaries_with_caps( @@ -65,7 +69,7 @@ pub fn collect_root_summaries_with_caps( per_namespace_cap: usize, total_cap: usize, ) -> Vec<(String, String, DateTime)> { - tinycortex::memory::tree::runtime::store::collect_root_summaries_with_caps( + crate::engine::backend::tree::runtime::store::collect_root_summaries_with_caps( workspace_dir, per_namespace_cap, total_cap, @@ -73,11 +77,11 @@ pub fn collect_root_summaries_with_caps( } pub fn list_namespaces_with_root(config: &Config) -> Result> { - tinycortex::memory::tree::runtime::store::list_namespaces_with_root(&engine_config(config)) + crate::engine::backend::tree::runtime::store::list_namespaces_with_root(&engine_config(config)) } pub fn delete_tree(config: &Config, namespace: &str) -> Result { - tinycortex::memory::tree::runtime::store::delete_tree(&engine_config(config), namespace) + crate::engine::backend::tree::runtime::store::delete_tree(&engine_config(config), namespace) } pub fn buffer_write( @@ -87,7 +91,7 @@ pub fn buffer_write( ts: &DateTime, metadata: Option<&Value>, ) -> Result { - tinycortex::memory::tree::runtime::store::buffer_write( + crate::engine::backend::tree::runtime::store::buffer_write( &engine_config(config), namespace, content, @@ -97,11 +101,11 @@ pub fn buffer_write( } pub fn buffer_read(config: &Config, namespace: &str) -> Result> { - tinycortex::memory::tree::runtime::store::buffer_read(&engine_config(config), namespace) + crate::engine::backend::tree::runtime::store::buffer_read(&engine_config(config), namespace) } pub fn buffer_delete(config: &Config, namespace: &str, filenames: &[String]) -> Result<()> { - tinycortex::memory::tree::runtime::store::buffer_delete( + crate::engine::backend::tree::runtime::store::buffer_delete( &engine_config(config), namespace, filenames, @@ -109,9 +113,9 @@ pub fn buffer_delete(config: &Config, namespace: &str, filenames: &[String]) -> } pub fn buffer_drain(config: &Config, namespace: &str) -> Result> { - tinycortex::memory::tree::runtime::store::buffer_drain(&engine_config(config), namespace) + crate::engine::backend::tree::runtime::store::buffer_drain(&engine_config(config), namespace) } pub fn parse_node_markdown_pub(raw: &str, namespace: &str, node_id: &str) -> Result { - tinycortex::memory::tree::runtime::store::parse_node_markdown_pub(raw, namespace, node_id) + crate::engine::backend::tree::runtime::store::parse_node_markdown_pub(raw, namespace, node_id) } diff --git a/crates/tinymemory-module/Cargo.lock b/crates/tinymemory-module/Cargo.lock index e5599eb..6b52126 100644 --- a/crates/tinymemory-module/Cargo.lock +++ b/crates/tinymemory-module/Cargo.lock @@ -1935,6 +1935,7 @@ dependencies = [ "serde_json", "sha2 0.10.9", "thiserror 2.0.20", + "tinymemory-api", "uuid", ] diff --git a/crates/tinymemory-module/Cargo.toml b/crates/tinymemory-module/Cargo.toml index adbc845..26e2c18 100644 --- a/crates/tinymemory-module/Cargo.toml +++ b/crates/tinymemory-module/Cargo.toml @@ -79,6 +79,14 @@ tempfile = "3" # which is unusual but correct: these are the same submodules the parent builds # against, and pointing somewhere else would compile the module against a # different engine than the workspace it ships from. +# `tinycortex-api` takes the contract by git (tinymemory#18 §A1). This crate is +# its own workspace root, and patch tables only apply from the root being built, +# so the entry has to be repeated here — without it cargo resolves the git copy +# alongside the path copy and `MemoryTaint` from one is not the same type as +# from the other. +[patch."https://github.com/tinyhumansai/tinymemory"] +tinymemory-api = { path = "../../api" } + [patch.crates-io] tinycortex = { path = "../../vendor/tinycortex" } tinycortex-api = { path = "../../vendor/tinycortex/api" } From 474904e98fb73d7da2596cfafb53364f6316ff13 Mon Sep 17 00:00:00 2001 From: Shanu Date: Mon, 17 Aug 2026 20:18:48 +0530 Subject: [PATCH 5/9] Patch the contract's git dependency in the module crate too MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The module crate is its own workspace root, and a patch table applies only from the root being built — so the parent workspace's entry does not reach it. Without its own, cargo resolves the git copy of `tinymemory-api` alongside the path copy and `MemoryTaint` from one is not the same type as from the other. Belongs in this commit rather than a later one: this is where the git dependency arrives, so this is where its consequence has to be handled. CI caught it — the module lane failed with eight type mismatches while every other lane was green, because it is the only job that builds from that root. Refs #18 (§A1) --- crates/tinymemory-module/Cargo.lock | 1 + crates/tinymemory-module/Cargo.toml | 8 ++++++++ 2 files changed, 9 insertions(+) diff --git a/crates/tinymemory-module/Cargo.lock b/crates/tinymemory-module/Cargo.lock index e5599eb..6b52126 100644 --- a/crates/tinymemory-module/Cargo.lock +++ b/crates/tinymemory-module/Cargo.lock @@ -1935,6 +1935,7 @@ dependencies = [ "serde_json", "sha2 0.10.9", "thiserror 2.0.20", + "tinymemory-api", "uuid", ] diff --git a/crates/tinymemory-module/Cargo.toml b/crates/tinymemory-module/Cargo.toml index adbc845..ced3f1e 100644 --- a/crates/tinymemory-module/Cargo.toml +++ b/crates/tinymemory-module/Cargo.toml @@ -79,6 +79,14 @@ tempfile = "3" # which is unusual but correct: these are the same submodules the parent builds # against, and pointing somewhere else would compile the module against a # different engine than the workspace it ships from. +# `tinycortex-api` takes the contract by git (issue #18 §A1). This crate is its +# own workspace root, and a patch table only applies from the root being built, +# so the entry has to be repeated here — the parent workspace's identical entry +# does not reach it. Without this, cargo resolves the git copy alongside the +# path copy and `MemoryTaint` from one is not the same type as from the other. +[patch."https://github.com/tinyhumansai/tinymemory"] +tinymemory-api = { path = "../../api" } + [patch.crates-io] tinycortex = { path = "../../vendor/tinycortex" } tinycortex-api = { path = "../../vendor/tinycortex/api" } From 76f052ae819f9306bd00b4658d0d267758e70dd5 Mon Sep 17 00:00:00 2001 From: Shanu Date: Tue, 18 Aug 2026 12:12:35 +0530 Subject: [PATCH 6/9] Run the conformance suite against the three hosted adapters MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Issue #18's acceptance criterion 5. The suite ran against the in-memory reference driver and the null driver — both written alongside it, so passing proved the assertions were self-consistent and not much else. The premise the whole issue rests on, that an engine other than TinyCortex can satisfy the contract, had never been exercised. Each adapter now runs the full `assert_provider` over a real TCP socket against a double that speaks its own HTTP shapes and retains what it is sent. That retention is the point: `failure_test`'s doubles only have to misbehave, while these have to work, because the suite writes and reads back. All three pass — eleven assertions each, including taint preservation, upsert identity, namespace isolation, and export/import round trip. Each adapter is paired with a second test asserting its double genuinely retains, and that pairing earned itself immediately. `cognee_upholds_the_ contract` passed while `the_cognee_double_actually_retains` failed: the suite returns early when a driver does not retain, so it had run four assertions and skipped the seven that matter, and reported success. Without the probe this would have been reported as "Cognee passes". The cause was the double, not the adapter. Cognee's data listing has to carry a `name` ending `.tinymemory[.json]` — the adapter skips anything else, because Cognee's own text loader strips the extension — and the listing returned only `id`, so every record was filtered out before the fetch. What this proves is narrower than "the hosted engines uphold the contract", and the module docs say so: nobody here can prove that about someone else's service. It is that *the adapter* does, given a backend answering its own documented shapes. A violation on the adapter's side of the wire — a dropped taint, a non-terminating export cursor, an upsert that duplicates — is caught. `retains_writes` is exported from the conformance crate for this: a caller standing up its own backend double needs it, for exactly the reason above. Not covered here: the TinyCortex adapter. It needs `require_embedding_host()`, a process-global, so driving it means installing host seams — which makes the test order-dependent unless it is isolated in its own target. Criterion 5 names it alongside the three, so it remains open. Refs #18 (§E1, acceptance criterion 5) --- Cargo.lock | 1 + adapters/remote/Cargo.toml | 3 + adapters/remote/src/conformance_test.rs | 497 ++++++++++++++++++++++++ adapters/remote/src/lib.rs | 3 + conformance/src/lib.rs | 8 + 5 files changed, 512 insertions(+) create mode 100644 adapters/remote/src/conformance_test.rs diff --git a/Cargo.lock b/Cargo.lock index dd6b02f..8bd1b2c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1887,6 +1887,7 @@ dependencies = [ "sha2 0.10.9", "tinymemory", "tinymemory-api", + "tinymemory-conformance", "tokio", ] diff --git a/adapters/remote/Cargo.toml b/adapters/remote/Cargo.toml index d78ff7e..3bb4da5 100644 --- a/adapters/remote/Cargo.toml +++ b/adapters/remote/Cargo.toml @@ -25,6 +25,9 @@ serde_json = "1" sha2 = "0.10" [dev-dependencies] +# The behavioural contract suite, run against these adapters over their own +# native doubles (issue #18 §E1, acceptance criterion 5). +tinymemory-conformance = { path = "../../conformance" } # Adapter tests run lightweight native-API doubles over a real TCP transport. axum = { version = "0.8", features = ["multipart"] } tokio = { version = "1", features = ["macros", "rt-multi-thread", "net"] } diff --git a/adapters/remote/src/conformance_test.rs b/adapters/remote/src/conformance_test.rs new file mode 100644 index 0000000..3694b63 --- /dev/null +++ b/adapters/remote/src/conformance_test.rs @@ -0,0 +1,497 @@ +//! The conformance suite, run against the hosted adapters. +//! +//! Issue #18's acceptance criterion 5: "the conformance suite passes for +//! TinyCortex and all three remote adapters". Until now it ran against the +//! in-memory reference driver and the null driver — both written alongside the +//! suite, so passing proved the assertions were self-consistent and little +//! else. +//! +//! These run the same `assert_provider` against the real adapters, over a real +//! TCP socket, against a double that speaks each vendor's own HTTP shapes and +//! **actually retains what it is sent**. That is the difference from +//! `failure_test`, whose doubles only need to misbehave: here the double has to +//! be a working backend, because the suite writes and reads back. +//! +//! What this proves is narrow and worth stating precisely. It is not that +//! Supermemory, Mem0 or Cognee uphold the contract — nobody here can prove that +//! about someone else's service. It is that **the adapter** does, given a +//! backend that answers its own documented shapes. A contract violation on the +//! adapter's side of the wire — a dropped taint, an export cursor that never +//! terminates, an upsert that duplicates — is caught here. + +#![allow(clippy::expect_used, clippy::panic)] + +use std::collections::BTreeMap; +use std::sync::{Arc, Mutex}; + +use axum::extract::{Path, Query, State}; +use axum::routing::{delete, get, post, put}; +use axum::{Json, Router}; +use serde_json::{json, Value}; + +use crate::{mem0_provider, Mem0Memory}; + +/// A record as one of the vendor doubles holds it. +#[derive(Clone, Debug)] +struct Row { + id: String, + content: String, + metadata: Value, +} + +/// The doubles' shared store: `id -> Row`, plus a counter for fresh ids. +#[derive(Default, Debug)] +struct Backend { + rows: BTreeMap, + next: usize, +} + +impl Backend { + fn fresh_id(&mut self) -> String { + self.next += 1; + format!("rec-{}", self.next) + } +} + +type Store = Arc>; + +/// Serves `app` on an ephemeral port and returns its base URL. +async fn serve(app: Router) -> String { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("bind"); + let endpoint = format!("http://{}", listener.local_addr().expect("address")); + tokio::spawn(async move { + axum::serve(listener, app).await.expect("serve"); + }); + endpoint +} + +// ── Mem0's native shapes ───────────────────────────────────────────────────── +// +// Five routes, matching what `Mem0Dialect` issues: list, create, update, +// delete, search. The response envelopes (`results`, `memory`, `metadata`) are +// the ones its `decode` reads, so a shape drift on either side fails here +// rather than silently returning nothing. + +async fn mem0_list(State(store): State) -> Json { + let store = store.lock().expect("store lock"); + let results: Vec = store + .rows + .values() + .map(|r| { + json!({ + "id": r.id, + "memory": r.content, + "metadata": r.metadata, + "created_at": "1970-01-01T00:00:00Z", + }) + }) + .collect(); + Json(json!({ "results": results })) +} + +async fn mem0_create(State(store): State, Json(body): Json) -> Json { + let mut store = store.lock().expect("store lock"); + let id = store.fresh_id(); + let content = body["messages"][0]["content"] + .as_str() + .unwrap_or_default() + .to_owned(); + let metadata = body["metadata"].clone(); + store.rows.insert( + id.clone(), + Row { + id: id.clone(), + content, + metadata, + }, + ); + Json(json!({ "results": [{ "id": id }] })) +} + +async fn mem0_update( + State(store): State, + Path(id): Path, + Json(body): Json, +) -> Json { + let mut store = store.lock().expect("store lock"); + if let Some(row) = store.rows.get_mut(&id) { + if let Some(text) = body["text"].as_str() { + row.content = text.to_owned(); + } + if !body["metadata"].is_null() { + row.metadata = body["metadata"].clone(); + } + } + Json(json!({ "id": id })) +} + +async fn mem0_delete(State(store): State, Path(id): Path) -> Json { + store.lock().expect("store lock").rows.remove(&id); + Json(json!({ "deleted": true })) +} + +async fn mem0_search(State(store): State, Json(body): Json) -> Json { + // Substring matching is enough: the suite asserts that recall *narrows*, + // not that the backend ranks well. + let needle = body["query"].as_str().unwrap_or_default().to_lowercase(); + let limit = body["top_k"].as_u64().unwrap_or(100) as usize; + let store = store.lock().expect("store lock"); + let results: Vec = store + .rows + .values() + .filter(|r| r.content.to_lowercase().contains(&needle)) + .take(limit) + .map(|r| { + json!({ + "id": r.id, + "memory": r.content, + "metadata": r.metadata, + "score": 0.9, + }) + }) + .collect(); + Json(json!({ "results": results })) +} + +/// A Mem0 double that retains what it is sent. +async fn mem0_backend() -> String { + let store: Store = Arc::new(Mutex::new(Backend::default())); + let app = Router::new() + .route("/memories", get(mem0_list).post(mem0_create)) + .route("/memories/{id}", put(mem0_update).delete(mem0_delete)) + .route("/search", post(mem0_search)) + .with_state(store); + serve(app).await +} + +#[tokio::test] +async fn mem0_upholds_the_contract() { + let endpoint = mem0_backend().await; + let provider = mem0_provider(Mem0Memory::new(&endpoint, None).expect("client")); + tinymemory_conformance::assert_provider(Arc::new(provider)).await; +} + +/// The suite's write-path assertions only run when the driver retains, so a +/// double that silently dropped writes would let the whole run pass vacuously. +/// This pins that the Mem0 double is genuinely retaining. +#[tokio::test] +async fn the_mem0_double_actually_retains() { + let endpoint = mem0_backend().await; + let provider = mem0_provider(Mem0Memory::new(&endpoint, None).expect("client")); + assert!( + tinymemory_conformance::retains_writes(&provider).await, + "the Mem0 double must retain writes, or `assert_provider` skips every \ + assertion that matters and still reports success" + ); +} + +// ── Supermemory's native shapes ────────────────────────────────────────────── +// +// Container tags are Supermemory's namespace equivalent, and the adapter +// derives one per TinyMemory namespace. The double keeps a tag per row so the +// tag listing — which drives `entries()` — reflects what has actually been +// written, rather than a fixed set the adapter would then filter to nothing. + +/// The tag the adapter derives, as sent on create. +fn tag_of(row: &Row) -> String { + row.metadata + .get("tinymemory_namespace") + .and_then(Value::as_str) + .map(|ns| format!("tinymemory-{ns}")) + .unwrap_or_default() +} + +async fn sm_tags(State(store): State) -> Json { + let store = store.lock().expect("store lock"); + let mut tags: Vec = store.rows.values().map(tag_of).collect(); + tags.sort(); + tags.dedup(); + Json(Value::Array( + tags.into_iter() + .map(|t| json!({ "containerTag": t })) + .collect(), + )) +} + +async fn sm_list(State(store): State, Json(body): Json) -> Json { + // The adapter pages until a short page comes back, so a double that always + // returned a full page would spin. One page, then empty. + let page = body["page"].as_u64().unwrap_or(1); + let wanted = body["containerTags"][0].as_str().unwrap_or_default(); + let store = store.lock().expect("store lock"); + let entries: Vec = if page > 1 { + Vec::new() + } else { + store + .rows + .values() + .filter(|r| tag_of(r) == wanted) + .map(|r| { + json!({ + "id": r.id, + "content": r.content, + "metadata": r.metadata, + "createdAt": "1970-01-01T00:00:00Z", + "isLatest": true, + "isForgotten": false, + }) + }) + .collect() + }; + Json(json!({ "memoryEntries": entries })) +} + +async fn sm_create(State(store): State, Json(body): Json) -> Json { + let mut store = store.lock().expect("store lock"); + let id = store.fresh_id(); + let first = &body["memories"][0]; + store.rows.insert( + id.clone(), + Row { + id: id.clone(), + content: first["content"].as_str().unwrap_or_default().to_owned(), + metadata: first["metadata"].clone(), + }, + ); + Json(json!({ "memories": [{ "id": id }] })) +} + +async fn sm_update(State(store): State, Json(body): Json) -> Json { + let mut store = store.lock().expect("store lock"); + let id = body["id"].as_str().unwrap_or_default().to_owned(); + if let Some(row) = store.rows.get_mut(&id) { + if let Some(text) = body["newContent"].as_str() { + row.content = text.to_owned(); + } + if !body["metadata"].is_null() { + row.metadata = body["metadata"].clone(); + } + } + Json(json!({ "id": id })) +} + +async fn sm_delete(State(store): State, Json(body): Json) -> Json { + let id = body["id"].as_str().unwrap_or_default(); + store.lock().expect("store lock").rows.remove(id); + Json(json!({ "deleted": true })) +} + +async fn sm_search(State(store): State, Json(body): Json) -> Json { + let needle = body["q"] + .as_str() + .or_else(|| body["query"].as_str()) + .unwrap_or_default() + .to_lowercase(); + let limit = body["limit"].as_u64().unwrap_or(100) as usize; + let tag = body["containerTag"].as_str(); + let store = store.lock().expect("store lock"); + let results: Vec = store + .rows + .values() + .filter(|r| tag.is_none_or(|t| tag_of(r) == t)) + .filter(|r| r.content.to_lowercase().contains(&needle)) + .take(limit) + .map(|r| { + json!({ + "id": r.id, + "content": r.content, + "metadata": r.metadata, + "score": 0.9, + }) + }) + .collect(); + Json(json!({ "results": results })) +} + +async fn supermemory_backend() -> String { + let store: Store = Arc::new(Mutex::new(Backend::default())); + let app = Router::new() + .route("/v3/container-tags/list", get(sm_tags)) + .route("/v4/memories/list", post(sm_list)) + .route( + "/v4/memories", + post(sm_create).patch(sm_update).delete(sm_delete), + ) + .route("/v4/search", post(sm_search)) + .with_state(store); + serve(app).await +} + +#[tokio::test] +async fn supermemory_upholds_the_contract() { + let endpoint = supermemory_backend().await; + let provider = crate::supermemory_provider( + crate::SupermemoryMemory::new(&endpoint, None).expect("client"), + ); + tinymemory_conformance::assert_provider(Arc::new(provider)).await; +} + +#[tokio::test] +async fn the_supermemory_double_actually_retains() { + let endpoint = supermemory_backend().await; + let provider = crate::supermemory_provider( + crate::SupermemoryMemory::new(&endpoint, None).expect("client"), + ); + assert!( + tinymemory_conformance::retains_writes(&provider).await, + "the Supermemory double must retain writes, or the suite passes vacuously" + ); +} + +// ── Cognee's native shapes ─────────────────────────────────────────────────── +// +// The odd one out. Cognee has no per-record API: the adapter uploads each +// record as a JSON *file* into a per-namespace dataset, and reads it back +// through `/raw` — so the double stores the uploaded bytes verbatim and serves +// them unchanged. That is also why this double is the strictest of the three: +// the envelope it hands back is deserialised straight into `StoredEntry`, so a +// field the adapter fails to write is a parse failure here rather than a +// silently empty value. + +/// A dataset, keyed by the name the adapter derives from a namespace. +type Datasets = Arc>>>; + +async fn cg_datasets(State(sets): State) -> Json { + let sets = sets.lock().expect("store lock"); + Json(Value::Array( + sets.keys() + .map(|name| json!({ "id": name, "name": name })) + .collect(), + )) +} + +async fn cg_data(State(sets): State, Path(dataset): Path) -> Json { + let sets = sets.lock().expect("store lock"); + let ids: Vec = sets + .get(&dataset) + .map(|d| { + d.keys() + // `name` is required, and the adapter skips anything not + // ending `.tinymemory[.json]` — Cognee's own loader strips the + // extension, so both spellings are accepted. The data id here + // *is* the uploaded filename, which already carries it. + .map(|id| json!({ "id": id, "name": id })) + .collect() + }) + .unwrap_or_default(); + Json(Value::Array(ids)) +} + +async fn cg_raw( + State(sets): State, + Path((dataset, data_id)): Path<(String, String)>, +) -> String { + sets.lock() + .expect("store lock") + .get(&dataset) + .and_then(|d| d.get(&data_id)) + .cloned() + .unwrap_or_default() +} + +async fn cg_delete( + State(sets): State, + Path((dataset, data_id)): Path<(String, String)>, +) -> Json { + if let Some(d) = sets.lock().expect("store lock").get_mut(&dataset) { + d.remove(&data_id); + } + Json(json!({ "deleted": true })) +} + +/// Pulls the uploaded envelope and the dataset name out of a multipart body. +async fn multipart_parts(mut form: axum::extract::Multipart) -> (String, String, String) { + let (mut body, mut dataset, mut filename) = (String::new(), String::new(), String::new()); + while let Ok(Some(field)) = form.next_field().await { + match field.name().unwrap_or_default().to_owned().as_str() { + "datasetName" => dataset = field.text().await.unwrap_or_default(), + "data" | "file" | "files" => { + filename = field.file_name().unwrap_or_default().to_owned(); + body = field.text().await.unwrap_or_default(); + } + _ => { + let _ = field.bytes().await; + } + } + } + (body, dataset, filename) +} + +async fn cg_remember(State(sets): State, form: axum::extract::Multipart) -> Json { + let (body, dataset, filename) = multipart_parts(form).await; + let mut sets = sets.lock().expect("store lock"); + sets.entry(dataset).or_default().insert(filename, body); + Json(json!({ "status": "ok" })) +} + +async fn cg_update( + State(sets): State, + Query(q): Query>, + form: axum::extract::Multipart, +) -> Json { + let (body, _, _) = multipart_parts(form).await; + let dataset = q.get("dataset_id").cloned().unwrap_or_default(); + let data_id = q.get("data_id").cloned().unwrap_or_default(); + if let Some(d) = sets.lock().expect("store lock").get_mut(&dataset) { + d.insert(data_id, body); + } + Json(json!({ "status": "ok" })) +} + +async fn cg_recall(State(sets): State, Json(body): Json) -> Json { + let needle = body["query"].as_str().unwrap_or_default().to_lowercase(); + let limit = body["top_k"].as_u64().unwrap_or(100) as usize; + let wanted: Option> = body["datasets"].as_array().map(|a| { + a.iter() + .filter_map(|v| v.as_str().map(str::to_owned)) + .collect() + }); + let sets = sets.lock().expect("store lock"); + let hits: Vec = sets + .iter() + .filter(|(name, _)| wanted.as_ref().is_none_or(|w| w.contains(name))) + .flat_map(|(_, d)| d.values()) + .filter(|raw| raw.to_lowercase().contains(&needle)) + .take(limit) + .map(|raw| json!({ "text": raw })) + .collect(); + Json(json!({ "results": hits })) +} + +async fn cognee_backend() -> String { + let sets: Datasets = Arc::new(Mutex::new(BTreeMap::new())); + let app = Router::new() + .route("/api/v1/datasets", get(cg_datasets)) + .route("/api/v1/datasets/{dataset}/data", get(cg_data)) + .route("/api/v1/datasets/{dataset}/data/{data_id}/raw", get(cg_raw)) + .route( + "/api/v1/datasets/{dataset}/data/{data_id}", + delete(cg_delete), + ) + .route("/api/v1/remember", post(cg_remember)) + .route("/api/v1/update", axum::routing::patch(cg_update)) + .route("/api/v1/recall", post(cg_recall)) + .with_state(sets); + serve(app).await +} + +#[tokio::test] +async fn cognee_upholds_the_contract() { + let endpoint = cognee_backend().await; + let provider = + crate::cognee_provider(crate::CogneeMemory::self_hosted(&endpoint, None).expect("client")); + tinymemory_conformance::assert_provider(Arc::new(provider)).await; +} + +#[tokio::test] +async fn the_cognee_double_actually_retains() { + let endpoint = cognee_backend().await; + let provider = + crate::cognee_provider(crate::CogneeMemory::self_hosted(&endpoint, None).expect("client")); + assert!( + tinymemory_conformance::retains_writes(&provider).await, + "the Cognee double must retain writes, or the suite passes vacuously" + ); +} diff --git a/adapters/remote/src/lib.rs b/adapters/remote/src/lib.rs index dba72ed..1b9d559 100644 --- a/adapters/remote/src/lib.rs +++ b/adapters/remote/src/lib.rs @@ -41,3 +41,6 @@ pub fn cognee_provider(memory: CogneeMemory) -> MemoryTraitProvider { #[cfg(test)] mod failure_test; + +#[cfg(test)] +mod conformance_test; diff --git a/conformance/src/lib.rs b/conformance/src/lib.rs index 7bfca70..3e99c02 100644 --- a/conformance/src/lib.rs +++ b/conformance/src/lib.rs @@ -49,3 +49,11 @@ pub use suite::{ assert_store_get_round_trip, assert_taint_is_preserved, assert_upsert_replaces_rather_than_duplicates, }; +pub use suite::{ + // Exported alongside the assertions because a caller standing up its own + // backend double needs it: `assert_provider` skips every write-path + // assertion when the driver does not retain, so a double that silently + // dropped writes would let a whole run pass vacuously. Probing for that + // directly is how a caller proves its harness is real. + retains_writes, +}; From 9f1b80523157a81e8d7a9873a8f50ac410574d74 Mon Sep 17 00:00:00 2001 From: Shanu Date: Tue, 18 Aug 2026 12:21:46 +0530 Subject: [PATCH 7/9] Run the conformance suite against the TinyCortex driver MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Completes issue #18's acceptance criterion 5. With the three hosted adapters already covered, this is the last driver the criterion names. `crate::provider` needs only a `tinycortex::memory::Memory` backend, so the suite runs against the engine's own `InMemoryMemoryStore` with no host seams. That is also the sharper test: it is the engine's simplest backend, so anything the suite catches is the adapter's behaviour rather than the storage engine's. It failed on the first run, which is the point of running it: tinycortex: store of `empty` failed: memory content cannot be empty The reference driver, the null driver and all three hosted adapters accept empty content. TinyCortex refuses it. The contract settles which is right — `MemoryCore::store` documents `MemoryError::Invalid` "for caller input the driver rejects" — so refusing is conformant and the *suite* was over-asserting. It required every content shape to round trip, which the contract never promised. `assert_awkward_content_round_trips` now allows a driver to refuse a shape, and still requires that a shape it *accepts* comes back unmangled. A guard keeps that from becoming vacuous: a driver that refused all four shapes fails, because it would otherwise pass having stored nothing. That correction surfaced a second finding, left open deliberately. The refusal arrives as `MemoryError::Other`, not `Invalid`: DIAG variant = Other(memory content cannot be empty) The engine's typed error is flattened through `anyhow` before the mandatory composition sees it, so a validation refusal is indistinguishable from a backend failure. Recovering it would need downcasting or string matching, and the real fix is §A4 — one error type across the contract. The suite says so where the assertion is, so the tightening to require `Invalid` has an obvious home rather than being rediscovered. Not weakened to get green: the reference driver accepts empty content and is still held to round-tripping it faithfully, as are the three hosted adapters. Refs #18 (§E1, acceptance criterion 5) --- Cargo.lock | 1 + adapters/tinycortex/Cargo.toml | 3 ++ adapters/tinycortex/src/conformance_test.rs | 50 +++++++++++++++++++++ adapters/tinycortex/src/lib.rs | 3 ++ conformance/src/suite/mod.rs | 28 +++++++++++- 5 files changed, 83 insertions(+), 2 deletions(-) create mode 100644 adapters/tinycortex/src/conformance_test.rs diff --git a/Cargo.lock b/Cargo.lock index 8bd1b2c..5caf806 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1904,6 +1904,7 @@ dependencies = [ "tinycortex", "tinymemory", "tinymemory-api", + "tinymemory-conformance", "tinymemory-core", "tokio", "uuid", diff --git a/adapters/tinycortex/Cargo.toml b/adapters/tinycortex/Cargo.toml index 57f28a1..73ba1e7 100644 --- a/adapters/tinycortex/Cargo.toml +++ b/adapters/tinycortex/Cargo.toml @@ -51,6 +51,9 @@ async-trait = "0.1" anyhow = "1" [dev-dependencies] +# The behavioural contract suite, run against this crate's drivers +# (issue #18 §E1, acceptance criterion 5). +tinymemory-conformance = { path = "../../conformance" } tokio = { version = "1", features = ["macros", "rt-multi-thread"] } [lints.rust] diff --git a/adapters/tinycortex/src/conformance_test.rs b/adapters/tinycortex/src/conformance_test.rs new file mode 100644 index 0000000..88f0ada --- /dev/null +++ b/adapters/tinycortex/src/conformance_test.rs @@ -0,0 +1,50 @@ +//! The conformance suite, run against the TinyCortex driver. +//! +//! The last name in issue #18's acceptance criterion 5, alongside the three +//! hosted adapters covered in `tinymemory-remote`. +//! +//! # Which TinyCortex driver +//! +//! This crate binds two, and they are conformance-tested differently. +//! +//! [`crate::provider`] composes the three mandatory families over any +//! `tinycortex::memory::Memory` backend. It needs nothing but the backend, so +//! the suite runs against it here with the engine's own `InMemoryMemoryStore`. +//! +//! [`crate::engine::TinycortexProvider`] serves all eighteen families, and +//! needs a `MemoryClient` — which needs the host's process-global seams +//! (`set_embedding_host` and friends) installed before it will open. A test +//! that installs a process global is order-dependent, which `AGENTS.md` rules +//! out, so covering it needs its own integration target that owns the global +//! for the whole binary. That is not written yet, and criterion 5 is not +//! complete until it is. +//! +//! Running against `InMemoryMemoryStore` rather than a SQLite workspace is +//! deliberate and is also the sharper test: it is the engine's simplest +//! `Memory`, so anything the suite catches is the *adapter's* behaviour rather +//! than the storage engine's. + +#![allow(clippy::expect_used, clippy::panic)] + +use std::sync::Arc; + +use tinycortex::memory::store::InMemoryMemoryStore; + +#[tokio::test] +async fn the_tinycortex_driver_upholds_the_contract() { + let driver = crate::provider(Arc::new(InMemoryMemoryStore::new())); + tinymemory_conformance::assert_provider(Arc::new(driver)).await; +} + +/// The suite skips every write-path assertion when a driver does not retain, so +/// a backend that silently dropped writes would let the run above pass having +/// asserted almost nothing. This pins that it does retain. +#[tokio::test] +async fn the_backend_actually_retains() { + let driver = crate::provider(Arc::new(InMemoryMemoryStore::new())); + assert!( + tinymemory_conformance::retains_writes(&driver).await, + "the engine's in-memory store must retain writes, or the suite above \ + reports success having run four assertions of eleven" + ); +} diff --git a/adapters/tinycortex/src/lib.rs b/adapters/tinycortex/src/lib.rs index ec838a7..fba0063 100644 --- a/adapters/tinycortex/src/lib.rs +++ b/adapters/tinycortex/src/lib.rs @@ -75,3 +75,6 @@ pub fn provider(memory: Arc) -> MemoryTraitProvi TINYCORTEX_DRIVER_ID, ) } + +#[cfg(test)] +mod conformance_test; diff --git a/conformance/src/suite/mod.rs b/conformance/src/suite/mod.rs index 194770c..1766887 100644 --- a/conformance/src/suite/mod.rs +++ b/conformance/src/suite/mod.rs @@ -601,8 +601,21 @@ pub async fn assert_awkward_content_round_trips(provider: &dyn MemoryProvider) { ("large", "x".repeat(64 * 1024)), ("newlines", "a\nb\r\nc\0d".to_string()), ]; + let mut accepted = 0usize; for (key, content) in &cases { - provider + // A driver may refuse a shape outright — `MemoryCore::store` documents + // `Invalid` "for caller input the driver rejects", and the TinyCortex + // engine uses that to refuse empty content. What a driver may *not* do + // is accept a value and hand back something else. + // + // The refusal is not yet required to be `Invalid` specifically. The + // engine's own error is flattened through `anyhow` before the mandatory + // composition sees it, so a validation refusal currently arrives as + // `Other` and is indistinguishable from a backend failure. That is the + // gap §A4 closes; when it does, this should tighten to require + // `MemoryError::Invalid` so a genuine backend failure stops passing + // here. + if provider .store( &ns, key, @@ -612,7 +625,11 @@ pub async fn assert_awkward_content_round_trips(provider: &dyn MemoryProvider) { MemoryTaint::Internal, ) .await - .unwrap_or_else(|e| panic!("{who}: store of `{key}` failed: {e}")); + .is_err() + { + continue; + } + accepted += 1; if let Some(got) = provider .get(&ns, key) .await @@ -621,6 +638,13 @@ pub async fn assert_awkward_content_round_trips(provider: &dyn MemoryProvider) { assert_eq!(&got.content, content, "{who}: `{key}` content was mangled"); } } + // Without this a driver that refused every shape would pass having stored + // nothing, which is the vacuous reading of "may refuse". + assert!( + accepted > 0, + "{who}: refused every content shape — unicode, empty, large and \ + newlines were all rejected, so this assertion proved nothing" + ); let keys: Vec<&str> = cases.iter().map(|(k, _)| *k).collect(); cleanup(provider, &ns, &keys).await; } From da888f793c1474ed3e171b1cdd6b006c00b05c1f Mon Sep 17 00:00:00 2001 From: Shanu Date: Tue, 18 Aug 2026 12:50:08 +0530 Subject: [PATCH 8/9] Extract the Composio normalisers into an engine-neutral crate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Issue #18 §B3: "Payload normalisers are pure `Value -> Value` transforms with no engine dependency. Move them back into a `tinymemory-sync` crate — so a non-TinyCortex engine gets Composio sync for free." They were not in this workspace at all. They lived inside the TinyCortex engine, and `tinymemory-core` reached in through `tinycortex::memory::sync::composio::providers::normalize::*` to use them. A host binding a different memory engine therefore could not have Composio sync, despite none of this code caring which engine is bound. That is the coupling §B3 names, and it ran through the engine rather than around it. `tinymemory-sync` is fifteen files and 2,598 lines, depending on `serde_json`, two logging facades, and `chrono`. It links no engine, no storage, no async runtime — and no contract either. Two things found while moving, both stated rather than smoothed over. The crate is not quite the pure function of its input that §B3 describes. `format_email_local_time` renders in `chrono::Local`, so it reads the host's timezone, and `notion::now_ms` reads the clock. Both are deliberate upstream — the agent presents local times without doing UTC arithmetic, and Notion payloads carry no ingestion timestamp — and the raw UTC field is preserved alongside, so sorting and deduplication stay UTC-based. Documented at the crate root and at each function rather than left for someone whose output moves when they change `TZ`. The two logging facades are also inherited: `gmail_post_process` traces through `tracing`, `slack_post_process` through `log`. Preserved rather than unified, because §B3 is a move and swapping a facade changes where a host's log lines surface — a behaviour change hiding inside a relocation. The move is otherwise verbatim, with four exceptions, all forced by this workspace's lint configuration being stricter than the engine's gate reached: two `unwrap`s removed by checking presence immutably before fetching mutably, one `if let ... else { return None }` rewritten as `?`, and one `unwrap` in `ensure_object` turned into a scoped `expect` with the invariant spelled out — the case `AGENTS.md` explicitly permits. Doc links pointing at engine-internal paths are unlinked to prose, since this crate deliberately cannot see them. Acceptance, measured: `cargo tree -p tinymemory-sync` links zero of `tinycortex`, `rusqlite`, `tinymemory-core`, `tinymemory-api`. Core no longer names the engine's normalisers anywhere. The engine keeps its copy until tinyhumansai/tinycortex removes it; that side is a companion change, and the module is dead code there — its only remaining references are four doc links. Refs #18 (§B3) --- Cargo.lock | 11 + Cargo.toml | 5 +- core/Cargo.toml | 5 + .../sync/composio/providers/clickup/mod.rs | 2 +- .../src/sync/composio/providers/github/mod.rs | 2 +- core/src/sync/composio/providers/gmail/mod.rs | 2 +- core/src/sync/composio/providers/helpers.rs | 4 +- .../src/sync/composio/providers/linear/mod.rs | 2 +- core/src/sync/composio/providers/mod.rs | 2 +- .../src/sync/composio/providers/notion/mod.rs | 2 +- core/src/sync/composio/providers/slack/mod.rs | 2 +- crates/tinymemory-module/Cargo.lock | 11 + sync/Cargo.toml | 45 ++ sync/src/clickup.rs | 133 +++++ sync/src/clickup_tests.rs | 101 ++++ sync/src/github.rs | 130 +++++ sync/src/github_tests.rs | 123 +++++ sync/src/gmail_post_process.rs | 503 ++++++++++++++++++ sync/src/gmail_post_process_tests.rs | 359 +++++++++++++ sync/src/helpers.rs | 50 ++ sync/src/helpers_tests.rs | 40 ++ sync/src/lib.rs | 38 ++ sync/src/linear.rs | 157 ++++++ sync/src/linear_tests.rs | 189 +++++++ sync/src/notion.rs | 120 +++++ sync/src/notion_tests.rs | 142 +++++ sync/src/slack_post_process.rs | 323 +++++++++++ sync/src/slack_post_process_tests.rs | 262 +++++++++ 28 files changed, 2754 insertions(+), 11 deletions(-) create mode 100644 sync/Cargo.toml create mode 100644 sync/src/clickup.rs create mode 100644 sync/src/clickup_tests.rs create mode 100644 sync/src/github.rs create mode 100644 sync/src/github_tests.rs create mode 100644 sync/src/gmail_post_process.rs create mode 100644 sync/src/gmail_post_process_tests.rs create mode 100644 sync/src/helpers.rs create mode 100644 sync/src/helpers_tests.rs create mode 100644 sync/src/lib.rs create mode 100644 sync/src/linear.rs create mode 100644 sync/src/linear_tests.rs create mode 100644 sync/src/notion.rs create mode 100644 sync/src/notion_tests.rs create mode 100644 sync/src/slack_post_process.rs create mode 100644 sync/src/slack_post_process_tests.rs diff --git a/Cargo.lock b/Cargo.lock index 5caf806..ce9b175 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1867,6 +1867,7 @@ dependencies = [ "tinycortex-api", "tinymemory", "tinymemory-api", + "tinymemory-sync", "tokio", "tracing", "url", @@ -1891,6 +1892,16 @@ dependencies = [ "tokio", ] +[[package]] +name = "tinymemory-sync" +version = "0.1.0" +dependencies = [ + "chrono", + "log", + "serde_json", + "tracing", +] + [[package]] name = "tinymemory-tinycortex" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index 2d18feb..fa90487 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,7 @@ [workspace] -members = [".", "api", "core", "adapters/tinycortex", "adapters/remote", "conformance"] -default-members = [".", "api", "core", "adapters/tinycortex", "adapters/remote", "conformance"] +# `sync` is the engine-neutral Composio normalisers (issue #18 §B3). +members = [".", "api", "core", "sync", "adapters/tinycortex", "adapters/remote", "conformance"] +default-members = [".", "api", "core", "sync", "adapters/tinycortex", "adapters/remote", "conformance"] # `vendor/` holds engine submodules (tinycortex, tinybus, tinyagents), each of # which is its own workspace with its own lockfile. Same exclusion # `vendor/tinycortex` uses for its own nested vendor directory. diff --git a/core/Cargo.toml b/core/Cargo.toml index 628cafd..d876807 100644 --- a/core/Cargo.toml +++ b/core/Cargo.toml @@ -16,6 +16,11 @@ readme = "../README.md" # The contract. `tinymemory-core` implements and consumes it; the host seam # traits (config, event sink, embeddings, chat) live in `tinymemory_api::host`. tinymemory-api = { path = "../api" } +# Composio payload normalisers, extracted out of the engine (issue #18 §B3). +# They were reached through `tinycortex` until now, which meant a host binding a +# different engine could not have Composio sync despite none of this code +# caring which engine is bound. +tinymemory-sync = { path = "../sync" } tinymemory = { path = ".." } # The default embedded engine. `store/`, `tree/` and `sync/` drive it directly; diff --git a/core/src/sync/composio/providers/clickup/mod.rs b/core/src/sync/composio/providers/clickup/mod.rs index 807d7fe..f38484a 100644 --- a/core/src/sync/composio/providers/clickup/mod.rs +++ b/core/src/sync/composio/providers/clickup/mod.rs @@ -17,7 +17,7 @@ // The payload normalisers moved to tinycortex (they are pure Value // transforms, i.e. driver-side). Aliased under the old module name so // every `normalization::extract_*` call site below stays unchanged. -use crate::engine::backend::sync::composio::providers::normalize::clickup as normalization; +use tinymemory_sync::clickup as normalization; mod provider; #[cfg(test)] mod tests; diff --git a/core/src/sync/composio/providers/github/mod.rs b/core/src/sync/composio/providers/github/mod.rs index 58c3cd8..90633f3 100644 --- a/core/src/sync/composio/providers/github/mod.rs +++ b/core/src/sync/composio/providers/github/mod.rs @@ -16,7 +16,7 @@ // The payload normalisers moved to tinycortex (they are pure Value // transforms, i.e. driver-side). Aliased under the old module name so // every `normalization::extract_*` call site below stays unchanged. -use crate::engine::backend::sync::composio::providers::normalize::github as normalization; +use tinymemory_sync::github as normalization; mod provider; #[cfg(test)] mod tests; diff --git a/core/src/sync/composio/providers/gmail/mod.rs b/core/src/sync/composio/providers/gmail/mod.rs index f8c99a0..df067ec 100644 --- a/core/src/sync/composio/providers/gmail/mod.rs +++ b/core/src/sync/composio/providers/gmail/mod.rs @@ -1,7 +1,7 @@ // The Gmail post-processor moved to tinycortex (a pure Value transform, i.e. // driver-side). Aliased under the old module name so the single call site in // `provider.rs` stays unchanged. -use crate::engine::backend::sync::composio::providers::normalize::gmail_post_process as post_process; +use tinymemory_sync::gmail_post_process as post_process; mod provider; #[cfg(test)] mod tests; diff --git a/core/src/sync/composio/providers/helpers.rs b/core/src/sync/composio/providers/helpers.rs index d2aca25..e0358e2 100644 --- a/core/src/sync/composio/providers/helpers.rs +++ b/core/src/sync/composio/providers/helpers.rs @@ -1,11 +1,11 @@ //! Shared helpers for Composio provider implementations. //! //! `pick_str` used to live here. It is a provider payload normaliser, so it -//! moved to `crate::engine::backend::sync::composio::providers::normalize::helpers` +//! moved to `tinymemory_sync::helpers` //! and is re-exported from this module's parent. The helpers that remain are //! request-building rather than normalisation, and stay host-side. -use crate::engine::backend::sync::composio::providers::normalize::helpers::pick_str; +use tinymemory_sync::helpers::pick_str; /// Shallow-merge an `extra` JSON object into a (mutable) action-args /// object. Only object-typed extras are merged; non-object `extra` diff --git a/core/src/sync/composio/providers/linear/mod.rs b/core/src/sync/composio/providers/linear/mod.rs index af81bca..7ccce87 100644 --- a/core/src/sync/composio/providers/linear/mod.rs +++ b/core/src/sync/composio/providers/linear/mod.rs @@ -6,7 +6,7 @@ // The payload normalisers moved to tinycortex (they are pure Value // transforms, i.e. driver-side). Aliased under the old module name so // every `normalization::extract_*` call site below stays unchanged. -use crate::engine::backend::sync::composio::providers::normalize::linear as normalization; +use tinymemory_sync::linear as normalization; mod provider; #[cfg(test)] mod tests; diff --git a/core/src/sync/composio/providers/mod.rs b/core/src/sync/composio/providers/mod.rs index 7bb43ac..9637d30 100644 --- a/core/src/sync/composio/providers/mod.rs +++ b/core/src/sync/composio/providers/mod.rs @@ -281,11 +281,11 @@ pub(crate) use helpers::{first_array_str, merge_extra}; // re-exported here so the ~40 in-tree call sites keep resolving unchanged. // Note this is deliberately NOT `providers::common::pick_str`, which coerces // numbers to strings — see the doc comments on both definitions. -pub(crate) use crate::engine::backend::sync::composio::providers::normalize::helpers::pick_str; pub use registry::{ all_providers, get_provider, init_default_providers, register_provider, ProviderArc, }; pub use scope_lookup::{curated_scope_for, toolkit_has_scope}; +pub(crate) use tinymemory_sync::helpers::pick_str; pub use tool_scope::{classify_unknown, find_curated, toolkit_from_slug, CuratedTool, ToolScope}; pub use traits::{resolve_sync_interval_secs, sync_interval_env_var, ComposioProvider}; pub use types::{ diff --git a/core/src/sync/composio/providers/notion/mod.rs b/core/src/sync/composio/providers/notion/mod.rs index f781129..138fe60 100644 --- a/core/src/sync/composio/providers/notion/mod.rs +++ b/core/src/sync/composio/providers/notion/mod.rs @@ -1,7 +1,7 @@ // The payload normalisers moved to tinycortex (they are pure Value // transforms, i.e. driver-side). Aliased under the old module name so // every `normalization::extract_*` call site below stays unchanged. -use crate::engine::backend::sync::composio::providers::normalize::notion as normalization; +use tinymemory_sync::notion as normalization; mod provider; #[cfg(test)] mod tests; diff --git a/core/src/sync/composio/providers/slack/mod.rs b/core/src/sync/composio/providers/slack/mod.rs index 8d1e84e..39b745a 100644 --- a/core/src/sync/composio/providers/slack/mod.rs +++ b/core/src/sync/composio/providers/slack/mod.rs @@ -10,7 +10,7 @@ // driver-side). Re-exported under the old module name — `pub`, not a plain // `use`, because `tests/raw_coverage/memory_threads_raw_coverage_e2e.rs` // imports this path directly. -pub use crate::engine::backend::sync::composio::providers::normalize::slack_post_process as post_process; +pub use tinymemory_sync::slack_post_process as post_process; pub mod types; mod provider; diff --git a/crates/tinymemory-module/Cargo.lock b/crates/tinymemory-module/Cargo.lock index 6b52126..e9d5751 100644 --- a/crates/tinymemory-module/Cargo.lock +++ b/crates/tinymemory-module/Cargo.lock @@ -1992,6 +1992,7 @@ dependencies = [ "tinycortex-api", "tinymemory", "tinymemory-api", + "tinymemory-sync", "tokio", "tracing", "url", @@ -2022,6 +2023,16 @@ dependencies = [ "uuid", ] +[[package]] +name = "tinymemory-sync" +version = "0.1.0" +dependencies = [ + "chrono", + "log", + "serde_json", + "tracing", +] + [[package]] name = "tinymemory-tinycortex" version = "0.1.0" diff --git a/sync/Cargo.toml b/sync/Cargo.toml new file mode 100644 index 0000000..223e667 --- /dev/null +++ b/sync/Cargo.toml @@ -0,0 +1,45 @@ +[package] +name = "tinymemory-sync" +publish = false +version = "0.1.0" +edition = "2021" +rust-version = "1.96" +license = "MIT" +repository = "https://github.com/tinyhumansai/tinymemory" +description = "Engine-neutral Composio payload normalisers for TinyMemory" + +# The whole dependency list, and it is the point of the crate. These are pure +# `Value -> Value` transforms: no engine, no storage, no network, no async +# runtime. A dependency added here should have to argue for itself against that +# sentence (issue #18 §B3). +[dependencies] +serde_json = "1" +# Two logging facades, neither an implementation, both carried over from the +# engine layout this crate was extracted from: `gmail_post_process` traces +# through `tracing`, `slack_post_process` through `log`. Preserved rather than +# unified, because §B3 is a *move* and swapping a facade would change where a +# host's log lines surface — a behaviour change hiding inside a relocation. +# Worth reconciling in its own change. +tracing = "0.1" +log = "0.4" +# RFC 2822/3339 date handling for Gmail `Date:` headers. +# +# `clock` is on, and it is the one place this crate is not a pure function of +# its input: `format_email_local_time` renders in `chrono::Local`, so it reads +# the host's timezone. That is deliberate upstream — the agent presents local +# times without doing UTC arithmetic, and the raw UTC field is preserved +# alongside — but it means "pure `Value -> Value`" is true of every normaliser +# here except that one. Better said out loud than discovered by someone whose +# output moved when they changed TZ. +chrono = { version = "0.4", features = ["clock"] } + +[lints.rust] +unsafe_code = "forbid" +missing_docs = "warn" +unreachable_pub = "warn" + +[lints.clippy] +all = { level = "warn", priority = -1 } +unwrap_used = "warn" +expect_used = "warn" +panic = "warn" diff --git a/sync/src/clickup.rs b/sync/src/clickup.rs new file mode 100644 index 0000000..ae340b6 --- /dev/null +++ b/sync/src/clickup.rs @@ -0,0 +1,133 @@ +//! ClickUp host normalization helpers — result extraction, task-title extraction, +//! and time utilities. +//! +//! ClickUp's REST API (and therefore Composio's wrapping of it) returns +//! task lists in a small handful of shapes depending on which endpoint +//! is called. The functions here walk the union of common shapes so the +//! provider doesn't have to branch per Composio envelope variant. + +use serde_json::Value; + +use super::helpers::pick_str; + +/// Walk the Composio response envelope for ClickUp task list results. +/// +/// ClickUp's "filtered team tasks" endpoint returns `{ "tasks": [...] }` +/// at the top level; Composio re-wraps the upstream payload under +/// `data` or `data.data` depending on the action. We probe each shape +/// in order and return the first array we find. +pub fn extract_tasks(data: &Value) -> Vec { + let candidates = [ + data.pointer("/data/tasks"), + data.pointer("/tasks"), + data.pointer("/data/data/tasks"), + data.pointer("/data/results"), + data.pointer("/results"), + data.pointer("/data/items"), + data.pointer("/items"), + ]; + for cand in candidates.into_iter().flatten() { + if let Some(arr) = cand.as_array() { + return arr.clone(); + } + } + Vec::new() +} + +/// Extract a human-readable title from a ClickUp task object. +/// +/// ClickUp tasks store the name at `name` (or `data.name` after Composio +/// envelope wrapping). When the name is missing we fall back to the +/// task ID so chunks remain identifiable. +pub fn extract_task_name(task: &Value) -> Option { + pick_str(task, &["name", "data.name", "title", "data.title"]) +} + +/// Extract a stable cursor timestamp (milliseconds since epoch as a +/// string) from a ClickUp task object. +/// +/// The ClickUp API returns `date_updated` as a stringified epoch ms +/// (e.g. `"1733412345678"`); we keep it as a string so lexicographic +/// comparison against the stored cursor remains valid as long as the +/// length doesn't change (it won't until year 33658). +pub fn extract_task_updated(task: &Value) -> Option { + pick_str( + task, + &[ + "date_updated", + "data.date_updated", + "updated_at", + "data.updated_at", + "dateUpdated", + "data.dateUpdated", + ], + ) +} + +/// Current wall-clock time in milliseconds since the UNIX epoch. +pub fn now_ms() -> u64 { + use std::time::{SystemTime, UNIX_EPOCH}; + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|d| d.as_millis() as u64) + .unwrap_or(0) +} + +/// Extract the authorized user's numeric ID from the +/// `CLICKUP_GET_AUTHORIZED_USER` response. +/// +/// Composio wraps the upstream `{"user": {"id": …}}` shape; this walker +/// is defensive against both raw and wrapped payloads. Returns the ID +/// as a string because `CLICKUP_GET_FILTERED_TEAM_TASKS` accepts the +/// `assignees` filter as a string array. +pub fn extract_user_id(data: &Value) -> Option { + let candidates = [ + data.pointer("/user/id"), + data.pointer("/data/user/id"), + data.pointer("/id"), + data.pointer("/data/id"), + ]; + for cand in candidates.into_iter().flatten() { + if let Some(n) = cand.as_u64() { + return Some(n.to_string()); + } + if let Some(n) = cand.as_i64() { + return Some(n.to_string()); + } + if let Some(s) = cand.as_str() { + let trimmed = s.trim(); + if !trimmed.is_empty() { + return Some(trimmed.to_string()); + } + } + } + None +} + +/// Extract a list of workspace (team) IDs from the +/// `CLICKUP_GET_AUTHORIZED_TEAMS_WORKSPACES` response. +/// +/// ClickUp returns `{"teams": [{"id": "...", "name": "..."}, …]}`. We +/// keep the IDs as strings — `CLICKUP_GET_FILTERED_TEAM_TASKS` requires +/// a `team_id` (string) argument. +pub fn extract_workspace_ids(data: &Value) -> Vec { + let candidates = [ + data.pointer("/teams"), + data.pointer("/data/teams"), + data.pointer("/workspaces"), + data.pointer("/data/workspaces"), + ]; + for cand in candidates.into_iter().flatten() { + if let Some(arr) = cand.as_array() { + return arr + .iter() + .filter_map(|t| pick_str(t, &["id", "team_id", "workspace_id"])) + .collect(); + } + } + Vec::new() +} + +#[cfg(test)] +#[path = "clickup_tests.rs"] +mod tests; diff --git a/sync/src/clickup_tests.rs b/sync/src/clickup_tests.rs new file mode 100644 index 0000000..99b4667 --- /dev/null +++ b/sync/src/clickup_tests.rs @@ -0,0 +1,101 @@ +#![allow(clippy::expect_used, clippy::panic, clippy::unwrap_used)] +// +// A failing assertion in a test *is* a panic. The crate-wide lints exist to +// keep the library from panicking, not the tests. + +use super::*; +use serde_json::json; + +#[test] +fn extract_tasks_from_data_tasks() { + let data = json!({ "data": { "tasks": [{"id": "t1"}] } }); + assert_eq!(extract_tasks(&data).len(), 1); +} + +#[test] +fn extract_tasks_from_top_level_tasks() { + let data = json!({ "tasks": [{"id": "a"}, {"id": "b"}] }); + assert_eq!(extract_tasks(&data).len(), 2); +} + +#[test] +fn extract_tasks_empty_when_missing() { + let data = json!({ "foo": "bar" }); + assert!(extract_tasks(&data).is_empty()); +} + +#[test] +fn extract_task_name_from_top_level() { + let task = json!({ "id": "t1", "name": "Build feature X" }); + assert_eq!(extract_task_name(&task), Some("Build feature X".into())); +} + +#[test] +fn extract_task_name_falls_back_to_data_name() { + let task = json!({ "data": { "name": "Wrapped" } }); + assert_eq!(extract_task_name(&task), Some("Wrapped".into())); +} + +#[test] +fn extract_task_name_none_when_missing() { + let task = json!({ "id": "t1" }); + assert!(extract_task_name(&task).is_none()); +} + +#[test] +fn extract_task_updated_handles_string_form() { + let task = json!({ "date_updated": "1733412345678" }); + assert_eq!( + extract_task_updated(&task), + Some("1733412345678".to_string()) + ); +} + +#[test] +fn extract_task_updated_handles_nested_data() { + let task = json!({ "data": { "dateUpdated": "1700000000000" } }); + assert_eq!( + extract_task_updated(&task), + Some("1700000000000".to_string()) + ); +} + +#[test] +fn extract_user_id_handles_numeric_id() { + let data = json!({ "user": { "id": 12345 } }); + assert_eq!(extract_user_id(&data), Some("12345".to_string())); +} + +#[test] +fn extract_user_id_handles_wrapped_payload() { + let data = json!({ "data": { "user": { "id": "777" } } }); + assert_eq!(extract_user_id(&data), Some("777".to_string())); +} + +#[test] +fn extract_user_id_none_when_missing() { + let data = json!({ "foo": "bar" }); + assert!(extract_user_id(&data).is_none()); +} + +#[test] +fn extract_workspace_ids_from_teams_array() { + let data = json!({ + "teams": [ + { "id": "ws1", "name": "Personal" }, + { "id": "ws2", "name": "Acme" }, + ] + }); + assert_eq!(extract_workspace_ids(&data), vec!["ws1", "ws2"]); +} + +#[test] +fn extract_workspace_ids_empty_when_no_teams() { + let data = json!({ "foo": "bar" }); + assert!(extract_workspace_ids(&data).is_empty()); +} + +#[test] +fn now_ms_returns_nonzero() { + assert!(now_ms() > 0); +} diff --git a/sync/src/github.rs b/sync/src/github.rs new file mode 100644 index 0000000..393e93e --- /dev/null +++ b/sync/src/github.rs @@ -0,0 +1,130 @@ +//! GitHub host normalization helpers — result extraction, identity helpers, and time utilities. +//! +//! GitHub's REST API (proxied through Composio) returns search results and +//! authenticated-user payloads in a small number of shapes. The functions here +//! walk the union of common Composio envelope variants so the provider stays +//! clean and branch-free. + +use serde_json::Value; + +use super::helpers::pick_str; + +/// Walk the Composio response envelope for GitHub search issue results. +/// +/// `GITHUB_SEARCH_ISSUES_AND_PULL_REQUESTS` wraps GitHub's `GET /search/issues` response, which +/// returns `{"total_count": N, "items": [...]}`. Composio may re-wrap this under +/// `data` or `data.data`; we probe each shape in order. +pub fn extract_issues(data: &Value) -> Vec { + let candidates = [ + data.pointer("/data/items"), + data.pointer("/items"), + data.pointer("/data/data/items"), + data.pointer("/data/results"), + data.pointer("/results"), + ]; + for cand in candidates.into_iter().flatten() { + if let Some(arr) = cand.as_array() { + return arr.clone(); + } + } + Vec::new() +} + +/// Extract a stable, globally unique identifier for a GitHub issue or PR. +/// +/// GitHub's internal `id` field is a large integer unique across all issues +/// and PRs on github.com. We convert it to a string for use as a sync key. +/// Falls back to composing from `html_url` path if `id` is absent. +pub fn extract_issue_id(issue: &Value) -> Option { + // Primary: numeric internal GitHub ID. + if let Some(id) = issue.get("id").or_else(|| issue.pointer("/data/id")) { + if let Some(n) = id.as_u64() { + return Some(n.to_string()); + } + if let Some(s) = id.as_str() { + let trimmed = s.trim(); + if !trimmed.is_empty() { + return Some(trimmed.to_string()); + } + } + } + // Fallback: parse owner/repo/number from html_url path segments. + // URL shape: https://github.com/{owner}/{repo}/issues/{number} + if let Some(url) = pick_str(issue, &["html_url", "data.html_url", "url", "data.url"]) { + if let Some(slug) = github_url_to_slug(&url) { + return Some(slug); + } + } + None +} + +/// Build a human-readable document title for a GitHub issue/PR. +/// +/// Format: `GitHub: {owner}/{repo}#{number}: {title}`. +/// Falls back to just the title or a placeholder when fields are missing. +pub fn extract_issue_title(issue: &Value) -> Option { + let title = pick_str(issue, &["title", "data.title"])?; + + // Best-effort: extract owner/repo#N from html_url for the prefix. + let prefix = pick_str(issue, &["html_url", "data.html_url"]) + .and_then(|url| github_url_to_slug(&url)) + .unwrap_or_default(); + + if prefix.is_empty() { + Some(title) + } else { + Some(format!("GitHub: {prefix}: {title}")) + } +} + +/// Parse `https://github.com/{owner}/{repo}/issues/{number}` (or `/pull/`) +/// into `"{owner}/{repo}#{number}"`. Returns `None` for unrecognised shapes. +fn github_url_to_slug(url: &str) -> Option { + let segs: Vec<&str> = url.trim_end_matches('/').split('/').collect(); + // Minimum: ["https:", "", "github.com", owner, repo, "issues", number] + if segs.len() >= 7 { + let number = segs[segs.len() - 1]; + let _kind = segs[segs.len() - 2]; // "issues" or "pull" — ignored + let repo = segs[segs.len() - 3]; + let owner = segs[segs.len() - 4]; + if !owner.is_empty() && !repo.is_empty() && !number.is_empty() { + return Some(format!("{owner}/{repo}#{number}")); + } + } + None +} + +/// Extract the `updated_at` ISO 8601 timestamp from a GitHub issue. +/// +/// GitHub returns `updated_at` as `"2024-05-21T15:30:00Z"`. ISO 8601 strings +/// sort lexicographically, so we use them directly as the sync cursor. +pub fn extract_issue_updated_at(issue: &Value) -> Option { + pick_str( + issue, + &[ + "updated_at", + "data.updated_at", + "updatedAt", + "data.updatedAt", + ], + ) +} + +/// Extract the authenticated user's login handle from a +/// `GITHUB_GET_THE_AUTHENTICATED_USER` response. +pub fn extract_user_login(data: &Value) -> Option { + pick_str(data, &["login", "data.login"]) +} + +/// Current wall-clock time in milliseconds since the UNIX epoch. +pub fn now_ms() -> u64 { + use std::time::{SystemTime, UNIX_EPOCH}; + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|d| d.as_millis() as u64) + .unwrap_or(0) +} + +#[cfg(test)] +#[path = "github_tests.rs"] +mod tests; diff --git a/sync/src/github_tests.rs b/sync/src/github_tests.rs new file mode 100644 index 0000000..17aa138 --- /dev/null +++ b/sync/src/github_tests.rs @@ -0,0 +1,123 @@ +#![allow(clippy::expect_used, clippy::panic, clippy::unwrap_used)] +// +// A failing assertion in a test *is* a panic. The crate-wide lints exist to +// keep the library from panicking, not the tests. + +use super::*; +use serde_json::json; + +#[test] +fn extract_issues_from_data_items() { + let data = json!({ "data": { "items": [{"id": 1}] } }); + assert_eq!(extract_issues(&data).len(), 1); +} + +#[test] +fn extract_issues_from_top_level_items() { + let data = json!({ "items": [{"id": 1}, {"id": 2}] }); + assert_eq!(extract_issues(&data).len(), 2); +} + +#[test] +fn extract_issues_empty_when_missing() { + let data = json!({ "foo": "bar" }); + assert!(extract_issues(&data).is_empty()); +} + +#[test] +fn extract_issue_id_from_numeric_field() { + let issue = json!({ "id": 123456789u64, "title": "Fix bug" }); + assert_eq!(extract_issue_id(&issue), Some("123456789".to_string())); +} + +#[test] +fn extract_issue_id_from_wrapped_data() { + let issue = json!({ "data": { "id": 99u64 } }); + assert_eq!(extract_issue_id(&issue), Some("99".to_string())); +} + +#[test] +fn extract_issue_id_falls_back_to_html_url() { + let issue = json!({ + "html_url": "https://github.com/owner/repo/issues/42" + }); + assert_eq!(extract_issue_id(&issue), Some("owner/repo#42".to_string())); +} + +#[test] +fn extract_issue_id_none_when_missing() { + let issue = json!({ "title": "No ID here" }); + assert!(extract_issue_id(&issue).is_none()); +} + +#[test] +fn extract_issue_title_builds_prefixed_title() { + let issue = json!({ + "id": 1u64, + "title": "Fix race condition", + "html_url": "https://github.com/acme/core/issues/99" + }); + assert_eq!( + extract_issue_title(&issue), + Some("GitHub: acme/core#99: Fix race condition".to_string()) + ); +} + +#[test] +fn extract_issue_title_returns_raw_title_when_no_url() { + let issue = json!({ "title": "Bare title" }); + assert_eq!(extract_issue_title(&issue), Some("Bare title".to_string())); +} + +#[test] +fn extract_issue_title_none_when_missing() { + let issue = json!({ "id": 1u64 }); + assert!(extract_issue_title(&issue).is_none()); +} + +#[test] +fn extract_issue_updated_at_from_top_level() { + let issue = json!({ "updated_at": "2024-05-21T15:30:00Z" }); + assert_eq!( + extract_issue_updated_at(&issue), + Some("2024-05-21T15:30:00Z".to_string()) + ); +} + +#[test] +fn extract_issue_updated_at_from_data_wrapper() { + let issue = json!({ "data": { "updated_at": "2023-01-01T00:00:00Z" } }); + assert_eq!( + extract_issue_updated_at(&issue), + Some("2023-01-01T00:00:00Z".to_string()) + ); +} + +#[test] +fn extract_issue_updated_at_none_when_missing() { + let issue = json!({ "id": 1u64 }); + assert!(extract_issue_updated_at(&issue).is_none()); +} + +#[test] +fn extract_user_login_from_top_level() { + let data = json!({ "login": "octocat" }); + assert_eq!(extract_user_login(&data), Some("octocat".to_string())); +} + +#[test] +fn extract_user_login_from_data_wrapper() { + let data = json!({ "data": { "login": "monalisa" } }); + assert_eq!(extract_user_login(&data), Some("monalisa".to_string())); +} + +#[test] +fn extract_user_login_none_when_missing() { + let data = json!({ "id": 1u64 }); + assert!(extract_user_login(&data).is_none()); +} + +#[test] +fn now_ms_returns_nonzero() { + assert!(now_ms() > 0); +} diff --git a/sync/src/gmail_post_process.rs b/sync/src/gmail_post_process.rs new file mode 100644 index 0000000..327479a --- /dev/null +++ b/sync/src/gmail_post_process.rs @@ -0,0 +1,503 @@ +//! Gmail-specific post-processing of Composio action responses. +//! +//! The upstream `GMAIL_FETCH_EMAILS` payload is extremely verbose +//! (full MIME tree under `payload.parts[]`, 50+ `Received:` headers, +//! display-layer noise the model never uses). This module rewrites +//! it into a slim envelope per message: +//! +//! ```json +//! { +//! "messages": [ +//! { +//! "id": "…", +//! "threadId": "…", +//! "subject": "…", +//! "from": "…", +//! "to": "…", +//! "date": "…", +//! "labels": ["INBOX", "UNREAD"], +//! "markdown": "…body…", +//! "attachments": [ { "filename": "...", "mimeType": "..." } ] +//! } +//! ], +//! "nextPageToken": "…", +//! "resultSizeEstimate": 201 +//! } +//! ``` +//! +//! ## Body source +//! +//! Composio's backend ships a +//! `markdownFormatted` field on the response envelope — one string +//! per tool call, pre-rendered with HTML stripped, URLs shortened, +//! footers removed, whitespace normalised. We split it per message +//! along `\n---\n` boundaries (with `## ` heading fallbacks) and +//! pin each slice to the corresponding entry in `messages[]` via +//! [`apply_response_level_markdown`]. The reshape's +//! `extract_markdown_body` then prefers that pinned field over +//! falling back to the upstream `messageText`. +//! +//! No in-house HTML→markdown conversion lives here anymore — the +//! backend does the cleaning. If `markdownFormatted` is absent for +//! a given response we fall through to whatever plain text the +//! upstream provided in `messageText`. +//! +//! Callers that need the raw Composio shape can pass `raw_html: +//! true` (or `rawHtml: true`) in the action arguments — this +//! short-circuits the reshape entirely. +//! +//! Only `GMAIL_FETCH_EMAILS` is reshaped today; other Gmail action +//! responses are passed through unchanged. When we add envelopes for +//! more slugs they should live in this file, branched from +//! [`post_process`]. + +use serde_json::{json, Map, Value}; + +/// Entry point called from `GmailProvider::post_process_action_result`. +/// +/// Dispatches on the Composio action slug. Unknown Gmail slugs fall +/// through to a no-op. +pub fn post_process(slug: &str, arguments: Option<&Value>, data: &mut Value) { + if is_raw_html_flag_set(arguments) { + tracing::debug!( + slug, + "[composio:gmail][post-process] raw_html flag set, passing through" + ); + return; + } + if slug == "GMAIL_FETCH_EMAILS" { + reshape_fetch_emails(data) + } +} + +/// Stash per-message slices of the response-level `markdownFormatted` +/// onto the corresponding entries inside `data.messages[]`. +/// +/// The Composio backend (tinyhumansai/backend#683) ships ONE +/// `markdownFormatted` string per tool call covering all messages — +/// already URL-shortened, footer-stripped, and whitespace-normalised. +/// To get per-email files in the raw archive we split that string +/// along section boundaries (`## ` headings or `---` rules) and pin +/// each slice to the message at the same index. `extract_markdown_body` +/// then prefers `msg.markdownFormatted` over re-decoding the MIME +/// tree. +/// +/// **Must be called BEFORE [`post_process`]** because `post_process` +/// reshapes `data` into the slim envelope; once `messages[]` carries +/// our slim shape the upstream message ordering is already locked in +/// but we may have lost original ordering signals if any. +/// +/// No-op when the slice count doesn't match `messages.len()` — we +/// can't safely align segments to messages without an exact match, +/// so we let `extract_markdown_body` fall through to its MIME path. +pub fn apply_response_level_markdown(data: &mut Value, top_md: &str) { + let trimmed = top_md.trim(); + if trimmed.is_empty() { + return; + } + // Presence is checked immutably first, then fetched mutably. The original + // form re-fetched with `unwrap()` after a mutable probe, which is sound but + // relies on the reader to see why; this crate forbids `unwrap`, and the + // immutable probe expresses the same reasoning to the compiler. + let container = if data.get("messages").is_some() { + data + } else if data.get("data").and_then(Value::as_object).is_some() { + match data.get_mut("data") { + Some(inner) => inner, + None => return, + } + } else { + tracing::debug!( + "[composio:gmail][post-process] apply_response_level_markdown: \ + no messages container in response — skipping" + ); + return; + }; + let Some(messages) = container.get_mut("messages").and_then(|v| v.as_array_mut()) else { + return; + }; + let count = messages.len(); + if count == 0 { + return; + } + // Clone hints out of the messages array so the slice borrows + // don't conflict with the upcoming `messages.iter_mut()` mutation. + let hints: Vec = messages.clone(); + let Some(slices) = split_response_markdown_per_message_with_hint(trimmed, count, Some(&hints)) + else { + tracing::debug!( + messages = count, + md_len = trimmed.len(), + "[composio:gmail][post-process] could not split response-level markdownFormatted \ + into {count} slices — falling back to per-message MIME decode" + ); + return; + }; + for (msg, slice) in messages.iter_mut().zip(slices) { + if let Some(obj) = msg.as_object_mut() { + obj.insert("markdownFormatted".to_string(), Value::String(slice)); + } + } + tracing::debug!( + messages = count, + "[composio:gmail][post-process] stashed per-message markdownFormatted slices" + ); +} + +/// Split a top-level `markdownFormatted` string into per-message +/// segments. Returns `Some(slices)` only when the split yields +/// exactly `expected_count` entries — otherwise the format isn't one +/// of the patterns we know about and we let the caller fall back. +/// +/// Primary boundary is the `\n---\n` horizontal rule the backend +/// emits between messages (confirmed against real +/// `GMAIL_FETCH_EMAILS` output). H2/H3 headings are kept as +/// fallbacks for older renderings. The preamble (`# Inbox (N +/// messages)`-style intro, if present) is dropped — we accept +/// either `expected` parts (no preamble) or `expected + 1` +/// (preamble + N messages). +/// +/// `messages_hint` is the slim message array from the same response +/// — when present we use the per-message `subject` field to verify +/// each segment really does belong to the message at the same index. +/// Mismatches force a fallback so we never write a wrong-message body +/// to the raw archive. +pub fn split_response_markdown_per_message(md: &str, expected_count: usize) -> Option> { + split_response_markdown_per_message_with_hint(md, expected_count, None) +} + +/// Split a response-level markdown blob into one slice per message. +/// +/// `hint` carries the message ids in response order, which is what makes the +/// split reliable: the blob's own section headings are backend-rendered and +/// have changed shape between versions, so matching on them alone silently +/// mis-attributed bodies. +pub fn split_response_markdown_per_message_with_hint( + md: &str, + expected_count: usize, + messages_hint: Option<&[Value]>, +) -> Option> { + if expected_count == 0 { + return None; + } + if expected_count == 1 { + return Some(vec![md.to_string()]); + } + + // Boundary patterns to try, in priority order. `\n---\n` is the + // confirmed marker; the heading variants stay as belt-and-braces + // for older / variant backend renderings. + let candidates: &[(&str, &str)] = &[ + ("\n---\n", "---\n"), + ("\n\n## ", "## "), + ("\n\n### ", "### "), + ("\n\n# ", "# "), + ("\n***\n", "***\n"), + ]; + + for (sep, prefix) in candidates { + let parts: Vec<&str> = md.split(sep).collect(); + let (drop_preamble, prepend_first) = if parts.len() == expected_count { + (false, false) // no preamble; first segment had no prefix + } else if parts.len() == expected_count + 1 { + (true, true) // preamble dropped; every kept segment had a prefix + } else { + continue; + }; + let segments: Vec = parts + .into_iter() + .skip(if drop_preamble { 1 } else { 0 }) + .enumerate() + .map(|(i, s)| { + if i == 0 && !prepend_first { + s.to_string() + } else { + format!("{prefix}{s}") + } + }) + .collect(); + + // Validate alignment against the JSON message array: every + // segment whose corresponding message has a non-empty subject + // must mention that subject somewhere in its body. If a single + // pair fails, we treat the split as unreliable and try the + // next pattern. Empty / null subjects skip validation (e.g. + // notification mails where the subject is ""). + if let Some(hints) = messages_hint { + if !validate_segments_against_hints(&segments, hints) { + tracing::debug!( + expected = expected_count, + sep = sep, + "[composio:gmail][post-process] split candidate failed subject check" + ); + continue; + } + } + return Some(segments); + } + None +} + +/// True if every (segment, message) pair where the message has a +/// non-empty subject contains that subject somewhere in the segment +/// (case-insensitive substring match — a defensive heuristic, not a +/// strict equality check, since the backend may format subjects +/// inside markdown links or with surrounding decoration). +fn validate_segments_against_hints(segments: &[String], hints: &[Value]) -> bool { + if segments.len() != hints.len() { + return false; + } + for (seg, hint) in segments.iter().zip(hints.iter()) { + let subject = hint + .get("subject") + .and_then(|v| v.as_str()) + .unwrap_or("") + .trim(); + if subject.is_empty() { + continue; + } + if !seg + .to_ascii_lowercase() + .contains(&subject.to_ascii_lowercase()) + { + return false; + } + } + true +} + +/// Returns true when the caller explicitly set `raw_html: true` (or the +/// camelCase `rawHtml: true`) in the `arguments` object. +fn is_raw_html_flag_set(arguments: Option<&Value>) -> bool { + let Some(obj) = arguments.and_then(|v| v.as_object()) else { + return false; + }; + obj.get("raw_html") + .or_else(|| obj.get("rawHtml")) + .and_then(|v| v.as_bool()) + .unwrap_or(false) +} + +/// Rewrite a `GMAIL_FETCH_EMAILS` `data` object in place into the slim +/// envelope documented at the module level. +/// +/// The Composio response can be shaped either as `{ messages, nextPageToken, ... }` +/// directly, or wrapped one level deeper under `{ data: { messages: … } }` +/// depending on backend version; we handle both. +fn reshape_fetch_emails(data: &mut Value) { + // Unwrap an optional `data:` envelope so downstream logic only has + // to deal with one shape. + let container = if data.get("messages").is_some() { + data + } else if data.get("data").and_then(Value::as_object).is_some() { + match data.get_mut("data") { + Some(inner) => inner, + None => return, + } + } else { + return; + }; + + let Some(obj) = container.as_object_mut() else { + return; + }; + + let raw_messages = obj + .remove("messages") + .and_then(|v| match v { + Value::Array(arr) => Some(arr), + _ => None, + }) + .unwrap_or_default(); + let next_page_token = obj.remove("nextPageToken").unwrap_or(Value::Null); + let result_size_estimate = obj.remove("resultSizeEstimate").unwrap_or(Value::Null); + + let messages: Vec = raw_messages.into_iter().map(reshape_message).collect(); + + let mut envelope = Map::new(); + envelope.insert("messages".into(), Value::Array(messages)); + if !next_page_token.is_null() { + envelope.insert("nextPageToken".into(), next_page_token); + } + if !result_size_estimate.is_null() { + envelope.insert("resultSizeEstimate".into(), result_size_estimate); + } + + *container = Value::Object(envelope); +} + +/// Parse an RFC 3339 or RFC 2822 date string into a UTC `DateTime`. +pub fn parse_email_date(date_str: &str) -> Option> { + date_str + .parse::>() + .or_else(|_| { + chrono::DateTime::parse_from_rfc2822(date_str).map(|d| d.with_timezone(&chrono::Utc)) + }) + .ok() +} + +const EMAIL_LOCAL_TIME_FMT: &str = "%Y-%m-%d %I:%M %p %:z"; + +/// Format a UTC `DateTime` in the given timezone. Returns `None` when the +/// formatted result is identical to the UTC rendering (no-op for UTC hosts). +pub fn format_at_tz( + utc: chrono::DateTime, + tz: &Tz, +) -> Option +where + Tz::Offset: std::fmt::Display, +{ + let local_dt = utc.with_timezone(tz); + let formatted = local_dt.format(EMAIL_LOCAL_TIME_FMT).to_string(); + + let utc_formatted = utc.format(EMAIL_LOCAL_TIME_FMT).to_string(); + if formatted == utc_formatted { + return None; + } + Some(formatted) +} + +/// Convert a UTC email timestamp string to a human-readable local-time string. +/// +/// Accepts RFC 3339 (`"2026-05-31T10:33:00Z"`) or RFC 2822 +/// (`"Sat, 31 May 2026 10:33:00 +0000"`) input. Returns a formatted string +/// in the host's local timezone, e.g. `"2026-05-31 05:33 AM -05:00"`, +/// so the agent can present local times without UTC arithmetic. +/// +/// The raw `date` field is always preserved alongside this field so +/// internal sorting, deduplication, and debugging remain UTC-based. +/// +/// Returns `None` when the input cannot be parsed or the output format +/// would be identical to the UTC input (no-op for UTC hosts). +pub fn format_email_local_time(date_str: &str) -> Option { + let utc = parse_email_date(date_str)?; + format_at_tz(utc, &chrono::Local) +} + +/// Map one raw Composio message object to its slim counterpart. +/// +/// Body source picked by [`extract_markdown_body`]: +/// 1. The per-message `markdownFormatted` slice pinned by +/// [`apply_response_level_markdown`] (preferred — backend-rendered). +/// 2. The upstream `messageText` plaintext (fallback). +/// 3. Empty string. +fn reshape_message(raw: Value) -> Value { + let Value::Object(obj) = raw else { + return raw; + }; + + let id = obj.get("messageId").cloned().unwrap_or(Value::Null); + let thread_id = obj.get("threadId").cloned().unwrap_or(Value::Null); + let subject = obj.get("subject").cloned().unwrap_or(Value::Null); + let sender = obj.get("sender").cloned().unwrap_or(Value::Null); + let to = obj.get("to").cloned().unwrap_or(Value::Null); + let date = obj + .get("messageTimestamp") + .cloned() + .or_else(|| pick_header(&obj, "Date")) + .unwrap_or(Value::Null); + let labels = obj + .get("labelIds") + .cloned() + .unwrap_or_else(|| Value::Array(Vec::new())); + let list_unsubscribe = pick_header(&obj, "List-Unsubscribe").unwrap_or(Value::Null); + + let markdown = extract_markdown_body(&obj); + let attachments = extract_attachments(&obj); + + // Compute a local-time representation of the UTC `date` so the agent + // presents times in the user's timezone rather than quoting raw UTC. + let date_local = date.as_str().and_then(format_email_local_time); + + let mut out = Map::new(); + out.insert("id".into(), id); + out.insert("threadId".into(), thread_id); + out.insert("subject".into(), subject); + out.insert("from".into(), sender); + out.insert("to".into(), to); + out.insert("date".into(), date); + if let Some(local) = date_local { + out.insert("date_local".into(), Value::String(local)); + } + out.insert("labels".into(), labels); + if !list_unsubscribe.is_null() { + out.insert("list_unsubscribe".into(), list_unsubscribe); + } + out.insert("markdown".into(), Value::String(markdown)); + if !attachments.is_empty() { + out.insert("attachments".into(), Value::Array(attachments)); + } + Value::Object(out) +} + +/// Find a header value by (case-insensitive) name in the Composio +/// `payload.headers[]` array. Returns `Some(Value::String)` on hit. +fn pick_header(msg: &Map, name: &str) -> Option { + let headers = msg.get("payload")?.get("headers")?.as_array()?; + for h in headers { + let hn = h.get("name").and_then(|v| v.as_str()).unwrap_or(""); + if hn.eq_ignore_ascii_case(name) { + if let Some(v) = h.get("value").and_then(|v| v.as_str()) { + return Some(Value::String(v.to_string())); + } + } + } + None +} + +/// Pick a body for the slim envelope. +/// +/// We trust the Composio backend's pre-rendered `markdownFormatted` +/// (set per-message by [`apply_response_level_markdown`] from the +/// response-level field). When that's absent we fall back to the +/// upstream's plain-text `messageText` verbatim — no in-house +/// HTML→markdown decoding lives here anymore. The backend already +/// strips HTML, shortens URLs, and normalises whitespace; running +/// our own pipeline on top duplicated work and corrupted some +/// renderings. +fn extract_markdown_body(msg: &Map) -> String { + if let Some(formatted) = msg + .get("markdownFormatted") + .or_else(|| msg.get("markdown_formatted")) + .and_then(|v| v.as_str()) + .map(str::trim) + .filter(|s| !s.is_empty()) + { + return formatted.to_string(); + } + if let Some(text) = msg + .get("messageText") + .and_then(|v| v.as_str()) + .map(str::trim) + .filter(|s| !s.is_empty()) + { + return text.to_string(); + } + String::new() +} + +/// Pull a minimal attachments descriptor from the Composio +/// `attachmentList` array. +fn extract_attachments(msg: &Map) -> Vec { + if let Some(list) = msg.get("attachmentList").and_then(|v| v.as_array()) { + return list + .iter() + .filter_map(|a| { + let filename = a.get("filename").and_then(|v| v.as_str())?; + if filename.is_empty() { + return None; + } + let mime = a + .get("mimeType") + .and_then(|v| v.as_str()) + .unwrap_or_default(); + Some(json!({ "filename": filename, "mimeType": mime })) + }) + .collect(); + } + Vec::new() +} + +#[cfg(test)] +#[path = "gmail_post_process_tests.rs"] +mod tests; diff --git a/sync/src/gmail_post_process_tests.rs b/sync/src/gmail_post_process_tests.rs new file mode 100644 index 0000000..69eda63 --- /dev/null +++ b/sync/src/gmail_post_process_tests.rs @@ -0,0 +1,359 @@ +#![allow(clippy::expect_used, clippy::panic, clippy::unwrap_used)] +// +// A failing assertion in a test *is* a panic. The crate-wide lints exist to +// keep the library from panicking, not the tests. + +use super::*; +use serde_json::json; + +fn fixture_with_backend_markdown() -> Value { + json!({ + "messages": [ + { + "messageId": "m1", + "threadId": "t1", + "subject": "Hello", + "sender": "a@x.com", + "to": "b@y.com", + "messageTimestamp": "2026-04-17T12:00:00Z", + "labelIds": ["INBOX", "UNREAD"], + // Pre-rendered slice (set by `apply_response_level_markdown` + // in production; inline here for the reshape test). + "markdownFormatted": "# Hello\n\nbody copy", + "messageText": "fallback should not be used", + "display_url": "ignore-me", + "preview": { "body": "Hi plain", "subject": "Hello" }, + "attachmentList": [ + { "filename": "report.pdf", "mimeType": "application/pdf", "size": 12345 }, + { "filename": "", "mimeType": "text/html" } + ], + "payload": {} + } + ], + "nextPageToken": "tok-1", + "resultSizeEstimate": 42 + }) +} + +#[test] +fn reshape_emits_slim_envelope() { + let mut v = fixture_with_backend_markdown(); + post_process("GMAIL_FETCH_EMAILS", None, &mut v); + + assert_eq!(v["nextPageToken"], "tok-1"); + assert_eq!(v["resultSizeEstimate"], 42); + + let msgs = v["messages"].as_array().unwrap(); + assert_eq!(msgs.len(), 1); + let m = &msgs[0]; + + assert_eq!(m["id"], "m1"); + assert_eq!(m["threadId"], "t1"); + assert_eq!(m["subject"], "Hello"); + assert_eq!(m["from"], "a@x.com"); + assert_eq!(m["to"], "b@y.com"); + assert_eq!(m["date"], "2026-04-17T12:00:00Z"); + assert_eq!(m["labels"], json!(["INBOX", "UNREAD"])); + + let md = m["markdown"].as_str().unwrap(); + assert_eq!(md, "# Hello\n\nbody copy"); + + // Noise fields removed. + assert!(m.get("display_url").is_none()); + assert!(m.get("preview").is_none()); + assert!(m.get("payload").is_none()); + assert!(m.get("messageText").is_none()); + + // Attachments: empty filename entry is filtered. + let atts = m["attachments"].as_array().unwrap(); + assert_eq!(atts.len(), 1); + assert_eq!(atts[0]["filename"], "report.pdf"); + assert_eq!(atts[0]["mimeType"], "application/pdf"); +} + +#[test] +fn raw_html_flag_passes_through_unchanged() { + let mut v = fixture_with_backend_markdown(); + let original = v.clone(); + let args = json!({ "raw_html": true }); + post_process("GMAIL_FETCH_EMAILS", Some(&args), &mut v); + assert_eq!( + v, original, + "raw_html=true must preserve the Composio shape" + ); +} + +#[test] +fn camel_case_raw_html_also_recognized() { + let mut v = fixture_with_backend_markdown(); + let original = v.clone(); + let args = json!({ "rawHtml": true }); + post_process("GMAIL_FETCH_EMAILS", Some(&args), &mut v); + assert_eq!(v, original); +} + +#[test] +fn falls_back_to_message_text_when_no_backend_markdown() { + let mut v = json!({ + "messages": [{ + "messageId": "m1", + "threadId": "t1", + "subject": "s", + "sender": "a@x.com", + "to": "b@y.com", + "messageTimestamp": "2026-04-17", + "labelIds": [], + "messageText": " plain body text ", + "payload": {} + }], + "nextPageToken": null + }); + post_process("GMAIL_FETCH_EMAILS", None, &mut v); + let md = v["messages"][0]["markdown"].as_str().unwrap(); + assert_eq!(md, "plain body text"); + assert!(v.get("nextPageToken").is_none(), "null tokens dropped"); +} + +#[test] +fn unwraps_data_envelope() { + let mut v = json!({ + "data": { + "messages": [{ + "messageId": "m1", + "threadId": "t1", + "subject": "s", + "sender": "a@x.com", + "to": "b@y.com", + "messageTimestamp": "2026-04-17", + "labelIds": [], + "messageText": "body", + "payload": {} + }] + } + }); + post_process("GMAIL_FETCH_EMAILS", None, &mut v); + // Reshape writes into `data` in place. + let msgs = v["data"]["messages"].as_array().unwrap(); + assert_eq!(msgs.len(), 1); + assert_eq!(msgs[0]["markdown"], "body"); +} + +#[test] +fn non_fetch_slug_is_noop() { + let mut v = json!({ "messages": [{ "messageId": "m1", "messageText": "x" }] }); + let original = v.clone(); + post_process("GMAIL_SEND_EMAIL", None, &mut v); + assert_eq!(v, original); +} + +#[test] +fn prefers_backend_markdown_formatted_when_present() { + // Composio backend (tinyhumansai/backend#683 +) ships + // `markdownFormatted` already URL-shortened + footer-stripped + // per message (after `apply_response_level_markdown` slices the + // response-level field). When present, our post-processor must + // use it verbatim instead of falling back to `messageText`. + let mut v = json!({ + "messages": [{ + "messageId": "m1", + "threadId": "t1", + "subject": "s", + "sender": "a@x.com", + "to": "b@y.com", + "messageTimestamp": "2026-04-17", + "labelIds": [], + "markdownFormatted": "# Already nice\n\nShort URL: https://gh.io/abc", + "messageText": "fallback should not be used", + "payload": {} + }] + }); + post_process("GMAIL_FETCH_EMAILS", None, &mut v); + let md = v["messages"][0]["markdown"].as_str().unwrap(); + assert_eq!(md, "# Already nice\n\nShort URL: https://gh.io/abc"); +} + +#[test] +fn empty_markdown_formatted_falls_through_to_message_text() { + let mut v = json!({ + "messages": [{ + "messageId": "m1", + "threadId": "t1", + "subject": "s", + "sender": "a@x.com", + "to": "b@y.com", + "messageTimestamp": "2026-04-17", + "labelIds": [], + "markdownFormatted": " \n \n", + "messageText": "real body", + "payload": {} + }] + }); + post_process("GMAIL_FETCH_EMAILS", None, &mut v); + let md = v["messages"][0]["markdown"].as_str().unwrap(); + assert!(md.contains("real body")); +} + +// ── split_response_markdown_per_message ───────────────────────────────── + +#[test] +fn split_response_markdown_uses_horizontal_rule_marker() { + // The confirmed backend marker is `\n---\n`. Three messages → + // expect three slices when there's no preamble. + let md = "## Alice's update\n\nbody A with https://gh.io/abc\n---\n## Bob's reply\n\nbody B\n---\n## Carol\n\nbody C"; + let slices = super::split_response_markdown_per_message(md, 3).unwrap(); + assert_eq!(slices.len(), 3); + assert!(slices[0].contains("Alice's update")); + assert!(slices[1].contains("Bob's reply")); + assert!(slices[2].contains("Carol")); + // The `---\n` prefix is preserved on every-but-the-first segment + // so the section break survives the round-trip. + assert!(slices[1].starts_with("---\n")); + assert!(slices[2].starts_with("---\n")); +} + +#[test] +fn split_response_markdown_drops_preamble() { + // When a preamble like `# Inbox` precedes the first marker, we + // see N+1 parts after split — the preamble must be dropped. + let md = "# Inbox (2 messages)\n---\n## A\n\nbody A\n---\n## B\n\nbody B"; + let slices = super::split_response_markdown_per_message(md, 2).unwrap(); + assert_eq!(slices.len(), 2); + assert!(slices[0].contains("body A")); + assert!(slices[1].contains("body B")); + // Both segments should carry the prefix when preamble was dropped. + assert!(slices[0].starts_with("---\n")); + assert!(slices[1].starts_with("---\n")); +} + +#[test] +fn split_response_markdown_falls_back_to_h2_marker() { + // No `---` rules — backend used h2 headings as boundaries. + let md = "## Alice\n\nbody A\n\n## Bob\n\nbody B"; + let slices = super::split_response_markdown_per_message(md, 2).unwrap(); + assert_eq!(slices.len(), 2); + assert!(slices[0].contains("body A")); + assert!(slices[1].contains("body B")); +} + +#[test] +fn split_response_markdown_returns_none_on_count_mismatch() { + let md = "## only one section here"; + assert!(super::split_response_markdown_per_message(md, 3).is_none()); +} + +#[test] +fn split_response_markdown_single_message_returns_whole_input() { + let md = "## solo\n\nthe whole body"; + let slices = super::split_response_markdown_per_message(md, 1).unwrap(); + assert_eq!(slices, vec![md.to_string()]); +} + +#[test] +fn split_with_hint_rejects_when_subjects_dont_match() { + let md = "## Foo\nbody1\n---\n## Bar\nbody2"; + let hints = vec![ + json!({"subject": "Completely different subject A"}), + json!({"subject": "Completely different subject B"}), + ]; + let out = super::split_response_markdown_per_message_with_hint(md, 2, Some(&hints)); + assert!(out.is_none(), "subject mismatch must force fallback"); +} + +#[test] +fn split_with_hint_accepts_when_subjects_match() { + let md = "## Welcome to Gmail\nbody1\n---\n## Your invoice\nbody2"; + let hints = vec![ + json!({"subject": "Welcome to Gmail"}), + json!({"subject": "Your invoice"}), + ]; + let slices = super::split_response_markdown_per_message_with_hint(md, 2, Some(&hints)).unwrap(); + assert_eq!(slices.len(), 2); + assert!(slices[0].contains("Welcome to Gmail")); + assert!(slices[1].contains("Your invoice")); +} + +#[test] +fn split_with_hint_skips_messages_with_blank_subject() { + let md = "## A\nbody1\n---\n## B\nbody2"; + let hints = vec![json!({"subject": "A"}), json!({"subject": ""})]; + let slices = super::split_response_markdown_per_message_with_hint(md, 2, Some(&hints)).unwrap(); + assert_eq!(slices.len(), 2); +} + +// ── format_email_local_time ────────────────────────────────────────────────── + +#[test] +fn format_email_local_time_returns_none_for_unparseable_date() { + assert!(super::format_email_local_time("not-a-date").is_none()); + assert!(super::format_email_local_time("").is_none()); +} + +#[test] +fn format_email_local_time_preserves_utc_raw_date_in_reshape() { + let mut v = json!({ + "messages": [{ + "messageId": "m1", + "threadId": "t1", + "subject": "Test", + "sender": "a@example.com", + "to": "b@example.com", + "messageTimestamp": "2026-05-31T10:33:00Z", + "labelIds": [], + "messageText": "body", + "payload": {} + }] + }); + post_process("GMAIL_FETCH_EMAILS", None, &mut v); + let msg = &v["messages"][0]; + assert_eq!(msg["date"], "2026-05-31T10:33:00Z"); +} + +#[test] +fn parse_email_date_accepts_rfc3339_and_rfc2822() { + assert!(super::parse_email_date("2026-05-31T10:33:00Z").is_some()); + assert!(super::parse_email_date("Sun, 31 May 2026 10:33:00 +0000").is_some()); + assert!(super::parse_email_date("not-a-date").is_none()); +} + +#[test] +fn format_at_tz_deterministic_with_fixed_offset() { + use chrono::FixedOffset; + + let utc = super::parse_email_date("2026-05-31T10:33:00Z").unwrap(); + + let est = FixedOffset::west_opt(5 * 3600).unwrap(); + let result = super::format_at_tz(utc, &est).unwrap(); + assert_eq!(result, "2026-05-31 05:33 AM -05:00"); + + let ist = FixedOffset::east_opt(5 * 3600 + 1800).unwrap(); + let result = super::format_at_tz(utc, &ist).unwrap(); + assert_eq!(result, "2026-05-31 04:03 PM +05:30"); +} + +#[test] +fn format_at_tz_returns_none_for_utc() { + let utc = super::parse_email_date("2026-05-31T10:33:00Z").unwrap(); + let utc_tz = chrono::FixedOffset::east_opt(0).unwrap(); + assert!(super::format_at_tz(utc, &utc_tz).is_none()); +} + +#[test] +fn apply_response_level_markdown_stashes_per_message_field() { + let mut data = json!({ + "messages": [ + {"messageId": "m1", "subject": "Hello"}, + {"messageId": "m2", "subject": "World"}, + ] + }); + let top_md = "## Hello\nbody A — link https://gh.io/abc\n---\n## World\nbody B"; + super::apply_response_level_markdown(&mut data, top_md); + let m1 = data["messages"][0]["markdownFormatted"].as_str().unwrap(); + let m2 = data["messages"][1]["markdownFormatted"].as_str().unwrap(); + assert!(m1.contains("Hello")); + assert!( + m1.contains("https://gh.io/abc"), + "shortened URL must survive" + ); + assert!(m2.contains("World")); + assert!(!m1.contains("World"), "no cross-message bleed"); +} diff --git a/sync/src/helpers.rs b/sync/src/helpers.rs new file mode 100644 index 0000000..101239e --- /dev/null +++ b/sync/src/helpers.rs @@ -0,0 +1,50 @@ +//! Shared helpers for the provider normalisers in this module. + +/// Walk a JSON object using a list of dotted-path candidates and return the +/// first non-empty **string** match. +/// +/// # This is deliberately NOT `super::super::common::pick_str` +/// +/// The crate carries two `pick_str` functions with the same name and +/// genuinely different behaviour. Do not "deduplicate" them: +/// +/// | | this one (`normalize::helpers`) | `common::pick_str` | +/// |---|---|---| +/// | traversal | `Value::get` per `.`-separated segment — objects only | `Value::pointer` — also indexes into arrays | +/// | non-string leaf | rejected, returns `None` | `Number` is coerced via `to_string()` | +/// +/// The number case is the one that bites. A payload whose `id` is `42` +/// rather than `"42"` yields `None` here and `Some("42")` there, which +/// silently changes what a normaliser emits as a document id. The callers of +/// this function were written against the reject-non-strings behaviour and +/// have a test pinning it (`pick_str_rejects_non_string_values` below, and +/// the host-side mirror of it). +pub fn pick_str(value: &serde_json::Value, paths: &[&str]) -> Option { + for path in paths { + let mut cur = value; + let mut ok = true; + for segment in path.split('.') { + match cur.get(segment) { + Some(next) => cur = next, + None => { + ok = false; + break; + } + } + } + if !ok { + continue; + } + if let Some(s) = cur.as_str() { + let trimmed = s.trim(); + if !trimmed.is_empty() { + return Some(trimmed.to_string()); + } + } + } + None +} + +#[cfg(test)] +#[path = "helpers_tests.rs"] +mod tests; diff --git a/sync/src/helpers_tests.rs b/sync/src/helpers_tests.rs new file mode 100644 index 0000000..b6410b8 --- /dev/null +++ b/sync/src/helpers_tests.rs @@ -0,0 +1,40 @@ +#![allow(clippy::expect_used, clippy::panic, clippy::unwrap_used)] +// +// A failing assertion in a test *is* a panic. The crate-wide lints exist to +// keep the library from panicking, not the tests. + +use super::*; +use serde_json::json; + +#[test] +fn pick_str_finds_first_non_empty_match() { + let v = json!({"data": {"user": {"name": "Ada", "email": "ada@example.com"}}}); + assert_eq!( + pick_str(&v, &["data.user.name", "data.user.email"]), + Some("Ada".into()) + ); + assert_eq!( + pick_str(&v, &["data.missing", "data.user.email"]), + Some("ada@example.com".into()) + ); + assert_eq!(pick_str(&v, &["nope.nope"]), None); +} + +#[test] +fn pick_str_respects_path_order() { + let v = json!({"a": "first", "b": "second"}); + assert_eq!(pick_str(&v, &["a", "b"]), Some("first".into())); + assert_eq!(pick_str(&v, &["b", "a"]), Some("second".into())); +} + +/// The drift guard for the divergence documented on [`pick_str`]. If this +/// ever starts returning `Some("42")`, someone has re-pointed the +/// normalisers at `common::pick_str` and changed their output. +#[test] +fn pick_str_rejects_non_string_values() { + let v = json!({"count": 42, "flag": true, "empty": "", "whitespace": " "}); + assert_eq!(pick_str(&v, &["count"]), None); + assert_eq!(pick_str(&v, &["flag"]), None); + assert_eq!(pick_str(&v, &["empty"]), None); + assert_eq!(pick_str(&v, &["whitespace"]), None); +} diff --git a/sync/src/lib.rs b/sync/src/lib.rs new file mode 100644 index 0000000..4eb3ba6 --- /dev/null +++ b/sync/src/lib.rs @@ -0,0 +1,38 @@ +//! Composio provider payload normalisers, engine-neutral by construction. +//! +//! Issue #18 §B3: "Payload normalisers … are pure `Value → Value` transforms +//! with no engine dependency. Move them back into a `tinymemory-sync` crate … +//! so a non-TinyCortex engine gets Composio sync for free." +//! +//! They lived inside the TinyCortex engine, and `tinymemory-core` reached into +//! it to use them — which meant a host binding a *different* memory engine +//! could not have Composio sync at all, despite none of this code caring which +//! engine is bound. Nothing here reads a database, opens a socket, or names an +//! engine type; the dependency list is `serde_json`, two logging facades, and +//! `chrono`. +//! +//! One caveat on "pure", because it is load-bearing and easy to miss. +//! [`gmail_post_process::format_email_local_time`] renders in `chrono::Local`, +//! so it reads the host's timezone — every other normaliser here is a function +//! of its input alone. The raw UTC field is preserved alongside it, so sorting +//! and deduplication stay UTC-based; what varies by host is only the +//! presentation string. +//! +//! These are pure `serde_json::Value` → `Value` transforms: given a raw +//! Composio action response, pull out the fields that make up a task, an +//! issue, a page or a message. They hold no credentials, touch no network, +//! and make no scheduling decisions — provider-specific normalisation is +//! driver-side by definition (see the host's `docs/specs/kernel.md` §4). +//! + +pub mod clickup; +pub mod github; +pub mod helpers; +pub mod linear; +pub mod notion; + +// The `_post_process` suffix is kept from the engine layout it came from, where +// `slack.rs` and `github.rs` one directory up already held those names. Renaming +// on the way out would have made this a rename *and* a move in one diff. +pub mod gmail_post_process; +pub mod slack_post_process; diff --git a/sync/src/linear.rs b/sync/src/linear.rs new file mode 100644 index 0000000..843f98a --- /dev/null +++ b/sync/src/linear.rs @@ -0,0 +1,157 @@ +//! Linear host normalization helpers — result extraction, issue-title extraction, +//! viewer identity, cursor extraction, and time utilities. +//! +//! Linear's GraphQL API (and therefore Composio's wrapping of it) returns +//! connection-style lists (`{ nodes: [...], pageInfo: {...} }`) at the top +//! level or nested under `data`. The functions here walk the union of +//! common shapes so the provider does not have to branch per Composio +//! envelope variant. + +use serde_json::Value; + +use super::helpers::pick_str; + +/// Walk the Composio response envelope for Linear issue list results. +/// +/// Linear's list endpoints return `{ nodes: [...] }` or +/// `{ issues: { nodes: [...] } }` shapes; Composio may re-wrap the +/// upstream payload under `data` or `data.data`. We probe each shape +/// in order and return the first array we find. +pub fn extract_issues(data: &Value) -> Vec { + let candidates = [ + data.pointer("/data/nodes"), + data.pointer("/nodes"), + data.pointer("/data/issues/nodes"), + data.pointer("/issues/nodes"), + data.pointer("/data/data/nodes"), + data.pointer("/data/data/issues/nodes"), + data.pointer("/data/results"), + data.pointer("/results"), + data.pointer("/data/items"), + data.pointer("/items"), + ]; + for cand in candidates.into_iter().flatten() { + if let Some(arr) = cand.as_array() { + return arr.clone(); + } + } + Vec::new() +} + +/// Extract a human-readable title from a Linear issue object. +/// +/// Linear issues store the name at `title` (or `data.title` after +/// Composio envelope wrapping). Falls back to `name` / `identifier` +/// so the chunk remains identifiable even for unusual response shapes. +pub fn extract_issue_title(issue: &Value) -> Option { + pick_str( + issue, + &[ + "title", + "data.title", + "name", + "data.name", + "identifier", + "data.identifier", + ], + ) +} + +/// Extract a stable cursor timestamp from a Linear issue object. +/// +/// Linear uses ISO-8601 strings for timestamps (`updatedAt`). We keep +/// the value as a string so lexicographic comparison against the stored +/// cursor is valid. +pub fn extract_issue_updated(issue: &Value) -> Option { + pick_str( + issue, + &[ + "updatedAt", + "data.updatedAt", + "updated_at", + "data.updated_at", + ], + ) +} + +/// Extract the viewer (authenticated user) object from a +/// `LINEAR_LIST_LINEAR_USERS { isMe: true }` response. +/// +/// Linear's GraphQL viewer endpoint returns `{ nodes: [{ id, email, … }] }`. +/// Composio may wrap this under `data` or `data.data`. We probe each +/// shape and return the first element of the nodes array, falling back +/// to the payload itself if it looks like a direct user object (has +/// `id` or `email`). +pub fn extract_viewer(data: &Value) -> Option { + let array_candidates = [ + data.pointer("/data/nodes"), + data.pointer("/nodes"), + data.pointer("/data/data/nodes"), + data.pointer("/data/users/nodes"), + ]; + for cand in array_candidates.into_iter().flatten() { + if let Some(arr) = cand.as_array() { + if let Some(first) = arr.first() { + return Some(first.clone()); + } + } + } + // Fallback: if the payload itself looks like a user object, return it. + if data.get("id").is_some() || data.get("email").is_some() { + return Some(data.clone()); + } + None +} + +/// Extract the viewer's ID string from a `LINEAR_LIST_LINEAR_USERS` +/// response. Returns `None` if the payload does not contain a +/// recognizable user ID. +pub fn extract_viewer_id(data: &Value) -> Option { + let viewer = extract_viewer(data)?; + pick_str(&viewer, &["id", "data.id"]) +} + +/// Extract a pagination cursor from a Linear connection `pageInfo` block. +/// +/// Returns `Some(endCursor)` only when `hasNextPage` is `true`; +/// `None` when the last page has been reached or when the envelope does +/// not carry `pageInfo` at all. +pub fn extract_pagination_cursor(data: &Value) -> Option { + // Mirrors the `extract_issues` envelope shapes, so every shape that can + // carry a node list can also carry its `pageInfo` cursor. + let page_info_candidates = [ + data.pointer("/data/pageInfo"), + data.pointer("/pageInfo"), + data.pointer("/data/data/pageInfo"), + data.pointer("/data/issues/pageInfo"), + data.pointer("/data/data/issues/pageInfo"), + ]; + for cand in page_info_candidates.into_iter().flatten() { + let has_next = cand + .get("hasNextPage") + .and_then(|v| v.as_bool()) + .unwrap_or(false); + if has_next { + if let Some(cursor) = cand.get("endCursor").and_then(|v| v.as_str()) { + let trimmed = cursor.trim(); + if !trimmed.is_empty() { + return Some(trimmed.to_string()); + } + } + } + } + None +} + +/// Current wall-clock time in milliseconds since the UNIX epoch. +pub fn now_ms() -> u64 { + use std::time::{SystemTime, UNIX_EPOCH}; + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|d| d.as_millis() as u64) + .unwrap_or(0) +} + +#[cfg(test)] +#[path = "linear_tests.rs"] +mod tests; diff --git a/sync/src/linear_tests.rs b/sync/src/linear_tests.rs new file mode 100644 index 0000000..78b5bf1 --- /dev/null +++ b/sync/src/linear_tests.rs @@ -0,0 +1,189 @@ +#![allow(clippy::expect_used, clippy::panic, clippy::unwrap_used)] +// +// A failing assertion in a test *is* a panic. The crate-wide lints exist to +// keep the library from panicking, not the tests. + +use super::*; +use serde_json::json; + +// ── extract_issues ─────────────────────────────────────────────── + +#[test] +fn extract_issues_from_data_nodes() { + let data = json!({ "data": { "nodes": [{"id": "i1"}, {"id": "i2"}] } }); + assert_eq!(extract_issues(&data).len(), 2); +} + +#[test] +fn extract_issues_from_top_level_nodes() { + let data = json!({ "nodes": [{"id": "i3"}] }); + assert_eq!(extract_issues(&data).len(), 1); +} + +#[test] +fn extract_issues_from_data_issues_nodes() { + let data = + json!({ "data": { "issues": { "nodes": [{"id": "i4"}, {"id": "i5"}, {"id": "i6"}] } } }); + assert_eq!(extract_issues(&data).len(), 3); +} + +#[test] +fn extract_issues_from_top_level_issues_nodes() { + let data = json!({ "issues": { "nodes": [{"id": "i7"}] } }); + assert_eq!(extract_issues(&data).len(), 1); +} + +#[test] +fn extract_issues_from_doubly_nested_issues_nodes() { + let data = + json!({ "data": { "data": { "issues": { "nodes": [{"id": "i8"}, {"id": "i9"}] } } } }); + assert_eq!(extract_issues(&data).len(), 2); +} + +#[test] +fn extract_issues_from_results() { + let data = json!({ "results": [{"id": "i7"}] }); + assert_eq!(extract_issues(&data).len(), 1); +} + +#[test] +fn extract_issues_empty_when_missing() { + let data = json!({ "foo": "bar" }); + assert!(extract_issues(&data).is_empty()); +} + +// ── extract_issue_title ────────────────────────────────────────── + +#[test] +fn extract_issue_title_from_title_field() { + let issue = json!({ "id": "i1", "title": "Fix the login bug" }); + assert_eq!( + extract_issue_title(&issue), + Some("Fix the login bug".into()) + ); +} + +#[test] +fn extract_issue_title_falls_back_to_wrapped_data() { + let issue = json!({ "data": { "title": "Wrapped issue" } }); + assert_eq!(extract_issue_title(&issue), Some("Wrapped issue".into())); +} + +#[test] +fn extract_issue_title_falls_back_to_identifier() { + let issue = json!({ "identifier": "ENG-42" }); + assert_eq!(extract_issue_title(&issue), Some("ENG-42".into())); +} + +// ── extract_issue_updated ──────────────────────────────────────── + +#[test] +fn extract_issue_updated_from_updated_at() { + let issue = json!({ "updatedAt": "2026-03-01T12:00:00.000Z" }); + assert_eq!( + extract_issue_updated(&issue), + Some("2026-03-01T12:00:00.000Z".to_string()) + ); +} + +#[test] +fn extract_issue_updated_falls_back_to_snake_case() { + let issue = json!({ "data": { "updated_at": "2026-01-15T08:30:00.000Z" } }); + assert_eq!( + extract_issue_updated(&issue), + Some("2026-01-15T08:30:00.000Z".to_string()) + ); +} + +// ── extract_viewer ─────────────────────────────────────────────── + +#[test] +fn extract_viewer_from_data_nodes() { + let data = json!({ "data": { "nodes": [{ "id": "usr_1", "email": "a@b.com" }] } }); + let v = extract_viewer(&data).expect("should find viewer"); + assert_eq!(v["id"], "usr_1"); +} + +#[test] +fn extract_viewer_from_top_level_nodes() { + let data = json!({ "nodes": [{ "id": "usr_2" }] }); + let v = extract_viewer(&data).expect("should find viewer"); + assert_eq!(v["id"], "usr_2"); +} + +#[test] +fn extract_viewer_fallback_direct_object() { + let data = json!({ "id": "usr_direct", "name": "Direct User" }); + let v = extract_viewer(&data).expect("should return direct object"); + assert_eq!(v["id"], "usr_direct"); +} + +#[test] +fn extract_viewer_returns_none_when_absent() { + let data = json!({ "foo": "bar" }); + assert!(extract_viewer(&data).is_none()); +} + +// ── extract_pagination_cursor ──────────────────────────────────── + +#[test] +fn extract_pagination_cursor_returns_cursor_when_has_next_page() { + let data = json!({ + "data": { + "pageInfo": { + "hasNextPage": true, + "endCursor": "cursor_abc" + } + } + }); + assert_eq!( + extract_pagination_cursor(&data), + Some("cursor_abc".to_string()) + ); +} + +#[test] +fn extract_pagination_cursor_returns_none_when_last_page() { + let data = json!({ + "pageInfo": { + "hasNextPage": false, + "endCursor": "cursor_xyz" + } + }); + assert!(extract_pagination_cursor(&data).is_none()); +} + +#[test] +fn extract_pagination_cursor_from_doubly_nested_issues() { + // The same `data.data.issues` shape `extract_issues` reads must also + // expose its pageInfo cursor, or a doubly-nested payload never pages. + let data = json!({ + "data": { + "data": { + "issues": { + "pageInfo": { + "hasNextPage": true, + "endCursor": "cursor_issue_2" + } + } + } + } + }); + assert_eq!( + extract_pagination_cursor(&data), + Some("cursor_issue_2".to_string()) + ); +} + +#[test] +fn extract_pagination_cursor_returns_none_when_absent() { + let data = json!({ "nodes": [{"id": "i1"}] }); + assert!(extract_pagination_cursor(&data).is_none()); +} + +// ── now_ms ─────────────────────────────────────────────────────── + +#[test] +fn now_ms_returns_nonzero() { + assert!(now_ms() > 0); +} diff --git a/sync/src/notion.rs b/sync/src/notion.rs new file mode 100644 index 0000000..f13c17f --- /dev/null +++ b/sync/src/notion.rs @@ -0,0 +1,120 @@ +//! Notion host normalization helpers — result extraction, pagination cursor, +//! page title extraction, and time utilities. + +use serde_json::Value; + +use super::helpers::pick_str; + +/// Walk the Composio response envelope for Notion page results. +pub fn extract_results(data: &Value) -> Vec { + let candidates = [ + data.pointer("/data/results"), + data.pointer("/results"), + data.pointer("/data/data/results"), + data.pointer("/data/items"), + data.pointer("/items"), + ]; + for cand in candidates.into_iter().flatten() { + if let Some(arr) = cand.as_array() { + return arr.clone(); + } + } + Vec::new() +} + +/// Extract the rendered page body markdown from a `NOTION_GET_PAGE_MARKDOWN` +/// response. Composio wraps action output in varying envelope shapes, so we +/// try the common locations tolerantly and return the first non-empty string. +/// Returns `None` if no markdown field is found (caller falls back to the +/// metadata-only body and logs the raw shape for diagnosis). +pub fn extract_page_markdown(data: &Value) -> Option { + const PATHS: &[&str] = &[ + "/markdown", + "/data/markdown", + "/data/response_data/markdown", + "/response_data/markdown", + "/data/content", + "/content", + "/data/markdown_content", + "/markdown_content", + "/text", + "/data/text", + ]; + for p in PATHS { + if let Some(s) = data.pointer(p).and_then(Value::as_str) { + if !s.trim().is_empty() { + return Some(s.to_string()); + } + } + } + None +} + +/// Extract the Notion pagination cursor (for `start_cursor` on the +/// next request). +pub fn extract_notion_cursor(data: &Value) -> Option { + let candidates = [ + data.pointer("/data/next_cursor"), + data.pointer("/next_cursor"), + data.pointer("/data/data/next_cursor"), + ]; + for cand in candidates.into_iter().flatten() { + if let Some(s) = cand.as_str() { + let trimmed = s.trim(); + if !trimmed.is_empty() { + return Some(trimmed.to_string()); + } + } + } + None +} + +/// Try to extract a human-readable title from a Notion page object. +/// +/// Notion pages store the title in `properties.title` or +/// `properties.Name.title[0].plain_text`. We try several shapes. +pub fn extract_page_title(page: &Value) -> Option { + // Try the common `properties.title.title[0].plain_text` shape. + let props = page + .get("properties") + .or_else(|| page.get("data")?.get("properties")); + if let Some(props) = props { + // Walk all properties looking for a "title" type field. + if let Some(obj) = props.as_object() { + for (_key, val) in obj { + if val.get("type").and_then(Value::as_str) == Some("title") { + if let Some(arr) = val.get("title").and_then(Value::as_array) { + let text: String = arr + .iter() + .filter_map(|t| t.get("plain_text").and_then(Value::as_str)) + .collect::>() + .join(""); + if !text.is_empty() { + return Some(text); + } + } + } + } + } + } + + // Fallback: top-level "title" field (some Composio shapes). + pick_str(page, &["title", "data.title", "name", "data.name"]) +} + +/// Milliseconds since the Unix epoch. +/// +/// The one clock read in this crate. Notion payloads carry no ingestion +/// timestamp, so the normaliser stamps one; everything else here is a function +/// of its input alone. +pub fn now_ms() -> u64 { + use std::time::{SystemTime, UNIX_EPOCH}; + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|d| d.as_millis() as u64) + .unwrap_or(0) +} + +#[cfg(test)] +#[path = "notion_tests.rs"] +mod tests; diff --git a/sync/src/notion_tests.rs b/sync/src/notion_tests.rs new file mode 100644 index 0000000..84b46e1 --- /dev/null +++ b/sync/src/notion_tests.rs @@ -0,0 +1,142 @@ +#![allow(clippy::expect_used, clippy::panic, clippy::unwrap_used)] +// +// A failing assertion in a test *is* a panic. The crate-wide lints exist to +// keep the library from panicking, not the tests. + +use super::*; +use serde_json::json; + +#[test] +fn extract_results_from_data_results() { + let data = json!({"data": {"results": [{"id": "page1"}]}}); + let results = extract_results(&data); + assert_eq!(results.len(), 1); +} + +#[test] +fn extract_page_markdown_reads_top_level_field() { + // Matches the live GET_PAGE_MARKDOWN envelope observed empirically: + // {id, markdown, object, request_id, truncated, unknown_block_ids}. + let data = json!({ + "id": "p1", + "markdown": "# Heading\n\nbody text", + "object": "page", + "truncated": false, + }); + assert_eq!( + extract_page_markdown(&data).as_deref(), + Some("# Heading\n\nbody text") + ); +} + +#[test] +fn extract_page_markdown_reads_nested_envelope() { + let data = json!({ "data": { "markdown": "nested body" } }); + assert_eq!(extract_page_markdown(&data).as_deref(), Some("nested body")); +} + +#[test] +fn extract_page_markdown_none_for_empty_or_missing() { + // Empty markdown (a DB row with no body blocks) → None → metadata-only. + assert_eq!(extract_page_markdown(&json!({ "markdown": "" })), None); + assert_eq!(extract_page_markdown(&json!({ "markdown": " " })), None); + // No markdown field at all → None. + assert_eq!(extract_page_markdown(&json!({ "id": "p1" })), None); +} + +#[test] +fn extract_results_from_top_level() { + let data = json!({"results": [{"id": "a"}, {"id": "b"}]}); + let results = extract_results(&data); + assert_eq!(results.len(), 2); +} + +#[test] +fn extract_results_from_data_items() { + let data = json!({"data": {"items": [{"id": "x"}]}}); + let results = extract_results(&data); + assert_eq!(results.len(), 1); +} + +#[test] +fn extract_results_empty_when_no_match() { + let data = json!({"foo": "bar"}); + assert!(extract_results(&data).is_empty()); +} + +#[test] +fn extract_notion_cursor_from_data() { + let data = json!({"data": {"next_cursor": "cur123"}}); + assert_eq!(extract_notion_cursor(&data), Some("cur123".into())); +} + +#[test] +fn extract_notion_cursor_from_top_level() { + let data = json!({"next_cursor": "abc"}); + assert_eq!(extract_notion_cursor(&data), Some("abc".into())); +} + +#[test] +fn extract_notion_cursor_none_when_empty() { + let data = json!({"data": {"next_cursor": " "}}); + assert_eq!(extract_notion_cursor(&data), None); +} + +#[test] +fn extract_notion_cursor_none_when_missing() { + assert_eq!(extract_notion_cursor(&json!({})), None); +} + +#[test] +fn extract_page_title_from_properties_title_type() { + let page = json!({ + "properties": { + "Name": { + "type": "title", + "title": [{"plain_text": "Hello"}, {"plain_text": " World"}] + } + } + }); + assert_eq!(extract_page_title(&page), Some("Hello World".into())); +} + +#[test] +fn extract_page_title_from_nested_data_properties() { + let page = json!({ + "data": { + "properties": { + "Title": { + "type": "title", + "title": [{"plain_text": "My Page"}] + } + } + } + }); + assert_eq!(extract_page_title(&page), Some("My Page".into())); +} + +#[test] +fn extract_page_title_fallback_to_top_level_title() { + let page = json!({"title": "Fallback Title"}); + assert_eq!(extract_page_title(&page), Some("Fallback Title".into())); +} + +#[test] +fn extract_page_title_none_when_empty() { + let page = json!({"properties": {"Name": {"type": "title", "title": []}}}); + // Empty title array means no text + assert!( + extract_page_title(&page).is_none() || extract_page_title(&page) == Some(String::new()) + ); +} + +#[test] +fn extract_page_title_none_when_no_title_field() { + let page = json!({"id": "123"}); + assert!(extract_page_title(&page).is_none()); +} + +#[test] +fn now_ms_returns_nonzero() { + assert!(now_ms() > 0); +} diff --git a/sync/src/slack_post_process.rs b/sync/src/slack_post_process.rs new file mode 100644 index 0000000..28ce30b --- /dev/null +++ b/sync/src/slack_post_process.rs @@ -0,0 +1,323 @@ +//! Slack-specific post-processing of Composio action responses. +//! +//! Composio's Slack responses are verbose API envelopes. This module +//! rewrites each supported action's response into a slim, stable shape +//! that the ingest pipeline and enrichers can consume without walking +//! Composio's unstable nested envelopes. +//! +//! ## Supported slugs +//! +//! - `SLACK_FETCH_CONVERSATION_HISTORY` — reshapes into top-level +//! `messages[]` with `{ ts, user, text, thread_ts, channel_id }`. +//! Empty-text messages are dropped. `channel_id` is absent here (it's +//! in the request, not the response); the caller injects it via the +//! enricher in the host's `SlackSyncPipeline`. +//! +//! - `SLACK_LIST_CONVERSATIONS` — reshapes into top-level `channels[]` +//! with `{ id, name, is_private }` per channel. Entries with an empty +//! id are dropped. +//! +//! - `SLACK_SEARCH_MESSAGES` — reshapes `messages.matches[]` (possibly +//! nested) into top-level `messages[]` with `{ ts, user, text, +//! thread_ts, channel_id }`. `channel_id` is pulled from each match's +//! `channel.id` field. `paging.pages` is preserved at top-level for +//! caller pagination. +//! +//! ## Design note: user-id resolution is NOT here +//! +//! `SlackUsers` is a per-sync cache built from a separate API call — +//! not a function of any individual response. Resolving user ids +//! happens in the host's `SlackSyncPipeline` (the enricher layer), keeping +//! this module purely data-shape–oriented. +//! This matches Gmail's pattern of "post_process is data-only". +//! +//! Unknown slugs are silently no-ops so new Composio actions don't +//! break the provider. + +use serde_json::{Map, Value}; + +/// Entry point called from `SlackProvider::post_process_action_result`. +/// +/// Dispatches on the Composio action slug and rewrites `data` in place. +/// Unknown slugs are silently ignored. +pub fn post_process(slug: &str, _arguments: Option<&Value>, data: &mut Value) { + log::debug!("[composio:slack][post-process] slug={slug}"); + match slug { + "SLACK_FETCH_CONVERSATION_HISTORY" => reshape_fetch_history(data), + "SLACK_LIST_CONVERSATIONS" => reshape_list_conversations(data), + "SLACK_SEARCH_MESSAGES" => reshape_search_messages(data), + _ => { + log::debug!("[composio:slack][post-process] unknown slug={slug}, passing through"); + } + } +} + +// ─── SLACK_FETCH_CONVERSATION_HISTORY ────────────────────────────────────── + +/// Rewrite a `SLACK_FETCH_CONVERSATION_HISTORY` response in place. +/// +/// Walks possible nested envelopes (`/data/messages`, `/messages`, +/// `/data/data/messages`) to find the raw messages array, drops messages +/// with empty `text`, and emits a slim `{ ts, user, text, thread_ts }` +/// shape under a top-level `messages[]` key. The consumed nested array is +/// removed from the payload so the raw verbose rows don't linger alongside +/// the slim copy. The caller injects `channel_id` via +/// [`super::sync::extract_messages`]. +fn reshape_fetch_history(data: &mut Value) { + let arr = take_array( + data, + &["/data/messages", "/messages", "/data/data/messages"], + 0, + ); + let slim: Vec = arr.into_iter().filter_map(slim_history_message).collect(); + let obj = ensure_object(data); + obj.insert("messages".to_string(), Value::Array(slim)); + log::debug!("[composio:slack][post-process] SLACK_FETCH_CONVERSATION_HISTORY reshaped"); +} + +fn slim_history_message(raw: Value) -> Option { + let text = raw + .get("text") + .and_then(|v| v.as_str()) + .unwrap_or("") + .trim(); + if text.is_empty() { + return None; + } + let mut out = Map::new(); + // `ts` is required: without it a caller can neither cursor nor archive. + out.insert("ts".into(), raw.get("ts")?.clone()); + if let Some(user) = raw.get("user").or_else(|| raw.get("bot_id")) { + out.insert("user".into(), user.clone()); + } + out.insert("text".into(), Value::String(text.to_string())); + if let Some(thread_ts) = raw.get("thread_ts") { + out.insert("thread_ts".into(), thread_ts.clone()); + } + if let Some(permalink) = raw.get("permalink") { + out.insert("permalink".into(), permalink.clone()); + } + Some(Value::Object(out)) +} + +/// Find the first array at any of `candidates`, remove that field (plus +/// `envelope_depth` ancestor object envelopes) from `data`, and return the +/// array. Removing the consumed nested payload keeps the reshaped output from +/// carrying duplicate raw rows. +fn take_array(data: &mut Value, candidates: &[&str], envelope_depth: usize) -> Vec { + for path in candidates { + let arr = match data.pointer(path).and_then(|v| v.as_array().cloned()) { + Some(a) => a, + None => continue, + }; + let mut remove_path = path.to_string(); + for _ in 0..envelope_depth { + remove_path = match remove_path.rsplit_once('/') { + Some((parent, _)) => parent.to_string(), + None => break, + }; + } + remove_nested(data, &remove_path); + return arr; + } + Vec::new() +} + +/// Remove the field at `path` from `data`, pruning any ancestor object that +/// the removal left empty so a consumed `data` envelope disappears entirely +/// instead of lingering as `{}`. +fn remove_nested(data: &mut Value, path: &str) { + let segments: Vec<&str> = path + .trim_start_matches('/') + .split('/') + .filter(|s| !s.is_empty()) + .collect(); + if segments.is_empty() { + return; + } + + // Remove the leaf field. + let mut current = &mut *data; + for seg in &segments[..segments.len() - 1] { + current = match current.get_mut(*seg) { + Some(next) => next, + None => return, + }; + } + if let Value::Object(map) = current { + map.remove(segments[segments.len() - 1]); + } + + // Prune empty object ancestors, deepest first. + for depth in (0..segments.len().saturating_sub(1)).rev() { + // Re-walk to the object at `segments[..=depth]`. + let mut ancestor = &mut *data; + for seg in &segments[..=depth] { + ancestor = match ancestor.get_mut(*seg) { + Some(next) => next, + None => return, + }; + } + if !matches!(ancestor, Value::Object(m) if m.is_empty()) { + break; + } + // Remove it from its parent (`segments[..depth]`). For `depth == 0` + // the parent is the top-level object, so an emptied `data` envelope + // key disappears entirely. + let mut parent = &mut *data; + for seg in &segments[..depth] { + parent = match parent.get_mut(*seg) { + Some(next) => next, + None => return, + }; + } + if let Value::Object(map) = parent { + map.remove(segments[depth]); + } + } +} + +// ─── SLACK_LIST_CONVERSATIONS ─────────────────────────────────────────────── + +/// Rewrite a `SLACK_LIST_CONVERSATIONS` response in place. +/// +/// Reshapes into a top-level `channels[]` with `{ id, name, is_private }` +/// per channel; entries with an empty id are dropped. +fn reshape_list_conversations(data: &mut Value) { + let arr = take_array( + data, + &[ + "/data/channels", + "/channels", + "/data/data/channels", + "/data/conversations", + "/conversations", + ], + 0, + ); + + let slim: Vec = arr.into_iter().filter_map(slim_channel).collect(); + let obj = ensure_object(data); + obj.insert("channels".to_string(), Value::Array(slim)); + log::debug!("[composio:slack][post-process] SLACK_LIST_CONVERSATIONS reshaped"); +} + +fn slim_channel(raw: Value) -> Option { + let id = raw.get("id").and_then(|v| v.as_str()).unwrap_or("").trim(); + if id.is_empty() { + return None; + } + let name = raw + .get("name") + .and_then(|v| v.as_str()) + .unwrap_or(id) + .trim(); + let is_private = raw + .get("is_private") + .and_then(|v| v.as_bool()) + .unwrap_or(false); + Some(Value::Object({ + let mut m = Map::new(); + m.insert("id".into(), Value::String(id.to_string())); + m.insert("name".into(), Value::String(name.to_string())); + m.insert("is_private".into(), Value::Bool(is_private)); + m + })) +} + +// ─── SLACK_SEARCH_MESSAGES ────────────────────────────────────────────────── + +/// Rewrite a `SLACK_SEARCH_MESSAGES` response in place. +/// +/// Reshapes `messages.matches[]` (possibly nested under one or two +/// `data` envelopes) into top-level `messages[]`. `channel_id` is pulled +/// from each match's `channel.id` field. `paging.pages` is preserved at +/// top-level under `pages` for the caller to drive pagination. +fn reshape_search_messages(data: &mut Value) { + // Preserve paging info before mutating data (take_array below removes the + // envelope that carries it). + let pages = [ + data.pointer("/data/messages/paging/pages"), + data.pointer("/messages/paging/pages"), + data.pointer("/data/data/messages/paging/pages"), + ] + .into_iter() + .flatten() + .find_map(|v| v.as_u64()) + .unwrap_or(1); + + // Envelope depth 1 removes the `messages` object (matches + paging) that + // held the consumed rows, not just the `matches` array. + let arr = take_array( + data, + &[ + "/data/messages/matches", + "/messages/matches", + "/data/data/messages/matches", + ], + 1, + ); + + let slim: Vec = arr.into_iter().filter_map(slim_search_match).collect(); + let obj = ensure_object(data); + obj.insert("messages".to_string(), Value::Array(slim)); + obj.insert("pages".to_string(), Value::Number(pages.into())); + log::debug!("[composio:slack][post-process] SLACK_SEARCH_MESSAGES reshaped"); +} + +fn slim_search_match(raw: Value) -> Option { + let text = raw + .get("text") + .and_then(|v| v.as_str()) + .unwrap_or("") + .trim(); + if text.is_empty() { + return None; + } + let ts = raw.get("ts")?; + let channel_id = raw + .pointer("/channel/id") + .and_then(|v| v.as_str()) + .unwrap_or("") + .trim(); + + let mut out = Map::new(); + out.insert("ts".into(), ts.clone()); + if let Some(user) = raw.get("user").or_else(|| raw.get("bot_id")) { + out.insert("user".into(), user.clone()); + } + out.insert("text".into(), Value::String(text.to_string())); + if let Some(thread_ts) = raw.get("thread_ts") { + out.insert("thread_ts".into(), thread_ts.clone()); + } + if !channel_id.is_empty() { + out.insert("channel_id".into(), Value::String(channel_id.to_string())); + } + if let Some(permalink) = raw.get("permalink") { + out.insert("permalink".into(), permalink.clone()); + } + Some(Value::Object(out)) +} + +// ─── Helpers ──────────────────────────────────────────────────────────────── + +/// Ensure `data` is a JSON object, replacing it with an empty object if +/// not. Returns a mutable ref to the inner map. +// Scoped rather than blanket, for the case `AGENTS.md` names: "genuinely +// unreachable states — where `expect` must carry a message explaining the +// invariant." The line below assigns `Value::Object` whenever `data` is not +// one, so the read-back cannot fail; the compiler cannot see that across the +// assignment. The two `unwrap`s this crate inherited elsewhere were removed +// rather than allowed. +#[allow(clippy::expect_used)] +fn ensure_object(data: &mut Value) -> &mut Map { + if !data.is_object() { + *data = Value::Object(Map::new()); + } + data.as_object_mut() + .expect("assigned Value::Object immediately above when data was not one") +} + +#[cfg(test)] +#[path = "slack_post_process_tests.rs"] +mod tests; diff --git a/sync/src/slack_post_process_tests.rs b/sync/src/slack_post_process_tests.rs new file mode 100644 index 0000000..aeec63a --- /dev/null +++ b/sync/src/slack_post_process_tests.rs @@ -0,0 +1,262 @@ +#![allow(clippy::expect_used, clippy::panic, clippy::unwrap_used)] +// +// A failing assertion in a test *is* a panic. The crate-wide lints exist to +// keep the library from panicking, not the tests. + +use super::*; +use serde_json::json; + +// ─── SLACK_FETCH_CONVERSATION_HISTORY ───────────────────────────────────── + +#[test] +fn history_reshapes_top_level_messages() { + let mut data = json!({ + "messages": [ + { "ts": "1714003200.000100", "user": "U1", "text": "hello" }, + { "ts": "1714003300.000200", "user": "U2", "text": "world", "thread_ts": "1714003200.0" }, + { "ts": "1714003400.000300", "user": "U3", "text": " " }, // dropped: empty text + ], + "response_metadata": { "next_cursor": "abc" } + }); + post_process("SLACK_FETCH_CONVERSATION_HISTORY", None, &mut data); + + let msgs = data["messages"].as_array().unwrap(); + assert_eq!(msgs.len(), 2, "empty-text message must be dropped"); + assert_eq!(msgs[0]["ts"], "1714003200.000100"); + assert_eq!(msgs[0]["user"], "U1"); + assert_eq!(msgs[0]["text"], "hello"); + assert!(msgs[0].get("thread_ts").is_none()); + assert_eq!(msgs[1]["thread_ts"], "1714003200.0"); +} + +#[test] +fn history_reshapes_nested_data_envelope() { + let mut data = json!({ + "data": { + "messages": [ + { "ts": "1714003200.0", "user": "U1", "text": "hi" } + ] + } + }); + post_process("SLACK_FETCH_CONVERSATION_HISTORY", None, &mut data); + let msgs = data["messages"].as_array().unwrap(); + assert_eq!(msgs.len(), 1); + assert_eq!(msgs[0]["text"], "hi"); +} + +#[test] +fn history_reshapes_doubly_nested_envelope() { + let mut data = json!({ + "data": { + "data": { + "messages": [ + { "ts": "1714003200.0", "user": "U1", "text": "deep" } + ] + } + } + }); + post_process("SLACK_FETCH_CONVERSATION_HISTORY", None, &mut data); + let msgs = data["messages"].as_array().unwrap(); + assert_eq!(msgs.len(), 1); + assert_eq!(msgs[0]["text"], "deep"); +} + +#[test] +fn history_drops_message_without_ts() { + let mut data = json!({ + "messages": [ + { "user": "U1", "text": "no timestamp" }, + { "ts": "1714003200.0", "user": "U2", "text": "has ts" }, + ] + }); + post_process("SLACK_FETCH_CONVERSATION_HISTORY", None, &mut data); + let msgs = data["messages"].as_array().unwrap(); + assert_eq!(msgs.len(), 1); + assert_eq!(msgs[0]["text"], "has ts"); +} + +#[test] +fn history_removes_nested_envelope_after_reshape() { + let mut data = json!({ + "data": { + "messages": [ + { "ts": "1714003200.0", "user": "U1", "text": "hi" } + ] + } + }); + post_process("SLACK_FETCH_CONVERSATION_HISTORY", None, &mut data); + + let msgs = data["messages"].as_array().unwrap(); + assert_eq!(msgs.len(), 1); + assert_eq!(msgs[0]["text"], "hi"); + assert!( + data.pointer("/data").is_none(), + "consumed `data.messages` envelope must be removed, got: {data}" + ); +} + +// ─── SLACK_LIST_CONVERSATIONS ───────────────────────────────────────────── + +#[test] +fn list_conversations_reshapes_channels() { + let mut data = json!({ + "data": { + "channels": [ + { "id": "C1", "name": "eng", "is_private": false, "extra": "noise" }, + { "id": "G1", "name": "ops", "is_private": true }, + { "id": "", "name": "empty-id" }, // dropped + ] + } + }); + post_process("SLACK_LIST_CONVERSATIONS", None, &mut data); + let channels = data["channels"].as_array().unwrap(); + assert_eq!(channels.len(), 2, "empty-id entry must be dropped"); + assert_eq!(channels[0]["id"], "C1"); + assert_eq!(channels[0]["name"], "eng"); + assert_eq!(channels[0]["is_private"], false); + assert!( + channels[0].get("extra").is_none(), + "noise fields must be removed" + ); + assert_eq!(channels[1]["id"], "G1"); + assert_eq!(channels[1]["is_private"], true); +} + +#[test] +fn list_conversations_falls_back_to_conversations_key() { + let mut data = json!({ + "conversations": [ + { "id": "C2", "name": "dev", "is_private": false } + ] + }); + post_process("SLACK_LIST_CONVERSATIONS", None, &mut data); + let channels = data["channels"].as_array().unwrap(); + assert_eq!(channels.len(), 1); + assert_eq!(channels[0]["id"], "C2"); + assert!( + data.pointer("/conversations").is_none(), + "consumed `conversations` field must be removed" + ); +} + +// ─── SLACK_SEARCH_MESSAGES ──────────────────────────────────────────────── + +#[test] +fn search_messages_reshapes_matches() { + let mut data = json!({ + "messages": { + "matches": [ + { + "ts": "1714003200.0", + "user": "U1", + "text": "hello from search", + "channel": { "id": "C1" } + }, + { + "ts": "1714003300.0", + "user": "U2", + "text": " ", // dropped: whitespace only + "channel": { "id": "C1" } + }, + ], + "paging": { "pages": 3 } + } + }); + post_process("SLACK_SEARCH_MESSAGES", None, &mut data); + let msgs = data["messages"].as_array().unwrap(); + assert_eq!(msgs.len(), 1, "empty-text match must be dropped"); + assert_eq!(msgs[0]["ts"], "1714003200.0"); + assert_eq!(msgs[0]["text"], "hello from search"); + assert_eq!(msgs[0]["channel_id"], "C1"); + assert_eq!(data["pages"], 3, "paging.pages must be preserved"); +} + +#[test] +fn search_messages_nested_data_envelope() { + let mut data = json!({ + "data": { + "messages": { + "matches": [ + { "ts": "1714003200.0", "user": "U1", "text": "nested", "channel": { "id": "C2" } } + ], + "paging": { "pages": 1 } + } + } + }); + post_process("SLACK_SEARCH_MESSAGES", None, &mut data); + let msgs = data["messages"].as_array().unwrap(); + assert_eq!(msgs.len(), 1); + assert_eq!(msgs[0]["channel_id"], "C2"); + assert_eq!(data["pages"], 1_u64); +} + +#[test] +fn search_messages_no_matches_emits_empty_array() { + let mut data = json!({ "messages": { "matches": [] } }); + post_process("SLACK_SEARCH_MESSAGES", None, &mut data); + let msgs = data["messages"].as_array().unwrap(); + assert!(msgs.is_empty()); +} + +#[test] +fn search_messages_removes_nested_envelope_after_reshape() { + let mut data = json!({ + "data": { + "messages": { + "matches": [ + { "ts": "1714003200.0", "user": "U1", "text": "nested", "channel": { "id": "C2" } } + ], + "paging": { "pages": 1 } + } + } + }); + post_process("SLACK_SEARCH_MESSAGES", None, &mut data); + + let msgs = data["messages"].as_array().unwrap(); + assert_eq!(msgs.len(), 1); + assert_eq!(msgs[0]["channel_id"], "C2"); + assert_eq!(data["pages"], 1_u64); + assert!( + data.pointer("/data").is_none(), + "consumed `data.messages` envelope must be removed, got: {data}" + ); +} + +#[test] +fn search_messages_doubly_nested_paging_preserved() { + let mut data = json!({ + "data": { + "data": { + "messages": { + "matches": [ + { "ts": "1714003200.0", "user": "U1", "text": "deep", "channel": { "id": "C3" } } + ], + "paging": { "pages": 4 } + } + } + } + }); + post_process("SLACK_SEARCH_MESSAGES", None, &mut data); + + let msgs = data["messages"].as_array().unwrap(); + assert_eq!(msgs.len(), 1); + assert_eq!(msgs[0]["text"], "deep"); + assert_eq!( + data["pages"], 4_u64, + "doubly-nested paging must be preserved" + ); + assert!( + data.pointer("/data").is_none(), + "consumed `data.data.messages` envelope must be removed, got: {data}" + ); +} + +// ─── Unknown slug ───────────────────────────────────────────────────────── + +#[test] +fn unknown_slug_is_noop() { + let mut data = json!({ "foo": "bar" }); + let original = data.clone(); + post_process("SLACK_SEND_MESSAGE", None, &mut data); + assert_eq!(data, original, "unknown slug must not mutate data"); +} From 3d1faa3d2f445e6cb3a8c08ea686be09c021cfbb Mon Sep 17 00:00:00 2001 From: Shanu Date: Tue, 18 Aug 2026 16:35:16 +0530 Subject: [PATCH 9/9] Re-point the tinycortex gitlink at the merged commit tinycortex#149 landed as a squash (8401346b), discarding the branch head this pin pointed at; 34cbb6c is diverged from tinycortex main rather than an ancestor of it. The merged commit is also the one that deletes the 33 duplicated files under api/src/, which is the state this stack depends on. --- vendor/tinycortex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/tinycortex b/vendor/tinycortex index 34cbb6c..8401346 160000 --- a/vendor/tinycortex +++ b/vendor/tinycortex @@ -1 +1 @@ -Subproject commit 34cbb6cfa91ea74d62605bd57790782b0c748556 +Subproject commit 8401346b574cacb1dc0cf6b36bc608ff5ef9f6f5