diff --git a/Cargo.lock b/Cargo.lock index dd6b02f..5caf806 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1887,6 +1887,7 @@ dependencies = [ "sha2 0.10.9", "tinymemory", "tinymemory-api", + "tinymemory-conformance", "tokio", ] @@ -1903,6 +1904,7 @@ dependencies = [ "tinycortex", "tinymemory", "tinymemory-api", + "tinymemory-conformance", "tinymemory-core", "tokio", "uuid", diff --git a/adapters/remote/Cargo.toml b/adapters/remote/Cargo.toml index d78ff7e..3bb4da5 100644 --- a/adapters/remote/Cargo.toml +++ b/adapters/remote/Cargo.toml @@ -25,6 +25,9 @@ serde_json = "1" sha2 = "0.10" [dev-dependencies] +# The behavioural contract suite, run against these adapters over their own +# native doubles (issue #18 §E1, acceptance criterion 5). +tinymemory-conformance = { path = "../../conformance" } # Adapter tests run lightweight native-API doubles over a real TCP transport. axum = { version = "0.8", features = ["multipart"] } tokio = { version = "1", features = ["macros", "rt-multi-thread", "net"] } diff --git a/adapters/remote/src/conformance_test.rs b/adapters/remote/src/conformance_test.rs new file mode 100644 index 0000000..3694b63 --- /dev/null +++ b/adapters/remote/src/conformance_test.rs @@ -0,0 +1,497 @@ +//! The conformance suite, run against the hosted adapters. +//! +//! Issue #18's acceptance criterion 5: "the conformance suite passes for +//! TinyCortex and all three remote adapters". Until now it ran against the +//! in-memory reference driver and the null driver — both written alongside the +//! suite, so passing proved the assertions were self-consistent and little +//! else. +//! +//! These run the same `assert_provider` against the real adapters, over a real +//! TCP socket, against a double that speaks each vendor's own HTTP shapes and +//! **actually retains what it is sent**. That is the difference from +//! `failure_test`, whose doubles only need to misbehave: here the double has to +//! be a working backend, because the suite writes and reads back. +//! +//! What this proves is narrow and worth stating precisely. It is not that +//! Supermemory, Mem0 or Cognee uphold the contract — nobody here can prove that +//! about someone else's service. It is that **the adapter** does, given a +//! backend that answers its own documented shapes. A contract violation on the +//! adapter's side of the wire — a dropped taint, an export cursor that never +//! terminates, an upsert that duplicates — is caught here. + +#![allow(clippy::expect_used, clippy::panic)] + +use std::collections::BTreeMap; +use std::sync::{Arc, Mutex}; + +use axum::extract::{Path, Query, State}; +use axum::routing::{delete, get, post, put}; +use axum::{Json, Router}; +use serde_json::{json, Value}; + +use crate::{mem0_provider, Mem0Memory}; + +/// A record as one of the vendor doubles holds it. +#[derive(Clone, Debug)] +struct Row { + id: String, + content: String, + metadata: Value, +} + +/// The doubles' shared store: `id -> Row`, plus a counter for fresh ids. +#[derive(Default, Debug)] +struct Backend { + rows: BTreeMap, + next: usize, +} + +impl Backend { + fn fresh_id(&mut self) -> String { + self.next += 1; + format!("rec-{}", self.next) + } +} + +type Store = Arc>; + +/// Serves `app` on an ephemeral port and returns its base URL. +async fn serve(app: Router) -> String { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("bind"); + let endpoint = format!("http://{}", listener.local_addr().expect("address")); + tokio::spawn(async move { + axum::serve(listener, app).await.expect("serve"); + }); + endpoint +} + +// ── Mem0's native shapes ───────────────────────────────────────────────────── +// +// Five routes, matching what `Mem0Dialect` issues: list, create, update, +// delete, search. The response envelopes (`results`, `memory`, `metadata`) are +// the ones its `decode` reads, so a shape drift on either side fails here +// rather than silently returning nothing. + +async fn mem0_list(State(store): State) -> Json { + let store = store.lock().expect("store lock"); + let results: Vec = store + .rows + .values() + .map(|r| { + json!({ + "id": r.id, + "memory": r.content, + "metadata": r.metadata, + "created_at": "1970-01-01T00:00:00Z", + }) + }) + .collect(); + Json(json!({ "results": results })) +} + +async fn mem0_create(State(store): State, Json(body): Json) -> Json { + let mut store = store.lock().expect("store lock"); + let id = store.fresh_id(); + let content = body["messages"][0]["content"] + .as_str() + .unwrap_or_default() + .to_owned(); + let metadata = body["metadata"].clone(); + store.rows.insert( + id.clone(), + Row { + id: id.clone(), + content, + metadata, + }, + ); + Json(json!({ "results": [{ "id": id }] })) +} + +async fn mem0_update( + State(store): State, + Path(id): Path, + Json(body): Json, +) -> Json { + let mut store = store.lock().expect("store lock"); + if let Some(row) = store.rows.get_mut(&id) { + if let Some(text) = body["text"].as_str() { + row.content = text.to_owned(); + } + if !body["metadata"].is_null() { + row.metadata = body["metadata"].clone(); + } + } + Json(json!({ "id": id })) +} + +async fn mem0_delete(State(store): State, Path(id): Path) -> Json { + store.lock().expect("store lock").rows.remove(&id); + Json(json!({ "deleted": true })) +} + +async fn mem0_search(State(store): State, Json(body): Json) -> Json { + // Substring matching is enough: the suite asserts that recall *narrows*, + // not that the backend ranks well. + let needle = body["query"].as_str().unwrap_or_default().to_lowercase(); + let limit = body["top_k"].as_u64().unwrap_or(100) as usize; + let store = store.lock().expect("store lock"); + let results: Vec = store + .rows + .values() + .filter(|r| r.content.to_lowercase().contains(&needle)) + .take(limit) + .map(|r| { + json!({ + "id": r.id, + "memory": r.content, + "metadata": r.metadata, + "score": 0.9, + }) + }) + .collect(); + Json(json!({ "results": results })) +} + +/// A Mem0 double that retains what it is sent. +async fn mem0_backend() -> String { + let store: Store = Arc::new(Mutex::new(Backend::default())); + let app = Router::new() + .route("/memories", get(mem0_list).post(mem0_create)) + .route("/memories/{id}", put(mem0_update).delete(mem0_delete)) + .route("/search", post(mem0_search)) + .with_state(store); + serve(app).await +} + +#[tokio::test] +async fn mem0_upholds_the_contract() { + let endpoint = mem0_backend().await; + let provider = mem0_provider(Mem0Memory::new(&endpoint, None).expect("client")); + tinymemory_conformance::assert_provider(Arc::new(provider)).await; +} + +/// The suite's write-path assertions only run when the driver retains, so a +/// double that silently dropped writes would let the whole run pass vacuously. +/// This pins that the Mem0 double is genuinely retaining. +#[tokio::test] +async fn the_mem0_double_actually_retains() { + let endpoint = mem0_backend().await; + let provider = mem0_provider(Mem0Memory::new(&endpoint, None).expect("client")); + assert!( + tinymemory_conformance::retains_writes(&provider).await, + "the Mem0 double must retain writes, or `assert_provider` skips every \ + assertion that matters and still reports success" + ); +} + +// ── Supermemory's native shapes ────────────────────────────────────────────── +// +// Container tags are Supermemory's namespace equivalent, and the adapter +// derives one per TinyMemory namespace. The double keeps a tag per row so the +// tag listing — which drives `entries()` — reflects what has actually been +// written, rather than a fixed set the adapter would then filter to nothing. + +/// The tag the adapter derives, as sent on create. +fn tag_of(row: &Row) -> String { + row.metadata + .get("tinymemory_namespace") + .and_then(Value::as_str) + .map(|ns| format!("tinymemory-{ns}")) + .unwrap_or_default() +} + +async fn sm_tags(State(store): State) -> Json { + let store = store.lock().expect("store lock"); + let mut tags: Vec = store.rows.values().map(tag_of).collect(); + tags.sort(); + tags.dedup(); + Json(Value::Array( + tags.into_iter() + .map(|t| json!({ "containerTag": t })) + .collect(), + )) +} + +async fn sm_list(State(store): State, Json(body): Json) -> Json { + // The adapter pages until a short page comes back, so a double that always + // returned a full page would spin. One page, then empty. + let page = body["page"].as_u64().unwrap_or(1); + let wanted = body["containerTags"][0].as_str().unwrap_or_default(); + let store = store.lock().expect("store lock"); + let entries: Vec = if page > 1 { + Vec::new() + } else { + store + .rows + .values() + .filter(|r| tag_of(r) == wanted) + .map(|r| { + json!({ + "id": r.id, + "content": r.content, + "metadata": r.metadata, + "createdAt": "1970-01-01T00:00:00Z", + "isLatest": true, + "isForgotten": false, + }) + }) + .collect() + }; + Json(json!({ "memoryEntries": entries })) +} + +async fn sm_create(State(store): State, Json(body): Json) -> Json { + let mut store = store.lock().expect("store lock"); + let id = store.fresh_id(); + let first = &body["memories"][0]; + store.rows.insert( + id.clone(), + Row { + id: id.clone(), + content: first["content"].as_str().unwrap_or_default().to_owned(), + metadata: first["metadata"].clone(), + }, + ); + Json(json!({ "memories": [{ "id": id }] })) +} + +async fn sm_update(State(store): State, Json(body): Json) -> Json { + let mut store = store.lock().expect("store lock"); + let id = body["id"].as_str().unwrap_or_default().to_owned(); + if let Some(row) = store.rows.get_mut(&id) { + if let Some(text) = body["newContent"].as_str() { + row.content = text.to_owned(); + } + if !body["metadata"].is_null() { + row.metadata = body["metadata"].clone(); + } + } + Json(json!({ "id": id })) +} + +async fn sm_delete(State(store): State, Json(body): Json) -> Json { + let id = body["id"].as_str().unwrap_or_default(); + store.lock().expect("store lock").rows.remove(id); + Json(json!({ "deleted": true })) +} + +async fn sm_search(State(store): State, Json(body): Json) -> Json { + let needle = body["q"] + .as_str() + .or_else(|| body["query"].as_str()) + .unwrap_or_default() + .to_lowercase(); + let limit = body["limit"].as_u64().unwrap_or(100) as usize; + let tag = body["containerTag"].as_str(); + let store = store.lock().expect("store lock"); + let results: Vec = store + .rows + .values() + .filter(|r| tag.is_none_or(|t| tag_of(r) == t)) + .filter(|r| r.content.to_lowercase().contains(&needle)) + .take(limit) + .map(|r| { + json!({ + "id": r.id, + "content": r.content, + "metadata": r.metadata, + "score": 0.9, + }) + }) + .collect(); + Json(json!({ "results": results })) +} + +async fn supermemory_backend() -> String { + let store: Store = Arc::new(Mutex::new(Backend::default())); + let app = Router::new() + .route("/v3/container-tags/list", get(sm_tags)) + .route("/v4/memories/list", post(sm_list)) + .route( + "/v4/memories", + post(sm_create).patch(sm_update).delete(sm_delete), + ) + .route("/v4/search", post(sm_search)) + .with_state(store); + serve(app).await +} + +#[tokio::test] +async fn supermemory_upholds_the_contract() { + let endpoint = supermemory_backend().await; + let provider = crate::supermemory_provider( + crate::SupermemoryMemory::new(&endpoint, None).expect("client"), + ); + tinymemory_conformance::assert_provider(Arc::new(provider)).await; +} + +#[tokio::test] +async fn the_supermemory_double_actually_retains() { + let endpoint = supermemory_backend().await; + let provider = crate::supermemory_provider( + crate::SupermemoryMemory::new(&endpoint, None).expect("client"), + ); + assert!( + tinymemory_conformance::retains_writes(&provider).await, + "the Supermemory double must retain writes, or the suite passes vacuously" + ); +} + +// ── Cognee's native shapes ─────────────────────────────────────────────────── +// +// The odd one out. Cognee has no per-record API: the adapter uploads each +// record as a JSON *file* into a per-namespace dataset, and reads it back +// through `/raw` — so the double stores the uploaded bytes verbatim and serves +// them unchanged. That is also why this double is the strictest of the three: +// the envelope it hands back is deserialised straight into `StoredEntry`, so a +// field the adapter fails to write is a parse failure here rather than a +// silently empty value. + +/// A dataset, keyed by the name the adapter derives from a namespace. +type Datasets = Arc>>>; + +async fn cg_datasets(State(sets): State) -> Json { + let sets = sets.lock().expect("store lock"); + Json(Value::Array( + sets.keys() + .map(|name| json!({ "id": name, "name": name })) + .collect(), + )) +} + +async fn cg_data(State(sets): State, Path(dataset): Path) -> Json { + let sets = sets.lock().expect("store lock"); + let ids: Vec = sets + .get(&dataset) + .map(|d| { + d.keys() + // `name` is required, and the adapter skips anything not + // ending `.tinymemory[.json]` — Cognee's own loader strips the + // extension, so both spellings are accepted. The data id here + // *is* the uploaded filename, which already carries it. + .map(|id| json!({ "id": id, "name": id })) + .collect() + }) + .unwrap_or_default(); + Json(Value::Array(ids)) +} + +async fn cg_raw( + State(sets): State, + Path((dataset, data_id)): Path<(String, String)>, +) -> String { + sets.lock() + .expect("store lock") + .get(&dataset) + .and_then(|d| d.get(&data_id)) + .cloned() + .unwrap_or_default() +} + +async fn cg_delete( + State(sets): State, + Path((dataset, data_id)): Path<(String, String)>, +) -> Json { + if let Some(d) = sets.lock().expect("store lock").get_mut(&dataset) { + d.remove(&data_id); + } + Json(json!({ "deleted": true })) +} + +/// Pulls the uploaded envelope and the dataset name out of a multipart body. +async fn multipart_parts(mut form: axum::extract::Multipart) -> (String, String, String) { + let (mut body, mut dataset, mut filename) = (String::new(), String::new(), String::new()); + while let Ok(Some(field)) = form.next_field().await { + match field.name().unwrap_or_default().to_owned().as_str() { + "datasetName" => dataset = field.text().await.unwrap_or_default(), + "data" | "file" | "files" => { + filename = field.file_name().unwrap_or_default().to_owned(); + body = field.text().await.unwrap_or_default(); + } + _ => { + let _ = field.bytes().await; + } + } + } + (body, dataset, filename) +} + +async fn cg_remember(State(sets): State, form: axum::extract::Multipart) -> Json { + let (body, dataset, filename) = multipart_parts(form).await; + let mut sets = sets.lock().expect("store lock"); + sets.entry(dataset).or_default().insert(filename, body); + Json(json!({ "status": "ok" })) +} + +async fn cg_update( + State(sets): State, + Query(q): Query>, + form: axum::extract::Multipart, +) -> Json { + let (body, _, _) = multipart_parts(form).await; + let dataset = q.get("dataset_id").cloned().unwrap_or_default(); + let data_id = q.get("data_id").cloned().unwrap_or_default(); + if let Some(d) = sets.lock().expect("store lock").get_mut(&dataset) { + d.insert(data_id, body); + } + Json(json!({ "status": "ok" })) +} + +async fn cg_recall(State(sets): State, Json(body): Json) -> Json { + let needle = body["query"].as_str().unwrap_or_default().to_lowercase(); + let limit = body["top_k"].as_u64().unwrap_or(100) as usize; + let wanted: Option> = body["datasets"].as_array().map(|a| { + a.iter() + .filter_map(|v| v.as_str().map(str::to_owned)) + .collect() + }); + let sets = sets.lock().expect("store lock"); + let hits: Vec = sets + .iter() + .filter(|(name, _)| wanted.as_ref().is_none_or(|w| w.contains(name))) + .flat_map(|(_, d)| d.values()) + .filter(|raw| raw.to_lowercase().contains(&needle)) + .take(limit) + .map(|raw| json!({ "text": raw })) + .collect(); + Json(json!({ "results": hits })) +} + +async fn cognee_backend() -> String { + let sets: Datasets = Arc::new(Mutex::new(BTreeMap::new())); + let app = Router::new() + .route("/api/v1/datasets", get(cg_datasets)) + .route("/api/v1/datasets/{dataset}/data", get(cg_data)) + .route("/api/v1/datasets/{dataset}/data/{data_id}/raw", get(cg_raw)) + .route( + "/api/v1/datasets/{dataset}/data/{data_id}", + delete(cg_delete), + ) + .route("/api/v1/remember", post(cg_remember)) + .route("/api/v1/update", axum::routing::patch(cg_update)) + .route("/api/v1/recall", post(cg_recall)) + .with_state(sets); + serve(app).await +} + +#[tokio::test] +async fn cognee_upholds_the_contract() { + let endpoint = cognee_backend().await; + let provider = + crate::cognee_provider(crate::CogneeMemory::self_hosted(&endpoint, None).expect("client")); + tinymemory_conformance::assert_provider(Arc::new(provider)).await; +} + +#[tokio::test] +async fn the_cognee_double_actually_retains() { + let endpoint = cognee_backend().await; + let provider = + crate::cognee_provider(crate::CogneeMemory::self_hosted(&endpoint, None).expect("client")); + assert!( + tinymemory_conformance::retains_writes(&provider).await, + "the Cognee double must retain writes, or the suite passes vacuously" + ); +} diff --git a/adapters/remote/src/lib.rs b/adapters/remote/src/lib.rs index dba72ed..1b9d559 100644 --- a/adapters/remote/src/lib.rs +++ b/adapters/remote/src/lib.rs @@ -41,3 +41,6 @@ pub fn cognee_provider(memory: CogneeMemory) -> MemoryTraitProvider { #[cfg(test)] mod failure_test; + +#[cfg(test)] +mod conformance_test; diff --git a/adapters/tinycortex/Cargo.toml b/adapters/tinycortex/Cargo.toml index 57f28a1..73ba1e7 100644 --- a/adapters/tinycortex/Cargo.toml +++ b/adapters/tinycortex/Cargo.toml @@ -51,6 +51,9 @@ async-trait = "0.1" anyhow = "1" [dev-dependencies] +# The behavioural contract suite, run against this crate's drivers +# (issue #18 §E1, acceptance criterion 5). +tinymemory-conformance = { path = "../../conformance" } tokio = { version = "1", features = ["macros", "rt-multi-thread"] } [lints.rust] diff --git a/adapters/tinycortex/src/conformance_test.rs b/adapters/tinycortex/src/conformance_test.rs new file mode 100644 index 0000000..88f0ada --- /dev/null +++ b/adapters/tinycortex/src/conformance_test.rs @@ -0,0 +1,50 @@ +//! The conformance suite, run against the TinyCortex driver. +//! +//! The last name in issue #18's acceptance criterion 5, alongside the three +//! hosted adapters covered in `tinymemory-remote`. +//! +//! # Which TinyCortex driver +//! +//! This crate binds two, and they are conformance-tested differently. +//! +//! [`crate::provider`] composes the three mandatory families over any +//! `tinycortex::memory::Memory` backend. It needs nothing but the backend, so +//! the suite runs against it here with the engine's own `InMemoryMemoryStore`. +//! +//! [`crate::engine::TinycortexProvider`] serves all eighteen families, and +//! needs a `MemoryClient` — which needs the host's process-global seams +//! (`set_embedding_host` and friends) installed before it will open. A test +//! that installs a process global is order-dependent, which `AGENTS.md` rules +//! out, so covering it needs its own integration target that owns the global +//! for the whole binary. That is not written yet, and criterion 5 is not +//! complete until it is. +//! +//! Running against `InMemoryMemoryStore` rather than a SQLite workspace is +//! deliberate and is also the sharper test: it is the engine's simplest +//! `Memory`, so anything the suite catches is the *adapter's* behaviour rather +//! than the storage engine's. + +#![allow(clippy::expect_used, clippy::panic)] + +use std::sync::Arc; + +use tinycortex::memory::store::InMemoryMemoryStore; + +#[tokio::test] +async fn the_tinycortex_driver_upholds_the_contract() { + let driver = crate::provider(Arc::new(InMemoryMemoryStore::new())); + tinymemory_conformance::assert_provider(Arc::new(driver)).await; +} + +/// The suite skips every write-path assertion when a driver does not retain, so +/// a backend that silently dropped writes would let the run above pass having +/// asserted almost nothing. This pins that it does retain. +#[tokio::test] +async fn the_backend_actually_retains() { + let driver = crate::provider(Arc::new(InMemoryMemoryStore::new())); + assert!( + tinymemory_conformance::retains_writes(&driver).await, + "the engine's in-memory store must retain writes, or the suite above \ + reports success having run four assertions of eleven" + ); +} diff --git a/adapters/tinycortex/src/lib.rs b/adapters/tinycortex/src/lib.rs index ec838a7..fba0063 100644 --- a/adapters/tinycortex/src/lib.rs +++ b/adapters/tinycortex/src/lib.rs @@ -75,3 +75,6 @@ pub fn provider(memory: Arc) -> MemoryTraitProvi TINYCORTEX_DRIVER_ID, ) } + +#[cfg(test)] +mod conformance_test; diff --git a/conformance/src/lib.rs b/conformance/src/lib.rs index 7bfca70..3e99c02 100644 --- a/conformance/src/lib.rs +++ b/conformance/src/lib.rs @@ -49,3 +49,11 @@ pub use suite::{ assert_store_get_round_trip, assert_taint_is_preserved, assert_upsert_replaces_rather_than_duplicates, }; +pub use suite::{ + // Exported alongside the assertions because a caller standing up its own + // backend double needs it: `assert_provider` skips every write-path + // assertion when the driver does not retain, so a double that silently + // dropped writes would let a whole run pass vacuously. Probing for that + // directly is how a caller proves its harness is real. + retains_writes, +}; diff --git a/conformance/src/suite/mod.rs b/conformance/src/suite/mod.rs index 194770c..1766887 100644 --- a/conformance/src/suite/mod.rs +++ b/conformance/src/suite/mod.rs @@ -601,8 +601,21 @@ pub async fn assert_awkward_content_round_trips(provider: &dyn MemoryProvider) { ("large", "x".repeat(64 * 1024)), ("newlines", "a\nb\r\nc\0d".to_string()), ]; + let mut accepted = 0usize; for (key, content) in &cases { - provider + // A driver may refuse a shape outright — `MemoryCore::store` documents + // `Invalid` "for caller input the driver rejects", and the TinyCortex + // engine uses that to refuse empty content. What a driver may *not* do + // is accept a value and hand back something else. + // + // The refusal is not yet required to be `Invalid` specifically. The + // engine's own error is flattened through `anyhow` before the mandatory + // composition sees it, so a validation refusal currently arrives as + // `Other` and is indistinguishable from a backend failure. That is the + // gap §A4 closes; when it does, this should tighten to require + // `MemoryError::Invalid` so a genuine backend failure stops passing + // here. + if provider .store( &ns, key, @@ -612,7 +625,11 @@ pub async fn assert_awkward_content_round_trips(provider: &dyn MemoryProvider) { MemoryTaint::Internal, ) .await - .unwrap_or_else(|e| panic!("{who}: store of `{key}` failed: {e}")); + .is_err() + { + continue; + } + accepted += 1; if let Some(got) = provider .get(&ns, key) .await @@ -621,6 +638,13 @@ pub async fn assert_awkward_content_round_trips(provider: &dyn MemoryProvider) { assert_eq!(&got.content, content, "{who}: `{key}` content was mangled"); } } + // Without this a driver that refused every shape would pass having stored + // nothing, which is the vacuous reading of "may refuse". + assert!( + accepted > 0, + "{who}: refused every content shape — unicode, empty, large and \ + newlines were all rejected, so this assertion proved nothing" + ); let keys: Vec<&str> = cases.iter().map(|(k, _)| *k).collect(); cleanup(provider, &ns, &keys).await; }