diff --git a/CLAUDE.md b/CLAUDE.md index 1b7dc04..9a8881b 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -39,7 +39,7 @@ model::WorkflowGraph → validate → compiler::compile → engine::run feature. Not part of the engine: `engine::run` neither reads nor writes any of it. `store::HostPolicy` is where a host injects the judgements only it can make — which harnesses exist, which slugs resolve. -- `bindings.rs` — reading the `={{ ... }}` bindings a graph declares: which node +- `bindings.rs` — reading the `=expr` bindings a graph declares: which node an expression reads from, and whether it reads as prose rather than jq. - `gates/` — authoring gates: what is *guaranteed* wrong with a graph, refused before a write rather than surfacing as a silent null at run time. Only the diff --git a/Cargo.lock b/Cargo.lock index 7591798..4f2b9b8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8,6 +8,19 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" +[[package]] +name = "ahash" +version = "0.8.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" +dependencies = [ + "cfg-if", + "getrandom 0.3.4", + "once_cell", + "version_check", + "zerocopy", +] + [[package]] name = "async-trait" version = "0.1.92" @@ -132,6 +145,18 @@ version = "2.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4388bee8683e3d04af747c73422af53102d2bd24d9eadb6cbc100baef4b43f8" +[[package]] +name = "bitvec" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddcec3d12c579d40898fe0a9a358a803c23e9c52ca3c425707f81c9436211837" +dependencies = [ + "funty", + "radium", + "tap", + "wyz", +] + [[package]] name = "block-buffer" version = "0.10.4" @@ -141,6 +166,29 @@ dependencies = [ "generic-array", ] +[[package]] +name = "bson" +version = "2.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7969a9ba84b0ff843813e7249eed1678d9b6607ce5a3b8f0a47af3fcf7978e6e" +dependencies = [ + "ahash", + "base64", + "bitvec", + "getrandom 0.2.17", + "getrandom 0.3.4", + "hex", + "indexmap", + "js-sys", + "once_cell", + "rand 0.9.4", + "serde", + "serde_bytes", + "serde_json", + "time", + "uuid", +] + [[package]] name = "bstr" version = "1.12.3" @@ -230,6 +278,35 @@ dependencies = [ "memchr", ] +[[package]] +name = "const-random" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87e00182fe74b066627d63b85fd550ac2998d4b0bd86bfed477a0ae4c7c71359" +dependencies = [ + "const-random-macro", +] + +[[package]] +name = "const-random-macro" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9d839f2a20b0aee515dc581a6172f2321f96cab76c1a38a4c584a194955390e" +dependencies = [ + "getrandom 0.2.17", + "once_cell", + "tiny-keccak", +] + +[[package]] +name = "convert_case" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "633458d4ef8c78b72454de2d54fd6ab2e60f9e02be22f3c6104cdc8a4e0fceb9" +dependencies = [ + "unicode-segmentation", +] + [[package]] name = "core-foundation" version = "0.10.1" @@ -273,6 +350,42 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "critical-section" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "790eea4361631c5e7d22598ecd5723ff611904e3344ce8720784c93e3d83d40b" + +[[package]] +name = "crossbeam-channel" +version = "0.5.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d85363c37faeca707aef026efa9f3b34d077bce547e48f770770625c6013679e" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d6914041f254d6e9176c01941b21115dcfb7089e55135a35411081bd106ef3f" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17" + +[[package]] +name = "crunchy" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" + [[package]] name = "crypto-common" version = "0.1.7" @@ -283,12 +396,101 @@ dependencies = [ "typenum", ] +[[package]] +name = "darling" +version = "0.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9cdf337090841a411e2a7f3deb9187445851f91b309c0c0a29e05f74a00a48c0" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1247195ecd7e3c85f83c8d2a366e4210d588e802133e1e355180a9870b517ea4" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn 2.0.118", +] + +[[package]] +name = "darling_macro" +version = "0.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d38308df82d1080de0afee5d069fa14b0326a88c14f15c5ccda35b4a6c414c81" +dependencies = [ + "darling_core", + "quote", + "syn 2.0.118", +] + [[package]] name = "data-encoding" version = "2.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a4ae5f15dda3c708c0ade84bfee31ccab44a3da4f88015ed22f63732abe300c8" +[[package]] +name = "deranged" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" +dependencies = [ + "powerfmt", +] + +[[package]] +name = "derive-syn-parse" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d65d7ce8132b7c0e54497a4d9a55a1c2a0912a0d786cf894472ba818fba45762" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.118", +] + +[[package]] +name = "derive-where" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d08b3a0bcc0d079199cd476b2cae8435016ec11d1c0986c6901c5ac223041534" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.118", +] + +[[package]] +name = "derive_more" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d751e9e49156b02b44f9c1815bcb94b984cdcc4396ecc32521c739452808b134" +dependencies = [ + "derive_more-impl", +] + +[[package]] +name = "derive_more-impl" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "799a97264921d8623a957f6c3b9011f3b5492f557bbb7a5a19b7fa6d06ba8dcb" +dependencies = [ + "convert_case", + "proc-macro2", + "quote", + "rustc_version", + "syn 2.0.118", + "unicode-xid", +] + [[package]] name = "digest" version = "0.10.7" @@ -297,6 +499,7 @@ checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ "block-buffer", "crypto-common", + "subtle", ] [[package]] @@ -322,6 +525,18 @@ version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" +[[package]] +name = "enum-as-inner" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1e6a265c649f3f5979b601d26f1d05ada116434c87741c9493cb56218f76cbc" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 2.0.118", +] + [[package]] name = "equivalent" version = "1.0.2" @@ -338,6 +553,18 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "fallible-iterator" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649" + +[[package]] +name = "fallible-streaming-iterator" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" + [[package]] name = "fastrand" version = "2.4.1" @@ -381,6 +608,12 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" +[[package]] +name = "foldhash" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" + [[package]] name = "font8x8" version = "0.3.1" @@ -412,6 +645,12 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" +[[package]] +name = "funty" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" + [[package]] name = "futures-channel" version = "0.3.33" @@ -427,6 +666,23 @@ version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2cd50c473c80f6d7c3670a752354b8e569b1a7cbfdc0419ec88e5edad85e0dc7" +[[package]] +name = "futures-io" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53c0fa8157de1303bfffdaa1cc2a673bfffb60102f76b0ef4441659124373fed" + +[[package]] +name = "futures-macro" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d6d3cde68c518367be28956066ddfef33813991b77a55005a69dae04bf3b10b" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.118", +] + [[package]] name = "futures-sink" version = "0.3.33" @@ -452,8 +708,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a77a90a256fce34da66415271e30f94ee91c57b04b8a2c042d9cf3220179deaa" dependencies = [ "futures-core", + "futures-io", + "futures-macro", "futures-sink", "futures-task", + "memchr", "pin-project-lite", "slab", ] @@ -488,9 +747,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" dependencies = [ "cfg-if", + "js-sys", "libc", "r-efi 5.3.0", "wasip2", + "wasm-bindgen", ] [[package]] @@ -507,11 +768,90 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "hashbrown" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" +dependencies = [ + "foldhash 0.2.0", +] + [[package]] name = "hashbrown" version = "0.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" +dependencies = [ + "foldhash 0.2.0", +] + +[[package]] +name = "hashlink" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32069d97bb81e38fa67eab65e3393bf804bb85969f2bc06bf13f64aef5aba248" +dependencies = [ + "hashbrown 0.17.1", +] + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] +name = "hickory-proto" +version = "0.25.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8a6fe56c0038198998a6f217ca4e7ef3a5e51f46163bd6dd60b5c71ca6c6502" +dependencies = [ + "async-trait", + "cfg-if", + "data-encoding", + "enum-as-inner", + "futures-channel", + "futures-io", + "futures-util", + "idna", + "ipnet", + "once_cell", + "rand 0.9.4", + "ring", + "thiserror", + "tinyvec", + "tokio", + "tracing", + "url", +] + +[[package]] +name = "hickory-resolver" +version = "0.25.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc62a9a99b0bfb44d2ab95a7208ac952d31060efc16241c87eaf36406fecf87a" +dependencies = [ + "cfg-if", + "futures-util", + "hickory-proto", + "ipconfig", + "moka", + "once_cell", + "parking_lot", + "rand 0.9.4", + "resolv-conf", + "smallvec", + "thiserror", + "tokio", + "tracing", +] [[package]] name = "hifijson" @@ -519,6 +859,15 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "242402749acf71e6f32f5857598b7002c4058a4e3c3b22b4c7d51cab9aea754e" +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest", +] + [[package]] name = "http" version = "1.4.2" @@ -704,6 +1053,12 @@ dependencies = [ "zerovec", ] +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + [[package]] name = "idna" version = "1.1.0" @@ -747,7 +1102,20 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" dependencies = [ "equivalent", - "hashbrown", + "hashbrown 0.17.1", +] + +[[package]] +name = "ipconfig" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d40460c0ce33d6ce4b0630ad68ff63d6661961c48b6dba35e5a4d81cfb48222" +dependencies = [ + "socket2", + "widestring", + "windows-registry", + "windows-result", + "windows-sys 0.61.2", ] [[package]] @@ -781,7 +1149,7 @@ checksum = "48d801b0b57f10064c4e9f5a4f6c97d0ccf62649b179ff8ac23cd494a3120ee9" dependencies = [ "bstr", "bytes", - "foldhash", + "foldhash 0.1.5", "hifijson", "indexmap", "jaq-core", @@ -879,6 +1247,17 @@ version = "0.2.186" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" +[[package]] +name = "libsqlite3-sys" +version = "0.38.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1d20bef17f513b9b3004532233187769cd072d790971f4e4da0e346eb6401e8" +dependencies = [ + "cc", + "pkg-config", + "vcpkg", +] + [[package]] name = "linux-raw-sys" version = "0.12.1" @@ -891,6 +1270,15 @@ version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0" +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + [[package]] name = "log" version = "0.4.33" @@ -903,12 +1291,70 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" +[[package]] +name = "macro_magic" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc33f9f0351468d26fbc53d9ce00a096c8522ecb42f19b50f34f2c422f76d21d" +dependencies = [ + "macro_magic_core", + "macro_magic_macros", + "quote", + "syn 2.0.118", +] + +[[package]] +name = "macro_magic_core" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1687dc887e42f352865a393acae7cf79d98fab6351cde1f58e9e057da89bf150" +dependencies = [ + "const-random", + "derive-syn-parse", + "macro_magic_core_macros", + "proc-macro2", + "quote", + "syn 2.0.118", +] + +[[package]] +name = "macro_magic_core_macros" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b02abfe41815b5bd98dbd4260173db2c116dda171dc0fe7838cb206333b83308" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.118", +] + +[[package]] +name = "macro_magic_macros" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73ea28ee64b88876bf45277ed9a5817c1817df061a74f2b988971a12570e5869" +dependencies = [ + "macro_magic_core", + "quote", + "syn 2.0.118", +] + [[package]] name = "matchit" version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" +[[package]] +name = "md-5" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" +dependencies = [ + "cfg-if", + "digest", +] + [[package]] name = "memchr" version = "2.8.2" @@ -932,14 +1378,107 @@ dependencies = [ ] [[package]] -name = "mio" -version = "1.2.1" +name = "mio" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02bd0af71c67b473010cbbc60715ee815645a4dc942899111f494b4b737d6fda" +dependencies = [ + "libc", + "wasi", + "windows-sys 0.61.2", +] + +[[package]] +name = "moka" +version = "0.12.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4293f18e7567a1caf3c584855554377025c65e0aa445344d04171f5ad63d19b9" +dependencies = [ + "crossbeam-channel", + "crossbeam-epoch", + "crossbeam-utils", + "equivalent", + "parking_lot", + "portable-atomic", + "smallvec", + "tagptr", + "uuid", +] + +[[package]] +name = "mongocrypt" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8da0cd419a51a5fb44819e290fbdb0665a54f21dead8923446a799c7f4d26ad9" +dependencies = [ + "bson", + "mongocrypt-sys", + "once_cell", + "serde", +] + +[[package]] +name = "mongocrypt-sys" +version = "0.1.6+1.18.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "851fac73f7fe22f6a3ab87f720ce509cae7c9fd08e7dd27866cc232dee07ccf4" + +[[package]] +name = "mongodb" +version = "3.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ef2c933617431ad0246fb5b43c425ebdae18c7f7259c87de0726d93b0e7e91b" +dependencies = [ + "base64", + "bitflags", + "bson", + "derive-where", + "derive_more", + "futures-core", + "futures-io", + "futures-util", + "hex", + "hickory-proto", + "hickory-resolver", + "hmac", + "macro_magic", + "md-5", + "mongocrypt", + "mongodb-internal-macros", + "pbkdf2", + "percent-encoding", + "rand 0.9.4", + "rustc_version_runtime", + "rustls", + "rustversion", + "serde", + "serde_bytes", + "serde_with", + "sha1", + "sha2", + "socket2", + "stringprep", + "strsim", + "take_mut", + "thiserror", + "tokio", + "tokio-rustls", + "tokio-util", + "typed-builder", + "uuid", + "webpki-roots", +] + +[[package]] +name = "mongodb-internal-macros" +version = "3.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02bd0af71c67b473010cbbc60715ee815645a4dc942899111f494b4b737d6fda" +checksum = "9e5758dc828eb2d02ec30563cba365609d56ddd833190b192beaee2b475a7bb3" dependencies = [ - "libc", - "wasi", - "windows-sys 0.61.2", + "macro_magic", + "proc-macro2", + "quote", + "syn 2.0.118", ] [[package]] @@ -962,6 +1501,12 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-conv" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" + [[package]] name = "num-integer" version = "0.1.46" @@ -985,6 +1530,10 @@ name = "once_cell" version = "1.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" +dependencies = [ + "critical-section", + "portable-atomic", +] [[package]] name = "openssl-probe" @@ -992,6 +1541,38 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" +[[package]] +name = "parking_lot" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-link", +] + +[[package]] +name = "pbkdf2" +version = "0.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8ed6a7761f76e3b9f92dfb0a60a6a6477c61024b775147ff0973a02653abaf2" +dependencies = [ + "digest", +] + [[package]] name = "percent-encoding" version = "2.3.2" @@ -1023,6 +1604,12 @@ dependencies = [ "miniz_oxide", ] +[[package]] +name = "portable-atomic" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85" + [[package]] name = "potential_utf" version = "0.1.5" @@ -1032,6 +1619,12 @@ dependencies = [ "zerovec", ] +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + [[package]] name = "ppv-lite86" version = "0.2.21" @@ -1159,6 +1752,12 @@ version = "6.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" +[[package]] +name = "radium" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09" + [[package]] name = "rand" version = "0.9.4" @@ -1223,6 +1822,15 @@ dependencies = [ "rand_core 0.9.5", ] +[[package]] +name = "redox_syscall" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +dependencies = [ + "bitflags", +] + [[package]] name = "regex-automata" version = "0.4.14" @@ -1272,6 +1880,12 @@ dependencies = [ "web-sys", ] +[[package]] +name = "resolv-conf" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e061d1b48cb8d38042de4ae0a7a6401009d6143dc80d2e2d6f31f0bdd6470c7" + [[package]] name = "ring" version = "0.17.14" @@ -1286,6 +1900,31 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "rsqlite-vfs" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c51c9ae4df8a7fba42103df5c621fa3c37eccf3a3c650879e90fc48b11cc192c" +dependencies = [ + "hashbrown 0.16.1", + "thiserror", +] + +[[package]] +name = "rusqlite" +version = "0.40.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23f2a97da3e3873c73cb2a2e71b35c40ff95e0b1eefa8d72d8499a6928c3b5b3" +dependencies = [ + "bitflags", + "fallible-iterator", + "fallible-streaming-iterator", + "hashlink", + "libsqlite3-sys", + "smallvec", + "sqlite-wasm-rs", +] + [[package]] name = "rustc-hash" version = "2.1.3" @@ -1301,6 +1940,16 @@ dependencies = [ "semver", ] +[[package]] +name = "rustc_version_runtime" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dd18cd2bae1820af0b6ad5e54f4a51d0f3fcc53b05f845675074efcc7af071d" +dependencies = [ + "rustc_version", + "semver", +] + [[package]] name = "rustix" version = "1.1.4" @@ -1321,7 +1970,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0283386ce02abc0151e1761d08802dfe86c173b0b494af5cbc086574e453da06" dependencies = [ "aws-lc-rs", + "log", "once_cell", + "ring", "rustls-pki-types", "rustls-webpki", "subtle", @@ -1431,6 +2082,12 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + [[package]] name = "security-framework" version = "3.7.0" @@ -1476,6 +2133,16 @@ dependencies = [ "serde_derive", ] +[[package]] +name = "serde_bytes" +version = "0.11.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5d440709e79d88e51ac01c4b72fc6cb7314017bb7da9eeff678aa94c10e3ea8" +dependencies = [ + "serde", + "serde_core", +] + [[package]] name = "serde_core" version = "1.0.229" @@ -1502,6 +2169,7 @@ version = "1.0.151" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" dependencies = [ + "indexmap", "itoa", "memchr", "serde", @@ -1520,6 +2188,28 @@ dependencies = [ "serde_core", ] +[[package]] +name = "serde_with" +version = "3.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "381b283ce7bc6b476d903296fb59d0d36633652b633b27f64db4fb46dcbfc3b9" +dependencies = [ + "serde_core", + "serde_with_macros", +] + +[[package]] +name = "serde_with_macros" +version = "3.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6d4e30573c8cb306ed6ab1dca8423eec9a463ea0e155f45399455e0368b27e0" +dependencies = [ + "darling", + "proc-macro2", + "quote", + "syn 2.0.118", +] + [[package]] name = "sha1" version = "0.10.7" @@ -1602,12 +2292,41 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "sqlite-wasm-rs" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc3efc0da82635d7e1ced0053bbbfa8c7ab9645d0bf36ceb4f7127bb85315d75" +dependencies = [ + "cc", + "js-sys", + "rsqlite-vfs", + "wasm-bindgen", +] + [[package]] name = "stable_deref_trait" version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" +[[package]] +name = "stringprep" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b4df3d392d81bd458a8a621b8bffbd2302a12ffe288a9d931670948749463b1" +dependencies = [ + "unicode-bidi", + "unicode-normalization", + "unicode-properties", +] + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + [[package]] name = "subtle" version = "2.6.1" @@ -1656,6 +2375,24 @@ dependencies = [ "syn 2.0.118", ] +[[package]] +name = "tagptr" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b2093cf4c8eb1e67749a6762251bc9cd836b6fc171623bd0a9d324d37af2417" + +[[package]] +name = "take_mut" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f764005d11ee5f36500a149ace24e00e3da98b0158b3e2d53a7495660d3f4d60" + +[[package]] +name = "tap" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" + [[package]] name = "tempfile" version = "3.27.0" @@ -1689,6 +2426,46 @@ dependencies = [ "syn 3.0.3", ] +[[package]] +name = "time" +version = "0.3.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9e442fc33d7fdb45aa9bfeb312c095964abdf596f7567261062b2a7107aaabd" +dependencies = [ + "deranged", + "itoa", + "num-conv", + "powerfmt", + "serde_core", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b36ee98fd31ec7426d599183e8fe26932a8dc1fb76ddb6214d05493377d34ca" + +[[package]] +name = "time-macros" +version = "0.2.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71e552d1249bf61ac2a52db88179fd0673def1e1ad8243a00d9ec9ed71fee3dd" +dependencies = [ + "num-conv", + "time-core", +] + +[[package]] +name = "tiny-keccak" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c9d3793400a45f954c52e73d068316d76b6f4e36977e3fcebb13a2721e80237" +dependencies = [ + "crunchy", +] + [[package]] name = "tinyflows" version = "0.8.0" @@ -1715,6 +2492,20 @@ dependencies = [ "tracing", ] +[[package]] +name = "tinyflows-adaptive" +version = "0.1.0" +dependencies = [ + "async-trait", + "mongodb", + "rusqlite", + "serde", + "serde_json", + "thiserror", + "tinyflows", + "tokio", +] + [[package]] name = "tinystr" version = "0.8.3" @@ -1789,6 +2580,20 @@ dependencies = [ "tungstenite", ] +[[package]] +name = "tokio-util" +version = "0.7.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "494815d09bf52b5548659851081238f0ca39ff638363907596da739561c62c52" +dependencies = [ + "bytes", + "futures-core", + "futures-io", + "futures-sink", + "pin-project-lite", + "tokio", +] + [[package]] name = "tower" version = "0.5.3" @@ -1881,6 +2686,26 @@ version = "2.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6af6ae20167a9ece4bcb41af5b80f8a1f1df981f6391189ce00fd257af04126a" +[[package]] +name = "typed-builder" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "398a3a3c918c96de527dc11e6e846cd549d4508030b8a33e1da12789c856b81a" +dependencies = [ + "typed-builder-macro", +] + +[[package]] +name = "typed-builder-macro" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e48cea23f68d1f78eb7bc092881b6bb88d3d6b5b7e6234f6f9c911da1ffb221" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.118", +] + [[package]] name = "typenum" version = "1.20.1" @@ -1893,12 +2718,45 @@ version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eaea85b334db583fe3274d12b4cd1880032beab409c0d774be044d4480ab9a94" +[[package]] +name = "unicode-bidi" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c1cb5db39152898a79168971543b1cb5020dff7fe43c8dc468b0885f5e29df5" + [[package]] name = "unicode-ident" version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" +[[package]] +name = "unicode-normalization" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fd4f6878c9cb28d874b009da9e8d183b5abc80117c40bbd187a1fde336be6e8" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "unicode-properties" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7df058c713841ad818f1dc5d3fd88063241cc61f49f5fbea4b951e8cf5a8d71d" + +[[package]] +name = "unicode-segmentation" +version = "1.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8" + +[[package]] +name = "unicode-xid" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" + [[package]] name = "untrusted" version = "0.9.0" @@ -1923,6 +2781,24 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" +[[package]] +name = "uuid" +version = "1.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf3923a6f5c4c6382e0b653c4117f48d631ea17f38ed86e2a828e6f7412f5239" +dependencies = [ + "getrandom 0.4.3", + "js-sys", + "serde_core", + "wasm-bindgen", +] + +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + [[package]] name = "version_check" version = "0.9.5" @@ -2056,6 +2932,21 @@ dependencies = [ "rustls-pki-types", ] +[[package]] +name = "webpki-roots" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dcd9d09a39985f5344844e66b0c530a33843579125f23e21e9f0f220850f22a" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "widestring" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72069c3113ab32ab29e5584db3c6ec55d416895e60715417b5b883a357c3e471" + [[package]] name = "winapi" version = "0.3.9" @@ -2093,6 +2984,35 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" +[[package]] +name = "windows-registry" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02752bf7fbdcce7f2a27a742f798510f3e5ad88dbe84871e5168e2120c3d5720" +dependencies = [ + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link", +] + [[package]] name = "windows-sys" version = "0.52.0" @@ -2187,6 +3107,15 @@ version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" +[[package]] +name = "wyz" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05f360fc0b24296329c78fda852a1e9ae82de9cf7b27dae4b7f62f118f77b9ed" +dependencies = [ + "tap", +] + [[package]] name = "yoke" version = "0.8.3" diff --git a/Cargo.toml b/Cargo.toml index 0ba9c10..fd4e814 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,3 +1,10 @@ +# The adaptive loop lives beside the engine, never inside it. `crates/adaptive` +# decides *what* graph to run and judges what came back; this package decides +# nothing and executes one graph. Keeping them separate packages is what makes +# a merge from upstream a merge rather than a conflict resolution. +[workspace] +members = ["crates/adaptive"] + [package] name = "tinyflows" version = "0.8.0" diff --git a/crates/adaptive/Cargo.toml b/crates/adaptive/Cargo.toml new file mode 100644 index 0000000..7088c84 --- /dev/null +++ b/crates/adaptive/Cargo.toml @@ -0,0 +1,36 @@ +[package] +name = "tinyflows-adaptive" +version = "0.1.0" +edition = "2024" +rust-version = "1.85" +license = "GPL-3.0-or-later" +description = "An adaptive loop over the tinyflows engine: select or author a workflow, run it, judge it, learn." + +[dependencies] +tinyflows = { path = "../..", features = ["store"] } +serde = { version = "1", features = ["derive"] } +serde_json = "1" +async-trait = "0.1" +thiserror = "2" +rusqlite = { version = "0.40.2", features = ["bundled"], optional = true } +mongodb = { version = "3.6.0", optional = true } + +[features] +# Persistence out of the box. The alternative — no backend unless asked — meant +# `cargo add tinyflows-adaptive` gave you a crate whose whole value is that +# learning accumulates, and no way to make it accumulate. +# +# It costs a bundled SQLite build. A deployment that only wants Mongo turns it +# off: `default-features = false, features = ["mongo"]`. +default = ["sqlite"] +sqlite = ["dep:rusqlite"] +mongo = ["dep:mongodb"] + +[dev-dependencies] +tinyflows = { path = "../..", features = ["store", "mock"] } +tokio = { version = "1", features = ["macros", "rt", "rt-multi-thread", "sync", "time"] } + +[lints.rust] +unsafe_code = "forbid" +missing_docs = "warn" + diff --git a/crates/adaptive/README.md b/crates/adaptive/README.md new file mode 100644 index 0000000..b06745c --- /dev/null +++ b/crates/adaptive/README.md @@ -0,0 +1,546 @@ +# tinyflows-adaptive + +An adaptive loop over the tinyflows engine. It ingests a prompt, **selects a +stored workflow or authors one**, runs it on the engine, judges the result +against evidence, and learns — updating or replacing the workflow when the +graph itself was the problem. + +The engine is not modified. This crate sits beside it and decides *which* graph +to run; `tinyflows` decides nothing and runs one graph. + +``` +prompt ─▶ INTAKE ──────────▶ Runner ──▶ engine::run ──▶ CLOSING ──▶ answer + ├ goal (local (unmodified) ├ judge + ├ select, or or remote) ├ consolidate + └ author ├ score / promote + ▲ └ retry? + └──── rows + lessons ──────────────┘ +``` + +Every phase of the plan below is built. What closes the loop is the bottom +edge: the next attempt sees what this episode already spent and what earlier +ones learned, so a retry is a different idea rather than the same one reworded. + +## A worked host + +```text +cargo run -p tinyflows-adaptive --example service +``` + +[`docs/api.md`](docs/api.md) is the API reference for hosts — the traits you +implement (including the per-tier reply contract your `LlmProvider` must +honour), the constructors, the wire shapes, and the invariants. Full item-level +docs: `cargo doc -p tinyflows-adaptive --open`. + +[`examples/service.rs`](examples/service.rs) is the reference for embedding the +crate in a service: the tenant handles built once, a `Loop` per goal run, and — +the part most worth copying — a **`Relay`**: dispatch mints a unique wire id, +registers a oneshot waiter, serializes the `RunRequest`, sends, and awaits with +a deadline; `deliver()` is the socket receive handler that correlates the +echoed id back to its waiter. The transport is a pair of channels there so the +pattern is visible without a web framework in the way; every seam a production +host replaces is marked `HOST:`. It runs two goal runs end to end — the first +authors a workflow and the success gate files it, the second selects it off the +shelf. + +## Why it is a separate crate + +The engine's graph is **frozen at compile**: `CompiledWorkflow` is +`{ graph: WorkflowGraph }`, and nothing at run time adds, removes or rewires a +node. It is also persistence-free and has no concept of a goal. So it can +*repeat* — the `loop` node is real — but it cannot *re-decide*. + +Re-deciding is this crate's whole job, and it is a different shape: the graph +changes **between** runs, from evidence, against a record of what has already +been ruled out. Keeping the two in separate packages is what makes a merge from +upstream a merge rather than a conflict resolution. + +## The rule + +> The engine may know about one run. Anything that spans runs lives here. + +Ledger rows, lessons, workflow scoring, exclusion lists, promotion — none of it +crosses into `tinyflows`. That is not a discipline we maintain; `WorkflowRecord` +has nowhere to put it. + +## What is ported, and what is not + +Derived from medulla-v2 (Python). Most of it is **not** ported, because the +engine already does it better: + +| medulla-v2 | here | +|---|---| +| `Step`, `depends_on`, wave scheduling, `_dispatch_child` | **dropped** — nodes, edges, fan-out and the merge barrier are upstream | +| worktree pool, harness adapters, stream reader | **dropped** — `AgentRunner` is the seam | +| `WorkflowStore` | **dropped** — `tinyflows::store` has `WorkflowRecord`, `RunRecord`, notes, proposals, rollback | +| `Verdict`, `Blocker`, `Budget`, stall rule, `advanced` | **ported** — `contracts.rs` | +| planner / evaluator / consolidator prompts | **ported**, planner split into *select* and *author* | +| ledger rows, scored lessons, `record_use` | **ported** — upstream has notes, not scored lessons | + +What survives is exactly the loop. + +## Plan + +- [x] **0 · repo** — workspace member beside the engine; engine untouched. +- [x] **1a · contracts** — `Goal`, `Approach`, `Verdict`, `Blocker`, `Budget`. +- [x] **1b · ledger** — the `Ledger` trait plus two backends behind features, + `sqlite` and `mongo`, both checked by one conformance suite. Kept separate + from `WorkflowStore` so an upstream merge never contends with it. +- [x] **2 · intake** — `decide()`: select a stored workflow, else author one. + Selection sees the catalogue with both score counters and never sees a + workflow this episode already tried; authoring is grounded on the engine's + generated node catalogue and validated before it returns. +- [x] **2b · host facts** — `HostFacts`: what this machine permits, rendered into + the authoring prompt and checked after, plus the store's own + `HostPolicy::check_graph`. An absent fact means unknown, never forbidden. +- [x] **3 · execute** — the `Runner` port: `Local` runs the graph in this + process, `Remote` relays it to one elsewhere, and the loop cannot tell + which. Both are `serve()` → `RunReport` → `into_ran()`, so there is no + second path to drift. Thin on purpose — it holds no opinion, reads no + history, and never returns an error, because an attempt that leaves no + ledger row is one the next pass repeats. Not `run_with_checkpointer`: see + the field notes. +- [x] **4 · judge** — evidence from three sources: the `RunOutcome`, the + engine's own `Diagnosis` of what the steps did, and what changed outside + the run. Mechanical evidence settles three verdicts before any model is + asked; the judge never sees the ledger, so it cannot propose what to try + next. +- [x] **5 · consolidate** — `close()` records the row **whatever the verdict** + and scores the workflow that ran; `consolidate()` keeps only what a + different task could act on, and only with rows cited; `repair()` turns a + `GraphOp` batch into a **variant**, never an edit in place, and only when + the diagnosis says the graph is the thing at fault. +- [x] **5b · promotion** — a repaired family collapses to **one** catalogue row, + and which member holds it is decided on score. A variant is proven only + after `MIN_TRIALS` runs; until then the root keeps the position, so an + untested graph never displaces a 40/40 parent for everyone. Lineage lives + in the ledger, because *this graph came from that one* spans runs. +- [x] **6 · retry edge** — both planners see this episode's rows *and* the + lessons other episodes left. Closes the loop: `consolidate()` was + write-only until now. Authored attempts are fingerprinted by graph shape, + so two of them no longer fold into one exclusion-list entry. + +- [x] **7 · acquire** — a graph that was **authored and worked** becomes a + stored procedure, so the catalogue holds more than what a person put there + and `select` can choose something the loop worked out. Gated: exact + reusability check first (`reuse::baked_in`), a model asked only for the + name and description, and it can still refuse. + +## An instance is not a goal run + +Two lifetimes, and putting them in one object is the mistake worth naming. + +A **`Loop` is per tenant** — a scoped ledger, a store, capabilities, host facts, +a runner, a budget. Building one costs a database pool and an HTTP client, so it +is built once and shared. + +A **goal run is an episode id**, not an object. Its state lives in the `Episode` +record: goal, status, attempt, stalled. + +```rust +let engine = Loop { ledger: &ledger.for_tenant("user-7"), store, caps, .. }; +let finished = engine.run("ep-9f2", &goal).await?; // many of these, concurrently +``` + +That split buys both things at once. Many goal runs share one instance, because +the instance holds nothing per-episode. And an episode survives the process: +kill this one mid-run and `Loop::unfinished()` on the next boot hands back +everything that was in flight, each resumable by id. + +Had the instance *been* the goal run, both would be false — config rebuilt per +goal, and a deploy losing every episode's counters while leaving its rows behind +to look like progress. + +The record holds exactly what the rows cannot: the **goal** (unrecoverable), and +the **stall count** (recomputable only if `advanced` is stored, so it is, on the +row). `satisfied` is a field too — it used to be recoverable only by matching +`outcome == "satisfied"`, one reworded line from reporting every episode failed. + +## Inference: the crate names the job, the host picks the model + +Every request carries a `tier` — `select`, `author`, `judge`, `consolidate`, +`repair`, `generalise`. The crate never names a model, a vendor or a URL, which is the +host-agnostic rule it inherits; only the host knows what a job maps to. + +That is what makes a tier sweep a config change rather than a code change. +Judging is the expensive opinion — a judge that says yes wrongly ends the +episode — and selecting is a cheap one; without a name on the request a host +cannot route them differently. + +Called `tier` and not `role` because a chat request already has `role` on every +message. Six rather than medulla-v2's three: a host maps several tiers to one +model in a line of config and cannot split one tier into two at all. + +## Where the engine runs + +The loop and the engine may sit in one process or on opposite ends of a socket. +`Runner` is the seam; `execute::wire` is the contract. + +``` + server device + ┌────────────┐ RunRequest{graph,inputs} ┌────────────┐ + │ intake │ ──────────────────────────▶ │ serve() │ + │ closing │ ◀────────────────────────── │ engine │ + └────────────┘ RunReport{steps,…} └────────────┘ +``` + +**Steps cross, not `output`.** A run's final `output` is a lossy projection of +its steps: no status (so a swallowed error is invisible), no duration, no +null-binding diagnostics, a looped node collapsed to one entry — and a run that +returned `Err` has no `output` at all while its steps are all still there. +`Diagnosis` is not sent either: it is a pure function of the graph and the +steps, and the loop already has the graph. + +**Bounding is per node, at two budgets.** `bounded_within` is whole-value and +non-recursive, so applied to a map of nodes one fat entry replaces every other +node's output with a string preview. `RECORD_BUDGET` (256 KiB) bounds each step +for the durable record; `PROMPT_BUDGET` (4 KiB) bounds each node again in the +projection the judge reads. + +**Nothing about the episode crosses.** A runner sees one graph and its inputs — +no ledger, no lessons, no exclusion list, no verdict. It cannot reconstruct what +is being learned from it. + +**A runner that never answers is still an attempt.** `Remote` synthesizes a +report rather than propagating an error, and deliberately does *not* report an +empty `changed`: empty means "the host looked and saw nothing", which settles +mechanically as `MissingEvidence` — terminal. `ExternalWait` is terminal too, so +there is no safe blocker to pick. Saying the result is unknown routes it to the +judge, which can reach a continuable verdict, so a socket blip cannot end an +episode. + +## Node-level remote execution, and what it needs + +`StepAction::Interrupt` raises a real graph interrupt at a node and checkpoints +the run there; `resume_with_checkpointer` reloads that checkpoint and continues +from the boundary. So "reach a node, emit a call, park the graph durably, resume +when the reply lands" is a thing the engine does — the pieces are +`interception.rs` plus the checkpointed run. + +Two properties make it work at `StepPhase::Before` and only there. The interrupt +discards the activation's state update and re-runs the node from the top on +resume, which is **free before the node has run and doubles its side effects +after**. And an interceptor keyed by node id can answer `Replace { items }` on +the second pass, so the reply becomes the node's output without the executor +ever running. + +**The one gap**: no public entry point takes an interceptor *and* a host +checkpointer. `RunConfig` has `with_interceptor` and `with_checkpointer` and +they compose; nothing exposes the pair. That is one function in +`engine/resumable.rs`, mirroring `run_with_checkpointer_journaled_observed`. + +Worth knowing because it is the alternative to whole-graph dispatch: under it +the graph never leaves the server, only one node's call crosses, and a process +restart mid-wait costs nothing. + +## The contracts an external process touches + +Two kinds, and they fail differently. A **wire type** breaks when a derive is +dropped or a field renamed — silently, in another repository. A **trait** breaks +at compile time in the host that implements it. + +### Wire — serialized, crosses a boundary + +| Contract | Types | Casing | +|---|---|---| +| Execute | `RunRequest`, `RunReport`, `StepRecord`, `StepOutcome` | camelCase | +| Nested in those | `WorkflowGraph`, `Node`, `Edge`, `WorkflowInput`, `NullResolution` | **snake_case** | +| Run diagnosis | `Diagnosis`, `NullBinding`, `HiddenError`, `NeverRan` | camelCase | +| Loop state | `Goal`, `Approach`, `Verdict`, `Blocker`, `Budget` | snake_case | +| Knowledge | `LedgerRow`, `Lesson`, `LessonKind`, `Score` | snake_case | +| Host & repair | `HostFacts`, `GraphOp`, `WorkflowRecord` | snake_case | + +**One payload, two conventions.** The envelope this crate added is camelCase; +the engine's model types predate it and use serde's default, so the graph +*inside* a camelCase request stays snake_case: + +```json +{ "attemptId": "ep-1/3", + "graph": { "schema_version": 1, + "nodes": [{ "type_version": 1, "kind": "trigger" }], + "edges": [{ "from_node": "a", "from_port": "main" }] } } +``` + +Neither is wrong and changing either breaks something already shipped, so the +seam is asserted rather than tidied. A relay that assumes one convention +throughout produces a graph the engine refuses. + +`tests/contracts_surface.rs` asserts all of it at compile time. + +**Not serializable, deliberately:** `RunOutcome`, `ExecutionStep` and +`StepStatus` are `Debug + Clone` only upstream. That is why steps cross as +`StepRecord` and the outcome is rebuilt by `into_ran` rather than sent. + +### Traits — implemented in-process by a host + +| Trait | Who implements it | +|---|---| +| `Relay` | the service, to reach a runner elsewhere | +| `Workspace` | whatever can say what changed outside a run | +| `Ledger` | ships as `sqlite` and `mongo`; a third passes `conformance` | +| `Runner` | ships as `Local` and `Remote`; rarely custom | +| `LlmProvider`, `ToolInvoker`, `HttpClient`, `CodeRunner`, `StateStore`, `WorkflowResolver` | the engine's `Capabilities` bundle | +| `AgentRunner`, `MemoryProvider` | optional capabilities | +| `WorkflowStore`, `HostPolicy` | the engine's store seam — **synchronous** | + +`WorkflowStore` being synchronous is the one to plan around: a hosted service +with an async driver cannot implement it without blocking, so load a per-episode +snapshot before the loop and flush after. + +## Choosing a ledger backend + +```toml +tinyflows-adaptive = "0.1" # sqlite, on by default +tinyflows-adaptive = { version = "0.1", default-features = false, features = ["mongo"] } +``` + +**sqlite is a default feature**, so the crate persists out of the box. A crate +whose whole value is that learning accumulates should not ship unable to +accumulate it. It costs a bundled SQLite build; a deployment that only wants +Mongo turns it off with `default-features = false`. + +```rust +// The parent directory is created — `/var/lib/app/` on a first run need not exist. +let ledger = SqliteLedger::from_env_or("./adaptive.db")?; +``` + +`from_env_or` reads `TINYFLOWS_ADAPTIVE_DB` and uses the argument when it is +unset, so ops can move the file without a rebuild while the fallback stays +visible in your code. + +For a CLI or a desktop agent there is a convention instead: + +```rust +let ledger = SqliteLedger::at_default_location()?; +``` + +| Platform | Path | +|---|---| +| Linux / XDG | `$XDG_DATA_HOME/tinyflows/adaptive.db`, else `~/.local/share/tinyflows/adaptive.db` | +| macOS | `~/Library/Application Support/tinyflows/adaptive.db` | +| Windows | `%LOCALAPPDATA%\tinyflows\adaptive.db` | + +Four decisions in that table, each of which could have gone the other way: + +- **Data, not cache or config.** A ledger is not regenerable, so a cache sweeper + finding it deletes everything the loop has learned; and it is not something a + person edits, so a config directory would invite exactly that. +- **`%LOCALAPPDATA%`, not `%APPDATA%`.** The roaming profile syncs between + machines, and a SQLite file copied mid-write between two that both think they + own it is a corrupted database. +- **Namespaced `tinyflows/`, not `tinyflows-adaptive/`**, so a sibling crate + shares the folder rather than scattering one per crate across a disk. +- **No directory is an error, not a guess.** A daemon under a user with no home + has nowhere by convention; the error says to set the variable rather than + putting a database somewhere nobody looks. + +`TINYFLOWS_ADAPTIVE_DB` still wins. And a container or a service should name its +own path — a volume mount is the point, and a convention that lands the database +inside an ephemeral layer is worse than no convention at all. + +Three implementations, all checked by the same public +[`ledger::conformance`] suite — so "it works on sqlite" cannot quietly mean "it +works only on sqlite", and a host writing a fourth runs the identical cases. + +`MemoryLedger` is always compiled: no feature, no driver, no C library, so the +crate is usable the moment it is added. It **forgets everything on restart**, +and it is deliberately never selected for you. + +That last part is the design decision, not an oversight. A ledger silently +defaulting to memory is the worst failure this crate could have: the loop runs, +the exclusion list works within an episode, lessons are written and scored, the +tests pass — and every restart throws all of it away. Nobody notices, because +the only symptom is that it never gets better. So there is no `Default` impl +that would hand it to a host that did not ask, it is named for what it does, and +`sqlite` or `mongo` is the answer the moment learning is supposed to outlive a +process. + +Workflow scores live here, not on `WorkflowRecord`: a score is a fact that spans +runs, and the engine's record is a fact about one document. + +## Workflows in the configured store too + +The ledger had three backends; workflows had a directory of JSON files. A +deployment picking Mongo for one half of its durable state and getting a +filesystem for the other is wrong, so `workflows::Vault` mirrors `Ledger` — +`memory`, `sqlite`, `mongo`, one conformance suite. + +```rust +let vault = MongoVault::connect(&uri, "adaptive").await?.for_tenant(&user); +let snapshot = Snapshot::load(&vault, policy).await?; +let store: Arc = Arc::new(snapshot.clone()); +// … the loop runs, repair and keep call store.save() … +snapshot.flush(&vault).await?; +``` + +**Why a snapshot rather than a store.** `WorkflowStore` is synchronous — ten +required methods, none `async`. A Mongo driver is not. `block_on` inside a sync +method deadlocks a current-thread runtime, and async-ifying the trait upstream +means rewriting the file store and the authoring module and contending with that +rewrite on every merge. So the async half is ours and the sync half reads from +memory: load once, buffer writes, flush after. That also suits how the loop uses +a store — several reads while deciding, one or two writes when closing. + +**Only what changed is flushed.** A workflow the loop read and did not touch is +never rewritten, so a human editing one in the meantime is not clobbered. And +every id this crate writes is content-derived, so two episodes arriving at the +same procedure write the same id with identical content — last-write-wins is not +a lost update. + +**Workflows are now tenant-scoped**, which closes the gap the tenancy section +used to name: a `Vault` scopes exactly as a `Ledger` does. + +The engine's authoring surface — run records, revisions, notes, proposals — +**refuses** rather than pretending. A run record accepted and then lost on the +next load is worse than an error, because nothing tells the caller it vanished. + +### When a graph becomes durable + +`store.save()` inside the loop writes to the snapshot's **buffer**, which is +what makes the persistence policy the host's. The canonical gate: + +```rust +let snapshot = Snapshot::load(&vault, policy).await?; +let finished = engine.run(&episode, &goal).await?; +if finished.status == EpisodeStatus::Satisfied { + snapshot.flush(&vault).await?; // variants + learned graphs land +} // stood down → drop the snapshot; no residue +``` + +A repair variant exists **mid-episode** — the retry selects it out of the +snapshot the moment its parent is excluded — but the vault receives it only +after the goal run succeeds. A failed episode leaves nothing behind on whatever +the vault fronts, which matters most when it fronts somebody's device. + +Dropping loses nothing. The ledger's rows, lineage and scores are durable +regardless, and a re-derived repair converges on the same content-derived id — +so when the graph finally does land, the evidence recorded earlier reattaches +rather than being orphaned. + +## Reading a catalogue that already exists + +The loop's own procedures live in a `Vault`. Everyone else's live in a +`WorkflowStore` — the engine's file store, a host's own, a device's local +catalogue. Same records, different way in, and without one the loop can only +select what it wrote itself. + +```rust +let theirs = Arc::new(DeviceVault::new(&relay, &user)); // read-only, fetched +let ours = Arc::new(MongoVault::….for_tenant(&user)); // writable +let vault = Layered::new(vec![("device".into(), theirs)], ours) + .degrading(Arc::new(|layer, why| warn!(%layer, %why, "catalogue unavailable"))); + +let snapshot = Snapshot::load(&vault, policy).await?; // once, per episode +``` + +`Vault::load` is the **only** async catalogue read, and it happens once when an +episode starts. `store.list()` inside `decide()` is synchronous and served from +the snapshot, so fetching per episode costs one round trip against a run that +takes minutes — cheap enough that freshness is the better trade. + +`StoreVault` makes any `WorkflowStore` a `Vault` — nothing is migrated or +rewritten to become selectable. + +`Layered` reads several and writes one, and that is what makes importing safe. +A device's workflow can be selected, judged and scored; when it falls short the +repaired variant lands in **our** layer with its own id, and their copy is never +touched. No second master, no question of whose version is current, and a +`delete` can never reach a machine that did not ask for it. + +Later layers shadow earlier ones by id, so order the writable one last: a copy +we have taken ownership of wins over the original it came from. + +**A sleeping device must not stop a tenant's goals.** `new` is strict — any +unreadable layer fails the load, and therefore the episode — which is right when +every layer is a database you own and wrong the moment one is somebody's laptop. +`degrading` skips a read-only layer that errors, and **requires a handler**: a +catalogue that quietly vanishes is this crate's worst failure shape, because the +loop then authors from scratch and looks like it is working. You cannot have the +degradation without being told each time. The writable layer stays fatal either +way — a loop that cannot read its own procedures should stop, not relearn them. + +One caveat worth reading twice: `StoreVault` is **unscoped**, because the +engine's store has no tenant concept to filter on. Scoping is by construction — +build one per tenant over that tenant's own store, or its records read as global +and every tenant sees them. + +## Tenancy + +The scope lives on the **handle**, not on every method, because the failure it +prevents is forgetting to pass it. `ledger.for_tenant("user-7")` at the edge of +a request is a thing a reviewer can see; six scope arguments threaded through +intake and closing is a thing that goes wrong once and leaks one tenant's +lessons into another's prompt. Nothing in the loop takes a tenant argument. + +One rule everywhere: **writes go to this handle's bucket; reads return this +handle's bucket plus the global one.** An unscoped handle's bucket *is* global, +so a single-tenant deployment that never calls `for_tenant` reads back exactly +what it wrote. + +This matters because a lesson is free text. Its `trigger` and `claim` are +written from one tenant's episode and can name their repositories, paths and +internals, and `consolidate()` renders every retrievable lesson into the +model's prompt. So `promote` stamps the handle's scope and **ignores whatever +the argument says** — a caller, or a model answer deserialized straight into a +`Lesson`, cannot publish into another bucket by asking. + +Episode rows were never at risk: they are keyed by episode and `tried()` reads +one episode at a time. It is the knowledge plane — lessons and workflow +scores — that needed the key. + +`ledger::conformance::run_tenants` is public alongside `run_all`, and takes +three handles onto one store because how a backend makes a scoped one is its +own business. + +## Deliberately out of scope + +- **Human-in-the-loop parking**, in the loop. `StopReason::Paused` from an + `agent` node is not routed into checkpoint/resume, so the loop treats a parked + approval as a terminal `NeedsInput` verdict rather than waiting. + *Node-level parking itself is not the gap* — see below. +- **Scheduling.** Nine trigger kinds are accepted and stored; whether one + dispatches unattended is a host concern, and on the hosts we run today only + `manual` fires. + +## Field notes + +Things that cost a day each if met in production instead. + +- **There are two resumes and they behave differently.** `engine::resume` is the + HITL convenience: it re-executes the workflow with the merged approval set, so + every node before the gate runs again. `engine::resume_with_checkpointer` is + not that — it reloads the state persisted under `thread_id` and continues from + the interrupt boundary, running only what had not run. Conflating them is easy + and expensive: it is the difference between "durable node-level parking is + impossible here" and "it is one missing entry point away". Our retry is still + a new run of a new graph, but that is a choice, not a limit. +- **There is no wait node.** A workflow cannot sleep. Long waits end the run and + are re-triggered. +- **`RenameNode` does not rewrite bindings.** Edges are rewired; + `=nodes.…` inside other nodes' configs is not. Validation passes and + the graph runs quietly wrong. An automated fixer must treat a rename as + touching every expression in the graph. +- **The envelope.** `agent`, `tool_call` and `http_request` wrap output in + `{json, text, raw}`. `=nodes.x.item.f` is null where `=nodes.x.item.json.f` + was meant — compiles, validates, dry-runs green, runs empty. +- **Bindings are `=expr`, and there are no braces.** `CLAUDE.md` and one node + doc said `={{ … }}`; the implementation never accepted it. `is_expression` is + `starts_with('=')`, and the remainder is either a simple dotted path + (`=nodes.fetch.item.json.body`) or a jq program (`=.items | length`). `{{ }}` + is neither: it routes to jq, fails to compile, and a failed program is + `Value::Null`. Measured, not reasoned about — `={{ nodes.fetch.item.json.body }}` + evaluates to `null` against a scope where the dotted form yields `"hello"`. + Both docs are corrected. +- **A dry run proves wiring, not work.** A `code` node's script and an `agent` + node's real reply are both invisible to one. +- **`run_with_checkpointer` installs a `NoopObserver`.** No observer means no + steps, and no steps means `diagnose` returns a blank `Diagnosis` — which is + not "nothing was wrong", it is "nobody looked". The judge's findings, the + three mechanical verdicts and `graph_is_suspect` all read it, so the durable + entry point silently disables repair. `run_with_checkpointer_journaled_observed` + keeps both, at the cost of a journal. We run observed and unpersisted: a + checkpointer buys durable *resume*, and resume is out of scope. +- **`never_ran` only reports `agent`, `tool_call` and `http_request`.** A + routed-past `transform` is not a surprise worth warning about, so it is + omitted by design. A test that asserts on skipped control-flow nodes will + fail against a correct engine. diff --git a/crates/adaptive/docs/api.md b/crates/adaptive/docs/api.md new file mode 100644 index 0000000..5c28503 --- /dev/null +++ b/crates/adaptive/docs/api.md @@ -0,0 +1,315 @@ +# tinyflows-adaptive · API reference for hosts + +How to embed the crate, ordered by what you do: implement the seams, construct +the handles, drive the loop, read the results. Signatures are the real ones; +for full detail every public item carries rustdoc — `cargo doc -p +tinyflows-adaptive --open`. The runnable companion is +[`examples/service.rs`](../examples/service.rs). + +```toml +[dependencies] +tinyflows-adaptive = "0.1" # sqlite ledger/vault on by default +# mongo-only service: +tinyflows-adaptive = { version = "0.1", default-features = false, features = ["mongo"] } +``` + +Modules: `contracts` · `driver` · `execute` · `intake` · `closing` · `ledger` · +`workflows` · `inventory` · `promotion` · `recall` · `reuse` · `host`. + +--- + +## 1 · The skeleton + +```rust +use std::sync::Arc; +use tinyflows::caps::Capabilities; +use tinyflows::store::WorkflowStore; +use tinyflows_adaptive::contracts::{Budget, Goal}; +use tinyflows_adaptive::driver::Loop; +use tinyflows_adaptive::execute::Remote; +use tinyflows_adaptive::host::HostFacts; +use tinyflows_adaptive::ledger::EpisodeStatus; +use tinyflows_adaptive::workflows::Snapshot; + +// once, at boot +let ledger_root = MongoLedger::connect(&uri, "adaptive").await?; +let vault_root = MongoVault::connect(&uri, "adaptive").await?; +let caps = Capabilities { llm: Arc::new(MyTieredClient::new(cfg)), /* … */ }; + +// per request (handles are free) +let ledger = ledger_root.for_tenant(&user_id); +let vault = vault_root.for_tenant(&user_id); + +// per goal run +let snapshot = Snapshot::load(&vault, policy.clone()).await?; +let store: Arc = Arc::new(snapshot.clone()); +let runner = Remote { relay: &my_relay, attempt_id: episode_id.clone() }; + +let engine = Loop { + ledger: &ledger, store: &store, caps: &caps, + facts: &facts, runner: &runner, clock: &my_clock, + budget: Budget::default(), conn: Some(&tenant_credential_ref), +}; +let finished = engine.run(&episode_id, &Goal::new(prompt)).await?; + +// the success gate — a device only ever receives graphs from satisfied runs +if finished.status == EpisodeStatus::Satisfied && snapshot.pending() > 0 { + snapshot.flush(&vault).await?; +} +``` + +--- + +## 2 · Traits you implement + +### `LlmProvider` (from `tinyflows::caps`) — required + +```rust +async fn complete(&self, request: Value, conn: Option<&str>) -> tinyflows::error::Result; +``` + +Every request the loop sends has this shape — `tier` is which job is asking, +`conn` is the opaque credential reference off the `Loop`: + +```json +{ "tier": "judge", + "messages": [ {"role":"system","content":"…"}, {"role":"user","content":"…"} ], + "response_format": { "type": "json_object" } } +``` + +Route `tier` → model in your config (select cheap, judge strong). The reply may +be a bare JSON object, an OpenAI-style envelope (`choices[0].message.content`), +or prose around an object — all three are parsed. What each tier must contain: + +| tier | expected reply | +|---|---| +| `select` | `{"workflow_id": str \| null, "why": str, "inputs": {name: value}}` — null declines; an unknown id reads as declining | +| `author` | `{"graph": , "why": str, "inputs": {name: value}}` — the graph is validated before it is accepted | +| `judge` | `{"satisfied": bool, "blocker": str, "gap": str, "attributed_to": str, "advanced": bool}` — blocker ∈ `goal_not_met · unverified · missing_evidence · needs_input · external_wait`; unrecognised coerces to `goal_not_met` | +| `consolidate` | `{"lessons": [{"kind","trigger","mechanism","claim","evidence":[row numbers]}], "corroborate": [lesson ids]}` — kind ∈ `strategy · constraint · failure_mode · calibration`; a lesson with no cited rows is dropped | +| `repair` | `{"ops": […], "why": str}` — empty/absent ops declines; `rename_node` is refused | +| `generalise` | `{"name": str, "description": str, "reusable": bool}` — prose only; `reusable: false` or an empty description declines | + +### `Relay` (`execute`) — required for remote execution + +```rust +async fn dispatch(&self, request: &RunRequest) -> Result; +``` + +Serialize, send, correlate the reply, apply a deadline; `Err(reason)` on +timeout or no-device — `Remote` turns it into a judgeable attempt, never a +crash. Mint your own unique wire id per dispatch (attempts within an episode +share `attempt_id`). Reference implementation: `ChannelRelay` in the example. + +### `Workspace` (`execute`) — device side, both methods default to empty + +```rust +async fn mark(&self) -> String; // baseline before the run +async fn changed_since(&self, mark: &str) -> String; // prose diff after +``` + +`Unobserved` is the honest no-op. What this returns is the judge's third +evidence source; empty means "nothing reported", never "nothing happened". + +### `Clock` (`driver`) — required, one method + +```rust +fn now(&self) -> String; // RFC 3339; opaque to the crate, drives tests frozen +``` + +### `Ledger` (`ledger`) / `Vault` (`workflows`) — only for a custom backend + +Three of each ship (`memory`, `sqlite`, `mongo`). A fourth implementation runs +the public conformance suites: `ledger::conformance::{run_all, run_tenants, +run_lineage, run_episodes, run_transcripts}` and +`workflows::conformance::{run_all, run_tenants}`. + +### `HostPolicy` (from `tinyflows::store`) — judgement only the host can make + +`check_graph(id, &graph)` vetoes a graph naming a harness/slug this deployment +lacks. A permissive default impl is two lines. + +--- + +## 3 · Configuration — every knob in one place + +**One storage setting drives both halves** (`storage::Config::parse` + +`Storage::open`), and `Storage::for_tenant` scopes ledger *and* vault in one +call — the two-handle scoping mistake cannot be made: + +```rust +let storage = Storage::open(&Config::parse(&cfg.storage)?).await?; // once, at boot +let tenant = storage.for_tenant(&user_id); // per request +// tenant.ledger() → &impl Ledger tenant.vault() → &impl Vault +``` + +| `storage` value | meaning | +|---|---| +| `memory` / `:memory:` | forgets on restart — must be asked for by name, never a fallback | +| `adaptive.db` or any path, `sqlite:` | one SQLite file holding ledger **and** vault | +| `mongodb://host:27017/adaptive` | one Mongo database, both halves; db name from the URI path, default `tinyflows_adaptive` | + +A URI for a backend the build lacks fails **at parse time**, naming the missing +feature. + +What a service configures, and who consumes it: + +| setting | consumed by | values / default | +|---|---|---| +| storage string | `storage::Config::parse`, or `Config::from_env()` / `Storage::from_env()` reading **`TINYFLOWS_ADAPTIVE_STORAGE`** | table above; unset = boot error naming the variable, never a default | +| `TINYFLOWS_ADAPTIVE_DB` env | `SqliteLedger::from_env_or` / `at_default_location` | overrides the sqlite path without a rebuild | +| `Budget { attempts, min_attempts, stall_limit }` | the loop, per `Loop` (per tenant if you like) | `12 / 3 / 2` | +| `conn` | passed verbatim to your `LlmProvider` | opaque tenant credential *reference*, never a secret | +| tier → model map | **your** `LlmProvider`, off the request's `tier` | e.g. select→flash, author/judge→strong, consolidate→mid | +| relay deadline | **your** `Relay` | example uses 30 s; size to your longest workflow | +| `HostFacts` | authoring prompt + post-author check | 15 fields describing the executing machine; `unknown()` forbids nothing | +| `HostPolicy` | store saves + authored graphs | your veto for harnesses/slugs this deployment lacks | +| Cargo features | build | `default = ["sqlite"]`; `mongo`; `default-features = false` for memory-only | + +**Deliberately not configurable** (behaviour, not policy): `MIN_TRIALS` = 3 +runs before a variant can take a family's slot; `RECALL_LIMIT` = all lessons in +scope; `RECORD_BUDGET`/`PROMPT_BUDGET` = 256 KiB / 4 KiB per node. + +## 3b · Storage construction (by hand) + +### Ledger + +| backend | construct | +|---|---| +| memory (always compiled; forgets) | `MemoryLedger::new()` | +| sqlite (default feature) | `SqliteLedger::open(path)` · `::in_memory()` · `::from_env_or(fallback)` · `::at_default_location()` | +| mongo (feature `mongo`) | `MongoLedger::connect(uri, db).await` · `::with_database(db)` | + +All three: `.for_tenant(scope)` → a cheap scoped handle sharing the +connection. `SqliteLedger::open` creates the parent directory; env var +`TINYFLOWS_ADAPTIVE_DB` overrides the path in `from_env_or` / +`at_default_location`. The ledger and the sqlite vault may share one file. + +**The scoping rule everywhere**: writes go to the handle's bucket; reads +return the handle's bucket **plus global** (an unscoped handle's bucket *is* +global). `promote`/`save_episode` stamp the handle's scope and ignore the +argument's. + +### Workflows + +```rust +// any backend → the sync WorkflowStore the loop needs +let snapshot = Snapshot::load(&vault, policy).await?; // one async read +let store: Arc = Arc::new(snapshot.clone()); +// … loop runs; save() buffers in memory, visible to the next attempt at once … +snapshot.pending(); // how many writes wait +snapshot.flush(&vault).await?; // only what changed goes back +``` + +Composing catalogues (`workflows::compat`): + +```rust +StoreVault::new(any_workflow_store) // any WorkflowStore as a Vault +Layered::new(vec![("device".into(), theirs)], ours) // read many, write one + .degrading(Arc::new(|layer, why| warn!(…))) // skip an unreachable read-only + // layer — handler is mandatory +``` + +Reads are the union, later layers shadow by id, writes/deletes reach only the +writable layer, and the writable layer failing is always fatal. + +--- + +## 4 · Driving the loop (`driver::Loop`) + +```rust +pub struct Loop<'a> { + pub ledger: &'a dyn Ledger, + pub store: &'a Arc, + pub caps: &'a Capabilities, + pub facts: &'a HostFacts, + pub runner: &'a dyn Runner, // Local { caps, workspace } | Remote { relay, attempt_id } + pub clock: &'a dyn Clock, + pub budget: Budget, // default: 12 attempts, min 3, stall 2 + pub conn: Option<&'a str>, +} +``` + +`Loop` is a bag of borrows — `Send + Sync`, no per-episode state, build one per +goal run or share one; any replica can pick up any episode. + +| method | returns | notes | +|---|---|---| +| `start(episode, goal)` | `Episode` | idempotent; resumes an existing record | +| `attempt(episode, goal)` | `Closed { verdict, row_id, next, stalled }` | one pass: decide → run → judge → record → repair-if-suspect | +| `run(episode, goal)` | `Finished { status, attempts, verdict, lessons }` | drives to `Satisfied`/`StoodDown`; consolidates once at the end | +| `unfinished()` | `Vec` | the boot recovery list for this tenant | + +Lower-level building blocks (same behaviour the driver composes): +`intake::decide` → `Attempt`, `execute::run_attempt`/`serve` → `Ran`/`RunReport`, +`closing::{close, judge, consolidate, repair, keep}`. + +--- + +## 5 · Reading back + +| read | signature | for | +|---|---|---| +| `inventory::shelf(&store, &ledger)` | `Vec` | a screen/audit — hides nothing, decides nothing | +| `ledger.rows(episode)` | `Vec` | one episode's attempt trail | +| `ledger.steps(row_id)` | `Vec` | one attempt's per-node transcript | +| `ledger.episodes(running_only, Page)` | `Vec` | listing; `Page { limit, offset }`, `Page::ALL`, `Page::first(n)` | +| `ledger.lessons(kind)` / `evidence(lesson_id)` | lessons + the rows behind one | the knowledge plane | +| `ledger.lineage(id)` / `workflow_score(id)` | family root-first / `Score { applied, helped }` | families and evidence | +| `promotion::{champion, standing}` | which family member is offered, and why | `MIN_TRIALS = 3` | +| `recall::{retrieve, render_history, render_lessons}` | what a planner is shown | default `RECALL_LIMIT` = everything in scope | +| `reuse::{baked_in, shape_id}` | pasted-input check / content-derived id | the keep gate, dedup | + +--- + +## 6 · Wire reference (`execute::wire`) + +Everything is `Serialize + Deserialize`; the envelope is **camelCase**, the +`WorkflowGraph` inside it keeps the engine's **snake_case** — both by contract, +pinned in `tests/contracts_surface.rs`. + +```jsonc +// server → device +{ "attemptId": "ep-1#0", + "graph": { "schema_version": 1, "nodes": [...], "edges": [...] }, + "inputs": { "repo": "acme/thing" } } + +// device → server +{ "attemptId": "ep-1#0", + "steps": [ { "nodeId": "report", "status": "success", // "success" | "error" + "output": { … }, // bounded per node, 256 KiB + "durationMs": 12, "nullBindings": [] } ], + "pendingApprovals": [], "cancelled": false, + "changed": "1 file changed", "failed": null, "costUsd": 0.42 } +``` + +Device obligation: `serde_json::from_str::` → `serve(&req, &caps, +&workspace).await` → `serde_json::to_string(&report)`. `RunReport::into_ran(&graph)` +on the server rebuilds outcome + diagnosis; steps cross the wire, `Diagnosis` +does not (re-derived server-side). Budgets: `RECORD_BUDGET` 256 KiB/node +stored, `PROMPT_BUDGET` 4 KiB/node shown to the judge. + +--- + +## 7 · Errors + +| type | variants | meaning | +|---|---|---| +| `intake::IntakeError` | `Store` · `Ledger` · `Inference` · `Invalid` · `Unsupported` · `Unbindable { id, missing }` | `Invalid` = the graph is wrong; `Unsupported` = the graph is fine, this machine is the constraint | +| `ledger::LedgerError` | `Backend` · `Corrupt` | deliberately coarse — retry or give up | +| execution | *never errors* | a failed compile/run/dispatch becomes a `Ran` with `failed: Some(reason)` and still reaches `close()` | + +--- + +## 8 · Invariants worth knowing before you build on top + +- Every inference reply is gated: graphs validated, ops applied to a copy, + lessons need cited rows, scope stamps are the handle's. +- An attempt always leaves a ledger row — including timeouts and compile + failures. `Remote`'s no-reply synthesis reports *unknown*, not "nothing + changed", so a socket blip cannot terminally end an episode. +- `store.save()` inside the loop is a **buffer**; nothing is durable until + `flush`, which is how the host gates persistence on success. +- Content-derived ids (`learned-…`, `…-fix-…`) mean identical work converges + instead of accumulating; evidence recorded early reattaches when the graph + lands. diff --git a/crates/adaptive/examples/service.rs b/crates/adaptive/examples/service.rs new file mode 100644 index 0000000..a2ac2c3 --- /dev/null +++ b/crates/adaptive/examples/service.rs @@ -0,0 +1,428 @@ +//! A worked host: the loop on a server, the engine on a "device", a relay +//! between them. +//! +//! Run it: +//! +//! ```text +//! cargo run -p tinyflows-adaptive --example service +//! ``` +//! +//! Everything here is the real crate driving real serialization — the only +//! stand-ins are the transport (tokio channels where production has a socket) +//! and the model (a script that routes on the `tier` field, where production +//! has an HTTP client). Every seam a production host implements is marked +//! `HOST:`. +//! +//! What it demonstrates, in order: +//! +//! 1. building the tenant handles once and the `Loop` per goal run; +//! 2. a [`Relay`] that serializes a [`RunRequest`], registers a waiter under a +//! unique wire id, sends the frame, and awaits the report with a deadline — +//! the exact shape a Socket.IO handler pair implements; +//! 3. the device side: one call to [`serve`] between deserialize and reply; +//! 4. the success gate: the learned workflow reaches the vault only because +//! the goal run satisfied; +//! 5. the second goal run selecting what the first one learned. + +use std::collections::HashMap; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +use async_trait::async_trait; +use serde_json::{Value, json}; +use tinyflows::caps::mock::mock_capabilities; +use tinyflows::caps::{Capabilities, LlmProvider}; +use tinyflows::error::Result as EngineResult; +use tinyflows::model::{Edge, InputType, Node, NodeKind, WorkflowGraph, WorkflowInput}; +use tinyflows::store::{HostPolicy, WorkflowStore}; +use tinyflows_adaptive::contracts::{Budget, Goal}; +use tinyflows_adaptive::driver::{Clock, Loop}; +use tinyflows_adaptive::execute::{Relay, Remote, RunReport, RunRequest, Unobserved, serve}; +use tinyflows_adaptive::host::HostFacts; +use tinyflows_adaptive::inventory; +use tinyflows_adaptive::ledger::memory::MemoryLedger; +use tinyflows_adaptive::ledger::{EpisodeStatus, Ledger}; +use tinyflows_adaptive::workflows::Snapshot; +use tinyflows_adaptive::workflows::memory::MemoryVault; +use tokio::sync::{mpsc, oneshot}; + +// --------------------------------------------------------------------------- +// The relay — the piece this example exists to show. +// --------------------------------------------------------------------------- + +/// Carries a [`RunRequest`] to wherever the engine is and correlates the +/// [`RunReport`] that comes back. +/// +/// The pattern, independent of transport: +/// +/// * **dispatch**: mint a unique wire id, register a oneshot waiter under it, +/// serialize, send, await with a deadline. The wire id is minted *here* +/// rather than trusting the request's own `attempt_id`, so two concurrent +/// episodes — or a retry racing a late reply — can never resolve each +/// other's waiters. +/// * **deliver**: parse the frame, look up the waiter by the echoed id, +/// resolve it. In production this body *is* your socket receive handler. +/// * **deadline**: return `Err` with a readable reason. [`Remote`] turns that +/// into a judgeable attempt rather than a crash — a device asleep is a fact +/// about the run, not an exception. +struct ChannelRelay { + /// HOST: `socket.emit("tinyflows:flow_run", frame)`. + to_device: mpsc::Sender, + waiting: Mutex>>, + sequence: AtomicU64, + deadline: Duration, +} + +impl ChannelRelay { + fn new(to_device: mpsc::Sender, deadline: Duration) -> Arc { + Arc::new(Self { + to_device, + waiting: Mutex::new(HashMap::new()), + sequence: AtomicU64::new(0), + deadline, + }) + } + + /// HOST: the body of your `socket.on("tinyflows:flow_result", …)` handler. + fn deliver(&self, frame: &str) { + let Ok(report) = serde_json::from_str::(frame) else { + eprintln!(" ! dropped an unparseable report frame"); + return; + }; + let waiter = self + .waiting + .lock() + .expect("waiter lock") + .remove(&report.attempt_id); + match waiter { + Some(tx) => { + let _ = tx.send(report); + } + // A reply after its deadline, or a duplicate. Log and drop — the + // dispatch side already synthesized an unreported attempt. + None => eprintln!(" ! late or unknown report `{}`", report.attempt_id), + } + } +} + +#[async_trait] +impl Relay for ChannelRelay { + async fn dispatch(&self, request: &RunRequest) -> Result { + // A unique wire id per dispatch. The loop's own attempt_id is not + // unique enough: attempts within an episode share it, and a late + // report from attempt 1 must not resolve attempt 2's waiter. + let wire_id = format!( + "{}#{}", + request.attempt_id, + self.sequence.fetch_add(1, Ordering::Relaxed) + ); + let mut framed = request.clone(); + framed.attempt_id = wire_id.clone(); + let frame = serde_json::to_string(&framed).map_err(|e| e.to_string())?; + println!(" → RunRequest {}", peek(&frame)); + + let (tx, rx) = oneshot::channel(); + self.waiting + .lock() + .expect("waiter lock") + .insert(wire_id.clone(), tx); + + if self.to_device.send(frame).await.is_err() { + self.waiting.lock().expect("waiter lock").remove(&wire_id); + return Err("no device connected".to_string()); + } + + match tokio::time::timeout(self.deadline, rx).await { + Ok(Ok(mut report)) => { + println!( + " ← RunReport {} steps, failed: {:?}", + report.steps.len(), + report.failed + ); + // Hand the loop back its own id; the wire salt was ours. + report.attempt_id = request.attempt_id.clone(); + Ok(report) + } + Ok(Err(_)) => Err("the delivery side dropped the waiter".to_string()), + Err(_) => { + self.waiting.lock().expect("waiter lock").remove(&wire_id); + Err(format!("no report within {:?}", self.deadline)) + } + } + } +} + +fn peek(frame: &str) -> String { + let head: String = frame.chars().take(88).collect(); + format!("{head}… ({} bytes)", frame.len()) +} + +// --------------------------------------------------------------------------- +// The device. In production this is medulla behind the socket. +// --------------------------------------------------------------------------- + +/// Deserialize, [`serve`], serialize. That is the whole device obligation. +fn spawn_device(mut from_server: mpsc::Receiver, to_server: mpsc::Sender) { + tokio::spawn(async move { + // HOST: the device's real Capabilities — its harness behind + // `AgentRunner`, its HTTP client, its sandboxed code runner. The mock + // bundle keeps this example self-contained. + let caps = mock_capabilities(); + while let Some(frame) = from_server.recv().await { + let Ok(request) = serde_json::from_str::(&frame) else { + continue; + }; + // HOST: a real Workspace here (git mark / git diff) is what fills + // the `changed` evidence the judge reads. + let report = serve(&request, &caps, &Unobserved).await; + let Ok(reply) = serde_json::to_string(&report) else { + continue; + }; + let _ = to_server.send(reply).await; + } + }); +} + +// --------------------------------------------------------------------------- +// Inference. In production: an HTTP client routing `tier` → model. +// --------------------------------------------------------------------------- + +/// A script standing where the model client goes. The one production-relevant +/// thing about it is the match: every request carries `tier`, and routing on +/// it — select to a cheap model, judge to a strong one — is host config, not +/// crate code. +struct TierRouter; + +#[async_trait] +impl LlmProvider for TierRouter { + async fn complete(&self, request: Value, _conn: Option<&str>) -> EngineResult { + let shown = request["messages"][1]["content"] + .as_str() + .unwrap_or_default(); + Ok(match request["tier"].as_str().unwrap_or_default() { + // Reads the candidate listing it was shown, like a real selector. + "select" => { + let first = shown + .lines() + .find_map(|line| line.trim().strip_prefix("- id: ")); + json!({ + "workflow_id": first, + "why": "matches the goal", + "inputs": { "repo": "acme/rust-lib" }, + }) + } + "author" => json!({ + "graph": review_graph(), + "why": "nothing stored fits yet", + "inputs": { "repo": "acme/thing" }, + }), + "judge" => json!({ "satisfied": true, "gap": "" }), + "generalise" => json!({ + "name": "Review a repository's pull requests", + "description": "Reviews the open pull requests on a repository \ + and posts a summary. Takes the repository as an input.", + "reusable": true, + }), + "consolidate" => json!({ "lessons": [], "corroborate": [] }), + other => json!({ "error": format!("unexpected tier {other}") }), + }) + } +} + +/// The graph the "author" writes: parameterised, which is what lets `keep` +/// file it — the repo arrives through a declared input, never pasted in. +fn review_graph() -> Value { + serde_json::to_value(WorkflowGraph { + schema_version: 1, + id: None, + name: "review-prs".into(), + inputs: vec![WorkflowInput::new("repo", InputType::String).required()], + agents: Vec::new(), + nodes: vec![ + Node { + id: "start".into(), + kind: NodeKind::Trigger, + type_version: 1, + name: "manual".into(), + config: json!({ "trigger_kind": "manual" }), + ports: Vec::new(), + position: None, + }, + Node { + id: "report".into(), + kind: NodeKind::Transform, + type_version: 1, + name: "report".into(), + config: json!({ "set": { "target": "=run.inputs.repo" } }), + ports: Vec::new(), + position: None, + }, + ], + edges: vec![Edge { + from_node: "start".into(), + from_port: "main".into(), + to_node: "report".into(), + to_port: "main".into(), + }], + }) + .expect("a graph serializes") +} + +// --------------------------------------------------------------------------- +// Small host pieces. +// --------------------------------------------------------------------------- + +struct WallClock; +impl Clock for WallClock { + fn now(&self) -> String { + // Opaque to the crate; a real host writes RFC 3339. Zero-padded so the + // episode listing's string ordering matches time ordering. + let secs = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_secs()) + .unwrap_or(0); + format!("{secs:020}") + } +} + +fn permissive() -> Arc { + #[derive(Debug, Default)] + struct Permissive; + impl HostPolicy for Permissive {} + Arc::new(Permissive) +} + +// --------------------------------------------------------------------------- +// The service. +// --------------------------------------------------------------------------- + +#[tokio::main(flavor = "multi_thread")] +async fn main() { + // ---- process scope: once, at boot ----------------------------------- + // HOST: MongoLedger::connect / SqliteLedger::at_default_location, a real + // HTTP-backed LlmProvider, real HostFacts from the device's probe. + let ledger_root = MemoryLedger::new(); + let vault_root = MemoryVault::new(); + let caps = Capabilities { + llm: Arc::new(TierRouter), + ..mock_capabilities() + }; + let facts = HostFacts::unknown(); + + // The wire: two channels where production has one socket. + let (to_device_tx, to_device_rx) = mpsc::channel::(16); + let (to_server_tx, mut to_server_rx) = mpsc::channel::(16); + let relay = ChannelRelay::new(to_device_tx, Duration::from_secs(30)); + spawn_device(to_device_rx, to_server_tx); + { + // HOST: this task is your socket receive handler. + let relay = Arc::clone(&relay); + tokio::spawn(async move { + while let Some(frame) = to_server_rx.recv().await { + relay.deliver(&frame); + } + }); + } + + // ---- tenant scope: per request, free -------------------------------- + let tenant = "user-7"; + let ledger = ledger_root.for_tenant(tenant); + let vault = vault_root.for_tenant(tenant); + + // ---- goal run 1: a cold catalogue, so the loop authors -------------- + println!("── goal run 1 · cold start ──"); + run_goal( + "ep-1", + &Goal::new("review the open pull requests on acme/thing"), + &ledger, + &vault, + &caps, + &facts, + &relay, + ) + .await; + + // ---- goal run 2: the catalogue now holds what run 1 learned --------- + println!("\n── goal run 2 · the loop reuses what it learned ──"); + run_goal( + "ep-2", + &Goal::new("review the open pull requests on acme/rust-lib"), + &ledger, + &vault, + &caps, + &facts, + &relay, + ) + .await; + + // ---- what is on the shelf, and what the trail says ------------------ + println!("\n── the tenant's shelf ──"); + let snapshot = Snapshot::load(&vault, permissive()).await.expect("load"); + let store: Arc = Arc::new(snapshot); + for listing in inventory::shelf(&store, &ledger).await.expect("shelf") { + println!( + " {} · {:?} · run {}× satisfied {}× · learned: {}", + listing.id, + listing.standing, + listing.score.applied, + listing.score.helped, + listing.learned + ); + } + + println!("\n── the trail ──"); + for episode in ["ep-1", "ep-2"] { + for row in ledger.rows(episode).await.expect("rows") { + println!( + " {episode} attempt {} · [{}] → {}", + row.attempt, row.approach_sig, row.outcome + ); + } + } +} + +/// One goal run, end to end: fetch the catalogue, drive the loop over the +/// relay, and persist what was learned only if the goal was achieved. +async fn run_goal( + episode: &str, + goal: &Goal, + ledger: &MemoryLedger, + vault: &MemoryVault, + caps: &Capabilities, + facts: &HostFacts, + relay: &Arc, +) { + // Fetched fresh each goal run. HOST: a Layered vault puts a device + // catalogue (read-only, degrading) in front of this one. + let snapshot = Snapshot::load(vault, permissive()).await.expect("load"); + let store: Arc = Arc::new(snapshot.clone()); + + let runner = Remote { + relay: relay.as_ref(), + attempt_id: episode.to_string(), + }; + let engine = Loop { + ledger, + store: &store, + caps, + facts, + runner: &runner, + clock: &WallClock, + budget: Budget::default(), + conn: None, // HOST: the tenant's credential reference + }; + + let finished = engine.run(episode, goal).await.expect("the loop ran"); + println!( + " {episode}: {:?} after {} attempt(s)", + finished.status, finished.attempts + ); + + // The success gate: the vault — and through it a device — only ever + // receives workflows from goal runs that succeeded. + if finished.status == EpisodeStatus::Satisfied && snapshot.pending() > 0 { + let landed = snapshot.flush(vault).await.expect("flush"); + println!(" flushed {landed} learned workflow(s) to the vault"); + } +} diff --git a/crates/adaptive/src/closing/consolidate.rs b/crates/adaptive/src/closing/consolidate.rs new file mode 100644 index 0000000..2cb4a60 --- /dev/null +++ b/crates/adaptive/src/closing/consolidate.rs @@ -0,0 +1,312 @@ +//! What a finished episode is worth remembering. +//! +//! The ledger records *what happened*; this decides what generalises out of it. +//! Those are different questions, and conflating them is how a knowledge store +//! fills with rows nobody can retrieve: a lesson whose trigger names the +//! original prompt matches exactly one task, forever. +//! +//! Two properties are deliberate and cost something to keep. +//! +//! **Most episodes are worth nothing.** A run that simply worked, or simply did +//! not, teaches nothing a different task could act on. The prompt says so, and +//! keeping nothing is the expected answer rather than a failure. +//! +//! **Consolidation cannot fail the episode.** It happens after the outcome is +//! already settled, so a provider hiccup or an unreadable answer keeps nothing +//! and leaves the real result standing. Every error path here returns an empty +//! list. + +use tinyflows::caps::Capabilities; + +use crate::contracts::{Goal, Tier}; +use crate::intake::ask; +use crate::ledger::{Ledger, LedgerRow, Lesson, LessonKind}; + +const SYSTEM: &str = "\ +You decide what a finished episode is worth remembering. + +You see every attempt it made, what each produced, and why each fell short. +Most episodes are worth nothing: if it simply worked, or simply did not, say so +and keep nothing. Only keep something a *different* task could act on. + +Return JSON: {\"lessons\": [...], \"corroborate\": [...]} + +Each lesson: {\"kind\", \"trigger\", \"mechanism\", \"claim\", \"evidence\": [row numbers]} + +kind is one of: +- strategy X works where Y fails. Lands in the next plan's approach. +- constraint a limit no approach here can cross. Rules approaches out. +- failure_mode a way this silently looks done when it is not. Becomes + something the next run checks for. +- calibration an estimate that was systematically wrong, and by how much. + +trigger is what decides whether this is ever found again, and it is the easiest +thing to get wrong in both directions: + good \"a CPU-bound scan over ~1M items with a sub-100ms target\" + bad \"Project Euler 14 in pure Python\" — names this one task, never matches + anything again + bad \"a task that needs to be fast\" — matches everything, says nothing +Describe the *class* of situation, never the specific instance. + +mechanism is why it is true. claim is what to do about it. +evidence lists the row numbers the lesson is drawn from — a claim with no rows +behind it is a guess, so cite them. + +corroborate lists ids of lessons already stored that this episode independently +confirms. Prefer it over restating one: a lesson confirmed twice is stronger +than two lessons saying the same thing. + +Keep nothing rather than keep something vague."; + +/// Read the episode's ledger and keep what generalises. +/// +/// Returns the lessons written, which is usually none. Never returns an error: +/// see the module note — this runs after the outcome is settled, and failing +/// here would turn a bookkeeping problem into a failed episode. +pub async fn consolidate( + goal: &Goal, + episode: &str, + satisfied: bool, + ledger: &dyn Ledger, + caps: &Capabilities, + conn: Option<&str>, +) -> Vec { + let Ok(rows) = ledger.rows(episode).await else { + return Vec::new(); + }; + if rows.is_empty() { + return Vec::new(); + } + // Everything already stored, not a retrieval view. Retrieval answers "what + // applies to this task" and cuts by help rate, so a lesson written moments + // ago — nothing has had the chance to apply it — sorts last and is dropped. + // Here the question is "does this already exist", and a lesson the model is + // not shown is one it cannot corroborate. + let existing = ledger.lessons(None).await.unwrap_or_default(); + + let user = render(goal, satisfied, &rows, &existing); + let Ok(answer) = ask(caps, conn, Tier::Consolidate, SYSTEM, &user).await else { + return Vec::new(); + }; + + let mut kept = Vec::new(); + for raw in answer["lessons"].as_array().unwrap_or(&Vec::new()) { + let Some(lesson) = read_lesson(raw) else { + continue; + }; + let cites = cited(raw, &rows); + // A claim with no rows behind it is a guess. The prompt asks for + // citations; a lesson that arrives without them is dropped rather than + // stored uncited, because `evidence()` is what makes it auditable + // later. + if cites.is_empty() { + continue; + } + if let Ok(id) = ledger.promote(&lesson, &cites).await { + kept.push(Lesson { id, ..lesson }); + } + } + + // Corroboration is a score, not a new row. It moves both counters, which + // is right: an episode that independently confirmed a lesson both applied + // it and was helped by it. The ordinary denominator comes from the driver, + // which scores every lesson a planner was shown against what happened. + // An id that no longer exists is ignored by the backend. + for id in answer["corroborate"].as_array().unwrap_or(&Vec::new()) { + if let Some(id) = id.as_str().filter(|s| !s.is_empty()) { + let _ = ledger.score_lesson(id, true).await; + } + } + + kept +} + +/// One line per attempt, numbered, because the model cites rows by number. +fn render(goal: &Goal, satisfied: bool, rows: &[LedgerRow], existing: &[Lesson]) -> String { + let attempts = rows + .iter() + .enumerate() + .map(|(i, r)| { + let because = if r.cause.is_empty() { + String::new() + } else { + format!(" (because {})", r.cause) + }; + format!( + "{i}. [{}] {} → {}{because}", + r.approach_sig, r.approach_desc, r.outcome + ) + }) + .collect::>() + .join("\n"); + + let mut out = format!( + "Goal: {}\n\nOutcome: {} after {} attempts\n\nAttempts:\n{attempts}", + goal.text.trim(), + if satisfied { + "satisfied" + } else { + "not satisfied" + }, + rows.len() + ); + if !existing.is_empty() { + out.push_str("\n\nAlready stored (corroborate by id rather than restating):\n"); + for lesson in existing { + out.push_str(&format!( + "- {}: [{:?}] when {} — {}\n", + lesson.id, lesson.kind, lesson.trigger, lesson.claim + )); + } + } + out +} + +/// A lesson is only worth storing when it says both *when* and *what*. +fn read_lesson(raw: &serde_json::Value) -> Option { + let trigger = raw["trigger"].as_str().unwrap_or_default().trim(); + let claim = raw["claim"].as_str().unwrap_or_default().trim(); + if trigger.is_empty() || claim.is_empty() { + return None; + } + Some(Lesson { + id: String::new(), + kind: LessonKind::parse(raw["kind"].as_str().unwrap_or_default()), + trigger: trigger.to_string(), + mechanism: raw["mechanism"] + .as_str() + .unwrap_or_default() + .trim() + .to_string(), + claim: claim.to_string(), + applied: 0, + helped: 0, + // Stamped by the ledger from its own handle, never chosen here. + scope_key: None, + }) +} + +/// Row numbers back to row ids, dropping any the model invented. +fn cited(raw: &serde_json::Value, rows: &[LedgerRow]) -> Vec { + // Membership, not `dedup()`: the model cites rows in the order it thought + // of them, so `[0, 1, 0]` is a legal answer and adjacent-only dedup would + // store the same row twice as evidence for one lesson. + let mut ids: Vec = Vec::new(); + for id in raw["evidence"] + .as_array() + .map(|a| { + a.iter() + .filter_map(serde_json::Value::as_u64) + .filter_map(|i| usize::try_from(i).ok()) + .filter_map(|i| rows.get(i)) + .map(|r| r.id.clone()) + .collect::>() + }) + .unwrap_or_default() + { + if !ids.contains(&id) { + ids.push(id); + } + } + ids +} + +#[cfg(test)] +mod tests { + use super::*; + + fn row(id: &str, sig: &str) -> LedgerRow { + LedgerRow { + id: id.into(), + episode: "e".into(), + attempt: 1, + approach_sig: sig.into(), + approach_desc: "tried the obvious thing".into(), + workflow_id: None, + outcome: "fell short".into(), + cause: "the file was never written".into(), + cost_usd: 0.0, + at: "2026-01-01T00:00:00Z".into(), + satisfied: false, + advanced: false, + } + } + + #[test] + fn a_lesson_without_a_trigger_is_not_worth_storing() { + let raw = serde_json::json!({"kind": "strategy", "claim": "do the thing"}); + assert!(read_lesson(&raw).is_none()); + } + + #[test] + fn a_lesson_without_a_claim_is_not_worth_storing() { + let raw = serde_json::json!({"kind": "strategy", "trigger": "a class of task"}); + assert!(read_lesson(&raw).is_none()); + } + + #[test] + fn an_unrecognised_kind_still_keeps_the_lesson() { + let raw = serde_json::json!({ + "kind": "vibes", "trigger": "a class of task", "claim": "do the thing" + }); + let lesson = read_lesson(&raw).expect("kept"); + assert_eq!(lesson.kind, LessonKind::Strategy); + } + + #[test] + fn citations_resolve_row_numbers_to_row_ids() { + let rows = vec![row("r1", "a"), row("r2", "b")]; + let raw = serde_json::json!({"evidence": [0, 1]}); + assert_eq!(cited(&raw, &rows), vec!["r1", "r2"]); + } + + #[test] + fn a_row_cited_twice_non_adjacently_is_stored_once() { + // `[0, 1, 0]` — Vec::dedup only removes adjacent repeats. + let rows = vec![row("r1", "a"), row("r2", "b")]; + let raw = serde_json::json!({"evidence": [0, 1, 0]}); + assert_eq!(cited(&raw, &rows), vec!["r1", "r2"]); + } + + #[test] + fn a_row_number_that_does_not_exist_is_dropped_not_fatal() { + let rows = vec![row("r1", "a")]; + let raw = serde_json::json!({"evidence": [0, 9]}); + assert_eq!(cited(&raw, &rows), vec!["r1"]); + } + + #[test] + fn the_rendering_numbers_attempts_from_zero_as_the_prompt_cites_them() { + let goal = Goal::new("make it fast"); + let rows = vec![row("r1", "sig-a"), row("r2", "sig-b")]; + let rendered = render(&goal, false, &rows, &[]); + assert!(rendered.contains("0. [sig-a]"), "{rendered}"); + assert!(rendered.contains("1. [sig-b]"), "{rendered}"); + assert!( + rendered.contains("not satisfied after 2 attempts"), + "{rendered}" + ); + assert!( + rendered.contains("because the file was never written"), + "{rendered}" + ); + } + + #[test] + fn stored_lessons_are_shown_by_id_so_they_can_be_corroborated() { + let goal = Goal::new("make it fast"); + let existing = vec![Lesson { + id: "L7".into(), + kind: LessonKind::Constraint, + trigger: "a sub-100ms target".into(), + mechanism: String::new(), + claim: "pure Python will not get there".into(), + applied: 3, + helped: 2, + scope_key: None, + }]; + let rendered = render(&goal, true, &[row("r1", "a")], &existing); + assert!(rendered.contains("- L7:"), "{rendered}"); + assert!(rendered.contains("corroborate by id"), "{rendered}"); + } +} diff --git a/crates/adaptive/src/closing/judge.rs b/crates/adaptive/src/closing/judge.rs new file mode 100644 index 0000000..b7cc452 --- /dev/null +++ b/crates/adaptive/src/closing/judge.rs @@ -0,0 +1,389 @@ +//! Deciding whether a run actually did the job. +//! +//! Two stages, and the order is the whole point. **Mechanical evidence first**: +//! the engine's own diagnosis of the run says, deterministically, that a +//! binding resolved to null or that half the graph never executed. Those are +//! facts, they cost nothing, and a model asked to weigh them will sometimes +//! decide the run went fine anyway. +//! +//! Only what mechanism cannot settle goes to a model — and it is shown the +//! diagnosis rather than asked to infer it. +//! +//! The judge is deliberately context-poor: goal, outcome, diagnosis. It does +//! not see the ledger, so it cannot propose what to try next, because it does +//! not know what has already been ruled out. That is the planner's job. + +use tinyflows::caps::Capabilities; +use tinyflows::diagnostics::Diagnosis; +use tinyflows::engine::RunOutcome; +use tinyflows::evidence::bounded_evidence; + +use crate::contracts::{Blocker, Goal, Tier, Verdict}; +use crate::intake::{Result, ask}; + +const SYSTEM: &str = "\ +You judge whether a workflow run achieved a goal. + +Return JSON: {\"satisfied\": bool, \"blocker\": str, \"gap\": str, + \"attributed_to\": str, \"advanced\": bool} + +- satisfied: did the run achieve the goal. Not \"did it finish\" — a run can + complete every node and achieve nothing. +- blocker: when not satisfied, one of + goal_not_met it tried and fell short. The ordinary case. + unverified something was produced but the evidence does not show it + working. + missing_evidence nothing was produced and there is nothing to judge. + needs_input a person has to answer something first. + external_wait waiting on something outside this system. +- gap: one line on what is still missing. It is read by whoever plans the next + attempt, so name the missing thing, not the feeling. +- attributed_to: the node id that fell short, when the evidence says which. +- advanced: did this run get closer to the goal than the state before it. + A run can fail and still advance — establishing what the problem is counts. + A run that produced the same nothing as the last one did not. + +Judge the EVIDENCE, not the run's own account of itself. A node reporting +success having written nothing is the failure this exists to catch, and the +diagnosis below is the engine's own reading of what the steps actually did."; + +/// What the run left behind, as the judge sees it. +/// +/// Assembled by the caller so the judge cannot reach for anything else: it gets +/// the outcome, the diagnosis, and nothing about history. +#[derive(Debug, Clone)] +pub struct Evidence<'a> { + /// What the engine returned. + pub outcome: &'a RunOutcome, + /// The engine's own reading of the steps — the four things a green outcome + /// hides. + pub diagnosis: &'a Diagnosis, + /// What changed outside the run state, when the host can say. A workspace + /// diff, a list of files, whatever the host counts as proof. Empty is + /// honest; a fabricated summary is not. + pub changed: String, +} + +impl Evidence<'_> { + /// The parts of the diagnosis worth a sentence each. + /// + /// `unverifiable` null bindings are dropped: the engine marks an expression + /// it could not evaluate even in principle, and reporting those as findings + /// buries the ones that are real. + fn findings(&self) -> Vec { + let mut out = Vec::new(); + for binding in &self.diagnosis.null_bindings { + if binding.unverifiable { + continue; + } + let from = binding + .reads_from + .as_deref() + .map_or(String::new(), |n| format!(", reading from `{n}`")); + out.push(format!( + "node `{}`: `{}` at {} resolved to null{from} — {}", + binding.node_id, binding.expression, binding.location, binding.suggestion + )); + } + for node in &self.diagnosis.empty_prompts { + out.push(format!( + "node `{node}`: dispatched an agent session with an empty prompt" + )); + } + for hidden in &self.diagnosis.hidden_errors { + out.push(format!( + "node `{}`: errored, and its on_error policy swallowed it{}", + hidden.node_id, + hidden + .message + .as_deref() + .map_or(String::new(), |m| format!(" — {m}")) + )); + } + for skipped in &self.diagnosis.never_ran { + out.push(format!( + "node `{}`: never ran{}", + skipped.node_id, + skipped + .routed_by + .as_deref() + .map_or(String::new(), |n| format!(", routed past by `{n}`")) + )); + } + out + } + + pub(super) fn render(&self) -> String { + let findings = self.findings(); + let diagnosis = if findings.is_empty() { + "the engine found nothing wrong with the steps".to_string() + } else { + findings.join("\n- ") + }; + format!( + "# Run outcome\n{}\n\n# What the engine's diagnosis found\n- {diagnosis}\n\n\ + # What changed outside the run\n{}", + serde_json::to_string_pretty(&bounded_evidence(&self.outcome.output)) + .unwrap_or_else(|_| "(unreadable)".into()), + if self.changed.is_empty() { + "(nothing reported)" + } else { + &self.changed + } + ) + } +} + +/// Judge a finished run. +/// +/// Three outcomes are decided without a model at all, because they are facts +/// rather than judgements and paying for an opinion on a fact is how a loop +/// gets expensive: +/// +/// * a parked approval is `needs_input`; +/// * a cancelled run is `external_wait` — it did not fail, it was stopped; +/// * a run that produced nothing *and* whose diagnosis says nothing ran is +/// `missing_evidence`, which is terminal, because a retry with the same +/// inputs produces the same nothing. +/// +/// # Errors +/// When inference fails or answers with nothing usable. +pub async fn judge( + goal: &Goal, + evidence: &Evidence<'_>, + caps: &Capabilities, + conn: Option<&str>, +) -> Result { + if let Some(settled) = without_a_model(evidence) { + return Ok(settled); + } + + let criteria = if goal.success_criteria.trim().is_empty() { + String::new() + } else { + format!("\n\n# Done when\n{}", goal.success_criteria.trim()) + }; + let user = format!( + "# Goal\n{}{criteria}\n\n{}", + goal.text.trim(), + evidence.render() + ); + + let answer = ask(caps, conn, Tier::Judge, SYSTEM, &user).await?; + let satisfied = answer["satisfied"].as_bool().unwrap_or(false); + Ok(Verdict { + satisfied, + // A satisfied verdict has no blocker whatever the model wrote in the + // field; the two disagreeing is a state nothing downstream can read. + blocker: if satisfied { + Blocker::None + } else { + Blocker::parse(answer["blocker"].as_str().unwrap_or_default()) + }, + gap: answer["gap"].as_str().unwrap_or_default().to_string(), + attributed_to: answer["attributed_to"] + .as_str() + .unwrap_or_default() + .to_string(), + evidence: evidence.findings().join("; "), + // Absent must not read as "made no progress" — that would stall a run + // for a field the model simply did not write. + advanced: answer["advanced"].as_bool().unwrap_or(true), + }) +} + +/// The verdicts that are facts rather than opinions. +fn without_a_model(evidence: &Evidence<'_>) -> Option { + let outcome = evidence.outcome; + + if !outcome.pending_approvals.is_empty() { + return Some(Verdict { + satisfied: false, + blocker: Blocker::NeedsInput, + gap: format!( + "parked for approval at: {}", + outcome.pending_approvals.join(", ") + ), + attributed_to: outcome + .pending_approvals + .first() + .cloned() + .unwrap_or_default(), + evidence: String::new(), + // It got as far as the gate. That is progress, and calling it a + // stall would count a waiting run against the stall limit. + advanced: true, + }); + } + + if outcome.cancelled { + return Some(Verdict { + satisfied: false, + blocker: Blocker::ExternalWait, + gap: "the run was cancelled before it finished".to_string(), + attributed_to: String::new(), + evidence: String::new(), + advanced: true, + }); + } + + // Nothing ran and nothing changed. There is no judgement to make and no + // second opinion worth buying. + let nothing_ran = !evidence.diagnosis.never_ran.is_empty() + && outcome + .output + .get("nodes") + .is_none_or(|n| n.as_object().is_none_or(serde_json::Map::is_empty)); + if nothing_ran && evidence.changed.is_empty() { + return Some(Verdict { + satisfied: false, + blocker: Blocker::MissingEvidence, + gap: "no node produced anything and nothing changed outside the run".to_string(), + attributed_to: String::new(), + evidence: String::new(), + advanced: false, + }); + } + + None +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + use tinyflows::diagnostics::{HiddenError, NeverRan, NullBinding}; + + fn outcome(output: serde_json::Value) -> RunOutcome { + RunOutcome { + output, + pending_approvals: Vec::new(), + cancelled: false, + } + } + + fn evidence<'a>(o: &'a RunOutcome, d: &'a Diagnosis) -> Evidence<'a> { + Evidence { + outcome: o, + diagnosis: d, + changed: String::new(), + } + } + + #[test] + fn a_parked_approval_needs_no_model() { + let mut o = outcome(json!({})); + o.pending_approvals = vec!["gate".into()]; + let d = Diagnosis::default(); + let verdict = without_a_model(&evidence(&o, &d)).expect("settled without a model"); + assert_eq!(verdict.blocker, Blocker::NeedsInput); + assert!( + verdict.advanced, + "reaching the gate is progress, not a stall" + ); + } + + #[test] + fn a_cancelled_run_did_not_fail_it_was_stopped() { + let mut o = outcome(json!({ "nodes": { "a": {} } })); + o.cancelled = true; + let d = Diagnosis::default(); + let verdict = without_a_model(&evidence(&o, &d)).expect("settled"); + assert_eq!(verdict.blocker, Blocker::ExternalWait); + assert!( + !verdict.blocker.continuable(), + "retrying now is not retrying later" + ); + } + + #[test] + fn a_run_where_nothing_ran_and_nothing_changed_is_terminal() { + let o = outcome(json!({})); + let d = Diagnosis { + never_ran: vec![NeverRan { + node_id: "work".into(), + routed_by: Some("gate".into()), + }], + ..Diagnosis::default() + }; + let verdict = without_a_model(&evidence(&o, &d)).expect("settled"); + assert_eq!(verdict.blocker, Blocker::MissingEvidence); + assert!(!verdict.blocker.continuable()); + } + + #[test] + fn a_run_that_produced_something_goes_to_the_model() { + let o = outcome(json!({ "nodes": { "a": { "items": [1] } } })); + let d = Diagnosis::default(); + assert!( + without_a_model(&evidence(&o, &d)).is_none(), + "a real outcome is a judgement, not a fact" + ); + } + + #[test] + fn an_unverifiable_null_binding_is_not_reported_as_a_finding() { + // The engine marks expressions it could not evaluate even in principle. + // Reporting those buries the ones that are real. + let o = outcome(json!({})); + let d = Diagnosis { + null_bindings: vec![NullBinding { + node_id: "a".into(), + location: "config.prompt".into(), + expression: "=nodes.x.item".into(), + unverifiable: true, + reads_from: None, + suggestion: "n/a".into(), + }], + ..Diagnosis::default() + }; + assert!(evidence(&o, &d).findings().is_empty()); + } + + #[test] + fn a_swallowed_error_reaches_the_judge() { + // The failure a naive reading misses entirely: the step is marked + // failed and its diagnostics are empty. + let o = outcome(json!({})); + let d = Diagnosis { + hidden_errors: vec![HiddenError { + node_id: "fetch".into(), + message: Some("404".into()), + }], + ..Diagnosis::default() + }; + let findings = evidence(&o, &d).findings(); + assert_eq!(findings.len(), 1); + assert!(findings[0].contains("swallowed"), "{findings:?}"); + assert!(findings[0].contains("404")); + } + + #[test] + fn a_null_binding_names_the_node_it_should_have_read_from() { + let o = outcome(json!({})); + let d = Diagnosis { + null_bindings: vec![NullBinding { + node_id: "review".into(), + location: "config.prompt".into(), + expression: "=nodes.fetch.item.body".into(), + unverifiable: false, + reads_from: Some("fetch".into()), + suggestion: "did you mean .item.json.body".into(), + }], + ..Diagnosis::default() + }; + let findings = evidence(&o, &d).findings(); + assert!(findings[0].contains("reading from `fetch`"), "{findings:?}"); + assert!( + findings[0].contains("item.json.body"), + "the suggestion carries" + ); + } + + #[test] + fn a_clean_run_says_so_rather_than_showing_an_empty_list() { + let o = outcome(json!({ "nodes": {} })); + let d = Diagnosis::default(); + assert!(evidence(&o, &d).render().contains("found nothing wrong")); + } +} diff --git a/crates/adaptive/src/closing/keep.rs b/crates/adaptive/src/closing/keep.rs new file mode 100644 index 0000000..59126e1 --- /dev/null +++ b/crates/adaptive/src/closing/keep.rs @@ -0,0 +1,155 @@ +//! Turning a graph that worked into a procedure that stays. +//! +//! The missing half of *"selects a stored workflow or authors one"*. Authoring +//! ran, produced a graph, the graph achieved the goal — and then the graph was +//! discarded, so the next episode of the same shape authored it again from +//! nothing. The catalogue only ever held what a person had put there, and +//! `select` could never choose something the loop itself worked out. +//! +//! Three gates, cheapest first, and each rules out a different kind of mistake. +//! +//! 1. **It has to have worked.** A graph that fell short is the repair path's +//! business, not this one's. +//! 2. **It has to be reusable** — [`crate::reuse::baked_in`], which is exact +//! rather than a judgement: an input value pasted into a node instead of +//! read through a binding means the graph matches one task and never +//! another. No model is asked, because a fuzzy gate on a store that grows +//! forever is a store that fills with near-misses. +//! 3. **It has to be describable as a class.** Only then is a model asked, and +//! only for prose — the graph is already fixed. `select` reads descriptions +//! to choose, so a stored workflow described by the goal that produced it +//! ("summarise /docs/q3.pdf") is unfindable by the next goal of its kind. +//! +//! The name and description are the whole of what inference contributes here, +//! and the [`Tier::Generalise`] request says so. It is the same judgement the +//! consolidator makes about a lesson's `trigger`: describe the situation, never +//! the instance. + +use std::sync::Arc; + +use tinyflows::caps::Capabilities; +use tinyflows::model::WorkflowGraph; +use tinyflows::store::{WorkflowRecord, WorkflowStore}; + +use crate::contracts::{Goal, Tier}; +use crate::intake::{IntakeError, Result, ask}; +use crate::reuse::baked_in; + +const SYSTEM: &str = "\ +You name a workflow that just achieved a goal, so it can be found again. + +Return JSON: {\"name\": str, \"description\": str, \"reusable\": bool} + +The graph is finished and you are not editing it. You are writing the two lines +a planner reads when deciding whether this procedure does what a NEW goal asks. + +- name: a few words. What it does, not what it was for. +- description: one or two sentences naming the CLASS of task and what the + workflow needs to be given. It is the only thing a planner sees besides the + step count, so a description that restates the original goal makes this + findable exactly once. + + good \"Reviews the open pull requests on a repository and posts a summary. + Takes the repository as an input.\" + bad \"Reviews the open PRs on acme/thing.\" — names one instance + bad \"Does the thing that was asked.\" — names nothing + +- reusable: false when this graph only makes sense for the one goal it was + written for, whatever its inputs say. A one-off kept in the catalogue is a row + every future planner reads and none can use, so say so rather than reaching + for a description that sounds general."; + +/// What was kept, when anything was. +#[derive(Debug, Clone)] +pub struct Kept { + /// The stored record. Its id is derived from the graph's shape, so the same + /// procedure arrived at twice converges rather than accumulating. + pub record: WorkflowRecord, + /// The class of task it was described as, in the model's words. + pub description: String, +} + +/// Keep an authored graph that achieved its goal, if it is worth keeping. +/// +/// `Ok(None)` is the ordinary answer and not a failure: the graph baked its +/// specifics in, or the model judged it a one-off. +/// +/// # Errors +/// When inference fails, or the store refuses the record. +pub async fn keep( + goal: &Goal, + graph: &WorkflowGraph, + inputs: &serde_json::Map, + store: &Arc, + caps: &Capabilities, + conn: Option<&str>, +) -> Result> { + // Exact, and free. A graph that pasted its inputs matches one task, and no + // description can make it match another. + let pasted = baked_in(graph, inputs); + if !pasted.is_empty() { + return Ok(None); + } + + let declared = if graph.inputs.is_empty() { + "(none)".to_string() + } else { + graph + .inputs + .iter() + .map(|input| format!("- {}", input.name)) + .collect::>() + .join("\n") + }; + let user = format!( + "# The goal it achieved\n{}\n\n# Its declared inputs\n{declared}\n\n# The graph\n{}", + goal.text.trim(), + serde_json::to_string_pretty(graph).map_err(|e| IntakeError::Store(e.to_string()))? + ); + + let answer = ask(caps, conn, Tier::Generalise, SYSTEM, &user).await?; + if !answer["reusable"].as_bool().unwrap_or(false) { + return Ok(None); + } + let description = answer["description"] + .as_str() + .unwrap_or_default() + .trim() + .to_string(); + // A workflow nobody can choose on purpose is a row that costs a planner + // attention and returns nothing, so an empty description is a refusal. + if description.is_empty() { + return Ok(None); + } + + let name = answer["name"] + .as_str() + .unwrap_or_default() + .trim() + .to_string(); + let id = crate::reuse::shape_id(graph); + let record = WorkflowRecord { + id: id.clone(), + name: if name.is_empty() { id.clone() } else { name }, + description, + enabled: true, + defaults: tinyflows::store::types::WorkflowDefaults::default(), + graph: WorkflowGraph { + id: Some(id), + ..graph.clone() + }, + // Never inherited and never invented: this graph came from a model, not + // from a file, and claiming a path would make the store think it owns + // something on disk. + source_path: None, + }; + store + .save(&record) + .map_err(|e| IntakeError::Store(e.to_string()))?; + + let description = record.description.clone(); + Ok(Some(Kept { + record, + description, + })) +} diff --git a/crates/adaptive/src/closing/mod.rs b/crates/adaptive/src/closing/mod.rs new file mode 100644 index 0000000..6e22654 --- /dev/null +++ b/crates/adaptive/src/closing/mod.rs @@ -0,0 +1,312 @@ +//! What happens after a run: judge it, record it, score it, decide. +//! +//! The closing half of the loop. Intake decided *how* to attempt the goal and +//! the engine carried it out; this reads what came back and turns it into the +//! two things that outlive the attempt — a ledger row, and a score against the +//! workflow that ran. +//! +//! The order matters and is not obvious. **Recording happens whatever the +//! verdict**, before any decision about retrying. A run that failed and was not +//! written down is a run the next attempt will repeat, so the ledger write is +//! not conditional on success — it is most valuable when the news is bad. + +mod consolidate; +mod judge; +mod keep; +mod repair; + +pub use consolidate::consolidate; +pub use judge::{Evidence, judge}; +pub use keep::{Kept, keep}; +pub use repair::{Variant, graph_is_suspect, repair}; + +use crate::contracts::{Approach, Budget, Goal, Verdict}; +use crate::intake::Result; +use crate::ledger::{Episode, EpisodeStatus, Ledger, LedgerRow}; +use tinyflows::caps::Capabilities; + +/// What the loop should do next. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Next { + /// The goal was met. + Done, + /// Attempt it again. The planner will see the exclusion list this closing + /// pass just added to. + Retry, + /// Stop without success: the blocker is terminal, or the budget is spent, + /// or the run stopped advancing. The reason is worth keeping because + /// "stood down" and "failed" read very differently to whoever asked. + StandDown(String), +} + +/// One finished attempt, closed out. +#[derive(Debug, Clone)] +pub struct Closed { + /// What the judge concluded. + pub verdict: Verdict, + /// The ledger row this attempt left behind. + pub row_id: String, + /// What to do next. + pub next: Next, + /// Consecutive non-advancing attempts, to carry into the next pass. + pub stalled: u32, +} + +/// Judge a finished run, record it, score it, and say what to do next. +/// +/// Takes the whole [`crate::execute::Ran`] rather than just its +/// [`Evidence`]: the cost and the per-node transcript are on it, both were +/// being measured and then dropped here, and a signature that cannot see them +/// is a signature that will drop them again. +/// +/// The stall count is **read from and written back to the episode record**, +/// not threaded by the caller. It used to be a parameter, on the reasoning that +/// two episodes sharing one closing layer must not share a counter — true, but +/// the fix was keying it by episode, not making the caller hold it. A counter +/// that lives only in the caller's memory is a counter a deploy loses, and an +/// episode whose stall count silently resets to zero will keep retrying an +/// approach that stopped working four attempts ago. +/// +/// The episode record is created here when it does not exist, so +/// [`Ledger::save_episode`] is optional for a caller that only wants the loop. +/// +/// # Errors +/// When inference fails, or the ledger cannot be read or written. +#[allow(clippy::too_many_arguments)] +pub async fn close( + goal: &Goal, + episode: &str, + attempt: u32, + approach: &Approach, + ran: &crate::execute::Ran, + budget: &Budget, + ledger: &dyn Ledger, + caps: &Capabilities, + conn: Option<&str>, + now: &str, +) -> Result { + let verdict = judge(goal, &ran.evidence(), caps, conn).await?; + let mut record = ledger.episode(episode).await?.unwrap_or(Episode { + id: episode.to_string(), + goal: goal.clone(), + scope_key: None, + status: EpisodeStatus::Running, + attempt: 0, + stalled: 0, + started_at: now.to_string(), + updated_at: now.to_string(), + }); + let stalled = record.stalled; + + // Recorded before anything is decided, and whatever the verdict. A failed + // attempt nobody wrote down is one the next attempt repeats. + let workflow_id = match approach { + // The id that ran, which for a repaired graph is the variant's own — + // scoring its parent instead would leave the two indistinguishable and + // the promotion gate with nothing to compare. + Approach::Selected { workflow_id, .. } => Some(workflow_id.clone()), + Approach::Authored { .. } => None, + }; + let row_id = ledger + .append(&LedgerRow { + id: String::new(), + episode: episode.to_string(), + attempt, + approach_sig: approach.signature(), + approach_desc: why(approach), + workflow_id: workflow_id.clone(), + outcome: outcome_line(&verdict), + cause: verdict.gap.clone(), + // What the runner measured. It was on the wire and on `Ran` all + // along; writing zero here made every row claim the attempt was + // free, which is indistinguishable from a host that does not meter. + cost_usd: ran.cost_usd, + at: now.to_string(), + satisfied: verdict.satisfied, + advanced: verdict.advanced, + }) + .await?; + + // The per-node record behind the row. Best-effort: the attempt is judged + // and scored either way, and losing the transcript costs a reader detail + // rather than costing the loop its result. + let _ = ledger.save_steps(&row_id, &ran.steps).await; + + // The rung medulla-v2 never had: without this nothing distinguishes a + // procedure that has worked forty times from one that has never run, and + // the promotion gate has no evidence to read. + if let Some(id) = workflow_id { + ledger.score_workflow(&id, verdict.satisfied).await?; + } + + let stalled = if verdict.satisfied || verdict.advanced { + 0 + } else { + stalled + 1 + }; + let next = decide_next(&verdict, attempt, stalled, budget); + + // Written after the row and the score, so a checkpoint never claims an + // attempt the ledger has no record of. + record.attempt = attempt; + record.stalled = stalled; + record.updated_at = now.to_string(); + record.status = match &next { + Next::Done => EpisodeStatus::Satisfied, + Next::Retry => EpisodeStatus::Running, + Next::StandDown(reason) => EpisodeStatus::StoodDown(reason.clone()), + }; + ledger.save_episode(&record).await?; + + Ok(Closed { + verdict, + row_id, + next, + stalled, + }) +} + +fn decide_next(verdict: &Verdict, attempt: u32, stalled: u32, budget: &Budget) -> Next { + if verdict.satisfied { + return Next::Done; + } + if verdict.should_retry(attempt, stalled, budget) { + return Next::Retry; + } + // Each reason is worth distinguishing: a terminal blocker is the goal's + // fault, a spent budget is ours, and a stall is the approach running out + // of ideas. Collapsing them to "failed" loses the only thing a reader can + // act on. + Next::StandDown(if !verdict.blocker.continuable() { + format!("{:?} — {}", verdict.blocker, verdict.gap) + } else if budget.exhausted(attempt) { + format!("out of attempts after {attempt}") + } else { + format!("{stalled} attempts in a row made no progress") + }) +} + +fn why(approach: &Approach) -> String { + match approach { + Approach::Selected { why, .. } | Approach::Authored { why, .. } => why.clone(), + } +} + +fn outcome_line(verdict: &Verdict) -> String { + if verdict.satisfied { + "satisfied".to_string() + } else if verdict.gap.is_empty() { + format!("{:?}", verdict.blocker) + } else { + verdict.gap.clone() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::contracts::Blocker; + + fn verdict(satisfied: bool, blocker: Blocker, advanced: bool) -> Verdict { + Verdict { + satisfied, + blocker, + gap: "something is missing".into(), + attributed_to: String::new(), + evidence: String::new(), + advanced, + } + } + + #[test] + fn a_satisfied_verdict_is_done() { + let next = decide_next( + &verdict(true, Blocker::None, true), + 1, + 0, + &Budget::default(), + ); + assert_eq!(next, Next::Done); + } + + #[test] + fn an_ordinary_shortfall_retries() { + let next = decide_next( + &verdict(false, Blocker::GoalNotMet, true), + 1, + 0, + &Budget::default(), + ); + assert_eq!(next, Next::Retry); + } + + #[test] + fn a_terminal_blocker_stands_down_naming_itself() { + let next = decide_next( + &verdict(false, Blocker::NeedsInput, true), + 1, + 0, + &Budget::default(), + ); + match next { + Next::StandDown(reason) => assert!(reason.contains("NeedsInput"), "{reason}"), + other => panic!("expected a stand-down, got {other:?}"), + } + } + + #[test] + fn a_spent_budget_says_so_rather_than_blaming_the_approach() { + let next = decide_next( + &verdict(false, Blocker::GoalNotMet, true), + 12, + 0, + &Budget::default(), + ); + match next { + Next::StandDown(reason) => assert!(reason.contains("out of attempts"), "{reason}"), + other => panic!("expected a stand-down, got {other:?}"), + } + } + + #[test] + fn a_stall_says_so_rather_than_blaming_the_budget() { + let next = decide_next( + &verdict(false, Blocker::GoalNotMet, false), + 5, + 2, + &Budget::default(), + ); + match next { + Next::StandDown(reason) => assert!(reason.contains("no progress"), "{reason}"), + other => panic!("expected a stand-down, got {other:?}"), + } + } + + #[test] + fn an_advancing_attempt_clears_the_stall_count() { + // The whole reason `advanced` exists: a run converging over five + // attempts must not accumulate a stall from the two that looked flat. + let budget = Budget::default(); + assert_eq!( + decide_next(&verdict(false, Blocker::GoalNotMet, true), 9, 0, &budget), + Next::Retry + ); + } + + #[test] + fn the_ledger_row_records_a_failure_in_its_own_words() { + let v = verdict(false, Blocker::GoalNotMet, true); + assert_eq!(outcome_line(&v), "something is missing"); + assert_eq!( + outcome_line(&verdict(true, Blocker::None, true)), + "satisfied" + ); + } + + #[test] + fn a_blockers_name_is_the_outcome_when_the_judge_gave_no_gap() { + let mut v = verdict(false, Blocker::MissingEvidence, false); + v.gap = String::new(); + assert_eq!(outcome_line(&v), "MissingEvidence"); + } +} diff --git a/crates/adaptive/src/closing/repair.rs b/crates/adaptive/src/closing/repair.rs new file mode 100644 index 0000000..d2b5d2c --- /dev/null +++ b/crates/adaptive/src/closing/repair.rs @@ -0,0 +1,413 @@ +//! Fixing the graph, when the graph is what fell short. +//! +//! The other half of learning. A lesson changes what the *next plan* thinks; +//! this changes the procedure itself — an edge that was never wired, a binding +//! that read the envelope instead of its `json` field, a node routed past by a +//! condition that could not be true. +//! +//! Three rules make this safe to run unattended. +//! +//! **A repair is a variant, never an overwrite.** The parent has a score +//! ([`crate::ledger::Ledger::workflow_score`]) built from every run it has ever +//! had. Editing it in place destroys that evidence and leaves nothing to +//! compare the fix against — a "learning" system that cannot tell whether it +//! learned. The variant starts at 0/0 and has to earn its way past the parent. +//! +//! **Only when the graph is actually at fault.** An agent that ran, was wired +//! correctly, and simply did a poor job is not a graph problem, and rewriting +//! the graph in response churns the store while fixing nothing. The gate is +//! mechanical and runs before any inference. +//! +//! **No renames.** [`GraphOp::RenameNode`] rewires edges but does not rewrite +//! `=nodes.…` expressions inside other nodes' configs — the graph +//! validates and then runs quietly wrong. A person doing this by hand can +//! re-point the bindings; a batch arriving from a model cannot be trusted to, +//! so the op is refused here. + +use std::sync::Arc; + +use tinyflows::caps::Capabilities; +use tinyflows::graph_ops::{GraphOp, apply_ops}; +use tinyflows::store::{WorkflowRecord, WorkflowStore}; +use tinyflows::validate::validate_all; + +use super::judge::Evidence; +use crate::contracts::{Goal, Tier, Verdict}; +use crate::intake::{IntakeError, Result, ask}; +use crate::ledger::Ledger; + +const SYSTEM: &str = "\ +You repair a workflow graph that ran and fell short. + +You are given the graph, the engine's own diagnosis of what its steps did, and +the judge's account of what is still missing. Return the smallest batch of edits +that would fix it. + +Return JSON: {\"ops\": [...], \"why\": str} + +Each op is one object, tagged by `op`: + {\"op\": \"add_node\", \"node\": {...}} + {\"op\": \"update_node_config\", \"id\": str, \"config\": {...}} + A JSON merge patch: keys merge onto the existing config, a null deletes. + This is the op for fixing a wrong binding, and usually the only one needed. + {\"op\": \"set_node_name\", \"id\": str, \"name\": str} + {\"op\": \"remove_node\", \"id\": str} + {\"op\": \"add_edge\", \"edge\": {\"from_node\": str, \"to_node\": str}} + {\"op\": \"remove_edge\", \"from_node\": str, \"to_node\": str} + {\"op\": \"set_workflow_inputs\", \"inputs\": [...]} + +Do not rename nodes. A rename rewires edges but leaves every `=nodes.` +expression pointing at a node that no longer exists, and the graph will validate +and then run wrong. + +Return an empty ops list when the graph is not the problem. A workflow whose +steps were wired correctly and whose agent simply did poor work does not get +better by being edited, and an edit made anyway costs the next run its +procedure. + +Prefer one precise edit to several speculative ones. You will see the result of +this batch before anything else changes."; + +/// A repaired copy of a workflow that fell short. +#[derive(Debug, Clone)] +pub struct Variant { + /// The saved record. Its id is derived from the parent and the edits, so an + /// identical repair proposed twice lands on one variant rather than two. + pub record: WorkflowRecord, + /// The workflow this was derived from, whose score it must beat. + pub parent_id: String, + /// The edits that produced it. + pub ops: Vec, + /// Why, in the model's words. Carried into `Approach::Variant`. + pub why: String, +} + +/// Is this a shortfall a graph edit could plausibly fix? +/// +/// Runs before inference, because the common case — an agent ran, was wired +/// right, and fell short on the work — must not pay for a repair proposal it +/// will discard. A null binding, an empty prompt, a swallowed error or a node +/// that never ran are all structural; so is a judge that named a node. +#[must_use] +pub fn graph_is_suspect(verdict: &Verdict, evidence: &Evidence<'_>) -> bool { + let d = evidence.diagnosis; + d.null_bindings.iter().any(|b| !b.unverifiable) + || !d.empty_prompts.is_empty() + || !d.hidden_errors.is_empty() + || !d.never_ran.is_empty() + || !verdict.attributed_to.trim().is_empty() +} + +/// Propose a graph fix and save it as a variant of `parent_id`. +/// +/// Returns `Ok(None)` when nothing is worth changing — the graph is not +/// suspect, or the model declined. That is the expected answer most of the time +/// and is not a failure. +/// +/// # Errors +/// When the store cannot be read or written, inference fails, or the proposed +/// batch does not apply, does not validate, or names something this host does +/// not have. A refused batch is an error rather than a silent `None` because +/// the caller records it: a repair that keeps failing the same gate is itself +/// evidence about the goal. +#[allow(clippy::too_many_arguments)] +pub async fn repair( + goal: &Goal, + verdict: &Verdict, + evidence: &Evidence<'_>, + parent_id: &str, + store: &Arc, + ledger: &dyn Ledger, + caps: &Capabilities, + conn: Option<&str>, +) -> Result> { + if !graph_is_suspect(verdict, evidence) { + return Ok(None); + } + let parent = store + .get(parent_id) + .map_err(|e| IntakeError::Store(e.to_string()))? + .ok_or_else(|| IntakeError::Store(format!("no workflow '{parent_id}'")))?; + + let user = format!( + "# Goal\n{}\n\n# The workflow that ran\n{}\n\n# What is still missing\n{}{}\n\n{}", + goal.text.trim(), + serde_json::to_string_pretty(&parent.graph) + .map_err(|e| IntakeError::Store(e.to_string()))?, + verdict.gap, + if verdict.attributed_to.trim().is_empty() { + String::new() + } else { + format!( + "\n\nThe judge attributed this to node `{}`.", + verdict.attributed_to + ) + }, + evidence.render() + ); + + let answer = ask(caps, conn, Tier::Repair, SYSTEM, &user).await?; + let ops = read_ops(&answer)?; + if ops.is_empty() { + return Ok(None); + } + + // Applied to a copy and validated before anything is saved — the same order + // the engine's own authoring path uses, and for the same reason: a store + // whose listings are trustworthy is one nothing unrunnable can enter. + let graph = apply_ops(&parent.graph, &ops) + .map_err(|e| IntakeError::Invalid(format!("the repair does not apply: {e}")))?; + let problems = validate_all(&graph); + if !problems.is_empty() { + return Err(IntakeError::Invalid( + problems + .iter() + .map(ToString::to_string) + .collect::>() + .join("; "), + )); + } + + let id = variant_id(parent_id, &ops); + let why = answer["why"] + .as_str() + .unwrap_or_default() + .trim() + .to_string(); + let record = WorkflowRecord { + id: id.clone(), + name: format!("{} (repaired)", parent.name), + description: if why.is_empty() { + format!("Variant of {parent_id}.") + } else { + format!("Variant of {parent_id}: {why}") + }, + enabled: parent.enabled, + defaults: parent.defaults.clone(), + graph, + // Never inherited: it points at the parent's file, and saving under it + // would overwrite the very record this exists to leave intact. + source_path: None, + }; + store + .policy() + .check_graph(&id, &record.graph) + .map_err(|e| IntakeError::Unsupported(e.to_string()))?; + store + .save(&record) + .map_err(|e| IntakeError::Store(e.to_string()))?; + + // Recorded after the save, so a link never points at a graph the store + // refused. Without it the variant is just another row in the catalogue and + // the promotion gate has no family to compare within. + // + // The converse can happen under a buffering store, and is fine: this link + // is durable now, while the graph lands only when the host flushes — and a + // host may gate that flush on the episode succeeding, so a failed episode + // leaves a link with no graph behind it. That degrades to "not offerable" + // in the catalogue rather than breaking anything, and the same failure + // re-derives the same repair onto the same content-derived id later, so + // the lineage and score recorded now reattach instead of being orphaned. + ledger.link_variant(parent_id, &id).await?; + + Ok(Some(Variant { + record, + parent_id: parent_id.to_string(), + ops, + why, + })) +} + +/// Read the batch, refusing renames. +fn read_ops(answer: &serde_json::Value) -> Result> { + let Some(raw) = answer.get("ops") else { + return Ok(Vec::new()); + }; + if raw.is_null() { + return Ok(Vec::new()); + } + let ops: Vec = serde_json::from_value(raw.clone()) + .map_err(|e| IntakeError::Invalid(format!("not a batch of graph ops: {e}")))?; + if ops + .iter() + .any(|op| matches!(op, GraphOp::RenameNode { .. })) + { + return Err(IntakeError::Invalid( + "a repair may not rename a node: edges are rewired but `=nodes.` \ + expressions in other nodes are not, and the graph would validate and \ + run wrong" + .to_string(), + )); + } + Ok(ops) +} + +/// `-fix-`. +/// +/// Derived rather than counted so it needs no clock and no read of what already +/// exists, and so the same repair proposed twice converges on one variant +/// instead of filling the store with near-identical copies. +fn variant_id(parent_id: &str, ops: &[GraphOp]) -> String { + // The stable digest — this id keys workflow records, scores and lineage, + // so it must survive toolchain upgrades. See `reuse::digest_hex`. + format!( + "{parent_id}-fix-{}", + crate::reuse::digest_hex(&serde_json::to_vec(ops).unwrap_or_default()) + ) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::contracts::Blocker; + use tinyflows::diagnostics::{Diagnosis, NeverRan, NullBinding}; + use tinyflows::engine::RunOutcome; + + fn verdict(attributed_to: &str) -> Verdict { + Verdict { + satisfied: false, + blocker: Blocker::GoalNotMet, + gap: "the report was never written".into(), + attributed_to: attributed_to.into(), + evidence: String::new(), + advanced: true, + } + } + + fn outcome() -> RunOutcome { + RunOutcome { + output: serde_json::json!({}), + pending_approvals: Vec::new(), + cancelled: false, + } + } + + #[test] + fn a_clean_run_that_simply_fell_short_is_not_a_graph_problem() { + let d = Diagnosis::default(); + let out = outcome(); + let evidence = Evidence { + outcome: &out, + diagnosis: &d, + changed: "wrote report.md".into(), + }; + assert!(!graph_is_suspect(&verdict(""), &evidence)); + } + + #[test] + fn a_node_the_judge_named_makes_the_graph_suspect() { + let d = Diagnosis::default(); + let out = outcome(); + let evidence = Evidence { + outcome: &out, + diagnosis: &d, + changed: String::new(), + }; + assert!(graph_is_suspect(&verdict("summarise"), &evidence)); + } + + #[test] + fn a_node_that_never_ran_makes_the_graph_suspect() { + let d = Diagnosis { + never_ran: vec![NeverRan { + node_id: "publish".into(), + routed_by: None, + }], + ..Diagnosis::default() + }; + let out = outcome(); + let evidence = Evidence { + outcome: &out, + diagnosis: &d, + changed: String::new(), + }; + assert!(graph_is_suspect(&verdict(""), &evidence)); + } + + #[test] + fn an_unverifiable_null_binding_alone_does_not_make_it_suspect() { + // The engine could not evaluate the expression even in principle, so it + // is not evidence the graph is wrong — and repairing on it would edit a + // correct graph every run. + let d = Diagnosis { + null_bindings: vec![NullBinding { + node_id: "fetch".into(), + location: "config.prompt".into(), + expression: "=nodes.agent.item.body".into(), + unverifiable: true, + reads_from: Some("agent".into()), + suggestion: "run it for real".into(), + }], + ..Diagnosis::default() + }; + let out = outcome(); + let evidence = Evidence { + outcome: &out, + diagnosis: &d, + changed: String::new(), + }; + assert!(!graph_is_suspect(&verdict(""), &evidence)); + } + + #[test] + fn declining_to_edit_is_read_as_no_ops_not_as_a_malformed_reply() { + assert!( + read_ops(&serde_json::json!({"ops": [], "why": "the graph is fine"})) + .expect("no ops") + .is_empty() + ); + assert!( + read_ops(&serde_json::json!({"why": "nothing to do"})) + .expect("no ops") + .is_empty() + ); + assert!( + read_ops(&serde_json::json!({"ops": null})) + .expect("no ops") + .is_empty() + ); + } + + #[test] + fn a_rename_is_refused_even_though_the_engine_would_apply_it() { + let batch = serde_json::json!({ + "ops": [{"op": "rename_node", "id": "a", "new_id": "b"}] + }); + let err = read_ops(&batch).expect_err("refused"); + assert!(err.to_string().contains("rename"), "{err}"); + } + + #[test] + fn an_ordinary_config_patch_reads_as_one_op() { + let batch = serde_json::json!({ + "ops": [{ + "op": "update_node_config", + "id": "summarise", + "config": {"prompt": "=nodes.fetch.item.json.body"} + }] + }); + let ops = read_ops(&batch).expect("read"); + assert_eq!(ops.len(), 1); + assert_eq!(ops[0].name(), "update_node_config"); + } + + #[test] + fn the_same_repair_twice_lands_on_the_same_variant_id() { + let ops = vec![GraphOp::SetNodeName { + id: "a".into(), + name: "A".into(), + }]; + assert_eq!(variant_id("weekly", &ops), variant_id("weekly", &ops)); + let other = vec![GraphOp::SetNodeName { + id: "a".into(), + name: "B".into(), + }]; + assert_ne!(variant_id("weekly", &ops), variant_id("weekly", &other)); + } + + #[test] + fn a_variant_id_names_its_parent() { + let ops = vec![GraphOp::RemoveNode { id: "a".into() }]; + assert!(variant_id("weekly-report", &ops).starts_with("weekly-report-fix-")); + } +} diff --git a/crates/adaptive/src/contracts.rs b/crates/adaptive/src/contracts.rs new file mode 100644 index 0000000..a5defc9 --- /dev/null +++ b/crates/adaptive/src/contracts.rs @@ -0,0 +1,423 @@ +//! The types the loop turns on. +//! +//! Ported from medulla-v2, where each of them was arrived at by a failure +//! rather than by design. The comments record which failure, because the shape +//! is not obvious from the type and a later reader will otherwise simplify one +//! of them back into the thing that broke. +//! +//! What is deliberately absent: anything shaped like a plan. A plan here is a +//! `WorkflowGraph` — the engine's own type — and nothing in this module +//! duplicates it. The loop decides *which* graph; the engine runs it. + +use serde::{Deserialize, Serialize}; + +/// Why a run did not satisfy its goal. +/// +/// A fixed vocabulary rather than free text, because the loop branches on it: +/// two of these mean "try again", two mean "stop", and one means "ask". Free +/// text cannot be branched on, and a model asked for a category invents a new +/// one every third call. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum Blocker { + /// Satisfied. No blocker. + None, + /// Something was produced but the evidence does not show it working. + /// Continuable: another attempt can verify it. + Unverified, + /// The goal was not met and the attempt made a real try at it. + /// Continuable: this is the ordinary retry case. + GoalNotMet, + /// Nothing was produced and there is nothing to judge. Terminal, because a + /// retry with the same inputs produces the same nothing. + MissingEvidence, + /// A person has to answer something before this can continue. + NeedsInput, + /// Waiting on something outside the system — a deploy, a review, a rate + /// limit. Retrying now is not the same as retrying later. + ExternalWait, +} + +impl Blocker { + /// Whether another attempt could plausibly do better. + #[must_use] + pub fn continuable(self) -> bool { + matches!(self, Self::Unverified | Self::GoalNotMet) + } + + /// Reads a model's answer, coercing anything unrecognised to the safest + /// continuable value. + /// + /// A misspelling used to end runs: `goal_not_meet` fell through to a + /// terminal default and killed a run at attempt 3 of 12. The model is not + /// going to stop misspelling, so the boundary absorbs it. + #[must_use] + pub fn parse(raw: &str) -> Self { + match raw.trim().to_ascii_lowercase().as_str() { + "none" | "" => Self::None, + "unverified" => Self::Unverified, + "missing_evidence" => Self::MissingEvidence, + "needs_input" => Self::NeedsInput, + "external_wait" => Self::ExternalWait, + _ => Self::GoalNotMet, + } + } +} + +/// What the judge produced after a run. +/// +/// Carries no plan-shaped field, on purpose. The judge runs context-poor — goal, +/// outcome and evidence only — so it can diagnose but cannot sensibly propose +/// what to do next: it does not know what has already been ruled out. Deciding +/// that is the planner's job, and the planner has the ledger. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct Verdict { + /// Whether the goal was met. + pub satisfied: bool, + /// Why not, when it was not. + pub blocker: Blocker, + /// What is still missing, in one sentence, for the next plan to read. + #[serde(default)] + pub gap: String, + /// Which node or step fell short, when the judge can tell. + #[serde(default)] + pub attributed_to: String, + /// What the judge actually looked at. Recorded so a wrong verdict can be + /// argued with later. + #[serde(default)] + pub evidence: String, + /// Did this attempt move the goal closer than it was before it ran? + /// + /// The decision to try again used to be a counter, which cannot tell a run + /// that is converging from one that is spinning — two live runs were killed + /// at 7 of 10 and climbing, while a third thrashed 10 → 2 → 1 and only the + /// counter stopped it. All three reported `goal_not_met`. + #[serde(default = "yes")] + pub advanced: bool, +} + +fn yes() -> bool { + true +} + +impl Verdict { + /// Whether the loop should attempt again, given how many attempts have run. + /// + /// Three gates, in order. `min_attempts` comes first because early attempts + /// routinely look flat while a run is still orienting — the first often + /// only establishes what it is dealing with — so a stall call on attempt one + /// ends runs that had not started. + #[must_use] + pub fn should_retry(&self, spent: u32, stalled: u32, budget: &Budget) -> bool { + if self.satisfied || !self.blocker.continuable() { + return false; + } + if budget.exhausted(spent) { + return false; + } + if spent < budget.min_attempts { + return true; + } + stalled < budget.stall_limit + } +} + +/// What one episode may spend. +#[derive(Debug, Clone, Copy, Serialize, Deserialize)] +pub struct Budget { + /// A backstop, not the stop rule. A run normally ends because the judge says + /// two attempts in a row went nowhere. + pub attempts: u32, + /// Attempts before the stall rule may end a run at all. + pub min_attempts: u32, + /// Consecutive non-advancing attempts that end a run. + pub stall_limit: u32, +} + +impl Default for Budget { + fn default() -> Self { + Self { + attempts: 12, + min_attempts: 3, + stall_limit: 2, + } + } +} + +impl Budget { + /// Whether the attempt ceiling has been reached. + #[must_use] + pub fn exhausted(&self, spent: u32) -> bool { + spent >= self.attempts + } +} + +/// Which job the loop is asking a model to do. +/// +/// Emitted on every inference request as `tier`, and that is the whole of it — +/// the crate names the **job**, never a model, a vendor or a URL, because only +/// the host knows which of those a job maps to. That is the host-agnostic rule +/// the engine sits on and this crate keeps. +/// +/// It is what makes a tier sweep a config change rather than a code change: +/// judging is the expensive opinion and selecting is a cheap one, and without a +/// name on the request a host cannot route them differently. +/// +/// Called `tier` rather than `role` on the wire because a chat request already +/// has `role` on every message, and two meanings of one key in one payload is a +/// bug waiting for a hurried reader. +/// +/// Six, not medulla-v2's three: a host can map several tiers to one model in a +/// line of config, and cannot split one tier into two at all. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum Tier { + /// Does a stored workflow already do this? Cheap; a short list and a yes/no. + Select, + /// Write a graph. The hardest reasoning the loop does. + Author, + /// Did the run achieve the goal? The opinion worth paying for — a judge + /// that says yes wrongly ends the episode. + Judge, + /// What is this episode worth remembering? Off the critical path, and + /// nothing downstream blocks on it. + Consolidate, + /// Repair a graph that fell short. Structured editing against a diagnosis. + Repair, + /// Name a graph that worked, so a later goal can find it. Prose only — the + /// graph is already fixed. + Generalise, +} + +impl Tier { + /// The name that goes on the wire. + #[must_use] + pub fn as_str(self) -> &'static str { + match self { + Self::Select => "select", + Self::Author => "author", + Self::Judge => "judge", + Self::Consolidate => "consolidate", + Self::Repair => "repair", + Self::Generalise => "generalise", + } + } +} + +/// What the user asked for, and what would prove it done. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct Goal { + /// The prompt, verbatim. Never paraphrased on its way anywhere: a detail + /// misremembered in a restatement becomes the only version an agent sees. + pub text: String, + /// What would show it satisfied. Empty when the user gave no criterion and + /// the judge has to infer one from the goal. + #[serde(default)] + pub success_criteria: String, +} + +impl Goal { + /// A goal with no stated success criterion; the judge infers one. + #[must_use] + pub fn new(text: impl Into) -> Self { + Self { + text: text.into(), + success_criteria: String::new(), + } + } +} + +/// How the loop decided to attempt a goal this time. +/// +/// Two, and the second is what makes this a loop rather than a router: when no +/// stored procedure fits, one is written. +/// +/// There is deliberately no `Variant` arm. A repaired graph is saved to the +/// store as a workflow in its own right, so the attempt that runs it is a +/// [`Selected`](Self::Selected) of *that* id — which is what the score has to +/// land on. A third arm naming the parent would score the parent for a run the +/// variant did, leaving the two indistinguishable and the promotion gate with +/// nothing to compare. What makes it a variant is the lineage in the ledger, +/// not the shape of this enum. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case", tag = "kind")] +pub enum Approach { + /// A stored workflow matched. The common case once anything has been + /// learned, and the cheap one: no authoring call. + Selected { + /// The stored workflow that matched. + workflow_id: String, + /// Why it was chosen, for the ledger row. + why: String, + }, + /// Nothing fitted, so a graph was written for this goal. + Authored { + /// Why nothing stored fitted. + why: String, + /// A digest of the graph that was written. + /// + /// The exclusion list is built from [`signature`](Self::signature), so + /// without this every authoring attempt in an episode signs as the same + /// string, `tried()` folds them to one entry, and attempt four can + /// re-author attempt two's graph word for word with nothing to notice. + /// The digest is what makes two authored attempts distinguishable — + /// and makes an identical re-author visible as the repeat it is. + fingerprint: String, + }, +} + +impl Approach { + /// The label a ledger row is keyed on, and the exclusion list is built from. + /// + /// Names the *kind* of attempt, not the task: a retry told not to repeat + /// `review_pr_5478` has nothing left to try, while one told not to repeat + /// `selected:pr-review` can still author. + #[must_use] + pub fn signature(&self) -> String { + match self { + Self::Selected { workflow_id, .. } => format!("selected:{workflow_id}"), + Self::Authored { fingerprint, .. } => format!("authored:{fingerprint}"), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn an_unrecognised_blocker_is_continuable_rather_than_terminal() { + // `goal_not_meet` — one letter — used to end a run at attempt 3 of 12. + assert_eq!(Blocker::parse("goal_not_meet"), Blocker::GoalNotMet); + assert_eq!(Blocker::parse("something new"), Blocker::GoalNotMet); + assert!(Blocker::parse("nonsense").continuable()); + } + + #[test] + fn the_terminal_blockers_stop_a_run() { + assert!(!Blocker::MissingEvidence.continuable()); + assert!(!Blocker::NeedsInput.continuable()); + assert!(!Blocker::ExternalWait.continuable()); + } + + #[test] + fn an_empty_blocker_reads_as_no_blocker() { + assert_eq!(Blocker::parse(""), Blocker::None); + } + + fn verdict(satisfied: bool, blocker: Blocker) -> Verdict { + Verdict { + satisfied, + blocker, + gap: String::new(), + attributed_to: String::new(), + evidence: String::new(), + advanced: false, + } + } + + #[test] + fn the_stall_rule_does_not_apply_before_min_attempts() { + // Early attempts look flat while a run is still orienting. + let budget = Budget::default(); + let v = verdict(false, Blocker::GoalNotMet); + assert!( + v.should_retry(1, 5, &budget), + "attempt 1 must not be stalled out" + ); + assert!(v.should_retry(2, 5, &budget)); + assert!( + !v.should_retry(3, 2, &budget), + "past min_attempts the rule bites" + ); + } + + #[test] + fn a_converging_run_is_not_killed_by_the_counter() { + let budget = Budget::default(); + let mut v = verdict(false, Blocker::GoalNotMet); + v.advanced = true; + // `stalled` is reset by the caller on every advancing attempt, so a run + // that keeps advancing never accumulates one. + assert!(v.should_retry(9, 0, &budget)); + } + + #[test] + fn a_satisfied_verdict_never_retries() { + assert!(!verdict(true, Blocker::None).should_retry(1, 0, &Budget::default())); + } + + #[test] + fn a_terminal_blocker_stops_even_with_budget_left() { + let v = verdict(false, Blocker::NeedsInput); + assert!(!v.should_retry(1, 0, &Budget::default())); + } + + #[test] + fn the_attempt_ceiling_is_still_a_backstop() { + let budget = Budget::default(); + let mut v = verdict(false, Blocker::GoalNotMet); + v.advanced = true; + assert!(!v.should_retry(12, 0, &budget)); + } + + #[test] + fn a_signature_names_the_kind_of_attempt_not_the_task() { + let selected = Approach::Selected { + workflow_id: "pr-review".into(), + why: "matches".into(), + }; + assert_eq!(selected.signature(), "selected:pr-review"); + } + + #[test] + fn two_authored_attempts_are_told_apart_by_their_graph() { + // Before the fingerprint every authoring attempt signed as "authored", + // `tried()` folded them to one entry, and attempt four could re-author + // attempt two word for word with nothing to notice. + let first = Approach::Authored { + why: "nothing fitted".into(), + fingerprint: "1111111".into(), + }; + let second = Approach::Authored { + why: "still nothing fitted".into(), + fingerprint: "2222222".into(), + }; + assert_ne!(first.signature(), second.signature()); + assert_eq!(first.signature(), "authored:1111111"); + } + + #[test] + fn the_same_graph_authored_twice_signs_the_same_and_is_caught() { + // The other half: a differently-worded `why` around an identical graph + // is the same attempt, and must read as the repeat it is. + let first = Approach::Authored { + why: "nothing fitted".into(), + fingerprint: "1111111".into(), + }; + let again = Approach::Authored { + why: "a fresh idea, honestly".into(), + fingerprint: "1111111".into(), + }; + assert_eq!(first.signature(), again.signature()); + } + + #[test] + fn a_verdict_round_trips_through_json() { + // The judge answers in JSON and the ledger stores JSON; a field lost in + // either direction is one that works in a test and never in a run. + let v = verdict(false, Blocker::Unverified); + let back: Verdict = serde_json::from_str(&serde_json::to_string(&v).unwrap()).unwrap(); + assert_eq!(back.blocker, Blocker::Unverified); + assert!(!back.advanced); + } + + #[test] + fn advanced_defaults_to_true_when_a_model_omits_it() { + // Absent must not read as "made no progress" — that would stall a run + // for a field the model simply did not write. + let v: Verdict = + serde_json::from_str(r#"{"satisfied":false,"blocker":"goal_not_met"}"#).unwrap(); + assert!(v.advanced); + } +} diff --git a/crates/adaptive/src/driver.rs b/crates/adaptive/src/driver.rs new file mode 100644 index 0000000..45cd3f6 --- /dev/null +++ b/crates/adaptive/src/driver.rs @@ -0,0 +1,467 @@ +//! Holding the pieces together, and driving an episode to an answer. +//! +//! Everything below this module is a free function taking seven or eight +//! arguments, which is right for a library and wearing to call. This bundles +//! them. +//! +//! # What is an instance, and what is a goal run +//! +//! These are different lifetimes and putting them in one object is the mistake +//! worth naming. +//! +//! A [`Loop`] is **per tenant**, long-lived, and holds only configuration and +//! adapters: a scoped ledger, a workflow store, capabilities, host facts, a +//! runner, a budget. Building one costs a database pool and an HTTP client, so +//! it is built once and shared. +//! +//! A **goal run is an episode id**, not an object. Its state — the goal, the +//! attempt number, the stall count, whether it finished — lives in the +//! [`Episode`] record in the ledger. Nothing about it is held here. +//! +//! That split is what makes two things true at once. Many goal runs share one +//! `Loop`, concurrently, because the `Loop` holds nothing per-episode. And an +//! episode survives the process running it: kill this one mid-run and +//! [`Ledger::episodes`] on the next boot hands back everything that was in +//! flight, each resumable from its own record by id. +//! +//! Had the instance *been* the goal run, both would be false — the config would +//! be rebuilt per goal, and a deploy would lose every episode's counters while +//! leaving its rows behind to look like progress. + +use std::sync::Arc; + +use tinyflows::caps::Capabilities; +use tinyflows::store::WorkflowStore; + +use crate::closing::{self, Closed, Next, graph_is_suspect}; +use crate::contracts::{Approach, Budget, Goal, Verdict}; +use crate::execute::Runner; +use crate::host::HostFacts; +use crate::intake::{Result, decide}; +use crate::ledger::{Episode, EpisodeStatus, Ledger, Lesson, Page}; + +/// Where timestamps come from. +/// +/// A seam rather than a dependency: the crate has no clock of its own, so a +/// frozen one drives tests and the host brings whatever it already uses. Every +/// stored time is caller-supplied for the same reason. +pub trait Clock: Send + Sync { + /// The current time, RFC 3339. + fn now(&self) -> String; +} + +/// One tenant's configuration and adapters. +/// +/// Cheap to hold, expensive to build. See the module note on why this is not +/// one per goal run. +pub struct Loop<'a> { + /// Scoped to this tenant — see [`Ledger::scope`]. + pub ledger: &'a dyn Ledger, + /// Where workflows are read and variants written. + pub store: &'a Arc, + /// Inference. The `tier` on each request says which job is asking. + pub caps: &'a Capabilities, + /// What the machine that runs graphs permits. + pub facts: &'a HostFacts, + /// In-process or relayed; the loop cannot tell. + pub runner: &'a dyn Runner, + /// Where timestamps come from. + pub clock: &'a dyn Clock, + /// How hard to try. + pub budget: Budget, + /// Opaque credential reference, passed to inference untouched. + pub conn: Option<&'a str>, +} + +/// How an episode ended. +#[derive(Debug, Clone)] +pub struct Finished { + /// Satisfied, or stood down with a reason. + pub status: EpisodeStatus, + /// How many attempts it took. + pub attempts: u32, + /// What the judge said about the last one. + pub verdict: Verdict, + /// What was worth remembering. Usually nothing. + pub lessons: Vec, +} + +impl Loop<'_> { + /// Begin an episode, or return the one already under way. + /// + /// Idempotent, so a service that retries a create does not restart a goal + /// that is four attempts in. + /// + /// # Errors + /// When the ledger cannot be read or written. + pub async fn start(&self, episode: &str, goal: &Goal) -> Result { + if let Some(existing) = self.ledger.episode(episode).await? { + return Ok(existing); + } + let now = self.clock.now(); + let record = Episode { + id: episode.to_string(), + goal: goal.clone(), + scope_key: None, + status: EpisodeStatus::Running, + attempt: 0, + stalled: 0, + started_at: now.clone(), + updated_at: now, + }; + self.ledger.save_episode(&record).await?; + Ok(record) + } + + /// One pass: decide, run, judge, record — and repair the graph if that is + /// what fell short. + /// + /// **One episode, one attempt at a time.** Concurrency is across episodes — + /// distinct ids — never within one: two concurrent `attempt` calls for the + /// same episode id would read the same attempt number, append two rows + /// under it, and race the checkpoint write. Nothing here serializes that, + /// because the natural caller — [`run`](Self::run), or a worker owning an + /// episode — is already sequential; a host that parallelizes within one + /// episode owns the lock. + /// + /// The attempt number comes from the episode record rather than the caller, + /// so a process that picks up an episode it did not start continues its + /// numbering instead of restarting at one. + /// + /// # Errors + /// When intake cannot decide, or the ledger cannot be read or written. + /// Running never errors — see [`crate::execute`]. + pub async fn attempt(&self, episode: &str, goal: &Goal) -> Result { + let record = self.start(episode, goal).await?; + let attempt = record.attempt + 1; + + let planned = decide( + goal, + episode, + self.store.as_ref(), + self.ledger, + self.facts, + self.caps, + self.conn, + ) + .await?; + + let ran = self.runner.run(&planned).await; + let closed = closing::close( + goal, + episode, + attempt, + &planned.approach, + &ran, + &self.budget, + self.ledger, + self.caps, + self.conn, + &self.clock.now(), + ) + .await?; + + // The other half of the knowledge ladder's scoring. `close` moves the + // workflow's counters; nothing was moving a lesson's, so a lesson that + // was read forty times and never helped looked exactly like one written + // this morning. + for lesson in &planned.lessons_shown { + let _ = self + .ledger + .score_lesson(lesson, closed.verdict.satisfied) + .await; + } + + if closed.verdict.satisfied { + self.keep_if_it_generalises(goal, &planned).await; + } else { + self.repair_if_the_graph_is_at_fault(goal, &closed, &planned.approach, &ran) + .await; + } + Ok(closed) + } + + /// Drive an episode until it is satisfied or stands down. + /// + /// Terminates without a bound of its own: `close` returns + /// [`Next::StandDown`] once the budget is spent or the run stops advancing, + /// so the exit condition lives in one place rather than two that can + /// disagree. + /// + /// # Errors + /// As [`attempt`](Self::attempt). + pub async fn run(&self, episode: &str, goal: &Goal) -> Result { + loop { + let closed = self.attempt(episode, goal).await?; + let status = match &closed.next { + Next::Retry => continue, + Next::Done => EpisodeStatus::Satisfied, + Next::StandDown(reason) => EpisodeStatus::StoodDown(reason.clone()), + }; + + // Once per episode, not per attempt: what generalises is visible + // from the whole trail and not from any one row of it. + let lessons = closing::consolidate( + goal, + episode, + closed.verdict.satisfied, + self.ledger, + self.caps, + self.conn, + ) + .await; + + let attempts = self + .ledger + .episode(episode) + .await? + .map_or(0, |record| record.attempt); + return Ok(Finished { + status, + attempts, + verdict: closed.verdict, + lessons, + }); + } + } + + /// Every episode of this tenant's that was still running. + /// + /// The boot recovery list. Without it a deploy abandons whatever was in + /// flight: the rows stay, nothing looks at them again, and the goal is + /// never answered. + /// + /// # Errors + /// When the ledger cannot be read. + pub async fn unfinished(&self) -> Result> { + Ok(self.ledger.episodes(true, Page::ALL).await?) + } + + /// Keep a graph that was authored for this goal and achieved it. + /// + /// Only an authored one: a selected workflow is already stored, and a + /// repaired variant was stored when it was proposed. + /// + /// Best-effort and silent on failure, like the other two post-outcome + /// passes. The goal is met either way; failing to file the procedure costs + /// the next episode an authoring call, not this one its result. + async fn keep_if_it_generalises(&self, goal: &Goal, planned: &crate::intake::Attempt) { + if !matches!(planned.approach, Approach::Authored { .. }) { + return; + } + let kept = closing::keep( + goal, + &planned.graph, + &planned.inputs, + self.store, + self.caps, + self.conn, + ) + .await; + + // Scored on the way in, from the run that earned it. A procedure + // entering the catalogue at 0/0 is indistinguishable from one nobody + // has ever run, and the evidence that it works is the episode that just + // finished. + if let Ok(Some(kept)) = kept { + let _ = self.ledger.score_workflow(&kept.record.id, true).await; + } + } + + /// Propose a variant when the diagnosis says the graph was the problem. + /// + /// Best-effort and deliberately silent on failure. It runs after the + /// outcome is already recorded, so a refused batch or a provider hiccup + /// must not turn a judged attempt into a failed one — the same reasoning as + /// [`crate::closing::consolidate`]. + async fn repair_if_the_graph_is_at_fault( + &self, + goal: &Goal, + closed: &Closed, + approach: &Approach, + ran: &crate::execute::Ran, + ) { + if closed.verdict.satisfied { + return; + } + // Whatever ran is the parent of the next repair — including a variant, + // which makes a second generation. `Ledger::lineage` walks to the root, + // so a grandchild is still compared inside one family. + let parent = match approach { + Approach::Selected { workflow_id, .. } => workflow_id, + // Nothing to repair: an authored graph was written for this goal + // and the next attempt writes another, seeing why this one fell + // short. A variant of a one-off is a stored procedure nobody asked + // for. + Approach::Authored { .. } => return, + }; + let evidence = ran.evidence(); + if !graph_is_suspect(&closed.verdict, &evidence) { + return; + } + let _ = closing::repair( + goal, + &closed.verdict, + &evidence, + parent, + self.store, + self.ledger, + self.caps, + self.conn, + ) + .await; + } +} + +#[cfg(test)] +mod tests { + use super::*; + + struct Frozen; + impl Clock for Frozen { + fn now(&self) -> String { + "2026-01-01T00:00:00Z".to_string() + } + } + + #[tokio::test] + async fn starting_an_episode_twice_does_not_restart_it() { + // A service that retries a create must not reset a goal four attempts + // in — the rows would stay and the counters would not, which reads as + // progress that never happened. + let ledger = crate::ledger::memory::MemoryLedger::new(); + let goal = Goal::new("write the weekly report"); + + let mut record = Episode { + id: "ep-1".into(), + goal: goal.clone(), + scope_key: None, + status: EpisodeStatus::Running, + attempt: 4, + stalled: 2, + started_at: "2026-01-01T00:00:00Z".into(), + updated_at: "2026-01-01T00:00:00Z".into(), + }; + ledger.save_episode(&record).await.expect("save"); + + // `start` short-circuits on an existing record, so this is what it sees. + let seen = ledger.episode("ep-1").await.expect("read").expect("exists"); + assert_eq!(seen.attempt, 4); + assert_eq!(seen.stalled, 2); + + record.attempt = 5; + ledger.save_episode(&record).await.expect("save"); + assert_eq!( + ledger + .episode("ep-1") + .await + .expect("read") + .expect("exists") + .attempt, + 5, + "a save updates rather than duplicating" + ); + } + + #[tokio::test] + async fn only_running_episodes_are_offered_for_recovery() { + let ledger = crate::ledger::memory::MemoryLedger::new(); + for (id, status) in [ + ("ep-live", EpisodeStatus::Running), + ("ep-done", EpisodeStatus::Satisfied), + ( + "ep-gave-up", + EpisodeStatus::StoodDown("out of attempts".into()), + ), + ] { + ledger + .save_episode(&Episode { + id: id.into(), + goal: Goal::new("something"), + scope_key: None, + status, + attempt: 1, + stalled: 0, + started_at: "2026-01-01T00:00:00Z".into(), + updated_at: "2026-01-01T00:00:00Z".into(), + }) + .await + .expect("save"); + } + + let running = ledger.episodes(true, Page::ALL).await.expect("episodes"); + assert_eq!(running.len(), 1); + assert_eq!(running[0].id, "ep-live"); + assert_eq!( + ledger + .episodes(false, Page::ALL) + .await + .expect("episodes") + .len(), + 3 + ); + } + + #[tokio::test] + async fn an_episode_round_trips_its_goal_and_its_reason_for_stopping() { + // Both are unrecoverable from the rows, which is the whole test for + // what belongs on the record. + let ledger = crate::ledger::memory::MemoryLedger::new(); + let mut goal = Goal::new("write the weekly report"); + goal.success_criteria = "cites the actual figures".into(); + + ledger + .save_episode(&Episode { + id: "ep-2".into(), + goal, + scope_key: None, + status: EpisodeStatus::StoodDown("3 attempts in a row made no progress".into()), + attempt: 7, + stalled: 3, + started_at: Frozen.now(), + updated_at: Frozen.now(), + }) + .await + .expect("save"); + + let back = ledger.episode("ep-2").await.expect("read").expect("exists"); + assert_eq!(back.goal.text, "write the weekly report"); + assert_eq!(back.goal.success_criteria, "cites the actual figures"); + assert_eq!(back.stalled, 3); + match back.status { + EpisodeStatus::StoodDown(reason) => assert!(reason.contains("no progress")), + other => panic!("expected a stand-down, got {other:?}"), + } + } + + #[tokio::test] + async fn one_tenants_episodes_are_invisible_to_another() { + let ledger = crate::ledger::memory::MemoryLedger::new(); + let a = ledger.for_tenant("user-a"); + let b = ledger.for_tenant("user-b"); + a.save_episode(&Episode { + id: "ep-private".into(), + goal: Goal::new("something of mine"), + scope_key: None, + status: EpisodeStatus::Running, + attempt: 1, + stalled: 0, + started_at: Frozen.now(), + updated_at: Frozen.now(), + }) + .await + .expect("save"); + + assert!(a.episode("ep-private").await.expect("read").is_some()); + assert!( + b.episode("ep-private").await.expect("read").is_none(), + "an episode carries a goal in the user's own words" + ); + assert!( + b.episodes(false, Page::ALL) + .await + .expect("episodes") + .is_empty() + ); + } +} diff --git a/crates/adaptive/src/execute/mod.rs b/crates/adaptive/src/execute/mod.rs new file mode 100644 index 0000000..48faa35 --- /dev/null +++ b/crates/adaptive/src/execute/mod.rs @@ -0,0 +1,416 @@ +//! Running one attempt, and coming back with something the judge can read. +//! +//! The middle of the loop, and deliberately the thinnest part of it. Intake +//! decided *what* to run; closing decides what it *meant*. This only runs it — +//! it holds no opinion about the result, reads no history, and makes no +//! decision the other two layers could make instead. +//! +//! Its one real job is that the engine hands back a [`RunOutcome`] and the +//! judge needs an [`Evidence`], and the difference between those is where runs +//! get misjudged. +//! +//! **A run is observed, always.** [`RunOutcome`] alone says the graph finished; +//! it does not say a binding resolved to null, that an `on_error` policy +//! swallowed a failure, or that half the nodes never executed. Those come from +//! [`diagnose`](tinyflows::diagnostics::diagnose), which needs the run's steps, which only exist if an observer +//! was attached. A run without one produces a green outcome and a blank +//! diagnosis — and a blank diagnosis is not "nothing was wrong", it is "nobody +//! looked". Every gate downstream reads it: the judge's findings, the three +//! mechanical verdicts, and [`crate::closing::graph_is_suspect`], which decides +//! whether a repair is even proposed. +//! +//! **An engine error is an attempt, not an escape.** [`run_attempt`] does not +//! return a `Result`. A graph that failed to compile or blew up mid-run still +//! has to reach `close()` and leave a ledger row, or the exclusion list never +//! learns it was tried and the next pass proposes it again in slightly +//! different words. The error becomes evidence like everything else. +//! +//! # Why no checkpointer +//! +//! The plan named [`tinyflows::engine::run_with_checkpointer`] for this phase. +//! It is the wrong entry point today, for two reasons that compound. +//! +//! It installs a `NoopObserver` — so taking it costs the diagnosis, and with it +//! every gate listed above. The variant that keeps both is +//! `run_with_checkpointer_journaled_observed`, which also demands a journal. +//! +//! And what a checkpointer buys is durable *resume*, which this crate does not +//! use. Note *use*, not *lack*: `resume_with_checkpointer` genuinely continues +//! from an interrupt boundary rather than replaying — it is `engine::resume`, +//! the HITL convenience, that re-runs every node before the gate. Our retry is +//! a new run of a new graph by choice, because a retry is a different idea and +//! not a continuation of the last one. So the cost is immediate and the benefit +//! is for a path we do not take. +//! +//! When HITL parking is wired upstream this becomes a one-line swap to the +//! journaled variant. Until then, taking a durability guarantee we cannot use +//! in exchange for the diagnosis we depend on is a bad trade made quietly. + +use std::sync::Arc; + +use async_trait::async_trait; +use serde_json::json; +use tinyflows::caps::Capabilities; +use tinyflows::compiler::compile; +use tinyflows::diagnostics::{Diagnosis, capturing}; +use tinyflows::engine::{RunInput, RunOutcome, run_with_observer}; + +use crate::closing::Evidence; +use crate::intake::Attempt; + +pub mod wire; +pub use wire::{PROMPT_BUDGET, RECORD_BUDGET, RunReport, RunRequest, StepOutcome, StepRecord}; + +/// What changed outside the run, according to the host. +/// +/// The engine cannot answer this: it hands back run state, not a view of the +/// machine. A file written, a commit made, a service called — that is the +/// difference between a run that did the job and one that reported success +/// having done nothing, and it is the only evidence that comes from outside the +/// system being judged. +/// +/// Two calls rather than one, because *what changed* is a comparison and needs +/// a before. A single "what is dirty now" reading cannot distinguish this run's +/// work from what was already on disk when it started. +/// +/// Both methods default to empty, so a host that cannot say anything gets +/// honest silence for free — `Evidence` treats an empty `changed` as "nothing +/// reported", never as "nothing happened". +#[async_trait] +pub trait Workspace: Send + Sync { + /// Take a baseline before the run. Opaque: a commit sha, a manifest hash, + /// a timestamp — whatever this host can compare against later. + async fn mark(&self) -> String { + String::new() + } + + /// Describe what changed since `mark`, for a reader. + /// + /// Prose, not a format anything parses. It is rendered into the judge's + /// prompt and stored nowhere. + async fn changed_since(&self, _mark: &str) -> String { + String::new() + } +} + +/// A host with nothing to report. +/// +/// The honest default, and the right one for a workflow that touches only +/// network services. Judging then rests on the run output and the diagnosis +/// alone, which is a weaker position — worth knowing you are in. +pub struct Unobserved; + +impl Workspace for Unobserved {} + +/// One attempt, run. +/// +/// Owns the outcome and diagnosis so [`evidence`](Self::evidence) can hand out +/// a borrowed [`Evidence`] without the caller keeping three variables alive. +#[derive(Debug, Clone)] +pub struct Ran { + /// What the run amounted to, reconstructed from the steps and bounded for + /// reading. Not the engine's own outcome value — see + /// [`RunReport::into_ran`]. + pub outcome: RunOutcome, + /// The engine's reading of what the steps actually did. + pub diagnosis: Diagnosis, + /// What the host says changed. Empty when it does not say. + pub changed: String, + /// The engine error, when the run did not complete. + /// + /// Present *and* recorded inside `outcome.output` under `error`, so the + /// judge sees it through the ordinary evidence rendering rather than + /// needing a special case. A caller that wants to branch on it — a retry + /// that distinguishes "the graph is broken" from "the work fell short" — + /// reads it here. + pub failed: Option, + /// Every node activation, at full record fidelity. The per-node transcript: + /// what to archive, and richer than what the judge is shown. + pub steps: Vec, + /// What the run cost, in the runner's unit. Zero means not measured. + pub cost_usd: f64, +} + +impl Ran { + /// The three sources, as the judge takes them. + #[must_use] + pub fn evidence(&self) -> Evidence<'_> { + Evidence { + outcome: &self.outcome, + diagnosis: &self.diagnosis, + changed: self.changed.clone(), + } + } +} + +/// Whatever runs a graph. +/// +/// The port the loop calls, and the reason the loop cannot tell whether the +/// engine is in this process or on a machine across a socket. Two +/// implementations ship — [`Local`] and [`Remote`] — and they are the *same +/// code either side of a serialization boundary*: both go through [`serve`] to +/// produce a [`RunReport`] and [`RunReport::into_ran`] to read it. There is no +/// second path that could drift. +#[async_trait] +pub trait Runner: Send + Sync { + /// Run one attempt. Never fails — see the module note. + async fn run(&self, attempt: &Attempt) -> Ran; +} + +/// Run the graph in this process. +pub struct Local<'a> { + /// The real capabilities: agents, tools, HTTP, code. + pub caps: &'a Capabilities, + /// What can say whether anything changed. + pub workspace: &'a dyn Workspace, +} + +#[async_trait] +impl Runner for Local<'_> { + async fn run(&self, attempt: &Attempt) -> Ran { + run_attempt(attempt, self.caps, self.workspace).await + } +} + +/// Carrying a request to an engine somewhere else, and a report back. +/// +/// The crate owns the contract and not the transport: Socket.IO, HTTP, a queue +/// and a unix socket are all the host's business. An implementation is expected +/// to apply its own deadline and return `Err` when it expires — [`Remote`] +/// treats that as an attempt, not as an exception. +#[async_trait] +pub trait Relay: Send + Sync { + /// Send `request` and wait for the matching report. + /// + /// # Errors + /// Whatever the transport calls a failure: no runner connected, a deadline, + /// a malformed reply. The string is recorded, so make it readable. + async fn dispatch(&self, request: &RunRequest) -> Result; +} + +/// Run the graph somewhere else, over a [`Relay`]. +pub struct Remote<'a> { + /// The transport. + pub relay: &'a dyn Relay, + /// Correlates request and reply, and appears in the ledger row. + pub attempt_id: String, +} + +#[async_trait] +impl Runner for Remote<'_> { + async fn run(&self, attempt: &Attempt) -> Ran { + let request = RunRequest { + attempt_id: self.attempt_id.clone(), + graph: attempt.graph.clone(), + inputs: attempt.inputs.clone(), + }; + match self.relay.dispatch(&request).await { + Ok(report) => report.into_ran(&attempt.graph), + Err(why) => unreported(&attempt.graph, &why), + } + } +} + +/// What a runner does when it receives a [`RunRequest`]. +/// +/// The far side of [`Remote`], and the whole of [`Local`]. A host embedding the +/// engine on a device calls this and sends the result back; a host running the +/// engine in-process gets the identical value without a wire. +pub async fn serve( + request: &RunRequest, + caps: &Capabilities, + workspace: &dyn Workspace, +) -> RunReport { + let mark = workspace.mark().await; + let (capture, observer) = capturing(); + + let failure = match compile(&request.graph) { + Ok(compiled) => { + let input = RunInput::new(json!({})).with_inputs(request.inputs.clone()); + match run_with_observer(&compiled, input, caps, &observer).await { + Ok(outcome) => { + return report(request, &capture, workspace, &mark, None, outcome).await; + } + Err(err) => err.to_string(), + } + } + // Nothing ran, so there are no steps — and `diagnose` against an empty + // step list reports every node as never-reached, which is exactly true. + Err(err) => err.to_string(), + }; + + let empty = RunOutcome { + output: json!({}), + pending_approvals: Vec::new(), + cancelled: false, + }; + report(request, &capture, workspace, &mark, Some(failure), empty).await +} + +/// Assemble the report, reading the workspace last. +/// +/// The reading happens after the run either way: a run that errored half way +/// through still wrote whatever it wrote before it did, and that is often the +/// only thing distinguishing "it broke" from "it broke having already done the +/// work". +async fn report( + request: &RunRequest, + capture: &Arc, + workspace: &dyn Workspace, + mark: &str, + failed: Option, + outcome: RunOutcome, +) -> RunReport { + RunReport { + attempt_id: request.attempt_id.clone(), + steps: capture + .steps() + .iter() + .map(|step| StepRecord::bounded(step, RECORD_BUDGET)) + .collect(), + pending_approvals: outcome.pending_approvals, + cancelled: outcome.cancelled, + changed: workspace.changed_since(mark).await, + failed, + // Not measurable from here. A host that meters its harness fills this + // in on the report before sending it. + cost_usd: 0.0, + } +} + +/// Compile and run one attempt in this process, observed. +/// +/// Never fails. Compilation errors, validation errors and mid-run failures all +/// come back as a [`Ran`] with `failed` set — see the module note: an attempt +/// that produced no ledger row is an attempt the next pass repeats. +pub async fn run_attempt(attempt: &Attempt, caps: &Capabilities, workspace: &dyn Workspace) -> Ran { + let request = RunRequest { + attempt_id: String::new(), + graph: attempt.graph.clone(), + inputs: attempt.inputs.clone(), + }; + serve(&request, caps, workspace) + .await + .into_ran(&attempt.graph) +} + +/// The reply that never came. +/// +/// Deliberately *not* an empty `changed`. Empty means "the host looked and saw +/// nothing"; here nobody looked, and the difference decides the episode. +/// +/// A run with no steps and an empty `changed` is settled mechanically as +/// [`crate::contracts::Blocker::MissingEvidence`], which is **terminal** — the +/// reasoning being that a retry with the same inputs produces the same nothing. +/// That reasoning is right for a graph that did nothing and wrong for a device +/// that dropped off: `ExternalWait` is terminal too, so either would strand the +/// episode permanently because a socket blipped. Saying plainly that the result +/// is unknown routes it to the judge, which can reach a continuable verdict. +fn unreported(graph: &tinyflows::model::WorkflowGraph, why: &str) -> Ran { + RunReport { + changed: format!( + "unknown — the runner did not report ({why}). Whether the run did any \ + of the work is not established either way." + ), + failed: Some(format!("no report from the runner: {why}")), + ..RunReport::default() + } + .into_ran(graph) +} + +#[cfg(test)] +mod tests { + use super::*; + + struct Git(&'static str); + + #[async_trait] + impl Workspace for Git { + async fn mark(&self) -> String { + "abc123".into() + } + async fn changed_since(&self, mark: &str) -> String { + format!("{} since {mark}", self.0) + } + } + + #[tokio::test] + async fn a_host_that_cannot_say_reports_nothing_rather_than_guessing() { + let quiet = Unobserved; + assert!(quiet.mark().await.is_empty()); + assert!(quiet.changed_since("").await.is_empty()); + } + + #[tokio::test] + async fn the_baseline_is_passed_back_to_the_comparison() { + // The reason this is a trait and not a closure: the mark taken before + // the run has to reach the reading taken after it. + let git = Git("1 file changed"); + let mark = git.mark().await; + assert_eq!( + git.changed_since(&mark).await, + "1 file changed since abc123" + ); + } + + fn bare_graph() -> tinyflows::model::WorkflowGraph { + tinyflows::model::WorkflowGraph { + schema_version: 1, + id: Some("g".into()), + name: "g".into(), + inputs: Vec::new(), + agents: Vec::new(), + nodes: Vec::new(), + edges: Vec::new(), + } + } + + #[test] + fn a_failure_is_readable_as_evidence_not_as_an_absence() { + let ran = RunReport { + failed: Some("node 'fetch' timed out".into()), + ..RunReport::default() + } + .into_ran(&bare_graph()); + let evidence = ran.evidence(); + assert_eq!( + evidence.outcome.output["error"], + json!("node 'fetch' timed out") + ); + // No `nodes` key: what the mechanical missing-evidence check reads. + assert!(evidence.outcome.output.get("nodes").is_none()); + } + + #[test] + fn an_unreported_run_does_not_claim_nothing_changed() { + // The bug this exists to prevent. Empty `changed` plus no steps is + // settled mechanically as MissingEvidence, which is terminal — so a + // socket blip would end the episode for good. `ExternalWait` is + // terminal too, so there is no safe blocker to pick; the fix is to stop + // asserting a fact nobody established. + let ran = unreported(&bare_graph(), "deadline elapsed after 600s"); + + assert!( + !ran.changed.is_empty(), + "empty means the host looked and saw nothing; nobody looked" + ); + assert!(ran.changed.contains("unknown"), "{}", ran.changed); + assert!( + ran.failed + .as_deref() + .unwrap_or_default() + .contains("deadline"), + "the transport's own words survive: {:?}", + ran.failed + ); + } + + #[test] + fn an_unreported_run_carries_no_invented_evidence() { + let ran = unreported(&bare_graph(), "no runner connected"); + assert!(ran.steps.is_empty()); + assert!(ran.outcome.pending_approvals.is_empty()); + assert!(!ran.outcome.cancelled); + assert!(ran.outcome.output.get("nodes").is_none()); + } +} diff --git a/crates/adaptive/src/execute/wire.rs b/crates/adaptive/src/execute/wire.rs new file mode 100644 index 0000000..e1821a6 --- /dev/null +++ b/crates/adaptive/src/execute/wire.rs @@ -0,0 +1,392 @@ +//! The contract between the loop and whatever runs the graph. +//! +//! The loop decides; an engine executes. Those two may sit in one process or on +//! opposite ends of a socket, and nothing above this module should be able to +//! tell which. This is the shape that crosses when they are apart — and, +//! deliberately, the shape used when they are together too. +//! +//! # Steps, not the final output +//! +//! A run's [`RunOutcome::output`] is a per-node map and looks like the obvious +//! thing to send. It is a lossy projection of the steps, and lossy in the four +//! places that matter to triage: +//! +//! * no `status`, so a node whose error an `on_error` policy swallowed is +//! indistinguishable from one that worked — that message survives *only* on +//! the step; +//! * no duration; +//! * no null-binding diagnostics; +//! * a looped node collapses to one entry however many times it ran; +//! * and a run that returned `Err` has **no output at all**, while its steps are +//! all still there. That is the run most in need of triage. +//! +//! So the steps cross, and the server reconstructs the rest. [`Diagnosis`](tinyflows::diagnostics::Diagnosis) is +//! not sent either: `diagnose` is a pure function of the graph and the steps, +//! the server already has the graph, and re-deriving it there is both smaller +//! and impossible to disagree about. +//! +//! # Two budgets, applied per node +//! +//! [`bounded_within`] is **whole-value and non-recursive**: hand it a map of +//! twelve nodes where one returned 300 KB and it replaces the entire map with a +//! truncated preview of the serialized string. Every other node's output is +//! gone — not trimmed, gone. +//! +//! So bounding happens **per node**, never on the aggregate, at two budgets: +//! +//! * [`RECORD_BUDGET`] on [`StepRecord::output`] — the durable record, written +//! once, generous. +//! * [`PROMPT_BUDGET`] on the reconstructed [`RunOutcome::output`] — what the +//! judge reads, where a dozen node outputs share one context window. +//! +//! Both come from the engine's own note on the function: a durable record uses +//! a generous budget because it is written once; a projection for a model uses +//! a much smaller one. + +use serde::{Deserialize, Serialize}; +use serde_json::{Map, Value, json}; +use tinyflows::engine::RunOutcome; +use tinyflows::evidence::bounded_within; +use tinyflows::expr::NullResolution; +use tinyflows::model::WorkflowGraph; +use tinyflows::observability::{ExecutionStep, StepStatus}; + +use super::Ran; + +/// Per-node budget for the durable record. Written once; generous. +pub const RECORD_BUDGET: usize = 256 * 1024; + +/// Per-node budget for what the judge reads. A dozen of these share one context +/// window, so it is much smaller than the record. +pub const PROMPT_BUDGET: usize = 4 * 1024; + +/// Whether a node succeeded. +/// +/// A mirror of [`StepStatus`], which does not derive `Serialize`. Mirrored +/// rather than patched upstream so the wire format can version independently of +/// the engine. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum StepOutcome { + /// The node executed and produced output. + Success, + /// The node's executor errored, after any retries. + Error, +} + +/// One node activation, as it crosses the wire. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct StepRecord { + /// The node that ran. Not unique across the list: a looped node appears + /// once per iteration, in order, which is the history `output` loses. + pub node_id: String, + /// Whether it succeeded. The only place a swallowed error is visible. + pub status: StepOutcome, + /// What it emitted, bounded to the budget it was recorded at. + pub output: Value, + /// Wall-clock milliseconds. `u64` rather than the engine's `u128`, which + /// has no faithful JSON representation; saturating, because a node that ran + /// for 584 million years has a different problem. + pub duration_ms: u64, + /// Config expressions that resolved to null during this activation. + #[serde(default)] + pub null_bindings: Vec, +} + +impl StepRecord { + /// Record a step, bounding its output to `budget`. + #[must_use] + pub fn bounded(step: &ExecutionStep, budget: usize) -> Self { + Self { + node_id: step.node_id.clone(), + status: match step.status { + StepStatus::Success => StepOutcome::Success, + StepStatus::Error => StepOutcome::Error, + }, + output: bounded_within(&step.output, budget), + duration_ms: u64::try_from(step.duration_ms).unwrap_or(u64::MAX), + null_bindings: step.diagnostics.clone(), + } + } + + /// Back to an engine step, so `diagnose` can read it on the far side. + #[must_use] + pub fn to_step(&self) -> ExecutionStep { + ExecutionStep { + node_id: self.node_id.clone(), + status: match self.status { + StepOutcome::Success => StepStatus::Success, + StepOutcome::Error => StepStatus::Error, + }, + output: self.output.clone(), + duration_ms: u128::from(self.duration_ms), + diagnostics: self.null_bindings.clone(), + } + } +} + +/// What the loop asks an engine to run. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct RunRequest { + /// Correlates the reply. The loop's own attempt identity, not a task id. + pub attempt_id: String, + /// The graph to run. Validated by intake before it ever gets here. + pub graph: WorkflowGraph, + /// Values for the graph's declared inputs. + pub inputs: Map, +} + +/// What comes back. +/// +/// Everything the closing layer reads, and nothing else: no history, no +/// workflow, no lessons. A device cannot see the episode it is part of. +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct RunReport { + /// Echoed from the request. + pub attempt_id: String, + /// Every node activation, in completion order. + pub steps: Vec, + /// Gates the run parked on. + #[serde(default)] + pub pending_approvals: Vec, + /// Whether it wound down on a cancellation. + #[serde(default)] + pub cancelled: bool, + /// What the host says changed outside the run. + #[serde(default)] + pub changed: String, + /// The engine error, when the run did not complete. + #[serde(default)] + pub failed: Option, + /// What it cost, in the host's unit. Zero means not measured. + /// + /// Carried from the start even though nothing consumes it yet: the runner + /// is the only thing that knows the number, and a column added later cannot + /// distinguish a genuine zero from a retrofitted one. + #[serde(default)] + pub cost_usd: f64, +} + +impl RunReport { + /// Rebuild what the closing layer takes. + /// + /// `graph` comes from the loop's own side — it authored or selected it — so + /// nothing here trusts the runner for the shape of the thing it ran. + /// + /// The reconstructed [`RunOutcome::output`] is bounded at + /// [`PROMPT_BUDGET`], not [`RECORD_BUDGET`]: it exists to be rendered into + /// the judge's prompt. The full-fidelity per-node record stays on + /// [`Ran::steps`]. + #[must_use] + pub fn into_ran(self, graph: &WorkflowGraph) -> Ran { + let steps: Vec = self.steps.iter().map(StepRecord::to_step).collect(); + let diagnosis = tinyflows::diagnostics::diagnose(graph, &steps); + + // Last activation wins, matching the engine's own final state: a looped + // node's latest output is what a downstream binding would have read. + // The per-iteration history is not lost — it is on `steps`. + let mut nodes = Map::new(); + for step in &self.steps { + nodes.insert( + step.node_id.clone(), + bounded_within(&step.output, PROMPT_BUDGET), + ); + } + + let mut output = Map::new(); + if !nodes.is_empty() { + output.insert("nodes".into(), Value::Object(nodes)); + } + if let Some(message) = &self.failed { + output.insert("error".into(), json!(message)); + } + + Ran { + outcome: RunOutcome { + output: Value::Object(output), + pending_approvals: self.pending_approvals, + cancelled: self.cancelled, + }, + diagnosis, + changed: self.changed, + failed: self.failed, + steps: self.steps, + cost_usd: self.cost_usd, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use tinyflows::evidence::is_truncated; + + fn step(node_id: &str, status: StepStatus, output: Value) -> ExecutionStep { + ExecutionStep { + node_id: node_id.into(), + status, + output, + duration_ms: 12, + diagnostics: Vec::new(), + } + } + + fn graph() -> WorkflowGraph { + WorkflowGraph { + schema_version: 1, + id: Some("g".into()), + name: "g".into(), + inputs: Vec::new(), + agents: Vec::new(), + nodes: Vec::new(), + edges: Vec::new(), + } + } + + #[test] + fn one_fat_node_does_not_take_the_rest_of_the_record_with_it() { + // The whole reason bounding is per node. `bounded_within` is + // non-recursive: applied to the aggregate, the big one would replace + // every other node's output with a string preview. + let big = json!({ "body": "x".repeat(600 * 1024) }); + let report = RunReport { + steps: vec![ + StepRecord::bounded( + &step("small", StepStatus::Success, json!({"ok": 1})), + RECORD_BUDGET, + ), + StepRecord::bounded(&step("huge", StepStatus::Success, big), RECORD_BUDGET), + ], + ..RunReport::default() + }; + + assert!( + !is_truncated(&report.steps[0].output), + "the small node is intact" + ); + assert!( + is_truncated(&report.steps[1].output), + "the big one is trimmed" + ); + assert_eq!(report.steps[0].output, json!({"ok": 1})); + } + + #[test] + fn a_swallowed_error_survives_the_round_trip() { + // `output` alone cannot express this, which is why steps cross. + let record = StepRecord::bounded( + &step( + "fetch", + StepStatus::Error, + json!({"error": "connection refused"}), + ), + RECORD_BUDGET, + ); + let json = serde_json::to_string(&record).expect("serializes"); + let back: StepRecord = serde_json::from_str(&json).expect("deserializes"); + assert_eq!(back.status, StepOutcome::Error); + assert!(matches!(back.to_step().status, StepStatus::Error)); + } + + #[test] + fn every_iteration_of_a_looped_node_is_kept() { + let report = RunReport { + steps: vec![ + StepRecord::bounded( + &step("body", StepStatus::Success, json!({"i": 1})), + RECORD_BUDGET, + ), + StepRecord::bounded( + &step("body", StepStatus::Success, json!({"i": 2})), + RECORD_BUDGET, + ), + StepRecord::bounded( + &step("body", StepStatus::Success, json!({"i": 3})), + RECORD_BUDGET, + ), + ], + ..RunReport::default() + }; + assert_eq!(report.steps.len(), 3); + + // The reconstructed final state keeps only the last, as the engine's own + // does — the history lives on `steps`. + let ran = report.into_ran(&graph()); + assert_eq!(ran.outcome.output["nodes"]["body"], json!({"i": 3})); + assert_eq!(ran.steps.len(), 3); + } + + #[test] + fn the_judges_view_is_bounded_tighter_than_the_record() { + let body = json!({ "body": "x".repeat(64 * 1024) }); + let report = RunReport { + steps: vec![StepRecord::bounded( + &step("agent", StepStatus::Success, body), + RECORD_BUDGET, + )], + ..RunReport::default() + }; + // Well under the record budget, so kept whole there... + assert!(!is_truncated(&report.steps[0].output)); + + let ran = report.into_ran(&graph()); + // ...and trimmed in the projection the model reads. + assert!(is_truncated(&ran.outcome.output["nodes"]["agent"])); + assert!( + !is_truncated(&ran.steps[0].output), + "the record is untouched" + ); + } + + #[test] + fn a_failed_run_still_carries_every_step_it_managed() { + // The case `output` cannot express at all: the engine returned Err, so + // there is no outcome, but eleven steps happened. + let report = RunReport { + steps: (0..11) + .map(|i| { + StepRecord::bounded( + &step("loop", StepStatus::Success, json!({ "i": i })), + RECORD_BUDGET, + ) + }) + .collect(), + failed: Some("loop node exceeded its maximum of 5 iterations".into()), + ..RunReport::default() + }; + let ran = report.into_ran(&graph()); + assert_eq!(ran.steps.len(), 11); + assert_eq!( + ran.outcome.output["error"], + json!("loop node exceeded its maximum of 5 iterations") + ); + // And the nodes are there too, so the judge sees what did happen rather + // than only that something broke. + assert!(ran.outcome.output["nodes"]["loop"].is_object()); + } + + #[test] + fn the_whole_report_round_trips_as_json() { + let report = RunReport { + attempt_id: "ep-1/3".into(), + steps: vec![StepRecord::bounded( + &step("write", StepStatus::Success, json!({"path": "report.md"})), + RECORD_BUDGET, + )], + pending_approvals: vec!["publish".into()], + cancelled: false, + changed: "1 file changed".into(), + failed: None, + cost_usd: 0.42, + }; + let text = serde_json::to_string(&report).expect("serializes"); + assert!(text.contains("attemptId"), "camelCase on the wire: {text}"); + let back: RunReport = serde_json::from_str(&text).expect("deserializes"); + assert_eq!(back.attempt_id, "ep-1/3"); + assert_eq!(back.pending_approvals, vec!["publish".to_string()]); + assert!((back.cost_usd - 0.42).abs() < f64::EPSILON); + } +} diff --git a/crates/adaptive/src/host.rs b/crates/adaptive/src/host.rs new file mode 100644 index 0000000..51ecac1 --- /dev/null +++ b/crates/adaptive/src/host.rs @@ -0,0 +1,627 @@ +//! What this host will actually permit a workflow to do. +//! +//! The grounding an author most needs and cannot derive. Every fact here is +//! enforced at **run** time by whoever runs the graph, so a graph that ignores +//! one saves cleanly, validates cleanly, and then fails the first time it +//! matters — usually overnight, to nobody watching. +//! +//! Two uses, and both matter: +//! +//! * **rendered into the authoring prompt**, so the model writes something this +//! machine can run; and +//! * **checked after authoring**, because a prompt is a request and a check is +//! a fact. The model will name a worker that does not exist however clearly +//! the list was given. +//! +//! **An absent fact means unknown, never forbidden.** A host that supplies no +//! worker list gets no worker check — not every graph refused. The opposite +//! reading turns an unconfigured host into one that can run nothing, and the +//! symptom is every authored graph failing for a reason the operator never set. + +use serde::{Deserialize, Serialize}; +use serde_json::Value; +use tinyflows::model::{NodeKind, WorkflowGraph}; + +/// What a host permits. Read from that host's configuration, never guessed. +/// +/// Construct it with [`HostFacts::unknown`] and fill in what is actually known: +/// every collection left empty and every `Option` left `None` disables its own +/// check rather than failing it. +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +pub struct HostFacts { + /// Where an `agent` node with no `agent_ref` goes. `None` makes + /// `agent_ref` **mandatory on every agent node** — a host fact that + /// changes a field from optional to required, which is why it cannot be + /// left to the model to infer. + pub default_worker: Option, + /// Workers an `agent_ref` may name. Empty means the list is unknown. + pub workers: Vec, + /// Harness names this host understands, built in or configured. + pub harnesses: Vec, + /// The harness used when a node and the document both stay silent. + pub default_harness: Option, + /// The model used when a node and the document both stay silent. + pub default_model: Option, + /// Tool slugs that resolve without an allowlist entry. + pub native_tools: Vec, + /// Slugs permitted beyond the native ones. Empty *and* `native_tools` + /// empty means slugs are unchecked. + pub tool_allowlist: Vec, + /// Hosts `http_request` may reach. Empty means unchecked. + pub http_allowlist: Vec, + /// Whether `code` nodes run at all. `None` means unknown. + pub allow_code: Option, + /// Whether a `shell` step may use a POSIX shell. `None` means unknown; + /// `Some(false)` is a Windows host, where `shell` is refused rather than + /// emulated. + pub shell_available: Option, + /// Trigger kinds that actually dispatch here. Empty means unchecked — and + /// a host that stores nine kinds while firing one should say so, because + /// the others save and validate and never run. + pub trigger_kinds: Vec, + /// The host's own ceiling, which a graph's `max_iterations` sits under. + pub max_loop_iterations: Option, + /// How many `agent` nodes may run at once. + pub max_parallel_agents: Option, + /// How long a whole run may take. + pub run_timeout_secs: Option, + /// Consequences of the facts above, in prose. + /// + /// Carried beside the data rather than derived from it because the + /// consequence is what the model needs: `default_worker: null` is a fact, + /// "every agent node must name `agent_ref`" is the instruction, and only + /// the host knows which of its facts have consequences worth stating. + pub notes: Vec, +} + +impl HostFacts { + /// A host that has told us nothing. Every check is skipped. + #[must_use] + pub fn unknown() -> Self { + Self::default() + } + + /// Whether anything here is worth showing an author. + #[must_use] + pub fn is_unknown(&self) -> bool { + self.default_worker.is_none() + && self.workers.is_empty() + && self.harnesses.is_empty() + && self.default_harness.is_none() + && self.default_model.is_none() + && self.max_parallel_agents.is_none() + && self.run_timeout_secs.is_none() + && self.native_tools.is_empty() + && self.tool_allowlist.is_empty() + && self.http_allowlist.is_empty() + && self.allow_code.is_none() + && self.shell_available.is_none() + && self.trigger_kinds.is_empty() + && self.max_loop_iterations.is_none() + && self.notes.is_empty() + } + + /// Everything about `graph` this host would refuse, all at once. + /// + /// Every failure rather than the first, for the same reason the validator + /// reports every failure: a model handed one problem fixes it and returns + /// with the next. + #[must_use] + pub fn check(&self, graph: &WorkflowGraph) -> Vec { + let mut problems = Vec::new(); + for node in &graph.nodes { + match node.kind { + NodeKind::Agent => self.check_agent(node, &mut problems), + NodeKind::ToolCall => self.check_tool(node, &mut problems), + NodeKind::HttpRequest => self.check_http(node, &mut problems), + NodeKind::Code => self.check_code(node, &mut problems), + NodeKind::Shell => self.check_shell(node, &mut problems), + NodeKind::Loop => self.check_loop(node, &mut problems), + NodeKind::Trigger => self.check_trigger(node, &mut problems), + _ => {} + } + } + problems + } + + fn check_agent(&self, node: &tinyflows::model::Node, out: &mut Vec) { + let named = text(&node.config, "agent_ref"); + match named { + None => { + if self.default_worker.is_none() && !self.workers.is_empty() { + out.push(format!( + "node `{}`: this host has no default worker, so every agent node must \ + name `config.agent_ref` (one of: {})", + node.id, + self.workers.join(", ") + )); + } + } + Some(reference) => { + if !self.workers.is_empty() && !self.workers.iter().any(|w| w == reference) { + out.push(format!( + "node `{}`: no worker named `{reference}` on this host (have: {})", + node.id, + self.workers.join(", ") + )); + } + } + } + if let Some(harness) = text(&node.config, "harness") + && !self.harnesses.is_empty() + && !self.harnesses.iter().any(|h| h == harness) + { + out.push(format!( + "node `{}`: no harness named `{harness}` here (have: {})", + node.id, + self.harnesses.join(", ") + )); + } + } + + fn check_tool(&self, node: &tinyflows::model::Node, out: &mut Vec) { + let Some(slug) = text(&node.config, "slug") else { + return; + }; + if self.native_tools.is_empty() && self.tool_allowlist.is_empty() { + return; + } + let known = self.native_tools.iter().chain(self.tool_allowlist.iter()); + if !known.into_iter().any(|s| s == slug) { + out.push(format!( + "node `{}`: the tool slug `{slug}` does not resolve here (native: {}; allowed: {})", + node.id, + render_list(&self.native_tools), + render_list(&self.tool_allowlist), + )); + } + } + + fn check_http(&self, node: &tinyflows::model::Node, out: &mut Vec) { + if self.http_allowlist.is_empty() { + return; + } + let Some(url) = text(&node.config, "url") else { + return; + }; + // A URL built from an expression is only known at run time. Refusing + // it here would refuse the correct way to write a parameterised + // request, so an unresolvable host is left to run time on purpose. + if url.starts_with('=') { + return; + } + // DNS names are case-insensitive; an allowlist that rejects + // `API.GitHub.com` against `github.com` costs the episode a spurious + // authoring round. + let Some(host) = host_of(url) else { return }; + let host = host.to_ascii_lowercase(); + if !self + .http_allowlist + .iter() + .map(|allowed| allowed.to_ascii_lowercase()) + .any(|allowed| host == allowed || host.ends_with(&format!(".{allowed}"))) + { + out.push(format!( + "node `{}`: this host may not reach `{host}` (allowed: {})", + node.id, + self.http_allowlist.join(", ") + )); + } + } + + fn check_code(&self, node: &tinyflows::model::Node, out: &mut Vec) { + if self.allow_code == Some(false) { + out.push(format!( + "node `{}`: `code` nodes are disabled on this host", + node.id + )); + } + } + + fn check_shell(&self, node: &tinyflows::model::Node, out: &mut Vec) { + if self.shell_available == Some(false) { + out.push(format!( + "node `{}`: this host refuses POSIX shell rather than emulating it — \ + use a `code` node with javascript or python", + node.id + )); + } + } + + fn check_loop(&self, node: &tinyflows::model::Node, out: &mut Vec) { + let (Some(ceiling), Some(asked)) = ( + self.max_loop_iterations, + node.config.get("max_iterations").and_then(Value::as_u64), + ) else { + return; + }; + if asked > ceiling { + out.push(format!( + "node `{}`: max_iterations {asked} is above this host's ceiling of {ceiling}, \ + so the loop stops earlier than the graph says", + node.id + )); + } + } + + fn check_trigger(&self, node: &tinyflows::model::Node, out: &mut Vec) { + if self.trigger_kinds.is_empty() { + return; + } + let kind = text(&node.config, "trigger_kind").unwrap_or("manual"); + if !self.trigger_kinds.iter().any(|k| k == kind) { + out.push(format!( + "node `{}`: a `{kind}` trigger is stored but never dispatched here — \ + this host fires: {}", + node.id, + self.trigger_kinds.join(", ") + )); + } + } + + /// The facts as an author should read them. + /// + /// Returns an empty string when nothing is known, so a caller can append it + /// unconditionally without producing an empty heading. + #[must_use] + pub fn render(&self) -> String { + if self.is_unknown() { + return String::new(); + } + let mut lines = vec!["# What this host permits — enforced at run time".to_string()]; + let mut say = |label: &str, value: String| { + if !value.is_empty() { + lines.push(format!("- {label}: {value}")); + } + }; + + say( + "default worker", + self.default_worker + .clone() + .unwrap_or_else(|| "none — every agent node must name config.agent_ref".into()), + ); + say("workers", render_list(&self.workers)); + say("harnesses", render_list(&self.harnesses)); + say( + "default harness", + self.default_harness.clone().unwrap_or_default(), + ); + say( + "default model", + self.default_model.clone().unwrap_or_default(), + ); + say("tool slugs that resolve", render_list(&self.native_tools)); + say("tool slugs also allowed", render_list(&self.tool_allowlist)); + say("http hosts reachable", render_list(&self.http_allowlist)); + if let Some(allowed) = self.allow_code { + say( + "code nodes", + if allowed { + "permitted".into() + } else { + "DISABLED".into() + }, + ); + } + if self.shell_available == Some(false) { + say( + "posix shell", + "refused, not emulated — use javascript or python".into(), + ); + } + say("triggers that fire", render_list(&self.trigger_kinds)); + if let Some(cap) = self.max_loop_iterations { + say( + "loop ceiling", + format!("{cap} iterations, whatever a graph asks for"), + ); + } + if let Some(cap) = self.max_parallel_agents { + say("agents at once", cap.to_string()); + } + if let Some(secs) = self.run_timeout_secs { + say("run timeout", format!("{secs}s")); + } + for note in &self.notes { + lines.push(format!("- {note}")); + } + lines.join("\n") + } +} + +fn text<'a>(config: &'a Value, key: &str) -> Option<&'a str> { + config + .get(key) + .and_then(Value::as_str) + .map(str::trim) + .filter(|s| !s.is_empty()) +} + +fn render_list(items: &[String]) -> String { + items.join(", ") +} + +/// The host part of a URL, without pulling in a URL parser for one field. +fn host_of(url: &str) -> Option<&str> { + let rest = url.split_once("://").map_or(url, |(_, rest)| rest); + let authority = rest.split(['/', '?', '#']).next()?; + let host = authority.rsplit_once('@').map_or(authority, |(_, h)| h); + let host = host.split(':').next()?; + (!host.is_empty()).then_some(host) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_host_that_configured_any_rendered_fact_is_not_unknown() { + // `is_unknown` must test every field `render` prints: a fact it skips + // is one that silently never reaches the authoring prompt. + for facts in [ + HostFacts { + default_harness: Some("codex".into()), + ..HostFacts::unknown() + }, + HostFacts { + default_model: Some("gpt-5".into()), + ..HostFacts::unknown() + }, + HostFacts { + max_parallel_agents: Some(2), + ..HostFacts::unknown() + }, + HostFacts { + run_timeout_secs: Some(600), + ..HostFacts::unknown() + }, + ] { + assert!(!facts.is_unknown(), "{facts:?}"); + assert!(!facts.render().is_empty(), "and it renders"); + } + } + + #[test] + fn host_names_compare_case_insensitively() { + // DNS is case-insensitive; `API.GitHub.com` against `github.com` must + // not cost the episode a spurious authoring round. + let facts = HostFacts { + http_allowlist: vec!["github.com".into()], + ..HostFacts::unknown() + }; + let graph = graph(vec![node( + "fetch", + NodeKind::HttpRequest, + serde_json::json!({ "url": "https://API.GitHub.com/repos/x", "method": "GET" }), + )]); + assert!(facts.check(&graph).is_empty(), "{:?}", facts.check(&graph)); + } + use serde_json::json; + use tinyflows::model::Node; + + fn node(id: &str, kind: NodeKind, config: Value) -> Node { + Node { + id: id.into(), + kind, + type_version: 1, + name: id.into(), + config, + ports: Vec::new(), + position: None, + } + } + + fn graph(nodes: Vec) -> WorkflowGraph { + WorkflowGraph { + nodes, + ..WorkflowGraph::default() + } + } + + #[test] + fn a_host_that_has_said_nothing_refuses_nothing() { + // The reading that would break every unconfigured deployment: empty + // meaning "deny" rather than "unknown". + let facts = HostFacts::unknown(); + let g = graph(vec![ + node("a", NodeKind::Agent, json!({ "agent_ref": "anyone" })), + node( + "t", + NodeKind::ToolCall, + json!({ "slug": "anything:at:all" }), + ), + node( + "c", + NodeKind::Code, + json!({ "language": "python", "source": "1" }), + ), + ]); + assert!(facts.check(&g).is_empty()); + assert!( + facts.render().is_empty(), + "nothing known renders as nothing" + ); + } + + #[test] + fn a_worker_this_host_does_not_have_is_named() { + let facts = HostFacts { + workers: vec!["laptop".into(), "ci".into()], + ..HostFacts::unknown() + }; + let problems = facts.check(&graph(vec![node( + "a", + NodeKind::Agent, + json!({ "agent_ref": "desktop" }), + )])); + assert_eq!(problems.len(), 1); + assert!(problems[0].contains("desktop"), "{problems:?}"); + assert!( + problems[0].contains("laptop, ci"), + "the alternatives are offered" + ); + } + + #[test] + fn no_default_worker_makes_agent_ref_mandatory() { + // A host fact that changes a field from optional to required. + let facts = HostFacts { + workers: vec!["laptop".into()], + default_worker: None, + ..HostFacts::unknown() + }; + let problems = facts.check(&graph(vec![node("a", NodeKind::Agent, json!({}))])); + assert_eq!(problems.len(), 1); + assert!(problems[0].contains("must name"), "{problems:?}"); + } + + #[test] + fn a_default_worker_makes_a_bare_agent_node_fine() { + let facts = HostFacts { + workers: vec!["laptop".into()], + default_worker: Some("laptop".into()), + ..HostFacts::unknown() + }; + assert!( + facts + .check(&graph(vec![node("a", NodeKind::Agent, json!({}))])) + .is_empty() + ); + } + + #[test] + fn a_slug_outside_both_lists_is_refused() { + let facts = HostFacts { + native_tools: vec!["medulla:shell".into()], + tool_allowlist: vec!["github".into()], + ..HostFacts::unknown() + }; + let g = graph(vec![ + node("ok", NodeKind::ToolCall, json!({ "slug": "medulla:shell" })), + node("no", NodeKind::ToolCall, json!({ "slug": "slack" })), + ]); + let problems = facts.check(&g); + assert_eq!(problems.len(), 1); + assert!(problems[0].contains("slack"), "{problems:?}"); + } + + #[test] + fn an_http_host_outside_the_allowlist_is_refused_but_a_subdomain_is_not() { + let facts = HostFacts { + http_allowlist: vec!["github.com".into()], + ..HostFacts::unknown() + }; + let g = graph(vec![ + node( + "ok", + NodeKind::HttpRequest, + json!({ "url": "https://api.github.com/x" }), + ), + node( + "no", + NodeKind::HttpRequest, + json!({ "url": "https://evil.test/x" }), + ), + ]); + let problems = facts.check(&g); + assert_eq!(problems.len(), 1, "{problems:?}"); + assert!(problems[0].contains("evil.test")); + } + + #[test] + fn a_url_built_from_an_expression_is_left_to_run_time() { + // Refusing it would refuse the correct way to write a parameterised + // request, which is the thing the authoring prompt asks for. + let facts = HostFacts { + http_allowlist: vec!["github.com".into()], + ..HostFacts::unknown() + }; + let g = graph(vec![node( + "u", + NodeKind::HttpRequest, + json!({ "url": "=\"https://\" + .inputs.host" }), + )]); + assert!(facts.check(&g).is_empty()); + } + + #[test] + fn disabled_code_and_refused_shell_are_both_reported() { + let facts = HostFacts { + allow_code: Some(false), + shell_available: Some(false), + ..HostFacts::unknown() + }; + let g = graph(vec![ + node( + "c", + NodeKind::Code, + json!({ "language": "python", "source": "1" }), + ), + node("s", NodeKind::Shell, json!({ "script": "ls" })), + ]); + assert_eq!( + facts.check(&g).len(), + 2, + "every failure at once, not the first" + ); + } + + #[test] + fn a_loop_above_the_host_ceiling_is_reported() { + // Otherwise it silently stops earlier than the graph says. + let facts = HostFacts { + max_loop_iterations: Some(10), + ..HostFacts::unknown() + }; + let g = graph(vec![node( + "l", + NodeKind::Loop, + json!({ "max_iterations": 50 }), + )]); + let problems = facts.check(&g); + assert_eq!(problems.len(), 1); + assert!(problems[0].contains("ceiling of 10"), "{problems:?}"); + } + + #[test] + fn a_trigger_kind_that_never_fires_is_reported() { + let facts = HostFacts { + trigger_kinds: vec!["manual".into()], + ..HostFacts::unknown() + }; + let g = graph(vec![node( + "t", + NodeKind::Trigger, + json!({ "trigger_kind": "schedule" }), + )]); + let problems = facts.check(&g); + assert_eq!(problems.len(), 1); + assert!(problems[0].contains("never dispatched"), "{problems:?}"); + } + + #[test] + fn the_rendering_states_consequences_not_just_values() { + let facts = HostFacts { + default_worker: None, + workers: vec!["laptop".into()], + allow_code: Some(false), + notes: vec!["Only manual triggers fire here.".into()], + ..HostFacts::unknown() + }; + let rendered = facts.render(); + assert!(rendered.contains("every agent node must name config.agent_ref")); + assert!(rendered.contains("DISABLED")); + assert!(rendered.contains("Only manual triggers fire here.")); + } + + #[test] + fn a_url_without_a_scheme_still_yields_its_host() { + assert_eq!(host_of("api.github.com/x"), Some("api.github.com")); + assert_eq!( + host_of("https://user:pw@api.github.com:443/x"), + Some("api.github.com") + ); + assert_eq!(host_of(""), None); + } +} diff --git a/crates/adaptive/src/intake/author.rs b/crates/adaptive/src/intake/author.rs new file mode 100644 index 0000000..417a06b --- /dev/null +++ b/crates/adaptive/src/intake/author.rs @@ -0,0 +1,262 @@ +//! Writing a graph when nothing stored fits. +//! +//! Two things make the difference between a graph that runs and one that +//! validates and then does nothing, and both are here rather than in the +//! prompt's good intentions: +//! +//! * the node catalogue is **generated from the engine**, not described from +//! memory, so a config field cannot be invented; and +//! * the result is **validated before it is returned**, so an authoring mistake +//! is an error from intake rather than a run-time failure that reads like the +//! work failing. + +use tinyflows::caps::Capabilities; +use tinyflows::catalog::{NodeKindContract, all_contracts}; +use tinyflows::model::WorkflowGraph; +use tinyflows::store::HostPolicy; +use tinyflows::validate::validate_all; + +use super::{Attempt, IntakeError, Result, ask}; +use crate::contracts::{Approach, Goal, Tier}; +use crate::host::HostFacts; + +const SYSTEM: &str = "\ +You write a workflow graph that achieves a goal. + +Return JSON: {\"graph\": , \"why\": str, \"inputs\": {name: value}} + +The graph is the engine's own format: +{\"schema_version\": 1, \"name\": str, \"inputs\": [{\"name\", \"required\", \"description\"}], + \"nodes\": [{\"id\", \"kind\", \"name\", \"config\": {...}}], + \"edges\": [{\"from_node\", \"from_port\", \"to_node\", \"to_port\"}]} + +Rules that are checked, not requested: + +- Exactly one `trigger` node, and it is where the graph starts. +- Every node kind and every config field must come from the catalogue below. + It is generated from the engine, so it is the truth; a field you remember + from elsewhere is a field that resolves to null at run time. +- Ports default to `main` on both ends. Name one only where the catalogue says + a node has others (a `condition` emits `true`/`false`; a `loop` emits `body` + and `done`). +- Declare in `inputs` anything the goal supplies as data — a repository, a path, + an id — and read it in config rather than pasting the literal. A graph with + the value baked in is a graph that works once. + +How one node reads another. A config string starting with `=` is an expression; +everything else is a literal. + + =item.name a field of the direct predecessor's output + =nodes.fetch.item.json.body a field of any completed node, by node id + =run.trigger.payload what the trigger carried + =.items | length a leading dot makes the rest a jq program + +There are no braces. `={{ ... }}` is not a binding — it is a jq program that +fails to compile, and a failed program is null, so the step runs with an empty +value and reports success. + +`agent`, `tool_call` and `http_request` wrap their output in +`{json, text, raw}`. Their fields are under `.json`: write +`=nodes.fetch.item.json.body`, never `=nodes.fetch.item.body`. The second form +validates, dry-runs green, and resolves to null every time. + +Design guidance, which is judgement rather than a check: + +- Fewer nodes is better. An `agent` node is a whole coding-agent session on some + hosts — minutes, not seconds — so a graph of eight is usually a worse answer + than a graph of three. +- Use `agent` for work that cannot be specified, and the determined kinds for + everything else. Fetching, reshaping and branching are not agent work. +- Say what a step is for, concretely. The agent running it sees the goal and + that instruction and nothing else — not the other nodes, not what they found. + +Where a section below states what this host permits, it is the machine's own +configuration and is enforced when the graph runs. A graph that ignores it saves +cleanly, validates cleanly, and fails the first time it matters. + +Where a section lists what this episode already tried, write something +DIFFERENT. Not the same graph with a reworded prompt — a different shape: other +nodes, another order, a step that checks what the last attempt assumed. If every +approach you can think of is already on that list, say so in `why` and write the +smallest graph that would establish which assumption is wrong."; + +/// Write a graph for `goal`, grounded on the engine's own node catalogue. +/// +/// # Errors +/// When inference fails, the reply holds no graph, or the graph does not +/// validate. An invalid graph is never returned: the caller would hand it +/// straight to `compile`, and the resulting failure would be attributed to the +/// work rather than to the authoring. +pub async fn author( + goal: &Goal, + facts: &HostFacts, + policy: &dyn HostPolicy, + past: &str, + caps: &Capabilities, + conn: Option<&str>, +) -> Result { + let permitted = facts.render(); + let user = format!( + "# Goal\n{}\n\n# Node catalogue — the only kinds and fields that exist\n{}{}{past}", + goal.text.trim(), + catalogue(), + if permitted.is_empty() { + String::new() + } else { + format!("\n\n{permitted}") + } + ); + + let answer = ask(caps, conn, Tier::Author, SYSTEM, &user).await?; + let raw = answer + .get("graph") + .cloned() + .ok_or_else(|| IntakeError::Inference("the reply has no `graph` key".to_string()))?; + + let graph: WorkflowGraph = serde_json::from_value(raw) + .map_err(|e| IntakeError::Invalid(format!("not a workflow graph: {e}")))?; + + // Every failure at once, not the first. A model handed one error fixes it + // and returns with the next; handed all four it fixes all four. + let problems = validate_all(&graph); + if !problems.is_empty() { + return Err(IntakeError::Invalid( + problems + .iter() + .map(ToString::to_string) + .collect::>() + .join("; "), + )); + } + + // Three gates, and the order is cost. `validate_all` is structural and + // free. `HostFacts::check` is our own reading of the machine's config. + // `check_graph` is the host's, which may know things we were not told — + // it runs last because it is the one that can reach outside this process. + let refused = facts.check(&graph); + if !refused.is_empty() { + return Err(IntakeError::Unsupported(refused.join("; "))); + } + if let Err(err) = policy.check_graph(graph.id.as_deref().unwrap_or("authored"), &graph) { + return Err(IntakeError::Unsupported(err.to_string())); + } + + Ok(Attempt { + approach: Approach::Authored { + why: answer["why"].as_str().unwrap_or_default().to_string(), + fingerprint: fingerprint(&graph), + }, + graph, + inputs: answer["inputs"].as_object().cloned().unwrap_or_default(), + // Filled by `decide`, which is what knows what the planner was shown. + lessons_shown: Vec::new(), + }) +} + +/// A digest of the graph's runnable shape. +/// +/// Nodes, edges and declared inputs — not the name, not the description. Two +/// graphs that run identically and differ in prose are the same attempt, and +/// the whole point of the exclusion list is that the second one is recognised +/// as a repeat rather than counted as a fresh idea. Inputs are in because a +/// graph that requires a value behaves differently from one that does not, +/// even when every node matches. +fn fingerprint(graph: &WorkflowGraph) -> String { + // The stable digest, because this string is persisted in ledger rows as + // the exclusion-list signature — see `reuse::digest_hex` on why not + // `DefaultHasher`. + crate::reuse::digest_hex(&crate::reuse::shape_bytes(graph)) +} + +/// The node catalogue, rendered for a prompt. +/// +/// Generated from [`all_contracts`] rather than written out here, so a node +/// kind the engine gains appears without this file being touched — and a field +/// this file could describe wrongly cannot exist. +fn catalogue() -> String { + all_contracts() + .iter() + .map(render) + .collect::>() + .join("\n") +} + +fn render(contract: &NodeKindContract) -> String { + let fields = contract + .config_fields + .iter() + .map(|field| { + let mark = if field.required { "*" } else { " " }; + let allowed = match field.enum_values.as_ref() { + Some(values) if !values.is_empty() => format!(" [{}]", values.join("|")), + _ => String::new(), + }; + format!(" {mark}{}: {}{allowed}", field.name, field.value_type) + }) + .collect::>() + .join("\n"); + + // Only the outputs, and only when they are not the default. `from_port` is + // the field an author gets wrong; inputs are almost always `main` and + // listing them on every kind is noise that hides the one that matters. + let outputs = &contract.ports.outputs; + let ports = if outputs.as_slice() == ["main".to_string()] || outputs.is_empty() { + String::new() + } else { + format!(" out ports: {}\n", outputs.join(", ")) + }; + + format!("{}: {}\n{ports}{fields}", contract.kind, contract.summary) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn the_catalogue_is_generated_from_the_engine() { + // If this list were written by hand it would already be wrong: it is + // the thing the prompt calls the truth. + let rendered = catalogue(); + for kind in [ + "trigger", + "agent", + "tool_call", + "http_request", + "condition", + "loop", + ] { + assert!(rendered.contains(kind), "catalogue is missing {kind}"); + } + } + + #[test] + fn required_fields_are_marked() { + let rendered = catalogue(); + // `trigger_kind` is required on a trigger; a model that misses it + // authors a graph that cannot start. + assert!(rendered.contains("*trigger_kind"), "{rendered}"); + } + + #[test] + fn enum_fields_show_their_allowed_values() { + assert!( + catalogue().contains("manual"), + "trigger_kind's values must be listed" + ); + } + + #[test] + fn a_graph_with_no_trigger_is_refused_rather_than_returned() { + // Not reachable through `author` without a provider, so the invariant + // is asserted against the validator this module gates on. + let graph = WorkflowGraph { + name: "no trigger".to_string(), + ..WorkflowGraph::default() + }; + assert!( + !validate_all(&graph).is_empty(), + "an empty graph must not validate — intake gates on exactly this" + ); + } +} diff --git a/crates/adaptive/src/intake/mod.rs b/crates/adaptive/src/intake/mod.rs new file mode 100644 index 0000000..34297dd --- /dev/null +++ b/crates/adaptive/src/intake/mod.rs @@ -0,0 +1,375 @@ +//! Prompt in, runnable graph out. +//! +//! Two paths and one rule between them: **prefer a stored workflow, author only +//! when nothing fits.** That ordering is the whole economic argument of the +//! loop — a procedure that has already worked costs one cheap selection call to +//! reuse and a full authoring call to reinvent, and reinventing it also throws +//! away every score it had accumulated. +//! +//! Neither path names a model or a provider. Both reach inference through the +//! engine's own [`LlmProvider`](tinyflows::caps::LlmProvider), so the host decides who answers and supplies +//! the credential as an opaque `conn` reference this crate never inspects. +//! +//! What comes out is an [`Attempt`]: an [`Approach`] saying how the decision was +//! reached, a graph that has been **validated**, and the inputs to run it with. +//! A graph leaves here compilable or not at all. + +mod author; +mod select; + +pub use author::author; +pub use select::{Candidate, bind, select}; + +use serde_json::{Map, Value}; +use tinyflows::caps::Capabilities; +use tinyflows::model::WorkflowGraph; +use tinyflows::store::WorkflowStore; + +use crate::contracts::{Approach, Goal}; +use crate::host::HostFacts; +use crate::ledger::Ledger; + +/// What intake decided to run, and how it got there. +#[derive(Debug, Clone)] +pub struct Attempt { + /// Selected, authored, or a variant — and why. Becomes the ledger row's + /// signature, and therefore the next attempt's exclusion list. + pub approach: Approach, + /// Validated. An invalid graph is an error from intake, never a return + /// value: handing one to the engine turns an authoring mistake into a + /// run-time failure that looks like the work failing. + pub graph: WorkflowGraph, + /// Values for the graph's declared inputs, by name. + pub inputs: Map, + /// The lessons this attempt's planner was shown. + /// + /// Carried so the closing pass can score them against what happened. A + /// lesson's `applied` counter is the denominator of its help rate, and + /// nothing was incrementing it: `score_lesson` had exactly one caller, the + /// corroboration loop, which moves both numbers together. So every lesson + /// read either 0/0 or n/n, the rate carried no information, and the + /// ordering built on it could not order anything. + pub lessons_shown: Vec, +} + +/// What went wrong deciding. +#[derive(Debug, thiserror::Error)] +pub enum IntakeError { + /// The store could not be read. + #[error("workflow store: {0}")] + Store(String), + /// The ledger could not be read. + #[error("ledger: {0}")] + Ledger(#[from] crate::ledger::LedgerError), + /// The model was unreachable, or answered with something unusable. + #[error("inference: {0}")] + Inference(String), + /// The model authored a graph the engine would refuse. + #[error("authored an invalid graph: {0}")] + Invalid(String), + /// The graph is well formed but names something this host does not have — + /// a worker, a tool slug, a reachable address. Distinct from `Invalid` + /// because the graph is fine and the *machine* is the constraint, which is + /// what the retry has to be told. + #[error("this host cannot run that graph: {0}")] + Unsupported(String), + /// A stored workflow was chosen whose declared inputs cannot be filled. + #[error("workflow {id} needs an input nothing supplied: {missing}")] + Unbindable { + /// The workflow that could not be bound. + id: String, + /// The first input with no value. + missing: String, + }, +} + +/// Convenience alias for intake results. +pub type Result = std::result::Result; + +/// Decide how to attempt `goal`, given what this episode has already tried. +/// +/// Selection runs first and authoring is the fallback, not the default. The +/// exclusion list matters more here than anywhere else in the loop: without it +/// attempt four re-selects the workflow attempt two already failed on, and the +/// episode pays twice for one dead end. +/// +/// # Errors +/// When the store or ledger cannot be read, inference fails, or the authored +/// graph does not validate. +pub async fn decide( + goal: &Goal, + episode: &str, + store: &dyn WorkflowStore, + ledger: &dyn Ledger, + facts: &HostFacts, + caps: &Capabilities, + conn: Option<&str>, +) -> Result { + // One read, two uses. The exclusion list and the rendered history are the + // same rows seen two ways, and `Ledger::tried` is a fresh query — calling + // it here as well would pay for the identical result twice on every + // attempt, against whatever database the host brought. + let rows = ledger.rows(episode).await?; + let tried = crate::ledger::signatures(&rows); + let candidates = catalogue(store, ledger, &tried).await?; + + // Both planners see the same past, in the same words. The exclusion list + // stops a *selection* being repeated, but nothing structural stops the + // author writing attempt two's graph again on attempt four — only being + // shown attempt two does. And the lessons were being written and never + // read, which is a knowledge store that costs money and returns nothing. + let lessons = crate::recall::retrieve( + ledger.lessons(None).await?, + None, + crate::recall::RECALL_LIMIT, + ); + let past = format!( + "{}{}", + crate::recall::render_history(&rows), + crate::recall::render_lessons(&lessons) + ); + + let shown: Vec = lessons.iter().map(|l| l.id.clone()).collect(); + + if let Some(chosen) = select(goal, &candidates, &past, caps, conn).await? { + // `select` answers with an id; the graph and the input check come from + // the store. Returning the choice unbound would hand the engine an + // empty graph, which compiles to nothing and reads as the work failing. + return bind(chosen, store).map(|attempt| Attempt { + lessons_shown: shown, + ..attempt + }); + } + author(goal, facts, store.policy(), &past, caps, conn) + .await + .map(|attempt| Attempt { + lessons_shown: shown, + ..attempt + }) +} + +/// The stored workflows worth offering, with what is known about each. +/// +/// Three filters, each removing something a planner must not be shown: +/// +/// * **disabled** — the operator turned it off; offering it invites a choice +/// that cannot be honoured. +/// * **already tried this episode** — its signature is in the exclusion list. +/// * **not selectable** — a `draft` variant is a proposal, not a procedure. It +/// is run deliberately by whoever proposed it, never chosen by a planner that +/// has not seen its evidence. +/// +/// The scores come from our ledger rather than the record, because +/// `WorkflowRecord` has no place for them: a score is a fact that spans runs. +/// +/// Then one more pass: a repaired family collapses to a single row, its +/// [`champion`]. Four near-identical graphs whose descriptions differ by a +/// clause is not a choice, it is noise, and a planner asked to make it is being +/// asked to guess. Which member survives is decided on score, never on being +/// the newest — see [`crate::promotion`]. +async fn catalogue( + store: &dyn WorkflowStore, + ledger: &dyn Ledger, + tried: &[String], +) -> Result> { + let listed = store + .list() + .map_err(|e| IntakeError::Store(e.to_string()))?; + + let mut out = Vec::new(); + for summary in listed { + if !summary.enabled { + continue; + } + let signature = format!("selected:{}", summary.id); + if tried.iter().any(|t| t == &signature) { + continue; + } + let score = ledger.workflow_score(&summary.id).await?; + out.push(Candidate { + id: summary.id, + name: summary.name, + description: summary.description, + node_count: summary.node_count, + applied: score.applied, + helped: score.helped, + }); + } + collapse_families(out, ledger).await +} + +/// Reduce each repaired family to its champion. +/// +/// A member excluded earlier — disabled, or already tried this episode — is +/// still counted when picking the champion but cannot be the one offered. That +/// matters: if the champion is the workflow this episode just failed with, +/// dropping the whole family would hide a variant that exists precisely because +/// the champion fell short. So the family's *best still-offerable* member is +/// what survives. +async fn collapse_families( + candidates: Vec, + ledger: &dyn Ledger, +) -> Result> { + let mut kept: Vec = Vec::new(); + let mut settled: Vec = Vec::new(); + + for candidate in candidates.iter() { + if settled.contains(&candidate.id) { + continue; + } + let lineage = ledger.lineage(&candidate.id).await?; + if lineage.len() <= 1 { + kept.push(candidate.clone()); + settled.push(candidate.id.clone()); + continue; + } + + // Scores for the whole family, including members not on offer — a + // parent that is disabled still counts as evidence about its variants. + let mut family = Vec::with_capacity(lineage.len()); + for id in &lineage { + family.push((id.clone(), ledger.workflow_score(id).await?)); + } + let best = crate::promotion::champion(&family).unwrap_or(&candidate.id); + + let offer = candidates + .iter() + .find(|c| c.id == best) + .or_else(|| { + // The champion is not offerable. Fall back to the best of what + // is, in family order, rather than dropping the family whole. + lineage + .iter() + .find_map(|id| candidates.iter().find(|c| &c.id == id)) + }) + .unwrap_or(candidate); + if !settled.contains(&offer.id) { + kept.push(offer.clone()); + } + settled.extend(lineage); + } + Ok(kept) +} + +/// Ask the host's model for one JSON object. +/// +/// Every intake call has this shape, and the failure modes are shared: a model +/// that answers with prose around its JSON, or with nothing. Both become +/// [`IntakeError::Inference`] here rather than at three call sites. +/// +/// # Errors +/// When the provider fails, or its answer holds no JSON object. +pub(crate) async fn ask( + caps: &Capabilities, + conn: Option<&str>, + tier: crate::contracts::Tier, + system: &str, + user: &str, +) -> Result { + let request = serde_json::json!({ + // Which job, never which model. A host reads this to route judging and + // selecting to different places; one that ignores it gets the old + // behaviour, which is why it is a plain field and not a required one. + "tier": tier.as_str(), + "messages": [ + { "role": "system", "content": system }, + { "role": "user", "content": user }, + ], + // A hint, not a guarantee: hosts differ in whether they honour it, so + // `extract` still has to cope with prose around the object. + "response_format": { "type": "json_object" }, + }); + + let answer = caps + .llm + .complete(request, conn) + .await + .map_err(|e| IntakeError::Inference(e.to_string()))?; + + extract(&answer).ok_or_else(|| { + IntakeError::Inference(format!("no JSON object in the reply: {}", peek(&answer))) + }) +} + +/// The JSON object inside a completion response, wherever the host put it. +/// +/// Hosts wrap differently — some return the object, some an OpenAI-shaped +/// envelope, some a string of JSON in a `text` field. Rather than demand one +/// shape from every host, this reads all three, because the alternative is a +/// crate that only works against the provider it was written for. +fn extract(answer: &Value) -> Option { + if answer.is_object() && !answer["choices"].is_array() && answer.get("text").is_none() { + return Some(answer.clone()); + } + let text = answer["choices"][0]["message"]["content"] + .as_str() + .or_else(|| answer["text"].as_str()) + .or_else(|| answer["content"].as_str())?; + from_text(text) +} + +/// A JSON object out of text that may have prose around it. +fn from_text(text: &str) -> Option { + if let Ok(value) = serde_json::from_str::(text.trim()) { + return Some(value); + } + // A fenced block, or a sentence before the object. Bounded by the first `{` + // and the last `}` rather than by parsing markdown, which a model will + // eventually emit in a form no parser expected. + let start = text.find('{')?; + let end = text.rfind('}')?; + serde_json::from_str(text.get(start..=end)?).ok() +} + +fn peek(value: &Value) -> String { + let mut text = value.to_string(); + // Floor to a char boundary: `truncate` panics mid-codepoint, and this runs + // on exactly the path that should become an `Inference` error — a provider + // reply with a multi-byte character at byte 200 must not abort the task. + let mut end = text.len().min(200); + while !text.is_char_boundary(end) { + end -= 1; + } + text.truncate(end); + text +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_bare_object_is_read_as_itself() { + let answer = serde_json::json!({ "workflow_id": "pr-review" }); + assert_eq!(extract(&answer).unwrap()["workflow_id"], "pr-review"); + } + + #[test] + fn an_openai_shaped_envelope_is_unwrapped() { + let answer = serde_json::json!({ + "choices": [{ "message": { "content": "{\"workflow_id\":\"pr-review\"}" } }] + }); + assert_eq!(extract(&answer).unwrap()["workflow_id"], "pr-review"); + } + + #[test] + fn a_text_field_holding_json_is_read() { + let answer = serde_json::json!({ "text": "{\"workflow_id\":\"x\"}" }); + assert_eq!(extract(&answer).unwrap()["workflow_id"], "x"); + } + + #[test] + fn prose_around_the_object_does_not_lose_it() { + // Models do this whatever the response_format asked for. + let answer = serde_json::json!({ + "text": "Sure! Here you go:\n```json\n{\"workflow_id\":\"x\"}\n```\nHope that helps." + }); + assert_eq!(extract(&answer).unwrap()["workflow_id"], "x"); + } + + #[test] + fn an_answer_with_no_object_at_all_is_none_rather_than_a_panic() { + assert!(extract(&serde_json::json!({ "text": "I could not decide." })).is_none()); + assert!(extract(&serde_json::json!("just a string")).is_none()); + } +} diff --git a/crates/adaptive/src/intake/select.rs b/crates/adaptive/src/intake/select.rs new file mode 100644 index 0000000..500bc35 --- /dev/null +++ b/crates/adaptive/src/intake/select.rs @@ -0,0 +1,239 @@ +//! Choosing a stored workflow, or declining to. +//! +//! The cheap path, and the one that should win once anything has been learned. +//! A selection is one small call against a list; authoring is a large call that +//! also discards whatever the existing procedure had proved about itself. +//! +//! Declining is a first-class answer, not a failure. A model pushed to always +//! pick something will pick the nearest thing, and a near-miss workflow runs to +//! completion producing confidently wrong work — which is more expensive than +//! authoring, not less. + +use serde_json::{Map, Value}; +use tinyflows::caps::Capabilities; +use tinyflows::model::WorkflowGraph; +use tinyflows::store::WorkflowStore; + +use super::{Attempt, IntakeError, Result, ask}; +use crate::contracts::{Approach, Goal, Tier}; + +/// One stored workflow as the chooser sees it. +#[derive(Debug, Clone)] +pub struct Candidate { + /// The id the choice is made on. + pub id: String, + /// Display name; falls back to the id when blank. + pub name: String, + /// What the model actually reads to decide. A workflow with none is a row + /// nobody can choose on purpose. + pub description: String, + /// A rough cost signal. + pub node_count: usize, + /// Times chosen and run. + pub applied: u32, + /// Times that ended satisfied. + pub helped: u32, +} + +impl Candidate { + fn render(&self) -> String { + let name = if self.name.is_empty() { + &self.id + } else { + &self.name + }; + let description = if self.description.is_empty() { + "(no description — nobody can choose this on purpose)" + } else { + &self.description + }; + // Both numbers, never a rate: 1/1 and 40/40 are the same rate and are + // not the same evidence, and the model is being asked to weigh exactly + // that difference. + let record = match self.applied { + 0 => "never run".to_string(), + applied => format!("run {applied}×, satisfied {}×", self.helped), + }; + format!( + "- id: {}\n name: {name}\n steps: {}, {record}\n {description}", + self.id, self.node_count + ) + } +} + +const SYSTEM: &str = "\ +You choose whether a saved workflow already does what a goal asks for. + +Return JSON: {\"workflow_id\": str | null, \"why\": str, \"inputs\": {name: value}} + +- workflow_id: the id of the workflow that does this, or null. +- why: one line. When you decline, say what is missing — it is read by whoever + writes the replacement. +- inputs: values for that workflow's declared inputs, taken from the goal. Only + what the goal actually states; never invent a repository, a path or an id. + +Choose one ONLY when it does what the goal asks. A workflow that does something +adjacent is worse than none: it will run to completion and produce confident +work for a job nobody wanted, which costs more than writing a new one. + +Prefer a workflow with a record over one without, and weigh both numbers rather +than the ratio — run 40× satisfied 30× is a known quantity, run 1× satisfied 1× +is a coin landing once. A workflow that has never run is still a fair choice +when it plainly matches; it just carries no evidence. + +When this episode has already tried something, decline rather than choose a +workflow that would fall short the same way. Being told a second time that the +report has no numbers in it costs a full run and establishes nothing."; + +/// Ask whether any candidate does the job, and bind its inputs if one does. +/// +/// `Ok(None)` means nothing fitted — the ordinary case on a cold store, and the +/// caller's cue to author. +/// +/// # Errors +/// When inference fails, or the chosen workflow cannot be loaded or bound. +pub async fn select( + goal: &Goal, + candidates: &[Candidate], + past: &str, + caps: &Capabilities, + conn: Option<&str>, +) -> Result> { + // Not a shortcut — a correctness point. With nothing to choose from the + // answer can only be "none", and asking costs a call to be told so. + if candidates.is_empty() { + return Ok(None); + } + + let listing = candidates + .iter() + .map(Candidate::render) + .collect::>() + .join("\n"); + let user = format!( + "# Goal\n{}\n\n# Saved workflows\n{listing}{past}", + goal.text.trim() + ); + + let answer = ask(caps, conn, Tier::Select, SYSTEM, &user).await?; + let Some(id) = answer["workflow_id"] + .as_str() + .filter(|s| !s.trim().is_empty()) + else { + return Ok(None); + }; + // A model naming something that is not on the list has hallucinated an id; + // treat it as a decline rather than looking it up, or a typo becomes a + // store read for a workflow nobody offered. + if !candidates.iter().any(|c| c.id == id) { + return Ok(None); + } + + Ok(Some(Attempt { + approach: Approach::Selected { + workflow_id: id.to_string(), + why: answer["why"].as_str().unwrap_or_default().to_string(), + }, + graph: WorkflowGraph::default(), + inputs: inputs_of(&answer), + // Filled by `decide`, which is what knows what the planner was shown. + lessons_shown: Vec::new(), + })) +} + +/// Load the chosen workflow and check every declared input has a value. +/// +/// Binding is checked here, *after* the model picks and before anything runs. +/// The model is confident about inputs it did not actually find in the goal, so +/// the cheap deterministic check catches what the expensive one asserted. +/// +/// # Errors +/// When the workflow is gone, or an input has no value. +pub fn bind(attempt: Attempt, store: &dyn WorkflowStore) -> Result { + let Approach::Selected { + ref workflow_id, .. + } = attempt.approach + else { + return Ok(attempt); + }; + let record = store + .get(workflow_id) + .map_err(|e| IntakeError::Store(e.to_string()))? + .ok_or_else(|| IntakeError::Store(format!("workflow {workflow_id} vanished")))?; + + for declared in &record.graph.inputs { + if !declared.required { + continue; + } + let filled = attempt + .inputs + .get(&declared.name) + .is_some_and(|v| !v.is_null() && v.as_str() != Some("")); + if !filled { + return Err(IntakeError::Unbindable { + id: workflow_id.clone(), + missing: declared.name.clone(), + }); + } + } + + Ok(Attempt { + graph: record.graph, + ..attempt + }) +} + +fn inputs_of(answer: &Value) -> Map { + answer["inputs"].as_object().cloned().unwrap_or_default() +} + +#[cfg(test)] +mod tests { + use super::*; + + fn candidate(id: &str, applied: u32, helped: u32) -> Candidate { + Candidate { + id: id.to_string(), + name: format!("the {id} workflow"), + description: "reviews a closed issue end to end".to_string(), + node_count: 4, + applied, + helped, + } + } + + #[test] + fn a_listing_shows_both_counters_not_a_rate() { + let rendered = candidate("pr-review", 40, 30).render(); + assert!(rendered.contains("run 40×, satisfied 30×"), "{rendered}"); + assert!(!rendered.contains("75"), "a rate hides the sample size"); + } + + #[test] + fn a_workflow_that_has_never_run_says_so_rather_than_showing_zeroes() { + let rendered = candidate("fresh", 0, 0).render(); + assert!(rendered.contains("never run"), "{rendered}"); + } + + #[test] + fn a_workflow_with_no_description_says_it_cannot_be_chosen_on_purpose() { + let mut c = candidate("bare", 0, 0); + c.description = String::new(); + assert!(c.render().contains("nobody can choose this on purpose")); + } + + #[test] + fn a_blank_name_falls_back_to_the_id() { + let mut c = candidate("only-an-id", 1, 1); + c.name = String::new(); + assert!(c.render().contains("name: only-an-id")); + } + + #[test] + fn the_prompt_tells_the_model_that_declining_is_allowed() { + // The single most important line in it: a model pushed to always pick + // will pick the nearest thing, and a near miss runs to completion. + assert!(SYSTEM.contains("or null")); + assert!(SYSTEM.contains("worse than none")); + } +} diff --git a/crates/adaptive/src/inventory.rs b/crates/adaptive/src/inventory.rs new file mode 100644 index 0000000..e4a2da9 --- /dev/null +++ b/crates/adaptive/src/inventory.rs @@ -0,0 +1,209 @@ +//! What a tenant has, for a reader rather than for a planner. +//! +//! [`crate::intake`] builds a catalogue too, and it is a different question. +//! That one answers *what may this attempt choose* — so it drops what is +//! disabled, what this episode already tried, and every family member but the +//! champion. Answering "what does this tenant have" with that view would hide a +//! workflow the moment an episode used it. +//! +//! This one hides nothing and decides nothing. It is the read behind a screen, +//! an audit, or a support question, which is why the standing is reported +//! rather than applied. + +use std::sync::Arc; + +use tinyflows::store::WorkflowStore; + +use crate::intake::{IntakeError, Result}; +use crate::ledger::{Ledger, Score}; +use crate::promotion::{Standing, standing}; + +/// One stored workflow, with everything known about it. +#[derive(Debug, Clone)] +pub struct Listing { + /// The id it is stored and scored under. + pub id: String, + /// Display name. + pub name: String, + /// What a planner reads to choose it. + pub description: String, + /// A rough cost signal. + pub node_count: usize, + /// Whether an operator has switched it off. Reported, not filtered: a + /// disabled workflow is exactly what someone asking this question is often + /// looking for. + pub enabled: bool, + /// Runs and successes, for this tenant. + pub score: Score, + /// Where it sits in its family. + pub standing: Standing, + /// The workflow it was repaired from, when it was. + pub parent: Option, + /// Whether the loop wrote it, rather than a person. + /// + /// Read off the id rather than stored, because the alternative is a flag on + /// `WorkflowRecord` — the engine's type, which an upstream merge would + /// contend with for a fact only we care about. + pub learned: bool, +} + +/// Every workflow this tenant can see, with its record. +/// +/// # Errors +/// When the store or the ledger cannot be read. +pub async fn shelf(store: &Arc, ledger: &dyn Ledger) -> Result> { + let listed = store + .list() + .map_err(|e| IntakeError::Store(e.to_string()))?; + + let mut out = Vec::with_capacity(listed.len()); + for summary in listed { + let lineage = ledger.lineage(&summary.id).await?; + let mut family: Vec<(String, Score)> = Vec::with_capacity(lineage.len()); + for id in &lineage { + family.push((id.clone(), ledger.workflow_score(id).await?)); + } + let score = family + .iter() + .find(|(id, _)| id == &summary.id) + .map_or_else(Score::default, |(_, score)| *score); + + out.push(Listing { + standing: standing(&summary.id, &family), + parent: ledger.parent_of(&summary.id).await?, + learned: summary.id.starts_with("learned-"), + score, + id: summary.id, + name: summary.name, + description: summary.description, + node_count: summary.node_count, + enabled: summary.enabled, + }); + } + Ok(out) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::ledger::memory::MemoryLedger; + use tinyflows::model::WorkflowGraph; + use tinyflows::store::{FileWorkflowStore, types::WorkflowRecord}; + + /// The store validates on save, so a fixture needs a graph that compiles. + fn tiny_graph(id: &str) -> WorkflowGraph { + WorkflowGraph { + schema_version: 1, + id: Some(id.into()), + name: id.into(), + inputs: Vec::new(), + agents: Vec::new(), + nodes: vec![tinyflows::model::Node { + id: "start".into(), + kind: tinyflows::model::NodeKind::Trigger, + type_version: 1, + name: "manual".into(), + config: serde_json::json!({ "trigger_kind": "manual" }), + ports: Vec::new(), + position: None, + }], + edges: Vec::new(), + } + } + + fn stored(id: &str, enabled: bool) -> WorkflowRecord { + WorkflowRecord { + id: id.into(), + name: id.into(), + description: "does a thing".into(), + enabled, + defaults: tinyflows::store::types::WorkflowDefaults::default(), + graph: tiny_graph(id), + source_path: None, + } + } + + fn store(tag: &str) -> Arc { + let root = + std::env::temp_dir().join(format!("adaptive-shelf-{}-{tag}", std::process::id())); + let _ = std::fs::remove_dir_all(&root); + std::fs::create_dir_all(root.join("workflows")).expect("temp dir"); + Arc::new(FileWorkflowStore::new( + vec![root.join("workflows")], + root.join("runs"), + )) + } + + #[tokio::test] + async fn it_reports_the_disabled_and_the_already_tried_rather_than_hiding_them() { + // The difference from intake's catalogue, which drops both. Someone + // asking what a tenant has is often asking precisely about the one that + // is switched off. + let store = store("all"); + store.save(&stored("weekly", true)).expect("save"); + store.save(&stored("retired", false)).expect("save"); + let ledger = MemoryLedger::new(); + + let shelf = shelf(&store, &ledger).await.expect("shelf"); + assert_eq!(shelf.len(), 2); + assert!(shelf.iter().any(|l| l.id == "retired" && !l.enabled)); + } + + #[tokio::test] + async fn a_family_is_reported_whole_with_each_members_standing() { + // intake collapses this to one row. A reader wants to see that the + // variant exists and where it stands. + let store = store("family"); + store.save(&stored("weekly", true)).expect("save"); + store.save(&stored("weekly-fix-1", true)).expect("save"); + let ledger = MemoryLedger::new(); + ledger + .link_variant("weekly", "weekly-fix-1") + .await + .expect("link"); + for _ in 0..4 { + ledger.score_workflow("weekly", true).await.expect("score"); + } + + let shelf = shelf(&store, &ledger).await.expect("shelf"); + assert_eq!(shelf.len(), 2, "both members, not just the champion"); + + let parent = shelf.iter().find(|l| l.id == "weekly").expect("parent"); + assert_eq!(parent.standing, Standing::Champion); + assert_eq!((parent.score.applied, parent.score.helped), (4, 4)); + assert_eq!(parent.parent, None); + + let variant = shelf + .iter() + .find(|l| l.id == "weekly-fix-1") + .expect("variant"); + assert_eq!(variant.standing, Standing::Unproven, "no trials yet"); + assert_eq!(variant.parent.as_deref(), Some("weekly")); + } + + #[tokio::test] + async fn what_the_loop_wrote_is_distinguishable_from_what_a_person_did() { + let store = store("learned"); + store.save(&stored("weekly", true)).expect("save"); + store.save(&stored("learned-a1b2c3d", true)).expect("save"); + + let shelf = shelf(&store, &MemoryLedger::new()).await.expect("shelf"); + assert!(!shelf.iter().find(|l| l.id == "weekly").expect("w").learned); + assert!( + shelf + .iter() + .find(|l| l.id == "learned-a1b2c3d") + .expect("l") + .learned + ); + } + + #[tokio::test] + async fn a_workflow_nobody_has_run_reports_zero_rather_than_erroring() { + let store = store("cold"); + store.save(&stored("fresh", true)).expect("save"); + let shelf = shelf(&store, &MemoryLedger::new()).await.expect("shelf"); + assert_eq!((shelf[0].score.applied, shelf[0].score.helped), (0, 0)); + assert_eq!(shelf[0].standing, Standing::Unproven); + } +} diff --git a/crates/adaptive/src/ledger/conformance.rs b/crates/adaptive/src/ledger/conformance.rs new file mode 100644 index 0000000..8d8392b --- /dev/null +++ b/crates/adaptive/src/ledger/conformance.rs @@ -0,0 +1,753 @@ +//! One suite every [`Ledger`] backend must pass. +//! +//! Two backends with separate test files drift: sqlite gets a case, Mongo does +//! not, and the difference surfaces in production as "it worked locally". So +//! the cases live here, take `&dyn Ledger`, and each backend's own tests are a +//! four-line call into this module. +//! +//! Compiled always, not behind `cfg(test)`, so a host writing its own backend +//! can run the same suite against it. + +use super::{Episode, EpisodeStatus, Ledger, LedgerRow, Lesson, LessonKind}; + +/// A row with the fields a test does not care about filled in. +#[must_use] +pub fn row(episode: &str, attempt: u32, sig: &str) -> LedgerRow { + LedgerRow { + id: String::new(), + episode: episode.to_string(), + attempt, + approach_sig: sig.to_string(), + approach_desc: format!("attempt {attempt} via {sig}"), + workflow_id: None, + outcome: String::new(), + cause: String::new(), + cost_usd: 0.0, + at: format!("2026-01-01T00:00:{attempt:02}Z"), + satisfied: false, + advanced: false, + } +} + +/// A lesson with a trigger that describes a class rather than an instance. +#[must_use] +pub fn lesson(trigger: &str) -> Lesson { + Lesson { + id: String::new(), + kind: LessonKind::Constraint, + trigger: trigger.to_string(), + mechanism: "because the API caps a page at 100".to_string(), + claim: "page the listing rather than raising per_page".to_string(), + applied: 0, + helped: 0, + scope_key: None, + } +} + +/// Run every case against `store`. Panics with a named assertion on failure, +/// so a backend's own test is one line and the failure still says what broke. +/// +/// # Panics +/// On any conformance failure, or if the backend errors on a call the contract +/// says must succeed. +pub async fn run_all(store: &dyn Ledger) { + appended_rows_come_back_in_order(store).await; + an_episode_sees_only_its_own_rows(store).await; + tried_is_the_deduplicated_exclusion_list(store).await; + an_unknown_episode_is_empty_not_an_error(store).await; + a_lesson_round_trips_with_its_evidence(store).await; + lessons_filter_by_kind(store).await; + scoring_a_lesson_moves_applied_always_and_helped_conditionally(store).await; + a_workflow_nobody_has_run_scores_zero_rather_than_erroring(store).await; + workflow_scores_accumulate(store).await; + run_lineage(store).await; + run_episodes(store).await; + run_transcripts(store).await; +} + +async fn appended_rows_come_back_in_order(store: &dyn Ledger) { + let ep = "ep-order"; + for n in 1..=3 { + store.append(&row(ep, n, "authored")).await.expect("append"); + } + let got = store.rows(ep).await.expect("rows"); + assert_eq!( + got.iter().map(|r| r.attempt).collect::>(), + vec![1, 2, 3], + "rows must read oldest first — a ledger read backwards makes every gap analysis wrong" + ); + assert!(!got[0].id.is_empty(), "append must assign an id"); +} + +async fn an_episode_sees_only_its_own_rows(store: &dyn Ledger) { + store + .append(&row("ep-a", 1, "authored")) + .await + .expect("append"); + store + .append(&row("ep-b", 1, "authored")) + .await + .expect("append"); + assert_eq!(store.rows("ep-a").await.expect("rows").len(), 1); + assert_eq!(store.rows("ep-b").await.expect("rows").len(), 1); +} + +async fn tried_is_the_deduplicated_exclusion_list(store: &dyn Ledger) { + let ep = "ep-tried"; + store + .append(&row(ep, 1, "selected:pr-review")) + .await + .expect("append"); + store.append(&row(ep, 2, "authored")).await.expect("append"); + store.append(&row(ep, 3, "authored")).await.expect("append"); + + let tried = store.tried(ep).await.expect("tried"); + assert_eq!( + tried, + vec!["selected:pr-review".to_string(), "authored".to_string()], + "each signature once, in the order first spent" + ); +} + +async fn an_unknown_episode_is_empty_not_an_error(store: &dyn Ledger) { + // A first-time goal must read its (absent) history without failing, or + // every episode's first attempt errors. + assert!(store.rows("never-seen").await.expect("rows").is_empty()); + assert!(store.tried("never-seen").await.expect("tried").is_empty()); +} + +async fn a_lesson_round_trips_with_its_evidence(store: &dyn Ledger) { + let ep = "ep-lesson"; + let a = store.append(&row(ep, 1, "authored")).await.expect("append"); + let b = store.append(&row(ep, 2, "authored")).await.expect("append"); + + let id = store + .promote( + &lesson("a paginated listing API with a hard per-page cap"), + &[a.clone(), b.clone()], + ) + .await + .expect("promote"); + assert!(!id.is_empty()); + + let cited = store.evidence(&id).await.expect("evidence"); + let mut ids: Vec = cited.into_iter().map(|r| r.id).collect(); + ids.sort(); + let mut want = vec![a, b]; + want.sort(); + assert_eq!( + ids, want, + "a lesson must be able to show the rows behind it" + ); +} + +async fn lessons_filter_by_kind(store: &dyn Ledger) { + let mut strategy = lesson("a wide fan-out over independent items"); + strategy.kind = LessonKind::Strategy; + store.promote(&strategy, &[]).await.expect("promote"); + + let only = store + .lessons(Some(LessonKind::Strategy)) + .await + .expect("lessons"); + assert!(!only.is_empty()); + assert!(only.iter().all(|l| l.kind == LessonKind::Strategy)); + + let all = store.lessons(None).await.expect("lessons"); + assert!(all.len() >= only.len(), "None must not filter"); +} + +async fn scoring_a_lesson_moves_applied_always_and_helped_conditionally(store: &dyn Ledger) { + let id = store + .promote(&lesson("a scoring probe"), &[]) + .await + .expect("promote"); + + store.score_lesson(&id, true).await.expect("score"); + store.score_lesson(&id, false).await.expect("score"); + + let found = store + .lessons(None) + .await + .expect("lessons") + .into_iter() + .find(|l| l.id == id) + .expect("the lesson just promoted"); + + assert_eq!(found.applied, 2, "shown twice"); + assert_eq!(found.helped, 1, "only one of those runs was satisfied"); +} + +async fn a_workflow_nobody_has_run_scores_zero_rather_than_erroring(store: &dyn Ledger) { + let score = store.workflow_score("never-run").await.expect("score"); + assert_eq!(score.applied, 0); + assert_eq!(score.helped, 0); +} + +async fn workflow_scores_accumulate(store: &dyn Ledger) { + let id = "wf-accumulate"; + store.score_workflow(id, true).await.expect("score"); + store.score_workflow(id, true).await.expect("score"); + store.score_workflow(id, false).await.expect("score"); + + let score = store.workflow_score(id).await.expect("score"); + assert_eq!(score.applied, 3); + assert_eq!( + score.helped, 2, + "2 of 3 — the evidence a promotion gate reads" + ); +} + +/// Run every tenant-isolation case. +/// +/// Separate from [`run_all`] because it needs three handles onto **one** +/// store — global, and two tenants — and how a backend makes a scoped handle +/// is its own business (`for_tenant` on both that ship). A backend that does +/// not support scoping simply does not call this. +/// +/// # Panics +/// On any isolation failure. Each one is a leak of one tenant's knowledge into +/// another's prompt, so none of them is a soft assertion. +pub async fn run_tenants(global: &dyn Ledger, a: &dyn Ledger, b: &dyn Ledger) { + assert_eq!(global.scope(), None, "the global handle must be unscoped"); + assert!(a.scope().is_some() && b.scope().is_some(), "both scoped"); + assert_ne!(a.scope(), b.scope(), "two different tenants"); + + a_tenants_lesson_is_invisible_to_another(a, b).await; + a_global_lesson_is_visible_to_every_tenant(global, a, b).await; + promote_stamps_the_handle_not_the_argument(a).await; + workflow_scores_do_not_bleed_between_tenants(a, b).await; + a_tenant_writing_does_not_move_the_global_score(global, a).await; + an_episode_id_alone_does_not_reach_another_tenants_attempts(a, b).await; + naming_another_tenants_lesson_id_does_not_move_its_score(global, a, b).await; +} + +async fn naming_another_tenants_lesson_id_does_not_move_its_score( + global: &dyn Ledger, + a: &dyn Ledger, + b: &dyn Ledger, +) { + // The ids reaching `score_lesson` come from model output (corroboration), + // so this is a hole a prompt injection walks through if the backend + // updates by id alone. + let private = a + .promote(&lesson("a private class of situation"), &[]) + .await + .expect("promote"); + b.score_lesson(&private, true) + .await + .expect("no-op, not error"); + let untouched = a + .lessons(None) + .await + .expect("lessons") + .into_iter() + .find(|l| l.id == private) + .expect("still there"); + assert_eq!( + (untouched.applied, untouched.helped), + (0, 0), + "tenant {:?} moved tenant {:?}'s score by naming its id", + b.scope(), + a.scope() + ); + + // A global lesson is visible to every tenant, so scoring it is legitimate. + let shared = global + .promote(&lesson("a class anyone can hit"), &[]) + .await + .expect("promote"); + b.score_lesson(&shared, true).await.expect("score"); + let moved = b + .lessons(None) + .await + .expect("lessons") + .into_iter() + .find(|l| l.id == shared) + .expect("visible"); + assert_eq!((moved.applied, moved.helped), (1, 1)); +} + +async fn an_episode_id_alone_does_not_reach_another_tenants_attempts( + a: &dyn Ledger, + b: &dyn Ledger, +) { + // An episode id is opaque and a service may hand one straight through from + // a request path. Being keyed by episode is not isolation — guessing an id + // would be enough — so the rows carry the bucket too. + a.append(&row("ep-secret", 1, "authored:aaa")) + .await + .expect("append"); + assert_eq!(a.rows("ep-secret").await.expect("rows").len(), 1); + assert!( + b.rows("ep-secret").await.expect("rows").is_empty(), + "tenant {:?} read tenant {:?}'s attempts by knowing the episode id", + b.scope(), + a.scope() + ); +} + +async fn a_tenants_lesson_is_invisible_to_another(a: &dyn Ledger, b: &dyn Ledger) { + let mut mine = lesson("a private class of task"); + mine.claim = "names an internal repository path".into(); + let id = a.promote(&mine, &[]).await.expect("promote"); + + let seen_by_a = a.lessons(None).await.expect("lessons"); + assert!( + seen_by_a.iter().any(|l| l.id == id), + "a tenant must see its own lesson" + ); + + let seen_by_b = b.lessons(None).await.expect("lessons"); + assert!( + !seen_by_b.iter().any(|l| l.id == id), + "tenant {:?} can read tenant {:?}'s lesson — this is the leak the scope exists to stop", + b.scope(), + a.scope() + ); +} + +async fn a_global_lesson_is_visible_to_every_tenant( + global: &dyn Ledger, + a: &dyn Ledger, + b: &dyn Ledger, +) { + let id = global + .promote(&lesson("a class of task anyone can hit"), &[]) + .await + .expect("promote"); + for tenant in [a, b] { + let seen = tenant.lessons(None).await.expect("lessons"); + assert!( + seen.iter().any(|l| l.id == id), + "tenant {:?} cannot see a global lesson", + tenant.scope() + ); + } +} + +async fn promote_stamps_the_handle_not_the_argument(a: &dyn Ledger) { + // A caller — or a model whose answer was deserialized straight into a + // `Lesson` — must not be able to publish into another bucket by asking. + let mut forged = lesson("a class of task claiming to be someone else's"); + forged.scope_key = Some("some-other-tenant".to_string()); + let id = a.promote(&forged, &[]).await.expect("promote"); + + let stored = a + .lessons(None) + .await + .expect("lessons") + .into_iter() + .find(|l| l.id == id) + .expect("stored"); + assert_eq!( + stored.scope_key.as_deref(), + a.scope(), + "promote must stamp the handle's scope, whatever the argument said" + ); +} + +async fn workflow_scores_do_not_bleed_between_tenants(a: &dyn Ledger, b: &dyn Ledger) { + let id = "wf-shared-id"; + a.score_workflow(id, true).await.expect("score"); + a.score_workflow(id, true).await.expect("score"); + b.score_workflow(id, false).await.expect("score"); + + let for_a = a.workflow_score(id).await.expect("score"); + let for_b = b.workflow_score(id).await.expect("score"); + assert_eq!( + (for_a.applied, for_a.helped), + (2, 2), + "tenant a's own record" + ); + assert_eq!( + (for_b.applied, for_b.helped), + (1, 0), + "tenant b's own record" + ); +} + +async fn a_tenant_writing_does_not_move_the_global_score(global: &dyn Ledger, a: &dyn Ledger) { + let id = "wf-tenant-only"; + a.score_workflow(id, true).await.expect("score"); + let seen = global.workflow_score(id).await.expect("score"); + assert_eq!( + (seen.applied, seen.helped), + (0, 0), + "the global bucket is its own bucket, not a union of every tenant's" + ); +} + +/// Run every lineage case. Part of [`run_all`]'s contract for any backend that +/// stores variant links, which is both that ship. +/// +/// # Panics +/// On any lineage failure. +pub async fn run_lineage(store: &dyn Ledger) { + an_unlinked_workflow_is_a_family_of_one(store).await; + lineage_reads_the_same_from_any_member(store).await; + linking_the_same_pair_twice_is_a_no_op(store).await; + a_variant_of_a_variant_stays_in_one_family(store).await; + a_cycle_is_truncated_rather_than_hung(store).await; +} + +async fn an_unlinked_workflow_is_a_family_of_one(store: &dyn Ledger) { + let family = store.lineage("wf-lonely").await.expect("lineage"); + assert_eq!(family, vec!["wf-lonely".to_string()]); +} + +async fn lineage_reads_the_same_from_any_member(store: &dyn Ledger) { + store + .link_variant("wf-a", "wf-a-fix-1") + .await + .expect("link"); + store + .link_variant("wf-a", "wf-a-fix-2") + .await + .expect("link"); + + let from_root = store.lineage("wf-a").await.expect("lineage"); + let from_leaf = store.lineage("wf-a-fix-2").await.expect("lineage"); + assert_eq!( + from_root, from_leaf, + "the champion must not depend on which member was asked" + ); + assert_eq!( + from_root[0], "wf-a", + "root first — the fallback relies on it" + ); + assert_eq!(from_root.len(), 3); +} + +async fn linking_the_same_pair_twice_is_a_no_op(store: &dyn Ledger) { + // A repair converging on an existing variant id will re-link. It must not + // duplicate the family member. + store + .link_variant("wf-b", "wf-b-fix-1") + .await + .expect("link"); + store + .link_variant("wf-b", "wf-b-fix-1") + .await + .expect("link"); + assert_eq!(store.lineage("wf-b").await.expect("lineage").len(), 2); +} + +async fn a_variant_of_a_variant_stays_in_one_family(store: &dyn Ledger) { + // `repair` takes whatever ran as the parent, and what ran may itself be a + // variant. Two generations are still one family, or the grandchild would be + // compared against nothing. + store + .link_variant("wf-c", "wf-c-fix-1") + .await + .expect("link"); + store + .link_variant("wf-c-fix-1", "wf-c-fix-2") + .await + .expect("link"); + + let family = store.lineage("wf-c-fix-2").await.expect("lineage"); + assert_eq!(family[0], "wf-c"); + assert_eq!(family.len(), 3, "{family:?}"); +} + +async fn a_cycle_is_truncated_rather_than_hung(store: &dyn Ledger) { + // Nothing should write this, but the ledger is read on the hot path of + // every attempt and a hang there stops the whole loop. Bounded walks mean + // a corrupt link costs a truncated answer instead. + store.link_variant("wf-y", "wf-x").await.expect("link"); + store.link_variant("wf-x", "wf-y").await.expect("link"); + let family = store.lineage("wf-x").await.expect("lineage"); + assert!(family.len() <= super::MAX_FAMILY, "{family:?}"); +} + +/// Run every episode-checkpoint case. +/// +/// # Panics +/// On any failure. Each is a way a restarted process would lose an episode. +pub async fn run_episodes(store: &dyn Ledger) { + an_unknown_episode_is_absent_not_an_error(store).await; + an_episode_round_trips_everything_the_rows_cannot_hold(store).await; + saving_twice_updates_rather_than_duplicating(store).await; + running_only_filters_to_the_recovery_list(store).await; + a_rows_verdict_survives_as_fields_not_as_prose(store).await; + the_episode_list_is_newest_first_on_every_backend(store).await; +} + +async fn the_episode_list_is_newest_first_on_every_backend(store: &dyn Ledger) { + // `Page` documents newest-first, and paging an unordered list returns + // opposite ends on different backends — `Page::first(1)` must mean the + // same episode everywhere. + for (id, at) in [ + ("ep-ord-old", "2026-02-01T00:00:01Z"), + ("ep-ord-new", "2026-02-01T00:00:03Z"), + ("ep-ord-mid", "2026-02-01T00:00:02Z"), + ] { + let mut e = episode(id, EpisodeStatus::Running, 1, 0); + e.updated_at = at.to_string(); + store.save_episode(&e).await.expect("save"); + } + let ordered: Vec = store + .episodes(false, super::Page::ALL) + .await + .expect("episodes") + .into_iter() + .map(|e| e.id) + .filter(|id| id.starts_with("ep-ord-")) + .collect(); + assert_eq!( + ordered, + ["ep-ord-new", "ep-ord-mid", "ep-ord-old"], + "newest first, on this backend as on every other" + ); +} + +fn episode(id: &str, status: EpisodeStatus, attempt: u32, stalled: u32) -> Episode { + Episode { + id: id.to_string(), + goal: crate::contracts::Goal::new("write the weekly report"), + scope_key: None, + status, + attempt, + stalled, + started_at: "2026-01-01T00:00:00Z".to_string(), + updated_at: "2026-01-01T00:00:05Z".to_string(), + } +} + +async fn an_unknown_episode_is_absent_not_an_error(store: &dyn Ledger) { + assert!( + store + .episode("never-started") + .await + .expect("read") + .is_none() + ); +} + +async fn an_episode_round_trips_everything_the_rows_cannot_hold(store: &dyn Ledger) { + let mut want = episode("ep-round", EpisodeStatus::Running, 3, 2); + want.goal.success_criteria = "cites the actual figures".to_string(); + store.save_episode(&want).await.expect("save"); + + let got = store + .episode("ep-round") + .await + .expect("read") + .expect("saved"); + assert_eq!( + got.goal.text, want.goal.text, + "the goal is unrecoverable from rows" + ); + assert_eq!(got.goal.success_criteria, "cites the actual figures"); + assert_eq!(got.attempt, 3); + assert_eq!(got.stalled, 2, "the stall count cannot be recomputed"); + assert_eq!(got.status, EpisodeStatus::Running); +} + +async fn saving_twice_updates_rather_than_duplicating(store: &dyn Ledger) { + store + .save_episode(&episode("ep-twice", EpisodeStatus::Running, 1, 0)) + .await + .expect("save"); + store + .save_episode(&episode( + "ep-twice", + EpisodeStatus::StoodDown("out of attempts after 12".to_string()), + 12, + 0, + )) + .await + .expect("save"); + + let got = store + .episode("ep-twice") + .await + .expect("read") + .expect("saved"); + assert_eq!(got.attempt, 12); + match got.status { + EpisodeStatus::StoodDown(reason) => assert!(reason.contains("out of attempts")), + other => panic!("expected the second write to win, got {other:?}"), + } + let all = store + .episodes(false, super::Page::ALL) + .await + .expect("episodes"); + assert_eq!( + all.iter().filter(|e| e.id == "ep-twice").count(), + 1, + "one episode, not two" + ); +} + +async fn running_only_filters_to_the_recovery_list(store: &dyn Ledger) { + store + .save_episode(&episode("ep-live", EpisodeStatus::Running, 1, 0)) + .await + .expect("save"); + store + .save_episode(&episode("ep-won", EpisodeStatus::Satisfied, 2, 0)) + .await + .expect("save"); + + let running = store + .episodes(true, super::Page::ALL) + .await + .expect("episodes"); + assert!(running.iter().any(|e| e.id == "ep-live")); + assert!( + !running.iter().any(|e| e.id == "ep-won"), + "a finished episode is not resumed" + ); +} + +async fn a_rows_verdict_survives_as_fields_not_as_prose(store: &dyn Ledger) { + // `satisfied` used to be recoverable only by matching the outcome string, + // and `advanced` not at all — so a restart could not recompute the stall. + let mut won = row("ep-fields", 1, "authored:aaa"); + won.satisfied = true; + won.advanced = true; + store.append(&won).await.expect("append"); + + let back = &store.rows("ep-fields").await.expect("rows")[0]; + assert!(back.satisfied); + assert!(back.advanced); +} + +/// Run every transcript and paging case. +/// +/// # Panics +/// On any failure. +pub async fn run_transcripts(store: &dyn Ledger) { + an_attempt_with_no_transcript_is_empty_not_an_error(store).await; + a_transcript_round_trips_in_order(store).await; + a_looped_node_keeps_every_iteration(store).await; + saving_a_transcript_twice_replaces_rather_than_appends(store).await; + a_page_windows_the_episode_list(store).await; +} + +fn step(node_id: &str, n: u64) -> crate::execute::StepRecord { + crate::execute::StepRecord { + node_id: node_id.to_string(), + status: crate::execute::StepOutcome::Success, + output: serde_json::json!({ "i": n }), + duration_ms: n, + null_bindings: Vec::new(), + } +} + +async fn an_attempt_with_no_transcript_is_empty_not_an_error(store: &dyn Ledger) { + assert!(store.steps("ldg_nothing").await.expect("steps").is_empty()); +} + +async fn a_transcript_round_trips_in_order(store: &dyn Ledger) { + let mut errored = step("fetch", 7); + errored.status = crate::execute::StepOutcome::Error; + errored.null_bindings = vec![tinyflows::expr::NullResolution { + location: "args.to".to_string(), + expression: "=nodes.x.item.email".to_string(), + }]; + store + .save_steps("ldg_a", &[step("start", 1), errored]) + .await + .expect("save"); + + let back = store.steps("ldg_a").await.expect("steps"); + assert_eq!(back.len(), 2); + assert_eq!(back[0].node_id, "start", "execution order is the record"); + assert_eq!(back[1].status, crate::execute::StepOutcome::Error); + assert_eq!(back[1].duration_ms, 7); + assert_eq!( + back[1].null_bindings.len(), + 1, + "the nested list survives both a JSON column and a native array" + ); + assert_eq!(back[1].output, serde_json::json!({ "i": 7 })); +} + +async fn a_looped_node_keeps_every_iteration(store: &dyn Ledger) { + // The reason this is a record per step rather than one blob per attempt. + let steps: Vec<_> = (0..12).map(|n| step("body", n)).collect(); + store.save_steps("ldg_loop", &steps).await.expect("save"); + + let back = store.steps("ldg_loop").await.expect("steps"); + assert_eq!(back.len(), 12); + assert_eq!( + back.iter().map(|s| s.duration_ms).collect::>(), + (0..12).collect::>(), + "iterations in order, not deduplicated by node id" + ); +} + +async fn saving_a_transcript_twice_replaces_rather_than_appends(store: &dyn Ledger) { + // A retried write must not double the record. + store + .save_steps("ldg_twice", &[step("a", 1), step("b", 2)]) + .await + .expect("save"); + store + .save_steps("ldg_twice", &[step("a", 1), step("b", 2)]) + .await + .expect("save"); + assert_eq!(store.steps("ldg_twice").await.expect("steps").len(), 2); + + // And a SHORTER re-save must not leave the old tail behind — an upsert + // keyed by sequence replaces only the sequences present, and the stitched + // result would read as one transcript mixing two attempts. + store + .save_steps("ldg_twice", &[step("a", 9)]) + .await + .expect("save"); + let back = store.steps("ldg_twice").await.expect("steps"); + assert_eq!(back.len(), 1, "{back:?}"); + assert_eq!( + back[0].duration_ms, 9, + "and it is the new save, not the old" + ); +} + +async fn a_page_windows_the_episode_list(store: &dyn Ledger) { + for n in 0..5 { + store + .save_episode(&episode( + &format!("ep-page-{n}"), + EpisodeStatus::Running, + 1, + 0, + )) + .await + .expect("save"); + } + let all = store + .episodes(false, super::Page::ALL) + .await + .expect("episodes"); + assert!(all.len() >= 5); + + let first_two = store + .episodes(false, super::Page::first(2)) + .await + .expect("episodes"); + assert_eq!(first_two.len(), 2); + assert_eq!( + first_two.iter().map(|e| &e.id).collect::>(), + all[..2].iter().map(|e| &e.id).collect::>(), + "the same order, windowed" + ); + + let past_the_end = store + .episodes( + false, + super::Page { + limit: 10, + offset: all.len() + 5, + }, + ) + .await + .expect("episodes"); + assert!( + past_the_end.is_empty(), + "an offset past the end is empty, not a panic" + ); +} diff --git a/crates/adaptive/src/ledger/memory.rs b/crates/adaptive/src/ledger/memory.rs new file mode 100644 index 0000000..3e99245 --- /dev/null +++ b/crates/adaptive/src/ledger/memory.rs @@ -0,0 +1,378 @@ +//! A ledger that forgets. +//! +//! Always compiled — no feature, no driver, no C library — so the crate is +//! usable the moment it is added rather than only after a backend has been +//! chosen. Tests, examples and a first look all want this. +//! +//! # What it is not +//! +//! It is **not the default**, and there is deliberately no `Default` impl on +//! anything that would hand it to a host that did not ask. That is not fussiness +//! about ergonomics; it is the single worst failure this crate could have. +//! +//! Everything else here is built so that a system which appears to be working +//! actually is: a green run with a blank diagnosis means nobody looked, an empty +//! `changed` means nobody checked, an attempt with no ledger row is one the next +//! pass repeats. A ledger silently defaulting to memory is the same shape and +//! worse — the loop runs, the exclusion list works, lessons are written and +//! scored, the tests pass, and every restart throws all of it away. Nobody +//! notices, because the only symptom is that it never gets better. +//! +//! So it is named for what it does, has to be constructed on purpose, and says +//! so in one line at the top. Reach for `super::sqlite` or +//! `super::mongo` the moment learning is supposed to outlive a +//! process. +//! +//! # What it is good for +//! +//! A reference implementation. It passes the same +//! [`conformance`](super::conformance) suite as both real backends, which is +//! worth more than it sounds: it proves the trait is implementable without a +//! database, so a host writing a third backend has a complete, readable example +//! that is checked by the same cases theirs will be. + +use std::collections::HashMap; +use std::sync::{Arc, Mutex}; + +use async_trait::async_trait; + +use super::{Episode, Ledger, LedgerError, LedgerRow, Lesson, LessonKind, Result, Score}; + +#[derive(Default)] +struct Inner { + /// Append-only; the index is the sequence, so insertion order survives a + /// timestamp tie the way both durable backends guarantee. Paired with the + /// bucket that wrote it, which is a column on the row in both durable + /// backends and has nowhere to live on `LedgerRow` itself. + rows: Vec<(String, LedgerRow)>, + lessons: Vec, + /// `(lesson_id, row_id)`, deduplicated on insert. + evidence: Vec<(String, String)>, + /// Keyed by `(bucket, workflow_id)` — the same composite key sqlite makes a + /// primary key and mongo matches on. + scores: HashMap<(String, String), Score>, + /// `(bucket, variant) -> parent`. + variants: HashMap<(String, String), String>, + episodes: Vec, + /// `(bucket, row_id)` to that attempt's steps, in execution order. + steps: HashMap<(String, String), Vec>, +} + +/// A ledger held in memory, which learns nothing across restarts. +/// +/// See the module note before using it for anything but tests. +#[derive(Clone, Default)] +pub struct MemoryLedger { + inner: Arc>, + scope: Option, +} + +impl MemoryLedger { + /// An empty ledger. + #[must_use] + pub fn new() -> Self { + Self::default() + } + + /// A handle onto the same store, scoped to one tenant. + #[must_use] + pub fn for_tenant(&self, scope: impl Into) -> Self { + Self { + inner: Arc::clone(&self.inner), + scope: Some(scope.into()), + } + } + + fn bucket(&self) -> String { + self.scope.clone().unwrap_or_default() + } + + /// A poisoned lock means a previous caller panicked mid-write. Every write + /// here is a single statement under the lock, so the data is intact; + /// refusing every later call would turn one panic into a dead loop — the + /// same reasoning as the sqlite backend's guard. + fn guard(&self) -> std::sync::MutexGuard<'_, Inner> { + self.inner + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + } + + /// This bucket plus global, the one read rule everywhere. + fn visible(&self, scope: Option<&str>) -> bool { + scope.is_none() || scope == self.scope.as_deref() + } +} + +#[async_trait] +impl Ledger for MemoryLedger { + fn scope(&self) -> Option<&str> { + self.scope.as_deref() + } + + async fn append(&self, row: &LedgerRow) -> Result { + let mut inner = self.guard(); + let id = format!("ldg_{:08}", inner.rows.len() + 1); + let bucket = self.bucket(); + inner.rows.push(( + bucket, + LedgerRow { + id: id.clone(), + ..row.clone() + }, + )); + Ok(id) + } + + async fn rows(&self, episode: &str) -> Result> { + let bucket = self.bucket(); + Ok(self + .guard() + .rows + .iter() + .filter(|(scope, r)| scope == &bucket && r.episode == episode) + .map(|(_, r)| r.clone()) + .collect()) + } + + async fn promote(&self, lesson: &Lesson, cites: &[String]) -> Result { + let mut inner = self.guard(); + let id = format!("les_{:08}", inner.lessons.len() + 1); + inner.lessons.push(Lesson { + id: id.clone(), + // The handle's scope, never the argument's. + scope_key: self.scope.clone(), + ..lesson.clone() + }); + for row_id in cites { + let edge = (id.clone(), row_id.clone()); + if !inner.evidence.contains(&edge) { + inner.evidence.push(edge); + } + } + Ok(id) + } + + async fn lessons(&self, kind: Option) -> Result> { + Ok(self + .guard() + .lessons + .iter() + .filter(|l| self.visible(l.scope_key.as_deref())) + .filter(|l| kind.is_none_or(|want| l.kind == want)) + .cloned() + .collect()) + } + + async fn evidence(&self, lesson_id: &str) -> Result> { + let inner = self.guard(); + let cited: Vec<&str> = inner + .evidence + .iter() + .filter(|(lesson, _)| lesson == lesson_id) + .map(|(_, row)| row.as_str()) + .collect(); + let bucket = self.bucket(); + Ok(inner + .rows + .iter() + .filter(|(scope, r)| scope == &bucket && cited.contains(&r.id.as_str())) + .map(|(_, r)| r.clone()) + .collect()) + } + + async fn score_lesson(&self, lesson_id: &str, helped: bool) -> Result<()> { + // Only what this handle can see: its bucket, or global. The ids reach + // here from model output (corroboration), and a tenant must not be + // able to move another tenant's score by naming its id. + let visible = |l: &&mut Lesson| { + l.scope_key.is_none() || l.scope_key.as_deref() == self.scope.as_deref() + }; + let mut inner = self.guard(); + if let Some(lesson) = inner + .lessons + .iter_mut() + .filter(visible) + .find(|l| l.id == lesson_id) + { + lesson.applied += 1; + lesson.helped += u32::from(helped); + } + Ok(()) + } + + async fn score_workflow(&self, workflow_id: &str, helped: bool) -> Result<()> { + let key = (self.bucket(), workflow_id.to_string()); + let mut inner = self.guard(); + let score = inner.scores.entry(key).or_default(); + score.applied += 1; + score.helped += u32::from(helped); + Ok(()) + } + + async fn workflow_score(&self, workflow_id: &str) -> Result { + Ok(self + .guard() + .scores + .get(&(self.bucket(), workflow_id.to_string())) + .copied() + .unwrap_or_default()) + } + + async fn link_variant(&self, parent: &str, variant: &str) -> Result<()> { + self.guard() + .variants + .entry((self.bucket(), variant.to_string())) + .or_insert_with(|| parent.to_string()); + Ok(()) + } + + async fn parent_of(&self, id: &str) -> Result> { + Ok(self + .guard() + .variants + .get(&(self.bucket(), id.to_string())) + .cloned()) + } + + async fn children_of(&self, id: &str) -> Result> { + let bucket = self.bucket(); + let inner = self.guard(); + let mut found: Vec = inner + .variants + .iter() + .filter(|((scope, _), parent)| scope == &bucket && parent.as_str() == id) + .map(|((_, variant), _)| variant.clone()) + .collect(); + // A HashMap has no order and `lineage` must read the same twice. + found.sort(); + Ok(found) + } + + async fn save_episode(&self, episode: &Episode) -> Result<()> { + let stored = Episode { + scope_key: self.scope.clone(), + ..episode.clone() + }; + let mut inner = self.guard(); + match inner.episodes.iter_mut().find(|e| e.id == episode.id) { + Some(existing) => { + // `started_at` and the scope are facts about creation, not + // progress, so an update leaves them alone — matching mongo's + // `$setOnInsert`. + let started = existing.started_at.clone(); + let scope = existing.scope_key.clone(); + *existing = Episode { + started_at: started, + scope_key: scope, + ..stored + }; + } + None => inner.episodes.push(stored), + } + Ok(()) + } + + async fn episode(&self, id: &str) -> Result> { + Ok(self + .guard() + .episodes + .iter() + .find(|e| e.id == id && e.scope_key.as_deref() == self.scope.as_deref()) + .cloned()) + } + + async fn episodes(&self, running_only: bool, page: super::Page) -> Result> { + let mut found: Vec = self + .guard() + .episodes + .iter() + .filter(|e| e.scope_key.as_deref() == self.scope.as_deref()) + .filter(|e| !running_only || e.status == super::EpisodeStatus::Running) + .cloned() + .collect(); + // Newest first, ids breaking ties — `Page` documents that order, the + // durable backends sort in the query, and this backend is the + // reference the conformance suite pins. Insertion order is the + // opposite end of the list. + found.sort_by(|a, b| b.updated_at.cmp(&a.updated_at).then(a.id.cmp(&b.id))); + Ok(page.apply(found)) + } + + async fn save_steps(&self, row_id: &str, steps: &[crate::execute::StepRecord]) -> Result<()> { + self.guard() + .steps + .insert((self.bucket(), row_id.to_string()), steps.to_vec()); + Ok(()) + } + + async fn steps(&self, row_id: &str) -> Result> { + Ok(self + .guard() + .steps + .get(&(self.bucket(), row_id.to_string())) + .cloned() + .unwrap_or_default()) + } +} + +/// Kept so the unused-import lint stays honest if the error type is ever needed +/// here: nothing in memory can fail, which is itself worth stating. +const _: Option = None; + +#[cfg(test)] +mod tests { + use super::*; + use crate::ledger::conformance; + + #[tokio::test] + async fn passes_the_conformance_suite() { + // The same cases both durable backends pass. That the trait is + // implementable in std alone is the point: a host writing a third + // backend has a complete example checked by the cases theirs will be. + conformance::run_all(&MemoryLedger::new()).await; + } + + #[tokio::test] + async fn passes_the_tenant_isolation_suite() { + let store = MemoryLedger::new(); + let a = store.for_tenant("user-a"); + let b = store.for_tenant("user-b"); + conformance::run_tenants(&store, &a, &b).await; + } + + #[tokio::test] + async fn a_scoped_handle_shares_the_store_rather_than_copying_it() { + // Two handles for the SAME tenant must see each other's writes — that + // is what "shares" means. Probing it across scopes would now fail by + // design, because rows carry the bucket that wrote them. + let store = MemoryLedger::new(); + let one = store.for_tenant("user-a"); + let two = store.for_tenant("user-a"); + one.append(&conformance::row("ep-shared", 1, "authored")) + .await + .expect("append"); + assert_eq!(two.rows("ep-shared").await.expect("rows").len(), 1); + assert!( + store.rows("ep-shared").await.expect("rows").is_empty(), + "and the global bucket is its own, not a union" + ); + } + + #[tokio::test] + async fn it_forgets_which_is_the_whole_point_of_the_name() { + // Not a limitation being tested around — the behaviour, pinned, so the + // difference from a durable backend is visible in the test names. + let first = MemoryLedger::new(); + first + .append(&conformance::row("ep-gone", 1, "authored")) + .await + .expect("append"); + assert_eq!(first.rows("ep-gone").await.expect("rows").len(), 1); + + let second = MemoryLedger::new(); + assert!( + second.rows("ep-gone").await.expect("rows").is_empty(), + "a new ledger is a new memory; nothing crosses between them" + ); + } +} diff --git a/crates/adaptive/src/ledger/mod.rs b/crates/adaptive/src/ledger/mod.rs new file mode 100644 index 0000000..8ad38e3 --- /dev/null +++ b/crates/adaptive/src/ledger/mod.rs @@ -0,0 +1,555 @@ +//! Everything that spans runs. +//! +//! The engine's own [`tinyflows::store`] holds workflows, run records, notes and +//! proposals — all of it *about one run* or one document. This holds the other +//! half: what was tried across attempts, what generalised out of that, and +//! which stored procedures have actually earned their place. +//! +//! Kept as a separate trait rather than as more methods on `WorkflowStore`, for +//! two reasons that are really one. The engine's store is upstream's type and a +//! merge should never contend with our additions; and the boundary this project +//! rests on — *the engine may know about one run, anything that spans runs is +//! ours* — is worth having in the type system rather than in a document. +//! +//! Three implementations ship. `sqlite` and `mongo` are behind features, +//! because the choice is the host's and a deployment that wants one should not +//! build the other's driver. [`memory`] is always compiled, needs no driver, +//! and **forgets everything on restart** — it exists so the crate is usable the +//! moment it is added, and it is never selected for you. +//! +//! All three are checked by the same conformance suite ([`conformance`]), so +//! "it works on sqlite" cannot quietly mean "it works only on sqlite" — and so +//! a host writing a fourth backend has a std-only reference implementation +//! checked by the cases theirs will be. + +use async_trait::async_trait; +use serde::{Deserialize, Serialize}; + +pub mod memory; +#[cfg(feature = "mongo")] +pub mod mongo; +#[cfg(feature = "sqlite")] +pub mod sqlite; + +pub mod conformance; + +/// What went wrong reaching the ledger. +/// +/// Deliberately coarse. A caller can retry or give up; it cannot repair a +/// backend, so a taxonomy of driver errors would be detail nobody branches on. +#[derive(Debug, thiserror::Error)] +pub enum LedgerError { + /// The backend refused or was unreachable. + #[error("ledger backend: {0}")] + Backend(String), + /// Something was stored that no longer parses — a schema moved under us. + #[error("ledger holds a row it cannot read: {0}")] + Corrupt(String), +} + +/// Convenience alias for ledger results. +pub type Result = std::result::Result; + +/// One attempt, recorded as it finishes. +/// +/// The unit is an *attempt*, not a run: a single episode may run three +/// workflows and author a fourth, and the exclusion list that stops attempt +/// four repeating attempt two is built from these rows. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +pub struct LedgerRow { + /// Assigned by the backend on append; empty when not yet stored. + #[serde(default)] + pub id: String, + /// The episode this attempt belongs to — one goal, many attempts. + pub episode: String, + /// 1-based, so a row reads the way a person counts. + pub attempt: u32, + /// [`crate::contracts::Approach::signature`]. What the exclusion list is + /// built from, and what a lesson is keyed against. + pub approach_sig: String, + /// The approach in a sentence, for a human reading the trail. + #[serde(default)] + pub approach_desc: String, + /// The workflow that ran, when one did. Absent for an authoring attempt + /// that never reached a graph. + #[serde(default)] + pub workflow_id: Option, + /// What happened, in the judge's words. + #[serde(default)] + pub outcome: String, + /// Why it fell short. Empty when it did not. + #[serde(default)] + pub cause: String, + /// What it cost, in whatever unit the host counts. Zero is "not measured", + /// which is honest; a made-up estimate is not. + #[serde(default)] + pub cost_usd: f64, + /// RFC 3339. Supplied by the caller so a frozen clock can drive tests. + pub at: String, + /// Whether the judge called this attempt satisfied. + /// + /// A field rather than `outcome == "satisfied"`: that string match works + /// and is one reworded line away from silently reporting every episode as + /// failed. + #[serde(default)] + pub satisfied: bool, + /// Whether it got closer than the state before it. + /// + /// Stored because the stall rule is computed from it, and an episode a + /// restarted process cannot recompute is an episode it has to start over. + #[serde(default)] + pub advanced: bool, +} + +/// The four kinds of thing an episode can teach. +/// +/// A closed set because retrieval filters on it and a prompt asks for it; an +/// open one becomes a synonym pile within a week. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum LessonKind { + /// X works where Y fails. Lands in the next plan's approach. + Strategy, + /// A limit no approach here can cross. Rules approaches out. + Constraint, + /// A way this silently looks done when it is not. Becomes something the + /// next run checks for. + FailureMode, + /// An estimate that was systematically wrong, and by how much. + Calibration, +} + +impl LessonKind { + /// Reads a model's answer, defaulting to the least actionable kind. + /// + /// Unrecognised becomes `Strategy` rather than an error: a lesson with a + /// misfiled kind is still worth keeping, and refusing the write loses it. + #[must_use] + pub fn parse(raw: &str) -> Self { + match raw.trim().to_ascii_lowercase().as_str() { + "constraint" => Self::Constraint, + "failure_mode" => Self::FailureMode, + "calibration" => Self::Calibration, + _ => Self::Strategy, + } + } +} + +/// Something a *different* task could act on. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +pub struct Lesson { + /// Assigned by the backend on promote. + #[serde(default)] + pub id: String, + /// Which kind, so retrieval can filter. + pub kind: LessonKind, + /// What decides whether this is ever found again — and the easiest thing + /// to get wrong in both directions. It must describe the *class* of + /// situation: "a CPU-bound scan over ~1M items with a sub-100ms target", + /// never "Project Euler 14" (matches once, never again) and never "a task + /// that needs to be fast" (matches everything, says nothing). + pub trigger: String, + /// Why it is true. + #[serde(default)] + pub mechanism: String, + /// What to do about it. + pub claim: String, + /// How many times it was put in front of a planner. + #[serde(default)] + pub applied: u32, + /// How many of those ended satisfied. + #[serde(default)] + pub helped: u32, + /// Whose lesson this is. `None` is global — visible to everyone. + /// + /// Never set by a caller: [`Ledger::promote`] stamps it from the handle's + /// own [`scope`](Ledger::scope). A lesson's `trigger` and `claim` are free + /// text drawn from one tenant's episode and can name their repositories, + /// paths and internals, so which tenant owns it is not a decision a model + /// or a caller gets to make. + #[serde(default)] + pub scope_key: Option, +} + +impl Lesson { + /// Both numbers are kept rather than a rate, because 1/1 and 40/40 are the + /// same rate and are not the same evidence. This is for ordering only. + #[must_use] + pub fn help_rate(&self) -> f64 { + if self.applied == 0 { + 0.0 + } else { + f64::from(self.helped) / f64::from(self.applied) + } + } +} + +/// How a stored workflow has actually performed. +/// +/// Not on `WorkflowRecord`: a score is a fact that spans runs, and the engine's +/// record is a fact about one document. Keyed by workflow id on our side. +#[derive(Debug, Clone, Copy, Default, Serialize, Deserialize, PartialEq, Eq)] +pub struct Score { + /// Times this workflow was chosen and run. + pub applied: u32, + /// Times that ended satisfied. + pub helped: u32, +} + +impl Score { + /// Both numbers are kept rather than a rate, because 1/1 and 40/40 are the + /// same rate and are not the same evidence. This is for ordering only. + #[must_use] + pub fn help_rate(&self) -> f64 { + if self.applied == 0 { + 0.0 + } else { + f64::from(self.helped) / f64::from(self.applied) + } + } +} + +/// How far up a variant chain [`Ledger::lineage`] will walk before giving up. +pub const MAX_LINEAGE_DEPTH: usize = 8; + +/// How many members of one family [`Ledger::lineage`] will return. +pub const MAX_FAMILY: usize = 64; + +/// The exclusion list, from rows already in hand. +/// +/// [`Ledger::tried`] is this over a fresh read, which is the right shape for a +/// caller that wants only the signatures. A caller that also renders the +/// history — [`crate::intake::decide`] does both — reads the rows once and +/// calls this, rather than paying for the same query twice per attempt. +/// +/// First-seen order, deduplicated. Order matters because it is rendered into a +/// prompt, and a list that reshuffles between attempts is one a planner cannot +/// be reasoned about against. +#[must_use] +pub fn signatures(rows: &[LedgerRow]) -> Vec { + let mut seen: Vec = Vec::new(); + for row in rows { + if !seen.contains(&row.approach_sig) { + seen.push(row.approach_sig.clone()); + } + } + seen +} + +/// A window onto a list that grows without bound. +/// +/// Only [`Ledger::episodes`] takes one. An episode's *rows* are bounded by +/// [`crate::contracts::Budget::attempts`] — a dozen — so paging them would be +/// ceremony around a list that cannot get long. A tenant's episodes accumulate +/// forever, which is a different thing. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Page { + /// How many at most. + pub limit: usize, + /// How many to skip, newest first. + pub offset: usize, +} + +impl Page { + /// Everything. What the loop's own recovery pass wants. + pub const ALL: Self = Self { + limit: usize::MAX, + offset: 0, + }; + + /// The first `n`, from the top. + #[must_use] + pub fn first(n: usize) -> Self { + Self { + limit: n, + offset: 0, + } + } + + /// Apply to an already-ordered list. + /// + /// Applied in the backend after ordering rather than pushed into each + /// query, because two of the three have no query language and the third + /// would then be the only one whose paging could disagree. + #[must_use] + pub fn apply(self, mut items: Vec) -> Vec { + if self.offset >= items.len() { + return Vec::new(); + } + items.drain(..self.offset); + items.truncate(self.limit); + items + } +} + +/// How an episode ended, or that it has not. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case", tag = "state", content = "reason")] +pub enum EpisodeStatus { + /// Still going. + Running, + /// The goal was met. + Satisfied, + /// Stopped without success, for the reason + /// [`crate::closing::Next::StandDown`] gave. + StoodDown(String), +} + +/// One goal, from the first attempt to whatever ended it. +/// +/// The loop's own checkpoint, and the thing that makes an episode survive the +/// process running it. Everything here is either unrecoverable from the rows +/// (the goal) or expensive and error-prone to recompute (the counters), which +/// is the test for what belongs on it. +/// +/// Not the engine's `Checkpointer`: that holds mid-run superstep state for +/// `engine::resume`, which this crate deliberately does not use. This is +/// between runs, which is the boundary the whole crate sits on. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct Episode { + /// The caller's id. Minted by whoever owns episodes — a service, a CLI — + /// and opaque here. + pub id: String, + /// What was asked. Unrecoverable from the rows, so a restart without it + /// cannot continue. + pub goal: crate::contracts::Goal, + /// Whose it is, stamped from the handle's scope on write. + #[serde(default)] + pub scope_key: Option, + /// Where it is. + pub status: EpisodeStatus, + /// Attempts spent. + #[serde(default)] + pub attempt: u32, + /// Consecutive attempts that made no progress. + #[serde(default)] + pub stalled: u32, + /// RFC 3339, caller-supplied. + pub started_at: String, + /// RFC 3339, caller-supplied. + pub updated_at: String, +} + +/// Everything that spans runs. +/// +/// Every method is fallible and none of them panics on an absent row: a missing +/// lesson or an unknown workflow is an empty answer, not an error. A loop that +/// cannot read its own history should degrade to a first-time run, never stop. +#[async_trait] +pub trait Ledger: Send + Sync { + /// Whose knowledge this handle reads and writes. `None` is the global + /// bucket. + /// + /// The scope lives on the handle rather than on every method because the + /// failure it prevents is *forgetting to pass it*. One `for_tenant` at the + /// edge of a request is a thing a reviewer can see; six scope arguments + /// threaded through intake and closing is a thing that goes wrong once and + /// leaks one tenant's lessons into another's prompt. + /// + /// One rule, everywhere: + /// + /// * **writes** go to this handle's bucket; + /// * **reads** return this handle's bucket plus the global one. + /// + /// An unscoped handle's bucket *is* global, so a single-tenant deployment + /// that never calls `for_tenant` reads back exactly what it wrote and + /// nothing changes for it. + /// + /// Episode rows are not affected: they are already keyed by episode, and + /// [`tried`](Ledger::tried) reads one episode at a time. + fn scope(&self) -> Option<&str> { + None + } + + /// Record one finished attempt. Returns the assigned id. + async fn append(&self, row: &LedgerRow) -> Result; + + /// Every attempt in one episode, oldest first. + async fn rows(&self, episode: &str) -> Result>; + + /// The approach signatures already spent on this episode. + /// + /// This is the exclusion list, and it is the reason the ledger exists at + /// all: without it a planner re-proposes attempt two's idea at attempt four + /// in slightly different words, and the run pays twice for the same dead + /// end. + async fn tried(&self, episode: &str) -> Result> { + Ok(signatures(&self.rows(episode).await?)) + } + + /// Keep a lesson, citing the rows it was drawn from. + /// + /// The stored lesson's [`scope_key`](Lesson::scope_key) is this handle's + /// [`scope`](Ledger::scope), whatever the argument says. + /// + /// A claim with no rows behind it is a guess, so the citation is part of + /// the call rather than an optional extra. + async fn promote(&self, lesson: &Lesson, cites: &[String]) -> Result; + + /// Lessons in scope, optionally of one kind. + async fn lessons(&self, kind: Option) -> Result>; + + /// The rows a lesson cited, for a reader arguing with it. + async fn evidence(&self, lesson_id: &str) -> Result>; + + /// Note that a lesson was shown to a planner, and whether that run ended + /// satisfied. Both counters move; only the second is conditional. + async fn score_lesson(&self, lesson_id: &str, helped: bool) -> Result<()>; + + /// The same for a workflow. This is the missing rung: without it nothing + /// distinguishes a procedure that has worked forty times from one that has + /// never run, and a promotion gate has no evidence to read. + async fn score_workflow(&self, workflow_id: &str, helped: bool) -> Result<()>; + + /// How a workflow has performed. Unknown ids answer `Score::default()` + /// rather than erroring — a workflow nobody has run yet is 0/0, not a bug. + async fn workflow_score(&self, workflow_id: &str) -> Result; + + /// Record that `variant` was derived from `parent`. + /// + /// Lineage lives here rather than on `WorkflowRecord` for the usual reason: + /// the engine's record is a fact about one document, and *this graph came + /// from that one after it fell short* is a fact that spans runs. It is also + /// what stops a repaired family from filling the catalogue with six + /// near-identical rows a planner has to choose between blindly. + /// + /// Idempotent: re-linking the same pair is a no-op, because a repair that + /// converges on an existing variant id will try. + async fn link_variant(&self, parent: &str, variant: &str) -> Result<()>; + + /// What `id` was derived from, if anything. + async fn parent_of(&self, id: &str) -> Result>; + + /// What was derived directly from `id`. + async fn children_of(&self, id: &str) -> Result>; + + /// Write an episode, creating or replacing it. + /// + /// The [`scope_key`](Episode::scope_key) stored is this handle's, whatever + /// the argument says — the same rule as [`promote`](Ledger::promote), for + /// the same reason. + async fn save_episode(&self, episode: &Episode) -> Result<()>; + + /// Read one episode, if this handle's scope can see it. + /// + /// This is resume: a process that restarts mid-episode reads the goal and + /// the counters back and carries on, rather than starting the goal over + /// with a ledger that says it has already been attempted four times. + async fn episode(&self, id: &str) -> Result>; + + /// Every episode in this handle's scope, optionally filtered by state. + /// + /// `Running` on boot is the recovery list. Without it a deploy silently + /// abandons every episode that was in flight — the rows stay, nothing ever + /// looks at them again, and the goal is never answered. + async fn episodes(&self, running_only: bool, page: Page) -> Result>; + + /// Keep an attempt's per-node record, addressed by its ledger row. + /// + /// The transcript: what each node emitted, whether it errored, how long it + /// took, and which of its bindings resolved to null. The judge is shown a + /// bounded projection of it and the ledger row holds a sentence; this is + /// the detail behind both, and without it "show me what that attempt did" + /// has nothing to answer with. + /// + /// **One record per step, never one blob per attempt.** A `loop` node + /// produces a step per iteration, and at + /// [`RECORD_BUDGET`](crate::execute::RECORD_BUDGET) that reaches megabytes + /// — past what a Mongo document may hold. A blob would work on sqlite, work + /// in testing, and fail in production on exactly the runs most worth + /// reading. + async fn save_steps(&self, row_id: &str, steps: &[crate::execute::StepRecord]) -> Result<()>; + + /// One attempt's per-node record, in execution order. + async fn steps(&self, row_id: &str) -> Result>; + + /// Every workflow in `id`'s family, **root first**, including `id`. + /// + /// Works from any member: it walks up to the root, then breadth-first down. + /// Both walks are bounded, so a cycle written by a buggy caller costs a + /// truncated answer rather than a loop that never returns — the ledger is + /// read on the hot path of every attempt, and a hang there stops + /// everything. + async fn lineage(&self, id: &str) -> Result> { + let mut root = id.to_string(); + for _ in 0..MAX_LINEAGE_DEPTH { + match self.parent_of(&root).await? { + Some(parent) if parent != root => root = parent, + _ => break, + } + } + let mut family = vec![root]; + let mut next = 0; + while next < family.len() && family.len() < MAX_FAMILY { + for child in self.children_of(&family[next]).await? { + if !family.contains(&child) { + family.push(child); + } + } + next += 1; + } + Ok(family) + } +} + +#[cfg(test)] +mod signature_tests { + use super::{LedgerRow, signatures}; + + fn row(attempt: u32, sig: &str) -> LedgerRow { + LedgerRow { + id: format!("r{attempt}"), + episode: "ep".into(), + attempt, + approach_sig: sig.into(), + approach_desc: String::new(), + workflow_id: None, + outcome: String::new(), + cause: String::new(), + cost_usd: 0.0, + at: "2026-01-01T00:00:00Z".into(), + satisfied: false, + advanced: false, + } + } + + #[test] + fn an_approach_tried_twice_appears_once() { + let got = signatures(&[ + row(1, "selected:weekly"), + row(2, "authored:aaa"), + row(3, "selected:weekly"), + ]); + assert_eq!(got, vec!["selected:weekly", "authored:aaa"]); + } + + #[test] + fn first_seen_order_is_kept() { + // It is rendered into a prompt, and a list that reshuffles between + // attempts is one a planner cannot be reasoned about against. + let got = signatures(&[row(1, "c"), row(2, "a"), row(3, "b")]); + assert_eq!(got, vec!["c", "a", "b"]); + } + + #[test] + fn no_rows_is_an_empty_list_rather_than_a_surprise() { + assert!(signatures(&[]).is_empty()); + } + + #[tokio::test] + async fn the_trait_method_agrees_with_the_function_it_now_calls() { + // `tried` is this over a fresh read. If the two ever disagree, one + // caller's exclusion list is not the other's. + use super::Ledger; + let ledger = super::memory::MemoryLedger::new(); + for (attempt, sig) in [ + (1u32, "selected:weekly"), + (2, "authored:aaa"), + (3, "selected:weekly"), + ] { + ledger.append(&row(attempt, sig)).await.expect("append"); + } + let rows = ledger.rows("ep").await.expect("rows"); + assert_eq!(ledger.tried("ep").await.expect("tried"), signatures(&rows)); + } +} diff --git a/crates/adaptive/src/ledger/mongo.rs b/crates/adaptive/src/ledger/mongo.rs new file mode 100644 index 0000000..0ab4b77 --- /dev/null +++ b/crates/adaptive/src/ledger/mongo.rs @@ -0,0 +1,560 @@ +//! A [`Ledger`] on MongoDB, for a hosted deployment. +//! +//! Four collections mirroring the sqlite tables, and the same conformance suite +//! runs against both. Where the two differ is concurrency: this one is a real +//! async driver and several loops may write the same ledger at once, so the two +//! counter updates use `$inc` rather than read-modify-write. A read-modify-write +//! here loses increments under exactly the load a hosted deployment has. + +use async_trait::async_trait; +use mongodb::bson::{Document, doc}; +use mongodb::options::{IndexOptions, ReturnDocument}; +use mongodb::{Client, Collection, Database, IndexModel}; + +use super::{ + Episode, EpisodeStatus, Ledger, LedgerError, LedgerRow, Lesson, LessonKind, Result, Score, +}; + +impl From for LedgerError { + fn from(err: mongodb::error::Error) -> Self { + Self::Backend(err.to_string()) + } +} + +impl From for LedgerError { + fn from(err: mongodb::bson::ser::Error) -> Self { + Self::Corrupt(err.to_string()) + } +} + +impl From for LedgerError { + fn from(err: mongodb::bson::de::Error) -> Self { + Self::Corrupt(err.to_string()) + } +} + +const ROWS: &str = "ledger_rows"; +const LESSONS: &str = "lessons"; +const EVIDENCE: &str = "lesson_evidence"; +const SCORES: &str = "workflow_scores"; +const VARIANTS: &str = "variants"; +const EPISODES: &str = "episodes"; +const STEPS: &str = "attempt_steps"; +const COUNTERS: &str = "counters"; + +/// A ledger backed by a MongoDB database. +pub struct MongoLedger { + db: Database, + scope: Option, +} + +impl MongoLedger { + /// Connect to `uri` and use the database named `database`. + /// + /// # Errors + /// When the URI is malformed, the server is unreachable, or an index + /// cannot be created. + pub async fn connect(uri: &str, database: &str) -> Result { + let client = Client::with_uri_str(uri).await?; + Self::with_database(client.database(database)).await + } + + /// Use an already-connected database. For a host that manages its own + /// client and pool. + /// + /// # Errors + /// When an index cannot be created. + pub async fn with_database(db: Database) -> Result { + let store = Self { db, scope: None }; + store.ensure_indexes().await?; + Ok(store) + } + + /// A handle onto the same database, scoped to one tenant. + /// + /// Cheap — a `Database` is a handle over a shared pool. Construct one per + /// request at the edge of the service and hand it to the loop; everything + /// downstream reads and writes the right bucket without knowing a tenant + /// exists. + #[must_use] + pub fn for_tenant(&self, scope: impl Into) -> Self { + Self { + db: self.db.clone(), + scope: Some(scope.into()), + } + } + + /// This handle's bucket, as stored: `""` for global. Stored as a present + /// empty string rather than an absent field, so the upsert filter on + /// workflow scores matches one document instead of creating a new one each + /// time — the same reason sqlite makes the column NOT NULL. + fn bucket(&self) -> &str { + self.scope.as_deref().unwrap_or_default() + } + + async fn ensure_indexes(&self) -> Result<()> { + // Ordered by `seq`, never by timestamp: two attempts finishing in the + // same second would otherwise read back in an arbitrary order, which + // silently reorders the exclusion list. + self.rows() + .create_index( + IndexModel::builder() + .keys(doc! { "episode": 1, "seq": 1 }) + .build(), + ) + .await?; + self.evidence() + .create_index(IndexModel::builder().keys(doc! { "lesson_id": 1 }).build()) + .await?; + // The score key is (scope_key, workflow_id) since tenancy landed. The + // old single-field unique index would reject the same workflow id in a + // second tenant's bucket, so it is dropped if present — failure means + // it never existed, which is the ordinary case. + let _ = self.scores().drop_index("workflow_id_1").await; + let unique = IndexOptions::builder().unique(true).build(); + self.scores() + .create_index( + IndexModel::builder() + .keys(doc! { "scope_key": 1, "workflow_id": 1 }) + .options(unique) + .build(), + ) + .await?; + Ok(()) + } + + fn rows(&self) -> Collection { + self.db.collection(ROWS) + } + fn lessons_c(&self) -> Collection { + self.db.collection(LESSONS) + } + fn evidence(&self) -> Collection { + self.db.collection(EVIDENCE) + } + fn scores(&self) -> Collection { + self.db.collection(SCORES) + } + fn variants(&self) -> Collection { + self.db.collection(VARIANTS) + } + fn episodes_c(&self) -> Collection { + self.db.collection(EPISODES) + } + fn steps_c(&self) -> Collection { + self.db.collection(STEPS) + } + + /// The next value in a named sequence. + /// + /// A counter document rather than a `count()` of the collection: counting + /// races with a concurrent insert and hands two writers the same number, + /// while `findAndModify` with `$inc` is atomic on the server. + async fn next_seq(&self, name: &str) -> Result { + let updated = self + .db + .collection::(COUNTERS) + .find_one_and_update(doc! { "_id": name }, doc! { "$inc": { "seq": 1 } }) + .upsert(true) + .return_document(ReturnDocument::After) + .await?; + Ok(updated.and_then(|d| d.get_i64("seq").ok()).unwrap_or(1)) + } +} + +fn kind_str(kind: LessonKind) -> &'static str { + match kind { + LessonKind::Strategy => "strategy", + LessonKind::Constraint => "constraint", + LessonKind::FailureMode => "failure_mode", + LessonKind::Calibration => "calibration", + } +} + +fn as_u32(doc: &Document, key: &str) -> u32 { + doc.get_i64(key) + .ok() + .and_then(|v| u32::try_from(v).ok()) + .or_else(|| doc.get_i32(key).ok().and_then(|v| u32::try_from(v).ok())) + .unwrap_or(0) +} + +fn text(doc: &Document, key: &str) -> String { + doc.get_str(key).unwrap_or_default().to_string() +} + +fn read_row(doc: &Document) -> LedgerRow { + LedgerRow { + id: text(doc, "_id"), + episode: text(doc, "episode"), + attempt: as_u32(doc, "attempt"), + approach_sig: text(doc, "approach_sig"), + approach_desc: text(doc, "approach_desc"), + // An absent key and a stored null are the same thing to a reader. + workflow_id: doc.get_str("workflow_id").ok().map(ToString::to_string), + outcome: text(doc, "outcome"), + cause: text(doc, "cause"), + cost_usd: doc.get_f64("cost_usd").unwrap_or(0.0), + at: text(doc, "at"), + satisfied: doc.get_bool("satisfied").unwrap_or(false), + advanced: doc.get_bool("advanced").unwrap_or(false), + } +} + +fn read_episode(doc: &Document) -> Result { + let scope = text(doc, "scope_key"); + Ok(Episode { + id: text(doc, "_id"), + goal: serde_json::from_str(&text(doc, "goal")) + .map_err(|e| LedgerError::Corrupt(e.to_string()))?, + scope_key: (!scope.is_empty()).then_some(scope), + status: serde_json::from_str(&text(doc, "status")) + .map_err(|e| LedgerError::Corrupt(e.to_string()))?, + attempt: as_u32(doc, "attempt"), + stalled: as_u32(doc, "stalled"), + started_at: text(doc, "started_at"), + updated_at: text(doc, "updated_at"), + }) +} + +#[async_trait] +impl Ledger for MongoLedger { + fn scope(&self) -> Option<&str> { + self.scope.as_deref() + } + + async fn append(&self, row: &LedgerRow) -> Result { + let seq = self.next_seq(ROWS).await?; + let id = format!("ldg_{seq:08}"); + self.rows() + .insert_one(doc! { + "_id": &id, + "episode": &row.episode, + "attempt": i64::from(row.attempt), + "approach_sig": &row.approach_sig, + "approach_desc": &row.approach_desc, + "workflow_id": row.workflow_id.clone(), + "outcome": &row.outcome, + "cause": &row.cause, + "cost_usd": row.cost_usd, + "at": &row.at, + "satisfied": row.satisfied, + "advanced": row.advanced, + "scope_key": self.bucket(), + "seq": seq, + }) + .await?; + Ok(id) + } + + async fn rows(&self, episode: &str) -> Result> { + let mut cursor = self + .rows() + .find(doc! { "episode": episode, "scope_key": self.bucket() }) + .sort(doc! { "seq": 1 }) + .await?; + let mut out = Vec::new(); + while cursor.advance().await? { + out.push(read_row(&cursor.deserialize_current()?)); + } + Ok(out) + } + + async fn promote(&self, lesson: &Lesson, cites: &[String]) -> Result { + let seq = self.next_seq(LESSONS).await?; + let id = format!("les_{seq:08}"); + self.lessons_c() + .insert_one(doc! { + "_id": &id, + "kind": kind_str(lesson.kind), + "trigger": &lesson.trigger, + "mechanism": &lesson.mechanism, + "claim": &lesson.claim, + "applied": i64::from(lesson.applied), + "helped": i64::from(lesson.helped), + // The handle's, never the argument's. + "scope_key": self.bucket(), + "seq": seq, + }) + .await?; + for row_id in cites { + // Upsert on the pair so re-promoting the same citation is a no-op + // rather than a duplicate edge. + self.evidence() + .update_one( + doc! { "lesson_id": &id, "row_id": row_id }, + doc! { "$setOnInsert": { "lesson_id": &id, "row_id": row_id } }, + ) + .upsert(true) + .await?; + } + Ok(id) + } + + async fn lessons(&self, kind: Option) -> Result> { + // This bucket plus global. An unscoped handle's bucket is global, so + // the two halves coincide and it sees exactly what it wrote. `null` is + // in the set because `$in` only matches a *missing* field when the + // array contains null — and a lesson written before scoping existed + // has no field at all; those read as global, which is what they were. + let mine = doc! { "$in": [self.bucket(), "", mongodb::bson::Bson::Null] }; + let filter = match kind { + Some(want) => doc! { "kind": kind_str(want), "scope_key": mine }, + None => doc! { "scope_key": mine }, + }; + let mut cursor = self + .lessons_c() + .find(filter) + .sort(doc! { "seq": 1 }) + .await?; + let mut out = Vec::new(); + while cursor.advance().await? { + let d = cursor.deserialize_current()?; + out.push(Lesson { + id: text(&d, "_id"), + kind: LessonKind::parse(&text(&d, "kind")), + trigger: text(&d, "trigger"), + mechanism: text(&d, "mechanism"), + claim: text(&d, "claim"), + applied: as_u32(&d, "applied"), + helped: as_u32(&d, "helped"), + scope_key: Some(text(&d, "scope_key")).filter(|s| !s.is_empty()), + }); + } + Ok(out) + } + + async fn evidence(&self, lesson_id: &str) -> Result> { + let mut cursor = self + .evidence() + .find(doc! { "lesson_id": lesson_id }) + .await?; + let mut ids = Vec::new(); + while cursor.advance().await? { + ids.push(text(&cursor.deserialize_current()?, "row_id")); + } + if ids.is_empty() { + return Ok(Vec::new()); + } + let mut found = self + .rows() + .find(doc! { "_id": { "$in": ids }, "scope_key": self.bucket() }) + .sort(doc! { "seq": 1 }) + .await?; + let mut out = Vec::new(); + while found.advance().await? { + out.push(read_row(&found.deserialize_current()?)); + } + Ok(out) + } + + async fn score_lesson(&self, lesson_id: &str, helped: bool) -> Result<()> { + // Constrained to what this handle can see — the id arrives from model + // output, and naming another tenant's lesson must not move its score. + self.lessons_c() + .update_one( + doc! { "_id": lesson_id, + "scope_key": { "$in": [self.bucket(), "", mongodb::bson::Bson::Null] } }, + doc! { "$inc": { "applied": 1_i64, "helped": i64::from(helped) } }, + ) + .await?; + Ok(()) + } + + async fn score_workflow(&self, workflow_id: &str, helped: bool) -> Result<()> { + // `$inc` on an upsert, not read-modify-write: several loops may finish + // the same workflow at once, and a lost increment is a promotion gate + // reading the wrong evidence. + self.scores() + .update_one( + doc! { "workflow_id": workflow_id, "scope_key": self.bucket() }, + doc! { "$inc": { "applied": 1_i64, "helped": i64::from(helped) } }, + ) + .upsert(true) + .await?; + Ok(()) + } + + async fn workflow_score(&self, workflow_id: &str) -> Result { + let found = self + .scores() + .find_one(doc! { "workflow_id": workflow_id, "scope_key": self.bucket() }) + .await?; + Ok(found.map_or_else(Score::default, |d| Score { + applied: as_u32(&d, "applied"), + helped: as_u32(&d, "helped"), + })) + } + + async fn link_variant(&self, parent: &str, variant: &str) -> Result<()> { + self.variants() + .update_one( + doc! { "scope_key": self.bucket(), "variant": variant }, + doc! { "$setOnInsert": { + "scope_key": self.bucket(), "variant": variant, "parent": parent + } }, + ) + .upsert(true) + .await?; + Ok(()) + } + + async fn parent_of(&self, id: &str) -> Result> { + let found = self + .variants() + .find_one(doc! { "scope_key": self.bucket(), "variant": id }) + .await?; + Ok(found.map(|d| text(&d, "parent")).filter(|p| !p.is_empty())) + } + + async fn save_episode(&self, episode: &Episode) -> Result<()> { + let goal = serde_json::to_string(&episode.goal) + .map_err(|e| LedgerError::Corrupt(e.to_string()))?; + let status = serde_json::to_string(&episode.status) + .map_err(|e| LedgerError::Corrupt(e.to_string()))?; + self.episodes_c() + .update_one( + doc! { "_id": &episode.id }, + doc! { + "$set": { + "goal": goal, + "status": status, + "attempt": i64::from(episode.attempt), + "stalled": i64::from(episode.stalled), + "updated_at": &episode.updated_at, + }, + // Set once: the handle's scope and the first timestamp are + // facts about the episode's creation, not its progress. + "$setOnInsert": { + "scope_key": self.bucket(), + "started_at": &episode.started_at, + }, + }, + ) + .upsert(true) + .await?; + Ok(()) + } + + async fn episode(&self, id: &str) -> Result> { + let found = self + .episodes_c() + .find_one(doc! { "_id": id, "scope_key": self.bucket() }) + .await?; + found.as_ref().map(read_episode).transpose() + } + + async fn save_steps(&self, row_id: &str, steps: &[crate::execute::StepRecord]) -> Result<()> { + // Replace, not overlay: a shorter re-save must not leave the old tail + // behind it, or `steps()` returns two attempts stitched together. + self.steps_c() + .delete_many(doc! { "scope_key": self.bucket(), "row_id": row_id }) + .await?; + // A document per step. One per attempt would exceed the 16 MB cap on a + // looped graph, and would do it only in production. + for (seq, step) in steps.iter().enumerate() { + let seq = i64::try_from(seq).unwrap_or(i64::MAX); + self.steps_c() + .update_one( + doc! { "scope_key": self.bucket(), "row_id": row_id, "seq": seq }, + doc! { "$set": { + "node_id": &step.node_id, + "status": serde_json::to_string(&step.status) + .map_err(|e| LedgerError::Corrupt(e.to_string()))? + .trim_matches('"'), + "output": serde_json::to_string(&step.output) + .map_err(|e| LedgerError::Corrupt(e.to_string()))?, + "duration_ms": i64::try_from(step.duration_ms).unwrap_or(i64::MAX), + "null_bindings": serde_json::to_string(&step.null_bindings) + .map_err(|e| LedgerError::Corrupt(e.to_string()))?, + } }, + ) + .upsert(true) + .await?; + } + Ok(()) + } + + async fn steps(&self, row_id: &str) -> Result> { + let mut cursor = self + .steps_c() + .find(doc! { "scope_key": self.bucket(), "row_id": row_id }) + .sort(doc! { "seq": 1 }) + .await?; + let mut out = Vec::new(); + while cursor.advance().await? { + let d = cursor.deserialize_current()?; + out.push(crate::execute::StepRecord { + node_id: text(&d, "node_id"), + status: if text(&d, "status") == "error" { + crate::execute::StepOutcome::Error + } else { + crate::execute::StepOutcome::Success + }, + output: serde_json::from_str(&text(&d, "output")) + .map_err(|e| LedgerError::Corrupt(e.to_string()))?, + duration_ms: u64::from(as_u32(&d, "duration_ms")), + null_bindings: serde_json::from_str(&text(&d, "null_bindings")) + .map_err(|e| LedgerError::Corrupt(e.to_string()))?, + }); + } + Ok(out) + } + + async fn episodes(&self, running_only: bool, page: super::Page) -> Result> { + let mut cursor = self + .episodes_c() + .find(doc! { "scope_key": self.bucket() }) + .sort(doc! { "updated_at": -1, "_id": 1 }) + .await?; + let mut out = Vec::new(); + while cursor.advance().await? { + let episode = read_episode(&cursor.deserialize_current()?)?; + if !running_only || episode.status == EpisodeStatus::Running { + out.push(episode); + } + } + Ok(page.apply(out)) + } + + async fn children_of(&self, id: &str) -> Result> { + let mut cursor = self + .variants() + .find(doc! { "scope_key": self.bucket(), "parent": id }) + .sort(doc! { "variant": 1 }) + .await?; + let mut out = Vec::new(); + while cursor.advance().await? { + out.push(text(&cursor.deserialize_current()?, "variant")); + } + Ok(out) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::ledger::conformance; + + /// Runs the same suite the sqlite backend passes, against a real server. + /// + /// Ignored by default: it needs one. Point `ADAPTIVE_MONGO_URI` at a + /// throwaway database and run with `--ignored`. Skipping silently when the + /// variable is absent would let this rot unnoticed, so the case is + /// `#[ignore]` and visible in the run summary instead. + #[tokio::test] + #[ignore = "needs a MongoDB server; set ADAPTIVE_MONGO_URI"] + async fn passes_the_conformance_suite() { + let uri = std::env::var("ADAPTIVE_MONGO_URI").expect("ADAPTIVE_MONGO_URI"); + let name = format!("adaptive_conformance_{}", std::process::id()); + let store = MongoLedger::connect(&uri, &name).await.expect("connect"); + conformance::run_all(&store).await; + conformance::run_tenants( + &store, + &store.for_tenant("user-a"), + &store.for_tenant("user-b"), + ) + .await; + store.db.drop().await.expect("drop the throwaway database"); + } +} diff --git a/crates/adaptive/src/ledger/sqlite.rs b/crates/adaptive/src/ledger/sqlite.rs new file mode 100644 index 0000000..4c9d4ca --- /dev/null +++ b/crates/adaptive/src/ledger/sqlite.rs @@ -0,0 +1,936 @@ +//! A [`Ledger`] on sqlite, for a single-process deployment. +//! +//! Synchronous work behind an async trait, on purpose. Every call here is one +//! or two short statements against a local file; wrapping them in +//! `spawn_blocking` would add a thread hop and a tokio dependency to save +//! microseconds nobody can measure. If a deployment ever puts this behind +//! enough concurrency for the lock to matter, that is the moment to move — +//! not before, and the trait means the move costs one file. + +use std::sync::Mutex; + +use async_trait::async_trait; +use rusqlite::{Connection, OptionalExtension, params}; + +use super::{ + Episode, EpisodeStatus, Ledger, LedgerError, LedgerRow, Lesson, LessonKind, Result, Score, +}; + +impl From for LedgerError { + fn from(err: rusqlite::Error) -> Self { + Self::Backend(err.to_string()) + } +} + +/// The schema, applied on open. +/// +/// `IF NOT EXISTS` throughout so opening an existing ledger is a no-op, and +/// every table carries its own id rather than relying on rowid — a row id +/// leaves this process (a lesson cites them) and rowid is not stable across a +/// vacuum. +const DDL: &[&str] = &[ + "CREATE TABLE IF NOT EXISTS ledger_rows ( + id TEXT PRIMARY KEY, + episode TEXT NOT NULL, + attempt INTEGER NOT NULL, + approach_sig TEXT NOT NULL, + approach_desc TEXT NOT NULL DEFAULT '', + workflow_id TEXT, + outcome TEXT NOT NULL DEFAULT '', + cause TEXT NOT NULL DEFAULT '', + cost_usd REAL NOT NULL DEFAULT 0, + at TEXT NOT NULL, + satisfied INTEGER NOT NULL DEFAULT 0, + advanced INTEGER NOT NULL DEFAULT 0, + scope_key TEXT NOT NULL DEFAULT '', + seq INTEGER NOT NULL + )", + // Ordered by `seq`, not by `at`: two attempts finishing in the same second + // are common, and a timestamp tie makes the ledger read in an arbitrary + // order — which silently reorders the exclusion list. + "CREATE INDEX IF NOT EXISTS ix_rows_episode ON ledger_rows(episode, seq)", + // `scope_key` is NOT NULL with '' for global rather than nullable: it is + // part of the workflow-scores primary key, and SQLite does not treat two + // NULLs as equal, so a nullable column there would let every global score + // insert a fresh row instead of upserting the same one. + "CREATE TABLE IF NOT EXISTS lessons ( + id TEXT PRIMARY KEY, + kind TEXT NOT NULL, + trigger TEXT NOT NULL, + mechanism TEXT NOT NULL DEFAULT '', + claim TEXT NOT NULL, + applied INTEGER NOT NULL DEFAULT 0, + helped INTEGER NOT NULL DEFAULT 0, + scope_key TEXT NOT NULL DEFAULT '', + seq INTEGER NOT NULL + )", + "CREATE INDEX IF NOT EXISTS ix_lessons_scope ON lessons(scope_key, seq)", + "CREATE TABLE IF NOT EXISTS lesson_evidence ( + lesson_id TEXT NOT NULL, + row_id TEXT NOT NULL, + PRIMARY KEY (lesson_id, row_id) + )", + "CREATE TABLE IF NOT EXISTS workflow_scores ( + scope_key TEXT NOT NULL DEFAULT '', + workflow_id TEXT NOT NULL, + applied INTEGER NOT NULL DEFAULT 0, + helped INTEGER NOT NULL DEFAULT 0, + PRIMARY KEY (scope_key, workflow_id) + )", + "CREATE TABLE IF NOT EXISTS variants ( + scope_key TEXT NOT NULL DEFAULT '', + variant TEXT NOT NULL, + parent TEXT NOT NULL, + PRIMARY KEY (scope_key, variant) + )", + "CREATE INDEX IF NOT EXISTS ix_variants_parent ON variants(scope_key, parent)", + "CREATE TABLE IF NOT EXISTS episodes ( + id TEXT PRIMARY KEY, + scope_key TEXT NOT NULL DEFAULT '', + goal TEXT NOT NULL, + status TEXT NOT NULL, + attempt INTEGER NOT NULL DEFAULT 0, + stalled INTEGER NOT NULL DEFAULT 0, + started_at TEXT NOT NULL, + updated_at TEXT NOT NULL + )", + "CREATE INDEX IF NOT EXISTS ix_episodes_scope ON episodes(scope_key, updated_at)", + // One row per step, never one blob per attempt: a looped node produces a + // step per iteration, and at RECORD_BUDGET that reaches past what a Mongo + // document may hold. Uniform across backends beats convenient on one. + "CREATE TABLE IF NOT EXISTS attempt_steps ( + scope_key TEXT NOT NULL DEFAULT '', + row_id TEXT NOT NULL, + seq INTEGER NOT NULL, + node_id TEXT NOT NULL, + status TEXT NOT NULL, + output TEXT NOT NULL, + duration_ms INTEGER NOT NULL DEFAULT 0, + null_bindings TEXT NOT NULL DEFAULT '[]', + PRIMARY KEY (scope_key, row_id, seq) + )", +]; + +/// Applied after [`DDL`], failures ignored. +/// +/// A ledger written before scoping existed has the columns missing rather than +/// empty, and `CREATE TABLE IF NOT EXISTS` will not add them. `ADD COLUMN` +/// errors once the column is there, which is the expected case on every start +/// after the first — so these are the statements whose failure means success. +const MIGRATIONS: &[&str] = &[ + "ALTER TABLE lessons ADD COLUMN scope_key TEXT NOT NULL DEFAULT ''", + "ALTER TABLE workflow_scores ADD COLUMN scope_key TEXT NOT NULL DEFAULT ''", + "ALTER TABLE ledger_rows ADD COLUMN satisfied INTEGER NOT NULL DEFAULT 0", + "ALTER TABLE ledger_rows ADD COLUMN advanced INTEGER NOT NULL DEFAULT 0", + "ALTER TABLE ledger_rows ADD COLUMN scope_key TEXT NOT NULL DEFAULT ''", +]; + +/// Where the ledger lives, when the environment says. +pub const DB_PATH_VAR: &str = "TINYFLOWS_ADAPTIVE_DB"; + +/// Where a platform keeps application **data**. +/// +/// Data, not cache and not config. A ledger is not regenerable, so a cache +/// sweeper finding it would delete everything the loop has learned; and it is +/// not something a person edits, so a config directory would invite exactly +/// that. Every platform below distinguishes the three, and this picks the one +/// whose contract is "keep this". +/// +/// Taken as a parameter rather than read from `cfg!` so all three rules are +/// tested on whichever machine runs the suite. A rule that only compiles on the +/// platform it is wrong for is a rule nobody checks. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Platform { + /// XDG Base Directory Specification. + Xdg, + /// Apple's File System Programming Guide. + MacOs, + /// Windows known folders. + Windows, +} + +impl Platform { + /// What this build is running on. + #[must_use] + pub fn host() -> Self { + if cfg!(target_os = "macos") { + Self::MacOs + } else if cfg!(windows) { + Self::Windows + } else { + Self::Xdg + } + } +} + +/// The documented data directory, or `None` when the environment does not say. +/// +/// * **XDG** — `$XDG_DATA_HOME`, else `$HOME/.local/share`. The spec names that +/// fallback, so an unset variable is normal rather than a failure. +/// * **macOS** — `$HOME/Library/Application Support`. +/// * **Windows** — `%LOCALAPPDATA%`, **not** `%APPDATA%`. The roaming profile +/// syncs between machines, and a SQLite file copied mid-write between two +/// machines that both think they own it is a corrupted database. Local is the +/// right shelf for anything a process holds open. +/// +/// `None` is a real answer: a daemon under a user with no home has nowhere by +/// convention, and inventing one would put a database somewhere nobody looks. +fn data_dir( + platform: Platform, + env: &dyn Fn(&str) -> Option, +) -> Option { + let read = |key: &str| env(key).filter(|v| !v.trim().is_empty()); + match platform { + Platform::Xdg => read("XDG_DATA_HOME") + .map(std::path::PathBuf::from) + .or_else(|| read("HOME").map(|h| std::path::PathBuf::from(h).join(".local/share"))), + Platform::MacOs => { + read("HOME").map(|h| std::path::PathBuf::from(h).join("Library/Application Support")) + } + Platform::Windows => read("LOCALAPPDATA").map(std::path::PathBuf::from), + } +} + +/// The directory this project owns inside the platform's data directory. +/// +/// Named for the project, not this crate, so a sibling shares the folder rather +/// than scattering one per crate across a user's disk. +const APP_DIR: &str = "tinyflows"; + +/// The file, inside that. +const DB_FILE: &str = "adaptive.db"; + +/// Which path wins: the environment when it names one, the caller otherwise. +/// +/// Pulled out as a pure function so the rule is tested without any test setting +/// a process-wide variable — `unsafe_code` is forbidden here, and an env-mutating +/// test is a test that fails when another one runs beside it. +/// +/// Blank and whitespace-only are treated as unset: an empty variable is what a +/// shell leaves behind when a value was meant to be interpolated and was not, +/// and opening `""` fails in a way that names nothing useful. +fn chosen_path(configured: Option<&str>, fallback: &std::path::Path) -> std::path::PathBuf { + match configured.map(str::trim).filter(|p| !p.is_empty()) { + Some(path) => std::path::PathBuf::from(path), + None => fallback.to_path_buf(), + } +} + +/// The conventional path, or an error naming the way out. +fn default_path( + platform: Platform, + env: &dyn Fn(&str) -> Option, +) -> Result { + if let Some(configured) = env(DB_PATH_VAR).filter(|v| !v.trim().is_empty()) { + return Ok(std::path::PathBuf::from(configured.trim())); + } + data_dir(platform, env) + .map(|dir| dir.join(APP_DIR).join(DB_FILE)) + .ok_or_else(|| { + LedgerError::Backend(format!( + "no data directory on this platform; set {DB_PATH_VAR} to a writable path" + )) + }) +} + +/// A ledger backed by one sqlite file. +pub struct SqliteLedger { + conn: std::sync::Arc>, + scope: Option, +} + +impl SqliteLedger { + /// Open (or create) a ledger at `path`. + /// + /// # Errors + /// When the file cannot be opened or the schema cannot be applied. + pub fn open(path: impl AsRef) -> Result { + let path = path.as_ref(); + // Create the parent, because `Connection::open` creates the file and + // not the directory holding it. Every sensible location for a ledger — + // `~/.config/something/`, `/var/lib/something/`, a data volume — is a + // directory that may not exist on a first run, and failing there reads + // as "the database is broken" rather than "make the folder". + if let Some(parent) = path.parent().filter(|p| !p.as_os_str().is_empty()) { + std::fs::create_dir_all(parent) + .map_err(|e| LedgerError::Backend(format!("{}: {e}", parent.display())))?; + } + Self::from_connection(Connection::open(path)?) + } + + /// Open the path in `TINYFLOWS_ADAPTIVE_DB`, or `fallback` when it is unset. + /// + /// The library does not invent a location on your disk. A crate that writes + /// to a home directory nobody named is a crate that surprises an operator + /// once and is distrusted afterwards, and the right place differs entirely + /// between a CLI, a container and a service with a mounted volume. + /// + /// So the fallback stays visible in your code and the environment can move + /// it without a rebuild — which is what a deployment actually needs. Either + /// way the parent directory is created. + /// + /// # Errors + /// As [`open`](Self::open). + pub fn from_env_or(fallback: impl AsRef) -> Result { + let configured = std::env::var(DB_PATH_VAR).ok(); + Self::open(chosen_path(configured.as_deref(), fallback.as_ref())) + } + + /// Open the ledger where this platform keeps application data. + /// + /// `TINYFLOWS_ADAPTIVE_DB` still wins when it is set. Otherwise: + /// + /// | Platform | Path | + /// |---|---| + /// | Linux and other XDG | `$XDG_DATA_HOME/tinyflows/adaptive.db`, else `~/.local/share/tinyflows/adaptive.db` | + /// | macOS | `~/Library/Application Support/tinyflows/adaptive.db` | + /// | Windows | `%LOCALAPPDATA%\tinyflows\adaptive.db` | + /// + /// Right for a CLI or a desktop agent, which is what a convention is for. + /// A container or a service should name its own path — a volume mount is + /// the whole point, and a convention that lands the database inside an + /// ephemeral layer is worse than no convention. + /// + /// # Errors + /// When the platform's data directory cannot be determined — a daemon under + /// a user with no home has nowhere by convention, and the error says to set + /// the variable rather than guessing somewhere nobody looks. Also as + /// [`open`](Self::open). + pub fn at_default_location() -> Result { + Self::open(default_path(Platform::host(), &|key| { + std::env::var(key).ok() + })?) + } + + /// A ledger held entirely in memory. For tests, and for a host that wants + /// the loop to run without learning anything durable. + /// + /// # Errors + /// When the schema cannot be applied. + pub fn in_memory() -> Result { + Self::from_connection(Connection::open_in_memory()?) + } + + fn from_connection(conn: Connection) -> Result { + conn.execute_batch("PRAGMA journal_mode = WAL; PRAGMA busy_timeout = 5000;") + .ok(); + for statement in DDL { + conn.execute(statement, [])?; + } + for statement in MIGRATIONS { + let _ = conn.execute(statement, []); + } + Ok(Self { + conn: std::sync::Arc::new(Mutex::new(conn)), + scope: None, + }) + } + + /// A handle onto the same database, scoped to one tenant. + /// + /// Cheap — it shares the connection. Construct one per request at the edge + /// of the service and hand it to the loop; everything downstream reads and + /// writes the right bucket without knowing a tenant exists. + #[must_use] + pub fn for_tenant(&self, scope: impl Into) -> Self { + Self { + conn: std::sync::Arc::clone(&self.conn), + scope: Some(scope.into()), + } + } + + /// This handle's bucket, as stored: `''` for global. + fn bucket(&self) -> &str { + self.scope.as_deref().unwrap_or_default() + } + + fn guard(&self) -> Result> { + // A poisoned lock means a previous caller panicked mid-write. The + // ledger is append-mostly and every write is a single statement, so + // the data is intact; refusing every later call would turn one panic + // into a dead loop. + Ok(self + .conn + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner())) + } +} + +fn next_seq(conn: &Connection, table: &str) -> Result { + let current: Option = conn + .query_row(&format!("SELECT MAX(seq) FROM {table}"), [], |r| r.get(0)) + .optional()? + .flatten(); + Ok(current.unwrap_or(0) + 1) +} + +fn new_id(prefix: &str, seq: i64) -> String { + format!("{prefix}_{seq:08}") +} + +fn read_row(r: &rusqlite::Row<'_>) -> rusqlite::Result { + Ok(LedgerRow { + id: r.get("id")?, + episode: r.get("episode")?, + attempt: r.get::<_, i64>("attempt")?.try_into().unwrap_or(0), + approach_sig: r.get("approach_sig")?, + approach_desc: r.get("approach_desc")?, + workflow_id: r.get("workflow_id")?, + outcome: r.get("outcome")?, + cause: r.get("cause")?, + cost_usd: r.get("cost_usd")?, + at: r.get("at")?, + satisfied: r.get::<_, i64>("satisfied")? != 0, + advanced: r.get::<_, i64>("advanced")? != 0, + }) +} + +/// Read an episode row, deferring the JSON columns' failure to the caller. +/// +/// The inner `Result` is deliberate: `query_map` cannot carry a +/// [`LedgerError`], and swallowing a goal that no longer parses would hand the +/// loop an empty goal and let it run against nothing. +fn read_episode(r: &rusqlite::Row<'_>) -> rusqlite::Result> { + let goal: String = r.get("goal")?; + let status: String = r.get("status")?; + let scope: String = r.get("scope_key")?; + Ok((|| { + Ok(Episode { + id: r.get("id").unwrap_or_default(), + goal: serde_json::from_str(&goal).map_err(|e| LedgerError::Corrupt(e.to_string()))?, + scope_key: (!scope.is_empty()).then_some(scope), + status: serde_json::from_str(&status) + .map_err(|e| LedgerError::Corrupt(e.to_string()))?, + attempt: r + .get::<_, i64>("attempt") + .unwrap_or(0) + .try_into() + .unwrap_or(0), + stalled: r + .get::<_, i64>("stalled") + .unwrap_or(0) + .try_into() + .unwrap_or(0), + started_at: r.get("started_at").unwrap_or_default(), + updated_at: r.get("updated_at").unwrap_or_default(), + }) + })()) +} + +#[async_trait] +impl Ledger for SqliteLedger { + fn scope(&self) -> Option<&str> { + self.scope.as_deref() + } + + async fn append(&self, row: &LedgerRow) -> Result { + let conn = self.guard()?; + let seq = next_seq(&conn, "ledger_rows")?; + let id = new_id("ldg", seq); + conn.execute( + "INSERT INTO ledger_rows(id, episode, attempt, approach_sig, approach_desc, + workflow_id, outcome, cause, cost_usd, at, + satisfied, advanced, scope_key, seq) + VALUES(?1,?2,?3,?4,?5,?6,?7,?8,?9,?10,?11,?12,?13,?14)", + params![ + id, + row.episode, + i64::from(row.attempt), + row.approach_sig, + row.approach_desc, + row.workflow_id, + row.outcome, + row.cause, + row.cost_usd, + row.at, + i64::from(row.satisfied), + i64::from(row.advanced), + self.bucket(), + seq, + ], + )?; + Ok(id) + } + + async fn rows(&self, episode: &str) -> Result> { + let conn = self.guard()?; + // Scoped as well as keyed by episode. An episode id is opaque and a + // service may hand one straight through from a request path, so this + // must not be the one read where guessing an id is enough. + let mut stmt = conn.prepare( + "SELECT * FROM ledger_rows WHERE episode = ?1 AND scope_key = ?2 ORDER BY seq", + )?; + let found = stmt + .query_map(params![episode, self.bucket()], read_row)? + .collect::>>()?; + Ok(found) + } + + async fn promote(&self, lesson: &Lesson, cites: &[String]) -> Result { + let conn = self.guard()?; + let seq = next_seq(&conn, "lessons")?; + let id = new_id("les", seq); + conn.execute( + "INSERT INTO lessons(id, kind, trigger, mechanism, claim, applied, helped, + scope_key, seq) + VALUES(?1,?2,?3,?4,?5,?6,?7,?8,?9)", + params![ + id, + serde_json::to_string(&lesson.kind) + .map_err(|e| LedgerError::Corrupt(e.to_string()))? + .trim_matches('"'), + lesson.trigger, + lesson.mechanism, + lesson.claim, + i64::from(lesson.applied), + i64::from(lesson.helped), + // The handle's, never the argument's. + self.bucket(), + seq, + ], + )?; + for row_id in cites { + conn.execute( + "INSERT OR IGNORE INTO lesson_evidence(lesson_id, row_id) VALUES(?1,?2)", + params![id, row_id], + )?; + } + Ok(id) + } + + async fn lessons(&self, kind: Option) -> Result> { + let conn = self.guard()?; + // This bucket plus global. An unscoped handle's bucket is global, so + // the two halves coincide and it sees exactly what it wrote. + let mut stmt = conn + .prepare("SELECT * FROM lessons WHERE scope_key = ?1 OR scope_key = '' ORDER BY seq")?; + let all = stmt + .query_map([self.bucket()], |r| { + let scope: String = r.get("scope_key")?; + Ok(Lesson { + id: r.get("id")?, + kind: LessonKind::parse(&r.get::<_, String>("kind")?), + trigger: r.get("trigger")?, + mechanism: r.get("mechanism")?, + claim: r.get("claim")?, + applied: r.get::<_, i64>("applied")?.try_into().unwrap_or(0), + helped: r.get::<_, i64>("helped")?.try_into().unwrap_or(0), + scope_key: (!scope.is_empty()).then_some(scope), + }) + })? + .collect::>>()?; + Ok(match kind { + Some(want) => all.into_iter().filter(|l| l.kind == want).collect(), + None => all, + }) + } + + async fn evidence(&self, lesson_id: &str) -> Result> { + let conn = self.guard()?; + let mut stmt = conn.prepare( + "SELECT r.* FROM ledger_rows r + JOIN lesson_evidence e ON e.row_id = r.id + WHERE e.lesson_id = ?1 AND r.scope_key = ?2 ORDER BY r.seq", + )?; + let found = stmt + .query_map(params![lesson_id, self.bucket()], read_row)? + .collect::>>()?; + Ok(found) + } + + async fn score_lesson(&self, lesson_id: &str, helped: bool) -> Result<()> { + let conn = self.guard()?; + conn.execute( + // Constrained to what this handle can see — the id arrives from + // model output, and naming another tenant's lesson must not move + // its score. + "UPDATE lessons SET applied = applied + 1, helped = helped + ?2 + WHERE id = ?1 AND (scope_key = ?3 OR scope_key = '')", + params![lesson_id, i64::from(helped), self.bucket()], + )?; + Ok(()) + } + + async fn score_workflow(&self, workflow_id: &str, helped: bool) -> Result<()> { + let conn = self.guard()?; + // Upsert: the first run of a workflow is the common case and must not + // need a separate registration step. + conn.execute( + "INSERT INTO workflow_scores(scope_key, workflow_id, applied, helped) + VALUES(?1, ?2, 1, ?3) + ON CONFLICT(scope_key, workflow_id) DO UPDATE SET + applied = applied + 1, + helped = helped + ?3", + params![self.bucket(), workflow_id, i64::from(helped)], + )?; + Ok(()) + } + + async fn workflow_score(&self, workflow_id: &str) -> Result { + let conn = self.guard()?; + let found = conn + .query_row( + "SELECT applied, helped FROM workflow_scores + WHERE scope_key = ?1 AND workflow_id = ?2", + params![self.bucket(), workflow_id], + |r| { + Ok(Score { + applied: r.get::<_, i64>(0)?.try_into().unwrap_or(0), + helped: r.get::<_, i64>(1)?.try_into().unwrap_or(0), + }) + }, + ) + .optional()?; + Ok(found.unwrap_or_default()) + } + + async fn link_variant(&self, parent: &str, variant: &str) -> Result<()> { + let conn = self.guard()?; + conn.execute( + "INSERT OR IGNORE INTO variants(scope_key, variant, parent) VALUES(?1,?2,?3)", + params![self.bucket(), variant, parent], + )?; + Ok(()) + } + + async fn parent_of(&self, id: &str) -> Result> { + let conn = self.guard()?; + let found = conn + .query_row( + "SELECT parent FROM variants WHERE scope_key = ?1 AND variant = ?2", + params![self.bucket(), id], + |r| r.get(0), + ) + .optional()?; + Ok(found) + } + + async fn save_episode(&self, episode: &Episode) -> Result<()> { + let conn = self.guard()?; + conn.execute( + "INSERT INTO episodes(id, scope_key, goal, status, attempt, stalled, + started_at, updated_at) + VALUES(?1,?2,?3,?4,?5,?6,?7,?8) + ON CONFLICT(id) DO UPDATE SET + goal = ?3, status = ?4, attempt = ?5, stalled = ?6, updated_at = ?8", + params![ + episode.id, + self.bucket(), + serde_json::to_string(&episode.goal) + .map_err(|e| LedgerError::Corrupt(e.to_string()))?, + serde_json::to_string(&episode.status) + .map_err(|e| LedgerError::Corrupt(e.to_string()))?, + i64::from(episode.attempt), + i64::from(episode.stalled), + episode.started_at, + episode.updated_at, + ], + )?; + Ok(()) + } + + async fn episode(&self, id: &str) -> Result> { + let conn = self.guard()?; + let found = conn + .query_row( + "SELECT * FROM episodes WHERE id = ?1 AND scope_key = ?2", + params![id, self.bucket()], + read_episode, + ) + .optional()?; + found.transpose() + } + + async fn save_steps(&self, row_id: &str, steps: &[crate::execute::StepRecord]) -> Result<()> { + let conn = self.guard()?; + // Replace, not overlay: `INSERT OR REPLACE` only touches the sequence + // numbers present in `steps`, so a shorter re-save would leave the old + // tail behind and `steps()` would stitch two attempts together. + conn.execute( + "DELETE FROM attempt_steps WHERE scope_key = ?1 AND row_id = ?2", + params![self.bucket(), row_id], + )?; + for (seq, step) in steps.iter().enumerate() { + conn.execute( + "INSERT OR REPLACE INTO attempt_steps(scope_key, row_id, seq, node_id, status, + output, duration_ms, null_bindings) + VALUES(?1,?2,?3,?4,?5,?6,?7,?8)", + params![ + self.bucket(), + row_id, + i64::try_from(seq).unwrap_or(i64::MAX), + step.node_id, + serde_json::to_string(&step.status) + .map_err(|e| LedgerError::Corrupt(e.to_string()))? + .trim_matches('"'), + serde_json::to_string(&step.output) + .map_err(|e| LedgerError::Corrupt(e.to_string()))?, + i64::try_from(step.duration_ms).unwrap_or(i64::MAX), + serde_json::to_string(&step.null_bindings) + .map_err(|e| LedgerError::Corrupt(e.to_string()))?, + ], + )?; + } + Ok(()) + } + + async fn steps(&self, row_id: &str) -> Result> { + let conn = self.guard()?; + let mut stmt = conn.prepare( + "SELECT node_id, status, output, duration_ms, null_bindings FROM attempt_steps + WHERE scope_key = ?1 AND row_id = ?2 ORDER BY seq", + )?; + let found = stmt + .query_map(params![self.bucket(), row_id], |r| { + Ok(( + r.get::<_, String>(0)?, + r.get::<_, String>(1)?, + r.get::<_, String>(2)?, + r.get::<_, i64>(3)?, + r.get::<_, String>(4)?, + )) + })? + .collect::>>()?; + + found + .into_iter() + .map(|(node_id, status, output, duration_ms, bindings)| { + Ok(crate::execute::StepRecord { + node_id, + status: if status == "error" { + crate::execute::StepOutcome::Error + } else { + crate::execute::StepOutcome::Success + }, + output: serde_json::from_str(&output) + .map_err(|e| LedgerError::Corrupt(e.to_string()))?, + duration_ms: u64::try_from(duration_ms).unwrap_or(0), + null_bindings: serde_json::from_str(&bindings) + .map_err(|e| LedgerError::Corrupt(e.to_string()))?, + }) + }) + .collect() + } + + async fn episodes(&self, running_only: bool, page: super::Page) -> Result> { + let conn = self.guard()?; + let mut stmt = conn + .prepare("SELECT * FROM episodes WHERE scope_key = ?1 ORDER BY updated_at DESC, id")?; + let all = stmt + .query_map([self.bucket()], read_episode)? + .collect::>>()?; + let kept: Result> = all + .into_iter() + .filter(|e| { + !running_only || e.as_ref().is_ok_and(|e| e.status == EpisodeStatus::Running) + }) + .collect(); + Ok(page.apply(kept?)) + } + + async fn children_of(&self, id: &str) -> Result> { + let conn = self.guard()?; + let mut stmt = conn.prepare( + "SELECT variant FROM variants WHERE scope_key = ?1 AND parent = ?2 ORDER BY variant", + )?; + let found = stmt + .query_map(params![self.bucket(), id], |r| r.get(0))? + .collect::>>()?; + Ok(found) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::ledger::conformance; + + #[tokio::test] + async fn passes_the_conformance_suite() { + let store = SqliteLedger::in_memory().expect("open in-memory ledger"); + conformance::run_all(&store).await; + } + + #[tokio::test] + async fn passes_the_tenant_isolation_suite() { + let store = SqliteLedger::in_memory().expect("open in-memory ledger"); + let a = store.for_tenant("user-a"); + let b = store.for_tenant("user-b"); + conformance::run_tenants(&store, &a, &b).await; + } + + #[tokio::test] + async fn a_scoped_handle_shares_the_connection_rather_than_the_file() { + // Two handles for the SAME tenant must see each other's writes — that + // is what "shares" means. Probing it across scopes would now fail by + // design, because rows carry the bucket that wrote them. + let store = SqliteLedger::in_memory().expect("open in-memory ledger"); + let one = store.for_tenant("user-a"); + let two = store.for_tenant("user-a"); + one.append(&conformance::row("ep-shared", 1, "authored")) + .await + .expect("append"); + assert_eq!(two.rows("ep-shared").await.expect("rows").len(), 1); + assert!( + store.rows("ep-shared").await.expect("rows").is_empty(), + "and the global bucket is its own, not a union" + ); + } + + #[tokio::test] + async fn opening_a_path_creates_the_directory_holding_it() { + // A first run against `/var/lib/whatever/ledger.db` must not fail + // because nobody made the folder. + let root = std::env::temp_dir().join(format!("adaptive-mkdir-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&root); + let path = root.join("deep").join("nested").join("ledger.db"); + + let store = SqliteLedger::open(&path).expect("open"); + store + .append(&conformance::row("ep-mkdir", 1, "authored")) + .await + .expect("append"); + assert!(path.exists(), "{}", path.display()); + let _ = std::fs::remove_dir_all(&root); + } + + #[test] + fn the_environment_moves_the_ledger_without_a_rebuild() { + let fallback = std::path::Path::new("/srv/app/ledger.db"); + assert_eq!( + chosen_path(Some("/mnt/data/ledger.db"), fallback), + std::path::PathBuf::from("/mnt/data/ledger.db") + ); + } + + #[test] + fn an_unset_environment_falls_back_to_the_path_in_the_code() { + let fallback = std::path::Path::new("/srv/app/ledger.db"); + assert_eq!(chosen_path(None, fallback), fallback); + } + + #[test] + fn a_blank_variable_reads_as_unset_rather_than_as_an_empty_path() { + // What a shell leaves behind when a value was meant to be interpolated + // and was not. Opening "" fails in a way that names nothing useful. + let fallback = std::path::Path::new("/srv/app/ledger.db"); + assert_eq!(chosen_path(Some(""), fallback), fallback); + assert_eq!(chosen_path(Some(" "), fallback), fallback); + } + + fn fake_env(pairs: &[(&str, &str)]) -> impl Fn(&str) -> Option + use<> { + let owned: Vec<(String, String)> = pairs + .iter() + .map(|(k, v)| ((*k).to_string(), (*v).to_string())) + .collect(); + move |key: &str| owned.iter().find(|(k, _)| k == key).map(|(_, v)| v.clone()) + } + + #[test] + fn each_platform_uses_its_own_documented_directory() { + let home = fake_env(&[("HOME", "/home/ada")]); + assert_eq!( + data_dir(Platform::Xdg, &home), + Some("/home/ada/.local/share".into()) + ); + assert_eq!( + data_dir(Platform::MacOs, &fake_env(&[("HOME", "/Users/ada")])), + Some("/Users/ada/Library/Application Support".into()) + ); + assert_eq!( + data_dir( + Platform::Windows, + &fake_env(&[("LOCALAPPDATA", "C:\\Users\\ada\\AppData\\Local")]) + ), + Some("C:\\Users\\ada\\AppData\\Local".into()) + ); + } + + #[test] + fn xdg_data_home_wins_over_the_spec_s_own_fallback() { + let env = fake_env(&[("XDG_DATA_HOME", "/data"), ("HOME", "/home/ada")]); + assert_eq!(data_dir(Platform::Xdg, &env), Some("/data".into())); + } + + #[test] + fn windows_uses_the_local_profile_not_the_roaming_one() { + // A roaming profile syncs between machines, and a SQLite file copied + // mid-write between two that both think they own it is a corrupted + // database. Setting only APPDATA must therefore find nothing. + let roaming = fake_env(&[("APPDATA", "C:\\Users\\ada\\AppData\\Roaming")]); + assert_eq!(data_dir(Platform::Windows, &roaming), None); + } + + #[test] + fn the_conventional_path_is_namespaced_by_project_and_named_for_the_crate() { + let env = fake_env(&[("HOME", "/home/ada")]); + assert_eq!( + default_path(Platform::Xdg, &env).expect("path"), + std::path::PathBuf::from("/home/ada/.local/share/tinyflows/adaptive.db") + ); + } + + #[test] + fn the_variable_still_wins_over_the_convention() { + let env = fake_env(&[(DB_PATH_VAR, "/mnt/data/ledger.db"), ("HOME", "/home/ada")]); + assert_eq!( + default_path(Platform::Xdg, &env).expect("path"), + std::path::PathBuf::from("/mnt/data/ledger.db") + ); + } + + #[test] + fn nowhere_conventional_is_an_error_that_says_what_to_set() { + // A daemon under a user with no home. Guessing would put a database + // somewhere nobody looks, and losing it silently is the failure this + // whole crate is written to avoid. + let err = default_path(Platform::Xdg, &fake_env(&[])).expect_err("no home"); + assert!(err.to_string().contains(DB_PATH_VAR), "{err}"); + } + + #[test] + fn a_configured_path_is_trimmed() { + let fallback = std::path::Path::new("/srv/app/ledger.db"); + assert_eq!( + chosen_path(Some(" /mnt/data/ledger.db\n"), fallback), + std::path::PathBuf::from("/mnt/data/ledger.db") + ); + } + + #[tokio::test] + async fn a_reopened_ledger_still_has_its_rows() { + // The whole point of the sqlite backend over the in-memory one. + let dir = std::env::temp_dir().join(format!("adaptive-ledger-{}", std::process::id())); + std::fs::create_dir_all(&dir).expect("temp dir"); + let path = dir.join("ledger.db"); + let _ = std::fs::remove_file(&path); + + { + let store = SqliteLedger::open(&path).expect("open"); + store + .append(&conformance::row("ep", 1, "authored")) + .await + .expect("append"); + } + let reopened = SqliteLedger::open(&path).expect("reopen"); + assert_eq!(reopened.rows("ep").await.expect("rows").len(), 1); + + let _ = std::fs::remove_dir_all(&dir); + } + + #[tokio::test] + async fn insertion_order_survives_a_timestamp_tie() { + // Two attempts finishing in the same second is common; ordering by `at` + // would make the exclusion list arbitrary. + let store = SqliteLedger::in_memory().expect("open"); + for sig in ["first", "second", "third"] { + let mut r = conformance::row("tie", 1, sig); + r.at = "2026-01-01T00:00:00Z".to_string(); + store.append(&r).await.expect("append"); + } + assert_eq!( + store.tried("tie").await.expect("tried"), + vec!["first", "second", "third"] + ); + } +} diff --git a/crates/adaptive/src/lib.rs b/crates/adaptive/src/lib.rs new file mode 100644 index 0000000..9107b51 --- /dev/null +++ b/crates/adaptive/src/lib.rs @@ -0,0 +1,29 @@ +//! An adaptive loop over the tinyflows engine. +//! +//! Ingests a prompt, selects a stored workflow or authors one, runs it on the +//! engine, judges the result against evidence, and learns — updating or +//! replacing the workflow when the graph itself was at fault. +//! +//! The engine is not modified. This crate decides *which* graph to run; +//! [`tinyflows`] decides nothing and runs one graph. See the crate README for +//! why that split is structural rather than stylistic. +//! +//! The rule it enforces: **the engine may know about one run; anything that +//! spans runs lives here.** + +#![forbid(unsafe_code)] +#![warn(missing_docs)] + +pub mod closing; +pub mod contracts; +pub mod driver; +pub mod execute; +pub mod host; +pub mod intake; +pub mod inventory; +pub mod ledger; +pub mod promotion; +pub mod recall; +pub mod reuse; +pub mod storage; +pub mod workflows; diff --git a/crates/adaptive/src/promotion.rs b/crates/adaptive/src/promotion.rs new file mode 100644 index 0000000..496efc8 --- /dev/null +++ b/crates/adaptive/src/promotion.rs @@ -0,0 +1,237 @@ +//! Which member of a repaired family the catalogue offers. +//! +//! [`crate::closing::repair`] never edits a workflow in place — it saves a +//! variant, so the parent's score survives to be compared against. That leaves +//! a question this module answers: after three repairs, which of the four +//! graphs does a planner get to see? +//! +//! Showing all four is the wrong answer. They are near-identical, their +//! descriptions differ by a clause, and a planner choosing between them is +//! choosing noise. Showing the newest is also wrong — that is promotion by +//! having been written, which is what the whole variant mechanism exists to +//! avoid. +//! +//! So the catalogue offers **one member per family**, and this decides which. +//! +//! # The rule +//! +//! Three bands, in order, and the first non-empty one wins: +//! +//! 1. **Proven and has helped** — [`MIN_TRIALS`] runs behind it and at least one +//! success. Best help rate, ties broken by more trials: 40/40 beats 1/1 at +//! the same rate, because they are not the same evidence. +//! 2. **Unproven** — too few runs to say. Ordered by what thin evidence there +//! is, then by lineage, so a family where *nothing* has been tried keeps the +//! graph a person wrote. +//! 3. **Proven and never helped** — enough runs to be sure it does not work. +//! +//! The bands exist because "not yet tried" and "tried and never worked" are +//! both a help rate of `0.0`, and a single number cannot tell them apart. With +//! one number the filter ran first, so if the *only* proven member had never +//! helped it won by default — a root that failed three times out of three +//! holding the slot against a variant that had succeeded twice out of two. The +//! same shape as the bug in [`crate::recall`], arrived at independently. +//! +//! # Why there is no exploration policy +//! +//! A fresh variant has zero trials, so it can never become proven if it is +//! never offered — the usual explore/exploit trap, and the usual fix is to +//! offer unproven candidates some fraction of the time. +//! +//! That machinery is not needed here, because of where variants come from. A +//! variant is written by the closing pass of an episode whose *parent just +//! failed*, and that parent is already in the episode's exclusion list. The +//! next attempt of that same episode cannot pick the parent, so the variant +//! gets its trials exactly where the evidence is most relevant — against the +//! goal that broke the parent — without anyone writing a bandit. +//! +//! The cost of getting this wrong in the other direction is what the rule +//! protects: an unproven variant that displaced a 40/40 parent for everyone +//! would spend other people's episodes discovering it was worse. + +use crate::ledger::Score; + +/// Runs before a member's score is treated as evidence. +/// +/// Three, not one: a single satisfied run is 1/1, indistinguishable by rate +/// from forty, and promoting on it means promoting on luck. Three is small +/// enough that a genuinely better variant takes over quickly and large enough +/// that a coin flip usually does not. +pub const MIN_TRIALS: u32 = 3; + +/// Where one member of a family stands. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Standing { + /// Not enough runs to say. Gets its trials from the episode that made it. + Unproven, + /// Proven, and the best of the family. This is what the catalogue offers. + Champion, + /// Proven, and something else in the family is better. + Beaten, +} + +/// Which band a member sits in. Lower is better; see the module note. +fn band(score: Score) -> u8 { + match (score.applied >= MIN_TRIALS, score.helped > 0) { + (true, true) => 0, + (false, _) => 1, + (true, false) => 2, + } +} + +/// Pick the member to offer. +/// +/// `family` is `(id, score)` in [`crate::ledger::Ledger::lineage`] order — +/// **root first**, which is what decides an unproven family: nothing has +/// established anything, so the graph a person wrote keeps the position. +/// Returns `None` only for an empty family. +#[must_use] +pub fn champion(family: &[(String, Score)]) -> Option<&str> { + family + .iter() + .enumerate() + .min_by(|(i, (_, a)), (j, (_, b))| { + band(*a) + .cmp(&band(*b)) + .then_with(|| b.help_rate().total_cmp(&a.help_rate())) + .then_with(|| b.applied.cmp(&a.applied)) + // Lineage order last, so a tie inside a band keeps the root. + .then_with(|| i.cmp(j)) + }) + .map(|(_, (id, _))| id.as_str()) +} + +/// Where `id` stands within its family. +#[must_use] +pub fn standing(id: &str, family: &[(String, Score)]) -> Standing { + let Some((_, score)) = family.iter().find(|(member, _)| member == id) else { + return Standing::Unproven; + }; + if band(*score) == 1 { + return Standing::Unproven; + } + if champion(family) == Some(id) { + Standing::Champion + } else { + Standing::Beaten + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn family(members: &[(&str, u32, u32)]) -> Vec<(String, Score)> { + members + .iter() + .map(|(id, applied, helped)| { + ( + (*id).to_string(), + Score { + applied: *applied, + helped: *helped, + }, + ) + }) + .collect() + } + + #[test] + fn a_lone_workflow_is_its_own_champion() { + assert_eq!(champion(&family(&[("weekly", 0, 0)])), Some("weekly")); + } + + #[test] + fn a_fresh_variant_does_not_displace_a_proven_parent() { + // The expensive mistake: an untested graph taking over for everyone and + // spending other people's episodes finding out it was worse. + let f = family(&[("weekly", 40, 40), ("weekly-fix-abc", 0, 0)]); + assert_eq!(champion(&f), Some("weekly")); + assert_eq!(standing("weekly-fix-abc", &f), Standing::Unproven); + } + + #[test] + fn a_variant_takes_over_once_it_has_proven_better() { + let f = family(&[("weekly", 10, 5), ("weekly-fix-abc", 4, 4)]); + assert_eq!(champion(&f), Some("weekly-fix-abc")); + assert_eq!(standing("weekly", &f), Standing::Beaten); + assert_eq!(standing("weekly-fix-abc", &f), Standing::Champion); + } + + #[test] + fn a_variant_proven_worse_stays_out() { + let f = family(&[("weekly", 10, 9), ("weekly-fix-abc", 5, 1)]); + assert_eq!(champion(&f), Some("weekly")); + assert_eq!(standing("weekly-fix-abc", &f), Standing::Beaten); + } + + #[test] + fn more_trials_win_the_tie_because_they_are_not_the_same_evidence() { + // 40/40 and 3/3 are the same rate. They are not the same claim. + let f = family(&[("weekly", 40, 40), ("weekly-fix-abc", 3, 3)]); + assert_eq!(champion(&f), Some("weekly")); + } + + #[test] + fn an_untried_family_keeps_the_graph_a_person_wrote() { + // The principle the lineage tie-break exists for: with no evidence at + // all, nothing displaces the root. + let f = family(&[("weekly", 0, 0), ("weekly-fix-abc", 0, 0)]); + assert_eq!(champion(&f), Some("weekly")); + } + + #[test] + fn a_fresh_variant_does_not_displace_an_only_slightly_tried_root() { + let f = family(&[("weekly", 1, 1), ("weekly-fix-abc", 0, 0)]); + assert_eq!(champion(&f), Some("weekly")); + } + + #[test] + fn thin_evidence_still_decides_between_two_unproven_members() { + // This case used to assert the root wins, on the reading that neither + // is proven so neither takes it. But the root here has been tried once + // and failed, and the variant twice and worked twice — "unproven" is + // not "untested", and offering the one that has only ever failed wastes + // the attempt that would have told us either way. + let f = family(&[("weekly", 1, 0), ("weekly-fix-abc", 2, 2)]); + assert_eq!(champion(&f), Some("weekly-fix-abc")); + } + + #[test] + fn one_proven_member_wins_even_when_the_root_is_unproven() { + let f = family(&[("weekly", 2, 0), ("weekly-fix-abc", 3, 2)]); + assert_eq!(champion(&f), Some("weekly-fix-abc")); + } + + #[test] + fn a_workflow_outside_the_family_reads_as_unproven_rather_than_panicking() { + let f = family(&[("weekly", 40, 40)]); + assert_eq!(standing("something-else", &f), Standing::Unproven); + } + + #[test] + fn a_workflow_proven_useless_does_not_outrank_an_untested_variant() { + // The mirror of the recall bug. There, an untried lesson sorted level + // with useless ones and was cut. Here, the proven filter runs first, so + // if the ONLY proven member has never helped it wins by default — a + // root that failed three times out of three keeping the slot against a + // variant that has succeeded twice out of two. + let f = family(&[("weekly", 3, 0), ("weekly-fix-1", 2, 2)]); + assert_eq!(champion(&f), Some("weekly-fix-1")); + assert_eq!(standing("weekly", &f), Standing::Beaten); + } + + #[test] + fn one_success_still_beats_an_untested_variant() { + // The other direction: a member that has actually worked keeps the slot + // against something with no record, which is the whole point of the + // trial threshold. + let f = family(&[("weekly", 4, 1), ("weekly-fix-1", 2, 2)]); + assert_eq!(champion(&f), Some("weekly")); + } + + #[test] + fn an_empty_family_has_no_champion() { + assert_eq!(champion(&[]), None); + } +} diff --git a/crates/adaptive/src/recall.rs b/crates/adaptive/src/recall.rs new file mode 100644 index 0000000..94c1694 --- /dev/null +++ b/crates/adaptive/src/recall.rs @@ -0,0 +1,281 @@ +//! What a planner is told about the past. +//! +//! Two different pasts, and conflating them is how a retry becomes a repeat. +//! +//! **This episode's attempts** are specific: three rows saying what was tried +//! and why each fell short. They are the reason attempt four is not attempt +//! two in different words. Without them the author writes the same graph again, +//! confidently, because nothing told it otherwise. +//! +//! **Lessons** are general: what generalised out of *other* episodes. They come +//! from [`crate::closing::consolidate`], which until now was write-only — +//! lessons were being kept and never read, which is a knowledge store that +//! costs money and returns nothing. +//! +//! Both are rendered for a prompt here rather than at the two call sites, so +//! `select` and `author` see the same history in the same words. + +use crate::ledger::{LedgerRow, Lesson, LessonKind}; + +/// How many lessons a planner sees, beyond the kinds that always load. +/// +/// **Everything, and that is the right answer at this scale.** With tens of +/// lessons in scope, every one of them is relevant to the planner reading them, +/// and the ordering below is a placeholder for matching nobody has written yet. +/// Capping on an unvalidated order does not select the best five, it discards +/// four-fifths of what was learned on a guess. +/// +/// It was five, and that was a bug rather than a trade: a lesson written +/// moments ago has `applied == 0`, so its help rate is `0.0`, so it sorted +/// level with lessons proven useless and was cut the moment five others had any +/// success. Never shown, so never applied, so never able to earn a rate — the +/// trap [`crate::promotion`] avoids by giving a variant its trials, with +/// nothing here doing the same. +/// +/// The seam stays because a host with hundreds of lessons has a real prompt-size +/// problem: pass your own `k` to [`retrieve`], and the ordering below decides +/// what survives. +pub const RECALL_LIMIT: usize = usize::MAX; + +/// Kinds that load wholesale, exempt from [`RECALL_LIMIT`]. +/// +/// A constraint is a limit no approach can cross. Inside its scope it is always +/// relevant, there are few of them, and dropping one because five strategies +/// outranked it means proposing something already known to be impossible. +const LOAD_ALL: [LessonKind; 1] = [LessonKind::Constraint]; + +/// Where a lesson sorts, when only some of them can be shown. +/// +/// Three bands rather than one number, because a rate cannot tell "has not been +/// tried" from "has been tried and never helped" — both are `0.0`, and +/// collapsing them means a cap silently prefers a known failure to an untested +/// idea. +fn band(lesson: &Lesson) -> u8 { + match (lesson.applied, lesson.helped) { + // Demonstrably useful at least once. + (a, h) if a > 0 && h > 0 => 0, + // Never put in front of a planner. Unjudged, not bad. + (0, _) => 1, + // Applied, and never once helped. + _ => 2, + } +} + +/// Choose which lessons a planner sees. +/// +/// Everything in scope by default — see [`RECALL_LIMIT`]. The order matters +/// only when a host passes a smaller `k`, and then it is by band first (useful, +/// untried, useless), rate within the first band, and id to break ties so a +/// planner does not see a different set each attempt. +#[must_use] +pub fn retrieve(lessons: Vec, kind: Option, k: usize) -> Vec { + let mut pool: Vec = lessons + .into_iter() + .filter(|lesson| kind.is_none_or(|want| lesson.kind == want)) + .collect(); + pool.sort_by(|a, b| { + band(a) + .cmp(&band(b)) + .then_with(|| b.help_rate().total_cmp(&a.help_rate())) + .then_with(|| a.id.cmp(&b.id)) + }); + + let (always, rest): (Vec, Vec) = + pool.into_iter().partition(|l| LOAD_ALL.contains(&l.kind)); + always.into_iter().chain(rest.into_iter().take(k)).collect() +} + +/// What generalised out of other episodes, for a prompt. Empty when nothing has. +#[must_use] +pub fn render_lessons(lessons: &[Lesson]) -> String { + if lessons.is_empty() { + return String::new(); + } + let body = lessons + .iter() + .map(|lesson| { + let mechanism = if lesson.mechanism.is_empty() { + String::new() + } else { + format!(" ({})", lesson.mechanism) + }; + let record = match lesson.applied { + 0 => "not yet applied".to_string(), + applied => format!("applied {applied}×, helped {}×", lesson.helped), + }; + format!( + "- when {}: {}{mechanism} [{record}]", + lesson.trigger, lesson.claim + ) + }) + .collect::>() + .join("\n"); + format!("\n\n# Learned from earlier episodes\n{body}") +} + +/// What this episode has already spent, for a prompt. Empty on attempt one. +/// +/// Numbered from one, the way a person counts attempts, and each line carries +/// the signature — the planner is being asked not to propose one of these +/// again, so it needs to see them the way the exclusion list does. +#[must_use] +pub fn render_history(rows: &[LedgerRow]) -> String { + if rows.is_empty() { + return String::new(); + } + let body = rows + .iter() + .map(|row| { + let because = if row.cause.is_empty() { + String::new() + } else { + format!("\n still missing: {}", row.cause) + }; + format!( + "{}. [{}] {} → {}{because}", + row.attempt, row.approach_sig, row.approach_desc, row.outcome + ) + }) + .collect::>() + .join("\n"); + format!("\n\n# Already tried this episode — do not propose any of these again\n{body}") +} + +#[cfg(test)] +mod tests { + use super::*; + + fn lesson(id: &str, kind: LessonKind, applied: u32, helped: u32) -> Lesson { + Lesson { + id: id.into(), + kind, + trigger: format!("the {id} situation"), + mechanism: String::new(), + claim: format!("do {id}"), + applied, + helped, + scope_key: None, + } + } + + fn row(attempt: u32, sig: &str, cause: &str) -> LedgerRow { + LedgerRow { + id: format!("r{attempt}"), + episode: "ep".into(), + attempt, + approach_sig: sig.into(), + approach_desc: "tried the obvious thing".into(), + workflow_id: None, + outcome: "fell short".into(), + cause: cause.into(), + cost_usd: 0.0, + at: "2026-01-01T00:00:00Z".into(), + satisfied: false, + advanced: false, + } + } + + #[test] + fn everything_in_scope_reaches_the_planner_by_default() { + // The default is not a selection. With tens of lessons every one is + // relevant, and cutting on an unvalidated order discards what was + // learned on a guess. + let pool: Vec = (0..20) + .map(|n| lesson(&format!("l{n}"), LessonKind::Strategy, 10, 10)) + .collect(); + assert_eq!(retrieve(pool, None, RECALL_LIMIT).len(), 20); + } + + #[test] + fn a_brand_new_lesson_is_not_cut_before_a_useless_one() { + // The bug the default hid. A lesson written moments ago has no rate, so + // it sorted level with lessons proven useless and was dropped first — + // never shown, so never applied, so never able to earn a rate. + let pool = vec![ + lesson("useless", LessonKind::Strategy, 9, 0), + lesson("brand-new", LessonKind::Strategy, 0, 0), + ]; + let got = retrieve(pool, None, 1); + assert_eq!(got.len(), 1); + assert_eq!(got[0].id, "brand-new", "untried outranks proven useless"); + } + + #[test] + fn a_lesson_that_has_helped_still_outranks_an_untried_one() { + let pool = vec![ + lesson("untried", LessonKind::Strategy, 0, 0), + lesson("works", LessonKind::Strategy, 4, 3), + ]; + let got = retrieve(pool, None, 1); + assert_eq!(got[0].id, "works"); + } + + #[test] + fn the_best_helping_lessons_come_first() { + let got = retrieve( + vec![ + lesson("weak", LessonKind::Strategy, 10, 1), + lesson("strong", LessonKind::Strategy, 10, 9), + ], + None, + 5, + ); + assert_eq!(got[0].id, "strong"); + } + + #[test] + fn the_order_is_stable_when_two_lessons_are_equally_good() { + // A planner shown a different five each attempt cannot be reasoned about. + let pool = vec![ + lesson("b", LessonKind::Strategy, 4, 2), + lesson("a", LessonKind::Strategy, 4, 2), + ]; + let once = retrieve(pool.clone(), None, 5); + let twice = retrieve(pool, None, 5); + assert_eq!(once[0].id, "a"); + assert_eq!( + once.iter().map(|l| &l.id).collect::>(), + twice.iter().map(|l| &l.id).collect::>() + ); + } + + #[test] + fn constraints_load_wholesale_past_the_cap() { + // Dropping a constraint because five strategies outranked it means + // proposing something already known to be impossible. + let mut pool: Vec = (0..8) + .map(|n| lesson(&format!("s{n}"), LessonKind::Strategy, 10, 10)) + .collect(); + pool.push(lesson("hard-limit", LessonKind::Constraint, 0, 0)); + + let got = retrieve(pool, None, 2); + assert!(got.iter().any(|l| l.id == "hard-limit"), "{got:?}"); + assert_eq!(got.len(), 3, "the constraint plus the two-strategy cap"); + } + + #[test] + fn nothing_learned_yet_renders_to_nothing_rather_than_an_empty_heading() { + assert!(render_lessons(&[]).is_empty()); + assert!(render_history(&[]).is_empty()); + } + + #[test] + fn the_history_names_the_signature_the_exclusion_list_uses() { + let rendered = render_history(&[row(1, "selected:weekly", "no numbers in it")]); + assert!(rendered.contains("[selected:weekly]"), "{rendered}"); + assert!(rendered.contains("do not propose any of these again")); + assert!(rendered.contains("still missing: no numbers in it")); + } + + #[test] + fn a_row_with_no_stated_cause_says_nothing_rather_than_an_empty_line() { + let rendered = render_history(&[row(2, "authored:abc", "")]); + assert!(!rendered.contains("still missing"), "{rendered}"); + } + + #[test] + fn an_unapplied_lesson_says_so_rather_than_showing_zero_of_zero() { + let rendered = render_lessons(&[lesson("new", LessonKind::Strategy, 0, 0)]); + assert!(rendered.contains("not yet applied"), "{rendered}"); + } +} diff --git a/crates/adaptive/src/reuse.rs b/crates/adaptive/src/reuse.rs new file mode 100644 index 0000000..3336162 --- /dev/null +++ b/crates/adaptive/src/reuse.rs @@ -0,0 +1,289 @@ +//! Whether an authored graph is a procedure or a one-off. +//! +//! The authoring prompt asks for a graph that is generic with declared inputs: +//! *"read it in config rather than pasting the literal. A graph with the value +//! baked in is a graph that works once."* Nothing checked that, and nothing +//! kept the result — so a graph authored for a goal, which then achieved it, +//! was thrown away and re-authored from scratch the next time the same kind of +//! thing was asked. +//! +//! Keeping it needs a gate, because keeping *every* one is worse than keeping +//! none: a catalogue full of graphs that each match one task makes selection +//! harder, not easier, and every row is a row the planner reads. +//! +//! # The gate is exact, not a judgement +//! +//! Authoring returns two things — the graph, and the concrete input values for +//! this run. So the question "did it bake the specifics in" has a precise +//! answer: **does a value it handed us as an input appear as a literal inside a +//! node's config?** +//! +//! ```text +//! inputs: { "repo": "acme/thing" } +//! +//! reusable { "prompt": "review the PRs on =run.inputs.repo" } +//! one-off { "prompt": "review the PRs on acme/thing" } +//! ``` +//! +//! Both run. Both may satisfy the goal. Only the first is worth keeping, and +//! telling them apart needs no model and no guessing — which matters, because a +//! fuzzy gate on a store that grows forever is a store that fills with +//! near-misses. + +use serde_json::Value; +use tinyflows::model::WorkflowGraph; + +/// Length alone at which a value is distinctive enough to be evidence. +const LONG_ENOUGH: usize = 8; + +/// Characters that make a short value distinctive anyway. +/// +/// A path, a repository, an id, an address — `acme/thing`, `/docs/q3.pdf`, +/// `PROJ-1234`, `ops@example.com` all carry one. A bare short word does not. +const DISTINCTIVE_CHARS: [char; 6] = ['/', '.', ':', '@', '_', '-']; + +/// Whether finding this value in a config proves anything. +/// +/// `"1"`, `"true"`, `"main"` are values an input can legitimately carry *and* a +/// node can legitimately contain for unrelated reasons — `main` is the default +/// port name on every edge in the graph. Treating those as pasted would refuse +/// to keep perfectly reusable procedures, and a gate that fires on noise is one +/// nobody trusts. +fn distinctive(value: &str) -> bool { + let length = value.chars().count(); + // A digit only counts alongside some length: `"1"` proves nothing and, via + // the substring test, would match any config containing that character — + // reporting a paste and discarding a perfectly reusable procedure. + length >= LONG_ENOUGH + || value.contains(DISTINCTIVE_CHARS) + || (length >= 4 && value.chars().any(|c| c.is_ascii_digit())) +} + +/// Input values this graph pasted into a node instead of reading. +/// +/// Empty means it is reusable: every specific it was given arrives through a +/// declared input, so the same graph serves the next goal of this shape. +/// +/// Only leaf strings are examined. A value appearing as a *key* is not evidence +/// — a config may legitimately be keyed by something the goal also named. +#[must_use] +pub fn baked_in(graph: &WorkflowGraph, inputs: &serde_json::Map) -> Vec { + let distinctive: Vec<&str> = inputs + .values() + .filter_map(Value::as_str) + .map(str::trim) + .filter(|value| distinctive(value)) + .collect(); + if distinctive.is_empty() { + return Vec::new(); + } + + let mut found: Vec = Vec::new(); + for node in &graph.nodes { + let mut leaves = Vec::new(); + collect_strings(&node.config, &mut leaves); + for leaf in leaves { + for value in &distinctive { + // An expression that happens to mention the value is still + // reading it from somewhere; a literal is not. + if leaf.starts_with('=') { + continue; + } + if leaf.contains(value) && !found.iter().any(|f| f == value) { + found.push((*value).to_string()); + } + } + } + } + found +} + +/// A stable id for a graph's runnable shape. +/// +/// Derived rather than counted, so the same procedure arrived at twice +/// converges on one stored workflow instead of accumulating near-duplicates — +/// and so keeping needs no read of what already exists. +/// +/// The same digest the authoring fingerprint uses, for the same reason: nodes, +/// edges and declared inputs are what runs; the name and description are prose +/// a later pass may improve without making it a different procedure. +#[must_use] +pub fn shape_id(graph: &WorkflowGraph) -> String { + format!("learned-{}", digest_hex(&shape_bytes(graph))) +} + +/// The canonical bytes an identity is derived from. +pub(crate) fn shape_bytes(graph: &WorkflowGraph) -> Vec { + serde_json::to_vec(&serde_json::json!({ + "nodes": &graph.nodes, + "edges": &graph.edges, + "inputs": &graph.inputs, + })) + .unwrap_or_default() +} + +/// FNV-1a over the bytes, 64 bits, rendered as 16 hex chars. +/// +/// Not `DefaultHasher`: these digests become **persisted identifiers** — +/// workflow ids, lineage keys, exclusion-list signatures — and `DefaultHasher` +/// is explicitly unstable across Rust releases, so a toolchain upgrade would +/// silently stop identical work converging and orphan every stored score. The +/// old 28-bit truncation also put birthday collisions within reach of a few +/// tens of thousands of records; 64 bits does not. FNV-1a is fixed forever, +/// fits in six lines, and needs no dependency. +pub(crate) fn digest_hex(bytes: &[u8]) -> String { + let mut hash: u64 = 0xcbf2_9ce4_8422_2325; + for byte in bytes { + hash ^= u64::from(*byte); + hash = hash.wrapping_mul(0x0000_0100_0000_01b3); + } + format!("{hash:016x}") +} + +/// Every string leaf in a config, keys excluded. +fn collect_strings(value: &Value, out: &mut Vec) { + match value { + Value::String(text) => out.push(text.clone()), + Value::Array(items) => items.iter().for_each(|item| collect_strings(item, out)), + Value::Object(map) => map.values().for_each(|v| collect_strings(v, out)), + _ => {} + } +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + use tinyflows::model::{Node, NodeKind}; + + fn graph_with(config: Value) -> WorkflowGraph { + WorkflowGraph { + schema_version: 1, + id: Some("g".into()), + name: "g".into(), + inputs: Vec::new(), + agents: Vec::new(), + nodes: vec![Node { + id: "step".into(), + kind: NodeKind::Agent, + type_version: 1, + name: "step".into(), + config, + ports: Vec::new(), + position: None, + }], + edges: Vec::new(), + } + } + + fn inputs(pairs: &[(&str, &str)]) -> serde_json::Map { + pairs + .iter() + .map(|(k, v)| ((*k).to_string(), json!(v))) + .collect() + } + + #[test] + fn a_value_read_through_an_input_is_reusable() { + let graph = graph_with(json!({ "prompt": "review the PRs on =run.inputs.repo" })); + assert!(baked_in(&graph, &inputs(&[("repo", "acme/thing")])).is_empty()); + } + + #[test] + fn the_same_value_pasted_in_is_a_one_off() { + let graph = graph_with(json!({ "prompt": "review the PRs on acme/thing" })); + assert_eq!( + baked_in(&graph, &inputs(&[("repo", "acme/thing")])), + vec!["acme/thing"] + ); + } + + #[test] + fn it_looks_inside_nested_config_not_just_the_top_level() { + let graph = graph_with(json!({ + "args": { "targets": ["/docs/q3.pdf", "=run.inputs.other"] } + })); + assert_eq!( + baked_in(&graph, &inputs(&[("path", "/docs/q3.pdf")])), + vec!["/docs/q3.pdf"] + ); + } + + #[test] + fn an_expression_mentioning_the_value_is_still_reading_it() { + // `=run.inputs.repo | ascii_downcase` names nothing literally, but a jq + // program can contain the text and still be a binding rather than a + // paste. Anything starting with `=` is resolved at run time. + let graph = graph_with(json!({ "prompt": "=\"acme/thing\" " })); + assert!(baked_in(&graph, &inputs(&[("repo", "acme/thing")])).is_empty()); + } + + #[test] + fn plain_short_words_prove_nothing_and_are_not_evidence() { + // `main` is the default port name on every edge in the graph, so a node + // containing it says nothing about where the input went. A gate that + // fires on that refuses perfectly reusable procedures. + let graph = graph_with(json!({ "branch": "main", "mode": "on" })); + assert!(baked_in(&graph, &inputs(&[("branch", "main"), ("mode", "on")])).is_empty()); + } + + #[test] + fn a_bare_short_digit_is_not_evidence() { + // "1" appears in half of all configs; treating it as a paste would + // refuse reusable procedures on noise. + let graph = graph_with(json!({ "max_items": "10", "prompt": "top 1 result" })); + assert!(baked_in(&graph, &inputs(&[("n", "1"), ("count", "10")])).is_empty()); + } + + #[test] + fn a_short_value_with_structure_is_still_evidence() { + // Short but unmistakable: nothing else in a config is `a/b` or has a + // ticket number in it by coincidence. + for (key, value) in [("repo", "a/b"), ("ticket", "P-91")] { + let graph = graph_with(json!({ "prompt": format!("do {value}") })); + assert_eq!( + baked_in(&graph, &inputs(&[(key, value)])), + vec![value.to_string()], + "{value} should read as pasted" + ); + } + } + + #[test] + fn a_key_that_matches_is_not_a_paste() { + // Configs are keyed by field names, and a goal may name one. Only the + // values a node would send are evidence. + let graph = graph_with(json!({ "acme/thing": "=run.inputs.repo" })); + assert!(baked_in(&graph, &inputs(&[("repo", "acme/thing")])).is_empty()); + } + + #[test] + fn a_graph_with_no_inputs_at_all_is_reusable_by_default() { + // "summarise today's pull requests" has no parameters. Nothing was + // given, so nothing could have been baked in. + let graph = graph_with(json!({ "prompt": "summarise today's pull requests" })); + assert!(baked_in(&graph, &inputs(&[])).is_empty()); + } + + #[test] + fn the_digest_is_the_documented_algorithm_not_a_std_implementation_detail() { + // Pinned to FNV-1a's published test vectors: if this fails, persisted + // identifiers changed and every stored score is orphaned. + assert_eq!(digest_hex(b""), "cbf29ce484222325"); + assert_eq!(digest_hex(b"a"), "af63dc4c8601ec8c"); + } + + #[test] + fn every_pasted_value_is_reported_not_only_the_first() { + // A caller renders these into an explanation of why a graph was not + // kept, and one at a time turns that into a conversation. + let graph = graph_with(json!({ + "prompt": "review acme/thing at /docs/q3.pdf" + })); + let found = baked_in( + &graph, + &inputs(&[("repo", "acme/thing"), ("path", "/docs/q3.pdf")]), + ); + assert_eq!(found.len(), 2, "{found:?}"); + } +} diff --git a/crates/adaptive/src/storage.rs b/crates/adaptive/src/storage.rs new file mode 100644 index 0000000..76770de --- /dev/null +++ b/crates/adaptive/src/storage.rs @@ -0,0 +1,479 @@ +//! One config value, the whole persistence stack. +//! +//! The ledger and the vault are chosen the same way, from the same setting, so +//! every host was writing the same two `match`es — and scoping the two handles +//! separately, which is the leak waiting to happen: a request that calls +//! `for_tenant` on the ledger and forgets the vault has isolated the learning +//! and shared the graphs. +//! +//! [`Storage::open`] does the picking; [`Storage::for_tenant`] scopes **both +//! halves in one call**, so there is nothing to forget. +//! +//! ```text +//! "memory" → forgets on restart; tests, first look +//! "adaptive.db" or "sqlite:…" → one SQLite file holding BOTH halves +//! "mongodb://host/db" → one Mongo database holding both +//! ``` +//! +//! A URI for a backend this build does not carry fails **at parse time**, with +//! the feature named — a config error at boot, not a missing symbol at the +//! first write. + +use std::path::PathBuf; + +use async_trait::async_trait; +use tinyflows::store::types::{WorkflowError, WorkflowRecord}; + +use crate::execute::StepRecord; +use crate::ledger::memory::MemoryLedger; +use crate::ledger::{ + Episode, Ledger, LedgerRow, Lesson, LessonKind, Page, Result as LedgerResult, Score, +}; +use crate::workflows::Vault; +use crate::workflows::memory::MemoryVault; + +/// What went wrong turning a config value into storage. +#[derive(Debug, thiserror::Error)] +pub enum StorageError { + /// The value does not name a storage this build can open. + #[error("storage config: {0}")] + Config(String), + /// The ledger backend refused to open. + #[error("ledger: {0}")] + Ledger(#[from] crate::ledger::LedgerError), + /// The vault backend refused to open. + #[error("vault: {0}")] + Vault(#[from] WorkflowError), +} + +/// Where the storage setting is read from, when the environment supplies it. +pub const STORAGE_VAR: &str = "TINYFLOWS_ADAPTIVE_STORAGE"; + +/// Where everything durable goes, parsed from one setting. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Config { + /// In memory; forgets on restart. Never picked implicitly — the value has + /// to literally say `memory`. + Memory, + /// One SQLite file, holding the ledger and the vault side by side. + #[cfg(feature = "sqlite")] + Sqlite(PathBuf), + /// One MongoDB database, holding both. + #[cfg(feature = "mongo")] + Mongo { + /// The connection string, passed to the driver untouched. + uri: String, + /// The database name, taken from the URI's path or defaulted. + database: String, + }, +} + +impl Config { + /// Read a storage setting. + /// + /// * `memory` (or `:memory:`) — the ledger and vault that forget; + /// * `mongodb://…` / `mongodb+srv://…` — Mongo, database from the URI's + /// first path segment, `tinyflows_adaptive` when it has none; + /// * `sqlite:` — SQLite at that path; + /// * anything else — treated as a filesystem path, SQLite. + /// + /// Read the setting from the environment: [`STORAGE_VAR`]. + /// + /// An unset variable is an **error naming the variable**, never a default. + /// The tempting fallbacks are both wrong: defaulting to a disk location + /// invents a path on the operator's machine nobody named, and defaulting + /// to memory is a service that runs perfectly and learns nothing — the + /// failure shape this crate is built to refuse. + /// + /// # Errors + /// When the variable is unset, or its value fails [`parse`](Self::parse). + pub fn from_env() -> Result { + Self::from_setting(std::env::var(STORAGE_VAR).ok().as_deref()) + } + + /// [`from_env`](Self::from_env) with the read made explicit, so the rule is + /// testable without any test mutating process-wide state. + pub fn from_setting(value: Option<&str>) -> Result { + match value.map(str::trim).filter(|v| !v.is_empty()) { + Some(value) => Self::parse(value), + None => Err(StorageError::Config(format!( + "{STORAGE_VAR} is not set; expected `memory`, a sqlite path, or a mongodb:// URI" + ))), + } + } + + /// # Errors + /// When the value names a backend this build was compiled without — caught + /// here so it fails at boot with the feature named, not at first use. + pub fn parse(value: &str) -> Result { + let value = value.trim(); + if value.is_empty() { + return Err(StorageError::Config( + "empty storage setting; expected `memory`, a sqlite path, or a mongodb:// URI" + .to_string(), + )); + } + if value == "memory" || value == ":memory:" { + return Ok(Self::Memory); + } + if value.starts_with("mongodb://") || value.starts_with("mongodb+srv://") { + #[cfg(feature = "mongo")] + return Ok(Self::Mongo { + uri: value.to_string(), + database: mongo_database(value), + }); + #[cfg(not(feature = "mongo"))] + return Err(StorageError::Config( + "a mongodb:// URI, but this build has no `mongo` feature".to_string(), + )); + } + let path = value.strip_prefix("sqlite:").unwrap_or(value); + #[cfg(feature = "sqlite")] + return Ok(Self::Sqlite(PathBuf::from(path))); + #[cfg(not(feature = "sqlite"))] + { + let _ = path; + Err(StorageError::Config(format!( + "`{value}` reads as a sqlite path, but this build has no `sqlite` feature" + ))) + } + } +} + +/// The database named by a Mongo URI's first path segment, or the default. +#[cfg(feature = "mongo")] +fn mongo_database(uri: &str) -> String { + let after_scheme = uri.split_once("://").map_or(uri, |(_, rest)| rest); + let path = after_scheme.split_once('/').map(|(_, path)| path); + let database = path + .map(|p| p.split(['?', '/']).next().unwrap_or("")) + .unwrap_or(""); + if database.is_empty() { + "tinyflows_adaptive".to_string() + } else { + database.to_string() + } +} + +/// A ledger and a vault, opened from one [`Config`] and scoped together. +pub struct Storage { + ledger: AnyLedger, + vault: AnyVault, +} + +impl Storage { + /// [`Config::from_env`] and [`open`](Self::open) in one call — the whole + /// persistence stack from the environment. + /// + /// # Errors + /// As both halves. + pub async fn from_env() -> Result { + Self::open(&Config::from_env()?).await + } + + /// Open both halves of the configured backend. + /// + /// SQLite puts them in **one file** — their schemas share no table — so a + /// single-node deployment backs up exactly one thing. Mongo puts them in + /// one database, in separate collections. + /// + /// # Errors + /// When the backend cannot be opened or reached. + pub async fn open(config: &Config) -> Result { + Ok(match config { + Config::Memory => Self { + ledger: AnyLedger::Memory(MemoryLedger::new()), + vault: AnyVault::Memory(MemoryVault::new()), + }, + #[cfg(feature = "sqlite")] + Config::Sqlite(path) => Self { + ledger: AnyLedger::Sqlite(crate::ledger::sqlite::SqliteLedger::open(path)?), + vault: AnyVault::Sqlite(crate::workflows::sqlite::SqliteVault::open(path)?), + }, + #[cfg(feature = "mongo")] + Config::Mongo { uri, database } => Self { + ledger: AnyLedger::Mongo( + crate::ledger::mongo::MongoLedger::connect(uri, database).await?, + ), + vault: AnyVault::Mongo( + crate::workflows::mongo::MongoVault::connect(uri, database).await?, + ), + }, + }) + } + + /// Both halves, scoped to one tenant, in one call. + /// + /// One call rather than two because the failure this module exists to + /// prevent is scoping the ledger and forgetting the vault — isolated + /// learning over shared graphs, or the reverse. + #[must_use] + pub fn for_tenant(&self, scope: &str) -> Self { + Self { + ledger: self.ledger.for_tenant(scope), + vault: self.vault.for_tenant(scope), + } + } + + /// The ledger half. + #[must_use] + pub fn ledger(&self) -> &AnyLedger { + &self.ledger + } + + /// The vault half. + #[must_use] + pub fn vault(&self) -> &AnyVault { + &self.vault + } +} + +/// Whichever ledger the config picked, behind the one trait. +pub enum AnyLedger { + /// Forgets on restart. + Memory(MemoryLedger), + /// One SQLite file. + #[cfg(feature = "sqlite")] + Sqlite(crate::ledger::sqlite::SqliteLedger), + /// A MongoDB database. + #[cfg(feature = "mongo")] + Mongo(crate::ledger::mongo::MongoLedger), +} + +impl AnyLedger { + /// A handle onto the same store, scoped to one tenant. + #[must_use] + pub fn for_tenant(&self, scope: &str) -> Self { + match self { + Self::Memory(l) => Self::Memory(l.for_tenant(scope)), + #[cfg(feature = "sqlite")] + Self::Sqlite(l) => Self::Sqlite(l.for_tenant(scope)), + #[cfg(feature = "mongo")] + Self::Mongo(l) => Self::Mongo(l.for_tenant(scope)), + } + } +} + +/// Delegate one call to whichever backend is inside. +macro_rules! on_ledger { + ($self:ident, $l:ident => $call:expr) => { + match $self { + AnyLedger::Memory($l) => $call, + #[cfg(feature = "sqlite")] + AnyLedger::Sqlite($l) => $call, + #[cfg(feature = "mongo")] + AnyLedger::Mongo($l) => $call, + } + }; +} + +#[async_trait] +impl Ledger for AnyLedger { + fn scope(&self) -> Option<&str> { + on_ledger!(self, l => l.scope()) + } + async fn append(&self, row: &LedgerRow) -> LedgerResult { + on_ledger!(self, l => l.append(row).await) + } + async fn rows(&self, episode: &str) -> LedgerResult> { + on_ledger!(self, l => l.rows(episode).await) + } + async fn promote(&self, lesson: &Lesson, cites: &[String]) -> LedgerResult { + on_ledger!(self, l => l.promote(lesson, cites).await) + } + async fn lessons(&self, kind: Option) -> LedgerResult> { + on_ledger!(self, l => l.lessons(kind).await) + } + async fn evidence(&self, lesson_id: &str) -> LedgerResult> { + on_ledger!(self, l => l.evidence(lesson_id).await) + } + async fn score_lesson(&self, lesson_id: &str, helped: bool) -> LedgerResult<()> { + on_ledger!(self, l => l.score_lesson(lesson_id, helped).await) + } + async fn score_workflow(&self, workflow_id: &str, helped: bool) -> LedgerResult<()> { + on_ledger!(self, l => l.score_workflow(workflow_id, helped).await) + } + async fn workflow_score(&self, workflow_id: &str) -> LedgerResult { + on_ledger!(self, l => l.workflow_score(workflow_id).await) + } + async fn link_variant(&self, parent: &str, variant: &str) -> LedgerResult<()> { + on_ledger!(self, l => l.link_variant(parent, variant).await) + } + async fn parent_of(&self, id: &str) -> LedgerResult> { + on_ledger!(self, l => l.parent_of(id).await) + } + async fn children_of(&self, id: &str) -> LedgerResult> { + on_ledger!(self, l => l.children_of(id).await) + } + async fn save_episode(&self, episode: &Episode) -> LedgerResult<()> { + on_ledger!(self, l => l.save_episode(episode).await) + } + async fn episode(&self, id: &str) -> LedgerResult> { + on_ledger!(self, l => l.episode(id).await) + } + async fn episodes(&self, running_only: bool, page: Page) -> LedgerResult> { + on_ledger!(self, l => l.episodes(running_only, page).await) + } + async fn save_steps(&self, row_id: &str, steps: &[StepRecord]) -> LedgerResult<()> { + on_ledger!(self, l => l.save_steps(row_id, steps).await) + } + async fn steps(&self, row_id: &str) -> LedgerResult> { + on_ledger!(self, l => l.steps(row_id).await) + } +} + +/// Whichever vault the config picked, behind the one trait. +pub enum AnyVault { + /// Forgets on restart. + Memory(MemoryVault), + /// One SQLite file — the same one the ledger may use. + #[cfg(feature = "sqlite")] + Sqlite(crate::workflows::sqlite::SqliteVault), + /// A MongoDB database. + #[cfg(feature = "mongo")] + Mongo(crate::workflows::mongo::MongoVault), +} + +impl AnyVault { + /// A handle onto the same store, scoped to one tenant. + #[must_use] + pub fn for_tenant(&self, scope: &str) -> Self { + match self { + Self::Memory(v) => Self::Memory(v.for_tenant(scope)), + #[cfg(feature = "sqlite")] + Self::Sqlite(v) => Self::Sqlite(v.for_tenant(scope)), + #[cfg(feature = "mongo")] + Self::Mongo(v) => Self::Mongo(v.for_tenant(scope)), + } + } +} + +macro_rules! on_vault { + ($self:ident, $v:ident => $call:expr) => { + match $self { + AnyVault::Memory($v) => $call, + #[cfg(feature = "sqlite")] + AnyVault::Sqlite($v) => $call, + #[cfg(feature = "mongo")] + AnyVault::Mongo($v) => $call, + } + }; +} + +#[async_trait] +impl Vault for AnyVault { + fn scope(&self) -> Option<&str> { + on_vault!(self, v => v.scope()) + } + async fn load(&self) -> Result, WorkflowError> { + on_vault!(self, v => v.load().await) + } + async fn put(&self, record: &WorkflowRecord) -> Result<(), WorkflowError> { + on_vault!(self, v => v.put(record).await) + } + async fn remove(&self, id: &str) -> Result<(), WorkflowError> { + on_vault!(self, v => v.remove(id).await) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn memory_has_to_be_asked_for_by_name() { + assert_eq!(Config::parse("memory").expect("parse"), Config::Memory); + assert_eq!(Config::parse(":memory:").expect("parse"), Config::Memory); + } + + #[cfg(feature = "sqlite")] + #[test] + fn a_bare_path_reads_as_sqlite() { + assert_eq!( + Config::parse("/var/lib/app/adaptive.db").expect("parse"), + Config::Sqlite(PathBuf::from("/var/lib/app/adaptive.db")) + ); + assert_eq!( + Config::parse("sqlite:./adaptive.db").expect("parse"), + Config::Sqlite(PathBuf::from("./adaptive.db")) + ); + } + + #[cfg(feature = "mongo")] + #[test] + fn a_mongo_uri_carries_its_database_or_gets_the_default() { + match Config::parse("mongodb://db.internal:27017/adaptive?replicaSet=rs0").expect("parse") { + Config::Mongo { database, .. } => assert_eq!(database, "adaptive"), + other => panic!("{other:?}"), + } + match Config::parse("mongodb+srv://cluster.example.net").expect("parse") { + Config::Mongo { database, .. } => assert_eq!(database, "tinyflows_adaptive"), + other => panic!("{other:?}"), + } + } + + #[test] + fn an_unset_variable_errors_naming_the_variable_rather_than_defaulting() { + // Defaulting to a path invents a location nobody named; defaulting to + // memory is a service that runs perfectly and learns nothing. + let err = Config::from_setting(None).expect_err("unset"); + assert!(err.to_string().contains(STORAGE_VAR), "{err}"); + let err = Config::from_setting(Some(" ")).expect_err("blank is unset"); + assert!(err.to_string().contains(STORAGE_VAR), "{err}"); + } + + #[test] + fn a_set_variable_goes_through_the_same_parse() { + assert_eq!( + Config::from_setting(Some("memory")).expect("parse"), + Config::Memory + ); + } + + #[test] + fn an_empty_setting_is_an_error_that_lists_the_choices() { + let err = Config::parse(" ").expect_err("empty"); + assert!(err.to_string().contains("memory"), "{err}"); + } + + #[tokio::test] + async fn one_call_scopes_both_halves() { + // The failure this module exists to prevent: scoping the ledger and + // forgetting the vault, or the reverse. + let storage = Storage::open(&Config::Memory).await.expect("open"); + let tenant = storage.for_tenant("user-7"); + assert_eq!(tenant.ledger().scope(), Some("user-7")); + assert_eq!(tenant.vault().scope(), Some("user-7")); + assert_eq!(storage.ledger().scope(), None, "the root stays unscoped"); + } + + #[cfg(feature = "sqlite")] + #[tokio::test] + async fn one_sqlite_setting_yields_one_file_holding_both_halves() { + let dir = std::env::temp_dir().join(format!("adaptive-storage-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + let path = dir.join("adaptive.db"); + + let config = Config::parse(path.to_str().expect("utf8 path")).expect("parse"); + let storage = Storage::open(&config).await.expect("open"); + let tenant = storage.for_tenant("user-7"); + + tenant + .ledger() + .append(&crate::ledger::conformance::row("ep-1", 1, "authored")) + .await + .expect("append"); + tenant + .vault() + .put(&crate::workflows::conformance::record("weekly")) + .await + .expect("put"); + + // Reopen from the same setting: both halves are still there, scoped. + let again = Storage::open(&config).await.expect("reopen"); + let tenant = again.for_tenant("user-7"); + assert_eq!(tenant.ledger().rows("ep-1").await.expect("rows").len(), 1); + assert_eq!(tenant.vault().load().await.expect("load").len(), 1); + let _ = std::fs::remove_dir_all(&dir); + } +} diff --git a/crates/adaptive/src/workflows/compat.rs b/crates/adaptive/src/workflows/compat.rs new file mode 100644 index 0000000..98709b7 --- /dev/null +++ b/crates/adaptive/src/workflows/compat.rs @@ -0,0 +1,399 @@ +//! Reaching workflows that already exist somewhere else. +//! +//! The loop's own procedures live in a [`Vault`]. Everyone else's live in a +//! [`WorkflowStore`] — the engine's file store, a host's own implementation, a +//! device's local catalogue. Those are the same records; only the way in +//! differs, and without a way in the loop can only ever select what it wrote +//! itself. +//! +//! Two adapters, and between them the loop reads any catalogue that exists. +//! +//! [`StoreVault`] makes any `WorkflowStore` a `Vault`, so nothing has to be +//! rewritten or migrated to be selectable. +//! +//! [`Layered`] reads several and writes one. That is the shape that solves the +//! problem importing otherwise creates: a device's catalogue is **read-only**, +//! so a workflow of theirs can be selected, judged and scored, and when it +//! falls short the repaired variant lands in *our* writable layer with its own +//! id. Their copy is never touched, so there is no second master and no +//! question of whose version is current. + +use std::sync::Arc; + +use async_trait::async_trait; +use tinyflows::store::WorkflowStore; +use tinyflows::store::types::{WorkflowError, WorkflowRecord}; + +use super::Vault; + +/// Any [`WorkflowStore`] as a [`Vault`]. +/// +/// Unscoped, and it cannot be otherwise: the engine's store has no tenant +/// concept to filter on. So scoping here is **by construction** — build one +/// per tenant over that tenant's own store. An unscoped vault's records read as +/// global, which is right for a shared catalogue and wrong for a device's, so +/// this is worth getting right at the call site. +pub struct StoreVault { + inner: Arc, +} + +impl StoreVault { + /// Wrap a store. + #[must_use] + pub fn new(inner: Arc) -> Self { + Self { inner } + } +} + +#[async_trait] +impl Vault for StoreVault { + async fn load(&self) -> Result, WorkflowError> { + // `list` gives summaries, so each record is a second call. One pass per + // episode over a catalogue of tens, against a store that is already + // synchronous and therefore local. + let mut out = Vec::new(); + for summary in self.inner.list()? { + if let Some(record) = self.inner.get(&summary.id)? { + out.push(record); + } + } + Ok(out) + } + + async fn put(&self, record: &WorkflowRecord) -> Result<(), WorkflowError> { + self.inner.save(record) + } + + async fn remove(&self, id: &str) -> Result<(), WorkflowError> { + self.inner.delete(id) + } +} + +/// Told which read-only layer could not be read, and why. +pub type OnUnavailable = Arc; + +/// Several catalogues to read, one to write. +/// +/// Reads are the union, and **later layers shadow earlier ones** by id — so +/// order the writable layer last and a copy we have taken ownership of wins +/// over the original it came from. +/// +/// Writes go only to the writable layer, which is the whole point. A variant of +/// somebody else's workflow is ours; their record is evidence, not something to +/// edit. +/// +/// # When a layer cannot be read +/// +/// [`new`](Self::new) is **strict**: any failure fails the load, and therefore +/// the episode. That is right when every layer is a database you own. +/// +/// It is wrong the moment a layer is a device. Fetching a device's catalogue +/// per episode is cheap and keeps it current, but a device is sometimes asleep, +/// and a machine being asleep must not stop a tenant's goals — their own +/// procedures are in another layer and perfectly readable. +/// +/// [`degrading`](Self::degrading) skips a read-only layer that errors. It +/// **requires a handler**, and that is deliberate: a catalogue that quietly +/// vanishes is this crate's worst failure shape — the loop runs, authors from +/// scratch, and looks like it is working. You cannot have the degradation +/// without being told each time it happens. +/// +/// The writable layer is fatal either way. It is your own store, and a loop +/// that cannot read its own procedures should stop rather than relearn them. +pub struct Layered { + /// Consulted in order, each shadowing the last. Named so a report can say + /// which one was missing. + read_only: Vec<(String, Arc)>, + /// Read last, and the only one written to. + writable: Arc, + /// Set by [`degrading`](Self::degrading). `None` means strict. + on_unavailable: Option, +} + +impl Layered { + /// Read `read_only` in order, then `writable`; write only `writable`. + /// + /// Strict: an unreadable layer fails the load. + #[must_use] + pub fn new(read_only: Vec<(String, Arc)>, writable: Arc) -> Self { + Self { + read_only, + writable, + on_unavailable: None, + } + } + + /// Skip a read-only layer that cannot be read, telling `on_unavailable`. + /// + /// For layers that are somebody else's machine. See the type note on why + /// the handler is required rather than optional. + #[must_use] + pub fn degrading(mut self, on_unavailable: OnUnavailable) -> Self { + self.on_unavailable = Some(on_unavailable); + self + } +} + +#[async_trait] +impl Vault for Layered { + fn scope(&self) -> Option<&str> { + // The scope that matters is the one writes land in. A read-only layer + // may be unscoped — a device store has no tenant concept — and + // reporting *that* would understate who this handle belongs to. + self.writable.scope() + } + + async fn load(&self) -> Result, WorkflowError> { + let mut merged: std::collections::BTreeMap = + std::collections::BTreeMap::new(); + for (name, layer) in &self.read_only { + let records = match (layer.load().await, self.on_unavailable.as_ref()) { + (Ok(records), _) => records, + // Skipped, and reported. A device asleep is a catalogue we do + // not have this episode, not a tenant who cannot run anything. + (Err(why), Some(tell)) => { + tell(name, &why); + continue; + } + (Err(why), None) => return Err(why), + }; + for record in records { + merged.insert(record.id.clone(), record); + } + } + // Last, so ours wins an id collision. + for record in self.writable.load().await? { + merged.insert(record.id.clone(), record); + } + Ok(merged.into_values().collect()) + } + + async fn put(&self, record: &WorkflowRecord) -> Result<(), WorkflowError> { + self.writable.put(record).await + } + + async fn remove(&self, id: &str) -> Result<(), WorkflowError> { + // Only ever ours. Removing from a read-only layer would delete + // something on a machine that never asked. + self.writable.remove(id).await + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::workflows::conformance::record; + use crate::workflows::memory::MemoryVault; + + async fn layer(ids: &[&str]) -> Arc { + let vault = Arc::new(MemoryVault::new()); + for id in ids { + vault.put(&record(id)).await.expect("put"); + } + vault + } + + #[tokio::test] + async fn a_plain_store_becomes_selectable_without_being_migrated() { + let dir = std::env::temp_dir().join(format!("adaptive-compat-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(dir.join("workflows")).expect("temp dir"); + let store: Arc = Arc::new(tinyflows::store::FileWorkflowStore::new( + vec![dir.join("workflows")], + dir.join("runs"), + )); + store.save(&record("theirs")).expect("save"); + + let vault = StoreVault::new(store); + let loaded = vault.load().await.expect("load"); + assert_eq!(loaded.len(), 1); + assert_eq!(loaded[0].description, "does the theirs thing"); + let _ = std::fs::remove_dir_all(&dir); + } + + #[tokio::test] + async fn reads_are_the_union_of_every_layer() { + let theirs = layer(&["device-a", "device-b"]).await; + let ours = layer(&["learned-1"]).await; + let stack = Layered::new(vec![("device".into(), theirs)], ours); + + let mut ids: Vec = stack + .load() + .await + .expect("load") + .into_iter() + .map(|r| r.id) + .collect(); + ids.sort(); + assert_eq!(ids, ["device-a", "device-b", "learned-1"]); + } + + #[tokio::test] + async fn what_we_wrote_shadows_what_we_read() { + let theirs = Arc::new(MemoryVault::new()); + let mut original = record("shared-id"); + original.description = "the device's version".into(); + theirs.put(&original).await.expect("put"); + + let ours = Arc::new(MemoryVault::new()); + let mut taken = record("shared-id"); + taken.description = "the copy we took ownership of".into(); + ours.put(&taken).await.expect("put"); + + let stack = Layered::new(vec![("device".into(), theirs)], ours); + let loaded = stack.load().await.expect("load"); + assert_eq!(loaded.len(), 1, "one id, one record"); + assert_eq!(loaded[0].description, "the copy we took ownership of"); + } + + #[tokio::test] + async fn a_variant_of_their_workflow_lands_in_our_layer_not_theirs() { + // The reason this is layered rather than merged. Their catalogue is + // evidence; the repair is ours, and their machine never changes. + let theirs = Arc::new(MemoryVault::new()); + theirs.put(&record("device-weekly")).await.expect("put"); + let ours = Arc::new(MemoryVault::new()); + + let stack = Layered::new(vec![("device".into(), theirs.clone())], ours.clone()); + stack + .put(&record("device-weekly-fix-a1b2c3d")) + .await + .expect("put"); + + assert_eq!( + theirs.load().await.expect("load").len(), + 1, + "their catalogue is untouched" + ); + assert_eq!(ours.load().await.expect("load").len(), 1, "ours gained it"); + } + + #[tokio::test] + async fn a_delete_never_reaches_a_read_only_layer() { + // Otherwise the loop could remove a workflow from a machine that never + // asked it to. + let theirs = Arc::new(MemoryVault::new()); + theirs.put(&record("device-weekly")).await.expect("put"); + let stack = Layered::new( + vec![("device".into(), theirs.clone())], + Arc::new(MemoryVault::new()), + ); + + stack.remove("device-weekly").await.expect("remove"); + assert_eq!( + theirs.load().await.expect("load").len(), + 1, + "still theirs, still there" + ); + } + + #[tokio::test] + async fn the_scope_reported_is_the_one_writes_land_in() { + // A device store has no tenant concept, so a read-only layer over it is + // unscoped. Reporting that would understate who the handle belongs to. + let unscoped_device = Arc::new(MemoryVault::new()); + let ours = Arc::new(MemoryVault::new().for_tenant("user-7")); + let stack = Layered::new(vec![("device".into(), unscoped_device)], ours); + assert_eq!(stack.scope(), Some("user-7")); + } +} + +#[cfg(test)] +mod degradation_tests { + use super::*; + use crate::workflows::conformance::record; + use crate::workflows::memory::MemoryVault; + use std::sync::Mutex; + + /// A layer that is asleep. + struct Offline; + + #[async_trait] + impl Vault for Offline { + async fn load(&self) -> Result, WorkflowError> { + Err(WorkflowError::Engine("device not connected".into())) + } + async fn put(&self, _record: &WorkflowRecord) -> Result<(), WorkflowError> { + Err(WorkflowError::Engine("device not connected".into())) + } + async fn remove(&self, _id: &str) -> Result<(), WorkflowError> { + Err(WorkflowError::Engine("device not connected".into())) + } + } + + async fn ours_with(id: &str) -> Arc { + let vault = Arc::new(MemoryVault::new()); + vault.put(&record(id)).await.expect("put"); + vault + } + + #[tokio::test] + async fn strict_is_the_default_and_an_unreadable_layer_fails_the_load() { + // Right when every layer is a database you own: a store that will not + // answer is a fault, not a shrug. + let stack = Layered::new( + vec![("db".into(), Arc::new(Offline))], + ours_with("learned-1").await, + ); + assert!(stack.load().await.is_err()); + } + + #[tokio::test] + async fn a_sleeping_device_costs_its_catalogue_and_nothing_else() { + // The case per-episode fetching creates. Without this, one machine + // being asleep stops every goal that tenant has, though their own + // procedures are in another layer and perfectly readable. + let told: Arc>> = Arc::new(Mutex::new(Vec::new())); + let sink = Arc::clone(&told); + + let stack = Layered::new( + vec![("device".into(), Arc::new(Offline))], + ours_with("learned-1").await, + ) + .degrading(Arc::new(move |name: &str, why: &WorkflowError| { + sink.lock().expect("lock").push(format!("{name}: {why}")); + })); + + let loaded = stack.load().await.expect("the episode still starts"); + assert_eq!(loaded.len(), 1); + assert_eq!(loaded[0].id, "learned-1", "our own catalogue survives"); + + let told = told.lock().expect("lock").clone(); + assert_eq!(told.len(), 1, "and it did not happen quietly"); + assert!(told[0].contains("device"), "{}", told[0]); + assert!(told[0].contains("not connected"), "{}", told[0]); + } + + #[tokio::test] + async fn the_writable_layer_is_fatal_even_when_degrading() { + // Our own store. A loop that cannot read the procedures it wrote should + // stop, not quietly relearn them and file duplicates. + let stack = Layered::new( + vec![("device".into(), ours_with("device-1").await)], + Arc::new(Offline), + ) + .degrading(Arc::new(|_: &str, _: &WorkflowError| {})); + assert!(stack.load().await.is_err()); + } + + #[tokio::test] + async fn one_layer_failing_does_not_hide_the_others() { + let told: Arc> = Arc::new(Mutex::new(0)); + let sink = Arc::clone(&told); + + let stack = Layered::new( + vec![ + ("device-a".into(), Arc::new(Offline)), + ("device-b".into(), ours_with("device-b-1").await), + ], + ours_with("learned-1").await, + ) + .degrading(Arc::new(move |_: &str, _: &WorkflowError| { + *sink.lock().expect("lock") += 1; + })); + + let loaded = stack.load().await.expect("load"); + assert_eq!(loaded.len(), 2, "b and ours: {loaded:?}"); + assert_eq!(*told.lock().expect("lock"), 1, "only a was missing"); + } +} diff --git a/crates/adaptive/src/workflows/conformance.rs b/crates/adaptive/src/workflows/conformance.rs new file mode 100644 index 0000000..2fd372a --- /dev/null +++ b/crates/adaptive/src/workflows/conformance.rs @@ -0,0 +1,162 @@ +//! One suite every vault backend passes. +//! +//! Public for the same reason [`crate::ledger::conformance`] is: a host writing +//! a fourth backend runs the identical cases against it, so "it works on +//! sqlite" cannot quietly mean "it works only on sqlite". + +use tinyflows::model::{Node, NodeKind, WorkflowGraph}; +use tinyflows::store::types::{WorkflowDefaults, WorkflowRecord}; + +use super::Vault; + +/// A record that a validating store would accept. +#[must_use] +pub fn record(id: &str) -> WorkflowRecord { + WorkflowRecord { + id: id.to_string(), + name: id.to_string(), + description: format!("does the {id} thing"), + enabled: true, + defaults: WorkflowDefaults::default(), + graph: WorkflowGraph { + schema_version: 1, + id: Some(id.to_string()), + name: id.to_string(), + inputs: Vec::new(), + agents: Vec::new(), + nodes: vec![Node { + id: "start".to_string(), + kind: NodeKind::Trigger, + type_version: 1, + name: "manual".to_string(), + config: serde_json::json!({ "trigger_kind": "manual" }), + ports: Vec::new(), + position: None, + }], + edges: Vec::new(), + }, + source_path: None, + } +} + +/// Run every case against `vault`. +/// +/// # Panics +/// On any conformance failure. +pub async fn run_all(vault: &dyn Vault) { + an_empty_vault_loads_nothing_rather_than_erroring(vault).await; + a_record_round_trips_whole(vault).await; + putting_the_same_id_twice_replaces_rather_than_duplicating(vault).await; + removing_what_is_not_there_is_not_an_error(vault).await; + a_removed_record_stops_loading(vault).await; +} + +async fn an_empty_vault_loads_nothing_rather_than_erroring(vault: &dyn Vault) { + assert!( + vault + .load() + .await + .expect("load") + .iter() + .all(|r| r.id != "never-written"), + "nothing was written under that id" + ); +} + +async fn a_record_round_trips_whole(vault: &dyn Vault) { + let mut want = record("wf-round"); + want.description = "carries prose a planner reads".to_string(); + want.enabled = false; + vault.put(&want).await.expect("put"); + + let got = vault + .load() + .await + .expect("load") + .into_iter() + .find(|r| r.id == "wf-round") + .expect("stored"); + assert_eq!(got.description, want.description); + assert!(!got.enabled, "an operator's off switch survives the trip"); + assert_eq!(got.graph.nodes.len(), 1, "the graph is the point"); + assert_eq!(got.graph.nodes[0].kind, NodeKind::Trigger); +} + +async fn putting_the_same_id_twice_replaces_rather_than_duplicating(vault: &dyn Vault) { + // Two episodes arriving at the same procedure write the same content-derived + // id. That must converge, not accumulate. + vault.put(&record("wf-twice")).await.expect("put"); + vault.put(&record("wf-twice")).await.expect("put"); + assert_eq!( + vault + .load() + .await + .expect("load") + .iter() + .filter(|r| r.id == "wf-twice") + .count(), + 1 + ); +} + +async fn removing_what_is_not_there_is_not_an_error(vault: &dyn Vault) { + vault.remove("wf-absent").await.expect("remove"); +} + +async fn a_removed_record_stops_loading(vault: &dyn Vault) { + vault.put(&record("wf-gone")).await.expect("put"); + vault.remove("wf-gone").await.expect("remove"); + assert!( + !vault + .load() + .await + .expect("load") + .iter() + .any(|r| r.id == "wf-gone"), + "a delete that only hides the row is a delete nobody can trust" + ); +} + +/// Run every tenant-isolation case. Three handles onto one store. +/// +/// # Panics +/// On any isolation failure — each is one tenant's procedure appearing in +/// another's catalogue. +pub async fn run_tenants(global: &dyn Vault, a: &dyn Vault, b: &dyn Vault) { + assert_eq!(global.scope(), None); + assert_ne!(a.scope(), b.scope()); + + a.put(&record("wf-mine")).await.expect("put"); + assert!( + a.load() + .await + .expect("load") + .iter() + .any(|r| r.id == "wf-mine"), + "a tenant sees its own" + ); + assert!( + !b.load() + .await + .expect("load") + .iter() + .any(|r| r.id == "wf-mine"), + "tenant {:?} can read tenant {:?}'s workflow", + b.scope(), + a.scope() + ); + + global.put(&record("wf-shared")).await.expect("put"); + for tenant in [a, b] { + assert!( + tenant + .load() + .await + .expect("load") + .iter() + .any(|r| r.id == "wf-shared"), + "tenant {:?} cannot see a global workflow", + tenant.scope() + ); + } +} diff --git a/crates/adaptive/src/workflows/memory.rs b/crates/adaptive/src/workflows/memory.rs new file mode 100644 index 0000000..709dbc6 --- /dev/null +++ b/crates/adaptive/src/workflows/memory.rs @@ -0,0 +1,105 @@ +//! A vault that forgets, for tests and for a first look. +//! +//! Same posture as [`crate::ledger::memory`]: always compiled, never the +//! default, named for what it does. + +use std::collections::BTreeMap; +use std::sync::{Arc, Mutex}; + +use async_trait::async_trait; +use tinyflows::store::types::{WorkflowError, WorkflowRecord}; + +use super::Vault; + +/// A vault held in memory, which keeps nothing across restarts. +#[derive(Clone, Default)] +pub struct MemoryVault { + /// `(bucket, id)` so scoping behaves exactly as the durable backends do. + inner: Arc>>, + scope: Option, +} + +impl MemoryVault { + /// An empty vault. + #[must_use] + pub fn new() -> Self { + Self::default() + } + + /// A handle onto the same store, scoped to one tenant. + #[must_use] + pub fn for_tenant(&self, scope: impl Into) -> Self { + Self { + inner: Arc::clone(&self.inner), + scope: Some(scope.into()), + } + } + + fn bucket(&self) -> String { + self.scope.clone().unwrap_or_default() + } +} + +#[async_trait] +impl Vault for MemoryVault { + fn scope(&self) -> Option<&str> { + self.scope.as_deref() + } + + async fn load(&self) -> Result, WorkflowError> { + let bucket = self.bucket(); + let inner = self + .inner + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + // This bucket plus global, one record per id, and the bucket's own + // record shadows a global one — explicitly, not as an accident of map + // iteration order. + let mut chosen: std::collections::BTreeMap = + std::collections::BTreeMap::new(); + for ((scope, id), record) in inner.iter() { + if scope.is_empty() { + chosen.insert(id.clone(), record.clone()); + } + } + for ((scope, id), record) in inner.iter() { + if !bucket.is_empty() && scope == &bucket { + chosen.insert(id.clone(), record.clone()); + } + } + Ok(chosen.into_values().collect()) + } + + async fn put(&self, record: &WorkflowRecord) -> Result<(), WorkflowError> { + self.inner + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .insert((self.bucket(), record.id.clone()), record.clone()); + Ok(()) + } + + async fn remove(&self, id: &str) -> Result<(), WorkflowError> { + self.inner + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .remove(&(self.bucket(), id.to_string())); + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::workflows::conformance; + + #[tokio::test] + async fn passes_the_conformance_suite() { + conformance::run_all(&MemoryVault::new()).await; + } + + #[tokio::test] + async fn passes_the_tenant_isolation_suite() { + let vault = MemoryVault::new(); + conformance::run_tenants(&vault, &vault.for_tenant("a"), &vault.for_tenant("b")).await; + } +} diff --git a/crates/adaptive/src/workflows/mod.rs b/crates/adaptive/src/workflows/mod.rs new file mode 100644 index 0000000..103c57c --- /dev/null +++ b/crates/adaptive/src/workflows/mod.rs @@ -0,0 +1,443 @@ +//! Workflows in whatever the host configured, behind the engine's own trait. +//! +//! The ledger has three backends chosen at boot. Workflows had one — a +//! directory of JSON files — which is wrong for a hosted service and wrong for +//! the symmetry: a deployment picks Mongo for one half of its durable state and +//! gets a filesystem for the other. +//! +//! # Why this is a snapshot and not a store +//! +//! [`tinyflows::store::WorkflowStore`] is **synchronous** — ten required +//! methods, none of them `async`. A Mongo driver is not. The obvious fixes are +//! both bad: `block_on` inside a sync method deadlocks a current-thread +//! runtime, and async-ifying the trait upstream means rewriting the file store +//! and the authoring module and then contending with that rewrite on every +//! merge from upstream. The fork stays mergeable or it stops being a fork. +//! +//! So the async half is ours ([`Vault`]) and the sync half is a snapshot over +//! it: load once, serve every read from memory, buffer writes, flush after. +//! That fits how the loop actually uses a store — a handful of reads while +//! deciding, at most one or two writes when closing — and it makes the reads +//! free rather than a round trip each. +//! +//! # Two things fall out of it +//! +//! **Workflows become tenant-scoped**, which they were not. The engine's store +//! has no scope, so a repaired variant of one tenant's workflow appeared in +//! every tenant's catalogue. A `Vault` is scoped like a `Ledger`, so this +//! closes that as a side effect rather than as a separate feature. +//! +//! **Concurrent flushes are safe by construction**, because every id this crate +//! writes is content-derived — [`crate::reuse::shape_id`] for a learned graph, a +//! digest of the edits for a variant. Two episodes that arrive at the same +//! procedure write the same id with byte-identical content, so last-write-wins +//! is not a lost update. A snapshot only flushes what it actually changed, so a +//! human editing a workflow the loop never touched is never clobbered. + +use std::collections::BTreeMap; +use std::sync::{Arc, Mutex}; + +use async_trait::async_trait; +use tinyflows::store::types::{ + RunRecord, WorkflowError, WorkflowRecord, WorkflowRevision, WorkflowSummary, +}; +use tinyflows::store::{HostPolicy, WorkflowStore}; + +pub mod memory; +#[cfg(feature = "mongo")] +pub mod mongo; +#[cfg(feature = "sqlite")] +pub mod sqlite; + +pub mod compat; +pub mod conformance; + +/// Durable workflow storage, in whatever the host configured. +/// +/// Deliberately narrower than [`WorkflowStore`]: load everything, write one, +/// delete one. Run records, revisions, notes and proposals are the engine's +/// authoring surface and this crate neither reads nor writes them — a `Vault` +/// that had to implement them would be ten methods of `unimplemented!` in every +/// backend. +#[async_trait] +pub trait Vault: Send + Sync { + /// Whose workflows this handle sees. `None` is the global bucket, and the + /// rule is the ledger's: writes go to this bucket, reads return this bucket + /// plus global. + fn scope(&self) -> Option<&str> { + None + } + + /// Every workflow in scope, **at most one record per id**: when the same + /// id exists in this handle's bucket and in global, the handle's own wins. + /// + /// The precedence is each backend's obligation rather than the caller's, + /// because the caller dedupes by id in arrival order — leaving it to + /// storage iteration order would make "whose record wins" an + /// implementation accident that differs across backends. + /// + /// The whole catalogue in one call, because a snapshot loads once and a + /// tenant's procedures number in the tens, not the millions. A host that + /// outgrows that wants a different seam, not a paged version of this one. + /// + /// # Errors + /// When the backend is unreachable or holds a record that no longer parses. + async fn load(&self) -> Result, WorkflowError>; + + /// Write one, replacing any with the same id. + /// + /// # Errors + /// When the backend refuses the write. + async fn put(&self, record: &WorkflowRecord) -> Result<(), WorkflowError>; + + /// Remove one. Removing what is not there is not an error. + /// + /// # Errors + /// When the backend refuses. + async fn remove(&self, id: &str) -> Result<(), WorkflowError>; +} + +/// The engine's synchronous store, served from memory. +/// +/// Cheap to clone — clones share the same buffer, so a `Snapshot` handed to the +/// loop as `Arc` and the one you flush are the same state. +#[derive(Clone)] +pub struct Snapshot { + records: Arc>>, + /// Ids written or deleted through the sync surface. Only these are flushed, + /// so a concurrent editor of an untouched workflow is never clobbered. + dirty: Arc>>>, + policy: Arc, +} + +impl Snapshot { + /// Read a vault into memory. + /// + /// # Errors + /// When the vault cannot be read. + pub async fn load( + vault: &dyn Vault, + policy: Arc, + ) -> Result { + let records = vault + .load() + .await? + .into_iter() + .map(|record| (record.id.clone(), record)) + .collect(); + Ok(Self { + records: Arc::new(Mutex::new(records)), + dirty: Arc::new(Mutex::new(BTreeMap::new())), + policy, + }) + } + + /// An empty snapshot, for a caller with nothing stored yet. + #[must_use] + pub fn empty(policy: Arc) -> Self { + Self { + records: Arc::new(Mutex::new(BTreeMap::new())), + dirty: Arc::new(Mutex::new(BTreeMap::new())), + policy, + } + } + + /// Push everything written since the load back to the vault. + /// + /// Only what changed: a workflow the loop read and did not touch is not + /// rewritten, so this cannot undo an edit someone else made in the + /// meantime. + /// + /// Clears the dirty set on success, so flushing twice is not two writes. + /// + /// # Errors + /// On the first write the vault refuses. Earlier writes stand — this is not + /// a transaction, and pretending otherwise across three backends with + /// different guarantees would be a lie. + pub async fn flush(&self, vault: &dyn Vault) -> Result { + let pending: Vec<(String, Option)> = { + let dirty = self.guard_dirty(); + dirty.iter().map(|(k, v)| (k.clone(), v.clone())).collect() + }; + let mut written = 0; + for (id, record) in &pending { + match record { + Some(record) => vault.put(record).await?, + None => vault.remove(id).await?, + } + written += 1; + } + // Remove only what was flushed, and only if it has not changed since + // the snapshot of `pending` was taken. Clearing the whole map would + // drop a save that landed *during* the awaits above — the record would + // exist only in memory and be gone after a restart, silently. + let mut dirty = self.guard_dirty(); + for (id, record) in &pending { + if dirty.get(id) == Some(record) { + dirty.remove(id); + } + } + Ok(written) + } + + /// How many writes are waiting. Zero after a `flush`. + #[must_use] + pub fn pending(&self) -> usize { + self.guard_dirty().len() + } + + fn guard(&self) -> std::sync::MutexGuard<'_, BTreeMap> { + self.records + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + } + + fn guard_dirty(&self) -> std::sync::MutexGuard<'_, BTreeMap>> { + self.dirty + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + } +} + +impl WorkflowStore for Snapshot { + fn policy(&self) -> &dyn HostPolicy { + self.policy.as_ref() + } + + fn list(&self) -> Result, WorkflowError> { + Ok(self + .guard() + .values() + .map(|record| WorkflowSummary { + id: record.id.clone(), + name: record.name.clone(), + description: record.description.clone(), + enabled: record.enabled, + node_count: record.graph.nodes.len(), + inputs: record.graph.inputs.clone(), + // What the summary carries instead of a path: the one node kind + // a lister filters on. + trigger_kind: record + .graph + .nodes + .iter() + .find(|n| n.kind == tinyflows::model::NodeKind::Trigger) + .and_then(|n| n.config.get("trigger_kind")) + .and_then(|v| v.as_str()) + .map(ToString::to_string), + }) + .collect()) + } + + fn get(&self, id: &str) -> Result, WorkflowError> { + Ok(self.guard().get(id).cloned()) + } + + fn save(&self, record: &WorkflowRecord) -> Result<(), WorkflowError> { + self.guard().insert(record.id.clone(), record.clone()); + self.guard_dirty() + .insert(record.id.clone(), Some(record.clone())); + Ok(()) + } + + fn delete(&self, id: &str) -> Result<(), WorkflowError> { + self.guard().remove(id); + self.guard_dirty().insert(id.to_string(), None); + Ok(()) + } + + // The engine's authoring surface. This crate does not use it, and a + // snapshot that pretended to would give a caller a run history that + // vanishes on the next load rather than an honest refusal. + fn record_run(&self, _run: &RunRecord) -> Result<(), WorkflowError> { + Err(unsupported("run records")) + } + + fn get_run(&self, _run_id: &str) -> Result, WorkflowError> { + Ok(None) + } + + fn list_runs(&self, _workflow_id: &str) -> Result, WorkflowError> { + Ok(Vec::new()) + } + + fn list_revisions(&self, _workflow_id: &str) -> Result, WorkflowError> { + Ok(Vec::new()) + } + + fn revision( + &self, + _workflow_id: &str, + _revision_id: &str, + ) -> Result, WorkflowError> { + Ok(None) + } +} + +/// A refusal that names what is missing rather than panicking. +fn unsupported(what: &str) -> WorkflowError { + WorkflowError::Engine(format!( + "this store keeps workflows only; {what} are the engine's authoring \ + surface and a snapshot does not carry them" + )) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::workflows::conformance::record; + use crate::workflows::memory::MemoryVault; + use std::sync::Mutex; + + fn policy() -> Arc { + #[derive(Debug, Default)] + struct Permissive; + impl HostPolicy for Permissive {} + Arc::new(Permissive) + } + + #[tokio::test] + async fn reads_are_served_from_memory_after_one_load() { + let vault = MemoryVault::new(); + vault.put(&record("weekly")).await.expect("put"); + + let snapshot = Snapshot::load(&vault, policy()).await.expect("load"); + assert_eq!(snapshot.list().expect("list").len(), 1); + assert!(snapshot.get("weekly").expect("get").is_some()); + assert!(snapshot.get("absent").expect("get").is_none()); + assert_eq!(snapshot.pending(), 0, "reading dirties nothing"); + } + + #[tokio::test] + async fn a_write_is_visible_at_once_and_flushed_later() { + // The loop saves a variant mid-episode and the next attempt has to see + // it. Buffering must not mean "invisible until flush". + let vault = MemoryVault::new(); + let snapshot = Snapshot::load(&vault, policy()).await.expect("load"); + + snapshot.save(&record("learned-abc")).expect("save"); + assert!(snapshot.get("learned-abc").expect("get").is_some()); + assert!( + vault.load().await.expect("load").is_empty(), + "not yet in the vault" + ); + + assert_eq!(snapshot.flush(&vault).await.expect("flush"), 1); + assert_eq!(vault.load().await.expect("load").len(), 1); + assert_eq!(snapshot.pending(), 0); + } + + #[tokio::test] + async fn only_what_changed_is_written_back() { + // The property that makes this safe beside a human editor: a workflow + // the loop read and did not touch is never rewritten, so an edit made + // elsewhere in the meantime survives. + let vault = MemoryVault::new(); + vault.put(&record("untouched")).await.expect("put"); + let snapshot = Snapshot::load(&vault, policy()).await.expect("load"); + + let _ = snapshot.list().expect("list"); + let _ = snapshot.get("untouched").expect("get"); + snapshot.save(&record("new-one")).expect("save"); + + assert_eq!( + snapshot.flush(&vault).await.expect("flush"), + 1, + "one write, not two" + ); + } + + #[tokio::test] + async fn flushing_twice_is_not_two_writes() { + let vault = MemoryVault::new(); + let snapshot = Snapshot::load(&vault, policy()).await.expect("load"); + snapshot.save(&record("once")).expect("save"); + assert_eq!(snapshot.flush(&vault).await.expect("flush"), 1); + assert_eq!(snapshot.flush(&vault).await.expect("flush"), 0); + } + + #[tokio::test] + async fn a_delete_survives_the_flush() { + let vault = MemoryVault::new(); + vault.put(&record("doomed")).await.expect("put"); + let snapshot = Snapshot::load(&vault, policy()).await.expect("load"); + + snapshot.delete("doomed").expect("delete"); + assert!(snapshot.get("doomed").expect("get").is_none()); + snapshot.flush(&vault).await.expect("flush"); + assert!(vault.load().await.expect("load").is_empty()); + } + + #[tokio::test] + async fn clones_share_the_buffer_so_the_loop_and_the_flusher_agree() { + // The loop is handed `Arc`; the caller keeps a + // `Snapshot` to flush. Those must be the same state. + let vault = MemoryVault::new(); + let snapshot = Snapshot::load(&vault, policy()).await.expect("load"); + let handed_to_the_loop: Arc = Arc::new(snapshot.clone()); + + handed_to_the_loop + .save(&record("via-the-loop")) + .expect("save"); + assert_eq!(snapshot.pending(), 1, "the flusher sees the loop's write"); + snapshot.flush(&vault).await.expect("flush"); + assert_eq!(vault.load().await.expect("load").len(), 1); + } + + #[tokio::test] + async fn a_save_landing_during_a_flush_is_not_dropped() { + // The vault's put() writes back into the snapshot through a clone — + // the shape of a second episode saving while the first one flushes. + // Clearing the whole dirty map would silently lose that record. + struct Reentrant { + inner: MemoryVault, + target: Mutex>, + } + #[async_trait] + impl Vault for Reentrant { + async fn load(&self) -> Result, WorkflowError> { + self.inner.load().await + } + async fn put(&self, incoming: &WorkflowRecord) -> Result<(), WorkflowError> { + if let Some(snapshot) = self.target.lock().expect("lock").take() { + snapshot.save(&record("late")).expect("save mid-flush"); + } + self.inner.put(incoming).await + } + async fn remove(&self, id: &str) -> Result<(), WorkflowError> { + self.inner.remove(id).await + } + } + + let vault = Reentrant { + inner: MemoryVault::new(), + target: Mutex::new(None), + }; + let snapshot = Snapshot::empty(policy()); + snapshot.save(&record("first")).expect("save"); + *vault.target.lock().expect("lock") = Some(snapshot.clone()); + + assert_eq!(snapshot.flush(&vault).await.expect("flush"), 1); + assert_eq!( + snapshot.pending(), + 1, + "the save that landed mid-flush survives to the next flush" + ); + assert_eq!(snapshot.flush(&vault).await.expect("flush"), 1); + assert_eq!(snapshot.pending(), 0); + } + + #[tokio::test] + async fn the_authoring_surface_refuses_rather_than_pretending() { + // A run record accepted and then lost on the next load is worse than a + // refusal, because nothing tells the caller it vanished. + let snapshot = Snapshot::empty(policy()); + assert!(snapshot.list_runs("any").expect("empty").is_empty()); + assert!(snapshot.get_run("any").expect("none").is_none()); + let run: tinyflows::store::types::RunRecord = serde_json::from_value(serde_json::json!({ + "id": "r1", "workflowId": "weekly", "status": "succeeded", "startedAt": 0 + })) + .expect("a minimal run record"); + assert!(snapshot.record_run(&run).is_err()); + } +} diff --git a/crates/adaptive/src/workflows/mongo.rs b/crates/adaptive/src/workflows/mongo.rs new file mode 100644 index 0000000..901096a --- /dev/null +++ b/crates/adaptive/src/workflows/mongo.rs @@ -0,0 +1,173 @@ +//! Workflows in the same MongoDB database as the ledger. + +use async_trait::async_trait; +use mongodb::bson::{Document, doc}; +use mongodb::{Client, Collection, Database}; +use tinyflows::store::types::{WorkflowError, WorkflowRecord}; + +use super::Vault; + +const WORKFLOWS: &str = "workflows"; + +/// A vault backed by a MongoDB database. +#[derive(Clone)] +pub struct MongoVault { + db: Database, + scope: Option, +} + +impl MongoVault { + /// Connect to `uri` and use the database named `database`. + /// + /// # Errors + /// When the URI is malformed or the server is unreachable. + pub async fn connect(uri: &str, database: &str) -> Result { + let client = Client::with_uri_str(uri) + .await + .map_err(|e| WorkflowError::Engine(e.to_string()))?; + Self::with_database(client.database(database)).await + } + + /// Use an already-connected database, for a host managing its own pool. + /// + /// Async because it creates the unique `(scope_key, workflow_id)` index — + /// without it, two replicas upserting the same workflow at once can insert + /// duplicate documents, and `load` would return whichever the cursor met + /// first. + /// + /// # Errors + /// When the index cannot be created. + pub async fn with_database(db: Database) -> Result { + let vault = Self { db, scope: None }; + vault.ensure_indexes().await?; + Ok(vault) + } + + async fn ensure_indexes(&self) -> Result<(), WorkflowError> { + let unique = mongodb::options::IndexOptions::builder() + .unique(true) + .build(); + self.workflows() + .create_index( + mongodb::IndexModel::builder() + .keys(doc! { "scope_key": 1, "workflow_id": 1 }) + .options(unique) + .build(), + ) + .await + .map_err(mongo)?; + Ok(()) + } + + /// A handle onto the same database, scoped to one tenant. + #[must_use] + pub fn for_tenant(&self, scope: impl Into) -> Self { + Self { + db: self.db.clone(), + scope: Some(scope.into()), + } + } + + /// Stored as a present empty string rather than an absent field, so the + /// upsert filter matches one document — the same reason the ledger does it. + fn bucket(&self) -> &str { + self.scope.as_deref().unwrap_or_default() + } + + fn workflows(&self) -> Collection { + self.db.collection(WORKFLOWS) + } +} + +fn mongo(err: mongodb::error::Error) -> WorkflowError { + WorkflowError::Engine(err.to_string()) +} + +#[async_trait] +impl Vault for MongoVault { + fn scope(&self) -> Option<&str> { + self.scope.as_deref() + } + + async fn load(&self) -> Result, WorkflowError> { + // This bucket plus global. A record written before scoping existed has + // no field at all, which `$in` with "" does not match — but nothing + // wrote one, because this collection is new. + // Global first, then this bucket, so the bucket's own record shadows a + // global one with the same id — precedence by construction, not by + // whatever order the cursor happens to walk. + let mut cursor = self + .workflows() + .find(doc! { "scope_key": { "$in": [self.bucket(), ""] } }) + .sort(doc! { "scope_key": 1, "workflow_id": 1 }) + .await + .map_err(mongo)?; + + let mut chosen: std::collections::BTreeMap = + std::collections::BTreeMap::new(); + while cursor.advance().await.map_err(mongo)? { + let document = cursor.deserialize_current().map_err(mongo)?; + let raw = document.get_str("document").unwrap_or_default(); + let record: WorkflowRecord = serde_json::from_str(raw).map_err(|e| { + WorkflowError::Engine(format!("stored workflow no longer parses: {e}")) + })?; + chosen.insert(record.id.clone(), record); + } + Ok(chosen.into_values().collect()) + } + + async fn put(&self, record: &WorkflowRecord) -> Result<(), WorkflowError> { + let document = serde_json::to_string(record) + .map_err(|e| WorkflowError::Engine(format!("workflow will not serialize: {e}")))?; + // Stored as a JSON string rather than a BSON subdocument: a node config + // is arbitrary JSON, and BSON refuses keys containing a dot — which a + // config keyed by a filename or a version has. + // + // One retry on a duplicate-key race: the unique index stops two + // concurrent upserts both inserting, but the loser errors rather than + // updating — its second pass finds the document and updates it. + for attempt in 0..2 { + let outcome = self + .workflows() + .update_one( + doc! { "scope_key": self.bucket(), "workflow_id": &record.id }, + doc! { "$set": { "document": &document } }, + ) + .upsert(true) + .await; + match outcome { + Ok(_) => return Ok(()), + Err(e) if attempt == 0 && e.to_string().contains("E11000") => continue, + Err(e) => return Err(mongo(e)), + } + } + unreachable!("the loop returns on every branch of its final pass") + } + + async fn remove(&self, id: &str) -> Result<(), WorkflowError> { + self.workflows() + .delete_one(doc! { "scope_key": self.bucket(), "workflow_id": id }) + .await + .map_err(mongo)?; + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::workflows::conformance; + + /// Needs a real server, so it is `#[ignore]` and visible in the run summary + /// rather than silently skipped — the same posture as the mongo ledger. + #[tokio::test] + #[ignore = "needs a MongoDB server; set ADAPTIVE_MONGO_URI"] + async fn passes_the_conformance_suite() { + let uri = std::env::var("ADAPTIVE_MONGO_URI").expect("ADAPTIVE_MONGO_URI"); + let name = format!("adaptive_vault_{}", std::process::id()); + let vault = MongoVault::connect(&uri, &name).await.expect("connect"); + conformance::run_all(&vault).await; + conformance::run_tenants(&vault, &vault.for_tenant("a"), &vault.for_tenant("b")).await; + vault.db.drop().await.expect("drop the throwaway database"); + } +} diff --git a/crates/adaptive/src/workflows/sqlite.rs b/crates/adaptive/src/workflows/sqlite.rs new file mode 100644 index 0000000..dc1bcb4 --- /dev/null +++ b/crates/adaptive/src/workflows/sqlite.rs @@ -0,0 +1,222 @@ +//! Workflows in the same sqlite file as the ledger. +//! +//! One file for everything durable, so a deployment backs up one thing and a +//! developer inspects one thing. + +use std::sync::{Arc, Mutex}; + +use async_trait::async_trait; +use rusqlite::{Connection, params}; +use tinyflows::store::types::{WorkflowError, WorkflowRecord}; + +use super::Vault; + +const DDL: &str = "CREATE TABLE IF NOT EXISTS workflows ( + scope_key TEXT NOT NULL DEFAULT '', + id TEXT NOT NULL, + document TEXT NOT NULL, + PRIMARY KEY (scope_key, id) + )"; + +/// A vault backed by one sqlite file. +#[derive(Clone)] +pub struct SqliteVault { + conn: Arc>, + scope: Option, +} + +impl SqliteVault { + /// Open (or create) a vault at `path`, creating the parent directory. + /// + /// # Errors + /// When the file or its directory cannot be opened. + pub fn open(path: impl AsRef) -> Result { + let path = path.as_ref(); + if let Some(parent) = path.parent().filter(|p| !p.as_os_str().is_empty()) { + std::fs::create_dir_all(parent) + .map_err(|e| WorkflowError::Engine(format!("{}: {e}", parent.display())))?; + } + Self::from_connection(Connection::open(path).map_err(sql)?) + } + + /// A vault held entirely in memory. For tests. + /// + /// # Errors + /// When the schema cannot be applied. + pub fn in_memory() -> Result { + Self::from_connection(Connection::open_in_memory().map_err(sql)?) + } + + fn from_connection(conn: Connection) -> Result { + conn.execute_batch("PRAGMA journal_mode = WAL; PRAGMA busy_timeout = 5000;") + .ok(); + conn.execute(DDL, []).map_err(sql)?; + Ok(Self { + conn: Arc::new(Mutex::new(conn)), + scope: None, + }) + } + + /// A handle onto the same file, scoped to one tenant. + #[must_use] + pub fn for_tenant(&self, scope: impl Into) -> Self { + Self { + conn: Arc::clone(&self.conn), + scope: Some(scope.into()), + } + } + + fn bucket(&self) -> String { + self.scope.clone().unwrap_or_default() + } + + fn guard(&self) -> std::sync::MutexGuard<'_, Connection> { + self.conn + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + } +} + +fn sql(err: rusqlite::Error) -> WorkflowError { + WorkflowError::Engine(err.to_string()) +} + +#[async_trait] +impl Vault for SqliteVault { + fn scope(&self) -> Option<&str> { + self.scope.as_deref() + } + + async fn load(&self) -> Result, WorkflowError> { + let conn = self.guard(); + let mut stmt = conn + .prepare( + "SELECT document FROM workflows WHERE scope_key = ?1 OR scope_key = '' ORDER BY id", + ) + .map_err(sql)?; + let documents = stmt + .query_map([self.bucket()], |r| r.get::<_, String>(0)) + .map_err(sql)? + .collect::>>() + .map_err(sql)?; + + documents + .iter() + .map(|document| { + serde_json::from_str(document).map_err(|e| { + WorkflowError::Engine(format!("stored workflow no longer parses: {e}")) + }) + }) + .collect() + } + + async fn put(&self, record: &WorkflowRecord) -> Result<(), WorkflowError> { + let document = serde_json::to_string(record) + .map_err(|e| WorkflowError::Engine(format!("workflow will not serialize: {e}")))?; + self.guard() + .execute( + "INSERT INTO workflows(scope_key, id, document) VALUES(?1,?2,?3) + ON CONFLICT(scope_key, id) DO UPDATE SET document = ?3", + params![self.bucket(), record.id, document], + ) + .map_err(sql)?; + Ok(()) + } + + async fn remove(&self, id: &str) -> Result<(), WorkflowError> { + self.guard() + .execute( + "DELETE FROM workflows WHERE scope_key = ?1 AND id = ?2", + params![self.bucket(), id], + ) + .map_err(sql)?; + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::workflows::conformance; + + #[tokio::test] + async fn passes_the_conformance_suite() { + conformance::run_all(&SqliteVault::in_memory().expect("open")).await; + } + + #[tokio::test] + async fn passes_the_tenant_isolation_suite() { + let vault = SqliteVault::in_memory().expect("open"); + conformance::run_tenants(&vault, &vault.for_tenant("a"), &vault.for_tenant("b")).await; + } + + #[tokio::test] + async fn a_reopened_vault_still_has_its_workflows() { + let dir = std::env::temp_dir().join(format!("adaptive-vault-{}", std::process::id())); + let path = dir.join("nested").join("workflows.db"); + let _ = std::fs::remove_dir_all(&dir); + + SqliteVault::open(&path) + .expect("open") + .put(&conformance::record("wf-durable")) + .await + .expect("put"); + + let again = SqliteVault::open(&path).expect("reopen"); + assert_eq!(again.load().await.expect("load").len(), 1); + let _ = std::fs::remove_dir_all(&dir); + } + + #[tokio::test] + async fn the_ledger_and_the_vault_share_one_file_without_colliding() { + // The module note claims "one file for everything durable". Two + // `Connection`s onto one path is only fine because the two schemas + // share no table name — ledger_rows/lessons/episodes/… beside + // workflows — and asserting it here is cheaper than finding out when a + // deployment points both at the same DSN. + use crate::ledger::Ledger; + + let dir = std::env::temp_dir().join(format!("adaptive-onefile-{}", std::process::id())); + let path = dir.join("adaptive.db"); + let _ = std::fs::remove_dir_all(&dir); + + let ledger = crate::ledger::sqlite::SqliteLedger::open(&path).expect("ledger"); + let vault = SqliteVault::open(&path).expect("vault"); + + vault + .put(&conformance::record("weekly")) + .await + .expect("put"); + ledger + .append(&crate::ledger::conformance::row("ep-1", 1, "authored")) + .await + .expect("append"); + + assert_eq!(vault.load().await.expect("load").len(), 1); + assert_eq!(ledger.rows("ep-1").await.expect("rows").len(), 1); + + // And both survive a reopen of the same file, which is the point of + // putting them there. + drop(ledger); + drop(vault); + let ledger = crate::ledger::sqlite::SqliteLedger::open(&path).expect("reopen ledger"); + let vault = SqliteVault::open(&path).expect("reopen vault"); + assert_eq!(vault.load().await.expect("load").len(), 1); + assert_eq!(ledger.rows("ep-1").await.expect("rows").len(), 1); + + let _ = std::fs::remove_dir_all(&dir); + } + + #[tokio::test] + async fn the_stored_document_is_the_whole_record_not_just_the_graph() { + // `description` is what a planner reads to choose. A vault that kept + // only the graph would file every workflow as unfindable. + let vault = SqliteVault::in_memory().expect("open"); + vault + .put(&conformance::record("wf-prose")) + .await + .expect("put"); + let back = &vault.load().await.expect("load")[0]; + assert_eq!(back.description, "does the wf-prose thing"); + } +} diff --git a/crates/adaptive/tests/closing.rs b/crates/adaptive/tests/closing.rs new file mode 100644 index 0000000..33d8185 --- /dev/null +++ b/crates/adaptive/tests/closing.rs @@ -0,0 +1,468 @@ +//! Closing, end to end, against a scripted model and a real ledger. +//! +//! The unit tests cover the decision table and the parsing. These cover the +//! properties that only show up once a ledger is actually written to: that a +//! *failed* attempt still leaves a row and still moves the score, that the row +//! it leaves is the one the next attempt's exclusion list reads, and that the +//! three mechanical verdicts never reach the model at all. + +use std::sync::Mutex; + +use async_trait::async_trait; +use serde_json::{Value, json}; +use tinyflows::caps::mock::mock_capabilities; +use tinyflows::caps::{Capabilities, LlmProvider}; +use tinyflows::diagnostics::{Diagnosis, NeverRan}; +use tinyflows::engine::RunOutcome; +use tinyflows::error::Result as EngineResult; +use tinyflows_adaptive::closing::{Next, close, consolidate}; +use tinyflows_adaptive::contracts::{Approach, Blocker, Budget, Goal}; +use tinyflows_adaptive::execute::Ran; +use tinyflows_adaptive::ledger::{Ledger, LessonKind, memory::MemoryLedger}; + +/// A provider that answers from a script and counts what it was asked. +struct Scripted { + replies: Mutex>, + calls: Mutex>, +} + +impl Scripted { + fn new(replies: Vec) -> std::sync::Arc { + std::sync::Arc::new(Self { + replies: Mutex::new(replies), + calls: Mutex::new(Vec::new()), + }) + } + + fn call_count(&self) -> usize { + self.calls.lock().expect("lock").len() + } + + fn last_prompt(&self) -> String { + self.calls + .lock() + .expect("lock") + .last() + .cloned() + .unwrap_or_default() + } +} + +#[async_trait] +impl LlmProvider for Scripted { + async fn complete(&self, request: Value, _conn: Option<&str>) -> EngineResult { + let text = request["messages"] + .as_array() + .map(|m| { + m.iter() + .filter_map(|msg| msg["content"].as_str()) + .collect::>() + .join("\n") + }) + .unwrap_or_default(); + self.calls.lock().expect("lock").push(text); + let mut replies = self.replies.lock().expect("lock"); + assert!( + !replies.is_empty(), + "the model was asked more times than the script has answers" + ); + Ok(replies.remove(0)) + } +} + +fn caps_with(llm: std::sync::Arc) -> Capabilities { + Capabilities { + llm, + ..mock_capabilities() + } +} + +fn completed(output: Value) -> RunOutcome { + RunOutcome { + output, + pending_approvals: Vec::new(), + cancelled: false, + } +} + +/// A finished run, as `close` now takes it. The judge still reads only the +/// evidence; the cost and the transcript ride along because they are recorded. +fn ran(outcome: &RunOutcome, diagnosis: &Diagnosis, changed: &str) -> Ran { + Ran { + outcome: outcome.clone(), + diagnosis: diagnosis.clone(), + changed: changed.to_string(), + failed: None, + steps: Vec::new(), + cost_usd: 0.0, + } +} + +fn selected(id: &str) -> Approach { + Approach::Selected { + workflow_id: id.to_string(), + why: "it matched".into(), + } +} + +#[tokio::test] +async fn a_failed_attempt_is_still_recorded_and_still_scored() { + // The property the whole retry edge rests on. An attempt that fell short + // and left no trace is one the next attempt repeats verbatim. + let llm = Scripted::new(vec![json!({ + "satisfied": false, + "blocker": "goal_not_met", + "gap": "the report has no numbers in it", + "advanced": true + })]); + let ledger = MemoryLedger::new(); + let diagnosis = Diagnosis::default(); + let outcome = completed(json!({"nodes": {"write": {"ok": true}}})); + + let closed = close( + &Goal::new("write the weekly report"), + "ep-1", + 1, + &selected("weekly"), + &ran(&outcome, &diagnosis, "wrote report.md"), + &Budget::default(), + &ledger, + &caps_with(llm), + None, + "2026-01-01T00:00:00Z", + ) + .await + .expect("closed"); + + assert_eq!(closed.next, Next::Retry); + assert_eq!(closed.stalled, 0, "it advanced, so nothing is stalling yet"); + + let rows = ledger.rows("ep-1").await.expect("rows"); + assert_eq!(rows.len(), 1); + assert_eq!(rows[0].outcome, "the report has no numbers in it"); + assert_eq!(rows[0].workflow_id.as_deref(), Some("weekly")); + + // The exclusion list the next attempt reads. + assert_eq!(ledger.tried("ep-1").await.expect("tried").len(), 1); + + let score = ledger.workflow_score("weekly").await.expect("score"); + assert_eq!( + (score.applied, score.helped), + (1, 0), + "a run that failed still counts as a run" + ); +} + +#[tokio::test] +async fn a_satisfied_attempt_moves_both_halves_of_the_score() { + let llm = Scripted::new(vec![json!({"satisfied": true, "gap": ""})]); + let ledger = MemoryLedger::new(); + let diagnosis = Diagnosis::default(); + let outcome = completed(json!({"nodes": {"write": {"ok": true}}})); + + let closed = close( + &Goal::new("write the weekly report"), + "ep-2", + 1, + &selected("weekly"), + &ran(&outcome, &diagnosis, "wrote report.md"), + &Budget::default(), + &ledger, + &caps_with(llm), + None, + "2026-01-01T00:00:00Z", + ) + .await + .expect("closed"); + + assert_eq!(closed.next, Next::Done); + assert_eq!(closed.verdict.blocker, Blocker::None); + let score = ledger.workflow_score("weekly").await.expect("score"); + assert_eq!((score.applied, score.helped), (1, 1)); +} + +#[tokio::test] +async fn a_run_where_nothing_happened_never_reaches_the_model() { + // Mechanical evidence first. The script is empty on purpose: if the judge + // asks anything at all, `Scripted` panics and this test fails. + let llm = Scripted::new(Vec::new()); + let caps = caps_with(llm.clone()); + let ledger = MemoryLedger::new(); + let diagnosis = Diagnosis { + never_ran: vec![NeverRan { + node_id: "write".into(), + routed_by: Some("is_due".into()), + }], + ..Diagnosis::default() + }; + let outcome = completed(json!({})); + + let closed = close( + &Goal::new("write the weekly report"), + "ep-3", + 1, + &selected("weekly"), + &ran(&outcome, &diagnosis, ""), + &Budget::default(), + &ledger, + &caps, + None, + "2026-01-01T00:00:00Z", + ) + .await + .expect("closed"); + + assert_eq!(llm.call_count(), 0, "a fact does not need an opinion"); + assert_eq!(closed.verdict.blocker, Blocker::MissingEvidence); + // Terminal: a retry with the same inputs produces the same nothing. + assert!( + matches!(closed.next, Next::StandDown(_)), + "{:?}", + closed.next + ); + // And it is still on the record. + assert_eq!(ledger.rows("ep-3").await.expect("rows").len(), 1); +} + +#[tokio::test] +async fn a_parked_approval_is_not_a_failure() { + let llm = Scripted::new(Vec::new()); + let caps = caps_with(llm.clone()); + let ledger = MemoryLedger::new(); + let diagnosis = Diagnosis::default(); + let outcome = RunOutcome { + output: json!({"nodes": {"draft": {"ok": true}}}), + pending_approvals: vec!["publish".into()], + cancelled: false, + }; + + let closed = close( + &Goal::new("publish the post"), + "ep-4", + 1, + &selected("blog"), + &ran(&outcome, &diagnosis, ""), + &Budget::default(), + &ledger, + &caps, + None, + "2026-01-01T00:00:00Z", + ) + .await + .expect("closed"); + + assert_eq!(llm.call_count(), 0); + assert_eq!(closed.verdict.blocker, Blocker::NeedsInput); + assert_eq!( + closed.stalled, 0, + "reaching an approval gate is progress, not a stall" + ); +} + +#[tokio::test] +async fn two_flat_attempts_in_a_row_stand_down_on_the_stall_rule() { + let llm = Scripted::new(vec![ + json!({"satisfied": false, "blocker": "goal_not_met", "gap": "same as before", "advanced": false}), + json!({"satisfied": false, "blocker": "goal_not_met", "gap": "same as before", "advanced": false}), + ]); + let caps = caps_with(llm); + let ledger = MemoryLedger::new(); + let diagnosis = Diagnosis::default(); + let outcome = completed(json!({"nodes": {"write": {}}})); + let budget = Budget::default(); + + let mut stalled = 0; + let mut last = None; + for attempt in 4..6 { + let closed = close( + &Goal::new("write the weekly report"), + "ep-5", + attempt, + &Approach::Authored { + why: format!("attempt {attempt}"), + fingerprint: "0000000".into(), + }, + &ran(&outcome, &diagnosis, ""), + &budget, + &ledger, + &caps, + None, + "2026-01-01T00:00:00Z", + ) + .await + .expect("closed"); + stalled = closed.stalled; + last = Some(closed.next); + } + + assert_eq!(stalled, 2); + match last.expect("a second pass ran") { + Next::StandDown(reason) => assert!(reason.contains("no progress"), "{reason}"), + other => panic!("expected a stand-down after two flat attempts, got {other:?}"), + } + // Authoring attempts have no workflow to score, and scoring one anyway + // would credit whichever workflow happened to run last. + assert_eq!( + ledger.rows("ep-5").await.expect("rows")[0].workflow_id, + None + ); +} + +#[tokio::test] +async fn consolidation_keeps_a_lesson_and_cites_the_rows_behind_it() { + let ledger = MemoryLedger::new(); + for (attempt, sig) in [(1u32, "sig-a"), (2, "sig-b")] { + ledger + .append(&tinyflows_adaptive::ledger::LedgerRow { + id: String::new(), + episode: "ep-6".into(), + attempt, + approach_sig: sig.into(), + approach_desc: "tried it".into(), + workflow_id: None, + outcome: "fell short".into(), + cause: "the loop never terminated".into(), + cost_usd: 0.0, + at: "2026-01-01T00:00:00Z".into(), + satisfied: false, + advanced: false, + }) + .await + .expect("appended"); + } + + let llm = Scripted::new(vec![json!({ + "lessons": [{ + "kind": "constraint", + "trigger": "a scan over ~1M items with a sub-100ms target", + "mechanism": "the interpreter overhead dominates", + "claim": "reach for a compiled step instead of tuning the loop", + "evidence": [0, 1] + }], + "corroborate": [] + })]); + let caps = caps_with(llm.clone()); + + let kept = consolidate( + &Goal::new("make the scan fast"), + "ep-6", + false, + &ledger, + &caps, + None, + ) + .await; + + assert_eq!(kept.len(), 1); + assert_eq!(kept[0].kind, LessonKind::Constraint); + assert!(!kept[0].id.is_empty(), "it was actually stored"); + + // The rows the claim was drawn from, readable back. + let cites = ledger.evidence(&kept[0].id).await.expect("evidence"); + assert_eq!(cites.len(), 2); + + // Both attempts were shown, numbered the way the prompt asks it to cite. + let prompt = llm.last_prompt(); + assert!(prompt.contains("0. [sig-a]"), "{prompt}"); + assert!(prompt.contains("1. [sig-b]"), "{prompt}"); +} + +#[tokio::test] +async fn a_lesson_with_nothing_behind_it_is_not_kept() { + // A claim with no rows cited is a guess, and a guess in the knowledge store + // is worse than nothing: it will be retrieved and believed. + let ledger = MemoryLedger::new(); + ledger + .append(&tinyflows_adaptive::ledger::LedgerRow { + id: String::new(), + episode: "ep-7".into(), + attempt: 1, + approach_sig: "sig-a".into(), + approach_desc: "tried it".into(), + workflow_id: None, + outcome: "fell short".into(), + cause: String::new(), + cost_usd: 0.0, + at: "2026-01-01T00:00:00Z".into(), + satisfied: false, + advanced: false, + }) + .await + .expect("appended"); + + let llm = Scripted::new(vec![json!({ + "lessons": [ + {"kind": "strategy", "trigger": "a class of task", "claim": "do the thing"}, + {"kind": "strategy", "claim": "no trigger, so nothing could ever match it", + "evidence": [0]} + ] + })]); + + let kept = consolidate( + &Goal::new("make the scan fast"), + "ep-7", + false, + &ledger, + &caps_with(llm), + None, + ) + .await; + + assert!(kept.is_empty(), "{kept:?}"); + assert!(ledger.lessons(None).await.expect("lessons").is_empty()); +} + +#[tokio::test] +async fn consolidation_failing_does_not_fail_the_episode() { + // It runs after the outcome is settled. A provider hiccup keeps nothing and + // leaves the real result standing — note the signature has no `Result`. + let ledger = MemoryLedger::new(); + ledger + .append(&tinyflows_adaptive::ledger::LedgerRow { + id: String::new(), + episode: "ep-8".into(), + attempt: 1, + approach_sig: "sig-a".into(), + approach_desc: "tried it".into(), + workflow_id: None, + outcome: "fell short".into(), + cause: String::new(), + cost_usd: 0.0, + at: "2026-01-01T00:00:00Z".into(), + satisfied: false, + advanced: false, + }) + .await + .expect("appended"); + + // Not JSON the reader can use. + let llm = Scripted::new(vec![json!("the model wandered off into prose")]); + let kept = consolidate( + &Goal::new("make the scan fast"), + "ep-8", + false, + &ledger, + &caps_with(llm), + None, + ) + .await; + assert!(kept.is_empty()); +} + +#[tokio::test] +async fn an_episode_with_no_attempts_asks_nothing() { + let llm = Scripted::new(Vec::new()); + let caps = caps_with(llm.clone()); + let ledger = MemoryLedger::new(); + let kept = consolidate( + &Goal::new("anything"), + "ep-none", + true, + &ledger, + &caps, + None, + ) + .await; + assert!(kept.is_empty()); + assert_eq!(llm.call_count(), 0); +} diff --git a/crates/adaptive/tests/contracts_surface.rs b/crates/adaptive/tests/contracts_surface.rs new file mode 100644 index 0000000..2ea244d --- /dev/null +++ b/crates/adaptive/tests/contracts_surface.rs @@ -0,0 +1,140 @@ +//! What crosses a process boundary, asserted rather than assumed. +//! +//! Every type here is part of a contract some other process — a device runner, +//! a TypeScript relay, a third ledger backend — has to produce or read. A +//! derive quietly dropped from one of them is a runtime failure in a different +//! repository, so the requirement is checked at compile time here. + +use serde::Serialize; +use serde::de::DeserializeOwned; + +const fn wire() {} + +#[test] +fn every_wire_type_still_serializes_both_ways() { + // The execute contract: what a runner receives and returns. + wire::(); + wire::(); + wire::(); + wire::(); + + // Inside those: engine types the runner must round-trip untouched. + wire::(); + wire::(); + wire::(); + wire::(); + wire::(); + + // Derived on the loop's side from the steps, but stored and shipped by + // hosts that keep a run record. + wire::(); + wire::(); + wire::(); + wire::(); + + // The loop's own persisted state: anything a hosted service stores. + wire::(); + wire::(); + wire::(); + wire::(); + wire::(); + wire::(); + wire::(); + wire::(); + wire::(); + + // What a device reports about itself, and what a repair proposes. + wire::(); + wire::(); + wire::(); +} + +#[test] +fn the_envelope_is_camel_case_and_the_graph_inside_it_is_not() { + // Worth pinning because it will bite whoever writes the other side. The + // wire types this crate added use camelCase; the engine's own model types + // predate them and use serde's default. One payload, two conventions. + let request = tinyflows_adaptive::execute::RunRequest { + attempt_id: "ep-1/3".into(), + graph: tinyflows::model::WorkflowGraph { + schema_version: 1, + id: Some("g".into()), + name: "g".into(), + inputs: Vec::new(), + agents: Vec::new(), + nodes: vec![tinyflows::model::Node { + id: "start".into(), + kind: tinyflows::model::NodeKind::Trigger, + type_version: 1, + name: "start".into(), + config: serde_json::json!({"trigger_kind": "manual"}), + ports: Vec::new(), + position: None, + }], + edges: vec![tinyflows::model::Edge { + from_node: "start".into(), + from_port: "main".into(), + to_node: "start".into(), + to_port: "main".into(), + }], + }, + inputs: serde_json::Map::new(), + }; + let text = serde_json::to_string(&request).expect("serializes"); + + assert!( + text.contains("\"attemptId\""), + "envelope is camelCase: {text}" + ); + assert!( + text.contains("\"schema_version\""), + "the graph keeps the engine's snake_case: {text}" + ); + assert!(text.contains("\"from_node\""), "{text}"); + assert!(text.contains("\"type_version\""), "{text}"); + + println!("REQUEST {text}"); + println!( + "REPORT {}", + serde_json::to_string(&tinyflows_adaptive::execute::RunReport { + attempt_id: "ep-1/3".into(), + steps: vec![tinyflows_adaptive::execute::StepRecord { + node_id: "start".into(), + status: tinyflows_adaptive::execute::StepOutcome::Success, + output: serde_json::json!({"ok": true}), + duration_ms: 12, + null_bindings: Vec::new(), + }], + pending_approvals: vec!["publish".into()], + cancelled: false, + changed: "1 file changed".into(), + failed: None, + cost_usd: 0.42, + }) + .expect("serializes") + ); +} + +const fn shareable() {} + +#[test] +fn a_loop_can_be_shared_across_tasks_and_replicas() { + // The operational half of statelessness. `Loop` holds only borrows of + // `Send + Sync` adapters and no state of its own, so one instance serves + // many concurrent episodes and any replica can serve any request. If this + // stops compiling, something acquired state that has to be owned — and the + // microservice story goes with it. + shareable::>(); + + // The adapters a host injects, for the same reason. + shareable::<&dyn tinyflows_adaptive::ledger::Ledger>(); + shareable::<&dyn tinyflows_adaptive::execute::Runner>(); + shareable::<&dyn tinyflows_adaptive::execute::Relay>(); + shareable::<&dyn tinyflows_adaptive::execute::Workspace>(); + shareable::<&dyn tinyflows_adaptive::driver::Clock>(); + + // And the values that cross between them. + shareable::(); + shareable::(); + shareable::(); +} diff --git a/crates/adaptive/tests/driver.rs b/crates/adaptive/tests/driver.rs new file mode 100644 index 0000000..1539afe --- /dev/null +++ b/crates/adaptive/tests/driver.rs @@ -0,0 +1,888 @@ +//! One instance, many goal runs, and an episode that outlives the process. +//! +//! These test the claim the `driver` module is built on, because it is the one +//! that is expensive to be wrong about: a `Loop` is per **tenant** and a goal +//! run is an **episode id**, so the same instance drives many episodes at once +//! and any instance can pick up an episode any other one started. + +use std::sync::{Arc, Mutex}; + +use async_trait::async_trait; +use serde_json::{Value, json}; +use tinyflows::caps::mock::mock_capabilities; +use tinyflows::caps::{Capabilities, LlmProvider}; +use tinyflows::error::Result as EngineResult; +use tinyflows::model::{Edge, Node, NodeKind, WorkflowGraph}; +use tinyflows::store::{FileWorkflowStore, WorkflowStore}; +use tinyflows_adaptive::contracts::Goal; +use tinyflows_adaptive::driver::{Clock, Loop}; +use tinyflows_adaptive::execute::{Local, Unobserved}; +use tinyflows_adaptive::host::HostFacts; +use tinyflows_adaptive::ledger::{EpisodeStatus, Ledger, Page, memory::MemoryLedger}; + +struct Frozen; +impl Clock for Frozen { + fn now(&self) -> String { + "2026-01-01T00:00:00Z".to_string() + } +} + +/// Answers every authoring call the same way, and keeps every request so the +/// tier can be read back off the wire. +struct Always { + reply: Value, + seen: Mutex>, +} + +impl Always { + fn new(reply: Value) -> Arc { + Arc::new(Self { + reply, + seen: Mutex::new(Vec::new()), + }) + } + fn tiers(&self) -> Vec { + self.seen + .lock() + .expect("lock") + .iter() + .map(|r| r["tier"].as_str().unwrap_or("(absent)").to_string()) + .collect() + } +} + +#[async_trait] +impl LlmProvider for Always { + async fn complete(&self, request: Value, _conn: Option<&str>) -> EngineResult { + self.seen.lock().expect("lock").push(request.clone()); + // The tier says which job is asking, so one double can answer them all. + Ok(match request["tier"].as_str().unwrap_or_default() { + "judge" => json!({ + "satisfied": false, "blocker": "goal_not_met", + "gap": "the report has no numbers in it", "advanced": false + }), + "consolidate" => json!({ "lessons": [], "corroborate": [] }), + "select" => json!({ "workflow_id": null, "why": "nothing fits" }), + _ => self.reply.clone(), + }) + } +} + +fn caps_with(llm: Arc) -> Capabilities { + Capabilities { + llm, + ..mock_capabilities() + } +} + +fn tiny(name: &str) -> WorkflowGraph { + WorkflowGraph { + schema_version: 1, + id: Some(name.into()), + name: name.into(), + inputs: Vec::new(), + agents: Vec::new(), + nodes: vec![ + Node { + id: "start".into(), + kind: NodeKind::Trigger, + type_version: 1, + name: "manual".into(), + config: json!({ "trigger_kind": "manual" }), + ports: Vec::new(), + position: None, + }, + Node { + id: "done".into(), + kind: NodeKind::Transform, + type_version: 1, + name: "done".into(), + config: json!({ "set": { "ok": true } }), + ports: Vec::new(), + position: None, + }, + ], + edges: vec![Edge { + from_node: "start".into(), + from_port: "main".into(), + to_node: "done".into(), + to_port: "main".into(), + }], + } +} + +fn store(tag: &str) -> Arc { + let root = std::env::temp_dir().join(format!("adaptive-driver-{}-{tag}", std::process::id())); + let _ = std::fs::remove_dir_all(&root); + std::fs::create_dir_all(root.join("workflows")).expect("temp dir"); + Arc::new(FileWorkflowStore::new( + vec![root.join("workflows")], + root.join("runs"), + )) +} + +fn authoring() -> Arc { + Always::new(json!({ + "graph": tiny("attempt"), + "why": "nothing stored fits", + "inputs": {}, + })) +} + +#[tokio::test] +async fn one_instance_drives_two_goal_runs_with_independent_counters() { + // The claim the split rests on: the instance holds no per-episode state, so + // two episodes interleaved through it cannot contaminate each other. + let llm = authoring(); + let caps = caps_with(llm); + let ledger = MemoryLedger::new(); + let store = store("two"); + let runner = Local { + caps: &caps, + workspace: &Unobserved, + }; + let engine = Loop { + ledger: &ledger, + store: &store, + caps: &caps, + facts: &HostFacts::unknown(), + runner: &runner, + clock: &Frozen, + budget: Default::default(), + conn: None, + }; + + let goal = Goal::new("write the weekly report"); + engine.attempt("ep-a", &goal).await.expect("a1"); + engine.attempt("ep-b", &goal).await.expect("b1"); + engine.attempt("ep-a", &goal).await.expect("a2"); + + let a = ledger.episode("ep-a").await.expect("read").expect("exists"); + let b = ledger.episode("ep-b").await.expect("read").expect("exists"); + assert_eq!(a.attempt, 2); + assert_eq!(b.attempt, 1, "b is untouched by a's two passes"); + assert_eq!(a.stalled, 2, "neither of a's attempts advanced"); + assert_eq!(b.stalled, 1); + + assert_eq!(ledger.rows("ep-a").await.expect("rows").len(), 2); + assert_eq!(ledger.rows("ep-b").await.expect("rows").len(), 1); +} + +#[tokio::test] +async fn a_second_instance_picks_up_an_episode_the_first_one_started() { + // Kill the process mid-episode. Everything the loop needs is in the ledger, + // so a fresh instance continues the numbering rather than starting over + // with a trail that says it has already tried twice. + let ledger = MemoryLedger::new(); + let store = store("resume"); + let goal = Goal::new("write the weekly report"); + + { + let caps = caps_with(authoring()); + let runner = Local { + caps: &caps, + workspace: &Unobserved, + }; + let first = Loop { + ledger: &ledger, + store: &store, + caps: &caps, + facts: &HostFacts::unknown(), + runner: &runner, + clock: &Frozen, + budget: Default::default(), + conn: None, + }; + first.attempt("ep-resume", &goal).await.expect("1"); + first.attempt("ep-resume", &goal).await.expect("2"); + } // the instance goes away, as a deploy would take it + + let unfinished = ledger.episodes(true, Page::ALL).await.expect("episodes"); + assert_eq!(unfinished.len(), 1, "the recovery list a boot reads"); + let recovered = &unfinished[0]; + assert_eq!(recovered.id, "ep-resume"); + assert_eq!(recovered.goal.text, "write the weekly report"); + assert_eq!(recovered.stalled, 2); + + let caps = caps_with(authoring()); + let runner = Local { + caps: &caps, + workspace: &Unobserved, + }; + let second = Loop { + ledger: &ledger, + store: &store, + caps: &caps, + facts: &HostFacts::unknown(), + runner: &runner, + clock: &Frozen, + budget: Default::default(), + conn: None, + }; + let closed = second + .attempt(&recovered.id, &recovered.goal) + .await + .expect("3"); + + assert_eq!( + ledger + .episode("ep-resume") + .await + .expect("read") + .expect("exists") + .attempt, + 3, + "it continued rather than restarting at one" + ); + assert_eq!( + closed.stalled, 3, + "the stall count survived the process that was counting it" + ); +} + +#[tokio::test] +async fn every_inference_request_says_which_job_is_asking() { + // Without this a host cannot route judging and selecting to different + // models, which is the whole point of the tier. + let llm = authoring(); + let caps = caps_with(llm.clone()); + let ledger = MemoryLedger::new(); + let store = store("tiers"); + let runner = Local { + caps: &caps, + workspace: &Unobserved, + }; + let engine = Loop { + ledger: &ledger, + store: &store, + caps: &caps, + facts: &HostFacts::unknown(), + runner: &runner, + clock: &Frozen, + budget: Default::default(), + conn: None, + }; + + engine + .attempt("ep-tiers", &Goal::new("write the weekly report")) + .await + .expect("attempt"); + + let tiers = llm.tiers(); + assert!(!tiers.iter().any(|t| t == "(absent)"), "{tiers:?}"); + assert!(tiers.contains(&"author".to_string()), "{tiers:?}"); + assert!(tiers.contains(&"judge".to_string()), "{tiers:?}"); +} + +#[tokio::test] +async fn a_run_drives_to_a_stand_down_and_consolidates_once() { + // The judge never says satisfied and nothing advances, so the stall rule + // ends it. `run` must stop on its own rather than needing a bound of its + // own alongside the one `close` already applies. + let llm = authoring(); + let caps = caps_with(llm.clone()); + let ledger = MemoryLedger::new(); + let store = store("drive"); + let runner = Local { + caps: &caps, + workspace: &Unobserved, + }; + let engine = Loop { + ledger: &ledger, + store: &store, + caps: &caps, + facts: &HostFacts::unknown(), + runner: &runner, + clock: &Frozen, + budget: Default::default(), + conn: None, + }; + + let finished = engine + .run("ep-drive", &Goal::new("write the weekly report")) + .await + .expect("run"); + + match &finished.status { + EpisodeStatus::StoodDown(reason) => assert!(reason.contains("no progress"), "{reason}"), + other => panic!("expected a stand-down, got {other:?}"), + } + assert!(finished.attempts >= 2, "{finished:?}"); + assert!(finished.lessons.is_empty(), "nothing generalised"); + + // Consolidation is per episode, not per attempt. + assert_eq!( + llm.tiers().iter().filter(|t| *t == "consolidate").count(), + 1 + ); + + let record = ledger + .episode("ep-drive") + .await + .expect("read") + .expect("exists"); + assert!(matches!(record.status, EpisodeStatus::StoodDown(_))); + assert_ne!( + record.status, + EpisodeStatus::Running, + "a finished episode must leave the recovery list" + ); +} + +// --------------------------------------------------------------------------- +// The loop acquires a skill: authored, worked, kept, then selected. +// --------------------------------------------------------------------------- + +/// A graph parameterised by a declared input, which is what the authoring +/// prompt asks for and what makes a procedure worth keeping. +fn parameterised() -> WorkflowGraph { + let mut g = tiny("review"); + g.inputs = vec![ + tinyflows::model::WorkflowInput::new("repo", tinyflows::model::InputType::String) + .required(), + ]; + g.nodes[1].config = json!({ "set": { "target": "=run.inputs.repo" } }); + g +} + +/// Authors `graph`, judges every run satisfied, and answers the naming call. +struct Succeeds { + graph: WorkflowGraph, + reusable: bool, + seen: Mutex>, +} + +#[async_trait] +impl LlmProvider for Succeeds { + async fn complete(&self, request: Value, _conn: Option<&str>) -> EngineResult { + self.seen.lock().expect("lock").push(request.clone()); + Ok(match request["tier"].as_str().unwrap_or_default() { + "judge" => json!({ "satisfied": true, "gap": "" }), + "consolidate" => json!({ "lessons": [], "corroborate": [] }), + "select" => json!({ "workflow_id": null, "why": "nothing fits yet" }), + "generalise" => json!({ + "name": "Review a repository's pull requests", + "description": "Reviews the open pull requests on a repository. Takes the repository as an input.", + "reusable": self.reusable, + }), + _ => json!({ + "graph": self.graph, + "why": "nothing stored fits", + "inputs": { "repo": "acme/thing" }, + }), + }) + } +} + +fn succeeding(graph: WorkflowGraph, reusable: bool) -> Arc { + Arc::new(Succeeds { + graph, + reusable, + seen: Mutex::new(Vec::new()), + }) +} + +fn engine_over<'a>( + ledger: &'a dyn Ledger, + store: &'a Arc, + caps: &'a Capabilities, + runner: &'a Local<'a>, + facts: &'a HostFacts, +) -> Loop<'a> { + Loop { + ledger, + store, + caps, + facts, + runner, + clock: &Frozen, + budget: Default::default(), + conn: None, + } +} + +#[tokio::test] +async fn a_graph_that_was_authored_and_worked_becomes_a_stored_procedure() { + // The headline claim: "selects a stored workflow or authors one" is only + // half true if authoring never becomes stored, because then the catalogue + // holds exactly what a person put there and the loop never acquires a skill. + let llm = succeeding(parameterised(), true); + let caps = Capabilities { + llm: llm.clone(), + ..mock_capabilities() + }; + let ledger = MemoryLedger::new(); + let store = store("keep"); + let runner = Local { + caps: &caps, + workspace: &Unobserved, + }; + + assert!(store.list().expect("list").is_empty(), "a cold store"); + + let finished = engine_over(&ledger, &store, &caps, &runner, &HostFacts::unknown()) + .run("ep-learn", &Goal::new("review the PRs on acme/thing")) + .await + .expect("run"); + assert_eq!(finished.status, EpisodeStatus::Satisfied); + + let listed = store.list().expect("list"); + assert_eq!(listed.len(), 1, "the procedure was filed: {listed:?}"); + assert!(listed[0].id.starts_with("learned-"), "{}", listed[0].id); + assert!( + listed[0].description.contains("a repository"), + "described as a class, not as the goal: {}", + listed[0].description + ); + + // Scored from the run that earned it — entering at 0/0 would be + // indistinguishable from a procedure nobody has ever run. + let score = ledger.workflow_score(&listed[0].id).await.expect("score"); + assert_eq!((score.applied, score.helped), (1, 1)); +} + +#[tokio::test] +async fn a_graph_that_pasted_its_inputs_is_not_kept() { + // Same run, same success — but the goal's specifics are welded into a node, + // so it matches one task and never another. No model is asked. + let mut baked = parameterised(); + baked.nodes[1].config = json!({ "set": { "target": "acme/thing" } }); + + let llm = succeeding(baked, true); + let caps = Capabilities { + llm: llm.clone(), + ..mock_capabilities() + }; + let ledger = MemoryLedger::new(); + let store = store("baked"); + let runner = Local { + caps: &caps, + workspace: &Unobserved, + }; + + let finished = engine_over(&ledger, &store, &caps, &runner, &HostFacts::unknown()) + .run("ep-baked", &Goal::new("review the PRs on acme/thing")) + .await + .expect("run"); + + assert_eq!(finished.status, EpisodeStatus::Satisfied, "it still worked"); + assert!( + store.list().expect("list").is_empty(), + "but it is a one-off" + ); + + let tiers: Vec = llm + .seen + .lock() + .expect("lock") + .iter() + .map(|r| r["tier"].as_str().unwrap_or_default().to_string()) + .collect(); + assert!( + !tiers.iter().any(|t| t == "generalise"), + "the mechanical gate settled it without paying for an opinion: {tiers:?}" + ); +} + +#[tokio::test] +async fn the_model_can_still_refuse_a_graph_the_gate_let_through() { + // Parameterised and reusable-looking, but only meaningful for the one goal + // it was written for. The gate cannot see that; a reader can. + let llm = succeeding(parameterised(), false); + let caps = Capabilities { + llm: llm.clone(), + ..mock_capabilities() + }; + let ledger = MemoryLedger::new(); + let store = store("refused"); + let runner = Local { + caps: &caps, + workspace: &Unobserved, + }; + + engine_over(&ledger, &store, &caps, &runner, &HostFacts::unknown()) + .run("ep-refused", &Goal::new("review the PRs on acme/thing")) + .await + .expect("run"); + + assert!(store.list().expect("list").is_empty()); +} + +#[tokio::test] +async fn the_next_episode_selects_what_the_last_one_learned() { + // The whole point, end to end. Episode one finds a cold store and authors; + // episode two finds the procedure episode one filed. + let llm = succeeding(parameterised(), true); + let caps = Capabilities { + llm: llm.clone(), + ..mock_capabilities() + }; + let ledger = MemoryLedger::new(); + let store = store("acquire"); + let runner = Local { + caps: &caps, + workspace: &Unobserved, + }; + + engine_over(&ledger, &store, &caps, &runner, &HostFacts::unknown()) + .run("ep-first", &Goal::new("review the PRs on acme/thing")) + .await + .expect("first"); + + let learned = store.list().expect("list")[0].id.clone(); + llm.seen.lock().expect("lock").clear(); + + engine_over(&ledger, &store, &caps, &runner, &HostFacts::unknown()) + .run("ep-second", &Goal::new("review the PRs on other/repo")) + .await + .expect("second"); + + // The selector was offered it, with the evidence from episode one. + let offered = llm.seen.lock().expect("lock")[0]["messages"][1]["content"] + .as_str() + .unwrap_or_default() + .to_string(); + assert!(offered.contains(&learned), "{offered}"); + assert!( + offered.contains("run 1×, satisfied 1×"), + "carrying what it earned: {offered}" + ); +} + +// --------------------------------------------------------------------------- +// The two stores are independent: any backend beside any other. +// --------------------------------------------------------------------------- + +#[tokio::test] +async fn a_ledger_and_a_vault_of_different_kinds_drive_the_same_loop() { + // `Ledger` and `Vault` are separate traits with separate handles, so the + // host mixes them freely — sqlite ledger beside a Mongo vault, or either + // beside memory. Nothing in the loop knows which it got. + use tinyflows_adaptive::ledger::sqlite::SqliteLedger; + use tinyflows_adaptive::workflows::{Snapshot, Vault, memory::MemoryVault}; + + let dir = std::env::temp_dir().join(format!("adaptive-mixed-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + + // Durable ledger, ephemeral vault. A deliberately silly pairing, chosen + // because if this compiles and runs then every sensible one does. + let ledger = SqliteLedger::open(dir.join("ledger.db")).expect("ledger"); + let vault = MemoryVault::new(); + + let llm = succeeding(parameterised(), true); + let caps = Capabilities { + llm: llm.clone(), + ..mock_capabilities() + }; + let policy: Arc = { + #[derive(Debug, Default)] + struct Permissive; + impl tinyflows::store::HostPolicy for Permissive {} + Arc::new(Permissive) + }; + let snapshot = Snapshot::load(&vault, policy).await.expect("snapshot"); + let store: Arc = Arc::new(snapshot.clone()); + let runner = Local { + caps: &caps, + workspace: &Unobserved, + }; + + let finished = engine_over(&ledger, &store, &caps, &runner, &HostFacts::unknown()) + .run("ep-mixed", &Goal::new("review the PRs on acme/thing")) + .await + .expect("run"); + assert_eq!(finished.status, EpisodeStatus::Satisfied); + + // The episode landed in sqlite; the learned procedure is waiting in the + // snapshot for a flush into the vault. + assert!(ledger.episode("ep-mixed").await.expect("read").is_some()); + assert_eq!(snapshot.pending(), 1, "the procedure it learned"); + snapshot.flush(&vault).await.expect("flush"); + assert_eq!(vault.load().await.expect("load").len(), 1); + + let _ = std::fs::remove_dir_all(&dir); +} + +#[tokio::test] +async fn a_lesson_put_in_front_of_a_planner_is_scored_against_what_happened() { + // `applied` is the denominator of a lesson's help rate, and nothing was + // moving it: `score_lesson` had one caller, the corroboration loop, which + // moves both counters together. So every lesson read 0/0 or n/n, the rate + // carried no information, and every ordering built on it was inert. + use tinyflows_adaptive::ledger::{Lesson, LessonKind}; + + let llm = succeeding(parameterised(), true); + let caps = Capabilities { + llm: llm.clone(), + ..mock_capabilities() + }; + let ledger = MemoryLedger::new(); + let id = ledger + .promote( + &Lesson { + id: String::new(), + kind: LessonKind::Strategy, + trigger: "a report that must cite figures".into(), + mechanism: String::new(), + claim: "read them from the source".into(), + applied: 0, + helped: 0, + scope_key: None, + }, + &[], + ) + .await + .expect("promote"); + + let store = store("scored"); + let runner = Local { + caps: &caps, + workspace: &Unobserved, + }; + engine_over(&ledger, &store, &caps, &runner, &HostFacts::unknown()) + .run("ep-scored", &Goal::new("review the PRs on acme/thing")) + .await + .expect("run"); + + let back = ledger + .lessons(None) + .await + .expect("lessons") + .into_iter() + .find(|l| l.id == id) + .expect("still there"); + assert_eq!(back.applied, 1, "it was shown to the planner"); + assert_eq!(back.helped, 1, "and the episode was satisfied"); +} + +#[tokio::test] +async fn a_lesson_shown_before_a_failure_moves_only_its_denominator() { + use tinyflows_adaptive::ledger::{Lesson, LessonKind}; + + let llm = authoring(); // its judge always says not-satisfied + let caps = caps_with(llm); + let ledger = MemoryLedger::new(); + let id = ledger + .promote( + &Lesson { + id: String::new(), + kind: LessonKind::Strategy, + trigger: "a class of task".into(), + mechanism: String::new(), + claim: "does not actually help".into(), + applied: 0, + helped: 0, + scope_key: None, + }, + &[], + ) + .await + .expect("promote"); + + let store = store("unhelpful"); + let runner = Local { + caps: &caps, + workspace: &Unobserved, + }; + engine_over(&ledger, &store, &caps, &runner, &HostFacts::unknown()) + .run("ep-unhelpful", &Goal::new("write the weekly report")) + .await + .expect("run"); + + let back = ledger + .lessons(None) + .await + .expect("lessons") + .into_iter() + .find(|l| l.id == id) + .expect("still there"); + assert!( + back.applied >= 2, + "shown on every attempt: {}", + back.applied + ); + assert_eq!(back.helped, 0, "and it never helped"); +} + +// --------------------------------------------------------------------------- +// The success gate: a variant exists mid-episode, the device gets it after. +// --------------------------------------------------------------------------- + +/// Drives the whole repair story from a script: select the parent, fail it +/// with a node named, propose a fix, select the fix, and — depending on +/// `satisfied_on` — let it win or keep failing until the stall rule ends it. +struct RepairFlow { + judged: Mutex, + satisfied_on: usize, +} + +impl RepairFlow { + fn new(satisfied_on: usize) -> Arc { + Arc::new(Self { + judged: Mutex::new(0), + satisfied_on, + }) + } +} + +#[async_trait] +impl LlmProvider for RepairFlow { + async fn complete(&self, request: Value, _conn: Option<&str>) -> EngineResult { + let user = request["messages"][1]["content"] + .as_str() + .unwrap_or_default() + .to_string(); + Ok(match request["tier"].as_str().unwrap_or_default() { + "select" => { + // Variant ids are content-derived, so the script cannot know + // them ahead — it reads the listing it was shown, the way a + // real selector would. + let ids: Vec<&str> = user + .lines() + .filter_map(|line| line.trim().strip_prefix("- id: ")) + .collect(); + let chosen = ids + .iter() + .find(|id| id.contains("-fix-")) + .or_else(|| ids.first()); + json!({ "workflow_id": chosen, "why": "it matches", "inputs": {} }) + } + "judge" => { + let mut judged = self.judged.lock().expect("lock"); + *judged += 1; + if *judged >= self.satisfied_on { + json!({ "satisfied": true, "gap": "" }) + } else { + json!({ + "satisfied": false, "blocker": "goal_not_met", + "gap": "the summary never landed", + "attributed_to": "start", "advanced": false + }) + } + } + "repair" => json!({ + "ops": [{ "op": "update_node_config", "id": "start", + "config": { "note": "fixed" } }], + "why": "repointed the binding" + }), + "consolidate" => json!({ "lessons": [], "corroborate": [] }), + other => panic!("no `{other}` call belongs in this flow"), + }) + } +} + +fn permissive() -> Arc { + #[derive(Debug, Default)] + struct Permissive; + impl tinyflows::store::HostPolicy for Permissive {} + Arc::new(Permissive) +} + +#[tokio::test] +async fn the_device_receives_a_variant_only_after_the_goal_run_succeeds() { + use tinyflows_adaptive::workflows::compat::Layered; + use tinyflows_adaptive::workflows::conformance::record; + use tinyflows_adaptive::workflows::memory::MemoryVault; + use tinyflows_adaptive::workflows::{Snapshot, Vault}; + + // The device owns the original; our writable layer starts empty. + let device = Arc::new(MemoryVault::new()); + device.put(&record("pr-review")).await.expect("put"); + let ours = Arc::new(MemoryVault::new()); + let stacked = Layered::new( + vec![("device".into(), device.clone() as Arc)], + ours.clone(), + ); + + let snapshot = Snapshot::load(&stacked, permissive()).await.expect("load"); + let store: Arc = Arc::new(snapshot.clone()); + let caps = Capabilities { + llm: RepairFlow::new(2), + ..mock_capabilities() + }; + let ledger = MemoryLedger::new(); + let runner = Local { + caps: &caps, + workspace: &Unobserved, + }; + + let finished = engine_over(&ledger, &store, &caps, &runner, &HostFacts::unknown()) + .run("ep-gate", &Goal::new("summarise the open pull requests")) + .await + .expect("run"); + + assert_eq!(finished.status, EpisodeStatus::Satisfied); + assert_eq!( + finished.attempts, 2, + "the parent failed once and its variant closed the goal" + ); + + // Mid-episode the variant lived only in the snapshot. The gate is the + // host's one `if`, and it is open: + assert_eq!(snapshot.pending(), 1); + snapshot.flush(&stacked).await.expect("flush"); + + let landed = ours.load().await.expect("load"); + assert_eq!(landed.len(), 1); + assert!( + landed[0].id.starts_with("pr-review-fix-"), + "{}", + landed[0].id + ); + assert_eq!( + device.load().await.expect("load").len(), + 1, + "the parent's home holds exactly what it held before" + ); +} + +#[tokio::test] +async fn a_failed_goal_run_leaves_no_residue_anywhere_durable() { + use tinyflows_adaptive::workflows::compat::Layered; + use tinyflows_adaptive::workflows::conformance::record; + use tinyflows_adaptive::workflows::memory::MemoryVault; + use tinyflows_adaptive::workflows::{Snapshot, Vault}; + + let device = Arc::new(MemoryVault::new()); + device.put(&record("pr-review")).await.expect("put"); + let ours = Arc::new(MemoryVault::new()); + let stacked = Layered::new( + vec![("device".into(), device.clone() as Arc)], + ours.clone(), + ); + + let snapshot = Snapshot::load(&stacked, permissive()).await.expect("load"); + let store: Arc = Arc::new(snapshot.clone()); + let caps = Capabilities { + llm: RepairFlow::new(usize::MAX), // never satisfied; the stall ends it + ..mock_capabilities() + }; + let ledger = MemoryLedger::new(); + let runner = Local { + caps: &caps, + workspace: &Unobserved, + }; + + let finished = engine_over(&ledger, &store, &caps, &runner, &HostFacts::unknown()) + .run( + "ep-no-residue", + &Goal::new("summarise the open pull requests"), + ) + .await + .expect("run"); + + assert!(matches!(finished.status, EpisodeStatus::StoodDown(_))); + assert!( + snapshot.pending() >= 1, + "repairs were proposed and buffered along the way" + ); + + // The gate stays closed: no flush. The knowledge is not lost with the + // graphs — the ledger kept the trail, durably, on the server side. + assert!(ours.load().await.expect("load").is_empty()); + assert_eq!(device.load().await.expect("load").len(), 1); + assert!( + !ledger.rows("ep-no-residue").await.expect("rows").is_empty(), + "the attempts are on the record even though no graph was kept" + ); +} diff --git a/crates/adaptive/tests/execute.rs b/crates/adaptive/tests/execute.rs new file mode 100644 index 0000000..e45cf56 --- /dev/null +++ b/crates/adaptive/tests/execute.rs @@ -0,0 +1,391 @@ +//! Execute, against the real engine. +//! +//! Not a mocked engine: these compile and run actual graphs, because the whole +//! point of the layer is the gap between what the engine returns and what the +//! judge needs, and a mock of the engine would be a mock of exactly that gap. + +use async_trait::async_trait; +use serde_json::{Map, Value, json}; +use tinyflows::caps::mock::mock_capabilities; +use tinyflows::model::{Edge, Node, NodeKind, WorkflowGraph}; +use tinyflows_adaptive::contracts::Approach; +use tinyflows_adaptive::execute::{Unobserved, Workspace, run_attempt}; +use tinyflows_adaptive::intake::Attempt; + +/// Records whether the baseline was taken before the run. +#[derive(Default)] +struct Recording { + calls: std::sync::Mutex>, +} + +#[async_trait] +impl Workspace for Recording { + async fn mark(&self) -> String { + self.calls.lock().expect("lock").push("mark".into()); + "baseline-7".into() + } + async fn changed_since(&self, mark: &str) -> String { + self.calls + .lock() + .expect("lock") + .push(format!("changed_since({mark})")); + "wrote report.md".into() + } +} + +fn node(id: &str, kind: NodeKind, config: Value) -> Node { + Node { + id: id.into(), + kind, + type_version: 1, + name: id.into(), + config, + ports: Vec::new(), + position: None, + } +} + +fn edge(from: &str, to: &str) -> Edge { + Edge { + from_node: from.into(), + from_port: "main".into(), + to_node: to.into(), + to_port: "main".into(), + } +} + +fn graph(nodes: Vec, edges: Vec) -> WorkflowGraph { + WorkflowGraph { + schema_version: 1, + id: Some("t".into()), + name: "t".into(), + inputs: Vec::new(), + agents: Vec::new(), + nodes, + edges, + } +} + +fn attempt(graph: WorkflowGraph) -> Attempt { + Attempt { + approach: Approach::Authored { + why: "for the test".into(), + fingerprint: "0000000".into(), + }, + graph, + inputs: Map::new(), + lessons_shown: Vec::new(), + } +} + +/// One trigger into one transform: the smallest graph that actually does work. +fn working() -> WorkflowGraph { + graph( + vec![ + node( + "start", + NodeKind::Trigger, + json!({"trigger_kind": "manual"}), + ), + node("done", NodeKind::Transform, json!({"set": {"ok": true}})), + ], + vec![edge("start", "done")], + ) +} + +#[tokio::test] +async fn a_clean_run_comes_back_with_a_clean_diagnosis_and_the_host_s_reading() { + let workspace = Recording::default(); + let ran = run_attempt(&attempt(working()), &mock_capabilities(), &workspace).await; + + assert!(ran.failed.is_none(), "{:?}", ran.failed); + assert_eq!(ran.changed, "wrote report.md"); + assert!( + ran.diagnosis.never_ran.is_empty(), + "both nodes ran: {:?}", + ran.diagnosis.never_ran + ); + + // The ordering the trait exists for: a baseline, then the run, then the + // comparison against that same baseline. + let calls = workspace.calls.lock().expect("lock").clone(); + assert_eq!(calls, vec!["mark", "changed_since(baseline-7)"]); +} + +#[tokio::test] +async fn the_diagnosis_is_populated_which_is_the_reason_an_observer_is_attached() { + // A condition that routes past a node. `RunOutcome` alone cannot say this + // happened — the run is green either way — and every downstream gate reads + // `never_ran` to find out. + let g = graph( + vec![ + node( + "start", + NodeKind::Trigger, + json!({"trigger_kind": "manual"}), + ), + node( + "gate", + NodeKind::Condition, + json!({"conditions": [{"left": "=item.nope", "operator": "equals", "right": "yes"}]}), + ), + // An `http_request`, not a transform: `never_ran` deliberately + // reports only the kinds that do outside work, because a routed-past + // transform is not a surprise worth warning about. + node( + "skipped", + NodeKind::HttpRequest, + json!({"url": "https://example.invalid/report", "method": "GET"}), + ), + ], + vec![ + edge("start", "gate"), + Edge { + from_node: "gate".into(), + from_port: "true".into(), + to_node: "skipped".into(), + to_port: "main".into(), + }, + ], + ); + + let ran = run_attempt(&attempt(g), &mock_capabilities(), &Unobserved).await; + + assert!( + ran.failed.is_none(), + "the run itself is fine: {:?}", + ran.failed + ); + assert!( + ran.diagnosis + .never_ran + .iter() + .any(|n| n.node_id == "skipped"), + "a blank diagnosis here would mean nobody looked: {:?}", + ran.diagnosis + ); +} + +#[tokio::test] +async fn a_graph_that_does_not_compile_is_an_attempt_not_an_error() { + // No trigger node. Intake would never return this, but a caller that hand- + // builds an `Attempt` can, and it still has to leave a ledger row. + let g = graph( + vec![node( + "lonely", + NodeKind::Transform, + json!({"set": {"ok": true}}), + )], + Vec::new(), + ); + + let ran = run_attempt(&attempt(g), &mock_capabilities(), &Unobserved).await; + + let failure = ran.failed.expect("it did not compile"); + assert!(!failure.is_empty()); + // Readable through the ordinary evidence path, with no special case. + assert_eq!(ran.outcome.output["error"], json!(failure)); + // And no `nodes` key, so the mechanical missing-evidence check fires. + assert!(ran.outcome.output.get("nodes").is_none()); +} + +#[tokio::test] +async fn a_silent_host_is_silent_rather_than_wrong() { + let ran = run_attempt(&attempt(working()), &mock_capabilities(), &Unobserved).await; + assert!(ran.changed.is_empty()); + assert!(ran.failed.is_none()); + // Empty reads as "nothing reported", never as "nothing happened". + assert!(ran.evidence().changed.is_empty()); +} + +#[tokio::test] +async fn the_evidence_borrows_what_ran_owns() { + let ran = run_attempt(&attempt(working()), &mock_capabilities(), &Unobserved).await; + let evidence = ran.evidence(); + assert!(std::ptr::eq(evidence.outcome, &ran.outcome)); + assert!(std::ptr::eq(evidence.diagnosis, &ran.diagnosis)); +} + +// --------------------------------------------------------------------------- +// The port: local and remote must be indistinguishable to the loop. +// --------------------------------------------------------------------------- + +use tinyflows_adaptive::execute::{Local, Relay, Remote, RunReport, RunRequest, Runner, serve}; + +/// A relay that actually serializes, so the round trip is the real one. +struct Loopback { + seen: std::sync::Mutex>, +} + +#[async_trait] +impl Relay for Loopback { + async fn dispatch(&self, request: &RunRequest) -> Result { + // Out over a wire... + let wire = serde_json::to_string(request).expect("request serializes"); + self.seen.lock().expect("lock").push(wire.clone()); + let received: RunRequest = serde_json::from_str(&wire).expect("request deserializes"); + + // ...run on the far side, exactly as a device would... + let report = serve(&received, &mock_capabilities(), &Unobserved).await; + + // ...and back. + let wire = serde_json::to_string(&report).expect("report serializes"); + Ok(serde_json::from_str(&wire).expect("report deserializes")) + } +} + +/// A model that answers once from a script and counts the asking. +struct Scripted { + replies: std::sync::Mutex>, + calls: std::sync::Mutex, +} + +impl Scripted { + fn new(replies: Vec) -> std::sync::Arc { + std::sync::Arc::new(Self { + replies: std::sync::Mutex::new(replies), + calls: std::sync::Mutex::new(0), + }) + } + fn call_count(&self) -> usize { + *self.calls.lock().expect("lock") + } +} + +#[async_trait] +impl tinyflows::caps::LlmProvider for Scripted { + async fn complete( + &self, + _request: Value, + _conn: Option<&str>, + ) -> tinyflows::error::Result { + *self.calls.lock().expect("lock") += 1; + let mut replies = self.replies.lock().expect("lock"); + assert!( + !replies.is_empty(), + "asked more times than the script answers" + ); + Ok(replies.remove(0)) + } +} + +fn caps_with(llm: std::sync::Arc) -> tinyflows::caps::Capabilities { + tinyflows::caps::Capabilities { + llm, + ..mock_capabilities() + } +} + +struct Dead(&'static str); + +#[async_trait] +impl Relay for Dead { + async fn dispatch(&self, _request: &RunRequest) -> Result { + Err(self.0.to_string()) + } +} + +#[tokio::test] +async fn a_run_relayed_over_a_wire_judges_the_same_as_one_run_in_process() { + // The property the whole port rests on: the loop cannot tell the + // difference, because both paths are serve() + into_ran() with only a + // serialization boundary between them. + let a = attempt(working()); + let caps = mock_capabilities(); + + let here = Local { + caps: &caps, + workspace: &Unobserved, + } + .run(&a) + .await; + + let relay = Loopback { + seen: std::sync::Mutex::new(Vec::new()), + }; + let there = Remote { + relay: &relay, + attempt_id: "ep-1/1".into(), + } + .run(&a) + .await; + + assert_eq!(here.outcome.output, there.outcome.output); + assert_eq!(here.diagnosis, there.diagnosis); + assert_eq!(here.failed, there.failed); + assert_eq!(here.steps.len(), there.steps.len()); + assert_eq!(here.changed, there.changed); +} + +#[tokio::test] +async fn the_graph_crosses_and_the_history_does_not() { + let relay = Loopback { + seen: std::sync::Mutex::new(Vec::new()), + }; + Remote { + relay: &relay, + attempt_id: "ep-9/2".into(), + } + .run(&attempt(working())) + .await; + + let sent = relay.seen.lock().expect("lock")[0].clone(); + assert!(sent.contains("attemptId"), "correlation: {sent}"); + assert!(sent.contains("\"nodes\""), "the graph itself crosses"); + // A runner sees one graph and nothing about the episode it belongs to. + for leak in ["episode", "lesson", "ledger", "approachSig", "verdict"] { + assert!(!sent.contains(leak), "`{leak}` must not cross: {sent}"); + } +} + +#[tokio::test] +async fn a_runner_that_never_answers_still_produces_a_judgeable_attempt() { + let ran = Remote { + relay: &Dead("deadline elapsed after 600s"), + attempt_id: "ep-2/4".into(), + } + .run(&attempt(working())) + .await; + + assert!(ran.failed.is_some()); + assert!(ran.steps.is_empty()); + // And crucially: it does not claim nothing changed, because nobody looked. + // Claiming it would settle the verdict as MissingEvidence, which is + // terminal — ending the episode because a socket blipped. + assert!(!ran.changed.is_empty(), "{}", ran.changed); +} + +#[tokio::test] +async fn an_unanswered_run_is_judged_rather_than_settled_terminally() { + // The end-to-end version of the above: the judge is asked, which is only + // possible because `changed` is not empty. A model that is never called + // panics here, proving the mechanical path did not swallow it. + use tinyflows_adaptive::closing::judge; + use tinyflows_adaptive::contracts::{Blocker, Goal}; + + let ran = Remote { + relay: &Dead("no runner connected"), + attempt_id: "ep-3/1".into(), + } + .run(&attempt(working())) + .await; + + let llm = Scripted::new(vec![json!({ + "satisfied": false, + "blocker": "goal_not_met", + "gap": "the runner never reported, so nothing is established", + "advanced": false + })]); + let verdict = judge( + &Goal::new("write the weekly report"), + &ran.evidence(), + &caps_with(llm.clone()), + None, + ) + .await + .expect("judged"); + + assert_eq!(llm.call_count(), 1, "it reached the judge"); + assert_eq!(verdict.blocker, Blocker::GoalNotMet); + assert!(verdict.blocker.continuable(), "the episode can still retry"); +} diff --git a/crates/adaptive/tests/intake.rs b/crates/adaptive/tests/intake.rs new file mode 100644 index 0000000..bac37fc --- /dev/null +++ b/crates/adaptive/tests/intake.rs @@ -0,0 +1,840 @@ +//! Intake, end to end, against a scripted model and a real store. +//! +//! The unit tests cover rendering and parsing. These cover the decision: that +//! selection is preferred, that authoring is the fallback rather than the +//! default, and that the exclusion list actually excludes — which is the +//! property the whole retry edge rests on and the one that is invisible until +//! an episode has spent an attempt. + +use std::sync::Mutex; + +use async_trait::async_trait; +use serde_json::{Value, json}; +use tinyflows::caps::mock::mock_capabilities; +use tinyflows::caps::{Capabilities, LlmProvider}; +use tinyflows::error::Result as EngineResult; +use tinyflows::model::{Edge, InputType, Node, NodeKind, WorkflowGraph, WorkflowInput}; +use tinyflows::store::types::WorkflowRecord; +use tinyflows::store::{FileWorkflowStore, WorkflowStore}; +use tinyflows_adaptive::contracts::{Approach, Goal}; +use tinyflows_adaptive::host::HostFacts; +use tinyflows_adaptive::intake::decide; +use tinyflows_adaptive::ledger::{Ledger, memory::MemoryLedger}; + +/// A provider that answers from a script and records what it was asked. +struct Scripted { + replies: Mutex>, + seen: Mutex>, +} + +impl Scripted { + fn new(replies: Vec) -> Self { + Self { + replies: Mutex::new(replies), + seen: Mutex::new(Vec::new()), + } + } + + fn prompts(&self) -> Vec { + self.seen.lock().expect("lock").clone() + } +} + +#[async_trait] +impl LlmProvider for Scripted { + async fn complete(&self, request: Value, _conn: Option<&str>) -> EngineResult { + let text = request["messages"] + .as_array() + .map(|m| { + m.iter() + .filter_map(|msg| msg["content"].as_str()) + .collect::>() + .join("\n") + }) + .unwrap_or_default(); + self.seen.lock().expect("lock").push(text); + + let mut replies = self.replies.lock().expect("lock"); + if replies.is_empty() { + panic!("the model was asked more times than the script has answers"); + } + Ok(replies.remove(0)) + } +} + +/// The engine's mock bundle with only the model replaced: nothing in intake +/// touches tools, HTTP, code or state, so scripting those too would be noise. +fn caps_with(llm: std::sync::Arc) -> Capabilities { + Capabilities { + llm, + ..mock_capabilities() + } +} + +/// A store on a fresh temp directory, so each case starts empty. +fn empty_store(tag: &str) -> (FileWorkflowStore, std::path::PathBuf) { + let root = std::env::temp_dir().join(format!("adaptive-intake-{}-{tag}", std::process::id())); + let _ = std::fs::remove_dir_all(&root); + std::fs::create_dir_all(root.join("workflows")).expect("temp dir"); + let store = FileWorkflowStore::new(vec![root.join("workflows")], root.join("runs")); + (store, root) +} + +/// A minimal graph that validates: one trigger, one transform. +fn tiny_graph(name: &str, required_input: Option<&str>) -> WorkflowGraph { + WorkflowGraph { + schema_version: 1, + id: Some(name.to_string()), + name: name.to_string(), + inputs: required_input + .map(|n| vec![WorkflowInput::new(n, InputType::String).required()]) + .unwrap_or_default(), + agents: Vec::new(), + nodes: vec![ + Node { + id: "start".into(), + kind: NodeKind::Trigger, + type_version: 1, + name: "manual".into(), + config: json!({ "trigger_kind": "manual" }), + ports: Vec::new(), + position: None, + }, + Node { + id: "done".into(), + kind: NodeKind::Transform, + type_version: 1, + name: "done".into(), + config: json!({ "set": { "ok": true } }), + ports: Vec::new(), + position: None, + }, + ], + edges: vec![Edge { + from_node: "start".into(), + from_port: "main".into(), + to_node: "done".into(), + to_port: "main".into(), + }], + } +} + +fn stored(id: &str, description: &str, required_input: Option<&str>) -> WorkflowRecord { + WorkflowRecord { + id: id.to_string(), + name: id.to_string(), + description: description.to_string(), + enabled: true, + defaults: Default::default(), + graph: tiny_graph(id, required_input), + source_path: None, + } +} + +#[tokio::test] +async fn an_empty_store_authors_without_asking_whether_to_select() { + // With nothing to choose from the answer can only be "none". Spending a + // call to be told so is the cost of every cold start. + let llm = std::sync::Arc::new(Scripted::new(vec![json!({ + "graph": tiny_graph("fresh", None), + "why": "nothing stored", + "inputs": {}, + })])); + let caps = caps_with(llm.clone()); + let (store, _root) = empty_store("1"); + let ledger = MemoryLedger::new(); + + let attempt = decide( + &Goal::new("do a new thing"), + "ep1", + &store, + &ledger, + &HostFacts::unknown(), + &caps, + None, + ) + .await + .expect("decide"); + + assert!(matches!(attempt.approach, Approach::Authored { .. })); + assert_eq!( + llm.prompts().len(), + 1, + "exactly one call: the authoring one" + ); + assert!( + llm.prompts()[0].contains("Node catalogue"), + "authoring must be grounded on the catalogue" + ); +} + +#[tokio::test] +async fn a_matching_workflow_is_selected_and_its_graph_is_loaded() { + let llm = std::sync::Arc::new(Scripted::new(vec![json!({ + "workflow_id": "pr-review", + "why": "does exactly this", + "inputs": {}, + })])); + let caps = caps_with(llm.clone()); + let (store, _root) = empty_store("2"); + store + .save(&stored("pr-review", "reviews a closed issue", None)) + .expect("save"); + let ledger = MemoryLedger::new(); + + let attempt = decide( + &Goal::new("review a closed issue"), + "ep1", + &store, + &ledger, + &HostFacts::unknown(), + &caps, + None, + ) + .await + .expect("decide"); + + match attempt.approach { + Approach::Selected { workflow_id, .. } => assert_eq!(workflow_id, "pr-review"), + other => panic!("expected a selection, got {other:?}"), + } + // The bug this catches: `select` answers with an id, and returning that + // unbound hands the engine an empty graph that compiles to nothing. + assert_eq!( + attempt.graph.nodes.len(), + 2, + "the stored graph must be loaded" + ); + assert_eq!(llm.prompts().len(), 1, "a hit must not also author"); +} + +#[tokio::test] +async fn declining_falls_through_to_authoring() { + let llm = std::sync::Arc::new(Scripted::new(vec![ + json!({ "workflow_id": null, "why": "none of these fetch anything" }), + json!({ "graph": tiny_graph("written", None), "why": "had to write one", "inputs": {} }), + ])); + let caps = caps_with(llm.clone()); + let (store, _root) = empty_store("3"); + store + .save(&stored("unrelated", "does something else", None)) + .expect("save"); + let ledger = MemoryLedger::new(); + + let attempt = decide( + &Goal::new("something new"), + "ep1", + &store, + &ledger, + &HostFacts::unknown(), + &caps, + None, + ) + .await + .expect("decide"); + + assert!(matches!(attempt.approach, Approach::Authored { .. })); + assert_eq!( + llm.prompts().len(), + 2, + "selection was asked first, then authoring" + ); +} + +#[tokio::test] +async fn a_workflow_already_tried_this_episode_is_not_offered_again() { + // The property the whole retry edge rests on. Without it attempt two + // re-selects what attempt one already failed on, and the episode pays + // twice for one dead end. + let llm = std::sync::Arc::new(Scripted::new(vec![json!({ + "graph": tiny_graph("written", None), + "why": "the only candidate was already spent", + "inputs": {}, + })])); + let caps = caps_with(llm.clone()); + let (store, _root) = empty_store("4"); + store + .save(&stored("pr-review", "reviews a closed issue", None)) + .expect("save"); + + let ledger = MemoryLedger::new(); + let mut spent = tinyflows_adaptive::ledger::conformance::row("ep1", 1, "selected:pr-review"); + spent.workflow_id = Some("pr-review".to_string()); + ledger.append(&spent).await.expect("append"); + + let attempt = decide( + &Goal::new("review a closed issue"), + "ep1", + &store, + &ledger, + &HostFacts::unknown(), + &caps, + None, + ) + .await + .expect("decide"); + + assert!( + matches!(attempt.approach, Approach::Authored { .. }), + "the only stored workflow was excluded, so authoring is the only path left" + ); + assert_eq!( + llm.prompts().len(), + 1, + "with every candidate excluded the list is empty and selection is skipped entirely" + ); +} + +#[tokio::test] +async fn a_selection_whose_required_input_is_missing_is_refused_before_it_runs() { + // The model is confident about inputs it did not find in the goal. The + // cheap deterministic check catches what the expensive one asserted. + let llm = std::sync::Arc::new(Scripted::new(vec![json!({ + "workflow_id": "needs-repo", + "why": "matches", + "inputs": {}, + })])); + let caps = caps_with(llm); + let (store, _root) = empty_store("5"); + store + .save(&stored("needs-repo", "reviews PRs in a repo", Some("repo"))) + .expect("save"); + let ledger = MemoryLedger::new(); + + let err = decide( + &Goal::new("review the PRs"), + "ep1", + &store, + &ledger, + &HostFacts::unknown(), + &caps, + None, + ) + .await + .expect_err("an unbindable selection must not reach the engine"); + + assert!( + err.to_string().contains("repo"), + "the error names the missing input: {err}" + ); +} + +#[tokio::test] +async fn a_hallucinated_workflow_id_reads_as_a_decline() { + let llm = std::sync::Arc::new(Scripted::new(vec![ + json!({ "workflow_id": "pr-reviewer", "why": "close, but no such id" }), + json!({ "graph": tiny_graph("written", None), "why": "wrote one", "inputs": {} }), + ])); + let caps = caps_with(llm.clone()); + let (store, _root) = empty_store("6"); + store + .save(&stored("pr-review", "reviews a closed issue", None)) + .expect("save"); + let ledger = MemoryLedger::new(); + + let attempt = decide( + &Goal::new("review something"), + "ep1", + &store, + &ledger, + &HostFacts::unknown(), + &caps, + None, + ) + .await + .expect("decide"); + + assert!( + matches!(attempt.approach, Approach::Authored { .. }), + "a name that is not on the list is a hallucination, not a lookup" + ); +} + +#[tokio::test] +async fn an_authored_graph_that_does_not_validate_is_an_error_not_a_return_value() { + // Handing it back would turn an authoring mistake into a run-time failure + // that reads like the work failing. + let llm = std::sync::Arc::new(Scripted::new(vec![json!({ + "graph": { "schema_version": 1, "name": "empty", "nodes": [], "edges": [] }, + "why": "forgot the trigger", + "inputs": {}, + })])); + let caps = caps_with(llm); + let (store, _root) = empty_store("7"); + let ledger = MemoryLedger::new(); + + let err = decide( + &Goal::new("anything"), + "ep1", + &store, + &ledger, + &HostFacts::unknown(), + &caps, + None, + ) + .await + .expect_err("an invalid graph must not leave intake"); + assert!(err.to_string().contains("invalid"), "{err}"); +} + +#[tokio::test] +async fn a_disabled_workflow_is_never_offered() { + let llm = std::sync::Arc::new(Scripted::new(vec![json!({ + "graph": tiny_graph("written", None), + "why": "the only one was disabled", + "inputs": {}, + })])); + let caps = caps_with(llm.clone()); + let (store, _root) = empty_store("8"); + let mut off = stored("switched-off", "would have matched", None); + off.enabled = false; + store.save(&off).expect("save"); + let ledger = MemoryLedger::new(); + + decide( + &Goal::new("do the thing"), + "ep1", + &store, + &ledger, + &HostFacts::unknown(), + &caps, + None, + ) + .await + .expect("decide"); + + assert_eq!( + llm.prompts().len(), + 1, + "offering a disabled workflow invites a choice that cannot be honoured" + ); +} + +#[tokio::test] +async fn a_graph_naming_a_worker_this_host_lacks_is_refused_before_it_runs() { + // The whole point of collecting host facts. Without this the graph saves + // cleanly, validates cleanly, and fails at run time — usually overnight, + // to nobody watching. + let mut agent_graph = tiny_graph("uses-an-agent", None); + agent_graph.nodes[1] = Node { + id: "work".into(), + kind: NodeKind::Agent, + type_version: 1, + name: "do it".into(), + config: json!({ "prompt": "do the thing", "agent_ref": "desktop" }), + ports: Vec::new(), + position: None, + }; + agent_graph.edges[0].to_node = "work".into(); + + let llm = std::sync::Arc::new(Scripted::new(vec![json!({ + "graph": agent_graph, "why": "needs an agent", "inputs": {}, + })])); + let caps = caps_with(llm); + let (store, _root) = empty_store("gated"); + let ledger = MemoryLedger::new(); + + let facts = HostFacts { + workers: vec!["laptop".into(), "ci".into()], + default_worker: Some("laptop".into()), + ..HostFacts::unknown() + }; + + let err = decide( + &Goal::new("do the thing"), + "ep1", + &store, + &ledger, + &facts, + &caps, + None, + ) + .await + .expect_err("a worker this host lacks must not reach the engine"); + + assert!( + err.to_string().contains("desktop"), + "the error names it: {err}" + ); + assert!( + err.to_string().contains("laptop"), + "and offers the alternatives: {err}" + ); +} + +#[tokio::test] +async fn the_authoring_prompt_carries_what_the_host_permits() { + let llm = std::sync::Arc::new(Scripted::new(vec![json!({ + "graph": tiny_graph("fine", None), "why": "ok", "inputs": {}, + })])); + let caps = caps_with(llm.clone()); + let (store, _root) = empty_store("facts-rendered"); + let ledger = MemoryLedger::new(); + + let facts = HostFacts { + workers: vec!["laptop".into()], + default_worker: None, + allow_code: Some(false), + notes: vec!["Only manual triggers fire here.".into()], + ..HostFacts::unknown() + }; + + decide( + &Goal::new("anything"), + "ep1", + &store, + &ledger, + &facts, + &caps, + None, + ) + .await + .expect("decide"); + + let prompt = &llm.prompts()[0]; + assert!(prompt.contains("What this host permits"), "{prompt}"); + assert!(prompt.contains("every agent node must name config.agent_ref")); + assert!(prompt.contains("Only manual triggers fire here.")); +} + +// --------------------------------------------------------------------------- +// Promotion: a repaired family is one row, and score decides which. +// --------------------------------------------------------------------------- + +/// A parent and one variant, both stored and linked, with scores applied. +async fn repaired_family( + tag: &str, + parent: (u32, u32), + variant: (u32, u32), +) -> (FileWorkflowStore, MemoryLedger, std::path::PathBuf) { + let (store, root) = empty_store(tag); + store + .save(&stored("weekly", "writes the weekly report", None)) + .expect("save"); + store + .save(&stored( + "weekly-fix-1", + "writes the weekly report, with the binding corrected", + None, + )) + .expect("save"); + + let ledger = MemoryLedger::new(); + ledger + .link_variant("weekly", "weekly-fix-1") + .await + .expect("link"); + for (id, (applied, helped)) in [("weekly", parent), ("weekly-fix-1", variant)] { + for n in 0..applied { + ledger.score_workflow(id, n < helped).await.expect("score"); + } + } + (store, ledger, root) +} + +/// What the selector was actually shown. +async fn offered(store: &FileWorkflowStore, ledger: &MemoryLedger) -> String { + let llm = std::sync::Arc::new(Scripted::new(vec![ + json!({"workflow_id": "none"}), + json!({ + "graph": tiny_graph("fallback", None), + "why": "declined", + "inputs": {}, + }), + ])); + let caps = caps_with(llm.clone()); + let _ = decide( + &Goal::new("write the weekly report"), + "ep-promo", + store, + ledger, + &HostFacts::unknown(), + &caps, + None, + ) + .await; + llm.prompts().first().cloned().unwrap_or_default() +} + +#[tokio::test] +async fn a_repaired_family_is_offered_as_one_row_not_two() { + // Two near-identical graphs whose descriptions differ by a clause is not a + // choice, it is noise. + let (store, ledger, _root) = repaired_family("promo-1", (40, 40), (0, 0)).await; + let shown = offered(&store, &ledger).await; + let rows = shown.matches("weekly").count(); + assert!(rows > 0, "the family must be offered at all: {shown}"); + assert!( + !shown.contains("weekly-fix-1"), + "an unproven variant must not appear beside its proven parent: {shown}" + ); +} + +#[tokio::test] +async fn a_fresh_variant_does_not_displace_a_proven_parent() { + let (store, ledger, _root) = repaired_family("promo-2", (40, 40), (0, 0)).await; + let shown = offered(&store, &ledger).await; + assert!(shown.contains("weekly"), "{shown}"); + assert!(!shown.contains("weekly-fix-1"), "{shown}"); +} + +#[tokio::test] +async fn a_variant_that_has_proven_better_is_the_one_offered() { + // Promotion on score, not on having been written. + let (store, ledger, _root) = repaired_family("promo-3", (10, 5), (4, 4)).await; + let shown = offered(&store, &ledger).await; + assert!( + shown.contains("weekly-fix-1"), + "the better member must take the position: {shown}" + ); +} + +#[tokio::test] +async fn a_family_whose_champion_was_already_tried_still_offers_its_variant() { + // The case that matters most and is easiest to get wrong: this episode just + // failed with the parent, so the parent is excluded — and the variant + // exists *because* the parent fell short. Dropping the whole family would + // hide the one graph written for this exact situation. + let (store, ledger, _root) = repaired_family("promo-4", (40, 40), (0, 0)).await; + ledger + .append(&tinyflows_adaptive::ledger::LedgerRow { + id: String::new(), + episode: "ep-promo".into(), + attempt: 1, + approach_sig: "selected:weekly".into(), + approach_desc: "the champion".into(), + workflow_id: Some("weekly".into()), + outcome: "fell short".into(), + cause: String::new(), + cost_usd: 0.0, + at: "2026-01-01T00:00:00Z".into(), + satisfied: false, + advanced: false, + }) + .await + .expect("append"); + + let shown = offered(&store, &ledger).await; + assert!( + shown.contains("weekly-fix-1"), + "the variant must survive its champion being excluded: {shown}" + ); +} + +// --------------------------------------------------------------------------- +// The retry edge: attempt four must not be attempt two in different words. +// --------------------------------------------------------------------------- + +async fn with_history(tag: &str) -> (FileWorkflowStore, MemoryLedger, std::path::PathBuf) { + let (store, root) = empty_store(tag); + let ledger = MemoryLedger::new(); + for (attempt, sig, desc, cause) in [ + ( + 1u32, + "authored:aaa", + "fetched the log and summarised it", + "no numbers in it", + ), + ( + 2, + "authored:bbb", + "asked an agent to write it from memory", + "it invented the figures", + ), + ] { + ledger + .append(&tinyflows_adaptive::ledger::LedgerRow { + id: String::new(), + episode: "ep-retry".into(), + attempt, + approach_sig: sig.into(), + approach_desc: desc.into(), + workflow_id: None, + outcome: "fell short".into(), + cause: cause.into(), + cost_usd: 0.0, + at: "2026-01-01T00:00:00Z".into(), + satisfied: false, + advanced: false, + }) + .await + .expect("append"); + } + (store, ledger, root) +} + +#[tokio::test] +async fn the_author_is_shown_what_this_episode_already_tried() { + // Without this the author writes attempt two's graph again, confidently, + // because nothing told it otherwise. The exclusion list only guards + // *selection*; authoring has no structural guard at all. + let (store, ledger, _root) = with_history("retry-1").await; + let llm = std::sync::Arc::new(Scripted::new(vec![json!({ + "graph": tiny_graph("third-idea", None), + "why": "the first two both trusted the model for figures", + "inputs": {}, + })])); + let caps = caps_with(llm.clone()); + + decide( + &Goal::new("write the weekly report"), + "ep-retry", + &store, + &ledger, + &HostFacts::unknown(), + &caps, + None, + ) + .await + .expect("decide"); + + let prompt = &llm.prompts()[0]; + assert!(prompt.contains("Already tried this episode"), "{prompt}"); + assert!( + prompt.contains("asked an agent to write it from memory"), + "{prompt}" + ); + assert!(prompt.contains("it invented the figures"), "{prompt}"); + assert!(prompt.contains("write something\nDIFFERENT"), "{prompt}"); +} + +#[tokio::test] +async fn the_selector_is_shown_the_same_history_in_the_same_words() { + let (store, ledger, _root) = with_history("retry-2").await; + store + .save(&stored("weekly", "writes the weekly report", None)) + .expect("save"); + + let llm = std::sync::Arc::new(Scripted::new(vec![json!({ + "workflow_id": "weekly", + "why": "it does this", + "inputs": {}, + })])); + let caps = caps_with(llm.clone()); + + decide( + &Goal::new("write the weekly report"), + "ep-retry", + &store, + &ledger, + &HostFacts::unknown(), + &caps, + None, + ) + .await + .expect("decide"); + + let prompt = &llm.prompts()[0]; + assert!(prompt.contains("Already tried this episode"), "{prompt}"); + assert!(prompt.contains("no numbers in it"), "{prompt}"); +} + +#[tokio::test] +async fn lessons_from_other_episodes_reach_the_planner() { + // consolidate() was writing these and nothing was reading them — a + // knowledge store that costs money and returns nothing. + let (store, root) = empty_store("retry-3"); + let _ = root; + let ledger = MemoryLedger::new(); + ledger + .promote( + &tinyflows_adaptive::ledger::Lesson { + id: String::new(), + kind: tinyflows_adaptive::ledger::LessonKind::Constraint, + trigger: "a report that must cite figures".into(), + mechanism: "the model has no access to the numbers".into(), + claim: "read them from the source rather than asking an agent".into(), + applied: 0, + helped: 0, + scope_key: None, + }, + &[], + ) + .await + .expect("promote"); + + let llm = std::sync::Arc::new(Scripted::new(vec![json!({ + "graph": tiny_graph("informed", None), + "why": "nothing stored", + "inputs": {}, + })])); + let caps = caps_with(llm.clone()); + + decide( + &Goal::new("write the weekly report"), + "ep-fresh", + &store, + &ledger, + &HostFacts::unknown(), + &caps, + None, + ) + .await + .expect("decide"); + + let prompt = &llm.prompts()[0]; + assert!(prompt.contains("Learned from earlier episodes"), "{prompt}"); + assert!(prompt.contains("read them from the source"), "{prompt}"); +} + +#[tokio::test] +async fn a_first_attempt_is_told_nothing_it_would_have_to_ignore() { + // An empty history section is noise a model has to read past, and an + // empty "already tried" heading reads as a claim that something was. + let (store, _root) = empty_store("retry-4"); + let ledger = MemoryLedger::new(); + let llm = std::sync::Arc::new(Scripted::new(vec![json!({ + "graph": tiny_graph("first", None), + "why": "nothing stored", + "inputs": {}, + })])); + let caps = caps_with(llm.clone()); + + decide( + &Goal::new("write the weekly report"), + "ep-first", + &store, + &ledger, + &HostFacts::unknown(), + &caps, + None, + ) + .await + .expect("decide"); + + let prompt = &llm.prompts()[0]; + assert!(!prompt.contains("Already tried"), "{prompt}"); + assert!(!prompt.contains("Learned from earlier"), "{prompt}"); +} + +#[tokio::test] +async fn two_authored_attempts_leave_two_distinct_signatures() { + // The fingerprint end to end: a differently-shaped graph must not fold into + // the same exclusion-list entry as the one before it. + let (store, _root) = empty_store("retry-5"); + let ledger = MemoryLedger::new(); + + let mut signatures = Vec::new(); + for (n, name) in [(0, "shape-one"), (1, "shape-two")] { + let llm = std::sync::Arc::new(Scripted::new(vec![json!({ + "graph": tiny_graph(name, if n == 1 { Some("repo") } else { None }), + "why": "nothing stored", + "inputs": { "repo": "acme/thing" }, + })])); + let attempt = decide( + &Goal::new("write the weekly report"), + "ep-sigs", + &store, + &ledger, + &HostFacts::unknown(), + &caps_with(llm), + None, + ) + .await + .expect("decide"); + signatures.push(attempt.approach.signature()); + } + + assert_ne!(signatures[0], signatures[1], "{signatures:?}"); + assert!(signatures[0].starts_with("authored:"), "{signatures:?}"); +} diff --git a/src/nodes/control_flow/void.rs b/src/nodes/control_flow/void.rs index e94383b..1fc5ad3 100644 --- a/src/nodes/control_flow/void.rs +++ b/src/nodes/control_flow/void.rs @@ -42,7 +42,7 @@ //! None. A `void` node's `name` is where the human reason goes ("Fire and //! forget: audit log") — it is already required, and unlike a config key it is //! rendered by [`crate::visualization`]. Config is ignored entirely, including -//! `={{ … }}` expressions, so this node can emit no binding diagnostics. +//! `=`-expressions, so this node can emit no binding diagnostics. //! //! # What it leaves behind //!