From 4d4c065087be9807e532d9e358cd620c7f2e5937 Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Tue, 6 Oct 2026 01:10:06 -0600 Subject: [PATCH] refactor(memory)!: remove the unused memory experiment, improvement, and benchmark runners Nothing outside this repository imports runAgentMemoryExperiment, runAgentMemoryLearningExperiment, runAgentMemoryImprovement, runKnowledgeBenchmarkSuite, or runMemoryAdapterBenchmark. They duplicated experiment and improvement machinery that Agent Eval owns. Memory adapters, branches, holdout, lifecycle bounds, play memory tools, and the benchmark adapters Supervisor Lab imports stay. BREAKING CHANGE: the memory experiment, memory improvement, and benchmark suite exports are removed. --- CHANGELOG.md | 3 + README.md | 69 +- api-surface.json | 212 +----- scripts/verify-package.mjs | 2 +- skills/build-with-agent-knowledge/SKILL.md | 1 - src/benchmarks/catalog.ts | 661 ---------------- src/benchmarks/index.ts | 60 -- src/benchmarks/memory-recovery.ts | 531 ------------- src/benchmarks/memory-responder.ts | 263 ------- src/benchmarks/memory-runner.ts | 548 ------------- src/benchmarks/metrics.ts | 117 --- src/benchmarks/retrieval.ts | 95 --- src/benchmarks/scoring.ts | 231 ------ src/benchmarks/suite.ts | 180 ----- src/benchmarks/types.ts | 347 --------- src/benchmarks/utils.ts | 21 - src/benchmarks/validation.ts | 83 -- src/candidate-ranking.test.ts | 85 --- src/candidate-ranking.ts | 43 -- src/memory/attempt-log.ts | 287 ------- src/memory/experiment.ts | 45 -- src/memory/experiment/cases.ts | 132 ---- src/memory/experiment/cell.ts | 493 ------------ src/memory/experiment/comparison-ref.ts | 85 --- src/memory/experiment/cost.ts | 58 -- src/memory/experiment/execution-context.ts | 171 ----- src/memory/experiment/learning-evidence.ts | 178 ----- src/memory/experiment/learning-metrics.ts | 470 ------------ src/memory/experiment/learning-pairs.ts | 457 ----------- src/memory/experiment/learning.ts | 111 --- src/memory/experiment/metrics.ts | 126 --- src/memory/experiment/probe-evaluation.ts | 53 -- src/memory/experiment/recovery.ts | 515 ------------- src/memory/experiment/run.ts | 375 --------- src/memory/experiment/runtime.ts | 98 --- src/memory/experiment/types.ts | 458 ----------- src/memory/experiment/validation.ts | 245 ------ src/memory/improvement.ts | 13 - src/memory/improvement/activation.ts | 247 ------ src/memory/improvement/candidate.ts | 109 --- src/memory/improvement/evaluation.ts | 459 ----------- src/memory/improvement/identity.ts | 106 --- src/memory/improvement/output.ts | 30 - src/memory/improvement/promotion.ts | 205 ----- src/memory/improvement/run.ts | 290 ------- src/memory/improvement/types.ts | 209 ----- src/memory/improvement/validation.ts | 143 ---- src/memory/index.ts | 3 - src/memory/run-control.ts | 93 --- src/statistics.test.ts | 16 - tests/benchmarks/catalog.test.ts | 309 -------- tests/benchmarks/cost.test.ts | 288 ------- tests/benchmarks/execution.test.ts | 201 ----- tests/benchmarks/recovery.test.ts | 420 ---------- tests/memory/experiment-cost.test.ts | 693 ----------------- tests/memory/experiment-execution.test.ts | 328 -------- .../experiment-learning-validation.test.ts | 55 -- tests/memory/experiment-learning.test.ts | 506 ------------ tests/memory/experiment-parallel.test.ts | 149 ---- tests/memory/experiment-privacy.test.ts | 306 -------- .../memory/experiment-recovery-safety.test.ts | 302 -------- tests/memory/experiment-recovery.test.ts | 719 ------------------ tests/memory/experiment-safety.test.ts | 610 --------------- tests/memory/graphiti.test.ts | 24 - tests/memory/improvement.test.ts | 624 --------------- tests/memory/mem0.test.ts | 25 - tests/support/benchmarks.ts | 13 - tests/support/memory-learning.ts | 143 ---- tests/support/memory.ts | 32 +- 69 files changed, 11 insertions(+), 15568 deletions(-) delete mode 100644 src/benchmarks/catalog.ts delete mode 100644 src/benchmarks/memory-recovery.ts delete mode 100644 src/benchmarks/memory-responder.ts delete mode 100644 src/benchmarks/memory-runner.ts delete mode 100644 src/benchmarks/metrics.ts delete mode 100644 src/benchmarks/retrieval.ts delete mode 100644 src/benchmarks/scoring.ts delete mode 100644 src/benchmarks/suite.ts delete mode 100644 src/benchmarks/types.ts delete mode 100644 src/benchmarks/utils.ts delete mode 100644 src/benchmarks/validation.ts delete mode 100644 src/candidate-ranking.test.ts delete mode 100644 src/candidate-ranking.ts delete mode 100644 src/memory/attempt-log.ts delete mode 100644 src/memory/experiment.ts delete mode 100644 src/memory/experiment/cases.ts delete mode 100644 src/memory/experiment/cell.ts delete mode 100644 src/memory/experiment/comparison-ref.ts delete mode 100644 src/memory/experiment/cost.ts delete mode 100644 src/memory/experiment/execution-context.ts delete mode 100644 src/memory/experiment/learning-evidence.ts delete mode 100644 src/memory/experiment/learning-metrics.ts delete mode 100644 src/memory/experiment/learning-pairs.ts delete mode 100644 src/memory/experiment/learning.ts delete mode 100644 src/memory/experiment/metrics.ts delete mode 100644 src/memory/experiment/probe-evaluation.ts delete mode 100644 src/memory/experiment/recovery.ts delete mode 100644 src/memory/experiment/run.ts delete mode 100644 src/memory/experiment/runtime.ts delete mode 100644 src/memory/experiment/types.ts delete mode 100644 src/memory/experiment/validation.ts delete mode 100644 src/memory/improvement.ts delete mode 100644 src/memory/improvement/activation.ts delete mode 100644 src/memory/improvement/candidate.ts delete mode 100644 src/memory/improvement/evaluation.ts delete mode 100644 src/memory/improvement/identity.ts delete mode 100644 src/memory/improvement/output.ts delete mode 100644 src/memory/improvement/promotion.ts delete mode 100644 src/memory/improvement/run.ts delete mode 100644 src/memory/improvement/types.ts delete mode 100644 src/memory/improvement/validation.ts delete mode 100644 src/memory/run-control.ts delete mode 100644 tests/benchmarks/catalog.test.ts delete mode 100644 tests/benchmarks/cost.test.ts delete mode 100644 tests/benchmarks/execution.test.ts delete mode 100644 tests/benchmarks/recovery.test.ts delete mode 100644 tests/memory/experiment-cost.test.ts delete mode 100644 tests/memory/experiment-execution.test.ts delete mode 100644 tests/memory/experiment-learning-validation.test.ts delete mode 100644 tests/memory/experiment-learning.test.ts delete mode 100644 tests/memory/experiment-parallel.test.ts delete mode 100644 tests/memory/experiment-privacy.test.ts delete mode 100644 tests/memory/experiment-recovery-safety.test.ts delete mode 100644 tests/memory/experiment-recovery.test.ts delete mode 100644 tests/memory/experiment-safety.test.ts delete mode 100644 tests/memory/improvement.test.ts delete mode 100644 tests/support/benchmarks.ts delete mode 100644 tests/support/memory-learning.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 2ed7de6..45d7eb6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,9 @@ Knowledge no longer browses or calls models itself; supply research through `run Removed: `runVerifiedResearchLoop`, `createWebResearchWorker`, `createVerifyingResearchDriver`, `createTangleRouterClient`, `RouterError`, `createAdaptiveResearchDriver`, `createCollectionResearchDriver`, `createResearchDrivingDriver`, `createPersistentResearchDrivingDriver`, `createClaimGroundingVerifier`, `createClaimDecorator`, the cited-claim helpers, the investment-thesis set, task, and material-facts metric, and their types. `groundClaimInText`, claim ledgers, `TrackedClaim`, store formats, and every other public operation are unchanged. The verified-research paper and its results remain as a historical record linked to the last commit with that code. +Remove the memory experiment, memory improvement, and benchmark-suite runners, which no consumer imported: `runAgentMemoryExperiment`, `runAgentMemoryLearningExperiment`, `runAgentMemoryImprovement`, `runKnowledgeBenchmarkSuite`, `runMemoryAdapterBenchmark`, the industry benchmark catalogs, qrels import, benchmark scoring, and their run-lease, cost, and comparison types. +Compare memory providers on product tasks with Agent Eval. +Memory adapters, branches, holdout, lifecycle bounds, play memory tools, and the `/benchmarks` in-memory and no-op adapters are unchanged. ## 19.1.5 diff --git a/README.md b/README.md index f53f83b..7c9288d 100644 --- a/README.md +++ b/README.md @@ -31,9 +31,8 @@ Use Knowledge 17 with older Eval releases. | Optimize retrieval or a complete RAG configuration | `runRetrievalImprovementLoop`, `runRagOptimization` | package root | | Optimize a KB maintenance policy | `optimizeKnowledgeBasePolicy` | package root | | Run retrieval, research, answer checks, and promotion as one process | `runRagKnowledgeImprovementLoop` | package root | -| Compare providers or optimize memory configuration | `AgentMemoryAdapter`, `runAgentMemoryImprovement` | `/memory` | +| Connect a memory provider or branch its state | `AgentMemoryAdapter`, `createAgentMemoryBranch` | `/memory` | | Read from external authorities | `KnowledgeSource` and source adapters | `/sources` | -| Run retrieval, answer, KB, or memory benchmark cases | `runKnowledgeBenchmarkSuite` | `/benchmarks` | | Use live research or coding agents | `runKnowledgeImprovementJob` | `@tangle-network/agent-runtime` | ## Create and search a knowledge base @@ -461,75 +460,17 @@ The application must enforce evaluator and auditor separation and retain evidenc The package is not a memory database. Install the provider you use, create its client, and pass that client to the adapter. -The memory APIs support scoped reads and writes, isolated branches, ordered histories, independent train, selection, and final comparisons, and adapter experiments. -Use them to compare a provider against no memory or another provider on the same tasks before changing production behavior. -`runAgentMemoryImprovement` accepts a complete `OptimizationMethod`, evaluates each serialized configuration in an isolated provider branch, and activates only a winner that passes a separate final comparison. -Set `implementationRef` to `git:<40 lowercase hex>` or `sha256:<64 lowercase hex>` covering the installed implementation, method configuration, candidate construction, execution behavior, and external configuration so incompatible state cannot resume. -The run records one immutable candidate reference for each memory configuration and refuses cached results if that reference changes. -Critical dimensions use Eval's deciding interval and observation minimum. -Missing, insufficient, or indeterminate safety evidence holds activation. -An interval that crosses the safety margin remains uncertain; it is not reported as an observed regression. -Set `significance.independentUnitByScenarioId` when final sequences share a source unit. -The run captures this mapping before execution and binds it to resume identity. -Paired repetitions retain their coverage without increasing the independent-unit count. -Each improvement candidate declares a maximum for one sequence and one recovery attempt. -The adapter must enforce that maximum with its provider before starting external work. -The adapter callback must call `recordExternalCost()` with each observed charge. -Positive external work without a receipt is recorded as incomplete cost accounting, not as the configured maximum. -Use `0` only for a free local path. -Paid memory improvement defaults to a zero-dollar total limit; set `maxTotalCostUsd` and `maximumEvaluationCostUsd` before enabling paid work. - -Use `runAgentMemoryLearningExperiment` to measure whether retained memory helps across ordered steps: - -```ts -import { runAgentMemoryLearningExperiment } from '@tangle-network/agent-knowledge/memory' - -const result = await runAgentMemoryLearningExperiment({ - experimentId: 'support-memory', - runDir: 'support-memory', - candidates: [memoryCandidate], - sequences, - seed: 42, - reps: 5, - armOrder: 'stateful-first', - costCeiling: 10, -}) - -console.log(result.comparison.gain) -``` - -The function runs matched stateful and stateless arms with the same immutable candidate, tasks, executor, policy, seed, and repetitions. -The stateless arm clears declared scopes between steps; adapters must support scoped `clear`. -Gain excludes first-step probes and averages candidates and repetitions within each independent sequence. -Use `transferKey` on later probes for transfer and repeat one `retentionKey` across steps for forgetting. -Unmarked probes are not assigned those meanings. - -Both arms share one cost limit and must have identical comparison references. -Run independent experiments with opposite `armOrder` values when provider behavior may drift. -Each saved probe includes the exact scoring input and content hash, so protect the run directory like the memory data itself. -Pass `signal` to cancel; rerun the same options and directory to resume completed work and cost records. - -## Run benchmarks - -`@tangle-network/agent-knowledge/benchmarks` provides common case and report types for: - -- retrieval from qrels datasets -- RAG answer quality and unsupported claims -- knowledge-base improvement -- multi-turn memory behavior and provider comparison - -The bundled industry cases are small smoke checks for adapter wiring. -They are not copies of full BEIR, MTEB, MS MARCO, LongMemEval, or other external datasets, and they do not produce a public leaderboard by themselves. -Import the real dataset rows or qrels and run them through `runKnowledgeBenchmarkSuite` for benchmark results. +The memory APIs support scoped reads and writes and isolated branches. +Compare providers on product tasks with `@tangle-network/agent-eval` before changing production behavior. ## Package boundaries | Import | Contents | |---|---| | `@tangle-network/agent-knowledge` | KB files, indexes, search, validation, research callbacks, RAG evaluation, and candidate improvement | -| `@tangle-network/agent-knowledge/memory` | Memory contracts, provider adapters, branches, and experiments | +| `@tangle-network/agent-knowledge/memory` | Memory contracts, provider adapters, branches, holdout, and play memory tools | | `@tangle-network/agent-knowledge/sources` | HTTP and authority-specific source adapters | -| `@tangle-network/agent-knowledge/benchmarks` | Benchmark cases, qrels import, execution, and reports | +| `@tangle-network/agent-knowledge/benchmarks` | In-memory and no-op memory adapters for benchmark harnesses | | `@tangle-network/agent-knowledge/viz` | Dependency-free graph analysis helpers | Use the `agent-knowledge` binary for CLI commands rather than importing its CLI module. diff --git a/api-surface.json b/api-surface.json index ead1350..8e7e399 100644 --- a/api-surface.json +++ b/api-surface.json @@ -4,60 +4,23 @@ ".": { "AddSourceOptions": "type bee9a2194b6c", "AddSourceTextInput": "type 9619930752c2", - "AgentMemoryAcquireRunLease": "type 4ff237e19d6d", - "AgentMemoryActivation": "type 61c2ab159978", - "AgentMemoryActivationDriver": "type 7cf464394ada", "AgentMemoryAdapter": "type 33f1cff7fc1a", - "AgentMemoryAttemptEvent": "type 7661dd996f22", "AgentMemoryBranch": "type 44f54100fad5", "AgentMemoryBranchIsolation": "type 33ffe7155eba", "AgentMemoryBranchLifetime": "type 0b553b25b156", "AgentMemoryBranchSnapshot": "type 13e0d7dc0c86", "AgentMemoryContext": "type d2ad9aae008c", - "AgentMemoryControllerMode": "type 3490191a2dd0", - "AgentMemoryDimensionComparison": "type 9bb6eed6d1d8", - "AgentMemoryEvidenceRef": "type 50e0ea9f0043", - "AgentMemoryExecutionContext": "type 5c2b654fd8e6", - "AgentMemoryExecutionCostMeter": "type ff5e1a82a393", - "AgentMemoryExecutionCostReceipt": "type 4ef14d239e82", - "AgentMemoryExecutionPaidCallInput": "type f3f5f7829439", - "AgentMemoryExecutionPaidCallResult": "type 9f31cd4fb11c", - "AgentMemoryExecutionStep": "type 34fd96933047", - "AgentMemoryExperimentCandidate": "type 7edf762c841c", - "AgentMemoryExperimentCandidateRef": "type 1cc6471638ef", - "AgentMemoryExperimentComparisonRef": "type 50e0ea9f0043", - "AgentMemoryExperimentRankingRow": "type 11b9f9d8b026", - "AgentMemoryExperimentRunLease": "type 9431eb380a3e", - "AgentMemoryFinalEvaluation": "type fa3494059c13", - "AgentMemoryFinalPair": "type cca58dd2fff1", - "AgentMemoryForgettingComparison": "type 7c0081efbbd1", "AgentMemoryHit": "type e7b45aadc418", "AgentMemoryHitSchema": "value 373728f5643d", - "AgentMemoryImprovementRunLease": "type 9431eb380a3e", "AgentMemoryJournalEntry": "type ceadedad84df", "AgentMemoryKind": "type 6d96b298a5a6", "AgentMemoryKindSchema": "value 373728f5643d", - "AgentMemoryLearningArmOrder": "type e8132d5b34fe", - "AgentMemoryLearningCandidateSummary": "type bbbac950c0d3", - "AgentMemoryLearningCellComparison": "type 13cc314ad280", - "AgentMemoryLearningComparison": "type 527489bcaeb1", "AgentMemoryLifecycleTimeoutError": "value 3d8784b1e48a", "AgentMemoryLifecycleUnsafeError": "value 76e25c9235a8", - "AgentMemoryMode": "type 9fa3c0482c12", - "AgentMemoryPromotionDecision": "type d34a89427912", - "AgentMemoryRunLease": "type 3833198f7ee0", "AgentMemoryScope": "type 879ac4259f3c", "AgentMemoryScopeSchema": "value 373728f5643d", "AgentMemorySearchOptions": "type 088e75c091e0", - "AgentMemorySequence": "type f25f429c08a6", - "AgentMemorySequenceArtifact": "type 8be8e6375531", - "AgentMemorySequenceProbe": "type 512ad45e62ef", - "AgentMemorySequenceProbeResult": "type 1f8aae34dcfd", - "AgentMemorySequenceScenario": "type d9094ae4cf99", - "AgentMemorySequenceStep": "type 98f8223e2ec2", "AgentMemorySharingPolicy": "type d14707f95351", - "AgentMemoryTransferCellComparison": "type 40f6d39f4c6a", - "AgentMemoryTransferStepSummary": "type 6842cb405469", "AgentMemoryVisibility": "type 0c3bef3036f1", "AgentMemoryWriteInput": "type 7f41599c27b7", "AgentMemoryWriteInputSchema": "value 373728f5643d", @@ -67,10 +30,8 @@ "AuditKnowledgeCitationsOptions": "type 04e80cd4835a", "Bm25Hit": "type a6c513ea2447", "Bm25Options": "type 9da64a8973e6", - "BuildAgentMemorySequencesFromBenchmarkCasesOptions": "type 210382d1dfe1", "BuildEvalKnowledgeBundleOptions": "type 8951c7835e42", "BuildKnowledgeRelationGraphInput": "type f08e22cb9d75", - "BuildRetrievalBenchmarkCasesFromQrelsOptions": "type 3b89d8bbc625", "BuildRetrievalEvalDispatchOptions": "type 2a1fe2b2da73", "CHECKABLE_RUNG_THRESHOLD": "value 534bcde62c80", "CITES_INVALIDATED_FIELD": "value 3fff28ee6d1f", @@ -83,7 +44,6 @@ "ClaimLedgerStore": "type c402ec258adc", "ClaimRef": "type e200b4c5eeec", "ClaimVerdict": "type 18f044018844", - "CompareAgentMemoryLearningOptions": "type 33bff9c3b28a", "CornellLiiSelector": "type c92fb79c4d6b", "CornellLiiSourceOptions": "type 3d1a7a174053", "CreateAgentMemoryBranchOptions": "type 924c7b6dcd11", @@ -135,8 +95,6 @@ "HindsightClientLike": "type e4b6e0bd1488", "HindsightMemoryAdapterOptions": "type 995d4468eb3c", "HindsightOperationUnknownError": "value 408246fe51ae", - "INDUSTRY_MEMORY_BENCHMARKS": "value 373728f5643d", - "INDUSTRY_RAG_BENCHMARKS": "value 373728f5643d", "IRS_DIMENSION_HINTS": "value 373728f5643d", "ImproveSelectedKnowledgeCandidateOptions": "type c738ecaa49fa", "ImproveSelectedKnowledgeCandidateResult": "type 3a8454d1031d", @@ -153,27 +111,11 @@ "KNOWLEDGE_SEARCH_RETRIEVER_ID": "value 9d17865b30b2", "KNOWLEDGE_USE_RECEIPT_SCHEMA_VERSION": "value 373728f5643d", "KbStore": "type 4a9e2c071e37", - "KnowledgeAnswerBenchmarkCase": "type 57b2b8188102", - "KnowledgeAnswerBenchmarkTaskKind": "type 19a203258010", "KnowledgeBaseCandidate": "type 2af41362cac3", "KnowledgeBaseCandidateSchema": "value 373728f5643d", "KnowledgeBaseQualityOptions": "type 97ca97df37a8", "KnowledgeBaseQualityReport": "type d35040b8624a", "KnowledgeBaseReadinessEvaluation": "type 5de5279c9dce", - "KnowledgeBenchmarkArtifact": "type 49da84673f55", - "KnowledgeBenchmarkCase": "type a4778ccebde4", - "KnowledgeBenchmarkCaseBase": "type c993c96958e9", - "KnowledgeBenchmarkDistribution": "type 08cc35362f22", - "KnowledgeBenchmarkEvaluation": "type 02071a7913ec", - "KnowledgeBenchmarkFamily": "type 3e2708add6a6", - "KnowledgeBenchmarkReport": "type f57a38b9325a", - "KnowledgeBenchmarkResponder": "type 8b7ad462c749", - "KnowledgeBenchmarkScenario": "type 34ec1dfca0d8", - "KnowledgeBenchmarkSliceSummary": "type 95a155c7cd62", - "KnowledgeBenchmarkSource": "type a1f4436ae164", - "KnowledgeBenchmarkSpec": "type 4037593be53d", - "KnowledgeBenchmarkSplit": "type 47605b0ef333", - "KnowledgeBenchmarkTaskKind": "type acb24a777cc7", "KnowledgeBrief": "type 01ca10f80a06", "KnowledgeBriefOptions": "type ceb76227d749", "KnowledgeChange": "type 7b71ca9dd658", @@ -189,7 +131,6 @@ "KnowledgeCitationResolutionError": "value 30aa37f0ad9f", "KnowledgeCitationResolutionStatus": "type 37f5c6476c90", "KnowledgeClaim": "type 0d2e985846b4", - "KnowledgeClaimMatcher": "type 42d6c45b4ed0", "KnowledgeConsumerKind": "type e68d53c21645", "KnowledgeConsumerRef": "type d2193df21037", "KnowledgeControlLoopAction": "type 278d5c67af20", @@ -250,10 +191,6 @@ "KnowledgeLexicalIndexOptions": "type f617cd13b723", "KnowledgeLexicalPosting": "type 096bcba20c1c", "KnowledgeLintFinding": "type 8a05fe6abfad", - "KnowledgeMemoryBenchmarkCase": "type e9132a0be619", - "KnowledgeMemoryBenchmarkTaskKind": "type 33dd3b696a1d", - "KnowledgeMemoryEvent": "type b5b297e3f9b1", - "KnowledgeMemoryFactMatcher": "type 095c9075c340", "KnowledgeMutationHold": "type 353c820b01ff", "KnowledgeMutationLock": "type 974f26ecfc42", "KnowledgeMutationOptions": "type 1ab92dd5d3de", @@ -295,9 +232,6 @@ "KnowledgeResearchLoopDecision": "type 648813512b86", "KnowledgeResearchLoopResult": "type 817d2eacc456", "KnowledgeResearchLoopStep": "type af5b4466e758", - "KnowledgeRetrievalBenchmarkCase": "type d2135c53958f", - "KnowledgeRetrievalBenchmarkQrel": "type 159153ab63cd", - "KnowledgeRetrievalBenchmarkQuery": "type d9ee05c62de0", "KnowledgeRetrievalDisposition": "type c3bb481999cf", "KnowledgeRetrievalDispositionRelation": "type 9c68bc5da478", "KnowledgeRetrievalReceipt": "type 63de26c92cfe", @@ -332,9 +266,6 @@ "Mem0MemoryAdapterOptions": "type c3d953afa968", "Mem0OssClient": "type 641eca0aecd6", "Mem0OssMemoryAdapterOptions": "type b7c55e102a21", - "MemoryAdapterBenchmarkCandidate": "type 761edb34d291", - "MemoryAdapterBenchmarkRankingRow": "type 9a8fa5ffee7a", - "MemoryConfigScenario": "type c81d5214e789", "MemoryKbStore": "value 95d4715b610e", "NearDuplicateDetectionOptions": "type cc1c77ce4d80", "NearDuplicatePair": "type 31bc35ca4a75", @@ -344,7 +275,6 @@ "OptimizeKnowledgeBasePolicyResult": "type 1e3b03c7a0a3", "OriginatedKnowledgeSearchResult": "type 99110190f58f", "OriginatedPage": "type 62ba6f531a4a", - "OwnedAgentMemoryRunLease": "type 4680998178f2", "POLITE_USER_AGENT": "value 3eb07665c028", "PageOrigin": "type df33a580b8a9", "ParsedFrontmatter": "type 7aebd4ab7c31", @@ -433,20 +363,10 @@ "RetrievalScenarioPartitions": "type a8a61ddef80a", "RetrievedKnowledgeHit": "type 9dfb968780ec", "RetrievedSourceSpan": "type d421429a842f", - "RunAgentMemoryExperimentOptions": "type ea599b678ed8", - "RunAgentMemoryExperimentResult": "type 4c59c13e22c2", - "RunAgentMemoryImprovementOptions": "type d4fc3b183ac9", - "RunAgentMemoryImprovementResult": "type d1da30a3cde1", - "RunAgentMemoryLearningExperimentOptions": "type 781b9c320d5b", - "RunAgentMemoryLearningExperimentResult": "type 8472cdc7f052", "RunDiscoveryLoopOptions": "type bfcdb3617601", - "RunKnowledgeBenchmarkSuiteOptions": "type 8f7c9a385609", - "RunKnowledgeBenchmarkSuiteResult": "type 46bed11cb991", "RunKnowledgeResearchLoopOptions": "type ac668b866256", "RunLineageAuthority": "type aa33159b2c96", "RunLineageRecord": "type 9ba2102ed82c", - "RunMemoryAdapterBenchmarkOptions": "type 9e6710497bb1", - "RunMemoryAdapterBenchmarkResult": "type 17500f018600", "RunRagKnowledgeImprovementLoopOptions": "type 415d465f5ad4", "RunRagKnowledgeImprovementLoopResult": "type 73696e62b829", "RunRagOptimizationOptions": "type 6083351ac8e2", @@ -481,10 +401,8 @@ "VerifyGradeableEvidenceOptions": "type 01ef698f9b40", "WIKILINK_REGEX": "value 373728f5643d", "__resetHttpThrottle": "value e47f3b6734a6", - "acquireAgentMemoryRunLease": "value 7d23f0e042cc", "addSourcePath": "value 9dabb587c73c", "addSourceText": "value 3a5c6107b137", - "agentMemorySequenceJudge": "value 6a8ab90bcbce", "applyKnowledgeWriteBlocks": "value 08c999b38722", "applyKnowledgeWriteBlocksFile": "value 99b85a997642", "applyRetrievalHoldout": "value f293047a46cd", @@ -504,19 +422,12 @@ "assertTrackedClaimIntegrity": "value 477a08d5355f", "auditCurrentRunCitations": "value 9c88cca6cd25", "auditKnowledgeCitations": "value 0bd1b4cba66e", - "buildAgentMemorySequenceScenarios": "value 5982b6b905c3", - "buildAgentMemorySequencesFromBenchmarkCases": "value 59d75f8c3a58", "buildEvalKnowledgeBundle": "value 15e65db67a64", - "buildFirstPartyMemoryLifecycleBenchmarkCases": "value 24a4438ada5b", - "buildIndustryMemoryBenchmarkSmokeCases": "value ab9a83e291d9", - "buildIndustryRagBenchmarkSmokeCases": "value a9327d14d1e5", - "buildKnowledgeBenchmarkScenarios": "value 507de6064e73", "buildKnowledgeBrief": "value e28953443489", "buildKnowledgeGraph": "value f4bc0817e156", "buildKnowledgeIndex": "value 7a198b5e15bd", "buildKnowledgeLexicalIndex": "value 61e7f22cdce2", "buildKnowledgeRelationGraph": "value 2c4502c049ed", - "buildRetrievalBenchmarkCasesFromQrels": "value 0cee488c25c6", "buildRetrievalEvalDispatch": "value 4f12d1f4c065", "calibrateRagAnswerJudge": "value 74f91dafd1b1", "canonicalPathsEqual": "value 1a1d22b3b1c3", @@ -526,7 +437,6 @@ "claimEvidenceId": "value 10ea894124db", "claimId": "value 4c9eca896a37", "claimSourceHost": "value a8a18cde9fcc", - "compareAgentMemoryLearning": "value 80e066a5a265", "createAgentMemoryBranch": "value bae206df8be9", "createCornellLiiSource": "value 5708478c5b8a", "createD1FreshnessStoreStub": "value e89ca59074b9", @@ -590,7 +500,6 @@ "inspectKnowledgeIndex": "value 79b67f7de1cd", "inspectPendingKnowledgeMutation": "value 38b2e30827bd", "isKernelAnchoredPath": "value 2819fa14db65", - "isKnowledgeMemoryBenchmarkCase": "value 8634e2c95fec", "isKnowledgeMutationHeld": "value e8f0de09e802", "isKnowledgePagePath": "value 2819fa14db65", "isMissingFile": "value 73ca49f2b72a", @@ -599,7 +508,6 @@ "isScaffoldPath": "value 2819fa14db65", "jsonCandidateCodec": "value 6520c69bb348", "jsonObjectCandidateCodec": "value 8eb601cd6de7", - "knowledgeBenchmarkJudge": "value e7c74f6e007a", "knowledgeCitationAuditFindings": "value 639f15656570", "knowledgeImprovementCandidateRef": "value 2c85f9c2a1a0", "knowledgeImprovementRunDir": "value d3fdafeff129", @@ -639,8 +547,6 @@ "optimizeKnowledgeBasePolicy": "value 3cec15621ce3", "originatedPages": "value 38bee2f6341e", "parseFrontmatter": "value 62f43b77d35a", - "parseKnowledgeBenchmarkJsonl": "value d3574e164bc5", - "parseKnowledgeBenchmarkQrels": "value 168db534a024", "parseKnowledgeCitationReference": "value 098a42106c17", "parseKnowledgeWriteBlocks": "value d6bd24d2858d", "partitionRetrievalScenarios": "value 363802debe8a", @@ -658,7 +564,6 @@ "relativeWithinRoot": "value bd8a9ab05814", "removeDurable": "value 6218521600a7", "renameDurable": "value ea60b9d3f720", - "renderKnowledgeBenchmarkReportMarkdown": "value 54198213283b", "renderMemoryContext": "value 7e8daf0436d5", "researchSourceVersionKey": "value 663796baec39", "resetRetrievalHoldoutRegistry": "value e47f3b6734a6", @@ -667,22 +572,15 @@ "resolveMemoryCleanupTimeoutMs": "value d86a780529d7", "resolveRunScopedCitation": "value 8f5d4c6546fb", "resolveRunScopedCitations": "value d3bfe5a3758c", - "respondToIndustryMemoryBenchmarkSmokeCase": "value 0ec725eb5621", - "respondToIndustryRagBenchmarkSmokeCase": "value 2f118a2e049c", "restoreKnowledgeCandidateBaseline": "value fe9fe35244bf", "retrievalConfigFromSurface": "value 1348f52cc5d8", "retrievalConfigSurface": "value e7c5f7d43629", "retrievalHoldoutConfigHash": "value d2a79c9fe68f", "retrievalRecallJudge": "value 725c9a0b731c", - "runAgentMemoryExperiment": "value 8d007b71d3cc", - "runAgentMemoryImprovement": "value d6708f756231", - "runAgentMemoryLearningExperiment": "value f11e54703bb5", "runBoundedMemoryLifecycle": "value 75f3e3b274a0", "runDiscoveryLoop": "value addb8a464a58", "runInKnowledgeMutationScope": "value 83a5698000d7", - "runKnowledgeBenchmarkSuite": "value ce72ffe18eef", "runKnowledgeResearchLoop": "value 4d00a0a2e19d", - "runMemoryAdapterBenchmark": "value e783141bcd1d", "runRagKnowledgeImprovementLoop": "value c943e69b947e", "runRagOptimization": "value 9493b6dd6742", "runRetrievalImprovementLoop": "value d1f8efdb025d", @@ -690,8 +588,6 @@ "scenarioContentFingerprint": "value c6788a57974b", "scoreBm25": "value 42139506975a", "scoreKnowledgeBaseIndex": "value c16e27e90d97", - "scoreKnowledgeBenchmarkArtifact": "value 79e3e2470439", - "scoreMemoryBenchmarkArtifact": "value c398635ef376", "scoreRagAnswerArtifact": "value 3594d420d068", "scoreRetrievalArtifact": "value ebfa02d93879", "searchKnowledge": "value 0fabd6d7d1e3", @@ -703,7 +599,6 @@ "sourceRegistryPath": "value 5fe2ec842844", "stableId": "value a7bc5069602e", "stripFrontmatter": "value 335a29d6a1cd", - "summarizeKnowledgeBenchmarkCampaign": "value 681c7995df88", "syncDirectory": "value 6218521600a7", "textSourceAdapter": "value 373728f5643d", "textSourceId": "value 3ea77a7de22d", @@ -738,121 +633,32 @@ "writeSourceRegistry": "value b526fdf1c35d" }, "./benchmarks": { - "BuildRetrievalBenchmarkCasesFromQrelsOptions": "type 3b89d8bbc625", - "INDUSTRY_MEMORY_BENCHMARKS": "value 373728f5643d", - "INDUSTRY_RAG_BENCHMARKS": "value 373728f5643d", - "KnowledgeAnswerBenchmarkCase": "type 57b2b8188102", - "KnowledgeAnswerBenchmarkTaskKind": "type 19a203258010", - "KnowledgeBenchmarkArtifact": "type 49da84673f55", - "KnowledgeBenchmarkCase": "type a4778ccebde4", - "KnowledgeBenchmarkCaseBase": "type c993c96958e9", - "KnowledgeBenchmarkDistribution": "type 08cc35362f22", - "KnowledgeBenchmarkEvaluation": "type 02071a7913ec", - "KnowledgeBenchmarkFamily": "type 3e2708add6a6", - "KnowledgeBenchmarkReport": "type f57a38b9325a", - "KnowledgeBenchmarkResponder": "type 8b7ad462c749", - "KnowledgeBenchmarkScenario": "type 34ec1dfca0d8", - "KnowledgeBenchmarkSliceSummary": "type 95a155c7cd62", - "KnowledgeBenchmarkSource": "type a1f4436ae164", - "KnowledgeBenchmarkSpec": "type 4037593be53d", - "KnowledgeBenchmarkSplit": "type 47605b0ef333", - "KnowledgeBenchmarkTaskKind": "type acb24a777cc7", - "KnowledgeClaimMatcher": "type 42d6c45b4ed0", - "KnowledgeMemoryBenchmarkCase": "type e9132a0be619", - "KnowledgeMemoryBenchmarkTaskKind": "type 33dd3b696a1d", - "KnowledgeMemoryEvent": "type b5b297e3f9b1", - "KnowledgeMemoryFactMatcher": "type 095c9075c340", - "KnowledgeRetrievalBenchmarkCase": "type d2135c53958f", - "KnowledgeRetrievalBenchmarkQrel": "type 159153ab63cd", - "KnowledgeRetrievalBenchmarkQuery": "type d9ee05c62de0", - "MemoryAdapterBenchmarkCandidate": "type 761edb34d291", - "MemoryAdapterBenchmarkRankingRow": "type 9a8fa5ffee7a", - "RunKnowledgeBenchmarkSuiteOptions": "type 8f7c9a385609", - "RunKnowledgeBenchmarkSuiteResult": "type 46bed11cb991", - "RunMemoryAdapterBenchmarkOptions": "type 9e6710497bb1", - "RunMemoryAdapterBenchmarkResult": "type 17500f018600", - "buildFirstPartyMemoryLifecycleBenchmarkCases": "value 24a4438ada5b", - "buildIndustryMemoryBenchmarkSmokeCases": "value ab9a83e291d9", - "buildIndustryRagBenchmarkSmokeCases": "value a9327d14d1e5", - "buildKnowledgeBenchmarkScenarios": "value 507de6064e73", - "buildRetrievalBenchmarkCasesFromQrels": "value 0cee488c25c6", "createInMemoryBenchmarkAdapter": "value f4c546ff2fff", - "createNoopMemoryBenchmarkAdapter": "value d0ec3fcca863", - "isKnowledgeMemoryBenchmarkCase": "value 8634e2c95fec", - "knowledgeBenchmarkJudge": "value e7c74f6e007a", - "parseKnowledgeBenchmarkJsonl": "value d3574e164bc5", - "parseKnowledgeBenchmarkQrels": "value 168db534a024", - "renderKnowledgeBenchmarkReportMarkdown": "value 54198213283b", - "respondToIndustryMemoryBenchmarkSmokeCase": "value 0ec725eb5621", - "respondToIndustryRagBenchmarkSmokeCase": "value 2f118a2e049c", - "runKnowledgeBenchmarkSuite": "value ce72ffe18eef", - "runMemoryAdapterBenchmark": "value e783141bcd1d", - "scoreKnowledgeBenchmarkArtifact": "value 79e3e2470439", - "scoreMemoryBenchmarkArtifact": "value c398635ef376", - "summarizeKnowledgeBenchmarkCampaign": "value 681c7995df88" + "createNoopMemoryBenchmarkAdapter": "value d0ec3fcca863" }, "./cli": {}, "./memory": { - "AgentMemoryAcquireRunLease": "type 4ff237e19d6d", - "AgentMemoryActivation": "type 61c2ab159978", - "AgentMemoryActivationDriver": "type 7cf464394ada", "AgentMemoryAdapter": "type 33f1cff7fc1a", - "AgentMemoryAttemptEvent": "type 7661dd996f22", "AgentMemoryBranch": "type 44f54100fad5", "AgentMemoryBranchIsolation": "type 33ffe7155eba", "AgentMemoryBranchLifetime": "type 0b553b25b156", "AgentMemoryBranchSnapshot": "type 13e0d7dc0c86", "AgentMemoryContext": "type d2ad9aae008c", - "AgentMemoryControllerMode": "type 3490191a2dd0", - "AgentMemoryDimensionComparison": "type 9bb6eed6d1d8", - "AgentMemoryEvidenceRef": "type 50e0ea9f0043", - "AgentMemoryExecutionContext": "type 5c2b654fd8e6", - "AgentMemoryExecutionCostMeter": "type ff5e1a82a393", - "AgentMemoryExecutionCostReceipt": "type 4ef14d239e82", - "AgentMemoryExecutionPaidCallInput": "type f3f5f7829439", - "AgentMemoryExecutionPaidCallResult": "type 9f31cd4fb11c", - "AgentMemoryExecutionStep": "type 34fd96933047", - "AgentMemoryExperimentCandidate": "type 7edf762c841c", - "AgentMemoryExperimentCandidateRef": "type 1cc6471638ef", - "AgentMemoryExperimentComparisonRef": "type 50e0ea9f0043", - "AgentMemoryExperimentRankingRow": "type 11b9f9d8b026", - "AgentMemoryExperimentRunLease": "type 9431eb380a3e", - "AgentMemoryFinalEvaluation": "type fa3494059c13", - "AgentMemoryFinalPair": "type cca58dd2fff1", - "AgentMemoryForgettingComparison": "type 7c0081efbbd1", "AgentMemoryHit": "type e7b45aadc418", "AgentMemoryHitSchema": "value 373728f5643d", - "AgentMemoryImprovementRunLease": "type 9431eb380a3e", "AgentMemoryJournalEntry": "type ceadedad84df", "AgentMemoryKind": "type 6d96b298a5a6", "AgentMemoryKindSchema": "value 373728f5643d", - "AgentMemoryLearningArmOrder": "type e8132d5b34fe", - "AgentMemoryLearningCandidateSummary": "type bbbac950c0d3", - "AgentMemoryLearningCellComparison": "type 13cc314ad280", - "AgentMemoryLearningComparison": "type 527489bcaeb1", "AgentMemoryLifecycleTimeoutError": "value 3d8784b1e48a", "AgentMemoryLifecycleUnsafeError": "value 76e25c9235a8", - "AgentMemoryMode": "type 9fa3c0482c12", - "AgentMemoryPromotionDecision": "type d34a89427912", - "AgentMemoryRunLease": "type 3833198f7ee0", "AgentMemoryScope": "type 879ac4259f3c", "AgentMemoryScopeSchema": "value 373728f5643d", "AgentMemorySearchOptions": "type 088e75c091e0", - "AgentMemorySequence": "type f25f429c08a6", - "AgentMemorySequenceArtifact": "type 8be8e6375531", - "AgentMemorySequenceProbe": "type 512ad45e62ef", - "AgentMemorySequenceProbeResult": "type 1f8aae34dcfd", - "AgentMemorySequenceScenario": "type d9094ae4cf99", - "AgentMemorySequenceStep": "type 98f8223e2ec2", "AgentMemorySharingPolicy": "type d14707f95351", - "AgentMemoryTransferCellComparison": "type 40f6d39f4c6a", - "AgentMemoryTransferStepSummary": "type 6842cb405469", "AgentMemoryVisibility": "type 0c3bef3036f1", "AgentMemoryWriteInput": "type 7f41599c27b7", "AgentMemoryWriteInputSchema": "value 373728f5643d", "AgentMemoryWriteResult": "type 4c444521a5ca", - "BuildAgentMemorySequencesFromBenchmarkCasesOptions": "type 210382d1dfe1", - "CompareAgentMemoryLearningOptions": "type 33bff9c3b28a", "CreateAgentMemoryBranchOptions": "type 924c7b6dcd11", "CreatePlayMemoryToolsOptions": "type d659926093f3", "DEFAULT_MEMORY_CLEANUP_TIMEOUT_MS": "value 10d1ef3e7731", @@ -869,9 +675,7 @@ "Mem0MemoryAdapterOptions": "type c3d953afa968", "Mem0OssClient": "type 641eca0aecd6", "Mem0OssMemoryAdapterOptions": "type b7c55e102a21", - "MemoryConfigScenario": "type c81d5214e789", "Neo4jAgentMemoryAdapterOptions": "type c0746e451047", - "OwnedAgentMemoryRunLease": "type 4680998178f2", "PlayMemoryRetrievalReceipt": "type 0118f85065b1", "RetrievalHoldoutBypassReason": "type 57ea26248eb4", "RetrievalHoldoutCallContext": "type d7fa5e2b47a6", @@ -883,19 +687,8 @@ "RetrievalHoldoutResult": "type 5713de3c84a1", "RetrievalHoldoutSessionState": "type 76de736b5743", "RetrievalHoldoutSessionSummary": "type 545aa1319ad0", - "RunAgentMemoryExperimentOptions": "type ea599b678ed8", - "RunAgentMemoryExperimentResult": "type 4c59c13e22c2", - "RunAgentMemoryImprovementOptions": "type d4fc3b183ac9", - "RunAgentMemoryImprovementResult": "type d1da30a3cde1", - "RunAgentMemoryLearningExperimentOptions": "type 781b9c320d5b", - "RunAgentMemoryLearningExperimentResult": "type 8472cdc7f052", - "acquireAgentMemoryRunLease": "value 7d23f0e042cc", - "agentMemorySequenceJudge": "value 6a8ab90bcbce", "applyRetrievalHoldout": "value f293047a46cd", "applySessionStickyRetrievalHoldout": "value b05959ec08ff", - "buildAgentMemorySequenceScenarios": "value 5982b6b905c3", - "buildAgentMemorySequencesFromBenchmarkCases": "value 59d75f8c3a58", - "compareAgentMemoryLearning": "value 80e066a5a265", "createAgentMemoryBranch": "value bae206df8be9", "createGraphitiMemoryAdapter": "value 512201c0dd04", "createHindsightMemoryAdapter": "value 97fcfb086aee", @@ -917,9 +710,6 @@ "resetRetrievalHoldoutRegistry": "value e47f3b6734a6", "resolveMemoryCleanupTimeoutMs": "value d86a780529d7", "retrievalHoldoutConfigHash": "value d2a79c9fe68f", - "runAgentMemoryExperiment": "value 8d007b71d3cc", - "runAgentMemoryImprovement": "value d6708f756231", - "runAgentMemoryLearningExperiment": "value f11e54703bb5", "runBoundedMemoryLifecycle": "value 75f3e3b274a0", "sleepForMemoryRecovery": "value 4a6818f984e4", "toOffPolicyTrajectory": "value 934dec24305c" diff --git a/scripts/verify-package.mjs b/scripts/verify-package.mjs index 2bf862f..e8d0e58 100644 --- a/scripts/verify-package.mjs +++ b/scripts/verify-package.mjs @@ -42,7 +42,7 @@ const requiredRootExports = [ // without executing, so it is structurally blind to this class — this check is // what stands in for it. const edgeUnsafeStaticImports = ['proper-lockfile', 'graceful-fs'] -const requiredMemoryExports = ['runAgentMemoryImprovement'] +const requiredMemoryExports = ['createAgentMemoryBranch'] const requiredAgentEvalExports = ['gepaOptimizationMethod', 'skillOptOptimizationMethod'] const agentEvalPackage = '@tangle-network/agent-eval' const agentCorePackage = '@tangle-network/agent-core' diff --git a/skills/build-with-agent-knowledge/SKILL.md b/skills/build-with-agent-knowledge/SKILL.md index 7a6b20c..7485ce6 100644 --- a/skills/build-with-agent-knowledge/SKILL.md +++ b/skills/build-with-agent-knowledge/SKILL.md @@ -67,7 +67,6 @@ Reject stale promotion rather than replacing newer knowledge. Report service and measurement failures separately from product failures. Keep candidate-generation cases separate from the final decision set. -Bundled benchmark samples prove adapter wiring only; use complete external datasets for benchmark claims. ## Completion diff --git a/src/benchmarks/catalog.ts b/src/benchmarks/catalog.ts deleted file mode 100644 index 31f70d0..0000000 --- a/src/benchmarks/catalog.ts +++ /dev/null @@ -1,661 +0,0 @@ -import type { RetrievalGoldTarget, RetrievedKnowledgeHit } from '../retrieval-eval' - -import type { - KnowledgeAnswerBenchmarkTaskKind, - KnowledgeBenchmarkArtifact, - KnowledgeBenchmarkCase, - KnowledgeBenchmarkFamily, - KnowledgeBenchmarkSource, - KnowledgeBenchmarkSpec, - KnowledgeBenchmarkSplit, - KnowledgeMemoryBenchmarkCase, - KnowledgeMemoryBenchmarkTaskKind, -} from './types' - -import { unique } from './utils' - -import { isKnowledgeMemoryBenchmarkCase } from './validation' - -export const INDUSTRY_RAG_BENCHMARKS: readonly (KnowledgeBenchmarkSpec & { - taskKind: 'retrieval' | KnowledgeAnswerBenchmarkTaskKind -})[] = [ - { - id: 'beir', - family: 'beir', - taskKind: 'retrieval', - primaryMetrics: ['nDCG@10', 'Recall@100', 'MRR@10'], - adapter: 'buildRetrievalBenchmarkCasesFromQrels', - notes: 'Classic zero-shot retrieval suites using query/corpus/qrels files.', - }, - { - id: 'mteb-retrieval', - family: 'mteb-retrieval', - taskKind: 'retrieval', - primaryMetrics: ['nDCG@10', 'Recall@100'], - adapter: 'buildRetrievalBenchmarkCasesFromQrels', - notes: 'MTEB retrieval task shape; same qrels bridge, different dataset provenance.', - }, - { - id: 'msmarco', - family: 'msmarco', - taskKind: 'retrieval', - primaryMetrics: ['MRR@10', 'Recall@100'], - adapter: 'buildRetrievalBenchmarkCasesFromQrels', - notes: 'Passage retrieval and reranking smoke for web-style questions.', - }, - { - id: 'trec-dl', - family: 'trec-dl', - taskKind: 'retrieval', - primaryMetrics: ['nDCG@10', 'MAP', 'Recall@100'], - adapter: 'buildRetrievalBenchmarkCasesFromQrels', - notes: 'Deep Learning Track judgments over MS MARCO-derived corpora.', - }, - { - id: 'miracl', - family: 'miracl', - taskKind: 'retrieval', - primaryMetrics: ['nDCG@10', 'Recall@100'], - adapter: 'buildRetrievalBenchmarkCasesFromQrels', - notes: 'Multilingual retrieval; use language tags on cases.', - }, - { - id: 'lotte', - family: 'lotte', - taskKind: 'retrieval', - primaryMetrics: ['Success@5', 'Recall@100'], - adapter: 'buildRetrievalBenchmarkCasesFromQrels', - notes: 'Long-tail search tasks; map collection/domain into tags.', - }, - { - id: 'bright', - family: 'bright', - taskKind: 'retrieval', - primaryMetrics: ['nDCG@10', 'Recall@100'], - adapter: 'buildRetrievalBenchmarkCasesFromQrels', - notes: 'Reasoning-heavy retrieval; preserve domain tags for slice reporting.', - }, - { - id: 'crag', - family: 'crag', - taskKind: 'rag-answer', - primaryMetrics: ['claim_recall', 'citation_recall', 'hallucination_safe'], - adapter: 'KnowledgeAnswerBenchmarkCase', - notes: 'Answer quality and freshness cases; use required/forbidden claims plus citations.', - }, - { - id: 'hotpotqa', - family: 'hotpotqa', - taskKind: 'rag-answer', - primaryMetrics: ['claim_recall', 'citation_recall'], - adapter: 'KnowledgeAnswerBenchmarkCase', - notes: 'Multihop QA; encode each supporting fact as a required claim.', - }, - { - id: 'kilt', - family: 'kilt', - taskKind: 'rag-answer', - primaryMetrics: ['claim_recall', 'citation_recall'], - adapter: 'KnowledgeAnswerBenchmarkCase', - notes: 'Knowledge-intensive generation with provenance; encode expected pages/sources.', - }, - { - id: 'ragtruth', - family: 'ragtruth', - taskKind: 'hallucination', - primaryMetrics: ['hallucination_safe', 'forbidden_claim_rate'], - adapter: 'KnowledgeAnswerBenchmarkCase', - notes: 'Hallucination detection; encode hallucinated spans as forbidden claims.', - }, - { - id: 'faithbench', - family: 'faithbench', - taskKind: 'hallucination', - primaryMetrics: ['hallucination_safe', 'forbidden_claim_rate'], - adapter: 'KnowledgeAnswerBenchmarkCase', - notes: 'Faithfulness benchmark; score unsupported claims as forbidden claims.', - }, - { - id: 'first-party/kb-improvement', - family: 'first-party', - taskKind: 'kb-improvement', - primaryMetrics: ['claim_recall', 'hallucination_safe', 'score'], - adapter: 'KnowledgeAnswerBenchmarkCase', - notes: 'Project-owned candidate-KB validation; grade the produced KB text or answer bundle.', - }, -] - -export const INDUSTRY_MEMORY_BENCHMARKS: readonly (KnowledgeBenchmarkSpec & { - taskKind: KnowledgeMemoryBenchmarkTaskKind -})[] = [ - { - id: 'locomo/qa', - family: 'locomo', - taskKind: 'memory-recall', - primaryMetrics: ['memory_fact_recall', 'memory_event_recall'], - adapter: 'KnowledgeMemoryBenchmarkCase', - notes: 'Long-term conversational QA over multi-session histories.', - }, - { - id: 'locomo/event-summary', - family: 'locomo', - taskKind: 'memory-summarization', - primaryMetrics: ['memory_fact_recall', 'memory_event_recall'], - adapter: 'KnowledgeMemoryBenchmarkCase', - notes: 'Event summarization over long conversational histories.', - }, - { - id: 'longmemeval', - family: 'longmemeval', - taskKind: 'memory-temporal', - primaryMetrics: ['memory_fact_recall', 'memory_stale_safe', 'memory_event_recall'], - adapter: 'KnowledgeMemoryBenchmarkCase', - notes: 'Long-term assistant memory with temporal and update-sensitive probes.', - }, - { - id: 'longmemeval-v2', - family: 'longmemeval-v2', - taskKind: 'memory-reasoning', - primaryMetrics: ['memory_fact_recall', 'memory_event_recall', 'score'], - adapter: 'KnowledgeMemoryBenchmarkCase', - notes: 'Experience reuse from long agent histories; track accuracy and latency.', - }, - { - id: 'memora', - family: 'memora', - taskKind: 'memory-update', - primaryMetrics: ['memory_fact_recall', 'memory_stale_safe', 'memory_stale_rate'], - adapter: 'KnowledgeMemoryBenchmarkCase', - notes: 'Forgetting-aware memory accuracy: reward current facts and penalize obsolete ones.', - }, - { - id: 'memoryagentbench', - family: 'memoryagentbench', - taskKind: 'memory-ingest', - primaryMetrics: ['memory_fact_recall', 'memory_event_recall'], - adapter: 'KnowledgeMemoryBenchmarkCase', - notes: 'Incremental multi-turn information intake before later recall.', - }, - { - id: 'memorybank', - family: 'memorybank', - taskKind: 'memory-recommendation', - primaryMetrics: ['memory_fact_recall', 'memory_stale_safe'], - adapter: 'KnowledgeMemoryBenchmarkCase', - notes: 'Personalized memory use for preference-aware downstream choices.', - }, - { - id: 'groupmembench', - family: 'groupmembench', - taskKind: 'memory-multiparty', - primaryMetrics: ['memory_fact_recall', 'memory_actor_recall', 'memory_stale_safe'], - adapter: 'KnowledgeMemoryBenchmarkCase', - notes: 'Multi-party memory with speaker attribution, update, and term ambiguity pressure.', - }, - { - id: 'first-party/memory-lifecycle', - family: 'first-party', - taskKind: 'memory-forgetting', - primaryMetrics: ['memory_fact_recall', 'memory_stale_safe', 'memory_event_recall'], - adapter: 'KnowledgeMemoryBenchmarkCase', - notes: 'Project-owned lifecycle pack for ingest, recall, update, forgetting, and ambiguity.', - }, -] - -export function buildIndustryRagBenchmarkSmokeCases( - specs: readonly (KnowledgeBenchmarkSpec & { - taskKind: 'retrieval' | KnowledgeAnswerBenchmarkTaskKind - })[] = INDUSTRY_RAG_BENCHMARKS, -): KnowledgeBenchmarkCase[] { - return specs.map((spec) => { - const source = { - name: spec.id, - version: 'smoke', - } - const split = spec.taskKind === 'retrieval' ? 'search' : 'holdout' - const tags = unique(['industry-smoke', spec.id, spec.family, spec.taskKind]) - if (spec.taskKind === 'retrieval') { - return { - id: `${spec.id}/smoke:q1`, - family: spec.family, - taskKind: 'retrieval', - split, - tags, - source, - query: `${spec.id} smoke retrieval query`, - expected: [{ kind: 'page', pageId: `${spec.id}:doc-1` }], - k: 5, - metadata: { - adapter: spec.adapter, - primaryMetrics: spec.primaryMetrics, - }, - } - } - - return { - id: `${spec.id}/smoke:q1`, - family: spec.family, - taskKind: spec.taskKind, - split, - tags, - source, - prompt: `${spec.id} smoke benchmark prompt`, - requiredClaims: [ - { - id: `${spec.id}:required`, - anyOf: [`${spec.id} supported answer`], - }, - ], - forbiddenClaims: [ - { - id: `${spec.id}:unsupported`, - anyOf: [`${spec.id} unsupported claim`], - }, - ], - expectedSourceIds: [`${spec.id}:source-1`], - referenceAnswer: `${spec.id} supported answer`, - metadata: { - adapter: spec.adapter, - primaryMetrics: spec.primaryMetrics, - }, - } - }) -} - -export function respondToIndustryRagBenchmarkSmokeCase(input: { - case: KnowledgeBenchmarkCase -}): KnowledgeBenchmarkArtifact { - const testCase = input.case - if (isKnowledgeMemoryBenchmarkCase(testCase)) { - return respondToIndustryMemoryBenchmarkSmokeCase({ case: testCase }) - } - if (testCase.taskKind === 'retrieval') { - const expected = Array.isArray(testCase.expected) ? testCase.expected[0] : testCase.expected - const hit = hitForExpectedTarget(expected, testCase.id) - return { - hits: [hit], - durationMs: 1, - metadata: { - smoke: true, - }, - } - } - - return { - answer: (testCase.requiredClaims ?? []) - .map((claim) => claim.anyOf[0]) - .filter((fragment): fragment is string => Boolean(fragment)) - .join(' '), - citedSourceIds: testCase.expectedSourceIds ?? [], - durationMs: 1, - metadata: { - smoke: true, - }, - } -} - -export function buildIndustryMemoryBenchmarkSmokeCases( - specs: readonly (KnowledgeBenchmarkSpec & { - taskKind: KnowledgeMemoryBenchmarkTaskKind - })[] = INDUSTRY_MEMORY_BENCHMARKS, -): KnowledgeMemoryBenchmarkCase[] { - return specs.map((spec) => { - const currentEventId = `${spec.id}:event-current` - const staleEventId = `${spec.id}:event-stale` - const actorId = spec.taskKind === 'memory-multiparty' ? 'teammate-ada' : 'user' - const currentFact = `${spec.id} current memory` - const staleFact = `${spec.id} stale memory` - return { - id: `${spec.id}/smoke:q1`, - family: spec.family, - taskKind: spec.taskKind, - split: spec.taskKind === 'memory-forgetting' ? 'holdout' : 'dev', - tags: unique(['memory-smoke', spec.id, spec.family, spec.taskKind]), - source: { - name: spec.id, - version: 'smoke', - }, - events: [ - { - id: staleEventId, - actorId, - sessionId: `${spec.id}:session-1`, - timestamp: '2026-01-01T00:00:00.000Z', - text: `${actorId} once had this obsolete fact: ${staleFact}.`, - }, - { - id: currentEventId, - actorId, - sessionId: `${spec.id}:session-2`, - timestamp: '2026-02-01T00:00:00.000Z', - text: `${actorId} updated the durable fact to: ${currentFact}.`, - }, - ], - prompt: `Use memory to answer the ${spec.id} smoke probe.`, - requiredFacts: [ - { - id: `${spec.id}:current`, - anyOf: [currentFact], - sourceEventIds: [currentEventId], - }, - ], - forbiddenFacts: [ - { - id: `${spec.id}:stale`, - anyOf: [staleFact], - sourceEventIds: [staleEventId], - obsolete: true, - }, - ], - expectedEventIds: [currentEventId], - expectedActorIds: [actorId], - referenceAnswer: currentFact, - metadata: { - adapter: spec.adapter, - primaryMetrics: spec.primaryMetrics, - }, - } - }) -} - -export function respondToIndustryMemoryBenchmarkSmokeCase(input: { - case: KnowledgeMemoryBenchmarkCase -}): KnowledgeBenchmarkArtifact { - const testCase = input.case - const facts = testCase.requiredFacts - ?.map((fact) => fact.anyOf[0]) - .filter((fragment): fragment is string => Boolean(fragment)) - return { - answer: facts?.join(' ') ?? '', - rememberedFacts: facts ?? [], - citedEventIds: testCase.expectedEventIds ?? [], - actorIds: testCase.expectedActorIds ?? [], - durationMs: 1, - metadata: { - smoke: true, - }, - } -} - -export function buildFirstPartyMemoryLifecycleBenchmarkCases(): KnowledgeMemoryBenchmarkCase[] { - const base = { - family: 'first-party', - source: { - name: 'first-party/memory-lifecycle', - version: 'real-v1', - }, - } as const - return [ - memoryLifecycleCase({ - ...base, - id: 'first-party/memory-lifecycle:allergy', - taskKind: 'memory-ingest', - split: 'dev', - actorId: 'user', - prompt: 'What food restriction should catering remember for this user?', - staleText: 'The user said they had no food allergies on the first onboarding form.', - currentText: 'The user later corrected the profile: they have a severe peanut allergy.', - required: 'severe peanut allergy', - forbidden: 'no food allergies', - }), - memoryLifecycleCase({ - ...base, - id: 'first-party/memory-lifecycle:account-tier', - taskKind: 'memory-recall', - split: 'dev', - actorId: 'sales-ops', - prompt: 'What is the customer account tier now?', - staleText: 'Sales ops originally marked the customer as starter tier.', - currentText: 'Sales ops updated the customer to enterprise tier after procurement approval.', - required: 'enterprise tier', - forbidden: 'starter tier', - }), - memoryLifecycleCase({ - ...base, - id: 'first-party/memory-lifecycle:launch-date', - taskKind: 'memory-temporal', - split: 'holdout', - actorId: 'pm', - prompt: 'What launch date should the agent use?', - staleText: 'The PM first planned the launch for April 3.', - currentText: 'The PM moved the launch date to April 17 after legal review.', - required: 'April 17', - forbidden: 'April 3', - }), - memoryLifecycleCase({ - ...base, - id: 'first-party/memory-lifecycle:briefing-channel', - taskKind: 'memory-update', - split: 'holdout', - actorId: 'user', - prompt: 'How should daily briefings be delivered now?', - staleText: 'The user used to want daily briefings by SMS.', - currentText: 'The user changed daily briefings to email only.', - required: 'email only', - forbidden: 'SMS', - }), - memoryLifecycleCase({ - ...base, - id: 'first-party/memory-lifecycle:shipping-address', - taskKind: 'memory-forgetting', - split: 'holdout', - actorId: 'user', - prompt: 'What shipping address is current?', - staleText: 'The old shipping address was 14 Pine Street, Apartment 2.', - currentText: 'The current shipping address is 88 Cedar Avenue, Suite 9.', - required: '88 Cedar Avenue, Suite 9', - forbidden: '14 Pine Street', - }), - memoryLifecycleCase({ - ...base, - id: 'first-party/memory-lifecycle:approval-owner', - taskKind: 'memory-reasoning', - split: 'holdout', - actorId: 'finance', - prompt: 'Who owns travel approval now?', - staleText: 'Finance said Liam owned travel approvals last quarter.', - currentText: 'Finance reassigned travel approvals to Maya this quarter.', - required: 'Maya', - forbidden: 'Liam', - }), - memoryLifecycleCase({ - ...base, - id: 'first-party/memory-lifecycle:project-summary', - taskKind: 'memory-summarization', - split: 'dev', - actorId: 'pm', - prompt: 'Summarize the current Project Orion risks.', - staleText: 'The old Project Orion risk was logo color churn.', - currentText: 'The current Project Orion risks are vendor delay and a QA staffing gap.', - required: 'vendor delay', - extraRequired: 'QA staffing gap', - forbidden: 'logo color churn', - }), - memoryLifecycleCase({ - ...base, - id: 'first-party/memory-lifecycle:meeting-format', - taskKind: 'memory-recommendation', - split: 'holdout', - actorId: 'user', - prompt: 'What meeting format should the agent recommend?', - staleText: 'The user previously preferred long video calls for planning.', - currentText: 'The user now prefers async docs for planning instead of video calls.', - required: 'async docs', - forbidden: 'long video calls', - }), - memoryLifecycleCase({ - ...base, - id: 'first-party/memory-lifecycle:multiparty-ada', - taskKind: 'memory-multiparty', - split: 'holdout', - actorId: 'ada', - prompt: 'Which SDK language did Ada ask for?', - staleText: 'Ben asked for a Python notebook example.', - currentText: 'Ada asked for a Rust SDK example.', - required: 'Rust SDK', - forbidden: 'Python notebook', - }), - memoryLifecycleCase({ - ...base, - id: 'first-party/memory-lifecycle:timezone', - taskKind: 'memory-recall', - split: 'dev', - actorId: 'user', - prompt: 'What timezone should scheduling use for this user?', - staleText: 'The user profile originally listed Pacific time.', - currentText: 'The user corrected scheduling to America/Denver time.', - required: 'America/Denver', - forbidden: 'Pacific time', - }), - memoryLifecycleCase({ - ...base, - id: 'first-party/memory-lifecycle:dinner-preference', - taskKind: 'memory-update', - split: 'holdout', - actorId: 'user', - prompt: 'What dinner preference should the assistant use?', - staleText: 'The user was previously vegetarian for team dinners.', - currentText: 'The user updated dinner restrictions to no shellfish.', - required: 'no shellfish', - forbidden: 'vegetarian', - }), - memoryLifecycleCase({ - ...base, - id: 'first-party/memory-lifecycle:support-sla', - taskKind: 'memory-temporal', - split: 'holdout', - actorId: 'support-lead', - prompt: 'What support SLA is current?', - staleText: 'Support used to promise a 24 hour response SLA.', - currentText: 'Support changed the current response SLA to 2 business hours.', - required: '2 business hours', - forbidden: '24 hour', - }), - ] -} - -function hitForExpectedTarget( - expected: RetrievalGoldTarget | undefined, - fallbackId: string, -): RetrievedKnowledgeHit { - if (!expected) { - return { - pageId: fallbackId, - path: `${fallbackId}.md`, - rank: 1, - } - } - switch (expected.kind) { - case 'page': - return { - pageId: expected.pageId, - path: `${expected.pageId}.md`, - rank: 1, - } - case 'page-path': - return { - pageId: expected.path, - path: expected.path, - rank: 1, - } - case 'source': - return { - pageId: expected.sourceId, - path: `${expected.sourceId}.md`, - sourceIds: [expected.sourceId], - rank: 1, - } - case 'source-anchor': - return { - pageId: expected.sourceId, - path: `${expected.sourceId}.md`, - sourceIds: [expected.sourceId], - sourceSpans: [{ sourceId: expected.sourceId, anchorId: expected.anchorId }], - rank: 1, - } - case 'source-span': - return { - pageId: expected.sourceId, - path: `${expected.sourceId}.md`, - sourceIds: [expected.sourceId], - sourceSpans: [ - { - sourceId: expected.sourceId, - charStart: expected.charStart, - charEnd: expected.charEnd, - }, - ], - rank: 1, - } - } -} - -function memoryLifecycleCase(input: { - id: string - family: KnowledgeBenchmarkFamily - source: KnowledgeBenchmarkSource - taskKind: KnowledgeMemoryBenchmarkTaskKind - split: KnowledgeBenchmarkSplit - actorId: string - prompt: string - staleText: string - currentText: string - required: string - extraRequired?: string - forbidden: string -}): KnowledgeMemoryBenchmarkCase { - const staleEventId = `${input.id}:stale` - const currentEventId = `${input.id}:current` - return { - id: input.id, - family: input.family, - taskKind: input.taskKind, - split: input.split, - tags: unique(['first-party-memory-lifecycle', input.taskKind, input.split]), - source: input.source, - events: [ - { - id: staleEventId, - actorId: input.actorId, - sessionId: `${input.id}:session-1`, - timestamp: '2026-01-01T00:00:00.000Z', - text: input.staleText, - }, - { - id: currentEventId, - actorId: input.actorId, - sessionId: `${input.id}:session-2`, - timestamp: '2026-02-01T00:00:00.000Z', - text: input.currentText, - }, - ], - prompt: input.prompt, - requiredFacts: [ - { - id: `${input.id}:required-1`, - anyOf: [input.required], - sourceEventIds: [currentEventId], - }, - ...(input.extraRequired - ? [ - { - id: `${input.id}:required-2`, - anyOf: [input.extraRequired], - sourceEventIds: [currentEventId], - }, - ] - : []), - ], - forbiddenFacts: [ - { - id: `${input.id}:stale`, - anyOf: [input.forbidden], - sourceEventIds: [staleEventId], - obsolete: true, - }, - ], - expectedEventIds: [currentEventId], - expectedActorIds: [input.actorId], - referenceAnswer: input.required, - } -} diff --git a/src/benchmarks/index.ts b/src/benchmarks/index.ts index 5ced2b4..f168dd1 100644 --- a/src/benchmarks/index.ts +++ b/src/benchmarks/index.ts @@ -2,63 +2,3 @@ export { createInMemoryBenchmarkAdapter, createNoopMemoryBenchmarkAdapter, } from './adapters' -export { - buildFirstPartyMemoryLifecycleBenchmarkCases, - buildIndustryMemoryBenchmarkSmokeCases, - buildIndustryRagBenchmarkSmokeCases, - INDUSTRY_MEMORY_BENCHMARKS, - INDUSTRY_RAG_BENCHMARKS, - respondToIndustryMemoryBenchmarkSmokeCase, - respondToIndustryRagBenchmarkSmokeCase, -} from './catalog' -export { runMemoryAdapterBenchmark } from './memory-runner' -export { summarizeKnowledgeBenchmarkCampaign } from './metrics' -export { - buildRetrievalBenchmarkCasesFromQrels, - parseKnowledgeBenchmarkJsonl, - parseKnowledgeBenchmarkQrels, -} from './retrieval' -export { - scoreKnowledgeBenchmarkArtifact, - scoreMemoryBenchmarkArtifact, -} from './scoring' -export { - buildKnowledgeBenchmarkScenarios, - knowledgeBenchmarkJudge, - renderKnowledgeBenchmarkReportMarkdown, - runKnowledgeBenchmarkSuite, -} from './suite' -export type { - BuildRetrievalBenchmarkCasesFromQrelsOptions, - KnowledgeAnswerBenchmarkCase, - KnowledgeAnswerBenchmarkTaskKind, - KnowledgeBenchmarkArtifact, - KnowledgeBenchmarkCase, - KnowledgeBenchmarkCaseBase, - KnowledgeBenchmarkDistribution, - KnowledgeBenchmarkEvaluation, - KnowledgeBenchmarkFamily, - KnowledgeBenchmarkReport, - KnowledgeBenchmarkResponder, - KnowledgeBenchmarkScenario, - KnowledgeBenchmarkSliceSummary, - KnowledgeBenchmarkSource, - KnowledgeBenchmarkSpec, - KnowledgeBenchmarkSplit, - KnowledgeBenchmarkTaskKind, - KnowledgeClaimMatcher, - KnowledgeMemoryBenchmarkCase, - KnowledgeMemoryBenchmarkTaskKind, - KnowledgeMemoryEvent, - KnowledgeMemoryFactMatcher, - KnowledgeRetrievalBenchmarkCase, - KnowledgeRetrievalBenchmarkQrel, - KnowledgeRetrievalBenchmarkQuery, - MemoryAdapterBenchmarkCandidate, - MemoryAdapterBenchmarkRankingRow, - RunKnowledgeBenchmarkSuiteOptions, - RunKnowledgeBenchmarkSuiteResult, - RunMemoryAdapterBenchmarkOptions, - RunMemoryAdapterBenchmarkResult, -} from './types' -export { isKnowledgeMemoryBenchmarkCase } from './validation' diff --git a/src/benchmarks/memory-recovery.ts b/src/benchmarks/memory-recovery.ts deleted file mode 100644 index 47001f6..0000000 --- a/src/benchmarks/memory-recovery.ts +++ /dev/null @@ -1,531 +0,0 @@ -import { join } from 'node:path' - -import { canonicalJson } from '@tangle-network/agent-eval' - -import type { CampaignStorage, CostLedgerHandle } from '@tangle-network/agent-eval/campaign' - -import { stableId } from '../ids' - -import { - appendAttemptJournalEvent, - assertNoInterruptedPaidCalls, - hasSettledPaidCall, - readActiveAttemptJournal, - reconcileInterruptedMemoryPaidCalls, - reserveRecoveryAttempts, -} from '../memory/attempt-log' - -import { - createMemoryExecutionPool, - memoryRecoveryDelayMs, - releaseMemoryAdapterCreatedAfterAbort, - runBoundedMemoryLifecycle, - sleepForMemoryRecovery, -} from '../memory/lifecycle' - -import type { OwnedAgentMemoryRunLease } from '../memory/run-control' - -import type { AgentMemoryAdapter, AgentMemoryScope } from '../memory/types' - -import type { MemoryAdapterBenchmarkCandidate } from './types' - -export function memoryAdapterBenchmarkExpectedId( - candidate: MemoryAdapterBenchmarkCandidate, -): string { - return candidate.adapterId ?? candidate.id -} - -export async function recoverMemoryAdapterBenchmarkAttempts(input: { - candidates: readonly MemoryAdapterBenchmarkCandidate[] - storage: CampaignStorage - attemptLogPath: string - lease: OwnedAgentMemoryRunLease - cleanupTimeoutMs: number - maxConcurrency: number - now?: () => Date - runDir: string - costLedger: CostLedgerHandle - costPhase: string - maxRecoveryAttempts: number - recoveryLogPath: string - maxRecoveryRetriesPerAttempt: number -}): Promise { - let attempts = readActiveMemoryBenchmarkAttempts(input.storage, input.attemptLogPath) - if (attempts.length > input.maxRecoveryAttempts) { - throw new Error( - `memory adapter benchmark has ${attempts.length} unfinished attempts; maxRecoveryAttempts is ${input.maxRecoveryAttempts}`, - ) - } - const candidateById = new Map(input.candidates.map((candidate) => [candidate.id, candidate])) - for (const attempt of attempts) { - const candidate = candidateById.get(attempt.candidateId) - if (!candidate) { - throw new Error( - `cannot recover memory benchmark attempts: candidate '${attempt.candidateId}' is missing; pass it in recoveryCandidates`, - ) - } - assertMemoryBenchmarkAttemptCandidateMatches(attempt, candidate) - } - - reconcileInterruptedMemoryPaidCalls(input.costLedger) - assertNoInterruptedPaidCalls(input.costLedger, 'memory adapter benchmark recovery') - - for (const attempt of attempts) { - const candidate = candidateById.get(attempt.candidateId)! - const costUsd = candidate.costUsdPerCase ?? 0 - if ( - costUsd > 0 && - !hasSettledPaidCall(input.costLedger, memoryBenchmarkCostCallId(attempt, 'execute', 0)) - ) { - appendMemoryBenchmarkAttemptEvent(input.storage, input.attemptLogPath, { - ...attempt, - status: 'cleaned', - recovery: true, - recordedAt: (input.now ?? (() => new Date()))().toISOString(), - }) - } - } - attempts = readActiveMemoryBenchmarkAttempts(input.storage, input.attemptLogPath) - const recoveryGenerations = reserveRecoveryAttempts({ - storage: input.storage, - path: input.recoveryLogPath, - attemptIds: attempts.map((attempt) => attempt.attemptId), - maxRetriesPerAttempt: input.maxRecoveryRetriesPerAttempt, - label: 'memory benchmark recovery attempt log', - now: input.now, - }) - const grouped = groupMemoryBenchmarkAttempts(attempts) - const pool = createMemoryExecutionPool(input.maxConcurrency) - const settled = await Promise.allSettled( - [...grouped] - .sort(([left], [right]) => left.localeCompare(right)) - .map(([candidateId, candidateAttempts]) => - pool.run(async () => { - await input.lease.assertOwned() - const candidate = candidateById.get(candidateId) - if (!candidate) { - throw new Error( - `cannot recover memory benchmark attempts: candidate '${candidateId}' is missing; pass it in recoveryCandidates`, - ) - } - for (const attempt of candidateAttempts) { - assertMemoryBenchmarkAttemptCandidateMatches(attempt, candidate) - if (recoveryGenerations.get(attempt.attemptId) === undefined) { - throw new Error( - `missing recovery generation for memory benchmark attempt '${attempt.attemptId}'`, - ) - } - } - - let recoveryAttemptsStarted = 0 - let adapterCreationExternalCallAttempted = false - const cleared: MemoryAdapterBenchmarkAttemptEvent[] = [] - const recover = async (): Promise => { - let adapter: AgentMemoryAdapter | undefined - let primaryError: unknown - try { - const abortController = new AbortController() - const creation = Promise.resolve().then(() => - candidate.createAdapter({ - purpose: 'recovery', - signal: abortController.signal, - markExternalCall: () => { - adapterCreationExternalCallAttempted = true - }, - }), - ) - releaseMemoryAdapterCreatedAfterAbort({ - creation, - signal: abortController.signal, - }) - adapter = await runBoundedMemoryLifecycle({ - operation: `${candidate.id}: benchmark recovery adapter creation`, - timeoutMs: input.cleanupTimeoutMs, - abortController, - run: () => creation, - }) - assertScopedMemoryBenchmarkAdapter(adapter) - for (const attempt of candidateAttempts) { - if (adapter.id !== attempt.adapterId) { - throw new Error( - `cannot recover memory benchmark attempt '${attempt.attemptId}': adapter changed from '${attempt.adapterId}' to '${adapter.id}'`, - ) - } - } - await sleepForMemoryRecovery( - memoryRecoveryDelayMs(adapter), - () => input.lease.assertOwned(), - input.cleanupTimeoutMs, - `${candidate.id}: benchmark recovery visibility wait`, - ) - for (const attempt of candidateAttempts.sort((left, right) => - left.attemptId.localeCompare(right.attemptId), - )) { - await input.lease.assertOwned() - try { - recoveryAttemptsStarted += 1 - await runBoundedMemoryLifecycle({ - operation: `${candidate.id}: abandoned benchmark attempt cleanup`, - timeoutMs: input.cleanupTimeoutMs, - resource: adapter, - run: () => adapter!.clear!(attempt.scope), - }) - await input.lease.assertOwned() - cleared.push(attempt) - } catch (error) { - primaryError = primaryError - ? new AggregateError( - [primaryError, error], - `${candidate.id}: multiple benchmark attempts failed recovery`, - ) - : error - } - } - } catch (error) { - primaryError = primaryError - ? new AggregateError( - [primaryError, error], - `${candidate.id}: benchmark recovery failed in multiple operations`, - ) - : error - } - - let closeError: unknown - if (adapter) { - try { - await runBoundedMemoryLifecycle({ - operation: `${candidate.id}: benchmark recovery adapter close`, - timeoutMs: input.cleanupTimeoutMs, - resource: adapter, - run: () => adapter!.close?.(), - }) - } catch (error) { - closeError = error - } - } - let journalError: unknown - if (!closeError) { - try { - for (const attempt of cleared) { - await input.lease.assertOwned() - appendMemoryBenchmarkAttemptEvent(input.storage, input.attemptLogPath, { - ...attempt, - status: 'cleaned', - recovery: true, - recordedAt: (input.now ?? (() => new Date()))().toISOString(), - }) - } - } catch (error) { - journalError = error - } - } - const failures = [primaryError, closeError, journalError].filter( - (error) => error !== undefined, - ) - if (failures.length > 1) { - throw new AggregateError( - failures, - `${candidate.id}: benchmark attempt recovery, adapter close, or cleanup journal failed`, - ) - } - if (primaryError) throw primaryError - if (closeError) throw closeError - if (journalError) throw journalError - } - - const recoveryCostUsd = candidate.recoveryCostUsdPerAttempt ?? 0 - const adapterCreationCostUsd = candidate.adapterCreationCostUsd ?? 0 - const maximumCostUsd = adapterCreationCostUsd + recoveryCostUsd * candidateAttempts.length - const actualCostUsd = (): number => - (adapterCreationExternalCallAttempted ? adapterCreationCostUsd : 0) + - recoveryCostUsd * recoveryAttemptsStarted - let recoveryError: unknown - if (maximumCostUsd === 0) { - try { - await recover() - } catch (error) { - recoveryError = error - } - } else { - const receipt = { - model: candidate.id, - inputTokens: 0, - outputTokens: 0, - } as const - const tags = memoryBenchmarkRecoveryCostTags(input.runDir, candidate.id) - const paid = await input.costLedger.runPaidCall({ - callId: memoryBenchmarkRecoveryCostCallId( - candidate, - candidateAttempts, - recoveryGenerations, - ), - channel: 'driver', - phase: `${input.costPhase}.${candidate.id}.recovery`, - actor: `agent-knowledge:memory-adapter-recovery:${candidate.id}`, - model: candidate.id, - tags, - maximumCharge: { externallyEnforcedMaximumUsd: maximumCostUsd }, - execute: recover, - receipt: () => ({ ...receipt, actualCostUsd: actualCostUsd() }), - receiptFromError: () => ({ - ...receipt, - actualCostUsd: actualCostUsd(), - }), - }) - if (!paid.succeeded) recoveryError = paid.error - } - if (recoveryError) throw recoveryError - }), - ), - ) - const failures = settled.flatMap((result) => - result.status === 'rejected' ? [result.reason] : [], - ) - if (failures.length === 1) throw failures[0] - if (failures.length > 1) { - throw new AggregateError(failures, 'multiple memory benchmark candidates failed recovery') - } -} - -function groupMemoryBenchmarkAttempts( - attempts: readonly MemoryAdapterBenchmarkAttemptEvent[], -): Map { - const grouped = new Map() - for (const attempt of attempts) { - const group = grouped.get(attempt.candidateId) ?? [] - group.push(attempt) - grouped.set(attempt.candidateId, group) - } - return grouped -} - -export function memoryBenchmarkCostCallId( - attempt: MemoryAdapterBenchmarkAttemptEvent, - purpose: 'execute' | 'recovery', - generation: number, -): string { - return stableId( - 'memory_benchmark_cost_call', - canonicalJson({ - purpose, - generation, - attemptId: attempt.attemptId, - candidateId: attempt.candidateId, - candidateRef: attempt.candidateRef, - adapterId: attempt.adapterId, - adapterCreationCostUsd: attempt.adapterCreationCostUsd, - costUsdPerCase: attempt.costUsdPerCase, - recoveryCostUsdPerAttempt: attempt.recoveryCostUsdPerAttempt, - caseId: attempt.caseId, - cellId: attempt.cellId, - }), - ) -} - -function memoryBenchmarkRecoveryCostCallId( - candidate: MemoryAdapterBenchmarkCandidate, - attempts: readonly MemoryAdapterBenchmarkAttemptEvent[], - generations: ReadonlyMap, -): string { - return stableId( - 'memory_benchmark_recovery_cost_call', - canonicalJson({ - candidateId: candidate.id, - candidateRef: candidate.ref, - adapterCreationCostUsd: candidate.adapterCreationCostUsd ?? 0, - recoveryCostUsdPerAttempt: candidate.recoveryCostUsdPerAttempt ?? 0, - attempts: attempts - .map((attempt) => ({ - attemptId: attempt.attemptId, - generation: generations.get(attempt.attemptId), - })) - .sort((left, right) => left.attemptId.localeCompare(right.attemptId)), - }), - ) -} - -function memoryBenchmarkRecoveryCostTags( - runDir: string, - candidateId: string, -): Record { - return { - runDir: join(runDir, candidateId), - candidateId, - memoryRecovery: 'benchmark', - } -} - -export function appendMemoryBenchmarkAttemptEvent( - storage: CampaignStorage, - path: string, - event: MemoryAdapterBenchmarkAttemptEvent, -): void { - appendAttemptJournalEvent({ - storage, - path, - event, - label: 'memory benchmark attempt log', - }) -} - -function readActiveMemoryBenchmarkAttempts( - storage: CampaignStorage, - path: string, -): MemoryAdapterBenchmarkAttemptEvent[] { - return readActiveAttemptJournal({ - storage, - path, - label: 'memory benchmark attempt log', - parse: parseMemoryBenchmarkAttemptEvent, - id: (event) => event.attemptId, - sameAttempt: sameMemoryBenchmarkAttempt, - }) -} - -function parseMemoryBenchmarkAttemptEvent( - value: unknown, - path: string, - line: number, -): MemoryAdapterBenchmarkAttemptEvent { - const event = value as Partial | null - const valid = - typeof event === 'object' && - event !== null && - (event.status === 'started' || event.status === 'cleaned') && - isNonEmptyString(event.attemptId) && - isNonEmptyString(event.candidateId) && - isNonEmptyString(event.candidateRef) && - isNonEmptyString(event.adapterId) && - isNonEmptyString(event.caseId) && - isNonEmptyString(event.cellId) && - isAgentMemoryScope(event.scope) && - typeof event.adapterCreationCostUsd === 'number' && - Number.isFinite(event.adapterCreationCostUsd) && - event.adapterCreationCostUsd >= 0 && - typeof event.costUsdPerCase === 'number' && - Number.isFinite(event.costUsdPerCase) && - event.costUsdPerCase >= 0 && - typeof event.recoveryCostUsdPerAttempt === 'number' && - Number.isFinite(event.recoveryCostUsdPerAttempt) && - event.recoveryCostUsdPerAttempt >= 0 && - typeof event.recordedAt === 'string' && - !Number.isNaN(Date.parse(event.recordedAt)) && - typeof event.recovery === 'boolean' - if (!valid) { - throw new Error(`invalid memory benchmark attempt event in '${path}' line ${line}`) - } - return value as MemoryAdapterBenchmarkAttemptEvent -} - -function sameMemoryBenchmarkAttempt( - left: MemoryAdapterBenchmarkAttemptEvent, - right: MemoryAdapterBenchmarkAttemptEvent, -): boolean { - return ( - left.attemptId === right.attemptId && - left.candidateId === right.candidateId && - left.candidateRef === right.candidateRef && - left.adapterId === right.adapterId && - left.caseId === right.caseId && - left.cellId === right.cellId && - canonicalJson(left.scope) === canonicalJson(right.scope) && - left.adapterCreationCostUsd === right.adapterCreationCostUsd && - left.costUsdPerCase === right.costUsdPerCase && - left.recoveryCostUsdPerAttempt === right.recoveryCostUsdPerAttempt - ) -} - -function assertMemoryBenchmarkAttemptCandidateMatches( - attempt: MemoryAdapterBenchmarkAttemptEvent, - candidate: MemoryAdapterBenchmarkCandidate, -): void { - if (attempt.candidateRef !== candidate.ref) { - throw new Error( - `cannot recover memory benchmark attempt '${attempt.attemptId}': candidate ref changed from '${attempt.candidateRef}' to '${candidate.ref}'`, - ) - } - const expectedAdapterId = memoryAdapterBenchmarkExpectedId(candidate) - if (attempt.adapterId !== expectedAdapterId) { - throw new Error( - `cannot recover memory benchmark attempt '${attempt.attemptId}': adapter id changed from '${attempt.adapterId}' to '${expectedAdapterId}'`, - ) - } - const executionCost = candidate.costUsdPerCase ?? 0 - const adapterCreationCost = candidate.adapterCreationCostUsd ?? 0 - const recoveryCost = candidate.recoveryCostUsdPerAttempt ?? 0 - if ( - adapterCreationCost !== attempt.adapterCreationCostUsd || - executionCost !== attempt.costUsdPerCase || - recoveryCost !== attempt.recoveryCostUsdPerAttempt - ) { - throw new Error( - `cannot recover memory benchmark attempt '${attempt.attemptId}': candidate cost settings changed; start a new run or restore the recorded costs`, - ) - } -} - -export function assertScopedMemoryBenchmarkAdapter(adapter: AgentMemoryAdapter): void { - if (adapter.branchIsolation?.mode !== 'scoped') { - throw new Error( - `${adapter.id}: direct memory benchmark requires branchIsolation mode scoped; use runAgentMemoryExperiment for dedicated instances`, - ) - } - if (!adapter.clear) { - throw new Error(`${adapter.id}: direct memory benchmark requires exact scoped clear`) - } -} - -function isAgentMemoryScope(value: unknown): value is AgentMemoryScope { - if (!isRecordValue(value)) return false - const allowed = new Set([ - 'tenantId', - 'userId', - 'agentId', - 'teamId', - 'runId', - 'sessionId', - 'namespace', - 'tags', - ]) - if (Object.keys(value).some((key) => !allowed.has(key))) return false - for (const key of [ - 'tenantId', - 'userId', - 'agentId', - 'teamId', - 'runId', - 'sessionId', - 'namespace', - ]) { - if (value[key] !== undefined && typeof value[key] !== 'string') return false - } - if (value.tags === undefined) return true - return ( - isRecordValue(value.tags) && - Object.values(value.tags).every((entry) => typeof entry === 'string') - ) -} - -function isNonEmptyString(value: unknown): value is string { - return typeof value === 'string' && value.trim().length > 0 -} - -function isRecordValue(value: unknown): value is Record { - return typeof value === 'object' && value !== null && !Array.isArray(value) -} - -export interface MemoryAdapterBenchmarkAttemptEvent { - status: 'started' | 'cleaned' - attemptId: string - candidateId: string - candidateRef: string - adapterId: string - caseId: string - cellId: string - scope: AgentMemoryScope - adapterCreationCostUsd: number - costUsdPerCase: number - recoveryCostUsdPerAttempt: number - recordedAt: string - recovery: boolean -} diff --git a/src/benchmarks/memory-responder.ts b/src/benchmarks/memory-responder.ts deleted file mode 100644 index 6c64362..0000000 --- a/src/benchmarks/memory-responder.ts +++ /dev/null @@ -1,263 +0,0 @@ -import { randomUUID } from 'node:crypto' -import type { CampaignStorage } from '@tangle-network/agent-eval/campaign' -import { runBoundedMemoryLifecycle } from '../memory/lifecycle' -import type { OwnedAgentMemoryRunLease } from '../memory/run-control' -import type { AgentMemoryAdapter, AgentMemoryScope } from '../memory/types' -import { memoryActorId, memoryEventId } from './adapters' -import { - appendMemoryBenchmarkAttemptEvent, - assertScopedMemoryBenchmarkAdapter, - type MemoryAdapterBenchmarkAttemptEvent, - memoryBenchmarkCostCallId, -} from './memory-recovery' -import type { - KnowledgeBenchmarkArtifact, - KnowledgeBenchmarkResponder, - KnowledgeMemoryBenchmarkCase, -} from './types' -import { compactObject, unique } from './utils' -import { isKnowledgeMemoryBenchmarkCase } from './validation' - -export function createMemoryAdapterBenchmarkResponder(options: { - adapter: AgentMemoryAdapter - candidateId: string - candidateRef: string - storage: CampaignStorage - attemptLogPath: string - lease: OwnedAgentMemoryRunLease - cleanupTimeoutMs: number - searchLimit?: number - scope?: AgentMemoryScope - costUsdPerCase?: number - adapterCreationCostUsd?: number - recoveryCostUsdPerAttempt?: number - now?: () => Date -}): KnowledgeBenchmarkResponder { - assertScopedMemoryBenchmarkAdapter(options.adapter) - return async ({ case: testCase, context: dispatchContext }) => { - if (!isKnowledgeMemoryBenchmarkCase(testCase)) { - return { answer: '', metadata: { candidateId: options.candidateId, skipped: true } } - } - const costUsd = options.costUsdPerCase ?? 0 - if (!Number.isFinite(costUsd) || costUsd < 0) { - throw new Error(`memory adapter costUsdPerCase must be non-negative finite, got ${costUsd}`) - } - - dispatchContext.signal.throwIfAborted() - await options.lease.assertOwned() - const startedAt = Date.now() - const attemptId = randomUUID() - const scope = benchmarkMemoryScope( - options.candidateId, - testCase, - dispatchContext.cellId, - attemptId, - options.scope, - ) - const attempt: MemoryAdapterBenchmarkAttemptEvent = { - status: 'started', - attemptId, - candidateId: options.candidateId, - candidateRef: options.candidateRef, - adapterId: options.adapter.id, - caseId: testCase.id, - cellId: dispatchContext.cellId, - scope, - adapterCreationCostUsd: options.adapterCreationCostUsd ?? 0, - costUsdPerCase: costUsd, - recoveryCostUsdPerAttempt: options.recoveryCostUsdPerAttempt ?? 0, - recordedAt: (options.now ?? (() => new Date()))().toISOString(), - recovery: false, - } - appendMemoryBenchmarkAttemptEvent(options.storage, options.attemptLogPath, attempt) - - let externalCallAttempted = false - const appendCleanedAttempt = (priorError?: unknown): void => { - try { - appendMemoryBenchmarkAttemptEvent(options.storage, options.attemptLogPath, { - ...attempt, - status: 'cleaned', - recordedAt: (options.now ?? (() => new Date()))().toISOString(), - }) - } catch (error) { - throw new MemoryAdapterBenchmarkCleanupError( - [...(priorError ? [priorError] : []), error], - `${options.candidateId}: memory benchmark cleanup could not be recorded`, - ) - } - } - const execute = async (): Promise => { - dispatchContext.signal.throwIfAborted() - await options.lease.assertOwned() - let artifact: KnowledgeBenchmarkArtifact | undefined - let primaryError: unknown - try { - for (const event of testCase.events) { - dispatchContext.signal.throwIfAborted() - await options.lease.assertOwned() - externalCallAttempted = true - await options.adapter.write({ - id: event.id, - kind: 'message', - text: event.text, - role: event.actorId === 'user' ? 'user' : 'assistant', - title: `${testCase.id}:${event.id}`, - scope, - metadata: compactObject({ - benchmarkCaseId: testCase.id, - benchmarkCellId: dispatchContext.cellId, - benchmarkAttemptId: attemptId, - eventId: event.id, - actorId: event.actorId, - sessionId: event.sessionId, - timestamp: event.timestamp, - ...event.metadata, - }) as Record, - }) - dispatchContext.signal.throwIfAborted() - await options.lease.assertOwned() - } - externalCallAttempted = true - await options.adapter.flush?.() - dispatchContext.signal.throwIfAborted() - await options.lease.assertOwned() - - externalCallAttempted = true - const adapterContext = await options.adapter.getContext(testCase.prompt, { - scope, - limit: options.searchLimit ?? 1, - metadata: { - benchmarkCaseId: testCase.id, - benchmarkCellId: dispatchContext.cellId, - benchmarkAttemptId: attemptId, - candidateId: options.candidateId, - }, - }) - dispatchContext.signal.throwIfAborted() - await options.lease.assertOwned() - const hits = adapterContext.hits - artifact = { - answer: adapterContext.text, - rememberedFacts: hits.map((hit) => hit.text), - citedEventIds: unique(hits.map(memoryEventId).filter((id): id is string => Boolean(id))), - usedMemoryIds: hits.map((hit) => hit.id), - actorIds: unique(hits.map(memoryActorId).filter((id): id is string => Boolean(id))), - costUsd, - durationMs: Math.max(0, Date.now() - startedAt), - metadata: { - candidateId: options.candidateId, - adapterId: options.adapter.id, - hitCount: hits.length, - }, - } - } catch (error) { - primaryError = error - } - - const cleanupErrors: unknown[] = [] - let cleanupOwned = true - try { - await options.lease.assertOwned() - } catch (error) { - cleanupOwned = false - cleanupErrors.push(error) - } - if (cleanupOwned) { - try { - await runBoundedMemoryLifecycle({ - operation: `${options.candidateId}: benchmark attempt flush`, - timeoutMs: options.cleanupTimeoutMs, - resource: options.adapter, - run: () => { - externalCallAttempted = true - return options.adapter.flush?.() - }, - }) - } catch (error) { - cleanupErrors.push(error) - } - try { - await runBoundedMemoryLifecycle({ - operation: `${options.candidateId}: benchmark attempt cleanup`, - timeoutMs: options.cleanupTimeoutMs, - resource: options.adapter, - run: () => { - externalCallAttempted = true - return options.adapter.clear!(scope) - }, - }) - } catch (error) { - cleanupErrors.push(error) - } - } - if (cleanupOwned && cleanupErrors.length === 0) appendCleanedAttempt(primaryError) - if (cleanupErrors.length > 0) { - const errors = [...(primaryError ? [primaryError] : []), ...cleanupErrors] - throw new MemoryAdapterBenchmarkCleanupError( - errors, - `${options.candidateId}: memory benchmark attempt cleanup failed`, - ) - } - if (primaryError) throw primaryError - if (!artifact) - throw new Error(`${options.candidateId}: memory benchmark produced no artifact`) - return artifact - } - - if (costUsd === 0) { - let artifact: KnowledgeBenchmarkArtifact | undefined - let error: unknown - try { - artifact = await execute() - } catch (caught) { - error = caught - } - if (error) throw error - if (!artifact) - throw new Error(`${options.candidateId}: memory benchmark produced no artifact`) - return artifact - } - const receipt = { - model: options.adapter.id, - inputTokens: 0, - outputTokens: 0, - actualCostUsd: costUsd, - } as const - const paid = await dispatchContext.cost.runPaidCall({ - callId: memoryBenchmarkCostCallId(attempt, 'execute', 0), - actor: `agent-knowledge:memory-adapter:${options.adapter.id}`, - model: options.adapter.id, - maximumCharge: { externallyEnforcedMaximumUsd: costUsd }, - execute, - receipt: () => receipt, - receiptFromError: () => ({ - ...receipt, - actualCostUsd: externalCallAttempted ? costUsd : 0, - }), - }) - if (!paid.succeeded) throw paid.error - return paid.value - } -} - -export class MemoryAdapterBenchmarkCleanupError extends AggregateError {} - -function benchmarkMemoryScope( - candidateId: string, - testCase: KnowledgeMemoryBenchmarkCase, - cellId: string, - attemptId: string, - scope: AgentMemoryScope = {}, -): AgentMemoryScope { - return { - ...scope, - namespace: `${scope.namespace ?? 'agent-knowledge-memory-benchmark'}:${attemptId}`, - tags: { - ...(scope.tags ?? {}), - benchmarkCandidateId: candidateId, - benchmarkCaseId: testCase.id, - benchmarkCellId: cellId, - benchmarkAttemptId: attemptId, - }, - } -} diff --git a/src/benchmarks/memory-runner.ts b/src/benchmarks/memory-runner.ts deleted file mode 100644 index 9ba9c68..0000000 --- a/src/benchmarks/memory-runner.ts +++ /dev/null @@ -1,548 +0,0 @@ -import { join } from 'node:path' -import { canonicalJson } from '@tangle-network/agent-eval' -import { - type CampaignStorage, - type CostLedgerHandle, - createRunCostLedger, - fsCampaignStorage, - resolveRunDir, -} from '@tangle-network/agent-eval/campaign' -import { rankCandidates } from '../candidate-ranking' - -import { stableId } from '../ids' - -import { DEFAULT_MEMORY_RECOVERY_RETRIES_PER_ATTEMPT } from '../memory/attempt-log' - -import { - releaseMemoryAdapterCreatedAfterAbort, - resolveMemoryCleanupTimeoutMs, - runBoundedMemoryLifecycle, -} from '../memory/lifecycle' - -import { acquireAgentMemoryRunLease, type OwnedAgentMemoryRunLease } from '../memory/run-control' - -import type { AgentMemoryAdapter } from '../memory/types' - -import { - assertScopedMemoryBenchmarkAdapter, - memoryAdapterBenchmarkExpectedId, - recoverMemoryAdapterBenchmarkAttempts, -} from './memory-recovery' - -import { - createMemoryAdapterBenchmarkResponder, - MemoryAdapterBenchmarkCleanupError, -} from './memory-responder' - -import { runKnowledgeBenchmarkSuite } from './suite' - -import type { - KnowledgeBenchmarkArtifact, - KnowledgeBenchmarkResponder, - MemoryAdapterBenchmarkCandidate, - MemoryAdapterBenchmarkRankingRow, - RunKnowledgeBenchmarkSuiteResult, - RunMemoryAdapterBenchmarkOptions, - RunMemoryAdapterBenchmarkResult, -} from './types' - -import { formatNumber, normalizeUsd } from './utils' - -import { assertNonEmptyBenchmarkString, assertUniqueNonEmptyStrings } from './validation' - -const MEMORY_ADAPTER_BENCHMARK_IMPLEMENTATION_REF = 'agent-knowledge:memory-adapter-benchmark:v7' - -export async function runMemoryAdapterBenchmark( - options: RunMemoryAdapterBenchmarkOptions, -): Promise { - if (options.candidates.length === 0) - throw new Error('memory adapter benchmark requires candidates') - const allCandidates = [...options.candidates, ...(options.recoveryCandidates ?? [])] - assertUniqueNonEmptyStrings( - allCandidates.map((candidate) => candidate.id), - 'memory adapter candidate id', - ) - for (const candidate of allCandidates) { - assertNonEmptyBenchmarkString(candidate.ref, `memory adapter candidate ${candidate.id} ref`) - if (candidate.adapterId !== undefined) { - assertNonEmptyBenchmarkString( - candidate.adapterId, - `memory adapter candidate ${candidate.id} adapterId`, - ) - } - if (!/^[A-Za-z0-9][A-Za-z0-9._-]*$/.test(candidate.id)) { - throw new Error( - `memory adapter candidate id '${candidate.id}' must be a safe directory segment`, - ) - } - if ( - candidate.adapterCreationCostUsd !== undefined && - (!Number.isFinite(candidate.adapterCreationCostUsd) || candidate.adapterCreationCostUsd < 0) - ) { - throw new Error( - `${candidate.id}: adapterCreationCostUsd must be a non-negative finite number`, - ) - } - if ( - candidate.costUsdPerCase !== undefined && - (!Number.isFinite(candidate.costUsdPerCase) || candidate.costUsdPerCase < 0) - ) { - throw new Error(`${candidate.id}: costUsdPerCase must be a non-negative finite number`) - } - if ( - candidate.recoveryCostUsdPerAttempt !== undefined && - (!Number.isFinite(candidate.recoveryCostUsdPerAttempt) || - candidate.recoveryCostUsdPerAttempt < 0) - ) { - throw new Error( - `${candidate.id}: recoveryCostUsdPerAttempt must be a non-negative finite number`, - ) - } - } - const storage = options.storage ?? fsCampaignStorage() - if (!storage.append) { - throw new Error('memory adapter benchmark requires CampaignStorage.append') - } - const runDir = resolveRunDir(options.runDir, options.repo) - const cleanupTimeoutMs = resolveMemoryCleanupTimeoutMs( - options.cleanupTimeoutMs, - 'memory adapter benchmark', - ) - const maxRecoveryAttempts = options.maxRecoveryAttempts ?? 1_000 - if (!Number.isSafeInteger(maxRecoveryAttempts) || maxRecoveryAttempts <= 0) { - throw new Error('memory adapter benchmark maxRecoveryAttempts must be a positive safe integer') - } - const maxRecoveryRetriesPerAttempt = - options.maxRecoveryRetriesPerAttempt ?? DEFAULT_MEMORY_RECOVERY_RETRIES_PER_ATTEMPT - if (!Number.isSafeInteger(maxRecoveryRetriesPerAttempt) || maxRecoveryRetriesPerAttempt <= 0) { - throw new Error( - 'memory adapter benchmark maxRecoveryRetriesPerAttempt must be a positive safe integer', - ) - } - storage.ensureDir(runDir) - const lease = await acquireAgentMemoryRunLease({ - experimentId: `memory-adapter-benchmark:${runDir}`, - runDir, - storage, - customStorage: options.storage !== undefined, - lockFileName: 'memory-adapter-benchmark.lock', - label: 'memory adapter benchmark', - controllerMode: options.controllerMode, - acquireRunLease: options.acquireRunLease, - }) - let result: RunMemoryAdapterBenchmarkResult | undefined - let primaryError: unknown - try { - result = await runOwnedMemoryAdapterBenchmark( - options, - storage, - runDir, - lease, - cleanupTimeoutMs, - maxRecoveryAttempts, - maxRecoveryRetriesPerAttempt, - ) - } catch (error) { - primaryError = error - } - let releaseError: unknown - try { - await lease.release() - } catch (error) { - releaseError = error - } - if (primaryError && releaseError) { - throw new AggregateError( - [primaryError, releaseError], - 'memory adapter benchmark failed and its controller lease could not be released', - ) - } - if (primaryError) throw primaryError - if (releaseError) throw releaseError - if (!result) throw new Error('memory adapter benchmark produced no result') - return result -} - -async function runOwnedMemoryAdapterBenchmark( - options: RunMemoryAdapterBenchmarkOptions, - storage: CampaignStorage, - runDir: string, - lease: OwnedAgentMemoryRunLease, - cleanupTimeoutMs: number, - maxRecoveryAttempts: number, - maxRecoveryRetriesPerAttempt: number, -): Promise { - const maxConcurrency = options.maxConcurrency ?? 2 - if (!Number.isSafeInteger(maxConcurrency) || maxConcurrency <= 0) { - throw new Error('memory adapter benchmark maxConcurrency must be a positive safe integer') - } - const costCeiling = options.costCeiling ?? options.costLedger?.costCeilingUsd ?? 0 - const costLedger = - options.costLedger ?? - createRunCostLedger({ - storage, - runDir, - costCeilingUsd: costCeiling, - }) - if (costLedger.costCeilingUsd !== costCeiling) { - throw new Error( - 'memory adapter benchmark costCeiling must match the shared cost ledger ceiling', - ) - } - const attemptLogPath = join(runDir, 'memory-adapter-attempts.jsonl') - const recoveryLogPath = join(runDir, 'memory-adapter-recovery-attempts.jsonl') - await recoverMemoryAdapterBenchmarkAttempts({ - candidates: [...options.candidates, ...(options.recoveryCandidates ?? [])], - storage, - attemptLogPath, - lease, - cleanupTimeoutMs, - maxConcurrency, - now: options.now, - runDir, - costLedger, - costPhase: options.costPhase ?? 'memory.adapter-benchmark', - maxRecoveryAttempts, - recoveryLogPath, - maxRecoveryRetriesPerAttempt, - }) - await lease.assertOwned() - const rows: MemoryAdapterBenchmarkRankingRow[] = [] - for (const candidate of options.candidates) { - await lease.assertOwned() - const expectedAdapterId = memoryAdapterBenchmarkExpectedId(candidate) - let adapter: AgentMemoryAdapter | undefined - let adapterPromise: Promise | undefined - let adapterCreationError: unknown - const getAdapter = (): Promise => { - if (!adapterPromise) { - const abortController = new AbortController() - const creation = createMemoryAdapterBenchmarkAdapter({ - candidate, - purpose: 'execute', - signal: abortController.signal, - costLedger, - runDir, - costPhase: options.costPhase ?? 'memory.adapter-benchmark', - }) - releaseMemoryAdapterCreatedAfterAbort({ creation, signal: abortController.signal }) - adapterPromise = runBoundedMemoryLifecycle({ - operation: `${candidate.id}: benchmark execute adapter creation`, - timeoutMs: Math.min(cleanupTimeoutMs, options.dispatchTimeoutMs ?? cleanupTimeoutMs), - abortController, - run: () => creation, - }) - .then((created) => { - adapter = created - if (created.id !== expectedAdapterId) { - throw new Error( - `${candidate.id}: createAdapter returned id '${created.id}', expected '${expectedAdapterId}'`, - ) - } - assertScopedMemoryBenchmarkAdapter(created) - return created - }) - .catch((error) => { - adapterCreationError = error - throw error - }) - } - return adapterPromise - } - const dispatchedExecutions: Promise[] = [] - let run: RunKnowledgeBenchmarkSuiteResult | undefined - let primaryError: unknown - try { - await lease.assertOwned() - let respond: KnowledgeBenchmarkResponder | undefined - run = await runKnowledgeBenchmarkSuite({ - cases: options.cases, - respond(input) { - const operation = getAdapter().then((activeAdapter) => { - respond ??= createMemoryAdapterBenchmarkResponder({ - adapter: activeAdapter, - candidateId: candidate.id, - candidateRef: candidate.ref, - storage, - attemptLogPath, - lease, - cleanupTimeoutMs, - searchLimit: candidate.searchLimit, - scope: candidate.scope, - adapterCreationCostUsd: candidate.adapterCreationCostUsd, - costUsdPerCase: candidate.costUsdPerCase, - recoveryCostUsdPerAttempt: candidate.recoveryCostUsdPerAttempt, - now: options.now, - }) - return respond(input) - }) - dispatchedExecutions.push(operation) - return operation - }, - respondRef: stableId( - 'memory_adapter_benchmark', - canonicalJson({ - implementationRef: MEMORY_ADAPTER_BENCHMARK_IMPLEMENTATION_REF, - candidateRef: candidate.ref, - adapterId: expectedAdapterId, - searchLimit: candidate.searchLimit ?? null, - adapterCreationCostUsd: candidate.adapterCreationCostUsd ?? 0, - costUsdPerCase: candidate.costUsdPerCase ?? 0, - recoveryCostUsdPerAttempt: candidate.recoveryCostUsdPerAttempt ?? 0, - scope: candidate.scope ?? null, - }), - ), - runDir: join(runDir, candidate.id), - storage, - seed: options.seed, - reps: options.reps, - resumable: options.resumable, - costCeiling, - costLedger, - costPhase: `${options.costPhase ?? 'memory.adapter-benchmark'}.${candidate.id}`, - maxConcurrency: options.maxConcurrency, - dispatchTimeoutMs: options.dispatchTimeoutMs, - expectUsage: options.expectUsage ?? 'off', - now: options.now, - }) - } catch (error) { - primaryError = error - } - - const settledExecutions = await Promise.allSettled(dispatchedExecutions) - const dispatchCleanupErrors = settledExecutions.flatMap((settled) => - settled.status === 'rejected' && settled.reason instanceof MemoryAdapterBenchmarkCleanupError - ? [settled.reason] - : [], - ) - if (!primaryError && adapterCreationError) primaryError = adapterCreationError - if (run) { - rows.push({ - rank: 0, - candidateId: candidate.id, - label: candidate.label ?? candidate.id, - adapterId: expectedAdapterId, - scoreMean: run.report.score.mean, - passRate: run.report.dimensions.passed?.mean ?? 0, - totalCases: run.report.totalCases, - totalCells: run.report.totalCells, - cellsFailed: run.report.cellsFailed, - totalCostUsd: run.report.totalCostUsd, - reportJsonPath: run.reportJsonPath, - reportMarkdownPath: run.reportMarkdownPath, - report: run.report, - }) - } - const cleanupErrors: unknown[] = [] - let cleanupOwned = true - try { - await lease.assertOwned() - } catch (error) { - cleanupOwned = false - cleanupErrors.push(error) - } - if (cleanupOwned && adapter && !adapterCreationError) { - const activeAdapter = adapter - try { - await runBoundedMemoryLifecycle({ - operation: `${candidate.id}: benchmark adapter flush`, - timeoutMs: cleanupTimeoutMs, - resource: activeAdapter, - run: () => activeAdapter.flush?.(), - }) - } catch (error) { - cleanupErrors.push(error) - } - } - if (adapter) { - try { - await runBoundedMemoryLifecycle({ - operation: `${candidate.id}: benchmark adapter close`, - timeoutMs: cleanupTimeoutMs, - resource: adapter, - run: () => adapter!.close?.(), - }) - } catch (error) { - cleanupErrors.push(error) - } - } - if (primaryError || dispatchCleanupErrors.length > 0 || cleanupErrors.length > 0) { - const errors = [ - ...(primaryError ? [primaryError] : []), - ...dispatchCleanupErrors, - ...cleanupErrors, - ] - if (errors.length === 1) throw errors[0] - throw new AggregateError(errors, `${candidate.id}: memory adapter benchmark cleanup failed`) - } - await lease.assertOwned() - } - - const costByCandidate = memoryAdapterBenchmarkCostByCandidate(costLedger, runDir, [ - ...options.candidates, - ...(options.recoveryCandidates ?? []), - ]) - const ranked = rankCandidates( - rows.map((row) => { - const totalCostUsd = normalizeUsd(costByCandidate.get(row.candidateId) ?? 0) - return { - ...row, - totalCostUsd, - } - }), - ) - - const rankingJsonPath = join(runDir, 'memory-adapter-ranking.json') - const rankingMarkdownPath = join(runDir, 'memory-adapter-ranking.md') - const unrankedRecoveryCostUsd = normalizeUsd( - (options.recoveryCandidates ?? []).reduce( - (sum, candidate) => sum + (costByCandidate.get(candidate.id) ?? 0), - 0, - ), - ) - const totalCostUsd = normalizeUsd( - [...costByCandidate.values()].reduce((sum, cost) => sum + cost, 0), - ) - storage.write( - rankingJsonPath, - `${JSON.stringify({ totalCostUsd, unrankedRecoveryCostUsd, rows: ranked }, null, 2)}\n`, - ) - storage.write( - rankingMarkdownPath, - renderMemoryAdapterRankingMarkdown(ranked, totalCostUsd, unrankedRecoveryCostUsd), - ) - return { - rows: ranked, - totalCostUsd, - unrankedRecoveryCostUsd, - rankingJsonPath, - rankingMarkdownPath, - attemptLogPath, - recoveryLogPath, - } -} - -async function createMemoryAdapterBenchmarkAdapter(input: { - candidate: MemoryAdapterBenchmarkCandidate - purpose: 'execute' | 'recovery' - signal: AbortSignal - costLedger: CostLedgerHandle - runDir: string - costPhase: string -}): Promise { - const { candidate, purpose, signal, costLedger, runDir, costPhase } = input - const costUsd = candidate.adapterCreationCostUsd ?? 0 - let externalCallAttempted = false - const create = async (): Promise => { - const adapter = await candidate.createAdapter({ - purpose, - signal, - markExternalCall: () => { - externalCallAttempted = true - }, - }) - if (!adapter || typeof adapter !== 'object') { - throw new Error(`${candidate.id}: createAdapter returned no ${purpose} adapter`) - } - return adapter - } - if (costUsd === 0) return create() - - const tags = memoryAdapterCreationCostTags(runDir, candidate.id, purpose) - const generation = costLedger.list({ tags }).length - const receipt = { - model: candidate.id, - inputTokens: 0, - outputTokens: 0, - actualCostUsd: costUsd, - } as const - const paid = await costLedger.runPaidCall({ - callId: memoryAdapterCreationCostCallId(candidate, purpose, generation), - channel: 'driver', - phase: `${costPhase}.${candidate.id}.adapter-${purpose}`, - actor: `agent-knowledge:memory-adapter:${candidate.id}`, - model: candidate.id, - tags, - maximumCharge: { externallyEnforcedMaximumUsd: costUsd }, - execute: create, - receipt: () => ({ - ...receipt, - actualCostUsd: externalCallAttempted ? costUsd : 0, - }), - receiptFromError: () => ({ - ...receipt, - actualCostUsd: externalCallAttempted ? costUsd : 0, - }), - }) - if (!paid.succeeded) throw paid.error - return paid.value -} - -function memoryAdapterCreationCostTags( - runDir: string, - candidateId: string, - purpose: 'execute' | 'recovery', -): Record { - return { - runDir: join(runDir, candidateId), - candidateId, - memoryAdapterCreation: purpose, - } -} - -function memoryAdapterCreationCostCallId( - candidate: MemoryAdapterBenchmarkCandidate, - purpose: 'execute' | 'recovery', - generation: number, -): string { - return stableId( - 'memory_adapter_creation_cost_call', - canonicalJson({ - purpose, - generation, - candidateId: candidate.id, - candidateRef: candidate.ref, - adapterCreationCostUsd: candidate.adapterCreationCostUsd ?? 0, - }), - ) -} - -function memoryAdapterBenchmarkCostByCandidate( - costLedger: CostLedgerHandle, - runDir: string, - candidates: readonly MemoryAdapterBenchmarkCandidate[], -): ReadonlyMap { - const candidateByRunDir = new Map( - candidates.map((candidate) => [join(runDir, candidate.id), candidate.id]), - ) - const totals = new Map() - for (const receipt of costLedger.list()) { - const candidateId = receipt.tags?.runDir - ? candidateByRunDir.get(receipt.tags.runDir) - : undefined - if (!candidateId) continue - totals.set(candidateId, (totals.get(candidateId) ?? 0) + receipt.costUsd) - } - return totals -} - -function renderMemoryAdapterRankingMarkdown( - rows: readonly MemoryAdapterBenchmarkRankingRow[], - totalCostUsd: number, - unrankedRecoveryCostUsd: number, -): string { - return [ - '# Memory Adapter Ranking', - '', - `- total cost: $${formatNumber(totalCostUsd)}`, - `- retired-candidate recovery cost: $${formatNumber(unrankedRecoveryCostUsd)}`, - '', - '| rank | candidate | adapter | cases | cells | failed | mean score | pass rate | cost |', - '| ---: | --- | --- | ---: | ---: | ---: | ---: | ---: | ---: |', - ...rows.map( - (row) => - `| ${row.rank} | ${row.label} | ${row.adapterId} | ${row.totalCases} | ${row.totalCells} | ${row.cellsFailed} | ${formatNumber(row.scoreMean)} | ${formatNumber(row.passRate)} | $${formatNumber(row.totalCostUsd)} |`, - ), - '', - ].join('\n') -} diff --git a/src/benchmarks/metrics.ts b/src/benchmarks/metrics.ts deleted file mode 100644 index 371a066..0000000 --- a/src/benchmarks/metrics.ts +++ /dev/null @@ -1,117 +0,0 @@ -import type { CampaignResult } from '@tangle-network/agent-eval/campaign' - -import type { - KnowledgeBenchmarkDistribution, - KnowledgeBenchmarkReport, - KnowledgeBenchmarkScenario, - KnowledgeBenchmarkSliceSummary, -} from './types' - -import { formatNumber, mean } from './utils' - -export function summarizeKnowledgeBenchmarkCampaign(input: { - scenarios: readonly KnowledgeBenchmarkScenario[] - campaign: CampaignResult -}): KnowledgeBenchmarkReport { - const scenariosById = new Map(input.scenarios.map((scenario) => [scenario.id, scenario])) - const rows = input.campaign.cells.map((cell) => { - const score = Object.values(cell.judgeScores)[0] - const scenario = scenariosById.get(cell.scenarioId) - return { - cell, - scenario, - composite: score?.composite ?? 0, - passed: (score?.dimensions.passed ?? 0) >= 1, - dimensions: score?.dimensions ?? {}, - } - }) - const successful = rows.filter((row) => !row.cell.error) - return { - totalCases: input.scenarios.length, - totalCells: input.campaign.cells.length, - cellsFailed: input.campaign.aggregates.cellsFailed, - cellsCached: input.campaign.aggregates.cellsCached, - totalCostUsd: input.campaign.aggregates.cost.totalCostUsd, - bySplit: summarizeSlices(successful, (row) => row.scenario?.splitTag ?? 'unknown'), - byFamily: summarizeSlices(successful, (row) => row.scenario?.family ?? 'unknown'), - byTaskKind: summarizeSlices(successful, (row) => row.scenario?.taskKind ?? 'unknown'), - dimensions: summarizeDimensions(successful.map((row) => row.dimensions)), - score: distribution(successful.map((row) => row.composite)), - } -} - -function summarizeDimensions( - rows: Array>, -): Record { - const values = new Map() - for (const row of rows) { - for (const [key, value] of Object.entries(row)) { - if (!Number.isFinite(value)) continue - const list = values.get(key) ?? [] - list.push(value) - values.set(key, list) - } - } - return Object.fromEntries([...values.entries()].map(([key, vals]) => [key, distribution(vals)])) -} - -function summarizeSlices( - rows: T[], - keyOf: (row: T) => string, -): Record { - const grouped = new Map() - for (const row of rows) { - const key = keyOf(row) - const list = grouped.get(key) ?? [] - list.push(row) - grouped.set(key, list) - } - return Object.fromEntries( - [...grouped.entries()].map(([key, list]) => { - const withShape = list as Array<{ composite: number; passed: boolean }> - return [ - key, - { - n: list.length, - meanScore: mean(withShape.map((row) => row.composite)), - passRate: mean(withShape.map((row) => (row.passed ? 1 : 0))), - score: distribution(withShape.map((row) => row.composite)), - }, - ] - }), - ) -} - -function distribution(values: readonly number[]): KnowledgeBenchmarkDistribution { - const finite = [...values].filter(Number.isFinite).sort((a, b) => a - b) - if (finite.length === 0) return { n: 0, min: 0, mean: 0, median: 0, p90: 0, max: 0 } - return { - n: finite.length, - min: finite[0]!, - mean: mean(finite), - median: percentile(finite, 0.5), - p90: percentile(finite, 0.9), - max: finite[finite.length - 1]!, - } -} - -function percentile(sortedValues: readonly number[], p: number): number { - if (sortedValues.length === 0) return 0 - const index = Math.min( - sortedValues.length - 1, - Math.max(0, Math.ceil(p * sortedValues.length) - 1), - ) - return sortedValues[index]! -} - -export function renderSliceTable(slices: Record): string { - const rows = Object.entries(slices).map( - ([key, slice]) => - `| ${key} | ${slice.n} | ${formatNumber(slice.meanScore)} | ${formatNumber(slice.passRate)} | ${formatNumber(slice.score.p90)} |`, - ) - return [ - '| slice | n | mean score | pass rate | score p90 |', - '| --- | ---: | ---: | ---: | ---: |', - ...(rows.length ? rows : ['| none | 0 | 0 | 0 | 0 |']), - ].join('\n') -} diff --git a/src/benchmarks/retrieval.ts b/src/benchmarks/retrieval.ts deleted file mode 100644 index a03b071..0000000 --- a/src/benchmarks/retrieval.ts +++ /dev/null @@ -1,95 +0,0 @@ -import type { RetrievalGoldTarget } from '../retrieval-eval' - -import type { - BuildRetrievalBenchmarkCasesFromQrelsOptions, - KnowledgeRetrievalBenchmarkCase, - KnowledgeRetrievalBenchmarkQrel, -} from './types' - -import { compactObject, unique } from './utils' - -export function parseKnowledgeBenchmarkJsonl(text: string): T[] { - return text - .split(/\r?\n/) - .map((line) => line.trim()) - .filter(Boolean) - .map((line, index) => { - try { - return JSON.parse(line) as T - } catch (error) { - throw new Error(`invalid JSONL row ${index + 1}: ${(error as Error).message}`) - } - }) -} - -export function parseKnowledgeBenchmarkQrels(text: string): KnowledgeRetrievalBenchmarkQrel[] { - return text - .split(/\r?\n/) - .map((line) => line.trim()) - .filter((line) => line && !line.startsWith('#')) - .flatMap((line, index) => { - const parts = line.split(/\t|\s+/) - if (parts.length < 3) return [] - const [queryId, maybeZeroOrDocId, maybeDocIdOrScore, maybeScore] = parts - if (!queryId || !maybeZeroOrDocId || !maybeDocIdOrScore) return [] - if (queryId.toLowerCase() === 'qid' || queryId.toLowerCase() === 'query-id') return [] - const documentId = maybeScore === undefined ? maybeZeroOrDocId : maybeDocIdOrScore - const scoreText = maybeScore === undefined ? maybeDocIdOrScore : maybeScore - const score = Number(scoreText) - if (!documentId || !Number.isFinite(score)) { - throw new Error(`invalid qrels row ${index + 1}: expected query id, doc id, score`) - } - return [{ queryId, documentId, score }] - }) -} - -export function buildRetrievalBenchmarkCasesFromQrels( - options: BuildRetrievalBenchmarkCasesFromQrelsOptions, -): KnowledgeRetrievalBenchmarkCase[] { - const qrelsByQuery = new Map() - for (const qrel of options.qrels) { - if (qrel.score <= 0) continue - const list = qrelsByQuery.get(qrel.queryId) ?? [] - list.push(qrel) - qrelsByQuery.set(qrel.queryId, list) - } - - return options.queries.flatMap((query) => { - const qrels = qrelsByQuery.get(query.id) ?? [] - if (qrels.length === 0) return [] - const split = query.split ?? options.splitOf?.(query.id) - const expected = qrels.map((qrel) => - options.documentTarget - ? options.documentTarget(qrel.documentId, qrel) - : defaultDocumentTarget(qrel.documentId, options.targetKind ?? 'page'), - ) - return [ - compactObject({ - id: `${options.benchmarkId}:${query.id}`, - family: options.family, - taskKind: 'retrieval' as const, - query: query.text, - expected, - k: options.k, - split, - tags: unique([...(options.tags ?? []), ...(query.tags ?? []), ...(split ? [split] : [])]), - source: options.source, - metadata: query.metadata, - }) as KnowledgeRetrievalBenchmarkCase, - ] - }) -} - -function defaultDocumentTarget( - documentId: string, - targetKind: 'page' | 'page-path' | 'source', -): RetrievalGoldTarget { - switch (targetKind) { - case 'page': - return { kind: 'page', pageId: documentId } - case 'page-path': - return { kind: 'page-path', path: documentId } - case 'source': - return { kind: 'source', sourceId: documentId } - } -} diff --git a/src/benchmarks/scoring.ts b/src/benchmarks/scoring.ts deleted file mode 100644 index f767073..0000000 --- a/src/benchmarks/scoring.ts +++ /dev/null @@ -1,231 +0,0 @@ -import { - type RetrievalEvalArtifact, - type RetrievalEvalScenario, - scoreRetrievalArtifact, -} from '../retrieval-eval' - -import type { - KnowledgeBenchmarkArtifact, - KnowledgeBenchmarkCase, - KnowledgeBenchmarkEvaluation, - KnowledgeClaimMatcher, - KnowledgeMemoryBenchmarkCase, - KnowledgeRetrievalBenchmarkCase, -} from './types' - -import { mean, unique } from './utils' - -import { isKnowledgeMemoryBenchmarkCase } from './validation' - -export function scoreKnowledgeBenchmarkArtifact( - testCase: KnowledgeBenchmarkCase, - artifact: TArtifact, -): KnowledgeBenchmarkEvaluation { - if (testCase.taskKind === 'retrieval') { - const retrievalArtifact = normalizeRetrievalArtifact(testCase, artifact) - const metrics = scoreRetrievalArtifact(retrievalArtifact, retrievalScenarioForCase(testCase)) - return { - score: metrics.recall, - passed: metrics.recall >= 1, - dimensions: { - recall: metrics.recall, - mrr: metrics.mrr, - ndcg: metrics.ndcg, - precision_at_k: metrics.precisionAtK, - expected_count: metrics.expectedCount, - matched_count: metrics.matchedCount, - }, - notes: `matched ${metrics.matchedCount}/${metrics.expectedCount}; first_hit_rank=${metrics.firstHitRank ?? 'none'}`, - raw: { matchedTargetIds: metrics.matchedTargetIds }, - } - } - if (isKnowledgeMemoryBenchmarkCase(testCase)) { - return scoreMemoryBenchmarkArtifact(testCase, artifact) - } - - const answerArtifact = artifact as KnowledgeBenchmarkArtifact - const text = answerArtifact.text ?? answerArtifact.answer ?? '' - const required = scoreClaims(text, testCase.requiredClaims ?? []) - const forbidden = scoreForbiddenClaims(text, testCase.forbiddenClaims ?? []) - const citation = scoreCitationRecall( - answerArtifact.citedSourceIds ?? [], - testCase.expectedSourceIds ?? [], - ) - const components = [ - required.totalWeight > 0 ? required.recall : undefined, - testCase.expectedSourceIds && testCase.expectedSourceIds.length > 0 ? citation : undefined, - forbidden.safe, - ].filter((value): value is number => value !== undefined) - const score = mean(components) - return { - score, - passed: score >= 1, - dimensions: { - claim_recall: required.recall, - citation_recall: citation, - hallucination_safe: forbidden.safe, - forbidden_claim_rate: forbidden.rate, - required_claim_count: required.total, - matched_claim_count: required.matched, - forbidden_claim_count: forbidden.total, - matched_forbidden_claim_count: forbidden.matched, - }, - notes: `required=${required.matched}/${required.total}; forbidden=${forbidden.matched}/${forbidden.total}; citation_recall=${citation.toFixed(3)}`, - raw: { - matchedRequiredClaimIds: required.matchedIds, - matchedForbiddenClaimIds: forbidden.matchedIds, - }, - } -} - -export function scoreMemoryBenchmarkArtifact( - testCase: KnowledgeMemoryBenchmarkCase, - artifact: TArtifact, -): KnowledgeBenchmarkEvaluation { - const memoryArtifact = artifact as KnowledgeBenchmarkArtifact - const text = [ - memoryArtifact.text, - memoryArtifact.answer, - ...(memoryArtifact.rememberedFacts ?? []), - ] - .filter((part): part is string => typeof part === 'string' && part.length > 0) - .join('\n') - const required = scoreClaims(text, testCase.requiredFacts ?? []) - const forbidden = scoreForbiddenClaims(text, testCase.forbiddenFacts ?? []) - const eventIds = unique([ - ...(memoryArtifact.citedEventIds ?? []), - ...(memoryArtifact.usedMemoryIds ?? []), - ]) - const eventRecall = scoreCitationRecall(eventIds, testCase.expectedEventIds ?? []) - const actorRecall = scoreCitationRecall( - memoryArtifact.actorIds ?? [], - testCase.expectedActorIds ?? [], - ) - const components = [ - required.totalWeight > 0 ? required.recall : undefined, - testCase.expectedEventIds && testCase.expectedEventIds.length > 0 ? eventRecall : undefined, - testCase.expectedActorIds && testCase.expectedActorIds.length > 0 ? actorRecall : undefined, - testCase.forbiddenFacts && testCase.forbiddenFacts.length > 0 ? forbidden.safe : undefined, - ].filter((value): value is number => value !== undefined) - const score = mean(components) - const dimensions: Record = {} - if (required.totalWeight > 0) { - dimensions.memory_fact_recall = required.recall - dimensions.memory_required_fact_count = required.total - dimensions.memory_matched_fact_count = required.matched - } - if (testCase.expectedEventIds && testCase.expectedEventIds.length > 0) { - dimensions.memory_event_recall = eventRecall - } - if (testCase.expectedActorIds && testCase.expectedActorIds.length > 0) { - dimensions.memory_actor_recall = actorRecall - } - if (testCase.forbiddenFacts && testCase.forbiddenFacts.length > 0) { - dimensions.memory_stale_safe = forbidden.safe - dimensions.memory_stale_rate = forbidden.rate - dimensions.memory_forbidden_fact_count = forbidden.total - dimensions.memory_matched_forbidden_fact_count = forbidden.matched - } - return { - score, - passed: score >= 1, - dimensions, - applicableDimensions: Object.keys(dimensions), - notes: `memory required=${required.matched}/${required.total}; stale=${forbidden.matched}/${forbidden.total}; event_recall=${eventRecall.toFixed(3)}; actor_recall=${actorRecall.toFixed(3)}`, - raw: { - matchedRequiredFactIds: required.matchedIds, - matchedForbiddenFactIds: forbidden.matchedIds, - citedEventIds: eventIds, - actorIds: memoryArtifact.actorIds ?? [], - }, - } -} - -function retrievalScenarioForCase( - testCase: KnowledgeRetrievalBenchmarkCase, -): RetrievalEvalScenario { - return { - id: testCase.id, - kind: 'retrieval-eval', - query: testCase.query, - expected: testCase.expected, - ...(testCase.k !== undefined ? { k: testCase.k } : {}), - } -} - -function normalizeRetrievalArtifact( - testCase: KnowledgeRetrievalBenchmarkCase, - artifact: TArtifact, -): RetrievalEvalArtifact { - const maybe = artifact as Partial & KnowledgeBenchmarkArtifact - const hits = maybe.hits ?? [] - if (Array.isArray(maybe.hits) && maybe.query && maybe.requestedK !== undefined) { - return maybe as RetrievalEvalArtifact - } - return { - config: {}, - query: testCase.query, - requestedK: testCase.k ?? Math.max(1, hits.length), - hits, - durationMs: maybe.durationMs ?? 0, - ...(maybe.costUsd !== undefined ? { costUsd: maybe.costUsd } : {}), - ...(maybe.metadata ? { metadata: maybe.metadata } : {}), - } -} - -function scoreClaims(text: string, claims: readonly KnowledgeClaimMatcher[]) { - let matched = 0 - let matchedWeight = 0 - let totalWeight = 0 - const matchedIds: string[] = [] - const haystack = text.toLowerCase() - for (const claim of claims) { - if ( - !claim.id.trim() || - claim.anyOf.length === 0 || - claim.anyOf.some((value) => !value.trim()) - ) { - throw new Error( - 'claim matchers require a non-empty id and at least one non-empty alternative', - ) - } - const weight = claim.weight ?? 1 - if (!Number.isFinite(weight) || weight <= 0) { - throw new Error(`claim matcher ${claim.id} weight must be a positive finite number`) - } - totalWeight += weight - if (claim.anyOf.some((fragment) => haystack.includes(fragment.toLowerCase()))) { - matched += 1 - matchedWeight += weight - matchedIds.push(claim.id) - } - } - return { - total: claims.length, - matched, - totalWeight, - recall: totalWeight === 0 ? 1 : matchedWeight / totalWeight, - matchedIds, - } -} - -function scoreForbiddenClaims(text: string, claims: readonly KnowledgeClaimMatcher[]) { - const matched = scoreClaims(text, claims) - return { - total: claims.length, - matched: matched.matched, - matchedIds: matched.matchedIds, - rate: claims.length === 0 ? 0 : matched.matched / claims.length, - safe: matched.matched === 0 ? 1 : 0, - } -} - -function scoreCitationRecall( - citedSourceIds: readonly string[], - expectedSourceIds: readonly string[], -): number { - if (expectedSourceIds.length === 0) return 1 - const cited = new Set(citedSourceIds) - const matched = expectedSourceIds.filter((sourceId) => cited.has(sourceId)).length - return matched / expectedSourceIds.length -} diff --git a/src/benchmarks/suite.ts b/src/benchmarks/suite.ts deleted file mode 100644 index 15f7f82..0000000 --- a/src/benchmarks/suite.ts +++ /dev/null @@ -1,180 +0,0 @@ -import { join } from 'node:path' - -import { canonicalJson } from '@tangle-network/agent-eval' - -import { - fsCampaignStorage, - type JudgeConfig, - type RunCampaignOptions, - runCampaign, -} from '@tangle-network/agent-eval/campaign' - -import { stableId } from '../ids' - -import { renderSliceTable, summarizeKnowledgeBenchmarkCampaign } from './metrics' - -import { scoreKnowledgeBenchmarkArtifact } from './scoring' - -import type { - KnowledgeBenchmarkArtifact, - KnowledgeBenchmarkCase, - KnowledgeBenchmarkReport, - KnowledgeBenchmarkScenario, - KnowledgeBenchmarkSplit, - RunKnowledgeBenchmarkSuiteOptions, - RunKnowledgeBenchmarkSuiteResult, -} from './types' - -import { compactObject, formatNumber, unique } from './utils' - -import { assertKnowledgeBenchmarkCases, assertNonEmptyBenchmarkString } from './validation' - -const KNOWLEDGE_BENCHMARK_IMPLEMENTATION_REF = 'agent-knowledge:benchmark-suite:v2' - -export async function runKnowledgeBenchmarkSuite( - options: RunKnowledgeBenchmarkSuiteOptions, -): Promise> { - assertKnowledgeBenchmarkCases(options.cases) - if (options.respondRef !== undefined) { - assertNonEmptyBenchmarkString(options.respondRef, 'knowledge benchmark respondRef') - } else if (options.resumable !== false) { - throw new Error('knowledge benchmark respondRef is required when resumable is enabled') - } - const storage = options.storage ?? fsCampaignStorage() - const costCeiling = options.costCeiling ?? options.costLedger?.costCeilingUsd ?? 0 - if (options.costLedger && options.costLedger.costCeilingUsd !== costCeiling) { - throw new Error('knowledge benchmark costCeiling must match the shared cost ledger ceiling') - } - const scenarios = buildKnowledgeBenchmarkScenarios(options.cases, options.splits) - const dispatch: RunCampaignOptions['dispatch'] = async ( - scenario, - context, - ) => { - const artifact = await options.respond({ case: scenario.case, scenario, context }) - return artifact - } - const campaign = await runCampaign({ - scenarios, - dispatch, - dispatchRef: stableId( - 'knowledge_benchmark', - canonicalJson({ - implementationRef: KNOWLEDGE_BENCHMARK_IMPLEMENTATION_REF, - respondRef: options.respondRef ?? 'non-resumable', - }), - ), - judges: [knowledgeBenchmarkJudge()], - runDir: options.runDir, - repo: options.repo, - seed: options.seed, - reps: options.reps, - resumable: options.resumable, - costCeiling, - costLedger: options.costLedger, - costPhase: options.costPhase, - maxConcurrency: options.maxConcurrency, - dispatchTimeoutMs: options.dispatchTimeoutMs, - expectUsage: options.expectUsage ?? 'off', - storage, - now: options.now, - }) - const report = summarizeKnowledgeBenchmarkCampaign({ scenarios, campaign }) - const reportJsonPath = join(campaign.runDir, 'knowledge-benchmark-report.json') - const reportMarkdownPath = join(campaign.runDir, 'knowledge-benchmark-report.md') - storage.write(reportJsonPath, `${JSON.stringify(report, null, 2)}\n`) - storage.write(reportMarkdownPath, renderKnowledgeBenchmarkReportMarkdown(report)) - return { - scenarios, - campaign, - report, - reportJsonPath, - reportMarkdownPath, - } -} - -export function renderKnowledgeBenchmarkReportMarkdown(report: KnowledgeBenchmarkReport): string { - return [ - '# Knowledge Benchmark Report', - '', - `- cases: ${report.totalCases}`, - `- cells: ${report.totalCells} total, ${report.cellsFailed} failed, ${report.cellsCached} cached`, - `- cost: $${formatNumber(report.totalCostUsd)}`, - `- score: mean ${formatNumber(report.score.mean)}, median ${formatNumber(report.score.median)}, p90 ${formatNumber(report.score.p90)}, n=${report.score.n}`, - '', - '## Task Kinds', - '', - renderSliceTable(report.byTaskKind), - '', - '## Splits', - '', - renderSliceTable(report.bySplit), - '', - '## Dimensions', - '', - '| dimension | n | mean | p90 |', - '| --- | ---: | ---: | ---: |', - ...Object.entries(report.dimensions) - .sort(([a], [b]) => a.localeCompare(b)) - .map( - ([key, dist]) => - `| ${key} | ${dist.n} | ${formatNumber(dist.mean)} | ${formatNumber(dist.p90)} |`, - ), - '', - ].join('\n') -} - -export function buildKnowledgeBenchmarkScenarios( - cases: readonly KnowledgeBenchmarkCase[], - splits?: readonly KnowledgeBenchmarkSplit[], -): KnowledgeBenchmarkScenario[] { - const splitSet = splits ? new Set(splits) : null - return cases.flatMap((testCase) => { - const splitTag = testCase.split ?? 'dev' - if (splitSet && !splitSet.has(splitTag)) return [] - return [ - compactObject({ - id: testCase.id, - kind: 'knowledge-benchmark' as const, - family: testCase.family, - taskKind: testCase.taskKind, - splitTag, - tags: unique([splitTag, ...(testCase.tags ?? [])]), - case: compactObject(testCase), - }) as KnowledgeBenchmarkScenario, - ] - }) -} - -export function knowledgeBenchmarkJudge(): JudgeConfig< - TArtifact, - KnowledgeBenchmarkScenario -> { - return { - name: 'knowledge-benchmark', - judgeVersion: 'agent-knowledge:knowledge-benchmark:v2', - dimensions: [ - { key: 'score', description: 'primary knowledge benchmark score' }, - { key: 'passed', description: '1 when the benchmark case passes' }, - { key: 'claim_recall', description: 'required claim coverage' }, - { key: 'citation_recall', description: 'expected citation/source coverage' }, - { key: 'hallucination_safe', description: '1 when no forbidden claim appears' }, - { key: 'memory_fact_recall', description: 'current memory fact coverage' }, - { key: 'memory_event_recall', description: 'expected memory event/source coverage' }, - { key: 'memory_stale_safe', description: '1 when obsolete memory is not reused' }, - { key: 'memory_actor_recall', description: 'expected speaker/user attribution coverage' }, - ], - appliesTo: (scenario) => scenario.kind === 'knowledge-benchmark', - score({ artifact, scenario }) { - const evaluation = scoreKnowledgeBenchmarkArtifact(scenario.case, artifact) - return { - dimensions: { - score: evaluation.score, - passed: evaluation.passed ? 1 : 0, - ...evaluation.dimensions, - }, - composite: evaluation.score, - notes: evaluation.notes, - } - }, - } -} diff --git a/src/benchmarks/types.ts b/src/benchmarks/types.ts deleted file mode 100644 index 871c87d..0000000 --- a/src/benchmarks/types.ts +++ /dev/null @@ -1,347 +0,0 @@ -import type { - CampaignResult, - CampaignStorage, - CostLedgerHandle, - DispatchContext, - Scenario, -} from '@tangle-network/agent-eval/campaign' - -import type { AgentMemoryAcquireRunLease, AgentMemoryControllerMode } from '../memory/run-control' - -import type { AgentMemoryAdapter, AgentMemoryScope } from '../memory/types' - -import type { RetrievalGoldTarget, RetrievedKnowledgeHit } from '../retrieval-eval' - -export type KnowledgeBenchmarkTaskKind = - | 'retrieval' - | 'rag-answer' - | 'hallucination' - | 'kb-improvement' - | 'memory-ingest' - | 'memory-recall' - | 'memory-temporal' - | 'memory-update' - | 'memory-forgetting' - | 'memory-reasoning' - | 'memory-summarization' - | 'memory-recommendation' - | 'memory-multiparty' - -export type KnowledgeAnswerBenchmarkTaskKind = 'rag-answer' | 'hallucination' | 'kb-improvement' - -export type KnowledgeMemoryBenchmarkTaskKind = Exclude< - KnowledgeBenchmarkTaskKind, - 'retrieval' | KnowledgeAnswerBenchmarkTaskKind -> - -export type KnowledgeBenchmarkFamily = - | 'beir' - | 'mteb-retrieval' - | 'msmarco' - | 'trec-dl' - | 'miracl' - | 'lotte' - | 'bright' - | 'crag' - | 'hotpotqa' - | 'kilt' - | 'ragtruth' - | 'faithbench' - | 'locomo' - | 'longmemeval' - | 'longmemeval-v2' - | 'memora' - | 'memoryagentbench' - | 'memorybank' - | 'groupmembench' - | 'first-party' - | 'custom' - -export type KnowledgeBenchmarkSplit = 'search' | 'dev' | 'holdout' | string - -export interface KnowledgeBenchmarkSource { - name?: string - url?: string - version?: string - license?: string - citation?: string -} - -export interface KnowledgeBenchmarkSpec { - id: string - family: KnowledgeBenchmarkFamily - taskKind: KnowledgeBenchmarkTaskKind - primaryMetrics: readonly string[] - adapter: string - notes: string -} - -export interface KnowledgeBenchmarkCaseBase { - id: string - family: KnowledgeBenchmarkFamily | string - taskKind: KnowledgeBenchmarkTaskKind - split?: KnowledgeBenchmarkSplit - tags?: readonly string[] - source?: KnowledgeBenchmarkSource - metadata?: Record -} - -export interface KnowledgeRetrievalBenchmarkCase extends KnowledgeBenchmarkCaseBase { - taskKind: 'retrieval' - query: string - expected: RetrievalGoldTarget | readonly RetrievalGoldTarget[] - k?: number -} - -export interface KnowledgeClaimMatcher { - id: string - anyOf: readonly string[] - weight?: number -} - -export interface KnowledgeMemoryEvent { - id: string - text: string - actorId?: string - sessionId?: string - timestamp?: string - metadata?: Record -} - -export interface KnowledgeMemoryFactMatcher extends KnowledgeClaimMatcher { - sourceEventIds?: readonly string[] - validAt?: string - obsolete?: boolean -} - -export interface KnowledgeAnswerBenchmarkCase extends KnowledgeBenchmarkCaseBase { - taskKind: KnowledgeAnswerBenchmarkTaskKind - prompt: string - requiredClaims?: readonly KnowledgeClaimMatcher[] - forbiddenClaims?: readonly KnowledgeClaimMatcher[] - expectedSourceIds?: readonly string[] - referenceAnswer?: string -} - -export interface KnowledgeMemoryBenchmarkCase extends KnowledgeBenchmarkCaseBase { - taskKind: KnowledgeMemoryBenchmarkTaskKind - events: readonly KnowledgeMemoryEvent[] - prompt: string - requiredFacts?: readonly KnowledgeMemoryFactMatcher[] - forbiddenFacts?: readonly KnowledgeMemoryFactMatcher[] - expectedEventIds?: readonly string[] - expectedActorIds?: readonly string[] - referenceAnswer?: string -} - -export type KnowledgeBenchmarkCase = - | KnowledgeRetrievalBenchmarkCase - | KnowledgeAnswerBenchmarkCase - | KnowledgeMemoryBenchmarkCase - -export interface KnowledgeBenchmarkArtifact { - answer?: string - text?: string - hits?: readonly RetrievedKnowledgeHit[] - citedSourceIds?: readonly string[] - rememberedFacts?: readonly string[] - citedEventIds?: readonly string[] - usedMemoryIds?: readonly string[] - actorIds?: readonly string[] - /** Informational copy. Billable responders account through context.cost.runPaidCall. */ - costUsd?: number - durationMs?: number - metadata?: Record -} - -export interface KnowledgeBenchmarkEvaluation { - score: number - passed: boolean - dimensions: Record - /** Dimensions for which this case declared an actual target. */ - applicableDimensions?: readonly string[] - notes: string - raw: Record -} - -export interface KnowledgeBenchmarkScenario extends Scenario { - kind: 'knowledge-benchmark' - family: KnowledgeBenchmarkFamily | string - taskKind: KnowledgeBenchmarkTaskKind - splitTag: KnowledgeBenchmarkSplit - case: KnowledgeBenchmarkCase -} - -export type KnowledgeBenchmarkResponder = (input: { - case: KnowledgeBenchmarkCase - scenario: KnowledgeBenchmarkScenario - context: DispatchContext -}) => Promise | TArtifact - -export interface RunKnowledgeBenchmarkSuiteOptions { - cases: readonly KnowledgeBenchmarkCase[] - respond: KnowledgeBenchmarkResponder - /** Versioned identity for the model, prompt, retrieval, and runtime behavior. */ - respondRef?: string - runDir: string - splits?: readonly KnowledgeBenchmarkSplit[] - repo?: string - seed?: number - reps?: number - resumable?: boolean - costCeiling?: number - /** Shared across nested benchmark suites when an outer run owns spend. */ - costLedger?: CostLedgerHandle - costPhase?: string - maxConcurrency?: number - dispatchTimeoutMs?: number - expectUsage?: 'assert' | 'warn' | 'off' - storage?: CampaignStorage - now?: () => Date -} - -export interface KnowledgeBenchmarkDistribution { - n: number - min: number - mean: number - median: number - p90: number - max: number -} - -export interface KnowledgeBenchmarkSliceSummary { - n: number - meanScore: number - passRate: number - score: KnowledgeBenchmarkDistribution -} - -export interface KnowledgeBenchmarkReport { - totalCases: number - totalCells: number - cellsFailed: number - cellsCached: number - totalCostUsd: number - bySplit: Record - byFamily: Record - byTaskKind: Record - dimensions: Record - score: KnowledgeBenchmarkDistribution -} - -export interface RunKnowledgeBenchmarkSuiteResult { - scenarios: readonly KnowledgeBenchmarkScenario[] - campaign: CampaignResult - report: KnowledgeBenchmarkReport - reportJsonPath: string - reportMarkdownPath: string -} - -export interface MemoryAdapterBenchmarkCandidate { - id: string - /** Versioned adapter and configuration identity used by resumable caches. */ - ref: string - /** Expected adapter.id. Defaults to candidate id and permits lazy no-work resume. */ - adapterId?: string - label?: string - /** Local construction is free; call markExternalCall before billable provisioning or reconnects. */ - createAdapter: (input: { - purpose: 'execute' | 'recovery' - signal: AbortSignal - markExternalCall(): void - }) => AgentMemoryAdapter | Promise - /** Conservative charge for one billable adapter provisioning or reconnect call. */ - adapterCreationCostUsd?: number - searchLimit?: number - costUsdPerCase?: number - /** Conservative extra provider charge for recovering one interrupted case. */ - recoveryCostUsdPerAttempt?: number - scope?: AgentMemoryScope -} - -export interface RunMemoryAdapterBenchmarkOptions { - cases: readonly KnowledgeMemoryBenchmarkCase[] - candidates: readonly MemoryAdapterBenchmarkCandidate[] - /** Retired candidates retained only so interrupted scopes can be cleaned on resume. */ - recoveryCandidates?: readonly MemoryAdapterBenchmarkCandidate[] - runDir: string - storage?: CampaignStorage - repo?: string - seed?: number - reps?: number - resumable?: boolean - costCeiling?: number - /** Shared with nested benchmark suites so the dollar limit applies to the whole comparison. */ - costLedger?: CostLedgerHandle - costPhase?: string - maxConcurrency?: number - dispatchTimeoutMs?: number - cleanupTimeoutMs?: number - /** Refuse a damaged run with more unfinished attempts than this. Default 1000. */ - maxRecoveryAttempts?: number - /** Bound repeated provider cleanup after process crashes. Default 3 per attempt. */ - maxRecoveryRetriesPerAttempt?: number - expectUsage?: 'assert' | 'warn' | 'off' - now?: () => Date - /** Required with custom storage when all controllers are confined to one process. */ - controllerMode?: AgentMemoryControllerMode - /** Required for distributed controllers that share custom storage. */ - acquireRunLease?: AgentMemoryAcquireRunLease -} - -export interface MemoryAdapterBenchmarkRankingRow { - rank: number - candidateId: string - label: string - adapterId: string - scoreMean: number - passRate: number - totalCases: number - totalCells: number - cellsFailed: number - totalCostUsd: number - reportJsonPath: string - reportMarkdownPath: string - report: KnowledgeBenchmarkReport -} - -export interface RunMemoryAdapterBenchmarkResult { - rows: readonly MemoryAdapterBenchmarkRankingRow[] - totalCostUsd: number - /** Recovery spend for retired candidates, excluded from ranking rows but included in totalCostUsd. */ - unrankedRecoveryCostUsd: number - rankingJsonPath: string - rankingMarkdownPath: string - attemptLogPath: string - recoveryLogPath: string -} - -export interface KnowledgeRetrievalBenchmarkQuery { - id: string - text: string - split?: KnowledgeBenchmarkSplit - tags?: readonly string[] - metadata?: Record -} - -export interface KnowledgeRetrievalBenchmarkQrel { - queryId: string - documentId: string - score: number -} - -export interface BuildRetrievalBenchmarkCasesFromQrelsOptions { - benchmarkId: string - family: KnowledgeBenchmarkFamily | string - queries: readonly KnowledgeRetrievalBenchmarkQuery[] - qrels: readonly KnowledgeRetrievalBenchmarkQrel[] - source?: KnowledgeBenchmarkSource - tags?: readonly string[] - k?: number - targetKind?: 'page' | 'page-path' | 'source' - documentTarget?: ( - documentId: string, - qrel: KnowledgeRetrievalBenchmarkQrel, - ) => RetrievalGoldTarget - splitOf?: (queryId: string) => KnowledgeBenchmarkSplit -} diff --git a/src/benchmarks/utils.ts b/src/benchmarks/utils.ts deleted file mode 100644 index d9bfe37..0000000 --- a/src/benchmarks/utils.ts +++ /dev/null @@ -1,21 +0,0 @@ -export { normalizeUsd } from '../candidate-ranking' -export { mean } from '../statistics' - -export function unique(values: readonly string[]): string[] { - return [...new Set(values.filter(Boolean))] -} - -export function formatNumber(value: number): string { - if (!Number.isFinite(value)) return '0' - return value.toFixed(value === 0 || Math.abs(value) >= 10 ? 0 : 3) -} - -export function compactObject(value: unknown): unknown { - if (Array.isArray(value)) return value.map(compactObject) - if (!value || typeof value !== 'object') return value - return Object.fromEntries( - Object.entries(value as Record) - .filter(([, entry]) => entry !== undefined) - .map(([key, entry]) => [key, compactObject(entry)]), - ) -} diff --git a/src/benchmarks/validation.ts b/src/benchmarks/validation.ts deleted file mode 100644 index 6d2f0ff..0000000 --- a/src/benchmarks/validation.ts +++ /dev/null @@ -1,83 +0,0 @@ -import type { - KnowledgeBenchmarkCase, - KnowledgeClaimMatcher, - KnowledgeMemoryBenchmarkCase, - KnowledgeMemoryFactMatcher, -} from './types' - -export function isKnowledgeMemoryBenchmarkCase( - testCase: KnowledgeBenchmarkCase, -): testCase is KnowledgeMemoryBenchmarkCase { - return testCase.taskKind.startsWith('memory-') -} - -export function assertKnowledgeBenchmarkCases(cases: readonly KnowledgeBenchmarkCase[]): void { - if (cases.length === 0) throw new Error('knowledge benchmark requires cases') - assertUniqueNonEmptyStrings( - cases.map((testCase) => testCase.id), - 'knowledge benchmark case id', - ) - for (const testCase of cases) { - if (typeof testCase.family !== 'string' || !testCase.family.trim()) { - throw new Error(`knowledge benchmark case ${testCase.id} requires a family`) - } - if (testCase.taskKind === 'retrieval') continue - if (isKnowledgeMemoryBenchmarkCase(testCase)) { - assertUniqueNonEmptyStrings( - testCase.events.map((event) => event.id), - `${testCase.id} memory event id`, - ) - for (const event of testCase.events) { - assertNonEmptyBenchmarkString(event.text, `${testCase.id} memory event ${event.id} text`) - } - assertClaimMatchers(testCase.requiredFacts ?? [], `${testCase.id} requiredFacts`) - assertClaimMatchers(testCase.forbiddenFacts ?? [], `${testCase.id} forbiddenFacts`) - assertUniqueNonEmptyStrings( - testCase.expectedEventIds ?? [], - `${testCase.id} expected event id`, - ) - assertUniqueNonEmptyStrings( - testCase.expectedActorIds ?? [], - `${testCase.id} expected actor id`, - ) - } else { - assertClaimMatchers(testCase.requiredClaims ?? [], `${testCase.id} requiredClaims`) - assertClaimMatchers(testCase.forbiddenClaims ?? [], `${testCase.id} forbiddenClaims`) - assertUniqueNonEmptyStrings(testCase.expectedSourceIds ?? [], `${testCase.id} source id`) - } - } -} - -function assertClaimMatchers(claims: readonly KnowledgeClaimMatcher[], label: string): void { - assertUniqueNonEmptyStrings( - claims.map((claim) => claim.id), - `${label} matcher id`, - ) - for (const claim of claims) { - if (claim.anyOf.length === 0) throw new Error(`${label} matcher ${claim.id} requires anyOf`) - assertUniqueNonEmptyStrings(claim.anyOf, `${label} matcher ${claim.id} anyOf`) - if (claim.weight !== undefined && (!Number.isFinite(claim.weight) || claim.weight <= 0)) { - throw new Error(`${label} matcher ${claim.id} weight must be a positive finite number`) - } - const sourceEventIds = (claim as Partial).sourceEventIds - if (sourceEventIds !== undefined) { - assertUniqueNonEmptyStrings(sourceEventIds, `${label} matcher ${claim.id} source event id`) - } - } -} - -export function assertUniqueNonEmptyStrings(values: readonly string[], label: string): void { - const seen = new Set() - for (const value of values) { - assertNonEmptyBenchmarkString(value, label) - if (seen.has(value)) throw new Error(`duplicate ${label}: ${value}`) - seen.add(value) - } -} - -export function assertNonEmptyBenchmarkString( - value: unknown, - label: string, -): asserts value is string { - if (typeof value !== 'string' || !value.trim()) throw new Error(`${label} must be non-empty`) -} diff --git a/src/candidate-ranking.test.ts b/src/candidate-ranking.test.ts deleted file mode 100644 index 01af655..0000000 --- a/src/candidate-ranking.test.ts +++ /dev/null @@ -1,85 +0,0 @@ -import { describe, expect, it } from 'vitest' - -import { normalizeUsd, rankCandidates } from './candidate-ranking' - -const base = { scoreMean: 0.5, passRate: 0.5, cellsFailed: 0, totalCostUsd: 1 } - -function ordered(rows: Parameters[0]): string[] { - return rankCandidates(rows).map((row) => row.candidateId) -} - -describe('rankCandidates', () => { - it('places a candidate with a failed cell below every complete candidate', () => { - expect( - ordered([ - { ...base, candidateId: 'incomplete', scoreMean: 0.99, cellsFailed: 1 }, - { ...base, candidateId: 'complete', scoreMean: 0.1 }, - ]), - ).toEqual(['complete', 'incomplete']) - }) - - it('breaks each tie in turn: score, then pass rate, then cost, then id', () => { - expect( - ordered([ - { ...base, candidateId: 'lower-score', scoreMean: 0.4 }, - { ...base, candidateId: 'higher-score', scoreMean: 0.6 }, - ]), - ).toEqual(['higher-score', 'lower-score']) - - expect( - ordered([ - { ...base, candidateId: 'lower-pass', passRate: 0.4 }, - { ...base, candidateId: 'higher-pass', passRate: 0.6 }, - ]), - ).toEqual(['higher-pass', 'lower-pass']) - - expect( - ordered([ - { ...base, candidateId: 'dearer', totalCostUsd: 2 }, - { ...base, candidateId: 'cheaper', totalCostUsd: 1 }, - ]), - ).toEqual(['cheaper', 'dearer']) - - expect( - ordered([ - { ...base, candidateId: 'b' }, - { ...base, candidateId: 'a' }, - ]), - ).toEqual(['a', 'b']) - }) - - it('stamps ranks from one and leaves the caller rows untouched', () => { - const rows = [ - { ...base, candidateId: 'second', scoreMean: 0.2 }, - { ...base, candidateId: 'first', scoreMean: 0.8 }, - ] - expect(rankCandidates(rows).map((row) => row.rank)).toEqual([1, 2]) - expect(rows.map((row) => row.candidateId)).toEqual(['second', 'first']) - }) - - it('orders identically however the same results arrive', () => { - const rows = [ - { ...base, candidateId: 'a', scoreMean: 0.5 }, - { ...base, candidateId: 'b', scoreMean: 0.5 }, - { ...base, candidateId: 'c', scoreMean: 0.9 }, - ] - expect(ordered(rows)).toEqual(ordered([...rows].reverse())) - }) -}) - -describe('normalizeUsd', () => { - it('removes the float noise that would otherwise decide a cost tie', () => { - const accumulated = [0.1, 0.2].reduce((sum, value) => sum + value, 0) - expect(accumulated).not.toBe(0.3) - expect(normalizeUsd(accumulated)).toBe(normalizeUsd(0.3)) - }) - - it('keeps two candidates that spent the same amount in id order', () => { - const accumulated = [0.1, 0.2].reduce((sum, value) => sum + value, 0) - const ranked = rankCandidates([ - { ...base, candidateId: 'b', totalCostUsd: normalizeUsd(accumulated) }, - { ...base, candidateId: 'a', totalCostUsd: normalizeUsd(0.3) }, - ]) - expect(ranked.map((row) => row.candidateId)).toEqual(['a', 'b']) - }) -}) diff --git a/src/candidate-ranking.ts b/src/candidate-ranking.ts deleted file mode 100644 index 802d6a5..0000000 --- a/src/candidate-ranking.ts +++ /dev/null @@ -1,43 +0,0 @@ -/** The measured facts that decide where a candidate places against its peers. */ -export interface RankedCandidateFacts { - candidateId: string - scoreMean: number - passRate: number - cellsFailed: number - totalCostUsd: number -} - -/** - * Round a US-dollar amount to the precision a cost ledger records. - * - * Costs are accumulated by addition, so two runs that spent the same amount - * can differ in the last bits of the float. Ranking breaks a tie on cost, and - * without a common precision that float noise decides the order. - */ -export function normalizeUsd(value: number): number { - return Number(value.toFixed(12)) -} - -/** - * Order candidates best-first and stamp each row with the rank it earned. - * - * A candidate with a failed cell measured less than it claims to have - * measured, so it places below every candidate that completed. Among those, - * the higher mean score wins, then the higher pass rate, then the lower cost. - * The candidate id breaks the final tie, so ranking the same results twice - * gives the same order. - */ -export function rankCandidates( - rows: readonly Row[], -): (Row & { rank: number })[] { - return [...rows] - .sort( - (a, b) => - Number(a.cellsFailed > 0) - Number(b.cellsFailed > 0) || - b.scoreMean - a.scoreMean || - b.passRate - a.passRate || - a.totalCostUsd - b.totalCostUsd || - a.candidateId.localeCompare(b.candidateId), - ) - .map((row, index) => ({ ...row, rank: index + 1 })) -} diff --git a/src/memory/attempt-log.ts b/src/memory/attempt-log.ts deleted file mode 100644 index 9f69384..0000000 --- a/src/memory/attempt-log.ts +++ /dev/null @@ -1,287 +0,0 @@ -import type { CampaignStorage, CostLedgerHandle } from '@tangle-network/agent-eval/campaign' - -export interface AttemptJournalEvent { - status: 'started' | 'cleaned' - recordedAt: string -} - -interface RecoveryAttemptEvent { - attemptId: string - generation: number - recordedAt: string -} - -const journalByteLengths = new WeakMap>() - -export const DEFAULT_MEMORY_RECOVERY_RETRIES_PER_ATTEMPT = 3 - -const MEMORY_PROVIDER_ACTOR_PREFIXES = [ - 'agent-knowledge:memory-adapter:', - 'agent-knowledge:memory-adapter-recovery:', - 'agent-knowledge:memory-experiment:', - 'agent-knowledge:memory-recovery:', -] as const - -export function reconcileInterruptedMemoryPaidCalls(costLedger: CostLedgerHandle): void { - reconcileInterruptedPaidCallsAtMaximum( - costLedger, - (actor) => isMemoryProviderActor(actor), - 'external memory provider', - ) -} - -export function reconcileInterruptedRunPaidCalls( - costLedger: CostLedgerHandle, - label: string, -): void { - reconcileInterruptedPaidCallsAtMaximum(costLedger, () => true, label) -} - -function reconcileInterruptedPaidCallsAtMaximum( - costLedger: CostLedgerHandle, - ownsActor: (actor: string) => boolean, - label: string, -): void { - if (costLedger.summary().unresolvedCalls === 0) return - const pending = requirePendingCostCallInspection(costLedger) - for (const call of pending) { - if (call.state !== 'interrupted' || !ownsActor(call.actor)) continue - if (call.maximumCostUsd === undefined) { - throw new Error( - `cannot recover unbounded interrupted ${label} call '${call.callId}' for '${call.actor}'`, - ) - } - try { - costLedger.reconcile( - call.callId, - { - model: call.model, - inputTokens: 0, - outputTokens: 0, - actualCostUsd: call.maximumCostUsd, - }, - { - error: `process exited before the ${label} receipt for '${call.actor}' was recorded; charged the reserved maximum`, - }, - ) - } catch (error) { - if (isMissingPendingCostCall(error, call.callId)) continue - throw error - } - } -} - -export function assertNoInterruptedPaidCalls(costLedger: CostLedgerHandle, label: string): void { - const unresolved = costLedger.summary().unresolvedCalls - if (unresolved === 0) return - const pending = requirePendingCostCallInspection(costLedger) - const blocked = pending.filter((call) => call.state !== 'active') - throw new Error( - `${label} has ${unresolved} unresolved paid call(s) that cannot be resumed: ${blocked - .map((call) => `${call.callId} (${call.actor}, ${call.state})`) - .join(', ')}`, - ) -} - -export function hasSettledPaidCall(costLedger: CostLedgerHandle, callId: string): boolean { - return costLedger.list().some((receipt) => receipt.callId === callId) -} - -export function appendAttemptJournalEvent(input: { - storage: CampaignStorage - path: string - event: TEvent - label: string -}): void { - appendDurableJournalEvent(input) -} - -export function appendDurableJournalEvent(input: { - storage: CampaignStorage - path: string - event: TEvent - label: string -}): void { - const { storage, path, event, label } = input - if (!storage.append) throw new Error(`${label} requires CampaignStorage.append`) - const line = `${JSON.stringify(event)}\n` - const byteLengths = journalByteLengths.get(storage) ?? new Map() - journalByteLengths.set(storage, byteLengths) - for (let retry = 0; retry < 100; retry += 1) { - let expectedBytes = byteLengths.get(path) - if (expectedBytes === undefined) { - const current = storage.read(path) - if (current === undefined && storage.exists(path)) { - throw new Error(`cannot read ${label} '${path}'`) - } - expectedBytes = new TextEncoder().encode(current ?? '').byteLength - byteLengths.set(path, expectedBytes) - } - const appendedBytes = storage.append(path, line, expectedBytes) - if (appendedBytes !== undefined) { - byteLengths.set(path, appendedBytes) - return - } - byteLengths.delete(path) - } - throw new Error(`${label} '${path}' remained contended after 100 retries`) -} - -/** Reserves one durable recovery generation per attempt before provider work starts. */ -export function reserveRecoveryAttempts(input: { - storage: CampaignStorage - path: string - attemptIds: readonly string[] - maxRetriesPerAttempt: number - label: string - now?: () => Date -}): ReadonlyMap { - const { storage, path, attemptIds, maxRetriesPerAttempt, label } = input - if (!Number.isSafeInteger(maxRetriesPerAttempt) || maxRetriesPerAttempt <= 0) { - throw new Error(`${label} maxRetriesPerAttempt must be a positive safe integer`) - } - const uniqueAttemptIds = new Set(attemptIds) - if (uniqueAttemptIds.size !== attemptIds.length || attemptIds.some((id) => !id.trim())) { - throw new Error(`${label} attempt ids must be unique non-empty strings`) - } - const generations = readRecoveryAttemptGenerations(storage, path, label) - for (const attemptId of attemptIds) { - const prior = generations.get(attemptId) ?? 0 - if (prior >= maxRetriesPerAttempt) { - throw new Error( - `${label} '${attemptId}' exhausted ${maxRetriesPerAttempt} recovery attempts; increase maxRecoveryRetriesPerAttempt only after inspecting provider state`, - ) - } - } - const reserved = new Map() - for (const attemptId of attemptIds) { - const generation = (generations.get(attemptId) ?? 0) + 1 - appendDurableJournalEvent({ - storage, - path, - label, - event: { - attemptId, - generation, - recordedAt: (input.now ?? (() => new Date()))().toISOString(), - } satisfies RecoveryAttemptEvent, - }) - generations.set(attemptId, generation) - reserved.set(attemptId, generation) - } - return reserved -} - -export function readActiveAttemptJournal(input: { - storage: CampaignStorage - path: string - label: string - parse(value: unknown, path: string, line: number): TEvent - id(event: TEvent): string - sameAttempt(left: TEvent, right: TEvent): boolean -}): TEvent[] { - const { storage, path, label, parse, id, sameAttempt } = input - const stored = storage.read(path) - if (stored === undefined) { - if (storage.exists(path)) throw new Error(`cannot read ${label} '${path}'`) - return [] - } - const active = new Map() - const completed = new Map() - for (const [index, line] of stored.split('\n').entries()) { - if (!line) continue - let raw: unknown - try { - raw = JSON.parse(line) - } catch (error) { - throw new Error(`invalid ${label} '${path}' line ${index + 1}`, { cause: error }) - } - const event = parse(raw, path, index + 1) - const attemptId = id(event) - if (event.status === 'started') { - if (active.has(attemptId) || completed.has(attemptId)) { - throw new Error(`${label} '${path}' repeats attempt '${attemptId}'`) - } - active.set(attemptId, event) - continue - } - - const started = active.get(attemptId) - if (started && sameAttempt(started, event)) { - active.delete(attemptId) - completed.set(attemptId, started) - continue - } - - const prior = completed.get(attemptId) - if (prior && sameAttempt(prior, event)) continue - throw new Error(`${label} '${path}' has an unmatched cleanup for '${attemptId}'`) - } - return [...active.values()] -} - -function readRecoveryAttemptGenerations( - storage: CampaignStorage, - path: string, - label: string, -): Map { - const stored = storage.read(path) - if (stored === undefined) { - if (storage.exists(path)) throw new Error(`cannot read ${label} '${path}'`) - return new Map() - } - const generations = new Map() - for (const [index, line] of stored.split('\n').entries()) { - if (!line) continue - let raw: unknown - try { - raw = JSON.parse(line) - } catch (error) { - throw new Error(`invalid ${label} '${path}' line ${index + 1}`, { cause: error }) - } - const event = parseRecoveryAttemptEvent(raw, path, index + 1, label) - const expected = (generations.get(event.attemptId) ?? 0) + 1 - if (event.generation !== expected) { - throw new Error( - `${label} '${path}' has recovery generation ${event.generation} for '${event.attemptId}', expected ${expected}`, - ) - } - generations.set(event.attemptId, event.generation) - } - return generations -} - -function parseRecoveryAttemptEvent( - value: unknown, - path: string, - line: number, - label: string, -): RecoveryAttemptEvent { - const event = value as Partial | null - const valid = - typeof event === 'object' && - event !== null && - typeof event.attemptId === 'string' && - event.attemptId.length > 0 && - Number.isSafeInteger(event.generation) && - event.generation! > 0 && - typeof event.recordedAt === 'string' && - !Number.isNaN(Date.parse(event.recordedAt)) - if (!valid) throw new Error(`invalid ${label} '${path}' line ${line}`) - return value as RecoveryAttemptEvent -} - -function isMissingPendingCostCall(error: unknown, callId: string): boolean { - return error instanceof Error && error.message === `CostLedger: no pending call '${callId}'` -} - -function requirePendingCostCallInspection(costLedger: CostLedgerHandle) { - if (!costLedger.listPending) { - throw new Error('interrupted memory recovery requires CostLedger.listPending() support') - } - return costLedger.listPending() -} - -function isMemoryProviderActor(actor: string): boolean { - return MEMORY_PROVIDER_ACTOR_PREFIXES.some((prefix) => actor.startsWith(prefix)) -} diff --git a/src/memory/experiment.ts b/src/memory/experiment.ts deleted file mode 100644 index 1f01b85..0000000 --- a/src/memory/experiment.ts +++ /dev/null @@ -1,45 +0,0 @@ -export { - agentMemorySequenceJudge, - buildAgentMemorySequenceScenarios, - buildAgentMemorySequencesFromBenchmarkCases, -} from './experiment/cases' -export { - compareAgentMemoryLearning, - runAgentMemoryLearningExperiment, -} from './experiment/learning' -export { runAgentMemoryExperiment } from './experiment/run' -export type { - AgentMemoryAttemptEvent, - AgentMemoryEvidenceRef, - AgentMemoryExecutionContext, - AgentMemoryExecutionCostMeter, - AgentMemoryExecutionCostReceipt, - AgentMemoryExecutionPaidCallInput, - AgentMemoryExecutionPaidCallResult, - AgentMemoryExecutionStep, - AgentMemoryExperimentCandidate, - AgentMemoryExperimentCandidateRef, - AgentMemoryExperimentComparisonRef, - AgentMemoryExperimentRankingRow, - AgentMemoryExperimentRunLease, - AgentMemoryForgettingComparison, - AgentMemoryLearningArmOrder, - AgentMemoryLearningCandidateSummary, - AgentMemoryLearningCellComparison, - AgentMemoryLearningComparison, - AgentMemoryMode, - AgentMemorySequence, - AgentMemorySequenceArtifact, - AgentMemorySequenceProbe, - AgentMemorySequenceProbeResult, - AgentMemorySequenceScenario, - AgentMemorySequenceStep, - AgentMemoryTransferCellComparison, - AgentMemoryTransferStepSummary, - BuildAgentMemorySequencesFromBenchmarkCasesOptions, - CompareAgentMemoryLearningOptions, - RunAgentMemoryExperimentOptions, - RunAgentMemoryExperimentResult, - RunAgentMemoryLearningExperimentOptions, - RunAgentMemoryLearningExperimentResult, -} from './experiment/types' diff --git a/src/memory/experiment/cases.ts b/src/memory/experiment/cases.ts deleted file mode 100644 index fca04cb..0000000 --- a/src/memory/experiment/cases.ts +++ /dev/null @@ -1,132 +0,0 @@ -/** Converts existing ordered memory benchmark cases into executable histories. */ -import type { JudgeConfig, Scenario } from '@tangle-network/agent-eval/campaign' -import type { KnowledgeMemoryBenchmarkCase } from '../../benchmarks/index' -import { stableId } from '../../ids' -import type { - AgentMemoryExperimentCandidate, - AgentMemorySequence, - AgentMemorySequenceArtifact, - AgentMemorySequenceScenario, - BuildAgentMemorySequencesFromBenchmarkCasesOptions, -} from './types' -import { compactRecord, compactScope } from './validation' - -export function buildAgentMemorySequencesFromBenchmarkCases( - cases: readonly KnowledgeMemoryBenchmarkCase[], - options: BuildAgentMemorySequencesFromBenchmarkCasesOptions = {}, -): AgentMemorySequence[] { - const memoryAgentId = options.memoryAgentId ?? 'benchmark-agent' - return cases.map((testCase) => ({ - id: testCase.id, - family: testCase.family, - ...(testCase.split !== undefined ? { split: testCase.split } : {}), - ...(testCase.tags !== undefined ? { tags: testCase.tags } : {}), - metadata: compactRecord({ - ...(testCase.metadata ?? {}), - taskKind: testCase.taskKind, - source: testCase.source, - }), - steps: [ - ...testCase.events.map((event, eventIndex) => ({ - id: `event:${event.id}`, - scope: compactScope( - options.eventScope?.({ event, case: testCase, eventIndex }) ?? { - agentId: memoryAgentId, - sessionId: 'benchmark-session', - }, - ), - writes: [ - { - id: event.id, - kind: 'observation' as const, - text: event.text, - metadata: compactRecord({ - ...(event.metadata ?? {}), - eventId: event.id, - actorId: event.actorId, - sessionId: event.sessionId, - timestamp: event.timestamp, - }), - }, - ], - metadata: { eventIndex }, - })), - { - id: 'probe', - scope: compactScope( - options.probeScope?.(testCase) ?? { - agentId: memoryAgentId, - sessionId: 'benchmark-session', - }, - ), - probes: [ - { - id: 'answer', - query: testCase.prompt, - taskKind: testCase.taskKind, - ...(testCase.requiredFacts !== undefined - ? { requiredFacts: testCase.requiredFacts } - : {}), - ...(testCase.forbiddenFacts !== undefined - ? { forbiddenFacts: testCase.forbiddenFacts } - : {}), - ...(testCase.expectedEventIds !== undefined - ? { expectedEventIds: testCase.expectedEventIds } - : {}), - ...(testCase.expectedActorIds !== undefined - ? { expectedActorIds: testCase.expectedActorIds } - : {}), - ...(testCase.referenceAnswer !== undefined - ? { referenceAnswer: testCase.referenceAnswer } - : {}), - }, - ], - }, - ], - })) -} - -export function buildAgentMemorySequenceScenarios( - sequences: readonly AgentMemorySequence[], - candidates: readonly Pick[], -): AgentMemorySequenceScenario[] { - return candidates.flatMap((candidate) => - sequences.map((sequence) => ({ - id: `${stableId('candidate', candidate.id)}:${sequence.id}`, - kind: 'agent-memory-sequence' as const, - candidateId: candidate.id, - sequenceId: sequence.id, - sequence, - seedGroup: sequence.id, - tags: [...new Set([sequence.split ?? 'dev', ...(sequence.tags ?? []), candidate.id])], - })), - ) -} - -export function agentMemorySequenceJudge< - TScenario extends Scenario = AgentMemorySequenceScenario, ->(): JudgeConfig { - return { - name: 'agent-memory-sequence', - judgeVersion: 'agent-knowledge:memory-sequence:v2', - dimensions: [ - { key: 'score', description: 'mean memory probe score' }, - { key: 'passed', description: '1 when every memory probe passes' }, - { key: 'memory_fact_recall', description: 'current memory fact coverage' }, - { key: 'memory_event_recall', description: 'memory source event coverage' }, - { key: 'memory_actor_recall', description: 'memory actor attribution coverage' }, - { key: 'memory_stale_safe', description: '1 when obsolete memory is not reused' }, - ], - score({ artifact }) { - return { - composite: artifact.score, - dimensions: { - score: artifact.score, - passed: artifact.passed ? 1 : 0, - ...artifact.dimensions, - }, - notes: `${artifact.probes.filter((probe) => probe.passed).length}/${artifact.probes.length} probes passed`, - } - }, - } -} diff --git a/src/memory/experiment/cell.ts b/src/memory/experiment/cell.ts deleted file mode 100644 index e605e07..0000000 --- a/src/memory/experiment/cell.ts +++ /dev/null @@ -1,493 +0,0 @@ -import { randomUUID } from 'node:crypto' -import { - type CampaignStorage, - canonicalDigest, - type DispatchContext, -} from '@tangle-network/agent-eval/campaign' -import { stableId } from '../../ids' -import { type AgentMemoryBranch, createAgentMemoryBranch } from '../branch' -import { - createBoundedMemoryAdapter, - MEMORY_OPERATION_CANCELLATION_TIMEOUT_MS, - resolveMemoryCleanupTimeoutMs, - runBoundedMemoryLifecycle, -} from '../lifecycle' -import type { AgentMemoryAdapter } from '../types' -import { createAgentMemoryCostRecorder } from './cost' -import { createAgentMemoryExecutionContext } from './execution-context' -import { countDimensions, mean, meanDimensions } from './metrics' -import { type AgentMemoryProbeScoringEvidence, scoreAgentMemoryProbe } from './probe-evaluation' -import { appendMemoryAttemptEvent, memoryAttemptCostCallId, memoryAttemptEvent } from './recovery' -import { - AgentMemoryCleanupError, - clearSequenceScopes, - memoryExperimentBaseScope, - mergeScopes, - sequenceCleanupScopes, - trackExternalMemoryCalls, -} from './runtime' -import type { - AgentMemoryExecutionStep, - AgentMemoryExperimentCandidate, - AgentMemoryExperimentComparisonRef, - AgentMemoryMode, - AgentMemorySequence, - AgentMemorySequenceArtifact, - AgentMemorySequenceProbe, - AgentMemorySequenceProbeResult, - AgentMemorySequenceScenario, - AgentMemorySequenceStep, - OwnedMemoryExperimentRunLease, - RunAgentMemoryExperimentOptions, -} from './types' -import { stringArray, uniqueStrings } from './validation' - -export class AgentMemoryAdapterCapabilityError extends Error { - override readonly name = 'AgentMemoryAdapterCapabilityError' -} - -export async function runSequenceCell(input: { - options: RunAgentMemoryExperimentOptions - candidate: AgentMemoryExperimentCandidate - scenario: AgentMemorySequenceScenario - context: DispatchContext - runIdentity: string - storage: CampaignStorage - attemptLogPath: string - lease: OwnedMemoryExperimentRunLease - memoryMode: AgentMemoryMode - comparisonRef: AgentMemoryExperimentComparisonRef -}): Promise { - const { - options, - candidate, - scenario, - context, - runIdentity, - storage, - attemptLogPath, - lease, - memoryMode, - comparisonRef, - } = input - const cleanupBranches = options.cleanupBranches ?? true - const costUsd = candidate.externalCostUsdPerSequence ?? 0 - if (!Number.isFinite(costUsd) || costUsd < 0) { - throw new Error( - `${candidate.id}: externalCostUsdPerSequence must be a non-negative finite number`, - ) - } - if (!cleanupBranches && !candidate.disposeAdapter) { - throw new Error( - `${candidate.id}: cleanupBranches=false requires disposeAdapter to delete isolated external state`, - ) - } - const cleanupTimeoutMs = resolveMemoryCleanupTimeoutMs( - options.cleanupTimeoutMs, - `${candidate.id}: memory sequence`, - ) - context.signal.throwIfAborted() - await lease.assertOwned() - const startedAt = Date.now() - const branchId = stableId( - 'memory_branch', - `${runIdentity}:${context.cellId}:${context.seed}:${randomUUID()}`, - ) - const attempt = memoryAttemptEvent({ - status: 'started', - branchId, - candidate, - sequence: scenario.sequence, - rep: context.rep, - seed: context.seed, - recovery: false, - cleanupBranches, - now: options.now, - }) - await lease.assertOwned() - appendMemoryAttemptEvent(storage, attemptLogPath, attempt) - - let externalCallAttempted = false - const costRecorder = createAgentMemoryCostRecorder({ - candidateRef: candidate.ref, - maximumCostUsd: costUsd, - operation: `${candidate.id}: memory sequence`, - }) - const appendCleanedAttempt = (priorError?: unknown): void => { - try { - appendMemoryAttemptEvent(storage, attemptLogPath, { - ...attempt, - status: 'cleaned', - recordedAt: (options.now ?? (() => new Date()))().toISOString(), - }) - } catch (error) { - throw new AgentMemoryCleanupError( - [...(priorError ? [priorError] : []), error], - `${candidate.id}: memory branch cleanup could not be recorded`, - ) - } - } - const execute = async (): Promise => { - context.signal.throwIfAborted() - await lease.assertOwned() - let rawAdapter: AgentMemoryAdapter | undefined - let adapter: AgentMemoryAdapter | undefined - let memory: AgentMemoryBranch | undefined - let primaryError: unknown - let completedArtifact: AgentMemorySequenceArtifact | undefined - let finalClearStarted = false - let finalClearCompleted = false - try { - const created = await createBoundedMemoryAdapter({ - operation: `${candidate.id}: execution adapter creation`, - timeoutMs: cleanupTimeoutMs, - signal: context.signal, - create: (signal) => - candidate.createAdapter({ - branchId, - purpose: 'execute', - signal, - maximumCostUsd: costUsd, - markExternalCall: () => { - externalCallAttempted = true - }, - recordExternalCost: (actualCostUsd) => { - externalCallAttempted = true - costRecorder.record(actualCostUsd) - }, - }), - dispose: candidate.disposeAdapter - ? async (lateAdapter) => { - externalCallAttempted = true - await candidate.disposeAdapter?.(lateAdapter) - } - : undefined, - }) - if (!created) throw new Error(`${candidate.id}: createAdapter returned no execution adapter`) - rawAdapter = created - adapter = trackExternalMemoryCalls(created, () => { - externalCallAttempted = true - }) - await lease.assertOwned() - context.signal.throwIfAborted() - if (memoryMode === 'stateless' && !adapter.clear) { - throw new AgentMemoryAdapterCapabilityError( - `${candidate.id}: stateless memoryMode requires scoped clear support`, - ) - } - if (cleanupBranches && !adapter.clear) { - throw new Error( - `${candidate.id}: cleanupBranches requires an adapter with scoped clear support`, - ) - } - memory = createAgentMemoryBranch({ - adapter, - branchId, - lifetime: 'attempt', - policy: candidate.policy, - allowedWriteScopes: sequenceCleanupScopes(scenario.sequence), - baseScope: memoryExperimentBaseScope(candidate), - }) - const runProviderOperation = (operation: string, run: () => Promise): Promise => - runBoundedMemoryLifecycle({ - operation, - timeoutMs: cleanupTimeoutMs, - resource: adapter, - signal: context.signal, - cancellationTimeoutMs: MEMORY_OPERATION_CANCELLATION_TIMEOUT_MS, - run, - }) - const probes: AgentMemorySequenceProbeResult[] = [] - for (const [ordinal, step] of scenario.sequence.steps.entries()) { - context.signal.throwIfAborted() - await lease.assertOwned() - if (memoryMode === 'stateless' && ordinal > 0) { - await runBoundedMemoryLifecycle({ - operation: `${candidate.id}: stateless reset before step ${step.id}`, - timeoutMs: cleanupTimeoutMs, - resource: adapter, - run: () => clearSequenceScopes(memory!, scenario.sequence), - }) - context.signal.throwIfAborted() - await lease.assertOwned() - } - await runProviderOperation(`${candidate.id}: writes for step ${step.id}`, () => - writeStep(memory!, step), - ) - await lease.assertOwned() - if (options.executeStep) { - const execution = createAgentMemoryExecutionContext(context, scenario.sequence) - try { - await runProviderOperation(`${candidate.id}: execution for step ${step.id}`, () => - options.executeStep!({ - memory: memory!, - candidateId: candidate.id, - step: executionStep(step, ordinal), - context: execution.context, - }), - ) - } catch (error) { - execution.abort() - throw error - } finally { - execution.dispose() - } - } - context.signal.throwIfAborted() - await lease.assertOwned() - const stepProbes = await runProviderOperation( - `${candidate.id}: probes for step ${step.id}`, - () => probeStep(memory!, scenario.sequence, step, ordinal, context), - ) - await lease.assertOwned() - probes.push(...stepProbes) - } - const snapshot = await runProviderOperation(`${candidate.id}: branch snapshot`, () => - memory!.snapshot(), - ) - await lease.assertOwned() - if (options.onBranchSnapshot) { - await runBoundedMemoryLifecycle({ - operation: `${candidate.id}: branch snapshot callback`, - timeoutMs: cleanupTimeoutMs, - signal: context.signal, - cancellationTimeoutMs: MEMORY_OPERATION_CANCELLATION_TIMEOUT_MS, - run: () => - options.onBranchSnapshot!({ - candidateId: candidate.id, - sequenceId: scenario.sequenceId, - cellId: context.cellId, - snapshot, - }), - }) - } - await lease.assertOwned() - const dimensions = meanDimensions(probes.map((probe) => probe.dimensions)) - const dimensionSampleCounts = countDimensions( - probes.map((probe) => probe.applicableDimensions), - ) - const artifact: AgentMemorySequenceArtifact = { - candidateId: candidate.id, - sequenceId: scenario.sequenceId, - memoryMode, - comparisonRef, - score: mean(probes.map((probe) => probe.score)), - passed: probes.length > 0 && probes.every((probe) => probe.passed), - dimensions, - dimensionSampleCounts, - probes, - branchDigest: snapshot.digest, - journalEntries: snapshot.journal.length, - durationMs: Math.max(0, Date.now() - startedAt), - } - if (cleanupBranches) { - finalClearStarted = true - await runBoundedMemoryLifecycle({ - operation: `${candidate.id}: final branch cleanup`, - timeoutMs: cleanupTimeoutMs, - resource: adapter, - run: () => clearSequenceScopes(memory!, scenario.sequence), - }) - finalClearCompleted = true - await lease.assertOwned() - } - completedArtifact = artifact - } catch (error) { - primaryError = error - } - const cleanupErrors: unknown[] = [] - let cleanupOwned = true - let ownershipError: unknown - try { - await lease.assertOwned() - } catch (error) { - cleanupOwned = false - ownershipError = error - } - if (finalClearStarted && !finalClearCompleted && primaryError) { - cleanupErrors.push(primaryError) - } - if ( - primaryError && - cleanupOwned && - !finalClearStarted && - memory && - cleanupBranches && - adapter?.clear - ) { - try { - await runBoundedMemoryLifecycle({ - operation: `${candidate.id}: failed branch cleanup`, - timeoutMs: cleanupTimeoutMs, - resource: adapter, - run: () => clearSequenceScopes(memory!, scenario.sequence), - }) - } catch (error) { - cleanupErrors.push(error) - } - } - if (adapter) { - try { - await runBoundedMemoryLifecycle({ - operation: `${candidate.id}: adapter close`, - timeoutMs: cleanupTimeoutMs, - resource: adapter, - run: async () => { - if (memory && cleanupOwned) await memory.close?.() - else { - if (cleanupOwned) await adapter!.flush?.() - await adapter!.close?.() - } - }, - }) - } catch (error) { - cleanupErrors.push(error) - } - if (cleanupOwned) { - try { - await runBoundedMemoryLifecycle({ - operation: `${candidate.id}: adapter disposal`, - timeoutMs: cleanupTimeoutMs, - resource: adapter, - run: () => { - if (candidate.disposeAdapter) externalCallAttempted = true - return candidate.disposeAdapter?.(rawAdapter!) - }, - }) - } catch (error) { - cleanupErrors.push(error) - } - } - } else { - cleanupErrors.push( - new Error(`${candidate.id}: adapter creation failed before cleanup could be confirmed`), - ) - } - if (cleanupOwned && cleanupErrors.length === 0) { - try { - await lease.assertOwned() - } catch (error) { - cleanupOwned = false - ownershipError = error - } - } - if (cleanupOwned && cleanupErrors.length === 0) appendCleanedAttempt(primaryError) - if (!cleanupOwned && cleanupErrors.length === 0) { - if (primaryError) throw primaryError - throw ownershipError - } - if (cleanupErrors.length > 0) { - const primaryMessage = - primaryError instanceof Error ? primaryError.message : String(primaryError ?? '') - throw new AgentMemoryCleanupError( - [ - ...(primaryError && !cleanupErrors.includes(primaryError) ? [primaryError] : []), - ...(ownershipError ? [ownershipError] : []), - ...cleanupErrors, - ], - `${candidate.id}: memory branch cleanup failed${primaryMessage ? ` after: ${primaryMessage}` : ''}`, - ) - } - if (primaryError) throw primaryError - if (!completedArtifact) throw new Error(`${candidate.id}: memory sequence produced no result`) - return completedArtifact - } - - if (costUsd === 0) { - let artifact: AgentMemorySequenceArtifact | undefined - let error: unknown - try { - artifact = await execute() - } catch (caught) { - error = caught - } - if (error) throw error - if (!artifact) throw new Error(`${candidate.id}: memory sequence produced no result`) - return artifact - } - const paid = await context.cost.runPaidCall({ - callId: memoryAttemptCostCallId(attempt, 'execute', 0), - actor: `agent-knowledge:memory-experiment:${candidate.id}`, - model: candidate.ref, - maximumCharge: { externallyEnforcedMaximumUsd: costUsd }, - execute, - receipt: () => costRecorder.receipt(externalCallAttempted), - receiptFromError: () => costRecorder.receipt(externalCallAttempted), - }) - if (!paid.succeeded) throw paid.error - return paid.value -} - -function executionStep(step: AgentMemorySequenceStep, ordinal: number): AgentMemoryExecutionStep { - return { - ordinal, - ...(step.instruction !== undefined ? { instruction: step.instruction } : {}), - ...(step.scope !== undefined ? { scope: structuredClone(step.scope) } : {}), - } -} - -async function writeStep(memory: AgentMemoryBranch, step: AgentMemorySequenceStep): Promise { - const writes = (step.writes ?? []).map((write) => ({ - ...write, - scope: mergeScopes(step.scope, write.scope), - })) - if (step.parallelWrites) { - await Promise.all(writes.map((write) => memory.write(write))) - return - } - for (const write of writes) await memory.write(write) -} - -async function probeStep( - memory: AgentMemoryBranch, - sequence: AgentMemorySequence, - step: AgentMemorySequenceStep, - stepOrdinal: number, - dispatch: DispatchContext, -): Promise { - const run = async (probe: AgentMemorySequenceProbe): Promise => { - const scope = mergeScopes(step.scope, probe.scope) - const context = await memory.getContext(probe.query, { scope, limit: probe.limit }) - const scoringEvidence: AgentMemoryProbeScoringEvidence = { - answer: context.text, - rememberedFacts: context.hits.map((hit) => hit.text), - citedEventIds: uniqueStrings([ - ...context.hits.map((hit) => hit.metadata?.eventId), - ...context.hits.flatMap((hit) => stringArray(hit.metadata?.eventIds)), - ]), - usedMemoryIds: context.hits.map((hit) => hit.id), - actorIds: uniqueStrings([ - ...context.hits.map((hit) => hit.metadata?.actorId), - ...context.hits.flatMap((hit) => stringArray(hit.metadata?.actorIds)), - ]), - } - const evidenceRef = canonicalDigest(scoringEvidence) - const evidencePath = await dispatch.artifacts.writeJson( - `memory-evidence/${evidenceRef.slice(7)}.json`, - scoringEvidence, - ) - const evaluation = scoreAgentMemoryProbe(sequence, step, probe, scoringEvidence) - return { - id: probe.id, - stepId: step.id, - stepOrdinal, - ...(probe.retentionKey !== undefined ? { retentionKey: probe.retentionKey } : {}), - ...(probe.transferKey !== undefined ? { transferKey: probe.transferKey } : {}), - query: probe.query, - score: evaluation.score, - passed: evaluation.passed, - dimensions: evaluation.dimensions, - applicableDimensions: evaluation.applicableDimensions ?? Object.keys(evaluation.dimensions), - notes: evaluation.notes, - hitIds: context.hits.map((hit) => hit.id), - evidenceRef, - evidencePath, - } - } - if (step.parallelProbes === false) { - const results: AgentMemorySequenceProbeResult[] = [] - for (const probe of step.probes ?? []) results.push(await run(probe)) - return results - } - return Promise.all((step.probes ?? []).map(run)) -} diff --git a/src/memory/experiment/comparison-ref.ts b/src/memory/experiment/comparison-ref.ts deleted file mode 100644 index c00c78f..0000000 --- a/src/memory/experiment/comparison-ref.ts +++ /dev/null @@ -1,85 +0,0 @@ -import { canonicalJson } from '@tangle-network/agent-eval' -import { sha256 } from '../../ids' -import { DEFAULT_MEMORY_RECOVERY_RETRIES_PER_ATTEMPT } from '../attempt-log' -import { DEFAULT_MEMORY_CLEANUP_TIMEOUT_MS } from '../lifecycle' -import type { - AgentMemoryExperimentCandidate, - AgentMemoryExperimentComparisonRef, - AgentMemoryMode, - RunAgentMemoryExperimentOptions, -} from './types' - -export const MEMORY_EXPERIMENT_IMPLEMENTATION_REF = `sha256:${sha256( - canonicalJson({ - stateful: 'retain declared sequence scopes across ordered steps', - stateless: 'clear every declared sequence scope before each step after the first', - isolation: 'provider scopes receive an opaque branch id but no experiment or sequence labels', - recovery: 'unfinished attempts bind cleanup to the exact sequence sha256', - gain: 'post-first-step stateful minus stateless reward', - inference: 'average candidates and repetitions within each sequence before paired bootstrap', - preTreatment: 'report first-step score balance separately from gain', - transfer: 'only probes with explicit transferKey after the first step', - forgetting: 'signed prior peak minus final score for exact repeated retentionKey probes', - evidence: 'full sha256 design, manifest, split, artifact, and probe scoring-input references', - }), -)}` - -const COMPARISON_REF_PATTERN = /^sha256:[a-f0-9]{64}$/ - -export function resolveAgentMemoryMode(value: unknown): AgentMemoryMode { - if (value === undefined) return 'stateful' - if (value !== 'stateful' && value !== 'stateless') { - throw new Error("memory experiment memoryMode must be 'stateful' or 'stateless'") - } - return value -} - -export function memoryExperimentComparisonRef( - options: RunAgentMemoryExperimentOptions, - resolvedRunDir: string, -): AgentMemoryExperimentComparisonRef { - const material = canonicalJson({ - implementationRef: MEMORY_EXPERIMENT_IMPLEMENTATION_REF, - experimentId: options.experimentId, - experimentRunId: options.experimentRunId ?? resolvedRunDir, - sequences: options.sequences, - candidates: options.candidates.map(candidateConditions), - recoveryCandidates: (options.recoveryCandidates ?? []).map(candidateConditions), - executeStepRef: options.executeStepRef ?? 'fixtures', - cleanupBranches: options.cleanupBranches ?? true, - seed: options.seed ?? 42, - reps: options.reps ?? 1, - resumable: options.resumable ?? true, - costCeilingUsd: options.costCeiling ?? options.costLedger?.costCeilingUsd ?? 0, - costPhase: options.costPhase ?? null, - maxConcurrency: options.maxConcurrency ?? 2, - dispatchTimeoutMs: options.dispatchTimeoutMs || null, - cleanupTimeoutMs: options.cleanupTimeoutMs ?? DEFAULT_MEMORY_CLEANUP_TIMEOUT_MS, - maxRecoveryAttempts: options.maxRecoveryAttempts ?? 1_000, - maxRecoveryRetriesPerAttempt: - options.maxRecoveryRetriesPerAttempt ?? DEFAULT_MEMORY_RECOVERY_RETRIES_PER_ATTEMPT, - }) - return `sha256:${sha256(material)}` -} - -export function assertAgentMemoryExperimentComparisonRef( - value: unknown, - label: string, -): asserts value is AgentMemoryExperimentComparisonRef { - if (typeof value !== 'string' || !COMPARISON_REF_PATTERN.test(value)) { - throw new Error(`${label} must be a full lowercase sha256 reference`) - } -} - -function candidateConditions(candidate: AgentMemoryExperimentCandidate) { - return { - id: candidate.id, - label: candidate.label ?? null, - ref: candidate.ref, - policy: candidate.policy ?? null, - baseScope: candidate.baseScope ?? null, - externalCostUsdPerSequence: candidate.externalCostUsdPerSequence ?? 0, - externalRecoveryCostUsdPerAttempt: candidate.externalRecoveryCostUsdPerAttempt ?? 0, - externalCostAccounting: candidate.externalCostAccounting ?? 'exact', - } -} diff --git a/src/memory/experiment/cost.ts b/src/memory/experiment/cost.ts deleted file mode 100644 index 7c1de9d..0000000 --- a/src/memory/experiment/cost.ts +++ /dev/null @@ -1,58 +0,0 @@ -export interface AgentMemoryCostReceipt { - model: string - inputTokens: 0 - outputTokens: 0 - actualCostUsd?: number - costUnknown?: boolean -} - -export interface AgentMemoryCostRecorder { - record(actualCostUsd: number): void - receipt(externalCallAttempted: boolean): AgentMemoryCostReceipt -} - -export function createAgentMemoryCostRecorder(input: { - candidateRef: string - maximumCostUsd: number - operation: string -}): AgentMemoryCostRecorder { - let observedCostUsd = 0 - let receiptCount = 0 - - return { - record(actualCostUsd) { - if (!Number.isFinite(actualCostUsd) || actualCostUsd < 0) { - throw new Error( - `${input.operation}: observed external cost must be non-negative and finite`, - ) - } - const next = observedCostUsd + actualCostUsd - if (exceedsMaximum(next, input.maximumCostUsd)) { - throw new Error( - `${input.operation}: observed external cost ${next} exceeds declared maximum ${input.maximumCostUsd}`, - ) - } - observedCostUsd = next - receiptCount += 1 - }, - receipt(externalCallAttempted) { - const base = { - model: input.candidateRef, - inputTokens: 0 as const, - outputTokens: 0 as const, - } - if (input.maximumCostUsd === 0 || !externalCallAttempted) { - return { ...base, actualCostUsd: 0 } - } - if (receiptCount === 0) { - return { ...base, costUnknown: true } - } - return { ...base, actualCostUsd: observedCostUsd } - }, - } -} - -function exceedsMaximum(actual: number, maximum: number): boolean { - const tolerance = Number.EPSILON * Math.max(1, Math.abs(actual), Math.abs(maximum)) * 8 - return actual - maximum > tolerance -} diff --git a/src/memory/experiment/execution-context.ts b/src/memory/experiment/execution-context.ts deleted file mode 100644 index 68984ca..0000000 --- a/src/memory/experiment/execution-context.ts +++ /dev/null @@ -1,171 +0,0 @@ -import type { CostReceipt, PaidCallResult } from '@tangle-network/agent-eval' -import type { DispatchContext } from '@tangle-network/agent-eval/campaign' -import type { - AgentMemoryExecutionContext, - AgentMemoryExecutionCostMeter, - AgentMemoryExecutionCostReceipt, - AgentMemoryExecutionPaidCallInput, - AgentMemoryExecutionPaidCallResult, - AgentMemorySequence, -} from './types' - -export interface OwnedAgentMemoryExecutionContext { - context: AgentMemoryExecutionContext - abort(): void - dispose(): void -} - -export function createAgentMemoryExecutionContext( - dispatch: DispatchContext, - sequence: AgentMemorySequence, -): OwnedAgentMemoryExecutionContext { - const privateValues = memoryEvaluationPrivateValues(dispatch.cellId, sequence) - const signal = relayAbortWithoutReason(dispatch.signal) - const cost: AgentMemoryExecutionCostMeter = Object.freeze({ - async runPaidCall( - input: AgentMemoryExecutionPaidCallInput, - ): Promise> { - const result = await dispatch.cost.runPaidCall({ - ...input, - execute: (sourceSignal, callId) => - withRedactedAbortSignal(sourceSignal, (redactedSignal) => - input.execute(redactedSignal, callId), - ), - }) - return sanitizePaidCallResult(result, privateValues) - }, - }) - return { - context: Object.freeze({ signal: signal.signal, cost }), - abort: signal.abort, - dispose: signal.dispose, - } -} - -function sanitizePaidCallResult( - result: PaidCallResult, - privateValues: readonly string[], -): AgentMemoryExecutionPaidCallResult { - if (result.succeeded) { - return Object.freeze({ - succeeded: true, - callId: result.callId, - value: result.value, - receipt: sanitizeCostReceipt(result.receipt, privateValues), - }) - } - return Object.freeze({ - succeeded: false, - ...(result.callId === undefined ? {} : { callId: result.callId }), - error: sanitizeError(result.error, privateValues), - ...(result.receipt === undefined - ? {} - : { receipt: sanitizeCostReceipt(result.receipt, privateValues) }), - }) -} - -function sanitizeCostReceipt( - receipt: CostReceipt, - privateValues: readonly string[], -): AgentMemoryExecutionCostReceipt { - const { tags: _tags, phase: _phase, error, ...visible } = receipt - return Object.freeze({ - ...visible, - ...(error === undefined ? {} : { error: redactPrivateValues(error, privateValues) }), - }) -} - -function sanitizeError(error: Error, privateValues: readonly string[]): Error { - const sanitized = new Error(redactPrivateValues(error.message, privateValues)) - sanitized.name = redactPrivateValues(error.name, privateValues) - return sanitized -} - -async function withRedactedAbortSignal( - source: AbortSignal, - run: (signal: AbortSignal) => Promise, -): Promise { - const relay = relayAbortWithoutReason(source) - try { - return await run(relay.signal) - } finally { - relay.dispose() - } -} - -function relayAbortWithoutReason(source: AbortSignal): { - signal: AbortSignal - abort(): void - dispose(): void -} { - const controller = new AbortController() - const abort = () => controller.abort(memoryExecutionAbortError()) - if (source.aborted) abort() - else source.addEventListener('abort', abort, { once: true }) - return { - signal: controller.signal, - abort, - dispose() { - source.removeEventListener('abort', abort) - }, - } -} - -function memoryExecutionAbortError(): Error { - const error = new Error('memory execution aborted') - error.name = 'AbortError' - return error -} - -function memoryEvaluationPrivateValues( - cellId: string, - sequence: AgentMemorySequence, -): readonly string[] { - const values = new Set([cellId, sequence.id]) - collectUnknownStrings(sequence.metadata, values) - for (const step of sequence.steps) { - values.add(step.id) - collectUnknownStrings(step.metadata, values) - for (const probe of step.probes ?? []) { - values.add(probe.id) - values.add(probe.query) - if (probe.referenceAnswer !== undefined) values.add(probe.referenceAnswer) - for (const matcher of [...(probe.requiredFacts ?? []), ...(probe.forbiddenFacts ?? [])]) { - values.add(matcher.id) - for (const expected of matcher.anyOf) values.add(expected) - } - for (const expected of probe.expectedEventIds ?? []) values.add(expected) - for (const expected of probe.expectedActorIds ?? []) values.add(expected) - } - } - return [...values].filter(Boolean).sort((left, right) => right.length - left.length) -} - -function collectUnknownStrings( - value: unknown, - output: Set, - seen = new WeakSet(), -): void { - if (typeof value === 'string') { - output.add(value) - return - } - if (!value || typeof value !== 'object' || seen.has(value)) return - seen.add(value) - if (Array.isArray(value)) { - for (const item of value) collectUnknownStrings(item, output, seen) - return - } - for (const [key, item] of Object.entries(value)) { - output.add(key) - collectUnknownStrings(item, output, seen) - } -} - -function redactPrivateValues(value: string, privateValues: readonly string[]): string { - let redacted = value - for (const privateValue of privateValues) { - redacted = redacted.split(privateValue).join('[redacted]') - } - return redacted -} diff --git a/src/memory/experiment/learning-evidence.ts b/src/memory/experiment/learning-evidence.ts deleted file mode 100644 index ff6a612..0000000 --- a/src/memory/experiment/learning-evidence.ts +++ /dev/null @@ -1,178 +0,0 @@ -import { canonicalJson } from '@tangle-network/agent-eval' -import { type CampaignStorage, canonicalDigest } from '@tangle-network/agent-eval/campaign' -import { countDimensions, mean, meanDimensions } from './metrics' -import { type AgentMemoryProbeScoringEvidence, scoreAgentMemoryProbe } from './probe-evaluation' -import type { - AgentMemorySequence, - AgentMemorySequenceArtifact, - AgentMemorySequenceProbe, - AgentMemorySequenceProbeResult, - AgentMemorySequenceStep, - RunAgentMemoryExperimentResult, -} from './types' - -interface ProbeDefinition { - step: AgentMemorySequenceStep - stepOrdinal: number - probe: AgentMemorySequenceProbe -} - -export function assertStoredAgentMemoryEvidence( - storage: CampaignStorage, - result: RunAgentMemoryExperimentResult, - sequences: readonly AgentMemorySequence[], -): void { - const sequenceById = new Map(sequences.map((sequence) => [sequence.id, sequence])) - for (const cell of result.campaign.cells) { - if (cell.error || !cell.artifact) { - throw new Error( - `memory learning cell '${cell.cellId}' has no valid artifact${cell.error ? `: ${cell.error}` : ''}`, - ) - } - const sequence = sequenceById.get(cell.artifact.sequenceId) - if (!sequence) { - throw new Error( - `memory learning cell '${cell.cellId}' references unknown sequence '${cell.artifact.sequenceId}'`, - ) - } - assertStoredCellEvidence(storage, cell.cellId, cell.artifact, sequence) - } -} - -function assertStoredCellEvidence( - storage: CampaignStorage, - cellId: string, - artifact: AgentMemorySequenceArtifact, - sequence: AgentMemorySequence, -): void { - const definitions = indexProbeDefinitions(sequence) - if (artifact.probes.length !== definitions.size) { - throw new Error( - `memory learning cell '${cellId}' has ${artifact.probes.length} probes; expected ${definitions.size}`, - ) - } - const seen = new Set() - for (const probe of artifact.probes) { - const key = probeKey(probe.stepOrdinal, probe.stepId, probe.id) - const definition = definitions.get(key) - if (!definition || seen.has(key)) { - throw new Error(`memory learning cell '${cellId}' has unexpected probe '${key}'`) - } - seen.add(key) - assertStoredProbeEvidence(storage, cellId, probe, definition, sequence) - } - const expectedScore = mean(artifact.probes.map((probe) => probe.score)) - const expectedPassed = - artifact.probes.length > 0 && artifact.probes.every((probe) => probe.passed) - const expectedDimensions = meanDimensions(artifact.probes.map((probe) => probe.dimensions)) - const expectedCounts = countDimensions(artifact.probes.map((probe) => probe.applicableDimensions)) - if ( - artifact.score !== expectedScore || - artifact.passed !== expectedPassed || - canonicalJson(artifact.dimensions) !== canonicalJson(expectedDimensions) || - canonicalJson(artifact.dimensionSampleCounts) !== canonicalJson(expectedCounts) - ) { - throw new Error(`memory learning cell '${cellId}' aggregate does not match its probes`) - } -} - -function assertStoredProbeEvidence( - storage: CampaignStorage, - cellId: string, - observed: AgentMemorySequenceProbeResult, - definition: ProbeDefinition, - sequence: AgentMemorySequence, -): void { - const { probe, step } = definition - if ( - observed.query !== probe.query || - observed.retentionKey !== probe.retentionKey || - observed.transferKey !== probe.transferKey - ) { - throw new Error(`memory learning cell '${cellId}' probe '${observed.id}' changed definition`) - } - const raw = storage.read(observed.evidencePath) - if (raw === undefined) { - throw new Error(`memory learning probe evidence is missing at '${observed.evidencePath}'`) - } - let parsed: unknown - try { - parsed = JSON.parse(raw) - } catch (error) { - throw new Error(`memory learning probe evidence is malformed at '${observed.evidencePath}'`, { - cause: error, - }) - } - if (canonicalDigest(parsed) !== observed.evidenceRef) { - throw new Error( - `memory learning probe evidence hash does not match at '${observed.evidencePath}'`, - ) - } - const evidence = parseProbeEvidence(parsed, observed.evidencePath) - const expected = scoreAgentMemoryProbe(sequence, step, probe, evidence) - const observedEvaluation = { - score: observed.score, - passed: observed.passed, - dimensions: observed.dimensions, - applicableDimensions: observed.applicableDimensions, - notes: observed.notes, - } - const expectedEvaluation = { - score: expected.score, - passed: expected.passed, - dimensions: expected.dimensions, - applicableDimensions: expected.applicableDimensions ?? Object.keys(expected.dimensions), - notes: expected.notes, - } - if (canonicalJson(observedEvaluation) !== canonicalJson(expectedEvaluation)) { - throw new Error( - `memory learning probe score does not match saved evidence at '${observed.evidencePath}'`, - ) - } - if (canonicalJson(observed.hitIds) !== canonicalJson(evidence.usedMemoryIds)) { - throw new Error( - `memory learning probe hits do not match saved evidence at '${observed.evidencePath}'`, - ) - } -} - -function indexProbeDefinitions(sequence: AgentMemorySequence): Map { - const definitions = new Map() - for (const [stepOrdinal, step] of sequence.steps.entries()) { - for (const probe of step.probes ?? []) { - definitions.set(probeKey(stepOrdinal, step.id, probe.id), { step, stepOrdinal, probe }) - } - } - return definitions -} - -function probeKey(stepOrdinal: number, stepId: string, probeId: string): string { - return JSON.stringify([stepOrdinal, stepId, probeId]) -} - -function parseProbeEvidence(value: unknown, path: string): AgentMemoryProbeScoringEvidence { - if (!value || typeof value !== 'object' || Array.isArray(value)) { - throw new Error(`memory learning probe evidence has invalid shape at '${path}'`) - } - const record = value as Record - if ( - typeof record.answer !== 'string' || - !isStringArray(record.rememberedFacts) || - !isStringArray(record.citedEventIds) || - !isStringArray(record.usedMemoryIds) || - !isStringArray(record.actorIds) - ) { - throw new Error(`memory learning probe evidence has invalid shape at '${path}'`) - } - return { - answer: record.answer, - rememberedFacts: record.rememberedFacts, - citedEventIds: record.citedEventIds, - usedMemoryIds: record.usedMemoryIds, - actorIds: record.actorIds, - } -} - -function isStringArray(value: unknown): value is string[] { - return Array.isArray(value) && value.every((item) => typeof item === 'string') -} diff --git a/src/memory/experiment/learning-metrics.ts b/src/memory/experiment/learning-metrics.ts deleted file mode 100644 index 94cc67d..0000000 --- a/src/memory/experiment/learning-metrics.ts +++ /dev/null @@ -1,470 +0,0 @@ -import { comparePairedArms, type PairedBootstrapResult } from '@tangle-network/agent-eval' -import type { PairedMemoryLearningCell, PairedMemoryLearningProbe } from './learning-pairs' -import { mean } from './metrics' -import type { - AgentMemoryEvidenceRef, - AgentMemoryForgettingComparison, - AgentMemoryLearningCandidateSummary, - AgentMemoryLearningCellComparison, - AgentMemoryLearningComparison, - AgentMemoryTransferCellComparison, - AgentMemoryTransferStepSummary, - RunAgentMemoryExperimentResult, -} from './types' - -export function measureAgentMemoryLearning( - paired: readonly PairedMemoryLearningCell[], - stateful: RunAgentMemoryExperimentResult, - stateless: RunAgentMemoryExperimentResult, -): AgentMemoryLearningComparison { - const cells = paired.map(toCellComparison) - const transferCells = paired.flatMap(toTransferCells) - const forgettingProbes = paired.flatMap(toForgettingComparisons) - const gainUnits = collapseLearningCells(cells) - const forgetting = summarizeForgetting(forgettingProbes, stateful.campaign.seed) - const preTreatment = measurePreTreatment(paired, stateful.campaign.seed) - const first = paired[0] - if (!first) throw new Error('cannot compare memory learning: no paired cells') - - return { - comparisonRef: stateful.comparisonRef, - evidence: { - splitRef: evidenceRef(stateful.campaign.splitDigest, 'task split'), - statefulManifestRef: evidenceRef( - stateful.campaign.manifestHash, - 'stateful campaign manifest', - ), - statelessManifestRef: evidenceRef( - stateless.campaign.manifestHash, - 'stateless campaign manifest', - ), - statefulRunDir: stateful.campaign.runDir, - statelessRunDir: stateless.campaign.runDir, - candidateRefs: stateful.candidateRefs.map((candidate) => ({ ...candidate })), - executionRef: stateful.executionRef, - }, - cells, - preTreatment, - gain: comparePairedRewards(gainUnits, stateful.campaign.seed, 'overall gain'), - gainByCandidate: summarizeCandidateGain(cells, stateful.campaign.seed), - transfer: { - definition: 'explicit-transfer-probes', - cells: transferCells, - byStep: summarizeTransfer(transferCells, stateful.campaign.seed), - }, - forgetting: { - definition: 'prior-peak-minus-final', - probes: forgettingProbes, - ...forgetting, - }, - } -} - -function measurePreTreatment( - paired: readonly PairedMemoryLearningCell[], - seed: number, -): AgentMemoryLearningComparison['preTreatment'] { - const cells = paired.flatMap((pair) => { - const probes = pair.probes.filter((probe) => probe.stateful.stepOrdinal === 0) - if (probes.length === 0) return [] - return [ - { - sequenceId: pair.stateful.artifact.sequenceId, - statefulReward: mean(probes.map((probe) => probe.stateful.score)), - statelessReward: mean(probes.map((probe) => probe.stateless.score)), - exact: probes.every((probe) => probe.stateful.score === probe.stateless.score), - }, - ] - }) - const bySequence = new Map< - string, - { - statefulRewards: number[] - statelessRewards: number[] - exact: boolean - } - >() - for (const cell of cells) { - const unit = bySequence.get(cell.sequenceId) ?? { - statefulRewards: [], - statelessRewards: [], - exact: true, - } - unit.statefulRewards.push(cell.statefulReward) - unit.statelessRewards.push(cell.statelessReward) - unit.exact &&= cell.exact - bySequence.set(cell.sequenceId, unit) - } - const units = [...bySequence.entries()].map(([sequenceId, unit]) => ({ - pairKey: sequenceId, - statefulReward: mean(unit.statefulRewards), - statelessReward: mean(unit.statelessRewards), - exact: unit.exact, - })) - return { - definition: 'first-step-probes', - cells: cells.length, - n: units.length, - exactMatchRate: units.length === 0 ? null : mean(units.map((unit) => (unit.exact ? 1 : 0))), - difference: - units.length === 0 ? null : comparePairedRewards(units, seed, 'pre-treatment difference'), - } -} - -function toCellComparison(pair: PairedMemoryLearningCell): AgentMemoryLearningCellComparison { - const probes = pair.probes.filter((probe) => probe.stateful.stepOrdinal > 0) - if (probes.length === 0) { - throw new Error( - `cannot compare memory learning: cell ${pair.stateful.cell.cellId} has no probe after the first step`, - ) - } - const statefulReward = mean(probes.map((probe) => probe.stateful.score)) - const statelessReward = mean(probes.map((probe) => probe.stateless.score)) - return { - cellId: pair.stateful.cell.cellId, - candidateId: pair.stateful.artifact.candidateId, - sequenceId: pair.stateful.artifact.sequenceId, - rep: pair.stateful.cell.rep, - seed: pair.stateful.cell.seed, - statefulArtifactRef: pair.stateful.artifactRef, - statelessArtifactRef: pair.stateless.artifactRef, - probeCount: probes.length, - statefulReward, - statelessReward, - gain: difference(statefulReward, statelessReward), - } -} - -function toTransferCells(pair: PairedMemoryLearningCell): AgentMemoryTransferCellComparison[] { - const groups = new Map< - string, - { stepOrdinal: number; transferKey: string; probes: PairedMemoryLearningProbe[] } - >() - for (const probe of pair.probes) { - const transferKey = probe.stateful.transferKey - if (transferKey === undefined) continue - const key = JSON.stringify([probe.stateful.stepOrdinal, transferKey]) - const group = groups.get(key) ?? { - stepOrdinal: probe.stateful.stepOrdinal, - transferKey, - probes: [], - } - group.probes.push(probe) - groups.set(key, group) - } - return [...groups.values()] - .map(({ stepOrdinal, transferKey, probes }) => { - const stepIds = sortedUnique(probes.map((probe) => probe.stateful.stepId)) - if (stepIds.length !== 1) { - throw new Error( - `cannot compare memory learning: step ordinal ${stepOrdinal} maps to multiple step ids in ${pair.stateful.cell.cellId}`, - ) - } - const statefulReward = mean(probes.map((probe) => probe.stateful.score)) - const statelessReward = mean(probes.map((probe) => probe.stateless.score)) - return { - cellId: pair.stateful.cell.cellId, - candidateId: pair.stateful.artifact.candidateId, - sequenceId: pair.stateful.artifact.sequenceId, - rep: pair.stateful.cell.rep, - seed: pair.stateful.cell.seed, - stepId: stepIds[0]!, - stepOrdinal, - transferKey, - probeCount: probes.length, - statefulArtifactRef: pair.stateful.artifactRef, - statelessArtifactRef: pair.stateless.artifactRef, - statefulReward, - statelessReward, - gain: difference(statefulReward, statelessReward), - } - }) - .sort( - (left, right) => - left.stepOrdinal - right.stepOrdinal || left.transferKey.localeCompare(right.transferKey), - ) -} - -function toForgettingComparisons( - pair: PairedMemoryLearningCell, -): AgentMemoryForgettingComparison[] { - const byRetentionKey = new Map() - for (const probe of pair.probes) { - const retentionKey = probe.stateful.retentionKey - if (retentionKey === undefined) continue - const probes = byRetentionKey.get(retentionKey) ?? [] - probes.push(probe) - byRetentionKey.set(retentionKey, probes) - } - return [...byRetentionKey.entries()] - .map(([retentionKey, observations]) => { - observations.sort((left, right) => left.stateful.stepOrdinal - right.stateful.stepOrdinal) - if (observations.length < 2) { - throw new Error( - `cannot compare memory learning: retention key ${retentionKey} in ${pair.stateful.cell.cellId} has fewer than two observations`, - ) - } - const ordinals = observations.map((probe) => probe.stateful.stepOrdinal) - if (new Set(ordinals).size !== ordinals.length) { - throw new Error( - `cannot compare memory learning: retention key ${retentionKey} in ${pair.stateful.cell.cellId} repeats within one step`, - ) - } - const final = observations[observations.length - 1]! - const prior = observations.slice(0, -1) - const statefulPeak = maxProbe(prior, (probe) => probe.stateful.score) - const statelessPeak = maxProbe(prior, (probe) => probe.stateless.score) - const statefulForgetting = difference(statefulPeak.reward, final.stateful.score) - const statelessForgetting = difference(statelessPeak.reward, final.stateless.score) - return { - cellId: pair.stateful.cell.cellId, - candidateId: pair.stateful.artifact.candidateId, - sequenceId: pair.stateful.artifact.sequenceId, - rep: pair.stateful.cell.rep, - seed: pair.stateful.cell.seed, - retentionKey, - observations: observations.length, - firstStepOrdinal: observations[0]!.stateful.stepOrdinal, - finalStepOrdinal: final.stateful.stepOrdinal, - statefulArtifactRef: pair.stateful.artifactRef, - statelessArtifactRef: pair.stateless.artifactRef, - statefulPriorPeakReward: statefulPeak.reward, - statefulPriorPeakStepOrdinal: statefulPeak.stepOrdinal, - statefulFinalReward: final.stateful.score, - statefulForgetting, - statelessPriorPeakReward: statelessPeak.reward, - statelessPriorPeakStepOrdinal: statelessPeak.stepOrdinal, - statelessFinalReward: final.stateless.score, - statelessForgetting, - excessForgetting: difference(statefulForgetting, statelessForgetting), - } - }) - .sort( - (left, right) => - left.retentionKey.localeCompare(right.retentionKey) || - left.sequenceId.localeCompare(right.sequenceId) || - left.rep - right.rep || - left.seed - right.seed, - ) -} - -function summarizeCandidateGain( - cells: readonly AgentMemoryLearningCellComparison[], - seed: number, -): AgentMemoryLearningCandidateSummary[] { - return sortedUnique(cells.map((cell) => cell.candidateId)).map((candidateId) => { - const units = collapseLearningCells(cells.filter((cell) => cell.candidateId === candidateId)) - return { - candidateId, - cells: cells.filter((cell) => cell.candidateId === candidateId).length, - gain: comparePairedRewards(units, seed, `candidate ${candidateId} gain`), - } - }) -} - -function summarizeTransfer( - cells: readonly AgentMemoryTransferCellComparison[], - seed: number, -): AgentMemoryTransferStepSummary[] { - const groups = new Map< - string, - { - candidateId: string - transferKey: string - stepOrdinal: number - rewardsBySequence: Map - } - >() - for (const cell of cells) { - const key = JSON.stringify([cell.candidateId, cell.transferKey, cell.stepOrdinal]) - const group = groups.get(key) ?? { - candidateId: cell.candidateId, - transferKey: cell.transferKey, - stepOrdinal: cell.stepOrdinal, - rewardsBySequence: new Map< - string, - { - stateful: number[] - stateless: number[] - } - >(), - } - const rewards = group.rewardsBySequence.get(cell.sequenceId) ?? { - stateful: [], - stateless: [], - } - rewards.stateful.push(cell.statefulReward) - rewards.stateless.push(cell.statelessReward) - group.rewardsBySequence.set(cell.sequenceId, rewards) - groups.set(key, group) - } - return [...groups.values()] - .sort( - (left, right) => - left.candidateId.localeCompare(right.candidateId) || - left.transferKey.localeCompare(right.transferKey) || - left.stepOrdinal - right.stepOrdinal, - ) - .map(({ candidateId, transferKey, stepOrdinal, rewardsBySequence }) => { - const sequenceRewards = [...rewardsBySequence.entries()] - .sort(([left], [right]) => left.localeCompare(right)) - .map(([sequenceId, rewards]) => ({ - pairKey: sequenceId, - statefulReward: mean(rewards.stateful), - statelessReward: mean(rewards.stateless), - })) - return { - candidateId, - transferKey, - stepOrdinal, - gain: comparePairedRewards( - sequenceRewards, - seed, - `transfer ${candidateId}:${transferKey}:${stepOrdinal}`, - ), - } - }) -} - -function collapseLearningCells( - cells: readonly AgentMemoryLearningCellComparison[], -): Array<{ pairKey: string; statefulReward: number; statelessReward: number }> { - const bySequence = new Map() - for (const cell of cells) { - const unit = bySequence.get(cell.sequenceId) ?? { - statefulRewards: [], - statelessRewards: [], - } - unit.statefulRewards.push(cell.statefulReward) - unit.statelessRewards.push(cell.statelessReward) - bySequence.set(cell.sequenceId, unit) - } - return [...bySequence.entries()] - .sort(([left], [right]) => left.localeCompare(right)) - .map(([sequenceId, unit]) => ({ - pairKey: sequenceId, - statefulReward: mean(unit.statefulRewards), - statelessReward: mean(unit.statelessRewards), - })) -} - -function summarizeForgetting( - probes: readonly AgentMemoryForgettingComparison[], - seed: number, -): { - n: number - meanStatefulForgetting: number | null - meanStatelessForgetting: number | null - meanExcessForgetting: number | null - excess: PairedBootstrapResult | null -} { - const byTarget = new Map< - string, - { - stateful: number[] - stateless: number[] - excess: number[] - } - >() - for (const probe of probes) { - const key = JSON.stringify([probe.sequenceId, probe.retentionKey]) - const unit = byTarget.get(key) ?? { stateful: [], stateless: [], excess: [] } - unit.stateful.push(probe.statefulForgetting) - unit.stateless.push(probe.statelessForgetting) - unit.excess.push(probe.excessForgetting) - byTarget.set(key, unit) - } - const units = [...byTarget.entries()].map(([pairKey, unit]) => ({ - pairKey, - statefulReward: mean(unit.stateful), - statelessReward: mean(unit.stateless), - excess: mean(unit.excess), - })) - return { - n: units.length, - meanStatefulForgetting: optionalMean(units.map((unit) => unit.statefulReward)), - meanStatelessForgetting: optionalMean(units.map((unit) => unit.statelessReward)), - meanExcessForgetting: optionalMean(units.map((unit) => unit.excess)), - excess: units.length === 0 ? null : comparePairedRewards(units, seed, 'excess forgetting'), - } -} - -function comparePairedRewards( - units: readonly { - pairKey: string - statefulReward: number - statelessReward: number - }[], - seed: number, - label: string, -): PairedBootstrapResult { - const comparison = comparePairedArms( - units.flatMap((unit) => [ - { - pairKey: unit.pairKey, - arm: 'stateless', - metrics: { reward: unit.statelessReward }, - }, - { - pairKey: unit.pairKey, - arm: 'stateful', - metrics: { reward: unit.statefulReward }, - }, - ]), - { - baselineArm: 'stateless', - treatmentArm: 'stateful', - metricNames: ['reward'], - bootstrap: { statistic: 'mean', seed }, - }, - ) - const metric = comparison.metricDeltas[0] - if ( - comparison.nPairs !== units.length || - comparison.nUnpairedBaseline !== 0 || - comparison.nUnpairedTreatment !== 0 || - !metric || - metric.n !== units.length || - metric.nMissing !== 0 || - !metric.bootstrapCi - ) { - throw new Error(`cannot compare memory learning: incomplete paired ${label}`) - } - return metric.bootstrapCi -} - -function maxProbe( - probes: readonly PairedMemoryLearningProbe[], - reward: (probe: PairedMemoryLearningProbe) => number, -): { reward: number; stepOrdinal: number } { - let best = probes[0]! - for (const probe of probes.slice(1)) { - if (reward(probe) > reward(best)) best = probe - } - return { reward: reward(best), stepOrdinal: best.stateful.stepOrdinal } -} - -function optionalMean(values: readonly number[]): number | null { - return values.length === 0 ? null : mean(values) -} - -function difference(left: number, right: number): number { - const value = left - right - return Object.is(value, -0) ? 0 : value -} - -function sortedUnique(values: readonly string[]): string[] { - return [...new Set(values)].sort() -} - -function evidenceRef(value: unknown, label: string): AgentMemoryEvidenceRef { - if (typeof value === 'string' && /^sha256:[a-f0-9]{64}$/.test(value)) { - return value as AgentMemoryEvidenceRef - } - if (typeof value !== 'string' || !/^[a-f0-9]{64}$/.test(value)) { - throw new Error( - `cannot compare memory learning: ${label} must be a full lowercase sha256 digest; received ${String(value)}`, - ) - } - return `sha256:${value}` -} diff --git a/src/memory/experiment/learning-pairs.ts b/src/memory/experiment/learning-pairs.ts deleted file mode 100644 index 5e22254..0000000 --- a/src/memory/experiment/learning-pairs.ts +++ /dev/null @@ -1,457 +0,0 @@ -import { - assertCampaignSplitIdentity, - type CampaignCellResult, - canonicalDigest, -} from '@tangle-network/agent-eval/campaign' -import { type PairedArmRow, pairArms } from '@tangle-network/agent-eval/experiment' -import { stableId } from '../../ids' -import { assertImmutableRef } from '../../immutable-ref' -import { assertAgentMemoryExperimentComparisonRef } from './comparison-ref' -import { mean } from './metrics' -import type { - AgentMemoryEvidenceRef, - AgentMemorySequenceArtifact, - AgentMemorySequenceProbeResult, - CompareAgentMemoryLearningOptions, - RunAgentMemoryExperimentResult, -} from './types' - -export interface IndexedMemoryLearningCell { - cell: CampaignCellResult - artifact: AgentMemorySequenceArtifact - artifactRef: AgentMemoryEvidenceRef -} - -export interface PairedMemoryLearningProbe { - stateful: AgentMemorySequenceProbeResult - stateless: AgentMemorySequenceProbeResult -} - -export interface PairedMemoryLearningCell { - stateful: IndexedMemoryLearningCell - stateless: IndexedMemoryLearningCell - probes: PairedMemoryLearningProbe[] -} - -interface MemoryLearningArmRow extends PairedArmRow { - indexed: IndexedMemoryLearningCell -} - -export function pairAgentMemoryLearningRuns( - options: CompareAgentMemoryLearningOptions, -): PairedMemoryLearningCell[] { - assertComparableDesign(options.stateful, options.stateless) - const statefulCells = indexCompleteCells('stateful', options.stateful) - const statelessCells = indexCompleteCells('stateless', options.stateless) - const matched = pairArms( - [ - ...memoryLearningArmRows('stateful', statefulCells.values()), - ...memoryLearningArmRows('stateless', statelessCells.values()), - ], - { baselineArm: 'stateless', treatmentArm: 'stateful' }, - ) - if (matched.unpairedBaseline.length > 0 || matched.unpairedTreatment.length > 0) { - throw new Error( - `cannot compare memory learning: unmatched experiment cell identities; stateful-only=${unpairedCellIds(matched.unpairedTreatment).join(', ') || 'none'}; stateless-only=${unpairedCellIds(matched.unpairedBaseline).join(', ') || 'none'}`, - ) - } - - return matched.pairs - .map((pair): PairedMemoryLearningCell => { - const stateful = (pair.treatment as MemoryLearningArmRow).indexed - const stateless = (pair.baseline as MemoryLearningArmRow).indexed - if (stateful.cell.cellId !== stateless.cell.cellId) { - throw new Error( - `cannot compare memory learning: cell identity differs for ${pair.pairKey} (${stateful.cell.cellId} vs ${stateless.cell.cellId})`, - ) - } - return { stateful, stateless, probes: pairProbes(stateful, stateless) } - }) - .sort(comparePairedCells) -} - -function memoryLearningArmRows( - arm: 'stateful' | 'stateless', - cells: Iterable, -): MemoryLearningArmRow[] { - return [...cells].map((indexed) => ({ - arm, - pairKey: JSON.stringify([indexed.artifact.candidateId, indexed.artifact.sequenceId]), - repKey: JSON.stringify([indexed.cell.rep, indexed.cell.seed]), - indexed, - })) -} - -function unpairedCellIds(rows: readonly PairedArmRow[]): string[] { - return rows - .map((row) => (row as MemoryLearningArmRow).indexed) - .map((indexed) => cellIdentity(indexed.artifact, indexed.cell.rep, indexed.cell.seed)) - .sort() -} - -function assertComparableDesign( - stateful: RunAgentMemoryExperimentResult, - stateless: RunAgentMemoryExperimentResult, -): void { - if (stateful.memoryMode !== 'stateful') { - throw new Error( - `cannot compare memory learning: stateful arm is labeled ${String(stateful.memoryMode)}`, - ) - } - if (stateless.memoryMode !== 'stateless') { - throw new Error( - `cannot compare memory learning: stateless arm is labeled ${String(stateless.memoryMode)}`, - ) - } - assertAgentMemoryExperimentComparisonRef( - stateful.comparisonRef, - 'cannot compare memory learning: stateful comparisonRef', - ) - assertAgentMemoryExperimentComparisonRef( - stateless.comparisonRef, - 'cannot compare memory learning: stateless comparisonRef', - ) - if (stateful.comparisonRef !== stateless.comparisonRef) { - throw new Error( - `cannot compare memory learning: experimental conditions differ (${stateful.comparisonRef} vs ${stateless.comparisonRef})`, - ) - } - if (stateful.campaign.manifestHash === stateless.campaign.manifestHash) { - throw new Error( - 'cannot compare memory learning: campaign cache identity does not distinguish memoryMode', - ) - } - assertRunSplitIdentity('stateful', stateful) - assertRunSplitIdentity('stateless', stateless) - if (stateful.campaign.splitDigest !== stateless.campaign.splitDigest) { - throw new Error('cannot compare memory learning: stateful and stateless task identities differ') - } - if (stateful.campaign.reps !== stateless.campaign.reps) { - throw new Error( - `cannot compare memory learning: repetitions differ (${stateful.campaign.reps} vs ${stateless.campaign.reps})`, - ) - } - if (stateful.campaign.seed !== stateless.campaign.seed) { - throw new Error( - `cannot compare memory learning: campaign seeds differ (${stateful.campaign.seed} vs ${stateless.campaign.seed})`, - ) - } - if ( - JSON.stringify(stateful.candidateRefs) !== JSON.stringify(stateless.candidateRefs) || - stateful.candidateRefs.length === 0 - ) { - throw new Error('cannot compare memory learning: candidate references differ between arms') - } - const candidateIds = new Set() - for (const candidate of stateful.candidateRefs) { - assertNonEmpty(candidate.id, 'candidate reference id') - if (candidateIds.has(candidate.id)) { - throw new Error( - `cannot compare memory learning: duplicate candidate reference ${candidate.id}`, - ) - } - candidateIds.add(candidate.id) - try { - assertImmutableRef(candidate.ref, `${candidate.id}: candidate reference`) - } catch (error) { - throw new Error( - `cannot compare memory learning: ${error instanceof Error ? error.message : String(error)}`, - ) - } - } - if (stateful.executionRef !== stateless.executionRef) { - throw new Error('cannot compare memory learning: executor references differ between arms') - } - if (stateful.executionRef !== 'fixtures') { - try { - assertImmutableRef(stateful.executionRef, 'memory learning executor reference') - } catch (error) { - throw new Error( - `cannot compare memory learning: ${error instanceof Error ? error.message : String(error)}`, - ) - } - } -} - -function assertRunSplitIdentity( - arm: 'stateful' | 'stateless', - result: RunAgentMemoryExperimentResult, -): void { - try { - assertCampaignSplitIdentity( - result.campaign.scenarios, - result.campaign.reps, - result.campaign.splitDigest, - ) - } catch (error) { - throw new Error( - `cannot compare memory learning: ${arm} run has invalid task identity: ${error instanceof Error ? error.message : String(error)}`, - ) - } -} - -function indexCompleteCells( - arm: 'stateful' | 'stateless', - result: RunAgentMemoryExperimentResult, -): Map { - const expectedCellIds = new Set( - result.campaign.scenarios.flatMap((scenario) => - Array.from({ length: result.campaign.reps }, (_, rep) => `${scenario.id}:${rep}`), - ), - ) - const byIdentity = new Map() - const seenCellIds = new Set() - for (const cell of result.campaign.cells) { - if (!expectedCellIds.has(cell.cellId)) { - throw new Error( - `cannot compare memory learning: ${arm} run has unexpected cell ${cell.cellId}`, - ) - } - if (seenCellIds.has(cell.cellId)) { - throw new Error(`cannot compare memory learning: ${arm} run duplicates cell ${cell.cellId}`) - } - seenCellIds.add(cell.cellId) - if (cell.cellId !== `${cell.scenarioId}:${cell.rep}`) { - throw new Error( - `cannot compare memory learning: ${arm} cell ${cell.cellId} does not match its scenario and repetition`, - ) - } - if (cell.manifestHash !== result.campaign.manifestHash) { - throw new Error( - `cannot compare memory learning: ${arm} cell ${cell.cellId} manifest differs from its campaign`, - ) - } - if (cell.error) { - throw new Error( - `cannot compare memory learning: ${arm} cell ${cell.cellId} failed: ${cell.error}`, - ) - } - const artifact = cell.artifact as AgentMemorySequenceArtifact | null | undefined - if (!artifact) { - throw new Error(`cannot compare memory learning: ${arm} cell ${cell.cellId} has no artifact`) - } - assertArtifactIdentity(arm, result, cell, artifact) - if (!Number.isSafeInteger(cell.rep) || cell.rep < 0 || cell.rep >= result.campaign.reps) { - throw new Error(`cannot compare memory learning: ${arm} cell ${cell.cellId} has invalid rep`) - } - if (!Number.isSafeInteger(cell.seed)) { - throw new Error(`cannot compare memory learning: ${arm} cell ${cell.cellId} has invalid seed`) - } - assertReward(artifact.score, `${arm} cell ${cell.cellId} reward`) - if (artifact.probes.length === 0) { - throw new Error(`cannot compare memory learning: ${arm} cell ${cell.cellId} has no probes`) - } - for (const probe of artifact.probes) assertProbe(arm, cell.cellId, probe) - const probeMean = mean(artifact.probes.map((probe) => probe.score)) - if (Math.abs(probeMean - artifact.score) > 1e-12) { - throw new Error( - `cannot compare memory learning: ${arm} cell ${cell.cellId} reward does not equal its probe mean`, - ) - } - const identity = cellIdentity(artifact, cell.rep, cell.seed) - if (byIdentity.has(identity)) { - throw new Error(`cannot compare memory learning: ${arm} run duplicates identity ${identity}`) - } - byIdentity.set(identity, { cell, artifact, artifactRef: canonicalDigest(artifact) }) - } - const missing = [...expectedCellIds].filter((cellId) => !seenCellIds.has(cellId)) - if (missing.length > 0) { - throw new Error( - `cannot compare memory learning: ${arm} run is missing cells ${missing.sort().join(', ')}`, - ) - } - return byIdentity -} - -function assertArtifactIdentity( - arm: 'stateful' | 'stateless', - result: RunAgentMemoryExperimentResult, - cell: CampaignCellResult, - artifact: AgentMemorySequenceArtifact, -): void { - assertNonEmpty(artifact.candidateId, `${arm} cell ${cell.cellId} candidateId`) - assertNonEmpty(artifact.sequenceId, `${arm} cell ${cell.cellId} sequenceId`) - if (!result.candidateRefs.some((candidate) => candidate.id === artifact.candidateId)) { - throw new Error( - `cannot compare memory learning: ${arm} cell ${cell.cellId} uses an undeclared candidate`, - ) - } - if (artifact.memoryMode !== arm) { - throw new Error( - `cannot compare memory learning: ${arm} cell ${cell.cellId} artifact is labeled ${String(artifact.memoryMode)}`, - ) - } - if (artifact.comparisonRef !== result.comparisonRef) { - throw new Error( - `cannot compare memory learning: ${arm} cell ${cell.cellId} comparisonRef differs from its run`, - ) - } - if (!/^sha256:[a-f0-9]{64}$/.test(artifact.branchDigest)) { - throw new Error( - `cannot compare memory learning: ${arm} cell ${cell.cellId} has invalid branch evidence`, - ) - } - const expectedScenarioId = `${stableId('candidate', artifact.candidateId)}:${artifact.sequenceId}` - if (cell.scenarioId !== expectedScenarioId) { - throw new Error( - `cannot compare memory learning: ${arm} cell ${cell.cellId} artifact identity does not match its scenario`, - ) - } -} - -function pairProbes( - stateful: IndexedMemoryLearningCell, - stateless: IndexedMemoryLearningCell, -): PairedMemoryLearningProbe[] { - const statefulProbes = indexProbes('stateful', stateful) - const statelessProbes = indexProbes('stateless', stateless) - assertSameKeys(statefulProbes, statelessProbes, `probe in cell ${stateful.cell.cellId}`) - return [...statefulProbes.entries()] - .map(([key, statefulProbe]): PairedMemoryLearningProbe => { - const statelessProbe = statelessProbes.get(key) - if (!statelessProbe) { - throw new Error( - `cannot compare memory learning: missing stateless probe ${key} in ${stateful.cell.cellId}`, - ) - } - if (statefulProbe.query !== statelessProbe.query) { - throw new Error( - `cannot compare memory learning: probe query differs for ${key} in ${stateful.cell.cellId}`, - ) - } - if (statefulProbe.retentionKey !== statelessProbe.retentionKey) { - throw new Error( - `cannot compare memory learning: retention identity differs for ${key} in ${stateful.cell.cellId}`, - ) - } - if (statefulProbe.transferKey !== statelessProbe.transferKey) { - throw new Error( - `cannot compare memory learning: transfer identity differs for ${key} in ${stateful.cell.cellId}`, - ) - } - if ( - sortedUnique(statefulProbe.applicableDimensions).join('\0') !== - sortedUnique(statelessProbe.applicableDimensions).join('\0') - ) { - throw new Error( - `cannot compare memory learning: measured dimensions differ for ${key} in ${stateful.cell.cellId}`, - ) - } - return { stateful: statefulProbe, stateless: statelessProbe } - }) - .sort((left, right) => - left.stateful.stepOrdinal !== right.stateful.stepOrdinal - ? left.stateful.stepOrdinal - right.stateful.stepOrdinal - : left.stateful.id.localeCompare(right.stateful.id), - ) -} - -function indexProbes( - arm: 'stateful' | 'stateless', - indexed: IndexedMemoryLearningCell, -): Map { - const probes = new Map() - for (const probe of indexed.artifact.probes) { - const key = probeIdentity(probe) - if (probes.has(key)) { - throw new Error( - `cannot compare memory learning: ${arm} cell ${indexed.cell.cellId} duplicates probe ${key}`, - ) - } - probes.set(key, probe) - } - return probes -} - -function assertProbe( - arm: 'stateful' | 'stateless', - cellId: string, - probe: AgentMemorySequenceProbeResult, -): void { - assertNonEmpty(probe.id, `${arm} cell ${cellId} probe id`) - assertNonEmpty(probe.stepId, `${arm} cell ${cellId} probe ${probe.id} stepId`) - assertNonEmpty(probe.query, `${arm} cell ${cellId} probe ${probe.id} query`) - if (!Number.isSafeInteger(probe.stepOrdinal) || probe.stepOrdinal < 0) { - throw new Error( - `cannot compare memory learning: ${arm} cell ${cellId} probe ${probe.id} has invalid stepOrdinal`, - ) - } - if (probe.retentionKey !== undefined) { - assertNonEmpty(probe.retentionKey, `${arm} cell ${cellId} probe ${probe.id} retentionKey`) - if (probe.retentionKey !== probe.retentionKey.trim()) { - throw new Error( - `cannot compare memory learning: ${arm} cell ${cellId} probe ${probe.id} retentionKey has surrounding whitespace`, - ) - } - } - if (probe.transferKey !== undefined) { - assertNonEmpty(probe.transferKey, `${arm} cell ${cellId} probe ${probe.id} transferKey`) - if (probe.transferKey !== probe.transferKey.trim()) { - throw new Error( - `cannot compare memory learning: ${arm} cell ${cellId} probe ${probe.id} transferKey has surrounding whitespace`, - ) - } - if (probe.retentionKey !== undefined || probe.stepOrdinal === 0) { - throw new Error( - `cannot compare memory learning: ${arm} cell ${cellId} probe ${probe.id} has invalid transfer placement`, - ) - } - } - if (!/^sha256:[a-f0-9]{64}$/.test(probe.evidenceRef)) { - throw new Error( - `cannot compare memory learning: ${arm} cell ${cellId} probe ${probe.id} has invalid evidence`, - ) - } - const expectedEvidenceSuffix = `/memory-evidence/${probe.evidenceRef.slice(7)}.json` - if (!probe.evidencePath.endsWith(expectedEvidenceSuffix)) { - throw new Error( - `cannot compare memory learning: ${arm} cell ${cellId} probe ${probe.id} evidence is missing`, - ) - } - assertReward(probe.score, `${arm} cell ${cellId} probe ${probe.id} reward`) -} - -function assertReward(value: number, label: string): void { - if (!Number.isFinite(value) || value < 0 || value > 1) { - throw new Error(`cannot compare memory learning: ${label} must be finite and between 0 and 1`) - } -} - -function assertNonEmpty(value: string, label: string): void { - if (typeof value !== 'string' || value.trim().length === 0) { - throw new Error(`cannot compare memory learning: ${label} must be a non-empty string`) - } -} - -function assertSameKeys(left: Map, right: Map, label: string): void { - const missingRight = [...left.keys()].filter((key) => !right.has(key)) - const missingLeft = [...right.keys()].filter((key) => !left.has(key)) - if (missingRight.length === 0 && missingLeft.length === 0) return - throw new Error( - `cannot compare memory learning: unmatched ${label} identities; stateful-only=${missingRight.sort().join(', ') || 'none'}; stateless-only=${missingLeft.sort().join(', ') || 'none'}`, - ) -} - -function cellIdentity(artifact: AgentMemorySequenceArtifact, rep: number, seed: number): string { - return JSON.stringify([artifact.candidateId, artifact.sequenceId, rep, seed]) -} - -function probeIdentity(probe: AgentMemorySequenceProbeResult): string { - return JSON.stringify([probe.stepOrdinal, probe.stepId, probe.id]) -} - -function comparePairedCells( - left: PairedMemoryLearningCell, - right: PairedMemoryLearningCell, -): number { - return ( - left.stateful.artifact.candidateId.localeCompare(right.stateful.artifact.candidateId) || - left.stateful.artifact.sequenceId.localeCompare(right.stateful.artifact.sequenceId) || - left.stateful.cell.rep - right.stateful.cell.rep || - left.stateful.cell.seed - right.stateful.cell.seed - ) -} - -function sortedUnique(values: readonly string[]): string[] { - return [...new Set(values)].sort() -} diff --git a/src/memory/experiment/learning.ts b/src/memory/experiment/learning.ts deleted file mode 100644 index bd51883..0000000 --- a/src/memory/experiment/learning.ts +++ /dev/null @@ -1,111 +0,0 @@ -import { join } from 'node:path' -import { - canonicalDigest, - createRunCostLedger, - fsCampaignStorage, - resolveRunDir, -} from '@tangle-network/agent-eval/campaign' -import { normalizeUsd } from '../../candidate-ranking' -import { assertImmutableRef } from '../../immutable-ref' -import { assertStoredAgentMemoryEvidence } from './learning-evidence' -import { measureAgentMemoryLearning } from './learning-metrics' -import { pairAgentMemoryLearningRuns } from './learning-pairs' -import { runAgentMemoryExperiment } from './run' -import type { - AgentMemoryLearningComparison, - CompareAgentMemoryLearningOptions, - RunAgentMemoryLearningExperimentOptions, - RunAgentMemoryLearningExperimentResult, -} from './types' -import { assertMemoryLearningSequences, assertNonEmptyString } from './validation' - -/** Compare exact matched stateful and stateless memory runs. */ -export function compareAgentMemoryLearning( - options: CompareAgentMemoryLearningOptions, -): AgentMemoryLearningComparison { - const paired = pairAgentMemoryLearningRuns(options) - return measureAgentMemoryLearning(paired, options.stateful, options.stateless) -} - -/** Run a matched stateful-versus-stateless memory experiment under one cost limit. */ -export async function runAgentMemoryLearningExperiment( - options: RunAgentMemoryLearningExperimentOptions, -): Promise { - assertNonEmptyString(options.runDir, 'memory learning experiment runDir') - options.signal?.throwIfAborted() - assertMemoryLearningSequences(options.sequences) - for (const candidate of [...options.candidates, ...(options.recoveryCandidates ?? [])]) { - assertImmutableRef(candidate.ref, `${candidate.id}: memory learning candidate ref`) - } - if (options.executeStep) { - assertImmutableRef(options.executeStepRef, 'memory learning experiment executeStepRef') - } - const storage = options.storage ?? fsCampaignStorage() - const runDir = resolveRunDir(options.runDir, options.repo) - storage.ensureDir(runDir) - const costCeiling = options.costCeiling ?? options.costLedger?.costCeilingUsd ?? 0 - const costLedger = - options.costLedger ?? createRunCostLedger({ storage, runDir, costCeilingUsd: costCeiling }) - if (costLedger.costCeilingUsd !== costCeiling) { - throw new Error( - 'memory learning experiment costCeiling must match the shared cost ledger ceiling', - ) - } - const armOrder = options.armOrder ?? 'stateful-first' - if (armOrder !== 'stateful-first' && armOrder !== 'stateless-first') { - throw new Error( - "memory learning experiment armOrder must be 'stateful-first' or 'stateless-first'", - ) - } - const { armOrder: _armOrder, ...experimentOptions } = options - const shared = { - ...experimentOptions, - experimentRunId: options.experimentRunId ?? runDir, - costCeiling, - costLedger, - } - const runArm = async (memoryMode: 'stateful' | 'stateless') => { - const result = await runAgentMemoryExperiment({ - ...shared, - memoryMode, - runDir: armRunDir(runDir, memoryMode), - }) - assertStoredAgentMemoryEvidence(storage, result, options.sequences) - return result - } - const firstMode = armOrder === 'stateful-first' ? 'stateful' : 'stateless' - const secondMode = firstMode === 'stateful' ? 'stateless' : 'stateful' - const first = await runArm(firstMode) - const second = await runArm(secondMode) - const stateful = firstMode === 'stateful' ? first : second - const stateless = firstMode === 'stateless' ? first : second - const comparison = compareAgentMemoryLearning({ stateful, stateless }) - const ledgerSummary = costLedger.summary() - const cost = { - experimentUsd: normalizeUsd(stateful.totalCostUsd + stateless.totalCostUsd), - ledgerUsd: normalizeUsd(ledgerSummary.totalCostUsd), - ceilingUsd: costLedger.costCeilingUsd, - accountingComplete: ledgerSummary.accountingComplete, - } - const report = { armOrder, comparison, cost } - const evidenceRef = canonicalDigest(report) - const comparisonDir = armRunDir(runDir, 'comparisons') - storage.ensureDir(comparisonDir) - const comparisonPath = armRunDir(comparisonDir, `${evidenceRef.slice(7)}.json`) - storage.write(comparisonPath, `${JSON.stringify({ evidenceRef, ...report }, null, 2)}\n`) - return { - armOrder, - stateful, - stateless, - comparison, - evidenceRef, - comparisonPath, - cost, - } -} - -function armRunDir(runDir: string, child: string): string { - return runDir.startsWith('mem://') - ? `${runDir.replace(/\/+$/, '')}/${child}` - : join(runDir, child) -} diff --git a/src/memory/experiment/metrics.ts b/src/memory/experiment/metrics.ts deleted file mode 100644 index 5602771..0000000 --- a/src/memory/experiment/metrics.ts +++ /dev/null @@ -1,126 +0,0 @@ -import type { CampaignResult, CostLedgerHandle } from '@tangle-network/agent-eval/campaign' -import { normalizeUsd, rankCandidates } from '../../candidate-ranking' - -export { mean } from '../../statistics' - -import { stableId } from '../../ids' -import { mean } from '../../statistics' -import type { - AgentMemoryExperimentCandidate, - AgentMemoryExperimentRankingRow, - AgentMemorySequenceArtifact, - AgentMemorySequenceScenario, -} from './types' - -export function rankAgentMemoryExperiment( - candidates: readonly AgentMemoryExperimentCandidate[], - scenarios: readonly AgentMemorySequenceScenario[], - campaign: CampaignResult, - costByCandidate: ReadonlyMap, -): AgentMemoryExperimentRankingRow[] { - const scenarioById = new Map(scenarios.map((scenario) => [scenario.id, scenario])) - const rows = candidates.map((candidate): AgentMemoryExperimentRankingRow => { - const candidateScenarios = scenarios.filter((scenario) => scenario.candidateId === candidate.id) - const cells = campaign.cells.filter( - (cell) => scenarioById.get(cell.scenarioId)?.candidateId === candidate.id, - ) - const successful = cells.filter((cell) => !cell.error && cell.artifact) - const dimensionRows = successful.map((cell) => cell.artifact.dimensions) - return { - rank: 0, - candidateId: candidate.id, - label: candidate.label ?? candidate.id, - scoreMean: mean( - cells.map((cell) => (!cell.error && cell.artifact ? cell.artifact.score : 0)), - ), - passRate: mean(cells.map((cell) => (!cell.error && cell.artifact?.passed ? 1 : 0))), - totalSequences: candidateScenarios.length, - totalCells: cells.length, - totalProbes: successful.reduce((sum, cell) => sum + cell.artifact.probes.length, 0), - cellsFailed: cells.filter((cell) => Boolean(cell.error)).length, - totalCostUsd: normalizeUsd(costByCandidate.get(candidate.id) ?? 0), - durationMs: cells.reduce((sum, cell) => sum + cell.durationMs, 0), - dimensions: meanDimensions(dimensionRows), - } - }) - return rankCandidates(rows) -} - -export function memoryExperimentCostByCandidate( - costLedger: CostLedgerHandle, - runDir: string, - scenarios: readonly AgentMemorySequenceScenario[], - candidateIdsInput: readonly string[], -): ReadonlyMap { - const candidateByScenario = new Map( - scenarios.map((scenario) => [scenario.id, scenario.candidateId]), - ) - const candidateIds = new Set(candidateIdsInput) - const sequenceIds = new Set(scenarios.map((scenario) => scenario.sequenceId)) - for (const candidateId of candidateIds) { - for (const sequenceId of sequenceIds) { - candidateByScenario.set(`${stableId('candidate', candidateId)}:${sequenceId}`, candidateId) - } - } - const totals = new Map() - for (const receipt of costLedger.list()) { - if (receipt.tags?.runDir !== runDir) continue - const scenarioCandidate = receipt.tags.scenarioId - ? candidateByScenario.get(receipt.tags.scenarioId) - : undefined - const recoveryCandidate = - receipt.tags.memoryRecovery === 'attempt' && receipt.tags.candidateId - ? receipt.tags.candidateId - : undefined - const candidateId = scenarioCandidate ?? recoveryCandidate - if (!candidateId || !candidateIds.has(candidateId)) continue - totals.set(candidateId, (totals.get(candidateId) ?? 0) + receipt.costUsd) - } - return totals -} - -export function renderAgentMemoryExperimentRanking( - rows: readonly AgentMemoryExperimentRankingRow[], - totalCostUsd: number, - unrankedRecoveryCostUsd: number, -): string { - return [ - '# Agent Memory Experiment', - '', - `- total cost: $${format(totalCostUsd)}`, - `- retired-candidate cost: $${format(unrankedRecoveryCostUsd)}`, - '', - '| rank | candidate | sequences | cells | probes | failed | score | pass rate | cost | duration ms |', - '| ---: | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |', - ...rows.map( - (row) => - `| ${row.rank} | ${row.label} | ${row.totalSequences} | ${row.totalCells} | ${row.totalProbes} | ${row.cellsFailed} | ${format(row.scoreMean)} | ${format(row.passRate)} | $${format(row.totalCostUsd)} | ${format(row.durationMs)} |`, - ), - '', - ].join('\n') -} - -export function meanDimensions(rows: readonly Record[]): Record { - const values = new Map() - for (const row of rows) { - for (const [key, value] of Object.entries(row)) { - if (!Number.isFinite(value)) continue - const bucket = values.get(key) ?? [] - bucket.push(value) - values.set(key, bucket) - } - } - return Object.fromEntries([...values].map(([key, bucket]) => [key, mean(bucket)])) -} - -export function countDimensions(rows: readonly (readonly string[])[]): Record { - const counts = new Map() - for (const row of rows) { - for (const key of new Set(row)) counts.set(key, (counts.get(key) ?? 0) + 1) - } - return Object.fromEntries(counts) -} - -function format(value: number): string { - return Number.isFinite(value) ? value.toFixed(4) : '0.0000' -} diff --git a/src/memory/experiment/probe-evaluation.ts b/src/memory/experiment/probe-evaluation.ts deleted file mode 100644 index 4585058..0000000 --- a/src/memory/experiment/probe-evaluation.ts +++ /dev/null @@ -1,53 +0,0 @@ -import { - type KnowledgeBenchmarkArtifact, - type KnowledgeBenchmarkEvaluation, - type KnowledgeMemoryBenchmarkCase, - scoreMemoryBenchmarkArtifact, -} from '../../benchmarks/index' -import type { - AgentMemorySequence, - AgentMemorySequenceProbe, - AgentMemorySequenceStep, -} from './types' - -export interface AgentMemoryProbeScoringEvidence extends KnowledgeBenchmarkArtifact { - answer: string - rememberedFacts: readonly string[] - citedEventIds: readonly string[] - usedMemoryIds: readonly string[] - actorIds: readonly string[] -} - -export function scoreAgentMemoryProbe( - sequence: AgentMemorySequence, - step: AgentMemorySequenceStep, - probe: AgentMemorySequenceProbe, - evidence: AgentMemoryProbeScoringEvidence, -): KnowledgeBenchmarkEvaluation { - return scoreMemoryBenchmarkArtifact(memoryProbeBenchmarkCase(sequence, step, probe), evidence) -} - -function memoryProbeBenchmarkCase( - sequence: AgentMemorySequence, - step: AgentMemorySequenceStep, - probe: AgentMemorySequenceProbe, -): KnowledgeMemoryBenchmarkCase { - return { - id: `${sequence.id}:${step.id}:${probe.id}`, - family: sequence.family, - taskKind: probe.taskKind ?? 'memory-recall', - split: sequence.split, - events: [], - prompt: probe.query, - requiredFacts: - probe.requiredFacts && probe.requiredFacts.length > 0 - ? probe.requiredFacts - : probe.referenceAnswer - ? [{ id: `${probe.id}:reference`, anyOf: [probe.referenceAnswer] }] - : undefined, - forbiddenFacts: probe.forbiddenFacts, - expectedEventIds: probe.expectedEventIds, - expectedActorIds: probe.expectedActorIds, - referenceAnswer: probe.referenceAnswer, - } -} diff --git a/src/memory/experiment/recovery.ts b/src/memory/experiment/recovery.ts deleted file mode 100644 index 9370281..0000000 --- a/src/memory/experiment/recovery.ts +++ /dev/null @@ -1,515 +0,0 @@ -import { canonicalJson } from '@tangle-network/agent-eval' -import { - type CampaignStorage, - type CostLedgerHandle, - canonicalDigest, -} from '@tangle-network/agent-eval/campaign' -import { stableId } from '../../ids' -import { - appendAttemptJournalEvent, - assertNoInterruptedPaidCalls, - hasSettledPaidCall, - readActiveAttemptJournal, - reconcileInterruptedMemoryPaidCalls, - reserveRecoveryAttempts, -} from '../attempt-log' -import { type AgentMemoryBranch, createAgentMemoryBranch } from '../branch' -import { - createBoundedMemoryAdapter, - createMemoryExecutionPool, - memoryRecoveryDelayMs, - resolveMemoryCleanupTimeoutMs, - runBoundedMemoryLifecycle, - sleepForMemoryRecovery, -} from '../lifecycle' -import type { AgentMemoryAdapter } from '../types' -import { createAgentMemoryCostRecorder } from './cost' -import { - AgentMemoryCleanupError, - clearSequenceScopes, - memoryExperimentBaseScope, - sequenceCleanupScopes, - trackExternalMemoryCalls, -} from './runtime' -import type { - AgentMemoryAttemptEvent, - AgentMemoryExperimentCandidate, - AgentMemorySequence, - OwnedMemoryExperimentRunLease, - RunAgentMemoryExperimentOptions, -} from './types' - -export async function recoverAbandonedMemoryAttempts(input: { - options: RunAgentMemoryExperimentOptions - storage: CampaignStorage - runDir: string - attemptLogPath: string - candidateById: ReadonlyMap - sequenceById: ReadonlyMap - lease: OwnedMemoryExperimentRunLease - maxConcurrency: number - costLedger: CostLedgerHandle - maxRecoveryAttempts: number - recoveryLogPath: string - maxRecoveryRetriesPerAttempt: number -}): Promise { - input.options.signal?.throwIfAborted() - let attempts = readActiveMemoryAttempts(input.storage, input.attemptLogPath) - if (attempts.length > input.maxRecoveryAttempts) { - throw new Error( - `memory experiment has ${attempts.length} unfinished attempts; maxRecoveryAttempts is ${input.maxRecoveryAttempts}`, - ) - } - for (const attempt of attempts) { - const candidate = input.candidateById.get(attempt.candidateId) - if (!candidate) { - throw new Error( - `cannot recover memory branch '${attempt.branchId}': candidate '${attempt.candidateId}' is missing; pass it in recoveryCandidates`, - ) - } - assertMemoryAttemptCandidateMatches(attempt, candidate) - const sequence = input.sequenceById.get(attempt.sequenceId) - if (!sequence) { - throw new Error( - `cannot recover memory branch '${attempt.branchId}': sequence '${attempt.sequenceId}' is missing`, - ) - } - assertMemoryAttemptSequenceMatches(attempt, sequence) - if ((input.options.cleanupBranches ?? true) !== attempt.cleanupBranches) { - throw new Error(`cannot recover memory branch '${attempt.branchId}': cleanupBranches changed`) - } - } - - reconcileInterruptedMemoryPaidCalls(input.costLedger) - assertNoInterruptedPaidCalls(input.costLedger, 'memory experiment recovery') - - for (const attempt of attempts) { - const candidate = input.candidateById.get(attempt.candidateId)! - const executionCostUsd = candidate.externalCostUsdPerSequence ?? 0 - if ( - executionCostUsd > 0 && - !hasSettledPaidCall(input.costLedger, memoryAttemptCostCallId(attempt, 'execute', 0)) - ) { - await input.lease.assertOwned() - appendMemoryAttemptEvent(input.storage, input.attemptLogPath, { - ...attempt, - status: 'cleaned', - recovery: true, - recordedAt: (input.options.now ?? (() => new Date()))().toISOString(), - }) - } - } - attempts = readActiveMemoryAttempts(input.storage, input.attemptLogPath) - const recoveryGenerations = new Map() - for (const attempt of attempts.sort((left, right) => - left.branchId.localeCompare(right.branchId), - )) { - await input.lease.assertOwned() - const reserved = reserveRecoveryAttempts({ - storage: input.storage, - path: input.recoveryLogPath, - attemptIds: [attempt.branchId], - maxRetriesPerAttempt: input.maxRecoveryRetriesPerAttempt, - label: 'memory recovery attempt log', - now: input.options.now, - }) - recoveryGenerations.set(attempt.branchId, reserved.get(attempt.branchId)!) - } - const pool = createMemoryExecutionPool(input.maxConcurrency) - const settled = await Promise.allSettled( - attempts.map((attempt) => - pool.run(async () => { - input.options.signal?.throwIfAborted() - await input.lease.assertOwned() - const candidate = input.candidateById.get(attempt.candidateId) - if (!candidate) { - throw new Error( - `cannot recover memory branch '${attempt.branchId}': candidate '${attempt.candidateId}' is missing; pass it in recoveryCandidates`, - ) - } - assertMemoryAttemptCandidateMatches(attempt, candidate) - const sequence = input.sequenceById.get(attempt.sequenceId) - if (!sequence) { - throw new Error( - `cannot recover memory branch '${attempt.branchId}': sequence '${attempt.sequenceId}' is missing`, - ) - } - assertMemoryAttemptSequenceMatches(attempt, sequence) - if ((input.options.cleanupBranches ?? true) !== attempt.cleanupBranches) { - throw new Error( - `cannot recover memory branch '${attempt.branchId}': cleanupBranches changed`, - ) - } - const recoveryCostUsd = candidate.externalRecoveryCostUsdPerAttempt ?? 0 - const recoveryGeneration = recoveryGenerations.get(attempt.branchId) - if (recoveryGeneration === undefined) { - throw new Error(`missing recovery generation for memory branch '${attempt.branchId}'`) - } - let externalRecoveryAttempted = false - const costRecorder = createAgentMemoryCostRecorder({ - candidateRef: candidate.ref, - maximumCostUsd: recoveryCostUsd, - operation: `${candidate.id}: memory recovery`, - }) - const recover = async (): Promise => { - await recoverMemoryAttempt({ - options: input.options, - candidate, - sequence, - attempt, - lease: input.lease, - onExternalCall: () => { - externalRecoveryAttempted = true - }, - recordExternalCost: (actualCostUsd) => { - externalRecoveryAttempted = true - costRecorder.record(actualCostUsd) - }, - }) - await input.lease.assertOwned() - appendMemoryAttemptEvent(input.storage, input.attemptLogPath, { - ...attempt, - status: 'cleaned', - recovery: true, - recordedAt: (input.options.now ?? (() => new Date()))().toISOString(), - }) - } - if (recoveryCostUsd === 0) { - await recover() - } else { - const tags = memoryRecoveryCostTags(input.runDir, candidate.id, attempt.branchId) - const paid = await input.costLedger.runPaidCall({ - callId: memoryAttemptCostCallId(attempt, 'recovery', recoveryGeneration), - channel: 'driver', - phase: `${input.options.costPhase ?? 'memory.experiment'}.recovery`, - actor: `agent-knowledge:memory-recovery:${candidate.id}`, - model: candidate.ref, - tags, - maximumCharge: { externallyEnforcedMaximumUsd: recoveryCostUsd }, - execute: recover, - receipt: () => costRecorder.receipt(externalRecoveryAttempted), - receiptFromError: () => costRecorder.receipt(externalRecoveryAttempted), - }) - if (!paid.succeeded) throw paid.error - } - }), - ), - ) - const failures = settled.flatMap((result) => - result.status === 'rejected' ? [result.reason] : [], - ) - if (failures.length === 1) throw failures[0] - if (failures.length > 1) { - throw new AggregateError(failures, 'multiple abandoned memory branches failed recovery') - } -} - -async function recoverMemoryAttempt(input: { - options: RunAgentMemoryExperimentOptions - candidate: AgentMemoryExperimentCandidate - sequence: AgentMemorySequence - attempt: AgentMemoryAttemptEvent - lease: OwnedMemoryExperimentRunLease - onExternalCall(): void - recordExternalCost(actualCostUsd: number): void -}): Promise { - const { options, candidate, sequence, attempt, lease, onExternalCall, recordExternalCost } = input - const cleanupBranches = attempt.cleanupBranches - const cleanupTimeoutMs = resolveMemoryCleanupTimeoutMs( - options.cleanupTimeoutMs, - `${candidate.id}: abandoned branch recovery`, - ) - let rawAdapter: AgentMemoryAdapter | undefined - let adapter: AgentMemoryAdapter | undefined - let memory: AgentMemoryBranch | undefined - let primaryError: unknown - try { - options.signal?.throwIfAborted() - const recovered = await createBoundedMemoryAdapter({ - operation: `${candidate.id}: recovery adapter creation`, - timeoutMs: cleanupTimeoutMs, - signal: options.signal, - create: (signal) => - candidate.createAdapter({ - branchId: attempt.branchId, - purpose: 'recovery', - signal, - maximumCostUsd: candidate.externalRecoveryCostUsdPerAttempt ?? 0, - markExternalCall: onExternalCall, - recordExternalCost, - }), - dispose: candidate.disposeAdapter - ? async (created) => { - onExternalCall() - await candidate.disposeAdapter?.(created) - } - : undefined, - }) - options.signal?.throwIfAborted() - await lease.assertOwned() - if (recovered === null) return - rawAdapter = recovered - adapter = trackExternalMemoryCalls(recovered, onExternalCall) - const recoveryDelayMs = memoryRecoveryDelayMs(adapter) - await sleepForMemoryRecovery( - recoveryDelayMs, - () => lease.assertOwned(), - cleanupTimeoutMs, - `${candidate.id}: abandoned branch recovery visibility wait`, - options.signal, - ) - if (cleanupBranches) { - if (!adapter.clear) { - throw new Error( - `${candidate.id}: abandoned branch recovery requires an adapter with scoped clear support`, - ) - } - memory = createAgentMemoryBranch({ - adapter, - branchId: attempt.branchId, - lifetime: 'attempt', - policy: candidate.policy, - allowedWriteScopes: sequenceCleanupScopes(sequence), - baseScope: memoryExperimentBaseScope(candidate), - }) - await runBoundedMemoryLifecycle({ - operation: `${candidate.id}: abandoned branch cleanup`, - timeoutMs: cleanupTimeoutMs, - resource: adapter, - run: () => clearSequenceScopes(memory!, sequence), - }) - await lease.assertOwned() - } - } catch (error) { - primaryError = error - } - - const cleanupErrors: unknown[] = [] - let cleanupOwned = true - try { - await lease.assertOwned() - } catch (error) { - cleanupOwned = false - cleanupErrors.push(error) - } - if (adapter) { - try { - await runBoundedMemoryLifecycle({ - operation: `${candidate.id}: recovery adapter close`, - timeoutMs: cleanupTimeoutMs, - resource: adapter, - run: async () => { - if (memory && cleanupOwned) { - await memory.close?.() - } else { - if (cleanupOwned && adapter!.flush) { - await adapter!.flush() - } - await adapter!.close?.() - } - }, - }) - } catch (error) { - cleanupErrors.push(error) - } - if (cleanupOwned) { - try { - if (candidate.disposeAdapter) onExternalCall() - await runBoundedMemoryLifecycle({ - operation: `${candidate.id}: recovery adapter disposal`, - timeoutMs: cleanupTimeoutMs, - resource: adapter, - run: () => candidate.disposeAdapter?.(rawAdapter!), - }) - } catch (error) { - cleanupErrors.push(error) - } - } - } else { - cleanupErrors.push( - new Error( - `${candidate.id}: recovery adapter creation failed before cleanup could be confirmed`, - ), - ) - } - if (!cleanupOwned || primaryError || cleanupErrors.length > 0) { - throw new AgentMemoryCleanupError( - [...(primaryError ? [primaryError] : []), ...cleanupErrors], - `${candidate.id}: abandoned memory branch '${attempt.branchId}' recovery failed`, - ) - } -} - -export function memoryAttemptCostCallId( - attempt: AgentMemoryAttemptEvent, - purpose: 'execute' | 'recovery', - generation: number, -): string { - return stableId( - 'memory_cost_call', - canonicalJson({ - purpose, - generation, - branchId: attempt.branchId, - candidateId: attempt.candidateId, - candidateRef: attempt.candidateRef, - externalCostUsdPerSequence: attempt.externalCostUsdPerSequence, - externalRecoveryCostUsdPerAttempt: attempt.externalRecoveryCostUsdPerAttempt, - sequenceId: attempt.sequenceId, - sequenceRef: attempt.sequenceRef, - rep: attempt.rep, - seed: attempt.seed, - }), - ) -} - -function memoryRecoveryCostTags( - runDir: string, - candidateId: string, - branchId: string, -): Record { - return { - runDir, - candidateId, - branchId, - memoryRecovery: 'attempt', - } -} - -export function memoryAttemptEvent(input: { - status: AgentMemoryAttemptEvent['status'] - branchId: string - candidate: AgentMemoryExperimentCandidate - sequence: AgentMemorySequence - rep: number - seed: number - recovery: boolean - cleanupBranches?: boolean - now?: () => Date -}): AgentMemoryAttemptEvent { - return { - status: input.status, - branchId: input.branchId, - candidateId: input.candidate.id, - candidateRef: input.candidate.ref, - sequenceId: input.sequence.id, - sequenceRef: canonicalDigest(input.sequence), - rep: input.rep, - seed: input.seed, - cleanupBranches: input.cleanupBranches ?? true, - externalCostUsdPerSequence: input.candidate.externalCostUsdPerSequence ?? 0, - externalRecoveryCostUsdPerAttempt: input.candidate.externalRecoveryCostUsdPerAttempt ?? 0, - recordedAt: (input.now ?? (() => new Date()))().toISOString(), - recovery: input.recovery, - } -} - -export function appendMemoryAttemptEvent( - storage: CampaignStorage, - path: string, - event: AgentMemoryAttemptEvent, -): void { - appendAttemptJournalEvent({ - storage, - path, - event, - label: 'memory attempt log', - }) -} - -function readActiveMemoryAttempts( - storage: CampaignStorage, - path: string, -): AgentMemoryAttemptEvent[] { - return readActiveAttemptJournal({ - storage, - path, - label: 'memory attempt log', - parse: parseMemoryAttemptEvent, - id: (event) => event.branchId, - sameAttempt: sameMemoryAttempt, - }) -} - -function parseMemoryAttemptEvent( - value: unknown, - path: string, - line: number, -): AgentMemoryAttemptEvent { - const event = value as Partial | null - const valid = - typeof event === 'object' && - event !== null && - (event.status === 'started' || event.status === 'cleaned') && - typeof event.branchId === 'string' && - event.branchId.length > 0 && - typeof event.candidateId === 'string' && - event.candidateId.length > 0 && - typeof event.candidateRef === 'string' && - event.candidateRef.length > 0 && - typeof event.sequenceId === 'string' && - event.sequenceId.length > 0 && - typeof event.sequenceRef === 'string' && - /^sha256:[a-f0-9]{64}$/.test(event.sequenceRef) && - Number.isSafeInteger(event.rep) && - Number.isSafeInteger(event.seed) && - typeof event.cleanupBranches === 'boolean' && - typeof event.externalCostUsdPerSequence === 'number' && - Number.isFinite(event.externalCostUsdPerSequence) && - event.externalCostUsdPerSequence >= 0 && - typeof event.externalRecoveryCostUsdPerAttempt === 'number' && - Number.isFinite(event.externalRecoveryCostUsdPerAttempt) && - event.externalRecoveryCostUsdPerAttempt >= 0 && - typeof event.recordedAt === 'string' && - !Number.isNaN(Date.parse(event.recordedAt)) && - typeof event.recovery === 'boolean' - if (!valid) throw new Error(`invalid memory attempt event in '${path}' line ${line}`) - return value as AgentMemoryAttemptEvent -} - -function sameMemoryAttempt(left: AgentMemoryAttemptEvent, right: AgentMemoryAttemptEvent): boolean { - return ( - left.branchId === right.branchId && - left.candidateId === right.candidateId && - left.candidateRef === right.candidateRef && - left.sequenceId === right.sequenceId && - left.sequenceRef === right.sequenceRef && - left.rep === right.rep && - left.seed === right.seed && - left.cleanupBranches === right.cleanupBranches && - left.externalCostUsdPerSequence === right.externalCostUsdPerSequence && - left.externalRecoveryCostUsdPerAttempt === right.externalRecoveryCostUsdPerAttempt - ) -} - -function assertMemoryAttemptSequenceMatches( - attempt: AgentMemoryAttemptEvent, - sequence: AgentMemorySequence, -): void { - if (canonicalDigest(sequence) !== attempt.sequenceRef) { - throw new Error( - `cannot recover memory branch '${attempt.branchId}': sequence '${attempt.sequenceId}' changed; restore the recorded sequence before cleanup`, - ) - } -} - -function assertMemoryAttemptCandidateMatches( - attempt: AgentMemoryAttemptEvent, - candidate: AgentMemoryExperimentCandidate, -): void { - if (candidate.ref !== attempt.candidateRef) { - throw new Error( - `cannot recover memory branch '${attempt.branchId}': candidate ref changed from '${attempt.candidateRef}' to '${candidate.ref}'`, - ) - } - const executionCost = candidate.externalCostUsdPerSequence ?? 0 - const recoveryCost = candidate.externalRecoveryCostUsdPerAttempt ?? 0 - if ( - executionCost !== attempt.externalCostUsdPerSequence || - recoveryCost !== attempt.externalRecoveryCostUsdPerAttempt - ) { - throw new Error( - `cannot recover memory branch '${attempt.branchId}': candidate cost settings changed; start a new run or restore the recorded costs`, - ) - } -} diff --git a/src/memory/experiment/run.ts b/src/memory/experiment/run.ts deleted file mode 100644 index 7756cc9..0000000 --- a/src/memory/experiment/run.ts +++ /dev/null @@ -1,375 +0,0 @@ -import { join } from 'node:path' -import { canonicalJson } from '@tangle-network/agent-eval' -import { - type CampaignResult, - type CampaignStorage, - createRunCostLedger, - fsCampaignStorage, - resolveRunDir, - runCampaign, -} from '@tangle-network/agent-eval/campaign' -import { normalizeUsd } from '../../candidate-ranking' -import { stableId } from '../../ids' -import { DEFAULT_MEMORY_RECOVERY_RETRIES_PER_ATTEMPT } from '../attempt-log' -import { - createMemoryExecutionPool, - MEMORY_CAMPAIGN_DISPATCH_SHUTDOWN_TIMEOUT_MS, - resolveMemoryCleanupTimeoutMs, -} from '../lifecycle' -import { acquireAgentMemoryRunLease } from '../run-control' -import { agentMemorySequenceJudge, buildAgentMemorySequenceScenarios } from './cases' -import { AgentMemoryAdapterCapabilityError, runSequenceCell } from './cell' -import { - MEMORY_EXPERIMENT_IMPLEMENTATION_REF, - memoryExperimentComparisonRef, - resolveAgentMemoryMode, -} from './comparison-ref' -import { - memoryExperimentCostByCandidate, - rankAgentMemoryExperiment, - renderAgentMemoryExperimentRanking, -} from './metrics' -import { recoverAbandonedMemoryAttempts } from './recovery' -import { AgentMemoryCleanupError } from './runtime' -import type { - AgentMemoryExperimentComparisonRef, - AgentMemoryMode, - AgentMemorySequenceArtifact, - AgentMemorySequenceScenario, - OwnedMemoryExperimentRunLease, - RunAgentMemoryExperimentOptions, - RunAgentMemoryExperimentResult, -} from './types' -import { assertMemorySequences, assertNonEmptyString, assertUnique } from './validation' - -/** Runs ordered, branch-isolated memory histories across candidate systems. */ -export async function runAgentMemoryExperiment( - options: RunAgentMemoryExperimentOptions, -): Promise { - options.signal?.throwIfAborted() - assertNonEmptyString(options.experimentId, 'memory experiment experimentId') - assertNonEmptyString(options.runDir, 'memory experiment runDir') - if (options.experimentRunId !== undefined) { - assertNonEmptyString(options.experimentRunId, 'memory experiment experimentRunId') - } - resolveAgentMemoryMode(options.memoryMode) - if (options.sequences.length === 0) throw new Error('memory experiment requires sequences') - if (options.candidates.length === 0) throw new Error('memory experiment requires candidates') - if (options.executeStep && !options.executeStepRef) { - throw new Error('memory experiment executeStepRef is required when executeStep is configured') - } - assertUnique( - options.sequences.map((sequence) => sequence.id), - 'sequence', - ) - assertUnique( - [...options.candidates, ...(options.recoveryCandidates ?? [])].map((candidate) => candidate.id), - 'candidate', - ) - assertMemorySequences(options.sequences) - for (const candidate of [...options.candidates, ...(options.recoveryCandidates ?? [])]) { - if (options.cleanupBranches === false && !candidate.disposeAdapter) { - throw new Error( - `${candidate.id}: cleanupBranches=false requires disposeAdapter to delete isolated external state`, - ) - } - if ( - candidate.externalCostUsdPerSequence !== undefined && - (!Number.isFinite(candidate.externalCostUsdPerSequence) || - candidate.externalCostUsdPerSequence < 0) - ) { - throw new Error( - `${candidate.id}: externalCostUsdPerSequence must be a non-negative finite number`, - ) - } - if ( - candidate.externalRecoveryCostUsdPerAttempt !== undefined && - (!Number.isFinite(candidate.externalRecoveryCostUsdPerAttempt) || - candidate.externalRecoveryCostUsdPerAttempt < 0) - ) { - throw new Error( - `${candidate.id}: externalRecoveryCostUsdPerAttempt must be a non-negative finite number`, - ) - } - if ( - candidate.externalCostAccounting !== undefined && - candidate.externalCostAccounting !== 'exact' - ) { - throw new Error(`${candidate.id}: externalCostAccounting must be 'exact'`) - } - if ( - ((candidate.externalCostUsdPerSequence ?? 0) > 0 || - (candidate.externalRecoveryCostUsdPerAttempt ?? 0) > 0) && - candidate.externalCostAccounting !== 'exact' - ) { - throw new Error(`${candidate.id}: positive external charges require exact cost accounting`) - } - assertNonEmptyString(candidate.ref, `${candidate.id} ref`) - } - - const storage = options.storage ?? fsCampaignStorage() - const runDir = resolveRunDir(options.runDir, options.repo) - if (!storage.append) { - throw new Error('memory experiment requires CampaignStorage.append for durable attempt state') - } - resolveMemoryCleanupTimeoutMs(options.cleanupTimeoutMs, 'memory experiment') - const maxRecoveryAttempts = options.maxRecoveryAttempts ?? 1_000 - if (!Number.isSafeInteger(maxRecoveryAttempts) || maxRecoveryAttempts <= 0) { - throw new Error('memory experiment maxRecoveryAttempts must be a positive safe integer') - } - const maxRecoveryRetriesPerAttempt = - options.maxRecoveryRetriesPerAttempt ?? DEFAULT_MEMORY_RECOVERY_RETRIES_PER_ATTEMPT - if (!Number.isSafeInteger(maxRecoveryRetriesPerAttempt) || maxRecoveryRetriesPerAttempt <= 0) { - throw new Error( - 'memory experiment maxRecoveryRetriesPerAttempt must be a positive safe integer', - ) - } - storage.ensureDir(runDir) - const lease = await acquireAgentMemoryRunLease({ - experimentId: options.experimentId, - runDir, - storage, - customStorage: options.storage !== undefined, - lockFileName: 'memory-experiment.lock', - label: 'memory experiment', - controllerMode: options.controllerMode, - acquireRunLease: options.acquireRunLease, - }) - let result: RunAgentMemoryExperimentResult | undefined - let primaryError: unknown - try { - await lease.assertOwned() - result = await runOwnedAgentMemoryExperiment(options, storage, runDir, lease) - } catch (error) { - primaryError = error - } - let releaseError: unknown - try { - await lease.release() - } catch (error) { - releaseError = error - } - if (primaryError && releaseError) { - throw new AggregateError( - [primaryError, releaseError], - 'memory experiment failed and its controller lease could not be released', - ) - } - if (primaryError) throw primaryError - if (releaseError) throw releaseError - if (!result) throw new Error('memory experiment produced no result') - return result -} - -async function runOwnedAgentMemoryExperiment( - options: RunAgentMemoryExperimentOptions, - storage: CampaignStorage, - runDir: string, - lease: OwnedMemoryExperimentRunLease, -): Promise { - const memoryMode = resolveAgentMemoryMode(options.memoryMode) - const comparisonRef = memoryExperimentComparisonRef(options, runDir) - const runIdentity = stableId( - 'memory_run', - canonicalJson({ - experimentId: options.experimentId, - experimentRunId: options.experimentRunId ?? runDir, - memoryMode, - }), - ) - const maxConcurrency = options.maxConcurrency ?? 2 - if (!Number.isSafeInteger(maxConcurrency) || maxConcurrency <= 0) { - throw new Error('memory experiment maxConcurrency must be a positive safe integer') - } - const executionPool = createMemoryExecutionPool(maxConcurrency) - const dispatchedExecutions: Promise[] = [] - const candidateById = new Map(options.candidates.map((candidate) => [candidate.id, candidate])) - const recoveryCandidateById = new Map( - [...options.candidates, ...(options.recoveryCandidates ?? [])].map((candidate) => [ - candidate.id, - candidate, - ]), - ) - const sequenceById = new Map(options.sequences.map((sequence) => [sequence.id, sequence])) - const scenarios = buildAgentMemorySequenceScenarios(options.sequences, options.candidates) - const attemptLogPath = join(runDir, 'memory-attempts.jsonl') - const recoveryLogPath = join(runDir, 'memory-recovery-attempts.jsonl') - const costCeiling = options.costCeiling ?? options.costLedger?.costCeilingUsd ?? 0 - const costLedger = - options.costLedger ?? - createRunCostLedger({ - storage, - runDir, - costCeilingUsd: costCeiling, - }) - if (costLedger.costCeilingUsd !== costCeiling) { - throw new Error('memory experiment costCeiling must match the shared cost ledger ceiling') - } - storage.ensureDir(runDir) - await recoverAbandonedMemoryAttempts({ - options, - storage, - runDir, - attemptLogPath, - candidateById: recoveryCandidateById, - sequenceById, - lease, - maxConcurrency, - costLedger, - maxRecoveryAttempts: options.maxRecoveryAttempts ?? 1_000, - recoveryLogPath, - maxRecoveryRetriesPerAttempt: - options.maxRecoveryRetriesPerAttempt ?? DEFAULT_MEMORY_RECOVERY_RETRIES_PER_ATTEMPT, - }) - options.signal?.throwIfAborted() - await lease.assertOwned() - let campaign: CampaignResult | undefined - let campaignError: unknown - let settledExecutions: PromiseSettledResult[] = [] - try { - campaign = await runCampaign({ - scenarios, - signal: options.signal, - dispatch: (scenario, context) => { - const candidate = candidateById.get(scenario.candidateId) - if (!candidate) throw new Error(`unknown memory candidate ${scenario.candidateId}`) - const operation = executionPool.run(() => - runSequenceCell({ - options, - candidate, - scenario, - context, - runIdentity, - storage, - attemptLogPath, - lease, - memoryMode, - comparisonRef, - }), - ) - dispatchedExecutions.push(operation) - return operation - }, - dispatchRef: memoryExperimentDispatchRef(memoryMode, comparisonRef), - judges: [agentMemorySequenceJudge()], - runDir, - storage, - seed: options.seed, - reps: options.reps, - resumable: options.resumable, - costCeiling, - costLedger, - costPhase: options.costPhase, - maxConcurrency, - dispatchTimeoutMs: options.dispatchTimeoutMs, - dispatchShutdownTimeoutMs: MEMORY_CAMPAIGN_DISPATCH_SHUTDOWN_TIMEOUT_MS, - expectUsage: 'off', - now: options.now, - }) - } catch (error) { - campaignError = error - } finally { - settledExecutions = await Promise.allSettled(dispatchedExecutions) - } - const cleanupFailures = settledExecutions.flatMap((settled) => - settled.status === 'rejected' && settled.reason instanceof AgentMemoryCleanupError - ? [settled.reason] - : [], - ) - const capabilityFailures = settledExecutions.flatMap((settled) => - settled.status === 'rejected' && settled.reason instanceof AgentMemoryAdapterCapabilityError - ? [settled.reason] - : [], - ) - const terminalFailures = [...cleanupFailures, ...capabilityFailures] - if (campaignError && terminalFailures.length > 0) { - throw new AggregateError( - [campaignError, ...terminalFailures], - 'memory experiment failed with terminal dispatch errors', - ) - } - if (campaignError) throw campaignError - if (cleanupFailures.length > 0 && capabilityFailures.length > 0) { - throw new AggregateError( - terminalFailures, - 'memory experiment adapter requirements and provider cleanup failed', - ) - } - if (cleanupFailures.length > 0) { - throw new AggregateError(cleanupFailures, 'memory experiment cleanup failed after dispatch') - } - if (capabilityFailures.length === 1) throw capabilityFailures[0] - if (capabilityFailures.length > 1) { - throw new AggregateError(capabilityFailures, 'memory experiment adapter requirements failed') - } - if (!campaign) throw new Error('memory experiment produced no campaign result') - await lease.assertOwned() - const costByCandidate = memoryExperimentCostByCandidate( - costLedger, - campaign.runDir, - scenarios, - [...options.candidates, ...(options.recoveryCandidates ?? [])].map((candidate) => candidate.id), - ) - const unrankedRecoveryCostUsd = normalizeUsd( - (options.recoveryCandidates ?? []).reduce( - (sum, candidate) => sum + (costByCandidate.get(candidate.id) ?? 0), - 0, - ), - ) - const totalCostUsd = normalizeUsd( - [...costByCandidate.values()].reduce((sum, cost) => sum + cost, 0), - ) - const rows = rankAgentMemoryExperiment(options.candidates, scenarios, campaign, costByCandidate) - const rankingJsonPath = join(campaign.runDir, 'memory-experiment-ranking.json') - const rankingMarkdownPath = join(campaign.runDir, 'memory-experiment-ranking.md') - storage.write( - rankingJsonPath, - `${JSON.stringify( - { - experimentId: options.experimentId, - memoryMode, - comparisonRef, - candidateRefs: options.candidates.map(({ id, ref }) => ({ id, ref })), - executionRef: options.executeStepRef ?? 'fixtures', - totalCostUsd, - unrankedRecoveryCostUsd, - rows, - }, - null, - 2, - )}\n`, - ) - storage.write( - rankingMarkdownPath, - renderAgentMemoryExperimentRanking(rows, totalCostUsd, unrankedRecoveryCostUsd), - ) - return { - memoryMode, - comparisonRef, - candidateRefs: options.candidates.map(({ id, ref }) => ({ id, ref })), - executionRef: options.executeStepRef ?? 'fixtures', - campaign, - rows, - totalCostUsd, - unrankedRecoveryCostUsd, - leaderCandidateId: rows.find((row) => row.cellsFailed === 0)?.candidateId, - rankingJsonPath, - rankingMarkdownPath, - attemptLogPath, - recoveryLogPath, - } -} - -function memoryExperimentDispatchRef( - memoryMode: AgentMemoryMode, - comparisonRef: AgentMemoryExperimentComparisonRef, -): string { - return stableId( - 'memory_experiment', - canonicalJson({ - implementationRef: MEMORY_EXPERIMENT_IMPLEMENTATION_REF, - memoryMode, - comparisonRef, - }), - ) -} diff --git a/src/memory/experiment/runtime.ts b/src/memory/experiment/runtime.ts deleted file mode 100644 index f9f4c0c..0000000 --- a/src/memory/experiment/runtime.ts +++ /dev/null @@ -1,98 +0,0 @@ -import type { AgentMemoryBranch } from '../branch' -import type { AgentMemoryAdapter, AgentMemoryScope } from '../types' -import type { AgentMemoryExperimentCandidate, AgentMemorySequence } from './types' -import { normalizeCleanupScope } from './validation' - -export class AgentMemoryCleanupError extends AggregateError { - constructor(errors: Iterable, message: string) { - super(errors, message) - this.name = 'AgentMemoryCleanupError' - } -} - -export function trackExternalMemoryCalls( - adapter: AgentMemoryAdapter, - onExternalCall: () => void, -): AgentMemoryAdapter { - return { - id: adapter.id, - branchIsolation: adapter.branchIsolation, - search(query, options) { - onExternalCall() - return adapter.search.call(adapter, query, options) - }, - getContext(query, options) { - onExternalCall() - return adapter.getContext.call(adapter, query, options) - }, - write(input) { - onExternalCall() - return adapter.write.call(adapter, input) - }, - ...(adapter.clear - ? { - clear(scope?: AgentMemoryScope) { - onExternalCall() - return adapter.clear!.call(adapter, scope) - }, - } - : {}), - ...(adapter.flush - ? { - flush() { - onExternalCall() - return adapter.flush!.call(adapter) - }, - } - : {}), - ...(adapter.close - ? { - close() { - onExternalCall() - return adapter.close!.call(adapter) - }, - } - : {}), - } -} - -export function mergeScopes(base?: AgentMemoryScope, extra?: AgentMemoryScope): AgentMemoryScope { - return { - ...(base ?? {}), - ...(extra ?? {}), - tags: { ...(base?.tags ?? {}), ...(extra?.tags ?? {}) }, - } -} - -export function memoryExperimentBaseScope( - candidate: Pick, -): AgentMemoryScope { - return mergeScopes(candidate.baseScope) -} - -export function sequenceCleanupScopes(sequence: AgentMemorySequence): AgentMemoryScope[] { - const scopes = new Map() - for (const scope of sequence.cleanupScopes ?? []) { - const normalized = normalizeCleanupScope(scope) - scopes.set(JSON.stringify(normalized), normalized) - } - for (const step of sequence.steps) { - const candidates = [ - ...(step.scope ? [step.scope] : []), - ...(step.writes ?? []).map((write) => mergeScopes(step.scope, write.scope)), - ...(step.probes ?? []).map((probe) => mergeScopes(step.scope, probe.scope)), - ] - for (const scope of candidates) { - const normalized = normalizeCleanupScope(scope) - scopes.set(JSON.stringify(normalized), normalized) - } - } - return [...scopes.values()] -} - -export async function clearSequenceScopes( - memory: AgentMemoryBranch, - sequence: AgentMemorySequence, -): Promise { - for (const scope of sequenceCleanupScopes(sequence)) await memory.clear?.(scope) -} diff --git a/src/memory/experiment/types.ts b/src/memory/experiment/types.ts deleted file mode 100644 index dc9ac3f..0000000 --- a/src/memory/experiment/types.ts +++ /dev/null @@ -1,458 +0,0 @@ -import type { - CostChannel, - CostReceipt, - PairedBootstrapResult, - RunPaidCallInput, -} from '@tangle-network/agent-eval' -import type { - CampaignResult, - CampaignStorage, - CostLedgerHandle, - Scenario, -} from '@tangle-network/agent-eval/campaign' -import type { - KnowledgeBenchmarkFamily, - KnowledgeBenchmarkSplit, - KnowledgeMemoryBenchmarkCase, - KnowledgeMemoryBenchmarkTaskKind, - KnowledgeMemoryEvent, - KnowledgeMemoryFactMatcher, -} from '../../benchmarks/index' -import type { - AgentMemoryBranch, - AgentMemoryBranchSnapshot, - AgentMemorySharingPolicy, -} from '../branch' -import type { - AgentMemoryAcquireRunLease, - AgentMemoryControllerMode, - AgentMemoryRunLease, - OwnedAgentMemoryRunLease, -} from '../run-control' -import type { AgentMemoryAdapter, AgentMemoryScope, AgentMemoryWriteInput } from '../types' - -export type AgentMemoryMode = 'stateful' | 'stateless' -export type AgentMemoryLearningArmOrder = 'stateful-first' | 'stateless-first' -export type AgentMemoryExperimentComparisonRef = `sha256:${string}` -export type AgentMemoryEvidenceRef = `sha256:${string}` - -export interface AgentMemoryExperimentCandidateRef { - id: string - ref: string -} - -export interface AgentMemorySequenceProbe { - id: string - /** Stable identity shared by exact repeated measurements of one retention target. */ - retentionKey?: string - /** Explicitly marks a later-step probe as measuring transfer from prior steps. */ - transferKey?: string - query: string - scope?: AgentMemoryScope - limit?: number - taskKind?: KnowledgeMemoryBenchmarkTaskKind - requiredFacts?: readonly KnowledgeMemoryFactMatcher[] - forbiddenFacts?: readonly KnowledgeMemoryFactMatcher[] - expectedEventIds?: readonly string[] - expectedActorIds?: readonly string[] - referenceAnswer?: string -} - -export interface AgentMemorySequenceStep { - id: string - instruction?: string - scope?: AgentMemoryScope - writes?: readonly AgentMemoryWriteInput[] - parallelWrites?: boolean - probes?: readonly AgentMemorySequenceProbe[] - parallelProbes?: boolean - metadata?: Record -} - -/** Runtime-visible step fields. Evaluation labels and dataset identity are excluded. */ -export interface AgentMemoryExecutionStep { - ordinal: number - instruction?: string - scope?: AgentMemoryScope -} - -export type AgentMemoryExecutionPaidCallInput = Omit< - RunPaidCallInput, - 'channel' | 'phase' | 'tags' | 'signal' -> & { - channel?: CostChannel -} - -export type AgentMemoryExecutionCostReceipt = Omit - -export type AgentMemoryExecutionPaidCallResult = - | { - succeeded: true - callId: string - value: T - receipt: AgentMemoryExecutionCostReceipt - } - | { - succeeded: false - callId?: string - error: Error - receipt?: AgentMemoryExecutionCostReceipt - } - -export interface AgentMemoryExecutionCostMeter { - runPaidCall( - input: AgentMemoryExecutionPaidCallInput, - ): Promise> -} - -/** Execution capabilities with all campaign and evaluation identity removed. */ -export interface AgentMemoryExecutionContext { - readonly signal: AbortSignal - readonly cost: AgentMemoryExecutionCostMeter -} - -export interface AgentMemorySequence { - id: string - family: KnowledgeBenchmarkFamily | string - split?: KnowledgeBenchmarkSplit - steps: readonly AgentMemorySequenceStep[] - /** Exact scopes a runtime callback may write beyond scopes declared by steps. */ - cleanupScopes?: readonly AgentMemoryScope[] - tags?: readonly string[] - metadata?: Record -} - -export interface BuildAgentMemorySequencesFromBenchmarkCasesOptions { - memoryAgentId?: string - eventScope?: (input: { - event: KnowledgeMemoryEvent - case: KnowledgeMemoryBenchmarkCase - eventIndex: number - }) => AgentMemoryScope - probeScope?: (testCase: KnowledgeMemoryBenchmarkCase) => AgentMemoryScope -} - -export interface AgentMemoryExperimentCandidate { - id: string - label?: string - /** Change when provider configuration changes so cached cells cannot be reused. */ - ref: string - /** Local construction is free; call markExternalCall before billable provisioning or reconnects. */ - createAdapter(input: { - branchId: string - purpose: 'execute' | 'recovery' - signal: AbortSignal - /** Maximum the adapter must enforce with its provider before external work. */ - maximumCostUsd: number - markExternalCall(): void - /** Record each observed provider charge. Required for complete positive-cost accounting. */ - recordExternalCost(actualCostUsd: number): void - }): AgentMemoryAdapter | null | Promise - policy?: AgentMemorySharingPolicy - baseScope?: AgentMemoryScope - /** Maximum the adapter must enforce for one complete history. Zero declares a free path. */ - externalCostUsdPerSequence?: number - /** Maximum the adapter must enforce for one recovery attempt. Zero declares free recovery. */ - externalRecoveryCostUsdPerAttempt?: number - /** Requires observed provider receipts for positive external charges. */ - externalCostAccounting?: 'exact' - /** Release resources and, when cleanupBranches is false, delete the isolated state. */ - disposeAdapter?(adapter: AgentMemoryAdapter): Promise -} - -export interface AgentMemorySequenceProbeResult { - id: string - stepId: string - stepOrdinal: number - retentionKey?: string - transferKey?: string - query: string - score: number - passed: boolean - dimensions: Record - applicableDimensions: readonly string[] - notes: string - hitIds: readonly string[] - /** Exact scoring input, saved under the cell artifact directory. */ - evidenceRef: AgentMemoryEvidenceRef - evidencePath: string -} - -export interface AgentMemorySequenceArtifact { - candidateId: string - sequenceId: string - memoryMode: AgentMemoryMode - comparisonRef: AgentMemoryExperimentComparisonRef - score: number - passed: boolean - dimensions: Record - dimensionSampleCounts: Record - probes: readonly AgentMemorySequenceProbeResult[] - branchDigest: string - journalEntries: number - durationMs: number -} - -export interface AgentMemorySequenceScenario extends Scenario { - kind: 'agent-memory-sequence' - candidateId: string - sequenceId: string - sequence: AgentMemorySequence - seedGroup: string -} - -export interface AgentMemoryExperimentRankingRow { - rank: number - candidateId: string - label: string - scoreMean: number - passRate: number - totalSequences: number - totalCells: number - totalProbes: number - cellsFailed: number - totalCostUsd: number - durationMs: number - dimensions: Record -} - -export interface AgentMemoryLearningCellComparison { - cellId: string - candidateId: string - sequenceId: string - rep: number - seed: number - statefulArtifactRef: AgentMemoryEvidenceRef - statelessArtifactRef: AgentMemoryEvidenceRef - /** Probes after the first step used for this contrast. */ - probeCount: number - statefulReward: number - statelessReward: number - gain: number -} - -export interface AgentMemoryLearningCandidateSummary { - candidateId: string - cells: number - /** One value per independent sequence after averaging repetitions. */ - gain: PairedBootstrapResult -} - -export interface AgentMemoryTransferCellComparison { - cellId: string - candidateId: string - sequenceId: string - rep: number - seed: number - stepId: string - stepOrdinal: number - transferKey: string - probeCount: number - statefulArtifactRef: AgentMemoryEvidenceRef - statelessArtifactRef: AgentMemoryEvidenceRef - statefulReward: number - statelessReward: number - gain: number -} - -export interface AgentMemoryTransferStepSummary { - candidateId: string - transferKey: string - stepOrdinal: number - /** Stateful minus stateless transfer reward, one pair per independent sequence. */ - gain: PairedBootstrapResult -} - -export interface AgentMemoryForgettingComparison { - cellId: string - candidateId: string - sequenceId: string - rep: number - seed: number - retentionKey: string - observations: number - firstStepOrdinal: number - finalStepOrdinal: number - statefulArtifactRef: AgentMemoryEvidenceRef - statelessArtifactRef: AgentMemoryEvidenceRef - statefulPriorPeakReward: number - statefulPriorPeakStepOrdinal: number - statefulFinalReward: number - /** Prior peak minus final reward. Negative means the final result improved. */ - statefulForgetting: number - statelessPriorPeakReward: number - statelessPriorPeakStepOrdinal: number - statelessFinalReward: number - /** Prior peak minus final reward. Negative means the final result improved. */ - statelessForgetting: number - /** Stateful forgetting minus stateless forgetting. Negative favors stateful memory. */ - excessForgetting: number -} - -export interface AgentMemoryLearningComparison { - comparisonRef: AgentMemoryExperimentComparisonRef - evidence: { - splitRef: AgentMemoryEvidenceRef - statefulManifestRef: AgentMemoryEvidenceRef - statelessManifestRef: AgentMemoryEvidenceRef - statefulRunDir: string - statelessRunDir: string - candidateRefs: readonly AgentMemoryExperimentCandidateRef[] - executionRef: string - } - cells: readonly AgentMemoryLearningCellComparison[] - preTreatment: { - definition: 'first-step-probes' - /** Raw matched cells with at least one first-step probe. */ - cells: number - /** Independent sequences after averaging candidates and repetitions. */ - n: number - /** Fraction of independent sequences whose first-step scores matched exactly in every cell. */ - exactMatchRate: number | null - /** Stateful minus stateless first-step reward. Null when no first-step probes exist. */ - difference: PairedBootstrapResult | null - } - /** - * Paired stateful minus stateless reward on probes after the first step. - * Repetitions and candidates are averaged within each sequence before bootstrapping. - */ - gain: PairedBootstrapResult - gainByCandidate: readonly AgentMemoryLearningCandidateSummary[] - /** Later-step probes explicitly labeled with transferKey. No task meaning is inferred. */ - transfer: { - definition: 'explicit-transfer-probes' - cells: readonly AgentMemoryTransferCellComparison[] - byStep: readonly AgentMemoryTransferStepSummary[] - } - forgetting: { - /** Signed prior peak minus final reward. Negative values mean improvement. */ - definition: 'prior-peak-minus-final' - probes: readonly AgentMemoryForgettingComparison[] - n: number - meanStatefulForgetting: number | null - meanStatelessForgetting: number | null - meanExcessForgetting: number | null - /** Stateful minus stateless forgetting, one pair per independent retention target. */ - excess: PairedBootstrapResult | null - } -} - -export interface CompareAgentMemoryLearningOptions { - stateful: RunAgentMemoryExperimentResult - stateless: RunAgentMemoryExperimentResult -} - -export interface RunAgentMemoryExperimentOptions { - experimentId: string - /** Stable external branch namespace; distributed workers must use the same value. */ - experimentRunId?: string - /** Stateful by default. Stateless clears every declared sequence scope between steps. */ - memoryMode?: AgentMemoryMode - sequences: readonly AgentMemorySequence[] - candidates: readonly AgentMemoryExperimentCandidate[] - /** Retired candidates retained only so interrupted branches can be cleaned on resume. */ - recoveryCandidates?: readonly AgentMemoryExperimentCandidate[] - runDir: string - executeStep?: (input: { - memory: AgentMemoryBranch - candidateId: string - step: AgentMemoryExecutionStep - context: AgentMemoryExecutionContext - }) => Promise - /** Required with executeStep; identify the runtime/profile behavior in cache keys. */ - executeStepRef?: string - onBranchSnapshot?: (input: { - candidateId: string - sequenceId: string - cellId: string - snapshot: AgentMemoryBranchSnapshot - }) => Promise | void - cleanupBranches?: boolean - storage?: CampaignStorage - repo?: string - seed?: number - reps?: number - resumable?: boolean - costCeiling?: number - /** Shared across nested experiments when an outer improvement run owns spend. */ - costLedger?: CostLedgerHandle - costPhase?: string - maxConcurrency?: number - dispatchTimeoutMs?: number - /** Total deadline for each provider cleanup, close, or recovery operation. */ - cleanupTimeoutMs?: number - /** Refuse a damaged run with more unfinished attempts than this. Default 1000. */ - maxRecoveryAttempts?: number - /** Bound repeated provider cleanup after process crashes. Default 3 per attempt. */ - maxRecoveryRetriesPerAttempt?: number - now?: () => Date - /** Required with custom storage when all controllers are confined to one process. */ - controllerMode?: AgentMemoryControllerMode - /** Required for distributed controllers that share custom storage. */ - acquireRunLease?: AgentMemoryAcquireRunLease - /** Cancels active cells; completed cells remain resumable. Cleanup still drains safely. */ - signal?: AbortSignal -} - -export type AgentMemoryExperimentRunLease = AgentMemoryRunLease - -export interface AgentMemoryAttemptEvent { - status: 'started' | 'cleaned' - branchId: string - candidateId: string - candidateRef: string - sequenceId: string - sequenceRef: AgentMemoryEvidenceRef - rep: number - seed: number - cleanupBranches: boolean - externalCostUsdPerSequence: number - externalRecoveryCostUsdPerAttempt: number - recordedAt: string - recovery: boolean -} - -export interface RunAgentMemoryExperimentResult { - memoryMode: AgentMemoryMode - comparisonRef: AgentMemoryExperimentComparisonRef - candidateRefs: readonly AgentMemoryExperimentCandidateRef[] - /** Immutable executor identity, or 'fixtures' when only declared writes are used. */ - executionRef: string - campaign: CampaignResult - rows: readonly AgentMemoryExperimentRankingRow[] - totalCostUsd: number - /** Spend attributed to retired recovery candidates, excluded from ranking rows. */ - unrankedRecoveryCostUsd: number - leaderCandidateId?: string - rankingJsonPath: string - rankingMarkdownPath: string - attemptLogPath: string - recoveryLogPath: string -} - -export type RunAgentMemoryLearningExperimentOptions = Omit< - RunAgentMemoryExperimentOptions, - 'memoryMode' -> & { - /** Counterbalance this across independent runs when provider behavior may drift over time. */ - armOrder?: AgentMemoryLearningArmOrder -} - -export interface RunAgentMemoryLearningExperimentResult { - armOrder: AgentMemoryLearningArmOrder - stateful: RunAgentMemoryExperimentResult - stateless: RunAgentMemoryExperimentResult - comparison: AgentMemoryLearningComparison - evidenceRef: AgentMemoryEvidenceRef - comparisonPath: string - cost: { - /** Receipts attributed only to these two arms, including interrupted attempts. */ - experimentUsd: number - /** All receipts in the shared ledger, including an owning workflow's other work. */ - ledgerUsd: number - ceilingUsd: number - accountingComplete: boolean - } -} - -export type OwnedMemoryExperimentRunLease = OwnedAgentMemoryRunLease diff --git a/src/memory/experiment/validation.ts b/src/memory/experiment/validation.ts deleted file mode 100644 index 7694822..0000000 --- a/src/memory/experiment/validation.ts +++ /dev/null @@ -1,245 +0,0 @@ -import { canonicalJson } from '@tangle-network/agent-eval' -import type { KnowledgeMemoryFactMatcher } from '../../benchmarks/index' -import type { AgentMemoryScope } from '../types' -import type { AgentMemorySequence, AgentMemorySequenceProbe } from './types' - -export function assertMemorySequences(sequences: readonly AgentMemorySequence[]): void { - for (const sequence of sequences) { - assertNonEmptyString(sequence.family, `memory experiment sequence ${sequence.id} family`) - if (sequence.steps.length === 0) { - throw new Error(`memory experiment sequence ${sequence.id} has no steps`) - } - assertUnique( - sequence.steps.map((step) => step.id), - `step in sequence ${sequence.id}`, - ) - let probeCount = 0 - const retentionMeasurements = new Map< - string, - { definition: string; stepOrdinals: Set } - >() - for (const [stepOrdinal, step] of sequence.steps.entries()) { - for (const write of step.writes ?? []) { - assertNonEmptyString(write.text, `memory experiment write in ${sequence.id}/${step.id}`) - if (write.id !== undefined) { - assertNonEmptyString(write.id, `memory experiment write id in ${sequence.id}/${step.id}`) - } - } - assertUnique( - (step.probes ?? []).map((probe) => probe.id), - `probe in sequence ${sequence.id} step ${step.id}`, - ) - for (const probe of step.probes ?? []) { - probeCount += 1 - assertNonEmptyString( - probe.query, - `memory experiment probe query ${sequence.id}/${step.id}/${probe.id}`, - ) - if (probe.limit !== undefined && (!Number.isSafeInteger(probe.limit) || probe.limit <= 0)) { - throw new Error( - `memory experiment probe limit ${sequence.id}/${step.id}/${probe.id} must be a positive safe integer`, - ) - } - assertMemoryFactMatchers( - probe.requiredFacts ?? [], - `${sequence.id}/${step.id}/${probe.id} requiredFacts`, - ) - assertMemoryFactMatchers( - probe.forbiddenFacts ?? [], - `${sequence.id}/${step.id}/${probe.id} forbiddenFacts`, - ) - assertStringList(probe.expectedEventIds, `${sequence.id}/${step.id}/${probe.id} event ids`) - assertStringList(probe.expectedActorIds, `${sequence.id}/${step.id}/${probe.id} actor ids`) - if (probe.referenceAnswer !== undefined) { - assertNonEmptyString( - probe.referenceAnswer, - `memory experiment reference answer ${sequence.id}/${step.id}/${probe.id}`, - ) - } - if (probe.retentionKey !== undefined) { - assertNonEmptyString( - probe.retentionKey, - `memory experiment retention key ${sequence.id}/${step.id}/${probe.id}`, - ) - if (probe.retentionKey !== probe.retentionKey.trim()) { - throw new Error( - `memory experiment retention key ${sequence.id}/${step.id}/${probe.id} must not have surrounding whitespace`, - ) - } - const definition = retentionMeasurementDefinition(step.scope, probe) - const prior = retentionMeasurements.get(probe.retentionKey) - if (prior?.stepOrdinals.has(stepOrdinal)) { - throw new Error( - `memory experiment retention key ${sequence.id}/${probe.retentionKey} may appear only once per step`, - ) - } - if (prior && prior.definition !== definition) { - throw new Error( - `memory experiment retention key ${sequence.id}/${probe.retentionKey} must repeat the exact same measurement`, - ) - } - const measurement = prior ?? { definition, stepOrdinals: new Set() } - measurement.stepOrdinals.add(stepOrdinal) - retentionMeasurements.set(probe.retentionKey, measurement) - } - if (probe.transferKey !== undefined) { - assertNonEmptyString( - probe.transferKey, - `memory experiment transfer key ${sequence.id}/${step.id}/${probe.id}`, - ) - if (probe.transferKey !== probe.transferKey.trim()) { - throw new Error( - `memory experiment transfer key ${sequence.id}/${step.id}/${probe.id} must not have surrounding whitespace`, - ) - } - if (probe.retentionKey !== undefined) { - throw new Error( - `memory experiment probe ${sequence.id}/${step.id}/${probe.id} cannot be both transfer and retention`, - ) - } - if (stepOrdinal === 0) { - throw new Error( - `memory experiment transfer probe ${sequence.id}/${step.id}/${probe.id} must run after the first step`, - ) - } - } - const hasTarget = - (probe.requiredFacts?.length ?? 0) > 0 || - (probe.forbiddenFacts?.length ?? 0) > 0 || - (probe.expectedEventIds?.length ?? 0) > 0 || - (probe.expectedActorIds?.length ?? 0) > 0 || - Boolean(probe.referenceAnswer?.trim()) - if (!hasTarget) { - throw new Error( - `memory experiment probe ${sequence.id}/${step.id}/${probe.id} has no measurable target`, - ) - } - } - } - if (probeCount === 0) { - throw new Error(`memory experiment sequence ${sequence.id} has no probes`) - } - for (const [retentionKey, measurement] of retentionMeasurements) { - if (measurement.stepOrdinals.size < 2) { - throw new Error( - `memory experiment retention key ${sequence.id}/${retentionKey} must appear in at least two distinct steps`, - ) - } - } - } -} - -export function assertMemoryLearningSequences(sequences: readonly AgentMemorySequence[]): void { - for (const sequence of sequences) { - if (sequence.steps.length < 2) { - throw new Error(`memory learning sequence ${sequence.id} requires at least two ordered steps`) - } - if (!sequence.steps.slice(1).some((step) => (step.probes?.length ?? 0) > 0)) { - throw new Error( - `memory learning sequence ${sequence.id} requires a probe after the first step`, - ) - } - } -} - -function retentionMeasurementDefinition( - stepScope: AgentMemoryScope | undefined, - probe: AgentMemorySequenceProbe, -): string { - return canonicalJson({ - query: probe.query, - scope: compactScope({ - ...(stepScope ?? {}), - ...(probe.scope ?? {}), - tags: { ...(stepScope?.tags ?? {}), ...(probe.scope?.tags ?? {}) }, - }), - limit: probe.limit ?? null, - taskKind: probe.taskKind ?? 'memory-recall', - requiredFacts: normalizeFactMatchers(probe.requiredFacts ?? []), - forbiddenFacts: normalizeFactMatchers(probe.forbiddenFacts ?? []), - expectedEventIds: [...(probe.expectedEventIds ?? [])], - expectedActorIds: [...(probe.expectedActorIds ?? [])], - referenceAnswer: probe.referenceAnswer ?? null, - }) -} - -function normalizeFactMatchers(matchers: readonly KnowledgeMemoryFactMatcher[]) { - return matchers.map((matcher) => ({ - id: matcher.id, - anyOf: [...matcher.anyOf], - weight: matcher.weight ?? 1, - sourceEventIds: [...(matcher.sourceEventIds ?? [])], - validAt: matcher.validAt ?? null, - obsolete: matcher.obsolete ?? false, - })) -} - -export function normalizeCleanupScope(scope: AgentMemoryScope): AgentMemoryScope { - const normalized = compactScope(scope) - if (normalized.tags && Object.keys(normalized.tags).length === 0) { - delete normalized.tags - } - return normalized -} - -export function compactScope(scope: AgentMemoryScope): AgentMemoryScope { - return Object.fromEntries( - Object.entries(scope).filter(([, value]) => value !== undefined), - ) as AgentMemoryScope -} - -export function compactRecord(record: Record): Record { - return Object.fromEntries(Object.entries(record).filter(([, value]) => value !== undefined)) -} - -export function uniqueStrings(values: readonly unknown[]): string[] { - return [...new Set(values.filter((value): value is string => typeof value === 'string'))] -} - -export function stringArray(value: unknown): string[] { - return Array.isArray(value) - ? value.filter((entry): entry is string => typeof entry === 'string') - : [] -} - -export function assertUnique(values: readonly string[], label: string): void { - const seen = new Set() - for (const value of values) { - assertNonEmptyString(value, `memory experiment ${label} id`) - if (seen.has(value)) throw new Error(`duplicate memory experiment ${label} id: ${value}`) - seen.add(value) - } -} - -function assertMemoryFactMatchers( - matchers: readonly KnowledgeMemoryFactMatcher[], - label: string, -): void { - assertUnique( - matchers.map((matcher) => matcher.id), - `${label} matcher`, - ) - for (const matcher of matchers) { - if (matcher.anyOf.length === 0) { - throw new Error(`memory experiment ${label} matcher ${matcher.id} requires anyOf`) - } - assertStringList(matcher.anyOf, `${label} matcher ${matcher.id} anyOf`) - assertStringList(matcher.sourceEventIds, `${label} matcher ${matcher.id} source event ids`) - if (matcher.weight !== undefined && (!Number.isFinite(matcher.weight) || matcher.weight <= 0)) { - throw new Error( - `memory experiment ${label} matcher ${matcher.id} weight must be a positive finite number`, - ) - } - } -} - -function assertStringList(values: readonly string[] | undefined, label: string): void { - if (values === undefined) return - assertUnique(values, label) -} - -export function assertNonEmptyString(value: unknown, label: string): asserts value is string { - if (typeof value !== 'string' || value.trim().length === 0) { - throw new Error(`${label} must be a non-empty string`) - } -} diff --git a/src/memory/improvement.ts b/src/memory/improvement.ts deleted file mode 100644 index c72fde4..0000000 --- a/src/memory/improvement.ts +++ /dev/null @@ -1,13 +0,0 @@ -export { runAgentMemoryImprovement } from './improvement/run' -export type { - AgentMemoryActivation, - AgentMemoryActivationDriver, - AgentMemoryDimensionComparison, - AgentMemoryFinalEvaluation, - AgentMemoryFinalPair, - AgentMemoryImprovementRunLease, - AgentMemoryPromotionDecision, - MemoryConfigScenario, - RunAgentMemoryImprovementOptions, - RunAgentMemoryImprovementResult, -} from './improvement/types' diff --git a/src/memory/improvement/activation.ts b/src/memory/improvement/activation.ts deleted file mode 100644 index 10e9eb9..0000000 --- a/src/memory/improvement/activation.ts +++ /dev/null @@ -1,247 +0,0 @@ -import { type CampaignStorage, surfaceHash } from '@tangle-network/agent-eval/campaign' -import type { AgentCandidateJsonValue as JsonValue } from '@tangle-network/agent-interface' -import { appendDurableJournalEvent } from '../attempt-log' -import { runBoundedMemoryLifecycle } from '../lifecycle' -import { memoryConfigCodec } from './evaluation' -import type { - AgentMemoryActivationEvent, - AgentMemoryActivationJournalState, - OwnedRunLease, - RunAgentMemoryImprovementOptions, - RunAgentMemoryImprovementResult, -} from './types' - -export function appendMemoryActivationEvent( - storage: CampaignStorage, - path: string, - event: AgentMemoryActivationEvent, -): void { - appendDurableJournalEvent({ - storage, - path, - event, - label: 'memory activation journal', - }) -} - -export function readMemoryActivationJournal( - storage: CampaignStorage, - path: string, - expected: Omit, -): AgentMemoryActivationJournalState { - const stored = storage.read(path) - if (stored === undefined) { - if (storage.exists(path)) throw new Error(`cannot read memory activation journal '${path}'`) - return { prepared: false } - } - let prepared = false - let activated: AgentMemoryActivationEvent | undefined - for (const [index, line] of stored.split('\n').entries()) { - if (!line) continue - let raw: unknown - try { - raw = JSON.parse(line) - } catch (error) { - throw new Error(`invalid memory activation journal '${path}' line ${index + 1}`, { - cause: error, - }) - } - const event = parseMemoryActivationEvent(raw, path, index + 1, expected) - if (event.status === 'prepared') { - if (prepared || activated) { - throw new Error(`memory activation journal '${path}' repeats its prepared event`) - } - prepared = true - continue - } - if (!prepared || activated) { - throw new Error(`memory activation journal '${path}' has an out-of-order activated event`) - } - activated = event - } - return { prepared, ...(activated ? { activated } : {}) } -} - -export async function assertActivatedMemoryWinner(input: { - options: RunAgentMemoryImprovementOptions - lease: OwnedRunLease - result: Pick, 'winnerSurfaceHash'> -}): Promise { - const activationDriver = input.options.activation - if (!activationDriver) { - throw new Error('an activated memory journal requires its activation driver') - } - await input.lease.assertOwned() - const currentConfig = await runBoundedMemoryLifecycle({ - operation: `${activationDriver.ref}: confirm active memory configuration`, - timeoutMs: input.options.activationTimeoutMs ?? 60_000, - run: () => activationDriver.readCurrent(), - }) - await input.lease.assertOwned() - const currentHash = surfaceHash(memoryConfigCodec(input.options).serialize(currentConfig)) - if (currentHash !== input.result.winnerSurfaceHash) { - throw new Error( - `memory activation target '${activationDriver.ref}' drifted from measured winner '${input.result.winnerSurfaceHash}' to '${currentHash}'`, - ) - } -} - -export async function activateMemoryWinner(input: { - options: RunAgentMemoryImprovementOptions - storage: CampaignStorage - lease: OwnedRunLease - result: RunAgentMemoryImprovementResult - activationEventIdentity: Omit - activationJournalDir: string - activationJournalPath: string - hadPreparedEvent: boolean -}): Promise { - const activationDriver = input.options.activation! - const activationTimeoutMs = input.options.activationTimeoutMs ?? 60_000 - if (!input.hadPreparedEvent) { - await input.lease.assertOwned() - input.storage.ensureDir(input.activationJournalDir) - appendMemoryActivationEvent(input.storage, input.activationJournalPath, { - ...input.activationEventIdentity, - status: 'prepared', - recordedAt: (input.options.now ?? (() => new Date()))().toISOString(), - }) - } - - await input.lease.assertOwned() - const currentConfig = await runBoundedMemoryLifecycle({ - operation: `${activationDriver.ref}: read current memory configuration`, - timeoutMs: activationTimeoutMs, - run: () => activationDriver.readCurrent(), - }) - await input.lease.assertOwned() - const codec = memoryConfigCodec(input.options) - const currentHash = surfaceHash(codec.serialize(currentConfig)) - if ( - currentHash !== input.result.baselineSurfaceHash && - currentHash !== input.result.winnerSurfaceHash - ) { - throw new Error( - `memory activation target '${activationDriver.ref}' changed concurrently; expected '${input.result.baselineSurfaceHash}' or '${input.result.winnerSurfaceHash}', found '${currentHash}'`, - ) - } - - let outcome: NonNullable - if (currentHash === input.result.winnerSurfaceHash) { - outcome = input.hadPreparedEvent ? 'recovered' : 'already-current' - input.result.activation.status = 'recovered' - } else { - let compareError: unknown - try { - await runBoundedMemoryLifecycle({ - operation: `${activationDriver.ref}: activate memory configuration`, - timeoutMs: activationTimeoutMs, - run: () => - activationDriver.compareAndSet({ - activationId: input.result.activation.id, - expectedConfig: input.result.baselineConfig, - expectedSurfaceHash: input.result.baselineSurfaceHash, - config: input.result.winnerConfig, - surfaceHash: input.result.winnerSurfaceHash, - decision: input.result.decision, - optimization: input.result.optimization, - finalEvaluation: input.result.finalEvaluation, - }), - }) - } catch (error) { - compareError = error - } - await input.lease.assertOwned() - - let observedConfig: TConfig - try { - observedConfig = await runBoundedMemoryLifecycle({ - operation: `${activationDriver.ref}: confirm memory configuration`, - timeoutMs: activationTimeoutMs, - run: () => activationDriver.readCurrent(), - }) - } catch (error) { - if (compareError) { - throw new AggregateError( - [compareError, error], - `memory activation '${input.result.activation.id}' failed and its live state could not be confirmed`, - ) - } - throw error - } - await input.lease.assertOwned() - const observedHash = surfaceHash(codec.serialize(observedConfig)) - if (observedHash !== input.result.winnerSurfaceHash) { - const mismatch = new Error( - `memory activation '${input.result.activation.id}' did not install the measured winner; found '${observedHash}'`, - ) - if (compareError) { - throw new AggregateError( - [compareError, mismatch], - `memory activation '${input.result.activation.id}' failed without applying the measured winner`, - ) - } - throw mismatch - } - outcome = compareError ? 'recovered' : 'applied' - input.result.activation.status = compareError ? 'recovered' : 'activated' - } - - await input.lease.assertOwned() - appendMemoryActivationEvent(input.storage, input.activationJournalPath, { - ...input.activationEventIdentity, - status: 'activated', - outcome, - recordedAt: (input.options.now ?? (() => new Date()))().toISOString(), - }) -} - -function parseMemoryActivationEvent( - value: unknown, - path: string, - line: number, - expected: Omit, -): AgentMemoryActivationEvent { - if (!value || typeof value !== 'object' || Array.isArray(value)) { - throw new Error(`invalid memory activation journal '${path}' line ${line}`) - } - const event = value as Record - for (const [key, expectedValue] of Object.entries(expected)) { - if (event[key] !== expectedValue) { - throw new Error( - `memory activation journal '${path}' line ${line} does not match the measured winner`, - ) - } - } - if (event.status !== 'prepared' && event.status !== 'activated') { - throw new Error(`invalid memory activation journal '${path}' line ${line} status`) - } - const expectedKeys = [ - ...Object.keys(expected), - 'status', - 'recordedAt', - ...(event.status === 'activated' ? ['outcome'] : []), - ] - const actualKeys = Object.keys(event) - if ( - actualKeys.length !== expectedKeys.length || - actualKeys.some((key) => !expectedKeys.includes(key)) - ) { - throw new Error(`invalid memory activation journal '${path}' line ${line} fields`) - } - if (typeof event.recordedAt !== 'string' || !Number.isFinite(Date.parse(event.recordedAt))) { - throw new Error(`invalid memory activation journal '${path}' line ${line} recordedAt`) - } - if ( - event.status === 'activated' && - event.outcome !== 'applied' && - event.outcome !== 'recovered' && - event.outcome !== 'already-current' - ) { - throw new Error(`invalid memory activation journal '${path}' line ${line} outcome`) - } - if (event.status === 'prepared' && event.outcome !== undefined) { - throw new Error(`invalid memory activation journal '${path}' line ${line} prepared outcome`) - } - return value as AgentMemoryActivationEvent -} diff --git a/src/memory/improvement/candidate.ts b/src/memory/improvement/candidate.ts deleted file mode 100644 index d23bbca..0000000 --- a/src/memory/improvement/candidate.ts +++ /dev/null @@ -1,109 +0,0 @@ -import { dirname, join } from 'node:path' -import { canonicalJson } from '@tangle-network/agent-eval' -import type { CampaignStorage } from '@tangle-network/agent-eval/campaign' -import type { AgentCandidateJsonValue as JsonValue } from '@tangle-network/agent-interface' -import { assertImmutableRef } from '../../immutable-ref' -import type { AgentMemoryExperimentCandidate, RunAgentMemoryExperimentOptions } from '../experiment' -import type { OwnedRunLease, RunAgentMemoryImprovementOptions } from './types' - -export async function buildCandidate( - options: RunAgentMemoryImprovementOptions, - config: TConfig, - hash: string, -): Promise { - const id = `memory-config-${hash}` - const built = await options.createCandidate({ - config, - candidateId: id, - surfaceHash: hash, - }) - assertImmutableRef(built.ref, `${id}: ref`) - assertDeclaredCost(built.externalCostUsdPerSequence, `${id}: externalCostUsdPerSequence`) - assertDeclaredCost( - built.externalRecoveryCostUsdPerAttempt, - `${id}: externalRecoveryCostUsdPerAttempt`, - ) - if (built.externalCostAccounting !== 'exact') { - throw new Error(`${id}: memory improvement requires exact external cost accounting`) - } - return { - ...built, - id, - ref: built.ref, - } -} - -export async function buildRegisteredCandidate( - options: RunAgentMemoryImprovementOptions, - storage: CampaignStorage, - runDir: string, - config: TConfig, - hash: string, -): Promise { - const candidate = await buildCandidate(options, config, hash) - const path = join(runDir, 'memory-candidate-identities', `${hash}.json`) - const record = { surfaceHash: hash, candidateRef: candidate.ref } - const stored = storage.read(path) - if (stored !== undefined) { - let parsed: unknown - try { - parsed = JSON.parse(stored) - } catch (error) { - throw new Error(`invalid memory candidate identity '${path}'`, { cause: error }) - } - if (canonicalJson(parsed as JsonValue) !== canonicalJson(record)) { - throw new Error( - `memory config '${hash}' changed candidate identity within the same improvement run`, - ) - } - return candidate - } - if (storage.exists(path)) throw new Error(`cannot read memory candidate identity '${path}'`) - storage.ensureDir(dirname(path)) - storage.write(path, `${JSON.stringify(record, null, 2)}\n`) - return candidate -} - -function assertDeclaredCost(value: unknown, label: string): asserts value is number { - if (typeof value !== 'number' || !Number.isFinite(value) || value < 0) { - throw new Error(`${label} must be a declared non-negative finite number`) - } -} - -export function experimentOptions( - options: RunAgentMemoryImprovementOptions, - storage: CampaignStorage, - lease: OwnedRunLease, -): Pick< - RunAgentMemoryExperimentOptions, - | 'storage' - | 'resumable' - | 'dispatchTimeoutMs' - | 'cleanupTimeoutMs' - | 'maxRecoveryAttempts' - | 'maxRecoveryRetriesPerAttempt' - | 'executeStep' - | 'executeStepRef' - | 'onBranchSnapshot' - | 'cleanupBranches' - | 'acquireRunLease' - | 'now' -> { - return { - storage, - resumable: options.resumable, - dispatchTimeoutMs: options.dispatchTimeoutMs, - cleanupTimeoutMs: options.cleanupTimeoutMs, - maxRecoveryAttempts: options.maxRecoveryAttempts, - maxRecoveryRetriesPerAttempt: options.maxRecoveryRetriesPerAttempt, - executeStep: options.executeStep, - executeStepRef: options.executeStepRef, - onBranchSnapshot: options.onBranchSnapshot, - cleanupBranches: options.cleanupBranches ?? true, - acquireRunLease: async () => ({ - assertOwned: () => lease.assertOwned(), - release() {}, - }), - now: options.now, - } -} diff --git a/src/memory/improvement/evaluation.ts b/src/memory/improvement/evaluation.ts deleted file mode 100644 index 338cb85..0000000 --- a/src/memory/improvement/evaluation.ts +++ /dev/null @@ -1,459 +0,0 @@ -import { dirname, join } from 'node:path' -import { canonicalJson } from '@tangle-network/agent-eval' -import { - type CampaignCostMeter, - type CampaignStorage, - surfaceHash, -} from '@tangle-network/agent-eval/campaign' -import type { AgentCandidateJsonValue as JsonValue } from '@tangle-network/agent-interface' -import { stableId } from '../../ids' -import type { SerializedCandidateCodec } from '../../optimization' -import { - type AgentMemoryExperimentCandidate, - type AgentMemorySequence, - type AgentMemorySequenceArtifact, - runAgentMemoryExperiment, -} from '../experiment' -import { assertAgentMemoryExperimentComparisonRef } from '../experiment/comparison-ref' -import { experimentOptions } from './candidate' -import { memorySequenceFingerprint, parseMemoryConfig, serializeMemoryConfig } from './identity' -import type { - AgentMemoryFinalEvaluation, - AgentMemoryFinalPair, - MemoryConfigScenario, - OwnedRunLease, - RunAgentMemoryImprovementOptions, -} from './types' - -interface StoredMemoryArtifact { - surfaceHash: string - candidateRef: string - sequenceFingerprint: string - sequenceId: string - rep: number - seed: number - artifact: AgentMemorySequenceArtifact -} - -export function memoryConfigCodec( - options: RunAgentMemoryImprovementOptions, -): SerializedCandidateCodec { - const serializeRaw = options.serializeConfig ?? ((config: TConfig) => canonicalJson(config)) - const parseRaw = options.parseConfig ?? ((surface: string) => JSON.parse(surface) as TConfig) - return { - serialize: (config) => serializeMemoryConfig(serializeRaw, config), - parse: (surface) => parseMemoryConfig(parseRaw, surface), - } -} - -export function memoryConfigScenarios( - sequences: readonly AgentMemorySequence[], -): MemoryConfigScenario[] { - return sequences.map(memoryConfigScenario) -} - -export async function evaluateMemoryCandidate(input: { - options: RunAgentMemoryImprovementOptions - storage: CampaignStorage - runDir: string - lease: OwnedRunLease - candidate: AgentMemoryExperimentCandidate - surfaceHash: string - scenario: MemoryConfigScenario - rep: number - seed: number - final: boolean - cost: CampaignCostMeter - signal: AbortSignal -}): Promise { - await input.lease.assertOwned() - const artifactPath = memoryArtifactPath( - input.runDir, - input.surfaceHash, - input.scenario, - input.rep, - input.seed, - ) - const stored = readStoredMemoryArtifact(input.storage, artifactPath, { - surfaceHash: input.surfaceHash, - candidateRef: input.candidate.ref, - scenario: input.scenario, - rep: input.rep, - seed: input.seed, - }) - if (stored) { - if (input.final) { - writeStoredMemoryArtifact( - input.storage, - memoryFinalArtifactPath(input.runDir, input.surfaceHash, input.scenario, input.rep), - storedMemoryArtifactRecord(input, stored), - ) - } - return stored - } - - const evaluationCostLimit = input.options.maximumEvaluationCostUsd ?? 0 - const evaluationId = stableId( - 'memory_eval', - canonicalJson({ - surfaceHash: input.surfaceHash, - sequence: memorySequenceFingerprint(input.scenario.sequence), - rep: input.rep, - seed: input.seed, - }), - ) - const paid = await input.cost.runPaidCall({ - actor: 'agent-knowledge:memory-config-evaluation', - model: input.candidate.ref, - signal: input.signal, - maximumCharge: { - externallyEnforcedMaximumUsd: evaluationCostLimit, - }, - execute: async () => { - const experiment = await runAgentMemoryExperiment({ - ...experimentOptions(input.options, input.storage, input.lease), - experimentId: `${input.options.experimentId}:${evaluationId}`, - experimentRunId: evaluationId, - sequences: [input.scenario.sequence], - candidates: [input.candidate], - runDir: join(input.runDir, 'evaluations', evaluationId), - seed: input.seed, - reps: 1, - maxConcurrency: 1, - costCeiling: evaluationCostLimit, - costPhase: `memory.config.${input.surfaceHash}`, - signal: input.signal, - }) - const cell = experiment.campaign.cells[0] - if (!cell || cell.error || cell.artifact.candidateId !== input.candidate.id) { - throw new Error( - `${input.surfaceHash}/${input.scenario.sequenceId}: memory config evaluation did not complete`, - ) - } - const cost = experiment.campaign.aggregates.cost - return { - artifact: cell.artifact, - costUsd: cell.cached ? 0 : experiment.totalCostUsd, - cost, - } - }, - receipt: (value) => ({ - model: input.candidate.ref, - inputTokens: value.cost.inputTokens, - outputTokens: value.cost.outputTokens, - ...(value.cost.reasoningTokens !== undefined - ? { reasoningTokens: value.cost.reasoningTokens } - : {}), - cachedTokens: value.cost.cachedTokens, - ...(value.cost.cacheWriteTokens !== undefined - ? { cacheWriteTokens: value.cost.cacheWriteTokens } - : {}), - actualCostUsd: value.costUsd, - costUnknown: !value.cost.accountingComplete, - usageUnknown: !value.cost.usageComplete, - }), - }) - if (!paid.succeeded) throw paid.error - await input.lease.assertOwned() - const record = storedMemoryArtifactRecord(input, paid.value.artifact) - writeStoredMemoryArtifact(input.storage, artifactPath, record) - if (input.final) { - writeStoredMemoryArtifact( - input.storage, - memoryFinalArtifactPath(input.runDir, input.surfaceHash, input.scenario, input.rep), - record, - ) - } - return paid.value.artifact -} - -export function loadFinalEvaluation(input: { - options: RunAgentMemoryImprovementOptions - storage: CampaignStorage - runDir: string - baselineSurfaceHash: string - baselineCandidateRef: string - winnerSurfaceHash: string - winnerCandidateRef: string -}): AgentMemoryFinalEvaluation { - const pairs: AgentMemoryFinalPair[] = [] - const reps = input.options.reps ?? 1 - for (const sequence of input.options.finalSequences) { - const scenario = memoryConfigScenario(sequence) - for (let rep = 0; rep < reps; rep += 1) { - const baseline = readStoredMemoryArtifact( - input.storage, - memoryFinalArtifactPath(input.runDir, input.baselineSurfaceHash, scenario, rep), - { - surfaceHash: input.baselineSurfaceHash, - candidateRef: input.baselineCandidateRef, - scenario, - rep, - }, - ) - const winner = readStoredMemoryArtifact( - input.storage, - memoryFinalArtifactPath(input.runDir, input.winnerSurfaceHash, scenario, rep), - { - surfaceHash: input.winnerSurfaceHash, - candidateRef: input.winnerCandidateRef, - scenario, - rep, - }, - ) - if (!baseline || !winner) { - throw new Error( - `memory final artifact is missing for sequence '${sequence.id}' repetition ${rep}`, - ) - } - pairs.push({ sequenceId: sequence.id, rep, baseline, winner }) - } - } - return { - manifestHash: surfaceHash( - canonicalJson({ - baselineSurfaceHash: input.baselineSurfaceHash, - baselineCandidateRef: input.baselineCandidateRef, - winnerSurfaceHash: input.winnerSurfaceHash, - winnerCandidateRef: input.winnerCandidateRef, - pairs, - }), - ), - baselineCandidateRef: input.baselineCandidateRef, - winnerCandidateRef: input.winnerCandidateRef, - pairs, - } -} - -export function memoryArtifactPath( - runDir: string, - candidateSurfaceHash: string, - scenario: MemoryConfigScenario, - rep: number, - seed: number, -): string { - return join( - runDir, - 'memory-config-artifacts', - candidateSurfaceHash, - stableId('sequence', scenario.sequenceId), - `rep-${rep}-${stableId('seed', String(seed))}.json`, - ) -} - -function memoryFinalArtifactPath( - runDir: string, - candidateSurfaceHash: string, - scenario: MemoryConfigScenario, - rep: number, -): string { - return join( - runDir, - 'memory-final-artifacts', - candidateSurfaceHash, - stableId('sequence', scenario.sequenceId), - `rep-${rep}.json`, - ) -} - -function memoryConfigScenario(sequence: AgentMemorySequence): MemoryConfigScenario { - return { - id: sequence.id, - kind: 'agent-memory-config-search', - sequenceId: sequence.id, - sequence, - } -} - -function readStoredMemoryArtifact( - storage: CampaignStorage, - path: string, - expected: { - surfaceHash: string - candidateRef: string - scenario: MemoryConfigScenario - rep: number - seed?: number - }, -): AgentMemorySequenceArtifact | undefined { - const stored = storage.read(path) - if (stored === undefined) { - if (storage.exists(path)) throw new Error(`cannot read memory config artifact '${path}'`) - return undefined - } - let record: unknown - try { - record = JSON.parse(stored) - } catch (error) { - throw new Error(`invalid memory config artifact '${path}'`, { cause: error }) - } - if (!record || typeof record !== 'object' || Array.isArray(record)) { - throw new Error(`invalid memory config artifact '${path}'`) - } - const value = record as Partial - if ( - !hasExactKeys(record, [ - 'surfaceHash', - 'candidateRef', - 'sequenceFingerprint', - 'sequenceId', - 'rep', - 'seed', - 'artifact', - ]) || - value.surfaceHash !== expected.surfaceHash || - value.candidateRef !== expected.candidateRef || - value.sequenceFingerprint !== memorySequenceFingerprint(expected.scenario.sequence) || - value.sequenceId !== expected.scenario.sequenceId || - value.rep !== expected.rep || - !Number.isSafeInteger(value.seed) || - (expected.seed !== undefined && value.seed !== expected.seed) - ) { - throw new Error(`memory config artifact '${path}' does not match its evaluation cell`) - } - return parseMemoryArtifact( - value.artifact, - `memory-config-${expected.surfaceHash}`, - expected.scenario.sequenceId, - path, - ) -} - -function writeStoredMemoryArtifact( - storage: CampaignStorage, - path: string, - record: StoredMemoryArtifact, -): void { - const serialized = `${JSON.stringify(record, null, 2)}\n` - const existing = storage.read(path) - if (existing !== undefined) { - if (canonicalJson(JSON.parse(existing)) !== canonicalJson(record)) { - throw new Error(`memory config artifact '${path}' conflicts with durable content`) - } - return - } - if (storage.exists(path)) throw new Error(`cannot read memory config artifact '${path}'`) - storage.ensureDir(dirname(path)) - storage.write(path, serialized) -} - -function storedMemoryArtifactRecord( - input: { - surfaceHash: string - candidate: AgentMemoryExperimentCandidate - scenario: MemoryConfigScenario - rep: number - seed: number - }, - artifact: AgentMemorySequenceArtifact, -): StoredMemoryArtifact { - return { - surfaceHash: input.surfaceHash, - candidateRef: input.candidate.ref, - sequenceFingerprint: memorySequenceFingerprint(input.scenario.sequence), - sequenceId: input.scenario.sequenceId, - rep: input.rep, - seed: input.seed, - artifact, - } -} - -function parseMemoryArtifact( - value: unknown, - candidateId: string, - sequenceId: string, - path: string, -): AgentMemorySequenceArtifact { - if (!value || typeof value !== 'object' || Array.isArray(value)) { - throw new Error(`memory config artifact '${path}' has no artifact`) - } - const artifact = value as Partial - try { - assertAgentMemoryExperimentComparisonRef( - artifact.comparisonRef, - `memory config artifact '${path}' comparisonRef`, - ) - } catch { - throw new Error(`memory config artifact '${path}' is malformed`) - } - if ( - artifact.candidateId !== candidateId || - artifact.sequenceId !== sequenceId || - artifact.memoryMode !== 'stateful' || - typeof artifact.score !== 'number' || - !Number.isFinite(artifact.score) || - artifact.score < 0 || - artifact.score > 1 || - typeof artifact.passed !== 'boolean' || - !isFiniteNumberRecord(artifact.dimensions) || - !isNonnegativeIntegerRecord(artifact.dimensionSampleCounts) || - !Array.isArray(artifact.probes) || - !artifact.probes.every(isMemoryProbeResult) || - typeof artifact.branchDigest !== 'string' || - !/^sha256:[a-f0-9]{64}$/.test(artifact.branchDigest) || - typeof artifact.journalEntries !== 'number' || - !Number.isSafeInteger(artifact.journalEntries) || - artifact.journalEntries < 0 || - typeof artifact.durationMs !== 'number' || - !Number.isFinite(artifact.durationMs) || - artifact.durationMs < 0 - ) { - throw new Error(`memory config artifact '${path}' is malformed`) - } - return artifact as AgentMemorySequenceArtifact -} - -function isMemoryProbeResult(value: unknown): boolean { - if (!value || typeof value !== 'object' || Array.isArray(value)) return false - const probe = value as Record - if ( - typeof probe.id !== 'string' || - !probe.id || - typeof probe.stepId !== 'string' || - !probe.stepId || - !Number.isSafeInteger(probe.stepOrdinal) || - (probe.stepOrdinal as number) < 0 || - typeof probe.query !== 'string' || - !probe.query || - typeof probe.score !== 'number' || - !Number.isFinite(probe.score) || - probe.score < 0 || - probe.score > 1 || - typeof probe.passed !== 'boolean' || - !isFiniteNumberRecord(probe.dimensions) || - !Array.isArray(probe.applicableDimensions) || - !probe.applicableDimensions.every((entry) => typeof entry === 'string' && entry.length > 0) || - typeof probe.notes !== 'string' || - !Array.isArray(probe.hitIds) || - !probe.hitIds.every((entry) => typeof entry === 'string') || - typeof probe.evidenceRef !== 'string' || - !/^sha256:[a-f0-9]{64}$/.test(probe.evidenceRef) || - typeof probe.evidencePath !== 'string' || - !probe.evidencePath.endsWith(`/memory-evidence/${probe.evidenceRef.slice(7)}.json`) - ) { - return false - } - return ( - (probe.retentionKey === undefined || - (typeof probe.retentionKey === 'string' && probe.retentionKey.length > 0)) && - (probe.transferKey === undefined || - (typeof probe.transferKey === 'string' && probe.transferKey.length > 0)) - ) -} - -function isFiniteNumberRecord(value: unknown): value is Record { - if (!value || typeof value !== 'object' || Array.isArray(value)) return false - return Object.values(value).every((entry) => typeof entry === 'number' && Number.isFinite(entry)) -} - -function isNonnegativeIntegerRecord(value: unknown): value is Record { - return ( - isFiniteNumberRecord(value) && - Object.values(value).every((entry) => Number.isSafeInteger(entry) && entry >= 0) - ) -} - -function hasExactKeys(value: object, expected: readonly string[]): boolean { - const keys = Object.keys(value) - return keys.length === expected.length && keys.every((key) => expected.includes(key)) -} diff --git a/src/memory/improvement/identity.ts b/src/memory/improvement/identity.ts deleted file mode 100644 index cf3dcfd..0000000 --- a/src/memory/improvement/identity.ts +++ /dev/null @@ -1,106 +0,0 @@ -import { join } from 'node:path' -import { canonicalJson } from '@tangle-network/agent-eval' -import { type CampaignStorage, surfaceHash } from '@tangle-network/agent-eval/campaign' -import type { AgentCandidateJsonValue as JsonValue } from '@tangle-network/agent-interface' -import type { AgentMemorySequence } from '../experiment' -import { normalizedPromotionPolicy } from './promotion' -import type { RunAgentMemoryImprovementOptions } from './types' - -export function assertMemoryImprovementIdentity( - options: RunAgentMemoryImprovementOptions, - storage: CampaignStorage, - runDir: string, - serialize: (config: TConfig) => string, -): void { - const path = join(runDir, 'memory-improvement-manifest.json') - const identity = { - experimentId: options.experimentId, - implementationRef: options.implementationRef, - method: options.method.name, - activationRef: options.activation?.ref ?? null, - baselineConfig: serialize(options.baselineConfig), - executeStepRef: options.executeStepRef ?? null, - cleanupBranches: options.cleanupBranches ?? true, - promotionPolicy: normalizedPromotionPolicy(options), - seed: options.seed ?? 42, - reps: options.reps ?? 1, - maxTotalCostUsd: options.maxTotalCostUsd ?? null, - maximumEvaluationCostUsd: options.maximumEvaluationCostUsd ?? null, - allowIncompleteCostAccounting: options.allowIncompleteCostAccounting ?? false, - trainSequences: options.trainSequences, - selectionSequences: options.selectionSequences, - finalSequences: options.finalSequences, - } - const identityHash = surfaceHash(canonicalJson(identity)) - const stored = storage.read(path) - if (stored === undefined) { - if (storage.exists(path)) throw new Error(`cannot read memory improvement manifest '${path}'`) - if (storage.exists(join(runDir, 'memory-improvement-result.json'))) { - throw new Error(`memory improvement run '${runDir}' has state without an identity manifest`) - } - storage.write(path, `${JSON.stringify({ identityHash, identity }, null, 2)}\n`) - return - } - let manifest: unknown - try { - manifest = JSON.parse(stored) - } catch (error) { - throw new Error(`invalid memory improvement manifest '${path}'`, { cause: error }) - } - if ( - !manifest || - typeof manifest !== 'object' || - (manifest as Record).identityHash !== identityHash || - canonicalJson((manifest as Record).identity) !== canonicalJson(identity) - ) { - throw new Error( - `memory improvement run '${runDir}' does not match its persisted inputs or implementation`, - ) - } -} - -export function serializeMemoryConfig( - serialize: (config: TConfig) => string, - config: TConfig, -): string { - let surface: unknown - try { - surface = serialize(config) - } catch (error) { - throw new Error('memory config serializer failed', { cause: error }) - } - if (typeof surface !== 'string' || surface.trim().length === 0) { - throw new Error('memory config serializer must return a non-empty string') - } - return surface -} - -export function parseMemoryConfig( - parse: (surface: string) => TConfig, - surface: string, -): TConfig { - try { - const config = parse(surface) - if (config === undefined) throw new Error('parser returned undefined') - return config - } catch (error) { - throw new Error('memory config surface could not be parsed', { cause: error }) - } -} - -export function assertMemoryConfigRoundTrip( - config: TConfig, - serialize: (config: TConfig) => string, - parse: (surface: string) => TConfig, -): void { - const first = serialize(config) - const second = serialize(parse(first)) - if (first !== second) { - throw new Error('memory config serializer and parser must round-trip exactly') - } -} - -export function memorySequenceFingerprint(sequence: AgentMemorySequence): string { - const { id: _id, split: _split, tags: _tags, ...content } = sequence - return surfaceHash(canonicalJson(content)) -} diff --git a/src/memory/improvement/output.ts b/src/memory/improvement/output.ts deleted file mode 100644 index 67dc92f..0000000 --- a/src/memory/improvement/output.ts +++ /dev/null @@ -1,30 +0,0 @@ -import type { CampaignStorage } from '@tangle-network/agent-eval/campaign' -import type { AgentCandidateJsonValue as JsonValue } from '@tangle-network/agent-interface' -import type { RunAgentMemoryImprovementResult } from './types' - -export function writeMemoryImprovementResult( - storage: CampaignStorage, - experimentId: string, - result: RunAgentMemoryImprovementResult, -): void { - storage.write( - result.resultJsonPath, - `${JSON.stringify( - { - experimentId, - baselineSurfaceHash: result.baselineSurfaceHash, - winnerSurfaceHash: result.winnerSurfaceHash, - baselineSurface: result.baselineSurface, - winnerSurface: result.winnerSurface, - decision: result.decision, - method: result.optimization.methodName, - comparison: result.optimization.comparison, - finalEvaluation: result.finalEvaluation, - activation: result.activation, - totalCostUsd: result.totalCostUsd, - }, - null, - 2, - )}\n`, - ) -} diff --git a/src/memory/improvement/promotion.ts b/src/memory/improvement/promotion.ts deleted file mode 100644 index c1a844d..0000000 --- a/src/memory/improvement/promotion.ts +++ /dev/null @@ -1,205 +0,0 @@ -import type { PairedHoldout } from '@tangle-network/agent-eval/campaign' -import { heldoutSignificance } from '@tangle-network/agent-eval/campaign' -import type { AgentCandidateJsonValue as JsonValue } from '@tangle-network/agent-interface' -import type { RunSerializedKnowledgeOptimizationResult } from '../../optimization' -import type { AgentMemorySequence, AgentMemorySequenceProbe } from '../experiment' -import { - type AgentMemoryFinalEvaluation, - type AgentMemoryPromotionDecision, - DEFAULT_CRITICAL_DIMENSIONS, - type RunAgentMemoryImprovementOptions, -} from './types' - -export function decidePromotion(input: { - options: RunAgentMemoryImprovementOptions - optimization: RunSerializedKnowledgeOptimizationResult - finalEvaluation: AgentMemoryFinalEvaluation - unchanged: boolean -}): AgentMemoryPromotionDecision { - const { options, finalEvaluation } = input - const baselineScores = finalEvaluation.pairs.map((pair) => pair.baseline.score) - const winnerScores = finalEvaluation.pairs.map((pair) => pair.winner.score) - const baselineScore = mean(baselineScores) - const winnerScore = mean(winnerScores) - if (input.unchanged) { - return { - status: 'no-change', - reasons: ['optimization selected the baseline configuration'], - baselineScore, - winnerScore, - lift: 0, - criticalDimensions: [], - } - } - - const significance = heldoutSignificance( - { - before: baselineScores, - after: winnerScores, - cellIds: finalEvaluation.pairs.map((pair) => `${pair.sequenceId}:${pair.rep}`), - }, - options.significance, - ) - const tolerance = options.criticalDimensionTolerance ?? 0.05 - const criticalChecks = (options.criticalDimensions ?? DEFAULT_CRITICAL_DIMENSIONS).map( - (dimension) => { - const expectedN = - applicableSequenceCount(options.finalSequences, dimension) * (options.reps ?? 1) - const pairs = pairedDimension(finalEvaluation, dimension) - const comparison = heldoutSignificance(pairs, { - ...options.significance, - deltaThreshold: -tolerance, - }) - const measured = expectedN > 0 && pairs.before.length === expectedN - return { - decision: comparison.decision, - report: { - dimension, - n: comparison.n, - expectedN, - measured, - meanDelta: comparison.bootstrap.mean, - low: comparison.decision.low, - high: comparison.decision.high, - tolerance, - regressed: - measured && !comparison.decision.indeterminate && comparison.decision.high < -tolerance, - }, - } - }, - ) - const reasons: string[] = [] - const totalCost = input.optimization.comparison.totalCost - if ( - !input.optimization.comparison.totalCost.accountingComplete && - !options.allowIncompleteCostAccounting - ) { - reasons.push('optimization or final cost accounting is incomplete') - } - const optimizerSource = input.optimization.comparison.best.provenance?.source - if (optimizerSource && optimizerSource.evidence !== 'observed') { - reasons.push('external optimizer package identity was not observed') - } - if (totalCost.totalCostUsd > (options.maxTotalCostUsd ?? 0)) { - reasons.push( - `total cost ${totalCost.totalCostUsd} exceeds the configured limit ${options.maxTotalCostUsd ?? 0}`, - ) - } - if (!significance.significant) { - reasons.push( - significance.fewRuns - ? `only ${significance.n} paired final observations; at least ${significance.minimumRequired} are required` - : 'final lift is not confidently above the promotion threshold', - ) - } - if (winnerScore < (options.minFinalScore ?? 0)) { - reasons.push(`winner final score ${winnerScore} is below the required minimum`) - } - for (const { report: dimension, decision } of criticalChecks) { - if (!dimension.measured) { - reasons.push( - dimension.expectedN === 0 - ? `critical dimension ${dimension.dimension} has no applicable final histories` - : `critical dimension ${dimension.dimension} was measured on ${dimension.n}/${dimension.expectedN} applicable paired final cells`, - ) - } else if (!decision.sufficient) { - reasons.push( - `critical dimension ${dimension.dimension} needs at least ${decision.minimumPairs} paired final observations; measured ${decision.n}`, - ) - } else if (decision.indeterminate) { - reasons.push(`critical dimension ${dimension.dimension} has an indeterminate final interval`) - } else if (dimension.regressed) { - reasons.push(`critical dimension ${dimension.dimension} regresses beyond ${tolerance}`) - } else if (!decision.promote) { - reasons.push( - `critical dimension ${dimension.dimension} does not exclude a regression beyond ${tolerance}`, - ) - } - } - return { - status: reasons.length === 0 ? 'promote' : 'hold', - reasons, - baselineScore, - winnerScore, - lift: winnerScore - baselineScore, - significance, - criticalDimensions: criticalChecks.map((check) => check.report), - } -} - -export function normalizedPromotionPolicy( - options: RunAgentMemoryImprovementOptions, -): Record { - return { - significance: { - deltaThreshold: options.significance?.deltaThreshold ?? 0, - minProductiveRuns: options.significance?.minProductiveRuns ?? 3, - confidence: options.significance?.confidence ?? 0.95, - resamples: options.significance?.resamples ?? 2000, - seed: options.significance?.seed ?? 1337, - statistic: options.significance?.statistic ?? 'mean', - ...(options.significance?.independentUnitByScenarioId === undefined - ? {} - : { - independentUnitByScenarioId: [...options.significance.independentUnitByScenarioId].sort( - ([left], [right]) => (left < right ? -1 : left > right ? 1 : 0), - ), - }), - }, - criticalDimensions: [...(options.criticalDimensions ?? DEFAULT_CRITICAL_DIMENSIONS)], - criticalDimensionTolerance: options.criticalDimensionTolerance ?? 0.05, - minFinalScore: options.minFinalScore ?? 0, - maxTotalCostUsd: options.maxTotalCostUsd ?? 0, - allowIncompleteCostAccounting: options.allowIncompleteCostAccounting ?? false, - } -} - -function pairedDimension(result: AgentMemoryFinalEvaluation, dimension: string): PairedHoldout { - const applicable = result.pairs.filter( - (pair) => - (pair.baseline.dimensionSampleCounts[dimension] ?? 0) > 0 && - (pair.winner.dimensionSampleCounts[dimension] ?? 0) > 0, - ) - return { - before: applicable.map((pair) => pair.baseline.dimensions[dimension]!), - after: applicable.map((pair) => pair.winner.dimensions[dimension]!), - cellIds: applicable.map((pair) => `${pair.sequenceId}:${pair.rep}`), - } -} - -function applicableSequenceCount( - sequences: readonly AgentMemorySequence[], - dimension: string, -): number { - return sequences.filter((sequence) => - sequence.steps.some((step) => - (step.probes ?? []).some((probe) => probeAppliesToDimension(probe, dimension)), - ), - ).length -} - -function probeAppliesToDimension(probe: AgentMemorySequenceProbe, dimension: string): boolean { - switch (dimension) { - case 'memory_fact_recall': - case 'memory_required_fact_count': - case 'memory_matched_fact_count': - return Boolean( - (probe.requiredFacts && probe.requiredFacts.length > 0) || probe.referenceAnswer, - ) - case 'memory_event_recall': - return Boolean(probe.expectedEventIds && probe.expectedEventIds.length > 0) - case 'memory_actor_recall': - return Boolean(probe.expectedActorIds && probe.expectedActorIds.length > 0) - case 'memory_stale_safe': - case 'memory_stale_rate': - case 'memory_forbidden_fact_count': - case 'memory_matched_forbidden_fact_count': - return Boolean(probe.forbiddenFacts && probe.forbiddenFacts.length > 0) - default: - return true - } -} - -function mean(values: readonly number[]): number { - return values.reduce((sum, value) => sum + value, 0) / values.length -} diff --git a/src/memory/improvement/run.ts b/src/memory/improvement/run.ts deleted file mode 100644 index 29f24b9..0000000 --- a/src/memory/improvement/run.ts +++ /dev/null @@ -1,290 +0,0 @@ -import { join } from 'node:path' -import { canonicalJson } from '@tangle-network/agent-eval' -import { - type CampaignStorage, - fsCampaignStorage, - resolveRunDir, - surfaceHash, -} from '@tangle-network/agent-eval/campaign' -import type { AgentCandidateJsonValue as JsonValue } from '@tangle-network/agent-interface' -import { runSerializedKnowledgeOptimization } from '../../optimization' -import { - type AgentMemoryExperimentCandidate, - type AgentMemorySequenceArtifact, - agentMemorySequenceJudge, -} from '../experiment' -import { acquireAgentMemoryRunLease } from '../run-control' -import { - activateMemoryWinner, - assertActivatedMemoryWinner, - readMemoryActivationJournal, -} from './activation' -import { buildRegisteredCandidate } from './candidate' -import { - evaluateMemoryCandidate, - loadFinalEvaluation, - memoryArtifactPath, - memoryConfigCodec, - memoryConfigScenarios, -} from './evaluation' -import { - assertMemoryConfigRoundTrip, - assertMemoryImprovementIdentity, - memorySequenceFingerprint, -} from './identity' -import { writeMemoryImprovementResult } from './output' -import { decidePromotion, normalizedPromotionPolicy } from './promotion' -import type { - AgentMemoryActivation, - MemoryConfigScenario, - OwnedRunLease, - RunAgentMemoryImprovementOptions, - RunAgentMemoryImprovementResult, -} from './types' -import { assertMemoryImprovementOptions } from './validation' - -/** Optimizes memory configuration with an external method and activates only a fresh final win. */ -export async function runAgentMemoryImprovement( - options: RunAgentMemoryImprovementOptions, -): Promise> { - if (options.significance !== undefined) { - options = { - ...options, - significance: { - ...options.significance, - ...(options.significance.independentUnitByScenarioId === undefined - ? {} - : { - independentUnitByScenarioId: new Map( - options.significance.independentUnitByScenarioId, - ), - }), - }, - } - } - assertMemoryImprovementOptions(options) - const storage = options.storage ?? fsCampaignStorage() - const runDir = resolveRunDir(options.runDir, options.repo) - storage.ensureDir(runDir) - const lease = await acquireAgentMemoryRunLease({ - experimentId: options.experimentId, - runDir, - storage, - customStorage: options.storage !== undefined, - lockFileName: 'memory-improvement.lock', - label: 'memory improvement', - controllerMode: options.controllerMode, - acquireRunLease: options.acquireRunLease, - }) - let result: RunAgentMemoryImprovementResult | undefined - let primaryError: unknown - try { - result = await runAgentMemoryImprovementOwned(options, storage, runDir, lease) - } catch (error) { - primaryError = error - } - let releaseError: unknown - try { - await lease.release() - } catch (error) { - releaseError = error - } - if (primaryError && releaseError) { - throw new AggregateError( - [primaryError, releaseError], - 'memory improvement failed and its controller lease could not be released', - ) - } - if (primaryError) throw primaryError - if (releaseError) throw releaseError - if (!result) throw new Error('memory improvement produced no result') - return result -} - -async function runAgentMemoryImprovementOwned( - options: RunAgentMemoryImprovementOptions, - storage: CampaignStorage, - runDir: string, - lease: OwnedRunLease, -): Promise> { - await lease.assertOwned() - const codec = memoryConfigCodec(options) - assertMemoryConfigRoundTrip(options.baselineConfig, codec.serialize, codec.parse) - if (options.activation && !storage.append) { - throw new Error('memory activation requires CampaignStorage.append') - } - assertMemoryImprovementIdentity(options, storage, runDir, codec.serialize) - - const trainScenarios = memoryConfigScenarios(options.trainSequences) - const selectionScenarios = memoryConfigScenarios(options.selectionSequences) - const finalScenarios = memoryConfigScenarios(options.finalSequences) - const finalScenarioIds = new Set(finalScenarios.map((scenario) => scenario.id)) - const pending = new Map>() - const candidates = new Map>() - const candidateFor = (config: TConfig, hash: string) => { - let candidate = candidates.get(hash) - if (!candidate) { - candidate = buildRegisteredCandidate(options, storage, runDir, config, hash) - candidates.set(hash, candidate) - void candidate.catch(() => candidates.delete(hash)) - } - return candidate - } - const optimizationRunOptions = { - ...(options.optimizationRunOptions ?? {}), - storage, - ...(options.reps !== undefined ? { reps: options.reps } : {}), - ...(options.resumable !== undefined ? { resumable: options.resumable } : {}), - ...(options.sequenceConcurrency !== undefined - ? { maxConcurrency: options.sequenceConcurrency } - : {}), - ...(options.dispatchTimeoutMs !== undefined - ? { dispatchTimeoutMs: options.dispatchTimeoutMs } - : {}), - expectUsage: 'off' as const, - } - const optimization = await runSerializedKnowledgeOptimization({ - executionRef: options.implementationRef, - baseline: options.baselineConfig, - method: options.method, - trainScenarios, - selectionScenarios, - finalScenarios, - codec, - scenarioFingerprint: (scenario) => memorySequenceFingerprint(scenario.sequence), - dispatchCandidate: async ({ candidate, candidateSurfaceHash, scenario, context }) => { - const key = memoryArtifactPath( - runDir, - candidateSurfaceHash, - scenario, - context.rep, - context.seed, - ) - let operation = pending.get(key) - if (!operation) { - operation = candidateFor(candidate, candidateSurfaceHash).then((builtCandidate) => - evaluateMemoryCandidate({ - options, - storage, - runDir, - lease, - candidate: builtCandidate, - surfaceHash: candidateSurfaceHash, - scenario, - rep: context.rep, - seed: context.seed, - final: finalScenarioIds.has(scenario.id), - cost: context.cost, - signal: context.signal, - }), - ) - pending.set(key, operation) - void operation.catch(() => pending.delete(key)) - } - return operation - }, - judges: [agentMemorySequenceJudge()], - runDir, - repo: options.repo, - storage, - seed: options.seed, - reps: options.reps, - resumable: options.resumable, - costCeiling: options.maxTotalCostUsd ?? 0, - maxConcurrency: options.sequenceConcurrency, - dispatchTimeoutMs: options.dispatchTimeoutMs, - expectUsage: 'off', - optimizationRunOptions, - now: options.now, - }) - await lease.assertOwned() - const [baselineCandidate, winnerCandidate] = await Promise.all([ - candidateFor(optimization.baseline.value, optimization.baseline.surfaceHash), - candidateFor(optimization.winner.value, optimization.winner.surfaceHash), - ]) - - const finalEvaluation = loadFinalEvaluation({ - options, - storage, - runDir, - baselineSurfaceHash: optimization.baseline.surfaceHash, - baselineCandidateRef: baselineCandidate.ref, - winnerSurfaceHash: optimization.winner.surfaceHash, - winnerCandidateRef: winnerCandidate.ref, - }) - const unchanged = optimization.baseline.surfaceHash === optimization.winner.surfaceHash - const decision = decidePromotion({ options, optimization, finalEvaluation, unchanged }) - const resultJsonPath = join(runDir, 'memory-improvement-result.json') - const activationRef = options.activation?.ref ?? 'not-configured' - const activationId = `memory-activation-${surfaceHash( - canonicalJson({ - experimentId: options.experimentId, - implementationRef: options.implementationRef, - method: optimization.methodName, - activationRef, - baselineSurfaceHash: optimization.baseline.surfaceHash, - winnerSurfaceHash: optimization.winner.surfaceHash, - finalEvaluationHash: finalEvaluation.manifestHash, - promotionPolicy: normalizedPromotionPolicy(options), - }), - )}` - const activationJournalDir = join(runDir, 'activations') - const activationJournalPath = join(activationJournalDir, `${activationId}.jsonl`) - const activationEligible = decision.status === 'promote' - const activationEventIdentity = { - activationId, - experimentId: options.experimentId, - activationRef, - baselineSurfaceHash: optimization.baseline.surfaceHash, - winnerSurfaceHash: optimization.winner.surfaceHash, - finalEvaluationHash: finalEvaluation.manifestHash, - } - const activationJournal = activationEligible - ? readMemoryActivationJournal(storage, activationJournalPath, activationEventIdentity) - : { prepared: false } - const activation: AgentMemoryActivation = { - id: activationId, - status: !activationEligible - ? 'not-eligible' - : activationJournal.activated - ? 'already-activated' - : options.activation - ? 'pending' - : 'not-configured', - journalPath: activationJournalPath, - } - const result = { - optimization, - baselineConfig: optimization.baseline.value, - winnerConfig: optimization.winner.value, - baselineSurface: optimization.baseline.surface, - winnerSurface: optimization.winner.surface, - baselineSurfaceHash: optimization.baseline.surfaceHash, - winnerSurfaceHash: optimization.winner.surfaceHash, - decision, - finalEvaluation, - activation, - totalCostUsd: optimization.comparison.totalCost.totalCostUsd, - resultJsonPath, - } satisfies RunAgentMemoryImprovementResult - await lease.assertOwned() - if (activationJournal.activated) { - await assertActivatedMemoryWinner({ options, lease, result }) - } - writeMemoryImprovementResult(storage, options.experimentId, result) - - if (activationEligible && !activationJournal.activated && options.activation) { - await activateMemoryWinner({ - options, - storage, - lease, - result, - activationEventIdentity, - activationJournalDir, - activationJournalPath, - hadPreparedEvent: activationJournal.prepared, - }) - writeMemoryImprovementResult(storage, options.experimentId, result) - } - return result -} diff --git a/src/memory/improvement/types.ts b/src/memory/improvement/types.ts deleted file mode 100644 index dfd5cf0..0000000 --- a/src/memory/improvement/types.ts +++ /dev/null @@ -1,209 +0,0 @@ -import type { - CampaignStorage, - HeldoutSignificance, - HeldoutSignificanceOptions, - OptimizationMethod, - OptimizationMethodRunOptions, - Scenario, -} from '@tangle-network/agent-eval/campaign' -import type { AgentCandidateJsonValue as JsonValue } from '@tangle-network/agent-interface' -import type { RunSerializedKnowledgeOptimizationResult } from '../../optimization' -import type { - AgentMemoryExperimentCandidate, - AgentMemorySequence, - AgentMemorySequenceArtifact, - RunAgentMemoryExperimentOptions, -} from '../experiment' -import type { - AgentMemoryAcquireRunLease, - AgentMemoryControllerMode, - AgentMemoryRunLease, - OwnedAgentMemoryRunLease, -} from '../run-control' - -export interface AgentMemoryDimensionComparison { - dimension: string - n: number - expectedN: number - measured: boolean - meanDelta: number - low: number - high: number - tolerance: number - regressed: boolean -} - -export interface AgentMemoryPromotionDecision { - status: 'promote' | 'hold' | 'no-change' - reasons: readonly string[] - baselineScore: number - winnerScore: number - lift: number - significance?: HeldoutSignificance - criticalDimensions: readonly AgentMemoryDimensionComparison[] -} - -export interface AgentMemoryFinalPair { - sequenceId: string - rep: number - baseline: AgentMemorySequenceArtifact - winner: AgentMemorySequenceArtifact -} - -export interface AgentMemoryFinalEvaluation { - manifestHash: string - baselineCandidateRef: string - winnerCandidateRef: string - pairs: readonly AgentMemoryFinalPair[] -} - -export interface AgentMemoryActivation { - id: string - status: - | 'not-eligible' - | 'not-configured' - | 'pending' - | 'activated' - | 'recovered' - | 'already-activated' - journalPath: string -} - -export interface AgentMemoryActivationDriver { - /** Change whenever activation behavior or the external target changes. */ - ref: string - /** Return the exact currently active configuration. */ - readCurrent(): Promise - /** Atomically replace expectedConfig with config, or fail on a concurrent change. */ - compareAndSet(input: { - activationId: string - expectedConfig: TConfig - expectedSurfaceHash: string - config: TConfig - surfaceHash: string - decision: AgentMemoryPromotionDecision - optimization: RunSerializedKnowledgeOptimizationResult - finalEvaluation: AgentMemoryFinalEvaluation - }): Promise -} - -export interface AgentMemoryActivationEvent { - status: 'prepared' | 'activated' - activationId: string - experimentId: string - activationRef: string - baselineSurfaceHash: string - winnerSurfaceHash: string - finalEvaluationHash: string - recordedAt: string - outcome?: 'applied' | 'recovered' | 'already-current' -} - -export interface AgentMemoryActivationJournalState { - prepared: boolean - activated?: AgentMemoryActivationEvent -} - -export type AgentMemoryImprovementRunLease = AgentMemoryRunLease - -export type AgentMemoryImprovementCandidate = Omit< - AgentMemoryExperimentCandidate, - | 'id' - | 'externalCostUsdPerSequence' - | 'externalRecoveryCostUsdPerAttempt' - | 'externalCostAccounting' -> & - Required< - Pick< - AgentMemoryExperimentCandidate, - 'externalCostUsdPerSequence' | 'externalRecoveryCostUsdPerAttempt' - > - > & { - externalCostAccounting: 'exact' - } - -export interface RunAgentMemoryImprovementOptions { - experimentId: string - baselineConfig: TConfig - method: OptimizationMethod - trainSequences: readonly AgentMemorySequence[] - selectionSequences: readonly AgentMemorySequence[] - finalSequences: readonly AgentMemorySequence[] - createCandidate(input: { - config: TConfig - candidateId: string - surfaceHash: string - }): AgentMemoryImprovementCandidate | Promise - /** - * Immutable digest covering the installed implementation, method config, - * candidate construction, execution behavior, and external settings. - */ - implementationRef: string - runDir: string - repo?: string - storage?: CampaignStorage - /** Required with custom storage when all controllers are confined to one process. */ - controllerMode?: AgentMemoryControllerMode - /** Required for distributed controllers using custom storage. */ - acquireRunLease?: AgentMemoryAcquireRunLease - seed?: number - reps?: number - resumable?: boolean - sequenceConcurrency?: number - dispatchTimeoutMs?: number - cleanupTimeoutMs?: number - maxRecoveryAttempts?: number - maxRecoveryRetriesPerAttempt?: number - /** Total spend limit across method search and final comparison. Default 0. */ - maxTotalCostUsd?: number - /** Enforced maximum for one config and one sequence. Required with either spend limit. */ - maximumEvaluationCostUsd?: number - /** Allow activation when a method cannot fully account for cost. Default false. */ - allowIncompleteCostAccounting?: boolean - optimizationRunOptions?: OptimizationMethodRunOptions< - MemoryConfigScenario, - AgentMemorySequenceArtifact - > - executeStep?: RunAgentMemoryExperimentOptions['executeStep'] - executeStepRef?: string - onBranchSnapshot?: RunAgentMemoryExperimentOptions['onBranchSnapshot'] - cleanupBranches?: boolean - serializeConfig?: (config: TConfig) => string - parseConfig?: (surface: string) => TConfig - significance?: HeldoutSignificanceOptions - criticalDimensions?: readonly string[] - criticalDimensionTolerance?: number - minFinalScore?: number - activation?: AgentMemoryActivationDriver - activationTimeoutMs?: number - now?: () => Date -} - -export interface RunAgentMemoryImprovementResult { - optimization: RunSerializedKnowledgeOptimizationResult - baselineConfig: TConfig - winnerConfig: TConfig - baselineSurface: string - winnerSurface: string - baselineSurfaceHash: string - winnerSurfaceHash: string - decision: AgentMemoryPromotionDecision - finalEvaluation: AgentMemoryFinalEvaluation - activation: AgentMemoryActivation - totalCostUsd: number - resultJsonPath: string -} - -export interface MemoryConfigScenario extends Scenario { - kind: 'agent-memory-config-search' - sequenceId: string - sequence: AgentMemorySequence -} - -export const DEFAULT_CRITICAL_DIMENSIONS = [ - 'memory_stale_safe', - 'memory_actor_recall', - 'memory_event_recall', -] as const - -export type OwnedRunLease = OwnedAgentMemoryRunLease diff --git a/src/memory/improvement/validation.ts b/src/memory/improvement/validation.ts deleted file mode 100644 index 47af544..0000000 --- a/src/memory/improvement/validation.ts +++ /dev/null @@ -1,143 +0,0 @@ -import type { AgentCandidateJsonValue as JsonValue } from '@tangle-network/agent-interface' -import { assertImmutableRef } from '../../immutable-ref' -import type { AgentMemorySequence } from '../experiment' -import { memorySequenceFingerprint } from './identity' -import { DEFAULT_CRITICAL_DIMENSIONS, type RunAgentMemoryImprovementOptions } from './types' - -export function assertMemoryImprovementOptions( - options: RunAgentMemoryImprovementOptions, -): void { - for (const [name, value] of [ - ['experimentId', options.experimentId], - ['runDir', options.runDir], - ] as const) { - if (typeof value !== 'string' || !value.trim()) { - throw new Error(`memory improvement ${name} must be a non-empty string`) - } - } - assertImmutableRef(options.implementationRef, 'memory improvement implementationRef') - if ( - !options.method || - typeof options.method.name !== 'string' || - !options.method.name.trim() || - typeof options.method.optimize !== 'function' - ) { - throw new Error('memory improvement method must be a complete OptimizationMethod') - } - if (typeof options.createCandidate !== 'function') { - throw new Error('memory improvement createCandidate must be a function') - } - if (options.activation !== undefined) { - assertImmutableRef(options.activation.ref, 'memory improvement activation.ref') - if (typeof options.activation.readCurrent !== 'function') { - throw new Error('memory improvement activation.readCurrent must be a function') - } - if (typeof options.activation.compareAndSet !== 'function') { - throw new Error('memory improvement activation.compareAndSet must be a function') - } - } - if (options.executeStep) { - assertImmutableRef(options.executeStepRef, 'memory improvement executeStepRef') - } - if (options.serializeConfig !== undefined && typeof options.serializeConfig !== 'function') { - throw new Error('memory improvement serializeConfig must be a function') - } - if (options.parseConfig !== undefined && typeof options.parseConfig !== 'function') { - throw new Error('memory improvement parseConfig must be a function') - } - for (const [name, value] of [ - ['sequenceConcurrency', options.sequenceConcurrency], - ['reps', options.reps], - ['maxRecoveryAttempts', options.maxRecoveryAttempts], - ['maxRecoveryRetriesPerAttempt', options.maxRecoveryRetriesPerAttempt], - ] as const) { - if (value !== undefined && (!Number.isSafeInteger(value) || value <= 0)) { - throw new Error(`memory improvement ${name} must be a positive safe integer`) - } - } - for (const [name, value] of [['maxTotalCostUsd', options.maxTotalCostUsd]] as const) { - if (value !== undefined && (!Number.isFinite(value) || value < 0)) { - throw new Error(`memory improvement ${name} must be a non-negative finite number`) - } - } - if ( - options.maximumEvaluationCostUsd !== undefined && - (!Number.isFinite(options.maximumEvaluationCostUsd) || options.maximumEvaluationCostUsd <= 0) - ) { - throw new Error('memory improvement maximumEvaluationCostUsd must be a positive finite number') - } - if ((options.maxTotalCostUsd ?? 0) > 0 && options.maximumEvaluationCostUsd === undefined) { - throw new Error( - 'memory improvement maximumEvaluationCostUsd is required when a spend limit is configured', - ) - } - if ( - options.allowIncompleteCostAccounting !== undefined && - typeof options.allowIncompleteCostAccounting !== 'boolean' - ) { - throw new Error('memory improvement allowIncompleteCostAccounting must be boolean') - } - if ( - options.activationTimeoutMs !== undefined && - (!Number.isSafeInteger(options.activationTimeoutMs) || options.activationTimeoutMs <= 0) - ) { - throw new Error('memory improvement activationTimeoutMs must be a positive safe integer') - } - const tolerance = options.criticalDimensionTolerance - if (tolerance !== undefined && (!Number.isFinite(tolerance) || tolerance < 0 || tolerance > 1)) { - throw new Error('memory improvement criticalDimensionTolerance must be between 0 and 1') - } - const minimum = options.minFinalScore - if (minimum !== undefined && (!Number.isFinite(minimum) || minimum < 0 || minimum > 1)) { - throw new Error('memory improvement minFinalScore must be between 0 and 1') - } - const dimensions = options.criticalDimensions ?? DEFAULT_CRITICAL_DIMENSIONS - if (dimensions.some((dimension) => typeof dimension !== 'string' || !dimension.trim())) { - throw new Error('memory improvement criticalDimensions must contain non-empty names') - } - if (new Set(dimensions).size !== dimensions.length) { - throw new Error('memory improvement criticalDimensions must be unique') - } - assertIndependentSequences( - options.trainSequences, - options.selectionSequences, - options.finalSequences, - ) -} - -function assertIndependentSequences( - train: readonly AgentMemorySequence[], - selection: readonly AgentMemorySequence[], - final: readonly AgentMemorySequence[], -): void { - if (train.length === 0) throw new Error('memory improvement requires training sequences') - if (selection.length === 0) throw new Error('memory improvement requires selection sequences') - if (final.length < 2) { - throw new Error('memory improvement requires at least 2 final sequences') - } - const ids = new Map() - const content = new Map() - for (const [split, sequences] of [ - ['train', train], - ['selection', selection], - ['final', final], - ] as const) { - for (const sequence of sequences) { - const priorId = ids.get(sequence.id) - if (priorId) { - throw new Error( - `memory improvement ${priorId}/${split} sequences share id '${sequence.id}'`, - ) - } - ids.set(sequence.id, split) - const fingerprint = memorySequenceFingerprint(sequence) - const priorContent = content.get(fingerprint) - if (priorContent) { - throw new Error( - `memory improvement ${priorContent.split}/${split} histories duplicate content at '${priorContent.sequenceId}'/'${sequence.id}'`, - ) - } - content.set(fingerprint, { split, sequenceId: sequence.id }) - } - } -} diff --git a/src/memory/index.ts b/src/memory/index.ts index d2905d0..c54e709 100644 --- a/src/memory/index.ts +++ b/src/memory/index.ts @@ -1,10 +1,8 @@ export * from './adapter' export * from './branch' -export * from './experiment' export * from './graphiti' export * from './hindsight' export * from './holdout' -export * from './improvement' export { AgentMemoryLifecycleTimeoutError, AgentMemoryLifecycleUnsafeError, @@ -17,7 +15,6 @@ export { } from './lifecycle' export * from './mem0' export * from './neo4j' -export * from './run-control' export * from './schemas' export * from './source-record' export * from './tools' diff --git a/src/memory/run-control.ts b/src/memory/run-control.ts deleted file mode 100644 index bf78273..0000000 --- a/src/memory/run-control.ts +++ /dev/null @@ -1,93 +0,0 @@ -import { join } from 'node:path' -import { acquireSingleRunLock, type CampaignStorage } from '@tangle-network/agent-eval/campaign' - -export interface AgentMemoryRunLease { - assertOwned(): Promise | void - release(): Promise | void -} - -export type AgentMemoryAcquireRunLease = (input: { - experimentId: string - runDir: string -}) => AgentMemoryRunLease | Promise - -export type AgentMemoryControllerMode = 'process-local' - -export interface OwnedAgentMemoryRunLease { - assertOwned(): Promise - release(): Promise -} - -const activeProcessLocalRuns = new Set() - -export async function acquireAgentMemoryRunLease(input: { - experimentId: string - runDir: string - storage: CampaignStorage - customStorage: boolean - lockFileName: string - label: string - controllerMode?: AgentMemoryControllerMode - acquireRunLease?: AgentMemoryAcquireRunLease -}): Promise { - if (input.acquireRunLease && input.controllerMode) { - throw new Error(`${input.label} cannot combine acquireRunLease with controllerMode`) - } - if (input.acquireRunLease) { - const lease = await input.acquireRunLease({ - experimentId: input.experimentId, - runDir: input.runDir, - }) - if (!lease || typeof lease.assertOwned !== 'function' || typeof lease.release !== 'function') { - throw new Error(`${input.label} acquireRunLease must return assertOwned and release`) - } - return { - async assertOwned() { - await lease.assertOwned() - }, - async release() { - await lease.release() - }, - } - } - - if (!input.customStorage) { - if (input.controllerMode) { - throw new Error(`${input.label} controllerMode is only valid with custom storage`) - } - const lock = acquireSingleRunLock({ - lockPath: join(input.runDir, input.lockFileName), - releaseOnExit: false, - }) - return { - async assertOwned() {}, - async release() { - lock.release() - }, - } - } - - if (input.controllerMode !== 'process-local') { - throw new Error( - `${input.label} custom storage requires acquireRunLease or controllerMode='process-local'`, - ) - } - const runKey = `${input.label}:${input.runDir}` - if (activeProcessLocalRuns.has(runKey)) { - throw new Error(`${input.label} run '${input.runDir}' already has an active controller`) - } - activeProcessLocalRuns.add(runKey) - let released = false - return { - async assertOwned() { - if (released || !activeProcessLocalRuns.has(runKey)) { - throw new Error(`${input.label} run '${input.runDir}' controller lease was lost`) - } - }, - async release() { - if (released) return - released = true - activeProcessLocalRuns.delete(runKey) - }, - } -} diff --git a/src/statistics.test.ts b/src/statistics.test.ts index d1d2e7f..95df4a3 100644 --- a/src/statistics.test.ts +++ b/src/statistics.test.ts @@ -1,5 +1,4 @@ import { describe, expect, it } from 'vitest' -import { rankCandidates } from './candidate-ranking' import { mean } from './statistics' describe('mean', () => { @@ -12,19 +11,4 @@ describe('mean', () => { expect(mean([1, Number.NaN, 3])).toBe(2) expect(mean([1, Number.POSITIVE_INFINITY, 3])).toBe(2) }) - - it('keeps a ranked candidate placed by what it scored, not by its name', () => { - // A non-finite scoreMean fails every numeric comparison in the ranking - // comparator, so the candidate carrying it falls through to the identifier - // tie-break: 'broken' would outrank two candidates with real scores purely - // because of where its name sorts. - const base = { passRate: 0.5, cellsFailed: 0, totalCostUsd: 1 } - const ranked = rankCandidates([ - { ...base, candidateId: 'alpha', scoreMean: mean([0.9]) }, - { ...base, candidateId: 'broken', scoreMean: mean([Number.NaN, 0.2]) }, - { ...base, candidateId: 'gamma', scoreMean: mean([0.1]) }, - { ...base, candidateId: 'delta', scoreMean: mean([0.5]) }, - ]) - expect(ranked.map((row) => row.candidateId)).toEqual(['alpha', 'delta', 'broken', 'gamma']) - }) }) diff --git a/tests/benchmarks/catalog.test.ts b/tests/benchmarks/catalog.test.ts deleted file mode 100644 index 2cbf279..0000000 --- a/tests/benchmarks/catalog.test.ts +++ /dev/null @@ -1,309 +0,0 @@ -import { inMemoryCampaignStorage } from '@tangle-network/agent-eval/campaign' -import { describe, expect, it } from 'vitest' -import { - buildIndustryMemoryBenchmarkSmokeCases, - buildIndustryRagBenchmarkSmokeCases, - buildRetrievalBenchmarkCasesFromQrels, - INDUSTRY_MEMORY_BENCHMARKS, - INDUSTRY_RAG_BENCHMARKS, - isKnowledgeMemoryBenchmarkCase, - type KnowledgeAnswerBenchmarkCase, - type KnowledgeMemoryBenchmarkCase, - knowledgeBenchmarkJudge, - parseKnowledgeBenchmarkJsonl, - parseKnowledgeBenchmarkQrels, - respondToIndustryMemoryBenchmarkSmokeCase, - respondToIndustryRagBenchmarkSmokeCase, - runKnowledgeBenchmarkSuite, - scoreKnowledgeBenchmarkArtifact, - scoreMemoryBenchmarkArtifact, -} from '../../src/benchmarks/index' - -describe('knowledge benchmark catalog and scoring', () => { - it('versions benchmark scoring for resumable cache safety', () => { - expect(knowledgeBenchmarkJudge().judgeVersion).toBe('agent-knowledge:knowledge-benchmark:v2') - }) - - it('requires a responder version for resumable benchmark rows', async () => { - await expect( - runKnowledgeBenchmarkSuite({ - cases: buildIndustryRagBenchmarkSmokeCases().slice(0, 1), - runDir: '/runs/missing-responder-ref', - storage: inMemoryCampaignStorage(), - respond: respondToIndustryRagBenchmarkSmokeCase, - }), - ).rejects.toThrow('respondRef is required when resumable is enabled') - }) - - it('parses qrels/jsonl and builds retrieval cases for public benchmark formats', () => { - expect(parseKnowledgeBenchmarkJsonl('{"id":"q1"}\n{"id":"q2"}\n')).toEqual([ - { id: 'q1' }, - { id: 'q2' }, - ]) - const qrels = parseKnowledgeBenchmarkQrels('q1 0 page-1 1\nq1 0 page-2 0\nq2 page-3 2') - expect(qrels).toEqual([ - { queryId: 'q1', documentId: 'page-1', score: 1 }, - { queryId: 'q1', documentId: 'page-2', score: 0 }, - { queryId: 'q2', documentId: 'page-3', score: 2 }, - ]) - - const cases = buildRetrievalBenchmarkCasesFromQrels({ - benchmarkId: 'beir/smoke', - family: 'beir', - queries: [ - { id: 'q1', text: 'refund policy', split: 'search' }, - { id: 'q2', text: 'shipping speed', split: 'holdout', tags: ['commerce'] }, - ], - qrels, - targetKind: 'page', - }) - - expect(cases).toHaveLength(2) - expect(cases[0]).toMatchObject({ - id: 'beir/smoke:q1', - taskKind: 'retrieval', - split: 'search', - expected: [{ kind: 'page', pageId: 'page-1' }], - }) - expect(cases[1]?.tags).toEqual(['commerce', 'holdout']) - }) - - it('runs retrieval benchmark cases through the campaign-backed suite', async () => { - const storage = inMemoryCampaignStorage() - const cases = buildRetrievalBenchmarkCasesFromQrels({ - benchmarkId: 'beir/suite-smoke', - family: 'beir', - queries: [ - { id: 'q1', text: 'refund policy', split: 'search' }, - { id: 'q2', text: 'shipping speed', split: 'holdout' }, - ], - qrels: [ - { queryId: 'q1', documentId: 'page-1', score: 1 }, - { queryId: 'q2', documentId: 'page-2', score: 1 }, - ], - targetKind: 'page', - k: 2, - }) - const result = await runKnowledgeBenchmarkSuite({ - cases, - runDir: '/runs/knowledge-benchmark-smoke', - storage, - respondRef: 'retriever-fixture:v1', - costCeiling: 1, - respond: async ({ case: testCase, context }) => { - const paid = await context.cost.runPaidCall({ - actor: 'benchmark-retriever', - model: 'retriever-fixture', - maximumCharge: { externallyEnforcedMaximumUsd: 0.01 }, - execute: async () => ({ - costUsd: 0.01, - hits: [ - testCase.id.endsWith('q1') - ? { pageId: 'page-1', path: 'knowledge/page-1.md', rank: 1 } - : { pageId: 'miss', path: 'knowledge/miss.md', rank: 1 }, - ], - }), - receipt: () => ({ - model: 'retriever-fixture', - inputTokens: 0, - outputTokens: 0, - usageUnknown: true, - actualCostUsd: 0.01, - }), - }) - if (!paid.succeeded) throw paid.error - return paid.value - }, - }) - - expect(result.report.totalCases).toBe(2) - expect(result.report.cellsFailed).toBe(0) - expect(result.report.score.mean).toBe(0.5) - expect(result.report.byFamily.beir?.n).toBe(2) - expect(result.report.bySplit.search?.meanScore).toBe(1) - expect(result.report.bySplit.holdout?.meanScore).toBe(0) - expect(result.report.totalCostUsd).toBe(0.02) - expect(storage.read(result.reportJsonPath)).toContain('"totalCases": 2') - expect(storage.read(result.reportMarkdownPath)).toContain('# Knowledge Benchmark Report') - }) - - it('scores RAG answer, hallucination, and KB-improvement cases with claim/source checks', () => { - const testCase: KnowledgeAnswerBenchmarkCase = { - id: 'crag/smoke:q1', - family: 'crag', - taskKind: 'rag-answer', - prompt: 'What is the refund policy?', - requiredClaims: [{ id: 'refund-window', anyOf: ['30 day refund', '30-day refund'] }], - forbiddenClaims: [{ id: 'unsupported-lifetime', anyOf: ['lifetime refund'] }], - expectedSourceIds: ['src-policy', 'src-terms'], - } - - const partial = scoreKnowledgeBenchmarkArtifact(testCase, { - answer: 'The product has a 30-day refund period.', - citedSourceIds: ['src-policy'], - }) - expect(partial.dimensions.claim_recall).toBe(1) - expect(partial.dimensions.citation_recall).toBe(0.5) - expect(partial.dimensions.hallucination_safe).toBe(1) - expect(partial.score).toBeCloseTo(5 / 6) - expect(partial.passed).toBe(false) - - const hallucinated = scoreKnowledgeBenchmarkArtifact( - { ...testCase, taskKind: 'hallucination' }, - { answer: 'The product has a lifetime refund.' }, - ) - expect(hallucinated.dimensions.hallucination_safe).toBe(0) - expect(hallucinated.raw.matchedForbiddenClaimIds).toEqual(['unsupported-lifetime']) - }) - - it('declares every requested industry benchmark family', () => { - const ids = new Set(INDUSTRY_RAG_BENCHMARKS.map((benchmark) => benchmark.id)) - for (const id of [ - 'beir', - 'mteb-retrieval', - 'msmarco', - 'trec-dl', - 'miracl', - 'lotte', - 'bright', - 'crag', - 'hotpotqa', - 'kilt', - 'ragtruth', - 'faithbench', - 'first-party/kb-improvement', - ]) { - expect(ids.has(id)).toBe(true) - } - }) - - it('runs one persisted benchmark cell for every declared industry family', async () => { - const storage = inMemoryCampaignStorage() - const cases = buildIndustryRagBenchmarkSmokeCases() - const result = await runKnowledgeBenchmarkSuite({ - cases, - runDir: '/runs/knowledge-benchmark-family-smoke', - storage, - respondRef: 'industry-rag-smoke:v1', - respond: respondToIndustryRagBenchmarkSmokeCase, - }) - - expect(cases).toHaveLength(INDUSTRY_RAG_BENCHMARKS.length) - expect(result.report.totalCases).toBe(INDUSTRY_RAG_BENCHMARKS.length) - expect(result.report.totalCells).toBe(INDUSTRY_RAG_BENCHMARKS.length) - expect(result.report.cellsFailed).toBe(0) - expect(result.report.score.mean).toBe(1) - expect(result.report.byTaskKind.retrieval?.n).toBe(7) - expect(result.report.byTaskKind['rag-answer']?.n).toBe(3) - expect(result.report.byTaskKind.hallucination?.n).toBe(2) - expect(result.report.byTaskKind['kb-improvement']?.n).toBe(1) - expect(result.report.totalCostUsd).toBe(0) - for (const benchmark of INDUSTRY_RAG_BENCHMARKS) { - expect(result.report.byFamily[benchmark.family]?.n).toBeGreaterThanOrEqual(1) - } - expect(storage.read(result.reportJsonPath)).toContain('"totalCases": 13') - expect(storage.read(result.reportMarkdownPath)).toContain('## Task Kinds') - }) - - it('calibrates memory scoring against stale and current facts', () => { - const testCase: KnowledgeMemoryBenchmarkCase = { - id: 'memora/smoke:q1', - family: 'memora', - taskKind: 'memory-update', - prompt: 'What does the user currently prefer for daily briefings?', - events: [ - { - id: 'e-old', - actorId: 'user', - timestamp: '2026-01-01T00:00:00.000Z', - text: 'The user used to prefer SMS briefings.', - }, - { - id: 'e-new', - actorId: 'user', - timestamp: '2026-02-01T00:00:00.000Z', - text: 'The user now prefers email briefings.', - }, - ], - requiredFacts: [{ id: 'current-channel', anyOf: ['email briefings'] }], - forbiddenFacts: [{ id: 'stale-channel', anyOf: ['SMS briefings'], obsolete: true }], - expectedEventIds: ['e-new'], - expectedActorIds: ['user'], - } - - const strong = scoreMemoryBenchmarkArtifact(testCase, { - answer: 'The user currently prefers email briefings.', - citedEventIds: ['e-new'], - actorIds: ['user'], - }) - const weak = scoreMemoryBenchmarkArtifact(testCase, { - answer: 'The user prefers SMS briefings.', - citedEventIds: ['e-old'], - actorIds: ['user'], - }) - - expect(strong.score).toBeGreaterThanOrEqual(0.7) - expect(strong.passed).toBe(true) - expect(weak.score).toBeLessThanOrEqual(0.3) - expect(weak.passed).toBe(false) - expect(strong.score - weak.score).toBeGreaterThanOrEqual(0.4) - expect(weak.dimensions.memory_stale_safe).toBe(0) - expect(strong.applicableDimensions).toEqual( - expect.arrayContaining([ - 'memory_fact_recall', - 'memory_event_recall', - 'memory_actor_recall', - 'memory_stale_safe', - ]), - ) - - const recallOnly = scoreMemoryBenchmarkArtifact( - { ...testCase, forbiddenFacts: undefined, expectedActorIds: undefined }, - { answer: 'The user currently prefers email briefings.', citedEventIds: ['e-new'] }, - ) - expect(recallOnly.dimensions).not.toHaveProperty('memory_stale_safe') - expect(recallOnly.dimensions).not.toHaveProperty('memory_actor_recall') - expect(recallOnly.applicableDimensions).not.toContain('memory_stale_safe') - }) - - it('runs one persisted benchmark cell for every declared memory benchmark family', async () => { - const storage = inMemoryCampaignStorage() - const cases = buildIndustryMemoryBenchmarkSmokeCases() - const result = await runKnowledgeBenchmarkSuite({ - cases, - runDir: '/runs/memory-benchmark-family-smoke', - storage, - respondRef: 'industry-memory-smoke:v1', - respond: ({ case: testCase }) => { - expect(isKnowledgeMemoryBenchmarkCase(testCase)).toBe(true) - if (!isKnowledgeMemoryBenchmarkCase(testCase)) { - throw new Error(`expected memory case, got ${testCase.taskKind}`) - } - return respondToIndustryMemoryBenchmarkSmokeCase({ - case: testCase, - }) - }, - }) - - expect(cases).toHaveLength(INDUSTRY_MEMORY_BENCHMARKS.length) - expect(result.report.totalCases).toBe(INDUSTRY_MEMORY_BENCHMARKS.length) - expect(result.report.totalCells).toBe(INDUSTRY_MEMORY_BENCHMARKS.length) - expect(result.report.cellsFailed).toBe(0) - expect(result.report.score.mean).toBe(1) - expect(result.report.byTaskKind['memory-ingest']?.n).toBe(1) - expect(result.report.byTaskKind['memory-recall']?.n).toBe(1) - expect(result.report.byTaskKind['memory-temporal']?.n).toBe(1) - expect(result.report.byTaskKind['memory-update']?.n).toBe(1) - expect(result.report.byTaskKind['memory-forgetting']?.n).toBe(1) - expect(result.report.byTaskKind['memory-reasoning']?.n).toBe(1) - expect(result.report.byTaskKind['memory-summarization']?.n).toBe(1) - expect(result.report.byTaskKind['memory-recommendation']?.n).toBe(1) - expect(result.report.byTaskKind['memory-multiparty']?.n).toBe(1) - expect(result.report.totalCostUsd).toBe(0) - for (const benchmark of INDUSTRY_MEMORY_BENCHMARKS) { - expect(result.report.byFamily[benchmark.family]?.n).toBeGreaterThanOrEqual(1) - } - expect(storage.read(result.reportJsonPath)).toContain('"totalCases": 9') - expect(storage.read(result.reportMarkdownPath)).toContain('memory_stale_safe') - }) -}) diff --git a/tests/benchmarks/cost.test.ts b/tests/benchmarks/cost.test.ts deleted file mode 100644 index 296ab6c..0000000 --- a/tests/benchmarks/cost.test.ts +++ /dev/null @@ -1,288 +0,0 @@ -import { createRunCostLedger, inMemoryCampaignStorage } from '@tangle-network/agent-eval/campaign' -import { describe, expect, it } from 'vitest' -import { - buildFirstPartyMemoryLifecycleBenchmarkCases, - createInMemoryBenchmarkAdapter, -} from '../../src/benchmarks/index' -import { runMemoryAdapterBenchmark } from '../support/benchmarks' - -describe('memory adapter benchmark cost and resume', () => { - it('counts billable adapter creation in the shared benchmark budget', async () => { - const storage = inMemoryCampaignStorage() - let creates = 0 - const result = await runMemoryAdapterBenchmark({ - cases: buildFirstPartyMemoryLifecycleBenchmarkCases().slice(0, 1), - runDir: '/runs/paid-adapter-creation', - storage, - costCeiling: 0.5, - candidates: [ - { - id: 'paid-creation', - ref: 'paid-creation:v1', - adapterId: 'paid-creation-provider', - adapterCreationCostUsd: 0.2, - createAdapter({ markExternalCall }) { - markExternalCall() - creates += 1 - return createInMemoryBenchmarkAdapter({ id: 'paid-creation-provider' }) - }, - }, - ], - }) - - const ledger = createRunCostLedger({ - storage, - runDir: '/runs/paid-adapter-creation', - costCeilingUsd: 0.5, - }) - expect(creates).toBe(1) - expect(result.rows[0]).toMatchObject({ candidateId: 'paid-creation', totalCostUsd: 0.2 }) - expect(result.totalCostUsd).toBe(0.2) - expect(ledger.list()).toMatchObject([ - { - actor: 'agent-knowledge:memory-adapter:paid-creation', - costUsd: 0.2, - tags: { - candidateId: 'paid-creation', - memoryAdapterCreation: 'execute', - runDir: '/runs/paid-adapter-creation/paid-creation', - }, - }, - ]) - }) - - it('does not recreate or recharge a billable adapter when every cell resumes', async () => { - const storage = inMemoryCampaignStorage() - let creates = 0 - const run = () => - runMemoryAdapterBenchmark({ - cases: buildFirstPartyMemoryLifecycleBenchmarkCases().slice(0, 1), - runDir: '/runs/resumed-paid-adapter-creation', - storage, - costCeiling: 0.5, - candidates: [ - { - id: 'paid-resume', - ref: 'paid-resume:v1', - adapterId: 'paid-resume-provider', - adapterCreationCostUsd: 0.2, - createAdapter({ markExternalCall }) { - markExternalCall() - creates += 1 - return createInMemoryBenchmarkAdapter({ id: 'paid-resume-provider' }) - }, - }, - ], - }) - - const initial = await run() - const resumed = await run() - - expect(creates).toBe(1) - expect(initial).toMatchObject({ totalCostUsd: 0.2 }) - expect(resumed).toMatchObject({ totalCostUsd: 0.2 }) - expect(resumed.rows[0]).toMatchObject({ adapterId: 'paid-resume-provider' }) - expect(resumed.rows[0]?.report.cellsCached).toBe(1) - }) - - it('closes a mismatched lazy adapter before any benchmark case writes', async () => { - const storage = inMemoryCampaignStorage() - let closes = 0 - let writes = 0 - const adapter = createInMemoryBenchmarkAdapter({ id: 'actual-provider' }) - const write = adapter.write.bind(adapter) - adapter.write = async (input) => { - writes += 1 - return write(input) - } - adapter.close = async () => { - closes += 1 - } - - await expect( - runMemoryAdapterBenchmark({ - cases: buildFirstPartyMemoryLifecycleBenchmarkCases().slice(0, 1), - runDir: '/runs/mismatched-lazy-adapter', - storage, - candidates: [ - { - id: 'candidate', - ref: 'candidate:v1', - adapterId: 'expected-provider', - createAdapter: () => adapter, - }, - ], - }), - ).rejects.toThrow("returned id 'actual-provider', expected 'expected-provider'") - expect({ closes, writes }).toEqual({ closes: 1, writes: 0 }) - expect( - storage.read('/runs/mismatched-lazy-adapter/memory-adapter-attempts.jsonl'), - ).toBeUndefined() - }) - - it('aborts execute adapter creation at the configured timeout', async () => { - const storage = inMemoryCampaignStorage() - let aborted = false - const startedAt = Date.now() - - await expect( - runMemoryAdapterBenchmark({ - cases: buildFirstPartyMemoryLifecycleBenchmarkCases().slice(0, 1), - runDir: '/runs/timed-out-adapter-creation', - storage, - cleanupTimeoutMs: 10, - candidates: [ - { - id: 'hung-factory', - ref: 'hung-factory:v1', - createAdapter: ({ signal }) => - new Promise((_resolve, reject) => { - signal.addEventListener( - 'abort', - () => { - aborted = true - reject(signal.reason) - }, - { once: true }, - ) - }), - }, - ], - }), - ).rejects.toThrow('benchmark execute adapter creation did not finish within 10ms') - expect(aborted).toBe(true) - expect(Date.now() - startedAt).toBeLessThan(500) - expect( - storage.read('/runs/timed-out-adapter-creation/memory-adapter-attempts.jsonl'), - ).toBeUndefined() - }) - - it('does not charge a local adapter factory failure', async () => { - const storage = inMemoryCampaignStorage() - await expect( - runMemoryAdapterBenchmark({ - cases: buildFirstPartyMemoryLifecycleBenchmarkCases().slice(0, 1), - runDir: '/runs/local-adapter-creation-failure', - storage, - costCeiling: 0.5, - candidates: [ - { - id: 'local-failure', - ref: 'local-failure:v1', - adapterCreationCostUsd: 0.2, - createAdapter() { - throw new Error('invalid local configuration') - }, - }, - ], - }), - ).rejects.toThrow('invalid local configuration') - - const ledger = createRunCostLedger({ - storage, - runDir: '/runs/local-adapter-creation-failure', - costCeilingUsd: 0.5, - }) - expect(ledger.summary()).toMatchObject({ totalCostUsd: 0, unresolvedCalls: 0 }) - }) - - it('enforces one cost ceiling across every compared adapter', async () => { - const result = await runMemoryAdapterBenchmark({ - cases: buildFirstPartyMemoryLifecycleBenchmarkCases().slice(0, 1), - runDir: '/runs/shared-adapter-cost-ceiling', - storage: inMemoryCampaignStorage(), - costCeiling: 0.75, - candidates: ['first', 'second'].map((id) => ({ - id, - ref: `${id}:v1`, - costUsdPerCase: 0.5, - createAdapter: () => createInMemoryBenchmarkAdapter({ id }), - })), - }) - - expect(result.totalCostUsd).toBe(0.5) - expect(result.rows).toHaveLength(2) - expect(result.rows.find((row) => row.candidateId === 'first')).toMatchObject({ - cellsFailed: 0, - totalCostUsd: 0.5, - }) - expect(result.rows.find((row) => row.candidateId === 'second')).toMatchObject({ - cellsFailed: 1, - totalCostUsd: 0, - }) - }) - - it('runs sequential paid adapter cases after exact external-cost receipts', async () => { - const result = await runMemoryAdapterBenchmark({ - cases: buildFirstPartyMemoryLifecycleBenchmarkCases().slice(0, 2), - runDir: '/runs/sequential-adapter-cost-accounting', - storage: inMemoryCampaignStorage(), - costCeiling: 0.2, - maxConcurrency: 1, - candidates: [ - { - id: 'sequential-paid', - ref: 'sequential-paid:v1', - costUsdPerCase: 0.1, - createAdapter: () => createInMemoryBenchmarkAdapter({ id: 'sequential-paid' }), - }, - ], - }) - - expect(result.rows[0]).toMatchObject({ cellsFailed: 0, totalCostUsd: 0.2 }) - expect(result.totalCostUsd).toBe(0.2) - }) - - it('refuses paid adapter calls when no dollar limit is configured', async () => { - let providerCalls = 0 - const adapter = createInMemoryBenchmarkAdapter({ id: 'paid-by-default' }) - const write = adapter.write.bind(adapter) - adapter.write = async (input) => { - providerCalls += 1 - return write(input) - } - - const result = await runMemoryAdapterBenchmark({ - cases: buildFirstPartyMemoryLifecycleBenchmarkCases().slice(0, 1), - runDir: '/runs/default-zero-cost-ceiling', - storage: inMemoryCampaignStorage(), - candidates: [ - { - id: 'paid-by-default', - ref: 'paid-by-default:v1', - costUsdPerCase: 0.01, - createAdapter: () => adapter, - }, - ], - }) - - expect(result.rows[0]).toMatchObject({ cellsFailed: 1, totalCostUsd: 0 }) - expect(providerCalls).toBe(0) - }) - - it('does not reuse resumable rows when adapter benchmark options change', async () => { - const storage = inMemoryCampaignStorage() - const run = (searchLimit: number) => - runMemoryAdapterBenchmark({ - cases: buildFirstPartyMemoryLifecycleBenchmarkCases().slice(0, 1), - runDir: '/runs/memory-adapter-cache-identity', - storage, - candidates: [ - { - id: 'in-memory', - ref: 'in-memory:v1', - createAdapter: () => createInMemoryBenchmarkAdapter(), - searchLimit, - }, - ], - }) - - const initial = await run(1) - const changed = await run(2) - const repeated = await run(2) - - expect(initial.rows[0]?.report.cellsCached).toBe(0) - expect(changed.rows[0]?.report.cellsCached).toBe(0) - expect(repeated.rows[0]?.report.cellsCached).toBe(1) - }) -}) diff --git a/tests/benchmarks/execution.test.ts b/tests/benchmarks/execution.test.ts deleted file mode 100644 index c1977a4..0000000 --- a/tests/benchmarks/execution.test.ts +++ /dev/null @@ -1,201 +0,0 @@ -import { inMemoryCampaignStorage } from '@tangle-network/agent-eval/campaign' -import { describe, expect, it } from 'vitest' -import { - buildFirstPartyMemoryLifecycleBenchmarkCases, - createInMemoryBenchmarkAdapter, - createNoopMemoryBenchmarkAdapter, - runMemoryAdapterBenchmark as runMemoryAdapterBenchmarkRaw, -} from '../../src/benchmarks/index' -import type { AgentMemoryAdapter, AgentMemoryScope } from '../../src/memory/types' -import { runMemoryAdapterBenchmark } from '../support/benchmarks' - -describe('memory adapter benchmark execution', () => { - it('ranks actual memory adapters on the first-party lifecycle benchmark', async () => { - const storage = inMemoryCampaignStorage() - const cases = buildFirstPartyMemoryLifecycleBenchmarkCases() - const result = await runMemoryAdapterBenchmark({ - cases, - runDir: '/runs/memory-adapter-ranking', - storage, - candidates: [ - { - id: 'no-memory', - ref: 'no-memory:v1', - createAdapter: () => createNoopMemoryBenchmarkAdapter(), - }, - { - id: 'in-memory', - ref: 'in-memory:v1', - createAdapter: () => createInMemoryBenchmarkAdapter(), - searchLimit: 1, - }, - ], - }) - - expect(cases).toHaveLength(12) - expect(result.rows).toHaveLength(2) - expect(result.rows[0]?.candidateId).toBe('in-memory') - expect(result.rows[0]?.scoreMean).toBeGreaterThan(0.9) - expect(result.rows[0]?.totalCells).toBe(12) - expect(result.rows[0]?.cellsFailed).toBe(0) - expect(result.rows[1]?.candidateId).toBe('no-memory') - expect(result.rows[1]?.scoreMean).toBeLessThan(0.3) - expect(storage.read(result.rankingJsonPath)).toContain('"candidateId": "in-memory"') - expect(storage.read(result.rankingMarkdownPath)).toContain('| 1 | in-memory |') - expect(storage.read(result.rows[0]!.reportJsonPath)).toContain('"memory_stale_safe"') - }) - - it('requires an explicit controller policy for custom benchmark storage', async () => { - await expect( - runMemoryAdapterBenchmarkRaw({ - cases: buildFirstPartyMemoryLifecycleBenchmarkCases().slice(0, 1), - runDir: '/runs/custom-benchmark-storage', - storage: inMemoryCampaignStorage(), - candidates: [ - { - id: 'in-memory', - ref: 'in-memory:v1', - createAdapter: () => createInMemoryBenchmarkAdapter(), - }, - ], - }), - ).rejects.toThrow("requires acquireRunLease or controllerMode='process-local'") - }) - - it('isolates and clears every concurrent adapter repetition', async () => { - const adapter = createInMemoryBenchmarkAdapter() - const clearedScopes: AgentMemoryScope[] = [] - const clear = adapter.clear! - adapter.clear = async (scope) => { - clearedScopes.push(scope ?? {}) - await clear(scope) - } - - const result = await runMemoryAdapterBenchmark({ - cases: buildFirstPartyMemoryLifecycleBenchmarkCases().slice(0, 1), - runDir: '/runs/isolated-memory-repetitions', - storage: inMemoryCampaignStorage(), - reps: 3, - maxConcurrency: 3, - candidates: [ - { - id: 'in-memory', - ref: 'in-memory:v1', - createAdapter: () => adapter, - }, - ], - }) - - expect(result.rows[0]?.totalCells).toBe(3) - expect(result.rows[0]?.cellsFailed).toBe(0) - expect(clearedScopes).toHaveLength(3) - expect(new Set(clearedScopes.map((scope) => scope.namespace)).size).toBe(3) - expect(new Set(clearedScopes.map((scope) => scope.tags?.benchmarkAttemptId)).size).toBe(3) - }) - - it('settles timed-out provider work before clearing and closing the adapter', async () => { - let closed = false - let writes = 0 - let writesAfterClose = 0 - let clears = 0 - let clearsAfterClose = 0 - let contextReads = 0 - const adapter: AgentMemoryAdapter = { - id: 'delayed-provider', - branchIsolation: { mode: 'scoped' }, - async search() { - return [] - }, - async getContext(query) { - contextReads += 1 - return { query, text: '', hits: [], sourceRecords: [] } - }, - async write(input) { - await new Promise((resolve) => setTimeout(resolve, 25)) - writes += 1 - if (closed) writesAfterClose += 1 - return { - accepted: true, - id: input.id ?? String(writes), - uri: `memory://delayed-provider/${writes}`, - kind: input.kind, - } - }, - async clear() { - await new Promise((resolve) => setTimeout(resolve, 1)) - clears += 1 - if (closed) clearsAfterClose += 1 - }, - async close() { - closed = true - }, - } - - const result = await runMemoryAdapterBenchmark({ - cases: buildFirstPartyMemoryLifecycleBenchmarkCases().slice(0, 1), - runDir: '/runs/timed-out-memory-provider', - storage: inMemoryCampaignStorage(), - dispatchTimeoutMs: 5, - candidates: [ - { - id: 'delayed-provider', - ref: 'delayed-provider:v1', - createAdapter: () => adapter, - }, - { - id: 'no-memory', - ref: 'no-memory:v1', - createAdapter: () => createNoopMemoryBenchmarkAdapter(), - }, - ], - }) - - expect(result.rows[0]).toMatchObject({ candidateId: 'no-memory', cellsFailed: 0 }) - expect(result.rows[1]).toMatchObject({ candidateId: 'delayed-provider', cellsFailed: 1 }) - expect({ closed, writes, writesAfterClose, clears, clearsAfterClose, contextReads }).toEqual({ - closed: true, - writes: 1, - writesAfterClose: 0, - clears: 1, - clearsAfterClose: 0, - contextReads: 0, - }) - await new Promise((resolve) => setTimeout(resolve, 50)) - expect({ writes, writesAfterClose, clears, clearsAfterClose }).toEqual({ - writes: 1, - writesAfterClose: 0, - clears: 1, - clearsAfterClose: 0, - }) - }) - - it('rejects every unsafe candidate id before creating any adapter', async () => { - let creates = 0 - await expect( - runMemoryAdapterBenchmark({ - cases: buildFirstPartyMemoryLifecycleBenchmarkCases().slice(0, 1), - runDir: '/runs/invalid-memory-candidate', - storage: inMemoryCampaignStorage(), - candidates: [ - { - id: 'valid', - ref: 'valid:v1', - createAdapter() { - creates += 1 - return createInMemoryBenchmarkAdapter() - }, - }, - { - id: '../invalid', - ref: 'invalid:v1', - createAdapter() { - creates += 1 - return createInMemoryBenchmarkAdapter() - }, - }, - ], - }), - ).rejects.toThrow('must be a safe directory segment') - expect(creates).toBe(0) - }) -}) diff --git a/tests/benchmarks/recovery.test.ts b/tests/benchmarks/recovery.test.ts deleted file mode 100644 index 99564b5..0000000 --- a/tests/benchmarks/recovery.test.ts +++ /dev/null @@ -1,420 +0,0 @@ -import { createRunCostLedger, inMemoryCampaignStorage } from '@tangle-network/agent-eval/campaign' -import { describe, expect, it } from 'vitest' -import { - buildFirstPartyMemoryLifecycleBenchmarkCases, - createInMemoryBenchmarkAdapter, -} from '../../src/benchmarks/index' -import type { AgentMemoryAdapter, AgentMemoryHit } from '../../src/memory/types' -import { runMemoryAdapterBenchmark } from '../support/benchmarks' - -describe('memory adapter benchmark recovery', () => { - it('recovers an unfinished provider scope before retrying a direct benchmark cell', async () => { - const storage = inMemoryCampaignStorage() - const providerRows = new Map() - const operations: string[] = [] - const purposes: string[] = [] - let firstExecution = true - const candidate = { - id: 'recoverable', - ref: 'recoverable:v1', - adapterId: 'recoverable-provider', - recoveryCostUsdPerAttempt: 0.1, - createAdapter({ purpose }: { purpose: 'execute' | 'recovery' }) { - purposes.push(purpose) - const label = purpose === 'recovery' ? 'recovery' : firstExecution ? 'first' : 'retry' - if (purpose === 'execute') firstExecution = false - const adapter: AgentMemoryAdapter = { - id: 'recoverable-provider', - branchIsolation: { mode: 'scoped' }, - async search(_query, options) { - return [...(providerRows.get(options?.scope?.namespace ?? '') ?? [])] - }, - async getContext(query, options) { - const hits = await adapter.search(query, options) - return { query, text: hits.map((hit) => hit.text).join('\n'), hits, sourceRecords: [] } - }, - async write(input) { - operations.push(`write:${label}`) - const namespace = input.scope?.namespace ?? '' - const hit: AgentMemoryHit = { - id: input.id ?? `${label}:memory`, - uri: `memory://recoverable/${label}`, - kind: input.kind, - text: input.text, - metadata: input.metadata, - } - providerRows.set(namespace, [...(providerRows.get(namespace) ?? []), hit]) - return { accepted: true, id: hit.id, uri: hit.uri, kind: hit.kind } - }, - async clear(scope) { - operations.push(`clear:${label}`) - if (label === 'first') throw new Error('provider cleanup unavailable') - providerRows.delete(scope?.namespace ?? '') - }, - async close() {}, - } - return adapter - }, - } - const run = () => - runMemoryAdapterBenchmark({ - cases: buildFirstPartyMemoryLifecycleBenchmarkCases().slice(0, 1), - runDir: '/runs/direct-benchmark-recovery', - storage, - candidates: [candidate], - costCeiling: 1, - }) - - await expect(run()).rejects.toThrow('memory benchmark attempt cleanup failed') - expect(providerRows.size).toBe(1) - - const result = await run() - expect(result.rows[0]).toMatchObject({ candidateId: 'recoverable', cellsFailed: 0 }) - expect(result.rows[0]?.totalCostUsd).toBe(0.1) - expect(result.totalCostUsd).toBe(0.1) - expect(purposes).toEqual(['execute', 'recovery', 'execute']) - expect(operations.indexOf('clear:recovery')).toBeLessThan(operations.indexOf('write:retry')) - expect(providerRows.size).toBe(0) - const events = storage - .read(result.attemptLogPath)! - .trim() - .split('\n') - .map((line) => JSON.parse(line) as { status: string; recovery: boolean }) - expect(events.map(({ status, recovery }) => ({ status, recovery }))).toEqual([ - { status: 'started', recovery: false }, - { status: 'cleaned', recovery: true }, - { status: 'started', recovery: false }, - { status: 'cleaned', recovery: false }, - ]) - }) - - it('includes paid cleanup for a retired candidate in the benchmark total', async () => { - const storage = inMemoryCampaignStorage() - const runDir = '/runs/retired-candidate-recovery-cost' - storage.write( - `${runDir}/memory-adapter-attempts.jsonl`, - `${JSON.stringify({ - status: 'started', - attemptId: 'retired-attempt', - candidateId: 'retired', - candidateRef: 'retired:v1', - adapterId: 'retired-provider', - caseId: 'old-case', - cellId: 'old-cell', - scope: { namespace: 'retired-scope' }, - adapterCreationCostUsd: 0.05, - costUsdPerCase: 0, - recoveryCostUsdPerAttempt: 0.1, - recordedAt: '2026-01-01T00:00:00.000Z', - recovery: false, - })}\n`, - ) - let retiredClears = 0 - const result = await runMemoryAdapterBenchmark({ - cases: buildFirstPartyMemoryLifecycleBenchmarkCases().slice(0, 1), - runDir, - storage, - costCeiling: 1, - candidates: [ - { - id: 'active', - ref: 'active:v1', - adapterId: 'active-provider', - createAdapter: () => createInMemoryBenchmarkAdapter({ id: 'active-provider' }), - }, - ], - recoveryCandidates: [ - { - id: 'retired', - ref: 'retired:v1', - adapterId: 'retired-provider', - adapterCreationCostUsd: 0.05, - recoveryCostUsdPerAttempt: 0.1, - createAdapter({ markExternalCall }) { - markExternalCall() - const adapter = createInMemoryBenchmarkAdapter({ id: 'retired-provider' }) - adapter.clear = async () => { - retiredClears += 1 - } - return adapter - }, - }, - ], - }) - - expect(retiredClears).toBe(1) - expect(result.rows[0]).toMatchObject({ candidateId: 'active', totalCostUsd: 0 }) - expect(result).toMatchObject({ totalCostUsd: 0.15, unrankedRecoveryCostUsd: 0.15 }) - expect(storage.read(result.rankingJsonPath)).toContain('"unrankedRecoveryCostUsd": 0.15') - }) - - it('refuses benchmark recovery when candidate cost settings changed', async () => { - const storage = inMemoryCampaignStorage() - const runDir = '/runs/benchmark-changed-recovery-costs' - storage.write( - `${runDir}/memory-adapter-attempts.jsonl`, - `${JSON.stringify({ - status: 'started', - attemptId: 'unfinished-attempt', - candidateId: 'memory', - candidateRef: 'memory:v1', - adapterId: 'memory-provider', - caseId: 'old-case', - cellId: 'old-cell', - scope: { namespace: 'unfinished-scope' }, - adapterCreationCostUsd: 0, - costUsdPerCase: 0, - recoveryCostUsdPerAttempt: 0, - recordedAt: '2026-01-01T00:00:00.000Z', - recovery: false, - })}\n`, - ) - let adapterCreates = 0 - - await expect( - runMemoryAdapterBenchmark({ - cases: buildFirstPartyMemoryLifecycleBenchmarkCases().slice(0, 1), - runDir, - storage, - costCeiling: 1, - candidates: [ - { - id: 'memory', - ref: 'memory:v1', - adapterId: 'memory-provider', - costUsdPerCase: 0.1, - createAdapter() { - adapterCreates += 1 - return createInMemoryBenchmarkAdapter({ id: 'memory-provider' }) - }, - }, - ], - }), - ).rejects.toThrow('candidate cost settings changed') - - expect(adapterCreates).toBe(0) - expect( - storage.read(`${runDir}/memory-adapter-attempts.jsonl`)?.trim().split('\n'), - ).toHaveLength(1) - }) - - it('bounds repeated direct benchmark recovery across process restarts', async () => { - const storage = inMemoryCampaignStorage() - const runDir = '/runs/benchmark-recovery-retry-limit' - storage.write( - `${runDir}/memory-adapter-attempts.jsonl`, - `${JSON.stringify({ - status: 'started', - attemptId: 'unfinished-attempt', - candidateId: 'memory', - candidateRef: 'memory:v1', - adapterId: 'memory-provider', - caseId: 'old-case', - cellId: 'old-cell', - scope: { namespace: 'unfinished-scope' }, - adapterCreationCostUsd: 0, - costUsdPerCase: 0, - recoveryCostUsdPerAttempt: 0, - recordedAt: '2026-01-01T00:00:00.000Z', - recovery: false, - })}\n`, - ) - let recoveryCreates = 0 - const run = () => - runMemoryAdapterBenchmark({ - cases: buildFirstPartyMemoryLifecycleBenchmarkCases().slice(0, 1), - runDir, - storage, - maxRecoveryRetriesPerAttempt: 2, - candidates: [ - { - id: 'memory', - ref: 'memory:v1', - adapterId: 'memory-provider', - createAdapter({ purpose }) { - if (purpose === 'recovery') recoveryCreates += 1 - throw new Error('provider recovery unavailable') - }, - }, - ], - }) - - await expect(run()).rejects.toThrow('provider recovery unavailable') - await expect(run()).rejects.toThrow('provider recovery unavailable') - await expect(run()).rejects.toThrow('exhausted 2 recovery attempts') - expect(recoveryCreates).toBe(2) - expect( - storage.read(`${runDir}/memory-adapter-recovery-attempts.jsonl`)?.trim().split('\n'), - ).toHaveLength(2) - }) - - it('closes a recovery adapter that arrives after its factory timeout', async () => { - const storage = inMemoryCampaignStorage() - const runDir = '/runs/late-benchmark-recovery-adapter' - storage.write( - `${runDir}/memory-adapter-attempts.jsonl`, - `${JSON.stringify({ - status: 'started', - attemptId: 'unfinished-attempt', - candidateId: 'memory', - candidateRef: 'memory:v1', - adapterId: 'memory-provider', - caseId: 'old-case', - cellId: 'old-cell', - scope: { namespace: 'unfinished-scope' }, - adapterCreationCostUsd: 0, - costUsdPerCase: 0, - recoveryCostUsdPerAttempt: 0, - recordedAt: '2026-01-01T00:00:00.000Z', - recovery: false, - })}\n`, - ) - let resolveCreation!: (adapter: AgentMemoryAdapter) => void - const creation = new Promise((resolve) => { - resolveCreation = resolve - }) - let reportClosed!: () => void - const closed = new Promise((resolve) => { - reportClosed = resolve - }) - let closeCalls = 0 - const lateAdapter = createInMemoryBenchmarkAdapter({ id: 'memory-provider' }) - lateAdapter.close = async () => { - closeCalls += 1 - reportClosed() - } - - await expect( - runMemoryAdapterBenchmark({ - cases: buildFirstPartyMemoryLifecycleBenchmarkCases().slice(0, 1), - runDir, - storage, - cleanupTimeoutMs: 10, - candidates: [ - { - id: 'memory', - ref: 'memory:v1', - adapterId: 'memory-provider', - createAdapter: ({ purpose }) => - purpose === 'recovery' ? creation : createInMemoryBenchmarkAdapter(), - }, - ], - }), - ).rejects.toThrow('benchmark recovery adapter creation did not finish within 10ms') - - resolveCreation(lateAdapter) - await closed - expect(closeCalls).toBe(1) - }) - - it('reconciles a crash after direct recovery but before its cost receipt', async () => { - const storage = inMemoryCampaignStorage() - const append = storage.append!.bind(storage) - let failRecoveryReceipt = false - storage.append = (path, value, expectedBytes) => { - if ( - failRecoveryReceipt && - path.endsWith('/cost-ledger.jsonl') && - value.includes('"status":"settled"') && - value.includes('memory-adapter-recovery') - ) { - failRecoveryReceipt = false - throw new Error('simulated process exit before benchmark recovery receipt') - } - return append(path, value, expectedBytes) - } - - let firstExecution = true - let recoveryClears = 0 - const candidate = { - id: 'receipt-crash', - ref: 'receipt-crash:v1', - adapterId: 'receipt-crash-provider', - costUsdPerCase: 0.1, - recoveryCostUsdPerAttempt: 0.1, - createAdapter({ purpose }: { purpose: 'execute' | 'recovery' }) { - const adapter = createInMemoryBenchmarkAdapter({ id: 'receipt-crash-provider' }) - const clear = adapter.clear! - const failThisExecution = purpose === 'execute' && firstExecution - if (purpose === 'execute') firstExecution = false - adapter.clear = async (scope) => { - if (purpose === 'recovery') recoveryClears += 1 - if (failThisExecution) throw new Error('leave direct benchmark state active') - await clear(scope) - } - return adapter - }, - } - const run = () => - runMemoryAdapterBenchmark({ - cases: buildFirstPartyMemoryLifecycleBenchmarkCases().slice(0, 1), - runDir: '/runs/direct-recovery-receipt-crash', - storage, - costCeiling: 1, - candidates: [candidate], - }) - - await expect(run()).rejects.toThrow('memory benchmark attempt cleanup failed') - failRecoveryReceipt = true - await expect(run()).rejects.toThrow('failed to persist') - expect( - storage - .read('/runs/direct-recovery-receipt-crash/memory-adapter-attempts.jsonl') - ?.trim() - .split('\n'), - ).toHaveLength(2) - - const result = await run() - const costLedger = createRunCostLedger({ - storage, - runDir: '/runs/direct-recovery-receipt-crash', - costCeilingUsd: 1, - }) - - expect(result.rows[0]).toMatchObject({ - candidateId: 'receipt-crash', - cellsFailed: 0, - }) - expect(result.rows[0]?.totalCostUsd).toBeCloseTo(0.3) - expect(result.totalCostUsd).toBeCloseTo(0.3) - expect(recoveryClears).toBe(1) - expect(costLedger.summary()).toMatchObject({ unresolvedCalls: 0, accountingComplete: true }) - expect(costLedger.summary().totalCostUsd).toBeCloseTo(0.3) - }) - - it('bounds direct benchmark cleanup and preserves its recovery record', async () => { - const storage = inMemoryCampaignStorage() - const adapter = createInMemoryBenchmarkAdapter({ id: 'hung-cleanup' }) - let closeCalls = 0 - adapter.clear = () => new Promise(() => {}) - adapter.close = async () => { - closeCalls += 1 - } - const startedAt = Date.now() - - await expect( - runMemoryAdapterBenchmark({ - cases: buildFirstPartyMemoryLifecycleBenchmarkCases().slice(0, 1), - runDir: '/runs/direct-benchmark-cleanup-timeout', - storage, - cleanupTimeoutMs: 10, - candidates: [ - { - id: 'hung-cleanup', - ref: 'hung-cleanup:v1', - createAdapter: () => adapter, - }, - ], - }), - ).rejects.toThrow('memory adapter benchmark cleanup failed') - - expect(Date.now() - startedAt).toBeLessThan(500) - expect( - storage - .read('/runs/direct-benchmark-cleanup-timeout/memory-adapter-attempts.jsonl') - ?.trim() - .split('\n'), - ).toHaveLength(1) - expect(closeCalls).toBe(0) - }) -}) diff --git a/tests/memory/experiment-cost.test.ts b/tests/memory/experiment-cost.test.ts deleted file mode 100644 index f1c9e81..0000000 --- a/tests/memory/experiment-cost.test.ts +++ /dev/null @@ -1,693 +0,0 @@ -import { createRunCostLedger, inMemoryCampaignStorage } from '@tangle-network/agent-eval/campaign' -import { describe, expect, it } from 'vitest' -import { - type AgentMemoryAdapter, - agentMemorySequenceJudge, - buildAgentMemorySequenceScenarios, - buildAgentMemorySequencesFromBenchmarkCases, - runAgentMemoryExperiment as runAgentMemoryExperimentRaw, -} from '../../src/memory/index' -import { createScopedTestAdapter, runAgentMemoryExperiment } from '../support/memory' - -describe('agent memory experiment cost and resume', () => { - it('versions memory scoring for resumable cache safety', () => { - expect(agentMemorySequenceJudge().judgeVersion).toBe('agent-knowledge:memory-sequence:v2') - }) - - it('requires an explicit controller policy for custom storage', async () => { - await expect( - runAgentMemoryExperimentRaw({ - experimentId: 'custom-storage-controller', - sequences: [ - { - id: 'history', - family: 'first-party', - steps: [{ id: 'probe', probes: [{ id: 'probe', query: 'x', referenceAnswer: 'x' }] }], - }, - ], - candidates: [ - { - id: 'memory', - ref: 'memory:v1', - createAdapter: () => createScopedTestAdapter('memory'), - }, - ], - runDir: '/runs/custom-storage-controller', - storage: inMemoryCampaignStorage(), - }), - ).rejects.toThrow("requires acquireRunLease or controllerMode='process-local'") - }) - - it('bounds provider cleanup and leaves the attempt available for recovery', async () => { - const storage = inMemoryCampaignStorage() - let clearCalls = 0 - const adapter = createScopedTestAdapter('bounded-cleanup') - const clear = adapter.clear! - adapter.clear = async (scope) => { - clearCalls += 1 - if (clearCalls === 1) return new Promise(() => {}) - await clear(scope) - } - const startedAt = Date.now() - - await expect( - runAgentMemoryExperiment({ - experimentId: 'bounded-cleanup', - sequences: [ - { - id: 'history', - family: 'first-party', - steps: [ - { - id: 'remember', - scope: { agentId: 'worker' }, - writes: [{ kind: 'fact', text: 'bounded fact' }], - probes: [{ id: 'probe', query: 'bounded', referenceAnswer: 'bounded fact' }], - }, - ], - }, - ], - candidates: [{ id: 'memory', ref: 'memory:v1', createAdapter: () => adapter }], - runDir: '/runs/bounded-cleanup', - storage, - cleanupTimeoutMs: 10, - }), - ).rejects.toThrow('memory experiment cleanup failed after dispatch') - - expect(Date.now() - startedAt).toBeLessThan(500) - expect( - storage.read('/runs/bounded-cleanup/memory-attempts.jsonl')?.trim().split('\n'), - ).toHaveLength(1) - }) - - it('does not charge provider cost when side-effect-free adapter construction fails', async () => { - const storage = inMemoryCampaignStorage() - const costLedger = createRunCostLedger({ - storage, - runDir: '/runs/create-adapter-cost', - costCeilingUsd: 1, - }) - - await expect( - runAgentMemoryExperiment({ - experimentId: 'create-adapter-cost', - sequences: [ - { - id: 'history', - family: 'first-party', - steps: [{ id: 'probe', probes: [{ id: 'probe', query: 'x', referenceAnswer: 'x' }] }], - }, - ], - candidates: [ - { - id: 'paid-memory', - ref: 'paid-memory:v1', - externalCostUsdPerSequence: 0.25, - externalCostAccounting: 'exact', - createAdapter() { - throw new Error('provider setup rejected') - }, - }, - ], - runDir: '/runs/create-adapter-cost', - storage, - costLedger, - }), - ).rejects.toThrow('memory experiment cleanup failed after dispatch') - - expect(costLedger.summary().totalCostUsd).toBe(0) - }) - - it('charges a factory failure after dedicated provider provisioning starts', async () => { - const storage = inMemoryCampaignStorage() - const costLedger = createRunCostLedger({ - storage, - runDir: '/runs/create-adapter-provider-cost', - costCeilingUsd: 1, - }) - - await expect( - runAgentMemoryExperiment({ - experimentId: 'create-adapter-provider-cost', - sequences: [ - { - id: 'history', - family: 'first-party', - steps: [{ id: 'probe', probes: [{ id: 'probe', query: 'x', referenceAnswer: 'x' }] }], - }, - ], - candidates: [ - { - id: 'paid-memory', - ref: 'paid-memory:v1', - externalCostUsdPerSequence: 0.25, - externalCostAccounting: 'exact', - createAdapter({ markExternalCall, recordExternalCost }) { - markExternalCall() - recordExternalCost(0.25) - throw new Error('provider provisioning failed') - }, - }, - ], - runDir: '/runs/create-adapter-provider-cost', - storage, - costLedger, - }), - ).rejects.toThrow('memory experiment cleanup failed after dispatch') - - expect(costLedger.summary()).toMatchObject({ totalCalls: 1, totalCostUsd: 0.25 }) - }) - - it('marks positive external spend incomplete when the provider emits no receipt', async () => { - const storage = inMemoryCampaignStorage() - const costLedger = createRunCostLedger({ - storage, - runDir: '/runs/missing-provider-receipt', - costCeilingUsd: 1, - }) - - const result = await runAgentMemoryExperiment({ - experimentId: 'missing-provider-receipt', - sequences: [ - { - id: 'history', - family: 'first-party', - steps: [ - { - id: 'remember', - scope: { agentId: 'worker' }, - writes: [{ kind: 'fact', text: 'durable fact' }], - probes: [ - { - id: 'recall', - query: 'durable', - scope: { agentId: 'worker' }, - referenceAnswer: 'durable fact', - }, - ], - }, - ], - }, - ], - candidates: [ - { - id: 'paid-memory', - ref: 'paid-memory:v1', - externalCostUsdPerSequence: 0.25, - externalCostAccounting: 'exact', - createAdapter({ maximumCostUsd, markExternalCall }) { - expect(maximumCostUsd).toBe(0.25) - markExternalCall() - return createScopedTestAdapter('missing-provider-receipt') - }, - }, - ], - runDir: '/runs/missing-provider-receipt', - storage, - costLedger, - }) - - expect(result.campaign.aggregates.cost).toMatchObject({ - accountingComplete: false, - totalCostUsd: 0, - }) - expect(costLedger.summary()).toMatchObject({ - accountingComplete: false, - totalCostUsd: 0, - }) - }) - - it('records provider cleanup before a paid-call receipt can be interrupted', async () => { - const storage = inMemoryCampaignStorage() - const append = storage.append!.bind(storage) - let interruptReceipt = true - storage.append = (path, value, expectedBytes) => { - if ( - interruptReceipt && - path.endsWith('/cost-ledger.jsonl') && - value.includes('"status":"settled"') && - value.includes('agent-knowledge:memory-experiment') - ) { - interruptReceipt = false - throw new Error('simulated process exit before execute receipt') - } - return append(path, value, expectedBytes) - } - const purposes: string[] = [] - const sequence = { - id: 'history', - family: 'first-party' as const, - steps: [ - { - id: 'remember', - scope: { agentId: 'worker' }, - writes: [{ kind: 'fact' as const, text: 'durable fact' }], - probes: [{ id: 'recall', query: 'durable', referenceAnswer: 'durable fact' }], - }, - ], - } - const run = () => - runAgentMemoryExperiment({ - experimentId: 'execute-receipt-crash', - sequences: [sequence], - candidates: [ - { - id: 'memory', - ref: 'memory:v1', - externalCostUsdPerSequence: 0.1, - externalCostAccounting: 'exact', - createAdapter({ purpose, recordExternalCost }) { - purposes.push(purpose) - recordExternalCost(0.1) - return createScopedTestAdapter(`memory:${purpose}`) - }, - }, - ], - runDir: '/runs/execute-receipt-crash', - storage, - costCeiling: 1, - }) - - await expect(run()).rejects.toThrow( - /paid calls for failed cell .* did not settle .* no complete failure receipt was produced/, - ) - expect( - storage.read('/runs/execute-receipt-crash/memory-attempts.jsonl')?.trim().split('\n'), - ).toHaveLength(2) - - const result = await run() - expect(purposes).toEqual(['execute', 'execute']) - expect(result.rows[0]).toMatchObject({ cellsFailed: 0, totalCostUsd: 0.2 }) - }) - - it('accounts for parallel candidates against one shared dollar limit', async () => { - let releaseFirstCalls: (() => void) | undefined - const firstCallsReady = new Promise((resolve) => { - releaseFirstCalls = resolve - }) - let activeCalls = 0 - let firstCalls = 0 - let maxActiveCalls = 0 - const createAdapter = (id: string): AgentMemoryAdapter => { - const adapter = createScopedTestAdapter(id) - const clear = adapter.clear! - let entered = false - adapter.clear = async (scope) => { - if (!entered) { - entered = true - firstCalls += 1 - activeCalls += 1 - maxActiveCalls = Math.max(maxActiveCalls, activeCalls) - if (firstCalls === 2) releaseFirstCalls?.() - await firstCallsReady - activeCalls -= 1 - } - await clear(scope) - } - return adapter - } - - const result = await runAgentMemoryExperiment({ - experimentId: 'parallel-shared-cost', - sequences: [ - { - id: 'history', - family: 'first-party', - steps: [ - { - id: 'remember', - scope: { agentId: 'worker' }, - writes: [{ kind: 'fact', text: 'parallel fact' }], - probes: [ - { - id: 'recall', - query: 'parallel', - scope: { agentId: 'worker' }, - referenceAnswer: 'parallel fact', - }, - ], - }, - ], - }, - ], - candidates: ['first', 'second'].map((id) => ({ - id, - ref: `${id}:v1`, - externalCostUsdPerSequence: 0.1, - externalCostAccounting: 'exact' as const, - createAdapter: ({ recordExternalCost }) => { - recordExternalCost(0.1) - return createAdapter(id) - }, - })), - runDir: '/runs/parallel-shared-cost', - storage: inMemoryCampaignStorage(), - costCeiling: 0.2, - maxConcurrency: 2, - }) - - expect(maxActiveCalls).toBe(2) - expect(result.rows.map((row) => row.totalCostUsd).sort()).toEqual([0.1, 0.1]) - expect(result.campaign.aggregates.cost.totalCostUsd).toBe(0.2) - }) - - it('runs sequential paid histories after exact external-cost receipts', async () => { - const storage = inMemoryCampaignStorage() - const costLedger = createRunCostLedger({ - storage, - runDir: '/runs/sequential-shared-cost', - costCeilingUsd: 0.2, - }) - const sequences = ['first', 'second'].map((id) => ({ - id, - family: 'first-party', - steps: [ - { - id: 'remember', - scope: { agentId: 'worker' }, - writes: [{ kind: 'fact' as const, text: `${id} fact` }], - probes: [ - { - id: 'recall', - query: id, - scope: { agentId: 'worker' }, - referenceAnswer: `${id} fact`, - }, - ], - }, - ], - })) - - const result = await runAgentMemoryExperiment({ - experimentId: 'sequential-shared-cost', - sequences, - candidates: [ - { - id: 'memory', - ref: 'memory:v1', - externalCostUsdPerSequence: 0.1, - externalCostAccounting: 'exact', - createAdapter: ({ branchId, recordExternalCost }) => { - recordExternalCost(0.1) - return createScopedTestAdapter(branchId) - }, - }, - ], - runDir: '/runs/sequential-shared-cost', - storage, - costLedger, - maxConcurrency: 1, - }) - - expect(result.rows[0]).toMatchObject({ cellsFailed: 0, totalCostUsd: 0.2 }) - expect(costLedger.summary()).toMatchObject({ - totalCalls: 2, - totalCostUsd: 0.2, - accountingComplete: true, - }) - }) - - it('reports the reserved cost of an interrupted prior cell attempt', async () => { - const storage = inMemoryCampaignStorage() - const runDir = '/runs/interrupted-cell-cost-reporting' - const sequence = { - id: 'history', - family: 'first-party' as const, - steps: [ - { - id: 'remember', - scope: { agentId: 'worker' }, - writes: [{ kind: 'fact' as const, text: 'resumed fact' }], - probes: [{ id: 'recall', query: 'resumed', referenceAnswer: 'resumed fact' }], - }, - ], - } - const candidate = { - id: 'memory', - ref: 'memory:v1', - externalCostUsdPerSequence: 0.1, - externalCostAccounting: 'exact' as const, - createAdapter: ({ recordExternalCost }) => { - recordExternalCost(0.1) - return createScopedTestAdapter('memory') - }, - } - const scenarioId = buildAgentMemorySequenceScenarios([sequence], [candidate])[0]!.id - const abandonedLedger = createRunCostLedger({ storage, runDir, costCeilingUsd: 1 }) - const controller = new AbortController() - let releaseProvider!: () => void - let reportStarted!: () => void - const started = new Promise((resolve) => { - reportStarted = resolve - }) - const abandoned = abandonedLedger.runPaidCall({ - callId: 'prior-cell-attempt', - channel: 'agent', - phase: 'memory.experiment', - actor: 'agent-knowledge:memory-experiment:memory', - model: 'memory', - signal: controller.signal, - tags: { - runDir, - scenarioId, - cellId: 'prior-cell', - rep: '0', - runAttemptId: 'prior-attempt', - }, - maximumCharge: { externallyEnforcedMaximumUsd: 0.1 }, - async execute() { - reportStarted() - return await new Promise((resolve) => { - releaseProvider = () => resolve('late result') - }) - }, - receipt: () => ({ - model: 'memory', - inputTokens: 0, - outputTokens: 0, - actualCostUsd: 0.05, - }), - }) - await started - controller.abort(new Error('simulated process exit')) - await abandoned - - const resumedLedger = createRunCostLedger({ storage, runDir, costCeilingUsd: 1 }) - const result = await runAgentMemoryExperiment({ - experimentId: 'interrupted-cell-cost-reporting', - sequences: [sequence], - candidates: [candidate], - runDir, - storage, - costLedger: resumedLedger, - }) - - expect(result.rows[0]).toMatchObject({ candidateId: 'memory', totalCostUsd: 0.2 }) - expect(resumedLedger.summary()).toMatchObject({ totalCalls: 2, totalCostUsd: 0.2 }) - releaseProvider() - await abandonedLedger.waitForIdle() - }) - - it('reconciles interrupted paid calls from every parallel memory branch before resuming', async () => { - const storage = inMemoryCampaignStorage() - const runDir = '/runs/parallel-interrupted-cost-recovery' - const abandonedLedger = createRunCostLedger({ storage, runDir, costCeilingUsd: 1 }) - const controllers = [new AbortController(), new AbortController()] - const releases: Array<() => void> = [] - let started = 0 - let reportStarted: (() => void) | undefined - const bothStarted = new Promise((resolve) => { - reportStarted = resolve - }) - const abandonedCalls = ['one', 'two'].map((branch, index) => - abandonedLedger.runPaidCall({ - callId: `abandoned-${branch}`, - channel: 'agent', - phase: 'memory.train', - actor: `agent-knowledge:memory-experiment:${branch}`, - model: `provider-${branch}`, - signal: controllers[index]!.signal, - maximumCharge: { externallyEnforcedMaximumUsd: 0.1 }, - async execute() { - started += 1 - if (started === 2) reportStarted?.() - return await new Promise((resolve) => { - releases[index] = () => resolve('late provider result') - }) - }, - receipt: () => ({ - model: `provider-${branch}`, - inputTokens: 0, - outputTokens: 0, - actualCostUsd: 0.05, - }), - }), - ) - - await bothStarted - for (const controller of controllers) { - controller.abort(new Error('simulated process exit')) - } - await Promise.all(abandonedCalls) - expect(abandonedLedger.listPending().map((call) => call.state)).toEqual(['late', 'late']) - - const resumedLedger = createRunCostLedger({ storage, runDir, costCeilingUsd: 1 }) - expect(resumedLedger.listPending().map((call) => call.state)).toEqual([ - 'interrupted', - 'interrupted', - ]) - - const result = await runAgentMemoryExperiment({ - experimentId: 'parallel-interrupted-cost-recovery', - sequences: [ - { - id: 'history', - family: 'first-party', - steps: [ - { - id: 'remember', - scope: { agentId: 'worker' }, - writes: [{ kind: 'fact', text: 'resumed fact' }], - probes: [ - { - id: 'recall', - query: 'resumed', - referenceAnswer: 'resumed fact', - }, - ], - }, - ], - }, - ], - candidates: [ - { - id: 'resumed', - ref: 'resumed:v1', - createAdapter: () => createScopedTestAdapter('resumed'), - }, - ], - runDir, - storage, - costLedger: resumedLedger, - }) - - expect(result.rows[0]).toMatchObject({ candidateId: 'resumed', cellsFailed: 0 }) - expect(resumedLedger.summary()).toMatchObject({ - totalCalls: 2, - unresolvedCalls: 0, - totalCostUsd: 0.2, - accountingComplete: true, - }) - expect(resumedLedger.list().map((receipt) => receipt.error)).toEqual([ - expect.stringContaining('charged the reserved maximum'), - expect.stringContaining('charged the reserved maximum'), - ]) - - for (const release of releases) release() - await abandonedLedger.waitForIdle() - }) - - it('does not reuse cells after the candidate implementation reference changes', async () => { - const storage = inMemoryCampaignStorage() - const sequence = { - id: 'candidate-ref', - family: 'first-party' as const, - steps: [ - { - id: 'remember', - scope: { agentId: 'worker' }, - writes: [{ kind: 'fact' as const, text: 'remember me' }], - probes: [ - { - id: 'recall', - query: 'remember', - requiredFacts: [{ id: 'fact', anyOf: ['remember me'] }], - }, - ], - }, - ], - } - let creates = 0 - const run = (ref: string, visible: boolean) => - runAgentMemoryExperiment({ - experimentId: 'candidate-ref-cache', - sequences: [sequence], - candidates: [ - { - id: 'memory', - ref, - createAdapter() { - creates += 1 - const adapter = createScopedTestAdapter(`memory:${ref}`) - if (visible) return adapter - return { - ...adapter, - async search() { - return [] - }, - } - }, - }, - ], - runDir: '/runs/candidate-ref-cache', - storage, - }) - - const first = await run('memory:v1', true) - const cached = await run('memory:v1', false) - const changed = await run('memory:v2', false) - - expect(first.rows[0]?.scoreMean).toBe(1) - expect(cached.campaign.cells[0]?.cached).toBe(true) - expect(cached.rows[0]?.scoreMean).toBe(1) - expect(changed.campaign.cells[0]?.cached).toBe(false) - expect(changed.rows[0]?.scoreMean).toBe(0) - expect(creates).toBe(2) - }) - - it('runs existing ordered memory benchmark cases without reshaping the dataset', async () => { - const sequences = buildAgentMemorySequencesFromBenchmarkCases([ - { - id: 'launch-update', - family: 'longmemeval', - taskKind: 'memory-temporal', - split: 'holdout', - events: [ - { id: 'old', actorId: 'pm', text: 'Launch is April 3.' }, - { id: 'current', actorId: 'pm', text: 'Launch moved to April 17.' }, - ], - prompt: 'When is launch?', - requiredFacts: [{ id: 'current-date', anyOf: ['April 17'] }], - forbiddenFacts: [{ id: 'old-date', anyOf: ['April 3'], obsolete: true }], - expectedEventIds: ['current'], - expectedActorIds: ['pm'], - }, - ]) - const result = await runAgentMemoryExperiment({ - experimentId: 'benchmark-conversion', - sequences, - candidates: [ - { - id: 'literal-memory', - ref: 'literal-memory:v1', - createAdapter: () => createScopedTestAdapter('literal-memory'), - }, - ], - runDir: '/runs/benchmark-conversion', - storage: inMemoryCampaignStorage(), - }) - - expect(sequences[0]?.steps.map((step) => step.id)).toEqual([ - 'event:old', - 'event:current', - 'probe', - ]) - expect(result.rows[0]).toMatchObject({ - scoreMean: 0.75, - totalSequences: 1, - totalProbes: 1, - cellsFailed: 0, - }) - expect(result.rows[0]?.dimensions).toMatchObject({ memory_stale_safe: 0 }) - }) -}) diff --git a/tests/memory/experiment-execution.test.ts b/tests/memory/experiment-execution.test.ts deleted file mode 100644 index dd084cd..0000000 --- a/tests/memory/experiment-execution.test.ts +++ /dev/null @@ -1,328 +0,0 @@ -import { inMemoryCampaignStorage } from '@tangle-network/agent-eval/campaign' -import { describe, expect, it } from 'vitest' -import type { AgentMemoryAdapter, AgentMemoryHit } from '../../src/memory/index' -import { createScopedTestAdapter, hitText, runAgentMemoryExperiment } from '../support/memory' - -describe('agent memory experiment execution', () => { - it('keeps each history ordered while comparing candidate branches in parallel', async () => { - const storage = inMemoryCampaignStorage() - const snapshots: string[] = [] - const stepOrder = new Map() - let active = 0 - let maxActive = 0 - let release: (() => void) | undefined - const twoActive = new Promise((resolve) => { - release = resolve - }) - const sequence = (id: string) => ({ - id, - family: 'first-party' as const, - split: 'holdout' as const, - steps: [ - { - id: 'research', - scope: { agentId: 'researcher', teamId: 'team-1' }, - writes: [ - { - id: `${id}-event`, - kind: 'fact' as const, - text: `${id} launch date is Friday`, - metadata: { eventId: `${id}-event`, actorId: 'researcher' }, - }, - ], - }, - { - id: 'delivery', - scope: { agentId: 'builder', teamId: 'team-1' }, - probes: [ - { - id: 'launch-date', - query: `${id} launch date`, - requiredFacts: [{ id: 'current', anyOf: [`${id} launch date is Friday`] }], - expectedEventIds: [`${id}-event`], - expectedActorIds: ['researcher'], - }, - ], - }, - ], - }) - - const result = await runAgentMemoryExperiment({ - experimentId: 'team-sharing-vs-private', - sequences: [sequence('alpha'), sequence('beta')], - candidates: [ - { - id: 'private', - ref: 'private:v1', - policy: { read: ['private'], write: 'private' }, - createAdapter: ({ branchId }) => createScopedTestAdapter(`private:${branchId}`), - }, - { - id: 'team', - ref: 'team:v1', - policy: { read: ['team'], write: 'team' }, - createAdapter: ({ branchId }) => createScopedTestAdapter(`team:${branchId}`), - }, - ], - runDir: '/runs/team-sharing-vs-private', - storage, - maxConcurrency: 4, - cleanupBranches: true, - executeStepRef: 'test-runtime/v1', - executeStep: async ({ memory, step }) => { - const order = stepOrder.get(memory.branchId) ?? [] - order.push(String(step.ordinal)) - stepOrder.set(memory.branchId, order) - if (step.ordinal === 0) { - active += 1 - maxActive = Math.max(maxActive, active) - if (active === 2) release?.() - await twoActive - active -= 1 - } - }, - onBranchSnapshot: ({ snapshot }) => { - snapshots.push(snapshot.digest) - }, - }) - - expect(maxActive).toBeGreaterThanOrEqual(2) - expect(result.rows[0]).toMatchObject({ - rank: 1, - candidateId: 'team', - scoreMean: 1, - passRate: 1, - totalSequences: 2, - totalCells: 2, - totalProbes: 2, - cellsFailed: 0, - }) - expect(result.rows[1]?.candidateId).toBe('private') - expect(result.rows[1]?.scoreMean).toBeLessThan(0.3) - expect(result.campaign.cells).toHaveLength(4) - expect(snapshots).toHaveLength(4) - expect([...stepOrder.values()].every((steps) => steps.join(',') === '0,1')).toBe(true) - expect(storage.read(result.rankingJsonPath)).toContain('"candidateId": "team"') - expect(storage.read(result.rankingMarkdownPath)).toContain('| 1 | team |') - }) - - it('uses distinct external branch ids for distinct run directories', async () => { - const branchIds: string[] = [] - const sequence = { - id: 'branch-id', - family: 'first-party' as const, - steps: [ - { - id: 'probe', - scope: { agentId: 'worker' }, - probes: [{ id: 'state', query: 'state', referenceAnswer: 'state' }], - }, - ], - } - const run = (runDir: string) => - runAgentMemoryExperiment({ - experimentId: 'same-experiment', - sequences: [sequence], - candidates: [ - { - id: 'memory', - ref: 'memory:v1', - createAdapter({ branchId }) { - branchIds.push(branchId) - return createScopedTestAdapter(branchId) - }, - }, - ], - runDir, - storage: inMemoryCampaignStorage(), - }) - - await run('/runs/branch-id-a') - await run('/runs/branch-id-b') - - expect(branchIds).toHaveLength(2) - expect(branchIds[0]).not.toBe(branchIds[1]) - }) - - it('uses a fresh external branch id for each distributed execution attempt', async () => { - const branchIds: string[] = [] - const run = (runDir: string) => - runAgentMemoryExperiment({ - experimentId: 'distributed-experiment', - experimentRunId: 'distributed-run-17', - sequences: [ - { - id: 'shared-history', - family: 'first-party', - steps: [ - { - id: 'probe', - scope: { agentId: 'worker' }, - probes: [{ id: 'state', query: 'state', referenceAnswer: 'state' }], - }, - ], - }, - ], - candidates: [ - { - id: 'memory', - ref: 'memory:v1', - createAdapter({ branchId }) { - branchIds.push(branchId) - return createScopedTestAdapter(branchId) - }, - }, - ], - runDir, - storage: inMemoryCampaignStorage(), - }) - - await run('/worker-a/run') - await run('/worker-b/run') - - expect(branchIds).toHaveLength(2) - expect(branchIds[0]).not.toBe(branchIds[1]) - }) - - it('waits for timed-out provider work to finish cleanup before returning', async () => { - let releaseWrite!: () => void - const writeMayFinish = new Promise((resolve) => { - releaseWrite = resolve - }) - let reportWriteStarted!: () => void - const writeStarted = new Promise((resolve) => { - reportWriteStarted = resolve - }) - const rows: AgentMemoryHit[] = [] - let clears = 0 - const adapter: AgentMemoryAdapter = { - id: 'slow-provider', - branchIsolation: { mode: 'scoped' }, - async search() { - return [...rows] - }, - async getContext(query) { - return { query, text: rows.map(hitText).join('\n'), hits: [...rows], sourceRecords: [] } - }, - async write(input) { - reportWriteStarted() - await writeMayFinish - const hit = { - id: 'late-write', - uri: 'memory://slow-provider/late-write', - kind: input.kind, - text: input.text, - } - rows.push(hit) - return { accepted: true, id: hit.id, uri: hit.uri, kind: hit.kind } - }, - async clear() { - clears += 1 - rows.length = 0 - }, - } - const run = runAgentMemoryExperiment({ - experimentId: 'timeout-cleanup', - sequences: [ - { - id: 'slow-history', - family: 'first-party', - steps: [ - { - id: 'write', - scope: { agentId: 'worker' }, - writes: [{ kind: 'fact', text: 'late fact' }], - probes: [{ id: 'fact', query: 'fact', referenceAnswer: 'late fact' }], - }, - ], - }, - ], - candidates: [{ id: 'slow', ref: 'slow:v1', createAdapter: () => adapter }], - runDir: '/runs/timeout-cleanup', - storage: inMemoryCampaignStorage(), - dispatchTimeoutMs: 5, - }) - await writeStarted - let settled = false - void run.then( - () => { - settled = true - }, - () => { - settled = true - }, - ) - await new Promise((resolve) => setTimeout(resolve, 15)) - - expect(settled).toBe(false) - releaseWrite() - const result = await run - - expect(result.rows[0]).toMatchObject({ cellsFailed: 1 }) - expect(result.campaign.cells[0]?.error).toContain('dispatch exceeded 5ms') - expect(clears).toBe(1) - expect(rows).toEqual([]) - }) - - it('fails when cleanup after a timed-out provider write fails', async () => { - let releaseWrite!: () => void - const writeMayFinish = new Promise((resolve) => { - releaseWrite = resolve - }) - let reportWriteStarted!: () => void - const writeStarted = new Promise((resolve) => { - reportWriteStarted = resolve - }) - let clears = 0 - const adapter: AgentMemoryAdapter = { - id: 'cleanup-failure', - branchIsolation: { mode: 'scoped' }, - async search() { - return [] - }, - async getContext(query) { - return { query, text: '', hits: [], sourceRecords: [] } - }, - async write(input) { - reportWriteStarted() - await writeMayFinish - return { - accepted: true, - id: 'late-write', - uri: 'memory://cleanup-failure/late-write', - kind: input.kind, - } - }, - async clear() { - clears += 1 - throw new Error('provider cleanup unavailable') - }, - } - const run = runAgentMemoryExperiment({ - experimentId: 'timeout-cleanup-failure', - sequences: [ - { - id: 'slow-history', - family: 'first-party', - steps: [ - { - id: 'write', - scope: { agentId: 'worker' }, - writes: [{ kind: 'fact', text: 'late fact' }], - probes: [{ id: 'fact', query: 'fact', referenceAnswer: 'late fact' }], - }, - ], - }, - ], - candidates: [{ id: 'slow', ref: 'slow:v1', createAdapter: () => adapter }], - runDir: '/runs/timeout-cleanup-failure', - storage: inMemoryCampaignStorage(), - dispatchTimeoutMs: 5, - }) - await writeStarted - await new Promise((resolve) => setTimeout(resolve, 10)) - releaseWrite() - - await expect(run).rejects.toThrow('memory experiment cleanup failed after dispatch') - }) -}) diff --git a/tests/memory/experiment-learning-validation.test.ts b/tests/memory/experiment-learning-validation.test.ts deleted file mode 100644 index a382c22..0000000 --- a/tests/memory/experiment-learning-validation.test.ts +++ /dev/null @@ -1,55 +0,0 @@ -import { describe, expect, it } from 'vitest' -import { controlProbe, retentionProbe, runInvalidSequence } from '../support/memory-learning' - -describe('memory learning measurement validation', () => { - it('requires a retention key to repeat across distinct steps', async () => { - await expect(runInvalidSequence([{ id: 'only', probes: [retentionProbe()] }])).rejects.toThrow( - 'must appear in at least two distinct steps', - ) - }) - - it('requires repeated retention probes to be the exact same measurement', async () => { - await expect( - runInvalidSequence([ - { id: 'first', probes: [retentionProbe()] }, - { - id: 'second', - probes: [{ ...retentionProbe(), query: 'A different query' }], - }, - ]), - ).rejects.toThrow('must repeat the exact same measurement') - }) - - it('allows at most one observation of a retention target per step', async () => { - await expect( - runInvalidSequence([ - { - id: 'first', - probes: [retentionProbe(), { ...retentionProbe(), id: 'duplicate' }], - }, - { id: 'second', probes: [retentionProbe()] }, - ]), - ).rejects.toThrow('may appear only once per step') - }) - - it('requires transfer probes to be explicit, later, and distinct from retention', async () => { - await expect( - runInvalidSequence([ - { - id: 'first', - probes: [{ ...controlProbe(), transferKey: 'transfer' }], - }, - ]), - ).rejects.toThrow('must run after the first step') - - await expect( - runInvalidSequence([ - { id: 'first', probes: [retentionProbe()] }, - { - id: 'second', - probes: [{ ...retentionProbe(), transferKey: 'transfer' }], - }, - ]), - ).rejects.toThrow('cannot be both transfer and retention') - }) -}) diff --git a/tests/memory/experiment-learning.test.ts b/tests/memory/experiment-learning.test.ts deleted file mode 100644 index 85fde99..0000000 --- a/tests/memory/experiment-learning.test.ts +++ /dev/null @@ -1,506 +0,0 @@ -import { - canonicalDigest, - createRunCostLedger, - inMemoryCampaignStorage, -} from '@tangle-network/agent-eval/campaign' -import { beforeAll, describe, expect, it } from 'vitest' -import { - compareAgentMemoryLearning, - type RunAgentMemoryLearningExperimentResult, - runAgentMemoryExperiment, - runAgentMemoryLearningExperiment, -} from '../../src/memory/index' -import { - continualSequences, - immutableRef, - memoryCandidate, - memoryCandidateWithoutClear, - runDirectArm, - twoStepControlSequence, -} from '../support/memory-learning' - -describe('agent memory controlled learning experiment', () => { - let result: RunAgentMemoryLearningExperimentResult - let sharedCostUsd = 0 - let executorCalls = 0 - let comparisonJson = '' - let storage: ReturnType - const observedBranchScopes = new Set() - const observedProviderTags: Record[] = [] - - beforeAll(async () => { - storage = inMemoryCampaignStorage() - const costLedger = createRunCostLedger({ - storage, - runDir: '/runs/controlled-memory-learning', - costCeilingUsd: 0.1, - }) - const candidate = memoryCandidate('memory:v1', 0.01, (scope) => { - const branchId = scope.tags?.memoryBranchId - if (branchId) observedBranchScopes.add(branchId) - observedProviderTags.push({ ...(scope.tags ?? {}) }) - }) - result = await runAgentMemoryLearningExperiment({ - experimentId: 'controlled-memory-learning', - sequences: continualSequences(), - candidates: [candidate], - runDir: '/runs/controlled-memory-learning', - storage, - controllerMode: 'process-local', - costCeiling: 0.1, - costLedger, - seed: 17, - reps: 2, - executeStepRef: immutableRef('test:shared-executor'), - async executeStep() { - executorCalls += 1 - }, - }) - sharedCostUsd = costLedger.summary().totalCostUsd - comparisonJson = storage.read(result.comparisonPath) ?? '' - }) - - it('changes only memory persistence and shares identity, execution, and spend', () => { - expect(result.stateful.memoryMode).toBe('stateful') - expect(result.stateless.memoryMode).toBe('stateless') - expect(result.armOrder).toBe('stateful-first') - expect(result.stateful.comparisonRef).toBe(result.stateless.comparisonRef) - expect(result.stateful.campaign.manifestHash).not.toBe(result.stateless.campaign.manifestHash) - expect( - result.stateful.campaign.cells.every((cell) => cell.artifact.memoryMode === 'stateful'), - ).toBe(true) - expect( - result.stateless.campaign.cells.every((cell) => cell.artifact.memoryMode === 'stateless'), - ).toBe(true) - expect(sharedCostUsd).toBeCloseTo(0.08, 12) - expect(result.stateful.totalCostUsd + result.stateless.totalCostUsd).toBeCloseTo(0.08, 12) - expect(executorCalls).toBe(16) - expect(observedBranchScopes.size).toBe(8) - for (const tags of observedProviderTags) { - expect(tags).not.toHaveProperty('memoryExperimentId') - expect(tags).not.toHaveProperty('memoryCandidateId') - expect(tags).not.toHaveProperty('memorySequenceId') - } - }) - - it('defaults a single-arm memory experiment to stateful mode', async () => { - const single = await runAgentMemoryExperiment({ - experimentId: 'default-stateful-memory', - sequences: [twoStepControlSequence()], - candidates: [memoryCandidate('memory:default-stateful')], - runDir: '/runs/default-stateful-memory', - storage: inMemoryCampaignStorage(), - controllerMode: 'process-local', - }) - - expect(single.memoryMode).toBe('stateful') - expect(single.campaign.cells[0]!.artifact.memoryMode).toBe('stateful') - }) - - it('records and executes a counterbalanced stateless-first arm order', async () => { - const storage = inMemoryCampaignStorage() - const append = storage.append!.bind(storage) - const startedPaths: string[] = [] - storage.append = (path, value, expectedBytes) => { - if (path.endsWith('/memory-attempts.jsonl') && value.includes('"status":"started"')) { - startedPaths.push(path) - } - return append(path, value, expectedBytes) - } - - const counterbalanced = await runAgentMemoryLearningExperiment({ - experimentId: 'counterbalanced-memory-learning', - runDir: '/runs/counterbalanced-memory-learning', - armOrder: 'stateless-first', - sequences: [twoStepControlSequence()], - candidates: [memoryCandidate('memory:counterbalanced')], - storage, - controllerMode: 'process-local', - }) - - expect(counterbalanced.armOrder).toBe('stateless-first') - expect(startedPaths[0]).toContain('/stateless/') - expect(startedPaths.at(-1)).toContain('/stateful/') - expect(JSON.parse(storage.read(counterbalanced.comparisonPath)!)).toMatchObject({ - armOrder: 'stateless-first', - }) - }) - - it('reports paired gain, explicit transfer, and explicit forgetting', () => { - const { comparison } = result - - expect(comparison.comparisonRef).toBe(result.stateful.comparisonRef) - expect(comparison.cells).toHaveLength(4) - expect(comparison.gain).toMatchObject({ - n: 2, - mean: 0.375, - median: 0.375, - confidence: 0.95, - resamples: 2_000, - gateEligible: false, - }) - expect(comparison.gainByCandidate).toHaveLength(1) - expect(comparison.gainByCandidate[0]).toMatchObject({ - candidateId: 'memory', - cells: 4, - gain: { n: 2, mean: 0.375, gateEligible: false }, - }) - expect(comparison.preTreatment).toMatchObject({ - definition: 'first-step-probes', - cells: 4, - n: 2, - exactMatchRate: 1, - difference: { n: 2, mean: 0 }, - }) - expect(comparison.cells.find((cell) => cell.sequenceId === 'retention')).toMatchObject({ - candidateId: 'memory', - statefulReward: 1, - statelessReward: 0.25, - gain: 0.75, - }) - - expect(comparison.transfer.definition).toBe('explicit-transfer-probes') - expect(comparison.transfer.cells).toHaveLength(2) - expect(comparison.transfer.cells.every((cell) => cell.probeCount === 1)).toBe(true) - expect(comparison.transfer.byStep).toMatchObject([ - { - candidateId: 'memory', - transferKey: 'cross-capital-recall', - stepOrdinal: 1, - gain: { n: 1, mean: 0.5, gateEligible: false }, - }, - ]) - - expect(comparison.forgetting.probes).toHaveLength(2) - expect(comparison.forgetting.probes[0]).toMatchObject({ - retentionKey: 'capital-of-france', - observations: 2, - firstStepOrdinal: 0, - finalStepOrdinal: 1, - statefulPriorPeakReward: 1, - statefulFinalReward: 1, - statefulForgetting: 0, - statelessPriorPeakReward: 1, - statelessFinalReward: 0, - statelessForgetting: 1, - excessForgetting: -1, - }) - expect(comparison.forgetting).toMatchObject({ - n: 1, - meanStatefulForgetting: 0, - meanStatelessForgetting: 1, - meanExcessForgetting: -1, - excess: { n: 1, mean: -1, gateEligible: false }, - }) - }) - - it('binds the report to exact tasks, manifests, candidates, and artifacts', () => { - expect(result.evidenceRef).toMatch(/^sha256:[a-f0-9]{64}$/) - expect(result.comparisonPath).toContain(result.evidenceRef.slice(7)) - expect(result.cost).toEqual({ - experimentUsd: 0.08, - ledgerUsd: 0.08, - ceilingUsd: 0.1, - accountingComplete: true, - }) - expect(result.comparison.evidence).toMatchObject({ - splitRef: expect.stringMatching(/^sha256:[a-f0-9]{64}$/), - statefulManifestRef: expect.stringMatching(/^sha256:[a-f0-9]{64}$/), - statelessManifestRef: expect.stringMatching(/^sha256:[a-f0-9]{64}$/), - candidateRefs: [{ id: 'memory', ref: immutableRef('memory:v1') }], - executionRef: immutableRef('test:shared-executor'), - }) - expect(result.comparison.evidence.statefulManifestRef).not.toBe( - result.comparison.evidence.statelessManifestRef, - ) - expect( - result.comparison.cells.every( - (cell) => - /^sha256:[a-f0-9]{64}$/.test(cell.statefulArtifactRef) && - /^sha256:[a-f0-9]{64}$/.test(cell.statelessArtifactRef) && - cell.statefulArtifactRef !== cell.statelessArtifactRef, - ), - ).toBe(true) - for (const arm of [result.stateful, result.stateless]) { - for (const cell of arm.campaign.cells) { - for (const probe of cell.artifact.probes) { - expect(Object.values(arm.campaign.artifactsByPath)).toContain(probe.evidencePath) - const raw = storage.read(probe.evidencePath) - expect(raw).toBeTruthy() - expect(canonicalDigest(JSON.parse(raw!))).toBe(probe.evidenceRef) - } - } - } - - const stored = JSON.parse(comparisonJson) - expect(stored).toEqual({ - evidenceRef: result.evidenceRef, - armOrder: 'stateful-first', - comparison: result.comparison, - cost: result.cost, - }) - expect( - canonicalDigest({ - armOrder: stored.armOrder, - comparison: stored.comparison, - cost: stored.cost, - }), - ).toBe(stored.evidenceRef) - }) - - it('does not infer forgetting from repeated probe ids or prompt text', () => { - expect( - result.comparison.forgetting.probes.some((probe) => probe.sequenceId === 'control'), - ).toBe(false) - }) - - it('reports first-step arm drift instead of hiding it in post-step gain', () => { - const drifted = structuredClone(result.stateless) - const artifact = drifted.campaign.cells.find( - (cell) => cell.artifact.sequenceId === 'retention', - )!.artifact - const firstProbe = artifact.probes.find((probe) => probe.stepOrdinal === 0)! - firstProbe.score = 0 - artifact.score = - artifact.probes.reduce((sum, probe) => sum + probe.score, 0) / artifact.probes.length - - const comparison = compareAgentMemoryLearning({ - stateful: result.stateful, - stateless: drifted, - }) - expect(comparison.preTreatment).toMatchObject({ - n: 2, - exactMatchRate: 0.5, - difference: { mean: 0.25 }, - }) - }) - - it('rejects swapped labels, incomplete runs, and mismatched probes', () => { - expect(() => - compareAgentMemoryLearning({ stateful: result.stateless, stateless: result.stateful }), - ).toThrow('stateful arm is labeled stateless') - - const missingCell = structuredClone(result.stateless) - missingCell.campaign.cells.pop() - expect(() => - compareAgentMemoryLearning({ stateful: result.stateful, stateless: missingCell }), - ).toThrow('stateless run is missing cells') - - const mismatchedProbe = structuredClone(result.stateless) - const artifact = mismatchedProbe.campaign.cells[0]!.artifact - artifact.probes = artifact.probes.map((probe, index) => - index === 0 ? { ...probe, id: 'different-probe' } : probe, - ) - expect(() => - compareAgentMemoryLearning({ stateful: result.stateful, stateless: mismatchedProbe }), - ).toThrow('unmatched probe in cell') - }) - - it('rejects damaged task identity and failed cells before reporting gain', () => { - const damagedIdentity = structuredClone(result.stateless) - damagedIdentity.campaign.splitDigest = `sha256:${'0'.repeat(64)}` - expect(() => - compareAgentMemoryLearning({ stateful: result.stateful, stateless: damagedIdentity }), - ).toThrow('stateless run has invalid task identity') - - const failed = structuredClone(result.stateless) - failed.campaign.cells[0]!.error = 'provider failed' - expect(() => - compareAgentMemoryLearning({ stateful: result.stateful, stateless: failed }), - ).toThrow('stateless cell') - - const damagedEvidence = structuredClone(result.stateless) - damagedEvidence.campaign.cells[0]!.artifact.branchDigest = 'sha256:not-evidence' - expect(() => - compareAgentMemoryLearning({ stateful: result.stateful, stateless: damagedEvidence }), - ).toThrow('invalid branch evidence') - }) - - it('requires scoped clear support for the stateless arm', async () => { - await expect( - runAgentMemoryExperiment({ - experimentId: 'missing-stateless-clear', - memoryMode: 'stateless', - sequences: [twoStepControlSequence()], - candidates: [memoryCandidateWithoutClear()], - runDir: '/runs/missing-stateless-clear', - storage: inMemoryCampaignStorage(), - controllerMode: 'process-local', - cleanupBranches: false, - }), - ).rejects.toThrow('stateless memoryMode requires scoped clear support') - }) - - it('rejects arms whose candidate implementation references differ', async () => { - const experimentRunId = 'unequal-memory-arms' - const [stateful, stateless] = await Promise.all([ - runDirectArm('stateful', 'memory:a', experimentRunId), - runDirectArm('stateless', 'memory:b', experimentRunId), - ]) - - expect(() => compareAgentMemoryLearning({ stateful, stateless })).toThrow( - 'experimental conditions differ', - ) - }) - - it('rejects a mutable executor reference in separately run arms', async () => { - const storage = inMemoryCampaignStorage() - const common = { - experimentId: 'mutable-executor-arms', - experimentRunId: 'mutable-executor-arms', - sequences: [twoStepControlSequence()], - candidates: [memoryCandidate('memory:mutable-executor')], - storage, - controllerMode: 'process-local' as const, - executeStepRef: 'executor:v1', - async executeStep() {}, - } - const [stateful, stateless] = await Promise.all([ - runAgentMemoryExperiment({ - ...common, - memoryMode: 'stateful', - runDir: '/runs/mutable-executor-arms/stateful', - }), - runAgentMemoryExperiment({ - ...common, - memoryMode: 'stateless', - runDir: '/runs/mutable-executor-arms/stateless', - }), - ]) - - expect(() => compareAgentMemoryLearning({ stateful, stateless })).toThrow( - 'executor reference must be lowercase sha256', - ) - }) - - it('aborts active work and resumes without replaying completed cells', async () => { - const storage = inMemoryCampaignStorage() - const runDir = '/runs/resumable-memory-learning' - const costLedger = createRunCostLedger({ storage, runDir, costCeilingUsd: 1 }) - const abortController = new AbortController() - let executeCalls = 0 - const base = { - experimentId: 'resumable-memory-learning', - experimentRunId: 'resumable-memory-learning-run', - sequences: continualSequences(), - candidates: [memoryCandidate('memory:resumable', 0.01)], - runDir, - storage, - controllerMode: 'process-local' as const, - costCeiling: 1, - costLedger, - seed: 31, - reps: 1, - maxConcurrency: 1, - executeStepRef: immutableRef('test:resumable-executor'), - async executeStep() { - executeCalls += 1 - if (executeCalls === 5) abortController.abort(new Error('stop after stateful arm')) - }, - } - - await expect( - runAgentMemoryLearningExperiment({ ...base, signal: abortController.signal }), - ).rejects.toThrow() - - const resumed = await runAgentMemoryLearningExperiment({ - ...base, - signal: new AbortController().signal, - }) - - expect(resumed.stateful.campaign.cells.every((cell) => cell.cached)).toBe(true) - expect(resumed.stateless.campaign.cells.every((cell) => !cell.error)).toBe(true) - expect(resumed.comparison.cells).toHaveLength(2) - expect(executeCalls).toBe(9) - expect(costLedger.summary()).toMatchObject({ - totalCalls: 5, - totalCostUsd: 0.05, - accountingComplete: true, - unresolvedCalls: 0, - }) - expect(resumed.cost).toEqual({ - experimentUsd: 0.05, - ledgerUsd: 0.05, - ceilingUsd: 1, - accountingComplete: true, - }) - - const evidencePath = resumed.stateful.campaign.cells[0]!.artifact.probes[0]!.evidencePath - storage.write(evidencePath, '{}\n') - await expect( - runAgentMemoryLearningExperiment({ - ...base, - signal: new AbortController().signal, - }), - ).rejects.toThrow('probe evidence hash does not match') - }) - - it('checks cached stateful evidence before starting the stateless arm', async () => { - const storage = inMemoryCampaignStorage() - const runDir = '/runs/corrupt-stateful-memory-learning' - let executeCalls = 0 - const base = { - experimentId: 'corrupt-stateful-memory-learning', - experimentRunId: runDir, - sequences: continualSequences(), - candidates: [memoryCandidate('memory:corrupt-stateful')], - storage, - controllerMode: 'process-local' as const, - seed: 41, - reps: 1, - executeStepRef: immutableRef('test:corrupt-stateful-executor'), - async executeStep() { - executeCalls += 1 - }, - } - const stateful = await runAgentMemoryExperiment({ - ...base, - memoryMode: 'stateful', - runDir: `${runDir}/stateful`, - }) - expect(executeCalls).toBe(4) - const evidencePath = stateful.campaign.cells[0]!.artifact.probes[0]!.evidencePath - storage.write(evidencePath, '{}\n') - - await expect( - runAgentMemoryLearningExperiment({ - ...base, - runDir, - }), - ).rejects.toThrow('probe evidence hash does not match') - expect(executeCalls).toBe(4) - }) - - it('recomputes cached probe scores from their saved retrieval evidence', async () => { - const storage = inMemoryCampaignStorage() - const runDir = '/runs/tampered-memory-learning-score' - const base = { - experimentId: 'tampered-memory-learning-score', - experimentRunId: runDir, - sequences: [twoStepControlSequence()], - candidates: [memoryCandidate('memory:tampered-score')], - runDir, - storage, - controllerMode: 'process-local' as const, - } - const first = await runAgentMemoryLearningExperiment(base) - const cell = first.stateful.campaign.cells[0]! - const cachePath = `${first.stateful.campaign.runDir}/${cell.cellId.replace( - /[^a-zA-Z0-9_-]/g, - '_', - )}/cached-result.json` - const cached = JSON.parse(storage.read(cachePath)!) - cached.artifact.probes[1].score = 0 - cached.artifact.score = - cached.artifact.probes.reduce( - (sum: number, probe: { score: number }) => sum + probe.score, - 0, - ) / cached.artifact.probes.length - cached.artifact.passed = false - storage.write(cachePath, JSON.stringify(cached)) - - await expect(runAgentMemoryLearningExperiment(base)).rejects.toThrow( - 'probe score does not match saved evidence', - ) - }) -}) diff --git a/tests/memory/experiment-parallel.test.ts b/tests/memory/experiment-parallel.test.ts deleted file mode 100644 index 689bb80..0000000 --- a/tests/memory/experiment-parallel.test.ts +++ /dev/null @@ -1,149 +0,0 @@ -import { inMemoryCampaignStorage } from '@tangle-network/agent-eval/campaign' -import { describe, expect, it } from 'vitest' -import type { - AgentMemoryAdapter, - AgentMemoryBranchSnapshot, - AgentMemoryScope, -} from '../../src/memory/index' -import { createScopedTestAdapter, runAgentMemoryExperiment } from '../support/memory' - -describe('agent memory experiment parallel operations', () => { - it('runs independent writes concurrently and keeps journal order deterministic', async () => { - let activeWrites = 0 - let maxActiveWrites = 0 - let snapshot: AgentMemoryBranchSnapshot | undefined - const adapter = createScopedTestAdapter('parallel-writes', async (_scope, text) => { - activeWrites += 1 - maxActiveWrites = Math.max(maxActiveWrites, activeWrites) - await new Promise((resolve) => setTimeout(resolve, text.includes('first') ? 10 : 1)) - activeWrites -= 1 - }) - - const result = await runAgentMemoryExperiment({ - experimentId: 'parallel-write-order', - sequences: [ - { - id: 'history', - family: 'first-party', - steps: [ - { - id: 'write', - parallelWrites: true, - parallelProbes: false, - writes: [ - { - id: 'first', - kind: 'fact', - text: 'first fact', - scope: { agentId: 'first-agent' }, - }, - { - id: 'second', - kind: 'fact', - text: 'second fact', - scope: { agentId: 'second-agent' }, - }, - ], - probes: [ - { - id: 'first', - query: 'first', - scope: { agentId: 'first-agent' }, - referenceAnswer: 'first fact', - }, - { - id: 'second', - query: 'second', - scope: { agentId: 'second-agent' }, - referenceAnswer: 'second fact', - }, - ], - }, - ], - }, - ], - candidates: [ - { - id: 'memory', - ref: 'memory:parallel', - createAdapter: () => adapter, - }, - ], - runDir: '/runs/parallel-write-order', - storage: inMemoryCampaignStorage(), - controllerMode: 'process-local', - onBranchSnapshot(input) { - snapshot = input.snapshot - }, - }) - - expect(result.rows[0]).toMatchObject({ cellsFailed: 0, scoreMean: 1 }) - expect(maxActiveWrites).toBe(2) - expect(snapshot?.journal.map((entry) => entry.sequence)).toEqual([0, 1]) - expect(snapshot?.journal.map((entry) => entry.input.id)).toEqual(['first', 'second']) - }) - - it('drains successful sibling writes before clearing a partially failed step', async () => { - let adapter: AgentMemoryAdapter | undefined - const touchedScopes: AgentMemoryScope[] = [] - adapter = createScopedTestAdapter('parallel-write-failure', async (scope, text) => { - touchedScopes.push(structuredClone(scope)) - if (text.includes('fail')) throw new Error('simulated write failure') - await new Promise((resolve) => setTimeout(resolve, 10)) - }) - - const result = await runAgentMemoryExperiment({ - experimentId: 'parallel-write-failure', - sequences: [ - { - id: 'history', - family: 'first-party', - steps: [ - { - id: 'write', - parallelWrites: true, - writes: [ - { - id: 'survivor', - kind: 'fact', - text: 'successful sibling', - scope: { agentId: 'survivor' }, - }, - { - id: 'failure', - kind: 'fact', - text: 'fail this write', - scope: { agentId: 'failure' }, - }, - ], - probes: [ - { - id: 'unused', - query: 'sibling', - scope: { agentId: 'survivor' }, - referenceAnswer: 'successful sibling', - }, - ], - }, - ], - }, - ], - candidates: [ - { - id: 'memory', - ref: 'memory:parallel-failure', - createAdapter: () => adapter!, - }, - ], - runDir: '/runs/parallel-write-failure', - storage: inMemoryCampaignStorage(), - controllerMode: 'process-local', - }) - - expect(result.rows[0]).toMatchObject({ cellsFailed: 1 }) - expect(touchedScopes).toHaveLength(2) - for (const scope of touchedScopes) { - await expect(adapter.search('sibling', { scope })).resolves.toEqual([]) - } - }) -}) diff --git a/tests/memory/experiment-privacy.test.ts b/tests/memory/experiment-privacy.test.ts deleted file mode 100644 index 2035d49..0000000 --- a/tests/memory/experiment-privacy.test.ts +++ /dev/null @@ -1,306 +0,0 @@ -import { inMemoryCampaignStorage } from '@tangle-network/agent-eval/campaign' -import { describe, expect, it } from 'vitest' -import { buildAgentMemorySequencesFromBenchmarkCases } from '../../src/memory/index' -import { createScopedTestAdapter, runAgentMemoryExperiment } from '../support/memory' - -describe('agent memory experiment privacy', () => { - it('does not expose raw benchmark case ids in default provider scopes', async () => { - const caseId = 'FINAL_BENCHMARK_CASE_SECRET' - const scopes: unknown[] = [] - const sequences = buildAgentMemorySequencesFromBenchmarkCases([ - { - id: caseId, - family: 'first-party', - taskKind: 'memory-recall', - events: [{ id: 'fact', actorId: 'user', text: 'The answer is green.' }], - prompt: 'What is the answer?', - referenceAnswer: 'answer is green', - }, - ]) - - await runAgentMemoryExperiment({ - experimentId: 'redacted-benchmark-case', - sequences, - candidates: [ - { - id: 'memory', - ref: 'memory:redacted-case', - createAdapter: () => - createScopedTestAdapter('redacted-case', async (scope) => { - scopes.push(structuredClone(scope)) - }), - }, - ], - runDir: '/runs/redacted-benchmark-case', - storage: inMemoryCampaignStorage(), - }) - - expect(scopes).not.toHaveLength(0) - expect(JSON.stringify(scopes)).not.toContain(caseId) - expect(scopes).toEqual( - expect.arrayContaining([expect.objectContaining({ sessionId: 'benchmark-session' })]), - ) - }) - - it('does not expose random seeds or repetitions to candidate adapters', async () => { - const adapterInputs: unknown[] = [] - await runAgentMemoryExperiment({ - experimentId: 'paired-seeds', - sequences: [ - { - id: 'paired-history', - family: 'first-party', - steps: [ - { - id: 'probe', - scope: { agentId: 'worker' }, - probes: [{ id: 'state', query: 'state', referenceAnswer: 'state' }], - }, - ], - }, - ], - candidates: ['a', 'b'].map((candidateId) => ({ - id: candidateId, - ref: `${candidateId}:v1`, - createAdapter: (input) => { - const { - markExternalCall: _markExternalCall, - recordExternalCost: _recordExternalCost, - signal: _signal, - ...visible - } = input - adapterInputs.push(structuredClone(visible)) - return createScopedTestAdapter(candidateId) - }, - })), - reps: 2, - runDir: '/runs/paired-seeds', - storage: inMemoryCampaignStorage(), - maxConcurrency: 4, - }) - - expect(adapterInputs).toHaveLength(4) - for (const input of adapterInputs) { - expect(input).not.toHaveProperty('rep') - expect(input).not.toHaveProperty('seed') - } - }) - - it('never exposes evaluation labels or dataset identity to candidate callbacks', async () => { - const adapterInputs: unknown[] = [] - const stepInputs: unknown[] = [] - const executionContexts: object[] = [] - const executionCostResults: unknown[] = [] - await runAgentMemoryExperiment({ - experimentId: 'redacted-candidate-input', - sequences: [ - { - id: 'FINAL_SEQUENCE_SECRET', - family: 'first-party', - split: 'test', - steps: [ - { - id: 'FINAL_STEP_SECRET', - instruction: 'Remember the supplied launch state.', - scope: { agentId: 'worker' }, - writes: [{ kind: 'fact', text: 'launch state' }], - probes: [ - { - id: 'FINAL_PROBE_SECRET', - query: 'launch state', - requiredFacts: [ - { id: 'EXPECTED_FACT_ID_SECRET', anyOf: ['EXPECTED_FACT_SECRET'] }, - ], - referenceAnswer: 'REFERENCE_ANSWER_SECRET', - }, - ], - metadata: { privateLabel: 'STEP_METADATA_SECRET' }, - }, - ], - metadata: { privateLabel: 'SEQUENCE_METADATA_SECRET' }, - }, - ], - candidates: [ - { - id: 'redacted', - ref: 'redacted:v1', - createAdapter(input) { - const { - markExternalCall: _markExternalCall, - recordExternalCost: _recordExternalCost, - signal: _signal, - ...visible - } = input - adapterInputs.push(structuredClone(visible)) - return createScopedTestAdapter('redacted') - }, - }, - ], - executeStepRef: 'redacted-worker/v1', - async executeStep(input) { - executionContexts.push(input.context) - executionCostResults.push( - await input.context.cost.runPaidCall({ - actor: 'redacted-worker', - model: 'free-local-test', - maximumCharge: { externallyEnforcedMaximumUsd: 0 }, - execute: async () => 'complete', - receipt: () => ({ - model: 'free-local-test', - inputTokens: 0, - outputTokens: 0, - actualCostUsd: 0, - }), - }), - ) - const { memory: _memory, context: _context, ...visible } = input - stepInputs.push(structuredClone(visible)) - }, - runDir: '/runs/redacted-candidate-input', - storage: inMemoryCampaignStorage(), - }) - - expect(executionContexts).toHaveLength(1) - const actualContext = executionContexts[0]! - expect(Reflect.ownKeys(actualContext).map(String).sort()).toEqual(['cost', 'signal']) - for (const privateKey of [ - 'cellId', - 'rep', - 'generation', - 'seed', - 'trace', - 'artifacts', - 'cycleId', - 'resumedFrom', - 'placement', - ]) { - expect(privateKey in actualContext).toBe(false) - } - expect(Reflect.ownKeys((actualContext as { cost: object }).cost).map(String)).toEqual([ - 'runPaidCall', - ]) - const costResult = executionCostResults[0] as { - succeeded: boolean - receipt?: Record - } - expect(costResult.succeeded).toBe(true) - expect(costResult.receipt).toBeDefined() - expect(costResult.receipt).not.toHaveProperty('tags') - expect(costResult.receipt).not.toHaveProperty('phase') - - const visible = JSON.stringify( - publicRuntimeSurface({ - adapterInputs, - stepInputs, - executionContexts, - executionCostResults, - }), - ) - for (const privateLabel of [ - 'FINAL_SEQUENCE_SECRET', - 'FINAL_STEP_SECRET', - 'FINAL_PROBE_SECRET', - 'EXPECTED_FACT_ID_SECRET', - 'EXPECTED_FACT_SECRET', - 'REFERENCE_ANSWER_SECRET', - 'STEP_METADATA_SECRET', - 'SEQUENCE_METADATA_SECRET', - ]) { - expect(visible).not.toContain(privateLabel) - } - expect(stepInputs).toEqual([ - { - candidateId: 'redacted', - step: { - ordinal: 0, - instruction: 'Remember the supplied launch state.', - scope: { agentId: 'worker' }, - }, - }, - ]) - }) - - it('redacts campaign cell identity from execution cancellation', async () => { - let abortReason: unknown - await runAgentMemoryExperiment({ - experimentId: 'redacted-execution-abort', - sequences: [ - { - id: 'FINAL_ABORT_SEQUENCE_SECRET', - family: 'first-party', - steps: [ - { - id: 'FINAL_ABORT_STEP_SECRET', - probes: [ - { - id: 'FINAL_ABORT_PROBE_SECRET', - query: 'state', - referenceAnswer: 'FINAL_ABORT_EXPECTED_SECRET', - }, - ], - }, - ], - }, - ], - candidates: [ - { - id: 'redacted', - ref: 'redacted:v1', - createAdapter: () => createScopedTestAdapter('redacted-abort'), - }, - ], - executeStepRef: 'redacted-abort-worker/v1', - async executeStep({ context }) { - if (context.signal.aborted) { - abortReason = context.signal.reason - return - } - await new Promise((resolve) => { - context.signal.addEventListener( - 'abort', - () => { - abortReason = context.signal.reason - resolve() - }, - { once: true }, - ) - }) - }, - dispatchTimeoutMs: 5, - runDir: '/runs/redacted-execution-abort', - storage: inMemoryCampaignStorage(), - }).catch(() => undefined) - - expect(abortReason).toBeInstanceOf(Error) - expect((abortReason as Error).name).toBe('AbortError') - expect((abortReason as Error).message).toBe('memory execution aborted') - expect(String(abortReason)).not.toContain('FINAL_ABORT_SEQUENCE_SECRET') - expect(String(abortReason)).not.toContain('FINAL_ABORT_STEP_SECRET') - expect(String(abortReason)).not.toContain('FINAL_ABORT_PROBE_SECRET') - expect(String(abortReason)).not.toContain('FINAL_ABORT_EXPECTED_SECRET') - }) -}) - -function publicRuntimeSurface(value: unknown, seen = new WeakSet()): unknown { - if ( - value === null || - typeof value === 'string' || - typeof value === 'number' || - typeof value === 'boolean' || - value === undefined - ) { - return value - } - if (typeof value === 'function') return '[function]' - if (typeof value !== 'object') return String(value) - if (seen.has(value)) return '[circular]' - seen.add(value) - if (value instanceof Error) return { name: value.name, message: value.message } - if (Array.isArray(value)) return value.map((item) => publicRuntimeSurface(item, seen)) - return Object.fromEntries( - Object.keys(value).map((key) => [ - key, - publicRuntimeSurface((value as Record)[key], seen), - ]), - ) -} diff --git a/tests/memory/experiment-recovery-safety.test.ts b/tests/memory/experiment-recovery-safety.test.ts deleted file mode 100644 index 001d957..0000000 --- a/tests/memory/experiment-recovery-safety.test.ts +++ /dev/null @@ -1,302 +0,0 @@ -import { canonicalDigest, inMemoryCampaignStorage } from '@tangle-network/agent-eval/campaign' -import { describe, expect, it } from 'vitest' -import type { AgentMemoryAdapter } from '../../src/memory/index' -import { createScopedTestAdapter, runAgentMemoryExperiment } from '../support/memory' - -describe('agent memory experiment recovery safety', () => { - it('returns with recoverable state when a provider write never settles', async () => { - const storage = inMemoryCampaignStorage() - const runDir = '/runs/hung-provider-write' - let writeStarted!: () => void - const started = new Promise((resolve) => { - writeStarted = resolve - }) - const adapter: AgentMemoryAdapter = { - id: 'hung-provider', - branchIsolation: { mode: 'scoped' }, - async search() { - return [] - }, - async getContext(query) { - return { query, text: '', hits: [], sourceRecords: [] } - }, - async write() { - writeStarted() - return new Promise(() => undefined) - }, - async clear() {}, - } - const run = runAgentMemoryExperiment({ - experimentId: 'hung-provider-write', - sequences: [ - { - id: 'history', - family: 'first-party', - steps: [ - { - id: 'write', - scope: { agentId: 'worker' }, - writes: [{ kind: 'fact', text: 'fact' }], - probes: [{ id: 'recall', query: 'fact', referenceAnswer: 'fact' }], - }, - ], - }, - ], - candidates: [{ id: 'memory', ref: 'memory:v1', createAdapter: () => adapter }], - runDir, - storage, - dispatchTimeoutMs: 5, - cleanupTimeoutMs: 20, - }) - - await started - let timeout: ReturnType | undefined - const outcome = await Promise.race([ - run.then( - () => ({ status: 'resolved' as const }), - (error: unknown) => ({ status: 'rejected' as const, error }), - ), - new Promise<{ status: 'hung' }>((resolve) => { - timeout = setTimeout(() => resolve({ status: 'hung' }), 500) - }), - ]).finally(() => { - if (timeout) clearTimeout(timeout) - }) - - expect(outcome.status).toBe('rejected') - if (outcome.status !== 'rejected') return - expect(outcome.error).toBeInstanceOf(Error) - expect((outcome.error as Error).message).toContain('cleanup failed after dispatch') - const attempts = storage.read(`${runDir}/memory-attempts.jsonl`)!.trim().split('\n') - expect(attempts).toHaveLength(1) - expect(JSON.parse(attempts[0]!)).toMatchObject({ status: 'started', recovery: false }) - }) - - it('cancels adapter creation and disposes an execution adapter that arrives late', async () => { - const storage = inMemoryCampaignStorage() - const controller = new AbortController() - let resolveCreation!: (adapter: AgentMemoryAdapter) => void - const creation = new Promise((resolve) => { - resolveCreation = resolve - }) - let reportStarted!: () => void - const started = new Promise((resolve) => { - reportStarted = resolve - }) - let reportDisposed!: () => void - const disposed = new Promise((resolve) => { - reportDisposed = resolve - }) - let providerSignal: AbortSignal | undefined - let closeCalls = 0 - let disposeCalls = 0 - const lateAdapter = createScopedTestAdapter('late-execution-provider') - lateAdapter.close = async () => { - closeCalls += 1 - } - const run = runAgentMemoryExperiment({ - experimentId: 'late-execution-adapter', - sequences: [ - { - id: 'history', - family: 'first-party', - steps: [ - { - id: 'probe', - scope: { agentId: 'worker' }, - probes: [{ id: 'recall', query: 'fact', referenceAnswer: 'fact' }], - }, - ], - }, - ], - candidates: [ - { - id: 'memory', - ref: 'memory:v1', - createAdapter({ signal }) { - providerSignal = signal - reportStarted() - return creation - }, - async disposeAdapter(adapter) { - expect(adapter).toBe(lateAdapter) - disposeCalls += 1 - reportDisposed() - }, - }, - ], - runDir: '/runs/late-execution-adapter', - storage, - signal: controller.signal, - }) - - await started - controller.abort(new Error('caller cancellation')) - await expect(run).rejects.toThrow() - expect(providerSignal?.aborted).toBe(true) - expect((providerSignal!.reason as Error).message).toBe( - 'memory: execution adapter creation aborted', - ) - - resolveCreation(lateAdapter) - await disposed - expect({ closeCalls, disposeCalls }).toEqual({ closeCalls: 1, disposeCalls: 1 }) - }) - - it('cancels an abandoned-write visibility wait without clearing the branch', async () => { - const storage = inMemoryCampaignStorage() - const runDir = '/runs/cancelled-recovery-delay' - const sequence = { - id: 'history', - family: 'first-party' as const, - steps: [ - { - id: 'probe', - scope: { agentId: 'worker' }, - probes: [{ id: 'recall', query: 'fact', referenceAnswer: 'fact' }], - }, - ], - } - storage.write( - `${runDir}/memory-attempts.jsonl`, - `${JSON.stringify({ - status: 'started', - branchId: 'unfinished-branch', - candidateId: 'memory', - candidateRef: 'memory:v1', - sequenceId: sequence.id, - sequenceRef: canonicalDigest(sequence), - rep: 0, - seed: 1, - cleanupBranches: true, - externalCostUsdPerSequence: 0, - externalRecoveryCostUsdPerAttempt: 0, - recordedAt: '2026-01-01T00:00:00.000Z', - recovery: false, - })}\n`, - ) - const controller = new AbortController() - let recoveryStarted!: () => void - const started = new Promise((resolve) => { - recoveryStarted = resolve - }) - let clearCalls = 0 - let closeCalls = 0 - const run = runAgentMemoryExperiment({ - experimentId: 'cancelled-recovery-delay', - sequences: [sequence], - candidates: [ - { - id: 'memory', - ref: 'memory:v1', - createAdapter({ purpose }) { - if (purpose === 'recovery') recoveryStarted() - return { - id: 'delayed-provider', - branchIsolation: { - mode: 'scoped', - processExitSafe: false, - recoveryDelayMs: 1_000, - }, - async search() { - return [] - }, - async getContext(query) { - return { query, text: '', hits: [], sourceRecords: [] } - }, - async write(input) { - return { - accepted: true, - id: 'write', - uri: 'memory://delayed-provider/write', - kind: input.kind, - } - }, - async clear() { - clearCalls += 1 - }, - async close() { - closeCalls += 1 - }, - } - }, - }, - ], - runDir, - storage, - signal: controller.signal, - }) - - await started - controller.abort(new Error('caller cancellation')) - await expect(run).rejects.toThrow('recovery failed') - expect({ clearCalls, closeCalls }).toEqual({ clearCalls: 0, closeCalls: 1 }) - expect(storage.read(`${runDir}/memory-attempts.jsonl`)!.trim().split('\n')).toHaveLength(1) - }) - - it('refuses to clean an unfinished branch with changed sequence scopes', async () => { - const storage = inMemoryCampaignStorage() - const runDir = '/runs/changed-recovery-sequence' - const recordedSequence = { - id: 'history', - family: 'first-party' as const, - steps: [ - { - id: 'probe', - scope: { agentId: 'original-worker' }, - probes: [{ id: 'recall', query: 'fact', referenceAnswer: 'fact' }], - }, - ], - } - storage.write( - `${runDir}/memory-attempts.jsonl`, - `${JSON.stringify({ - status: 'started', - branchId: 'unfinished-branch', - candidateId: 'memory', - candidateRef: 'memory:v1', - sequenceId: recordedSequence.id, - sequenceRef: canonicalDigest(recordedSequence), - rep: 0, - seed: 1, - cleanupBranches: true, - externalCostUsdPerSequence: 0, - externalRecoveryCostUsdPerAttempt: 0, - recordedAt: '2026-01-01T00:00:00.000Z', - recovery: false, - })}\n`, - ) - let adapterCreates = 0 - - await expect( - runAgentMemoryExperiment({ - experimentId: 'changed-recovery-sequence', - sequences: [ - { - ...recordedSequence, - steps: [ - { - ...recordedSequence.steps[0], - scope: { agentId: 'different-worker' }, - }, - ], - }, - ], - candidates: [ - { - id: 'memory', - ref: 'memory:v1', - createAdapter() { - adapterCreates += 1 - return createScopedTestAdapter('memory') - }, - }, - ], - runDir, - storage, - }), - ).rejects.toThrow("sequence 'history' changed") - - expect(adapterCreates).toBe(0) - }) -}) diff --git a/tests/memory/experiment-recovery.test.ts b/tests/memory/experiment-recovery.test.ts deleted file mode 100644 index 4e7dfd8..0000000 --- a/tests/memory/experiment-recovery.test.ts +++ /dev/null @@ -1,719 +0,0 @@ -import { - canonicalDigest, - createRunCostLedger, - inMemoryCampaignStorage, -} from '@tangle-network/agent-eval/campaign' -import { describe, expect, it } from 'vitest' -import { - type AgentMemoryAdapter, - type AgentMemoryHit, - buildAgentMemorySequenceScenarios, -} from '../../src/memory/index' -import { createScopedTestAdapter, hitText, runAgentMemoryExperiment } from '../support/memory' - -describe('agent memory experiment recovery', () => { - it('recovers an unfinished provider branch before retrying the history', async () => { - const storage = inMemoryCampaignStorage() - const rows = new Map() - const operations: string[] = [] - const branchIds: Record<'first' | 'recovery' | 'retry', string | undefined> = { - first: undefined, - recovery: undefined, - retry: undefined, - } - let firstExecution = true - const sequence = { - id: 'recover-history', - family: 'first-party' as const, - steps: [ - { - id: 'remember', - scope: { agentId: 'worker' }, - writes: [{ kind: 'fact' as const, text: 'sensitive provider fact' }], - probes: [ - { - id: 'recall', - query: 'provider fact', - referenceAnswer: 'sensitive provider fact', - }, - ], - }, - ], - } - const candidate = { - id: 'recoverable', - ref: 'recoverable:v1', - externalRecoveryCostUsdPerAttempt: 0.1, - externalCostAccounting: 'exact' as const, - createAdapter({ branchId, purpose, recordExternalCost }) { - if (purpose === 'recovery') recordExternalCost(0.1) - const executionLabel = - purpose === 'recovery' ? 'recovery' : firstExecution ? 'first' : 'retry' - branchIds[executionLabel] = branchId - operations.push(`create:${executionLabel}`) - let clearCalls = 0 - const adapter: AgentMemoryAdapter = { - id: 'recoverable-provider', - branchIsolation: { mode: 'scoped' }, - async search(_query, options) { - return [...(rows.get(options.scope?.namespace ?? '') ?? [])] - }, - async getContext(query, options) { - const hits = await adapter.search(query, options) - return { query, text: hits.map(hitText).join('\n'), hits, sourceRecords: [] } - }, - async write(input) { - operations.push(`write:${executionLabel}`) - const namespace = input.scope?.namespace ?? '' - const hit = { - id: input.id ?? `${executionLabel}-fact`, - uri: `memory://recoverable/${executionLabel}`, - kind: input.kind, - text: input.text, - } - rows.set(namespace, [...(rows.get(namespace) ?? []), hit]) - return { accepted: true, id: hit.id, uri: hit.uri, kind: hit.kind } - }, - async clear(scope) { - clearCalls += 1 - operations.push(`clear:${executionLabel}`) - if (executionLabel === 'first' && clearCalls === 1) { - throw new Error('provider cleanup unavailable') - } - rows.delete(scope?.namespace ?? '') - }, - } - if (purpose === 'execute') firstExecution = false - return adapter - }, - } - const run = () => - runAgentMemoryExperiment({ - experimentId: 'restart-recovery', - sequences: [sequence], - candidates: [candidate], - runDir: '/runs/restart-recovery', - storage, - costCeiling: 1, - }) - - await expect(run()).rejects.toThrow('memory experiment cleanup failed after dispatch') - expect(rows.size).toBe(1) - - const result = await run() - - expect(result.rows[0]).toMatchObject({ candidateId: 'recoverable', cellsFailed: 0 }) - expect(result.rows[0]?.totalCostUsd).toBe(0.1) - expect(result.campaign.aggregates.cost.totalCostUsd).toBe(0.1) - expect(branchIds.recovery).toBe(branchIds.first) - expect(branchIds.retry).not.toBe(branchIds.first) - expect(operations.indexOf('clear:recovery')).toBeLessThan(operations.indexOf('write:retry')) - expect(rows.size).toBe(0) - const attemptEvents = storage - .read(result.attemptLogPath)! - .trim() - .split('\n') - .map((line) => JSON.parse(line) as { status: string; recovery: boolean }) - expect(attemptEvents.map(({ status, recovery }) => ({ status, recovery }))).toEqual([ - { status: 'started', recovery: false }, - { status: 'cleaned', recovery: true }, - { status: 'started', recovery: false }, - { status: 'cleaned', recovery: false }, - ]) - }) - - it('closes and disposes a recovery adapter that arrives after its factory timeout', async () => { - const storage = inMemoryCampaignStorage() - const runDir = '/runs/late-recovery-adapter' - const sequence = { - id: 'history', - family: 'first-party' as const, - steps: [ - { - id: 'remember', - scope: { agentId: 'worker' }, - writes: [{ kind: 'fact' as const, text: 'fact' }], - probes: [{ id: 'recall', query: 'fact', referenceAnswer: 'fact' }], - }, - ], - } - storage.write( - `${runDir}/memory-attempts.jsonl`, - `${JSON.stringify({ - status: 'started', - branchId: 'unfinished-branch', - candidateId: 'memory', - candidateRef: 'memory:v1', - sequenceId: 'history', - sequenceRef: canonicalDigest(sequence), - rep: 0, - seed: 42, - cleanupBranches: true, - externalCostUsdPerSequence: 0, - externalRecoveryCostUsdPerAttempt: 0, - recordedAt: '2026-01-01T00:00:00.000Z', - recovery: false, - })}\n`, - ) - let resolveCreation!: (adapter: AgentMemoryAdapter) => void - const creation = new Promise((resolve) => { - resolveCreation = resolve - }) - let reportDisposed!: () => void - const disposed = new Promise((resolve) => { - reportDisposed = resolve - }) - let closeCalls = 0 - let disposeCalls = 0 - const lateAdapter = createScopedTestAdapter('memory-provider') - lateAdapter.close = async () => { - closeCalls += 1 - } - - await expect( - runAgentMemoryExperiment({ - experimentId: 'late-recovery-adapter', - sequences: [sequence], - candidates: [ - { - id: 'memory', - ref: 'memory:v1', - createAdapter: ({ purpose }) => - purpose === 'recovery' ? creation : createScopedTestAdapter('memory-provider'), - async disposeAdapter(adapter) { - expect(adapter).toBe(lateAdapter) - disposeCalls += 1 - reportDisposed() - }, - }, - ], - runDir, - storage, - cleanupTimeoutMs: 10, - }), - ).rejects.toThrow("memory: abandoned memory branch 'unfinished-branch' recovery failed") - - resolveCreation(lateAdapter) - await disposed - expect({ closeCalls, disposeCalls }).toEqual({ closeCalls: 1, disposeCalls: 1 }) - }) - - it('reconciles a crash after provider recovery but before its cost receipt', async () => { - const storage = inMemoryCampaignStorage() - const append = storage.append!.bind(storage) - let failRecoveryReceipt = false - storage.append = (path, value, expectedBytes) => { - if ( - failRecoveryReceipt && - path.endsWith('/cost-ledger.jsonl') && - value.includes('"status":"settled"') && - value.includes('agent-knowledge:memory-recovery') - ) { - failRecoveryReceipt = false - throw new Error('simulated process exit before recovery receipt') - } - return append(path, value, expectedBytes) - } - - let firstExecution = true - let recoveryClears = 0 - const candidate = { - id: 'crash-recoverable', - ref: 'crash-recoverable:v1', - externalCostUsdPerSequence: 0.1, - externalCostAccounting: 'exact' as const, - externalRecoveryCostUsdPerAttempt: 0.1, - createAdapter({ purpose, recordExternalCost }) { - recordExternalCost(0.1) - const adapter = createScopedTestAdapter(`crash-recoverable:${purpose}`) - const clear = adapter.clear! - let clearCalls = 0 - const failThisExecution = purpose === 'execute' && firstExecution - if (purpose === 'execute') firstExecution = false - adapter.clear = async (scope) => { - clearCalls += 1 - if (purpose === 'recovery') recoveryClears += 1 - if (failThisExecution && clearCalls === 1) { - throw new Error('leave the first branch active') - } - await clear(scope) - } - return adapter - }, - } - const run = () => - runAgentMemoryExperiment({ - experimentId: 'recovery-receipt-crash', - sequences: [ - { - id: 'history', - family: 'first-party', - steps: [ - { - id: 'remember', - scope: { agentId: 'worker' }, - writes: [{ kind: 'fact', text: 'durable recovery fact' }], - probes: [ - { - id: 'recall', - query: 'durable', - referenceAnswer: 'durable recovery fact', - }, - ], - }, - ], - }, - ], - candidates: [candidate], - runDir: '/runs/recovery-receipt-crash', - storage, - costCeiling: 1, - }) - - await expect(run()).rejects.toThrow('memory experiment cleanup failed after dispatch') - failRecoveryReceipt = true - await expect(run()).rejects.toThrow('failed to persist') - expect( - storage.read('/runs/recovery-receipt-crash/memory-attempts.jsonl')?.trim().split('\n'), - ).toHaveLength(2) - - const result = await run() - const costLedger = createRunCostLedger({ - storage, - runDir: '/runs/recovery-receipt-crash', - costCeilingUsd: 1, - }) - - expect(result.rows[0]).toMatchObject({ - candidateId: 'crash-recoverable', - cellsFailed: 0, - }) - expect(result.rows[0]?.totalCostUsd).toBeCloseTo(0.3) - expect(recoveryClears).toBe(1) - expect(costLedger.summary()).toMatchObject({ unresolvedCalls: 0, accountingComplete: true }) - expect(costLedger.summary().totalCostUsd).toBeCloseTo(0.3) - }) - - it('recovers independent abandoned branches in parallel before retrying them', async () => { - const storage = inMemoryCampaignStorage() - let phase: 'leave-active' | 'recover' = 'leave-active' - let activeRecoveries = 0 - let maxActiveRecoveries = 0 - let releaseRecoveries: (() => void) | undefined - const recoveriesStarted = new Promise((resolve) => { - releaseRecoveries = resolve - }) - const sequences = ['one', 'two'].map((id) => ({ - id, - family: 'first-party' as const, - steps: [ - { - id: 'remember', - scope: { agentId: 'worker' }, - writes: [{ kind: 'fact' as const, text: `${id} fact` }], - probes: [{ id: 'recall', query: id, referenceAnswer: `${id} fact` }], - }, - ], - })) - const run = () => - runAgentMemoryExperiment({ - experimentId: 'parallel-recovery', - sequences, - candidates: [ - { - id: 'memory', - ref: 'memory:v1', - async createAdapter({ purpose }) { - if (purpose === 'recovery') { - activeRecoveries += 1 - maxActiveRecoveries = Math.max(maxActiveRecoveries, activeRecoveries) - if (activeRecoveries === 2) releaseRecoveries?.() - await recoveriesStarted - activeRecoveries -= 1 - return null - } - if (phase === 'leave-active') return null - return createScopedTestAdapter('parallel-recovery') - }, - }, - ], - runDir: '/runs/parallel-recovery', - storage, - maxConcurrency: 2, - }) - - await expect(run()).rejects.toThrow('memory experiment cleanup failed after dispatch') - phase = 'recover' - const result = await run() - - expect(maxActiveRecoveries).toBe(2) - expect(result.rows[0]).toMatchObject({ candidateId: 'memory', cellsFailed: 0 }) - const recovered = storage - .read(result.attemptLogPath)! - .trim() - .split('\n') - .map((line) => JSON.parse(line) as { status: string; recovery: boolean }) - .filter((event) => event.status === 'cleaned' && event.recovery) - expect(recovered).toHaveLength(2) - }) - - it('closes an attempt whose provider was never created before retrying it', async () => { - const storage = inMemoryCampaignStorage() - const purposes: Array<'execute' | 'recovery'> = [] - let executionCalls = 0 - const sequence = { - id: 'provider-creation', - family: 'first-party' as const, - steps: [ - { - id: 'remember', - scope: { agentId: 'worker' }, - writes: [{ kind: 'fact' as const, text: 'created provider fact' }], - probes: [ - { - id: 'recall', - query: 'provider fact', - referenceAnswer: 'created provider fact', - }, - ], - }, - ], - } - const run = () => - runAgentMemoryExperiment({ - experimentId: 'provider-creation-recovery', - sequences: [sequence], - candidates: [ - { - id: 'sometimes-created', - ref: 'sometimes-created:v1', - externalRecoveryCostUsdPerAttempt: 0.1, - externalCostAccounting: 'exact', - createAdapter({ purpose }) { - purposes.push(purpose) - if (purpose === 'recovery') return null - executionCalls += 1 - if (executionCalls === 1) return null - return createScopedTestAdapter('created-provider') - }, - }, - ], - runDir: '/runs/provider-creation-recovery', - storage, - costCeiling: 1, - }) - - await expect(run()).rejects.toThrow('memory experiment cleanup failed after dispatch') - const result = await run() - - expect(result.rows[0]).toMatchObject({ candidateId: 'sometimes-created', cellsFailed: 0 }) - expect(result.rows[0]?.totalCostUsd).toBe(0) - expect(purposes).toEqual(['execute', 'recovery', 'execute']) - const attemptEvents = storage - .read(result.attemptLogPath)! - .trim() - .split('\n') - .map((line) => JSON.parse(line) as { status: string; recovery: boolean }) - expect(attemptEvents.map(({ status, recovery }) => ({ status, recovery }))).toEqual([ - { status: 'started', recovery: false }, - { status: 'cleaned', recovery: true }, - { status: 'started', recovery: false }, - { status: 'cleaned', recovery: false }, - ]) - }) - - it('uses a retired candidate only to clean its unfinished branch', async () => { - const storage = inMemoryCampaignStorage() - const runDir = '/runs/retired-recovery-candidate' - const sequence = { - id: 'history', - family: 'first-party' as const, - steps: [ - { - id: 'remember', - scope: { agentId: 'worker' }, - writes: [{ kind: 'fact' as const, text: 'active fact' }], - probes: [{ id: 'recall', query: 'active', referenceAnswer: 'active fact' }], - }, - ], - } - storage.write( - `${runDir}/memory-attempts.jsonl`, - `${JSON.stringify({ - status: 'started', - branchId: 'retired-branch', - candidateId: 'retired', - candidateRef: 'retired:v1', - sequenceId: sequence.id, - sequenceRef: canonicalDigest(sequence), - rep: 0, - seed: 1, - cleanupBranches: true, - externalCostUsdPerSequence: 0, - externalRecoveryCostUsdPerAttempt: 0.1, - recordedAt: '2026-01-01T00:00:00.000Z', - recovery: false, - })}\n`, - ) - const costLedger = createRunCostLedger({ storage, runDir, costCeilingUsd: 1 }) - const retiredScenarioId = buildAgentMemorySequenceScenarios([sequence], [{ id: 'retired' }])[0]! - .id - await costLedger.runPaidCall({ - callId: 'retired-interrupted-execution', - channel: 'agent', - phase: 'memory.experiment', - actor: 'retired-worker', - model: 'retired-worker', - tags: { - runDir, - scenarioId: retiredScenarioId, - cellId: `${retiredScenarioId}:0`, - rep: '0', - runAttemptId: 'retired-attempt', - }, - maximumCharge: { externallyEnforcedMaximumUsd: 0.2 }, - execute: async () => undefined, - receipt: () => ({ - model: 'retired-worker', - inputTokens: 0, - outputTokens: 0, - actualCostUsd: 0.2, - }), - }) - const purposes: string[] = [] - let retiredClears = 0 - const result = await runAgentMemoryExperiment({ - experimentId: 'retired-recovery-candidate', - sequences: [sequence], - candidates: [ - { - id: 'active', - ref: 'active:v1', - createAdapter({ purpose }) { - purposes.push(`active:${purpose}`) - return createScopedTestAdapter('active') - }, - }, - ], - recoveryCandidates: [ - { - id: 'retired', - ref: 'retired:v1', - externalRecoveryCostUsdPerAttempt: 0.1, - externalCostAccounting: 'exact', - createAdapter({ purpose, recordExternalCost }) { - purposes.push(`retired:${purpose}`) - recordExternalCost(0.1) - const adapter = createScopedTestAdapter('retired') - adapter.clear = async () => { - retiredClears += 1 - } - return adapter - }, - }, - ], - runDir, - storage, - costLedger, - resumable: false, - costCeiling: 1, - }) - - expect(result.rows).toHaveLength(1) - expect(result.rows[0]?.candidateId).toBe('active') - expect(result.rows[0]?.totalCostUsd).toBe(0) - expect(result).toMatchObject({ totalCostUsd: 0.3, unrankedRecoveryCostUsd: 0.3 }) - expect(costLedger.summary().totalCostUsd).toBeCloseTo(0.3, 12) - expect(costLedger.summary().accountingComplete).toBe(true) - expect(purposes).toEqual(['retired:recovery', 'active:execute']) - expect(retiredClears).toBeGreaterThan(0) - }) - - it('refuses to hide an unfinished branch when candidate cost settings change', async () => { - const storage = inMemoryCampaignStorage() - const runDir = '/runs/changed-recovery-costs' - const sequence = { - id: 'history', - family: 'first-party' as const, - steps: [{ id: 'probe', probes: [{ id: 'recall', query: 'fact', referenceAnswer: 'fact' }] }], - } - storage.write( - `${runDir}/memory-attempts.jsonl`, - `${JSON.stringify({ - status: 'started', - branchId: 'unfinished-branch', - candidateId: 'memory', - candidateRef: 'memory:v1', - sequenceId: sequence.id, - sequenceRef: canonicalDigest(sequence), - rep: 0, - seed: 1, - cleanupBranches: true, - externalCostUsdPerSequence: 0, - externalRecoveryCostUsdPerAttempt: 0, - recordedAt: '2026-01-01T00:00:00.000Z', - recovery: false, - })}\n`, - ) - let adapterCreates = 0 - - await expect( - runAgentMemoryExperiment({ - experimentId: 'changed-recovery-costs', - sequences: [sequence], - candidates: [ - { - id: 'memory', - ref: 'memory:v1', - externalCostUsdPerSequence: 0.1, - externalCostAccounting: 'exact', - createAdapter() { - adapterCreates += 1 - return createScopedTestAdapter('memory') - }, - }, - ], - runDir, - storage, - costCeiling: 1, - }), - ).rejects.toThrow('candidate cost settings changed') - - expect(adapterCreates).toBe(0) - expect(storage.read(`${runDir}/memory-attempts.jsonl`)?.trim().split('\n')).toHaveLength(1) - }) - - it('refuses a recovery backlog larger than maxRecoveryAttempts', async () => { - const storage = inMemoryCampaignStorage() - const runDir = '/runs/recovery-backlog-limit' - const sequence = { - id: 'history', - family: 'first-party' as const, - steps: [ - { - id: 'probe', - scope: { agentId: 'worker' }, - probes: [{ id: 'recall', query: 'fact', referenceAnswer: 'fact' }], - }, - ], - } - storage.write( - `${runDir}/memory-attempts.jsonl`, - `${[ - { - status: 'started', - branchId: 'branch-1', - candidateId: 'memory', - candidateRef: 'memory:v1', - sequenceId: sequence.id, - sequenceRef: canonicalDigest(sequence), - rep: 0, - seed: 1, - cleanupBranches: true, - externalCostUsdPerSequence: 0, - externalRecoveryCostUsdPerAttempt: 0, - recordedAt: '2026-01-01T00:00:00.000Z', - recovery: false, - }, - { - status: 'started', - branchId: 'branch-2', - candidateId: 'memory', - candidateRef: 'memory:v1', - sequenceId: sequence.id, - sequenceRef: canonicalDigest(sequence), - rep: 0, - seed: 2, - cleanupBranches: true, - externalCostUsdPerSequence: 0, - externalRecoveryCostUsdPerAttempt: 0, - recordedAt: '2026-01-01T00:00:00.000Z', - recovery: false, - }, - ] - .map((event) => JSON.stringify(event)) - .join('\n')}\n`, - ) - let providerCreates = 0 - - await expect( - runAgentMemoryExperiment({ - experimentId: 'recovery-backlog-limit', - sequences: [sequence], - candidates: [ - { - id: 'memory', - ref: 'memory:v1', - createAdapter() { - providerCreates += 1 - return createScopedTestAdapter('memory') - }, - }, - ], - runDir, - storage, - maxRecoveryAttempts: 1, - }), - ).rejects.toThrow('2 unfinished attempts; maxRecoveryAttempts is 1') - expect(providerCreates).toBe(0) - }) - - it('bounds repeated provider recovery across process restarts', async () => { - const storage = inMemoryCampaignStorage() - const runDir = '/runs/recovery-retry-limit' - const sequence = { - id: 'history', - family: 'first-party' as const, - steps: [{ id: 'probe', probes: [{ id: 'recall', query: 'fact', referenceAnswer: 'fact' }] }], - } - storage.write( - `${runDir}/memory-attempts.jsonl`, - `${JSON.stringify({ - status: 'started', - branchId: 'unfinished-branch', - candidateId: 'memory', - candidateRef: 'memory:v1', - sequenceId: sequence.id, - sequenceRef: canonicalDigest(sequence), - rep: 0, - seed: 1, - cleanupBranches: true, - externalCostUsdPerSequence: 0, - externalRecoveryCostUsdPerAttempt: 0, - recordedAt: '2026-01-01T00:00:00.000Z', - recovery: false, - })}\n`, - ) - let recoveryCreates = 0 - const run = () => - runAgentMemoryExperiment({ - experimentId: 'recovery-retry-limit', - sequences: [sequence], - candidates: [ - { - id: 'memory', - ref: 'memory:v1', - createAdapter({ purpose }) { - if (purpose === 'recovery') recoveryCreates += 1 - throw new Error('provider recovery unavailable') - }, - }, - ], - runDir, - storage, - maxRecoveryRetriesPerAttempt: 2, - }) - - await expect(run()).rejects.toThrow( - "abandoned memory branch 'unfinished-branch' recovery failed", - ) - await expect(run()).rejects.toThrow( - "abandoned memory branch 'unfinished-branch' recovery failed", - ) - await expect(run()).rejects.toThrow('exhausted 2 recovery attempts') - expect(recoveryCreates).toBe(2) - expect( - storage.read(`${runDir}/memory-recovery-attempts.jsonl`)?.trim().split('\n'), - ).toHaveLength(2) - }) -}) diff --git a/tests/memory/experiment-safety.test.ts b/tests/memory/experiment-safety.test.ts deleted file mode 100644 index d19c0fe..0000000 --- a/tests/memory/experiment-safety.test.ts +++ /dev/null @@ -1,610 +0,0 @@ -import { inMemoryCampaignStorage } from '@tangle-network/agent-eval/campaign' -import { describe, expect, it } from 'vitest' -import type { AgentMemoryAdapter } from '../../src/memory/index' -import { createScopedTestAdapter, hitText, runAgentMemoryExperiment } from '../support/memory' - -describe('agent memory experiment safety', () => { - it('allows one controller per run while its history workers run in parallel', async () => { - const storage = inMemoryCampaignStorage() - let reportStarted: (() => void) | undefined - const started = new Promise((resolve) => { - reportStarted = resolve - }) - let releaseWorker: (() => void) | undefined - const continueWorker = new Promise((resolve) => { - releaseWorker = resolve - }) - const options: RunAgentMemoryExperimentOptions = { - experimentId: 'single-controller', - sequences: [ - { - id: 'one', - family: 'first-party', - steps: [ - { - id: 'work', - scope: { agentId: 'worker' }, - writes: [{ kind: 'fact', text: 'parallel worker fact' }], - probes: [ - { - id: 'recall', - query: 'worker fact', - referenceAnswer: 'parallel worker fact', - }, - ], - }, - ], - }, - ], - candidates: [ - { - id: 'memory', - ref: 'memory:v1', - createAdapter: () => createScopedTestAdapter('single-controller'), - }, - ], - runDir: '/runs/single-controller', - storage, - executeStepRef: 'blocking-step:v1', - async executeStep() { - reportStarted?.() - await continueWorker - }, - } - - const firstRun = runAgentMemoryExperiment(options) - await started - await expect( - runAgentMemoryExperiment({ ...options, storage: inMemoryCampaignStorage() }), - ).rejects.toThrow('active controller') - releaseWorker?.() - const result = await firstRun - - expect(result.rows[0]).toMatchObject({ candidateId: 'memory', cellsFailed: 0 }) - }) - - it('leaves a lost controller branch for the next owner and accepts duplicate cleanup receipts', async () => { - const storage = inMemoryCampaignStorage() - let firstControllerOwned = true - let expireFirstController = true - let controllerCount = 0 - let searches = 0 - let clears = 0 - let closes = 0 - let factVisible = false - const purposes: string[] = [] - const options: RunAgentMemoryExperimentOptions = { - experimentId: 'lost-controller', - sequences: [ - { - id: 'one', - family: 'first-party', - steps: [ - { - id: 'work', - scope: { agentId: 'worker' }, - writes: [{ kind: 'fact', text: 'must be cleaned' }], - probes: [{ id: 'probe', query: 'must', referenceAnswer: 'must be cleaned' }], - }, - ], - }, - ], - candidates: [ - { - id: 'memory', - ref: 'memory:v1', - createAdapter({ purpose }) { - purposes.push(purpose) - const adapter: AgentMemoryAdapter = { - id: 'lost-controller', - branchIsolation: { mode: 'scoped' }, - async search() { - searches += 1 - return factVisible - ? [ - { - id: 'write', - uri: 'memory://lost-controller/write', - kind: 'fact', - text: 'must be cleaned', - }, - ] - : [] - }, - async getContext(query, searchOptions) { - const hits = await adapter.search(query, searchOptions) - return { query, text: hits.map(hitText).join('\n'), hits, sourceRecords: [] } - }, - async write(input) { - factVisible = true - return { - accepted: true, - id: 'write', - uri: 'memory://lost-controller/write', - kind: input.kind, - } - }, - async clear() { - clears += 1 - factVisible = false - }, - async close() { - closes += 1 - }, - } - return adapter - }, - }, - ], - runDir: '/runs/lost-controller', - storage, - acquireRunLease: async () => { - controllerCount += 1 - const controller = controllerCount - return { - assertOwned() { - if (controller === 1 && !firstControllerOwned) { - throw new Error('controller ownership expired') - } - }, - release() {}, - } - }, - executeStepRef: 'expire-controller:v1', - async executeStep() { - if (expireFirstController) { - expireFirstController = false - firstControllerOwned = false - } - }, - } - const run = () => runAgentMemoryExperiment(options) - - await expect(run()).rejects.toThrow('controller ownership expired') - expect(searches).toBe(0) - expect(clears).toBe(0) - expect(closes).toBe(1) - expect(factVisible).toBe(true) - - const result = await run() - expect(result.rows[0]).toMatchObject({ candidateId: 'memory', cellsFailed: 0 }) - expect(purposes).toEqual(['execute', 'recovery', 'execute']) - expect(clears).toBe(2) - expect(closes).toBe(3) - expect(factVisible).toBe(false) - - const journal = storage.read(result.attemptLogPath)! - const lines = journal.trim().split('\n') - const last = lines.at(-1)! - expect(storage.append!(result.attemptLogPath, `${last}\n`, Buffer.byteLength(journal))).toBe( - Buffer.byteLength(journal) + Buffer.byteLength(`${last}\n`), - ) - const callsBeforeCachedRun = { - purposes: purposes.length, - searches, - clears, - closes, - } - const cached = await run() - expect(cached.campaign.aggregates.cellsCached).toBe(1) - expect({ purposes: purposes.length, searches, clears, closes }).toEqual(callsBeforeCachedRun) - }) - - it('does not mark an attempt clean after ownership expires during adapter close', async () => { - const storage = inMemoryCampaignStorage() - const runDir = '/runs/ownership-lost-during-close' - let owned = true - const adapter = createScopedTestAdapter('ownership-lost-during-close') - adapter.close = async () => { - owned = false - } - - await expect( - runAgentMemoryExperiment({ - experimentId: 'ownership-lost-during-close', - sequences: [ - { - id: 'history', - family: 'first-party', - steps: [ - { - id: 'probe', - scope: { agentId: 'worker' }, - probes: [{ id: 'recall', query: 'fact', referenceAnswer: 'fact' }], - }, - ], - }, - ], - candidates: [ - { - id: 'memory', - ref: 'memory:v1', - createAdapter: () => adapter, - }, - ], - runDir, - storage, - acquireRunLease: async () => ({ - assertOwned() { - if (!owned) throw new Error('controller ownership expired during close') - }, - release() {}, - }), - }), - ).rejects.toThrow('controller ownership expired during close') - - const attempts = storage.read(`${runDir}/memory-attempts.jsonl`)!.trim().split('\n') - expect(attempts).toHaveLength(1) - expect(JSON.parse(attempts[0]!)).toMatchObject({ status: 'started' }) - }) - - it('preserves both the run failure and controller release failure', async () => { - const error = await runAgentMemoryExperiment({ - experimentId: 'run-and-release-failure', - sequences: [ - { - id: 'one', - family: 'first-party', - steps: [ - { - id: 'probe', - probes: [{ id: 'probe', query: 'state', referenceAnswer: 'state' }], - }, - ], - }, - ], - candidates: [ - { - id: 'unused', - ref: 'unused:v1', - createAdapter: () => createScopedTestAdapter('unused'), - }, - ], - runDir: '/runs/run-and-release-failure', - storage: inMemoryCampaignStorage(), - acquireRunLease: async () => ({ - assertOwned() { - throw new Error('run ownership failed') - }, - release() { - throw new Error('release failed') - }, - }), - }).catch((caught: unknown) => caught) - - expect(error).toBeInstanceOf(AggregateError) - expect((error as AggregateError).errors.map((item) => String(item))).toEqual([ - 'Error: run ownership failed', - 'Error: release failed', - ]) - }) - - it('fails the experiment when accepted writes cannot be cleared after a failed step', async () => { - let clears = 0 - let closes = 0 - let disposals = 0 - const run = runAgentMemoryExperiment({ - experimentId: 'failed-step-cleanup', - sequences: [ - { - id: 'failure', - family: 'first-party', - steps: [ - { - id: 'write-then-fail', - scope: { agentId: 'worker' }, - writes: [{ kind: 'fact', text: 'partial state' }], - probes: [{ id: 'state', query: 'partial state', referenceAnswer: 'partial state' }], - }, - ], - }, - ], - candidates: [ - { - id: 'scoped', - ref: 'scoped:v1', - createAdapter() { - const base = createScopedTestAdapter('failure') - return { - ...base, - async clear(scope) { - clears += 1 - await base.clear?.(scope) - throw new Error('provider clear failed') - }, - async close() { - closes += 1 - }, - } - }, - async disposeAdapter() { - disposals += 1 - }, - }, - ], - runDir: '/runs/failed-step-cleanup', - storage: inMemoryCampaignStorage(), - cleanupBranches: true, - executeStepRef: 'failing-worker/v1', - async executeStep() { - throw new Error('worker failed') - }, - }) - - await expect(run).rejects.toThrow('memory experiment cleanup failed after dispatch') - expect(clears).toBe(1) - expect(closes).toBe(1) - expect(disposals).toBe(1) - }) - - it('cleans an unjournaled provider side effect and ranks failed histories below complete ones', async () => { - let dirtyRows = 0 - let clears = 0 - const result = await runAgentMemoryExperiment({ - experimentId: 'provider-side-effect-cleanup', - sequences: [ - { - id: 'one', - family: 'first-party', - steps: [ - { - id: 'remember', - scope: { agentId: 'worker' }, - writes: [{ kind: 'fact', text: 'Launch is Friday.' }], - probes: [{ id: 'launch', query: 'launch', referenceAnswer: 'Launch is Friday.' }], - }, - ], - }, - ], - candidates: [ - { - id: 'side-effect-then-error', - ref: 'side-effect-then-error:v1', - createAdapter: () => ({ - id: 'side-effect-then-error', - branchIsolation: { mode: 'scoped' }, - async search() { - return [] - }, - async getContext(query) { - return { query, text: '', hits: [], sourceRecords: [] } - }, - async write() { - dirtyRows += 1 - throw new Error('provider disconnected after commit') - }, - async clear() { - clears += 1 - dirtyRows = 0 - }, - }), - }, - { - id: 'complete', - ref: 'complete:v1', - createAdapter: () => createScopedTestAdapter('complete'), - }, - ], - runDir: '/runs/provider-side-effect-cleanup', - storage: inMemoryCampaignStorage(), - cleanupBranches: true, - maxConcurrency: 2, - }) - - expect(dirtyRows).toBe(0) - expect(clears).toBe(1) - expect(result.rows[0]).toMatchObject({ - candidateId: 'complete', - cellsFailed: 0, - scoreMean: 1, - }) - expect(result.rows[1]).toMatchObject({ - candidateId: 'side-effect-then-error', - cellsFailed: 1, - scoreMean: 0, - passRate: 0, - }) - }) - - it('closes and disposes an adapter when branch validation fails', async () => { - let closes = 0 - let disposals = 0 - const result = await runAgentMemoryExperiment({ - experimentId: 'invalid-branch-cleanup', - sequences: [ - { - id: 'one', - family: 'first-party', - steps: [ - { - id: 'probe', - scope: { agentId: 'worker' }, - probes: [{ id: 'state', query: 'state', referenceAnswer: 'state' }], - }, - ], - }, - ], - candidates: [ - { - id: 'legacy', - ref: 'legacy:v1', - createAdapter() { - const { branchIsolation: _branchIsolation, ...legacy } = - createScopedTestAdapter('legacy') - return { - ...legacy, - async close() { - closes += 1 - }, - } - }, - async disposeAdapter() { - disposals += 1 - }, - }, - ], - runDir: '/runs/invalid-branch-cleanup', - storage: inMemoryCampaignStorage(), - cleanupBranches: false, - }) - - expect(result.rows[0]).toMatchObject({ cellsFailed: 1 }) - expect(result.campaign.cells[0]?.error).toContain('adapter must declare branchIsolation') - expect(closes).toBe(1) - expect(disposals).toBe(1) - }) - - it('fails closed when cleanup is requested for an adapter without scoped clear', async () => { - let disposals = 0 - const result = await runAgentMemoryExperiment({ - experimentId: 'unsupported-cleanup', - sequences: [ - { - id: 'one', - family: 'first-party', - steps: [ - { - id: 'probe', - scope: { agentId: 'worker' }, - probes: [{ id: 'state', query: 'state', referenceAnswer: 'state' }], - }, - ], - }, - ], - candidates: [ - { - id: 'no-clear', - ref: 'no-clear:v1', - createAdapter() { - const { clear: _clear, ...adapter } = createScopedTestAdapter('no-clear') - return adapter - }, - async disposeAdapter() { - disposals += 1 - }, - }, - ], - runDir: '/runs/unsupported-cleanup', - storage: inMemoryCampaignStorage(), - cleanupBranches: true, - }) - - expect(result.rows[0]).toMatchObject({ cellsFailed: 1 }) - expect(result.campaign.cells[0]?.error).toContain('requires an adapter with scoped clear') - expect(disposals).toBe(1) - }) - - it('rejects dynamic writes whose scope cannot be cleaned after a crash', async () => { - let providerWrites = 0 - const result = await runAgentMemoryExperiment({ - experimentId: 'undeclared-dynamic-scope', - sequences: [ - { - id: 'one', - family: 'first-party', - steps: [ - { - id: 'agent-step', - scope: { agentId: 'declared' }, - probes: [{ id: 'state', query: 'state', referenceAnswer: 'state' }], - }, - ], - }, - ], - candidates: [ - { - id: 'scoped', - ref: 'scoped:v1', - createAdapter: () => - createScopedTestAdapter('scoped', async () => { - providerWrites += 1 - }), - }, - ], - runDir: '/runs/undeclared-dynamic-scope', - storage: inMemoryCampaignStorage(), - cleanupBranches: true, - executeStepRef: 'dynamic-scope-test/v1', - executeStep: async ({ memory }) => { - await memory.write({ - kind: 'fact', - text: 'must not reach the provider', - scope: { agentId: 'undeclared' }, - }) - }, - }) - - expect(result.rows[0]).toMatchObject({ cellsFailed: 1 }) - expect(result.campaign.cells[0]?.error).toContain( - 'write scope was not declared in the experiment sequence cleanupScopes', - ) - expect(providerWrites).toBe(0) - }) - - it('requires an external-state disposer when scoped cleanup is disabled', async () => { - let creates = 0 - await expect( - runAgentMemoryExperiment({ - experimentId: 'missing-disposer', - sequences: [ - { - id: 'one', - family: 'first-party', - steps: [ - { - id: 'probe', - scope: { agentId: 'worker' }, - probes: [{ id: 'state', query: 'state', referenceAnswer: 'state' }], - }, - ], - }, - ], - candidates: [ - { - id: 'persistent', - ref: 'persistent:v1', - createAdapter() { - creates += 1 - return createScopedTestAdapter('persistent') - }, - }, - ], - runDir: '/runs/missing-disposer', - storage: inMemoryCampaignStorage(), - cleanupBranches: false, - }), - ).rejects.toThrow('requires disposeAdapter') - - expect(creates).toBe(0) - }) - - it('rejects probes that cannot distinguish a useful memory system', async () => { - await expect( - runAgentMemoryExperiment({ - experimentId: 'no-target', - sequences: [ - { - id: 'one', - family: 'first-party', - steps: [ - { - id: 'probe', - scope: { agentId: 'worker' }, - probes: [{ id: 'state', query: 'state' }], - }, - ], - }, - ], - candidates: [ - { - id: 'memory', - ref: 'memory:v1', - createAdapter: () => createScopedTestAdapter('memory'), - }, - ], - runDir: '/runs/no-target', - storage: inMemoryCampaignStorage(), - }), - ).rejects.toThrow('has no measurable target') - }) -}) diff --git a/tests/memory/graphiti.test.ts b/tests/memory/graphiti.test.ts index e934a89..bb6849f 100644 --- a/tests/memory/graphiti.test.ts +++ b/tests/memory/graphiti.test.ts @@ -1,12 +1,10 @@ import { describe, expect, it } from 'vitest' -import { buildCandidate } from '../../src/memory/improvement/candidate' import { type AgentMemoryScope, createAgentMemoryBranch, createGraphitiMemoryAdapter, type GraphitiMcpClientLike, graphitiMemoryAdapterIdentity, - type RunAgentMemoryImprovementOptions, } from '../../src/memory/index' describe('Graphiti adapter', () => { @@ -455,26 +453,4 @@ describe('Graphiti adapter', () => { }), ) }) - - it('uses its identity as a memory improvement candidate ref', async () => { - const config = { provider: 'graphiti' } as const - const ref = graphitiMemoryAdapterIdentity({ - id: 'graphiti', - backendRef: 'graphiti-cluster-a', - }) - const options = { - createCandidate: () => ({ - ref, - externalCostUsdPerSequence: 0, - externalRecoveryCostUsdPerAttempt: 0, - externalCostAccounting: 'exact' as const, - createAdapter: () => null, - }), - } as RunAgentMemoryImprovementOptions - - const candidate = await buildCandidate(options, config, 'graphiti') - - expect(candidate.ref).toBe(ref) - expect(candidate.ref).toMatch(/^sha256:[a-f0-9]{64}$/) - }) }) diff --git a/tests/memory/improvement.test.ts b/tests/memory/improvement.test.ts deleted file mode 100644 index 5d0ae4f..0000000 --- a/tests/memory/improvement.test.ts +++ /dev/null @@ -1,624 +0,0 @@ -import { createHash } from 'node:crypto' -import { canonicalJson } from '@tangle-network/agent-eval' -import { - campaignMeanComposite, - inMemoryCampaignStorage, - type JsonValue, - type OptimizationMethod, - runCampaign, - surfaceHash, -} from '@tangle-network/agent-eval/campaign' -import { describe, expect, it } from 'vitest' -import { stableId } from '../../src/ids' -import { buildCandidate } from '../../src/memory/improvement/candidate' -import { - type AgentMemorySequence, - type AgentMemorySequenceArtifact, - type MemoryConfigScenario, - type RunAgentMemoryImprovementOptions, - runAgentMemoryImprovement as runAgentMemoryImprovementRaw, -} from '../../src/memory/index' -import { createScopedTestAdapter, runAgentMemoryImprovement } from '../support/memory' - -type Config = { visibility: 'private' | 'team' } - -// Tied binary safety outcomes need enough evidence to clear the 0.05 regression margin. -const FINAL_SEQUENCE_IDS = Array.from({ length: 14 }, (_, batch) => - ['a', 'b', 'c', 'd', 'e', 'f'].map((suffix) => - batch === 0 ? `final-${suffix}` : `final-${suffix}-${batch}`, - ), -).flat() - -function immutableRef(value: string): string { - return `sha256:${createHash('sha256').update(value).digest('hex')}` -} - -describe('agent memory improvement', () => { - it('holds activation when tied safety outcomes cannot exclude the allowed regression', async () => { - let activationCalls = 0 - let activeConfig: Config = { visibility: 'private' } - const result = await runAgentMemoryImprovement( - baseOptions({ - finalSequences: Array.from({ length: 20 }, (_, index) => - improvementSequence(`safety-final-${index}`, 'test', index % 2), - ), - criticalDimensions: ['memory_stale_safe'], - criticalDimensionTolerance: 0.05, - activation: { - ref: immutableRef('safety-policy'), - async readCurrent() { - return activeConfig - }, - async compareAndSet({ config }) { - activationCalls += 1 - activeConfig = config - }, - }, - }), - ) - - expect(result.decision.significance?.significant).toBe(true) - expect(result.decision.status).toBe('hold') - expect(result.decision.criticalDimensions[0]?.low).toBeLessThan(-0.05) - expect(result.decision.criticalDimensions[0]?.high).toBeGreaterThan(0.05) - expect(result.decision.criticalDimensions[0]?.regressed).toBe(false) - expect(result.decision.reasons).toContain( - 'critical dimension memory_stale_safe does not exclude a regression beyond 0.05', - ) - expect(result.activation.status).toBe('not-eligible') - expect(activationCalls).toBe(0) - expect(activeConfig).toEqual({ visibility: 'private' }) - }) - - it('runs a complete method, keeps final data private, resumes, and activates once', async () => { - const storage = inMemoryCampaignStorage() - const methodInputs: string[][] = [] - let candidateConstructions = 0 - let activeConfig: Config = { visibility: 'private' } - const activationIds: string[] = [] - const method = selectingMethod( - [{ visibility: 'private' }, { visibility: 'team' }], - methodInputs, - ) - const options: RunAgentMemoryImprovementOptions = { - experimentId: 'complete-method-memory', - baselineConfig: { visibility: 'private' }, - method, - trainSequences: [improvementSequence('train-a', 'train')], - selectionSequences: [improvementSequence('selection-a', 'validation')], - finalSequences: finalImprovementSequences(), - implementationRef: immutableRef('team-memory-policy'), - runDir: '/runs/complete-method-memory', - storage, - controllerMode: 'process-local', - sequenceConcurrency: 4, - significance: { minProductiveRuns: 6, resamples: 200, seed: 7 }, - createCandidate: ({ config, candidateId }) => { - candidateConstructions += 1 - return { - ref: immutableRef(`visibility/${config.visibility}`), - policy: { read: [config.visibility], write: config.visibility }, - externalCostUsdPerSequence: 0, - externalRecoveryCostUsdPerAttempt: 0, - externalCostAccounting: 'exact', - createAdapter: ({ branchId }) => createScopedTestAdapter(`${candidateId}:${branchId}`), - } - }, - activation: { - ref: immutableRef('memory-policy'), - async readCurrent() { - return structuredClone(activeConfig) - }, - async compareAndSet({ activationId, expectedConfig, config, optimization }) { - expect(activeConfig).toEqual(expectedConfig) - expect(optimization.winner.surfaceHash).toBe(surfaceHash(canonicalJson(config))) - activationIds.push(activationId) - activeConfig = structuredClone(config) - }, - }, - } - - const result = await runAgentMemoryImprovement(options) - - expect(methodInputs).toEqual([['train-a', 'selection-a']]) - expect(result.winnerConfig).toEqual({ visibility: 'team' }) - expect(result.winnerSurface).toBe('{"visibility":"team"}') - expect(result.finalEvaluation.pairs).toHaveLength(FINAL_SEQUENCE_IDS.length) - expect(result.finalEvaluation.pairs.map((pair) => pair.sequenceId)).toEqual(FINAL_SEQUENCE_IDS) - expect(result.decision).toMatchObject({ status: 'promote', winnerScore: 1 }) - expect(result.decision.baselineScore).toBeCloseTo(61 / 96) - expect(result.decision.lift).toBeCloseTo(35 / 96) - expect(result.activation.status).toBe('activated') - expect(activeConfig).toEqual({ visibility: 'team' }) - expect(activationIds).toEqual([result.activation.id]) - const identity = JSON.parse( - storage.read('/runs/complete-method-memory/memory-improvement-manifest.json') ?? '{}', - ).identity - expect(identity).toMatchObject({ - experimentId: 'complete-method-memory', - implementationRef: immutableRef('team-memory-policy'), - method: 'fixture-selection', - }) - expect(identity).not.toHaveProperty('schema') - expect(identity).not.toHaveProperty('builtInFunctionHash') - expect(identity).not.toHaveProperty('callbackFunctionHash') - expect(result.finalEvaluation).toMatchObject({ - baselineCandidateRef: immutableRef('visibility/private'), - winnerCandidateRef: immutableRef('visibility/team'), - }) - expect( - storage.read( - `/runs/complete-method-memory/memory-config-artifacts/${result.winnerSurfaceHash}/${stableId('sequence', 'final-a')}/rep-0-${stableId('seed', '42')}.json`, - ), - ).toContain(`"candidateRef": "${immutableRef('visibility/team')}"`) - expect( - storage.read( - `/runs/complete-method-memory/memory-final-artifacts/${result.winnerSurfaceHash}/${stableId('sequence', 'final-a')}/rep-0.json`, - ), - ).toContain('"seed": 42') - - const constructionsAfterFirstRun = candidateConstructions - const resumed = await runAgentMemoryImprovement(options) - - expect(candidateConstructions).toBe(constructionsAfterFirstRun + 2) - expect(activationIds).toEqual([result.activation.id]) - expect(resumed.activation.status).toBe('already-activated') - expect(resumed.winnerSurfaceHash).toBe(result.winnerSurfaceHash) - expect(resumed.finalEvaluation.manifestHash).toBe(result.finalEvaluation.manifestHash) - }) - - it('rejects an activated journal when the live memory configuration drifted', async () => { - const storage = inMemoryCampaignStorage() - let activeConfig: Config = { visibility: 'private' } - const options = baseOptions({ - experimentId: 'activation-drift', - runDir: '/runs/activation-drift', - storage, - method: selectingMethod([{ visibility: 'private' }, { visibility: 'team' }]), - activation: { - ref: immutableRef('activation-drift-target'), - async readCurrent() { - return structuredClone(activeConfig) - }, - async compareAndSet({ expectedConfig, config }) { - expect(activeConfig).toEqual(expectedConfig) - activeConfig = structuredClone(config) - }, - }, - }) - - const activated = await runAgentMemoryImprovement(options) - expect(activated.activation.status).toBe('activated') - activeConfig = { visibility: 'private' } - - await expect(runAgentMemoryImprovement(options)).rejects.toThrow( - "memory activation target '" + - immutableRef('activation-drift-target') + - "' drifted from measured winner", - ) - }) - - it('shares one paid evaluation while identical concurrent cells await candidate construction', async () => { - let teamAdapterCreations = 0 - let candidateBuildStarted!: () => void - const buildStarted = new Promise((resolve) => { - candidateBuildStarted = resolve - }) - let releaseCandidateBuild!: () => void - const candidateBuildReleased = new Promise((resolve) => { - releaseCandidateBuild = resolve - }) - const team: Config = { visibility: 'team' } - const method: OptimizationMethod = { - name: 'concurrent-duplicate-dispatch', - async optimize(input) { - const surface = canonicalJson(team) - const dispatch = (suffix: string) => - runCampaign({ - ...input.runOptions, - scenarios: [input.trainScenarios[0]!], - dispatch: (scenario, context) => input.dispatchWithSurface(surface, scenario, context), - judges: [...input.judges], - runDir: `${input.runDir}/duplicate/${suffix}`, - seed: input.seed, - }) - const duplicateRuns = Promise.all([dispatch('a'), dispatch('b')]) - await buildStarted - await new Promise((resolve) => setTimeout(resolve, 20)) - releaseCandidateBuild() - await duplicateRuns - return { - winnerSurface: surface, - cost: { - totalCostUsd: 0, - costProvenance: { kind: 'observed', usd: 0 }, - accountingComplete: true, - incompleteReasons: [], - }, - } - }, - } - const result = await runAgentMemoryImprovement( - baseOptions({ - experimentId: 'deduplicate-concurrent-evaluation', - runDir: '/runs/deduplicate-concurrent-evaluation', - method, - async createCandidate({ config, candidateId }) { - if (config.visibility === 'team') { - candidateBuildStarted() - await candidateBuildReleased - } - return { - ref: immutableRef(`visibility/${config.visibility}`), - policy: { read: [config.visibility], write: config.visibility }, - externalCostUsdPerSequence: 0, - externalRecoveryCostUsdPerAttempt: 0, - externalCostAccounting: 'exact', - createAdapter: ({ branchId }) => { - if (config.visibility === 'team') teamAdapterCreations += 1 - return createScopedTestAdapter(`${candidateId}:${branchId}`) - }, - } - }, - }), - ) - - expect(result.winnerConfig).toEqual(team) - expect(teamAdapterCreations).toBe(FINAL_SEQUENCE_IDS.length + 1) - }) - - it('rejects a changed candidate identity before reusing a resumed result', async () => { - const storage = inMemoryCampaignStorage() - let candidateRevision = 'v1' - const options = baseOptions({ - experimentId: 'changed-candidate-identity', - runDir: '/runs/changed-candidate-identity', - storage, - createCandidate: ({ config, candidateId }) => ({ - ref: immutableRef(`visibility/${config.visibility}/${candidateRevision}`), - policy: { read: [config.visibility], write: config.visibility }, - externalCostUsdPerSequence: 0, - externalRecoveryCostUsdPerAttempt: 0, - externalCostAccounting: 'exact', - createAdapter: ({ branchId }) => createScopedTestAdapter(`${candidateId}:${branchId}`), - }), - }) - - await runAgentMemoryImprovement(options) - candidateRevision = 'v2' - - await expect(runAgentMemoryImprovement(options)).rejects.toThrow( - 'changed candidate identity within the same improvement run', - ) - }) - - it('recovers an applied activation whose final journal write was interrupted', async () => { - const storage = inMemoryCampaignStorage() - const append = storage.append!.bind(storage) - let rejectActivatedEvent = true - storage.append = (path, value, expectedBytes) => { - if ( - rejectActivatedEvent && - path.includes('/activations/') && - value.includes('"status":"activated"') - ) { - rejectActivatedEvent = false - throw new Error('activation journal unavailable') - } - return append(path, value, expectedBytes) - } - let activeConfig: Config = { visibility: 'private' } - let compareAndSetCalls = 0 - const options = baseOptions({ - experimentId: 'recover-memory-activation', - runDir: '/runs/recover-memory-activation', - storage, - method: selectingMethod([{ visibility: 'private' }, { visibility: 'team' }]), - activation: { - ref: immutableRef('memory-policy'), - async readCurrent() { - return structuredClone(activeConfig) - }, - async compareAndSet({ expectedConfig, config }) { - expect(activeConfig).toEqual(expectedConfig) - compareAndSetCalls += 1 - activeConfig = structuredClone(config) - }, - }, - }) - - await expect(runAgentMemoryImprovement(options)).rejects.toThrow( - 'activation journal unavailable', - ) - expect(activeConfig).toEqual({ visibility: 'team' }) - expect(compareAndSetCalls).toBe(1) - - const recovered = await runAgentMemoryImprovement(options) - expect(recovered.activation.status).toBe('recovered') - expect(compareAndSetCalls).toBe(1) - - const resumed = await runAgentMemoryImprovement(options) - expect(resumed.activation.status).toBe('already-activated') - expect(compareAndSetCalls).toBe(1) - }) - - it('rejects a resumed final artifact whose embedded candidate identity changed', async () => { - const storage = inMemoryCampaignStorage() - const options = baseOptions({ - experimentId: 'corrupt-memory-artifact', - runDir: '/runs/corrupt-memory-artifact', - storage, - }) - const first = await runAgentMemoryImprovement(options) - const artifactPath = - `/runs/corrupt-memory-artifact/memory-final-artifacts/${first.winnerSurfaceHash}` + - `/${stableId('sequence', 'final-a')}/rep-0.json` - const record = JSON.parse(storage.read(artifactPath)!) - record.artifact.candidateId = 'memory-config-from-another-surface' - storage.write(artifactPath, `${JSON.stringify(record, null, 2)}\n`) - - await expect(runAgentMemoryImprovement(options)).rejects.toThrow( - `memory config artifact '${artifactPath}' is malformed`, - ) - }) - - it('rejects copied data across train, selection, and final partitions', async () => { - const train = improvementSequence('train-original', 'train') - const options = baseOptions({ - experimentId: 'copied-memory-data', - runDir: '/runs/copied-memory-data', - trainSequences: [train], - finalSequences: [ - { ...train, id: 'renamed-final', split: 'test' }, - improvementSequence('final-b', 'test'), - ], - }) - - await expect(runAgentMemoryImprovement(options)).rejects.toThrow( - "train/final histories duplicate content at 'train-original'/'renamed-final'", - ) - }) - - it('requires an explicit controller policy for custom storage', async () => { - const options = baseOptions({ - experimentId: 'custom-memory-storage', - runDir: '/runs/custom-memory-storage', - storage: inMemoryCampaignStorage(), - }) - delete (options as { controllerMode?: string }).controllerMode - - await expect(runAgentMemoryImprovementRaw(options)).rejects.toThrow( - "requires acquireRunLease or controllerMode='process-local'", - ) - }) - - it('requires a per-evaluation maximum before enabling paid work', async () => { - await expect( - runAgentMemoryImprovement( - baseOptions({ - experimentId: 'missing-evaluation-maximum', - runDir: '/runs/missing-evaluation-maximum', - maxTotalCostUsd: 1, - }), - ), - ).rejects.toThrow('maximumEvaluationCostUsd is required when a spend limit is configured') - }) - - it('rejects memory candidates without explicit provider cost declarations', async () => { - const options = baseOptions({ - experimentId: 'missing-provider-costs', - runDir: '/runs/missing-provider-costs', - }) - options.createCandidate = (({ - config, - candidateId, - }: Parameters[0]) => ({ - ref: immutableRef(`visibility/${config.visibility}`), - policy: { read: [config.visibility], write: config.visibility }, - createAdapter: ({ branchId }: { branchId: string }) => - createScopedTestAdapter(`${candidateId}:${branchId}`), - })) as typeof options.createCandidate - - await expect(buildCandidate(options, options.baselineConfig, 'missing-costs')).rejects.toThrow( - 'externalCostUsdPerSequence must be a declared non-negative finite number', - ) - }) - - it('holds activation when the method cannot fully account for optimization cost', async () => { - let activationCalls = 0 - const result = await runAgentMemoryImprovement( - baseOptions({ - experimentId: 'incomplete-method-cost', - runDir: '/runs/incomplete-method-cost', - method: selectingMethod([{ visibility: 'private' }, { visibility: 'team' }], undefined, { - totalCostUsd: 0, - costProvenance: { kind: 'uncaptured', usd: null }, - accountingComplete: false, - incompleteReasons: ['external optimizer usage unavailable'], - }), - activation: { - ref: immutableRef('memory-policy'), - async readCurrent() { - return { visibility: 'private' } - }, - async compareAndSet() { - activationCalls += 1 - }, - }, - }), - ) - - expect(result.winnerConfig).toEqual({ visibility: 'team' }) - expect(result.decision.status).toBe('hold') - expect(result.decision.reasons).toContain('optimization or final cost accounting is incomplete') - expect(result.activation.status).toBe('not-eligible') - expect(activationCalls).toBe(0) - }) - - it('binds independent source units before resuming a memory improvement', async () => { - const finalSequences = finalImprovementSequences().slice(0, 6) - const independentUnitByScenarioId = new Map( - finalSequences.map(({ id }, index) => [id, `source-${index}`]), - ) - const method = selectingMethod([{ visibility: 'private' }, { visibility: 'team' }]) - const options = baseOptions({ - finalSequences, - reps: 2, - significance: { resamples: 200, seed: 7, independentUnitByScenarioId }, - method: { - name: method.name, - async optimize(input) { - independentUnitByScenarioId.set(finalSequences[0]!.id, 'source-1') - return method.optimize(input) - }, - }, - }) - const result = await runAgentMemoryImprovement(options) - expect(result.decision.significance).toMatchObject({ - n: 6, - pairedCellN: 12, - observationUnit: 'registered', - }) - expect(result.decision.criticalDimensions[0]).toMatchObject({ - n: 6, - expectedN: 12, - measured: true, - }) - - independentUnitByScenarioId.set(FINAL_SEQUENCE_IDS[0]!, 'changed-source') - await expect(runAgentMemoryImprovement(options).then(() => undefined)).rejects.toThrow( - 'does not match its persisted inputs or implementation', - ) - }) -}) - -function baseOptions( - overrides: Partial> = {}, -): RunAgentMemoryImprovementOptions { - return { - experimentId: 'memory-improvement', - baselineConfig: { visibility: 'private' }, - method: selectingMethod([{ visibility: 'private' }, { visibility: 'team' }]), - trainSequences: [improvementSequence('train-a', 'train')], - selectionSequences: [improvementSequence('selection-a', 'validation')], - finalSequences: finalImprovementSequences(), - createCandidate: ({ config, candidateId }) => ({ - ref: immutableRef(`visibility/${config.visibility}`), - policy: { read: [config.visibility], write: config.visibility }, - externalCostUsdPerSequence: 0, - externalRecoveryCostUsdPerAttempt: 0, - externalCostAccounting: 'exact', - createAdapter: ({ branchId }) => createScopedTestAdapter(`${candidateId}:${branchId}`), - }), - implementationRef: immutableRef('memory-improvement'), - runDir: '/runs/memory-improvement', - storage: inMemoryCampaignStorage(), - controllerMode: 'process-local', - significance: { minProductiveRuns: 6, resamples: 200, seed: 7 }, - ...overrides, - } -} - -function selectingMethod( - configs: readonly TConfig[], - inputs?: string[][], - cost = { - totalCostUsd: 0, - costProvenance: { kind: 'observed' as const, usd: 0 }, - accountingComplete: true, - incompleteReasons: [], - }, -): OptimizationMethod { - return { - name: 'fixture-selection', - async optimize(input) { - inputs?.push([ - ...input.trainScenarios.map((scenario) => scenario.id), - ...input.selectionScenarios.map((scenario) => scenario.id), - ]) - expect('testScenarios' in input).toBe(false) - const scored = await Promise.all( - configs.map(async (config) => { - const surface = canonicalJson(config) - await runCampaign({ - ...input.runOptions, - scenarios: [...input.trainScenarios], - dispatch: (scenario, context) => input.dispatchWithSurface(surface, scenario, context), - judges: [...input.judges], - runDir: `${input.runDir}/fixture/${surfaceHash(surface)}/train`, - seed: input.seed, - }) - const selection = await runCampaign({ - ...input.runOptions, - scenarios: [...input.selectionScenarios], - dispatch: (scenario, context) => input.dispatchWithSurface(surface, scenario, context), - judges: [...input.judges], - runDir: `${input.runDir}/fixture/${surfaceHash(surface)}/selection`, - seed: input.seed, - }) - return { surface, score: campaignMeanComposite(selection) } - }), - ) - scored.sort((left, right) => right.score - left.score) - return { - winnerSurface: scored[0]!.surface, - cost, - } - }, - } -} - -function improvementSequence( - id: string, - split: 'train' | 'validation' | 'test', - sameAgentProbeCount = 0, -): AgentMemorySequence { - const launchDateProbe = (suffix: string) => ({ - id: `launch-date-${suffix}`, - query: `${id} launch date`, - requiredFacts: [{ id: `current-${suffix}`, anyOf: [`${id} launch date is Friday`] }], - forbiddenFacts: [ - { - id: `stale-${suffix}`, - anyOf: [`${id} launch date is Thursday`], - obsolete: true, - }, - ], - expectedEventIds: [`${id}-event`], - expectedActorIds: ['researcher'], - }) - return { - id, - family: 'first-party', - split, - steps: [ - { - id: 'research', - scope: { agentId: 'researcher', teamId: 'team-1' }, - writes: [ - { - id: `${id}-event`, - kind: 'fact', - text: `${id} launch date is Friday`, - metadata: { eventId: `${id}-event`, actorId: 'researcher' }, - }, - ], - probes: Array.from({ length: sameAgentProbeCount }, (_, index) => - launchDateProbe(`research-${index + 1}`), - ), - }, - { - id: 'delivery', - scope: { agentId: 'builder', teamId: 'team-1' }, - probes: [launchDateProbe('delivery')], - }, - ], - } -} - -function finalImprovementSequences(): AgentMemorySequence[] { - const sameAgentProbeCounts = [0, 1, 2, 3, 1, 2] - return FINAL_SEQUENCE_IDS.map((id, index) => - improvementSequence(id, 'test', sameAgentProbeCounts[index % sameAgentProbeCounts.length]), - ) -} diff --git a/tests/memory/mem0.test.ts b/tests/memory/mem0.test.ts index 98788a4..51b075d 100644 --- a/tests/memory/mem0.test.ts +++ b/tests/memory/mem0.test.ts @@ -1,11 +1,9 @@ import { describe, expect, it } from 'vitest' -import { buildCandidate } from '../../src/memory/improvement/candidate' import { createAgentMemoryBranch, createMem0MemoryAdapter, type Mem0HostedClient, mem0MemoryAdapterIdentity, - type RunAgentMemoryImprovementOptions, } from '../../src/memory/index' describe('Mem0 adapter', () => { @@ -276,29 +274,6 @@ describe('Mem0 adapter', () => { ) }) - it('uses its identity as a memory improvement candidate ref', async () => { - const config = { provider: 'mem0' } as const - const ref = mem0MemoryAdapterIdentity({ - mode: 'hosted', - id: 'mem0', - backendRef: 'mem0-account-a', - }) - const options = { - createCandidate: () => ({ - ref, - externalCostUsdPerSequence: 0, - externalRecoveryCostUsdPerAttempt: 0, - externalCostAccounting: 'exact' as const, - createAdapter: () => null, - }), - } as RunAgentMemoryImprovementOptions - - const candidate = await buildCandidate(options, config, 'mem0') - - expect(candidate.ref).toBe(ref) - expect(candidate.ref).toMatch(/^sha256:[a-f0-9]{64}$/) - }) - it('passes OSS entity filters on add', async () => { let addOptions: Record | undefined const adapter = createMem0MemoryAdapter({ diff --git a/tests/support/benchmarks.ts b/tests/support/benchmarks.ts deleted file mode 100644 index 9b046fa..0000000 --- a/tests/support/benchmarks.ts +++ /dev/null @@ -1,13 +0,0 @@ -import { - type RunMemoryAdapterBenchmarkOptions, - runMemoryAdapterBenchmark as runMemoryAdapterBenchmarkRaw, -} from '../../src/benchmarks/index' - -export function runMemoryAdapterBenchmark(options: RunMemoryAdapterBenchmarkOptions) { - return runMemoryAdapterBenchmarkRaw({ - ...options, - ...(options.storage && !options.controllerMode && !options.acquireRunLease - ? { controllerMode: 'process-local' as const } - : {}), - }) -} diff --git a/tests/support/memory-learning.ts b/tests/support/memory-learning.ts deleted file mode 100644 index 28252bc..0000000 --- a/tests/support/memory-learning.ts +++ /dev/null @@ -1,143 +0,0 @@ -import { createHash } from 'node:crypto' -import { inMemoryCampaignStorage } from '@tangle-network/agent-eval/campaign' -import { - type AgentMemoryExperimentCandidate, - type AgentMemoryMode, - type AgentMemoryScope, - type AgentMemorySequence, - runAgentMemoryExperiment, -} from '../../src/memory/index' -import { createScopedTestAdapter } from './memory' - -export function continualSequences(): AgentMemorySequence[] { - return [retentionSequence(), twoStepControlSequence()] -} - -export function retentionSequence(): AgentMemorySequence { - return { - id: 'retention', - family: 'first-party', - steps: [ - { - id: 'learn', - scope: { agentId: 'worker' }, - writes: [{ id: 'france', kind: 'fact', text: 'The capital of France is Paris.' }], - probes: [retentionProbe()], - }, - { - id: 'interference', - scope: { agentId: 'worker' }, - writes: [{ id: 'germany', kind: 'fact', text: 'The capital of Germany is Berlin.' }], - probes: [ - retentionProbe(), - { - id: 'both-capitals', - transferKey: 'cross-capital-recall', - query: 'Recall both learned capitals.', - requiredFacts: [ - { id: 'france', anyOf: ['capital of France is Paris'] }, - { id: 'germany', anyOf: ['capital of Germany is Berlin'] }, - ], - }, - ], - }, - ], - } -} - -export function twoStepControlSequence(): AgentMemorySequence { - return { - id: 'control', - family: 'first-party', - steps: [ - { - id: 'first', - scope: { agentId: 'worker' }, - writes: [{ id: 'green-1', kind: 'fact', text: 'The control value is green.' }], - probes: [controlProbe()], - }, - { - id: 'second', - scope: { agentId: 'worker' }, - writes: [{ id: 'green-2', kind: 'fact', text: 'The control value is green.' }], - probes: [controlProbe()], - }, - ], - } -} - -export function retentionProbe() { - return { - id: 'capital', - retentionKey: 'capital-of-france', - query: 'What is the capital of France?', - referenceAnswer: 'capital of France is Paris', - } -} - -export function controlProbe() { - return { - id: 'same-local-id', - query: 'What is the control value?', - referenceAnswer: 'control value is green', - } -} - -export function memoryCandidate( - ref: string, - costUsd = 0, - onWriteScope?: (scope: AgentMemoryScope) => void, -): AgentMemoryExperimentCandidate { - return { - id: 'memory', - ref: immutableRef(ref), - externalCostUsdPerSequence: costUsd, - externalRecoveryCostUsdPerAttempt: 0, - externalCostAccounting: 'exact', - createAdapter({ branchId, recordExternalCost }) { - if (costUsd > 0) recordExternalCost(costUsd) - return createScopedTestAdapter(`${ref}:${branchId}`, async (scope) => onWriteScope?.(scope)) - }, - } -} - -export function immutableRef(value: string): string { - return `sha256:${createHash('sha256').update(value).digest('hex')}` -} - -export function memoryCandidateWithoutClear(): AgentMemoryExperimentCandidate { - return { - id: 'memory', - ref: 'memory:no-clear', - createAdapter({ branchId }) { - const { clear: _clear, ...adapter } = createScopedTestAdapter(branchId) - return adapter - }, - async disposeAdapter() {}, - } -} - -export function runDirectArm(mode: AgentMemoryMode, ref: string, experimentRunId: string) { - return runAgentMemoryExperiment({ - experimentId: 'unequal-memory-arms', - experimentRunId, - memoryMode: mode, - sequences: [twoStepControlSequence()], - candidates: [memoryCandidate(ref)], - runDir: `/runs/unequal-memory-arms/${mode}`, - storage: inMemoryCampaignStorage(), - controllerMode: 'process-local', - seed: 23, - }) -} - -export function runInvalidSequence(steps: AgentMemorySequence['steps']) { - return runAgentMemoryExperiment({ - experimentId: 'invalid-retention-measurement', - sequences: [{ id: 'invalid', family: 'first-party', steps }], - candidates: [memoryCandidate('memory:validation')], - runDir: '/runs/invalid-retention-measurement', - storage: inMemoryCampaignStorage(), - controllerMode: 'process-local', - }) -} diff --git a/tests/support/memory.ts b/tests/support/memory.ts index 982ea4b..4102a1b 100644 --- a/tests/support/memory.ts +++ b/tests/support/memory.ts @@ -1,34 +1,4 @@ -import type { JsonValue } from '@tangle-network/agent-eval/campaign' -import { - type AgentMemoryAdapter, - type AgentMemoryHit, - type AgentMemoryScope, - type RunAgentMemoryExperimentOptions, - type RunAgentMemoryImprovementOptions, - runAgentMemoryExperiment as runAgentMemoryExperimentRaw, - runAgentMemoryImprovement as runAgentMemoryImprovementRaw, -} from '../../src/memory/index' - -function withProcessLocalController< - T extends { - storage?: unknown - controllerMode?: 'process-local' - acquireRunLease?: unknown - }, ->(options: T): T { - if (!options.storage || options.controllerMode || options.acquireRunLease) return options - return { ...options, controllerMode: 'process-local' } -} - -export function runAgentMemoryExperiment(options: RunAgentMemoryExperimentOptions) { - return runAgentMemoryExperimentRaw(withProcessLocalController(options)) -} - -export function runAgentMemoryImprovement( - options: RunAgentMemoryImprovementOptions, -) { - return runAgentMemoryImprovementRaw(withProcessLocalController(options)) -} +import type { AgentMemoryAdapter, AgentMemoryHit, AgentMemoryScope } from '../../src/memory/index' export function hitText(hit: AgentMemoryHit): string { return hit.text