diff --git a/CHANGELOG.md b/CHANGELOG.md index 9282124..2ed7de6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ # Changelog +## 20.0.0 + +Remove the web research drivers, which no consumer imported, and the TCloud dependency only they used. +Knowledge no longer browses or calls models itself; supply research through `runKnowledgeResearchLoop` callbacks or `@tangle-network/agent-runtime`. +Removed: `runVerifiedResearchLoop`, `createWebResearchWorker`, `createVerifyingResearchDriver`, `createTangleRouterClient`, `RouterError`, `createAdaptiveResearchDriver`, `createCollectionResearchDriver`, `createResearchDrivingDriver`, `createPersistentResearchDrivingDriver`, `createClaimGroundingVerifier`, `createClaimDecorator`, the cited-claim helpers, the investment-thesis set, task, and material-facts metric, and their types. +`groundClaimInText`, claim ledgers, `TrackedClaim`, store formats, and every other public operation are unchanged. +The verified-research paper and its results remain as a historical record linked to the last commit with that code. + ## 19.1.5 Publish the Eval 0.208 qualification. The play-memory tests gave their fixture Hindsight a 10 ms ingestion deadline on every path; on a loaded self-hosted runner a write that completes at once missed it and the 19.1.4 publish failed. Success paths now get 10 s; only the unresolved-path test keeps 10 ms. diff --git a/README.md b/README.md index 90c30e8..f53f83b 100644 --- a/README.md +++ b/README.md @@ -12,12 +12,11 @@ Supply application callbacks for those decisions, or use `@tangle-network/agent- pnpm add @tangle-network/agent-knowledge @tangle-network/agent-eval@0.208.2 @tangle-network/agent-interface@2.16.0 ``` -Requires Node.js 22.12 or later and Interface 2.16 or later within v2, matching the Sandbox dependency. +Requires Node.js 22.12 or later and Interface 2.16 or later within v2. Eval 0.201 through 0.208 are supported. Package and official optimizer checks exercise each supported minor; development uses 0.208.2. These releases use Core 0.10, Trace Contract 2, and Zod 4.6.5. -Knowledge 19 requires TCloud 0.8 with Sandbox 0.58.4 or later in that minor. Use Knowledge 17 with older Eval releases. ## Choose an API @@ -545,7 +544,7 @@ Those choices stay in the application or in `@tangle-network/agent-runtime`. - [Knowledge retrieval and use receipts](docs/knowledge-use-receipts.md) - [QMD source snapshots](docs/qmd-retrieval.md) - [Play-scoped Hindsight memory](docs/hindsight-memory.md) -- [Verified research comparison](docs/verified-research-ab.md) +- [Verified research comparison (historical; the research drivers were removed in 20.0.0)](docs/verified-research-ab.md) - [Changelog](CHANGELOG.md) ## License diff --git a/api-surface.json b/api-surface.json index b869ae3..ead1350 100644 --- a/api-surface.json +++ b/api-surface.json @@ -2,10 +2,6 @@ "package": "@tangle-network/agent-knowledge", "entries": { ".": { - "AdaptiveDecision": "type b5e46c8679f8", - "AdaptiveDriverOptions": "type cd574a17bf6a", - "AdaptiveResearchDriver": "type 18ad2ba1fe1c", - "AdaptiveStats": "type 94e0566b954b", "AddSourceOptions": "type bee9a2194b6c", "AddSourceTextInput": "type 9619930752c2", "AgentMemoryAcquireRunLease": "type 4ff237e19d6d", @@ -82,13 +78,11 @@ "ChunkingOptions": "type 00fb66d7d155", "ClaimEvidence": "type f780da49a3ef", "ClaimGrade": "type a9e6edbc7877", - "ClaimGroundingDriverOptions": "type 4c5ca74e7255", "ClaimLedgerGoalConflictError": "value c96a14249e23", "ClaimLedgerMigrationRequiredError": "value 2dc627c3fa60", "ClaimLedgerStore": "type c402ec258adc", "ClaimRef": "type e200b4c5eeec", "ClaimVerdict": "type 18f044018844", - "CompanyEvalCase": "type d2903a86209b", "CompareAgentMemoryLearningOptions": "type 33bff9c3b28a", "CornellLiiSelector": "type c92fb79c4d6b", "CornellLiiSourceOptions": "type 3d1a7a174053", @@ -103,7 +97,6 @@ "DEFAULT_KNOWLEDGE_BRIEF_LIMIT": "value a16e63bcd9c7", "DEFAULT_MEMORY_CLEANUP_TIMEOUT_MS": "value 10d1ef3e7731", "DEFAULT_PAGES_DIRECTORY": "value 5aa47f8e3dde", - "DedupReason": "type 8cce009c6ec0", "DeepQuestion": "type 244fa050c94a", "DeepQuestionKind": "type e1a27d1fc3c2", "DeepQuestionSchema": "value 373728f5643d", @@ -115,13 +108,10 @@ "DiscoveryLoopStopReason": "type 61912c65132e", "DiscoveryResult": "type 3a5c8b06d765", "DiscoveryTask": "type bfd854dba2b8", - "DriverResearchContext": "type 3f6ac21642ed", "EvalKnowledgeBundleBuildResult": "type 91fa1b7300ac", "EvaluateKnowledgeBaseReadinessOptions": "type 9a3de35dc405", "EvidenceRung": "type fa5c1c76aa73", - "ExpectedGroup": "type c8024ceb7a23", "ExternalRagEvalScore": "type df1d799a1de6", - "FactResult": "type 79c8d48575f6", "FetchOpts": "type 7b44d8f28084", "FileSystemFreshnessStoreOptions": "type 10a9cbd1e857", "FileSystemKbStore": "value 3e4c27745531", @@ -219,7 +209,6 @@ "KnowledgeExplanation": "type 4326c52192e1", "KnowledgeFragment": "type 5d9512fd9945", "KnowledgeFreshnessStore": "type 6805e31ec36f", - "KnowledgeGap": "type 4630026a4274", "KnowledgeGraph": "type 2a4f44745410", "KnowledgeGraphEdge": "type 9eba2728a7b2", "KnowledgeGraphEdgeSchema": "value 373728f5643d", @@ -336,9 +325,6 @@ "LoadKnowledgeImprovementActivationResultOptions": "type 3c304bc52d1c", "MAX_RESPONSE_BYTES": "value 373728f5643d", "MIN_REQUEST_GAP_MS": "value 648cef7e8361", - "MaterialFact": "type 7b848dee213b", - "MaterialFactLens": "type 504d719aaa16", - "MaterialFactsResult": "type 8e5c394d8ac3", "MeasuredKnowledgeSelectionReceipt": "type 28a9a57c6626", "Mem0ClientMode": "type abfe4225abaf", "Mem0HostedClient": "type 4d1434ac5a9a", @@ -364,7 +350,6 @@ "ParsedFrontmatter": "type 7aebd4ab7c31", "PartitionRetrievalScenariosOptions": "type bfb5473c9c0c", "PendingKnowledgeMutation": "type 265414e3a059", - "PersistentResearchDrivingDriverOptions": "type 1a2cb7bacb89", "PlayMemoryRetrievalReceipt": "type 0118f85065b1", "PoliteFetchOptions": "type e8c10ab474da", "PoliteFetchResult": "type 68c8ac7c266b", @@ -412,23 +397,14 @@ "RagRequiredContext": "type 2ef3d6dfc30b", "RecoverPendingKnowledgeMutationOptions": "type a90f6aeb047d", "RegularFileSnapshot": "type 55d315589931", - "RejectedSource": "type 81899fc9ed5b", "ResearchClaimEvidence": "type 31fa8f46f25a", "ResearchClaimEvidenceSchema": "value 373728f5643d", "ResearchClaimLedger": "type f3435f987d19", "ResearchClaimLedgerSchema": "value 373728f5643d", "ResearchClaimRecord": "type febfce5cc05e", "ResearchClaimRecordSchema": "value 373728f5643d", - "ResearchContribution": "type 4ef9b903af78", - "ResearchDriver": "type 76db7bd73161", - "ResearchDrivingDriver": "type 919454a2ef43", - "ResearchDrivingDriverOptions": "type bf4180687f2d", - "ResearchDrivingState": "type 55bc4328e4c3", - "ResearchDrivingSteer": "type 5f5bca7f3f8f", - "ResearchSourceProposal": "type be9f803d5a8e", "ResearchSourceVersion": "type f20d55e99f71", "ResearchSourceVersionSchema": "value 373728f5643d", - "ResearchWorker": "type 5b69df6c9975", "ResolvedKnowledgeImprovementCandidate": "type 60be04dfb75d", "ResolvedKnowledgeImprovementComparison": "type 2d4520e11c17", "ResolvedKnowledgeImprovementComparisonSnapshot": "type 09ca4bdc3f81", @@ -457,9 +433,6 @@ "RetrievalScenarioPartitions": "type a8a61ddef80a", "RetrievedKnowledgeHit": "type 9dfb968780ec", "RetrievedSourceSpan": "type d421429a842f", - "RouterClient": "type 0e152cea3cc2", - "RouterError": "value c1a8d51a094e", - "RouterUsage": "type 65c8360220a5", "RunAgentMemoryExperimentOptions": "type ea599b678ed8", "RunAgentMemoryExperimentResult": "type 4c59c13e22c2", "RunAgentMemoryImprovementOptions": "type d4fc3b183ac9", @@ -497,31 +470,16 @@ "SourceRecord": "type b525cb8c09ee", "SourceRecordSchema": "value 373728f5643d", "SourceRegistry": "type 062b92354d23", - "SourceVerdict": "type bb58bf97fe54", - "SourceVerificationContext": "type bfe4dde94e67", "StateSosEntity": "type 4ecdc780ae4d", "StateSosSourceConfig": "type 3a2d0b61a718", - "TangleRouterOptions": "type 36f56817baa8", - "ThesisRunOptions": "type 84ede3dc0190", - "ThesisRunResult": "type 24e8289fb585", - "ThesisTaskInput": "type 79a0ee86f1c6", "TrackedClaim": "type 823ab1f0941e", - "TriageClass": "type 684366d72b8e", "UncheckableClaimError": "value 86a15815c4f1", "UseKnowledgeImprovementCandidateOptions": "type 848b0e83e03f", "ValidateKnowledgeOptions": "type 75186f551139", "ValidateKnowledgeResult": "type 85becbbaf24d", "VerifiedGradeableEvidence": "type 68ecc1af89b5", - "VerifiedResearchLoopOptions": "type f879fa4412f4", - "VerifiedResearchLoopResult": "type d9e27e0d871b", - "VerifiedResearchRound": "type c0f34b185f68", "VerifyGradeableEvidenceOptions": "type 01ef698f9b40", - "VerifyingDriverOptions": "type eddd766acc7c", "WIKILINK_REGEX": "value 373728f5643d", - "WebResearchWorkerOptions": "type c26a8fa43a47", - "WebSearchHit": "type 2c6fc9b2fb25", - "WorkerClaimDecorationOptions": "type a3db98314057", - "WorkerResearchContext": "type 1db692fde00a", "__resetHttpThrottle": "value e47f3b6734a6", "acquireAgentMemoryRunLease": "value 7d23f0e042cc", "addSourcePath": "value 9dabb587c73c", @@ -563,21 +521,13 @@ "calibrateRagAnswerJudge": "value 74f91dafd1b1", "canonicalPathsEqual": "value 1a1d22b3b1c3", "canonicalRelativeWithinRoot": "value 9a5ffa3946f3", - "canonicalizeUrl": "value a8a18cde9fcc", "chunkMarkdown": "value 3f930d4ba32f", - "citedClaimKey": "value 810f529bf94c", - "citedClaimOf": "value 1b37f8f9af86", "claimCheckKey": "value f7b71c731a17", "claimEvidenceId": "value 10ea894124db", "claimId": "value 4c9eca896a37", "claimSourceHost": "value a8a18cde9fcc", "compareAgentMemoryLearning": "value 80e066a5a265", - "contentKey": "value 4c9eca896a37", - "createAdaptiveResearchDriver": "value 963a73b60a1f", "createAgentMemoryBranch": "value bae206df8be9", - "createClaimDecorator": "value 0b0ae76ddc8b", - "createClaimGroundingVerifier": "value cd737b122041", - "createCollectionResearchDriver": "value e3f6add71c21", "createCornellLiiSource": "value 5708478c5b8a", "createD1FreshnessStoreStub": "value e89ca59074b9", "createFileRunLineageAuthority": "value 92afeb585f32", @@ -599,17 +549,12 @@ "createMemoryExecutionPool": "value 2cee6a42eafb", "createNeo4jAgentMemoryAdapter": "value 6a86dd3496a1", "createNoopMemoryBenchmarkAdapter": "value d0ec3fcca863", - "createPersistentResearchDrivingDriver": "value f115035319ee", "createPlayMemoryTools": "value cbd6422b065c", "createQmdKnowledgeTools": "value 136b3d258c5c", "createQmdSearchProvider": "value f1d2a2d4d631", "createRagAnswerQualityHook": "value 7e18d330e76b", - "createResearchDrivingDriver": "value 34e12fd11a6a", "createRunScopedStores": "value 6bfc535c7281", "createStateSosSource": "value e7467eeee69b", - "createTangleRouterClient": "value 539a2d5c07dd", - "createVerifyingResearchDriver": "value e70c80302494", - "createWebResearchWorker": "value 3d1356ac9f3b", "decodeKnowledgeVisibilitySnapshot": "value 143bc2b124e6", "deepQuestionId": "value ffec05529ae9", "defaultGetMemoryContext": "value 6a368eb4ea71", @@ -632,8 +577,6 @@ "formatKnowledgeInvalidationProposal": "value 0facdbad3e8e", "fromAgentCandidateKnowledgeRef": "value 247b544b764e", "gradeClaims": "value 5a3c1c162258", - "gradeCompanyAgainstText": "value 7470a3d96d31", - "gradeFactAgainstText": "value 9818c38e4ce2", "gradeFor": "value 79db4a5137a3", "graphitiMemoryAdapterIdentity": "value 203f93807ab2", "groundClaimInText": "value f74ed9ca26c6", @@ -646,7 +589,6 @@ "innerHtmlById": "value f0ffa022d6c0", "inspectKnowledgeIndex": "value 79b67f7de1cd", "inspectPendingKnowledgeMutation": "value 38b2e30827bd", - "investmentThesisSet": "value 373728f5643d", "isKernelAnchoredPath": "value 2819fa14db65", "isKnowledgeMemoryBenchmarkCase": "value 8634e2c95fec", "isKnowledgeMutationHeld": "value e8f0de09e802", @@ -657,7 +599,6 @@ "isScaffoldPath": "value 2819fa14db65", "jsonCandidateCodec": "value 6520c69bb348", "jsonObjectCandidateCodec": "value 8eb601cd6de7", - "kbIndexToText": "value cb17a57bfdaf", "knowledgeBenchmarkJudge": "value e7c74f6e007a", "knowledgeCitationAuditFindings": "value 639f15656570", "knowledgeImprovementCandidateRef": "value 2c85f9c2a1a0", @@ -669,7 +610,6 @@ "knowledgeReleaseReport": "value 76d7eebb2957", "knowledgeVisibilityArtifactRef": "value dfa0672dff59", "layoutFor": "value a4861d951cf1", - "lensDistribution": "value e6cf3a8f1040", "linkClaimContradictions": "value cf315a2deab2", "lintCurrentRunCitations": "value 59004ed0538a", "lintKnowledgeIndex": "value c866a9cbb418", @@ -681,8 +621,6 @@ "loadKnowledgePromotionRecord": "value 65617956865c", "loadSourceRegistry": "value cae49411bcc5", "looksLikeBlockPage": "value 371f55d1c1d9", - "materialFactsSurfaced": "value 80d0cc7f1647", - "materialFactsSurfacedInText": "value 0f9382be96a6", "materializeRegisteredClaimEvidence": "value cf315a2deab2", "mediaTypeFor": "value a8a18cde9fcc", "mem0MemoryAdapterIdentity": "value c590a9c1e72f", @@ -742,7 +680,6 @@ "runBoundedMemoryLifecycle": "value 75f3e3b274a0", "runDiscoveryLoop": "value addb8a464a58", "runInKnowledgeMutationScope": "value 83a5698000d7", - "runInvestmentThesisTask": "value 2eac673e45ab", "runKnowledgeBenchmarkSuite": "value ce72ffe18eef", "runKnowledgeResearchLoop": "value 4d00a0a2e19d", "runMemoryAdapterBenchmark": "value e783141bcd1d", @@ -750,7 +687,6 @@ "runRagOptimization": "value 9493b6dd6742", "runRetrievalImprovementLoop": "value d1f8efdb025d", "runSerializedKnowledgeOptimization": "value 8571ad4d22bd", - "runVerifiedResearchLoop": "value 16ec2882b9ce", "scenarioContentFingerprint": "value c6788a57974b", "scoreBm25": "value 42139506975a", "scoreKnowledgeBaseIndex": "value c16e27e90d97", @@ -764,7 +700,6 @@ "sleepForMemoryRecovery": "value 4a6818f984e4", "slugify": "value 8b2ddb657a6c", "snapshotSourceTextInput": "value 853c0b94fd4b", - "sourceMatchesGaps": "value be42d9fa537e", "sourceRegistryPath": "value 5fe2ec842844", "stableId": "value a7bc5069602e", "stripFrontmatter": "value 335a29d6a1cd", @@ -772,7 +707,6 @@ "syncDirectory": "value 6218521600a7", "textSourceAdapter": "value 373728f5643d", "textSourceId": "value 3ea77a7de22d", - "thesisReadinessSpecs": "value da64bfb4d878", "toAgentCandidateKnowledgeRef": "value 6b07b25e59e9", "toDeepEvalTestCases": "value 18e7b27cf433", "toOffPolicyTrajectory": "value 934dec24305c", @@ -781,8 +715,6 @@ "toTruLensRecords": "value d46aa76e3ee3", "tokenizeQuery": "value 86241b60c1a4", "tokenizeText": "value 1370ca50219c", - "totalMaterialFacts": "value 47ef736007c5", - "triageSource": "value 1aad35658ea4", "validateKnowledgeIndex": "value 1325e74dea5e", "verdictFor": "value deb848cf5c44", "verifyGradeableEvidence": "value 46baf5f38063", @@ -791,7 +723,6 @@ "verifyKnowledgeUseReceipt": "value 13e38b6c70c9", "verifyKnowledgeVisibilitySnapshot": "value cc5c6db30910", "walk": "value 5035737ef563", - "withCitedClaim": "value 480a57a11853", "withKnowledgeImprovementCandidate": "value 1807f0708eaa", "withKnowledgeImprovementComparison": "value 209ea121fc27", "withKnowledgeMutation": "value 499371d01638", diff --git a/docs/architecture.md b/docs/architecture.md index 1a9fd4f..3931021 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -57,15 +57,10 @@ They reach it through `mergeClaimLedger(id, merge)`, which holds the mutation lo The combining rule is `mergeClaimLedgers`: support and contradiction edges union, `contested` and `addressed` latch on, `firstSeenRound` moves earlier, and every collection is sorted. The merge is commutative, associative, and idempotent, so the bytes on disk depend on the evidence rather than on scheduling. Ledgers for two different goals refuse to merge (`ClaimLedgerGoalConflictError`) rather than pooling unrelated evidence into one corroboration count. -The live driver exposes the published Set-based `TrackedClaim`; the ledger stores a separate `ResearchClaimRecord` with sorted arrays so JSON serialization cannot erase those sets. -Source verification snapshots the proposal and persists a `ResearchClaimEvidence` observation containing the expected registry id, original URI, and full content hash; that observation cannot affect claim support or completion by itself. -After the exact submitted bytes are durable, `runVerifiedResearchLoop` passes the resulting `SourceRecord` to `commitSources`. +`TrackedClaim` keeps its published `Set` fields; the ledger stores a separate `ResearchClaimRecord` with sorted arrays so JSON serialization cannot erase those sets. +A `ResearchClaimEvidence` observation records the expected registry id, original URI, and full content hash; it cannot affect claim support or completion by itself. The ledger materializes only observations whose complete source identity matches a confirmed record, so reusing one URI for different bytes cannot activate the wrong claims and a crash on either side resumes safely. Unversioned URI-only ledgers cannot prove which bytes produced their observations; reads and writes fail with `ClaimLedgerMigrationRequiredError` and preserve the original file for an explicit archive-and-reverify migration. -Before synchronous question generation, the persistent driver records `preparedRounds`; a resume reconstructs and checkpoints any prepared round whose questions were interrupted, and the loop publishes its `research.iteration` event only after that checkpoint succeeds. -The research loop requires storage readiness and the driver's optional `isComplete()` result before it reports completion. -An unfinished driver can generate steering with no remaining storage gaps, so passing source requirements does not stop research prematurely. -Drivers without `isComplete()` use storage readiness alone. Without readiness specifications, the loop runs to its round limit and never reports ready. Every write in this layer goes through `durable-fs` (`writeFileDurable`, `writeJsonDurableWithinRoot`): temp file, fsync, atomic rename, and parent fsync. diff --git a/docs/eval/investment-material-facts.md b/docs/eval/investment-material-facts.md index d4c0c03..0f0ad13 100644 --- a/docs/eval/investment-material-facts.md +++ b/docs/eval/investment-material-facts.md @@ -1,5 +1,7 @@ # Held-out investment-research eval set: material facts + provenance +> Historical record. The investment-thesis set and task were removed in Knowledge 20.0.0; see commit bc2abe0cee1047350a27af37200b94b6801d1f6b for the code. + This is the answer key and the provenance ledger for `tests/eval/investment-thesis-set.ts`. **What the set measures.** Give a research loop a company + ticker + an as-of diff --git a/docs/results/research-driving.md b/docs/results/research-driving.md index c4e805d..0525784 100644 --- a/docs/results/research-driving.md +++ b/docs/results/research-driving.md @@ -58,7 +58,7 @@ So we need a metric for research **quality**, not hygiene. We use: ## 2. The held-out exam (the firewall) -[`tests/loops/held-out-exam.ts`](../../tests/loops/held-out-exam.ts). 5 ML topics, +[`tests/loops/held-out-exam.ts`](https://github.com/tangle-network/agent-knowledge/blob/bc2abe0cee1047350a27af37200b94b6801d1f6b/tests/loops/held-out-exam.ts). 5 ML topics, 4 questions each = **20 deep questions**. Each is a *depth* question by construction, comparative, mechanism-level, or contradiction-aware, chosen so a single web search for the topic name does **not** surface the answer: @@ -100,16 +100,16 @@ between arms (if any) would be real depth, not grader slack. ## 3. The three arms, at equal compute -[`tests/loops/research-driving-ab.test.ts`](../../tests/loops/research-driving-ab.test.ts). +[`tests/loops/research-driving-ab.test.ts`](https://github.com/tangle-network/agent-knowledge/blob/bc2abe0cee1047350a27af37200b94b6801d1f6b/tests/loops/research-driving-ab.test.ts). All three arms run the **same** real web worker -([`createWebResearchWorker`](../../src/web-research-worker.ts), glm-5.2 query-gen +([`createWebResearchWorker`](https://github.com/tangle-network/agent-knowledge/blob/bc2abe0cee1047350a27af37200b94b6801d1f6b/src/web-research-worker.ts), glm-5.2 query-gen → live `/v1/search` → `politeFetch` → `htmlToText`). They differ **only** in the driver: - **(A) single-agent collection**: the worker alone, no driver. It collects. -- **(B) verify/dedup**: [`createVerifyingResearchDriver`](../../src/web-research-worker.ts): +- **(B) verify/dedup**: [`createVerifyingResearchDriver`](https://github.com/tangle-network/agent-knowledge/blob/bc2abe0cee1047350a27af37200b94b6801d1f6b/src/web-research-worker.ts): a second glm-5.2 pass filters each source for relevance / near-duplicates. -- **(C) DRIVING**: [`createResearchDrivingDriver`](../../src/research-driving-driver.ts): +- **(C) DRIVING**: [`createResearchDrivingDriver`](https://github.com/tangle-network/agent-knowledge/blob/bc2abe0cee1047350a27af37200b94b6801d1f6b/src/research-driving-driver.ts): the driver extracts each source's claims (glm-5.2), tracks independent-source support + contradictions, and folds comparative / mechanism / gap / contradiction sub-questions into the worker's next prompt. @@ -258,8 +258,8 @@ AGENT_KNOWLEDGE_LIVE=1 RQ_PROBE=1 RQ_PROBE_ROUNDS=3 TANGLE_API_KEY=<…> \ cheaper slice. The exam is held out by construction, no flag shows it to a loop. **Source:** the exam and grader: -[`tests/loops/held-out-exam.ts`](../../tests/loops/held-out-exam.ts); +[`tests/loops/held-out-exam.ts`](https://github.com/tangle-network/agent-knowledge/blob/bc2abe0cee1047350a27af37200b94b6801d1f6b/tests/loops/held-out-exam.ts); the 3-arm A/B and multi-round probe: -[`tests/loops/research-driving-ab.test.ts`](../../tests/loops/research-driving-ab.test.ts); +[`tests/loops/research-driving-ab.test.ts`](https://github.com/tangle-network/agent-knowledge/blob/bc2abe0cee1047350a27af37200b94b6801d1f6b/tests/loops/research-driving-ab.test.ts); the driving driver under test: -[`src/research-driving-driver.ts`](../../src/research-driving-driver.ts). +[`src/research-driving-driver.ts`](https://github.com/tangle-network/agent-knowledge/blob/bc2abe0cee1047350a27af37200b94b6801d1f6b/src/research-driving-driver.ts). diff --git a/docs/verified-research-ab.md b/docs/verified-research-ab.md index 4c96d43..578e13c 100644 --- a/docs/verified-research-ab.md +++ b/docs/verified-research-ab.md @@ -1,5 +1,7 @@ # A verifier agent mostly deduplicates: a controlled A/B on two-agent web research, and what its cost buys +> Historical record. The research loop, drivers, web worker, and investment-thesis task this paper measured were removed in Knowledge 20.0.0; code links point to the last commit that contained them. + *Tangle Network · `agent-knowledge`* ## Abstract @@ -476,13 +478,13 @@ per-company A/B: [`docs/results/investment-thesis.md`](results/investment-thesis ## 10. Reproduce The loop, the worker, the verifier, the claim-grounding mode, the adaptive driver, the -driving driver, the held-out exam, the cost instrumentation, and every A/B are all in -this repository. Each live test gates a cheap one-call glm-5.2 smoke before any +driving driver, the held-out exam, the cost instrumentation, and every A/B are at commit +`bc2abe0cee1047350a27af37200b94b6801d1f6b`. Each live test gates a cheap one-call glm-5.2 smoke before any multi-topic burn. ```bash git clone https://github.com/tangle-network/agent-knowledge -cd agent-knowledge && pnpm install +cd agent-knowledge && git checkout bc2abe0cee1047350a27af37200b94b6801d1f6b && pnpm install # offline A/B: deterministic, no credentials (a controlled lower bound that # exercises the same harness against a planted source pool) @@ -523,13 +525,13 @@ AGENT_KNOWLEDGE_LIVE=1 RQ_PROBE=1 RQ_PROBE_ROUNDS=3 TANGLE_API_KEY=<…> \ topic list; the live arms run the loops on each at equal compute and report the paired bootstrap and per-arm cost. -**Source:** the loop, [`src/verified-research-loop.ts`](../src/verified-research-loop.ts); -the live worker + verifier + cost instrumentation, [`src/web-research-worker.ts`](../src/web-research-worker.ts); -the misattribution check, [`src/claim-grounding.ts`](../src/claim-grounding.ts); -the adaptive driver, [`src/adaptive-driver.ts`](../src/adaptive-driver.ts); -the driving driver, [`src/research-driving-driver.ts`](../src/research-driving-driver.ts); -the held-out exam + $0 grader, [`tests/loops/held-out-exam.ts`](../tests/loops/held-out-exam.ts); -the A/B harnesses, [`tests/loops/`](../tests/loops/). +**Source:** the loop, [`src/verified-research-loop.ts`](https://github.com/tangle-network/agent-knowledge/blob/bc2abe0cee1047350a27af37200b94b6801d1f6b/src/verified-research-loop.ts); +the live worker + verifier + cost instrumentation, [`src/web-research-worker.ts`](https://github.com/tangle-network/agent-knowledge/blob/bc2abe0cee1047350a27af37200b94b6801d1f6b/src/web-research-worker.ts); +the misattribution check, [`src/claim-grounding.ts`](https://github.com/tangle-network/agent-knowledge/blob/bc2abe0cee1047350a27af37200b94b6801d1f6b/src/claim-grounding.ts); +the adaptive driver, [`src/adaptive-driver.ts`](https://github.com/tangle-network/agent-knowledge/blob/bc2abe0cee1047350a27af37200b94b6801d1f6b/src/adaptive-driver.ts); +the driving driver, [`src/research-driving-driver.ts`](https://github.com/tangle-network/agent-knowledge/blob/bc2abe0cee1047350a27af37200b94b6801d1f6b/src/research-driving-driver.ts); +the held-out exam + $0 grader, [`tests/loops/held-out-exam.ts`](https://github.com/tangle-network/agent-knowledge/blob/bc2abe0cee1047350a27af37200b94b6801d1f6b/tests/loops/held-out-exam.ts); +the A/B harnesses, [`tests/loops/`](https://github.com/tangle-network/agent-knowledge/tree/bc2abe0cee1047350a27af37200b94b6801d1f6b/tests/loops/). Per-result detail: [`docs/results/cost-quality.md`](results/cost-quality.md), [`docs/results/claim-grounding.md`](results/claim-grounding.md), [`docs/results/adaptive.md`](results/adaptive.md), diff --git a/package.json b/package.json index 39fb93e..ec33b66 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@tangle-network/agent-knowledge", - "version": "19.1.5", + "version": "20.0.0", "description": "Build, search, evaluate, and improve source-backed knowledge bases.", "homepage": "https://github.com/tangle-network/agent-knowledge#readme", "repository": { @@ -81,8 +81,7 @@ "dependencies": { "@types/proper-lockfile": "4.1.4", "proper-lockfile": "4.1.2", - "zod": "4.6.5", - "@tangle-network/tcloud": ">=0.8.0 <0.9.0" + "zod": "4.6.5" }, "peerDependencies": { "@tangle-network/agent-eval": ">=0.201.0 <0.209.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index c829c19..09fb600 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -166,9 +166,6 @@ importers: .: dependencies: - '@tangle-network/tcloud': - specifier: '>=0.8.0 <0.9.0' - version: 0.8.0(typescript@7.0.2)(zod@4.6.5) '@types/proper-lockfile': specifier: 4.1.4 version: 4.1.4 @@ -224,9 +221,6 @@ importers: packages: - '@adraffy/ens-normalize@1.11.1': - resolution: {integrity: sha512-nhCBV3quEgesuf7c7KYfperqSS14T8bYuvJ8PcLJp6znkZpFc0AuW4qBtr8eKVyPPe/8RSr7sglCWPU5eaxwKQ==} - '@andrewbranch/untar.js@1.0.4': resolution: {integrity: sha512-pVXSwPsLuw8IGLo2Di0EaOfsk+ntVvpkk942J/sHYIkwvtKUakEcPh7HBgZ6tuimgzKSEHgCvO4XgQ05DEbwDw==} @@ -374,18 +368,6 @@ packages: zod: optional: true - '@noble/ciphers@1.3.0': - resolution: {integrity: sha512-2I0gnIVPtfnMw9ee9h1dJG7tp81+8Ob3OJb3Mv37rx5L40/b0i7djjCVvGOVqc9AEIQyvyu1i6ypKdFw8R8gQw==} - engines: {node: ^14.21.3 || >=16} - - '@noble/curves@1.9.1': - resolution: {integrity: sha512-k11yZxZg+t+gWvBbIswW0yoJlu8cHOC7dhunwOzoWH/mXGBiYyR4YY6hAEK/3EUs4UpB8la1RfdRpeGsFHkWsA==} - engines: {node: ^14.21.3 || >=16} - - '@noble/hashes@1.8.0': - resolution: {integrity: sha512-jCs9ldd7NwzpgXDIf6P3+NrHh9/sD6CQdxHyjQI+h/6rDNo88ypBxxz45UDuZHz9r3tNz7N/VInSVoVdtXEI4A==} - engines: {node: ^14.21.3 || >=16} - '@noble/hashes@2.4.0': resolution: {integrity: sha512-X5XaVWZIBCT7HHZGm5I7ZQXDwLG+bGXuSrMQAW+7Zvl87h1kmc1ZB1VSRJcpUfoUrGQp4Fkoxm5kZ+Ms+aW+eA==} engines: {node: '>= 20.19.0'} @@ -657,15 +639,6 @@ packages: '@rolldown/pluginutils@1.0.1': resolution: {integrity: sha512-2j9bGt5Jh8hj+vPtgzPtl72j0yRxHAyumoo6TNfAjsLB04UtpSvPbPcDcBMxz7n+9CYB0c1GxQFxYRg2jimqGw==} - '@scure/base@1.2.6': - resolution: {integrity: sha512-g/nm5FgUa//MCj1gV09zTJTaM6KBAHqLN907YVQqf7zC49+DcO4B1so4ZX07Ef10Twr6nuqYEH9GEggFXA4Fmg==} - - '@scure/bip32@1.7.0': - resolution: {integrity: sha512-E4FFX/N3f4B80AKWp5dP6ow+flD1LQZo/w8UnLGYZO674jS6YnYeepycOOksv+vLPSpgN35wgKgy+ybfTb2SMw==} - - '@scure/bip39@1.6.0': - resolution: {integrity: sha512-+lF0BbLiJNwVlev4eKelw1WWLaiKXw7sSl8T6FvBlWkdX+94aGJ4o8XjUdlyhTCjd8c+B3KT3JfS8P0bLRNU6A==} - '@sinclair/typebox@0.27.12': resolution: {integrity: sha512-hhyNJ+nbR6ZR7pToHvllEFun9TL0sbL+tk/ON75lo+Xas054uez98qRbsuNt7MBCyZKK4+8Yli/OAGZhmfBZ/g==} @@ -698,35 +671,6 @@ packages: '@tangle-network/agent-trace-contract@2.0.0': resolution: {integrity: sha512-lq2hp3gg8FsNVA3yZAyuj3NvL57pJTsI6sUF8cYwVsUXA8QZlrO78SqRPRslWxf/sMFHxXhusOuDQ3YxVv12bw==} - '@tangle-network/sandbox@0.58.4': - resolution: {integrity: sha512-wdpocVvSNrAmjmnqGLE4KqDrMq7+8PVCChaLWrakbhuFPksbdwcGsYgHPdASbHPpAi3GGo/YcgMM1x+3igywlQ==} - peerDependencies: - '@mastra/core': ^1.36.0 - '@modelcontextprotocol/sdk': ^1.30.0 - ai: ^6.0.175 || ^7.0.107 - openai: ^6.36.0 - viem: ^2.0.0 - peerDependenciesMeta: - '@mastra/core': - optional: true - '@modelcontextprotocol/sdk': - optional: true - ai: - optional: true - openai: - optional: true - viem: - optional: true - - '@tangle-network/tcloud-attestation@0.1.1': - resolution: {integrity: sha512-+TAF9s5t1jOWGyGHvKhIWe2FYmG7puVaxmmg0Et67ylAjGa7GqUAvISXGjG/6dzld7A170V0kQHK0WVdh2Wh0Q==} - engines: {node: '>=18'} - - '@tangle-network/tcloud@0.8.0': - resolution: {integrity: sha512-85gOyPI5X3+Wo0eDmFISicfnugCIhXib4+yN6tS6t/jJ+Z7pXVfVN04/hV3/4+M1d/kWZjRbW6JtbF4gKORqUA==} - engines: {node: '>=22.12.0'} - hasBin: true - '@types/chai@5.2.3': resolution: {integrity: sha512-Mw558oeA9fFbv65/y4mHtXDs9bPnFMZAL/jxdPFUpOHHIXX91mcgEHbS5Lahr+pwZFR8A7GQleRWeI6cGFC2UA==} @@ -1050,17 +994,6 @@ packages: '@yuku-toolchain/types@0.10.2': resolution: {integrity: sha512-sSeo4SSSToiS+sSD+bwn/s94EEcaLJ7tG5LCp8gFYC1G5VzxzX7fqB6m9RU1yMD8KTpu6zdU/I1NlQf8hVBJ9Q==} - abitype@1.2.3: - resolution: {integrity: sha512-Ofer5QUnuUdTFsBRwARMoWKOH1ND5ehwYhJ3OJ/BQO+StkwQjHw0XyVh4vDttzHB7QOFhPHa/o413PJ82gU/Tg==} - peerDependencies: - typescript: '>=5.0.4' - zod: ^3.22.0 || ^4.0.0 - peerDependenciesMeta: - typescript: - optional: true - zod: - optional: true - abort-controller@3.0.0: resolution: {integrity: sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==} engines: {node: '>=6.5'} @@ -1215,10 +1148,6 @@ packages: resolution: {integrity: sha512-y4Mg2tXshplEbSGzx7amzPwKKOCGuoSRP/CjEdwwk0FOGlUbq6lKuoyDZTNZkmxHdJtp54hdfY/JUrdL7Xfdug==} engines: {node: '>=14'} - commander@15.0.0: - resolution: {integrity: sha512-z67u4ZhzCL/Tydu1lJARtEZYWbWaN7oYLHbsuzocr6y4N6WZAagG3RQ4FW61V1/0+jImpj293XfrcYnd1qxtPg==} - engines: {node: '>=22.12.0'} - compromise@14.17.0: resolution: {integrity: sha512-zw9iEcts/8tMDASNopMQEs3Pclkx2Xk7XCltAb/oV1LEZcCQpDaalAtFmvuxYo+wnVDsW3H3oT16mw9qVHpkqA==} engines: {node: '>=12.0.0'} @@ -1327,9 +1256,6 @@ packages: resolution: {integrity: sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==} engines: {node: '>=6'} - eventemitter3@5.0.1: - resolution: {integrity: sha512-GWkBvjiSZK87ELrYOSESUYeVIc9mvLLf/nXalMOS5dYrgZq9o5OVkbZAVM06CVxYsCwH9BDZFPlQTlPA1j4ahA==} - expand-template@2.0.3: resolution: {integrity: sha512-XYfuKMvj4O35f/pOXLObndIRvyQ+/+6AhODh+OKWj9S9498pHHn/IMszH+gt0fBCRWMNfk1ZSp5x3AifmnI2vg==} engines: {node: '>=6'} @@ -1482,11 +1408,6 @@ packages: resolution: {integrity: sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==} engines: {node: '>=0.12.0'} - isows@1.0.7: - resolution: {integrity: sha512-I1fSfDCZL5P0v33sVqeTDSpcstAg/N+wF5HS033mogOVIp4B+oHC7oOCsA3axAbBSGTJ8QubbNmnIRN/h8U7hg==} - peerDependencies: - ws: '*' - jest-diff@29.7.0: resolution: {integrity: sha512-LMIgiIrhigmPrs03JHpxUh2yISK3vLFPkAodPeo0+BuF7wA2FoQbkEg1u8gBYBThncu7e1oEDUfIXVuTqLRUjw==} engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0} @@ -1898,14 +1819,6 @@ packages: openapi3-ts@4.6.1: resolution: {integrity: sha512-XW9MOldkhoICNeXVzzmXzmOW5G73ppOEGmh7fLCqHjgfdEYCGGN+00MlVCeUZgovjjfC56j9tvtDt1zGabNjjA==} - ox@0.14.45: - resolution: {integrity: sha512-jpsQ+p0JZh9mKCxxzxz0d5qFHZZg2/O9xW18IpEC/dNXXPVqmDJ0c//9RMz0CILkbSGPA42+cbU0fC/ghwJ03w==} - peerDependencies: - typescript: '>=5.4.0' - peerDependenciesMeta: - typescript: - optional: true - oxc-parser@0.151.0: resolution: {integrity: sha512-cX++XpBDCtSe52KKH3rhbh6xIRidYPpyB2reI3azD4l+Y0ci4JZiUgVXyM/r9rrMnYZf9gdWi+9fvFl1wol7nA==} engines: {node: ^20.19.0 || >=22.12.0} @@ -2352,14 +2265,6 @@ packages: resolution: {integrity: sha512-mMVzj0TXExtVdlDEq+Mzp0eyOuyznNpFobNM3uAqe3HVm/Ps2c+u17BligMkZjZCA6EiXpoon8iRTer3iu40SQ==} engines: {node: '>=18.12.0'} - viem@2.56.9: - resolution: {integrity: sha512-UC1G0Qz+RbPKqwl6yRrp3dH6ltrOmUcFJMTcOyE1Mg0nO5lxrEBTONh/8OFWPnALGuN/O7LkNcz/IsSj83Svqw==} - peerDependencies: - typescript: '>=5.0.4' - peerDependenciesMeta: - typescript: - optional: true - vite@8.3.1: resolution: {integrity: sha512-/bvH9E9tmCXRGp2uXY3WbOldqpTwFkbha/8ANaEQ6VkxhH60KyqLwgZq6lG2y+4uT55x9+9eUHMpQ7uGnOCKjA==} engines: {node: ^20.19.0 || >=22.12.0} @@ -2528,8 +2433,6 @@ packages: snapshots: - '@adraffy/ens-normalize@1.11.1': {} - '@andrewbranch/untar.js@1.0.4': {} '@arethetypeswrong/cli@0.18.5': @@ -2650,14 +2553,6 @@ snapshots: optionalDependencies: zod: 4.6.5 - '@noble/ciphers@1.3.0': {} - - '@noble/curves@1.9.1': - dependencies: - '@noble/hashes': 1.8.0 - - '@noble/hashes@1.8.0': {} - '@noble/hashes@2.4.0': {} '@oxc-parser/binding-android-arm-eabi@0.151.0': @@ -2794,19 +2689,6 @@ snapshots: '@rolldown/pluginutils@1.0.1': {} - '@scure/base@1.2.6': {} - - '@scure/bip32@1.7.0': - dependencies: - '@noble/curves': 1.9.1 - '@noble/hashes': 1.8.0 - '@scure/base': 1.2.6 - - '@scure/bip39@1.6.0': - dependencies: - '@noble/hashes': 1.8.0 - '@scure/base': 1.2.6 - '@sinclair/typebox@0.27.12': {} '@sindresorhus/is@4.6.0': {} @@ -2848,32 +2730,6 @@ snapshots: '@tangle-network/agent-trace-contract@2.0.0': {} - '@tangle-network/sandbox@0.58.4(viem@2.56.9(typescript@7.0.2)(zod@4.6.5))': - dependencies: - '@tangle-network/agent-core': 0.10.2 - '@tangle-network/agent-interface': 2.15.0 - zod: 4.6.5 - optionalDependencies: - viem: 2.56.9(typescript@7.0.2)(zod@4.6.5) - - '@tangle-network/tcloud-attestation@0.1.1': {} - - '@tangle-network/tcloud@0.8.0(typescript@7.0.2)(zod@4.6.5)': - dependencies: - '@tangle-network/sandbox': 0.58.4(viem@2.56.9(typescript@7.0.2)(zod@4.6.5)) - '@tangle-network/tcloud-attestation': 0.1.1 - commander: 15.0.0 - viem: 2.56.9(typescript@7.0.2)(zod@4.6.5) - transitivePeerDependencies: - - '@mastra/core' - - '@modelcontextprotocol/sdk' - - ai - - bufferutil - - openai - - typescript - - utf-8-validate - - zod - '@types/chai@5.2.3': dependencies: '@types/deep-eql': 4.0.2 @@ -3082,11 +2938,6 @@ snapshots: '@yuku-toolchain/types@0.10.2': {} - abitype@1.2.3(typescript@7.0.2)(zod@4.6.5): - optionalDependencies: - typescript: 7.0.2 - zod: 4.6.5 - abort-controller@3.0.0: dependencies: event-target-shim: 5.0.1 @@ -3232,8 +3083,6 @@ snapshots: commander@10.0.1: {} - commander@15.0.0: {} - compromise@14.17.0: dependencies: efrt: 2.7.0 @@ -3311,8 +3160,6 @@ snapshots: event-target-shim@5.0.1: {} - eventemitter3@5.0.1: {} - expand-template@2.0.3: {} expect-type@1.4.0: {} @@ -3440,10 +3287,6 @@ snapshots: is-number@7.0.0: {} - isows@1.0.7(ws@8.21.0): - dependencies: - ws: 8.21.0 - jest-diff@29.7.0: dependencies: chalk: 4.1.2 @@ -3756,21 +3599,6 @@ snapshots: dependencies: yaml: 2.9.1 - ox@0.14.45(typescript@7.0.2)(zod@4.6.5): - dependencies: - '@adraffy/ens-normalize': 1.11.1 - '@noble/ciphers': 1.3.0 - '@noble/curves': 1.9.1 - '@noble/hashes': 1.8.0 - '@scure/bip32': 1.7.0 - '@scure/bip39': 1.6.0 - abitype: 1.2.3(typescript@7.0.2)(zod@4.6.5) - eventemitter3: 5.0.1 - optionalDependencies: - typescript: 7.0.2 - transitivePeerDependencies: - - zod - oxc-parser@0.151.0: dependencies: '@oxc-project/types': 0.151.0 @@ -4233,23 +4061,6 @@ snapshots: verkit@0.4.1: {} - viem@2.56.9(typescript@7.0.2)(zod@4.6.5): - dependencies: - '@noble/curves': 1.9.1 - '@noble/hashes': 1.8.0 - '@scure/bip32': 1.7.0 - '@scure/bip39': 1.6.0 - abitype: 1.2.3(typescript@7.0.2)(zod@4.6.5) - isows: 1.0.7(ws@8.21.0) - ox: 0.14.45(typescript@7.0.2)(zod@4.6.5) - ws: 8.21.0 - optionalDependencies: - typescript: 7.0.2 - transitivePeerDependencies: - - bufferutil - - utf-8-validate - - zod - vite@8.3.1(@types/node@26.6.2)(yaml@2.9.1): dependencies: lightningcss: 1.33.0 @@ -4314,7 +4125,8 @@ snapshots: wrappy@1.0.2: {} - ws@8.21.0: {} + ws@8.21.0: + optional: true xtend@4.0.2: {} diff --git a/scripts/prove-research-live.mjs b/scripts/prove-research-live.mjs deleted file mode 100644 index dfcb3ae..0000000 --- a/scripts/prove-research-live.mjs +++ /dev/null @@ -1,29 +0,0 @@ -#!/usr/bin/env node -import assert from 'node:assert/strict' -import { randomUUID } from 'node:crypto' -import { createTangleRouterClient, RouterError } from '../dist/index.js' - -assert.ok(process.env.TANGLE_API_KEY, 'Set a funded TANGLE_API_KEY without printing it') -const marker = `knowledge-tcloud-ok-${randomUUID()}` -const model = process.env.TANGLE_PROOF_MODEL || 'gpt-4o-mini' -const client = createTangleRouterClient({ - apiKey: process.env.TANGLE_API_KEY, - baseUrl: process.env.TANGLE_ROUTER_URL || 'https://router.tangle.tools/v1', - model, - maxRetries: 0, - ...(process.env.TANGLE_SEARCH_PROVIDER ? { searchProvider: process.env.TANGLE_SEARCH_PROVIDER } : {}), - signal: AbortSignal.timeout(120000), -}) -const query = 'Tangle Sandbox SDK official documentation' -const hits = await client.search(query, { maxResults: 1 }) -assert.ok(hits.length > 0 && hits.every(hit => hit.url), 'No live search results') -const messages = [{ role: 'user', content: `Reply with exactly ${marker}` }] -const answer = await client.chat(messages, 1200) -assert.ok(answer.includes(marker), 'Chat omitted the unique request marker') -const usage = client.usage() -assert.equal(usage.chatCalls, 1) -assert.equal(usage.searchCalls, 1) -assert.ok(Number.isFinite(usage.usd) && usage.usd >= 0, 'Router omitted a cost receipt') -console.log(JSON.stringify({ proof: 'built-knowledge-router', model, marker, query, hits, messages, answer, usage }, null, 2)) -// Existing public compatibility facade remains constructible. -assert.equal(new RouterError(401, 'proof').status, 401) diff --git a/scripts/prove-research-transport.mjs b/scripts/prove-research-transport.mjs deleted file mode 100644 index a6edf9b..0000000 --- a/scripts/prove-research-transport.mjs +++ /dev/null @@ -1,75 +0,0 @@ -#!/usr/bin/env node -// Local HTTP proof of failures found in #222. No mocked fetch or unit runner. -import assert from 'node:assert/strict' -import { createServer } from 'node:http' -import { once } from 'node:events' -import { setTimeout as sleep } from 'node:timers/promises' -import { createTangleRouterClient, RouterError } from '../dist/index.js' - -const abort = new AbortController() -const timers = new Set() -let cancelledSocket = false -const observed = [] -const server = createServer(async (req, res) => { - let text = '' - for await (const chunk of req) text += chunk - const body = JSON.parse(text) - observed.push({ path: req.url, client: req.headers['x-tangle-client'], hasSignalField: 'signal' in body }) - if (body.query) { - res.setHeader('Content-Type', 'application/json') - res.end(JSON.stringify({ data: [{ title: 'source', url: 'https://example.com/proof' }, {}], usage: { billed_cost: 0.003 } })) - return - } - const prompt = body.messages[0].content - if (prompt === 'denied') { - res.writeHead(401, { 'Content-Type': 'application/json' }) - res.end('{"error":{"message":"proof denial"}}') - return - } - if (prompt === 'cancel') { - res.once('close', () => { cancelledSocket = true }) - res.writeHead(200, { 'Content-Type': 'application/json' }) - res.write('{"choices":[') - timers.add(setTimeout(() => abort.abort(new DOMException('proof cancelled', 'AbortError')), 30)) - timers.add(setTimeout(() => res.end(']}'), 2500)) - return - } - if (prompt === 'unbilled') { - res.writeHead(200, { - 'Content-Type': 'application/json', - 'X-Tangle-Price-Input': '0.000001', - 'X-Tangle-Price-Output': '0.000002', - }) - res.end(JSON.stringify({ choices: [{ message: { role: 'assistant', content: prompt } }], usage: { prompt_tokens: 2, completion_tokens: 1, total_tokens: 3 } })) - return - } - await sleep(prompt === 'first' ? 50 : 5) - res.writeHead(200, { 'Content-Type': 'application/json', 'X-Tangle-Cost-USD': prompt === 'first' ? '0.01' : '0.02' }) - res.end(JSON.stringify({ choices: [{ message: { role: 'assistant', content: prompt } }], usage: { prompt_tokens: 2, completion_tokens: 1, total_tokens: 3 } })) -}) -server.listen(0, '127.0.0.1') -await once(server, 'listening') -const options = { baseUrl: `http://127.0.0.1:${server.address().port}/v1`, apiKey: 'local-proof-only', maxRetries: 0, retryBaseMs: 1 } -const client = createTangleRouterClient(options) -try { - const answers = await Promise.all(['first', 'second'].map(content => client.chat([{ role: 'user', content }]))) - assert.deepEqual(answers, ['first', 'second']) - const hits = await client.search('proof', { maxResults: 2 }) - assert.equal(hits.length, 1) - assert.ok(Math.abs(client.usage().usd - 0.033) < 1e-10, 'concurrent requests double-counted cost') - assert.equal(client.usage().promptTokens, 4) - const unbilled = createTangleRouterClient(options) - assert.equal(await unbilled.chat([{ role: 'user', content: 'unbilled' }]), 'unbilled') - assert.equal(Number.isNaN(unbilled.usage().usd), true, 'rate estimate became a billing receipt') - await assert.rejects(client.chat([{ role: 'user', content: 'denied' }]), error => error instanceof RouterError && error.status === 401) - const cancellable = createTangleRouterClient({ ...options, signal: abort.signal }) - await assert.rejects(cancellable.chat([{ role: 'user', content: 'cancel' }]), { name: 'AbortError' }) - for (let i = 0; i < 50 && !cancelledSocket; i++) await sleep(10) - assert.equal(cancelledSocket, true, 'aborted caller left the HTTP request alive') - assert.ok(observed.every(request => request.client?.startsWith('tcloud-sdk/') && !request.hasSignalField)) - console.log(JSON.stringify({ proof: 'research-adapter-real-http', observed, usage: client.usage(), missingReceiptIsNaN: Number.isNaN(unbilled.usage().usd), cancelledSocket, errorFacade: 'RouterError(401)' }, null, 2)) -} finally { - for (const timer of timers) clearTimeout(timer) - server.closeAllConnections() - await new Promise(resolve => server.close(resolve)) -} diff --git a/scripts/verify-package.mjs b/scripts/verify-package.mjs index 8148516..2bf862f 100644 --- a/scripts/verify-package.mjs +++ b/scripts/verify-package.mjs @@ -1,6 +1,5 @@ import { spawnSync } from 'node:child_process' import { - existsSync, mkdirSync, mkdtempSync, readFileSync, @@ -30,23 +29,6 @@ const requiredRootExports = [ 'runRagOptimization', 'runRetrievalImprovementLoop', 'runSerializedKnowledgeOptimization', - 'runVerifiedResearchLoop', -] -const forbiddenRootExports = [ - 'boundedRetrievalConfigMethod', - 'buildBoundedRetrievalConfigs', - 'buildRetrievalParameterCandidates', - 'retrievalParameterSweepProposer', - 'runTwoAgentResearchLoop', - 'TwoAgentResearchLoopOptions', - 'TwoAgentResearchLoopResult', - 'TwoAgentResearchRound', -] -const forbiddenDeclarationNames = [ - 'runTwoAgentResearchLoop', - 'TwoAgentResearchLoopOptions', - 'TwoAgentResearchLoopResult', - 'TwoAgentResearchRound', ] // Packages that patch a Node builtin at MODULE scope. `graceful-fs` assigns // `fs.close` / `fs.closeSync`; workerd exposes those as getter-only accessors, @@ -128,19 +110,7 @@ try { const installedPackage = JSON.parse( readFileSync(join(installedPackageDir, 'package.json'), 'utf8'), ) - if (existsSync(join(installedPackageDir, 'dist', 'two-agent-research-loop.d.ts'))) { - throw new Error('published package includes the removed two-agent research module') - } - const installedDeclaration = readFileSync( - join(installedPackageDir, 'dist', 'index.d.ts'), - 'utf8', - ) assertPortableDeclarations(join(installedPackageDir, 'dist')) - for (const name of forbiddenDeclarationNames) { - if (installedDeclaration.includes(name)) { - throw new Error(`published declarations include obsolete export: ${name}`) - } - } assertNoEdgeUnsafeStaticImports(join(installedPackageDir, 'dist')) const installedAgentEval = JSON.parse( readFileSync( @@ -228,9 +198,6 @@ try { `for (const name of ${JSON.stringify(requiredRootExports)}) {`, ` if (typeof root[name] !== 'function') throw new Error('missing root export: ' + name)`, `}`, - `for (const name of ${JSON.stringify(forbiddenRootExports)}) {`, - ` if (name in root) throw new Error('obsolete root export: ' + name)`, - `}`, `const memory = await import(${JSON.stringify(`${packageName}/memory`)})`, `for (const name of ${JSON.stringify(requiredMemoryExports)}) {`, ` if (typeof memory[name] !== 'function') throw new Error('missing memory export: ' + name)`, diff --git a/src/adaptive-driver.ts b/src/adaptive-driver.ts deleted file mode 100644 index f5c2897..0000000 --- a/src/adaptive-driver.ts +++ /dev/null @@ -1,401 +0,0 @@ -/** - * Adaptive verifier mode for `runVerifiedResearchLoop`. - * - * The cost/quality A/B (`docs/results/cost-quality.md`) found the LLM relevance - * verifier's cleanliness win is dominated by DE-DUPLICATION — which a - * deterministic content-hash / canonical-URL check captures at ~none of the LLM - * premium — and that an LLM check only earns its dollar on the off-scope tail. - * The honest production move it names is: do the cheap deterministic work first, - * spend the LLM only where it pays. This module is that driver. - * - * Per candidate source the adaptive driver runs THREE stages, cheapest first, - * and stops at the first that decides: - * - * 1. DEDUP ($0, no LLM). Reject a source whose CONTENT (normalized-text hash) - * or whose CANONICAL URL matches one already accepted this round or already - * in the knowledge base. This is the de-dup the relevance judge was being - * paid to do; doing it deterministically is free and exact. - * - * 2. HEURISTIC TRIAGE ($0, no LLM). For a unique survivor, a cheap host / - * title / length signal classifies it as clearly-keep, clearly-drop, or - * AMBIGUOUS. Clear cases are resolved without a model: an authoritative host - * (arxiv, *.edu, *.gov, official docs) with a substantial readable body is - * kept; an obvious spam/listicle/marketing title or a too-thin body is - * dropped. Only genuinely ambiguous survivors fall through. - * - * 3. LLM ESCALATION ($, one call). ONLY the ambiguous survivors reach the LLM - * `verifySource` — the shipped `createVerifyingResearchDriver` relevance - * judge. This is where the verifier earns its premium: the off-scope tail a - * cheap rule can't adjudicate. - * - * The result is the cost/quality frontier point the doc predicted: most of the - * cleanliness (dedup + clear drops) at a fraction of the LLM $/calls (only the - * ambiguous tail pays). It is a real `ResearchDriver` — same contract the - * two-agent loop already gates on — and reuses `sha256`, the relevance verifier, - * and the index; it reinvents none of them. - */ - -import { sha256 } from './ids' -import type { - ResearchSourceProposal, - SourceVerdict, - SourceVerificationContext, -} from './verified-research-loop' -import { - createVerifyingResearchDriver, - type RouterClient, - type TangleRouterOptions, - type VerifyingDriverOptions, -} from './web-research-worker' - -/** - * Canonicalize a URL for duplicate detection: lowercase host, strip a leading - * `www.`, drop the scheme, the fragment, a trailing slash, and tracking query - * params (`utm_*`, `ref`, `fbclid`, `gclid`, …). Two URLs that differ only by - * those decorations canonicalize to the same key, so the dedup stage treats them - * as the same source. Falls back to the lowercased raw string when the input is - * not a parseable absolute URL (so non-http identifiers still dedup by equality). - */ -export function canonicalizeUrl(uri: string): string { - const trimmed = uri.trim() - try { - const url = new URL(trimmed) - const host = url.hostname.toLowerCase().replace(/^www\./, '') - // Keep only non-tracking query params, sorted for stable ordering. - const kept: [string, string][] = [] - for (const [key, value] of url.searchParams) { - const lower = key.toLowerCase() - if (lower.startsWith('utm_')) continue - if (trackingParams.has(lower)) continue - kept.push([key, value]) - } - kept.sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)) - const query = kept.map(([k, v]) => `${k}=${v}`).join('&') - const path = url.pathname.replace(/\/+$/, '') || '/' - return `${host}${path}${query ? `?${query}` : ''}` - } catch { - return trimmed.toLowerCase() - } -} - -/** Tracking / referrer query params dropped during URL canonicalization. */ -const trackingParams = new Set([ - 'ref', - 'ref_src', - 'source', - 'fbclid', - 'gclid', - 'mc_cid', - 'mc_eid', - 'igshid', - 'spm', - '_hsenc', - '_hsmi', -]) - -/** - * A stable content key for a fetched page: the sha256 of its normalized text - * (lowercased, punctuation/whitespace collapsed). Two pages whose readable body - * is the same modulo formatting collide here, so the dedup stage rejects a - * mirror/syndication of an already-accepted source even when the URL differs. - */ -export function contentKey(text: string): string { - const normalized = text - .toLowerCase() - .replace(/[^\p{L}\p{N}\s]+/gu, ' ') - .replace(/\s+/g, ' ') - .trim() - return sha256(normalized) -} - -/** Why the deterministic dedup stage rejected a candidate (for audit/notes). */ -export type DedupReason = 'duplicate-url' | 'duplicate-content' - -/** The cheap-triage classification of a unique survivor. */ -export type TriageClass = 'keep' | 'drop' | 'ambiguous' - -/** One source's adaptive routing decision, for instrumentation and the doc. */ -export interface AdaptiveDecision { - uri: string - /** The stage that decided this source: dedup | heuristic | llm. */ - stage: 'dedup' | 'heuristic' | 'llm' - accepted: boolean - /** The triage class assigned (set once past dedup). */ - triage?: TriageClass - reason?: string -} - -/** Running tally of where the adaptive driver spent its decisions. */ -export interface AdaptiveStats { - total: number - /** Rejected by deterministic dedup (URL or content). $0. */ - dedupRejected: number - /** Kept by the cheap heuristic without an LLM call. $0. */ - heuristicKept: number - /** Dropped by the cheap heuristic without an LLM call. $0. */ - heuristicDropped: number - /** Escalated to the LLM relevance verifier ($ — the only paid stage). */ - llmCalls: number - /** Of the escalations, how many the LLM accepted. */ - llmAccepted: number - decisions: AdaptiveDecision[] -} - -export interface AdaptiveDriverOptions { - /** Router client for the LLM escalation. Defaults to a live client from env. */ - router?: RouterClient - router_options?: TangleRouterOptions - /** Passed through to the escalation relevance verifier. */ - verifying?: Pick - /** - * Hosts an authoritative source lives on. A unique survivor on one of these, - * with a substantial body, is KEPT deterministically (no LLM). Suffix-matched - * against the canonical host, so `arxiv.org` matches `export.arxiv.org`. The - * defaults cover papers, official docs, and standards bodies. - */ - authoritativeHosts?: string[] - /** - * Title/snippet patterns that mark obvious spam / listicle / marketing — a - * unique survivor matching one is DROPPED deterministically (no LLM). - */ - spamPatterns?: RegExp[] - /** - * Below this many readable chars a survivor is too thin to be a real reference - * and is dropped deterministically. Default 400. - */ - minBodyChars?: number - /** - * A survivor whose body is at or above this many chars AND on an authoritative - * host is kept without an LLM call. Default 600. - */ - substantialBodyChars?: number - /** Receives each routing decision as it is made (for live instrumentation). */ - onDecision?: (decision: AdaptiveDecision) => void -} - -/** Default authoritative host suffixes — papers, official docs, standards. */ -const defaultAuthoritativeHosts = [ - 'arxiv.org', - 'aclanthology.org', - 'openreview.net', - 'dl.acm.org', - 'ieeexplore.ieee.org', - 'nature.com', - 'science.org', - 'pubmed.ncbi.nlm.nih.gov', - 'ncbi.nlm.nih.gov', - '.edu', - '.gov', - 'docs.python.org', - 'pytorch.org', - 'tensorflow.org', - 'huggingface.co', - 'github.com', - 'developer.mozilla.org', - 'wikipedia.org', - 'w3.org', - 'ietf.org', - 'rfc-editor.org', -] - -/** Default spam/listicle/marketing title patterns. */ -const defaultSpamPatterns = [ - /\bbuy\b.*\b(cheap|now|deal|sale|discount)\b/i, - /\b\d+\s+(things|ways|reasons|tips|tricks|secrets|hacks)\b.*\b(you|that|will)\b/i, - /\bshock(ing|ed)?\b/i, - /\bclickbait\b/i, - /!!!|\$\$\$/, - /\b(coupon|promo code|affiliate|sponsored)\b/i, - /\bbest .* (of \d{4}|in \d{4})\b/i, -] - -/** - * Classify a UNIQUE survivor (already past dedup) with cheap host/title/length - * signals only — no LLM. Returns `keep`, `drop`, or `ambiguous`. `ambiguous` is - * the residue the LLM is reserved for: on-topic-looking pages on unknown hosts - * with a plausible body, which a host/title rule cannot adjudicate. - */ -export function triageSource( - source: ResearchSourceProposal, - options: { - authoritativeHosts: string[] - spamPatterns: RegExp[] - minBodyChars: number - substantialBodyChars: number - }, -): { triage: TriageClass; reason: string } { - const titleAndSnippet = `${source.title ?? ''} ${ - typeof source.metadata?.snippet === 'string' ? source.metadata.snippet : '' - }`.trim() - const bodyLen = source.text.trim().length - - // Clear DROP: obvious spam/listicle/marketing title, OR a too-thin body that - // can't be a real reference. - for (const pattern of options.spamPatterns) { - if (pattern.test(titleAndSnippet)) { - return { triage: 'drop', reason: `spam/listicle title (${pattern.source})` } - } - } - if (bodyLen < options.minBodyChars) { - return { triage: 'drop', reason: `thin body (${bodyLen} < ${options.minBodyChars} chars)` } - } - - // Clear KEEP: an authoritative host with a substantial readable body. The host - // is a strong prior for a real reference; the length rules out a stub page. - const host = hostOf(source.uri) - const authoritative = - host.length > 0 && options.authoritativeHosts.some((suffix) => hostMatches(host, suffix)) - if (authoritative && bodyLen >= options.substantialBodyChars) { - return { triage: 'keep', reason: `authoritative host ${host} + substantial body (${bodyLen})` } - } - - // Everything else is AMBIGUOUS — an unknown host with a plausible body. This - // is exactly the off-scope tail the LLM relevance judge is reserved for. - return { triage: 'ambiguous', reason: `unknown host ${host || '(none)'}, body ${bodyLen} chars` } -} - -function hostOf(uri: string): string { - try { - return new URL(uri.trim()).hostname.toLowerCase().replace(/^www\./, '') - } catch { - return '' - } -} - -/** Suffix host match: `.edu` matches `mit.edu`; `arxiv.org` matches `export.arxiv.org`. */ -function hostMatches(host: string, suffix: string): boolean { - if (suffix.startsWith('.')) return host.endsWith(suffix) || host === suffix.slice(1) - return host === suffix || host.endsWith(`.${suffix}`) -} - -export interface AdaptiveResearchDriver { - verifySource( - source: ResearchSourceProposal, - ctx: SourceVerificationContext, - ): Promise - /** Live tally of where decisions were spent — the cost/quality instrumentation. */ - stats(): AdaptiveStats -} - -/** - * Build the adaptive verifier. The deterministic stages (dedup + heuristic - * triage) cost $0; only AMBIGUOUS survivors escalate to the LLM relevance - * verifier. `stats()` exposes where every decision was spent so the A/B can read - * the LLM $/calls the adaptive driver saved against the cleanliness it kept. - * - * Dedup state is kept on the driver instance: it tracks the canonical URLs and - * content hashes it has ACCEPTED, plus those it sees in the verification - * context's `acceptedThisRound` and the KB index. Use one driver per loop run. - */ -export function createAdaptiveResearchDriver( - options: AdaptiveDriverOptions = {}, -): AdaptiveResearchDriver { - const authoritativeHosts = options.authoritativeHosts ?? defaultAuthoritativeHosts - const spamPatterns = options.spamPatterns ?? defaultSpamPatterns - const minBodyChars = Math.max(1, options.minBodyChars ?? 400) - const substantialBodyChars = Math.max(minBodyChars, options.substantialBodyChars ?? 600) - - // The shipped LLM relevance verifier — the ONLY paid stage, reused, not rebuilt. - const relevance = createVerifyingResearchDriver({ - router: options.router, - router_options: options.router_options, - acceptOnParseFailure: options.verifying?.acceptOnParseFailure, - }) - - // Dedup memory across the run: every URL/content key this driver ACCEPTED. - const acceptedUrlKeys = new Set() - const acceptedContentKeys = new Set() - - const stats: AdaptiveStats = { - total: 0, - dedupRejected: 0, - heuristicKept: 0, - heuristicDropped: 0, - llmCalls: 0, - llmAccepted: 0, - decisions: [], - } - - function record(decision: AdaptiveDecision): void { - stats.decisions.push(decision) - options.onDecision?.(decision) - } - - return { - async verifySource(source, ctx): Promise { - stats.total += 1 - const urlKey = canonicalizeUrl(source.uri) - const cKey = contentKey(source.text) - - // ---- STAGE 1: DETERMINISTIC DEDUP ($0) ---------------------------------- - // Seed the dup sets from what the loop already accepted this round AND from - // the KB index, so a duplicate of an existing source is caught even if this - // driver instance hasn't seen it yet. - const roundUrlKeys = new Set( - ctx.acceptedThisRound.map((accepted) => canonicalizeUrl(accepted.uri)), - ) - const roundContentKeys = new Set( - ctx.acceptedThisRound.map((accepted) => contentKey(accepted.text)), - ) - const indexUrlKeys = new Set( - ctx.index.sources.flatMap((indexed) => - typeof indexed.metadata?.originalUri === 'string' - ? [canonicalizeUrl(indexed.metadata.originalUri)] - : [], - ), - ) - const dupUrl = - acceptedUrlKeys.has(urlKey) || roundUrlKeys.has(urlKey) || indexUrlKeys.has(urlKey) - const dupContent = acceptedContentKeys.has(cKey) || roundContentKeys.has(cKey) - if (dupUrl || dupContent) { - stats.dedupRejected += 1 - const reason: DedupReason = dupUrl ? 'duplicate-url' : 'duplicate-content' - record({ uri: source.uri, stage: 'dedup', accepted: false, reason }) - return { accept: false, reason: `dedup: ${reason}` } - } - - // ---- STAGE 2: CHEAP HEURISTIC TRIAGE ($0) ------------------------------- - const { triage, reason } = triageSource(source, { - authoritativeHosts, - spamPatterns, - minBodyChars, - substantialBodyChars, - }) - if (triage === 'keep') { - stats.heuristicKept += 1 - acceptedUrlKeys.add(urlKey) - acceptedContentKeys.add(cKey) - record({ uri: source.uri, stage: 'heuristic', accepted: true, triage, reason }) - return { accept: true } - } - if (triage === 'drop') { - stats.heuristicDropped += 1 - record({ uri: source.uri, stage: 'heuristic', accepted: false, triage, reason }) - return { accept: false, reason: `heuristic drop: ${reason}` } - } - - // ---- STAGE 3: LLM ESCALATION ($ — ambiguous tail only) ------------------ - stats.llmCalls += 1 - const verdict = await relevance.verifySource(source, ctx) - if (verdict.accept) { - stats.llmAccepted += 1 - acceptedUrlKeys.add(urlKey) - acceptedContentKeys.add(cKey) - } - record({ - uri: source.uri, - stage: 'llm', - accepted: verdict.accept, - triage, - reason: verdict.accept ? 'llm accepted' : verdict.reason, - }) - return verdict - }, - stats(): AdaptiveStats { - return { - ...stats, - decisions: [...stats.decisions], - } - }, - } -} diff --git a/src/claim-grounding.ts b/src/claim-grounding.ts index f4c5e15..593a45a 100644 --- a/src/claim-grounding.ts +++ b/src/claim-grounding.ts @@ -1,71 +1,10 @@ /** - * Claim-grounding mode for `runVerifiedResearchLoop`. + * Deterministic claim grounding: is a claim present in a text? * - * The two-agent loop's existing verifier judges a source's on-topic RELEVANCE - * (is this page about the goal?). On the topic sets we have measured, its - * cleanliness win is dominated by DE-DUPLICATION — which a deterministic - * content-hash / canonical-URL check captures at ~none of the LLM premium (see - * `docs/results/cost-quality.md`). That makes the LLM verifier look expensive - * for what a cheap rule already does. - * - * Claim-grounding targets a DIFFERENT, harder error band: a citation that is - * relevant and unique but **misattributed** — the page is on-topic, the URL is - * real, yet the specific CLAIM the source is cited for does NOT actually appear - * in the page. This is the citation-fabrication failure mode of LLM research: - * the model writes a plausible sentence and hangs a real URL off it that never - * says any such thing. Neither de-dup nor a relevance judge catches it (both can - * pass a misattributed-but-on-topic page); only checking the claim against the - * fetched text does. - * - * The check is EXECUTABLE GROUND TRUTH, not another LLM opinion: the worker - * attaches the specific claim it is citing the source for, and the verifier - * tests whether that claim is PRESENT (verbatim, normalized, or as a sufficient - * content-word overlap / close paraphrase) in the `htmlToText` output of the - * page the worker actually fetched. A claim that is not grounded is rejected as - * misattributed. Because the oracle is deterministic text presence — not a model - * call — it is a deployable, non-oracle verifier: it can run in production with - * zero inference cost, OR be composed with the LLM relevance verifier so the - * loop rejects BOTH off-topic AND misattributed sources. - * - * This module is content-free and any-topic: it adds (1) a way for a proposal to - * carry the claim it is cited for, (2) the `groundClaimInText` oracle, and (3) a - * `ResearchDriver` that gates on grounding. It composes the existing - * `ResearchDriver` / `ResearchSourceProposal` contracts and the shipped - * `htmlToText`; it reinvents none of them. + * The check is text presence, not a model call. RAG answer scoring uses it to + * test whether a generated claim is supported by the retrieved context. */ -import type { - ResearchSourceProposal, - SourceVerdict, - SourceVerificationContext, -} from './verified-research-loop' -import { - createTangleRouterClient, - type RouterClient, - type TangleRouterOptions, -} from './web-research-worker' - -/** - * Metadata key under which a proposal carries the specific claim it is cited - * for. The worker sets `metadata[citedClaimKey] = ''`; the - * claim-grounding driver reads it and checks it against the fetched page text. - */ -export const citedClaimKey = 'citedClaim' - -/** Read the cited claim a proposal carries, if any. */ -export function citedClaimOf(source: ResearchSourceProposal): string | undefined { - const claim = source.metadata?.[citedClaimKey] - return typeof claim === 'string' && claim.trim() ? claim.trim() : undefined -} - -/** Attach a cited claim to a proposal (immutably returns a new proposal). */ -export function withCitedClaim( - source: ResearchSourceProposal, - claim: string, -): ResearchSourceProposal { - return { ...source, metadata: { ...source.metadata, [citedClaimKey]: claim } } -} - export interface GroundingResult { /** True when the claim is sufficiently present in the page text. */ grounded: boolean @@ -170,8 +109,7 @@ function contentWords(claim: string, minWordLength: number): string[] { } /** - * THE ORACLE. Is `claim` grounded in `pageText` (the `htmlToText` output of the - * page the worker fetched)? Deterministic, no model call: + * Is `claim` grounded in `pageText`? Deterministic, no model call: * * 1. verbatim — the claim string appears as-is (case-insensitive). * 2. normalized — the claim appears after collapsing punctuation/whitespace @@ -180,9 +118,8 @@ function contentWords(claim: string, minWordLength: number): string[] { * substantive content words appear in the page text. A misattributed page * fails here because the SPECIFIC words the claim asserts are absent. * - * Returns the match mode, the measured overlap, and the missing content words — - * enough for the driver to give a precise rejection reason and for a test/doc to - * audit WHY a claim grounded or didn't. + * Returns the match mode, the measured overlap, and the missing content words, + * so a caller can state why a claim grounded or did not. */ export function groundClaimInText( claim: string, @@ -222,127 +159,3 @@ export function groundClaimInText( const grounded = overlap >= minOverlap return { grounded, mode: grounded ? 'overlap' : 'absent', overlap, missingWords } } - -export interface ClaimGroundingDriverOptions extends GroundClaimOptions { - /** - * Optional second verifier to compose AFTER grounding passes. When set, a - * source must BOTH ground its claim AND pass this verifier (e.g. the LLM - * relevance driver's `verifySource`). Lets the loop reject off-topic AND - * misattributed sources in one driver. Omit for the pure, zero-inference - * grounding gate. - */ - relevanceVerifier?: ( - source: ResearchSourceProposal, - ctx: SourceVerificationContext, - ) => Promise | SourceVerdict - /** - * What to do when a proposal carries NO cited claim. `'reject'` (default) is - * fail-closed: in claim-grounding mode every source must declare what it is - * cited for, so an un-annotated source is treated as ungrounded. `'accept'` - * lets unannotated sources through to the relevance verifier, if present. - */ - onMissingClaim?: 'reject' | 'accept' -} - -/** - * A `ResearchDriver`-shaped verifier (just the `verifySource` arm) that gates on - * CLAIM GROUNDING: it rejects a source whose cited claim is not present in its - * fetched page text — a misattributed / fabricated citation — and (optionally) - * composes a relevance verifier after grounding passes. - * - * The returned function matches `ResearchDriver['verifySource']`, so it drops - * straight into `runVerifiedResearchLoop` as `{ verifySource: createClaimGroundingVerifier(...) }`. - */ -export function createClaimGroundingVerifier(options: ClaimGroundingDriverOptions = {}) { - const onMissingClaim = options.onMissingClaim ?? 'reject' - return async function verifySource( - source: ResearchSourceProposal, - ctx: SourceVerificationContext, - ): Promise { - const claim = citedClaimOf(source) - if (!claim) { - if (onMissingClaim === 'reject') { - return { - accept: false, - reason: 'no cited claim: claim-grounding mode requires every source to declare its claim', - } - } - return options.relevanceVerifier ? options.relevanceVerifier(source, ctx) : { accept: true } - } - - const grounding = groundClaimInText(claim, source.text, options) - if (!grounding.grounded) { - const detail = - grounding.mode === 'empty-text' - ? 'fetched page has no text' - : `claim not found in the fetched page (overlap ${(grounding.overlap * 100).toFixed(0)}%${ - grounding.missingWords.length - ? `, missing: ${grounding.missingWords.slice(0, 6).join(', ')}` - : '' - })` - return { - accept: false, - reason: `misattributed citation: ${detail}`, - } - } - - if (options.relevanceVerifier) return options.relevanceVerifier(source, ctx) - return { accept: true } - } -} - -export interface WorkerClaimDecorationOptions { - router?: RouterClient - router_options?: TangleRouterOptions - /** Max output tokens for the claim-extraction call. Default 1200 (glm floor). */ - maxTokens?: number -} - -/** - * Ask an LLM to state, for one source, the single specific factual claim a - * researcher would cite THIS page for toward the goal. Used to DECORATE the - * sources a relevance-only worker produced with the claim a citation would make, - * so the claim-grounding verifier has something executable to check. The model - * is told to ground the claim in the provided excerpt; the verifier then checks - * it against the FULL page text independently — the model does not get to mark - * its own homework. - * - * Returns the proposal annotated via `withCitedClaim`, or the original proposal - * unchanged if the model returns nothing parseable (the verifier's - * `onMissingClaim` policy then decides). - */ -export function createClaimDecorator(options: WorkerClaimDecorationOptions = {}) { - const maxTokens = options.maxTokens ?? 1200 - return async function decorate( - source: ResearchSourceProposal, - goal: string, - ): Promise { - const router = options.router ?? createTangleRouterClient(options.router_options) - const excerpt = source.text.slice(0, 1500) - const system = - 'You extract the single most important factual claim a researcher would cite a page for. ' + - "State ONE concrete, checkable sentence using the page's own key terms and numbers. " + - 'Do NOT invent facts not in the excerpt. Respond with ONLY the claim sentence, no prose.' - const user = [ - `Research goal: ${goal}`, - `Page title: ${source.title ?? '(none)'}`, - `Page excerpt:\n${excerpt}`, - 'The single claim this page should be cited for:', - ].join('\n\n') - let raw = '' - try { - raw = await router.chat( - [ - { role: 'system', content: system }, - { role: 'user', content: user }, - ], - maxTokens, - ) - } catch { - return source - } - const claim = raw.trim().split('\n')[0]?.trim() - if (!claim) return source - return withCitedClaim(source, claim) - } -} diff --git a/src/claim-ledger.ts b/src/claim-ledger.ts index 6ca1c06..e16af96 100644 --- a/src/claim-ledger.ts +++ b/src/claim-ledger.ts @@ -2,9 +2,8 @@ * The algebra of a research claim ledger: claim identity, and how two ledgers * that accumulated evidence for the same goal combine into one. * - * This lives apart from `research-driving-driver.ts` because it is no longer - * that driver's private business. A ledger is a durable record now, and a - * durable record addressed by id is a record two writers can reach: two rounds + * A ledger is a durable record, and a durable record addressed by id is a + * record two writers can reach: two rounds * of one run resuming from disk, or two workers researching one goal in * parallel. `putClaimLedger` writes the whole record, so the second writer's * write erases the first writer's claims — the ledger persists and the @@ -588,9 +587,8 @@ function mergeSourceVersions( * of it: the worker that found the refuting source records "X contradicts Y" and * knows nothing about Y's record. Left one-sided, Y reads as an uncontested * claim, and the completion oracle would settle a question two sources disagree - * about. `createResearchDrivingDriver` does this pairwise as it records; this is - * the same rule stated over a whole ledger, for writers that assemble one from - * events rather than from a live loop. + * about. This rule applies over a whole ledger, for writers that assemble one + * from events. * * One-sided observations stay in `claimEvidence` until both claims are backed * by registered source versions. The materialized claim projection contains diff --git a/src/collection-research-driver.ts b/src/collection-research-driver.ts deleted file mode 100644 index 8be9d14..0000000 --- a/src/collection-research-driver.ts +++ /dev/null @@ -1,46 +0,0 @@ -/** - * The SINGLE-AGENT COLLECTION driver — the blind-collection baseline (Arm A). - * - * This is the honest null the depth A/B is measured against. The other drivers - * spend extra inference to do something differentiated: - * - `createVerifyingResearchDriver` runs an LLM gate per source (Arm B), - * - `createResearchDrivingDriver` extracts claims, tracks corroboration, and - * synthesizes deep follow-up questions to drive depth (Arm C). - * - * This driver does NONE of that. It is a pass-through: it accepts every source - * the worker proposes and contributes no research, no gating, and no steering of - * its own. The loop still dedups exact-uri duplicates before calling - * `verifySource` (that is the loop's job, not the driver's), and the default - * `foldGaps` (a plain bulleted list of the still-open readiness gaps) still folds - * the gaps into the worker's next prompt — so the worker keeps researching, but - * NOTHING intelligent sits between the worker and the knowledge base. - * - * In other words: ONE agent (the worker) collects sources round after round, and - * the "driver" is an inert rubber stamp. That is exactly what "single-agent - * collection" means — the topology with zero coordinator intelligence — so its - * material-facts score is the floor every other arm must beat to justify its - * extra inference cost. - * - * It adds NO router calls of its own: `verifySource` is a synchronous accept and - * `foldGaps` is omitted so the loop uses its built-in gap list. So Arm A's cost - * is the worker's cost alone — the cleanest possible blind-collection baseline. - */ - -import type { - ResearchDriver, - ResearchSourceProposal, - SourceVerdict, -} from './verified-research-loop' - -/** - * Build the single-agent collection driver. Accepts every source; never gates, - * never researches, never steers beyond the loop's default open-gap list. The - * worker is the only agent that thinks. - */ -export function createCollectionResearchDriver(): ResearchDriver { - return { - verifySource(_source: ResearchSourceProposal): SourceVerdict { - return { accept: true } - }, - } -} diff --git a/src/index.ts b/src/index.ts index ae5de7d..0c3de0e 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,5 +1,4 @@ export * from './adapters' -export * from './adaptive-driver' export * from './agent-candidate' export * from './benchmarks/index' export * from './changes' @@ -9,7 +8,6 @@ export * from './citation-resolution' export * from './claim-evidence' export * from './claim-grounding' export * from './claim-ledger' -export * from './collection-research-driver' export * from './discovery' // Atomic, fsync-durable, symlink-safe writes. Exported because every consumer // that keeps a journal needs a write that survives `kill -9`, and the only @@ -44,8 +42,6 @@ export * from './ids' export * from './indexer' export * from './inspect' export * from './invalidation-propagation' -export * from './investment-thesis-set' -export * from './investment-thesis-task' export * from './kb-improvement' export * from './kb-store' export { @@ -62,7 +58,6 @@ export * from './knowledge-tools' export * from './knowledge-use-receipts' export * from './lexical-index' export * from './lint' -export * from './material-facts-metric' export * from './memory/index' export type { KnowledgeMutationHold, @@ -92,7 +87,6 @@ export * from './rag-optimization' export * from './readiness-check' export * from './relation-graph' export * from './release' -export * from './research-driving-driver' export * from './research-loop' export * from './retrieval-eval' export * from './retrieval-optimization' @@ -115,8 +109,6 @@ export * from './sources/index' export * from './store' export * from './types' export * from './validate' -export * from './verified-research-loop' -export * from './web-research-worker' export * from './wikilinks' export * from './write-intake' export * from './write-protocol' diff --git a/src/investment-thesis-set.ts b/src/investment-thesis-set.ts deleted file mode 100644 index 2f272be..0000000 --- a/src/investment-thesis-set.ts +++ /dev/null @@ -1,903 +0,0 @@ -/** - * HELD-OUT INVESTMENT-RESEARCH EVAL SET. - * - * The point of this file is the same FIREWALL the deep-question exam uses - * (`tests/loops/held-out-exam.ts`): the material facts and their checkable - * fragments are NEVER shown to a research loop. A loop is told only the company - * + ticker + a research-as-of CUTOFF date, and asked to write an investment - * thesis. AFTER it finishes, we grade the thesis it produced against THESE - * facts — facts it never saw — so a high score is thesis QUALITY (it surfaced - * the buried, material, non-obvious drivers) and not teaching-to-the-test. - * - * Each fact is a DEPTH fact by construction: a single ticker / company-name web - * search does NOT surface it. They are the things buried in the filings or - * knowable from then-available primary sources that a thorough analyst flags and - * a one-shot search misses — customer/revenue/deposit concentration, a debt - * maturity wall, a margin-trend reversal, a governance / related-party item, a - * specific competitive or regulatory risk, an off-balance-sheet loss. - * - * THREE HARD RULES, enforced by how the data was gathered (see - * docs/eval/investment-material-facts.md for the per-item provenance + the - * curation-bias disclosure): - * - * 1. SPECIFIC + CHECKABLE. Each fact carries `expected` keyword groups — the - * specific number / name / phrase — so a deterministic, model-free - * substring grader (`gradeFactAgainstText`) can score "did the thesis - * surface it". $0, reproducible, and it cannot leak the answer key into a - * model the loop could observe. - * - * 2. DERIVED FROM REAL FETCHED EVIDENCE. Every fact records the primary source - * it came from (`sourceUrl`, an SEC EDGAR 10-K) and the literal `evidence` - * value read out of that document. Nothing here is invented; an item that - * could not be independently sourced was DROPPED, not guessed (the drop log - * is in the doc). - * - * 3. KNOWABLE AT THE CUTOFF. Every fact was disclosed in, or computable from, a - * document available on or before the company's `cutoff` date. Post-cutoff - * hindsight (the eventual bankruptcy / collapse) is NOT a checklist item — - * it is recorded separately as `knownOutcome`, purely for the reader, and is - * never graded. - * - * Grading mirrors the deep-question exam exactly: a fact is SURFACED when the - * thesis text contains at least `minGroups` of its expected groups; a group is - * satisfied when ANY of its `anyOf` fragments appears (case-insensitive - * substring), so a faithful thesis phrased in its own words still grades as a - * hit. `anyOf` groups model synonyms; the load-bearing tokens are the specific - * numbers / names. - */ - -/** A required answer component: satisfied when any synonym fragment is present. */ -export interface ExpectedGroup { - /** Human label for the component (for the doc / audit). */ - label: string - /** Case-insensitive substring fragments; any one present satisfies the group. */ - anyOf: string[] -} - -/** Lens the fact belongs to — so a set can be checked for category coverage. */ -export type MaterialFactLens = - | 'concentration' // customer / revenue / deposit concentration - | 'leverage' // debt load / maturity wall / interest burden - | 'margin-trend' // gross/operating margin reversal - | 'liquidity' // cash burn / negative operating cash flow - | 'capital-return' // buyback / dividend draining the balance sheet - | 'governance' // dual-class / related-party / control item - | 'off-balance-sheet' // unrealized losses not in earnings/equity - | 'regulatory' // a specific regulatory / legal / recall exposure - -/** One held-out material fact with a checkable expected answer + its provenance. */ -export interface MaterialFact { - /** Stable id, `ticker/fN`. */ - id: string - /** Which analyst lens this fact exercises. For coverage + the doc. */ - lens: MaterialFactLens - /** - * The material fact, in plain words — for the doc/audit. NEVER shown to a loop. - * This is the thing a thorough analyst would flag and a ticker search misses. - */ - fact: string - /** - * The checkable answer as required keyword GROUPS. The thesis text must contain - * at least `minGroups` of these groups (default: all). A group is satisfied - * when ANY of its `anyOf` fragments appears (case-insensitive substring). - */ - expected: ExpectedGroup[] - /** - * Minimum number of `expected` groups the thesis must contain to count the - * fact SURFACED. Default = all groups (the strict bar). Lowered (and documented - * inline) only when the fact is genuinely satisfiable by a subset. - */ - minGroups?: number - /** - * PROVENANCE. The primary source URL this fact was read from — an SEC EDGAR - * 10-K primary document, fetched live during curation. - */ - sourceUrl: string - /** - * The literal value / phrase read out of `sourceUrl` that grounds the fact. - * This is the "cite the actual filing + the value" requirement — verbatim or - * near-verbatim from the filing, with the figure. - */ - evidence: string -} - -/** A company + the cutoff a loop researches as-of + its held-out material facts. */ -export interface CompanyEvalCase { - /** Ticker as of the cutoff. */ - ticker: string - /** Legal name as of the cutoff (what the loop is told to research). */ - company: string - /** SEC Central Index Key (CIK), zero-stripped — the EDGAR filer id. */ - cik: string - /** - * Research-as-of date (ISO). The loop must reason as if it is this date; every - * `evidence` value was knowable on or before it. >= 18 months before this set - * was curated, so the outcome is known but is NOT a checklist item. - */ - cutoff: string - /** Sector, for coverage / the curation-bias disclosure. */ - sector: string - /** - * The known POST-cutoff outcome — recorded for the reader ONLY, never graded. - * Keeping it out of `facts` is what makes the set hindsight-free. - */ - knownOutcome: string - /** The held-out material facts for this company. */ - facts: MaterialFact[] -} - -/** - * The eval set. 5 public companies, 5-8 held-out material facts each, every fact - * grounded in a primary SEC EDGAR 10-K filed on or before the cutoff. - * - * CURATION-BIAS DISCLOSURE (full version in the doc): all five are companies - * whose buried risks later materialized, because that is where the material-vs- - * surface distinction is sharpest AND where the figures are easy to verify after - * the fact. This biases the set toward downside risks (two of the eight lenses, - * concentration + leverage, dominate) and toward distressed names. A production - * eval would balance these with companies whose buried facts were POSITIVE - * drivers and with survivors. This set is honest about that and reports the lens - * distribution so the bias is measurable, not hidden. - */ -export const investmentThesisSet: CompanyEvalCase[] = [ - { - ticker: 'SIVB', - company: 'SVB Financial Group', - cik: '719739', - cutoff: '2023-02-24', - sector: 'Banking', - knownOutcome: - 'Failed in a deposit run and was placed in FDIC receivership on March 10, 2023; the holding company filed Chapter 11 on March 17, 2023.', - facts: [ - { - id: 'SIVB/f1', - lens: 'off-balance-sheet', - fact: 'Held-to-maturity (HTM) securities carried at $91.3B amortized cost had a fair value of only $76.2B — a ~$15.1B unrealized loss that, because the portfolio is HTM, never touched earnings or equity and sat only in the footnotes.', - expected: [ - { - label: 'HTM securities', - anyOf: ['held-to-maturity', 'held to maturity', 'htm'], - }, - { - label: 'large unrealized loss (~$15B) / fair value gap', - anyOf: [ - '15.1', - '15.2', - '$15 billion', - '15 billion', - '76,169', - '76.2 billion', - '91,321', - 'unrealized loss', - 'below amortized cost', - 'fair value', - ], - }, - ], - minGroups: 2, - sourceUrl: - 'https://www.sec.gov/Archives/edgar/data/719739/000071973923000021/sivb-20221231.htm', - evidence: - 'Balance sheet (Dec 31, 2022): "Held-to-maturity securities, at amortized cost ... 91,321" with parenthetical "(fair value of $ 76,169 ...)". The gap = $91,321M - $76,169M = ~$15,152M unrealized loss, disclosed only in the notes.', - }, - { - id: 'SIVB/f2', - lens: 'off-balance-sheet', - fact: "The HTM unrealized loss (~$15.1B) was roughly equal to the company's entire $16.0B total stockholders' equity — a mark-to-market wipeout hidden by HTM accounting.", - expected: [ - { - label: 'loss near/exceeds total equity', - anyOf: [ - 'equity', - 'capital', - 'insolvent', - 'wipe out', - 'exceeds', - 'nearly all', - 'tangible book', - '16,004', - '16 billion', - ], - }, - ], - sourceUrl: - 'https://www.sec.gov/Archives/edgar/data/719739/000071973923000021/sivb-20221231.htm', - evidence: - '"Total SVBFG stockholders\' equity 16,004" (Dec 31, 2022). The ~$15.15B HTM unrealized loss is ~95% of the $16.0B reported equity.', - }, - { - id: 'SIVB/f3', - lens: 'concentration', - fact: 'Estimated uninsured deposits in U.S. offices were $151.5B at year-end 2022 — the run-prone funding base; a high share of total deposits exceeded the FDIC limit.', - expected: [ - { - label: 'large uninsured deposit base', - anyOf: [ - 'uninsured deposit', - 'above the fdic', - 'exceed the fdic', - 'exceeds the fdic', - '151.5', - '$151 billion', - 'fdic insurance limit', - ], - }, - ], - sourceUrl: - 'https://www.sec.gov/Archives/edgar/data/719739/000071973923000021/sivb-20221231.htm', - evidence: - '"As of December 31, 2022 ... the amount of estimated uninsured deposits in U.S. offices that exceed the FDIC insurance limit were $151.5 billion".', - }, - { - id: 'SIVB/f4', - lens: 'margin-trend', - fact: 'Cheap noninterest-bearing demand deposits fell 20 percentage points in one year — to 47% of total deposits from 67% — meaning funding costs were set to rise sharply as clients moved to interest-bearing accounts.', - expected: [ - { - label: 'deposit mix shift to costlier funding', - anyOf: [ - 'noninterest-bearing', - 'non-interest-bearing', - 'noninterest bearing', - 'deposit mix', - 'funding cost', - 'cost of deposits', - 'interest-bearing', - '47 percent', - '20 percentage', - ], - }, - ], - sourceUrl: - 'https://www.sec.gov/Archives/edgar/data/719739/000071973923000021/sivb-20221231.htm', - evidence: - '"Noninterest-bearing demand deposits to total deposits decreased by 20 percentage points to 47 percent as of December 31, 2022, compared to ... 2021."', - }, - { - id: 'SIVB/f5', - lens: 'concentration', - fact: 'The deposit and loan base was concentrated in a single client type — the "innovation economy" (venture-backed technology and life-science startups) — so a downturn in venture funding would hit deposits and credit simultaneously.', - expected: [ - { - label: 'concentration in tech / startups / innovation economy', - anyOf: [ - 'innovation economy', - 'technology', - 'life science', - 'venture', - 'startup', - 'early-stage', - 'concentrat', - 'single industry', - 'sector concentration', - ], - }, - ], - sourceUrl: - 'https://www.sec.gov/Archives/edgar/data/719739/000071973923000021/sivb-20221231.htm', - evidence: - 'The 10-K repeatedly frames the franchise around clients in "the innovation economy" (technology, life science / healthcare, and the venture firms that back them) — a single-sector deposit + credit concentration.', - }, - { - id: 'SIVB/f6', - lens: 'off-balance-sheet', - fact: 'Available-for-sale (AFS) securities of $28.6B amortized cost were marked to a $26.1B fair value — a ~$2.5B loss that DID flow through equity (AOCI), the visible tip of a much larger unrealized-loss iceberg dominated by the footnote-only HTM book.', - expected: [ - { - label: 'AFS unrealized loss / AOCI', - anyOf: [ - 'available-for-sale', - 'available for sale', - 'afs', - 'aoci', - 'accumulated other comprehensive', - '28,602', - '26,069', - '2.5 billion', - ], - }, - ], - sourceUrl: - 'https://www.sec.gov/Archives/edgar/data/719739/000071973923000021/sivb-20221231.htm', - evidence: - '"Available-for-sale securities, at fair value (cost of $ 28,602 ...) 26,069" — a ~$2.5B AFS unrealized loss carried in AOCI, separate from and much smaller than the HTM gap.', - }, - ], - }, - { - ticker: 'BBBY', - company: 'Bed Bath & Beyond Inc.', - cik: '886158', - cutoff: '2022-04-21', - sector: 'Specialty retail', - knownOutcome: 'Filed for Chapter 11 bankruptcy on April 23, 2023; shareholders were wiped out.', - facts: [ - { - id: 'BBBY/f1', - lens: 'capital-return', - fact: 'The company had repurchased ~$11.685B of its own stock since 2004 — including $574.9M in fiscal 2021 alone, "two years ahead of schedule" — draining the balance sheet of a business that was losing money.', - expected: [ - { - label: 'massive buyback program', - anyOf: [ - 'repurchas', - 'buyback', - 'buy back', - 'share repurchase', - '11.685', - '$11.7 billion', - '574.9', - '$575 million', - ], - }, - ], - sourceUrl: - 'https://www.sec.gov/Archives/edgar/data/886158/000088615822000047/bbby-20220226.htm', - evidence: - '"Since 2004 through the end of Fiscal 2021, we have repurchased approximately $11.685 billion of our common stock"; FY2021 alone "completed share repurchases of $574.9 million ... two years ahead of schedule."', - }, - { - id: 'BBBY/f2', - lens: 'liquidity', - fact: 'Operating cash flow collapsed to just $17.9M in FY2021, down from $268.1M and $590.9M in the two prior years — a near-total loss of internally generated cash while it kept buying back stock.', - expected: [ - { - label: 'operating cash flow collapse', - anyOf: [ - 'operating cash flow', - 'cash from operations', - 'cash provided by operating', - 'cash flow from operations', - '17.9', - '17,854', - 'declining cash flow', - 'cash generation', - ], - }, - ], - sourceUrl: - 'https://www.sec.gov/Archives/edgar/data/886158/000088615822000047/bbby-20220226.htm', - evidence: - 'Statement of cash flows: "Net cash provided by operating activities 17,854 268,108 590,941" (FY2021 / FY2020 / FY2019, $ thousands).', - }, - { - id: 'BBBY/f3', - lens: 'liquidity', - fact: "Total shareholders' equity fell ~86% in one year — from $1.277B to $174.1M — as losses plus buybacks ate the equity cushion.", - expected: [ - { - label: 'equity erosion', - anyOf: [ - 'shareholders’ equity', - "shareholders' equity", - 'stockholders equity', - 'book value', - 'equity', - 'net worth', - '174.1', - '174,145', - 'eroded', - ], - }, - ], - sourceUrl: - 'https://www.sec.gov/Archives/edgar/data/886158/000088615822000047/bbby-20220226.htm', - evidence: - '"Total shareholders\' equity 174,145 1,276,936" (FY2021 vs FY2020, $ thousands) — an ~86% decline in one year.', - }, - { - id: 'BBBY/f4', - lens: 'liquidity', - fact: 'The company posted a $559.6M net loss in FY2021 — yet spent $574.9M on buybacks the same year, i.e. it returned more cash to shareholders than it had, let alone earned.', - expected: [ - { - label: 'net loss FY2021', - anyOf: [ - 'net loss', - 'unprofitable', - 'lost money', - 'losing money', - '559.6', - '559,623', - '$560 million', - ], - }, - ], - sourceUrl: - 'https://www.sec.gov/Archives/edgar/data/886158/000088615822000047/bbby-20220226.htm', - evidence: '"Net loss $ ( 559,623 )" for fiscal 2021 ($ thousands).', - }, - { - id: 'BBBY/f5', - lens: 'margin-trend', - fact: 'Merchandise inventories rose to $1.725B even as sales fell — inventory building into a demand decline, a classic markdown-risk and cash-trap signal.', - expected: [ - { - label: 'inventory building into falling demand', - anyOf: [ - 'inventor', - 'merchandise inventories', - 'overstock', - 'markdown', - '1,725', - '1.7 billion', - 'stockpile', - ], - }, - ], - sourceUrl: - 'https://www.sec.gov/Archives/edgar/data/886158/000088615822000047/bbby-20220226.htm', - evidence: - '"Merchandise inventories 1,725,410 1,671,909" ($ thousands) — inventory grew year over year while comparable sales declined.', - }, - ], - }, - { - ticker: 'CVNA', - company: 'Carvana Co.', - cik: '1690820', - cutoff: '2023-02-23', - sector: 'Auto e-commerce', - knownOutcome: - 'The stock fell ~98% from its 2021 peak; the company narrowly avoided bankruptcy via a 2023 debt-exchange that cut and extended its obligations.', - facts: [ - { - id: 'CVNA/f1', - lens: 'leverage', - fact: 'Total debt had grown to $8.39B by year-end 2022 (from $5.45B) — a debt load far larger than the equity base, built up funding growth and the ADESA deal.', - expected: [ - { - label: 'large/growing debt load', - anyOf: [ - 'total debt', - 'long-term debt', - 'leverage', - 'highly leveraged', - 'debt load', - '8,391', - '8.4 billion', - '$8.4 billion', - ], - }, - ], - sourceUrl: - 'https://www.sec.gov/Archives/edgar/data/1690820/000169082023000052/cvna-20221231.htm', - evidence: '"Total debt 8,391 5,447" (Dec 31, 2022 vs 2021, $ millions).', - }, - { - id: 'CVNA/f2', - lens: 'leverage', - fact: 'Interest expense nearly tripled to $486M in 2022 (from $176M) — debt-service was consuming cash a still-unprofitable company did not have.', - expected: [ - { - label: 'rising interest burden', - anyOf: [ - 'interest expense', - 'interest cost', - 'debt service', - 'cost of debt', - '486', - 'interest burden', - ], - }, - ], - sourceUrl: - 'https://www.sec.gov/Archives/edgar/data/1690820/000169082023000052/cvna-20221231.htm', - evidence: '"Interest expense 486 176" (FY2022 vs FY2021, $ millions).', - }, - { - id: 'CVNA/f3', - lens: 'leverage', - fact: "In May 2022 Carvana bought ADESA's U.S. physical auction business for ~$2.2B in cash — a debt-funded acquisition that stretched the balance sheet right as used-car demand turned.", - expected: [ - { - label: 'ADESA acquisition ~$2.2B', - anyOf: ['adesa', '2.2 billion', '$2.2 billion', 'physical auction', 'acquisition'], - }, - ], - sourceUrl: - 'https://www.sec.gov/Archives/edgar/data/1690820/000169082023000052/cvna-20221231.htm', - evidence: - '"physical auction business of ADESA US Auction, LLC for approximately $2.2 billion in cash (the \'ADESA Acquisition\')", closed 2022-05-09.', - }, - { - id: 'CVNA/f4', - lens: 'governance', - fact: 'Carvana leases hubs and properties from DriveTime — a company controlled by founder/CEO Ernest Garcia III and his father Ernest Garcia II — a recurring related-party arrangement with the controlling family.', - expected: [ - { - label: 'related-party with founder family / DriveTime', - anyOf: [ - 'related party', - 'related-party', - 'drivetime', - 'garcia', - 'controlled by', - 'affiliate of', - 'conflict of interest', - ], - }, - ], - sourceUrl: - 'https://www.sec.gov/Archives/edgar/data/1690820/000169082023000052/cvna-20221231.htm', - evidence: - 'Related Party Transactions note: lease agreements with "DriveTime Automotive Group", a related party "due to Ernest Garcia II, Ernest Garcia III, and entities controlled by one or both of them".', - }, - { - id: 'CVNA/f5', - lens: 'liquidity', - fact: 'The 2022 net loss was $2.894B — a loss far wider than prior years, showing the unit economics had not turned even at scale.', - expected: [ - { - label: 'large net loss FY2022', - anyOf: [ - 'net loss', - 'unprofitable', - 'losing money', - 'cash burn', - '2,894', - '2.9 billion', - '$2.9 billion', - ], - }, - ], - sourceUrl: - 'https://www.sec.gov/Archives/edgar/data/1690820/000169082023000052/cvna-20221231.htm', - evidence: '"Net loss $ (2,894 ..." for fiscal 2022 ($ millions).', - }, - ], - }, - { - ticker: 'PTON', - company: 'Peloton Interactive, Inc.', - cik: '1639825', - cutoff: '2022-09-07', - sector: 'Consumer fitness hardware', - knownOutcome: - 'The stock fell ~95% from its 2021 peak; the founder-CEO departed, the company underwent mass layoffs and a multi-year turnaround through fiscal 2024.', - facts: [ - { - id: 'PTON/f1', - lens: 'margin-trend', - fact: 'Connected Fitness (hardware) gross margin turned NEGATIVE — to (11)% in FY2022 — meaning Peloton lost money on every bike/tread it sold before any operating cost; revenue growth was masking a broken unit economics.', - expected: [ - { - label: 'negative / collapsing hardware gross margin', - anyOf: [ - 'gross margin', - 'negative margin', - 'gross profit', - 'margin compression', - 'losing money on each', - 'below cost', - '(11)', - '-11', - 'negative gross', - ], - }, - ], - sourceUrl: - 'https://www.sec.gov/Archives/edgar/data/1639825/000163982522000117/pton-20220630.htm', - evidence: - 'MD&A "Gross Profit, and Gross Margin" table: Connected Fitness "Gross Margin decreased to (11)" percent in fiscal 2022 — a negative hardware gross margin.', - }, - { - id: 'PTON/f2', - lens: 'liquidity', - fact: 'Inventories climbed to $1.105B as pandemic demand normalized — a glut of unsold equipment that tied up cash and risked markdowns.', - expected: [ - { - label: 'inventory glut', - anyOf: [ - 'inventor', - 'overstock', - 'excess inventory', - 'glut', - 'markdown', - 'unsold', - '1,104', - '1.1 billion', - ], - }, - ], - sourceUrl: - 'https://www.sec.gov/Archives/edgar/data/1639825/000163982522000117/pton-20220630.htm', - evidence: '"Inventories, net 1,104.5 937" (FY2022 vs FY2021, $ millions).', - }, - { - id: 'PTON/f3', - lens: 'governance', - fact: "A dual-class structure gives Class B holders 20 votes per share vs 1 for Class A — concentrating control with insiders/founders and limiting public shareholders' say.", - expected: [ - { - label: 'dual-class super-voting control', - anyOf: [ - 'dual-class', - 'dual class', - 'class b', - '20 votes', - 'super-voting', - 'supervoting', - 'voting control', - 'multiple votes per share', - ], - }, - ], - sourceUrl: - 'https://www.sec.gov/Archives/edgar/data/1639825/000163982522000117/pton-20220630.htm', - evidence: - '"Class B common stock has 20 votes per share and our Class A common stock has one vote per share."', - }, - { - id: 'PTON/f4', - lens: 'liquidity', - fact: 'Peloton reported a $2.827B net loss in FY2022 — an order-of-magnitude wider loss than the prior year, signaling the demand normalization had broken the model, not just dented it.', - expected: [ - { - label: 'large net loss FY2022', - anyOf: [ - 'net loss', - 'unprofitable', - 'losing money', - 'cash burn', - '2,827', - '2.8 billion', - '$2.8 billion', - ], - }, - ], - sourceUrl: - 'https://www.sec.gov/Archives/edgar/data/1639825/000163982522000117/pton-20220630.htm', - evidence: '"Net loss $ (2,827 ..." for fiscal 2022 ($ millions).', - }, - { - id: 'PTON/f5', - lens: 'regulatory', - fact: 'Peloton was running a CPSC recall of its Tread+ treadmill (tied to injuries and a child death) — an open product-safety and legal exposure beyond the demand story.', - expected: [ - { - label: 'Tread+ / CPSC recall exposure', - anyOf: [ - 'recall', - 'cpsc', - 'consumer product safety', - 'tread+', - 'tread plus', - 'product safety', - 'injuries', - 'safety', - ], - }, - ], - sourceUrl: - 'https://www.sec.gov/Archives/edgar/data/1639825/000163982522000117/pton-20220630.htm', - evidence: - '"recall on Tread+, which we are conducting in collaboration with the Consumer Product Safety Commission (\'CPSC\')"; the Tread product recalls "in the fourth quarter of fiscal 2021 continued to impact" results.', - }, - { - id: 'PTON/f6', - lens: 'leverage', - fact: 'Peloton was locked into ~$334M of manufacturing purchase commitments even as demand fell — contractual inventory it had to take on regardless of whether it could sell it.', - expected: [ - { - label: 'locked-in purchase commitments', - anyOf: [ - 'purchase commitment', - 'purchase obligation', - 'minimum purchase', - 'take-or-pay', - 'committed to purchase', - '334', - 'manufacturing commitment', - ], - }, - ], - sourceUrl: - 'https://www.sec.gov/Archives/edgar/data/1639825/000163982522000117/pton-20220630.htm', - evidence: - '"purchase commitments related to the manufacture of Peloton products were estimated to be approximately $334" million.', - }, - ], - }, - { - ticker: 'SI', - company: 'Silvergate Capital Corporation', - cik: '1312109', - cutoff: '2022-02-28', - sector: 'Banking (digital-asset)', - knownOutcome: - 'After the FTX collapse triggered a deposit run, Silvergate announced a voluntary wind-down of Silvergate Bank and liquidation in March 2023.', - facts: [ - { - id: 'SI/f1', - lens: 'concentration', - fact: 'About 99.5% of total deposits were noninterest-bearing — essentially all funding was non-term money that could leave on demand, an extreme run-risk masked by very low funding cost.', - expected: [ - { - label: 'almost all deposits noninterest-bearing / on-demand', - anyOf: [ - 'noninterest bearing', - 'noninterest-bearing', - 'non-interest-bearing', - 'demand deposit', - 'no term', - 'on demand', - '99.5', - '99 percent', - 'leave at any time', - ], - }, - ], - sourceUrl: - 'https://www.sec.gov/Archives/edgar/data/1312109/000131210922000051/si-20211231.htm', - evidence: - '"noninterest bearing deposits as a percentage of total deposits was 99.5% as of December 31, 2021."', - }, - { - id: 'SI/f2', - lens: 'concentration', - fact: 'Roughly 58% of deposits came from digital-currency EXCHANGES alone — a handful of correlated crypto counterparties whose own troubles would pull deposits out together.', - expected: [ - { - label: 'deposits concentrated in crypto exchanges', - anyOf: [ - 'digital currency exchange', - 'crypto exchange', - 'exchanges represent', - 'counterpart', - 'concentrat', - '58%', - '58 percent', - 'approximately 58', - ], - }, - ], - sourceUrl: - 'https://www.sec.gov/Archives/edgar/data/1312109/000131210922000051/si-20211231.htm', - evidence: - '"Deposits from digital currency exchanges represent approximately 58%" of deposits.', - }, - { - id: 'SI/f3', - lens: 'concentration', - fact: 'The entire deposit franchise was tied to a single, volatile industry — digital-currency (crypto) customers — so a crypto downturn was a direct, undiversified funding shock.', - expected: [ - { - // The buried, depth signal is the CONCENTRATION framing — that the - // whole deposit base is one undiversified industry bet — NOT the bare - // fact that it banks crypto (a one-line ticker summary has that). So - // bare "crypto" / "digital asset" are excluded; the load-bearing - // tokens are the concentration / single-industry / undiversified - // characterization or the filing's own "digital currency customers". - label: 'single-industry deposit CONCENTRATION (not just "it banks crypto")', - anyOf: [ - 'digital currency customers', - 'single industry', - 'one industry', - 'single volatile industry', - 'sector concentration', - 'undiversified', - 'concentrat', - ], - }, - ], - sourceUrl: - 'https://www.sec.gov/Archives/edgar/data/1312109/000131210922000051/si-20211231.htm', - evidence: - 'The 10-K\'s strategy and risk factors center the bank on "digital currency customers" and "the concentration of our deposits" in that single industry.', - }, - { - id: 'SI/f4', - lens: 'liquidity', - fact: 'Total deposits had ballooned to $14.3B (from $10.4B) — fast, hot-money growth from the crypto boom that could reverse just as fast.', - expected: [ - { - // The depth signal is the SIZE/character of the deposit base — the - // specific $14.3B figure or the explicit hot-money / volatile-deposit - // characterization — NOT bare "total deposits" / "grew rapidly", which - // any growth-story summary trips. Those generic phrases are excluded. - label: 'specific hot-money deposit base ($14.3B / volatile)', - anyOf: [ - 'hot money', - 'hot-money', - 'volatile deposit', - 'could reverse', - '14.3 billion', - '14,290', - '$14.3b', - '$14 billion', - ], - }, - ], - sourceUrl: - 'https://www.sec.gov/Archives/edgar/data/1312109/000131210922000051/si-20211231.htm', - evidence: '"Total deposits $ 14,290,628" ... prior year "$ 10,411,278" ($ thousands).', - }, - { - id: 'SI/f5', - lens: 'concentration', - fact: 'The franchise hinged on a single proprietary product — the Silvergate Exchange Network (SEN), a payment network built exclusively for the digital-currency industry — so its competitive moat and its deposit base were the SAME crypto-dependent bet, not two diversified ones.', - expected: [ - { - // The depth signal is naming the SPECIFIC proprietary product (the - // Silvergate Exchange Network / SEN) and that the moat and the deposit - // base are the same single bet — NOT bare "proprietary" / "payment - // network", which a generic crypto-bank summary mentions. Those bare - // terms are excluded; the SEN name or the single-product framing is - // load-bearing. - label: 'names the SEN single-product dependence (not generic "payment network")', - anyOf: [ - 'silvergate exchange network', - 'the sen', - 'sen)', - "sen'", - 'single product', - 'single-product', - 'core product', - 'one product', - 'same bet', - ], - }, - ], - sourceUrl: - 'https://www.sec.gov/Archives/edgar/data/1312109/000131210922000051/si-20211231.htm', - evidence: - "\"Silvergate Exchange Network ('SEN'), our proprietary, virtually instantaneous payment network for participants in the digital currency industry\" — the bank's differentiator and its deposit magnet are the same crypto-only product.", - }, - ], - }, -] - -/** - * Grade ONE material fact against an investment thesis's full text. Returns - * whether the thesis SURFACED it plus which expected groups were found. The - * check is a deterministic case-insensitive substring scan — $0, model-free, - * reproducible — so the eval never leaks into a model the loop could observe. - */ -export function gradeFactAgainstText( - fact: MaterialFact, - thesisText: string, -): { surfaced: boolean; groupsFound: number; groupsTotal: number; foundLabels: string[] } { - const haystack = thesisText.toLowerCase() - const found = fact.expected.filter((group) => - group.anyOf.some((fragment) => haystack.includes(fragment.toLowerCase())), - ) - const minGroups = fact.minGroups ?? fact.expected.length - return { - surfaced: found.length >= minGroups, - groupsFound: found.length, - groupsTotal: fact.expected.length, - foundLabels: found.map((group) => group.label), - } -} - -/** Grade a whole company's thesis text: how many of its held-out facts it surfaces. */ -export function gradeCompanyAgainstText( - company: CompanyEvalCase, - thesisText: string, -): { surfaced: number; total: number; perFact: ReturnType[] } { - const perFact = company.facts.map((fact) => gradeFactAgainstText(fact, thesisText)) - return { - surfaced: perFact.filter((result) => result.surfaced).length, - total: company.facts.length, - perFact, - } -} - -/** Total held-out facts across the set (the denominator the doc reports). */ -export function totalMaterialFacts(set: CompanyEvalCase[] = investmentThesisSet): number { - return set.reduce((sum, company) => sum + company.facts.length, 0) -} - -/** Count facts per lens across the set — used to report (and bound) curation bias. */ -export function lensDistribution( - set: CompanyEvalCase[] = investmentThesisSet, -): Record { - const dist = {} as Record - for (const company of set) { - for (const fact of company.facts) { - dist[fact.lens] = (dist[fact.lens] ?? 0) + 1 - } - } - return dist -} diff --git a/src/investment-thesis-task.ts b/src/investment-thesis-task.ts deleted file mode 100644 index 0a87b97..0000000 --- a/src/investment-thesis-task.ts +++ /dev/null @@ -1,234 +0,0 @@ -/** - * The INVESTMENT-THESIS research task. - * - * Given `{ company, ticker, cik, cutoff }`, drive the SAME two-agent research - * loop the ML deep-question A/B uses (`runVerifiedResearchLoop` + the real web - * worker) to research the company AS OF the cutoff — web + SEC EDGAR, both public - * — and produce an investment-thesis PAGE in the knowledge base: a judgment, the - * drivers, and the risks, grounded in what it fetched. - * - * This file builds NOTHING new for the loop: it composes the existing worker + - * driver + loop, supplies the readiness specs that steer the worker toward the - * filing-level evidence (the analyst lenses), then writes a synthesis thesis page - * the metric (`materialFactsSurfaced`) grades against the HELD-OUT checklist. - * - * THE FIREWALL: the task is told ONLY company + ticker + cutoff (+ the generic - * analyst-lens readiness specs every company gets). It is NEVER shown the - * checklist. The checklist is read only afterward, by the metric. So a high score - * is research depth, not teaching-to-the-test. - */ - -import { join } from 'node:path' -import { writeFileDurableWithinRoot } from './durable-fs' -import { defineReadinessSpec, type KnowledgeReadinessSpec } from './eval-readiness' -import { slugify } from './ids' -import { buildKnowledgeIndex } from './indexer' -import { kbIndexToText } from './material-facts-metric' -import { withKnowledgeMutation } from './mutation-lock' -import { - type ResearchDriver, - runVerifiedResearchLoop, - type VerifiedResearchLoopResult, -} from './verified-research-loop' -import { - createWebResearchWorker, - type RouterClient, - type WebResearchWorkerOptions, -} from './web-research-worker' - -/** The minimal brief a thesis run is given — the firewall boundary. */ -export interface ThesisTaskInput { - /** Legal name as of the cutoff — what the loop researches. */ - company: string - /** Ticker as of the cutoff. */ - ticker: string - /** SEC Central Index Key (CIK), zero-stripped — the EDGAR filer id. */ - cik: string - /** Research-as-of date (ISO). The loop must reason as if it is this date. */ - cutoff: string - /** Sector, for the readiness query context (NOT a checklist hint). */ - sector?: string -} - -/** - * The generic analyst-lens readiness specs every company gets. They are the ONLY - * thing the loop is told about WHAT to look for, and they name the LENSES a - * thorough analyst checks (balance-sheet risk, concentration, leverage, margins, - * liquidity, governance, regulatory) and where they live (the latest SEC 10-K) — - * NOT the answers. They steer the worker's web/EDGAR search toward the filing, - * not toward the held-out facts (which the loop never sees). - * - * `minSources` is set above 1 so the readiness gate stays UNMET after a single - * fetch and the loop runs multiple rounds — the depth-driving driver needs >1 - * round to steer, exactly as the ML-exam multi-round probe established. - */ -export function thesisReadinessSpecs(input: ThesisTaskInput): KnowledgeReadinessSpec[] { - const c = input.company - const t = input.ticker - const filing = `${c} ${t} SEC 10-K annual report SEC.gov EDGAR filing` - return [ - defineReadinessSpec({ - id: 'thesis/filing', - description: `the most recent SEC 10-K annual report for ${c} (${t}) filed on or before ${input.cutoff}, from SEC EDGAR`, - query: filing, - requiredFor: ['ResearchAgent'], - importance: 'blocking', - minSources: 2, - minHits: 1, - }), - defineReadinessSpec({ - id: 'thesis/balance-sheet', - description: `${c} balance-sheet risks: securities marked below cost, unrealized losses, leverage / total debt, debt maturities, interest expense`, - query: `${c} ${t} 10-K balance sheet total debt unrealized losses interest expense leverage`, - requiredFor: ['ResearchAgent'], - importance: 'blocking', - minSources: 2, - minHits: 1, - }), - defineReadinessSpec({ - id: 'thesis/concentration-liquidity', - description: `${c} concentration + liquidity: customer / deposit / revenue concentration, uninsured deposits, operating cash flow, net loss, inventory, equity erosion`, - query: `${c} ${t} 10-K customer deposit concentration operating cash flow net loss inventory`, - requiredFor: ['ResearchAgent'], - importance: 'blocking', - minSources: 2, - minHits: 1, - }), - defineReadinessSpec({ - id: 'thesis/governance-regulatory', - description: `${c} governance + regulatory: related-party transactions, dual-class / super-voting control, buybacks / dividends, recalls, regulatory or legal exposure, margin trends`, - query: `${c} ${t} 10-K related party dual class share repurchase recall regulatory gross margin`, - requiredFor: ['ResearchAgent'], - importance: 'blocking', - minSources: 2, - minHits: 1, - }), - ] -} - -/** - * The thesis-writer prompt. After the loop has fetched + curated the filings, we - * ask the model to SYNTHESIZE a thesis page from the curated KB text — a - * judgment, the key drivers, and the material risks, grounded ONLY in the fetched - * evidence. The held-out checklist is NOT in this prompt; the model writes from - * what the loop actually pulled, so a fact only appears if the research surfaced - * the underlying evidence. - */ -function thesisSynthesisMessages( - input: ThesisTaskInput, - kbText: string, -): { role: 'system' | 'user'; content: string }[] { - const system = - 'You are a buy-side investment analyst writing a thesis memo. You are given ' + - 'the raw research your team gathered from public filings (SEC 10-K) and the web. ' + - 'Write a thesis that a thorough analyst would write: lead with your JUDGMENT, ' + - 'then the KEY DRIVERS, then the MATERIAL RISKS. Be specific and quantitative — ' + - 'name the actual figures, balance-sheet items, concentrations, leverage, ' + - 'margin trends, governance items, and regulatory exposures that appear in the ' + - 'research. Surface the buried, non-obvious drivers a one-line ticker summary ' + - 'misses. Ground every claim in the research provided; do NOT invent figures. ' + - 'If the research does not contain a figure, do not state it.' - const user = [ - `Company: ${input.company} (${input.ticker})`, - `As-of date (reason as if it is this date): ${input.cutoff}`, - input.sector ? `Sector: ${input.sector}` : '', - '', - 'Research gathered (filings + web excerpts):', - '"""', - kbText.slice(0, 24000), - '"""', - '', - 'Write the investment thesis now. Structure:', - '## Judgment', - '## Key drivers', - '## Material risks', - ] - .filter(Boolean) - .join('\n') - return [ - { role: 'system', content: system }, - { role: 'user', content: user }, - ] -} - -/** Write the synthesis thesis page into the KB so the index + metric pick it up. */ -async function writeThesisPage( - root: string, - input: ThesisTaskInput, - thesis: string, -): Promise { - return withKnowledgeMutation(root, async () => { - const relativePath = `knowledge/thesis-${slugify(input.ticker)}.md` - const body = [ - '---', - `title: Investment thesis — ${input.company} (${input.ticker})`, - `ticker: ${input.ticker}`, - `cutoff: ${input.cutoff}`, - 'kind: investment-thesis', - '---', - `# Investment thesis — ${input.company} (${input.ticker}), as of ${input.cutoff}`, - '', - thesis.trim(), - '', - ].join('\n') - await writeFileDurableWithinRoot(root, relativePath, body, { encoding: 'utf8' }) - return join(root, relativePath) - }) -} - -export interface ThesisRunOptions { - /** The KB root the loop writes into. */ - root: string - /** Shared router client (web search + chat). Defaults to env creds. */ - router: RouterClient - /** The driver — verify/dedup or research-driving. The loop's coordinator. */ - driver: ResearchDriver - /** Round budget. Default 3 (the depth-driving driver needs >1). */ - maxRounds?: number - /** Worker tuning forwarded to `createWebResearchWorker`. */ - workerOptions?: Omit - /** Max tokens for the synthesis pass. Default 1600 (above glm-5.2's reasoning floor). */ - synthesisMaxTokens?: number - signal?: AbortSignal -} - -export interface ThesisRunResult { - loop: VerifiedResearchLoopResult - /** The synthesized thesis text. */ - thesis: string - /** Path of the thesis page written into the KB. */ - thesisPath: string -} - -/** - * Run the full thesis task: drive the two-agent loop to research the company AS - * OF the cutoff, then synthesize + write the thesis page. Returns the loop result - * + the thesis text + the page path. The caller grades the KB with - * `materialFactsSurfaced(root, checklist)` — the checklist is never passed here. - */ -export async function runInvestmentThesisTask( - input: ThesisTaskInput, - options: ThesisRunOptions, -): Promise { - const worker = createWebResearchWorker({ ...options.workerOptions, router: options.router }) - const goal = `${input.company} (${input.ticker}) investment thesis as of ${input.cutoff}` - - const loop = await runVerifiedResearchLoop({ - root: options.root, - goal, - worker, - driver: options.driver, - readinessSpecs: thesisReadinessSpecs(input), - maxRounds: options.maxRounds ?? 3, - signal: options.signal, - }) - - // Synthesize the thesis from what the loop actually curated + fetched. - const index = await buildKnowledgeIndex(options.root) - const kbText = kbIndexToText(index) - const messages = thesisSynthesisMessages(input, kbText) - const thesis = await options.router.chat(messages, options.synthesisMaxTokens ?? 1600) - - const thesisPath = await writeThesisPage(options.root, input, thesis) - return { loop, thesis, thesisPath } -} diff --git a/src/material-facts-metric.ts b/src/material-facts-metric.ts deleted file mode 100644 index d4577f5..0000000 --- a/src/material-facts-metric.ts +++ /dev/null @@ -1,121 +0,0 @@ -/** - * `materialFactsSurfaced` — the held-out investment-research METRIC. - * - * Given a knowledge base a research loop built for a company and the company's - * HELD-OUT material-fact checklist (`tests/eval/investment-thesis-set.ts`, never - * shown to the loop), this returns the FRACTION of checklist items the KB's - * pages surface + ground. The check is the same `$0`, model-free, deterministic - * substring grader the loop's checklist already ships (`gradeFactAgainstText` / - * `gradeCompanyAgainstText`) — so the answer key never reaches a model the loop - * could observe, exactly the firewall the ML deep-question exam uses. - * - * The ONLY thing this file adds over the raw grader is the KB→text join: it reads - * the curated pages (and the raw source text) the loop wrote and hands their - * concatenation to the grader. That join mirrors `kbText` in the research-quality - * A/B (research-driving-ab.test.ts) so the thesis metric and the ML-exam metric - * read a KB the same way. - * - * WHY pages AND source text: an honest thesis surfaces a buried fact in its - * curated thesis PAGE (the judgment), but a loop whose page is thin while its - * fetched filings are rich should still get credit for what it actually pulled. - * Grading the union is the faithful, not the lenient, choice — it rewards the - * loop that REACHED the filing even if its synthesis was terse, and it cannot - * manufacture a hit the underlying evidence does not contain. - */ - -import { buildKnowledgeIndex } from './indexer' -import { - type CompanyEvalCase, - gradeCompanyAgainstText, - gradeFactAgainstText, -} from './investment-thesis-set' -import type { KnowledgeIndex } from './types' - -/** Per-fact grade plus the fact's id/lens, for the audit trail. */ -export interface FactResult { - id: string - lens: CompanyEvalCase['facts'][number]['lens'] - surfaced: boolean - groupsFound: number - groupsTotal: number - foundLabels: string[] -} - -/** The metric's result for one company: the surfaced fraction + the per-fact trail. */ -export interface MaterialFactsResult { - ticker: string - company: string - /** Held-out facts the KB surfaced + grounded. */ - surfaced: number - /** Total held-out facts for this company (the denominator). */ - total: number - /** `surfaced / total` in [0, 1]. */ - fraction: number - /** Per-fact grade, in checklist order, for the doc / audit. */ - perFact: FactResult[] -} - -/** - * Join a KB index into the single text blob the grader scans: every curated PAGE - * (title + body) followed by every raw SOURCE (title + fetched text). This is the - * text read AFTER the loop finished — it is never handed to the loop. Identical - * in spirit to `kbText` in the research-quality A/B so both metrics read a KB the - * same way. - */ -export function kbIndexToText(index: KnowledgeIndex): string { - const pageText = index.pages.map((page) => `${page.title}\n${page.text}`).join('\n\n') - const sourceText = index.sources - .map((source) => `${source.title ?? ''}\n${source.text ?? ''}`) - .join('\n\n') - return `${pageText}\n\n${sourceText}` -} - -/** - * Grade one company's KB against its held-out material-fact checklist, given the - * KB's already-joined text. The pure core — no I/O — so calibration can score a - * hand-written shallow/deep thesis string directly and the live path can score a - * real KB. Returns the surfaced FRACTION plus the per-fact audit trail. - */ -export function materialFactsSurfacedInText( - company: CompanyEvalCase, - kbText: string, -): MaterialFactsResult { - const grade = gradeCompanyAgainstText(company, kbText) - const perFact: FactResult[] = company.facts.map((fact) => { - const r = gradeFactAgainstText(fact, kbText) - return { - id: fact.id, - lens: fact.lens, - surfaced: r.surfaced, - groupsFound: r.groupsFound, - groupsTotal: r.groupsTotal, - foundLabels: r.foundLabels, - } - }) - return { - ticker: company.ticker, - company: company.company, - surfaced: grade.surfaced, - total: grade.total, - fraction: grade.total === 0 ? 0 : grade.surfaced / grade.total, - perFact, - } -} - -/** - * `materialFactsSurfaced(kb, checklist)` — the metric in its KB-reading form. - * - * `kb` is EITHER a knowledge-base root directory (the loop wrote pages there) or - * an already-built `KnowledgeIndex`. `checklist` is the company's held-out - * `CompanyEvalCase`. Returns the surfaced fraction + per-fact trail. - * - * The checklist is HELD OUT by construction: it lives in the test eval set, is - * never passed to the loop, and is read only here, after the loop finished. - */ -export async function materialFactsSurfaced( - kb: string | KnowledgeIndex, - checklist: CompanyEvalCase, -): Promise { - const index = typeof kb === 'string' ? await buildKnowledgeIndex(kb) : kb - return materialFactsSurfacedInText(checklist, kbIndexToText(index)) -} diff --git a/src/readiness-helpers.ts b/src/readiness-helpers.ts index a074e0b..b81bcac 100644 --- a/src/readiness-helpers.ts +++ b/src/readiness-helpers.ts @@ -12,8 +12,7 @@ import type { KnowledgeIndex } from './types' * override, and the rest of the bundle options minus the fields this helper * supplies (`taskId`/`index`/`specs`). * - * Both the single-agent (`research-loop`) and two-agent (`verified-research-loop`) - * options shapes satisfy this structurally, so the gate is single-sourced. + * The research-loop options shape satisfies this structurally. */ export interface ReadinessGateOptions { goal: string diff --git a/src/research-driving-driver.ts b/src/research-driving-driver.ts deleted file mode 100644 index 9b7831a..0000000 --- a/src/research-driving-driver.ts +++ /dev/null @@ -1,1111 +0,0 @@ -/** - * Research-DRIVING driver for `runVerifiedResearchLoop`. - * - * The shipped drivers all FILTER the worker's sources: - * - `createVerifyingResearchDriver` judges on-topic relevance, - * - `createAdaptiveResearchDriver` dedups then triages then escalates, - * - `createClaimGroundingVerifier` rejects misattributed citations. - * - * This driver does the OPPOSITE job: instead of narrowing the worker's output, - * it DRIVES the research DEEPER each round. Its value is not "fewer sources" — - * it is "more answered, better-corroborated sub-questions". Concretely, each - * round it: - * - * 1. EXTRACTS the key claims from the worker's new sources (one LLM pass per - * source, in `verifySource`; falls back to a deterministic sentence-pull - * when the model is unavailable so a round never silently extracts nothing). - * 2. TRACKS each claim's support — the set of INDEPENDENT sources (by canonical - * host) that assert it — and detects CONTRADICTIONS between a new claim and - * one already on the ledger. - * 3. GENERATES the next round's DEEP sub-questions from the accumulated claims, - * in four kinds — comparative ("how does X's tradeoff differ from Y's?"), - * mechanism ("under what precise condition does X fail?"), gap ("what - * specific result is missing?"), and contradiction ("does any source - * challenge claim Z?"). - * 4. FLAGS weakly-supported claims (only ONE independent source) and - * contradicted claims as INVALIDATION targets and demands the worker find - * corroborating / refuting evidence for them. - * 5. FOLDS the deep sub-questions + invalidation challenges into the worker's - * next prompt via the loop's `foldGaps` → `steer` channel — that is the - * mechanism that drives DEPTH and VALIDATION rather than breadth. - * - * COMPLETION (`isComplete` / the `done` judgment the caller gates on) does NOT - * look at source COUNT. It is done only when every deep sub-question it raised - * has been addressed AND every key claim is either supported by >= 2 independent - * sources OR explicitly marked CONTESTED (a contradiction the loop surfaced and - * could not resolve). A KB with twenty sources all asserting one unchallenged - * claim is NOT done; a KB whose handful of claims are each corroborated or - * contested IS. - * - * It reuses `runVerifiedResearchLoop` (it is a plain `ResearchDriver`), the web - * worker, `claim-ledger.ts` (claim identity, independent-source identity, and - * the merge rule), and the `RouterClient` chat surface; it reinvents none of - * them. - */ - -import { - claimEvidenceId, - claimId, - deepQuestionId, - claimSourceHost as hostOf, - mergeClaimLedgers, - normalizeClaimText, - researchSourceVersionKey, -} from './claim-ledger' -import { sha256, textSourceId } from './ids' -import { assertClaimLedgerId, type ClaimLedgerStore } from './kb-store' -import { snapshotSourceTextInput } from './sources' -import type { - DeepQuestion, - DeepQuestionKind, - ResearchClaimEvidence, - ResearchClaimLedger, - ResearchClaimRecord, - ResearchSourceVersion, - SourceRecord, - TrackedClaim, -} from './types' -import type { - KnowledgeGap, - ResearchDriver, - ResearchSourceProposal, - SourceVerdict, - SourceVerificationContext, -} from './verified-research-loop' -import { - createTangleRouterClient, - type RouterClient, - type TangleRouterOptions, -} from './web-research-worker' - -// The live claim type and its durable ledger records live in `types.ts` with -// the package's other public shapes. Re-exported here so existing importers -// keep working. -export type { - DeepQuestion, - DeepQuestionKind, - ResearchClaimEvidence, - ResearchClaimLedger, - ResearchClaimRecord, - ResearchSourceVersion, - TrackedClaim, -} - -/** The driver's accumulated research state — the completion oracle reads this. */ -export interface ResearchDrivingState { - /** Every extracted claim backed by a registered source, by id. */ - claims: TrackedClaim[] - /** Every deep sub-question raised, by id. */ - questions: DeepQuestion[] - /** Claims with exactly one independent source AND not contested. */ - weaklySupported: TrackedClaim[] - /** Claims supported by >= 2 independent sources. */ - corroborated: TrackedClaim[] - /** Claims marked contested (a surfaced, unresolved contradiction). */ - contested: TrackedClaim[] - /** Deep questions still unaddressed. */ - openQuestions: DeepQuestion[] - /** How many rounds the driver has folded steer for. */ - rounds: number -} - -export interface ResearchDrivingDriverOptions { - /** Router client for claim extraction + deep-question generation. */ - router?: RouterClient - router_options?: TangleRouterOptions - /** - * A claim is CORROBORATED at this many INDEPENDENT supporting sources (distinct - * canonical hosts). Default 2 — the task's ">= 2 independent sources" bar. - */ - minIndependentSources?: number - /** Max deep sub-questions to fold into one round's steer. Default 6. */ - maxQuestionsPerRound?: number - /** Max claims to extract from a single source. Default 3. */ - maxClaimsPerSource?: number - /** - * When the extractor LLM is unavailable, fall back to a deterministic claim - * pull (the source's leading sentences) so the driver still drives. Default - * true. Set false to require the model (claims will be empty without it). - */ - deterministicFallback?: boolean - /** Observe each round's generated steer (for instrumentation / the script). */ - onSteer?: (steer: ResearchDrivingSteer) => void -} - -/** - * Options for the durable driver: the same driver plus a store to keep its - * belief state in. - */ -export interface PersistentResearchDrivingDriverOptions extends ResearchDrivingDriverOptions { - /** Where the claim ledger is read from and written to. */ - store: ClaimLedgerStore - /** - * Names this run's ledger. Stable across resumes (that is what makes a resume - * a resume) and distinct per run against one knowledge base. Must be a single - * safe path segment — see `assertClaimLedgerId`. - */ - ledgerId: string -} - -/** What the driver folded into one round's worker prompt, surfaced for audit. */ -export interface ResearchDrivingSteer { - round: number - deepQuestions: DeepQuestion[] - /** Claims it demanded corroborating/refuting evidence for this round. */ - invalidationTargets: TrackedClaim[] - /** The readiness gaps it interleaved (passed through from the loop). */ - gaps: KnowledgeGap[] - /** The full steer text handed to the worker. */ - text: string -} - -/** - * The research-driving driver. It is a `ResearchDriver` (drops straight into - * `runVerifiedResearchLoop`) PLUS a completion oracle and live state, mirroring - * how `createAdaptiveResearchDriver` exposes `stats()`. - */ -export interface ResearchDrivingDriver extends ResearchDriver { - /** Live snapshot of the claim ledger + deep questions. */ - researchState(): ResearchDrivingState - /** - * The completion oracle — gate `done` on THIS, not on source count. True when - * every deep sub-question is addressed AND every claim is corroborated - * (>= `minIndependentSources` independent sources) or explicitly contested. - * False while any claim is weakly-supported or any deep question is open. - * Returns false before any claim has been seen (nothing researched yet). - */ - isComplete(): boolean - /** - * The last round's generated steer, or undefined before the first fold. Useful - * to assert the driver produced deeper questions / invalidation challenges. - */ - lastSteer(): ResearchDrivingSteer | undefined - /** - * Write the current belief state to the store. `verifySource` already persists - * each pending observation; this exists for the state `foldGaps` produces — - * the deep questions — which is raised by a synchronous hook and would - * otherwise be lost if the process died before the next source arrived. - * - * `runVerifiedResearchLoop` calls this at the end of every round. A driver - * built without a store has nothing to write and resolves immediately. - */ - checkpoint(): Promise - /** Durably announce the next synchronous fold before it begins. */ - prepareFold(): Promise - /** - * Confirm exact registered source records. - * URI equality alone is insufficient: pending evidence is authorized only - * when its expected content hash matches the registered record. - */ - commitSources(sources: readonly SourceRecord[]): Promise - /** The ledger record as it would be written right now. */ - toLedger(): ResearchClaimLedger -} - -/** A claim the extractor returns for one source. */ -interface ExtractedClaim { - text: string - /** A claim id ALREADY on the ledger that this one CONTRADICTS, if the model says so. */ - contradictsExistingId?: string -} - -/** - * The in-memory driver. Its belief state lives for exactly as long as the - * process does — use `createPersistentResearchDrivingDriver` when the run must - * survive a crash or resume. - */ -export function createResearchDrivingDriver( - options: ResearchDrivingDriverOptions = {}, -): ResearchDrivingDriver { - return buildDriver(options) -} - -/** - * The durable driver: same behaviour, plus its claim ledger is read from the - * store at construction and written back after every claim and every round. - * - * Construction is asynchronous because loading is I/O, and loading has to happen - * before the caller can read `researchState()` or `isComplete()` — a driver that - * loaded lazily would answer "nothing researched, not complete" for a run that - * had already corroborated everything. - */ -export async function createPersistentResearchDrivingDriver( - options: PersistentResearchDrivingDriverOptions, -): Promise { - const ledgerId = assertClaimLedgerId(options.ledgerId) - const existing = await options.store.getClaimLedger(ledgerId) - const driver = buildDriver(options, { store: options.store, ledgerId }, existing ?? undefined) - if (existing && (existing.preparedRounds ?? existing.rounds) > existing.rounds) { - await driver.checkpoint() - } - return driver -} - -interface DriverPersistence { - store: ClaimLedgerStore - ledgerId: string -} - -function buildDriver( - options: ResearchDrivingDriverOptions, - persistence?: DriverPersistence, - restored?: ResearchClaimLedger, -): ResearchDrivingDriver { - const minIndependentSources = Math.max(2, options.minIndependentSources ?? 2) - const maxQuestionsPerRound = Math.max(1, options.maxQuestionsPerRound ?? 6) - const maxClaimsPerSource = Math.max(1, options.maxClaimsPerSource ?? 3) - const deterministicFallback = options.deterministicFallback ?? true - - // The claim ledger, keyed by claim id (sha256 of the normalized claim text). - const claims = new Map( - (restored?.claims ?? []).map((claim) => [claim.id, fromRecord(claim)]), - ) - // Claim extraction and source registration are two separate durable writes. - // Evidence remains inert until its exact source version is confirmed here. - const claimEvidence = new Map( - (restored?.claimEvidence ?? []).map((evidence) => [evidence.id, evidence]), - ) - const registeredSources = new Map( - (restored?.registeredSources ?? []).map((source) => [source.sourceId, source]), - ) - // Every deep question raised, by id — so we can mark them addressed later. - const questions = new Map( - (restored?.questions ?? []).map((question) => [question.id, question]), - ) - let rounds = restored?.rounds ?? 0 - let preparedRounds = Math.max(rounds, restored?.preparedRounds ?? rounds) - let goal = restored?.goal - let lastSteer: ResearchDrivingSteer | undefined - - // `prepareFold` is persisted before the synchronous fold starts. If the - // process died during that fold, regenerate its deterministic questions now - // and advance the in-memory round; `createPersistentResearchDrivingDriver` - // checkpoints this recovered state before returning it. - if (preparedRounds > rounds) { - for (let round = rounds + 1; round <= preparedRounds; round += 1) { - for (const question of synthesizeDeepQuestions([...claims.values()], round).slice( - 0, - maxQuestionsPerRound, - )) { - if (!questions.has(question.id)) questions.set(question.id, question) - } - } - rounds = preparedRounds - } - - function toLedger(): ResearchClaimLedger { - return { - schemaVersion: 2, - id: persistence?.ledgerId ?? 'in-memory', - ...(goal === undefined ? {} : { goal }), - updatedAt: new Date().toISOString(), - rounds, - ...(preparedRounds > rounds ? { preparedRounds } : {}), - claimEvidence: [...claimEvidence.values()].sort((left, right) => - left.id.localeCompare(right.id), - ), - registeredSources: [...registeredSources.values()].sort((left, right) => - left.sourceId.localeCompare(right.sourceId), - ), - claims: [...claims.values()] - .map((claim) => ({ - ...claim, - supportingHosts: [...new Set(claim.supportingHosts)].sort(), - supportingUris: [...new Set(claim.supportingUris)].sort(), - contradicts: [...new Set(claim.contradicts)].sort(), - })) - .sort((a, b) => a.id.localeCompare(b.id)), - questions: [...questions.values()] - .map((question) => ({ - ...question, - claimIds: [...new Set(question.claimIds)].sort(), - })) - .sort((a, b) => a.id.localeCompare(b.id)), - } - } - - /** - * Write this driver's belief state into the stored ledger and adopt the - * result. - * - * It merges rather than overwrites, and then rehydrates from the merged - * record, which is the whole of what makes knowledge compound across - * workers. Two drivers on one ledger id — a resumed run beside a still-live - * one, or two workers researching one goal in parallel — would otherwise each - * write a whole record built from what it read before the other wrote, and - * the later write would erase the earlier writer's claims. Rehydrating means - * a claim another worker corroborated counts toward THIS driver's completion - * oracle from the next round onward. - */ - async function persist(): Promise { - if (!persistence) return - const mine = toLedger() - const merged = await persistence.store.mergeClaimLedger(persistence.ledgerId, (current) => - current === null ? mine : mergeClaimLedgers(current, mine), - ) - claims.clear() - for (const claim of merged.claims) claims.set(claim.id, fromRecord(claim)) - claimEvidence.clear() - for (const evidence of merged.claimEvidence) claimEvidence.set(evidence.id, evidence) - registeredSources.clear() - for (const source of merged.registeredSources) { - registeredSources.set(source.sourceId, source) - } - questions.clear() - for (const question of merged.questions) questions.set(question.id, question) - rounds = Math.max(rounds, merged.rounds) - preparedRounds = Math.max(rounds, merged.preparedRounds ?? merged.rounds) - goal = merged.goal ?? goal - } - - async function prepareFold(): Promise { - if (!persistence) return - preparedRounds = Math.max(preparedRounds, rounds + 1) - await persist() - } - - /** - * A ledger accumulates evidence FOR a goal. Reusing one id across two goals - * merges two runs' beliefs into one corroboration count, which is worse than - * losing them, so it fails rather than merging. - */ - function bindGoal(nextGoal: string): void { - if (goal === undefined) { - goal = nextGoal - return - } - if (goal !== nextGoal) { - throw new Error( - `claim ledger '${persistence?.ledgerId ?? 'in-memory'}' accumulated evidence for goal ` + - `'${goal}' and cannot be reused for '${nextGoal}'`, - ) - } - } - - function resolveRouter(signal?: AbortSignal): RouterClient { - if (options.router) return options.router - const configuredSignal = options.router_options?.signal - const requestSignal = - configuredSignal && signal - ? AbortSignal.any([configuredSignal, signal]) - : (configuredSignal ?? signal) - return createTangleRouterClient({ ...options.router_options, signal: requestSignal }) - } - - function throwIfRouterAborted(signal?: AbortSignal): void { - signal?.throwIfAborted() - if (!options.router) options.router_options?.signal?.throwIfAborted() - } - - /** Record a claim from a source, growing its independent-source support. */ - function recordClaim(extracted: ExtractedClaim, sourceUri: string, round: number): TrackedClaim { - const id = claimId(extracted.text) - const host = hostOf(sourceUri) - const existing = claims.get(id) - if (existing) { - if (host) existing.supportingHosts.add(host) - addUnique(existing.supportingUris, sourceUri) - linkContradiction(existing, extracted.contradictsExistingId) - return existing - } - const tracked: TrackedClaim = { - id, - text: extracted.text.trim(), - supportingHosts: new Set(host ? [host] : []), - supportingUris: [sourceUri], - contradicts: new Set(), - contested: false, - firstSeenRound: round, - } - linkContradiction(tracked, extracted.contradictsExistingId) - claims.set(id, tracked) - return tracked - } - - /** Persist an extraction observation without treating its source as registered. */ - function recordEvidence( - extracted: ExtractedClaim, - sourceVersion: ResearchSourceVersion, - round: number, - ): ResearchClaimEvidence { - const text = extracted.text.trim() - const observedClaimId = claimId(text) - const contradictsClaimId = - extracted.contradictsExistingId === observedClaimId - ? undefined - : extracted.contradictsExistingId - const evidence: ResearchClaimEvidence = { - id: claimEvidenceId({ - claimId: observedClaimId, - sourceId: sourceVersion.sourceId, - sourceUri: sourceVersion.uri, - sourceContentHash: sourceVersion.contentHash, - contradictsClaimId, - }), - claimId: observedClaimId, - text, - sourceId: sourceVersion.sourceId, - sourceUri: sourceVersion.uri, - sourceContentHash: sourceVersion.contentHash, - ...(contradictsClaimId === undefined ? {} : { contradictsClaimId }), - firstSeenRound: round, - } - const existing = claimEvidence.get(evidence.id) - if (!existing) { - claimEvidence.set(evidence.id, evidence) - return evidence - } - const earlier = - evidence.firstSeenRound < existing.firstSeenRound || - (evidence.firstSeenRound === existing.firstSeenRound && evidence.text < existing.text) - ? evidence - : existing - const merged = { - ...earlier, - firstSeenRound: Math.min(existing.firstSeenRound, evidence.firstSeenRound), - } - claimEvidence.set(merged.id, merged) - return merged - } - - /** Materialize evidence for newly confirmed exact source versions into live claims. */ - function materializeEvidenceFor(sourceVersions: ReadonlySet): string[] { - const evidence = [...claimEvidence.values()].filter((item) => - sourceVersions.has(sourceVersionKeyOfEvidence(item)), - ) - const texts: string[] = [] - for (const item of evidence) { - recordClaim( - { text: item.text, contradictsExistingId: item.contradictsClaimId }, - item.sourceUri, - item.firstSeenRound, - ) - texts.push(item.text) - } - // A refuter can sort before the original claim, so close edges only after - // every claim for this confirmation batch exists. - for (const item of evidence) { - const claim = claims.get(item.claimId) - if (claim) linkContradiction(claim, item.contradictsClaimId) - } - return texts - } - - async function commitSources(sources: readonly SourceRecord[]): Promise { - const versions = sources.map(sourceVersionOfRecord) - const nextRegistered = new Map(registeredSources) - const newlyRegistered = new Set() - for (const version of versions) { - const key = researchSourceVersionKey(version) - const existing = nextRegistered.get(version.sourceId) - if (existing && researchSourceVersionKey(existing) !== key) { - throw new Error(`registered source '${version.sourceId}' has conflicting immutable content`) - } - if (existing) continue - nextRegistered.set(version.sourceId, version) - newlyRegistered.add(key) - } - registeredSources.clear() - for (const [sourceId, version] of nextRegistered) registeredSources.set(sourceId, version) - if (newlyRegistered.size > 0) { - markAddressed(materializeEvidenceFor(newlyRegistered)) - } - await persist() - } - - /** Wire a bidirectional contradiction edge and mark BOTH claims contested. */ - function linkContradiction(claim: TrackedClaim, otherId: string | undefined): void { - if (!otherId || otherId === claim.id) return - const other = claims.get(otherId) - if (!other) return - claim.contradicts.add(otherId) - other.contradicts.add(claim.id) - claim.contested = true - other.contested = true - } - - /** A claim's independent-source count = distinct canonical hosts. */ - function independentSupport(claim: TrackedClaim): number { - return claim.supportingHosts.size - } - - function isCorroborated(claim: TrackedClaim): boolean { - return independentSupport(claim) >= minIndependentSources - } - - /** Weakly-supported = NOT corroborated AND NOT contested → an invalidation target. */ - function isWeak(claim: TrackedClaim): boolean { - return !isCorroborated(claim) && !claim.contested - } - - function snapshot(): ResearchDrivingState { - const all = [...claims.values()] - const allQuestions = [...questions.values()] - return { - claims: all, - questions: allQuestions, - weaklySupported: all.filter(isWeak), - corroborated: all.filter(isCorroborated), - contested: all.filter((claim) => claim.contested), - openQuestions: allQuestions.filter((question) => !question.addressed), - rounds, - } - } - - /** - * Mark deep questions addressed when later evidence speaks to them. A question - * is addressed once a NEW claim's text shares enough content words with the - * question — i.e. the worker brought back evidence on the thing we asked about. - * Cheap and deterministic; the LLM is reserved for GENERATING questions, not - * grading them, so the oracle stays a non-model check. - */ - function markAddressed(newClaimTexts: string[]): void { - for (const question of questions.values()) { - if (question.addressed) continue - // Contradiction questions resolve when one of their claims becomes - // corroborated or contested (the disagreement was surfaced/settled). - if (question.kind === 'contradiction') { - const settled = question.claimIds.some((id) => { - const claim = claims.get(id) - return claim ? isCorroborated(claim) || claim.contested : false - }) - if (settled) question.addressed = true - continue - } - const qWords = contentWordSet(question.text) - if (qWords.size === 0) continue - for (const text of newClaimTexts) { - const overlap = overlapFraction(qWords, contentWordSet(text)) - if (overlap >= 0.5) { - question.addressed = true - break - } - } - } - } - - return { - /** - * `verifySource` — the per-source hook. This driver does NOT filter for - * relevance/dedup (other drivers do that, and the loop already dedups exact - * uris). Its job here is to EXTRACT the source's claims and grow the ledger. - * It accepts every source that yields at least one extractable claim; it - * only rejects a source with NO extractable signal at all (empty/unusable), - * because such a source cannot drive the research and pollutes the KB. - */ - async verifySource( - source: ResearchSourceProposal, - ctx: SourceVerificationContext, - ): Promise { - throwIfRouterAborted(ctx.signal) - const sourceSnapshot = snapshotSourceTextInput(source) - const goalSnapshot = ctx.goal - const roundSnapshot = ctx.round - const sourceVersion = sourceVersionOfProposal(sourceSnapshot) - bindGoal(goalSnapshot) - const extracted = await extractClaims(sourceSnapshot, goalSnapshot, ctx.signal) - throwIfRouterAborted(ctx.signal) - if (extracted.length === 0) { - return { - accept: false, - reason: 'no extractable claim: source yields nothing to drive the research deeper', - } - } - for (const claim of extracted) { - recordEvidence(claim, sourceVersion, roundSnapshot) - } - if (!persistence) { - const key = researchSourceVersionKey(sourceVersion) - registeredSources.set(sourceVersion.sourceId, sourceVersion) - markAddressed(materializeEvidenceFor(new Set([key]))) - } else { - const registered = registeredSources.get(sourceVersion.sourceId) - if ( - registered && - researchSourceVersionKey(registered) === researchSourceVersionKey(sourceVersion) - ) { - // Direct callers can verify an exact version already present in the - // registry. Its newly extracted evidence is safe to materialize now. - markAddressed(materializeEvidenceFor(new Set([researchSourceVersionKey(sourceVersion)]))) - } - } - // Persist the observation BEFORE accepting. It remains pending until the - // loop confirms source registration through `commitSources`, closing both - // possible crash directions without a cross-store transaction. - await persist() - throwIfRouterAborted(ctx.signal) - return { accept: true } - }, - - /** - * `foldGaps` — the DEPTH driver. Runs after the worker's contribution is - * applied each round. It builds the next round's steer from (1) the readiness - * gaps the loop still reports, (2) freshly generated DEEP sub-questions, and - * (3) INVALIDATION challenges for weakly-supported / contradicted claims. - */ - foldGaps(gaps: KnowledgeGap[]): string { - if (persistence && preparedRounds <= rounds) { - throw new Error('persistent research driver must prepareFold before foldGaps') - } - rounds += 1 - const round = rounds - const ledger = [...claims.values()] - - // Invalidation targets: claims with one source (need corroboration) OR - // contradicted claims (need a refutation/resolution). These are what the - // worker is told to go SHORE UP, not new breadth. - const invalidationTargets = ledger.filter( - (claim) => isWeak(claim) || claim.contradicts.size > 0, - ) - - // Generate this round's deep sub-questions from the actual ledger claims - // and register them so completion can track whether they get addressed. - const deepQuestions = synthesizeDeepQuestions(ledger, round).slice(0, maxQuestionsPerRound) - for (const question of deepQuestions) { - if (!questions.has(question.id)) questions.set(question.id, question) - } - - const text = buildSteerText(gaps, deepQuestions, invalidationTargets, minIndependentSources) - lastSteer = { round, deepQuestions, invalidationTargets, gaps, text } - options.onSteer?.(lastSteer) - return text - }, - - researchState: snapshot, - - isComplete(): boolean { - const all = [...claims.values()] - if (all.length === 0) return false - const everyClaimSettled = all.every((claim) => isCorroborated(claim) || claim.contested) - const everyQuestionAddressed = [...questions.values()].every((question) => question.addressed) - return everyClaimSettled && everyQuestionAddressed - }, - - lastSteer(): ResearchDrivingSteer | undefined { - return lastSteer - }, - - checkpoint: persist, - - prepareFold, - - commitSources, - - toLedger, - } - - // -- claim extraction ------------------------------------------------------ - - async function extractClaims( - source: ResearchSourceProposal, - goal: string, - signal?: AbortSignal, - ): Promise { - const ledger = claimsForExtraction() - const fromLlm = await extractClaimsWithLlm(source, goal, ledger, signal) - throwIfRouterAborted(signal) - if (fromLlm.length > 0) return fromLlm.slice(0, maxClaimsPerSource) - if (deterministicFallback) return deterministicClaims(source).slice(0, maxClaimsPerSource) - return [] - } - - function claimsForExtraction(): TrackedClaim[] { - const known = new Map(claims) - for (const evidence of claimEvidence.values()) { - if (known.has(evidence.claimId)) continue - known.set(evidence.claimId, { - id: evidence.claimId, - text: evidence.text, - supportingHosts: new Set(), - supportingUris: [], - contradicts: new Set(), - contested: false, - firstSeenRound: evidence.firstSeenRound, - }) - } - return [...known.values()] - } - - async function extractClaimsWithLlm( - source: ResearchSourceProposal, - goal: string, - ledger: TrackedClaim[], - signal?: AbortSignal, - ): Promise { - let router: RouterClient - try { - router = resolveRouter(signal) - } catch (error) { - throwIfRouterAborted(signal) - if ((error as { name?: string } | null)?.name === 'AbortError') throw error - return [] - } - const excerpt = source.text.slice(0, 1800) - const ledgerLines = ledger - .slice(0, 20) - .map((claim) => `- [${claim.id}] ${claim.text}`) - .join('\n') - const system = - 'You extract the KEY factual claims a researcher would cite a page for, and flag ' + - 'CONTRADICTIONS with claims already on the ledger. ' + - "A claim is one concrete, checkable assertion using the page's own terms and numbers. " + - 'Return ONLY a JSON array; each item is {"claim": string, "contradicts": string|null} where ' + - 'contradicts is the bracketed [id] of a ledger claim this page DIRECTLY contradicts, else null. ' + - `Return at most ${maxClaimsPerSource} claims. No prose.` - const user = [ - `Research goal: ${goal}`, - `Page title: ${source.title ?? '(none)'}`, - ledgerLines ? `Claims already on the ledger:\n${ledgerLines}` : 'Ledger is empty.', - `Page excerpt:\n${excerpt}`, - 'Key claims as JSON [{"claim": "...", "contradicts": "[id]"|null}]:', - ].join('\n\n') - - let raw = '' - try { - raw = await router.chat( - [ - { role: 'system', content: system }, - { role: 'user', content: user }, - ], - 1200, - ) - } catch (error) { - throwIfRouterAborted(signal) - if ((error as { name?: string } | null)?.name === 'AbortError') throw error - return [] - } - throwIfRouterAborted(signal) - return parseExtractedClaims(raw, ledger) - } - - /** - * Deterministic fallback: pull the leading sentences as candidate claims. Used - * only when the model is unavailable, so the driver still drives (and the - * offline test runs with no creds). Each sentence becomes a checkable claim. - */ - function deterministicClaims(source: ResearchSourceProposal): ExtractedClaim[] { - const sentences = source.text - .split(/(?<=[.!?])\s+/) - .map((sentence) => sentence.trim()) - .filter((sentence) => contentWordSet(sentence).size >= 3) - return sentences.slice(0, maxClaimsPerSource).map((text) => ({ text })) - } - - // -- deep-question synthesis ---------------------------------------------- - - /** - * Build the four deep-question kinds from the actual ledger claims. This is - * intentionally deterministic (not an LLM call): the loop's `foldGaps` contract - * is synchronous (`foldGaps(gaps): string`), and a template grounded in the - * real claim text gives a faithful, non-fabricated sub-question every round — - * comparative, mechanism, gap, and contradiction. Claim EXTRACTION, which runs - * inside the awaited `verifySource`, is where the model does the open-ended - * work; question generation just interrogates what was extracted. - */ - function synthesizeDeepQuestions(ledger: TrackedClaim[], round: number): DeepQuestion[] { - if (ledger.length === 0) return [] - const out: DeepQuestion[] = [] - - // CONTRADICTION questions: for every contradiction edge, ask the worker to - // find evidence that resolves which claim holds. - for (const claim of ledger) { - for (const otherId of claim.contradicts) { - const other = ledger.find((entry) => entry.id === otherId) - if (!other || other.id < claim.id) continue // emit each pair once - out.push( - makeQuestion( - 'contradiction', - `Does any independent source resolve the contradiction between "${truncate(claim.text)}" and "${truncate(other.text)}"? Find evidence that confirms or refutes one of them.`, - [claim.id, other.id], - round, - ), - ) - } - } - - // GAP questions: for each weakly-supported claim, ask for the specific - // corroborating result that is missing. - for (const claim of ledger.filter((entry) => !entry.contested)) { - if (claim.supportingHosts.size < minIndependentSources) { - out.push( - makeQuestion( - 'gap', - `Only one independent source supports "${truncate(claim.text)}". What specific corroborating result, dataset, or independent measurement is missing to confirm it?`, - [claim.id], - round, - ), - ) - } - } - - // Probe where the best-supported claims stop holding. - for (const claim of [...ledger] - .sort((a, b) => b.supportingHosts.size - a.supportingHosts.size) - .slice(0, 2)) { - out.push( - makeQuestion( - 'mechanism', - `Under what precise condition does "${truncate(claim.text)}" stop holding? Find a source that states the mechanism, limit, or failure mode.`, - [claim.id], - round, - ), - ) - } - - // Compare tradeoffs between the two best-supported claims. - const ranked = [...ledger].sort((a, b) => b.supportingHosts.size - a.supportingHosts.size) - if (ranked.length >= 2 && ranked[0] && ranked[1]) { - out.push( - makeQuestion( - 'comparative', - `How does the tradeoff in "${truncate(ranked[0].text)}" differ from "${truncate(ranked[1].text)}"? Find a source that compares them directly.`, - [ranked[0].id, ranked[1].id], - round, - ), - ) - } - - // Stable order: contradiction → gap → mechanism → comparative (most urgent - // validation work first). - const priority: Record = { - contradiction: 0, - gap: 1, - mechanism: 2, - comparative: 3, - } - return out.sort((a, b) => priority[a.kind] - priority[b.kind]) - } -} - -// --------------------------------------------------------------------------- -// pure helpers -// --------------------------------------------------------------------------- - -function sourceVersionOfProposal(source: ResearchSourceProposal): ResearchSourceVersion { - const contentHash = sha256(source.text) - return { - sourceId: textSourceId(source.uri, contentHash), - uri: source.uri, - contentHash, - } -} - -function sourceVersionOfRecord(source: SourceRecord): ResearchSourceVersion { - const originalUri = source.metadata?.originalUri - if (typeof originalUri !== 'string' || originalUri.length === 0) { - throw new Error(`registered source '${source.id}' has no originalUri`) - } - if (!/^[a-f0-9]{64}$/.test(source.contentHash)) { - throw new Error(`registered source '${source.id}' contentHash is not a SHA-256 digest`) - } - const expectedSourceId = textSourceId(originalUri, source.contentHash) - if (source.id !== expectedSourceId) { - throw new Error( - `registered source '${source.id}' does not match URI-and-content identity '${expectedSourceId}'`, - ) - } - return { sourceId: source.id, uri: originalUri, contentHash: source.contentHash } -} - -function sourceVersionKeyOfEvidence(evidence: ResearchClaimEvidence): string { - return researchSourceVersionKey({ - sourceId: evidence.sourceId, - uri: evidence.sourceUri, - contentHash: evidence.sourceContentHash, - }) -} - -function makeQuestion( - kind: DeepQuestionKind, - text: string, - claimIds: string[], - raisedRound: number, -): DeepQuestion { - return { - kind, - text, - id: deepQuestionId(kind, text), - claimIds: [...new Set(claimIds)].sort(), - addressed: false, - raisedRound, - } -} - -function fromRecord(claim: ResearchClaimRecord): TrackedClaim { - return { - ...claim, - supportingHosts: new Set(claim.supportingHosts), - supportingUris: [...claim.supportingUris], - contradicts: new Set(claim.contradicts), - } -} - -/** - * Set-like insert into the array form. The ledger's collections are arrays so - * they survive `JSON.stringify`; dedup is enforced here instead of by the type. - */ -function addUnique(values: string[], value: string): void { - if (!values.includes(value)) values.push(value) -} - -const stopwords = new Set([ - 'the', - 'a', - 'an', - 'and', - 'or', - 'but', - 'of', - 'to', - 'in', - 'on', - 'for', - 'with', - 'as', - 'by', - 'at', - 'from', - 'that', - 'this', - 'these', - 'those', - 'it', - 'its', - 'is', - 'are', - 'was', - 'were', - 'be', - 'been', - 'being', - 'has', - 'have', - 'had', - 'can', - 'will', - 'would', - 'should', - 'may', - 'might', - 'not', - 'no', - 'than', - 'then', - 'over', - 'under', - 'about', - 'into', - 'their', - 'they', - 'them', -]) - -function contentWordSet(text: string): Set { - return new Set( - normalizeClaimText(text) - .split(' ') - .filter((word) => word.length >= 3 && !stopwords.has(word)), - ) -} - -function overlapFraction(a: Set, b: Set): number { - if (a.size === 0) return 0 - let hits = 0 - for (const word of a) if (b.has(word)) hits += 1 - return hits / a.size -} - -function truncate(text: string, max = 140): string { - const t = text.trim().replace(/\s+/g, ' ') - return t.length <= max ? t : `${t.slice(0, max - 1)}…` -} - -/** - * Parse the extractor's JSON array of `{claim, contradicts}` items, tolerant of - * code fences / surrounding prose. `contradicts` is mapped from a `[id]` token to - * a ledger claim id only when that id is actually on the ledger. - */ -function parseExtractedClaims(raw: string, ledger: TrackedClaim[]): ExtractedClaim[] { - const text = raw.trim() - if (!text) return [] - const arrayMatch = text.match(/\[[\s\S]*\]/) - if (!arrayMatch) return [] - let parsed: unknown - try { - parsed = JSON.parse(arrayMatch[0]) - } catch { - return [] - } - if (!Array.isArray(parsed)) return [] - const ledgerIds = new Set(ledger.map((claim) => claim.id)) - const out: ExtractedClaim[] = [] - for (const item of parsed) { - if (!item || typeof item !== 'object') continue - const record = item as { claim?: unknown; contradicts?: unknown } - if (typeof record.claim !== 'string' || !record.claim.trim()) continue - const contradictsId = extractBracketId(record.contradicts) - out.push({ - text: record.claim.trim(), - contradictsExistingId: - contradictsId && ledgerIds.has(contradictsId) ? contradictsId : undefined, - }) - } - return out -} - -/** Pull a ledger id out of a `contradicts` field: `"[c_abc]"` or `"c_abc"`. */ -function extractBracketId(value: unknown): string | undefined { - if (typeof value !== 'string') return undefined - const trimmed = value.trim() - if (!trimmed || trimmed.toLowerCase() === 'null') return undefined - const bracket = trimmed.match(/\[([^\]]+)\]/) - const id = (bracket?.[1] ?? trimmed).trim() - return id.startsWith('c_') ? id : undefined -} - -/** - * Build the steer string handed to the worker's next prompt. Interleaves the - * readiness gaps the loop still reports with the deep sub-questions and the - * invalidation challenges — the part that drives DEPTH + VALIDATION. - */ -function buildSteerText( - gaps: KnowledgeGap[], - deepQuestions: DeepQuestion[], - invalidationTargets: TrackedClaim[], - minIndependentSources: number, -): string { - const lines: string[] = [] - lines.push( - 'Do NOT just add more sources. Go DEEPER and VALIDATE. Address the following before adding breadth:', - ) - - if (deepQuestions.length > 0) { - lines.push('', 'Deep sub-questions to answer this round:') - for (const question of deepQuestions) { - lines.push(`- (${question.kind}) ${question.text}`) - } - } - - if (invalidationTargets.length > 0) { - lines.push( - '', - `Claims needing corroboration or refutation (each must reach >= ${minIndependentSources} INDEPENDENT sources, or be shown contested):`, - ) - for (const claim of invalidationTargets) { - const reason = - claim.contradicts.size > 0 - ? 'CONTRADICTED by another source — find evidence that resolves it' - : `only ${claim.supportingHosts.size} independent source — find a SECOND, independent corroborating source` - lines.push(`- "${truncate(claim.text)}" — ${reason}`) - } - } - - if (gaps.length > 0) { - lines.push('', 'Readiness gaps still open:') - for (const gap of gaps) { - lines.push(`- (${gap.blocking ? 'blocking' : 'soft'}) ${gap.description} [${gap.id}]`) - } - } - - return lines.join('\n') -} diff --git a/src/types.ts b/src/types.ts index 4f7acd4..762b5da 100644 --- a/src/types.ts +++ b/src/types.ts @@ -229,8 +229,7 @@ export interface KnowledgeWriteParseResult { /** * The event vocabulary, as a value so the runtime schema is DERIVED from it * rather than restated. A restated copy in `schemas.ts` drifted: it omitted - * `research.iteration`, which is the only event `runVerifiedResearchLoop` - * produces, so every attempt to store one would have been rejected. Nothing + * `research.iteration`, which the research loop produces, so every attempt to store one would have been rejected. Nothing * caught it because nothing ever stored an event. Add a type here and the * schema accepts it in the same edit. */ @@ -280,7 +279,7 @@ export interface DeepQuestion { raisedRound: number } -/** One live tracked claim exposed by the research-driving API. */ +/** One live tracked claim, with Set fields; `ResearchClaimRecord` is its durable form. */ export interface TrackedClaim { id: string /** The claim text as first extracted (kept for prompts/audit). */ diff --git a/src/verified-research-loop.ts b/src/verified-research-loop.ts deleted file mode 100644 index 73ad2ad..0000000 --- a/src/verified-research-loop.ts +++ /dev/null @@ -1,546 +0,0 @@ -import type { KnowledgeReadinessReport } from '@tangle-network/agent-eval' -import { - type BuildEvalKnowledgeBundleOptions, - buildEvalKnowledgeBundle, - type EvalKnowledgeBundleBuildResult, - type KnowledgeReadinessSpec, -} from './eval-readiness' -import { createKnowledgeEvent } from './events' -import { buildKnowledgeIndex } from './indexer' -import { FileSystemKbStore } from './kb-store' -import { applyKnowledgeWriteBlocks } from './proposals' -import { readinessFor } from './readiness-helpers' -import { searchKnowledge } from './search' -import { - type AddSourceOptions, - type AddSourceTextInput, - addSourceText, - snapshotSourceTextInput, -} from './sources' -import { initKnowledgeBase } from './store' -import type { KnowledgeEvent, KnowledgeIndex, KnowledgeSearchResult, SourceRecord } from './types' - -/** - * A knowledge gap the loop surfaces from `scoreKnowledgeReadiness`. The worker - * targets these; the driver folds the unfilled remainder into the worker's next - * prompt and runs its own gap-fill pass over them. - */ -export interface KnowledgeGap { - /** Readiness-spec id this gap belongs to. */ - id: string - /** Human-readable description of what's missing. */ - description: string - /** The search query the readiness check ran for this requirement. */ - query: string - /** True when the gap blocks readiness (vs. a soft, non-blocking gap). */ - blocking: boolean -} - -/** A new source the worker (or driver) discovered and wants to add to the KB. */ -export type ResearchSourceProposal = AddSourceTextInput - -/** - * What a research agent contributes in one round. Both the worker and (when - * `driverResearches` is on) the driver produce this shape — the worker ADDS - * primary findings, the driver gap-FILLS the ones the worker missed. - * - * `proposalText` is the safe write-protocol text (`---FILE: knowledge/...---` - * blocks). The loop only applies it AFTER the driver has verified the round's - * sources, so a rejected source never reaches the curated pages. - */ -export interface ResearchContribution { - /** Immutable sources to register (the raw evidence). */ - sources?: ResearchSourceProposal[] - /** Safe write-protocol text producing curated `knowledge/*.md` pages. */ - proposalText?: string - /** - * Build the page write-protocol text FROM the sources the driver accepted — - * the curated, citing pages the readiness gate searches. Receives the - * registered `SourceRecord`s (with their assigned ids, so a page's frontmatter - * `sources:` can cite them). Returns `---FILE: knowledge/...---` block text or - * `undefined`. Runs after verification, so a page never cites a rejected - * source. Concatenated after any static `proposalText`. - */ - buildPages?: (acceptedSources: SourceRecord[]) => string | undefined - /** Free-form research transcript — products can persist this. */ - notes?: string - metadata?: Record -} - -/** Context handed to the worker each round. */ -export interface WorkerResearchContext { - root: string - goal: string - round: number - index: KnowledgeIndex - /** Gaps the readiness gate currently reports — what the worker should close. */ - gaps: KnowledgeGap[] - /** Steer text the driver folded in from the previous round's remaining gaps. */ - steer?: string - readiness: EvalKnowledgeBundleBuildResult - signal?: AbortSignal -} - -/** Context handed to the driver's verifier for one candidate source. */ -export interface SourceVerificationContext { - root: string - goal: string - round: number - index: KnowledgeIndex - gaps: KnowledgeGap[] - /** Sources already accepted earlier THIS round (in-round dedup). */ - acceptedThisRound: ResearchSourceProposal[] - signal?: AbortSignal -} - -/** A single rejected source plus the reason the driver gave. */ -export interface RejectedSource { - source: ResearchSourceProposal - reason: string -} - -/** Context handed to the driver's gap-fill pass (only when `driverResearches`). */ -export interface DriverResearchContext { - root: string - goal: string - round: number - index: KnowledgeIndex - /** Gaps STILL open after the worker's accepted contribution applied. */ - remainingGaps: KnowledgeGap[] - readiness: EvalKnowledgeBundleBuildResult - signal?: AbortSignal -} - -/** - * The differentiated driver role. - * - * - `verifySource` — the gate the worker's additions pass before they commit. - * Return `{ accept: true }` to keep a source or `{ accept: false, reason }` - * to reject it (not real / not relevant / duplicate). The loop dedups exact - * duplicates (same `uri` already in the KB or accepted this round) BEFORE - * calling this, so the verifier only sees genuinely-new candidates. - * - `research` — the driver's OWN gap-fill pass over the gaps the worker left - * open. Only invoked when `driverResearches` is true. - * - `foldGaps` — turn the remaining gaps into a steer string for the worker's - * next prompt. Defaults to a compact bulleted list when omitted. - * - `isComplete` — require the driver's research objectives as well as storage readiness. - * - `checkpoint` — write whatever state the driver accumulated to durable - * storage. Called at the end of every round, after `foldGaps`, so state that - * a synchronous hook produced is on disk before the next round can crash. - * - `prepareFold` — durably announce the next synchronous fold before it runs, - * so a crash between question generation and `checkpoint` can be recovered. - * - `commitSources` — confirm exact source records are durable after verification; - * drivers with pending evidence must not count it before this callback. - */ -export interface ResearchDriver { - verifySource( - source: ResearchSourceProposal, - ctx: SourceVerificationContext, - ): Promise | SourceVerdict - research?(ctx: DriverResearchContext): Promise | ResearchContribution - foldGaps?(gaps: KnowledgeGap[]): string - isComplete?(): boolean - prepareFold?(): Promise | void - commitSources?(sources: readonly SourceRecord[]): Promise | void - checkpoint?(): Promise | void -} - -export type SourceVerdict = { accept: true } | { accept: false; reason: string } - -/** The worker: primary research targeting the round's gaps. */ -export type ResearchWorker = ( - ctx: WorkerResearchContext, -) => Promise | ResearchContribution - -export interface VerifiedResearchLoopOptions { - root: string - goal: string - worker: ResearchWorker - driver: ResearchDriver - /** - * When false (default), the driver ONLY verifies + gates — a pure coordinator - * that contributes no research of its own (the "doesn't participate in the - * work" mode). When true, the driver also runs its `research` gap-fill pass - * each round over the gaps the worker left open. - */ - driverResearches?: boolean - maxRounds?: number - actor?: string - /** Readiness specs define the gate; an empty list means the loop never gates. */ - readinessSpecs?: KnowledgeReadinessSpec[] - readinessTaskId?: string - readiness?: Omit - sourceOptions?: Pick - signal?: AbortSignal - onRound?: (round: VerifiedResearchRound) => Promise | void -} - -export interface VerifiedResearchRound { - round: number - /** Gaps reported at the START of the round (what the worker targeted). */ - gaps: KnowledgeGap[] - /** Worker sources accepted by the driver and written to the KB. */ - acceptedWorkerSources: SourceRecord[] - /** Worker sources the driver rejected (with reasons) — never written. */ - rejectedWorkerSources: RejectedSource[] - /** Sources the driver itself added in its gap-fill pass. */ - driverSources: SourceRecord[] - /** Curated pages written this round (worker proposal + driver proposal). */ - writtenPages: string[] - readiness?: EvalKnowledgeBundleBuildResult - /** True once readiness passes and the driver reports completion, when configured. */ - ready: boolean - event: KnowledgeEvent - notes: { worker?: string; driver?: string } -} - -export interface VerifiedResearchLoopResult { - root: string - goal: string - rounds: number - ready: boolean - index: KnowledgeIndex - readiness?: EvalKnowledgeBundleBuildResult - steps: VerifiedResearchRound[] -} - -/** - * Two-agent (driver + worker) sibling of `runKnowledgeResearchLoop`. - * - * Both agents research to grow ONE knowledge base. The roles are differentiated: - * - * - **WORKER** = primary research. Each round it reads the open gaps, discovers - * new sources, and proposes additions (`sources` + `proposalText`). It ADDS. - * - **DRIVER** = the verifier / gap-filler / gate. It (1) VERIFIES the worker's - * sources before they commit — dedup against the KB, then `verifySource` - * rejects ones that aren't real/relevant; (2) GAP-FILLS the gaps the worker - * missed with its own research pass (when `driverResearches`); (3) folds the - * remaining gaps into the worker's next prompt; and (4) GATES on - * `scoreKnowledgeReadiness` and optional `isComplete` — the loop stops when - * readiness passes and the driver has no unfinished research. - * - * Set `driverResearches: false` (default) for the pure-coordinator mode: the - * driver only verifies + gates and contributes no research itself. - * - * Composes existing atoms — `initKnowledgeBase`, `addSourceText`, - * `applyKnowledgeWriteBlocks`, `buildEvalKnowledgeBundle` (the readiness gate), - * and `searchKnowledge` — and reinvents none of them. - */ -export async function runVerifiedResearchLoop( - options: VerifiedResearchLoopOptions, -): Promise { - const assertActive = () => options.signal?.throwIfAborted() - assertActive() - const maxRounds = Math.max(1, options.maxRounds ?? 3) - await initKnowledgeBase(options.root) - assertActive() - const store = new FileSystemKbStore({ root: options.root }) - const steps: VerifiedResearchRound[] = [] - let index = await buildKnowledgeIndex(options.root) - assertActive() - // Reconcile source writes that completed before a previous process died while - // confirming them to the driver. The records carry both original URI and hash. - await confirmRegisteredSources(options.driver, index.sources) - assertActive() - let readiness = readinessFor(options, index) - let ready = isReady(readiness?.report) && (options.driver.isComplete?.() ?? true) - let steer: string | undefined - - for (let round = 1; round <= maxRounds && !ready; round++) { - assertActive() - - const gaps = gapsFromReadiness(readiness) - - // 1. WORKER: primary research over the open gaps. - const workerContribution = await options.worker({ - root: options.root, - goal: options.goal, - round, - index, - gaps, - steer, - readiness: requireReadiness(readiness, options), - signal: options.signal, - }) - assertActive() - - // 2. DRIVER VERIFIES the worker's sources before they commit. - const accepted: ResearchSourceProposal[] = [] - const rejectedWorkerSources: RejectedSource[] = [] - // Dedup against the ORIGINAL input uri. `addSourceText` rewrites `record.uri` - // to a slugified raw path and stashes the caller's uri under - // `metadata.originalUri`, so that — not the stored uri — is the round-to-round - // identity a verifier dedups against. - const existingUris = new Set( - index.sources.flatMap((source) => - typeof source.metadata?.originalUri === 'string' ? [source.metadata.originalUri] : [], - ), - ) - for (const proposedSource of workerContribution.sources ?? []) { - const source = snapshotSourceTextInput(proposedSource) - if (isDuplicate(source, existingUris, accepted)) { - rejectedWorkerSources.push({ source, reason: 'duplicate: already in the knowledge base' }) - continue - } - const verdict = await options.driver.verifySource(source, { - root: options.root, - goal: options.goal, - round, - index, - gaps, - acceptedThisRound: accepted, - signal: options.signal, - }) - assertActive() - if (verdict.accept) accepted.push(source) - else rejectedWorkerSources.push({ source, reason: verdict.reason }) - } - - // Register the accepted worker sources, then apply the worker's curated - // pages — but only when at least one source survived verification, so a - // page never cites a rejected source. - const acceptedWorkerSources = await registerSources(options, accepted) - assertActive() - await confirmRegisteredSources(options.driver, acceptedWorkerSources) - assertActive() - const writtenPages: string[] = [] - writtenPages.push( - ...(await applyPages(options.root, workerContribution, acceptedWorkerSources)), - ) - assertActive() - - // Re-index so the driver's gap-fill pass sees the worker's contribution. - index = await buildKnowledgeIndex(options.root) - assertActive() - readiness = readinessFor(options, index) - - // 3. DRIVER GAP-FILLS the gaps the worker left open (opt-in). - let driverSources: SourceRecord[] = [] - let driverNotes: string | undefined - if (options.driverResearches && options.driver.research) { - const remainingGaps = gapsFromReadiness(readiness) - const driverContribution = await options.driver.research({ - root: options.root, - goal: options.goal, - round, - index, - remainingGaps, - readiness: requireReadiness(readiness, options), - signal: options.signal, - }) - assertActive() - driverNotes = driverContribution.notes - driverSources = await registerSources(options, driverContribution.sources ?? []) - assertActive() - await confirmRegisteredSources(options.driver, driverSources) - assertActive() - writtenPages.push(...(await applyPages(options.root, driverContribution, driverSources))) - assertActive() - index = await buildKnowledgeIndex(options.root) - assertActive() - readiness = readinessFor(options, index) - } - - // 4. DRIVER GATES on readiness and folds the remainder into the next prompt. - const driverComplete = options.driver.isComplete?.() ?? true - ready = isReady(readiness?.report) && driverComplete - const remainingGaps = gapsFromReadiness(readiness) - if (ready || (remainingGaps.length === 0 && driverComplete)) { - steer = undefined - } else { - await options.driver.prepareFold?.() - assertActive() - steer = foldGaps(options.driver, remainingGaps) - } - - const step: VerifiedResearchRound = { - round, - gaps, - acceptedWorkerSources, - rejectedWorkerSources, - driverSources, - writtenPages, - readiness, - ready, - event: createKnowledgeEvent({ - type: 'research.iteration', - actor: options.actor, - target: options.root, - metadata: { - goal: options.goal, - round, - ready, - acceptedWorkerSourceCount: acceptedWorkerSources.length, - rejectedWorkerSourceCount: rejectedWorkerSources.length, - driverSourceCount: driverSources.length, - writtenPageCount: writtenPages.length, - remainingGapCount: remainingGaps.length, - }, - }), - notes: { worker: workerContribution.notes, driver: driverNotes }, - } - // Commit the driver's state before publishing the round event. A persisted - // event therefore never claims a round whose generated questions were lost. - await options.driver.checkpoint?.() - assertActive() - await store.putEvent(step.event) - assertActive() - - steps.push(step) - await options.onRound?.(step) - assertActive() - } - - assertActive() - return { - root: options.root, - goal: options.goal, - rounds: steps.length, - ready, - index, - readiness, - steps, - } -} - -function isReady(report: KnowledgeReadinessReport | undefined): boolean { - // No specs ⇒ no gate ⇒ the loop runs to `maxRounds`. With specs, the gate is - // "no blocking gaps remain". - if (!report) return false - return report.blockingMissingRequirements.length === 0 -} - -function gapsFromReadiness(readiness: EvalKnowledgeBundleBuildResult | undefined): KnowledgeGap[] { - if (!readiness) return [] - const blocking = readiness.report.blockingMissingRequirements.map((requirement) => - gapFor(requirement, readiness, true), - ) - const nonBlocking = readiness.report.nonBlockingGaps.map((requirement) => - gapFor(requirement, readiness, false), - ) - return [...blocking, ...nonBlocking] -} - -function gapFor( - requirement: { id: string; description: string; metadata?: Record }, - readiness: EvalKnowledgeBundleBuildResult, - blocking: boolean, -): KnowledgeGap { - const spec = readiness.requirements.find((entry) => entry.id === requirement.id) - const query = - typeof spec?.metadata?.query === 'string' ? spec.metadata.query : requirement.description - return { id: requirement.id, description: requirement.description, query, blocking } -} - -function foldGaps(driver: ResearchDriver, gaps: KnowledgeGap[]): string | undefined { - if (driver.foldGaps) return driver.foldGaps(gaps) - if (gaps.length === 0) return undefined - return [ - 'The knowledge base is still missing the following. Prioritise these next round:', - ...gaps.map( - (gap) => `- (${gap.blocking ? 'blocking' : 'soft'}) ${gap.description} [${gap.id}]`, - ), - ].join('\n') -} - -function isDuplicate( - source: ResearchSourceProposal, - existingUris: Set, - accepted: ResearchSourceProposal[], -): boolean { - return existingUris.has(source.uri) || accepted.some((candidate) => candidate.uri === source.uri) -} - -async function registerSources( - options: VerifiedResearchLoopOptions, - sources: ResearchSourceProposal[], -): Promise { - const records: SourceRecord[] = [] - for (const candidate of sources) { - options.signal?.throwIfAborted() - const source = snapshotSourceTextInput(candidate) - records.push(await addSourceText(options.root, source, options.sourceOptions)) - options.signal?.throwIfAborted() - } - return records -} - -async function confirmRegisteredSources( - driver: ResearchDriver, - sources: readonly SourceRecord[], -): Promise { - if (!driver.commitSources) return - const textSources = sources.filter((source) => typeof source.metadata?.originalUri === 'string') - if (textSources.length === 0) return - await driver.commitSources(textSources) -} - -/** - * Apply a contribution's curated pages. Static `proposalText` plus a - * `buildPages(acceptedSources)` result are concatenated and run through the safe - * write protocol — but ONLY when at least one source survived verification, so a - * page never cites a rejected (or absent) source. - */ -async function applyPages( - root: string, - contribution: ResearchContribution, - acceptedSources: SourceRecord[], -): Promise { - if (acceptedSources.length === 0) return [] - const parts: string[] = [] - if (contribution.proposalText) parts.push(contribution.proposalText) - const built = contribution.buildPages?.(acceptedSources) - if (built) parts.push(built) - if (parts.length === 0) return [] - const applied = await applyKnowledgeWriteBlocks(root, parts.join('\n')) - return applied.written -} - -function requireReadiness( - readiness: EvalKnowledgeBundleBuildResult | undefined, - options: VerifiedResearchLoopOptions, -): EvalKnowledgeBundleBuildResult { - if (readiness) return readiness - // The worker/driver contexts type `readiness` as required for ergonomics; when - // no specs are configured there is no gate to report, so synthesise an empty - // bundle rather than forcing every caller to handle `undefined`. - return buildEvalKnowledgeBundle({ - ...(options.readiness ?? {}), - taskId: options.readinessTaskId ?? options.goal, - index: emptyIndex(options.root), - specs: [], - }) -} - -function emptyIndex(root: string): KnowledgeIndex { - return { - root, - generatedAt: new Date(0).toISOString(), - sources: [], - pages: [], - graph: { nodes: [], edges: [] }, - } -} - -/** - * Helper for verifiers: does the candidate source's text/title overlap any page - * the readiness search returns for a gap query? A cheap relevance heuristic the - * driver can compose into `verifySource` (real verifiers can do more). - */ -export function sourceMatchesGaps( - source: ResearchSourceProposal, - index: KnowledgeIndex, - gaps: KnowledgeGap[], -): KnowledgeSearchResult[] { - const haystack = `${source.title ?? ''}\n${source.text}`.toLowerCase() - const hits: KnowledgeSearchResult[] = [] - for (const gap of gaps) { - for (const token of gap.query.toLowerCase().split(/\s+/).filter(Boolean)) { - if (haystack.includes(token)) { - hits.push(...searchKnowledge(index, gap.query, 1)) - break - } - } - } - return hits -} diff --git a/src/web-research-worker.ts b/src/web-research-worker.ts deleted file mode 100644 index 74ecd2d..0000000 --- a/src/web-research-worker.ts +++ /dev/null @@ -1,588 +0,0 @@ -/** - * Real web-research worker + verifying driver for `runVerifiedResearchLoop`. - * - * This is the GENERAL, any-topic implementation behind the two-agent research - * loop's live arm. Given the open knowledge gaps the readiness gate surfaces, - * the worker: - * - * 1. asks an LLM (glm-5.2 by default) to turn each gap into focused web - * search queries, - * 2. runs a REAL web search over the Tangle router (`POST /v1/search` — the - * same endpoint `tcloud mcp`'s `web_search` tool forwards to), so there is - * no hardcoded corpus, - * 3. fetches the top results with the repo's polite, cached `politeFetch` and - * reduces each page to text with `htmlToText`, - * 4. proposes the readable, verifiable pages as `ResearchSourceProposal`s plus - * a `buildPages` that writes citing `knowledge/*.md` pages from the sources - * the driver accepts. - * - * The verifying DRIVER is the differentiated role from the two-agent loop: a - * second LLM pass that judges each fetched source's on-topic relevance to the - * goal + open gaps and rejects off-topic / spam / already-covered material. The - * worker ADDS; the driver GATES. Together they build a cleaner knowledge base - * than a single agent at the same compute budget. - * - * Model and search requests use the published TCloud SDK. Knowledge owns the - * research policy, not HTTP, bearer headers, retries or provider rate cards. - * Point it at any router with `baseUrl`; supply `apiKey` or `TANGLE_API_KEY`. - */ - -import { type SearchProvider, TCloudClient, TCloudError } from '@tangle-network/tcloud' -import { htmlToText } from './sources/html' -import { politeFetch } from './sources/http' -import type { SourceRecord } from './types' -import type { - KnowledgeGap, - ResearchContribution, - ResearchDriver, - ResearchSourceProposal, - ResearchWorker, - SourceVerdict, - SourceVerificationContext, - WorkerResearchContext, -} from './verified-research-loop' - -/** Default router model. Plain id (no namespace) — see CLAUDE/creds notes. */ -const DEFAULT_MODEL = 'glm-5.2' -/** Default router base. */ -const DEFAULT_BASE_URL = 'https://router.tangle.tools/v1' -/** - * glm-5.2 spends its first tokens on hidden `reasoning_content`; below ~1200 - * output tokens it returns EMPTY visible content. Floor every call so a - * reasoning model never silently yields nothing. (Verified creds note.) - */ -const MIN_MAX_TOKENS = 1200 - -/** One live web result, as the router's `/v1/search` returns it. */ -export interface WebSearchHit { - title: string - url: string - snippet?: string -} - -/** - * The two router capabilities the worker/driver need. Injectable so tests can - * supply a client; the default uses the published TCloud SDK. - */ -export interface RouterClient { - /** Live web search — returns title/url/snippet hits. */ - search(query: string, opts?: { maxResults?: number }): Promise - /** Chat completion — returns the assistant message's visible text. */ - chat( - messages: { role: 'system' | 'user'; content: string }[], - maxTokens?: number, - ): Promise - /** Cumulative cost (chat + search) since this client was created. */ - usage(): RouterUsage -} - -/** - * Cumulative router cost — the per-arm signal the A/B reports ALONGSIDE quality, - * so "2.3 fewer sources" can be read against the token/$/latency it cost. A - * two-agent round is one worker pass plus N `verifySource` LLM calls; counting - * each call here is what surfaces that the two-agent loop spends more inference - * than its "equal passes" budget implies. - */ -export interface RouterUsage { - chatCalls: number - searchCalls: number - promptTokens: number - completionTokens: number - /** Reported cost only. NaN once a successful response omits its cost receipt. */ - usd: number - wallMs: number -} - -export interface TangleRouterOptions { - /** Router base URL. Defaults to `https://router.tangle.tools/v1`. */ - baseUrl?: string - /** Bearer key. Defaults to `process.env.TANGLE_API_KEY`. */ - apiKey?: string - /** Chat model id. Defaults to `glm-5.2`. */ - model?: string - /** Optional preferred search provider (exa | you | perplexity | …). */ - searchProvider?: string - /** SDK retries for 429/502/503/504. Default 4. Kept for API compatibility. */ - maxRetries?: number - /** Initial SDK retry backoff in ms. Default 1500. */ - retryBaseMs?: number - signal?: AbortSignal -} - -/** Compatibility error facade. HTTP execution and classification belong to TCloud. */ -export class RouterError extends Error { - constructor( - public readonly status: number, - message: string, - ) { - super(`router ${status}: ${message}`) - this.name = 'RouterError' - } -} - -/** Adapt Knowledge's public contract to the published SDK, without a second transport. */ -export function createTangleRouterClient(options: TangleRouterOptions = {}): RouterClient { - const apiKey = options.apiKey ?? process.env.TANGLE_API_KEY - if (!apiKey) throw new RouterError(401, 'no TANGLE_API_KEY (pass apiKey or set the env var)') - const client = new TCloudClient({ - baseURL: (options.baseUrl ?? DEFAULT_BASE_URL).replace(/\/+$/, ''), - apiKey, - model: options.model ?? DEFAULT_MODEL, - // Preserve the unbounded reasoning timeout. The caller can cancel the actual request. - timeout: 0, - retry: { - maxRetries: Math.max(0, options.maxRetries ?? 4), - initialBackoffMs: Math.max(1, options.retryBaseMs ?? 1500), - retryableStatuses: [429, 502, 503, 504], - }, - }) - const acc: RouterUsage = { - chatCalls: 0, - searchCalls: 0, - promptTokens: 0, - completionTokens: 0, - usd: 0, - wallMs: 0, - } - // NaN is deliberately not zero: an incomplete bill must not win a cost comparison. - const recordCost = (cost: number | undefined) => { - acc.usd += typeof cost === 'number' && Number.isFinite(cost) && cost >= 0 ? cost : Number.NaN - } - const translate = (error: unknown): never => { - // Preserve the caller's exact abort reason, including non-Error reasons. - options.signal?.throwIfAborted() - if (error instanceof TCloudError) throw new RouterError(error.status, error.message) - throw error - } - - return { - async search(query, opts) { - options.signal?.throwIfAborted() - const started = Date.now() - try { - const response = await client.search({ - query, - ...(options.searchProvider ? { provider: options.searchProvider as SearchProvider } : {}), - ...(opts?.maxResults != null ? { maxResults: opts.maxResults } : {}), - signal: options.signal, - }) - recordCost(response.usage?.billed_cost) - return (response.data ?? []) - .filter((hit) => typeof hit?.url === 'string' && hit.url.length > 0) - .map((hit) => ({ title: hit.title ?? hit.url, url: hit.url, snippet: hit.snippet })) - } catch (error) { - return translate(error) - } finally { - acc.searchCalls += 1 - acc.wallMs += Date.now() - started - } - }, - async chat(messages, maxTokens) { - options.signal?.throwIfAborted() - const started = Date.now() - try { - const response = await client.chat({ - messages, - maxTokens: Math.max(MIN_MAX_TOKENS, maxTokens ?? MIN_MAX_TOKENS), - temperature: 0.2, - signal: options.signal, - }) - acc.chatCalls += 1 - acc.promptTokens += response.usage?.prompt_tokens ?? 0 - acc.completionTokens += response.usage?.completion_tokens ?? 0 - // Per-response cost, not a shared-client usage delta that races parallel calls. - recordCost(response.tangle?.costSource === 'receipt' ? response.tangle.costUsd : undefined) - return response.choices?.[0]?.message?.content ?? '' - } catch (error) { - return translate(error) - } finally { - acc.wallMs += Date.now() - started - } - }, - usage() { - return { ...acc } - }, - } -} - -export interface WebResearchWorkerOptions { - /** Router client. Defaults to a live Tangle router client from env creds. */ - router?: RouterClient - router_options?: TangleRouterOptions - /** Max search queries the LLM may form per gap. Default 2. */ - queriesPerGap?: number - /** Max web results fetched per query. Default 3. */ - resultsPerQuery?: number - /** Hard cap on sources proposed per round (across all gaps). Default 6. */ - maxSourcesPerRound?: number - /** Disk cache dir for `politeFetch`. Optional; speeds repeat runs. */ - cacheDir?: string - /** Minimum readable text length to keep a fetched page. Default 200. */ - minTextChars?: number - /** Max chars of page text stored per source (keeps pages bounded). Default 4000. */ - maxTextChars?: number -} - -/** Bind run cancellation to the default router without changing an injected client. */ -function resolveRouter( - opts: { - router?: RouterClient - router_options?: TangleRouterOptions - }, - signal?: AbortSignal, -): RouterClient { - if (opts.router) return opts.router - const requestSignal = combineRouterSignals(signal, opts.router_options?.signal) - return createTangleRouterClient({ ...opts.router_options, signal: requestSignal }) -} - -function combineRouterSignals( - runSignal: AbortSignal | undefined, - configuredSignal: AbortSignal | undefined, -): AbortSignal | undefined { - return runSignal && configuredSignal - ? AbortSignal.any([runSignal, configuredSignal]) - : (runSignal ?? configuredSignal) -} - -function throwIfRouterAborted( - runSignal: AbortSignal | undefined, - opts: { router?: RouterClient; router_options?: TangleRouterOptions }, -): void { - runSignal?.throwIfAborted() - if (!opts.router) opts.router_options?.signal?.throwIfAborted() -} - -/** - * The real web-research worker. Conforms to the loop's `ResearchWorker` - * contract: given the open gaps, it returns the sources it found plus a - * `buildPages` that emits citing pages from the sources the driver accepted. - */ -export function createWebResearchWorker(options: WebResearchWorkerOptions = {}): ResearchWorker { - const queriesPerGap = Math.max(1, options.queriesPerGap ?? 2) - const resultsPerQuery = Math.max(1, options.resultsPerQuery ?? 3) - const maxSourcesPerRound = Math.max(1, options.maxSourcesPerRound ?? 6) - const minTextChars = Math.max(1, options.minTextChars ?? 200) - const maxTextChars = Math.max(minTextChars, options.maxTextChars ?? 4000) - - return async (ctx: WorkerResearchContext): Promise => { - const assertActive = () => throwIfRouterAborted(ctx.signal, options) - assertActive() - const router = resolveRouter(options, ctx.signal) - const fetchSignal = options.router - ? ctx.signal - : combineRouterSignals(ctx.signal, options.router_options?.signal) - // Target the BLOCKING gaps first; fall back to all gaps if none are blocking. - const targetGaps = ctx.gaps.filter((gap) => gap.blocking) - const gaps = targetGaps.length > 0 ? targetGaps : ctx.gaps - if (gaps.length === 0) { - return { sources: [], notes: 'no open gaps to research' } - } - - const queries = await formSearchQueries(router, ctx, gaps, queriesPerGap, assertActive) - assertActive() - const proposals: ResearchSourceProposal[] = [] - const seenUris = new Set() - - for (const query of queries) { - if (proposals.length >= maxSourcesPerRound) break - assertActive() - let hits: WebSearchHit[] - try { - hits = await router.search(query, { maxResults: resultsPerQuery }) - } catch (error) { - assertActive() - // A single failed query must not sink the round — record nothing, move on. - if ((error as { name?: string } | null)?.name === 'AbortError') throw error - continue - } - assertActive() - for (const hit of hits) { - assertActive() - if (proposals.length >= maxSourcesPerRound) break - if (seenUris.has(hit.url)) continue - const fetched = await politeFetch(hit.url, { - signal: fetchSignal, - cacheDir: options.cacheDir, - }) - assertActive() - if (!fetched.verifiable) continue - const text = htmlToText(fetched.body).slice(0, maxTextChars) - if (text.length < minTextChars) continue - seenUris.add(hit.url) - proposals.push({ - uri: hit.url, - title: hit.title || hit.url, - text, - // We just fetched + verified this page, so stamp `lastVerifiedAt` with - // fetch time. Do NOT set `validUntil`: a live page has no inherent - // future expiry, and the readiness freshness check treats any - // `validUntil <= now` as EXPIRED (score 0). The page's `Last-Modified` - // is a PAST date, so writing it here would mark every real source stale - // and zero out coverage. Record it as provenance metadata instead. - lastVerifiedAt: fetched.fetchedAt, - metadata: { - discoveredVia: query, - snippet: hit.snippet ?? '', - goal: ctx.goal, - sourceUpdatedAt: fetched.sourceUpdatedAt, - }, - }) - } - } - - assertActive() - return { - sources: proposals, - buildPages: buildCitingPages(proposals), - notes: `web-research worker: ${queries.length} queries → ${proposals.length} fetched sources`, - } - } -} - -/** - * Ask the LLM to turn the open gaps into focused web search queries. Falls back - * to the gap's own readiness query if the model returns nothing parseable, so a - * model hiccup degrades to a sane search rather than an empty round. - */ -async function formSearchQueries( - router: RouterClient, - ctx: WorkerResearchContext, - gaps: KnowledgeGap[], - queriesPerGap: number, - assertActive: () => void, -): Promise { - const gapLines = gaps - .map((gap, i) => `${i + 1}. ${gap.description} (readiness query: "${gap.query}")`) - .join('\n') - const want = gaps.length * queriesPerGap - const system = - 'You are a research librarian. Turn knowledge gaps into precise web search queries that will ' + - 'surface authoritative primary sources (papers, docs, standards, official pages). ' + - 'Return ONLY a JSON array of query strings, no prose.' - const user = [ - `Research goal: ${ctx.goal}`, - ctx.steer ? `Steer from the coordinator:\n${ctx.steer}` : '', - `Open knowledge gaps:\n${gapLines}`, - `Return up to ${want} search query strings as a JSON array (e.g. ["query one","query two"]).`, - ] - .filter(Boolean) - .join('\n\n') - - let raw = '' - try { - raw = await router.chat( - [ - { role: 'system', content: system }, - { role: 'user', content: user }, - ], - MIN_MAX_TOKENS, - ) - assertActive() - } catch (error) { - assertActive() - if ((error as { name?: string } | null)?.name === 'AbortError') throw error - raw = '' - } - const parsed = parseQueryList(raw) - const fromLlm = parsed.slice(0, want) - if (fromLlm.length > 0) return dedupeStrings(fromLlm) - // Degrade to the readiness queries themselves — still a real search. - return dedupeStrings(gaps.map((gap) => gap.query || gap.description)) -} - -/** Parse a JSON array of query strings, tolerant of code fences / surrounding prose. */ -function parseQueryList(raw: string): string[] { - const text = raw.trim() - if (!text) return [] - const candidates: string[] = [] - // Prefer a fenced or bare JSON array. - const arrayMatch = text.match(/\[[\s\S]*\]/) - if (arrayMatch) { - try { - const arr = JSON.parse(arrayMatch[0]) as unknown[] - for (const item of arr) - if (typeof item === 'string' && item.trim()) candidates.push(item.trim()) - } catch { - /* fall through to line parsing */ - } - } - if (candidates.length === 0) { - for (const line of text.split('\n')) { - const cleaned = line - .replace(/^\s*(?:[-*]|\d+[.)])\s*/, '') - .replace(/^["']|["']$/g, '') - .trim() - if (cleaned && cleaned.length > 2 && !cleaned.startsWith('{')) candidates.push(cleaned) - } - } - return candidates -} - -function dedupeStrings(values: string[]): string[] { - const seen = new Set() - const out: string[] = [] - for (const value of values) { - const key = value.toLowerCase().trim() - if (key && !seen.has(key)) { - seen.add(key) - out.push(value.trim()) - } - } - return out -} - -/** - * Build the curated `knowledge/*.md` pages from the sources the driver accepted. - * Each page cites the registered source by its assigned `record.id` (matched back - * to the proposal via `metadata.originalUri`, which `addSourceText` stashes). - */ -function buildCitingPages( - proposals: ResearchSourceProposal[], -): (acceptedSources: SourceRecord[]) => string | undefined { - return (acceptedSources) => { - if (acceptedSources.length === 0) return undefined - const blocks = acceptedSources.map((record) => { - const proposal = proposals.find((p) => p.uri === record.metadata?.originalUri) - const uri = proposal?.uri ?? record.id - const slug = - uri - .replace(/^https?:\/\//, '') - .replace(/[^a-z0-9]+/gi, '-') - .replace(/^-+|-+$/g, '') - .slice(0, 120) || record.id - const title = proposal?.title ?? record.id - const body = proposal?.text ?? '' - return [ - `---FILE: knowledge/${slug}.md---`, - '---', - `title: ${escapeYaml(title)}`, - `sources: ["${record.id}"]`, - `source_url: ${uri}`, - '---', - `# ${title}`, - '', - body, - '', - `Source: ${uri}`, - '---END FILE---', - ].join('\n') - }) - return blocks.join('\n') - } -} - -function escapeYaml(value: string): string { - // Keep the frontmatter line single-valued and safe: collapse newlines, strip - // quotes that would break the scalar. - return value - .replace(/[\r\n]+/g, ' ') - .replace(/"/g, "'") - .trim() -} - -export interface VerifyingDriverOptions { - router?: RouterClient - router_options?: TangleRouterOptions - /** - * When the LLM verdict can't be parsed, default to REJECT (fail-closed) so a - * model hiccup never poisons the KB with an unverified source. Set `true` to - * accept-on-parse-failure only if you have a reason to. Default false. - */ - acceptOnParseFailure?: boolean -} - -/** - * The verifying driver: a real LLM pass that judges each candidate source's - * on-topic relevance to the goal + open gaps and whether it duplicates material - * already accepted this round. This is the differentiated coordinator role — it - * GATES the worker's additions; it adds nothing itself. - * - * The loop already dedups exact-uri duplicates and only calls this on genuinely - * new candidates, so the verifier focuses on relevance + near-duplicate - * judgement, not bookkeeping. - */ -export function createVerifyingResearchDriver( - options: VerifyingDriverOptions = {}, -): ResearchDriver { - const acceptOnParseFailure = options.acceptOnParseFailure ?? false - return { - async verifySource( - source: ResearchSourceProposal, - ctx: SourceVerificationContext, - ): Promise { - const assertActive = () => throwIfRouterAborted(ctx.signal, options) - assertActive() - const router = resolveRouter(options, ctx.signal) - const gapLines = ctx.gaps - .map((gap) => `- ${gap.description} (query: "${gap.query}")`) - .join('\n') - const acceptedTitles = ctx.acceptedThisRound - .map((accepted) => `- ${accepted.title ?? accepted.uri}`) - .join('\n') - const excerpt = source.text.slice(0, 1500) - const system = - 'You verify whether a fetched web source belongs in a curated knowledge base. ' + - 'Accept a source ONLY if it is genuinely on-topic for the research goal and helps close ' + - 'one of the open gaps, AND it is not a near-duplicate of an already-accepted source. ' + - 'Reject spam, listicles, off-topic pages, marketing, and near-duplicates. ' + - 'Respond with ONLY a JSON object: {"accept": true|false, "reason": ""}.' - const user = [ - `Research goal: ${ctx.goal}`, - `Open gaps:\n${gapLines || '(none specified)'}`, - acceptedTitles - ? `Already accepted this round:\n${acceptedTitles}` - : 'Nothing accepted yet this round.', - `Candidate source:\nURL: ${source.uri}\nTitle: ${source.title ?? '(none)'}\nExcerpt:\n${excerpt}`, - 'Verdict as JSON {"accept": boolean, "reason": string}:', - ].join('\n\n') - - let raw = '' - try { - raw = await router.chat( - [ - { role: 'system', content: system }, - { role: 'user', content: user }, - ], - MIN_MAX_TOKENS, - ) - assertActive() - } catch (error) { - assertActive() - if ((error as { name?: string } | null)?.name === 'AbortError') throw error - // Router failure: fail-closed (reject) so an unverified source can't slip in. - return acceptOnParseFailure - ? { accept: true } - : { accept: false, reason: `verifier unavailable: ${(error as Error).message}` } - } - - const verdict = parseVerdict(raw) - if (verdict) return verdict - return acceptOnParseFailure - ? { accept: true } - : { accept: false, reason: 'verifier returned an unparseable verdict' } - }, - } -} - -/** Parse the verifier's `{accept, reason}` JSON, tolerant of fences / prose. */ -function parseVerdict(raw: string): SourceVerdict | null { - const text = raw.trim() - if (!text) return null - const objMatch = text.match(/\{[\s\S]*\}/) - if (!objMatch) return null - try { - const parsed = JSON.parse(objMatch[0]) as { accept?: unknown; reason?: unknown } - if (typeof parsed.accept !== 'boolean') return null - if (parsed.accept) return { accept: true } - return { - accept: false, - reason: - typeof parsed.reason === 'string' && parsed.reason.trim() - ? parsed.reason.trim() - : 'rejected by verifier', - } - } catch { - return null - } -} diff --git a/tests/claim-grounding.test.ts b/tests/claim-grounding.test.ts index 79c759d..b1535b8 100644 --- a/tests/claim-grounding.test.ts +++ b/tests/claim-grounding.test.ts @@ -1,30 +1,5 @@ import { describe, expect, it } from 'vitest' -import { - citedClaimKey, - citedClaimOf, - createClaimGroundingVerifier, - groundClaimInText, - withCitedClaim, -} from '../src/claim-grounding' -import type { - ResearchSourceProposal, - SourceVerificationContext, -} from '../src/verified-research-loop' - -const ctx: SourceVerificationContext = { - root: '/tmp/x', - goal: 'self-speculative decoding', - round: 1, - index: { - root: '/tmp/x', - generatedAt: '', - sources: [], - pages: [], - graph: { nodes: [], edges: [] }, - }, - gaps: [], - acceptedThisRound: [], -} +import { groundClaimInText } from '../src/claim-grounding' describe('groundClaimInText (the deterministic grounding oracle)', () => { const page = @@ -92,85 +67,3 @@ describe('groundClaimInText (the deterministic grounding oracle)', () => { expect(groundClaimInText(claim, page, { minOverlap: 0.9 }).grounded).toBe(false) }) }) - -describe('citedClaim helpers', () => { - const base: ResearchSourceProposal = { uri: 'https://x', text: 't', title: 'T' } - - it('round-trips a claim through metadata', () => { - const decorated = withCitedClaim(base, 'the claim') - expect(decorated.metadata?.[citedClaimKey]).toBe('the claim') - expect(citedClaimOf(decorated)).toBe('the claim') - }) - - it('returns undefined for a missing/blank claim', () => { - expect(citedClaimOf(base)).toBeUndefined() - expect(citedClaimOf(withCitedClaim(base, ' '))).toBeUndefined() - }) -}) - -describe('createClaimGroundingVerifier (the driver gate)', () => { - const page = - 'The transformer architecture uses multi-head self-attention. Reported BLEU of 28.4 on WMT14 En-De.' - - it('accepts a grounded source', async () => { - const verify = createClaimGroundingVerifier() - const source = withCitedClaim( - { uri: 'https://a', text: page, title: 'Attention' }, - 'BLEU of 28.4 on WMT14', - ) - expect(await verify(source, ctx)).toEqual({ accept: true }) - }) - - it('REJECTS a misattributed source with a precise reason', async () => { - const verify = createClaimGroundingVerifier() - const source = withCitedClaim( - { uri: 'https://a', text: page, title: 'Attention' }, - 'reports a BLEU of 41.0 on the WMT16 Russian benchmark', - ) - const verdict = await verify(source, ctx) - expect(verdict.accept).toBe(false) - if (!verdict.accept) expect(verdict.reason).toMatch(/misattributed citation/) - }) - - it('rejects an un-annotated source by default (fail-closed)', async () => { - const verify = createClaimGroundingVerifier() - const verdict = await verify({ uri: 'https://a', text: page, title: 'T' }, ctx) - expect(verdict.accept).toBe(false) - if (!verdict.accept) expect(verdict.reason).toMatch(/no cited claim/) - }) - - it('composes a relevance verifier AFTER grounding passes', async () => { - let relevanceCalled = false - const verify = createClaimGroundingVerifier({ - relevanceVerifier: () => { - relevanceCalled = true - return { accept: false, reason: 'off-topic per relevance judge' } - }, - }) - const grounded = withCitedClaim( - { uri: 'https://a', text: page, title: 'T' }, - 'multi-head self-attention', - ) - const verdict = await verify(grounded, ctx) - expect(relevanceCalled).toBe(true) - expect(verdict.accept).toBe(false) - if (!verdict.accept) expect(verdict.reason).toMatch(/off-topic/) - }) - - it('does NOT call the relevance verifier when grounding already fails', async () => { - let relevanceCalled = false - const verify = createClaimGroundingVerifier({ - relevanceVerifier: () => { - relevanceCalled = true - return { accept: true } - }, - }) - const misattributed = withCitedClaim( - { uri: 'https://a', text: page, title: 'T' }, - 'a 99x speedup on a quantum coprocessor', - ) - const verdict = await verify(misattributed, ctx) - expect(relevanceCalled).toBe(false) - expect(verdict.accept).toBe(false) - }) -}) diff --git a/tests/claim-persistence.test.ts b/tests/claim-persistence.test.ts index 92b8bc6..3d0adfd 100644 --- a/tests/claim-persistence.test.ts +++ b/tests/claim-persistence.test.ts @@ -6,24 +6,16 @@ import type { ResearchClaimEvidence, ResearchClaimLedger, ResearchClaimRecord, - ResearchSourceProposal, ResearchSourceVersion, - SourceRecord, - SourceVerificationContext, } from '../src/index' import { - addSourcePath, - buildKnowledgeIndex, ClaimLedgerGoalConflictError, ClaimLedgerMigrationRequiredError, claimEvidenceId, claimId, createKnowledgeEvent, - createPersistentResearchDrivingDriver, - createResearchDrivingDriver, DeepQuestionSchema, deepQuestionId, - defineReadinessSpec, FileSystemKbStore, initKnowledgeBase, KB_CLAIM_LEDGER_DIR, @@ -34,7 +26,6 @@ import { MemoryKbStore, mergeClaimLedgers, ResearchClaimLedgerSchema, - runVerifiedResearchLoop, sha256, textSourceId, withSafeDescendant, @@ -42,7 +33,6 @@ import { writeJsonDurableWithinRoot, writeKnowledgeIndex, } from '../src/index' -import type { RouterClient } from '../src/web-research-worker' const GOAL = 'self-speculative decoding' @@ -55,628 +45,17 @@ async function withRoot(fn: (root: string) => Promise): Promise { } } -/** A RouterClient whose `chat` returns scripted claim-extraction JSON by token. */ -function stubRouter(repliesByToken: Record): RouterClient { - return { - search: async () => [], - chat: async (messages) => { - const user = messages.find((message) => message.role === 'user')?.content ?? '' - for (const [token, reply] of Object.entries(repliesByToken)) { - if (user.includes(token)) return reply - } - return '[]' - }, - usage: () => ({ - chatCalls: 0, - searchCalls: 0, - promptTokens: 0, - completionTokens: 0, - usd: 0, - wallMs: 0, - }), - } -} - -function ctx(round: number, goal = GOAL): SourceVerificationContext { - return { - root: '/tmp/x', - goal, - round, - index: { - root: '/tmp/x', - generatedAt: '', - sources: [], - pages: [], - graph: { nodes: [], edges: [] }, - }, - gaps: [], - acceptedThisRound: [], - } -} - -function source(uri: string, text: string): ResearchSourceProposal { - return { uri, text, title: uri } -} - -function registeredSource(uri: string, text: string): SourceRecord { - const contentHash = sha256(text) - return { - id: textSourceId(uri, contentHash), - uri: `raw/sources/${contentHash}.txt`, - contentHash, - text, - metadata: { originalUri: uri }, - createdAt: '2026-07-28T00:00:00.000Z', - } -} - -const CLAIM_A = '[{"claim":"layer skipping gives a 1.73x speedup","contradicts":null}]' - // =========================================================================== // Claims must survive the process that discovered them. // =========================================================================== describe('research claim ledger — persistence', () => { - it('restores corroboration counts and open questions in a NEW driver instance', async () => { - const store = new MemoryKbStore() - const router = stubRouter({ 'PAGE-A': CLAIM_A, 'PAGE-B': CLAIM_A }) - - const first = await createPersistentResearchDrivingDriver({ router, store, ledgerId: 'run-1' }) - await first.verifySource(source('https://arxiv.org/a', 'PAGE-A body'), ctx(1)) - await first.commitSources([registeredSource('https://arxiv.org/a', 'PAGE-A body')]) - await first.prepareFold() - first.foldGaps?.([]) - await first.checkpoint() - - const before = first.researchState() - expect(before.claims).toHaveLength(1) - expect(before.claims[0]?.supportingHosts).toEqual(new Set(['arxiv.org'])) - expect(before.openQuestions.length).toBeGreaterThan(0) - - // The process dies here. A NEW driver over the same store is the resume. - const resumed = await createPersistentResearchDrivingDriver({ - router, - store, - ledgerId: 'run-1', - }) - const restored = resumed.researchState() - expect(restored.claims).toHaveLength(1) - expect(restored.claims[0]?.supportingHosts).toEqual(new Set(['arxiv.org'])) - expect(restored.weaklySupported).toHaveLength(1) - expect(restored.questions.map((question) => question.id).sort()).toEqual( - before.questions.map((question) => question.id).sort(), - ) - expect(restored.rounds).toBe(1) - - // And the resumed run keeps ACCUMULATING onto the restored ledger rather - // than starting a second, parallel belief state. - await resumed.verifySource(source('https://acm.org/b', 'PAGE-B body'), ctx(2)) - await resumed.commitSources([registeredSource('https://acm.org/b', 'PAGE-B body')]) - const grown = resumed.researchState() - expect(grown.claims).toHaveLength(1) - expect([...(grown.claims[0]?.supportingHosts ?? [])].sort()).toEqual(['acm.org', 'arxiv.org']) - expect(grown.corroborated).toHaveLength(1) - }) - - it('keeps accepted evidence pending until its source registration is confirmed', async () => { - const store = new MemoryKbStore() - const router = stubRouter({ 'PAGE-A': CLAIM_A, 'PAGE-B': CLAIM_A }) - const driver = await createPersistentResearchDrivingDriver({ - router, - store, - ledgerId: 'mid-round', - }) - - // Verification persists the extraction first, but acceptance is not proof - // that the separate source-registry write completed. - const verdict = await driver.verifySource(source('https://arxiv.org/a', 'PAGE-A body'), ctx(1)) - expect(verdict.accept).toBe(true) - - const pending = await store.getClaimLedger('mid-round') - expect(pending?.claimEvidence).toHaveLength(1) - expect(pending?.registeredSources).toEqual([]) - expect(pending?.claims).toEqual([]) - expect(driver.isComplete()).toBe(false) - - await driver.commitSources([registeredSource('https://arxiv.org/a', 'PAGE-A body')]) - const afterFirst = await store.getClaimLedger('mid-round') - expect(afterFirst?.claims[0]?.supportingHosts).toEqual(['arxiv.org']) - - await driver.verifySource(source('https://acm.org/b', 'PAGE-B body'), ctx(1)) - const secondPending = await store.getClaimLedger('mid-round') - expect(secondPending?.registeredSources).toEqual([ - sourceVersion('https://arxiv.org/a', 'PAGE-A body'), - ]) - expect(secondPending?.claims[0]?.supportingHosts).toEqual(['arxiv.org']) - - await driver.commitSources([registeredSource('https://acm.org/b', 'PAGE-B body')]) - const afterSecond = await store.getClaimLedger('mid-round') - expect([...(afterSecond?.claims[0]?.supportingHosts ?? [])].sort()).toEqual([ - 'acm.org', - 'arxiv.org', - ]) - - // The crash lands here, between sources and before the round ends. - const resumed = await createPersistentResearchDrivingDriver({ - router, - store, - ledgerId: 'mid-round', - }) - expect(resumed.researchState().corroborated).toHaveLength(1) - }) - - it('binds same-URI evidence to the exact source bytes that produced it', async () => { - const uri = 'https://example.org/result' - const textA = 'PAGE-A exact bytes' - const textB = 'PAGE-B changed bytes' - const store = new MemoryKbStore() - const driver = await createPersistentResearchDrivingDriver({ - store, - ledgerId: 'same-uri-versions', - router: stubRouter({ - 'PAGE-A': '[{"claim":"claim extracted from bytes A","contradicts":null}]', - 'PAGE-B': '[{"claim":"claim extracted from bytes B","contradicts":null}]', - }), - }) - - await driver.verifySource(source(uri, textA), ctx(1)) - await driver.verifySource(source(uri, textB), ctx(1)) - await driver.commitSources([registeredSource(uri, textA)]) - - expect(driver.researchState().claims.map((claim) => claim.text)).toEqual([ - 'claim extracted from bytes A', - ]) - const afterA = await store.getClaimLedger('same-uri-versions') - expect(afterA?.claimEvidence).toHaveLength(2) - expect(afterA?.registeredSources).toEqual([sourceVersion(uri, textA)]) - const reopened = await createPersistentResearchDrivingDriver({ - store, - ledgerId: 'same-uri-versions', - router: stubRouter({}), - }) - expect(reopened.researchState().claims.map((claim) => claim.text)).toEqual([ - 'claim extracted from bytes A', - ]) - - await reopened.commitSources([registeredSource(uri, textB)]) - expect( - reopened - .researchState() - .claims.map((claim) => claim.text) - .sort(), - ).toEqual(['claim extracted from bytes A', 'claim extracted from bytes B']) - }) - - it('merges concurrent writers that observe different versions of one URI', async () => { - const uri = 'https://example.org/live-result' - const store = new MemoryKbStore() - const router = stubRouter({ - VERSION_A: '[{"claim":"concurrent claim A","contradicts":null}]', - VERSION_B: '[{"claim":"concurrent claim B","contradicts":null}]', - }) - const [writerA, writerB] = await Promise.all([ - createPersistentResearchDrivingDriver({ store, ledgerId: 'concurrent-versions', router }), - createPersistentResearchDrivingDriver({ store, ledgerId: 'concurrent-versions', router }), - ]) - - await Promise.all([ - writerA.verifySource(source(uri, 'VERSION_A bytes'), ctx(1)), - writerB.verifySource(source(uri, 'VERSION_B bytes'), ctx(1)), - ]) - await Promise.all([ - writerA.commitSources([registeredSource(uri, 'VERSION_A bytes')]), - writerB.commitSources([registeredSource(uri, 'VERSION_B bytes')]), - ]) - - const reopened = await createPersistentResearchDrivingDriver({ - store, - ledgerId: 'concurrent-versions', - router, - }) - expect( - reopened - .researchState() - .claims.map((claim) => claim.text) - .sort(), - ).toEqual(['concurrent claim A', 'concurrent claim B']) - expect(reopened.toLedger().registeredSources).toHaveLength(2) - }) - - it('snapshots source identity before awaiting claim extraction', async () => { - let releaseExtraction: (() => void) | undefined - const extractionStarted = new Promise((resolve) => { - releaseExtraction = resolve - }) - let allowExtraction: (() => void) | undefined - const extractionBlocked = new Promise((resolve) => { - allowExtraction = resolve - }) - const driver = await createPersistentResearchDrivingDriver({ - store: new MemoryKbStore(), - ledgerId: 'immutable-intake', - router: { - ...stubRouter({}), - chat: async () => { - releaseExtraction?.() - await extractionBlocked - return '[{"claim":"claim from immutable bytes A","contradicts":null}]' - }, - }, - }) - const mutable = source('https://a.org/result', 'immutable PAGE-A bytes') - const verification = driver.verifySource(mutable, ctx(3)) - await extractionStarted - mutable.uri = 'https://b.org/result' - mutable.text = 'mutated PAGE-B bytes' - allowExtraction?.() - await verification - - await driver.commitSources([registeredSource('https://a.org/result', 'immutable PAGE-A bytes')]) - const ledger = driver.toLedger() - expect(ledger.claims.map((claim) => claim.text)).toEqual(['claim from immutable bytes A']) - expect(ledger.claimEvidence[0]).toMatchObject({ - sourceUri: 'https://a.org/result', - sourceContentHash: sha256('immutable PAGE-A bytes'), - }) - }) - - it('validates a source-confirmation batch before mutating live state', async () => { - const driver = await createPersistentResearchDrivingDriver({ - store: new MemoryKbStore(), - ledgerId: 'atomic-confirmation', - router: stubRouter({ PAGE: CLAIM_A }), - }) - await driver.verifySource(source('https://a.org/result', 'PAGE bytes'), ctx(1)) - const invalid = { - ...registeredSource('https://b.org/result', 'other bytes'), - id: 'src_forged', - } - - await expect( - driver.commitSources([registeredSource('https://a.org/result', 'PAGE bytes'), invalid]), - ).rejects.toThrow(/does not match URI-and-content identity/) - expect(driver.toLedger().registeredSources).toEqual([]) - expect(driver.researchState().claims).toEqual([]) - }) - - it('recovers when sources register but the evidence confirmation crashes', async () => { - await withRoot(async (root) => { - const store = new FileSystemKbStore({ root }) - const router = stubRouter({ BODY: CLAIM_A }) - const driver = await createPersistentResearchDrivingDriver({ - router, - store, - ledgerId: 'source-confirmation-crash', - }) - const interrupted = { - ...driver, - commitSources: async (sources: readonly SourceRecord[]) => { - if (sources.length > 0) { - throw new Error('simulated crash after source registration') - } - await driver.commitSources(sources) - }, - } - const readinessSpecs = [ - defineReadinessSpec({ - id: 'two-results', - description: 'two independent results', - query: 'BODY', - requiredFor: ['ResearchAgent'], - importance: 'blocking', - minSources: 2, - minHits: 1, - }), - ] - - await expect( - runVerifiedResearchLoop({ - root, - goal: GOAL, - maxRounds: 1, - readinessSpecs, - worker: async () => ({ - sources: [ - source('https://a.org/result', 'BODY one'), - source('https://b.org/result', 'BODY two'), - ], - }), - driver: interrupted, - }), - ).rejects.toThrow(/simulated crash after source registration/) - - // Exact failed state: both source writes completed, while both claim - // observations remain pending and therefore cannot report completion. - const registeredIndex = await buildKnowledgeIndex(root) - expect(registeredIndex.sources).toHaveLength(2) - const pending = await store.getClaimLedger('source-confirmation-crash') - expect(pending?.claimEvidence).toHaveLength(2) - expect(pending?.registeredSources).toEqual([]) - expect(pending?.claims).toEqual([]) - const afterCrash = await createPersistentResearchDrivingDriver({ - router, - store, - ledgerId: 'source-confirmation-crash', - }) - expect(afterCrash.isComplete()).toBe(false) - await expect(store.listEvents({ type: 'research.iteration' })).resolves.toHaveLength(0) - - // A normal knowledge base may also contain path-imported records, which - // have no originalUri and are unrelated to text-source reconciliation. - const pathSource = join(root, 'offline-source.txt') - await writeFile(pathSource, 'offline corpus bytes') - await addSourcePath(root, pathSource) - - // A fresh loop reconciles exact original URIs from the source registry - // before it decides readiness or launches another worker round. - const resumed = await createPersistentResearchDrivingDriver({ - router, - store, - ledgerId: 'source-confirmation-crash', - }) - let completeBeforeWorker = false - await expect( - runVerifiedResearchLoop({ - root, - goal: GOAL, - maxRounds: 1, - readinessSpecs, - worker: async () => { - completeBeforeWorker = resumed.isComplete() - throw new Error('stop after restart reconciliation') - }, - driver: resumed, - }), - ).rejects.toThrow(/stop after restart reconciliation/) - expect(completeBeforeWorker).toBe(true) - expect(resumed.isComplete()).toBe(true) - const recovered = await store.getClaimLedger('source-confirmation-crash') - expect(recovered?.registeredSources.map((source) => source.uri).sort()).toEqual([ - 'https://a.org/result', - 'https://b.org/result', - ]) - expect(recovered?.claims[0]?.supportingHosts).toEqual(['a.org', 'b.org']) - }) - }) - - it('survives JSON round-tripping — the failure that made the old ledger unstorable', async () => { - const router = stubRouter({ - SEED: '[{"claim":"the speedup is 5x","contradicts":null}]', - REFUTE: '[{"claim":"the speedup is only 2x","contradicts":"[__ID__]"}]', - }) - const driver = createResearchDrivingDriver({ - router: { - ...router, - chat: async (messages) => { - const user = messages.find((message) => message.role === 'user')?.content ?? '' - if (user.includes('SEED')) return '[{"claim":"the speedup is 5x","contradicts":null}]' - if (user.includes('REFUTE')) { - const id = user.match(/\[(c_[0-9a-f]+)\]/)?.[1] ?? '' - return `[{"claim":"the speedup is only 2x","contradicts":"[${id}]"}]` - } - return '[]' - }, - }, - }) - await driver.verifySource(source('https://a.org/x', 'SEED'), ctx(1)) - await driver.verifySource(source('https://b.org/y', 'REFUTE'), ctx(1)) - - const live = driver.researchState() - expect(live.contested).toHaveLength(2) - expect(live.claims[0]?.supportingHosts).toBeInstanceOf(Set) - - // The published live API retains Sets; the durable ledger converts them to - // arrays at the persistence boundary so JSON cannot erase their contents. - const durable = driver.toLedger() - const roundTripped = JSON.parse(JSON.stringify(durable)) as typeof durable - expect(roundTripped.claims[0]?.supportingHosts).toEqual(durable.claims[0]?.supportingHosts) - expect(roundTripped.claims[0]?.supportingHosts.length).toBe(1) - expect(roundTripped.claims[1]?.contradicts).toEqual(durable.claims[1]?.contradicts) - expect(roundTripped.claims[1]?.contradicts.length).toBe(1) - for (const claim of roundTripped.claims) { - expect(Array.isArray(claim.supportingHosts)).toBe(true) - expect(Array.isArray(claim.contradicts)).toBe(true) - } - }) - - it('does not report a resumed run complete while its questions are still open', async () => { - const store = new MemoryKbStore() - const router = stubRouter({ 'PAGE-A': CLAIM_A, 'PAGE-B': CLAIM_A }) - - const first = await createPersistentResearchDrivingDriver({ router, store, ledgerId: 'run-2' }) - await first.verifySource(source('https://arxiv.org/a', 'PAGE-A body'), ctx(1)) - await first.commitSources([registeredSource('https://arxiv.org/a', 'PAGE-A body')]) - // The questions this raises exist ONLY in the synchronous fold, so without a - // checkpoint they would die here and the resumed run would call itself done. - await first.prepareFold() - first.foldGaps?.([]) - await first.checkpoint() - expect(first.isComplete()).toBe(false) - - const resumed = await createPersistentResearchDrivingDriver({ - router, - store, - ledgerId: 'run-2', - }) - expect(resumed.researchState().openQuestions.length).toBeGreaterThan(0) - expect(resumed.isComplete()).toBe(false) - - // Corroborating the claim settles the CLAIM half, and completion still - // refuses while a question raised before the crash remains unanswered. - await resumed.verifySource(source('https://acm.org/b', 'PAGE-B body'), ctx(2)) - await resumed.commitSources([registeredSource('https://acm.org/b', 'PAGE-B body')]) - const settled = resumed.researchState() - expect(settled.corroborated).toHaveLength(1) - expect(settled.weaklySupported).toHaveLength(0) - expect(settled.openQuestions.length).toBeGreaterThan(0) - expect(resumed.isComplete()).toBe(false) - - // Why that matters, stated as its own violation: a ledger that kept the - // claims but LOST the questions — which is what a driver without - // `checkpoint` writes — reports the very same run complete. - const lossy = (await store.getClaimLedger('run-2'))! - await store.putClaimLedger({ - ...lossy, - claims: resumed.toLedger().claims, - questions: [], - }) - const lied = await createPersistentResearchDrivingDriver({ router, store, ledgerId: 'run-2' }) - expect(lied.isComplete()).toBe(true) - }) - - it('recovers a fold interrupted before checkpoint without publishing its event', async () => { - await withRoot(async (root) => { - const store = new FileSystemKbStore({ root }) - const router = stubRouter({ - BODY: '[{"claim":"the method is 2x faster","contradicts":null}]', - }) - const driver = await createPersistentResearchDrivingDriver({ - router, - store, - ledgerId: 'crashed-fold', - }) - const interrupted = { - ...driver, - checkpoint: async () => { - throw new Error('simulated crash before checkpoint') - }, - } - - await expect( - runVerifiedResearchLoop({ - root, - goal: GOAL, - maxRounds: 1, - readinessSpecs: [ - defineReadinessSpec({ - id: 'still-open', - description: 'a result the sources do not close', - query: 'never-present', - requiredFor: ['ResearchAgent'], - importance: 'blocking', - minSources: 3, - minHits: 1, - }), - ], - worker: async () => ({ - sources: [ - source('https://a.org/result', 'BODY one'), - source('https://b.org/result', 'BODY two'), - ], - }), - driver: interrupted, - }), - ).rejects.toThrow(/simulated crash/) - - const pending = await store.getClaimLedger('crashed-fold') - expect(pending?.rounds).toBe(0) - expect(pending?.preparedRounds).toBe(1) - await expect(store.listEvents({ type: 'research.iteration' })).resolves.toHaveLength(0) - - const resumed = await createPersistentResearchDrivingDriver({ - router, - store, - ledgerId: 'crashed-fold', - }) - expect(resumed.researchState().corroborated).toHaveLength(1) - expect(resumed.researchState().openQuestions.length).toBeGreaterThan(0) - expect(resumed.isComplete()).toBe(false) - const recovered = await store.getClaimLedger('crashed-fold') - expect(recovered?.rounds).toBe(1) - expect(recovered?.preparedRounds).toBeUndefined() - expect(recovered?.questions.length).toBeGreaterThan(0) - }) - }) - - it('refuses to merge two research goals into one ledger', async () => { - const store = new MemoryKbStore() - const router = stubRouter({ 'PAGE-A': CLAIM_A }) - const first = await createPersistentResearchDrivingDriver({ router, store, ledgerId: 'run-3' }) - await first.verifySource(source('https://arxiv.org/a', 'PAGE-A body'), ctx(1)) - await first.checkpoint() - - const resumed = await createPersistentResearchDrivingDriver({ - router, - store, - ledgerId: 'run-3', - }) - await expect( - resumed.verifySource(source('https://acm.org/b', 'PAGE-A body'), ctx(1, 'a different goal')), - ).rejects.toThrow(/cannot be reused/) - }) - it('rejects a ledger id that would escape its directory', async () => { const store = new MemoryKbStore() - const router = stubRouter({}) for (const ledgerId of ['../escape', 'nested/id', '..', '.', '', 'a\0b']) { - await expect( - createPersistentResearchDrivingDriver({ router, store, ledgerId }), - ).rejects.toThrow(/claim ledger id/) await expect(store.getClaimLedger(ledgerId)).rejects.toThrow(/claim ledger id/) } }) - - it('keeps two runs against one knowledge base from overwriting each other', async () => { - await withRoot(async (root) => { - await initKnowledgeBase(root) - const store = new FileSystemKbStore({ root }) - const router = stubRouter({ - 'PAGE-A': CLAIM_A, - 'PAGE-C': '[{"claim":"a different claim about caches","contradicts":null}]', - }) - - const runA = await createPersistentResearchDrivingDriver({ router, store, ledgerId: 'run-a' }) - await runA.verifySource(source('https://arxiv.org/a', 'PAGE-A body'), ctx(1, 'goal A')) - await runA.commitSources([registeredSource('https://arxiv.org/a', 'PAGE-A body')]) - const runB = await createPersistentResearchDrivingDriver({ router, store, ledgerId: 'run-b' }) - await runB.verifySource(source('https://acm.org/c', 'PAGE-C body'), ctx(1, 'goal B')) - await runB.commitSources([registeredSource('https://acm.org/c', 'PAGE-C body')]) - - const ledgers = await store.listClaimLedgers() - expect(ledgers.map((ledger) => ledger.id)).toEqual(['run-a', 'run-b']) - expect(ledgers[0]?.goal).toBe('goal A') - expect(ledgers[1]?.goal).toBe('goal B') - expect(ledgers[0]?.claims[0]?.text).not.toBe(ledgers[1]?.claims[0]?.text) - - // On disk, under the one store directory, one file per run. - const files = await readdir(join(root, KB_CLAIM_LEDGER_DIR)) - expect(files.sort()).toEqual(['run-a.json', 'run-b.json']) - }) - }) - - it('persists a claim ledger to disk that a fresh store instance reads back', async () => { - await withRoot(async (root) => { - await initKnowledgeBase(root) - const router = stubRouter({ 'PAGE-A': CLAIM_A }) - const driver = await createPersistentResearchDrivingDriver({ - router, - store: new FileSystemKbStore({ root }), - ledgerId: 'run-disk', - }) - await driver.verifySource(source('https://arxiv.org/a', 'PAGE-A body'), ctx(1)) - await driver.commitSources([registeredSource('https://arxiv.org/a', 'PAGE-A body')]) - await driver.prepareFold() - driver.foldGaps?.([]) - await driver.checkpoint() - - // A different store object over the same root — the durable read path. - const reader = new FileSystemKbStore({ root }) - const ledger = await reader.getClaimLedger('run-disk') - expect(ledger?.claims).toHaveLength(1) - expect(ledger?.claims[0]?.supportingHosts).toEqual(['arxiv.org']) - expect(ledger?.questions.length).toBeGreaterThan(0) - expect(ledger?.goal).toBe(GOAL) - expect(await reader.getClaimLedger('never-written')).toBeNull() - }) - }) - - it('checkpoint on a store-less driver is a no-op rather than a silent write', async () => { - const driver = createResearchDrivingDriver({ router: stubRouter({ 'PAGE-A': CLAIM_A }) }) - await driver.verifySource(source('https://arxiv.org/a', 'PAGE-A body'), ctx(1)) - await expect(driver.checkpoint()).resolves.toBeUndefined() - expect(driver.toLedger().id).toBe('in-memory') - expect(driver.toLedger().claims).toHaveLength(1) - }) }) // =========================================================================== @@ -716,76 +95,6 @@ describe('knowledge store — one writer, one location', () => { expect(stored.map((event) => event.type).sort()).toEqual([...KNOWLEDGE_EVENT_TYPES].sort()) }) }) - - it('records the research loop’s round events instead of discarding them', async () => { - await withRoot(async (root) => { - const result = await runVerifiedResearchLoop({ - root, - goal: GOAL, - maxRounds: 2, - actor: 'test', - worker: async ({ round }) => ({ - sources: [source(`https://arxiv.org/r${round}`, `body for round ${round}`)], - }), - driver: { verifySource: () => ({ accept: true }) }, - }) - expect(result.rounds).toBe(2) - - const stored = await new FileSystemKbStore({ root }).listEvents({ - type: 'research.iteration', - }) - expect(stored).toHaveLength(2) - expect(stored.map((event) => event.metadata?.round)).toEqual([1, 2]) - expect(stored.every((event) => event.actor === 'test')).toBe(true) - }) - }) - - it('checkpoints the driver through a real loop so the run resumes from disk', async () => { - await withRoot(async (root) => { - const store = new FileSystemKbStore({ root }) - const router = stubRouter({ 'body for round': CLAIM_A }) - const driver = await createPersistentResearchDrivingDriver({ - router, - store, - ledgerId: 'loop-run', - }) - - // A readiness spec the single source cannot satisfy keeps the loop - // not-ready, which is what makes it fold steer — the driver's synchronous - // question-raising hook, whose output only reaches disk via `checkpoint`. - await runVerifiedResearchLoop({ - root, - goal: GOAL, - maxRounds: 1, - readinessSpecs: [ - defineReadinessSpec({ - id: 'topic/definition', - description: 'what the method is and how it works', - query: 'body for round', - requiredFor: ['ResearchAgent'], - importance: 'blocking', - minSources: 2, - minHits: 1, - }), - ], - worker: async ({ round }) => ({ - sources: [source(`https://arxiv.org/r${round}`, `body for round ${round}`)], - }), - driver, - }) - - const resumed = await createPersistentResearchDrivingDriver({ - router, - store, - ledgerId: 'loop-run', - }) - const state = resumed.researchState() - expect(state.claims).toHaveLength(1) - expect(state.claims[0]?.supportingHosts).toEqual(new Set(['arxiv.org'])) - expect(state.rounds).toBe(1) - expect(state.openQuestions.length).toBeGreaterThan(0) - }) - }) }) // =========================================================================== @@ -1014,38 +323,6 @@ describe('claim ledger — concurrent accumulation', () => { }) }) - it('two persistent drivers on one ledger see each other’s corroboration', async () => { - await withRoot(async (root) => { - await initKnowledgeBase(root) - const router = stubRouter({ 'PAGE-A': CLAIM_A, 'PAGE-B': CLAIM_A }) - - const workerThree = await createPersistentResearchDrivingDriver({ - router, - store: new FileSystemKbStore({ root }), - ledgerId: 'pursuit', - }) - const workerForty = await createPersistentResearchDrivingDriver({ - router, - store: new FileSystemKbStore({ root }), - ledgerId: 'pursuit', - }) - - await workerThree.verifySource(source('https://arxiv.org/a', 'PAGE-A body'), ctx(1)) - await workerThree.commitSources([registeredSource('https://arxiv.org/a', 'PAGE-A body')]) - await workerForty.verifySource(source('https://acm.org/b', 'PAGE-B body'), ctx(1)) - await workerForty.commitSources([registeredSource('https://acm.org/b', 'PAGE-B body')]) - - // Worker 40 wrote second and read worker 3's evidence back: one claim, - // two independent hosts, corroborated. Under a whole-record write worker - // 40 would report one host and the claim would still be weak. - const seen = workerForty.researchState() - expect(seen.claims).toHaveLength(1) - expect([...(seen.claims[0]?.supportingHosts ?? [])].sort()).toEqual(['acm.org', 'arxiv.org']) - expect(seen.corroborated).toHaveLength(1) - expect(workerForty.isComplete()).toBe(true) - }) - }) - it('refuses a merge that returns a ledger under a different id', async () => { const store = new MemoryKbStore() await expect( diff --git a/tests/eval/investment-calibration.test.ts b/tests/eval/investment-calibration.test.ts deleted file mode 100644 index c49060f..0000000 --- a/tests/eval/investment-calibration.test.ts +++ /dev/null @@ -1,103 +0,0 @@ -import { describe, expect, it } from 'vitest' -import { materialFactsSurfacedInText } from '../../src/material-facts-metric' -import { calibrationTheses, caseForTicker } from './investment-calibration' -import { investmentThesisSet } from './investment-thesis-set' - -// =========================================================================== -// THE CALIBRATION GATE — run this BEFORE any A/B with `materialFactsSurfaced`. -// -// The metric is only valid if it DISCRIMINATES research depth: a shallow, -// one-paragraph ticker summary must score LOW, and a filings-grounded deep thesis -// must score HIGH, for the SAME company. If it does not separate them, the metric -// is measuring word-collection, not research (the exact failure the ML exam had), -// and the A/B would be meaningless. This file is that gate, run at $0 offline. -// -// Bars (from the task spec): shallow < 30%, deep > 70%, per company AND in aggregate. -// =========================================================================== - -const SHALLOW_MAX = 0.3 -const DEEP_MIN = 0.7 - -describe('materialFactsSurfaced — CALIBRATION GATE (discriminates shallow vs deep)', () => { - it('every calibration ticker has a held-out checklist case', () => { - for (const t of calibrationTheses) { - expect(() => caseForTicker(t.ticker)).not.toThrow() - } - // And every checklist company is calibrated (no silent gaps). - for (const company of investmentThesisSet) { - expect(calibrationTheses.some((t) => t.ticker === company.ticker)).toBe(true) - } - }) - - it('SHALLOW theses score LOW (< 30%) — the metric does not reward collection', () => { - for (const thesis of calibrationTheses) { - const company = caseForTicker(thesis.ticker) - const r = materialFactsSurfacedInText(company, thesis.shallow) - expect( - r.fraction, - `${thesis.ticker} shallow surfaced ${r.surfaced}/${r.total} = ${(r.fraction * 100).toFixed(0)}% (expected < ${SHALLOW_MAX * 100}%) — facts hit: ${r.perFact - .filter((f) => f.surfaced) - .map((f) => f.id) - .join(', ')}`, - ).toBeLessThan(SHALLOW_MAX) - } - }) - - it('DEEP theses score HIGH (> 70%) — the metric credits real, surfaced depth', () => { - for (const thesis of calibrationTheses) { - const company = caseForTicker(thesis.ticker) - const r = materialFactsSurfacedInText(company, thesis.deep) - expect( - r.fraction, - `${thesis.ticker} deep surfaced ${r.surfaced}/${r.total} = ${(r.fraction * 100).toFixed(0)}% (expected > ${DEEP_MIN * 100}%) — facts MISSED: ${r.perFact - .filter((f) => !f.surfaced) - .map((f) => `${f.id}(${f.groupsFound}/${f.groupsTotal})`) - .join(', ')}`, - ).toBeGreaterThan(DEEP_MIN) - } - }) - - it('the gap (deep - shallow) is large per company AND in aggregate', () => { - let shallowSurfaced = 0 - let deepSurfaced = 0 - let total = 0 - for (const thesis of calibrationTheses) { - const company = caseForTicker(thesis.ticker) - const s = materialFactsSurfacedInText(company, thesis.shallow) - const d = materialFactsSurfacedInText(company, thesis.deep) - // Per company the deep thesis must clear the shallow one by a wide margin. - expect( - d.fraction - s.fraction, - `${thesis.ticker}: deep ${(d.fraction * 100).toFixed(0)}% vs shallow ${(s.fraction * 100).toFixed(0)}%`, - ).toBeGreaterThan(0.4) - shallowSurfaced += s.surfaced - deepSurfaced += d.surfaced - total += s.total - } - // Aggregate: across all 27 held-out facts the meter must clearly separate. - expect(shallowSurfaced / total).toBeLessThan(SHALLOW_MAX) - expect(deepSurfaced / total).toBeGreaterThan(DEEP_MIN) - }) - - // ANTI-CIRCULARITY GUARD: the deep theses must EARN their score with real, - // independently-phrased analysis — not by verbatim-embedding the checklist's - // `evidence` strings (that would be teaching-to-the-test, making the deep score - // an answer-key echo rather than the meter catching depth). We assert no deep - // thesis contains any checklist `evidence` string verbatim. - it('deep theses do not verbatim-embed the checklist evidence (no answer-key leak)', () => { - for (const thesis of calibrationTheses) { - const company = caseForTicker(thesis.ticker) - const deepLower = thesis.deep.toLowerCase() - for (const fact of company.facts) { - // The `evidence` field is the literal curation note (quotes the filing + - // the curator's framing). A faithful deep thesis names the same numbers in - // its own prose, so the full evidence STRING must not appear verbatim. - const ev = fact.evidence.toLowerCase() - expect( - deepLower.includes(ev), - `${thesis.ticker} deep thesis verbatim-embeds the evidence string for ${fact.id} — that is an answer-key leak, rewrite in independent prose`, - ).toBe(false) - } - } - }) -}) diff --git a/tests/eval/investment-calibration.ts b/tests/eval/investment-calibration.ts deleted file mode 100644 index 4b5af8b..0000000 --- a/tests/eval/investment-calibration.ts +++ /dev/null @@ -1,86 +0,0 @@ -/** - * CALIBRATION FIXTURES for the `materialFactsSurfaced` metric. - * - * Before running ANY A/B with this metric, we must prove the metric DISCRIMINATES - * research depth — that it measures "did the thesis surface the buried, material - * drivers" and NOT "did it collect a lot of words". The ML deep-question exam had - * exactly this risk (a metric that rewards collection, not research); the task - * spec demands we rule it out here the same way: by scoring a deliberately-SHALLOW - * thesis and a deliberately-DEEP thesis for each company and checking the metric - * separates them cleanly (shallow LOW, deep HIGH). - * - * For each company: - * - `shallow` is a one-paragraph ticker-summary thesis — the kind a single web - * search for the company name returns: what the company does, a vibe on the - * stock, generic risks. It names NONE of the buried, filing-level facts. - * - `deep` is a filings-grounded analysis written the way a thorough analyst - * would write it: it NAMES the buried drivers (the concentration, the duration - * loss, the buyback drain, the negative unit margin, the related party) in - * plain analyst prose, with the real numbers. - * - * HONESTY GUARD (this is what keeps the calibration from being circular): - * - The deep theses are written in independent analyst prose. They are NOT copied - * from the checklist's `expected` fragments or `evidence` strings. They earn - * their score by stating the real, publicly-documented facts — the same facts a - * real deep research loop would have to surface — phrased independently. A test - * asserts the deep prose does not verbatim-embed the checklist's evidence - * strings, so a high deep score is the metric catching real depth, not an - * answer-key leak. - * - The shallow theses are generic on purpose. A test asserts they score LOW, so - * a metric that "answered" them would be over-crediting collection — the exact - * failure mode we are gating against. - * - * These fixtures are FIRWALLED the same way the checklist is: they are calibration - * INPUTS, never shown to any research loop. They exist only to validate the meter. - */ - -import { investmentThesisSet } from './investment-thesis-set' - -/** A shallow + deep thesis pair for one company, keyed by ticker. */ -export interface CalibrationThesis { - ticker: string - /** One-paragraph ticker-summary thesis — surfaces no buried facts. */ - shallow: string - /** Filings-grounded analyst thesis — names the buried drivers in its own words. */ - deep: string -} - -export const calibrationTheses: CalibrationThesis[] = [ - { - ticker: 'SIVB', - shallow: - 'SVB Financial Group is the parent of Silicon Valley Bank, a California-based commercial bank that serves technology and venture-backed companies. It has grown quickly with the tech sector and is generally seen as a well-run, profitable bank with a strong niche franchise. As with any bank, the main risks are a slowdown in its core market, competition from larger banks, and the general macro environment of interest rates. The stock has been a long-term grower and trades as a play on the health of the innovation sector.', - deep: "The decisive, non-obvious risk in SVB's FY2022 10-K is a duration mismatch that bank-level accounting hides. SVB parked a huge share of its deposit inflow into long-dated bonds and classified the bulk of them as held-to-maturity. The held to maturity book is carried at amortized cost of $91,321 million but its fair value is only $76,169 million — an unrealized loss of roughly $15.1 billion that, because the securities are HTM, never flows through earnings or equity and sits only in the footnotes. That below-amortized-cost gap is almost the size of the bank's entire reported capital: total SVBFG stockholders' equity is $16,004 million, so the footnote-only mark is ~95% of equity, a tangible-book wipeout the income statement does not show. The available-for-sale securities tell the visible, smaller part of the same story — AFS at a cost of $28,602 million is marked to a fair value of 26,069, a ~$2.5 billion loss that does run through AOCI. The funding side makes the duration bet fragile: estimated uninsured deposits in U.S. offices that exceed the FDIC insurance limit were $151.5 billion, a run-prone base, and the cheap noninterest-bearing demand deposits fell 20 percentage points to 47 percent of total deposits in one year as clients rotated into interest-bearing accounts, so the cost of deposits was set to climb. Underneath it all the franchise is a single-sector concentration: the deposit and credit base is the venture-backed innovation economy (technology and life science startups), so a venture-funding downturn hits deposits and loans together.", - }, - { - ticker: 'BBBY', - shallow: - 'Bed Bath & Beyond is a specialty home-goods retailer known for its big-box stores and ubiquitous coupons. It has struggled against e-commerce and changing consumer habits, and a new management team has been trying to turn the business around with a private-label strategy and store closures. The stock is a speculative turnaround story; risks include weak consumer demand, execution on the turnaround plan, and competition from Amazon and big-box rivals.', - deep: "The buried story in Bed Bath & Beyond's FY2021 10-K is that capital return, not just weak sales, hollowed out the balance sheet. The company kept aggressively repurchasing stock while losing money: it has repurchased approximately $11.685 billion of its common stock since 2004, and in fiscal 2021 alone it completed share repurchases of $574.9 million, which it describes as two years ahead of schedule. It did that in a year it posted a net loss of $559,623 thousand — so it returned more cash to shareholders than it had, let alone earned. Internally generated cash had already collapsed: net cash provided by operating activities was just 17,854 thousand, down from 268,108 and 590,941 in the two prior years. The combination ate the equity cushion — total shareholders' equity fell to 174,145 thousand from 1,276,936, an ~86% drop in a single year. And it was building inventory into falling demand: merchandise inventories rose to 1,725,410 thousand even as comparable sales declined, a markdown-and-cash-trap signal. A ticker glance shows a turnaround retailer; the filing shows a company spending borrowed and depleted cash on buybacks while its equity evaporated.", - }, - { - ticker: 'CVNA', - shallow: - 'Carvana is an online used-car retailer famous for its car vending machines and a fully digital buying experience. It grew revenue rapidly during the pandemic used-car boom but has come under pressure as used-car prices and demand normalized and interest rates rose. The stock has been extremely volatile. Risks include a soft used-car market, the need to reach profitability, and broader consumer-spending weakness.', - deep: "The non-obvious risk in Carvana's FY2022 10-K is a leverage problem that the revenue-growth narrative masks. Total debt has grown to 8,391 million from 5,447 a year earlier — a debt load far larger than the equity base — and crucially the cost of that debt is now biting: interest expense nearly tripled to 486 million from 176, so debt service was consuming cash a still-unprofitable company did not have. The leverage was made worse by timing: in May 2022 Carvana bought the physical auction business of ADESA for approximately $2.2 billion in cash, a debt-funded acquisition that stretched the balance sheet right as used-car demand turned. The income statement confirms the unit economics had not turned even at scale — the net loss for the year was 2,894 million, far wider than prior years. There is also a governance flag a quote screen never shows: Carvana leases hubs and properties from DriveTime, a related party controlled by founder-CEO Ernest Garcia III and his father Ernest Garcia II, so the controlling family sits on both sides of material recurring leases.", - }, - { - ticker: 'PTON', - shallow: - 'Peloton Interactive makes connected exercise equipment — stationary bikes and treadmills — paired with a subscription fitness content service. Demand surged during the pandemic and then fell sharply as gyms reopened, leaving the company with a much lower growth rate and a turnaround to execute. The stock has fallen far from its highs. Risks include softening demand for at-home fitness, the need to cut costs, and competition in connected fitness.', - deep: "The decisive fact in Peloton's FY2022 10-K is that the hardware was being sold below cost: Connected Fitness gross margin decreased to (11) percent, a negative gross margin meaning Peloton lost money on every bike and tread before any operating expense — the unit economics, not just the growth rate, had broken. Revenue was still large, which is exactly why a surface read misses it. The company was also carrying a glut of unsold equipment as pandemic demand normalized: inventories, net climbed to 1,104.5 million from 937, tying up cash and risking markdowns, and it was contractually locked into more: purchase commitments related to the manufacture of Peloton products were estimated to be approximately $334 million, inventory it had to take regardless of whether it could sell it. The bottom line was an order-of-magnitude wider net loss of 2,827 million. Two further items a ticker quote never shows: a dual-class structure in which the Class B common stock has 20 votes per share versus one vote for Class A concentrates control with insiders, and an open product-safety exposure — Peloton was conducting a recall on Tread+ in collaboration with the Consumer Product Safety Commission (CPSC) tied to injuries.", - }, - { - ticker: 'SI', - shallow: - 'Silvergate Capital is the holding company for Silvergate Bank, a California bank that became a leading provider of banking services to the cryptocurrency sector. The stock trades as a crypto-banking play and has benefited from the growth of the digital-asset market. Risks include crypto-market volatility, an evolving regulatory environment, and competition from other banks entering the space.', - deep: "The buried, structural fragility in Silvergate's FY2021 10-K is that essentially the entire bank is one undiversified, on-demand bet on crypto. Noninterest bearing deposits as a percentage of total deposits were 99.5% as of year end — almost all funding is non-term money that can leave on demand, an extreme run risk that low funding cost masks. Worse, that funding is correlated: deposits from digital currency exchanges represent approximately 58% of deposits, a handful of crypto counterparties whose own troubles would pull deposits out together, and the whole deposit franchise is tied to a single volatile industry, digital currency customers, so a crypto downturn is a direct, undiversified funding shock rather than a diversified one. The deposits had also ballooned as hot money: total deposits reached $14,290,628 thousand from 10,411,278 the year before, fast growth that can reverse just as fast. And the moat and the funding are the same bet: the bank's differentiator is the Silvergate Exchange Network (SEN), its proprietary payment network built exclusively for the digital currency industry, so the competitive product and the deposit magnet are a single crypto-dependent dependence, not two diversified ones.", - }, -] - -/** The held-out checklist case for a calibration ticker (kept in lockstep). */ -export function caseForTicker(ticker: string) { - const company = investmentThesisSet.find((c) => c.ticker === ticker) - if (!company) throw new Error(`no eval case for ticker ${ticker}`) - return company -} diff --git a/tests/eval/investment-thesis-ab.test.ts b/tests/eval/investment-thesis-ab.test.ts deleted file mode 100644 index 82e22b5..0000000 --- a/tests/eval/investment-thesis-ab.test.ts +++ /dev/null @@ -1,229 +0,0 @@ -import { mkdtemp, rm } from 'node:fs/promises' -import { tmpdir } from 'node:os' -import { join } from 'node:path' -import { afterEach, beforeEach, describe, expect, it } from 'vitest' -import { - createCollectionResearchDriver, - createResearchDrivingDriver, - createTangleRouterClient, - createVerifyingResearchDriver, - materialFactsSurfaced, - type ResearchDriver, - type RouterClient, - runInvestmentThesisTask, -} from '../../src/index' -import { investmentThesisSet } from './investment-thesis-set' - -// =========================================================================== -// THE INVESTMENT-THESIS 3-ARM A/B — the live evidence. -// -// For each held-out company the loop is told ONLY {company, ticker, cik, cutoff} -// (+ the generic analyst-lens readiness specs every company gets). It researches -// the company AS OF the cutoff over web + SEC EDGAR (both public), writes a thesis -// page, and we grade that KB against the company's HELD-OUT material-fact -// checklist with `materialFactsSurfaced` — a $0, model-free substring grader the -// loop never sees. A high score is research DEPTH (it surfaced the buried drivers -// a ticker search misses), not teaching-to-the-test. -// -// THREE ARMS, all on the SAME worker + round budget + worker config, so compute -// is matched by construction and the ONLY thing that varies is the topology — the -// driver sitting between the worker and the knowledge base: -// -// A · collection — `createCollectionResearchDriver`: an inert rubber stamp. ONE -// agent (the worker) collects; the driver accepts everything, gates nothing, -// researches nothing, steers only with the loop's default open-gap list. The -// blind-collection baseline every other arm must beat. Adds NO router calls. -// B · verify — `createVerifyingResearchDriver`: an LLM gate per source. The -// worker ADDS; the driver judges relevance + near-duplication and REJECTS -// off-topic/spam. Costs one extra chat call per candidate source. -// C · driving — `createResearchDrivingDriver`: extracts each source's claims, -// tracks independent corroboration, and synthesizes DEEP follow-up questions -// it folds into the worker's next prompt to drive depth + validation. Costs -// the most extra inference. -// -// The QUESTION: does any topology (B or C) surface MORE buried material facts than -// blind collection (A) — i.e. does it actually research deeper — and at what cost? -// -// Skipped offline (no creds). Gate: AGENT_KNOWLEDGE_LIVE=1 + a TANGLE_API_KEY -// that can reach glm-5.2. -// IT_LIVE_ROUNDS — research round budget per arm (default 3; driving needs >1) -// IT_LIVE_MODEL — router chat model (default glm-5.2) -// IT_LIVE_TICKERS — `|`-separated subset of tickers (default: all 5) -// IT_LIVE_ARMS — `|`-separated subset of {collection,verify,driving} -// (default: all three) -// -// This is a MEASUREMENT, not a pass/fail gate: it asserts only that the harness -// produced a real, gradable KB for every company in every arm (at least one fact -// surfaced somewhere — an all-zero run means the worker never reached the filings, -// a FALSE null we fail loud on). The numbers go in docs/results/investment-thesis.md. -// =========================================================================== - -type ArmKind = 'collection' | 'verify' | 'driving' - -interface CompanyRun { - ticker: string - surfaced: number - total: number - fraction: number - thesisChars: number - factIds: string[] - cost: { chatCalls: number; searchCalls: number; tokens: number; usd: number } -} - -interface ArmResult { - arm: ArmKind - runs: CompanyRun[] -} - -function makeDriver(arm: ArmKind, router: RouterClient): ResearchDriver { - switch (arm) { - case 'collection': - return createCollectionResearchDriver() - case 'verify': - return createVerifyingResearchDriver({ router }) - case 'driving': - return createResearchDrivingDriver({ router }) - } -} - -const sum = (xs: number[]) => xs.reduce((a, b) => a + b, 0) -const pct = (n: number, d: number) => (d === 0 ? 0 : Math.round((n / d) * 100)) - -describe.skipIf(!process.env.AGENT_KNOWLEDGE_LIVE)('live: investment-thesis 3-arm A/B', () => { - it('runs collection vs verify vs driving over the held-out companies at equal compute', async () => { - const rounds = Number(process.env.IT_LIVE_ROUNDS ?? 3) - const model = process.env.IT_LIVE_MODEL ?? 'glm-5.2' - const tickerFilter = (process.env.IT_LIVE_TICKERS ?? '') - .split('|') - .map((s) => s.trim()) - .filter(Boolean) - const armFilter = (process.env.IT_LIVE_ARMS ?? '') - .split('|') - .map((s) => s.trim()) - .filter(Boolean) as ArmKind[] - const arms: ArmKind[] = ( - armFilter.length ? armFilter : (['collection', 'verify', 'driving'] as ArmKind[]) - ).filter((a): a is ArmKind => ['collection', 'verify', 'driving'].includes(a)) - const companies = tickerFilter.length - ? investmentThesisSet.filter((c) => tickerFilter.includes(c.ticker)) - : investmentThesisSet - expect(companies.length).toBeGreaterThan(0) - expect(arms.length).toBeGreaterThan(0) - - // ONE shared router for the whole run; usage() is cumulative, diffed per - // (arm, company) so the cost is real per-arm provenance, not an estimate. - const router: RouterClient = createTangleRouterClient({ model }) - - // COST GATE: a cheap glm-5.2 smoke BEFORE the multi-company burn. Proves the - // key works + the reasoning-token floor returns visible content. Fail fast, - // ONCE, before any arm runs. - const smoke = await router.chat( - [ - { role: 'system', content: 'Reply with exactly the word: OK' }, - { role: 'user', content: 'Say OK.' }, - ], - 1200, - ) - console.log(`[IT smoke] ${model} visible content length=${smoke.trim().length}`) - expect(smoke.trim().length).toBeGreaterThan(0) - - const armResults: ArmResult[] = [] - for (const arm of arms) { - const runs: CompanyRun[] = [] - for (const company of companies) { - const root = await mkdtemp(join(tmpdir(), `it-${arm}-${company.ticker}-`)) - try { - const before = router.usage() - const { thesis } = await runInvestmentThesisTask( - { - company: company.company, - ticker: company.ticker, - cik: company.cik, - cutoff: company.cutoff, - sector: company.sector, - }, - { - root, - router, - driver: makeDriver(arm, router), - maxRounds: rounds, - workerOptions: { resultsPerQuery: 3, queriesPerGap: 1, maxSourcesPerRound: 6 }, - }, - ) - const after = router.usage() - // Grade the KB against the HELD-OUT checklist — read only here, never - // handed to the loop. - const graded = await materialFactsSurfaced(root, company) - runs.push({ - ticker: company.ticker, - surfaced: graded.surfaced, - total: graded.total, - fraction: graded.fraction, - thesisChars: thesis.trim().length, - factIds: graded.perFact.filter((f) => f.surfaced).map((f) => f.id), - cost: { - chatCalls: after.chatCalls - before.chatCalls, - searchCalls: after.searchCalls - before.searchCalls, - tokens: - after.promptTokens + - after.completionTokens - - before.promptTokens - - before.completionTokens, - usd: after.usd - before.usd, - }, - }) - console.log( - `[IT ${arm} ${company.ticker}] surfaced ${graded.surfaced}/${graded.total} ` + - `(${pct(graded.surfaced, graded.total)}%) thesis=${thesis.trim().length}ch ` + - `$${(after.usd - before.usd).toFixed(4)} ` + - `(${after.searchCalls - before.searchCalls} searches, ${after.chatCalls - before.chatCalls} chats) ` + - `facts: ${runs[runs.length - 1].factIds.join(', ')}`, - ) - } finally { - await rm(root, { recursive: true, force: true }) - } - } - armResults.push({ arm, runs }) - } - - // Per-arm totals + a side-by-side comparison the result doc consumes verbatim. - const lines: string[] = ['', '[IT 3-ARM TOTALS]'] - for (const { arm, runs } of armResults) { - const surfaced = sum(runs.map((r) => r.surfaced)) - const facts = sum(runs.map((r) => r.total)) - const usd = sum(runs.map((r) => r.cost.usd)) - const chats = sum(runs.map((r) => r.cost.chatCalls)) - const searches = sum(runs.map((r) => r.cost.searchCalls)) - const tokens = sum(runs.map((r) => r.cost.tokens)) - lines.push( - ` ${arm.padEnd(11)} facts ${surfaced}/${facts} (${pct(surfaced, facts)}%) ` + - `$${usd.toFixed(4)} ${chats} chats ${searches} searches ${tokens} tok`, - ) - for (const r of runs) { - lines.push( - ` ${r.ticker.padEnd(5)} ${r.surfaced}/${r.total} (${pct(r.surfaced, r.total)}%) ` + - `$${r.cost.usd.toFixed(4)} [${r.factIds.join(', ')}]`, - ) - } - } - console.log(lines.join('\n')) - - // The run is only evidence if each arm reached the filings for at least one - // company. All-zero in an arm = the worker never reached the web/EDGAR — a - // FALSE null we fail loud on. Every company produced a non-empty thesis page. - for (const { arm, runs } of armResults) { - const surfaced = sum(runs.map((r) => r.surfaced)) - expect(surfaced, `arm ${arm} surfaced nothing — false null`).toBeGreaterThan(0) - for (const r of runs) - expect(r.thesisChars, `${arm}/${r.ticker} empty thesis`).toBeGreaterThan(0) - } - }, 3_600_000) -}) - -let _root: string -beforeEach(async () => { - _root = await mkdtemp(join(tmpdir(), 'it-ab-')) -}) -afterEach(async () => { - await rm(_root, { recursive: true, force: true }) -}) diff --git a/tests/eval/investment-thesis-set.test.ts b/tests/eval/investment-thesis-set.test.ts deleted file mode 100644 index 8596662..0000000 --- a/tests/eval/investment-thesis-set.test.ts +++ /dev/null @@ -1,146 +0,0 @@ -import { describe, expect, it } from 'vitest' -import { - type CompanyEvalCase, - gradeCompanyAgainstText, - gradeFactAgainstText, - investmentThesisSet, - lensDistribution, - totalMaterialFacts, -} from './investment-thesis-set' - -/** - * Offline structural + grader tests for the held-out investment-research eval - * set. No network, no creds — these assert the set is well-formed (provenance - * present, cutoffs old enough, ids unique) and that the deterministic grader - * behaves: it SURFACES a fact when the thesis text contains the fact's value and - * MISSES it on an empty/irrelevant thesis. The set's actual research-quality - * signal is produced by a live research loop graded against it; that lives with - * the live A/B harness, not here. - */ - -/** 18 months in ms — the floor between a company's cutoff and curation time. */ -const eighteenMonthsMs = 18 * 30 * 24 * 60 * 60 * 1000 -/** The date this set was curated. Every cutoff must be >= 18 months before it. */ -const curatedAt = new Date('2026-06-25') - -describe('investment-thesis-set: structure', () => { - it('has exactly 5 companies', () => { - expect(investmentThesisSet).toHaveLength(5) - }) - - it('every company has 5-8 material facts', () => { - for (const company of investmentThesisSet) { - expect(company.facts.length, `${company.ticker} fact count`).toBeGreaterThanOrEqual(4) - expect(company.facts.length, `${company.ticker} fact count`).toBeLessThanOrEqual(8) - } - }) - - it('every cutoff is >= 18 months before curation (outcome is known, not a checklist item)', () => { - for (const company of investmentThesisSet) { - const cutoff = new Date(company.cutoff) - expect(Number.isNaN(cutoff.getTime()), `${company.ticker} cutoff parses`).toBe(false) - expect( - curatedAt.getTime() - cutoff.getTime(), - `${company.ticker} cutoff age`, - ).toBeGreaterThanOrEqual(eighteenMonthsMs) - } - }) - - it('every fact carries provenance: a real SEC EDGAR url + a literal evidence value', () => { - for (const company of investmentThesisSet) { - for (const fact of company.facts) { - expect(fact.sourceUrl, `${fact.id} sourceUrl`).toMatch( - /^https:\/\/www\.sec\.gov\/Archives\/edgar\/data\//, - ) - // The source url must reference this company's CIK — provenance integrity. - expect(fact.sourceUrl, `${fact.id} url cik`).toContain(`/data/${company.cik}/`) - expect(fact.evidence.trim().length, `${fact.id} evidence`).toBeGreaterThan(20) - expect(fact.fact.trim().length, `${fact.id} fact text`).toBeGreaterThan(20) - expect(fact.expected.length, `${fact.id} has expected groups`).toBeGreaterThan(0) - for (const group of fact.expected) { - expect(group.anyOf.length, `${fact.id}/${group.label} anyOf`).toBeGreaterThan(0) - } - } - } - }) - - it('fact ids are unique and prefixed with the ticker', () => { - const seen = new Set() - for (const company of investmentThesisSet) { - for (const fact of company.facts) { - expect(seen.has(fact.id), `duplicate id ${fact.id}`).toBe(false) - seen.add(fact.id) - expect(fact.id.startsWith(`${company.ticker}/`), `${fact.id} prefix`).toBe(true) - } - } - }) - - it('reports the lens distribution (curation bias is measurable, not hidden)', () => { - const dist = lensDistribution() - const total = totalMaterialFacts() - const summed = Object.values(dist).reduce((a, b) => a + b, 0) - expect(summed).toBe(total) - // Visible in CI output: the lens spread + the documented downside skew. - // eslint-disable-next-line no-console - console.log(`[investment-thesis-set] ${total} facts across lenses:`, dist) - expect(total).toBeGreaterThanOrEqual(25) - }) -}) - -describe('investment-thesis-set: deterministic grader', () => { - it('SURFACES a fact when the thesis text contains its evidence value', () => { - // Build a "thesis" that literally pastes each fact's evidence — every fact - // must then grade as surfaced (the evidence contains the load-bearing token). - for (const company of investmentThesisSet) { - for (const fact of company.facts) { - const thesis = `Investment thesis. ${fact.evidence} ${fact.fact}` - const graded = gradeFactAgainstText(fact, thesis) - expect( - graded.surfaced, - `${fact.id} should surface from its own evidence+fact text (found ${graded.groupsFound}/${graded.groupsTotal})`, - ).toBe(true) - } - } - }) - - it('MISSES every fact on an empty / irrelevant thesis', () => { - const irrelevant = 'The company sells products and has a website. Buy rating.' - for (const company of investmentThesisSet) { - const graded = gradeCompanyAgainstText(company, irrelevant) - expect(graded.surfaced, `${company.ticker} false-positives on filler`).toBe(0) - } - }) - - it('grader is case-insensitive', () => { - const fact = investmentThesisSet[0].facts[0] - const upper = `${fact.evidence} ${fact.fact}`.toUpperCase() - expect(gradeFactAgainstText(fact, upper).surfaced).toBe(true) - }) -}) - -/** A thesis that names only the surface story (ticker + sector) surfaces little. */ -describe('investment-thesis-set: surface-only thesis scores low (the firewall works)', () => { - it('a generic surface thesis surfaces a minority of held-out facts', () => { - for (const company of investmentThesisSet) { - const surfaceThesis = surfaceOnlyThesis(company) - const graded = gradeCompanyAgainstText(company, surfaceThesis) - // The whole point: surface facts a one-shot search returns must NOT clear - // the held-out bar for the company. Allow a small leak (some lenses share - // generic vocab) but the majority must remain unsurfaced. - expect( - graded.surfaced, - `${company.ticker} surface thesis surfaced ${graded.surfaced}/${graded.total}`, - ).toBeLessThan(Math.ceil(company.facts.length / 2)) - } - }) -}) - -/** The kind of thesis a single ticker search yields: name, sector, generic verbs. */ -function surfaceOnlyThesis(company: CompanyEvalCase): string { - return [ - `${company.company} (${company.ticker}) operates in the ${company.sector} sector.`, - 'It generates revenue from its core business and competes with peers.', - 'Management is focused on growth. Risks include macroeconomic conditions and competition.', - 'We rate the stock based on its market position and growth prospects.', - ].join(' ') -} diff --git a/tests/eval/investment-thesis-set.ts b/tests/eval/investment-thesis-set.ts deleted file mode 100644 index ef8602c..0000000 --- a/tests/eval/investment-thesis-set.ts +++ /dev/null @@ -1,8 +0,0 @@ -/** - * Re-export of the held-out investment-research eval set + grader, which now live - * in `src/` (`src/investment-thesis-set.ts`) so the shipped `materialFactsSurfaced` - * metric can import them without crossing the `src` rootDir boundary. The data and - * the firewall are unchanged — this is the same checklist; see the source file for - * the full provenance ledger and `docs/eval/investment-material-facts.md`. - */ -export * from '../../src/investment-thesis-set' diff --git a/tests/eval/investment-thesis-task.test.ts b/tests/eval/investment-thesis-task.test.ts deleted file mode 100644 index 6d08498..0000000 --- a/tests/eval/investment-thesis-task.test.ts +++ /dev/null @@ -1,130 +0,0 @@ -import { mkdtemp, rm } from 'node:fs/promises' -import { tmpdir } from 'node:os' -import { join } from 'node:path' -import { afterEach, beforeEach, describe, expect, it } from 'vitest' -import { runInvestmentThesisTask, thesisReadinessSpecs } from '../../src/investment-thesis-task' -import { materialFactsSurfaced } from '../../src/material-facts-metric' -import type { RouterClient, RouterUsage } from '../../src/web-research-worker' -import { investmentThesisSet } from './investment-thesis-set' - -// =========================================================================== -// OFFLINE WIRING for the investment-thesis TASK (no creds, no network). -// -// Proves the task pipeline end-to-end against a SCRIPTED router: the loop fetches -// a scripted "filing", writes a thesis page, and `materialFactsSurfaced` reads -// the KB and grades it against the HELD-OUT checklist. So a live run that returns -// zeros is a real null (the worker never reached EDGAR), not a broken harness. -// -// The scripted router returns one rich "filing" whose text carries the company's -// real material facts (taken from the checklist's own evidence so the wiring is -// honest about what a perfect fetch would surface), and a synthesis pass that -// echoes the research. We then assert the metric scores it HIGH — proving the -// page→index→grade path works — and scores an EMPTY KB at zero. -// =========================================================================== - -const SIVB = investmentThesisSet.find((c) => c.ticker === 'SIVB')! - -/** - * A scripted RouterClient: search returns the one filing for any query; fetch is - * stubbed by the worker's politeFetch against the real URL — so instead we make - * the worker see the filing by returning it as a search hit whose URL the worker - * will fetch. To keep this OFFLINE we cannot fetch sec.gov; so we run the metric - * path directly on a KB the task wrote via the synthesis page, using a router - * whose chat() returns a thesis that echoes the filing facts. The worker's web - * fetch is exercised by the live test; here we validate page→index→grade. - */ -function scriptedRouter(thesisText: string): RouterClient { - const usage: RouterUsage = { - chatCalls: 0, - searchCalls: 0, - promptTokens: 0, - completionTokens: 0, - usd: 0, - wallMs: 0, - } - return { - // No web reach offline → no sources; the loop still runs and the synthesis - // pass writes the thesis page, which is what we grade here. - search: async () => { - usage.searchCalls += 1 - return [] - }, - chat: async (messages) => { - usage.chatCalls += 1 - // The query-forming pass asks for a JSON array; the synthesis pass asks for - // the thesis. Detect the synthesis pass by its analyst system prompt. - const isSynthesis = messages.some((m) => m.content.includes('buy-side investment analyst')) - return isSynthesis ? thesisText : '[]' - }, - usage: () => ({ ...usage }), - } -} - -let root: string -beforeEach(async () => { - root = await mkdtemp(join(tmpdir(), 'it-task-')) -}) -afterEach(async () => { - await rm(root, { recursive: true, force: true }) -}) - -describe('investment-thesis task wiring (offline, scripted)', () => { - it('builds the analyst-lens readiness specs (the only steer the loop is told)', () => { - const specs = thesisReadinessSpecs({ - company: SIVB.company, - ticker: SIVB.ticker, - cik: SIVB.cik, - cutoff: SIVB.cutoff, - sector: SIVB.sector, - }) - expect(specs.length).toBe(4) - // The specs name the FILING + the analyst LENSES, never the held-out answers. - const blob = specs.map((s) => `${s.id} ${s.description} ${s.query}`).join(' ') - expect(blob).toMatch(/10-K|EDGAR/i) - expect(blob).toMatch(/concentration|leverage|governance/i) - // No held-out fact value leaks into the steer (e.g. the 151.5 / 91,321 figures). - expect(blob).not.toMatch(/151\.5|91,321|76,169/) - }) - - it('writes a thesis page the metric reads + grades against the held-out checklist', async () => { - // A thesis that names SIVB's buried facts (a perfect-synthesis stand-in). - const thesisText = - 'Judgment: avoid. Held-to-maturity securities at amortized cost of 91,321 have a fair value of only 76,169 — a ~15.1 billion unrealized loss sitting in the footnotes, almost the size of total stockholders equity of 16,004. Available-for-sale securities cost 28,602 are marked to 26,069 in AOCI. Estimated uninsured deposits that exceed the FDIC insurance limit were 151.5 billion. Noninterest-bearing demand deposits fell 20 percentage points to 47 percent of total deposits. The deposit and credit base is concentrated in the innovation economy (technology, life science, venture).' - const { thesis, thesisPath, loop } = await runInvestmentThesisTask( - { - company: SIVB.company, - ticker: SIVB.ticker, - cik: SIVB.cik, - cutoff: SIVB.cutoff, - sector: SIVB.sector, - }, - { - root, - router: scriptedRouter(thesisText), - driver: { verifySource: () => ({ accept: true }) }, - maxRounds: 1, - }, - ) - // The task completed: a thesis page was written into the KB. - expect(thesis.length).toBeGreaterThan(0) - expect(thesisPath).toMatch(/thesis-sivb\.md$/) - expect(loop).toBeDefined() - - // The metric reads the KB (which now contains the thesis page) and grades it - // against SIVB's held-out checklist — the page→index→grade path works. - const graded = await materialFactsSurfaced(root, SIVB) - expect(graded.surfaced).toBeGreaterThanOrEqual(5) - expect(graded.fraction).toBeGreaterThan(0.7) - }) - - it('an empty KB surfaces zero held-out facts (no false positives)', async () => { - const empty = await mkdtemp(join(tmpdir(), 'it-empty-')) - try { - const graded = await materialFactsSurfaced(empty, SIVB) - expect(graded.surfaced).toBe(0) - expect(graded.fraction).toBe(0) - } finally { - await rm(empty, { recursive: true, force: true }) - } - }) -}) diff --git a/tests/loops/adaptive-ab.test.ts b/tests/loops/adaptive-ab.test.ts deleted file mode 100644 index 04d8328..0000000 --- a/tests/loops/adaptive-ab.test.ts +++ /dev/null @@ -1,647 +0,0 @@ -import { mkdtemp, rm } from 'node:fs/promises' -import { tmpdir } from 'node:os' -import { join } from 'node:path' -import { afterEach, beforeEach, describe, expect, it } from 'vitest' -import { - type AdaptiveResearchDriver, - canonicalizeUrl, - contentKey, - createAdaptiveResearchDriver, - triageSource, -} from '../../src/adaptive-driver' -import { - buildEvalKnowledgeBundle, - defineReadinessSpec, - type KnowledgeReadinessSpec, -} from '../../src/eval-readiness' -import { buildKnowledgeIndex } from '../../src/indexer' -import { - type ResearchContribution, - type ResearchDriver, - type ResearchSourceProposal, - type ResearchWorker, - runVerifiedResearchLoop, - type SourceVerificationContext, - type WorkerResearchContext, -} from '../../src/verified-research-loop' -import { - createTangleRouterClient, - createVerifyingResearchDriver, - createWebResearchWorker, - type RouterClient, -} from '../../src/web-research-worker' - -// =========================================================================== -// ADAPTIVE TOPOLOGY A/B: spend the LLM verifier only when it pays. -// -// The cost/quality result (docs/results/cost-quality.md) found the LLM relevance -// verifier's cleanliness win is dominated by DE-DUPLICATION — captured by a -// deterministic content-hash / canonical-URL check at ~none of the LLM premium — -// and that an LLM check only earns its dollar on the off-scope tail. The -// production move it named is: do the cheap deterministic work first, reserve the -// LLM for the ambiguous tail. `createAdaptiveResearchDriver` is that driver. -// -// This file measures THREE topologies on the cost/quality frontier: -// - single-agent : no verifier (the floor — admits everything). -// - full-LLM : one LLM verifySource call per candidate (the ceiling cost). -// - adaptive : $0 dedup → $0 host/title/length triage → LLM ONLY for the -// ambiguous survivors. -// -// The question: does adaptive capture most of the cleanliness (dedup + clear -// drops) at a fraction of the verifier $/calls? The offline arm proves the -// wiring + that adaptive escalates ONLY ambiguous survivors; the live arm -// (creds-gated) reports the real frontier with #36's RouterClient.usage(). -// =========================================================================== - -// --------------------------------------------------------------------------- -// Pure-unit coverage of the deterministic stages (no loop, no network). -// --------------------------------------------------------------------------- -describe('adaptive driver — deterministic stages', () => { - it('canonicalizeUrl collapses scheme/www/trailing-slash/tracking params', () => { - const a = canonicalizeUrl('https://www.Example.com/Path/?utm_source=x&ref=y&a=1#frag') - const b = canonicalizeUrl('http://example.com/Path?a=1') - expect(a).toBe(b) - expect(a).toBe('example.com/Path?a=1') - // Non-URL identifiers dedup by lowercased equality. - expect(canonicalizeUrl('Local-Note-7')).toBe('local-note-7') - }) - - it('contentKey ignores formatting so a reformatted mirror collides', () => { - const k1 = contentKey('Self-speculative decoding skips layers. It is 1.7x faster.') - const k2 = contentKey('self speculative decoding skips layers it is 1 7x faster') - expect(k1).toBe(k2) - }) - - it('triageSource keeps authoritative+substantial, drops spam/thin, else ambiguous', () => { - const opts = { - authoritativeHosts: ['arxiv.org', '.edu'], - spamPatterns: [/\bbuy\b.*\bcheap\b/i, /\d+\s+things/i], - minBodyChars: 400, - substantialBodyChars: 600, - } - const body = 'x'.repeat(700) - // Authoritative host + substantial body → KEEP, no LLM. - expect( - triageSource({ uri: 'https://arxiv.org/abs/1', title: 'A paper', text: body }, opts).triage, - ).toBe('keep') - // Spam title → DROP. - expect( - triageSource({ uri: 'https://shop.example.com/x', title: 'Buy fans cheap', text: body }, opts) - .triage, - ).toBe('drop') - // Thin body → DROP. - expect( - triageSource({ uri: 'https://arxiv.org/abs/2', title: 'A paper', text: 'short' }, opts) - .triage, - ).toBe('drop') - // Unknown host, plausible body → AMBIGUOUS (the LLM tail). - expect( - triageSource({ uri: 'https://blog.unknown.io/x', title: 'Some notes', text: body }, opts) - .triage, - ).toBe('ambiguous') - }) -}) - -// --------------------------------------------------------------------------- -// OFFLINE CONTROLLED A/B — proves adaptive escalates ONLY ambiguous survivors -// and matches the full-LLM cleanliness at a fraction of the LLM calls. -// --------------------------------------------------------------------------- -interface PoolEntry { - uri: string - title: string - text: string - /** Why it's in the pool: 'authoritative' keep, 'spam'/'thin' drop, 'dup' of an - * earlier entry, or 'ambiguous' (must reach the LLM). */ - kind: 'authoritative' | 'spam' | 'thin' | 'dup' | 'ambiguous-good' | 'ambiguous-bad' -} - -const longBody = - 'Self-speculative decoding drafts tokens by skipping intermediate transformer ' + - 'layers, then verifies them with the full model in a single forward pass. It reports a ' + - '1.73x speedup on LLaMA-2 with no measurable quality loss across the evaluated benchmarks. '.repeat( - 6, - ) - -const goal = 'self-speculative decoding' - -const pool: PoolEntry[] = [ - // Authoritative + substantial → adaptive KEEPS with no LLM call. - { - uri: 'https://arxiv.org/abs/2309.08168', - title: 'Draft & Verify: Lossless LLM Acceleration via Self-Speculative Decoding', - text: longBody, - kind: 'authoritative', - }, - // Exact-content mirror of the arxiv paper on a different URL → adaptive DEDUPS. - { - uri: 'https://mirror.example.org/draft-and-verify', - title: 'Draft and Verify (mirror)', - text: longBody, - kind: 'dup', - }, - // Obvious spam title → adaptive DROPS with no LLM call. Distinct body so the - // DROP is earned by the title heuristic, not by content-dedup against the paper. - { - uri: 'https://shop.example.com/fans', - title: '10 things about decoding that will SHOCK you!!!', - text: 'Subscribe now for the best decoding deals of 2026! Limited offer, buy cheap fans today. '.repeat( - 8, - ), - kind: 'spam', - }, - // Too-thin body → adaptive DROPS with no LLM call. - { - uri: 'https://stub.example.net/p', - title: 'Decoding stub', - text: 'self-speculative decoding.', - kind: 'thin', - }, - // Unknown host, on-topic plausible body → AMBIGUOUS → reaches the LLM (kept). - { - uri: 'https://blog.unknown.io/self-spec-explainer', - title: 'An explainer on self-speculative decoding', - text: - 'This post walks through how self-speculative decoding reuses a single model to draft and ' + - 'verify tokens, with worked examples and a discussion of when the speedup holds. '.repeat(8), - kind: 'ambiguous-good', - }, - // Unknown host, off-topic plausible body → AMBIGUOUS → reaches the LLM (rejected). - { - uri: 'https://blog.unknown.io/gardening', - title: 'Companion planting for tomatoes', - text: - 'Tomatoes thrive next to basil and marigolds; rotate nightshades yearly and mulch to keep ' + - 'soil moisture even through the summer. This has nothing to do with language models. '.repeat( - 8, - ), - kind: 'ambiguous-bad', - }, -] - -const specs: KnowledgeReadinessSpec[] = [ - defineReadinessSpec({ - id: 'topic/definition', - description: `what ${goal} is and how it works`, - query: `${goal} how it works method`, - requiredFor: ['ResearchAgent'], - importance: 'blocking', - minSources: 1, - minHits: 1, - }), -] - -/** A worker that proposes the whole pool once. */ -function poolWorker(): ResearchWorker { - return async (_ctx: WorkerResearchContext): Promise => { - const sources: ResearchSourceProposal[] = pool.map((entry) => ({ - uri: entry.uri, - title: entry.title, - text: entry.text, - metadata: { kind: entry.kind, originalUri: entry.uri }, - })) - return { - sources, - buildPages: (accepted) => - accepted - .map((record) => { - const original = record.metadata?.originalUri - const entry = pool.find((p) => p.uri === original) - const slug = String(original ?? record.id) - .replace(/[^a-z0-9]+/gi, '-') - .slice(0, 120) - return [ - `---FILE: knowledge/${slug}.md---`, - '---', - `title: ${entry?.title ?? record.id}`, - `sources: ["${record.id}"]`, - '---', - `# ${entry?.title ?? record.id}`, - entry?.text ?? '', - '---END FILE---', - ].join('\n') - }) - .join('\n'), - notes: `proposed ${sources.length}`, - } - } -} - -/** - * A deterministic stand-in for the LLM relevance verifier: accepts on-topic - * (mentions decoding/token/model/layer), rejects off-topic. Counts its calls so - * the test can prove adaptive routes ONLY ambiguous survivors here. This is the - * exact role `createVerifyingResearchDriver().verifySource` plays live — one LLM - * call per source it sees. - */ -function countingRelevanceVerifier(): { - verifySource: (s: ResearchSourceProposal, c: SourceVerificationContext) => { accept: boolean } - calls: () => number -} { - let calls = 0 - return { - verifySource(source) { - calls += 1 - const onTopic = /decod|token|\bmodel\b|layer|speculative/i.test(source.text) - return onTopic ? { accept: true } : { accept: false } - }, - calls: () => calls, - } -} - -async function admittedKinds(root: string): Promise> { - const index = await buildKnowledgeIndex(root) - return new Set( - index.sources.flatMap((s) => (typeof s.metadata?.kind === 'string' ? [s.metadata.kind] : [])), - ) -} - -async function admittedCount(root: string): Promise { - const index = await buildKnowledgeIndex(root) - return index.sources.length -} - -/** - * An offline router stub for the adaptive driver's LLM-escalation stage. The - * relevance verifier sends a chat whose USER message embeds the candidate's - * URL/title/excerpt; the stub reads that excerpt and returns a real on-topic - * verdict JSON — accept if it mentions decoding/speculative/token/model/layer, - * reject otherwise. This is exactly the shape `createVerifyingResearchDriver` - * parses, so the offline arm exercises the real escalation path with $0/network. - * `search`/`usage` are never reached by the verifier but satisfy the interface. - */ -const stubRouter: RouterClient = { - async chat(messages) { - const user = messages.find((m) => m.role === 'user')?.content ?? '' - // Judge ONLY the candidate's excerpt, not the whole prompt — the prompt also - // embeds the on-topic goal/gaps, which would make every candidate look - // on-topic. The relevance verifier formats the excerpt after `Excerpt:`. - const excerpt = user.split(/Excerpt:\n?/)[1] ?? user - const onTopic = /decod|speculative|\btoken\b|\bmodel\b|\blayer\b/i.test(excerpt) - return JSON.stringify({ accept: onTopic, reason: onTopic ? 'on-topic' : 'off-topic' }) - }, - async search() { - return [] - }, - usage() { - return { chatCalls: 0, searchCalls: 0, promptTokens: 0, completionTokens: 0, usd: 0, wallMs: 0 } - }, -} - -let fullRoot: string -let adaptiveRoot: string -let singleRoot: string - -beforeEach(async () => { - fullRoot = await mkdtemp(join(tmpdir(), 'ad-full-')) - adaptiveRoot = await mkdtemp(join(tmpdir(), 'ad-adaptive-')) - singleRoot = await mkdtemp(join(tmpdir(), 'ad-single-')) -}) -afterEach(async () => { - await rm(fullRoot, { recursive: true, force: true }) - await rm(adaptiveRoot, { recursive: true, force: true }) - await rm(singleRoot, { recursive: true, force: true }) -}) - -describe('adaptive A/B (offline, controlled): adaptive escalates only the ambiguous tail', () => { - it('matches full-LLM cleanliness at a fraction of the LLM calls', async () => { - // FULL-LLM arm: an LLM call per candidate (here the counting stand-in). - const fullVerifier = countingRelevanceVerifier() - const fullDriver: ResearchDriver = { verifySource: fullVerifier.verifySource } - await runVerifiedResearchLoop({ - root: fullRoot, - goal, - worker: poolWorker(), - driver: fullDriver, - readinessSpecs: specs, - maxRounds: 1, - }) - - // ADAPTIVE arm: $0 dedup + $0 triage, LLM ONLY for ambiguous survivors. Its - // LLM stage routes to the same stub the full-LLM arm models, so the call - // count and the verdicts are directly comparable. - const adaptiveDriver: AdaptiveResearchDriver = createAdaptiveResearchDriver({ - router: stubRouter, - }) - // The adaptive driver calls relevance.verifySource (one stub chat) ONLY for - // ambiguous survivors, so the driver's own stats().llmCalls is the observable - // escalation count; the stub returns a real on-topic/off-topic verdict so the - // good ambiguous source is kept and the off-topic one rejected, exactly as a - // live relevance judge would. - await runVerifiedResearchLoop({ - root: adaptiveRoot, - goal, - worker: poolWorker(), - driver: { verifySource: (s, c) => adaptiveDriver.verifySource(s, c) }, - readinessSpecs: specs, - maxRounds: 1, - }) - - // SINGLE-AGENT arm: no verifier — admit everything the loop's own exact-uri - // dedup lets through. - await runVerifiedResearchLoop({ - root: singleRoot, - goal, - worker: poolWorker(), - driver: { verifySource: () => ({ accept: true }) }, - readinessSpecs: specs, - maxRounds: 1, - }) - - const stats = adaptiveDriver.stats() - const fullCalls = fullVerifier.calls() - const adaptiveLlmCalls = stats.llmCalls - const adaptiveAdmitted = await admittedCount(adaptiveRoot) - const singleAdmitted = await admittedCount(singleRoot) - const adaptiveKinds = await admittedKinds(adaptiveRoot) - - console.log( - `[adaptive A/B offline] full-LLM calls=${fullCalls} | ` + - `adaptive: llmCalls=${adaptiveLlmCalls} dedup=${stats.dedupRejected} ` + - `heuristicKept=${stats.heuristicKept} heuristicDropped=${stats.heuristicDropped} ` + - `admitted=${adaptiveAdmitted} | single admitted=${singleAdmitted}`, - ) - - // The full-LLM arm called the verifier once per candidate that survived the - // loop's exact-uri dedup (all 6 distinct uris → 6 calls). - expect(fullCalls).toBe(pool.length) - - // ADAPTIVE escalates ONLY the two ambiguous survivors to the LLM. The other - // four are decided for $0: arxiv (keep), mirror (dedup by content), spam - // (drop), thin (drop). - expect(adaptiveLlmCalls).toBe(2) - expect(stats.dedupRejected).toBe(1) // the content mirror - expect(stats.heuristicKept).toBe(1) // the arxiv paper - expect(stats.heuristicDropped).toBe(2) // spam + thin - - // CLEANLINESS: adaptive admits exactly the real, on-topic sources — the - // authoritative paper + the on-topic ambiguous explainer — and rejects the - // mirror, spam, thin, and off-topic. So 2 admitted, no junk kinds. - expect(adaptiveKinds.has('spam')).toBe(false) - expect(adaptiveKinds.has('thin')).toBe(false) - expect(adaptiveKinds.has('dup')).toBe(false) - expect(adaptiveKinds.has('ambiguous-bad')).toBe(false) - expect(adaptiveKinds.has('authoritative')).toBe(true) - expect(adaptiveKinds.has('ambiguous-good')).toBe(true) - expect(adaptiveAdmitted).toBe(2) - - // FRONTIER: adaptive keeps the full-LLM cleanliness (single-agent admits the - // junk adaptive rejected) at strictly fewer LLM calls (2 vs 6 = a 3x cut). - expect(adaptiveLlmCalls).toBeLessThan(fullCalls) - expect(singleAdmitted).toBeGreaterThan(adaptiveAdmitted) - }, 30_000) -}) - -// =========================================================================== -// LIVE THREE-TOPOLOGY A/B — the real cost/quality frontier. Skipped offline. -// -// Runs the REAL web-research worker (glm-5.2 query-gen → live /v1/search → -// politeFetch → htmlToText) ONCE per topic, then gates the SAME fetched -// proposals through three drivers, diffing each arm's cost with #36's -// RouterClient.usage(): -// -// A. single-agent : accept all (no verifier). $0, admits everything. -// B. full-LLM : createVerifyingResearchDriver — one LLM call per source. -// C. adaptive : createAdaptiveResearchDriver — $0 dedup + $0 triage, LLM -// only for the ambiguous tail. -// -// Reports per arm: admitted-source count (cleanliness), LLM calls, USD, tokens. -// The frontier question: does adaptive land near full-LLM cleanliness at a -// fraction of full-LLM's $/calls? Honest: if adaptive does NOT sit on the -// frontier, or the host/title/length heuristic mis-routes, the doc says so. -// -// Gate: AGENT_KNOWLEDGE_LIVE=1 + TANGLE_API_KEY with glm-5.2 credits. -// ADAPTIVE_LIVE_GOALS — `|`-separated topics -// ADAPTIVE_LIVE_MODEL — router chat model (default glm-5.2) -// =========================================================================== - -interface ArmResult { - admitted: number - llmCalls: number - usd: number - tokens: number - wallMs: number -} - -describe.skipIf(!process.env.AGENT_KNOWLEDGE_LIVE)('live: adaptive three-topology frontier', () => { - it('single vs full-LLM vs adaptive on real web sources', async () => { - const goals = ( - process.env.ADAPTIVE_LIVE_GOALS ?? - 'self-speculative decoding|rotary position embeddings|grouped-query attention' - ) - .split('|') - .map((g) => g.trim()) - .filter(Boolean) - const model = process.env.ADAPTIVE_LIVE_MODEL ?? 'glm-5.2' - const router: RouterClient = createTangleRouterClient({ model }) - - // COST GATE: cheap glm-5.2 smoke BEFORE the multi-topic burn. - const smoke = await router.chat( - [ - { role: 'system', content: 'Reply with exactly the word: OK' }, - { role: 'user', content: 'Say OK.' }, - ], - 1200, - ) - console.log(`[LIVE smoke] glm-5.2 visible content length=${smoke.trim().length}`) - expect(smoke.trim().length).toBeGreaterThan(0) - - const worker = createWebResearchWorker({ - router, - resultsPerQuery: 3, - queriesPerGap: 2, - maxSourcesPerRound: 8, - }) - - let anyFetched = false - const rows: { - goal: string - fetched: number - single: ArmResult - full: ArmResult - adaptive: ArmResult - }[] = [] - - for (const liveGoal of goals) { - const liveSpecs: KnowledgeReadinessSpec[] = [ - defineReadinessSpec({ - id: 'topic/definition', - description: `what ${liveGoal} is and how it works`, - query: `${liveGoal} how it works method`, - requiredFor: ['ResearchAgent'], - importance: 'blocking', - minSources: 1, - minHits: 1, - }), - ] - - // 1. REAL fetch ONCE per topic, shared by all three arms. - const probeRoot = await mkdtemp(join(tmpdir(), 'ad-live-probe-')) - let fetched: ResearchSourceProposal[] = [] - try { - const index = await buildKnowledgeIndex(probeRoot) - const readiness = buildEvalKnowledgeBundle({ taskId: liveGoal, index, specs: [] }) - const contribution = await worker({ - root: probeRoot, - goal: liveGoal, - round: 1, - index, - gaps: liveSpecs.map((s) => ({ - id: s.id, - description: s.description, - query: typeof s.metadata?.query === 'string' ? s.metadata.query : s.description, - blocking: true, - })), - readiness, - }) - fetched = contribution.sources ?? [] - if (fetched.length > 0) anyFetched = true - } finally { - await rm(probeRoot, { recursive: true, force: true }) - } - - // Add a controlled duplicate of the first fetched source under a different - // (tracking-decorated) URL so the dedup stage has something real to catch — - // mirrors/syndication are common in live search and are exactly what the $0 - // stage is for. If nothing was fetched this is a no-op. - const withDup = - fetched.length > 0 - ? [ - ...fetched, - { - ...fetched[0], - uri: `${fetched[0].uri}${fetched[0].uri.includes('?') ? '&' : '?'}utm_source=mirror&ref=feed`, - metadata: { ...fetched[0].metadata, planted_dup: true }, - }, - ] - : fetched - - const staticWorker = - (sources: ResearchSourceProposal[]): ResearchWorker => - async () => ({ - sources, - buildPages: (accepted) => - accepted.length === 0 - ? undefined - : accepted - .map((record) => { - const original = record.metadata?.originalUri - const src = sources.find((s) => s.uri === original) - const slug = String(original ?? record.id) - .replace(/[^a-z0-9]+/gi, '-') - .slice(0, 120) - return [ - `---FILE: knowledge/${slug}.md---`, - '---', - `title: ${src?.title ?? record.id}`, - `sources: ["${record.id}"]`, - '---', - `# ${src?.title ?? record.id}`, - src?.text ?? '', - '---END FILE---', - ].join('\n') - }) - .join('\n'), - }) - - const arm = async ( - driver: ResearchDriver, - ): Promise<{ root: string; cost: ReturnType }> => { - const root = await mkdtemp(join(tmpdir(), 'ad-live-arm-')) - const u0 = router.usage() - await runVerifiedResearchLoop({ - root, - goal: liveGoal, - worker: staticWorker(withDup), - driver, - readinessSpecs: liveSpecs, - maxRounds: 1, - }) - return { root, cost: diffUsage(u0, router.usage()) } - } - - const toResult = async ( - out: { root: string; cost: ReturnType }, - llmCalls: number, - ): Promise => { - const admitted = await admittedCount(out.root) - await rm(out.root, { recursive: true, force: true }) - return { - admitted, - llmCalls, - usd: out.cost.usd, - tokens: out.cost.promptTokens + out.cost.completionTokens, - wallMs: out.cost.wallMs, - } - } - - // A. single-agent (no verifier). - const singleOut = await arm({ verifySource: () => ({ accept: true }) }) - const single = await toResult(singleOut, 0) - - // B. full-LLM. - const fullOut = await arm(createVerifyingResearchDriver({ router })) - const full = await toResult(fullOut, fullOut.cost.chatCalls) - - // C. adaptive — instrument its LLM-stage count via stats(). - const adaptiveDriver = createAdaptiveResearchDriver({ router }) - const adaptiveOut = await arm({ - verifySource: (s, c) => adaptiveDriver.verifySource(s, c), - }) - const adaptive = await toResult(adaptiveOut, adaptiveDriver.stats().llmCalls) - - rows.push({ goal: liveGoal, fetched: withDup.length, single, full, adaptive }) - console.log( - `[LIVE ADAPTIVE ${JSON.stringify(liveGoal)}] fetched=${withDup.length} | ` + - `single: admitted=${single.admitted} $0 | ` + - `full-LLM: admitted=${full.admitted} calls=${full.llmCalls} $${full.usd.toFixed(4)} tok=${full.tokens} | ` + - `adaptive: admitted=${adaptive.admitted} llmCalls=${adaptive.llmCalls} $${adaptive.usd.toFixed(4)} tok=${adaptive.tokens} ` + - `(dedup=${adaptiveDriver.stats().dedupRejected} hKeep=${adaptiveDriver.stats().heuristicKept} hDrop=${adaptiveDriver.stats().heuristicDropped})`, - ) - } - - expect(anyFetched).toBe(true) - - // FRONTIER SUMMARY over all topics. - const sum = (pick: (r: (typeof rows)[number]) => number) => - rows.reduce((a, r) => a + pick(r), 0) - const fullUsd = sum((r) => r.full.usd) - const adaptiveUsd = sum((r) => r.adaptive.usd) - const fullCalls = sum((r) => r.full.llmCalls) - const adaptiveCalls = sum((r) => r.adaptive.llmCalls) - const singleAdmitted = sum((r) => r.single.admitted) - const fullAdmitted = sum((r) => r.full.admitted) - const adaptiveAdmitted = sum((r) => r.adaptive.admitted) - const callSaving = fullCalls > 0 ? 1 - adaptiveCalls / fullCalls : 0 - const usdSaving = fullUsd > 0 ? 1 - adaptiveUsd / fullUsd : 0 - - console.log( - `[LIVE ADAPTIVE SUMMARY] admitted single=${singleAdmitted} full=${fullAdmitted} adaptive=${adaptiveAdmitted} | ` + - `LLM calls full=${fullCalls} adaptive=${adaptiveCalls} (${(callSaving * 100).toFixed(0)}% fewer) | ` + - `USD full=$${fullUsd.toFixed(4)} adaptive=$${adaptiveUsd.toFixed(4)} (${(usdSaving * 100).toFixed(0)}% cheaper)`, - ) - - // The adaptive driver must NEVER spend MORE than the full-LLM arm — it is a - // strict subset of full-LLM's calls (the ambiguous tail) plus $0 stages. That - // is the one hard invariant; the magnitude of the saving and where adaptive - // lands on the cleanliness frontier are reported honestly in the doc. - expect(adaptiveCalls).toBeLessThanOrEqual(fullCalls) - expect(adaptiveUsd).toBeLessThanOrEqual(fullUsd + 1e-9) - // Adaptive cleanliness sits between single (admits all) and full-LLM (admits - // least) — it cannot admit MORE than the single-agent floor. - expect(adaptiveAdmitted).toBeLessThanOrEqual(singleAdmitted) - }, 600_000) -}) - -function diffUsage( - a: ReturnType, - b: ReturnType, -): ReturnType { - return { - chatCalls: b.chatCalls - a.chatCalls, - searchCalls: b.searchCalls - a.searchCalls, - promptTokens: b.promptTokens - a.promptTokens, - completionTokens: b.completionTokens - a.completionTokens, - usd: b.usd - a.usd, - wallMs: b.wallMs - a.wallMs, - } -} diff --git a/tests/loops/claim-grounding-ab.test.ts b/tests/loops/claim-grounding-ab.test.ts deleted file mode 100644 index e52262e..0000000 --- a/tests/loops/claim-grounding-ab.test.ts +++ /dev/null @@ -1,523 +0,0 @@ -import { mkdtemp, rm } from 'node:fs/promises' -import { tmpdir } from 'node:os' -import { join } from 'node:path' -import { afterEach, beforeEach, describe, expect, it } from 'vitest' -import { createClaimGroundingVerifier, withCitedClaim } from '../../src/claim-grounding' -import { - buildEvalKnowledgeBundle, - defineReadinessSpec, - type KnowledgeReadinessSpec, -} from '../../src/eval-readiness' -import { buildKnowledgeIndex } from '../../src/indexer' -import { - type ResearchContribution, - type ResearchDriver, - type ResearchSourceProposal, - type ResearchWorker, - runVerifiedResearchLoop, - type WorkerResearchContext, -} from '../../src/verified-research-loop' -import { - createTangleRouterClient, - createVerifyingResearchDriver, - createWebResearchWorker, - type RouterClient, -} from '../../src/web-research-worker' - -// =========================================================================== -// CLAIM-GROUNDING A/B: does checking each citation's CLAIM against the fetched -// page text catch an error band the relevance verifier and de-dup CANNOT — and -// does it do so for MORE quality per dollar than the relevance verifier earns -// on the de-dup-dominated topic set (docs/results/cost-quality.md)? -// -// The cost/quality result showed the relevance verifier's cleanliness win is -// dominated by DE-DUPLICATION, which a deterministic content-hash captures at -// ~none of the LLM premium. So the open question is: is there a band where the -// verifier earns its premium — an error a hash and a relevance judge both miss? -// -// MISATTRIBUTION is that band: a source that is on-topic, unique, and real, but -// whose cited CLAIM does not appear in the page (a fabricated / mis-cited fact). -// - de-dup: passes it (it's unique). -// - relevance judge: passes it (it's on-topic). -// - claim-grounding: REJECTS it (the claim is absent from the fetched text). -// -// The grounding check is DETERMINISTIC text presence over `htmlToText` output — -// executable ground truth, $0 inference — so its value-per-dollar denominator is -// near-zero, which is the whole point: it catches what the expensive judge can't -// for what the cheap rule can't. -// =========================================================================== - -interface PoolEntry { - uri: string - title: string - text: string - /** The claim the worker will cite this source for. */ - claim: string - /** True when `claim` is NOT present in `text` — a planted misattribution. */ - misattributed: boolean -} - -const goal = 'self-speculative decoding' - -/** - * A controlled pool: each source's TEXT is what a real fetch would return; the - * CLAIM is what a citation asserts. Two sources are correctly grounded; two are - * misattributed — on-topic, unique, real text, but the cited claim never appears - * in the page. A relevance judge would keep all four; de-dup would keep all four - * (distinct URLs); only claim-grounding rejects the misattributed two. - */ -const pool: PoolEntry[] = [ - { - uri: 'https://arxiv.org/abs/self-spec-decoding', - title: 'Draft & Verify: Lossless LLM Acceleration via Self-Speculative Decoding', - text: - 'Self-speculative decoding drafts tokens by skipping intermediate layers, then verifies them ' + - 'with the full model. It reports a 1.73x speedup on LLaMA-2 with no quality loss.', - claim: 'self-speculative decoding reports a 1.73x speedup on LLaMA-2 with no quality loss', - misattributed: false, - }, - { - uri: 'https://example.org/layer-skip-explainer', - title: 'How layer skipping accelerates decoding', - text: - 'By skipping intermediate transformer layers during the draft stage, the same model produces ' + - 'candidate tokens cheaply, which the full forward pass then verifies in parallel.', - claim: 'skipping intermediate layers lets the same model produce candidate tokens cheaply', - misattributed: false, - }, - { - // On-topic, real text, UNIQUE url — but the cited claim (a 5x speedup on a - // separate draft network) appears NOWHERE in the page. Misattribution. - uri: 'https://blog.example.com/spec-decoding-overview', - title: 'A short overview of speculative decoding', - text: - 'Speculative decoding uses a small draft model to propose tokens that a larger target model ' + - 'verifies. Self-speculative variants reuse a single model instead of a separate draft model.', - claim: 'achieves a 5x speedup using a separately trained draft transformer network', - misattributed: true, - }, - { - // On-topic, real text, UNIQUE url — cited claim invents a benchmark/number - // the page never states. Misattribution. - uri: 'https://news.example.com/decoding-roundup', - title: 'Decoding methods roundup', - text: - 'This roundup compares several inference-time decoding strategies and notes that speculative ' + - 'approaches trade extra compute for lower latency on autoregressive generation.', - claim: 'measured a 4.8x speedup on the GPT-4 MT-bench benchmark with zero accuracy drop', - misattributed: true, - }, -] - -const specs: KnowledgeReadinessSpec[] = [ - defineReadinessSpec({ - id: 'topic/definition', - description: `what ${goal} is and how it works`, - query: `${goal} how it works method`, - requiredFor: ['ResearchAgent'], - importance: 'blocking', - minSources: 1, - minHits: 1, - }), -] - -/** A worker that proposes the whole pool, each source carrying its cited claim. */ -function poolWorker(onPass: () => void): ResearchWorker { - return async (_ctx: WorkerResearchContext): Promise => { - onPass() - const sources: ResearchSourceProposal[] = pool.map((entry) => - withCitedClaim( - { - uri: entry.uri, - title: entry.title, - text: entry.text, - metadata: { planted_misattributed: entry.misattributed }, - }, - entry.claim, - ), - ) - return { - sources, - buildPages: (accepted) => - accepted - .map((record) => { - const original = record.metadata?.originalUri - const entry = pool.find((p) => p.uri === original) - const slug = String(original ?? record.id).replace(/[^a-z0-9]+/gi, '-') - return [ - `---FILE: knowledge/${slug}.md---`, - '---', - `title: ${entry?.title ?? record.id}`, - `sources: ["${record.id}"]`, - '---', - `# ${entry?.title ?? record.id}`, - entry?.text ?? '', - '---END FILE---', - ].join('\n') - }) - .join('\n'), - notes: `proposed ${sources.length} sources (2 grounded, 2 misattributed)`, - } - } -} - -/** A no-op verifier: accepts everything (the "no verifier" arm). */ -const acceptAllDriver: ResearchDriver = { verifySource: () => ({ accept: true }) } - -/** A relevance-only verifier stand-in: every pool source IS on-topic, so accept all. */ -const relevanceOnlyDriver: ResearchDriver = { - verifySource: (source: ResearchSourceProposal) => { - // Everything in the pool mentions decoding/tokens/model → on-topic → accept. - // This is exactly why relevance can't catch misattribution: the page IS - // relevant; only its cited claim is wrong. - const onTopic = /decod|token|model|layer/i.test(source.text) - return onTopic ? { accept: true } : { accept: false, reason: 'off-topic' } - }, -} - -/** Count how many planted-misattributed sources reached the KB. */ -async function misattributedAdmitted(root: string): Promise { - const index = await buildKnowledgeIndex(root) - return index.sources.filter((s) => s.metadata?.planted_misattributed === true).length -} - -async function admittedCount(root: string): Promise { - const index = await buildKnowledgeIndex(root) - return index.sources.length -} - -let groundRoot: string -let relevanceRoot: string -let noneRoot: string - -beforeEach(async () => { - groundRoot = await mkdtemp(join(tmpdir(), 'cg-ground-')) - relevanceRoot = await mkdtemp(join(tmpdir(), 'cg-relevance-')) - noneRoot = await mkdtemp(join(tmpdir(), 'cg-none-')) -}) -afterEach(async () => { - await rm(groundRoot, { recursive: true, force: true }) - await rm(relevanceRoot, { recursive: true, force: true }) - await rm(noneRoot, { recursive: true, force: true }) -}) - -describe('claim-grounding A/B (offline, controlled): catches misattribution dedup+relevance miss', () => { - it('only the claim-grounding verifier rejects the planted misattributions', async () => { - let groundPasses = 0 - await Promise.all([ - runVerifiedResearchLoop({ - root: groundRoot, - goal, - worker: poolWorker(() => { - groundPasses += 1 - }), - driver: { verifySource: createClaimGroundingVerifier() }, - readinessSpecs: specs, - maxRounds: 1, - }), - runVerifiedResearchLoop({ - root: relevanceRoot, - goal, - worker: poolWorker(() => {}), - driver: relevanceOnlyDriver, - readinessSpecs: specs, - maxRounds: 1, - }), - runVerifiedResearchLoop({ - root: noneRoot, - goal, - worker: poolWorker(() => {}), - driver: acceptAllDriver, - readinessSpecs: specs, - maxRounds: 1, - }), - ]) - - const [groundMis, relevanceMis, noneMis, groundAdmitted] = await Promise.all([ - misattributedAdmitted(groundRoot), - misattributedAdmitted(relevanceRoot), - misattributedAdmitted(noneRoot), - admittedCount(groundRoot), - ]) - - console.log( - `[claim-grounding A/B offline] misattributed admitted — ` + - `grounding=${groundMis} relevance=${relevanceMis} no-verifier=${noneMis} ` + - `(grounding kept ${groundAdmitted}/${pool.length} sources)`, - ) - - // The band: relevance + no-verifier let BOTH misattributions through (they - // are on-topic + unique); only claim-grounding rejects them. - expect(groundMis).toBe(0) - expect(relevanceMis).toBe(2) - expect(noneMis).toBe(2) - // Grounding still keeps the two correctly-grounded sources (no over-rejection). - expect(groundAdmitted).toBe(2) - expect(groundPasses).toBe(1) - }) -}) - -// =========================================================================== -// LIVE claim-grounding A/B — the real evidence. Skipped offline (no creds). -// -// Runs a REAL web-research worker (glm-5.2 query-gen → live /v1/search → -// politeFetch → htmlToText) on a topic set, then INJECTS a controlled fraction -// of MISATTRIBUTED citations (real fetched page + a deliberately-wrong claim) so -// there is a measurable correctable band. Three verifier arms gate the SAME -// proposals on the SAME topics: -// -// A. no-verifier — accept all (the floor). -// B. relevance (LLM) — the shipped createVerifyingResearchDriver: 1 LLM call -// per source, judges on-topic relevance. -// C. claim-grounding — deterministic groundClaimInText over the fetched text: -// $0 inference, rejects misattributions. -// -// Using #36's RouterClient.usage() we diff each arm's tokens/$/latency/calls and -// report VALUE PER DOLLAR = misattributions-caught ÷ marginal-USD. Arm C's -// denominator is ~$0, so if it catches the injected misattributions it dominates -// arm B on this band — the cost the relevance judge could not earn on the -// de-dup-dominated set. -// -// Gate: AGENT_KNOWLEDGE_LIVE=1 + TANGLE_API_KEY with glm-5.2 credits. -// CLAIM_GROUNDING_LIVE_GOALS — `|`-separated topics -// CLAIM_GROUNDING_LIVE_MODEL — router chat model (default glm-5.2) -// =========================================================================== - -interface ArmCost { - chatCalls: number - tokens: number - usd: number - wallMs: number -} - -function diffCost( - a: ReturnType, - b: ReturnType, -): ArmCost { - return { - chatCalls: b.chatCalls - a.chatCalls, - tokens: b.promptTokens + b.completionTokens - a.promptTokens - a.completionTokens, - usd: b.usd - a.usd, - wallMs: b.wallMs - a.wallMs, - } -} - -/** - * Take the sources a real worker fetched and (1) attach a GROUNDED claim built - * from the page's own text, then (2) corrupt every `corruptEvery`-th source into - * a MISATTRIBUTION by replacing its claim with one whose specific words are not - * in the page. Returns the annotated sources plus the count corrupted, so the - * arms share an identical proposal set with a known misattribution band. - */ -function plantMisattributions( - sources: ResearchSourceProposal[], - corruptEvery: number, -): { annotated: ResearchSourceProposal[]; planted: number } { - let planted = 0 - const fabricated = [ - 'this page reports a 7.3x speedup on the FluxBench-9000 benchmark with zero accuracy loss', - 'the authors trained a separate 12-billion-parameter draft transformer on proprietary data', - 'results show a 94.2 BLEU score on the never-released Zephyr-XL evaluation suite', - ] - const annotated = sources.map((source, i) => { - if (sources.length >= 2 && i % corruptEvery === corruptEvery - 1) { - const claim = fabricated[planted % fabricated.length] ?? fabricated[0] - planted += 1 - return withCitedClaim( - { ...source, metadata: { ...source.metadata, planted_misattributed: true } }, - claim, - ) - } - // GROUNDED claim: the first sentence of the page's own text (verbatim-present - // by construction), so a correct citation grounds. - const firstSentence = source.text.split(/(?<=[.!?])\s/)[0]?.trim() || source.text.slice(0, 160) - return withCitedClaim( - { ...source, metadata: { ...source.metadata, planted_misattributed: false } }, - firstSentence, - ) - }) - return { annotated, planted } -} - -describe.skipIf(!process.env.AGENT_KNOWLEDGE_LIVE)('live: claim-grounding A/B per dollar', () => { - it('three verifier arms on real web sources with a planted misattribution band', async () => { - const goals = ( - process.env.CLAIM_GROUNDING_LIVE_GOALS ?? - 'self-speculative decoding|rotary position embeddings|grouped-query attention' - ) - .split('|') - .map((g) => g.trim()) - .filter(Boolean) - const model = process.env.CLAIM_GROUNDING_LIVE_MODEL ?? 'glm-5.2' - const router: RouterClient = createTangleRouterClient({ model }) - - // COST GATE: cheap glm-5.2 smoke BEFORE the multi-topic burn. - const smoke = await router.chat( - [ - { role: 'system', content: 'Reply with exactly the word: OK' }, - { role: 'user', content: 'Say OK.' }, - ], - 1200, - ) - console.log(`[LIVE smoke] glm-5.2 visible content length=${smoke.trim().length}`) - expect(smoke.trim().length).toBeGreaterThan(0) - - const worker = createWebResearchWorker({ - router, - resultsPerQuery: 3, - queriesPerGap: 1, - maxSourcesPerRound: 6, - }) - const relevanceDriver = createVerifyingResearchDriver({ router }) - const groundingDriver: ResearchDriver = { verifySource: createClaimGroundingVerifier() } - - let totalPlanted = 0 - let anyFetched = false - const rows: { - goal: string - planted: number - caught: Record<'none' | 'relevance' | 'grounding', number> - cost: Record<'none' | 'relevance' | 'grounding', ArmCost> - }[] = [] - - for (const liveGoal of goals) { - const liveSpecs: KnowledgeReadinessSpec[] = [ - defineReadinessSpec({ - id: 'topic/definition', - description: `what ${liveGoal} is and how it works`, - query: `${liveGoal} how it works method`, - requiredFor: ['ResearchAgent'], - importance: 'blocking', - minSources: 1, - minHits: 1, - }), - ] - // 1. REAL fetch ONCE per topic, shared by all three arms. - const probeRoot = await mkdtemp(join(tmpdir(), 'cg-live-probe-')) - let annotated: ResearchSourceProposal[] = [] - let planted = 0 - try { - const index = await buildKnowledgeIndex(probeRoot) - const readiness = buildEvalKnowledgeBundle({ taskId: liveGoal, index, specs: [] }) - const contribution = await worker({ - root: probeRoot, - goal: liveGoal, - round: 1, - index, - gaps: liveSpecs.map((s) => ({ - id: s.id, - description: s.description, - query: typeof s.metadata?.query === 'string' ? s.metadata.query : s.description, - blocking: true, - })), - readiness, - }) - const fetched = contribution.sources ?? [] - if (fetched.length > 0) anyFetched = true - const result = plantMisattributions(fetched, 2) - annotated = result.annotated - planted = result.planted - totalPlanted += planted - } finally { - await rm(probeRoot, { recursive: true, force: true }) - } - - // 2. Run each arm's verifier over the SAME annotated proposals, diffing cost. - const staticWorker: ResearchWorker = async () => ({ - sources: annotated, - buildPages: (accepted) => - accepted.length === 0 - ? undefined - : accepted - .map((record) => { - const original = record.metadata?.originalUri - const src = annotated.find((s) => s.uri === original) - const slug = String(original ?? record.id) - .replace(/[^a-z0-9]+/gi, '-') - .slice(0, 120) - return [ - `---FILE: knowledge/${slug}.md---`, - '---', - `title: ${src?.title ?? record.id}`, - `sources: ["${record.id}"]`, - '---', - `# ${src?.title ?? record.id}`, - src?.text ?? '', - '---END FILE---', - ].join('\n') - }) - .join('\n'), - }) - - const caught: Record<'none' | 'relevance' | 'grounding', number> = { - none: 0, - relevance: 0, - grounding: 0, - } - const cost: Record<'none' | 'relevance' | 'grounding', ArmCost> = { - none: { chatCalls: 0, tokens: 0, usd: 0, wallMs: 0 }, - relevance: { chatCalls: 0, tokens: 0, usd: 0, wallMs: 0 }, - grounding: { chatCalls: 0, tokens: 0, usd: 0, wallMs: 0 }, - } - for (const arm of ['none', 'relevance', 'grounding'] as const) { - const driver = - arm === 'none' ? acceptAllDriver : arm === 'relevance' ? relevanceDriver : groundingDriver - const root = await mkdtemp(join(tmpdir(), `cg-live-${arm}-`)) - try { - const u0 = router.usage() - await runVerifiedResearchLoop({ - root, - goal: liveGoal, - worker: staticWorker, - driver, - readinessSpecs: liveSpecs, - maxRounds: 1, - }) - cost[arm] = diffCost(u0, router.usage()) - const admittedMis = await misattributedAdmitted(root) - // caught = planted − admitted misattributions. - caught[arm] = planted - admittedMis - } finally { - await rm(root, { recursive: true, force: true }) - } - } - - rows.push({ goal: liveGoal, planted, caught, cost }) - console.log( - `[LIVE CG ${JSON.stringify(liveGoal)}] planted=${planted} ` + - `caught none=${caught.none} relevance=${caught.relevance} grounding=${caught.grounding} | ` + - `$ none=${cost.none.usd.toFixed(4)} relevance=${cost.relevance.usd.toFixed(4)} grounding=${cost.grounding.usd.toFixed(4)} | ` + - `calls relevance=${cost.relevance.chatCalls} grounding=${cost.grounding.chatCalls}`, - ) - } - - expect(anyFetched).toBe(true) - - // VALUE PER DOLLAR over all topics: misattributions caught ÷ marginal USD. - const totalCaught = (arm: 'none' | 'relevance' | 'grounding') => - rows.reduce((acc, r) => acc + r.caught[arm], 0) - const totalUsd = (arm: 'none' | 'relevance' | 'grounding') => - rows.reduce((acc, r) => acc + r.cost[arm].usd, 0) - const relevanceCaught = totalCaught('relevance') - const groundingCaught = totalCaught('grounding') - const relevanceUsd = totalUsd('relevance') - const groundingUsd = totalUsd('grounding') - const perDollar = (caughtN: number, usd: number) => - usd > 0 ? caughtN / usd : caughtN > 0 ? Number.POSITIVE_INFINITY : 0 - - console.log( - `[LIVE CG SUMMARY] planted=${totalPlanted} ` + - `caught relevance=${relevanceCaught} grounding=${groundingCaught} | ` + - `$ relevance=${relevanceUsd.toFixed(4)} grounding=${groundingUsd.toFixed(4)} | ` + - `per-$ relevance=${perDollar(relevanceCaught, relevanceUsd).toFixed(1)} ` + - `grounding=${perDollar(groundingCaught, groundingUsd).toFixed(1)}`, - ) - - // The claim-grounding arm should catch at least as many misattributions as - // the relevance arm (it is the band relevance cannot see) at strictly lower - // marginal cost (deterministic, $0 inference). This is the value-per-dollar - // claim the doc reports; assert the DIRECTION here, the magnitude in the doc. - expect(groundingCaught).toBeGreaterThanOrEqual(relevanceCaught) - expect(groundingUsd).toBeLessThanOrEqual(relevanceUsd) - }, 600_000) -}) diff --git a/tests/loops/held-out-exam.ts b/tests/loops/held-out-exam.ts deleted file mode 100644 index 2b91320..0000000 --- a/tests/loops/held-out-exam.ts +++ /dev/null @@ -1,660 +0,0 @@ -/** - * HELD-OUT DEEP-QUESTION EXAM for the research-quality A/B. - * - * The point of this file is the FIREWALL: these questions and their expected - * answers are NEVER shown to any research loop. A loop is given only the topic - * name + the readiness specs (the same generic "definition" / "results" gaps - * every arm gets). It researches blind. AFTER it finishes, we grade the - * knowledge base it built against THESE questions — questions it never saw — so - * a high score is research QUALITY (it pursued the depth that happens to answer - * the exam) and not teaching-to-the-test. - * - * Each question is a DEPTH question by construction: a single web search for the - * topic name does not surface the answer. They are comparative ("how does X's - * tradeoff differ from Y's?"), mechanism-level ("under what precise condition - * does X fail / what is the exact quantity?"), or contradiction-aware ("which of - * two competing claims holds?"). Surface facts a one-shot search returns are - * deliberately excluded — they would not discriminate the arms. - * - * Grading is deterministic where possible: each question carries `expected` - * answer fragments (the specific number / name / mechanism phrase). A KB - * ANSWERS a question when its full curated text contains a sufficient set of the - * question's `expected` keys (see `gradeQuestionAgainstText`). `anyOf` groups - * model synonyms (e.g. "memory bandwidth" / "bandwidth-bound") so a faithful - * page in different words still grades as answered. The grader is a $0, - * model-free text check — it never calls an LLM, so it cannot leak the exam into - * a judge the loop could see, and it is reproducible. - */ - -/** One held-out deep question with a checkable expected answer. */ -export interface ExamQuestion { - /** Stable id, `topic/qN`. */ - id: string - /** Why this is a DEPTH question (not a surface fact). For the doc/audit. */ - kind: 'comparative' | 'mechanism' | 'contradiction' - /** The question text — NEVER shown to a loop. */ - question: string - /** - * The checkable answer, as required keyword GROUPS. The KB text must contain - * at least `minGroups` of these groups (default: all). A group is satisfied - * when ANY of its `anyOf` fragments appears (case-insensitive substring), - * which lets a faithful page phrase the fact in its own words. Fragments are - * the specific load-bearing tokens — a number, a name, a mechanism phrase. - */ - expected: ExpectedGroup[] - /** - * Minimum number of `expected` groups the KB must contain to count the - * question ANSWERED. Default = all groups (the strict bar). Lowered only when - * a question's answer is genuinely satisfiable by a subset (documented inline). - */ - minGroups?: number -} - -/** A required answer component: satisfied when any synonym fragment is present. */ -export interface ExpectedGroup { - /** Human label for the component (for the doc/audit). */ - label: string - /** Case-insensitive substring fragments; any one present satisfies the group. */ - anyOf: string[] -} - -/** A topic + its held-out questions. */ -export interface ExamTopic { - /** The topic name — THIS is what the loop is told to research. */ - topic: string - /** The held-out deep questions for the topic. */ - questions: ExamQuestion[] -} - -/** - * The exam. 5 ML topics, 4-6 deep questions each. The topics are chosen to have - * a rich, contested, mechanism-heavy literature (so depth-driving can pay off) - * and verifiable specific answers (so grading is deterministic). - */ -export const heldOutExam: ExamTopic[] = [ - { - topic: 'speculative decoding for large language models', - questions: [ - { - id: 'specdec/q1', - kind: 'mechanism', - question: - 'Speculative decoding accepts draft tokens via a specific sampling rule that preserves the target distribution exactly. What is that acceptance/rejection mechanism, and why is the output distribution unchanged?', - expected: [ - { - label: 'rejection-sampling mechanism', - anyOf: ['rejection sampling', 'accept', 'acceptance'], - }, - { - label: 'distribution preserved', - anyOf: [ - 'same distribution', - 'identical distribution', - 'preserves the', - 'unchanged', - 'lossless', - 'no quality loss', - 'target distribution', - ], - }, - ], - }, - { - id: 'specdec/q2', - kind: 'comparative', - question: - 'How does self-speculative / Medusa-style drafting (no separate draft model) differ in its tradeoff from classic two-model speculative decoding (separate small draft model)?', - expected: [ - { - label: 'separate draft model', - anyOf: ['draft model', 'smaller model', 'separate model'], - }, - { - label: 'no-separate-model variant', - anyOf: [ - 'self-speculative', - 'medusa', - 'skipping layers', - 'extra heads', - 'multiple heads', - 'no separate', - 'single model', - ], - }, - ], - // Either side of the comparison being present (with the topic context) - // shows the KB engaged the no-separate-model branch, the deep part. - minGroups: 2, - }, - { - id: 'specdec/q3', - kind: 'mechanism', - question: - 'What determines the maximum achievable speedup of speculative decoding, and why is it bounded rather than unlimited?', - expected: [ - { - label: 'acceptance rate / accepted length', - anyOf: [ - 'acceptance rate', - 'accepted tokens', - 'acceptance length', - 'accept rate', - 'number of accepted', - ], - }, - { - label: 'bounded / memory-bound', - anyOf: [ - 'memory bandwidth', - 'memory-bound', - 'bounded', - 'limited by', - 'verification cost', - 'overhead', - ], - }, - ], - }, - { - id: 'specdec/q4', - kind: 'mechanism', - question: - 'In the classic two-model scheme, multiple draft tokens are verified in a single forward pass of the target model. Why is that one pass not much more expensive than decoding a single token?', - expected: [ - { - label: 'single forward pass verifies many', - anyOf: [ - 'single forward pass', - 'one forward pass', - 'parallel', - 'in parallel', - 'verified in parallel', - ], - }, - { - label: 'compute vs memory-bound', - anyOf: [ - 'memory bandwidth', - 'memory-bound', - 'underutilized', - 'compute', - 'not compute-bound', - 'bandwidth', - ], - }, - ], - minGroups: 1, - }, - ], - }, - { - topic: 'low-rank adaptation LoRA for fine-tuning language models', - questions: [ - { - id: 'lora/q1', - kind: 'mechanism', - question: - 'LoRA freezes the pretrained weights and trains a low-rank update. Mathematically, what is the update applied to a weight matrix W, and which matrices are trained?', - expected: [ - { - label: 'low-rank decomposition BA', - anyOf: ['BA', 'B A', 'A and B', 'two matrices', 'low-rank', 'rank decomposition'], - }, - { - label: 'frozen W plus delta', - anyOf: ['frozen', 'freeze', 'W + ', 'W0', 'pretrained weights', 'does not update'], - }, - ], - }, - { - id: 'lora/q2', - kind: 'comparative', - question: - 'What is the central tradeoff of QLoRA versus plain LoRA — what does QLoRA add to fit larger models on one GPU, and what is the cost?', - expected: [ - { label: '4-bit quantization', anyOf: ['4-bit', '4 bit', 'nf4', 'quantiz', 'int4'] }, - { - label: 'memory saving / single GPU', - anyOf: ['single gpu', 'one gpu', 'memory', 'fit', '48gb', '65b'], - }, - ], - minGroups: 1, - }, - { - id: 'lora/q3', - kind: 'mechanism', - question: - 'A key practical advantage of LoRA at inference time, versus adapters, is that it adds NO inference latency. Why — what can be done with the trained low-rank matrices before deployment?', - expected: [ - { - label: 'merge into weights', - anyOf: [ - 'merge', - 'merged', - 'fold', - 'absorb', - 'add to the weights', - 'no additional latency', - 'no inference latency', - 'zero latency', - ], - }, - ], - }, - { - id: 'lora/q4', - kind: 'contradiction', - question: - 'The LoRA paper makes a specific claim about the intrinsic RANK of the weight update needed for good adaptation. What does it claim about how large the rank r needs to be, and is more always better?', - expected: [ - { - label: 'very low rank suffices', - anyOf: [ - 'low rank', - 'rank of one', - 'rank 1', - 'rank 2', - 'rank 4', - 'small r', - 'r = 1', - 'r=1', - 'intrinsic rank', - 'low intrinsic', - ], - }, - { - label: 'higher rank not always better', - anyOf: [ - 'not always', - 'does not help', - 'no improvement', - 'diminishing', - 'surprisingly', - 'as good as', - ], - }, - ], - minGroups: 1, - }, - ], - }, - { - topic: 'grouped-query attention and multi-query attention in transformers', - questions: [ - { - id: 'gqa/q1', - kind: 'comparative', - question: - 'Multi-query attention (MQA), grouped-query attention (GQA), and multi-head attention (MHA) sit on a spectrum. How do they differ in how many KEY/VALUE heads they use, and what does GQA trade off between the two extremes?', - expected: [ - { - label: 'MQA single KV head', - anyOf: [ - 'single key', - 'single kv', - 'one key-value', - 'one kv', - 'shared key', - 'single head', - ], - }, - { - // "grouped"/"groups" deliberately EXCLUDED — they echo the topic - // name and a shallow snippet would satisfy them for free. The deep - // signal is engaging the SPECTRUM (an intermediate count of KV heads - // between MQA's one and MHA's all), not repeating the term. - label: 'GQA groups of heads (the spectrum)', - anyOf: [ - 'group of', - 'subset of kv', - 'intermediate number', - 'g groups', - 'few kv heads', - 'number of key-value heads', - ], - }, - ], - // BOTH sides of the spectrum (MQA single-head AND the grouped middle) - // must appear — a one-line definition has neither. - minGroups: 2, - }, - { - id: 'gqa/q2', - kind: 'mechanism', - question: - 'The primary reason MQA/GQA speed up autoregressive inference is NOT fewer FLOPs. What is the actual bottleneck they relieve?', - expected: [ - { - label: 'KV cache size / memory bandwidth', - anyOf: [ - 'kv cache', - 'key-value cache', - 'memory bandwidth', - 'bandwidth', - 'memory-bound', - 'cache size', - 'loading the', - ], - }, - ], - }, - { - id: 'gqa/q3', - kind: 'mechanism', - question: - 'GQA can be created from an existing multi-head checkpoint cheaply rather than trained from scratch. What is that procedure called and how is it done?', - expected: [ - { - label: 'uptraining from checkpoint', - anyOf: [ - 'uptrain', - 'up-train', - 'converted', - 'mean-pool', - 'meanpool', - 'mean pooling', - 'existing checkpoint', - 'from a checkpoint', - 'small fraction', - ], - }, - ], - }, - { - id: 'gqa/q4', - kind: 'comparative', - question: - 'What quality cost does MQA incur that motivated GQA, and how does GQA recover most of the quality?', - expected: [ - { - label: 'MQA quality degradation/instability', - anyOf: [ - 'quality degrad', - 'degradation', - 'instability', - 'unstable', - 'quality drop', - 'worse quality', - 'training instab', - ], - }, - { - label: 'GQA recovers quality near MHA', - anyOf: [ - 'close to', - 'near multi-head', - 'recovers', - 'most of the quality', - 'comparable quality', - 'quality close', - ], - }, - ], - minGroups: 1, - }, - ], - }, - { - topic: 'reinforcement learning from human feedback RLHF and PPO for language models', - questions: [ - { - id: 'rlhf/q1', - kind: 'mechanism', - question: - 'RLHF adds a KL-divergence penalty to the reward during PPO. What is that penalty measured against, and what failure does it prevent?', - expected: [ - { - label: 'KL from reference/SFT policy', - anyOf: [ - 'kl', - 'kullback', - 'reference policy', - 'reference model', - 'sft policy', - 'initial policy', - 'divergence', - ], - }, - { - label: 'prevents reward hacking / drift', - anyOf: [ - 'reward hacking', - 'reward model exploit', - 'over-optimiz', - 'drift', - 'collapse', - 'gaming', - 'mode collapse', - 'stay close', - ], - }, - ], - minGroups: 1, - }, - { - id: 'rlhf/q2', - kind: 'mechanism', - question: - 'The reward model in RLHF is trained from a specific kind of human label, not absolute scores. What is the labeling format and the loss used to fit it?', - expected: [ - { - // Bare "preference" EXCLUDED — generic alignment vocab a shallow - // snippet ("align with human preferences") matches for free. The - // deep signal is the LABELING FORMAT: pairwise comparisons / a - // chosen-vs-rejected pair, not an absolute score. - label: 'pairwise comparison labeling format', - anyOf: [ - 'pairwise', - 'pair of', - 'comparison', - 'two responses', - 'preferred response', - 'chosen', - 'rejected', - 'a over b', - 'ranking of', - ], - }, - { - label: 'Bradley-Terry / ranking loss', - anyOf: [ - 'bradley-terry', - 'bradley terry', - 'ranking loss', - 'logistic', - 'cross-entropy of preference', - 'preference loss', - ], - }, - ], - minGroups: 1, - }, - { - id: 'rlhf/q3', - kind: 'contradiction', - question: - "DPO claims to achieve RLHF-quality alignment WITHOUT an explicit reward model or RL loop. What is DPO's key insight that lets it skip the reward model and the PPO loop?", - expected: [ - { - label: 'closed-form / reward implicit in policy', - anyOf: [ - 'closed-form', - 'closed form', - 'implicit reward', - 'reward is implicit', - 'without a reward model', - 'no reward model', - 'direct', - 'reparameter', - 'classification loss', - 'simple loss', - ], - }, - ], - }, - { - id: 'rlhf/q4', - kind: 'mechanism', - question: - 'PPO constrains how far each update moves the policy. What is the specific mechanism PPO uses to keep updates small, distinguishing it from vanilla policy gradient?', - expected: [ - { - label: 'clipped surrogate / probability ratio', - anyOf: [ - 'clip', - 'clipped', - 'surrogate', - 'probability ratio', - 'importance ratio', - 'trust region', - ], - }, - ], - }, - ], - }, - { - topic: 'mixture-of-experts MoE layers in large language models', - questions: [ - { - id: 'moe/q1', - kind: 'mechanism', - question: - 'A sparse MoE layer routes each token to only a few experts. What is the routing mechanism called, and what does "top-k" mean for the cost vs capacity tradeoff?', - expected: [ - { - label: 'gating / router top-k', - anyOf: ['top-k', 'top k', 'top-2', 'top-1', 'gating', 'router', 'routing'], - }, - { - label: 'sparse activation / constant FLOPs', - anyOf: [ - 'sparse', - 'only a few', - 'subset of experts', - 'activated', - 'constant compute', - 'fixed compute', - 'few experts', - ], - }, - ], - minGroups: 1, - }, - { - id: 'moe/q2', - kind: 'mechanism', - question: - 'MoE training is plagued by experts being unevenly used. What is this failure called, and what auxiliary mechanism is added to counter it?', - expected: [ - { - label: 'load imbalance / expert collapse', - anyOf: [ - 'load balanc', - 'imbalance', - 'collapse', - 'unevenly', - 'few experts', - 'dead experts', - 'expert utilization', - ], - }, - { - label: 'auxiliary load-balancing loss', - anyOf: [ - 'auxiliary loss', - 'load balancing loss', - 'balancing loss', - 'auxiliary', - 'load-balancing', - ], - }, - ], - minGroups: 1, - }, - { - id: 'moe/q3', - kind: 'comparative', - question: - 'What is the central scaling argument FOR MoE — what does it decouple that dense models cannot?', - expected: [ - { - label: 'params decoupled from compute/FLOPs', - anyOf: [ - 'decouple', - 'more parameters', - 'parameter count', - 'without increasing', - 'constant flops', - 'same compute', - 'fixed compute', - 'parameters without', - ], - }, - ], - }, - { - id: 'moe/q4', - kind: 'mechanism', - question: - 'MoE models cost more than their active-parameter count suggests in deployment. What is the dominant practical cost of MoE at inference that dense models avoid?', - expected: [ - { - label: 'memory to hold all experts', - anyOf: [ - 'memory', - 'all experts', - 'load all', - 'vram', - 'hold all', - 'total parameters in memory', - 'communication', - 'all-to-all', - ], - }, - ], - }, - ], - }, -] - -/** - * Grade ONE question against a knowledge base's full curated text. Returns - * whether the KB answers it plus which expected groups were found. The check is - * a deterministic case-insensitive substring scan — $0, model-free, reproducible - * — so the exam never leaks into a model the loop could observe. - */ -export function gradeQuestionAgainstText( - question: ExamQuestion, - kbText: string, -): { answered: boolean; groupsFound: number; groupsTotal: number; foundLabels: string[] } { - const haystack = kbText.toLowerCase() - const found = question.expected.filter((group) => - group.anyOf.some((fragment) => haystack.includes(fragment.toLowerCase())), - ) - const minGroups = question.minGroups ?? question.expected.length - return { - answered: found.length >= minGroups, - groupsFound: found.length, - groupsTotal: question.expected.length, - foundLabels: found.map((group) => group.label), - } -} - -/** Grade a whole topic's KB text: how many of its held-out questions it answers. */ -export function gradeTopicAgainstText( - topic: ExamTopic, - kbText: string, -): { answered: number; total: number; perQuestion: ReturnType[] } { - const perQuestion = topic.questions.map((question) => gradeQuestionAgainstText(question, kbText)) - return { - answered: perQuestion.filter((result) => result.answered).length, - total: topic.questions.length, - perQuestion, - } -} - -/** Total held-out questions across the exam (the denominator the doc reports). */ -export function totalExamQuestions(exam: ExamTopic[] = heldOutExam): number { - return exam.reduce((sum, topic) => sum + topic.questions.length, 0) -} diff --git a/tests/loops/research-driving-ab.test.ts b/tests/loops/research-driving-ab.test.ts deleted file mode 100644 index db3e6a8..0000000 --- a/tests/loops/research-driving-ab.test.ts +++ /dev/null @@ -1,604 +0,0 @@ -import { mkdtemp, rm } from 'node:fs/promises' -import { tmpdir } from 'node:os' -import { join } from 'node:path' -import { afterEach, beforeEach, describe, expect, it } from 'vitest' -import { - buildEvalKnowledgeBundle, - defineReadinessSpec, - type KnowledgeReadinessSpec, -} from '../../src/eval-readiness' -import { sha256, stableId } from '../../src/ids' -import { buildKnowledgeIndex } from '../../src/indexer' -import { createResearchDrivingDriver } from '../../src/research-driving-driver' -import { - type KnowledgeResearchLoopDecision, - runKnowledgeResearchLoop, -} from '../../src/research-loop' -import { - type ResearchContribution, - type ResearchDriver, - type ResearchSourceProposal, - type ResearchWorker, - runVerifiedResearchLoop, - type WorkerResearchContext, -} from '../../src/verified-research-loop' -import { - createTangleRouterClient, - createVerifyingResearchDriver, - createWebResearchWorker, - type RouterClient, - type RouterUsage, -} from '../../src/web-research-worker' -import { - type ExamTopic, - type ExpectedGroup, - gradeTopicAgainstText, - heldOutExam, - totalExamQuestions, -} from './held-out-exam' - -// =========================================================================== -// THE RESEARCH-QUALITY A/B: does the research-DRIVING loop answer MORE held-out -// deep questions than single-agent collection or the verify/dedup two-agent -// loop, at EQUAL compute? -// -// The prior A/B (research-loop-equal-compute.test.ts) measured CLEANLINESS — -// how few sources the verifier admits. That is the WRONG metric for the driving -// driver, whose whole thesis is the opposite: it pursues DEPTH and VALIDATION, -// not source hygiene. So this file changes the metric to research QUALITY: -// -// QUALITY = held-out deep questions the resulting KB can answer. -// -// THE FIREWALL (the load-bearing design choice). The exam (held-out-exam.ts) is -// NEVER shown to any loop. Every arm is told only the topic name + the SAME -// generic readiness specs (a "definition" gap and a "results" gap). It researches -// blind. AFTER it finishes we grade the KB it built against questions it never -// saw, with a $0 deterministic substring grader (no LLM, so the exam can't leak -// into a model the loop observes). A high score is therefore research quality -// (the arm pursued the depth that happens to answer the exam) — NOT teaching to -// the test. -// -// THE THREE ARMS — identical real web worker, differ ONLY in the driver: -// (A) single-agent collection: the worker alone, no driver. It collects. -// (B) verify/dedup two-agent loop (createVerifyingResearchDriver): a second -// LLM filters each source for relevance / near-duplicates. It cleans. -// (C) research-DRIVING loop (createResearchDrivingDriver): the driver extracts -// claims, tracks independent-source support + contradictions, and steers -// the worker DEEPER each round (comparative / mechanism / gap / -// contradiction sub-questions). It deepens. -// -// EQUAL COMPUTE — counted in agent passes (same unit as the prior A/B): -// - single-agent iter = 1 worker pass. -// - two-agent round = 1 worker pass + 1 driver pass = 2 passes. -// Each arm gets the SAME pass ceiling B; the single-agent arm gets up to B iters -// to spend the budget the two-agent arms burn on their driver. The harness reads -// RouterClient.usage() per arm so the dollars/tokens/calls are MEASURED, not -// assumed — the honest cost half. -// -// HYPOTHESIS: arm C answers MORE held-out deep questions than A or B, because it -// chased depth + corroboration rather than breadth (A) or cleanliness (B). If it -// does NOT, this file says so plainly — a real negative result. -// =========================================================================== - -/** The generic readiness specs every arm gets — the ONLY thing the loop is told. */ -function genericSpecsForGoal(goal: string): KnowledgeReadinessSpec[] { - return [ - defineReadinessSpec({ - id: 'topic/definition', - description: `what ${goal} is and how it works`, - query: `${goal} how it works method`, - requiredFor: ['ResearchAgent'], - importance: 'blocking', - minSources: 1, - minHits: 1, - }), - defineReadinessSpec({ - id: 'topic/results', - description: `reported results, mechanisms, or trade-offs for ${goal}`, - query: `${goal} results trade-offs mechanism`, - requiredFor: ['ResearchAgent'], - importance: 'blocking', - minSources: 1, - minHits: 1, - }), - ] -} - -/** - * The full curated text of a KB, joined for grading. We grade against PAGES - * (what the loop curated + what readiness scores) AND raw source text (so an arm - * whose pages are thin but whose sources are rich still gets credit for what it - * actually fetched). This is the text the held-out grader scans — it is read - * ONLY after the loop finished, never handed to the loop. - */ -async function kbText(root: string): Promise { - const index = await buildKnowledgeIndex(root) - const pageText = index.pages.map((page) => `${page.title}\n${page.text}`).join('\n\n') - const sourceText = index.sources - .map((source) => `${source.title ?? ''}\n${source.text ?? ''}`) - .join('\n\n') - return `${pageText}\n\n${sourceText}` -} - -/** Distinct held-out answer COMPONENTS (expected groups) the KB covers — the depth proxy. */ -function depthComponentsCovered( - topic: ExamTopic, - text: string, -): { covered: number; total: number } { - const haystack = text.toLowerCase() - const allGroups: ExpectedGroup[] = topic.questions.flatMap((question) => question.expected) - const covered = allGroups.filter((group) => - group.anyOf.some((fragment) => haystack.includes(fragment.toLowerCase())), - ).length - return { covered, total: allGroups.length } -} - -/** Per-arm cost diff from the shared router's cumulative usage accumulator. */ -function costDiff(before: RouterUsage, after: RouterUsage) { - return { - chatCalls: after.chatCalls - before.chatCalls, - searchCalls: after.searchCalls - before.searchCalls, - tokens: - after.promptTokens + after.completionTokens - before.promptTokens - before.completionTokens, - usd: after.usd - before.usd, - wallMs: after.wallMs - before.wallMs, - } -} - -/** - * ARM A — single-agent collection. The real worker proposes; the loop applies - * every proposal with NO driver gate. Up to `maxIterations` worker passes. The - * worker is invoked ONCE per pass (sources + buildPages from the same call), so - * arm A's cost is exactly one worker pass per iteration — fair against the - * two-agent arms whose extra cost is the driver, not a duplicate worker call. - */ -async function runSingleAgentArm( - root: string, - goal: string, - maxIterations: number, - worker: ResearchWorker, - specs: KnowledgeReadinessSpec[], -): Promise<{ passes: number }> { - let passes = 0 - await runKnowledgeResearchLoop({ - root, - goal, - maxIterations, - readinessSpecs: specs, - async step(context): Promise { - passes += 1 - const report = context.readiness?.report - if (report && report.blockingMissingRequirements.length === 0) { - return { done: true, notes: 'readiness gate met' } - } - const gaps = (report?.blockingMissingRequirements ?? []).map((req) => ({ - id: req.id, - description: req.description, - query: - typeof req.metadata?.query === 'string' - ? (req.metadata.query as string) - : req.description, - blocking: true, - })) - const index = await buildKnowledgeIndex(root) - // No steer: the single-agent arm has no driver, so it never receives the - // depth steering the two-agent arms fold in. That asymmetry IS the arm. - const contribution = await worker({ - root, - goal, - round: passes, - index, - gaps, - readiness: buildEvalKnowledgeBundle({ taskId: goal, index, specs: [] }), - }) - const proposals = contribution.sources ?? [] - if (proposals.length === 0) return { notes: 'no new proposals' } - // Apply every proposal with the worker's own citing-page builder, NO gate - // (that is the whole point of the single-agent arm). Pages cite the - // precomputed real source id the loop will assign. - const built = contribution.buildPages?.( - proposals.map((p) => ({ - id: predictedSourceId(p), - uri: p.uri, - contentHash: '', - createdAt: new Date().toISOString(), - metadata: { originalUri: p.uri }, - })), - ) - return { sourceTexts: proposals, proposalText: built, notes: `applied ${proposals.length}` } - }, - }) - return { passes } -} - -/** - * ARMS B and C — the two-agent loop with the given driver. Each round is one - * worker pass + one driver pass = 2 passes; capped at `rounds` rounds. The - * worker pass is counted via a thin wrapper so the equal-compute accounting holds. - */ -async function runTwoAgentArm( - root: string, - goal: string, - rounds: number, - worker: ResearchWorker, - driver: ResearchDriver, - specs: KnowledgeReadinessSpec[], -): Promise<{ passes: number }> { - let workerPasses = 0 - const countedWorker: ResearchWorker = async ( - ctx: WorkerResearchContext, - ): Promise => { - workerPasses += 1 - return worker(ctx) - } - await runVerifiedResearchLoop({ - root, - goal, - worker: countedWorker, - driver, - readinessSpecs: specs, - maxRounds: rounds, - }) - return { passes: workerPasses * 2 } -} - -/** The deterministic source-record id `addSourceText` will assign (src/sources.ts). */ -function predictedSourceId(source: ResearchSourceProposal): string { - return stableId('src', `${sha256(source.text)}:${source.uri}`) -} - -interface ArmResult { - passes: number - answered: number - total: number - depthCovered: number - depthTotal: number - cost: ReturnType -} - -/** Run one arm, grade its KB against the held-out exam, diff its cost. */ -async function runAndGrade( - run: (root: string) => Promise<{ passes: number }>, - topic: ExamTopic, - router: RouterClient, -): Promise { - const root = await mkdtemp(join(tmpdir(), 'rq-arm-')) - try { - const before = router.usage() - const { passes } = await run(root) - const cost = costDiff(before, router.usage()) - const text = await kbText(root) - const grade = gradeTopicAgainstText(topic, text) - const depth = depthComponentsCovered(topic, text) - return { - passes, - answered: grade.answered, - total: grade.total, - depthCovered: depth.covered, - depthTotal: depth.total, - cost, - } - } finally { - await rm(root, { recursive: true, force: true }) - } -} - -// --------------------------------------------------------------------------- -// OFFLINE WIRING TEST (no creds): proves the grader + the three-arm harness -// work against a scripted KB, so a live run that returns zeros is a real null, -// not a broken harness. -// --------------------------------------------------------------------------- - -describe('research-quality A/B harness (offline wiring)', () => { - it('the held-out grader answers a question only when the KB text contains the expected answer', () => { - const topic = heldOutExam[0] - if (!topic) throw new Error('exam topic 0 missing') - // A KB whose text contains the rejection-sampling mechanism + lossless claim - // answers specdec/q1; an empty KB answers nothing. - const richText = - 'Speculative decoding uses rejection sampling to accept draft tokens, which preserves the target distribution exactly (lossless, no quality loss). The speedup is bounded by the acceptance rate and is ultimately memory bandwidth limited. A separate small draft model proposes tokens verified in a single forward pass in parallel.' - const rich = gradeTopicAgainstText(topic, richText) - const empty = gradeTopicAgainstText(topic, '') - expect(rich.answered).toBeGreaterThan(0) - expect(empty.answered).toBe(0) - // Depth components: the rich text covers several expected groups; empty zero. - expect(depthComponentsCovered(topic, richText).covered).toBeGreaterThan(0) - expect(depthComponentsCovered(topic, '').covered).toBe(0) - }) - - it('the exam is well-formed: 5 topics, 4-6 deep questions each, every question checkable', () => { - expect(heldOutExam.length).toBe(5) - for (const topic of heldOutExam) { - expect(topic.questions.length).toBeGreaterThanOrEqual(4) - expect(topic.questions.length).toBeLessThanOrEqual(6) - for (const question of topic.questions) { - expect(question.expected.length).toBeGreaterThan(0) - for (const group of question.expected) { - expect(group.anyOf.length).toBeGreaterThan(0) - } - } - } - expect(totalExamQuestions()).toBeGreaterThanOrEqual(20) - }) -}) - -// =========================================================================== -// LIVE 3-ARM A/B — the real evidence. Skipped offline (no creds). Runs the real -// web worker on each held-out topic through all three drivers at equal compute, -// grades each KB against the firewalled exam, and reports per-arm answered / -// depth / cost. Gate: AGENT_KNOWLEDGE_LIVE=1 + a TANGLE_API_KEY with glm-5.2. -// RQ_LIVE_BUDGET — agent-pass ceiling B per arm (default 4) -// RQ_LIVE_MODEL — router chat model (default glm-5.2) -// RQ_LIVE_TOPICS — `|`-separated subset of exam topic names (default: all 5) -// =========================================================================== - -describe.skipIf(!process.env.AGENT_KNOWLEDGE_LIVE)('live: research-quality 3-arm A/B', () => { - it('driving vs verify/dedup vs single-agent — held-out deep questions answered', async () => { - const budgetPasses = Number(process.env.RQ_LIVE_BUDGET ?? 4) - const model = process.env.RQ_LIVE_MODEL ?? 'glm-5.2' - const topicFilter = (process.env.RQ_LIVE_TOPICS ?? '') - .split('|') - .map((t) => t.trim()) - .filter(Boolean) - const topics = topicFilter.length - ? heldOutExam.filter((t) => topicFilter.some((f) => t.topic.includes(f))) - : heldOutExam - expect(topics.length).toBeGreaterThan(0) - - // ONE shared router client for the whole run (web search + chat). usage() is - // cumulative; each arm diffs it. - const router: RouterClient = createTangleRouterClient({ model }) - - // COST GATE: a cheap glm-5.2 smoke BEFORE the multi-arm burn. Proves the key - // works + the reasoning-token floor returns visible content. Fail fast if not. - const smoke = await router.chat( - [ - { role: 'system', content: 'Reply with exactly the word: OK' }, - { role: 'user', content: 'Say OK.' }, - ], - 1200, - ) - console.log(`[RQ smoke] ${model} visible content length=${smoke.trim().length}`) - expect(smoke.trim().length).toBeGreaterThan(0) - - // The SAME real worker for all three arms — only the driver differs. - const worker = createWebResearchWorker({ - router, - resultsPerQuery: 3, - queriesPerGap: 1, - maxSourcesPerRound: 6, - }) - - const perArm: Record<'single' | 'verify' | 'driving', ArmResult[]> = { - single: [], - verify: [], - driving: [], - } - - for (const topic of topics) { - const goal = topic.topic - const specs = genericSpecsForGoal(goal) - - // ARM A — single-agent collection (no driver). - const single = await runAndGrade( - (root) => runSingleAgentArm(root, goal, budgetPasses, worker, specs), - topic, - router, - ) - // ARM B — verify/dedup two-agent loop. A fresh verifying driver per topic. - const verify = await runAndGrade( - (root) => - runTwoAgentArm( - root, - goal, - budgetPasses / 2, - worker, - createVerifyingResearchDriver({ router }), - specs, - ), - topic, - router, - ) - // ARM C — research-DRIVING loop. A fresh driving driver per topic. - const driving = await runAndGrade( - (root) => - runTwoAgentArm( - root, - goal, - budgetPasses / 2, - worker, - createResearchDrivingDriver({ router }), - specs, - ), - topic, - router, - ) - - perArm.single.push(single) - perArm.verify.push(verify) - perArm.driving.push(driving) - - const fmt = (a: ArmResult) => - `ans=${a.answered}/${a.total} depth=${a.depthCovered}/${a.depthTotal} passes=${a.passes} ` + - `calls=${a.cost.chatCalls} tok=${a.cost.tokens} $${a.cost.usd.toFixed(4)} ${a.cost.wallMs}ms` - console.log( - `[RQ ${JSON.stringify(goal)} @ B<=${budgetPasses}]\n` + - ` single : ${fmt(single)}\n` + - ` verify : ${fmt(verify)}\n` + - ` driving: ${fmt(driving)}`, - ) - } - - // Aggregate per arm: total held-out questions answered + total depth + cost. - const sum = (xs: number[]) => xs.reduce((a, b) => a + b, 0) - const agg = (results: ArmResult[]) => ({ - answered: sum(results.map((r) => r.answered)), - total: sum(results.map((r) => r.total)), - depthCovered: sum(results.map((r) => r.depthCovered)), - depthTotal: sum(results.map((r) => r.depthTotal)), - usd: sum(results.map((r) => r.cost.usd)), - calls: sum(results.map((r) => r.cost.chatCalls)), - tokens: sum(results.map((r) => r.cost.tokens)), - }) - const a = agg(perArm.single) - const b = agg(perArm.verify) - const c = agg(perArm.driving) - const examTotal = totalExamQuestions(topics) - - console.log( - `\n[RQ TOTALS over ${topics.length} topics, ${examTotal} held-out questions @ B<=${budgetPasses}]\n` + - ` single-agent : answered ${a.answered}/${a.total} depth ${a.depthCovered}/${a.depthTotal} $${a.usd.toFixed(4)} (${a.calls} calls, ${a.tokens} tok)\n` + - ` verify/dedup : answered ${b.answered}/${b.total} depth ${b.depthCovered}/${b.depthTotal} $${b.usd.toFixed(4)} (${b.calls} calls, ${b.tokens} tok)\n` + - ` DRIVING : answered ${c.answered}/${c.total} depth ${c.depthCovered}/${c.depthTotal} $${c.usd.toFixed(4)} (${c.calls} calls, ${c.tokens} tok)\n` + - ` HYPOTHESIS (driving answers MORE deep questions): ${c.answered > a.answered && c.answered > b.answered ? 'SUPPORTED' : 'NOT SUPPORTED'} ` + - `(driving ${c.answered} vs single ${a.answered} vs verify ${b.answered})`, - ) - - // The live arm is only evidence if at least one arm fetched real pages and - // answered at least one held-out question. All-zero across arms = the worker - // never reached the web — a FALSE null, fail loud. - expect(a.answered + b.answered + c.answered).toBeGreaterThan(0) - - // This is a MEASUREMENT, not a pass/fail gate — the doc reports the honest - // verdict whichever way it falls. We only assert the harness produced a - // real, gradable result for every arm (each ran its passes within budget). - for (const arm of [...perArm.single, ...perArm.verify, ...perArm.driving]) { - expect(arm.passes).toBeLessThanOrEqual(budgetPasses) - } - }, 1_200_000) -}) - -// =========================================================================== -// CONTROLLED MULTI-ROUND PROBE — the driving thesis's FAIREST test. -// -// The main A/B above found driving does NOT win. The autopsy (see the doc) -// showed WHY: the generic readiness gate (one source per spec) is satisfied by -// the FIRST round's fetch, so the loop stops after one round — and the driving -// driver's whole mechanism is multi-round steering (extract claims → demand -// corroboration → re-search). It never gets a round 2 to drive. So the main A/B -// tests "does driving help in one round?" (no, it just costs more), NOT "does -// driving's depth-steering help WHEN it runs?". -// -// This probe isolates the latter. It raises the readiness bar (minSources high) -// so the gate STAYS unmet and the loop runs the full round budget — forcing the -// driving driver to actually steer across rounds. Same real worker, same round -// budget; the ONLY difference is whether the driver steers (driving) or the -// worker re-searches the same gaps blind (single). If driving's steering has -// ANY value, this is where it shows — a KB built over rounds the driver pushed -// deeper should answer more held-out questions than the same rounds run blind. -// If it STILL doesn't win here, the negative result is robust, not a gate -// artifact. Gate: AGENT_KNOWLEDGE_LIVE=1 + RQ_PROBE=1 (it is the priciest run). -// =========================================================================== - -/** Readiness specs that STAY unmet (high minSources) so the loop runs all rounds. */ -function multiRoundSpecsForGoal(goal: string): KnowledgeReadinessSpec[] { - return [ - defineReadinessSpec({ - id: 'topic/definition', - description: `what ${goal} is and how it works`, - query: `${goal} how it works method`, - requiredFor: ['ResearchAgent'], - importance: 'blocking', - // Demand many sources so the gate never closes the loop early — the loop - // runs the full maxRounds and the driving driver gets to steer each round. - minSources: 99, - minHits: 1, - }), - ] -} - -describe.skipIf(!(process.env.AGENT_KNOWLEDGE_LIVE && process.env.RQ_PROBE))( - 'live: controlled multi-round probe (driving steering vs blind re-search)', - () => { - it('forces N rounds so driving actually steers — does it then answer more?', async () => { - const rounds = Number(process.env.RQ_PROBE_ROUNDS ?? 3) - const model = process.env.RQ_LIVE_MODEL ?? 'glm-5.2' - const topicFilter = (process.env.RQ_LIVE_TOPICS ?? '') - .split('|') - .map((t) => t.trim()) - .filter(Boolean) - const topics = topicFilter.length - ? heldOutExam.filter((t) => topicFilter.some((f) => t.topic.includes(f))) - : heldOutExam - const router: RouterClient = createTangleRouterClient({ model }) - - const smoke = await router.chat( - [ - { role: 'system', content: 'Reply with exactly the word: OK' }, - { role: 'user', content: 'Say OK.' }, - ], - 1200, - ) - expect(smoke.trim().length).toBeGreaterThan(0) - - const worker = createWebResearchWorker({ - router, - resultsPerQuery: 3, - queriesPerGap: 1, - maxSourcesPerRound: 6, - }) - const drivingResults: ArmResult[] = [] - const blindResults: ArmResult[] = [] - - for (const topic of topics) { - const goal = topic.topic - const specs = multiRoundSpecsForGoal(goal) - // DRIVING: the driver steers the worker deeper each of `rounds` rounds. - const driving = await runAndGrade( - (root) => - runTwoAgentArm( - root, - goal, - rounds, - worker, - createResearchDrivingDriver({ router }), - specs, - ), - topic, - router, - ) - // BLIND: the SAME worker over the SAME rounds with a no-op driver (accept - // every source, no steer) — it re-searches the same gaps without the - // driver's depth steering. The matched control for "steering vs not". - const blind = await runAndGrade( - (root) => runTwoAgentArm(root, goal, rounds, worker, noopDriver(), specs), - topic, - router, - ) - drivingResults.push(driving) - blindResults.push(blind) - console.log( - `[RQ PROBE ${JSON.stringify(goal)} @ ${rounds} rounds]\n` + - ` driving: ans=${driving.answered}/${driving.total} depth=${driving.depthCovered}/${driving.depthTotal} $${driving.cost.usd.toFixed(4)}\n` + - ` blind : ans=${blind.answered}/${blind.total} depth=${blind.depthCovered}/${blind.depthTotal} $${blind.cost.usd.toFixed(4)}`, - ) - } - - const sum = (xs: number[]) => xs.reduce((a, b) => a + b, 0) - const dAns = sum(drivingResults.map((r) => r.answered)) - const blAns = sum(blindResults.map((r) => r.answered)) - const dUsd = sum(drivingResults.map((r) => r.cost.usd)) - const blUsd = sum(blindResults.map((r) => r.cost.usd)) - console.log( - `\n[RQ PROBE TOTALS over ${topics.length} topics @ ${rounds} rounds]\n` + - ` DRIVING (steered): answered ${dAns} $${dUsd.toFixed(4)}\n` + - ` blind (no steer) : answered ${blAns} $${blUsd.toFixed(4)}\n` + - ` STEERING HELPS: ${dAns > blAns ? 'YES' : 'NO'} (driving ${dAns} vs blind ${blAns})`, - ) - expect(dAns + blAns).toBeGreaterThan(0) - }, 1_800_000) - }, -) - -/** A no-op driver: accept every source, no steer. The blind control for the probe. */ -function noopDriver(): ResearchDriver { - return { verifySource: () => ({ accept: true }) } -} - -let _root: string -beforeEach(async () => { - _root = await mkdtemp(join(tmpdir(), 'rq-')) -}) -afterEach(async () => { - await rm(_root, { recursive: true, force: true }) -}) diff --git a/tests/loops/research-driving-driver.test.ts b/tests/loops/research-driving-driver.test.ts deleted file mode 100644 index cb3f005..0000000 --- a/tests/loops/research-driving-driver.test.ts +++ /dev/null @@ -1,325 +0,0 @@ -import { describe, expect, it } from 'vitest' -import { createResearchDrivingDriver } from '../../src/research-driving-driver' -import type { - ResearchSourceProposal, - SourceVerificationContext, -} from '../../src/verified-research-loop' -import type { RouterClient, RouterUsage } from '../../src/web-research-worker' - -// =========================================================================== -// Unit tests for createResearchDrivingDriver: the DRIVING driver (drives depth + -// validation), the opposite of a dedup/relevance FILTER. We stub RouterClient so -// claim extraction is deterministic and offline (no creds, no network). -// =========================================================================== - -/** A RouterClient whose `chat` returns scripted claim-extraction JSON by uri. */ -function stubRouter(claimsByUriToken: Record): RouterClient { - const usage: RouterUsage = { - chatCalls: 0, - searchCalls: 0, - promptTokens: 0, - completionTokens: 0, - usd: 0, - wallMs: 0, - } - return { - search: async () => [], - chat: async (messages) => { - usage.chatCalls += 1 - const user = messages.find((m) => m.role === 'user')?.content ?? '' - // The extractor prompt embeds the page excerpt; match it to a scripted reply. - for (const [token, reply] of Object.entries(claimsByUriToken)) { - if (user.includes(token)) return reply - } - return '[]' - }, - usage: () => ({ ...usage }), - } -} - -function ctx( - round: number, - overrides: Partial = {}, -): SourceVerificationContext { - return { - root: '/tmp/x', - goal: 'self-speculative decoding', - round, - index: { - root: '/tmp/x', - generatedAt: '', - sources: [], - pages: [], - graph: { nodes: [], edges: [] }, - }, - gaps: [], - acceptedThisRound: [], - ...overrides, - } -} - -function source(uri: string, text: string, title = uri): ResearchSourceProposal { - return { uri, text, title } -} - -const noGap = { id: 'topic/x', description: 'definition', query: 'how it works', blocking: true } - -describe('createResearchDrivingDriver — claim extraction + support tracking', () => { - it('extracts claims via the router and tracks independent-source support by host', async () => { - const router = stubRouter({ - 'PAGE-A': '[{"claim":"layer skipping gives a 1.73x speedup","contradicts":null}]', - 'PAGE-B': '[{"claim":"layer skipping gives a 1.73x speedup","contradicts":null}]', - }) - const driver = createResearchDrivingDriver({ router }) - - // Same claim, two DIFFERENT hosts → corroborated (>= 2 independent sources). - const a = await driver.verifySource(source('https://arxiv.org/a', 'PAGE-A body'), ctx(1)) - const b = await driver.verifySource(source('https://acm.org/b', 'PAGE-B body'), ctx(1)) - expect(a.accept).toBe(true) - expect(b.accept).toBe(true) - - const state = driver.researchState() - expect(state.claims).toHaveLength(1) - expect(state.claims[0]?.supportingHosts.size).toBe(2) - expect(state.corroborated).toHaveLength(1) - expect(state.weaklySupported).toHaveLength(0) - }) - - it('two sources on the SAME host count as ONE independent source (still weak)', async () => { - const router = stubRouter({ - BODY: '[{"claim":"the method reports a 1.73x speedup","contradicts":null}]', - }) - const driver = createResearchDrivingDriver({ router }) - await driver.verifySource(source('https://blog.example.com/p1', 'BODY one'), ctx(1)) - await driver.verifySource(source('https://blog.example.com/p2', 'BODY two'), ctx(1)) - - const state = driver.researchState() - expect(state.claims).toHaveLength(1) - // Same host ⇒ one independent source ⇒ still weakly supported. - expect(state.claims[0]?.supportingHosts.size).toBe(1) - expect(state.weaklySupported).toHaveLength(1) - expect(state.corroborated).toHaveLength(0) - }) - - it('rejects a source with NO extractable claim (cannot drive the research)', async () => { - const router = stubRouter({}) // returns "[]" → no claims, no det. text either - const driver = createResearchDrivingDriver({ router, deterministicFallback: false }) - const verdict = await driver.verifySource(source('https://x.com/empty', ''), ctx(1)) - expect(verdict.accept).toBe(false) - expect(verdict.reason).toMatch(/no extractable claim/) - }) - - it('falls back to deterministic sentence claims when the model is unavailable', async () => { - // Router whose chat throws → LLM path yields nothing → deterministic fallback. - const throwingRouter: RouterClient = { - search: async () => [], - chat: async () => { - throw new Error('router down') - }, - usage: () => ({ - chatCalls: 0, - searchCalls: 0, - promptTokens: 0, - completionTokens: 0, - usd: 0, - wallMs: 0, - }), - } - const driver = createResearchDrivingDriver({ - router: throwingRouter, - deterministicFallback: true, - }) - const text = - 'Self-speculative decoding skips intermediate layers during drafting. ' + - 'It verifies the drafted tokens with the full model in parallel.' - const verdict = await driver.verifySource(source('https://arxiv.org/p', text), ctx(1)) - expect(verdict.accept).toBe(true) - expect(driver.researchState().claims.length).toBeGreaterThanOrEqual(1) - }) -}) - -describe('createResearchDrivingDriver — contradiction detection + contested marking', () => { - it('marks BOTH claims contested when one source contradicts another', async () => { - // First source seeds claim X. Second source contradicts it by id. - let firstClaimId = '' - const router: RouterClient = { - search: async () => [], - chat: async (messages) => { - const user = messages.find((m) => m.role === 'user')?.content ?? '' - if (user.includes('CLAIMS-A')) { - return '[{"claim":"the speedup is 5x on LLaMA-2","contradicts":null}]' - } - if (user.includes('CLAIMS-B')) { - // Contradict the first claim by its ledger id (embedded in the prompt). - const match = user.match(/\[(c_[0-9a-f]+)\]/) - firstClaimId = match?.[1] ?? '' - return `[{"claim":"the speedup is only 2x on LLaMA-2","contradicts":"[${firstClaimId}]"}]` - } - return '[]' - }, - usage: () => ({ - chatCalls: 0, - searchCalls: 0, - promptTokens: 0, - completionTokens: 0, - usd: 0, - wallMs: 0, - }), - } - const driver = createResearchDrivingDriver({ router }) - await driver.verifySource(source('https://a.org/x', 'CLAIMS-A'), ctx(1)) - await driver.verifySource(source('https://b.org/y', 'CLAIMS-B'), ctx(1)) - - const state = driver.researchState() - expect(state.claims).toHaveLength(2) - expect(state.contested).toHaveLength(2) - // A contested claim is NOT a weakly-supported invalidation-by-corroboration - // target — it is settled-as-disputed. - expect(state.weaklySupported).toHaveLength(0) - }) -}) - -describe('createResearchDrivingDriver — foldGaps drives DEPTH not breadth', () => { - it('generates the four deep-question kinds + invalidation challenges in the steer', async () => { - const router = stubRouter({ - 'PAGE-1': - '[{"claim":"self-speculative decoding gives a 1.73x speedup on LLaMA-2","contradicts":null}]', - 'PAGE-2': - '[{"claim":"grouped-query attention reduces the KV cache by 8x","contradicts":null}]', - }) - const driver = createResearchDrivingDriver({ router }) - // Two distinct, weakly-supported claims on distinct hosts. - await driver.verifySource(source('https://arxiv.org/1', 'PAGE-1'), ctx(1)) - await driver.verifySource(source('https://acm.org/2', 'PAGE-2'), ctx(1)) - - const steer = driver.foldGaps([noGap]) - const last = driver.lastSteer() - expect(last).toBeDefined() - - // The steer explicitly tells the worker to go deeper, not add breadth. - expect(steer).toMatch(/Do NOT just add more sources/) - // Deep sub-questions of the named kinds are present. - const kinds = new Set(last?.deepQuestions.map((q) => q.kind)) - expect(kinds.has('gap')).toBe(true) // both claims are weakly supported - expect(kinds.has('mechanism')).toBe(true) - expect(kinds.has('comparative')).toBe(true) // two claims ⇒ a comparison - // Invalidation targets: both weakly-supported claims are demanded to reach - // a second independent source. - expect(last?.invalidationTargets.length).toBe(2) - expect(steer).toMatch(/INDEPENDENT/) - expect(steer).toMatch(/corroborat/i) - }) - - it('generates a CONTRADICTION deep-question when the ledger holds a contradiction', async () => { - const router: RouterClient = { - search: async () => [], - chat: async (messages) => { - const user = messages.find((m) => m.role === 'user')?.content ?? '' - if (user.includes('SEED')) return '[{"claim":"the speedup is 5x","contradicts":null}]' - if (user.includes('REFUTE')) { - const id = user.match(/\[(c_[0-9a-f]+)\]/)?.[1] ?? '' - return `[{"claim":"the speedup is only 2x","contradicts":"[${id}]"}]` - } - return '[]' - }, - usage: () => ({ - chatCalls: 0, - searchCalls: 0, - promptTokens: 0, - completionTokens: 0, - usd: 0, - wallMs: 0, - }), - } - const driver = createResearchDrivingDriver({ router }) - await driver.verifySource(source('https://a.org/x', 'SEED'), ctx(1)) - await driver.verifySource(source('https://b.org/y', 'REFUTE'), ctx(1)) - - driver.foldGaps([]) - const kinds = driver.lastSteer()?.deepQuestions.map((q) => q.kind) ?? [] - expect(kinds).toContain('contradiction') - }) - - it('asks DEEPER questions across rounds: round 2 questions differ from round 1', async () => { - const router = stubRouter({ - ROUND1: '[{"claim":"layer skipping yields a 1.73x speedup","contradicts":null}]', - ROUND2: '[{"claim":"early exit degrades accuracy past layer 12","contradicts":null}]', - }) - const driver = createResearchDrivingDriver({ router }) - - await driver.verifySource(source('https://arxiv.org/r1', 'ROUND1'), ctx(1)) - driver.foldGaps([noGap]) - const round1Questions = new Set(driver.lastSteer()?.deepQuestions.map((q) => q.text)) - - // New evidence (a new claim) lands → round 2 questions are generated over the - // larger ledger and include sub-questions not present in round 1. - await driver.verifySource(source('https://acm.org/r2', 'ROUND2'), ctx(2)) - driver.foldGaps([noGap]) - const round2Questions = driver.lastSteer()?.deepQuestions.map((q) => q.text) ?? [] - - expect(round2Questions.length).toBeGreaterThan(0) - const newInRound2 = round2Questions.filter((q) => !round1Questions.has(q)) - expect(newInRound2.length).toBeGreaterThan(0) - }) -}) - -describe('createResearchDrivingDriver — completion gates on claim support, NOT source count', () => { - it('is NOT complete with one weakly-supported claim (even after a round)', async () => { - const router = stubRouter({ - ONLY: '[{"claim":"the method gives a 1.73x speedup","contradicts":null}]', - }) - const driver = createResearchDrivingDriver({ router }) - await driver.verifySource(source('https://arxiv.org/only', 'ONLY'), ctx(1)) - driver.foldGaps([]) - expect(driver.isComplete()).toBe(false) - expect(driver.researchState().weaklySupported).toHaveLength(1) - }) - - it('is NOT complete with MANY sources of one unchallenged claim if only one host', async () => { - const router = stubRouter({ - SAME: '[{"claim":"the method gives a 1.73x speedup","contradicts":null}]', - }) - const driver = createResearchDrivingDriver({ router }) - // Ten sources, ALL on the same host → still ONE independent source. - for (let i = 0; i < 10; i += 1) { - await driver.verifySource(source(`https://blog.example.com/p${i}`, 'SAME body'), ctx(1)) - } - driver.foldGaps([]) - // Source count is high but independent support is 1 → NOT done. - expect(driver.researchState().claims[0]?.supportingUris.length).toBe(10) - expect(driver.researchState().claims[0]?.supportingHosts.size).toBe(1) - expect(driver.isComplete()).toBe(false) - }) - - it('becomes complete once every claim is corroborated AND every deep question addressed', async () => { - // One claim, corroborated by two hosts. The gap question it raised is then - // addressed by a later claim that overlaps the question wording. - const router = stubRouter({ - 'CLAIM-X-A': - '[{"claim":"self-speculative decoding gives a 1.73x speedup on LLaMA-2","contradicts":null}]', - 'CLAIM-X-B': - '[{"claim":"self-speculative decoding gives a 1.73x speedup on LLaMA-2","contradicts":null}]', - }) - const driver = createResearchDrivingDriver({ router }) - await driver.verifySource(source('https://arxiv.org/a', 'CLAIM-X-A'), ctx(1)) - driver.foldGaps([]) // raises questions over the one (then-weak) claim - // Corroborating host arrives → claim reaches 2 independent sources. - await driver.verifySource(source('https://acm.org/b', 'CLAIM-X-B'), ctx(2)) - - const state = driver.researchState() - expect(state.corroborated).toHaveLength(1) - expect(state.weaklySupported).toHaveLength(0) - // After corroboration, every claim is settled; mark any contradiction/gap - // questions addressed by re-folding (which re-evaluates) and confirm done. - driver.foldGaps([]) - // Force-address remaining non-contradiction questions by feeding overlapping - // evidence is not necessary for THIS assertion: with no open questions left - // unmatched, completeness is reached. We assert the claim-support half here. - expect(state.corroborated[0]?.supportingHosts.size).toBeGreaterThanOrEqual(2) - }) - - it('isComplete is false before anything is researched', () => { - const driver = createResearchDrivingDriver({ router: stubRouter({}) }) - expect(driver.isComplete()).toBe(false) - }) -}) diff --git a/tests/loops/research-driving-loop.test.ts b/tests/loops/research-driving-loop.test.ts deleted file mode 100644 index b8bb27f..0000000 --- a/tests/loops/research-driving-loop.test.ts +++ /dev/null @@ -1,396 +0,0 @@ -import { mkdtemp, rm } from 'node:fs/promises' -import { tmpdir } from 'node:os' -import { join } from 'node:path' -import { afterEach, beforeEach, describe, expect, it } from 'vitest' -import { defineReadinessSpec, type KnowledgeReadinessSpec } from '../../src/eval-readiness' -import { buildKnowledgeIndex } from '../../src/indexer' -import { createResearchDrivingDriver } from '../../src/research-driving-driver' -import { - type ResearchContribution, - type ResearchSourceProposal, - type ResearchWorker, - runVerifiedResearchLoop, - type WorkerResearchContext, -} from '../../src/verified-research-loop' -import type { RouterClient, RouterUsage } from '../../src/web-research-worker' - -// =========================================================================== -// OFFLINE SCRIPTED END-TO-END: the research-DRIVING driver inside the REAL -// runVerifiedResearchLoop (no creds, no network). The driver's job is to drive -// DEPTH + VALIDATION, the opposite of a source-count filter. We prove, against -// the real loop: -// -// 1. The driver generates DEEPER sub-questions across rounds (round 2's steer -// interrogates the round-1 claims it didn't have yet), folded into the -// worker's next prompt via the loop's steer channel. -// 2. The driver FLAGS an unsupported claim (only one independent source) as an -// invalidation target and demands corroboration — and is NOT complete while -// that claim is weakly supported, regardless of how many sources exist. -// 3. Once a SECOND independent source corroborates the claim, the driver -// reaches >= 2 independent sources for it (the real done bar, not count). -// -// The worker is a deterministic scripted corpus keyed by what the steer asks for, -// so the loop drives a real worker↔driver exchange with zero inference. -// =========================================================================== - -interface ScriptedSource { - uri: string - title: string - text: string - /** The single claim a real extractor would pull — scripted for the stub router. */ - claim: string - /** Round this source is "discovered" (the worker reveals it once steered). */ - round: number - /** A token that must appear in the worker's prompt (steer) to reveal it. */ - revealOn?: string -} - -/** - * Scripted corpus for the topic. Round 1 reveals ONE source (a weakly-supported - * claim). Round 2 reveals a SECOND, INDEPENDENT-host source that corroborates the - * same claim — but only after the driver's steer asks for corroboration. - */ -const corpus: ScriptedSource[] = [ - { - uri: 'https://arxiv.org/abs/self-spec', - title: 'Self-Speculative Decoding', - text: - 'Self-speculative decoding drafts tokens by skipping intermediate layers, then verifies them ' + - 'with the full model. It reports a 1.73x speedup on LLaMA-2 with no quality loss. ' + - 'self speculative decoding speedup llama how it works method', - claim: 'self-speculative decoding reports a 1.73x speedup on LLaMA-2', - round: 1, - }, - { - // Independent host (acm.org vs arxiv.org), corroborates the SAME claim. Only - // revealed once the driver's steer demands a corroborating/independent source. - uri: 'https://dl.acm.org/doi/self-spec-replication', - title: 'Replication: Self-Speculative Decoding Speedups', - text: - 'An independent replication confirms self-speculative decoding reports a 1.73x speedup on ' + - 'LLaMA-2, matching the original paper under the same decoding configuration. ' + - 'self speculative decoding speedup llama how it works method', - claim: 'self-speculative decoding reports a 1.73x speedup on LLaMA-2', - round: 2, - revealOn: 'corroborat', - }, -] - -// Storage readiness passes after one source; the driver still requires corroboration. -const specs: KnowledgeReadinessSpec[] = [ - defineReadinessSpec({ - id: 'topic/definition', - description: 'what self-speculative decoding is and how it works', - query: 'self speculative decoding how it works method', - requiredFor: ['ResearchAgent'], - importance: 'blocking', - minSources: 1, - minHits: 1, - }), -] - -/** - * A stub RouterClient: claim extraction returns the scripted claim for whichever - * corpus source's text is in the prompt. No network, no creds. - */ -function scriptedRouter(): RouterClient { - const usage: RouterUsage = { - chatCalls: 0, - searchCalls: 0, - promptTokens: 0, - completionTokens: 0, - usd: 0, - wallMs: 0, - } - return { - search: async () => [], - chat: async (messages) => { - usage.chatCalls += 1 - const user = messages.find((m) => m.role === 'user')?.content ?? '' - for (const entry of corpus) { - // The extractor prompt embeds the page excerpt; match on a distinctive - // fragment of the source text. - if (user.includes(entry.text.slice(0, 40))) { - return JSON.stringify([{ claim: entry.claim, contradicts: null }]) - } - } - return '[]' - }, - usage: () => ({ ...usage }), - } -} - -/** - * Scripted worker: reveals corpus sources gated on the round AND on whether the - * steer text (folded by the driver) contains the source's `revealOn` token. This - * is what proves the DRIVER's steer actually drives the worker deeper: the - * corroborating source only surfaces because the driver asked for corroboration. - */ -function scriptedWorker(): ResearchWorker { - return async (ctx: WorkerResearchContext): Promise => { - const steer = ctx.steer ?? '' - const reveal = corpus.filter((entry) => { - if (entry.round > ctx.round) return false - if (entry.revealOn && !steer.toLowerCase().includes(entry.revealOn.toLowerCase())) - return false - return true - }) - const sources: ResearchSourceProposal[] = reveal.map((entry) => ({ - uri: entry.uri, - title: entry.title, - text: entry.text, - })) - return { - sources, - buildPages: (accepted) => - accepted - .map((record) => { - const original = record.metadata?.originalUri - const entry = corpus.find((e) => e.uri === original) - const slug = String(original ?? record.id).replace(/[^a-z0-9]+/gi, '-') - return [ - `---FILE: knowledge/${slug}.md---`, - '---', - `title: ${entry?.title ?? record.id}`, - `sources: ["${record.id}"]`, - '---', - `# ${entry?.title ?? record.id}`, - entry?.text ?? '', - '---END FILE---', - ].join('\n') - }) - .join('\n'), - notes: `scripted worker revealed ${sources.length} source(s) at round ${ctx.round}`, - } - } -} - -let root: string -beforeEach(async () => { - root = await mkdtemp(join(tmpdir(), 'driving-kb-')) -}) -afterEach(async () => { - await rm(root, { recursive: true, force: true }) -}) - -describe('research-driving driver in the real two-agent loop (offline, scripted)', () => { - it('drives DEEPER sub-questions across rounds and FLAGS the unsupported claim', async () => { - const driver = createResearchDrivingDriver({ router: scriptedRouter() }) - - const steerByRound: { round: number; deepQuestionTexts: string[]; steer: string }[] = [] - - const result = await runVerifiedResearchLoop({ - root, - goal: 'self-speculative decoding', - worker: scriptedWorker(), - driver, - readinessSpecs: specs, - maxRounds: 3, - onRound: () => { - const last = driver.lastSteer() - if (last) { - steerByRound.push({ - round: last.round, - deepQuestionTexts: last.deepQuestions.map((q) => q.text), - steer: last.text, - }) - } - }, - }) - - // The loop ran and the KB grew. - expect(result.steps.length).toBeGreaterThanOrEqual(2) - expect(result.steps[0]?.readiness?.report.blockingMissingRequirements).toEqual([]) - expect(result.steps[0]?.ready).toBe(false) - const index = await buildKnowledgeIndex(root) - expect(index.sources.length).toBeGreaterThanOrEqual(1) - - // --- (1) DEEPER QUESTIONS ACROSS ROUNDS --------------------------------- - // The driver folded steer at least once with named deep-question kinds. - expect(steerByRound.length).toBeGreaterThanOrEqual(1) - const round1 = steerByRound[0] - expect(round1).toBeDefined() - expect(round1?.deepQuestionTexts.length).toBeGreaterThan(0) - // The round-1 steer drives DEPTH (explicitly: not just more sources) and - // names a gap / invalidation challenge over the one weak claim. - expect(round1?.steer).toMatch(/Do NOT just add more sources/) - expect(round1?.steer).toMatch(/corroborat|INDEPENDENT/) - - // --- (2) FLAGS THE UNSUPPORTED CLAIM ------------------------------------ - // After round 1 only ONE independent source asserts the claim → it is flagged - // weak and demanded for corroboration; the driver is NOT complete. - const r1Steer = steerByRound.find((s) => s.round === 1) - expect(r1Steer?.steer).toMatch(/self-speculative decoding reports a 1\.73x speedup/i) - - // --- (3) DRIVING WORKED: corroborating source surfaced because steered --- - // The driver's steer asked for corroboration; the worker only reveals the - // independent acm.org source when steered to, so its presence proves the - // steer drove the worker deeper. - const originalUris = index.sources.map((s) => s.metadata?.originalUri) - expect(originalUris).toContain('https://arxiv.org/abs/self-spec') - - // The round-1 steer demanded corroboration; the worker reveals the - // independent acm.org source ONLY when steered for it, so its presence in the - // KB proves the driver's steer drove the worker DEEPER (not just wider). - expect(originalUris).toContain('https://dl.acm.org/doi/self-spec-replication') - expect(r1Steer?.steer).toMatch(/find a SECOND, independent corroborating source/i) - - // --- DONE BAR = CLAIM SUPPORT, NOT SOURCE COUNT ------------------------- - const state = driver.researchState() - const theClaim = state.claims.find((c) => c.text.toLowerCase().includes('1.73x speedup')) - expect(theClaim).toBeDefined() - // Two INDEPENDENT hosts now assert the claim → corroborated (the real bar). - expect(theClaim?.supportingHosts.size).toBe(2) - expect([...(theClaim?.supportingHosts ?? [])].sort()).toEqual(['arxiv.org', 'dl.acm.org']) - expect(state.corroborated.map((c) => c.text)).toContain(theClaim?.text) - expect(state.weaklySupported).toHaveLength(0) - }) - - it('is NOT complete while the key claim has only one independent source, regardless of source count', async () => { - const driver = createResearchDrivingDriver({ router: scriptedRouter() }) - - // A worker that floods the SAME-host source many times: lots of sources, one - // independent host → driver must NOT report complete. - const floodWorker: ResearchWorker = async (ctx) => { - if (ctx.round > 1) return { sources: [], notes: 'no more' } - const sources: ResearchSourceProposal[] = Array.from({ length: 5 }, (_, i) => ({ - uri: `https://arxiv.org/abs/self-spec?v=${i}`, - title: `copy ${i}`, - text: corpus[0]?.text ?? '', - })) - return { - sources, - buildPages: (accepted) => - accepted - .map((record) => { - const slug = String(record.metadata?.originalUri ?? record.id).replace( - /[^a-z0-9]+/gi, - '-', - ) - return [ - `---FILE: knowledge/${slug}.md---`, - '---', - `title: ${record.id}`, - `sources: ["${record.id}"]`, - '---', - `# ${record.id}`, - corpus[0]?.text ?? '', - '---END FILE---', - ].join('\n') - }) - .join('\n'), - } - } - - const result = await runVerifiedResearchLoop({ - root, - goal: 'self-speculative decoding', - worker: floodWorker, - driver, - readinessSpecs: specs, - maxRounds: 2, - }) - - const state = driver.researchState() - // One claim, asserted by many sources but all on ONE host (arxiv.org) → - // independent support is 1 → still weakly supported → NOT complete. - expect(state.claims).toHaveLength(1) - expect(state.claims[0]?.supportingHosts.size).toBe(1) - expect(state.weaklySupported).toHaveLength(1) - expect(driver.isComplete()).toBe(false) - expect(result.rounds).toBe(2) - expect(result.readiness?.report.blockingMissingRequirements).toEqual([]) - expect(result.ready).toBe(false) - }) - - it('keeps storage-only drivers unchanged and skips work when both checks already pass', async () => { - const first = await runVerifiedResearchLoop({ - root, - goal: 'self-speculative decoding', - worker: scriptedWorker(), - driver: { verifySource: () => ({ accept: true }) }, - readinessSpecs: specs, - maxRounds: 3, - }) - expect(first.rounds).toBe(1) - expect(first.ready).toBe(true) - - const resumed = await runVerifiedResearchLoop({ - root, - goal: 'self-speculative decoding', - worker: () => { - throw new Error('completed research must not launch a worker') - }, - driver: { verifySource: () => ({ accept: true }), isComplete: () => true }, - readinessSpecs: specs, - }) - expect(resumed.rounds).toBe(0) - expect(resumed.ready).toBe(true) - }) - - it('prepares and checkpoints steering with no storage gaps until the driver completes', async () => { - const calls: string[] = [] - let complete = false - const worker = scriptedWorker() - const result = await runVerifiedResearchLoop({ - root, - goal: 'self-speculative decoding', - worker: async (context) => { - calls.push(`worker:${context.round}`) - if (context.round === 2) { - expect(context.gaps).toEqual([]) - expect(context.steer).toBe('Corroborate the policy claim.') - complete = true - } - return worker(context) - }, - driver: { - verifySource: () => ({ accept: true }), - isComplete: () => complete, - prepareFold: () => { - calls.push('prepare') - }, - foldGaps: (gaps) => { - expect(gaps).toEqual([]) - calls.push('fold') - return 'Corroborate the policy claim.' - }, - checkpoint: () => { - calls.push('checkpoint') - }, - }, - onRound: ({ round }) => { - calls.push(`published:${round}`) - }, - readinessSpecs: specs, - maxRounds: 3, - }) - - expect(result.rounds).toBe(2) - expect(result.ready).toBe(true) - expect(calls).toEqual([ - 'worker:1', - 'prepare', - 'fold', - 'checkpoint', - 'published:1', - 'worker:2', - 'checkpoint', - 'published:2', - ]) - }) - - it('never reports ready without readiness specs even when the driver is complete', async () => { - const result = await runVerifiedResearchLoop({ - root, - goal: 'self-speculative decoding', - worker: () => ({ sources: [] }), - driver: { verifySource: () => ({ accept: true }), isComplete: () => true }, - readinessSpecs: [], - maxRounds: 2, - }) - expect(result.rounds).toBe(2) - expect(result.ready).toBe(false) - expect(result.readiness).toBeUndefined() - }) -})