From bd405db1e68e59d848e92eb71c6bb39527a9ac3c Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Tue, 6 Oct 2026 01:12:29 -0600 Subject: [PATCH] refactor(sources)!: remove the unused authority source adapters, change detection, and freshness stores No consumer imports the /sources entry point, its Cornell LII, IRS, and state Secretary of State adapters, politeFetch, htmlToText, detectChanges, or the freshness stores. The source registry and readiness freshness scoring stay. CI no longer sets AGENT_KNOWLEDGE_RUN_NETWORK_TESTS, which only the deleted live source test read. BREAKING CHANGE: the ./sources entry point and the change-detection and freshness-store exports are removed. --- .github/workflows/ci.yml | 2 - CHANGELOG.md | 2 + README.md | 2 - api-surface.json | 66 -------- package.json | 5 - scripts/verify-package.mjs | 1 - src/changes.ts | 177 --------------------- src/freshness.ts | 208 ------------------------- src/index.ts | 3 - src/sources/cornell-lii.ts | 188 ----------------------- src/sources/html.ts | 89 ----------- src/sources/http.ts | 262 -------------------------------- src/sources/index.ts | 17 --- src/sources/irs-publications.ts | 223 --------------------------- src/sources/state-sos.ts | 151 ------------------ src/sources/types.ts | 153 ------------------- tests/changes.test.ts | 121 --------------- tests/freshness.test.ts | 159 ------------------- tests/http-cache.test.ts | 192 ----------------------- tests/sources-live.test.ts | 158 ------------------- tests/sources-mocked.test.ts | 245 ----------------------------- tests/sources-types.test.ts | 109 ------------- tsdown.config.ts | 1 - 23 files changed, 2 insertions(+), 2532 deletions(-) delete mode 100644 src/changes.ts delete mode 100644 src/freshness.ts delete mode 100644 src/sources/cornell-lii.ts delete mode 100644 src/sources/html.ts delete mode 100644 src/sources/http.ts delete mode 100644 src/sources/index.ts delete mode 100644 src/sources/irs-publications.ts delete mode 100644 src/sources/state-sos.ts delete mode 100644 src/sources/types.ts delete mode 100644 tests/changes.test.ts delete mode 100644 tests/freshness.test.ts delete mode 100644 tests/http-cache.test.ts delete mode 100644 tests/sources-live.test.ts delete mode 100644 tests/sources-mocked.test.ts delete mode 100644 tests/sources-types.test.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index afbf51a..1a3b5b1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -34,8 +34,6 @@ jobs: - name: Test run: pnpm test - env: - AGENT_KNOWLEDGE_RUN_NETWORK_TESTS: '1' - name: Build run: pnpm build diff --git a/CHANGELOG.md b/CHANGELOG.md index 45d7eb6..944283a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,8 @@ The verified-research paper and its results remain as a historical record linked Remove the memory experiment, memory improvement, and benchmark-suite runners, which no consumer imported: `runAgentMemoryExperiment`, `runAgentMemoryLearningExperiment`, `runAgentMemoryImprovement`, `runKnowledgeBenchmarkSuite`, `runMemoryAdapterBenchmark`, the industry benchmark catalogs, qrels import, benchmark scoring, and their run-lease, cost, and comparison types. Compare memory providers on product tasks with Agent Eval. Memory adapters, branches, holdout, lifecycle bounds, play memory tools, and the `/benchmarks` in-memory and no-op adapters are unchanged. +Remove the `/sources` entry point and the unused authority adapters behind it (Cornell LII, IRS publications, state Secretary of State, polite HTTP fetch, HTML extraction), plus `detectChanges` and the freshness stores. No consumer imported them. +The source registry (`addSourceText`, `addSourcePath`, `loadSourceRegistry`), source adapters such as `textSourceAdapter`, and readiness freshness scoring are unchanged. ## 19.1.5 diff --git a/README.md b/README.md index 7c9288d..99e56c6 100644 --- a/README.md +++ b/README.md @@ -32,7 +32,6 @@ Use Knowledge 17 with older Eval releases. | Optimize a KB maintenance policy | `optimizeKnowledgeBasePolicy` | package root | | Run retrieval, research, answer checks, and promotion as one process | `runRagKnowledgeImprovementLoop` | package root | | Connect a memory provider or branch its state | `AgentMemoryAdapter`, `createAgentMemoryBranch` | `/memory` | -| Read from external authorities | `KnowledgeSource` and source adapters | `/sources` | | Use live research or coding agents | `runKnowledgeImprovementJob` | `@tangle-network/agent-runtime` | ## Create and search a knowledge base @@ -469,7 +468,6 @@ Compare providers on product tasks with `@tangle-network/agent-eval` before chan |---|---| | `@tangle-network/agent-knowledge` | KB files, indexes, search, validation, research callbacks, RAG evaluation, and candidate improvement | | `@tangle-network/agent-knowledge/memory` | Memory contracts, provider adapters, branches, holdout, and play memory tools | -| `@tangle-network/agent-knowledge/sources` | HTTP and authority-specific source adapters | | `@tangle-network/agent-knowledge/benchmarks` | In-memory and no-op memory adapters for benchmark harnesses | | `@tangle-network/agent-knowledge/viz` | Dependency-free graph analysis helpers | diff --git a/api-surface.json b/api-surface.json index 8e7e399..99d8202 100644 --- a/api-surface.json +++ b/api-surface.json @@ -44,15 +44,12 @@ "ClaimLedgerStore": "type c402ec258adc", "ClaimRef": "type e200b4c5eeec", "ClaimVerdict": "type 18f044018844", - "CornellLiiSelector": "type c92fb79c4d6b", - "CornellLiiSourceOptions": "type 3d1a7a174053", "CreateAgentMemoryBranchOptions": "type 924c7b6dcd11", "CreateKnowledgeRetrievalDispositionInput": "type 27fb6228c8c1", "CreateKnowledgeRetrievalReceiptInput": "type 2b6c8c6a078b", "CreateKnowledgeToolsOptions": "type 359d4edb185e", "CreateKnowledgeUseReceiptInput": "type eb120a93f4c7", "CreatePlayMemoryToolsOptions": "type d659926093f3", - "D1Adapter": "type fd8669db1f2e", "DEADLINE_EXIT_CODE": "value 16590df13e07", "DEFAULT_KNOWLEDGE_BRIEF_LIMIT": "value a16e63bcd9c7", "DEFAULT_MEMORY_CLEANUP_TIMEOUT_MS": "value 10d1ef3e7731", @@ -61,8 +58,6 @@ "DeepQuestionKind": "type e1a27d1fc3c2", "DeepQuestionSchema": "value 373728f5643d", "DefineReadinessSpecInput": "type f54fcc87f848", - "DetectChangesOptions": "type 39c50fdf35ae", - "DetectChangesResult": "type 830df4ec6d85", "DiscoveryLoopResult": "type 80ccac2ce500", "DiscoveryLoopRound": "type 4a67fa7bfc48", "DiscoveryLoopStopReason": "type 61912c65132e", @@ -72,19 +67,12 @@ "EvaluateKnowledgeBaseReadinessOptions": "type 9a3de35dc405", "EvidenceRung": "type fa5c1c76aa73", "ExternalRagEvalScore": "type df1d799a1de6", - "FetchOpts": "type 7b44d8f28084", - "FileSystemFreshnessStoreOptions": "type 10a9cbd1e857", "FileSystemKbStore": "value 3e4c27745531", "FileSystemKbStoreOptions": "type 10a9cbd1e857", "FileSystemSearchOptions": "type ab0c98cd100a", "FileSystemSearchProvider": "value cf2101caec6e", "FileSystemSearchProviderOptions": "type f8bf1cf8926d", "ForkAgentMemoryBranchSnapshotOptions": "type 4daac6457273", - "FragmentProvenance": "type a719c43aeb25", - "FreshnessKey": "type 65cdc4ab2341", - "FreshnessMark": "type 93bc4803d9b3", - "FreshnessRecord": "type 65bdf72051de", - "FreshnessTtl": "type e3c26a4f6c3e", "GradeableClaim": "type bf465a1848d5", "GradedClaim": "type acfba8ff752d", "GraphitiMcpClientLike": "type a6c48bcfcdd6", @@ -95,10 +83,8 @@ "HindsightClientLike": "type e4b6e0bd1488", "HindsightMemoryAdapterOptions": "type 995d4468eb3c", "HindsightOperationUnknownError": "value 408246fe51ae", - "IRS_DIMENSION_HINTS": "value 373728f5643d", "ImproveSelectedKnowledgeCandidateOptions": "type c738ecaa49fa", "ImproveSelectedKnowledgeCandidateResult": "type 3a8454d1031d", - "IrsPublicationsSourceOptions": "type 6b60bfe9f1bc", "KB_CLAIM_LEDGER_DIR": "value 2e94ea580600", "KB_EVENTS_PATH": "value 7c4a3ddbac83", "KB_INDEX_PATH": "value f450045c9510", @@ -118,8 +104,6 @@ "KnowledgeBaseReadinessEvaluation": "type 5de5279c9dce", "KnowledgeBrief": "type 01ca10f80a06", "KnowledgeBriefOptions": "type ceb76227d749", - "KnowledgeChange": "type 7b71ca9dd658", - "KnowledgeChangeKind": "type 02226535c8f0", "KnowledgeChunk": "type 5c73eb000db2", "KnowledgeCitationAuditError": "value e64c40482f65", "KnowledgeCitationAuditIssue": "type bb84ee6c9f6c", @@ -148,8 +132,6 @@ "KnowledgeEventSchema": "value 373728f5643d", "KnowledgeEventType": "type 7071c221c684", "KnowledgeExplanation": "type 4326c52192e1", - "KnowledgeFragment": "type 5d9512fd9945", - "KnowledgeFreshnessStore": "type 6805e31ec36f", "KnowledgeGraph": "type 2a4f44745410", "KnowledgeGraphEdge": "type 9eba2728a7b2", "KnowledgeGraphEdgeSchema": "value 373728f5643d", @@ -239,7 +221,6 @@ "KnowledgeRetrieverIdentity": "type fb46de336ac2", "KnowledgeSearchHit": "type d13a101a4b4f", "KnowledgeSearchResult": "type c443b7efb49e", - "KnowledgeSource": "type 6ebb3f0a0ef2", "KnowledgeStateScope": "type 61b8ac32e052", "KnowledgeUnit": "type bbd850260f7f", "KnowledgeUseReceipt": "type 95b469ec3c07", @@ -257,8 +238,6 @@ "KnowledgeWriteIntakeRequest": "type 5074f0dab514", "KnowledgeWriteParseResult": "type 3ec744fcaf93", "LoadKnowledgeImprovementActivationResultOptions": "type 3c304bc52d1c", - "MAX_RESPONSE_BYTES": "value 373728f5643d", - "MIN_REQUEST_GAP_MS": "value 648cef7e8361", "MeasuredKnowledgeSelectionReceipt": "type 28a9a57c6626", "Mem0ClientMode": "type abfe4225abaf", "Mem0HostedClient": "type 4d1434ac5a9a", @@ -275,14 +254,11 @@ "OptimizeKnowledgeBasePolicyResult": "type 1e3b03c7a0a3", "OriginatedKnowledgeSearchResult": "type 99110190f58f", "OriginatedPage": "type 62ba6f531a4a", - "POLITE_USER_AGENT": "value 3eb07665c028", "PageOrigin": "type df33a580b8a9", "ParsedFrontmatter": "type 7aebd4ab7c31", "PartitionRetrievalScenariosOptions": "type bfb5473c9c0c", "PendingKnowledgeMutation": "type 265414e3a059", "PlayMemoryRetrievalReceipt": "type 0118f85065b1", - "PoliteFetchOptions": "type e8c10ab474da", - "PoliteFetchResult": "type 68c8ac7c266b", "PromoteKnowledgeCandidateOptions": "type 804348b78767", "PromoteRunScopedPagesOptions": "type 314295c13674", "ProposeFromFindingsResult": "type 979e50440bbd", @@ -390,8 +366,6 @@ "SourceRecord": "type b525cb8c09ee", "SourceRecordSchema": "value 373728f5643d", "SourceRegistry": "type 062b92354d23", - "StateSosEntity": "type 4ecdc780ae4d", - "StateSosSourceConfig": "type 3a2d0b61a718", "TrackedClaim": "type 823ab1f0941e", "UncheckableClaimError": "value 86a15815c4f1", "UseKnowledgeImprovementCandidateOptions": "type 848b0e83e03f", @@ -400,7 +374,6 @@ "VerifiedGradeableEvidence": "type 68ecc1af89b5", "VerifyGradeableEvidenceOptions": "type 01ef698f9b40", "WIKILINK_REGEX": "value 373728f5643d", - "__resetHttpThrottle": "value e47f3b6734a6", "addSourcePath": "value 9dabb587c73c", "addSourceText": "value 3a5c6107b137", "applyKnowledgeWriteBlocks": "value 08c999b38722", @@ -438,15 +411,11 @@ "claimId": "value 4c9eca896a37", "claimSourceHost": "value a8a18cde9fcc", "createAgentMemoryBranch": "value bae206df8be9", - "createCornellLiiSource": "value 5708478c5b8a", - "createD1FreshnessStoreStub": "value e89ca59074b9", "createFileRunLineageAuthority": "value 92afeb585f32", - "createFileSystemFreshnessStore": "value 56d2aaf47eb6", "createFileSystemSearchProvider": "value c88b43a7ac80", "createGraphitiMemoryAdapter": "value 512201c0dd04", "createHindsightMemoryAdapter": "value 97fcfb086aee", "createInMemoryBenchmarkAdapter": "value f4c546ff2fff", - "createIrsPublicationsSource": "value da388cf92c5c", "createKnowledgeControlLoopAdapter": "value bfc8f1be188f", "createKnowledgeEvent": "value 4b9e73e096bd", "createKnowledgeRetrievalDisposition": "value 838fadfc5636", @@ -464,12 +433,10 @@ "createQmdSearchProvider": "value f1d2a2d4d631", "createRagAnswerQualityHook": "value 7e18d330e76b", "createRunScopedStores": "value 6bfc535c7281", - "createStateSosSource": "value e7467eeee69b", "decodeKnowledgeVisibilitySnapshot": "value 143bc2b124e6", "deepQuestionId": "value ffec05529ae9", "defaultGetMemoryContext": "value 6a368eb4ea71", "defineReadinessSpec": "value 9128b5b31bfa", - "detectChanges": "value 741122c9f8e7", "detectNearDuplicatePages": "value 457a9cbea181", "deterministicRng": "value f2af0cd5da5c", "diagnoseRagAnswerFailure": "value a8382d9e2133", @@ -478,9 +445,7 @@ "encodeKnowledgeVisibilitySnapshot": "value c673fa060267", "evaluateKnowledgeBaseReadiness": "value a981becc92c2", "explainKnowledgeTarget": "value 1266c029bf54", - "extractLinks": "value 930437fc2aa4", "extractWikilinks": "value 2a806a857c29", - "firstMatch": "value 0fa78f90a44f", "forkAgentMemoryBranchSnapshot": "value 0c258d0909ac", "formatFrontmatter": "value e0a4508d4ded", "formatKnowledgeCitationReference": "value 8c9301778549", @@ -492,11 +457,9 @@ "groundClaimInText": "value f74ed9ca26c6", "hashKnowledgeBase": "value bdb2d9eea5f9", "hindsightMemoryBankId": "value b828b7e949f0", - "htmlToText": "value c58c3f5d8c75", "improveKnowledgeBase": "value a4ebce095bdf", "improveSelectedKnowledgeCandidate": "value 6d48e36af283", "initKnowledgeBase": "value 65692669c5c8", - "innerHtmlById": "value f0ffa022d6c0", "inspectKnowledgeIndex": "value 79b67f7de1cd", "inspectPendingKnowledgeMutation": "value 38b2e30827bd", "isKernelAnchoredPath": "value 2819fa14db65", @@ -528,7 +491,6 @@ "loadKnowledgePages": "value baac5eb22201", "loadKnowledgePromotionRecord": "value 65617956865c", "loadSourceRegistry": "value cae49411bcc5", - "looksLikeBlockPage": "value 371f55d1c1d9", "materializeRegisteredClaimEvidence": "value cf315a2deab2", "mediaTypeFor": "value a8a18cde9fcc", "mem0MemoryAdapterIdentity": "value c590a9c1e72f", @@ -551,7 +513,6 @@ "parseKnowledgeWriteBlocks": "value d6bd24d2858d", "partitionRetrievalScenarios": "value 363802debe8a", "planInvalidationPropagation": "value 39da8e2c1a14", - "politeFetch": "value 10d577fc5694", "promoteKnowledgeCandidate": "value ea1e94a3d6a2", "promoteRunScopedPages": "value 58ff1ec22528", "proposeFromFinding": "value 1dcecd58e869", @@ -714,33 +675,6 @@ "sleepForMemoryRecovery": "value 4a6818f984e4", "toOffPolicyTrajectory": "value 934dec24305c" }, - "./sources": { - "CornellLiiSelector": "type c92fb79c4d6b", - "CornellLiiSourceOptions": "type 3d1a7a174053", - "FetchOpts": "type 7b44d8f28084", - "FragmentProvenance": "type a719c43aeb25", - "IRS_DIMENSION_HINTS": "value 373728f5643d", - "IrsPublicationsSourceOptions": "type 6b60bfe9f1bc", - "KnowledgeFragment": "type 5d9512fd9945", - "KnowledgeSource": "type 6ebb3f0a0ef2", - "MAX_RESPONSE_BYTES": "value 373728f5643d", - "MIN_REQUEST_GAP_MS": "value 648cef7e8361", - "POLITE_USER_AGENT": "value 3eb07665c028", - "PoliteFetchOptions": "type e8c10ab474da", - "PoliteFetchResult": "type 68c8ac7c266b", - "StateSosEntity": "type 4ecdc780ae4d", - "StateSosSourceConfig": "type 3a2d0b61a718", - "__resetHttpThrottle": "value e47f3b6734a6", - "createCornellLiiSource": "value 5708478c5b8a", - "createIrsPublicationsSource": "value da388cf92c5c", - "createStateSosSource": "value e7467eeee69b", - "extractLinks": "value 930437fc2aa4", - "firstMatch": "value 0fa78f90a44f", - "htmlToText": "value c58c3f5d8c75", - "innerHtmlById": "value f0ffa022d6c0", - "looksLikeBlockPage": "value 371f55d1c1d9", - "politeFetch": "value 10d577fc5694" - }, "./viz": { "KnowledgeCommunity": "type 945ac89694c9", "KnowledgeGap": "type afb49eb9f42f", diff --git a/package.json b/package.json index ec33b66..ab19ac2 100644 --- a/package.json +++ b/package.json @@ -34,11 +34,6 @@ "import": "./dist/memory/index.js", "default": "./dist/memory/index.js" }, - "./sources": { - "types": "./dist/sources/index.d.ts", - "import": "./dist/sources/index.js", - "default": "./dist/sources/index.js" - }, "./benchmarks": { "types": "./dist/benchmarks/index.d.ts", "import": "./dist/benchmarks/index.js", diff --git a/scripts/verify-package.mjs b/scripts/verify-package.mjs index e8d0e58..2a1932a 100644 --- a/scripts/verify-package.mjs +++ b/scripts/verify-package.mjs @@ -19,7 +19,6 @@ const publicImports = [ packageName, `${packageName}/viz`, `${packageName}/memory`, - `${packageName}/sources`, `${packageName}/benchmarks`, ] const requiredRootExports = [ diff --git a/src/changes.ts b/src/changes.ts deleted file mode 100644 index d5624fe..0000000 --- a/src/changes.ts +++ /dev/null @@ -1,177 +0,0 @@ -import type { KnowledgeFragment } from './sources/types' - -/** - * Change detection across snapshots of one source's fragments. - * - * The output drives the continuous-ingestion loop: each `KnowledgeChange` - * carries the eval dimensions affected (`affectedDimensions`), which an - * agent-eval campaign scheduler consumes to decide which campaigns to - * re-run. Three change kinds: - * - * - `added` — fragment id appears in `next` but not `prev`. - * - `removed` — fragment id appears in `prev` but not `next`. Typical - * trigger: an authority retires a Wex slug, or a state SOS reorganises - * its forms catalogue. - * - `modified` — fragment id appears in both, body hash differs. This - * is the dominant change kind in practice — the Ryan-LLC v. FTC - * vacatur case manifests as a `modified` on the Wex non-compete - * fragment, NOT as a removed one. - * - * Unverifiable fragments are filtered out before diffing — comparing a - * captcha-blocked snapshot against a real one would falsely fire every - * fragment as removed. The caller can inspect the raw lists if they want - * to surface block-page failures. - * - * Within-snapshot duplicate ids are an upstream bug; this function picks - * the LAST one and emits a diagnostic via the `warnings` field. - * - * @stable - */ - -export type KnowledgeChangeKind = 'added' | 'removed' | 'modified' - -export interface KnowledgeChange { - /** Source-scoped id (matches `KnowledgeFragment.id`). */ - fragmentId: string - kind: KnowledgeChangeKind - /** - * For `added`: full body of the new fragment. - * For `removed`: full body of the prior fragment. - * For `modified`: unified-diff-style payload `{ before, after }` body strings. - */ - diff?: { before?: string; after?: string } - /** - * Eval dimensions to re-score. Computed as the union of both fragments' - * `dimensionHints`. The eval cron treats this as a set of campaign tags. - */ - affectedDimensions: string[] - /** URL of the affected authority page (from whichever side has it). */ - url?: string - /** - * Source-attested change time. For `modified`, takes the NEXT fragment's - * `sourceUpdatedAt`. For `removed`, takes the PRIOR fragment's - * `sourceUpdatedAt`. For `added`, takes the NEXT fragment's - * `sourceUpdatedAt`. Consumers index changes by this date. - */ - detectedAt: string -} - -export interface DetectChangesResult { - changes: KnowledgeChange[] - /** Counts by kind — handy for dashboards. */ - summary: { added: number; removed: number; modified: number } - /** Non-fatal diagnostics (duplicate ids, dropped unverifiable fragments). */ - warnings: string[] -} - -export interface DetectChangesOptions { - /** - * When true (default), unverifiable fragments are dropped from both - * sides before comparison. Set false ONLY when debugging block-page - * issues — comparing against unverifiable content emits false - * `removed`/`modified` changes. - */ - skipUnverifiable?: boolean - /** - * When provided, only changes whose `affectedDimensions` intersect this - * set are returned. Useful for cron loops that schedule per-dimension - * eval campaigns and only care about a subset. - */ - filterDimensions?: string[] -} - -export function detectChanges( - prev: KnowledgeFragment[], - next: KnowledgeFragment[], - options: DetectChangesOptions = {}, -): DetectChangesResult { - const skipUnverifiable = options.skipUnverifiable ?? true - const warnings: string[] = [] - - const { map: prevMap, warnings: prevWarn } = indexFragments(prev, skipUnverifiable, 'prev') - const { map: nextMap, warnings: nextWarn } = indexFragments(next, skipUnverifiable, 'next') - warnings.push(...prevWarn, ...nextWarn) - - const changes: KnowledgeChange[] = [] - const seen = new Set() - - for (const [id, nextFragment] of nextMap) { - seen.add(id) - const prevFragment = prevMap.get(id) - if (!prevFragment) { - changes.push({ - fragmentId: id, - kind: 'added', - diff: { after: nextFragment.body }, - affectedDimensions: dedup(nextFragment.dimensionHints), - url: nextFragment.provenance.url, - detectedAt: nextFragment.provenance.sourceUpdatedAt, - }) - continue - } - if (prevFragment.bodyHash !== nextFragment.bodyHash) { - changes.push({ - fragmentId: id, - kind: 'modified', - diff: { before: prevFragment.body, after: nextFragment.body }, - affectedDimensions: dedup([...prevFragment.dimensionHints, ...nextFragment.dimensionHints]), - url: nextFragment.provenance.url, - detectedAt: nextFragment.provenance.sourceUpdatedAt, - }) - } - } - - for (const [id, prevFragment] of prevMap) { - if (seen.has(id)) continue - changes.push({ - fragmentId: id, - kind: 'removed', - diff: { before: prevFragment.body }, - affectedDimensions: dedup(prevFragment.dimensionHints), - url: prevFragment.provenance.url, - detectedAt: prevFragment.provenance.sourceUpdatedAt, - }) - } - - const filtered = options.filterDimensions - ? changes.filter((c) => c.affectedDimensions.some((d) => options.filterDimensions?.includes(d))) - : changes - - return { - changes: filtered, - summary: { - added: filtered.filter((c) => c.kind === 'added').length, - removed: filtered.filter((c) => c.kind === 'removed').length, - modified: filtered.filter((c) => c.kind === 'modified').length, - }, - warnings, - } -} - -function indexFragments( - fragments: KnowledgeFragment[], - skipUnverifiable: boolean, - side: string, -): { map: Map; warnings: string[] } { - const map = new Map() - const warnings: string[] = [] - let dropped = 0 - for (const fragment of fragments) { - if (skipUnverifiable && !fragment.provenance.verifiable) { - dropped += 1 - continue - } - if (map.has(fragment.id)) { - warnings.push(`${side}: duplicate fragment id ${fragment.id} — keeping last`) - } - map.set(fragment.id, fragment) - } - if (dropped > 0) { - warnings.push(`${side}: dropped ${dropped} unverifiable fragment(s) before diff`) - } - return { map, warnings } -} - -function dedup(items: T[]): T[] { - return [...new Set(items)] -} diff --git a/src/freshness.ts b/src/freshness.ts deleted file mode 100644 index 22b1450..0000000 --- a/src/freshness.ts +++ /dev/null @@ -1,208 +0,0 @@ -import { readFile } from 'node:fs/promises' -import { join } from 'node:path' -import { z } from 'zod' -import { isMissingFile, writeJsonDurableWithinRoot } from './durable-fs' -import { withKnowledgeMutation, withKnowledgeRead } from './mutation-lock' - -/** - * Knowledge freshness store: tracks when each `(workspaceId, sourceId)` pair - * was last successfully refreshed, and reports staleness against a TTL. - * - * The contract is intentionally minimal and supports scheduled refresh loops: - * - * ```ts - * const store = createFileSystemFreshnessStore({ root: '.agent-knowledge' }) - * for (const source of sources) { - * if (await store.stale({ workspaceId, sourceId: source.id, ttlMs: DAY })) { - * const fragments = await source.fetch({ cacheDir }) - * await persistFragments(fragments) - * await store.mark({ workspaceId, sourceId: source.id, when: new Date() }) - * } - * } - * ``` - * - * Per-tenant isolation is enforced by `workspaceId` keying. There is no - * global mutable state across workspaces. - * - * Two adapters ship in-package: - * - * - `createFileSystemFreshnessStore`: JSON file under the knowledge root, - * mirrors the layout convention already used by `sources.json`. - * - `createD1FreshnessStoreStub`: complete bridge from the `D1Adapter` port - * to this package's freshness interface. The port can be backed by D1, - * PostgreSQL, SQLite, or another application-owned store. - */ - -/** Identity for one freshness record. */ -export interface FreshnessKey { - workspaceId: string - sourceId: string -} - -/** TTL bound for staleness checks. */ -export interface FreshnessTtl extends FreshnessKey { - /** Milliseconds; the record is stale when `Date.now() - last() > ttlMs`. */ - ttlMs: number - /** Injected clock for deterministic tests; defaults to system time. */ - now?: Date -} - -/** Mark argument. */ -export interface FreshnessMark extends FreshnessKey { - when: Date - /** Optional content hash captured at refresh time; aids debugging. */ - contentHash?: string -} - -export interface KnowledgeFreshnessStore { - /** Last refresh time, or null if never refreshed. */ - last(key: FreshnessKey): Promise - /** Record a successful refresh. */ - mark(input: FreshnessMark): Promise - /** True iff `last(key)` is null or older than `ttlMs`. */ - stale(input: FreshnessTtl): Promise - /** All records for a workspace. */ - list(workspaceId: string): Promise -} - -export interface FreshnessRecord { - workspaceId: string - sourceId: string - lastRefreshedAt: string - contentHash?: string -} - -export interface FileSystemFreshnessStoreOptions { - /** - * Knowledge root. The store writes to `/.agent-knowledge/freshness.json`, - * mirroring the convention used by `sources.json`. - */ - root: string -} - -const freshnessRecordSchema = z - .object({ - workspaceId: z.string().min(1), - sourceId: z.string().min(1), - lastRefreshedAt: z.iso.datetime(), - contentHash: z.string().min(1).optional(), - }) - .strict() -const freshnessFileSchema = z - .object({ records: z.record(z.string(), freshnessRecordSchema) }) - .strict() - -/** - * Filesystem-backed implementation. Single JSON file per knowledge root, - * indexed by `${workspaceId}::${sourceId}`. Reads parse on every call; - * cron tick rate is well below the cost of one JSON parse. - * - * Writes share the package-wide filesystem lock, so multiple workers cannot - * overwrite one another or run through an interrupted knowledge promotion. - */ -export function createFileSystemFreshnessStore( - options: FileSystemFreshnessStoreOptions, -): KnowledgeFreshnessStore { - const path = join(options.root, '.agent-knowledge', 'freshness.json') - - const read = async (): Promise> => { - return withKnowledgeRead(options.root, async () => { - try { - return freshnessFileSchema.parse(JSON.parse(await readFile(path, 'utf8'))).records - } catch (error) { - if (isMissingFile(error)) return {} - throw error - } - }) - } - - const write = async (records: Record): Promise => { - await writeJsonDurableWithinRoot( - options.root, - '.agent-knowledge/freshness.json', - freshnessFileSchema.parse({ records }), - ) - } - - return { - async last(key) { - const records = await read() - const record = records[buildKey(key)] - return record ? new Date(record.lastRefreshedAt) : null - }, - async mark(input) { - await withKnowledgeMutation(options.root, async () => { - const records = await read() - records[buildKey(input)] = { - workspaceId: input.workspaceId, - sourceId: input.sourceId, - lastRefreshedAt: input.when.toISOString(), - ...(input.contentHash === undefined ? {} : { contentHash: input.contentHash }), - } - await write(records) - }) - }, - async stale(input) { - const last = await this.last(input) - if (!last) return true - const now = input.now ?? new Date() - return now.getTime() - last.getTime() > input.ttlMs - }, - async list(workspaceId) { - const records = await read() - return Object.values(records).filter((r) => r.workspaceId === workspaceId) - }, - } -} - -/** - * Bridge an application-owned database adapter to `KnowledgeFreshnessStore`. - * The adapter may use D1, PostgreSQL, SQLite, or another durable store. - * - * The expected schema: - * - * ```sql - * CREATE TABLE knowledge_freshness ( - * workspace_id TEXT NOT NULL, - * source_id TEXT NOT NULL, - * last_refreshed_at TEXT NOT NULL, - * content_hash TEXT, - * PRIMARY KEY (workspace_id, source_id) - * ); - * ``` - */ -export interface D1Adapter { - get(workspaceId: string, sourceId: string): Promise - upsert(record: FreshnessRecord): Promise - listByWorkspace(workspaceId: string): Promise -} - -export function createD1FreshnessStoreStub(adapter: D1Adapter): KnowledgeFreshnessStore { - return { - async last(key) { - const record = await adapter.get(key.workspaceId, key.sourceId) - return record ? new Date(record.lastRefreshedAt) : null - }, - async mark(input) { - await adapter.upsert({ - workspaceId: input.workspaceId, - sourceId: input.sourceId, - lastRefreshedAt: input.when.toISOString(), - contentHash: input.contentHash, - }) - }, - async stale(input) { - const last = await this.last(input) - if (!last) return true - const now = input.now ?? new Date() - return now.getTime() - last.getTime() > input.ttlMs - }, - async list(workspaceId) { - return adapter.listByWorkspace(workspaceId) - }, - } -} - -function buildKey(key: FreshnessKey): string { - return `${key.workspaceId}::${key.sourceId}` -} diff --git a/src/index.ts b/src/index.ts index 0c3de0e..c12abaf 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,7 +1,6 @@ export * from './adapters' export * from './agent-candidate' export * from './benchmarks/index' -export * from './changes' export * from './chunking' export * from './citation-lint' export * from './citation-resolution' @@ -35,7 +34,6 @@ export { export * from './eval-readiness' export * from './events' export * from './filesystem-search-provider' -export * from './freshness' export * from './frontmatter' export * from './graph' export * from './ids' @@ -105,7 +103,6 @@ export { export * from './schemas' export * from './search' export * from './sources' -export * from './sources/index' export * from './store' export * from './types' export * from './validate' diff --git a/src/sources/cornell-lii.ts b/src/sources/cornell-lii.ts deleted file mode 100644 index 41c6c9c..0000000 --- a/src/sources/cornell-lii.ts +++ /dev/null @@ -1,188 +0,0 @@ -import { sha256 } from '../ids' -import { htmlToText, innerHtmlById } from './html' -import { politeFetch } from './http' -import type { FetchOpts, KnowledgeFragment, KnowledgeSource } from './types' - -/** - * Cornell Legal Information Institute (LII) source. - * - * Pulls federal US Code sections and Wex encyclopedia entries — the two - * Cornell LII surfaces an agent typically grounds against. The Wex - * "non-compete" page is the canonical test case for the Ryan-LLC v. FTC - * vacatur drift the continuous-ingestion story is designed to catch. - * - * @stable - */ - -const BASE_URL = 'https://www.law.cornell.edu' - -export interface CornellLiiSelector { - /** Either 'uscode' or 'wex'. */ - kind: 'uscode' | 'wex' - /** - * For `uscode`: `/<section>` (e.g. `'18/1836'` for DTSA). - * For `wex`: the slug (e.g. `'non-compete'`). - */ - path: string - /** - * Optional pre-declared eval dimensions affected by this section. If - * omitted, defaults are chosen from `kind` + path heuristics. - */ - dimensionHints?: string[] -} - -export interface CornellLiiSourceOptions { - /** - * Selectors to fetch on each `fetch()` call. The caller (a per-tenant - * workspace config, typically) lists exactly the authorities they need - * tracked. There is no auto-discovery; that would crawl Cornell at - * cron speed, which is what the polite-fetch contract exists to avoid. - */ - selectors: CornellLiiSelector[] - /** Source id override; default is `'cornell-lii'`. */ - id?: string -} - -/** - * Build a Cornell LII source for the listed selectors. - * - * Example: track DTSA + non-compete: - * ``` - * createCornellLiiSource({ - * selectors: [ - * { kind: 'uscode', path: '18/1836' }, - * { kind: 'wex', path: 'non-compete', dimensionHints: ['jurisdictional_accuracy'] }, - * ], - * }) - * ``` - */ -export function createCornellLiiSource(options: CornellLiiSourceOptions): KnowledgeSource { - const id = options.id ?? 'cornell-lii' - return { - id, - name: 'Cornell Legal Information Institute', - description: - 'Federal US Code sections (uscode/text/...) and Wex legal encyclopedia entries from law.cornell.edu.', - async fetch(opts: FetchOpts): Promise<KnowledgeFragment[]> { - const limit = opts.limit ?? options.selectors.length - const selectors = options.selectors.slice(0, limit) - const out: KnowledgeFragment[] = [] - for (const selector of selectors) { - out.push(await fetchOne(id, selector, opts)) - } - return out - }, - } -} - -async function fetchOne( - sourceId: string, - selector: CornellLiiSelector, - opts: FetchOpts, -): Promise<KnowledgeFragment> { - const path = selector.path.replace(/^\/+/, '') - const url = - selector.kind === 'uscode' ? `${BASE_URL}/uscode/text/${path}` : `${BASE_URL}/wex/${path}` - - const response = await politeFetch(url, { - signal: opts.signal, - cacheDir: opts.cacheDir, - }) - - const fragmentId = `${selector.kind}:${selector.path}` - const dimensionHints = selector.dimensionHints ?? defaultDimensionHints(selector) - - if (!response.verifiable) { - return { - id: fragmentId, - title: `Cornell LII ${selector.kind} ${selector.path}`, - body: '', - bodyHash: sha256(''), - provenance: { - url, - sourceUpdatedAt: response.sourceUpdatedAt, - fetchedAt: response.fetchedAt, - jurisdiction: 'US-FED', - verifiable: false, - unverifiableReason: response.unverifiableReason, - }, - dimensionHints, - metadata: { sourceId, status: response.status, fromCache: response.fromCache }, - } - } - - const html = response.body - const title = extractTitle(html, selector) - const body = extractBody(html, selector) - const effective = extractEffectiveDate(html) ?? response.sourceUpdatedAt - - const verifiable = body.length > 50 - return { - id: fragmentId, - title, - body, - bodyHash: sha256(body), - provenance: { - url, - sourceUpdatedAt: effective, - fetchedAt: response.fetchedAt, - jurisdiction: 'US-FED', - verifiable, - unverifiableReason: verifiable ? undefined : 'extracted body too short', - }, - dimensionHints, - metadata: { sourceId, status: response.status, fromCache: response.fromCache }, - } -} - -function extractTitle(html: string, selector: CornellLiiSelector): string { - const h1 = /<h1[^>]*\bid=["']page_title["'][^>]*>([\s\S]*?)<\/h1>/i.exec(html)?.[1] - if (h1) return htmlToText(h1) - const t = /<title>([\s\S]*?)<\/title>/i.exec(html)?.[1] - if (t) return htmlToText(t).split(' | ')[0] ?? `Cornell LII ${selector.path}` - return `Cornell LII ${selector.kind} ${selector.path}` -} - -function extractBody(html: string, selector: CornellLiiSelector): string { - if (selector.kind === 'uscode') { - // The statute text lives inside a <text><div class="text">…</div></text> - // block on US Code section pages. Prefer it; fall back to #tab_default_1 - // which always contains the section body. - const text = /<text>([\s\S]*?)<\/text>/i.exec(html)?.[1] - if (text) return htmlToText(text) - const tab = innerHtmlById(html, 'tab_default_1') - if (tab) return htmlToText(tab) - } - // Wex pages wrap the encyclopedia entry under <div id="main-content"> (newer - // Drupal template) or directly inside <div id="extracted-content"> (older - // template). Try both — lazy regex matching against a nested-div container - // returns the wrong (shorter) slice, so we anchor on the leaf containers. - const mainContent = innerHtmlById(html, 'main-content') - if (mainContent) { - return htmlToText(mainContent.replace(/<h1[\s\S]*?<\/h1>/i, '')) - } - const extracted = innerHtmlById(html, 'extracted-content') - if (extracted) { - return htmlToText(extracted.replace(/<h1[\s\S]*?<\/h1>/i, '')) - } - return htmlToText(html) -} - -function extractEffectiveDate(html: string): string | undefined { - // Cornell LII includes "Editorial Notes" / "Amendments" blocks with - // dates; the most reliable machine-readable signal is the last - // amendment year embedded near the section text. - const amend = /Amendments[\s\S]{0,200}?(\d{4})/i.exec(html)?.[1] - if (amend) { - const y = Number.parseInt(amend, 10) - if (Number.isFinite(y) && y > 1900 && y <= new Date().getUTCFullYear() + 1) { - return new Date(Date.UTC(y, 11, 31)).toISOString() - } - } - return undefined -} - -function defaultDimensionHints(selector: CornellLiiSelector): string[] { - if (selector.kind === 'uscode') return ['jurisdictional_accuracy', 'citation_hygiene'] - return ['citation_hygiene'] -} diff --git a/src/sources/html.ts b/src/sources/html.ts deleted file mode 100644 index ce6ade5..0000000 --- a/src/sources/html.ts +++ /dev/null @@ -1,89 +0,0 @@ -/** - * Minimal HTML helpers used by the shipped sources. - * - * Deliberately not a full DOM parser: every authority we ship against - * (Cornell LII, IRS.gov, state SOS portals) has well-behaved server-rendered - * HTML where regex-based extraction is correct and cheap. Bringing in cheerio - * would add a 1.5MB dependency to a package whose purpose is shipping - * primitives, not parsing arbitrary web pages. - * - * If a future source needs real DOM traversal, it should depend on its own - * parser locally rather than promoting one into the package-wide deps. - * - * @stable - */ - -/** - * Strip HTML tags, collapse whitespace, decode common entities. - * - * Preserves paragraph and line breaks (`</p>`, `<br>`, `</li>`, `</div>`, - * `</h*>`) as `\n` so statute text retains its subsection structure. - */ -export function htmlToText(html: string): string { - return html - .replace(/<script[\s\S]*?<\/script>/gi, '') - .replace(/<style[\s\S]*?<\/style>/gi, '') - .replace(/<noscript[\s\S]*?<\/noscript>/gi, '') - .replace(/<!--([\s\S]*?)-->/g, '') - .replace(/<\s*br\s*\/?>/gi, '\n') - .replace(/<\/(p|li|div|tr|h[1-6]|blockquote|section|article)>/gi, '\n') - .replace(/<[^>]+>/g, '') - .replace(/ /gi, ' ') - .replace(/&/gi, '&') - .replace(/</gi, '<') - .replace(/>/gi, '>') - .replace(/"/gi, '"') - .replace(/'/gi, "'") - .replace(/§/gi, '§') - .replace(/—/gi, '—') - .replace(/–/gi, '–') - .replace(/&#(\d+);/g, (_, code) => String.fromCodePoint(Number(code))) - .replace(/&#x([0-9a-f]+);/gi, (_, code) => String.fromCodePoint(Number.parseInt(code, 16))) - .split('\n') - .map((line) => line.replace(/[\t  ]+/g, ' ').trim()) - .filter((line, idx, all) => !(line === '' && all[idx - 1] === '')) - .join('\n') - .trim() -} - -/** Extract the first match of a regex's first capture group, or undefined. */ -export function firstMatch(html: string, pattern: RegExp): string | undefined { - return pattern.exec(html)?.[1]?.trim() -} - -/** Extract the inner HTML of the first matching tag with id `id`. */ -export function innerHtmlById(html: string, id: string): string | undefined { - const escaped = id.replace(/[.*+?^${}()|[\]\\]/g, '\\$&') - const tagPattern = new RegExp( - `<([a-z][a-z0-9]*)\\b[^>]*\\sid=["']${escaped}["'][^>]*>([\\s\\S]*?)<\\/\\1>`, - 'i', - ) - return tagPattern.exec(html)?.[2] -} - -/** - * Extract every (href, text) pair matching the URL regex. - * Returns absolute URLs by resolving against `baseUrl`. - */ -export function extractLinks( - html: string, - hrefPattern: RegExp, - baseUrl: string, -): { href: string; text: string }[] { - const out: { href: string; text: string }[] = [] - const anchor = /<a\b[^>]*\shref=["']([^"']+)["'][^>]*>([\s\S]*?)<\/a>/gi - for (const match of html.matchAll(anchor)) { - const href = match[1] - const inner = match[2] - if (!href || !inner) continue - if (!hrefPattern.test(href)) continue - const text = htmlToText(inner) - if (!text) continue - try { - out.push({ href: new URL(href, baseUrl).toString(), text }) - } catch { - /* skip malformed URL */ - } - } - return out -} diff --git a/src/sources/http.ts b/src/sources/http.ts deleted file mode 100644 index fada4cc..0000000 --- a/src/sources/http.ts +++ /dev/null @@ -1,262 +0,0 @@ -import { mkdir, readFile, stat, writeFile } from 'node:fs/promises' -import { dirname, join } from 'node:path' -import { sha256 } from '../ids' - -/** - * Polite HTTP fetcher shared by remote sources. - * - * Independent sources share a per-origin throttle because rate-limited sites - * may return block pages instead of 429 responses. Responses are cached by URL - * because many publishers omit reliable ETag and Last-Modified headers. Bodies - * are checked even after a 2xx response because captcha and block pages often - * use successful status codes. - */ - -/** User-Agent string sent on every outbound request. */ -export const POLITE_USER_AGENT = - 'agent-knowledge (+https://github.com/tangle-network/agent-knowledge)' - -/** Minimum gap between successive requests to the same origin (ms). */ -export const MIN_REQUEST_GAP_MS = 1_000 - -/** Maximum response body we will buffer in memory (bytes). */ -export const MAX_RESPONSE_BYTES = 8 * 1024 * 1024 - -const hostThrottle = new Map<string, Promise<void>>() - -export interface PoliteFetchOptions { - signal?: AbortSignal - cacheDir?: string - /** - * Cache age beyond which we re-fetch. Default 1 hour, long enough to - * batch a cron sweep across many selectors, short enough that hourly - * authoritative-page changes get picked up next tick. - */ - cacheTtlMs?: number - /** - * Extra request headers. The fetcher always sets `User-Agent` and - * `Accept`; callers can add `Accept-Language` etc. - */ - headers?: Record<string, string> -} - -export interface PoliteFetchResult { - url: string - status: number - /** Decoded UTF-8 body. Truncated to `MAX_RESPONSE_BYTES`. */ - body: string - /** - * Best-effort source-attested timestamp. Reads `Last-Modified`, - * falling back to `Date`, falling back to fetch time. Always ISO 8601. - */ - sourceUpdatedAt: string - fetchedAt: string - /** True iff the response was satisfied from disk cache. */ - fromCache: boolean - /** - * False on: non-2xx status, captcha/block page heuristic match, or - * decoded body below 200 chars from a host known to serve real content - * (Cornell, IRS, state SOS). `unverifiableReason` carries the why. - */ - verifiable: boolean - unverifiableReason?: string -} - -/** - * Fetch one URL with per-host throttling, on-disk cache, and block-page - * detection. Never throws on network/HTTP failure. It returns a result with - * `verifiable: false` and `unverifiableReason` set so the caller can decide - * whether to skip, retry, or surface. - * - * Throws ONLY on `AbortError` (caller asked to stop) and on cache-write - * failures that indicate a misconfigured filesystem. - */ -export async function politeFetch( - url: string, - options: PoliteFetchOptions = {}, -): Promise<PoliteFetchResult> { - const cacheTtl = options.cacheTtlMs ?? 60 * 60 * 1000 - const cached = options.cacheDir ? await readCache(options.cacheDir, url, cacheTtl) : undefined - if (cached) return cached - - const host = safeHost(url) - await throttleHost(host) - - const fetchedAt = new Date().toISOString() - let response: Response - try { - response = await fetch(url, { - signal: options.signal, - redirect: 'follow', - headers: { - 'User-Agent': POLITE_USER_AGENT, - Accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', - 'Accept-Language': 'en-US,en;q=0.9', - ...(options.headers ?? {}), - }, - }) - } catch (error) { - if ((error as { name?: string }).name === 'AbortError') throw error - const result: PoliteFetchResult = { - url, - status: 0, - body: '', - sourceUpdatedAt: fetchedAt, - fetchedAt, - fromCache: false, - verifiable: false, - unverifiableReason: `network error: ${(error as Error).message}`, - } - if (options.cacheDir) await writeCache(options.cacheDir, url, result) - return result - } - - const text = await readBoundedText(response) - const lastModified = response.headers.get('last-modified') - const dateHeader = response.headers.get('date') - const sourceUpdatedAt = parseHttpDate(lastModified) ?? parseHttpDate(dateHeader) ?? fetchedAt - - const result: PoliteFetchResult = { - url, - status: response.status, - body: text, - sourceUpdatedAt, - fetchedAt, - fromCache: false, - verifiable: true, - } - - if (response.status < 200 || response.status >= 300) { - result.verifiable = false - result.unverifiableReason = `non-2xx status: ${response.status}` - } else if (looksLikeBlockPage(text)) { - result.verifiable = false - result.unverifiableReason = 'block-page heuristic matched' - } else if (text.length < 200 && knownLargeAuthority(host)) { - result.verifiable = false - result.unverifiableReason = `body shorter than expected (${text.length} chars)` - } - - if (options.cacheDir) await writeCache(options.cacheDir, url, result) - return result -} - -/** Reset the in-process throttle map. Test-only. */ -export function __resetHttpThrottle(): void { - hostThrottle.clear() -} - -function safeHost(url: string): string { - try { - return new URL(url).host - } catch { - return 'unknown' - } -} - -async function throttleHost(host: string): Promise<void> { - const prev = hostThrottle.get(host) ?? Promise.resolve() - let release: () => void = () => {} - const next = new Promise<void>((resolve) => { - release = resolve - }) - hostThrottle.set( - host, - prev.then(() => next), - ) - await prev - setTimeout(release, MIN_REQUEST_GAP_MS) -} - -async function readBoundedText(response: Response): Promise<string> { - if (!response.body) return '' - const reader = response.body.getReader() - const chunks: Uint8Array[] = [] - let total = 0 - while (true) { - const { done, value } = await reader.read() - if (done) break - if (!value) continue - total += value.length - if (total > MAX_RESPONSE_BYTES) { - // Stop reading; release the underlying connection. - await reader.cancel() - break - } - chunks.push(value) - } - const merged = new Uint8Array(Math.min(total, MAX_RESPONSE_BYTES)) - let offset = 0 - for (const chunk of chunks) { - const take = Math.min(chunk.length, merged.length - offset) - if (take <= 0) break - merged.set(chunk.subarray(0, take), offset) - offset += take - } - return new TextDecoder('utf-8', { fatal: false }).decode(merged) -} - -function parseHttpDate(value: string | null): string | undefined { - if (!value) return undefined - const ms = Date.parse(value) - return Number.isFinite(ms) ? new Date(ms).toISOString() : undefined -} - -/** Cheap heuristic that catches CAPTCHA, WAF block pages, and "Just a moment" interstitials. */ -export function looksLikeBlockPage(body: string): boolean { - if (!body) return false - const lower = body.toLowerCase() - const markers = [ - 'verify you are human', - 'please enable javascript and cookies', - 'just a moment', - 'access denied', - 'request unsuccessful', - 'cf-error-details', - 'captcha', - 'incapsula', - 'pardon our interruption', - ] - for (const marker of markers) { - if (lower.includes(marker)) return true - } - return false -} - -function knownLargeAuthority(host: string): boolean { - return ( - host.endsWith('law.cornell.edu') || - host.endsWith('irs.gov') || - host.endsWith('sos.ca.gov') || - host.endsWith('sos.state.tx.us') || - host.endsWith('sos.state.us') - ) -} - -function cachePath(cacheDir: string, url: string): string { - const key = sha256(url) - return join(cacheDir, 'http', `${key.slice(0, 2)}`, `${key}.json`) -} - -async function readCache( - cacheDir: string, - url: string, - ttlMs: number, -): Promise<PoliteFetchResult | undefined> { - const path = cachePath(cacheDir, url) - try { - const info = await stat(path) - if (Date.now() - info.mtimeMs > ttlMs) return undefined - const raw = await readFile(path, 'utf8') - const parsed = JSON.parse(raw) as PoliteFetchResult - return { ...parsed, fromCache: true } - } catch { - return undefined - } -} - -async function writeCache(cacheDir: string, url: string, value: PoliteFetchResult): Promise<void> { - const path = cachePath(cacheDir, url) - await mkdir(dirname(path), { recursive: true }) - await writeFile(path, JSON.stringify(value), 'utf8') -} diff --git a/src/sources/index.ts b/src/sources/index.ts deleted file mode 100644 index 03b1a4b..0000000 --- a/src/sources/index.ts +++ /dev/null @@ -1,17 +0,0 @@ -/** - * Pluggable knowledge sources. - * - * @stable types — `KnowledgeSource`, `KnowledgeFragment`, `FetchOpts`, - * `FragmentProvenance` - * @stable http — `politeFetch`, `looksLikeBlockPage`, `POLITE_USER_AGENT` - * @stable html — `htmlToText`, `extractLinks` - * @stable shipped sources — `createCornellLiiSource`, - * `createIrsPublicationsSource`, `createStateSosSource` - */ - -export * from './cornell-lii' -export * from './html' -export * from './http' -export * from './irs-publications' -export * from './state-sos' -export * from './types' diff --git a/src/sources/irs-publications.ts b/src/sources/irs-publications.ts deleted file mode 100644 index c7cbf2d..0000000 --- a/src/sources/irs-publications.ts +++ /dev/null @@ -1,223 +0,0 @@ -import { sha256 } from '../ids' -import { htmlToText } from './html' -import { politeFetch } from './http' -import type { FetchOpts, KnowledgeFragment, KnowledgeSource } from './types' - -/** - * IRS publications source. - * - * Two surfaces: - * - * 1. The publications index at https://www.irs.gov/publications enumerates - * every active publication with its revision year — a single fragment - * with the full table lets change detection notice when a publication - * year flips (e.g. Pub 15 (2025) → Pub 15 (2026)). - * - * 2. Individual publication landing pages at /publications/p<N>[<suffix>] - * return one fragment per publication with summary text. Callers list - * the publications they need tracked via `selectors`. - * - * Revenue procedures are fetched under their numbered URLs; the IRS does - * not maintain a stable HTML index of rev-procs, so the caller passes the - * specific rev-proc paths they care about. - * - * @stable - */ - -const BASE_URL = 'https://www.irs.gov' -const INDEX_URL = `${BASE_URL}/publications` - -export interface IrsPublicationsSourceOptions { - /** - * Specific publication slugs to fetch (e.g. `['p15', 'p17', 'p463']`). - * When `includeIndex` is true (default), the publications index page is - * also fetched as a single fragment so change detection can notice - * year/revision shifts across the whole catalogue. - */ - publications?: string[] - /** - * Revenue procedure paths to fetch (e.g. `['/irb/2024-31_IRB']`). The - * caller passes the exact path; this source does not auto-discover. - */ - revenueProcedures?: string[] - includeIndex?: boolean - id?: string -} - -/** Default eval dimensions for IRS-sourced fragments. */ -export const IRS_DIMENSION_HINTS = ['tax_compliance', 'regulatory_currency', 'citation_hygiene'] - -export function createIrsPublicationsSource( - options: IrsPublicationsSourceOptions = {}, -): KnowledgeSource { - const id = options.id ?? 'irs-publications' - const includeIndex = options.includeIndex ?? true - return { - id, - name: 'IRS Publications', - description: - 'Internal Revenue Service publications index and individual publication landing pages from irs.gov.', - async fetch(opts: FetchOpts): Promise<KnowledgeFragment[]> { - const out: KnowledgeFragment[] = [] - const limit = opts.limit ?? Number.POSITIVE_INFINITY - - if (includeIndex && out.length < limit) { - out.push(await fetchIndex(id, opts)) - } - for (const slug of options.publications ?? []) { - if (out.length >= limit) break - out.push(await fetchPublication(id, slug, opts)) - } - for (const path of options.revenueProcedures ?? []) { - if (out.length >= limit) break - out.push(await fetchRevenueProcedure(id, path, opts)) - } - return out - }, - } -} - -async function fetchIndex(sourceId: string, opts: FetchOpts): Promise<KnowledgeFragment> { - const response = await politeFetch(INDEX_URL, { signal: opts.signal, cacheDir: opts.cacheDir }) - const tablePattern = /<table[\s\S]*?<\/table>/gi - const matches = response.body.match(tablePattern) ?? [] - // Extract the table that lists current-year publications. IRS publishes - // one table per year on the index; the most recent table is always the - // first that mentions a year ≥ current. - const tables = matches.map((t) => htmlToText(t)) - const body = tables - .filter((t) => /Publication\s*\d+/i.test(t)) - .join('\n\n') - .slice(0, 200_000) - - const verifiable = response.verifiable && body.length > 200 - return { - id: 'index', - title: 'IRS Publications Index', - body, - bodyHash: sha256(body), - provenance: { - url: INDEX_URL, - sourceUpdatedAt: response.sourceUpdatedAt, - fetchedAt: response.fetchedAt, - jurisdiction: 'US-FED', - verifiable, - unverifiableReason: - response.unverifiableReason ?? (verifiable ? undefined : 'no publication rows extracted'), - }, - dimensionHints: IRS_DIMENSION_HINTS, - metadata: { sourceId, status: response.status, fromCache: response.fromCache, kind: 'index' }, - } -} - -async function fetchPublication( - sourceId: string, - slug: string, - opts: FetchOpts, -): Promise<KnowledgeFragment> { - const url = `${BASE_URL}/publications/${slug.replace(/^\/+/, '')}` - const response = await politeFetch(url, { signal: opts.signal, cacheDir: opts.cacheDir }) - - const title = extractTitle(response.body, `IRS Publication ${slug}`) - const body = extractMainContent(response.body) - const verifiable = response.verifiable && body.length > 200 - - return { - id: `publication:${slug}`, - title, - body, - bodyHash: sha256(body), - provenance: { - url, - sourceUpdatedAt: extractRevisionDate(response.body) ?? response.sourceUpdatedAt, - fetchedAt: response.fetchedAt, - jurisdiction: 'US-FED', - verifiable, - unverifiableReason: - response.unverifiableReason ?? (verifiable ? undefined : 'no publication body extracted'), - }, - dimensionHints: IRS_DIMENSION_HINTS, - metadata: { - sourceId, - status: response.status, - fromCache: response.fromCache, - kind: 'publication', - slug, - }, - } -} - -async function fetchRevenueProcedure( - sourceId: string, - path: string, - opts: FetchOpts, -): Promise<KnowledgeFragment> { - const url = `${BASE_URL}${path.startsWith('/') ? path : `/${path}`}` - const response = await politeFetch(url, { signal: opts.signal, cacheDir: opts.cacheDir }) - const body = extractMainContent(response.body) - const verifiable = response.verifiable && body.length > 200 - return { - id: `rev-proc:${path}`, - title: extractTitle(response.body, `IRS Revenue Procedure ${path}`), - body, - bodyHash: sha256(body), - provenance: { - url, - sourceUpdatedAt: response.sourceUpdatedAt, - fetchedAt: response.fetchedAt, - jurisdiction: 'US-FED', - verifiable, - unverifiableReason: - response.unverifiableReason ?? - (verifiable ? undefined : 'no revenue-procedure body extracted'), - }, - dimensionHints: [...IRS_DIMENSION_HINTS, 'procedural_currency'], - metadata: { - sourceId, - status: response.status, - fromCache: response.fromCache, - kind: 'rev-proc', - path, - }, - } -} - -function extractTitle(html: string, fallback: string): string { - const og = /<meta\s+property=["']og:title["']\s+content=["']([^"']+)["']/i.exec(html)?.[1] - if (og) return decodeHtml(og) - const title = /<title>([\s\S]*?)<\/title>/i.exec(html)?.[1] - if (title) return htmlToText(title).split(' | ')[0] ?? fallback - return fallback -} - -function extractMainContent(html: string): string { - // IRS uses Drupal — the main publication body is inside <main role="main"> - // or under .field--name-body. We try main first; on miss, body. - const main = /<main\b[\s\S]*?<\/main>/i.exec(html)?.[0] - if (main) { - const noNav = main - .replace(/<nav[\s\S]*?<\/nav>/gi, '') - .replace(/<header[\s\S]*?<\/header>/gi, '') - .replace(/<footer[\s\S]*?<\/footer>/gi, '') - return htmlToText(noNav).slice(0, 200_000) - } - const body = /<body\b[\s\S]*?<\/body>/i.exec(html)?.[0] - return body ? htmlToText(body).slice(0, 200_000) : htmlToText(html).slice(0, 200_000) -} - -function extractRevisionDate(html: string): string | undefined { - // IRS publication pages typically show "Publication X (YYYY)" in the title; - // pulling the year gives a stable revision marker. - const m = /Publication\s+\S+\s*\((\d{4})\)/i.exec(html) - if (m?.[1]) { - const year = Number.parseInt(m[1], 10) - if (Number.isFinite(year) && year >= 2000 && year <= new Date().getUTCFullYear() + 1) { - return new Date(Date.UTC(year, 0, 1)).toISOString() - } - } - return undefined -} - -function decodeHtml(value: string): string { - return htmlToText(value) -} diff --git a/src/sources/state-sos.ts b/src/sources/state-sos.ts deleted file mode 100644 index 40268c5..0000000 --- a/src/sources/state-sos.ts +++ /dev/null @@ -1,151 +0,0 @@ -import { sha256 } from '../ids' -import { htmlToText } from './html' -import { politeFetch } from './http' -import type { FetchOpts, KnowledgeFragment, KnowledgeSource } from './types' - -/** - * Generic Secretary-of-State source. - * - * Every US state SOS surfaces LLC/Corp formation requirements differently - * (CA via static forms pages, DE via division of corporations pages, TX - * via SOSDirect content pages). Rather than baking 50 state-specific - * parsers into this package, the source takes a config that names the URL - * pattern + CSS-equivalent selector + jurisdiction tag. Callers supply one - * config per state they need tracked. - * - * The selector is interpreted as a substring/regex of an HTML element id - * or class — see `StateSosSourceConfig` for the contract. This is - * intentionally minimal; richer extraction belongs in a state-specific - * adapter the consumer authors. - * - * @experimental Interface will likely grow as we add more state coverage. - */ - -export interface StateSosEntity { - /** Stable id for this fragment within the state (e.g. 'llc-formation', 'corp-formation'). */ - id: string - /** Path under the configured `baseUrl` for this entity. */ - path: string - /** - * Extraction selector. Choose one: - * - `{ kind: 'id', value: 'main-content' }` — innermost match of element with that id - * - `{ kind: 'class', value: 'field--name-body' }` — innermost match of element with that class - * - `{ kind: 'regex', value: /<article[\s\S]*?<\/article>/i }` — raw regex - * - `{ kind: 'whole' }` — full body, tags stripped (fallback for unstructured pages) - */ - selector: - | { kind: 'id'; value: string } - | { kind: 'class'; value: string } - | { kind: 'regex'; value: RegExp } - | { kind: 'whole' } - title: string - /** Eval dimensions this entity feeds. */ - dimensionHints?: string[] -} - -export interface StateSosSourceConfig { - /** US state postal code, e.g. 'CA', 'DE', 'TX'. */ - state: string - /** Base URL for the state SOS — e.g. 'https://www.sos.ca.gov'. */ - baseUrl: string - /** Entities this state exposes (LLC, Corp, etc). */ - entities: StateSosEntity[] - /** Source id; default `state-sos:<state>`. */ - id?: string - /** Display name; default `<state> Secretary of State`. */ - name?: string -} - -export function createStateSosSource(config: StateSosSourceConfig): KnowledgeSource { - const id = config.id ?? `state-sos:${config.state.toLowerCase()}` - const name = config.name ?? `${config.state} Secretary of State` - return { - id, - name, - description: `${config.state} Secretary of State filings and formation guidance pages.`, - async fetch(opts: FetchOpts): Promise<KnowledgeFragment[]> { - const limit = opts.limit ?? config.entities.length - const entities = config.entities.slice(0, limit) - const out: KnowledgeFragment[] = [] - for (const entity of entities) { - out.push(await fetchEntity(id, config, entity, opts)) - } - return out - }, - } -} - -async function fetchEntity( - sourceId: string, - config: StateSosSourceConfig, - entity: StateSosEntity, - opts: FetchOpts, -): Promise<KnowledgeFragment> { - const url = joinUrl(config.baseUrl, entity.path) - const response = await politeFetch(url, { signal: opts.signal, cacheDir: opts.cacheDir }) - - const body = response.verifiable ? extractBySelector(response.body, entity.selector) : '' - const verifiable = response.verifiable && body.length > 100 - - return { - id: entity.id, - title: entity.title, - body, - bodyHash: sha256(body), - provenance: { - url, - sourceUpdatedAt: response.sourceUpdatedAt, - fetchedAt: response.fetchedAt, - jurisdiction: `US-${config.state.toUpperCase()}`, - verifiable, - unverifiableReason: - response.unverifiableReason ?? (verifiable ? undefined : 'extracted body too short'), - }, - dimensionHints: entity.dimensionHints ?? [ - 'jurisdictional_accuracy', - 'corporate_formation', - 'citation_hygiene', - ], - metadata: { - sourceId, - status: response.status, - fromCache: response.fromCache, - state: config.state, - }, - } -} - -function extractBySelector(html: string, selector: StateSosEntity['selector']): string { - if (selector.kind === 'whole') { - const main = /<main\b[\s\S]*?<\/main>/i.exec(html)?.[0] - return htmlToText(main ?? html).slice(0, 200_000) - } - if (selector.kind === 'regex') { - const m = selector.value.exec(html)?.[0] - return m ? htmlToText(m).slice(0, 200_000) : '' - } - if (selector.kind === 'id') { - const escaped = selector.value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&') - const pattern = new RegExp( - `<([a-z][a-z0-9]*)\\b[^>]*\\sid=["']${escaped}["'][^>]*>([\\s\\S]*?)<\\/\\1>`, - 'i', - ) - const inner = pattern.exec(html)?.[2] - return inner ? htmlToText(inner).slice(0, 200_000) : '' - } - const escaped = selector.value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&') - const pattern = new RegExp( - `<([a-z][a-z0-9]*)\\b[^>]*\\sclass=["'][^"']*\\b${escaped}\\b[^"']*["'][^>]*>([\\s\\S]*?)<\\/\\1>`, - 'i', - ) - const inner = pattern.exec(html)?.[2] - return inner ? htmlToText(inner).slice(0, 200_000) : '' -} - -function joinUrl(base: string, path: string): string { - try { - return new URL(path, base.endsWith('/') ? base : `${base}/`).toString() - } catch { - return `${base.replace(/\/+$/, '')}/${path.replace(/^\/+/, '')}` - } -} diff --git a/src/sources/types.ts b/src/sources/types.ts deleted file mode 100644 index 38b7789..0000000 --- a/src/sources/types.ts +++ /dev/null @@ -1,153 +0,0 @@ -/** - * Pluggable knowledge source contract. - * - * A `KnowledgeSource` is one external provider of authoritative content that - * an agent's knowledge base should track over time (e.g. Cornell LII US Code, - * IRS publications, a state secretary-of-state filing portal). It returns - * hashable, embed-ready `KnowledgeFragment`s plus enough provenance metadata - * for downstream consumers to: - * - * 1. detect change against a previous snapshot (see `./changes`) - * 2. score freshness on a per-source-id basis (see `./freshness`) - * 3. decide which evals to re-run when the underlying authority moves - * (the `dimensionHints` field is the binding contract for that decision) - * - * Sources MUST be pure with respect to local filesystem state outside the - * cache directory the caller hands them — they read remote authorities and - * return data. They MUST mark `verifiable: false` on any fragment they could - * not fetch and extract (block page, 4xx, parse failure) rather than silently - * substituting empty or partial content. The control loop downstream uses - * `verifiable` to refuse promotion of unusable content. - * - * @stable - */ - -/** - * Per-fetch options the host (control loop / cron / CLI) passes in. - * - * `signal` lets the host abort long-running fetches (rate-limited authority, - * congested network). `cacheDir` is where the source SHOULD write its disk - * cache; an undefined value disables caching (useful in tests). `now` is - * injected for deterministic tests of change-detection windows. - */ -export interface FetchOpts { - /** Abort signal forwarded to the underlying HTTP fetcher. */ - signal?: AbortSignal - /** Absolute path under which the source may cache raw bytes. */ - cacheDir?: string - /** Clock injection for deterministic tests. */ - now?: () => Date - /** - * Maximum number of authority pages the source should fetch in this call. - * Sources MUST respect this bound — exhaustively crawling Cornell LII on - * every cron tick would be both rude and slow. Default is source-specific. - */ - limit?: number - /** - * Source-specific selector string. Examples: - * - cornell-lii: `'uscode/text/18/1836'` or `'wex/non-compete'` - * - irs-publications: `'index'` or `'p15'` - * - state-sos: opaque, see `StateSosSourceConfig` - * - * Sources that don't need a selector ignore this field. - */ - selector?: string -} - -/** - * The standard provenance shape every fragment carries. Kept separate from - * `KnowledgeFragment` so freshness/change code can pass it around without - * also dragging the body text. - */ -export interface FragmentProvenance { - /** Canonical URL the fragment was extracted from. */ - url: string - /** - * Source-attested timestamp: the time the AUTHORITY last updated this - * content, as reported by the source (Last-Modified header, in-page - * effective date, registry generated-at, etc). Falls back to the fetch - * time only when the authority publishes no timestamp. - */ - sourceUpdatedAt: string - /** ISO timestamp the fragment was fetched. */ - fetchedAt: string - /** - * Jurisdiction the content is binding within, if applicable. Use ISO - * country code, US state abbreviation, or 'US-FED' for federal scope. - * Statute sources MUST populate this; reference / encyclopedia sources - * MAY leave it undefined. - */ - jurisdiction?: string - /** - * True iff the configured URL returned an acceptable response and the - * expected content was extracted. False on a block page, rate-limit - * response, 4xx/5xx, or selector miss. This is not publisher authentication - * or cryptographic content verification. Consumers MUST refuse to promote - * `verifiable: false` fragments into citable knowledge. - */ - verifiable: boolean - /** If `verifiable === false`, the reason — surfaced to operators. */ - unverifiableReason?: string -} - -/** - * One unit of authoritative content. Stable hash on `(id, body)` lets change - * detection reason about identity across snapshots. - */ -export interface KnowledgeFragment { - /** - * Stable identity within (sourceId, selector-space). Two fetches against - * the same authority section MUST produce the same `id`. The (sourceId, - * id) pair is the primary key for change detection. - */ - id: string - /** Free-form title — section heading, publication name, etc. */ - title: string - /** Body text, normalised: no HTML tags, line breaks preserved. */ - body: string - /** SHA-256 of `body`. Pre-computed so consumers don't re-hash on diff. */ - bodyHash: string - provenance: FragmentProvenance - /** - * Eval dimensions an agent-eval campaign should re-score when this - * fragment changes. Examples: `citation_hygiene`, `jurisdictional_accuracy`, - * `tax_compliance`, `regulatory_currency`. The eval cron treats this as a - * set, not a contract — adding a new dimension is non-breaking. - * - * This is the load-bearing field for the continuous-ingestion story: a - * Ryan-LLC-style ruling vacates the FTC non-compete rule → the source - * returns a fragment with `jurisdictional_accuracy` in this list → - * `detectChanges()` emits a `KnowledgeChange` carrying that hint → the - * cron knows exactly which agent-eval campaigns to re-run. - */ - dimensionHints: string[] - /** Arbitrary source-specific metadata for debugging / connector wiring. */ - metadata?: Record<string, unknown> -} - -/** - * One pluggable knowledge source. - * - * Implementations: see `./cornell-lii`, `./irs-publications`, `./state-sos`. - * To author a new source, follow the same shape and register it in your - * application's source list — there is no global registry by design (per - * the per-tenant isolation contract; see README). - */ -export interface KnowledgeSource { - /** Stable id — used to key freshness state. MUST NOT change once shipped. */ - id: string - /** Human-readable name for dashboards. */ - name: string - /** One-sentence description: what authority + scope. */ - description: string - /** - * Pull fragments for this source. Sources MUST: - * - rate-limit themselves (>=1 req/sec per source by convention) - * - send a polite User-Agent - * - cache to disk when `opts.cacheDir` is set - * - mark `verifiable: false` rather than throwing on parse/block - * - honour `opts.signal` - * - honour `opts.limit` - */ - fetch(opts: FetchOpts): Promise<KnowledgeFragment[]> -} diff --git a/tests/changes.test.ts b/tests/changes.test.ts deleted file mode 100644 index 46487a4..0000000 --- a/tests/changes.test.ts +++ /dev/null @@ -1,121 +0,0 @@ -import { describe, expect, it } from 'vitest' -import { detectChanges } from '../src/changes' -import { sha256 } from '../src/ids' -import type { KnowledgeFragment } from '../src/sources/types' - -function fragment( - id: string, - body: string, - opts: Partial<KnowledgeFragment & { hints: string[]; verifiable: boolean }> = {}, -): KnowledgeFragment { - return { - id, - title: opts.title ?? id, - body, - bodyHash: sha256(body), - provenance: { - url: opts.provenance?.url ?? `https://example.test/${id}`, - sourceUpdatedAt: opts.provenance?.sourceUpdatedAt ?? '2026-05-14T12:00:00.000Z', - fetchedAt: opts.provenance?.fetchedAt ?? '2026-05-14T12:00:00.000Z', - jurisdiction: opts.provenance?.jurisdiction, - verifiable: opts.verifiable ?? opts.provenance?.verifiable ?? true, - }, - dimensionHints: opts.hints ?? opts.dimensionHints ?? ['citation_hygiene'], - } -} - -describe('detectChanges', () => { - it('flags an added fragment with after-body diff', () => { - const result = detectChanges([], [fragment('wex:non-compete', 'BODY-V1')]) - expect(result.summary).toEqual({ added: 1, removed: 0, modified: 0 }) - expect(result.changes[0]?.kind).toBe('added') - expect(result.changes[0]?.diff?.after).toBe('BODY-V1') - expect(result.changes[0]?.diff?.before).toBeUndefined() - }) - - it('flags a removed fragment with before-body diff', () => { - const result = detectChanges([fragment('uscode:18/1836', 'BODY-V1')], []) - expect(result.summary).toEqual({ added: 0, removed: 1, modified: 0 }) - expect(result.changes[0]?.kind).toBe('removed') - expect(result.changes[0]?.diff?.before).toBe('BODY-V1') - expect(result.changes[0]?.diff?.after).toBeUndefined() - }) - - it('flags a modification when body hash changes', () => { - const prev = [fragment('wex:non-compete', 'BEFORE')] - const next = [fragment('wex:non-compete', 'AFTER')] - const result = detectChanges(prev, next) - expect(result.summary).toEqual({ added: 0, removed: 0, modified: 1 }) - expect(result.changes[0]?.kind).toBe('modified') - expect(result.changes[0]?.diff).toEqual({ before: 'BEFORE', after: 'AFTER' }) - }) - - it('does not flag identical-hash fragments', () => { - const result = detectChanges( - [fragment('wex:non-compete', 'SAME')], - [fragment('wex:non-compete', 'SAME')], - ) - expect(result.summary).toEqual({ added: 0, removed: 0, modified: 0 }) - }) - - it('unions and dedupes dimension hints across before/after', () => { - const prev = [fragment('wex:non-compete', 'BEFORE', { hints: ['citation_hygiene'] })] - const next = [ - fragment('wex:non-compete', 'AFTER', { - hints: ['citation_hygiene', 'jurisdictional_accuracy'], - }), - ] - const result = detectChanges(prev, next) - expect(result.changes[0]?.affectedDimensions.sort()).toEqual([ - 'citation_hygiene', - 'jurisdictional_accuracy', - ]) - }) - - it('drops unverifiable fragments before diffing (no false `removed`)', () => { - const real = fragment('wex:non-compete', 'REAL', { hints: ['jurisdictional_accuracy'] }) - const blocked = fragment('wex:non-compete', '', { verifiable: false, hints: [] }) - const result = detectChanges([real], [blocked]) - expect(result.summary).toEqual({ added: 0, removed: 1, modified: 0 }) - expect(result.warnings.join('\n')).toMatch(/dropped 1 unverifiable/) - }) - - it('filterDimensions narrows the result set', () => { - const next = [ - fragment('a', 'A', { hints: ['citation_hygiene'] }), - fragment('b', 'B', { hints: ['jurisdictional_accuracy'] }), - ] - const result = detectChanges([], next, { filterDimensions: ['jurisdictional_accuracy'] }) - expect(result.changes).toHaveLength(1) - expect(result.changes[0]?.fragmentId).toBe('b') - expect(result.summary).toEqual({ added: 1, removed: 0, modified: 0 }) - }) - - it('warns on duplicate fragment ids', () => { - const result = detectChanges([], [fragment('dup', 'A'), fragment('dup', 'B')]) - expect(result.warnings.join('\n')).toMatch(/duplicate fragment id dup/) - expect(result.changes[0]?.diff?.after).toBe('B') - }) - - it('emits `modified` change tagged for the eval-cron worked example', () => { - // Worked example from the README/PR body: Cornell LII Wex non-compete - // page changes after Ryan-LLC v. FTC. The KnowledgeChange the eval cron - // consumes carries `jurisdictional_accuracy` so it knows to re-run the - // legal-compliance campaign. - const prev = [ - fragment('wex:non-compete', 'Federal non-compete rule effective 2024-09-04', { - hints: ['jurisdictional_accuracy'], - }), - ] - const next = [ - fragment( - 'wex:non-compete', - 'On 2024-08-20 the U.S. District Court for the Northern District of Texas set aside the FTC rule', - { hints: ['jurisdictional_accuracy'] }, - ), - ] - const result = detectChanges(prev, next) - expect(result.summary.modified).toBe(1) - expect(result.changes[0]?.affectedDimensions).toContain('jurisdictional_accuracy') - }) -}) diff --git a/tests/freshness.test.ts b/tests/freshness.test.ts deleted file mode 100644 index fd5db55..0000000 --- a/tests/freshness.test.ts +++ /dev/null @@ -1,159 +0,0 @@ -import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises' -import { tmpdir } from 'node:os' -import { join } from 'node:path' -import { describe, expect, it } from 'vitest' -import { - createD1FreshnessStoreStub, - createFileSystemFreshnessStore, - type D1Adapter, - type FreshnessRecord, -} from '../src/freshness' - -async function withTempRoot<T>(fn: (root: string) => Promise<T>): Promise<T> { - const root = await mkdtemp(join(tmpdir(), 'agent-knowledge-freshness-')) - try { - return await fn(root) - } finally { - await rm(root, { recursive: true, force: true }) - } -} - -describe('createFileSystemFreshnessStore', () => { - it('starts empty — every source is stale', async () => { - await withTempRoot(async (root) => { - const store = createFileSystemFreshnessStore({ root }) - expect(await store.last({ workspaceId: 'w1', sourceId: 'cornell-lii' })).toBeNull() - expect(await store.stale({ workspaceId: 'w1', sourceId: 'cornell-lii', ttlMs: 60_000 })).toBe( - true, - ) - }) - }) - - it('round-trips mark → last → stale=false within TTL', async () => { - await withTempRoot(async (root) => { - const store = createFileSystemFreshnessStore({ root }) - const when = new Date('2026-05-14T12:00:00.000Z') - await store.mark({ workspaceId: 'w1', sourceId: 'cornell-lii', when, contentHash: 'abc' }) - - expect(await store.last({ workspaceId: 'w1', sourceId: 'cornell-lii' })).toEqual(when) - expect( - await store.stale({ - workspaceId: 'w1', - sourceId: 'cornell-lii', - ttlMs: 60_000, - now: new Date('2026-05-14T12:00:30.000Z'), - }), - ).toBe(false) - }) - }) - - it('reports stale once TTL elapses', async () => { - await withTempRoot(async (root) => { - const store = createFileSystemFreshnessStore({ root }) - const when = new Date('2026-05-14T12:00:00.000Z') - await store.mark({ workspaceId: 'w1', sourceId: 'cornell-lii', when }) - expect( - await store.stale({ - workspaceId: 'w1', - sourceId: 'cornell-lii', - ttlMs: 60_000, - now: new Date('2026-05-14T12:02:00.000Z'), - }), - ).toBe(true) - }) - }) - - it('isolates workspaces — w2 cannot read w1 freshness', async () => { - await withTempRoot(async (root) => { - const store = createFileSystemFreshnessStore({ root }) - await store.mark({ - workspaceId: 'w1', - sourceId: 'cornell-lii', - when: new Date('2026-05-14T12:00:00.000Z'), - }) - expect(await store.last({ workspaceId: 'w2', sourceId: 'cornell-lii' })).toBeNull() - expect(await store.stale({ workspaceId: 'w2', sourceId: 'cornell-lii', ttlMs: 60_000 })).toBe( - true, - ) - }) - }) - - it('list returns only that workspace', async () => { - await withTempRoot(async (root) => { - const store = createFileSystemFreshnessStore({ root }) - const t = new Date('2026-05-14T12:00:00.000Z') - await store.mark({ workspaceId: 'w1', sourceId: 'cornell-lii', when: t }) - await store.mark({ workspaceId: 'w1', sourceId: 'irs-publications', when: t }) - await store.mark({ workspaceId: 'w2', sourceId: 'cornell-lii', when: t }) - - const w1 = await store.list('w1') - expect(w1.map((r) => r.sourceId).sort()).toEqual(['cornell-lii', 'irs-publications']) - - const w2 = await store.list('w2') - expect(w2.map((r) => r.sourceId)).toEqual(['cornell-lii']) - }) - }) - - it('serializes concurrent marks without losing writes', async () => { - await withTempRoot(async (root) => { - const stores = [ - createFileSystemFreshnessStore({ root }), - createFileSystemFreshnessStore({ root }), - createFileSystemFreshnessStore({ root }), - ] - const t = new Date('2026-05-14T12:00:00.000Z') - await Promise.all([ - stores[0]!.mark({ workspaceId: 'w1', sourceId: 'a', when: t }), - stores[1]!.mark({ workspaceId: 'w1', sourceId: 'b', when: t }), - stores[2]!.mark({ workspaceId: 'w1', sourceId: 'c', when: t }), - ]) - const list = await stores[0]!.list('w1') - expect(list.map((r) => r.sourceId).sort()).toEqual(['a', 'b', 'c']) - }) - }) - - it('does not replace malformed freshness data', async () => { - await withTempRoot(async (root) => { - const store = createFileSystemFreshnessStore({ root }) - await store.mark({ - workspaceId: 'w1', - sourceId: 'first', - when: new Date('2026-05-14T12:00:00.000Z'), - }) - const path = join(root, '.agent-knowledge', 'freshness.json') - await writeFile(path, '{broken') - - await expect( - store.mark({ - workspaceId: 'w1', - sourceId: 'second', - when: new Date('2026-05-14T12:01:00.000Z'), - }), - ).rejects.toThrow() - await expect(readFile(path, 'utf8')).resolves.toBe('{broken') - }) - }) -}) - -describe('createD1FreshnessStoreStub', () => { - it('delegates last/mark/stale to the adapter', async () => { - const records: Record<string, FreshnessRecord> = {} - const adapter: D1Adapter = { - async get(workspaceId, sourceId) { - return records[`${workspaceId}::${sourceId}`] ?? null - }, - async upsert(record) { - records[`${record.workspaceId}::${record.sourceId}`] = record - }, - async listByWorkspace(workspaceId) { - return Object.values(records).filter((r) => r.workspaceId === workspaceId) - }, - } - const store = createD1FreshnessStoreStub(adapter) - expect(await store.last({ workspaceId: 'w1', sourceId: 'irs-publications' })).toBeNull() - const when = new Date('2026-05-14T12:00:00.000Z') - await store.mark({ workspaceId: 'w1', sourceId: 'irs-publications', when }) - expect(await store.last({ workspaceId: 'w1', sourceId: 'irs-publications' })).toEqual(when) - expect(records['w1::irs-publications']?.lastRefreshedAt).toBe(when.toISOString()) - }) -}) diff --git a/tests/http-cache.test.ts b/tests/http-cache.test.ts deleted file mode 100644 index 1df889f..0000000 --- a/tests/http-cache.test.ts +++ /dev/null @@ -1,192 +0,0 @@ -import { mkdir, mkdtemp, readdir, readFile, rm, writeFile } from 'node:fs/promises' -import { tmpdir } from 'node:os' -import { join } from 'node:path' -import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' -import { sha256 } from '../src/ids' -import { __resetHttpThrottle, POLITE_USER_AGENT, politeFetch } from '../src/sources/index' - -let cacheDir: string -const originalFetch = globalThis.fetch - -beforeEach(async () => { - __resetHttpThrottle() - cacheDir = await mkdtemp(join(tmpdir(), 'agent-knowledge-http-cache-')) -}) - -afterEach(async () => { - await rm(cacheDir, { recursive: true, force: true }) - globalThis.fetch = originalFetch -}) - -function mockFetch(handler: (url: string, init?: RequestInit) => Response): void { - globalThis.fetch = vi.fn(async (input: RequestInfo | URL, init?: RequestInit) => { - const url = typeof input === 'string' ? input : input.toString() - return handler(url, init) - }) as unknown as typeof globalThis.fetch -} - -function html(body: string, status = 200, headers: Record<string, string> = {}): Response { - return new Response(body, { - status, - headers: { 'content-type': 'text/html', ...headers }, - }) -} - -describe('politeFetch', () => { - it('identifies the package without embedding a stale release version', async () => { - let headers: Headers | undefined - mockFetch((_url, init) => { - headers = new Headers(init?.headers) - return html(`<html><body>${'X'.repeat(500)}</body></html>`) - }) - - await politeFetch('https://headers.test/source') - - expect(POLITE_USER_AGENT).toBe( - 'agent-knowledge (+https://github.com/tangle-network/agent-knowledge)', - ) - expect(headers?.get('User-Agent')).toBe(POLITE_USER_AGENT) - }) - - it('returns verifiable=true for a normal 200', async () => { - mockFetch(() => - html(`<html><body>${'X'.repeat(500)}</body></html>`, 200, { - 'last-modified': 'Wed, 01 Jan 2025 00:00:00 GMT', - }), - ) - const result = await politeFetch('https://www.law.cornell.edu/uscode/text/18/1836') - expect(result.status).toBe(200) - expect(result.verifiable).toBe(true) - expect(result.unverifiableReason).toBeUndefined() - expect(result.sourceUpdatedAt).toBe('2025-01-01T00:00:00.000Z') - }) - - it('returns verifiable=false on 404 with reason', async () => { - mockFetch(() => html('Not Found', 404)) - const result = await politeFetch('https://www.law.cornell.edu/uscode/text/99/9999') - expect(result.verifiable).toBe(false) - expect(result.unverifiableReason).toMatch(/non-2xx status: 404/) - }) - - it('returns verifiable=false on block page even with 200', async () => { - mockFetch(() => - html( - `<html><body>${'pad '.repeat(100)}Just a moment — please enable JavaScript</body></html>`, - ), - ) - const result = await politeFetch('https://www.law.cornell.edu/wex/non-compete') - expect(result.status).toBe(200) - expect(result.verifiable).toBe(false) - expect(result.unverifiableReason).toMatch(/block-page heuristic/) - }) - - it('returns verifiable=false on short body from known authority', async () => { - mockFetch(() => html('too short')) - const result = await politeFetch('https://www.irs.gov/publications') - expect(result.verifiable).toBe(false) - expect(result.unverifiableReason).toMatch(/body shorter than expected/) - }) - - it('writes to disk cache and serves the second call from cache', async () => { - const calls: string[] = [] - mockFetch((url) => { - calls.push(url) - return html(`<html><body>${'X'.repeat(500)}</body></html>`) - }) - const url = 'https://www.law.cornell.edu/uscode/text/18/1836' - const a = await politeFetch(url, { cacheDir }) - const b = await politeFetch(url, { cacheDir }) - expect(calls).toHaveLength(1) - expect(a.fromCache).toBe(false) - expect(b.fromCache).toBe(true) - expect(b.body).toBe(a.body) - }) - - it('respects cache TTL — expired entry re-fetches', async () => { - // Plant a stale cache file directly so the test controls its age. - const url = 'https://www.law.cornell.edu/uscode/text/18/1836' - const key = sha256(url) - const path = join(cacheDir, 'http', key.slice(0, 2), `${key}.json`) - await mkdir(join(cacheDir, 'http', key.slice(0, 2)), { recursive: true }) - await writeFile( - path, - JSON.stringify({ - url, - status: 200, - body: 'STALE', - sourceUpdatedAt: '2020-01-01T00:00:00.000Z', - fetchedAt: '2020-01-01T00:00:00.000Z', - fromCache: false, - verifiable: true, - }), - ) - // A one-day-old mtime must exceed the configured one-minute TTL. - const { utimes } = await import('node:fs/promises') - const dayAgo = new Date(Date.now() - 24 * 60 * 60 * 1000) - await utimes(path, dayAgo, dayAgo) - - mockFetch(() => html(`<html><body>${'FRESH'.repeat(100)}</body></html>`)) - const result = await politeFetch(url, { cacheDir, cacheTtlMs: 60_000 }) - expect(result.fromCache).toBe(false) - expect(result.body).toContain('FRESH') - }) - - it("caches failures too so a transient block doesn't storm the authority", async () => { - mockFetch(() => html('Just a moment', 200)) - const url = 'https://www.law.cornell.edu/wex/non-compete' - const first = await politeFetch(url, { cacheDir }) - expect(first.verifiable).toBe(false) - const cached = await readdir(join(cacheDir, 'http'), { recursive: true }).catch(() => []) - expect(cached.length).toBeGreaterThan(0) - }) - - it('serialises requests to the same host (>=1s gap)', async () => { - const timestamps: number[] = [] - mockFetch(() => { - timestamps.push(Date.now()) - return html(`<html><body>${'X'.repeat(500)}</body></html>`) - }) - // Distinct URLs bypass the URL cache but still share the host throttle. - const t0 = Date.now() - await Promise.all([ - politeFetch('https://throttle.test/a'), - politeFetch('https://throttle.test/b'), - ]) - const gap = (timestamps[1] ?? 0) - (timestamps[0] ?? 0) - expect(gap).toBeGreaterThanOrEqual(900) // some leeway for timer precision - // Total elapsed time must include the per-host delay. - expect(Date.now() - t0).toBeGreaterThanOrEqual(900) - }, 10_000) - - it('never throws on a network error — returns verifiable=false', async () => { - mockFetch(() => { - throw new TypeError('network unreachable') - }) - const result = await politeFetch('https://throw.test/x') - expect(result.verifiable).toBe(false) - expect(result.unverifiableReason).toMatch(/network error: network unreachable/) - expect(result.status).toBe(0) - }) - - it('cache entry on success is reusable via subsequent fetches without remocking', async () => { - mockFetch(() => html(`<html><body>${'X'.repeat(500)}</body></html>`)) - const url = 'https://www.law.cornell.edu/uscode/text/18/1836' - await politeFetch(url, { cacheDir }) - - // A network fetch would return 500, so success proves the cache was used. - mockFetch(() => html('boom', 500)) - const second = await politeFetch(url, { cacheDir }) - expect(second.status).toBe(200) - expect(second.fromCache).toBe(true) - }) - - it('cache files are organised by URL hash prefix', async () => { - mockFetch(() => html(`<html><body>${'Y'.repeat(500)}</body></html>`)) - await politeFetch('https://hash.test/foo', { cacheDir }) - const entries = await readdir(join(cacheDir, 'http'), { recursive: true }) - expect(entries.some((e) => e.toString().endsWith('.json'))).toBe(true) - const jsons = entries.filter((e) => e.toString().endsWith('.json')) - const content = await readFile(join(cacheDir, 'http', jsons[0]!.toString()), 'utf8') - expect(JSON.parse(content).url).toBe('https://hash.test/foo') - }) -}) diff --git a/tests/sources-live.test.ts b/tests/sources-live.test.ts deleted file mode 100644 index b51ef51..0000000 --- a/tests/sources-live.test.ts +++ /dev/null @@ -1,158 +0,0 @@ -import { mkdtemp, rm } from 'node:fs/promises' -import { tmpdir } from 'node:os' -import { join } from 'node:path' -import { afterEach, beforeEach, describe, expect, it } from 'vitest' -import { - __resetHttpThrottle, - createCornellLiiSource, - createIrsPublicationsSource, - createStateSosSource, -} from '../src/sources/index' - -/** - * Optional network compatibility checks for Cornell LII, IRS.gov, and CA SOS. - * Each check uses a fresh cache and returns early when the authority reports - * that its response cannot be verified. - */ - -const LIVE_ENABLED = process.env.AGENT_KNOWLEDGE_RUN_NETWORK_TESTS === '1' -const TIMEOUT_MS = 30_000 - -let cacheDir: string - -beforeEach(async () => { - __resetHttpThrottle() - cacheDir = await mkdtemp(join(tmpdir(), 'agent-knowledge-live-cache-')) -}) - -afterEach(async () => { - await rm(cacheDir, { recursive: true, force: true }) -}) - -describe.skipIf(!LIVE_ENABLED)('live: Cornell LII', () => { - it( - 'fetches DTSA 18 USC § 1836 with verifiable=true and statute text', - async () => { - const source = createCornellLiiSource({ - selectors: [{ kind: 'uscode', path: '18/1836' }], - }) - const fragments = await source.fetch({ cacheDir }) - expect(fragments).toHaveLength(1) - const f = fragments[0]! - if (!f.provenance.verifiable) { - console.warn(`Cornell LII unreachable: ${f.provenance.unverifiableReason} — skipping`) - return - } - expect(f.id).toBe('uscode:18/1836') - expect(f.title.toLowerCase()).toContain('1836') - expect(f.provenance.url).toBe('https://www.law.cornell.edu/uscode/text/18/1836') - expect(f.provenance.jurisdiction).toBe('US-FED') - // The statute text must include the Attorney General clause. - expect(f.body.toLowerCase()).toMatch(/attorney general/) - expect(f.dimensionHints).toContain('jurisdictional_accuracy') - // Body hash must be deterministic + non-empty. - expect(f.bodyHash).toMatch(/^[0-9a-f]{64}$/) - }, - TIMEOUT_MS, - ) - - it( - 'fetches a Wex entry (restraint_of_trade — covers the non-compete doctrine surface)', - async () => { - // Cornell's Wex doesn't currently carry a /wex/non-compete slug. The - // doctrinal surface for the Ryan-LLC v. FTC drift sits under - // /wex/restraint_of_trade. If Wex later adds a more specific slug we - // should add it as a second selector — change detection across slugs - // is exactly what `KnowledgeChange.added` is for. - const source = createCornellLiiSource({ - selectors: [ - { kind: 'wex', path: 'restraint_of_trade', dimensionHints: ['jurisdictional_accuracy'] }, - ], - }) - const fragments = await source.fetch({ cacheDir }) - const f = fragments[0]! - if (!f.provenance.verifiable) { - console.warn(`Cornell LII Wex unreachable: ${f.provenance.unverifiableReason} — skipping`) - return - } - expect(f.id).toBe('wex:restraint_of_trade') - expect(f.provenance.url).toBe('https://www.law.cornell.edu/wex/restraint_of_trade') - expect(f.body.length).toBeGreaterThan(200) - expect(f.dimensionHints).toEqual(['jurisdictional_accuracy']) - }, - TIMEOUT_MS, - ) -}) - -describe.skipIf(!LIVE_ENABLED)('live: IRS publications', () => { - it( - 'fetches the publications index with table rows extracted', - async () => { - const source = createIrsPublicationsSource({ publications: [] }) - const fragments = await source.fetch({ cacheDir }) - const index = fragments.find((f) => f.id === 'index') - expect(index).toBeDefined() - if (!index?.provenance.verifiable) { - console.warn(`IRS index unreachable: ${index?.provenance.unverifiableReason} — skipping`) - return - } - expect(index.provenance.url).toBe('https://www.irs.gov/publications') - expect(index.provenance.jurisdiction).toBe('US-FED') - // The publications index must mention at least one current pub. - expect(index.body).toMatch(/Publication\s+15\b/i) - expect(index.dimensionHints).toContain('tax_compliance') - }, - TIMEOUT_MS, - ) - - it( - 'fetches Publication 15 landing page', - async () => { - const source = createIrsPublicationsSource({ - includeIndex: false, - publications: ['p15'], - }) - const fragments = await source.fetch({ cacheDir }) - const pub = fragments[0]! - if (!pub.provenance.verifiable) { - console.warn(`IRS p15 unreachable: ${pub.provenance.unverifiableReason} — skipping`) - return - } - expect(pub.id).toBe('publication:p15') - expect(pub.provenance.url).toBe('https://www.irs.gov/publications/p15') - expect(pub.body.length).toBeGreaterThan(500) - }, - TIMEOUT_MS, - ) -}) - -describe.skipIf(!LIVE_ENABLED)('live: state SOS (California)', () => { - it( - 'fetches CA SOS forms page', - async () => { - const source = createStateSosSource({ - state: 'CA', - baseUrl: 'https://www.sos.ca.gov', - entities: [ - { - id: 'business-entities-forms', - path: '/business-programs/business-entities/forms', - title: 'CA Business Entities Forms', - selector: { kind: 'whole' }, - }, - ], - }) - const fragments = await source.fetch({ cacheDir }) - const f = fragments[0]! - if (!f.provenance.verifiable) { - console.warn(`CA SOS unreachable: ${f.provenance.unverifiableReason} — skipping`) - return - } - expect(f.id).toBe('business-entities-forms') - expect(f.provenance.jurisdiction).toBe('US-CA') - expect(f.body.toLowerCase()).toMatch(/llc|limited liability|forms/) - expect(f.dimensionHints).toContain('jurisdictional_accuracy') - }, - TIMEOUT_MS, - ) -}) diff --git a/tests/sources-mocked.test.ts b/tests/sources-mocked.test.ts deleted file mode 100644 index 664bf59..0000000 --- a/tests/sources-mocked.test.ts +++ /dev/null @@ -1,245 +0,0 @@ -import { mkdtemp, rm } from 'node:fs/promises' -import { tmpdir } from 'node:os' -import { join } from 'node:path' -import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' -import { - __resetHttpThrottle, - createCornellLiiSource, - createIrsPublicationsSource, - createStateSosSource, -} from '../src/sources/index' - -/** Representative authority HTML keeps parser tests deterministic and offline. */ - -let cacheDir: string -const originalFetch = globalThis.fetch - -beforeEach(async () => { - __resetHttpThrottle() - cacheDir = await mkdtemp(join(tmpdir(), 'agent-knowledge-sources-mock-')) -}) - -afterEach(async () => { - await rm(cacheDir, { recursive: true, force: true }) - globalThis.fetch = originalFetch -}) - -function mockOnce(html: string, status = 200): void { - globalThis.fetch = vi.fn( - async () => - new Response(html, { - status, - headers: { 'content-type': 'text/html', 'last-modified': 'Wed, 01 Jan 2025 00:00:00 GMT' }, - }), - ) as unknown as typeof globalThis.fetch -} - -const CORNELL_USCODE_HTML = `<!doctype html><html><head> -<title>18 U.S. Code § 1836 - Civil proceedings | LII / Legal Information Institute - -
NAV
-
-

18 U.S. Code § 1836 - Civil proceedings

-
-
(a) The Attorney General may, in a civil action, obtain appropriate injunctive relief against any violation of this chapter.
-
(b) Private Civil Actions. — An owner of a trade secret may bring a civil action under this subsection.
-
 -
Amendments 2016 — Pub. L. 114–153
-
-
FOOT
-` - -const CORNELL_WEX_HTML = ` -Non-compete | Wex | US Law | LII / Legal Information Institute - -
-

Non-compete

-

${'A non-compete agreement is a contract between an employer and employee. '.repeat(20)}

-

On August 20, 2024, the U.S. District Court for the Northern District of Texas set aside the FTC rule.

-
-` - -const IRS_INDEX_HTML = ` -Publications | Internal Revenue Service - -
- - - -
Publication 15 (2026), (Circular E), Employer's Tax GuidePublication 15 (2026)p15.pdf
Publication 17 (2025), Your Federal Income TaxPublication 17 (2025)p17.pdf
-
` - -const IRS_PUB_HTML = ` -Publication 15 (2026), (Circular E), Employer's Tax Guide | Internal Revenue Service - - -
-

Publication 15 (2026), (Circular E), Employer's Tax Guide

-

${'For use in 2026. This publication explains your tax responsibilities as an employer. '.repeat(10)}

-
` - -const SOS_CA_HTML = ` -Forms, Samples and Fees :: California Secretary of State - -
-

Forms, Samples and Fees

-
-

${'LLC formation is governed by RULLCA. Filing fee is $70. '.repeat(8)}

-
-
` - -describe('cornell-lii parsing', () => { - it('extracts statute body and tags US-FED jurisdiction', async () => { - mockOnce(CORNELL_USCODE_HTML) - const source = createCornellLiiSource({ selectors: [{ kind: 'uscode', path: '18/1836' }] }) - const [f] = await source.fetch({ cacheDir }) - expect(f!.id).toBe('uscode:18/1836') - expect(f!.provenance.url).toBe('https://www.law.cornell.edu/uscode/text/18/1836') - expect(f!.provenance.jurisdiction).toBe('US-FED') - expect(f!.provenance.verifiable).toBe(true) - expect(f!.body).toContain('Attorney General') - expect(f!.body).toContain('Private Civil Actions') - expect(f!.dimensionHints).toContain('jurisdictional_accuracy') - expect(f!.title).toContain('1836') - }) - - it('extracts effective date from the Amendments block', async () => { - mockOnce(CORNELL_USCODE_HTML) - const source = createCornellLiiSource({ selectors: [{ kind: 'uscode', path: '18/1836' }] }) - const [f] = await source.fetch({ cacheDir }) - expect(f!.provenance.sourceUpdatedAt).toBe('2016-12-31T00:00:00.000Z') - }) - - it('handles Wex slugs and uses fallback dimension hints', async () => { - mockOnce(CORNELL_WEX_HTML) - const source = createCornellLiiSource({ selectors: [{ kind: 'wex', path: 'non-compete' }] }) - const [f] = await source.fetch({ cacheDir }) - expect(f!.id).toBe('wex:non-compete') - expect(f!.provenance.url).toBe('https://www.law.cornell.edu/wex/non-compete') - expect(f!.body).toMatch(/Northern District of Texas/) - expect(f!.dimensionHints).toEqual(['citation_hygiene']) - }) - - it('surfaces verifiable=false when authority serves a 4xx', async () => { - mockOnce('not found', 404) - const source = createCornellLiiSource({ selectors: [{ kind: 'uscode', path: '99/9999' }] }) - const [f] = await source.fetch({ cacheDir }) - expect(f!.provenance.verifiable).toBe(false) - expect(f!.body).toBe('') - expect(f!.provenance.unverifiableReason).toMatch(/non-2xx/) - }) -}) - -describe('irs-publications parsing', () => { - it('index fragment captures publication rows', async () => { - mockOnce(IRS_INDEX_HTML) - const source = createIrsPublicationsSource() - const fragments = await source.fetch({ cacheDir }) - const index = fragments.find((f) => f.id === 'index') - expect(index).toBeDefined() - expect(index!.body).toMatch(/Publication\s+15\s*\(2026\)/i) - expect(index!.body).toMatch(/Publication\s+17\s*\(2025\)/i) - expect(index!.provenance.jurisdiction).toBe('US-FED') - expect(index!.dimensionHints).toContain('tax_compliance') - }) - - it('publication fragment captures revision year as sourceUpdatedAt', async () => { - let callCount = 0 - globalThis.fetch = vi.fn(async () => { - callCount += 1 - if (callCount === 1) { - return new Response(IRS_INDEX_HTML, { - status: 200, - headers: { 'content-type': 'text/html' }, - }) - } - return new Response(IRS_PUB_HTML, { - status: 200, - headers: { 'content-type': 'text/html' }, - }) - }) as unknown as typeof globalThis.fetch - - const source = createIrsPublicationsSource({ publications: ['p15'] }) - const fragments = await source.fetch({ cacheDir }) - const pub = fragments.find((f) => f.id === 'publication:p15') - expect(pub).toBeDefined() - expect(pub!.body).toMatch(/Publication 15/) - expect(pub!.provenance.sourceUpdatedAt).toBe('2026-01-01T00:00:00.000Z') - expect(pub!.metadata).toMatchObject({ kind: 'publication', slug: 'p15' }) - }) - - it('respects includeIndex=false and limit', async () => { - mockOnce(IRS_PUB_HTML) - const source = createIrsPublicationsSource({ - includeIndex: false, - publications: ['p15', 'p17'], - }) - const fragments = await source.fetch({ cacheDir, limit: 1 }) - expect(fragments).toHaveLength(1) - expect(fragments[0]!.id).toBe('publication:p15') - }) -}) - -describe('state-sos parsing', () => { - it('extracts via id selector and tags jurisdiction', async () => { - mockOnce(SOS_CA_HTML) - const source = createStateSosSource({ - state: 'CA', - baseUrl: 'https://www.sos.ca.gov', - entities: [ - { - id: 'llc-formation', - path: '/business-programs/business-entities/forms', - title: 'CA LLC Formation', - selector: { kind: 'id', value: 'main-content' }, - }, - ], - }) - const [f] = await source.fetch({ cacheDir }) - expect(f!.id).toBe('llc-formation') - expect(f!.provenance.jurisdiction).toBe('US-CA') - expect(f!.body).toMatch(/RULLCA/) - expect(f!.dimensionHints).toContain('jurisdictional_accuracy') - }) - - it('whole-page selector falls back to main when no id match', async () => { - mockOnce(SOS_CA_HTML) - const source = createStateSosSource({ - state: 'CA', - baseUrl: 'https://www.sos.ca.gov', - entities: [ - { - id: 'forms', - path: '/forms', - title: 'CA Forms', - selector: { kind: 'whole' }, - }, - ], - }) - const [f] = await source.fetch({ cacheDir }) - expect(f!.body).toMatch(/Forms, Samples and Fees/) - expect(f!.body).toMatch(/RULLCA/) - }) - - it('regex selector picks the configured block', async () => { - mockOnce(SOS_CA_HTML) - const source = createStateSosSource({ - state: 'CA', - baseUrl: 'https://www.sos.ca.gov', - entities: [ - { - id: 'forms-h1', - path: '/forms', - title: 'CA Forms', - selector: { kind: 'regex', value: //i }, - }, - ], - }) - const [f] = await source.fetch({ cacheDir }) - // Body is just the h1 text; short, so verifiable=false expected. - expect(f!.title).toBe('CA Forms') - expect(f!.body.toLowerCase()).toContain('forms, samples and fees') - // Short body ⇒ source flags it as not verifiable. - expect(f!.provenance.verifiable).toBe(false) - }) -}) diff --git a/tests/sources-types.test.ts b/tests/sources-types.test.ts deleted file mode 100644 index 4e2b4d8..0000000 --- a/tests/sources-types.test.ts +++ /dev/null @@ -1,109 +0,0 @@ -import { describe, expect, it } from 'vitest' -import { - createCornellLiiSource, - createIrsPublicationsSource, - createStateSosSource, - extractLinks, - htmlToText, - looksLikeBlockPage, -} from '../src/sources/index' - -describe('source factories', () => { - it('cornell-lii default id is stable', () => { - const source = createCornellLiiSource({ selectors: [{ kind: 'uscode', path: '18/1836' }] }) - expect(source.id).toBe('cornell-lii') - expect(source.name).toMatch(/Cornell/) - }) - - it('cornell-lii id override is honoured', () => { - const source = createCornellLiiSource({ - selectors: [{ kind: 'wex', path: 'non-compete' }], - id: 'cornell-lii-trade-secrets', - }) - expect(source.id).toBe('cornell-lii-trade-secrets') - }) - - it('irs-publications default id is stable', () => { - const source = createIrsPublicationsSource() - expect(source.id).toBe('irs-publications') - }) - - it('state-sos id derived from postal code (lower-cased)', () => { - const source = createStateSosSource({ - state: 'CA', - baseUrl: 'https://www.sos.ca.gov', - entities: [], - }) - expect(source.id).toBe('state-sos:ca') - expect(source.name).toBe('CA Secretary of State') - }) -}) - -describe('looksLikeBlockPage', () => { - it('catches Cloudflare interstitial', () => { - expect(looksLikeBlockPage('Just a moment...
Verify you are human')).toBe(true) - }) - - it('catches CAPTCHA pages', () => { - expect(looksLikeBlockPage('
Please complete the CAPTCHA
')).toBe(true) - }) - - it('catches Incapsula block pages', () => { - expect(looksLikeBlockPage(' Request unsuccessful.')).toBe(true) - }) - - it('does not false-positive on real statute text', () => { - expect( - looksLikeBlockPage( - '18 U.S. Code § 1836 - Civil proceedings. The Attorney General may, in a civil action, obtain appropriate injunctive relief...', - ), - ).toBe(false) - }) - - it('empty body is not a block page (different failure path)', () => { - expect(looksLikeBlockPage('')).toBe(false) - }) -}) - -describe('htmlToText', () => { - it('preserves
and

as newlines', () => { - const text = htmlToText('

alpha

beta

gamma
delta
') - expect(text.split('\n')).toEqual(['alpha', 'beta', 'gamma', 'delta']) - }) - - it('strips scripts and styles entirely', () => { - const text = htmlToText('

visible

') - expect(text).toBe('visible') - }) - - it('decodes the section sign and common entities', () => { - const text = htmlToText('

18 U.S. Code § 1836 — "trade secret"

') - expect(text).toContain('§') - expect(text).toContain('—') - expect(text).toContain('"trade secret"') - }) - - it('decodes numeric entities', () => { - expect(htmlToText('

§1836

')).toBe('§1836') - expect(htmlToText('

§1836

')).toBe('§1836') - }) -}) - -describe('extractLinks', () => { - it('filters by href pattern and resolves against base', () => { - const html = - 'Pub 15' + - 'About' + - 'Pub 17' - const links = extractLinks(html, /\/publications\/p\d+/i, 'https://www.irs.gov') - expect(links).toEqual([ - { href: 'https://www.irs.gov/publications/p15', text: 'Pub 15' }, - { href: 'https://www.irs.gov/publications/p17', text: 'Pub 17' }, - ]) - }) - - it('skips empty link text', () => { - const html = '' - expect(extractLinks(html, /\/publications\//i, 'https://www.irs.gov')).toEqual([]) - }) -}) diff --git a/tsdown.config.ts b/tsdown.config.ts index d6367d6..bdbdfaf 100644 --- a/tsdown.config.ts +++ b/tsdown.config.ts @@ -6,7 +6,6 @@ export default defineConfig({ 'viz/index': 'src/viz/index.ts', cli: 'src/cli.ts', 'memory/index': 'src/memory/index.ts', - 'sources/index': 'src/sources/index.ts', 'benchmarks/index': 'src/benchmarks/index.ts', }, format: ['esm'],