diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index afbf51a..1a3b5b1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -34,8 +34,6 @@ jobs: - name: Test run: pnpm test - env: - AGENT_KNOWLEDGE_RUN_NETWORK_TESTS: '1' - name: Build run: pnpm build diff --git a/CHANGELOG.md b/CHANGELOG.md index 45d7eb6..944283a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,8 @@ The verified-research paper and its results remain as a historical record linked Remove the memory experiment, memory improvement, and benchmark-suite runners, which no consumer imported: `runAgentMemoryExperiment`, `runAgentMemoryLearningExperiment`, `runAgentMemoryImprovement`, `runKnowledgeBenchmarkSuite`, `runMemoryAdapterBenchmark`, the industry benchmark catalogs, qrels import, benchmark scoring, and their run-lease, cost, and comparison types. Compare memory providers on product tasks with Agent Eval. Memory adapters, branches, holdout, lifecycle bounds, play memory tools, and the `/benchmarks` in-memory and no-op adapters are unchanged. +Remove the `/sources` entry point and the unused authority adapters behind it (Cornell LII, IRS publications, state Secretary of State, polite HTTP fetch, HTML extraction), plus `detectChanges` and the freshness stores. No consumer imported them. +The source registry (`addSourceText`, `addSourcePath`, `loadSourceRegistry`), source adapters such as `textSourceAdapter`, and readiness freshness scoring are unchanged. ## 19.1.5 diff --git a/README.md b/README.md index 7c9288d..99e56c6 100644 --- a/README.md +++ b/README.md @@ -32,7 +32,6 @@ Use Knowledge 17 with older Eval releases. | Optimize a KB maintenance policy | `optimizeKnowledgeBasePolicy` | package root | | Run retrieval, research, answer checks, and promotion as one process | `runRagKnowledgeImprovementLoop` | package root | | Connect a memory provider or branch its state | `AgentMemoryAdapter`, `createAgentMemoryBranch` | `/memory` | -| Read from external authorities | `KnowledgeSource` and source adapters | `/sources` | | Use live research or coding agents | `runKnowledgeImprovementJob` | `@tangle-network/agent-runtime` | ## Create and search a knowledge base @@ -469,7 +468,6 @@ Compare providers on product tasks with `@tangle-network/agent-eval` before chan |---|---| | `@tangle-network/agent-knowledge` | KB files, indexes, search, validation, research callbacks, RAG evaluation, and candidate improvement | | `@tangle-network/agent-knowledge/memory` | Memory contracts, provider adapters, branches, holdout, and play memory tools | -| `@tangle-network/agent-knowledge/sources` | HTTP and authority-specific source adapters | | `@tangle-network/agent-knowledge/benchmarks` | In-memory and no-op memory adapters for benchmark harnesses | | `@tangle-network/agent-knowledge/viz` | Dependency-free graph analysis helpers | diff --git a/api-surface.json b/api-surface.json index 8e7e399..99d8202 100644 --- a/api-surface.json +++ b/api-surface.json @@ -44,15 +44,12 @@ "ClaimLedgerStore": "type c402ec258adc", "ClaimRef": "type e200b4c5eeec", "ClaimVerdict": "type 18f044018844", - "CornellLiiSelector": "type c92fb79c4d6b", - "CornellLiiSourceOptions": "type 3d1a7a174053", "CreateAgentMemoryBranchOptions": "type 924c7b6dcd11", "CreateKnowledgeRetrievalDispositionInput": "type 27fb6228c8c1", "CreateKnowledgeRetrievalReceiptInput": "type 2b6c8c6a078b", "CreateKnowledgeToolsOptions": "type 359d4edb185e", "CreateKnowledgeUseReceiptInput": "type eb120a93f4c7", "CreatePlayMemoryToolsOptions": "type d659926093f3", - "D1Adapter": "type fd8669db1f2e", "DEADLINE_EXIT_CODE": "value 16590df13e07", "DEFAULT_KNOWLEDGE_BRIEF_LIMIT": "value a16e63bcd9c7", "DEFAULT_MEMORY_CLEANUP_TIMEOUT_MS": "value 10d1ef3e7731", @@ -61,8 +58,6 @@ "DeepQuestionKind": "type e1a27d1fc3c2", "DeepQuestionSchema": "value 373728f5643d", "DefineReadinessSpecInput": "type f54fcc87f848", - "DetectChangesOptions": "type 39c50fdf35ae", - "DetectChangesResult": "type 830df4ec6d85", "DiscoveryLoopResult": "type 80ccac2ce500", "DiscoveryLoopRound": "type 4a67fa7bfc48", "DiscoveryLoopStopReason": "type 61912c65132e", @@ -72,19 +67,12 @@ "EvaluateKnowledgeBaseReadinessOptions": "type 9a3de35dc405", "EvidenceRung": "type fa5c1c76aa73", "ExternalRagEvalScore": "type df1d799a1de6", - "FetchOpts": "type 7b44d8f28084", - "FileSystemFreshnessStoreOptions": "type 10a9cbd1e857", "FileSystemKbStore": "value 3e4c27745531", "FileSystemKbStoreOptions": "type 10a9cbd1e857", "FileSystemSearchOptions": "type ab0c98cd100a", "FileSystemSearchProvider": "value cf2101caec6e", "FileSystemSearchProviderOptions": "type f8bf1cf8926d", "ForkAgentMemoryBranchSnapshotOptions": "type 4daac6457273", - "FragmentProvenance": "type a719c43aeb25", - "FreshnessKey": "type 65cdc4ab2341", - "FreshnessMark": "type 93bc4803d9b3", - "FreshnessRecord": "type 65bdf72051de", - "FreshnessTtl": "type e3c26a4f6c3e", "GradeableClaim": "type bf465a1848d5", "GradedClaim": "type acfba8ff752d", "GraphitiMcpClientLike": "type a6c48bcfcdd6", @@ -95,10 +83,8 @@ "HindsightClientLike": "type e4b6e0bd1488", "HindsightMemoryAdapterOptions": "type 995d4468eb3c", "HindsightOperationUnknownError": "value 408246fe51ae", - "IRS_DIMENSION_HINTS": "value 373728f5643d", "ImproveSelectedKnowledgeCandidateOptions": "type c738ecaa49fa", "ImproveSelectedKnowledgeCandidateResult": "type 3a8454d1031d", - "IrsPublicationsSourceOptions": "type 6b60bfe9f1bc", "KB_CLAIM_LEDGER_DIR": "value 2e94ea580600", "KB_EVENTS_PATH": "value 7c4a3ddbac83", "KB_INDEX_PATH": "value f450045c9510", @@ -118,8 +104,6 @@ "KnowledgeBaseReadinessEvaluation": "type 5de5279c9dce", "KnowledgeBrief": "type 01ca10f80a06", "KnowledgeBriefOptions": "type ceb76227d749", - "KnowledgeChange": "type 7b71ca9dd658", - "KnowledgeChangeKind": "type 02226535c8f0", "KnowledgeChunk": "type 5c73eb000db2", "KnowledgeCitationAuditError": "value e64c40482f65", "KnowledgeCitationAuditIssue": "type bb84ee6c9f6c", @@ -148,8 +132,6 @@ "KnowledgeEventSchema": "value 373728f5643d", "KnowledgeEventType": "type 7071c221c684", "KnowledgeExplanation": "type 4326c52192e1", - "KnowledgeFragment": "type 5d9512fd9945", - "KnowledgeFreshnessStore": "type 6805e31ec36f", "KnowledgeGraph": "type 2a4f44745410", "KnowledgeGraphEdge": "type 9eba2728a7b2", "KnowledgeGraphEdgeSchema": "value 373728f5643d", @@ -239,7 +221,6 @@ "KnowledgeRetrieverIdentity": "type fb46de336ac2", "KnowledgeSearchHit": "type d13a101a4b4f", "KnowledgeSearchResult": "type c443b7efb49e", - "KnowledgeSource": "type 6ebb3f0a0ef2", "KnowledgeStateScope": "type 61b8ac32e052", "KnowledgeUnit": "type bbd850260f7f", "KnowledgeUseReceipt": "type 95b469ec3c07", @@ -257,8 +238,6 @@ "KnowledgeWriteIntakeRequest": "type 5074f0dab514", "KnowledgeWriteParseResult": "type 3ec744fcaf93", "LoadKnowledgeImprovementActivationResultOptions": "type 3c304bc52d1c", - "MAX_RESPONSE_BYTES": "value 373728f5643d", - "MIN_REQUEST_GAP_MS": "value 648cef7e8361", "MeasuredKnowledgeSelectionReceipt": "type 28a9a57c6626", "Mem0ClientMode": "type abfe4225abaf", "Mem0HostedClient": "type 4d1434ac5a9a", @@ -275,14 +254,11 @@ "OptimizeKnowledgeBasePolicyResult": "type 1e3b03c7a0a3", "OriginatedKnowledgeSearchResult": "type 99110190f58f", "OriginatedPage": "type 62ba6f531a4a", - "POLITE_USER_AGENT": "value 3eb07665c028", "PageOrigin": "type df33a580b8a9", "ParsedFrontmatter": "type 7aebd4ab7c31", "PartitionRetrievalScenariosOptions": "type bfb5473c9c0c", "PendingKnowledgeMutation": "type 265414e3a059", "PlayMemoryRetrievalReceipt": "type 0118f85065b1", - "PoliteFetchOptions": "type e8c10ab474da", - "PoliteFetchResult": "type 68c8ac7c266b", "PromoteKnowledgeCandidateOptions": "type 804348b78767", "PromoteRunScopedPagesOptions": "type 314295c13674", "ProposeFromFindingsResult": "type 979e50440bbd", @@ -390,8 +366,6 @@ "SourceRecord": "type b525cb8c09ee", "SourceRecordSchema": "value 373728f5643d", "SourceRegistry": "type 062b92354d23", - "StateSosEntity": "type 4ecdc780ae4d", - "StateSosSourceConfig": "type 3a2d0b61a718", "TrackedClaim": "type 823ab1f0941e", "UncheckableClaimError": "value 86a15815c4f1", "UseKnowledgeImprovementCandidateOptions": "type 848b0e83e03f", @@ -400,7 +374,6 @@ "VerifiedGradeableEvidence": "type 68ecc1af89b5", "VerifyGradeableEvidenceOptions": "type 01ef698f9b40", "WIKILINK_REGEX": "value 373728f5643d", - "__resetHttpThrottle": "value e47f3b6734a6", "addSourcePath": "value 9dabb587c73c", "addSourceText": "value 3a5c6107b137", "applyKnowledgeWriteBlocks": "value 08c999b38722", @@ -438,15 +411,11 @@ "claimId": "value 4c9eca896a37", "claimSourceHost": "value a8a18cde9fcc", "createAgentMemoryBranch": "value bae206df8be9", - "createCornellLiiSource": "value 5708478c5b8a", - "createD1FreshnessStoreStub": "value e89ca59074b9", "createFileRunLineageAuthority": "value 92afeb585f32", - "createFileSystemFreshnessStore": "value 56d2aaf47eb6", "createFileSystemSearchProvider": "value c88b43a7ac80", "createGraphitiMemoryAdapter": "value 512201c0dd04", "createHindsightMemoryAdapter": "value 97fcfb086aee", "createInMemoryBenchmarkAdapter": "value f4c546ff2fff", - "createIrsPublicationsSource": "value da388cf92c5c", "createKnowledgeControlLoopAdapter": "value bfc8f1be188f", "createKnowledgeEvent": "value 4b9e73e096bd", "createKnowledgeRetrievalDisposition": "value 838fadfc5636", @@ -464,12 +433,10 @@ "createQmdSearchProvider": "value f1d2a2d4d631", "createRagAnswerQualityHook": "value 7e18d330e76b", "createRunScopedStores": "value 6bfc535c7281", - "createStateSosSource": "value e7467eeee69b", "decodeKnowledgeVisibilitySnapshot": "value 143bc2b124e6", "deepQuestionId": "value ffec05529ae9", "defaultGetMemoryContext": "value 6a368eb4ea71", "defineReadinessSpec": "value 9128b5b31bfa", - "detectChanges": "value 741122c9f8e7", "detectNearDuplicatePages": "value 457a9cbea181", "deterministicRng": "value f2af0cd5da5c", "diagnoseRagAnswerFailure": "value a8382d9e2133", @@ -478,9 +445,7 @@ "encodeKnowledgeVisibilitySnapshot": "value c673fa060267", "evaluateKnowledgeBaseReadiness": "value a981becc92c2", "explainKnowledgeTarget": "value 1266c029bf54", - "extractLinks": "value 930437fc2aa4", "extractWikilinks": "value 2a806a857c29", - "firstMatch": "value 0fa78f90a44f", "forkAgentMemoryBranchSnapshot": "value 0c258d0909ac", "formatFrontmatter": "value e0a4508d4ded", "formatKnowledgeCitationReference": "value 8c9301778549", @@ -492,11 +457,9 @@ "groundClaimInText": "value f74ed9ca26c6", "hashKnowledgeBase": "value bdb2d9eea5f9", "hindsightMemoryBankId": "value b828b7e949f0", - "htmlToText": "value c58c3f5d8c75", "improveKnowledgeBase": "value a4ebce095bdf", "improveSelectedKnowledgeCandidate": "value 6d48e36af283", "initKnowledgeBase": "value 65692669c5c8", - "innerHtmlById": "value f0ffa022d6c0", "inspectKnowledgeIndex": "value 79b67f7de1cd", "inspectPendingKnowledgeMutation": "value 38b2e30827bd", "isKernelAnchoredPath": "value 2819fa14db65", @@ -528,7 +491,6 @@ "loadKnowledgePages": "value baac5eb22201", "loadKnowledgePromotionRecord": "value 65617956865c", "loadSourceRegistry": "value cae49411bcc5", - "looksLikeBlockPage": "value 371f55d1c1d9", "materializeRegisteredClaimEvidence": "value cf315a2deab2", "mediaTypeFor": "value a8a18cde9fcc", "mem0MemoryAdapterIdentity": "value c590a9c1e72f", @@ -551,7 +513,6 @@ "parseKnowledgeWriteBlocks": "value d6bd24d2858d", "partitionRetrievalScenarios": "value 363802debe8a", "planInvalidationPropagation": "value 39da8e2c1a14", - "politeFetch": "value 10d577fc5694", "promoteKnowledgeCandidate": "value ea1e94a3d6a2", "promoteRunScopedPages": "value 58ff1ec22528", "proposeFromFinding": "value 1dcecd58e869", @@ -714,33 +675,6 @@ "sleepForMemoryRecovery": "value 4a6818f984e4", "toOffPolicyTrajectory": "value 934dec24305c" }, - "./sources": { - "CornellLiiSelector": "type c92fb79c4d6b", - "CornellLiiSourceOptions": "type 3d1a7a174053", - "FetchOpts": "type 7b44d8f28084", - "FragmentProvenance": "type a719c43aeb25", - "IRS_DIMENSION_HINTS": "value 373728f5643d", - "IrsPublicationsSourceOptions": "type 6b60bfe9f1bc", - "KnowledgeFragment": "type 5d9512fd9945", - "KnowledgeSource": "type 6ebb3f0a0ef2", - "MAX_RESPONSE_BYTES": "value 373728f5643d", - "MIN_REQUEST_GAP_MS": "value 648cef7e8361", - "POLITE_USER_AGENT": "value 3eb07665c028", - "PoliteFetchOptions": "type e8c10ab474da", - "PoliteFetchResult": "type 68c8ac7c266b", - "StateSosEntity": "type 4ecdc780ae4d", - "StateSosSourceConfig": "type 3a2d0b61a718", - "__resetHttpThrottle": "value e47f3b6734a6", - "createCornellLiiSource": "value 5708478c5b8a", - "createIrsPublicationsSource": "value da388cf92c5c", - "createStateSosSource": "value e7467eeee69b", - "extractLinks": "value 930437fc2aa4", - "firstMatch": "value 0fa78f90a44f", - "htmlToText": "value c58c3f5d8c75", - "innerHtmlById": "value f0ffa022d6c0", - "looksLikeBlockPage": "value 371f55d1c1d9", - "politeFetch": "value 10d577fc5694" - }, "./viz": { "KnowledgeCommunity": "type 945ac89694c9", "KnowledgeGap": "type afb49eb9f42f", diff --git a/package.json b/package.json index ec33b66..ab19ac2 100644 --- a/package.json +++ b/package.json @@ -34,11 +34,6 @@ "import": "./dist/memory/index.js", "default": "./dist/memory/index.js" }, - "./sources": { - "types": "./dist/sources/index.d.ts", - "import": "./dist/sources/index.js", - "default": "./dist/sources/index.js" - }, "./benchmarks": { "types": "./dist/benchmarks/index.d.ts", "import": "./dist/benchmarks/index.js", diff --git a/scripts/verify-package.mjs b/scripts/verify-package.mjs index e8d0e58..2a1932a 100644 --- a/scripts/verify-package.mjs +++ b/scripts/verify-package.mjs @@ -19,7 +19,6 @@ const publicImports = [ packageName, `${packageName}/viz`, `${packageName}/memory`, - `${packageName}/sources`, `${packageName}/benchmarks`, ] const requiredRootExports = [ diff --git a/src/changes.ts b/src/changes.ts deleted file mode 100644 index d5624fe..0000000 --- a/src/changes.ts +++ /dev/null @@ -1,177 +0,0 @@ -import type { KnowledgeFragment } from './sources/types' - -/** - * Change detection across snapshots of one source's fragments. - * - * The output drives the continuous-ingestion loop: each `KnowledgeChange` - * carries the eval dimensions affected (`affectedDimensions`), which an - * agent-eval campaign scheduler consumes to decide which campaigns to - * re-run. Three change kinds: - * - * - `added` — fragment id appears in `next` but not `prev`. - * - `removed` — fragment id appears in `prev` but not `next`. Typical - * trigger: an authority retires a Wex slug, or a state SOS reorganises - * its forms catalogue. - * - `modified` — fragment id appears in both, body hash differs. This - * is the dominant change kind in practice — the Ryan-LLC v. FTC - * vacatur case manifests as a `modified` on the Wex non-compete - * fragment, NOT as a removed one. - * - * Unverifiable fragments are filtered out before diffing — comparing a - * captcha-blocked snapshot against a real one would falsely fire every - * fragment as removed. The caller can inspect the raw lists if they want - * to surface block-page failures. - * - * Within-snapshot duplicate ids are an upstream bug; this function picks - * the LAST one and emits a diagnostic via the `warnings` field. - * - * @stable - */ - -export type KnowledgeChangeKind = 'added' | 'removed' | 'modified' - -export interface KnowledgeChange { - /** Source-scoped id (matches `KnowledgeFragment.id`). */ - fragmentId: string - kind: KnowledgeChangeKind - /** - * For `added`: full body of the new fragment. - * For `removed`: full body of the prior fragment. - * For `modified`: unified-diff-style payload `{ before, after }` body strings. - */ - diff?: { before?: string; after?: string } - /** - * Eval dimensions to re-score. Computed as the union of both fragments' - * `dimensionHints`. The eval cron treats this as a set of campaign tags. - */ - affectedDimensions: string[] - /** URL of the affected authority page (from whichever side has it). */ - url?: string - /** - * Source-attested change time. For `modified`, takes the NEXT fragment's - * `sourceUpdatedAt`. For `removed`, takes the PRIOR fragment's - * `sourceUpdatedAt`. For `added`, takes the NEXT fragment's - * `sourceUpdatedAt`. Consumers index changes by this date. - */ - detectedAt: string -} - -export interface DetectChangesResult { - changes: KnowledgeChange[] - /** Counts by kind — handy for dashboards. */ - summary: { added: number; removed: number; modified: number } - /** Non-fatal diagnostics (duplicate ids, dropped unverifiable fragments). */ - warnings: string[] -} - -export interface DetectChangesOptions { - /** - * When true (default), unverifiable fragments are dropped from both - * sides before comparison. Set false ONLY when debugging block-page - * issues — comparing against unverifiable content emits false - * `removed`/`modified` changes. - */ - skipUnverifiable?: boolean - /** - * When provided, only changes whose `affectedDimensions` intersect this - * set are returned. Useful for cron loops that schedule per-dimension - * eval campaigns and only care about a subset. - */ - filterDimensions?: string[] -} - -export function detectChanges( - prev: KnowledgeFragment[], - next: KnowledgeFragment[], - options: DetectChangesOptions = {}, -): DetectChangesResult { - const skipUnverifiable = options.skipUnverifiable ?? true - const warnings: string[] = [] - - const { map: prevMap, warnings: prevWarn } = indexFragments(prev, skipUnverifiable, 'prev') - const { map: nextMap, warnings: nextWarn } = indexFragments(next, skipUnverifiable, 'next') - warnings.push(...prevWarn, ...nextWarn) - - const changes: KnowledgeChange[] = [] - const seen = new Set() - - for (const [id, nextFragment] of nextMap) { - seen.add(id) - const prevFragment = prevMap.get(id) - if (!prevFragment) { - changes.push({ - fragmentId: id, - kind: 'added', - diff: { after: nextFragment.body }, - affectedDimensions: dedup(nextFragment.dimensionHints), - url: nextFragment.provenance.url, - detectedAt: nextFragment.provenance.sourceUpdatedAt, - }) - continue - } - if (prevFragment.bodyHash !== nextFragment.bodyHash) { - changes.push({ - fragmentId: id, - kind: 'modified', - diff: { before: prevFragment.body, after: nextFragment.body }, - affectedDimensions: dedup([...prevFragment.dimensionHints, ...nextFragment.dimensionHints]), - url: nextFragment.provenance.url, - detectedAt: nextFragment.provenance.sourceUpdatedAt, - }) - } - } - - for (const [id, prevFragment] of prevMap) { - if (seen.has(id)) continue - changes.push({ - fragmentId: id, - kind: 'removed', - diff: { before: prevFragment.body }, - affectedDimensions: dedup(prevFragment.dimensionHints), - url: prevFragment.provenance.url, - detectedAt: prevFragment.provenance.sourceUpdatedAt, - }) - } - - const filtered = options.filterDimensions - ? changes.filter((c) => c.affectedDimensions.some((d) => options.filterDimensions?.includes(d))) - : changes - - return { - changes: filtered, - summary: { - added: filtered.filter((c) => c.kind === 'added').length, - removed: filtered.filter((c) => c.kind === 'removed').length, - modified: filtered.filter((c) => c.kind === 'modified').length, - }, - warnings, - } -} - -function indexFragments( - fragments: KnowledgeFragment[], - skipUnverifiable: boolean, - side: string, -): { map: Map; warnings: string[] } { - const map = new Map() - const warnings: string[] = [] - let dropped = 0 - for (const fragment of fragments) { - if (skipUnverifiable && !fragment.provenance.verifiable) { - dropped += 1 - continue - } - if (map.has(fragment.id)) { - warnings.push(`${side}: duplicate fragment id ${fragment.id} — keeping last`) - } - map.set(fragment.id, fragment) - } - if (dropped > 0) { - warnings.push(`${side}: dropped ${dropped} unverifiable fragment(s) before diff`) - } - return { map, warnings } -} - -function dedup(items: T[]): T[] { - return [...new Set(items)] -} diff --git a/src/freshness.ts b/src/freshness.ts deleted file mode 100644 index 22b1450..0000000 --- a/src/freshness.ts +++ /dev/null @@ -1,208 +0,0 @@ -import { readFile } from 'node:fs/promises' -import { join } from 'node:path' -import { z } from 'zod' -import { isMissingFile, writeJsonDurableWithinRoot } from './durable-fs' -import { withKnowledgeMutation, withKnowledgeRead } from './mutation-lock' - -/** - * Knowledge freshness store: tracks when each `(workspaceId, sourceId)` pair - * was last successfully refreshed, and reports staleness against a TTL. - * - * The contract is intentionally minimal and supports scheduled refresh loops: - * - * ```ts - * const store = createFileSystemFreshnessStore({ root: '.agent-knowledge' }) - * for (const source of sources) { - * if (await store.stale({ workspaceId, sourceId: source.id, ttlMs: DAY })) { - * const fragments = await source.fetch({ cacheDir }) - * await persistFragments(fragments) - * await store.mark({ workspaceId, sourceId: source.id, when: new Date() }) - * } - * } - * ``` - * - * Per-tenant isolation is enforced by `workspaceId` keying. There is no - * global mutable state across workspaces. - * - * Two adapters ship in-package: - * - * - `createFileSystemFreshnessStore`: JSON file under the knowledge root, - * mirrors the layout convention already used by `sources.json`. - * - `createD1FreshnessStoreStub`: complete bridge from the `D1Adapter` port - * to this package's freshness interface. The port can be backed by D1, - * PostgreSQL, SQLite, or another application-owned store. - */ - -/** Identity for one freshness record. */ -export interface FreshnessKey { - workspaceId: string - sourceId: string -} - -/** TTL bound for staleness checks. */ -export interface FreshnessTtl extends FreshnessKey { - /** Milliseconds; the record is stale when `Date.now() - last() > ttlMs`. */ - ttlMs: number - /** Injected clock for deterministic tests; defaults to system time. */ - now?: Date -} - -/** Mark argument. */ -export interface FreshnessMark extends FreshnessKey { - when: Date - /** Optional content hash captured at refresh time; aids debugging. */ - contentHash?: string -} - -export interface KnowledgeFreshnessStore { - /** Last refresh time, or null if never refreshed. */ - last(key: FreshnessKey): Promise - /** Record a successful refresh. */ - mark(input: FreshnessMark): Promise - /** True iff `last(key)` is null or older than `ttlMs`. */ - stale(input: FreshnessTtl): Promise - /** All records for a workspace. */ - list(workspaceId: string): Promise -} - -export interface FreshnessRecord { - workspaceId: string - sourceId: string - lastRefreshedAt: string - contentHash?: string -} - -export interface FileSystemFreshnessStoreOptions { - /** - * Knowledge root. The store writes to `/.agent-knowledge/freshness.json`, - * mirroring the convention used by `sources.json`. - */ - root: string -} - -const freshnessRecordSchema = z - .object({ - workspaceId: z.string().min(1), - sourceId: z.string().min(1), - lastRefreshedAt: z.iso.datetime(), - contentHash: z.string().min(1).optional(), - }) - .strict() -const freshnessFileSchema = z - .object({ records: z.record(z.string(), freshnessRecordSchema) }) - .strict() - -/** - * Filesystem-backed implementation. Single JSON file per knowledge root, - * indexed by `${workspaceId}::${sourceId}`. Reads parse on every call; - * cron tick rate is well below the cost of one JSON parse. - * - * Writes share the package-wide filesystem lock, so multiple workers cannot - * overwrite one another or run through an interrupted knowledge promotion. - */ -export function createFileSystemFreshnessStore( - options: FileSystemFreshnessStoreOptions, -): KnowledgeFreshnessStore { - const path = join(options.root, '.agent-knowledge', 'freshness.json') - - const read = async (): Promise> => { - return withKnowledgeRead(options.root, async () => { - try { - return freshnessFileSchema.parse(JSON.parse(await readFile(path, 'utf8'))).records - } catch (error) { - if (isMissingFile(error)) return {} - throw error - } - }) - } - - const write = async (records: Record): Promise => { - await writeJsonDurableWithinRoot( - options.root, - '.agent-knowledge/freshness.json', - freshnessFileSchema.parse({ records }), - ) - } - - return { - async last(key) { - const records = await read() - const record = records[buildKey(key)] - return record ? new Date(record.lastRefreshedAt) : null - }, - async mark(input) { - await withKnowledgeMutation(options.root, async () => { - const records = await read() - records[buildKey(input)] = { - workspaceId: input.workspaceId, - sourceId: input.sourceId, - lastRefreshedAt: input.when.toISOString(), - ...(input.contentHash === undefined ? {} : { contentHash: input.contentHash }), - } - await write(records) - }) - }, - async stale(input) { - const last = await this.last(input) - if (!last) return true - const now = input.now ?? new Date() - return now.getTime() - last.getTime() > input.ttlMs - }, - async list(workspaceId) { - const records = await read() - return Object.values(records).filter((r) => r.workspaceId === workspaceId) - }, - } -} - -/** - * Bridge an application-owned database adapter to `KnowledgeFreshnessStore`. - * The adapter may use D1, PostgreSQL, SQLite, or another durable store. - * - * The expected schema: - * - * ```sql - * CREATE TABLE knowledge_freshness ( - * workspace_id TEXT NOT NULL, - * source_id TEXT NOT NULL, - * last_refreshed_at TEXT NOT NULL, - * content_hash TEXT, - * PRIMARY KEY (workspace_id, source_id) - * ); - * ``` - */ -export interface D1Adapter { - get(workspaceId: string, sourceId: string): Promise - upsert(record: FreshnessRecord): Promise - listByWorkspace(workspaceId: string): Promise -} - -export function createD1FreshnessStoreStub(adapter: D1Adapter): KnowledgeFreshnessStore { - return { - async last(key) { - const record = await adapter.get(key.workspaceId, key.sourceId) - return record ? new Date(record.lastRefreshedAt) : null - }, - async mark(input) { - await adapter.upsert({ - workspaceId: input.workspaceId, - sourceId: input.sourceId, - lastRefreshedAt: input.when.toISOString(), - contentHash: input.contentHash, - }) - }, - async stale(input) { - const last = await this.last(input) - if (!last) return true - const now = input.now ?? new Date() - return now.getTime() - last.getTime() > input.ttlMs - }, - async list(workspaceId) { - return adapter.listByWorkspace(workspaceId) - }, - } -} - -function buildKey(key: FreshnessKey): string { - return `${key.workspaceId}::${key.sourceId}` -} diff --git a/src/index.ts b/src/index.ts index 0c3de0e..c12abaf 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,7 +1,6 @@ export * from './adapters' export * from './agent-candidate' export * from './benchmarks/index' -export * from './changes' export * from './chunking' export * from './citation-lint' export * from './citation-resolution' @@ -35,7 +34,6 @@ export { export * from './eval-readiness' export * from './events' export * from './filesystem-search-provider' -export * from './freshness' export * from './frontmatter' export * from './graph' export * from './ids' @@ -105,7 +103,6 @@ export { export * from './schemas' export * from './search' export * from './sources' -export * from './sources/index' export * from './store' export * from './types' export * from './validate' diff --git a/src/sources/cornell-lii.ts b/src/sources/cornell-lii.ts deleted file mode 100644 index 41c6c9c..0000000 --- a/src/sources/cornell-lii.ts +++ /dev/null @@ -1,188 +0,0 @@ -import { sha256 } from '../ids' -import { htmlToText, innerHtmlById } from './html' -import { politeFetch } from './http' -import type { FetchOpts, KnowledgeFragment, KnowledgeSource } from './types' - -/** - * Cornell Legal Information Institute (LII) source. - * - * Pulls federal US Code sections and Wex encyclopedia entries — the two - * Cornell LII surfaces an agent typically grounds against. The Wex - * "non-compete" page is the canonical test case for the Ryan-LLC v. FTC - * vacatur drift the continuous-ingestion story is designed to catch. - * - * @stable - */ - -const BASE_URL = 'https://www.law.cornell.edu' - -export interface CornellLiiSelector { - /** Either 'uscode' or 'wex'. */ - kind: 'uscode' | 'wex' - /** - * For `uscode`: `/<section>` (e.g. `'18/1836'` for DTSA). - * For `wex`: the slug (e.g. `'non-compete'`). - */ - path: string - /** - * Optional pre-declared eval dimensions affected by this section. If - * omitted, defaults are chosen from `kind` + path heuristics. - */ - dimensionHints?: string[] -} - -export interface CornellLiiSourceOptions { - /** - * Selectors to fetch on each `fetch()` call. The caller (a per-tenant - * workspace config, typically) lists exactly the authorities they need - * tracked. There is no auto-discovery; that would crawl Cornell at - * cron speed, which is what the polite-fetch contract exists to avoid. - */ - selectors: CornellLiiSelector[] - /** Source id override; default is `'cornell-lii'`. */ - id?: string -} - -/** - * Build a Cornell LII source for the listed selectors. - * - * Example: track DTSA + non-compete: - * ``` - * createCornellLiiSource({ - * selectors: [ - * { kind: 'uscode', path: '18/1836' }, - * { kind: 'wex', path: 'non-compete', dimensionHints: ['jurisdictional_accuracy'] }, - * ], - * }) - * ``` - */ -export function createCornellLiiSource(options: CornellLiiSourceOptions): KnowledgeSource { - const id = options.id ?? 'cornell-lii' - return { - id, - name: 'Cornell Legal Information Institute', - description: - 'Federal US Code sections (uscode/text/...) and Wex legal encyclopedia entries from law.cornell.edu.', - async fetch(opts: FetchOpts): Promise<KnowledgeFragment[]> { - const limit = opts.limit ?? options.selectors.length - const selectors = options.selectors.slice(0, limit) - const out: KnowledgeFragment[] = [] - for (const selector of selectors) { - out.push(await fetchOne(id, selector, opts)) - } - return out - }, - } -} - -async function fetchOne( - sourceId: string, - selector: CornellLiiSelector, - opts: FetchOpts, -): Promise<KnowledgeFragment> { - const path = selector.path.replace(/^\/+/, '') - const url = - selector.kind === 'uscode' ? `${BASE_URL}/uscode/text/${path}` : `${BASE_URL}/wex/${path}` - - const response = await politeFetch(url, { - signal: opts.signal, - cacheDir: opts.cacheDir, - }) - - const fragmentId = `${selector.kind}:${selector.path}` - const dimensionHints = selector.dimensionHints ?? defaultDimensionHints(selector) - - if (!response.verifiable) { - return { - id: fragmentId, - title: `Cornell LII ${selector.kind} ${selector.path}`, - body: '', - bodyHash: sha256(''), - provenance: { - url, - sourceUpdatedAt: response.sourceUpdatedAt, - fetchedAt: response.fetchedAt, - jurisdiction: 'US-FED', - verifiable: false, - unverifiableReason: response.unverifiableReason, - }, - dimensionHints, - metadata: { sourceId, status: response.status, fromCache: response.fromCache }, - } - } - - const html = response.body - const title = extractTitle(html, selector) - const body = extractBody(html, selector) - const effective = extractEffectiveDate(html) ?? response.sourceUpdatedAt - - const verifiable = body.length > 50 - return { - id: fragmentId, - title, - body, - bodyHash: sha256(body), - provenance: { - url, - sourceUpdatedAt: effective, - fetchedAt: response.fetchedAt, - jurisdiction: 'US-FED', - verifiable, - unverifiableReason: verifiable ? undefined : 'extracted body too short', - }, - dimensionHints, - metadata: { sourceId, status: response.status, fromCache: response.fromCache }, - } -} - -function extractTitle(html: string, selector: CornellLiiSelector): string { - const h1 = /<h1[^>]*\bid=["']page_title["'][^>]*>([\s\S]*?)<\/h1>/i.exec(html)?.[1] - if (h1) return htmlToText(h1) - const t = /<title>([\s\S]*?)<\/title>/i.exec(html)?.[1] - if (t) return htmlToText(t).split(' | ')[0] ?? `Cornell LII ${selector.path}` - return `Cornell LII ${selector.kind} ${selector.path}` -} - -function extractBody(html: string, selector: CornellLiiSelector): string { - if (selector.kind === 'uscode') { - // The statute text lives inside a <text><div class="text">…</div></text> - // block on US Code section pages. Prefer it; fall back to #tab_default_1 - // which always contains the section body. - const text = /<text>([\s\S]*?)<\/text>/i.exec(html)?.[1] - if (text) return htmlToText(text) - const tab = innerHtmlById(html, 'tab_default_1') - if (tab) return htmlToText(tab) - } - // Wex pages wrap the encyclopedia entry under <div id="main-content"> (newer - // Drupal template) or directly inside <div id="extracted-content"> (older - // template). Try both — lazy regex matching against a nested-div container - // returns the wrong (shorter) slice, so we anchor on the leaf containers. - const mainContent = innerHtmlById(html, 'main-content') - if (mainContent) { - return htmlToText(mainContent.replace(/<h1[\s\S]*?<\/h1>/i, '')) - } - const extracted = innerHtmlById(html, 'extracted-content') - if (extracted) { - return htmlToText(extracted.replace(/<h1[\s\S]*?<\/h1>/i, '')) - } - return htmlToText(html) -} - -function extractEffectiveDate(html: string): string | undefined { - // Cornell LII includes "Editorial Notes" / "Amendments" blocks with - // dates; the most reliable machine-readable signal is the last - // amendment year embedded near the section text. - const amend = /Amendments[\s\S]{0,200}?(\d{4})/i.exec(html)?.[1] - if (amend) { - const y = Number.parseInt(amend, 10) - if (Number.isFinite(y) && y > 1900 && y <= new Date().getUTCFullYear() + 1) { - return new Date(Date.UTC(y, 11, 31)).toISOString() - } - } - return undefined -} - -function defaultDimensionHints(selector: CornellLiiSelector): string[] { - if (selector.kind === 'uscode') return ['jurisdictional_accuracy', 'citation_hygiene'] - return ['citation_hygiene'] -} diff --git a/src/sources/html.ts b/src/sources/html.ts deleted file mode 100644 index ce6ade5..0000000 --- a/src/sources/html.ts +++ /dev/null @@ -1,89 +0,0 @@ -/** - * Minimal HTML helpers used by the shipped sources. - * - * Deliberately not a full DOM parser: every authority we ship against - * (Cornell LII, IRS.gov, state SOS portals) has well-behaved server-rendered - * HTML where regex-based extraction is correct and cheap. Bringing in cheerio - * would add a 1.5MB dependency to a package whose purpose is shipping - * primitives, not parsing arbitrary web pages. - * - * If a future source needs real DOM traversal, it should depend on its own - * parser locally rather than promoting one into the package-wide deps. - * - * @stable - */ - -/** - * Strip HTML tags, collapse whitespace, decode common entities. - * - * Preserves paragraph and line breaks (`</p>`, `<br>`, `</li>`, `</div>`, - * `</h*>`) as `\n` so statute text retains its subsection structure. - */ -export function htmlToText(html: string): string { - return html - .replace(/<script[\s\S]*?<\/script>/gi, '') - .replace(/<style[\s\S]*?<\/style>/gi, '') - .replace(/<noscript[\s\S]*?<\/noscript>/gi, '') - .replace(/<!--([\s\S]*?)-->/g, '') - .replace(/<\s*br\s*\/?>/gi, '\n') - .replace(/<\/(p|li|div|tr|h[1-6]|blockquote|section|article)>/gi, '\n') - .replace(/<[^>]+>/g, '') - .replace(/ /gi, ' ') - .replace(/&/gi, '&') - .replace(/</gi, '<') - .replace(/>/gi, '>') - .replace(/"/gi, '"') - .replace(/'/gi, "'") - .replace(/§/gi, '§') - .replace(/—/gi, '—') - .replace(/–/gi, '–') - .replace(/&#(\d+);/g, (_, code) => String.fromCodePoint(Number(code))) - .replace(/&#x([0-9a-f]+);/gi, (_, code) => String.fromCodePoint(Number.parseInt(code, 16))) - .split('\n') - .map((line) => line.replace(/[\t  ]+/g, ' ').trim()) - .filter((line, idx, all) => !(line === '' && all[idx - 1] === '')) - .join('\n') - .trim() -} - -/** Extract the first match of a regex's first capture group, or undefined. */ -export function firstMatch(html: string, pattern: RegExp): string | undefined { - return pattern.exec(html)?.[1]?.trim() -} - -/** Extract the inner HTML of the first matching tag with id `id`. */ -export function innerHtmlById(html: string, id: string): string | undefined { - const escaped = id.replace(/[.*+?^${}()|[\]\\]/g, '\\$&') - const tagPattern = new RegExp( - `<([a-z][a-z0-9]*)\\b[^>]*\\sid=["']${escaped}["'][^>]*>([\\s\\S]*?)<\\/\\1>`, - 'i', - ) - return tagPattern.exec(html)?.[2] -} - -/** - * Extract every (href, text) pair matching the URL regex. - * Returns absolute URLs by resolving against `baseUrl`. - */ -export function extractLinks( - html: string, - hrefPattern: RegExp, - baseUrl: string, -): { href: string; text: string }[] { - const out: { href: string; text: string }[] = [] - const anchor = /<a\b[^>]*\shref=["']([^"']+)["'][^>]*>([\s\S]*?)<\/a>/gi - for (const match of html.matchAll(anchor)) { - const href = match[1] - const inner = match[2] - if (!href || !inner) continue - if (!hrefPattern.test(href)) continue - const text = htmlToText(inner) - if (!text) continue - try { - out.push({ href: new URL(href, baseUrl).toString(), text }) - } catch { - /* skip malformed URL */ - } - } - return out -} diff --git a/src/sources/http.ts b/src/sources/http.ts deleted file mode 100644 index fada4cc..0000000 --- a/src/sources/http.ts +++ /dev/null @@ -1,262 +0,0 @@ -import { mkdir, readFile, stat, writeFile } from 'node:fs/promises' -import { dirname, join } from 'node:path' -import { sha256 } from '../ids' - -/** - * Polite HTTP fetcher shared by remote sources. - * - * Independent sources share a per-origin throttle because rate-limited sites - * may return block pages instead of 429 responses. Responses are cached by URL - * because many publishers omit reliable ETag and Last-Modified headers. Bodies - * are checked even after a 2xx response because captcha and block pages often - * use successful status codes. - */ - -/** User-Agent string sent on every outbound request. */ -export const POLITE_USER_AGENT = - 'agent-knowledge (+https://github.com/tangle-network/agent-knowledge)' - -/** Minimum gap between successive requests to the same origin (ms). */ -export const MIN_REQUEST_GAP_MS = 1_000 - -/** Maximum response body we will buffer in memory (bytes). */ -export const MAX_RESPONSE_BYTES = 8 * 1024 * 1024 - -const hostThrottle = new Map<string, Promise<void>>() - -export interface PoliteFetchOptions { - signal?: AbortSignal - cacheDir?: string - /** - * Cache age beyond which we re-fetch. Default 1 hour, long enough to - * batch a cron sweep across many selectors, short enough that hourly - * authoritative-page changes get picked up next tick. - */ - cacheTtlMs?: number - /** - * Extra request headers. The fetcher always sets `User-Agent` and - * `Accept`; callers can add `Accept-Language` etc. - */ - headers?: Record<string, string> -} - -export interface PoliteFetchResult { - url: string - status: number - /** Decoded UTF-8 body. Truncated to `MAX_RESPONSE_BYTES`. */ - body: string - /** - * Best-effort source-attested timestamp. Reads `Last-Modified`, - * falling back to `Date`, falling back to fetch time. Always ISO 8601. - */ - sourceUpdatedAt: string - fetchedAt: string - /** True iff the response was satisfied from disk cache. */ - fromCache: boolean - /** - * False on: non-2xx status, captcha/block page heuristic match, or - * decoded body below 200 chars from a host known to serve real content - * (Cornell, IRS, state SOS). `unverifiableReason` carries the why. - */ - verifiable: boolean - unverifiableReason?: string -} - -/** - * Fetch one URL with per-host throttling, on-disk cache, and block-page - * detection. Never throws on network/HTTP failure. It returns a result with - * `verifiable: false` and `unverifiableReason` set so the caller can decide - * whether to skip, retry, or surface. - * - * Throws ONLY on `AbortError` (caller asked to stop) and on cache-write - * failures that indicate a misconfigured filesystem. - */ -export async function politeFetch( - url: string, - options: PoliteFetchOptions = {}, -): Promise<PoliteFetchResult> { - const cacheTtl = options.cacheTtlMs ?? 60 * 60 * 1000 - const cached = options.cacheDir ? await readCache(options.cacheDir, url, cacheTtl) : undefined - if (cached) return cached - - const host = safeHost(url) - await throttleHost(host) - - const fetchedAt = new Date().toISOString() - let response: Response - try { - response = await fetch(url, { - signal: options.signal, - redirect: 'follow', - headers: { - 'User-Agent': POLITE_USER_AGENT, - Accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', - 'Accept-Language': 'en-US,en;q=0.9', - ...(options.headers ?? {}), - }, - }) - } catch (error) { - if ((error as { name?: string }).name === 'AbortError') throw error - const result: PoliteFetchResult = { - url, - status: 0, - body: '', - sourceUpdatedAt: fetchedAt, - fetchedAt, - fromCache: false, - verifiable: false, - unverifiableReason: `network error: ${(error as Error).message}`, - } - if (options.cacheDir) await writeCache(options.cacheDir, url, result) - return result - } - - const text = await readBoundedText(response) - const lastModified = response.headers.get('last-modified') - const dateHeader = response.headers.get('date') - const sourceUpdatedAt = parseHttpDate(lastModified) ?? parseHttpDate(dateHeader) ?? fetchedAt - - const result: PoliteFetchResult = { - url, - status: response.status, - body: text, - sourceUpdatedAt, - fetchedAt, - fromCache: false, - verifiable: true, - } - - if (response.status < 200 || response.status >= 300) { - result.verifiable = false - result.unverifiableReason = `non-2xx status: ${response.status}` - } else if (looksLikeBlockPage(text)) { - result.verifiable = false - result.unverifiableReason = 'block-page heuristic matched' - } else if (text.length < 200 && knownLargeAuthority(host)) { - result.verifiable = false - result.unverifiableReason = `body shorter than expected (${text.length} chars)` - } - - if (options.cacheDir) await writeCache(options.cacheDir, url, result) - return result -} - -/** Reset the in-process throttle map. Test-only. */ -export function __resetHttpThrottle(): void { - hostThrottle.clear() -} - -function safeHost(url: string): string { - try { - return new URL(url).host - } catch { - return 'unknown' - } -} - -async function throttleHost(host: string): Promise<void> { - const prev = hostThrottle.get(host) ?? Promise.resolve() - let release: () => void = () => {} - const next = new Promise<void>((resolve) => { - release = resolve - }) - hostThrottle.set( - host, - prev.then(() => next), - ) - await prev - setTimeout(release, MIN_REQUEST_GAP_MS) -} - -async function readBoundedText(response: Response): Promise<string> { - if (!response.body) return '' - const reader = response.body.getReader() - const chunks: Uint8Array[] = [] - let total = 0 - while (true) { - const { done, value } = await reader.read() - if (done) break - if (!value) continue - total += value.length - if (total > MAX_RESPONSE_BYTES) { - // Stop reading; release the underlying connection. - await reader.cancel() - break - } - chunks.push(value) - } - const merged = new Uint8Array(Math.min(total, MAX_RESPONSE_BYTES)) - let offset = 0 - for (const chunk of chunks) { - const take = Math.min(chunk.length, merged.length - offset) - if (take <= 0) break - merged.set(chunk.subarray(0, take), offset) - offset += take - } - return new TextDecoder('utf-8', { fatal: false }).decode(merged) -} - -function parseHttpDate(value: string | null): string | undefined { - if (!value) return undefined - const ms = Date.parse(value) - return Number.isFinite(ms) ? new Date(ms).toISOString() : undefined -} - -/** Cheap heuristic that catches CAPTCHA, WAF block pages, and "Just a moment" interstitials. */ -export function looksLikeBlockPage(body: string): boolean { - if (!body) return false - const lower = body.toLowerCase() - const markers = [ - 'verify you are human', - 'please enable javascript and cookies', - 'just a moment', - 'access denied', - 'request unsuccessful', - 'cf-error-details', - 'captcha', - 'incapsula', - 'pardon our interruption', - ] - for (const marker of markers) { - if (lower.includes(marker)) return true - } - return false -} - -function knownLargeAuthority(host: string): boolean { - return ( - host.endsWith('law.cornell.edu') || - host.endsWith('irs.gov') || - host.endsWith('sos.ca.gov') || - host.endsWith('sos.state.tx.us') || - host.endsWith('sos.state.us') - ) -} - -function cachePath(cacheDir: string, url: string): string { - const key = sha256(url) - return join(cacheDir, 'http', `${key.slice(0, 2)}`, `${key}.json`) -} - -async function readCache( - cacheDir: string, - url: string, - ttlMs: number, -): Promise<PoliteFetchResult | undefined> { - const path = cachePath(cacheDir, url) - try { - const info = await stat(path) - if (Date.now() - info.mtimeMs > ttlMs) return undefined - const raw = await readFile(path, 'utf8') - const parsed = JSON.parse(raw) as PoliteFetchResult - return { ...parsed, fromCache: true } - } catch { - return undefined - } -} - -async function writeCache(cacheDir: string, url: string, value: PoliteFetchResult): Promise<void> { - const path = cachePath(cacheDir, url) - await mkdir(dirname(path), { recursive: true }) - await writeFile(path, JSON.stringify(value), 'utf8') -} diff --git a/src/sources/index.ts b/src/sources/index.ts deleted file mode 100644 index 03b1a4b..0000000 --- a/src/sources/index.ts +++ /dev/null @@ -1,17 +0,0 @@ -/** - * Pluggable knowledge sources. - * - * @stable types — `KnowledgeSource`, `KnowledgeFragment`, `FetchOpts`, - * `FragmentProvenance` - * @stable http — `politeFetch`, `looksLikeBlockPage`, `POLITE_USER_AGENT` - * @stable html — `htmlToText`, `extractLinks` - * @stable shipped sources — `createCornellLiiSource`, - * `createIrsPublicationsSource`, `createStateSosSource` - */ - -export * from './cornell-lii' -export * from './html' -export * from './http' -export * from './irs-publications' -export * from './state-sos' -export * from './types' diff --git a/src/sources/irs-publications.ts b/src/sources/irs-publications.ts deleted file mode 100644 index c7cbf2d..0000000 --- a/src/sources/irs-publications.ts +++ /dev/null @@ -1,223 +0,0 @@ -import { sha256 } from '../ids' -import { htmlToText } from './html' -import { politeFetch } from './http' -import type { FetchOpts, KnowledgeFragment, KnowledgeSource } from './types' - -/** - * IRS publications source. - * - * Two surfaces: - * - * 1. The publications index at https://www.irs.gov/publications enumerates - * every active publication with its revision year — a single fragment - * with the full table lets change detection notice when a publication - * year flips (e.g. Pub 15 (2025) → Pub 15 (2026)). - * - * 2. Individual publication landing pages at /publications/p<N>[<suffix>] - * return one fragment per publication with summary text. Callers list - * the publications they need tracked via `selectors`. - * - * Revenue procedures are fetched under their numbered URLs; the IRS does - * not maintain a stable HTML index of rev-procs, so the caller passes the - * specific rev-proc paths they care about. - * - * @stable - */ - -const BASE_URL = 'https://www.irs.gov' -const INDEX_URL = `${BASE_URL}/publications` - -export interface IrsPublicationsSourceOptions { - /** - * Specific publication slugs to fetch (e.g. `['p15', 'p17', 'p463']`). - * When `includeIndex` is true (default), the publications index page is - * also fetched as a single fragment so change detection can notice - * year/revision shifts across the whole catalogue. - */ - publications?: string[] - /** - * Revenue procedure paths to fetch (e.g. `['/irb/2024-31_IRB']`). The - * caller passes the exact path; this source does not auto-discover. - */ - revenueProcedures?: string[] - includeIndex?: boolean - id?: string -} - -/** Default eval dimensions for IRS-sourced fragments. */ -export const IRS_DIMENSION_HINTS = ['tax_compliance', 'regulatory_currency', 'citation_hygiene'] - -export function createIrsPublicationsSource( - options: IrsPublicationsSourceOptions = {}, -): KnowledgeSource { - const id = options.id ?? 'irs-publications' - const includeIndex = options.includeIndex ?? true - return { - id, - name: 'IRS Publications', - description: - 'Internal Revenue Service publications index and individual publication landing pages from irs.gov.', - async fetch(opts: FetchOpts): Promise<KnowledgeFragment[]> { - const out: KnowledgeFragment[] = [] - const limit = opts.limit ?? Number.POSITIVE_INFINITY - - if (includeIndex && out.length < limit) { - out.push(await fetchIndex(id, opts)) - } - for (const slug of options.publications ?? []) { - if (out.length >= limit) break - out.push(await fetchPublication(id, slug, opts)) - } - for (const path of options.revenueProcedures ?? []) { - if (out.length >= limit) break - out.push(await fetchRevenueProcedure(id, path, opts)) - } - return out - }, - } -} - -async function fetchIndex(sourceId: string, opts: FetchOpts): Promise<KnowledgeFragment> { - const response = await politeFetch(INDEX_URL, { signal: opts.signal, cacheDir: opts.cacheDir }) - const tablePattern = /<table[\s\S]*?<\/table>/gi - const matches = response.body.match(tablePattern) ?? [] - // Extract the table that lists current-year publications. IRS publishes - // one table per year on the index; the most recent table is always the - // first that mentions a year ≥ current. - const tables = matches.map((t) => htmlToText(t)) - const body = tables - .filter((t) => /Publication\s*\d+/i.test(t)) - .join('\n\n') - .slice(0, 200_000) - - const verifiable = response.verifiable && body.length > 200 - return { - id: 'index', - title: 'IRS Publications Index', - body, - bodyHash: sha256(body), - provenance: { - url: INDEX_URL, - sourceUpdatedAt: response.sourceUpdatedAt, - fetchedAt: response.fetchedAt, - jurisdiction: 'US-FED', - verifiable, - unverifiableReason: - response.unverifiableReason ?? (verifiable ? undefined : 'no publication rows extracted'), - }, - dimensionHints: IRS_DIMENSION_HINTS, - metadata: { sourceId, status: response.status, fromCache: response.fromCache, kind: 'index' }, - } -} - -async function fetchPublication( - sourceId: string, - slug: string, - opts: FetchOpts, -): Promise<KnowledgeFragment> { - const url = `${BASE_URL}/publications/${slug.replace(/^\/+/, '')}` - const response = await politeFetch(url, { signal: opts.signal, cacheDir: opts.cacheDir }) - - const title = extractTitle(response.body, `IRS Publication ${slug}`) - const body = extractMainContent(response.body) - const verifiable = response.verifiable && body.length > 200 - - return { - id: `publication:${slug}`, - title, - body, - bodyHash: sha256(body), - provenance: { - url, - sourceUpdatedAt: extractRevisionDate(response.body) ?? response.sourceUpdatedAt, - fetchedAt: response.fetchedAt, - jurisdiction: 'US-FED', - verifiable, - unverifiableReason: - response.unverifiableReason ?? (verifiable ? undefined : 'no publication body extracted'), - }, - dimensionHints: IRS_DIMENSION_HINTS, - metadata: { - sourceId, - status: response.status, - fromCache: response.fromCache, - kind: 'publication', - slug, - }, - } -} - -async function fetchRevenueProcedure( - sourceId: string, - path: string, - opts: FetchOpts, -): Promise<KnowledgeFragment> { - const url = `${BASE_URL}${path.startsWith('/') ? path : `/${path}`}` - const response = await politeFetch(url, { signal: opts.signal, cacheDir: opts.cacheDir }) - const body = extractMainContent(response.body) - const verifiable = response.verifiable && body.length > 200 - return { - id: `rev-proc:${path}`, - title: extractTitle(response.body, `IRS Revenue Procedure ${path}`), - body, - bodyHash: sha256(body), - provenance: { - url, - sourceUpdatedAt: response.sourceUpdatedAt, - fetchedAt: response.fetchedAt, - jurisdiction: 'US-FED', - verifiable, - unverifiableReason: - response.unverifiableReason ?? - (verifiable ? undefined : 'no revenue-procedure body extracted'), - }, - dimensionHints: [...IRS_DIMENSION_HINTS, 'procedural_currency'], - metadata: { - sourceId, - status: response.status, - fromCache: response.fromCache, - kind: 'rev-proc', - path, - }, - } -} - -function extractTitle(html: string, fallback: string): string { - const og = /<meta\s+property=["']og:title["']\s+content=["']([^"']+)["']/i.exec(html)?.[1] - if (og) return decodeHtml(og) - const title = /<title>([\s\S]*?)<\/title>/i.exec(html)?.[1] - if (title) return htmlToText(title).split(' | ')[0] ?? fallback - return fallback -} - -function extractMainContent(html: string): string { - // IRS uses Drupal — the main publication body is inside <main role="main"> - // or under .field--name-body. We try main first; on miss, body. - const main = /<main\b[\s\S]*?<\/main>/i.exec(html)?.[0] - if (main) { - const noNav = main - .replace(/<nav[\s\S]*?<\/nav>/gi, '') - .replace(/<header[\s\S]*?<\/header>/gi, '') - .replace(/<footer[\s\S]*?<\/footer>/gi, '') - return htmlToText(noNav).slice(0, 200_000) - } - const body = /<body\b[\s\S]*?<\/body>/i.exec(html)?.[0] - return body ? htmlToText(body).slice(0, 200_000) : htmlToText(html).slice(0, 200_000) -} - -function extractRevisionDate(html: string): string | undefined { - // IRS publication pages typically show "Publication X (YYYY)" in the title; - // pulling the year gives a stable revision marker. - const m = /Publication\s+\S+\s*\((\d{4})\)/i.exec(html) - if (m?.[1]) { - const year = Number.parseInt(m[1], 10) - if (Number.isFinite(year) && year >= 2000 && year <= new Date().getUTCFullYear() + 1) { - return new Date(Date.UTC(year, 0, 1)).toISOString() - } - } - return undefined -} - -function decodeHtml(value: string): string { - return htmlToText(value) -} diff --git a/src/sources/state-sos.ts b/src/sources/state-sos.ts deleted file mode 100644 index 40268c5..0000000 --- a/src/sources/state-sos.ts +++ /dev/null @@ -1,151 +0,0 @@ -import { sha256 } from '../ids' -import { htmlToText } from './html' -import { politeFetch } from './http' -import type { FetchOpts, KnowledgeFragment, KnowledgeSource } from './types' - -/** - * Generic Secretary-of-State source. - * - * Every US state SOS surfaces LLC/Corp formation requirements differently - * (CA via static forms pages, DE via division of corporations pages, TX - * via SOSDirect content pages). Rather than baking 50 state-specific - * parsers into this package, the source takes a config that names the URL - * pattern + CSS-equivalent selector + jurisdiction tag. Callers supply one - * config per state they need tracked. - * - * The selector is interpreted as a substring/regex of an HTML element id - * or class — see `StateSosSourceConfig` for the contract. This is - * intentionally minimal; richer extraction belongs in a state-specific - * adapter the consumer authors. - * - * @experimental Interface will likely grow as we add more state coverage. - */ - -export interface StateSosEntity { - /** Stable id for this fragment within the state (e.g. 'llc-formation', 'corp-formation'). */ - id: string - /** Path under the configured `baseUrl` for this entity. */ - path: string - /** - * Extraction selector. Choose one: - * - `{ kind: 'id', value: 'main-content' }` — innermost match of element with that id - * - `{ kind: 'class', value: 'field--name-body' }` — innermost match of element with that class - * - `{ kind: 'regex', value: /<article[\s\S]*?<\/article>/i }` — raw regex - * - `{ kind: 'whole' }` — full body, tags stripped (fallback for unstructured pages) - */ - selector: - | { kind: 'id'; value: string } - | { kind: 'class'; value: string } - | { kind: 'regex'; value: RegExp } - | { kind: 'whole' } - title: string - /** Eval dimensions this entity feeds. */ - dimensionHints?: string[] -} - -export interface StateSosSourceConfig { - /** US state postal code, e.g. 'CA', 'DE', 'TX'. */ - state: string - /** Base URL for the state SOS — e.g. 'https://www.sos.ca.gov'. */ - baseUrl: string - /** Entities this state exposes (LLC, Corp, etc). */ - entities: StateSosEntity[] - /** Source id; default `state-sos:<state>`. */ - id?: string - /** Display name; default `<state> Secretary of State`. */ - name?: string -} - -export function createStateSosSource(config: StateSosSourceConfig): KnowledgeSource { - const id = config.id ?? `state-sos:${config.state.toLowerCase()}` - const name = config.name ?? `${config.state} Secretary of State` - return { - id, - name, - description: `${config.state} Secretary of State filings and formation guidance pages.`, - async fetch(opts: FetchOpts): Promise<KnowledgeFragment[]> { - const limit = opts.limit ?? config.entities.length - const entities = config.entities.slice(0, limit) - const out: KnowledgeFragment[] = [] - for (const entity of entities) { - out.push(await fetchEntity(id, config, entity, opts)) - } - return out - }, - } -} - -async function fetchEntity( - sourceId: string, - config: StateSosSourceConfig, - entity: StateSosEntity, - opts: FetchOpts, -): Promise<KnowledgeFragment> { - const url = joinUrl(config.baseUrl, entity.path) - const response = await politeFetch(url, { signal: opts.signal, cacheDir: opts.cacheDir }) - - const body = response.verifiable ? extractBySelector(response.body, entity.selector) : '' - const verifiable = response.verifiable && body.length > 100 - - return { - id: entity.id, - title: entity.title, - body, - bodyHash: sha256(body), - provenance: { - url, - sourceUpdatedAt: response.sourceUpdatedAt, - fetchedAt: response.fetchedAt, - jurisdiction: `US-${config.state.toUpperCase()}`, - verifiable, - unverifiableReason: - response.unverifiableReason ?? (verifiable ? undefined : 'extracted body too short'), - }, - dimensionHints: entity.dimensionHints ?? [ - 'jurisdictional_accuracy', - 'corporate_formation', - 'citation_hygiene', - ], - metadata: { - sourceId, - status: response.status, - fromCache: response.fromCache, - state: config.state, - }, - } -} - -function extractBySelector(html: string, selector: StateSosEntity['selector']): string { - if (selector.kind === 'whole') { - const main = /<main\b[\s\S]*?<\/main>/i.exec(html)?.[0] - return htmlToText(main ?? html).slice(0, 200_000) - } - if (selector.kind === 'regex') { - const m = selector.value.exec(html)?.[0] - return m ? htmlToText(m).slice(0, 200_000) : '' - } - if (selector.kind === 'id') { - const escaped = selector.value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&') - const pattern = new RegExp( - `<([a-z][a-z0-9]*)\\b[^>]*\\sid=["']${escaped}["'][^>]*>([\\s\\S]*?)<\\/\\1>`, - 'i', - ) - const inner = pattern.exec(html)?.[2] - return inner ? htmlToText(inner).slice(0, 200_000) : '' - } - const escaped = selector.value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&') - const pattern = new RegExp( - `<([a-z][a-z0-9]*)\\b[^>]*\\sclass=["'][^"']*\\b${escaped}\\b[^"']*["'][^>]*>([\\s\\S]*?)<\\/\\1>`, - 'i', - ) - const inner = pattern.exec(html)?.[2] - return inner ? htmlToText(inner).slice(0, 200_000) : '' -} - -function joinUrl(base: string, path: string): string { - try { - return new URL(path, base.endsWith('/') ? base : `${base}/`).toString() - } catch { - return `${base.replace(/\/+$/, '')}/${path.replace(/^\/+/, '')}` - } -} diff --git a/src/sources/types.ts b/src/sources/types.ts deleted file mode 100644 index 38b7789..0000000 --- a/src/sources/types.ts +++ /dev/null @@ -1,153 +0,0 @@ -/** - * Pluggable knowledge source contract. - * - * A `KnowledgeSource` is one external provider of authoritative content that - * an agent's knowledge base should track over time (e.g. Cornell LII US Code, - * IRS publications, a state secretary-of-state filing portal). It returns - * hashable, embed-ready `KnowledgeFragment`s plus enough provenance metadata - * for downstream consumers to: - * - * 1. detect change against a previous snapshot (see `./changes`) - * 2. score freshness on a per-source-id basis (see `./freshness`) - * 3. decide which evals to re-run when the underlying authority moves - * (the `dimensionHints` field is the binding contract for that decision) - * - * Sources MUST be pure with respect to local filesystem state outside the - * cache directory the caller hands them — they read remote authorities and - * return data. They MUST mark `verifiable: false` on any fragment they could - * not fetch and extract (block page, 4xx, parse failure) rather than silently - * substituting empty or partial content. The control loop downstream uses - * `verifiable` to refuse promotion of unusable content. - * - * @stable - */ - -/** - * Per-fetch options the host (control loop / cron / CLI) passes in. - * - * `signal` lets the host abort long-running fetches (rate-limited authority, - * congested network). `cacheDir` is where the source SHOULD write its disk - * cache; an undefined value disables caching (useful in tests). `now` is - * injected for deterministic tests of change-detection windows. - */ -export interface FetchOpts { - /** Abort signal forwarded to the underlying HTTP fetcher. */ - signal?: AbortSignal - /** Absolute path under which the source may cache raw bytes. */ - cacheDir?: string - /** Clock injection for deterministic tests. */ - now?: () => Date - /** - * Maximum number of authority pages the source should fetch in this call. - * Sources MUST respect this bound — exhaustively crawling Cornell LII on - * every cron tick would be both rude and slow. Default is source-specific. - */ - limit?: number - /** - * Source-specific selector string. Examples: - * - cornell-lii: `'uscode/text/18/1836'` or `'wex/non-compete'` - * - irs-publications: `'index'` or `'p15'` - * - state-sos: opaque, see `StateSosSourceConfig` - * - * Sources that don't need a selector ignore this field. - */ - selector?: string -} - -/** - * The standard provenance shape every fragment carries. Kept separate from - * `KnowledgeFragment` so freshness/change code can pass it around without - * also dragging the body text. - */ -export interface FragmentProvenance { - /** Canonical URL the fragment was extracted from. */ - url: string - /** - * Source-attested timestamp: the time the AUTHORITY last updated this - * content, as reported by the source (Last-Modified header, in-page - * effective date, registry generated-at, etc). Falls back to the fetch - * time only when the authority publishes no timestamp. - */ - sourceUpdatedAt: string - /** ISO timestamp the fragment was fetched. */ - fetchedAt: string - /** - * Jurisdiction the content is binding within, if applicable. Use ISO - * country code, US state abbreviation, or 'US-FED' for federal scope. - * Statute sources MUST populate this; reference / encyclopedia sources - * MAY leave it undefined. - */ - jurisdiction?: string - /** - * True iff the configured URL returned an acceptable response and the - * expected content was extracted. False on a block page, rate-limit - * response, 4xx/5xx, or selector miss. This is not publisher authentication - * or cryptographic content verification. Consumers MUST refuse to promote - * `verifiable: false` fragments into citable knowledge. - */ - verifiable: boolean - /** If `verifiable === false`, the reason — surfaced to operators. */ - unverifiableReason?: string -} - -/** - * One unit of authoritative content. Stable hash on `(id, body)` lets change - * detection reason about identity across snapshots. - */ -export interface KnowledgeFragment { - /** - * Stable identity within (sourceId, selector-space). Two fetches against - * the same authority section MUST produce the same `id`. The (sourceId, - * id) pair is the primary key for change detection. - */ - id: string - /** Free-form title — section heading, publication name, etc. */ - title: string - /** Body text, normalised: no HTML tags, line breaks preserved. */ - body: string - /** SHA-256 of `body`. Pre-computed so consumers don't re-hash on diff. */ - bodyHash: string - provenance: FragmentProvenance - /** - * Eval dimensions an agent-eval campaign should re-score when this - * fragment changes. Examples: `citation_hygiene`, `jurisdictional_accuracy`, - * `tax_compliance`, `regulatory_currency`. The eval cron treats this as a - * set, not a contract — adding a new dimension is non-breaking. - * - * This is the load-bearing field for the continuous-ingestion story: a - * Ryan-LLC-style ruling vacates the FTC non-compete rule → the source - * returns a fragment with `jurisdictional_accuracy` in this list → - * `detectChanges()` emits a `KnowledgeChange` carrying that hint → the - * cron knows exactly which agent-eval campaigns to re-run. - */ - dimensionHints: string[] - /** Arbitrary source-specific metadata for debugging / connector wiring. */ - metadata?: Record<string, unknown> -} - -/** - * One pluggable knowledge source. - * - * Implementations: see `./cornell-lii`, `./irs-publications`, `./state-sos`. - * To author a new source, follow the same shape and register it in your - * application's source list — there is no global registry by design (per - * the per-tenant isolation contract; see README). - */ -export interface KnowledgeSource { - /** Stable id — used to key freshness state. MUST NOT change once shipped. */ - id: string - /** Human-readable name for dashboards. */ - name: string - /** One-sentence description: what authority + scope. */ - description: string - /** - * Pull fragments for this source. Sources MUST: - * - rate-limit themselves (>=1 req/sec per source by convention) - * - send a polite User-Agent - * - cache to disk when `opts.cacheDir` is set - * - mark `verifiable: false` rather than throwing on parse/block - * - honour `opts.signal` - * - honour `opts.limit` - */ - fetch(opts: FetchOpts): Promise<KnowledgeFragment[]> -} diff --git a/tests/changes.test.ts b/tests/changes.test.ts deleted file mode 100644 index 46487a4..0000000 --- a/tests/changes.test.ts +++ /dev/null @@ -1,121 +0,0 @@ -import { describe, expect, it } from 'vitest' -import { detectChanges } from '../src/changes' -import { sha256 } from '../src/ids' -import type { KnowledgeFragment } from '../src/sources/types' - -function fragment( - id: string, - body: string, - opts: Partial<KnowledgeFragment & { hints: string[]; verifiable: boolean }> = {}, -): KnowledgeFragment { - return { - id, - title: opts.title ?? id, - body, - bodyHash: sha256(body), - provenance: { - url: opts.provenance?.url ?? `https://example.test/${id}`, - sourceUpdatedAt: opts.provenance?.sourceUpdatedAt ?? '2026-05-14T12:00:00.000Z', - fetchedAt: opts.provenance?.fetchedAt ?? '2026-05-14T12:00:00.000Z', - jurisdiction: opts.provenance?.jurisdiction, - verifiable: opts.verifiable ?? opts.provenance?.verifiable ?? true, - }, - dimensionHints: opts.hints ?? opts.dimensionHints ?? ['citation_hygiene'], - } -} - -describe('detectChanges', () => { - it('flags an added fragment with after-body diff', () => { - const result = detectChanges([], [fragment('wex:non-compete', 'BODY-V1')]) - expect(result.summary).toEqual({ added: 1, removed: 0, modified: 0 }) - expect(result.changes[0]?.kind).toBe('added') - expect(result.changes[0]?.diff?.after).toBe('BODY-V1') - expect(result.changes[0]?.diff?.before).toBeUndefined() - }) - - it('flags a removed fragment with before-body diff', () => { - const result = detectChanges([fragment('uscode:18/1836', 'BODY-V1')], []) - expect(result.summary).toEqual({ added: 0, removed: 1, modified: 0 }) - expect(result.changes[0]?.kind).toBe('removed') - expect(result.changes[0]?.diff?.before).toBe('BODY-V1') - expect(result.changes[0]?.diff?.after).toBeUndefined() - }) - - it('flags a modification when body hash changes', () => { - const prev = [fragment('wex:non-compete', 'BEFORE')] - const next = [fragment('wex:non-compete', 'AFTER')] - const result = detectChanges(prev, next) - expect(result.summary).toEqual({ added: 0, removed: 0, modified: 1 }) - expect(result.changes[0]?.kind).toBe('modified') - expect(result.changes[0]?.diff).toEqual({ before: 'BEFORE', after: 'AFTER' }) - }) - - it('does not flag identical-hash fragments', () => { - const result = detectChanges( - [fragment('wex:non-compete', 'SAME')], - [fragment('wex:non-compete', 'SAME')], - ) - expect(result.summary).toEqual({ added: 0, removed: 0, modified: 0 }) - }) - - it('unions and dedupes dimension hints across before/after', () => { - const prev = [fragment('wex:non-compete', 'BEFORE', { hints: ['citation_hygiene'] })] - const next = [ - fragment('wex:non-compete', 'AFTER', { - hints: ['citation_hygiene', 'jurisdictional_accuracy'], - }), - ] - const result = detectChanges(prev, next) - expect(result.changes[0]?.affectedDimensions.sort()).toEqual([ - 'citation_hygiene', - 'jurisdictional_accuracy', - ]) - }) - - it('drops unverifiable fragments before diffing (no false `removed`)', () => { - const real = fragment('wex:non-compete', 'REAL', { hints: ['jurisdictional_accuracy'] }) - const blocked = fragment('wex:non-compete', '', { verifiable: false, hints: [] }) - const result = detectChanges([real], [blocked]) - expect(result.summary).toEqual({ added: 0, removed: 1, modified: 0 }) - expect(result.warnings.join('\n')).toMatch(/dropped 1 unverifiable/) - }) - - it('filterDimensions narrows the result set', () => { - const next = [ - fragment('a', 'A', { hints: ['citation_hygiene'] }), - fragment('b', 'B', { hints: ['jurisdictional_accuracy'] }), - ] - const result = detectChanges([], next, { filterDimensions: ['jurisdictional_accuracy'] }) - expect(result.changes).toHaveLength(1) - expect(result.changes[0]?.fragmentId).toBe('b') - expect(result.summary).toEqual({ added: 1, removed: 0, modified: 0 }) - }) - - it('warns on duplicate fragment ids', () => { - const result = detectChanges([], [fragment('dup', 'A'), fragment('dup', 'B')]) - expect(result.warnings.join('\n')).toMatch(/duplicate fragment id dup/) - expect(result.changes[0]?.diff?.after).toBe('B') - }) - - it('emits `modified` change tagged for the eval-cron worked example', () => { - // Worked example from the README/PR body: Cornell LII Wex non-compete - // page changes after Ryan-LLC v. FTC. The KnowledgeChange the eval cron - // consumes carries `jurisdictional_accuracy` so it knows to re-run the - // legal-compliance campaign. - const prev = [ - fragment('wex:non-compete', 'Federal non-compete rule effective 2024-09-04', { - hints: ['jurisdictional_accuracy'], - }), - ] - const next = [ - fragment( - 'wex:non-compete', - 'On 2024-08-20 the U.S. District Court for the Northern District of Texas set aside the FTC rule', - { hints: ['jurisdictional_accuracy'] }, - ), - ] - const result = detectChanges(prev, next) - expect(result.summary.modified).toBe(1) - expect(result.changes[0]?.affectedDimensions).toContain('jurisdictional_accuracy') - }) -}) diff --git a/tests/freshness.test.ts b/tests/freshness.test.ts deleted file mode 100644 index fd5db55..0000000 --- a/tests/freshness.test.ts +++ /dev/null @@ -1,159 +0,0 @@ -import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises' -import { tmpdir } from 'node:os' -import { join } from 'node:path' -import { describe, expect, it } from 'vitest' -import { - createD1FreshnessStoreStub, - createFileSystemFreshnessStore, - type D1Adapter, - type FreshnessRecord, -} from '../src/freshness' - -async function withTempRoot<T>(fn: (root: string) => Promise<T>): Promise<T> { - const root = await mkdtemp(join(tmpdir(), 'agent-knowledge-freshness-')) - try { - return await fn(root) - } finally { - await rm(root, { recursive: true, force: true }) - } -} - -describe('createFileSystemFreshnessStore', () => { - it('starts empty — every source is stale', async () => { - await withTempRoot(async (root) => { - const store = createFileSystemFreshnessStore({ root }) - expect(await store.last({ workspaceId: 'w1', sourceId: 'cornell-lii' })).toBeNull() - expect(await store.stale({ workspaceId: 'w1', sourceId: 'cornell-lii', ttlMs: 60_000 })).toBe( - true, - ) - }) - }) - - it('round-trips mark → last → stale=false within TTL', async () => { - await withTempRoot(async (root) => { - const store = createFileSystemFreshnessStore({ root }) - const when = new Date('2026-05-14T12:00:00.000Z') - await store.mark({ workspaceId: 'w1', sourceId: 'cornell-lii', when, contentHash: 'abc' }) - - expect(await store.last({ workspaceId: 'w1', sourceId: 'cornell-lii' })).toEqual(when) - expect( - await store.stale({ - workspaceId: 'w1', - sourceId: 'cornell-lii', - ttlMs: 60_000, - now: new Date('2026-05-14T12:00:30.000Z'), - }), - ).toBe(false) - }) - }) - - it('reports stale once TTL elapses', async () => { - await withTempRoot(async (root) => { - const store = createFileSystemFreshnessStore({ root }) - const when = new Date('2026-05-14T12:00:00.000Z') - await store.mark({ workspaceId: 'w1', sourceId: 'cornell-lii', when }) - expect( - await store.stale({ - workspaceId: 'w1', - sourceId: 'cornell-lii', - ttlMs: 60_000, - now: new Date('2026-05-14T12:02:00.000Z'), - }), - ).toBe(true) - }) - }) - - it('isolates workspaces — w2 cannot read w1 freshness', async () => { - await withTempRoot(async (root) => { - const store = createFileSystemFreshnessStore({ root }) - await store.mark({ - workspaceId: 'w1', - sourceId: 'cornell-lii', - when: new Date('2026-05-14T12:00:00.000Z'), - }) - expect(await store.last({ workspaceId: 'w2', sourceId: 'cornell-lii' })).toBeNull() - expect(await store.stale({ workspaceId: 'w2', sourceId: 'cornell-lii', ttlMs: 60_000 })).toBe( - true, - ) - }) - }) - - it('list returns only that workspace', async () => { - await withTempRoot(async (root) => { - const store = createFileSystemFreshnessStore({ root }) - const t = new Date('2026-05-14T12:00:00.000Z') - await store.mark({ workspaceId: 'w1', sourceId: 'cornell-lii', when: t }) - await store.mark({ workspaceId: 'w1', sourceId: 'irs-publications', when: t }) - await store.mark({ workspaceId: 'w2', sourceId: 'cornell-lii', when: t }) - - const w1 = await store.list('w1') - expect(w1.map((r) => r.sourceId).sort()).toEqual(['cornell-lii', 'irs-publications']) - - const w2 = await store.list('w2') - expect(w2.map((r) => r.sourceId)).toEqual(['cornell-lii']) - }) - }) - - it('serializes concurrent marks without losing writes', async () => { - await withTempRoot(async (root) => { - const stores = [ - createFileSystemFreshnessStore({ root }), - createFileSystemFreshnessStore({ root }), - createFileSystemFreshnessStore({ root }), - ] - const t = new Date('2026-05-14T12:00:00.000Z') - await Promise.all([ - stores[0]!.mark({ workspaceId: 'w1', sourceId: 'a', when: t }), - stores[1]!.mark({ workspaceId: 'w1', sourceId: 'b', when: t }), - stores[2]!.mark({ workspaceId: 'w1', sourceId: 'c', when: t }), - ]) - const list = await stores[0]!.list('w1') - expect(list.map((r) => r.sourceId).sort()).toEqual(['a', 'b', 'c']) - }) - }) - - it('does not replace malformed freshness data', async () => { - await withTempRoot(async (root) => { - const store = createFileSystemFreshnessStore({ root }) - await store.mark({ - workspaceId: 'w1', - sourceId: 'first', - when: new Date('2026-05-14T12:00:00.000Z'), - }) - const path = join(root, '.agent-knowledge', 'freshness.json') - await writeFile(path, '{broken') - - await expect( - store.mark({ - workspaceId: 'w1', - sourceId: 'second', - when: new Date('2026-05-14T12:01:00.000Z'), - }), - ).rejects.toThrow() - await expect(readFile(path, 'utf8')).resolves.toBe('{broken') - }) - }) -}) - -describe('createD1FreshnessStoreStub', () => { - it('delegates last/mark/stale to the adapter', async () => { - const records: Record<string, FreshnessRecord> = {} - const adapter: D1Adapter = { - async get(workspaceId, sourceId) { - return records[`${workspaceId}::${sourceId}`] ?? null - }, - async upsert(record) { - records[`${record.workspaceId}::${record.sourceId}`] = record - }, - async listByWorkspace(workspaceId) { - return Object.values(records).filter((r) => r.workspaceId === workspaceId) - }, - } - const store = createD1FreshnessStoreStub(adapter) - expect(await store.last({ workspaceId: 'w1', sourceId: 'irs-publications' })).toBeNull() - const when = new Date('2026-05-14T12:00:00.000Z') - await store.mark({ workspaceId: 'w1', sourceId: 'irs-publications', when }) - expect(await store.last({ workspaceId: 'w1', sourceId: 'irs-publications' })).toEqual(when) - expect(records['w1::irs-publications']?.lastRefreshedAt).toBe(when.toISOString()) - }) -}) diff --git a/tests/http-cache.test.ts b/tests/http-cache.test.ts deleted file mode 100644 index 1df889f..0000000 --- a/tests/http-cache.test.ts +++ /dev/null @@ -1,192 +0,0 @@ -import { mkdir, mkdtemp, readdir, readFile, rm, writeFile } from 'node:fs/promises' -import { tmpdir } from 'node:os' -import { join } from 'node:path' -import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' -import { sha256 } from '../src/ids' -import { __resetHttpThrottle, POLITE_USER_AGENT, politeFetch } from '../src/sources/index' - -let cacheDir: string -const originalFetch = globalThis.fetch - -beforeEach(async () => { - __resetHttpThrottle() - cacheDir = await mkdtemp(join(tmpdir(), 'agent-knowledge-http-cache-')) -}) - -afterEach(async () => { - await rm(cacheDir, { recursive: true, force: true }) - globalThis.fetch = originalFetch -}) - -function mockFetch(handler: (url: string, init?: RequestInit) => Response): void { - globalThis.fetch = vi.fn(async (input: RequestInfo | URL, init?: RequestInit) => { - const url = typeof input === 'string' ? input : input.toString() - return handler(url, init) - }) as unknown as typeof globalThis.fetch -} - -function html(body: string, status = 200, headers: Record<string, string> = {}): Response { - return new Response(body, { - status, - headers: { 'content-type': 'text/html', ...headers }, - }) -} - -describe('politeFetch', () => { - it('identifies the package without embedding a stale release version', async () => { - let headers: Headers | undefined - mockFetch((_url, init) => { - headers = new Headers(init?.headers) - return html(`<html><body>${'X'.repeat(500)}</body></html>`) - }) - - await politeFetch('https://headers.test/source') - - expect(POLITE_USER_AGENT).toBe( - 'agent-knowledge (+https://github.com/tangle-network/agent-knowledge)', - ) - expect(headers?.get('User-Agent')).toBe(POLITE_USER_AGENT) - }) - - it('returns verifiable=true for a normal 200', async () => { - mockFetch(() => - html(`<html><body>${'X'.repeat(500)}</body></html>`, 200, { - 'last-modified': 'Wed, 01 Jan 2025 00:00:00 GMT', - }), - ) - const result = await politeFetch('https://www.law.cornell.edu/uscode/text/18/1836') - expect(result.status).toBe(200) - expect(result.verifiable).toBe(true) - expect(result.unverifiableReason).toBeUndefined() - expect(result.sourceUpdatedAt).toBe('2025-01-01T00:00:00.000Z') - }) - - it('returns verifiable=false on 404 with reason', async () => { - mockFetch(() => html('Not Found', 404)) - const result = await politeFetch('https://www.law.cornell.edu/uscode/text/99/9999') - expect(result.verifiable).toBe(false) - expect(result.unverifiableReason).toMatch(/non-2xx status: 404/) - }) - - it('returns verifiable=false on block page even with 200', async () => { - mockFetch(() => - html( - `<html><body>${'pad '.repeat(100)}Just a moment — please enable JavaScript</body></html>`, - ), - ) - const result = await politeFetch('https://www.law.cornell.edu/wex/non-compete') - expect(result.status).toBe(200) - expect(result.verifiable).toBe(false) - expect(result.unverifiableReason).toMatch(/block-page heuristic/) - }) - - it('returns verifiable=false on short body from known authority', async () => { - mockFetch(() => html('too short')) - const result = await politeFetch('https://www.irs.gov/publications') - expect(result.verifiable).toBe(false) - expect(result.unverifiableReason).toMatch(/body shorter than expected/) - }) - - it('writes to disk cache and serves the second call from cache', async () => { - const calls: string[] = [] - mockFetch((url) => { - calls.push(url) - return html(`<html><body>${'X'.repeat(500)}</body></html>`) - }) - const url = 'https://www.law.cornell.edu/uscode/text/18/1836' - const a = await politeFetch(url, { cacheDir }) - const b = await politeFetch(url, { cacheDir }) - expect(calls).toHaveLength(1) - expect(a.fromCache).toBe(false) - expect(b.fromCache).toBe(true) - expect(b.body).toBe(a.body) - }) - - it('respects cache TTL — expired entry re-fetches', async () => { - // Plant a stale cache file directly so the test controls its age. - const url = 'https://www.law.cornell.edu/uscode/text/18/1836' - const key = sha256(url) - const path = join(cacheDir, 'http', key.slice(0, 2), `${key}.json`) - await mkdir(join(cacheDir, 'http', key.slice(0, 2)), { recursive: true }) - await writeFile( - path, - JSON.stringify({ - url, - status: 200, - body: 'STALE', - sourceUpdatedAt: '2020-01-01T00:00:00.000Z', - fetchedAt: '2020-01-01T00:00:00.000Z', - fromCache: false, - verifiable: true, - }), - ) - // A one-day-old mtime must exceed the configured one-minute TTL. - const { utimes } = await import('node:fs/promises') - const dayAgo = new Date(Date.now() - 24 * 60 * 60 * 1000) - await utimes(path, dayAgo, dayAgo) - - mockFetch(() => html(`<html><body>${'FRESH'.repeat(100)}</body></html>`)) - const result = await politeFetch(url, { cacheDir, cacheTtlMs: 60_000 }) - expect(result.fromCache).toBe(false) - expect(result.body).toContain('FRESH') - }) - - it("caches failures too so a transient block doesn't storm the authority", async () => { - mockFetch(() => html('Just a moment', 200)) - const url = 'https://www.law.cornell.edu/wex/non-compete' - const first = await politeFetch(url, { cacheDir }) - expect(first.verifiable).toBe(false) - const cached = await readdir(join(cacheDir, 'http'), { recursive: true }).catch(() => []) - expect(cached.length).toBeGreaterThan(0) - }) - - it('serialises requests to the same host (>=1s gap)', async () => { - const timestamps: number[] = [] - mockFetch(() => { - timestamps.push(Date.now()) - return html(`<html><body>${'X'.repeat(500)}</body></html>`) - }) - // Distinct URLs bypass the URL cache but still share the host throttle. - const t0 = Date.now() - await Promise.all([ - politeFetch('https://throttle.test/a'), - politeFetch('https://throttle.test/b'), - ]) - const gap = (timestamps[1] ?? 0) - (timestamps[0] ?? 0) - expect(gap).toBeGreaterThanOrEqual(900) // some leeway for timer precision - // Total elapsed time must include the per-host delay. - expect(Date.now() - t0).toBeGreaterThanOrEqual(900) - }, 10_000) - - it('never throws on a network error — returns verifiable=false', async () => { - mockFetch(() => { - throw new TypeError('network unreachable') - }) - const result = await politeFetch('https://throw.test/x') - expect(result.verifiable).toBe(false) - expect(result.unverifiableReason).toMatch(/network error: network unreachable/) - expect(result.status).toBe(0) - }) - - it('cache entry on success is reusable via subsequent fetches without remocking', async () => { - mockFetch(() => html(`<html><body>${'X'.repeat(500)}</body></html>`)) - const url = 'https://www.law.cornell.edu/uscode/text/18/1836' - await politeFetch(url, { cacheDir }) - - // A network fetch would return 500, so success proves the cache was used. - mockFetch(() => html('boom', 500)) - const second = await politeFetch(url, { cacheDir }) - expect(second.status).toBe(200) - expect(second.fromCache).toBe(true) - }) - - it('cache files are organised by URL hash prefix', async () => { - mockFetch(() => html(`<html><body>${'Y'.repeat(500)}</body></html>`)) - await politeFetch('https://hash.test/foo', { cacheDir }) - const entries = await readdir(join(cacheDir, 'http'), { recursive: true }) - expect(entries.some((e) => e.toString().endsWith('.json'))).toBe(true) - const jsons = entries.filter((e) => e.toString().endsWith('.json')) - const content = await readFile(join(cacheDir, 'http', jsons[0]!.toString()), 'utf8') - expect(JSON.parse(content).url).toBe('https://hash.test/foo') - }) -}) diff --git a/tests/sources-live.test.ts b/tests/sources-live.test.ts deleted file mode 100644 index b51ef51..0000000 --- a/tests/sources-live.test.ts +++ /dev/null @@ -1,158 +0,0 @@ -import { mkdtemp, rm } from 'node:fs/promises' -import { tmpdir } from 'node:os' -import { join } from 'node:path' -import { afterEach, beforeEach, describe, expect, it } from 'vitest' -import { - __resetHttpThrottle, - createCornellLiiSource, - createIrsPublicationsSource, - createStateSosSource, -} from '../src/sources/index' - -/** - * Optional network compatibility checks for Cornell LII, IRS.gov, and CA SOS. - * Each check uses a fresh cache and returns early when the authority reports - * that its response cannot be verified. - */ - -const LIVE_ENABLED = process.env.AGENT_KNOWLEDGE_RUN_NETWORK_TESTS === '1' -const TIMEOUT_MS = 30_000 - -let cacheDir: string - -beforeEach(async () => { - __resetHttpThrottle() - cacheDir = await mkdtemp(join(tmpdir(), 'agent-knowledge-live-cache-')) -}) - -afterEach(async () => { - await rm(cacheDir, { recursive: true, force: true }) -}) - -describe.skipIf(!LIVE_ENABLED)('live: Cornell LII', () => { - it( - 'fetches DTSA 18 USC § 1836 with verifiable=true and statute text', - async () => { - const source = createCornellLiiSource({ - selectors: [{ kind: 'uscode', path: '18/1836' }], - }) - const fragments = await source.fetch({ cacheDir }) - expect(fragments).toHaveLength(1) - const f = fragments[0]! - if (!f.provenance.verifiable) { - console.warn(`Cornell LII unreachable: ${f.provenance.unverifiableReason} — skipping`) - return - } - expect(f.id).toBe('uscode:18/1836') - expect(f.title.toLowerCase()).toContain('1836') - expect(f.provenance.url).toBe('https://www.law.cornell.edu/uscode/text/18/1836') - expect(f.provenance.jurisdiction).toBe('US-FED') - // The statute text must include the Attorney General clause. - expect(f.body.toLowerCase()).toMatch(/attorney general/) - expect(f.dimensionHints).toContain('jurisdictional_accuracy') - // Body hash must be deterministic + non-empty. - expect(f.bodyHash).toMatch(/^[0-9a-f]{64}$/) - }, - TIMEOUT_MS, - ) - - it( - 'fetches a Wex entry (restraint_of_trade — covers the non-compete doctrine surface)', - async () => { - // Cornell's Wex doesn't currently carry a /wex/non-compete slug. The - // doctrinal surface for the Ryan-LLC v. FTC drift sits under - // /wex/restraint_of_trade. If Wex later adds a more specific slug we - // should add it as a second selector — change detection across slugs - // is exactly what `KnowledgeChange.added` is for. - const source = createCornellLiiSource({ - selectors: [ - { kind: 'wex', path: 'restraint_of_trade', dimensionHints: ['jurisdictional_accuracy'] }, - ], - }) - const fragments = await source.fetch({ cacheDir }) - const f = fragments[0]! - if (!f.provenance.verifiable) { - console.warn(`Cornell LII Wex unreachable: ${f.provenance.unverifiableReason} — skipping`) - return - } - expect(f.id).toBe('wex:restraint_of_trade') - expect(f.provenance.url).toBe('https://www.law.cornell.edu/wex/restraint_of_trade') - expect(f.body.length).toBeGreaterThan(200) - expect(f.dimensionHints).toEqual(['jurisdictional_accuracy']) - }, - TIMEOUT_MS, - ) -}) - -describe.skipIf(!LIVE_ENABLED)('live: IRS publications', () => { - it( - 'fetches the publications index with table rows extracted', - async () => { - const source = createIrsPublicationsSource({ publications: [] }) - const fragments = await source.fetch({ cacheDir }) - const index = fragments.find((f) => f.id === 'index') - expect(index).toBeDefined() - if (!index?.provenance.verifiable) { - console.warn(`IRS index unreachable: ${index?.provenance.unverifiableReason} — skipping`) - return - } - expect(index.provenance.url).toBe('https://www.irs.gov/publications') - expect(index.provenance.jurisdiction).toBe('US-FED') - // The publications index must mention at least one current pub. - expect(index.body).toMatch(/Publication\s+15\b/i) - expect(index.dimensionHints).toContain('tax_compliance') - }, - TIMEOUT_MS, - ) - - it( - 'fetches Publication 15 landing page', - async () => { - const source = createIrsPublicationsSource({ - includeIndex: false, - publications: ['p15'], - }) - const fragments = await source.fetch({ cacheDir }) - const pub = fragments[0]! - if (!pub.provenance.verifiable) { - console.warn(`IRS p15 unreachable: ${pub.provenance.unverifiableReason} — skipping`) - return - } - expect(pub.id).toBe('publication:p15') - expect(pub.provenance.url).toBe('https://www.irs.gov/publications/p15') - expect(pub.body.length).toBeGreaterThan(500) - }, - TIMEOUT_MS, - ) -}) - -describe.skipIf(!LIVE_ENABLED)('live: state SOS (California)', () => { - it( - 'fetches CA SOS forms page', - async () => { - const source = createStateSosSource({ - state: 'CA', - baseUrl: 'https://www.sos.ca.gov', - entities: [ - { - id: 'business-entities-forms', - path: '/business-programs/business-entities/forms', - title: 'CA Business Entities Forms', - selector: { kind: 'whole' }, - }, - ], - }) - const fragments = await source.fetch({ cacheDir }) - const f = fragments[0]! - if (!f.provenance.verifiable) { - console.warn(`CA SOS unreachable: ${f.provenance.unverifiableReason} — skipping`) - return - } - expect(f.id).toBe('business-entities-forms') - expect(f.provenance.jurisdiction).toBe('US-CA') - expect(f.body.toLowerCase()).toMatch(/llc|limited liability|forms/) - expect(f.dimensionHints).toContain('jurisdictional_accuracy') - }, - TIMEOUT_MS, - ) -}) diff --git a/tests/sources-mocked.test.ts b/tests/sources-mocked.test.ts deleted file mode 100644 index 664bf59..0000000 --- a/tests/sources-mocked.test.ts +++ /dev/null @@ -1,245 +0,0 @@ -import { mkdtemp, rm } from 'node:fs/promises' -import { tmpdir } from 'node:os' -import { join } from 'node:path' -import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' -import { - __resetHttpThrottle, - createCornellLiiSource, - createIrsPublicationsSource, - createStateSosSource, -} from '../src/sources/index' - -/** Representative authority HTML keeps parser tests deterministic and offline. */ - -let cacheDir: string -const originalFetch = globalThis.fetch - -beforeEach(async () => { - __resetHttpThrottle() - cacheDir = await mkdtemp(join(tmpdir(), 'agent-knowledge-sources-mock-')) -}) - -afterEach(async () => { - await rm(cacheDir, { recursive: true, force: true }) - globalThis.fetch = originalFetch -}) - -function mockOnce(html: string, status = 200): void { - globalThis.fetch = vi.fn( - async () => - new Response(html, { - status, - headers: { 'content-type': 'text/html', 'last-modified': 'Wed, 01 Jan 2025 00:00:00 GMT' }, - }), - ) as unknown as typeof globalThis.fetch -} - -const CORNELL_USCODE_HTML = `<!doctype html><html><head> -<title>18 U.S. Code § 1836 - Civil proceedings | LII / Legal Information Institute - -
NAV
-
-

18 U.S. Code § 1836 - Civil proceedings

-
-
(a) The Attorney General may, in a civil action, obtain appropriate injunctive relief against any violation of this chapter.
-
(b) Private Civil Actions. — An owner of a trade secret may bring a civil action under this subsection.
-
 -
Amendments 2016 — Pub. L. 114–153
-
-
FOOT
-` - -const CORNELL_WEX_HTML = ` -Non-compete | Wex | US Law | LII / Legal Information Institute - -
-

Non-compete

-

${'A non-compete agreement is a contract between an employer and employee. '.repeat(20)}

-

On August 20, 2024, the U.S. District Court for the Northern District of Texas set aside the FTC rule.

-
-` - -const IRS_INDEX_HTML = ` -Publications | Internal Revenue Service - -
- - - -
Publication 15 (2026), (Circular E), Employer's Tax GuidePublication 15 (2026)p15.pdf
Publication 17 (2025), Your Federal Income TaxPublication 17 (2025)p17.pdf
-
` - -const IRS_PUB_HTML = ` -Publication 15 (2026), (Circular E), Employer's Tax Guide | Internal Revenue Service - - -
-

Publication 15 (2026), (Circular E), Employer's Tax Guide

-

${'For use in 2026. This publication explains your tax responsibilities as an employer. '.repeat(10)}

-
` - -const SOS_CA_HTML = ` -Forms, Samples and Fees :: California Secretary of State - -
-

Forms, Samples and Fees

-
-

${'LLC formation is governed by RULLCA. Filing fee is $70. '.repeat(8)}

-
-
` - -describe('cornell-lii parsing', () => { - it('extracts statute body and tags US-FED jurisdiction', async () => { - mockOnce(CORNELL_USCODE_HTML) - const source = createCornellLiiSource({ selectors: [{ kind: 'uscode', path: '18/1836' }] }) - const [f] = await source.fetch({ cacheDir }) - expect(f!.id).toBe('uscode:18/1836') - expect(f!.provenance.url).toBe('https://www.law.cornell.edu/uscode/text/18/1836') - expect(f!.provenance.jurisdiction).toBe('US-FED') - expect(f!.provenance.verifiable).toBe(true) - expect(f!.body).toContain('Attorney General') - expect(f!.body).toContain('Private Civil Actions') - expect(f!.dimensionHints).toContain('jurisdictional_accuracy') - expect(f!.title).toContain('1836') - }) - - it('extracts effective date from the Amendments block', async () => { - mockOnce(CORNELL_USCODE_HTML) - const source = createCornellLiiSource({ selectors: [{ kind: 'uscode', path: '18/1836' }] }) - const [f] = await source.fetch({ cacheDir }) - expect(f!.provenance.sourceUpdatedAt).toBe('2016-12-31T00:00:00.000Z') - }) - - it('handles Wex slugs and uses fallback dimension hints', async () => { - mockOnce(CORNELL_WEX_HTML) - const source = createCornellLiiSource({ selectors: [{ kind: 'wex', path: 'non-compete' }] }) - const [f] = await source.fetch({ cacheDir }) - expect(f!.id).toBe('wex:non-compete') - expect(f!.provenance.url).toBe('https://www.law.cornell.edu/wex/non-compete') - expect(f!.body).toMatch(/Northern District of Texas/) - expect(f!.dimensionHints).toEqual(['citation_hygiene']) - }) - - it('surfaces verifiable=false when authority serves a 4xx', async () => { - mockOnce('not found', 404) - const source = createCornellLiiSource({ selectors: [{ kind: 'uscode', path: '99/9999' }] }) - const [f] = await source.fetch({ cacheDir }) - expect(f!.provenance.verifiable).toBe(false) - expect(f!.body).toBe('') - expect(f!.provenance.unverifiableReason).toMatch(/non-2xx/) - }) -}) - -describe('irs-publications parsing', () => { - it('index fragment captures publication rows', async () => { - mockOnce(IRS_INDEX_HTML) - const source = createIrsPublicationsSource() - const fragments = await source.fetch({ cacheDir }) - const index = fragments.find((f) => f.id === 'index') - expect(index).toBeDefined() - expect(index!.body).toMatch(/Publication\s+15\s*\(2026\)/i) - expect(index!.body).toMatch(/Publication\s+17\s*\(2025\)/i) - expect(index!.provenance.jurisdiction).toBe('US-FED') - expect(index!.dimensionHints).toContain('tax_compliance') - }) - - it('publication fragment captures revision year as sourceUpdatedAt', async () => { - let callCount = 0 - globalThis.fetch = vi.fn(async () => { - callCount += 1 - if (callCount === 1) { - return new Response(IRS_INDEX_HTML, { - status: 200, - headers: { 'content-type': 'text/html' }, - }) - } - return new Response(IRS_PUB_HTML, { - status: 200, - headers: { 'content-type': 'text/html' }, - }) - }) as unknown as typeof globalThis.fetch - - const source = createIrsPublicationsSource({ publications: ['p15'] }) - const fragments = await source.fetch({ cacheDir }) - const pub = fragments.find((f) => f.id === 'publication:p15') - expect(pub).toBeDefined() - expect(pub!.body).toMatch(/Publication 15/) - expect(pub!.provenance.sourceUpdatedAt).toBe('2026-01-01T00:00:00.000Z') - expect(pub!.metadata).toMatchObject({ kind: 'publication', slug: 'p15' }) - }) - - it('respects includeIndex=false and limit', async () => { - mockOnce(IRS_PUB_HTML) - const source = createIrsPublicationsSource({ - includeIndex: false, - publications: ['p15', 'p17'], - }) - const fragments = await source.fetch({ cacheDir, limit: 1 }) - expect(fragments).toHaveLength(1) - expect(fragments[0]!.id).toBe('publication:p15') - }) -}) - -describe('state-sos parsing', () => { - it('extracts via id selector and tags jurisdiction', async () => { - mockOnce(SOS_CA_HTML) - const source = createStateSosSource({ - state: 'CA', - baseUrl: 'https://www.sos.ca.gov', - entities: [ - { - id: 'llc-formation', - path: '/business-programs/business-entities/forms', - title: 'CA LLC Formation', - selector: { kind: 'id', value: 'main-content' }, - }, - ], - }) - const [f] = await source.fetch({ cacheDir }) - expect(f!.id).toBe('llc-formation') - expect(f!.provenance.jurisdiction).toBe('US-CA') - expect(f!.body).toMatch(/RULLCA/) - expect(f!.dimensionHints).toContain('jurisdictional_accuracy') - }) - - it('whole-page selector falls back to main when no id match', async () => { - mockOnce(SOS_CA_HTML) - const source = createStateSosSource({ - state: 'CA', - baseUrl: 'https://www.sos.ca.gov', - entities: [ - { - id: 'forms', - path: '/forms', - title: 'CA Forms', - selector: { kind: 'whole' }, - }, - ], - }) - const [f] = await source.fetch({ cacheDir }) - expect(f!.body).toMatch(/Forms, Samples and Fees/) - expect(f!.body).toMatch(/RULLCA/) - }) - - it('regex selector picks the configured block', async () => { - mockOnce(SOS_CA_HTML) - const source = createStateSosSource({ - state: 'CA', - baseUrl: 'https://www.sos.ca.gov', - entities: [ - { - id: 'forms-h1', - path: '/forms', - title: 'CA Forms', - selector: { kind: 'regex', value: //i }, - }, - ], - }) - const [f] = await source.fetch({ cacheDir }) - // Body is just the h1 text; short, so verifiable=false expected. - expect(f!.title).toBe('CA Forms') - expect(f!.body.toLowerCase()).toContain('forms, samples and fees') - // Short body ⇒ source flags it as not verifiable. - expect(f!.provenance.verifiable).toBe(false) - }) -}) diff --git a/tests/sources-types.test.ts b/tests/sources-types.test.ts deleted file mode 100644 index 4e2b4d8..0000000 --- a/tests/sources-types.test.ts +++ /dev/null @@ -1,109 +0,0 @@ -import { describe, expect, it } from 'vitest' -import { - createCornellLiiSource, - createIrsPublicationsSource, - createStateSosSource, - extractLinks, - htmlToText, - looksLikeBlockPage, -} from '../src/sources/index' - -describe('source factories', () => { - it('cornell-lii default id is stable', () => { - const source = createCornellLiiSource({ selectors: [{ kind: 'uscode', path: '18/1836' }] }) - expect(source.id).toBe('cornell-lii') - expect(source.name).toMatch(/Cornell/) - }) - - it('cornell-lii id override is honoured', () => { - const source = createCornellLiiSource({ - selectors: [{ kind: 'wex', path: 'non-compete' }], - id: 'cornell-lii-trade-secrets', - }) - expect(source.id).toBe('cornell-lii-trade-secrets') - }) - - it('irs-publications default id is stable', () => { - const source = createIrsPublicationsSource() - expect(source.id).toBe('irs-publications') - }) - - it('state-sos id derived from postal code (lower-cased)', () => { - const source = createStateSosSource({ - state: 'CA', - baseUrl: 'https://www.sos.ca.gov', - entities: [], - }) - expect(source.id).toBe('state-sos:ca') - expect(source.name).toBe('CA Secretary of State') - }) -}) - -describe('looksLikeBlockPage', () => { - it('catches Cloudflare interstitial', () => { - expect(looksLikeBlockPage('Just a moment...
Verify you are human')).toBe(true) - }) - - it('catches CAPTCHA pages', () => { - expect(looksLikeBlockPage('
Please complete the CAPTCHA
')).toBe(true) - }) - - it('catches Incapsula block pages', () => { - expect(looksLikeBlockPage(' Request unsuccessful.')).toBe(true) - }) - - it('does not false-positive on real statute text', () => { - expect( - looksLikeBlockPage( - '18 U.S. Code § 1836 - Civil proceedings. The Attorney General may, in a civil action, obtain appropriate injunctive relief...', - ), - ).toBe(false) - }) - - it('empty body is not a block page (different failure path)', () => { - expect(looksLikeBlockPage('')).toBe(false) - }) -}) - -describe('htmlToText', () => { - it('preserves
and

as newlines', () => { - const text = htmlToText('

alpha

beta

gamma
delta
') - expect(text.split('\n')).toEqual(['alpha', 'beta', 'gamma', 'delta']) - }) - - it('strips scripts and styles entirely', () => { - const text = htmlToText('

visible

') - expect(text).toBe('visible') - }) - - it('decodes the section sign and common entities', () => { - const text = htmlToText('

18 U.S. Code § 1836 — "trade secret"

') - expect(text).toContain('§') - expect(text).toContain('—') - expect(text).toContain('"trade secret"') - }) - - it('decodes numeric entities', () => { - expect(htmlToText('

§1836

')).toBe('§1836') - expect(htmlToText('

§1836

')).toBe('§1836') - }) -}) - -describe('extractLinks', () => { - it('filters by href pattern and resolves against base', () => { - const html = - 'Pub 15' + - 'About' + - 'Pub 17' - const links = extractLinks(html, /\/publications\/p\d+/i, 'https://www.irs.gov') - expect(links).toEqual([ - { href: 'https://www.irs.gov/publications/p15', text: 'Pub 15' }, - { href: 'https://www.irs.gov/publications/p17', text: 'Pub 17' }, - ]) - }) - - it('skips empty link text', () => { - const html = '' - expect(extractLinks(html, /\/publications\//i, 'https://www.irs.gov')).toEqual([]) - }) -}) diff --git a/tsdown.config.ts b/tsdown.config.ts index d6367d6..bdbdfaf 100644 --- a/tsdown.config.ts +++ b/tsdown.config.ts @@ -6,7 +6,6 @@ export default defineConfig({ 'viz/index': 'src/viz/index.ts', cli: 'src/cli.ts', 'memory/index': 'src/memory/index.ts', - 'sources/index': 'src/sources/index.ts', 'benchmarks/index': 'src/benchmarks/index.ts', }, format: ['esm'],