From 5a4fd7833e5d8e4125929a3db13dcb577cedafee Mon Sep 17 00:00:00 2001 From: Teal Larson Date: Wed, 5 Aug 2026 14:27:42 -0400 Subject: [PATCH 1/4] test: add the toolkit join parity harness Build the oracle S5 and S6 are judged by: does joining the extracted layers (enrichment + curation) against a catalog snapshot reproduce the toolkit JSON we ship today? - capture-catalog-snapshot.ts pulls a raw /v1/tool_metadata snapshot (paginated at 1000), records total_count, and refuses to write a truncated fetch. The snapshot is gitignored (~10 MB, never committed). - verify-toolkit-join.ts joins a snapshot with the enrichment and curation layers through the real merger (no LLM, committed toolkit as previousToolkit) and reports the first structural difference per file by JSON path. Comparison is structural, not raw-byte: generatedAt is volatile and summary's key position varies across generator versions. Until S3/S5 land the enrichment/ and curation/ directories, the layers are read straight out of the committed toolkit JSON (the default). Proven offline: projecting every committed tool back to its pre-merge shape and re-joining reproduces all 117 toolkits with zero differences; an injected one-character enrichment change and a missing catalog item each fail with a named JSON path. Co-Authored-By: Claude Opus 4.8 --- toolkit-docs-generator/.gitignore | 5 + .../scripts/capture-catalog-snapshot.ts | 151 ++++++ .../scripts/verify-toolkit-join.ts | 487 ++++++++++++++++++ .../tests/scripts/verify-toolkit-join.test.ts | 211 ++++++++ 4 files changed, 854 insertions(+) create mode 100644 toolkit-docs-generator/scripts/capture-catalog-snapshot.ts create mode 100644 toolkit-docs-generator/scripts/verify-toolkit-join.ts create mode 100644 toolkit-docs-generator/tests/scripts/verify-toolkit-join.test.ts diff --git a/toolkit-docs-generator/.gitignore b/toolkit-docs-generator/.gitignore index 38c0afc46..a7f6e0b71 100644 --- a/toolkit-docs-generator/.gitignore +++ b/toolkit-docs-generator/.gitignore @@ -25,3 +25,8 @@ npm-debug.log* # Environment .env .env.local + +# Catalog snapshot for the join parity harness (verify-toolkit-join.ts). +# Raw /v1/tool_metadata capture, ~10 MB, never committed. +catalog-snapshot.json +*.catalog-snapshot.json diff --git a/toolkit-docs-generator/scripts/capture-catalog-snapshot.ts b/toolkit-docs-generator/scripts/capture-catalog-snapshot.ts new file mode 100644 index 000000000..884fa565c --- /dev/null +++ b/toolkit-docs-generator/scripts/capture-catalog-snapshot.ts @@ -0,0 +1,151 @@ +#!/usr/bin/env npx tsx +/** + * Capture a raw catalog snapshot from the Engine `/v1/tool_metadata` endpoint + * to a local file, for the join parity harness (see verify-toolkit-join.ts). + * + * The snapshot is the raw API response items, paginated and concatenated, so + * the verifier can reshape them with the same production code path the build + * will use. It is ~10 MB and must never be committed — `.gitignore` covers the + * default path. + * + * Requires only two env vars: + * ENGINE_API_URL base URL (this script appends /v1/tool_metadata) + * ENGINE_API_KEY bearer token; the key alone scopes the catalog + * + * Usage from the generator package root: + * ENGINE_API_URL=... ENGINE_API_KEY=... pnpm dlx tsx \ + * scripts/capture-catalog-snapshot.ts [--out catalog-snapshot.json] + * + * The request mirrors EngineApiSource: latest-only (the server default), page + * size 1000, `Authorization: Bearer`. `total_count` is recorded so a truncated + * fetch is detectable both here and by the verifier. + */ +import { writeFile } from "fs/promises"; + +const DEFAULT_OUT = "catalog-snapshot.json"; +const PAGE_SIZE = 1000; +const JSON_INDENT = 2; + +type CliOptions = { out: string }; + +type ToolMetadataResponse = { + items: unknown[]; + total_count: number; +}; + +/** Raw payload written to disk; the verifier reads `items` and `totalCount`. */ +type CatalogSnapshot = { + capturedAt: string; + source: string; + totalCount: number; + items: unknown[]; +}; + +const parseArgs = (argv: string[]): CliOptions => { + let out = DEFAULT_OUT; + for (let i = 0; i < argv.length; i++) { + if (argv[i] === "--out" && argv[i + 1]) { + out = argv[i + 1]; + i++; + } + } + return { out }; +}; + +/** Mirror of EngineApiSource.buildEndpointUrl so the request path matches. */ +const buildEndpointUrl = (baseUrl: string): string => { + const normalized = baseUrl.replace(/\/+$/, ""); + return normalized.endsWith("/v1") + ? `${normalized}/tool_metadata` + : `${normalized}/v1/tool_metadata`; +}; + +const requireEnv = (name: string): string => { + const value = process.env[name]; + if (!value) { + throw new Error(`Missing required environment variable: ${name}`); + } + return value; +}; + +const fetchPage = async ( + endpoint: string, + apiKey: string, + offset: number +): Promise => { + const url = new URL(endpoint); + url.searchParams.set("limit", String(PAGE_SIZE)); + url.searchParams.set("offset", String(offset)); + + const response = await fetch(url.toString(), { + headers: { + Authorization: `Bearer ${apiKey}`, + Accept: "application/json", + }, + }); + + if (!response.ok) { + throw new Error( + `Engine API error ${response.status} at offset ${offset}: ${response.statusText}` + ); + } + + const payload = (await response.json()) as ToolMetadataResponse; + if ( + !Array.isArray(payload.items) || + typeof payload.total_count !== "number" + ) { + throw new Error( + `Unexpected response shape at offset ${offset}: missing items[] or total_count` + ); + } + return payload; +}; + +async function main(): Promise { + const { out } = parseArgs(process.argv.slice(2)); + const baseUrl = requireEnv("ENGINE_API_URL"); + const apiKey = requireEnv("ENGINE_API_KEY"); + const endpoint = buildEndpointUrl(baseUrl); + + const items: unknown[] = []; + let totalCount = Number.POSITIVE_INFINITY; + + for (let offset = 0; items.length < totalCount; offset += PAGE_SIZE) { + const page = await fetchPage(endpoint, apiKey, offset); + totalCount = page.total_count; + if (page.items.length === 0) { + // Guard against an endless loop if the server reports more than it returns. + break; + } + items.push(...page.items); + process.stdout.write(`\r fetched ${items.length}/${totalCount} tools`); + } + process.stdout.write("\n"); + + if (items.length !== totalCount) { + throw new Error( + `Truncated fetch: collected ${items.length} tools but total_count is ${totalCount}. ` + + "Refusing to write a partial snapshot." + ); + } + + const snapshot: CatalogSnapshot = { + capturedAt: new Date().toISOString(), + source: endpoint, + totalCount, + items, + }; + + await writeFile( + out, + `${JSON.stringify(snapshot, null, JSON_INDENT)}\n`, + "utf-8" + ); + console.log(`Wrote ${items.length} tools to ${out}`); +} + +main().catch((error) => { + console.error("Snapshot capture failed:", error); + process.exit(1); +}); diff --git a/toolkit-docs-generator/scripts/verify-toolkit-join.ts b/toolkit-docs-generator/scripts/verify-toolkit-join.ts new file mode 100644 index 000000000..8077f1618 --- /dev/null +++ b/toolkit-docs-generator/scripts/verify-toolkit-join.ts @@ -0,0 +1,487 @@ +#!/usr/bin/env npx tsx +/** + * Join parity harness. + * + * Answers one question mechanically: does joining the extracted layers + * (enrichment + curation) against a catalog snapshot reproduce the toolkit + * JSON we ship today? S5 and S6 both rest on that answer, so this harness is + * the oracle they are judged by rather than a human eyeballing diffs. + * + * The join reuses the real generator pipeline — `mergeToolkit` with no LLM and + * the committed toolkit as `previousToolkit` — so every reshape, provider + * resolution, and enrichment carry-forward matches production. The only thing + * this harness supplies is the catalog (from the snapshot) and the layers. + * + * Comparison is structural (by JSON path), not raw-byte: `generatedAt` is the + * one volatile field, and `summary`'s key *position* varies across generator + * versions without changing content, so a byte compare would report false + * differences. Any real content difference exits non-zero and names the file + * and JSON path. + * + * Until S3/S5 land the `enrichment/` and `curation/` directories, run this + * with enrichment and curation read straight out of the committed toolkit + * JSON (the default when `--enrichment` / `--curation` are omitted). + * + * Usage from the generator package root: + * pnpm dlx tsx scripts/verify-toolkit-join.ts \ + * --snapshot catalog-snapshot.json \ + * --reference data/toolkits \ + * [--enrichment data/toolkits] [--curation data/toolkits] \ + * [--metadata tests/fixtures/metadata.json] + * + * Exit code is non-zero if any toolkit differs or any reference tool has no + * matching catalog item. + */ +import { readdir, readFile } from "fs/promises"; +import { join } from "path"; +import { fileURLToPath } from "url"; +import { + groupToolsByToolkit, + mergeToolkit, +} from "../src/merger/data-merger.js"; +import { createDesignSystemMetadataSource } from "../src/sources/design-system-metadata.js"; +import type { IMetadataSource } from "../src/sources/interfaces.js"; +import { createMockMetadataSource } from "../src/sources/mock-metadata.js"; +import { + createDesignSystemProviderIdResolver, + type ProviderIdResolver, +} from "../src/sources/oauth-provider-resolver.js"; +import { parseToolMetadataResponse } from "../src/sources/tool-metadata-schema.js"; +import type { + CustomSections, + MergedToolkit, + ToolDefinition, + ToolkitMetadata, +} from "../src/types/index.js"; + +// ============================================================================ +// Structural diff +// ============================================================================ + +export type JsonPathDiff = { + path: string; + reason: string; + expected: unknown; + actual: unknown; +}; + +/** The only field that legitimately changes run-to-run. */ +const VOLATILE_ROOT_KEYS = ["generatedAt"]; + +const kindOf = (value: unknown): string => { + if (Array.isArray(value)) { + return "array"; + } + if (value === null) { + return "null"; + } + return typeof value; +}; + +/** A key counts as present only when it holds a defined value. */ +const hasValue = (obj: Record, key: string): boolean => + key in obj && obj[key] !== undefined; + +const diffArrays = ( + expected: unknown[], + actual: unknown[], + path: string +): JsonPathDiff | null => { + if (expected.length !== actual.length) { + return { + path, + reason: `array length ${expected.length} vs ${actual.length}`, + expected: expected.length, + actual: actual.length, + }; + } + for (let i = 0; i < expected.length; i++) { + const diff = deepDiff(expected[i], actual[i], `${path}[${i}]`); + if (diff) { + return diff; + } + } + return null; +}; + +/** + * Compare one key's presence. Absent, undefined, and explicit null are all + * equivalent for an optional key (the generator emits `metadata: null` where + * older files omit it). Returns "skip" when equivalent, "recurse" when both + * sides hold a value to compare, or a diff for a real presence mismatch. + */ +const compareKeyPresence = ( + expected: Record, + actual: Record, + key: string, + childPath: string +): JsonPathDiff | "skip" | "recurse" => { + const inExpected = hasValue(expected, key); + const inActual = hasValue(actual, key); + if (!(inExpected || inActual)) { + return "skip"; + } + if (!inExpected) { + return actual[key] === null + ? "skip" + : { + path: childPath, + reason: "unexpected key", + expected: undefined, + actual: actual[key], + }; + } + if (!inActual) { + return expected[key] === null + ? "skip" + : { + path: childPath, + reason: "missing key", + expected: expected[key], + actual: undefined, + }; + } + return "recurse"; +}; + +const diffObjects = ( + expected: Record, + actual: Record, + path: string +): JsonPathDiff | null => { + const keys = [ + ...new Set([...Object.keys(expected), ...Object.keys(actual)]), + ].sort(); + for (const key of keys) { + const childPath = path ? `${path}.${key}` : key; + const verdict = compareKeyPresence(expected, actual, key, childPath); + if (verdict === "skip") { + continue; + } + const diff = + verdict === "recurse" + ? deepDiff(expected[key], actual[key], childPath) + : verdict; + if (diff) { + return diff; + } + } + return null; +}; + +const deepDiff = ( + expected: unknown, + actual: unknown, + path: string +): JsonPathDiff | null => { + const expectedKind = kindOf(expected); + const actualKind = kindOf(actual); + if (expectedKind !== actualKind) { + return { + path, + reason: `type ${expectedKind} vs ${actualKind}`, + expected, + actual, + }; + } + if (expectedKind === "array") { + return diffArrays(expected as unknown[], actual as unknown[], path); + } + if (expectedKind === "object") { + return diffObjects( + expected as Record, + actual as Record, + path + ); + } + return expected === actual + ? null + : { path, reason: "value mismatch", expected, actual }; +}; + +/** + * First structural difference between the expected (committed) toolkit and the + * emitted one, ignoring the volatile `generatedAt`. Returns null when equal. + */ +export const firstDifference = ( + expected: Record, + actual: Record +): JsonPathDiff | null => { + const strip = (value: Record): Record => { + const copy = { ...value }; + for (const key of VOLATILE_ROOT_KEYS) { + delete copy[key]; + } + return copy; + }; + return deepDiff(strip(expected), strip(actual), ""); +}; + +// ============================================================================ +// Join +// ============================================================================ + +/** Read the curation layer out of a committed/merged toolkit. */ +export const curationFromToolkit = ( + toolkit: MergedToolkit +): CustomSections => ({ + documentationChunks: toolkit.documentationChunks ?? [], + customImports: toolkit.customImports ?? [], + subPages: toolkit.subPages ?? [], + toolChunks: {}, +}); + +export type JoinParams = { + toolkitId: string; + catalogTools: readonly ToolDefinition[]; + /** Committed toolkit supplying codeExample / secretsInfo / summary. */ + enrichment: MergedToolkit; + curation: CustomSections | null; + metadata: ToolkitMetadata | null; + resolveProviderId?: ProviderIdResolver; +}; + +/** + * Join catalog tools with the enrichment and curation layers via the real + * merger, then overlay the enrichment layer verbatim. + * + * The merger supplies the catalog-derived fields, curation, and structure. The + * enrichment layer (per-tool `codeExample` / `secretsInfo`, toolkit `summary`) + * is LLM output with no upstream source, so the harness holds it fixed and + * places it directly rather than depending on the merger's carry-forward + * heuristics — those decide whether to *regenerate*, which is a different + * question from whether the frozen layers reproduce today's output. + */ +export const joinToolkit = async ( + params: JoinParams +): Promise => { + const { toolkit } = await mergeToolkit( + params.toolkitId, + params.catalogTools, + params.metadata, + params.curation, + undefined, + { + previousToolkit: params.enrichment, + ...(params.resolveProviderId + ? { resolveProviderId: params.resolveProviderId } + : {}), + } + ); + + const enrichmentByName = new Map( + (params.enrichment.tools ?? []).map((tool) => [tool.qualifiedName, tool]) + ); + for (const tool of toolkit.tools ?? []) { + const enriched = enrichmentByName.get(tool.qualifiedName); + if (!enriched) { + continue; + } + if (enriched.secretsInfo !== undefined) { + tool.secretsInfo = enriched.secretsInfo; + } + // undefined here reads as "absent" — the diff treats it the same as a + // committed tool that omits codeExample entirely. + tool.codeExample = enriched.codeExample; + } + + if (params.enrichment.summary !== undefined) { + toolkit.summary = params.enrichment.summary; + } + if (params.enrichment.summaryStale !== undefined) { + toolkit.summaryStale = params.enrichment.summaryStale; + } + if (params.enrichment.summaryStaleReason !== undefined) { + toolkit.summaryStaleReason = params.enrichment.summaryStaleReason; + } + + return toolkit; +}; + +/** Reference tools whose qualified name has no matching catalog item. */ +export const missingCatalogTools = ( + reference: MergedToolkit, + catalogTools: readonly ToolDefinition[] +): string[] => { + const available = new Set(catalogTools.map((tool) => tool.qualifiedName)); + return (reference.tools ?? []) + .map((tool) => tool.qualifiedName) + .filter((qualifiedName) => !available.has(qualifiedName)); +}; + +export type ToolkitVerification = { + toolkitId: string; + missing: string[]; + diff: JsonPathDiff | null; +}; + +export const verifyOneToolkit = async (params: { + reference: MergedToolkit; + catalogTools: readonly ToolDefinition[]; + enrichment: MergedToolkit; + curation: CustomSections | null; + metadata: ToolkitMetadata | null; + resolveProviderId?: ProviderIdResolver; +}): Promise => { + const toolkitId = params.reference.id; + const missing = missingCatalogTools(params.reference, params.catalogTools); + if (missing.length > 0) { + return { toolkitId, missing, diff: null }; + } + + const emitted = await joinToolkit({ + toolkitId, + catalogTools: params.catalogTools, + enrichment: params.enrichment, + curation: params.curation, + metadata: params.metadata, + resolveProviderId: params.resolveProviderId, + }); + + const diff = firstDifference( + params.reference as unknown as Record, + emitted as unknown as Record + ); + return { toolkitId, missing: [], diff }; +}; + +// ============================================================================ +// CLI +// ============================================================================ + +type CliOptions = { + snapshot: string; + reference: string; + enrichment: string; + curation: string; + metadata?: string; +}; + +const parseArgs = (argv: string[]): CliOptions => { + const values: Record = {}; + for (let i = 0; i < argv.length; i++) { + const flag = argv[i]; + if (flag.startsWith("--") && argv[i + 1]) { + values[flag.slice(2)] = argv[i + 1]; + i++; + } + } + if (!(values.snapshot && values.reference)) { + throw new Error( + "Usage: verify-toolkit-join.ts --snapshot --reference " + + "[--enrichment ] [--curation ] [--metadata ]" + ); + } + return { + snapshot: values.snapshot, + reference: values.reference, + enrichment: values.enrichment ?? values.reference, + curation: values.curation ?? values.reference, + metadata: values.metadata, + }; +}; + +const readToolkitFile = async ( + dir: string, + file: string +): Promise => + JSON.parse(await readFile(join(dir, file), "utf-8")) as MergedToolkit; + +/** Parse the snapshot (capture format or a raw /v1 response) into tools. */ +const loadCatalogTools = async ( + snapshotPath: string +): Promise> => { + const raw = JSON.parse(await readFile(snapshotPath, "utf-8")) as { + items: unknown[]; + total_count?: number; + totalCount?: number; + }; + const total_count = raw.total_count ?? raw.totalCount ?? raw.items.length; + const { items } = parseToolMetadataResponse({ + items: raw.items, + total_count, + }); + return groupToolsByToolkit(items); +}; + +const truncate = (value: unknown): string => { + const text = typeof value === "string" ? value : JSON.stringify(value); + if (text === undefined) { + return "undefined"; + } + return text.length > 120 ? `${text.slice(0, 117)}...` : text; +}; + +async function main(): Promise { + const options = parseArgs(process.argv.slice(2)); + + const catalog = await loadCatalogTools(options.snapshot); + const metadataSource: IMetadataSource = options.metadata + ? createMockMetadataSource(options.metadata) + : await createDesignSystemMetadataSource(); + const resolveProviderId = + (await createDesignSystemProviderIdResolver()) ?? undefined; + + const files = (await readdir(options.reference)) + .filter((file) => file.endsWith(".json") && file !== "index.json") + .sort(); + + let matched = 0; + const problems: string[] = []; + + for (const file of files) { + const reference = await readToolkitFile(options.reference, file); + const toolkitId = reference.id; + const catalogTools = catalog.get(toolkitId) ?? []; + const enrichment = await readToolkitFile(options.enrichment, file); + const curationToolkit = await readToolkitFile(options.curation, file); + const metadata = await metadataSource.getToolkitMetadata(toolkitId); + + const result = await verifyOneToolkit({ + reference, + catalogTools, + enrichment, + curation: curationFromToolkit(curationToolkit), + metadata, + resolveProviderId, + }); + + if (result.missing.length > 0) { + problems.push( + `✗ ${file}: ${result.missing.length} reference tool(s) missing from catalog: ` + + `${result.missing.slice(0, 5).join(", ")}` + ); + } else if (result.diff) { + problems.push( + `✗ ${file}: ${result.diff.path} — ${result.diff.reason}\n` + + ` expected: ${truncate(result.diff.expected)}\n` + + ` actual: ${truncate(result.diff.actual)}` + ); + } else { + matched++; + } + } + + console.log( + `\n${matched}/${files.length} toolkits reproduce the committed output.` + ); + if (problems.length > 0) { + console.log(`\n${problems.length} problem(s):\n`); + for (const problem of problems) { + console.log(problem); + } + process.exit(1); + } + console.log("Parity verified: 0 differences."); +} + +const invokedDirectly = + process.argv[1] !== undefined && + fileURLToPath(import.meta.url) === process.argv[1]; + +if (invokedDirectly) { + main().catch((error) => { + console.error("Verification failed:", error); + process.exit(1); + }); +} diff --git a/toolkit-docs-generator/tests/scripts/verify-toolkit-join.test.ts b/toolkit-docs-generator/tests/scripts/verify-toolkit-join.test.ts new file mode 100644 index 000000000..84e029eb5 --- /dev/null +++ b/toolkit-docs-generator/tests/scripts/verify-toolkit-join.test.ts @@ -0,0 +1,211 @@ +/** + * Tests for the join parity harness (scripts/verify-toolkit-join.ts). + * + * The harness's promise is: joining a catalog against the enrichment and + * curation layers reproduces the committed toolkit JSON. These tests prove + * that offline, without a live catalog, by treating the committed data as its + * own consistent catalog — every committed tool is projected back to the + * pre-merge `ToolDefinition` shape and re-joined. If the merge round-trips, + * the harness reports zero differences; the failure-mode tests confirm it also + * catches injected drift and missing catalog items. + */ +import { readdirSync, readFileSync } from "fs"; +import { join } from "path"; +import { describe, expect, it } from "vitest"; +import { + curationFromToolkit, + firstDifference, + joinToolkit, + missingCatalogTools, + verifyOneToolkit, +} from "../../scripts/verify-toolkit-join.js"; +import { parseToolMetadataResponse } from "../../src/sources/tool-metadata-schema.js"; +import type { + MergedToolkit, + ToolDefinition, + ToolkitMetadata, +} from "../../src/types/index.js"; + +const TOOLKITS_DIR = join(__dirname, "../../data/toolkits"); +const FIXTURES_DIR = join(__dirname, "../fixtures"); + +const listToolkitFiles = (): string[] => + readdirSync(TOOLKITS_DIR) + .filter((file) => file.endsWith(".json") && file !== "index.json") + .sort(); + +const loadToolkit = (file: string): MergedToolkit => + JSON.parse(readFileSync(join(TOOLKITS_DIR, file), "utf-8")) as MergedToolkit; + +/** Reconstruct the design-system metadata that produced a committed toolkit. */ +const metadataFromToolkit = (toolkit: MergedToolkit): ToolkitMetadata => + ({ + id: toolkit.id, + label: toolkit.label, + ...toolkit.metadata, + }) as ToolkitMetadata; + +/** Project a committed tool back to its pre-merge catalog shape. */ +const catalogToolsFromToolkit = (toolkit: MergedToolkit): ToolDefinition[] => + (toolkit.tools ?? []).map((tool) => ({ + name: tool.name, + qualifiedName: tool.qualifiedName, + fullyQualifiedName: tool.fullyQualifiedName, + description: tool.description, + toolkitDescription: toolkit.description, + parameters: tool.parameters, + auth: tool.auth, + secrets: tool.secrets, + output: tool.output, + metadata: tool.metadata ?? null, + })); + +describe("firstDifference", () => { + it("returns null for equal objects", () => { + expect( + firstDifference({ a: 1, b: [1, 2] }, { a: 1, b: [1, 2] }) + ).toBeNull(); + }); + + it("ignores the volatile generatedAt field", () => { + expect( + firstDifference( + { a: 1, generatedAt: "2026-01-01" }, + { a: 1, generatedAt: "2026-08-05" } + ) + ).toBeNull(); + }); + + it("reports the JSON path of a nested value mismatch", () => { + const diff = firstDifference( + { tools: [{ codeExample: { tabLabel: "A" } }] }, + { tools: [{ codeExample: { tabLabel: "B" } }] } + ); + expect(diff?.path).toBe("tools[0].codeExample.tabLabel"); + expect(diff?.reason).toBe("value mismatch"); + }); + + it("reports array length differences", () => { + const diff = firstDifference({ xs: [1, 2] }, { xs: [1] }); + expect(diff?.path).toBe("xs"); + expect(diff?.reason).toContain("array length"); + }); + + it("reports a missing key", () => { + const diff = firstDifference({ a: 1, b: 2 }, { a: 1 }); + expect(diff?.path).toBe("b"); + expect(diff?.reason).toBe("missing key"); + }); +}); + +describe("missingCatalogTools", () => { + const reference = { + id: "Github", + tools: [{ qualifiedName: "Github.A" }, { qualifiedName: "Github.B" }], + } as unknown as MergedToolkit; + + it("returns reference tools absent from the catalog", () => { + const catalog = [{ qualifiedName: "Github.A" }] as ToolDefinition[]; + expect(missingCatalogTools(reference, catalog)).toEqual(["Github.B"]); + }); + + it("returns empty when every reference tool is present", () => { + const catalog = [ + { qualifiedName: "Github.A" }, + { qualifiedName: "Github.B" }, + ] as ToolDefinition[]; + expect(missingCatalogTools(reference, catalog)).toEqual([]); + }); +}); + +describe("raw /v1/tool_metadata reshape", () => { + it("turns API items into pre-merge tools (value_schema.val_type -> type)", () => { + const raw = JSON.parse( + readFileSync(join(FIXTURES_DIR, "engine-api-response.json"), "utf-8") + ) as { items: unknown[]; total_count: number }; + const { items } = parseToolMetadataResponse(raw); + + expect(items.length).toBeGreaterThan(0); + const withParams = items.find((tool) => tool.parameters.length > 0); + expect(withParams).toBeDefined(); + for (const parameter of withParams?.parameters ?? []) { + expect(typeof parameter.type).toBe("string"); + expect(parameter).not.toHaveProperty("value_schema"); + } + }); +}); + +describe("join reproduces committed output", () => { + it("reports zero differences across every committed toolkit", async () => { + const files = listToolkitFiles(); + expect(files.length).toBeGreaterThan(0); + + const mismatches: string[] = []; + for (const file of files) { + const reference = loadToolkit(file); + const result = await verifyOneToolkit({ + reference, + catalogTools: catalogToolsFromToolkit(reference), + enrichment: reference, + curation: curationFromToolkit(reference), + metadata: metadataFromToolkit(reference), + }); + if (result.missing.length > 0) { + mismatches.push(`${file}: missing ${result.missing.join(", ")}`); + } else if (result.diff) { + mismatches.push(`${file}: ${result.diff.path} — ${result.diff.reason}`); + } + } + + expect(mismatches).toEqual([]); + }, 120_000); +}); + +describe("join detects drift", () => { + it("flags a one-character change to an enrichment value", async () => { + const reference = loadToolkit("github.json"); + const toolWithExample = (reference.tools ?? []).find( + (tool) => tool.codeExample !== undefined + ); + expect(toolWithExample).toBeDefined(); + + const enrichment = JSON.parse(JSON.stringify(reference)) as MergedToolkit; + const mutated = (enrichment.tools ?? []).find( + (tool) => tool.qualifiedName === toolWithExample?.qualifiedName + ); + if (mutated?.codeExample) { + mutated.codeExample.tabLabel = `${mutated.codeExample.tabLabel ?? ""}X`; + } + + const emitted = await joinToolkit({ + toolkitId: reference.id, + catalogTools: catalogToolsFromToolkit(reference), + enrichment, + curation: curationFromToolkit(reference), + metadata: metadataFromToolkit(reference), + }); + + const diff = firstDifference( + reference as unknown as Record, + emitted as unknown as Record + ); + expect(diff).not.toBeNull(); + expect(diff?.path).toContain("codeExample"); + }); + + it("flags a reference tool missing from the catalog", async () => { + const reference = loadToolkit("github.json"); + const catalogTools = catalogToolsFromToolkit(reference).slice(1); + + const result = await verifyOneToolkit({ + reference, + catalogTools, + enrichment: reference, + curation: curationFromToolkit(reference), + metadata: metadataFromToolkit(reference), + }); + + expect(result.missing.length).toBeGreaterThan(0); + expect(result.diff).toBeNull(); + }); +}); From c0420ea1d38f49b680e4a18b485834f20693a4cf Mon Sep 17 00:00:00 2001 From: Teal Larson Date: Thu, 6 Aug 2026 11:33:23 -0400 Subject: [PATCH 2/4] fix: satisfy the strict generator typecheck in the harness scripts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit These scripts live in scripts/, which nothing type-checks today — #1106 widens the generator project to cover it. Under that project's exactOptionalPropertyTypes and noUncheckedIndexedAccess they do not compile, so #1106 landing would break this PR. - IMetadataSource was imported from src/sources/interfaces, which does not export it; it lives in src/sources/internal, where all eight other consumers import it from. It is an 'import type', so it erased at runtime and no test could catch it. - Bind the indexed argv reads before use: a truthiness check on argv[i + 1] does not narrow a later, separate read of the same index. - Declare the two optional properties that legitimately receive an explicit undefined as '| undefined' rather than spreading conditionally at each call site. No behaviour change; the harness still reproduces all 117 toolkits. Co-Authored-By: Claude Opus 4.8 --- .../scripts/capture-catalog-snapshot.ts | 5 +++-- .../scripts/verify-toolkit-join.ts | 13 +++++++------ 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/toolkit-docs-generator/scripts/capture-catalog-snapshot.ts b/toolkit-docs-generator/scripts/capture-catalog-snapshot.ts index 884fa565c..75e9aec3c 100644 --- a/toolkit-docs-generator/scripts/capture-catalog-snapshot.ts +++ b/toolkit-docs-generator/scripts/capture-catalog-snapshot.ts @@ -44,8 +44,9 @@ type CatalogSnapshot = { const parseArgs = (argv: string[]): CliOptions => { let out = DEFAULT_OUT; for (let i = 0; i < argv.length; i++) { - if (argv[i] === "--out" && argv[i + 1]) { - out = argv[i + 1]; + const value = argv[i + 1]; + if (argv[i] === "--out" && value) { + out = value; i++; } } diff --git a/toolkit-docs-generator/scripts/verify-toolkit-join.ts b/toolkit-docs-generator/scripts/verify-toolkit-join.ts index 8077f1618..95624bc9a 100644 --- a/toolkit-docs-generator/scripts/verify-toolkit-join.ts +++ b/toolkit-docs-generator/scripts/verify-toolkit-join.ts @@ -40,7 +40,7 @@ import { mergeToolkit, } from "../src/merger/data-merger.js"; import { createDesignSystemMetadataSource } from "../src/sources/design-system-metadata.js"; -import type { IMetadataSource } from "../src/sources/interfaces.js"; +import type { IMetadataSource } from "../src/sources/internal.js"; import { createMockMetadataSource } from "../src/sources/mock-metadata.js"; import { createDesignSystemProviderIdResolver, @@ -238,7 +238,7 @@ export type JoinParams = { enrichment: MergedToolkit; curation: CustomSections | null; metadata: ToolkitMetadata | null; - resolveProviderId?: ProviderIdResolver; + resolveProviderId?: ProviderIdResolver | undefined; }; /** @@ -321,7 +321,7 @@ export const verifyOneToolkit = async (params: { enrichment: MergedToolkit; curation: CustomSections | null; metadata: ToolkitMetadata | null; - resolveProviderId?: ProviderIdResolver; + resolveProviderId?: ProviderIdResolver | undefined; }): Promise => { const toolkitId = params.reference.id; const missing = missingCatalogTools(params.reference, params.catalogTools); @@ -354,15 +354,16 @@ type CliOptions = { reference: string; enrichment: string; curation: string; - metadata?: string; + metadata?: string | undefined; }; const parseArgs = (argv: string[]): CliOptions => { const values: Record = {}; for (let i = 0; i < argv.length; i++) { const flag = argv[i]; - if (flag.startsWith("--") && argv[i + 1]) { - values[flag.slice(2)] = argv[i + 1]; + const value = argv[i + 1]; + if (flag?.startsWith("--") && value) { + values[flag.slice(2)] = value; i++; } } From 2c15de0ea1005a92a8cd9b9719473d2e4525fac4 Mon Sep 17 00:00:00 2001 From: Teal Larson Date: Thu, 6 Aug 2026 17:33:20 -0400 Subject: [PATCH 3/4] fix: retain per-tool chunks in join parity curation --- .../scripts/verify-toolkit-join.ts | 26 +++++++++++----- .../tests/scripts/verify-toolkit-join.test.ts | 31 +++++++++++++++++++ 2 files changed, 49 insertions(+), 8 deletions(-) diff --git a/toolkit-docs-generator/scripts/verify-toolkit-join.ts b/toolkit-docs-generator/scripts/verify-toolkit-join.ts index 95624bc9a..57f4f0142 100644 --- a/toolkit-docs-generator/scripts/verify-toolkit-join.ts +++ b/toolkit-docs-generator/scripts/verify-toolkit-join.ts @@ -222,14 +222,24 @@ export const firstDifference = ( // ============================================================================ /** Read the curation layer out of a committed/merged toolkit. */ -export const curationFromToolkit = ( - toolkit: MergedToolkit -): CustomSections => ({ - documentationChunks: toolkit.documentationChunks ?? [], - customImports: toolkit.customImports ?? [], - subPages: toolkit.subPages ?? [], - toolChunks: {}, -}); +export const curationFromToolkit = (toolkit: MergedToolkit): CustomSections => { + const toolChunks: Record< + string, + NonNullable[string] + > = {}; + for (const tool of toolkit.tools ?? []) { + if (tool.documentationChunks && tool.documentationChunks.length > 0) { + toolChunks[tool.qualifiedName] = tool.documentationChunks; + } + } + + return { + documentationChunks: toolkit.documentationChunks ?? [], + customImports: toolkit.customImports ?? [], + subPages: toolkit.subPages ?? [], + toolChunks, + }; +}; export type JoinParams = { toolkitId: string; diff --git a/toolkit-docs-generator/tests/scripts/verify-toolkit-join.test.ts b/toolkit-docs-generator/tests/scripts/verify-toolkit-join.test.ts index 84e029eb5..1e5f6bbc3 100644 --- a/toolkit-docs-generator/tests/scripts/verify-toolkit-join.test.ts +++ b/toolkit-docs-generator/tests/scripts/verify-toolkit-join.test.ts @@ -118,6 +118,37 @@ describe("missingCatalogTools", () => { }); }); +describe("curationFromToolkit", () => { + it("keeps per-tool documentation chunks in the curation layer", () => { + const sections = curationFromToolkit({ + tools: [ + { + qualifiedName: "Github.CreateIssue", + documentationChunks: [ + { + type: "warning", + location: "description", + position: "after", + content: "Use a narrow repository scope.", + }, + ], + }, + ], + } as MergedToolkit); + + expect(sections.toolChunks).toEqual({ + "Github.CreateIssue": [ + { + type: "warning", + location: "description", + position: "after", + content: "Use a narrow repository scope.", + }, + ], + }); + }); +}); + describe("raw /v1/tool_metadata reshape", () => { it("turns API items into pre-merge tools (value_schema.val_type -> type)", () => { const raw = JSON.parse( From e9a38deba3307db1cc4aee2ce7db52fbc28057a2 Mon Sep 17 00:00:00 2001 From: Teal Larson Date: Fri, 7 Aug 2026 13:32:14 -0400 Subject: [PATCH 4/4] fix: harden join parity harness invocation and curation --- .../scripts/verify-toolkit-join.ts | 14 ++++++++----- .../tests/scripts/verify-toolkit-join.test.ts | 20 +++++++++++++++++-- 2 files changed, 27 insertions(+), 7 deletions(-) diff --git a/toolkit-docs-generator/scripts/verify-toolkit-join.ts b/toolkit-docs-generator/scripts/verify-toolkit-join.ts index 57f4f0142..c07788827 100644 --- a/toolkit-docs-generator/scripts/verify-toolkit-join.ts +++ b/toolkit-docs-generator/scripts/verify-toolkit-join.ts @@ -33,7 +33,7 @@ * matching catalog item. */ import { readdir, readFile } from "fs/promises"; -import { join } from "path"; +import { join, resolve } from "path"; import { fileURLToPath } from "url"; import { groupToolsByToolkit, @@ -229,7 +229,7 @@ export const curationFromToolkit = (toolkit: MergedToolkit): CustomSections => { > = {}; for (const tool of toolkit.tools ?? []) { if (tool.documentationChunks && tool.documentationChunks.length > 0) { - toolChunks[tool.qualifiedName] = tool.documentationChunks; + toolChunks[tool.name] = tool.documentationChunks; } } @@ -486,9 +486,13 @@ async function main(): Promise { console.log("Parity verified: 0 differences."); } -const invokedDirectly = - process.argv[1] !== undefined && - fileURLToPath(import.meta.url) === process.argv[1]; +export const isInvokedDirectly = ( + moduleUrl: string, + argv1: string | undefined +): boolean => + argv1 !== undefined && fileURLToPath(moduleUrl) === resolve(argv1); + +const invokedDirectly = isInvokedDirectly(import.meta.url, process.argv[1]); if (invokedDirectly) { main().catch((error) => { diff --git a/toolkit-docs-generator/tests/scripts/verify-toolkit-join.test.ts b/toolkit-docs-generator/tests/scripts/verify-toolkit-join.test.ts index 1e5f6bbc3..0ccc5189f 100644 --- a/toolkit-docs-generator/tests/scripts/verify-toolkit-join.test.ts +++ b/toolkit-docs-generator/tests/scripts/verify-toolkit-join.test.ts @@ -10,11 +10,13 @@ * catches injected drift and missing catalog items. */ import { readdirSync, readFileSync } from "fs"; -import { join } from "path"; +import { join, resolve } from "path"; +import { pathToFileURL } from "url"; import { describe, expect, it } from "vitest"; import { curationFromToolkit, firstDifference, + isInvokedDirectly, joinToolkit, missingCatalogTools, verifyOneToolkit, @@ -123,6 +125,7 @@ describe("curationFromToolkit", () => { const sections = curationFromToolkit({ tools: [ { + name: "CreateIssue", qualifiedName: "Github.CreateIssue", documentationChunks: [ { @@ -137,7 +140,7 @@ describe("curationFromToolkit", () => { } as MergedToolkit); expect(sections.toolChunks).toEqual({ - "Github.CreateIssue": [ + CreateIssue: [ { type: "warning", location: "description", @@ -149,6 +152,19 @@ describe("curationFromToolkit", () => { }); }); +describe("isInvokedDirectly", () => { + it("recognizes a relative script path", () => { + expect( + isInvokedDirectly( + pathToFileURL( + resolve("toolkit-docs-generator/scripts/verify-toolkit-join.ts") + ).href, + "toolkit-docs-generator/scripts/verify-toolkit-join.ts" + ) + ).toBe(true); + }); +}); + describe("raw /v1/tool_metadata reshape", () => { it("turns API items into pre-merge tools (value_schema.val_type -> type)", () => { const raw = JSON.parse(