diff --git a/toolkit-docs-generator/.gitignore b/toolkit-docs-generator/.gitignore index 38c0afc46..a7f6e0b71 100644 --- a/toolkit-docs-generator/.gitignore +++ b/toolkit-docs-generator/.gitignore @@ -25,3 +25,8 @@ npm-debug.log* # Environment .env .env.local + +# Catalog snapshot for the join parity harness (verify-toolkit-join.ts). +# Raw /v1/tool_metadata capture, ~10 MB, never committed. +catalog-snapshot.json +*.catalog-snapshot.json diff --git a/toolkit-docs-generator/scripts/capture-catalog-snapshot.ts b/toolkit-docs-generator/scripts/capture-catalog-snapshot.ts new file mode 100644 index 000000000..75e9aec3c --- /dev/null +++ b/toolkit-docs-generator/scripts/capture-catalog-snapshot.ts @@ -0,0 +1,152 @@ +#!/usr/bin/env npx tsx +/** + * Capture a raw catalog snapshot from the Engine `/v1/tool_metadata` endpoint + * to a local file, for the join parity harness (see verify-toolkit-join.ts). + * + * The snapshot is the raw API response items, paginated and concatenated, so + * the verifier can reshape them with the same production code path the build + * will use. It is ~10 MB and must never be committed — `.gitignore` covers the + * default path. + * + * Requires only two env vars: + * ENGINE_API_URL base URL (this script appends /v1/tool_metadata) + * ENGINE_API_KEY bearer token; the key alone scopes the catalog + * + * Usage from the generator package root: + * ENGINE_API_URL=... ENGINE_API_KEY=... pnpm dlx tsx \ + * scripts/capture-catalog-snapshot.ts [--out catalog-snapshot.json] + * + * The request mirrors EngineApiSource: latest-only (the server default), page + * size 1000, `Authorization: Bearer`. `total_count` is recorded so a truncated + * fetch is detectable both here and by the verifier. + */ +import { writeFile } from "fs/promises"; + +const DEFAULT_OUT = "catalog-snapshot.json"; +const PAGE_SIZE = 1000; +const JSON_INDENT = 2; + +type CliOptions = { out: string }; + +type ToolMetadataResponse = { + items: unknown[]; + total_count: number; +}; + +/** Raw payload written to disk; the verifier reads `items` and `totalCount`. */ +type CatalogSnapshot = { + capturedAt: string; + source: string; + totalCount: number; + items: unknown[]; +}; + +const parseArgs = (argv: string[]): CliOptions => { + let out = DEFAULT_OUT; + for (let i = 0; i < argv.length; i++) { + const value = argv[i + 1]; + if (argv[i] === "--out" && value) { + out = value; + i++; + } + } + return { out }; +}; + +/** Mirror of EngineApiSource.buildEndpointUrl so the request path matches. */ +const buildEndpointUrl = (baseUrl: string): string => { + const normalized = baseUrl.replace(/\/+$/, ""); + return normalized.endsWith("/v1") + ? `${normalized}/tool_metadata` + : `${normalized}/v1/tool_metadata`; +}; + +const requireEnv = (name: string): string => { + const value = process.env[name]; + if (!value) { + throw new Error(`Missing required environment variable: ${name}`); + } + return value; +}; + +const fetchPage = async ( + endpoint: string, + apiKey: string, + offset: number +): Promise => { + const url = new URL(endpoint); + url.searchParams.set("limit", String(PAGE_SIZE)); + url.searchParams.set("offset", String(offset)); + + const response = await fetch(url.toString(), { + headers: { + Authorization: `Bearer ${apiKey}`, + Accept: "application/json", + }, + }); + + if (!response.ok) { + throw new Error( + `Engine API error ${response.status} at offset ${offset}: ${response.statusText}` + ); + } + + const payload = (await response.json()) as ToolMetadataResponse; + if ( + !Array.isArray(payload.items) || + typeof payload.total_count !== "number" + ) { + throw new Error( + `Unexpected response shape at offset ${offset}: missing items[] or total_count` + ); + } + return payload; +}; + +async function main(): Promise { + const { out } = parseArgs(process.argv.slice(2)); + const baseUrl = requireEnv("ENGINE_API_URL"); + const apiKey = requireEnv("ENGINE_API_KEY"); + const endpoint = buildEndpointUrl(baseUrl); + + const items: unknown[] = []; + let totalCount = Number.POSITIVE_INFINITY; + + for (let offset = 0; items.length < totalCount; offset += PAGE_SIZE) { + const page = await fetchPage(endpoint, apiKey, offset); + totalCount = page.total_count; + if (page.items.length === 0) { + // Guard against an endless loop if the server reports more than it returns. + break; + } + items.push(...page.items); + process.stdout.write(`\r fetched ${items.length}/${totalCount} tools`); + } + process.stdout.write("\n"); + + if (items.length !== totalCount) { + throw new Error( + `Truncated fetch: collected ${items.length} tools but total_count is ${totalCount}. ` + + "Refusing to write a partial snapshot." + ); + } + + const snapshot: CatalogSnapshot = { + capturedAt: new Date().toISOString(), + source: endpoint, + totalCount, + items, + }; + + await writeFile( + out, + `${JSON.stringify(snapshot, null, JSON_INDENT)}\n`, + "utf-8" + ); + console.log(`Wrote ${items.length} tools to ${out}`); +} + +main().catch((error) => { + console.error("Snapshot capture failed:", error); + process.exit(1); +}); diff --git a/toolkit-docs-generator/scripts/verify-toolkit-join.ts b/toolkit-docs-generator/scripts/verify-toolkit-join.ts new file mode 100644 index 000000000..c07788827 --- /dev/null +++ b/toolkit-docs-generator/scripts/verify-toolkit-join.ts @@ -0,0 +1,502 @@ +#!/usr/bin/env npx tsx +/** + * Join parity harness. + * + * Answers one question mechanically: does joining the extracted layers + * (enrichment + curation) against a catalog snapshot reproduce the toolkit + * JSON we ship today? S5 and S6 both rest on that answer, so this harness is + * the oracle they are judged by rather than a human eyeballing diffs. + * + * The join reuses the real generator pipeline — `mergeToolkit` with no LLM and + * the committed toolkit as `previousToolkit` — so every reshape, provider + * resolution, and enrichment carry-forward matches production. The only thing + * this harness supplies is the catalog (from the snapshot) and the layers. + * + * Comparison is structural (by JSON path), not raw-byte: `generatedAt` is the + * one volatile field, and `summary`'s key *position* varies across generator + * versions without changing content, so a byte compare would report false + * differences. Any real content difference exits non-zero and names the file + * and JSON path. + * + * Until S3/S5 land the `enrichment/` and `curation/` directories, run this + * with enrichment and curation read straight out of the committed toolkit + * JSON (the default when `--enrichment` / `--curation` are omitted). + * + * Usage from the generator package root: + * pnpm dlx tsx scripts/verify-toolkit-join.ts \ + * --snapshot catalog-snapshot.json \ + * --reference data/toolkits \ + * [--enrichment data/toolkits] [--curation data/toolkits] \ + * [--metadata tests/fixtures/metadata.json] + * + * Exit code is non-zero if any toolkit differs or any reference tool has no + * matching catalog item. + */ +import { readdir, readFile } from "fs/promises"; +import { join, resolve } from "path"; +import { fileURLToPath } from "url"; +import { + groupToolsByToolkit, + mergeToolkit, +} from "../src/merger/data-merger.js"; +import { createDesignSystemMetadataSource } from "../src/sources/design-system-metadata.js"; +import type { IMetadataSource } from "../src/sources/internal.js"; +import { createMockMetadataSource } from "../src/sources/mock-metadata.js"; +import { + createDesignSystemProviderIdResolver, + type ProviderIdResolver, +} from "../src/sources/oauth-provider-resolver.js"; +import { parseToolMetadataResponse } from "../src/sources/tool-metadata-schema.js"; +import type { + CustomSections, + MergedToolkit, + ToolDefinition, + ToolkitMetadata, +} from "../src/types/index.js"; + +// ============================================================================ +// Structural diff +// ============================================================================ + +export type JsonPathDiff = { + path: string; + reason: string; + expected: unknown; + actual: unknown; +}; + +/** The only field that legitimately changes run-to-run. */ +const VOLATILE_ROOT_KEYS = ["generatedAt"]; + +const kindOf = (value: unknown): string => { + if (Array.isArray(value)) { + return "array"; + } + if (value === null) { + return "null"; + } + return typeof value; +}; + +/** A key counts as present only when it holds a defined value. */ +const hasValue = (obj: Record, key: string): boolean => + key in obj && obj[key] !== undefined; + +const diffArrays = ( + expected: unknown[], + actual: unknown[], + path: string +): JsonPathDiff | null => { + if (expected.length !== actual.length) { + return { + path, + reason: `array length ${expected.length} vs ${actual.length}`, + expected: expected.length, + actual: actual.length, + }; + } + for (let i = 0; i < expected.length; i++) { + const diff = deepDiff(expected[i], actual[i], `${path}[${i}]`); + if (diff) { + return diff; + } + } + return null; +}; + +/** + * Compare one key's presence. Absent, undefined, and explicit null are all + * equivalent for an optional key (the generator emits `metadata: null` where + * older files omit it). Returns "skip" when equivalent, "recurse" when both + * sides hold a value to compare, or a diff for a real presence mismatch. + */ +const compareKeyPresence = ( + expected: Record, + actual: Record, + key: string, + childPath: string +): JsonPathDiff | "skip" | "recurse" => { + const inExpected = hasValue(expected, key); + const inActual = hasValue(actual, key); + if (!(inExpected || inActual)) { + return "skip"; + } + if (!inExpected) { + return actual[key] === null + ? "skip" + : { + path: childPath, + reason: "unexpected key", + expected: undefined, + actual: actual[key], + }; + } + if (!inActual) { + return expected[key] === null + ? "skip" + : { + path: childPath, + reason: "missing key", + expected: expected[key], + actual: undefined, + }; + } + return "recurse"; +}; + +const diffObjects = ( + expected: Record, + actual: Record, + path: string +): JsonPathDiff | null => { + const keys = [ + ...new Set([...Object.keys(expected), ...Object.keys(actual)]), + ].sort(); + for (const key of keys) { + const childPath = path ? `${path}.${key}` : key; + const verdict = compareKeyPresence(expected, actual, key, childPath); + if (verdict === "skip") { + continue; + } + const diff = + verdict === "recurse" + ? deepDiff(expected[key], actual[key], childPath) + : verdict; + if (diff) { + return diff; + } + } + return null; +}; + +const deepDiff = ( + expected: unknown, + actual: unknown, + path: string +): JsonPathDiff | null => { + const expectedKind = kindOf(expected); + const actualKind = kindOf(actual); + if (expectedKind !== actualKind) { + return { + path, + reason: `type ${expectedKind} vs ${actualKind}`, + expected, + actual, + }; + } + if (expectedKind === "array") { + return diffArrays(expected as unknown[], actual as unknown[], path); + } + if (expectedKind === "object") { + return diffObjects( + expected as Record, + actual as Record, + path + ); + } + return expected === actual + ? null + : { path, reason: "value mismatch", expected, actual }; +}; + +/** + * First structural difference between the expected (committed) toolkit and the + * emitted one, ignoring the volatile `generatedAt`. Returns null when equal. + */ +export const firstDifference = ( + expected: Record, + actual: Record +): JsonPathDiff | null => { + const strip = (value: Record): Record => { + const copy = { ...value }; + for (const key of VOLATILE_ROOT_KEYS) { + delete copy[key]; + } + return copy; + }; + return deepDiff(strip(expected), strip(actual), ""); +}; + +// ============================================================================ +// Join +// ============================================================================ + +/** Read the curation layer out of a committed/merged toolkit. */ +export const curationFromToolkit = (toolkit: MergedToolkit): CustomSections => { + const toolChunks: Record< + string, + NonNullable[string] + > = {}; + for (const tool of toolkit.tools ?? []) { + if (tool.documentationChunks && tool.documentationChunks.length > 0) { + toolChunks[tool.name] = tool.documentationChunks; + } + } + + return { + documentationChunks: toolkit.documentationChunks ?? [], + customImports: toolkit.customImports ?? [], + subPages: toolkit.subPages ?? [], + toolChunks, + }; +}; + +export type JoinParams = { + toolkitId: string; + catalogTools: readonly ToolDefinition[]; + /** Committed toolkit supplying codeExample / secretsInfo / summary. */ + enrichment: MergedToolkit; + curation: CustomSections | null; + metadata: ToolkitMetadata | null; + resolveProviderId?: ProviderIdResolver | undefined; +}; + +/** + * Join catalog tools with the enrichment and curation layers via the real + * merger, then overlay the enrichment layer verbatim. + * + * The merger supplies the catalog-derived fields, curation, and structure. The + * enrichment layer (per-tool `codeExample` / `secretsInfo`, toolkit `summary`) + * is LLM output with no upstream source, so the harness holds it fixed and + * places it directly rather than depending on the merger's carry-forward + * heuristics — those decide whether to *regenerate*, which is a different + * question from whether the frozen layers reproduce today's output. + */ +export const joinToolkit = async ( + params: JoinParams +): Promise => { + const { toolkit } = await mergeToolkit( + params.toolkitId, + params.catalogTools, + params.metadata, + params.curation, + undefined, + { + previousToolkit: params.enrichment, + ...(params.resolveProviderId + ? { resolveProviderId: params.resolveProviderId } + : {}), + } + ); + + const enrichmentByName = new Map( + (params.enrichment.tools ?? []).map((tool) => [tool.qualifiedName, tool]) + ); + for (const tool of toolkit.tools ?? []) { + const enriched = enrichmentByName.get(tool.qualifiedName); + if (!enriched) { + continue; + } + if (enriched.secretsInfo !== undefined) { + tool.secretsInfo = enriched.secretsInfo; + } + // undefined here reads as "absent" — the diff treats it the same as a + // committed tool that omits codeExample entirely. + tool.codeExample = enriched.codeExample; + } + + if (params.enrichment.summary !== undefined) { + toolkit.summary = params.enrichment.summary; + } + if (params.enrichment.summaryStale !== undefined) { + toolkit.summaryStale = params.enrichment.summaryStale; + } + if (params.enrichment.summaryStaleReason !== undefined) { + toolkit.summaryStaleReason = params.enrichment.summaryStaleReason; + } + + return toolkit; +}; + +/** Reference tools whose qualified name has no matching catalog item. */ +export const missingCatalogTools = ( + reference: MergedToolkit, + catalogTools: readonly ToolDefinition[] +): string[] => { + const available = new Set(catalogTools.map((tool) => tool.qualifiedName)); + return (reference.tools ?? []) + .map((tool) => tool.qualifiedName) + .filter((qualifiedName) => !available.has(qualifiedName)); +}; + +export type ToolkitVerification = { + toolkitId: string; + missing: string[]; + diff: JsonPathDiff | null; +}; + +export const verifyOneToolkit = async (params: { + reference: MergedToolkit; + catalogTools: readonly ToolDefinition[]; + enrichment: MergedToolkit; + curation: CustomSections | null; + metadata: ToolkitMetadata | null; + resolveProviderId?: ProviderIdResolver | undefined; +}): Promise => { + const toolkitId = params.reference.id; + const missing = missingCatalogTools(params.reference, params.catalogTools); + if (missing.length > 0) { + return { toolkitId, missing, diff: null }; + } + + const emitted = await joinToolkit({ + toolkitId, + catalogTools: params.catalogTools, + enrichment: params.enrichment, + curation: params.curation, + metadata: params.metadata, + resolveProviderId: params.resolveProviderId, + }); + + const diff = firstDifference( + params.reference as unknown as Record, + emitted as unknown as Record + ); + return { toolkitId, missing: [], diff }; +}; + +// ============================================================================ +// CLI +// ============================================================================ + +type CliOptions = { + snapshot: string; + reference: string; + enrichment: string; + curation: string; + metadata?: string | undefined; +}; + +const parseArgs = (argv: string[]): CliOptions => { + const values: Record = {}; + for (let i = 0; i < argv.length; i++) { + const flag = argv[i]; + const value = argv[i + 1]; + if (flag?.startsWith("--") && value) { + values[flag.slice(2)] = value; + i++; + } + } + if (!(values.snapshot && values.reference)) { + throw new Error( + "Usage: verify-toolkit-join.ts --snapshot --reference " + + "[--enrichment ] [--curation ] [--metadata ]" + ); + } + return { + snapshot: values.snapshot, + reference: values.reference, + enrichment: values.enrichment ?? values.reference, + curation: values.curation ?? values.reference, + metadata: values.metadata, + }; +}; + +const readToolkitFile = async ( + dir: string, + file: string +): Promise => + JSON.parse(await readFile(join(dir, file), "utf-8")) as MergedToolkit; + +/** Parse the snapshot (capture format or a raw /v1 response) into tools. */ +const loadCatalogTools = async ( + snapshotPath: string +): Promise> => { + const raw = JSON.parse(await readFile(snapshotPath, "utf-8")) as { + items: unknown[]; + total_count?: number; + totalCount?: number; + }; + const total_count = raw.total_count ?? raw.totalCount ?? raw.items.length; + const { items } = parseToolMetadataResponse({ + items: raw.items, + total_count, + }); + return groupToolsByToolkit(items); +}; + +const truncate = (value: unknown): string => { + const text = typeof value === "string" ? value : JSON.stringify(value); + if (text === undefined) { + return "undefined"; + } + return text.length > 120 ? `${text.slice(0, 117)}...` : text; +}; + +async function main(): Promise { + const options = parseArgs(process.argv.slice(2)); + + const catalog = await loadCatalogTools(options.snapshot); + const metadataSource: IMetadataSource = options.metadata + ? createMockMetadataSource(options.metadata) + : await createDesignSystemMetadataSource(); + const resolveProviderId = + (await createDesignSystemProviderIdResolver()) ?? undefined; + + const files = (await readdir(options.reference)) + .filter((file) => file.endsWith(".json") && file !== "index.json") + .sort(); + + let matched = 0; + const problems: string[] = []; + + for (const file of files) { + const reference = await readToolkitFile(options.reference, file); + const toolkitId = reference.id; + const catalogTools = catalog.get(toolkitId) ?? []; + const enrichment = await readToolkitFile(options.enrichment, file); + const curationToolkit = await readToolkitFile(options.curation, file); + const metadata = await metadataSource.getToolkitMetadata(toolkitId); + + const result = await verifyOneToolkit({ + reference, + catalogTools, + enrichment, + curation: curationFromToolkit(curationToolkit), + metadata, + resolveProviderId, + }); + + if (result.missing.length > 0) { + problems.push( + `✗ ${file}: ${result.missing.length} reference tool(s) missing from catalog: ` + + `${result.missing.slice(0, 5).join(", ")}` + ); + } else if (result.diff) { + problems.push( + `✗ ${file}: ${result.diff.path} — ${result.diff.reason}\n` + + ` expected: ${truncate(result.diff.expected)}\n` + + ` actual: ${truncate(result.diff.actual)}` + ); + } else { + matched++; + } + } + + console.log( + `\n${matched}/${files.length} toolkits reproduce the committed output.` + ); + if (problems.length > 0) { + console.log(`\n${problems.length} problem(s):\n`); + for (const problem of problems) { + console.log(problem); + } + process.exit(1); + } + console.log("Parity verified: 0 differences."); +} + +export const isInvokedDirectly = ( + moduleUrl: string, + argv1: string | undefined +): boolean => + argv1 !== undefined && fileURLToPath(moduleUrl) === resolve(argv1); + +const invokedDirectly = isInvokedDirectly(import.meta.url, process.argv[1]); + +if (invokedDirectly) { + main().catch((error) => { + console.error("Verification failed:", error); + process.exit(1); + }); +} diff --git a/toolkit-docs-generator/tests/scripts/verify-toolkit-join.test.ts b/toolkit-docs-generator/tests/scripts/verify-toolkit-join.test.ts new file mode 100644 index 000000000..0ccc5189f --- /dev/null +++ b/toolkit-docs-generator/tests/scripts/verify-toolkit-join.test.ts @@ -0,0 +1,258 @@ +/** + * Tests for the join parity harness (scripts/verify-toolkit-join.ts). + * + * The harness's promise is: joining a catalog against the enrichment and + * curation layers reproduces the committed toolkit JSON. These tests prove + * that offline, without a live catalog, by treating the committed data as its + * own consistent catalog — every committed tool is projected back to the + * pre-merge `ToolDefinition` shape and re-joined. If the merge round-trips, + * the harness reports zero differences; the failure-mode tests confirm it also + * catches injected drift and missing catalog items. + */ +import { readdirSync, readFileSync } from "fs"; +import { join, resolve } from "path"; +import { pathToFileURL } from "url"; +import { describe, expect, it } from "vitest"; +import { + curationFromToolkit, + firstDifference, + isInvokedDirectly, + joinToolkit, + missingCatalogTools, + verifyOneToolkit, +} from "../../scripts/verify-toolkit-join.js"; +import { parseToolMetadataResponse } from "../../src/sources/tool-metadata-schema.js"; +import type { + MergedToolkit, + ToolDefinition, + ToolkitMetadata, +} from "../../src/types/index.js"; + +const TOOLKITS_DIR = join(__dirname, "../../data/toolkits"); +const FIXTURES_DIR = join(__dirname, "../fixtures"); + +const listToolkitFiles = (): string[] => + readdirSync(TOOLKITS_DIR) + .filter((file) => file.endsWith(".json") && file !== "index.json") + .sort(); + +const loadToolkit = (file: string): MergedToolkit => + JSON.parse(readFileSync(join(TOOLKITS_DIR, file), "utf-8")) as MergedToolkit; + +/** Reconstruct the design-system metadata that produced a committed toolkit. */ +const metadataFromToolkit = (toolkit: MergedToolkit): ToolkitMetadata => + ({ + id: toolkit.id, + label: toolkit.label, + ...toolkit.metadata, + }) as ToolkitMetadata; + +/** Project a committed tool back to its pre-merge catalog shape. */ +const catalogToolsFromToolkit = (toolkit: MergedToolkit): ToolDefinition[] => + (toolkit.tools ?? []).map((tool) => ({ + name: tool.name, + qualifiedName: tool.qualifiedName, + fullyQualifiedName: tool.fullyQualifiedName, + description: tool.description, + toolkitDescription: toolkit.description, + parameters: tool.parameters, + auth: tool.auth, + secrets: tool.secrets, + output: tool.output, + metadata: tool.metadata ?? null, + })); + +describe("firstDifference", () => { + it("returns null for equal objects", () => { + expect( + firstDifference({ a: 1, b: [1, 2] }, { a: 1, b: [1, 2] }) + ).toBeNull(); + }); + + it("ignores the volatile generatedAt field", () => { + expect( + firstDifference( + { a: 1, generatedAt: "2026-01-01" }, + { a: 1, generatedAt: "2026-08-05" } + ) + ).toBeNull(); + }); + + it("reports the JSON path of a nested value mismatch", () => { + const diff = firstDifference( + { tools: [{ codeExample: { tabLabel: "A" } }] }, + { tools: [{ codeExample: { tabLabel: "B" } }] } + ); + expect(diff?.path).toBe("tools[0].codeExample.tabLabel"); + expect(diff?.reason).toBe("value mismatch"); + }); + + it("reports array length differences", () => { + const diff = firstDifference({ xs: [1, 2] }, { xs: [1] }); + expect(diff?.path).toBe("xs"); + expect(diff?.reason).toContain("array length"); + }); + + it("reports a missing key", () => { + const diff = firstDifference({ a: 1, b: 2 }, { a: 1 }); + expect(diff?.path).toBe("b"); + expect(diff?.reason).toBe("missing key"); + }); +}); + +describe("missingCatalogTools", () => { + const reference = { + id: "Github", + tools: [{ qualifiedName: "Github.A" }, { qualifiedName: "Github.B" }], + } as unknown as MergedToolkit; + + it("returns reference tools absent from the catalog", () => { + const catalog = [{ qualifiedName: "Github.A" }] as ToolDefinition[]; + expect(missingCatalogTools(reference, catalog)).toEqual(["Github.B"]); + }); + + it("returns empty when every reference tool is present", () => { + const catalog = [ + { qualifiedName: "Github.A" }, + { qualifiedName: "Github.B" }, + ] as ToolDefinition[]; + expect(missingCatalogTools(reference, catalog)).toEqual([]); + }); +}); + +describe("curationFromToolkit", () => { + it("keeps per-tool documentation chunks in the curation layer", () => { + const sections = curationFromToolkit({ + tools: [ + { + name: "CreateIssue", + qualifiedName: "Github.CreateIssue", + documentationChunks: [ + { + type: "warning", + location: "description", + position: "after", + content: "Use a narrow repository scope.", + }, + ], + }, + ], + } as MergedToolkit); + + expect(sections.toolChunks).toEqual({ + CreateIssue: [ + { + type: "warning", + location: "description", + position: "after", + content: "Use a narrow repository scope.", + }, + ], + }); + }); +}); + +describe("isInvokedDirectly", () => { + it("recognizes a relative script path", () => { + expect( + isInvokedDirectly( + pathToFileURL( + resolve("toolkit-docs-generator/scripts/verify-toolkit-join.ts") + ).href, + "toolkit-docs-generator/scripts/verify-toolkit-join.ts" + ) + ).toBe(true); + }); +}); + +describe("raw /v1/tool_metadata reshape", () => { + it("turns API items into pre-merge tools (value_schema.val_type -> type)", () => { + const raw = JSON.parse( + readFileSync(join(FIXTURES_DIR, "engine-api-response.json"), "utf-8") + ) as { items: unknown[]; total_count: number }; + const { items } = parseToolMetadataResponse(raw); + + expect(items.length).toBeGreaterThan(0); + const withParams = items.find((tool) => tool.parameters.length > 0); + expect(withParams).toBeDefined(); + for (const parameter of withParams?.parameters ?? []) { + expect(typeof parameter.type).toBe("string"); + expect(parameter).not.toHaveProperty("value_schema"); + } + }); +}); + +describe("join reproduces committed output", () => { + it("reports zero differences across every committed toolkit", async () => { + const files = listToolkitFiles(); + expect(files.length).toBeGreaterThan(0); + + const mismatches: string[] = []; + for (const file of files) { + const reference = loadToolkit(file); + const result = await verifyOneToolkit({ + reference, + catalogTools: catalogToolsFromToolkit(reference), + enrichment: reference, + curation: curationFromToolkit(reference), + metadata: metadataFromToolkit(reference), + }); + if (result.missing.length > 0) { + mismatches.push(`${file}: missing ${result.missing.join(", ")}`); + } else if (result.diff) { + mismatches.push(`${file}: ${result.diff.path} — ${result.diff.reason}`); + } + } + + expect(mismatches).toEqual([]); + }, 120_000); +}); + +describe("join detects drift", () => { + it("flags a one-character change to an enrichment value", async () => { + const reference = loadToolkit("github.json"); + const toolWithExample = (reference.tools ?? []).find( + (tool) => tool.codeExample !== undefined + ); + expect(toolWithExample).toBeDefined(); + + const enrichment = JSON.parse(JSON.stringify(reference)) as MergedToolkit; + const mutated = (enrichment.tools ?? []).find( + (tool) => tool.qualifiedName === toolWithExample?.qualifiedName + ); + if (mutated?.codeExample) { + mutated.codeExample.tabLabel = `${mutated.codeExample.tabLabel ?? ""}X`; + } + + const emitted = await joinToolkit({ + toolkitId: reference.id, + catalogTools: catalogToolsFromToolkit(reference), + enrichment, + curation: curationFromToolkit(reference), + metadata: metadataFromToolkit(reference), + }); + + const diff = firstDifference( + reference as unknown as Record, + emitted as unknown as Record + ); + expect(diff).not.toBeNull(); + expect(diff?.path).toContain("codeExample"); + }); + + it("flags a reference tool missing from the catalog", async () => { + const reference = loadToolkit("github.json"); + const catalogTools = catalogToolsFromToolkit(reference).slice(1); + + const result = await verifyOneToolkit({ + reference, + catalogTools, + enrichment: reference, + curation: curationFromToolkit(reference), + metadata: metadataFromToolkit(reference), + }); + + expect(result.missing.length).toBeGreaterThan(0); + expect(result.diff).toBeNull(); + }); +});