diff --git a/app/_lib/toolkit-data.ts b/app/_lib/toolkit-data.ts index 27a9b1c3e..6645d27da 100644 --- a/app/_lib/toolkit-data.ts +++ b/app/_lib/toolkit-data.ts @@ -121,11 +121,14 @@ type ToolkitDataMap = { }; /** - * The production data directory is immutable for the lifetime of a process, - * so retain one successful load for it. Explicit fixture/override directories - * are intentionally not retained here: callers can point them at arbitrary - * paths, and keeping every path would turn test and dev runs into an - * unbounded process-global cache. + * One process-wide load per data directory in production. Keyed by directory + * (not a single flat variable) because tests point `TOOLKIT_DATA_DIR` at + * scratch fixtures and must not see another test's cached data. + * + * Failed loads are removed from the cache so a transient read or deployment + * error can recover. Development is deliberately excluded from this cache: + * the generator can update JSON while `next dev` is running, and a refresh + * should see that new snapshot. */ const loadsByDataDir = new Map>(); const DEFAULT_DATA_DIR = resolveToolkitDataDir(); @@ -184,7 +187,10 @@ const loadAllToolkitDataUncached = async ( */ export const loadAllToolkitData = cache( async (dataDir: string): Promise => { - if (dataDir !== DEFAULT_DATA_DIR) { + if ( + process.env.NODE_ENV === "development" || + dataDir !== DEFAULT_DATA_DIR + ) { return await loadAllToolkitDataUncached(dataDir); } diff --git a/tests/toolkit-data-cache.test.ts b/tests/toolkit-data-cache.test.ts index 3f7f1c561..46a2e802b 100644 --- a/tests/toolkit-data-cache.test.ts +++ b/tests/toolkit-data-cache.test.ts @@ -1,7 +1,7 @@ -import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; -import { afterAll, describe, expect, test } from "vitest"; +import { afterAll, describe, expect, test, vi } from "vitest"; import { readToolkitData, readToolkitIndex } from "@/app/_lib/toolkit-data"; /** @@ -75,11 +75,29 @@ describe("readToolkitData against a clean fixture directory", () => { expect(data).toBeNull(); }); - test("materializes defaults from the shared schema", async () => { - const data = await readToolkitData("ValidToolkitOne", { dataDir }); - expect(data?.documentationChunks).toEqual([]); - expect(data?.customImports).toEqual([]); - expect(data?.subPages).toEqual([]); + test("development reads see regenerated files and materialize defaults", async () => { + vi.stubEnv("NODE_ENV", "development"); + + try { + await readToolkitData("ValidToolkitOne", { dataDir }); + writeFileSync( + join(dataDir, "validtoolkitone.json"), + JSON.stringify({ + ...JSON.parse( + readFileSync(join(dataDir, "validtoolkitone.json"), "utf8") + ), + label: "RegeneratedToolkit", + }) + ); + + const data = await readToolkitData("ValidToolkitOne", { dataDir }); + expect(data?.label).toBe("RegeneratedToolkit"); + expect(data?.documentationChunks).toEqual([]); + expect(data?.customImports).toEqual([]); + expect(data?.subPages).toEqual([]); + } finally { + vi.unstubAllEnvs(); + } }); }); diff --git a/tests/toolkit-generation-site.test.ts b/tests/toolkit-generation-site.test.ts new file mode 100644 index 000000000..440d0f66c --- /dev/null +++ b/tests/toolkit-generation-site.test.ts @@ -0,0 +1,81 @@ +import { mkdtemp, readFile, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, describe, expect, test } from "vitest"; +import { readToolkitData, readToolkitIndex } from "@/app/_lib/toolkit-data"; +import { listToolkitRoutes } from "@/app/_lib/toolkit-static-params"; +import { createJsonGenerator } from "@/toolkit-docs-generator/src/generator/json-generator"; +import type { MergedToolkit } from "@/toolkit-docs-generator/src/shared/toolkit-schemas"; + +const temporaryDirectories: string[] = []; + +afterEach(async () => { + await Promise.all( + temporaryDirectories + .splice(0) + .map((directory) => rm(directory, { recursive: true, force: true })) + ); +}); + +const fixtureToolkit: MergedToolkit = { + id: "FixtureApi", + label: "Fixture API", + version: "1.0.0", + description: "A generated fixture toolkit.", + metadata: { + category: "development", + iconUrl: "https://example.com/fixture.svg", + isBYOC: false, + isPro: false, + type: "arcade_starter", + docsLink: + "https://docs.arcade.dev/en/resources/integrations/development/fixture-api", + isComingSoon: false, + isHidden: false, + }, + auth: null, + tools: [], + documentationChunks: [], + customImports: [], + subPages: [], +}; + +describe("generated toolkit output through the docs app", () => { + test("generation, index loading, data loading, and route enumeration agree", async () => { + const dataDir = await mkdtemp(join(tmpdir(), "toolkit-generation-site-")); + temporaryDirectories.push(dataDir); + + const generator = createJsonGenerator({ + outputDir: dataDir, + generateIndex: true, + }); + const result = await generator.generateAll([fixtureToolkit]); + + expect(result.errors).toEqual([]); + expect(result.filesWritten).toHaveLength(2); + + const index = await readToolkitIndex({ dataDir }); + const data = await readToolkitData("fixture-api", { dataDir }); + const routes = await listToolkitRoutes({ + dataDir, + toolkitsCatalog: [], + }); + + expect(index?.toolkits).toEqual([ + expect.objectContaining({ + id: "FixtureApi", + category: "development", + toolCount: 0, + }), + ]); + expect(data).toMatchObject(fixtureToolkit); + expect(routes).toEqual([ + { toolkitId: "fixture-api", category: "development" }, + ]); + + const generatedFile = JSON.parse( + await readFile(join(dataDir, "fixtureapi.json"), "utf8") + ) as MergedToolkit; + expect(generatedFile.id).toBe("FixtureApi"); + }); +}); diff --git a/toolkit-docs-generator/ARCHITECTURE.md b/toolkit-docs-generator/ARCHITECTURE.md index 86654dd08..18a760238 100644 --- a/toolkit-docs-generator/ARCHITECTURE.md +++ b/toolkit-docs-generator/ARCHITECTURE.md @@ -1,87 +1,191 @@ # Toolkit docs generator architecture -This document explains how the toolkit docs generator assembles JSON output and how that output is rendered in the docs site. +This document explains where toolkit documentation data comes from, how the generator +assembles it, and how the docs site renders it. ## Overview -The generator builds toolkit JSON files from multiple sources and writes them to an output directory. The docs site reads these JSON files and renders them into pages at build time. - -The generator does **not** render HTML. It produces structured JSON and optional markdown snippets that the app renders later. - -## Data flow - -1. Fetch tool definitions from the Engine API or Arcade API. -2. Load toolkit metadata from the design system or mock metadata. -3. Load custom sections from JSON files (optional). -4. Merge all data into `MergedToolkit` objects. -5. Write a JSON file per toolkit and an `index.json` file. -6. Optionally verify output and compute diffs. +The generator builds one JSON file per toolkit from several sources and commits them to +`data/toolkits/`. The docs site reads those committed files and renders them into pages at +build time. + +The generator does **not** render HTML, and it does **not** run during a Vercel build. The +nightly workflow commits JSON and navigation changes through a pull request; Vercel then +runs the root `pnpm build` and compiles the committed files with Next.js. That separation is +deliberate — a docs deploy never depends on the Engine being reachable. + +## Pipeline + +```mermaid +graph TD + Engine[Arcade Engine v1 tool_metadata] --> Merger[DataMerger] + DesignSystem[Design system package] --> Merger + LLM[Anthropic summaries and examples] --> Merger + Merger --> Artifact[Committed toolkit JSON and index] + Artifact -.->|previous output| Merger + Artifact --> Sidebar[Generated navigation meta files] + Artifact --> Build[next build] + Sidebar --> Build + Build --> Pages[Toolkit pages and API route] + Build --> Exports[Markdown export, llms.txt, sitemap] +``` + +![Rendered toolkit documentation pipeline showing the data sources, merge step, committed artifacts, and build outputs](./assets/toolkit-docs-pipeline.png) + +The dotted edge is the most important thing on this diagram: **the output directory is also +an input.** See [Source-of-truth rules](#source-of-truth-rules). + +## Where each field comes from + +Roughly 117 toolkits and 8,000 tools, refreshed nightly at 11:00 UTC and on +`porter_deploy_succeeded`. + +| Field | Source | Changes when | +|---|---|---| +| `tools[].name`, `description`, `parameters`, `output`, `secrets` | Arcade Engine | Engine deploys a toolkit | +| `tools[].auth.providerId`, `providerType`, `scopes` | Arcade Engine | same | +| `tools[].metadata.behavior` (`readOnly`, `destructive`, `idempotent`, `openWorld`) | Arcade Engine | same | +| `version` | Arcade Engine — parsed from `fullyQualifiedName` after the `@` | same | +| `description` (toolkit) | Arcade Engine — `tools[0].toolkitDescription ?? tools[0].description` | same | +| `label` | design system, falling back to the toolkit description, then a humanized id | dependency bump | +| `metadata.category`, `iconUrl`, `type`, `isPro`, `isBYOC`, `isComingSoon`, `isHidden`, `docsLink` | design system | dependency bump | +| **the page URL** | derived from `metadata.category` + the last segment of `docsLink` | dependency bump | +| `summary` | Anthropic, keyed on `buildToolkitSummarySignature` | toolkit signature changes | +| `tools[].codeExample`, `secretsInfo` | Anthropic, keyed on `buildComparableToolSignature` | tool signature changes | +| `documentationChunks`, `customImports`, `subPages` | **hand-authored, stored only in the committed JSON** | a human edits the file | + +`buildComparableToolSignature` deliberately ignores descriptions and normalizes enum and +output representations, so cosmetic upstream churn does not trigger LLM regeneration. + +When the design system has no entry for a toolkit, `getDefaultMetadata` supplies a +placeholder. Every field in it is a guess rather than a fact, so it forces `isHidden: true` +and nothing routes to it, and `--require-complete` — which the nightly workflow passes — +turns the omission into a failure that names the toolkit. + +## Source-of-truth rules + +These are the constraints that are not obvious from reading any single file. + +### The committed JSON is the system of record for hand-authored prose + +`documentationChunks`, `customImports`, and `subPages` have no upstream source. The +generator supports loading them from a file (`--custom-sections`), but the nightly workflow +does not pass that flag, so on every run the custom-sections source is empty and these +fields survive only because the merger carries the previous value forward when the incoming +value is empty (`mergeCustomSectionsArrays` in `src/merger/data-merger.ts`). + +Consequences to respect when changing the generator: + +- **Anything that disables previous-output loading discards prose.** `--force-regenerate` + and `--overwrite-output` both set the previous-output directory to `undefined`, which + removes the only copy of this content from the run. `--force-regenerate` is documented as + refreshing examples and summaries; dropping prose is a side effect, not an intent. +- `src/diff/previous-output.ts` carries these three fields forward **even when the previous + file fails schema validation**, for the same reason. That leniency is what keeps a schema + mismatch from wiping prose, and should not be "cleaned up" without first giving prose + another home. + +### One schema, imported by both halves + +`src/shared/toolkit-schemas.ts` holds the Zod definitions for the output format. The +generator validates against them on write and the docs site validates against them on read, +so there is no second hand-written description of the same data to drift against. The app's +types are `z.infer` of those schemas, and `src/shared/toolkit-primitives.ts` holds the id, +slug, and category helpers both halves share. + +### A design-system bump can move page URLs + +`metadata.category` and `metadata.docsLink` come from `@arcadeai/design-system`, which is a +pinned dependency. A version bump is therefore a routing change: recategorizing a toolkit +changes its canonical URL and needs a redirect. Review metadata diffs before bumping it. + +A category the site does not recognize is a build failure, not a fallback. There is no +`others` catch-all — `normalizeCategory` throws, because every category needs a matching +route directory, and silently absorbing an unknown one produced clickable catalog cards +pointing at pages that could not exist. + +### Toolkits are derived from tools, not listed + +There is no "list toolkits" call. `fetchAllToolkitsData` fetches every tool and groups by +the first segment of `qualifiedName`, so a toolkit exists only if it has at least one tool. +`/v1/tool_metadata_summary` does return an authoritative toolkit list with per-toolkit tool +counts; `fetchToolkitsSummary` implements it but nothing calls it today. + +### Absence and corruption are different failures + +A missing file is normal — an optional toolkit, and the reader returns `null`. A file that +is present but unparseable or schema-invalid is a defect that fails the build, because this +data arrives through an automated pull request where a silent page deletion would ship. +Both halves enforce this: `src/generator/output-verifier.ts` reports read, syntax, and +schema errors distinctly on write, and `app/_lib/toolkit-data.ts` throws with the file path +and the underlying issue on read. ## Core components ### Data sources -- `EngineApiSource` fetches tool metadata from the Engine API. -- `ArcadeApiSource` fetches tool metadata from the Arcade API. -- `DesignSystemMetadataSource` loads toolkit metadata from `@arcadeai/design-system`. -- `CustomSectionsFileSource` loads custom documentation chunks from a JSON file. -- `CombinedToolkitDataSource` merges tools and metadata into one interface. +- `EngineApiSource` — tool definitions from `GET /v1/tool_metadata`, paginated at 1000 per + page, Bearer auth. Selected by `--api-source tool-metadata`; this is what CI uses. +- `ArcadeApiSource` — tool definitions from `GET /v1/tools`. Different response schema, no + server-side toolkit filtering. Selected by `--api-source list-tools`. +- `DesignSystemMetadataSource` — toolkit metadata from `@arcadeai/design-system`. Matching is + by normalized id with a `*Api` → provider-id fallback. +- `CustomSectionsFileSource` — hand-authored chunks from a JSON file, when `--custom-sections` + is supplied. +- `CombinedToolkitDataSource` — joins tools and metadata behind one interface, so the rest of + the pipeline does not know they are separate sources. +- Mock equivalents (`src/sources/mock-*.ts`) back `--api-source mock`, which needs no Engine + or Anthropic credentials and is the right way to exercise the pipeline locally. + +Two toolkit lists control what gets processed, and they are not synonyms: +`skip-toolkits.txt` (`--ignore-file`) skips a toolkit and leaves any previously generated +output in place; `remove-toolkits.txt` (`--exclude-file`) skips it **and** deletes its +output file. ### Merger -`DataMerger` creates `MergedToolkit` objects by combining tools, metadata, and custom sections. - -It also: -- Computes tool signatures for change detection. -- Generates optional tool examples and summaries with LLMs. -- Tracks warnings and failed tools. +`DataMerger` builds `MergedToolkit` objects from tools, metadata, and custom sections. It +also computes tool signatures for change detection, generates examples and summaries through +the LLM, runs the secret-coherence scan, and collects warnings, failed tools, and which +toolkits fell back to placeholder metadata. ### Generator -`JsonGenerator` writes the final output: - -- `.json` for each toolkit -- `index.json` for lookup and metadata - -It can also verify output consistency with `OutputVerifier`. +`JsonGenerator` writes `.json` per toolkit plus `index.json`, using atomic +writes and rejecting unsafe or colliding filenames. `OutputVerifier` re-reads the output +directory and validates it. ### Diffing -`ToolkitDiff` compares new output to previous output. It reports: -- new toolkits -- removed toolkits -- modified toolkits -- version-only changes +`ToolkitDiff` compares new output against previous output and reports new, removed, +modified, and version-only changes. `--skip-unchanged` uses it to regenerate only what +changed, which is what keeps the nightly pull request reviewable. ## Rendering in the docs site -The generator output is consumed by the Next.js app: - -- The app loads JSON from `toolkit-docs-generator/data/toolkits/`. -- `generateStaticParams` enumerates the toolkit routes and disables unknown dynamic parameters. -- Custom documentation chunks are rendered as MDX in the UI. - -If you need HTML output, add a separate build step in the app. The generator intentionally avoids HTML to keep the pipeline deterministic. - -## Vercel build - -The generator does not run during a Vercel build. The generation workflow commits -the JSON files and navigation changes through a pull request. Vercel then runs the -root `pnpm build` command and compiles the committed files with Next.js. - -`app/_lib/toolkit-static-params.ts` enumerates routes from `index.json` and the -per-toolkit files. `app/_lib/toolkit-data.ts` reads the same files for page -rendering and the `/api/toolkit-data/[toolkitId]` route. The root layout no -longer reads request headers — the locale it needs is a hardcoded constant, -since `proxy.ts` redirects every request to an `/en` path — so Vercel can -statically render the toolkit routes at build time from the committed JSON. +- `app/_lib/toolkit-data.ts` reads the committed JSON for page rendering and for the + `/api/toolkit-data/[toolkitId]` route. `loadAllToolkitData` is wrapped in React's `cache()` + so a build reads the directory once and every caller shares one map. +- `app/_lib/toolkit-static-params.ts` enumerates routes from `index.json` and the per-toolkit + files, and disables unknown dynamic parameters. +- Pages live at `/en/resources/integrations//` behind a `[toolkitId]` dynamic + route per category, and prerender to static HTML at build time. +- Initial HTML carries a stripped summary; per-tool detail is fetched from + `/api/toolkit-data/[toolkitId]` on expand, which keeps the largest reference pages under + Googlebot's 2 MB crawl limit. +- `documentationChunks` are rendered as MDX. Two consumers implement the + location/position injection model: `documentation-chunk-renderer.tsx` for the page and + `app/_lib/toolkit-markdown.ts` for the markdown export. Both must stay in agreement. + +If you need HTML output from the generator, add a separate build step in the app. The +generator intentionally avoids HTML to keep the pipeline deterministic. ## Search indexing -Search uses an external Algolia crawler. There is no Pagefind or local search -index build step in this repository. After deployment, the crawler indexes the -rendered site. `app/_components/algolia-search.tsx` queries that index with the -public, read-only values configured through these Vercel environment variables: +Search uses an external Algolia crawler. There is no Pagefind or local search index build +step in this repository. After deployment, the crawler indexes the rendered site. +`app/_components/algolia-search.tsx` queries that index with the public, read-only values +configured through these Vercel environment variables: - `NEXT_PUBLIC_ALGOLIA_APP_ID` - `NEXT_PUBLIC_ALGOLIA_SEARCH_API_KEY` @@ -89,10 +193,16 @@ public, read-only values configured through these Vercel environment variables: ## Key files -- `src/sources/engine-api.ts` — tool metadata from Engine API +- `src/shared/toolkit-schemas.ts` — the output contract, imported by generator and app +- `src/shared/toolkit-primitives.ts` — shared id, slug, and category helpers +- `src/shared/toolkit-data-dir.ts` — data directory resolution (Node-only, kept separate so + it never enters a client component's import graph) +- `src/sources/engine-api.ts` — tool metadata from the Engine - `src/sources/toolkit-data-source.ts` — unified data source -- `src/merger/data-merger.ts` — merge pipeline +- `src/merger/data-merger.ts` — merge pipeline, signatures, carry-forward +- `src/diff/previous-output.ts` — lenient re-parse of committed output - `src/generator/json-generator.ts` — output writer - `src/generator/output-verifier.ts` — output validation - `src/diff/toolkit-diff.ts` — change detection - `src/cli/index.ts` — CLI entry point +- `scripts/sync-toolkit-sidebar.ts` — writes the integrations `_meta.tsx` navigation diff --git a/toolkit-docs-generator/assets/toolkit-docs-pipeline.png b/toolkit-docs-generator/assets/toolkit-docs-pipeline.png new file mode 100644 index 000000000..7411f4af5 Binary files /dev/null and b/toolkit-docs-generator/assets/toolkit-docs-pipeline.png differ diff --git a/toolkit-docs-generator/scripts/check-stale-summaries.ts b/toolkit-docs-generator/scripts/check-stale-summaries.ts index 18b06b830..404bc1fb8 100644 --- a/toolkit-docs-generator/scripts/check-stale-summaries.ts +++ b/toolkit-docs-generator/scripts/check-stale-summaries.ts @@ -39,27 +39,47 @@ const listToolkitFiles = (): string[] => { } }; +const readToolkit = (file: string): ToolkitShape | null => { + try { + return JSON.parse( + readFileSync(join(TOOLKITS_DIR, file), "utf8") + ) as ToolkitShape; + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + console.error(`✗ Cannot parse ${file}: ${message}`); + return null; + } +}; + +const getStaleFinding = ( + file: string, + toolkit: ToolkitShape +): { file: string; id: string; reason: string } | null => { + if (toolkit.summaryStale !== true) { + return null; + } + + return { + file, + id: typeof toolkit.id === "string" ? toolkit.id : file, + reason: + typeof toolkit.summaryStaleReason === "string" + ? toolkit.summaryStaleReason + : "unknown", + }; +}; + const main = (): number => { const stale: Array<{ file: string; id: string; reason: string }> = []; for (const file of listToolkitFiles()) { - let toolkit: ToolkitShape; - try { - const raw = readFileSync(join(TOOLKITS_DIR, file), "utf8"); - toolkit = JSON.parse(raw) as ToolkitShape; - } catch (error) { - const message = error instanceof Error ? error.message : String(error); - console.error(`✗ Cannot parse ${file}: ${message}`); + const toolkit = readToolkit(file); + if (!toolkit) { return 1; } - if (toolkit.summaryStale === true) { - stale.push({ - file, - id: typeof toolkit.id === "string" ? toolkit.id : file, - reason: - typeof toolkit.summaryStaleReason === "string" - ? toolkit.summaryStaleReason - : "unknown", - }); + + const finding = getStaleFinding(file, toolkit); + if (finding) { + stale.push(finding); } } diff --git a/toolkit-docs-generator/src/cli/exclusion-cleanup.ts b/toolkit-docs-generator/src/cli/exclusion-cleanup.ts index 3426f9798..5cfe5b1e3 100644 --- a/toolkit-docs-generator/src/cli/exclusion-cleanup.ts +++ b/toolkit-docs-generator/src/cli/exclusion-cleanup.ts @@ -18,6 +18,38 @@ export interface CleanupExcludedToolkitOutputResult { warnings: string[]; } +const appendReadWarnings = ( + warnings: string[], + result: RebuildIndexResult, + verbose: boolean +): void => { + if (result.readErrors.length > 0) { + warnings.push( + `Index rebuild skipped ${result.readErrors.length} unreadable toolkit file(s).` + ); + if (verbose) { + warnings.push( + ...result.readErrors.map( + (error) => `Index rebuild read error: ${error}` + ) + ); + } + } + + if (result.readWarnings.length > 0) { + warnings.push( + `Index rebuild reported ${result.readWarnings.length} warning(s).` + ); + if (verbose) { + warnings.push( + ...result.readWarnings.map( + (warning) => `Index rebuild warning: ${warning}` + ) + ); + } + } +}; + export const cleanupExcludedToolkitOutput = async ( options: CleanupExcludedToolkitOutputOptions ): Promise => { @@ -38,27 +70,7 @@ export const cleanupExcludedToolkitOutput = async ( try { const rebuildResult = await options.generator.rebuildIndexFromOutput(); - if (rebuildResult.readErrors.length > 0) { - warnings.push( - `Index rebuild skipped ${rebuildResult.readErrors.length} unreadable toolkit file(s).` - ); - if (options.verbose) { - for (const error of rebuildResult.readErrors) { - warnings.push(`Index rebuild read error: ${error}`); - } - } - } - - if (rebuildResult.readWarnings.length > 0) { - warnings.push( - `Index rebuild reported ${rebuildResult.readWarnings.length} warning(s).` - ); - if (options.verbose) { - for (const warning of rebuildResult.readWarnings) { - warnings.push(`Index rebuild warning: ${warning}`); - } - } - } + appendReadWarnings(warnings, rebuildResult, options.verbose); } catch (error) { warnings.push( `Excluded toolkit files were deleted, but index rebuild failed: ${error instanceof Error ? error.message : String(error)}` diff --git a/toolkit-docs-generator/src/diff/previous-output.ts b/toolkit-docs-generator/src/diff/previous-output.ts index 043a7bb90..2b03416f7 100644 --- a/toolkit-docs-generator/src/diff/previous-output.ts +++ b/toolkit-docs-generator/src/diff/previous-output.ts @@ -259,6 +259,29 @@ type FallbackToolkitResult = { droppedParameterCount: number; }; +const getOptionalString = ( + record: Record, + key: string +): string | undefined => + typeof record[key] === "string" ? record[key] : undefined; + +const getFallbackCustomSections = ( + record: Record +): Pick< + MergedToolkit, + "documentationChunks" | "customImports" | "subPages" +> => ({ + documentationChunks: Array.isArray(record.documentationChunks) + ? (record.documentationChunks as MergedToolkit["documentationChunks"]) + : [], + customImports: Array.isArray(record.customImports) + ? (record.customImports as MergedToolkit["customImports"]) + : [], + subPages: Array.isArray(record.subPages) + ? (record.subPages as MergedToolkit["subPages"]) + : [], +}); + const buildFallbackToolkit = ( record: Record, fallbackId: string @@ -274,29 +297,12 @@ const buildFallbackToolkit = ( const metadataResult = MergedToolkitMetadataSchema.safeParse(record.metadata); const authResult = MergedToolkitAuthSchema.safeParse(record.auth); - const summary = - typeof record.summary === "string" ? record.summary : undefined; + const summary = getOptionalString(record, "summary"); const summaryStale = typeof record.summaryStale === "boolean" ? record.summaryStale : undefined; - const summaryStaleReason = - typeof record.summaryStaleReason === "string" - ? record.summaryStaleReason - : undefined; - const generatedAt = - typeof record.generatedAt === "string" ? record.generatedAt : undefined; - - // Carry forward custom sections even when strict schema parse fails. - // These are hand-authored fields — losing them silently on any schema - // mismatch would permanently wipe content on the next write. - const documentationChunks = Array.isArray(record.documentationChunks) - ? (record.documentationChunks as MergedToolkit["documentationChunks"]) - : []; - const customImports = Array.isArray(record.customImports) - ? (record.customImports as MergedToolkit["customImports"]) - : []; - const subPages = Array.isArray(record.subPages) - ? (record.subPages as MergedToolkit["subPages"]) - : []; + const summaryStaleReason = getOptionalString(record, "summaryStaleReason"); + const generatedAt = getOptionalString(record, "generatedAt"); + const customSections = getFallbackCustomSections(record); return { toolkit: { @@ -316,9 +322,7 @@ const buildFallbackToolkit = ( secretsInfo: [], documentationChunks: [], })), - documentationChunks, - customImports, - subPages, + ...customSections, ...(generatedAt ? { generatedAt } : {}), }, droppedToolCount,