docs(generator): document data provenance and source-of-truth rules

app/_lib/toolkit-data.ts

@@ -121,11 +121,14 @@ type ToolkitDataMap = {
 };
 
 /**
- * The production data directory is immutable for the lifetime of a process,
- * so retain one successful load for it. Explicit fixture/override directories
- * are intentionally not retained here: callers can point them at arbitrary
- * paths, and keeping every path would turn test and dev runs into an
- * unbounded process-global cache.
+ * One process-wide load per data directory in production. Keyed by directory
+ * (not a single flat variable) because tests point `TOOLKIT_DATA_DIR` at
+ * scratch fixtures and must not see another test's cached data.
+ *
+ * Failed loads are removed from the cache so a transient read or deployment
+ * error can recover. Development is deliberately excluded from this cache:
+ * the generator can update JSON while `next dev` is running, and a refresh
+ * should see that new snapshot.
  */
 const loadsByDataDir = new Map<string, Promise<ToolkitDataMap>>();
 const DEFAULT_DATA_DIR = resolveToolkitDataDir();
@@ -184,7 +187,10 @@ const loadAllToolkitDataUncached = async (
  */
 export const loadAllToolkitData = cache(
   async (dataDir: string): Promise<ToolkitDataMap> => {
-    if (dataDir !== DEFAULT_DATA_DIR) {
+    if (
+      process.env.NODE_ENV === "development" ||
+      dataDir !== DEFAULT_DATA_DIR
+    ) {
       return await loadAllToolkitDataUncached(dataDir);
     }
 

tests/toolkit-data-cache.test.ts

@@ -1,7 +1,7 @@
-import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
+import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
 import { tmpdir } from "node:os";
 import { join } from "node:path";
-import { afterAll, describe, expect, test } from "vitest";
+import { afterAll, describe, expect, test, vi } from "vitest";
 import { readToolkitData, readToolkitIndex } from "@/app/_lib/toolkit-data";
 
 /**
@@ -75,11 +75,29 @@ describe("readToolkitData against a clean fixture directory", () => {
     expect(data).toBeNull();
   });
 
-  test("materializes defaults from the shared schema", async () => {
-    const data = await readToolkitData("ValidToolkitOne", { dataDir });
-    expect(data?.documentationChunks).toEqual([]);
-    expect(data?.customImports).toEqual([]);
-    expect(data?.subPages).toEqual([]);
+  test("development reads see regenerated files and materialize defaults", async () => {
+    vi.stubEnv("NODE_ENV", "development");
+
+    try {
+      await readToolkitData("ValidToolkitOne", { dataDir });
+      writeFileSync(
+        join(dataDir, "validtoolkitone.json"),
+        JSON.stringify({
+          ...JSON.parse(
+            readFileSync(join(dataDir, "validtoolkitone.json"), "utf8")
+          ),
+          label: "RegeneratedToolkit",
+        })
+      );
+
+      const data = await readToolkitData("ValidToolkitOne", { dataDir });
+      expect(data?.label).toBe("RegeneratedToolkit");
+      expect(data?.documentationChunks).toEqual([]);
+      expect(data?.customImports).toEqual([]);
+      expect(data?.subPages).toEqual([]);
+    } finally {
+      vi.unstubAllEnvs();
+    }
   });
 });
 

tests/toolkit-generation-site.test.ts

@@ -0,0 +1,81 @@
+import { mkdtemp, readFile, rm } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { afterEach, describe, expect, test } from "vitest";
+import { readToolkitData, readToolkitIndex } from "@/app/_lib/toolkit-data";
+import { listToolkitRoutes } from "@/app/_lib/toolkit-static-params";
+import { createJsonGenerator } from "@/toolkit-docs-generator/src/generator/json-generator";
+import type { MergedToolkit } from "@/toolkit-docs-generator/src/shared/toolkit-schemas";
+
+const temporaryDirectories: string[] = [];
+
+afterEach(async () => {
+  await Promise.all(
+    temporaryDirectories
+      .splice(0)
+      .map((directory) => rm(directory, { recursive: true, force: true }))
+  );
+});
+
+const fixtureToolkit: MergedToolkit = {
+  id: "FixtureApi",
+  label: "Fixture API",
+  version: "1.0.0",
+  description: "A generated fixture toolkit.",
+  metadata: {
+    category: "development",
+    iconUrl: "https://example.com/fixture.svg",
+    isBYOC: false,
+    isPro: false,
+    type: "arcade_starter",
+    docsLink:
+      "https://docs.arcade.dev/en/resources/integrations/development/fixture-api",
+    isComingSoon: false,
+    isHidden: false,
+  },
+  auth: null,
+  tools: [],
+  documentationChunks: [],
+  customImports: [],
+  subPages: [],
+};
+
+describe("generated toolkit output through the docs app", () => {
+  test("generation, index loading, data loading, and route enumeration agree", async () => {
+    const dataDir = await mkdtemp(join(tmpdir(), "toolkit-generation-site-"));
+    temporaryDirectories.push(dataDir);
+
+    const generator = createJsonGenerator({
+      outputDir: dataDir,
+      generateIndex: true,
+    });
+    const result = await generator.generateAll([fixtureToolkit]);
+
+    expect(result.errors).toEqual([]);
+    expect(result.filesWritten).toHaveLength(2);
+
+    const index = await readToolkitIndex({ dataDir });
+    const data = await readToolkitData("fixture-api", { dataDir });
+    const routes = await listToolkitRoutes({
+      dataDir,
+      toolkitsCatalog: [],
+    });
+
+    expect(index?.toolkits).toEqual([
+      expect.objectContaining({
+        id: "FixtureApi",
+        category: "development",
+        toolCount: 0,
+      }),
+    ]);
+    expect(data).toMatchObject(fixtureToolkit);
+    expect(routes).toEqual([
+      { toolkitId: "fixture-api", category: "development" },
+    ]);
+
+    const generatedFile = JSON.parse(
+      await readFile(join(dataDir, "fixtureapi.json"), "utf8")
+    ) as MergedToolkit;
+    expect(generatedFile.id).toBe("FixtureApi");
+  });
+});