From e95db374c55b1bab8863542904ef511c92bf4d98 Mon Sep 17 00:00:00 2001 From: Teal Larson Date: Fri, 31 Jul 2026 09:49:38 -0400 Subject: [PATCH 01/17] chore: remove dead deps, stale docs, and a no-op nightly build step Deletion and documentation accuracy only; no behavior change. - CLAUDE.md documented `pnpm build` as a three-stage pipeline ending in pagefind. Pagefind does not exist in this repo (search is an external Algolia crawler); the build is a single `next build --webpack`. Adds a Vale install note, since `pnpm vale:check` is documented as required but vale is a Go binary with no npm dependency. - .gitignore reserved `public/toolkit-markdown/` for a build step that no longer exists, and both .gitignore and the Makefile referenced `make_toolkit_docs/`, a Python directory that was removed. `make mcp-server-docs` was therefore a broken target. - `data/toolkits/jira.json` was an unreferenced 133 KB copy at the repo root; the live data is under `toolkit-docs-generator/data/toolkits/`. - Drops unused dependencies (zustand, turndown, @mdx-js/react) and redundant direct declarations that are supplied transitively (@theguild/remark-mermaid via nextra, baseline-browser-mapping via next, unist-util-visit-parents, mdast-util-to-string). Moves chalk to devDependencies and consolidates the two colour libraries onto it. - Adds @types/hast so neutralize-emails.tsx can use unist-util-visit instead of a hand-rolled tree walk. - The nightly generator workflow ran `pnpm build` with a working-directory that has no package.json, so pnpm resolved upward and executed the root Next production build. The step that follows runs the CLI through tsx and needs no build. - Renames ignored-toolkits.txt/excluded-toolkits.txt to skip-toolkits.txt/remove-toolkits.txt, which say what they do. Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/generate-toolkit-docs.yml | 8 +- .gitignore | 8 - CLAUDE.md | 11 +- Makefile | 3 - .../toolkit-docs/lib/neutralize-emails.tsx | 53 +- data/toolkits/jira.json | 3710 ----------------- package.json | 14 +- pnpm-lock.yaml | 148 +- scripts/generate-llmstxt.ts | 52 +- toolkit-docs-generator/README.md | 6 +- ...luded-toolkits.txt => remove-toolkits.txt} | 0 ...ignored-toolkits.txt => skip-toolkits.txt} | 0 toolkit-docs-generator/src/cli/index.ts | 8 +- .../scenarios/removed-toolkit-cleanup.test.ts | 2 +- .../workflows/generate-toolkit-docs.test.ts | 12 + 15 files changed, 125 insertions(+), 3910 deletions(-) delete mode 100644 data/toolkits/jira.json rename toolkit-docs-generator/{excluded-toolkits.txt => remove-toolkits.txt} (100%) rename toolkit-docs-generator/{ignored-toolkits.txt => skip-toolkits.txt} (100%) diff --git a/.github/workflows/generate-toolkit-docs.yml b/.github/workflows/generate-toolkit-docs.yml index b342c21df..605b29fbd 100644 --- a/.github/workflows/generate-toolkit-docs.yml +++ b/.github/workflows/generate-toolkit-docs.yml @@ -48,10 +48,6 @@ jobs: - name: Install dependencies run: pnpm install --frozen-lockfile - - name: Build toolkit docs generator - run: pnpm build - working-directory: toolkit-docs-generator - - name: Generate toolkit docs run: | pnpm dlx tsx src/cli/index.ts generate \ @@ -71,8 +67,8 @@ jobs: --llm-editor-api-key "$ANTHROPIC_API_KEY" \ --toolkit-concurrency 8 \ --llm-concurrency 15 \ - --exclude-file ./excluded-toolkits.txt \ - --ignore-file ./ignored-toolkits.txt \ + --exclude-file ./remove-toolkits.txt \ + --ignore-file ./skip-toolkits.txt \ --output data/toolkits working-directory: toolkit-docs-generator env: diff --git a/.gitignore b/.gitignore index d869b805e..596a0104d 100644 --- a/.gitignore +++ b/.gitignore @@ -8,11 +8,6 @@ public/sitemap*.xml # TypeScript *.tsbuildinfo -# Toolkit docs -make_toolkit_docs/.venv/ -make_toolkit_docs/.env -make_toolkit_docs/__pycache__/ -make_toolkit_docs/uv.lock *.bak # Vale synced packages (re-sync with `vale sync`) @@ -24,9 +19,6 @@ styles/write-good/ toolkit-docs-generator/overview-input/ toolkit-docs-generator-verification/logs/ -# Generated toolkit markdown (built at build time, not committed) -public/toolkit-markdown/ - # Git worktrees .worktrees/ .cursor/* diff --git a/CLAUDE.md b/CLAUDE.md index d0d643b17..0bdfbf7b9 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -6,7 +6,7 @@ Arcade documentation site built with Next.js + Nextra (App Router), using pnpm a ```bash pnpm dev # Local dev server (port 3000) -pnpm build # Full production build (toolkit-markdown → next build → pagefind) +pnpm build # Production build (next build --webpack) pnpm lint # Lint with Ultracite (Biome-based) pnpm format # Auto-format with Ultracite pnpm test # Run all Vitest tests @@ -27,7 +27,7 @@ pnpm vitest run tests/broken-link-check.test.ts - **`app/_lib/`** — Data-fetching utilities (toolkit catalog, slug generation, static params). - **`app/api/`** — API routes (markdown export, toolkit-data, glossary). - **`toolkit-docs-generator/`** — Generates MCP toolkit documentation from server metadata JSON files in `toolkit-docs-generator/data/toolkits/`. -- **`scripts/`** — Build/CI scripts (Vale style fixes, redirect checking, pagefind indexing, i18n sync). +- **`scripts/`** — Build/CI scripts (Vale style fixes, redirect checking, llms.txt generation, Algolia crawler config, i18n sync). - **`tests/`** — Vitest tests (broken links, internal link validation, sitemap, smoke tests). - **`lib/`** — Next.js utilities (glossary remark plugin, llmstxt plugin). - **`next.config.ts`** — Contains ~138 redirect rules. @@ -42,6 +42,13 @@ Follow **STYLEGUIDE.md** for writing standards and **AUTHORING.md** for formatti - Code snippets: 4 spaces for Python, 2 spaces for other languages. - Run `pnpm vale:check` before submitting docs changes. +Vale is a Go binary with no npm dependency, so a clean checkout does not have it. Install it once, then fetch the style packages (`styles/Google/`, `styles/alex/`, `styles/write-good/` are gitignored): + +```bash +brew install vale # or see https://vale.sh/docs/install +pnpm vale:sync +``` + ## Pre-commit Hooks Husky runs on commit: Vale style checks on `.md/.mdx`, `_meta.tsx` key validation, redirect checking for deleted/renamed pages, internal link updates, and Ultracite formatting. You MUST fix any issues surfaced by the pre-commit hooks. NEVER bypass hooks with `--no-verify` or similar flags. diff --git a/Makefile b/Makefile index e1bb58c99..ee83ba55a 100644 --- a/Makefile +++ b/Makefile @@ -21,9 +21,6 @@ test: ## Run the tests run: ## Run the docs site locally @pnpm dev -mcp-server-docs: ## Generate documentation for an MCP Server toolkit - @cd make_toolkit_docs && uv sync && uv run python __main__.py - ruin: @echo "\033[31m\033[1m💀☠️💀☠️💀☠️💀☠️💀☠️💀☠️💀☠️💀☠️💀☠️💀☠️💀☠️💀☠️💀☠️💀☠️💀☠️\033[0m" @echo "\033[31m\033[1m👻 👻\033[0m" diff --git a/app/_components/toolkit-docs/lib/neutralize-emails.tsx b/app/_components/toolkit-docs/lib/neutralize-emails.tsx index 88b545d66..ea695d66f 100644 --- a/app/_components/toolkit-docs/lib/neutralize-emails.tsx +++ b/app/_components/toolkit-docs/lib/neutralize-emails.tsx @@ -1,4 +1,6 @@ +import type { Element, Root, Text } from "hast"; import { Fragment, type ReactNode } from "react"; +import { visit } from "unist-util-visit"; /** * Matches the email-like text runs that Cloudflare's Email Obfuscation (Scrape @@ -46,22 +48,10 @@ export function splitEmails(text: string): ReactNode { return nodes; } -/** Structural view over hast nodes — avoids depending on `unist-util-visit`. */ -type WalkNode = { - type: string; - value?: string; - tagName?: string; - properties?: Record; - children?: WalkNode[]; -}; - -function neutralizeTextValue(value: string): WalkNode[] { +/** Splits `value` into text/`` element pairs at each email `@` break. */ +function neutralizeTextValue(value: string): Array { const breaks = atBreakOffsets(value); - if (breaks.length === 0) { - return [{ type: "text", value }]; - } - - const out: WalkNode[] = []; + const out: Array = []; let cursor = 0; for (const offset of breaks) { out.push({ type: "text", value: value.slice(cursor, offset) }); @@ -72,30 +62,23 @@ function neutralizeTextValue(value: string): WalkNode[] { return out; } -function walk(node: WalkNode): void { - if (!node.children) { - return; - } - const next: WalkNode[] = []; - for (const child of node.children) { - if (child.type === "text" && typeof child.value === "string") { - next.push(...neutralizeTextValue(child.value)); - } else { - walk(child); - next.push(child); - } - } - node.children = next; -} - /** * rehype plugin (for react-markdown) that applies the same `` break to * email-like text inside rendered markdown — e.g. a toolkit `summary` that * contains a `mongodb+srv://user:pass@host.tld` connection string. - * - * Typed structurally against the hast tree (a `WalkNode`) to avoid a direct - * dependency on `@types/hast`, which pnpm only exposes transitively. */ export function rehypeNeutralizeEmails() { - return (tree: WalkNode): void => walk(tree); + return (tree: Root): void => { + visit(tree, "text", (node, index, parent) => { + if (index === undefined || !parent) { + return; + } + const replacement = neutralizeTextValue(node.value); + if (replacement.length <= 1) { + return; + } + parent.children.splice(index, 1, ...replacement); + return index + replacement.length; + }); + }; } diff --git a/data/toolkits/jira.json b/data/toolkits/jira.json deleted file mode 100644 index b881a4bef..000000000 --- a/data/toolkits/jira.json +++ /dev/null @@ -1,3710 +0,0 @@ -{ - "id": "Jira", - "label": "Jira", - "version": "3.0.2", - "description": "Arcade.dev LLM tools for interacting with Atlassian Jira", - "summary": "The Jira MCP Server provides a comprehensive set of tools for interacting with Jira, enabling users and AI applications to efficiently manage issues and projects. With this MCP Server, you can:\n\n- Create, update, and search for Jira issues using various parameters.\n- Retrieve detailed information about issues, projects, users, and issue types.\n- Manage issue labels and attachments, including adding and removing them.\n- Transition issues between different statuses and manage comments on issues.\n- Browse and list available projects, priorities, and users within Jira.\n- Browse and list information of available boards and sprints within a Jira cloud.\n\nThis MCP Server streamlines the process of issue management, making it easier to integrate Jira functionalities into applications and workflows.", - "metadata": { - "category": "productivity", - "iconUrl": "https://design-system.arcade.dev/icons/jira.svg", - "isBYOC": false, - "isPro": false, - "type": "auth", - "docsLink": "https://docs.arcade.dev/en/mcp-servers/productivity/jira", - "isComingSoon": false, - "isHidden": false - }, - "auth": { - "type": "oauth2", - "providerId": "atlassian", - "allScopes": [ - "manage:jira-configuration", - "read:board-scope.admin:jira-software", - "read:board-scope:jira-software", - "read:issue-details:jira", - "read:jira-user", - "read:jira-work", - "read:jql:jira", - "read:project:jira", - "read:sprint:jira-software", - "write:board-scope:jira-software", - "write:jira-work", - "write:sprint:jira-software" - ] - }, - "tools": [ - { - "name": "AddCommentToIssue", - "qualifiedName": "Jira.AddCommentToIssue", - "fullyQualifiedName": "Jira.AddCommentToIssue@3.0.2", - "description": "Add a comment to a Jira issue.", - "parameters": [ - { - "name": "issue", - "type": "string", - "required": true, - "description": "The ID or key of the issue to comment on.", - "enum": null, - "inferrable": true - }, - { - "name": "body", - "type": "string", - "required": true, - "description": "The body of the comment to add to the issue.", - "enum": null, - "inferrable": true - }, - { - "name": "reply_to_comment", - "type": "string", - "required": false, - "description": "Quote a previous comment as a reply to it. Provide the comment's ID. Must be a comment from the same issue. Defaults to None (no quoted comment).", - "enum": null, - "inferrable": true - }, - { - "name": "mention_users", - "type": "array", - "innerType": "string", - "required": false, - "description": "The users to mention in the comment. Provide the user display name, email address, or ID. Ex: 'John Doe' or 'john.doe@example.com'. Defaults to None (no user mentions).", - "enum": null, - "inferrable": true - }, - { - "name": "atlassian_cloud_id", - "type": "string", - "required": false, - "description": "The ID of the Atlassian Cloud to use (defaults to None). If not provided and the user has a single cloud authorized, the tool will use that. Otherwise, an error will be raised.", - "enum": null, - "inferrable": true - } - ], - "auth": { - "providerId": "atlassian", - "providerType": "oauth2", - "scopes": ["write:jira-work", "read:jira-work", "read:jira-user"] - }, - "secrets": [], - "secretsInfo": [], - "output": { - "type": "json", - "description": "Information about the comment created" - }, - "documentationChunks": [], - "codeExample": { - "toolName": "Jira.AddCommentToIssue", - "parameters": { - "issue": { - "value": "PROJ-123", - "type": "string", - "required": true - }, - "body": { - "value": "This is a comment added via the API.", - "type": "string", - "required": true - }, - "reply_to_comment": { - "value": "456", - "type": "string", - "required": false - }, - "mention_users": { - "value": ["john.doe@example.com", "Jane Smith"], - "type": "array", - "required": false - }, - "atlassian_cloud_id": { - "value": "cloud-789", - "type": "string", - "required": false - } - }, - "requiresAuth": true, - "authProvider": "atlassian", - "tabLabel": "Call the Tool with User Authorization" - } - }, - { - "name": "AddIssuesToSprint", - "qualifiedName": "Jira.AddIssuesToSprint", - "fullyQualifiedName": "Jira.AddIssuesToSprint@3.0.2", - "description": "Add a list of issues to a sprint.\nMaximum of 50 issues per operation.", - "parameters": [ - { - "name": "sprint_id", - "type": "string", - "required": true, - "description": "The numeric Jira sprint ID that identifies the sprint in Jira's API.", - "enum": null, - "inferrable": true - }, - { - "name": "issue_ids", - "type": "array", - "innerType": "string", - "required": true, - "description": "List of issue IDs or keys to add to the sprint. Must not be empty and cannot exceed 50 issues.", - "enum": null, - "inferrable": true - }, - { - "name": "atlassian_cloud_id", - "type": "string", - "required": false, - "description": "The ID of the Atlassian Cloud to use (defaults to None). If not provided and the user has a single cloud authorized, the tool will use that. Otherwise, an error will be raised.", - "enum": null, - "inferrable": true - } - ], - "auth": { - "providerId": "atlassian", - "providerType": "oauth2", - "scopes": [ - "write:sprint:jira-software", - "read:sprint:jira-software", - "read:board-scope:jira-software", - "read:issue-details:jira", - "read:jira-work" - ] - }, - "secrets": [], - "secretsInfo": [], - "output": { - "type": "json", - "description": "A dictionary containing the results of adding issues to the sprint. Includes lists of successfully added issues, issues that were already in the sprint, issues not found, and issues that cannot be moved due to board restrictions. " - }, - "documentationChunks": [], - "codeExample": { - "toolName": "Jira.AddIssuesToSprint", - "parameters": { - "sprint_id": { - "value": "123", - "type": "string", - "required": true - }, - "issue_ids": { - "value": ["ISSUE-1", "ISSUE-2", "ISSUE-3", "ISSUE-4", "ISSUE-5"], - "type": "array", - "required": true - }, - "atlassian_cloud_id": { - "value": "cloud-abc123", - "type": "string", - "required": false - } - }, - "requiresAuth": true, - "authProvider": "atlassian", - "tabLabel": "Call the Tool with User Authorization" - } - }, - { - "name": "AddLabelsToIssue", - "qualifiedName": "Jira.AddLabelsToIssue", - "fullyQualifiedName": "Jira.AddLabelsToIssue@3.0.2", - "description": "Add labels to an existing Jira issue.", - "parameters": [ - { - "name": "issue", - "type": "string", - "required": true, - "description": "The ID or key of the issue to update", - "enum": null, - "inferrable": true - }, - { - "name": "labels", - "type": "array", - "innerType": "string", - "required": true, - "description": "The labels to add to the issue. A label cannot contain spaces. If a label is provided with spaces, they will be trimmed and replaced by underscores.", - "enum": null, - "inferrable": true - }, - { - "name": "notify_watchers", - "type": "boolean", - "required": false, - "description": "Whether to notify the issue's watchers. Defaults to True (notifies watchers).", - "enum": null, - "inferrable": true - }, - { - "name": "atlassian_cloud_id", - "type": "string", - "required": false, - "description": "The ID of the Atlassian Cloud to use (defaults to None). If not provided and the user has a single cloud authorized, the tool will use that. Otherwise, an error will be raised.", - "enum": null, - "inferrable": true - } - ], - "auth": { - "providerId": "atlassian", - "providerType": "oauth2", - "scopes": [ - "read:jira-work", - "write:jira-work", - "read:jira-user", - "manage:jira-configuration" - ] - }, - "secrets": [], - "secretsInfo": [], - "output": { - "type": "json", - "description": "The updated issue" - }, - "documentationChunks": [], - "codeExample": { - "toolName": "Jira.AddLabelsToIssue", - "parameters": { - "issue": { - "value": "JIRA-1234", - "type": "string", - "required": true - }, - "labels": { - "value": ["bug", "urgent", "frontend"], - "type": "array", - "required": true - }, - "notify_watchers": { - "value": true, - "type": "boolean", - "required": false - }, - "atlassian_cloud_id": { - "value": "cloud-5678", - "type": "string", - "required": false - } - }, - "requiresAuth": true, - "authProvider": "atlassian", - "tabLabel": "Call the Tool with User Authorization" - } - }, - { - "name": "AttachFileToIssue", - "qualifiedName": "Jira.AttachFileToIssue", - "fullyQualifiedName": "Jira.AttachFileToIssue@3.0.2", - "description": "Add an attachment to an issue.\n\nMust provide exactly one of file_content_str or file_content_base64.", - "parameters": [ - { - "name": "issue", - "type": "string", - "required": true, - "description": "The issue ID or key to add the attachment to", - "enum": null, - "inferrable": true - }, - { - "name": "filename", - "type": "string", - "required": true, - "description": "The name of the file to add as an attachment. The filename should contain the file extension (e.g. 'test.txt', 'report.pdf'), but it is not mandatory.", - "enum": null, - "inferrable": true - }, - { - "name": "file_content_str", - "type": "string", - "required": false, - "description": "The string content of the file to attach. Use this if the file is a text file. Defaults to None.", - "enum": null, - "inferrable": true - }, - { - "name": "file_content_base64", - "type": "string", - "required": false, - "description": "The base64-encoded binary contents of the file. Use this for binary files like images or PDFs. Defaults to None.", - "enum": null, - "inferrable": true - }, - { - "name": "file_encoding", - "type": "string", - "required": false, - "description": "The encoding of the file to attach. Only used with file_content_str. Defaults to 'utf-8'.", - "enum": null, - "inferrable": true - }, - { - "name": "file_type", - "type": "string", - "required": false, - "description": "The type of the file to attach. E.g. 'application/pdf', 'text', 'image/png'. If not provided, the tool will try to infer the type from the filename. If the filename is not recognized, it will attach the file without specifying a type. Defaults to None (infer from filename or attach without type).", - "enum": null, - "inferrable": true - }, - { - "name": "atlassian_cloud_id", - "type": "string", - "required": false, - "description": "The ID of the Atlassian Cloud to use (defaults to None). If not provided and the user has a single cloud authorized, the tool will use that. Otherwise, an error will be raised.", - "enum": null, - "inferrable": true - } - ], - "auth": { - "providerId": "atlassian", - "providerType": "oauth2", - "scopes": ["write:jira-work", "read:jira-user"] - }, - "secrets": [], - "secretsInfo": [], - "output": { - "type": "json", - "description": "Metadata about the attachment" - }, - "documentationChunks": [], - "codeExample": { - "toolName": "Jira.AttachFileToIssue", - "parameters": { - "issue": { - "value": "PROJ-123", - "type": "string", - "required": true - }, - "filename": { - "value": "report.pdf", - "type": "string", - "required": true - }, - "file_content_str": { - "value": null, - "type": "string", - "required": false - }, - "file_content_base64": { - "value": "JVBERi0xLjQKJcfs...", - "type": "string", - "required": false - }, - "file_encoding": { - "value": "utf-8", - "type": "string", - "required": false - }, - "file_type": { - "value": "application/pdf", - "type": "string", - "required": false - }, - "atlassian_cloud_id": { - "value": "cloud_id_456", - "type": "string", - "required": false - } - }, - "requiresAuth": true, - "authProvider": "atlassian", - "tabLabel": "Call the Tool with User Authorization" - } - }, - { - "name": "CreateIssue", - "qualifiedName": "Jira.CreateIssue", - "fullyQualifiedName": "Jira.CreateIssue@3.0.2", - "description": "Create a new Jira issue.\n\nProvide a value to one of `project` or `parent_issue` arguments. If `project` and\n`parent_issue` are not provided, the tool will select the single project available.\nIf the user has multiple, an error will be returned with the available projects to choose from.\n\nIF YOU DO NOT FOLLOW THE INSTRUCTIONS BELOW AND UNNECESSARILY CALL MULTIPLE TOOLS IN ORDER TO\nCREATE AN ISSUE, TOO MUCH CO2 WILL BE RELEASED IN THE ATMOSPHERE AND YOU WILL CAUSE THE\nDESTRUCTION OF PLANET EARTH BY CATASTROPHIC CLIMATE CHANGE.\n\nIf you have an issue type name, or a project key/name, a priority name, an assignee\nname/key/email, or a reporter name/key/email, DO NOT CALL OTHER TOOLS only to list available\nprojects, priorities, issue types, or users. Provide the name, key, or email and the tool\nwill figure out the ID, WITHOUT CAUSING CATASTROPHIC CLIMATE CHANGE.", - "parameters": [ - { - "name": "title", - "type": "string", - "required": true, - "description": "The title of the issue.", - "enum": null, - "inferrable": true - }, - { - "name": "issue_type", - "type": "string", - "required": true, - "description": "The name or ID of the issue type. If a name is provided, the tool will try to find a unique exact match among the available issue types.", - "enum": null, - "inferrable": true - }, - { - "name": "project", - "type": "string", - "required": false, - "description": "The ID, key or name of the project to associate the issue with. If a name is provided, the tool will try to find a unique exact match among the available projects. Defaults to None (no project). If `project` and `parent_issue` are not provided, the tool will select the single project available. If the user has multiple, an error will be returned with the available projects to choose from.", - "enum": null, - "inferrable": true - }, - { - "name": "due_date", - "type": "string", - "required": false, - "description": "The due date of the issue. Format: YYYY-MM-DD. Ex: '2025-01-01'. Defaults to None (no due date).", - "enum": null, - "inferrable": true - }, - { - "name": "description", - "type": "string", - "required": false, - "description": "The description of the issue. Defaults to None (no description).", - "enum": null, - "inferrable": true - }, - { - "name": "environment", - "type": "string", - "required": false, - "description": "The environment of the issue. Defaults to None (no environment).", - "enum": null, - "inferrable": true - }, - { - "name": "labels", - "type": "array", - "innerType": "string", - "required": false, - "description": "The labels of the issue. Defaults to None (no labels). A label cannot contain spaces. If a label is provided with spaces, they will be trimmed and replaced by underscores.", - "enum": null, - "inferrable": true - }, - { - "name": "parent_issue", - "type": "string", - "required": false, - "description": "The ID or key of the parent issue. Defaults to None (no parent issue). Must provide at least one of `parent_issue` or `project` arguments.", - "enum": null, - "inferrable": true - }, - { - "name": "priority", - "type": "string", - "required": false, - "description": "The ID or name of the priority to use for the issue. If a name is provided, the tool will try to find a unique exact match among the available priorities. Defaults to None (the issue is created with Jira's default priority for the specified project).", - "enum": null, - "inferrable": true - }, - { - "name": "assignee", - "type": "string", - "required": false, - "description": "The name, email or ID of the user to assign the issue to. If a name or email is provided, the tool will try to find a unique exact match among the available users. Defaults to None (no assignee).", - "enum": null, - "inferrable": true - }, - { - "name": "reporter", - "type": "string", - "required": false, - "description": "The name, email or ID of the user who is the reporter of the issue. If a name or email is provided, the tool will try to find a unique exact match among the available users. Defaults to None (no reporter).", - "enum": null, - "inferrable": true - }, - { - "name": "atlassian_cloud_id", - "type": "string", - "required": false, - "description": "The ID of the Atlassian Cloud to use (defaults to None). If not provided and the user has a single cloud authorized, the tool will use that. Otherwise, an error will be raised.", - "enum": null, - "inferrable": true - } - ], - "auth": { - "providerId": "atlassian", - "providerType": "oauth2", - "scopes": [ - "read:jira-work", - "write:jira-work", - "read:jira-user", - "manage:jira-configuration" - ] - }, - "secrets": [], - "secretsInfo": [], - "output": { - "type": "json", - "description": "The created issue" - }, - "documentationChunks": [], - "codeExample": { - "toolName": "Jira.CreateIssue", - "parameters": { - "title": { - "value": "Fix login issue", - "type": "string", - "required": true - }, - "issue_type": { - "value": "Bug", - "type": "string", - "required": true - }, - "project": { - "value": "WEB", - "type": "string", - "required": false - }, - "due_date": { - "value": "2023-12-01", - "type": "string", - "required": false - }, - "description": { - "value": "Users are unable to log in using their credentials.", - "type": "string", - "required": false - }, - "environment": { - "value": null, - "type": "string", - "required": false - }, - "labels": { - "value": ["login_issue", "urgent"], - "type": "array", - "required": false - }, - "parent_issue": { - "value": null, - "type": "string", - "required": false - }, - "priority": { - "value": "High", - "type": "string", - "required": false - }, - "assignee": { - "value": "john.doe@example.com", - "type": "string", - "required": false - }, - "reporter": { - "value": "jane.smith@example.com", - "type": "string", - "required": false - }, - "atlassian_cloud_id": { - "value": null, - "type": "string", - "required": false - } - }, - "requiresAuth": true, - "authProvider": "atlassian", - "tabLabel": "Call the Tool with User Authorization" - } - }, - { - "name": "DownloadAttachment", - "qualifiedName": "Jira.DownloadAttachment", - "fullyQualifiedName": "Jira.DownloadAttachment@3.0.2", - "description": "Download the contents of an attachment associated with an issue.", - "parameters": [ - { - "name": "attachment_id", - "type": "string", - "required": true, - "description": "The ID of the attachment to download", - "enum": null, - "inferrable": true - }, - { - "name": "atlassian_cloud_id", - "type": "string", - "required": false, - "description": "The ID of the Atlassian Cloud to use (defaults to None). If not provided and the user has a single cloud authorized, the tool will use that. Otherwise, an error will be raised.", - "enum": null, - "inferrable": true - } - ], - "auth": { - "providerId": "atlassian", - "providerType": "oauth2", - "scopes": ["read:jira-work", "read:jira-user"] - }, - "secrets": [], - "secretsInfo": [], - "output": { - "type": "json", - "description": "The content of the attachment" - }, - "documentationChunks": [], - "codeExample": { - "toolName": "Jira.DownloadAttachment", - "parameters": { - "attachment_id": { - "value": "ATT-123456", - "type": "string", - "required": true - }, - "atlassian_cloud_id": { - "value": "cloud-7890", - "type": "string", - "required": false - } - }, - "requiresAuth": true, - "authProvider": "atlassian", - "tabLabel": "Call the Tool with User Authorization" - } - }, - { - "name": "GetAttachmentMetadata", - "qualifiedName": "Jira.GetAttachmentMetadata", - "fullyQualifiedName": "Jira.GetAttachmentMetadata@3.0.2", - "description": "Get the metadata of an attachment.", - "parameters": [ - { - "name": "attachment_id", - "type": "string", - "required": true, - "description": "The ID of the attachment to retrieve", - "enum": null, - "inferrable": true - }, - { - "name": "atlassian_cloud_id", - "type": "string", - "required": false, - "description": "The ID of the Atlassian Cloud to use (defaults to None). If not provided and the user has a single cloud authorized, the tool will use that. Otherwise, an error will be raised.", - "enum": null, - "inferrable": true - } - ], - "auth": { - "providerId": "atlassian", - "providerType": "oauth2", - "scopes": ["read:jira-work", "read:jira-user"] - }, - "secrets": [], - "secretsInfo": [], - "output": { - "type": "json", - "description": "The metadata of the attachment" - }, - "documentationChunks": [], - "codeExample": { - "toolName": "Jira.GetAttachmentMetadata", - "parameters": { - "attachment_id": { - "value": "ATT-123456", - "type": "string", - "required": true - }, - "atlassian_cloud_id": { - "value": "cloud-987654321", - "type": "string", - "required": false - } - }, - "requiresAuth": true, - "authProvider": "atlassian", - "tabLabel": "Call the Tool with User Authorization" - } - }, - { - "name": "GetAvailableAtlassianClouds", - "qualifiedName": "Jira.GetAvailableAtlassianClouds", - "fullyQualifiedName": "Jira.GetAvailableAtlassianClouds@3.0.2", - "description": "Get available Atlassian Clouds.", - "parameters": [], - "auth": { - "providerId": "atlassian", - "providerType": "oauth2", - "scopes": ["read:jira-user"] - }, - "secrets": [], - "secretsInfo": [], - "output": { - "type": "json", - "description": "Available Atlassian Clouds" - }, - "documentationChunks": [], - "codeExample": { - "toolName": "Jira.GetAvailableAtlassianClouds", - "parameters": {}, - "requiresAuth": true, - "authProvider": "atlassian", - "tabLabel": "Call the Tool with User Authorization" - } - }, - { - "name": "GetBoardBacklogIssues", - "qualifiedName": "Jira.GetBoardBacklogIssues", - "fullyQualifiedName": "Jira.GetBoardBacklogIssues@3.0.2", - "description": "Get all issues in a board's backlog with pagination support.\nReturns issues that are not currently assigned to any active sprint.\n\nThe backlog contains issues that are ready to be planned into future sprints.\nOnly boards that support backlogs (like Scrum and Kanban boards) will return results.", - "parameters": [ - { - "name": "board_id", - "type": "string", - "required": true, - "description": "The ID of the board to retrieve backlog issues from. Must be a valid board ID that supports backlogs (typically Scrum or Kanban boards).", - "enum": null, - "inferrable": true - }, - { - "name": "limit", - "type": "integer", - "required": false, - "description": "The maximum number of issues to return. Must be between 1 and 100 inclusive. Controls pagination and determines how many issues are fetched and returned. Defaults to 50 for improved performance.", - "enum": null, - "inferrable": true - }, - { - "name": "offset", - "type": "integer", - "required": false, - "description": "The number of issues to skip before starting to return results. Used for pagination when the backlog has many issues. For example, offset=50 with limit=50 would return issues 51-100. Must be 0 or greater. Defaults to 0.", - "enum": null, - "inferrable": true - }, - { - "name": "atlassian_cloud_id", - "type": "string", - "required": false, - "description": "The ID of the Atlassian Cloud to use (defaults to None). If not provided and the user has a single cloud authorized, the tool will use that. Otherwise, an error will be raised.", - "enum": null, - "inferrable": true - } - ], - "auth": { - "providerId": "atlassian", - "providerType": "oauth2", - "scopes": ["read:board-scope:jira-software", "read:issue-details:jira"] - }, - "secrets": [], - "secretsInfo": [], - "output": { - "type": "json", - "description": "A dictionary containing the board information, list of backlog issues, and pagination metadata. Issues are returned with full details including summary, status, assignee, and other fields. If the board doesn't support backlogs or doesn't exist, appropriate error information is returned." - }, - "documentationChunks": [], - "codeExample": { - "toolName": "Jira.GetBoardBacklogIssues", - "parameters": { - "board_id": { - "value": "12345", - "type": "string", - "required": true - }, - "limit": { - "value": 50, - "type": "integer", - "required": false - }, - "offset": { - "value": 0, - "type": "integer", - "required": false - }, - "atlassian_cloud_id": { - "value": "abc-def-ghi", - "type": "string", - "required": false - } - }, - "requiresAuth": true, - "authProvider": "atlassian", - "tabLabel": "Call the Tool with User Authorization" - } - }, - { - "name": "GetBoards", - "qualifiedName": "Jira.GetBoards", - "fullyQualifiedName": "Jira.GetBoards@3.0.2", - "description": "Retrieve Jira boards either by specifying their names or IDs, or get all\navailable boards.\nAll requests support offset and limit with a maximum of 50 boards returned per call.\n\nMANDATORY ACTION: ALWAYS when you need to get multiple boards, you must\ninclude all the board identifiers in a single call rather than making\nmultiple separate tool calls, as this provides much better performance, not doing that will\nbring huge performance penalties.\n\nThe tool automatically handles mixed identifier types (names and IDs), deduplicates results, and\nfalls back from ID lookup to name lookup when needed.", - "parameters": [ - { - "name": "board_identifiers_list", - "type": "array", - "innerType": "string", - "required": false, - "description": "List of board names or numeric IDs (as strings) to retrieve using pagination. Include all mentioned boards in a single list for best performance. Default None retrieves all boards. Maximum 50 boards returned per call.", - "enum": null, - "inferrable": true - }, - { - "name": "limit", - "type": "integer", - "required": false, - "description": "Maximum number of boards to return (1-50). Defaults to max that is 50.", - "enum": null, - "inferrable": true - }, - { - "name": "offset", - "type": "integer", - "required": false, - "description": "Number of boards to skip for pagination. Must be 0 or greater. Defaults to 0.", - "enum": null, - "inferrable": true - }, - { - "name": "atlassian_cloud_id", - "type": "string", - "required": false, - "description": "Atlassian Cloud ID to use. Defaults to None (uses single authorized cloud).", - "enum": null, - "inferrable": true - } - ], - "auth": { - "providerId": "atlassian", - "providerType": "oauth2", - "scopes": [ - "read:board-scope:jira-software", - "read:project:jira", - "read:issue-details:jira", - "read:jira-user" - ] - }, - "secrets": [], - "secretsInfo": [], - "output": { - "type": "json", - "description": "Dictionary with 'boards' list containing board metadata (ID, name, type, location) and 'errors' array for not found boards. Includes pagination metadata and deduplication." - }, - "documentationChunks": [], - "codeExample": { - "toolName": "Jira.GetBoards", - "parameters": { - "board_identifiers_list": { - "value": ["123", "project-board", "456", "design-board"], - "type": "array", - "required": false - }, - "limit": { - "value": 25, - "type": "integer", - "required": false - }, - "offset": { - "value": 0, - "type": "integer", - "required": false - }, - "atlassian_cloud_id": { - "value": "cloud-12345", - "type": "string", - "required": false - } - }, - "requiresAuth": true, - "authProvider": "atlassian", - "tabLabel": "Call the Tool with User Authorization" - } - }, - { - "name": "GetCommentById", - "qualifiedName": "Jira.GetCommentById", - "fullyQualifiedName": "Jira.GetCommentById@3.0.2", - "description": "Get a comment by its ID.", - "parameters": [ - { - "name": "issue_id", - "type": "string", - "required": true, - "description": "The ID or key of the issue to retrieve the comment from.", - "enum": null, - "inferrable": true - }, - { - "name": "comment_id", - "type": "string", - "required": true, - "description": "The ID of the comment to retrieve", - "enum": null, - "inferrable": true - }, - { - "name": "include_adf_content", - "type": "boolean", - "required": false, - "description": "Whether to include the ADF (Atlassian Document Format) content of the comment in the response. Defaults to False (return only the HTML rendered content).", - "enum": null, - "inferrable": true - }, - { - "name": "atlassian_cloud_id", - "type": "string", - "required": false, - "description": "The ID of the Atlassian Cloud to use (defaults to None). If not provided and the user has a single cloud authorized, the tool will use that. Otherwise, an error will be raised.", - "enum": null, - "inferrable": true - } - ], - "auth": { - "providerId": "atlassian", - "providerType": "oauth2", - "scopes": ["read:jira-work", "read:jira-user"] - }, - "secrets": [], - "secretsInfo": [], - "output": { - "type": "json", - "description": "Information about the comment" - }, - "documentationChunks": [], - "codeExample": { - "toolName": "Jira.GetCommentById", - "parameters": { - "issue_id": { - "value": "PROJ-123", - "type": "string", - "required": true - }, - "comment_id": { - "value": "456", - "type": "string", - "required": true - }, - "include_adf_content": { - "value": true, - "type": "boolean", - "required": false - }, - "atlassian_cloud_id": { - "value": "cloud-789", - "type": "string", - "required": false - } - }, - "requiresAuth": true, - "authProvider": "atlassian", - "tabLabel": "Call the Tool with User Authorization" - } - }, - { - "name": "GetIssueById", - "qualifiedName": "Jira.GetIssueById", - "fullyQualifiedName": "Jira.GetIssueById@3.0.2", - "description": "Get the details of a Jira issue by its ID.", - "parameters": [ - { - "name": "issue", - "type": "string", - "required": true, - "description": "The ID or key of the issue to retrieve", - "enum": null, - "inferrable": true - }, - { - "name": "atlassian_cloud_id", - "type": "string", - "required": false, - "description": "The ID of the Atlassian Cloud to use (defaults to None). If not provided and the user has a single cloud authorized, the tool will use that. Otherwise, an error will be raised.", - "enum": null, - "inferrable": true - } - ], - "auth": { - "providerId": "atlassian", - "providerType": "oauth2", - "scopes": ["read:jira-work", "read:jira-user"] - }, - "secrets": [], - "secretsInfo": [], - "output": { - "type": "json", - "description": "Information about the issue" - }, - "documentationChunks": [], - "codeExample": { - "toolName": "Jira.GetIssueById", - "parameters": { - "issue": { - "value": "JIRA-12345", - "type": "string", - "required": true - }, - "atlassian_cloud_id": { - "value": "cloud-abc123", - "type": "string", - "required": false - } - }, - "requiresAuth": true, - "authProvider": "atlassian", - "tabLabel": "Call the Tool with User Authorization" - } - }, - { - "name": "GetIssueComments", - "qualifiedName": "Jira.GetIssueComments", - "fullyQualifiedName": "Jira.GetIssueComments@3.0.2", - "description": "Get the comments of a Jira issue by its ID.", - "parameters": [ - { - "name": "issue", - "type": "string", - "required": true, - "description": "The ID or key of the issue to retrieve", - "enum": null, - "inferrable": true - }, - { - "name": "limit", - "type": "integer", - "required": false, - "description": "The maximum number of comments to retrieve. Min 1, max 100, default 100.", - "enum": null, - "inferrable": true - }, - { - "name": "offset", - "type": "integer", - "required": false, - "description": "The number of comments to skip. Defaults to 0 (start from the first comment).", - "enum": null, - "inferrable": true - }, - { - "name": "order_by", - "type": "string", - "required": false, - "description": "The order in which to return the comments. Defaults to 'created_date_descending' (most recent first).", - "enum": ["created_date_ascending", "created_date_descending"], - "inferrable": true - }, - { - "name": "include_adf_content", - "type": "boolean", - "required": false, - "description": "Whether to include the ADF (Atlassian Document Format) content of the comment in the response. Defaults to False (return only the HTML rendered content).", - "enum": null, - "inferrable": true - }, - { - "name": "atlassian_cloud_id", - "type": "string", - "required": false, - "description": "The ID of the Atlassian Cloud to use (defaults to None). If not provided and the user has a single cloud authorized, the tool will use that. Otherwise, an error will be raised.", - "enum": null, - "inferrable": true - } - ], - "auth": { - "providerId": "atlassian", - "providerType": "oauth2", - "scopes": ["read:jira-work", "read:jira-user"] - }, - "secrets": [], - "secretsInfo": [], - "output": { - "type": "json", - "description": "Information about the issue comments" - }, - "documentationChunks": [], - "codeExample": { - "toolName": "Jira.GetIssueComments", - "parameters": { - "issue": { - "value": "JIRA-123", - "type": "string", - "required": true - }, - "limit": { - "value": 50, - "type": "integer", - "required": false - }, - "offset": { - "value": 10, - "type": "integer", - "required": false - }, - "order_by": { - "value": "created_date_ascending", - "type": "string", - "required": false - }, - "include_adf_content": { - "value": true, - "type": "boolean", - "required": false - }, - "atlassian_cloud_id": { - "value": "cloud-789", - "type": "string", - "required": false - } - }, - "requiresAuth": true, - "authProvider": "atlassian", - "tabLabel": "Call the Tool with User Authorization" - } - }, - { - "name": "GetIssuesWithoutId", - "qualifiedName": "Jira.GetIssuesWithoutId", - "fullyQualifiedName": "Jira.GetIssuesWithoutId@3.0.2", - "description": "Search for Jira issues when you don't have the issue ID(s).\n\nAll text-based arguments (keywords, assignee, project, labels) are case-insensitive.\n\nALWAYS PREFER THIS TOOL OVER THE `Jira.SearchIssuesWithJql` TOOL, UNLESS IT'S ABSOLUTELY\nNECESSARY TO USE A JQL QUERY TO FILTER IN A WAY THAT IS NOT SUPPORTED BY THIS TOOL.", - "parameters": [ - { - "name": "keywords", - "type": "string", - "required": false, - "description": "Keywords to search for issues. Matches against the issue name, description, comments, and any custom field of type text. Defaults to None (no keywords filtering).", - "enum": null, - "inferrable": true - }, - { - "name": "due_from", - "type": "string", - "required": false, - "description": "Match issues due on or after this date. Format: YYYY-MM-DD. Ex: '2025-01-01'. Defaults to None (no due date filtering).", - "enum": null, - "inferrable": true - }, - { - "name": "due_until", - "type": "string", - "required": false, - "description": "Match issues due on or before this date. Format: YYYY-MM-DD. Ex: '2025-01-01'. Defaults to None (no due date filtering).", - "enum": null, - "inferrable": true - }, - { - "name": "status", - "type": "string", - "required": false, - "description": "Match issues that are in this status. Provide a status name. Ex: 'To Do', 'In Progress', 'Done'. Defaults to None (any status).", - "enum": null, - "inferrable": true - }, - { - "name": "priority", - "type": "string", - "required": false, - "description": "Match issues that have this priority. Provide a priority name. E.g. 'Highest'. Defaults to None (any priority).", - "enum": null, - "inferrable": true - }, - { - "name": "assignee", - "type": "string", - "required": false, - "description": "Match issues that are assigned to this user. Provide the user's name or email address. Ex: 'John Doe' or 'john.doe@example.com'. Defaults to None (any assignee).", - "enum": null, - "inferrable": true - }, - { - "name": "project", - "type": "string", - "required": false, - "description": "Match issues that are associated with this project. Provide the project's name, ID, or key. If a project name is provided, the tool will try to find a unique exact match among the available projects. Defaults to None (search across all projects).", - "enum": null, - "inferrable": true - }, - { - "name": "issue_type", - "type": "string", - "required": false, - "description": "Match issues that are of this issue type. Provide an issue type name or ID. E.g. 'Task', 'Epic', '12345'. If a name is provided, the tool will try to find a unique exact match among the available issue types. Defaults to None (any issue type).", - "enum": null, - "inferrable": true - }, - { - "name": "labels", - "type": "array", - "innerType": "string", - "required": false, - "description": "Match issues that are in these labels. Defaults to None (any label).", - "enum": null, - "inferrable": true - }, - { - "name": "parent_issue", - "type": "string", - "required": false, - "description": "Match issues that are a child of this issue. Provide the issue's ID or key. Defaults to None (no parent issue filtering).", - "enum": null, - "inferrable": true - }, - { - "name": "limit", - "type": "integer", - "required": false, - "description": "The maximum number of issues to retrieve. Min 1, max 100, default 50.", - "enum": null, - "inferrable": true - }, - { - "name": "next_page_token", - "type": "string", - "required": false, - "description": "The token to use to get the next page of issues. Defaults to None (first page).", - "enum": null, - "inferrable": true - }, - { - "name": "atlassian_cloud_id", - "type": "string", - "required": false, - "description": "The ID of the Atlassian Cloud to use (defaults to None). If not provided and the user has a single cloud authorized, the tool will use that. Otherwise, an error will be raised.", - "enum": null, - "inferrable": true - } - ], - "auth": { - "providerId": "atlassian", - "providerType": "oauth2", - "scopes": [ - "read:jira-work", - "read:jira-user", - "manage:jira-configuration" - ] - }, - "secrets": [], - "secretsInfo": [], - "output": { - "type": "json", - "description": "Information about the issues matching the search criteria" - }, - "documentationChunks": [], - "codeExample": { - "toolName": "Jira.GetIssuesWithoutId", - "parameters": { - "keywords": { - "value": "bug fix", - "type": "string", - "required": false - }, - "due_from": { - "value": "2023-11-01", - "type": "string", - "required": false - }, - "due_until": { - "value": "2023-12-31", - "type": "string", - "required": false - }, - "status": { - "value": "In Progress", - "type": "string", - "required": false - }, - "priority": { - "value": "Highest", - "type": "string", - "required": false - }, - "assignee": { - "value": "jane.doe@example.com", - "type": "string", - "required": false - }, - "project": { - "value": "ProjectX", - "type": "string", - "required": false - }, - "issue_type": { - "value": "Task", - "type": "string", - "required": false - }, - "labels": { - "value": ["urgent", "backend"], - "type": "array", - "required": false - }, - "parent_issue": { - "value": "PROJ-123", - "type": "string", - "required": false - }, - "limit": { - "value": 25, - "type": "integer", - "required": false - }, - "next_page_token": { - "value": "abc123", - "type": "string", - "required": false - }, - "atlassian_cloud_id": { - "value": "cloud-456", - "type": "string", - "required": false - } - }, - "requiresAuth": true, - "authProvider": "atlassian", - "tabLabel": "Call the Tool with User Authorization" - } - }, - { - "name": "GetIssueTypeById", - "qualifiedName": "Jira.GetIssueTypeById", - "fullyQualifiedName": "Jira.GetIssueTypeById@3.0.2", - "description": "Get the details of a Jira issue type by its ID.", - "parameters": [ - { - "name": "issue_type_id", - "type": "string", - "required": true, - "description": "The ID of the issue type to retrieve", - "enum": null, - "inferrable": true - }, - { - "name": "atlassian_cloud_id", - "type": "string", - "required": false, - "description": "The ID of the Atlassian Cloud to use (defaults to None). If not provided and the user has a single cloud authorized, the tool will use that. Otherwise, an error will be raised.", - "enum": null, - "inferrable": true - } - ], - "auth": { - "providerId": "atlassian", - "providerType": "oauth2", - "scopes": ["read:jira-work", "read:jira-user"] - }, - "secrets": [], - "secretsInfo": [], - "output": { - "type": "json", - "description": "Information about the issue type" - }, - "documentationChunks": [], - "codeExample": { - "toolName": "Jira.GetIssueTypeById", - "parameters": { - "issue_type_id": { - "value": "10001", - "type": "string", - "required": true - }, - "atlassian_cloud_id": { - "value": "cloud-12345", - "type": "string", - "required": false - } - }, - "requiresAuth": true, - "authProvider": "atlassian", - "tabLabel": "Call the Tool with User Authorization" - } - }, - { - "name": "GetPriorityById", - "qualifiedName": "Jira.GetPriorityById", - "fullyQualifiedName": "Jira.GetPriorityById@3.0.2", - "description": "Get the details of a priority by its ID.", - "parameters": [ - { - "name": "priority_id", - "type": "string", - "required": true, - "description": "The ID of the priority to retrieve.", - "enum": null, - "inferrable": true - }, - { - "name": "atlassian_cloud_id", - "type": "string", - "required": false, - "description": "The ID of the Atlassian Cloud to use (defaults to None). If not provided and the user has a single cloud authorized, the tool will use that. Otherwise, an error will be raised.", - "enum": null, - "inferrable": true - } - ], - "auth": { - "providerId": "atlassian", - "providerType": "oauth2", - "scopes": ["read:jira-work", "read:jira-user"] - }, - "secrets": [], - "secretsInfo": [], - "output": { - "type": "json", - "description": "The priority" - }, - "documentationChunks": [], - "codeExample": { - "toolName": "Jira.GetPriorityById", - "parameters": { - "priority_id": { - "value": "10001", - "type": "string", - "required": true - }, - "atlassian_cloud_id": { - "value": "abcd1234", - "type": "string", - "required": false - } - }, - "requiresAuth": true, - "authProvider": "atlassian", - "tabLabel": "Call the Tool with User Authorization" - } - }, - { - "name": "GetProjectById", - "qualifiedName": "Jira.GetProjectById", - "fullyQualifiedName": "Jira.GetProjectById@3.0.2", - "description": "Get the details of a Jira project by its ID or key.", - "parameters": [ - { - "name": "project", - "type": "string", - "required": true, - "description": "The ID or key of the project to retrieve", - "enum": null, - "inferrable": true - }, - { - "name": "atlassian_cloud_id", - "type": "string", - "required": false, - "description": "The ID of the Atlassian Cloud to use (defaults to None). If not provided and the user has a single cloud authorized, the tool will use that. Otherwise, an error will be raised.", - "enum": null, - "inferrable": true - } - ], - "auth": { - "providerId": "atlassian", - "providerType": "oauth2", - "scopes": ["read:jira-work", "read:jira-user"] - }, - "secrets": [], - "secretsInfo": [], - "output": { - "type": "json", - "description": "Information about the project" - }, - "documentationChunks": [], - "codeExample": { - "toolName": "Jira.GetProjectById", - "parameters": { - "project": { - "value": "PROJ-123", - "type": "string", - "required": true - }, - "atlassian_cloud_id": { - "value": "cloud_456", - "type": "string", - "required": false - } - }, - "requiresAuth": true, - "authProvider": "atlassian", - "tabLabel": "Call the Tool with User Authorization" - } - }, - { - "name": "GetSprintIssues", - "qualifiedName": "Jira.GetSprintIssues", - "fullyQualifiedName": "Jira.GetSprintIssues@3.0.2", - "description": "Get all issues that are currently assigned to a specific sprint with pagination support.\nReturns issues that are planned for or being worked on in the sprint.", - "parameters": [ - { - "name": "sprint_id", - "type": "string", - "required": true, - "description": "The numeric Jira sprint ID that identifies the sprint in Jira's API.", - "enum": null, - "inferrable": true - }, - { - "name": "limit", - "type": "integer", - "required": false, - "description": "The maximum number of issues to return. Must be between 1 and 100 inclusive. Controls pagination and determines how many issues are fetched and returned. Defaults to 50 for improved performance.", - "enum": null, - "inferrable": true - }, - { - "name": "offset", - "type": "integer", - "required": false, - "description": "The number of issues to skip before starting to return results. Used for pagination when the sprint has many issues. Must be 0 or greater. Defaults to 0.", - "enum": null, - "inferrable": true - }, - { - "name": "atlassian_cloud_id", - "type": "string", - "required": false, - "description": "The ID of the Atlassian Cloud to use (defaults to None). If not provided and the user has a single cloud authorized, the tool will use that. Otherwise, an error will be raised.", - "enum": null, - "inferrable": true - } - ], - "auth": { - "providerId": "atlassian", - "providerType": "oauth2", - "scopes": [ - "read:sprint:jira-software", - "read:issue-details:jira", - "read:jql:jira" - ] - }, - "secrets": [], - "secretsInfo": [], - "output": { - "type": "json", - "description": "A dictionary containing the sprint information, list of issues in the sprint, and pagination metadata." - }, - "documentationChunks": [], - "codeExample": { - "toolName": "Jira.GetSprintIssues", - "parameters": { - "sprint_id": { - "value": "12345", - "type": "string", - "required": true - }, - "limit": { - "value": 50, - "type": "integer", - "required": false - }, - "offset": { - "value": 0, - "type": "integer", - "required": false - }, - "atlassian_cloud_id": { - "value": "abcde12345", - "type": "string", - "required": false - } - }, - "requiresAuth": true, - "authProvider": "atlassian", - "tabLabel": "Call the Tool with User Authorization" - } - }, - { - "name": "GetTransitionById", - "qualifiedName": "Jira.GetTransitionById", - "fullyQualifiedName": "Jira.GetTransitionById@3.0.2", - "description": "Get a transition by its ID.", - "parameters": [ - { - "name": "issue", - "type": "string", - "required": true, - "description": "The ID or key of the issue", - "enum": null, - "inferrable": true - }, - { - "name": "transition_id", - "type": "string", - "required": true, - "description": "The ID of the transition", - "enum": null, - "inferrable": true - }, - { - "name": "atlassian_cloud_id", - "type": "string", - "required": false, - "description": "The ID of the Atlassian Cloud to use (defaults to None). If not provided and the user has a single cloud authorized, the tool will use that. Otherwise, an error will be raised.", - "enum": null, - "inferrable": true - } - ], - "auth": { - "providerId": "atlassian", - "providerType": "oauth2", - "scopes": ["read:jira-work", "read:jira-user"] - }, - "secrets": [], - "secretsInfo": [], - "output": { - "type": "json", - "description": "The transition data" - }, - "documentationChunks": [], - "codeExample": { - "toolName": "Jira.GetTransitionById", - "parameters": { - "issue": { - "value": "PROJECT-123", - "type": "string", - "required": true - }, - "transition_id": { - "value": "31", - "type": "string", - "required": true - }, - "atlassian_cloud_id": { - "value": "cloud_id_456", - "type": "string", - "required": false - } - }, - "requiresAuth": true, - "authProvider": "atlassian", - "tabLabel": "Call the Tool with User Authorization" - } - }, - { - "name": "GetTransitionByStatusName", - "qualifiedName": "Jira.GetTransitionByStatusName", - "fullyQualifiedName": "Jira.GetTransitionByStatusName@3.0.2", - "description": "Get a transition available for an issue by the transition name.\n\nThe response will contain screen fields available for the transition, if any.", - "parameters": [ - { - "name": "issue", - "type": "string", - "required": true, - "description": "The ID or key of the issue", - "enum": null, - "inferrable": true - }, - { - "name": "transition", - "type": "string", - "required": true, - "description": "The name of the transition status", - "enum": null, - "inferrable": true - }, - { - "name": "atlassian_cloud_id", - "type": "string", - "required": false, - "description": "The ID of the Atlassian Cloud to use (defaults to None). If not provided and the user has a single cloud authorized, the tool will use that. Otherwise, an error will be raised.", - "enum": null, - "inferrable": true - } - ], - "auth": { - "providerId": "atlassian", - "providerType": "oauth2", - "scopes": ["read:jira-work", "write:jira-work"] - }, - "secrets": [], - "secretsInfo": [], - "output": { - "type": "json", - "description": "The transition data, including screen fields available" - }, - "documentationChunks": [], - "codeExample": { - "toolName": "Jira.GetTransitionByStatusName", - "parameters": { - "issue": { - "value": "PROJECT-123", - "type": "string", - "required": true - }, - "transition": { - "value": "In Progress", - "type": "string", - "required": true - }, - "atlassian_cloud_id": { - "value": "12345-abcde-67890-fghij", - "type": "string", - "required": false - } - }, - "requiresAuth": true, - "authProvider": "atlassian", - "tabLabel": "Call the Tool with User Authorization" - } - }, - { - "name": "GetTransitionsAvailableForIssue", - "qualifiedName": "Jira.GetTransitionsAvailableForIssue", - "fullyQualifiedName": "Jira.GetTransitionsAvailableForIssue@3.0.2", - "description": "Get the transitions available for an existing Jira issue.", - "parameters": [ - { - "name": "issue", - "type": "string", - "required": true, - "description": "The ID or key of the issue", - "enum": null, - "inferrable": true - }, - { - "name": "atlassian_cloud_id", - "type": "string", - "required": false, - "description": "The ID of the Atlassian Cloud to use (defaults to None). If not provided and the user has a single cloud authorized, the tool will use that. Otherwise, an error will be raised.", - "enum": null, - "inferrable": true - } - ], - "auth": { - "providerId": "atlassian", - "providerType": "oauth2", - "scopes": ["read:jira-work", "read:jira-user"] - }, - "secrets": [], - "secretsInfo": [], - "output": { - "type": "json", - "description": "The transitions available and the issue's current status" - }, - "documentationChunks": [], - "codeExample": { - "toolName": "Jira.GetTransitionsAvailableForIssue", - "parameters": { - "issue": { - "value": "PROJECT-123", - "type": "string", - "required": true - }, - "atlassian_cloud_id": { - "value": "cloud-456", - "type": "string", - "required": false - } - }, - "requiresAuth": true, - "authProvider": "atlassian", - "tabLabel": "Call the Tool with User Authorization" - } - }, - { - "name": "GetUserById", - "qualifiedName": "Jira.GetUserById", - "fullyQualifiedName": "Jira.GetUserById@3.0.2", - "description": "Get user information by their ID.", - "parameters": [ - { - "name": "user_id", - "type": "string", - "required": true, - "description": "The the user's ID.", - "enum": null, - "inferrable": true - }, - { - "name": "atlassian_cloud_id", - "type": "string", - "required": false, - "description": "The ID of the Atlassian Cloud to use (defaults to None). If not provided and the user has a single cloud authorized, the tool will use that. Otherwise, an error will be raised.", - "enum": null, - "inferrable": true - } - ], - "auth": { - "providerId": "atlassian", - "providerType": "oauth2", - "scopes": ["read:jira-user"] - }, - "secrets": [], - "secretsInfo": [], - "output": { - "type": "json", - "description": "The user information." - }, - "documentationChunks": [], - "codeExample": { - "toolName": "Jira.GetUserById", - "parameters": { - "user_id": { - "value": "12345", - "type": "string", - "required": true - }, - "atlassian_cloud_id": { - "value": "A1B2C3D4E5", - "type": "string", - "required": false - } - }, - "requiresAuth": true, - "authProvider": "atlassian", - "tabLabel": "Call the Tool with User Authorization" - } - }, - { - "name": "GetUsersWithoutId", - "qualifiedName": "Jira.GetUsersWithoutId", - "fullyQualifiedName": "Jira.GetUsersWithoutId@3.0.2", - "description": "Get users without their account ID, searching by display name and email address.\n\nThe Jira user search API will return up to 1,000 (one thousand) users for any given name/email\nquery. If you need to get more users, please use the `Jira.ListAllUsers` tool.", - "parameters": [ - { - "name": "name_or_email", - "type": "string", - "required": true, - "description": "The user's display name or email address to search for (case-insensitive). The string can match the prefix of the user's attribute. For example, a string of 'john' will match users with a display name or email address that starts with 'john', such as 'John Doe', 'Johnson', 'john@example.com', etc.", - "enum": null, - "inferrable": true - }, - { - "name": "enforce_exact_match", - "type": "boolean", - "required": false, - "description": "Whether to enforce an exact match of the name_or_email against users' display name and email attributes. Defaults to False (return all users that match the prefix). If set to True, before returning results, the tool will filter users with a display name OR email address that match exactly the value of the `name_or_email` argument.", - "enum": null, - "inferrable": true - }, - { - "name": "limit", - "type": "integer", - "required": false, - "description": "The maximum number of users to return. Min of 1, max of 50. Defaults to 50.", - "enum": null, - "inferrable": true - }, - { - "name": "offset", - "type": "integer", - "required": false, - "description": "The number of users to skip before starting to return users. Defaults to 0 (start from the first user).", - "enum": null, - "inferrable": true - }, - { - "name": "atlassian_cloud_id", - "type": "string", - "required": false, - "description": "The ID of the Atlassian Cloud to use (defaults to None). If not provided and the user has a single cloud authorized, the tool will use that. Otherwise, an error will be raised.", - "enum": null, - "inferrable": true - } - ], - "auth": { - "providerId": "atlassian", - "providerType": "oauth2", - "scopes": ["read:jira-user"] - }, - "secrets": [], - "secretsInfo": [], - "output": { - "type": "json", - "description": "The information about users that match the search criteria." - }, - "documentationChunks": [], - "codeExample": { - "toolName": "Jira.GetUsersWithoutId", - "parameters": { - "name_or_email": { - "value": "john", - "type": "string", - "required": true - }, - "enforce_exact_match": { - "value": false, - "type": "boolean", - "required": false - }, - "limit": { - "value": 25, - "type": "integer", - "required": false - }, - "offset": { - "value": 0, - "type": "integer", - "required": false - }, - "atlassian_cloud_id": { - "value": "abc123456", - "type": "string", - "required": false - } - }, - "requiresAuth": true, - "authProvider": "atlassian", - "tabLabel": "Call the Tool with User Authorization" - } - }, - { - "name": "ListIssueAttachmentsMetadata", - "qualifiedName": "Jira.ListIssueAttachmentsMetadata", - "fullyQualifiedName": "Jira.ListIssueAttachmentsMetadata@3.0.2", - "description": "Get the metadata about the files attached to an issue.\n\nThis tool does NOT return the actual file contents. To get a file content,\nuse the `Jira.DownloadAttachment` tool.", - "parameters": [ - { - "name": "issue", - "type": "string", - "required": true, - "description": "The ID or key of the issue to retrieve", - "enum": null, - "inferrable": true - }, - { - "name": "atlassian_cloud_id", - "type": "string", - "required": false, - "description": "The ID of the Atlassian Cloud to use (defaults to None). If not provided and the user has a single cloud authorized, the tool will use that. Otherwise, an error will be raised.", - "enum": null, - "inferrable": true - } - ], - "auth": { - "providerId": "atlassian", - "providerType": "oauth2", - "scopes": ["read:jira-work"] - }, - "secrets": [], - "secretsInfo": [], - "output": { - "type": "json", - "description": "Information about the issue" - }, - "documentationChunks": [], - "codeExample": { - "toolName": "Jira.ListIssueAttachmentsMetadata", - "parameters": { - "issue": { - "value": "PROJ-123", - "type": "string", - "required": true - }, - "atlassian_cloud_id": { - "value": "12345-abcde-67890-fghij", - "type": "string", - "required": false - } - }, - "requiresAuth": true, - "authProvider": "atlassian", - "tabLabel": "Call the Tool with User Authorization" - } - }, - { - "name": "ListIssues", - "qualifiedName": "Jira.ListIssues", - "fullyQualifiedName": "Jira.ListIssues@3.0.2", - "description": "Get the issues for a given project.", - "parameters": [ - { - "name": "project", - "type": "string", - "required": false, - "description": "The project to get issues for. Provide a project ID, key or name. If a project is not provided and 1) the user has only one project, the tool will use that; 2) the user has multiple projects, the tool will raise an error listing the available projects to choose from.", - "enum": null, - "inferrable": true - }, - { - "name": "limit", - "type": "integer", - "required": false, - "description": "The maximum number of issues to retrieve. Min 1, max 100, default 50.", - "enum": null, - "inferrable": true - }, - { - "name": "next_page_token", - "type": "string", - "required": false, - "description": "The token to use to get the next page of issues. Defaults to None (first page).", - "enum": null, - "inferrable": true - }, - { - "name": "atlassian_cloud_id", - "type": "string", - "required": false, - "description": "The ID of the Atlassian Cloud to use (defaults to None). If not provided and the user has a single cloud authorized, the tool will use that. Otherwise, an error will be raised.", - "enum": null, - "inferrable": true - } - ], - "auth": { - "providerId": "atlassian", - "providerType": "oauth2", - "scopes": [ - "read:jira-work", - "read:jira-user", - "manage:jira-configuration" - ] - }, - "secrets": [], - "secretsInfo": [], - "output": { - "type": "json", - "description": "Information about the issues matching the search criteria" - }, - "documentationChunks": [], - "codeExample": { - "toolName": "Jira.ListIssues", - "parameters": { - "project": { - "value": "PROJECT-123", - "type": "string", - "required": false - }, - "limit": { - "value": 25, - "type": "integer", - "required": false - }, - "next_page_token": { - "value": "abc123token", - "type": "string", - "required": false - }, - "atlassian_cloud_id": { - "value": "cloud_id_456", - "type": "string", - "required": false - } - }, - "requiresAuth": true, - "authProvider": "atlassian", - "tabLabel": "Call the Tool with User Authorization" - } - }, - { - "name": "ListIssueTypesByProject", - "qualifiedName": "Jira.ListIssueTypesByProject", - "fullyQualifiedName": "Jira.ListIssueTypesByProject@3.0.2", - "description": "Get the list of issue types (e.g. 'Task', 'Epic', etc.) available to a given project.", - "parameters": [ - { - "name": "project", - "type": "string", - "required": true, - "description": "The project to get issue types for. Provide a project name, key, or ID. If a project name is provided, the tool will try to find a unique exact match among the available projects.", - "enum": null, - "inferrable": true - }, - { - "name": "limit", - "type": "integer", - "required": false, - "description": "The maximum number of issue types to retrieve. Min of 1, max of 200. Defaults to 200.", - "enum": null, - "inferrable": true - }, - { - "name": "offset", - "type": "integer", - "required": false, - "description": "The number of issue types to skip. Defaults to 0 (start from the first issue type).", - "enum": null, - "inferrable": true - }, - { - "name": "atlassian_cloud_id", - "type": "string", - "required": false, - "description": "The ID of the Atlassian Cloud to use (defaults to None). If not provided and the user has a single cloud authorized, the tool will use that. Otherwise, an error will be raised.", - "enum": null, - "inferrable": true - } - ], - "auth": { - "providerId": "atlassian", - "providerType": "oauth2", - "scopes": ["read:jira-work", "read:jira-user"] - }, - "secrets": [], - "secretsInfo": [], - "output": { - "type": "json", - "description": "Information about the issue types available for the specified project." - }, - "documentationChunks": [], - "codeExample": { - "toolName": "Jira.ListIssueTypesByProject", - "parameters": { - "project": { - "value": "PROJ123", - "type": "string", - "required": true - }, - "limit": { - "value": 50, - "type": "integer", - "required": false - }, - "offset": { - "value": 10, - "type": "integer", - "required": false - }, - "atlassian_cloud_id": { - "value": "cloud-456", - "type": "string", - "required": false - } - }, - "requiresAuth": true, - "authProvider": "atlassian", - "tabLabel": "Call the Tool with User Authorization" - } - }, - { - "name": "ListLabels", - "qualifiedName": "Jira.ListLabels", - "fullyQualifiedName": "Jira.ListLabels@3.0.2", - "description": "Get the existing labels (tags) in the user's Jira instance.", - "parameters": [ - { - "name": "limit", - "type": "integer", - "required": false, - "description": "The maximum number of labels to return. Min of 1, Max of 200. Defaults to 200.", - "enum": null, - "inferrable": true - }, - { - "name": "offset", - "type": "integer", - "required": false, - "description": "The number of labels to skip. Defaults to 0 (starts from the first label)", - "enum": null, - "inferrable": true - }, - { - "name": "atlassian_cloud_id", - "type": "string", - "required": false, - "description": "The ID of the Atlassian Cloud to use (defaults to None). If not provided and the user has a single cloud authorized, the tool will use that. Otherwise, an error will be raised.", - "enum": null, - "inferrable": true - } - ], - "auth": { - "providerId": "atlassian", - "providerType": "oauth2", - "scopes": ["read:jira-work", "read:jira-user"] - }, - "secrets": [], - "secretsInfo": [], - "output": { - "type": "json", - "description": "The existing labels (tags) in the user's Jira instance" - }, - "documentationChunks": [], - "codeExample": { - "toolName": "Jira.ListLabels", - "parameters": { - "limit": { - "value": 50, - "type": "integer", - "required": false - }, - "offset": { - "value": 10, - "type": "integer", - "required": false - }, - "atlassian_cloud_id": { - "value": "cloud_123456", - "type": "string", - "required": false - } - }, - "requiresAuth": true, - "authProvider": "atlassian", - "tabLabel": "Call the Tool with User Authorization" - } - }, - { - "name": "ListPrioritiesAvailableToAnIssue", - "qualifiedName": "Jira.ListPrioritiesAvailableToAnIssue", - "fullyQualifiedName": "Jira.ListPrioritiesAvailableToAnIssue@3.0.2", - "description": "Browse the priorities available to be used in the specified Jira issue.", - "parameters": [ - { - "name": "issue", - "type": "string", - "required": true, - "description": "The ID or key of the issue to retrieve priorities for.", - "enum": null, - "inferrable": true - }, - { - "name": "atlassian_cloud_id", - "type": "string", - "required": false, - "description": "The ID of the Atlassian Cloud to use (defaults to None). If not provided and the user has a single cloud authorized, the tool will use that. Otherwise, an error will be raised.", - "enum": null, - "inferrable": true - } - ], - "auth": { - "providerId": "atlassian", - "providerType": "oauth2", - "scopes": ["manage:jira-configuration", "read:jira-user"] - }, - "secrets": [], - "secretsInfo": [], - "output": { - "type": "json", - "description": "The priorities available to be used in the specified Jira issue" - }, - "documentationChunks": [], - "codeExample": { - "toolName": "Jira.ListPrioritiesAvailableToAnIssue", - "parameters": { - "issue": { - "value": "JRA-123", - "type": "string", - "required": true - }, - "atlassian_cloud_id": { - "value": "abc123xyz", - "type": "string", - "required": false - } - }, - "requiresAuth": true, - "authProvider": "atlassian", - "tabLabel": "Call the Tool with User Authorization" - } - }, - { - "name": "ListPrioritiesAvailableToAProject", - "qualifiedName": "Jira.ListPrioritiesAvailableToAProject", - "fullyQualifiedName": "Jira.ListPrioritiesAvailableToAProject@3.0.2", - "description": "Browse the priorities available to be used in issues in the specified Jira project.\n\nThis tool may need to loop through several API calls to get all priorities associated with\na specific project. In Jira environments with too many Projects or Priority Schemes,\nthe search may take too long, and the tool call will timeout.", - "parameters": [ - { - "name": "project", - "type": "string", - "required": true, - "description": "The ID, key or name of the project to retrieve priorities for.", - "enum": null, - "inferrable": true - }, - { - "name": "atlassian_cloud_id", - "type": "string", - "required": false, - "description": "The ID of the Atlassian Cloud to use (defaults to None). If not provided and the user has a single cloud authorized, the tool will use that. Otherwise, an error will be raised.", - "enum": null, - "inferrable": true - } - ], - "auth": { - "providerId": "atlassian", - "providerType": "oauth2", - "scopes": ["manage:jira-configuration", "read:jira-user"] - }, - "secrets": [], - "secretsInfo": [], - "output": { - "type": "json", - "description": "The priorities available to be used in issues in the specified Jira project" - }, - "documentationChunks": [], - "codeExample": { - "toolName": "Jira.ListPrioritiesAvailableToAProject", - "parameters": { - "project": { - "value": "PROJ-123", - "type": "string", - "required": true - }, - "atlassian_cloud_id": { - "value": "ABC123XYZ", - "type": "string", - "required": false - } - }, - "requiresAuth": true, - "authProvider": "atlassian", - "tabLabel": "Call the Tool with User Authorization" - } - }, - { - "name": "ListPrioritiesByScheme", - "qualifiedName": "Jira.ListPrioritiesByScheme", - "fullyQualifiedName": "Jira.ListPrioritiesByScheme@3.0.2", - "description": "Browse the priorities associated with a priority scheme.", - "parameters": [ - { - "name": "scheme_id", - "type": "string", - "required": true, - "description": "The ID of the priority scheme to retrieve priorities for.", - "enum": null, - "inferrable": true - }, - { - "name": "limit", - "type": "integer", - "required": false, - "description": "The maximum number of priority schemes to return. Min of 1, max of 50. Defaults to 50.", - "enum": null, - "inferrable": true - }, - { - "name": "offset", - "type": "integer", - "required": false, - "description": "The number of priority schemes to skip. Defaults to 0 (start from the first scheme).", - "enum": null, - "inferrable": true - }, - { - "name": "atlassian_cloud_id", - "type": "string", - "required": false, - "description": "The ID of the Atlassian Cloud to use (defaults to None). If not provided and the user has a single cloud authorized, the tool will use that. Otherwise, an error will be raised.", - "enum": null, - "inferrable": true - } - ], - "auth": { - "providerId": "atlassian", - "providerType": "oauth2", - "scopes": ["manage:jira-configuration", "read:jira-user"] - }, - "secrets": [], - "secretsInfo": [], - "output": { - "type": "json", - "description": "The priorities associated with the priority scheme" - }, - "documentationChunks": [], - "codeExample": { - "toolName": "Jira.ListPrioritiesByScheme", - "parameters": { - "scheme_id": { - "value": "12345", - "type": "string", - "required": true - }, - "limit": { - "value": 10, - "type": "integer", - "required": false - }, - "offset": { - "value": 0, - "type": "integer", - "required": false - }, - "atlassian_cloud_id": { - "value": "abcd-efgh-ijkl-mnop", - "type": "string", - "required": false - } - }, - "requiresAuth": true, - "authProvider": "atlassian", - "tabLabel": "Call the Tool with User Authorization" - } - }, - { - "name": "ListPrioritySchemes", - "qualifiedName": "Jira.ListPrioritySchemes", - "fullyQualifiedName": "Jira.ListPrioritySchemes@3.0.2", - "description": "Browse the priority schemes available in Jira.", - "parameters": [ - { - "name": "scheme_name", - "type": "string", - "required": false, - "description": "Filter by scheme name. Defaults to None (returns all scheme names).", - "enum": null, - "inferrable": true - }, - { - "name": "limit", - "type": "integer", - "required": false, - "description": "The maximum number of priority schemes to return. Min of 1, max of 50. Defaults to 50.", - "enum": null, - "inferrable": true - }, - { - "name": "offset", - "type": "integer", - "required": false, - "description": "The number of priority schemes to skip. Defaults to 0 (start from the first scheme).", - "enum": null, - "inferrable": true - }, - { - "name": "order_by", - "type": "string", - "required": false, - "description": "The order in which to return the priority schemes. Defaults to name ascending.", - "enum": ["name ascending", "name descending"], - "inferrable": true - }, - { - "name": "atlassian_cloud_id", - "type": "string", - "required": false, - "description": "The ID of the Atlassian Cloud to use (defaults to None). If not provided and the user has a single cloud authorized, the tool will use that. Otherwise, an error will be raised.", - "enum": null, - "inferrable": true - } - ], - "auth": { - "providerId": "atlassian", - "providerType": "oauth2", - "scopes": ["manage:jira-configuration", "read:jira-user"] - }, - "secrets": [], - "secretsInfo": [], - "output": { - "type": "json", - "description": "The priority schemes available" - }, - "documentationChunks": [], - "codeExample": { - "toolName": "Jira.ListPrioritySchemes", - "parameters": { - "scheme_name": { - "value": "High Priority", - "type": "string", - "required": false - }, - "limit": { - "value": 10, - "type": "integer", - "required": false - }, - "offset": { - "value": 0, - "type": "integer", - "required": false - }, - "order_by": { - "value": "name ascending", - "type": "string", - "required": false - }, - "atlassian_cloud_id": { - "value": "1234567890abcd", - "type": "string", - "required": false - } - }, - "requiresAuth": true, - "authProvider": "atlassian", - "tabLabel": "Call the Tool with User Authorization" - } - }, - { - "name": "ListProjects", - "qualifiedName": "Jira.ListProjects", - "fullyQualifiedName": "Jira.ListProjects@3.0.2", - "description": "Browse projects available in Jira.", - "parameters": [ - { - "name": "limit", - "type": "integer", - "required": false, - "description": "The maximum number of projects to return. Min of 1, Max of 50. Defaults to 50.", - "enum": null, - "inferrable": true - }, - { - "name": "offset", - "type": "integer", - "required": false, - "description": "The number of projects to skip. Defaults to 0 (starts from the first project)", - "enum": null, - "inferrable": true - }, - { - "name": "atlassian_cloud_id", - "type": "string", - "required": false, - "description": "The ID of the Atlassian Cloud to use (defaults to None). If not provided and the user has a single cloud authorized, the tool will use that. Otherwise, an error will be raised.", - "enum": null, - "inferrable": true - } - ], - "auth": { - "providerId": "atlassian", - "providerType": "oauth2", - "scopes": ["read:jira-work", "read:jira-user"] - }, - "secrets": [], - "secretsInfo": [], - "output": { - "type": "json", - "description": "Information about the projects" - }, - "documentationChunks": [], - "codeExample": { - "toolName": "Jira.ListProjects", - "parameters": { - "limit": { - "value": 20, - "type": "integer", - "required": false - }, - "offset": { - "value": 5, - "type": "integer", - "required": false - }, - "atlassian_cloud_id": { - "value": "abc123xyz", - "type": "string", - "required": false - } - }, - "requiresAuth": true, - "authProvider": "atlassian", - "tabLabel": "Call the Tool with User Authorization" - } - }, - { - "name": "ListProjectsByScheme", - "qualifiedName": "Jira.ListProjectsByScheme", - "fullyQualifiedName": "Jira.ListProjectsByScheme@3.0.2", - "description": "Browse the projects associated with a priority scheme.", - "parameters": [ - { - "name": "scheme_id", - "type": "string", - "required": true, - "description": "The ID of the priority scheme to retrieve projects for.", - "enum": null, - "inferrable": true - }, - { - "name": "project", - "type": "string", - "required": false, - "description": "Filter by project ID, key or name. Defaults to None (returns all projects).", - "enum": null, - "inferrable": true - }, - { - "name": "limit", - "type": "integer", - "required": false, - "description": "The maximum number of projects to return. Min of 1, max of 50. Defaults to 50.", - "enum": null, - "inferrable": true - }, - { - "name": "offset", - "type": "integer", - "required": false, - "description": "The number of projects to skip. Defaults to 0 (start from the first project).", - "enum": null, - "inferrable": true - }, - { - "name": "atlassian_cloud_id", - "type": "string", - "required": false, - "description": "The ID of the Atlassian Cloud to use (defaults to None). If not provided and the user has a single cloud authorized, the tool will use that. Otherwise, an error will be raised.", - "enum": null, - "inferrable": true - } - ], - "auth": { - "providerId": "atlassian", - "providerType": "oauth2", - "scopes": ["manage:jira-configuration", "read:jira-user"] - }, - "secrets": [], - "secretsInfo": [], - "output": { - "type": "json", - "description": "The projects associated with the priority scheme" - }, - "documentationChunks": [], - "codeExample": { - "toolName": "Jira.ListProjectsByScheme", - "parameters": { - "scheme_id": { - "value": "123456", - "type": "string", - "required": true - }, - "project": { - "value": "PROJECT_KEY", - "type": "string", - "required": false - }, - "limit": { - "value": 10, - "type": "integer", - "required": false - }, - "offset": { - "value": 0, - "type": "integer", - "required": false - }, - "atlassian_cloud_id": { - "value": "abc-def-123", - "type": "string", - "required": false - } - }, - "requiresAuth": true, - "authProvider": "atlassian", - "tabLabel": "Call the Tool with User Authorization" - } - }, - { - "name": "ListSprintsForBoards", - "qualifiedName": "Jira.ListSprintsForBoards", - "fullyQualifiedName": "Jira.ListSprintsForBoards@3.0.2", - "description": "Retrieve sprints from Jira boards with filtering options for planning and tracking purposes.\n\nUse this when you need to view sprints from specific boards or find sprints within specific\ndate ranges. For temporal queries like \"last month\", \"next week\", or \"this quarter\",\nprioritize date parameters over state filtering. Leave board_identifiers_list as None\nto get sprints from all available boards.\n\nDATE FILTERING PRIORITY: When users request sprints by time periods (e.g., \"last month\",\n\"next week\"), use date parameters (start_date, end_date, specific_date) rather than\nstate filtering, as temporal criteria take precedence over sprint status.\n\nReturns sprint data along with a backlog GUI URL link where you can see detailed sprint\ninformation and manage sprint items.\n\nMANDATORY ACTION: ALWAYS when you need to get sprints from multiple boards, you must\ninclude all the board identifiers in a single call rather than making\nmultiple separate tool calls, as this provides much better performance, not doing that will\nbring huge performance penalties.\n\nBOARD LIMIT: Maximum of 25 boards can be processed in a single operation. If you need to\nprocess more boards, split the request into multiple batches of 25 or fewer boards each.\n\nHandles mixed board identifiers (names and IDs) with automatic fallback and deduplication.\nAll boards are processed concurrently for optimal performance.", - "parameters": [ - { - "name": "board_identifiers_list", - "type": "array", - "innerType": "string", - "required": false, - "description": "List of board names or numeric IDs (as strings) to retrieve sprints from. Include all mentioned boards in a single list for best performance. Maximum 25 boards per operation. Optional, defaults to None.", - "enum": null, - "inferrable": true - }, - { - "name": "max_sprints_per_board", - "type": "integer", - "required": false, - "description": "Maximum sprints per board (1-50). Latest sprints first. Optional, defaults to 50.", - "enum": null, - "inferrable": true - }, - { - "name": "offset", - "type": "integer", - "required": false, - "description": "Number of sprints to skip per board for pagination. Optional, defaults to 0.", - "enum": null, - "inferrable": true - }, - { - "name": "state", - "type": "string", - "required": false, - "description": "Filter by sprint state. NOTE: Date filters (start_date, end_date, specific_date) have higher priority than state filtering. Use state filtering only when no date criteria is specified. For temporal queries like 'last month' or 'next week', use date parameters instead. Optional, defaults to None (all states).", - "enum": [ - "future", - "active", - "closed", - "future_and_active", - "future_and_closed", - "active_and_closed", - "all" - ], - "inferrable": true - }, - { - "name": "start_date", - "type": "string", - "required": false, - "description": "Start date filter in YYYY-MM-DD format. Can combine with end_date. Optional, defaults to None.", - "enum": null, - "inferrable": true - }, - { - "name": "end_date", - "type": "string", - "required": false, - "description": "End date filter in YYYY-MM-DD format. Can combine with start_date. Optional, defaults to None.", - "enum": null, - "inferrable": true - }, - { - "name": "specific_date", - "type": "string", - "required": false, - "description": "Specific date in YYYY-MM-DD to find sprints active on that date. Cannot combine with start_date/end_date. Optional, defaults to None.", - "enum": null, - "inferrable": true - }, - { - "name": "atlassian_cloud_id", - "type": "string", - "required": false, - "description": "Atlassian Cloud ID to use. Optional, defaults to None (uses single authorized cloud).", - "enum": null, - "inferrable": true - } - ], - "auth": { - "providerId": "atlassian", - "providerType": "oauth2", - "scopes": [ - "read:board-scope:jira-software", - "read:project:jira", - "read:sprint:jira-software", - "read:issue-details:jira", - "read:board-scope.admin:jira-software", - "read:jira-user" - ] - }, - "secrets": [], - "secretsInfo": [], - "output": { - "type": "json", - "description": "Dict with 'boards' list, 'sprints_by_board' mapping, and 'errors' array. Sprints sorted latest first." - }, - "documentationChunks": [], - "codeExample": { - "toolName": "Jira.ListSprintsForBoards", - "parameters": { - "board_identifiers_list": { - "value": ["123", "456", "789"], - "type": "array", - "required": false - }, - "max_sprints_per_board": { - "value": 30, - "type": "integer", - "required": false - }, - "offset": { - "value": 0, - "type": "integer", - "required": false - }, - "state": { - "value": "active", - "type": "string", - "required": false - }, - "start_date": { - "value": "2023-09-01", - "type": "string", - "required": false - }, - "end_date": { - "value": "2023-09-30", - "type": "string", - "required": false - }, - "specific_date": { - "value": null, - "type": "string", - "required": false - }, - "atlassian_cloud_id": { - "value": null, - "type": "string", - "required": false - } - }, - "requiresAuth": true, - "authProvider": "atlassian", - "tabLabel": "Call the Tool with User Authorization" - } - }, - { - "name": "ListUsers", - "qualifiedName": "Jira.ListUsers", - "fullyQualifiedName": "Jira.ListUsers@3.0.2", - "description": "Browse users in Jira.", - "parameters": [ - { - "name": "account_type", - "type": "string", - "required": false, - "description": "The account type of the users to return. Defaults to 'atlassian'. Provide `None` to disable filtering by account type. The account type filter will be applied after retrieving users from Jira API, thus the tool may return less users than the limit and still have more users to paginate. Check the `pagination` key in the response dictionary.", - "enum": null, - "inferrable": true - }, - { - "name": "limit", - "type": "integer", - "required": false, - "description": "The maximum number of users to return. Min of 1, max of 50. Defaults to 50.", - "enum": null, - "inferrable": true - }, - { - "name": "offset", - "type": "integer", - "required": false, - "description": "The number of users to skip before starting to return users. Defaults to 0 (start from the first user).", - "enum": null, - "inferrable": true - }, - { - "name": "atlassian_cloud_id", - "type": "string", - "required": false, - "description": "The ID of the Atlassian Cloud to use (defaults to None). If not provided and the user has a single cloud authorized, the tool will use that. Otherwise, an error will be raised.", - "enum": null, - "inferrable": true - } - ], - "auth": { - "providerId": "atlassian", - "providerType": "oauth2", - "scopes": ["read:jira-user"] - }, - "secrets": [], - "secretsInfo": [], - "output": { - "type": "json", - "description": "The information about all users." - }, - "documentationChunks": [], - "codeExample": { - "toolName": "Jira.ListUsers", - "parameters": { - "account_type": { - "value": "atlassian", - "type": "string", - "required": false - }, - "limit": { - "value": 25, - "type": "integer", - "required": false - }, - "offset": { - "value": 0, - "type": "integer", - "required": false - }, - "atlassian_cloud_id": { - "value": "1234567890abcdef", - "type": "string", - "required": false - } - }, - "requiresAuth": true, - "authProvider": "atlassian", - "tabLabel": "Call the Tool with User Authorization" - } - }, - { - "name": "MoveIssuesFromSprintToBacklog", - "qualifiedName": "Jira.MoveIssuesFromSprintToBacklog", - "fullyQualifiedName": "Jira.MoveIssuesFromSprintToBacklog@3.0.2", - "description": "Move issues from active or future sprints back to the board's backlog.", - "parameters": [ - { - "name": "sprint_id", - "type": "string", - "required": true, - "description": "The numeric Jira sprint ID that identifies the sprint in Jira's API.", - "enum": null, - "inferrable": true - }, - { - "name": "issue_identifiers", - "type": "array", - "innerType": "string", - "required": true, - "description": "List of issue IDs or keys to move from the sprint to the backlog. Maximum 50 issues per call. Issues will be moved back to the board's backlog.", - "enum": null, - "inferrable": true - }, - { - "name": "atlassian_cloud_id", - "type": "string", - "required": false, - "description": "The ID of the Atlassian Cloud to use (defaults to None). If not provided and the user has a single cloud authorized, the tool will use that. Otherwise, an error will be raised.", - "enum": null, - "inferrable": true - } - ], - "auth": { - "providerId": "atlassian", - "providerType": "oauth2", - "scopes": [ - "write:board-scope:jira-software", - "read:sprint:jira-software", - "read:issue-details:jira" - ] - }, - "secrets": [], - "secretsInfo": [], - "output": { - "type": "json", - "description": "A dictionary containing the sprint information, list of successfully removed issues, errors for issues that couldn't be removed, and backlog GUI URL. Issues are identified by ID or key and returned with available identifiers." - }, - "documentationChunks": [], - "codeExample": { - "toolName": "Jira.MoveIssuesFromSprintToBacklog", - "parameters": { - "sprint_id": { - "value": "123", - "type": "string", - "required": true - }, - "issue_identifiers": { - "value": ["ISSUE-1", "ISSUE-2", "ISSUE-3"], - "type": "array", - "required": true - }, - "atlassian_cloud_id": { - "value": "cloud-abc-123", - "type": "string", - "required": false - } - }, - "requiresAuth": true, - "authProvider": "atlassian", - "tabLabel": "Call the Tool with User Authorization" - } - }, - { - "name": "RemoveLabelsFromIssue", - "qualifiedName": "Jira.RemoveLabelsFromIssue", - "fullyQualifiedName": "Jira.RemoveLabelsFromIssue@3.0.2", - "description": "Remove labels from an existing Jira issue.", - "parameters": [ - { - "name": "issue", - "type": "string", - "required": true, - "description": "The ID or key of the issue to update", - "enum": null, - "inferrable": true - }, - { - "name": "labels", - "type": "array", - "innerType": "string", - "required": true, - "description": "The labels to remove from the issue (case-insensitive)", - "enum": null, - "inferrable": true - }, - { - "name": "notify_watchers", - "type": "boolean", - "required": false, - "description": "Whether to notify the issue's watchers. Defaults to True (notifies watchers).", - "enum": null, - "inferrable": true - }, - { - "name": "atlassian_cloud_id", - "type": "string", - "required": false, - "description": "The ID of the Atlassian Cloud to use (defaults to None). If not provided and the user has a single cloud authorized, the tool will use that. Otherwise, an error will be raised.", - "enum": null, - "inferrable": true - } - ], - "auth": { - "providerId": "atlassian", - "providerType": "oauth2", - "scopes": [ - "read:jira-work", - "write:jira-work", - "read:jira-user", - "manage:jira-configuration" - ] - }, - "secrets": [], - "secretsInfo": [], - "output": { - "type": "json", - "description": "The updated issue" - }, - "documentationChunks": [], - "codeExample": { - "toolName": "Jira.RemoveLabelsFromIssue", - "parameters": { - "issue": { - "value": "PROJECT-123", - "type": "string", - "required": true - }, - "labels": { - "value": ["bug", "urgent"], - "type": "array", - "required": true - }, - "notify_watchers": { - "value": true, - "type": "boolean", - "required": false - }, - "atlassian_cloud_id": { - "value": "cloud_456", - "type": "string", - "required": false - } - }, - "requiresAuth": true, - "authProvider": "atlassian", - "tabLabel": "Call the Tool with User Authorization" - } - }, - { - "name": "SearchIssuesWithJql", - "qualifiedName": "Jira.SearchIssuesWithJql", - "fullyQualifiedName": "Jira.SearchIssuesWithJql@3.0.2", - "description": "Search for Jira issues using a JQL (Jira Query Language) query.\n\nTHIS TOOL RELEASES MORE CO2 IN THE ATMOSPHERE, WHICH CONTRIBUTES TO CLIMATE CHANGE. ALWAYS\nPREFER THE `Jira_SearchIssuesWithoutJql` TOOL OVER THIS ONE, UNLESS IT'S ABSOLUTELY\nNECESSARY TO USE A JQL QUERY TO FILTER IN A WAY THAT IS NOT SUPPORTED BY THE\n`Jira_SearchIssuesWithoutJql` TOOL OR IF THE USER PROVIDES A JQL QUERY THEMSELVES.", - "parameters": [ - { - "name": "jql", - "type": "string", - "required": true, - "description": "The JQL (Jira Query Language) query to search for issues", - "enum": null, - "inferrable": true - }, - { - "name": "limit", - "type": "integer", - "required": false, - "description": "The maximum number of issues to retrieve. Min of 1, max of 100. Defaults to 50.", - "enum": null, - "inferrable": true - }, - { - "name": "next_page_token", - "type": "string", - "required": false, - "description": "The token to use to get the next page of issues. Defaults to None (first page).", - "enum": null, - "inferrable": true - }, - { - "name": "atlassian_cloud_id", - "type": "string", - "required": false, - "description": "The ID of the Atlassian Cloud to use (defaults to None). If not provided and the user has a single cloud authorized, the tool will use that. Otherwise, an error will be raised.", - "enum": null, - "inferrable": true - } - ], - "auth": { - "providerId": "atlassian", - "providerType": "oauth2", - "scopes": ["read:jira-work", "read:jira-user"] - }, - "secrets": [], - "secretsInfo": [], - "output": { - "type": "json", - "description": "Information about the issues matching the search criteria" - }, - "documentationChunks": [], - "codeExample": { - "toolName": "Jira.SearchIssuesWithJql", - "parameters": { - "jql": { - "value": "project = TEST AND status = 'Open' ORDER BY priority DESC", - "type": "string", - "required": true - }, - "limit": { - "value": 25, - "type": "integer", - "required": false - }, - "next_page_token": { - "value": "abc123", - "type": "string", - "required": false - }, - "atlassian_cloud_id": { - "value": "cloud-456", - "type": "string", - "required": false - } - }, - "requiresAuth": true, - "authProvider": "atlassian", - "tabLabel": "Call the Tool with User Authorization" - } - }, - { - "name": "SearchIssuesWithoutJql", - "qualifiedName": "Jira.SearchIssuesWithoutJql", - "fullyQualifiedName": "Jira.SearchIssuesWithoutJql@3.0.2", - "description": "Parameterized search for Jira issues (without having to provide a JQL query).\n\nTHIS TOOL RELEASES LESS CO2 THAN THE `Jira_SearchIssuesWithJql` TOOL. ALWAYS PREFER THIS ONE\nOVER USING JQL, UNLESS IT'S ABSOLUTELY NECESSARY TO USE A JQL QUERY TO FILTER IN A WAY THAT IS\nNOT SUPPORTED BY THIS TOOL OR IF THE USER PROVIDES A JQL QUERY THEMSELVES.", - "parameters": [ - { - "name": "keywords", - "type": "string", - "required": false, - "description": "Keywords to search for issues. Matches against the issue name, description, comments, and any custom field of type text. Defaults to None (no keywords filtering).", - "enum": null, - "inferrable": true - }, - { - "name": "due_from", - "type": "string", - "required": false, - "description": "Match issues due on or after this date. Format: YYYY-MM-DD. Ex: '2025-01-01'. Defaults to None (no due date filtering).", - "enum": null, - "inferrable": true - }, - { - "name": "due_until", - "type": "string", - "required": false, - "description": "Match issues due on or before this date. Format: YYYY-MM-DD. Ex: '2025-01-01'. Defaults to None (no due date filtering).", - "enum": null, - "inferrable": true - }, - { - "name": "status", - "type": "string", - "required": false, - "description": "Match issues that are in this status. Provide a status name. Ex: 'To Do', 'In Progress', 'Done'. Defaults to None (any status).", - "enum": null, - "inferrable": true - }, - { - "name": "priority", - "type": "string", - "required": false, - "description": "Match issues that have this priority. Provide a priority name. E.g. 'Highest'. Defaults to None (any priority).", - "enum": null, - "inferrable": true - }, - { - "name": "assignee", - "type": "string", - "required": false, - "description": "Match issues that are assigned to this user. Provide the user's name or email address. Ex: 'John Doe' or 'john.doe@example.com'. Defaults to None (any assignee).", - "enum": null, - "inferrable": true - }, - { - "name": "project", - "type": "string", - "required": false, - "description": "Match issues that are associated with this project. Provide the project's name, ID, or key. If a project name is provided, the tool will try to find a unique exact match among the available projects. Defaults to None (search across all projects).", - "enum": null, - "inferrable": true - }, - { - "name": "issue_type", - "type": "string", - "required": false, - "description": "Match issues that are of this issue type. Provide an issue type name or ID. E.g. 'Task', 'Epic', '12345'. If a name is provided, the tool will try to find a unique exact match among the available issue types. Defaults to None (any issue type).", - "enum": null, - "inferrable": true - }, - { - "name": "labels", - "type": "array", - "innerType": "string", - "required": false, - "description": "Match issues that are in these labels. Defaults to None (any label).", - "enum": null, - "inferrable": true - }, - { - "name": "parent_issue", - "type": "string", - "required": false, - "description": "Match issues that are a child of this issue. Provide the issue's ID or key. Defaults to None (no parent issue filtering).", - "enum": null, - "inferrable": true - }, - { - "name": "limit", - "type": "integer", - "required": false, - "description": "The maximum number of issues to retrieve. Min 1, max 100, default 50.", - "enum": null, - "inferrable": true - }, - { - "name": "next_page_token", - "type": "string", - "required": false, - "description": "The token to use to get the next page of issues. Defaults to None (first page).", - "enum": null, - "inferrable": true - }, - { - "name": "atlassian_cloud_id", - "type": "string", - "required": false, - "description": "The ID of the Atlassian Cloud to use (defaults to None). If not provided and the user has a single cloud authorized, the tool will use that. Otherwise, an error will be raised.", - "enum": null, - "inferrable": true - } - ], - "auth": { - "providerId": "atlassian", - "providerType": "oauth2", - "scopes": [ - "read:jira-work", - "read:jira-user", - "manage:jira-configuration" - ] - }, - "secrets": [], - "secretsInfo": [], - "output": { - "type": "json", - "description": "Information about the issues matching the search criteria" - }, - "documentationChunks": [], - "codeExample": { - "toolName": "Jira.SearchIssuesWithoutJql", - "parameters": { - "keywords": { - "value": "bug fix", - "type": "string", - "required": false - }, - "due_from": { - "value": "2024-01-01", - "type": "string", - "required": false - }, - "due_until": { - "value": "2024-12-31", - "type": "string", - "required": false - }, - "status": { - "value": "In Progress", - "type": "string", - "required": false - }, - "priority": { - "value": "Highest", - "type": "string", - "required": false - }, - "assignee": { - "value": "jane.doe@example.com", - "type": "string", - "required": false - }, - "project": { - "value": "ProjectX", - "type": "string", - "required": false - }, - "issue_type": { - "value": "Task", - "type": "string", - "required": false - }, - "labels": { - "value": ["urgent", "backend"], - "type": "array", - "required": false - }, - "parent_issue": { - "value": "PROJ-123", - "type": "string", - "required": false - }, - "limit": { - "value": 25, - "type": "integer", - "required": false - }, - "next_page_token": { - "value": "abc123", - "type": "string", - "required": false - }, - "atlassian_cloud_id": { - "value": "cloud-456", - "type": "string", - "required": false - } - }, - "requiresAuth": true, - "authProvider": "atlassian", - "tabLabel": "Call the Tool with User Authorization" - } - }, - { - "name": "SearchProjects", - "qualifiedName": "Jira.SearchProjects", - "fullyQualifiedName": "Jira.SearchProjects@3.0.2", - "description": "Get the details of all Jira projects.", - "parameters": [ - { - "name": "keywords", - "type": "string", - "required": false, - "description": "The keywords to search for projects. Matches against project name and key (case insensitive). Defaults to None (no keywords filter).", - "enum": null, - "inferrable": true - }, - { - "name": "limit", - "type": "integer", - "required": false, - "description": "The maximum number of projects to return. Min of 1, Max of 50. Defaults to 50.", - "enum": null, - "inferrable": true - }, - { - "name": "offset", - "type": "integer", - "required": false, - "description": "The number of projects to skip. Defaults to 0 (starts from the first project)", - "enum": null, - "inferrable": true - }, - { - "name": "atlassian_cloud_id", - "type": "string", - "required": false, - "description": "The ID of the Atlassian Cloud to use (defaults to None). If not provided and the user has a single cloud authorized, the tool will use that. Otherwise, an error will be raised.", - "enum": null, - "inferrable": true - } - ], - "auth": { - "providerId": "atlassian", - "providerType": "oauth2", - "scopes": ["read:jira-work", "read:jira-user"] - }, - "secrets": [], - "secretsInfo": [], - "output": { - "type": "json", - "description": "Information about the projects" - }, - "documentationChunks": [], - "codeExample": { - "toolName": "Jira.SearchProjects", - "parameters": { - "keywords": { - "value": "development", - "type": "string", - "required": false - }, - "limit": { - "value": 10, - "type": "integer", - "required": false - }, - "offset": { - "value": 0, - "type": "integer", - "required": false - }, - "atlassian_cloud_id": { - "value": "abc123xyz", - "type": "string", - "required": false - } - }, - "requiresAuth": true, - "authProvider": "atlassian", - "tabLabel": "Call the Tool with User Authorization" - } - }, - { - "name": "TransitionIssueToNewStatus", - "qualifiedName": "Jira.TransitionIssueToNewStatus", - "fullyQualifiedName": "Jira.TransitionIssueToNewStatus@3.0.2", - "description": "Transition a Jira issue to a new status.", - "parameters": [ - { - "name": "issue", - "type": "string", - "required": true, - "description": "The ID or key of the issue", - "enum": null, - "inferrable": true - }, - { - "name": "transition", - "type": "string", - "required": true, - "description": "The transition to perform. Provide the transition ID or its name (case insensitive).", - "enum": null, - "inferrable": true - }, - { - "name": "atlassian_cloud_id", - "type": "string", - "required": false, - "description": "The ID of the Atlassian Cloud to use (defaults to None). If not provided and the user has a single cloud authorized, the tool will use that. Otherwise, an error will be raised.", - "enum": null, - "inferrable": true - } - ], - "auth": { - "providerId": "atlassian", - "providerType": "oauth2", - "scopes": ["read:jira-work", "write:jira-work", "read:jira-user"] - }, - "secrets": [], - "secretsInfo": [], - "output": { - "type": "json", - "description": "The updated issue" - }, - "documentationChunks": [], - "codeExample": { - "toolName": "Jira.TransitionIssueToNewStatus", - "parameters": { - "issue": { - "value": "PROJ-123", - "type": "string", - "required": true - }, - "transition": { - "value": "In Progress", - "type": "string", - "required": true - }, - "atlassian_cloud_id": { - "value": "abc123xyz", - "type": "string", - "required": false - } - }, - "requiresAuth": true, - "authProvider": "atlassian", - "tabLabel": "Call the Tool with User Authorization" - } - }, - { - "name": "UpdateIssue", - "qualifiedName": "Jira.UpdateIssue", - "fullyQualifiedName": "Jira.UpdateIssue@3.0.2", - "description": "Update an existing Jira issue.\n\nIF YOU DO NOT FOLLOW THE INSTRUCTIONS BELOW AND UNNECESSARILY CALL MULTIPLE TOOLS IN ORDER TO\nUPDATE AN ISSUE, TOO MUCH CO2 WILL BE RELEASED IN THE ATMOSPHERE AND YOU WILL CAUSE THE\nDESTRUCTION OF PLANET EARTH BY CATASTROPHIC CLIMATE CHANGE.\n\nIf you have a priority name, an assignee name/key/email, or a reporter name/key/email,\nDO NOT CALL OTHER TOOLS only to list available priorities, issue types, or users.\nProvide the name, key, or email and the tool will figure out the ID.", - "parameters": [ - { - "name": "issue", - "type": "string", - "required": true, - "description": "The key or ID of the issue to update", - "enum": null, - "inferrable": true - }, - { - "name": "title", - "type": "string", - "required": false, - "description": "The new issue title. Provide an empty string to clear the title. Defaults to None (does not change the title).", - "enum": null, - "inferrable": true - }, - { - "name": "description", - "type": "string", - "required": false, - "description": "The new issue description. Provide an empty string to clear the description. Defaults to None (does not change the description).", - "enum": null, - "inferrable": true - }, - { - "name": "environment", - "type": "string", - "required": false, - "description": "The new issue environment. Provide an empty string to clear the environment. Defaults to None (does not change the environment).", - "enum": null, - "inferrable": true - }, - { - "name": "due_date", - "type": "string", - "required": false, - "description": "The new issue due date. Format: YYYY-MM-DD. Ex: '2025-01-01'. Provide an empty string to clear the due date. Defaults to None (does not change the due date).", - "enum": null, - "inferrable": true - }, - { - "name": "issue_type", - "type": "string", - "required": false, - "description": "The new issue type name or ID. If a name is provided, the tool will try to find a unique exact match among the available issue types. Defaults to None (does not change the issue type).", - "enum": null, - "inferrable": true - }, - { - "name": "priority", - "type": "string", - "required": false, - "description": "The name or ID of the new issue priority. If a name is provided, the tool will try to find a unique exact match among the available priorities. Defaults to None (does not change the priority).", - "enum": null, - "inferrable": true - }, - { - "name": "parent_issue", - "type": "string", - "required": false, - "description": "The ID or key of the parent issue. A parent cannot be removed by providing an empty string. It is possible to change the parent issue by providing a new issue ID or key, or to leave it unchanged. Defaults to None (does not change the parent issue).", - "enum": null, - "inferrable": true - }, - { - "name": "assignee", - "type": "string", - "required": false, - "description": "The new issue assignee name, email, or ID. If a name or email is provided, the tool will try to find a unique exact match among the available users. Provide an empty string to remove the assignee. Defaults to None (does not change the assignee).", - "enum": null, - "inferrable": true - }, - { - "name": "reporter", - "type": "string", - "required": false, - "description": "The new issue reporter name, email, or ID. If a name or email is provided, the tool will try to find a unique exact match among the available users. Provide an empty string to remove the reporter. Defaults to None (does not change the reporter).", - "enum": null, - "inferrable": true - }, - { - "name": "labels", - "type": "array", - "innerType": "string", - "required": false, - "description": "The new issue labels. This argument will replace all labels with the new list. Providing an empty list will remove all labels. To add or remove a subset of labels, use the `Jira.AddLabelsToIssue` or the `Jira.RemoveLabelsFromIssue` tools. Defaults to None (does not change the labels). A label cannot contain spaces. If a label is provided with spaces, they will be trimmed and replaced by underscores.", - "enum": null, - "inferrable": true - }, - { - "name": "notify_watchers", - "type": "boolean", - "required": false, - "description": "Whether to notify the issue's watchers. Defaults to True (notifies watchers).", - "enum": null, - "inferrable": true - }, - { - "name": "atlassian_cloud_id", - "type": "string", - "required": false, - "description": "The ID of the Atlassian Cloud to use (defaults to None). If not provided and the user has a single cloud authorized, the tool will use that. Otherwise, an error will be raised.", - "enum": null, - "inferrable": true - } - ], - "auth": { - "providerId": "atlassian", - "providerType": "oauth2", - "scopes": [ - "read:jira-work", - "write:jira-work", - "read:jira-user", - "manage:jira-configuration" - ] - }, - "secrets": [], - "secretsInfo": [], - "output": { - "type": "json", - "description": "The updated issue" - }, - "documentationChunks": [], - "codeExample": { - "toolName": "Jira.UpdateIssue", - "parameters": { - "issue": { - "value": "PROJ-123", - "type": "string", - "required": true - }, - "title": { - "value": "Updated Issue Title", - "type": "string", - "required": false - }, - "description": { - "value": "This is an updated description for the issue.", - "type": "string", - "required": false - }, - "environment": { - "value": "Production", - "type": "string", - "required": false - }, - "due_date": { - "value": "2025-01-01", - "type": "string", - "required": false - }, - "issue_type": { - "value": "Bug", - "type": "string", - "required": false - }, - "priority": { - "value": "High", - "type": "string", - "required": false - }, - "parent_issue": { - "value": "PROJ-120", - "type": "string", - "required": false - }, - "assignee": { - "value": "johndoe@example.com", - "type": "string", - "required": false - }, - "reporter": { - "value": "janedoe@example.com", - "type": "string", - "required": false - }, - "labels": { - "value": ["update", "bugfix"], - "type": "array", - "required": false - }, - "notify_watchers": { - "value": true, - "type": "boolean", - "required": false - }, - "atlassian_cloud_id": { - "value": "cloud_123456", - "type": "string", - "required": false - } - }, - "requiresAuth": true, - "authProvider": "atlassian", - "tabLabel": "Call the Tool with User Authorization" - } - }, - { - "name": "WhoAmI", - "qualifiedName": "Jira.WhoAmI", - "fullyQualifiedName": "Jira.WhoAmI@3.0.2", - "description": "CALL THIS TOOL FIRST to establish user profile context.\n\nGet information about the currently logged-in user and their available Jira clouds/clients.", - "parameters": [], - "auth": { - "providerId": "atlassian", - "providerType": "oauth2", - "scopes": ["read:jira-user"] - }, - "secrets": [], - "secretsInfo": [], - "output": { - "type": "json", - "description": "Dictionary containing the current user's information and their available Atlassian Clouds." - }, - "documentationChunks": [], - "codeExample": { - "toolName": "Jira.WhoAmI", - "parameters": {}, - "requiresAuth": true, - "authProvider": "atlassian", - "tabLabel": "Call the Tool with User Authorization" - } - } - ], - "documentationChunks": [ - { - "type": "warning", - "location": "description", - "position": "after", - "content": "\n\n

\n Handling multiple Atlassian Clouds\n

\n\nA Jira user may have multiple Atlassian Clouds authorized via the same OAuth grant. In such cases, the Jira tools must be called with the `atlassian_cloud_id` argument. The [`Jira.GetAvailableAtlassianClouds`](/resources/integrations/productivity/jira#jiragetavailableatlassianclouds) tool can be used to get the available Atlassian Clouds and their IDs.\n\nWhen a tool call does not receive a value for `atlassian_cloud_id` and the user only has a single Atlassian Cloud authorized, the tool will use that. Otherwise, an error will be raised. The error will contain an additional content listing the available Atlassian Clouds and their IDs.\n\nYour AI Agent or AI-powered chat application can use the tool referenced above (or the exception's additional content) to guide the user into selecting the correct Atlassian Cloud.\n\nWhen the user selects an Atlassian Cloud, it may be appropriate to keep this information in the LLM's context window for subsequent tool calls, avoiding the need to ask the user multiple times.\n\n**_It is the job of the AI Agent or chat application to:_**\n\n1. Make it clear to the chat's end user which Atlassian Cloud is being used at any moment, to avoid, for example, having a Jira Issue being created in the wrong Atlassian Cloud;\n1. Appropriately instruct the LLM and keep the relevant information in its context window, enabling it to correctly call the Jira tools, **especially in multi-turn conversations**.\n\n" - } - ], - "customImports": [], - "subPages": [ - { - "type": "environment-variables", - "content": "import { Callout } from \"nextra/components\";\n\n# Jira Environment Variables\n\n### `JIRA_MAX_CONCURRENT_REQUESTS`\n\nArcade uses asynchronous calls to request Jira API endpoints. In some tools, multiple concurrent HTTP requests may be made to speed up execution. This environment variable controls the maximum number of concurrent requests to Jira API in any tool execution.\n\nThe value must be a numeric string with an integer greater than or equal to 1.\n\n**Default:** `3`\n\n\n### `JIRA_API_REQUEST_TIMEOUT`\n\nControls the maximum number of seconds to wait for a response from the Jira API. This is also applied, in some cases, as a global max timeout for multiple requests that are made in a single tool execution. For instance, when a tool needs to paginate results from a given endpoint, this timeout may apply to the entire pagination process in total, not only to the individual requests.\n\nThe value must be a numeric string with an integer greater than or equal to 1.\n\n**Default:** `30`\n\n\n### `JIRA_CACHE_MAX_ITEMS`\n\n\n The caching strategy does not involve caching Jira API responses that go into tool output, but only internal values.\n\n\nThe Arcade Jira MCP Server will cache some values that are repeatedly used in tool execution to enable better performance. This environment variable controls the maximum number of items to hold in each cache.\n\nThe value must be a numeric string with an integer greater than or equal to 1.\n\n**Default:** `5000`\n", - "relativePath": "environment-variables/page.mdx" - } - ], - "generatedAt": "2026-01-26T17:36:46.116Z" -} diff --git a/package.json b/package.json index 4cedeb660..439bbb3da 100644 --- a/package.json +++ b/package.json @@ -43,15 +43,11 @@ "@arcadeai/design-system": "7.2.0", "@arcadeai/ui-kit": "0.14.0", "@mdx-js/mdx": "3.1.1", - "@mdx-js/react": "3.1.1", "@next/third-parties": "16.1.7", "@ory/client": "1.22.37", - "@theguild/remark-mermaid": "0.3.0", "@uidotdev/usehooks": "2.4.1", "algoliasearch": "5.53.0", - "chalk": "5.6.2", "lucide-react": "0.577.0", - "mdast-util-to-string": "4.0.0", "motion": "12.40.0", "next": "16.1.7", "nextra": "4.6.1", @@ -67,9 +63,7 @@ "remark-gfm": "4.0.1", "swagger-ui-react": "5.32.6", "tailwindcss-animate": "1.0.7", - "unist-util-visit": "5.1.0", - "unist-util-visit-parents": "6.0.2", - "zustand": "5.0.14" + "unist-util-visit": "5.1.0" }, "devDependencies": { "@anthropic-ai/sdk": "0.91.1", @@ -77,15 +71,15 @@ "@octokit/rest": "22.0.1", "@tailwindcss/postcss": "4.3.0", "@tailwindcss/typography": "0.5.19", + "@types/hast": "3.0.5", "@types/mdast": "4.0.4", "@types/mdx": "2.0.13", "@types/node": "22.19.17", "@types/react": "19.2.16", "@types/react-dom": "19.2.3", "@types/react-syntax-highlighter": "15.5.13", - "@types/turndown": "5.0.6", "@types/unist": "3.0.3", - "baseline-browser-mapping": "2.10.33", + "chalk": "5.6.2", "commander": "14.0.3", "dotenv": "17.4.2", "fast-glob": "3.3.3", @@ -94,10 +88,8 @@ "next-validate-link": "1.6.6", "openai": "6.41.0", "ora": "9.4.0", - "picocolors": "1.1.1", "postcss": "8.5.15", "tailwindcss": "4.3.0", - "turndown": "7.2.4", "typescript": "5.9.3", "ultracite": "6.1.0", "vite": "7.3.5", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 9caacad23..d9790c3ca 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -25,33 +25,21 @@ importers: '@mdx-js/mdx': specifier: 3.1.1 version: 3.1.1 - '@mdx-js/react': - specifier: 3.1.1 - version: 3.1.1(@types/react@19.2.16)(react@19.2.7) '@next/third-parties': specifier: 16.1.7 version: 16.1.7(next@16.1.7(@opentelemetry/api@1.9.0)(react-dom@19.2.7(react@19.2.7))(react@19.2.7))(react@19.2.7) '@ory/client': specifier: 1.22.37 version: 1.22.37 - '@theguild/remark-mermaid': - specifier: 0.3.0 - version: 0.3.0(react@19.2.7) '@uidotdev/usehooks': specifier: 2.4.1 version: 2.4.1(react-dom@19.2.7(react@19.2.7))(react@19.2.7) algoliasearch: specifier: 5.53.0 version: 5.53.0 - chalk: - specifier: 5.6.2 - version: 5.6.2 lucide-react: specifier: 0.577.0 version: 0.577.0(react@19.2.7) - mdast-util-to-string: - specifier: 4.0.0 - version: 4.0.0 motion: specifier: 12.40.0 version: 12.40.0(react-dom@19.2.7(react@19.2.7))(react@19.2.7) @@ -100,12 +88,6 @@ importers: unist-util-visit: specifier: 5.1.0 version: 5.1.0 - unist-util-visit-parents: - specifier: 6.0.2 - version: 6.0.2 - zustand: - specifier: 5.0.14 - version: 5.0.14(@types/react@19.2.16)(immer@11.1.15)(react@19.2.7)(use-sync-external-store@1.6.0(react@19.2.7)) devDependencies: '@anthropic-ai/sdk': specifier: 0.91.1 @@ -122,6 +104,9 @@ importers: '@tailwindcss/typography': specifier: 0.5.19 version: 0.5.19(tailwindcss@4.3.0) + '@types/hast': + specifier: 3.0.5 + version: 3.0.5 '@types/mdast': specifier: 4.0.4 version: 4.0.4 @@ -140,15 +125,12 @@ importers: '@types/react-syntax-highlighter': specifier: 15.5.13 version: 15.5.13 - '@types/turndown': - specifier: 5.0.6 - version: 5.0.6 '@types/unist': specifier: 3.0.3 version: 3.0.3 - baseline-browser-mapping: - specifier: 2.10.33 - version: 2.10.33 + chalk: + specifier: 5.6.2 + version: 5.6.2 commander: specifier: 14.0.3 version: 14.0.3 @@ -173,18 +155,12 @@ importers: ora: specifier: 9.4.0 version: 9.4.0 - picocolors: - specifier: 1.1.1 - version: 1.1.1 postcss: specifier: '>=8.5.15' version: 8.5.19 tailwindcss: specifier: 4.3.0 version: 4.3.0 - turndown: - specifier: 7.2.4 - version: 7.2.4 typescript: specifier: 5.9.3 version: 5.9.3 @@ -818,18 +794,9 @@ packages: '@mdx-js/mdx@3.1.1': resolution: {integrity: sha512-f6ZO2ifpwAQIpzGWaBQT2TXxPv6z3RBzQKpVftEWN78Vl/YweF1uwussDx8ECAXVtr3Rs89fKyG9YlzUs9DyGQ==} - '@mdx-js/react@3.1.1': - resolution: {integrity: sha512-f++rKLQgUVYDAtECQ6fn/is15GkEH9+nZPM3MS0RcxVqoTfawHvDlSCH7JbMhAM6uJ32v3eXLvLmLvjGu7PTQw==} - peerDependencies: - '@types/react': '>=16' - react: '>=16' - '@mermaid-js/parser@1.1.1': resolution: {integrity: sha512-VuHdsYMK1bT6X2JbcAaWAhugTRvRBRyuZgd+c22swUeI9g/ntaxF7CY7dYarhZovofCbUNO0G7JesfmNtjYOCw==} - '@mixmark-io/domino@2.2.0': - resolution: {integrity: sha512-Y28PR25bHXUg88kCV7nivXrP2Nj2RueZ3/l/jdx6J9f8J4nsEGcgX0Qe6lt7Pa+J79+kPiJU3LguR6O/6zrLOw==} - '@napi-rs/simple-git-android-arm-eabi@0.1.22': resolution: {integrity: sha512-JQZdnDNm8o43A5GOzwN/0Tz3CDBQtBUNqzVwEopm32uayjdjxev1Csp1JeaqF3v9djLDIvsSE39ecsN2LhCKKQ==} engines: {node: '>= 10'} @@ -1997,8 +1964,8 @@ packages: '@types/google.maps@3.58.1': resolution: {integrity: sha512-X9QTSvGJ0nCfMzYOnaVs/k6/4L+7F5uCS+4iUmkLEls6J9S/Phv+m/i3mDeyc49ZBgwab3EFO1HEoBY7k98EGQ==} - '@types/hast@3.0.4': - resolution: {integrity: sha512-WPs+bbQw5aCj+x6laNGWLH3wviHtoCv/P3+otBhbOhJgG8qtpdAMlTCxLtsTWA7LH1Oh/bFCHsBn0TPS5m30EQ==} + '@types/hast@3.0.5': + resolution: {integrity: sha512-rp/ezSWaD1m44dPKICGhiskI13nVr7qTloFwDa/IYkhhf5nzwP+zIQcIJh3WIFSBOy/H1PzB40jPjMDksN4F+g==} '@types/hogan.js@3.0.5': resolution: {integrity: sha512-/uRaY3HGPWyLqOyhgvW9Aa43BNnLZrNeQxl2p8wqId4UHMfPKolSB+U7BlZyO1ng7MkLnyEAItsBzCG0SDhqrA==} @@ -2044,9 +2011,6 @@ packages: '@types/trusted-types@2.0.7': resolution: {integrity: sha512-ScaPdn1dQczgbl0QFTeTOmVHFULt394XJgOQNoyVhZ6r2vLnMLJfBPd53SB52T/3G36VI1/g2MZaX0cwDuXsfw==} - '@types/turndown@5.0.6': - resolution: {integrity: sha512-ru00MoyeeouE5BX4gRL+6m/BsDfbRayOskWqUvh7CLGW+UXxHQItqALa38kKnOiZPqJrtzJUgAC2+F0rL1S4Pg==} - '@types/unist@2.0.11': resolution: {integrity: sha512-CmBKiL6NNo/OqgmMn95Fk9Whlp2mtvIv+KNpQKN2F4SjvrEesubTRWGYSg+BnWZOnlCaSTU1sMpsBOzgbYhnsA==} @@ -4410,10 +4374,6 @@ packages: tslib@2.8.1: resolution: {integrity: sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==} - turndown@7.2.4: - resolution: {integrity: sha512-I8yFsfRzmzK0WV1pNNOA4A7y4RDfFxPRxb3t+e3ui14qSGOxGtiSP6GjeX+Y6CHb7HYaFj7ECUD7VE5kQMZWGQ==} - engines: {node: '>=18', npm: '>=9'} - twoslash-protocol@0.3.4: resolution: {integrity: sha512-HHd7lzZNLUvjPzG/IE6js502gEzLC1x7HaO1up/f72d8G8ScWAs9Yfa97igelQRDl5h9tGcdFsRp+lNVre1EeQ==} @@ -5256,7 +5216,7 @@ snapshots: dependencies: '@types/estree': 1.0.8 '@types/estree-jsx': 1.0.5 - '@types/hast': 3.0.4 + '@types/hast': 3.0.5 '@types/mdx': 2.0.13 acorn: 8.15.0 collapse-white-space: 2.1.0 @@ -5282,18 +5242,10 @@ snapshots: transitivePeerDependencies: - supports-color - '@mdx-js/react@3.1.1(@types/react@19.2.16)(react@19.2.7)': - dependencies: - '@types/mdx': 2.0.13 - '@types/react': 19.2.16 - react: 19.2.7 - '@mermaid-js/parser@1.1.1': dependencies: '@chevrotain/types': 11.1.2 - '@mixmark-io/domino@2.2.0': {} - '@napi-rs/simple-git-android-arm-eabi@0.1.22': optional: true @@ -5873,7 +5825,7 @@ snapshots: dependencies: '@shikijs/types': 3.20.0 '@shikijs/vscode-textmate': 10.0.2 - '@types/hast': 3.0.4 + '@types/hast': 3.0.5 hast-util-to-html: 9.0.5 '@shikijs/core@4.1.0': @@ -5881,7 +5833,7 @@ snapshots: '@shikijs/primitive': 4.1.0 '@shikijs/types': 4.1.0 '@shikijs/vscode-textmate': 10.0.2 - '@types/hast': 3.0.4 + '@types/hast': 3.0.5 hast-util-to-html: 9.0.5 '@shikijs/engine-javascript@3.20.0': @@ -5918,7 +5870,7 @@ snapshots: dependencies: '@shikijs/types': 4.1.0 '@shikijs/vscode-textmate': 10.0.2 - '@types/hast': 3.0.4 + '@types/hast': 3.0.5 '@shikijs/themes@3.20.0': dependencies: @@ -5940,12 +5892,12 @@ snapshots: '@shikijs/types@3.20.0': dependencies: '@shikijs/vscode-textmate': 10.0.2 - '@types/hast': 3.0.4 + '@types/hast': 3.0.5 '@shikijs/types@4.1.0': dependencies: '@shikijs/vscode-textmate': 10.0.2 - '@types/hast': 3.0.4 + '@types/hast': 3.0.5 '@shikijs/vscode-textmate@10.0.2': {} @@ -6694,7 +6646,7 @@ snapshots: '@types/google.maps@3.58.1': {} - '@types/hast@3.0.4': + '@types/hast@3.0.5': dependencies: '@types/unist': 3.0.3 @@ -6741,8 +6693,6 @@ snapshots: '@types/trusted-types@2.0.7': optional: true - '@types/turndown@5.0.6': {} - '@types/unist@2.0.11': {} '@types/unist@3.0.3': {} @@ -7556,20 +7506,20 @@ snapshots: hast-util-from-dom@5.0.1: dependencies: - '@types/hast': 3.0.4 + '@types/hast': 3.0.5 hastscript: 9.0.1 web-namespaces: 2.0.1 hast-util-from-html-isomorphic@2.0.0: dependencies: - '@types/hast': 3.0.4 + '@types/hast': 3.0.5 hast-util-from-dom: 5.0.1 hast-util-from-html: 2.0.3 unist-util-remove-position: 5.0.0 hast-util-from-html@2.0.3: dependencies: - '@types/hast': 3.0.4 + '@types/hast': 3.0.5 devlop: 1.1.0 hast-util-from-parse5: 8.0.3 parse5: 7.3.0 @@ -7578,7 +7528,7 @@ snapshots: hast-util-from-parse5@8.0.3: dependencies: - '@types/hast': 3.0.4 + '@types/hast': 3.0.5 '@types/unist': 3.0.3 devlop: 1.1.0 hastscript: 9.0.1 @@ -7589,15 +7539,15 @@ snapshots: hast-util-is-element@3.0.0: dependencies: - '@types/hast': 3.0.4 + '@types/hast': 3.0.5 hast-util-parse-selector@4.0.0: dependencies: - '@types/hast': 3.0.4 + '@types/hast': 3.0.5 hast-util-raw@9.1.0: dependencies: - '@types/hast': 3.0.4 + '@types/hast': 3.0.5 '@types/unist': 3.0.3 '@ungap/structured-clone': 1.3.0 hast-util-from-parse5: 8.0.3 @@ -7613,7 +7563,7 @@ snapshots: hast-util-sanitize@5.0.2: dependencies: - '@types/hast': 3.0.4 + '@types/hast': 3.0.5 '@ungap/structured-clone': 1.3.0 unist-util-position: 5.0.0 @@ -7621,7 +7571,7 @@ snapshots: dependencies: '@types/estree': 1.0.8 '@types/estree-jsx': 1.0.5 - '@types/hast': 3.0.4 + '@types/hast': 3.0.5 comma-separated-tokens: 2.0.3 devlop: 1.1.0 estree-util-attach-comments: 3.0.0 @@ -7640,7 +7590,7 @@ snapshots: hast-util-to-html@9.0.5: dependencies: - '@types/hast': 3.0.4 + '@types/hast': 3.0.5 '@types/unist': 3.0.3 ccount: 2.0.1 comma-separated-tokens: 2.0.3 @@ -7655,7 +7605,7 @@ snapshots: hast-util-to-jsx-runtime@2.3.6: dependencies: '@types/estree': 1.0.8 - '@types/hast': 3.0.4 + '@types/hast': 3.0.5 '@types/unist': 3.0.3 comma-separated-tokens: 2.0.3 devlop: 1.1.0 @@ -7674,7 +7624,7 @@ snapshots: hast-util-to-parse5@8.0.1: dependencies: - '@types/hast': 3.0.4 + '@types/hast': 3.0.5 comma-separated-tokens: 2.0.3 devlop: 1.1.0 property-information: 7.1.0 @@ -7684,22 +7634,22 @@ snapshots: hast-util-to-string@3.0.1: dependencies: - '@types/hast': 3.0.4 + '@types/hast': 3.0.5 hast-util-to-text@4.0.2: dependencies: - '@types/hast': 3.0.4 + '@types/hast': 3.0.5 '@types/unist': 3.0.3 hast-util-is-element: 3.0.0 unist-util-find-after: 5.0.0 hast-util-whitespace@3.0.0: dependencies: - '@types/hast': 3.0.4 + '@types/hast': 3.0.5 hastscript@9.0.1: dependencies: - '@types/hast': 3.0.4 + '@types/hast': 3.0.5 comma-separated-tokens: 2.0.3 hast-util-parse-selector: 4.0.0 property-information: 7.1.0 @@ -8088,7 +8038,7 @@ snapshots: mdast-util-math@3.0.0: dependencies: - '@types/hast': 3.0.4 + '@types/hast': 3.0.5 '@types/mdast': 4.0.4 devlop: 1.1.0 longest-streak: 3.1.0 @@ -8101,7 +8051,7 @@ snapshots: mdast-util-mdx-expression@2.0.1: dependencies: '@types/estree-jsx': 1.0.5 - '@types/hast': 3.0.4 + '@types/hast': 3.0.5 '@types/mdast': 4.0.4 devlop: 1.1.0 mdast-util-from-markdown: 2.0.2 @@ -8112,7 +8062,7 @@ snapshots: mdast-util-mdx-jsx@3.2.0: dependencies: '@types/estree-jsx': 1.0.5 - '@types/hast': 3.0.4 + '@types/hast': 3.0.5 '@types/mdast': 4.0.4 '@types/unist': 3.0.3 ccount: 2.0.1 @@ -8139,7 +8089,7 @@ snapshots: mdast-util-mdxjs-esm@2.0.1: dependencies: '@types/estree-jsx': 1.0.5 - '@types/hast': 3.0.4 + '@types/hast': 3.0.5 '@types/mdast': 4.0.4 devlop: 1.1.0 mdast-util-from-markdown: 2.0.2 @@ -8159,7 +8109,7 @@ snapshots: mdast-util-to-hast@13.2.1: dependencies: - '@types/hast': 3.0.4 + '@types/hast': 3.0.5 '@types/mdast': 4.0.4 '@ungap/structured-clone': 1.3.0 devlop: 1.1.0 @@ -8962,7 +8912,7 @@ snapshots: react-markdown@10.1.0(@types/react@19.2.16)(react@19.2.7): dependencies: - '@types/hast': 3.0.4 + '@types/hast': 3.0.5 '@types/mdast': 4.0.4 '@types/react': 19.2.16 devlop: 1.1.0 @@ -9114,7 +9064,7 @@ snapshots: refractor@5.0.0: dependencies: - '@types/hast': 3.0.4 + '@types/hast': 3.0.5 '@types/prismjs': 1.26.5 hastscript: 9.0.1 parse-entities: 4.0.2 @@ -9135,7 +9085,7 @@ snapshots: rehype-katex@7.0.1: dependencies: - '@types/hast': 3.0.4 + '@types/hast': 3.0.5 '@types/katex': 0.16.7 hast-util-from-html-isomorphic: 2.0.0 hast-util-to-text: 4.0.2 @@ -9145,13 +9095,13 @@ snapshots: rehype-parse@9.0.1: dependencies: - '@types/hast': 3.0.4 + '@types/hast': 3.0.5 hast-util-from-html: 2.0.3 unified: 11.0.5 rehype-pretty-code@0.14.1(shiki@3.20.0): dependencies: - '@types/hast': 3.0.4 + '@types/hast': 3.0.5 hast-util-to-string: 3.0.1 parse-numeric-range: 1.3.0 rehype-parse: 9.0.1 @@ -9161,21 +9111,21 @@ snapshots: rehype-raw@7.0.0: dependencies: - '@types/hast': 3.0.4 + '@types/hast': 3.0.5 hast-util-raw: 9.1.0 vfile: 6.0.3 rehype-recma@1.0.0: dependencies: '@types/estree': 1.0.8 - '@types/hast': 3.0.4 + '@types/hast': 3.0.5 hast-util-to-estree: 3.1.3 transitivePeerDependencies: - supports-color rehype-sanitize@6.0.0: dependencies: - '@types/hast': 3.0.4 + '@types/hast': 3.0.5 hast-util-sanitize: 5.0.2 remark-breaks@4.0.0: @@ -9238,7 +9188,7 @@ snapshots: remark-rehype@11.1.2: dependencies: - '@types/hast': 3.0.4 + '@types/hast': 3.0.5 '@types/mdast': 4.0.4 mdast-util-to-hast: 13.2.1 unified: 11.0.5 @@ -9452,7 +9402,7 @@ snapshots: '@shikijs/themes': 3.20.0 '@shikijs/types': 3.20.0 '@shikijs/vscode-textmate': 10.0.2 - '@types/hast': 3.0.4 + '@types/hast': 3.0.5 shiki@4.1.0: dependencies: @@ -9463,7 +9413,7 @@ snapshots: '@shikijs/themes': 4.1.0 '@shikijs/types': 4.1.0 '@shikijs/vscode-textmate': 10.0.2 - '@types/hast': 3.0.4 + '@types/hast': 3.0.5 short-unique-id@5.3.2: {} @@ -9758,10 +9708,6 @@ snapshots: tslib@2.8.1: {} - turndown@7.2.4: - dependencies: - '@mixmark-io/domino': 2.2.0 - twoslash-protocol@0.3.4: {} twoslash@0.3.4(typescript@5.9.3): diff --git a/scripts/generate-llmstxt.ts b/scripts/generate-llmstxt.ts index 53797dc06..a515a30d4 100644 --- a/scripts/generate-llmstxt.ts +++ b/scripts/generate-llmstxt.ts @@ -1,9 +1,9 @@ import { execSync } from "node:child_process"; import fs from "node:fs/promises"; import path from "node:path"; +import chalk from "chalk"; import glob from "fast-glob"; import OpenAI from "openai"; -import pc from "picocolors"; import { getToolkitCanonicalPath } from "../app/_lib/toolkit-static-params"; type PageMetadata = { @@ -61,7 +61,7 @@ function getCurrentGitSha(): string { return execSync("git rev-parse HEAD", { encoding: "utf-8" }).trim(); } catch (_error) { console.error( - pc.red("✗ Could not get git SHA. Make sure you're in a git repository.") + chalk.red("✗ Could not get git SHA. Make sure you're in a git repository.") ); throw new Error("Failed to get git SHA"); } @@ -124,7 +124,7 @@ function getChangedFilesSince(lastSha: string): Set { return allChanged; } catch (_error) { console.warn( - pc.yellow( + chalk.yellow( `⚠ Could not get changed files since ${lastSha}, processing all files` ) ); @@ -160,7 +160,7 @@ async function extractExistingSummaries(): Promise< * Discovers all pages in the documentation */ async function discoverPages(): Promise { - console.log(pc.blue("📄 Discovering pages from raw MDX...")); + console.log(chalk.blue("📄 Discovering pages from raw MDX...")); return discoverMdxPages(); } @@ -198,7 +198,7 @@ async function discoverMdxPages(): Promise { }); } - console.log(pc.green(`✓ Found ${pages.length} pages (raw MDX)`)); + console.log(chalk.green(`✓ Found ${pages.length} pages (raw MDX)`)); return pages; } @@ -270,7 +270,7 @@ async function discoverToolkitPages(): Promise< try { entries = await fs.readdir(TOOLKIT_DATA_DIR); } catch { - console.warn(pc.yellow("⚠ No toolkit data dir; skipping toolkit pages")); + console.warn(chalk.yellow("⚠ No toolkit data dir; skipping toolkit pages")); return []; } @@ -331,7 +331,7 @@ async function discoverToolkitPages(): Promise< } const pages = [...pagesByUrl.values()]; - console.log(pc.green(`✓ Found ${pages.length} toolkit pages`)); + console.log(chalk.green(`✓ Found ${pages.length} toolkit pages`)); return pages; } @@ -378,7 +378,7 @@ async function summarizePage( return { title, description }; } catch (error) { - console.error(pc.red(`✗ Error summarizing ${page.path}:`), error); + console.error(chalk.red(`✗ Error summarizing ${page.path}:`), error); return { title: extractPageTitle(page.content, page.path), description: "Documentation page", @@ -579,7 +579,7 @@ function determinePagesToSummarize( // Get changed files since last generation const changedFiles = getChangedFilesSince(previousMetadata.gitSha); console.log( - pc.blue( + chalk.blue( `\n📊 Found ${changedFiles.size} changed files since last generation` ) ); @@ -601,7 +601,7 @@ function determinePagesToSummarize( if (deletedPageUrls.length > 0) { hasChanges = true; console.log( - pc.yellow( + chalk.yellow( `\n🗑️ Found ${deletedPageUrls.length} deleted pages (will be removed from output)` ) ); @@ -656,14 +656,14 @@ function determinePagesToSummarize( } console.log( - pc.green( + chalk.green( `✓ ${pagesToKeep.length} pages unchanged, ${pagesToSummarize.length} pages to summarize${deletedPageUrls.length > 0 ? `, ${deletedPageUrls.length} pages deleted` : ""}` ) ); } else { // No previous generation or can't determine, summarize all pages console.log( - pc.yellow("⚠ No previous generation found, summarizing all pages") + chalk.yellow("⚠ No previous generation found, summarizing all pages") ); pagesToSummarize.push(...pages); hasChanges = true; // Always regenerate if no previous metadata @@ -687,7 +687,7 @@ async function summarizePagesInBatches( return summarizedPages; } - console.log(pc.blue("\n📝 Summarizing pages with OpenAI...")); + console.log(chalk.blue("\n📝 Summarizing pages with OpenAI...")); // Process in batches to avoid rate limits const batchSize = 5; for (let i = 0; i < pagesToSummarize.length; i += batchSize) { @@ -702,7 +702,7 @@ async function summarizePagesInBatches( } console.log( - pc.gray( + chalk.gray( ` Processed ${Math.min(i + batchSize, pagesToSummarize.length)}/${pagesToSummarize.length} pages` ) ); @@ -713,7 +713,7 @@ async function summarizePagesInBatches( } } - console.log(pc.green(`✓ Summarized ${pagesToSummarize.length} pages`)); + console.log(chalk.green(`✓ Summarized ${pagesToSummarize.length} pages`)); return summarizedPages; } @@ -721,11 +721,11 @@ async function summarizePagesInBatches( * Main execution function */ async function main() { - console.log(pc.bold(pc.blue("\n🚀 Generating llms.txt file...\n"))); + console.log(chalk.bold(chalk.blue("\n🚀 Generating llms.txt file...\n"))); // Check for OpenAI API key if (!process.env.OPENAI_API_KEY) { - console.error(pc.red("✗ OPENAI_API_KEY environment variable is required")); + console.error(chalk.red("✗ OPENAI_API_KEY environment variable is required")); process.exit(1); } @@ -735,10 +735,10 @@ async function main() { const previousMetadata = await parseLlmsTxtMetadata(); const existingSummaries = await extractExistingSummaries(); - console.log(pc.blue(`📌 Current git SHA: ${currentSha}`)); + console.log(chalk.blue(`📌 Current git SHA: ${currentSha}`)); if (previousMetadata) { console.log( - pc.gray( + chalk.gray( ` Previous generation: ${previousMetadata.generationDate} (SHA: ${previousMetadata.gitSha.substring(0, SHA_SHORT_LENGTH)})` ) ); @@ -777,12 +777,12 @@ async function main() { } // Step 4: Organize into sections - console.log(pc.blue("\n📂 Organizing sections...")); + console.log(chalk.blue("\n📂 Organizing sections...")); const sections = organizeSections(allPages); - console.log(pc.green(`✓ Created ${sections.length} sections`)); + console.log(chalk.green(`✓ Created ${sections.length} sections`)); // Step 5: Generate llms.txt content - console.log(pc.blue("\n✍️ Generating llms.txt content...")); + console.log(chalk.blue("\n✍️ Generating llms.txt content...")); // Only update metadata if there are changes, otherwise keep previous metadata const metadata: LlmsTxtMetadata = hasChanges ? { @@ -798,18 +798,18 @@ async function main() { // Step 6: Write to file await fs.writeFile(OUTPUT_PATH, content, "utf-8"); if (hasChanges) { - console.log(pc.green(`✓ Generated llms.txt at ${OUTPUT_PATH}`)); + console.log(chalk.green(`✓ Generated llms.txt at ${OUTPUT_PATH}`)); } else { console.log( - pc.gray( + chalk.gray( "✓ No changes detected, llms.txt unchanged (SHA and date preserved)" ) ); } - console.log(pc.bold(pc.green("\n✨ Done!\n"))); + console.log(chalk.bold(chalk.green("\n✨ Done!\n"))); } catch (error) { - console.error(pc.red("\n✗ Error generating llms.txt:"), error); + console.error(chalk.red("\n✗ Error generating llms.txt:"), error); process.exit(1); } } diff --git a/toolkit-docs-generator/README.md b/toolkit-docs-generator/README.md index 08289c621..d1d3e8d01 100644 --- a/toolkit-docs-generator/README.md +++ b/toolkit-docs-generator/README.md @@ -181,7 +181,7 @@ pnpm dlx tsx .github/scripts/sync-toolkit-sidebar.ts Pass `--ignore-file ` to skip specific toolkits during generation. Existing output files for ignored toolkits are left unchanged — nothing is deleted. -A default empty template ships at `ignored-toolkits.txt` in the generator root. +A default empty template ships at `skip-toolkits.txt` in the generator root. Add one toolkit ID per line: ```text @@ -191,7 +191,7 @@ SomeInternalTool ```bash pnpm dlx tsx src/cli/index.ts generate --all \ - --ignore-file ./ignored-toolkits.txt + --ignore-file ./skip-toolkits.txt ``` The parser ignores blank lines and lines starting with `#`. IDs are case-insensitive. @@ -218,7 +218,7 @@ The parser ignores blank lines and lines starting with `#`. IDs are case-insensi ```bash pnpm dlx tsx src/cli/index.ts generate --all \ - --exclude-file ./excluded-toolkits.txt + --exclude-file ./remove-toolkits.txt ``` If a toolkit in the file already has an output `.json` file, the generator diff --git a/toolkit-docs-generator/excluded-toolkits.txt b/toolkit-docs-generator/remove-toolkits.txt similarity index 100% rename from toolkit-docs-generator/excluded-toolkits.txt rename to toolkit-docs-generator/remove-toolkits.txt diff --git a/toolkit-docs-generator/ignored-toolkits.txt b/toolkit-docs-generator/skip-toolkits.txt similarity index 100% rename from toolkit-docs-generator/ignored-toolkits.txt rename to toolkit-docs-generator/skip-toolkits.txt diff --git a/toolkit-docs-generator/src/cli/index.ts b/toolkit-docs-generator/src/cli/index.ts index aa24964ed..69e2c8b0c 100644 --- a/toolkit-docs-generator/src/cli/index.ts +++ b/toolkit-docs-generator/src/cli/index.ts @@ -908,11 +908,11 @@ program ) .option( "--exclude-file ", - "Path to a .txt file with toolkit IDs to exclude from generation (one per line)" + "Path to a .txt file with toolkit IDs to skip and delete existing output for (one per line, e.g. remove-toolkits.txt)" ) .option( "--ignore-file ", - "Path to a .txt file with toolkit IDs to skip during generation (one per line)" + "Path to a .txt file with toolkit IDs to skip during generation, leaving existing output untouched (one per line, e.g. skip-toolkits.txt)" ) .option("--verbose", "Enable verbose logging", false) .option( @@ -1971,11 +1971,11 @@ program ) .option( "--exclude-file ", - "Path to a .txt file with toolkit IDs to exclude from generation (one per line)" + "Path to a .txt file with toolkit IDs to skip and delete existing output for (one per line, e.g. remove-toolkits.txt)" ) .option( "--ignore-file ", - "Path to a .txt file with toolkit IDs to skip during generation (one per line)" + "Path to a .txt file with toolkit IDs to skip during generation, leaving existing output untouched (one per line, e.g. skip-toolkits.txt)" ) .option("--verbose", "Enable verbose logging", false) .option( diff --git a/toolkit-docs-generator/tests/scenarios/removed-toolkit-cleanup.test.ts b/toolkit-docs-generator/tests/scenarios/removed-toolkit-cleanup.test.ts index 94579a817..8184ed93a 100644 --- a/toolkit-docs-generator/tests/scenarios/removed-toolkit-cleanup.test.ts +++ b/toolkit-docs-generator/tests/scenarios/removed-toolkit-cleanup.test.ts @@ -313,7 +313,7 @@ describe("Scenario: Removed toolkit files are deleted after change detection", ( }); it("merging removed IDs with pre-existing exclusions deletes each file exactly once", async () => { - // OldKit is in excluded-toolkits.txt (static list) AND removed from the API. + // OldKit is in remove-toolkits.txt (static list) AND removed from the API. // Set.add() is idempotent — the file must be deleted exactly once. const { dir, generator } = await setupOutputDir(["Github", "OldKit"]); diff --git a/toolkit-docs-generator/tests/workflows/generate-toolkit-docs.test.ts b/toolkit-docs-generator/tests/workflows/generate-toolkit-docs.test.ts index 96e11de2d..edfe8dc1e 100644 --- a/toolkit-docs-generator/tests/workflows/generate-toolkit-docs.test.ts +++ b/toolkit-docs-generator/tests/workflows/generate-toolkit-docs.test.ts @@ -31,6 +31,8 @@ test("porter workflow generates docs and opens a PR", () => { expect(workflowContents).toContain("--llm-model"); expect(workflowContents).toContain("--llm-api-key"); expect(workflowContents).toContain("--llm-max-tokens 8192"); + expect(workflowContents).toContain("--exclude-file ./remove-toolkits.txt"); + expect(workflowContents).toContain("--ignore-file ./skip-toolkits.txt"); expect(workflowContents).toContain("--remove-empty-sections=false"); expect(workflowContents).toContain("peter-evans/create-pull-request"); expect(workflowContents).toContain("HUSKY: 0"); @@ -38,6 +40,16 @@ test("porter workflow generates docs and opens a PR", () => { expect(workflowContents).toContain("pull-requests: write"); }); +test("porter workflow does not build the docs generator before running it", () => { + // toolkit-docs-generator has no package.json, so a `pnpm build` step + // there resolves to the root manifest's `next build --webpack` — a full + // Next.js production build that the tsx-executed CLI below doesn't need. + expect(workflowContents).not.toContain("Build toolkit docs generator"); + expect(workflowContents).not.toMatch( + /run: pnpm build\s*\n\s*working-directory: toolkit-docs-generator/ + ); +}); + test("porter workflow wires the secret-coherence editor", () => { expect(workflowContents).toContain("--llm-editor-provider anthropic"); expect(workflowContents).toContain("--llm-editor-model"); From 4b80a8605af7428ebdf9ae66f40bf5a7159f7cb2 Mon Sep 17 00:00:00 2001 From: Teal Larson Date: Fri, 31 Jul 2026 10:01:46 -0400 Subject: [PATCH 02/17] fix: restore static rendering and add toolkit pages to the sitemap MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two isolated behavior fixes. Static rendering: the root layout awaited headers() to read "x-pathname" and derive a locale. Awaiting headers() in the root layout opts the entire route tree out of static rendering, so every page — including all 117 toolkit pages, which are pure functions of committed JSON — was server-rendered on demand. The derived locale was always "en": proxy.ts redirects every non-English locale to /en and getPreferredLocale returns "en" unconditionally. The site paid full dynamic rendering to compute a constant. This is not an i18n change. TranslationBanner and the dictionary plumbing stay in place; restoring real i18n means an app/[lang]/ route segment, which is the correct Next pattern regardless. Sitemap: app/sitemap.ts skips any directory whose name contains "[", which is right for directory walking but meant all 117 toolkit pages were absent from sitemap.xml — the largest content section on the site. Merges in listValidIntegrationLinks() from app/_lib/toolkit-static-params.ts, the same enumeration the integrations index uses, and dedupes against the authored partner pages the disk walk already finds. Co-Authored-By: Claude Opus 5 (1M context) --- app/layout.tsx | 21 ++++----- app/sitemap.ts | 63 ++++++++++++++++++++++++-- tests/sitemap.test.ts | 8 ++++ toolkit-docs-generator/ARCHITECTURE.md | 7 +-- 4 files changed, 81 insertions(+), 18 deletions(-) diff --git a/app/layout.tsx b/app/layout.tsx index 2bd3723f7..421c1e73a 100644 --- a/app/layout.tsx +++ b/app/layout.tsx @@ -10,7 +10,6 @@ import { TranslationBanner } from "@/app/_components/translation-banner"; import "@/app/globals.css"; import { Discord, Github } from "@arcadeai/design-system"; import { GoogleTagManager } from "@next/third-parties/google"; -import { headers } from "next/headers"; import Link from "next/link"; import Script from "next/script"; import { Head } from "nextra/components"; @@ -22,8 +21,6 @@ import { Footer as NextraFooter, } from "nextra-theme-docs"; -const REGEX_LOCALE = /^\/([a-z]{2}(?:-[A-Z]{2})?)(?:\/|$)/; - /** * Nextra's active-state detection only checks `item.route`, never `item.href`. * Toolkit sidebar entries use `href` (required so Nextra doesn't fail validation @@ -94,19 +91,21 @@ export function generateMetadata() { }; } -function getLocaleFromPathname(pathname: string): string { - const localeMatch = pathname.match(REGEX_LOCALE); - return localeMatch?.[1] || "en"; -} - export default async function RootLayout({ children, }: { children: React.ReactNode; }) { - const headersList = await headers(); - const pathname = headersList.get("x-pathname") || "/"; - const lang = getLocaleFromPathname(pathname); + // proxy.ts redirects every request to a "/en/..." path — "es" and + // "pt-BR" routes bounce to their "/en" equivalent and unlocaled routes + // pick up "/en" from getPreferredLocale, which is hardcoded to return + // "en" unconditionally. So this layout only ever renders under "/en", + // and reading the locale here can be a constant instead of a header + // lookup. Awaiting headers() in the root layout previously forced the + // entire route tree into dynamic rendering. Restoring real i18n means + // moving this layout under an `app/[lang]/` route segment so the + // locale comes from routing params, not a request header. + const lang = "en"; const dictionary = await getDictionary(lang); const rawPageMap = await getPageMap(`/${lang}`); diff --git a/app/sitemap.ts b/app/sitemap.ts index 45b6615cf..54a2442e0 100644 --- a/app/sitemap.ts +++ b/app/sitemap.ts @@ -1,6 +1,7 @@ import fs from "node:fs/promises"; import path from "node:path"; import type { MetadataRoute } from "next"; +import { listValidIntegrationLinks } from "./_lib/toolkit-static-params"; const SITE_URL = process.env.SITE_URL ?? "https://docs.arcade.dev"; const NORMALIZED_SITE_URL = SITE_URL.replace(/\/+$/, ""); @@ -43,12 +44,66 @@ async function collectRoutes(dir: string): Promise { return entries; } +/** + * `[toolkitId]` directories are skipped above because they aren't literal + * URLs — but `listValidIntegrationLinks()` (the same enumeration the + * integrations index page uses) already resolves every toolkit that dynamic + * route serves, plus a handful of authored static partner pages living + * alongside it. Skip any link the directory walk already found (the static + * ones) so it isn't listed twice. + */ +async function collectToolkitRoutes( + existingPaths: Set +): Promise { + const links = await listValidIntegrationLinks(); + const entries: MetadataRoute.Sitemap = []; + const categoryPageMtime = new Map(); + + for (const link of links) { + if (existingPaths.has(link)) { + continue; + } + + const category = link.split("/").at(-2) ?? ""; + let mtime = categoryPageMtime.get(category); + if (!mtime) { + const pageFile = path.join( + APP_DIR, + "en", + "resources", + "integrations", + category, + "[toolkitId]", + "page.mdx" + ); + mtime = (await fs.stat(pageFile)).mtime; + categoryPageMtime.set(category, mtime); + } + + entries.push({ + url: `${NORMALIZED_SITE_URL}${link}`, + lastModified: mtime, + changeFrequency: "weekly", + priority: 0.7, + }); + } + + return entries; +} + export default function sitemap(): Promise { if (!cachedRoutes) { - cachedRoutes = collectRoutes(APP_DIR).then((routes) => { - routes.sort((a, b) => a.url.localeCompare(b.url)); - return routes; - }); + cachedRoutes = (async () => { + const routes = await collectRoutes(APP_DIR); + const existingPaths = new Set( + routes.map((route) => route.url.slice(NORMALIZED_SITE_URL.length)) + ); + const toolkitRoutes = await collectToolkitRoutes(existingPaths); + + const allRoutes = [...routes, ...toolkitRoutes]; + allRoutes.sort((a, b) => a.url.localeCompare(b.url)); + return allRoutes; + })(); } return cachedRoutes; diff --git a/tests/sitemap.test.ts b/tests/sitemap.test.ts index 854fa6234..d8daf65cc 100644 --- a/tests/sitemap.test.ts +++ b/tests/sitemap.test.ts @@ -22,6 +22,14 @@ test("sitemap lists expected URLs", async () => { // Known page should be present expect(urls).toContain("https://example.test/en/references/changelog"); + // Generated toolkit pages (served by the `[toolkitId]` dynamic route, + // which the directory walk above can't see) must still make it into the + // sitemap. This fails if the toolkit-route merge in app/sitemap.ts is + // reverted. + expect(urls).toContain( + "https://example.test/en/resources/integrations/development/github" + ); + // No duplicates const duplicates = urls.filter( (url, index, arr) => arr.indexOf(url) !== index diff --git a/toolkit-docs-generator/ARCHITECTURE.md b/toolkit-docs-generator/ARCHITECTURE.md index 67022ac8c..86654dd08 100644 --- a/toolkit-docs-generator/ARCHITECTURE.md +++ b/toolkit-docs-generator/ARCHITECTURE.md @@ -71,9 +71,10 @@ root `pnpm build` command and compiles the committed files with Next.js. `app/_lib/toolkit-static-params.ts` enumerates routes from `index.json` and the per-toolkit files. `app/_lib/toolkit-data.ts` reads the same files for page -rendering and the `/api/toolkit-data/[toolkitId]` route. The root layout reads -request headers, so Vercel reports the docs routes as dynamic even though the -toolkit parameter set is fixed at build time. +rendering and the `/api/toolkit-data/[toolkitId]` route. The root layout no +longer reads request headers — the locale it needs is a hardcoded constant, +since `proxy.ts` redirects every request to an `/en` path — so Vercel can +statically render the toolkit routes at build time from the committed JSON. ## Search indexing From dce839ebf37836ea8acebfdbe5a788eff12335cc Mon Sep 17 00:00:00 2001 From: Teal Larson Date: Fri, 31 Jul 2026 10:08:41 -0400 Subject: [PATCH 03/17] chore: drop removed build step from workflow flow comment The header comment still listed "Build the toolkit docs generator" as step 1 after that step was removed. Renumber the remaining three. Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/generate-toolkit-docs.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/generate-toolkit-docs.yml b/.github/workflows/generate-toolkit-docs.yml index 605b29fbd..e57a3484a 100644 --- a/.github/workflows/generate-toolkit-docs.yml +++ b/.github/workflows/generate-toolkit-docs.yml @@ -1,10 +1,9 @@ name: Generate toolkit docs # Description: Generate toolkit JSON from Engine API, then sync sidebar navigation. # Flow: -# 1) Build the toolkit docs generator -# 2) Generate toolkit JSON into toolkit-docs-generator/data/toolkits -# 3) Sync integrations sidebar _meta.tsx from toolkit-docs-generator/data/toolkits -# 4) Create or update a PR if changes were produced +# 1) Generate toolkit JSON into toolkit-docs-generator/data/toolkits +# 2) Sync integrations sidebar _meta.tsx from toolkit-docs-generator/data/toolkits +# 3) Create or update a PR if changes were produced on: repository_dispatch: From 072b21b467f3674e3158356bd8915ce9ec4d0006 Mon Sep 17 00:00:00 2001 From: Teal Larson Date: Fri, 31 Jul 2026 13:23:35 -0400 Subject: [PATCH 04/17] refactor: make the toolkit data contract single and enforced MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Eliminates the duplication that lets these subsystems drift. Redirects were a 909-line array inside next.config.ts, so three consumers regex-parsed the config as text — one of them carrying a second "reversed" regex that existed only because a human might write {destination, source} instead of {source, destination}, a problem that only exists when you parse text instead of importing data. The array now lives in a typed redirects.ts and every consumer imports it. Two further consumers that also parsed the config as text (scripts/update-internal-links.ts and tests/integration-index-links.test.ts) would have silently found zero redirects, so they move to the import too. --auto-fix now appends to the data file, and its six scattered "Auto-added redirects" marker blocks collapse to one append point. The resolved array is byte-identical: 156 entries, same order. check-redirects-utils.ts had 425 lines of tests but was imported by nothing except its own test file, while the code that actually runs — the pre-commit hook's scripts/check-redirects.ts — kept private copies of the same six helpers. The tests guarded a copy while the shipping implementation was untested. The module moves to scripts/lib/ and the shipping script now imports it. Toolkit primitives (data dir, toKebabCase, normalizeToolkitId, the category list, the *Api heuristic, docsLink→slug) existed in 2-7 copies, one pair carrying a "must stay in sync" comment. They collapse into toolkit-docs-generator/src/shared/, which both halves can import. All seven data-dir consumers now honor TOOLKIT_DATA_DIR; previously only two did. The Node-only path resolution lives in its own module because client components reach the primitives through the integrations index, and a node:* import anywhere in that graph fails the webpack browser build. The consumer-side contract was a four-field duck-check that never verified tools was an array, followed by an unchecked cast — while toToolkitSummary immediately calls data.tools.map(). The generator's Zod schemas are now the single shared contract, the 522-line hand-written mirror is z.infer, and zod moves to dependencies because it enters the app's runtime path. Corruption is now loud and absence stays quiet: three catch blocks treated missing, unparseable, and schema-invalid identically, so a malformed file from the nightly PR silently dropped a toolkit and 404'd. Unparseable or invalid now throws with the file path and the Zod issues, failing the build. An unrecognized category throws instead of being coerced to "others", which had no route directory and would have made every toolkit in a new category a clickable card pointing at a 404. One cache()-wrapped loader replaces 11 full passes over the data directory per build, and removes the scan-every-file miss path that a burst of unknown IDs could otherwise trigger at request time. Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/check-redirects.yml | 8 +- .husky/pre-commit | 18 +- app/_components/toolkit-docs/types/index.ts | 349 ++----- app/_lib/integration-catalog.ts | 3 +- app/_lib/integration-index.ts | 3 +- app/_lib/toolkit-data.ts | 248 +++-- app/_lib/toolkit-slug.ts | 53 - app/_lib/toolkit-static-params.ts | 137 +-- .../integrations/_lib/toolkit-docs-page.tsx | 24 +- .../integrations/components/filter-params.ts | 43 +- next.config.ts | 923 +----------------- package.json | 6 +- pnpm-lock.yaml | 6 +- redirects.ts | 875 +++++++++++++++++ scripts/check-redirects.ts | 399 ++------ scripts/generate-llmstxt.ts | 38 +- .../lib}/check-redirects-utils.ts | 88 +- scripts/update-internal-links.ts | 65 +- tests/integration-category-routes.test.ts | 58 ++ tests/integration-index-links.test.ts | 20 +- .../scripts/check-redirects-utils.test.ts | 76 +- tests/sitemap.test.ts | 10 +- tests/toolkit-data-cache.test.ts | 118 +++ tests/toolkit-data-parity.test.ts | 58 ++ .../scripts/check-stale-summaries.ts | 7 +- .../scripts/report-tool-metadata.ts | 10 +- .../scripts/sync-toolkit-sidebar.ts | 72 +- .../scripts/validate-merge.ts | 27 +- .../src/merger/data-merger.ts | 15 +- .../src/shared/toolkit-data-dir.ts | 35 + .../src/shared/toolkit-primitives.ts | 126 +++ .../src/shared/toolkit-schemas.ts | 420 ++++++++ .../src/sources/design-system-metadata.ts | 15 +- .../src/sources/toolkit-data-source.ts | 4 +- toolkit-docs-generator/src/types/index.ts | 410 +------- .../tests/app-lib/toolkit-data.test.ts | 10 + .../tests/app-lib/toolkit-slug.test.ts | 2 +- .../app-lib/toolkit-static-params.test.ts | 41 +- .../scripts/sync-toolkit-sidebar.test.ts | 14 + 39 files changed, 2344 insertions(+), 2490 deletions(-) create mode 100644 redirects.ts rename {toolkit-docs-generator/scripts => scripts/lib}/check-redirects-utils.ts (76%) create mode 100644 tests/integration-category-routes.test.ts rename {toolkit-docs-generator/tests => tests}/scripts/check-redirects-utils.test.ts (86%) create mode 100644 tests/toolkit-data-cache.test.ts create mode 100644 tests/toolkit-data-parity.test.ts create mode 100644 toolkit-docs-generator/src/shared/toolkit-data-dir.ts create mode 100644 toolkit-docs-generator/src/shared/toolkit-primitives.ts create mode 100644 toolkit-docs-generator/src/shared/toolkit-schemas.ts diff --git a/.github/workflows/check-redirects.yml b/.github/workflows/check-redirects.yml index 1482842be..ccaa1bef8 100644 --- a/.github/workflows/check-redirects.yml +++ b/.github/workflows/check-redirects.yml @@ -6,7 +6,7 @@ on: paths: - "app/**/*.md" - "app/**/*.mdx" - - "next.config.ts" + - "redirects.ts" permissions: contents: read @@ -52,7 +52,7 @@ jobs: // Extract the missing redirects and suggestions from output const body = `## 🔗 Missing Redirects Detected - This PR deletes markdown files that don't have corresponding redirects in \`next.config.ts\`. + This PR deletes markdown files that don't have corresponding redirects in \`redirects.ts\`. When you delete a page, you must add a redirect to prevent broken links for users who have bookmarked the old URL. @@ -67,8 +67,8 @@ jobs: ### How to fix - 1. Open \`next.config.ts\` - 2. Find the \`redirects()\` function + 1. Open \`redirects.ts\` + 2. Find the \`redirects\` array 3. Add redirect entries for each deleted file (see suggestions above) 4. Push the changes diff --git a/.husky/pre-commit b/.husky/pre-commit index 4ffa0437b..452307eb9 100755 --- a/.husky/pre-commit +++ b/.husky/pre-commit @@ -61,18 +61,18 @@ if [ -n "$DELETED_PAGES" ] || [ -n "$RENAMED_PAGES" ]; then echo "🔗 Detected deleted/renamed page(s), checking for redirects..." # Run the TypeScript redirect checker with auto-fix (only checks staged changes) - # This will add redirect entries to next.config.ts if missing + # This will add redirect entries to redirects.ts if missing if ! pnpm check-redirects --auto-fix --staged-only 2>&1; then - # Stage next.config.ts if it was modified - if git diff --name-only next.config.ts 2>/dev/null | grep -q "next.config.ts"; then - git add next.config.ts + # Stage redirects.ts if it was modified + if git diff --name-only redirects.ts 2>/dev/null | grep -q "redirects.ts"; then + git add redirects.ts echo "" - echo "📝 Redirect entries added to next.config.ts and staged." + echo "📝 Redirect entries added to redirects.ts and staged." fi echo "" # Check if there are placeholders vs other errors - if grep -q "REPLACE_WITH_NEW_PATH" next.config.ts 2>/dev/null; then - echo "❌ Commit blocked: Please update the placeholder destinations in next.config.ts" + if grep -q "REPLACE_WITH_NEW_PATH" redirects.ts 2>/dev/null; then + echo "❌ Commit blocked: Please update the placeholder destinations in redirects.ts" echo " Search for 'REPLACE_WITH_NEW_PATH' and provide actual redirect paths." else echo "❌ Commit blocked: Please fix the redirect issues shown above." @@ -82,8 +82,8 @@ if [ -n "$DELETED_PAGES" ] || [ -n "$RENAMED_PAGES" ]; then fi # --- Update Internal Links (when redirects are added) --- -# If next.config.ts is staged, update any internal links pointing to redirected paths -if git diff --cached --name-only | grep -q "next.config.ts"; then +# If redirects.ts is staged, update any internal links pointing to redirected paths +if git diff --cached --name-only | grep -q "redirects.ts"; then echo "🔗 Updating internal links for new redirects..." # Capture files already modified in working directory BEFORE running update-links diff --git a/app/_components/toolkit-docs/types/index.ts b/app/_components/toolkit-docs/types/index.ts index f091219b6..ad56a5be2 100644 --- a/app/_components/toolkit-docs/types/index.ts +++ b/app/_components/toolkit-docs/types/index.ts @@ -1,214 +1,97 @@ /** * Type definitions for toolkit documentation MDX components * - * These types are designed for React component props and are compatible - * with the JSON data structure from toolkit-docs-generator. + * The data-shape types below (everything through `ToolkitData`) are + * `z.infer` types derived from the Zod schemas in + * toolkit-docs-generator/src/shared/toolkit-schemas.ts — the same schemas + * the generator validates its JSON output against. Only `import type` is + * used here: this file is imported by client components, and a runtime + * import of the Zod schemas would ship the Zod library to the browser for + * no benefit (the client only needs the types, never runs `.parse()`). + * + * Everything below `ToolkitData` (ToolSummary, ToolkitSummary, and the + * component prop types) is app-specific shaping of that data for React + * props and has no generator equivalent. */ +import type { z } from "zod"; +import type { + DocumentationChunkSchema, + ExampleParameterValueSchema, + MergedToolkitAuthSchema, + MergedToolkitMetadataSchema, + MergedToolkitSchema, + MergedToolSchema, + SecretTypeSchema, + ToolAuthSchema, + ToolCodeExampleSchema, + ToolkitAuthTypeSchema, + ToolkitCategorySchema, + ToolkitTypeSchema, + ToolMetadataBehaviorSchema, + ToolMetadataClassificationSchema, + ToolMetadataSchema, + ToolOutputSchema, + ToolParameterSchema, + ToolSecretSchema, +} from "@/toolkit-docs-generator/src/shared/toolkit-schemas"; // ============================================================================ // Documentation Chunk Types // ============================================================================ -/** - * Type of documentation chunk content - */ -export type DocumentationChunkType = - | "callout" - | "markdown" - | "code" - | "warning" - | "info" - | "tip" - | "section"; - -/** - * Location where the chunk should be injected - */ -export type DocumentationChunkLocation = - | "header" - | "description" - | "parameters" - | "auth" - | "secrets" - | "output" - | "footer" - | "before_available_tools" - | "after_available_tools" - | "custom_section"; - -/** - * Position relative to the location - */ -export type DocumentationChunkPosition = "before" | "after" | "replace"; - -/** - * Callout variant for styling - */ -export type DocumentationChunkVariant = - | "default" - | "destructive" - | "warning" - | "info" - | "success"; - -/** - * A documentation chunk represents custom content to inject into docs - */ -export type DocumentationChunk = { - /** Type of content */ - type: DocumentationChunkType; - /** Where to inject the content */ - location: DocumentationChunkLocation; - /** Position relative to location */ - position: DocumentationChunkPosition; - /** The actual content (markdown string) */ - content: string; - /** Optional title for callouts */ - title?: string; - /** Optional variant for styling */ - variant?: DocumentationChunkVariant; - /** Optional section header for sidebar navigation (e.g., "## Auth Setup") */ - header?: string; - /** Optional priority for ordering (lower = earlier, default = 100) */ - priority?: number; -}; +export type DocumentationChunk = z.infer; +export type DocumentationChunkType = DocumentationChunk["type"]; +export type DocumentationChunkLocation = DocumentationChunk["location"]; +export type DocumentationChunkPosition = DocumentationChunk["position"]; +export type DocumentationChunkVariant = NonNullable< + DocumentationChunk["variant"] +>; // ============================================================================ // Tool Parameter Types // ============================================================================ -/** - * Tool parameter definition - */ -export type ToolParameter = { - /** Parameter name */ - name: string; - /** Parameter type (string, integer, boolean, array, object) */ - type: string; - /** For array types, the inner element type */ - innerType?: string; - /** Whether the parameter is required */ - required: boolean; - /** Parameter description */ - description: string | null; - /** Enum values if this is an enum parameter */ - enum: string[] | null; - /** Whether the parameter can be inferred by an LLM */ - inferrable?: boolean; - /** Default value if not provided */ - default?: unknown; -}; +export type ToolParameter = z.infer; // ============================================================================ // Tool Auth Types // ============================================================================ -/** - * Tool-level authentication requirements - */ -export type ToolAuth = { - /** Auth provider ID (e.g., "github", "google") */ - providerId: string | null; - /** Provider type (e.g., "oauth2", "api_key") */ - providerType: string; - /** Required OAuth scopes for this specific tool */ - scopes: string[]; -}; +export type ToolAuth = z.infer; // ============================================================================ // Tool Output Types // ============================================================================ -/** - * Tool output schema - */ -export type ToolOutput = { - /** Output type (object, array, string, etc.) */ - type: string; - /** Output description */ - description: string | null; -}; +export type ToolOutput = z.infer; // ============================================================================ // Tool Secrets Types // ============================================================================ -export type SecretType = - | "api_key" - | "token" - | "client_secret" - | "webhook_secret" - | "private_key" - | "password" - | "unknown"; - -export type ToolSecret = { - /** Secret name */ - name: string; - /** Secret type classification */ - type: SecretType; -}; +export type SecretType = z.infer; +export type ToolSecret = z.infer; // ============================================================================ // Code Example Types // ============================================================================ -/** - * Parameter value with type information for code generation - */ -export type ExampleParameterValue = { - /** The example value to use in generated code */ - value: unknown; - /** Parameter type for proper serialization */ - type: "string" | "integer" | "boolean" | "array" | "object"; - /** Whether this parameter is required */ - required: boolean; -}; - -/** - * Tool code example configuration - * Used to generate Python/JavaScript example code - */ -export type ToolCodeExample = { - /** Full tool name (e.g., "Github.SetStarred") */ - toolName: string; - /** Parameter values with type info */ - parameters: Record; - /** Whether this tool requires user authorization */ - requiresAuth: boolean; - /** Auth provider ID if auth is required */ - authProvider?: string; - /** Optional tab label for the code example */ - tabLabel?: string; -}; +export type ExampleParameterValue = z.infer; +export type ToolCodeExample = z.infer; // ============================================================================ // Tool Metadata Types // ============================================================================ -export type ToolMetadataClassification = { - serviceDomains: string[]; -}; - -export type ToolMetadataBehavior = { - operations: string[]; - readOnly?: boolean; - destructive?: boolean; - idempotent?: boolean; - openWorld?: boolean; -}; +export type ToolMetadataClassification = z.infer< + typeof ToolMetadataClassificationSchema +>; +export type ToolMetadataBehavior = z.infer; -export type BehaviorFlagKey = - | "readOnly" - | "destructive" - | "idempotent" - | "openWorld"; +/** UI-only helper: the boolean behavior flags, excluding `operations`. */ +export type BehaviorFlagKey = Exclude; -export type ToolMetadata = { - classification: ToolMetadataClassification; - behavior: ToolMetadataBehavior; - extras?: Record | null; -}; +export type ToolMetadata = z.infer; // ============================================================================ // Tool Definition Types @@ -217,32 +100,7 @@ export type ToolMetadata = { /** * Complete tool definition with all documentation data */ -export type ToolDefinition = { - /** Tool name (e.g., "CreateIssue") */ - name: string; - /** Qualified name (e.g., "Github.CreateIssue") */ - qualifiedName: string; - /** Fully qualified name with version (e.g., "Github.CreateIssue@1.0.0") */ - fullyQualifiedName: string; - /** Tool description */ - description: string | null; - /** Tool parameters */ - parameters: ToolParameter[]; - /** Tool authentication requirements */ - auth: ToolAuth | null; - /** Required secrets */ - secrets: string[]; - /** Classified secrets (LLM-generated) */ - secretsInfo?: ToolSecret[]; - /** Tool output schema */ - output: ToolOutput | null; - /** Per-tool metadata from Engine API */ - metadata?: ToolMetadata | null; - /** Custom documentation chunks for this tool */ - documentationChunks: DocumentationChunk[]; - /** Generated code example configuration */ - codeExample?: ToolCodeExample; -}; +export type ToolDefinition = z.infer; /** * A tool with its heavy detail fields stripped — everything needed to render the @@ -258,72 +116,16 @@ export type ToolSummary = Omit< // Toolkit Metadata Types // ============================================================================ -/** - * Toolkit category for navigation grouping - */ -export type ToolkitCategory = - | "productivity" - | "social" - | "development" - | "entertainment" - | "search" - | "payments" - | "sales" - | "databases" - | "customer-support"; - -/** - * Toolkit type classification - */ -export type ToolkitType = - | "arcade" - | "arcade_starter" - | "verified" - | "community" - | "auth"; - -/** - * Toolkit metadata from Design System - */ -export type ToolkitMetadata = { - /** Category for navigation grouping */ - category: ToolkitCategory; - /** Icon URL */ - iconUrl: string; - /** Whether this toolkit requires BYOC (Bring Your Own Credentials) */ - isBYOC: boolean; - /** Whether this is a Pro feature */ - isPro: boolean; - /** Toolkit type classification */ - type: ToolkitType; - /** Link to documentation */ - docsLink: string; - /** Whether this toolkit is coming soon */ - isComingSoon?: boolean; - /** Whether this toolkit is hidden */ - isHidden?: boolean; -}; +export type ToolkitCategory = z.infer; +export type ToolkitType = z.infer; +export type ToolkitMetadata = z.infer; // ============================================================================ // Toolkit Auth Types // ============================================================================ -/** - * Toolkit-level authentication type - */ -export type ToolkitAuthType = "oauth2" | "api_key" | "mixed" | "none"; - -/** - * Toolkit-level authentication summary - */ -export type ToolkitAuth = { - /** Auth type */ - type: ToolkitAuthType; - /** Auth provider ID */ - providerId: string | null; - /** Union of all scopes required by tools in this toolkit */ - allScopes: string[]; -}; +export type ToolkitAuthType = z.infer; +export type ToolkitAuth = z.infer; // ============================================================================ // Complete Toolkit Data Type @@ -333,38 +135,7 @@ export type ToolkitAuth = { * Complete toolkit data structure for rendering documentation * This is the main type consumed by the ToolkitPage component */ -export type ToolkitData = { - /** Unique toolkit ID (e.g., "Github") */ - id: string; - /** Human-readable label (e.g., "GitHub") */ - label: string; - /** Toolkit version (e.g., "1.0.0") */ - version: string; - /** Toolkit description */ - description: string | null; - /** LLM-generated summary */ - summary?: string; - /** Metadata from Design System */ - metadata: ToolkitMetadata; - /** Authentication requirements */ - auth: ToolkitAuth | null; - /** All tools in this toolkit */ - tools: ToolDefinition[]; - /** Toolkit-level documentation chunks */ - documentationChunks?: DocumentationChunk[]; - /** Custom imports for MDX */ - customImports: string[]; - /** - * Sub-pages that exist for this toolkit. - * Each entry is either a string (legacy slug) or a rich object with - * { type, content, relativePath } for inline MDX sub-page content. - */ - subPages: (string | Record)[]; - /** Optional pip package name override */ - pipPackageName?: string; - /** Generation timestamp */ - generatedAt?: string; -}; +export type ToolkitData = z.infer; /** * Toolkit data with each tool's heavy detail fields stripped. This is what the diff --git a/app/_lib/integration-catalog.ts b/app/_lib/integration-catalog.ts index f92cc7d31..110f466f4 100644 --- a/app/_lib/integration-catalog.ts +++ b/app/_lib/integration-catalog.ts @@ -1,8 +1,9 @@ import type { Toolkit } from "@arcadeai/design-system"; import { TOOLKITS } from "@arcadeai/design-system/metadata/toolkits"; import { PARTNER_TOOLKITS } from "@/app/_data/partner-toolkits"; +import { normalizeToolkitId } from "@/toolkit-docs-generator/src/shared/toolkit-primitives"; import { readToolkitData } from "./toolkit-data"; -import { normalizeToolkitId, type ToolkitWithDocsLink } from "./toolkit-slug"; +import type { ToolkitWithDocsLink } from "./toolkit-slug"; const getToolkitDocsLink = (toolkit: Toolkit): string | undefined => { if ("docsLink" in toolkit) { diff --git a/app/_lib/integration-index.ts b/app/_lib/integration-index.ts index 12d9cacbf..0e062a54a 100644 --- a/app/_lib/integration-index.ts +++ b/app/_lib/integration-index.ts @@ -1,4 +1,5 @@ -import { getToolkitSlug, type ToolkitWithDocsLink } from "./toolkit-slug"; +import { getToolkitSlug } from "@/toolkit-docs-generator/src/shared/toolkit-primitives"; +import type { ToolkitWithDocsLink } from "./toolkit-slug"; const INTEGRATIONS_BASE = "/en/resources/integrations"; diff --git a/app/_lib/toolkit-data.ts b/app/_lib/toolkit-data.ts index f0299c74e..f724b03b2 100644 --- a/app/_lib/toolkit-data.ts +++ b/app/_lib/toolkit-data.ts @@ -1,11 +1,22 @@ import { readdir, readFile } from "node:fs/promises"; import { join } from "node:path"; +import { cache } from "react"; +import type { z } from "zod"; import type { ToolkitData, ToolkitSummary, ToolSummary, } from "@/app/_components/toolkit-docs/types"; -import { getToolkitSlug, normalizeToolkitId } from "./toolkit-slug"; +import { resolveToolkitDataDir } from "@/toolkit-docs-generator/src/shared/toolkit-data-dir"; +import { + getToolkitSlug, + normalizeToolkitId, +} from "@/toolkit-docs-generator/src/shared/toolkit-primitives"; +import { + MergedToolkitSchema, + type ToolkitIndexEntrySchema, + type ToolkitIndexSchema, +} from "@/toolkit-docs-generator/src/shared/toolkit-schemas"; /** * Strip each tool's heavy fields (parameters, output, codeExample) so the @@ -35,87 +46,156 @@ export function toToolkitSummary(data: ToolkitData): ToolkitSummary { }; } -export type ToolkitIndexEntry = { - id: string; - label: string; - version: string; - category: string; - type?: string; - toolCount: number; - authType: string; -}; - -export type ToolkitIndex = { - generatedAt: string; - version: string; - toolkits: ToolkitIndexEntry[]; -}; +export type ToolkitIndexEntry = z.infer; +export type ToolkitIndex = z.infer; type ToolkitDataOptions = { dataDir?: string; }; -const DEFAULT_DATA_DIR = join( - process.cwd(), - "toolkit-docs-generator", - "data", - "toolkits" -); - const resolveDataDir = (options?: ToolkitDataOptions): string => - options?.dataDir ?? process.env.TOOLKIT_DATA_DIR ?? DEFAULT_DATA_DIR; - -const isValidToolkitData = (parsed: unknown): parsed is ToolkitData => - typeof parsed === "object" && - parsed !== null && - "id" in parsed && - ("label" in parsed || "name" in parsed) && - "metadata" in parsed && - typeof (parsed as Record).metadata === "object" && - (parsed as Record).metadata !== null; - -const readToolkitFile = async ( + resolveToolkitDataDir(options?.dataDir); + +const isEnoent = (error: unknown): boolean => + error instanceof Error && (error as NodeJS.ErrnoException).code === "ENOENT"; + +const describeError = (error: unknown): string => + error instanceof Error ? error.message : String(error); + +/** + * Read and validate a single merged-toolkit JSON file. + * + * A missing file is a legitimate, quiet outcome (`null`) — not every toolkit + * has generated docs yet. Anything else wrong with the file — unreadable, + * not valid JSON, or valid JSON that doesn't match `MergedToolkitSchema` — is + * corruption, not absence, and throws with the file path and the underlying + * error so a bad nightly-generated file fails `next build` loudly instead of + * quietly dropping the toolkit from the site. Mirrors the read/parse/schema + * split in toolkit-docs-generator/src/generator/output-verifier.ts. + */ +export const readToolkitFile = async ( filePath: string ): Promise => { + let content: string; try { - const content = await readFile(filePath, "utf-8"); - const parsed: unknown = JSON.parse(content); - return isValidToolkitData(parsed) ? (parsed as ToolkitData) : null; - } catch { - return null; + content = await readFile(filePath, "utf-8"); + } catch (error) { + if (isEnoent(error)) { + return null; + } + throw new Error( + `Failed to read toolkit file ${filePath}: ${describeError(error)}` + ); + } + + let parsed: unknown; + try { + parsed = JSON.parse(content); + } catch (error) { + throw new Error( + `Invalid JSON in toolkit file ${filePath}: ${describeError(error)}` + ); + } + + const result = MergedToolkitSchema.safeParse(parsed); + if (!result.success) { + throw new Error( + `Invalid toolkit schema in ${filePath}: ${result.error.message}` + ); } + + return result.data; }; -const findToolkitDataBySlug = async ( - dataDir: string, - slug: string -): Promise => { - const entries = await readdir(dataDir); - const slugKey = slug.toLowerCase(); +/** + * Every toolkit's data, indexed two ways: by the normalized id its filename + * is derived from (the common case — an id-shaped lookup), and by its docs + * slug (a hand-authored `docsLink` can diverge from the id, e.g. a route + * reached by "posthog-api" for a file whose id normalizes differently). + * `readToolkitData` below tries the id map first, then the slug map, mirroring + * the direct-file-then-scan order the old implementation used. + */ +type ToolkitDataMap = { + byNormalizedId: Map; + bySlug: Map; +}; + +/** + * One process-wide load per data directory. Keyed by directory (not a single + * flat variable) because tests point `TOOLKIT_DATA_DIR` at scratch fixtures + * and must not see another test's cached data. + * + * A failed load (a corrupt file — see readToolkitFile) is kept in this map + * rather than retried: the underlying files are static build output that + * only change on a new deploy, so a bad file stays bad for the rest of this + * process's life, and re-scanning 21.6 MB on every subsequent lookup hoping + * it healed itself would only add cost without ever succeeding. + */ +const loadsByDataDir = new Map>(); + +const loadAllToolkitDataUncached = async ( + dataDir: string +): Promise => { + let entries: string[]; + try { + entries = await readdir(dataDir); + } catch (error) { + throw new Error( + `Failed to read toolkit data directory ${dataDir}: ${describeError(error)}` + ); + } + + const byNormalizedId = new Map(); + const bySlug = new Map(); for (const entry of entries) { if (!entry.endsWith(".json") || entry === "index.json") { continue; } + // Throws on a corrupt file (see readToolkitFile) — a malformed file here + // is never legitimately "absent", so it fails the build/request loudly + // rather than being dropped from the map. const data = await readToolkitFile(join(dataDir, entry)); if (!data) { continue; } - const candidateSlug = getToolkitSlug({ + byNormalizedId.set(normalizeToolkitId(data.id), data); + const slug = getToolkitSlug({ id: data.id, docsLink: data.metadata?.docsLink, - }).toLowerCase(); - - if (candidateSlug === slugKey) { - return data; - } + }); + bySlug.set(slug.toLowerCase(), data); } - return null; + return { byNormalizedId, bySlug }; }; +/** + * Load every toolkit's data from `dataDir` into one shared map, read once per + * process rather than once per caller. + * + * Wrapped in React's `cache()` so, when a live cache scope exists (build-time + * static generation, a Route Handler, a Server Component render), concurrent + * callers within that same scope share one in-flight read rather than each + * independently reading the directory. `cache()` is a no-op outside a cache + * scope (Vitest, plain scripts) — see its implementation in + * react/cjs/react.react-server.development.js — so `loadsByDataDir` is the + * mechanism that actually guarantees one read per directory everywhere, with + * `cache()` as the layer that also dedupes concurrent build-time work. + */ +export const loadAllToolkitData = cache( + async (dataDir: string): Promise => { + let promise = loadsByDataDir.get(dataDir); + if (!promise) { + promise = loadAllToolkitDataUncached(dataDir); + loadsByDataDir.set(dataDir, promise); + } + return await promise; + } +); + export const readToolkitData = async ( toolkitId: string, options?: ToolkitDataOptions @@ -128,15 +208,14 @@ export const readToolkitData = async ( return null; } - const fileName = `${normalizedId}.json`; const dataDir = resolveDataDir(options); - const filePath = join(dataDir, fileName); - const direct = await readToolkitFile(filePath); - if (direct) { - return direct; - } + const { byNormalizedId, bySlug } = await loadAllToolkitData(dataDir); - return await findToolkitDataBySlug(dataDir, toolkitId); + return ( + byNormalizedId.get(normalizedId) ?? + bySlug.get(toolkitId.toLowerCase()) ?? + null + ); }; export const readToolkitIndex = async ( @@ -144,22 +223,45 @@ export const readToolkitIndex = async ( ): Promise => { const filePath = join(resolveDataDir(options), "index.json"); + let content: string; try { - const content = await readFile(filePath, "utf-8"); - const parsed: unknown = JSON.parse(content); - - // Basic runtime validation - ensure it's an object with required fields - if ( - typeof parsed !== "object" || - parsed === null || - !("toolkits" in parsed) || - !Array.isArray((parsed as { toolkits: unknown }).toolkits) - ) { + content = await readFile(filePath, "utf-8"); + } catch (error) { + if (isEnoent(error)) { return null; } + throw new Error( + `Failed to read toolkit index ${filePath}: ${describeError(error)}` + ); + } - return parsed as ToolkitIndex; - } catch { - return null; + let parsed: unknown; + try { + parsed = JSON.parse(content); + } catch (error) { + throw new Error( + `Invalid JSON in toolkit index ${filePath}: ${describeError(error)}` + ); } + + // Deliberately looser than a full ToolkitIndexSchema.safeParse: unlike + // per-toolkit data, entries here are only ever used to look up a + // toolkit's id/category, with the toolkit's own JSON file as the real + // source of truth (see resolveToolkitRoute in toolkit-static-params.ts). + // Rejecting the whole index over one entry missing a field the callers + // don't read would cost every route on the site, not just one page. But + // the file as a whole not even having the shape of an index is + // corruption, not a missing-field nuance, so that still throws. + if ( + typeof parsed !== "object" || + parsed === null || + !("toolkits" in parsed) || + !Array.isArray((parsed as { toolkits: unknown }).toolkits) + ) { + throw new Error( + `Invalid toolkit index shape in ${filePath}: expected an object with a "toolkits" array.` + ); + } + + return parsed as ToolkitIndex; }; diff --git a/app/_lib/toolkit-slug.ts b/app/_lib/toolkit-slug.ts index 5ff34e21d..c99566dc3 100644 --- a/app/_lib/toolkit-slug.ts +++ b/app/_lib/toolkit-slug.ts @@ -1,13 +1,5 @@ import type { Toolkit } from "@arcadeai/design-system"; -const TOOLKIT_ID_NORMALIZER = /[^a-z0-9]+/g; -const CAMEL_BOUNDARY = /([a-z0-9])([A-Z])/g; - -export type ToolkitSlugSource = { - id: string; - docsLink?: string | null; -}; - /** * Toolkit with optional `docsLink` and `isPartner` properties. * The design-system `Toolkit` type doesn't include either field, but some @@ -19,48 +11,3 @@ export type ToolkitWithDocsLink = Toolkit & { docsLink?: string | null; isPartner?: boolean; }; - -/** - * Strip all non-alphanumeric characters and lowercase. - * Used for case-insensitive matching of toolkit IDs to filenames. - */ -export function normalizeToolkitId(value: string): string { - return value.toLowerCase().replace(TOOLKIT_ID_NORMALIZER, ""); -} - -/** - * Convert a CamelCase toolkit ID to a kebab-case URL slug. - * - * Examples: - * PosthogApi → posthog-api - * GoogleCalendar → google-calendar - * E2b → e2b - * HubspotCrmApi → hubspot-crm-api - */ -export function toKebabCase(value: string): string { - return value.replace(CAMEL_BOUNDARY, "$1-$2").toLowerCase(); -} - -const extractSlugFromPath = (path: string): string | null => { - const segments = path.split("/").filter(Boolean); - return segments.at(-1) ?? null; -}; - -export function getToolkitSlug({ id, docsLink }: ToolkitSlugSource): string { - if (docsLink) { - try { - const url = new URL(docsLink); - const slug = extractSlugFromPath(url.pathname); - if (slug) { - return slug; - } - } catch { - const slug = extractSlugFromPath(docsLink); - if (slug) { - return slug; - } - } - } - - return toKebabCase(id); -} diff --git a/app/_lib/toolkit-static-params.ts b/app/_lib/toolkit-static-params.ts index 1e07c5c33..d93bdd8f1 100644 --- a/app/_lib/toolkit-static-params.ts +++ b/app/_lib/toolkit-static-params.ts @@ -1,23 +1,18 @@ -import { readdir, readFile } from "node:fs/promises"; +import { readdir } from "node:fs/promises"; import { join } from "node:path"; import { TOOLKITS as DESIGN_SYSTEM_TOOLKITS } from "@arcadeai/design-system/metadata/toolkits"; -import { readToolkitData, readToolkitIndex } from "./toolkit-data"; -import { getToolkitSlug, normalizeToolkitId } from "./toolkit-slug"; - -export const INTEGRATION_CATEGORIES = [ - "productivity", - "social", - "entertainment", - "development", - "payments", - "search", - "sales", - "databases", - "customer-support", - "others", -] as const; - -export type IntegrationCategory = (typeof INTEGRATION_CATEGORIES)[number]; +import { resolveToolkitDataDir } from "@/toolkit-docs-generator/src/shared/toolkit-data-dir"; +import { + getToolkitSlug, + INTEGRATION_CATEGORIES, + type IntegrationCategory, + normalizeToolkitId, +} from "@/toolkit-docs-generator/src/shared/toolkit-primitives"; +import { + loadAllToolkitData, + readToolkitData, + readToolkitIndex, +} from "./toolkit-data"; export type ToolkitCatalogEntry = { id: string; @@ -42,16 +37,40 @@ const DESIGN_SYSTEM_TOOLKITS_FOR_ROUTES: ToolkitCatalogEntry[] = const loadDesignSystemToolkits = async (): Promise => DESIGN_SYSTEM_TOOLKITS_FOR_ROUTES; +/** + * Normalize a category value read from toolkit data into a routable + * category, or `null` when there is nothing to route by. + * + * `undefined`/`null`/empty string means no category information was + * available at all — typically a fallback source (the design-system catalog + * used only when a toolkit's own JSON file is absent) that simply doesn't + * carry one. That's a quiet "nothing to go on," not corruption: callers skip + * the toolkit rather than invent a page for it. + * + * A non-empty string that isn't one of `INTEGRATION_CATEGORIES`, though, is + * a real value someone set — most likely a new category introduced upstream + * (the Engine / design-system catalog) that this docs site doesn't have a + * route for yet. There is no "others" catch-all to silently absorb it (see + * INTEGRATION_CATEGORIES's doc comment): every toolkit in an unrecognized + * category would otherwise render as a clickable catalog card pointing at a + * route that 404s, with nothing failing the build to surface it. So this + * throws instead. + */ export function normalizeCategory( value: string | null | undefined -): IntegrationCategory { +): IntegrationCategory | null { if (!value) { - return "others"; + return null; } - return INTEGRATION_CATEGORIES.includes(value as IntegrationCategory) - ? (value as IntegrationCategory) - : "others"; + if (INTEGRATION_CATEGORIES.includes(value as IntegrationCategory)) { + return value as IntegrationCategory; + } + + throw new Error( + `Unrecognized integration category "${value}". Expected one of: ${INTEGRATION_CATEGORIES.join(", ")}. ` + + "A new category needs a matching app/en/resources/integrations//[toolkitId] route directory before toolkits can use it." + ); } /** @@ -62,6 +81,11 @@ export function normalizeCategory( * alias (e.g. `development/pagerduty-api` when its category is `customer-support`) * must canonicalize to the one generated, index-linked page instead of * orphaning itself. Mirrors the slug + category logic in `listToolkitRoutes`. + * + * Only called for a toolkit that already has a generated page (it's building + * that page's own canonical tag), so a `null` category here means the page + * exists but its routing information doesn't — an internal inconsistency, + * not a toolkit to quietly skip. That throws too. */ export function getToolkitCanonicalPath(toolkit: { id: string; @@ -69,60 +93,45 @@ export function getToolkitCanonicalPath(toolkit: { docsLink?: string | null; }): string { const category = normalizeCategory(toolkit.category); + if (!category) { + throw new Error( + `Cannot build a canonical path for toolkit "${toolkit.id}": it has no integration category.` + ); + } const slug = getToolkitSlug({ id: toolkit.id, docsLink: toolkit.docsLink }); return `/en/resources/integrations/${category}/${slug}`; } -const DEFAULT_DATA_DIR = join( - process.cwd(), - "toolkit-docs-generator", - "data", - "toolkits" -); - const resolveDataDir = (dataDir?: string): string => - dataDir ?? process.env.TOOLKIT_DATA_DIR ?? DEFAULT_DATA_DIR; + resolveToolkitDataDir(dataDir); const listToolkitRoutesFromDataDir = async (options?: { dataDir?: string; }): Promise => { const dataDir = resolveDataDir(options?.dataDir); - const entries = await readdir(dataDir); + + // loadAllToolkitData validates every file against MergedToolkitSchema and + // throws on a corrupt one (see app/_lib/toolkit-data.ts) — a malformed file + // in this directory listing is never legitimately "absent", so it should + // fail the build rather than be skipped here. + const { byNormalizedId } = await loadAllToolkitData(dataDir); + const unique = new Map(); - for (const entry of entries) { - if (!entry.endsWith(".json") || entry === "index.json") { + for (const data of byNormalizedId.values()) { + if (data.metadata?.isHidden) { continue; } - try { - const content = await readFile(join(dataDir, entry), "utf-8"); - const parsed = JSON.parse(content) as { - id?: string; - metadata?: { - category?: string; - docsLink?: string; - isHidden?: boolean; - }; - }; - - if (!parsed?.id) { - continue; - } - - if (parsed.metadata?.isHidden) { - continue; - } - - const slug = getToolkitSlug({ - id: parsed.id, - docsLink: parsed.metadata?.docsLink, - }); - const category = normalizeCategory(parsed.metadata?.category); - unique.set(slug, { toolkitId: slug, category }); - } catch { - // Ignore malformed toolkit data files. + const slug = getToolkitSlug({ + id: data.id, + docsLink: data.metadata?.docsLink, + }); + const category = normalizeCategory(data.metadata?.category); + if (!category) { + continue; } + unique.set(slug, { toolkitId: slug, category }); } return [...unique.values()]; @@ -158,6 +167,12 @@ const resolveToolkitRoute = async ( const category = normalizeCategory( data?.metadata?.category ?? catalogEntry?.category ?? toolkit.category ); + // No category info anywhere for this toolkit: nothing to route it under. + // Skip it quietly (same treatment as a hidden toolkit) rather than + // fabricate a page under a category that doesn't exist. + if (!category) { + return null; + } return { toolkitId: slug, category }; }; diff --git a/app/en/resources/integrations/_lib/toolkit-docs-page.tsx b/app/en/resources/integrations/_lib/toolkit-docs-page.tsx index c0522c52a..bf585d585 100644 --- a/app/en/resources/integrations/_lib/toolkit-docs-page.tsx +++ b/app/en/resources/integrations/_lib/toolkit-docs-page.tsx @@ -2,33 +2,23 @@ import type { Metadata } from "next"; import { notFound } from "next/navigation"; import { ToolkitPage } from "@/app/_components/toolkit-docs"; import { readToolkitData, toToolkitSummary } from "@/app/_lib/toolkit-data"; -import { normalizeToolkitId } from "@/app/_lib/toolkit-slug"; import { getToolkitCanonicalPath, getToolkitStaticParamsForCategory, - type IntegrationCategory, } from "@/app/_lib/toolkit-static-params"; +import type { IntegrationCategory } from "@/toolkit-docs-generator/src/shared/toolkit-primitives"; type ToolkitDocsParams = { toolkitId: string; }; export function createToolkitDocsPage(category: IntegrationCategory) { - const dataCache = new Map>(); - - const getToolkitData = async (toolkitId: string) => { - const cacheKey = normalizeToolkitId(toolkitId); - const cached = dataCache.get(cacheKey); - if (cached) { - return await cached; - } - - // Pass the original toolkitId (not normalized) so readToolkitData's - // findToolkitDataBySlug fallback can match hyphenated slugs like "posthog-api". - const promise = readToolkitData(toolkitId); - dataCache.set(cacheKey, promise); - return await promise; - }; + // readToolkitData is itself backed by a shared, process-wide cache (see + // loadAllToolkitData in app/_lib/toolkit-data.ts), so generateMetadata and + // Page below calling it separately for the same toolkitId costs one map + // lookup each rather than a second file read — no per-factory cache needed + // here. + const getToolkitData = (toolkitId: string) => readToolkitData(toolkitId); const generateStaticParams = async () => await getToolkitStaticParamsForCategory(category); diff --git a/app/en/resources/integrations/components/filter-params.ts b/app/en/resources/integrations/components/filter-params.ts index 432088986..71e4722c3 100644 --- a/app/en/resources/integrations/components/filter-params.ts +++ b/app/en/resources/integrations/components/filter-params.ts @@ -1,25 +1,26 @@ import type { ToolkitCategory, ToolkitType } from "@arcadeai/design-system"; - -const TOOLKIT_TYPES: readonly ToolkitType[] = [ - "arcade", - "arcade_starter", - "verified", - "community", - "auth", -]; - -const TOOLKIT_CATEGORIES: readonly ToolkitCategory[] = [ - "all", - "productivity", - "social", - "development", - "entertainment", - "search", - "payments", - "sales", - "databases", - "customer-support", -]; +import { CATEGORIES } from "@arcadeai/design-system/metadata/toolkits"; + +// The design system exports CATEGORIES (id + display name) as the runtime +// source of truth for ToolkitCategory. Derive the filter list from it +// instead of hand-copying the ids, so a new category can't silently drop +// out of this list the way it could with a separately maintained array. +const TOOLKIT_CATEGORIES: readonly ToolkitCategory[] = CATEGORIES.map( + (category) => category.id +); + +// ToolkitType has no runtime export from the design system, so this list is +// hand-maintained. `satisfies` turns a missing or extra entry into a +// compile error the next time the union changes, instead of a silent drop. +const TOOLKIT_TYPE_MEMBERSHIP = { + arcade: true, + arcade_starter: true, + verified: true, + community: true, + auth: true, +} satisfies Record; + +const TOOLKIT_TYPES = Object.keys(TOOLKIT_TYPE_MEMBERSHIP) as ToolkitType[]; export const PARAM_CATEGORY = "category"; export const PARAM_TYPE = "type"; diff --git a/next.config.ts b/next.config.ts index b7634c2e8..1c6c928ec 100644 --- a/next.config.ts +++ b/next.config.ts @@ -2,6 +2,7 @@ import type { NextConfig } from "next"; import nextra from "nextra"; import { withLlmsTxt } from "./lib/next-plugin-llmstxt"; import { remarkGlossary } from "./lib/remark-glossary"; +import { redirects } from "./redirects"; // Set up Nextra with its configuration const withNextra = nextra({ @@ -23,915 +24,19 @@ const nextConfig: NextConfig = withLlmsTxt({ })( withNextra({ async redirects() { - return [ - // The toolkit-page breadcrumb links "Resources" -> /resources, which has - // no index page. Send it to the integrations registry instead of 404ing. - { - source: "/:locale/resources", - destination: "/:locale/resources/integrations", - permanent: true, - }, - // The auth provider is "square"; an external/stale link points at the - // old "squareup" slug, which 404s. Send it to the real page. - { - source: "/:locale/references/auth-providers/squareup", - destination: "/:locale/references/auth-providers/square", - permanent: true, - }, - // Dissolved guides/security section - { - source: "/:locale/guides/security/security-research-program", - destination: "/:locale/resources/security-research-program", - permanent: true, - }, - { - source: "/:locale/guides/security/securing-arcade-mcp", - destination: "/:locale/guides/create-tools/secure-your-server", - permanent: true, - }, - { - source: "/:locale/guides/security/secure-your-mcp-server", - destination: - "/:locale/guides/create-tools/secure-your-server/secure-your-mcp-server", - permanent: true, - }, - { - source: "/:locale/guides/security", - destination: "/:locale/guides/create-tools/secure-your-server", - permanent: true, - }, - // Auto-added redirects for deleted pages - { - source: "/:locale/references/mcp/python/transports", - destination: "/:locale/references/mcp/python", - permanent: true, - }, - { - source: "/:locale/references/mcp/python/types", - destination: "/:locale/references/mcp/python", - permanent: true, - }, - // CrewAI custom auth flow redirect to use-arcade-tools - { - source: - "/:locale/get-started/agent-frameworks/crewai/custom-auth-flow", - destination: - "/:locale/get-started/agent-frameworks/crewai/use-arcade-tools", - permanent: true, - }, - // "others" category removed — toolkits moved to proper categories - { - source: "/:locale/resources/integrations/others/:path*", - destination: "/:locale/resources/integrations", - permanent: false, - }, - // Google ADK tutorial consolidation - redirect old URL to new - { - source: - "/:locale/get-started/agent-frameworks/google-adk/use-arcade-tools", - destination: - "/:locale/get-started/agent-frameworks/google-adk/overview", - permanent: true, - }, - // Auto-added redirects for deleted pages - { - source: "/:locale/references/logic-extensions-api", - destination: "/:locale/references/contextual-access-webhook-api", - permanent: true, - }, - // Auto-added redirects for deleted pages - { - source: "/:locale/guides/logic-extensions", - destination: "/:locale/guides/contextual-access", - permanent: true, - }, - { - source: "/:locale/guides/logic-extensions/build-your-own", - destination: "/:locale/guides/contextual-access/build-your-own", - permanent: true, - }, - { - source: "/:locale/guides/logic-extensions/examples", - destination: "/:locale/guides/contextual-access/examples", - permanent: true, - }, - { - source: "/:locale/guides/logic-extensions/how-hooks-work", - destination: "/:locale/guides/contextual-access/how-hooks-work", - permanent: true, - }, - // Auto-added redirects for deleted pages - { - source: "/:locale/resources/integrations/preview", - destination: "/:locale/resources/integrations", - permanent: true, - }, - // Auto-added redirects for deleted pages - { - source: - "/:locale/resources/integrations/customer-support/zendesk/reference", - destination: - "/:locale/resources/integrations/customer-support/zendesk", - permanent: true, - }, - { - source: - "/:locale/resources/integrations/development/firecrawl/reference", - destination: "/:locale/resources/integrations/development/firecrawl", - permanent: true, - }, - { - source: - "/:locale/resources/integrations/productivity/asana/reference", - destination: "/:locale/resources/integrations/productivity/asana", - permanent: true, - }, - { - source: - "/:locale/resources/integrations/productivity/clickup/reference", - destination: "/:locale/resources/integrations/productivity/clickup", - permanent: true, - }, - { - source: - "/:locale/resources/integrations/productivity/dropbox/reference", - destination: "/:locale/resources/integrations/productivity/dropbox", - permanent: true, - }, - { - source: - "/:locale/resources/integrations/productivity/gmail/reference", - destination: "/:locale/resources/integrations/productivity/gmail", - permanent: true, - }, - { - source: - "/:locale/resources/integrations/productivity/google-calendar/reference", - destination: - "/:locale/resources/integrations/productivity/google-calendar", - permanent: true, - }, - { - source: - "/:locale/resources/integrations/productivity/google-docs/reference", - destination: - "/:locale/resources/integrations/productivity/google-docs", - permanent: true, - }, - { - source: - "/:locale/resources/integrations/productivity/google-drive/reference", - destination: - "/:locale/resources/integrations/productivity/google-drive", - permanent: true, - }, - { - source: - "/:locale/resources/integrations/productivity/google-sheets/reference", - destination: - "/:locale/resources/integrations/productivity/google-sheets", - permanent: true, - }, - { - source: - "/:locale/resources/integrations/productivity/jira/environment-variables", - destination: "/:locale/resources/integrations/productivity/jira", - permanent: true, - }, - { - source: "/:locale/resources/integrations/productivity/jira/reference", - destination: "/:locale/resources/integrations/productivity/jira", - permanent: true, - }, - { - source: "/:locale/resources/integrations/sales/hubspot/reference", - destination: "/:locale/resources/integrations/sales/hubspot", - permanent: true, - }, - { - source: - "/:locale/resources/integrations/social-communication/discord", - destination: "/:locale/resources/integrations", - permanent: true, - }, - { - source: - "/:locale/resources/integrations/social-communication/linkedin", - destination: "/:locale/resources/integrations/social/linkedin", - permanent: true, - }, - { - source: - "/:locale/resources/integrations/social-communication/microsoft-teams", - destination: "/:locale/resources/integrations/social/microsoft-teams", - permanent: true, - }, - { - source: - "/:locale/resources/integrations/social-communication/microsoft-teams/reference", - destination: "/:locale/resources/integrations/social/microsoft-teams", - permanent: true, - }, - { - source: "/:locale/resources/integrations/social-communication/reddit", - destination: "/:locale/resources/integrations/social/reddit", - permanent: true, - }, - { - source: - "/:locale/resources/integrations/social-communication/slack-api", - destination: "/:locale/resources/integrations/social/slack-api", - permanent: true, - }, - { - source: - "/:locale/resources/integrations/social-communication/slack/environment-variables", - destination: "/:locale/resources/integrations/social/slack", - permanent: true, - }, - { - source: - "/:locale/resources/integrations/social-communication/slack/install", - destination: "/:locale/resources/integrations/social/slack", - permanent: true, - }, - { - source: "/:locale/resources/integrations/social-communication/slack", - destination: "/:locale/resources/integrations/social/slack", - permanent: true, - }, - { - source: - "/:locale/resources/integrations/social-communication/slack/reference", - destination: "/:locale/resources/integrations/social/slack", - permanent: true, - }, - { - source: - "/:locale/resources/integrations/social-communication/teams/reference", - destination: "/:locale/resources/integrations/social/microsoft-teams", - permanent: true, - }, - { - source: "/:locale/resources/integrations/social-communication/twilio", - destination: "/:locale/resources/integrations", - permanent: true, - }, - { - source: - "/:locale/resources/integrations/social-communication/twilio/reference", - destination: "/:locale/resources/integrations", - permanent: true, - }, - { - source: "/:locale/resources/integrations/social-communication/x", - destination: "/:locale/resources/integrations/social/x", - permanent: true, - }, - { - source: - "/:locale/resources/integrations/social-communication/zoom/install", - destination: "/:locale/resources/integrations/social/zoom", - permanent: true, - }, - { - source: "/:locale/resources/integrations/social-communication/zoom", - destination: "/:locale/resources/integrations/social/zoom", - permanent: true, - }, - // Auto-added redirects for deleted pages - { - source: - "/:locale/guides/create-tools/contribute/registry-early-access", - destination: "/:locale/resources/registry-early-access", - permanent: true, - }, - { - source: "/:locale/resources/integrations/contribute-a-server", - destination: "/:locale/resources/registry-early-access", - permanent: true, - }, - // Moved MCP Gateway UI guide to guides - { - source: "/:locale/guides/create-tools/mcp-gateways", - destination: "/:locale/guides/mcp-gateways", - permanent: true, - }, - // Removed LangChain old stuff - { - source: - "/:locale/get-started/agent-frameworks/langchain/use-arcade-tools", - destination: - "/:locale/get-started/agent-frameworks/langchain/use-arcade-with-langchain-py", - permanent: true, - }, - { - source: - "/:locale/get-started/agent-frameworks/langchain/user-auth-interrupts", - destination: - "/:locale/get-started/agent-frameworks/langchain/use-arcade-with-langchain-py", - permanent: true, - }, - // Mastra tutorial consolidation - { - source: "/:locale/get-started/agent-frameworks/mastra/overview", - destination: "/:locale/get-started/agent-frameworks/mastra", - permanent: true, - }, - { - source: - "/:locale/get-started/agent-frameworks/mastra/use-arcade-tools", - destination: "/:locale/get-started/agent-frameworks/mastra", - permanent: true, - }, - { - source: - "/:locale/get-started/agent-frameworks/mastra/user-auth-interrupts", - destination: "/:locale/get-started/agent-frameworks/mastra", - permanent: true, - }, - // OpenAI Agents tutorial consolidation - { - source: - "/:locale/get-started/agent-frameworks/openai-agents/use-arcade-with-openai-agents", - destination: - "/:locale/get-started/agent-frameworks/openai-agents/overview", - permanent: true, - }, - { - source: - "/:locale/get-started/agent-frameworks/openai-agents/use-arcade-tools", - destination: - "/:locale/get-started/agent-frameworks/openai-agents/overview", - permanent: true, - }, - { - source: - "/:locale/get-started/agent-frameworks/openai-agents/user-auth-interrupts", - destination: - "/:locale/get-started/agent-frameworks/openai-agents/overview", - permanent: true, - }, - // Moved from guides to get-started - { - source: - "/:locale/guides/agent-frameworks/setup-arcade-with-your-llm-python", - destination: - "/:locale/get-started/agent-frameworks/setup-arcade-with-your-llm-python", - permanent: true, - }, - // Old /home/* paths to new structure - { - source: "/:locale/home/langchain/use-arcade-tools", - destination: - "/:locale/get-started/agent-frameworks/langchain/use-arcade-with-langchain-py", - permanent: true, - }, - { - source: "/:locale/guides/agent-frameworks/langchain/use-arcade-tools", - destination: - "/:locale/get-started/agent-frameworks/langchain/use-arcade-with-langchain-py", - permanent: true, - }, - { - source: "/:locale/home/langchain/user-auth-interrupts", - destination: - "/:locale/get-started/agent-frameworks/langchain/use-arcade-with-langchain-py", - permanent: true, - }, - { - source: - "/:locale/guides/agent-frameworks/langchain/user-auth-interrupts", - destination: - "/:locale/get-started/agent-frameworks/langchain/use-arcade-with-langchain-py", - permanent: true, - }, - { - source: - "/:locale/get-started/agent-frameworks/langchain/use-arcade-with-langchain", - destination: - "/:locale/get-started/agent-frameworks/langchain/use-arcade-with-langchain-py", - permanent: true, - }, - { - source: "/:locale/home/oai-agents/user-auth-interrupts", - destination: - "/:locale/get-started/agent-frameworks/openai-agents/overview", - permanent: true, - }, - { - source: "/:locale/home/mastra/user-auth-interrupts", - destination: "/:locale/get-started/agent-frameworks/mastra", - permanent: true, - }, - { - source: "/:locale/home/build-tools/server-level-vs-tool-level-auth", - destination: "/:locale/learn/server-level-vs-tool-level-auth", - permanent: true, - }, - { - source: "/:locale/home/build-tools/secure-your-mcp-server", - destination: - "/:locale/guides/create-tools/secure-your-server/secure-your-mcp-server", - permanent: true, - }, - { - source: "/:locale/home/agent-frameworks-overview", - destination: "/:locale/get-started/agent-frameworks", - permanent: true, - }, - { - source: "/:locale/home/agentic-development", - destination: "/:locale/get-started/setup/connect-arcade-docs", - permanent: true, - }, - { - source: "/:locale/home/api-keys", - destination: "/:locale/get-started/setup/api-keys", - permanent: true, - }, - { - source: - "/:locale/guides/agent-frameworks/vercelai/using-arcade-tools", - destination: "/:locale/get-started/agent-frameworks/vercelai", - permanent: true, - }, - { - source: "/:locale/home/arcade-cli", - destination: "/:locale/references/arcade-cli", - permanent: true, - }, - { - source: "/:locale/home/auth-providers", - destination: "/:locale/references/auth-providers", - permanent: true, - }, - { - source: "/:locale/home/auth-providers/:path*", - destination: "/:locale/references/auth-providers/:path*", - permanent: true, - }, - { - source: "/:locale/home/auth/auth-tool-calling", - destination: - "/:locale/guides/tool-calling/custom-apps/auth-tool-calling", - permanent: true, - }, - { - source: "/:locale/home/auth/call-third-party-apis-directly", - destination: "/:locale/guides/tool-calling/call-third-party-apis", - permanent: true, - }, - { - source: "/:locale/home/auth/how-arcade-helps", - destination: "/:locale/get-started/about-arcade", - permanent: true, - }, - { - source: "/:locale/home/auth/secure-auth-production", - destination: - "/:locale/guides/user-facing-agents/secure-auth-production", - permanent: true, - }, - { - source: "/:locale/home/auth/tool-auth-status", - destination: - "/:locale/guides/tool-calling/custom-apps/check-auth-status", - permanent: true, - }, - { - source: "/:locale/home/build-tools/call-tools-from-mcp-clients", - destination: - "/:locale/guides/create-tools/tool-basics/call-tools-mcp", - permanent: true, - }, - { - source: "/:locale/home/build-tools/create-a-mcp-server", - destination: - "/:locale/guides/create-tools/tool-basics/build-mcp-server", - permanent: true, - }, - { - source: "/:locale/home/build-tools/create-a-tool-with-auth", - destination: - "/:locale/guides/create-tools/tool-basics/create-tool-auth", - permanent: true, - }, - { - source: "/:locale/home/build-tools/create-a-tool-with-secrets", - destination: - "/:locale/guides/create-tools/tool-basics/create-tool-secrets", - permanent: true, - }, - { - source: "/:locale/home/build-tools/migrate-from-toolkits", - destination: "/:locale/guides/create-tools/migrate-toolkits", - permanent: true, - }, - { - source: "/:locale/home/build-tools/organize-mcp-server-tools", - destination: - "/:locale/guides/create-tools/tool-basics/organize-mcp-tools", - permanent: true, - }, - { - source: "/:locale/home/build-tools/providing-useful-tool-errors", - destination: - "/:locale/guides/create-tools/error-handling/useful-tool-errors", - permanent: true, - }, - { - source: "/:locale/home/build-tools/retry-tools-with-improved-prompt", - destination: - "/:locale/guides/create-tools/error-handling/retry-tools", - permanent: true, - }, - { - source: "/:locale/home/build-tools/tool-context", - destination: - "/:locale/guides/create-tools/tool-basics/runtime-data-access", - permanent: true, - }, - { - source: "/:locale/home/changelog", - destination: "/:locale/references/changelog", - permanent: true, - }, - { - source: "/:locale/home/compare-server-types", - destination: - "/:locale/guides/create-tools/tool-basics/compare-server-types", - permanent: true, - }, - { - source: "/:locale/home/contact-us", - destination: "/:locale/resources/contact-us", - permanent: true, - }, - { - source: "/:locale/home/crewai/custom-auth-flow", - destination: - "/:locale/get-started/agent-frameworks/crewai/use-arcade-tools", - permanent: true, - }, - { - source: "/:locale/home/crewai/use-arcade-tools", - destination: - "/:locale/get-started/agent-frameworks/crewai/use-arcade-tools", - permanent: true, - }, - { - source: "/:locale/home/custom-mcp-server-quickstart", - destination: "/:locale/get-started/quickstarts/mcp-server-quickstart", - permanent: true, - }, - { - source: "/:locale/home/deployment/arcade-cloud-infra", - destination: "/:locale/guides/deployment-hosting/arcade-cloud", - permanent: true, - }, - { - source: "/:locale/home/deployment/engine-configuration", - destination: "/:locale/guides/deployment-hosting/helm", - permanent: true, - }, - { - source: "/:locale/home/evaluate-tools/create-an-evaluation-suite", - destination: - "/:locale/guides/create-tools/evaluate-tools/create-evaluation-suite", - permanent: true, - }, - { - source: "/:locale/home/evaluate-tools/run-evaluations", - destination: - "/:locale/guides/create-tools/evaluate-tools/run-evaluations", - permanent: true, - }, - { - source: "/:locale/home/evaluate-tools/why-evaluate-tools", - destination: - "/:locale/guides/create-tools/evaluate-tools/why-evaluate", - permanent: true, - }, - { - source: "/:locale/home/examples", - destination: "/:locale/resources/examples", - permanent: true, - }, - { - source: "/:locale/home/faq", - destination: "/:locale/resources/faq", - permanent: true, - }, - { - source: "/:locale/home/glossary", - destination: "/:locale/resources/glossary", - permanent: true, - }, - { - source: "/:locale/home/google-adk/use-arcade-tools", - destination: - "/:locale/get-started/agent-frameworks/google-adk/setup-python", - permanent: true, - }, - { - source: "/:locale/home/hosting-overview", - destination: "/:locale/guides/deployment-hosting", - permanent: true, - }, - { - source: "/:locale/home/langchain/auth-langchain-tools", - destination: - "/:locale/get-started/agent-frameworks/langchain/auth-langchain-tools", - permanent: true, - }, - { - source: "/:locale/home/mastra/use-arcade-tools", - destination: "/:locale/get-started/agent-frameworks/mastra", - permanent: true, - }, - { - source: "/:locale/home/mcp-clients/claude-desktop", - destination: "/:locale/get-started/mcp-clients/claude-desktop", - permanent: true, - }, - { - source: "/:locale/home/mcp-clients/claude-code", - destination: "/:locale/get-started/mcp-clients/claude-code", - permanent: true, - }, - { - source: "/:locale/home/mcp-clients/cursor", - destination: "/:locale/get-started/mcp-clients/cursor", - permanent: true, - }, - { - source: "/:locale/home/mcp-clients/visual-studio-code", - destination: "/:locale/get-started/mcp-clients/visual-studio-code", - permanent: true, - }, - { - source: "/:locale/home/mcp-gateway-quickstart", - destination: "/:locale/get-started/quickstarts/call-tool-client", - permanent: true, - }, - { - source: "/:locale/home/mcp-gateways", - destination: "/:locale/guides/mcp-gateways", - permanent: true, - }, - { - source: "/:locale/home/oai-agents/use-arcade-tools", - destination: - "/:locale/get-started/agent-frameworks/openai-agents/overview", - permanent: true, - }, - { - source: "/:locale/home/quickstart", - destination: "/:locale/get-started/quickstarts/call-tool-agent", - permanent: true, - }, - { - source: "/:locale/home/registry-early-access", - destination: "/:locale/resources/registry-early-access", - permanent: true, - }, - { - source: "/:locale/home/serve-tools/arcade-deploy", - destination: "/:locale/guides/deployment-hosting/arcade-deploy", - permanent: true, - }, - { - source: "/:locale/home/serve-tools/hybrid-worker", - destination: "/:locale/guides/deployment-hosting/on-prem", - permanent: true, - }, - { - source: "/:locale/home/use-tools/get-tool-definitions", - destination: - "/:locale/guides/tool-calling/custom-apps/get-tool-definitions", - permanent: true, - }, - { - source: "/:locale/home/use-tools/tools-overview", - destination: "/:locale/guides/tool-calling", - permanent: true, - }, - { - source: "/:locale/home/use-tools/types-of-tools", - destination: "/:locale/guides/create-tools/improve/types-of-tools", - permanent: true, - }, - { - source: "/:locale/home/use-tools/error-handling", - destination: "/:locale/guides/tool-calling/error-handling", - permanent: true, - }, - { - source: "/:locale/home/vercelai/using-arcade-tools", - destination: "/:locale/get-started/agent-frameworks/vercelai", - permanent: true, - }, - // Legacy /integrations path - // NOTE: :locale is constrained to actual locale values to prevent - // collisions with locale-less paths like /resources/integrations, - // which would otherwise match with :locale="resources" and redirect - // to /resources/resources/integrations (a 404). - { - source: "/:locale(en|es|pt-BR)/integrations", - destination: "/:locale/resources/integrations", - permanent: true, - }, - { - source: "/:locale(en|es|pt-BR)/integrations/:path*", - destination: "/:locale/resources/integrations/:path*", - permanent: true, - }, - // MCP servers to integrations - { - source: "/:locale(en|es|pt-BR)/mcp-servers", - destination: "/:locale/resources/integrations", - permanent: true, - }, - { - source: "/:locale(en|es|pt-BR)/mcp-servers/:path*", - destination: "/:locale/resources/integrations/:path*", - permanent: true, - }, - // References fixes - { - source: "/:locale/references/mcp", - destination: "/:locale/references/mcp/python", - permanent: true, - }, - { - source: "/:locale/references/mcp/python/overview", - destination: "/:locale/references/mcp/python", - permanent: true, - }, - { - source: "/:locale/references/arcade-cliarcade-configure", - destination: "/:locale/references/arcade-cli", - permanent: true, - }, - // Path corrections (typos, renames) - { - source: "/:locale/get-started/setup/api-key", - destination: "/:locale/get-started/setup/api-keys", - permanent: true, - }, - { - source: - "/:locale/guides/tool-calling/custom-apps/authorized-tool-calling", - destination: - "/:locale/guides/tool-calling/custom-apps/auth-tool-calling", - permanent: true, - }, - { - source: "/:locale/guides/user-facing-agents/brand-provider", - destination: - "/:locale/guides/user-facing-agents/secure-auth-production", - permanent: true, - }, - { - source: "/:locale/guides/user-facing-agents/configure-oauth-provider", - destination: - "/:locale/guides/user-facing-agents/secure-auth-production", - permanent: true, - }, - { - source: "/:locale/guides/tool-calling/mcp-client/:client", - destination: "/:locale/get-started/mcp-clients/:client", - permanent: true, - }, - { - source: "/:locale/guides/tool-calling/get-tool-definitions", - destination: - "/:locale/guides/tool-calling/custom-apps/get-tool-definitions", - permanent: true, - }, - { - source: "/:locale/guides/deployment-hosting/engine-configuration", - destination: "/:locale/guides/deployment-hosting/helm", - permanent: true, - }, - { - source: "/:locale/guides/deployment-hosting/configure-engine", - destination: "/:locale/guides/deployment-hosting/helm", - permanent: true, - }, - { - source: "/:locale/guides/create-tools/performance/run-evaluations", - destination: - "/:locale/guides/create-tools/evaluate-tools/run-evaluations", - permanent: true, - }, - { - source: "/:locale/guides/create-tools/contribute/registry", - destination: "/:locale/resources/registry-early-access", - permanent: true, - }, - // Framework path aliases (old naming conventions) - { - source: "/:locale/guides/agent-frameworks/crewai/python", - destination: - "/:locale/get-started/agent-frameworks/crewai/use-arcade-tools", - permanent: true, - }, - { - source: "/:locale/guides/agent-frameworks/langchain/python", - destination: - "/:locale/get-started/agent-frameworks/langchain/use-arcade-with-langchain-py", - permanent: true, - }, - { - source: "/:locale/guides/agent-frameworks/langchain/tools", - destination: - "/:locale/get-started/agent-frameworks/langchain/auth-langchain-tools", - permanent: true, - }, - { - source: "/:locale/guides/agent-frameworks/mastra/typescript", - destination: "/:locale/get-started/agent-frameworks/mastra", - permanent: true, - }, - { - source: "/:locale/guides/agent-frameworks/google-adk/python", - destination: - "/:locale/get-started/agent-frameworks/google-adk/setup-python", - permanent: true, - }, - { - source: "/:locale/guides/agent-frameworks/openai/python", - destination: - "/:locale/get-started/agent-frameworks/openai-agents/overview", - permanent: true, - }, - { - source: "/:locale/guides/agent-frameworks/vercel-ai/typescript", - destination: "/:locale/get-started/agent-frameworks/vercelai", - permanent: true, - }, - // Old resource paths - { - source: "/:locale/resources/mastra/user-auth-interrupts", - destination: "/:locale/get-started/agent-frameworks/mastra", - permanent: true, - }, - { - source: "/:locale/resources/oai-agents/overview", - destination: - "/:locale/get-started/agent-frameworks/openai-agents/overview", - permanent: true, - }, - { - source: "/:locale/resources/creating-tools/:path*", - destination: "/:locale/guides/create-tools/:path*", - permanent: true, - }, - // Agent frameworks moved from guides to get-started - { - source: "/:locale/guides/agent-frameworks", - destination: "/:locale/get-started/agent-frameworks", - permanent: true, - }, - { - source: "/:locale/guides/agent-frameworks/:path*", - destination: "/:locale/get-started/agent-frameworks/:path*", - permanent: true, - }, - // MCP clients moved from guides/tool-calling to get-started - { - source: "/:locale/guides/tool-calling/mcp-clients", - destination: "/:locale/get-started/mcp-clients", - permanent: true, - }, - { - source: "/:locale/guides/tool-calling/mcp-clients/:path*", - destination: "/:locale/get-started/mcp-clients/:path*", - permanent: true, - }, - // Deprecated toolkit renames (microsoft_* prefix, ArcadeAI/monorepo#601) - { - source: "/:locale/resources/integrations/productivity/sharepoint", - destination: - "/:locale/resources/integrations/productivity/microsoft-sharepoint", - permanent: true, - }, - { - source: "/:locale/resources/integrations/productivity/outlook-mail", - destination: - "/:locale/resources/integrations/productivity/microsoft-outlook-mail", - permanent: true, - }, - { - source: - "/:locale/resources/integrations/productivity/outlook-calendar", - destination: - "/:locale/resources/integrations/productivity/microsoft-outlook-calendar", - permanent: true, - }, - ]; + return redirects; + }, + // The app imports shared modules out of toolkit-docs-generator/src/shared/, + // which compiles under "moduleResolution": "NodeNext" and therefore writes + // its internal relative imports with a ".js" extension. Webpack resolves + // with bundler semantics and would look for a literal ".js" file that + // never exists on disk, so teach it to try ".ts" first. + webpack: (config) => { + config.resolve.extensionAlias = { + ...config.resolve.extensionAlias, + ".js": [".ts", ".tsx", ".js"], + }; + return config; }, headers: async () => [ { diff --git a/package.json b/package.json index 439bbb3da..657540f93 100644 --- a/package.json +++ b/package.json @@ -63,7 +63,8 @@ "remark-gfm": "4.0.1", "swagger-ui-react": "5.32.6", "tailwindcss-animate": "1.0.7", - "unist-util-visit": "5.1.0" + "unist-util-visit": "5.1.0", + "zod": "4.3.6" }, "devDependencies": { "@anthropic-ai/sdk": "0.91.1", @@ -93,8 +94,7 @@ "typescript": "5.9.3", "ultracite": "6.1.0", "vite": "7.3.5", - "vitest": "4.1.8", - "zod": "4.3.6" + "vitest": "4.1.8" }, "engines": { "node": "22.x", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index d9790c3ca..761fc4215 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -88,6 +88,9 @@ importers: unist-util-visit: specifier: 5.1.0 version: 5.1.0 + zod: + specifier: 4.3.6 + version: 4.3.6 devDependencies: '@anthropic-ai/sdk': specifier: 0.91.1 @@ -173,9 +176,6 @@ importers: vitest: specifier: 4.1.8 version: 4.1.8(@opentelemetry/api@1.9.0)(@types/node@22.19.17)(vite@7.3.5(@types/node@22.19.17)(jiti@2.6.1)(lightningcss@1.32.0)(yaml@2.8.3)) - zod: - specifier: 4.3.6 - version: 4.3.6 packages: diff --git a/redirects.ts b/redirects.ts new file mode 100644 index 000000000..7e4b338b5 --- /dev/null +++ b/redirects.ts @@ -0,0 +1,875 @@ +/** + * Redirect rules for renamed, merged, or deleted pages. + * + * This is a plain data module (not next.config.ts) so it can be imported + * directly by scripts/check-redirects.ts and tests/sitemap.test.ts instead of + * regex-parsing next.config.ts as text. + * + * `pnpm check-redirects --auto-fix` appends new entries under the + * "Auto-added redirects" comment near the end of this file. + */ + +export type Redirect = { + source: string; + destination: string; + permanent: boolean; +}; + +export const redirects: Redirect[] = [ + // The toolkit-page breadcrumb links "Resources" -> /resources, which has + // no index page. Send it to the integrations registry instead of 404ing. + { + source: "/:locale/resources", + destination: "/:locale/resources/integrations", + permanent: true, + }, + // The auth provider is "square"; an external/stale link points at the + // old "squareup" slug, which 404s. Send it to the real page. + { + source: "/:locale/references/auth-providers/squareup", + destination: "/:locale/references/auth-providers/square", + permanent: true, + }, + // Dissolved guides/security section + { + source: "/:locale/guides/security/security-research-program", + destination: "/:locale/resources/security-research-program", + permanent: true, + }, + { + source: "/:locale/guides/security/securing-arcade-mcp", + destination: "/:locale/guides/create-tools/secure-your-server", + permanent: true, + }, + { + source: "/:locale/guides/security/secure-your-mcp-server", + destination: + "/:locale/guides/create-tools/secure-your-server/secure-your-mcp-server", + permanent: true, + }, + { + source: "/:locale/guides/security", + destination: "/:locale/guides/create-tools/secure-your-server", + permanent: true, + }, + { + source: "/:locale/references/mcp/python/transports", + destination: "/:locale/references/mcp/python", + permanent: true, + }, + { + source: "/:locale/references/mcp/python/types", + destination: "/:locale/references/mcp/python", + permanent: true, + }, + // CrewAI custom auth flow redirect to use-arcade-tools + { + source: "/:locale/get-started/agent-frameworks/crewai/custom-auth-flow", + destination: + "/:locale/get-started/agent-frameworks/crewai/use-arcade-tools", + permanent: true, + }, + // "others" category removed — toolkits moved to proper categories + { + source: "/:locale/resources/integrations/others/:path*", + destination: "/:locale/resources/integrations", + permanent: false, + }, + // Google ADK tutorial consolidation - redirect old URL to new + { + source: "/:locale/get-started/agent-frameworks/google-adk/use-arcade-tools", + destination: "/:locale/get-started/agent-frameworks/google-adk/overview", + permanent: true, + }, + { + source: "/:locale/references/logic-extensions-api", + destination: "/:locale/references/contextual-access-webhook-api", + permanent: true, + }, + { + source: "/:locale/guides/logic-extensions", + destination: "/:locale/guides/contextual-access", + permanent: true, + }, + { + source: "/:locale/guides/logic-extensions/build-your-own", + destination: "/:locale/guides/contextual-access/build-your-own", + permanent: true, + }, + { + source: "/:locale/guides/logic-extensions/examples", + destination: "/:locale/guides/contextual-access/examples", + permanent: true, + }, + { + source: "/:locale/guides/logic-extensions/how-hooks-work", + destination: "/:locale/guides/contextual-access/how-hooks-work", + permanent: true, + }, + { + source: "/:locale/resources/integrations/preview", + destination: "/:locale/resources/integrations", + permanent: true, + }, + { + source: + "/:locale/resources/integrations/customer-support/zendesk/reference", + destination: "/:locale/resources/integrations/customer-support/zendesk", + permanent: true, + }, + { + source: "/:locale/resources/integrations/development/firecrawl/reference", + destination: "/:locale/resources/integrations/development/firecrawl", + permanent: true, + }, + { + source: "/:locale/resources/integrations/productivity/asana/reference", + destination: "/:locale/resources/integrations/productivity/asana", + permanent: true, + }, + { + source: "/:locale/resources/integrations/productivity/clickup/reference", + destination: "/:locale/resources/integrations/productivity/clickup", + permanent: true, + }, + { + source: "/:locale/resources/integrations/productivity/dropbox/reference", + destination: "/:locale/resources/integrations/productivity/dropbox", + permanent: true, + }, + { + source: "/:locale/resources/integrations/productivity/gmail/reference", + destination: "/:locale/resources/integrations/productivity/gmail", + permanent: true, + }, + { + source: + "/:locale/resources/integrations/productivity/google-calendar/reference", + destination: "/:locale/resources/integrations/productivity/google-calendar", + permanent: true, + }, + { + source: + "/:locale/resources/integrations/productivity/google-docs/reference", + destination: "/:locale/resources/integrations/productivity/google-docs", + permanent: true, + }, + { + source: + "/:locale/resources/integrations/productivity/google-drive/reference", + destination: "/:locale/resources/integrations/productivity/google-drive", + permanent: true, + }, + { + source: + "/:locale/resources/integrations/productivity/google-sheets/reference", + destination: "/:locale/resources/integrations/productivity/google-sheets", + permanent: true, + }, + { + source: + "/:locale/resources/integrations/productivity/jira/environment-variables", + destination: "/:locale/resources/integrations/productivity/jira", + permanent: true, + }, + { + source: "/:locale/resources/integrations/productivity/jira/reference", + destination: "/:locale/resources/integrations/productivity/jira", + permanent: true, + }, + { + source: "/:locale/resources/integrations/sales/hubspot/reference", + destination: "/:locale/resources/integrations/sales/hubspot", + permanent: true, + }, + { + source: "/:locale/resources/integrations/social-communication/discord", + destination: "/:locale/resources/integrations", + permanent: true, + }, + { + source: "/:locale/resources/integrations/social-communication/linkedin", + destination: "/:locale/resources/integrations/social/linkedin", + permanent: true, + }, + { + source: + "/:locale/resources/integrations/social-communication/microsoft-teams", + destination: "/:locale/resources/integrations/social/microsoft-teams", + permanent: true, + }, + { + source: + "/:locale/resources/integrations/social-communication/microsoft-teams/reference", + destination: "/:locale/resources/integrations/social/microsoft-teams", + permanent: true, + }, + { + source: "/:locale/resources/integrations/social-communication/reddit", + destination: "/:locale/resources/integrations/social/reddit", + permanent: true, + }, + { + source: "/:locale/resources/integrations/social-communication/slack-api", + destination: "/:locale/resources/integrations/social/slack-api", + permanent: true, + }, + { + source: + "/:locale/resources/integrations/social-communication/slack/environment-variables", + destination: "/:locale/resources/integrations/social/slack", + permanent: true, + }, + { + source: + "/:locale/resources/integrations/social-communication/slack/install", + destination: "/:locale/resources/integrations/social/slack", + permanent: true, + }, + { + source: "/:locale/resources/integrations/social-communication/slack", + destination: "/:locale/resources/integrations/social/slack", + permanent: true, + }, + { + source: + "/:locale/resources/integrations/social-communication/slack/reference", + destination: "/:locale/resources/integrations/social/slack", + permanent: true, + }, + { + source: + "/:locale/resources/integrations/social-communication/teams/reference", + destination: "/:locale/resources/integrations/social/microsoft-teams", + permanent: true, + }, + { + source: "/:locale/resources/integrations/social-communication/twilio", + destination: "/:locale/resources/integrations", + permanent: true, + }, + { + source: + "/:locale/resources/integrations/social-communication/twilio/reference", + destination: "/:locale/resources/integrations", + permanent: true, + }, + { + source: "/:locale/resources/integrations/social-communication/x", + destination: "/:locale/resources/integrations/social/x", + permanent: true, + }, + { + source: "/:locale/resources/integrations/social-communication/zoom/install", + destination: "/:locale/resources/integrations/social/zoom", + permanent: true, + }, + { + source: "/:locale/resources/integrations/social-communication/zoom", + destination: "/:locale/resources/integrations/social/zoom", + permanent: true, + }, + { + source: "/:locale/guides/create-tools/contribute/registry-early-access", + destination: "/:locale/resources/registry-early-access", + permanent: true, + }, + { + source: "/:locale/resources/integrations/contribute-a-server", + destination: "/:locale/resources/registry-early-access", + permanent: true, + }, + // Moved MCP Gateway UI guide to guides + { + source: "/:locale/guides/create-tools/mcp-gateways", + destination: "/:locale/guides/mcp-gateways", + permanent: true, + }, + // Removed LangChain old stuff + { + source: "/:locale/get-started/agent-frameworks/langchain/use-arcade-tools", + destination: + "/:locale/get-started/agent-frameworks/langchain/use-arcade-with-langchain-py", + permanent: true, + }, + { + source: + "/:locale/get-started/agent-frameworks/langchain/user-auth-interrupts", + destination: + "/:locale/get-started/agent-frameworks/langchain/use-arcade-with-langchain-py", + permanent: true, + }, + // Mastra tutorial consolidation + { + source: "/:locale/get-started/agent-frameworks/mastra/overview", + destination: "/:locale/get-started/agent-frameworks/mastra", + permanent: true, + }, + { + source: "/:locale/get-started/agent-frameworks/mastra/use-arcade-tools", + destination: "/:locale/get-started/agent-frameworks/mastra", + permanent: true, + }, + { + source: "/:locale/get-started/agent-frameworks/mastra/user-auth-interrupts", + destination: "/:locale/get-started/agent-frameworks/mastra", + permanent: true, + }, + // OpenAI Agents tutorial consolidation + { + source: + "/:locale/get-started/agent-frameworks/openai-agents/use-arcade-with-openai-agents", + destination: "/:locale/get-started/agent-frameworks/openai-agents/overview", + permanent: true, + }, + { + source: + "/:locale/get-started/agent-frameworks/openai-agents/use-arcade-tools", + destination: "/:locale/get-started/agent-frameworks/openai-agents/overview", + permanent: true, + }, + { + source: + "/:locale/get-started/agent-frameworks/openai-agents/user-auth-interrupts", + destination: "/:locale/get-started/agent-frameworks/openai-agents/overview", + permanent: true, + }, + // Moved from guides to get-started + { + source: + "/:locale/guides/agent-frameworks/setup-arcade-with-your-llm-python", + destination: + "/:locale/get-started/agent-frameworks/setup-arcade-with-your-llm-python", + permanent: true, + }, + // Old /home/* paths to new structure + { + source: "/:locale/home/langchain/use-arcade-tools", + destination: + "/:locale/get-started/agent-frameworks/langchain/use-arcade-with-langchain-py", + permanent: true, + }, + { + source: "/:locale/guides/agent-frameworks/langchain/use-arcade-tools", + destination: + "/:locale/get-started/agent-frameworks/langchain/use-arcade-with-langchain-py", + permanent: true, + }, + { + source: "/:locale/home/langchain/user-auth-interrupts", + destination: + "/:locale/get-started/agent-frameworks/langchain/use-arcade-with-langchain-py", + permanent: true, + }, + { + source: "/:locale/guides/agent-frameworks/langchain/user-auth-interrupts", + destination: + "/:locale/get-started/agent-frameworks/langchain/use-arcade-with-langchain-py", + permanent: true, + }, + { + source: + "/:locale/get-started/agent-frameworks/langchain/use-arcade-with-langchain", + destination: + "/:locale/get-started/agent-frameworks/langchain/use-arcade-with-langchain-py", + permanent: true, + }, + { + source: "/:locale/home/oai-agents/user-auth-interrupts", + destination: "/:locale/get-started/agent-frameworks/openai-agents/overview", + permanent: true, + }, + { + source: "/:locale/home/mastra/user-auth-interrupts", + destination: "/:locale/get-started/agent-frameworks/mastra", + permanent: true, + }, + { + source: "/:locale/home/build-tools/server-level-vs-tool-level-auth", + destination: "/:locale/learn/server-level-vs-tool-level-auth", + permanent: true, + }, + { + source: "/:locale/home/build-tools/secure-your-mcp-server", + destination: + "/:locale/guides/create-tools/secure-your-server/secure-your-mcp-server", + permanent: true, + }, + { + source: "/:locale/home/agent-frameworks-overview", + destination: "/:locale/get-started/agent-frameworks", + permanent: true, + }, + { + source: "/:locale/home/agentic-development", + destination: "/:locale/get-started/setup/connect-arcade-docs", + permanent: true, + }, + { + source: "/:locale/home/api-keys", + destination: "/:locale/get-started/setup/api-keys", + permanent: true, + }, + { + source: "/:locale/guides/agent-frameworks/vercelai/using-arcade-tools", + destination: "/:locale/get-started/agent-frameworks/vercelai", + permanent: true, + }, + { + source: "/:locale/home/arcade-cli", + destination: "/:locale/references/arcade-cli", + permanent: true, + }, + { + source: "/:locale/home/auth-providers", + destination: "/:locale/references/auth-providers", + permanent: true, + }, + { + source: "/:locale/home/auth-providers/:path*", + destination: "/:locale/references/auth-providers/:path*", + permanent: true, + }, + { + source: "/:locale/home/auth/auth-tool-calling", + destination: "/:locale/guides/tool-calling/custom-apps/auth-tool-calling", + permanent: true, + }, + { + source: "/:locale/home/auth/call-third-party-apis-directly", + destination: "/:locale/guides/tool-calling/call-third-party-apis", + permanent: true, + }, + { + source: "/:locale/home/auth/how-arcade-helps", + destination: "/:locale/get-started/about-arcade", + permanent: true, + }, + { + source: "/:locale/home/auth/secure-auth-production", + destination: "/:locale/guides/user-facing-agents/secure-auth-production", + permanent: true, + }, + { + source: "/:locale/home/auth/tool-auth-status", + destination: "/:locale/guides/tool-calling/custom-apps/check-auth-status", + permanent: true, + }, + { + source: "/:locale/home/build-tools/call-tools-from-mcp-clients", + destination: "/:locale/guides/create-tools/tool-basics/call-tools-mcp", + permanent: true, + }, + { + source: "/:locale/home/build-tools/create-a-mcp-server", + destination: "/:locale/guides/create-tools/tool-basics/build-mcp-server", + permanent: true, + }, + { + source: "/:locale/home/build-tools/create-a-tool-with-auth", + destination: "/:locale/guides/create-tools/tool-basics/create-tool-auth", + permanent: true, + }, + { + source: "/:locale/home/build-tools/create-a-tool-with-secrets", + destination: "/:locale/guides/create-tools/tool-basics/create-tool-secrets", + permanent: true, + }, + { + source: "/:locale/home/build-tools/migrate-from-toolkits", + destination: "/:locale/guides/create-tools/migrate-toolkits", + permanent: true, + }, + { + source: "/:locale/home/build-tools/organize-mcp-server-tools", + destination: "/:locale/guides/create-tools/tool-basics/organize-mcp-tools", + permanent: true, + }, + { + source: "/:locale/home/build-tools/providing-useful-tool-errors", + destination: + "/:locale/guides/create-tools/error-handling/useful-tool-errors", + permanent: true, + }, + { + source: "/:locale/home/build-tools/retry-tools-with-improved-prompt", + destination: "/:locale/guides/create-tools/error-handling/retry-tools", + permanent: true, + }, + { + source: "/:locale/home/build-tools/tool-context", + destination: "/:locale/guides/create-tools/tool-basics/runtime-data-access", + permanent: true, + }, + { + source: "/:locale/home/changelog", + destination: "/:locale/references/changelog", + permanent: true, + }, + { + source: "/:locale/home/compare-server-types", + destination: + "/:locale/guides/create-tools/tool-basics/compare-server-types", + permanent: true, + }, + { + source: "/:locale/home/contact-us", + destination: "/:locale/resources/contact-us", + permanent: true, + }, + { + source: "/:locale/home/crewai/custom-auth-flow", + destination: + "/:locale/get-started/agent-frameworks/crewai/use-arcade-tools", + permanent: true, + }, + { + source: "/:locale/home/crewai/use-arcade-tools", + destination: + "/:locale/get-started/agent-frameworks/crewai/use-arcade-tools", + permanent: true, + }, + { + source: "/:locale/home/custom-mcp-server-quickstart", + destination: "/:locale/get-started/quickstarts/mcp-server-quickstart", + permanent: true, + }, + { + source: "/:locale/home/deployment/arcade-cloud-infra", + destination: "/:locale/guides/deployment-hosting/arcade-cloud", + permanent: true, + }, + { + source: "/:locale/home/deployment/engine-configuration", + destination: "/:locale/guides/deployment-hosting/helm", + permanent: true, + }, + { + source: "/:locale/home/evaluate-tools/create-an-evaluation-suite", + destination: + "/:locale/guides/create-tools/evaluate-tools/create-evaluation-suite", + permanent: true, + }, + { + source: "/:locale/home/evaluate-tools/run-evaluations", + destination: "/:locale/guides/create-tools/evaluate-tools/run-evaluations", + permanent: true, + }, + { + source: "/:locale/home/evaluate-tools/why-evaluate-tools", + destination: "/:locale/guides/create-tools/evaluate-tools/why-evaluate", + permanent: true, + }, + { + source: "/:locale/home/examples", + destination: "/:locale/resources/examples", + permanent: true, + }, + { + source: "/:locale/home/faq", + destination: "/:locale/resources/faq", + permanent: true, + }, + { + source: "/:locale/home/glossary", + destination: "/:locale/resources/glossary", + permanent: true, + }, + { + source: "/:locale/home/google-adk/use-arcade-tools", + destination: + "/:locale/get-started/agent-frameworks/google-adk/setup-python", + permanent: true, + }, + { + source: "/:locale/home/hosting-overview", + destination: "/:locale/guides/deployment-hosting", + permanent: true, + }, + { + source: "/:locale/home/langchain/auth-langchain-tools", + destination: + "/:locale/get-started/agent-frameworks/langchain/auth-langchain-tools", + permanent: true, + }, + { + source: "/:locale/home/mastra/use-arcade-tools", + destination: "/:locale/get-started/agent-frameworks/mastra", + permanent: true, + }, + { + source: "/:locale/home/mcp-clients/claude-desktop", + destination: "/:locale/get-started/mcp-clients/claude-desktop", + permanent: true, + }, + { + source: "/:locale/home/mcp-clients/claude-code", + destination: "/:locale/get-started/mcp-clients/claude-code", + permanent: true, + }, + { + source: "/:locale/home/mcp-clients/cursor", + destination: "/:locale/get-started/mcp-clients/cursor", + permanent: true, + }, + { + source: "/:locale/home/mcp-clients/visual-studio-code", + destination: "/:locale/get-started/mcp-clients/visual-studio-code", + permanent: true, + }, + { + source: "/:locale/home/mcp-gateway-quickstart", + destination: "/:locale/get-started/quickstarts/call-tool-client", + permanent: true, + }, + { + source: "/:locale/home/mcp-gateways", + destination: "/:locale/guides/mcp-gateways", + permanent: true, + }, + { + source: "/:locale/home/oai-agents/use-arcade-tools", + destination: "/:locale/get-started/agent-frameworks/openai-agents/overview", + permanent: true, + }, + { + source: "/:locale/home/quickstart", + destination: "/:locale/get-started/quickstarts/call-tool-agent", + permanent: true, + }, + { + source: "/:locale/home/registry-early-access", + destination: "/:locale/resources/registry-early-access", + permanent: true, + }, + { + source: "/:locale/home/serve-tools/arcade-deploy", + destination: "/:locale/guides/deployment-hosting/arcade-deploy", + permanent: true, + }, + { + source: "/:locale/home/serve-tools/hybrid-worker", + destination: "/:locale/guides/deployment-hosting/on-prem", + permanent: true, + }, + { + source: "/:locale/home/use-tools/get-tool-definitions", + destination: + "/:locale/guides/tool-calling/custom-apps/get-tool-definitions", + permanent: true, + }, + { + source: "/:locale/home/use-tools/tools-overview", + destination: "/:locale/guides/tool-calling", + permanent: true, + }, + { + source: "/:locale/home/use-tools/types-of-tools", + destination: "/:locale/guides/create-tools/improve/types-of-tools", + permanent: true, + }, + { + source: "/:locale/home/use-tools/error-handling", + destination: "/:locale/guides/tool-calling/error-handling", + permanent: true, + }, + { + source: "/:locale/home/vercelai/using-arcade-tools", + destination: "/:locale/get-started/agent-frameworks/vercelai", + permanent: true, + }, + // Legacy /integrations path + // NOTE: :locale is constrained to actual locale values to prevent + // collisions with locale-less paths like /resources/integrations, + // which would otherwise match with :locale="resources" and redirect + // to /resources/resources/integrations (a 404). + { + source: "/:locale(en|es|pt-BR)/integrations", + destination: "/:locale/resources/integrations", + permanent: true, + }, + { + source: "/:locale(en|es|pt-BR)/integrations/:path*", + destination: "/:locale/resources/integrations/:path*", + permanent: true, + }, + // MCP servers to integrations + { + source: "/:locale(en|es|pt-BR)/mcp-servers", + destination: "/:locale/resources/integrations", + permanent: true, + }, + { + source: "/:locale(en|es|pt-BR)/mcp-servers/:path*", + destination: "/:locale/resources/integrations/:path*", + permanent: true, + }, + // References fixes + { + source: "/:locale/references/mcp", + destination: "/:locale/references/mcp/python", + permanent: true, + }, + { + source: "/:locale/references/mcp/python/overview", + destination: "/:locale/references/mcp/python", + permanent: true, + }, + { + source: "/:locale/references/arcade-cliarcade-configure", + destination: "/:locale/references/arcade-cli", + permanent: true, + }, + // Path corrections (typos, renames) + { + source: "/:locale/get-started/setup/api-key", + destination: "/:locale/get-started/setup/api-keys", + permanent: true, + }, + { + source: "/:locale/guides/tool-calling/custom-apps/authorized-tool-calling", + destination: "/:locale/guides/tool-calling/custom-apps/auth-tool-calling", + permanent: true, + }, + { + source: "/:locale/guides/user-facing-agents/brand-provider", + destination: "/:locale/guides/user-facing-agents/secure-auth-production", + permanent: true, + }, + { + source: "/:locale/guides/user-facing-agents/configure-oauth-provider", + destination: "/:locale/guides/user-facing-agents/secure-auth-production", + permanent: true, + }, + { + source: "/:locale/guides/tool-calling/mcp-client/:client", + destination: "/:locale/get-started/mcp-clients/:client", + permanent: true, + }, + { + source: "/:locale/guides/tool-calling/get-tool-definitions", + destination: + "/:locale/guides/tool-calling/custom-apps/get-tool-definitions", + permanent: true, + }, + { + source: "/:locale/guides/deployment-hosting/engine-configuration", + destination: "/:locale/guides/deployment-hosting/helm", + permanent: true, + }, + { + source: "/:locale/guides/deployment-hosting/configure-engine", + destination: "/:locale/guides/deployment-hosting/helm", + permanent: true, + }, + { + source: "/:locale/guides/create-tools/performance/run-evaluations", + destination: "/:locale/guides/create-tools/evaluate-tools/run-evaluations", + permanent: true, + }, + { + source: "/:locale/guides/create-tools/contribute/registry", + destination: "/:locale/resources/registry-early-access", + permanent: true, + }, + // Framework path aliases (old naming conventions) + { + source: "/:locale/guides/agent-frameworks/crewai/python", + destination: + "/:locale/get-started/agent-frameworks/crewai/use-arcade-tools", + permanent: true, + }, + { + source: "/:locale/guides/agent-frameworks/langchain/python", + destination: + "/:locale/get-started/agent-frameworks/langchain/use-arcade-with-langchain-py", + permanent: true, + }, + { + source: "/:locale/guides/agent-frameworks/langchain/tools", + destination: + "/:locale/get-started/agent-frameworks/langchain/auth-langchain-tools", + permanent: true, + }, + { + source: "/:locale/guides/agent-frameworks/mastra/typescript", + destination: "/:locale/get-started/agent-frameworks/mastra", + permanent: true, + }, + { + source: "/:locale/guides/agent-frameworks/google-adk/python", + destination: + "/:locale/get-started/agent-frameworks/google-adk/setup-python", + permanent: true, + }, + { + source: "/:locale/guides/agent-frameworks/openai/python", + destination: "/:locale/get-started/agent-frameworks/openai-agents/overview", + permanent: true, + }, + { + source: "/:locale/guides/agent-frameworks/vercel-ai/typescript", + destination: "/:locale/get-started/agent-frameworks/vercelai", + permanent: true, + }, + // Old resource paths + { + source: "/:locale/resources/mastra/user-auth-interrupts", + destination: "/:locale/get-started/agent-frameworks/mastra", + permanent: true, + }, + { + source: "/:locale/resources/oai-agents/overview", + destination: "/:locale/get-started/agent-frameworks/openai-agents/overview", + permanent: true, + }, + { + source: "/:locale/resources/creating-tools/:path*", + destination: "/:locale/guides/create-tools/:path*", + permanent: true, + }, + // Agent frameworks moved from guides to get-started + { + source: "/:locale/guides/agent-frameworks", + destination: "/:locale/get-started/agent-frameworks", + permanent: true, + }, + { + source: "/:locale/guides/agent-frameworks/:path*", + destination: "/:locale/get-started/agent-frameworks/:path*", + permanent: true, + }, + // MCP clients moved from guides/tool-calling to get-started + { + source: "/:locale/guides/tool-calling/mcp-clients", + destination: "/:locale/get-started/mcp-clients", + permanent: true, + }, + { + source: "/:locale/guides/tool-calling/mcp-clients/:path*", + destination: "/:locale/get-started/mcp-clients/:path*", + permanent: true, + }, + // Deprecated toolkit renames (microsoft_* prefix, ArcadeAI/monorepo#601) + { + source: "/:locale/resources/integrations/productivity/sharepoint", + destination: + "/:locale/resources/integrations/productivity/microsoft-sharepoint", + permanent: true, + }, + { + source: "/:locale/resources/integrations/productivity/outlook-mail", + destination: + "/:locale/resources/integrations/productivity/microsoft-outlook-mail", + permanent: true, + }, + { + source: "/:locale/resources/integrations/productivity/outlook-calendar", + destination: + "/:locale/resources/integrations/productivity/microsoft-outlook-calendar", + permanent: true, + }, + + // Auto-added redirects for deleted pages. + // `pnpm check-redirects --auto-fix` appends new entries here. +]; diff --git a/scripts/check-redirects.ts b/scripts/check-redirects.ts index 23b9716af..23733e0c3 100644 --- a/scripts/check-redirects.ts +++ b/scripts/check-redirects.ts @@ -1,21 +1,32 @@ #!/usr/bin/env npx tsx /** - * Check that deleted/renamed markdown files have corresponding redirects in next.config.ts + * Check that deleted/renamed markdown files have corresponding redirects in redirects.ts * * Usage: * pnpm check-redirects [--auto-fix] [--staged-only] [base_branch] * * Features: * - Detects deleted AND renamed markdown files without redirects - * - Auto-fix mode: automatically inserts redirect entries into next.config.ts + * - Auto-fix mode: automatically inserts redirect entries into redirects.ts * - Validates existing redirects for circular references and invalid destinations * - Collapses redirect chains automatically * - --staged-only: Only check staged changes (for pre-commit hook) */ import { execSync } from "node:child_process"; -import { existsSync, readFileSync, writeFileSync } from "node:fs"; +import { readFileSync, writeFileSync } from "node:fs"; +import { redirects as configuredRedirects } from "../redirects"; +import { + checkWildcardMatch, + type DynamicRouteMove, + dynamicRouteExists, + fileToUrl, + isMoveCoveredByRedirect, + pageExists, + parseDynamicRouteMoves, + type Redirect, +} from "./lib/check-redirects-utils"; // Colors for terminal output const colors = { @@ -31,31 +42,12 @@ const autoFix = args.includes("--auto-fix"); const stagedOnly = args.includes("--staged-only"); const baseBranch = args.find((arg) => !arg.startsWith("--")) || "main"; -const CONFIG_FILE = "next.config.ts"; - -// Magic number constant for "return [" offset -const RETURN_BRACKET_LENGTH = 8; +const REDIRECTS_FILE = "redirects.ts"; // Top-level regex patterns for performance -const APP_LOCALE_PREFIX_REGEX = /^app\/[a-z]{2}\//; -const PAGE_FILE_SUFFIX_REGEX = /\/?page\.mdx?$/; -const LOCALE_PREFIX_REGEX = /^\/:locale\/?/; const PAGE_FILE_MATCH_REGEX = /page\.mdx?$/; const LOCALE_PATH_PREFIX_REGEX = /^\/:locale\//; -const WILDCARD_PATH_REGEX = /\/:path\*.*$/; -const MDX_EXTENSION_REGEX = /\.mdx$/; const SPECIAL_REGEX_CHARS_REGEX = /[.*+?^${}()|[\]\\]/g; -const REDIRECT_REGEX = - /\{\s*source:\s*["']([^"']+)["']\s*,\s*destination:\s*["']([^"']+)["']/g; -const REVERSED_REDIRECT_REGEX = - /\{\s*destination:\s*["']([^"']+)["']\s*,\s*source:\s*["']([^"']+)["']/g; -const DYNAMIC_ROUTE_REGEX = /\[[^\]]+\]/; - -type Redirect = { - source: string; - destination: string; - permanent?: boolean; -}; type RedirectChain = { source: string; @@ -63,142 +55,6 @@ type RedirectChain = { newDest: string; }; -type DynamicRouteMove = { - oldPath: string; - newPath: string; - oldUrl: string; - newUrl: string; -}; - -/** - * Convert file path to URL path - * e.g., app/en/guides/foo/page.mdx -> /:locale/guides/foo - */ -function fileToUrl(filePath: string): string { - const urlPath = filePath - .replace(APP_LOCALE_PREFIX_REGEX, "") - .replace(PAGE_FILE_SUFFIX_REGEX, ""); - - return urlPath ? `/:locale/${urlPath}` : "/:locale"; -} - -/** - * Convert URL path to file path - * e.g., /:locale/guides/foo -> app/en/guides/foo/page.mdx - */ -function urlToFile(urlPath: string): string { - const pathWithoutLocale = urlPath.replace(LOCALE_PREFIX_REGEX, ""); - return pathWithoutLocale - ? `app/en/${pathWithoutLocale}/page.mdx` - : "app/en/page.mdx"; -} - -/** - * Check if a dynamic route exists that could serve this URL path. - * e.g., for /resources/integrations/productivity/gmail, - * check if /resources/integrations/productivity/[toolkitId]/page.mdx exists - */ -function dynamicRouteExists(urlPath: string): boolean { - const pathWithoutLocale = urlPath.replace(LOCALE_PREFIX_REGEX, ""); - const segments = pathWithoutLocale.split("/").filter(Boolean); - - // Try replacing the last segment with common dynamic route patterns - const dynamicPatterns = ["[toolkitId]", "[slug]", "[id]", "[...slug]"]; - - for (let i = segments.length - 1; i >= 0; i--) { - for (const pattern of dynamicPatterns) { - const testSegments = [...segments]; - testSegments[i] = pattern; - const testPath = `app/en/${testSegments.join("/")}/page.mdx`; - if (existsSync(testPath)) { - return true; - } - const testPathMd = testPath.replace(MDX_EXTENSION_REGEX, ".md"); - if (existsSync(testPathMd)) { - return true; - } - } - } - - return false; -} - -/** - * Check if a page exists on disk - */ -function pageExists(urlPath: string): boolean { - if (urlPath.includes(":path*") || urlPath.includes(":path")) { - return true; - } - - const filePath = urlToFile(urlPath); - if (existsSync(filePath)) { - return true; - } - - const mdPath = filePath.replace(MDX_EXTENSION_REGEX, ".md"); - if (existsSync(mdPath)) { - return true; - } - - // Check if a dynamic route could serve this URL - if (dynamicRouteExists(urlPath)) { - return true; - } - - return false; -} - -/** - * Execute regex and collect all matches (avoids assignment in expression) - */ -function collectRegexMatches( - regex: RegExp, - content: string, - sourceIndex: number, - destIndex: number -): Array<{ source: string; destination: string }> { - const results: Array<{ source: string; destination: string }> = []; - regex.lastIndex = 0; - - let match = regex.exec(content); - while (match !== null) { - results.push({ - source: match[sourceIndex], - destination: match[destIndex], - }); - match = regex.exec(content); - } - - return results; -} - -/** - * Parse redirects from next.config.ts - */ -function parseRedirects(content: string): Redirect[] { - const results: Redirect[] = []; - - // Collect standard format: { source: "...", destination: "..." } - const standardMatches = collectRegexMatches(REDIRECT_REGEX, content, 1, 2); - for (const m of standardMatches) { - results.push(m); - } - - // Collect reversed format: { destination: "...", source: "..." } - const reversedMatches = collectRegexMatches( - REVERSED_REDIRECT_REGEX, - content, - 2, - 1 - ); - for (const m of reversedMatches) { - results.push(m); - } - - return results; -} - /** * Parse git diff output for deleted and renamed files */ @@ -227,77 +83,6 @@ function parseGitDiffOutput( } } -/** - * Convert a file path containing a dynamic route to a URL pattern. - * Replaces [param] with :param and [...param] with :param* - * e.g., app/en/resources/[toolkitId]/page.mdx -> /:locale/resources/:toolkitId - */ -function dynamicFileToUrlPattern(filePath: string): string { - const urlPath = filePath - .replace(APP_LOCALE_PREFIX_REGEX, "") - .replace(PAGE_FILE_SUFFIX_REGEX, ""); - - // Replace [...param] with :param* (catch-all routes) - // Replace [param] with :param (dynamic segments) - const patternPath = urlPath - .replace(/\[\.\.\.([^\]]+)\]/g, ":$1*") - .replace(/\[([^\]]+)\]/g, ":$1"); - - return patternPath ? `/:locale/${patternPath}` : "/:locale"; -} - -/** - * Parse git diff output for renamed dynamic route page files. - * Detects when a page.mdx inside a dynamic route folder is moved. - */ -function parseDynamicRouteMoves( - output: string, - moves: DynamicRouteMove[] -): void { - for (const line of output.split("\n")) { - if (!line) { - continue; - } - const parts = line.split("\t"); - const status = parts[0]; - - // Only look at renames (R followed by similarity percentage) - if (!status?.startsWith("R")) { - continue; - } - - const oldPath = parts[1]; - const newPath = parts[2]; - - if (!oldPath || !newPath) { - continue; - } - - // Check if either path contains a dynamic route segment - const oldHasDynamic = DYNAMIC_ROUTE_REGEX.test(oldPath); - const newHasDynamic = DYNAMIC_ROUTE_REGEX.test(newPath); - - // We care about moves where the URL pattern changes - if (!PAGE_FILE_MATCH_REGEX.test(oldPath)) { - continue; - } - - const oldUrl = dynamicFileToUrlPattern(oldPath); - const newUrl = dynamicFileToUrlPattern(newPath); - - // Skip if the URL pattern hasn't actually changed - if (oldUrl === newUrl) { - continue; - } - - // Record the move if either path has a dynamic route - // or if the directory structure changed significantly - if (oldHasDynamic || newHasDynamic) { - moves.push({ oldPath, newPath, oldUrl, newUrl }); - } - } -} - /** * Get moved dynamic routes by comparing branches */ @@ -312,7 +97,7 @@ function getMovedDynamicRoutes( const stagedChanges = execSync("git diff --cached --name-status", { encoding: "utf-8", }); - parseDynamicRouteMoves(stagedChanges, moves); + moves.push(...parseDynamicRouteMoves(stagedChanges)); } catch { // Ignore errors } @@ -324,7 +109,7 @@ function getMovedDynamicRoutes( `git diff --name-status ${branch}...HEAD`, { encoding: "utf-8" } ); - parseDynamicRouteMoves(committedChanges, moves); + moves.push(...parseDynamicRouteMoves(committedChanges)); } catch { // Ignore errors } @@ -333,7 +118,7 @@ function getMovedDynamicRoutes( const uncommittedChanges = execSync("git diff --name-status HEAD", { encoding: "utf-8", }); - parseDynamicRouteMoves(uncommittedChanges, moves); + moves.push(...parseDynamicRouteMoves(uncommittedChanges)); } catch { // Ignore errors } @@ -350,36 +135,6 @@ function getMovedDynamicRoutes( }); } -/** - * Check if a wildcard redirect already covers a dynamic route move - */ -function isMoveCoveredByRedirect( - move: DynamicRouteMove, - redirects: Redirect[] -): boolean { - // Check for exact match or wildcard that covers the path - for (const redirect of redirects) { - // Exact pattern match - if (redirect.source === move.oldUrl) { - return true; - } - - // Check if a wildcard redirect covers this path - if (redirect.source.includes(":path*")) { - const prefix = redirect.source - .replace(WILDCARD_PATH_REGEX, "") - .replace(LOCALE_PATH_PREFIX_REGEX, ""); - const movePrefix = move.oldUrl.replace(LOCALE_PATH_PREFIX_REGEX, ""); - - if (movePrefix.startsWith(`${prefix}/`) || movePrefix === prefix) { - return true; - } - } - } - - return false; -} - /** * Ensure base branch exists locally */ @@ -457,30 +212,6 @@ function getDeletedAndRenamedFiles( }; } -/** - * Check if a wildcard redirect covers a path - */ -function checkWildcardMatch(path: string, redirectList: Redirect[]): boolean { - const pathWithoutLocale = path.replace(LOCALE_PATH_PREFIX_REGEX, ""); - - for (const redirect of redirectList) { - if (redirect.source.includes(":path*")) { - const prefix = redirect.source - .replace(WILDCARD_PATH_REGEX, "") - .replace(LOCALE_PATH_PREFIX_REGEX, ""); - - if ( - pathWithoutLocale.startsWith(`${prefix}/`) || - pathWithoutLocale === prefix - ) { - return true; - } - } - } - - return false; -} - /** * Find the final destination in a redirect chain (follows all hops) * Returns null if the path doesn't redirect anywhere @@ -510,30 +241,32 @@ function findFinalRedirectDestination( } /** - * Insert redirect entries into next.config.ts + * Insert redirect entries into redirects.ts, just before the closing `];` of + * the `redirects` array (i.e. below the "Auto-added redirects" comment). + * This is the single append point for every auto-fix run, rather than the + * next.config.ts approach of inserting at the top of the array each time. */ function insertRedirects(entries: string[]): void { - const content = readFileSync(CONFIG_FILE, "utf-8"); + const content = readFileSync(REDIRECTS_FILE, "utf-8"); - const insertPoint = content.indexOf("return ["); + const insertPoint = content.lastIndexOf("\n];"); if (insertPoint === -1) { - console.error(colors.red("ERROR: Could not find 'return [' in config")); + console.error( + colors.red(`ERROR: Could not find closing '];' in ${REDIRECTS_FILE}`) + ); process.exit(1); } - const beforeReturn = content.substring( - 0, - insertPoint + RETURN_BRACKET_LENGTH - ); - const afterReturn = content.substring(insertPoint + RETURN_BRACKET_LENGTH); + const before = content.slice(0, insertPoint); + const after = content.slice(insertPoint); - const newContent = `${beforeReturn}\n // Auto-added redirects for deleted pages\n${entries.join("\n")}${afterReturn}`; + const newContent = `${before}\n${entries.join("\n")}${after}`; - writeFileSync(CONFIG_FILE, newContent); + writeFileSync(REDIRECTS_FILE, newContent); } /** - * Update a redirect destination in the config + * Update a redirect destination in redirects.ts */ function updateRedirectDestination( oldDest: string, @@ -555,8 +288,9 @@ console.log("Checking for deleted markdown files without redirects..."); console.log(`Comparing current branch to: ${baseBranch}`); console.log(""); -const configContent = readFileSync(CONFIG_FILE, "utf-8"); -const redirects = parseRedirects(configContent); +// Copy the imported entries into plain objects so PART 1b can update +// `destination` in place without mutating the imported module's array. +const redirects: Redirect[] = configuredRedirects.map((r) => ({ ...r })); let exitCode = 0; const invalidRedirects: string[] = []; @@ -565,7 +299,9 @@ const chains: RedirectChain[] = []; // ============================================================ // PART 1: Validate existing redirects // ============================================================ -console.log(colors.blue(`Validating existing redirects in ${CONFIG_FILE}...`)); +console.log( + colors.blue(`Validating existing redirects in ${REDIRECTS_FILE}...`) +); console.log(""); for (const redirect of redirects) { @@ -624,22 +360,33 @@ if (chains.length > 0) { ); console.log(""); - let updatedConfig = configContent; + let updatedRedirectsFile = readFileSync(REDIRECTS_FILE, "utf-8"); for (const chain of chains) { console.log(`${colors.green(" ✓")} ${chain.source}`); console.log(` was: ${chain.oldDest}`); console.log(` now: ${chain.newDest}`); - updatedConfig = updateRedirectDestination( + updatedRedirectsFile = updateRedirectDestination( chain.oldDest, chain.newDest, - updatedConfig + updatedRedirectsFile ); + + // Mirror the same (deliberately global, not source-scoped) replacement + // in memory so later parts see the collapsed destinations without + // re-reading the file. + for (const r of redirects) { + if (r.destination === chain.oldDest) { + r.destination = chain.newDest; + } + } } - writeFileSync(CONFIG_FILE, updatedConfig); + writeFileSync(REDIRECTS_FILE, updatedRedirectsFile); console.log(""); - console.log(colors.green(`✓ Redirect chains collapsed in ${CONFIG_FILE}`)); + console.log( + colors.green(`✓ Redirect chains collapsed in ${REDIRECTS_FILE}`) + ); console.log(""); } else { console.log( @@ -678,7 +425,9 @@ console.log(""); const missingRedirects: string[] = []; const suggestedEntries: string[] = []; -const latestRedirects = parseRedirects(readFileSync(CONFIG_FILE, "utf-8")); +// `redirects` already reflects PART 1b's chain collapses (see above), so it +// doubles as "the latest known state" without re-reading the file. +const latestRedirects = redirects; for (const file of allDeletedOrRenamed) { const urlPath = fileToUrl(file); @@ -701,11 +450,11 @@ for (const file of allDeletedOrRenamed) { console.log(colors.red(`✗ Missing redirect for: ${urlPath}`)); missingRedirects.push(urlPath); - suggestedEntries.push(` { - source: "${urlPath}", - destination: "/:locale/REPLACE_WITH_NEW_PATH", - permanent: true, - },`); + suggestedEntries.push(` { + source: "${urlPath}", + destination: "/:locale/REPLACE_WITH_NEW_PATH", + permanent: true, + },`); exitCode = 1; } @@ -725,7 +474,7 @@ if (missingRedirects.length > 0) { ); console.log( colors.blue( - `Auto-fixing: Adding ${missingRedirects.length} redirect(s) to ${CONFIG_FILE}` + `Auto-fixing: Adding ${missingRedirects.length} redirect(s) to ${REDIRECTS_FILE}` ) ); console.log( @@ -737,7 +486,7 @@ if (missingRedirects.length > 0) { insertRedirects(suggestedEntries); - console.log(colors.green(`✓ Added redirect entries to ${CONFIG_FILE}`)); + console.log(colors.green(`✓ Added redirect entries to ${REDIRECTS_FILE}`)); console.log(""); console.log( colors.red( @@ -764,7 +513,7 @@ if (missingRedirects.length > 0) { } console.log(""); console.log( - `Open ${CONFIG_FILE} and search for 'REPLACE_WITH_NEW_PATH' to find them.` + `Open ${REDIRECTS_FILE} and search for 'REPLACE_WITH_NEW_PATH' to find them.` ); console.log(""); @@ -787,7 +536,7 @@ if (missingRedirects.length > 0) { ); console.log(""); console.log( - "When you delete a markdown file, you must add a redirect in next.config.ts" + "When you delete a markdown file, you must add a redirect in redirects.ts" ); console.log( "to prevent broken links for users who have bookmarked the old URL." @@ -799,9 +548,7 @@ if (missingRedirects.length > 0) { } console.log(""); console.log( - colors.yellow( - "Add the following to the redirects array in next.config.ts:" - ) + colors.yellow("Add the following to the redirects array in redirects.ts:") ); console.log(""); for (const entry of suggestedEntries) { @@ -830,7 +577,7 @@ if (invalidRedirects.length > 0) { } console.log(""); console.log(colors.yellow("How to fix:")); - console.log(" 1. Open next.config.ts"); + console.log(" 1. Open redirects.ts"); console.log(" 2. Find the redirect(s) listed above"); console.log(" 3. Update the destination to a valid page path"); console.log(" (Check that the path exists under app/en/)"); @@ -871,11 +618,11 @@ if (uncoveredMoves.length > 0) { console.log(colors.blue(` → ${move.newPath}`)); console.log(""); console.log(colors.yellow(" Suggested redirect:")); - console.log(` { - source: "${move.oldUrl}/:path*", - destination: "${move.newUrl}/:path*", - permanent: true, - },`); + console.log(` { + source: "${move.oldUrl}/:path*", + destination: "${move.newUrl}/:path*", + permanent: true, + },`); console.log(""); } diff --git a/scripts/generate-llmstxt.ts b/scripts/generate-llmstxt.ts index a515a30d4..8f14c3088 100644 --- a/scripts/generate-llmstxt.ts +++ b/scripts/generate-llmstxt.ts @@ -5,6 +5,11 @@ import chalk from "chalk"; import glob from "fast-glob"; import OpenAI from "openai"; import { getToolkitCanonicalPath } from "../app/_lib/toolkit-static-params"; +import { resolveToolkitDataDir } from "../toolkit-docs-generator/src/shared/toolkit-data-dir"; +import type { + MergedToolkit, + MergedToolkitMetadata, +} from "../toolkit-docs-generator/src/shared/toolkit-schemas"; type PageMetadata = { path: string; @@ -202,29 +207,28 @@ async function discoverMdxPages(): Promise { return pages; } -const TOOLKIT_DATA_DIR = path.join( - process.cwd(), - "toolkit-docs-generator", - "data", - "toolkits" -); +const TOOLKIT_DATA_DIR = resolveToolkitDataDir(); const MAX_TOOLKIT_DESCRIPTION = 280; const MARKDOWN_LINK_REGEX = /\[([^\]]+)\]\([^)]+\)/g; const MARKDOWN_NOISE_REGEX = /[#*`>]/g; const WHITESPACE_REGEX = /\s+/g; -type ToolkitData = { - id?: string; - label?: string; - description?: string; - summary?: string; +/** + * This script only reads a handful of fields off each toolkit JSON file (it + * doesn't validate the whole document), so it declares the subset it needs + * as a `Pick` off the real generator schema types rather than re-describing + * the shape by hand. + */ +type ToolkitData = Partial< + Pick +> & { tools?: unknown[]; - metadata?: { - category?: string; - docsLink?: string; - isHidden?: boolean; - isComingSoon?: boolean; - }; + metadata?: Partial< + Pick< + MergedToolkitMetadata, + "category" | "docsLink" | "isHidden" | "isComingSoon" + > + >; }; /** diff --git a/toolkit-docs-generator/scripts/check-redirects-utils.ts b/scripts/lib/check-redirects-utils.ts similarity index 76% rename from toolkit-docs-generator/scripts/check-redirects-utils.ts rename to scripts/lib/check-redirects-utils.ts index 0259cbb95..48133192d 100644 --- a/toolkit-docs-generator/scripts/check-redirects-utils.ts +++ b/scripts/lib/check-redirects-utils.ts @@ -1,30 +1,32 @@ /** - * Utility functions for check-redirects.ts - * Extracted for testability. + * Shared helpers for scripts/check-redirects.ts. + * + * Split out so the logic that maps file paths to URLs, checks whether a + * page still exists on disk, and matches redirects against moved files can + * be unit tested without shelling out to git or touching the real + * filesystem. */ import { existsSync } from "node:fs"; -// Regex patterns -export const APP_LOCALE_PREFIX_REGEX = /^app\/[a-z]{2}\//; -export const PAGE_FILE_SUFFIX_REGEX = /\/?page\.mdx?$/; -export const LOCALE_PREFIX_REGEX = /^\/:locale\/?/; -export const PAGE_FILE_MATCH_REGEX = /page\.mdx?$/; -export const LOCALE_PATH_PREFIX_REGEX = /^\/:locale\//; -export const WILDCARD_PATH_REGEX = /\/:path\*.*$/; -export const MDX_EXTENSION_REGEX = /\.mdx$/; -export const DYNAMIC_ROUTE_REGEX = /\[[^\]]+\]/; -export const REDIRECT_REGEX = - /\{\s*source:\s*["']([^"']+)["']\s*,\s*destination:\s*["']([^"']+)["']/g; -export const REVERSED_REDIRECT_REGEX = - /\{\s*destination:\s*["']([^"']+)["']\s*,\s*source:\s*["']([^"']+)["']/g; - +// `permanent` is optional here (unlike redirects.ts's `Redirect`, where it's +// required) because none of these helpers read it — they only match on +// `source`/`destination`. A `redirects.ts` entry satisfies this type as-is. export type Redirect = { source: string; destination: string; permanent?: boolean; }; +const APP_LOCALE_PREFIX_REGEX = /^app\/[a-z]{2}\//; +const PAGE_FILE_SUFFIX_REGEX = /\/?page\.mdx?$/; +const LOCALE_PREFIX_REGEX = /^\/:locale\/?/; +const PAGE_FILE_MATCH_REGEX = /page\.mdx?$/; +const LOCALE_PATH_PREFIX_REGEX = /^\/:locale\//; +const WILDCARD_PATH_REGEX = /\/:path\*.*$/; +const MDX_EXTENSION_REGEX = /\.mdx$/; +const DYNAMIC_ROUTE_REGEX = /\[[^\]]+\]/; + export type DynamicRouteMove = { oldPath: string; newPath: string; @@ -200,10 +202,10 @@ export function parseDynamicRouteMoves(output: string): DynamicRouteMove[] { */ export function isMoveCoveredByRedirect( move: DynamicRouteMove, - redirects: Redirect[] + redirectList: Redirect[] ): boolean { // Check for exact match or wildcard that covers the path - for (const redirect of redirects) { + for (const redirect of redirectList) { // Exact pattern match if (redirect.source === move.oldUrl) { return true; @@ -225,56 +227,6 @@ export function isMoveCoveredByRedirect( return false; } -/** - * Execute regex and collect all matches - */ -export function collectRegexMatches( - regex: RegExp, - content: string, - sourceIndex: number, - destIndex: number -): Array<{ source: string; destination: string }> { - const results: Array<{ source: string; destination: string }> = []; - regex.lastIndex = 0; - - let match = regex.exec(content); - while (match !== null) { - results.push({ - source: match[sourceIndex], - destination: match[destIndex], - }); - match = regex.exec(content); - } - - return results; -} - -/** - * Parse redirects from next.config.ts content - */ -export function parseRedirects(content: string): Redirect[] { - const results: Redirect[] = []; - - // Collect standard format: { source: "...", destination: "..." } - const standardMatches = collectRegexMatches(REDIRECT_REGEX, content, 1, 2); - for (const m of standardMatches) { - results.push(m); - } - - // Collect reversed format: { destination: "...", source: "..." } - const reversedMatches = collectRegexMatches( - REVERSED_REDIRECT_REGEX, - content, - 2, - 1 - ); - for (const m of reversedMatches) { - results.push(m); - } - - return results; -} - /** * Check if a wildcard redirect covers a path */ diff --git a/scripts/update-internal-links.ts b/scripts/update-internal-links.ts index da10d93c9..e960a2702 100644 --- a/scripts/update-internal-links.ts +++ b/scripts/update-internal-links.ts @@ -6,12 +6,13 @@ * Usage: * pnpm update-links [--dry-run] * - * This script reads redirects from next.config.ts and updates any internal links + * This script reads redirects from redirects.ts and updates any internal links * in MDX/TSX files that point to redirected paths. */ import { readFileSync, writeFileSync } from "node:fs"; import fg from "fast-glob"; +import { redirects as configuredRedirects } from "../redirects"; // Colors for terminal output const colors = { @@ -24,71 +25,15 @@ const colors = { // Parse command line arguments const dryRun = process.argv.includes("--dry-run"); -const CONFIG_FILE = "next.config.ts"; - // Top-level regex patterns const LOCALE_PREFIX_REGEX = /^\/:locale/; const SPECIAL_REGEX_CHARS_REGEX = /[.*+?^${}()|[\]\\]/g; -const REDIRECT_REGEX = - /\{\s*source:\s*["']([^"']+)["']\s*,\s*destination:\s*["']([^"']+)["']/g; -const REVERSED_REDIRECT_REGEX = - /\{\s*destination:\s*["']([^"']+)["']\s*,\s*source:\s*["']([^"']+)["']/g; type Redirect = { source: string; destination: string; }; -/** - * Execute regex and collect all matches (avoids assignment in expression) - */ -function collectRegexMatches( - regex: RegExp, - content: string, - sourceIndex: number, - destIndex: number -): Array<{ source: string; destination: string }> { - const results: Array<{ source: string; destination: string }> = []; - regex.lastIndex = 0; - - let match = regex.exec(content); - while (match !== null) { - results.push({ - source: match[sourceIndex], - destination: match[destIndex], - }); - match = regex.exec(content); - } - - return results; -} - -/** - * Parse redirects from next.config.ts - */ -function parseRedirects(content: string): Redirect[] { - const results: Redirect[] = []; - - // Collect standard format: { source: "...", destination: "..." } - const standardMatches = collectRegexMatches(REDIRECT_REGEX, content, 1, 2); - for (const m of standardMatches) { - results.push(m); - } - - // Collect reversed format: { destination: "...", source: "..." } - const reversedMatches = collectRegexMatches( - REVERSED_REDIRECT_REGEX, - content, - 2, - 1 - ); - for (const m of reversedMatches) { - results.push(m); - } - - return results; -} - /** * Filter redirects to only those that can be auto-updated */ @@ -199,11 +144,9 @@ if (dryRun) { console.log(""); } -console.log(colors.blue(`Parsing redirects from ${CONFIG_FILE}...`)); +console.log(colors.blue("Parsing redirects from redirects.ts...")); -const configContent = readFileSync(CONFIG_FILE, "utf-8"); -const allRedirects = parseRedirects(configContent); -const redirects = getUpdatableRedirects(allRedirects); +const redirects = getUpdatableRedirects(configuredRedirects); console.log( `Found ${colors.green(String(redirects.length))} non-wildcard redirects to check` diff --git a/tests/integration-category-routes.test.ts b/tests/integration-category-routes.test.ts new file mode 100644 index 000000000..13a77434f --- /dev/null +++ b/tests/integration-category-routes.test.ts @@ -0,0 +1,58 @@ +import { existsSync, mkdirSync, mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { describe, expect, test } from "vitest"; +import { INTEGRATION_CATEGORIES } from "@/toolkit-docs-generator/src/shared/toolkit-primitives"; + +const INTEGRATIONS_APP_DIR = join( + process.cwd(), + "app", + "en", + "resources", + "integrations" +); + +/** + * normalizeCategory (app/_lib/toolkit-static-params.ts) trusts that every + * value in INTEGRATION_CATEGORIES has a real `[toolkitId]` route directory + * to route toolkits into. If a category is ever added to that list without + * the matching directory (or a directory is removed/renamed), toolkits in + * that category become clickable catalog cards pointing at a route that + * 404s — the same class of bug the "others" catch-all used to hide, since + * tests/integration-index-links.test.ts derives its notion of "valid link" + * from the same normalizeCategory output and can't see this gap. + */ +const missingCategoryDirs = (baseDir: string): string[] => + INTEGRATION_CATEGORIES.filter( + (category) => !existsSync(join(baseDir, category, "[toolkitId]")) + ); + +describe("integration category route directories", () => { + test("every INTEGRATION_CATEGORIES value has a matching [toolkitId] route directory", () => { + expect(missingCategoryDirs(INTEGRATIONS_APP_DIR)).toEqual([]); + }); + + test("the check fails when a category's route directory is missing", () => { + // Proves the check above actually catches drift, without touching any + // tracked directory: build a scratch tree with every category present, + // then remove one and confirm it's flagged. + const scratchDir = mkdtempSync(join(tmpdir(), "integration-categories-")); + try { + for (const category of INTEGRATION_CATEGORIES) { + mkdirSync(join(scratchDir, category, "[toolkitId]"), { + recursive: true, + }); + } + + const removedCategory = INTEGRATION_CATEGORIES[0]; + rmSync(join(scratchDir, removedCategory, "[toolkitId]"), { + recursive: true, + force: true, + }); + + expect(missingCategoryDirs(scratchDir)).toEqual([removedCategory]); + } finally { + rmSync(scratchDir, { recursive: true, force: true }); + } + }); +}); diff --git a/tests/integration-index-links.test.ts b/tests/integration-index-links.test.ts index de50d8f9d..45775c334 100644 --- a/tests/integration-index-links.test.ts +++ b/tests/integration-index-links.test.ts @@ -9,16 +9,17 @@ import { toIntegrationLink, } from "@/app/_lib/integration-index"; import { readToolkitData } from "@/app/_lib/toolkit-data"; -import { - getToolkitSlug, - type ToolkitWithDocsLink, -} from "@/app/_lib/toolkit-slug"; +import type { ToolkitWithDocsLink } from "@/app/_lib/toolkit-slug"; import { getToolkitCanonicalPath, - INTEGRATION_CATEGORIES, listToolkitRoutes, listValidIntegrationLinks, } from "@/app/_lib/toolkit-static-params"; +import { + getToolkitSlug, + INTEGRATION_CATEGORIES, +} from "@/toolkit-docs-generator/src/shared/toolkit-primitives"; +import { redirects } from "../redirects"; const TIMEOUT = 30_000; const ROOT = process.cwd(); @@ -198,13 +199,8 @@ const pageFileExists = (path: string): boolean => { ); }; -const readRedirectSources = async (): Promise> => { - const config = await readFile(join(ROOT, "next.config.ts"), "utf-8"); - const sources = [...config.matchAll(/source:\s*"([^"]+)"/g)].map( - (match) => match[1] - ); - return new Set(sources); -}; +const readRedirectSources = (): Set => + new Set(redirects.map((redirect) => redirect.source)); const extractInternalHrefs = async (relPath: string): Promise => { const content = await readFile(join(ROOT, relPath), "utf-8"); diff --git a/toolkit-docs-generator/tests/scripts/check-redirects-utils.test.ts b/tests/scripts/check-redirects-utils.test.ts similarity index 86% rename from toolkit-docs-generator/tests/scripts/check-redirects-utils.test.ts rename to tests/scripts/check-redirects-utils.test.ts index 47c0018e7..405d8caee 100644 --- a/toolkit-docs-generator/tests/scripts/check-redirects-utils.test.ts +++ b/tests/scripts/check-redirects-utils.test.ts @@ -8,10 +8,9 @@ import { isMoveCoveredByRedirect, pageExists, parseDynamicRouteMoves, - parseRedirects, type Redirect, urlToFile, -} from "../../scripts/check-redirects-utils"; +} from "../../scripts/lib/check-redirects-utils"; describe("fileToUrl", () => { it("converts file path to URL path", () => { @@ -313,79 +312,6 @@ describe("isMoveCoveredByRedirect", () => { }); }); -describe("parseRedirects", () => { - it("parses standard format redirects", () => { - const content = ` - { - source: "/:locale/old", - destination: "/:locale/new", - permanent: true, - }, - `; - - const redirects = parseRedirects(content); - - expect(redirects).toHaveLength(1); - expect(redirects[0]).toEqual({ - source: "/:locale/old", - destination: "/:locale/new", - }); - }); - - it("parses reversed format redirects", () => { - const content = ` - { - destination: "/:locale/new", - source: "/:locale/old", - permanent: true, - }, - `; - - const redirects = parseRedirects(content); - - expect(redirects).toHaveLength(1); - expect(redirects[0]).toEqual({ - source: "/:locale/old", - destination: "/:locale/new", - }); - }); - - it("parses multiple redirects", () => { - const content = ` - { - source: "/:locale/a", - destination: "/:locale/b", - }, - { - source: "/:locale/c", - destination: "/:locale/d", - }, - `; - - const redirects = parseRedirects(content); - - expect(redirects).toHaveLength(2); - }); - - it("handles single quotes", () => { - const content = ` - { - source: '/:locale/old', - destination: '/:locale/new', - }, - `; - - const redirects = parseRedirects(content); - - expect(redirects).toHaveLength(1); - expect(redirects[0].source).toBe("/:locale/old"); - }); - - it("handles empty content", () => { - expect(parseRedirects("")).toHaveLength(0); - }); -}); - describe("checkWildcardMatch", () => { it("returns true when wildcard prefix matches", () => { const redirects: Redirect[] = [ diff --git a/tests/sitemap.test.ts b/tests/sitemap.test.ts index d8daf65cc..ded32cc7b 100644 --- a/tests/sitemap.test.ts +++ b/tests/sitemap.test.ts @@ -1,6 +1,7 @@ import { readFileSync } from "node:fs"; import { join } from "node:path"; import { expect, test } from "vitest"; +import { redirects } from "../redirects"; test("sitemap lists expected URLs", async () => { const previousSiteUrl = process.env.SITE_URL; @@ -55,16 +56,15 @@ test("sitemap contains no URL that we redirect away", async () => { entry.url.replace("https://example.test", "") ); - // Every redirect `source` in next.config.ts is a path we 3xx away, so a live + // Every redirect `source` in redirects.ts is a path we 3xx away, so a live // page must never sit there — otherwise the sitemap ships a redirecting URL // (Ahrefs flags "3XX redirect in sitemap"). Guards against pages left behind // after a rename. - const config = readFileSync(join(process.cwd(), "next.config.ts"), "utf-8"); const exactSources = new Set(); const prefixSources: string[] = []; - for (const match of config.matchAll(/source:\s*"([^"]+)"/g)) { - const normalized = match[1] + for (const redirect of redirects) { + const normalized = redirect.source .replace(/:locale\([^)]*\)/g, "en") .replace(/:locale/g, "en"); @@ -90,7 +90,7 @@ test("sitemap contains no URL that we redirect away", async () => { for (const offender of offenders) { console.error( - `Sitemap includes ${offender}, which matches a redirect source in next.config.ts. ` + + `Sitemap includes ${offender}, which matches a redirect source in redirects.ts. ` + "Delete the stale page (or remove the redirect) so the sitemap ships no 3XX URLs." ); } diff --git a/tests/toolkit-data-cache.test.ts b/tests/toolkit-data-cache.test.ts new file mode 100644 index 000000000..45d9ac618 --- /dev/null +++ b/tests/toolkit-data-cache.test.ts @@ -0,0 +1,118 @@ +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterAll, describe, expect, test } from "vitest"; +import { readToolkitData } from "@/app/_lib/toolkit-data"; + +/** + * loadAllToolkitData (app/_lib/toolkit-data.ts) reads and validates every + * toolkit file in a data directory once, then serves all lookups from the + * resulting map. That eager read means one corrupt file can no longer be + * skipped by requesting a different, healthy toolkit — the whole directory + * load fails, and every lookup against it throws. These tests pin that + * behavior down explicitly, since it's a real change from the old + * direct-file-then-scan implementation (a corrupt sibling file was + * previously invisible to a direct hit). + */ + +const validToolkitJson = (id: string, docsSlug: string): string => + JSON.stringify({ + id, + label: id, + version: "1.0.0", + description: "A test toolkit fixture.", + metadata: { + category: "development", + iconUrl: "https://example.com/icon.svg", + isBYOC: false, + isPro: false, + type: "arcade", + docsLink: `https://docs.arcade.dev/en/resources/integrations/development/${docsSlug}`, + isComingSoon: false, + isHidden: false, + }, + auth: null, + tools: [], + }); + +const makeFixtureDir = (): string => { + const dir = mkdtempSync(join(tmpdir(), "toolkit-data-cache-test-")); + writeFileSync( + join(dir, "validtoolkitone.json"), + validToolkitJson("ValidToolkitOne", "valid-toolkit-one") + ); + writeFileSync( + join(dir, "validtoolkittwo.json"), + validToolkitJson("ValidToolkitTwo", "valid-toolkit-two") + ); + return dir; +}; + +const dirsToClean: string[] = []; + +afterAll(() => { + for (const dir of dirsToClean) { + rmSync(dir, { recursive: true, force: true }); + } +}); + +describe("readToolkitData against a clean fixture directory", () => { + const dataDir = makeFixtureDir(); + dirsToClean.push(dataDir); + + test("a known toolkit id resolves to its data", async () => { + const data = await readToolkitData("ValidToolkitOne", { dataDir }); + expect(data?.id).toBe("ValidToolkitOne"); + }); + + test("a known toolkit reached by its docs slug resolves to the same data", async () => { + const data = await readToolkitData("valid-toolkit-two", { dataDir }); + expect(data?.id).toBe("ValidToolkitTwo"); + }); + + test("an absent toolkit id yields null, not a throw", async () => { + const data = await readToolkitData("no-such-toolkit-at-all", { dataDir }); + expect(data).toBeNull(); + }); +}); + +describe("readToolkitData against a directory with one corrupt file", () => { + const dataDir = makeFixtureDir(); + dirsToClean.push(dataDir); + writeFileSync( + join(dataDir, "corrupttoolkit.json"), + "{ this is not valid json" + ); + + test("requesting the corrupt toolkit throws, naming the file path", async () => { + await expect( + readToolkitData("CorruptToolkit", { dataDir }) + ).rejects.toThrow(join(dataDir, "corrupttoolkit.json")); + }); + + test("the failure is cached, not retried: a second request throws the same way", async () => { + // Confirms the deliberate choice to cache a failed load rather than + // re-scanning the directory on every subsequent call: this directory's + // corruption doesn't heal between calls, so re-reading it every time + // would only add cost without ever succeeding. + await expect( + readToolkitData("CorruptToolkit", { dataDir }) + ).rejects.toThrow(join(dataDir, "corrupttoolkit.json")); + }); + + // A pre-existing property of the old scan-on-miss implementation too, not + // a regression introduced by the shared cache: any lookup that needs to + // rule out every file in the directory (a genuinely absent id, or a slug + // reached only via the full scan) surfaces a sibling file's corruption, + // because "is this id absent" can't be answered without reading everything. + // A healthy toolkit's *direct* id-shaped lookup, though, is unaffected by + // corruption elsewhere in the directory only when that toolkit was already + // resident in a load that happened before the corruption — once the whole + // directory's load has failed once, it stays failed (see the caching test + // above), so every subsequent lookup against this dataDir throws too. + test("a healthy toolkit id in the same directory also throws once the directory load has failed", async () => { + await expect( + readToolkitData("ValidToolkitOne", { dataDir }) + ).rejects.toThrow(join(dataDir, "corrupttoolkit.json")); + }); +}); diff --git a/tests/toolkit-data-parity.test.ts b/tests/toolkit-data-parity.test.ts new file mode 100644 index 000000000..ef4d5cbb2 --- /dev/null +++ b/tests/toolkit-data-parity.test.ts @@ -0,0 +1,58 @@ +import { readdirSync } from "node:fs"; +import { join } from "node:path"; +import { describe, expect, test } from "vitest"; +import { readToolkitFile, readToolkitIndex } from "@/app/_lib/toolkit-data"; +import { listToolkitRoutes } from "@/app/_lib/toolkit-static-params"; +import { resolveToolkitDataDir } from "@/toolkit-docs-generator/src/shared/toolkit-data-dir"; + +// resolveToolkitDataDir defaults to the real committed data, but also honors +// TOOLKIT_DATA_DIR (same as readToolkitIndex/listToolkitRoutes below), so +// pointing that env var at a scratch copy runs this exact test against it. +const DATA_DIR = resolveToolkitDataDir(); + +/** + * A malformed or missing nightly-generated toolkit file used to disappear + * from the site silently: readToolkitData/listToolkitRoutes swallowed the + * error and just dropped the toolkit, so index.json, the on-disk files, and + * the routes Next.js actually generates could drift apart with nothing + * failing the build. Runs against the real committed data (not a fixture) + * so it catches that drift for whatever toolkits are checked in right now. + */ +describe("toolkit data parity", () => { + test("index.json entries, parseable toolkit files, and generated routes agree", async () => { + const index = await readToolkitIndex(); + expect(index).not.toBeNull(); + + const jsonFileNames = readdirSync(DATA_DIR).filter( + (file) => file.endsWith(".json") && file !== "index.json" + ); + + // readToolkitFile throws on a corrupt file (see app/_lib/toolkit-data.ts), + // so a bad file fails this test loudly instead of quietly shrinking the + // "parseable" count below. + const toolkits = await Promise.all( + jsonFileNames.map((file) => readToolkitFile(join(DATA_DIR, file))) + ); + const parseableCount = toolkits.filter( + (toolkit) => toolkit !== null + ).length; + + // Every file on disk should be a real, schema-valid toolkit: no file + // silently failed to parse into null. + expect(parseableCount).toBe(jsonFileNames.length); + + // index.json is regenerated alongside the per-toolkit files, so its + // entry count should match the file count exactly. + expect(index?.toolkits.length).toBe(parseableCount); + + // Routes exclude hidden toolkits (they're intentionally unrouted, not + // corrupt), so compare against the non-hidden subset rather than the + // raw file count. + const visibleCount = toolkits.filter( + (toolkit) => toolkit && !toolkit.metadata?.isHidden + ).length; + + const routes = await listToolkitRoutes(); + expect(routes.length).toBe(visibleCount); + }); +}); diff --git a/toolkit-docs-generator/scripts/check-stale-summaries.ts b/toolkit-docs-generator/scripts/check-stale-summaries.ts index da093402d..73930978e 100644 --- a/toolkit-docs-generator/scripts/check-stale-summaries.ts +++ b/toolkit-docs-generator/scripts/check-stale-summaries.ts @@ -13,11 +13,10 @@ */ import { readdirSync, readFileSync } from "node:fs"; -import { dirname, join } from "node:path"; -import { fileURLToPath } from "node:url"; +import { join } from "node:path"; +import { resolveToolkitDataDir } from "../src/shared/toolkit-data-dir.ts"; -const here = dirname(fileURLToPath(import.meta.url)); -const TOOLKITS_DIR = join(here, "..", "data", "toolkits"); +const TOOLKITS_DIR = resolveToolkitDataDir(); type ToolkitShape = { id?: unknown; diff --git a/toolkit-docs-generator/scripts/report-tool-metadata.ts b/toolkit-docs-generator/scripts/report-tool-metadata.ts index 51624b8e0..428317a2a 100644 --- a/toolkit-docs-generator/scripts/report-tool-metadata.ts +++ b/toolkit-docs-generator/scripts/report-tool-metadata.ts @@ -1,16 +1,14 @@ #!/usr/bin/env node /** * CLI script to report tool metadata coverage and distinct enum values. - * Resolves data directory relative to this script, so it works regardless of cwd. + * Resolves the data directory via resolveToolkitDataDir, which works + * regardless of cwd and honors the TOOLKIT_DATA_DIR env var override. */ -import { dirname, join } from "node:path"; -import { fileURLToPath } from "node:url"; - +import { resolveToolkitDataDir } from "../src/shared/toolkit-data-dir.ts"; import { collectToolMetadataStats } from "../src/utils/tool-metadata-audit.ts"; -const __dirname = dirname(fileURLToPath(import.meta.url)); -const DATA_DIR = join(__dirname, "..", "data", "toolkits"); +const DATA_DIR = resolveToolkitDataDir(); async function main(): Promise { const stats = await collectToolMetadataStats({ dataDir: DATA_DIR }); diff --git a/toolkit-docs-generator/scripts/sync-toolkit-sidebar.ts b/toolkit-docs-generator/scripts/sync-toolkit-sidebar.ts index 01927bec6..a42ed0a25 100644 --- a/toolkit-docs-generator/scripts/sync-toolkit-sidebar.ts +++ b/toolkit-docs-generator/scripts/sync-toolkit-sidebar.ts @@ -28,6 +28,16 @@ import { import { dirname, join, resolve } from "node:path"; import { fileURLToPath } from "node:url"; import { TOOLKITS as DESIGN_SYSTEM_TOOLKITS } from "@arcadeai/design-system/metadata/toolkits"; +import { resolveToolkitDataDir } from "../src/shared/toolkit-data-dir.ts"; +import { + getToolkitSlug, + INTEGRATION_CATEGORIES, + isApiSuffixedToolkitId, +} from "../src/shared/toolkit-primitives.ts"; +import type { + MergedToolkit, + MergedToolkitMetadata, +} from "../src/shared/toolkit-schemas.ts"; const __filename = fileURLToPath(import.meta.url); const __dirname = dirname(__filename); @@ -51,7 +61,7 @@ const PROJECT_ROOT = resolve(__dirname, "..", ".."); // Configuration const CONFIG = { - dataDir: join(PROJECT_ROOT, "toolkit-docs-generator/data/toolkits"), + dataDir: resolveToolkitDataDir(), integrationsDir: join(PROJECT_ROOT, "app/en/resources/integrations"), integrationsBasePath: "/en/resources/integrations", }; @@ -71,43 +81,24 @@ const CATEGORY_NAMES: Record = { }; // Category order for main _meta.tsx -const CATEGORY_ORDER = [ - "productivity", - "social", - "entertainment", - "development", - "payments", - "search", - "sales", - "databases", - "customer-support", - "others", -]; +const CATEGORY_ORDER: readonly string[] = INTEGRATION_CATEGORIES; const CAPITAL_LETTER_REGEX = /([A-Z])/g; const FIRST_CHARACTER_REGEX = /^./; -const CAMEL_BOUNDARY = /([a-z0-9])([A-Z])/g; const IDENTIFIER_KEY_REGEX = /^[A-Za-z_$][A-Za-z0-9_$]*$/; /** - * Convert a CamelCase string to kebab-case. - * Must stay in sync with toKebabCase in app/_lib/toolkit-slug.ts. + * This script only reads a handful of fields off each toolkit JSON file (it + * doesn't validate the whole document), so it declares the subset it needs + * as a `Pick` off the real generator schema types rather than re-describing + * the shape by hand. */ -function toKebabCase(value: string): string { - return value.replace(CAMEL_BOUNDARY, "$1-$2").toLowerCase(); -} - -type ToolkitJson = { - id?: string; - label?: string; +type ToolkitJson = Partial> & { name?: string; - metadata?: { - category?: string; - docsLink?: string; - isHidden?: boolean; - type?: string; - }; + metadata?: Partial< + Pick + >; }; function renderObjectKey(key: string): string { @@ -231,21 +222,6 @@ function readToolkitJson(dataDir: string, slug: string): ToolkitJson | null { return null; } -function getDocsSlugFromLink(docsLink?: string | null): string | null { - if (!docsLink) { - return null; - } - - try { - const url = new URL(docsLink); - const segments = url.pathname.split("/").filter(Boolean); - return segments.at(-1) ?? null; - } catch { - const segments = docsLink.split("/").filter(Boolean); - return segments.at(-1) ?? null; - } -} - /** * Read toolkit JSON and extract label if available */ @@ -275,7 +251,7 @@ export function inferNavGroup( } // Heuristic fallback: "*Api" toolkits are starter. - return toolkitIdOrSlug.toLowerCase().endsWith("api") + return isApiSuffixedToolkitId(toolkitIdOrSlug) ? ("starter" as const) : ("optimized" as const); } @@ -294,8 +270,10 @@ function resolveToolkitInfo( ): ToolkitInfoEntry | null { const jsonData = readToolkitJson(dataDir, slug); const toolkitId = jsonData?.id ?? slug; - const docsSlug = - getDocsSlugFromLink(jsonData?.metadata?.docsLink) ?? toKebabCase(toolkitId); + const docsSlug = getToolkitSlug({ + id: toolkitId, + docsLink: jsonData?.metadata?.docsLink, + }); const designSystemToolkit = TOOLKITS.find( (t) => t.id.toLowerCase() === toolkitId.toLowerCase() ); diff --git a/toolkit-docs-generator/scripts/validate-merge.ts b/toolkit-docs-generator/scripts/validate-merge.ts index b51e79bfd..89d4ed2ed 100644 --- a/toolkit-docs-generator/scripts/validate-merge.ts +++ b/toolkit-docs-generator/scripts/validate-merge.ts @@ -11,22 +11,23 @@ */ import { existsSync, readdirSync, readFileSync } from "node:fs"; -import { dirname, join, resolve } from "node:path"; +import { join, resolve } from "node:path"; import { fileURLToPath } from "node:url"; +import { resolveToolkitDataDir } from "../src/shared/toolkit-data-dir.ts"; +import type { MergedToolkit } from "../src/shared/toolkit-schemas.ts"; -const __filename = fileURLToPath(import.meta.url); -const __dirname = dirname(__filename); +const DATA_DIR = resolveToolkitDataDir(); -const WORKSPACE_ROOT = join(__dirname, ".."); -const DATA_DIR = join(WORKSPACE_ROOT, "data", "toolkits"); - -type ToolkitJson = { - id: string; - label: string; - documentationChunks?: Record[]; - customImports?: string[]; - subPages?: Record[]; -}; +/** + * This script only reads a handful of fields off each toolkit JSON file (it + * doesn't validate the whole document), so it declares the subset it needs + * as a `Pick` off the real generator schema type rather than re-describing + * the shape by hand. + */ +type ToolkitJson = Pick< + MergedToolkit, + "id" | "label" | "documentationChunks" | "customImports" | "subPages" +>; export type ToolkitValidationDetail = { file: string; diff --git a/toolkit-docs-generator/src/merger/data-merger.ts b/toolkit-docs-generator/src/merger/data-merger.ts index d278b18ca..15d28def8 100644 --- a/toolkit-docs-generator/src/merger/data-merger.ts +++ b/toolkit-docs-generator/src/merger/data-merger.ts @@ -6,6 +6,10 @@ */ import type { ISecretEditGenerator } from "../llm/secret-edit-generator.js"; +import { + isApiSuffixedToolkitId, + normalizeToolkitId, +} from "../shared/toolkit-primitives.js"; import type { ICustomSectionsSource } from "../sources/interfaces.js"; import type { IToolkitDataSource, @@ -366,14 +370,10 @@ export const getProviderId = ( /** * Create default metadata for toolkits not found in Design System */ -const TOOLKIT_ID_NORMALIZER = /[^a-z0-9]/g; const TOOLKIT_ID_ACRONYM_BOUNDARY = /([A-Z]+)([A-Z][a-z])/g; const TOOLKIT_ID_WORD_BOUNDARY = /([a-z0-9])([A-Z])/g; const TOOLKIT_DESCRIPTION_LABEL_PREFIX = "Arcade.dev LLM tools for "; -const normalizeToolkitId = (toolkitId: string): string => - toolkitId.toLowerCase().replace(TOOLKIT_ID_NORMALIZER, ""); - const humanizeToolkitId = (toolkitId: string): string => toolkitId .replace(TOOLKIT_ID_ACRONYM_BOUNDARY, "$1 $2") @@ -415,9 +415,6 @@ const resolveToolkitLabel = (options: { extractLabelFromDescription(options.description) ?? humanizeToolkitId(options.toolkitId); -const isStarterToolkitId = (toolkitId: string): boolean => - normalizeToolkitId(toolkitId).endsWith("api"); - const getDefaultIconId = (toolkitId: string): string => { const normalized = normalizeToolkitId(toolkitId); // Prefer provider icons for "*Api" toolkits when possible. @@ -436,7 +433,7 @@ const applyToolkitTypeOverrides = ( toolkitId: string, metadata: MergedToolkitMetadata ): MergedToolkitMetadata => { - if (isStarterToolkitId(toolkitId) && metadata.type === "arcade") { + if (isApiSuffixedToolkitId(toolkitId) && metadata.type === "arcade") { return { ...metadata, type: "arcade_starter" }; } return metadata; @@ -1008,7 +1005,7 @@ export class DataMerger { iconUrl: "", isBYOC: false, isPro: false, - type: isStarterToolkitId(toolkitId) ? "arcade_starter" : "arcade", + type: isApiSuffixedToolkitId(toolkitId) ? "arcade_starter" : "arcade", docsLink: "", isComingSoon: false, isHidden: false, diff --git a/toolkit-docs-generator/src/shared/toolkit-data-dir.ts b/toolkit-docs-generator/src/shared/toolkit-data-dir.ts new file mode 100644 index 000000000..36bd19cf6 --- /dev/null +++ b/toolkit-docs-generator/src/shared/toolkit-data-dir.ts @@ -0,0 +1,35 @@ +/** + * Where the generated toolkit JSON lives, shared by the Next.js docs app and + * toolkit-docs-generator. Kept separate from `toolkit-primitives.ts` because + * this module reaches for `node:path` / `node:url`: the primitives are pure + * string helpers that client components pull in through the integrations + * index, and a Node built-in anywhere in that import graph fails the webpack + * browser build. + */ + +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; + +const HERE = dirname(fileURLToPath(import.meta.url)); + +/** + * toolkit-docs-generator/data/toolkits, resolved relative to this file rather + * than `process.cwd()` — correct regardless of which directory a script or the + * Next.js server happened to be started from. + */ +export const DEFAULT_TOOLKIT_DATA_DIR = join( + HERE, + "..", + "..", + "data", + "toolkits" +); + +/** + * Resolve the toolkit data directory: an explicit override wins, then the + * `TOOLKIT_DATA_DIR` env var (used by tests and CI to point at a fixture or + * scratch copy), then the real generator output directory. + */ +export function resolveToolkitDataDir(override?: string): string { + return override ?? process.env.TOOLKIT_DATA_DIR ?? DEFAULT_TOOLKIT_DATA_DIR; +} diff --git a/toolkit-docs-generator/src/shared/toolkit-primitives.ts b/toolkit-docs-generator/src/shared/toolkit-primitives.ts new file mode 100644 index 000000000..1764190d1 --- /dev/null +++ b/toolkit-docs-generator/src/shared/toolkit-primitives.ts @@ -0,0 +1,126 @@ +/** + * Toolkit primitives shared by the Next.js docs app (app/_lib and its + * consumers) and toolkit-docs-generator. Both halves need the same toolkit + * ID/slug/category logic, but the generator's tsconfig pins `rootDir` to its + * own `src/`, so a module outside that directory fails its build + * (`TS6059: File '...' is not under 'rootDir'`). Living here satisfies the + * generator's rootDir trivially, while the app side can still reach it with + * a normal relative or `@/`-aliased import — root tsconfig has no `rootDir` + * restriction, only a `toolkit-docs-generator` entry in `exclude`, which + * only affects automatic root-file discovery, not files reached via import. + * + * Everything here must stay free of Node built-ins: client components reach + * this module through the integrations index, so a `node:*` import anywhere + * in the graph fails the webpack browser build. Filesystem concerns live in + * `toolkit-data-dir.ts` instead. + */ + +// ============================================================================ +// Toolkit ID normalization +// ============================================================================ + +const TOOLKIT_ID_NORMALIZER = /[^a-z0-9]+/g; + +/** + * Strip all non-alphanumeric characters and lowercase. + * Used for case/punctuation-insensitive matching of toolkit IDs and labels + * (e.g. matching "GitHub API" against a design system entry keyed "Github"). + */ +export function normalizeToolkitId(value: string): string { + return value.toLowerCase().replace(TOOLKIT_ID_NORMALIZER, ""); +} + +/** + * Whether a toolkit ID looks like an auto-generated "*Api" wrapper toolkit + * (e.g. "GithubApi", "hubspot-crm-api", "stripe_api"). These get special- + * cased in several places: starter-type override, provider-id metadata + * fallback, and "-api"-suffixed docs slugs/icons. + */ +export function isApiSuffixedToolkitId(toolkitId: string): boolean { + return normalizeToolkitId(toolkitId).endsWith("api"); +} + +// ============================================================================ +// Slug generation +// ============================================================================ + +const CAMEL_BOUNDARY = /([a-z0-9])([A-Z])/g; + +/** + * Convert a CamelCase toolkit ID to a kebab-case URL slug. + * + * Examples: + * PosthogApi → posthog-api + * GoogleCalendar → google-calendar + * E2b → e2b + * HubspotCrmApi → hubspot-crm-api + */ +export function toKebabCase(value: string): string { + return value.replace(CAMEL_BOUNDARY, "$1-$2").toLowerCase(); +} + +export type ToolkitSlugSource = { + id: string; + docsLink?: string | null; +}; + +function extractSlugFromPath(path: string): string | null { + const segments = path.split("/").filter(Boolean); + return segments.at(-1) ?? null; +} + +/** + * The canonical docs slug for a toolkit: the last path segment of its + * `docsLink` when present (preserves hand-authored slugs like "stripe_api"), + * otherwise the kebab-case of its ID. + */ +export function getToolkitSlug({ id, docsLink }: ToolkitSlugSource): string { + if (docsLink) { + try { + const url = new URL(docsLink); + const slug = extractSlugFromPath(url.pathname); + if (slug) { + return slug; + } + } catch { + const slug = extractSlugFromPath(docsLink); + if (slug) { + return slug; + } + } + } + + return toKebabCase(id); +} + +// ============================================================================ +// Integration categories +// ============================================================================ + +/** + * The docs-generation category buckets. Each value corresponds to exactly + * one `app/en/resources/integrations//[toolkitId]` route directory + * (see tests/integration-category-routes.test.ts) and to the design system's + * own `ToolkitCategory` union (minus its "all" filter meta-value) — see + * app/en/resources/integrations/components/filter-params.ts. There is + * deliberately no "others" catch-all: a toolkit whose category doesn't match + * one of these has no page to render, so `normalizeCategory` in + * app/_lib/toolkit-static-params.ts throws instead of bucketing it here. + * + * `ToolkitCategorySchema` in ./toolkit-schemas.ts is built from this array, + * so the generator's contract and the docs app's route set can't drift + * apart. + */ +export const INTEGRATION_CATEGORIES = [ + "productivity", + "social", + "entertainment", + "development", + "payments", + "search", + "sales", + "databases", + "customer-support", +] as const; + +export type IntegrationCategory = (typeof INTEGRATION_CATEGORIES)[number]; diff --git a/toolkit-docs-generator/src/shared/toolkit-schemas.ts b/toolkit-docs-generator/src/shared/toolkit-schemas.ts new file mode 100644 index 000000000..7531105b2 --- /dev/null +++ b/toolkit-docs-generator/src/shared/toolkit-schemas.ts @@ -0,0 +1,420 @@ +/** + * Zod schemas for the merged toolkit JSON contract, shared by the Next.js + * docs app (app/_lib and app/_components/toolkit-docs) and + * toolkit-docs-generator. The generator validates its own output against + * these schemas on write (see src/generator/json-generator.ts); the app + * validates on read (see app/_lib/toolkit-data.ts) so a file that doesn't + * match this shape is rejected instead of crashing mid-render. + * + * This lives under toolkit-docs-generator/src/ (not app/_lib or a repo-root + * shared/ directory) because the generator's tsconfig pins `rootDir` to its + * own `src/`, so a shared module outside that directory fails the + * generator's build (`TS6059: File '...' is not under 'rootDir'`). The app + * reaches it via the `@/` alias, same as toolkit-primitives.ts next to this + * file. + * + * CLI-only input schemas (ProviderVersion, GenerateInput) and the raw, + * pre-merge Engine/Design-System schemas (ToolDefinition, ToolkitMetadata) + * stay in toolkit-docs-generator/src/types/index.ts: the app never sees + * that shape, only the merged output defined here. + */ +import { z } from "zod"; +import { INTEGRATION_CATEGORIES } from "./toolkit-primitives.js"; + +// ============================================================================ +// Tool Parameter Schema +// ============================================================================ + +export const ToolParameterSchema = z.object({ + name: z.string(), + type: z.string(), + innerType: z.string().optional(), + required: z.boolean(), + description: z.string().nullable(), + enum: z.array(z.string()).nullable(), + inferrable: z.boolean().default(true), +}); + +export type ToolParameter = z.infer; + +// ============================================================================ +// Tool Auth Schema +// ============================================================================ + +export const ToolAuthSchema = z.object({ + providerId: z.string().nullable(), + providerType: z.string(), + scopes: z.array(z.string()), +}); + +export type ToolAuth = z.infer; + +// ============================================================================ +// Tool Output Schema +// ============================================================================ + +export const ToolOutputSchema = z.object({ + type: z.string(), + description: z.string().nullable(), +}); + +export type ToolOutput = z.infer; + +// ============================================================================ +// Tool Secrets Schema +// ============================================================================ + +export const SecretTypeSchema = z.enum([ + "api_key", + "token", + "client_secret", + "webhook_secret", + "private_key", + "password", + "unknown", +]); + +export type SecretType = z.infer; + +export const ToolSecretSchema = z.object({ + name: z.string(), + type: SecretTypeSchema, +}); + +export type ToolSecret = z.infer; + +// ============================================================================ +// Tool Metadata Schema (per-tool metadata from Engine API) +// ============================================================================ + +export const ToolMetadataClassificationSchema = z.object({ + serviceDomains: z.array(z.string()).default([]), +}); +export type ToolMetadataClassification = z.infer< + typeof ToolMetadataClassificationSchema +>; + +export const ToolMetadataBehaviorSchema = z.object({ + operations: z.array(z.string()).default([]), + readOnly: z.boolean().optional(), + destructive: z.boolean().optional(), + idempotent: z.boolean().optional(), + openWorld: z.boolean().optional(), +}); +export type ToolMetadataBehavior = z.infer; + +export const ToolMetadataSchema = z.object({ + classification: ToolMetadataClassificationSchema, + behavior: ToolMetadataBehaviorSchema, + extras: z.record(z.string(), z.unknown()).optional().nullable(), +}); +export type ToolMetadata = z.infer; + +// ============================================================================ +// Toolkit Category / Type Schemas (from Design System) +// ============================================================================ + +// Built from INTEGRATION_CATEGORIES (toolkit-primitives.ts) rather than a +// hand-copied list of the same values, so the generator's output contract +// and the docs app's route set can never drift apart — see that constant's +// doc comment for why there's no "others" member. +export const ToolkitCategorySchema = z.enum(INTEGRATION_CATEGORIES); + +export type ToolkitCategory = z.infer; + +export const ToolkitTypeSchema = z.enum([ + "arcade", + "arcade_starter", + "verified", + "community", + "auth", +]); + +export type ToolkitType = z.infer; + +// ============================================================================ +// Documentation Chunk Schema (for custom content injection) +// ============================================================================ + +/** + * Type of documentation chunk content + * - callout: Warning, info, or tip box + * - markdown: Raw markdown content + * - code: Code block with language + * - warning: Highlighted warning message + * - info: Informational note + * - tip: Helpful tip + */ +export const DocumentationChunkTypeSchema = z.enum([ + "callout", + "markdown", + "code", + "warning", + "info", + "tip", + "section", +]); + +export type DocumentationChunkType = z.infer< + typeof DocumentationChunkTypeSchema +>; + +/** + * Location where the chunk should be injected + * - header: After the toolkit header, before tools list + * - description: Around the tool description + * - parameters: Around the parameters section + * - auth: Around the auth/scopes section + * - secrets: Around the secrets section + * - output: Around the output section + * - footer: After all tools, before the footer + * - before_available_tools: Before the available tools section (toolkit-level) + * - after_available_tools: After the available tools section (toolkit-level) + * - custom_section: Standalone custom section outside the tools list + */ +export const DocumentationChunkLocationSchema = z.enum([ + "header", + "description", + "parameters", + "auth", + "secrets", + "output", + "footer", + "before_available_tools", + "after_available_tools", + "custom_section", +]); + +export type DocumentationChunkLocation = z.infer< + typeof DocumentationChunkLocationSchema +>; + +/** + * Position relative to the location + */ +export const DocumentationChunkPositionSchema = z.enum([ + "before", + "after", + "replace", +]); + +export type DocumentationChunkPosition = z.infer< + typeof DocumentationChunkPositionSchema +>; + +/** + * A documentation chunk represents custom content to inject into docs + */ +export const DocumentationChunkSchema = z.object({ + /** Type of content */ + type: DocumentationChunkTypeSchema, + /** Where to inject the content */ + location: DocumentationChunkLocationSchema, + /** Position relative to location (before, after, replace) */ + position: DocumentationChunkPositionSchema, + /** The actual content (markdown string) */ + content: z.string(), + /** Optional title for callouts */ + title: z.string().optional(), + /** Optional variant for styling (e.g., "destructive" for warnings) */ + variant: z + .enum(["default", "destructive", "warning", "info", "success"]) + .optional(), + /** Optional section header for sidebar navigation (e.g., "## Auth Setup") */ + header: z.string().optional(), + /** Optional priority for ordering (lower = earlier, default = 100) */ + priority: z.number().optional(), +}); + +export type DocumentationChunk = z.infer; + +// ============================================================================ +// Tool Code Example Schema (for generating example code) +// ============================================================================ + +/** + * Parameter value with type information for code generation + */ +export const ExampleParameterValueSchema = z.object({ + /** The example value to use in code */ + value: z.unknown(), + /** Parameter type */ + type: z.enum(["string", "integer", "boolean", "array", "object"]), + /** Whether this parameter is required */ + required: z.boolean(), +}); + +export type ExampleParameterValue = z.infer; + +/** + * Tool code example configuration + * Used to generate Python/JavaScript example code + */ +export const ToolCodeExampleSchema = z.object({ + /** Full tool name (e.g., "Github.SetStarred") */ + toolName: z.string(), + /** Parameter values with type info */ + parameters: z.record(z.string(), ExampleParameterValueSchema), + /** Whether this tool requires user authorization */ + requiresAuth: z.boolean(), + /** Auth provider ID if auth is required */ + authProvider: z.string().optional(), + /** Optional tab label for the code example */ + tabLabel: z.string().optional(), +}); + +export type ToolCodeExample = z.infer; + +// ============================================================================ +// Toolkit Sub-Page Schema +// ============================================================================ + +/** + * A sub-page for a toolkit: either a string (legacy slug) or a rich object + * with { type, content, relativePath } for inline MDX sub-page content. + */ +export const ToolkitSubPageSchema = z.union([ + z.string(), + z.object({ + type: z.string().min(1), + content: z.string(), + relativePath: z.string().min(1), + }), +]); + +export type ToolkitSubPage = z.infer; + +// ============================================================================ +// Merged Tool Schema (output format) +// ============================================================================ + +export const MergedToolSchema = z.object({ + name: z.string(), + qualifiedName: z.string(), + fullyQualifiedName: z.string(), + description: z.string().nullable(), + parameters: z.array(ToolParameterSchema), + auth: ToolAuthSchema.nullable(), + secrets: z.array(z.string()), + secretsInfo: z.array(ToolSecretSchema).default([]), + output: ToolOutputSchema.nullable(), + /** Custom documentation chunks for this tool */ + documentationChunks: z.array(DocumentationChunkSchema).default([]), + /** Generated code example configuration */ + codeExample: ToolCodeExampleSchema.optional(), + metadata: ToolMetadataSchema.nullable().optional(), +}); + +export type MergedTool = z.infer; + +// ============================================================================ +// Merged Toolkit Schema (output format) +// ============================================================================ + +export const ToolkitAuthTypeSchema = z.enum([ + "oauth2", + "api_key", + "mixed", + "none", +]); + +export type ToolkitAuthType = z.infer; + +export const MergedToolkitMetadataSchema = z.object({ + category: ToolkitCategorySchema, + iconUrl: z.string(), + isBYOC: z.boolean(), + isPro: z.boolean(), + type: ToolkitTypeSchema, + docsLink: z.string(), + isComingSoon: z.boolean(), + isHidden: z.boolean(), +}); + +export type MergedToolkitMetadata = z.infer; + +export const MergedToolkitAuthSchema = z.object({ + type: ToolkitAuthTypeSchema, + providerId: z.string().nullable(), + allScopes: z.array(z.string()), +}); + +export type MergedToolkitAuth = z.infer; + +export const MergedToolkitSchema = z.object({ + /** Unique toolkit ID (e.g., "Github") */ + id: z.string(), + /** Human-readable label (e.g., "GitHub") */ + label: z.string(), + /** Toolkit version (e.g., "1.0.0") */ + version: z.string(), + /** Toolkit description */ + description: z.string().nullable(), + /** LLM-generated summary (optional) */ + summary: z.string().optional(), + /** + * True when the current `summary` is known to be out of date with the + * toolkit's current tools (the signature changed but regeneration was + * skipped or failed, so the previous summary was carried forward as a + * fallback). Cleared whenever a fresh summary is successfully generated + * or when the summary is verified against an unchanged signature. + */ + summaryStale: z.boolean().optional(), + /** + * Machine-readable reason the summary is stale (e.g. + * "llm_generator_unavailable", "llm_generation_failed"). Always set + * together with `summaryStale: true`. Cleared together with it. + */ + summaryStaleReason: z.string().optional(), + /** Metadata from Design System */ + metadata: MergedToolkitMetadataSchema, + /** Authentication requirements */ + auth: MergedToolkitAuthSchema.nullable(), + /** All tools in this toolkit */ + tools: z.array(MergedToolSchema), + /** Toolkit-level documentation chunks */ + documentationChunks: z.array(DocumentationChunkSchema).default([]), + /** Custom imports for MDX */ + customImports: z.array(z.string()).default([]), + /** + * Sub-pages that exist for this toolkit. + * Each entry is either a string (legacy slug) or a rich object with + * { type, content, relativePath } for inline MDX sub-page content. + */ + subPages: z.array(ToolkitSubPageSchema).default([]), + /** + * Optional override for the pip package name shown in the install + * snippet. Not currently emitted by the generator (toolkits derive it + * from `id` via `buildPipPackageName`), but the docs app has always + * accepted an explicit override here, so it stays part of the contract. + */ + pipPackageName: z.string().optional(), + /** Generation metadata */ + generatedAt: z.string().optional(), +}); + +export type MergedToolkit = z.infer; + +// ============================================================================ +// Index Output Schema +// ============================================================================ + +export const ToolkitIndexEntrySchema = z.object({ + id: z.string(), + label: z.string(), + version: z.string(), + category: ToolkitCategorySchema, + type: ToolkitTypeSchema, + toolCount: z.number(), + authType: ToolkitAuthTypeSchema, +}); + +export type ToolkitIndexEntry = z.infer; + +export const ToolkitIndexSchema = z.object({ + generatedAt: z.string(), + version: z.string(), + toolkits: z.array(ToolkitIndexEntrySchema), +}); + +export type ToolkitIndex = z.infer; diff --git a/toolkit-docs-generator/src/sources/design-system-metadata.ts b/toolkit-docs-generator/src/sources/design-system-metadata.ts index 9f0f67a11..b25946660 100644 --- a/toolkit-docs-generator/src/sources/design-system-metadata.ts +++ b/toolkit-docs-generator/src/sources/design-system-metadata.ts @@ -8,6 +8,7 @@ */ import { TOOLKITS as DESIGN_SYSTEM_TOOLKITS } from "@arcadeai/design-system/metadata/toolkits"; import { z } from "zod"; +import { normalizeToolkitId } from "../shared/toolkit-primitives.js"; import type { ToolkitMetadata } from "../types/index.js"; import { ToolkitMetadataSchema } from "../types/index.js"; import type { IMetadataSource } from "./internal.js"; @@ -38,12 +39,6 @@ type DesignSystemToolkit = z.infer; // Helpers // ============================================================================ -const LOOKUP_KEY_REGEX = /[^a-z0-9]/g; - -function normalizeLookupKey(value: string): string { - return value.toLowerCase().replace(LOOKUP_KEY_REGEX, ""); -} - function toToolkitMetadata(entry: DesignSystemToolkit): ToolkitMetadata | null { const iconUrl = entry.publicIconUrl ?? entry.iconUrl; if (!iconUrl) return null; @@ -83,18 +78,18 @@ export class DesignSystemMetadataSource implements IMetadataSource { this.indexByIdOrLabel = new Map(); for (const toolkit of toolkits) { - this.indexByIdOrLabel.set(normalizeLookupKey(toolkit.id), toolkit); - this.indexByIdOrLabel.set(normalizeLookupKey(toolkit.label), toolkit); + this.indexByIdOrLabel.set(normalizeToolkitId(toolkit.id), toolkit); + this.indexByIdOrLabel.set(normalizeToolkitId(toolkit.label), toolkit); } } async getToolkitMetadata(toolkitId: string): Promise { - const key = normalizeLookupKey(toolkitId); + const key = normalizeToolkitId(toolkitId); const direct = this.indexByIdOrLabel.get(key); if (direct) return direct; // Fallback 1: "github-api" / "github_api" style inputs. - // (normalizeLookupKey already strips separators) + // (normalizeToolkitId already strips separators) // Fallback 2: If this looks like an API toolkit, try the base provider. if (key.endsWith("api")) { diff --git a/toolkit-docs-generator/src/sources/toolkit-data-source.ts b/toolkit-docs-generator/src/sources/toolkit-data-source.ts index 1b6b191a6..de78a8951 100644 --- a/toolkit-docs-generator/src/sources/toolkit-data-source.ts +++ b/toolkit-docs-generator/src/sources/toolkit-data-source.ts @@ -7,8 +7,8 @@ */ import { join } from "path"; +import { isApiSuffixedToolkitId } from "../shared/toolkit-primitives.js"; import type { ToolDefinition, ToolkitMetadata } from "../types/index.js"; -import { normalizeId } from "../utils/fp.js"; import { filterToolsByHighestVersion } from "../utils/version-coherence.js"; import { type ArcadeApiSourceConfig, @@ -142,7 +142,7 @@ export class CombinedToolkitDataSource implements IToolkitDataSource { tools: readonly ToolDefinition[], directMetadata: ToolkitMetadata | null ): Promise { - if (directMetadata || !normalizeId(toolkitId).endsWith("api")) { + if (directMetadata || !isApiSuffixedToolkitId(toolkitId)) { return directMetadata; } diff --git a/toolkit-docs-generator/src/types/index.ts b/toolkit-docs-generator/src/types/index.ts index c8dba520c..0e9e431a6 100644 --- a/toolkit-docs-generator/src/types/index.ts +++ b/toolkit-docs-generator/src/types/index.ts @@ -1,7 +1,28 @@ /** * Core type definitions for the toolkit docs generator + * + * The merged/output schemas (MergedToolkit, ToolkitIndex, and everything + * they're built from) live in ../shared/toolkit-schemas.ts because the + * Next.js docs app imports them too — see that file's header comment for + * why the shared module has to live under this package's `src/`. Everything + * below is either CLI-only or describes a pre-merge shape the app never + * sees (raw Engine API / Design System data, extracted MDX custom + * sections), so it stays generator-local and re-exports the shared pieces + * it depends on. */ import { z } from "zod"; +import { + DocumentationChunkSchema, + ToolAuthSchema, + ToolkitCategorySchema, + ToolkitSubPageSchema, + ToolkitTypeSchema, + ToolMetadataSchema, + ToolOutputSchema, + ToolParameterSchema, +} from "../shared/toolkit-schemas.js"; + +export * from "../shared/toolkit-schemas.js"; // ============================================================================ // CLI Input Types @@ -29,96 +50,7 @@ export const GenerateInputSchema = z.object({ export type GenerateInput = z.infer; // ============================================================================ -// Tool Parameter Schema -// ============================================================================ - -export const ToolParameterSchema = z.object({ - name: z.string(), - type: z.string(), - innerType: z.string().optional(), - required: z.boolean(), - description: z.string().nullable(), - enum: z.array(z.string()).nullable(), - inferrable: z.boolean().default(true), -}); - -export type ToolParameter = z.infer; - -// ============================================================================ -// Tool Auth Schema -// ============================================================================ - -export const ToolAuthSchema = z.object({ - providerId: z.string().nullable(), - providerType: z.string(), - scopes: z.array(z.string()), -}); - -export type ToolAuth = z.infer; - -// ============================================================================ -// Tool Output Schema -// ============================================================================ - -export const ToolOutputSchema = z.object({ - type: z.string(), - description: z.string().nullable(), -}); - -export type ToolOutput = z.infer; - -// ============================================================================ -// Tool Secrets Schema -// ============================================================================ - -export const SecretTypeSchema = z.enum([ - "api_key", - "token", - "client_secret", - "webhook_secret", - "private_key", - "password", - "unknown", -]); - -export type SecretType = z.infer; - -export const ToolSecretSchema = z.object({ - name: z.string(), - type: SecretTypeSchema, -}); - -export type ToolSecret = z.infer; - -// ============================================================================ -// Tool Metadata Schema (per-tool metadata from Engine API) -// ============================================================================ - -export const ToolMetadataClassificationSchema = z.object({ - serviceDomains: z.array(z.string()).default([]), -}); -export type ToolMetadataClassification = z.infer< - typeof ToolMetadataClassificationSchema ->; - -export const ToolMetadataBehaviorSchema = z.object({ - operations: z.array(z.string()).default([]), - readOnly: z.boolean().optional(), - destructive: z.boolean().optional(), - idempotent: z.boolean().optional(), - openWorld: z.boolean().optional(), -}); -export type ToolMetadataBehavior = z.infer; - -export const ToolMetadataSchema = z.object({ - classification: ToolMetadataClassificationSchema, - behavior: ToolMetadataBehaviorSchema, - extras: z.record(z.string(), z.unknown()).optional().nullable(), -}); -export type ToolMetadata = z.infer; - -// ============================================================================ -// Tool Definition Schema (from Engine API) +// Tool Definition Schema (raw, from Engine API, pre-merge) // ============================================================================ export const ToolDefinitionSchema = z.object({ @@ -137,33 +69,9 @@ export const ToolDefinitionSchema = z.object({ export type ToolDefinition = z.infer; // ============================================================================ -// Toolkit Metadata Schema (from Design System) +// Toolkit Metadata Schema (raw, from Design System, pre-merge) // ============================================================================ -export const ToolkitCategorySchema = z.enum([ - "productivity", - "social", - "development", - "entertainment", - "search", - "payments", - "sales", - "databases", - "customer-support", -]); - -export type ToolkitCategory = z.infer; - -export const ToolkitTypeSchema = z.enum([ - "arcade", - "arcade_starter", - "verified", - "community", - "auth", -]); - -export type ToolkitType = z.infer; - export const ToolkitMetadataSchema = z.object({ id: z.string(), label: z.string(), @@ -180,151 +88,9 @@ export const ToolkitMetadataSchema = z.object({ export type ToolkitMetadata = z.infer; // ============================================================================ -// Documentation Chunk Schema (for custom content injection) -// ============================================================================ - -/** - * Type of documentation chunk content - * - callout: Warning, info, or tip box - * - markdown: Raw markdown content - * - code: Code block with language - * - warning: Highlighted warning message - * - info: Informational note - * - tip: Helpful tip - */ -export const DocumentationChunkTypeSchema = z.enum([ - "callout", - "markdown", - "code", - "warning", - "info", - "tip", - "section", -]); - -export type DocumentationChunkType = z.infer< - typeof DocumentationChunkTypeSchema ->; - -/** - * Location where the chunk should be injected - * - header: After the toolkit header, before tools list - * - description: Around the tool description - * - parameters: Around the parameters section - * - auth: Around the auth/scopes section - * - secrets: Around the secrets section - * - output: Around the output section - * - footer: After all tools, before the footer - * - before_available_tools: Before the available tools section (toolkit-level) - * - after_available_tools: After the available tools section (toolkit-level) - * - custom_section: Standalone custom section outside the tools list - */ -export const DocumentationChunkLocationSchema = z.enum([ - "header", - "description", - "parameters", - "auth", - "secrets", - "output", - "footer", - "before_available_tools", - "after_available_tools", - "custom_section", -]); - -export type DocumentationChunkLocation = z.infer< - typeof DocumentationChunkLocationSchema ->; - -/** - * Position relative to the location - */ -export const DocumentationChunkPositionSchema = z.enum([ - "before", - "after", - "replace", -]); - -export type DocumentationChunkPosition = z.infer< - typeof DocumentationChunkPositionSchema ->; - -/** - * A documentation chunk represents custom content to inject into docs - */ -export const DocumentationChunkSchema = z.object({ - /** Type of content */ - type: DocumentationChunkTypeSchema, - /** Where to inject the content */ - location: DocumentationChunkLocationSchema, - /** Position relative to location (before, after, replace) */ - position: DocumentationChunkPositionSchema, - /** The actual content (markdown string) */ - content: z.string(), - /** Optional title for callouts */ - title: z.string().optional(), - /** Optional variant for styling (e.g., "destructive" for warnings) */ - variant: z - .enum(["default", "destructive", "warning", "info", "success"]) - .optional(), - /** Optional section header for sidebar navigation (e.g., "## Auth Setup") */ - header: z.string().optional(), - /** Optional priority for ordering (lower = earlier, default = 100) */ - priority: z.number().optional(), -}); - -export type DocumentationChunk = z.infer; - -// ============================================================================ -// Tool Code Example Schema (for generating example code) -// ============================================================================ - -/** - * Parameter value with type information for code generation - */ -export const ExampleParameterValueSchema = z.object({ - /** The example value to use in code */ - value: z.unknown(), - /** Parameter type */ - type: z.enum(["string", "integer", "boolean", "array", "object"]), - /** Whether this parameter is required */ - required: z.boolean(), -}); - -export type ExampleParameterValue = z.infer; - -/** - * Tool code example configuration - * Used to generate Python/JavaScript example code - */ -export const ToolCodeExampleSchema = z.object({ - /** Full tool name (e.g., "Github.SetStarred") */ - toolName: z.string(), - /** Parameter values with type info */ - parameters: z.record(z.string(), ExampleParameterValueSchema), - /** Whether this tool requires user authorization */ - requiresAuth: z.boolean(), - /** Auth provider ID if auth is required */ - authProvider: z.string().optional(), - /** Optional tab label for the code example */ - tabLabel: z.string().optional(), -}); - -export type ToolCodeExample = z.infer; - -// ============================================================================ -// Custom Sections Schema (extracted from MDX) +// Custom Sections Schema (extracted from MDX, pre-merge) // ============================================================================ -export const ToolkitSubPageSchema = z.union([ - z.string(), - z.object({ - type: z.string().min(1), - content: z.string(), - relativePath: z.string().min(1), - }), -]); - export const CustomSectionsSchema = z.object({ /** Toolkit-level documentation chunks */ documentationChunks: z.array(DocumentationChunkSchema).default([]), @@ -339,131 +105,3 @@ export const CustomSectionsSchema = z.object({ }); export type CustomSections = z.infer; - -// ============================================================================ -// Merged Tool Schema (output format) -// ============================================================================ - -export const MergedToolSchema = z.object({ - name: z.string(), - qualifiedName: z.string(), - fullyQualifiedName: z.string(), - description: z.string().nullable(), - parameters: z.array(ToolParameterSchema), - auth: ToolAuthSchema.nullable(), - secrets: z.array(z.string()), - secretsInfo: z.array(ToolSecretSchema).default([]), - output: ToolOutputSchema.nullable(), - /** Custom documentation chunks for this tool */ - documentationChunks: z.array(DocumentationChunkSchema).default([]), - /** Generated code example configuration */ - codeExample: ToolCodeExampleSchema.optional(), - metadata: ToolMetadataSchema.nullable().optional(), -}); - -export type MergedTool = z.infer; - -// ============================================================================ -// Merged Toolkit Schema (output format) -// ============================================================================ - -export const ToolkitAuthTypeSchema = z.enum([ - "oauth2", - "api_key", - "mixed", - "none", -]); - -export type ToolkitAuthType = z.infer; - -export const MergedToolkitMetadataSchema = z.object({ - category: ToolkitCategorySchema, - iconUrl: z.string(), - isBYOC: z.boolean(), - isPro: z.boolean(), - type: ToolkitTypeSchema, - docsLink: z.string(), - isComingSoon: z.boolean(), - isHidden: z.boolean(), -}); - -export type MergedToolkitMetadata = z.infer; - -export const MergedToolkitAuthSchema = z.object({ - type: ToolkitAuthTypeSchema, - providerId: z.string().nullable(), - allScopes: z.array(z.string()), -}); - -export type MergedToolkitAuth = z.infer; - -export const MergedToolkitSchema = z.object({ - /** Unique toolkit ID (e.g., "Github") */ - id: z.string(), - /** Human-readable label (e.g., "GitHub") */ - label: z.string(), - /** Toolkit version (e.g., "1.0.0") */ - version: z.string(), - /** Toolkit description */ - description: z.string().nullable(), - /** LLM-generated summary (optional) */ - summary: z.string().optional(), - /** - * True when the current `summary` is known to be out of date with the - * toolkit's current tools (the signature changed but regeneration was - * skipped or failed, so the previous summary was carried forward as a - * fallback). Cleared whenever a fresh summary is successfully generated - * or when the summary is verified against an unchanged signature. - */ - summaryStale: z.boolean().optional(), - /** - * Machine-readable reason the summary is stale (e.g. - * "llm_generator_unavailable", "llm_generation_failed"). Always set - * together with `summaryStale: true`. Cleared together with it. - */ - summaryStaleReason: z.string().optional(), - /** Metadata from Design System */ - metadata: MergedToolkitMetadataSchema, - /** Authentication requirements */ - auth: MergedToolkitAuthSchema.nullable(), - /** All tools in this toolkit */ - tools: z.array(MergedToolSchema), - /** Toolkit-level documentation chunks */ - documentationChunks: z.array(DocumentationChunkSchema).default([]), - /** Custom imports for MDX */ - customImports: z.array(z.string()).default([]), - /** - * Sub-pages that exist for this toolkit. - * Each entry is either a string (legacy slug) or a rich object with - * { type, content, relativePath } for inline MDX sub-page content. - */ - subPages: z.array(ToolkitSubPageSchema).default([]), - /** Generation metadata */ - generatedAt: z.string().optional(), -}); - -export type MergedToolkit = z.infer; - -// ============================================================================ -// Index Output Schema -// ============================================================================ - -export const ToolkitIndexEntrySchema = z.object({ - id: z.string(), - label: z.string(), - version: z.string(), - category: ToolkitCategorySchema, - type: ToolkitTypeSchema, - toolCount: z.number(), - authType: ToolkitAuthTypeSchema, -}); - -export type ToolkitIndexEntry = z.infer; - -export const ToolkitIndexSchema = z.object({ - generatedAt: z.string(), - version: z.string(), - toolkits: z.array(ToolkitIndexEntrySchema), -}); - -export type ToolkitIndex = z.infer; diff --git a/toolkit-docs-generator/tests/app-lib/toolkit-data.test.ts b/toolkit-docs-generator/tests/app-lib/toolkit-data.test.ts index 91a0bdf36..fac395e4d 100644 --- a/toolkit-docs-generator/tests/app-lib/toolkit-data.test.ts +++ b/toolkit-docs-generator/tests/app-lib/toolkit-data.test.ts @@ -80,10 +80,20 @@ describe("toolkit data loader", () => { const toolkitData = { id: "PosthogApi", label: "PostHog API", + version: "1.0.0", + description: null, tools: [], + auth: null, metadata: { + category: "development", + iconUrl: "https://design-system.arcade.dev/icons/posthog.svg", + isBYOC: false, + isPro: false, + type: "arcade_starter", docsLink: "https://docs.arcade.dev/en/mcp-servers/development/posthog-api", + isComingSoon: false, + isHidden: false, }, }; await writeFile( diff --git a/toolkit-docs-generator/tests/app-lib/toolkit-slug.test.ts b/toolkit-docs-generator/tests/app-lib/toolkit-slug.test.ts index 74d7a4017..b18317bf8 100644 --- a/toolkit-docs-generator/tests/app-lib/toolkit-slug.test.ts +++ b/toolkit-docs-generator/tests/app-lib/toolkit-slug.test.ts @@ -4,7 +4,7 @@ import { normalizeToolkitId, type ToolkitSlugSource, toKebabCase, -} from "../../../app/_lib/toolkit-slug"; +} from "../../src/shared/toolkit-primitives"; // ============================================================================ // normalizeToolkitId diff --git a/toolkit-docs-generator/tests/app-lib/toolkit-static-params.test.ts b/toolkit-docs-generator/tests/app-lib/toolkit-static-params.test.ts index 8c5c59f2c..55a5a92c0 100644 --- a/toolkit-docs-generator/tests/app-lib/toolkit-static-params.test.ts +++ b/toolkit-docs-generator/tests/app-lib/toolkit-static-params.test.ts @@ -2,12 +2,12 @@ import { mkdtemp, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { describe, expect, it } from "vitest"; -import { normalizeToolkitId } from "../../../app/_lib/toolkit-slug"; import { getToolkitStaticParamsForCategory, listToolkitRoutes, type ToolkitCatalogEntry, } from "../../../app/_lib/toolkit-static-params"; +import { normalizeToolkitId } from "../../src/shared/toolkit-primitives"; const withTempDir = async (fn: (dir: string) => Promise) => { const dir = await mkdtemp(join(tmpdir(), "toolkit-static-params-")); @@ -43,8 +43,29 @@ const writeToolkitData = async ( } ) => { const fileName = `${normalizeToolkitId(toolkit.id)}.json`; + // Fill in the fields the merged toolkit schema requires but this test + // suite doesn't care about, so fixtures stay valid without every call + // site restating boilerplate. const toolkitFixture = JSON.stringify( - { label: toolkit.label ?? toolkit.id, ...toolkit }, + { + version: "1.0.0", + description: null, + tools: [], + auth: null, + label: toolkit.label ?? toolkit.id, + ...toolkit, + metadata: { + category: "productivity", + iconUrl: "https://design-system.arcade.dev/icons/placeholder.svg", + isBYOC: false, + isPro: false, + type: "arcade", + docsLink: "", + isComingSoon: false, + isHidden: false, + ...toolkit.metadata, + }, + }, null, 2 ); @@ -227,16 +248,28 @@ describe("toolkit static params", () => { }); }); - it('maps unknown categories to "others"', async () => { + it('throws on an unrecognized category instead of coercing it to "others"', async () => { await withTempDir(async (dir) => { await writeIndex(dir, [{ id: "Github", category: "weird" }]); + await expect( + listToolkitRoutes({ dataDir: dir, toolkitsCatalog: [] }) + ).rejects.toThrow(/weird/); + }); + }); + + it("skips a toolkit with no category anywhere instead of routing it to a fake bucket", async () => { + await withTempDir(async (dir) => { + // No JSON file, no catalog entry, and the index entry itself omits + // category — nothing to route this toolkit under. + await writeIndex(dir, [{ id: "Github" }]); + const routes = await listToolkitRoutes({ dataDir: dir, toolkitsCatalog: [], }); - expect(routes).toEqual([{ toolkitId: "github", category: "others" }]); + expect(routes).toEqual([]); }); }); }); diff --git a/toolkit-docs-generator/tests/scripts/sync-toolkit-sidebar.test.ts b/toolkit-docs-generator/tests/scripts/sync-toolkit-sidebar.test.ts index 273ff1e19..aa0f20397 100644 --- a/toolkit-docs-generator/tests/scripts/sync-toolkit-sidebar.test.ts +++ b/toolkit-docs-generator/tests/scripts/sync-toolkit-sidebar.test.ts @@ -296,13 +296,27 @@ describe("buildToolkitInfoList", () => { }); it("keeps sidebar href categories consistent with static params", async () => { + // This fixture also flows through getToolkitStaticParamsForCategory + // below, which validates it against the full merged toolkit schema — + // unlike the other fixtures in this file, it needs every required field, + // not just the ones buildToolkitInfoList itself reads. createToolkitJson("weaviateapi", { id: "WeaviateApi", label: "Weaviate API", + version: "1.0.0", + description: null, + auth: null, + tools: [], metadata: { category: "databases", docsLink: "https://docs.arcade.dev/en/mcp-servers/databases/weaviate-api", + iconUrl: "https://design-system.arcade.dev/icons/placeholder.svg", + isBYOC: false, + isPro: false, + type: "arcade", + isComingSoon: false, + isHidden: false, }, }); From 71d6ef1cd784f39a5d651a899ee894de21ba3a1b Mon Sep 17 00:00:00 2001 From: Teal Larson Date: Fri, 31 Jul 2026 14:27:48 -0400 Subject: [PATCH 05/17] chore: turn on the checks that were inert MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds a typecheck script covering ~18k lines that nothing checked: root tests/, and the generator's scripts/ and tests/. scripts/ already had a tsconfig and already passed — it was simply wired to no command. The generator's tsconfig excluded **/*.test.ts, which is what hid most of the errors. Fixing the 50 surfaced errors needed no suppressions and no loosened compiler options: six were .ts import specifiers that tsx tolerates and tsc does not, the rest were incomplete test fixtures and genuine exactOptionalPropertyTypes / noUncheckedIndexedAccess findings. test.yaml now runs it. The generator's vitest config set 80% coverage thresholds and enabled typecheck, but nothing pointed at it — the root config's default glob picked those tests up and ran them under root settings. It could not have worked if it had been wired up: @vitest/coverage-v8 is not installed, so the thresholds could never execute, and running under it breaks four suites for want of an @ alias. Deleting it stops the file implying guarantees it never provided. The six app-lib tests move to root tests/, where the @ alias means they no longer reach through ../../../. The pre-commit hook stashed unstaged work and re-formatted staged files after lint-staged had already formatted them — its own comment conceded the redundancy. That block also swallowed formatter failures by assigning FORMAT_EXIT_CODE=0 unconditionally, and called sha256sum, which stock macOS does not have, so under set -e a contributor without coreutils could not commit at all. 222 lines to 125. The generator fabricated metadata when the design system had none: category "development", an iconUrl, and a docsLink under /en/mcp-servers/, signalled only by a console warning inside an automated PR. One committed record runs on it today. Missing metadata now fails under --require-complete and names the toolkit; without the flag the omission reaches the run log rather than only stdout. The fabricated docsLink prefix now matches the route shape the other 116 records use, and the placeholder is marked hidden so a guessed category cannot file a toolkit under the wrong sidebar section. --require-complete was previously unreachable in --all mode, which is the mode CI uses: a pre-filter added those toolkits to the skip list before the merger ever saw them. Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/test.yaml | 3 + .husky/pre-commit | 97 ------ package.json | 1 + .../available-tools-filter-behavior.test.ts | 2 +- .../available-tools-filter-operations.test.ts | 2 +- tests/neutralize-emails.test.tsx | 39 +-- .../shared-service-domain.test.ts | 2 +- .../app-lib => tests}/toolkit-data.test.ts | 10 +- tests/toolkit-markdown.test.ts | 4 + .../app-lib => tests}/toolkit-slug.test.ts | 2 +- .../toolkit-static-params.test.ts | 8 +- .../scripts/check-stale-summaries.ts | 2 +- .../scripts/merge-custom-sections.ts | 12 +- .../scripts/report-tool-metadata.ts | 4 +- .../scripts/sync-toolkit-sidebar.ts | 8 +- .../scripts/validate-merge.ts | 4 +- toolkit-docs-generator/src/cli/index.ts | 57 +-- .../src/merger/data-merger.ts | 120 ++++++- .../tests/cli/generate-flow.test.ts | 2 +- .../tests/diff/previous-output.test.ts | 1 - .../llm/toolkit-summary-generator.test.ts | 2 + .../tests/merger/data-merger.test.ts | 161 +++++++-- .../tests/merger/metadata-freshness.test.ts | 2 +- .../tests/scenarios/skip-unchanged.test.ts | 3 +- .../scripts/sync-toolkit-sidebar.test.ts | 71 +++- .../tests/scripts/validate-merge.test.ts | 2 +- .../tests/sources/arcade-api.test.ts | 326 +++++++++++------- .../tests/sources/engine-api.test.ts | 10 +- .../tests/utils/output-dir.test.ts | 36 +- toolkit-docs-generator/tsconfig.json | 10 +- toolkit-docs-generator/vitest.config.ts | 41 --- tsconfig.json | 5 + 32 files changed, 631 insertions(+), 418 deletions(-) rename {toolkit-docs-generator/tests/app-lib => tests}/available-tools-filter-behavior.test.ts (94%) rename {toolkit-docs-generator/tests/app-lib => tests}/available-tools-filter-operations.test.ts (94%) rename {toolkit-docs-generator/tests/app-lib => tests}/shared-service-domain.test.ts (93%) rename {toolkit-docs-generator/tests/app-lib => tests}/toolkit-data.test.ts (95%) rename {toolkit-docs-generator/tests/app-lib => tests}/toolkit-slug.test.ts (98%) rename {toolkit-docs-generator/tests/app-lib => tests}/toolkit-static-params.test.ts (97%) delete mode 100644 toolkit-docs-generator/vitest.config.ts diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 8957e46d9..147e17586 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -41,6 +41,9 @@ jobs: - name: Run linter run: pnpm run lint + - name: Run typecheck + run: pnpm run typecheck + - name: Try a build run: pnpm build diff --git a/.husky/pre-commit b/.husky/pre-commit index 452307eb9..a55f31a75 100755 --- a/.husky/pre-commit +++ b/.husky/pre-commit @@ -2,16 +2,6 @@ # Exit on any error set -e -# Detect merge/rebase state — used later to skip the stash+format block -# which conflicts with merge state and corrupts it. -GIT_DIR="$(git rev-parse --git-dir)" -IS_MERGING=false -if [ -f "$GIT_DIR/MERGE_HEAD" ] || \ - [ -d "$GIT_DIR/rebase-merge" ] || \ - [ -d "$GIT_DIR/rebase-apply" ]; then - IS_MERGING=true -fi - # Check if there are any staged files if [ -z "$(git diff --cached --name-only)" ]; then echo "No staged files to check" @@ -133,90 +123,3 @@ fi # --- Lint Staged (formatting) --- pnpm exec lint-staged - -# --- Stash + Format --- -# Skip this block during merge/rebase: git stash --keep-index destroys -# MERGE_HEAD and corrupts the merge state, causing repeated failures. -# lint-staged (above) already handles formatting for staged files safely. -if [ "$IS_MERGING" = true ]; then - echo "⏭️ Skipping stash+format (merge/rebase in progress)" - exit 0 -fi - -# Store the hash of staged changes to detect modifications -STAGED_HASH=$(git diff --cached | sha256sum | cut -d' ' -f1) - -# Save list of staged files (handling all file states) -STAGED_FILES=$(git diff --cached --name-only --diff-filter=ACMR) -PARTIALLY_STAGED=$(git diff --name-only) - -# If a file is both staged and unstaged, stash/pop can produce conflicts. -# In that case rely on lint-staged only, which already ran above. -if [ -n "$PARTIALLY_STAGED" ] && [ -n "$STAGED_FILES" ]; then - for file in $PARTIALLY_STAGED; do - if [ -f "$file" ] && echo "$STAGED_FILES" | grep -qxF "$file"; then - echo "⏭️ Skipping stash+format (partially staged files detected)" - exit 0 - fi - done -fi - -# Stash unstaged changes to preserve working directory -# --keep-index keeps staged changes in working tree -STASH_CREATED=false -STASH_MESSAGE="pre-commit-stash-$$-$(date +%s)" -if ! git diff --quiet; then - git stash push --quiet --keep-index --message "$STASH_MESSAGE" - TOP_STASH_SUBJECT="$(git stash list -1 --format='%s' || true)" - case "$TOP_STASH_SUBJECT" in - *"$STASH_MESSAGE") - STASH_CREATED=true - ;; - esac -fi - -# Run formatter on the staged files -if [ -n "$STAGED_FILES" ]; then - for file in $STAGED_FILES; do - if [ -f "$file" ]; then - pnpm exec ultracite fix "$file" - fi - done -fi -FORMAT_EXIT_CODE=0 - -# Restore working directory state -if [ "$STASH_CREATED" = true ]; then - # Re-stage the formatted files - if [ -n "$STAGED_FILES" ]; then - echo "$STAGED_FILES" | while IFS= read -r file; do - if [ -f "$file" ]; then - git add "$file" - fi - done - fi - - # Restore unstaged changes - if ! git stash pop --quiet; then - echo "❌ Failed to restore stashed changes during pre-commit." - echo " Resolve conflicts, then re-stage files and commit again." - exit 1 - fi -else - # No stash was created, just re-add the formatted files - if [ -n "$STAGED_FILES" ]; then - echo "$STAGED_FILES" | while IFS= read -r file; do - if [ -f "$file" ]; then - git add "$file" - fi - done - fi -fi - -# Check if staged files actually changed -NEW_STAGED_HASH=$(git diff --cached | sha256sum | cut -d' ' -f1) -if [ "$STAGED_HASH" != "$NEW_STAGED_HASH" ]; then - echo "✨ Files formatted by Ultracite" -fi - -exit $FORMAT_EXIT_CODE diff --git a/package.json b/package.json index 657540f93..648ac89e8 100644 --- a/package.json +++ b/package.json @@ -9,6 +9,7 @@ "start": "next start", "lint": "pnpm exec ultracite check", "format": "pnpm exec ultracite fix", + "typecheck": "tsc -p tsconfig.json --noEmit && tsc -p scripts/tsconfig.json --noEmit && tsc -p toolkit-docs-generator/tsconfig.json --noEmit", "prepare": "husky install", "translate": "pnpm dlx tsx scripts/i18n-sync/index.ts && pnpm format", "llmstxt": "pnpm dlx tsx scripts/generate-llmstxt.ts", diff --git a/toolkit-docs-generator/tests/app-lib/available-tools-filter-behavior.test.ts b/tests/available-tools-filter-behavior.test.ts similarity index 94% rename from toolkit-docs-generator/tests/app-lib/available-tools-filter-behavior.test.ts rename to tests/available-tools-filter-behavior.test.ts index 7c1c2e24a..87fc97e13 100644 --- a/toolkit-docs-generator/tests/app-lib/available-tools-filter-behavior.test.ts +++ b/tests/available-tools-filter-behavior.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from "vitest"; -import { filterTools } from "../../../app/_components/toolkit-docs/components/available-tools-filter"; +import { filterTools } from "@/app/_components/toolkit-docs/components/available-tools-filter"; const makeTool = ( name: string, diff --git a/toolkit-docs-generator/tests/app-lib/available-tools-filter-operations.test.ts b/tests/available-tools-filter-operations.test.ts similarity index 94% rename from toolkit-docs-generator/tests/app-lib/available-tools-filter-operations.test.ts rename to tests/available-tools-filter-operations.test.ts index 223fc6fc6..56c9796aa 100644 --- a/toolkit-docs-generator/tests/app-lib/available-tools-filter-operations.test.ts +++ b/tests/available-tools-filter-operations.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from "vitest"; -import { filterTools } from "../../../app/_components/toolkit-docs/components/available-tools-filter"; +import { filterTools } from "@/app/_components/toolkit-docs/components/available-tools-filter"; const makeTool = (name: string, operations: string[]) => ({ name, diff --git a/tests/neutralize-emails.test.tsx b/tests/neutralize-emails.test.tsx index 47e7411f9..a6502c559 100644 --- a/tests/neutralize-emails.test.tsx +++ b/tests/neutralize-emails.test.tsx @@ -1,3 +1,4 @@ +import type { Element, Root, RootContent } from "hast"; import { renderToStaticMarkup } from "react-dom/server"; import { describe, expect, test } from "vitest"; import { @@ -39,27 +40,23 @@ describe("splitEmails", () => { }); }); -type HastNode = { - type: string; - value?: string; - tagName?: string; - properties?: Record; - children?: HastNode[]; +const collectText = (node: Root | RootContent): string => { + if (node.type === "text") { + return node.value; + } + return "children" in node ? node.children.map(collectText).join("") : ""; }; -const collectText = (node: HastNode): string => - node.type === "text" - ? (node.value ?? "") - : (node.children ?? []).map(collectText).join(""); - -const hasContiguousEmail = (node: HastNode): boolean => - node.type === "text" - ? EMAIL.test(node.value ?? "") - : (node.children ?? []).some(hasContiguousEmail); +const hasContiguousEmail = (node: Root | RootContent): boolean => { + if (node.type === "text") { + return EMAIL.test(node.value); + } + return "children" in node ? node.children.some(hasContiguousEmail) : false; +}; describe("rehypeNeutralizeEmails", () => { test("splits email text nodes and inserts a , losslessly", () => { - const tree: HastNode = { + const tree: Root = { type: "root", children: [ { @@ -73,10 +70,12 @@ describe("rehypeNeutralizeEmails", () => { rehypeNeutralizeEmails()(tree); - const paragraph = tree.children?.[0]; - expect(paragraph?.children?.some((child) => child.tagName === "wbr")).toBe( - true - ); + const paragraph = tree.children[0] as Element; + expect( + paragraph.children.some( + (child) => child.type === "element" && child.tagName === "wbr" + ) + ).toBe(true); // No single text node still holds a full email... expect(hasContiguousEmail(tree)).toBe(false); // ...and the concatenated text is unchanged. diff --git a/toolkit-docs-generator/tests/app-lib/shared-service-domain.test.ts b/tests/shared-service-domain.test.ts similarity index 93% rename from toolkit-docs-generator/tests/app-lib/shared-service-domain.test.ts rename to tests/shared-service-domain.test.ts index f3b2eecd3..49e2d8b41 100644 --- a/toolkit-docs-generator/tests/app-lib/shared-service-domain.test.ts +++ b/tests/shared-service-domain.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from "vitest"; -import { getSharedServiceDomain } from "../../../app/_components/toolkit-docs/components/toolkit-page-utils"; +import { getSharedServiceDomain } from "@/app/_components/toolkit-docs/components/toolkit-page-utils"; const makeTool = (domains: string[]) => ({ metadata: { diff --git a/toolkit-docs-generator/tests/app-lib/toolkit-data.test.ts b/tests/toolkit-data.test.ts similarity index 95% rename from toolkit-docs-generator/tests/app-lib/toolkit-data.test.ts rename to tests/toolkit-data.test.ts index fac395e4d..89a6f0050 100644 --- a/toolkit-docs-generator/tests/app-lib/toolkit-data.test.ts +++ b/tests/toolkit-data.test.ts @@ -3,13 +3,13 @@ import { tmpdir } from "node:os"; import { basename, join } from "node:path"; import { describe, expect, it } from "vitest"; -import { - readToolkitData, - readToolkitIndex, -} from "../../../app/_lib/toolkit-data"; +import { readToolkitData, readToolkitIndex } from "@/app/_lib/toolkit-data"; const loadFixture = async (fileName: string): Promise => { - const fixturesDir = new URL("../fixtures/", import.meta.url); + const fixturesDir = new URL( + "../toolkit-docs-generator/tests/fixtures/", + import.meta.url + ); const filePath = new URL(fileName, fixturesDir); return await readFile(filePath, "utf-8"); }; diff --git a/tests/toolkit-markdown.test.ts b/tests/toolkit-markdown.test.ts index 456358329..e2a166be1 100644 --- a/tests/toolkit-markdown.test.ts +++ b/tests/toolkit-markdown.test.ts @@ -20,8 +20,11 @@ const fixture: ToolkitData = { isPro: false, type: "arcade", docsLink: "", + isComingSoon: false, + isHidden: false, }, auth: null, + documentationChunks: [], customImports: [], subPages: [], tools: [ @@ -37,6 +40,7 @@ const fixture: ToolkitData = { required: true, description: "Who to do the thing for", enum: null, + inferrable: true, }, ], auth: { providerId: "demo", providerType: "oauth2", scopes: ["scope.a"] }, diff --git a/toolkit-docs-generator/tests/app-lib/toolkit-slug.test.ts b/tests/toolkit-slug.test.ts similarity index 98% rename from toolkit-docs-generator/tests/app-lib/toolkit-slug.test.ts rename to tests/toolkit-slug.test.ts index b18317bf8..faee2b6a8 100644 --- a/toolkit-docs-generator/tests/app-lib/toolkit-slug.test.ts +++ b/tests/toolkit-slug.test.ts @@ -4,7 +4,7 @@ import { normalizeToolkitId, type ToolkitSlugSource, toKebabCase, -} from "../../src/shared/toolkit-primitives"; +} from "@/toolkit-docs-generator/src/shared/toolkit-primitives"; // ============================================================================ // normalizeToolkitId diff --git a/toolkit-docs-generator/tests/app-lib/toolkit-static-params.test.ts b/tests/toolkit-static-params.test.ts similarity index 97% rename from toolkit-docs-generator/tests/app-lib/toolkit-static-params.test.ts rename to tests/toolkit-static-params.test.ts index 55a5a92c0..c18c410b5 100644 --- a/toolkit-docs-generator/tests/app-lib/toolkit-static-params.test.ts +++ b/tests/toolkit-static-params.test.ts @@ -6,8 +6,10 @@ import { getToolkitStaticParamsForCategory, listToolkitRoutes, type ToolkitCatalogEntry, -} from "../../../app/_lib/toolkit-static-params"; -import { normalizeToolkitId } from "../../src/shared/toolkit-primitives"; +} from "@/app/_lib/toolkit-static-params"; +import { normalizeToolkitId } from "@/toolkit-docs-generator/src/shared/toolkit-primitives"; + +const UNRECOGNIZED_CATEGORY_ERROR = /weird/; const withTempDir = async (fn: (dir: string) => Promise) => { const dir = await mkdtemp(join(tmpdir(), "toolkit-static-params-")); @@ -254,7 +256,7 @@ describe("toolkit static params", () => { await expect( listToolkitRoutes({ dataDir: dir, toolkitsCatalog: [] }) - ).rejects.toThrow(/weird/); + ).rejects.toThrow(UNRECOGNIZED_CATEGORY_ERROR); }); }); diff --git a/toolkit-docs-generator/scripts/check-stale-summaries.ts b/toolkit-docs-generator/scripts/check-stale-summaries.ts index 73930978e..18b06b830 100644 --- a/toolkit-docs-generator/scripts/check-stale-summaries.ts +++ b/toolkit-docs-generator/scripts/check-stale-summaries.ts @@ -14,7 +14,7 @@ import { readdirSync, readFileSync } from "node:fs"; import { join } from "node:path"; -import { resolveToolkitDataDir } from "../src/shared/toolkit-data-dir.ts"; +import { resolveToolkitDataDir } from "../src/shared/toolkit-data-dir.js"; const TOOLKITS_DIR = resolveToolkitDataDir(); diff --git a/toolkit-docs-generator/scripts/merge-custom-sections.ts b/toolkit-docs-generator/scripts/merge-custom-sections.ts index bc0298fbf..cd5349a5e 100644 --- a/toolkit-docs-generator/scripts/merge-custom-sections.ts +++ b/toolkit-docs-generator/scripts/merge-custom-sections.ts @@ -59,13 +59,15 @@ const parseArgs = (args: string[]): MergeOptions => { let verbose = false; for (let i = 0; i < args.length; i++) { - if (args[i] === "--custom-sections" && args[i + 1]) { - customSectionsPath = args[i + 1]; + const arg = args[i]; + const next = args[i + 1]; + if (arg === "--custom-sections" && next) { + customSectionsPath = next; i++; - } else if (args[i] === "--toolkits-dir" && args[i + 1]) { - toolkitsDir = args[i + 1]; + } else if (arg === "--toolkits-dir" && next) { + toolkitsDir = next; i++; - } else if (args[i] === "--verbose" || args[i] === "-v") { + } else if (arg === "--verbose" || arg === "-v") { verbose = true; } } diff --git a/toolkit-docs-generator/scripts/report-tool-metadata.ts b/toolkit-docs-generator/scripts/report-tool-metadata.ts index 428317a2a..82245018c 100644 --- a/toolkit-docs-generator/scripts/report-tool-metadata.ts +++ b/toolkit-docs-generator/scripts/report-tool-metadata.ts @@ -5,8 +5,8 @@ * regardless of cwd and honors the TOOLKIT_DATA_DIR env var override. */ -import { resolveToolkitDataDir } from "../src/shared/toolkit-data-dir.ts"; -import { collectToolMetadataStats } from "../src/utils/tool-metadata-audit.ts"; +import { resolveToolkitDataDir } from "../src/shared/toolkit-data-dir.js"; +import { collectToolMetadataStats } from "../src/utils/tool-metadata-audit.js"; const DATA_DIR = resolveToolkitDataDir(); diff --git a/toolkit-docs-generator/scripts/sync-toolkit-sidebar.ts b/toolkit-docs-generator/scripts/sync-toolkit-sidebar.ts index a42ed0a25..59512f824 100644 --- a/toolkit-docs-generator/scripts/sync-toolkit-sidebar.ts +++ b/toolkit-docs-generator/scripts/sync-toolkit-sidebar.ts @@ -28,16 +28,16 @@ import { import { dirname, join, resolve } from "node:path"; import { fileURLToPath } from "node:url"; import { TOOLKITS as DESIGN_SYSTEM_TOOLKITS } from "@arcadeai/design-system/metadata/toolkits"; -import { resolveToolkitDataDir } from "../src/shared/toolkit-data-dir.ts"; +import { resolveToolkitDataDir } from "../src/shared/toolkit-data-dir.js"; import { getToolkitSlug, INTEGRATION_CATEGORIES, isApiSuffixedToolkitId, -} from "../src/shared/toolkit-primitives.ts"; +} from "../src/shared/toolkit-primitives.js"; import type { MergedToolkit, MergedToolkitMetadata, -} from "../src/shared/toolkit-schemas.ts"; +} from "../src/shared/toolkit-schemas.js"; const __filename = fileURLToPath(import.meta.url); const __dirname = dirname(__filename); @@ -272,7 +272,7 @@ function resolveToolkitInfo( const toolkitId = jsonData?.id ?? slug; const docsSlug = getToolkitSlug({ id: toolkitId, - docsLink: jsonData?.metadata?.docsLink, + docsLink: jsonData?.metadata?.docsLink ?? null, }); const designSystemToolkit = TOOLKITS.find( (t) => t.id.toLowerCase() === toolkitId.toLowerCase() diff --git a/toolkit-docs-generator/scripts/validate-merge.ts b/toolkit-docs-generator/scripts/validate-merge.ts index 89d4ed2ed..ebddb70a0 100644 --- a/toolkit-docs-generator/scripts/validate-merge.ts +++ b/toolkit-docs-generator/scripts/validate-merge.ts @@ -13,8 +13,8 @@ import { existsSync, readdirSync, readFileSync } from "node:fs"; import { join, resolve } from "node:path"; import { fileURLToPath } from "node:url"; -import { resolveToolkitDataDir } from "../src/shared/toolkit-data-dir.ts"; -import type { MergedToolkit } from "../src/shared/toolkit-schemas.ts"; +import { resolveToolkitDataDir } from "../src/shared/toolkit-data-dir.js"; +import type { MergedToolkit } from "../src/shared/toolkit-schemas.js"; const DATA_DIR = resolveToolkitDataDir(); diff --git a/toolkit-docs-generator/src/cli/index.ts b/toolkit-docs-generator/src/cli/index.ts index 69e2c8b0c..3d911bc35 100644 --- a/toolkit-docs-generator/src/cli/index.ts +++ b/toolkit-docs-generator/src/cli/index.ts @@ -1501,20 +1501,12 @@ program ); } - if (requireComplete) { - const metadataExcludedToolkitIds = - getToolkitIdsWithoutMetadata(toolkitList); - for (const toolkitId of metadataExcludedToolkitIds) { - skipToolkitIds.add(toolkitId.toLowerCase()); - } - if (options.verbose && metadataExcludedToolkitIds.length > 0) { - console.log( - chalk.dim( - ` Excluding ${metadataExcludedToolkitIds.length} toolkit(s) without metadata` - ) - ); - } - } + // requireComplete no longer silently drops toolkits without + // design-system metadata into skipToolkitIds here. Silently + // excluding them was just as opaque as fabricating metadata for + // them — DataMerger.assertNoMissingMetadata (run from inside + // mergeAllToolkits below) now fails the whole run and names every + // affected toolkit instead. // If --skip-unchanged, only process changed toolkits // Add unchanged toolkits to skipToolkitIds @@ -1814,6 +1806,17 @@ program const failedToolkitsFromTools = Array.from( new Set(failedTools.map((tool) => tool.toolkitId)) ); + // Toolkits that fell back to getDefaultMetadata's guessed category, + // icon, and docsLink because the design system had no entry for + // them. --require-complete would have already failed the run for + // these (see DataMerger.assertNoMissingMetadata), so reaching here + // means requireComplete was off. The per-toolkit warning text + // already goes to stdout above; naming these explicitly in the run + // log means the omission survives past the CI log window instead + // of only ever being visible in real time. + const toolkitsWithDefaultMetadata = allResults + .filter((result) => result.usedDefaultMetadata) + .map((result) => result.toolkit.id); const runDetails = [ `output=${resolve(options.output)}`, @@ -1826,6 +1829,12 @@ program `writeErrors=${writeErrors.length}`, ]; + if (toolkitsWithDefaultMetadata.length > 0) { + runDetails.push( + `toolkitsWithDefaultMetadata=${toolkitsWithDefaultMetadata.join(", ")}` + ); + } + if (!runAll && providers) { runDetails.push( `providers=${providers.map((p) => p.provider).join(", ")}` @@ -2276,20 +2285,12 @@ program ); } - if (requireComplete) { - const metadataExcludedToolkitIds = - getToolkitIdsWithoutMetadata(toolkitList); - for (const toolkitId of metadataExcludedToolkitIds) { - skipToolkitIds.add(toolkitId.toLowerCase()); - } - if (options.verbose && metadataExcludedToolkitIds.length > 0) { - console.log( - chalk.dim( - ` Excluding ${metadataExcludedToolkitIds.length} toolkit(s) without metadata` - ) - ); - } - } + // requireComplete no longer silently drops toolkits without + // design-system metadata into skipToolkitIds here. Silently + // excluding them was just as opaque as fabricating metadata for + // them — DataMerger.assertNoMissingMetadata (run from inside + // mergeAllToolkits below) now fails the whole run and names every + // affected toolkit instead. const processingStats = computeProcessingStats( toolkitList, diff --git a/toolkit-docs-generator/src/merger/data-merger.ts b/toolkit-docs-generator/src/merger/data-merger.ts index 15d28def8..315725980 100644 --- a/toolkit-docs-generator/src/merger/data-merger.ts +++ b/toolkit-docs-generator/src/merger/data-merger.ts @@ -98,6 +98,15 @@ export interface MergeResult { warnings: string[]; failedTools: FailedTool[]; error?: string; + /** + * True when the design system had no metadata for this toolkit and + * `getDefaultMetadata`'s placeholder (category, icon, docsLink, and + * `isHidden: true`) was used instead. Also true for the last-known-good + * placeholder in `buildMergeErrorResult`, for the same reason. Callers + * use this to log which toolkits are running on fabricated metadata, + * since that's easy to miss in a warnings list read only on failure. + */ + usedDefaultMetadata: boolean; } export interface ToolExampleResult { @@ -439,16 +448,50 @@ const applyToolkitTypeOverrides = ( return metadata; }; +/** + * Category assigned to a toolkit when the design system has no metadata for + * it at all. `MergedToolkitMetadata.category` is a closed enum + * (`INTEGRATION_CATEGORIES`) with no catch-all value, so this placeholder + * has to be one of the real categories — there is nothing else the schema + * will accept. "development" is picked arbitrarily; it is almost certainly + * wrong for any given toolkit, which is exactly why `getDefaultMetadata` + * also forces `isHidden: true` below instead of trusting this value enough + * to publish a page under it. + */ +const DEFAULT_METADATA_CATEGORY: MergedToolkitMetadata["category"] = + "development"; + +/** + * Metadata used when the design system has no entry for a toolkit at all. + * + * Every field here is a guess, not a fact: none of it came from the design + * system, so none of it should be trusted enough to route or display. The + * category in particular can't be flagged as "unknown" — the schema is a + * closed enum with no catch-all — so a wrong-but-valid category would + * otherwise file the toolkit under the wrong sidebar section with a + * canonical URL nobody chose. `isHidden: true` is what actually neutralizes + * that: `app/_lib/toolkit-static-params.ts` drops hidden toolkits from + * routing entirely, so this placeholder record can exist on disk (and keep + * CI green when metadata truly is optional) without ever rendering under + * the wrong category. Real metadata — pulled in the next successful design + * system sync — clears the flag automatically, since `metadata` will no + * longer be null and this function won't run for that toolkit again. + * + * `DataMerger` only takes this path when `requireCompleteData` is false; + * under `--require-complete` (CI's mode) a missing design-system entry + * fails the run instead, naming the toolkit, before this function is ever + * called. See `DataMerger.assertNoMissingMetadata`. + */ const getDefaultMetadata = (toolkitId: string): MergedToolkitMetadata => applyToolkitTypeOverrides(toolkitId, { - category: "development", + category: DEFAULT_METADATA_CATEGORY, iconUrl: `https://design-system.arcade.dev/icons/${getDefaultIconId(toolkitId)}.svg`, isBYOC: false, isPro: false, type: "arcade", - docsLink: `https://docs.arcade.dev/en/mcp-servers/development/${getDefaultDocsSlug(toolkitId)}`, + docsLink: `https://docs.arcade.dev/en/resources/integrations/${DEFAULT_METADATA_CATEGORY}/${getDefaultDocsSlug(toolkitId)}`, isComingSoon: false, - isHidden: false, + isHidden: true, }); /** @@ -914,7 +957,12 @@ export const mergeToolkit = async ( warnings.push(...formatFreshnessWarnings(freshnessResult)); } - return { toolkit, warnings, failedTools }; + return { + toolkit, + warnings, + failedTools, + usedDefaultMetadata: metadata === null, + }; }; // ============================================================================ @@ -991,9 +1039,15 @@ export class DataMerger { warnings: [`Error processing toolkit: ${message}`], failedTools: [], error: message, + usedDefaultMetadata: false, }; } + // No previous toolkit to fall back on: this is a first-time toolkit + // whose merge threw before metadata even entered the picture. The + // placeholder below reuses the same "unhidden" category and forced + // `isHidden: true` as `getDefaultMetadata` and for the same reason — + // it's a guess, not a fact, so it must not be routable. return { toolkit: { id: toolkitId, @@ -1001,14 +1055,14 @@ export class DataMerger { version: "0.0.0", description: null, metadata: { - category: "development", + category: DEFAULT_METADATA_CATEGORY, iconUrl: "", isBYOC: false, isPro: false, type: isApiSuffixedToolkitId(toolkitId) ? "arcade_starter" : "arcade", docsLink: "", isComingSoon: false, - isHidden: false, + isHidden: true, }, auth: null, tools: [], @@ -1020,6 +1074,7 @@ export class DataMerger { warnings: [`Error processing toolkit: ${message}`], failedTools: [], error: message, + usedDefaultMetadata: true, }; } @@ -1323,20 +1378,56 @@ export class DataMerger { return result; } + /** + * Under `--require-complete`, a toolkit with no design-system metadata + * must fail the run instead of silently falling back to + * `getDefaultMetadata`'s guessed category/docsLink/icon. Silently + * dropping the toolkit (the old behavior) is just as bad as fabricating + * data for it — either way nobody finds out until a human notices a + * toolkit is missing or mis-filed. Naming every affected toolkit in one + * error, before any concurrent processing starts, keeps CI logs + * unambiguous about exactly what to fix upstream. + */ + private assertNoMissingMetadata( + toolkitEntries: ReadonlyArray + ): void { + if (!this.requireCompleteData) { + return; + } + + const missing = toolkitEntries + .filter( + ([toolkitId, toolkitData]) => + !this.skipToolkitIds.has(toolkitId.toLowerCase()) && + toolkitData.metadata === null + ) + .map(([toolkitId]) => toolkitId); + + if (missing.length > 0) { + throw new Error( + `--require-complete: missing design-system metadata for ${missing.length} toolkit(s): ${missing.join(", ")}. ` + + "Add the toolkit to the design system catalog, or drop --require-complete to continue with a hidden placeholder record." + ); + } + } + /** * Merge data for all toolkits */ async mergeAllToolkits(): Promise { const allToolkitsData = await this.toolkitDataSource.fetchAllToolkitsData(); - const toolkitEntries = Array.from(allToolkitsData.entries()); - // Filter out toolkits that should be skipped (for resume support) + this.assertNoMissingMetadata(toolkitEntries); + + // Filter out toolkits that should be skipped (for resume support) and, + // under --require-complete, toolkits with no tools. Missing metadata is + // no longer filtered here — assertNoMissingMetadata above already threw + // if any slipped through. const filteredEntries = toolkitEntries.filter( ([toolkitId, toolkitData]) => !this.skipToolkitIds.has(toolkitId.toLowerCase()) && - (!this.requireCompleteData || - (toolkitData.metadata !== null && toolkitData.tools.length > 0)) + (!this.requireCompleteData || toolkitData.tools.length > 0) ); const results = await mapWithConcurrency( @@ -1369,12 +1460,15 @@ export class DataMerger { skipped: number; }> { const allToolkitsData = await this.toolkitDataSource.fetchAllToolkitsData(); + const toolkitEntries = Array.from(allToolkitsData.entries()); + + this.assertNoMissingMetadata(toolkitEntries); + const total = allToolkitsData.size; - const skipped = Array.from(allToolkitsData.entries()).filter( + const skipped = toolkitEntries.filter( ([id, toolkitData]) => this.skipToolkitIds.has(id.toLowerCase()) || - (this.requireCompleteData && - (toolkitData.metadata === null || toolkitData.tools.length === 0)) + (this.requireCompleteData && toolkitData.tools.length === 0) ).length; return { total, diff --git a/toolkit-docs-generator/tests/cli/generate-flow.test.ts b/toolkit-docs-generator/tests/cli/generate-flow.test.ts index 12440b99a..083e0110f 100644 --- a/toolkit-docs-generator/tests/cli/generate-flow.test.ts +++ b/toolkit-docs-generator/tests/cli/generate-flow.test.ts @@ -162,7 +162,7 @@ describe("filterProvidersBySkipIds", () => { ); expect(providersToProcess).toHaveLength(1); - expect(providersToProcess[0].provider).toBe("Slack"); + expect(providersToProcess[0]?.provider).toBe("Slack"); expect(skippedProviders).toHaveLength(2); }); diff --git a/toolkit-docs-generator/tests/diff/previous-output.test.ts b/toolkit-docs-generator/tests/diff/previous-output.test.ts index b26899324..45854ad50 100644 --- a/toolkit-docs-generator/tests/diff/previous-output.test.ts +++ b/toolkit-docs-generator/tests/diff/previous-output.test.ts @@ -25,7 +25,6 @@ const createValidToolkit = (): MergedToolkit => ({ qualifiedName: "Github.CreateIssue", fullyQualifiedName: "Github.CreateIssue@1.0.0", description: null, - toolkitDescription: null, parameters: [ { name: "title", diff --git a/toolkit-docs-generator/tests/llm/toolkit-summary-generator.test.ts b/toolkit-docs-generator/tests/llm/toolkit-summary-generator.test.ts index 946159171..9fc35e9dd 100644 --- a/toolkit-docs-generator/tests/llm/toolkit-summary-generator.test.ts +++ b/toolkit-docs-generator/tests/llm/toolkit-summary-generator.test.ts @@ -53,6 +53,7 @@ const createToolkit = ( describe("LlmToolkitSummaryGenerator", () => { it("parses summary from a JSON response", async () => { const client: LlmClient = { + provider: "openai", generateText: async () => '```json\n{"summary":"Concise summary."}\n```', }; const generator = new LlmToolkitSummaryGenerator({ @@ -68,6 +69,7 @@ describe("LlmToolkitSummaryGenerator", () => { it("includes tool descriptions and auth info in the prompt", async () => { let capturedPrompt = ""; const client: LlmClient = { + provider: "openai", generateText: async ({ prompt }) => { capturedPrompt = prompt; return '{"summary":"OK"}'; diff --git a/toolkit-docs-generator/tests/merger/data-merger.test.ts b/toolkit-docs-generator/tests/merger/data-merger.test.ts index e2277f399..04b07b37b 100644 --- a/toolkit-docs-generator/tests/merger/data-merger.test.ts +++ b/toolkit-docs-generator/tests/merger/data-merger.test.ts @@ -22,6 +22,7 @@ import { InMemoryMetadataSource, InMemoryToolDataSource, } from "../../src/sources/in-memory.js"; +import type { ICustomSectionsSource } from "../../src/sources/interfaces.js"; import { createCombinedToolkitDataSource, type IToolkitDataSource, @@ -440,9 +441,33 @@ describe("mergeToolkit", () => { expect(result.toolkit.label).toBe("Unknown"); expect(result.toolkit.metadata.category).toBe("development"); + // The fabricated category can't be trusted enough to route or display — + // isHidden neutralizes it until real design-system metadata arrives. + expect(result.toolkit.metadata.isHidden).toBe(true); + // Must match the real integration route shape (see + // app/_lib/toolkit-static-params.ts getToolkitCanonicalPath), never the + // retired /en/mcp-servers/ prefix. + expect(result.toolkit.metadata.docsLink).toBe( + "https://docs.arcade.dev/en/resources/integrations/development/unknown" + ); expect(result.warnings).toContain( "No metadata found for toolkit: Unknown - using defaults" ); + expect(result.usedDefaultMetadata).toBe(true); + }); + + it("does not flag usedDefaultMetadata when design-system metadata is present", async () => { + const tools = [createTool({ qualifiedName: "TestKit.Tool1" })]; + + const result = await mergeToolkit( + "TestKit", + tools, + createMetadata(), + null, + createStubGenerator() + ); + + expect(result.usedDefaultMetadata).toBe(false); }); it("infers a readable label from toolkit description without metadata", async () => { @@ -1442,7 +1467,7 @@ describe("DataMerger", () => { }, ]; - const cleanupSpy = vi.fn( + const cleanupSpy = vi.fn( async () => "| Secret | Required For |\n| `GITHUB_SERVER_URL` | All tools |" ); @@ -1467,14 +1492,11 @@ describe("DataMerger", () => { const result = await merger.mergeToolkit("Github"); expect(cleanupSpy).toHaveBeenCalledTimes(1); - const cleanupCall = cleanupSpy.mock.calls[0]?.[0] as { - removedSecrets: string[]; - kind: string; - }; - expect(cleanupCall.removedSecrets).toEqual([ + const cleanupCall = cleanupSpy.mock.calls[0]?.[0]; + expect(cleanupCall?.removedSecrets).toEqual([ "GITHUB_CLASSIC_PERSONAL_ACCESS_TOKEN", ]); - expect(cleanupCall.kind).toBe("documentation_chunk"); + expect(cleanupCall?.kind).toBe("documentation_chunk"); // The chunk content in the result reflects the editor output. expect( result.toolkit.documentationChunks[0]?.content.includes( @@ -1887,10 +1909,13 @@ describe("DataMerger", () => { // buildMergeErrorResult is invoked by mergeToolkitEntry (called from // mergeAllToolkits). We trigger it by making the customSectionsSource throw, // which is caught by mergeToolkitEntry's try/catch. - const makeFailingCustomSectionsSource = () => ({ + const makeFailingCustomSectionsSource = (): ICustomSectionsSource => ({ getCustomSections: async () => { throw new Error("Custom sections source unavailable"); }, + getAllCustomSections: async () => { + throw new Error("Custom sections source unavailable"); + }, }); it("preserves documentationChunks and customImports from previous toolkit when merge throws", async () => { @@ -2006,21 +2031,11 @@ describe("DataMerger", () => { expect(slackResult?.toolkit.tools).toHaveLength(1); }); - it("skips toolkits missing metadata or tools when requireCompleteData is true", async () => { + it("skips toolkits with no tools (but present metadata) when requireCompleteData is true", async () => { const completeToolkitData: ToolkitData = { tools: [githubTool1], metadata: githubMetadata, }; - const missingMetadataToolkitData: ToolkitData = { - tools: [ - createTool({ - name: "Lookup", - qualifiedName: "Unknown.Lookup", - fullyQualifiedName: "Unknown.Lookup@1.0.0", - }), - ], - metadata: null, - }; const missingToolsToolkitData: ToolkitData = { tools: [], metadata: slackMetadata, @@ -2031,9 +2046,6 @@ describe("DataMerger", () => { if (toolkitId === "Github") { return completeToolkitData; } - if (toolkitId === "Unknown") { - return missingMetadataToolkitData; - } if (toolkitId === "Slack") { return missingToolsToolkitData; } @@ -2042,7 +2054,6 @@ describe("DataMerger", () => { fetchAllToolkitsData: async () => new Map([ ["Github", completeToolkitData], - ["Unknown", missingMetadataToolkitData], ["Slack", missingToolsToolkitData], ]), isAvailable: async () => true, @@ -2058,13 +2069,110 @@ describe("DataMerger", () => { const count = await merger.getToolkitCount(); const results = await merger.mergeAllToolkits(); - expect(count.total).toBe(3); + expect(count.total).toBe(2); expect(count.toProcess).toBe(1); - expect(count.skipped).toBe(2); + expect(count.skipped).toBe(1); expect(results).toHaveLength(1); expect(results[0]?.toolkit.id).toBe("Github"); }); + it("fails the run and names every toolkit missing design-system metadata when requireCompleteData is true", async () => { + // Silently dropping (the old behavior) or silently fabricating + // metadata for these toolkits are both worse than failing loudly: + // --require-complete exists so CI can't ship a wrong-but-valid + // category or a docsLink nobody chose. The error must name every + // affected toolkit, not just the first one found, so a single CI + // failure is enough to fix the whole batch. + const completeToolkitData: ToolkitData = { + tools: [githubTool1], + metadata: githubMetadata, + }; + const missingMetadataToolkitData: ToolkitData = { + tools: [ + createTool({ + name: "Lookup", + qualifiedName: "Unknown.Lookup", + fullyQualifiedName: "Unknown.Lookup@1.0.0", + }), + ], + metadata: null, + }; + const anotherMissingMetadataToolkitData: ToolkitData = { + tools: [ + createTool({ + name: "Ping", + qualifiedName: "AlsoUnknown.Ping", + fullyQualifiedName: "AlsoUnknown.Ping@1.0.0", + }), + ], + metadata: null, + }; + + const toolkitDataSource: IToolkitDataSource = { + fetchToolkitData: async () => { + throw new Error("not used by mergeAllToolkits"); + }, + fetchAllToolkitsData: async () => + new Map([ + ["Github", completeToolkitData], + ["Unknown", missingMetadataToolkitData], + ["AlsoUnknown", anotherMissingMetadataToolkitData], + ]), + isAvailable: async () => true, + }; + + const merger = new DataMerger({ + toolkitDataSource, + customSectionsSource: new EmptyCustomSectionsSource(), + toolExampleGenerator: createStubGenerator(), + requireCompleteData: true, + }); + + await expect(merger.mergeAllToolkits()).rejects.toThrow( + /missing design-system metadata.*Unknown.*AlsoUnknown/s + ); + await expect(merger.getToolkitCount()).rejects.toThrow( + /missing design-system metadata.*Unknown.*AlsoUnknown/s + ); + }); + + it("does not skip or fail toolkits missing metadata when requireCompleteData is false", async () => { + // Without --require-complete, generation must still complete — the + // toolkit falls back to getDefaultMetadata (hidden placeholder + // metadata) and usedDefaultMetadata reports the fact rather than the + // omission disappearing entirely. + const missingMetadataToolkitData: ToolkitData = { + tools: [ + createTool({ + name: "Lookup", + qualifiedName: "Unknown.Lookup", + fullyQualifiedName: "Unknown.Lookup@1.0.0", + }), + ], + metadata: null, + }; + + const toolkitDataSource: IToolkitDataSource = { + fetchToolkitData: async () => missingMetadataToolkitData, + fetchAllToolkitsData: async () => + new Map([["Unknown", missingMetadataToolkitData]]), + isAvailable: async () => true, + }; + + const merger = new DataMerger({ + toolkitDataSource, + customSectionsSource: new EmptyCustomSectionsSource(), + toolExampleGenerator: createStubGenerator(), + }); + + const results = await merger.mergeAllToolkits(); + + expect(results).toHaveLength(1); + expect(results[0]?.toolkit.id).toBe("Unknown"); + expect(results[0]?.usedDefaultMetadata).toBe(true); + expect(results[0]?.toolkit.metadata.isHidden).toBe(true); + }); + it("fails strict runs when a complete toolkit cannot be merged", async () => { const toolkitDataSource = createCombinedToolkitDataSource({ toolSource: new InMemoryToolDataSource([githubTool1]), @@ -2076,6 +2184,9 @@ describe("DataMerger", () => { getCustomSections: async () => { throw new Error("Custom sections source unavailable"); }, + getAllCustomSections: async () => { + throw new Error("Custom sections source unavailable"); + }, }, toolExampleGenerator: createStubGenerator(), requireCompleteData: true, diff --git a/toolkit-docs-generator/tests/merger/metadata-freshness.test.ts b/toolkit-docs-generator/tests/merger/metadata-freshness.test.ts index 4973bf9e5..a04fab6ae 100644 --- a/toolkit-docs-generator/tests/merger/metadata-freshness.test.ts +++ b/toolkit-docs-generator/tests/merger/metadata-freshness.test.ts @@ -34,7 +34,7 @@ const createMetadata = ( }); const createPreviousToolkit = ( - overrides: Partial & { + overrides: Omit, "metadata"> & { metadata?: Partial; label?: string; } = {} diff --git a/toolkit-docs-generator/tests/scenarios/skip-unchanged.test.ts b/toolkit-docs-generator/tests/scenarios/skip-unchanged.test.ts index c09caf0c8..5a38bc9f0 100644 --- a/toolkit-docs-generator/tests/scenarios/skip-unchanged.test.ts +++ b/toolkit-docs-generator/tests/scenarios/skip-unchanged.test.ts @@ -10,6 +10,7 @@ import { getChangedToolkitIds, hasChanges, } from "../../src/diff/index.js"; +import type { CurrentToolkitDiffInput } from "../../src/diff/toolkit-diff.js"; import type { MergedToolkit, ToolDefinition } from "../../src/types/index.js"; const createTool = ( @@ -197,7 +198,7 @@ describe("Scenario: Skip unchanged toolkits", () => { }); it("includes metadata-only changes in changed IDs", () => { - const currentToolkitData = new Map([ + const currentToolkitData = new Map([ [ "Github", { diff --git a/toolkit-docs-generator/tests/scripts/sync-toolkit-sidebar.test.ts b/toolkit-docs-generator/tests/scripts/sync-toolkit-sidebar.test.ts index aa0f20397..e2590cd10 100644 --- a/toolkit-docs-generator/tests/scripts/sync-toolkit-sidebar.test.ts +++ b/toolkit-docs-generator/tests/scripts/sync-toolkit-sidebar.test.ts @@ -8,7 +8,7 @@ import { existsSync, mkdirSync, rmSync, writeFileSync } from "node:fs"; import { join } from "node:path"; import { afterEach, beforeEach, describe, expect, it } from "vitest"; -import { getToolkitStaticParamsForCategory } from "../../../app/_lib/toolkit-static-params"; +import { getToolkitStaticParamsForCategory } from "../../../app/_lib/toolkit-static-params.js"; import { buildToolkitInfoList, generateCategoryMeta, @@ -23,7 +23,7 @@ import { setToolkitsForTesting, syncToolkitSidebar, type ToolkitInfo, -} from "../../scripts/sync-toolkit-sidebar"; +} from "../../scripts/sync-toolkit-sidebar.js"; setToolkitsForTesting([ { id: "Gmail", label: "Gmail", category: "productivity" }, @@ -370,12 +370,25 @@ describe("buildToolkitInfoList", () => { describe("groupByCategory", () => { it("should group toolkits by category", () => { const toolkits: ToolkitInfo[] = [ - { id: "gmail", slug: "gmail", label: "Gmail", category: "productivity" }, - { id: "slack", slug: "slack", label: "Slack", category: "social" }, + { + id: "gmail", + slug: "gmail", + label: "Gmail", + navGroup: "optimized", + category: "productivity", + }, + { + id: "slack", + slug: "slack", + label: "Slack", + navGroup: "optimized", + category: "social", + }, { id: "dropbox", slug: "dropbox", label: "Dropbox", + navGroup: "optimized", category: "productivity", }, ]; @@ -389,9 +402,27 @@ describe("groupByCategory", () => { it("should sort toolkits alphabetically by label", () => { const toolkits: ToolkitInfo[] = [ - { id: "zoom", slug: "zoom", label: "Zoom", category: "social" }, - { id: "slack", slug: "slack", label: "Slack", category: "social" }, - { id: "discord", slug: "discord", label: "Discord", category: "social" }, + { + id: "zoom", + slug: "zoom", + label: "Zoom", + navGroup: "optimized", + category: "social", + }, + { + id: "slack", + slug: "slack", + label: "Slack", + navGroup: "optimized", + category: "social", + }, + { + id: "discord", + slug: "discord", + label: "Discord", + navGroup: "optimized", + category: "social", + }, ]; const result = groupByCategory(toolkits); @@ -409,7 +440,13 @@ describe("groupByCategory", () => { it("should handle 'others' category", () => { const toolkits: ToolkitInfo[] = [ - { id: "custom", slug: "custom", label: "Custom", category: "others" }, + { + id: "custom", + slug: "custom", + label: "Custom", + navGroup: "optimized", + category: "others", + }, ]; const result = groupByCategory(toolkits); @@ -479,11 +516,18 @@ describe("remove empty section flags", () => { describe("generateCategoryMeta", () => { it("should generate valid _meta.tsx content", () => { const toolkits: ToolkitInfo[] = [ - { id: "gmail", slug: "gmail", label: "Gmail", category: "productivity" }, + { + id: "gmail", + slug: "gmail", + label: "Gmail", + navGroup: "optimized", + category: "productivity", + }, { id: "dropbox", slug: "dropbox", label: "Dropbox", + navGroup: "optimized", category: "productivity", }, ]; @@ -510,6 +554,7 @@ describe("generateCategoryMeta", () => { id: "test", slug: "test", label: 'Test "Quoted" Label', + navGroup: "optimized", category: "others", }, ]; @@ -529,7 +574,13 @@ describe("generateCategoryMeta", () => { it("should handle single toolkit", () => { const toolkits: ToolkitInfo[] = [ - { id: "gmail", slug: "gmail", label: "Gmail", category: "productivity" }, + { + id: "gmail", + slug: "gmail", + label: "Gmail", + navGroup: "optimized", + category: "productivity", + }, ]; const result = generateCategoryMeta(toolkits, "productivity", "/preview"); diff --git a/toolkit-docs-generator/tests/scripts/validate-merge.test.ts b/toolkit-docs-generator/tests/scripts/validate-merge.test.ts index b17c98bad..339e02d2e 100644 --- a/toolkit-docs-generator/tests/scripts/validate-merge.test.ts +++ b/toolkit-docs-generator/tests/scripts/validate-merge.test.ts @@ -5,7 +5,7 @@ import { afterEach, describe, expect, it } from "vitest"; import { buildValidationOutputLines, validateMergedCustomSections, -} from "../../scripts/validate-merge"; +} from "../../scripts/validate-merge.js"; const JSON_INDENT_SPACES = 2; const ZERO = 0; diff --git a/toolkit-docs-generator/tests/sources/arcade-api.test.ts b/toolkit-docs-generator/tests/sources/arcade-api.test.ts index d09122010..bbb0600ac 100644 --- a/toolkit-docs-generator/tests/sources/arcade-api.test.ts +++ b/toolkit-docs-generator/tests/sources/arcade-api.test.ts @@ -1,3 +1,4 @@ +import type { Mock } from "vitest"; import { beforeEach, describe, expect, it, vi } from "vitest"; import { ArcadeApiSource, @@ -214,11 +215,27 @@ const mockToolWithSecrets: ArcadeToolsResponse["items"][0] = { // Tests // ============================================================================ +/** + * ArcadeApiSource only reads `.ok`, `.status`, `.statusText`, `.headers.get(...)`, + * and `.json()` off the fetch response, so tests mock that subset and assert + * it as a `Response` rather than constructing a real one. + */ +type FetchResponseLike = { + ok: boolean; + status?: number; + statusText?: string; + headers?: { get(name: string): string | null | undefined }; + json?: () => Promise; +}; + +const asResponse = (value: FetchResponseLike): Response => + value as unknown as Response; + describe("ArcadeApiSource", () => { - let mockFetch: ReturnType; + let mockFetch: Mock; beforeEach(() => { - mockFetch = vi.fn(); + mockFetch = vi.fn(); }); describe("constructor and configuration", () => { @@ -232,10 +249,12 @@ describe("ArcadeApiSource", () => { }); it("should normalize base URL with trailing slash", () => { - mockFetch.mockResolvedValueOnce({ - ok: true, - json: () => Promise.resolve(createMockArcadeResponse([])), - }); + mockFetch.mockResolvedValueOnce( + asResponse({ + ok: true, + json: () => Promise.resolve(createMockArcadeResponse([])), + }) + ); const source = new ArcadeApiSource({ baseUrl: "https://api.arcade.dev/", @@ -252,10 +271,12 @@ describe("ArcadeApiSource", () => { }); it("should handle base URL with /v1 suffix", () => { - mockFetch.mockResolvedValueOnce({ - ok: true, - json: () => Promise.resolve(createMockArcadeResponse([])), - }); + mockFetch.mockResolvedValueOnce( + asResponse({ + ok: true, + json: () => Promise.resolve(createMockArcadeResponse([])), + }) + ); const source = new ArcadeApiSource({ baseUrl: "https://api.arcade.dev/v1", @@ -272,10 +293,12 @@ describe("ArcadeApiSource", () => { }); it("should cap page size at maximum", () => { - mockFetch.mockResolvedValueOnce({ - ok: true, - json: () => Promise.resolve(createMockArcadeResponse([])), - }); + mockFetch.mockResolvedValueOnce( + asResponse({ + ok: true, + json: () => Promise.resolve(createMockArcadeResponse([])), + }) + ); const source = new ArcadeApiSource({ baseUrl: "https://api.arcade.dev", @@ -293,13 +316,18 @@ describe("ArcadeApiSource", () => { describe("fetchAllTools", () => { it("should fetch and transform tools correctly", async () => { - mockFetch.mockResolvedValueOnce({ - ok: true, - json: () => - Promise.resolve( - createMockArcadeResponse([mockAirtableTool, mockGoogleCalendarTool]) - ), - }); + mockFetch.mockResolvedValueOnce( + asResponse({ + ok: true, + json: () => + Promise.resolve( + createMockArcadeResponse([ + mockAirtableTool, + mockGoogleCalendarTool, + ]) + ), + }) + ); const source = new ArcadeApiSource({ baseUrl: "https://api.arcade.dev", @@ -362,11 +390,13 @@ describe("ArcadeApiSource", () => { }); it("should handle tools with array parameters", async () => { - mockFetch.mockResolvedValueOnce({ - ok: true, - json: () => - Promise.resolve(createMockArcadeResponse([mockGoogleCalendarTool])), - }); + mockFetch.mockResolvedValueOnce( + asResponse({ + ok: true, + json: () => + Promise.resolve(createMockArcadeResponse([mockGoogleCalendarTool])), + }) + ); const source = new ArcadeApiSource({ baseUrl: "https://api.arcade.dev", @@ -391,11 +421,13 @@ describe("ArcadeApiSource", () => { }); it("should extract secrets from requirements", async () => { - mockFetch.mockResolvedValueOnce({ - ok: true, - json: () => - Promise.resolve(createMockArcadeResponse([mockToolWithSecrets])), - }); + mockFetch.mockResolvedValueOnce( + asResponse({ + ok: true, + json: () => + Promise.resolve(createMockArcadeResponse([mockToolWithSecrets])), + }) + ); const source = new ArcadeApiSource({ baseUrl: "https://api.arcade.dev", @@ -412,17 +444,19 @@ describe("ArcadeApiSource", () => { }); it("should filter by toolkit ID client-side", async () => { - mockFetch.mockResolvedValueOnce({ - ok: true, - json: () => - Promise.resolve( - createMockArcadeResponse([ - mockAirtableTool, - mockGoogleCalendarTool, - mockToolWithSecrets, - ]) - ), - }); + mockFetch.mockResolvedValueOnce( + asResponse({ + ok: true, + json: () => + Promise.resolve( + createMockArcadeResponse([ + mockAirtableTool, + mockGoogleCalendarTool, + mockToolWithSecrets, + ]) + ), + }) + ); const source = new ArcadeApiSource({ baseUrl: "https://api.arcade.dev", @@ -437,11 +471,13 @@ describe("ArcadeApiSource", () => { }); it("should filter by toolkit ID case-insensitively", async () => { - mockFetch.mockResolvedValueOnce({ - ok: true, - json: () => - Promise.resolve(createMockArcadeResponse([mockGoogleCalendarTool])), - }); + mockFetch.mockResolvedValueOnce( + asResponse({ + ok: true, + json: () => + Promise.resolve(createMockArcadeResponse([mockGoogleCalendarTool])), + }) + ); const source = new ArcadeApiSource({ baseUrl: "https://api.arcade.dev", @@ -455,17 +491,19 @@ describe("ArcadeApiSource", () => { }); it("should filter by version", async () => { - mockFetch.mockResolvedValueOnce({ - ok: true, - json: () => - Promise.resolve( - createMockArcadeResponse([ - mockAirtableTool, // @4.0.0 - mockGoogleCalendarTool, // @1.0.0 - mockToolWithSecrets, // @2.0.0 - ]) - ), - }); + mockFetch.mockResolvedValueOnce( + asResponse({ + ok: true, + json: () => + Promise.resolve( + createMockArcadeResponse([ + mockAirtableTool, // @4.0.0 + mockGoogleCalendarTool, // @1.0.0 + mockToolWithSecrets, // @2.0.0 + ]) + ), + }) + ); const source = new ArcadeApiSource({ baseUrl: "https://api.arcade.dev", @@ -485,28 +523,32 @@ describe("ArcadeApiSource", () => { describe("pagination", () => { it("should handle pagination correctly", async () => { // First page - mockFetch.mockResolvedValueOnce({ - ok: true, - json: () => - Promise.resolve({ - items: [mockAirtableTool], - limit: 1, - offset: 0, - total_count: 2, - }), - }); + mockFetch.mockResolvedValueOnce( + asResponse({ + ok: true, + json: () => + Promise.resolve({ + items: [mockAirtableTool], + limit: 1, + offset: 0, + total_count: 2, + }), + }) + ); // Second page - mockFetch.mockResolvedValueOnce({ - ok: true, - json: () => - Promise.resolve({ - items: [mockGoogleCalendarTool], - limit: 1, - offset: 1, - total_count: 2, - }), - }); + mockFetch.mockResolvedValueOnce( + asResponse({ + ok: true, + json: () => + Promise.resolve({ + items: [mockGoogleCalendarTool], + limit: 1, + offset: 1, + total_count: 2, + }), + }) + ); const source = new ArcadeApiSource({ baseUrl: "https://api.arcade.dev", @@ -522,27 +564,31 @@ describe("ArcadeApiSource", () => { }); it("should stop pagination when no more items", async () => { - mockFetch.mockResolvedValueOnce({ - ok: true, - json: () => - Promise.resolve({ - items: [mockAirtableTool], - limit: 100, - offset: 0, - total_count: 100, // Says 100 but only returns 1 - }), - }); + mockFetch.mockResolvedValueOnce( + asResponse({ + ok: true, + json: () => + Promise.resolve({ + items: [mockAirtableTool], + limit: 100, + offset: 0, + total_count: 100, // Says 100 but only returns 1 + }), + }) + ); - mockFetch.mockResolvedValueOnce({ - ok: true, - json: () => - Promise.resolve({ - items: [], - limit: 100, - offset: 1, - total_count: 100, - }), - }); + mockFetch.mockResolvedValueOnce( + asResponse({ + ok: true, + json: () => + Promise.resolve({ + items: [], + limit: 100, + offset: 1, + total_count: 100, + }), + }) + ); const source = new ArcadeApiSource({ baseUrl: "https://api.arcade.dev", @@ -558,11 +604,13 @@ describe("ArcadeApiSource", () => { describe("fetchToolsByToolkit", () => { it("should call fetchAllTools with toolkit filter", async () => { - mockFetch.mockResolvedValueOnce({ - ok: true, - json: () => - Promise.resolve(createMockArcadeResponse([mockGoogleCalendarTool])), - }); + mockFetch.mockResolvedValueOnce( + asResponse({ + ok: true, + json: () => + Promise.resolve(createMockArcadeResponse([mockGoogleCalendarTool])), + }) + ); const source = new ArcadeApiSource({ baseUrl: "https://api.arcade.dev", @@ -579,10 +627,12 @@ describe("ArcadeApiSource", () => { describe("isAvailable", () => { it("should return true when API is accessible", async () => { - mockFetch.mockResolvedValueOnce({ - ok: true, - json: () => Promise.resolve(createMockArcadeResponse([])), - }); + mockFetch.mockResolvedValueOnce( + asResponse({ + ok: true, + json: () => Promise.resolve(createMockArcadeResponse([])), + }) + ); const source = new ArcadeApiSource({ baseUrl: "https://api.arcade.dev", @@ -595,11 +645,13 @@ describe("ArcadeApiSource", () => { }); it("should return false when API returns error", async () => { - mockFetch.mockResolvedValueOnce({ - ok: false, - status: 401, - statusText: "Unauthorized", - }); + mockFetch.mockResolvedValueOnce( + asResponse({ + ok: false, + status: 401, + statusText: "Unauthorized", + }) + ); const source = new ArcadeApiSource({ baseUrl: "https://api.arcade.dev", @@ -627,13 +679,15 @@ describe("ArcadeApiSource", () => { describe("error handling", () => { it("should throw on API error with JSON detail", async () => { - mockFetch.mockResolvedValueOnce({ - ok: false, - status: 401, - statusText: "Unauthorized", - headers: new Map([["content-type", "application/json"]]), - json: () => Promise.resolve({ detail: "Invalid API key" }), - }); + mockFetch.mockResolvedValueOnce( + asResponse({ + ok: false, + status: 401, + statusText: "Unauthorized", + headers: new Map([["content-type", "application/json"]]), + json: () => Promise.resolve({ detail: "Invalid API key" }), + }) + ); const source = new ArcadeApiSource({ baseUrl: "https://api.arcade.dev", @@ -647,12 +701,14 @@ describe("ArcadeApiSource", () => { }); it("should throw on API error without JSON detail", async () => { - mockFetch.mockResolvedValueOnce({ - ok: false, - status: 500, - statusText: "Internal Server Error", - headers: new Map([["content-type", "text/plain"]]), - }); + mockFetch.mockResolvedValueOnce( + asResponse({ + ok: false, + status: 500, + statusText: "Internal Server Error", + headers: new Map([["content-type", "text/plain"]]), + }) + ); const source = new ArcadeApiSource({ baseUrl: "https://api.arcade.dev", @@ -666,10 +722,12 @@ describe("ArcadeApiSource", () => { }); it("should throw on invalid response schema", async () => { - mockFetch.mockResolvedValueOnce({ - ok: true, - json: () => Promise.resolve({ invalid: "response" }), - }); + mockFetch.mockResolvedValueOnce( + asResponse({ + ok: true, + json: () => Promise.resolve({ invalid: "response" }), + }) + ); const source = new ArcadeApiSource({ baseUrl: "https://api.arcade.dev", @@ -685,10 +743,12 @@ describe("ArcadeApiSource", () => { describe("authorization header", () => { it("should include Bearer token in request", async () => { - mockFetch.mockResolvedValueOnce({ - ok: true, - json: () => Promise.resolve(createMockArcadeResponse([])), - }); + mockFetch.mockResolvedValueOnce( + asResponse({ + ok: true, + json: () => Promise.resolve(createMockArcadeResponse([])), + }) + ); const source = new ArcadeApiSource({ baseUrl: "https://api.arcade.dev", diff --git a/toolkit-docs-generator/tests/sources/engine-api.test.ts b/toolkit-docs-generator/tests/sources/engine-api.test.ts index ebadf86a7..02852a058 100644 --- a/toolkit-docs-generator/tests/sources/engine-api.test.ts +++ b/toolkit-docs-generator/tests/sources/engine-api.test.ts @@ -39,7 +39,7 @@ type ToolMetadataItem = { provider_id: string | null; provider_type: string | null; scopes: string[]; - }>; + }> | null; secrets: Array<{ key: string }>; } | null; metadata?: { @@ -111,7 +111,7 @@ const createItems = (): ToolMetadataItem[] => [ const createFetchStub = (items: ToolMetadataItem[], status = 200) => - async (input: RequestInfo | URL) => { + async (input: string | URL | Request) => { if (status !== 200) { return new Response("error", { status }); } @@ -162,7 +162,7 @@ const createErrorFetchStub = (status: number, payload: unknown) => async () => const createInspectFetchStub = (inspect: (params: URLSearchParams) => void) => - async (input: RequestInfo | URL) => { + async (input: string | URL | Request) => { const url = new URL(input.toString()); inspect(url.searchParams); return new Response( @@ -179,7 +179,7 @@ const createInspectFetchStub = const createSummaryFetchStub = (payload: unknown, inspect?: (url: URL) => void) => - async (input: RequestInfo | URL) => { + async (input: string | URL | Request) => { const url = new URL(input.toString()); inspect?.(url); return new Response(JSON.stringify(payload), { @@ -228,7 +228,7 @@ describe("EngineApiSource", () => { description: "GitHub toolkit", }, input: { parameters: [] }, - output: {} as ToolMetadataItem["output"], + output: {} as NonNullable, requirements: { authorization: null, secrets: [], diff --git a/toolkit-docs-generator/tests/utils/output-dir.test.ts b/toolkit-docs-generator/tests/utils/output-dir.test.ts index 278cda2f9..15e96d032 100644 --- a/toolkit-docs-generator/tests/utils/output-dir.test.ts +++ b/toolkit-docs-generator/tests/utils/output-dir.test.ts @@ -8,11 +8,29 @@ import { resolveSafeOutputDir, } from "../../src/utils/output-dir.js"; +type ResolveOptions = { repoRoot?: string; homeDir?: string }; + describe("resolveSafeOutputDir", () => { const originalCwd = process.cwd(); let repoRoot: string | null = null; let homeDir: string | null = null; + /** + * `repoRoot`/`homeDir` are optional properties without `exactOptionalPropertyTypes` + * allowing an explicit `undefined` value, so this only sets a key when the + * corresponding temp dir has actually been created for the current test. + */ + const dirOptions = (): ResolveOptions => { + const opts: ResolveOptions = {}; + if (repoRoot) { + opts.repoRoot = repoRoot; + } + if (homeDir) { + opts.homeDir = homeDir; + } + return opts; + }; + beforeEach(async () => { repoRoot = await mkdtemp(join(tmpdir(), "generator-repo-")); homeDir = await mkdtemp(join(tmpdir(), "generator-home-")); @@ -35,8 +53,7 @@ describe("resolveSafeOutputDir", () => { await mkdir(join(repoRoot ?? "", "output"), { recursive: true }); const resolved = await resolveSafeOutputDir("output", { - repoRoot: repoRoot ?? undefined, - homeDir: homeDir ?? undefined, + ...dirOptions(), }); const expected = await realpath(join(repoRoot ?? "", "output")); @@ -46,8 +63,7 @@ describe("resolveSafeOutputDir", () => { it("rejects relative paths that escape the repo root", async () => { await expect( resolveSafeOutputDir("../outside", { - repoRoot: repoRoot ?? undefined, - homeDir: homeDir ?? undefined, + ...dirOptions(), }) ).rejects.toThrow("outside repo root"); }); @@ -55,8 +71,7 @@ describe("resolveSafeOutputDir", () => { it("rejects deleting the filesystem root", async () => { await expect( resolveSafeOutputDir("/", { - repoRoot: repoRoot ?? undefined, - homeDir: homeDir ?? undefined, + ...dirOptions(), }) ).rejects.toThrow("unsafe output directory"); }); @@ -64,8 +79,7 @@ describe("resolveSafeOutputDir", () => { it("rejects deleting the home directory", async () => { await expect( resolveSafeOutputDir(homeDir ?? "", { - repoRoot: repoRoot ?? undefined, - homeDir: homeDir ?? undefined, + ...dirOptions(), }) ).rejects.toThrow("unsafe output directory"); }); @@ -101,8 +115,7 @@ describe("resolveSafeOutputDir", () => { const expected = await realpath(outputDir); const cleared = await clearSafeOutputDir(outputDir, { - repoRoot: repoRoot ?? undefined, - homeDir: homeDir ?? undefined, + ...dirOptions(), }); expect(cleared).toBe(expected); @@ -116,8 +129,7 @@ describe("resolveSafeOutputDir", () => { try { await expect( clearSafeOutputDir(`../${outsideName}`, { - repoRoot: repoRoot ?? undefined, - homeDir: homeDir ?? undefined, + ...dirOptions(), }) ).rejects.toThrow("outside repo root"); expect(await realpath(outsideDir)).toContain(outsideName); diff --git a/toolkit-docs-generator/tsconfig.json b/toolkit-docs-generator/tsconfig.json index 8acdb3fea..7dbea96ae 100644 --- a/toolkit-docs-generator/tsconfig.json +++ b/toolkit-docs-generator/tsconfig.json @@ -12,7 +12,7 @@ "declarationMap": true, "sourceMap": true, "outDir": "./dist", - "rootDir": "./src", + "rootDir": ".", "resolveJsonModule": true, "noUnusedLocals": true, "noUnusedParameters": true, @@ -21,6 +21,10 @@ "exactOptionalPropertyTypes": true, "noUncheckedIndexedAccess": true }, - "include": ["src/**/*"], - "exclude": ["node_modules", "dist", "**/*.test.ts"] + "include": ["src/**/*", "scripts/**/*", "tests/**/*"], + "exclude": [ + "node_modules", + "dist", + "tests/scripts/sync-toolkit-sidebar.test.ts" + ] } diff --git a/toolkit-docs-generator/vitest.config.ts b/toolkit-docs-generator/vitest.config.ts deleted file mode 100644 index 3ff060c9f..000000000 --- a/toolkit-docs-generator/vitest.config.ts +++ /dev/null @@ -1,41 +0,0 @@ -import { fileURLToPath } from "node:url"; -import { defineConfig } from "vitest/config"; - -export default defineConfig({ - root: fileURLToPath(new URL(".", import.meta.url)), - test: { - // Enable globals like describe, it, expect without imports - globals: true, - - // Test environment - environment: "node", - - // Include test files - include: ["tests/**/*.test.ts"], - - // Coverage configuration - coverage: { - provider: "v8", - reporter: ["text", "json", "html"], - exclude: [ - "node_modules/", - "dist/", - "tests/", - "**/*.d.ts", - "vitest.config.ts", - ], - // Require 80% coverage - thresholds: { - lines: 80, - functions: 80, - branches: 80, - statements: 80, - }, - }, - - // TypeScript configuration - typecheck: { - enabled: true, - }, - }, -}); diff --git a/tsconfig.json b/tsconfig.json index ca0fc204d..1ebdde928 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -23,11 +23,16 @@ }, "strictNullChecks": true }, + "files": [ + "toolkit-docs-generator/tests/scripts/sync-toolkit-sidebar.test.ts" + ], "include": [ "next-env.d.ts", "app/**/*.ts", "app/**/*.tsx", "lib/**/*.ts", + "tests/**/*.ts", + "tests/**/*.tsx", "_dictionaries/**/*.ts", ".next/types/**/*.ts", "app/[lang]/page.mdx", From 29259648f160bf16ee376b377e0fb340f93d299a Mon Sep 17 00:00:00 2001 From: Teal Larson Date: Fri, 31 Jul 2026 15:10:51 -0400 Subject: [PATCH 06/17] fix: enforce --require-complete before skip-unchanged early exit The scheduled workflow uses both flags together; metadata-missing toolkits were excluded from change detection and the no-op path never validated them. Co-authored-by: Cursor --- toolkit-docs-generator/src/cli/index.ts | 36 ++++--------- .../src/merger/data-merger.ts | 34 +++++++----- .../tests/cli/generate-flow.test.ts | 52 +++++++++++++++++++ .../tests/merger/data-merger.test.ts | 35 +++++++++++++ 4 files changed, 117 insertions(+), 40 deletions(-) diff --git a/toolkit-docs-generator/src/cli/index.ts b/toolkit-docs-generator/src/cli/index.ts index 3d911bc35..8b22ff12e 100644 --- a/toolkit-docs-generator/src/cli/index.ts +++ b/toolkit-docs-generator/src/cli/index.ts @@ -38,7 +38,10 @@ import { LlmToolkitSummaryGenerator, } from "../llm/index.js"; import type { MergeResult } from "../merger/data-merger.js"; -import { createDataMerger } from "../merger/data-merger.js"; +import { + assertRequireCompleteMetadata, + createDataMerger, +} from "../merger/data-merger.js"; import { createCustomSectionsFileSource } from "../sources/custom-sections-file.js"; import { createDesignSystemMetadataSource } from "../sources/design-system-metadata.js"; import { createEmptyCustomSectionsSource } from "../sources/in-memory.js"; @@ -133,13 +136,6 @@ const buildLogPaths = (logDir: string) => ({ failedToolsPath: join(logDir, "failed-tools.json"), }); -const getToolkitIdsWithoutMetadata = ( - toolkitsData: ReadonlyMap -): string[] => - Array.from(toolkitsData.entries()) - .filter(([, toolkitData]) => toolkitData.metadata === null) - .map(([toolkitId]) => toolkitId); - const createMetadataSource = async (options: { metadataFile: string; useMetadataFile: boolean; @@ -1303,28 +1299,16 @@ program ); } - const metadataExcludedToolkitIds = requireComplete - ? getToolkitIdsWithoutMetadata(currentToolkitsData) - : []; - const metadataExcludedToolkitIdSet = new Set( - metadataExcludedToolkitIds.map((id) => id.toLowerCase()) - ); - if (options.verbose && metadataExcludedToolkitIds.length > 0) { - console.log( - chalk.dim( - ` Excluding ${metadataExcludedToolkitIds.length} toolkit(s) without metadata before change detection` - ) + if (requireComplete) { + assertRequireCompleteMetadata( + Array.from(currentToolkitsData.entries()) ); } // Build map of toolkit ID -> current toolkit data for comparison - const currentToolkitDataForDiff = new Map(); - for (const [id, data] of currentToolkitsData) { - if (metadataExcludedToolkitIdSet.has(id.toLowerCase())) { - continue; - } - currentToolkitDataForDiff.set(id, data); - } + const currentToolkitDataForDiff = new Map( + currentToolkitsData + ); assertSafeCurrentToolkitSnapshot( currentToolkitDataForDiff.size, previousToolkits?.size ?? 0 diff --git a/toolkit-docs-generator/src/merger/data-merger.ts b/toolkit-docs-generator/src/merger/data-merger.ts index 315725980..f0fdfcae7 100644 --- a/toolkit-docs-generator/src/merger/data-merger.ts +++ b/toolkit-docs-generator/src/merger/data-merger.ts @@ -122,6 +122,25 @@ export interface ToolkitSummaryGenerator { generate: (toolkit: MergedToolkit) => Promise; } +/** + * Under `--require-complete`, every toolkit must have design-system metadata. + * Fails the run with every affected toolkit named in one error. + */ +export const assertRequireCompleteMetadata = ( + toolkitEntries: ReadonlyArray +): void => { + const missing = toolkitEntries + .filter(([, toolkitData]) => toolkitData.metadata === null) + .map(([toolkitId]) => toolkitId); + + if (missing.length > 0) { + throw new Error( + `--require-complete: missing design-system metadata for ${missing.length} toolkit(s): ${missing.join(", ")}. ` + + "Add the toolkit to the design system catalog, or drop --require-complete to continue with a hidden placeholder record." + ); + } +}; + interface MergeToolkitOptions { previousToolkit?: MergedToolkit; /** Maximum concurrent LLM calls for tool examples (default: 5) */ @@ -1395,20 +1414,7 @@ export class DataMerger { return; } - const missing = toolkitEntries - .filter( - ([toolkitId, toolkitData]) => - !this.skipToolkitIds.has(toolkitId.toLowerCase()) && - toolkitData.metadata === null - ) - .map(([toolkitId]) => toolkitId); - - if (missing.length > 0) { - throw new Error( - `--require-complete: missing design-system metadata for ${missing.length} toolkit(s): ${missing.join(", ")}. ` + - "Add the toolkit to the design system catalog, or drop --require-complete to continue with a hidden placeholder record." - ); - } + assertRequireCompleteMetadata(toolkitEntries); } /** diff --git a/toolkit-docs-generator/tests/cli/generate-flow.test.ts b/toolkit-docs-generator/tests/cli/generate-flow.test.ts index 083e0110f..b403169d5 100644 --- a/toolkit-docs-generator/tests/cli/generate-flow.test.ts +++ b/toolkit-docs-generator/tests/cli/generate-flow.test.ts @@ -6,6 +6,8 @@ import { filterProvidersBySkipIds, } from "../../src/cli/generate-flow.js"; import type { ChangeDetectionResult } from "../../src/diff/index.js"; +import { assertRequireCompleteMetadata } from "../../src/merger/data-merger.js"; +import type { ToolkitData } from "../../src/sources/toolkit-data-source.js"; // ── Minimal ChangeDetectionResult builder ───────────────────────────────────── @@ -215,3 +217,53 @@ describe("filterProvidersBySkipIds", () => { expect(providerNames).toContain("Jira"); }); }); + +describe("assertRequireCompleteMetadata", () => { + it("throws when any toolkit is missing design-system metadata", () => { + const complete: ToolkitData = { + tools: [], + metadata: { + id: "Github", + label: "Github", + category: "development", + iconUrl: "https://example.com/icon.svg", + isBYOC: false, + isPro: false, + type: "arcade", + docsLink: "https://docs.example.com", + isComingSoon: false, + isHidden: false, + }, + }; + const missing: ToolkitData = { tools: [], metadata: null }; + + expect(() => + assertRequireCompleteMetadata([ + ["Github", complete], + ["Unknown", missing], + ]) + ).toThrow(/missing design-system metadata.*Unknown/); + }); + + it("passes when every toolkit has metadata", () => { + const complete: ToolkitData = { + tools: [], + metadata: { + id: "Github", + label: "Github", + category: "development", + iconUrl: "https://example.com/icon.svg", + isBYOC: false, + isPro: false, + type: "arcade", + docsLink: "https://docs.example.com", + isComingSoon: false, + isHidden: false, + }, + }; + + expect(() => + assertRequireCompleteMetadata([["Github", complete]]) + ).not.toThrow(); + }); +}); diff --git a/toolkit-docs-generator/tests/merger/data-merger.test.ts b/toolkit-docs-generator/tests/merger/data-merger.test.ts index 04b07b37b..b64da70e0 100644 --- a/toolkit-docs-generator/tests/merger/data-merger.test.ts +++ b/toolkit-docs-generator/tests/merger/data-merger.test.ts @@ -2136,6 +2136,41 @@ describe("DataMerger", () => { ); }); + it("fails strict runs for toolkits in skipToolkitIds when metadata is missing", async () => { + const missingMetadataToolkitData: ToolkitData = { + tools: [ + createTool({ + name: "Lookup", + qualifiedName: "Unknown.Lookup", + fullyQualifiedName: "Unknown.Lookup@1.0.0", + }), + ], + metadata: null, + }; + + const toolkitDataSource: IToolkitDataSource = { + fetchToolkitData: async () => missingMetadataToolkitData, + fetchAllToolkitsData: async () => + new Map([["Unknown", missingMetadataToolkitData]]), + isAvailable: async () => true, + }; + + const merger = new DataMerger({ + toolkitDataSource, + customSectionsSource: new EmptyCustomSectionsSource(), + toolExampleGenerator: createStubGenerator(), + requireCompleteData: true, + skipToolkitIds: new Set(["unknown"]), + }); + + await expect(merger.mergeAllToolkits()).rejects.toThrow( + /missing design-system metadata.*Unknown/s + ); + await expect(merger.getToolkitCount()).rejects.toThrow( + /missing design-system metadata.*Unknown/s + ); + }); + it("does not skip or fail toolkits missing metadata when requireCompleteData is false", async () => { // Without --require-complete, generation must still complete — the // toolkit falls back to getDefaultMetadata (hidden placeholder From 0654c852d181259ad1571174ce04376e3662f2fc Mon Sep 17 00:00:00 2001 From: Teal Larson Date: Mon, 3 Aug 2026 15:33:06 -0400 Subject: [PATCH 07/17] test: cover email HAST traversal --- .../toolkit-docs/lib/neutralize-emails.tsx | 4 +- tests/neutralize-emails.test.tsx | 96 +++++++++++++++---- 2 files changed, 78 insertions(+), 22 deletions(-) diff --git a/app/_components/toolkit-docs/lib/neutralize-emails.tsx b/app/_components/toolkit-docs/lib/neutralize-emails.tsx index ea695d66f..731d462a4 100644 --- a/app/_components/toolkit-docs/lib/neutralize-emails.tsx +++ b/app/_components/toolkit-docs/lib/neutralize-emails.tsx @@ -49,7 +49,7 @@ export function splitEmails(text: string): ReactNode { } /** Splits `value` into text/`` element pairs at each email `@` break. */ -function neutralizeTextValue(value: string): Array { +export function splitEmailText(value: string): Array { const breaks = atBreakOffsets(value); const out: Array = []; let cursor = 0; @@ -73,7 +73,7 @@ export function rehypeNeutralizeEmails() { if (index === undefined || !parent) { return; } - const replacement = neutralizeTextValue(node.value); + const replacement = splitEmailText(node.value); if (replacement.length <= 1) { return; } diff --git a/tests/neutralize-emails.test.tsx b/tests/neutralize-emails.test.tsx index 47e7411f9..54c16d664 100644 --- a/tests/neutralize-emails.test.tsx +++ b/tests/neutralize-emails.test.tsx @@ -1,8 +1,10 @@ +import type { Element, Root, RootContent } from "hast"; import { renderToStaticMarkup } from "react-dom/server"; import { describe, expect, test } from "vitest"; import { rehypeNeutralizeEmails, splitEmails, + splitEmailText, } from "@/app/_components/toolkit-docs/lib/neutralize-emails"; /** @@ -39,27 +41,51 @@ describe("splitEmails", () => { }); }); -type HastNode = { - type: string; - value?: string; - tagName?: string; - properties?: Record; - children?: HastNode[]; -}; +describe("splitEmailText", () => { + test("returns one text node when there are no emails", () => { + expect(splitEmailText("just some text")).toEqual([ + { type: "text", value: "just some text" }, + ]); + }); + + test("splits every email while preserving the text", () => { + const nodes = splitEmailText( + "contact jane@example.com or sam@example.org today" + ); + + expect(nodes).toEqual([ + { type: "text", value: "contact jane" }, + { type: "element", tagName: "wbr", properties: {}, children: [] }, + { type: "text", value: "@example.com or sam" }, + { type: "element", tagName: "wbr", properties: {}, children: [] }, + { type: "text", value: "@example.org today" }, + ]); + expect( + nodes + .filter((node) => node.type === "text") + .map((node) => node.value) + .join("") + ).toBe("contact jane@example.com or sam@example.org today"); + }); +}); -const collectText = (node: HastNode): string => - node.type === "text" - ? (node.value ?? "") - : (node.children ?? []).map(collectText).join(""); +const collectText = (node: Root | RootContent): string => { + if (node.type === "text") { + return node.value; + } + return "children" in node ? node.children.map(collectText).join("") : ""; +}; -const hasContiguousEmail = (node: HastNode): boolean => - node.type === "text" - ? EMAIL.test(node.value ?? "") - : (node.children ?? []).some(hasContiguousEmail); +const hasContiguousEmail = (node: Root | RootContent): boolean => { + if (node.type === "text") { + return EMAIL.test(node.value); + } + return "children" in node ? node.children.some(hasContiguousEmail) : false; +}; describe("rehypeNeutralizeEmails", () => { test("splits email text nodes and inserts a , losslessly", () => { - const tree: HastNode = { + const tree: Root = { type: "root", children: [ { @@ -73,13 +99,43 @@ describe("rehypeNeutralizeEmails", () => { rehypeNeutralizeEmails()(tree); - const paragraph = tree.children?.[0]; - expect(paragraph?.children?.some((child) => child.tagName === "wbr")).toBe( - true - ); + const paragraph = tree.children[0] as Element; + expect( + paragraph.children.some( + (child) => child.type === "element" && child.tagName === "wbr" + ) + ).toBe(true); // No single text node still holds a full email... expect(hasContiguousEmail(tree)).toBe(false); // ...and the concatenated text is unchanged. expect(collectText(tree)).toBe("reach user@example.com now"); }); + + test("visits nested elements and every matching text node", () => { + const tree: Root = { + type: "root", + children: [ + { + type: "element", + tagName: "blockquote", + properties: {}, + children: [ + { + type: "element", + tagName: "em", + properties: {}, + children: [{ type: "text", value: "jane@example.com" }], + }, + { type: "text", value: " and sam@example.org" }, + ], + }, + ], + }; + + rehypeNeutralizeEmails()(tree); + + expect(hasContiguousEmail(tree)).toBe(false); + expect(collectText(tree)).toBe("jane@example.com and sam@example.org"); + expect(JSON.stringify(tree).match(/"tagName":"wbr"/g)).toHaveLength(2); + }); }); From 849e85643d090a1ebcd338dcdfe967cf4486ed86 Mon Sep 17 00:00:00 2001 From: Teal Larson Date: Fri, 31 Jul 2026 10:01:46 -0400 Subject: [PATCH 08/17] fix: restore static rendering and add toolkit pages to the sitemap MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two isolated behavior fixes. Static rendering: the root layout awaited headers() to read "x-pathname" and derive a locale. Awaiting headers() in the root layout opts the entire route tree out of static rendering, so every page — including all 117 toolkit pages, which are pure functions of committed JSON — was server-rendered on demand. The derived locale was always "en": proxy.ts redirects every non-English locale to /en and getPreferredLocale returns "en" unconditionally. The site paid full dynamic rendering to compute a constant. This is not an i18n change. TranslationBanner and the dictionary plumbing stay in place; restoring real i18n means an app/[lang]/ route segment, which is the correct Next pattern regardless. Sitemap: app/sitemap.ts skips any directory whose name contains "[", which is right for directory walking but meant all 117 toolkit pages were absent from sitemap.xml — the largest content section on the site. Merges in listValidIntegrationLinks() from app/_lib/toolkit-static-params.ts, the same enumeration the integrations index uses, and dedupes against the authored partner pages the disk walk already finds. Co-Authored-By: Claude Opus 5 (1M context) --- app/layout.tsx | 21 ++++----- app/sitemap.ts | 63 ++++++++++++++++++++++++-- tests/sitemap.test.ts | 8 ++++ toolkit-docs-generator/ARCHITECTURE.md | 7 +-- 4 files changed, 81 insertions(+), 18 deletions(-) diff --git a/app/layout.tsx b/app/layout.tsx index 2bd3723f7..421c1e73a 100644 --- a/app/layout.tsx +++ b/app/layout.tsx @@ -10,7 +10,6 @@ import { TranslationBanner } from "@/app/_components/translation-banner"; import "@/app/globals.css"; import { Discord, Github } from "@arcadeai/design-system"; import { GoogleTagManager } from "@next/third-parties/google"; -import { headers } from "next/headers"; import Link from "next/link"; import Script from "next/script"; import { Head } from "nextra/components"; @@ -22,8 +21,6 @@ import { Footer as NextraFooter, } from "nextra-theme-docs"; -const REGEX_LOCALE = /^\/([a-z]{2}(?:-[A-Z]{2})?)(?:\/|$)/; - /** * Nextra's active-state detection only checks `item.route`, never `item.href`. * Toolkit sidebar entries use `href` (required so Nextra doesn't fail validation @@ -94,19 +91,21 @@ export function generateMetadata() { }; } -function getLocaleFromPathname(pathname: string): string { - const localeMatch = pathname.match(REGEX_LOCALE); - return localeMatch?.[1] || "en"; -} - export default async function RootLayout({ children, }: { children: React.ReactNode; }) { - const headersList = await headers(); - const pathname = headersList.get("x-pathname") || "/"; - const lang = getLocaleFromPathname(pathname); + // proxy.ts redirects every request to a "/en/..." path — "es" and + // "pt-BR" routes bounce to their "/en" equivalent and unlocaled routes + // pick up "/en" from getPreferredLocale, which is hardcoded to return + // "en" unconditionally. So this layout only ever renders under "/en", + // and reading the locale here can be a constant instead of a header + // lookup. Awaiting headers() in the root layout previously forced the + // entire route tree into dynamic rendering. Restoring real i18n means + // moving this layout under an `app/[lang]/` route segment so the + // locale comes from routing params, not a request header. + const lang = "en"; const dictionary = await getDictionary(lang); const rawPageMap = await getPageMap(`/${lang}`); diff --git a/app/sitemap.ts b/app/sitemap.ts index 45b6615cf..54a2442e0 100644 --- a/app/sitemap.ts +++ b/app/sitemap.ts @@ -1,6 +1,7 @@ import fs from "node:fs/promises"; import path from "node:path"; import type { MetadataRoute } from "next"; +import { listValidIntegrationLinks } from "./_lib/toolkit-static-params"; const SITE_URL = process.env.SITE_URL ?? "https://docs.arcade.dev"; const NORMALIZED_SITE_URL = SITE_URL.replace(/\/+$/, ""); @@ -43,12 +44,66 @@ async function collectRoutes(dir: string): Promise { return entries; } +/** + * `[toolkitId]` directories are skipped above because they aren't literal + * URLs — but `listValidIntegrationLinks()` (the same enumeration the + * integrations index page uses) already resolves every toolkit that dynamic + * route serves, plus a handful of authored static partner pages living + * alongside it. Skip any link the directory walk already found (the static + * ones) so it isn't listed twice. + */ +async function collectToolkitRoutes( + existingPaths: Set +): Promise { + const links = await listValidIntegrationLinks(); + const entries: MetadataRoute.Sitemap = []; + const categoryPageMtime = new Map(); + + for (const link of links) { + if (existingPaths.has(link)) { + continue; + } + + const category = link.split("/").at(-2) ?? ""; + let mtime = categoryPageMtime.get(category); + if (!mtime) { + const pageFile = path.join( + APP_DIR, + "en", + "resources", + "integrations", + category, + "[toolkitId]", + "page.mdx" + ); + mtime = (await fs.stat(pageFile)).mtime; + categoryPageMtime.set(category, mtime); + } + + entries.push({ + url: `${NORMALIZED_SITE_URL}${link}`, + lastModified: mtime, + changeFrequency: "weekly", + priority: 0.7, + }); + } + + return entries; +} + export default function sitemap(): Promise { if (!cachedRoutes) { - cachedRoutes = collectRoutes(APP_DIR).then((routes) => { - routes.sort((a, b) => a.url.localeCompare(b.url)); - return routes; - }); + cachedRoutes = (async () => { + const routes = await collectRoutes(APP_DIR); + const existingPaths = new Set( + routes.map((route) => route.url.slice(NORMALIZED_SITE_URL.length)) + ); + const toolkitRoutes = await collectToolkitRoutes(existingPaths); + + const allRoutes = [...routes, ...toolkitRoutes]; + allRoutes.sort((a, b) => a.url.localeCompare(b.url)); + return allRoutes; + })(); } return cachedRoutes; diff --git a/tests/sitemap.test.ts b/tests/sitemap.test.ts index 854fa6234..d8daf65cc 100644 --- a/tests/sitemap.test.ts +++ b/tests/sitemap.test.ts @@ -22,6 +22,14 @@ test("sitemap lists expected URLs", async () => { // Known page should be present expect(urls).toContain("https://example.test/en/references/changelog"); + // Generated toolkit pages (served by the `[toolkitId]` dynamic route, + // which the directory walk above can't see) must still make it into the + // sitemap. This fails if the toolkit-route merge in app/sitemap.ts is + // reverted. + expect(urls).toContain( + "https://example.test/en/resources/integrations/development/github" + ); + // No duplicates const duplicates = urls.filter( (url, index, arr) => arr.indexOf(url) !== index diff --git a/toolkit-docs-generator/ARCHITECTURE.md b/toolkit-docs-generator/ARCHITECTURE.md index 67022ac8c..86654dd08 100644 --- a/toolkit-docs-generator/ARCHITECTURE.md +++ b/toolkit-docs-generator/ARCHITECTURE.md @@ -71,9 +71,10 @@ root `pnpm build` command and compiles the committed files with Next.js. `app/_lib/toolkit-static-params.ts` enumerates routes from `index.json` and the per-toolkit files. `app/_lib/toolkit-data.ts` reads the same files for page -rendering and the `/api/toolkit-data/[toolkitId]` route. The root layout reads -request headers, so Vercel reports the docs routes as dynamic even though the -toolkit parameter set is fixed at build time. +rendering and the `/api/toolkit-data/[toolkitId]` route. The root layout no +longer reads request headers — the locale it needs is a hardcoded constant, +since `proxy.ts` redirects every request to an `/en` path — so Vercel can +statically render the toolkit routes at build time from the committed JSON. ## Search indexing From 35ae4f22c4353eea3d35e9e41a225afc74fdac81 Mon Sep 17 00:00:00 2001 From: Teal Larson Date: Fri, 31 Jul 2026 13:23:35 -0400 Subject: [PATCH 09/17] refactor: make the toolkit data contract single and enforced MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Eliminates the duplication that lets these subsystems drift. Redirects were a 909-line array inside next.config.ts, so three consumers regex-parsed the config as text — one of them carrying a second "reversed" regex that existed only because a human might write {destination, source} instead of {source, destination}, a problem that only exists when you parse text instead of importing data. The array now lives in a typed redirects.ts and every consumer imports it. Two further consumers that also parsed the config as text (scripts/update-internal-links.ts and tests/integration-index-links.test.ts) would have silently found zero redirects, so they move to the import too. --auto-fix now appends to the data file, and its six scattered "Auto-added redirects" marker blocks collapse to one append point. The resolved array is byte-identical: 156 entries, same order. check-redirects-utils.ts had 425 lines of tests but was imported by nothing except its own test file, while the code that actually runs — the pre-commit hook's scripts/check-redirects.ts — kept private copies of the same six helpers. The tests guarded a copy while the shipping implementation was untested. The module moves to scripts/lib/ and the shipping script now imports it. Toolkit primitives (data dir, toKebabCase, normalizeToolkitId, the category list, the *Api heuristic, docsLink→slug) existed in 2-7 copies, one pair carrying a "must stay in sync" comment. They collapse into toolkit-docs-generator/src/shared/, which both halves can import. All seven data-dir consumers now honor TOOLKIT_DATA_DIR; previously only two did. The Node-only path resolution lives in its own module because client components reach the primitives through the integrations index, and a node:* import anywhere in that graph fails the webpack browser build. The consumer-side contract was a four-field duck-check that never verified tools was an array, followed by an unchecked cast — while toToolkitSummary immediately calls data.tools.map(). The generator's Zod schemas are now the single shared contract, the 522-line hand-written mirror is z.infer, and zod moves to dependencies because it enters the app's runtime path. Corruption is now loud and absence stays quiet: three catch blocks treated missing, unparseable, and schema-invalid identically, so a malformed file from the nightly PR silently dropped a toolkit and 404'd. Unparseable or invalid now throws with the file path and the Zod issues, failing the build. An unrecognized category throws instead of being coerced to "others", which had no route directory and would have made every toolkit in a new category a clickable card pointing at a 404. One cache()-wrapped loader replaces 11 full passes over the data directory per build, and removes the scan-every-file miss path that a burst of unknown IDs could otherwise trigger at request time. Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/check-redirects.yml | 8 +- .husky/pre-commit | 18 +- app/_components/toolkit-docs/types/index.ts | 349 ++----- app/_lib/integration-catalog.ts | 3 +- app/_lib/integration-index.ts | 3 +- app/_lib/toolkit-data.ts | 248 +++-- app/_lib/toolkit-slug.ts | 53 - app/_lib/toolkit-static-params.ts | 137 +-- .../integrations/_lib/toolkit-docs-page.tsx | 24 +- .../integrations/components/filter-params.ts | 43 +- next.config.ts | 923 +----------------- package.json | 6 +- pnpm-lock.yaml | 6 +- redirects.ts | 875 +++++++++++++++++ scripts/check-redirects.ts | 399 ++------ scripts/generate-llmstxt.ts | 38 +- .../lib}/check-redirects-utils.ts | 88 +- scripts/update-internal-links.ts | 65 +- tests/integration-category-routes.test.ts | 58 ++ tests/integration-index-links.test.ts | 20 +- .../scripts/check-redirects-utils.test.ts | 76 +- tests/sitemap.test.ts | 10 +- tests/toolkit-data-cache.test.ts | 118 +++ tests/toolkit-data-parity.test.ts | 58 ++ .../scripts/check-stale-summaries.ts | 7 +- .../scripts/report-tool-metadata.ts | 10 +- .../scripts/sync-toolkit-sidebar.ts | 72 +- .../scripts/validate-merge.ts | 27 +- .../src/merger/data-merger.ts | 15 +- .../src/shared/toolkit-data-dir.ts | 35 + .../src/shared/toolkit-primitives.ts | 126 +++ .../src/shared/toolkit-schemas.ts | 420 ++++++++ .../src/sources/design-system-metadata.ts | 15 +- .../src/sources/toolkit-data-source.ts | 4 +- toolkit-docs-generator/src/types/index.ts | 410 +------- .../tests/app-lib/toolkit-data.test.ts | 10 + .../tests/app-lib/toolkit-slug.test.ts | 2 +- .../app-lib/toolkit-static-params.test.ts | 41 +- .../scripts/sync-toolkit-sidebar.test.ts | 14 + 39 files changed, 2344 insertions(+), 2490 deletions(-) create mode 100644 redirects.ts rename {toolkit-docs-generator/scripts => scripts/lib}/check-redirects-utils.ts (76%) create mode 100644 tests/integration-category-routes.test.ts rename {toolkit-docs-generator/tests => tests}/scripts/check-redirects-utils.test.ts (86%) create mode 100644 tests/toolkit-data-cache.test.ts create mode 100644 tests/toolkit-data-parity.test.ts create mode 100644 toolkit-docs-generator/src/shared/toolkit-data-dir.ts create mode 100644 toolkit-docs-generator/src/shared/toolkit-primitives.ts create mode 100644 toolkit-docs-generator/src/shared/toolkit-schemas.ts diff --git a/.github/workflows/check-redirects.yml b/.github/workflows/check-redirects.yml index 1482842be..ccaa1bef8 100644 --- a/.github/workflows/check-redirects.yml +++ b/.github/workflows/check-redirects.yml @@ -6,7 +6,7 @@ on: paths: - "app/**/*.md" - "app/**/*.mdx" - - "next.config.ts" + - "redirects.ts" permissions: contents: read @@ -52,7 +52,7 @@ jobs: // Extract the missing redirects and suggestions from output const body = `## 🔗 Missing Redirects Detected - This PR deletes markdown files that don't have corresponding redirects in \`next.config.ts\`. + This PR deletes markdown files that don't have corresponding redirects in \`redirects.ts\`. When you delete a page, you must add a redirect to prevent broken links for users who have bookmarked the old URL. @@ -67,8 +67,8 @@ jobs: ### How to fix - 1. Open \`next.config.ts\` - 2. Find the \`redirects()\` function + 1. Open \`redirects.ts\` + 2. Find the \`redirects\` array 3. Add redirect entries for each deleted file (see suggestions above) 4. Push the changes diff --git a/.husky/pre-commit b/.husky/pre-commit index 4ffa0437b..452307eb9 100755 --- a/.husky/pre-commit +++ b/.husky/pre-commit @@ -61,18 +61,18 @@ if [ -n "$DELETED_PAGES" ] || [ -n "$RENAMED_PAGES" ]; then echo "🔗 Detected deleted/renamed page(s), checking for redirects..." # Run the TypeScript redirect checker with auto-fix (only checks staged changes) - # This will add redirect entries to next.config.ts if missing + # This will add redirect entries to redirects.ts if missing if ! pnpm check-redirects --auto-fix --staged-only 2>&1; then - # Stage next.config.ts if it was modified - if git diff --name-only next.config.ts 2>/dev/null | grep -q "next.config.ts"; then - git add next.config.ts + # Stage redirects.ts if it was modified + if git diff --name-only redirects.ts 2>/dev/null | grep -q "redirects.ts"; then + git add redirects.ts echo "" - echo "📝 Redirect entries added to next.config.ts and staged." + echo "📝 Redirect entries added to redirects.ts and staged." fi echo "" # Check if there are placeholders vs other errors - if grep -q "REPLACE_WITH_NEW_PATH" next.config.ts 2>/dev/null; then - echo "❌ Commit blocked: Please update the placeholder destinations in next.config.ts" + if grep -q "REPLACE_WITH_NEW_PATH" redirects.ts 2>/dev/null; then + echo "❌ Commit blocked: Please update the placeholder destinations in redirects.ts" echo " Search for 'REPLACE_WITH_NEW_PATH' and provide actual redirect paths." else echo "❌ Commit blocked: Please fix the redirect issues shown above." @@ -82,8 +82,8 @@ if [ -n "$DELETED_PAGES" ] || [ -n "$RENAMED_PAGES" ]; then fi # --- Update Internal Links (when redirects are added) --- -# If next.config.ts is staged, update any internal links pointing to redirected paths -if git diff --cached --name-only | grep -q "next.config.ts"; then +# If redirects.ts is staged, update any internal links pointing to redirected paths +if git diff --cached --name-only | grep -q "redirects.ts"; then echo "🔗 Updating internal links for new redirects..." # Capture files already modified in working directory BEFORE running update-links diff --git a/app/_components/toolkit-docs/types/index.ts b/app/_components/toolkit-docs/types/index.ts index f091219b6..ad56a5be2 100644 --- a/app/_components/toolkit-docs/types/index.ts +++ b/app/_components/toolkit-docs/types/index.ts @@ -1,214 +1,97 @@ /** * Type definitions for toolkit documentation MDX components * - * These types are designed for React component props and are compatible - * with the JSON data structure from toolkit-docs-generator. + * The data-shape types below (everything through `ToolkitData`) are + * `z.infer` types derived from the Zod schemas in + * toolkit-docs-generator/src/shared/toolkit-schemas.ts — the same schemas + * the generator validates its JSON output against. Only `import type` is + * used here: this file is imported by client components, and a runtime + * import of the Zod schemas would ship the Zod library to the browser for + * no benefit (the client only needs the types, never runs `.parse()`). + * + * Everything below `ToolkitData` (ToolSummary, ToolkitSummary, and the + * component prop types) is app-specific shaping of that data for React + * props and has no generator equivalent. */ +import type { z } from "zod"; +import type { + DocumentationChunkSchema, + ExampleParameterValueSchema, + MergedToolkitAuthSchema, + MergedToolkitMetadataSchema, + MergedToolkitSchema, + MergedToolSchema, + SecretTypeSchema, + ToolAuthSchema, + ToolCodeExampleSchema, + ToolkitAuthTypeSchema, + ToolkitCategorySchema, + ToolkitTypeSchema, + ToolMetadataBehaviorSchema, + ToolMetadataClassificationSchema, + ToolMetadataSchema, + ToolOutputSchema, + ToolParameterSchema, + ToolSecretSchema, +} from "@/toolkit-docs-generator/src/shared/toolkit-schemas"; // ============================================================================ // Documentation Chunk Types // ============================================================================ -/** - * Type of documentation chunk content - */ -export type DocumentationChunkType = - | "callout" - | "markdown" - | "code" - | "warning" - | "info" - | "tip" - | "section"; - -/** - * Location where the chunk should be injected - */ -export type DocumentationChunkLocation = - | "header" - | "description" - | "parameters" - | "auth" - | "secrets" - | "output" - | "footer" - | "before_available_tools" - | "after_available_tools" - | "custom_section"; - -/** - * Position relative to the location - */ -export type DocumentationChunkPosition = "before" | "after" | "replace"; - -/** - * Callout variant for styling - */ -export type DocumentationChunkVariant = - | "default" - | "destructive" - | "warning" - | "info" - | "success"; - -/** - * A documentation chunk represents custom content to inject into docs - */ -export type DocumentationChunk = { - /** Type of content */ - type: DocumentationChunkType; - /** Where to inject the content */ - location: DocumentationChunkLocation; - /** Position relative to location */ - position: DocumentationChunkPosition; - /** The actual content (markdown string) */ - content: string; - /** Optional title for callouts */ - title?: string; - /** Optional variant for styling */ - variant?: DocumentationChunkVariant; - /** Optional section header for sidebar navigation (e.g., "## Auth Setup") */ - header?: string; - /** Optional priority for ordering (lower = earlier, default = 100) */ - priority?: number; -}; +export type DocumentationChunk = z.infer; +export type DocumentationChunkType = DocumentationChunk["type"]; +export type DocumentationChunkLocation = DocumentationChunk["location"]; +export type DocumentationChunkPosition = DocumentationChunk["position"]; +export type DocumentationChunkVariant = NonNullable< + DocumentationChunk["variant"] +>; // ============================================================================ // Tool Parameter Types // ============================================================================ -/** - * Tool parameter definition - */ -export type ToolParameter = { - /** Parameter name */ - name: string; - /** Parameter type (string, integer, boolean, array, object) */ - type: string; - /** For array types, the inner element type */ - innerType?: string; - /** Whether the parameter is required */ - required: boolean; - /** Parameter description */ - description: string | null; - /** Enum values if this is an enum parameter */ - enum: string[] | null; - /** Whether the parameter can be inferred by an LLM */ - inferrable?: boolean; - /** Default value if not provided */ - default?: unknown; -}; +export type ToolParameter = z.infer; // ============================================================================ // Tool Auth Types // ============================================================================ -/** - * Tool-level authentication requirements - */ -export type ToolAuth = { - /** Auth provider ID (e.g., "github", "google") */ - providerId: string | null; - /** Provider type (e.g., "oauth2", "api_key") */ - providerType: string; - /** Required OAuth scopes for this specific tool */ - scopes: string[]; -}; +export type ToolAuth = z.infer; // ============================================================================ // Tool Output Types // ============================================================================ -/** - * Tool output schema - */ -export type ToolOutput = { - /** Output type (object, array, string, etc.) */ - type: string; - /** Output description */ - description: string | null; -}; +export type ToolOutput = z.infer; // ============================================================================ // Tool Secrets Types // ============================================================================ -export type SecretType = - | "api_key" - | "token" - | "client_secret" - | "webhook_secret" - | "private_key" - | "password" - | "unknown"; - -export type ToolSecret = { - /** Secret name */ - name: string; - /** Secret type classification */ - type: SecretType; -}; +export type SecretType = z.infer; +export type ToolSecret = z.infer; // ============================================================================ // Code Example Types // ============================================================================ -/** - * Parameter value with type information for code generation - */ -export type ExampleParameterValue = { - /** The example value to use in generated code */ - value: unknown; - /** Parameter type for proper serialization */ - type: "string" | "integer" | "boolean" | "array" | "object"; - /** Whether this parameter is required */ - required: boolean; -}; - -/** - * Tool code example configuration - * Used to generate Python/JavaScript example code - */ -export type ToolCodeExample = { - /** Full tool name (e.g., "Github.SetStarred") */ - toolName: string; - /** Parameter values with type info */ - parameters: Record; - /** Whether this tool requires user authorization */ - requiresAuth: boolean; - /** Auth provider ID if auth is required */ - authProvider?: string; - /** Optional tab label for the code example */ - tabLabel?: string; -}; +export type ExampleParameterValue = z.infer; +export type ToolCodeExample = z.infer; // ============================================================================ // Tool Metadata Types // ============================================================================ -export type ToolMetadataClassification = { - serviceDomains: string[]; -}; - -export type ToolMetadataBehavior = { - operations: string[]; - readOnly?: boolean; - destructive?: boolean; - idempotent?: boolean; - openWorld?: boolean; -}; +export type ToolMetadataClassification = z.infer< + typeof ToolMetadataClassificationSchema +>; +export type ToolMetadataBehavior = z.infer; -export type BehaviorFlagKey = - | "readOnly" - | "destructive" - | "idempotent" - | "openWorld"; +/** UI-only helper: the boolean behavior flags, excluding `operations`. */ +export type BehaviorFlagKey = Exclude; -export type ToolMetadata = { - classification: ToolMetadataClassification; - behavior: ToolMetadataBehavior; - extras?: Record | null; -}; +export type ToolMetadata = z.infer; // ============================================================================ // Tool Definition Types @@ -217,32 +100,7 @@ export type ToolMetadata = { /** * Complete tool definition with all documentation data */ -export type ToolDefinition = { - /** Tool name (e.g., "CreateIssue") */ - name: string; - /** Qualified name (e.g., "Github.CreateIssue") */ - qualifiedName: string; - /** Fully qualified name with version (e.g., "Github.CreateIssue@1.0.0") */ - fullyQualifiedName: string; - /** Tool description */ - description: string | null; - /** Tool parameters */ - parameters: ToolParameter[]; - /** Tool authentication requirements */ - auth: ToolAuth | null; - /** Required secrets */ - secrets: string[]; - /** Classified secrets (LLM-generated) */ - secretsInfo?: ToolSecret[]; - /** Tool output schema */ - output: ToolOutput | null; - /** Per-tool metadata from Engine API */ - metadata?: ToolMetadata | null; - /** Custom documentation chunks for this tool */ - documentationChunks: DocumentationChunk[]; - /** Generated code example configuration */ - codeExample?: ToolCodeExample; -}; +export type ToolDefinition = z.infer; /** * A tool with its heavy detail fields stripped — everything needed to render the @@ -258,72 +116,16 @@ export type ToolSummary = Omit< // Toolkit Metadata Types // ============================================================================ -/** - * Toolkit category for navigation grouping - */ -export type ToolkitCategory = - | "productivity" - | "social" - | "development" - | "entertainment" - | "search" - | "payments" - | "sales" - | "databases" - | "customer-support"; - -/** - * Toolkit type classification - */ -export type ToolkitType = - | "arcade" - | "arcade_starter" - | "verified" - | "community" - | "auth"; - -/** - * Toolkit metadata from Design System - */ -export type ToolkitMetadata = { - /** Category for navigation grouping */ - category: ToolkitCategory; - /** Icon URL */ - iconUrl: string; - /** Whether this toolkit requires BYOC (Bring Your Own Credentials) */ - isBYOC: boolean; - /** Whether this is a Pro feature */ - isPro: boolean; - /** Toolkit type classification */ - type: ToolkitType; - /** Link to documentation */ - docsLink: string; - /** Whether this toolkit is coming soon */ - isComingSoon?: boolean; - /** Whether this toolkit is hidden */ - isHidden?: boolean; -}; +export type ToolkitCategory = z.infer; +export type ToolkitType = z.infer; +export type ToolkitMetadata = z.infer; // ============================================================================ // Toolkit Auth Types // ============================================================================ -/** - * Toolkit-level authentication type - */ -export type ToolkitAuthType = "oauth2" | "api_key" | "mixed" | "none"; - -/** - * Toolkit-level authentication summary - */ -export type ToolkitAuth = { - /** Auth type */ - type: ToolkitAuthType; - /** Auth provider ID */ - providerId: string | null; - /** Union of all scopes required by tools in this toolkit */ - allScopes: string[]; -}; +export type ToolkitAuthType = z.infer; +export type ToolkitAuth = z.infer; // ============================================================================ // Complete Toolkit Data Type @@ -333,38 +135,7 @@ export type ToolkitAuth = { * Complete toolkit data structure for rendering documentation * This is the main type consumed by the ToolkitPage component */ -export type ToolkitData = { - /** Unique toolkit ID (e.g., "Github") */ - id: string; - /** Human-readable label (e.g., "GitHub") */ - label: string; - /** Toolkit version (e.g., "1.0.0") */ - version: string; - /** Toolkit description */ - description: string | null; - /** LLM-generated summary */ - summary?: string; - /** Metadata from Design System */ - metadata: ToolkitMetadata; - /** Authentication requirements */ - auth: ToolkitAuth | null; - /** All tools in this toolkit */ - tools: ToolDefinition[]; - /** Toolkit-level documentation chunks */ - documentationChunks?: DocumentationChunk[]; - /** Custom imports for MDX */ - customImports: string[]; - /** - * Sub-pages that exist for this toolkit. - * Each entry is either a string (legacy slug) or a rich object with - * { type, content, relativePath } for inline MDX sub-page content. - */ - subPages: (string | Record)[]; - /** Optional pip package name override */ - pipPackageName?: string; - /** Generation timestamp */ - generatedAt?: string; -}; +export type ToolkitData = z.infer; /** * Toolkit data with each tool's heavy detail fields stripped. This is what the diff --git a/app/_lib/integration-catalog.ts b/app/_lib/integration-catalog.ts index f92cc7d31..110f466f4 100644 --- a/app/_lib/integration-catalog.ts +++ b/app/_lib/integration-catalog.ts @@ -1,8 +1,9 @@ import type { Toolkit } from "@arcadeai/design-system"; import { TOOLKITS } from "@arcadeai/design-system/metadata/toolkits"; import { PARTNER_TOOLKITS } from "@/app/_data/partner-toolkits"; +import { normalizeToolkitId } from "@/toolkit-docs-generator/src/shared/toolkit-primitives"; import { readToolkitData } from "./toolkit-data"; -import { normalizeToolkitId, type ToolkitWithDocsLink } from "./toolkit-slug"; +import type { ToolkitWithDocsLink } from "./toolkit-slug"; const getToolkitDocsLink = (toolkit: Toolkit): string | undefined => { if ("docsLink" in toolkit) { diff --git a/app/_lib/integration-index.ts b/app/_lib/integration-index.ts index 12d9cacbf..0e062a54a 100644 --- a/app/_lib/integration-index.ts +++ b/app/_lib/integration-index.ts @@ -1,4 +1,5 @@ -import { getToolkitSlug, type ToolkitWithDocsLink } from "./toolkit-slug"; +import { getToolkitSlug } from "@/toolkit-docs-generator/src/shared/toolkit-primitives"; +import type { ToolkitWithDocsLink } from "./toolkit-slug"; const INTEGRATIONS_BASE = "/en/resources/integrations"; diff --git a/app/_lib/toolkit-data.ts b/app/_lib/toolkit-data.ts index f0299c74e..f724b03b2 100644 --- a/app/_lib/toolkit-data.ts +++ b/app/_lib/toolkit-data.ts @@ -1,11 +1,22 @@ import { readdir, readFile } from "node:fs/promises"; import { join } from "node:path"; +import { cache } from "react"; +import type { z } from "zod"; import type { ToolkitData, ToolkitSummary, ToolSummary, } from "@/app/_components/toolkit-docs/types"; -import { getToolkitSlug, normalizeToolkitId } from "./toolkit-slug"; +import { resolveToolkitDataDir } from "@/toolkit-docs-generator/src/shared/toolkit-data-dir"; +import { + getToolkitSlug, + normalizeToolkitId, +} from "@/toolkit-docs-generator/src/shared/toolkit-primitives"; +import { + MergedToolkitSchema, + type ToolkitIndexEntrySchema, + type ToolkitIndexSchema, +} from "@/toolkit-docs-generator/src/shared/toolkit-schemas"; /** * Strip each tool's heavy fields (parameters, output, codeExample) so the @@ -35,87 +46,156 @@ export function toToolkitSummary(data: ToolkitData): ToolkitSummary { }; } -export type ToolkitIndexEntry = { - id: string; - label: string; - version: string; - category: string; - type?: string; - toolCount: number; - authType: string; -}; - -export type ToolkitIndex = { - generatedAt: string; - version: string; - toolkits: ToolkitIndexEntry[]; -}; +export type ToolkitIndexEntry = z.infer; +export type ToolkitIndex = z.infer; type ToolkitDataOptions = { dataDir?: string; }; -const DEFAULT_DATA_DIR = join( - process.cwd(), - "toolkit-docs-generator", - "data", - "toolkits" -); - const resolveDataDir = (options?: ToolkitDataOptions): string => - options?.dataDir ?? process.env.TOOLKIT_DATA_DIR ?? DEFAULT_DATA_DIR; - -const isValidToolkitData = (parsed: unknown): parsed is ToolkitData => - typeof parsed === "object" && - parsed !== null && - "id" in parsed && - ("label" in parsed || "name" in parsed) && - "metadata" in parsed && - typeof (parsed as Record).metadata === "object" && - (parsed as Record).metadata !== null; - -const readToolkitFile = async ( + resolveToolkitDataDir(options?.dataDir); + +const isEnoent = (error: unknown): boolean => + error instanceof Error && (error as NodeJS.ErrnoException).code === "ENOENT"; + +const describeError = (error: unknown): string => + error instanceof Error ? error.message : String(error); + +/** + * Read and validate a single merged-toolkit JSON file. + * + * A missing file is a legitimate, quiet outcome (`null`) — not every toolkit + * has generated docs yet. Anything else wrong with the file — unreadable, + * not valid JSON, or valid JSON that doesn't match `MergedToolkitSchema` — is + * corruption, not absence, and throws with the file path and the underlying + * error so a bad nightly-generated file fails `next build` loudly instead of + * quietly dropping the toolkit from the site. Mirrors the read/parse/schema + * split in toolkit-docs-generator/src/generator/output-verifier.ts. + */ +export const readToolkitFile = async ( filePath: string ): Promise => { + let content: string; try { - const content = await readFile(filePath, "utf-8"); - const parsed: unknown = JSON.parse(content); - return isValidToolkitData(parsed) ? (parsed as ToolkitData) : null; - } catch { - return null; + content = await readFile(filePath, "utf-8"); + } catch (error) { + if (isEnoent(error)) { + return null; + } + throw new Error( + `Failed to read toolkit file ${filePath}: ${describeError(error)}` + ); + } + + let parsed: unknown; + try { + parsed = JSON.parse(content); + } catch (error) { + throw new Error( + `Invalid JSON in toolkit file ${filePath}: ${describeError(error)}` + ); + } + + const result = MergedToolkitSchema.safeParse(parsed); + if (!result.success) { + throw new Error( + `Invalid toolkit schema in ${filePath}: ${result.error.message}` + ); } + + return result.data; }; -const findToolkitDataBySlug = async ( - dataDir: string, - slug: string -): Promise => { - const entries = await readdir(dataDir); - const slugKey = slug.toLowerCase(); +/** + * Every toolkit's data, indexed two ways: by the normalized id its filename + * is derived from (the common case — an id-shaped lookup), and by its docs + * slug (a hand-authored `docsLink` can diverge from the id, e.g. a route + * reached by "posthog-api" for a file whose id normalizes differently). + * `readToolkitData` below tries the id map first, then the slug map, mirroring + * the direct-file-then-scan order the old implementation used. + */ +type ToolkitDataMap = { + byNormalizedId: Map; + bySlug: Map; +}; + +/** + * One process-wide load per data directory. Keyed by directory (not a single + * flat variable) because tests point `TOOLKIT_DATA_DIR` at scratch fixtures + * and must not see another test's cached data. + * + * A failed load (a corrupt file — see readToolkitFile) is kept in this map + * rather than retried: the underlying files are static build output that + * only change on a new deploy, so a bad file stays bad for the rest of this + * process's life, and re-scanning 21.6 MB on every subsequent lookup hoping + * it healed itself would only add cost without ever succeeding. + */ +const loadsByDataDir = new Map>(); + +const loadAllToolkitDataUncached = async ( + dataDir: string +): Promise => { + let entries: string[]; + try { + entries = await readdir(dataDir); + } catch (error) { + throw new Error( + `Failed to read toolkit data directory ${dataDir}: ${describeError(error)}` + ); + } + + const byNormalizedId = new Map(); + const bySlug = new Map(); for (const entry of entries) { if (!entry.endsWith(".json") || entry === "index.json") { continue; } + // Throws on a corrupt file (see readToolkitFile) — a malformed file here + // is never legitimately "absent", so it fails the build/request loudly + // rather than being dropped from the map. const data = await readToolkitFile(join(dataDir, entry)); if (!data) { continue; } - const candidateSlug = getToolkitSlug({ + byNormalizedId.set(normalizeToolkitId(data.id), data); + const slug = getToolkitSlug({ id: data.id, docsLink: data.metadata?.docsLink, - }).toLowerCase(); - - if (candidateSlug === slugKey) { - return data; - } + }); + bySlug.set(slug.toLowerCase(), data); } - return null; + return { byNormalizedId, bySlug }; }; +/** + * Load every toolkit's data from `dataDir` into one shared map, read once per + * process rather than once per caller. + * + * Wrapped in React's `cache()` so, when a live cache scope exists (build-time + * static generation, a Route Handler, a Server Component render), concurrent + * callers within that same scope share one in-flight read rather than each + * independently reading the directory. `cache()` is a no-op outside a cache + * scope (Vitest, plain scripts) — see its implementation in + * react/cjs/react.react-server.development.js — so `loadsByDataDir` is the + * mechanism that actually guarantees one read per directory everywhere, with + * `cache()` as the layer that also dedupes concurrent build-time work. + */ +export const loadAllToolkitData = cache( + async (dataDir: string): Promise => { + let promise = loadsByDataDir.get(dataDir); + if (!promise) { + promise = loadAllToolkitDataUncached(dataDir); + loadsByDataDir.set(dataDir, promise); + } + return await promise; + } +); + export const readToolkitData = async ( toolkitId: string, options?: ToolkitDataOptions @@ -128,15 +208,14 @@ export const readToolkitData = async ( return null; } - const fileName = `${normalizedId}.json`; const dataDir = resolveDataDir(options); - const filePath = join(dataDir, fileName); - const direct = await readToolkitFile(filePath); - if (direct) { - return direct; - } + const { byNormalizedId, bySlug } = await loadAllToolkitData(dataDir); - return await findToolkitDataBySlug(dataDir, toolkitId); + return ( + byNormalizedId.get(normalizedId) ?? + bySlug.get(toolkitId.toLowerCase()) ?? + null + ); }; export const readToolkitIndex = async ( @@ -144,22 +223,45 @@ export const readToolkitIndex = async ( ): Promise => { const filePath = join(resolveDataDir(options), "index.json"); + let content: string; try { - const content = await readFile(filePath, "utf-8"); - const parsed: unknown = JSON.parse(content); - - // Basic runtime validation - ensure it's an object with required fields - if ( - typeof parsed !== "object" || - parsed === null || - !("toolkits" in parsed) || - !Array.isArray((parsed as { toolkits: unknown }).toolkits) - ) { + content = await readFile(filePath, "utf-8"); + } catch (error) { + if (isEnoent(error)) { return null; } + throw new Error( + `Failed to read toolkit index ${filePath}: ${describeError(error)}` + ); + } - return parsed as ToolkitIndex; - } catch { - return null; + let parsed: unknown; + try { + parsed = JSON.parse(content); + } catch (error) { + throw new Error( + `Invalid JSON in toolkit index ${filePath}: ${describeError(error)}` + ); } + + // Deliberately looser than a full ToolkitIndexSchema.safeParse: unlike + // per-toolkit data, entries here are only ever used to look up a + // toolkit's id/category, with the toolkit's own JSON file as the real + // source of truth (see resolveToolkitRoute in toolkit-static-params.ts). + // Rejecting the whole index over one entry missing a field the callers + // don't read would cost every route on the site, not just one page. But + // the file as a whole not even having the shape of an index is + // corruption, not a missing-field nuance, so that still throws. + if ( + typeof parsed !== "object" || + parsed === null || + !("toolkits" in parsed) || + !Array.isArray((parsed as { toolkits: unknown }).toolkits) + ) { + throw new Error( + `Invalid toolkit index shape in ${filePath}: expected an object with a "toolkits" array.` + ); + } + + return parsed as ToolkitIndex; }; diff --git a/app/_lib/toolkit-slug.ts b/app/_lib/toolkit-slug.ts index 5ff34e21d..c99566dc3 100644 --- a/app/_lib/toolkit-slug.ts +++ b/app/_lib/toolkit-slug.ts @@ -1,13 +1,5 @@ import type { Toolkit } from "@arcadeai/design-system"; -const TOOLKIT_ID_NORMALIZER = /[^a-z0-9]+/g; -const CAMEL_BOUNDARY = /([a-z0-9])([A-Z])/g; - -export type ToolkitSlugSource = { - id: string; - docsLink?: string | null; -}; - /** * Toolkit with optional `docsLink` and `isPartner` properties. * The design-system `Toolkit` type doesn't include either field, but some @@ -19,48 +11,3 @@ export type ToolkitWithDocsLink = Toolkit & { docsLink?: string | null; isPartner?: boolean; }; - -/** - * Strip all non-alphanumeric characters and lowercase. - * Used for case-insensitive matching of toolkit IDs to filenames. - */ -export function normalizeToolkitId(value: string): string { - return value.toLowerCase().replace(TOOLKIT_ID_NORMALIZER, ""); -} - -/** - * Convert a CamelCase toolkit ID to a kebab-case URL slug. - * - * Examples: - * PosthogApi → posthog-api - * GoogleCalendar → google-calendar - * E2b → e2b - * HubspotCrmApi → hubspot-crm-api - */ -export function toKebabCase(value: string): string { - return value.replace(CAMEL_BOUNDARY, "$1-$2").toLowerCase(); -} - -const extractSlugFromPath = (path: string): string | null => { - const segments = path.split("/").filter(Boolean); - return segments.at(-1) ?? null; -}; - -export function getToolkitSlug({ id, docsLink }: ToolkitSlugSource): string { - if (docsLink) { - try { - const url = new URL(docsLink); - const slug = extractSlugFromPath(url.pathname); - if (slug) { - return slug; - } - } catch { - const slug = extractSlugFromPath(docsLink); - if (slug) { - return slug; - } - } - } - - return toKebabCase(id); -} diff --git a/app/_lib/toolkit-static-params.ts b/app/_lib/toolkit-static-params.ts index 1e07c5c33..d93bdd8f1 100644 --- a/app/_lib/toolkit-static-params.ts +++ b/app/_lib/toolkit-static-params.ts @@ -1,23 +1,18 @@ -import { readdir, readFile } from "node:fs/promises"; +import { readdir } from "node:fs/promises"; import { join } from "node:path"; import { TOOLKITS as DESIGN_SYSTEM_TOOLKITS } from "@arcadeai/design-system/metadata/toolkits"; -import { readToolkitData, readToolkitIndex } from "./toolkit-data"; -import { getToolkitSlug, normalizeToolkitId } from "./toolkit-slug"; - -export const INTEGRATION_CATEGORIES = [ - "productivity", - "social", - "entertainment", - "development", - "payments", - "search", - "sales", - "databases", - "customer-support", - "others", -] as const; - -export type IntegrationCategory = (typeof INTEGRATION_CATEGORIES)[number]; +import { resolveToolkitDataDir } from "@/toolkit-docs-generator/src/shared/toolkit-data-dir"; +import { + getToolkitSlug, + INTEGRATION_CATEGORIES, + type IntegrationCategory, + normalizeToolkitId, +} from "@/toolkit-docs-generator/src/shared/toolkit-primitives"; +import { + loadAllToolkitData, + readToolkitData, + readToolkitIndex, +} from "./toolkit-data"; export type ToolkitCatalogEntry = { id: string; @@ -42,16 +37,40 @@ const DESIGN_SYSTEM_TOOLKITS_FOR_ROUTES: ToolkitCatalogEntry[] = const loadDesignSystemToolkits = async (): Promise => DESIGN_SYSTEM_TOOLKITS_FOR_ROUTES; +/** + * Normalize a category value read from toolkit data into a routable + * category, or `null` when there is nothing to route by. + * + * `undefined`/`null`/empty string means no category information was + * available at all — typically a fallback source (the design-system catalog + * used only when a toolkit's own JSON file is absent) that simply doesn't + * carry one. That's a quiet "nothing to go on," not corruption: callers skip + * the toolkit rather than invent a page for it. + * + * A non-empty string that isn't one of `INTEGRATION_CATEGORIES`, though, is + * a real value someone set — most likely a new category introduced upstream + * (the Engine / design-system catalog) that this docs site doesn't have a + * route for yet. There is no "others" catch-all to silently absorb it (see + * INTEGRATION_CATEGORIES's doc comment): every toolkit in an unrecognized + * category would otherwise render as a clickable catalog card pointing at a + * route that 404s, with nothing failing the build to surface it. So this + * throws instead. + */ export function normalizeCategory( value: string | null | undefined -): IntegrationCategory { +): IntegrationCategory | null { if (!value) { - return "others"; + return null; } - return INTEGRATION_CATEGORIES.includes(value as IntegrationCategory) - ? (value as IntegrationCategory) - : "others"; + if (INTEGRATION_CATEGORIES.includes(value as IntegrationCategory)) { + return value as IntegrationCategory; + } + + throw new Error( + `Unrecognized integration category "${value}". Expected one of: ${INTEGRATION_CATEGORIES.join(", ")}. ` + + "A new category needs a matching app/en/resources/integrations//[toolkitId] route directory before toolkits can use it." + ); } /** @@ -62,6 +81,11 @@ export function normalizeCategory( * alias (e.g. `development/pagerduty-api` when its category is `customer-support`) * must canonicalize to the one generated, index-linked page instead of * orphaning itself. Mirrors the slug + category logic in `listToolkitRoutes`. + * + * Only called for a toolkit that already has a generated page (it's building + * that page's own canonical tag), so a `null` category here means the page + * exists but its routing information doesn't — an internal inconsistency, + * not a toolkit to quietly skip. That throws too. */ export function getToolkitCanonicalPath(toolkit: { id: string; @@ -69,60 +93,45 @@ export function getToolkitCanonicalPath(toolkit: { docsLink?: string | null; }): string { const category = normalizeCategory(toolkit.category); + if (!category) { + throw new Error( + `Cannot build a canonical path for toolkit "${toolkit.id}": it has no integration category.` + ); + } const slug = getToolkitSlug({ id: toolkit.id, docsLink: toolkit.docsLink }); return `/en/resources/integrations/${category}/${slug}`; } -const DEFAULT_DATA_DIR = join( - process.cwd(), - "toolkit-docs-generator", - "data", - "toolkits" -); - const resolveDataDir = (dataDir?: string): string => - dataDir ?? process.env.TOOLKIT_DATA_DIR ?? DEFAULT_DATA_DIR; + resolveToolkitDataDir(dataDir); const listToolkitRoutesFromDataDir = async (options?: { dataDir?: string; }): Promise => { const dataDir = resolveDataDir(options?.dataDir); - const entries = await readdir(dataDir); + + // loadAllToolkitData validates every file against MergedToolkitSchema and + // throws on a corrupt one (see app/_lib/toolkit-data.ts) — a malformed file + // in this directory listing is never legitimately "absent", so it should + // fail the build rather than be skipped here. + const { byNormalizedId } = await loadAllToolkitData(dataDir); + const unique = new Map(); - for (const entry of entries) { - if (!entry.endsWith(".json") || entry === "index.json") { + for (const data of byNormalizedId.values()) { + if (data.metadata?.isHidden) { continue; } - try { - const content = await readFile(join(dataDir, entry), "utf-8"); - const parsed = JSON.parse(content) as { - id?: string; - metadata?: { - category?: string; - docsLink?: string; - isHidden?: boolean; - }; - }; - - if (!parsed?.id) { - continue; - } - - if (parsed.metadata?.isHidden) { - continue; - } - - const slug = getToolkitSlug({ - id: parsed.id, - docsLink: parsed.metadata?.docsLink, - }); - const category = normalizeCategory(parsed.metadata?.category); - unique.set(slug, { toolkitId: slug, category }); - } catch { - // Ignore malformed toolkit data files. + const slug = getToolkitSlug({ + id: data.id, + docsLink: data.metadata?.docsLink, + }); + const category = normalizeCategory(data.metadata?.category); + if (!category) { + continue; } + unique.set(slug, { toolkitId: slug, category }); } return [...unique.values()]; @@ -158,6 +167,12 @@ const resolveToolkitRoute = async ( const category = normalizeCategory( data?.metadata?.category ?? catalogEntry?.category ?? toolkit.category ); + // No category info anywhere for this toolkit: nothing to route it under. + // Skip it quietly (same treatment as a hidden toolkit) rather than + // fabricate a page under a category that doesn't exist. + if (!category) { + return null; + } return { toolkitId: slug, category }; }; diff --git a/app/en/resources/integrations/_lib/toolkit-docs-page.tsx b/app/en/resources/integrations/_lib/toolkit-docs-page.tsx index c0522c52a..bf585d585 100644 --- a/app/en/resources/integrations/_lib/toolkit-docs-page.tsx +++ b/app/en/resources/integrations/_lib/toolkit-docs-page.tsx @@ -2,33 +2,23 @@ import type { Metadata } from "next"; import { notFound } from "next/navigation"; import { ToolkitPage } from "@/app/_components/toolkit-docs"; import { readToolkitData, toToolkitSummary } from "@/app/_lib/toolkit-data"; -import { normalizeToolkitId } from "@/app/_lib/toolkit-slug"; import { getToolkitCanonicalPath, getToolkitStaticParamsForCategory, - type IntegrationCategory, } from "@/app/_lib/toolkit-static-params"; +import type { IntegrationCategory } from "@/toolkit-docs-generator/src/shared/toolkit-primitives"; type ToolkitDocsParams = { toolkitId: string; }; export function createToolkitDocsPage(category: IntegrationCategory) { - const dataCache = new Map>(); - - const getToolkitData = async (toolkitId: string) => { - const cacheKey = normalizeToolkitId(toolkitId); - const cached = dataCache.get(cacheKey); - if (cached) { - return await cached; - } - - // Pass the original toolkitId (not normalized) so readToolkitData's - // findToolkitDataBySlug fallback can match hyphenated slugs like "posthog-api". - const promise = readToolkitData(toolkitId); - dataCache.set(cacheKey, promise); - return await promise; - }; + // readToolkitData is itself backed by a shared, process-wide cache (see + // loadAllToolkitData in app/_lib/toolkit-data.ts), so generateMetadata and + // Page below calling it separately for the same toolkitId costs one map + // lookup each rather than a second file read — no per-factory cache needed + // here. + const getToolkitData = (toolkitId: string) => readToolkitData(toolkitId); const generateStaticParams = async () => await getToolkitStaticParamsForCategory(category); diff --git a/app/en/resources/integrations/components/filter-params.ts b/app/en/resources/integrations/components/filter-params.ts index 432088986..71e4722c3 100644 --- a/app/en/resources/integrations/components/filter-params.ts +++ b/app/en/resources/integrations/components/filter-params.ts @@ -1,25 +1,26 @@ import type { ToolkitCategory, ToolkitType } from "@arcadeai/design-system"; - -const TOOLKIT_TYPES: readonly ToolkitType[] = [ - "arcade", - "arcade_starter", - "verified", - "community", - "auth", -]; - -const TOOLKIT_CATEGORIES: readonly ToolkitCategory[] = [ - "all", - "productivity", - "social", - "development", - "entertainment", - "search", - "payments", - "sales", - "databases", - "customer-support", -]; +import { CATEGORIES } from "@arcadeai/design-system/metadata/toolkits"; + +// The design system exports CATEGORIES (id + display name) as the runtime +// source of truth for ToolkitCategory. Derive the filter list from it +// instead of hand-copying the ids, so a new category can't silently drop +// out of this list the way it could with a separately maintained array. +const TOOLKIT_CATEGORIES: readonly ToolkitCategory[] = CATEGORIES.map( + (category) => category.id +); + +// ToolkitType has no runtime export from the design system, so this list is +// hand-maintained. `satisfies` turns a missing or extra entry into a +// compile error the next time the union changes, instead of a silent drop. +const TOOLKIT_TYPE_MEMBERSHIP = { + arcade: true, + arcade_starter: true, + verified: true, + community: true, + auth: true, +} satisfies Record; + +const TOOLKIT_TYPES = Object.keys(TOOLKIT_TYPE_MEMBERSHIP) as ToolkitType[]; export const PARAM_CATEGORY = "category"; export const PARAM_TYPE = "type"; diff --git a/next.config.ts b/next.config.ts index b7634c2e8..1c6c928ec 100644 --- a/next.config.ts +++ b/next.config.ts @@ -2,6 +2,7 @@ import type { NextConfig } from "next"; import nextra from "nextra"; import { withLlmsTxt } from "./lib/next-plugin-llmstxt"; import { remarkGlossary } from "./lib/remark-glossary"; +import { redirects } from "./redirects"; // Set up Nextra with its configuration const withNextra = nextra({ @@ -23,915 +24,19 @@ const nextConfig: NextConfig = withLlmsTxt({ })( withNextra({ async redirects() { - return [ - // The toolkit-page breadcrumb links "Resources" -> /resources, which has - // no index page. Send it to the integrations registry instead of 404ing. - { - source: "/:locale/resources", - destination: "/:locale/resources/integrations", - permanent: true, - }, - // The auth provider is "square"; an external/stale link points at the - // old "squareup" slug, which 404s. Send it to the real page. - { - source: "/:locale/references/auth-providers/squareup", - destination: "/:locale/references/auth-providers/square", - permanent: true, - }, - // Dissolved guides/security section - { - source: "/:locale/guides/security/security-research-program", - destination: "/:locale/resources/security-research-program", - permanent: true, - }, - { - source: "/:locale/guides/security/securing-arcade-mcp", - destination: "/:locale/guides/create-tools/secure-your-server", - permanent: true, - }, - { - source: "/:locale/guides/security/secure-your-mcp-server", - destination: - "/:locale/guides/create-tools/secure-your-server/secure-your-mcp-server", - permanent: true, - }, - { - source: "/:locale/guides/security", - destination: "/:locale/guides/create-tools/secure-your-server", - permanent: true, - }, - // Auto-added redirects for deleted pages - { - source: "/:locale/references/mcp/python/transports", - destination: "/:locale/references/mcp/python", - permanent: true, - }, - { - source: "/:locale/references/mcp/python/types", - destination: "/:locale/references/mcp/python", - permanent: true, - }, - // CrewAI custom auth flow redirect to use-arcade-tools - { - source: - "/:locale/get-started/agent-frameworks/crewai/custom-auth-flow", - destination: - "/:locale/get-started/agent-frameworks/crewai/use-arcade-tools", - permanent: true, - }, - // "others" category removed — toolkits moved to proper categories - { - source: "/:locale/resources/integrations/others/:path*", - destination: "/:locale/resources/integrations", - permanent: false, - }, - // Google ADK tutorial consolidation - redirect old URL to new - { - source: - "/:locale/get-started/agent-frameworks/google-adk/use-arcade-tools", - destination: - "/:locale/get-started/agent-frameworks/google-adk/overview", - permanent: true, - }, - // Auto-added redirects for deleted pages - { - source: "/:locale/references/logic-extensions-api", - destination: "/:locale/references/contextual-access-webhook-api", - permanent: true, - }, - // Auto-added redirects for deleted pages - { - source: "/:locale/guides/logic-extensions", - destination: "/:locale/guides/contextual-access", - permanent: true, - }, - { - source: "/:locale/guides/logic-extensions/build-your-own", - destination: "/:locale/guides/contextual-access/build-your-own", - permanent: true, - }, - { - source: "/:locale/guides/logic-extensions/examples", - destination: "/:locale/guides/contextual-access/examples", - permanent: true, - }, - { - source: "/:locale/guides/logic-extensions/how-hooks-work", - destination: "/:locale/guides/contextual-access/how-hooks-work", - permanent: true, - }, - // Auto-added redirects for deleted pages - { - source: "/:locale/resources/integrations/preview", - destination: "/:locale/resources/integrations", - permanent: true, - }, - // Auto-added redirects for deleted pages - { - source: - "/:locale/resources/integrations/customer-support/zendesk/reference", - destination: - "/:locale/resources/integrations/customer-support/zendesk", - permanent: true, - }, - { - source: - "/:locale/resources/integrations/development/firecrawl/reference", - destination: "/:locale/resources/integrations/development/firecrawl", - permanent: true, - }, - { - source: - "/:locale/resources/integrations/productivity/asana/reference", - destination: "/:locale/resources/integrations/productivity/asana", - permanent: true, - }, - { - source: - "/:locale/resources/integrations/productivity/clickup/reference", - destination: "/:locale/resources/integrations/productivity/clickup", - permanent: true, - }, - { - source: - "/:locale/resources/integrations/productivity/dropbox/reference", - destination: "/:locale/resources/integrations/productivity/dropbox", - permanent: true, - }, - { - source: - "/:locale/resources/integrations/productivity/gmail/reference", - destination: "/:locale/resources/integrations/productivity/gmail", - permanent: true, - }, - { - source: - "/:locale/resources/integrations/productivity/google-calendar/reference", - destination: - "/:locale/resources/integrations/productivity/google-calendar", - permanent: true, - }, - { - source: - "/:locale/resources/integrations/productivity/google-docs/reference", - destination: - "/:locale/resources/integrations/productivity/google-docs", - permanent: true, - }, - { - source: - "/:locale/resources/integrations/productivity/google-drive/reference", - destination: - "/:locale/resources/integrations/productivity/google-drive", - permanent: true, - }, - { - source: - "/:locale/resources/integrations/productivity/google-sheets/reference", - destination: - "/:locale/resources/integrations/productivity/google-sheets", - permanent: true, - }, - { - source: - "/:locale/resources/integrations/productivity/jira/environment-variables", - destination: "/:locale/resources/integrations/productivity/jira", - permanent: true, - }, - { - source: "/:locale/resources/integrations/productivity/jira/reference", - destination: "/:locale/resources/integrations/productivity/jira", - permanent: true, - }, - { - source: "/:locale/resources/integrations/sales/hubspot/reference", - destination: "/:locale/resources/integrations/sales/hubspot", - permanent: true, - }, - { - source: - "/:locale/resources/integrations/social-communication/discord", - destination: "/:locale/resources/integrations", - permanent: true, - }, - { - source: - "/:locale/resources/integrations/social-communication/linkedin", - destination: "/:locale/resources/integrations/social/linkedin", - permanent: true, - }, - { - source: - "/:locale/resources/integrations/social-communication/microsoft-teams", - destination: "/:locale/resources/integrations/social/microsoft-teams", - permanent: true, - }, - { - source: - "/:locale/resources/integrations/social-communication/microsoft-teams/reference", - destination: "/:locale/resources/integrations/social/microsoft-teams", - permanent: true, - }, - { - source: "/:locale/resources/integrations/social-communication/reddit", - destination: "/:locale/resources/integrations/social/reddit", - permanent: true, - }, - { - source: - "/:locale/resources/integrations/social-communication/slack-api", - destination: "/:locale/resources/integrations/social/slack-api", - permanent: true, - }, - { - source: - "/:locale/resources/integrations/social-communication/slack/environment-variables", - destination: "/:locale/resources/integrations/social/slack", - permanent: true, - }, - { - source: - "/:locale/resources/integrations/social-communication/slack/install", - destination: "/:locale/resources/integrations/social/slack", - permanent: true, - }, - { - source: "/:locale/resources/integrations/social-communication/slack", - destination: "/:locale/resources/integrations/social/slack", - permanent: true, - }, - { - source: - "/:locale/resources/integrations/social-communication/slack/reference", - destination: "/:locale/resources/integrations/social/slack", - permanent: true, - }, - { - source: - "/:locale/resources/integrations/social-communication/teams/reference", - destination: "/:locale/resources/integrations/social/microsoft-teams", - permanent: true, - }, - { - source: "/:locale/resources/integrations/social-communication/twilio", - destination: "/:locale/resources/integrations", - permanent: true, - }, - { - source: - "/:locale/resources/integrations/social-communication/twilio/reference", - destination: "/:locale/resources/integrations", - permanent: true, - }, - { - source: "/:locale/resources/integrations/social-communication/x", - destination: "/:locale/resources/integrations/social/x", - permanent: true, - }, - { - source: - "/:locale/resources/integrations/social-communication/zoom/install", - destination: "/:locale/resources/integrations/social/zoom", - permanent: true, - }, - { - source: "/:locale/resources/integrations/social-communication/zoom", - destination: "/:locale/resources/integrations/social/zoom", - permanent: true, - }, - // Auto-added redirects for deleted pages - { - source: - "/:locale/guides/create-tools/contribute/registry-early-access", - destination: "/:locale/resources/registry-early-access", - permanent: true, - }, - { - source: "/:locale/resources/integrations/contribute-a-server", - destination: "/:locale/resources/registry-early-access", - permanent: true, - }, - // Moved MCP Gateway UI guide to guides - { - source: "/:locale/guides/create-tools/mcp-gateways", - destination: "/:locale/guides/mcp-gateways", - permanent: true, - }, - // Removed LangChain old stuff - { - source: - "/:locale/get-started/agent-frameworks/langchain/use-arcade-tools", - destination: - "/:locale/get-started/agent-frameworks/langchain/use-arcade-with-langchain-py", - permanent: true, - }, - { - source: - "/:locale/get-started/agent-frameworks/langchain/user-auth-interrupts", - destination: - "/:locale/get-started/agent-frameworks/langchain/use-arcade-with-langchain-py", - permanent: true, - }, - // Mastra tutorial consolidation - { - source: "/:locale/get-started/agent-frameworks/mastra/overview", - destination: "/:locale/get-started/agent-frameworks/mastra", - permanent: true, - }, - { - source: - "/:locale/get-started/agent-frameworks/mastra/use-arcade-tools", - destination: "/:locale/get-started/agent-frameworks/mastra", - permanent: true, - }, - { - source: - "/:locale/get-started/agent-frameworks/mastra/user-auth-interrupts", - destination: "/:locale/get-started/agent-frameworks/mastra", - permanent: true, - }, - // OpenAI Agents tutorial consolidation - { - source: - "/:locale/get-started/agent-frameworks/openai-agents/use-arcade-with-openai-agents", - destination: - "/:locale/get-started/agent-frameworks/openai-agents/overview", - permanent: true, - }, - { - source: - "/:locale/get-started/agent-frameworks/openai-agents/use-arcade-tools", - destination: - "/:locale/get-started/agent-frameworks/openai-agents/overview", - permanent: true, - }, - { - source: - "/:locale/get-started/agent-frameworks/openai-agents/user-auth-interrupts", - destination: - "/:locale/get-started/agent-frameworks/openai-agents/overview", - permanent: true, - }, - // Moved from guides to get-started - { - source: - "/:locale/guides/agent-frameworks/setup-arcade-with-your-llm-python", - destination: - "/:locale/get-started/agent-frameworks/setup-arcade-with-your-llm-python", - permanent: true, - }, - // Old /home/* paths to new structure - { - source: "/:locale/home/langchain/use-arcade-tools", - destination: - "/:locale/get-started/agent-frameworks/langchain/use-arcade-with-langchain-py", - permanent: true, - }, - { - source: "/:locale/guides/agent-frameworks/langchain/use-arcade-tools", - destination: - "/:locale/get-started/agent-frameworks/langchain/use-arcade-with-langchain-py", - permanent: true, - }, - { - source: "/:locale/home/langchain/user-auth-interrupts", - destination: - "/:locale/get-started/agent-frameworks/langchain/use-arcade-with-langchain-py", - permanent: true, - }, - { - source: - "/:locale/guides/agent-frameworks/langchain/user-auth-interrupts", - destination: - "/:locale/get-started/agent-frameworks/langchain/use-arcade-with-langchain-py", - permanent: true, - }, - { - source: - "/:locale/get-started/agent-frameworks/langchain/use-arcade-with-langchain", - destination: - "/:locale/get-started/agent-frameworks/langchain/use-arcade-with-langchain-py", - permanent: true, - }, - { - source: "/:locale/home/oai-agents/user-auth-interrupts", - destination: - "/:locale/get-started/agent-frameworks/openai-agents/overview", - permanent: true, - }, - { - source: "/:locale/home/mastra/user-auth-interrupts", - destination: "/:locale/get-started/agent-frameworks/mastra", - permanent: true, - }, - { - source: "/:locale/home/build-tools/server-level-vs-tool-level-auth", - destination: "/:locale/learn/server-level-vs-tool-level-auth", - permanent: true, - }, - { - source: "/:locale/home/build-tools/secure-your-mcp-server", - destination: - "/:locale/guides/create-tools/secure-your-server/secure-your-mcp-server", - permanent: true, - }, - { - source: "/:locale/home/agent-frameworks-overview", - destination: "/:locale/get-started/agent-frameworks", - permanent: true, - }, - { - source: "/:locale/home/agentic-development", - destination: "/:locale/get-started/setup/connect-arcade-docs", - permanent: true, - }, - { - source: "/:locale/home/api-keys", - destination: "/:locale/get-started/setup/api-keys", - permanent: true, - }, - { - source: - "/:locale/guides/agent-frameworks/vercelai/using-arcade-tools", - destination: "/:locale/get-started/agent-frameworks/vercelai", - permanent: true, - }, - { - source: "/:locale/home/arcade-cli", - destination: "/:locale/references/arcade-cli", - permanent: true, - }, - { - source: "/:locale/home/auth-providers", - destination: "/:locale/references/auth-providers", - permanent: true, - }, - { - source: "/:locale/home/auth-providers/:path*", - destination: "/:locale/references/auth-providers/:path*", - permanent: true, - }, - { - source: "/:locale/home/auth/auth-tool-calling", - destination: - "/:locale/guides/tool-calling/custom-apps/auth-tool-calling", - permanent: true, - }, - { - source: "/:locale/home/auth/call-third-party-apis-directly", - destination: "/:locale/guides/tool-calling/call-third-party-apis", - permanent: true, - }, - { - source: "/:locale/home/auth/how-arcade-helps", - destination: "/:locale/get-started/about-arcade", - permanent: true, - }, - { - source: "/:locale/home/auth/secure-auth-production", - destination: - "/:locale/guides/user-facing-agents/secure-auth-production", - permanent: true, - }, - { - source: "/:locale/home/auth/tool-auth-status", - destination: - "/:locale/guides/tool-calling/custom-apps/check-auth-status", - permanent: true, - }, - { - source: "/:locale/home/build-tools/call-tools-from-mcp-clients", - destination: - "/:locale/guides/create-tools/tool-basics/call-tools-mcp", - permanent: true, - }, - { - source: "/:locale/home/build-tools/create-a-mcp-server", - destination: - "/:locale/guides/create-tools/tool-basics/build-mcp-server", - permanent: true, - }, - { - source: "/:locale/home/build-tools/create-a-tool-with-auth", - destination: - "/:locale/guides/create-tools/tool-basics/create-tool-auth", - permanent: true, - }, - { - source: "/:locale/home/build-tools/create-a-tool-with-secrets", - destination: - "/:locale/guides/create-tools/tool-basics/create-tool-secrets", - permanent: true, - }, - { - source: "/:locale/home/build-tools/migrate-from-toolkits", - destination: "/:locale/guides/create-tools/migrate-toolkits", - permanent: true, - }, - { - source: "/:locale/home/build-tools/organize-mcp-server-tools", - destination: - "/:locale/guides/create-tools/tool-basics/organize-mcp-tools", - permanent: true, - }, - { - source: "/:locale/home/build-tools/providing-useful-tool-errors", - destination: - "/:locale/guides/create-tools/error-handling/useful-tool-errors", - permanent: true, - }, - { - source: "/:locale/home/build-tools/retry-tools-with-improved-prompt", - destination: - "/:locale/guides/create-tools/error-handling/retry-tools", - permanent: true, - }, - { - source: "/:locale/home/build-tools/tool-context", - destination: - "/:locale/guides/create-tools/tool-basics/runtime-data-access", - permanent: true, - }, - { - source: "/:locale/home/changelog", - destination: "/:locale/references/changelog", - permanent: true, - }, - { - source: "/:locale/home/compare-server-types", - destination: - "/:locale/guides/create-tools/tool-basics/compare-server-types", - permanent: true, - }, - { - source: "/:locale/home/contact-us", - destination: "/:locale/resources/contact-us", - permanent: true, - }, - { - source: "/:locale/home/crewai/custom-auth-flow", - destination: - "/:locale/get-started/agent-frameworks/crewai/use-arcade-tools", - permanent: true, - }, - { - source: "/:locale/home/crewai/use-arcade-tools", - destination: - "/:locale/get-started/agent-frameworks/crewai/use-arcade-tools", - permanent: true, - }, - { - source: "/:locale/home/custom-mcp-server-quickstart", - destination: "/:locale/get-started/quickstarts/mcp-server-quickstart", - permanent: true, - }, - { - source: "/:locale/home/deployment/arcade-cloud-infra", - destination: "/:locale/guides/deployment-hosting/arcade-cloud", - permanent: true, - }, - { - source: "/:locale/home/deployment/engine-configuration", - destination: "/:locale/guides/deployment-hosting/helm", - permanent: true, - }, - { - source: "/:locale/home/evaluate-tools/create-an-evaluation-suite", - destination: - "/:locale/guides/create-tools/evaluate-tools/create-evaluation-suite", - permanent: true, - }, - { - source: "/:locale/home/evaluate-tools/run-evaluations", - destination: - "/:locale/guides/create-tools/evaluate-tools/run-evaluations", - permanent: true, - }, - { - source: "/:locale/home/evaluate-tools/why-evaluate-tools", - destination: - "/:locale/guides/create-tools/evaluate-tools/why-evaluate", - permanent: true, - }, - { - source: "/:locale/home/examples", - destination: "/:locale/resources/examples", - permanent: true, - }, - { - source: "/:locale/home/faq", - destination: "/:locale/resources/faq", - permanent: true, - }, - { - source: "/:locale/home/glossary", - destination: "/:locale/resources/glossary", - permanent: true, - }, - { - source: "/:locale/home/google-adk/use-arcade-tools", - destination: - "/:locale/get-started/agent-frameworks/google-adk/setup-python", - permanent: true, - }, - { - source: "/:locale/home/hosting-overview", - destination: "/:locale/guides/deployment-hosting", - permanent: true, - }, - { - source: "/:locale/home/langchain/auth-langchain-tools", - destination: - "/:locale/get-started/agent-frameworks/langchain/auth-langchain-tools", - permanent: true, - }, - { - source: "/:locale/home/mastra/use-arcade-tools", - destination: "/:locale/get-started/agent-frameworks/mastra", - permanent: true, - }, - { - source: "/:locale/home/mcp-clients/claude-desktop", - destination: "/:locale/get-started/mcp-clients/claude-desktop", - permanent: true, - }, - { - source: "/:locale/home/mcp-clients/claude-code", - destination: "/:locale/get-started/mcp-clients/claude-code", - permanent: true, - }, - { - source: "/:locale/home/mcp-clients/cursor", - destination: "/:locale/get-started/mcp-clients/cursor", - permanent: true, - }, - { - source: "/:locale/home/mcp-clients/visual-studio-code", - destination: "/:locale/get-started/mcp-clients/visual-studio-code", - permanent: true, - }, - { - source: "/:locale/home/mcp-gateway-quickstart", - destination: "/:locale/get-started/quickstarts/call-tool-client", - permanent: true, - }, - { - source: "/:locale/home/mcp-gateways", - destination: "/:locale/guides/mcp-gateways", - permanent: true, - }, - { - source: "/:locale/home/oai-agents/use-arcade-tools", - destination: - "/:locale/get-started/agent-frameworks/openai-agents/overview", - permanent: true, - }, - { - source: "/:locale/home/quickstart", - destination: "/:locale/get-started/quickstarts/call-tool-agent", - permanent: true, - }, - { - source: "/:locale/home/registry-early-access", - destination: "/:locale/resources/registry-early-access", - permanent: true, - }, - { - source: "/:locale/home/serve-tools/arcade-deploy", - destination: "/:locale/guides/deployment-hosting/arcade-deploy", - permanent: true, - }, - { - source: "/:locale/home/serve-tools/hybrid-worker", - destination: "/:locale/guides/deployment-hosting/on-prem", - permanent: true, - }, - { - source: "/:locale/home/use-tools/get-tool-definitions", - destination: - "/:locale/guides/tool-calling/custom-apps/get-tool-definitions", - permanent: true, - }, - { - source: "/:locale/home/use-tools/tools-overview", - destination: "/:locale/guides/tool-calling", - permanent: true, - }, - { - source: "/:locale/home/use-tools/types-of-tools", - destination: "/:locale/guides/create-tools/improve/types-of-tools", - permanent: true, - }, - { - source: "/:locale/home/use-tools/error-handling", - destination: "/:locale/guides/tool-calling/error-handling", - permanent: true, - }, - { - source: "/:locale/home/vercelai/using-arcade-tools", - destination: "/:locale/get-started/agent-frameworks/vercelai", - permanent: true, - }, - // Legacy /integrations path - // NOTE: :locale is constrained to actual locale values to prevent - // collisions with locale-less paths like /resources/integrations, - // which would otherwise match with :locale="resources" and redirect - // to /resources/resources/integrations (a 404). - { - source: "/:locale(en|es|pt-BR)/integrations", - destination: "/:locale/resources/integrations", - permanent: true, - }, - { - source: "/:locale(en|es|pt-BR)/integrations/:path*", - destination: "/:locale/resources/integrations/:path*", - permanent: true, - }, - // MCP servers to integrations - { - source: "/:locale(en|es|pt-BR)/mcp-servers", - destination: "/:locale/resources/integrations", - permanent: true, - }, - { - source: "/:locale(en|es|pt-BR)/mcp-servers/:path*", - destination: "/:locale/resources/integrations/:path*", - permanent: true, - }, - // References fixes - { - source: "/:locale/references/mcp", - destination: "/:locale/references/mcp/python", - permanent: true, - }, - { - source: "/:locale/references/mcp/python/overview", - destination: "/:locale/references/mcp/python", - permanent: true, - }, - { - source: "/:locale/references/arcade-cliarcade-configure", - destination: "/:locale/references/arcade-cli", - permanent: true, - }, - // Path corrections (typos, renames) - { - source: "/:locale/get-started/setup/api-key", - destination: "/:locale/get-started/setup/api-keys", - permanent: true, - }, - { - source: - "/:locale/guides/tool-calling/custom-apps/authorized-tool-calling", - destination: - "/:locale/guides/tool-calling/custom-apps/auth-tool-calling", - permanent: true, - }, - { - source: "/:locale/guides/user-facing-agents/brand-provider", - destination: - "/:locale/guides/user-facing-agents/secure-auth-production", - permanent: true, - }, - { - source: "/:locale/guides/user-facing-agents/configure-oauth-provider", - destination: - "/:locale/guides/user-facing-agents/secure-auth-production", - permanent: true, - }, - { - source: "/:locale/guides/tool-calling/mcp-client/:client", - destination: "/:locale/get-started/mcp-clients/:client", - permanent: true, - }, - { - source: "/:locale/guides/tool-calling/get-tool-definitions", - destination: - "/:locale/guides/tool-calling/custom-apps/get-tool-definitions", - permanent: true, - }, - { - source: "/:locale/guides/deployment-hosting/engine-configuration", - destination: "/:locale/guides/deployment-hosting/helm", - permanent: true, - }, - { - source: "/:locale/guides/deployment-hosting/configure-engine", - destination: "/:locale/guides/deployment-hosting/helm", - permanent: true, - }, - { - source: "/:locale/guides/create-tools/performance/run-evaluations", - destination: - "/:locale/guides/create-tools/evaluate-tools/run-evaluations", - permanent: true, - }, - { - source: "/:locale/guides/create-tools/contribute/registry", - destination: "/:locale/resources/registry-early-access", - permanent: true, - }, - // Framework path aliases (old naming conventions) - { - source: "/:locale/guides/agent-frameworks/crewai/python", - destination: - "/:locale/get-started/agent-frameworks/crewai/use-arcade-tools", - permanent: true, - }, - { - source: "/:locale/guides/agent-frameworks/langchain/python", - destination: - "/:locale/get-started/agent-frameworks/langchain/use-arcade-with-langchain-py", - permanent: true, - }, - { - source: "/:locale/guides/agent-frameworks/langchain/tools", - destination: - "/:locale/get-started/agent-frameworks/langchain/auth-langchain-tools", - permanent: true, - }, - { - source: "/:locale/guides/agent-frameworks/mastra/typescript", - destination: "/:locale/get-started/agent-frameworks/mastra", - permanent: true, - }, - { - source: "/:locale/guides/agent-frameworks/google-adk/python", - destination: - "/:locale/get-started/agent-frameworks/google-adk/setup-python", - permanent: true, - }, - { - source: "/:locale/guides/agent-frameworks/openai/python", - destination: - "/:locale/get-started/agent-frameworks/openai-agents/overview", - permanent: true, - }, - { - source: "/:locale/guides/agent-frameworks/vercel-ai/typescript", - destination: "/:locale/get-started/agent-frameworks/vercelai", - permanent: true, - }, - // Old resource paths - { - source: "/:locale/resources/mastra/user-auth-interrupts", - destination: "/:locale/get-started/agent-frameworks/mastra", - permanent: true, - }, - { - source: "/:locale/resources/oai-agents/overview", - destination: - "/:locale/get-started/agent-frameworks/openai-agents/overview", - permanent: true, - }, - { - source: "/:locale/resources/creating-tools/:path*", - destination: "/:locale/guides/create-tools/:path*", - permanent: true, - }, - // Agent frameworks moved from guides to get-started - { - source: "/:locale/guides/agent-frameworks", - destination: "/:locale/get-started/agent-frameworks", - permanent: true, - }, - { - source: "/:locale/guides/agent-frameworks/:path*", - destination: "/:locale/get-started/agent-frameworks/:path*", - permanent: true, - }, - // MCP clients moved from guides/tool-calling to get-started - { - source: "/:locale/guides/tool-calling/mcp-clients", - destination: "/:locale/get-started/mcp-clients", - permanent: true, - }, - { - source: "/:locale/guides/tool-calling/mcp-clients/:path*", - destination: "/:locale/get-started/mcp-clients/:path*", - permanent: true, - }, - // Deprecated toolkit renames (microsoft_* prefix, ArcadeAI/monorepo#601) - { - source: "/:locale/resources/integrations/productivity/sharepoint", - destination: - "/:locale/resources/integrations/productivity/microsoft-sharepoint", - permanent: true, - }, - { - source: "/:locale/resources/integrations/productivity/outlook-mail", - destination: - "/:locale/resources/integrations/productivity/microsoft-outlook-mail", - permanent: true, - }, - { - source: - "/:locale/resources/integrations/productivity/outlook-calendar", - destination: - "/:locale/resources/integrations/productivity/microsoft-outlook-calendar", - permanent: true, - }, - ]; + return redirects; + }, + // The app imports shared modules out of toolkit-docs-generator/src/shared/, + // which compiles under "moduleResolution": "NodeNext" and therefore writes + // its internal relative imports with a ".js" extension. Webpack resolves + // with bundler semantics and would look for a literal ".js" file that + // never exists on disk, so teach it to try ".ts" first. + webpack: (config) => { + config.resolve.extensionAlias = { + ...config.resolve.extensionAlias, + ".js": [".ts", ".tsx", ".js"], + }; + return config; }, headers: async () => [ { diff --git a/package.json b/package.json index 439bbb3da..657540f93 100644 --- a/package.json +++ b/package.json @@ -63,7 +63,8 @@ "remark-gfm": "4.0.1", "swagger-ui-react": "5.32.6", "tailwindcss-animate": "1.0.7", - "unist-util-visit": "5.1.0" + "unist-util-visit": "5.1.0", + "zod": "4.3.6" }, "devDependencies": { "@anthropic-ai/sdk": "0.91.1", @@ -93,8 +94,7 @@ "typescript": "5.9.3", "ultracite": "6.1.0", "vite": "7.3.5", - "vitest": "4.1.8", - "zod": "4.3.6" + "vitest": "4.1.8" }, "engines": { "node": "22.x", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index d9790c3ca..761fc4215 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -88,6 +88,9 @@ importers: unist-util-visit: specifier: 5.1.0 version: 5.1.0 + zod: + specifier: 4.3.6 + version: 4.3.6 devDependencies: '@anthropic-ai/sdk': specifier: 0.91.1 @@ -173,9 +176,6 @@ importers: vitest: specifier: 4.1.8 version: 4.1.8(@opentelemetry/api@1.9.0)(@types/node@22.19.17)(vite@7.3.5(@types/node@22.19.17)(jiti@2.6.1)(lightningcss@1.32.0)(yaml@2.8.3)) - zod: - specifier: 4.3.6 - version: 4.3.6 packages: diff --git a/redirects.ts b/redirects.ts new file mode 100644 index 000000000..7e4b338b5 --- /dev/null +++ b/redirects.ts @@ -0,0 +1,875 @@ +/** + * Redirect rules for renamed, merged, or deleted pages. + * + * This is a plain data module (not next.config.ts) so it can be imported + * directly by scripts/check-redirects.ts and tests/sitemap.test.ts instead of + * regex-parsing next.config.ts as text. + * + * `pnpm check-redirects --auto-fix` appends new entries under the + * "Auto-added redirects" comment near the end of this file. + */ + +export type Redirect = { + source: string; + destination: string; + permanent: boolean; +}; + +export const redirects: Redirect[] = [ + // The toolkit-page breadcrumb links "Resources" -> /resources, which has + // no index page. Send it to the integrations registry instead of 404ing. + { + source: "/:locale/resources", + destination: "/:locale/resources/integrations", + permanent: true, + }, + // The auth provider is "square"; an external/stale link points at the + // old "squareup" slug, which 404s. Send it to the real page. + { + source: "/:locale/references/auth-providers/squareup", + destination: "/:locale/references/auth-providers/square", + permanent: true, + }, + // Dissolved guides/security section + { + source: "/:locale/guides/security/security-research-program", + destination: "/:locale/resources/security-research-program", + permanent: true, + }, + { + source: "/:locale/guides/security/securing-arcade-mcp", + destination: "/:locale/guides/create-tools/secure-your-server", + permanent: true, + }, + { + source: "/:locale/guides/security/secure-your-mcp-server", + destination: + "/:locale/guides/create-tools/secure-your-server/secure-your-mcp-server", + permanent: true, + }, + { + source: "/:locale/guides/security", + destination: "/:locale/guides/create-tools/secure-your-server", + permanent: true, + }, + { + source: "/:locale/references/mcp/python/transports", + destination: "/:locale/references/mcp/python", + permanent: true, + }, + { + source: "/:locale/references/mcp/python/types", + destination: "/:locale/references/mcp/python", + permanent: true, + }, + // CrewAI custom auth flow redirect to use-arcade-tools + { + source: "/:locale/get-started/agent-frameworks/crewai/custom-auth-flow", + destination: + "/:locale/get-started/agent-frameworks/crewai/use-arcade-tools", + permanent: true, + }, + // "others" category removed — toolkits moved to proper categories + { + source: "/:locale/resources/integrations/others/:path*", + destination: "/:locale/resources/integrations", + permanent: false, + }, + // Google ADK tutorial consolidation - redirect old URL to new + { + source: "/:locale/get-started/agent-frameworks/google-adk/use-arcade-tools", + destination: "/:locale/get-started/agent-frameworks/google-adk/overview", + permanent: true, + }, + { + source: "/:locale/references/logic-extensions-api", + destination: "/:locale/references/contextual-access-webhook-api", + permanent: true, + }, + { + source: "/:locale/guides/logic-extensions", + destination: "/:locale/guides/contextual-access", + permanent: true, + }, + { + source: "/:locale/guides/logic-extensions/build-your-own", + destination: "/:locale/guides/contextual-access/build-your-own", + permanent: true, + }, + { + source: "/:locale/guides/logic-extensions/examples", + destination: "/:locale/guides/contextual-access/examples", + permanent: true, + }, + { + source: "/:locale/guides/logic-extensions/how-hooks-work", + destination: "/:locale/guides/contextual-access/how-hooks-work", + permanent: true, + }, + { + source: "/:locale/resources/integrations/preview", + destination: "/:locale/resources/integrations", + permanent: true, + }, + { + source: + "/:locale/resources/integrations/customer-support/zendesk/reference", + destination: "/:locale/resources/integrations/customer-support/zendesk", + permanent: true, + }, + { + source: "/:locale/resources/integrations/development/firecrawl/reference", + destination: "/:locale/resources/integrations/development/firecrawl", + permanent: true, + }, + { + source: "/:locale/resources/integrations/productivity/asana/reference", + destination: "/:locale/resources/integrations/productivity/asana", + permanent: true, + }, + { + source: "/:locale/resources/integrations/productivity/clickup/reference", + destination: "/:locale/resources/integrations/productivity/clickup", + permanent: true, + }, + { + source: "/:locale/resources/integrations/productivity/dropbox/reference", + destination: "/:locale/resources/integrations/productivity/dropbox", + permanent: true, + }, + { + source: "/:locale/resources/integrations/productivity/gmail/reference", + destination: "/:locale/resources/integrations/productivity/gmail", + permanent: true, + }, + { + source: + "/:locale/resources/integrations/productivity/google-calendar/reference", + destination: "/:locale/resources/integrations/productivity/google-calendar", + permanent: true, + }, + { + source: + "/:locale/resources/integrations/productivity/google-docs/reference", + destination: "/:locale/resources/integrations/productivity/google-docs", + permanent: true, + }, + { + source: + "/:locale/resources/integrations/productivity/google-drive/reference", + destination: "/:locale/resources/integrations/productivity/google-drive", + permanent: true, + }, + { + source: + "/:locale/resources/integrations/productivity/google-sheets/reference", + destination: "/:locale/resources/integrations/productivity/google-sheets", + permanent: true, + }, + { + source: + "/:locale/resources/integrations/productivity/jira/environment-variables", + destination: "/:locale/resources/integrations/productivity/jira", + permanent: true, + }, + { + source: "/:locale/resources/integrations/productivity/jira/reference", + destination: "/:locale/resources/integrations/productivity/jira", + permanent: true, + }, + { + source: "/:locale/resources/integrations/sales/hubspot/reference", + destination: "/:locale/resources/integrations/sales/hubspot", + permanent: true, + }, + { + source: "/:locale/resources/integrations/social-communication/discord", + destination: "/:locale/resources/integrations", + permanent: true, + }, + { + source: "/:locale/resources/integrations/social-communication/linkedin", + destination: "/:locale/resources/integrations/social/linkedin", + permanent: true, + }, + { + source: + "/:locale/resources/integrations/social-communication/microsoft-teams", + destination: "/:locale/resources/integrations/social/microsoft-teams", + permanent: true, + }, + { + source: + "/:locale/resources/integrations/social-communication/microsoft-teams/reference", + destination: "/:locale/resources/integrations/social/microsoft-teams", + permanent: true, + }, + { + source: "/:locale/resources/integrations/social-communication/reddit", + destination: "/:locale/resources/integrations/social/reddit", + permanent: true, + }, + { + source: "/:locale/resources/integrations/social-communication/slack-api", + destination: "/:locale/resources/integrations/social/slack-api", + permanent: true, + }, + { + source: + "/:locale/resources/integrations/social-communication/slack/environment-variables", + destination: "/:locale/resources/integrations/social/slack", + permanent: true, + }, + { + source: + "/:locale/resources/integrations/social-communication/slack/install", + destination: "/:locale/resources/integrations/social/slack", + permanent: true, + }, + { + source: "/:locale/resources/integrations/social-communication/slack", + destination: "/:locale/resources/integrations/social/slack", + permanent: true, + }, + { + source: + "/:locale/resources/integrations/social-communication/slack/reference", + destination: "/:locale/resources/integrations/social/slack", + permanent: true, + }, + { + source: + "/:locale/resources/integrations/social-communication/teams/reference", + destination: "/:locale/resources/integrations/social/microsoft-teams", + permanent: true, + }, + { + source: "/:locale/resources/integrations/social-communication/twilio", + destination: "/:locale/resources/integrations", + permanent: true, + }, + { + source: + "/:locale/resources/integrations/social-communication/twilio/reference", + destination: "/:locale/resources/integrations", + permanent: true, + }, + { + source: "/:locale/resources/integrations/social-communication/x", + destination: "/:locale/resources/integrations/social/x", + permanent: true, + }, + { + source: "/:locale/resources/integrations/social-communication/zoom/install", + destination: "/:locale/resources/integrations/social/zoom", + permanent: true, + }, + { + source: "/:locale/resources/integrations/social-communication/zoom", + destination: "/:locale/resources/integrations/social/zoom", + permanent: true, + }, + { + source: "/:locale/guides/create-tools/contribute/registry-early-access", + destination: "/:locale/resources/registry-early-access", + permanent: true, + }, + { + source: "/:locale/resources/integrations/contribute-a-server", + destination: "/:locale/resources/registry-early-access", + permanent: true, + }, + // Moved MCP Gateway UI guide to guides + { + source: "/:locale/guides/create-tools/mcp-gateways", + destination: "/:locale/guides/mcp-gateways", + permanent: true, + }, + // Removed LangChain old stuff + { + source: "/:locale/get-started/agent-frameworks/langchain/use-arcade-tools", + destination: + "/:locale/get-started/agent-frameworks/langchain/use-arcade-with-langchain-py", + permanent: true, + }, + { + source: + "/:locale/get-started/agent-frameworks/langchain/user-auth-interrupts", + destination: + "/:locale/get-started/agent-frameworks/langchain/use-arcade-with-langchain-py", + permanent: true, + }, + // Mastra tutorial consolidation + { + source: "/:locale/get-started/agent-frameworks/mastra/overview", + destination: "/:locale/get-started/agent-frameworks/mastra", + permanent: true, + }, + { + source: "/:locale/get-started/agent-frameworks/mastra/use-arcade-tools", + destination: "/:locale/get-started/agent-frameworks/mastra", + permanent: true, + }, + { + source: "/:locale/get-started/agent-frameworks/mastra/user-auth-interrupts", + destination: "/:locale/get-started/agent-frameworks/mastra", + permanent: true, + }, + // OpenAI Agents tutorial consolidation + { + source: + "/:locale/get-started/agent-frameworks/openai-agents/use-arcade-with-openai-agents", + destination: "/:locale/get-started/agent-frameworks/openai-agents/overview", + permanent: true, + }, + { + source: + "/:locale/get-started/agent-frameworks/openai-agents/use-arcade-tools", + destination: "/:locale/get-started/agent-frameworks/openai-agents/overview", + permanent: true, + }, + { + source: + "/:locale/get-started/agent-frameworks/openai-agents/user-auth-interrupts", + destination: "/:locale/get-started/agent-frameworks/openai-agents/overview", + permanent: true, + }, + // Moved from guides to get-started + { + source: + "/:locale/guides/agent-frameworks/setup-arcade-with-your-llm-python", + destination: + "/:locale/get-started/agent-frameworks/setup-arcade-with-your-llm-python", + permanent: true, + }, + // Old /home/* paths to new structure + { + source: "/:locale/home/langchain/use-arcade-tools", + destination: + "/:locale/get-started/agent-frameworks/langchain/use-arcade-with-langchain-py", + permanent: true, + }, + { + source: "/:locale/guides/agent-frameworks/langchain/use-arcade-tools", + destination: + "/:locale/get-started/agent-frameworks/langchain/use-arcade-with-langchain-py", + permanent: true, + }, + { + source: "/:locale/home/langchain/user-auth-interrupts", + destination: + "/:locale/get-started/agent-frameworks/langchain/use-arcade-with-langchain-py", + permanent: true, + }, + { + source: "/:locale/guides/agent-frameworks/langchain/user-auth-interrupts", + destination: + "/:locale/get-started/agent-frameworks/langchain/use-arcade-with-langchain-py", + permanent: true, + }, + { + source: + "/:locale/get-started/agent-frameworks/langchain/use-arcade-with-langchain", + destination: + "/:locale/get-started/agent-frameworks/langchain/use-arcade-with-langchain-py", + permanent: true, + }, + { + source: "/:locale/home/oai-agents/user-auth-interrupts", + destination: "/:locale/get-started/agent-frameworks/openai-agents/overview", + permanent: true, + }, + { + source: "/:locale/home/mastra/user-auth-interrupts", + destination: "/:locale/get-started/agent-frameworks/mastra", + permanent: true, + }, + { + source: "/:locale/home/build-tools/server-level-vs-tool-level-auth", + destination: "/:locale/learn/server-level-vs-tool-level-auth", + permanent: true, + }, + { + source: "/:locale/home/build-tools/secure-your-mcp-server", + destination: + "/:locale/guides/create-tools/secure-your-server/secure-your-mcp-server", + permanent: true, + }, + { + source: "/:locale/home/agent-frameworks-overview", + destination: "/:locale/get-started/agent-frameworks", + permanent: true, + }, + { + source: "/:locale/home/agentic-development", + destination: "/:locale/get-started/setup/connect-arcade-docs", + permanent: true, + }, + { + source: "/:locale/home/api-keys", + destination: "/:locale/get-started/setup/api-keys", + permanent: true, + }, + { + source: "/:locale/guides/agent-frameworks/vercelai/using-arcade-tools", + destination: "/:locale/get-started/agent-frameworks/vercelai", + permanent: true, + }, + { + source: "/:locale/home/arcade-cli", + destination: "/:locale/references/arcade-cli", + permanent: true, + }, + { + source: "/:locale/home/auth-providers", + destination: "/:locale/references/auth-providers", + permanent: true, + }, + { + source: "/:locale/home/auth-providers/:path*", + destination: "/:locale/references/auth-providers/:path*", + permanent: true, + }, + { + source: "/:locale/home/auth/auth-tool-calling", + destination: "/:locale/guides/tool-calling/custom-apps/auth-tool-calling", + permanent: true, + }, + { + source: "/:locale/home/auth/call-third-party-apis-directly", + destination: "/:locale/guides/tool-calling/call-third-party-apis", + permanent: true, + }, + { + source: "/:locale/home/auth/how-arcade-helps", + destination: "/:locale/get-started/about-arcade", + permanent: true, + }, + { + source: "/:locale/home/auth/secure-auth-production", + destination: "/:locale/guides/user-facing-agents/secure-auth-production", + permanent: true, + }, + { + source: "/:locale/home/auth/tool-auth-status", + destination: "/:locale/guides/tool-calling/custom-apps/check-auth-status", + permanent: true, + }, + { + source: "/:locale/home/build-tools/call-tools-from-mcp-clients", + destination: "/:locale/guides/create-tools/tool-basics/call-tools-mcp", + permanent: true, + }, + { + source: "/:locale/home/build-tools/create-a-mcp-server", + destination: "/:locale/guides/create-tools/tool-basics/build-mcp-server", + permanent: true, + }, + { + source: "/:locale/home/build-tools/create-a-tool-with-auth", + destination: "/:locale/guides/create-tools/tool-basics/create-tool-auth", + permanent: true, + }, + { + source: "/:locale/home/build-tools/create-a-tool-with-secrets", + destination: "/:locale/guides/create-tools/tool-basics/create-tool-secrets", + permanent: true, + }, + { + source: "/:locale/home/build-tools/migrate-from-toolkits", + destination: "/:locale/guides/create-tools/migrate-toolkits", + permanent: true, + }, + { + source: "/:locale/home/build-tools/organize-mcp-server-tools", + destination: "/:locale/guides/create-tools/tool-basics/organize-mcp-tools", + permanent: true, + }, + { + source: "/:locale/home/build-tools/providing-useful-tool-errors", + destination: + "/:locale/guides/create-tools/error-handling/useful-tool-errors", + permanent: true, + }, + { + source: "/:locale/home/build-tools/retry-tools-with-improved-prompt", + destination: "/:locale/guides/create-tools/error-handling/retry-tools", + permanent: true, + }, + { + source: "/:locale/home/build-tools/tool-context", + destination: "/:locale/guides/create-tools/tool-basics/runtime-data-access", + permanent: true, + }, + { + source: "/:locale/home/changelog", + destination: "/:locale/references/changelog", + permanent: true, + }, + { + source: "/:locale/home/compare-server-types", + destination: + "/:locale/guides/create-tools/tool-basics/compare-server-types", + permanent: true, + }, + { + source: "/:locale/home/contact-us", + destination: "/:locale/resources/contact-us", + permanent: true, + }, + { + source: "/:locale/home/crewai/custom-auth-flow", + destination: + "/:locale/get-started/agent-frameworks/crewai/use-arcade-tools", + permanent: true, + }, + { + source: "/:locale/home/crewai/use-arcade-tools", + destination: + "/:locale/get-started/agent-frameworks/crewai/use-arcade-tools", + permanent: true, + }, + { + source: "/:locale/home/custom-mcp-server-quickstart", + destination: "/:locale/get-started/quickstarts/mcp-server-quickstart", + permanent: true, + }, + { + source: "/:locale/home/deployment/arcade-cloud-infra", + destination: "/:locale/guides/deployment-hosting/arcade-cloud", + permanent: true, + }, + { + source: "/:locale/home/deployment/engine-configuration", + destination: "/:locale/guides/deployment-hosting/helm", + permanent: true, + }, + { + source: "/:locale/home/evaluate-tools/create-an-evaluation-suite", + destination: + "/:locale/guides/create-tools/evaluate-tools/create-evaluation-suite", + permanent: true, + }, + { + source: "/:locale/home/evaluate-tools/run-evaluations", + destination: "/:locale/guides/create-tools/evaluate-tools/run-evaluations", + permanent: true, + }, + { + source: "/:locale/home/evaluate-tools/why-evaluate-tools", + destination: "/:locale/guides/create-tools/evaluate-tools/why-evaluate", + permanent: true, + }, + { + source: "/:locale/home/examples", + destination: "/:locale/resources/examples", + permanent: true, + }, + { + source: "/:locale/home/faq", + destination: "/:locale/resources/faq", + permanent: true, + }, + { + source: "/:locale/home/glossary", + destination: "/:locale/resources/glossary", + permanent: true, + }, + { + source: "/:locale/home/google-adk/use-arcade-tools", + destination: + "/:locale/get-started/agent-frameworks/google-adk/setup-python", + permanent: true, + }, + { + source: "/:locale/home/hosting-overview", + destination: "/:locale/guides/deployment-hosting", + permanent: true, + }, + { + source: "/:locale/home/langchain/auth-langchain-tools", + destination: + "/:locale/get-started/agent-frameworks/langchain/auth-langchain-tools", + permanent: true, + }, + { + source: "/:locale/home/mastra/use-arcade-tools", + destination: "/:locale/get-started/agent-frameworks/mastra", + permanent: true, + }, + { + source: "/:locale/home/mcp-clients/claude-desktop", + destination: "/:locale/get-started/mcp-clients/claude-desktop", + permanent: true, + }, + { + source: "/:locale/home/mcp-clients/claude-code", + destination: "/:locale/get-started/mcp-clients/claude-code", + permanent: true, + }, + { + source: "/:locale/home/mcp-clients/cursor", + destination: "/:locale/get-started/mcp-clients/cursor", + permanent: true, + }, + { + source: "/:locale/home/mcp-clients/visual-studio-code", + destination: "/:locale/get-started/mcp-clients/visual-studio-code", + permanent: true, + }, + { + source: "/:locale/home/mcp-gateway-quickstart", + destination: "/:locale/get-started/quickstarts/call-tool-client", + permanent: true, + }, + { + source: "/:locale/home/mcp-gateways", + destination: "/:locale/guides/mcp-gateways", + permanent: true, + }, + { + source: "/:locale/home/oai-agents/use-arcade-tools", + destination: "/:locale/get-started/agent-frameworks/openai-agents/overview", + permanent: true, + }, + { + source: "/:locale/home/quickstart", + destination: "/:locale/get-started/quickstarts/call-tool-agent", + permanent: true, + }, + { + source: "/:locale/home/registry-early-access", + destination: "/:locale/resources/registry-early-access", + permanent: true, + }, + { + source: "/:locale/home/serve-tools/arcade-deploy", + destination: "/:locale/guides/deployment-hosting/arcade-deploy", + permanent: true, + }, + { + source: "/:locale/home/serve-tools/hybrid-worker", + destination: "/:locale/guides/deployment-hosting/on-prem", + permanent: true, + }, + { + source: "/:locale/home/use-tools/get-tool-definitions", + destination: + "/:locale/guides/tool-calling/custom-apps/get-tool-definitions", + permanent: true, + }, + { + source: "/:locale/home/use-tools/tools-overview", + destination: "/:locale/guides/tool-calling", + permanent: true, + }, + { + source: "/:locale/home/use-tools/types-of-tools", + destination: "/:locale/guides/create-tools/improve/types-of-tools", + permanent: true, + }, + { + source: "/:locale/home/use-tools/error-handling", + destination: "/:locale/guides/tool-calling/error-handling", + permanent: true, + }, + { + source: "/:locale/home/vercelai/using-arcade-tools", + destination: "/:locale/get-started/agent-frameworks/vercelai", + permanent: true, + }, + // Legacy /integrations path + // NOTE: :locale is constrained to actual locale values to prevent + // collisions with locale-less paths like /resources/integrations, + // which would otherwise match with :locale="resources" and redirect + // to /resources/resources/integrations (a 404). + { + source: "/:locale(en|es|pt-BR)/integrations", + destination: "/:locale/resources/integrations", + permanent: true, + }, + { + source: "/:locale(en|es|pt-BR)/integrations/:path*", + destination: "/:locale/resources/integrations/:path*", + permanent: true, + }, + // MCP servers to integrations + { + source: "/:locale(en|es|pt-BR)/mcp-servers", + destination: "/:locale/resources/integrations", + permanent: true, + }, + { + source: "/:locale(en|es|pt-BR)/mcp-servers/:path*", + destination: "/:locale/resources/integrations/:path*", + permanent: true, + }, + // References fixes + { + source: "/:locale/references/mcp", + destination: "/:locale/references/mcp/python", + permanent: true, + }, + { + source: "/:locale/references/mcp/python/overview", + destination: "/:locale/references/mcp/python", + permanent: true, + }, + { + source: "/:locale/references/arcade-cliarcade-configure", + destination: "/:locale/references/arcade-cli", + permanent: true, + }, + // Path corrections (typos, renames) + { + source: "/:locale/get-started/setup/api-key", + destination: "/:locale/get-started/setup/api-keys", + permanent: true, + }, + { + source: "/:locale/guides/tool-calling/custom-apps/authorized-tool-calling", + destination: "/:locale/guides/tool-calling/custom-apps/auth-tool-calling", + permanent: true, + }, + { + source: "/:locale/guides/user-facing-agents/brand-provider", + destination: "/:locale/guides/user-facing-agents/secure-auth-production", + permanent: true, + }, + { + source: "/:locale/guides/user-facing-agents/configure-oauth-provider", + destination: "/:locale/guides/user-facing-agents/secure-auth-production", + permanent: true, + }, + { + source: "/:locale/guides/tool-calling/mcp-client/:client", + destination: "/:locale/get-started/mcp-clients/:client", + permanent: true, + }, + { + source: "/:locale/guides/tool-calling/get-tool-definitions", + destination: + "/:locale/guides/tool-calling/custom-apps/get-tool-definitions", + permanent: true, + }, + { + source: "/:locale/guides/deployment-hosting/engine-configuration", + destination: "/:locale/guides/deployment-hosting/helm", + permanent: true, + }, + { + source: "/:locale/guides/deployment-hosting/configure-engine", + destination: "/:locale/guides/deployment-hosting/helm", + permanent: true, + }, + { + source: "/:locale/guides/create-tools/performance/run-evaluations", + destination: "/:locale/guides/create-tools/evaluate-tools/run-evaluations", + permanent: true, + }, + { + source: "/:locale/guides/create-tools/contribute/registry", + destination: "/:locale/resources/registry-early-access", + permanent: true, + }, + // Framework path aliases (old naming conventions) + { + source: "/:locale/guides/agent-frameworks/crewai/python", + destination: + "/:locale/get-started/agent-frameworks/crewai/use-arcade-tools", + permanent: true, + }, + { + source: "/:locale/guides/agent-frameworks/langchain/python", + destination: + "/:locale/get-started/agent-frameworks/langchain/use-arcade-with-langchain-py", + permanent: true, + }, + { + source: "/:locale/guides/agent-frameworks/langchain/tools", + destination: + "/:locale/get-started/agent-frameworks/langchain/auth-langchain-tools", + permanent: true, + }, + { + source: "/:locale/guides/agent-frameworks/mastra/typescript", + destination: "/:locale/get-started/agent-frameworks/mastra", + permanent: true, + }, + { + source: "/:locale/guides/agent-frameworks/google-adk/python", + destination: + "/:locale/get-started/agent-frameworks/google-adk/setup-python", + permanent: true, + }, + { + source: "/:locale/guides/agent-frameworks/openai/python", + destination: "/:locale/get-started/agent-frameworks/openai-agents/overview", + permanent: true, + }, + { + source: "/:locale/guides/agent-frameworks/vercel-ai/typescript", + destination: "/:locale/get-started/agent-frameworks/vercelai", + permanent: true, + }, + // Old resource paths + { + source: "/:locale/resources/mastra/user-auth-interrupts", + destination: "/:locale/get-started/agent-frameworks/mastra", + permanent: true, + }, + { + source: "/:locale/resources/oai-agents/overview", + destination: "/:locale/get-started/agent-frameworks/openai-agents/overview", + permanent: true, + }, + { + source: "/:locale/resources/creating-tools/:path*", + destination: "/:locale/guides/create-tools/:path*", + permanent: true, + }, + // Agent frameworks moved from guides to get-started + { + source: "/:locale/guides/agent-frameworks", + destination: "/:locale/get-started/agent-frameworks", + permanent: true, + }, + { + source: "/:locale/guides/agent-frameworks/:path*", + destination: "/:locale/get-started/agent-frameworks/:path*", + permanent: true, + }, + // MCP clients moved from guides/tool-calling to get-started + { + source: "/:locale/guides/tool-calling/mcp-clients", + destination: "/:locale/get-started/mcp-clients", + permanent: true, + }, + { + source: "/:locale/guides/tool-calling/mcp-clients/:path*", + destination: "/:locale/get-started/mcp-clients/:path*", + permanent: true, + }, + // Deprecated toolkit renames (microsoft_* prefix, ArcadeAI/monorepo#601) + { + source: "/:locale/resources/integrations/productivity/sharepoint", + destination: + "/:locale/resources/integrations/productivity/microsoft-sharepoint", + permanent: true, + }, + { + source: "/:locale/resources/integrations/productivity/outlook-mail", + destination: + "/:locale/resources/integrations/productivity/microsoft-outlook-mail", + permanent: true, + }, + { + source: "/:locale/resources/integrations/productivity/outlook-calendar", + destination: + "/:locale/resources/integrations/productivity/microsoft-outlook-calendar", + permanent: true, + }, + + // Auto-added redirects for deleted pages. + // `pnpm check-redirects --auto-fix` appends new entries here. +]; diff --git a/scripts/check-redirects.ts b/scripts/check-redirects.ts index 23b9716af..23733e0c3 100644 --- a/scripts/check-redirects.ts +++ b/scripts/check-redirects.ts @@ -1,21 +1,32 @@ #!/usr/bin/env npx tsx /** - * Check that deleted/renamed markdown files have corresponding redirects in next.config.ts + * Check that deleted/renamed markdown files have corresponding redirects in redirects.ts * * Usage: * pnpm check-redirects [--auto-fix] [--staged-only] [base_branch] * * Features: * - Detects deleted AND renamed markdown files without redirects - * - Auto-fix mode: automatically inserts redirect entries into next.config.ts + * - Auto-fix mode: automatically inserts redirect entries into redirects.ts * - Validates existing redirects for circular references and invalid destinations * - Collapses redirect chains automatically * - --staged-only: Only check staged changes (for pre-commit hook) */ import { execSync } from "node:child_process"; -import { existsSync, readFileSync, writeFileSync } from "node:fs"; +import { readFileSync, writeFileSync } from "node:fs"; +import { redirects as configuredRedirects } from "../redirects"; +import { + checkWildcardMatch, + type DynamicRouteMove, + dynamicRouteExists, + fileToUrl, + isMoveCoveredByRedirect, + pageExists, + parseDynamicRouteMoves, + type Redirect, +} from "./lib/check-redirects-utils"; // Colors for terminal output const colors = { @@ -31,31 +42,12 @@ const autoFix = args.includes("--auto-fix"); const stagedOnly = args.includes("--staged-only"); const baseBranch = args.find((arg) => !arg.startsWith("--")) || "main"; -const CONFIG_FILE = "next.config.ts"; - -// Magic number constant for "return [" offset -const RETURN_BRACKET_LENGTH = 8; +const REDIRECTS_FILE = "redirects.ts"; // Top-level regex patterns for performance -const APP_LOCALE_PREFIX_REGEX = /^app\/[a-z]{2}\//; -const PAGE_FILE_SUFFIX_REGEX = /\/?page\.mdx?$/; -const LOCALE_PREFIX_REGEX = /^\/:locale\/?/; const PAGE_FILE_MATCH_REGEX = /page\.mdx?$/; const LOCALE_PATH_PREFIX_REGEX = /^\/:locale\//; -const WILDCARD_PATH_REGEX = /\/:path\*.*$/; -const MDX_EXTENSION_REGEX = /\.mdx$/; const SPECIAL_REGEX_CHARS_REGEX = /[.*+?^${}()|[\]\\]/g; -const REDIRECT_REGEX = - /\{\s*source:\s*["']([^"']+)["']\s*,\s*destination:\s*["']([^"']+)["']/g; -const REVERSED_REDIRECT_REGEX = - /\{\s*destination:\s*["']([^"']+)["']\s*,\s*source:\s*["']([^"']+)["']/g; -const DYNAMIC_ROUTE_REGEX = /\[[^\]]+\]/; - -type Redirect = { - source: string; - destination: string; - permanent?: boolean; -}; type RedirectChain = { source: string; @@ -63,142 +55,6 @@ type RedirectChain = { newDest: string; }; -type DynamicRouteMove = { - oldPath: string; - newPath: string; - oldUrl: string; - newUrl: string; -}; - -/** - * Convert file path to URL path - * e.g., app/en/guides/foo/page.mdx -> /:locale/guides/foo - */ -function fileToUrl(filePath: string): string { - const urlPath = filePath - .replace(APP_LOCALE_PREFIX_REGEX, "") - .replace(PAGE_FILE_SUFFIX_REGEX, ""); - - return urlPath ? `/:locale/${urlPath}` : "/:locale"; -} - -/** - * Convert URL path to file path - * e.g., /:locale/guides/foo -> app/en/guides/foo/page.mdx - */ -function urlToFile(urlPath: string): string { - const pathWithoutLocale = urlPath.replace(LOCALE_PREFIX_REGEX, ""); - return pathWithoutLocale - ? `app/en/${pathWithoutLocale}/page.mdx` - : "app/en/page.mdx"; -} - -/** - * Check if a dynamic route exists that could serve this URL path. - * e.g., for /resources/integrations/productivity/gmail, - * check if /resources/integrations/productivity/[toolkitId]/page.mdx exists - */ -function dynamicRouteExists(urlPath: string): boolean { - const pathWithoutLocale = urlPath.replace(LOCALE_PREFIX_REGEX, ""); - const segments = pathWithoutLocale.split("/").filter(Boolean); - - // Try replacing the last segment with common dynamic route patterns - const dynamicPatterns = ["[toolkitId]", "[slug]", "[id]", "[...slug]"]; - - for (let i = segments.length - 1; i >= 0; i--) { - for (const pattern of dynamicPatterns) { - const testSegments = [...segments]; - testSegments[i] = pattern; - const testPath = `app/en/${testSegments.join("/")}/page.mdx`; - if (existsSync(testPath)) { - return true; - } - const testPathMd = testPath.replace(MDX_EXTENSION_REGEX, ".md"); - if (existsSync(testPathMd)) { - return true; - } - } - } - - return false; -} - -/** - * Check if a page exists on disk - */ -function pageExists(urlPath: string): boolean { - if (urlPath.includes(":path*") || urlPath.includes(":path")) { - return true; - } - - const filePath = urlToFile(urlPath); - if (existsSync(filePath)) { - return true; - } - - const mdPath = filePath.replace(MDX_EXTENSION_REGEX, ".md"); - if (existsSync(mdPath)) { - return true; - } - - // Check if a dynamic route could serve this URL - if (dynamicRouteExists(urlPath)) { - return true; - } - - return false; -} - -/** - * Execute regex and collect all matches (avoids assignment in expression) - */ -function collectRegexMatches( - regex: RegExp, - content: string, - sourceIndex: number, - destIndex: number -): Array<{ source: string; destination: string }> { - const results: Array<{ source: string; destination: string }> = []; - regex.lastIndex = 0; - - let match = regex.exec(content); - while (match !== null) { - results.push({ - source: match[sourceIndex], - destination: match[destIndex], - }); - match = regex.exec(content); - } - - return results; -} - -/** - * Parse redirects from next.config.ts - */ -function parseRedirects(content: string): Redirect[] { - const results: Redirect[] = []; - - // Collect standard format: { source: "...", destination: "..." } - const standardMatches = collectRegexMatches(REDIRECT_REGEX, content, 1, 2); - for (const m of standardMatches) { - results.push(m); - } - - // Collect reversed format: { destination: "...", source: "..." } - const reversedMatches = collectRegexMatches( - REVERSED_REDIRECT_REGEX, - content, - 2, - 1 - ); - for (const m of reversedMatches) { - results.push(m); - } - - return results; -} - /** * Parse git diff output for deleted and renamed files */ @@ -227,77 +83,6 @@ function parseGitDiffOutput( } } -/** - * Convert a file path containing a dynamic route to a URL pattern. - * Replaces [param] with :param and [...param] with :param* - * e.g., app/en/resources/[toolkitId]/page.mdx -> /:locale/resources/:toolkitId - */ -function dynamicFileToUrlPattern(filePath: string): string { - const urlPath = filePath - .replace(APP_LOCALE_PREFIX_REGEX, "") - .replace(PAGE_FILE_SUFFIX_REGEX, ""); - - // Replace [...param] with :param* (catch-all routes) - // Replace [param] with :param (dynamic segments) - const patternPath = urlPath - .replace(/\[\.\.\.([^\]]+)\]/g, ":$1*") - .replace(/\[([^\]]+)\]/g, ":$1"); - - return patternPath ? `/:locale/${patternPath}` : "/:locale"; -} - -/** - * Parse git diff output for renamed dynamic route page files. - * Detects when a page.mdx inside a dynamic route folder is moved. - */ -function parseDynamicRouteMoves( - output: string, - moves: DynamicRouteMove[] -): void { - for (const line of output.split("\n")) { - if (!line) { - continue; - } - const parts = line.split("\t"); - const status = parts[0]; - - // Only look at renames (R followed by similarity percentage) - if (!status?.startsWith("R")) { - continue; - } - - const oldPath = parts[1]; - const newPath = parts[2]; - - if (!oldPath || !newPath) { - continue; - } - - // Check if either path contains a dynamic route segment - const oldHasDynamic = DYNAMIC_ROUTE_REGEX.test(oldPath); - const newHasDynamic = DYNAMIC_ROUTE_REGEX.test(newPath); - - // We care about moves where the URL pattern changes - if (!PAGE_FILE_MATCH_REGEX.test(oldPath)) { - continue; - } - - const oldUrl = dynamicFileToUrlPattern(oldPath); - const newUrl = dynamicFileToUrlPattern(newPath); - - // Skip if the URL pattern hasn't actually changed - if (oldUrl === newUrl) { - continue; - } - - // Record the move if either path has a dynamic route - // or if the directory structure changed significantly - if (oldHasDynamic || newHasDynamic) { - moves.push({ oldPath, newPath, oldUrl, newUrl }); - } - } -} - /** * Get moved dynamic routes by comparing branches */ @@ -312,7 +97,7 @@ function getMovedDynamicRoutes( const stagedChanges = execSync("git diff --cached --name-status", { encoding: "utf-8", }); - parseDynamicRouteMoves(stagedChanges, moves); + moves.push(...parseDynamicRouteMoves(stagedChanges)); } catch { // Ignore errors } @@ -324,7 +109,7 @@ function getMovedDynamicRoutes( `git diff --name-status ${branch}...HEAD`, { encoding: "utf-8" } ); - parseDynamicRouteMoves(committedChanges, moves); + moves.push(...parseDynamicRouteMoves(committedChanges)); } catch { // Ignore errors } @@ -333,7 +118,7 @@ function getMovedDynamicRoutes( const uncommittedChanges = execSync("git diff --name-status HEAD", { encoding: "utf-8", }); - parseDynamicRouteMoves(uncommittedChanges, moves); + moves.push(...parseDynamicRouteMoves(uncommittedChanges)); } catch { // Ignore errors } @@ -350,36 +135,6 @@ function getMovedDynamicRoutes( }); } -/** - * Check if a wildcard redirect already covers a dynamic route move - */ -function isMoveCoveredByRedirect( - move: DynamicRouteMove, - redirects: Redirect[] -): boolean { - // Check for exact match or wildcard that covers the path - for (const redirect of redirects) { - // Exact pattern match - if (redirect.source === move.oldUrl) { - return true; - } - - // Check if a wildcard redirect covers this path - if (redirect.source.includes(":path*")) { - const prefix = redirect.source - .replace(WILDCARD_PATH_REGEX, "") - .replace(LOCALE_PATH_PREFIX_REGEX, ""); - const movePrefix = move.oldUrl.replace(LOCALE_PATH_PREFIX_REGEX, ""); - - if (movePrefix.startsWith(`${prefix}/`) || movePrefix === prefix) { - return true; - } - } - } - - return false; -} - /** * Ensure base branch exists locally */ @@ -457,30 +212,6 @@ function getDeletedAndRenamedFiles( }; } -/** - * Check if a wildcard redirect covers a path - */ -function checkWildcardMatch(path: string, redirectList: Redirect[]): boolean { - const pathWithoutLocale = path.replace(LOCALE_PATH_PREFIX_REGEX, ""); - - for (const redirect of redirectList) { - if (redirect.source.includes(":path*")) { - const prefix = redirect.source - .replace(WILDCARD_PATH_REGEX, "") - .replace(LOCALE_PATH_PREFIX_REGEX, ""); - - if ( - pathWithoutLocale.startsWith(`${prefix}/`) || - pathWithoutLocale === prefix - ) { - return true; - } - } - } - - return false; -} - /** * Find the final destination in a redirect chain (follows all hops) * Returns null if the path doesn't redirect anywhere @@ -510,30 +241,32 @@ function findFinalRedirectDestination( } /** - * Insert redirect entries into next.config.ts + * Insert redirect entries into redirects.ts, just before the closing `];` of + * the `redirects` array (i.e. below the "Auto-added redirects" comment). + * This is the single append point for every auto-fix run, rather than the + * next.config.ts approach of inserting at the top of the array each time. */ function insertRedirects(entries: string[]): void { - const content = readFileSync(CONFIG_FILE, "utf-8"); + const content = readFileSync(REDIRECTS_FILE, "utf-8"); - const insertPoint = content.indexOf("return ["); + const insertPoint = content.lastIndexOf("\n];"); if (insertPoint === -1) { - console.error(colors.red("ERROR: Could not find 'return [' in config")); + console.error( + colors.red(`ERROR: Could not find closing '];' in ${REDIRECTS_FILE}`) + ); process.exit(1); } - const beforeReturn = content.substring( - 0, - insertPoint + RETURN_BRACKET_LENGTH - ); - const afterReturn = content.substring(insertPoint + RETURN_BRACKET_LENGTH); + const before = content.slice(0, insertPoint); + const after = content.slice(insertPoint); - const newContent = `${beforeReturn}\n // Auto-added redirects for deleted pages\n${entries.join("\n")}${afterReturn}`; + const newContent = `${before}\n${entries.join("\n")}${after}`; - writeFileSync(CONFIG_FILE, newContent); + writeFileSync(REDIRECTS_FILE, newContent); } /** - * Update a redirect destination in the config + * Update a redirect destination in redirects.ts */ function updateRedirectDestination( oldDest: string, @@ -555,8 +288,9 @@ console.log("Checking for deleted markdown files without redirects..."); console.log(`Comparing current branch to: ${baseBranch}`); console.log(""); -const configContent = readFileSync(CONFIG_FILE, "utf-8"); -const redirects = parseRedirects(configContent); +// Copy the imported entries into plain objects so PART 1b can update +// `destination` in place without mutating the imported module's array. +const redirects: Redirect[] = configuredRedirects.map((r) => ({ ...r })); let exitCode = 0; const invalidRedirects: string[] = []; @@ -565,7 +299,9 @@ const chains: RedirectChain[] = []; // ============================================================ // PART 1: Validate existing redirects // ============================================================ -console.log(colors.blue(`Validating existing redirects in ${CONFIG_FILE}...`)); +console.log( + colors.blue(`Validating existing redirects in ${REDIRECTS_FILE}...`) +); console.log(""); for (const redirect of redirects) { @@ -624,22 +360,33 @@ if (chains.length > 0) { ); console.log(""); - let updatedConfig = configContent; + let updatedRedirectsFile = readFileSync(REDIRECTS_FILE, "utf-8"); for (const chain of chains) { console.log(`${colors.green(" ✓")} ${chain.source}`); console.log(` was: ${chain.oldDest}`); console.log(` now: ${chain.newDest}`); - updatedConfig = updateRedirectDestination( + updatedRedirectsFile = updateRedirectDestination( chain.oldDest, chain.newDest, - updatedConfig + updatedRedirectsFile ); + + // Mirror the same (deliberately global, not source-scoped) replacement + // in memory so later parts see the collapsed destinations without + // re-reading the file. + for (const r of redirects) { + if (r.destination === chain.oldDest) { + r.destination = chain.newDest; + } + } } - writeFileSync(CONFIG_FILE, updatedConfig); + writeFileSync(REDIRECTS_FILE, updatedRedirectsFile); console.log(""); - console.log(colors.green(`✓ Redirect chains collapsed in ${CONFIG_FILE}`)); + console.log( + colors.green(`✓ Redirect chains collapsed in ${REDIRECTS_FILE}`) + ); console.log(""); } else { console.log( @@ -678,7 +425,9 @@ console.log(""); const missingRedirects: string[] = []; const suggestedEntries: string[] = []; -const latestRedirects = parseRedirects(readFileSync(CONFIG_FILE, "utf-8")); +// `redirects` already reflects PART 1b's chain collapses (see above), so it +// doubles as "the latest known state" without re-reading the file. +const latestRedirects = redirects; for (const file of allDeletedOrRenamed) { const urlPath = fileToUrl(file); @@ -701,11 +450,11 @@ for (const file of allDeletedOrRenamed) { console.log(colors.red(`✗ Missing redirect for: ${urlPath}`)); missingRedirects.push(urlPath); - suggestedEntries.push(` { - source: "${urlPath}", - destination: "/:locale/REPLACE_WITH_NEW_PATH", - permanent: true, - },`); + suggestedEntries.push(` { + source: "${urlPath}", + destination: "/:locale/REPLACE_WITH_NEW_PATH", + permanent: true, + },`); exitCode = 1; } @@ -725,7 +474,7 @@ if (missingRedirects.length > 0) { ); console.log( colors.blue( - `Auto-fixing: Adding ${missingRedirects.length} redirect(s) to ${CONFIG_FILE}` + `Auto-fixing: Adding ${missingRedirects.length} redirect(s) to ${REDIRECTS_FILE}` ) ); console.log( @@ -737,7 +486,7 @@ if (missingRedirects.length > 0) { insertRedirects(suggestedEntries); - console.log(colors.green(`✓ Added redirect entries to ${CONFIG_FILE}`)); + console.log(colors.green(`✓ Added redirect entries to ${REDIRECTS_FILE}`)); console.log(""); console.log( colors.red( @@ -764,7 +513,7 @@ if (missingRedirects.length > 0) { } console.log(""); console.log( - `Open ${CONFIG_FILE} and search for 'REPLACE_WITH_NEW_PATH' to find them.` + `Open ${REDIRECTS_FILE} and search for 'REPLACE_WITH_NEW_PATH' to find them.` ); console.log(""); @@ -787,7 +536,7 @@ if (missingRedirects.length > 0) { ); console.log(""); console.log( - "When you delete a markdown file, you must add a redirect in next.config.ts" + "When you delete a markdown file, you must add a redirect in redirects.ts" ); console.log( "to prevent broken links for users who have bookmarked the old URL." @@ -799,9 +548,7 @@ if (missingRedirects.length > 0) { } console.log(""); console.log( - colors.yellow( - "Add the following to the redirects array in next.config.ts:" - ) + colors.yellow("Add the following to the redirects array in redirects.ts:") ); console.log(""); for (const entry of suggestedEntries) { @@ -830,7 +577,7 @@ if (invalidRedirects.length > 0) { } console.log(""); console.log(colors.yellow("How to fix:")); - console.log(" 1. Open next.config.ts"); + console.log(" 1. Open redirects.ts"); console.log(" 2. Find the redirect(s) listed above"); console.log(" 3. Update the destination to a valid page path"); console.log(" (Check that the path exists under app/en/)"); @@ -871,11 +618,11 @@ if (uncoveredMoves.length > 0) { console.log(colors.blue(` → ${move.newPath}`)); console.log(""); console.log(colors.yellow(" Suggested redirect:")); - console.log(` { - source: "${move.oldUrl}/:path*", - destination: "${move.newUrl}/:path*", - permanent: true, - },`); + console.log(` { + source: "${move.oldUrl}/:path*", + destination: "${move.newUrl}/:path*", + permanent: true, + },`); console.log(""); } diff --git a/scripts/generate-llmstxt.ts b/scripts/generate-llmstxt.ts index a515a30d4..8f14c3088 100644 --- a/scripts/generate-llmstxt.ts +++ b/scripts/generate-llmstxt.ts @@ -5,6 +5,11 @@ import chalk from "chalk"; import glob from "fast-glob"; import OpenAI from "openai"; import { getToolkitCanonicalPath } from "../app/_lib/toolkit-static-params"; +import { resolveToolkitDataDir } from "../toolkit-docs-generator/src/shared/toolkit-data-dir"; +import type { + MergedToolkit, + MergedToolkitMetadata, +} from "../toolkit-docs-generator/src/shared/toolkit-schemas"; type PageMetadata = { path: string; @@ -202,29 +207,28 @@ async function discoverMdxPages(): Promise { return pages; } -const TOOLKIT_DATA_DIR = path.join( - process.cwd(), - "toolkit-docs-generator", - "data", - "toolkits" -); +const TOOLKIT_DATA_DIR = resolveToolkitDataDir(); const MAX_TOOLKIT_DESCRIPTION = 280; const MARKDOWN_LINK_REGEX = /\[([^\]]+)\]\([^)]+\)/g; const MARKDOWN_NOISE_REGEX = /[#*`>]/g; const WHITESPACE_REGEX = /\s+/g; -type ToolkitData = { - id?: string; - label?: string; - description?: string; - summary?: string; +/** + * This script only reads a handful of fields off each toolkit JSON file (it + * doesn't validate the whole document), so it declares the subset it needs + * as a `Pick` off the real generator schema types rather than re-describing + * the shape by hand. + */ +type ToolkitData = Partial< + Pick +> & { tools?: unknown[]; - metadata?: { - category?: string; - docsLink?: string; - isHidden?: boolean; - isComingSoon?: boolean; - }; + metadata?: Partial< + Pick< + MergedToolkitMetadata, + "category" | "docsLink" | "isHidden" | "isComingSoon" + > + >; }; /** diff --git a/toolkit-docs-generator/scripts/check-redirects-utils.ts b/scripts/lib/check-redirects-utils.ts similarity index 76% rename from toolkit-docs-generator/scripts/check-redirects-utils.ts rename to scripts/lib/check-redirects-utils.ts index 0259cbb95..48133192d 100644 --- a/toolkit-docs-generator/scripts/check-redirects-utils.ts +++ b/scripts/lib/check-redirects-utils.ts @@ -1,30 +1,32 @@ /** - * Utility functions for check-redirects.ts - * Extracted for testability. + * Shared helpers for scripts/check-redirects.ts. + * + * Split out so the logic that maps file paths to URLs, checks whether a + * page still exists on disk, and matches redirects against moved files can + * be unit tested without shelling out to git or touching the real + * filesystem. */ import { existsSync } from "node:fs"; -// Regex patterns -export const APP_LOCALE_PREFIX_REGEX = /^app\/[a-z]{2}\//; -export const PAGE_FILE_SUFFIX_REGEX = /\/?page\.mdx?$/; -export const LOCALE_PREFIX_REGEX = /^\/:locale\/?/; -export const PAGE_FILE_MATCH_REGEX = /page\.mdx?$/; -export const LOCALE_PATH_PREFIX_REGEX = /^\/:locale\//; -export const WILDCARD_PATH_REGEX = /\/:path\*.*$/; -export const MDX_EXTENSION_REGEX = /\.mdx$/; -export const DYNAMIC_ROUTE_REGEX = /\[[^\]]+\]/; -export const REDIRECT_REGEX = - /\{\s*source:\s*["']([^"']+)["']\s*,\s*destination:\s*["']([^"']+)["']/g; -export const REVERSED_REDIRECT_REGEX = - /\{\s*destination:\s*["']([^"']+)["']\s*,\s*source:\s*["']([^"']+)["']/g; - +// `permanent` is optional here (unlike redirects.ts's `Redirect`, where it's +// required) because none of these helpers read it — they only match on +// `source`/`destination`. A `redirects.ts` entry satisfies this type as-is. export type Redirect = { source: string; destination: string; permanent?: boolean; }; +const APP_LOCALE_PREFIX_REGEX = /^app\/[a-z]{2}\//; +const PAGE_FILE_SUFFIX_REGEX = /\/?page\.mdx?$/; +const LOCALE_PREFIX_REGEX = /^\/:locale\/?/; +const PAGE_FILE_MATCH_REGEX = /page\.mdx?$/; +const LOCALE_PATH_PREFIX_REGEX = /^\/:locale\//; +const WILDCARD_PATH_REGEX = /\/:path\*.*$/; +const MDX_EXTENSION_REGEX = /\.mdx$/; +const DYNAMIC_ROUTE_REGEX = /\[[^\]]+\]/; + export type DynamicRouteMove = { oldPath: string; newPath: string; @@ -200,10 +202,10 @@ export function parseDynamicRouteMoves(output: string): DynamicRouteMove[] { */ export function isMoveCoveredByRedirect( move: DynamicRouteMove, - redirects: Redirect[] + redirectList: Redirect[] ): boolean { // Check for exact match or wildcard that covers the path - for (const redirect of redirects) { + for (const redirect of redirectList) { // Exact pattern match if (redirect.source === move.oldUrl) { return true; @@ -225,56 +227,6 @@ export function isMoveCoveredByRedirect( return false; } -/** - * Execute regex and collect all matches - */ -export function collectRegexMatches( - regex: RegExp, - content: string, - sourceIndex: number, - destIndex: number -): Array<{ source: string; destination: string }> { - const results: Array<{ source: string; destination: string }> = []; - regex.lastIndex = 0; - - let match = regex.exec(content); - while (match !== null) { - results.push({ - source: match[sourceIndex], - destination: match[destIndex], - }); - match = regex.exec(content); - } - - return results; -} - -/** - * Parse redirects from next.config.ts content - */ -export function parseRedirects(content: string): Redirect[] { - const results: Redirect[] = []; - - // Collect standard format: { source: "...", destination: "..." } - const standardMatches = collectRegexMatches(REDIRECT_REGEX, content, 1, 2); - for (const m of standardMatches) { - results.push(m); - } - - // Collect reversed format: { destination: "...", source: "..." } - const reversedMatches = collectRegexMatches( - REVERSED_REDIRECT_REGEX, - content, - 2, - 1 - ); - for (const m of reversedMatches) { - results.push(m); - } - - return results; -} - /** * Check if a wildcard redirect covers a path */ diff --git a/scripts/update-internal-links.ts b/scripts/update-internal-links.ts index da10d93c9..e960a2702 100644 --- a/scripts/update-internal-links.ts +++ b/scripts/update-internal-links.ts @@ -6,12 +6,13 @@ * Usage: * pnpm update-links [--dry-run] * - * This script reads redirects from next.config.ts and updates any internal links + * This script reads redirects from redirects.ts and updates any internal links * in MDX/TSX files that point to redirected paths. */ import { readFileSync, writeFileSync } from "node:fs"; import fg from "fast-glob"; +import { redirects as configuredRedirects } from "../redirects"; // Colors for terminal output const colors = { @@ -24,71 +25,15 @@ const colors = { // Parse command line arguments const dryRun = process.argv.includes("--dry-run"); -const CONFIG_FILE = "next.config.ts"; - // Top-level regex patterns const LOCALE_PREFIX_REGEX = /^\/:locale/; const SPECIAL_REGEX_CHARS_REGEX = /[.*+?^${}()|[\]\\]/g; -const REDIRECT_REGEX = - /\{\s*source:\s*["']([^"']+)["']\s*,\s*destination:\s*["']([^"']+)["']/g; -const REVERSED_REDIRECT_REGEX = - /\{\s*destination:\s*["']([^"']+)["']\s*,\s*source:\s*["']([^"']+)["']/g; type Redirect = { source: string; destination: string; }; -/** - * Execute regex and collect all matches (avoids assignment in expression) - */ -function collectRegexMatches( - regex: RegExp, - content: string, - sourceIndex: number, - destIndex: number -): Array<{ source: string; destination: string }> { - const results: Array<{ source: string; destination: string }> = []; - regex.lastIndex = 0; - - let match = regex.exec(content); - while (match !== null) { - results.push({ - source: match[sourceIndex], - destination: match[destIndex], - }); - match = regex.exec(content); - } - - return results; -} - -/** - * Parse redirects from next.config.ts - */ -function parseRedirects(content: string): Redirect[] { - const results: Redirect[] = []; - - // Collect standard format: { source: "...", destination: "..." } - const standardMatches = collectRegexMatches(REDIRECT_REGEX, content, 1, 2); - for (const m of standardMatches) { - results.push(m); - } - - // Collect reversed format: { destination: "...", source: "..." } - const reversedMatches = collectRegexMatches( - REVERSED_REDIRECT_REGEX, - content, - 2, - 1 - ); - for (const m of reversedMatches) { - results.push(m); - } - - return results; -} - /** * Filter redirects to only those that can be auto-updated */ @@ -199,11 +144,9 @@ if (dryRun) { console.log(""); } -console.log(colors.blue(`Parsing redirects from ${CONFIG_FILE}...`)); +console.log(colors.blue("Parsing redirects from redirects.ts...")); -const configContent = readFileSync(CONFIG_FILE, "utf-8"); -const allRedirects = parseRedirects(configContent); -const redirects = getUpdatableRedirects(allRedirects); +const redirects = getUpdatableRedirects(configuredRedirects); console.log( `Found ${colors.green(String(redirects.length))} non-wildcard redirects to check` diff --git a/tests/integration-category-routes.test.ts b/tests/integration-category-routes.test.ts new file mode 100644 index 000000000..13a77434f --- /dev/null +++ b/tests/integration-category-routes.test.ts @@ -0,0 +1,58 @@ +import { existsSync, mkdirSync, mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { describe, expect, test } from "vitest"; +import { INTEGRATION_CATEGORIES } from "@/toolkit-docs-generator/src/shared/toolkit-primitives"; + +const INTEGRATIONS_APP_DIR = join( + process.cwd(), + "app", + "en", + "resources", + "integrations" +); + +/** + * normalizeCategory (app/_lib/toolkit-static-params.ts) trusts that every + * value in INTEGRATION_CATEGORIES has a real `[toolkitId]` route directory + * to route toolkits into. If a category is ever added to that list without + * the matching directory (or a directory is removed/renamed), toolkits in + * that category become clickable catalog cards pointing at a route that + * 404s — the same class of bug the "others" catch-all used to hide, since + * tests/integration-index-links.test.ts derives its notion of "valid link" + * from the same normalizeCategory output and can't see this gap. + */ +const missingCategoryDirs = (baseDir: string): string[] => + INTEGRATION_CATEGORIES.filter( + (category) => !existsSync(join(baseDir, category, "[toolkitId]")) + ); + +describe("integration category route directories", () => { + test("every INTEGRATION_CATEGORIES value has a matching [toolkitId] route directory", () => { + expect(missingCategoryDirs(INTEGRATIONS_APP_DIR)).toEqual([]); + }); + + test("the check fails when a category's route directory is missing", () => { + // Proves the check above actually catches drift, without touching any + // tracked directory: build a scratch tree with every category present, + // then remove one and confirm it's flagged. + const scratchDir = mkdtempSync(join(tmpdir(), "integration-categories-")); + try { + for (const category of INTEGRATION_CATEGORIES) { + mkdirSync(join(scratchDir, category, "[toolkitId]"), { + recursive: true, + }); + } + + const removedCategory = INTEGRATION_CATEGORIES[0]; + rmSync(join(scratchDir, removedCategory, "[toolkitId]"), { + recursive: true, + force: true, + }); + + expect(missingCategoryDirs(scratchDir)).toEqual([removedCategory]); + } finally { + rmSync(scratchDir, { recursive: true, force: true }); + } + }); +}); diff --git a/tests/integration-index-links.test.ts b/tests/integration-index-links.test.ts index de50d8f9d..45775c334 100644 --- a/tests/integration-index-links.test.ts +++ b/tests/integration-index-links.test.ts @@ -9,16 +9,17 @@ import { toIntegrationLink, } from "@/app/_lib/integration-index"; import { readToolkitData } from "@/app/_lib/toolkit-data"; -import { - getToolkitSlug, - type ToolkitWithDocsLink, -} from "@/app/_lib/toolkit-slug"; +import type { ToolkitWithDocsLink } from "@/app/_lib/toolkit-slug"; import { getToolkitCanonicalPath, - INTEGRATION_CATEGORIES, listToolkitRoutes, listValidIntegrationLinks, } from "@/app/_lib/toolkit-static-params"; +import { + getToolkitSlug, + INTEGRATION_CATEGORIES, +} from "@/toolkit-docs-generator/src/shared/toolkit-primitives"; +import { redirects } from "../redirects"; const TIMEOUT = 30_000; const ROOT = process.cwd(); @@ -198,13 +199,8 @@ const pageFileExists = (path: string): boolean => { ); }; -const readRedirectSources = async (): Promise> => { - const config = await readFile(join(ROOT, "next.config.ts"), "utf-8"); - const sources = [...config.matchAll(/source:\s*"([^"]+)"/g)].map( - (match) => match[1] - ); - return new Set(sources); -}; +const readRedirectSources = (): Set => + new Set(redirects.map((redirect) => redirect.source)); const extractInternalHrefs = async (relPath: string): Promise => { const content = await readFile(join(ROOT, relPath), "utf-8"); diff --git a/toolkit-docs-generator/tests/scripts/check-redirects-utils.test.ts b/tests/scripts/check-redirects-utils.test.ts similarity index 86% rename from toolkit-docs-generator/tests/scripts/check-redirects-utils.test.ts rename to tests/scripts/check-redirects-utils.test.ts index 47c0018e7..405d8caee 100644 --- a/toolkit-docs-generator/tests/scripts/check-redirects-utils.test.ts +++ b/tests/scripts/check-redirects-utils.test.ts @@ -8,10 +8,9 @@ import { isMoveCoveredByRedirect, pageExists, parseDynamicRouteMoves, - parseRedirects, type Redirect, urlToFile, -} from "../../scripts/check-redirects-utils"; +} from "../../scripts/lib/check-redirects-utils"; describe("fileToUrl", () => { it("converts file path to URL path", () => { @@ -313,79 +312,6 @@ describe("isMoveCoveredByRedirect", () => { }); }); -describe("parseRedirects", () => { - it("parses standard format redirects", () => { - const content = ` - { - source: "/:locale/old", - destination: "/:locale/new", - permanent: true, - }, - `; - - const redirects = parseRedirects(content); - - expect(redirects).toHaveLength(1); - expect(redirects[0]).toEqual({ - source: "/:locale/old", - destination: "/:locale/new", - }); - }); - - it("parses reversed format redirects", () => { - const content = ` - { - destination: "/:locale/new", - source: "/:locale/old", - permanent: true, - }, - `; - - const redirects = parseRedirects(content); - - expect(redirects).toHaveLength(1); - expect(redirects[0]).toEqual({ - source: "/:locale/old", - destination: "/:locale/new", - }); - }); - - it("parses multiple redirects", () => { - const content = ` - { - source: "/:locale/a", - destination: "/:locale/b", - }, - { - source: "/:locale/c", - destination: "/:locale/d", - }, - `; - - const redirects = parseRedirects(content); - - expect(redirects).toHaveLength(2); - }); - - it("handles single quotes", () => { - const content = ` - { - source: '/:locale/old', - destination: '/:locale/new', - }, - `; - - const redirects = parseRedirects(content); - - expect(redirects).toHaveLength(1); - expect(redirects[0].source).toBe("/:locale/old"); - }); - - it("handles empty content", () => { - expect(parseRedirects("")).toHaveLength(0); - }); -}); - describe("checkWildcardMatch", () => { it("returns true when wildcard prefix matches", () => { const redirects: Redirect[] = [ diff --git a/tests/sitemap.test.ts b/tests/sitemap.test.ts index d8daf65cc..ded32cc7b 100644 --- a/tests/sitemap.test.ts +++ b/tests/sitemap.test.ts @@ -1,6 +1,7 @@ import { readFileSync } from "node:fs"; import { join } from "node:path"; import { expect, test } from "vitest"; +import { redirects } from "../redirects"; test("sitemap lists expected URLs", async () => { const previousSiteUrl = process.env.SITE_URL; @@ -55,16 +56,15 @@ test("sitemap contains no URL that we redirect away", async () => { entry.url.replace("https://example.test", "") ); - // Every redirect `source` in next.config.ts is a path we 3xx away, so a live + // Every redirect `source` in redirects.ts is a path we 3xx away, so a live // page must never sit there — otherwise the sitemap ships a redirecting URL // (Ahrefs flags "3XX redirect in sitemap"). Guards against pages left behind // after a rename. - const config = readFileSync(join(process.cwd(), "next.config.ts"), "utf-8"); const exactSources = new Set(); const prefixSources: string[] = []; - for (const match of config.matchAll(/source:\s*"([^"]+)"/g)) { - const normalized = match[1] + for (const redirect of redirects) { + const normalized = redirect.source .replace(/:locale\([^)]*\)/g, "en") .replace(/:locale/g, "en"); @@ -90,7 +90,7 @@ test("sitemap contains no URL that we redirect away", async () => { for (const offender of offenders) { console.error( - `Sitemap includes ${offender}, which matches a redirect source in next.config.ts. ` + + `Sitemap includes ${offender}, which matches a redirect source in redirects.ts. ` + "Delete the stale page (or remove the redirect) so the sitemap ships no 3XX URLs." ); } diff --git a/tests/toolkit-data-cache.test.ts b/tests/toolkit-data-cache.test.ts new file mode 100644 index 000000000..45d9ac618 --- /dev/null +++ b/tests/toolkit-data-cache.test.ts @@ -0,0 +1,118 @@ +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterAll, describe, expect, test } from "vitest"; +import { readToolkitData } from "@/app/_lib/toolkit-data"; + +/** + * loadAllToolkitData (app/_lib/toolkit-data.ts) reads and validates every + * toolkit file in a data directory once, then serves all lookups from the + * resulting map. That eager read means one corrupt file can no longer be + * skipped by requesting a different, healthy toolkit — the whole directory + * load fails, and every lookup against it throws. These tests pin that + * behavior down explicitly, since it's a real change from the old + * direct-file-then-scan implementation (a corrupt sibling file was + * previously invisible to a direct hit). + */ + +const validToolkitJson = (id: string, docsSlug: string): string => + JSON.stringify({ + id, + label: id, + version: "1.0.0", + description: "A test toolkit fixture.", + metadata: { + category: "development", + iconUrl: "https://example.com/icon.svg", + isBYOC: false, + isPro: false, + type: "arcade", + docsLink: `https://docs.arcade.dev/en/resources/integrations/development/${docsSlug}`, + isComingSoon: false, + isHidden: false, + }, + auth: null, + tools: [], + }); + +const makeFixtureDir = (): string => { + const dir = mkdtempSync(join(tmpdir(), "toolkit-data-cache-test-")); + writeFileSync( + join(dir, "validtoolkitone.json"), + validToolkitJson("ValidToolkitOne", "valid-toolkit-one") + ); + writeFileSync( + join(dir, "validtoolkittwo.json"), + validToolkitJson("ValidToolkitTwo", "valid-toolkit-two") + ); + return dir; +}; + +const dirsToClean: string[] = []; + +afterAll(() => { + for (const dir of dirsToClean) { + rmSync(dir, { recursive: true, force: true }); + } +}); + +describe("readToolkitData against a clean fixture directory", () => { + const dataDir = makeFixtureDir(); + dirsToClean.push(dataDir); + + test("a known toolkit id resolves to its data", async () => { + const data = await readToolkitData("ValidToolkitOne", { dataDir }); + expect(data?.id).toBe("ValidToolkitOne"); + }); + + test("a known toolkit reached by its docs slug resolves to the same data", async () => { + const data = await readToolkitData("valid-toolkit-two", { dataDir }); + expect(data?.id).toBe("ValidToolkitTwo"); + }); + + test("an absent toolkit id yields null, not a throw", async () => { + const data = await readToolkitData("no-such-toolkit-at-all", { dataDir }); + expect(data).toBeNull(); + }); +}); + +describe("readToolkitData against a directory with one corrupt file", () => { + const dataDir = makeFixtureDir(); + dirsToClean.push(dataDir); + writeFileSync( + join(dataDir, "corrupttoolkit.json"), + "{ this is not valid json" + ); + + test("requesting the corrupt toolkit throws, naming the file path", async () => { + await expect( + readToolkitData("CorruptToolkit", { dataDir }) + ).rejects.toThrow(join(dataDir, "corrupttoolkit.json")); + }); + + test("the failure is cached, not retried: a second request throws the same way", async () => { + // Confirms the deliberate choice to cache a failed load rather than + // re-scanning the directory on every subsequent call: this directory's + // corruption doesn't heal between calls, so re-reading it every time + // would only add cost without ever succeeding. + await expect( + readToolkitData("CorruptToolkit", { dataDir }) + ).rejects.toThrow(join(dataDir, "corrupttoolkit.json")); + }); + + // A pre-existing property of the old scan-on-miss implementation too, not + // a regression introduced by the shared cache: any lookup that needs to + // rule out every file in the directory (a genuinely absent id, or a slug + // reached only via the full scan) surfaces a sibling file's corruption, + // because "is this id absent" can't be answered without reading everything. + // A healthy toolkit's *direct* id-shaped lookup, though, is unaffected by + // corruption elsewhere in the directory only when that toolkit was already + // resident in a load that happened before the corruption — once the whole + // directory's load has failed once, it stays failed (see the caching test + // above), so every subsequent lookup against this dataDir throws too. + test("a healthy toolkit id in the same directory also throws once the directory load has failed", async () => { + await expect( + readToolkitData("ValidToolkitOne", { dataDir }) + ).rejects.toThrow(join(dataDir, "corrupttoolkit.json")); + }); +}); diff --git a/tests/toolkit-data-parity.test.ts b/tests/toolkit-data-parity.test.ts new file mode 100644 index 000000000..ef4d5cbb2 --- /dev/null +++ b/tests/toolkit-data-parity.test.ts @@ -0,0 +1,58 @@ +import { readdirSync } from "node:fs"; +import { join } from "node:path"; +import { describe, expect, test } from "vitest"; +import { readToolkitFile, readToolkitIndex } from "@/app/_lib/toolkit-data"; +import { listToolkitRoutes } from "@/app/_lib/toolkit-static-params"; +import { resolveToolkitDataDir } from "@/toolkit-docs-generator/src/shared/toolkit-data-dir"; + +// resolveToolkitDataDir defaults to the real committed data, but also honors +// TOOLKIT_DATA_DIR (same as readToolkitIndex/listToolkitRoutes below), so +// pointing that env var at a scratch copy runs this exact test against it. +const DATA_DIR = resolveToolkitDataDir(); + +/** + * A malformed or missing nightly-generated toolkit file used to disappear + * from the site silently: readToolkitData/listToolkitRoutes swallowed the + * error and just dropped the toolkit, so index.json, the on-disk files, and + * the routes Next.js actually generates could drift apart with nothing + * failing the build. Runs against the real committed data (not a fixture) + * so it catches that drift for whatever toolkits are checked in right now. + */ +describe("toolkit data parity", () => { + test("index.json entries, parseable toolkit files, and generated routes agree", async () => { + const index = await readToolkitIndex(); + expect(index).not.toBeNull(); + + const jsonFileNames = readdirSync(DATA_DIR).filter( + (file) => file.endsWith(".json") && file !== "index.json" + ); + + // readToolkitFile throws on a corrupt file (see app/_lib/toolkit-data.ts), + // so a bad file fails this test loudly instead of quietly shrinking the + // "parseable" count below. + const toolkits = await Promise.all( + jsonFileNames.map((file) => readToolkitFile(join(DATA_DIR, file))) + ); + const parseableCount = toolkits.filter( + (toolkit) => toolkit !== null + ).length; + + // Every file on disk should be a real, schema-valid toolkit: no file + // silently failed to parse into null. + expect(parseableCount).toBe(jsonFileNames.length); + + // index.json is regenerated alongside the per-toolkit files, so its + // entry count should match the file count exactly. + expect(index?.toolkits.length).toBe(parseableCount); + + // Routes exclude hidden toolkits (they're intentionally unrouted, not + // corrupt), so compare against the non-hidden subset rather than the + // raw file count. + const visibleCount = toolkits.filter( + (toolkit) => toolkit && !toolkit.metadata?.isHidden + ).length; + + const routes = await listToolkitRoutes(); + expect(routes.length).toBe(visibleCount); + }); +}); diff --git a/toolkit-docs-generator/scripts/check-stale-summaries.ts b/toolkit-docs-generator/scripts/check-stale-summaries.ts index da093402d..73930978e 100644 --- a/toolkit-docs-generator/scripts/check-stale-summaries.ts +++ b/toolkit-docs-generator/scripts/check-stale-summaries.ts @@ -13,11 +13,10 @@ */ import { readdirSync, readFileSync } from "node:fs"; -import { dirname, join } from "node:path"; -import { fileURLToPath } from "node:url"; +import { join } from "node:path"; +import { resolveToolkitDataDir } from "../src/shared/toolkit-data-dir.ts"; -const here = dirname(fileURLToPath(import.meta.url)); -const TOOLKITS_DIR = join(here, "..", "data", "toolkits"); +const TOOLKITS_DIR = resolveToolkitDataDir(); type ToolkitShape = { id?: unknown; diff --git a/toolkit-docs-generator/scripts/report-tool-metadata.ts b/toolkit-docs-generator/scripts/report-tool-metadata.ts index 51624b8e0..428317a2a 100644 --- a/toolkit-docs-generator/scripts/report-tool-metadata.ts +++ b/toolkit-docs-generator/scripts/report-tool-metadata.ts @@ -1,16 +1,14 @@ #!/usr/bin/env node /** * CLI script to report tool metadata coverage and distinct enum values. - * Resolves data directory relative to this script, so it works regardless of cwd. + * Resolves the data directory via resolveToolkitDataDir, which works + * regardless of cwd and honors the TOOLKIT_DATA_DIR env var override. */ -import { dirname, join } from "node:path"; -import { fileURLToPath } from "node:url"; - +import { resolveToolkitDataDir } from "../src/shared/toolkit-data-dir.ts"; import { collectToolMetadataStats } from "../src/utils/tool-metadata-audit.ts"; -const __dirname = dirname(fileURLToPath(import.meta.url)); -const DATA_DIR = join(__dirname, "..", "data", "toolkits"); +const DATA_DIR = resolveToolkitDataDir(); async function main(): Promise { const stats = await collectToolMetadataStats({ dataDir: DATA_DIR }); diff --git a/toolkit-docs-generator/scripts/sync-toolkit-sidebar.ts b/toolkit-docs-generator/scripts/sync-toolkit-sidebar.ts index 01927bec6..a42ed0a25 100644 --- a/toolkit-docs-generator/scripts/sync-toolkit-sidebar.ts +++ b/toolkit-docs-generator/scripts/sync-toolkit-sidebar.ts @@ -28,6 +28,16 @@ import { import { dirname, join, resolve } from "node:path"; import { fileURLToPath } from "node:url"; import { TOOLKITS as DESIGN_SYSTEM_TOOLKITS } from "@arcadeai/design-system/metadata/toolkits"; +import { resolveToolkitDataDir } from "../src/shared/toolkit-data-dir.ts"; +import { + getToolkitSlug, + INTEGRATION_CATEGORIES, + isApiSuffixedToolkitId, +} from "../src/shared/toolkit-primitives.ts"; +import type { + MergedToolkit, + MergedToolkitMetadata, +} from "../src/shared/toolkit-schemas.ts"; const __filename = fileURLToPath(import.meta.url); const __dirname = dirname(__filename); @@ -51,7 +61,7 @@ const PROJECT_ROOT = resolve(__dirname, "..", ".."); // Configuration const CONFIG = { - dataDir: join(PROJECT_ROOT, "toolkit-docs-generator/data/toolkits"), + dataDir: resolveToolkitDataDir(), integrationsDir: join(PROJECT_ROOT, "app/en/resources/integrations"), integrationsBasePath: "/en/resources/integrations", }; @@ -71,43 +81,24 @@ const CATEGORY_NAMES: Record = { }; // Category order for main _meta.tsx -const CATEGORY_ORDER = [ - "productivity", - "social", - "entertainment", - "development", - "payments", - "search", - "sales", - "databases", - "customer-support", - "others", -]; +const CATEGORY_ORDER: readonly string[] = INTEGRATION_CATEGORIES; const CAPITAL_LETTER_REGEX = /([A-Z])/g; const FIRST_CHARACTER_REGEX = /^./; -const CAMEL_BOUNDARY = /([a-z0-9])([A-Z])/g; const IDENTIFIER_KEY_REGEX = /^[A-Za-z_$][A-Za-z0-9_$]*$/; /** - * Convert a CamelCase string to kebab-case. - * Must stay in sync with toKebabCase in app/_lib/toolkit-slug.ts. + * This script only reads a handful of fields off each toolkit JSON file (it + * doesn't validate the whole document), so it declares the subset it needs + * as a `Pick` off the real generator schema types rather than re-describing + * the shape by hand. */ -function toKebabCase(value: string): string { - return value.replace(CAMEL_BOUNDARY, "$1-$2").toLowerCase(); -} - -type ToolkitJson = { - id?: string; - label?: string; +type ToolkitJson = Partial> & { name?: string; - metadata?: { - category?: string; - docsLink?: string; - isHidden?: boolean; - type?: string; - }; + metadata?: Partial< + Pick + >; }; function renderObjectKey(key: string): string { @@ -231,21 +222,6 @@ function readToolkitJson(dataDir: string, slug: string): ToolkitJson | null { return null; } -function getDocsSlugFromLink(docsLink?: string | null): string | null { - if (!docsLink) { - return null; - } - - try { - const url = new URL(docsLink); - const segments = url.pathname.split("/").filter(Boolean); - return segments.at(-1) ?? null; - } catch { - const segments = docsLink.split("/").filter(Boolean); - return segments.at(-1) ?? null; - } -} - /** * Read toolkit JSON and extract label if available */ @@ -275,7 +251,7 @@ export function inferNavGroup( } // Heuristic fallback: "*Api" toolkits are starter. - return toolkitIdOrSlug.toLowerCase().endsWith("api") + return isApiSuffixedToolkitId(toolkitIdOrSlug) ? ("starter" as const) : ("optimized" as const); } @@ -294,8 +270,10 @@ function resolveToolkitInfo( ): ToolkitInfoEntry | null { const jsonData = readToolkitJson(dataDir, slug); const toolkitId = jsonData?.id ?? slug; - const docsSlug = - getDocsSlugFromLink(jsonData?.metadata?.docsLink) ?? toKebabCase(toolkitId); + const docsSlug = getToolkitSlug({ + id: toolkitId, + docsLink: jsonData?.metadata?.docsLink, + }); const designSystemToolkit = TOOLKITS.find( (t) => t.id.toLowerCase() === toolkitId.toLowerCase() ); diff --git a/toolkit-docs-generator/scripts/validate-merge.ts b/toolkit-docs-generator/scripts/validate-merge.ts index b51e79bfd..89d4ed2ed 100644 --- a/toolkit-docs-generator/scripts/validate-merge.ts +++ b/toolkit-docs-generator/scripts/validate-merge.ts @@ -11,22 +11,23 @@ */ import { existsSync, readdirSync, readFileSync } from "node:fs"; -import { dirname, join, resolve } from "node:path"; +import { join, resolve } from "node:path"; import { fileURLToPath } from "node:url"; +import { resolveToolkitDataDir } from "../src/shared/toolkit-data-dir.ts"; +import type { MergedToolkit } from "../src/shared/toolkit-schemas.ts"; -const __filename = fileURLToPath(import.meta.url); -const __dirname = dirname(__filename); +const DATA_DIR = resolveToolkitDataDir(); -const WORKSPACE_ROOT = join(__dirname, ".."); -const DATA_DIR = join(WORKSPACE_ROOT, "data", "toolkits"); - -type ToolkitJson = { - id: string; - label: string; - documentationChunks?: Record[]; - customImports?: string[]; - subPages?: Record[]; -}; +/** + * This script only reads a handful of fields off each toolkit JSON file (it + * doesn't validate the whole document), so it declares the subset it needs + * as a `Pick` off the real generator schema type rather than re-describing + * the shape by hand. + */ +type ToolkitJson = Pick< + MergedToolkit, + "id" | "label" | "documentationChunks" | "customImports" | "subPages" +>; export type ToolkitValidationDetail = { file: string; diff --git a/toolkit-docs-generator/src/merger/data-merger.ts b/toolkit-docs-generator/src/merger/data-merger.ts index d278b18ca..15d28def8 100644 --- a/toolkit-docs-generator/src/merger/data-merger.ts +++ b/toolkit-docs-generator/src/merger/data-merger.ts @@ -6,6 +6,10 @@ */ import type { ISecretEditGenerator } from "../llm/secret-edit-generator.js"; +import { + isApiSuffixedToolkitId, + normalizeToolkitId, +} from "../shared/toolkit-primitives.js"; import type { ICustomSectionsSource } from "../sources/interfaces.js"; import type { IToolkitDataSource, @@ -366,14 +370,10 @@ export const getProviderId = ( /** * Create default metadata for toolkits not found in Design System */ -const TOOLKIT_ID_NORMALIZER = /[^a-z0-9]/g; const TOOLKIT_ID_ACRONYM_BOUNDARY = /([A-Z]+)([A-Z][a-z])/g; const TOOLKIT_ID_WORD_BOUNDARY = /([a-z0-9])([A-Z])/g; const TOOLKIT_DESCRIPTION_LABEL_PREFIX = "Arcade.dev LLM tools for "; -const normalizeToolkitId = (toolkitId: string): string => - toolkitId.toLowerCase().replace(TOOLKIT_ID_NORMALIZER, ""); - const humanizeToolkitId = (toolkitId: string): string => toolkitId .replace(TOOLKIT_ID_ACRONYM_BOUNDARY, "$1 $2") @@ -415,9 +415,6 @@ const resolveToolkitLabel = (options: { extractLabelFromDescription(options.description) ?? humanizeToolkitId(options.toolkitId); -const isStarterToolkitId = (toolkitId: string): boolean => - normalizeToolkitId(toolkitId).endsWith("api"); - const getDefaultIconId = (toolkitId: string): string => { const normalized = normalizeToolkitId(toolkitId); // Prefer provider icons for "*Api" toolkits when possible. @@ -436,7 +433,7 @@ const applyToolkitTypeOverrides = ( toolkitId: string, metadata: MergedToolkitMetadata ): MergedToolkitMetadata => { - if (isStarterToolkitId(toolkitId) && metadata.type === "arcade") { + if (isApiSuffixedToolkitId(toolkitId) && metadata.type === "arcade") { return { ...metadata, type: "arcade_starter" }; } return metadata; @@ -1008,7 +1005,7 @@ export class DataMerger { iconUrl: "", isBYOC: false, isPro: false, - type: isStarterToolkitId(toolkitId) ? "arcade_starter" : "arcade", + type: isApiSuffixedToolkitId(toolkitId) ? "arcade_starter" : "arcade", docsLink: "", isComingSoon: false, isHidden: false, diff --git a/toolkit-docs-generator/src/shared/toolkit-data-dir.ts b/toolkit-docs-generator/src/shared/toolkit-data-dir.ts new file mode 100644 index 000000000..36bd19cf6 --- /dev/null +++ b/toolkit-docs-generator/src/shared/toolkit-data-dir.ts @@ -0,0 +1,35 @@ +/** + * Where the generated toolkit JSON lives, shared by the Next.js docs app and + * toolkit-docs-generator. Kept separate from `toolkit-primitives.ts` because + * this module reaches for `node:path` / `node:url`: the primitives are pure + * string helpers that client components pull in through the integrations + * index, and a Node built-in anywhere in that import graph fails the webpack + * browser build. + */ + +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; + +const HERE = dirname(fileURLToPath(import.meta.url)); + +/** + * toolkit-docs-generator/data/toolkits, resolved relative to this file rather + * than `process.cwd()` — correct regardless of which directory a script or the + * Next.js server happened to be started from. + */ +export const DEFAULT_TOOLKIT_DATA_DIR = join( + HERE, + "..", + "..", + "data", + "toolkits" +); + +/** + * Resolve the toolkit data directory: an explicit override wins, then the + * `TOOLKIT_DATA_DIR` env var (used by tests and CI to point at a fixture or + * scratch copy), then the real generator output directory. + */ +export function resolveToolkitDataDir(override?: string): string { + return override ?? process.env.TOOLKIT_DATA_DIR ?? DEFAULT_TOOLKIT_DATA_DIR; +} diff --git a/toolkit-docs-generator/src/shared/toolkit-primitives.ts b/toolkit-docs-generator/src/shared/toolkit-primitives.ts new file mode 100644 index 000000000..1764190d1 --- /dev/null +++ b/toolkit-docs-generator/src/shared/toolkit-primitives.ts @@ -0,0 +1,126 @@ +/** + * Toolkit primitives shared by the Next.js docs app (app/_lib and its + * consumers) and toolkit-docs-generator. Both halves need the same toolkit + * ID/slug/category logic, but the generator's tsconfig pins `rootDir` to its + * own `src/`, so a module outside that directory fails its build + * (`TS6059: File '...' is not under 'rootDir'`). Living here satisfies the + * generator's rootDir trivially, while the app side can still reach it with + * a normal relative or `@/`-aliased import — root tsconfig has no `rootDir` + * restriction, only a `toolkit-docs-generator` entry in `exclude`, which + * only affects automatic root-file discovery, not files reached via import. + * + * Everything here must stay free of Node built-ins: client components reach + * this module through the integrations index, so a `node:*` import anywhere + * in the graph fails the webpack browser build. Filesystem concerns live in + * `toolkit-data-dir.ts` instead. + */ + +// ============================================================================ +// Toolkit ID normalization +// ============================================================================ + +const TOOLKIT_ID_NORMALIZER = /[^a-z0-9]+/g; + +/** + * Strip all non-alphanumeric characters and lowercase. + * Used for case/punctuation-insensitive matching of toolkit IDs and labels + * (e.g. matching "GitHub API" against a design system entry keyed "Github"). + */ +export function normalizeToolkitId(value: string): string { + return value.toLowerCase().replace(TOOLKIT_ID_NORMALIZER, ""); +} + +/** + * Whether a toolkit ID looks like an auto-generated "*Api" wrapper toolkit + * (e.g. "GithubApi", "hubspot-crm-api", "stripe_api"). These get special- + * cased in several places: starter-type override, provider-id metadata + * fallback, and "-api"-suffixed docs slugs/icons. + */ +export function isApiSuffixedToolkitId(toolkitId: string): boolean { + return normalizeToolkitId(toolkitId).endsWith("api"); +} + +// ============================================================================ +// Slug generation +// ============================================================================ + +const CAMEL_BOUNDARY = /([a-z0-9])([A-Z])/g; + +/** + * Convert a CamelCase toolkit ID to a kebab-case URL slug. + * + * Examples: + * PosthogApi → posthog-api + * GoogleCalendar → google-calendar + * E2b → e2b + * HubspotCrmApi → hubspot-crm-api + */ +export function toKebabCase(value: string): string { + return value.replace(CAMEL_BOUNDARY, "$1-$2").toLowerCase(); +} + +export type ToolkitSlugSource = { + id: string; + docsLink?: string | null; +}; + +function extractSlugFromPath(path: string): string | null { + const segments = path.split("/").filter(Boolean); + return segments.at(-1) ?? null; +} + +/** + * The canonical docs slug for a toolkit: the last path segment of its + * `docsLink` when present (preserves hand-authored slugs like "stripe_api"), + * otherwise the kebab-case of its ID. + */ +export function getToolkitSlug({ id, docsLink }: ToolkitSlugSource): string { + if (docsLink) { + try { + const url = new URL(docsLink); + const slug = extractSlugFromPath(url.pathname); + if (slug) { + return slug; + } + } catch { + const slug = extractSlugFromPath(docsLink); + if (slug) { + return slug; + } + } + } + + return toKebabCase(id); +} + +// ============================================================================ +// Integration categories +// ============================================================================ + +/** + * The docs-generation category buckets. Each value corresponds to exactly + * one `app/en/resources/integrations//[toolkitId]` route directory + * (see tests/integration-category-routes.test.ts) and to the design system's + * own `ToolkitCategory` union (minus its "all" filter meta-value) — see + * app/en/resources/integrations/components/filter-params.ts. There is + * deliberately no "others" catch-all: a toolkit whose category doesn't match + * one of these has no page to render, so `normalizeCategory` in + * app/_lib/toolkit-static-params.ts throws instead of bucketing it here. + * + * `ToolkitCategorySchema` in ./toolkit-schemas.ts is built from this array, + * so the generator's contract and the docs app's route set can't drift + * apart. + */ +export const INTEGRATION_CATEGORIES = [ + "productivity", + "social", + "entertainment", + "development", + "payments", + "search", + "sales", + "databases", + "customer-support", +] as const; + +export type IntegrationCategory = (typeof INTEGRATION_CATEGORIES)[number]; diff --git a/toolkit-docs-generator/src/shared/toolkit-schemas.ts b/toolkit-docs-generator/src/shared/toolkit-schemas.ts new file mode 100644 index 000000000..7531105b2 --- /dev/null +++ b/toolkit-docs-generator/src/shared/toolkit-schemas.ts @@ -0,0 +1,420 @@ +/** + * Zod schemas for the merged toolkit JSON contract, shared by the Next.js + * docs app (app/_lib and app/_components/toolkit-docs) and + * toolkit-docs-generator. The generator validates its own output against + * these schemas on write (see src/generator/json-generator.ts); the app + * validates on read (see app/_lib/toolkit-data.ts) so a file that doesn't + * match this shape is rejected instead of crashing mid-render. + * + * This lives under toolkit-docs-generator/src/ (not app/_lib or a repo-root + * shared/ directory) because the generator's tsconfig pins `rootDir` to its + * own `src/`, so a shared module outside that directory fails the + * generator's build (`TS6059: File '...' is not under 'rootDir'`). The app + * reaches it via the `@/` alias, same as toolkit-primitives.ts next to this + * file. + * + * CLI-only input schemas (ProviderVersion, GenerateInput) and the raw, + * pre-merge Engine/Design-System schemas (ToolDefinition, ToolkitMetadata) + * stay in toolkit-docs-generator/src/types/index.ts: the app never sees + * that shape, only the merged output defined here. + */ +import { z } from "zod"; +import { INTEGRATION_CATEGORIES } from "./toolkit-primitives.js"; + +// ============================================================================ +// Tool Parameter Schema +// ============================================================================ + +export const ToolParameterSchema = z.object({ + name: z.string(), + type: z.string(), + innerType: z.string().optional(), + required: z.boolean(), + description: z.string().nullable(), + enum: z.array(z.string()).nullable(), + inferrable: z.boolean().default(true), +}); + +export type ToolParameter = z.infer; + +// ============================================================================ +// Tool Auth Schema +// ============================================================================ + +export const ToolAuthSchema = z.object({ + providerId: z.string().nullable(), + providerType: z.string(), + scopes: z.array(z.string()), +}); + +export type ToolAuth = z.infer; + +// ============================================================================ +// Tool Output Schema +// ============================================================================ + +export const ToolOutputSchema = z.object({ + type: z.string(), + description: z.string().nullable(), +}); + +export type ToolOutput = z.infer; + +// ============================================================================ +// Tool Secrets Schema +// ============================================================================ + +export const SecretTypeSchema = z.enum([ + "api_key", + "token", + "client_secret", + "webhook_secret", + "private_key", + "password", + "unknown", +]); + +export type SecretType = z.infer; + +export const ToolSecretSchema = z.object({ + name: z.string(), + type: SecretTypeSchema, +}); + +export type ToolSecret = z.infer; + +// ============================================================================ +// Tool Metadata Schema (per-tool metadata from Engine API) +// ============================================================================ + +export const ToolMetadataClassificationSchema = z.object({ + serviceDomains: z.array(z.string()).default([]), +}); +export type ToolMetadataClassification = z.infer< + typeof ToolMetadataClassificationSchema +>; + +export const ToolMetadataBehaviorSchema = z.object({ + operations: z.array(z.string()).default([]), + readOnly: z.boolean().optional(), + destructive: z.boolean().optional(), + idempotent: z.boolean().optional(), + openWorld: z.boolean().optional(), +}); +export type ToolMetadataBehavior = z.infer; + +export const ToolMetadataSchema = z.object({ + classification: ToolMetadataClassificationSchema, + behavior: ToolMetadataBehaviorSchema, + extras: z.record(z.string(), z.unknown()).optional().nullable(), +}); +export type ToolMetadata = z.infer; + +// ============================================================================ +// Toolkit Category / Type Schemas (from Design System) +// ============================================================================ + +// Built from INTEGRATION_CATEGORIES (toolkit-primitives.ts) rather than a +// hand-copied list of the same values, so the generator's output contract +// and the docs app's route set can never drift apart — see that constant's +// doc comment for why there's no "others" member. +export const ToolkitCategorySchema = z.enum(INTEGRATION_CATEGORIES); + +export type ToolkitCategory = z.infer; + +export const ToolkitTypeSchema = z.enum([ + "arcade", + "arcade_starter", + "verified", + "community", + "auth", +]); + +export type ToolkitType = z.infer; + +// ============================================================================ +// Documentation Chunk Schema (for custom content injection) +// ============================================================================ + +/** + * Type of documentation chunk content + * - callout: Warning, info, or tip box + * - markdown: Raw markdown content + * - code: Code block with language + * - warning: Highlighted warning message + * - info: Informational note + * - tip: Helpful tip + */ +export const DocumentationChunkTypeSchema = z.enum([ + "callout", + "markdown", + "code", + "warning", + "info", + "tip", + "section", +]); + +export type DocumentationChunkType = z.infer< + typeof DocumentationChunkTypeSchema +>; + +/** + * Location where the chunk should be injected + * - header: After the toolkit header, before tools list + * - description: Around the tool description + * - parameters: Around the parameters section + * - auth: Around the auth/scopes section + * - secrets: Around the secrets section + * - output: Around the output section + * - footer: After all tools, before the footer + * - before_available_tools: Before the available tools section (toolkit-level) + * - after_available_tools: After the available tools section (toolkit-level) + * - custom_section: Standalone custom section outside the tools list + */ +export const DocumentationChunkLocationSchema = z.enum([ + "header", + "description", + "parameters", + "auth", + "secrets", + "output", + "footer", + "before_available_tools", + "after_available_tools", + "custom_section", +]); + +export type DocumentationChunkLocation = z.infer< + typeof DocumentationChunkLocationSchema +>; + +/** + * Position relative to the location + */ +export const DocumentationChunkPositionSchema = z.enum([ + "before", + "after", + "replace", +]); + +export type DocumentationChunkPosition = z.infer< + typeof DocumentationChunkPositionSchema +>; + +/** + * A documentation chunk represents custom content to inject into docs + */ +export const DocumentationChunkSchema = z.object({ + /** Type of content */ + type: DocumentationChunkTypeSchema, + /** Where to inject the content */ + location: DocumentationChunkLocationSchema, + /** Position relative to location (before, after, replace) */ + position: DocumentationChunkPositionSchema, + /** The actual content (markdown string) */ + content: z.string(), + /** Optional title for callouts */ + title: z.string().optional(), + /** Optional variant for styling (e.g., "destructive" for warnings) */ + variant: z + .enum(["default", "destructive", "warning", "info", "success"]) + .optional(), + /** Optional section header for sidebar navigation (e.g., "## Auth Setup") */ + header: z.string().optional(), + /** Optional priority for ordering (lower = earlier, default = 100) */ + priority: z.number().optional(), +}); + +export type DocumentationChunk = z.infer; + +// ============================================================================ +// Tool Code Example Schema (for generating example code) +// ============================================================================ + +/** + * Parameter value with type information for code generation + */ +export const ExampleParameterValueSchema = z.object({ + /** The example value to use in code */ + value: z.unknown(), + /** Parameter type */ + type: z.enum(["string", "integer", "boolean", "array", "object"]), + /** Whether this parameter is required */ + required: z.boolean(), +}); + +export type ExampleParameterValue = z.infer; + +/** + * Tool code example configuration + * Used to generate Python/JavaScript example code + */ +export const ToolCodeExampleSchema = z.object({ + /** Full tool name (e.g., "Github.SetStarred") */ + toolName: z.string(), + /** Parameter values with type info */ + parameters: z.record(z.string(), ExampleParameterValueSchema), + /** Whether this tool requires user authorization */ + requiresAuth: z.boolean(), + /** Auth provider ID if auth is required */ + authProvider: z.string().optional(), + /** Optional tab label for the code example */ + tabLabel: z.string().optional(), +}); + +export type ToolCodeExample = z.infer; + +// ============================================================================ +// Toolkit Sub-Page Schema +// ============================================================================ + +/** + * A sub-page for a toolkit: either a string (legacy slug) or a rich object + * with { type, content, relativePath } for inline MDX sub-page content. + */ +export const ToolkitSubPageSchema = z.union([ + z.string(), + z.object({ + type: z.string().min(1), + content: z.string(), + relativePath: z.string().min(1), + }), +]); + +export type ToolkitSubPage = z.infer; + +// ============================================================================ +// Merged Tool Schema (output format) +// ============================================================================ + +export const MergedToolSchema = z.object({ + name: z.string(), + qualifiedName: z.string(), + fullyQualifiedName: z.string(), + description: z.string().nullable(), + parameters: z.array(ToolParameterSchema), + auth: ToolAuthSchema.nullable(), + secrets: z.array(z.string()), + secretsInfo: z.array(ToolSecretSchema).default([]), + output: ToolOutputSchema.nullable(), + /** Custom documentation chunks for this tool */ + documentationChunks: z.array(DocumentationChunkSchema).default([]), + /** Generated code example configuration */ + codeExample: ToolCodeExampleSchema.optional(), + metadata: ToolMetadataSchema.nullable().optional(), +}); + +export type MergedTool = z.infer; + +// ============================================================================ +// Merged Toolkit Schema (output format) +// ============================================================================ + +export const ToolkitAuthTypeSchema = z.enum([ + "oauth2", + "api_key", + "mixed", + "none", +]); + +export type ToolkitAuthType = z.infer; + +export const MergedToolkitMetadataSchema = z.object({ + category: ToolkitCategorySchema, + iconUrl: z.string(), + isBYOC: z.boolean(), + isPro: z.boolean(), + type: ToolkitTypeSchema, + docsLink: z.string(), + isComingSoon: z.boolean(), + isHidden: z.boolean(), +}); + +export type MergedToolkitMetadata = z.infer; + +export const MergedToolkitAuthSchema = z.object({ + type: ToolkitAuthTypeSchema, + providerId: z.string().nullable(), + allScopes: z.array(z.string()), +}); + +export type MergedToolkitAuth = z.infer; + +export const MergedToolkitSchema = z.object({ + /** Unique toolkit ID (e.g., "Github") */ + id: z.string(), + /** Human-readable label (e.g., "GitHub") */ + label: z.string(), + /** Toolkit version (e.g., "1.0.0") */ + version: z.string(), + /** Toolkit description */ + description: z.string().nullable(), + /** LLM-generated summary (optional) */ + summary: z.string().optional(), + /** + * True when the current `summary` is known to be out of date with the + * toolkit's current tools (the signature changed but regeneration was + * skipped or failed, so the previous summary was carried forward as a + * fallback). Cleared whenever a fresh summary is successfully generated + * or when the summary is verified against an unchanged signature. + */ + summaryStale: z.boolean().optional(), + /** + * Machine-readable reason the summary is stale (e.g. + * "llm_generator_unavailable", "llm_generation_failed"). Always set + * together with `summaryStale: true`. Cleared together with it. + */ + summaryStaleReason: z.string().optional(), + /** Metadata from Design System */ + metadata: MergedToolkitMetadataSchema, + /** Authentication requirements */ + auth: MergedToolkitAuthSchema.nullable(), + /** All tools in this toolkit */ + tools: z.array(MergedToolSchema), + /** Toolkit-level documentation chunks */ + documentationChunks: z.array(DocumentationChunkSchema).default([]), + /** Custom imports for MDX */ + customImports: z.array(z.string()).default([]), + /** + * Sub-pages that exist for this toolkit. + * Each entry is either a string (legacy slug) or a rich object with + * { type, content, relativePath } for inline MDX sub-page content. + */ + subPages: z.array(ToolkitSubPageSchema).default([]), + /** + * Optional override for the pip package name shown in the install + * snippet. Not currently emitted by the generator (toolkits derive it + * from `id` via `buildPipPackageName`), but the docs app has always + * accepted an explicit override here, so it stays part of the contract. + */ + pipPackageName: z.string().optional(), + /** Generation metadata */ + generatedAt: z.string().optional(), +}); + +export type MergedToolkit = z.infer; + +// ============================================================================ +// Index Output Schema +// ============================================================================ + +export const ToolkitIndexEntrySchema = z.object({ + id: z.string(), + label: z.string(), + version: z.string(), + category: ToolkitCategorySchema, + type: ToolkitTypeSchema, + toolCount: z.number(), + authType: ToolkitAuthTypeSchema, +}); + +export type ToolkitIndexEntry = z.infer; + +export const ToolkitIndexSchema = z.object({ + generatedAt: z.string(), + version: z.string(), + toolkits: z.array(ToolkitIndexEntrySchema), +}); + +export type ToolkitIndex = z.infer; diff --git a/toolkit-docs-generator/src/sources/design-system-metadata.ts b/toolkit-docs-generator/src/sources/design-system-metadata.ts index 9f0f67a11..b25946660 100644 --- a/toolkit-docs-generator/src/sources/design-system-metadata.ts +++ b/toolkit-docs-generator/src/sources/design-system-metadata.ts @@ -8,6 +8,7 @@ */ import { TOOLKITS as DESIGN_SYSTEM_TOOLKITS } from "@arcadeai/design-system/metadata/toolkits"; import { z } from "zod"; +import { normalizeToolkitId } from "../shared/toolkit-primitives.js"; import type { ToolkitMetadata } from "../types/index.js"; import { ToolkitMetadataSchema } from "../types/index.js"; import type { IMetadataSource } from "./internal.js"; @@ -38,12 +39,6 @@ type DesignSystemToolkit = z.infer; // Helpers // ============================================================================ -const LOOKUP_KEY_REGEX = /[^a-z0-9]/g; - -function normalizeLookupKey(value: string): string { - return value.toLowerCase().replace(LOOKUP_KEY_REGEX, ""); -} - function toToolkitMetadata(entry: DesignSystemToolkit): ToolkitMetadata | null { const iconUrl = entry.publicIconUrl ?? entry.iconUrl; if (!iconUrl) return null; @@ -83,18 +78,18 @@ export class DesignSystemMetadataSource implements IMetadataSource { this.indexByIdOrLabel = new Map(); for (const toolkit of toolkits) { - this.indexByIdOrLabel.set(normalizeLookupKey(toolkit.id), toolkit); - this.indexByIdOrLabel.set(normalizeLookupKey(toolkit.label), toolkit); + this.indexByIdOrLabel.set(normalizeToolkitId(toolkit.id), toolkit); + this.indexByIdOrLabel.set(normalizeToolkitId(toolkit.label), toolkit); } } async getToolkitMetadata(toolkitId: string): Promise { - const key = normalizeLookupKey(toolkitId); + const key = normalizeToolkitId(toolkitId); const direct = this.indexByIdOrLabel.get(key); if (direct) return direct; // Fallback 1: "github-api" / "github_api" style inputs. - // (normalizeLookupKey already strips separators) + // (normalizeToolkitId already strips separators) // Fallback 2: If this looks like an API toolkit, try the base provider. if (key.endsWith("api")) { diff --git a/toolkit-docs-generator/src/sources/toolkit-data-source.ts b/toolkit-docs-generator/src/sources/toolkit-data-source.ts index 1b6b191a6..de78a8951 100644 --- a/toolkit-docs-generator/src/sources/toolkit-data-source.ts +++ b/toolkit-docs-generator/src/sources/toolkit-data-source.ts @@ -7,8 +7,8 @@ */ import { join } from "path"; +import { isApiSuffixedToolkitId } from "../shared/toolkit-primitives.js"; import type { ToolDefinition, ToolkitMetadata } from "../types/index.js"; -import { normalizeId } from "../utils/fp.js"; import { filterToolsByHighestVersion } from "../utils/version-coherence.js"; import { type ArcadeApiSourceConfig, @@ -142,7 +142,7 @@ export class CombinedToolkitDataSource implements IToolkitDataSource { tools: readonly ToolDefinition[], directMetadata: ToolkitMetadata | null ): Promise { - if (directMetadata || !normalizeId(toolkitId).endsWith("api")) { + if (directMetadata || !isApiSuffixedToolkitId(toolkitId)) { return directMetadata; } diff --git a/toolkit-docs-generator/src/types/index.ts b/toolkit-docs-generator/src/types/index.ts index c8dba520c..0e9e431a6 100644 --- a/toolkit-docs-generator/src/types/index.ts +++ b/toolkit-docs-generator/src/types/index.ts @@ -1,7 +1,28 @@ /** * Core type definitions for the toolkit docs generator + * + * The merged/output schemas (MergedToolkit, ToolkitIndex, and everything + * they're built from) live in ../shared/toolkit-schemas.ts because the + * Next.js docs app imports them too — see that file's header comment for + * why the shared module has to live under this package's `src/`. Everything + * below is either CLI-only or describes a pre-merge shape the app never + * sees (raw Engine API / Design System data, extracted MDX custom + * sections), so it stays generator-local and re-exports the shared pieces + * it depends on. */ import { z } from "zod"; +import { + DocumentationChunkSchema, + ToolAuthSchema, + ToolkitCategorySchema, + ToolkitSubPageSchema, + ToolkitTypeSchema, + ToolMetadataSchema, + ToolOutputSchema, + ToolParameterSchema, +} from "../shared/toolkit-schemas.js"; + +export * from "../shared/toolkit-schemas.js"; // ============================================================================ // CLI Input Types @@ -29,96 +50,7 @@ export const GenerateInputSchema = z.object({ export type GenerateInput = z.infer; // ============================================================================ -// Tool Parameter Schema -// ============================================================================ - -export const ToolParameterSchema = z.object({ - name: z.string(), - type: z.string(), - innerType: z.string().optional(), - required: z.boolean(), - description: z.string().nullable(), - enum: z.array(z.string()).nullable(), - inferrable: z.boolean().default(true), -}); - -export type ToolParameter = z.infer; - -// ============================================================================ -// Tool Auth Schema -// ============================================================================ - -export const ToolAuthSchema = z.object({ - providerId: z.string().nullable(), - providerType: z.string(), - scopes: z.array(z.string()), -}); - -export type ToolAuth = z.infer; - -// ============================================================================ -// Tool Output Schema -// ============================================================================ - -export const ToolOutputSchema = z.object({ - type: z.string(), - description: z.string().nullable(), -}); - -export type ToolOutput = z.infer; - -// ============================================================================ -// Tool Secrets Schema -// ============================================================================ - -export const SecretTypeSchema = z.enum([ - "api_key", - "token", - "client_secret", - "webhook_secret", - "private_key", - "password", - "unknown", -]); - -export type SecretType = z.infer; - -export const ToolSecretSchema = z.object({ - name: z.string(), - type: SecretTypeSchema, -}); - -export type ToolSecret = z.infer; - -// ============================================================================ -// Tool Metadata Schema (per-tool metadata from Engine API) -// ============================================================================ - -export const ToolMetadataClassificationSchema = z.object({ - serviceDomains: z.array(z.string()).default([]), -}); -export type ToolMetadataClassification = z.infer< - typeof ToolMetadataClassificationSchema ->; - -export const ToolMetadataBehaviorSchema = z.object({ - operations: z.array(z.string()).default([]), - readOnly: z.boolean().optional(), - destructive: z.boolean().optional(), - idempotent: z.boolean().optional(), - openWorld: z.boolean().optional(), -}); -export type ToolMetadataBehavior = z.infer; - -export const ToolMetadataSchema = z.object({ - classification: ToolMetadataClassificationSchema, - behavior: ToolMetadataBehaviorSchema, - extras: z.record(z.string(), z.unknown()).optional().nullable(), -}); -export type ToolMetadata = z.infer; - -// ============================================================================ -// Tool Definition Schema (from Engine API) +// Tool Definition Schema (raw, from Engine API, pre-merge) // ============================================================================ export const ToolDefinitionSchema = z.object({ @@ -137,33 +69,9 @@ export const ToolDefinitionSchema = z.object({ export type ToolDefinition = z.infer; // ============================================================================ -// Toolkit Metadata Schema (from Design System) +// Toolkit Metadata Schema (raw, from Design System, pre-merge) // ============================================================================ -export const ToolkitCategorySchema = z.enum([ - "productivity", - "social", - "development", - "entertainment", - "search", - "payments", - "sales", - "databases", - "customer-support", -]); - -export type ToolkitCategory = z.infer; - -export const ToolkitTypeSchema = z.enum([ - "arcade", - "arcade_starter", - "verified", - "community", - "auth", -]); - -export type ToolkitType = z.infer; - export const ToolkitMetadataSchema = z.object({ id: z.string(), label: z.string(), @@ -180,151 +88,9 @@ export const ToolkitMetadataSchema = z.object({ export type ToolkitMetadata = z.infer; // ============================================================================ -// Documentation Chunk Schema (for custom content injection) -// ============================================================================ - -/** - * Type of documentation chunk content - * - callout: Warning, info, or tip box - * - markdown: Raw markdown content - * - code: Code block with language - * - warning: Highlighted warning message - * - info: Informational note - * - tip: Helpful tip - */ -export const DocumentationChunkTypeSchema = z.enum([ - "callout", - "markdown", - "code", - "warning", - "info", - "tip", - "section", -]); - -export type DocumentationChunkType = z.infer< - typeof DocumentationChunkTypeSchema ->; - -/** - * Location where the chunk should be injected - * - header: After the toolkit header, before tools list - * - description: Around the tool description - * - parameters: Around the parameters section - * - auth: Around the auth/scopes section - * - secrets: Around the secrets section - * - output: Around the output section - * - footer: After all tools, before the footer - * - before_available_tools: Before the available tools section (toolkit-level) - * - after_available_tools: After the available tools section (toolkit-level) - * - custom_section: Standalone custom section outside the tools list - */ -export const DocumentationChunkLocationSchema = z.enum([ - "header", - "description", - "parameters", - "auth", - "secrets", - "output", - "footer", - "before_available_tools", - "after_available_tools", - "custom_section", -]); - -export type DocumentationChunkLocation = z.infer< - typeof DocumentationChunkLocationSchema ->; - -/** - * Position relative to the location - */ -export const DocumentationChunkPositionSchema = z.enum([ - "before", - "after", - "replace", -]); - -export type DocumentationChunkPosition = z.infer< - typeof DocumentationChunkPositionSchema ->; - -/** - * A documentation chunk represents custom content to inject into docs - */ -export const DocumentationChunkSchema = z.object({ - /** Type of content */ - type: DocumentationChunkTypeSchema, - /** Where to inject the content */ - location: DocumentationChunkLocationSchema, - /** Position relative to location (before, after, replace) */ - position: DocumentationChunkPositionSchema, - /** The actual content (markdown string) */ - content: z.string(), - /** Optional title for callouts */ - title: z.string().optional(), - /** Optional variant for styling (e.g., "destructive" for warnings) */ - variant: z - .enum(["default", "destructive", "warning", "info", "success"]) - .optional(), - /** Optional section header for sidebar navigation (e.g., "## Auth Setup") */ - header: z.string().optional(), - /** Optional priority for ordering (lower = earlier, default = 100) */ - priority: z.number().optional(), -}); - -export type DocumentationChunk = z.infer; - -// ============================================================================ -// Tool Code Example Schema (for generating example code) -// ============================================================================ - -/** - * Parameter value with type information for code generation - */ -export const ExampleParameterValueSchema = z.object({ - /** The example value to use in code */ - value: z.unknown(), - /** Parameter type */ - type: z.enum(["string", "integer", "boolean", "array", "object"]), - /** Whether this parameter is required */ - required: z.boolean(), -}); - -export type ExampleParameterValue = z.infer; - -/** - * Tool code example configuration - * Used to generate Python/JavaScript example code - */ -export const ToolCodeExampleSchema = z.object({ - /** Full tool name (e.g., "Github.SetStarred") */ - toolName: z.string(), - /** Parameter values with type info */ - parameters: z.record(z.string(), ExampleParameterValueSchema), - /** Whether this tool requires user authorization */ - requiresAuth: z.boolean(), - /** Auth provider ID if auth is required */ - authProvider: z.string().optional(), - /** Optional tab label for the code example */ - tabLabel: z.string().optional(), -}); - -export type ToolCodeExample = z.infer; - -// ============================================================================ -// Custom Sections Schema (extracted from MDX) +// Custom Sections Schema (extracted from MDX, pre-merge) // ============================================================================ -export const ToolkitSubPageSchema = z.union([ - z.string(), - z.object({ - type: z.string().min(1), - content: z.string(), - relativePath: z.string().min(1), - }), -]); - export const CustomSectionsSchema = z.object({ /** Toolkit-level documentation chunks */ documentationChunks: z.array(DocumentationChunkSchema).default([]), @@ -339,131 +105,3 @@ export const CustomSectionsSchema = z.object({ }); export type CustomSections = z.infer; - -// ============================================================================ -// Merged Tool Schema (output format) -// ============================================================================ - -export const MergedToolSchema = z.object({ - name: z.string(), - qualifiedName: z.string(), - fullyQualifiedName: z.string(), - description: z.string().nullable(), - parameters: z.array(ToolParameterSchema), - auth: ToolAuthSchema.nullable(), - secrets: z.array(z.string()), - secretsInfo: z.array(ToolSecretSchema).default([]), - output: ToolOutputSchema.nullable(), - /** Custom documentation chunks for this tool */ - documentationChunks: z.array(DocumentationChunkSchema).default([]), - /** Generated code example configuration */ - codeExample: ToolCodeExampleSchema.optional(), - metadata: ToolMetadataSchema.nullable().optional(), -}); - -export type MergedTool = z.infer; - -// ============================================================================ -// Merged Toolkit Schema (output format) -// ============================================================================ - -export const ToolkitAuthTypeSchema = z.enum([ - "oauth2", - "api_key", - "mixed", - "none", -]); - -export type ToolkitAuthType = z.infer; - -export const MergedToolkitMetadataSchema = z.object({ - category: ToolkitCategorySchema, - iconUrl: z.string(), - isBYOC: z.boolean(), - isPro: z.boolean(), - type: ToolkitTypeSchema, - docsLink: z.string(), - isComingSoon: z.boolean(), - isHidden: z.boolean(), -}); - -export type MergedToolkitMetadata = z.infer; - -export const MergedToolkitAuthSchema = z.object({ - type: ToolkitAuthTypeSchema, - providerId: z.string().nullable(), - allScopes: z.array(z.string()), -}); - -export type MergedToolkitAuth = z.infer; - -export const MergedToolkitSchema = z.object({ - /** Unique toolkit ID (e.g., "Github") */ - id: z.string(), - /** Human-readable label (e.g., "GitHub") */ - label: z.string(), - /** Toolkit version (e.g., "1.0.0") */ - version: z.string(), - /** Toolkit description */ - description: z.string().nullable(), - /** LLM-generated summary (optional) */ - summary: z.string().optional(), - /** - * True when the current `summary` is known to be out of date with the - * toolkit's current tools (the signature changed but regeneration was - * skipped or failed, so the previous summary was carried forward as a - * fallback). Cleared whenever a fresh summary is successfully generated - * or when the summary is verified against an unchanged signature. - */ - summaryStale: z.boolean().optional(), - /** - * Machine-readable reason the summary is stale (e.g. - * "llm_generator_unavailable", "llm_generation_failed"). Always set - * together with `summaryStale: true`. Cleared together with it. - */ - summaryStaleReason: z.string().optional(), - /** Metadata from Design System */ - metadata: MergedToolkitMetadataSchema, - /** Authentication requirements */ - auth: MergedToolkitAuthSchema.nullable(), - /** All tools in this toolkit */ - tools: z.array(MergedToolSchema), - /** Toolkit-level documentation chunks */ - documentationChunks: z.array(DocumentationChunkSchema).default([]), - /** Custom imports for MDX */ - customImports: z.array(z.string()).default([]), - /** - * Sub-pages that exist for this toolkit. - * Each entry is either a string (legacy slug) or a rich object with - * { type, content, relativePath } for inline MDX sub-page content. - */ - subPages: z.array(ToolkitSubPageSchema).default([]), - /** Generation metadata */ - generatedAt: z.string().optional(), -}); - -export type MergedToolkit = z.infer; - -// ============================================================================ -// Index Output Schema -// ============================================================================ - -export const ToolkitIndexEntrySchema = z.object({ - id: z.string(), - label: z.string(), - version: z.string(), - category: ToolkitCategorySchema, - type: ToolkitTypeSchema, - toolCount: z.number(), - authType: ToolkitAuthTypeSchema, -}); - -export type ToolkitIndexEntry = z.infer; - -export const ToolkitIndexSchema = z.object({ - generatedAt: z.string(), - version: z.string(), - toolkits: z.array(ToolkitIndexEntrySchema), -}); - -export type ToolkitIndex = z.infer; diff --git a/toolkit-docs-generator/tests/app-lib/toolkit-data.test.ts b/toolkit-docs-generator/tests/app-lib/toolkit-data.test.ts index 91a0bdf36..fac395e4d 100644 --- a/toolkit-docs-generator/tests/app-lib/toolkit-data.test.ts +++ b/toolkit-docs-generator/tests/app-lib/toolkit-data.test.ts @@ -80,10 +80,20 @@ describe("toolkit data loader", () => { const toolkitData = { id: "PosthogApi", label: "PostHog API", + version: "1.0.0", + description: null, tools: [], + auth: null, metadata: { + category: "development", + iconUrl: "https://design-system.arcade.dev/icons/posthog.svg", + isBYOC: false, + isPro: false, + type: "arcade_starter", docsLink: "https://docs.arcade.dev/en/mcp-servers/development/posthog-api", + isComingSoon: false, + isHidden: false, }, }; await writeFile( diff --git a/toolkit-docs-generator/tests/app-lib/toolkit-slug.test.ts b/toolkit-docs-generator/tests/app-lib/toolkit-slug.test.ts index 74d7a4017..b18317bf8 100644 --- a/toolkit-docs-generator/tests/app-lib/toolkit-slug.test.ts +++ b/toolkit-docs-generator/tests/app-lib/toolkit-slug.test.ts @@ -4,7 +4,7 @@ import { normalizeToolkitId, type ToolkitSlugSource, toKebabCase, -} from "../../../app/_lib/toolkit-slug"; +} from "../../src/shared/toolkit-primitives"; // ============================================================================ // normalizeToolkitId diff --git a/toolkit-docs-generator/tests/app-lib/toolkit-static-params.test.ts b/toolkit-docs-generator/tests/app-lib/toolkit-static-params.test.ts index 8c5c59f2c..55a5a92c0 100644 --- a/toolkit-docs-generator/tests/app-lib/toolkit-static-params.test.ts +++ b/toolkit-docs-generator/tests/app-lib/toolkit-static-params.test.ts @@ -2,12 +2,12 @@ import { mkdtemp, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { describe, expect, it } from "vitest"; -import { normalizeToolkitId } from "../../../app/_lib/toolkit-slug"; import { getToolkitStaticParamsForCategory, listToolkitRoutes, type ToolkitCatalogEntry, } from "../../../app/_lib/toolkit-static-params"; +import { normalizeToolkitId } from "../../src/shared/toolkit-primitives"; const withTempDir = async (fn: (dir: string) => Promise) => { const dir = await mkdtemp(join(tmpdir(), "toolkit-static-params-")); @@ -43,8 +43,29 @@ const writeToolkitData = async ( } ) => { const fileName = `${normalizeToolkitId(toolkit.id)}.json`; + // Fill in the fields the merged toolkit schema requires but this test + // suite doesn't care about, so fixtures stay valid without every call + // site restating boilerplate. const toolkitFixture = JSON.stringify( - { label: toolkit.label ?? toolkit.id, ...toolkit }, + { + version: "1.0.0", + description: null, + tools: [], + auth: null, + label: toolkit.label ?? toolkit.id, + ...toolkit, + metadata: { + category: "productivity", + iconUrl: "https://design-system.arcade.dev/icons/placeholder.svg", + isBYOC: false, + isPro: false, + type: "arcade", + docsLink: "", + isComingSoon: false, + isHidden: false, + ...toolkit.metadata, + }, + }, null, 2 ); @@ -227,16 +248,28 @@ describe("toolkit static params", () => { }); }); - it('maps unknown categories to "others"', async () => { + it('throws on an unrecognized category instead of coercing it to "others"', async () => { await withTempDir(async (dir) => { await writeIndex(dir, [{ id: "Github", category: "weird" }]); + await expect( + listToolkitRoutes({ dataDir: dir, toolkitsCatalog: [] }) + ).rejects.toThrow(/weird/); + }); + }); + + it("skips a toolkit with no category anywhere instead of routing it to a fake bucket", async () => { + await withTempDir(async (dir) => { + // No JSON file, no catalog entry, and the index entry itself omits + // category — nothing to route this toolkit under. + await writeIndex(dir, [{ id: "Github" }]); + const routes = await listToolkitRoutes({ dataDir: dir, toolkitsCatalog: [], }); - expect(routes).toEqual([{ toolkitId: "github", category: "others" }]); + expect(routes).toEqual([]); }); }); }); diff --git a/toolkit-docs-generator/tests/scripts/sync-toolkit-sidebar.test.ts b/toolkit-docs-generator/tests/scripts/sync-toolkit-sidebar.test.ts index 273ff1e19..aa0f20397 100644 --- a/toolkit-docs-generator/tests/scripts/sync-toolkit-sidebar.test.ts +++ b/toolkit-docs-generator/tests/scripts/sync-toolkit-sidebar.test.ts @@ -296,13 +296,27 @@ describe("buildToolkitInfoList", () => { }); it("keeps sidebar href categories consistent with static params", async () => { + // This fixture also flows through getToolkitStaticParamsForCategory + // below, which validates it against the full merged toolkit schema — + // unlike the other fixtures in this file, it needs every required field, + // not just the ones buildToolkitInfoList itself reads. createToolkitJson("weaviateapi", { id: "WeaviateApi", label: "Weaviate API", + version: "1.0.0", + description: null, + auth: null, + tools: [], metadata: { category: "databases", docsLink: "https://docs.arcade.dev/en/mcp-servers/databases/weaviate-api", + iconUrl: "https://design-system.arcade.dev/icons/placeholder.svg", + isBYOC: false, + isPro: false, + type: "arcade", + isComingSoon: false, + isHidden: false, }, }); From b584d5e64b3af4b30f7b93521d3f8a32229da12d Mon Sep 17 00:00:00 2001 From: Teal Larson Date: Mon, 3 Aug 2026 16:36:35 -0400 Subject: [PATCH 10/17] fix: harden toolkit data loading and parity checks --- app/_lib/toolkit-data.ts | 60 ++++++++++------- tests/toolkit-data-cache.test.ts | 67 +++++++++++++------ tests/toolkit-data-parity.test.ts | 39 +++++++---- .../tests/app-lib/toolkit-data.test.ts | 1 + .../app-lib/toolkit-static-params.test.ts | 22 ++++-- 5 files changed, 123 insertions(+), 66 deletions(-) diff --git a/app/_lib/toolkit-data.ts b/app/_lib/toolkit-data.ts index f724b03b2..27a9b1c3e 100644 --- a/app/_lib/toolkit-data.ts +++ b/app/_lib/toolkit-data.ts @@ -15,7 +15,7 @@ import { import { MergedToolkitSchema, type ToolkitIndexEntrySchema, - type ToolkitIndexSchema, + ToolkitIndexSchema, } from "@/toolkit-docs-generator/src/shared/toolkit-schemas"; /** @@ -121,17 +121,14 @@ type ToolkitDataMap = { }; /** - * One process-wide load per data directory. Keyed by directory (not a single - * flat variable) because tests point `TOOLKIT_DATA_DIR` at scratch fixtures - * and must not see another test's cached data. - * - * A failed load (a corrupt file — see readToolkitFile) is kept in this map - * rather than retried: the underlying files are static build output that - * only change on a new deploy, so a bad file stays bad for the rest of this - * process's life, and re-scanning 21.6 MB on every subsequent lookup hoping - * it healed itself would only add cost without ever succeeding. + * The production data directory is immutable for the lifetime of a process, + * so retain one successful load for it. Explicit fixture/override directories + * are intentionally not retained here: callers can point them at arbitrary + * paths, and keeping every path would turn test and dev runs into an + * unbounded process-global cache. */ const loadsByDataDir = new Map>(); +const DEFAULT_DATA_DIR = resolveToolkitDataDir(); const loadAllToolkitDataUncached = async ( dataDir: string @@ -187,10 +184,23 @@ const loadAllToolkitDataUncached = async ( */ export const loadAllToolkitData = cache( async (dataDir: string): Promise => { + if (dataDir !== DEFAULT_DATA_DIR) { + return await loadAllToolkitDataUncached(dataDir); + } + let promise = loadsByDataDir.get(dataDir); if (!promise) { promise = loadAllToolkitDataUncached(dataDir); loadsByDataDir.set(dataDir, promise); + + // A transient read or deployment error must not poison the process + // forever. Keep successful loads warm, but allow the next lookup to + // retry after a failed load. + promise.catch(() => { + if (loadsByDataDir.get(dataDir) === promise) { + loadsByDataDir.delete(dataDir); + } + }); } return await promise; } @@ -209,6 +219,16 @@ export const readToolkitData = async ( } const dataDir = resolveDataDir(options); + // The API route normally receives the normalized toolkit id. Keep that + // common path O(1), especially on a cold serverless instance: eagerly + // loading every toolkit JSON file just to serve one toolkit adds tens of + // megabytes of parsing and memory overhead. The full directory index below + // is reserved for slug lookups and build-time enumeration. + const direct = await readToolkitFile(join(dataDir, `${normalizedId}.json`)); + if (direct) { + return direct; + } + const { byNormalizedId, bySlug } = await loadAllToolkitData(dataDir); return ( @@ -244,24 +264,12 @@ export const readToolkitIndex = async ( ); } - // Deliberately looser than a full ToolkitIndexSchema.safeParse: unlike - // per-toolkit data, entries here are only ever used to look up a - // toolkit's id/category, with the toolkit's own JSON file as the real - // source of truth (see resolveToolkitRoute in toolkit-static-params.ts). - // Rejecting the whole index over one entry missing a field the callers - // don't read would cost every route on the site, not just one page. But - // the file as a whole not even having the shape of an index is - // corruption, not a missing-field nuance, so that still throws. - if ( - typeof parsed !== "object" || - parsed === null || - !("toolkits" in parsed) || - !Array.isArray((parsed as { toolkits: unknown }).toolkits) - ) { + const result = ToolkitIndexSchema.safeParse(parsed); + if (!result.success) { throw new Error( - `Invalid toolkit index shape in ${filePath}: expected an object with a "toolkits" array.` + `Invalid toolkit index schema in ${filePath}: ${result.error.message}` ); } - return parsed as ToolkitIndex; + return result.data; }; diff --git a/tests/toolkit-data-cache.test.ts b/tests/toolkit-data-cache.test.ts index 45d9ac618..ede8c38e5 100644 --- a/tests/toolkit-data-cache.test.ts +++ b/tests/toolkit-data-cache.test.ts @@ -2,7 +2,7 @@ import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { afterAll, describe, expect, test } from "vitest"; -import { readToolkitData } from "@/app/_lib/toolkit-data"; +import { readToolkitData, readToolkitIndex } from "@/app/_lib/toolkit-data"; /** * loadAllToolkitData (app/_lib/toolkit-data.ts) reads and validates every @@ -90,29 +90,56 @@ describe("readToolkitData against a directory with one corrupt file", () => { ).rejects.toThrow(join(dataDir, "corrupttoolkit.json")); }); - test("the failure is cached, not retried: a second request throws the same way", async () => { - // Confirms the deliberate choice to cache a failed load rather than - // re-scanning the directory on every subsequent call: this directory's - // corruption doesn't heal between calls, so re-reading it every time - // would only add cost without ever succeeding. + test("a transient scan failure can recover after the file is repaired", async () => { await expect( readToolkitData("CorruptToolkit", { dataDir }) ).rejects.toThrow(join(dataDir, "corrupttoolkit.json")); + + writeFileSync( + join(dataDir, "corrupttoolkit.json"), + validToolkitJson("RecoveredToolkit", "recovered-toolkit") + ); + + const recovered = await readToolkitData("recovered-toolkit", { dataDir }); + expect(recovered?.id).toBe("RecoveredToolkit"); }); - // A pre-existing property of the old scan-on-miss implementation too, not - // a regression introduced by the shared cache: any lookup that needs to - // rule out every file in the directory (a genuinely absent id, or a slug - // reached only via the full scan) surfaces a sibling file's corruption, - // because "is this id absent" can't be answered without reading everything. - // A healthy toolkit's *direct* id-shaped lookup, though, is unaffected by - // corruption elsewhere in the directory only when that toolkit was already - // resident in a load that happened before the corruption — once the whole - // directory's load has failed once, it stays failed (see the caching test - // above), so every subsequent lookup against this dataDir throws too. - test("a healthy toolkit id in the same directory also throws once the directory load has failed", async () => { - await expect( - readToolkitData("ValidToolkitOne", { dataDir }) - ).rejects.toThrow(join(dataDir, "corrupttoolkit.json")); + test("a healthy toolkit id still uses the direct file after a failed scan", async () => { + const data = await readToolkitData("ValidToolkitOne", { dataDir }); + expect(data?.id).toBe("ValidToolkitOne"); + }); +}); + +describe("readToolkitData direct-file fast path", () => { + const dataDir = makeFixtureDir(); + dirsToClean.push(dataDir); + writeFileSync( + join(dataDir, "corrupttoolkit.json"), + "{ this is not valid json" + ); + + test("a normalized id does not scan or parse corrupt sibling files", async () => { + const data = await readToolkitData("ValidToolkitOne", { dataDir }); + expect(data?.id).toBe("ValidToolkitOne"); + }); +}); + +describe("readToolkitIndex schema validation", () => { + const dataDir = mkdtempSync(join(tmpdir(), "toolkit-index-schema-test-")); + dirsToClean.push(dataDir); + + test("rejects malformed index entries instead of casting them", async () => { + writeFileSync( + join(dataDir, "index.json"), + JSON.stringify({ + generatedAt: "2026-01-01T00:00:00Z", + version: "1", + toolkits: [{ id: "missing-required-fields" }], + }) + ); + + await expect(readToolkitIndex({ dataDir })).rejects.toThrow( + join(dataDir, "index.json") + ); }); }); diff --git a/tests/toolkit-data-parity.test.ts b/tests/toolkit-data-parity.test.ts index ef4d5cbb2..870bc29c0 100644 --- a/tests/toolkit-data-parity.test.ts +++ b/tests/toolkit-data-parity.test.ts @@ -4,6 +4,7 @@ import { describe, expect, test } from "vitest"; import { readToolkitFile, readToolkitIndex } from "@/app/_lib/toolkit-data"; import { listToolkitRoutes } from "@/app/_lib/toolkit-static-params"; import { resolveToolkitDataDir } from "@/toolkit-docs-generator/src/shared/toolkit-data-dir"; +import { getToolkitSlug } from "@/toolkit-docs-generator/src/shared/toolkit-primitives"; // resolveToolkitDataDir defaults to the real committed data, but also honors // TOOLKIT_DATA_DIR (same as readToolkitIndex/listToolkitRoutes below), so @@ -33,26 +34,38 @@ describe("toolkit data parity", () => { const toolkits = await Promise.all( jsonFileNames.map((file) => readToolkitFile(join(DATA_DIR, file))) ); - const parseableCount = toolkits.filter( - (toolkit) => toolkit !== null - ).length; + const parseableToolkits = toolkits.filter((toolkit) => toolkit !== null); // Every file on disk should be a real, schema-valid toolkit: no file // silently failed to parse into null. - expect(parseableCount).toBe(jsonFileNames.length); + expect(parseableToolkits).toHaveLength(jsonFileNames.length); - // index.json is regenerated alongside the per-toolkit files, so its - // entry count should match the file count exactly. - expect(index?.toolkits.length).toBe(parseableCount); + // index.json is regenerated alongside the per-toolkit files. Compare the + // actual IDs, not only counts, so a missing file replaced by a different + // file cannot make this check pass. + const indexIds = new Set(index?.toolkits.map((toolkit) => toolkit.id)); + const fileIds = new Set(parseableToolkits.map((toolkit) => toolkit.id)); + expect(indexIds).toEqual(fileIds); // Routes exclude hidden toolkits (they're intentionally unrouted, not - // corrupt), so compare against the non-hidden subset rather than the - // raw file count. - const visibleCount = toolkits.filter( - (toolkit) => toolkit && !toolkit.metadata?.isHidden - ).length; + // corrupt), so compare their complete category/slug identities rather + // than only the visible count. + const expectedRoutes = new Set( + parseableToolkits + .filter((toolkit) => !toolkit.metadata.isHidden) + .map((toolkit) => { + const slug = getToolkitSlug({ + id: toolkit.id, + docsLink: toolkit.metadata.docsLink, + }); + return `${toolkit.metadata.category}/${slug}`; + }) + ); const routes = await listToolkitRoutes(); - expect(routes.length).toBe(visibleCount); + const actualRoutes = new Set( + routes.map((route) => `${route.category}/${route.toolkitId}`) + ); + expect(actualRoutes).toEqual(expectedRoutes); }); }); diff --git a/toolkit-docs-generator/tests/app-lib/toolkit-data.test.ts b/toolkit-docs-generator/tests/app-lib/toolkit-data.test.ts index fac395e4d..af99af5e1 100644 --- a/toolkit-docs-generator/tests/app-lib/toolkit-data.test.ts +++ b/toolkit-docs-generator/tests/app-lib/toolkit-data.test.ts @@ -56,6 +56,7 @@ describe("toolkit data loader", () => { label: "GitHub", version: "1.0.0", category: "development", + type: "arcade", toolCount: 3, authType: "oauth2", }, diff --git a/toolkit-docs-generator/tests/app-lib/toolkit-static-params.test.ts b/toolkit-docs-generator/tests/app-lib/toolkit-static-params.test.ts index 55a5a92c0..040e49c34 100644 --- a/toolkit-docs-generator/tests/app-lib/toolkit-static-params.test.ts +++ b/toolkit-docs-generator/tests/app-lib/toolkit-static-params.test.ts @@ -22,11 +22,20 @@ const writeIndex = async ( dir: string, toolkits: Array<{ id: string; category?: string }> ) => { + const entries = toolkits.map((toolkit) => ({ + id: toolkit.id, + label: toolkit.id, + version: "1.0.0", + category: toolkit.category ?? "development", + type: "arcade", + toolCount: 0, + authType: "none", + })); const indexFixture = JSON.stringify( { generatedAt: "2026-01-15T00:00:00.000Z", version: "1.0.0", - toolkits, + toolkits: entries, }, null, 2 @@ -248,21 +257,20 @@ describe("toolkit static params", () => { }); }); - it('throws on an unrecognized category instead of coercing it to "others"', async () => { + it("rejects an unrecognized category in the index schema", async () => { await withTempDir(async (dir) => { await writeIndex(dir, [{ id: "Github", category: "weird" }]); await expect( listToolkitRoutes({ dataDir: dir, toolkitsCatalog: [] }) - ).rejects.toThrow(/weird/); + ).rejects.toThrow(/Invalid toolkit index schema/); }); }); - it("skips a toolkit with no category anywhere instead of routing it to a fake bucket", async () => { + it("skips a toolkit with no category when the index is absent", async () => { await withTempDir(async (dir) => { - // No JSON file, no catalog entry, and the index entry itself omits - // category — nothing to route this toolkit under. - await writeIndex(dir, [{ id: "Github" }]); + // No JSON file, no catalog entry, and no index — nothing to route this + // toolkit under. const routes = await listToolkitRoutes({ dataDir: dir, From dacb402af294a57b96996d9345dd55b8ea2eca02 Mon Sep 17 00:00:00 2001 From: Teal Larson Date: Tue, 4 Aug 2026 10:26:52 -0400 Subject: [PATCH 11/17] fix: remove fallback integration category --- app/_lib/integration-index.ts | 19 ++++++++----- .../integrations/components/tool-card.tsx | 4 +-- .../components/toolkits-client.tsx | 2 +- tests/integration-index-links.test.ts | 16 ++++++++--- tests/toolkit-data-cache.test.ts | 7 +++++ .../scripts/sync-toolkit-sidebar.ts | 19 ++++++++++--- .../scripts/sync-toolkit-sidebar.test.ts | 27 +++++++++++-------- 7 files changed, 66 insertions(+), 28 deletions(-) diff --git a/app/_lib/integration-index.ts b/app/_lib/integration-index.ts index 0e062a54a..036d7f3d9 100644 --- a/app/_lib/integration-index.ts +++ b/app/_lib/integration-index.ts @@ -12,9 +12,12 @@ export function toIntegrationLink(toolkit: { id: string; docsLink?: string | null; category?: string | null; -}): string { +}): string | null { const slug = getToolkitSlug({ id: toolkit.id, docsLink: toolkit.docsLink }); - const category = toolkit.category ?? "others"; + const category = toolkit.category; + if (!category) { + return null; + } return `${INTEGRATIONS_BASE}/${category}/${slug}`; } @@ -49,18 +52,20 @@ export function resolveIndexToolkits( } const link = toIntegrationLink(toolkit); - const hasPage = validLinks.has(link); + const hasPage = link !== null && validLinks.has(link); // A bare duplicate of a real "-api" toolkit: drop it; the real card stays. - if (!hasPage && validLinks.has(`${link}-api`)) { + if (link && !hasPage && validLinks.has(`${link}-api`)) { continue; } // Collapse multiple catalog entries that point at the same URL. - if (seen.has(link)) { - continue; + if (link) { + if (seen.has(link)) { + continue; + } + seen.add(link); } - seen.add(link); resolved.push({ ...toolkit, hasPage }); } diff --git a/app/en/resources/integrations/components/tool-card.tsx b/app/en/resources/integrations/components/tool-card.tsx index be92ac880..2473c89c4 100644 --- a/app/en/resources/integrations/components/tool-card.tsx +++ b/app/en/resources/integrations/components/tool-card.tsx @@ -21,7 +21,7 @@ type ToolCardProps = { name: string; icon?: React.ComponentType> | null; iconUrl?: string; - link: string; + link?: string; type: ToolkitType; isComingSoon?: boolean; isByoc?: boolean; @@ -152,7 +152,7 @@ export const ToolCard: React.FC = ({ return ( <> - {isComingSoon ? ( + {isComingSoon || !link ? (