diff --git a/README.md b/README.md index 1c83ca4..6ca905a 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # opencode-firecrawl -OpenCode plugin for [Firecrawl](https://firecrawl.dev) — gives your AI agent reliable web scraping, crawling, and search via the [Firecrawl CLI](https://github.com/firecrawl/cli). +OpenCode plugin for [Firecrawl](https://firecrawl.dev) — gives your AI agent primary-source answers from the Firecrawl developer index, plus reliable web scraping, crawling, and search via the [Firecrawl CLI](https://github.com/firecrawl/cli). ## Installation @@ -13,7 +13,9 @@ Add the plugin to your `opencode.json`: } ``` -Then install the Firecrawl CLI globally: +The `firecrawl_developer_search` tool works as soon as the plugin loads, with no API key and no CLI. + +For the web tools (scrape, crawl, map, search, agent), install the Firecrawl CLI globally: ```bash npm install -g firecrawl-cli @@ -35,6 +37,20 @@ Get an API key at [firecrawl.dev](https://firecrawl.dev). If `FIRECRAWL_API_KEY` is set in your environment, the plugin automatically passes it to shell commands. +## Developer index + +The plugin adds a `firecrawl_developer_search` tool that searches a curated index of GitHub issues, merged pull requests, READMEs, and library documentation, and returns the **matched passages** as markdown rather than a list of links. + +Use it when the question is how a library behaves, what an error means, whether a bug was fixed, or what an API contract guarantees. The agent gets to answer from the issue that reported the bug, the pull request that fixed it, or the doc page that defines the contract. + +``` +Why does my Playwright script hang on page.goto with a service worker registered? +``` + +The tool takes a `query` plus optional `types` (`doc`, `issue`, `pull_request`, `readme`), `repos` (`owner/name`), `sources`, `k`, and `passages`. It needs no API key; setting `FIRECRAWL_API_KEY` only raises the rate limit. The same index is available from the shell as `firecrawl developer `. + +The bundled `firecrawl-developer-index` skill teaches the agent which questions belong in the index, how to shape a query for an error string versus an API contract, and when to fall back to the open web instead. + ## What it does This plugin registers the Firecrawl CLI skill with OpenCode. Once installed, the agent can: @@ -44,6 +60,7 @@ This plugin registers the Firecrawl CLI skill with OpenCode. Once installed, the - **Map** all URLs on a website - **Crawl** entire websites recursively - **Agent** — AI-powered autonomous web data extraction +- **Developer search** — issues, merged PRs, READMEs, and docs, with the matched passages All output is written to a `.firecrawl/` directory to avoid flooding context. diff --git a/developer-search.ts b/developer-search.ts new file mode 100644 index 0000000..de07a48 --- /dev/null +++ b/developer-search.ts @@ -0,0 +1,154 @@ +import { tool } from "@opencode-ai/plugin"; + +const ENDPOINT = "https://api.firecrawl.dev/v2/search/developer"; +const TIMEOUT_MS = 30_000; + +type Passage = { text?: string; citation_url?: string }; + +type SearchResult = { + id?: string; + url?: string; + title?: string; + license?: string; + passages?: Passage[]; +}; + +type SearchResponse = { + success?: boolean; + partial?: boolean; + results?: SearchResult[]; + repos?: { repo?: string; indexed?: boolean }[]; + sources?: { source?: string; indexed?: boolean }[]; + error?: string; +}; + +type Scope = { repos?: string[]; sources?: string[] }; + +/** + * A scope the index doesn't hold can never match, so no rephrasing will help. + * Unknown repos are absent from the echo entirely while unknown sources come + * back with `indexed: false`, so treat "missing" and "not indexed" alike. + */ +function unmatchableScopes(response: SearchResponse, scope: Scope) { + const indexed = (echo: { indexed?: boolean } | undefined) => echo !== undefined && echo.indexed !== false; + const missing = [ + ...(scope.repos ?? []) + .filter((repo) => !indexed(response.repos?.find((echo) => echo.repo === repo))) + .map((repo) => `repo ${repo}`), + ...(scope.sources ?? []) + .filter((source) => !indexed(response.sources?.find((echo) => echo.source === source))) + .map((source) => `source ${source}`), + ]; + if (missing.length === 0) return []; + return [ + `Not in the developer index, so no query scoped to ${missing.length > 1 ? "them" : "it"} can ever match: ${missing.join(", ")}. Drop the scope and search the whole index, or use the Firecrawl CLI for the open web.`, + ]; +} + +function render(response: SearchResponse, scope: Scope) { + const results = response.results ?? []; + const notes = [ + ...unmatchableScopes(response, scope), + response.partial ? "The index returned a partial result set." : undefined, + ].filter((note): note is string => Boolean(note)); + + if (results.length === 0) { + return ["No results.", ...notes].join("\n\n"); + } + + const blocks = results.map((result, index) => { + const heading = `## ${index + 1}. ${result.title ?? result.url ?? result.id ?? "untitled"}`; + const meta = [ + // A doc id is just the url with a prefix, so printing both wastes context. + result.id && !result.id.startsWith("doc:") && `id: ${result.id}`, + result.url && `url: ${result.url}`, + result.license && `license: ${result.license}`, + ] + .filter(Boolean) + .join("\n"); + const passages = (result.passages ?? []) + .map((passage) => passage.text?.trim()) + .filter(Boolean) + .join("\n\n---\n\n"); + return [heading, meta, passages || "(no passage returned)"].filter(Boolean).join("\n\n"); + }); + + return [...blocks, ...notes].join("\n\n"); +} + +export const developerSearch = tool({ + description: `Search a curated index of GitHub issues, merged pull requests, READMEs, and library documentation, returning the matched passages as markdown. + +Reach for this before a web search whenever the question is how a library or API behaves, what an error message means, whether a bug was fixed, or what an API contract guarantees. It answers from the primary source: the issue where the bug was reported, the merged PR that fixed it, the doc page that defines the contract. A blog post describing a behaviour is a weaker answer than the passage defining it. + +Matching the query to the question: +- Literal error string or stack trace: search the string plus the library name with types ["issue", "pull_request"]. If nothing matches, strip the volatile parts (paths, line numbers, ids) and retry; the invariant middle of the message is what is indexed. +- Conceptual "how do I do X": ask the full question in natural language across all types. Raise passages before raising k. +- Known bug: the issue reports it, the merged pull request fixes it, and the fix is what you want. Search ["issue", "pull_request"], then re-query the issue's own terms scoped to its repo with types ["pull_request"]. +- API contract ("what does X return", "is Y required", "what is the default"): types ["readme", "doc"]. +- Version-specific behaviour: an issue's opening report describes the broken version and its resolution supersedes it, so read the resolution before answering. +- Scoped to one library: repos ["owner/name"]. If a scoped search comes back empty, read the "Not indexed" note before rephrasing. + +Search broadly first, then narrow with types or repos once you have seen what the hits look like; scoping first hides the result that would have told you where to look. Quote the passages and cite the url, falling back to the url when a doc result has no title. When the index has nothing to say, comparisons, opinion, news, or an unindexed project, use the Firecrawl CLI to search or scrape the open web instead.`, + args: { + query: tool.schema + .string() + .describe("The developer question, literal error string, or API contract to look up"), + k: tool.schema.number().min(1).max(100).default(10).describe("Number of results to return"), + types: tool.schema + .array(tool.schema.enum(["doc", "issue", "pull_request", "readme"])) + .optional() + .describe( + "Artifact kinds to search. Defaults to all four; narrowing here is the cheapest way to sharpen a query", + ), + repos: tool.schema + .array(tool.schema.string()) + .optional() + .describe("Scope the repository half (issue, pull_request, readme) to these owner/name slugs"), + sources: tool.schema + .array(tool.schema.string()) + .optional() + .describe( + "Scope the documentation half (doc) to these source ids, at most 20. Unions with repos rather than intersecting", + ), + passages: tool.schema + .number() + .min(1) + .max(5) + .default(1) + .describe( + "Maximum passages per result. Raise when a page is clearly right but the first passage is the wrong part of it", + ), + }, + async execute(args, context) { + const timeout = AbortSignal.timeout(TIMEOUT_MS); + const response = await fetch(ENDPOINT, { + method: "POST", + signal: AbortSignal.any([context.abort, timeout]), + headers: { + "Content-Type": "application/json", + // Keyless by default; a key only raises the rate limit. + ...(process.env.FIRECRAWL_API_KEY && { + Authorization: `Bearer ${process.env.FIRECRAWL_API_KEY}`, + }), + }, + body: JSON.stringify(args), + }); + + const body = (await response.json().catch(() => undefined)) as SearchResponse | undefined; + + if (!response.ok) { + const detail = body?.error ? `: ${body.error}` : ""; + throw new Error(`Developer index search failed with ${response.status} ${response.statusText}${detail}`); + } + + return { + title: args.query, + output: render(body ?? {}, args), + metadata: { + count: body?.results?.length ?? 0, + partial: body?.partial ?? false, + }, + }; + }, +}); diff --git a/index.ts b/index.ts index 849a0a4..ce45dce 100644 --- a/index.ts +++ b/index.ts @@ -1,6 +1,7 @@ import type { Plugin } from "@opencode-ai/plugin"; import { dirname, join } from "node:path"; import { fileURLToPath } from "node:url"; +import { developerSearch } from "./developer-search.ts"; const current_dir = dirname(fileURLToPath(import.meta.url)); @@ -26,6 +27,10 @@ export const plugin: Plugin = async () => { output.env.FIRECRAWL_API_KEY = process.env.FIRECRAWL_API_KEY; } }, + + tool: { + firecrawl_developer_search: developerSearch, + }, }; }; diff --git a/package-lock.json b/package-lock.json index 515f0c6..0780040 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,34 +1,159 @@ { "name": "opencode-firecrawl", - "version": "1.0.0", + "version": "1.1.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "opencode-firecrawl", - "version": "1.0.0", + "version": "1.1.0", "license": "ISC", "devDependencies": { - "@opencode-ai/plugin": "^1.0.0", + "@opencode-ai/plugin": "^1.18.25", "@types/node": "^25.2.2", "typescript": "^5.0.0" } }, + "node_modules/@ai-sdk/provider": { + "version": "3.0.8", + "resolved": "https://registry.npmjs.org/@ai-sdk/provider/-/provider-3.0.8.tgz", + "integrity": "sha512-oGMAgGoQdBXbZqNG0Ze56CHjDZ1IDYOwGYxYjO5KLSlz5HiNQ9udIXsPZ61VWaHGZ5XW/jyjmr6t2xz2jGVwbQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "json-schema": "^0.4.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@msgpackr-extract/msgpackr-extract-darwin-arm64": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-darwin-arm64/-/msgpackr-extract-darwin-arm64-3.0.4.tgz", + "integrity": "sha512-LCkGo6JDfaBhgST7UpPWgNgLINpcpabaHfyz5OBx75nUYxBsaEPxjnyNjWpeb/xBup/682QnBfRBy2/LvPutZQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@msgpackr-extract/msgpackr-extract-darwin-x64": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-darwin-x64/-/msgpackr-extract-darwin-x64-3.0.4.tgz", + "integrity": "sha512-zExlW9zUJKZH/tOtVMttwjKa4Xm/3KcNjnE3dPN92uCktwavMxpgCA3MoJK/DOnTWsQgo224OaST27/mPNAf+w==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@msgpackr-extract/msgpackr-extract-linux-arm": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-arm/-/msgpackr-extract-linux-arm-3.0.4.tgz", + "integrity": "sha512-Tg3yX65f5GbtXLkrYEHE5oibZG9epyYWas7FogTTEJeDEF9JlXJzKgXaNhT3UXlTOeA+AfZpYZYZ0uPj7Cfquw==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@msgpackr-extract/msgpackr-extract-linux-arm64": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-arm64/-/msgpackr-extract-linux-arm64-3.0.4.tgz", + "integrity": "sha512-dgX0P/9wGPJeHFBG+ZmhgE6bmtMt7NP5CRBGyyktpopdk/mW4POnrpQsSLtKI1dwpc+pPLuXHDh6vvskyQE/sw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@msgpackr-extract/msgpackr-extract-linux-x64": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-x64/-/msgpackr-extract-linux-x64-3.0.4.tgz", + "integrity": "sha512-8TNXMEjJc3QEy7R/x1INhgiU+XakDAFUzBhaz7+Rbrs8NH5UQeHQxxmzsSBJGyV6I1jW79undiQm8tOI+D+8FQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@msgpackr-extract/msgpackr-extract-win32-x64": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-win32-x64/-/msgpackr-extract-win32-x64-3.0.4.tgz", + "integrity": "sha512-CmCXPQrkbwExx3j946/PtHWHbYJiCRBRDl4BlkRQcJB/YOwQxJRTpoo7aTsortjgoJ1x7opzTSxn7C+ASSLVjQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, "node_modules/@opencode-ai/plugin": { - "version": "1.1.53", - "resolved": "https://registry.npmjs.org/@opencode-ai/plugin/-/plugin-1.1.53.tgz", - "integrity": "sha512-9ye7Wz2kESgt02AUDaMea4hXxj6XhWwKAG8NwFhrw09Ux54bGaMJFt1eIS8QQGIMaD+Lp11X4QdyEg96etEBJw==", + "version": "1.18.25", + "resolved": "https://registry.npmjs.org/@opencode-ai/plugin/-/plugin-1.18.25.tgz", + "integrity": "sha512-Kb34zFqYosFNiMd1IuYiZGjX17z+18Srm7tHZMCz+uMVRTYNkEw1FTrfAK2FLbggwYdgzifGwKMNF1slLT8eLw==", "dev": true, "license": "MIT", "dependencies": { - "@opencode-ai/sdk": "1.1.53", + "@ai-sdk/provider": "3.0.8", + "@opencode-ai/sdk": "1.18.25", + "effect": "4.0.0-beta.83", "zod": "4.1.8" + }, + "peerDependencies": { + "@opentui/core": ">=0.4.5", + "@opentui/keymap": ">=0.4.5", + "@opentui/solid": ">=0.4.5" + }, + "peerDependenciesMeta": { + "@opentui/core": { + "optional": true + }, + "@opentui/keymap": { + "optional": true + }, + "@opentui/solid": { + "optional": true + } } }, "node_modules/@opencode-ai/sdk": { - "version": "1.1.53", - "resolved": "https://registry.npmjs.org/@opencode-ai/sdk/-/sdk-1.1.53.tgz", - "integrity": "sha512-RUIVnPOP1CyyU32FrOOYuE7Ge51lOBuhaFp2NSX98ncApT7ffoNetmwzqrhOiJQgZB1KrbCHLYOCK6AZfacxag==", + "version": "1.18.25", + "resolved": "https://registry.npmjs.org/@opencode-ai/sdk/-/sdk-1.18.25.tgz", + "integrity": "sha512-GwgwhW+vE8FWSDw730SjzqNhsWXB0uJjbFOiqFkmM+USFuG13HuTlGe6SR2ixt+WXxoD6FV1hILWqsXyqej9hQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "cross-spawn": "7.0.6" + } + }, + "node_modules/@standard-schema/spec": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@standard-schema/spec/-/spec-1.1.0.tgz", + "integrity": "sha512-l2aFy5jALhniG5HgqrD6jXLi/rUWrKvqN/qJx6yoJsgKhblVd+iqqU4RCXavm/jPityDo5TCvKMnpjKnOriy0w==", "dev": true, "license": "MIT" }, @@ -42,6 +167,228 @@ "undici-types": "~7.16.0" } }, + "node_modules/cross-spawn": { + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", + "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==", + "dev": true, + "license": "MIT", + "dependencies": { + "path-key": "^3.1.0", + "shebang-command": "^2.0.0", + "which": "^2.0.1" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/detect-libc": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.1.2.tgz", + "integrity": "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==", + "dev": true, + "license": "Apache-2.0", + "optional": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/effect": { + "version": "4.0.0-beta.83", + "resolved": "https://registry.npmjs.org/effect/-/effect-4.0.0-beta.83.tgz", + "integrity": "sha512-0wsak8RtgGAr9UWSbVDgJHZcUqMSvicHcvaZv1MbMM7MCGgW4Rn/137J1MHQbwYPcwYGxT/IqehFd+UbYuj78w==", + "dev": true, + "license": "MIT", + "dependencies": { + "@standard-schema/spec": "^1.1.0", + "fast-check": "^4.8.0", + "find-my-way-ts": "^0.1.6", + "ini": "^7.0.0", + "kubernetes-types": "^1.30.0", + "msgpackr": "^2.0.1", + "multipasta": "^0.2.7", + "toml": "^4.1.1", + "uuid": "^14.0.0", + "yaml": "^2.9.0" + } + }, + "node_modules/fast-check": { + "version": "4.9.0", + "resolved": "https://registry.npmjs.org/fast-check/-/fast-check-4.9.0.tgz", + "integrity": "sha512-7ms6T7SybUev/PQITciI0yLM2pOSFy5zpG8Ty7tQofcVaQUvrMXp6CBwqF6fThLCLOrfBtuHAtwq6Yu4XPCllg==", + "dev": true, + "funding": [ + { + "type": "individual", + "url": "https://github.com/sponsors/dubzzz" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/fast-check" + } + ], + "license": "MIT", + "dependencies": { + "pure-rand": "^8.0.0" + }, + "engines": { + "node": ">=12.17.0" + } + }, + "node_modules/find-my-way-ts": { + "version": "0.1.6", + "resolved": "https://registry.npmjs.org/find-my-way-ts/-/find-my-way-ts-0.1.6.tgz", + "integrity": "sha512-a85L9ZoXtNAey3Y6Z+eBWW658kO/MwR7zIafkIUPUMf3isZG0NCs2pjW2wtjxAKuJPxMAsHUIP4ZPGv0o5gyTA==", + "dev": true, + "license": "MIT" + }, + "node_modules/ini": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/ini/-/ini-7.0.0.tgz", + "integrity": "sha512-ifK0CgjALofS5bkrcTy4RaQ9Vx2Knf/eLeIO+NaswQEpH1UblrtTSCIvN71qQDMq0PeQ/SSPojvEJp9vvvfr+w==", + "dev": true, + "license": "ISC", + "engines": { + "node": "^22.22.2 || ^24.15.0 || >=26.0.0" + } + }, + "node_modules/isexe": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", + "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", + "dev": true, + "license": "ISC" + }, + "node_modules/json-schema": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/json-schema/-/json-schema-0.4.0.tgz", + "integrity": "sha512-es94M3nTIfsEPisRafak+HDLfHXnKBhV3vU5eqPcS3flIWqcxJWgXHXiey3YrpaNsanY5ei1VoYEbOzijuq9BA==", + "dev": true, + "license": "(AFL-2.1 OR BSD-3-Clause)" + }, + "node_modules/kubernetes-types": { + "version": "1.30.0", + "resolved": "https://registry.npmjs.org/kubernetes-types/-/kubernetes-types-1.30.0.tgz", + "integrity": "sha512-Dew1okvhM/SQcIa2rcgujNndZwU8VnSapDgdxlYoB84ZlpAD43U6KLAFqYo17ykSFGHNPrg0qry0bP+GJd9v7Q==", + "dev": true, + "license": "Apache-2.0" + }, + "node_modules/msgpackr": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/msgpackr/-/msgpackr-2.1.0.tgz", + "integrity": "sha512-p/pBCVO63CsvvpkomUnNNag6+n38rULuDA6HHe70o2gtC8ODI52foF/4ko2qQcp6OiErJXTmrZeXmsGGHsIQNQ==", + "dev": true, + "license": "MIT", + "optionalDependencies": { + "msgpackr-extract": "^3.0.4" + } + }, + "node_modules/msgpackr-extract": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/msgpackr-extract/-/msgpackr-extract-3.0.4.tgz", + "integrity": "sha512-4kmO/MdyUIkLIvTPr8VHLil4AtoKIoniWPIEk5+CDy0xnWC84azhSFmuJ7PxZdsYtiP5kEeQsORAVIeMgxT+Hw==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "dependencies": { + "node-gyp-build-optional-packages": "5.2.2" + }, + "bin": { + "download-msgpackr-prebuilds": "bin/download-prebuilds.js" + }, + "optionalDependencies": { + "@msgpackr-extract/msgpackr-extract-darwin-arm64": "3.0.4", + "@msgpackr-extract/msgpackr-extract-darwin-x64": "3.0.4", + "@msgpackr-extract/msgpackr-extract-linux-arm": "3.0.4", + "@msgpackr-extract/msgpackr-extract-linux-arm64": "3.0.4", + "@msgpackr-extract/msgpackr-extract-linux-x64": "3.0.4", + "@msgpackr-extract/msgpackr-extract-win32-x64": "3.0.4" + } + }, + "node_modules/multipasta": { + "version": "0.2.8", + "resolved": "https://registry.npmjs.org/multipasta/-/multipasta-0.2.8.tgz", + "integrity": "sha512-ZPWuMKyv0cSO29f7hozp+k6+crZbQijV8ipMvxNxRf2SwtYGTX1ZX89Kd20VV4H9Znonx+EQn+iy1wGQsJ+b+Q==", + "dev": true, + "license": "MIT" + }, + "node_modules/node-gyp-build-optional-packages": { + "version": "5.2.2", + "resolved": "https://registry.npmjs.org/node-gyp-build-optional-packages/-/node-gyp-build-optional-packages-5.2.2.tgz", + "integrity": "sha512-s+w+rBWnpTMwSFbaE0UXsRlg7hU4FjekKU4eyAih5T8nJuNZT1nNsskXpxmeqSK9UzkBl6UgRlnKc8hz8IEqOw==", + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "detect-libc": "^2.0.1" + }, + "bin": { + "node-gyp-build-optional-packages": "bin.js", + "node-gyp-build-optional-packages-optional": "optional.js", + "node-gyp-build-optional-packages-test": "build-test.js" + } + }, + "node_modules/path-key": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", + "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/pure-rand": { + "version": "8.4.2", + "resolved": "https://registry.npmjs.org/pure-rand/-/pure-rand-8.4.2.tgz", + "integrity": "sha512-vvuOGgcuPJAirlHvuQw1TrOiw7ptaIXXmIbNuiNOY6lNGJJH49PQ1Kj4nd783nPdQhQdicgOjVI2yI/9BD6/Ng==", + "dev": true, + "funding": [ + { + "type": "individual", + "url": "https://github.com/sponsors/dubzzz" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/fast-check" + } + ], + "license": "MIT" + }, + "node_modules/shebang-command": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", + "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", + "dev": true, + "license": "MIT", + "dependencies": { + "shebang-regex": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/shebang-regex": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", + "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/toml": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/toml/-/toml-4.3.0.tgz", + "integrity": "sha512-lVb8X9BsPVuH0M4BKeS91tXAmJvCjQ5UIyAbQFaxkKGyUFK2RPkhwaFSQH8vbpl1d23eu/IBH+dwVMHWaq9A5A==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=20" + } + }, "node_modules/typescript": { "version": "5.9.3", "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", @@ -63,6 +410,52 @@ "dev": true, "license": "MIT" }, + "node_modules/uuid": { + "version": "14.0.2", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-14.0.2.tgz", + "integrity": "sha512-xZe/16rV4aa+HGSOCiY2YeLT1OybRLrrkL/Rqaq7p7GMVXjFh+6wN4oMYgjFmnSnhY8t6Xpdl2l9qmnHYuMHwQ==", + "dev": true, + "funding": [ + "https://github.com/sponsors/broofa", + "https://github.com/sponsors/ctavan" + ], + "license": "MIT", + "bin": { + "uuid": "dist-node/bin/uuid" + } + }, + "node_modules/which": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", + "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", + "dev": true, + "license": "ISC", + "dependencies": { + "isexe": "^2.0.0" + }, + "bin": { + "node-which": "bin/node-which" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/yaml": { + "version": "2.9.0", + "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.9.0.tgz", + "integrity": "sha512-2AvhNX3mb8zd6Zy7INTtSpl1F15HW6Wnqj0srWlkKLcpYl/gMIMJiyuGq2KeI2YFxUPjdlB+3Lc10seMLtL4cA==", + "dev": true, + "license": "ISC", + "bin": { + "yaml": "bin.mjs" + }, + "engines": { + "node": ">= 14.6" + }, + "funding": { + "url": "https://github.com/sponsors/eemeli" + } + }, "node_modules/zod": { "version": "4.1.8", "resolved": "https://registry.npmjs.org/zod/-/zod-4.1.8.tgz", diff --git a/package.json b/package.json index cce24a8..cd2a7ff 100644 --- a/package.json +++ b/package.json @@ -1,17 +1,20 @@ { "name": "opencode-firecrawl", - "version": "1.0.0", - "description": "Firecrawl plugin for OpenCode - web scraping, crawling, and search via the Firecrawl CLI", + "version": "1.1.0", + "description": "Firecrawl plugin for OpenCode - developer index search, web scraping, crawling, and search via the Firecrawl CLI", "type": "module", "main": "index.ts", "files": [ "index.ts", + "developer-search.ts", "skills" ], "keywords": [ "opencode", "opencode-plugin", "firecrawl", + "developer-search", + "documentation-search", "web-scraping", "crawling", "search", @@ -32,7 +35,7 @@ "check": "tsc --noEmit" }, "devDependencies": { - "@opencode-ai/plugin": "^1.0.0", + "@opencode-ai/plugin": "^1.18.25", "@types/node": "^25.2.2", "typescript": "^5.0.0" } diff --git a/skills/firecrawl-developer-index/SKILL.md b/skills/firecrawl-developer-index/SKILL.md new file mode 100644 index 0000000..affea1f --- /dev/null +++ b/skills/firecrawl-developer-index/SKILL.md @@ -0,0 +1,57 @@ +--- +name: firecrawl-developer-index +description: | + Search issues, merged pull requests, READMEs, and library documentation for primary-source answers to developer questions. + + USE THE DEVELOPER INDEX FOR: + - How a library or API behaves, what it returns, what is required, what the default is + - What an error message or stack trace means, and whether the bug was fixed + - Version-specific behaviour and regressions + - "why does X do Y", "is this a known bug", "what changed in this release" + + Prefer this over a general web search for any of the above: it returns the matched passages from the issue, the merged pull request, the README, or the doc page, so you can answer from the source instead of pointing at a page. Fall back to the Firecrawl CLI for the open web when the question is a comparison, an opinion, news, or a project that is not indexed. +--- + +# Firecrawl Developer Index + +Answer a developer question from the primary source: the issue where the bug was reported, the merged pull request that fixed it, the README or documentation page that states the contract. A blog post that describes a behaviour is a weaker answer than the passage that defines it, so reach for the index first and the open web second. + +There is **no fixed recipe**. Read the question, decide what kind it is, and choose the approach below. A literal error string wants a different move than "how do I do X". Don't run machinery a question doesn't call for. + +## The surfaces, and what each is uniquely good at + +- **`firecrawl_developer_search(query, k?, types?, repos?, sources?, passages?)`** + Ranked results over the whole index, returned as markdown. Each result carries an `id` (`issue:owner/repo#123`), a `url`, and the **matched passages**, so tables and code blocks survive. The artifact kind is the `id` prefix: `doc:`, `issue:`, `pull_request:`, or `readme:`. + This is the default first move for a developer question, and the only surface that hands you the passages. It works without an API key; a `FIRECRAWL_API_KEY` in the environment only raises the rate limit. + `k` is 1 to 100 and defaults to 10. `passages` is 1 to 5 and defaults to 1. + +- **`firecrawl developer [--limit ]`** (CLI) + The same index from the shell. Reach for it when you are already scripting a batch of lookups or want the output written to a file rather than into context. + +- **`firecrawl search ` / `firecrawl scrape `** (CLI) + General web search and fetch, for what no primary source states: a comparison between two libraries, an outage, a migration write-up, a project with no public repository or indexed docs. Also the follow-through when a hit is the right page but you need all of it, so `scrape` the result's `url`. + +## Filters, and what each one costs you + +- `types` picks which of `doc`, `issue`, `pull_request`, `readme` to search, and defaults to all four. Narrowing here is the cheapest way to sharpen a query. +- `repos` (`owner/name`) scopes the repository half, meaning `issue`, `pull_request`, and `readme`. `sources` (documentation source ids, at most 20) scopes the documentation half, meaning `doc`. Passing both **unions** the halves rather than intersecting them. +- A filter that cannot match any requested `type` is an error rather than an empty list, so don't pass `repos` without a repository type in `types`, or `sources` without `doc`. +- When a scope is not in the index the result says so explicitly. That note means no rephrasing will ever help: drop the scope and search the whole index, or go to the web. +- `passages` is the _maximum_ passages per result, not a guarantee. Raise it when one page is clearly the right page but the first passage is the wrong part of it. + +## Match the approach to the question + +- **Literal error message or stack-trace string** → search the string itself plus the library name, with `types: ["issue", "pull_request"]`. Whoever hit it filed it. If nothing matches, strip the volatile parts (paths, line numbers, ids, addresses) and retry; the invariant middle of the message is what is indexed. +- **Conceptual "how do I do X"** → the full question in natural language, all four types. The answer is usually a `doc` or a `readme`; raise `passages` before raising `k`. +- **Known bug** → the issue reports it, the merged pull request _fixes_ it, and the fix is what you want. Search `types: ["issue", "pull_request"]`, then re-query the issue's own terms scoped to its repo with `types: ["pull_request"]`. A merged PR's passages tell you what changed and in which direction. +- **API contract** ("what does X return", "is Y required", "what is the default") → `readme` and `doc` are authoritative and a blog post is not. Use `types: ["readme", "doc"]`. If the contract looks like it moved, follow up with `pull_request` for the change that moved it. +- **Version-specific behaviour** → an issue's opening report describes the broken version; its resolution supersedes it. Raise `passages` to see further into the thread, and read the resolution and the linked pull request before answering. Never answer from an opening report alone. +- **Scoped to one library** → `repos: ["owner/name"]` when you know the slug, plus `sources` if you want its docs in the same call. +- **Comparison, opinion, news, or an unindexed project** → the open web, via `firecrawl search` and then `firecrawl scrape` on whatever deserves a full read. Combining is often right: take the contract from the index and the trade-off from the web. + +## Principles + +- **Quote the passage, cite the `url`.** The passages are the evidence; hand them over rather than paraphrasing them into a claim the reader can't check. `title` is frequently absent on `doc` results, so fall back to the `url`. +- **A merge supersedes a report.** When an issue and a pull request disagree, the merged pull request is the current behaviour. Say which one you read. +- **Scope last, not first.** Search the whole index, then narrow with `types` or `repos` once you know what the hits look like. Scoping first hides the result that would have told you where to look. +- **Go to the web when the index has nothing to say.** Trade-offs, ecosystem opinion, and anything about an unindexed project are web questions. Don't force them through the index, and don't dress a general web page up as a primary source. diff --git a/tsconfig.json b/tsconfig.json index 976dde4..5b90154 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -5,7 +5,8 @@ "moduleResolution": "NodeNext", "strict": true, "noEmit": true, - "skipLibCheck": true + "skipLibCheck": true, + "allowImportingTsExtensions": true }, - "include": ["index.ts"] + "include": ["index.ts", "developer-search.ts"] }