diff --git a/.changeset/grep-exclude.md b/.changeset/grep-exclude.md new file mode 100644 index 00000000..7bbd6c24 --- /dev/null +++ b/.changeset/grep-exclude.md @@ -0,0 +1,26 @@ +--- +"@cloudflare/dofs": minor +"@cloudflare/computer": minor +--- + +Add an `exclude` option to `fs.grep`, matching `fs.find` + +`find` could already prune directories from a traversal but `grep` could not, so +there was no way to search a workspace while skipping `node_modules` — the one +thing callers most often want to skip. The exclusion globs are passed to the +same find walker `grep` already traverses with, so an excluded directory is +pruned before its children are queried rather than being read and filtered. + +```ts +const todos = await workspace.fs.grep("TODO", "/", { + include: "**/*.ts", + exclude: ["node_modules", "node_modules/**"], +}); +``` + +Exclusion is matched against the same directory-relative path as `include` and +applied first, so an exclusion always wins. As with `find`, name both the +directory and its contents to prune the subtree: `node_modules/**` matches what +is below `node_modules`, not `node_modules` itself. Grepping a single file +ignores `exclude`, since the caller named the file and there is no traversal to +prune. diff --git a/docs/04_filesystem_interface.md b/docs/04_filesystem_interface.md index d5f6eaa5..4563fc16 100644 --- a/docs/04_filesystem_interface.md +++ b/docs/04_filesystem_interface.md @@ -392,6 +392,7 @@ interface GrepOptions { limit?: number; offset?: number; include?: string; + exclude?: string[]; } interface WorkspaceGrepContextLine { @@ -420,14 +421,23 @@ letter case. `context` adds that many lines before and after each match. `include` is a glob relative to a searched directory. `limit` and `offset` paginate matching lines. +`exclude` takes globs of the same shape as `find`'s, matched against the same +directory-relative path and applied before `include`, so an exclusion always +wins. An excluded directory is pruned before its children are queried, so the +subtree costs nothing rather than being read and filtered. As with `find`, name +both the directory and its contents to skip a whole subtree: `node_modules/**` +matches what is below `node_modules`, not `node_modules` itself. + `path` may be a directory or a single file. Directory searches return matches in deterministic depth-first discovery order, then line order within each -file. Results are not globally sorted by full path. +file. Results are not globally sorted by full path. A single-file search has no +traversal to prune, so `exclude` does not apply to it. ```ts const hits = await fs.grep("TODO", "/workspace/src", { ignoreCase: true, include: "**/*.ts", + exclude: ["node_modules", "node_modules/**"], }); for (const hit of hits) { console.log(`${hit.path}:${hit.line}: ${hit.text}`); @@ -517,7 +527,7 @@ maps to `Workspace.fs`: | `watch` | — | Low-level primitive in `fs/watch.ts` (`createWatcher`, `createWatchAsyncIterable`, `WatchHandle`, `WatchOptions`); not exposed on the `WorkspaceFilesystem` class. | | `open` / `FileHandle` | — | Use streams instead. | | `glob` | `find` | Limited glob support (`*`, `**`, `**/`, and `?`), plus `exclude` for pruning subtrees. | -| — | `grep` | Not in `node:fs`; literal by default, with optional regular expressions. | +| — | `grep` | Not in `node:fs`; literal by default, with optional regular expressions. Shares `find`'s `include`/`exclude` globs. | | — | `find` | Recursive directory walk with an optional glob, relative-rooted. | | — | `ls` | Flat list of file paths under a directory (segment-aware). | diff --git a/packages/computer/src/tools/fs/grep.ts b/packages/computer/src/tools/fs/grep.ts index 43f35cf8..430df6f2 100644 --- a/packages/computer/src/tools/fs/grep.ts +++ b/packages/computer/src/tools/fs/grep.ts @@ -21,6 +21,7 @@ interface GrepOptions { limit?: number; offset?: number; include?: string; + exclude?: string[]; } export interface GrepWorkspaceLike { @@ -43,6 +44,12 @@ const inputSchema = z.object({ .string() .optional() .describe('Glob relative to path that limits searched files, for example "**/*.ts".'), + exclude: z + .array(z.string()) + .optional() + .describe( + 'Glob patterns to leave out, for example ["node_modules/**", "**/.git/**"]. An excluded directory is skipped along with everything below it.', + ), regex: z.boolean().optional().describe("Interpret query as a regular expression."), ignoreCase: z.boolean().optional().describe("Ignore letter case."), context: z.number().int().min(0).max(10).optional(), @@ -55,7 +62,17 @@ export function createGrepTool(options: GrepToolOptions): Tool { + execute: async ({ + path, + query, + include, + exclude, + regex, + ignoreCase, + context, + limit, + offset, + }) => { try { const pageSize = limit ?? DEFAULT_LIMIT; const pageOffset = offset ?? 0; @@ -67,6 +84,7 @@ export function createGrepTool(options: GrepToolOptions): Tool { }); }); + describe("exclude", () => { + it("leaves an excluded file out of the results", async () => { + await withDB(async (db) => { + await writeFile(db, "/keep.ts", "TODO keep\n", {}, () => 0); + await writeFile(db, "/skip.ts", "TODO skip\n", {}, () => 0); + + expect( + (await grep(db, "TODO", "/", { exclude: ["skip.ts"] })).map((match) => match.path), + ).toEqual(["/keep.ts"]); + }); + }); + + it("drops an excluded directory along with everything below it", async () => { + await withDB(async (db) => { + mkdir(db, "/src", { recursive: true }, () => 0); + mkdir(db, "/node_modules/dep", { recursive: true }, () => 0); + await writeFile(db, "/src/index.ts", "TODO mine\n", {}, () => 0); + await writeFile(db, "/node_modules/dep/index.ts", "TODO theirs\n", {}, () => 0); + + // Both forms, as the find tests do: `node_modules` prunes the + // directory and `node_modules/**` covers anything below it. + expect( + ( + await grep(db, "TODO", "/", { + exclude: ["node_modules", "node_modules/**"], + }) + ).map((match) => match.path), + ).toEqual(["/src/index.ts"]); + }); + }); + + it("never reads a file below an excluded directory", async () => { + await withDB(async (db) => { + mkdir(db, "/src", { recursive: true }, () => 0); + mkdir(db, "/vendor", { recursive: true }, () => 0); + await writeFile(db, "/src/a.ts", "TODO mine\n", {}, () => 0); + await writeFile(db, "/vendor/b.ts", "TODO theirs\n", {}, () => 0); + + // Pruning has to happen during the walk, not as a filter over + // results: the whole point is that the excluded subtree costs + // nothing. If the walker descended and grep then discarded the + // matches, the blob for b.ts would still be queried. + // + // The directory itself must be named to be pruned -- `vendor/**` + // matches what is *below* `vendor`, not `vendor` -- which is why the + // existing find tests pass both forms. With only `vendor/**` the + // walker still descends and excludes each child, costing a query. + const seen: string[] = []; + const all = db.all.bind(db); + vi.spyOn(db, "all").mockImplementation((query: unknown, ...args: unknown[]) => { + seen.push(String(query)); + return all(query as never, ...(args as never[])); + }); + + await grep(db, "TODO", "/", { exclude: ["vendor", "vendor/**"] }); + vi.restoreAllMocks(); + + // The walker reads children of / and of /src, but never of /vendor. + const childQueries = seen.filter((query) => query.includes("d.name > ?")); + expect(childQueries.length).toBe(2); + }); + }); + + it("applies exclusion before the inclusion glob", async () => { + await withDB(async (db) => { + mkdir(db, "/pkg", { recursive: true }, () => 0); + await writeFile(db, "/pkg/keep.ts", "TODO keep\n", {}, () => 0); + await writeFile(db, "/pkg/skip.ts", "TODO skip\n", {}, () => 0); + + expect( + ( + await grep(db, "TODO", "/", { + include: "**/*.ts", + exclude: ["pkg/skip.ts"], + }) + ).map((match) => match.path), + ).toEqual(["/pkg/keep.ts"]); + }); + }); + + it("ignores exclusion when the path names a single file", async () => { + await withDB(async (db) => { + await writeFile(db, "/only.ts", "TODO here\n", {}, () => 0); + + // The caller named the file, so there is no traversal to prune and + // nothing to second-guess. + expect( + (await grep(db, "TODO", "/only.ts", { exclude: ["only.ts"] })).map((match) => match.path), + ).toEqual(["/only.ts"]); + }); + }); + }); + it("walks each directory page once during a search", async () => { await withDB(async (db) => { for (let index = 0; index < 260; index += 1) { diff --git a/packages/dofs/src/fs/grep.ts b/packages/dofs/src/fs/grep.ts index 2eae9392..b2132cf0 100644 --- a/packages/dofs/src/fs/grep.ts +++ b/packages/dofs/src/fs/grep.ts @@ -31,6 +31,13 @@ export interface GrepOptions { offset?: number; /** Glob relative to a searched directory that limits files. */ include?: string; + /** + * Glob patterns whose matches are not searched. Matched against the + * same directory-relative path as the inclusion glob and applied + * first, so an exclusion always wins. An excluded directory is + * pruned: neither it nor anything below it is read. + */ + exclude?: string[]; } interface ScanState { @@ -68,7 +75,12 @@ export async function grep( }); const matches: WorkspaceGrepMatch[] = []; const state: ScanState = { seen: 0, accepted: 0 }; - const filePaths = node.type === "file" ? [canonical] : filesUnder(db, canonical, options.include); + // Grepping a single file has no traversal to prune, so `exclude` does not + // apply to it: the caller named the file explicitly. + const filePaths = + node.type === "file" + ? [canonical] + : filesUnder(db, canonical, options.include, options.exclude); for (const filePath of filePaths) { const complete = await scanFile( db, @@ -117,8 +129,12 @@ function* filesUnder( db: Database, directory: string, include: string | undefined, + exclude: string[] | undefined, ): Iterable { - for (const entry of iterateFoundEntries(db, directory, include)) { + // The find walker already prunes excluded directories before querying their + // children, so an excluded subtree costs nothing here rather than being + // walked and filtered. + for (const entry of iterateFoundEntries(db, directory, include, exclude)) { if (entry.type === "file") yield entry.path; } }