diff --git a/.changeset/batch-remote-filesystem-walks.md b/.changeset/batch-remote-filesystem-walks.md new file mode 100644 index 00000000000..9a16b8ec629 --- /dev/null +++ b/.changeset/batch-remote-filesystem-walks.md @@ -0,0 +1,5 @@ +--- +"@moonshot-ai/kimi-code": patch +--- + +Speed up remote file reads, edits, writes, search, and completion. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 19ecf46363a..fbe93c7ae3f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -62,6 +62,7 @@ jobs: cache: pnpm - run: pnpm install --frozen-lockfile + - run: sudo apt-get update && sudo apt-get install -y ripgrep - run: pnpm run test --shard=${{ matrix.shard }}/5 # pi-tui's suite runs on node:test (not vitest), so the root `pnpm run test` diff --git a/packages/acp-server/src/acp-fs/acpFsService.ts b/packages/acp-server/src/acp-fs/acpFsService.ts index 5dd422e72b6..5bd7aa8224d 100644 --- a/packages/acp-server/src/acp-fs/acpFsService.ts +++ b/packages/acp-server/src/acp-fs/acpFsService.ts @@ -20,6 +20,8 @@ import { HostFileSystem, type HostDirEntry, type HostFileStat, + type HostWalkOptions, + type HostWalkResult, IHostFileSystem, ISessionContext, LifecycleScope, @@ -151,6 +153,10 @@ export class AcpHostFileSystem implements IHostFileSystem { return this.inner.readdir(path); } + walk(path: string, options: HostWalkOptions): Promise { + return this.inner.walk(path, options); + } + mkdir(path: string, options?: { readonly recursive?: boolean }): Promise { return this.inner.mkdir(path, options); } diff --git a/packages/agent-core-v2/src/agent/tools/fileReadSource.ts b/packages/agent-core-v2/src/agent/tools/fileReadSource.ts index 611558af915..0f3c9ceb6eb 100644 --- a/packages/agent-core-v2/src/agent/tools/fileReadSource.ts +++ b/packages/agent-core-v2/src/agent/tools/fileReadSource.ts @@ -1,5 +1,5 @@ import { readUtf8Lines } from '#/_base/execEnv/decodeText'; -import type { HostFileStat, IHostFileSystem } from '#/os/interface/hostFileSystem'; +import type { HostFileStat, HostWholeFile, IHostFileSystem } from '#/os/interface/hostFileSystem'; import { parseDaemonFileUrl } from '#/agent/media/mediaRef'; import type { ISessionMediaStore } from '#/agent/media/sessionMediaStore'; import type { ExecutableToolResult } from '#/tool/toolContract'; @@ -8,6 +8,7 @@ export interface FileReadSource { readonly name: string; readonly localPath?: string; stat(): Promise; + readWholeFile?(): Promise; readBytes(n?: number): Promise; readLines(): AsyncIterable; } @@ -21,6 +22,7 @@ export function environmentFileSource(fs: IHostFileSystem, path: string): FileRe return { name: path, stat: () => fs.stat(path), + readWholeFile: fs.readWholeFile === undefined ? undefined : () => fs.readWholeFile!(path), readBytes: (n) => fs.readBytes(path, n), readLines: () => fs.readLines(path, { errors: 'strict' }), }; diff --git a/packages/agent-core-v2/src/agent/tools/os/grep/grepTool.ts b/packages/agent-core-v2/src/agent/tools/os/grep/grepTool.ts index 503ddff6c29..d02dae75765 100644 --- a/packages/agent-core-v2/src/agent/tools/os/grep/grepTool.ts +++ b/packages/agent-core-v2/src/agent/tools/os/grep/grepTool.ts @@ -44,7 +44,6 @@ import { type GrepInput, GrepInputSchema, IGrepTool } from './grep'; const RG_MAX_COLUMNS = 500; const DEFAULT_HEAD_LIMIT = 250; -const MTIME_STAT_CONCURRENCY = 32; const VCS_DIRECTORIES_TO_EXCLUDE = ['.git', '.svn', '.hg', '.bzr', '.jj', '.sl'] as const; const SENSITIVE_KEY_BASENAMES = ['id_rsa', 'id_ed25519', 'id_ecdsa'] as const; @@ -222,18 +221,7 @@ export class GrepTool implements IGrepTool { const filteredSensitive = new Set(); const keptLines = filterSensitiveLines(rawLines, mode, filteredSensitive, pathClass); - let orderedLines: ParsedGrepLine[]; - try { - orderedLines = - mode === 'files_with_matches' && !timedOut - ? await this.sortFilesWithMatchesByMtime(fs, keptLines, signal) - : keptLines; - } catch (error) { - if (error instanceof GrepAbortedError) { - return { isError: true, output: 'Grep aborted' }; - } - throw error; - } + const orderedLines = keptLines; const offset = args.offset ?? 0; const headLimit = args.head_limit ?? DEFAULT_HEAD_LIMIT; @@ -329,32 +317,6 @@ export class GrepTool implements IGrepTool { }; } - private async sortFilesWithMatchesByMtime( - fs: IHostFileSystem, - lines: readonly ParsedGrepLine[], - signal: AbortSignal, - ): Promise { - const entries = await mapWithConcurrency( - lines, - MTIME_STAT_CONCURRENCY, - signal, - async (line, index) => { - const path = - line.kind === 'record' ? line.filePath : line.kind === 'legacy' ? line.text : undefined; - let mtime = 0; - if (path !== undefined) { - try { - const mtimeMs = (await fs.stat(path)).mtimeMs ?? 0; - mtime = Math.trunc(mtimeMs / 1000); - } catch { - } - } - return { line, mtime, index }; - }, - ); - entries.sort((a, b) => b.mtime - a.mtime || a.index - b.index); - return entries.map((entry) => entry.line); - } } registerAgentToolService(IGrepTool, GrepTool, { @@ -387,41 +349,6 @@ type ParsedGrepLine = readonly text: string; }; -class GrepAbortedError extends Error { - constructor() { - super('Grep aborted'); - this.name = 'GrepAbortedError'; - } -} - -async function mapWithConcurrency( - items: readonly T[], - concurrency: number, - signal: AbortSignal, - mapper: (item: T, index: number) => Promise, -): Promise { - if (signal.aborted) throw new GrepAbortedError(); - if (items.length === 0) return []; - - const results: U[] = []; - results.length = items.length; - let nextIndex = 0; - const workerCount = Math.min(Math.max(1, concurrency), items.length); - await Promise.all( - Array.from({ length: workerCount }, async () => { - while (true) { - if (signal.aborted) return; - const index = nextIndex; - nextIndex += 1; - if (index >= items.length) return; - results[index] = await mapper(items[index] as T, index); - } - }), - ); - if (signal.aborted) throw new GrepAbortedError(); - return results; -} - function buildRgArgs( rgPath: string, args: GrepInput, @@ -440,7 +367,7 @@ function buildRgArgs( cmd.push('--glob', `!${dir}`); } - if (mode === 'files_with_matches') cmd.push('-l'); + if (mode === 'files_with_matches') cmd.push('-l', '--sortr=modified'); else if (mode === 'count_matches') { cmd.push('--count-matches', '--with-filename'); } diff --git a/packages/agent-core-v2/src/agent/tools/os/read/readTool.ts b/packages/agent-core-v2/src/agent/tools/os/read/readTool.ts index a37c494bf98..4d8797867fb 100644 --- a/packages/agent-core-v2/src/agent/tools/os/read/readTool.ts +++ b/packages/agent-core-v2/src/agent/tools/os/read/readTool.ts @@ -319,8 +319,10 @@ export class ReadTool implements IReadTool { ): Promise { try { let stat: Awaited>; + let whole: Awaited>> | undefined; try { - stat = await source.stat(); + if ((args.line_offset ?? 1) >= 0) whole = await source.readWholeFile?.(); + stat = whole?.stat ?? await source.stat(); } catch (error) { if (isHostFsNotFound(error) || isHostFsNotDirectory(error)) { return { isError: true, output: `"${args.path}" does not exist.` }; @@ -331,7 +333,7 @@ export class ReadTool implements IReadTool { return { isError: true, output: `"${args.path}" is not a file.` }; } - const header = await source.readBytes(MEDIA_SNIFF_BYTES); + const header = whole?.bytes.subarray(0, MEDIA_SNIFF_BYTES) ?? await source.readBytes(MEDIA_SNIFF_BYTES); const fileType = detectFileType(source.name, header); if (fileType.kind === 'image' || fileType.kind === 'video') { const kind = fileType.kind; @@ -369,7 +371,7 @@ export class ReadTool implements IReadTool { 'Convert it to UTF-8 first (e.g. with `iconv`).', }; } - const bytes = await source.readBytes(); + const bytes = whole?.bytes ?? await source.readBytes(); let decoded: string; try { decoded = new TextDecoder(detection.encoding, { fatal: true }).decode(bytes); @@ -386,6 +388,10 @@ export class ReadTool implements IReadTool { isError: true, output: notReadableFileOutput(args.path), }; + } else if (whole !== undefined) { + const decoded = new TextDecoder('utf-8', { fatal: true }).decode(whole.bytes); + const content = splitLinesKeepingTerminator(decoded); + readLines = () => decodedLines(content); } else { readLines = () => source.readLines(); } diff --git a/packages/agent-core-v2/src/agent/tools/os/write/writeTool.ts b/packages/agent-core-v2/src/agent/tools/os/write/writeTool.ts index 972419a4aac..24b0e6d0fbe 100644 --- a/packages/agent-core-v2/src/agent/tools/os/write/writeTool.ts +++ b/packages/agent-core-v2/src/agent/tools/os/write/writeTool.ts @@ -105,14 +105,18 @@ export class WriteTool implements IWriteTool { } private async execution(fs: IHostFileSystem, args: WriteInput, safePath: string, parent = dirname(safePath)): Promise { - const parentError = await this.ensureParentDirectory(fs, parent); - if (parentError !== undefined) { - return { isError: true, output: parentError }; + const mode = args.mode ?? 'overwrite'; + if (fs.writeTextWithParent === undefined) { + const parentError = await this.ensureParentDirectory(fs, parent); + if (parentError !== undefined) { + return { isError: true, output: parentError }; + } } try { - const mode = args.mode ?? 'overwrite'; - if (mode === 'append') { + if (fs.writeTextWithParent !== undefined) { + await fs.writeTextWithParent(safePath, args.content, mode, parent); + } else if (mode === 'append') { await fs.appendText(safePath, args.content); } else { await fs.writeText(safePath, args.content); diff --git a/packages/agent-core-v2/src/os/backends/node-local/hostFsService.ts b/packages/agent-core-v2/src/os/backends/node-local/hostFsService.ts index e141543c10d..214f2e68e94 100644 --- a/packages/agent-core-v2/src/os/backends/node-local/hostFsService.ts +++ b/packages/agent-core-v2/src/os/backends/node-local/hostFsService.ts @@ -14,8 +14,15 @@ import { } from 'node:fs/promises'; import { decodeTextWithErrors, readUtf8Lines, type TextDecodeErrors } from '#/_base/execEnv/decodeText'; -import { type HostDirEntry, type HostFileStat, IHostFileSystem } from '#/os/interface/hostFileSystem'; +import { + type HostDirEntry, + type HostFileStat, + type HostWalkOptions, + type HostWalkResult, + IHostFileSystem, +} from '#/os/interface/hostFileSystem'; import { toHostFsError } from '#/os/interface/hostFsErrors'; +import { walkHostDirectory } from '#/os/interface/hostFsWalk'; const READ_CHUNK_SIZE = 64 * 1024; @@ -192,6 +199,10 @@ export class HostFileSystem implements IHostFileSystem { } } + async walk(path: string, options: HostWalkOptions): Promise { + return walkHostDirectory(this, path, options); + } + async mkdir(path: string, options?: { readonly recursive?: boolean }): Promise { try { await mkdir(path, { recursive: options?.recursive ?? false }); diff --git a/packages/agent-core-v2/src/os/backends/node-local/tools/runRg.ts b/packages/agent-core-v2/src/os/backends/node-local/tools/runRg.ts index de3700a8ed2..e607da9adef 100644 --- a/packages/agent-core-v2/src/os/backends/node-local/tools/runRg.ts +++ b/packages/agent-core-v2/src/os/backends/node-local/tools/runRg.ts @@ -100,7 +100,9 @@ export async function runRgOnce( try { const isTerminating = (): boolean => timedOut || aborted || killed; const [stdoutResult, stderrResult, code] = await Promise.all([ - readStreamWithCap(proc.stdout, MAX_OUTPUT_BYTES, isTerminating), + readStreamWithCap(proc.stdout, MAX_OUTPUT_BYTES, isTerminating, () => { + void killProc(); + }), readStreamWithCap(proc.stderr, MAX_OUTPUT_BYTES, isTerminating), proc.wait(), ]); @@ -139,6 +141,7 @@ export function shouldRetryRipgrepEagain(result: RunRgResult): boolean { result.exitCode !== 0 && result.exitCode !== 1 && !result.timedOut && + !result.bufferTruncated && isEagainRipgrepError(result.stderrText) ); } @@ -163,6 +166,7 @@ async function readStreamWithCap( stream: Readable, maxBytes: number, suppressPrematureClose?: () => boolean, + onLimit?: () => void, ): Promise { const chunks: Buffer[] = []; let total = 0; @@ -177,6 +181,7 @@ async function readStreamWithCap( if (remaining > 0) chunks.push(buf.subarray(0, remaining)); total = maxBytes; truncated = true; + onLimit?.(); continue; } chunks.push(buf); diff --git a/packages/agent-core-v2/src/os/backends/remote/remoteFileSystem.ts b/packages/agent-core-v2/src/os/backends/remote/remoteFileSystem.ts index f74e978faf7..b9fb1931ceb 100644 --- a/packages/agent-core-v2/src/os/backends/remote/remoteFileSystem.ts +++ b/packages/agent-core-v2/src/os/backends/remote/remoteFileSystem.ts @@ -1,3 +1,5 @@ +import * as posixPath from 'node:path/posix'; + import { decodeTextWithErrors, readUtf8Lines, @@ -7,8 +9,15 @@ import { splitLinesKeepingTerminator } from '#/_base/text/line-endings'; import type { HostDirEntry, HostFileStat, + HostWalkOptions, + HostWalkResult, IHostFileSystem, } from '#/os/interface/hostFileSystem'; +import { + HOST_WALK_DEFAULT_TIMEOUT_MS, + HOST_WALK_MAX_IGNORE_BYTES, + walkHostDirectory, +} from '#/os/interface/hostFsWalk'; import { HostFsError, OsFsErrors, @@ -28,16 +37,23 @@ import { FS_READ_FILE_METHOD, FS_REMOVE_METHOD, FS_RENAME_METHOD, + FS_WALK_METHOD, FS_WRITE_FILE_CHUNK_BYTES, FS_WRITE_FILE_METHOD, type FsGetMetadataResult, type FsReadDirectoryResult, type FsReadFileResult, + type FsWalkResult, type FsWriteMode, } from '#/remote/protocol/methods'; import { rpcDomainError } from './rpcDomainError'; const HOST_FS_CODES: ReadonlySet = new Set(Object.values(OsFsErrors.codes)); +const REMOTE_WALK_PATH = { + join: (...paths: readonly string[]) => posixPath.join(...paths), + relative: (from: string, to: string) => posixPath.relative(from, to), + separator: posixPath.sep, +}; export function toRemoteFsError(error: unknown): Error { if (!(error instanceof RpcError)) { @@ -77,17 +93,30 @@ export class RemoteFileSystem implements IHostFileSystem { return this.call(FS_READ_FILE_METHOD, { path, offset, maxBytes }); } - private async writeMode(path: string, data: Uint8Array, mode: FsWriteMode): Promise { - await this.call(FS_WRITE_FILE_METHOD, { path, dataBase64: encodeBase64(data), mode }); + private async writeMode(path: string, data: Uint8Array, mode: FsWriteMode, parent?: string): Promise { + await this.call(FS_WRITE_FILE_METHOD, { path, dataBase64: encodeBase64(data), mode, parent }); } - private async writeChunked(path: string, data: Uint8Array, mode: FsWriteMode): Promise { + private async writeChunked(path: string, data: Uint8Array, mode: FsWriteMode, parent?: string): Promise { if (data.byteLength <= FS_WRITE_FILE_CHUNK_BYTES) { - await this.writeMode(path, data, mode); + await this.writeMode(path, data, mode, parent); return; } for (let offset = 0; offset < data.byteLength; offset += FS_WRITE_FILE_CHUNK_BYTES) { - await this.writeMode(path, data.subarray(offset, offset + FS_WRITE_FILE_CHUNK_BYTES), offset === 0 ? mode : 'append'); + await this.writeMode(path, data.subarray(offset, offset + FS_WRITE_FILE_CHUNK_BYTES), offset === 0 ? mode : 'append', offset === 0 ? parent : undefined); + } + } + + async readWholeFile(path: string) { + try { + const result = await this.readRange(path, 0); + const bytes = decodeBase64(result.dataBase64); + return { + bytes, + stat: { isFile: true, isDirectory: false, size: bytes.byteLength }, + }; + } catch { + return undefined; } } @@ -95,7 +124,8 @@ export class RemoteFileSystem implements IHostFileSystem { path: string, options?: { encoding?: BufferEncoding; errors?: TextDecodeErrors }, ): Promise { - const bytes = await this.readBytes(path); + const whole = await this.readWholeFile(path); + const bytes = whole === undefined ? await this.readBytes(path) : whole.bytes; const buffer = Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength); return decodeTextWithErrors(buffer, options?.encoding ?? 'utf-8', options?.errors ?? 'strict'); } @@ -104,6 +134,10 @@ export class RemoteFileSystem implements IHostFileSystem { await this.writeChunked(path, Buffer.from(data, 'utf8'), 'truncate'); } + async writeTextWithParent(path: string, data: string, mode: 'overwrite' | 'append', parent: string): Promise { + await this.writeChunked(path, Buffer.from(data, 'utf8'), mode === 'append' ? 'append' : 'truncate', parent); + } + async appendText(path: string, data: string): Promise { await this.writeChunked(path, Buffer.from(data, 'utf8'), 'append'); } @@ -232,6 +266,46 @@ export class RemoteFileSystem implements IHostFileSystem { })); } + async walk(path: string, options: HostWalkOptions): Promise { + const remainingMs = options.deadlineMs === undefined ? undefined : options.deadlineMs - Date.now(); + if (remainingMs !== undefined && remainingMs <= 0) { + return { entries: [], errors: [], truncated: true, responseBytes: 0 }; + } + const timeoutMs = Math.max(1, Math.min( + remainingMs ?? HOST_WALK_DEFAULT_TIMEOUT_MS, + HOST_WALK_DEFAULT_TIMEOUT_MS, + )); + if (options.ignore !== undefined && Buffer.byteLength(JSON.stringify(options.ignore)) > HOST_WALK_MAX_IGNORE_BYTES) { + return this.walkFallback(path, options); + } + try { + const result = await this.connection.call(FS_WALK_METHOD, { + path, + maxDepth: options.maxDepth, + maxDirectories: options.maxDirectories, + maxEntries: options.maxEntries, + maxResponseBytes: options.maxResponseBytes, + ignore: options.ignore, + timeoutMs, + }) as FsWalkResult; + return { + entries: result.entries, + errors: result.errors.map(({ path: errorPath, error }) => ({ + path: errorPath, + error: toRemoteFsError(RpcError.fromErrorBody(error)), + })), + truncated: result.truncated, + responseBytes: result.responseBytes, + }; + } catch (error) { + throw toRemoteFsError(error); + } + } + + private async walkFallback(path: string, options: HostWalkOptions): Promise { + return walkHostDirectory(this, path, options, { path: REMOTE_WALK_PATH }); + } + async mkdir(path: string, options?: { readonly recursive?: boolean }): Promise { await this.call(FS_CREATE_DIRECTORY_METHOD, { path, diff --git a/packages/agent-core-v2/src/os/interface/hostFileSystem.ts b/packages/agent-core-v2/src/os/interface/hostFileSystem.ts index 0199b65263d..9327692e364 100644 --- a/packages/agent-core-v2/src/os/interface/hostFileSystem.ts +++ b/packages/agent-core-v2/src/os/interface/hostFileSystem.ts @@ -11,6 +11,11 @@ export interface HostFileStat { readonly mode?: number; } +export interface HostWholeFile { + readonly stat: HostFileStat; + readonly bytes: Uint8Array; +} + export interface HostDirEntry { readonly name: string; readonly isFile: boolean; @@ -18,6 +23,27 @@ export interface HostDirEntry { readonly isSymbolicLink?: boolean; } +export interface HostWalkOptions { + readonly maxDepth: number; + readonly maxDirectories: number; + readonly maxEntries: number; + readonly maxResponseBytes: number; + readonly ignore?: readonly string[]; + readonly deadlineMs?: number; +} + +export interface HostWalkError { + readonly path: string; + readonly error: Error; +} + +export interface HostWalkResult { + readonly entries: readonly HostDirEntry[]; + readonly errors: readonly HostWalkError[]; + readonly truncated: boolean; + readonly responseBytes: number; +} + export interface IHostFileSystem { readonly _serviceBrand: undefined; @@ -25,7 +51,9 @@ export interface IHostFileSystem { path: string, options?: { encoding?: BufferEncoding; errors?: TextDecodeErrors }, ): Promise; + readWholeFile?(path: string): Promise; writeText(path: string, data: string): Promise; + writeTextWithParent?(path: string, data: string, mode: 'overwrite' | 'append', parent: string): Promise; appendText(path: string, data: string): Promise; readBytes(path: string, n?: number, offset?: number): Promise; writeBytes(path: string, data: Uint8Array | AsyncIterable): Promise; @@ -37,6 +65,7 @@ export interface IHostFileSystem { stat(path: string): Promise; lstat(path: string): Promise; readdir(path: string): Promise; + walk(path: string, options: HostWalkOptions): Promise; mkdir(path: string, options?: { readonly recursive?: boolean }): Promise; remove(path: string): Promise; rename?(from: string, to: string): Promise; diff --git a/packages/agent-core-v2/src/os/interface/hostFsWalk.ts b/packages/agent-core-v2/src/os/interface/hostFsWalk.ts new file mode 100644 index 00000000000..44e4a7619e1 --- /dev/null +++ b/packages/agent-core-v2/src/os/interface/hostFsWalk.ts @@ -0,0 +1,123 @@ +import { join, relative, sep } from 'node:path'; + +import ignore from 'ignore'; + +import type { HostDirEntry, HostWalkOptions, HostWalkResult } from './hostFileSystem'; +import { HostFsError, OsFsErrors } from './hostFsErrors'; + +export const HOST_WALK_MAX_DEPTH = 64; +export const HOST_WALK_MAX_DIRECTORIES = 10_000; +export const HOST_WALK_MAX_ENTRIES = 100_000; +export const HOST_WALK_MAX_RESPONSE_BYTES = 8 * 1024 * 1024; +export const HOST_WALK_MAX_IGNORE_BYTES = 1024 * 1024; +export const HOST_WALK_DEFAULT_TIMEOUT_MS = 55_000; +export const HOST_WALK_MAX_TIMEOUT_MS = 60_000; +export const HOST_WALK_MAX_DIRECTORY_ENTRIES = 50_000; + +interface HostWalkerFs { + readdir(path: string): Promise; +} + +interface HostWalkPath { + readonly separator: string; + join(...paths: readonly string[]): string; + relative(from: string, to: string): string; +} + +interface HostWalkHooks { + readonly path?: HostWalkPath; + readonly serializeError?: (path: string, error: Error) => unknown; +} + +export async function walkHostDirectory( + fs: HostWalkerFs, + root: string, + options: HostWalkOptions, + hooks: HostWalkHooks = {}, + readDirectory: (path: string) => Promise = fs.readdir.bind(fs), +): Promise { + validateLimit('maxDepth', options.maxDepth, 0, HOST_WALK_MAX_DEPTH); + validateLimit('maxDirectories', options.maxDirectories, 1, HOST_WALK_MAX_DIRECTORIES); + validateLimit('maxEntries', options.maxEntries, 1, HOST_WALK_MAX_ENTRIES); + validateLimit('maxResponseBytes', options.maxResponseBytes, 1, HOST_WALK_MAX_RESPONSE_BYTES); + if (options.deadlineMs !== undefined && !Number.isFinite(options.deadlineMs)) { + throw new RangeError('deadlineMs must be finite'); + } + const matcher = options.ignore === undefined ? undefined : ignore().add(options.ignore); + const serializeError = hooks.serializeError ?? ((_path, error) => ({ + name: error.name, + message: error.message, + })); + const pathApi = hooks.path ?? { join, relative, separator: sep }; + const deadlineMs = options.deadlineMs ?? Date.now() + HOST_WALK_DEFAULT_TIMEOUT_MS; + const entries: HostDirEntry[] = []; + const errors: { path: string; error: Error }[] = []; + const pending = [{ path: root, depth: 0 }]; + let responseBytes = 64; + let truncated = false; + for (let index = 0; index < pending.length && !truncated; index += 1) { + if (Date.now() >= deadlineMs) { + truncated = true; + break; + } + const { path: directory, depth } = pending[index]!; + let children: readonly HostDirEntry[]; + try { + children = await readDirectory(directory); + if (children.length > HOST_WALK_MAX_DIRECTORY_ENTRIES) { + throw new HostFsError( + OsFsErrors.codes.OS_FS_DIRECTORY_TOO_LARGE, + `readdir ${directory} exceeds the ${HOST_WALK_MAX_DIRECTORY_ENTRIES}-entry limit`, + { details: { path: directory, op: 'readdir', limit: HOST_WALK_MAX_DIRECTORY_ENTRIES } }, + ); + } + } catch (error) { + const errorBytes = Buffer.byteLength(JSON.stringify({ + path: directory, + error: serializeError(directory, asError(error)), + })) + 1; + if (responseBytes + errorBytes > options.maxResponseBytes) { + truncated = true; + break; + } + responseBytes += errorBytes; + errors.push({ path: directory, error: asError(error) }); + continue; + } + for (const child of children) { + if (Date.now() >= deadlineMs) { + truncated = true; + break; + } + const path = pathApi.join(directory, child.name); + const isDir = child.isDirectory && child.isSymbolicLink !== true; + const relPath = pathApi.relative(root, path).split(pathApi.separator).join('/'); + if (matcher?.ignores(isDir ? `${relPath}/` : relPath) === true) continue; + const entryBytes = Buffer.byteLength(JSON.stringify({ ...child, name: path })) + 1; + if (entries.length >= options.maxEntries || responseBytes + entryBytes > options.maxResponseBytes) { + truncated = true; + break; + } + responseBytes += entryBytes; + entries.push({ ...child, name: path }); + if (isDir && depth < options.maxDepth) { + if (pending.length >= options.maxDirectories) { + truncated = true; + break; + } + pending.push({ path, depth: depth + 1 }); + } + } + } + return { entries, errors, truncated, responseBytes }; +} + +function validateLimit(name: string, value: number, min: number, max: number): void { + if (!Number.isInteger(value) || value < min || value > max) { + throw new RangeError(`${name} must be an integer in [${min}, ${max}]`); + } +} + +function asError(error: unknown): Error { + return error instanceof Error ? error : new Error(String(error)); +} diff --git a/packages/agent-core-v2/src/remote/README.md b/packages/agent-core-v2/src/remote/README.md index 555debb269f..1cdc4a90467 100644 --- a/packages/agent-core-v2/src/remote/README.md +++ b/packages/agent-core-v2/src/remote/README.md @@ -63,10 +63,17 @@ deviations per the design spec, plus one forced addition: without `maxBytes` are capped well below the 64MiB frame cap. - `fs/writeFile` takes `mode: truncate|append|exclusive` (codex only has truncate); `exclusive` maps EEXIST to `os.fs.already_exists`, which the - client maps back to `createExclusive === false`. + client maps back to `createExclusive === false`. Optional `parent` folds + directory preparation into the first write while chunks remain sequential. - `fs/readDirectory` preserves the symlink marker on each entry. The client rejects a truncated listing with `os.fs.directory_too_large` rather than exposing an incomplete directory; the current limit is 50,000 entries. +- `fs/walk` performs a bounded recursive listing in one call. It accepts the + root ignore rules needed to prune ignored directories before traversal and + reports entries, recoverable directory errors, and truncation. Clients fall + back to per-directory reads when the JSON-encoded ignore rules exceed 1MiB. + Walks default to a 55-second budget so they can return a truncated result + before the 60-second RPC deadline. - Added `fs/rename` (no codex counterpart). - `fs/remove` takes only `path` and always removes recursively and ignores a missing path, matching `IHostFileSystem.remove` (codex has `recursive`/`force`). @@ -89,7 +96,7 @@ deviations per the design spec, plus one forced addition: - `environment` adds `osArch`/`osVersion`/`pathClass` over codex (rg artifact selection needs arch), plus `cwd`/`tempDir`. - No ws transport, no `resumeSessionId`, no sandbox parameter family, no - `fs/walk`/`fs/copy`/`capabilityRoots/discoverV1`/`environmentConfig/read`. + `fs/copy`/`capabilityRoots/discoverV1`/`environmentConfig/read`. Process output is delivered through notifications (`process/output`, `process/exited`, `process/closed`). The executor retains process handles until diff --git a/packages/agent-core-v2/src/remote/client/remoteEnvironmentProvider.ts b/packages/agent-core-v2/src/remote/client/remoteEnvironmentProvider.ts index 10950441377..b58c7ba4359 100644 --- a/packages/agent-core-v2/src/remote/client/remoteEnvironmentProvider.ts +++ b/packages/agent-core-v2/src/remote/client/remoteEnvironmentProvider.ts @@ -52,7 +52,9 @@ class DeclaredRemoteEnvironment implements Environment { readonly fs: IHostFileSystem = { _serviceBrand: undefined, readText: async (path, options) => this.connected().fs.readText(path, options), + readWholeFile: async (path) => this.connected().fs.readWholeFile!(path), writeText: async (path, data) => this.connected().fs.writeText(path, data), + writeTextWithParent: async (path, data, mode, parent) => this.connected().fs.writeTextWithParent!(path, data, mode, parent), appendText: async (path, data) => this.connected().fs.appendText(path, data), readBytes: async (path, n, offset) => this.connected().fs.readBytes(path, n, offset), writeBytes: async (path, data) => this.connected().fs.writeBytes(path, data), @@ -61,6 +63,7 @@ class DeclaredRemoteEnvironment implements Environment { stat: async (path) => this.connected().fs.stat(path), lstat: async (path) => this.connected().fs.lstat(path), readdir: async (path) => this.connected().fs.readdir(path), + walk: async (path, options) => this.connected().fs.walk(path, options), mkdir: async (path, options) => this.connected().fs.mkdir(path, options), remove: async (path) => this.connected().fs.remove(path), rename: async (from, to) => this.connected().fs.rename(from, to), diff --git a/packages/agent-core-v2/src/remote/protocol/methods.ts b/packages/agent-core-v2/src/remote/protocol/methods.ts index d7a9d121451..99f4bde03ad 100644 --- a/packages/agent-core-v2/src/remote/protocol/methods.ts +++ b/packages/agent-core-v2/src/remote/protocol/methods.ts @@ -7,6 +7,7 @@ export const FS_CREATE_DIRECTORY_METHOD = 'fs/createDirectory'; export const FS_GET_METADATA_METHOD = 'fs/getMetadata'; export const FS_CANONICALIZE_METHOD = 'fs/canonicalize'; export const FS_READ_DIRECTORY_METHOD = 'fs/readDirectory'; +export const FS_WALK_METHOD = 'fs/walk'; export const FS_REMOVE_METHOD = 'fs/remove'; export const FS_RENAME_METHOD = 'fs/rename'; @@ -63,6 +64,7 @@ export interface FsWriteFileParams { readonly path: string; readonly dataBase64: string; readonly mode: FsWriteMode; + readonly parent?: string; } export interface FsCreateDirectoryParams { @@ -105,8 +107,40 @@ export interface FsReadDirectoryEntry { export interface FsReadDirectoryResult { readonly entries: FsReadDirectoryEntry[]; + readonly truncated: boolean; +} + +export interface FsWalkParams { + readonly path: string; + readonly maxDepth: number; + readonly maxDirectories: number; + readonly maxEntries: number; + readonly maxResponseBytes: number; + readonly ignore?: readonly string[]; + readonly timeoutMs?: number; +} + +export interface FsWalkEntry { + readonly name: string; + readonly isFile: boolean; + readonly isDirectory: boolean; + readonly isSymbolicLink?: boolean; +} + +export interface FsWalkError { + readonly path: string; + readonly error: { + readonly code: number; + readonly message: string; + readonly data?: unknown; + }; +} +export interface FsWalkResult { + readonly entries: readonly FsWalkEntry[]; + readonly errors: readonly FsWalkError[]; readonly truncated: boolean; + readonly responseBytes: number; } export interface FsRemoveParams { diff --git a/packages/agent-core-v2/src/remote/server/fsHandler.ts b/packages/agent-core-v2/src/remote/server/fsHandler.ts index 5326bfe9682..0125f3ca397 100644 --- a/packages/agent-core-v2/src/remote/server/fsHandler.ts +++ b/packages/agent-core-v2/src/remote/server/fsHandler.ts @@ -1,7 +1,7 @@ import { isAbsolute } from 'node:path'; import type { HostFileSystem } from '#/os/backends/node-local/hostFsService'; -import { HostFsError, OsFsErrors, toHostFsError } from '#/os/interface/hostFsErrors'; +import { HostFsError, isHostFsNotFound, OsFsErrors, toHostFsError } from '#/os/interface/hostFsErrors'; import { RpcError, RpcErrorCode } from '#/remote/protocol/errors'; import { @@ -12,7 +12,18 @@ import { type FsGetMetadataResult, type FsReadDirectoryResult, type FsReadFileResult, + type FsWalkResult, } from '#/remote/protocol/methods'; +import { + HOST_WALK_DEFAULT_TIMEOUT_MS, + HOST_WALK_MAX_DEPTH, + HOST_WALK_MAX_DIRECTORIES, + HOST_WALK_MAX_ENTRIES, + HOST_WALK_MAX_IGNORE_BYTES, + HOST_WALK_MAX_RESPONSE_BYTES, + HOST_WALK_MAX_TIMEOUT_MS, + walkHostDirectory, +} from '#/os/interface/hostFsWalk'; type Params = Record; @@ -62,6 +73,15 @@ function optionalInteger( return value; } +function optionalStringArray(params: Params, name: string): readonly string[] | undefined { + const value = params[name]; + if (value === undefined) return undefined; + if (!Array.isArray(value) || value.some((item) => typeof item !== 'string')) { + throw new RpcError(RpcErrorCode.InvalidParams, `${name} must be an array of strings`); + } + return value; +} + function fsDomainError(error: unknown, ctx: { path: string; op: string }): RpcError { const hostError = toHostFsError(error, ctx); const code = @@ -119,6 +139,24 @@ export class FsHandler { throw new RpcError(RpcErrorCode.InvalidParams, 'mode must be truncate, append or exclusive'); } const data = Buffer.from(dataBase64, 'base64'); + if (params['parent'] !== undefined) { + const parent = requireAbsolutePath(params, 'parent'); + try { + const stat = await this.fs.stat(parent); + if (!stat.isDirectory) { + throw new RpcError(RpcErrorCode.InvalidRequest, `Parent path is not a directory: ${parent}.`); + } + } catch (error) { + if (error instanceof RpcError) throw error; + if (isHostFsNotFound(error)) { + try { + await this.fs.mkdir(parent, { recursive: true }); + } catch (mkdirError) { + throw new RpcError(RpcErrorCode.InvalidRequest, mkdirError instanceof Error ? mkdirError.message : String(mkdirError)); + } + } + } + } try { if (mode === 'truncate') { await this.fs.writeBytes(path, data); @@ -196,6 +234,45 @@ export class FsHandler { } } + async walk(rawParams: unknown): Promise { + const params = requireParams(rawParams); + const path = requireAbsolutePath(params, 'path'); + const maxDepth = optionalInteger(params, 'maxDepth', 0, HOST_WALK_MAX_DEPTH) ?? HOST_WALK_MAX_DEPTH; + const maxDirectories = optionalInteger(params, 'maxDirectories', 1, HOST_WALK_MAX_DIRECTORIES) ?? HOST_WALK_MAX_DIRECTORIES; + const maxEntries = optionalInteger(params, 'maxEntries', 1, HOST_WALK_MAX_ENTRIES) ?? HOST_WALK_MAX_ENTRIES; + const maxResponseBytes = optionalInteger(params, 'maxResponseBytes', 1, HOST_WALK_MAX_RESPONSE_BYTES) ?? HOST_WALK_MAX_RESPONSE_BYTES; + const ignore = optionalStringArray(params, 'ignore'); + if (ignore !== undefined && Buffer.byteLength(JSON.stringify(ignore)) > HOST_WALK_MAX_IGNORE_BYTES) { + throw new RpcError( + RpcErrorCode.InvalidParams, + `ignore must not exceed ${HOST_WALK_MAX_IGNORE_BYTES} bytes when encoded as JSON`, + ); + } + const timeoutMs = optionalInteger(params, 'timeoutMs', 1, HOST_WALK_MAX_TIMEOUT_MS) ?? HOST_WALK_DEFAULT_TIMEOUT_MS; + const result = await walkHostDirectory(this.fs, path, { + maxDepth, + maxDirectories, + maxEntries, + maxResponseBytes, + ignore, + deadlineMs: Date.now() + timeoutMs, + }, { + serializeError: (errorPath, error) => fsDomainError(error, { + path: errorPath, + op: 'readdir', + }).toErrorBody(), + }); + return { + entries: result.entries, + errors: result.errors.map(({ path: errorPath, error }) => ({ + path: errorPath, + error: fsDomainError(error, { path: errorPath, op: 'readdir' }).toErrorBody(), + })), + truncated: result.truncated, + responseBytes: result.responseBytes, + }; + } + async remove(rawParams: unknown): Promise { const params = requireParams(rawParams); const path = requireAbsolutePath(params, 'path'); diff --git a/packages/agent-core-v2/src/remote/server/stdioHost.ts b/packages/agent-core-v2/src/remote/server/stdioHost.ts index 4acfe2e0aff..78515f49214 100644 --- a/packages/agent-core-v2/src/remote/server/stdioHost.ts +++ b/packages/agent-core-v2/src/remote/server/stdioHost.ts @@ -19,6 +19,7 @@ import { FS_READ_FILE_METHOD, FS_REMOVE_METHOD, FS_RENAME_METHOD, + FS_WALK_METHOD, FS_WRITE_FILE_METHOD, INITIALIZE_METHOD, MAX_IN_FLIGHT_CALLS, @@ -166,6 +167,7 @@ export class StdioHost { [FS_GET_METADATA_METHOD, (p) => fs.getMetadata(p)], [FS_CANONICALIZE_METHOD, (p) => fs.canonicalize(p)], [FS_READ_DIRECTORY_METHOD, (p) => fs.readDirectory(p)], + [FS_WALK_METHOD, (p) => fs.walk(p)], [FS_REMOVE_METHOD, (p) => fs.remove(p)], [FS_RENAME_METHOD, (p) => fs.rename(p)], [PROCESS_START_METHOD, (p) => pm.start(p)], @@ -283,7 +285,9 @@ export class StdioHost { ); return; } - const lane: Lane = message.method === FS_READ_FILE_METHOD ? 'data' : 'control'; + const lane: Lane = message.method === FS_READ_FILE_METHOD || message.method === FS_WALK_METHOD + ? 'data' + : 'control'; if (CONTROL_METHODS.has(message.method)) { void this.runHandler(message.id, handler, message.params, lane, false); return; diff --git a/packages/agent-core-v2/src/workspace/workspaceFs/fsService.ts b/packages/agent-core-v2/src/workspace/workspaceFs/fsService.ts index ad7189ff5f2..7a8e142e35c 100644 --- a/packages/agent-core-v2/src/workspace/workspaceFs/fsService.ts +++ b/packages/agent-core-v2/src/workspace/workspaceFs/fsService.ts @@ -63,6 +63,12 @@ import { HandshakeError } from '#/remote/client/connection'; import { ITelemetryService } from '#/app/telemetry/telemetry'; import { IHostFileSystem, type HostDirEntry, type HostFileStat } from '#/os/interface/hostFileSystem'; import { isHostFsNotDirectory, isHostFsNotFound } from '#/os/interface/hostFsErrors'; +import { + HOST_WALK_MAX_DEPTH, + HOST_WALK_MAX_DIRECTORIES, + HOST_WALK_MAX_ENTRIES, + HOST_WALK_MAX_RESPONSE_BYTES, +} from '#/os/interface/hostFsWalk'; import { type EnvironmentPath } from '#/environment/environment'; import { POSIX_ENVIRONMENT_PATH } from '#/environment/environmentDefaults'; import { IEnvironmentService, type EnvironmentResolver } from '#/app/environment/environment'; @@ -97,8 +103,7 @@ const PROBE_MAX_DEPTH = 6; const PROBE_MAX_ENTRIES = 500; const GREP_TIMEOUT_MS = 30_000; const SUGGEST_TIMEOUT_MS = 10_000; -const SUGGEST_WALK_ABORTED = new Error('suggest walk aborted'); -const WALK_MAX_DEPTH = 64; +const WALK_DEFAULT_IGNORE = ['.git/']; interface SuggestRoot { readonly dir: string; @@ -114,7 +119,10 @@ const MACOS_NOISE = new Set(['.DS_Store', '.AppleDouble', '.LSOverride']); export class WorkspaceFsService implements IWorkspaceFsService { declare readonly _serviceBrand: undefined; - private readonly gitignoreCache = new Map(); + private readonly gitignoreCache = new Map(); private rgResolution: RgResolution | null | undefined = undefined; private realRootsCache: | { readonly key: string; readonly roots: readonly { dir: string; real: string }[] } @@ -693,9 +701,10 @@ export class WorkspaceFsService implements IWorkspaceFsService { const candidates: FsSearchHit[] = []; const queryLower = req.query.toLowerCase(); const seenPaths = new Set(); + let inventoryTruncated = false; for (const root of roots) { - const matcher = req.follow_gitignore ? await this.matcherFor(root.dir) : undefined; - await this.walk(root.dir, '', matcher, async (relPath, name, kind) => { + const ignoreRules = req.follow_gitignore ? await this.walkIgnoreFor(root.dir) : WALK_DEFAULT_IGNORE; + inventoryTruncated = await this.walk(root.dir, ignoreRules, async (relPath, name, kind) => { const pathKey = `${root.real}/${relPath}`; if (seenPaths.has(pathKey)) return; seenPaths.add(pathKey); @@ -715,7 +724,7 @@ export class WorkspaceFsService implements IWorkspaceFsService { score, match_positions: computeMatchPositions(path, queryLower), }); - }); + }) || inventoryTruncated; } candidates.sort((a, b) => { @@ -725,7 +734,7 @@ export class WorkspaceFsService implements IWorkspaceFsService { const effectiveCap = Math.min(req.limit, SEARCH_HARD_CAP); const truncated = candidates.length > effectiveCap; - return { items: candidates.slice(0, effectiveCap), truncated }; + return { items: candidates.slice(0, effectiveCap), truncated: truncated || inventoryTruncated }; } async suggest(req: FsSuggestRequest): Promise { @@ -751,6 +760,7 @@ export class WorkspaceFsService implements IWorkspaceFsService { }; const cap = req.limit; + const deadline = Date.now() + SUGGEST_TIMEOUT_MS; const controller = new AbortController(); const timer = setTimeout(() => controller.abort(), SUGGEST_TIMEOUT_MS); timer.unref?.(); @@ -767,11 +777,11 @@ export class WorkspaceFsService implements IWorkspaceFsService { } catch (error) { if (controller.signal.aborted) throw error; this.telemetry.track2('fs_suggest_node_fallback', { reason: 'rg_error' }); - return await this.suggestWithNode(query, cap, controller.signal, roots); + return await this.suggestWithNode(query, cap, controller.signal, roots, deadline); } } this.telemetry.track2('fs_suggest_node_fallback', { reason: 'rg_missing' }); - return await this.suggestWithNode(query, cap, controller.signal, roots); + return await this.suggestWithNode(query, cap, controller.signal, roots, deadline); } finally { clearTimeout(timer); } @@ -1026,29 +1036,27 @@ export class WorkspaceFsService implements IWorkspaceFsService { cap: number, signal: AbortSignal, roots: readonly SuggestRoot[], + deadline: number, ): Promise { const multi = roots.length > 1; const top = new SuggestTopHeap(cap); const seenPaths = new Set(); let matched = 0; - try { - for (const root of roots) { - const matcher = query.followGitignore ? await this.matcherFor(root.dir) : undefined; - await this.walk(root.dir, '', matcher, async (relPath, _name, kind) => { - if (signal.aborted) throw SUGGEST_WALK_ABORTED; - if (multi) { - const pathKey = `${root.real}/${relPath}`; - if (seenPaths.has(pathKey)) return; - seenPaths.add(pathKey); - } - const candidate = evaluateSuggestCandidate(relPath, kind, query); - if (candidate === null) return; - matched += 1; - top.push(this.displayCandidate(root, candidate)); - }); - } - } catch (error) { - if (error !== SUGGEST_WALK_ABORTED) throw error; + let inventoryTruncated = false; + for (const root of roots) { + const ignoreRules = query.followGitignore ? await this.walkIgnoreFor(root.dir) : WALK_DEFAULT_IGNORE; + inventoryTruncated = await this.walk(root.dir, ignoreRules, async (relPath, _name, kind) => { + if (multi) { + const pathKey = `${root.real}/${relPath}`; + if (seenPaths.has(pathKey)) return; + seenPaths.add(pathKey); + } + const candidate = evaluateSuggestCandidate(relPath, kind, query); + if (candidate === null) return; + matched += 1; + top.push(this.displayCandidate(root, candidate)); + }, { deadlineMs: deadline }) || inventoryTruncated; + if (signal.aborted || Date.now() >= deadline) break; } const items = top.drain().map((candidate) => ({ path: candidate.path, @@ -1057,7 +1065,7 @@ export class WorkspaceFsService implements IWorkspaceFsService { score: candidate.score, match_positions: [...candidate.positions], })); - return { items, truncated: matched > cap || signal.aborted }; + return { items, truncated: matched > cap || signal.aborted || inventoryTruncated }; } async grep(req: FsGrepRequest): Promise { @@ -1262,9 +1270,10 @@ export class WorkspaceFsService implements IWorkspaceFsService { const filePaths: { path: string; absolute: string }[] = []; const seenPaths = new Set(); const roots = await this.suggestRoots(); + const deadline = startedAt + GREP_TIMEOUT_MS; for (const root of roots) { - const matcher = req.follow_gitignore ? await this.matcherFor(root.dir) : undefined; - await this.walk(root.dir, '', matcher, async (rel, _name, kind) => { + const ignoreRules = req.follow_gitignore ? await this.walkIgnoreFor(root.dir) : WALK_DEFAULT_IGNORE; + truncated = await this.walk(root.dir, ignoreRules, async (rel, _name, kind) => { if (kind !== 'file') return; if (req.include_globs && !matchesAnyGlob(rel, req.include_globs)) return; if (req.exclude_globs && matchesAnyGlob(rel, req.exclude_globs)) return; @@ -1275,14 +1284,16 @@ export class WorkspaceFsService implements IWorkspaceFsService { path: this.suggestDisplayPath(root, rel), absolute: this.path.join(root.dir, rel), }); - }); + }, { deadlineMs: deadline }) || truncated; + if (signal.aborted || Date.now() >= deadline) break; + } + + if ((signal.aborted || Date.now() >= deadline) && filesScanned === 0 && filePaths.length === 0) { + throw new Error2(ErrorCodes.FS_GREP_TIMEOUT, `grep timed out after ${Date.now() - startedAt}ms`); } for (const file of filePaths) { - if (signal.aborted) { - if (totalMatches === 0 && filesScanned === 0) { - throw new Error2(ErrorCodes.FS_GREP_TIMEOUT, `grep timed out after ${Date.now() - startedAt}ms`); - } + if ((signal.aborted || Date.now() >= deadline) && filesScanned > 0) { truncated = true; break; } @@ -1331,58 +1342,82 @@ export class WorkspaceFsService implements IWorkspaceFsService { private async walk( baseAbs: string, - rootRel: string, - matcher: Ignore | undefined, + ignoreRules: readonly string[] | undefined, visit: ( relPath: string, name: string, kind: 'file' | 'directory' | 'symlink', ) => Promise, - depth = 0, - ): Promise { - if (depth > WALK_MAX_DEPTH) return; - let entries: readonly HostDirEntry[]; - try { - entries = await this.hostFs.readdir(rootRel === '' ? baseAbs : this.path.join(baseAbs, rootRel)); - } catch (error) { + options: { deadlineMs?: number } = {}, + ): Promise { + const result = await this.hostFs.walk(baseAbs, { + maxDepth: HOST_WALK_MAX_DEPTH, + maxDirectories: HOST_WALK_MAX_DIRECTORIES, + maxEntries: HOST_WALK_MAX_ENTRIES, + maxResponseBytes: HOST_WALK_MAX_RESPONSE_BYTES, + ignore: ignoreRules, + deadlineMs: options.deadlineMs, + }); + for (const { path, error } of result.errors) { if (error instanceof Error2 && error.code === ErrorCodes.OS_FS_DIRECTORY_TOO_LARGE) { - throw mapFsError(error, rootRel); + throw mapFsError(error, path); } - return; } - for (const entry of entries) { - const { name } = entry; - if (name === '.git') continue; - const childRel = rootRel === '' ? name : `${rootRel}/${name}`; - const isDir = entry.isDirectory && entry.isSymbolicLink !== true; - if (matcher) { - const probe = isDir ? `${childRel}/` : childRel; - if (matcher.ignores(probe)) continue; - } - const kind: 'file' | 'directory' | 'symlink' = entry.isSymbolicLink - ? 'symlink' - : isDir - ? 'directory' - : 'file'; - await visit(childRel, name, kind); - if (isDir) { - await this.walk(baseAbs, childRel, matcher, visit, depth + 1); + const byParent = new Map(); + for (const entry of result.entries) { + const relPath = this.path.relative(baseAbs, entry.name).split(this.path.separator).join('/'); + if (relPath === '' || relPath === '..' || relPath.startsWith('../')) continue; + const slash = relPath.lastIndexOf('/'); + const parent = slash < 0 ? '' : relPath.slice(0, slash); + const name = slash < 0 ? relPath : relPath.slice(slash + 1); + const siblings = byParent.get(parent) ?? []; + siblings.push({ entry: { ...entry, name }, relPath }); + byParent.set(parent, siblings); + } + const visitDirectory = async (parent: string): Promise => { + for (const { entry, relPath } of byParent.get(parent) ?? []) { + const { name } = entry; + if (name === '.git') continue; + const isDir = entry.isDirectory && entry.isSymbolicLink !== true; + const kind: 'file' | 'directory' | 'symlink' = entry.isSymbolicLink + ? 'symlink' + : isDir + ? 'directory' + : 'file'; + await visit(relPath, name, kind); + if (isDir) await visitDirectory(relPath); } - } + }; + await visitDirectory(''); + return result.truncated; } private async matcherFor(rootDir: string): Promise { + return (await this.ignoreFor(rootDir)).matcher; + } + + private async walkIgnoreFor(rootDir: string): Promise { + return (await this.ignoreFor(rootDir)).walkRules; + } + + private async ignoreFor(rootDir: string): Promise<{ + readonly matcher: Ignore; + readonly walkRules: readonly string[]; + }> { const cached = this.gitignoreCache.get(rootDir); if (cached !== undefined) return cached; const ig = ignore(); ig.add('.git/'); + let contents = ''; try { - const contents = await this.hostFs.readText(this.path.join(rootDir, '.gitignore')); + contents = await this.hostFs.readText(this.path.join(rootDir, '.gitignore')); ig.add(contents); } catch { } - this.gitignoreCache.set(rootDir, ig); - return ig; + const rules = ['.git/', ...contents.split(/\r?\n/)]; + const result = { matcher: ig, walkRules: rules }; + this.gitignoreCache.set(rootDir, result); + return result; } private async resolveRg(): Promise { diff --git a/packages/agent-core-v2/test/agent/tools/os/rgUnavailableGuidance.test.ts b/packages/agent-core-v2/test/agent/tools/os/rgUnavailableGuidance.test.ts index 8998335b8a2..6d0811ab750 100644 --- a/packages/agent-core-v2/test/agent/tools/os/rgUnavailableGuidance.test.ts +++ b/packages/agent-core-v2/test/agent/tools/os/rgUnavailableGuidance.test.ts @@ -56,6 +56,7 @@ function createBackend(options: { stat: async () => directoryStat(), lstat: async () => directoryStat(), readdir: () => notImplemented('readdir'), + walk: () => notImplemented('walk'), mkdir: () => notImplemented('mkdir'), remove: () => notImplemented('remove'), realpath: async (path: string) => path, diff --git a/packages/agent-core-v2/test/harness/agent.ts b/packages/agent-core-v2/test/harness/agent.ts index 5e1970b0847..d2c385edb33 100644 --- a/packages/agent-core-v2/test/harness/agent.ts +++ b/packages/agent-core-v2/test/harness/agent.ts @@ -588,6 +588,7 @@ function isFullHostFs(input: unknown): boolean { 'realpath', 'stat', 'readdir', + 'walk', 'mkdir', 'remove', ]; diff --git a/packages/agent-core-v2/test/os/backends/node-local/hostFsService.test.ts b/packages/agent-core-v2/test/os/backends/node-local/hostFsService.test.ts index 6d56f42ab85..d438f27e93a 100644 --- a/packages/agent-core-v2/test/os/backends/node-local/hostFsService.test.ts +++ b/packages/agent-core-v2/test/os/backends/node-local/hostFsService.test.ts @@ -2,9 +2,10 @@ import { chmod, mkdir, mkdtemp, rm, symlink, writeFile } from 'node:fs/promises' import { tmpdir } from 'node:os'; import { join } from 'pathe'; -import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { HostFileSystem } from '#/os/backends/node-local/hostFsService'; +import { walkHostDirectory } from '#/os/interface/hostFsWalk'; let dir: string; let fs: HostFileSystem; @@ -53,6 +54,164 @@ describe('HostFileSystem streamed writes', () => { }); }); +describe('HostFileSystem walk', () => { + const options = { + maxDepth: 64, + maxDirectories: 10_000, + maxEntries: 100_000, + maxResponseBytes: 8 * 1024 * 1024, + }; + + it('lists nested entries without following directory symlinks', async () => { + await mkdir(join(dir, 'a/b'), { recursive: true }); + await writeFile(join(dir, 'a/b/file.ts'), ''); + await writeFile(join(dir, 'root.ts'), ''); + await symlink(join(dir, 'a'), join(dir, 'link')); + + const result = await fs.walk(dir, options); + + const names = result.entries.map((entry) => entry.name.slice(dir.length + 1)); + expect(names).toHaveLength(5); + expect(names).toEqual(expect.arrayContaining([ + 'a', + 'root.ts', + 'link', + 'a/b', + 'a/b/file.ts', + ])); + expect(result.entries.find((entry) => entry.name === join(dir, 'link'))?.isSymbolicLink).toBe(true); + expect(result.errors).toEqual([]); + expect(result.truncated).toBe(false); + }); + + it('prunes ignored directories before traversing them', async () => { + await mkdir(join(dir, 'node_modules/pkg'), { recursive: true }); + await mkdir(join(dir, 'src'), { recursive: true }); + await writeFile(join(dir, 'node_modules/pkg/index.ts'), ''); + await writeFile(join(dir, 'src/app.ts'), ''); + + const result = await fs.walk(dir, { ...options, ignore: ['node_modules/'] }); + + expect(result.entries.map((entry) => entry.name.slice(dir.length + 1))).toEqual(['src', 'src/app.ts']); + expect(result.truncated).toBe(false); + }); + + it('returns descendants at maxDepth but does not traverse below them', async () => { + await mkdir(join(dir, 'a/b/c'), { recursive: true }); + await writeFile(join(dir, 'a/b/c/deep.ts'), ''); + + const result = await fs.walk(dir, { ...options, maxDepth: 1 }); + + expect(result.entries.map((entry) => entry.name.slice(dir.length + 1))).toEqual(['a', 'a/b']); + expect(result.truncated).toBe(false); + }); + + it('reports missing roots as recoverable errors', async () => { + const result = await fs.walk(join(dir, 'missing'), options); + + expect(result.entries).toEqual([]); + expect(result.errors).toHaveLength(1); + expect(result.errors[0]).toMatchObject({ + path: join(dir, 'missing'), + error: { code: 'os.fs.not_found' }, + }); + }); + + it('bounds directories and entries without reporting an exact fit as truncated', async () => { + const tree = new Map([ + ['/root', [ + { name: 'a', isFile: false, isDirectory: true }, + { name: 'b', isFile: false, isDirectory: true }, + ]], + ['/root/a', [{ name: 'one.ts', isFile: true, isDirectory: false }]], + ['/root/b', [{ name: 'two.ts', isFile: true, isDirectory: false }]], + ]); + const backend = { + readdir: async (path: string) => tree.get(path) ?? [], + }; + + const exact = await walkHostDirectory(backend, '/root', { + ...options, + maxDirectories: 3, + maxEntries: 4, + }); + const boundedDirectories = await walkHostDirectory(backend, '/root', { + ...options, + maxDirectories: 2, + maxEntries: 4, + }); + const boundedEntries = await walkHostDirectory(backend, '/root', { + ...options, + maxDirectories: 3, + maxEntries: 3, + }); + + expect(exact.truncated).toBe(false); + expect(exact.entries).toHaveLength(4); + expect(boundedDirectories.truncated).toBe(true); + expect(boundedDirectories.entries.map((entry) => entry.name)).toEqual(['/root/a', '/root/b']); + expect(boundedEntries.truncated).toBe(true); + expect(boundedEntries.entries.map((entry) => entry.name)).toEqual(['/root/a', '/root/b', '/root/a/one.ts']); + }); + + it('counts entry and error JSON in the response budget', async () => { + const backend = { + readdir: async () => [{ name: 'a', isFile: true, isDirectory: false }], + }; + const initial = await walkHostDirectory(backend, '/root', options); + const tooSmall = await walkHostDirectory(backend, '/root', { ...options, maxResponseBytes: initial.responseBytes - 1 }); + + expect(initial).toMatchObject({ truncated: false }); + expect(tooSmall).toMatchObject({ entries: [], truncated: true }); + + const readWithError = async (maxResponseBytes: number) => walkHostDirectory(backend, '/root', { + ...options, + maxResponseBytes, + }, {}, async (path) => { + if (path === '/root') return [{ name: 'a', isFile: false, isDirectory: true }]; + throw new Error('unreadable'); + }); + const withError = await readWithError(options.maxResponseBytes); + const beforeError = await readWithError(withError.responseBytes - 1); + + expect(withError.errors).toHaveLength(1); + expect(beforeError).toMatchObject({ errors: [], truncated: true }); + }); + + it('returns partial entries at the deadline', async () => { + let clock = 1_000; + const now = vi.spyOn(Date, 'now').mockImplementation(() => clock); + try { + const result = await walkHostDirectory({ readdir: async () => [] }, '/root', { + ...options, + deadlineMs: clock + 5, + }, {}, async (path) => { + if (path === '/root') return [{ name: 'a', isFile: false, isDirectory: true }]; + clock += 5; + return [{ name: 'file.ts', isFile: true, isDirectory: false }]; + }); + + expect(result.entries.map((entry) => entry.name)).toEqual(['/root/a']); + expect(result.truncated).toBe(true); + } finally { + now.mockRestore(); + } + }); + + it('does not start walking at an expired deadline', async () => { + let reads = 0; + const result = await walkHostDirectory({ + readdir: async () => { + reads += 1; + return []; + }, + }, '/root', { ...options, deadlineMs: Date.now() }); + + expect(result).toMatchObject({ entries: [], truncated: true }); + expect(reads).toBe(0); + }); +}); + describe('HostFileSystem stat / lstat', () => { it('stat follows a symlink to a regular file while lstat stats the link', async () => { const target = join(dir, 'target.txt'); diff --git a/packages/agent-core-v2/test/os/backends/node-local/tools/glob.test.ts b/packages/agent-core-v2/test/os/backends/node-local/tools/glob.test.ts index dd7099e4f2c..1cac4a398fe 100644 --- a/packages/agent-core-v2/test/os/backends/node-local/tools/glob.test.ts +++ b/packages/agent-core-v2/test/os/backends/node-local/tools/glob.test.ts @@ -459,13 +459,14 @@ describe('GlobTool', () => { }); it('does not present a capped capture as a complete search when returning all matches', async () => { - const { tool } = makeTool(workspace, { - exec: execReturning(`/workspace/${'x'.repeat(200)}.ts\n`.repeat(50_000)), - }); + const exec = vi.fn(async () => fakeProcess(`/workspace/${'x'.repeat(200)}.ts\n`.repeat(50_000))); + const { tool } = makeTool(workspace, { exec }); const result = await execute(tool, GlobInputSchema.parse({ pattern: '*.ts', head_limit: 0 })); expect(result.output).toContain('stdout truncated'); expect(result.output).toContain('collected matches (partial result set)'); expect(result.output).not.toContain('Continue with'); + const process = await exec.mock.results[0]!.value as IHostProcess; + expect(process.kill).toHaveBeenCalledWith('SIGTERM'); }); it('keeps the timeout warning when the requested page has no collected matches', async () => { diff --git a/packages/agent-core-v2/test/os/backends/node-local/tools/grep.test.ts b/packages/agent-core-v2/test/os/backends/node-local/tools/grep.test.ts index 3c41eeb76ef..5a7f99b690c 100644 --- a/packages/agent-core-v2/test/os/backends/node-local/tools/grep.test.ts +++ b/packages/agent-core-v2/test/os/backends/node-local/tools/grep.test.ts @@ -1,6 +1,6 @@ import { Readable, type Writable } from 'node:stream'; -import { afterEach, describe, expect, it, vi } from 'vitest'; +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest'; import { DisposableStore, toDisposable } from '#/_base/di/lifecycle'; import { Service } from '#/_base/di/service'; @@ -46,7 +46,8 @@ import { import { GrepTool as ProductionGrepTool } from '#/agent/tools/os/grep/grepTool'; import { IAgentEnvironmentService } from '#/agent/environmentBinding/agentEnvironment'; import { FakeEnvironment } from '#/environment/fakeEnvironment'; -import { ensureRgPath } from '#/os/backends/node-local/tools/rgLocator'; +import { ensureRgPath, type RgProbe } from '#/os/backends/node-local/tools/rgLocator'; +import { HostProcessService } from '#/os/backends/node-local/hostProcessService'; import { stubWorkspaceContext } from '../../../../session/workspaceContext/stub-workspace-context'; import { stubAgentEnvironment } from '../../../../environment/stubs'; import { recordingTelemetry, type TelemetryRecord } from '../../../../app/telemetry/stubs'; @@ -70,6 +71,20 @@ class TestContributionAssembly extends Service { } const workspace: WorkspaceConfig = { workspaceDir: '/workspace', additionalDirs: ['/extra'] }; const MAX_COLUMNS_RG_ARGS = ['--max-columns', '500'] as const; +const MTIME_RG_ARGS = ['-l', '--sortr=modified'] as const; +const realRgProbe: RgProbe = { + exec: async (args) => { + const [command, ...rest] = args; + if (command === undefined) return { exitCode: -1 }; + const proc = await new HostProcessService().spawn(command, rest); + proc.stdin.end(); + proc.stdout.resume(); + proc.stderr.resume(); + const exitCode = await proc.wait(); + await proc.dispose(); + return { exitCode }; + }, +}; const COMMON_RG_ARGS = [ '--null', '--glob', @@ -160,6 +175,7 @@ function createTestFs(kaos: FakeKaos): IHostFileSystem { stat: (path) => kaos.stat(path), lstat: (path) => kaos.stat(path), readdir: () => notImplemented('readdir'), + walk: () => notImplemented('walk'), mkdir: () => notImplemented('mkdir'), remove: () => notImplemented('remove'), realpath: () => notImplemented('realpath'), @@ -222,15 +238,6 @@ function processWithOutput(stdout: string, stderr = '', exitCode = 0): IHostProc }; } -function statResult(mtime: number): HostFileStat { - return { - isFile: true, - isDirectory: false, - size: 0, - mtimeMs: mtime * 1000, - }; -} - function processThatExitsOnKill(stdout: string, stderr = '', exitCode = 143): IHostProcess { let currentExitCode: number | null = null; let resolveWait: (code: number) => void; @@ -468,7 +475,7 @@ describe('GrepTool', () => { expect(exec).toHaveBeenCalledWith( '/mock/rg', ...DEFAULT_RG_ARGS, - '-l', + ...MTIME_RG_ARGS, ...SENSITIVE_RG_ARGS, '--', 'hit', @@ -486,7 +493,7 @@ describe('GrepTool', () => { expect(exec).toHaveBeenCalledWith( '/mock/rg', ...DEFAULT_RG_ARGS, - '-l', + ...MTIME_RG_ARGS, ...SENSITIVE_RG_ARGS, '--', 'hit', @@ -524,17 +531,14 @@ describe('GrepTool', () => { expect(result.output).toBe('No non-sensitive matches found'); }); - it('sorts files_with_matches by mtime before pagination after sensitive filtering', async () => { - const stdout = ['/workspace/src/old.ts', '/workspace/.env', '/workspace/src/new.ts', ''].join( + it('uses ripgrep mtime order without stat calls before pagination after sensitive filtering', async () => { + const stdout = ['/workspace/src/new.ts', '/workspace/.env', '/workspace/src/old.ts', ''].join( '\n', ); - const stat = vi.fn(async (path: string) => { - if (path === '/workspace/src/new.ts') return statResult(10); - if (path === '/workspace/src/old.ts') return statResult(1); - throw new Error(`unexpected stat: ${path}`); - }); + const exec = vi.fn().mockResolvedValue(processWithOutput(stdout)); + const stat = vi.fn(); const tool = new GrepTool( - createFakeKaos({ exec: vi.fn().mockResolvedValue(processWithOutput(stdout)), stat }), + createFakeKaos({ exec, stat }), { workspaceDir: '/workspace', additionalDirs: [] }, ); @@ -547,122 +551,64 @@ describe('GrepTool', () => { 'Results truncated to 1 lines (total: 2). Use offset=1 to see more.', ].join('\n'), ); - expect(stat).toHaveBeenCalledTimes(2); - expect(stat).toHaveBeenCalledWith('/workspace/src/old.ts'); - expect(stat).toHaveBeenCalledWith('/workspace/src/new.ts'); + expect(exec).toHaveBeenCalledWith( + '/mock/rg', + ...DEFAULT_RG_ARGS, + ...MTIME_RG_ARGS, + ...SENSITIVE_RG_ARGS, + '--', + 'hit', + '/workspace', + ); + expect(stat).not.toHaveBeenCalled(); }); - it('keeps v1 tie order for files modified within the same second', async () => { - const stdout = ['/workspace/src/a.ts', '/workspace/src/b.ts', '/workspace/src/c.ts', ''].join( - '\n', - ); - const stat = vi.fn(async (path: string) => { - if (path === '/workspace/src/a.ts') { - return { ...statResult(1), mtimeMs: 1000 }; - } - if (path === '/workspace/src/b.ts') { - return { ...statResult(1), mtimeMs: 1500 }; - } - if (path === '/workspace/src/c.ts') { - return { ...statResult(2), mtimeMs: 2000 }; + describe('with real ripgrep', () => { + let runRealRg = false; + + beforeAll(async () => { + try { + const actual = await vi.importActual( + '#/os/backends/node-local/tools/rgLocator', + ); + const resolution = await actual.ensureRgPath(realRgProbe, { allowCachedFallback: false }); + vi.mocked(ensureRgPath).mockResolvedValue(resolution); + runRealRg = true; + } catch { } - throw new Error(`unexpected stat: ${path}`); }); - const tool = new GrepTool( - createFakeKaos({ exec: vi.fn().mockResolvedValue(processWithOutput(stdout)), stat }), - { workspaceDir: '/workspace', additionalDirs: [] }, - ); - const result = await executeTool(tool, context({ pattern: 'hit', head_limit: 0 })); - - expect(toolContentString(result)).toBe(['src/c.ts', 'src/a.ts', 'src/b.ts'].join('\n')); - }); - - it('limits concurrent mtime stats while sorting files_with_matches', async () => { - const filePaths = Array.from( - { length: 40 }, - (_, index) => `/workspace/src/file-${String(index).padStart(2, '0')}.ts`, - ); - let activeStats = 0; - let maxActiveStats = 0; - const stat = vi.fn(async (path: string) => { - activeStats += 1; - maxActiveStats = Math.max(maxActiveStats, activeStats); - await new Promise((resolve) => { - setTimeout(resolve, 0); - }); - activeStats -= 1; - const mtime = Number(path.match(/file-(\d+)\.ts$/)?.[1] ?? 0); - return statResult(mtime); + beforeEach((testCtx) => { + if (!runRealRg) testCtx.skip(); }); - const tool = new GrepTool( - createFakeKaos({ - exec: vi.fn().mockResolvedValue(processWithOutput(`${filePaths.join('\n')}\n`)), - stat, - }), - { workspaceDir: '/workspace', additionalDirs: [] }, - ); - - const result = await executeTool(tool, context({ pattern: 'hit', head_limit: 0 })); - const lines = toolContentString(result).split('\n'); - - expect(stat).toHaveBeenCalledTimes(filePaths.length); - expect(maxActiveStats).toBeLessThanOrEqual(32); - expect(lines.at(0)).toBe('src/file-39.ts'); - expect(lines.at(-1)).toBe('src/file-00.ts'); - }); - it('stops scheduling mtime stats when aborted during files_with_matches sorting', async () => { - const filePaths = Array.from( - { length: 40 }, - (_, index) => `/workspace/src/file-${String(index).padStart(2, '0')}.ts`, - ); - const abortController = new AbortController(); - const stat = vi.fn(async () => { - abortController.abort(); - await new Promise((resolve) => { - setTimeout(resolve, 0); - }); - return statResult(1); + afterAll(() => { + vi.mocked(ensureRgPath).mockResolvedValue({ path: '/mock/rg', source: 'system-path' }); }); - const tool = new GrepTool( - createFakeKaos({ - exec: vi.fn().mockResolvedValue(processWithOutput(`${filePaths.join('\n')}\n`)), - stat, - }), - { workspaceDir: '/workspace', additionalDirs: [] }, - ); - - const result = await executeTool(tool, - context({ pattern: 'hit', head_limit: 0 }, abortController.signal), - ); - - expect(result).toMatchObject({ isError: true, output: 'Grep aborted' }); - expect(stat.mock.calls.length).toBeLessThan(filePaths.length); - }); - it('keeps files_with_matches entries when mtime stat fails', async () => { - const stdout = [ - '/workspace/src/old.ts', - '/workspace/src/missing.ts', - '/workspace/src/new.ts', - '', - ].join('\n'); - const stat = vi.fn(async (path: string) => { - if (path === '/workspace/src/new.ts') return statResult(10); - if (path === '/workspace/src/old.ts') return statResult(1); - throw new Error('stat failed'); + it('lets ripgrep order matching files newest-first', async () => { + const { mkdtemp, mkdir, rm, utimes, writeFile } = await import('node:fs/promises'); + const { tmpdir } = await import('node:os'); + const { join } = await import('node:path'); + const dir = await mkdtemp(join(tmpdir(), 'grep-rg-sort-')); + try { + await mkdir(join(dir, 'src')); + for (const [name, mtime] of [['old.ts', 1_000], ['new.ts', 2_000]] as const) { + const file = join(dir, 'src', name); + await writeFile(file, 'hit\n'); + await utimes(file, mtime, mtime); + } + const processService = new HostProcessService(); + const exec = vi.fn((command: string, ...args: string[]) => processService.spawn(command, args)); + const tool = new GrepTool(createFakeKaos({ exec }), { workspaceDir: dir, additionalDirs: [] }); + + const result = await executeTool(tool, context({ pattern: 'hit' })); + + expect(toolContentString(result)).toBe('src/new.ts\nsrc/old.ts'); + } finally { + await rm(dir, { recursive: true, force: true }); + } }); - const tool = new GrepTool( - createFakeKaos({ exec: vi.fn().mockResolvedValue(processWithOutput(stdout)), stat }), - { workspaceDir: '/workspace', additionalDirs: [] }, - ); - - const result = await executeTool(tool, context({ pattern: 'hit', head_limit: 0 })); - - expect(toolContentString(result)).toBe( - ['src/new.ts', 'src/old.ts', 'src/missing.ts'].join('\n'), - ); }); it('uses count-matches and ignores context flags outside content output mode', async () => { @@ -705,7 +651,7 @@ describe('GrepTool', () => { '-j', '1', ...DEFAULT_RG_ARGS, - '-l', + ...MTIME_RG_ARGS, ...SENSITIVE_RG_ARGS, '--', 'hit', @@ -730,7 +676,7 @@ describe('GrepTool', () => { expect(exec).toHaveBeenCalledWith( '/mock/rg', ...DEFAULT_RG_ARGS, - '-l', + ...MTIME_RG_ARGS, '-i', '--type', 'ts', @@ -821,7 +767,7 @@ describe('GrepTool', () => { expect(exec).toHaveBeenCalledWith( '/mock/rg', ...DEFAULT_RG_ARGS, - '-l', + ...MTIME_RG_ARGS, '--glob', '**/.env', ...SENSITIVE_RG_ARGS, diff --git a/packages/agent-core-v2/test/os/backends/node-local/tools/read.test.ts b/packages/agent-core-v2/test/os/backends/node-local/tools/read.test.ts index 87c84919874..4769a8a6253 100644 --- a/packages/agent-core-v2/test/os/backends/node-local/tools/read.test.ts +++ b/packages/agent-core-v2/test/os/backends/node-local/tools/read.test.ts @@ -455,6 +455,25 @@ describe('ReadTool', () => { }); }); + it('uses a whole-file capability without separate stat, sniff and line reads', async () => { + const { fs, stat, readBytes, readLines } = createSpiedFs('alpha\nbeta\n'); + const bytes = Buffer.from('alpha\nbeta\n'); + const readWholeFile = vi.fn(async () => ({ + stat: { isFile: true, isDirectory: false, size: bytes.length }, + bytes, + })); + fs.readWholeFile = readWholeFile; + const tool = createReadTool(fs, createTestEnv(), PERMISSIVE_WORKSPACE); + + const result = await execute(tool, { path: '/tmp/a.txt' }); + + expect(result.output).toBe('1\talpha\n2\tbeta'); + expect(readWholeFile).toHaveBeenCalledWith('/tmp/a.txt'); + expect(stat).not.toHaveBeenCalled(); + expect(readBytes).not.toHaveBeenCalled(); + expect(readLines).not.toHaveBeenCalled(); + }); + it('stats the resolved target so symlinked files stay readable', async () => { const { fs, stat } = createSpiedFs('alpha\n'); const tool = createReadTool(fs, createTestEnv(), PERMISSIVE_WORKSPACE); diff --git a/packages/agent-core-v2/test/os/backends/node-local/tools/write.test.ts b/packages/agent-core-v2/test/os/backends/node-local/tools/write.test.ts index e1565adc2ff..52a8d5392c1 100644 --- a/packages/agent-core-v2/test/os/backends/node-local/tools/write.test.ts +++ b/packages/agent-core-v2/test/os/backends/node-local/tools/write.test.ts @@ -257,6 +257,19 @@ describe('WriteTool', () => { expect(writeText).toHaveBeenCalledWith('/tmp/missing-dir/file.txt', 'data'); }); + it('uses the combined parent-and-write operation when available', async () => { + const { tool, fs, stat, writeText } = makeTool(); + const writeTextWithParent = vi.fn(async () => {}); + fs.writeTextWithParent = writeTextWithParent; + + const result = await execute(tool, { path: '/tmp/missing-dir/file.txt', content: 'data' }); + + expect(result.isError).toBeUndefined(); + expect(writeTextWithParent).toHaveBeenCalledWith('/tmp/missing-dir/file.txt', 'data', 'overwrite', '/tmp/missing-dir'); + expect(stat).not.toHaveBeenCalled(); + expect(writeText).not.toHaveBeenCalled(); + }); + it('surfaces mkdir failures when a missing parent cannot be created', async () => { const enoent = Object.assign(new Error('ENOENT: no such file or directory'), { code: 'ENOENT', diff --git a/packages/agent-core-v2/test/program/program.test.ts b/packages/agent-core-v2/test/program/program.test.ts index 0db0ad74b0e..1414585666b 100644 --- a/packages/agent-core-v2/test/program/program.test.ts +++ b/packages/agent-core-v2/test/program/program.test.ts @@ -353,6 +353,7 @@ function scopedFs(base: string, realBase: string, inner: IHostFileSystem, additi stat: (path) => { assert(path); return inner.stat(path); }, lstat: inner.lstat.bind(inner), readdir: (path) => { assert(path); return inner.readdir(path); }, + walk: (path, options) => { assert(path); return inner.walk(path, options); }, mkdir: (path, options) => { assert(path); return inner.mkdir(path, options); }, remove: (path) => { assert(path); return inner.remove(path); }, realpath: inner.realpath.bind(inner), diff --git a/packages/agent-core-v2/test/remote/fs.test.ts b/packages/agent-core-v2/test/remote/fs.test.ts index 1a671165b2d..f5689d2af95 100644 --- a/packages/agent-core-v2/test/remote/fs.test.ts +++ b/packages/agent-core-v2/test/remote/fs.test.ts @@ -1,15 +1,24 @@ -import { mkdtemp, rm, writeFile } from 'node:fs/promises'; +import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; -import { afterAll, beforeAll, describe, expect, it } from 'vitest'; +import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest'; +import { FileEditService } from '#/app/edit/fileEditService'; import { HostFsError } from '#/os/interface/hostFsErrors'; +import { HOST_WALK_MAX_IGNORE_BYTES } from '#/os/interface/hostFsWalk'; import { RemoteExecConnection } from '#/remote/client/connection'; import { RemoteFileSystem } from '#/os/backends/remote/remoteFileSystem'; -import { FS_READ_DIRECTORY_MAX_ENTRIES, FS_READ_FILE_WHOLE_MAX_BYTES } from '#/remote/protocol/methods'; import { + FS_READ_DIRECTORY_MAX_ENTRIES, + FS_READ_DIRECTORY_METHOD, + FS_READ_FILE_WHOLE_MAX_BYTES, + FS_WALK_METHOD, +} from '#/remote/protocol/methods'; +import { RpcErrorCode } from '#/remote/protocol/errors'; +import { + connectInProcess, connectSubprocess, createInProcessLoopback, RawClient, @@ -34,6 +43,54 @@ describe('fs group over a subprocess loopback', () => { await rm(workDir, { recursive: true, force: true }); }); + it('reads a small whole file in one RPC', async () => { + const path = join(workDir, 'whole.txt'); + await writeFile(path, 'alpha\nbeta\n'); + const calls = vi.spyOn(connection, 'call'); + try { + const result = await fs.readWholeFile(path); + expect(Buffer.from(result?.bytes ?? []).toString('utf8')).toBe('alpha\nbeta\n'); + expect(result?.stat).toEqual({ isFile: true, isDirectory: false, size: 11 }); + expect(calls.mock.calls.map(([method]) => method)).toEqual(['fs/readFile']); + } finally { + calls.mockRestore(); + } + }); + + it('edits a small file with one read and one write RPC', async () => { + const path = join(workDir, 'edit.txt'); + await writeFile(path, 'before\nold value\n'); + const editor = new FileEditService(fs); + const calls = vi.spyOn(connection, 'call'); + try { + const result = await editor.edit({ + path, + displayPath: 'edit.txt', + old_string: 'old value', + new_string: 'new value', + replace_all: false, + }); + expect(result).toEqual({ ok: true, count: 1 }); + expect(calls.mock.calls.map(([method]) => method)).toEqual(['fs/readFile', 'fs/writeFile']); + expect(await readFile(path, 'utf8')).toBe('before\nnew value\n'); + } finally { + calls.mockRestore(); + } + }); + + it('creates the parent and writes text in one RPC', async () => { + const parent = join(workDir, 'created', 'nested'); + const path = join(parent, 'out.txt'); + const calls = vi.spyOn(connection, 'call'); + try { + await fs.writeTextWithParent(path, 'hello', 'overwrite', parent); + expect(calls.mock.calls.map(([method]) => method)).toEqual(['fs/writeFile']); + expect(await readFile(path, 'utf8')).toBe('hello'); + } finally { + calls.mockRestore(); + } + }); + it('writes a byte stream progressively without retaining the complete input', async () => { const path = join(workDir, 'stream.bin'); await fs.writeText(path, 'old content'); @@ -126,6 +183,147 @@ describe('fs group over a subprocess loopback', () => { }); describe('fs protocol semantics', () => { + const walkOptions = { + maxDepth: 64, + maxDirectories: 10_000, + maxEntries: 100_000, + maxResponseBytes: 8 * 1024 * 1024, + }; + + it('falls back to bounded reads when a whole-file read is rejected', async () => { + const calls: Array<{ maxBytes?: number }> = []; + const fs = new RemoteFileSystem({ + call: async (method, params) => { + if (method !== 'fs/readFile') throw new Error(`unexpected method: ${method}`); + const maxBytes = (params as { maxBytes?: number }).maxBytes; + calls.push({ maxBytes }); + if (maxBytes === undefined) throw new Error('whole read rejected'); + return { dataBase64: Buffer.from('fallback').toString('base64'), eof: true }; + }, + notify: () => {}, + onNotification: () => () => {}, + onDidClose: () => {}, + }); + + expect(await fs.readText('/tmp/file.txt')).toBe('fallback'); + expect(calls).toEqual([{ maxBytes: undefined }, { maxBytes: 1024 * 1024 }]); + }); + + it('honors the caller deadline before and during a remote walk', async () => { + const calls: Array<{ method: string; timeoutMs?: number }> = []; + const fs = new RemoteFileSystem({ + call: async (method, params) => { + calls.push({ method, timeoutMs: (params as { timeoutMs: number }).timeoutMs }); + return { entries: [], errors: [], truncated: false, responseBytes: 64 }; + }, + notify: () => {}, + onNotification: () => () => {}, + onDidClose: () => {}, + }); + + const active = await fs.walk('/root', { ...walkOptions, deadlineMs: Date.now() + 1_000 }); + const expired = await fs.walk('/root', { ...walkOptions, deadlineMs: Date.now() }); + + expect(active.truncated).toBe(false); + expect(calls).toHaveLength(1); + expect(calls[0]?.method).toBe(FS_WALK_METHOD); + expect(calls[0]?.timeoutMs).toBeGreaterThan(0); + expect(calls[0]?.timeoutMs).toBeLessThanOrEqual(1_000); + expect(expired).toEqual({ entries: [], errors: [], truncated: true, responseBytes: 0 }); + }); + + it('walks nested directories in one rpc and prunes ignored subtrees', async () => { + const dir = await mkdtemp(join(tmpdir(), 'remote-exec-walk-')); + const { connection, loopback } = await connectInProcess(); + const methods: string[] = []; + loopback.clientInput.on('data', (chunk: Buffer) => { + for (const line of chunk.toString('utf8').trim().split('\n')) { + if (line.length > 0) methods.push((JSON.parse(line) as { method?: string }).method ?? ''); + } + }); + try { + await mkdir(join(dir, 'src/nested'), { recursive: true }); + await mkdir(join(dir, 'node_modules/pkg'), { recursive: true }); + await writeFile(join(dir, 'src/nested/app.ts'), ''); + await writeFile(join(dir, 'node_modules/pkg/index.ts'), ''); + const fs = new RemoteFileSystem(connection); + const result = await fs.walk(dir, { + ...walkOptions, + ignore: ['node_modules/'], + }); + + expect(result.entries.map((entry) => entry.name.slice(dir.length + 1)).toSorted()).toEqual([ + 'src', + 'src/nested', + 'src/nested/app.ts', + ]); + expect(result.errors).toEqual([]); + expect(result.truncated).toBe(false); + expect(methods.filter((method) => method === FS_WALK_METHOD)).toHaveLength(1); + expect(methods.filter((method) => method === FS_READ_DIRECTORY_METHOD)).toHaveLength(0); + } finally { + connection.close(); + await loopback.host.done; + await rm(dir, { recursive: true, force: true }); + } + }); + + it('falls back locally for oversized ignore rules without pruning them', async () => { + const dir = await mkdtemp(join(tmpdir(), 'remote-exec-walk-ignore-cap-')); + const { connection, loopback } = await connectInProcess(); + const methods: string[] = []; + loopback.clientInput.on('data', (chunk: Buffer) => { + for (const line of chunk.toString('utf8').trim().split('\n')) { + if (line.length > 0) methods.push((JSON.parse(line) as { method?: string }).method ?? ''); + } + }); + try { + await mkdir(join(dir, 'src')); + await mkdir(join(dir, 'ignored')); + await writeFile(join(dir, 'src/app.ts'), ''); + await writeFile(join(dir, 'ignored/app.ts'), ''); + const fs = new RemoteFileSystem(connection); + const unusedRule = `unused-${'x'.repeat(8 * 1024)}`; + const unusedRuleCount = Math.ceil(HOST_WALK_MAX_IGNORE_BYTES / Buffer.byteLength(JSON.stringify(unusedRule))); + + const result = await fs.walk(dir, { + ...walkOptions, + ignore: ['ignored/', ...Array.from({ length: unusedRuleCount }, () => unusedRule)], + }); + + expect(result.entries.map((entry) => entry.name.slice(dir.length + 1)).toSorted()).toEqual([ + 'src', + 'src/app.ts', + ]); + expect(result.truncated).toBe(false); + expect(methods.filter((method) => method === FS_WALK_METHOD)).toHaveLength(0); + expect(methods.filter((method) => method === FS_READ_DIRECTORY_METHOD)).toHaveLength(2); + } finally { + connection.close(); + await loopback.host.done; + await rm(dir, { recursive: true, force: true }); + } + }); + + it('rejects oversized ignore rules on the wire', async () => { + const loopback = createInProcessLoopback(); + const raw = new RawClient(loopback); + await raw.handshake(); + raw.send({ + id: 1, + method: FS_WALK_METHOD, + params: { + path: '/root', + ignore: ['x'.repeat(HOST_WALK_MAX_IGNORE_BYTES)], + }, + }); + + const response = await raw.nextResponse(1); + expect(response['error']).toMatchObject({ code: RpcErrorCode.InvalidParams }); + loopback.clientInput.end(); + await loopback.host.done; + }); + it('reports capped directory listings on the wire and rejects them through the filesystem interface', async () => { const dir = await mkdtemp(join(tmpdir(), 'remote-exec-readdir-cap-')); const smallDir = await mkdtemp(join(tmpdir(), 'remote-exec-readdir-small-')); diff --git a/packages/agent-core-v2/test/tools/fixtures/fake-exec.ts b/packages/agent-core-v2/test/tools/fixtures/fake-exec.ts index f8efa4c47d3..10ad7e060a3 100644 --- a/packages/agent-core-v2/test/tools/fixtures/fake-exec.ts +++ b/packages/agent-core-v2/test/tools/fixtures/fake-exec.ts @@ -28,6 +28,7 @@ export function createFakeHostFs(overrides: Partial = {}): IHos stat: () => notImplemented('FakeHostFs.stat'), lstat: () => notImplemented('FakeHostFs.lstat'), readdir: () => notImplemented('FakeHostFs.readdir'), + walk: () => notImplemented('FakeHostFs.walk'), mkdir: () => notImplemented('FakeHostFs.mkdir'), remove: () => notImplemented('FakeHostFs.remove'), realpath: () => notImplemented('FakeHostFs.realpath'), diff --git a/packages/agent-core-v2/test/workspace/workspaceFs/fsService.test.ts b/packages/agent-core-v2/test/workspace/workspaceFs/fsService.test.ts index 7b9f16c1a72..c92d197e568 100644 --- a/packages/agent-core-v2/test/workspace/workspaceFs/fsService.test.ts +++ b/packages/agent-core-v2/test/workspace/workspaceFs/fsService.test.ts @@ -1,7 +1,7 @@ import { isAbsolute, join, relative, resolve } from 'node:path'; import { Readable, Writable } from 'node:stream'; -import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { LifecycleScope } from '#/app/scopes'; import { ScopeActivation, @@ -15,6 +15,7 @@ import { EnvironmentError } from '#/environment/environmentRegistry'; import { HandshakeError } from '#/remote/client/connection'; import { type HostDirEntry, IHostFileSystem } from '#/os/interface/hostFileSystem'; import { HostFsError } from '#/os/interface/hostFsErrors'; +import { walkHostDirectory } from '#/os/interface/hostFsWalk'; import { IHostProcessService, type IHostProcess } from '#/os/interface/hostProcess'; import { IWorkspaceFsService } from '#/workspace/workspaceFs/fs'; import { WorkspaceFsService } from '#/workspace/workspaceFs/fsService'; @@ -158,7 +159,7 @@ function fakeFs( } throw enoent(p); }; - return { + const fs: IHostFileSystem = { _serviceBrand: undefined, readText: async (p) => { const c = fileMap.get(resolveSymlinks(p)); @@ -214,6 +215,7 @@ function fakeFs( for (const s of symlinkSet) visit(s, 'symlink'); return [...children.values()]; }, + walk: (path, options) => walkHostDirectory(fs, path, options), mkdir: async (p, options) => { const recursive = options?.recursive ?? false; const exists = isDir(p) || fileMap.has(p); @@ -266,6 +268,7 @@ function fakeFs( return current; }, }; + return fs; } function fakeProcess(stdout: string, stderr: string, exitCode: number): IHostProcess { @@ -791,6 +794,26 @@ describe('WorkspaceFsService.search', () => { expect(paths.some((p) => p.startsWith('src/link/'))).toBe(false); }); + it('keeps the original deep-directory boundary', async () => { + const files: Record = {}; + let current = 'root'; + files[`${current}/match.ts`] = ''; + for (let depth = 1; depth <= 66; depth += 1) { + current = `${current}/level-${depth}`; + if (depth === 63 || depth === 64 || depth === 65) files[`${current}/match.ts`] = ''; + } + const fs = makeSession(files, emptyHandler); + const deepPath = Array.from({ length: 63 }, (_, index) => `level-${index + 1}`).join('/'); + + const result = await fs.search({ query: 'match', limit: 50, follow_gitignore: false }); + + expect(result.items.map((item) => item.path).toSorted()).toEqual([ + 'root/match.ts', + `root/${deepPath}/match.ts`, + ].toSorted()); + expect(result.items.some((item) => item.path.includes('level-64/'))).toBe(false); + }); + it('lists the workspace root top-level entries when the query is empty', async () => { const fs = makeSession( { 'src/foo.ts': '', 'src/nested/deep.ts': '', 'README.md': '', '.hidden.ts': '' }, @@ -1179,6 +1202,79 @@ describe('WorkspaceFsService.suggest', () => { expect(on.items.map((i) => i.path)).toContain('.secret.ts'); }); + it('passes ignore rules to the walk and reports inventory truncation', async () => { + const backend = fakeFs({ + '.gitignore': 'node_modules/\n', + 'src/app.ts': '', + 'node_modules/pkg/index.ts': '', + }); + let receivedIgnore: readonly string[] | undefined; + backend.walk = async (_path, options) => { + receivedIgnore = options.ignore; + return { entries: [], errors: [], truncated: true, responseBytes: 64 }; + }; + const fs = makeRemoteSession( + { '.gitignore': 'node_modules/\n' }, + () => ({ stdout: '', exitCode: 127 }), + [], + 'ssh-dev', + '/home/target', + backend, + ); + + const result = await fs.suggest({ + query: 'app', + limit: 50, + show_hidden: false, + follow_gitignore: true, + }); + + expect(receivedIgnore).toEqual(['.git/', 'node_modules/', '']); + expect(result).toEqual({ items: [], truncated: true }); + }); + + it('keeps matching and file enumeration order when using batched walks', async () => { + const backend = fakeFs({ + '.git/config': 'hello\n', + 'a/first.ts': 'hello\n', + 'a/nested/second.ts': 'hello\n', + 'z.ts': 'hello\n', + }); + const readdir = backend.readdir.bind(backend); + const reads: string[] = []; + backend.readdir = async (path) => { + reads.push(path); + return readdir(path); + }; + const fs = makeRemoteSession( + {}, + () => ({ stdout: '', exitCode: 127 }), + [], + 'ssh-dev', + '/home/target', + backend, + ); + + const result = await fs.grep({ + pattern: 'hello', + regex: false, + case_sensitive: true, + follow_gitignore: false, + max_files: 200, + max_matches_per_file: 50, + max_total_matches: 5000, + context_lines: 0, + }); + + expect(result.files.map((file) => file.path)).toEqual([ + 'a/nested/second.ts', + 'a/first.ts', + 'z.ts', + ]); + expect(result.files.some((file) => file.path.startsWith('.git/'))).toBe(false); + expect(reads).toEqual(['/repo', '/repo/a', '/repo/a/nested']); + }); + it('falls back to the node walk when rg fails to spawn', async () => { const events: Array<{ event: string; properties: Record }> = []; const runner: IHostProcessService = { @@ -1493,6 +1589,89 @@ describe('WorkspaceFsService.suggest', () => { expect(skipped.items.map((i) => i.path)).toEqual([]); }); + it('keeps partial candidates at the shared deadline without walking later roots', async () => { + const backend = fakeFs({ 'src/foo.ts': '', '/extra/foo-extra.ts': '' }); + let clock = 1_000; + const now = vi.spyOn(Date, 'now').mockImplementation(() => clock); + let deadline: number | undefined; + const roots: string[] = []; + backend.walk = async (path, options) => { + roots.push(path); + deadline = options.deadlineMs; + clock += 10_000; + return { + entries: [ + { name: '/repo/src', isFile: false, isDirectory: true }, + { name: '/repo/src/foo.ts', isFile: true, isDirectory: false }, + ], + errors: [], + truncated: true, + responseBytes: 128, + }; + }; + try { + const fs = makeRemoteSession({}, () => ({ stdout: '', exitCode: 127 }), [], 'ssh-dev', '/home/target', backend); + Object.defineProperty(fs, 'suggestRoots', { + value: async () => [ + { dir: WORK_DIR, real: WORK_DIR, primary: true }, + { dir: '/extra', real: '/extra', primary: false }, + ], + }); + + const result = await fs.suggest({ + query: 'foo', + limit: 50, + follow_gitignore: false, + show_hidden: false, + }); + + expect(result.items.map((item) => item.path)).toEqual(['src/foo.ts']); + expect(result.truncated).toBe(true); + expect(roots).toEqual([WORK_DIR]); + expect(deadline).toBe(11_000); + } finally { + now.mockRestore(); + } + }); + + it('does not restart the suggest budget after the rg attempt', async () => { + const backend = fakeFs({ 'src/foo.ts': '' }); + let clock = 1_000; + const now = vi.spyOn(Date, 'now').mockImplementation(() => clock); + let deadline: number | undefined; + backend.walk = async (_path, options) => { + deadline = options.deadlineMs; + return { entries: [], errors: [], truncated: false, responseBytes: 64 }; + }; + const runner: IHostProcessService = { + _serviceBrand: undefined, + spawn: async (_command, args) => { + if (args?.[0] === '--version') return fakeProcess('ripgrep 15.0.0', '', 0); + clock += 4_000; + throw new Error('spawn failed'); + }, + }; + try { + const fs = makeRemoteSession({}, emptyHandler, [], 'ssh-dev', '/home/target', backend); + Object.defineProperty(fs, 'resolver', { + value: { + acquire: () => ({ + environment: { process: runner }, + track: (resource: unknown) => resource, + dispose: () => {}, + }), + }, + }); + Object.defineProperty(fs, 'rgResolution', { value: { path: 'rg' } }); + + await fs.suggest({ query: 'foo', limit: 50, follow_gitignore: false, show_hidden: false }); + + expect(deadline).toBe(11_000); + } finally { + now.mockRestore(); + } + }); + it('lists top-level entries of every root for an empty query', async () => { const fs = makeSession( { 'src/foo.ts': '', 'README.md': '', '/extra/lib/util.ts': '', '/extra/notes.md': '' }, @@ -1774,6 +1953,70 @@ describe('WorkspaceFsService.grep', () => { expect(result.files.map((file) => file.path)).toEqual(['src/a.ts', '/extra/b.ts']); expect(result.files_scanned).toBe(2); }); + + it('distinguishes partial matches from an empty timed-out inventory', async () => { + const backend = fakeFs({ 'src/a.ts': 'hello primary\n', '/extra/b.ts': 'hello extra\n' }); + let clock = 1_000; + const now = vi.spyOn(Date, 'now').mockImplementation(() => clock); + const roots: string[] = []; + backend.walk = async (path) => { + roots.push(path); + clock += 30_000; + return { + entries: [ + { name: '/repo/src', isFile: false, isDirectory: true }, + { name: '/repo/src/a.ts', isFile: true, isDirectory: false }, + ], + errors: [], + truncated: true, + responseBytes: 128, + }; + }; + try { + const fs = makeRemoteSession({}, () => ({ stdout: '', exitCode: 127 }), [], 'ssh-dev', '/home/target', backend); + Object.defineProperty(fs, 'suggestRoots', { + value: async () => [ + { dir: WORK_DIR, real: WORK_DIR, primary: true }, + { dir: '/extra', real: '/extra', primary: false }, + ], + }); + + const result = await fs.grep({ + pattern: 'hello', + regex: false, + case_sensitive: true, + follow_gitignore: false, + max_files: 200, + max_matches_per_file: 50, + max_total_matches: 5000, + context_lines: 0, + }); + + expect(result.files.map((file) => file.path)).toEqual(['src/a.ts']); + expect(result.files_scanned).toBe(1); + expect(result.truncated).toBe(true); + expect(roots).toEqual([WORK_DIR]); + backend.walk = async () => { + clock += 30_000; + return { entries: [], errors: [], truncated: true, responseBytes: 64 }; + }; + await expect(fs.grep({ + pattern: 'hello', + regex: false, + case_sensitive: true, + follow_gitignore: false, + max_files: 200, + max_matches_per_file: 50, + max_total_matches: 5000, + context_lines: 0, + })).rejects.toMatchObject({ + code: ErrorCodes.FS_GREP_TIMEOUT, + message: 'grep timed out after 30000ms', + }); + } finally { + now.mockRestore(); + } + }); }); describe('WorkspaceFsService.list', () => { diff --git a/packages/kap-server/test/workspaceFs.test.ts b/packages/kap-server/test/workspaceFs.test.ts index 047201befef..061bc190aed 100644 --- a/packages/kap-server/test/workspaceFs.test.ts +++ b/packages/kap-server/test/workspaceFs.test.ts @@ -463,6 +463,7 @@ function mappingHostFs(remoteRoot: string): IHostFileSystem { stat: (path) => inner.stat(map(path)), lstat: (path) => inner.lstat(map(path)), readdir: (path) => inner.readdir(map(path)), + walk: (path, options) => inner.walk(map(path), options), mkdir: (path, options) => inner.mkdir(map(path), options), remove: (path) => inner.remove(map(path)), realpath: async (path) => unmap(await inner.realpath(map(path))),