From 3f497c509998b36918691e988dcfc2c03a20fd42 Mon Sep 17 00:00:00 2001 From: 7Sageer Date: Fri, 30 Oct 2026 12:29:55 +0800 Subject: [PATCH 1/8] perf: batch remote filesystem walks --- .changeset/batch-remote-filesystem-walks.md | 5 + .../acp-server/src/acp-fs/acpFsService.ts | 6 + .../os/backends/node-local/hostFsService.ts | 13 +- .../os/backends/remote/remoteFileSystem.ts | 63 ++++ .../src/os/interface/hostFileSystem.ts | 22 ++ .../src/os/interface/hostFsWalk.ts | 123 ++++++++ packages/agent-core-v2/src/remote/README.md | 8 +- .../client/remoteEnvironmentProvider.ts | 1 + .../src/remote/protocol/methods.ts | 33 ++ .../src/remote/server/fsHandler.ts | 59 ++++ .../src/remote/server/stdioHost.ts | 6 +- .../src/workspace/workspaceFs/fsService.ts | 163 ++++++---- .../tools/os/rgUnavailableGuidance.test.ts | 1 + packages/agent-core-v2/test/harness/agent.ts | 1 + .../backends/node-local/hostFsService.test.ts | 180 +++++++++++ .../os/backends/node-local/tools/grep.test.ts | 1 + .../test/program/program.test.ts | 1 + packages/agent-core-v2/test/remote/fs.test.ts | 285 +++++++++++++++++- .../test/tools/fixtures/fake-exec.ts | 1 + .../workspace/workspaceFs/fsService.test.ts | 234 +++++++++++++- packages/kap-server/test/workspaceFs.test.ts | 1 + 21 files changed, 1135 insertions(+), 72 deletions(-) create mode 100644 .changeset/batch-remote-filesystem-walks.md create mode 100644 packages/agent-core-v2/src/os/interface/hostFsWalk.ts diff --git a/.changeset/batch-remote-filesystem-walks.md b/.changeset/batch-remote-filesystem-walks.md new file mode 100644 index 00000000000..bdf6664659a --- /dev/null +++ b/.changeset/batch-remote-filesystem-walks.md @@ -0,0 +1,5 @@ +--- +"@moonshot-ai/kimi-code": patch +--- + +Speed up file completion in remote environments without ripgrep. diff --git a/packages/acp-server/src/acp-fs/acpFsService.ts b/packages/acp-server/src/acp-fs/acpFsService.ts index 5dd422e72b6..5bd7aa8224d 100644 --- a/packages/acp-server/src/acp-fs/acpFsService.ts +++ b/packages/acp-server/src/acp-fs/acpFsService.ts @@ -20,6 +20,8 @@ import { HostFileSystem, type HostDirEntry, type HostFileStat, + type HostWalkOptions, + type HostWalkResult, IHostFileSystem, ISessionContext, LifecycleScope, @@ -151,6 +153,10 @@ export class AcpHostFileSystem implements IHostFileSystem { return this.inner.readdir(path); } + walk(path: string, options: HostWalkOptions): Promise { + return this.inner.walk(path, options); + } + mkdir(path: string, options?: { readonly recursive?: boolean }): Promise { return this.inner.mkdir(path, options); } diff --git a/packages/agent-core-v2/src/os/backends/node-local/hostFsService.ts b/packages/agent-core-v2/src/os/backends/node-local/hostFsService.ts index e141543c10d..214f2e68e94 100644 --- a/packages/agent-core-v2/src/os/backends/node-local/hostFsService.ts +++ b/packages/agent-core-v2/src/os/backends/node-local/hostFsService.ts @@ -14,8 +14,15 @@ import { } from 'node:fs/promises'; import { decodeTextWithErrors, readUtf8Lines, type TextDecodeErrors } from '#/_base/execEnv/decodeText'; -import { type HostDirEntry, type HostFileStat, IHostFileSystem } from '#/os/interface/hostFileSystem'; +import { + type HostDirEntry, + type HostFileStat, + type HostWalkOptions, + type HostWalkResult, + IHostFileSystem, +} from '#/os/interface/hostFileSystem'; import { toHostFsError } from '#/os/interface/hostFsErrors'; +import { walkHostDirectory } from '#/os/interface/hostFsWalk'; const READ_CHUNK_SIZE = 64 * 1024; @@ -192,6 +199,10 @@ export class HostFileSystem implements IHostFileSystem { } } + async walk(path: string, options: HostWalkOptions): Promise { + return walkHostDirectory(this, path, options); + } + async mkdir(path: string, options?: { readonly recursive?: boolean }): Promise { try { await mkdir(path, { recursive: options?.recursive ?? false }); diff --git a/packages/agent-core-v2/src/os/backends/remote/remoteFileSystem.ts b/packages/agent-core-v2/src/os/backends/remote/remoteFileSystem.ts index f74e978faf7..99f0b03aa8d 100644 --- a/packages/agent-core-v2/src/os/backends/remote/remoteFileSystem.ts +++ b/packages/agent-core-v2/src/os/backends/remote/remoteFileSystem.ts @@ -1,3 +1,5 @@ +import * as posixPath from 'node:path/posix'; + import { decodeTextWithErrors, readUtf8Lines, @@ -7,8 +9,15 @@ import { splitLinesKeepingTerminator } from '#/_base/text/line-endings'; import type { HostDirEntry, HostFileStat, + HostWalkOptions, + HostWalkResult, IHostFileSystem, } from '#/os/interface/hostFileSystem'; +import { + HOST_WALK_DEFAULT_TIMEOUT_MS, + HOST_WALK_MAX_IGNORE_BYTES, + walkHostDirectory, +} from '#/os/interface/hostFsWalk'; import { HostFsError, OsFsErrors, @@ -28,16 +37,23 @@ import { FS_READ_FILE_METHOD, FS_REMOVE_METHOD, FS_RENAME_METHOD, + FS_WALK_METHOD, FS_WRITE_FILE_CHUNK_BYTES, FS_WRITE_FILE_METHOD, type FsGetMetadataResult, type FsReadDirectoryResult, type FsReadFileResult, + type FsWalkResult, type FsWriteMode, } from '#/remote/protocol/methods'; import { rpcDomainError } from './rpcDomainError'; const HOST_FS_CODES: ReadonlySet = new Set(Object.values(OsFsErrors.codes)); +const REMOTE_WALK_PATH = { + join: (...paths: readonly string[]) => posixPath.join(...paths), + relative: (from: string, to: string) => posixPath.relative(from, to), + separator: posixPath.sep, +}; export function toRemoteFsError(error: unknown): Error { if (!(error instanceof RpcError)) { @@ -62,6 +78,7 @@ function isUtf8Encoding(encoding: BufferEncoding): boolean { export class RemoteFileSystem implements IHostFileSystem { declare readonly _serviceBrand: undefined; + private walkSupported: boolean | undefined; constructor(private readonly connection: ExecChannel) {} @@ -232,6 +249,52 @@ export class RemoteFileSystem implements IHostFileSystem { })); } + async walk(path: string, options: HostWalkOptions): Promise { + const remainingMs = options.deadlineMs === undefined ? undefined : options.deadlineMs - Date.now(); + if (remainingMs !== undefined && remainingMs <= 0) { + return { entries: [], errors: [], truncated: true, responseBytes: 0 }; + } + const timeoutMs = Math.max(1, Math.min( + remainingMs ?? HOST_WALK_DEFAULT_TIMEOUT_MS, + HOST_WALK_DEFAULT_TIMEOUT_MS, + )); + if (this.walkSupported === false) return this.walkFallback(path, options); + if (options.ignore !== undefined && Buffer.byteLength(JSON.stringify(options.ignore)) > HOST_WALK_MAX_IGNORE_BYTES) { + return this.walkFallback(path, options); + } + try { + const result = await this.connection.call(FS_WALK_METHOD, { + path, + maxDepth: options.maxDepth, + maxDirectories: options.maxDirectories, + maxEntries: options.maxEntries, + maxResponseBytes: options.maxResponseBytes, + ignore: options.ignore, + timeoutMs, + }) as FsWalkResult; + this.walkSupported = true; + return { + entries: result.entries, + errors: result.errors.map(({ path: errorPath, error }) => ({ + path: errorPath, + error: toRemoteFsError(RpcError.fromErrorBody(error)), + })), + truncated: result.truncated, + responseBytes: result.responseBytes, + }; + } catch (error) { + if (error instanceof RpcError && error.code === RpcErrorCode.MethodNotFound) { + this.walkSupported = false; + return this.walkFallback(path, options); + } + throw toRemoteFsError(error); + } + } + + private async walkFallback(path: string, options: HostWalkOptions): Promise { + return walkHostDirectory(this, path, options, { path: REMOTE_WALK_PATH }); + } + async mkdir(path: string, options?: { readonly recursive?: boolean }): Promise { await this.call(FS_CREATE_DIRECTORY_METHOD, { path, diff --git a/packages/agent-core-v2/src/os/interface/hostFileSystem.ts b/packages/agent-core-v2/src/os/interface/hostFileSystem.ts index 0199b65263d..e33c60273e2 100644 --- a/packages/agent-core-v2/src/os/interface/hostFileSystem.ts +++ b/packages/agent-core-v2/src/os/interface/hostFileSystem.ts @@ -18,6 +18,27 @@ export interface HostDirEntry { readonly isSymbolicLink?: boolean; } +export interface HostWalkOptions { + readonly maxDepth: number; + readonly maxDirectories: number; + readonly maxEntries: number; + readonly maxResponseBytes: number; + readonly ignore?: readonly string[]; + readonly deadlineMs?: number; +} + +export interface HostWalkError { + readonly path: string; + readonly error: Error; +} + +export interface HostWalkResult { + readonly entries: readonly HostDirEntry[]; + readonly errors: readonly HostWalkError[]; + readonly truncated: boolean; + readonly responseBytes: number; +} + export interface IHostFileSystem { readonly _serviceBrand: undefined; @@ -37,6 +58,7 @@ export interface IHostFileSystem { stat(path: string): Promise; lstat(path: string): Promise; readdir(path: string): Promise; + walk(path: string, options: HostWalkOptions): Promise; mkdir(path: string, options?: { readonly recursive?: boolean }): Promise; remove(path: string): Promise; rename?(from: string, to: string): Promise; diff --git a/packages/agent-core-v2/src/os/interface/hostFsWalk.ts b/packages/agent-core-v2/src/os/interface/hostFsWalk.ts new file mode 100644 index 00000000000..44e4a7619e1 --- /dev/null +++ b/packages/agent-core-v2/src/os/interface/hostFsWalk.ts @@ -0,0 +1,123 @@ +import { join, relative, sep } from 'node:path'; + +import ignore from 'ignore'; + +import type { HostDirEntry, HostWalkOptions, HostWalkResult } from './hostFileSystem'; +import { HostFsError, OsFsErrors } from './hostFsErrors'; + +export const HOST_WALK_MAX_DEPTH = 64; +export const HOST_WALK_MAX_DIRECTORIES = 10_000; +export const HOST_WALK_MAX_ENTRIES = 100_000; +export const HOST_WALK_MAX_RESPONSE_BYTES = 8 * 1024 * 1024; +export const HOST_WALK_MAX_IGNORE_BYTES = 1024 * 1024; +export const HOST_WALK_DEFAULT_TIMEOUT_MS = 55_000; +export const HOST_WALK_MAX_TIMEOUT_MS = 60_000; +export const HOST_WALK_MAX_DIRECTORY_ENTRIES = 50_000; + +interface HostWalkerFs { + readdir(path: string): Promise; +} + +interface HostWalkPath { + readonly separator: string; + join(...paths: readonly string[]): string; + relative(from: string, to: string): string; +} + +interface HostWalkHooks { + readonly path?: HostWalkPath; + readonly serializeError?: (path: string, error: Error) => unknown; +} + +export async function walkHostDirectory( + fs: HostWalkerFs, + root: string, + options: HostWalkOptions, + hooks: HostWalkHooks = {}, + readDirectory: (path: string) => Promise = fs.readdir.bind(fs), +): Promise { + validateLimit('maxDepth', options.maxDepth, 0, HOST_WALK_MAX_DEPTH); + validateLimit('maxDirectories', options.maxDirectories, 1, HOST_WALK_MAX_DIRECTORIES); + validateLimit('maxEntries', options.maxEntries, 1, HOST_WALK_MAX_ENTRIES); + validateLimit('maxResponseBytes', options.maxResponseBytes, 1, HOST_WALK_MAX_RESPONSE_BYTES); + if (options.deadlineMs !== undefined && !Number.isFinite(options.deadlineMs)) { + throw new RangeError('deadlineMs must be finite'); + } + const matcher = options.ignore === undefined ? undefined : ignore().add(options.ignore); + const serializeError = hooks.serializeError ?? ((_path, error) => ({ + name: error.name, + message: error.message, + })); + const pathApi = hooks.path ?? { join, relative, separator: sep }; + const deadlineMs = options.deadlineMs ?? Date.now() + HOST_WALK_DEFAULT_TIMEOUT_MS; + const entries: HostDirEntry[] = []; + const errors: { path: string; error: Error }[] = []; + const pending = [{ path: root, depth: 0 }]; + let responseBytes = 64; + let truncated = false; + for (let index = 0; index < pending.length && !truncated; index += 1) { + if (Date.now() >= deadlineMs) { + truncated = true; + break; + } + const { path: directory, depth } = pending[index]!; + let children: readonly HostDirEntry[]; + try { + children = await readDirectory(directory); + if (children.length > HOST_WALK_MAX_DIRECTORY_ENTRIES) { + throw new HostFsError( + OsFsErrors.codes.OS_FS_DIRECTORY_TOO_LARGE, + `readdir ${directory} exceeds the ${HOST_WALK_MAX_DIRECTORY_ENTRIES}-entry limit`, + { details: { path: directory, op: 'readdir', limit: HOST_WALK_MAX_DIRECTORY_ENTRIES } }, + ); + } + } catch (error) { + const errorBytes = Buffer.byteLength(JSON.stringify({ + path: directory, + error: serializeError(directory, asError(error)), + })) + 1; + if (responseBytes + errorBytes > options.maxResponseBytes) { + truncated = true; + break; + } + responseBytes += errorBytes; + errors.push({ path: directory, error: asError(error) }); + continue; + } + for (const child of children) { + if (Date.now() >= deadlineMs) { + truncated = true; + break; + } + const path = pathApi.join(directory, child.name); + const isDir = child.isDirectory && child.isSymbolicLink !== true; + const relPath = pathApi.relative(root, path).split(pathApi.separator).join('/'); + if (matcher?.ignores(isDir ? `${relPath}/` : relPath) === true) continue; + const entryBytes = Buffer.byteLength(JSON.stringify({ ...child, name: path })) + 1; + if (entries.length >= options.maxEntries || responseBytes + entryBytes > options.maxResponseBytes) { + truncated = true; + break; + } + responseBytes += entryBytes; + entries.push({ ...child, name: path }); + if (isDir && depth < options.maxDepth) { + if (pending.length >= options.maxDirectories) { + truncated = true; + break; + } + pending.push({ path, depth: depth + 1 }); + } + } + } + return { entries, errors, truncated, responseBytes }; +} + +function validateLimit(name: string, value: number, min: number, max: number): void { + if (!Number.isInteger(value) || value < min || value > max) { + throw new RangeError(`${name} must be an integer in [${min}, ${max}]`); + } +} + +function asError(error: unknown): Error { + return error instanceof Error ? error : new Error(String(error)); +} diff --git a/packages/agent-core-v2/src/remote/README.md b/packages/agent-core-v2/src/remote/README.md index 555debb269f..c8edba47120 100644 --- a/packages/agent-core-v2/src/remote/README.md +++ b/packages/agent-core-v2/src/remote/README.md @@ -67,6 +67,12 @@ deviations per the design spec, plus one forced addition: - `fs/readDirectory` preserves the symlink marker on each entry. The client rejects a truncated listing with `os.fs.directory_too_large` rather than exposing an incomplete directory; the current limit is 50,000 entries. +- `fs/walk` performs a bounded recursive listing in one call. It accepts the + root ignore rules needed to prune ignored directories before traversal and + reports entries, recoverable directory errors, and truncation. Clients fall + back to per-directory reads when an older executor does not implement it or + when the JSON-encoded ignore rules exceed 1MiB. Walks default to a 55-second + budget so they can return a truncated result before the 60-second RPC deadline. - Added `fs/rename` (no codex counterpart). - `fs/remove` takes only `path` and always removes recursively and ignores a missing path, matching `IHostFileSystem.remove` (codex has `recursive`/`force`). @@ -89,7 +95,7 @@ deviations per the design spec, plus one forced addition: - `environment` adds `osArch`/`osVersion`/`pathClass` over codex (rg artifact selection needs arch), plus `cwd`/`tempDir`. - No ws transport, no `resumeSessionId`, no sandbox parameter family, no - `fs/walk`/`fs/copy`/`capabilityRoots/discoverV1`/`environmentConfig/read`. + `fs/copy`/`capabilityRoots/discoverV1`/`environmentConfig/read`. Process output is delivered through notifications (`process/output`, `process/exited`, `process/closed`). The executor retains process handles until diff --git a/packages/agent-core-v2/src/remote/client/remoteEnvironmentProvider.ts b/packages/agent-core-v2/src/remote/client/remoteEnvironmentProvider.ts index 10950441377..cc1131748c5 100644 --- a/packages/agent-core-v2/src/remote/client/remoteEnvironmentProvider.ts +++ b/packages/agent-core-v2/src/remote/client/remoteEnvironmentProvider.ts @@ -61,6 +61,7 @@ class DeclaredRemoteEnvironment implements Environment { stat: async (path) => this.connected().fs.stat(path), lstat: async (path) => this.connected().fs.lstat(path), readdir: async (path) => this.connected().fs.readdir(path), + walk: async (path, options) => this.connected().fs.walk(path, options), mkdir: async (path, options) => this.connected().fs.mkdir(path, options), remove: async (path) => this.connected().fs.remove(path), rename: async (from, to) => this.connected().fs.rename(from, to), diff --git a/packages/agent-core-v2/src/remote/protocol/methods.ts b/packages/agent-core-v2/src/remote/protocol/methods.ts index d7a9d121451..478cfee3285 100644 --- a/packages/agent-core-v2/src/remote/protocol/methods.ts +++ b/packages/agent-core-v2/src/remote/protocol/methods.ts @@ -7,6 +7,7 @@ export const FS_CREATE_DIRECTORY_METHOD = 'fs/createDirectory'; export const FS_GET_METADATA_METHOD = 'fs/getMetadata'; export const FS_CANONICALIZE_METHOD = 'fs/canonicalize'; export const FS_READ_DIRECTORY_METHOD = 'fs/readDirectory'; +export const FS_WALK_METHOD = 'fs/walk'; export const FS_REMOVE_METHOD = 'fs/remove'; export const FS_RENAME_METHOD = 'fs/rename'; @@ -105,8 +106,40 @@ export interface FsReadDirectoryEntry { export interface FsReadDirectoryResult { readonly entries: FsReadDirectoryEntry[]; + readonly truncated: boolean; +} + +export interface FsWalkParams { + readonly path: string; + readonly maxDepth: number; + readonly maxDirectories: number; + readonly maxEntries: number; + readonly maxResponseBytes: number; + readonly ignore?: readonly string[]; + readonly timeoutMs?: number; +} + +export interface FsWalkEntry { + readonly name: string; + readonly isFile: boolean; + readonly isDirectory: boolean; + readonly isSymbolicLink?: boolean; +} + +export interface FsWalkError { + readonly path: string; + readonly error: { + readonly code: number; + readonly message: string; + readonly data?: unknown; + }; +} +export interface FsWalkResult { + readonly entries: readonly FsWalkEntry[]; + readonly errors: readonly FsWalkError[]; readonly truncated: boolean; + readonly responseBytes: number; } export interface FsRemoveParams { diff --git a/packages/agent-core-v2/src/remote/server/fsHandler.ts b/packages/agent-core-v2/src/remote/server/fsHandler.ts index 5326bfe9682..c30f4e50a98 100644 --- a/packages/agent-core-v2/src/remote/server/fsHandler.ts +++ b/packages/agent-core-v2/src/remote/server/fsHandler.ts @@ -12,7 +12,18 @@ import { type FsGetMetadataResult, type FsReadDirectoryResult, type FsReadFileResult, + type FsWalkResult, } from '#/remote/protocol/methods'; +import { + HOST_WALK_DEFAULT_TIMEOUT_MS, + HOST_WALK_MAX_DEPTH, + HOST_WALK_MAX_DIRECTORIES, + HOST_WALK_MAX_ENTRIES, + HOST_WALK_MAX_IGNORE_BYTES, + HOST_WALK_MAX_RESPONSE_BYTES, + HOST_WALK_MAX_TIMEOUT_MS, + walkHostDirectory, +} from '#/os/interface/hostFsWalk'; type Params = Record; @@ -62,6 +73,15 @@ function optionalInteger( return value; } +function optionalStringArray(params: Params, name: string): readonly string[] | undefined { + const value = params[name]; + if (value === undefined) return undefined; + if (!Array.isArray(value) || value.some((item) => typeof item !== 'string')) { + throw new RpcError(RpcErrorCode.InvalidParams, `${name} must be an array of strings`); + } + return value; +} + function fsDomainError(error: unknown, ctx: { path: string; op: string }): RpcError { const hostError = toHostFsError(error, ctx); const code = @@ -196,6 +216,45 @@ export class FsHandler { } } + async walk(rawParams: unknown): Promise { + const params = requireParams(rawParams); + const path = requireAbsolutePath(params, 'path'); + const maxDepth = optionalInteger(params, 'maxDepth', 0, HOST_WALK_MAX_DEPTH) ?? HOST_WALK_MAX_DEPTH; + const maxDirectories = optionalInteger(params, 'maxDirectories', 1, HOST_WALK_MAX_DIRECTORIES) ?? HOST_WALK_MAX_DIRECTORIES; + const maxEntries = optionalInteger(params, 'maxEntries', 1, HOST_WALK_MAX_ENTRIES) ?? HOST_WALK_MAX_ENTRIES; + const maxResponseBytes = optionalInteger(params, 'maxResponseBytes', 1, HOST_WALK_MAX_RESPONSE_BYTES) ?? HOST_WALK_MAX_RESPONSE_BYTES; + const ignore = optionalStringArray(params, 'ignore'); + if (ignore !== undefined && Buffer.byteLength(JSON.stringify(ignore)) > HOST_WALK_MAX_IGNORE_BYTES) { + throw new RpcError( + RpcErrorCode.InvalidParams, + `ignore must not exceed ${HOST_WALK_MAX_IGNORE_BYTES} bytes when encoded as JSON`, + ); + } + const timeoutMs = optionalInteger(params, 'timeoutMs', 1, HOST_WALK_MAX_TIMEOUT_MS) ?? HOST_WALK_DEFAULT_TIMEOUT_MS; + const result = await walkHostDirectory(this.fs, path, { + maxDepth, + maxDirectories, + maxEntries, + maxResponseBytes, + ignore, + deadlineMs: Date.now() + timeoutMs, + }, { + serializeError: (errorPath, error) => fsDomainError(error, { + path: errorPath, + op: 'readdir', + }).toErrorBody(), + }); + return { + entries: result.entries, + errors: result.errors.map(({ path: errorPath, error }) => ({ + path: errorPath, + error: fsDomainError(error, { path: errorPath, op: 'readdir' }).toErrorBody(), + })), + truncated: result.truncated, + responseBytes: result.responseBytes, + }; + } + async remove(rawParams: unknown): Promise { const params = requireParams(rawParams); const path = requireAbsolutePath(params, 'path'); diff --git a/packages/agent-core-v2/src/remote/server/stdioHost.ts b/packages/agent-core-v2/src/remote/server/stdioHost.ts index 4acfe2e0aff..78515f49214 100644 --- a/packages/agent-core-v2/src/remote/server/stdioHost.ts +++ b/packages/agent-core-v2/src/remote/server/stdioHost.ts @@ -19,6 +19,7 @@ import { FS_READ_FILE_METHOD, FS_REMOVE_METHOD, FS_RENAME_METHOD, + FS_WALK_METHOD, FS_WRITE_FILE_METHOD, INITIALIZE_METHOD, MAX_IN_FLIGHT_CALLS, @@ -166,6 +167,7 @@ export class StdioHost { [FS_GET_METADATA_METHOD, (p) => fs.getMetadata(p)], [FS_CANONICALIZE_METHOD, (p) => fs.canonicalize(p)], [FS_READ_DIRECTORY_METHOD, (p) => fs.readDirectory(p)], + [FS_WALK_METHOD, (p) => fs.walk(p)], [FS_REMOVE_METHOD, (p) => fs.remove(p)], [FS_RENAME_METHOD, (p) => fs.rename(p)], [PROCESS_START_METHOD, (p) => pm.start(p)], @@ -283,7 +285,9 @@ export class StdioHost { ); return; } - const lane: Lane = message.method === FS_READ_FILE_METHOD ? 'data' : 'control'; + const lane: Lane = message.method === FS_READ_FILE_METHOD || message.method === FS_WALK_METHOD + ? 'data' + : 'control'; if (CONTROL_METHODS.has(message.method)) { void this.runHandler(message.id, handler, message.params, lane, false); return; diff --git a/packages/agent-core-v2/src/workspace/workspaceFs/fsService.ts b/packages/agent-core-v2/src/workspace/workspaceFs/fsService.ts index ef36d69da56..835dbf82b66 100644 --- a/packages/agent-core-v2/src/workspace/workspaceFs/fsService.ts +++ b/packages/agent-core-v2/src/workspace/workspaceFs/fsService.ts @@ -49,6 +49,12 @@ import { ErrorCodes, Error2, unwrapErrorCause } from '#/errors'; import { ITelemetryService } from '#/app/telemetry/telemetry'; import { IHostFileSystem, type HostDirEntry, type HostFileStat } from '#/os/interface/hostFileSystem'; import { isHostFsNotDirectory, isHostFsNotFound } from '#/os/interface/hostFsErrors'; +import { + HOST_WALK_MAX_DEPTH, + HOST_WALK_MAX_DIRECTORIES, + HOST_WALK_MAX_ENTRIES, + HOST_WALK_MAX_RESPONSE_BYTES, +} from '#/os/interface/hostFsWalk'; import { type EnvironmentPath } from '#/environment/environment'; import { POSIX_ENVIRONMENT_PATH } from '#/environment/environmentDefaults'; import { IEnvironmentService, type EnvironmentResolver } from '#/app/environment/environment'; @@ -83,8 +89,7 @@ const PROBE_MAX_DEPTH = 6; const PROBE_MAX_ENTRIES = 500; const GREP_TIMEOUT_MS = 30_000; const SUGGEST_TIMEOUT_MS = 10_000; -const SUGGEST_WALK_ABORTED = new Error('suggest walk aborted'); -const WALK_MAX_DEPTH = 64; +const WALK_DEFAULT_IGNORE = ['.git/']; interface SuggestRoot { readonly dir: string; @@ -100,7 +105,10 @@ const MACOS_NOISE = new Set(['.DS_Store', '.AppleDouble', '.LSOverride']); export class WorkspaceFsService implements IWorkspaceFsService { declare readonly _serviceBrand: undefined; - private readonly gitignoreCache = new Map(); + private readonly gitignoreCache = new Map(); private rgResolution: RgResolution | null | undefined = undefined; private realRootsCache: | { readonly key: string; readonly roots: readonly { dir: string; real: string }[] } @@ -679,9 +687,10 @@ export class WorkspaceFsService implements IWorkspaceFsService { const candidates: FsSearchHit[] = []; const queryLower = req.query.toLowerCase(); const seenPaths = new Set(); + let inventoryTruncated = false; for (const root of roots) { - const matcher = req.follow_gitignore ? await this.matcherFor(root.dir) : undefined; - await this.walk(root.dir, '', matcher, async (relPath, name, kind) => { + const ignoreRules = req.follow_gitignore ? await this.walkIgnoreFor(root.dir) : WALK_DEFAULT_IGNORE; + inventoryTruncated = await this.walk(root.dir, ignoreRules, async (relPath, name, kind) => { const pathKey = `${root.real}/${relPath}`; if (seenPaths.has(pathKey)) return; seenPaths.add(pathKey); @@ -701,7 +710,7 @@ export class WorkspaceFsService implements IWorkspaceFsService { score, match_positions: computeMatchPositions(path, queryLower), }); - }); + }) || inventoryTruncated; } candidates.sort((a, b) => { @@ -711,7 +720,7 @@ export class WorkspaceFsService implements IWorkspaceFsService { const effectiveCap = Math.min(req.limit, SEARCH_HARD_CAP); const truncated = candidates.length > effectiveCap; - return { items: candidates.slice(0, effectiveCap), truncated }; + return { items: candidates.slice(0, effectiveCap), truncated: truncated || inventoryTruncated }; } async suggest(req: FsSuggestRequest): Promise { @@ -737,6 +746,7 @@ export class WorkspaceFsService implements IWorkspaceFsService { }; const cap = req.limit; + const deadline = Date.now() + SUGGEST_TIMEOUT_MS; const controller = new AbortController(); const timer = setTimeout(() => controller.abort(), SUGGEST_TIMEOUT_MS); timer.unref?.(); @@ -753,11 +763,11 @@ export class WorkspaceFsService implements IWorkspaceFsService { } catch (error) { if (controller.signal.aborted) throw error; this.telemetry.track2('fs_suggest_node_fallback', { reason: 'rg_error' }); - return await this.suggestWithNode(query, cap, controller.signal, roots); + return await this.suggestWithNode(query, cap, controller.signal, roots, deadline); } } this.telemetry.track2('fs_suggest_node_fallback', { reason: 'rg_missing' }); - return await this.suggestWithNode(query, cap, controller.signal, roots); + return await this.suggestWithNode(query, cap, controller.signal, roots, deadline); } finally { clearTimeout(timer); } @@ -1012,29 +1022,27 @@ export class WorkspaceFsService implements IWorkspaceFsService { cap: number, signal: AbortSignal, roots: readonly SuggestRoot[], + deadline: number, ): Promise { const multi = roots.length > 1; const top = new SuggestTopHeap(cap); const seenPaths = new Set(); let matched = 0; - try { - for (const root of roots) { - const matcher = query.followGitignore ? await this.matcherFor(root.dir) : undefined; - await this.walk(root.dir, '', matcher, async (relPath, _name, kind) => { - if (signal.aborted) throw SUGGEST_WALK_ABORTED; - if (multi) { - const pathKey = `${root.real}/${relPath}`; - if (seenPaths.has(pathKey)) return; - seenPaths.add(pathKey); - } - const candidate = evaluateSuggestCandidate(relPath, kind, query); - if (candidate === null) return; - matched += 1; - top.push(this.displayCandidate(root, candidate)); - }); - } - } catch (error) { - if (error !== SUGGEST_WALK_ABORTED) throw error; + let inventoryTruncated = false; + for (const root of roots) { + const ignoreRules = query.followGitignore ? await this.walkIgnoreFor(root.dir) : WALK_DEFAULT_IGNORE; + inventoryTruncated = await this.walk(root.dir, ignoreRules, async (relPath, _name, kind) => { + if (multi) { + const pathKey = `${root.real}/${relPath}`; + if (seenPaths.has(pathKey)) return; + seenPaths.add(pathKey); + } + const candidate = evaluateSuggestCandidate(relPath, kind, query); + if (candidate === null) return; + matched += 1; + top.push(this.displayCandidate(root, candidate)); + }, { deadlineMs: deadline }) || inventoryTruncated; + if (signal.aborted || Date.now() >= deadline) break; } const items = top.drain().map((candidate) => ({ path: candidate.path, @@ -1043,7 +1051,7 @@ export class WorkspaceFsService implements IWorkspaceFsService { score: candidate.score, match_positions: [...candidate.positions], })); - return { items, truncated: matched > cap || signal.aborted }; + return { items, truncated: matched > cap || signal.aborted || inventoryTruncated }; } async grep(req: FsGrepRequest): Promise { @@ -1180,9 +1188,10 @@ export class WorkspaceFsService implements IWorkspaceFsService { const filePaths: { path: string; absolute: string }[] = []; const seenPaths = new Set(); const roots = await this.suggestRoots(); + const deadline = startedAt + GREP_TIMEOUT_MS; for (const root of roots) { - const matcher = req.follow_gitignore ? await this.matcherFor(root.dir) : undefined; - await this.walk(root.dir, '', matcher, async (rel, _name, kind) => { + const ignoreRules = req.follow_gitignore ? await this.walkIgnoreFor(root.dir) : WALK_DEFAULT_IGNORE; + truncated = await this.walk(root.dir, ignoreRules, async (rel, _name, kind) => { if (kind !== 'file') return; if (req.include_globs && !matchesAnyGlob(rel, req.include_globs)) return; if (req.exclude_globs && matchesAnyGlob(rel, req.exclude_globs)) return; @@ -1193,11 +1202,13 @@ export class WorkspaceFsService implements IWorkspaceFsService { path: this.suggestDisplayPath(root, rel), absolute: this.path.join(root.dir, rel), }); - }); + }, { deadlineMs: deadline }) || truncated; + if (signal.aborted || Date.now() >= deadline) break; } for (const file of filePaths) { - if (signal.aborted) { + const outOfTime = signal.aborted || Date.now() >= deadline; + if (outOfTime && (filesScanned > 0 || filePaths.length === 0)) { if (totalMatches === 0 && filesScanned === 0) { throw new Error2(ErrorCodes.FS_GREP_TIMEOUT, `grep timed out after ${Date.now() - startedAt}ms`); } @@ -1249,58 +1260,82 @@ export class WorkspaceFsService implements IWorkspaceFsService { private async walk( baseAbs: string, - rootRel: string, - matcher: Ignore | undefined, + ignoreRules: readonly string[] | undefined, visit: ( relPath: string, name: string, kind: 'file' | 'directory' | 'symlink', ) => Promise, - depth = 0, - ): Promise { - if (depth > WALK_MAX_DEPTH) return; - let entries: readonly HostDirEntry[]; - try { - entries = await this.hostFs.readdir(rootRel === '' ? baseAbs : this.path.join(baseAbs, rootRel)); - } catch (error) { + options: { deadlineMs?: number } = {}, + ): Promise { + const result = await this.hostFs.walk(baseAbs, { + maxDepth: HOST_WALK_MAX_DEPTH, + maxDirectories: HOST_WALK_MAX_DIRECTORIES, + maxEntries: HOST_WALK_MAX_ENTRIES, + maxResponseBytes: HOST_WALK_MAX_RESPONSE_BYTES, + ignore: ignoreRules, + deadlineMs: options.deadlineMs, + }); + for (const { path, error } of result.errors) { if (error instanceof Error2 && error.code === ErrorCodes.OS_FS_DIRECTORY_TOO_LARGE) { - throw mapFsError(error, rootRel); + throw mapFsError(error, path); } - return; } - for (const entry of entries) { - const { name } = entry; - if (name === '.git') continue; - const childRel = rootRel === '' ? name : `${rootRel}/${name}`; - const isDir = entry.isDirectory && entry.isSymbolicLink !== true; - if (matcher) { - const probe = isDir ? `${childRel}/` : childRel; - if (matcher.ignores(probe)) continue; - } - const kind: 'file' | 'directory' | 'symlink' = entry.isSymbolicLink - ? 'symlink' - : isDir - ? 'directory' - : 'file'; - await visit(childRel, name, kind); - if (isDir) { - await this.walk(baseAbs, childRel, matcher, visit, depth + 1); - } + const byParent = new Map(); + for (const entry of result.entries) { + const relPath = this.path.relative(baseAbs, entry.name).split(this.path.separator).join('/'); + if (relPath === '' || relPath === '..' || relPath.startsWith('../')) continue; + const slash = relPath.lastIndexOf('/'); + const parent = slash < 0 ? '' : relPath.slice(0, slash); + const name = slash < 0 ? relPath : relPath.slice(slash + 1); + const siblings = byParent.get(parent) ?? []; + siblings.push({ entry: { ...entry, name }, relPath }); + byParent.set(parent, siblings); } + const visitDirectory = async (parent: string): Promise => { + for (const { entry, relPath } of byParent.get(parent) ?? []) { + const { name } = entry; + if (name === '.git') continue; + const isDir = entry.isDirectory && entry.isSymbolicLink !== true; + const kind: 'file' | 'directory' | 'symlink' = entry.isSymbolicLink + ? 'symlink' + : isDir + ? 'directory' + : 'file'; + await visit(relPath, name, kind); + if (isDir) await visitDirectory(relPath); + } + }; + await visitDirectory(''); + return result.truncated; } private async matcherFor(rootDir: string): Promise { + return (await this.ignoreFor(rootDir)).matcher; + } + + private async walkIgnoreFor(rootDir: string): Promise { + return (await this.ignoreFor(rootDir)).walkRules; + } + + private async ignoreFor(rootDir: string): Promise<{ + readonly matcher: Ignore; + readonly walkRules: readonly string[]; + }> { const cached = this.gitignoreCache.get(rootDir); if (cached !== undefined) return cached; const ig = ignore(); ig.add('.git/'); + let contents = ''; try { - const contents = await this.hostFs.readText(this.path.join(rootDir, '.gitignore')); + contents = await this.hostFs.readText(this.path.join(rootDir, '.gitignore')); ig.add(contents); } catch { } - this.gitignoreCache.set(rootDir, ig); - return ig; + const rules = ['.git/', ...contents.split(/\r?\n/)]; + const result = { matcher: ig, walkRules: rules }; + this.gitignoreCache.set(rootDir, result); + return result; } private async resolveRg(): Promise { diff --git a/packages/agent-core-v2/test/agent/tools/os/rgUnavailableGuidance.test.ts b/packages/agent-core-v2/test/agent/tools/os/rgUnavailableGuidance.test.ts index 8998335b8a2..6d0811ab750 100644 --- a/packages/agent-core-v2/test/agent/tools/os/rgUnavailableGuidance.test.ts +++ b/packages/agent-core-v2/test/agent/tools/os/rgUnavailableGuidance.test.ts @@ -56,6 +56,7 @@ function createBackend(options: { stat: async () => directoryStat(), lstat: async () => directoryStat(), readdir: () => notImplemented('readdir'), + walk: () => notImplemented('walk'), mkdir: () => notImplemented('mkdir'), remove: () => notImplemented('remove'), realpath: async (path: string) => path, diff --git a/packages/agent-core-v2/test/harness/agent.ts b/packages/agent-core-v2/test/harness/agent.ts index 399a6bb3cab..f52efda2b97 100644 --- a/packages/agent-core-v2/test/harness/agent.ts +++ b/packages/agent-core-v2/test/harness/agent.ts @@ -590,6 +590,7 @@ function isFullHostFs(input: unknown): boolean { 'realpath', 'stat', 'readdir', + 'walk', 'mkdir', 'remove', ]; diff --git a/packages/agent-core-v2/test/os/backends/node-local/hostFsService.test.ts b/packages/agent-core-v2/test/os/backends/node-local/hostFsService.test.ts index 6d56f42ab85..d99991e0cf6 100644 --- a/packages/agent-core-v2/test/os/backends/node-local/hostFsService.test.ts +++ b/packages/agent-core-v2/test/os/backends/node-local/hostFsService.test.ts @@ -5,6 +5,7 @@ import { join } from 'pathe'; import { afterEach, beforeEach, describe, expect, it } from 'vitest'; import { HostFileSystem } from '#/os/backends/node-local/hostFsService'; +import { walkHostDirectory } from '#/os/interface/hostFsWalk'; let dir: string; let fs: HostFileSystem; @@ -53,6 +54,185 @@ describe('HostFileSystem streamed writes', () => { }); }); +describe('HostFileSystem walk', () => { + const options = { + maxDepth: 64, + maxDirectories: 10_000, + maxEntries: 100_000, + maxResponseBytes: 8 * 1024 * 1024, + }; + + it('lists nested entries without following directory symlinks', async () => { + await mkdir(join(dir, 'a/b'), { recursive: true }); + await writeFile(join(dir, 'a/b/file.ts'), ''); + await writeFile(join(dir, 'root.ts'), ''); + await symlink(join(dir, 'a'), join(dir, 'link')); + + const result = await fs.walk(dir, options); + + const names = result.entries.map((entry) => entry.name.slice(dir.length + 1)); + expect(names).toHaveLength(5); + expect(names).toEqual(expect.arrayContaining([ + 'a', + 'root.ts', + 'link', + 'a/b', + 'a/b/file.ts', + ])); + expect(result.entries.find((entry) => entry.name === join(dir, 'link'))?.isSymbolicLink).toBe(true); + expect(result.errors).toEqual([]); + expect(result.truncated).toBe(false); + }); + + it('prunes ignored directories before traversing them', async () => { + await mkdir(join(dir, 'node_modules/pkg'), { recursive: true }); + await mkdir(join(dir, 'src'), { recursive: true }); + await writeFile(join(dir, 'node_modules/pkg/index.ts'), ''); + await writeFile(join(dir, 'src/app.ts'), ''); + + const result = await fs.walk(dir, { ...options, ignore: ['node_modules/'] }); + + expect(result.entries.map((entry) => entry.name.slice(dir.length + 1))).toEqual(['src', 'src/app.ts']); + expect(result.truncated).toBe(false); + }); + + it('returns descendants at maxDepth but does not traverse below them', async () => { + await mkdir(join(dir, 'a/b/c'), { recursive: true }); + await writeFile(join(dir, 'a/b/c/deep.ts'), ''); + + const result = await fs.walk(dir, { ...options, maxDepth: 1 }); + + expect(result.entries.map((entry) => entry.name.slice(dir.length + 1))).toEqual(['a', 'a/b']); + expect(result.truncated).toBe(false); + }); + + it('reports missing roots as recoverable errors', async () => { + const result = await fs.walk(join(dir, 'missing'), options); + + expect(result.entries).toEqual([]); + expect(result.errors).toHaveLength(1); + expect(result.errors[0]).toMatchObject({ + path: join(dir, 'missing'), + error: { code: 'os.fs.not_found' }, + }); + }); + + it('bounds directories and entries without reporting an exact fit as truncated', async () => { + const tree = new Map([ + ['/root', [ + { name: 'a', isFile: false, isDirectory: true }, + { name: 'b', isFile: false, isDirectory: true }, + ]], + ['/root/a', [{ name: 'one.ts', isFile: true, isDirectory: false }]], + ['/root/b', [{ name: 'two.ts', isFile: true, isDirectory: false }]], + ]); + const backend = { + readdir: async (path: string) => tree.get(path) ?? [], + }; + + const exact = await walkHostDirectory(backend, '/root', { + ...options, + maxDirectories: 3, + maxEntries: 4, + }); + const boundedDirectories = await walkHostDirectory(backend, '/root', { + ...options, + maxDirectories: 2, + maxEntries: 4, + }); + const boundedEntries = await walkHostDirectory(backend, '/root', { + ...options, + maxDirectories: 3, + maxEntries: 3, + }); + + expect(exact.truncated).toBe(false); + expect(exact.entries).toHaveLength(4); + expect(boundedDirectories.truncated).toBe(true); + expect(boundedDirectories.entries.map((entry) => entry.name)).toEqual(['/root/a', '/root/b']); + expect(boundedEntries.truncated).toBe(true); + expect(boundedEntries.entries.map((entry) => entry.name)).toEqual(['/root/a', '/root/b', '/root/a/one.ts']); + }); + + it('counts entry and error JSON in the response budget', async () => { + const backend = { + readdir: async (path: string) => { + if (path === '/root') return [{ name: 'a', isFile: true, isDirectory: false }]; + throw new Error('unreadable'); + }, + }; + const read = () => walkHostDirectory(backend, '/root', { + ...options, + maxResponseBytes: 256, + }); + const initial = await read(); + const result = await walkHostDirectory(backend, '/root', { + ...options, + maxResponseBytes: initial.responseBytes - 1, + }); + + expect(initial.truncated).toBe(false); + expect(result.entries).toEqual([]); + expect(result.truncated).toBe(true); + + const withError = await walkHostDirectory(backend, '/root', options, { + serializeError: (_path, error) => ({ message: error.message }), + }, async (path) => { + if (path === '/root') return [{ name: 'a', isFile: false, isDirectory: true }]; + throw new Error('unreadable'); + }); + const beforeError = await walkHostDirectory(backend, '/root', { + ...options, + maxResponseBytes: withError.responseBytes - 1, + }, { + serializeError: (_path, error) => ({ message: error.message }), + }, async (path) => { + if (path === '/root') return [{ name: 'a', isFile: false, isDirectory: true }]; + throw new Error('unreadable'); + }); + + expect(withError.errors).toHaveLength(1); + expect(beforeError.errors).toEqual([]); + expect(beforeError.truncated).toBe(true); + }); + + it('returns partial entries at the deadline', async () => { + let release!: () => void; + const gate = new Promise((resolve) => { + release = resolve; + }); + const timedOut = walkHostDirectory({ + readdir: async () => [], + }, '/root', { + ...options, + deadlineMs: Date.now() + 5, + }, {}, async (path) => { + if (path === '/root') return [{ name: 'a', isFile: false, isDirectory: true }]; + await gate; + return [{ name: 'file.ts', isFile: true, isDirectory: false }]; + }); + await new Promise((resolve) => setTimeout(resolve, 15)); + release(); + const afterTimeout = await timedOut; + + expect(afterTimeout.entries.map((entry) => entry.name)).toEqual(['/root/a']); + expect(afterTimeout.truncated).toBe(true); + }); + + it('does not start walking at an expired deadline', async () => { + let reads = 0; + const result = await walkHostDirectory({ + readdir: async () => { + reads += 1; + return []; + }, + }, '/root', { ...options, deadlineMs: Date.now() }); + + expect(result).toMatchObject({ entries: [], truncated: true }); + expect(reads).toBe(0); + }); +}); + describe('HostFileSystem stat / lstat', () => { it('stat follows a symlink to a regular file while lstat stats the link', async () => { const target = join(dir, 'target.txt'); diff --git a/packages/agent-core-v2/test/os/backends/node-local/tools/grep.test.ts b/packages/agent-core-v2/test/os/backends/node-local/tools/grep.test.ts index 3c41eeb76ef..a547538d2be 100644 --- a/packages/agent-core-v2/test/os/backends/node-local/tools/grep.test.ts +++ b/packages/agent-core-v2/test/os/backends/node-local/tools/grep.test.ts @@ -160,6 +160,7 @@ function createTestFs(kaos: FakeKaos): IHostFileSystem { stat: (path) => kaos.stat(path), lstat: (path) => kaos.stat(path), readdir: () => notImplemented('readdir'), + walk: () => notImplemented('walk'), mkdir: () => notImplemented('mkdir'), remove: () => notImplemented('remove'), realpath: () => notImplemented('realpath'), diff --git a/packages/agent-core-v2/test/program/program.test.ts b/packages/agent-core-v2/test/program/program.test.ts index 0db0ad74b0e..1414585666b 100644 --- a/packages/agent-core-v2/test/program/program.test.ts +++ b/packages/agent-core-v2/test/program/program.test.ts @@ -353,6 +353,7 @@ function scopedFs(base: string, realBase: string, inner: IHostFileSystem, additi stat: (path) => { assert(path); return inner.stat(path); }, lstat: inner.lstat.bind(inner), readdir: (path) => { assert(path); return inner.readdir(path); }, + walk: (path, options) => { assert(path); return inner.walk(path, options); }, mkdir: (path, options) => { assert(path); return inner.mkdir(path, options); }, remove: (path) => { assert(path); return inner.remove(path); }, realpath: inner.realpath.bind(inner), diff --git a/packages/agent-core-v2/test/remote/fs.test.ts b/packages/agent-core-v2/test/remote/fs.test.ts index 1a671165b2d..b0980bc5110 100644 --- a/packages/agent-core-v2/test/remote/fs.test.ts +++ b/packages/agent-core-v2/test/remote/fs.test.ts @@ -1,18 +1,29 @@ -import { mkdtemp, rm, writeFile } from 'node:fs/promises'; +import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; -import { afterAll, beforeAll, describe, expect, it } from 'vitest'; +import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest'; import { HostFsError } from '#/os/interface/hostFsErrors'; +import { HOST_WALK_MAX_IGNORE_BYTES } from '#/os/interface/hostFsWalk'; import { RemoteExecConnection } from '#/remote/client/connection'; import { RemoteFileSystem } from '#/os/backends/remote/remoteFileSystem'; -import { FS_READ_DIRECTORY_MAX_ENTRIES, FS_READ_FILE_WHOLE_MAX_BYTES } from '#/remote/protocol/methods'; import { + FS_READ_DIRECTORY_MAX_ENTRIES, + FS_READ_DIRECTORY_METHOD, + FS_READ_FILE_WHOLE_MAX_BYTES, + FS_WALK_METHOD, +} from '#/remote/protocol/methods'; +import { RpcError, RpcErrorCode } from '#/remote/protocol/errors'; +import { + connectInProcess, connectSubprocess, createInProcessLoopback, + createScriptedServer, RawClient, + testInitializeResult, + type ScriptedFrame, type SpawnedExecutor, } from './helpers/loopback'; @@ -126,6 +137,274 @@ describe('fs group over a subprocess loopback', () => { }); describe('fs protocol semantics', () => { + it('short-circuits expired walks without sending an RPC', async () => { + const calls: string[] = []; + const fs = new RemoteFileSystem({ + call: async (method) => { + calls.push(method); + throw new Error('unexpected call'); + }, + notify: () => {}, + onNotification: () => () => {}, + onDidClose: () => {}, + }); + + const result = await fs.walk('/root', { + maxDepth: 64, + maxDirectories: 10_000, + maxEntries: 100_000, + maxResponseBytes: 8 * 1024 * 1024, + deadlineMs: Date.now(), + }); + + expect(result).toEqual({ entries: [], errors: [], truncated: true, responseBytes: 0 }); + expect(calls).toEqual([]); + }); + + it('caps executor walk budgets at the caller deadline', async () => { + let timeoutMs: number | undefined; + const fs = new RemoteFileSystem({ + call: async (_method, params) => { + timeoutMs = (params as { timeoutMs: number }).timeoutMs; + return { entries: [], errors: [], truncated: false, responseBytes: 64 }; + }, + notify: () => {}, + onNotification: () => () => {}, + onDidClose: () => {}, + }); + + const result = await fs.walk('/root', { + maxDepth: 64, + maxDirectories: 10_000, + maxEntries: 100_000, + maxResponseBytes: 8 * 1024 * 1024, + deadlineMs: Date.now() + 1_000, + }); + + expect(result.truncated).toBe(false); + expect(timeoutMs).toBeGreaterThan(0); + expect(timeoutMs).toBeLessThanOrEqual(1_000); + }); + + it('bounds old executor fallback with the caller deadline', async () => { + let clock = 1_000; + const now = vi.spyOn(Date, 'now').mockImplementation(() => clock); + const seen: ScriptedFrame[] = []; + const pipe = createScriptedServer((frame, reply) => { + seen.push(frame); + if (frame.method === 'initialize') { + reply({ id: frame.id, result: testInitializeResult() }); + return; + } + if (frame.method === FS_WALK_METHOD) { + reply({ + id: frame.id, + error: new RpcError(RpcErrorCode.MethodNotFound, 'unknown method fs/walk').toErrorBody(), + }); + return; + } + if (frame.method === FS_READ_DIRECTORY_METHOD) { + const path = (frame.params as { path: string }).path; + reply({ + id: frame.id, + result: { + entries: path === '/root' + ? [{ fileName: 'src', isDirectory: true, isFile: false, isSymlink: false }] + : [{ fileName: 'app.ts', isDirectory: false, isFile: true, isSymlink: false }], + truncated: false, + }, + }); + } + }); + const connection = await RemoteExecConnection.connect(pipe, { + clientName: 'remote-exec-test', + clientVersion: '0.0.0', + }); + try { + const fs = new RemoteFileSystem(connection); + const result = await fs.walk('/root', { + maxDepth: 64, + maxDirectories: 10_000, + maxEntries: 100_000, + maxResponseBytes: 8 * 1024 * 1024, + deadlineMs: clock, + }); + + clock += 1; + const expired = await fs.walk('/root/child', { + maxDepth: 64, + maxDirectories: 10_000, + maxEntries: 100_000, + maxResponseBytes: 8 * 1024 * 1024, + deadlineMs: clock, + }); + + expect(result.entries).toEqual([]); + expect(result.truncated).toBe(true); + expect(expired.entries).toEqual([]); + expect(seen.filter((frame) => frame.method === FS_READ_DIRECTORY_METHOD)).toEqual([]); + } finally { + connection.close(); + now.mockRestore(); + } + }); + + it('walks nested directories in one rpc and prunes ignored subtrees', async () => { + const dir = await mkdtemp(join(tmpdir(), 'remote-exec-walk-')); + const { connection, loopback } = await connectInProcess(); + const methods: string[] = []; + loopback.clientInput.on('data', (chunk: Buffer) => { + for (const line of chunk.toString('utf8').trim().split('\n')) { + if (line.length > 0) methods.push((JSON.parse(line) as { method?: string }).method ?? ''); + } + }); + try { + await mkdir(join(dir, 'src/nested'), { recursive: true }); + await mkdir(join(dir, 'node_modules/pkg'), { recursive: true }); + await writeFile(join(dir, 'src/nested/app.ts'), ''); + await writeFile(join(dir, 'node_modules/pkg/index.ts'), ''); + const fs = new RemoteFileSystem(connection); + + const result = await fs.walk(dir, { + maxDepth: 64, + maxDirectories: 10_000, + maxEntries: 100_000, + maxResponseBytes: 8 * 1024 * 1024, + ignore: ['node_modules/'], + }); + + expect(result.entries.map((entry) => entry.name.slice(dir.length + 1)).toSorted()).toEqual([ + 'src', + 'src/nested', + 'src/nested/app.ts', + ]); + expect(result.errors).toEqual([]); + expect(result.truncated).toBe(false); + expect(methods.filter((method) => method === FS_WALK_METHOD)).toHaveLength(1); + expect(methods.filter((method) => method === 'fs/readDirectory')).toHaveLength(0); + } finally { + connection.close(); + await loopback.host.done; + await rm(dir, { recursive: true, force: true }); + } + }); + + it('falls back once for old executors and caches their walk capability', async () => { + const seen: ScriptedFrame[] = []; + const pipe = createScriptedServer((frame, reply) => { + seen.push(frame); + if (frame.method === 'initialize') { + reply({ id: frame.id, result: testInitializeResult() }); + return; + } + if (frame.method === FS_WALK_METHOD) { + reply({ + id: frame.id, + error: new RpcError(RpcErrorCode.MethodNotFound, 'unknown method fs/walk').toErrorBody(), + }); + return; + } + if (frame.method === FS_READ_DIRECTORY_METHOD) { + const path = (frame.params as { path: string }).path; + reply({ + id: frame.id, + result: { + entries: path === '/root' + ? [{ fileName: 'src', isDirectory: true, isFile: false, isSymlink: false }] + : [{ fileName: 'app.ts', isDirectory: false, isFile: true, isSymlink: false }], + truncated: false, + }, + }); + } + }); + const connection = await RemoteExecConnection.connect(pipe, { + clientName: 'remote-exec-test', + clientVersion: '0.0.0', + }); + try { + const fs = new RemoteFileSystem(connection); + const options = { + maxDepth: 64, + maxDirectories: 10_000, + maxEntries: 100_000, + maxResponseBytes: 8 * 1024 * 1024, + }; + + const first = await fs.walk('/root', options); + const second = await fs.walk('/root', options); + const third = await fs.walk('/root/child', options); + + expect(first.entries.map((entry) => entry.name)).toEqual(['/root/src', '/root/src/app.ts']); + expect(second.entries).toEqual(first.entries); + expect(third.entries).toEqual([ + { name: '/root/child/app.ts', isDirectory: false, isFile: true, isSymbolicLink: false }, + ]); + expect(seen.filter((frame) => frame.method === FS_WALK_METHOD)).toHaveLength(1); + expect(seen.filter((frame) => frame.method === FS_READ_DIRECTORY_METHOD)).toHaveLength(5); + } finally { + connection.close(); + } + }); + + it('falls back locally for oversized ignore rules without pruning them', async () => { + const dir = await mkdtemp(join(tmpdir(), 'remote-exec-walk-ignore-cap-')); + const { connection, loopback } = await connectInProcess(); + const methods: string[] = []; + loopback.clientInput.on('data', (chunk: Buffer) => { + for (const line of chunk.toString('utf8').trim().split('\n')) { + if (line.length > 0) methods.push((JSON.parse(line) as { method?: string }).method ?? ''); + } + }); + try { + await mkdir(join(dir, 'src')); + await mkdir(join(dir, 'ignored')); + await writeFile(join(dir, 'src/app.ts'), ''); + await writeFile(join(dir, 'ignored/app.ts'), ''); + const fs = new RemoteFileSystem(connection); + const unusedRule = `unused-${'x'.repeat(8 * 1024)}`; + const unusedRuleCount = Math.ceil(HOST_WALK_MAX_IGNORE_BYTES / Buffer.byteLength(JSON.stringify(unusedRule))); + + const result = await fs.walk(dir, { + maxDepth: 64, + maxDirectories: 10_000, + maxEntries: 100_000, + maxResponseBytes: 8 * 1024 * 1024, + ignore: ['ignored/', ...Array.from({ length: unusedRuleCount }, () => unusedRule)], + }); + + expect(result.entries.map((entry) => entry.name.slice(dir.length + 1)).toSorted()).toEqual([ + 'src', + 'src/app.ts', + ]); + expect(result.truncated).toBe(false); + expect(methods.filter((method) => method === FS_WALK_METHOD)).toHaveLength(0); + expect(methods.filter((method) => method === FS_READ_DIRECTORY_METHOD)).toHaveLength(2); + } finally { + connection.close(); + await loopback.host.done; + await rm(dir, { recursive: true, force: true }); + } + }); + + it('rejects oversized ignore rules on the wire', async () => { + const loopback = createInProcessLoopback(); + const raw = new RawClient(loopback); + await raw.handshake(); + raw.send({ + id: 1, + method: FS_WALK_METHOD, + params: { + path: '/root', + ignore: ['x'.repeat(HOST_WALK_MAX_IGNORE_BYTES)], + }, + }); + + const response = await raw.nextResponse(1); + expect(response['error']).toMatchObject({ code: RpcErrorCode.InvalidParams }); + loopback.clientInput.end(); + await loopback.host.done; + }); + it('reports capped directory listings on the wire and rejects them through the filesystem interface', async () => { const dir = await mkdtemp(join(tmpdir(), 'remote-exec-readdir-cap-')); const smallDir = await mkdtemp(join(tmpdir(), 'remote-exec-readdir-small-')); diff --git a/packages/agent-core-v2/test/tools/fixtures/fake-exec.ts b/packages/agent-core-v2/test/tools/fixtures/fake-exec.ts index f8efa4c47d3..10ad7e060a3 100644 --- a/packages/agent-core-v2/test/tools/fixtures/fake-exec.ts +++ b/packages/agent-core-v2/test/tools/fixtures/fake-exec.ts @@ -28,6 +28,7 @@ export function createFakeHostFs(overrides: Partial = {}): IHos stat: () => notImplemented('FakeHostFs.stat'), lstat: () => notImplemented('FakeHostFs.lstat'), readdir: () => notImplemented('FakeHostFs.readdir'), + walk: () => notImplemented('FakeHostFs.walk'), mkdir: () => notImplemented('FakeHostFs.mkdir'), remove: () => notImplemented('FakeHostFs.remove'), realpath: () => notImplemented('FakeHostFs.realpath'), diff --git a/packages/agent-core-v2/test/workspace/workspaceFs/fsService.test.ts b/packages/agent-core-v2/test/workspace/workspaceFs/fsService.test.ts index 35a0132b743..846c6d117fc 100644 --- a/packages/agent-core-v2/test/workspace/workspaceFs/fsService.test.ts +++ b/packages/agent-core-v2/test/workspace/workspaceFs/fsService.test.ts @@ -1,7 +1,7 @@ import { isAbsolute, join, relative, resolve } from 'node:path'; import { Readable, Writable } from 'node:stream'; -import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { LifecycleScope } from '#/app/scopes'; import { ScopeActivation, @@ -13,6 +13,7 @@ import { IGitService } from '#/app/git/git'; import { ErrorCodes, Error2 } from '#/errors'; import { type HostDirEntry, IHostFileSystem } from '#/os/interface/hostFileSystem'; import { HostFsError } from '#/os/interface/hostFsErrors'; +import { walkHostDirectory } from '#/os/interface/hostFsWalk'; import { IHostProcessService, type IHostProcess } from '#/os/interface/hostProcess'; import { IWorkspaceFsService } from '#/workspace/workspaceFs/fs'; import { WorkspaceFsService } from '#/workspace/workspaceFs/fsService'; @@ -151,7 +152,7 @@ function fakeFs( } throw enoent(p); }; - return { + const fs: IHostFileSystem = { _serviceBrand: undefined, readText: async (p) => { const c = fileMap.get(resolveSymlinks(p)); @@ -207,6 +208,7 @@ function fakeFs( for (const s of symlinkSet) visit(s, 'symlink'); return [...children.values()]; }, + walk: (path, options) => walkHostDirectory(fs, path, options), mkdir: async (p, options) => { const recursive = options?.recursive ?? false; const exists = isDir(p) || fileMap.has(p); @@ -259,6 +261,7 @@ function fakeFs( return current; }, }; + return fs; } function fakeProcess(stdout: string, stderr: string, exitCode: number): IHostProcess { @@ -566,6 +569,26 @@ describe('WorkspaceFsService.search', () => { expect(paths.some((p) => p.startsWith('src/link/'))).toBe(false); }); + it('keeps the original deep-directory boundary', async () => { + const files: Record = {}; + let current = 'root'; + files[`${current}/match.ts`] = ''; + for (let depth = 1; depth <= 66; depth += 1) { + current = `${current}/level-${depth}`; + if (depth === 63 || depth === 64 || depth === 65) files[`${current}/match.ts`] = ''; + } + const fs = makeSession(files, emptyHandler); + const deepPath = Array.from({ length: 63 }, (_, index) => `level-${index + 1}`).join('/'); + + const result = await fs.search({ query: 'match', limit: 50, follow_gitignore: false }); + + expect(result.items.map((item) => item.path).toSorted()).toEqual([ + 'root/match.ts', + `root/${deepPath}/match.ts`, + ].toSorted()); + expect(result.items.some((item) => item.path.includes('level-64/'))).toBe(false); + }); + it('lists the workspace root top-level entries when the query is empty', async () => { const fs = makeSession( { 'src/foo.ts': '', 'src/nested/deep.ts': '', 'README.md': '', '.hidden.ts': '' }, @@ -954,6 +977,79 @@ describe('WorkspaceFsService.suggest', () => { expect(on.items.map((i) => i.path)).toContain('.secret.ts'); }); + it('passes ignore rules to the walk and reports inventory truncation', async () => { + const backend = fakeFs({ + '.gitignore': 'node_modules/\n', + 'src/app.ts': '', + 'node_modules/pkg/index.ts': '', + }); + let receivedIgnore: readonly string[] | undefined; + backend.walk = async (_path, options) => { + receivedIgnore = options.ignore; + return { entries: [], errors: [], truncated: true, responseBytes: 64 }; + }; + const fs = makeRemoteSession( + { '.gitignore': 'node_modules/\n' }, + () => ({ stdout: '', exitCode: 127 }), + [], + 'ssh-dev', + '/home/target', + backend, + ); + + const result = await fs.suggest({ + query: 'app', + limit: 50, + show_hidden: false, + follow_gitignore: true, + }); + + expect(receivedIgnore).toEqual(['.git/', 'node_modules/', '']); + expect(result).toEqual({ items: [], truncated: true }); + }); + + it('keeps matching and file enumeration order when using batched walks', async () => { + const backend = fakeFs({ + '.git/config': 'hello\n', + 'a/first.ts': 'hello\n', + 'a/nested/second.ts': 'hello\n', + 'z.ts': 'hello\n', + }); + const readdir = backend.readdir.bind(backend); + const reads: string[] = []; + backend.readdir = async (path) => { + reads.push(path); + return readdir(path); + }; + const fs = makeRemoteSession( + {}, + () => ({ stdout: '', exitCode: 127 }), + [], + 'ssh-dev', + '/home/target', + backend, + ); + + const result = await fs.grep({ + pattern: 'hello', + regex: false, + case_sensitive: true, + follow_gitignore: false, + max_files: 200, + max_matches_per_file: 50, + max_total_matches: 5000, + context_lines: 0, + }); + + expect(result.files.map((file) => file.path)).toEqual([ + 'a/nested/second.ts', + 'a/first.ts', + 'z.ts', + ]); + expect(result.files.some((file) => file.path.startsWith('.git/'))).toBe(false); + expect(reads).toEqual(['/repo', '/repo/a', '/repo/a/nested']); + }); + it('falls back to the node walk when rg fails to spawn', async () => { const events: Array<{ event: string; properties: Record }> = []; const runner: IHostProcessService = { @@ -1268,6 +1364,93 @@ describe('WorkspaceFsService.suggest', () => { expect(skipped.items.map((i) => i.path)).toEqual([]); }); + it('keeps partial candidates at the shared deadline without walking later roots', async () => { + const backend = fakeFs({ 'src/foo.ts': '', '/extra/foo-extra.ts': '' }); + let clock = 1_000; + const now = vi.spyOn(Date, 'now').mockImplementation(() => clock); + const walk = backend.walk; + const deadlines: number[] = []; + const roots: string[] = []; + backend.walk = async (path, options) => { + roots.push(path); + deadlines.push(options.deadlineMs ?? 0); + if (path === WORK_DIR) { + clock += 10_000; + return { + entries: [ + { name: '/repo/src', isFile: false, isDirectory: true }, + { name: '/repo/src/foo.ts', isFile: true, isDirectory: false }, + ], + errors: [], + truncated: true, + responseBytes: 128, + }; + } + return walk(path, options); + }; + try { + const fs = makeRemoteSession({}, () => ({ stdout: '', exitCode: 127 }), [], 'ssh-dev', '/home/target', backend); + Object.defineProperty(fs, 'suggestRoots', { + value: async () => [ + { dir: WORK_DIR, real: WORK_DIR, primary: true }, + { dir: '/extra', real: '/extra', primary: false }, + ], + }); + + const result = await fs.suggest({ + query: 'foo', + limit: 50, + follow_gitignore: false, + show_hidden: false, + }); + + expect(result.items.map((item) => item.path)).toEqual(['src/foo.ts']); + expect(result.truncated).toBe(true); + expect(roots).toEqual([WORK_DIR]); + expect(deadlines).toEqual([11_000]); + } finally { + now.mockRestore(); + } + }); + + it('does not restart the suggest budget after the rg attempt', async () => { + const backend = fakeFs({ 'src/foo.ts': '' }); + let clock = 1_000; + const now = vi.spyOn(Date, 'now').mockImplementation(() => clock); + let deadline: number | undefined; + backend.walk = async (_path, options) => { + deadline = options.deadlineMs; + return { entries: [], errors: [], truncated: false, responseBytes: 64 }; + }; + const runner: IHostProcessService = { + _serviceBrand: undefined, + spawn: async (_command, args) => { + if (args?.[0] === '--version') return fakeProcess('ripgrep 15.0.0', '', 0); + clock += 4_000; + throw new Error('spawn failed'); + }, + }; + try { + const fs = makeRemoteSession({}, emptyHandler, [], 'ssh-dev', '/home/target', backend); + Object.defineProperty(fs, 'resolver', { + value: { + acquire: () => ({ + environment: { process: runner }, + track: (resource: unknown) => resource, + dispose: () => {}, + }), + }, + }); + Object.defineProperty(fs, 'rgResolution', { value: { path: 'rg' } }); + + await fs.suggest({ query: 'foo', limit: 50, follow_gitignore: false, show_hidden: false }); + + expect(deadline).toBe(11_000); + } finally { + now.mockRestore(); + } + }); + it('lists top-level entries of every root for an empty query', async () => { const fs = makeSession( { 'src/foo.ts': '', 'README.md': '', '/extra/lib/util.ts': '', '/extra/notes.md': '' }, @@ -1549,6 +1732,53 @@ describe('WorkspaceFsService.grep', () => { expect(result.files.map((file) => file.path)).toEqual(['src/a.ts', '/extra/b.ts']); expect(result.files_scanned).toBe(2); }); + + it('returns partial grep matches when the inventory reaches the shared deadline', async () => { + const backend = fakeFs({ 'src/a.ts': 'hello primary\n', '/extra/b.ts': 'hello extra\n' }); + let clock = 1_000; + const now = vi.spyOn(Date, 'now').mockImplementation(() => clock); + const roots: string[] = []; + backend.walk = async (path) => { + roots.push(path); + clock += 30_000; + return { + entries: [ + { name: '/repo/src', isFile: false, isDirectory: true }, + { name: '/repo/src/a.ts', isFile: true, isDirectory: false }, + ], + errors: [], + truncated: true, + responseBytes: 128, + }; + }; + try { + const fs = makeRemoteSession({}, () => ({ stdout: '', exitCode: 127 }), [], 'ssh-dev', '/home/target', backend); + Object.defineProperty(fs, 'suggestRoots', { + value: async () => [ + { dir: WORK_DIR, real: WORK_DIR, primary: true }, + { dir: '/extra', real: '/extra', primary: false }, + ], + }); + + const result = await fs.grep({ + pattern: 'hello', + regex: false, + case_sensitive: true, + follow_gitignore: false, + max_files: 200, + max_matches_per_file: 50, + max_total_matches: 5000, + context_lines: 0, + }); + + expect(result.files.map((file) => file.path)).toEqual(['src/a.ts']); + expect(result.files_scanned).toBe(1); + expect(result.truncated).toBe(true); + expect(roots).toEqual([WORK_DIR]); + } finally { + now.mockRestore(); + } + }); }); describe('WorkspaceFsService.list', () => { diff --git a/packages/kap-server/test/workspaceFs.test.ts b/packages/kap-server/test/workspaceFs.test.ts index 047201befef..061bc190aed 100644 --- a/packages/kap-server/test/workspaceFs.test.ts +++ b/packages/kap-server/test/workspaceFs.test.ts @@ -463,6 +463,7 @@ function mappingHostFs(remoteRoot: string): IHostFileSystem { stat: (path) => inner.stat(map(path)), lstat: (path) => inner.lstat(map(path)), readdir: (path) => inner.readdir(map(path)), + walk: (path, options) => inner.walk(map(path), options), mkdir: (path, options) => inner.mkdir(map(path), options), remove: (path) => inner.remove(map(path)), realpath: async (path) => unmap(await inner.realpath(map(path))), From ff2e2403b4a14ca4b23d071ceb80fc2cb9b8c8b5 Mon Sep 17 00:00:00 2001 From: 7Sageer Date: Fri, 30 Oct 2026 13:48:56 +0800 Subject: [PATCH 2/8] fix: preserve grep timeout on empty inventory --- .../src/workspace/workspaceFs/fsService.ts | 10 +++---- .../workspace/workspaceFs/fsService.test.ts | 29 +++++++++++++++++++ 2 files changed, 34 insertions(+), 5 deletions(-) diff --git a/packages/agent-core-v2/src/workspace/workspaceFs/fsService.ts b/packages/agent-core-v2/src/workspace/workspaceFs/fsService.ts index 835dbf82b66..6f837a0c569 100644 --- a/packages/agent-core-v2/src/workspace/workspaceFs/fsService.ts +++ b/packages/agent-core-v2/src/workspace/workspaceFs/fsService.ts @@ -1206,12 +1206,12 @@ export class WorkspaceFsService implements IWorkspaceFsService { if (signal.aborted || Date.now() >= deadline) break; } + if ((signal.aborted || Date.now() >= deadline) && filesScanned === 0 && filePaths.length === 0) { + throw new Error2(ErrorCodes.FS_GREP_TIMEOUT, `grep timed out after ${Date.now() - startedAt}ms`); + } + for (const file of filePaths) { - const outOfTime = signal.aborted || Date.now() >= deadline; - if (outOfTime && (filesScanned > 0 || filePaths.length === 0)) { - if (totalMatches === 0 && filesScanned === 0) { - throw new Error2(ErrorCodes.FS_GREP_TIMEOUT, `grep timed out after ${Date.now() - startedAt}ms`); - } + if ((signal.aborted || Date.now() >= deadline) && filesScanned > 0) { truncated = true; break; } diff --git a/packages/agent-core-v2/test/workspace/workspaceFs/fsService.test.ts b/packages/agent-core-v2/test/workspace/workspaceFs/fsService.test.ts index 846c6d117fc..fddf16a4aa8 100644 --- a/packages/agent-core-v2/test/workspace/workspaceFs/fsService.test.ts +++ b/packages/agent-core-v2/test/workspace/workspaceFs/fsService.test.ts @@ -1779,6 +1779,35 @@ describe('WorkspaceFsService.grep', () => { now.mockRestore(); } }); + + it('times out when the inventory deadline expires without discovering files', async () => { + const backend = fakeFs({ 'src/a.ts': 'hello\n' }); + let clock = 1_000; + const now = vi.spyOn(Date, 'now').mockImplementation(() => clock); + backend.walk = async () => { + clock += 30_000; + return { entries: [], errors: [], truncated: true, responseBytes: 64 }; + }; + try { + const fs = makeRemoteSession({}, () => ({ stdout: '', exitCode: 127 }), [], 'ssh-dev', '/home/target', backend); + + await expect(fs.grep({ + pattern: 'hello', + regex: false, + case_sensitive: true, + follow_gitignore: false, + max_files: 200, + max_matches_per_file: 50, + max_total_matches: 5000, + context_lines: 0, + })).rejects.toMatchObject({ + code: ErrorCodes.FS_GREP_TIMEOUT, + message: 'grep timed out after 30000ms', + }); + } finally { + now.mockRestore(); + } + }); }); describe('WorkspaceFsService.list', () => { From 61f5df60a762fd2b0711d25b4cfdeb72e78d3c17 Mon Sep 17 00:00:00 2001 From: 7Sageer Date: Fri, 30 Oct 2026 14:08:09 +0800 Subject: [PATCH 3/8] test: consolidate filesystem walk regressions --- .../backends/node-local/hostFsService.test.ts | 79 ++++----- packages/agent-core-v2/test/remote/fs.test.ts | 157 ++++-------------- .../workspace/workspaceFs/fsService.test.ts | 52 ++---- 3 files changed, 77 insertions(+), 211 deletions(-) diff --git a/packages/agent-core-v2/test/os/backends/node-local/hostFsService.test.ts b/packages/agent-core-v2/test/os/backends/node-local/hostFsService.test.ts index d99991e0cf6..d438f27e93a 100644 --- a/packages/agent-core-v2/test/os/backends/node-local/hostFsService.test.ts +++ b/packages/agent-core-v2/test/os/backends/node-local/hostFsService.test.ts @@ -2,7 +2,7 @@ import { chmod, mkdir, mkdtemp, rm, symlink, writeFile } from 'node:fs/promises' import { tmpdir } from 'node:os'; import { join } from 'pathe'; -import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { HostFileSystem } from '#/os/backends/node-local/hostFsService'; import { walkHostDirectory } from '#/os/interface/hostFsWalk'; @@ -156,67 +156,46 @@ describe('HostFileSystem walk', () => { it('counts entry and error JSON in the response budget', async () => { const backend = { - readdir: async (path: string) => { - if (path === '/root') return [{ name: 'a', isFile: true, isDirectory: false }]; - throw new Error('unreadable'); - }, + readdir: async () => [{ name: 'a', isFile: true, isDirectory: false }], }; - const read = () => walkHostDirectory(backend, '/root', { - ...options, - maxResponseBytes: 256, - }); - const initial = await read(); - const result = await walkHostDirectory(backend, '/root', { - ...options, - maxResponseBytes: initial.responseBytes - 1, - }); + const initial = await walkHostDirectory(backend, '/root', options); + const tooSmall = await walkHostDirectory(backend, '/root', { ...options, maxResponseBytes: initial.responseBytes - 1 }); - expect(initial.truncated).toBe(false); - expect(result.entries).toEqual([]); - expect(result.truncated).toBe(true); + expect(initial).toMatchObject({ truncated: false }); + expect(tooSmall).toMatchObject({ entries: [], truncated: true }); - const withError = await walkHostDirectory(backend, '/root', options, { - serializeError: (_path, error) => ({ message: error.message }), - }, async (path) => { - if (path === '/root') return [{ name: 'a', isFile: false, isDirectory: true }]; - throw new Error('unreadable'); - }); - const beforeError = await walkHostDirectory(backend, '/root', { + const readWithError = async (maxResponseBytes: number) => walkHostDirectory(backend, '/root', { ...options, - maxResponseBytes: withError.responseBytes - 1, - }, { - serializeError: (_path, error) => ({ message: error.message }), - }, async (path) => { + maxResponseBytes, + }, {}, async (path) => { if (path === '/root') return [{ name: 'a', isFile: false, isDirectory: true }]; throw new Error('unreadable'); }); + const withError = await readWithError(options.maxResponseBytes); + const beforeError = await readWithError(withError.responseBytes - 1); expect(withError.errors).toHaveLength(1); - expect(beforeError.errors).toEqual([]); - expect(beforeError.truncated).toBe(true); + expect(beforeError).toMatchObject({ errors: [], truncated: true }); }); it('returns partial entries at the deadline', async () => { - let release!: () => void; - const gate = new Promise((resolve) => { - release = resolve; - }); - const timedOut = walkHostDirectory({ - readdir: async () => [], - }, '/root', { - ...options, - deadlineMs: Date.now() + 5, - }, {}, async (path) => { - if (path === '/root') return [{ name: 'a', isFile: false, isDirectory: true }]; - await gate; - return [{ name: 'file.ts', isFile: true, isDirectory: false }]; - }); - await new Promise((resolve) => setTimeout(resolve, 15)); - release(); - const afterTimeout = await timedOut; - - expect(afterTimeout.entries.map((entry) => entry.name)).toEqual(['/root/a']); - expect(afterTimeout.truncated).toBe(true); + let clock = 1_000; + const now = vi.spyOn(Date, 'now').mockImplementation(() => clock); + try { + const result = await walkHostDirectory({ readdir: async () => [] }, '/root', { + ...options, + deadlineMs: clock + 5, + }, {}, async (path) => { + if (path === '/root') return [{ name: 'a', isFile: false, isDirectory: true }]; + clock += 5; + return [{ name: 'file.ts', isFile: true, isDirectory: false }]; + }); + + expect(result.entries.map((entry) => entry.name)).toEqual(['/root/a']); + expect(result.truncated).toBe(true); + } finally { + now.mockRestore(); + } }); it('does not start walking at an expired deadline', async () => { diff --git a/packages/agent-core-v2/test/remote/fs.test.ts b/packages/agent-core-v2/test/remote/fs.test.ts index b0980bc5110..761097126c5 100644 --- a/packages/agent-core-v2/test/remote/fs.test.ts +++ b/packages/agent-core-v2/test/remote/fs.test.ts @@ -137,35 +137,18 @@ describe('fs group over a subprocess loopback', () => { }); describe('fs protocol semantics', () => { - it('short-circuits expired walks without sending an RPC', async () => { - const calls: string[] = []; + const walkOptions = { + maxDepth: 64, + maxDirectories: 10_000, + maxEntries: 100_000, + maxResponseBytes: 8 * 1024 * 1024, + }; + + it('honors the caller deadline before and during a remote walk', async () => { + const calls: Array<{ method: string; timeoutMs?: number }> = []; const fs = new RemoteFileSystem({ - call: async (method) => { - calls.push(method); - throw new Error('unexpected call'); - }, - notify: () => {}, - onNotification: () => () => {}, - onDidClose: () => {}, - }); - - const result = await fs.walk('/root', { - maxDepth: 64, - maxDirectories: 10_000, - maxEntries: 100_000, - maxResponseBytes: 8 * 1024 * 1024, - deadlineMs: Date.now(), - }); - - expect(result).toEqual({ entries: [], errors: [], truncated: true, responseBytes: 0 }); - expect(calls).toEqual([]); - }); - - it('caps executor walk budgets at the caller deadline', async () => { - let timeoutMs: number | undefined; - const fs = new RemoteFileSystem({ - call: async (_method, params) => { - timeoutMs = (params as { timeoutMs: number }).timeoutMs; + call: async (method, params) => { + calls.push({ method, timeoutMs: (params as { timeoutMs: number }).timeoutMs }); return { entries: [], errors: [], truncated: false, responseBytes: 64 }; }, notify: () => {}, @@ -173,80 +156,15 @@ describe('fs protocol semantics', () => { onDidClose: () => {}, }); - const result = await fs.walk('/root', { - maxDepth: 64, - maxDirectories: 10_000, - maxEntries: 100_000, - maxResponseBytes: 8 * 1024 * 1024, - deadlineMs: Date.now() + 1_000, - }); - - expect(result.truncated).toBe(false); - expect(timeoutMs).toBeGreaterThan(0); - expect(timeoutMs).toBeLessThanOrEqual(1_000); - }); - - it('bounds old executor fallback with the caller deadline', async () => { - let clock = 1_000; - const now = vi.spyOn(Date, 'now').mockImplementation(() => clock); - const seen: ScriptedFrame[] = []; - const pipe = createScriptedServer((frame, reply) => { - seen.push(frame); - if (frame.method === 'initialize') { - reply({ id: frame.id, result: testInitializeResult() }); - return; - } - if (frame.method === FS_WALK_METHOD) { - reply({ - id: frame.id, - error: new RpcError(RpcErrorCode.MethodNotFound, 'unknown method fs/walk').toErrorBody(), - }); - return; - } - if (frame.method === FS_READ_DIRECTORY_METHOD) { - const path = (frame.params as { path: string }).path; - reply({ - id: frame.id, - result: { - entries: path === '/root' - ? [{ fileName: 'src', isDirectory: true, isFile: false, isSymlink: false }] - : [{ fileName: 'app.ts', isDirectory: false, isFile: true, isSymlink: false }], - truncated: false, - }, - }); - } - }); - const connection = await RemoteExecConnection.connect(pipe, { - clientName: 'remote-exec-test', - clientVersion: '0.0.0', - }); - try { - const fs = new RemoteFileSystem(connection); - const result = await fs.walk('/root', { - maxDepth: 64, - maxDirectories: 10_000, - maxEntries: 100_000, - maxResponseBytes: 8 * 1024 * 1024, - deadlineMs: clock, - }); - - clock += 1; - const expired = await fs.walk('/root/child', { - maxDepth: 64, - maxDirectories: 10_000, - maxEntries: 100_000, - maxResponseBytes: 8 * 1024 * 1024, - deadlineMs: clock, - }); + const active = await fs.walk('/root', { ...walkOptions, deadlineMs: Date.now() + 1_000 }); + const expired = await fs.walk('/root', { ...walkOptions, deadlineMs: Date.now() }); - expect(result.entries).toEqual([]); - expect(result.truncated).toBe(true); - expect(expired.entries).toEqual([]); - expect(seen.filter((frame) => frame.method === FS_READ_DIRECTORY_METHOD)).toEqual([]); - } finally { - connection.close(); - now.mockRestore(); - } + expect(active.truncated).toBe(false); + expect(calls).toHaveLength(1); + expect(calls[0]?.method).toBe(FS_WALK_METHOD); + expect(calls[0]?.timeoutMs).toBeGreaterThan(0); + expect(calls[0]?.timeoutMs).toBeLessThanOrEqual(1_000); + expect(expired).toEqual({ entries: [], errors: [], truncated: true, responseBytes: 0 }); }); it('walks nested directories in one rpc and prunes ignored subtrees', async () => { @@ -264,12 +182,8 @@ describe('fs protocol semantics', () => { await writeFile(join(dir, 'src/nested/app.ts'), ''); await writeFile(join(dir, 'node_modules/pkg/index.ts'), ''); const fs = new RemoteFileSystem(connection); - const result = await fs.walk(dir, { - maxDepth: 64, - maxDirectories: 10_000, - maxEntries: 100_000, - maxResponseBytes: 8 * 1024 * 1024, + ...walkOptions, ignore: ['node_modules/'], }); @@ -281,7 +195,7 @@ describe('fs protocol semantics', () => { expect(result.errors).toEqual([]); expect(result.truncated).toBe(false); expect(methods.filter((method) => method === FS_WALK_METHOD)).toHaveLength(1); - expect(methods.filter((method) => method === 'fs/readDirectory')).toHaveLength(0); + expect(methods.filter((method) => method === FS_READ_DIRECTORY_METHOD)).toHaveLength(0); } finally { connection.close(); await loopback.host.done; @@ -289,7 +203,9 @@ describe('fs protocol semantics', () => { } }); - it('falls back once for old executors and caches their walk capability', async () => { + it('caches old executor fallback and preserves its deadline', async () => { + let clock = 1_000; + const now = vi.spyOn(Date, 'now').mockImplementation(() => clock); const seen: ScriptedFrame[] = []; const pipe = createScriptedServer((frame, reply) => { seen.push(frame); @@ -323,26 +239,16 @@ describe('fs protocol semantics', () => { }); try { const fs = new RemoteFileSystem(connection); - const options = { - maxDepth: 64, - maxDirectories: 10_000, - maxEntries: 100_000, - maxResponseBytes: 8 * 1024 * 1024, - }; - - const first = await fs.walk('/root', options); - const second = await fs.walk('/root', options); - const third = await fs.walk('/root/child', options); + const first = await fs.walk('/root', walkOptions); + const expired = await fs.walk('/root', { ...walkOptions, deadlineMs: clock }); expect(first.entries.map((entry) => entry.name)).toEqual(['/root/src', '/root/src/app.ts']); - expect(second.entries).toEqual(first.entries); - expect(third.entries).toEqual([ - { name: '/root/child/app.ts', isDirectory: false, isFile: true, isSymbolicLink: false }, - ]); + expect(expired).toMatchObject({ entries: [], truncated: true }); expect(seen.filter((frame) => frame.method === FS_WALK_METHOD)).toHaveLength(1); - expect(seen.filter((frame) => frame.method === FS_READ_DIRECTORY_METHOD)).toHaveLength(5); + expect(seen.filter((frame) => frame.method === FS_READ_DIRECTORY_METHOD)).toHaveLength(2); } finally { connection.close(); + now.mockRestore(); } }); @@ -365,10 +271,7 @@ describe('fs protocol semantics', () => { const unusedRuleCount = Math.ceil(HOST_WALK_MAX_IGNORE_BYTES / Buffer.byteLength(JSON.stringify(unusedRule))); const result = await fs.walk(dir, { - maxDepth: 64, - maxDirectories: 10_000, - maxEntries: 100_000, - maxResponseBytes: 8 * 1024 * 1024, + ...walkOptions, ignore: ['ignored/', ...Array.from({ length: unusedRuleCount }, () => unusedRule)], }); diff --git a/packages/agent-core-v2/test/workspace/workspaceFs/fsService.test.ts b/packages/agent-core-v2/test/workspace/workspaceFs/fsService.test.ts index fddf16a4aa8..76b4761478c 100644 --- a/packages/agent-core-v2/test/workspace/workspaceFs/fsService.test.ts +++ b/packages/agent-core-v2/test/workspace/workspaceFs/fsService.test.ts @@ -1368,25 +1368,21 @@ describe('WorkspaceFsService.suggest', () => { const backend = fakeFs({ 'src/foo.ts': '', '/extra/foo-extra.ts': '' }); let clock = 1_000; const now = vi.spyOn(Date, 'now').mockImplementation(() => clock); - const walk = backend.walk; - const deadlines: number[] = []; + let deadline: number | undefined; const roots: string[] = []; backend.walk = async (path, options) => { roots.push(path); - deadlines.push(options.deadlineMs ?? 0); - if (path === WORK_DIR) { - clock += 10_000; - return { - entries: [ - { name: '/repo/src', isFile: false, isDirectory: true }, - { name: '/repo/src/foo.ts', isFile: true, isDirectory: false }, - ], - errors: [], - truncated: true, - responseBytes: 128, - }; - } - return walk(path, options); + deadline = options.deadlineMs; + clock += 10_000; + return { + entries: [ + { name: '/repo/src', isFile: false, isDirectory: true }, + { name: '/repo/src/foo.ts', isFile: true, isDirectory: false }, + ], + errors: [], + truncated: true, + responseBytes: 128, + }; }; try { const fs = makeRemoteSession({}, () => ({ stdout: '', exitCode: 127 }), [], 'ssh-dev', '/home/target', backend); @@ -1407,7 +1403,7 @@ describe('WorkspaceFsService.suggest', () => { expect(result.items.map((item) => item.path)).toEqual(['src/foo.ts']); expect(result.truncated).toBe(true); expect(roots).toEqual([WORK_DIR]); - expect(deadlines).toEqual([11_000]); + expect(deadline).toBe(11_000); } finally { now.mockRestore(); } @@ -1733,7 +1729,7 @@ describe('WorkspaceFsService.grep', () => { expect(result.files_scanned).toBe(2); }); - it('returns partial grep matches when the inventory reaches the shared deadline', async () => { + it('distinguishes partial matches from an empty timed-out inventory', async () => { const backend = fakeFs({ 'src/a.ts': 'hello primary\n', '/extra/b.ts': 'hello extra\n' }); let clock = 1_000; const now = vi.spyOn(Date, 'now').mockImplementation(() => clock); @@ -1775,22 +1771,10 @@ describe('WorkspaceFsService.grep', () => { expect(result.files_scanned).toBe(1); expect(result.truncated).toBe(true); expect(roots).toEqual([WORK_DIR]); - } finally { - now.mockRestore(); - } - }); - - it('times out when the inventory deadline expires without discovering files', async () => { - const backend = fakeFs({ 'src/a.ts': 'hello\n' }); - let clock = 1_000; - const now = vi.spyOn(Date, 'now').mockImplementation(() => clock); - backend.walk = async () => { - clock += 30_000; - return { entries: [], errors: [], truncated: true, responseBytes: 64 }; - }; - try { - const fs = makeRemoteSession({}, () => ({ stdout: '', exitCode: 127 }), [], 'ssh-dev', '/home/target', backend); - + backend.walk = async () => { + clock += 30_000; + return { entries: [], errors: [], truncated: true, responseBytes: 64 }; + }; await expect(fs.grep({ pattern: 'hello', regex: false, From 4023039cc2a84a5e46c0c92b2e93e6bfed2d5288 Mon Sep 17 00:00:00 2001 From: 7Sageer Date: Fri, 30 Oct 2026 14:23:15 +0800 Subject: [PATCH 4/8] perf: sort grep matches with ripgrep --- .changeset/batch-remote-filesystem-walks.md | 2 +- .../src/agent/tools/os/grep/grepTool.ts | 77 +------- .../os/backends/node-local/tools/grep.test.ts | 176 +++++------------- 3 files changed, 47 insertions(+), 208 deletions(-) diff --git a/.changeset/batch-remote-filesystem-walks.md b/.changeset/batch-remote-filesystem-walks.md index bdf6664659a..1c944b9f003 100644 --- a/.changeset/batch-remote-filesystem-walks.md +++ b/.changeset/batch-remote-filesystem-walks.md @@ -2,4 +2,4 @@ "@moonshot-ai/kimi-code": patch --- -Speed up file completion in remote environments without ripgrep. +Speed up remote file search and completion without per-file metadata requests. diff --git a/packages/agent-core-v2/src/agent/tools/os/grep/grepTool.ts b/packages/agent-core-v2/src/agent/tools/os/grep/grepTool.ts index 503ddff6c29..d02dae75765 100644 --- a/packages/agent-core-v2/src/agent/tools/os/grep/grepTool.ts +++ b/packages/agent-core-v2/src/agent/tools/os/grep/grepTool.ts @@ -44,7 +44,6 @@ import { type GrepInput, GrepInputSchema, IGrepTool } from './grep'; const RG_MAX_COLUMNS = 500; const DEFAULT_HEAD_LIMIT = 250; -const MTIME_STAT_CONCURRENCY = 32; const VCS_DIRECTORIES_TO_EXCLUDE = ['.git', '.svn', '.hg', '.bzr', '.jj', '.sl'] as const; const SENSITIVE_KEY_BASENAMES = ['id_rsa', 'id_ed25519', 'id_ecdsa'] as const; @@ -222,18 +221,7 @@ export class GrepTool implements IGrepTool { const filteredSensitive = new Set(); const keptLines = filterSensitiveLines(rawLines, mode, filteredSensitive, pathClass); - let orderedLines: ParsedGrepLine[]; - try { - orderedLines = - mode === 'files_with_matches' && !timedOut - ? await this.sortFilesWithMatchesByMtime(fs, keptLines, signal) - : keptLines; - } catch (error) { - if (error instanceof GrepAbortedError) { - return { isError: true, output: 'Grep aborted' }; - } - throw error; - } + const orderedLines = keptLines; const offset = args.offset ?? 0; const headLimit = args.head_limit ?? DEFAULT_HEAD_LIMIT; @@ -329,32 +317,6 @@ export class GrepTool implements IGrepTool { }; } - private async sortFilesWithMatchesByMtime( - fs: IHostFileSystem, - lines: readonly ParsedGrepLine[], - signal: AbortSignal, - ): Promise { - const entries = await mapWithConcurrency( - lines, - MTIME_STAT_CONCURRENCY, - signal, - async (line, index) => { - const path = - line.kind === 'record' ? line.filePath : line.kind === 'legacy' ? line.text : undefined; - let mtime = 0; - if (path !== undefined) { - try { - const mtimeMs = (await fs.stat(path)).mtimeMs ?? 0; - mtime = Math.trunc(mtimeMs / 1000); - } catch { - } - } - return { line, mtime, index }; - }, - ); - entries.sort((a, b) => b.mtime - a.mtime || a.index - b.index); - return entries.map((entry) => entry.line); - } } registerAgentToolService(IGrepTool, GrepTool, { @@ -387,41 +349,6 @@ type ParsedGrepLine = readonly text: string; }; -class GrepAbortedError extends Error { - constructor() { - super('Grep aborted'); - this.name = 'GrepAbortedError'; - } -} - -async function mapWithConcurrency( - items: readonly T[], - concurrency: number, - signal: AbortSignal, - mapper: (item: T, index: number) => Promise, -): Promise { - if (signal.aborted) throw new GrepAbortedError(); - if (items.length === 0) return []; - - const results: U[] = []; - results.length = items.length; - let nextIndex = 0; - const workerCount = Math.min(Math.max(1, concurrency), items.length); - await Promise.all( - Array.from({ length: workerCount }, async () => { - while (true) { - if (signal.aborted) return; - const index = nextIndex; - nextIndex += 1; - if (index >= items.length) return; - results[index] = await mapper(items[index] as T, index); - } - }), - ); - if (signal.aborted) throw new GrepAbortedError(); - return results; -} - function buildRgArgs( rgPath: string, args: GrepInput, @@ -440,7 +367,7 @@ function buildRgArgs( cmd.push('--glob', `!${dir}`); } - if (mode === 'files_with_matches') cmd.push('-l'); + if (mode === 'files_with_matches') cmd.push('-l', '--sortr=modified'); else if (mode === 'count_matches') { cmd.push('--count-matches', '--with-filename'); } diff --git a/packages/agent-core-v2/test/os/backends/node-local/tools/grep.test.ts b/packages/agent-core-v2/test/os/backends/node-local/tools/grep.test.ts index a547538d2be..22b54b8ebf2 100644 --- a/packages/agent-core-v2/test/os/backends/node-local/tools/grep.test.ts +++ b/packages/agent-core-v2/test/os/backends/node-local/tools/grep.test.ts @@ -70,6 +70,7 @@ class TestContributionAssembly extends Service { } const workspace: WorkspaceConfig = { workspaceDir: '/workspace', additionalDirs: ['/extra'] }; const MAX_COLUMNS_RG_ARGS = ['--max-columns', '500'] as const; +const MTIME_RG_ARGS = ['-l', '--sortr=modified'] as const; const COMMON_RG_ARGS = [ '--null', '--glob', @@ -223,15 +224,6 @@ function processWithOutput(stdout: string, stderr = '', exitCode = 0): IHostProc }; } -function statResult(mtime: number): HostFileStat { - return { - isFile: true, - isDirectory: false, - size: 0, - mtimeMs: mtime * 1000, - }; -} - function processThatExitsOnKill(stdout: string, stderr = '', exitCode = 143): IHostProcess { let currentExitCode: number | null = null; let resolveWait: (code: number) => void; @@ -469,7 +461,7 @@ describe('GrepTool', () => { expect(exec).toHaveBeenCalledWith( '/mock/rg', ...DEFAULT_RG_ARGS, - '-l', + ...MTIME_RG_ARGS, ...SENSITIVE_RG_ARGS, '--', 'hit', @@ -487,7 +479,7 @@ describe('GrepTool', () => { expect(exec).toHaveBeenCalledWith( '/mock/rg', ...DEFAULT_RG_ARGS, - '-l', + ...MTIME_RG_ARGS, ...SENSITIVE_RG_ARGS, '--', 'hit', @@ -525,17 +517,14 @@ describe('GrepTool', () => { expect(result.output).toBe('No non-sensitive matches found'); }); - it('sorts files_with_matches by mtime before pagination after sensitive filtering', async () => { - const stdout = ['/workspace/src/old.ts', '/workspace/.env', '/workspace/src/new.ts', ''].join( + it('uses ripgrep mtime order without stat calls before pagination after sensitive filtering', async () => { + const stdout = ['/workspace/src/new.ts', '/workspace/.env', '/workspace/src/old.ts', ''].join( '\n', ); - const stat = vi.fn(async (path: string) => { - if (path === '/workspace/src/new.ts') return statResult(10); - if (path === '/workspace/src/old.ts') return statResult(1); - throw new Error(`unexpected stat: ${path}`); - }); + const exec = vi.fn().mockResolvedValue(processWithOutput(stdout)); + const stat = vi.fn(); const tool = new GrepTool( - createFakeKaos({ exec: vi.fn().mockResolvedValue(processWithOutput(stdout)), stat }), + createFakeKaos({ exec, stat }), { workspaceDir: '/workspace', additionalDirs: [] }, ); @@ -548,122 +537,45 @@ describe('GrepTool', () => { 'Results truncated to 1 lines (total: 2). Use offset=1 to see more.', ].join('\n'), ); - expect(stat).toHaveBeenCalledTimes(2); - expect(stat).toHaveBeenCalledWith('/workspace/src/old.ts'); - expect(stat).toHaveBeenCalledWith('/workspace/src/new.ts'); - }); - - it('keeps v1 tie order for files modified within the same second', async () => { - const stdout = ['/workspace/src/a.ts', '/workspace/src/b.ts', '/workspace/src/c.ts', ''].join( - '\n', - ); - const stat = vi.fn(async (path: string) => { - if (path === '/workspace/src/a.ts') { - return { ...statResult(1), mtimeMs: 1000 }; - } - if (path === '/workspace/src/b.ts') { - return { ...statResult(1), mtimeMs: 1500 }; - } - if (path === '/workspace/src/c.ts') { - return { ...statResult(2), mtimeMs: 2000 }; - } - throw new Error(`unexpected stat: ${path}`); - }); - const tool = new GrepTool( - createFakeKaos({ exec: vi.fn().mockResolvedValue(processWithOutput(stdout)), stat }), - { workspaceDir: '/workspace', additionalDirs: [] }, - ); - - const result = await executeTool(tool, context({ pattern: 'hit', head_limit: 0 })); - - expect(toolContentString(result)).toBe(['src/c.ts', 'src/a.ts', 'src/b.ts'].join('\n')); - }); - - it('limits concurrent mtime stats while sorting files_with_matches', async () => { - const filePaths = Array.from( - { length: 40 }, - (_, index) => `/workspace/src/file-${String(index).padStart(2, '0')}.ts`, - ); - let activeStats = 0; - let maxActiveStats = 0; - const stat = vi.fn(async (path: string) => { - activeStats += 1; - maxActiveStats = Math.max(maxActiveStats, activeStats); - await new Promise((resolve) => { - setTimeout(resolve, 0); - }); - activeStats -= 1; - const mtime = Number(path.match(/file-(\d+)\.ts$/)?.[1] ?? 0); - return statResult(mtime); - }); - const tool = new GrepTool( - createFakeKaos({ - exec: vi.fn().mockResolvedValue(processWithOutput(`${filePaths.join('\n')}\n`)), - stat, - }), - { workspaceDir: '/workspace', additionalDirs: [] }, + expect(exec).toHaveBeenCalledWith( + '/mock/rg', + ...DEFAULT_RG_ARGS, + ...MTIME_RG_ARGS, + ...SENSITIVE_RG_ARGS, + '--', + 'hit', + '/workspace', ); - - const result = await executeTool(tool, context({ pattern: 'hit', head_limit: 0 })); - const lines = toolContentString(result).split('\n'); - - expect(stat).toHaveBeenCalledTimes(filePaths.length); - expect(maxActiveStats).toBeLessThanOrEqual(32); - expect(lines.at(0)).toBe('src/file-39.ts'); - expect(lines.at(-1)).toBe('src/file-00.ts'); + expect(stat).not.toHaveBeenCalled(); }); - it('stops scheduling mtime stats when aborted during files_with_matches sorting', async () => { - const filePaths = Array.from( - { length: 40 }, - (_, index) => `/workspace/src/file-${String(index).padStart(2, '0')}.ts`, - ); - const abortController = new AbortController(); - const stat = vi.fn(async () => { - abortController.abort(); - await new Promise((resolve) => { - setTimeout(resolve, 0); + it('lets ripgrep order matching files newest-first', async () => { + const { mkdtemp, mkdir, rm, utimes, writeFile } = await import('node:fs/promises'); + const { tmpdir } = await import('node:os'); + const { join } = await import('node:path'); + const { execFile } = await import('node:child_process'); + const { promisify } = await import('node:util'); + const dir = await mkdtemp(join(tmpdir(), 'grep-rg-sort-')); + try { + await mkdir(join(dir, 'src')); + for (const [name, mtime] of [['old.ts', 1_000], ['new.ts', 2_000]] as const) { + const file = join(dir, 'src', name); + await writeFile(file, 'hit\n'); + await utimes(file, mtime, mtime); + } + const exec = vi.fn(async (command: string, ...args: string[]) => { + const { stdout, stderr } = await promisify(execFile)(command, args); + return processWithOutput(stdout, stderr); }); - return statResult(1); - }); - const tool = new GrepTool( - createFakeKaos({ - exec: vi.fn().mockResolvedValue(processWithOutput(`${filePaths.join('\n')}\n`)), - stat, - }), - { workspaceDir: '/workspace', additionalDirs: [] }, - ); - - const result = await executeTool(tool, - context({ pattern: 'hit', head_limit: 0 }, abortController.signal), - ); - - expect(result).toMatchObject({ isError: true, output: 'Grep aborted' }); - expect(stat.mock.calls.length).toBeLessThan(filePaths.length); - }); - - it('keeps files_with_matches entries when mtime stat fails', async () => { - const stdout = [ - '/workspace/src/old.ts', - '/workspace/src/missing.ts', - '/workspace/src/new.ts', - '', - ].join('\n'); - const stat = vi.fn(async (path: string) => { - if (path === '/workspace/src/new.ts') return statResult(10); - if (path === '/workspace/src/old.ts') return statResult(1); - throw new Error('stat failed'); - }); - const tool = new GrepTool( - createFakeKaos({ exec: vi.fn().mockResolvedValue(processWithOutput(stdout)), stat }), - { workspaceDir: '/workspace', additionalDirs: [] }, - ); + vi.mocked(ensureRgPath).mockResolvedValueOnce({ path: 'rg', source: 'system-path' }); + const tool = new GrepTool(createFakeKaos({ exec }), { workspaceDir: dir, additionalDirs: [] }); - const result = await executeTool(tool, context({ pattern: 'hit', head_limit: 0 })); + const result = await executeTool(tool, context({ pattern: 'hit' })); - expect(toolContentString(result)).toBe( - ['src/new.ts', 'src/old.ts', 'src/missing.ts'].join('\n'), - ); + expect(toolContentString(result)).toBe('src/new.ts\nsrc/old.ts'); + } finally { + await rm(dir, { recursive: true, force: true }); + } }); it('uses count-matches and ignores context flags outside content output mode', async () => { @@ -706,7 +618,7 @@ describe('GrepTool', () => { '-j', '1', ...DEFAULT_RG_ARGS, - '-l', + ...MTIME_RG_ARGS, ...SENSITIVE_RG_ARGS, '--', 'hit', @@ -731,7 +643,7 @@ describe('GrepTool', () => { expect(exec).toHaveBeenCalledWith( '/mock/rg', ...DEFAULT_RG_ARGS, - '-l', + ...MTIME_RG_ARGS, '-i', '--type', 'ts', @@ -822,7 +734,7 @@ describe('GrepTool', () => { expect(exec).toHaveBeenCalledWith( '/mock/rg', ...DEFAULT_RG_ARGS, - '-l', + ...MTIME_RG_ARGS, '--glob', '**/.env', ...SENSITIVE_RG_ARGS, From 0f3a2c4f1e5d205c21dfb59aa9fe5b1ffd72aaf9 Mon Sep 17 00:00:00 2001 From: 7Sageer Date: Fri, 30 Oct 2026 14:55:16 +0800 Subject: [PATCH 5/8] ci: install ripgrep for filesystem tests --- .github/workflows/ci.yml | 1 + .../os/backends/node-local/tools/grep.test.ts | 85 +++++++++++++------ 2 files changed, 60 insertions(+), 26 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 19ecf46363a..fbe93c7ae3f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -62,6 +62,7 @@ jobs: cache: pnpm - run: pnpm install --frozen-lockfile + - run: sudo apt-get update && sudo apt-get install -y ripgrep - run: pnpm run test --shard=${{ matrix.shard }}/5 # pi-tui's suite runs on node:test (not vitest), so the root `pnpm run test` diff --git a/packages/agent-core-v2/test/os/backends/node-local/tools/grep.test.ts b/packages/agent-core-v2/test/os/backends/node-local/tools/grep.test.ts index 22b54b8ebf2..5a7f99b690c 100644 --- a/packages/agent-core-v2/test/os/backends/node-local/tools/grep.test.ts +++ b/packages/agent-core-v2/test/os/backends/node-local/tools/grep.test.ts @@ -1,6 +1,6 @@ import { Readable, type Writable } from 'node:stream'; -import { afterEach, describe, expect, it, vi } from 'vitest'; +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest'; import { DisposableStore, toDisposable } from '#/_base/di/lifecycle'; import { Service } from '#/_base/di/service'; @@ -46,7 +46,8 @@ import { import { GrepTool as ProductionGrepTool } from '#/agent/tools/os/grep/grepTool'; import { IAgentEnvironmentService } from '#/agent/environmentBinding/agentEnvironment'; import { FakeEnvironment } from '#/environment/fakeEnvironment'; -import { ensureRgPath } from '#/os/backends/node-local/tools/rgLocator'; +import { ensureRgPath, type RgProbe } from '#/os/backends/node-local/tools/rgLocator'; +import { HostProcessService } from '#/os/backends/node-local/hostProcessService'; import { stubWorkspaceContext } from '../../../../session/workspaceContext/stub-workspace-context'; import { stubAgentEnvironment } from '../../../../environment/stubs'; import { recordingTelemetry, type TelemetryRecord } from '../../../../app/telemetry/stubs'; @@ -71,6 +72,19 @@ class TestContributionAssembly extends Service { const workspace: WorkspaceConfig = { workspaceDir: '/workspace', additionalDirs: ['/extra'] }; const MAX_COLUMNS_RG_ARGS = ['--max-columns', '500'] as const; const MTIME_RG_ARGS = ['-l', '--sortr=modified'] as const; +const realRgProbe: RgProbe = { + exec: async (args) => { + const [command, ...rest] = args; + if (command === undefined) return { exitCode: -1 }; + const proc = await new HostProcessService().spawn(command, rest); + proc.stdin.end(); + proc.stdout.resume(); + proc.stderr.resume(); + const exitCode = await proc.wait(); + await proc.dispose(); + return { exitCode }; + }, +}; const COMMON_RG_ARGS = [ '--null', '--glob', @@ -549,33 +563,52 @@ describe('GrepTool', () => { expect(stat).not.toHaveBeenCalled(); }); - it('lets ripgrep order matching files newest-first', async () => { - const { mkdtemp, mkdir, rm, utimes, writeFile } = await import('node:fs/promises'); - const { tmpdir } = await import('node:os'); - const { join } = await import('node:path'); - const { execFile } = await import('node:child_process'); - const { promisify } = await import('node:util'); - const dir = await mkdtemp(join(tmpdir(), 'grep-rg-sort-')); - try { - await mkdir(join(dir, 'src')); - for (const [name, mtime] of [['old.ts', 1_000], ['new.ts', 2_000]] as const) { - const file = join(dir, 'src', name); - await writeFile(file, 'hit\n'); - await utimes(file, mtime, mtime); + describe('with real ripgrep', () => { + let runRealRg = false; + + beforeAll(async () => { + try { + const actual = await vi.importActual( + '#/os/backends/node-local/tools/rgLocator', + ); + const resolution = await actual.ensureRgPath(realRgProbe, { allowCachedFallback: false }); + vi.mocked(ensureRgPath).mockResolvedValue(resolution); + runRealRg = true; + } catch { } - const exec = vi.fn(async (command: string, ...args: string[]) => { - const { stdout, stderr } = await promisify(execFile)(command, args); - return processWithOutput(stdout, stderr); - }); - vi.mocked(ensureRgPath).mockResolvedValueOnce({ path: 'rg', source: 'system-path' }); - const tool = new GrepTool(createFakeKaos({ exec }), { workspaceDir: dir, additionalDirs: [] }); + }); + + beforeEach((testCtx) => { + if (!runRealRg) testCtx.skip(); + }); - const result = await executeTool(tool, context({ pattern: 'hit' })); + afterAll(() => { + vi.mocked(ensureRgPath).mockResolvedValue({ path: '/mock/rg', source: 'system-path' }); + }); - expect(toolContentString(result)).toBe('src/new.ts\nsrc/old.ts'); - } finally { - await rm(dir, { recursive: true, force: true }); - } + it('lets ripgrep order matching files newest-first', async () => { + const { mkdtemp, mkdir, rm, utimes, writeFile } = await import('node:fs/promises'); + const { tmpdir } = await import('node:os'); + const { join } = await import('node:path'); + const dir = await mkdtemp(join(tmpdir(), 'grep-rg-sort-')); + try { + await mkdir(join(dir, 'src')); + for (const [name, mtime] of [['old.ts', 1_000], ['new.ts', 2_000]] as const) { + const file = join(dir, 'src', name); + await writeFile(file, 'hit\n'); + await utimes(file, mtime, mtime); + } + const processService = new HostProcessService(); + const exec = vi.fn((command: string, ...args: string[]) => processService.spawn(command, args)); + const tool = new GrepTool(createFakeKaos({ exec }), { workspaceDir: dir, additionalDirs: [] }); + + const result = await executeTool(tool, context({ pattern: 'hit' })); + + expect(toolContentString(result)).toBe('src/new.ts\nsrc/old.ts'); + } finally { + await rm(dir, { recursive: true, force: true }); + } + }); }); it('uses count-matches and ignores context flags outside content output mode', async () => { From 8facc09ed3da01a9e100f9469ba7e7bf3c3d9d58 Mon Sep 17 00:00:00 2001 From: 7Sageer <125936732+7Sageer@users.noreply.github.com> Date: Fri, 30 Oct 2026 15:19:24 +0800 Subject: [PATCH 6/8] refactor: remove old executor walk fallback --- .../os/backends/remote/remoteFileSystem.ts | 7 --- packages/agent-core-v2/src/remote/README.md | 6 +- packages/agent-core-v2/test/remote/fs.test.ts | 56 +------------------ 3 files changed, 5 insertions(+), 64 deletions(-) diff --git a/packages/agent-core-v2/src/os/backends/remote/remoteFileSystem.ts b/packages/agent-core-v2/src/os/backends/remote/remoteFileSystem.ts index 99f0b03aa8d..a9a997793e6 100644 --- a/packages/agent-core-v2/src/os/backends/remote/remoteFileSystem.ts +++ b/packages/agent-core-v2/src/os/backends/remote/remoteFileSystem.ts @@ -78,7 +78,6 @@ function isUtf8Encoding(encoding: BufferEncoding): boolean { export class RemoteFileSystem implements IHostFileSystem { declare readonly _serviceBrand: undefined; - private walkSupported: boolean | undefined; constructor(private readonly connection: ExecChannel) {} @@ -258,7 +257,6 @@ export class RemoteFileSystem implements IHostFileSystem { remainingMs ?? HOST_WALK_DEFAULT_TIMEOUT_MS, HOST_WALK_DEFAULT_TIMEOUT_MS, )); - if (this.walkSupported === false) return this.walkFallback(path, options); if (options.ignore !== undefined && Buffer.byteLength(JSON.stringify(options.ignore)) > HOST_WALK_MAX_IGNORE_BYTES) { return this.walkFallback(path, options); } @@ -272,7 +270,6 @@ export class RemoteFileSystem implements IHostFileSystem { ignore: options.ignore, timeoutMs, }) as FsWalkResult; - this.walkSupported = true; return { entries: result.entries, errors: result.errors.map(({ path: errorPath, error }) => ({ @@ -283,10 +280,6 @@ export class RemoteFileSystem implements IHostFileSystem { responseBytes: result.responseBytes, }; } catch (error) { - if (error instanceof RpcError && error.code === RpcErrorCode.MethodNotFound) { - this.walkSupported = false; - return this.walkFallback(path, options); - } throw toRemoteFsError(error); } } diff --git a/packages/agent-core-v2/src/remote/README.md b/packages/agent-core-v2/src/remote/README.md index c8edba47120..5b83109b0bc 100644 --- a/packages/agent-core-v2/src/remote/README.md +++ b/packages/agent-core-v2/src/remote/README.md @@ -70,9 +70,9 @@ deviations per the design spec, plus one forced addition: - `fs/walk` performs a bounded recursive listing in one call. It accepts the root ignore rules needed to prune ignored directories before traversal and reports entries, recoverable directory errors, and truncation. Clients fall - back to per-directory reads when an older executor does not implement it or - when the JSON-encoded ignore rules exceed 1MiB. Walks default to a 55-second - budget so they can return a truncated result before the 60-second RPC deadline. + back to per-directory reads when the JSON-encoded ignore rules exceed 1MiB. + Walks default to a 55-second budget so they can return a truncated result + before the 60-second RPC deadline. - Added `fs/rename` (no codex counterpart). - `fs/remove` takes only `path` and always removes recursively and ignores a missing path, matching `IHostFileSystem.remove` (codex has `recursive`/`force`). diff --git a/packages/agent-core-v2/test/remote/fs.test.ts b/packages/agent-core-v2/test/remote/fs.test.ts index 761097126c5..b2a067f1255 100644 --- a/packages/agent-core-v2/test/remote/fs.test.ts +++ b/packages/agent-core-v2/test/remote/fs.test.ts @@ -2,7 +2,7 @@ import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; -import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest'; +import { afterAll, beforeAll, describe, expect, it } from 'vitest'; import { HostFsError } from '#/os/interface/hostFsErrors'; import { HOST_WALK_MAX_IGNORE_BYTES } from '#/os/interface/hostFsWalk'; @@ -15,15 +15,12 @@ import { FS_READ_FILE_WHOLE_MAX_BYTES, FS_WALK_METHOD, } from '#/remote/protocol/methods'; -import { RpcError, RpcErrorCode } from '#/remote/protocol/errors'; +import { RpcErrorCode } from '#/remote/protocol/errors'; import { connectInProcess, connectSubprocess, createInProcessLoopback, - createScriptedServer, RawClient, - testInitializeResult, - type ScriptedFrame, type SpawnedExecutor, } from './helpers/loopback'; @@ -203,55 +200,6 @@ describe('fs protocol semantics', () => { } }); - it('caches old executor fallback and preserves its deadline', async () => { - let clock = 1_000; - const now = vi.spyOn(Date, 'now').mockImplementation(() => clock); - const seen: ScriptedFrame[] = []; - const pipe = createScriptedServer((frame, reply) => { - seen.push(frame); - if (frame.method === 'initialize') { - reply({ id: frame.id, result: testInitializeResult() }); - return; - } - if (frame.method === FS_WALK_METHOD) { - reply({ - id: frame.id, - error: new RpcError(RpcErrorCode.MethodNotFound, 'unknown method fs/walk').toErrorBody(), - }); - return; - } - if (frame.method === FS_READ_DIRECTORY_METHOD) { - const path = (frame.params as { path: string }).path; - reply({ - id: frame.id, - result: { - entries: path === '/root' - ? [{ fileName: 'src', isDirectory: true, isFile: false, isSymlink: false }] - : [{ fileName: 'app.ts', isDirectory: false, isFile: true, isSymlink: false }], - truncated: false, - }, - }); - } - }); - const connection = await RemoteExecConnection.connect(pipe, { - clientName: 'remote-exec-test', - clientVersion: '0.0.0', - }); - try { - const fs = new RemoteFileSystem(connection); - const first = await fs.walk('/root', walkOptions); - const expired = await fs.walk('/root', { ...walkOptions, deadlineMs: clock }); - - expect(first.entries.map((entry) => entry.name)).toEqual(['/root/src', '/root/src/app.ts']); - expect(expired).toMatchObject({ entries: [], truncated: true }); - expect(seen.filter((frame) => frame.method === FS_WALK_METHOD)).toHaveLength(1); - expect(seen.filter((frame) => frame.method === FS_READ_DIRECTORY_METHOD)).toHaveLength(2); - } finally { - connection.close(); - now.mockRestore(); - } - }); - it('falls back locally for oversized ignore rules without pruning them', async () => { const dir = await mkdtemp(join(tmpdir(), 'remote-exec-walk-ignore-cap-')); const { connection, loopback } = await connectInProcess(); From 73c06abc60a6bd285d3777b12ee07c53f5b745dc Mon Sep 17 00:00:00 2001 From: 7Sageer Date: Fri, 30 Oct 2026 17:44:04 +0800 Subject: [PATCH 7/8] perf: reduce remote file operation round trips --- .changeset/batch-remote-filesystem-walks.md | 2 +- AGENTS.md | 2 +- .../scripts/check-import-boundaries.mjs | 14 +- .../agent-core-v2/src/_base/fs/pathFilters.ts | 85 +++ .../agent-core-v2/src/_base/media/fileType.ts | 368 +++++++++++++ .../src/_base/media/mimeTypes.ts | 44 ++ .../src/_base/text/outputAccumulator.ts | 94 ++++ .../src/_base/text/outputLimits.ts | 9 + .../src/_base/text/resultStatus.ts | 12 + .../agentsMdReminderService.ts | 49 +- .../agent/contextMemory/toolResultRender.ts | 17 +- .../src/agent/media/file-type.ts | 369 +------------ .../agent-core-v2/src/agent/media/mediaRef.ts | 48 +- .../src/agent/profile/context.ts | 59 +- .../src/agent/tools/fileReadSource.ts | 12 +- .../src/agent/tools/os/glob/globTool.ts | 218 +------- .../src/agent/tools/os/grep/grepTool.ts | 508 +----------------- .../src/agent/tools/os/read/read.ts | 2 +- .../src/agent/tools/os/read/readTool.ts | 505 +---------------- .../src/agent/tools/os/write/writeTool.ts | 54 +- .../agent-core-v2/src/app/edit/editService.ts | 42 +- .../agent-core-v2/src/app/edit/fileEdit.ts | 13 +- .../src/app/edit/fileEditService.ts | 35 +- .../agent-core-v2/src/app/edit/textModel.ts | 44 +- .../agent-core-v2/src/app/git/workTree.ts | 55 +- .../fileHistory/fileHistoryService.ts | 123 ++--- .../src/os/backends/node-local/tools/runRg.ts | 192 +------ .../os/backends/remote/remoteFileSystem.ts | 212 +++++++- .../src/os/backends/remote/remoteProcess.ts | 30 ++ .../src/os/interface/fileOperations.ts | 115 ++++ .../src/os/interface/hostFileSystem.ts | 3 + .../src/os/interface/hostProcess.ts | 4 + .../src/os/interface/searchOperations.ts | 33 ++ .../src/os/operations/editService.ts | 41 ++ .../src/os/operations/editText.ts | 32 ++ .../src/os/operations/gitWorkTree.ts | 50 ++ .../src/os/operations/instructions.ts | 107 ++++ .../src/os/operations/matchFiles.ts | 72 +++ .../src/os/operations/readTextPage.ts | 481 +++++++++++++++++ .../agent-core-v2/src/os/operations/runRg.ts | 202 +++++++ .../src/os/operations/searchGlob.ts | 208 +++++++ .../src/os/operations/searchGrep.ts | 502 +++++++++++++++++ .../src/os/operations/snapshotFiles.ts | 39 ++ .../src/os/operations/textModel.ts | 43 ++ .../src/os/operations/writeText.ts | 53 ++ packages/agent-core-v2/src/remote/README.md | 55 +- .../src/remote/client/connection.ts | 3 +- .../client/remoteEnvironmentProvider.ts | 10 + .../src/remote/protocol/methods.ts | 23 + .../remote/server/fileOperationsHandler.ts | 135 +++++ .../src/remote/server/fsHandler.ts | 59 +- .../src/remote/server/searchHandler.ts | 76 +++ .../src/remote/server/stdioHost.ts | 38 +- .../src/tool/output-accumulator.ts | 98 +--- .../agent-core-v2/src/tool/path-access.ts | 87 +-- .../agent-core-v2/src/tool/toolContract.ts | 11 +- .../src/workspace/workspaceFs/fsService.ts | 59 +- .../workspaceFs/internal/fsSearch.ts | 12 +- .../features/fileHistory/fileHistory.test.ts | 19 +- .../test/lint/import-boundaries.test.ts | 13 + packages/agent-core-v2/test/remote/fs.test.ts | 249 ++++++++- .../agent-core-v2/test/remote/process.test.ts | 98 +++- .../test/remote/protocol.test.ts | 11 + .../workspace/workspaceFs/fsService.test.ts | 15 + 64 files changed, 3770 insertions(+), 2503 deletions(-) create mode 100644 packages/agent-core-v2/src/_base/fs/pathFilters.ts create mode 100644 packages/agent-core-v2/src/_base/media/fileType.ts create mode 100644 packages/agent-core-v2/src/_base/media/mimeTypes.ts create mode 100644 packages/agent-core-v2/src/_base/text/outputAccumulator.ts create mode 100644 packages/agent-core-v2/src/_base/text/outputLimits.ts create mode 100644 packages/agent-core-v2/src/_base/text/resultStatus.ts create mode 100644 packages/agent-core-v2/src/os/interface/fileOperations.ts create mode 100644 packages/agent-core-v2/src/os/interface/searchOperations.ts create mode 100644 packages/agent-core-v2/src/os/operations/editService.ts create mode 100644 packages/agent-core-v2/src/os/operations/editText.ts create mode 100644 packages/agent-core-v2/src/os/operations/gitWorkTree.ts create mode 100644 packages/agent-core-v2/src/os/operations/instructions.ts create mode 100644 packages/agent-core-v2/src/os/operations/matchFiles.ts create mode 100644 packages/agent-core-v2/src/os/operations/readTextPage.ts create mode 100644 packages/agent-core-v2/src/os/operations/runRg.ts create mode 100644 packages/agent-core-v2/src/os/operations/searchGlob.ts create mode 100644 packages/agent-core-v2/src/os/operations/searchGrep.ts create mode 100644 packages/agent-core-v2/src/os/operations/snapshotFiles.ts create mode 100644 packages/agent-core-v2/src/os/operations/textModel.ts create mode 100644 packages/agent-core-v2/src/os/operations/writeText.ts create mode 100644 packages/agent-core-v2/src/remote/server/fileOperationsHandler.ts create mode 100644 packages/agent-core-v2/src/remote/server/searchHandler.ts diff --git a/.changeset/batch-remote-filesystem-walks.md b/.changeset/batch-remote-filesystem-walks.md index 1c944b9f003..69bdd20b7e1 100644 --- a/.changeset/batch-remote-filesystem-walks.md +++ b/.changeset/batch-remote-filesystem-walks.md @@ -2,4 +2,4 @@ "@moonshot-ai/kimi-code": patch --- -Speed up remote file search and completion without per-file metadata requests. +Reduce latency for file operations and searches in remote environments. diff --git a/AGENTS.md b/AGENTS.md index 463158d0ad4..b80b90dd160 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -26,7 +26,7 @@ This is a TypeScript monorepo built for agent-assisted development. Keep the roo - `packages/session-import`: standalone Node.js conversion of external sessions into a single user message that carries the conversation, full effective-history attachments, and image references. It has no engine or UI dependency; callers provide the destination session attachment directory and context budget. - `packages/kap-server`: the Kimi Code server, backed by `@moonshot-ai/agent-core-v2`; exposes sessions over REST + WebSocket (`/api/v1` + `/api/v1/ws`), plus the `/api/v1/debug/*` reflection RPC surface (`--debug-endpoints`, loopback bind + bearer auth). - `packages/remote-control`: the Kimi Remote Control tunnel client — registers this machine with the relay and forwards HTTP/WebSocket traffic to the local server, with a machine-wide single-instance lock; consumed by kap-server (the `/api/v1/remote-control` toggle) and by the CLI (`kimi web --remote-control`). -- `packages/agent-core-v2/src/remote`: the remote execution stack inside the engine — self-contained NDJSON line-framed RPC (codex exec-server dialect) between the local `ExecBridge` (ssh/docker/`command` launchers) and the light `kimi exec-server` executor, plus the provider that registers declared targets as `Environment`s (the fs/process RPC stubs are an OS backend under `src/os/backends/remote`, sibling of `node-local`); exported as `@moonshot-ai/agent-core-v2/remote` and `/remote/server`. The executor stays light — `src/remote/server` may import only node builtins, relative modules, `#/remote/protocol`, `#/os/interface`, `#/_base/execEnv` and the DI-free node-local fs/process backends it adapts (enforced by the package's `lint:imports`); the CLI reaches it only through the SDK's light `@moonshot-ai/kimi-code-sdk/exec-server` subpath, loaded dynamically in `src/cli/exec-server.ts`. +- `packages/agent-core-v2/src/remote`: the remote execution stack inside the engine — self-contained NDJSON line-framed RPC (codex exec-server dialect) between the local `ExecBridge` (ssh/docker/`command` launchers) and the light `kimi exec-server` executor, plus the provider that registers declared targets as `Environment`s (the fs/process RPC stubs are an OS backend under `src/os/backends/remote`, sibling of `node-local`); exported as `@moonshot-ai/agent-core-v2/remote` and `/remote/server`. The executor stays light — `src/remote/server` may import only node builtins, relative modules, `#/remote/protocol`, `#/os/interface`, the DI-free shared `#/os/operations`, `#/_base/execEnv` and the DI-free node-local fs/process backends it adapts (enforced by the package's `lint:imports`); the CLI reaches it only through the SDK's light `@moonshot-ai/kimi-code-sdk/exec-server` subpath, loaded dynamically in `src/cli/exec-server.ts`. - `packages/klient`: the client SDK — a contract-driven facade over agent-core-v2 (`global.*` / `session(id).*` / `agent(id).*`, zod-validated); transport via subpath entry (`@moonshot-ai/klient/ipc|memory`, both return the same `Klient`); also hosts the e2e suites. See `packages/klient/AGENTS.md`. - `packages/tree-sitter-bash`: a pure-TypeScript bash parser (no runtime deps, no wasm); `parse(source, { timeoutMs, maxNodes })` runs under a deterministic budget and returns a discriminated `ParseResult` — callers must treat aborted/hasError trees as "cannot analyze" and degrade. Parser only, no safety judgments; see the package README's "Known differences" section. - `packages/minidb`: the embedded JSON document store (`MiniDb`) behind kap-server's search index — snapshot + WAL persistence with an exclusive write lock, a larger-than-RAM full-text layer, and persistent index generations. See `packages/minidb/AGENTS.md`. diff --git a/packages/agent-core-v2/scripts/check-import-boundaries.mjs b/packages/agent-core-v2/scripts/check-import-boundaries.mjs index 8bff7391338..9c7439dc842 100644 --- a/packages/agent-core-v2/scripts/check-import-boundaries.mjs +++ b/packages/agent-core-v2/scripts/check-import-boundaries.mjs @@ -12,11 +12,13 @@ const LOOP_MACHINE_ADAPTER_ROOT = join(SRC_ROOT, 'agent/loop/machine'); const SESSION_LIFECYCLE_ADAPTER_ROOT = join(SRC_ROOT, 'session/agentLifecycle'); const REMOTE_SERVER_ROOT = join(SRC_ROOT, 'remote/server'); const REMOTE_SERVER_ALLOWED_RE = - /^#\/(?:(?:remote\/protocol|os\/interface|_base\/execEnv)(?:\/|$)|os\/backends\/node-local\/(?:hostFsService|hostProcessService)$)/; + /^#\/(?:(?:remote\/protocol|os\/interface|os\/operations|_base\/execEnv)(?:\/|$)|os\/backends\/node-local\/(?:hostFsService|hostProcessService)$)/; const REMOTE_SERVER_MODULE_RE = /\/remote\/server(?:\/|$)/; const REMOTE_ASKPASS_FILE = join(SRC_ROOT, 'remote/askpass.ts'); +const OS_OPERATIONS_ROOT = join(SRC_ROOT, 'os/operations'); +const OS_OPERATIONS_ALLOWED_RE = /^#\/(?:os\/(?:interface|operations)|_base\/(?:execEnv|errors|text|media|fs))(?:\/|$)/; const OS_REMOTE_BACKEND_ROOT = join(SRC_ROOT, 'os/backends/remote'); -const OS_REMOTE_BACKEND_ALLOWED_RE = /^#\/(?:remote\/protocol|os\/interface|_base)(?:\/|$)/; +const OS_REMOTE_BACKEND_ALLOWED_RE = /^#\/(?:remote\/protocol|os\/interface|os\/operations|_base)(?:\/|$)/; const ENVIRONMENT_ROOT = join(SRC_ROOT, 'environment'); const REMOTE_ROOT = join(SRC_ROOT, 'remote'); const REMOTE_DECLARATION_ROOT = join(SRC_ROOT, 'remote/declaration'); @@ -39,7 +41,7 @@ function remoteServerViolation(absFile, specifier) { if (!isInside(REMOTE_SERVER_ROOT, absFile)) return undefined; if (specifier.startsWith('.') || specifier.startsWith('node:')) return undefined; if (REMOTE_SERVER_ALLOWED_RE.test(specifier)) return undefined; - return `the remote executor stays light: src/remote/server may import only node builtins, relative modules, #/remote/protocol, #/os/interface, #/_base/execEnv and the node-local hostFsService / hostProcessService implementations ('${specifier}')`; + return `the remote executor stays light: src/remote/server may import only node builtins, relative modules, #/remote/protocol, #/os/interface, #/os/operations, #/_base/execEnv and the node-local hostFsService / hostProcessService implementations ('${specifier}')`; } function remoteAskpassViolation(absFile, specifier) { @@ -51,7 +53,7 @@ function osRemoteBackendViolation(absFile, specifier) { if (!isInside(OS_REMOTE_BACKEND_ROOT, absFile)) return undefined; if (specifier.startsWith('.') || specifier.startsWith('node:')) return undefined; if (OS_REMOTE_BACKEND_ALLOWED_RE.test(specifier)) return undefined; - return `the remote os backend is a sibling of node-local: src/os/backends/remote may import only node builtins, relative modules, #/remote/protocol, #/os/interface and #/_base ('${specifier}') — the transport and environment wiring live in src/remote/client`; + return `the remote os backend is a sibling of node-local: src/os/backends/remote may import only node builtins, relative modules, #/remote/protocol, #/os/interface, #/os/operations and #/_base ('${specifier}') — the transport and environment wiring live in src/remote/client`; } function remoteServerConsumerViolation(absFile, specifier) { @@ -182,6 +184,10 @@ export function checkSource(source, absFile) { if (!inSrc) continue; + if (isInside(OS_OPERATIONS_ROOT, absFile) && !specifier.startsWith('node:') && !specifier.startsWith('./') && specifier !== 'pathe' && !OS_OPERATIONS_ALLOWED_RE.test(specifier)) { + violations.push({ file: absFile, line, message: `shared host operations stay DI-free and cannot import engine or transport modules ('${specifier}')` }); + continue; + } const remoteServer = remoteServerViolation(absFile, specifier); if (remoteServer !== undefined) { violations.push({ file: absFile, line, message: remoteServer }); diff --git a/packages/agent-core-v2/src/_base/fs/pathFilters.ts b/packages/agent-core-v2/src/_base/fs/pathFilters.ts new file mode 100644 index 00000000000..adaff55d15a --- /dev/null +++ b/packages/agent-core-v2/src/_base/fs/pathFilters.ts @@ -0,0 +1,85 @@ +import * as pathe from 'pathe'; + +const SENSITIVE_BASENAMES = new Set([ + '.env', + 'id_rsa', + 'id_ed25519', + 'id_ecdsa', + 'credentials', +]); + +const SENSITIVE_PATH_SUFFIXES = [ + ['.aws', 'credentials'], + ['.gcp', 'credentials'], +]; + +const ENV_PREFIX = '.env.'; +const ENV_EXEMPTIONS = new Set(['.env.example', '.env.sample', '.env.template']); + +const SENSITIVE_BASENAME_PREFIXES = ['id_rsa', 'id_ed25519', 'id_ecdsa', 'credentials']; +const PUBLIC_KEY_BASENAMES = new Set(['id_rsa.pub', 'id_ed25519.pub', 'id_ecdsa.pub']); +export const SENSITIVE_DOT_VARIANT_SUFFIXES = [ + '.bak', + '.backup', + '.copy', + '.disabled', + '.key', + '.old', + '.orig', + '.pem', + '.save', + '.tmp', +] as const; +const SENSITIVE_DOT_VARIANT_SUFFIX_SET = new Set(SENSITIVE_DOT_VARIANT_SUFFIXES); + +function comparable(path: string): string { + return path.toLowerCase(); +} + +export function isSensitiveFile(path: string): boolean { + const name = pathe.basename(path); + const comparableName = comparable(name); + const comparablePath = comparable(path); + + if (ENV_EXEMPTIONS.has(comparableName)) return false; + if (PUBLIC_KEY_BASENAMES.has(comparableName)) return false; + if (SENSITIVE_BASENAMES.has(comparableName)) return true; + if (comparableName.startsWith(ENV_PREFIX)) return true; + + for (const prefix of SENSITIVE_BASENAME_PREFIXES) { + if (comparableName === prefix) return true; + if (comparableName.length > prefix.length && comparableName.startsWith(prefix)) { + const suffix = comparableName.slice(prefix.length); + const next = suffix[0]; + if (next === '-' || next === '_') return true; + if (next === '.' && SENSITIVE_DOT_VARIANT_SUFFIX_SET.has(suffix)) return true; + } + } + + for (const suffixParts of SENSITIVE_PATH_SUFFIXES) { + const suffix = suffixParts.join('/'); + const comparableSuffix = comparable(suffix); + if ( + comparablePath.endsWith(`/${comparableSuffix}`) || + comparablePath.includes(`/${comparableSuffix}/`) + ) { + return true; + } + } + + return false; +} + +export function isWithinDirectory( + candidate: string, + base: string, + pathClass: 'posix' | 'win32' = process.platform === 'win32' ? 'win32' : 'posix', +): boolean { + const nc = pathe.normalize(candidate); + const nb = pathe.normalize(base); + const comparableCandidate = pathClass === 'win32' ? nc.toLowerCase() : nc; + const comparableBase = pathClass === 'win32' ? nb.toLowerCase() : nb; + if (comparableCandidate === comparableBase) return true; + const prefix = comparableBase.endsWith('/') ? comparableBase : comparableBase + '/'; + return comparableCandidate.startsWith(prefix); +} diff --git a/packages/agent-core-v2/src/_base/media/fileType.ts b/packages/agent-core-v2/src/_base/media/fileType.ts new file mode 100644 index 00000000000..21e626c43cc --- /dev/null +++ b/packages/agent-core-v2/src/_base/media/fileType.ts @@ -0,0 +1,368 @@ +import { + AUDIO_MIME_BY_SUFFIX, + IMAGE_MIME_BY_SUFFIX, + VIDEO_MIME_BY_SUFFIX, +} from './mimeTypes'; + +export { AUDIO_MIME_BY_SUFFIX, IMAGE_MIME_BY_SUFFIX, VIDEO_MIME_BY_SUFFIX }; + +export const MEDIA_SNIFF_BYTES = 512; + +export interface FileType { + readonly kind: 'text' | 'image' | 'video' | 'unknown'; + readonly mimeType: string; +} + +export type DetectFileTypeMode = 'text' | 'media'; + +const TEXT_MIME_BY_SUFFIX: Readonly> = Object.freeze({ + '.svg': 'image/svg+xml', +}); + +export const NON_TEXT_SUFFIXES: ReadonlySet = new Set([ + '.icns', + '.psd', + '.ai', + '.eps', + '.pdf', + '.doc', + '.docx', + '.dot', + '.dotx', + '.rtf', + '.odt', + '.xls', + '.xlsx', + '.xlsm', + '.xlt', + '.xltx', + '.xltm', + '.ods', + '.ppt', + '.pptx', + '.pptm', + '.pps', + '.ppsx', + '.odp', + '.pages', + '.numbers', + '.key', + '.zip', + '.rar', + '.7z', + '.tar', + '.gz', + '.tgz', + '.bz2', + '.xz', + '.zst', + '.lz', + '.lz4', + '.br', + '.cab', + '.ar', + '.deb', + '.rpm', + '.mp3', + '.wav', + '.flac', + '.ogg', + '.oga', + '.opus', + '.aac', + '.m4a', + '.wma', + '.ttf', + '.otf', + '.woff', + '.woff2', + '.exe', + '.dll', + '.so', + '.dylib', + '.bin', + '.apk', + '.ipa', + '.jar', + '.class', + '.pyc', + '.pyo', + '.wasm', + '.dmg', + '.iso', + '.img', + '.sqlite', + '.sqlite3', + '.db', + '.db3', +]); + +const ASF_HEADER = Buffer.from([ + 0x30, 0x26, 0xb2, 0x75, 0x8e, 0x66, 0xcf, 0x11, 0xa6, 0xd9, 0x00, 0xaa, 0x00, 0x62, 0xce, 0x6c, +]); + +const FTYP_IMAGE_BRANDS: Readonly> = Object.freeze({ + avif: 'image/avif', + avis: 'image/avif', + heic: 'image/heic', + heif: 'image/heif', + heix: 'image/heif', + hevc: 'image/heic', + mif1: 'image/heif', + msf1: 'image/heif', +}); + +const FTYP_VIDEO_BRANDS: Readonly> = Object.freeze({ + isom: 'video/mp4', + iso2: 'video/mp4', + iso5: 'video/mp4', + mp41: 'video/mp4', + mp42: 'video/mp4', + avc1: 'video/mp4', + mp4v: 'video/mp4', + m4v: 'video/x-m4v', + qt: 'video/quicktime', + '3gp4': 'video/3gpp', + '3gp5': 'video/3gpp', + '3gp6': 'video/3gpp', + '3gp7': 'video/3gpp', + '3g2': 'video/3gpp2', +}); + +function toBuffer(data: Buffer | Uint8Array): Buffer { + return Buffer.isBuffer(data) ? data : Buffer.from(data.buffer, data.byteOffset, data.byteLength); +} + +function startsWith(buf: Buffer, prefix: Buffer | readonly number[]): boolean { + const needle = Buffer.isBuffer(prefix) ? prefix : Buffer.from(prefix); + if (buf.length < needle.length) return false; + for (let i = 0; i < needle.length; i += 1) { + if (buf[i] !== needle[i]) return false; + } + return true; +} + +function sniffFtypBrand(header: Buffer): string | null { + if (header.length < 12) return null; + if (header.subarray(4, 8).toString('latin1') !== 'ftyp') return null; + const raw = header.subarray(8, 12).toString('latin1').toLowerCase(); + return raw.replaceAll(/[\s\u0000]+$/g, '').trim(); +} + +export function sniffMediaFromMagic(data: Buffer | Uint8Array): FileType | null { + const buf = toBuffer(data); + const header = buf.length > MEDIA_SNIFF_BYTES ? buf.subarray(0, MEDIA_SNIFF_BYTES) : buf; + + if (startsWith(header, [0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a])) { + return { kind: 'image', mimeType: 'image/png' }; + } + if (startsWith(header, [0xff, 0xd8, 0xff])) { + return { kind: 'image', mimeType: 'image/jpeg' }; + } + if (startsWith(header, Buffer.from('GIF87a')) || startsWith(header, Buffer.from('GIF89a'))) { + return { kind: 'image', mimeType: 'image/gif' }; + } + if (startsWith(header, Buffer.from('BM'))) { + return { kind: 'image', mimeType: 'image/bmp' }; + } + if ( + startsWith(header, [0x49, 0x49, 0x2a, 0x00]) || + startsWith(header, [0x4d, 0x4d, 0x00, 0x2a]) + ) { + return { kind: 'image', mimeType: 'image/tiff' }; + } + if (startsWith(header, [0x00, 0x00, 0x01, 0x00])) { + return { kind: 'image', mimeType: 'image/x-icon' }; + } + if (startsWith(header, Buffer.from('RIFF')) && header.length >= 12) { + const chunk = header.subarray(8, 12).toString('latin1'); + if (chunk === 'WEBP') return { kind: 'image', mimeType: 'image/webp' }; + if (chunk === 'AVI ') return { kind: 'video', mimeType: 'video/x-msvideo' }; + } + if (startsWith(header, Buffer.from('FLV'))) { + return { kind: 'video', mimeType: 'video/x-flv' }; + } + if (startsWith(header, ASF_HEADER)) { + return { kind: 'video', mimeType: 'video/x-ms-wmv' }; + } + if (startsWith(header, [0x1a, 0x45, 0xdf, 0xa3])) { + const lowered = header.toString('latin1').toLowerCase(); + if (lowered.includes('webm')) return { kind: 'video', mimeType: 'video/webm' }; + if (lowered.includes('matroska')) return { kind: 'video', mimeType: 'video/x-matroska' }; + } + const brand = sniffFtypBrand(header); + if (brand !== null && brand !== '') { + if (brand in FTYP_IMAGE_BRANDS) { + return { kind: 'image', mimeType: FTYP_IMAGE_BRANDS[brand]! }; + } + if (brand in FTYP_VIDEO_BRANDS) { + return { kind: 'video', mimeType: FTYP_VIDEO_BRANDS[brand]! }; + } + } + return null; +} + +export interface ImageDimensions { + readonly width: number; + readonly height: number; + readonly transposed?: boolean; +} + +export function sniffImageDimensions(data: Buffer | Uint8Array): ImageDimensions | null { + const buf = toBuffer(data); + + if (startsWith(buf, [0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]) && buf.length >= 24) { + return { width: buf.readUInt32BE(16), height: buf.readUInt32BE(20) }; + } + + if ( + (startsWith(buf, Buffer.from('GIF87a')) || startsWith(buf, Buffer.from('GIF89a'))) && + buf.length >= 10 + ) { + return { width: buf.readUInt16LE(6), height: buf.readUInt16LE(8) }; + } + + if (startsWith(buf, Buffer.from('BM')) && buf.length >= 26) { + return { width: buf.readInt32LE(18), height: Math.abs(buf.readInt32LE(22)) }; + } + + if (startsWith(buf, Buffer.from('RIFF')) && buf.length >= 30) { + const fourCc = buf.subarray(12, 16).toString('latin1'); + if (fourCc === 'VP8 ') { + return { + width: buf.readUInt16LE(26) & 0x3fff, + height: buf.readUInt16LE(28) & 0x3fff, + }; + } + if (fourCc === 'VP8L' && buf.length >= 25) { + const bits = buf.readUInt32LE(21); + return { + width: (bits & 0x3fff) + 1, + height: ((bits >> 14) & 0x3fff) + 1, + }; + } + if (fourCc === 'VP8X') { + const width = 1 + (buf[24]! | (buf[25]! << 8) | (buf[26]! << 16)); + const height = 1 + (buf[27]! | (buf[28]! << 8) | (buf[29]! << 16)); + return { width, height }; + } + } + + if (startsWith(buf, [0xff, 0xd8])) { + let orientation: number | null = null; + let offset = 2; + while (offset + 9 < buf.length) { + if (buf[offset] !== 0xff) { + offset += 1; + continue; + } + const marker = buf[offset + 1]!; + if ( + marker >= 0xc0 && + marker <= 0xcf && + marker !== 0xc4 && + marker !== 0xc8 && + marker !== 0xcc + ) { + const height = buf.readUInt16BE(offset + 5); + const width = buf.readUInt16BE(offset + 7); + return orientation !== null && orientation >= 5 + ? { width: height, height: width, transposed: true } + : { width, height }; + } + if (marker === 0xd8 || marker === 0xd9 || (marker >= 0xd0 && marker <= 0xd7)) { + offset += 2; + continue; + } + const segmentLength = buf.readUInt16BE(offset + 2); + if (segmentLength < 2) break; + if (marker === 0xe1 && orientation === null) { + orientation = readExifOrientation(buf, offset + 4, offset + 2 + segmentLength); + } + offset += 2 + segmentLength; + } + } + + return null; +} + +function readExifOrientation(buf: Buffer, start: number, end: number): number | null { + const boundedEnd = Math.min(end, buf.length); + if (start + 6 > boundedEnd || buf.toString('latin1', start, start + 6) !== 'Exif\0\0') { + return null; + } + const tiff = start + 6; + if (tiff + 8 > boundedEnd) return null; + const byteOrder = buf.toString('latin1', tiff, tiff + 2); + const le = byteOrder === 'II'; + if (!le && byteOrder !== 'MM') return null; + const u16 = (offset: number): number => (le ? buf.readUInt16LE(offset) : buf.readUInt16BE(offset)); + const u32 = (offset: number): number => (le ? buf.readUInt32LE(offset) : buf.readUInt32BE(offset)); + if (u16(tiff + 2) !== 42) return null; + const ifd = tiff + u32(tiff + 4); + if (ifd + 2 > boundedEnd) return null; + const entryCount = u16(ifd); + for (let i = 0; i < entryCount; i += 1) { + const entry = ifd + 2 + i * 12; + if (entry + 12 > boundedEnd) return null; + if (u16(entry) === 0x0112) { + const value = u16(entry + 8); + return value >= 1 && value <= 8 ? value : null; + } + } + return null; +} + +function getSuffix(path: string): string { + const idx = path.lastIndexOf('.'); + if (idx === -1) return ''; + const lastSep = Math.max(path.lastIndexOf('/'), path.lastIndexOf('\\')); + if (idx <= lastSep + 1) return ''; + return path.slice(idx).toLowerCase(); +} + +export function detectFileType( + path: string, + header?: Buffer | Uint8Array, + type: DetectFileTypeMode = 'text', +): FileType { + const suffix = getSuffix(path); + let mediaHint: FileType | null = null; + if (suffix in TEXT_MIME_BY_SUFFIX) { + mediaHint = { kind: 'text', mimeType: TEXT_MIME_BY_SUFFIX[suffix]! }; + } else if (suffix in IMAGE_MIME_BY_SUFFIX) { + mediaHint = { kind: 'image', mimeType: IMAGE_MIME_BY_SUFFIX[suffix]! }; + } else if (suffix in VIDEO_MIME_BY_SUFFIX) { + mediaHint = { kind: 'video', mimeType: VIDEO_MIME_BY_SUFFIX[suffix]! }; + } + + if (header !== undefined) { + const buf = toBuffer(header); + const sniffed = sniffMediaFromMagic(buf); + if (sniffed) { + if (type === 'media') return sniffed; + if (mediaHint) { + if (sniffed.kind !== mediaHint.kind) { + return { kind: 'unknown', mimeType: '' }; + } + return mediaHint; + } + return sniffed; + } + if (mediaHint?.kind === 'image') { + return { kind: 'unknown', mimeType: '' }; + } + if (type === 'media' && mediaHint?.kind === 'video') { + return mediaHint; + } + if (buf.includes(0x00)) { + return { kind: 'unknown', mimeType: '' }; + } + } + + if (mediaHint) return mediaHint; + if (NON_TEXT_SUFFIXES.has(suffix)) { + return { kind: 'unknown', mimeType: '' }; + } + return { kind: 'text', mimeType: 'text/plain' }; +} diff --git a/packages/agent-core-v2/src/_base/media/mimeTypes.ts b/packages/agent-core-v2/src/_base/media/mimeTypes.ts new file mode 100644 index 00000000000..d1bb267008e --- /dev/null +++ b/packages/agent-core-v2/src/_base/media/mimeTypes.ts @@ -0,0 +1,44 @@ +export const IMAGE_MIME_BY_SUFFIX: Readonly> = Object.freeze({ + '.png': 'image/png', + '.jpg': 'image/jpeg', + '.jpeg': 'image/jpeg', + '.gif': 'image/gif', + '.bmp': 'image/bmp', + '.tif': 'image/tiff', + '.tiff': 'image/tiff', + '.webp': 'image/webp', + '.ico': 'image/x-icon', + '.heic': 'image/heic', + '.heif': 'image/heif', + '.avif': 'image/avif', + '.svgz': 'image/svg+xml', +}); + +export const VIDEO_MIME_BY_SUFFIX: Readonly> = Object.freeze({ + '.mp4': 'video/mp4', + '.mpg': 'video/mpeg', + '.mpeg': 'video/mpeg', + '.mkv': 'video/x-matroska', + '.avi': 'video/x-msvideo', + '.mov': 'video/quicktime', + '.ogv': 'video/ogg', + '.wmv': 'video/x-ms-wmv', + '.webm': 'video/webm', + '.m4v': 'video/x-m4v', + '.flv': 'video/x-flv', + '.3gp': 'video/3gpp', + '.3g2': 'video/3gpp2', +}); + +export const AUDIO_MIME_BY_SUFFIX: Readonly> = Object.freeze({ + '.mp3': 'audio/mpeg', + '.wav': 'audio/wav', + '.m4a': 'audio/mp4', + '.ogg': 'audio/ogg', + '.oga': 'audio/ogg', + '.flac': 'audio/flac', + '.aac': 'audio/aac', + '.opus': 'audio/opus', + '.weba': 'audio/webm', + '.wma': 'audio/x-ms-wma', +}); diff --git a/packages/agent-core-v2/src/_base/text/outputAccumulator.ts b/packages/agent-core-v2/src/_base/text/outputAccumulator.ts new file mode 100644 index 00000000000..16406631ba3 --- /dev/null +++ b/packages/agent-core-v2/src/_base/text/outputAccumulator.ts @@ -0,0 +1,94 @@ +import { sliceOnCodePointBoundary } from '#/_base/text/code-points'; +import { + DEFAULT_TOOL_RESULT_MAX_CHARS, + DEFAULT_TOOL_RESULT_MAX_RETAINED_CHARS, + type ToolResultSpill, +} from './outputLimits'; + +export interface ToolOutputAccumulatorResult { + readonly output: string; + readonly isError: boolean; + readonly spill?: ToolResultSpill; + readonly brief?: string; +} + +export class ToolOutputAccumulator { + private readonly buffer: string[] = []; + private retainedChars = 0; + private totalCharsValue = 0; + private saturated = false; + + get nChars(): number { + return this.retainedChars; + } + + get totalChars(): number { + return this.totalCharsValue; + } + + write(text: string): void { + this.totalCharsValue += text.length; + if (this.saturated) return; + const remainingRetention = DEFAULT_TOOL_RESULT_MAX_RETAINED_CHARS - this.retainedChars; + const kept = + text.length <= remainingRetention + ? text + : sliceOnCodePointBoundary(text, remainingRetention); + if (kept.length !== text.length) this.saturated = true; + this.buffer.push(kept); + this.retainedChars += kept.length; + } + + ok(message = '', options: { readonly brief?: string } = {}): ToolOutputAccumulatorResult { + let finalMessage = message; + if (finalMessage.length > 0 && !finalMessage.endsWith('.')) { + finalMessage += '.'; + } + const output = this.buffer.join(''); + return { + isError: false, + output: output.length === 0 ? finalMessage : output, + brief: options.brief, + spill: this.completionSpill(finalMessage), + }; + } + + error( + message: string, + options: { readonly brief?: string } = {}, + ): ToolOutputAccumulatorResult { + const output = this.buffer.join(''); + return { + isError: true, + output: + message.length === 0 + ? output + : output.length === 0 + ? message + : output.endsWith('\n') + ? `${output}${message}` + : `${output}\n${message}`, + brief: options.brief, + spill: this.retentionSpill(message), + }; + } + + private retentionSpill(suffix?: string): ToolResultSpill | undefined { + if (this.totalCharsValue <= this.retainedChars) return undefined; + return { + totalChars: this.totalCharsValue, + suffix: suffix !== undefined && suffix.length > 0 ? suffix : undefined, + }; + } + + private completionSpill(suffix: string): ToolResultSpill | undefined { + const retentionSpill = this.retentionSpill(); + if (retentionSpill !== undefined) { + return suffix.length > 0 ? { ...retentionSpill, suffix } : retentionSpill; + } + if (suffix.length === 0 || this.totalCharsValue <= DEFAULT_TOOL_RESULT_MAX_CHARS) { + return undefined; + } + return { suffix }; + } +} diff --git a/packages/agent-core-v2/src/_base/text/outputLimits.ts b/packages/agent-core-v2/src/_base/text/outputLimits.ts new file mode 100644 index 00000000000..430e1f951b3 --- /dev/null +++ b/packages/agent-core-v2/src/_base/text/outputLimits.ts @@ -0,0 +1,9 @@ +export const DEFAULT_TOOL_RESULT_MAX_CHARS = 50_000; + +export const DEFAULT_TOOL_RESULT_MAX_RETAINED_CHARS = 10_000_000; + +export interface ToolResultSpill { + readonly outputPath?: string; + readonly totalChars?: number; + readonly suffix?: string; +} diff --git a/packages/agent-core-v2/src/_base/text/resultStatus.ts b/packages/agent-core-v2/src/_base/text/resultStatus.ts new file mode 100644 index 00000000000..6a1bdc51a22 --- /dev/null +++ b/packages/agent-core-v2/src/_base/text/resultStatus.ts @@ -0,0 +1,12 @@ +export const TOOL_ERROR_STATUS = 'ERROR: Tool execution failed.'; +export const TOOL_EMPTY_STATUS = 'Tool output is empty.'; +export const TOOL_EMPTY_ERROR_STATUS = + 'ERROR: Tool execution failed. Tool output is empty.'; +export const TOOL_OUTPUT_EMPTY_TEXT = 'Tool output is empty.'; + +export function renderTextStatus(output: string, isError?: boolean): string { + if (isError === true) { + return output.length === 0 ? TOOL_EMPTY_ERROR_STATUS : TOOL_ERROR_STATUS + '\n' + output; + } + return output.trim().length === 0 || output.trim() === TOOL_OUTPUT_EMPTY_TEXT ? TOOL_EMPTY_STATUS : output; +} diff --git a/packages/agent-core-v2/src/agent/agentsMdReminder/agentsMdReminderService.ts b/packages/agent-core-v2/src/agent/agentsMdReminder/agentsMdReminderService.ts index 6bdca1e2306..94bb7907f47 100644 --- a/packages/agent-core-v2/src/agent/agentsMdReminder/agentsMdReminderService.ts +++ b/packages/agent-core-v2/src/agent/agentsMdReminder/agentsMdReminderService.ts @@ -9,7 +9,7 @@ import { IBashParserService } from '#/app/bashParser/bashParser'; import { IBootstrapService } from '#/app/bootstrap/bootstrap'; import type { AgentsMdReminderShownEvent } from '#/app/telemetry/events'; import { ITelemetryService } from '#/app/telemetry/telemetry'; -import type { IHostFileSystem } from '#/os/interface/hostFileSystem'; +import { discoverInstructions } from '#/os/operations/instructions'; import type { WatchChange } from '#human/utils/watch'; import { IAgentEnvironmentService } from '#/agent/environmentBinding/agentEnvironment'; import { acquireEnvironmentLease } from '#/agent/permissionPolicy/policies/environment-lease'; @@ -19,10 +19,6 @@ import { ISessionInstructionsProvider } from '#/session/sessionInstructions/inst import { normalizeUserPath } from '#/tool/path-access'; import { AGENTS_MD_PLAIN_NAMES, - agentsMdCandidatePaths, - dirsRootToLeaf, - findAgentsMdInDir, - findProjectRoot, extractAgentsMdPathsFromSystemPrompt, loadAgentsMdDetailed, } from '#/agent/profile/context'; @@ -190,13 +186,9 @@ export class AgentAgentsMdReminderService const { dirs, selfKnown } = this.targetDirs(ctx); const selfKnownSet = new Set(selfKnown); const discovered: string[] = []; - for (const dir of dirs) { - for (const path of await this.probeDir(dir)) { - if (this.known.has(path) || this.remindQueue.has(path) || selfKnownSet.has(path)) { - continue; - } - discovered.push(path); - } + for (const path of await this.probeDirs(dirs)) { + if (this.known.has(path) || this.remindQueue.has(path) || selfKnownSet.has(path)) continue; + discovered.push(path); } for (const path of selfKnown) { this.remindQueue.delete(path); @@ -310,43 +302,20 @@ export class AgentAgentsMdReminderService return { dirs: [...new Set(dirs)], selfKnown: [...new Set(selfKnown)] }; } - private async probeDir(dir: string): Promise { + private async probeDirs(dirs: readonly string[]): Promise { + if (dirs.length === 0) return []; const lease = acquireEnvironmentLease(this.environment, ['fs']); if (lease === undefined) return []; try { const fs = lease.environment.fs!; - const anchor = await this.nearestExistingDir(fs, dir); - if (anchor === undefined) return []; - const deps = { fs }; - const projectRoot = await findProjectRoot(deps, anchor); - const chain = dirsRootToLeaf(anchor, projectRoot); - const found: string[] = []; - for (const chainDir of chain) { - const candidates = agentsMdCandidatePaths(chainDir); - if (candidates.every((candidate) => this.known.has(normalize(candidate)))) continue; - for (const path of await findAgentsMdInDir(deps, chainDir)) { - found.push(normalize(path)); - } - } - return found; + return await (fs.operations === undefined + ? discoverInstructions(fs, dirs, [...this.known]) + : fs.operations.discoverInstructions(dirs, [...this.known])); } finally { lease.dispose(); } } - private async nearestExistingDir(fs: IHostFileSystem, path: string): Promise { - let current = path; - for (;;) { - const stat = await fs.stat(current).catch(() => undefined); - if (stat?.isDirectory === true) return current; - const parent = dirname(current); - if (parent === current) { - this.log.debug('agentsMdReminder probe found no existing anchor directory', { path }); - return undefined; - } - current = parent; - } - } } function hostPath(path: string, pathClass: 'posix' | 'win32'): string { diff --git a/packages/agent-core-v2/src/agent/contextMemory/toolResultRender.ts b/packages/agent-core-v2/src/agent/contextMemory/toolResultRender.ts index 2bf13ed32a0..f936f9ea417 100644 --- a/packages/agent-core-v2/src/agent/contextMemory/toolResultRender.ts +++ b/packages/agent-core-v2/src/agent/contextMemory/toolResultRender.ts @@ -1,13 +1,8 @@ +import { renderTextStatus, TOOL_EMPTY_STATUS, TOOL_EMPTY_ERROR_STATUS, TOOL_ERROR_STATUS } from '#/_base/text/resultStatus'; import type { ContentPart } from '#human/llm/message'; import { isMcpToolName } from '#/tool/toolContract'; -const TOOL_ERROR_STATUS = 'ERROR: Tool execution failed.'; -const TOOL_EMPTY_STATUS = 'Tool output is empty.'; -const TOOL_EMPTY_ERROR_STATUS = - 'ERROR: Tool execution failed. Tool output is empty.'; -const TOOL_OUTPUT_EMPTY_TEXT = 'Tool output is empty.'; - export interface RenderableToolResult { readonly output: string | readonly ContentPart[]; readonly note?: string; @@ -52,11 +47,7 @@ function renderStatus(result: RenderableToolResult): ContentPart[] { const output = result.output; const single = typeof output === 'string' ? output : singleTextPart(output); if (single !== undefined) { - if (result.isError === true) { - if (single.length === 0) return [textPart(TOOL_EMPTY_ERROR_STATUS)]; - return [textPart(TOOL_ERROR_STATUS + '\n' + single)]; - } - return isEmptyOutputText(single) ? [textPart(TOOL_EMPTY_STATUS)] : [textPart(single)]; + return [textPart(renderTextStatus(single, result.isError))]; } const parts = output as readonly ContentPart[]; @@ -76,10 +67,6 @@ function textPart(text: string): ContentPart { return { type: 'text', text }; } -function isEmptyOutputText(output: string): boolean { - return output.trim().length === 0 || output.trim() === TOOL_OUTPUT_EMPTY_TEXT; -} - function isEmptyEquivalentContentArray(output: readonly ContentPart[]): boolean { return output.every((part) => part.type === 'text' && part.text.trim().length === 0); } diff --git a/packages/agent-core-v2/src/agent/media/file-type.ts b/packages/agent-core-v2/src/agent/media/file-type.ts index d729c68231c..459e0d5db11 100644 --- a/packages/agent-core-v2/src/agent/media/file-type.ts +++ b/packages/agent-core-v2/src/agent/media/file-type.ts @@ -1,368 +1 @@ -import { - AUDIO_MIME_BY_SUFFIX, - IMAGE_MIME_BY_SUFFIX, - VIDEO_MIME_BY_SUFFIX, -} from './mediaRef'; - -export { AUDIO_MIME_BY_SUFFIX, IMAGE_MIME_BY_SUFFIX, VIDEO_MIME_BY_SUFFIX }; - -export const MEDIA_SNIFF_BYTES = 512; - -export interface FileType { - readonly kind: 'text' | 'image' | 'video' | 'unknown'; - readonly mimeType: string; -} - -export type DetectFileTypeMode = 'text' | 'media'; - -const TEXT_MIME_BY_SUFFIX: Readonly> = Object.freeze({ - '.svg': 'image/svg+xml', -}); - -export const NON_TEXT_SUFFIXES: ReadonlySet = new Set([ - '.icns', - '.psd', - '.ai', - '.eps', - '.pdf', - '.doc', - '.docx', - '.dot', - '.dotx', - '.rtf', - '.odt', - '.xls', - '.xlsx', - '.xlsm', - '.xlt', - '.xltx', - '.xltm', - '.ods', - '.ppt', - '.pptx', - '.pptm', - '.pps', - '.ppsx', - '.odp', - '.pages', - '.numbers', - '.key', - '.zip', - '.rar', - '.7z', - '.tar', - '.gz', - '.tgz', - '.bz2', - '.xz', - '.zst', - '.lz', - '.lz4', - '.br', - '.cab', - '.ar', - '.deb', - '.rpm', - '.mp3', - '.wav', - '.flac', - '.ogg', - '.oga', - '.opus', - '.aac', - '.m4a', - '.wma', - '.ttf', - '.otf', - '.woff', - '.woff2', - '.exe', - '.dll', - '.so', - '.dylib', - '.bin', - '.apk', - '.ipa', - '.jar', - '.class', - '.pyc', - '.pyo', - '.wasm', - '.dmg', - '.iso', - '.img', - '.sqlite', - '.sqlite3', - '.db', - '.db3', -]); - -const ASF_HEADER = Buffer.from([ - 0x30, 0x26, 0xb2, 0x75, 0x8e, 0x66, 0xcf, 0x11, 0xa6, 0xd9, 0x00, 0xaa, 0x00, 0x62, 0xce, 0x6c, -]); - -const FTYP_IMAGE_BRANDS: Readonly> = Object.freeze({ - avif: 'image/avif', - avis: 'image/avif', - heic: 'image/heic', - heif: 'image/heif', - heix: 'image/heif', - hevc: 'image/heic', - mif1: 'image/heif', - msf1: 'image/heif', -}); - -const FTYP_VIDEO_BRANDS: Readonly> = Object.freeze({ - isom: 'video/mp4', - iso2: 'video/mp4', - iso5: 'video/mp4', - mp41: 'video/mp4', - mp42: 'video/mp4', - avc1: 'video/mp4', - mp4v: 'video/mp4', - m4v: 'video/x-m4v', - qt: 'video/quicktime', - '3gp4': 'video/3gpp', - '3gp5': 'video/3gpp', - '3gp6': 'video/3gpp', - '3gp7': 'video/3gpp', - '3g2': 'video/3gpp2', -}); - -function toBuffer(data: Buffer | Uint8Array): Buffer { - return Buffer.isBuffer(data) ? data : Buffer.from(data.buffer, data.byteOffset, data.byteLength); -} - -function startsWith(buf: Buffer, prefix: Buffer | readonly number[]): boolean { - const needle = Buffer.isBuffer(prefix) ? prefix : Buffer.from(prefix); - if (buf.length < needle.length) return false; - for (let i = 0; i < needle.length; i += 1) { - if (buf[i] !== needle[i]) return false; - } - return true; -} - -function sniffFtypBrand(header: Buffer): string | null { - if (header.length < 12) return null; - if (header.subarray(4, 8).toString('latin1') !== 'ftyp') return null; - const raw = header.subarray(8, 12).toString('latin1').toLowerCase(); - return raw.replaceAll(/[\s\u0000]+$/g, '').trim(); -} - -export function sniffMediaFromMagic(data: Buffer | Uint8Array): FileType | null { - const buf = toBuffer(data); - const header = buf.length > MEDIA_SNIFF_BYTES ? buf.subarray(0, MEDIA_SNIFF_BYTES) : buf; - - if (startsWith(header, [0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a])) { - return { kind: 'image', mimeType: 'image/png' }; - } - if (startsWith(header, [0xff, 0xd8, 0xff])) { - return { kind: 'image', mimeType: 'image/jpeg' }; - } - if (startsWith(header, Buffer.from('GIF87a')) || startsWith(header, Buffer.from('GIF89a'))) { - return { kind: 'image', mimeType: 'image/gif' }; - } - if (startsWith(header, Buffer.from('BM'))) { - return { kind: 'image', mimeType: 'image/bmp' }; - } - if ( - startsWith(header, [0x49, 0x49, 0x2a, 0x00]) || - startsWith(header, [0x4d, 0x4d, 0x00, 0x2a]) - ) { - return { kind: 'image', mimeType: 'image/tiff' }; - } - if (startsWith(header, [0x00, 0x00, 0x01, 0x00])) { - return { kind: 'image', mimeType: 'image/x-icon' }; - } - if (startsWith(header, Buffer.from('RIFF')) && header.length >= 12) { - const chunk = header.subarray(8, 12).toString('latin1'); - if (chunk === 'WEBP') return { kind: 'image', mimeType: 'image/webp' }; - if (chunk === 'AVI ') return { kind: 'video', mimeType: 'video/x-msvideo' }; - } - if (startsWith(header, Buffer.from('FLV'))) { - return { kind: 'video', mimeType: 'video/x-flv' }; - } - if (startsWith(header, ASF_HEADER)) { - return { kind: 'video', mimeType: 'video/x-ms-wmv' }; - } - if (startsWith(header, [0x1a, 0x45, 0xdf, 0xa3])) { - const lowered = header.toString('latin1').toLowerCase(); - if (lowered.includes('webm')) return { kind: 'video', mimeType: 'video/webm' }; - if (lowered.includes('matroska')) return { kind: 'video', mimeType: 'video/x-matroska' }; - } - const brand = sniffFtypBrand(header); - if (brand !== null && brand !== '') { - if (brand in FTYP_IMAGE_BRANDS) { - return { kind: 'image', mimeType: FTYP_IMAGE_BRANDS[brand]! }; - } - if (brand in FTYP_VIDEO_BRANDS) { - return { kind: 'video', mimeType: FTYP_VIDEO_BRANDS[brand]! }; - } - } - return null; -} - -export interface ImageDimensions { - readonly width: number; - readonly height: number; - readonly transposed?: boolean; -} - -export function sniffImageDimensions(data: Buffer | Uint8Array): ImageDimensions | null { - const buf = toBuffer(data); - - if (startsWith(buf, [0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]) && buf.length >= 24) { - return { width: buf.readUInt32BE(16), height: buf.readUInt32BE(20) }; - } - - if ( - (startsWith(buf, Buffer.from('GIF87a')) || startsWith(buf, Buffer.from('GIF89a'))) && - buf.length >= 10 - ) { - return { width: buf.readUInt16LE(6), height: buf.readUInt16LE(8) }; - } - - if (startsWith(buf, Buffer.from('BM')) && buf.length >= 26) { - return { width: buf.readInt32LE(18), height: Math.abs(buf.readInt32LE(22)) }; - } - - if (startsWith(buf, Buffer.from('RIFF')) && buf.length >= 30) { - const fourCc = buf.subarray(12, 16).toString('latin1'); - if (fourCc === 'VP8 ') { - return { - width: buf.readUInt16LE(26) & 0x3fff, - height: buf.readUInt16LE(28) & 0x3fff, - }; - } - if (fourCc === 'VP8L' && buf.length >= 25) { - const bits = buf.readUInt32LE(21); - return { - width: (bits & 0x3fff) + 1, - height: ((bits >> 14) & 0x3fff) + 1, - }; - } - if (fourCc === 'VP8X') { - const width = 1 + (buf[24]! | (buf[25]! << 8) | (buf[26]! << 16)); - const height = 1 + (buf[27]! | (buf[28]! << 8) | (buf[29]! << 16)); - return { width, height }; - } - } - - if (startsWith(buf, [0xff, 0xd8])) { - let orientation: number | null = null; - let offset = 2; - while (offset + 9 < buf.length) { - if (buf[offset] !== 0xff) { - offset += 1; - continue; - } - const marker = buf[offset + 1]!; - if ( - marker >= 0xc0 && - marker <= 0xcf && - marker !== 0xc4 && - marker !== 0xc8 && - marker !== 0xcc - ) { - const height = buf.readUInt16BE(offset + 5); - const width = buf.readUInt16BE(offset + 7); - return orientation !== null && orientation >= 5 - ? { width: height, height: width, transposed: true } - : { width, height }; - } - if (marker === 0xd8 || marker === 0xd9 || (marker >= 0xd0 && marker <= 0xd7)) { - offset += 2; - continue; - } - const segmentLength = buf.readUInt16BE(offset + 2); - if (segmentLength < 2) break; - if (marker === 0xe1 && orientation === null) { - orientation = readExifOrientation(buf, offset + 4, offset + 2 + segmentLength); - } - offset += 2 + segmentLength; - } - } - - return null; -} - -function readExifOrientation(buf: Buffer, start: number, end: number): number | null { - const boundedEnd = Math.min(end, buf.length); - if (start + 6 > boundedEnd || buf.toString('latin1', start, start + 6) !== 'Exif\0\0') { - return null; - } - const tiff = start + 6; - if (tiff + 8 > boundedEnd) return null; - const byteOrder = buf.toString('latin1', tiff, tiff + 2); - const le = byteOrder === 'II'; - if (!le && byteOrder !== 'MM') return null; - const u16 = (offset: number): number => (le ? buf.readUInt16LE(offset) : buf.readUInt16BE(offset)); - const u32 = (offset: number): number => (le ? buf.readUInt32LE(offset) : buf.readUInt32BE(offset)); - if (u16(tiff + 2) !== 42) return null; - const ifd = tiff + u32(tiff + 4); - if (ifd + 2 > boundedEnd) return null; - const entryCount = u16(ifd); - for (let i = 0; i < entryCount; i += 1) { - const entry = ifd + 2 + i * 12; - if (entry + 12 > boundedEnd) return null; - if (u16(entry) === 0x0112) { - const value = u16(entry + 8); - return value >= 1 && value <= 8 ? value : null; - } - } - return null; -} - -function getSuffix(path: string): string { - const idx = path.lastIndexOf('.'); - if (idx === -1) return ''; - const lastSep = Math.max(path.lastIndexOf('/'), path.lastIndexOf('\\')); - if (idx <= lastSep + 1) return ''; - return path.slice(idx).toLowerCase(); -} - -export function detectFileType( - path: string, - header?: Buffer | Uint8Array, - type: DetectFileTypeMode = 'text', -): FileType { - const suffix = getSuffix(path); - let mediaHint: FileType | null = null; - if (suffix in TEXT_MIME_BY_SUFFIX) { - mediaHint = { kind: 'text', mimeType: TEXT_MIME_BY_SUFFIX[suffix]! }; - } else if (suffix in IMAGE_MIME_BY_SUFFIX) { - mediaHint = { kind: 'image', mimeType: IMAGE_MIME_BY_SUFFIX[suffix]! }; - } else if (suffix in VIDEO_MIME_BY_SUFFIX) { - mediaHint = { kind: 'video', mimeType: VIDEO_MIME_BY_SUFFIX[suffix]! }; - } - - if (header !== undefined) { - const buf = toBuffer(header); - const sniffed = sniffMediaFromMagic(buf); - if (sniffed) { - if (type === 'media') return sniffed; - if (mediaHint) { - if (sniffed.kind !== mediaHint.kind) { - return { kind: 'unknown', mimeType: '' }; - } - return mediaHint; - } - return sniffed; - } - if (mediaHint?.kind === 'image') { - return { kind: 'unknown', mimeType: '' }; - } - if (type === 'media' && mediaHint?.kind === 'video') { - return mediaHint; - } - if (buf.includes(0x00)) { - return { kind: 'unknown', mimeType: '' }; - } - } - - if (mediaHint) return mediaHint; - if (NON_TEXT_SUFFIXES.has(suffix)) { - return { kind: 'unknown', mimeType: '' }; - } - return { kind: 'text', mimeType: 'text/plain' }; -} +export * from '#/_base/media/fileType'; diff --git a/packages/agent-core-v2/src/agent/media/mediaRef.ts b/packages/agent-core-v2/src/agent/media/mediaRef.ts index d1ea3e2fd02..3f2bc71f3b4 100644 --- a/packages/agent-core-v2/src/agent/media/mediaRef.ts +++ b/packages/agent-core-v2/src/agent/media/mediaRef.ts @@ -1,53 +1,11 @@ +import { AUDIO_MIME_BY_SUFFIX, IMAGE_MIME_BY_SUFFIX, VIDEO_MIME_BY_SUFFIX } from '#/_base/media/mimeTypes'; import { join } from 'node:path'; import type { ContentPart } from '#human/llm/message'; -export type MediaKind = 'image' | 'video' | 'audio' | 'file'; +export { AUDIO_MIME_BY_SUFFIX, IMAGE_MIME_BY_SUFFIX, VIDEO_MIME_BY_SUFFIX } from '#/_base/media/mimeTypes'; -export const IMAGE_MIME_BY_SUFFIX: Readonly> = Object.freeze({ - '.png': 'image/png', - '.jpg': 'image/jpeg', - '.jpeg': 'image/jpeg', - '.gif': 'image/gif', - '.bmp': 'image/bmp', - '.tif': 'image/tiff', - '.tiff': 'image/tiff', - '.webp': 'image/webp', - '.ico': 'image/x-icon', - '.heic': 'image/heic', - '.heif': 'image/heif', - '.avif': 'image/avif', - '.svgz': 'image/svg+xml', -}); - -export const VIDEO_MIME_BY_SUFFIX: Readonly> = Object.freeze({ - '.mp4': 'video/mp4', - '.mpg': 'video/mpeg', - '.mpeg': 'video/mpeg', - '.mkv': 'video/x-matroska', - '.avi': 'video/x-msvideo', - '.mov': 'video/quicktime', - '.ogv': 'video/ogg', - '.wmv': 'video/x-ms-wmv', - '.webm': 'video/webm', - '.m4v': 'video/x-m4v', - '.flv': 'video/x-flv', - '.3gp': 'video/3gpp', - '.3g2': 'video/3gpp2', -}); - -export const AUDIO_MIME_BY_SUFFIX: Readonly> = Object.freeze({ - '.mp3': 'audio/mpeg', - '.wav': 'audio/wav', - '.m4a': 'audio/mp4', - '.ogg': 'audio/ogg', - '.oga': 'audio/ogg', - '.flac': 'audio/flac', - '.aac': 'audio/aac', - '.opus': 'audio/opus', - '.weba': 'audio/webm', - '.wma': 'audio/x-ms-wma', -}); +export type MediaKind = 'image' | 'video' | 'audio' | 'file'; const IMAGE_EXT_BY_MIME = invertMimeBySuffix(IMAGE_MIME_BY_SUFFIX); const VIDEO_EXT_BY_MIME = invertMimeBySuffix(VIDEO_MIME_BY_SUFFIX); diff --git a/packages/agent-core-v2/src/agent/profile/context.ts b/packages/agent-core-v2/src/agent/profile/context.ts index 2075e71ec37..5fb57a63efb 100644 --- a/packages/agent-core-v2/src/agent/profile/context.ts +++ b/packages/agent-core-v2/src/agent/profile/context.ts @@ -1,10 +1,13 @@ -import { basename, dirname, join, normalize } from 'pathe'; +import { AGENTS_MD_PLAIN_NAMES, dotKimiAgentsMdPath, dirsRootToLeaf } from '#/os/operations/instructions'; +import { basename, join, normalize } from 'pathe'; import { findGitWorkTree } from '#/app/git/workTree'; import type { IHostFileSystem } from '#/os/interface/hostFileSystem'; import type { SystemPromptContext } from './profile'; +export { AGENTS_MD_PLAIN_NAMES, dotKimiAgentsMdPath, agentsMdCandidatePaths, dirsRootToLeaf, findAgentsMdInDir } from '#/os/operations/instructions'; + export const AGENTS_MD_RECOMMENDED_MAX_BYTES = 32 * 1024; export const LIST_DIR_ROOT_WIDTH = 30; @@ -100,16 +103,6 @@ export function splitAgentsMdFiles( return files; } -export const AGENTS_MD_PLAIN_NAMES = ['AGENTS.md', 'agents.md'] as const; - -export function dotKimiAgentsMdPath(dir: string): string { - return join(dir, '.kimi-code', 'AGENTS.md'); -} - -export function agentsMdCandidatePaths(dir: string): string[] { - return [dotKimiAgentsMdPath(dir), ...AGENTS_MD_PLAIN_NAMES.map((name) => join(dir, name))]; -} - export function extractAgentsMdPathsFromSystemPrompt(systemPrompt: string): string[] { const paths: string[] = []; const seen = new Set(); @@ -129,35 +122,6 @@ export function extractAgentsMdPathsFromSystemPrompt(systemPrompt: string): stri return paths; } -export async function findAgentsMdInDir( - deps: { readonly fs: IHostFileSystem }, - dir: string, -): Promise { - const found: string[] = []; - const dotKimi = dotKimiAgentsMdPath(dir); - if (await isNonEmptyFile(deps, dotKimi)) found.push(dotKimi); - for (const fileName of AGENTS_MD_PLAIN_NAMES) { - const candidate = join(dir, fileName); - if (await isNonEmptyFile(deps, candidate)) { - found.push(candidate); - break; - } - } - return found; -} - -async function isNonEmptyFile( - deps: { readonly fs: IHostFileSystem }, - path: string, -): Promise { - try { - const content = await deps.fs.readText(path, { errors: 'ignore' }); - return content.trim().length > 0; - } catch { - return false; - } -} - export async function loadAgentsMdForRoots( deps: ProfileContextDeps, brandHome: string | undefined, @@ -277,21 +241,6 @@ export async function findProjectRoot( return (await findGitWorkTree(deps.fs, rootWorkDir))?.root ?? rootWorkDir; } -export function dirsRootToLeaf(workDir: string, projectRoot: string): string[] { - const dirs: string[] = []; - let current = normalize(workDir); - - while (true) { - dirs.push(current); - if (current === projectRoot) break; - const parent = dirname(current); - if (parent === current) break; - current = parent; - } - - return dirs.toReversed(); -} - interface AgentFile { readonly path: string; readonly content: string; diff --git a/packages/agent-core-v2/src/agent/tools/fileReadSource.ts b/packages/agent-core-v2/src/agent/tools/fileReadSource.ts index 611558af915..dad55321224 100644 --- a/packages/agent-core-v2/src/agent/tools/fileReadSource.ts +++ b/packages/agent-core-v2/src/agent/tools/fileReadSource.ts @@ -1,16 +1,11 @@ +import type { FileReadSource } from '#/os/interface/fileOperations'; import { readUtf8Lines } from '#/_base/execEnv/decodeText'; -import type { HostFileStat, IHostFileSystem } from '#/os/interface/hostFileSystem'; +import type { IHostFileSystem } from '#/os/interface/hostFileSystem'; import { parseDaemonFileUrl } from '#/agent/media/mediaRef'; import type { ISessionMediaStore } from '#/agent/media/sessionMediaStore'; import type { ExecutableToolResult } from '#/tool/toolContract'; -export interface FileReadSource { - readonly name: string; - readonly localPath?: string; - stat(): Promise; - readBytes(n?: number): Promise; - readLines(): AsyncIterable; -} +export type { FileReadSource } from '#/os/interface/fileOperations'; export function withAttachmentLocation(result: ExecutableToolResult, source: FileReadSource): ExecutableToolResult { if (!result.isError || source.localPath === undefined || typeof result.output !== 'string') return result; @@ -20,6 +15,7 @@ export function withAttachmentLocation(result: ExecutableToolResult, source: Fil export function environmentFileSource(fs: IHostFileSystem, path: string): FileReadSource { return { name: path, + readTextPage: fs.operations === undefined ? undefined : (options, signal) => fs.operations!.readTextPage(path, options, signal), stat: () => fs.stat(path), readBytes: (n) => fs.readBytes(path, n), readLines: () => fs.readLines(path, { errors: 'strict' }), diff --git a/packages/agent-core-v2/src/agent/tools/os/glob/globTool.ts b/packages/agent-core-v2/src/agent/tools/os/glob/globTool.ts index d34c9e7017d..2ed6b4c00a5 100644 --- a/packages/agent-core-v2/src/agent/tools/os/glob/globTool.ts +++ b/packages/agent-core-v2/src/agent/tools/os/glob/globTool.ts @@ -1,13 +1,8 @@ -import { normalize, resolve } from 'pathe'; +import { searchGlob } from '#/os/operations/searchGlob'; import { ensureRgPath, rgUnavailableMessage, type RgProbe } from '#/os/backends/node-local/tools/rgLocator'; import type { Environment } from '#/environment/environment'; -import { - DEFAULT_TIMEOUT_MS, - MAX_OUTPUT_BYTES, - runRgOnce, - shouldRetryRipgrepEagain, -} from '#/os/backends/node-local/tools/runRg'; + import type { IHostEnvironment } from '#/os/interface/hostEnvironment'; import type { IHostFileSystem } from '#/os/interface/hostFileSystem'; import { isHostFsNotFound } from '#/os/interface/hostFsErrors'; @@ -19,18 +14,13 @@ import { ISessionWorkspaceContext } from '#/session/workspaceContext/workspaceCo import { ITelemetryService } from '#/app/telemetry/telemetry'; import { IOAuthService } from '#/app/auth/auth'; import { - DEFAULT_TOOL_RESULT_MAX_RETAINED_CHARS, ToolAccesses, type ExecutableToolResult, type ToolExecution, } from '#/tool/toolContract'; import { registerAgentToolService } from '#/agent/toolRegistry/toolContribution'; import { - isWithinDirectory, resolvePathAccessPath, - type PathClass, - isSensitiveFile, - SENSITIVE_DOT_VARIANT_SUFFIXES, type WorkspaceConfig, } from '#/tool/path-access'; import { toInputJsonSchema } from '#/tool/input-schema'; @@ -40,26 +30,9 @@ import { type GlobInput, GlobInputSchema, IGlobTool, - DEFAULT_HEAD_LIMIT, WINDOWS_PATH_HINT, } from './glob'; -const VCS_DIRECTORIES_TO_EXCLUDE = ['.git', '.svn', '.hg', '.bzr', '.jj', '.sl'] as const; - -const SENSITIVE_KEY_BASENAMES = ['id_rsa', 'id_ed25519', 'id_ecdsa'] as const; -const SENSITIVE_GLOBS_TO_EXCLUDE: readonly string[] = [ - '**/.env', - ...SENSITIVE_KEY_BASENAMES.flatMap((name) => [ - `**/${name}`, - `**/${name}[-_]*`, - ...SENSITIVE_DOT_VARIANT_SUFFIXES.map((suffix) => `**/${name}${suffix}`), - ]), - '**/.aws/credentials', - '**/.aws/credentials/**', - '**/.gcp/credentials', - '**/.gcp/credentials/**', -]; - export class GlobTool implements IGlobTool { declare readonly _serviceBrand: undefined; readonly name = 'Glob' as const; @@ -203,140 +176,16 @@ export class GlobTool implements IGlobTool { return { isError: true, output: rgUnavailableMessage(error, environment) }; } - let run; try { - run = await runRgOnce(processService, buildRgArgs(rgPath, args), signal, { cwd: searchRoot }); + const request = { kind: 'glob' as const, args, rgPath, roots: searchRoots, workspaceDir: workspace.workspaceDir, pathClass: env.pathClass }; + return await (processService.search === undefined + ? searchGlob(processService, request, signal) + : processService.search(request, signal)); } catch (error) { return { isError: true, output: formatSpawnError(error, environment) }; } - if (run.kind === 'aborted') { - return { isError: true, output: 'Glob aborted' }; - } - - if (shouldRetryRipgrepEagain(run)) { - try { - run = await runRgOnce(processService, buildRgArgs(rgPath, args, true), signal, { cwd: searchRoot }); - } catch (error) { - return { isError: true, output: formatSpawnError(error, environment) }; - } - if (run.kind === 'aborted') { - return { isError: true, output: 'Glob aborted' }; - } - } - - const { exitCode, stdoutText, stderrText, bufferTruncated, timedOut } = run; - - let traversalWarning: string | undefined; - if (exitCode !== 0 && exitCode !== 1 && !timedOut) { - const rawPathsBeforeError = splitCompletePaths(stdoutText, true); - if (rawPathsBeforeError.length === 0) { - return { isError: true, output: formatGlobError(searchRoot, stderrText) }; - } - traversalWarning = formatGlobWarning(stderrText); - } - if (signal.aborted) { - return { isError: true, output: 'Glob aborted' }; - } - - const rawPaths = splitCompletePaths(stdoutText, bufferTruncated || timedOut).map((p) => - resolve(searchRoot, p), - ); - - const kept: string[] = []; - let filteredSensitive = 0; - for (const p of rawPaths) { - if (isSensitiveFile(p)) { - filteredSensitive++; - } else { - kept.push(p); - } - } - - const offset = args.offset ?? 0; - const headLimit = args.head_limit ?? DEFAULT_HEAD_LIMIT; - const limited = headLimit === 0 ? kept.slice(offset) : kept.slice(offset, offset + headLimit); - const partial = bufferTruncated || timedOut || traversalWarning !== undefined; - - const pathClass = env.pathClass; - const shouldRelativize = isWithinDirectory(searchRoot, workspace.workspaceDir, pathClass); - const candidates = limited.map((p) => - shouldRelativize ? relativizeIfUnder(p, searchRoot, pathClass) : p, - ); - - const warnings: string[] = []; - if (timedOut) { - warnings.push( - `Glob timed out after ${String(DEFAULT_TIMEOUT_MS / 1000)}s; partial results returned.`, - ); - } - if (bufferTruncated) { - warnings.push( - `[stdout truncated at ${String(MAX_OUTPUT_BYTES)} bytes; results may be incomplete — use a more specific pattern]`, - ); - } - if (traversalWarning !== undefined) { - warnings.push(traversalWarning); - } - const pageNotices = (count: number, characterLimited: boolean) => { - const lines = [...warnings]; - const footer: string[] = []; - const truncated = characterLimited || offset + count < kept.length; - if (count === 0) { - if (kept.length > 0) { - const resultSet = partial ? 'collected partial result set' : 'current result set'; - lines.push( - `No more matches at offset=${String(offset)} in the ${resultSet} (${String(kept.length)} matches).`, - ); - } else if (partial) { - lines.push('No matches collected; search incomplete.'); - } else if (filteredSensitive > 0) { - lines.push( - `No non-sensitive matches found (${String(filteredSensitive)} sensitive file(s) filtered).`, - ); - } else { - lines.push('No matches found'); - } - } else if (truncated || offset > 0 || partial) { - const total = partial - ? `${String(kept.length)} collected matches (partial result set)` - : String(kept.length); - lines.push(`Showing matches ${String(offset + 1)}–${String(offset + count)} of ${total}.`); - } - if (characterLimited) lines.push('Character limit reached; only complete paths are returned.'); - if (truncated) { - lines.push( - `Continue with the same search arguments and offset=${String(offset + count)}.`, - ); - if (!characterLimited) lines.push('To remove the match-count limit, omit offset and use head_limit=0.'); - } - if (filteredSensitive > 0 && (kept.length > 0 || partial)) { - footer.push(`Filtered ${String(filteredSensitive)} sensitive file(s).`); - } - if (!truncated && !partial && offset === 0 && headLimit > 0 && count === headLimit) { - footer.push(`Found ${String(count)} matches`); - } - return { lines, footer }; - }; - const noticeChars = Math.max(...[false, true].map((characterLimited) => { - const { lines, footer } = pageNotices(candidates.length, characterLimited); - return [...lines, ...footer].join('\n').length + 2; - })); - let remaining = DEFAULT_TOOL_RESULT_MAX_RETAINED_CHARS - noticeChars; - const displayLines: string[] = []; - for (const path of candidates) { - if (path.length + 1 > remaining) break; - displayLines.push(path); - remaining -= path.length + 1; - } - if (candidates.length > 0 && displayLines.length === 0) { - return { - isError: true, - output: 'Glob cannot fit a complete path and its diagnostics within the output limit. Narrow the search path or pattern.', - }; - } - const notices = pageNotices(displayLines.length, displayLines.length < candidates.length); - return { output: [...notices.lines, ...displayLines, ...notices.footer].join('\n') }; } + } registerAgentToolService(IGlobTool, GlobTool, { @@ -367,37 +216,6 @@ function createRgProbe(processService: IHostProcessService, cwd: string): RgProb }; } -function buildRgArgs(rgPath: string, args: GlobInput, singleThreaded = false): string[] { - const cmd: string[] = [rgPath]; - if (singleThreaded) cmd.push('-j', '1'); - cmd.push('--files', '--hidden', '--sortr=modified'); - for (const dir of VCS_DIRECTORIES_TO_EXCLUDE) { - cmd.push('--glob', `!${dir}`); - } - cmd.push('--glob', args.pattern); - for (const glob of SENSITIVE_GLOBS_TO_EXCLUDE) { - cmd.push('--glob', `!${glob}`); - } - if (args.include_ignored) cmd.push('--no-ignore'); - cmd.push('.'); - return cmd; -} - -function formatGlobError(searchRoot: string, stderr: string): string { - const trimmed = stderr.trim(); - if (/no such file or directory/i.test(trimmed)) { - return `${searchRoot} does not exist`; - } - return trimmed.length > 0 ? `Glob failed: ${trimmed}` : 'Glob failed'; -} - -function formatGlobWarning(stderr: string): string { - const trimmed = stderr.trim(); - return trimmed.length > 0 - ? `Glob completed with warnings; some directories could not be read: ${trimmed}` - : 'Glob completed with warnings; some directories could not be read.'; -} - function formatSpawnError(error: unknown, environment: Environment): string { return isHostFsNotFound(error) ? rgUnavailableMessage(error, environment) @@ -406,24 +224,4 @@ function formatSpawnError(error: unknown, environment: Environment): string { : String(error); } -export function splitCompletePaths(stdoutText: string, truncatedOutput: boolean): string[] { - let text = stdoutText; - if (truncatedOutput && !text.endsWith('\n')) { - const lastNewline = text.lastIndexOf('\n'); - text = lastNewline >= 0 ? text.slice(0, lastNewline + 1) : ''; - } - return text.split('\n').filter((p) => p.length > 0); -} - -function relativizeIfUnder(candidate: string, base: string, pathClass: PathClass): string { - const normCandidate = normalize(candidate); - const normBase = normalize(base); - const comparableCandidate = pathClass === 'win32' ? normCandidate.toLowerCase() : normCandidate; - const comparableBase = pathClass === 'win32' ? normBase.toLowerCase() : normBase; - if (comparableCandidate === comparableBase) return '.'; - const prefix = comparableBase.endsWith('/') ? comparableBase : comparableBase + '/'; - if (comparableCandidate.startsWith(prefix)) { - return normCandidate.slice(prefix.length); - } - return normCandidate; -} +export { splitCompletePaths } from '#/os/operations/searchGlob'; diff --git a/packages/agent-core-v2/src/agent/tools/os/grep/grepTool.ts b/packages/agent-core-v2/src/agent/tools/os/grep/grepTool.ts index d02dae75765..b3f4f7a0841 100644 --- a/packages/agent-core-v2/src/agent/tools/os/grep/grepTool.ts +++ b/packages/agent-core-v2/src/agent/tools/os/grep/grepTool.ts @@ -1,6 +1,5 @@ -import { normalize } from 'pathe'; +import { searchGrep } from '#/os/operations/searchGrep'; -import { ToolOutputAccumulator } from '#/tool/output-accumulator'; import { ToolAccesses, type ExecutableToolResult, @@ -20,9 +19,6 @@ import { ISessionSkillCatalog } from '#/features/skill/session/skillCatalog'; import { ISessionWorkspaceContext } from '#/session/workspaceContext/workspaceContext'; import { resolvePathAccessPath, - type PathClass, - isSensitiveFile, - SENSITIVE_DOT_VARIANT_SUFFIXES, type WorkspaceConfig, } from '#/tool/path-access'; import { toInputJsonSchema } from '#/tool/input-schema'; @@ -32,37 +28,10 @@ import { rgUnavailableMessage, type RgProbe, } from '#/os/backends/node-local/tools/rgLocator'; -import { - DEFAULT_TIMEOUT_MS, - MAX_OUTPUT_BYTES, - runRgOnce, - shouldRetryRipgrepEagain, - type RunRgResult, -} from '#/os/backends/node-local/tools/runRg'; + import GREP_DESCRIPTION from './grep.md?raw'; import { type GrepInput, GrepInputSchema, IGrepTool } from './grep'; -const RG_MAX_COLUMNS = 500; -const DEFAULT_HEAD_LIMIT = 250; - -const VCS_DIRECTORIES_TO_EXCLUDE = ['.git', '.svn', '.hg', '.bzr', '.jj', '.sl'] as const; -const SENSITIVE_KEY_BASENAMES = ['id_rsa', 'id_ed25519', 'id_ecdsa'] as const; -const SENSITIVE_KEY_GLOBS_TO_EXCLUDE = SENSITIVE_KEY_BASENAMES.flatMap((name) => [ - `**/${name}`, - `**/${name}[-_]*`, - ...SENSITIVE_DOT_VARIANT_SUFFIXES.map((suffix) => `**/${name}${suffix}`), -]); -const SENSITIVE_GLOBS_TO_EXCLUDE = [ - '**/.env', - ...SENSITIVE_KEY_GLOBS_TO_EXCLUDE, - '**/.aws/credentials', - '**/.aws/credentials/**', - '**/.gcp/credentials', - '**/.gcp/credentials/**', -] as const; - -const CONTENT_LINE_RE = /^(.*?)([:-])(\d+)\2/; - export class GrepTool implements IGrepTool { declare readonly _serviceBrand: undefined; readonly name = 'Grep' as const; @@ -142,7 +111,6 @@ export class GrepTool implements IGrepTool { return { isError: true, output: 'Aborted before search started' }; } - const pathClass = env.pathClass; let rgPath: string; try { const resolution = await ensureRgPath(this.createRgProbe(processService, workspace.workspaceDir), { @@ -166,135 +134,15 @@ export class GrepTool implements IGrepTool { return { isError: true, output: rgUnavailableMessage(error, environment) }; } - let runResult: RunRgResult; try { - const firstRun = await runRgOnce( - processService, - buildRgArgs(rgPath, args, searchPaths), - signal, - ); - if (firstRun.kind === 'aborted') { - return { isError: true, output: 'Grep aborted' }; - } - runResult = firstRun; - - if (shouldRetryRipgrepEagain(runResult)) { - const retryRun = await runRgOnce( - processService, - buildRgArgs(rgPath, args, searchPaths, true), - signal, - ); - if (retryRun.kind === 'aborted') { - return { isError: true, output: 'Grep aborted' }; - } - runResult = retryRun; - } + const request = { kind: 'grep' as const, args, rgPath, roots: searchPaths, workspaceDir: workspace.workspaceDir, pathClass: env.pathClass }; + return await (processService.search === undefined + ? searchGrep(processService, request, signal) + : processService.search(request, signal)); } catch (error) { return { isError: true, output: formatSpawnError(error, environment) }; } - - const { exitCode, stderrText, bufferTruncated, stderrTruncated, timedOut } = runResult; - let { stdoutText } = runResult; - - if (exitCode !== 0 && exitCode !== 1 && !timedOut) { - return { - isError: true, - output: formatRipgrepError(exitCode, stderrText, stderrTruncated), - }; - } - - const mode = args.output_mode ?? 'files_with_matches'; - if (bufferTruncated || timedOut) { - stdoutText = omitIncompleteTrailingRecord(stdoutText, mode); - } - if (timedOut && stdoutText.trim() === '') { - return { - isError: true, - output: `Grep timed out after ${String(DEFAULT_TIMEOUT_MS / 1000)}s. Try a more specific path or pattern.`, - }; - } - if (signal.aborted) { - return { isError: true, output: 'Grep aborted' }; - } - - const rawLines = parseRipgrepOutput(stdoutText, mode); - - const filteredSensitive = new Set(); - const keptLines = filterSensitiveLines(rawLines, mode, filteredSensitive, pathClass); - const orderedLines = keptLines; - - const offset = args.offset ?? 0; - const headLimit = args.head_limit ?? DEFAULT_HEAD_LIMIT; - const afterOffset = offset > 0 ? orderedLines.slice(offset) : orderedLines; - const limitActive = headLimit > 0; - const limited = limitActive ? afterOffset.slice(0, headLimit) : afterOffset; - const paginationTruncated = limitActive && afterOffset.length > headLimit; - - const headerLines: string[] = []; - const messages: string[] = []; - if (filteredSensitive.size > 0) { - const displayedFilteredPaths = [...filteredSensitive].map((path) => - relativizeIfUnder(path, workspace.workspaceDir, pathClass), - ); - messages.push( - `Filtered ${String(filteredSensitive.size)} sensitive file(s): ${displayedFilteredPaths.join(', ')}`, - ); - } - if (mode === 'count_matches' && orderedLines.length > 0) { - headerLines.push(formatCountSummary(orderedLines, filteredSensitive.size > 0)); - } - if (paginationTruncated) { - const total = afterOffset.length + offset; - const nextOffset = offset + headLimit; - const paginationNotice = - bufferTruncated || timedOut - ? `Results truncated to ${String(headLimit)} lines (total: ${String(total)} of a partial result set). Use offset=${String(nextOffset)} to see more.` - : `Results truncated to ${String(headLimit)} lines (total: ${String(total)}). Use offset=${String(nextOffset)} to see more.`; - if (mode === 'count_matches') { - headerLines.push(paginationNotice); - } else { - messages.push(paginationNotice); - } - } - if (bufferTruncated) { - messages.push( - `[Output truncated at ${String(MAX_OUTPUT_BYTES)} bytes of rg output — the result set is incomplete. Narrow the pattern, path, or glob filters and re-run to recover complete results.]`, - ); - } - if (timedOut) { - messages.push( - `Grep timed out after ${String(DEFAULT_TIMEOUT_MS / 1000)}s; partial results returned. Narrow the path, glob, or pattern and retry for complete results.`, - ); - } - - const contentIncludesLineNumbers = mode === 'content' && args['-n'] !== false; - const displayedLines = limited.map((line) => - formatDisplayLine( - line, - mode, - workspace.workspaceDir, - pathClass, - contentIncludesLineNumbers, - ), - ); - const contentBody = displayedLines.join('\n'); - const visibleBody = - orderedLines.length === 0 && filteredSensitive.size > 0 - ? 'No non-sensitive matches found' - : contentBody; - const emptyResultMessage = - SENSITIVE_GLOBS_TO_EXCLUDE.length > 0 ? 'No non-sensitive matches found' : 'No matches found'; - const body = - visibleBody === '' && headerLines.length === 0 && messages.length === 0 - ? emptyResultMessage - : visibleBody; - const combined = [...headerLines, body, ...messages].filter((part) => part !== '').join('\n'); - - const builder = new ToolOutputAccumulator(); - builder.write(combined); - return builder.ok(); } - private createRgProbe(processService: IHostProcessService, cwd: string): RgProbe { return { exec: async (args) => { @@ -332,347 +180,3 @@ function formatSpawnError(error: unknown, environment: Environment): string { ? error.message : String(error); } - -type GrepMode = 'content' | 'files_with_matches' | 'count_matches'; - -type ParsedGrepLine = - | { - readonly kind: 'record'; - readonly filePath: string; - readonly payload: string; - } - | { - readonly kind: 'separator'; - } - | { - readonly kind: 'legacy'; - readonly text: string; - }; - -function buildRgArgs( - rgPath: string, - args: GrepInput, - searchPaths: readonly string[], - singleThreaded = false, -): string[] { - const cmd: string[] = [rgPath]; - if (singleThreaded) cmd.push('-j', '1'); - cmd.push('--hidden'); - const mode = args.output_mode ?? 'files_with_matches'; - if (mode !== 'content') { - cmd.push('--max-columns', String(RG_MAX_COLUMNS)); - } - cmd.push('--null'); - for (const dir of VCS_DIRECTORIES_TO_EXCLUDE) { - cmd.push('--glob', `!${dir}`); - } - - if (mode === 'files_with_matches') cmd.push('-l', '--sortr=modified'); - else if (mode === 'count_matches') { - cmd.push('--count-matches', '--with-filename'); - } - - if (args['-i']) cmd.push('-i'); - if (mode === 'content') { - cmd.push('--with-filename'); - if (args['-n'] !== false) { - cmd.push('-n'); - } else { - cmd.push('--field-context-separator', ':'); - } - if (args['-C'] !== undefined) { - cmd.push('-C', String(args['-C'])); - } else { - if (args['-A'] !== undefined) cmd.push('-A', String(args['-A'])); - if (args['-B'] !== undefined) cmd.push('-B', String(args['-B'])); - } - } - if (args.glob !== undefined) cmd.push('--glob', args.glob); - if (args.type !== undefined) cmd.push('--type', args.type); - if (args.multiline) cmd.push('-U', '--multiline-dotall'); - if (args.include_ignored) cmd.push('--no-ignore'); - for (const glob of SENSITIVE_GLOBS_TO_EXCLUDE) { - cmd.push('--glob', `!${glob}`); - } - - cmd.push('--', args.pattern, ...searchPaths); - return cmd; -} - -function splitRgLines(text: string): string[] { - if (text === '') return []; - const lines = text.split('\n'); - while (lines.length > 0 && lines.at(-1) === '') { - lines.pop(); - } - return lines.map((line) => stripTrailingCarriageReturn(line)); -} - -function parseRipgrepOutput(text: string, mode: GrepMode): ParsedGrepLine[] { - if (text === '') return []; - if (!text.includes('\0')) { - return splitRgLines(text).map((line) => - mode === 'content' && line === '--' ? { kind: 'separator' } : { kind: 'legacy', text: line }, - ); - } - - if (mode === 'files_with_matches') { - return text - .split('\0') - .map((filePath) => stripTrailingCarriageReturn(filePath)) - .filter((filePath) => filePath !== '') - .map((filePath) => ({ kind: 'record', filePath, payload: '' })); - } - - const records: ParsedGrepLine[] = []; - let cursor = 0; - while (cursor < text.length) { - if (text[cursor] === '\n') { - cursor += 1; - continue; - } - if (text.startsWith('--\r\n', cursor)) { - records.push({ kind: 'separator' }); - cursor += 4; - continue; - } - if (text.startsWith('--\n', cursor)) { - records.push({ kind: 'separator' }); - cursor += 3; - continue; - } - - const nulIndex = text.indexOf('\0', cursor); - if (nulIndex < 0) { - const tail = stripTrailingCarriageReturn(text.slice(cursor)); - if (tail !== '') records.push({ kind: 'legacy', text: tail }); - break; - } - - const lineEnd = text.indexOf('\n', nulIndex + 1); - const payloadEnd = lineEnd >= 0 ? lineEnd : text.length; - const filePath = text.slice(cursor, nulIndex); - const payload = stripTrailingCarriageReturn(text.slice(nulIndex + 1, payloadEnd)); - records.push({ kind: 'record', filePath, payload }); - cursor = lineEnd >= 0 ? lineEnd + 1 : text.length; - } - return records; -} - -function formatDisplayLine( - line: ParsedGrepLine, - mode: GrepMode, - workspaceDir: string, - pathClass: PathClass, - contentIncludesLineNumbers: boolean, -): string { - if (line.kind === 'separator') return '--'; - if (line.kind === 'record') { - const displayPath = relativizeIfUnder(line.filePath, workspaceDir, pathClass); - if (mode === 'files_with_matches') return displayPath; - if (mode === 'count_matches') return `${displayPath}:${line.payload}`; - const separator = contentIncludesLineNumbers ? contentPayloadPathSeparator(line.payload) : ':'; - return `${displayPath}${separator}${line.payload}`; - } - - const text = line.text; - if (mode === 'files_with_matches') { - return relativizeIfUnder(text, workspaceDir, pathClass); - } - if (mode === 'count_matches') { - const idx = text.lastIndexOf(':'); - if (idx <= 0) return text; - return relativizeIfUnder(text.slice(0, idx), workspaceDir, pathClass) + text.slice(idx); - } - - const filePath = extractContentFilePath(text, pathClass); - if (filePath !== undefined) { - return relativizeIfUnder(filePath, workspaceDir, pathClass) + text.slice(filePath.length); - } - return text; -} - -function relativizeIfUnder(candidate: string, base: string, pathClass: PathClass): string { - const normCandidate = normalize(candidate); - const normBase = normalize(base); - const comparableCandidate = pathClass === 'win32' ? normCandidate.toLowerCase() : normCandidate; - const comparableBase = pathClass === 'win32' ? normBase.toLowerCase() : normBase; - if (comparableCandidate === comparableBase) return '.'; - const prefix = comparableBase.endsWith('/') ? comparableBase : comparableBase + '/'; - if (comparableCandidate.startsWith(prefix)) { - return normCandidate.slice(prefix.length); - } - return normCandidate; -} - -function omitIncompleteTrailingRecord(text: string, mode: GrepMode): string { - if (!text.includes('\0')) return omitIncompleteTrailingLine(text); - if (mode === 'files_with_matches') { - const lastNul = text.lastIndexOf('\0'); - return lastNul >= 0 ? text.slice(0, lastNul + 1) : ''; - } - - let cursor = 0; - let lastCompleteEnd = 0; - while (cursor < text.length) { - if (text[cursor] === '\n') { - cursor += 1; - lastCompleteEnd = cursor; - continue; - } - if (text.startsWith('--\r\n', cursor)) { - cursor += 4; - lastCompleteEnd = cursor; - continue; - } - if (text.startsWith('--\n', cursor)) { - cursor += 3; - lastCompleteEnd = cursor; - continue; - } - - const nulIndex = text.indexOf('\0', cursor); - if (nulIndex < 0) break; - const lineEnd = text.indexOf('\n', nulIndex + 1); - if (lineEnd < 0) break; - cursor = lineEnd + 1; - lastCompleteEnd = cursor; - } - return text.slice(0, lastCompleteEnd); -} - -function omitIncompleteTrailingLine(text: string): string { - const lastNewline = text.lastIndexOf('\n'); - return lastNewline >= 0 ? text.slice(0, lastNewline) : ''; -} - -function formatRipgrepError( - exitCode: number, - stderrText: string, - stderrTruncated: boolean, -): string { - const stderr = stderrText.trim(); - if (stderr.length === 0) { - return `Failed to grep: ripgrep exited with code ${String(exitCode)}`; - } - - const summary = summarizeRipgrepStderr(stderr); - const lines = [`Failed to grep: ${summary}`, '', 'ripgrep stderr:', stderr]; - if (stderrTruncated) { - lines.push(`[stderr truncated at ${String(MAX_OUTPUT_BYTES)} bytes]`); - } - return lines.join('\n'); -} - -function summarizeRipgrepStderr(stderr: string): string { - const lines = splitRgLines(stderr) - .map((line) => line.trim()) - .filter((line) => line.length > 0); - const errorLine = lines.findLast((line) => line.toLowerCase().startsWith('error:')); - return errorLine ?? lines.at(-1) ?? 'ripgrep error'; -} - -function filterSensitiveLines( - lines: readonly ParsedGrepLine[], - mode: GrepMode, - filteredPaths: Set, - pathClass: PathClass, -): ParsedGrepLine[] { - const kept: ParsedGrepLine[] = []; - for (const line of lines) { - if (line.kind === 'separator') { - kept.push(line); - continue; - } - const filePath = parsedFilePath(line, mode, pathClass); - if (filePath !== undefined && isSensitiveFile(filePath)) { - filteredPaths.add(filePath); - continue; - } - kept.push(line); - } - return mode === 'content' ? normalizeContextSeparators(kept) : kept; -} - -function normalizeContextSeparators(lines: readonly ParsedGrepLine[]): ParsedGrepLine[] { - const normalized: ParsedGrepLine[] = []; - for (const line of lines) { - if ( - line.kind === 'separator' && - (normalized.length === 0 || normalized.at(-1)?.kind === 'separator') - ) { - continue; - } - normalized.push(line); - } - while (normalized.length > 0 && normalized.at(-1)?.kind === 'separator') { - normalized.pop(); - } - return normalized; -} - -function parsedFilePath( - line: ParsedGrepLine, - mode: GrepMode, - pathClass: PathClass, -): string | undefined { - if (line.kind === 'record') return normalize(line.filePath); - if (line.kind === 'separator') return undefined; - const text = line.text; - if (mode === 'files_with_matches') return normalize(text); - if (mode === 'count_matches') { - const idx = text.lastIndexOf(':'); - return idx > 0 ? normalize(text.slice(0, idx)) : normalize(text); - } - return extractContentFilePath(text, pathClass); -} - -function extractContentFilePath(line: string, pathClass: PathClass): string | undefined { - const m = CONTENT_LINE_RE.exec(line); - if (m?.[1] !== undefined) return normalize(m[1]); - - const separatorIndex = noLineNumberContentSeparatorIndex(line, pathClass); - return separatorIndex > 0 ? normalize(line.slice(0, separatorIndex)) : undefined; -} - -function noLineNumberContentSeparatorIndex(line: string, pathClass: PathClass): number { - const searchFrom = pathClass === 'win32' && /^[A-Za-z]:/.test(line) ? 2 : 0; - return line.indexOf(':', searchFrom); -} - -function contentPayloadPathSeparator(payload: string): ':' | '-' { - const m = /^(\d+)([:-])/.exec(payload); - return m?.[2] === '-' ? '-' : ':'; -} - -function stripTrailingCarriageReturn(value: string): string { - return value.endsWith('\r') ? value.slice(0, -1) : value; -} - -function formatCountSummary(lines: readonly ParsedGrepLine[], redactedSensitive: boolean): string { - let totalMatches = 0; - let totalFiles = 0; - for (const line of lines) { - const rawCount = - line.kind === 'record' - ? line.payload - : line.kind === 'legacy' - ? countPayloadFromLegacyLine(line.text) - : undefined; - if (rawCount === undefined) continue; - const count = Number(rawCount); - if (!Number.isSafeInteger(count) || count < 0) continue; - totalMatches += count; - totalFiles++; - } - - const occurrenceWord = totalMatches === 1 ? 'occurrence' : 'occurrences'; - const fileWord = totalFiles === 1 ? 'file' : 'files'; - const scope = redactedSensitive ? 'total non-sensitive' : 'total'; - return `Found ${String(totalMatches)} ${scope} ${occurrenceWord} across ${String(totalFiles)} ${fileWord}.`; -} - -function countPayloadFromLegacyLine(line: string): string | undefined { - const idx = line.lastIndexOf(':'); - return idx > 0 ? line.slice(idx + 1) : undefined; -} diff --git a/packages/agent-core-v2/src/agent/tools/os/read/read.ts b/packages/agent-core-v2/src/agent/tools/os/read/read.ts index 060f82571c8..3650ebba17b 100644 --- a/packages/agent-core-v2/src/agent/tools/os/read/read.ts +++ b/packages/agent-core-v2/src/agent/tools/os/read/read.ts @@ -6,7 +6,7 @@ import { type AgentTool } from '#/tool/toolContract'; export const DEFAULT_MAX_CHARS = 100_000; export const DEFAULT_MAX_CHARS_LIMIT = 500_000; -export const TRANSCODE_MAX_BYTES: number = 10 * 1024 * 1024; +export { TRANSCODE_MAX_BYTES } from '#/os/operations/readTextPage'; const PositiveLineOffsetSchema = z.number().int().min(1); const TailLineOffsetSchema = z.number().int().negative(); diff --git a/packages/agent-core-v2/src/agent/tools/os/read/readTool.ts b/packages/agent-core-v2/src/agent/tools/os/read/readTool.ts index a37c494bf98..5a8d75258af 100644 --- a/packages/agent-core-v2/src/agent/tools/os/read/readTool.ts +++ b/packages/agent-core-v2/src/agent/tools/os/read/readTool.ts @@ -1,3 +1,4 @@ +import { readTextPage } from '#/os/operations/readTextPage'; import * as posixPath from 'node:path/posix'; import * as win32Path from 'node:path/win32'; @@ -6,7 +7,6 @@ import { ISessionContext } from '#/session/sessionContext/sessionContext'; import { IAgentScopeContext } from '#/agent/scopeContext/scopeContext'; import { isTaskOutputFilePath } from '#/agent/task/persist'; import { IHostFileSystem } from '#/os/interface/hostFileSystem'; -import { isHostFsNotDirectory, isHostFsNotFound } from '#/os/interface/hostFsErrors'; import { acquireOrWhenReady, IAgentEnvironmentService, pinnedGeneration } from '#/agent/environmentBinding/agentEnvironment'; import { ISessionMediaStore } from '#/agent/media/sessionMediaStore'; import { IAgentProfileService } from '#/agent/profile/profile'; @@ -16,11 +16,9 @@ import { IAgentToolRegistryService } from '#/agent/toolRegistry/toolRegistry'; import { isDaemonFileUrl } from '#/agent/media/mediaRef'; import { attachmentFileSource, environmentFileSource, withAttachmentLocation, type FileReadSource } from '#/agent/tools/fileReadSource'; import { EnvironmentWorkspaceView } from '#/environment/environmentWorkspaceView'; -import { unwrapErrorCause } from '#/_base/errors/errors'; import { ISessionSkillCatalog } from '#/features/skill/session/skillCatalog'; import { ISessionWorkspaceContext } from '#/session/workspaceContext/workspaceContext'; import { IConfigService } from '#/app/config/config'; -import { renderToolResultForModel } from '#/agent/contextMemory/toolResultRender'; import { ToolAccesses, type ExecutableToolResult, @@ -31,145 +29,20 @@ import { resolvePathAccessPath, type WorkspaceConfig, } from '#/tool/path-access'; -import { MEDIA_SNIFF_BYTES, detectFileType } from '#/agent/media/file-type'; import { toInputJsonSchema } from '#/tool/input-schema'; import { literalRulePattern, matchesGlobRuleSubject, matchesPathRuleSubject } from '#/tool/rule-match'; -import { makeCarriageReturnsVisible, splitLinesKeepingTerminator, type LineEndingStyle } from '#/_base/text/line-endings'; -import { detectTextEncoding, type UtfTextEncoding } from '#/_base/text/encoding'; import { renderPrompt } from '#/_base/utils/render-prompt'; import { DEFAULT_MAX_CHARS, DEFAULT_MAX_CHARS_LIMIT, IReadTool, ReadInputSchema, - TRANSCODE_MAX_BYTES, type ReadInput, } from './read'; import { IAgentToolResultTruncationService } from '#/agent/toolResultTruncation/toolResultTruncation'; import { READ_SECTION, type ReadConfig } from './configSection'; import readDescriptionTemplate from './read.md?raw'; -interface LineEndingFlags { - hasCrLf: boolean; - hasLf: boolean; - hasLoneCr: boolean; -} - -interface ReadLineEntry { - readonly lineNo: number; - readonly rawContent: string; -} - -interface ReadTailEntry extends ReadLineEntry { - readonly minChars: number; -} - -interface ReadRequest { - readonly args: ReadInput; - readonly maxChars: number; - readonly maxCharsLimit: number; - readonly detectedEncoding?: UtfTextEncoding; - readonly lossyDecoding: boolean; - readonly eventLog: boolean; -} - -interface ReadPage { - readonly request: ReadRequest; - readonly renderedLines: readonly string[]; - readonly startLine: number; - readonly rangeStart: number; - readonly rangeEnd: number; - readonly totalLines: number; - readonly fromTail: boolean; - readonly lineEndingStyle: LineEndingStyle; -} - -function stripTrailingLf(line: string): string { - return line.endsWith('\n') ? line.slice(0, -1) : line; -} - -function splitsSurrogatePair(text: string, offset: number): boolean { - // oxlint-disable-next-line unicorn/prefer-code-point -- raw UTF-16 halves are the point here - const previous = text.charCodeAt(offset - 1); - // oxlint-disable-next-line unicorn/prefer-code-point -- raw UTF-16 halves are the point here - const next = text.charCodeAt(offset); - return previous >= 0xd800 && previous <= 0xdbff && next >= 0xdc00 && next <= 0xdfff; -} - -function updateLineEndingFlags(flags: LineEndingFlags, text: string): void { - for (let i = 0; i < text.length; i += 1) { - const code = text.codePointAt(i); - if (code === 13) { - if (text.codePointAt(i + 1) === 10) { - flags.hasCrLf = true; - i += 1; - } else { - flags.hasLoneCr = true; - } - } else if (code === 10) { - flags.hasLf = true; - } - } -} - -function lineEndingStyleFromFlags(flags: LineEndingFlags): LineEndingStyle { - if (flags.hasLoneCr || (flags.hasCrLf && flags.hasLf)) return 'mixed'; - if (flags.hasCrLf) return 'crlf'; - return 'lf'; -} - -function renderLine(entry: ReadLineEntry, lineEndingStyle: LineEndingStyle): string { - const modelContent = - lineEndingStyle === 'crlf' && entry.rawContent.endsWith('\r') - ? entry.rawContent.slice(0, -1) - : entry.rawContent; - const renderedContent = - lineEndingStyle === 'mixed' ? makeCarriageReturnsVisible(modelContent) : modelContent; - return `${String(entry.lineNo)}\t${renderedContent}`; -} - -function isTextDecodeError(error: unknown): boolean { - const unwrapped = unwrapErrorCause(error); - if (typeof unwrapped !== 'object' || unwrapped === null) return false; - const code = (unwrapped as { code?: unknown })['code']; - if (code === 'ERR_ENCODING_INVALID_ENCODED_DATA') return true; - if (!(unwrapped instanceof Error)) return false; - return /encoded data was not valid|invalid.*encoding|invalid.*utf-?8/i.test(unwrapped.message); -} - -function containsNulByte(text: string): boolean { - return text.includes('\u0000'); -} - -function encodingDisplayName(encoding: UtfTextEncoding): string { - switch (encoding) { - case 'utf-16le': - return 'UTF-16 LE'; - case 'utf-16be': - return 'UTF-16 BE'; - default: - return 'UTF-8'; - } -} - -async function* decodedLines(lines: readonly string[]): AsyncGenerator { - yield* lines; -} - -const READ_MEDIA_FILE_TOOL_NAME = 'ReadMediaFile'; - -function notReadableFileOutput(path: string): string { - return `"${path}" is not readable as UTF-8 text. Only text files can be read.`; -} - -function notUtf8DecodableFileOutput(path: string): string { - return ( - `"${path}" is not valid UTF-8 or UTF-16 text. ` + - 'Only UTF-8 and UTF-16 text files can be read; ' + - 'for other encodings (e.g. GBK), convert the file to UTF-8 first (e.g. with `iconv`).' - ); -} - export class ReadTool implements IReadTool { declare readonly _serviceBrand: undefined; readonly name = 'Read' as const; @@ -246,14 +119,14 @@ export class ReadTool implements IReadTool { pathClass: env.pathClass, homeDir: env.homeDir, }), - execute: async () => { + execute: async ({ signal }) => { const lease = await acquireOrWhenReady(this.environment, ['fs']); try { if (expectedGeneration !== undefined && lease.environment.identity.generation !== expectedGeneration) { return { isError: true, output: 'Environment changed before execution. Retry the tool call.' }; } const eventLog = this.resultTruncation.isWireJournalPath(path); - const result = await this.execution(environmentFileSource(lease.environment.fs!, path), args, eventLog); + const result = await this.execution(environmentFileSource(lease.environment.fs!, path), args, eventLog, signal); return { ...result, spillExempt: true }; } finally { lease.dispose(); @@ -288,11 +161,11 @@ export class ReadTool implements IReadTool { matchesRule: (ruleArgs) => matchesPathRuleSubject(ruleArgs, path, { cwd: target.environmentPath.dirname(path), pathClass: target.pathClass, }), - execute: async () => { + execute: async ({ signal }) => { if (planFile && (await this.planService?.resolveFile(path, 'read'))?.path !== path) { return { isError: true, output: 'The active plan changed before execution. Retry the tool call.' }; } - return { ...await this.execution(environmentFileSource(target.fs, path), args, false), spillExempt: true }; + return { ...await this.execution(environmentFileSource(target.fs, path), args, false, signal), spillExempt: true }; }, }; } @@ -305,366 +178,26 @@ export class ReadTool implements IReadTool { display: { kind: 'file_io', operation: 'read', path: source.localPath ?? args.path }, approvalRule: literalRulePattern(this.name, args.path), matchesRule: (ruleArgs) => matchesGlobRuleSubject(ruleArgs, args.path), - execute: async () => ({ - ...withAttachmentLocation(await this.execution(source, args, false), source), + execute: async ({ signal }) => ({ + ...withAttachmentLocation(await this.execution(source, args, false, signal), source), spillExempt: true, }), }; } - private async execution( - source: FileReadSource, - args: ReadInput, - eventLog: boolean, - ): Promise { - try { - let stat: Awaited>; - try { - stat = await source.stat(); - } catch (error) { - if (isHostFsNotFound(error) || isHostFsNotDirectory(error)) { - return { isError: true, output: `"${args.path}" does not exist.` }; - } - throw error; - } - if (!stat.isFile) { - return { isError: true, output: `"${args.path}" is not a file.` }; - } - - const header = await source.readBytes(MEDIA_SNIFF_BYTES); - const fileType = detectFileType(source.name, header); - if (fileType.kind === 'image' || fileType.kind === 'video') { - const kind = fileType.kind; - const article = kind === 'image' ? 'an' : 'a'; - const capabilities = this.profile.getModelCapabilities(); - const supported = kind === 'image' ? capabilities.image_in : capabilities.video_in; - if (!supported) { - return { - isError: true, - output: `"${args.path}" is ${article} ${kind} file. The current model does not support ${kind} input (missing ${kind}_in capability), so this agent cannot view it. Only text files can be read.`, - }; - } - const mediaToolAvailable = - this.toolRegistry.resolve(READ_MEDIA_FILE_TOOL_NAME) !== undefined && - this.toolPolicy.isToolActive(READ_MEDIA_FILE_TOOL_NAME); - return { - isError: true, - output: mediaToolAvailable - ? `"${args.path}" is ${article} ${kind} file. Only text files can be read; use ReadMediaFile for image and video files.` - : `"${args.path}" is ${article} ${kind} file. Only text files can be read.`, - }; - } - - const detection = detectTextEncoding(header); - let readLines: () => AsyncIterable; - let detectedEncoding: UtfTextEncoding | undefined; - let lossyDecoding = false; - if (!detection.seemsBinary && detection.encoding !== 'utf-8') { - if (stat.size > TRANSCODE_MAX_BYTES) { - return { - isError: true, - output: - `"${args.path}" is ${encodingDisplayName(detection.encoding)} text but too large to transcode ` + - `(${String(stat.size)} bytes > ${String(TRANSCODE_MAX_BYTES)}). ` + - 'Convert it to UTF-8 first (e.g. with `iconv`).', - }; - } - const bytes = await source.readBytes(); - let decoded: string; - try { - decoded = new TextDecoder(detection.encoding, { fatal: true }).decode(bytes); - } catch (error) { - if (!isTextDecodeError(error)) throw error; - decoded = new TextDecoder(detection.encoding, { fatal: false }).decode(bytes); - lossyDecoding = true; - } - detectedEncoding = detection.encoding; - const decodedContent = splitLinesKeepingTerminator(decoded); - readLines = () => decodedLines(decodedContent); - } else if (fileType.kind === 'unknown') { - return { - isError: true, - output: notReadableFileOutput(args.path), - }; - } else { - readLines = () => source.readLines(); - } - - const limits = this.limits(); - const request: ReadRequest = { - args, - maxChars: Math.min(args.max_chars ?? limits.defaultMaxChars, limits.maxChars), - maxCharsLimit: limits.maxChars, - detectedEncoding, - lossyDecoding, - eventLog, - }; - const lineOffset = args.line_offset ?? 1; - if (lineOffset >= 0) return await this.readForward(readLines(), request); - const rereadsFile = detectedEncoding === undefined && (args.n_lines ?? Infinity) < -lineOffset; - const result = await this.readTail(readLines, request); - if (!result.isError && rereadsFile) { - const currentStat = await source.stat(); - if (!currentStat.isFile || currentStat.size !== stat.size || - currentStat.mtimeMs !== stat.mtimeMs || currentStat.ino !== stat.ino) { - return { isError: true, output: 'File changed while reading its tail. Retry Read with the updated file.' }; - } - } - return result; - } catch (error) { - if (isTextDecodeError(error)) { - return { isError: true, output: notUtf8DecodableFileOutput(args.path) }; - } - return { - isError: true, - output: error instanceof Error ? error.message : String(error), - }; - } - } - - private async readForward( - lines: AsyncIterable, - request: ReadRequest, - ): Promise { - const { args, maxChars } = request; - const lineOffset = args.line_offset ?? 1; - const columnOffset = args.column_offset ?? 0; - const requestedLines = args.n_lines ?? Infinity; - const selectedEntries: ReadLineEntry[] = []; - const flags: LineEndingFlags = { hasCrLf: false, hasLf: false, hasLoneCr: false }; - let currentLineNo = 0; - let collectionClosed = false; - let minimumChars = 0; - - for await (const rawLine of lines) { - if (containsNulByte(rawLine)) { - return { isError: true, output: notReadableFileOutput(args.path) }; - } - currentLineNo += 1; - updateLineEndingFlags(flags, rawLine); - if (collectionClosed) continue; - if (currentLineNo < lineOffset) continue; - if (selectedEntries.length >= requestedLines) { - collectionClosed = true; - continue; - } - const rawContent = stripTrailingLf(rawLine); - const lineChars = String(currentLineNo).length + 1 + Math.max( - 0, - rawContent.length - (rawContent.endsWith('\r') ? 1 : 0) - - (currentLineNo === lineOffset ? columnOffset : 0), - ) + (selectedEntries.length === 0 ? 0 : 1); - if (minimumChars + lineChars > maxChars && selectedEntries.length > 0) { - collectionClosed = true; - continue; - } - selectedEntries.push({ - lineNo: currentLineNo, - rawContent, - }); - minimumChars += lineChars; - if (selectedEntries.length >= requestedLines || minimumChars >= maxChars) { - collectionClosed = true; - } - } - - const lineEndingStyle = lineEndingStyleFromFlags(flags); - const renderedLines = selectedEntries.map((entry) => renderLine(entry, lineEndingStyle)); - const firstLine = renderedLines[0]; - if (columnOffset > 0) { - const prefix = `${String(lineOffset)}\t`; - const text = firstLine?.slice(prefix.length); - if (text === undefined || columnOffset > text.length) { - return { isError: true, output: `column_offset=${String(columnOffset)} is past the end of the starting line ${String(lineOffset)}. Read the line from column 0 to inspect its current contents.` }; - } - if (splitsSurrogatePair(text, columnOffset)) { - return { isError: true, output: `column_offset=${String(columnOffset)} splits a Unicode character in line ${String(lineOffset)}. Use a character boundary or the Next Read arguments.` }; - } - renderedLines[0] = prefix + text.slice(columnOffset); - } - return this.finishPage({ - request, - renderedLines, - startLine: lineOffset, - rangeStart: lineOffset, - rangeEnd: Math.min(currentLineNo, lineOffset + requestedLines - 1), - totalLines: currentLineNo, - fromTail: false, - lineEndingStyle, - }); - } - - private finishPage(page: ReadPage): ExecutableToolResult { - const { args, maxChars, maxCharsLimit, eventLog, detectedEncoding, lossyDecoding } = page.request; - let first = 0; - let end = page.renderedLines.length; - let contentChars = page.renderedLines.reduce((sum, line) => sum + line.length + 1, -1); - const firstColumn = page.fromTail ? 0 : args.column_offset ?? 0; - const firstPrefix = `${String(page.startLine)}\t`; - const firstText = page.renderedLines[0]?.slice(firstPrefix.length) ?? ''; - let fragmentEnd: number | undefined; - - while (true) { - const count = end - first; - const startLine = page.startLine + first; - const endLine = page.startLine + end - 1; - const lineIncomplete = fragmentEnd !== undefined; - const complete = page.rangeStart > page.rangeEnd || - (count > 0 && startLine === page.rangeStart && endLine === page.rangeEnd && !lineIncomplete); - const parts = [ - count > 0 - ? `${String(count)} ${count === 1 ? 'line' : 'lines'} read from file starting from line ${String(startLine)}.` - : 'No lines read from file.', - `Total lines in file: ${String(page.totalLines)}.`, - complete ? 'Requested range complete.' : 'Character limit reached.', - `Effective max_chars: ${String(maxChars)}.`, - ]; - if (!lineIncomplete && ((count > 0 && endLine === page.totalLines) || page.rangeStart > page.totalLines)) { - parts.push('End of file reached.'); - } - if (count > 0 && !page.fromTail && (firstColumn > 0 || fragmentEnd !== undefined)) { - parts.push(`Line ${String(startLine)} fragment: columns [${String(firstColumn)}, ${String(firstColumn + (fragmentEnd ?? firstText.length))}) of ${String(firstColumn + firstText.length)}. ${lineIncomplete ? 'Line continues.' : 'Line complete.'}`); - } - if (args.max_chars !== undefined && args.max_chars > maxCharsLimit) { - parts.push(`Requested max_chars=${String(args.max_chars)} was capped at the configured maximum ${String(maxCharsLimit)}.`); - } - if (!complete && (count > 0 || page.fromTail)) { - const nextStart = page.fromTail ? page.rangeStart : lineIncomplete ? startLine : endLine + 1; - const nextEnd = page.fromTail && count > 0 ? startLine - 1 : page.rangeEnd; - const next = { - path: args.path, - line_offset: nextStart, - column_offset: fragmentEnd !== undefined ? firstColumn + fragmentEnd : undefined, - n_lines: page.fromTail || args.n_lines !== undefined ? nextEnd - nextStart + 1 : undefined, - max_chars: maxChars, - }; - parts.push(`Next Read: ${JSON.stringify(next)}`); - } - if (eventLog) { - parts.push('Kimi Code agent event log: read one record at a time (n_lines=1); increase max_chars for a longer record or extract fields with Bash.'); - } - if (page.lineEndingStyle === 'mixed') { - parts.push('Mixed or lone carriage-return line endings are shown as \\r. Use exact \\r\\n or \\r escapes in Edit.old_string for those lines.'); - } - if (detectedEncoding !== undefined) { - parts.push(`Detected file encoding: ${encodingDisplayName(detectedEncoding)}; content transcoded to UTF-8 for display. Edit and Write expect UTF-8 — convert the file's encoding first (e.g. \`iconv\` via Bash).`); - } - if (lossyDecoding) { - parts.push('Lossy UTF-16 decoding: malformed sequences were replaced with U+FFFD. The decoded text may differ from the original file.'); - } - const note = `${parts.join(' ')}`; - const renderedChars = count === 0 - ? renderToolResultForModel({ output: '', note }).reduce( - (sum, part) => sum + (part.type === 'text' ? part.text.length : 0), - 0, - ) - : contentChars + 1 + note.length; - if (renderedChars <= maxChars && (complete || count > 0)) { - return { - output: fragmentEnd !== undefined - ? firstPrefix + firstText.slice(0, fragmentEnd) - : page.renderedLines.slice(first, end).join('\n'), - note, - truncated: complete ? undefined : true, - }; - } - if (count === 1 && !page.fromTail) { - const previousEnd = fragmentEnd ?? firstText.length; - fragmentEnd = Math.min(previousEnd - 1, previousEnd - (renderedChars - maxChars)); - if (splitsSurrogatePair(firstText, fragmentEnd)) fragmentEnd -= 1; - if (fragmentEnd <= 0) { - return { isError: true, output: `max_chars=${String(maxChars)} is too small for file text and the Read status. Increase max_chars.` }; - } - contentChars = firstPrefix.length + fragmentEnd; - continue; - } - if (count === 0) { - if (!complete) { - const recovery: ExecutableToolResult = { - isError: true, - output: 'No complete line fits. Continue with the forward Next Read.', - note, - truncated: true, - }; - const recoveryChars = renderToolResultForModel(recovery).reduce( - (sum, part) => sum + (part.type === 'text' ? part.text.length : 0), - 0, - ); - if (recoveryChars <= maxChars) return recovery; - } - return { isError: true, output: `max_chars=${String(maxChars)} is too small for the Read status. Increase max_chars.` }; - } - const dropped = page.fromTail ? first++ : --end; - contentChars -= page.renderedLines[dropped]!.length + 1; - } - } - - private async readTail( - readLines: () => AsyncIterable, - request: ReadRequest, - ): Promise { - const { args, maxChars } = request; - const lineOffset = args.line_offset ?? 1; - const requestedLines = args.n_lines ?? Infinity; - const tailCount = -lineOffset; - const singlePass = requestedLines >= tailCount; - let entries: (ReadTailEntry | undefined)[] = []; - let first = 0; - let chars = 0; - const retainLine = (rawLine: string, lineNo: number): void => { - const rawContent = stripTrailingLf(rawLine); - const minChars = String(lineNo).length + 2 + rawContent.length - (rawContent.endsWith('\r') ? 1 : 0); - entries.push({ lineNo, rawContent, minChars }); - chars += minChars; - while (first < entries.length && (chars - 1 > maxChars || entries.length - first > tailCount)) { - chars -= entries[first]!.minChars; - entries[first++] = undefined; - } - if (first > 1024 && first >= entries.length / 2) { - entries = entries.slice(first); - first = 0; - } + private async execution(source: FileReadSource, args: ReadInput, eventLog: boolean, signal: AbortSignal): Promise { + const limits = this.limits(); + const capabilities = this.profile.getModelCapabilities(); + const options = { + args, + maxChars: Math.min(args.max_chars ?? limits.defaultMaxChars, limits.maxChars), + maxCharsLimit: limits.maxChars, + eventLog, + imageSupported: capabilities.image_in, + videoSupported: capabilities.video_in, + mediaToolAvailable: this.toolRegistry.resolve('ReadMediaFile') !== undefined && this.toolPolicy.isToolActive('ReadMediaFile'), }; - const flags: LineEndingFlags = { hasCrLf: false, hasLf: false, hasLoneCr: false }; - let totalLines = 0; - for await (const rawLine of readLines()) { - if (containsNulByte(rawLine)) { - return { isError: true, output: notReadableFileOutput(args.path) }; - } - totalLines += 1; - updateLineEndingFlags(flags, rawLine); - if (singlePass) retainLine(rawLine, totalLines); - } - - const rangeStart = Math.max(1, totalLines + lineOffset + 1); - const rangeEnd = Math.min(totalLines, rangeStart + requestedLines - 1); - const lineEndingStyle = lineEndingStyleFromFlags(flags); - if (!singlePass) { - let currentLine = 0; - for await (const rawLine of readLines()) { - currentLine += 1; - if (currentLine > rangeEnd) break; - if (currentLine < rangeStart) continue; - if (containsNulByte(rawLine)) { - return { isError: true, output: notReadableFileOutput(args.path) }; - } - retainLine(rawLine, currentLine); - } - if (currentLine < rangeEnd || currentLine > totalLines) { - return { isError: true, output: 'File changed while reading its tail. Retry Read with the updated file.' }; - } - } - const selected = entries.slice(first).map((entry) => renderLine(entry!, lineEndingStyle)); - return this.finishPage({ - request, - renderedLines: selected, - startLine: rangeEnd - selected.length + 1, - rangeStart, - rangeEnd, - totalLines, - fromTail: true, - lineEndingStyle, - }); + return source.readTextPage === undefined ? readTextPage(source, options, signal) : source.readTextPage(options, signal); } } diff --git a/packages/agent-core-v2/src/agent/tools/os/write/writeTool.ts b/packages/agent-core-v2/src/agent/tools/os/write/writeTool.ts index 972419a4aac..5b3a73bdb7d 100644 --- a/packages/agent-core-v2/src/agent/tools/os/write/writeTool.ts +++ b/packages/agent-core-v2/src/agent/tools/os/write/writeTool.ts @@ -1,7 +1,7 @@ import { dirname } from 'pathe'; -import { type HostFileStat, type IHostFileSystem } from '#/os/interface/hostFileSystem'; -import { isHostFsNotFound } from '#/os/interface/hostFsErrors'; +import { type IHostFileSystem } from '#/os/interface/hostFileSystem'; +import { writeText } from '#/os/operations/writeText'; import { acquireOrWhenReady, IAgentEnvironmentService, pinnedGeneration } from '#/agent/environmentBinding/agentEnvironment'; import { IAgentPlanService } from '#/features/plan/plan'; import { EnvironmentWorkspaceView } from '#/environment/environmentWorkspaceView'; @@ -105,56 +105,10 @@ export class WriteTool implements IWriteTool { } private async execution(fs: IHostFileSystem, args: WriteInput, safePath: string, parent = dirname(safePath)): Promise { - const parentError = await this.ensureParentDirectory(fs, parent); - if (parentError !== undefined) { - return { isError: true, output: parentError }; - } - - try { - const mode = args.mode ?? 'overwrite'; - if (mode === 'append') { - await fs.appendText(safePath, args.content); - } else { - await fs.writeText(safePath, args.content); - } - const bytesWritten = Buffer.byteLength(args.content, 'utf8'); - return { - output: `${mode === 'append' ? 'Appended' : 'Wrote'} ${String(bytesWritten)} bytes to ${args.path}`, - }; - } catch (error) { - if (isHostFsNotFound(error)) { - return { - isError: true, - output: `Failed to write ${args.path}: parent directory does not exist.`, - }; - } - return { - isError: true, - output: error instanceof Error ? error.message : String(error), - }; - } + const input = { path: safePath, displayPath: args.path, parent, content: args.content, mode: args.mode ?? 'overwrite' as const }; + return fs.operations === undefined ? writeText(fs, input) : fs.operations.writeText(input); } - private async ensureParentDirectory(fs: IHostFileSystem, parent: string): Promise { - let stat: HostFileStat; - try { - stat = await fs.stat(parent); - } catch (error) { - if (isHostFsNotFound(error)) { - try { - await fs.mkdir(parent, { recursive: true }); - return undefined; - } catch (mkdirError) { - return mkdirError instanceof Error ? mkdirError.message : String(mkdirError); - } - } - return undefined; - } - if (!stat.isDirectory) { - return `Parent path is not a directory: ${parent}.`; - } - return undefined; - } } registerAgentToolService(IWriteTool, WriteTool, { diff --git a/packages/agent-core-v2/src/app/edit/editService.ts b/packages/agent-core-v2/src/app/edit/editService.ts index 3c261d4c9c6..2108d09ab0c 100644 --- a/packages/agent-core-v2/src/app/edit/editService.ts +++ b/packages/agent-core-v2/src/app/edit/editService.ts @@ -1,41 +1 @@ -import type { TextModel } from './textModel'; - -export interface EditApplyInput { - readonly path: string; - readonly old_string: string; - readonly new_string: string; - readonly replace_all: boolean; -} - -export type EditApplyResult = - | { readonly ok: true; readonly rawContent: string; readonly count: number } - | { readonly ok: false; readonly error: string }; - -function notFoundMessage(path: string): string { - return `old_string not found in ${path}, the file contents may be out of date. Please use the Read Tool to reload the content. -`; -} - -function notUniqueMessage(path: string, count: number): string { - return ( - `old_string is not unique in ${path} (found ${String(count)} occurrences). ` + - 'To replace every occurrence, set replace_all=true. To replace only one occurrence, include more surrounding context in old_string.' - ); -} - -export class EditService { - apply(model: TextModel, input: EditApplyInput): EditApplyResult { - if (input.replace_all) { - const { text, count } = model.replaceAll(input.old_string, input.new_string); - if (count === 0) return { ok: false, error: notFoundMessage(input.path) }; - return { ok: true, rawContent: model.materialize(text), count }; - } - - const count = model.countOccurrences(input.old_string); - if (count === 0) return { ok: false, error: notFoundMessage(input.path) }; - if (count > 1) return { ok: false, error: notUniqueMessage(input.path, count) }; - - const text = model.replaceOnce(input.old_string, input.new_string); - return { ok: true, rawContent: model.materialize(text), count: 1 }; - } -} +export * from '#/os/operations/editService'; diff --git a/packages/agent-core-v2/src/app/edit/fileEdit.ts b/packages/agent-core-v2/src/app/edit/fileEdit.ts index b2b10cc7274..a572bb98c0e 100644 --- a/packages/agent-core-v2/src/app/edit/fileEdit.ts +++ b/packages/agent-core-v2/src/app/edit/fileEdit.ts @@ -1,17 +1,8 @@ +import type { TextEditInput as FileEditInput, TextEditResult as FileEditResult } from '#/os/interface/fileOperations'; import { createDecorator, type ServiceIdentifier } from '#/_base/di/instantiation'; import type { IHostFileSystem } from '#/os/interface/hostFileSystem'; -export interface FileEditInput { - readonly path: string; - readonly displayPath: string; - readonly old_string: string; - readonly new_string: string; - readonly replace_all: boolean; -} - -export type FileEditResult = - | { readonly ok: true; readonly count: number } - | { readonly ok: false; readonly error: string }; +export type { TextEditInput as FileEditInput, TextEditResult as FileEditResult } from '#/os/interface/fileOperations'; export interface IFileEditService { readonly _serviceBrand: undefined; diff --git a/packages/agent-core-v2/src/app/edit/fileEditService.ts b/packages/agent-core-v2/src/app/edit/fileEditService.ts index 79e8510248e..c8904a4a71c 100644 --- a/packages/agent-core-v2/src/app/edit/fileEditService.ts +++ b/packages/agent-core-v2/src/app/edit/fileEditService.ts @@ -1,47 +1,18 @@ import { LifecycleScope } from '#/app/scopes'; import { ScopeActivation, registerScopedService } from '#/_base/di/scope'; -import { unwrapErrorCause } from '#/_base/errors/errors'; import { IHostFileSystem } from '#/os/interface/hostFileSystem'; -import { EditService } from './editService'; +import { editText } from '#/os/operations/editText'; import { type FileEditInput, type FileEditResult, IFileEditService } from './fileEdit'; -import { TextModel } from './textModel'; export class FileEditService implements IFileEditService { declare readonly _serviceBrand: undefined; - private readonly editor: EditService; - - constructor(@IHostFileSystem private readonly fs: IHostFileSystem) { - this.editor = new EditService(); - } + constructor(@IHostFileSystem private readonly fs: IHostFileSystem) {} async edit(input: FileEditInput, fs: IHostFileSystem = this.fs): Promise { - try { - const raw = await fs.readText(input.path, { errors: 'strict' }); - const model = new TextModel(raw); - const result = this.editor.apply(model, { - path: input.displayPath, - old_string: input.old_string, - new_string: input.new_string, - replace_all: input.replace_all, - }); - if (!result.ok) { - return { ok: false, error: result.error }; - } - await fs.writeText(input.path, result.rawContent); - return { ok: true, count: result.count }; - } catch (error) { - const code = (unwrapErrorCause(error) as { code?: unknown } | null)?.code; - if (code === 'EISDIR') { - return { ok: false, error: `${input.displayPath} is not a file.` }; - } - return { - ok: false, - error: error instanceof Error ? error.message : String(error), - }; - } + return fs.operations === undefined ? editText(fs, input) : fs.operations.editText(input); } } diff --git a/packages/agent-core-v2/src/app/edit/textModel.ts b/packages/agent-core-v2/src/app/edit/textModel.ts index ab6611d738f..05211bd786a 100644 --- a/packages/agent-core-v2/src/app/edit/textModel.ts +++ b/packages/agent-core-v2/src/app/edit/textModel.ts @@ -1,43 +1 @@ -import { - type LineEndingStyle, - materializeModelText, - toModelTextView, -} from '#/_base/text/line-endings'; - -export class TextModel { - readonly lineEndingStyle: LineEndingStyle; - readonly text: string; - - constructor(raw: string) { - const view = toModelTextView(raw); - this.text = view.text; - this.lineEndingStyle = view.lineEndingStyle; - } - - countOccurrences(needle: string): number { - let count = 0; - let pos = 0; - while (pos < this.text.length) { - const idx = this.text.indexOf(needle, pos); - if (idx === -1) break; - count += 1; - pos = idx + needle.length; - } - return count; - } - - replaceOnce(needle: string, replacement: string): string { - const index = this.text.indexOf(needle); - if (index === -1) return this.text; - return this.text.slice(0, index) + replacement + this.text.slice(index + needle.length); - } - - replaceAll(needle: string, replacement: string): { text: string; count: number } { - const parts = this.text.split(needle); - return { text: parts.join(replacement), count: parts.length - 1 }; - } - - materialize(modelText: string): string { - return materializeModelText(modelText, this.lineEndingStyle); - } -} +export * from '#/os/operations/textModel'; diff --git a/packages/agent-core-v2/src/app/git/workTree.ts b/packages/agent-core-v2/src/app/git/workTree.ts index 42b30154d8a..2868783c05e 100644 --- a/packages/agent-core-v2/src/app/git/workTree.ts +++ b/packages/agent-core-v2/src/app/git/workTree.ts @@ -1,54 +1,9 @@ -import { dirname, isAbsolute, join, normalize } from 'pathe'; - import type { IHostFileSystem } from '#/os/interface/hostFileSystem'; +import type { GitWorkTree } from '#/os/interface/fileOperations'; +import { findGitWorkTree as findLocalGitWorkTree } from '#/os/operations/gitWorkTree'; -export interface GitWorkTree { - readonly root: string; - readonly dotGitPath: string; - readonly controlDirPath: string; -} - -export async function findGitWorkTree( - fs: IHostFileSystem, - cwd: string, -): Promise { - if (cwd.length === 0 || !isAbsolute(cwd)) return null; - - let current = normalize(cwd); - while (true) { - const dotGitPath = join(current, '.git'); - const controlDirPath = await probeGitControlDir(fs, dotGitPath, current); - if (controlDirPath !== null) return { root: current, dotGitPath, controlDirPath }; - - const parent = dirname(current); - if (parent === current) return null; - current = parent; - } -} - -async function probeGitControlDir( - fs: IHostFileSystem, - dotGitPath: string, - markerParent: string, -): Promise { - try { - const stat = await fs.stat(dotGitPath); - if (stat.isDirectory) return dotGitPath; - if (!stat.isFile) return null; - - const content = await fs.readText(dotGitPath); - return parseGitDirPointer(content, markerParent) ?? null; - } catch { - return null; - } -} - -function parseGitDirPointer(content: string, markerParent: string): string | undefined { - const stripped = content.codePointAt(0) === 0xfeff ? content.slice(1) : content; - const line = stripped.trimStart().split(/\r?\n/, 1)[0]?.trim(); - if (line === undefined || !line.startsWith('gitdir:')) return undefined; +export type { GitWorkTree } from '#/os/interface/fileOperations'; - const rawPath = line.slice('gitdir:'.length).trim(); - if (rawPath.length === 0) return undefined; - return normalize(isAbsolute(rawPath) ? rawPath : join(markerParent, rawPath)); +export function findGitWorkTree(fs: IHostFileSystem, cwd: string): Promise { + return fs.operations === undefined ? findLocalGitWorkTree(fs, cwd) : fs.operations.findGitWorkTree(cwd); } diff --git a/packages/agent-core-v2/src/features/fileHistory/fileHistoryService.ts b/packages/agent-core-v2/src/features/fileHistory/fileHistoryService.ts index d343b45b504..a141275a640 100644 --- a/packages/agent-core-v2/src/features/fileHistory/fileHistoryService.ts +++ b/packages/agent-core-v2/src/features/fileHistory/fileHistoryService.ts @@ -1,3 +1,4 @@ +import { snapshotFiles } from '#/os/operations/snapshotFiles'; import { createHash } from 'node:crypto'; import { Service } from '#/_base/di/service'; @@ -10,7 +11,6 @@ import type { WillExecuteToolEvent } from '#/agent/toolExecutor/toolHooks'; import { TurnStarted } from '#/agent/loop/turnEvents'; import { ISessionContext } from '#/session/sessionContext/sessionContext'; import { IHostFileSystem } from '#/os/interface/hostFileSystem'; -import { isHostFsNotFound } from '#/os/interface/hostFsErrors'; import { IAtomicDocumentStore } from '#/persistence/interface/atomicDocumentStore'; import { TurnEnded } from '#/agent/loop/turnOps'; import { IEventBus } from '#/app/event/eventBus'; @@ -49,6 +49,8 @@ import { touchFileHistorySession } from './fileHistoryRetention'; export const FILE_HISTORY_MAX_FILE_BYTES = 4 * 1024 * 1024; export { FILE_HISTORY_BLOB_PREFIX } from './fileHistory'; +type CurrentFile = Uint8Array | 'missing' | 'unreadable' | { oversizeBytes: number; mtimeMs?: number }; + export class AgentFileHistoryService extends Service implements IAgentFileHistoryService { declare readonly _serviceBrand: undefined; @@ -299,9 +301,9 @@ export class AgentFileHistoryService extends Service implements IAgentFileHistor ); if (startCheckpoint !== undefined && Object.hasOwn(startCheckpoint.entries, pathKey)) return; - const current = await this.readCurrent(pathKey); - if (current === 'unreadable') return; const latest = latestEntry(state.checkpoints, pathKey); + const current = await this.readCurrent(pathKey, latest); + if (current === 'unreadable') return; const nextVersion = maxVersion(state.checkpoints, pathKey) + 1; let entry: FileBackupEntry; if (current === 'missing') { @@ -356,35 +358,40 @@ export class AgentFileHistoryService extends Service implements IAgentFileHistor string, FileBackupEntry >; - for (const [pathKey, before] of Object.entries(start.entries)) { - const nextVersion = maxVersion(state.checkpoints, pathKey) + 1; - const current = await this.readCurrent(pathKey); - if (current === 'unreadable') continue; - if (current === 'missing') { - if (before.key !== null || before.oversize === true) { - entries[pathKey] = { key: null, version: nextVersion }; + const tracked = Object.entries(start.entries); + for (let offset = 0; offset < tracked.length; offset += 4) { + const batch = tracked.slice(offset, offset + 4); + const currentFiles = await this.readCurrents(batch.map(([pathKey, previous]) => ({ pathKey, previous }))); + for (const [pathKey, before] of batch) { + const nextVersion = maxVersion(state.checkpoints, pathKey) + 1; + const current = currentFiles.get(pathKey) ?? 'unreadable'; + if (current === 'unreadable') continue; + if (current === 'missing') { + if (before.key !== null || before.oversize === true) { + entries[pathKey] = { key: null, version: nextVersion }; + } + continue; } - continue; - } - if (!(current instanceof Uint8Array)) { - if ( - before.oversize !== true || - before.size !== current.oversizeBytes || - before.mtimeMs !== current.mtimeMs - ) { - entries[pathKey] = { - key: null, - version: nextVersion, - oversize: true, - size: current.oversizeBytes, - mtimeMs: current.mtimeMs, - }; + if (!(current instanceof Uint8Array)) { + if ( + before.oversize !== true || + before.size !== current.oversizeBytes || + before.mtimeMs !== current.mtimeMs + ) { + entries[pathKey] = { + key: null, + version: nextVersion, + oversize: true, + size: current.oversizeBytes, + mtimeMs: current.mtimeMs, + }; + } + continue; } - continue; + const contentHash = sha256(current); + if (before.contentHash === contentHash) continue; + entries[pathKey] = await this.backup(pathKey, nextVersion, current, contentHash); } - const contentHash = sha256(current); - if (before.contentHash === contentHash) continue; - entries[pathKey] = await this.backup(pathKey, nextVersion, current, contentHash); } const evictable = displacedCheckpoints(state.checkpoints, turnId); @@ -464,48 +471,44 @@ export class AgentFileHistoryService extends Service implements IAgentFileHistor return this.blobs.get(this.agentCtx.scope(), entry.key); } - private async readCurrent( - pathKey: string, - ): Promise< - Uint8Array | 'missing' | 'unreadable' | { oversizeBytes: number; mtimeMs?: number } - > { + private async readCurrent(pathKey: string, previous?: FileBackupEntry): Promise { + return (await this.readCurrents([{ pathKey, previous }])).get(pathKey) ?? 'unreadable'; + } + + private async readCurrents(files: readonly { pathKey: string; previous?: FileBackupEntry }[]): Promise> { + const results = new Map(); let lease: EnvironmentLease; try { lease = this.environment.acquire(['fs']); } catch (error) { - if (error instanceof EnvironmentError) return 'unreadable'; + if (error instanceof EnvironmentError) return results; throw error; } try { const fs = lease.environment.fs; - if (fs === undefined) return 'unreadable'; const path = lease.environment.path; - if (path === undefined) return 'unreadable'; - const absolute = resolveWorkspacePath(path, this.workspaceCtx.workDir, pathKey); - let info; - try { - info = await fs.stat(absolute); - } catch (error) { - return isHostFsNotFound(error) ? 'missing' : 'unreadable'; - } - if (!info.isFile) return 'unreadable'; - if (info.size > FILE_HISTORY_MAX_FILE_BYTES) { - return { oversizeBytes: info.size, mtimeMs: info.mtimeMs }; - } - try { - const bytes = await fs.readBytes(absolute, info.size + 1); - if (bytes.byteLength > FILE_HISTORY_MAX_FILE_BYTES) { - const grown = await fs.stat(absolute).catch(() => undefined); - return { - oversizeBytes: grown?.size ?? bytes.byteLength, - mtimeMs: grown?.mtimeMs, - }; + if (fs === undefined || path === undefined) return results; + for (let offset = 0; offset < files.length; offset += 4) { + const batch = files.slice(offset, offset + 4); + const known = await Promise.all(batch.map((file) => this.entryBytes(file.previous))); + const requests = batch.map((file, index) => ({ + path: resolveWorkspacePath(path, this.workspaceCtx.workDir, file.pathKey), + knownHash: known[index] === undefined ? undefined : file.previous?.contentHash, + })); + const snapshots = await (fs.operations === undefined + ? snapshotFiles(fs, requests, FILE_HISTORY_MAX_FILE_BYTES) + : fs.operations.snapshotFiles(requests, FILE_HISTORY_MAX_FILE_BYTES)).catch(() => []); + for (let index = 0; index < batch.length; index += 1) { + const snapshot = snapshots[index]; + const current = snapshot?.kind === 'file' + ? snapshot.bytes ?? known[index] ?? 'unreadable' + : snapshot?.kind === 'oversize' + ? { oversizeBytes: snapshot.size, mtimeMs: snapshot.mtimeMs } + : snapshot?.kind ?? 'unreadable'; + results.set(batch[index]!.pathKey, current); } - if (bytes.byteLength !== info.size) return 'unreadable'; - return bytes; - } catch { - return 'unreadable'; } + return results; } finally { lease.dispose(); } diff --git a/packages/agent-core-v2/src/os/backends/node-local/tools/runRg.ts b/packages/agent-core-v2/src/os/backends/node-local/tools/runRg.ts index de3700a8ed2..957040d2cba 100644 --- a/packages/agent-core-v2/src/os/backends/node-local/tools/runRg.ts +++ b/packages/agent-core-v2/src/os/backends/node-local/tools/runRg.ts @@ -1,191 +1 @@ -import type { Readable } from 'node:stream'; - -import { BugIndicatingError } from '#/errors'; -import type { IHostProcess, IHostProcessService } from '#/os/interface/hostProcess'; - -export const DEFAULT_TIMEOUT_MS = 20_000; -export const SIGTERM_GRACE_MS = 5_000; -export const MAX_OUTPUT_BYTES = 10 * 1024 * 1024; - -export interface RunRgResult { - readonly kind: 'result'; - readonly exitCode: number; - readonly stdoutText: string; - readonly stderrText: string; - readonly bufferTruncated: boolean; - readonly stderrTruncated: boolean; - readonly timedOut: boolean; -} - -export type RunRgOutcome = RunRgResult | { readonly kind: 'aborted' }; - -function disposeProcess(proc: IHostProcess): void { - try { - void proc.dispose(); - } catch { - } -} - -export async function runRgOnce( - processService: IHostProcessService, - rgArgs: readonly string[], - signal: AbortSignal, - options?: { readonly cwd?: string }, -): Promise { - if (signal.aborted) { - return { kind: 'aborted' }; - } - - const [command, ...args] = rgArgs; - if (command === undefined) { - throw new BugIndicatingError('runRgOnce: rgArgs must not be empty'); - } - const proc: IHostProcess = await processService.spawn(command, args, { cwd: options?.cwd }); - - try { - proc.stdin.end(); - } catch { - } - - let timedOut = false; - let aborted = false; - let killed = false; - - const killProc = async (): Promise => { - if (killed) return; - killed = true; - try { - await proc.kill('SIGTERM'); - } catch { - } - const exited = proc - .wait() - .then(() => true) - .catch(() => true); - const raced = await Promise.race([ - exited, - new Promise((resolve) => { - setTimeout(() => { - resolve(false); - }, SIGTERM_GRACE_MS); - }), - ]); - if (!raced && proc.exitCode === null) { - try { - await proc.kill('SIGKILL'); - } catch { - } - } - disposeProcess(proc); - }; - - const onAbort = (): void => { - aborted = true; - void killProc(); - }; - signal.addEventListener('abort', onAbort); - if (signal.aborted) onAbort(); - - const timeoutHandle = setTimeout(() => { - timedOut = true; - void killProc(); - }, DEFAULT_TIMEOUT_MS); - - let exitCode = 0; - let stdoutText = ''; - let stderrText = ''; - let bufferTruncated = false; - let stderrTruncated = false; - - try { - const isTerminating = (): boolean => timedOut || aborted || killed; - const [stdoutResult, stderrResult, code] = await Promise.all([ - readStreamWithCap(proc.stdout, MAX_OUTPUT_BYTES, isTerminating), - readStreamWithCap(proc.stderr, MAX_OUTPUT_BYTES, isTerminating), - proc.wait(), - ]); - stdoutText = stdoutResult.text; - stderrText = stderrResult.text; - bufferTruncated = stdoutResult.truncated; - stderrTruncated = stderrResult.truncated; - exitCode = code; - } catch (error) { - if (!(isPrematureCloseError(error) && (timedOut || aborted || killed))) { - throw error; - } - } finally { - clearTimeout(timeoutHandle); - signal.removeEventListener('abort', onAbort); - disposeProcess(proc); - } - - if (aborted) { - return { kind: 'aborted' }; - } - - return { - kind: 'result', - exitCode, - stdoutText, - stderrText, - bufferTruncated, - stderrTruncated, - timedOut, - }; -} - -export function shouldRetryRipgrepEagain(result: RunRgResult): boolean { - return ( - result.exitCode !== 0 && - result.exitCode !== 1 && - !result.timedOut && - isEagainRipgrepError(result.stderrText) - ); -} - -function isEagainRipgrepError(stderr: string): boolean { - return stderr.includes('os error 11') || stderr.includes('Resource temporarily unavailable'); -} - -function isPrematureCloseError(error: unknown): boolean { - return ( - error instanceof Error && - (error as NodeJS.ErrnoException).code === 'ERR_STREAM_PREMATURE_CLOSE' - ); -} - -interface CappedStreamResult { - readonly text: string; - readonly truncated: boolean; -} - -async function readStreamWithCap( - stream: Readable, - maxBytes: number, - suppressPrematureClose?: () => boolean, -): Promise { - const chunks: Buffer[] = []; - let total = 0; - let truncated = false; - try { - for await (const chunk of stream) { - const buf: Buffer = - typeof chunk === 'string' ? Buffer.from(chunk, 'utf8') : (chunk as Buffer); - if (truncated) continue; - if (total + buf.length > maxBytes) { - const remaining = maxBytes - total; - if (remaining > 0) chunks.push(buf.subarray(0, remaining)); - total = maxBytes; - truncated = true; - continue; - } - chunks.push(buf); - total += buf.length; - } - } catch (error) { - if (!isPrematureCloseError(error) || suppressPrematureClose?.() !== true) { - throw error; - } - } - return { text: Buffer.concat(chunks).toString('utf8'), truncated }; -} +export * from '#/os/operations/runRg'; diff --git a/packages/agent-core-v2/src/os/backends/remote/remoteFileSystem.ts b/packages/agent-core-v2/src/os/backends/remote/remoteFileSystem.ts index a9a997793e6..6ac913048bd 100644 --- a/packages/agent-core-v2/src/os/backends/remote/remoteFileSystem.ts +++ b/packages/agent-core-v2/src/os/backends/remote/remoteFileSystem.ts @@ -1,3 +1,9 @@ +import { readTextPage } from '#/os/operations/readTextPage'; +import { EditService } from '#/os/operations/editService'; +import { TextModel } from '#/os/operations/textModel'; +import { createHash, randomUUID } from 'node:crypto'; +import { writeTextResult } from '#/os/operations/writeText'; +import { TEXT_PAGE_MAX_REMOTE_CHARS, type GitWorkTree, type FileSnapshot, type HostFileOperations } from '#/os/interface/fileOperations'; import * as posixPath from 'node:path/posix'; import { @@ -35,6 +41,15 @@ import { FS_READ_DIRECTORY_MAX_ENTRIES, FS_READ_FILE_MAX_BYTES, FS_READ_FILE_METHOD, + FS_READ_TEXT_PAGE_METHOD, + FS_CANCEL_READ_METHOD, + FS_MAX_ACTIVE_READS, + FS_SNAPSHOT_FILES_METHOD, + FS_MATCH_FILES_METHOD, + FS_FIND_GIT_WORK_TREE_METHOD, + FS_DISCOVER_INSTRUCTIONS_METHOD, + FS_SNAPSHOT_MAX_FILES, + FS_EDIT_TEXT_METHOD, FS_REMOVE_METHOD, FS_RENAME_METHOD, FS_WALK_METHOD, @@ -79,7 +94,115 @@ function isUtf8Encoding(encoding: BufferEncoding): boolean { export class RemoteFileSystem implements IHostFileSystem { declare readonly _serviceBrand: undefined; - constructor(private readonly connection: ExecChannel) {} + private readonly contents = new Map(); + private contentBytes = 0; + private activeReads = 0; + + constructor(private readonly connection: ExecChannel) { + connection.onDidClose(() => { + this.contents.clear(); + this.contentBytes = 0; + }); + } + + private rememberContent(path: string, data: Uint8Array): void { + const previous = this.contents.get(path); + if (previous !== undefined) { + this.contentBytes -= previous.bytes.byteLength; + this.contents.delete(path); + } + if (data.byteLength > 4 * 1024 * 1024) return; + const bytes = Uint8Array.from(data); + this.contents.set(path, { bytes, hash: createHash('sha256').update(bytes).digest('hex') }); + this.contentBytes += bytes.byteLength; + while (this.contentBytes > 16 * 1024 * 1024 || this.contents.size > 16) { + const first = this.contents.entries().next().value!; + this.contents.delete(first[0]); + this.contentBytes -= first[1].bytes.byteLength; + } + } + + + readonly operations: HostFileOperations = { + matchFiles: (files, options, timeoutMs) => this.call(FS_MATCH_FILES_METHOD, { files, options, timeoutMs }), + findGitWorkTree: async (cwd) => posixPath.isAbsolute(cwd) ? (await this.call<{ workTree: GitWorkTree | null }>(FS_FIND_GIT_WORK_TREE_METHOD, { cwd })).workTree : null, + discoverInstructions: async (dirs, known) => { + const paths = new Set(); + for (let offset = 0; offset < dirs.length; offset += 64) { + for (const path of await this.call(FS_DISCOVER_INSTRUCTIONS_METHOD, { dirs: dirs.slice(offset, offset + 64), known })) paths.add(path); + } + return [...paths]; + }, + snapshotFiles: async (files, maxFileBytes) => { + const results: FileSnapshot[] = []; + for (let offset = 0; offset < files.length; offset += FS_SNAPSHOT_MAX_FILES) { + const requests = files.slice(offset, offset + FS_SNAPSHOT_MAX_FILES); + const cached = requests.map((file) => this.contents.get(file.path)); + const batch = await this.call(FS_SNAPSHOT_FILES_METHOD, { + files: requests.map((file, index) => ({ path: file.path, knownHash: cached[index]?.hash ?? file.knownHash })), maxFileBytes, + }); + for (let index = 0; index < batch.length; index += 1) { + const result = batch[index]!; + const request = requests[index]!; + if (result.kind !== 'file') { + results.push(result); + continue; + } + const bytes = result.dataBase64 === undefined + ? cached[index]?.hash === result.contentHash ? cached[index]!.bytes : undefined + : decodeBase64(result.dataBase64); + if (bytes !== undefined) this.rememberContent(request.path, bytes); + results.push({ kind: 'file', contentHash: result.contentHash, bytes: request.knownHash === result.contentHash ? undefined : bytes }); + } + } + return results; + }, + writeText: (input) => writeTextResult(input, async () => { + const bytes = Buffer.from(input.content, 'utf8'); + const previous = input.mode === 'append' ? this.contents.get(input.path) : undefined; + await this.writeChunked(input.path, bytes, input.mode === 'append' ? 'append' : 'truncate', input.parent); + if (input.mode === 'overwrite') this.rememberContent(input.path, bytes); + else if (previous !== undefined) this.rememberContent(input.path, Buffer.concat([previous.bytes, bytes])); + }), + readTextPage: async (path, options, signal) => { + if (signal?.aborted === true) return { isError: true, output: 'Read aborted' }; + if (options.maxChars > TEXT_PAGE_MAX_REMOTE_CHARS || this.activeReads >= FS_MAX_ACTIVE_READS) { + return readTextPage({ name: path, stat: () => this.stat(path), readBytes: (n) => this.readBytes(path, n), readLines: () => this.readLines(path) }, options, signal); + } + this.activeReads += 1; + const readId = randomUUID(); + const cancel = (): void => { + try { this.connection.notify(FS_CANCEL_READ_METHOD, { readId }); } catch {} + }; + signal?.addEventListener('abort', cancel, { once: true }); + try { + return await this.call(FS_READ_TEXT_PAGE_METHOD, { path, ...options, readId }); + } catch (error) { + cancel(); + return { isError: true, output: error instanceof Error ? error.message : String(error) }; + } finally { + this.activeReads -= 1; + signal?.removeEventListener('abort', cancel); + } + }, + editText: async (input) => { + const previous = this.contents.get(input.path); + let result: Awaited>; + try { + result = await this.call(FS_EDIT_TEXT_METHOD, input); + } catch (error) { + return { ok: false, error: error instanceof Error ? error.message : String(error) }; + } + if (result.ok && previous !== undefined) { + try { + const raw = decodeTextWithErrors(Buffer.from(previous.bytes), 'utf8', 'strict'); + const applied = new EditService().apply(new TextModel(raw), { ...input, path: input.displayPath }); + if (applied.ok) this.rememberContent(input.path, Buffer.from(applied.rawContent)); + } catch {} + } + return result; + }, + }; private async call(method: string, params: unknown): Promise { try { @@ -93,17 +216,31 @@ export class RemoteFileSystem implements IHostFileSystem { return this.call(FS_READ_FILE_METHOD, { path, offset, maxBytes }); } - private async writeMode(path: string, data: Uint8Array, mode: FsWriteMode): Promise { - await this.call(FS_WRITE_FILE_METHOD, { path, dataBase64: encodeBase64(data), mode }); - } - - private async writeChunked(path: string, data: Uint8Array, mode: FsWriteMode): Promise { + private async writeChunked(path: string, data: Uint8Array, mode: FsWriteMode, parent?: string): Promise { if (data.byteLength <= FS_WRITE_FILE_CHUNK_BYTES) { - await this.writeMode(path, data, mode); + await this.call(FS_WRITE_FILE_METHOD, { path, dataBase64: encodeBase64(data), mode, parent }); return; } - for (let offset = 0; offset < data.byteLength; offset += FS_WRITE_FILE_CHUNK_BYTES) { - await this.writeMode(path, data.subarray(offset, offset + FS_WRITE_FILE_CHUNK_BYTES), offset === 0 ? mode : 'append'); + const transferId = randomUUID(); + let chunkIndex = 0; + for (let offset = 0; offset < data.byteLength;) { + const pending: Promise[] = []; + for (let window = 0; window < 4 && offset < data.byteLength; window += 1) { + const end = Math.min(data.byteLength, offset + FS_WRITE_FILE_CHUNK_BYTES); + pending.push(this.call(FS_WRITE_FILE_METHOD, { + path, + dataBase64: encodeBase64(data.subarray(offset, end)), + mode: offset === 0 ? mode : 'append', + parent: offset === 0 ? parent : undefined, + transferId, + chunkIndex: chunkIndex++, + final: end === data.byteLength, + })); + offset = end; + } + const results = await Promise.allSettled(pending); + const failure = results.find((result) => result.status === 'rejected'); + if (failure?.status === 'rejected') throw failure.reason; } } @@ -128,7 +265,7 @@ export class RemoteFileSystem implements IHostFileSystem { if (n === 0) return new Uint8Array(0); const chunks: Uint8Array[] = []; let total = 0; - for await (const chunk of this.readChunks(path, offset)) { + for await (const chunk of this.readChunks(path, offset, n)) { const length = n === undefined ? chunk.byteLength : Math.min(chunk.byteLength, n - total); if (length === 0) break; chunks.push(chunk.subarray(0, length)); @@ -149,25 +286,43 @@ export class RemoteFileSystem implements IHostFileSystem { await this.writeChunked(path, data, 'truncate'); return; } - let mode: FsWriteMode = 'truncate'; + const transferId = randomUUID(); + let chunkIndex = 0; const pending = new Uint8Array(FS_WRITE_FILE_CHUNK_BYTES); let pendingBytes = 0; - for await (const chunk of data) { - let offset = 0; - while (offset < chunk.byteLength) { - const length = Math.min(pending.byteLength - pendingBytes, chunk.byteLength - offset); - pending.set(chunk.subarray(offset, offset + length), pendingBytes); - pendingBytes += length; - offset += length; - if (pendingBytes === pending.byteLength) { - await this.writeMode(path, pending, mode); - mode = 'append'; - pendingBytes = 0; + let writes: Promise[] = []; + const send = (bytes: Uint8Array, final: boolean): void => { + writes.push(this.call(FS_WRITE_FILE_METHOD, { + path, dataBase64: encodeBase64(bytes), mode: chunkIndex === 0 ? 'truncate' : 'append', + transferId, chunkIndex: chunkIndex++, final, + })); + }; + const flush = async (): Promise => { + const results = await Promise.allSettled(writes); + writes = []; + const failure = results.find((result) => result.status === 'rejected'); + if (failure?.status === 'rejected') throw failure.reason; + }; + try { + for await (const chunk of data) { + let offset = 0; + while (offset < chunk.byteLength) { + const length = Math.min(pending.byteLength - pendingBytes, chunk.byteLength - offset); + pending.set(chunk.subarray(offset, offset + length), pendingBytes); + pendingBytes += length; + offset += length; + if (pendingBytes === pending.byteLength) { + send(pending, false); + pendingBytes = 0; + if (writes.length === 4) await flush(); + } } + await flush(); } - } - if (pendingBytes > 0 || mode === 'truncate') { - await this.writeMode(path, pending.subarray(0, pendingBytes), mode); + send(pending.subarray(0, pendingBytes), true); + await flush(); + } finally { + await Promise.allSettled(writes); } } @@ -185,14 +340,15 @@ export class RemoteFileSystem implements IHostFileSystem { yield* readUtf8Lines(this.readChunks(path), errors); } - private async *readChunks(path: string, offset = 0): AsyncGenerator { + private async *readChunks(path: string, offset = 0, remaining?: number): AsyncGenerator { for (;;) { - const result = await this.readRange(path, offset, FS_READ_FILE_MAX_BYTES); + const result = await this.readRange(path, offset, Math.min(remaining ?? FS_READ_FILE_MAX_BYTES, FS_READ_FILE_MAX_BYTES)); if (result.dataBase64.length === 0) return; const chunk = decodeBase64(result.dataBase64); yield chunk; offset += chunk.byteLength; - if (result.eof) return; + if (remaining !== undefined) remaining -= chunk.byteLength; + if (result.eof || remaining === 0) return; } } diff --git a/packages/agent-core-v2/src/os/backends/remote/remoteProcess.ts b/packages/agent-core-v2/src/os/backends/remote/remoteProcess.ts index 5dc2a7c6e12..3daad334307 100644 --- a/packages/agent-core-v2/src/os/backends/remote/remoteProcess.ts +++ b/packages/agent-core-v2/src/os/backends/remote/remoteProcess.ts @@ -1,3 +1,6 @@ +import { searchGlob } from '#/os/operations/searchGlob'; +import { searchGrep } from '#/os/operations/searchGrep'; +import type { SearchRequest, SearchResult } from '#/os/interface/searchOperations'; import { randomUUID } from 'node:crypto'; import { Readable, Writable } from 'node:stream'; @@ -20,6 +23,10 @@ import { PROCESS_OUTPUT_METHOD, PROCESS_SIGNAL_METHOD, PROCESS_START_METHOD, + PROCESS_SEARCH_METHOD, + PROCESS_CANCEL_SEARCH_METHOD, + PROCESS_MAX_ACTIVE_SEARCHES, + PROCESS_SEARCH_MAX_ROOTS, PROCESS_WRITE_METHOD, type ProcessClosedNotification, type ProcessExitedNotification, @@ -233,6 +240,7 @@ export class RemoteProcessService implements IHostProcessService { declare readonly _serviceBrand: undefined; private readonly processes = new Map(); + private activeSearches = 0; constructor( private readonly connection: ExecChannel, @@ -263,6 +271,28 @@ export class RemoteProcessService implements IHostProcessService { }); } + async search(request: SearchRequest, signal: AbortSignal): Promise { + if (signal.aborted) return { isError: true, output: request.kind === 'glob' ? 'Glob aborted' : 'Grep aborted' }; + if (request.roots.length > PROCESS_SEARCH_MAX_ROOTS || this.activeSearches >= PROCESS_MAX_ACTIVE_SEARCHES) { + return request.kind === 'glob' ? searchGlob(this, request, signal) : searchGrep(this, request, signal); + } + this.activeSearches += 1; + const searchId = randomUUID(); + const cancel = (): void => { + try { this.connection.notify(PROCESS_CANCEL_SEARCH_METHOD, { searchId }); } catch {} + }; + signal.addEventListener('abort', cancel, { once: true }); + try { + return await this.connection.call(PROCESS_SEARCH_METHOD, { ...request, searchId }) as SearchResult; + } catch (error) { + cancel(); + throw toRemoteProcessError(error); + } finally { + this.activeSearches -= 1; + signal.removeEventListener('abort', cancel); + } + } + async spawn( command: string, args: readonly string[] = [], diff --git a/packages/agent-core-v2/src/os/interface/fileOperations.ts b/packages/agent-core-v2/src/os/interface/fileOperations.ts new file mode 100644 index 00000000000..cfa8479f111 --- /dev/null +++ b/packages/agent-core-v2/src/os/interface/fileOperations.ts @@ -0,0 +1,115 @@ +import type { ToolResultSpill } from '#/_base/text/outputLimits'; +import type { HostFileStat } from './hostFileSystem'; + +export const TEXT_PAGE_MAX_REMOTE_CHARS = 4_000_000; + +export interface TextFileResult { + readonly output: string; + readonly note?: string; + readonly isError?: boolean; + readonly truncated?: boolean; + readonly spill?: ToolResultSpill; +} + +export interface TextPageInput { + readonly path: string; + readonly line_offset?: number; + readonly column_offset?: number; + readonly n_lines?: number; + readonly max_chars?: number; +} + +export interface TextPageOptions { + readonly args: TextPageInput; + readonly maxChars: number; + readonly maxCharsLimit: number; + readonly eventLog: boolean; + readonly imageSupported: boolean; + readonly videoSupported: boolean; + readonly mediaToolAvailable: boolean; +} + +export interface FileReadSource { + readonly name: string; + readonly localPath?: string; + stat(): Promise; + readBytes(n?: number): Promise; + readLines(): AsyncIterable; + readTextPage?(options: TextPageOptions, signal?: AbortSignal): Promise; +} + +export interface TextEditInput { + readonly path: string; + readonly displayPath: string; + readonly old_string: string; + readonly new_string: string; + readonly replace_all: boolean; +} + +export type TextEditResult = + | { readonly ok: true; readonly count: number } + | { readonly ok: false; readonly error: string }; + +export interface TextWriteInput { + readonly path: string; + readonly displayPath: string; + readonly parent: string; + readonly content: string; + readonly mode: 'overwrite' | 'append'; +} + +export interface FileSnapshotRequest { + readonly path: string; + readonly knownHash?: string; +} + +export type FileSnapshot = + | { readonly kind: 'missing' | 'unreadable' } + | { readonly kind: 'oversize'; readonly size: number; readonly mtimeMs?: number } + | { readonly kind: 'file'; readonly contentHash: string; readonly bytes?: Uint8Array }; + +export interface GitWorkTree { + readonly root: string; + readonly dotGitPath: string; + readonly controlDirPath: string; +} + +export interface FileMatchOptions { + readonly pattern: string; + readonly regex: boolean; + readonly case_sensitive: boolean; + readonly context_lines: number; + readonly max_files: number; + readonly max_matches_per_file: number; + readonly max_total_matches: number; +} + +export interface FileMatch { + readonly line: number; + readonly col: number; + readonly text: string; + readonly before: string[]; + readonly after: string[]; +} + +export interface FileMatches { + readonly responseTooLarge?: boolean; + readonly files: { path: string; matches: FileMatch[] }[]; + readonly files_scanned: number; + readonly truncated: boolean; +} + +export interface FileMatchPath { + readonly path: string; + readonly absolute: string; +} + +export interface HostFileOperations { + matchFiles(files: readonly FileMatchPath[], options: FileMatchOptions, timeoutMs: number): Promise; + findGitWorkTree(cwd: string): Promise; + discoverInstructions(dirs: readonly string[], known: readonly string[]): Promise; + snapshotFiles(files: readonly FileSnapshotRequest[], maxFileBytes: number): Promise; + writeText(input: TextWriteInput): Promise; + readTextPage(path: string, options: TextPageOptions, signal?: AbortSignal): Promise; + editText(input: TextEditInput): Promise; +} diff --git a/packages/agent-core-v2/src/os/interface/hostFileSystem.ts b/packages/agent-core-v2/src/os/interface/hostFileSystem.ts index e33c60273e2..69b02812575 100644 --- a/packages/agent-core-v2/src/os/interface/hostFileSystem.ts +++ b/packages/agent-core-v2/src/os/interface/hostFileSystem.ts @@ -1,4 +1,5 @@ import { createDecorator, type ServiceIdentifier } from '#/_base/di/instantiation'; +import type { HostFileOperations } from './fileOperations'; import type { TextDecodeErrors } from '#/_base/execEnv/decodeText'; export interface HostFileStat { @@ -42,6 +43,8 @@ export interface HostWalkResult { export interface IHostFileSystem { readonly _serviceBrand: undefined; + readonly operations?: HostFileOperations; + readText( path: string, options?: { encoding?: BufferEncoding; errors?: TextDecodeErrors }, diff --git a/packages/agent-core-v2/src/os/interface/hostProcess.ts b/packages/agent-core-v2/src/os/interface/hostProcess.ts index 6197026f790..376c63ef45b 100644 --- a/packages/agent-core-v2/src/os/interface/hostProcess.ts +++ b/packages/agent-core-v2/src/os/interface/hostProcess.ts @@ -1,3 +1,4 @@ +import type { SearchRequest, SearchResult } from './searchOperations'; import type { Readable, Writable } from 'node:stream'; import { registerErrorDomain, type ErrorDomain } from '#/_base/errors/codes'; @@ -27,9 +28,12 @@ export interface IHostProcess { dispose(): void | Promise; } + export interface IHostProcessService { readonly _serviceBrand: undefined; + search?(request: SearchRequest, signal: AbortSignal): Promise; + spawn( command: string, args?: readonly string[], diff --git a/packages/agent-core-v2/src/os/interface/searchOperations.ts b/packages/agent-core-v2/src/os/interface/searchOperations.ts new file mode 100644 index 00000000000..f880e02d4b7 --- /dev/null +++ b/packages/agent-core-v2/src/os/interface/searchOperations.ts @@ -0,0 +1,33 @@ +import type { TextFileResult } from './fileOperations'; + +export interface GlobSearchInput { + readonly pattern: string; + readonly head_limit?: number; + readonly offset?: number; + readonly path?: string; + readonly include_ignored?: boolean; + readonly include_dirs?: boolean; +} + +export interface GrepSearchInput extends GlobSearchInput { + readonly glob?: string; + readonly type?: string; + readonly output_mode?: 'content' | 'files_with_matches' | 'count_matches'; + readonly '-i'?: boolean; + readonly '-n'?: boolean; + readonly '-A'?: number; + readonly '-B'?: number; + readonly '-C'?: number; + readonly multiline?: boolean; +} + +export interface SearchRequest { + readonly kind: 'glob' | 'grep'; + readonly args: GrepSearchInput; + readonly rgPath: string; + readonly roots: readonly string[]; + readonly workspaceDir: string; + readonly pathClass: 'posix' | 'win32'; +} + +export type SearchResult = TextFileResult; diff --git a/packages/agent-core-v2/src/os/operations/editService.ts b/packages/agent-core-v2/src/os/operations/editService.ts new file mode 100644 index 00000000000..3c261d4c9c6 --- /dev/null +++ b/packages/agent-core-v2/src/os/operations/editService.ts @@ -0,0 +1,41 @@ +import type { TextModel } from './textModel'; + +export interface EditApplyInput { + readonly path: string; + readonly old_string: string; + readonly new_string: string; + readonly replace_all: boolean; +} + +export type EditApplyResult = + | { readonly ok: true; readonly rawContent: string; readonly count: number } + | { readonly ok: false; readonly error: string }; + +function notFoundMessage(path: string): string { + return `old_string not found in ${path}, the file contents may be out of date. Please use the Read Tool to reload the content. +`; +} + +function notUniqueMessage(path: string, count: number): string { + return ( + `old_string is not unique in ${path} (found ${String(count)} occurrences). ` + + 'To replace every occurrence, set replace_all=true. To replace only one occurrence, include more surrounding context in old_string.' + ); +} + +export class EditService { + apply(model: TextModel, input: EditApplyInput): EditApplyResult { + if (input.replace_all) { + const { text, count } = model.replaceAll(input.old_string, input.new_string); + if (count === 0) return { ok: false, error: notFoundMessage(input.path) }; + return { ok: true, rawContent: model.materialize(text), count }; + } + + const count = model.countOccurrences(input.old_string); + if (count === 0) return { ok: false, error: notFoundMessage(input.path) }; + if (count > 1) return { ok: false, error: notUniqueMessage(input.path, count) }; + + const text = model.replaceOnce(input.old_string, input.new_string); + return { ok: true, rawContent: model.materialize(text), count: 1 }; + } +} diff --git a/packages/agent-core-v2/src/os/operations/editText.ts b/packages/agent-core-v2/src/os/operations/editText.ts new file mode 100644 index 00000000000..faa24f57876 --- /dev/null +++ b/packages/agent-core-v2/src/os/operations/editText.ts @@ -0,0 +1,32 @@ +import type { IHostFileSystem } from '#/os/interface/hostFileSystem'; +import type { TextEditInput, TextEditResult } from '#/os/interface/fileOperations'; +import { unwrapErrorCause } from '#/_base/errors/errors'; +import { EditService } from './editService'; +import { TextModel } from './textModel'; + +export async function editText(fs: IHostFileSystem, input: TextEditInput): Promise { + try { + const raw = await fs.readText(input.path, { errors: 'strict' }); + const model = new TextModel(raw); + const result = new EditService().apply(model, { + path: input.displayPath, + old_string: input.old_string, + new_string: input.new_string, + replace_all: input.replace_all, + }); + if (!result.ok) { + return { ok: false, error: result.error }; + } + await fs.writeText(input.path, result.rawContent); + return { ok: true, count: result.count }; + } catch (error) { + const code = (unwrapErrorCause(error) as { code?: unknown } | null)?.code; + if (code === 'EISDIR') { + return { ok: false, error: `${input.displayPath} is not a file.` }; + } + return { + ok: false, + error: error instanceof Error ? error.message : String(error), + }; + } +} diff --git a/packages/agent-core-v2/src/os/operations/gitWorkTree.ts b/packages/agent-core-v2/src/os/operations/gitWorkTree.ts new file mode 100644 index 00000000000..40d0c5471ab --- /dev/null +++ b/packages/agent-core-v2/src/os/operations/gitWorkTree.ts @@ -0,0 +1,50 @@ +import { dirname, isAbsolute, join, normalize } from 'pathe'; + +import type { IHostFileSystem } from '#/os/interface/hostFileSystem'; + +import type { GitWorkTree } from '#/os/interface/fileOperations'; + +export async function findGitWorkTree( + fs: Pick, + cwd: string, +): Promise { + if (cwd.length === 0 || !isAbsolute(cwd)) return null; + + let current = normalize(cwd); + while (true) { + const dotGitPath = join(current, '.git'); + const controlDirPath = await probeGitControlDir(fs, dotGitPath, current); + if (controlDirPath !== null) return { root: current, dotGitPath, controlDirPath }; + + const parent = dirname(current); + if (parent === current) return null; + current = parent; + } +} + +async function probeGitControlDir( + fs: Pick, + dotGitPath: string, + markerParent: string, +): Promise { + try { + const stat = await fs.stat(dotGitPath); + if (stat.isDirectory) return dotGitPath; + if (!stat.isFile) return null; + + const content = await fs.readText(dotGitPath); + return parseGitDirPointer(content, markerParent) ?? null; + } catch { + return null; + } +} + +function parseGitDirPointer(content: string, markerParent: string): string | undefined { + const stripped = content.codePointAt(0) === 0xfeff ? content.slice(1) : content; + const line = stripped.trimStart().split(/\r?\n/, 1)[0]?.trim(); + if (line === undefined || !line.startsWith('gitdir:')) return undefined; + + const rawPath = line.slice('gitdir:'.length).trim(); + if (rawPath.length === 0) return undefined; + return normalize(isAbsolute(rawPath) ? rawPath : join(markerParent, rawPath)); +} diff --git a/packages/agent-core-v2/src/os/operations/instructions.ts b/packages/agent-core-v2/src/os/operations/instructions.ts new file mode 100644 index 00000000000..c5bf48ff044 --- /dev/null +++ b/packages/agent-core-v2/src/os/operations/instructions.ts @@ -0,0 +1,107 @@ +import { dirname, join, normalize } from 'pathe'; + +import type { HostFileStat, IHostFileSystem } from '#/os/interface/hostFileSystem'; +import { findGitWorkTree } from './gitWorkTree'; + +export const AGENTS_MD_PLAIN_NAMES = ['AGENTS.md', 'agents.md'] as const; + +export function dotKimiAgentsMdPath(dir: string): string { + return join(dir, '.kimi-code', 'AGENTS.md'); +} + +export function agentsMdCandidatePaths(dir: string): string[] { + return [dotKimiAgentsMdPath(dir), ...AGENTS_MD_PLAIN_NAMES.map((name) => join(dir, name))]; +} + +export async function findAgentsMdInDir( + deps: { readonly fs: Pick }, + dir: string, +): Promise { + const found: string[] = []; + const dotKimi = dotKimiAgentsMdPath(dir); + if (await isNonEmptyFile(deps, dotKimi)) found.push(dotKimi); + for (const fileName of AGENTS_MD_PLAIN_NAMES) { + const candidate = join(dir, fileName); + if (await isNonEmptyFile(deps, candidate)) { + found.push(candidate); + break; + } + } + return found; +} + +async function isNonEmptyFile( + deps: { readonly fs: Pick }, + path: string, +): Promise { + try { + const content = await deps.fs.readText(path, { errors: 'ignore' }); + return content.trim().length > 0; + } catch { + return false; + } +} + +export function dirsRootToLeaf(workDir: string, projectRoot: string): string[] { + const dirs: string[] = []; + let current = normalize(workDir); + + while (true) { + dirs.push(current); + if (current === projectRoot) break; + const parent = dirname(current); + if (parent === current) break; + current = parent; + } + + return dirs.toReversed(); +} + +export async function discoverInstructions(fs: IHostFileSystem, dirs: readonly string[], knownPaths: readonly string[]): Promise { + const known = new Set(knownPaths.map(normalize)); + const stats = new Map>(); + const contents = new Map>(); + const probe: Pick = { + stat: (path) => { + let pending = stats.get(path); + if (pending === undefined) { + pending = fs.stat(path); + stats.set(path, pending); + } + return pending; + }, + readText: (path, options) => { + const key = JSON.stringify([path, options]); + let pending = contents.get(key); + if (pending === undefined) { + pending = fs.readText(path, options); + contents.set(key, pending); + } + return pending; + }, + }; + const found = new Set(); + const inspected = new Set(); + for (const dir of new Set(dirs)) { + let anchor = dir; + let exists = false; + for (;;) { + if ((await probe.stat(anchor).catch(() => undefined))?.isDirectory === true) { + exists = true; + break; + } + const parent = dirname(anchor); + if (parent === anchor) break; + anchor = parent; + } + if (!exists) continue; + const projectRoot = (await findGitWorkTree(probe, anchor))?.root ?? anchor; + for (const chainDir of dirsRootToLeaf(anchor, projectRoot)) { + if (inspected.has(chainDir)) continue; + inspected.add(chainDir); + if (agentsMdCandidatePaths(chainDir).every((path) => known.has(normalize(path)))) continue; + for (const path of await findAgentsMdInDir({ fs }, chainDir)) found.add(normalize(path)); + } + } + return [...found]; +} diff --git a/packages/agent-core-v2/src/os/operations/matchFiles.ts b/packages/agent-core-v2/src/os/operations/matchFiles.ts new file mode 100644 index 00000000000..921047cb956 --- /dev/null +++ b/packages/agent-core-v2/src/os/operations/matchFiles.ts @@ -0,0 +1,72 @@ +import type { IHostFileSystem } from '#/os/interface/hostFileSystem'; +import type { FileMatch, FileMatches, FileMatchOptions, FileMatchPath } from '#/os/interface/fileOperations'; + +export function compileGrepPattern>(req: T): RegExp { + const flags = req.case_sensitive ? 'g' : 'gi'; + const body = req.regex ? req.pattern : escapeRegExp(req.pattern); + return new RegExp(body, flags); +} + +function escapeRegExp(s: string): string { + return s.replaceAll(/[.*+?^${}()|[\]\\]/g, '\\$&'); +} + +export async function matchFiles(fs: IHostFileSystem, paths: readonly FileMatchPath[], options: FileMatchOptions, timeoutMs: number, signal?: AbortSignal, maxResponseBytes = Infinity): Promise { + const re = compileGrepPattern(options); + const files: FileMatches['files'] = []; + let filesScanned = 0; + let totalMatches = 0; + let truncated = false; + const deadline = Date.now() + timeoutMs; + let responseBytes = 0; + for (const file of paths) { + if ((signal?.aborted === true || Date.now() >= deadline) && filesScanned > 0) { + truncated = true; + break; + } + if (filesScanned >= options.max_files) { + truncated = true; + break; + } + filesScanned += 1; + let content: string; + try { + content = await fs.readText(file.absolute); + } catch { + continue; + } + const lines = content.split(/\r?\n/); + const matches: FileMatch[] = []; + for (let i = 0; i < lines.length; i++) { + const line = lines[i]!; + re.lastIndex = 0; + const m = re.exec(line); + if (m === null) continue; + if (matches.length >= options.max_matches_per_file) break; + const before: string[] = []; + for (let k = Math.max(0, i - options.context_lines); k < i; k++) { + before.push(lines[k] ?? ''); + } + const after: string[] = []; + for (let k = i + 1; k < Math.min(lines.length, i + 1 + options.context_lines); k++) { + after.push(lines[k] ?? ''); + } + matches.push({ line: i + 1, col: m.index + 1, text: line, before, after }); + totalMatches += 1; + if (totalMatches >= options.max_total_matches) { + truncated = true; + break; + } + } + if (matches.length > 0) { + responseBytes += Buffer.byteLength(JSON.stringify({ path: file.path, matches })); + if (responseBytes > maxResponseBytes) { + return { files: [], files_scanned: 0, truncated: false, responseTooLarge: true }; + } + files.push({ path: file.path, matches }); + } + if (totalMatches >= options.max_total_matches) break; + } + + return { files, files_scanned: filesScanned, truncated }; +} diff --git a/packages/agent-core-v2/src/os/operations/readTextPage.ts b/packages/agent-core-v2/src/os/operations/readTextPage.ts new file mode 100644 index 00000000000..ea59f8e9e48 --- /dev/null +++ b/packages/agent-core-v2/src/os/operations/readTextPage.ts @@ -0,0 +1,481 @@ +import type { IHostFileSystem } from '#/os/interface/hostFileSystem'; +import type { FileReadSource, TextPageInput, TextPageOptions, TextFileResult } from '#/os/interface/fileOperations'; +import { isHostFsNotDirectory, isHostFsNotFound } from '#/os/interface/hostFsErrors'; +import { unwrapErrorCause } from '#/_base/errors/errors'; +import { MEDIA_SNIFF_BYTES, detectFileType } from '#/_base/media/fileType'; +import { makeCarriageReturnsVisible, splitLinesKeepingTerminator, type LineEndingStyle } from '#/_base/text/line-endings'; +import { detectTextEncoding, type UtfTextEncoding } from '#/_base/text/encoding'; +import { renderTextStatus } from '#/_base/text/resultStatus'; + +export const TRANSCODE_MAX_BYTES = 10 * 1024 * 1024; + +interface LineEndingFlags { + hasCrLf: boolean; + hasLf: boolean; + hasLoneCr: boolean; +} + +interface ReadLineEntry { + readonly lineNo: number; + readonly rawContent: string; +} + +interface ReadTailEntry extends ReadLineEntry { + readonly minChars: number; +} + +interface ReadRequest { + readonly args: TextPageInput; + readonly maxChars: number; + readonly maxCharsLimit: number; + readonly detectedEncoding?: UtfTextEncoding; + readonly lossyDecoding: boolean; + readonly eventLog: boolean; + readonly signal?: AbortSignal; +} + +interface ReadPage { + readonly request: ReadRequest; + readonly renderedLines: readonly string[]; + readonly startLine: number; + readonly rangeStart: number; + readonly rangeEnd: number; + readonly totalLines: number; + readonly fromTail: boolean; + readonly lineEndingStyle: LineEndingStyle; +} + +function stripTrailingLf(line: string): string { + return line.endsWith('\n') ? line.slice(0, -1) : line; +} + +function splitsSurrogatePair(text: string, offset: number): boolean { + // oxlint-disable-next-line unicorn/prefer-code-point -- raw UTF-16 halves are the point here + const previous = text.charCodeAt(offset - 1); + // oxlint-disable-next-line unicorn/prefer-code-point -- raw UTF-16 halves are the point here + const next = text.charCodeAt(offset); + return previous >= 0xd800 && previous <= 0xdbff && next >= 0xdc00 && next <= 0xdfff; +} + +function updateLineEndingFlags(flags: LineEndingFlags, text: string): void { + for (let i = 0; i < text.length; i += 1) { + const code = text.codePointAt(i); + if (code === 13) { + if (text.codePointAt(i + 1) === 10) { + flags.hasCrLf = true; + i += 1; + } else { + flags.hasLoneCr = true; + } + } else if (code === 10) { + flags.hasLf = true; + } + } +} + +function lineEndingStyleFromFlags(flags: LineEndingFlags): LineEndingStyle { + if (flags.hasLoneCr || (flags.hasCrLf && flags.hasLf)) return 'mixed'; + if (flags.hasCrLf) return 'crlf'; + return 'lf'; +} + +function renderLine(entry: ReadLineEntry, lineEndingStyle: LineEndingStyle): string { + const modelContent = + lineEndingStyle === 'crlf' && entry.rawContent.endsWith('\r') + ? entry.rawContent.slice(0, -1) + : entry.rawContent; + const renderedContent = + lineEndingStyle === 'mixed' ? makeCarriageReturnsVisible(modelContent) : modelContent; + return `${String(entry.lineNo)}\t${renderedContent}`; +} + +function isTextDecodeError(error: unknown): boolean { + const unwrapped = unwrapErrorCause(error); + if (typeof unwrapped !== 'object' || unwrapped === null) return false; + const code = (unwrapped as { code?: unknown })['code']; + if (code === 'ERR_ENCODING_INVALID_ENCODED_DATA') return true; + if (!(unwrapped instanceof Error)) return false; + return /encoded data was not valid|invalid.*encoding|invalid.*utf-?8/i.test(unwrapped.message); +} + +function containsNulByte(text: string): boolean { + return text.includes('\u0000'); +} + +function encodingDisplayName(encoding: UtfTextEncoding): string { + switch (encoding) { + case 'utf-16le': + return 'UTF-16 LE'; + case 'utf-16be': + return 'UTF-16 BE'; + case 'utf-8': + return 'UTF-8'; + } +} + +async function* decodedLines(lines: readonly string[]): AsyncGenerator { + yield* lines; +} + +function notReadableFileOutput(path: string): string { + return `"${path}" is not readable as UTF-8 text. Only text files can be read.`; +} + +function notUtf8DecodableFileOutput(path: string): string { + return ( + `"${path}" is not valid UTF-8 or UTF-16 text. ` + + 'Only UTF-8 and UTF-16 text files can be read; ' + + 'for other encodings (e.g. GBK), convert the file to UTF-8 first (e.g. with `iconv`).' + ); +} + +export async function readTextPage( + source: FileReadSource, + options: TextPageOptions, + signal?: AbortSignal, +): Promise { + const { args, eventLog } = options; + try { + signal?.throwIfAborted(); + let stat: Awaited>; + try { + stat = await source.stat(); + } catch (error) { + if (isHostFsNotFound(error) || isHostFsNotDirectory(error)) { + return { isError: true, output: `"${args.path}" does not exist.` }; + } + throw error; + } + if (!stat.isFile) { + return { isError: true, output: `"${args.path}" is not a file.` }; + } + + const header = await source.readBytes(MEDIA_SNIFF_BYTES); + const fileType = detectFileType(source.name, header); + if (fileType.kind === 'image' || fileType.kind === 'video') { + const kind = fileType.kind; + const article = kind === 'image' ? 'an' : 'a'; + const supported = kind === 'image' ? options.imageSupported : options.videoSupported; + if (!supported) { + return { + isError: true, + output: `"${args.path}" is ${article} ${kind} file. The current model does not support ${kind} input (missing ${kind}_in capability), so this agent cannot view it. Only text files can be read.`, + }; + } + const mediaToolAvailable = options.mediaToolAvailable; + return { + isError: true, + output: mediaToolAvailable + ? `"${args.path}" is ${article} ${kind} file. Only text files can be read; use ReadMediaFile for image and video files.` + : `"${args.path}" is ${article} ${kind} file. Only text files can be read.`, + }; + } + + const detection = detectTextEncoding(header); + let readLines: () => AsyncIterable; + let detectedEncoding: UtfTextEncoding | undefined; + let lossyDecoding = false; + if (!detection.seemsBinary && detection.encoding !== 'utf-8') { + if (stat.size > TRANSCODE_MAX_BYTES) { + return { + isError: true, + output: + `"${args.path}" is ${encodingDisplayName(detection.encoding)} text but too large to transcode ` + + `(${String(stat.size)} bytes > ${String(TRANSCODE_MAX_BYTES)}). ` + + 'Convert it to UTF-8 first (e.g. with `iconv`).', + }; + } + const bytes = await source.readBytes(); + let decoded: string; + try { + decoded = new TextDecoder(detection.encoding, { fatal: true }).decode(bytes); + } catch (error) { + if (!isTextDecodeError(error)) throw error; + decoded = new TextDecoder(detection.encoding, { fatal: false }).decode(bytes); + lossyDecoding = true; + } + detectedEncoding = detection.encoding; + const decodedContent = splitLinesKeepingTerminator(decoded); + readLines = () => decodedLines(decodedContent); + } else if (fileType.kind === 'unknown') { + return { + isError: true, + output: notReadableFileOutput(args.path), + }; + } else { + readLines = () => source.readLines(); + } + + const request: ReadRequest = { + args, + maxChars: options.maxChars, + maxCharsLimit: options.maxCharsLimit, + detectedEncoding, + lossyDecoding, + eventLog, + signal, + }; + const lineOffset = args.line_offset ?? 1; + if (lineOffset >= 0) return await readForward(readLines(), request); + const rereadsFile = detectedEncoding === undefined && (args.n_lines ?? Infinity) < -lineOffset; + const result = await readTail(readLines, request); + if (!result.isError && rereadsFile) { + const currentStat = await source.stat(); + if (!currentStat.isFile || currentStat.size !== stat.size || + currentStat.mtimeMs !== stat.mtimeMs || currentStat.ino !== stat.ino) { + return { isError: true, output: 'File changed while reading its tail. Retry Read with the updated file.' }; + } + } + return result; + } catch (error) { + if (isTextDecodeError(error)) { + return { isError: true, output: notUtf8DecodableFileOutput(args.path) }; + } + return { + isError: true, + output: error instanceof Error ? error.message : String(error), + }; + } +} + +async function readForward( + lines: AsyncIterable, + request: ReadRequest, +): Promise { + const { args, maxChars } = request; + const lineOffset = args.line_offset ?? 1; + const columnOffset = args.column_offset ?? 0; + const requestedLines = args.n_lines ?? Infinity; + const selectedEntries: ReadLineEntry[] = []; + const flags: LineEndingFlags = { hasCrLf: false, hasLf: false, hasLoneCr: false }; + let currentLineNo = 0; + let collectionClosed = false; + let minimumChars = 0; + + for await (const rawLine of lines) { + request.signal?.throwIfAborted(); + if (containsNulByte(rawLine)) { + return { isError: true, output: notReadableFileOutput(args.path) }; + } + currentLineNo += 1; + updateLineEndingFlags(flags, rawLine); + if (collectionClosed) continue; + if (currentLineNo < lineOffset) continue; + if (selectedEntries.length >= requestedLines) { + collectionClosed = true; + continue; + } + const rawContent = stripTrailingLf(rawLine); + const lineChars = String(currentLineNo).length + 1 + Math.max( + 0, + rawContent.length - (rawContent.endsWith('\r') ? 1 : 0) - + (currentLineNo === lineOffset ? columnOffset : 0), + ) + (selectedEntries.length === 0 ? 0 : 1); + if (minimumChars + lineChars > maxChars && selectedEntries.length > 0) { + collectionClosed = true; + continue; + } + selectedEntries.push({ + lineNo: currentLineNo, + rawContent, + }); + minimumChars += lineChars; + if (selectedEntries.length >= requestedLines || minimumChars >= maxChars) { + collectionClosed = true; + } + } + + const lineEndingStyle = lineEndingStyleFromFlags(flags); + const renderedLines = selectedEntries.map((entry) => renderLine(entry, lineEndingStyle)); + const firstLine = renderedLines[0]; + if (columnOffset > 0) { + const prefix = `${String(lineOffset)}\t`; + const text = firstLine?.slice(prefix.length); + if (text === undefined || columnOffset > text.length) { + return { isError: true, output: `column_offset=${String(columnOffset)} is past the end of the starting line ${String(lineOffset)}. Read the line from column 0 to inspect its current contents.` }; + } + if (splitsSurrogatePair(text, columnOffset)) { + return { isError: true, output: `column_offset=${String(columnOffset)} splits a Unicode character in line ${String(lineOffset)}. Use a character boundary or the Next Read arguments.` }; + } + renderedLines[0] = prefix + text.slice(columnOffset); + } + return finishPage({ + request, + renderedLines, + startLine: lineOffset, + rangeStart: lineOffset, + rangeEnd: Math.min(currentLineNo, lineOffset + requestedLines - 1), + totalLines: currentLineNo, + fromTail: false, + lineEndingStyle, + }); +} + +function finishPage(page: ReadPage): TextFileResult { + const { args, maxChars, maxCharsLimit, eventLog, detectedEncoding, lossyDecoding } = page.request; + let first = 0; + let end = page.renderedLines.length; + let contentChars = page.renderedLines.reduce((sum, line) => sum + line.length + 1, -1); + const firstColumn = page.fromTail ? 0 : args.column_offset ?? 0; + const firstPrefix = `${String(page.startLine)}\t`; + const firstText = page.renderedLines[0]?.slice(firstPrefix.length) ?? ''; + let fragmentEnd: number | undefined; + + while (true) { + const count = end - first; + const startLine = page.startLine + first; + const endLine = page.startLine + end - 1; + const lineIncomplete = fragmentEnd !== undefined; + const complete = page.rangeStart > page.rangeEnd || + (count > 0 && startLine === page.rangeStart && endLine === page.rangeEnd && !lineIncomplete); + const parts = [ + count > 0 + ? `${String(count)} ${count === 1 ? 'line' : 'lines'} read from file starting from line ${String(startLine)}.` + : 'No lines read from file.', + `Total lines in file: ${String(page.totalLines)}.`, + complete ? 'Requested range complete.' : 'Character limit reached.', + `Effective max_chars: ${String(maxChars)}.`, + ]; + if (!lineIncomplete && ((count > 0 && endLine === page.totalLines) || page.rangeStart > page.totalLines)) { + parts.push('End of file reached.'); + } + if (count > 0 && !page.fromTail && (firstColumn > 0 || fragmentEnd !== undefined)) { + parts.push(`Line ${String(startLine)} fragment: columns [${String(firstColumn)}, ${String(firstColumn + (fragmentEnd ?? firstText.length))}) of ${String(firstColumn + firstText.length)}. ${lineIncomplete ? 'Line continues.' : 'Line complete.'}`); + } + if (args.max_chars !== undefined && args.max_chars > maxCharsLimit) { + parts.push(`Requested max_chars=${String(args.max_chars)} was capped at the configured maximum ${String(maxCharsLimit)}.`); + } + if (!complete && (count > 0 || page.fromTail)) { + const nextStart = page.fromTail ? page.rangeStart : lineIncomplete ? startLine : endLine + 1; + const nextEnd = page.fromTail && count > 0 ? startLine - 1 : page.rangeEnd; + const next = { + path: args.path, + line_offset: nextStart, + column_offset: fragmentEnd !== undefined ? firstColumn + fragmentEnd : undefined, + n_lines: page.fromTail || args.n_lines !== undefined ? nextEnd - nextStart + 1 : undefined, + max_chars: maxChars, + }; + parts.push(`Next Read: ${JSON.stringify(next)}`); + } + if (eventLog) { + parts.push('Kimi Code agent event log: read one record at a time (n_lines=1); increase max_chars for a longer record or extract fields with Bash.'); + } + if (page.lineEndingStyle === 'mixed') { + parts.push('Mixed or lone carriage-return line endings are shown as \\r. Use exact \\r\\n or \\r escapes in Edit.old_string for those lines.'); + } + if (detectedEncoding !== undefined) { + parts.push(`Detected file encoding: ${encodingDisplayName(detectedEncoding)}; content transcoded to UTF-8 for display. Edit and Write expect UTF-8 — convert the file's encoding first (e.g. \`iconv\` via Bash).`); + } + if (lossyDecoding) { + parts.push('Lossy UTF-16 decoding: malformed sequences were replaced with U+FFFD. The decoded text may differ from the original file.'); + } + const note = `${parts.join(' ')}`; + const renderedChars = count === 0 + ? renderTextStatus('').length + 1 + note.length + : contentChars + 1 + note.length; + if (renderedChars <= maxChars && (complete || count > 0)) { + return { + output: fragmentEnd !== undefined + ? firstPrefix + firstText.slice(0, fragmentEnd) + : page.renderedLines.slice(first, end).join('\n'), + note, + truncated: complete ? undefined : true, + }; + } + if (count === 1 && !page.fromTail) { + const previousEnd = fragmentEnd ?? firstText.length; + fragmentEnd = Math.min(previousEnd - 1, previousEnd - (renderedChars - maxChars)); + if (splitsSurrogatePair(firstText, fragmentEnd)) fragmentEnd -= 1; + if (fragmentEnd <= 0) { + return { isError: true, output: `max_chars=${String(maxChars)} is too small for file text and the Read status. Increase max_chars.` }; + } + contentChars = firstPrefix.length + fragmentEnd; + continue; + } + if (count === 0) { + if (!complete) { + const recovery: TextFileResult = { + isError: true, + output: 'No complete line fits. Continue with the forward Next Read.', + note, + truncated: true, + }; + const recoveryChars = renderTextStatus(recovery.output, true).length + 1 + note.length; + if (recoveryChars <= maxChars) return recovery; + } + return { isError: true, output: `max_chars=${String(maxChars)} is too small for the Read status. Increase max_chars.` }; + } + const dropped = page.fromTail ? first++ : --end; + contentChars -= page.renderedLines[dropped]!.length + 1; + } +} + +async function readTail( + readLines: () => AsyncIterable, + request: ReadRequest, +): Promise { + const { args, maxChars } = request; + const lineOffset = args.line_offset ?? 1; + const requestedLines = args.n_lines ?? Infinity; + const tailCount = -lineOffset; + const singlePass = requestedLines >= tailCount; + let entries: (ReadTailEntry | undefined)[] = []; + let first = 0; + let chars = 0; + const retainLine = (rawLine: string, lineNo: number): void => { + const rawContent = stripTrailingLf(rawLine); + const minChars = String(lineNo).length + 2 + rawContent.length - (rawContent.endsWith('\r') ? 1 : 0); + entries.push({ lineNo, rawContent, minChars }); + chars += minChars; + while (first < entries.length && (chars - 1 > maxChars || entries.length - first > tailCount)) { + chars -= entries[first]!.minChars; + entries[first++] = undefined; + } + if (first > 1024 && first >= entries.length / 2) { + entries = entries.slice(first); + first = 0; + } + }; + const flags: LineEndingFlags = { hasCrLf: false, hasLf: false, hasLoneCr: false }; + let totalLines = 0; + for await (const rawLine of readLines()) { + request.signal?.throwIfAborted(); + if (containsNulByte(rawLine)) { + return { isError: true, output: notReadableFileOutput(args.path) }; + } + totalLines += 1; + updateLineEndingFlags(flags, rawLine); + if (singlePass) retainLine(rawLine, totalLines); + } + + const rangeStart = Math.max(1, totalLines + lineOffset + 1); + const rangeEnd = Math.min(totalLines, rangeStart + requestedLines - 1); + const lineEndingStyle = lineEndingStyleFromFlags(flags); + if (!singlePass) { + let currentLine = 0; + for await (const rawLine of readLines()) { + request.signal?.throwIfAborted(); + currentLine += 1; + if (currentLine > rangeEnd) break; + if (currentLine < rangeStart) continue; + if (containsNulByte(rawLine)) { + return { isError: true, output: notReadableFileOutput(args.path) }; + } + retainLine(rawLine, currentLine); + } + if (currentLine < rangeEnd || currentLine > totalLines) { + return { isError: true, output: 'File changed while reading its tail. Retry Read with the updated file.' }; + } + } + const selected = entries.slice(first).map((entry) => renderLine(entry!, lineEndingStyle)); + return finishPage({ + request, + renderedLines: selected, + startLine: rangeEnd - selected.length + 1, + rangeStart, + rangeEnd, + totalLines, + fromTail: true, + lineEndingStyle, + }); +} diff --git a/packages/agent-core-v2/src/os/operations/runRg.ts b/packages/agent-core-v2/src/os/operations/runRg.ts new file mode 100644 index 00000000000..bf48db4c448 --- /dev/null +++ b/packages/agent-core-v2/src/os/operations/runRg.ts @@ -0,0 +1,202 @@ +import type { Readable } from 'node:stream'; + +import { BugIndicatingError } from '#/_base/errors/errors'; +import type { IHostProcess, IHostProcessService } from '#/os/interface/hostProcess'; + +export const DEFAULT_TIMEOUT_MS = 20_000; +export const SIGTERM_GRACE_MS = 5_000; +export const MAX_OUTPUT_BYTES = 10 * 1024 * 1024; + +export interface RunRgResult { + readonly kind: 'result'; + readonly exitCode: number; + readonly stdoutText: string; + readonly stderrText: string; + readonly bufferTruncated: boolean; + readonly stderrTruncated: boolean; + readonly timedOut: boolean; + readonly outputLimitReached?: boolean; +} + +export type RunRgOutcome = RunRgResult | { readonly kind: 'aborted' }; + +function disposeProcess(proc: IHostProcess): void { + try { + void proc.dispose(); + } catch { + } +} + +export async function runRgOnce( + processService: IHostProcessService, + rgArgs: readonly string[], + signal: AbortSignal, + options?: { readonly cwd?: string }, +): Promise { + if (signal.aborted) { + return { kind: 'aborted' }; + } + + const [command, ...args] = rgArgs; + if (command === undefined) { + throw new BugIndicatingError('runRgOnce: rgArgs must not be empty'); + } + const proc: IHostProcess = await processService.spawn(command, args, { cwd: options?.cwd }); + + try { + proc.stdin.end(); + } catch { + } + + let timedOut = false; + let aborted = false; + let killed = false; + let outputLimitReached = false; + + const killProc = async (): Promise => { + if (killed) return; + killed = true; + try { + await proc.kill('SIGTERM'); + } catch { + } + const exited = proc + .wait() + .then(() => true) + .catch(() => true); + let graceTimer: ReturnType | undefined; + const raced = await Promise.race([ + exited, + new Promise((resolve) => { + graceTimer = setTimeout(() => { + resolve(false); + }, SIGTERM_GRACE_MS); + }), + ]); + clearTimeout(graceTimer); + if (!raced && proc.exitCode === null) { + try { + await proc.kill('SIGKILL'); + } catch { + } + } + disposeProcess(proc); + }; + + const onAbort = (): void => { + aborted = true; + void killProc(); + }; + signal.addEventListener('abort', onAbort); + if (signal.aborted) onAbort(); + + const timeoutHandle = setTimeout(() => { + timedOut = true; + void killProc(); + }, DEFAULT_TIMEOUT_MS); + + let exitCode = 0; + let stdoutText = ''; + let stderrText = ''; + let bufferTruncated = false; + let stderrTruncated = false; + + try { + const isTerminating = (): boolean => timedOut || aborted || killed; + const [stdoutResult, stderrResult, code] = await Promise.all([ + readStreamWithCap(proc.stdout, MAX_OUTPUT_BYTES, isTerminating, () => { + outputLimitReached = true; + void killProc(); + }), + readStreamWithCap(proc.stderr, MAX_OUTPUT_BYTES, isTerminating), + proc.wait(), + ]); + stdoutText = stdoutResult.text; + stderrText = stderrResult.text; + bufferTruncated = stdoutResult.truncated; + stderrTruncated = stderrResult.truncated; + exitCode = code; + } catch (error) { + if (!(isPrematureCloseError(error) && (timedOut || aborted || killed))) { + throw error; + } + } finally { + clearTimeout(timeoutHandle); + signal.removeEventListener('abort', onAbort); + disposeProcess(proc); + } + + if (aborted) { + return { kind: 'aborted' }; + } + + return { + kind: 'result', + exitCode, + stdoutText, + stderrText, + bufferTruncated, + stderrTruncated, + timedOut, + outputLimitReached, + }; +} + +export function shouldRetryRipgrepEagain(result: RunRgResult): boolean { + return ( + result.exitCode !== 0 && + result.exitCode !== 1 && + !result.timedOut && + !result.outputLimitReached && + isEagainRipgrepError(result.stderrText) + ); +} + +function isEagainRipgrepError(stderr: string): boolean { + return stderr.includes('os error 11') || stderr.includes('Resource temporarily unavailable'); +} + +function isPrematureCloseError(error: unknown): boolean { + return ( + error instanceof Error && + (error as NodeJS.ErrnoException).code === 'ERR_STREAM_PREMATURE_CLOSE' + ); +} + +interface CappedStreamResult { + readonly text: string; + readonly truncated: boolean; +} + +async function readStreamWithCap( + stream: Readable, + maxBytes: number, + suppressPrematureClose?: () => boolean, + onLimit?: () => void, +): Promise { + const chunks: Buffer[] = []; + let total = 0; + let truncated = false; + try { + for await (const chunk of stream) { + const buf: Buffer = + typeof chunk === 'string' ? Buffer.from(chunk, 'utf8') : (chunk as Buffer); + if (truncated) continue; + if (total + buf.length > maxBytes) { + const remaining = maxBytes - total; + if (remaining > 0) chunks.push(buf.subarray(0, remaining)); + total = maxBytes; + truncated = true; + onLimit?.(); + continue; + } + chunks.push(buf); + total += buf.length; + } + } catch (error) { + if (!isPrematureCloseError(error) || suppressPrematureClose?.() !== true) { + throw error; + } + } + return { text: Buffer.concat(chunks).toString('utf8'), truncated }; +} diff --git a/packages/agent-core-v2/src/os/operations/searchGlob.ts b/packages/agent-core-v2/src/os/operations/searchGlob.ts new file mode 100644 index 00000000000..3d914b2b3b6 --- /dev/null +++ b/packages/agent-core-v2/src/os/operations/searchGlob.ts @@ -0,0 +1,208 @@ +import { normalize, resolve } from 'pathe'; +import type { IHostProcessService } from '#/os/interface/hostProcess'; +import type { SearchRequest, SearchResult, GlobSearchInput as GlobInput } from '#/os/interface/searchOperations'; +import { isSensitiveFile, SENSITIVE_DOT_VARIANT_SUFFIXES, isWithinDirectory } from '#/_base/fs/pathFilters'; +import { DEFAULT_TOOL_RESULT_MAX_RETAINED_CHARS } from '#/_base/text/outputLimits'; +import { DEFAULT_TIMEOUT_MS, MAX_OUTPUT_BYTES, runRgOnce, shouldRetryRipgrepEagain } from './runRg'; + +type PathClass = 'posix' | 'win32'; +const DEFAULT_HEAD_LIMIT = 100; + +const VCS_DIRECTORIES_TO_EXCLUDE = ['.git', '.svn', '.hg', '.bzr', '.jj', '.sl'] as const; + +const SENSITIVE_KEY_BASENAMES = ['id_rsa', 'id_ed25519', 'id_ecdsa'] as const; +const SENSITIVE_GLOBS_TO_EXCLUDE: readonly string[] = [ + '**/.env', + ...SENSITIVE_KEY_BASENAMES.flatMap((name) => [ + `**/${name}`, + `**/${name}[-_]*`, + ...SENSITIVE_DOT_VARIANT_SUFFIXES.map((suffix) => `**/${name}${suffix}`), + ]), + '**/.aws/credentials', + '**/.aws/credentials/**', + '**/.gcp/credentials', + '**/.gcp/credentials/**', +]; + +export async function searchGlob(processService: IHostProcessService, request: SearchRequest, signal: AbortSignal): Promise { + const { args, rgPath, roots: searchPaths, workspaceDir, pathClass } = request; + const searchRoot = searchPaths[0] ?? workspaceDir; + let run; + run = await runRgOnce(processService, buildRgArgs(rgPath, args), signal, { cwd: searchRoot }); + if (run.kind === 'aborted') { + return { isError: true, output: 'Glob aborted' }; + } + + if (shouldRetryRipgrepEagain(run)) { + run = await runRgOnce(processService, buildRgArgs(rgPath, args, true), signal, { cwd: searchRoot }); + if (run.kind === 'aborted') { + return { isError: true, output: 'Glob aborted' }; + } + } + + const { exitCode, stdoutText, stderrText, bufferTruncated, timedOut } = run; + + let traversalWarning: string | undefined; + if (exitCode !== 0 && exitCode !== 1 && !timedOut && !run.outputLimitReached) { + const rawPathsBeforeError = splitCompletePaths(stdoutText, true); + if (rawPathsBeforeError.length === 0) { + return { isError: true, output: formatGlobError(searchRoot, stderrText) }; + } + traversalWarning = formatGlobWarning(stderrText); + } + if (signal.aborted) { + return { isError: true, output: 'Glob aborted' }; + } + + const rawPaths = splitCompletePaths(stdoutText, bufferTruncated || timedOut).map((p) => + resolve(searchRoot, p), + ); + + const kept: string[] = []; + let filteredSensitive = 0; + for (const p of rawPaths) { + if (isSensitiveFile(p)) { + filteredSensitive++; + } else { + kept.push(p); + } + } + + const offset = args.offset ?? 0; + const headLimit = args.head_limit ?? DEFAULT_HEAD_LIMIT; + const limited = headLimit === 0 ? kept.slice(offset) : kept.slice(offset, offset + headLimit); + const partial = bufferTruncated || timedOut || traversalWarning !== undefined; + + const shouldRelativize = isWithinDirectory(searchRoot, workspaceDir, pathClass); + const candidates = limited.map((p) => + shouldRelativize ? relativizeIfUnder(p, searchRoot, pathClass) : p, + ); + + const warnings: string[] = []; + if (timedOut) { + warnings.push( + `Glob timed out after ${String(DEFAULT_TIMEOUT_MS / 1000)}s; partial results returned.`, + ); + } + if (bufferTruncated) { + warnings.push( + `[stdout truncated at ${String(MAX_OUTPUT_BYTES)} bytes; results may be incomplete — use a more specific pattern]`, + ); + } + if (traversalWarning !== undefined) { + warnings.push(traversalWarning); + } + const pageNotices = (count: number, characterLimited: boolean) => { + const lines = [...warnings]; + const footer: string[] = []; + const truncated = characterLimited || offset + count < kept.length; + if (count === 0) { + if (kept.length > 0) { + const resultSet = partial ? 'collected partial result set' : 'current result set'; + lines.push( + `No more matches at offset=${String(offset)} in the ${resultSet} (${String(kept.length)} matches).`, + ); + } else if (partial) { + lines.push('No matches collected; search incomplete.'); + } else if (filteredSensitive > 0) { + lines.push( + `No non-sensitive matches found (${String(filteredSensitive)} sensitive file(s) filtered).`, + ); + } else { + lines.push('No matches found'); + } + } else if (truncated || offset > 0 || partial) { + const total = partial + ? `${String(kept.length)} collected matches (partial result set)` + : String(kept.length); + lines.push(`Showing matches ${String(offset + 1)}–${String(offset + count)} of ${total}.`); + } + if (characterLimited) lines.push('Character limit reached; only complete paths are returned.'); + if (truncated) { + lines.push( + `Continue with the same search arguments and offset=${String(offset + count)}.`, + ); + if (!characterLimited) lines.push('To remove the match-count limit, omit offset and use head_limit=0.'); + } + if (filteredSensitive > 0 && (kept.length > 0 || partial)) { + footer.push(`Filtered ${String(filteredSensitive)} sensitive file(s).`); + } + if (!truncated && !partial && offset === 0 && headLimit > 0 && count === headLimit) { + footer.push(`Found ${String(count)} matches`); + } + return { lines, footer }; + }; + const noticeChars = Math.max(...[false, true].map((characterLimited) => { + const { lines, footer } = pageNotices(candidates.length, characterLimited); + return [...lines, ...footer].join('\n').length + 2; + })); + let remaining = DEFAULT_TOOL_RESULT_MAX_RETAINED_CHARS - noticeChars; + const displayLines: string[] = []; + for (const path of candidates) { + if (path.length + 1 > remaining) break; + displayLines.push(path); + remaining -= path.length + 1; + } + if (candidates.length > 0 && displayLines.length === 0) { + return { + isError: true, + output: 'Glob cannot fit a complete path and its diagnostics within the output limit. Narrow the search path or pattern.', + }; + } + const notices = pageNotices(displayLines.length, displayLines.length < candidates.length); + return { output: [...notices.lines, ...displayLines, ...notices.footer].join('\n') }; + +} + +function buildRgArgs(rgPath: string, args: GlobInput, singleThreaded = false): string[] { + const cmd: string[] = [rgPath]; + if (singleThreaded) cmd.push('-j', '1'); + cmd.push('--files', '--hidden', '--sortr=modified'); + for (const dir of VCS_DIRECTORIES_TO_EXCLUDE) { + cmd.push('--glob', `!${dir}`); + } + cmd.push('--glob', args.pattern); + for (const glob of SENSITIVE_GLOBS_TO_EXCLUDE) { + cmd.push('--glob', `!${glob}`); + } + if (args.include_ignored) cmd.push('--no-ignore'); + cmd.push('.'); + return cmd; +} + +function formatGlobError(searchRoot: string, stderr: string): string { + const trimmed = stderr.trim(); + if (/no such file or directory/i.test(trimmed)) { + return `${searchRoot} does not exist`; + } + return trimmed.length > 0 ? `Glob failed: ${trimmed}` : 'Glob failed'; +} + +function formatGlobWarning(stderr: string): string { + const trimmed = stderr.trim(); + return trimmed.length > 0 + ? `Glob completed with warnings; some directories could not be read: ${trimmed}` + : 'Glob completed with warnings; some directories could not be read.'; +} + +export function splitCompletePaths(stdoutText: string, truncatedOutput: boolean): string[] { + let text = stdoutText; + if (truncatedOutput && !text.endsWith('\n')) { + const lastNewline = text.lastIndexOf('\n'); + text = lastNewline >= 0 ? text.slice(0, lastNewline + 1) : ''; + } + return text.split('\n').filter((p) => p.length > 0); +} + +function relativizeIfUnder(candidate: string, base: string, pathClass: PathClass): string { + const normCandidate = normalize(candidate); + const normBase = normalize(base); + const comparableCandidate = pathClass === 'win32' ? normCandidate.toLowerCase() : normCandidate; + const comparableBase = pathClass === 'win32' ? normBase.toLowerCase() : normBase; + if (comparableCandidate === comparableBase) return '.'; + const prefix = comparableBase.endsWith('/') ? comparableBase : comparableBase + '/'; + if (comparableCandidate.startsWith(prefix)) { + return normCandidate.slice(prefix.length); + } + return normCandidate; +} diff --git a/packages/agent-core-v2/src/os/operations/searchGrep.ts b/packages/agent-core-v2/src/os/operations/searchGrep.ts new file mode 100644 index 00000000000..f78f28904b8 --- /dev/null +++ b/packages/agent-core-v2/src/os/operations/searchGrep.ts @@ -0,0 +1,502 @@ +import { normalize } from 'pathe'; +import type { IHostProcessService } from '#/os/interface/hostProcess'; +import type { SearchRequest, SearchResult, GrepSearchInput as GrepInput } from '#/os/interface/searchOperations'; +import { isSensitiveFile, SENSITIVE_DOT_VARIANT_SUFFIXES } from '#/_base/fs/pathFilters'; +import { ToolOutputAccumulator } from '#/_base/text/outputAccumulator'; +import { DEFAULT_TIMEOUT_MS, MAX_OUTPUT_BYTES, runRgOnce, shouldRetryRipgrepEagain, type RunRgResult } from './runRg'; + +type PathClass = 'posix' | 'win32'; + +const RG_MAX_COLUMNS = 500; +const DEFAULT_HEAD_LIMIT = 250; + +const VCS_DIRECTORIES_TO_EXCLUDE = ['.git', '.svn', '.hg', '.bzr', '.jj', '.sl'] as const; +const SENSITIVE_KEY_BASENAMES = ['id_rsa', 'id_ed25519', 'id_ecdsa'] as const; +const SENSITIVE_KEY_GLOBS_TO_EXCLUDE = SENSITIVE_KEY_BASENAMES.flatMap((name) => [ + `**/${name}`, + `**/${name}[-_]*`, + ...SENSITIVE_DOT_VARIANT_SUFFIXES.map((suffix) => `**/${name}${suffix}`), +]); +const SENSITIVE_GLOBS_TO_EXCLUDE = [ + '**/.env', + ...SENSITIVE_KEY_GLOBS_TO_EXCLUDE, + '**/.aws/credentials', + '**/.aws/credentials/**', + '**/.gcp/credentials', + '**/.gcp/credentials/**', +] as const; + +const CONTENT_LINE_RE = /^(.*?)([:-])(\d+)\2/; + +export async function searchGrep(processService: IHostProcessService, request: SearchRequest, signal: AbortSignal): Promise { + const { args, rgPath, roots: searchPaths, workspaceDir, pathClass } = request; + let runResult: RunRgResult; + const firstRun = await runRgOnce( + processService, + buildRgArgs(rgPath, args, searchPaths), + signal, + ); + if (firstRun.kind === 'aborted') { + return { isError: true, output: 'Grep aborted' }; + } + runResult = firstRun; + + if (shouldRetryRipgrepEagain(runResult)) { + const retryRun = await runRgOnce( + processService, + buildRgArgs(rgPath, args, searchPaths, true), + signal, + ); + if (retryRun.kind === 'aborted') { + return { isError: true, output: 'Grep aborted' }; + } + runResult = retryRun; + } + + + const { exitCode, stderrText, bufferTruncated, stderrTruncated, timedOut } = runResult; + let { stdoutText } = runResult; + + if (exitCode !== 0 && exitCode !== 1 && !timedOut && !runResult.outputLimitReached) { + return { + isError: true, + output: formatRipgrepError(exitCode, stderrText, stderrTruncated), + }; + } + + const mode = args.output_mode ?? 'files_with_matches'; + if (bufferTruncated || timedOut) { + stdoutText = omitIncompleteTrailingRecord(stdoutText, mode); + } + if (timedOut && stdoutText.trim() === '') { + return { + isError: true, + output: `Grep timed out after ${String(DEFAULT_TIMEOUT_MS / 1000)}s. Try a more specific path or pattern.`, + }; + } + if (signal.aborted) { + return { isError: true, output: 'Grep aborted' }; + } + + const rawLines = parseRipgrepOutput(stdoutText, mode); + + const filteredSensitive = new Set(); + const keptLines = filterSensitiveLines(rawLines, mode, filteredSensitive, pathClass); + const orderedLines = keptLines; + + const offset = args.offset ?? 0; + const headLimit = args.head_limit ?? DEFAULT_HEAD_LIMIT; + const afterOffset = offset > 0 ? orderedLines.slice(offset) : orderedLines; + const limitActive = headLimit > 0; + const limited = limitActive ? afterOffset.slice(0, headLimit) : afterOffset; + const paginationTruncated = limitActive && afterOffset.length > headLimit; + + const headerLines: string[] = []; + const messages: string[] = []; + if (filteredSensitive.size > 0) { + const displayedFilteredPaths = [...filteredSensitive].map((path) => + relativizeIfUnder(path, workspaceDir, pathClass), + ); + messages.push( + `Filtered ${String(filteredSensitive.size)} sensitive file(s): ${displayedFilteredPaths.join(', ')}`, + ); + } + if (mode === 'count_matches' && orderedLines.length > 0) { + headerLines.push(formatCountSummary(orderedLines, filteredSensitive.size > 0)); + } + if (paginationTruncated) { + const total = afterOffset.length + offset; + const nextOffset = offset + headLimit; + const paginationNotice = + bufferTruncated || timedOut + ? `Results truncated to ${String(headLimit)} lines (total: ${String(total)} of a partial result set). Use offset=${String(nextOffset)} to see more.` + : `Results truncated to ${String(headLimit)} lines (total: ${String(total)}). Use offset=${String(nextOffset)} to see more.`; + if (mode === 'count_matches') { + headerLines.push(paginationNotice); + } else { + messages.push(paginationNotice); + } + } + if (bufferTruncated) { + messages.push( + `[Output truncated at ${String(MAX_OUTPUT_BYTES)} bytes of rg output — the result set is incomplete. Narrow the pattern, path, or glob filters and re-run to recover complete results.]`, + ); + } + if (timedOut) { + messages.push( + `Grep timed out after ${String(DEFAULT_TIMEOUT_MS / 1000)}s; partial results returned. Narrow the path, glob, or pattern and retry for complete results.`, + ); + } + + const contentIncludesLineNumbers = mode === 'content' && args['-n'] !== false; + const displayedLines = limited.map((line) => + formatDisplayLine( + line, + mode, + workspaceDir, + pathClass, + contentIncludesLineNumbers, + ), + ); + const contentBody = displayedLines.join('\n'); + const visibleBody = + orderedLines.length === 0 && filteredSensitive.size > 0 + ? 'No non-sensitive matches found' + : contentBody; + const emptyResultMessage = + SENSITIVE_GLOBS_TO_EXCLUDE.length > 0 ? 'No non-sensitive matches found' : 'No matches found'; + const body = + visibleBody === '' && headerLines.length === 0 && messages.length === 0 + ? emptyResultMessage + : visibleBody; + const combined = [...headerLines, body, ...messages].filter((part) => part !== '').join('\n'); + + const builder = new ToolOutputAccumulator(); + builder.write(combined); + return builder.ok(); + +} + +type GrepMode = 'content' | 'files_with_matches' | 'count_matches'; + +type ParsedGrepLine = + | { + readonly kind: 'record'; + readonly filePath: string; + readonly payload: string; + } + | { + readonly kind: 'separator'; + } + | { + readonly kind: 'legacy'; + readonly text: string; + }; + +function buildRgArgs( + rgPath: string, + args: GrepInput, + searchPaths: readonly string[], + singleThreaded = false, +): string[] { + const cmd: string[] = [rgPath]; + if (singleThreaded) cmd.push('-j', '1'); + cmd.push('--hidden'); + const mode = args.output_mode ?? 'files_with_matches'; + if (mode !== 'content') { + cmd.push('--max-columns', String(RG_MAX_COLUMNS)); + } + cmd.push('--null'); + for (const dir of VCS_DIRECTORIES_TO_EXCLUDE) { + cmd.push('--glob', `!${dir}`); + } + + if (mode === 'files_with_matches') cmd.push('-l', '--sortr=modified'); + else if (mode === 'count_matches') { + cmd.push('--count-matches', '--with-filename'); + } + + if (args['-i']) cmd.push('-i'); + if (mode === 'content') { + cmd.push('--with-filename'); + if (args['-n'] !== false) { + cmd.push('-n'); + } else { + cmd.push('--field-context-separator', ':'); + } + if (args['-C'] !== undefined) { + cmd.push('-C', String(args['-C'])); + } else { + if (args['-A'] !== undefined) cmd.push('-A', String(args['-A'])); + if (args['-B'] !== undefined) cmd.push('-B', String(args['-B'])); + } + } + if (args.glob !== undefined) cmd.push('--glob', args.glob); + if (args.type !== undefined) cmd.push('--type', args.type); + if (args.multiline) cmd.push('-U', '--multiline-dotall'); + if (args.include_ignored) cmd.push('--no-ignore'); + for (const glob of SENSITIVE_GLOBS_TO_EXCLUDE) { + cmd.push('--glob', `!${glob}`); + } + + cmd.push('--', args.pattern, ...searchPaths); + return cmd; +} + +function splitRgLines(text: string): string[] { + if (text === '') return []; + const lines = text.split('\n'); + while (lines.length > 0 && lines.at(-1) === '') { + lines.pop(); + } + return lines.map((line) => stripTrailingCarriageReturn(line)); +} + +function parseRipgrepOutput(text: string, mode: GrepMode): ParsedGrepLine[] { + if (text === '') return []; + if (!text.includes('\0')) { + return splitRgLines(text).map((line) => + mode === 'content' && line === '--' ? { kind: 'separator' } : { kind: 'legacy', text: line }, + ); + } + + if (mode === 'files_with_matches') { + return text + .split('\0') + .map((filePath) => stripTrailingCarriageReturn(filePath)) + .filter((filePath) => filePath !== '') + .map((filePath) => ({ kind: 'record', filePath, payload: '' })); + } + + const records: ParsedGrepLine[] = []; + let cursor = 0; + while (cursor < text.length) { + if (text[cursor] === '\n') { + cursor += 1; + continue; + } + if (text.startsWith('--\r\n', cursor)) { + records.push({ kind: 'separator' }); + cursor += 4; + continue; + } + if (text.startsWith('--\n', cursor)) { + records.push({ kind: 'separator' }); + cursor += 3; + continue; + } + + const nulIndex = text.indexOf('\0', cursor); + if (nulIndex < 0) { + const tail = stripTrailingCarriageReturn(text.slice(cursor)); + if (tail !== '') records.push({ kind: 'legacy', text: tail }); + break; + } + + const lineEnd = text.indexOf('\n', nulIndex + 1); + const payloadEnd = lineEnd >= 0 ? lineEnd : text.length; + const filePath = text.slice(cursor, nulIndex); + const payload = stripTrailingCarriageReturn(text.slice(nulIndex + 1, payloadEnd)); + records.push({ kind: 'record', filePath, payload }); + cursor = lineEnd >= 0 ? lineEnd + 1 : text.length; + } + return records; +} + +function formatDisplayLine( + line: ParsedGrepLine, + mode: GrepMode, + workspaceDir: string, + pathClass: PathClass, + contentIncludesLineNumbers: boolean, +): string { + if (line.kind === 'separator') return '--'; + if (line.kind === 'record') { + const displayPath = relativizeIfUnder(line.filePath, workspaceDir, pathClass); + if (mode === 'files_with_matches') return displayPath; + if (mode === 'count_matches') return `${displayPath}:${line.payload}`; + const separator = contentIncludesLineNumbers ? contentPayloadPathSeparator(line.payload) : ':'; + return `${displayPath}${separator}${line.payload}`; + } + + const text = line.text; + if (mode === 'files_with_matches') { + return relativizeIfUnder(text, workspaceDir, pathClass); + } + if (mode === 'count_matches') { + const idx = text.lastIndexOf(':'); + if (idx <= 0) return text; + return relativizeIfUnder(text.slice(0, idx), workspaceDir, pathClass) + text.slice(idx); + } + + const filePath = extractContentFilePath(text, pathClass); + if (filePath !== undefined) { + return relativizeIfUnder(filePath, workspaceDir, pathClass) + text.slice(filePath.length); + } + return text; +} + +function relativizeIfUnder(candidate: string, base: string, pathClass: PathClass): string { + const normCandidate = normalize(candidate); + const normBase = normalize(base); + const comparableCandidate = pathClass === 'win32' ? normCandidate.toLowerCase() : normCandidate; + const comparableBase = pathClass === 'win32' ? normBase.toLowerCase() : normBase; + if (comparableCandidate === comparableBase) return '.'; + const prefix = comparableBase.endsWith('/') ? comparableBase : comparableBase + '/'; + if (comparableCandidate.startsWith(prefix)) { + return normCandidate.slice(prefix.length); + } + return normCandidate; +} + +function omitIncompleteTrailingRecord(text: string, mode: GrepMode): string { + if (!text.includes('\0')) return omitIncompleteTrailingLine(text); + if (mode === 'files_with_matches') { + const lastNul = text.lastIndexOf('\0'); + return lastNul >= 0 ? text.slice(0, lastNul + 1) : ''; + } + + let cursor = 0; + let lastCompleteEnd = 0; + while (cursor < text.length) { + if (text[cursor] === '\n') { + cursor += 1; + lastCompleteEnd = cursor; + continue; + } + if (text.startsWith('--\r\n', cursor)) { + cursor += 4; + lastCompleteEnd = cursor; + continue; + } + if (text.startsWith('--\n', cursor)) { + cursor += 3; + lastCompleteEnd = cursor; + continue; + } + + const nulIndex = text.indexOf('\0', cursor); + if (nulIndex < 0) break; + const lineEnd = text.indexOf('\n', nulIndex + 1); + if (lineEnd < 0) break; + cursor = lineEnd + 1; + lastCompleteEnd = cursor; + } + return text.slice(0, lastCompleteEnd); +} + +function omitIncompleteTrailingLine(text: string): string { + const lastNewline = text.lastIndexOf('\n'); + return lastNewline >= 0 ? text.slice(0, lastNewline) : ''; +} + +function formatRipgrepError( + exitCode: number, + stderrText: string, + stderrTruncated: boolean, +): string { + const stderr = stderrText.trim(); + if (stderr.length === 0) { + return `Failed to grep: ripgrep exited with code ${String(exitCode)}`; + } + + const summary = summarizeRipgrepStderr(stderr); + const lines = [`Failed to grep: ${summary}`, '', 'ripgrep stderr:', stderr]; + if (stderrTruncated) { + lines.push(`[stderr truncated at ${String(MAX_OUTPUT_BYTES)} bytes]`); + } + return lines.join('\n'); +} + +function summarizeRipgrepStderr(stderr: string): string { + const lines = splitRgLines(stderr) + .map((line) => line.trim()) + .filter((line) => line.length > 0); + const errorLine = lines.findLast((line) => line.toLowerCase().startsWith('error:')); + return errorLine ?? lines.at(-1) ?? 'ripgrep error'; +} + +function filterSensitiveLines( + lines: readonly ParsedGrepLine[], + mode: GrepMode, + filteredPaths: Set, + pathClass: PathClass, +): ParsedGrepLine[] { + const kept: ParsedGrepLine[] = []; + for (const line of lines) { + if (line.kind === 'separator') { + kept.push(line); + continue; + } + const filePath = parsedFilePath(line, mode, pathClass); + if (filePath !== undefined && isSensitiveFile(filePath)) { + filteredPaths.add(filePath); + continue; + } + kept.push(line); + } + return mode === 'content' ? normalizeContextSeparators(kept) : kept; +} + +function normalizeContextSeparators(lines: readonly ParsedGrepLine[]): ParsedGrepLine[] { + const normalized: ParsedGrepLine[] = []; + for (const line of lines) { + if ( + line.kind === 'separator' && + (normalized.length === 0 || normalized.at(-1)?.kind === 'separator') + ) { + continue; + } + normalized.push(line); + } + while (normalized.length > 0 && normalized.at(-1)?.kind === 'separator') { + normalized.pop(); + } + return normalized; +} + +function parsedFilePath( + line: ParsedGrepLine, + mode: GrepMode, + pathClass: PathClass, +): string | undefined { + if (line.kind === 'record') return normalize(line.filePath); + if (line.kind === 'separator') return undefined; + const text = line.text; + if (mode === 'files_with_matches') return normalize(text); + if (mode === 'count_matches') { + const idx = text.lastIndexOf(':'); + return idx > 0 ? normalize(text.slice(0, idx)) : normalize(text); + } + return extractContentFilePath(text, pathClass); +} + +function extractContentFilePath(line: string, pathClass: PathClass): string | undefined { + const m = CONTENT_LINE_RE.exec(line); + if (m?.[1] !== undefined) return normalize(m[1]); + + const separatorIndex = noLineNumberContentSeparatorIndex(line, pathClass); + return separatorIndex > 0 ? normalize(line.slice(0, separatorIndex)) : undefined; +} + +function noLineNumberContentSeparatorIndex(line: string, pathClass: PathClass): number { + const searchFrom = pathClass === 'win32' && /^[A-Za-z]:/.test(line) ? 2 : 0; + return line.indexOf(':', searchFrom); +} + +function contentPayloadPathSeparator(payload: string): ':' | '-' { + const m = /^(\d+)([:-])/.exec(payload); + return m?.[2] === '-' ? '-' : ':'; +} + +function stripTrailingCarriageReturn(value: string): string { + return value.endsWith('\r') ? value.slice(0, -1) : value; +} + +function formatCountSummary(lines: readonly ParsedGrepLine[], redactedSensitive: boolean): string { + let totalMatches = 0; + let totalFiles = 0; + for (const line of lines) { + const rawCount = + line.kind === 'record' + ? line.payload + : line.kind === 'legacy' + ? countPayloadFromLegacyLine(line.text) + : undefined; + if (rawCount === undefined) continue; + const count = Number(rawCount); + if (!Number.isSafeInteger(count) || count < 0) continue; + totalMatches += count; + totalFiles++; + } + + const occurrenceWord = totalMatches === 1 ? 'occurrence' : 'occurrences'; + const fileWord = totalFiles === 1 ? 'file' : 'files'; + const scope = redactedSensitive ? 'total non-sensitive' : 'total'; + return `Found ${String(totalMatches)} ${scope} ${occurrenceWord} across ${String(totalFiles)} ${fileWord}.`; +} + +function countPayloadFromLegacyLine(line: string): string | undefined { + const idx = line.lastIndexOf(':'); + return idx > 0 ? line.slice(idx + 1) : undefined; +} diff --git a/packages/agent-core-v2/src/os/operations/snapshotFiles.ts b/packages/agent-core-v2/src/os/operations/snapshotFiles.ts new file mode 100644 index 00000000000..5a0e5048179 --- /dev/null +++ b/packages/agent-core-v2/src/os/operations/snapshotFiles.ts @@ -0,0 +1,39 @@ +import { createHash } from 'node:crypto'; + +import type { IHostFileSystem } from '#/os/interface/hostFileSystem'; +import type { FileSnapshotRequest, FileSnapshot } from '#/os/interface/fileOperations'; +import { isHostFsNotFound } from '#/os/interface/hostFsErrors'; + +export async function snapshotFiles(fs: IHostFileSystem, files: readonly FileSnapshotRequest[], maxFileBytes: number): Promise { + const results: FileSnapshot[] = []; + for (const file of files) { + let inspected = false; + try { + const before = await fs.stat(file.path); + inspected = true; + if (!before.isFile) { + results.push({ kind: 'unreadable' }); + continue; + } + if (before.size > maxFileBytes) { + results.push({ kind: 'oversize', size: before.size, mtimeMs: before.mtimeMs }); + continue; + } + const bytes = await fs.readBytes(file.path, before.size + 1); + const after = await fs.stat(file.path); + if (bytes.byteLength > maxFileBytes) { + results.push({ kind: 'oversize', size: after.size, mtimeMs: after.mtimeMs }); + continue; + } + if (!after.isFile || bytes.byteLength !== before.size || before.size !== after.size || before.mtimeMs !== after.mtimeMs || before.ino !== after.ino) { + results.push({ kind: 'unreadable' }); + continue; + } + const contentHash = createHash('sha256').update(bytes).digest('hex'); + results.push({ kind: 'file', contentHash, bytes: file.knownHash === contentHash ? undefined : bytes }); + } catch (error) { + results.push({ kind: !inspected && isHostFsNotFound(error) ? 'missing' : 'unreadable' }); + } + } + return results; +} diff --git a/packages/agent-core-v2/src/os/operations/textModel.ts b/packages/agent-core-v2/src/os/operations/textModel.ts new file mode 100644 index 00000000000..ab6611d738f --- /dev/null +++ b/packages/agent-core-v2/src/os/operations/textModel.ts @@ -0,0 +1,43 @@ +import { + type LineEndingStyle, + materializeModelText, + toModelTextView, +} from '#/_base/text/line-endings'; + +export class TextModel { + readonly lineEndingStyle: LineEndingStyle; + readonly text: string; + + constructor(raw: string) { + const view = toModelTextView(raw); + this.text = view.text; + this.lineEndingStyle = view.lineEndingStyle; + } + + countOccurrences(needle: string): number { + let count = 0; + let pos = 0; + while (pos < this.text.length) { + const idx = this.text.indexOf(needle, pos); + if (idx === -1) break; + count += 1; + pos = idx + needle.length; + } + return count; + } + + replaceOnce(needle: string, replacement: string): string { + const index = this.text.indexOf(needle); + if (index === -1) return this.text; + return this.text.slice(0, index) + replacement + this.text.slice(index + needle.length); + } + + replaceAll(needle: string, replacement: string): { text: string; count: number } { + const parts = this.text.split(needle); + return { text: parts.join(replacement), count: parts.length - 1 }; + } + + materialize(modelText: string): string { + return materializeModelText(modelText, this.lineEndingStyle); + } +} diff --git a/packages/agent-core-v2/src/os/operations/writeText.ts b/packages/agent-core-v2/src/os/operations/writeText.ts new file mode 100644 index 00000000000..461562ee83d --- /dev/null +++ b/packages/agent-core-v2/src/os/operations/writeText.ts @@ -0,0 +1,53 @@ +import type { HostFileStat, IHostFileSystem } from '#/os/interface/hostFileSystem'; +import type { TextWriteInput, TextFileResult } from '#/os/interface/fileOperations'; +import { isHostFsNotFound } from '#/os/interface/hostFsErrors'; + +export async function writeText(fs: IHostFileSystem, input: TextWriteInput): Promise { + const parentError = await ensureParentDirectory(fs, input.parent); + if (parentError !== undefined) return { isError: true, output: parentError }; + return writeTextResult(input, () => input.mode === 'append' ? fs.appendText(input.path, input.content) : fs.writeText(input.path, input.content)); +} + +export async function writeTextResult(input: TextWriteInput, write: () => Promise): Promise { + try { + const mode = input.mode; + await write(); + const bytesWritten = Buffer.byteLength(input.content, 'utf8'); + return { + output: `${mode === 'append' ? 'Appended' : 'Wrote'} ${String(bytesWritten)} bytes to ${input.displayPath}`, + }; + } catch (error) { + if (isHostFsNotFound(error)) { + return { + isError: true, + output: `Failed to write ${input.displayPath}: parent directory does not exist.`, + }; + } + return { + isError: true, + output: error instanceof Error ? error.message : String(error), + }; + } + +} + +export async function ensureParentDirectory(fs: IHostFileSystem, parent: string): Promise { + let stat: HostFileStat; + try { + stat = await fs.stat(parent); + } catch (error) { + if (isHostFsNotFound(error)) { + try { + await fs.mkdir(parent, { recursive: true }); + return undefined; + } catch (mkdirError) { + return mkdirError instanceof Error ? mkdirError.message : String(mkdirError); + } + } + return undefined; + } + if (!stat.isDirectory) { + return `Parent path is not a directory: ${parent}.`; + } + return undefined; +} diff --git a/packages/agent-core-v2/src/remote/README.md b/packages/agent-core-v2/src/remote/README.md index 5b83109b0bc..fec6e4973e6 100644 --- a/packages/agent-core-v2/src/remote/README.md +++ b/packages/agent-core-v2/src/remote/README.md @@ -12,6 +12,7 @@ node-local one rather than here: packages/agent-core-v2/src/ ├── os/ │ ├── interface/ IHostFileSystem / IHostProcessService contracts +│ ├── operations/ DI-free file and search algorithms shared with the controller │ └── backends/ │ ├── node-local/ direct node implementation (host, local environment, executor) │ └── remote/ RPC stubs over an ExecChannel (remoteFileSystem, remoteProcess) @@ -19,13 +20,13 @@ packages/agent-core-v2/src/ ├── protocol/ message types, error codes, NDJSON codec, ExecChannel (self-contained) ├── client/ execBridge, launchers, sshAskpass, connection, remoteSession, │ remoteEnvironmentProvider, connectGuidance (executor detection + guidance) - ├── server/ stdioHost, fsHandler, processManager, environment, entry + ├── server/ stdioHost, fsHandler, fileOperationsHandler, searchHandler, processManager ├── declaration/ [environments] config schema └── askpass.ts SSH_ASKPASS helper process (light entry, node builtins only) ``` -`os/backends/remote` may import only `#/remote/protocol`, `#/os/interface` and -`#/_base`; the connection lifecycle that feeds it an `ExecChannel` stays in +`os/backends/remote` may import only `#/remote/protocol`, `#/os/interface`, +`#/os/operations` and `#/_base`; the connection lifecycle that feeds it an `ExecChannel` stays in `remote/client`. ## Entries @@ -47,14 +48,14 @@ config/OAuth/telemetry/session services. Logs go to stderr; stdout carries protocol frames only. The light import graph is enforced by `scripts/check-import-boundaries.mjs`: `src/remote/server` may import only node builtins, relative modules, `#/remote/protocol`, `#/os/interface`, -`#/_base/execEnv` and the DI-free node-local `hostFsService` / +`#/os/operations`, `#/_base/execEnv` and the DI-free node-local `hostFsService` / `hostProcessService` modules (their App-scope registration lives in `os/backends/node-local/hostServices.ts`). ## Protocol notes (deviations from codex exec-server) Blueprint is codex exec-server (`codex-rs/exec-server*`, master `0265dd7b`); -deviations per the design spec, plus one forced addition: +deviations per the design spec and the shared host-operation extensions: - Bare absolute paths instead of `file:` URIs (our abstraction is bare-path end to end). @@ -63,7 +64,14 @@ deviations per the design spec, plus one forced addition: without `maxBytes` are capped well below the 64MiB frame cap. - `fs/writeFile` takes `mode: truncate|append|exclusive` (codex only has truncate); `exclusive` maps EEXIST to `os.fs.already_exists`, which the - client maps back to `createExclusive === false`. + client maps back to `createExclusive === false`. Optional `parent` folds + directory preparation into the first write. Large transfers attach + `transferId`/`chunkIndex`/`final`; the client pipelines at most four 1MiB + chunks, and the executor sequences each transfer and stops after an error. + It tracks at most 64 transfers, expires idle transfers after 60 seconds, + and rejects duplicate or out-of-order chunks while a transfer is live. + Writes are never automatically retried, and connection loss may leave a + partial write or an uncertain mutation outcome. - `fs/readDirectory` preserves the symlink marker on each entry. The client rejects a truncated listing with `os.fs.directory_too_large` rather than exposing an incomplete directory; the current limit is 50,000 entries. @@ -73,6 +81,34 @@ deviations per the design spec, plus one forced addition: back to per-directory reads when the JSON-encoded ignore rules exceed 1MiB. Walks default to a 55-second budget so they can return a truncated result before the 60-second RPC deadline. +- `fs/readTextPage` performs sniffing, text decoding, line counting and page + rendering on the executor. It returns the same bounded output and exact + totals as local Read. Up to eight scans may run at once; `fs/cancelRead` + aborts an active scan. Requests above 4,000,000 output characters or above + the active-scan limit fall back to controller-side streaming, preserving + larger configured Read budgets and concurrent calls. +- `fs/editText` applies the shared text-edit algorithm next to the file and + returns the replacement count or original edit error. It does not add an + atomic compare-and-swap guarantee against external writers. +- `fs/snapshotFiles` reads and hashes up to four files of at most 4MiB each. + Matching known hashes omit the bytes; the controller retains durable + history blobs before a mutation. Content hints from writes and edits are + bounded to 16 files / 16MiB and reused only after a fresh remote content hash. +- `fs/findGitWorkTree` and `fs/discoverInstructions` perform directory-chain + probes on the executor. Probe reuse is scoped to one instruction batch, + so a newly created Git marker or instruction file is visible on the next call. +- `fs/matchFiles` scans batches of up to 128 already-selected files for the + workspace grep fallback. Responses above 8MiB request controller-side + matching so existing large match/context outputs remain available. +- `process/search` runs the shared Glob/Grep algorithm on the executor and + returns the formatted page. The controller still resolves ripgrep and + handles permissions and telemetry. Up to eight searches may be active; + `process/cancelSearch` and disconnect abort their processes. More than 64 + roots or eight concurrent searches use controller-side processing with the + original process RPC path. The existing + 20-second timeout, retry, sorting, filtering and exact count semantics + remain; reaching the 10MiB stdout capture limit stops ripgrep early and + returns the existing partial-output diagnostics. - Added `fs/rename` (no codex counterpart). - `fs/remove` takes only `path` and always removes recursively and ignores a missing path, matching `IHostFileSystem.remove` (codex has `recursive`/`force`). @@ -117,7 +153,9 @@ Client-surface notes beyond the wire protocol: and a late response is discarded — the connection survives. `process/write` is exempt: its response is deliberately suspended until the child's stdin drains, so it carries no timeout and is never retried — a hung write fails - only with the connection. + only with the connection. `fs/readTextPage` is also exempt so long scans + retain Read's existing behavior; explicit cancellation and connection + liveness remain in force. - Whole-file reads without `maxBytes` are rejected server-side above 32MiB (base64 of the response must fit the 64MiB frame cap); larger files are read through `offset`/`maxBytes` range reads. @@ -180,6 +218,9 @@ handshake completes: any byte or unknown notification disconnects. ## Operational notes +- Remote execution is experimental; client and executor must be upgraded + together. The new operation methods do not negotiate compatibility with + earlier development executors. - The executor is posix-only; the client refuses non-posix environments and executors below `MIN_EXECUTOR_VERSION` at the handshake gate. - `test/remote/e2e/` holds the real-machine (ssh/docker) acceptance driver and diff --git a/packages/agent-core-v2/src/remote/client/connection.ts b/packages/agent-core-v2/src/remote/client/connection.ts index 1c1375f2ad8..840fbff8d30 100644 --- a/packages/agent-core-v2/src/remote/client/connection.ts +++ b/packages/agent-core-v2/src/remote/client/connection.ts @@ -16,6 +16,7 @@ import { MIN_EXECUTOR_VERSION, PING_METHOD, PROCESS_WRITE_METHOD, + FS_READ_TEXT_PAGE_METHOD, SERVER_NOTIFICATION_METHODS, type InitializeResult, type RemoteEnvironmentInfo, @@ -258,7 +259,7 @@ export class RemoteExecConnection implements ExecChannel { } return new Promise((resolve, reject) => { const pending: PendingCall = { resolve, reject }; - if (method !== PROCESS_WRITE_METHOD) { + if (method !== PROCESS_WRITE_METHOD && method !== FS_READ_TEXT_PAGE_METHOD) { pending.timer = setTimeout(() => { this.expireRequest(id, method); }, this.requestCallTimeoutMs); diff --git a/packages/agent-core-v2/src/remote/client/remoteEnvironmentProvider.ts b/packages/agent-core-v2/src/remote/client/remoteEnvironmentProvider.ts index cc1131748c5..bcfa92777ad 100644 --- a/packages/agent-core-v2/src/remote/client/remoteEnvironmentProvider.ts +++ b/packages/agent-core-v2/src/remote/client/remoteEnvironmentProvider.ts @@ -51,6 +51,15 @@ class DeclaredRemoteEnvironment implements Environment { private connectedAt = 0; readonly fs: IHostFileSystem = { _serviceBrand: undefined, + operations: { + matchFiles: async (files, options, timeoutMs) => this.connected().fs.operations.matchFiles(files, options, timeoutMs), + findGitWorkTree: async (cwd) => this.connected().fs.operations.findGitWorkTree(cwd), + discoverInstructions: async (dirs, known) => this.connected().fs.operations.discoverInstructions(dirs, known), + snapshotFiles: async (files, maxFileBytes) => this.connected().fs.operations.snapshotFiles(files, maxFileBytes), + writeText: async (input) => this.connected().fs.operations.writeText(input), + readTextPage: async (path, options, signal) => this.connected().fs.operations.readTextPage(path, options, signal), + editText: async (input) => this.connected().fs.operations.editText(input), + }, readText: async (path, options) => this.connected().fs.readText(path, options), writeText: async (path, data) => this.connected().fs.writeText(path, data), appendText: async (path, data) => this.connected().fs.appendText(path, data), @@ -69,6 +78,7 @@ class DeclaredRemoteEnvironment implements Environment { }; readonly process: IHostProcessService = { _serviceBrand: undefined, + search: async (request, signal) => this.connected().process.search(request, signal), spawn: async (command, args, options) => this.connected().process.spawn(command, args, options), }; diff --git a/packages/agent-core-v2/src/remote/protocol/methods.ts b/packages/agent-core-v2/src/remote/protocol/methods.ts index 478cfee3285..a25cc454b39 100644 --- a/packages/agent-core-v2/src/remote/protocol/methods.ts +++ b/packages/agent-core-v2/src/remote/protocol/methods.ts @@ -1,6 +1,22 @@ export const INITIALIZE_METHOD = 'initialize'; export const PING_METHOD = 'ping'; +export const FS_FIND_GIT_WORK_TREE_METHOD = 'fs/findGitWorkTree'; +export const FS_DISCOVER_INSTRUCTIONS_METHOD = 'fs/discoverInstructions'; + +export const FS_MATCH_FILES_METHOD = 'fs/matchFiles'; +export const FS_SNAPSHOT_FILES_METHOD = 'fs/snapshotFiles'; +export const FS_SNAPSHOT_MAX_FILES = 4; +export const FS_SNAPSHOT_MAX_FILE_BYTES = 4 * 1024 * 1024; + +export const FS_MAX_ACTIVE_READS = 8; +export const PROCESS_MAX_ACTIVE_SEARCHES = 8; +export const PROCESS_SEARCH_MAX_ROOTS = 64; + +export const FS_CANCEL_READ_METHOD = 'fs/cancelRead'; +export const FS_READ_TEXT_PAGE_METHOD = 'fs/readTextPage'; +export const FS_EDIT_TEXT_METHOD = 'fs/editText'; + export const FS_READ_FILE_METHOD = 'fs/readFile'; export const FS_WRITE_FILE_METHOD = 'fs/writeFile'; export const FS_CREATE_DIRECTORY_METHOD = 'fs/createDirectory'; @@ -11,6 +27,9 @@ export const FS_WALK_METHOD = 'fs/walk'; export const FS_REMOVE_METHOD = 'fs/remove'; export const FS_RENAME_METHOD = 'fs/rename'; +export const PROCESS_SEARCH_METHOD = 'process/search'; +export const PROCESS_CANCEL_SEARCH_METHOD = 'process/cancelSearch'; + export const PROCESS_START_METHOD = 'process/start'; export const PROCESS_OUTPUT_METHOD = 'process/output'; export const PROCESS_EXITED_METHOD = 'process/exited'; @@ -64,6 +83,10 @@ export interface FsWriteFileParams { readonly path: string; readonly dataBase64: string; readonly mode: FsWriteMode; + readonly parent?: string; + readonly transferId?: string; + readonly chunkIndex?: number; + readonly final?: boolean; } export interface FsCreateDirectoryParams { diff --git a/packages/agent-core-v2/src/remote/server/fileOperationsHandler.ts b/packages/agent-core-v2/src/remote/server/fileOperationsHandler.ts new file mode 100644 index 00000000000..62ff3f3c510 --- /dev/null +++ b/packages/agent-core-v2/src/remote/server/fileOperationsHandler.ts @@ -0,0 +1,135 @@ +import type { HostFileSystem } from '#/os/backends/node-local/hostFsService'; +import { TEXT_PAGE_MAX_REMOTE_CHARS, type TextFileResult, type TextEditResult } from '#/os/interface/fileOperations'; +import { findGitWorkTree } from '#/os/operations/gitWorkTree'; +import { discoverInstructions } from '#/os/operations/instructions'; +import { matchFiles } from '#/os/operations/matchFiles'; +import { snapshotFiles } from '#/os/operations/snapshotFiles'; +import { FS_MAX_ACTIVE_READS, FS_SNAPSHOT_MAX_FILES, FS_SNAPSHOT_MAX_FILE_BYTES } from '#/remote/protocol/methods'; +import { encodeBase64 } from '#/remote/protocol/codec'; +import { readTextPage } from '#/os/operations/readTextPage'; +import { editText } from '#/os/operations/editText'; +import { RpcError, RpcErrorCode } from '#/remote/protocol/errors'; +import { optionalBoolean, optionalInteger, requireAbsolutePath, requireParams, requireString } from './fsHandler'; + +export class FileOperationsHandler { + private readonly reads = new Map }>(); + + cancelRead(rawParams: unknown): void { + const params = requireParams(rawParams); + this.reads.get(requireString(params, 'readId'))?.controller.abort(new Error('Read aborted')); + } + + async dispose(): Promise { + const pending = [...this.reads.values()]; + for (const read of pending) read.controller.abort(new Error('Read aborted')); + await Promise.allSettled(pending.map((read) => read.done)); + } + + constructor(private readonly fs: HostFileSystem) {} + + async findGitWorkTree(rawParams: unknown): Promise<{ workTree: Awaited> }> { + const params = requireParams(rawParams); + return { workTree: await findGitWorkTree(this.fs, requireAbsolutePath(params, 'cwd')) }; + } + + discoverInstructions(rawParams: unknown): Promise { + const params = requireParams(rawParams); + const dirs = params['dirs']; + const known = params['known']; + if (!Array.isArray(dirs) || dirs.length > 64 || !Array.isArray(known) || known.some((path) => typeof path !== 'string')) throw new RpcError(RpcErrorCode.InvalidParams, 'invalid instruction directories'); + const paths = dirs.map((dir: unknown) => requireAbsolutePath({ dir }, 'dir')); + return discoverInstructions(this.fs, paths, known); + } + + matchFiles(rawParams: unknown): ReturnType { + const params = requireParams(rawParams); + const files = params['files']; + if (!Array.isArray(files) || files.length > 128) throw new RpcError(RpcErrorCode.InvalidParams, 'too many match files'); + const paths = files.map((file: unknown) => { + const value = requireParams(file); + return { path: requireString(value, 'path'), absolute: requireAbsolutePath(value, 'absolute') }; + }); + const options = requireParams(params['options']); + return matchFiles(this.fs, paths, { + pattern: requireString(options, 'pattern'), + regex: optionalBoolean(options, 'regex') ?? false, + case_sensitive: optionalBoolean(options, 'case_sensitive') ?? true, + context_lines: optionalInteger(options, 'context_lines', 0, 10) ?? 2, + max_files: optionalInteger(options, 'max_files', 1, 10_000) ?? 200, + max_matches_per_file: optionalInteger(options, 'max_matches_per_file', 1, 10_000) ?? 50, + max_total_matches: optionalInteger(options, 'max_total_matches', 1, 100_000) ?? 5000, + }, optionalInteger(params, 'timeoutMs', 1, 30_000) ?? 30_000, undefined, 8 * 1024 * 1024); + } + + async snapshotFiles(rawParams: unknown): Promise { + const params = requireParams(rawParams); + const files = params['files']; + if (!Array.isArray(files) || files.length > FS_SNAPSHOT_MAX_FILES) throw new RpcError(RpcErrorCode.InvalidParams, 'too many snapshot files'); + const requests = files.map((file: unknown) => { + const entry = requireParams(file); + const knownHash = entry['knownHash']; + if (knownHash !== undefined && (typeof knownHash !== 'string' || !/^[a-f0-9]{64}$/.test(knownHash))) throw new RpcError(RpcErrorCode.InvalidParams, 'invalid snapshot hash'); + return { path: requireAbsolutePath(entry, 'path'), knownHash }; + }); + const maxFileBytes = optionalInteger(params, 'maxFileBytes', 0, FS_SNAPSHOT_MAX_FILE_BYTES) ?? FS_SNAPSHOT_MAX_FILE_BYTES; + const results = await snapshotFiles(this.fs, requests, maxFileBytes); + return results.map((result) => result.kind === 'file' ? { + kind: result.kind, + contentHash: result.contentHash, + dataBase64: result.bytes === undefined ? undefined : encodeBase64(result.bytes), + } : result); + } + + async readTextPage(rawParams: unknown): Promise { + const params = requireParams(rawParams); + const path = requireAbsolutePath(params, 'path'); + const args = requireParams(params['args']); + const lineOffset = optionalInteger(args, 'line_offset', -Number.MAX_SAFE_INTEGER, Number.MAX_SAFE_INTEGER); + if (lineOffset === 0) throw new RpcError(RpcErrorCode.InvalidParams, 'line_offset must be nonzero'); + const maxChars = optionalInteger(params, 'maxChars', 1, TEXT_PAGE_MAX_REMOTE_CHARS); + const maxCharsLimit = optionalInteger(params, 'maxCharsLimit', 1, Number.MAX_SAFE_INTEGER); + if (maxChars === undefined || maxCharsLimit === undefined) throw new RpcError(RpcErrorCode.InvalidParams, 'read limits are required'); + const readId = requireString(params, 'readId'); + if (this.reads.has(readId) || this.reads.size >= FS_MAX_ACTIVE_READS) throw new RpcError(RpcErrorCode.InvalidRequest, 'read ID is active or too many active reads'); + const controller = new AbortController(); + const done = readTextPage({ + name: path, + stat: () => this.fs.stat(path), + readBytes: (n) => this.fs.readBytes(path, n), + readLines: () => this.fs.readLines(path, { errors: 'strict' }), + }, { + args: { + path: requireString(args, 'path'), + line_offset: lineOffset, + column_offset: optionalInteger(args, 'column_offset', 0, Number.MAX_SAFE_INTEGER), + n_lines: optionalInteger(args, 'n_lines', 1, Number.MAX_SAFE_INTEGER), + max_chars: optionalInteger(args, 'max_chars', 1, Number.MAX_SAFE_INTEGER), + }, + maxChars, + maxCharsLimit, + eventLog: optionalBoolean(params, 'eventLog') ?? false, + imageSupported: optionalBoolean(params, 'imageSupported') ?? false, + videoSupported: optionalBoolean(params, 'videoSupported') ?? false, + mediaToolAvailable: optionalBoolean(params, 'mediaToolAvailable') ?? false, + }, controller.signal); + this.reads.set(readId, { controller, done }); + try { + return await done; + } finally { + this.reads.delete(readId); + } + } + + editText(rawParams: unknown): Promise { + const params = requireParams(rawParams); + const newString = params['new_string']; + if (typeof newString !== 'string') throw new RpcError(RpcErrorCode.InvalidParams, 'new_string must be a string'); + return editText(this.fs, { + path: requireAbsolutePath(params, 'path'), + displayPath: requireString(params, 'displayPath'), + old_string: requireString(params, 'old_string'), + new_string: newString, + replace_all: optionalBoolean(params, 'replace_all') ?? false, + }); + } +} diff --git a/packages/agent-core-v2/src/remote/server/fsHandler.ts b/packages/agent-core-v2/src/remote/server/fsHandler.ts index c30f4e50a98..462732a01d3 100644 --- a/packages/agent-core-v2/src/remote/server/fsHandler.ts +++ b/packages/agent-core-v2/src/remote/server/fsHandler.ts @@ -1,3 +1,4 @@ +import { ensureParentDirectory } from '#/os/operations/writeText'; import { isAbsolute } from 'node:path'; import type { HostFileSystem } from '#/os/backends/node-local/hostFsService'; @@ -6,6 +7,7 @@ import { HostFsError, OsFsErrors, toHostFsError } from '#/os/interface/hostFsErr import { RpcError, RpcErrorCode } from '#/remote/protocol/errors'; import { FS_READ_DIRECTORY_MAX_ENTRIES, + FS_WRITE_FILE_CHUNK_BYTES, FS_READ_FILE_MAX_BYTES, FS_READ_FILE_WHOLE_MAX_BYTES, type FsCanonicalizeResult, @@ -59,7 +61,7 @@ export function optionalBoolean(params: Params, name: string): boolean | undefin return value; } -function optionalInteger( +export function optionalInteger( params: Params, name: string, min: number, @@ -99,7 +101,22 @@ function fsDomainError(error: unknown, ctx: { path: string; op: string }): RpcEr const EMPTY: Record = {}; +interface WriteTransfer { + readonly path: string; + nextIndex: number; + final: boolean; + tail: Promise; + timer: ReturnType; +} + export class FsHandler { + private readonly writes = new Map(); + + dispose(): void { + for (const transfer of this.writes.values()) clearTimeout(transfer.timer); + this.writes.clear(); + } + constructor(private readonly fs: HostFileSystem) {} async readFile(rawParams: unknown): Promise { @@ -128,6 +145,40 @@ export class FsHandler { } async writeFile(rawParams: unknown): Promise { + const params = requireParams(rawParams); + const transferId = params['transferId']; + if (transferId === undefined) return this.writeChunk(params); + if (typeof transferId !== 'string' || transferId.length > 128 || transferId.length === 0) { + throw new RpcError(RpcErrorCode.InvalidParams, 'invalid write transfer ID'); + } + const path = requireAbsolutePath(params, 'path'); + const chunkIndex = optionalInteger(params, 'chunkIndex', 0, Number.MAX_SAFE_INTEGER); + const final = optionalBoolean(params, 'final') ?? false; + let transfer = this.writes.get(transferId); + if (transfer === undefined) { + if (chunkIndex !== 0 || this.writes.size >= 64) throw new RpcError(RpcErrorCode.InvalidRequest, 'unknown write transfer or too many active writes'); + transfer = { path, nextIndex: 0, final: false, tail: Promise.resolve(), timer: setTimeout(() => this.writes.delete(transferId), 60_000).unref() }; + this.writes.set(transferId, transfer); + } + if (transfer.final || transfer.path !== path || transfer.nextIndex !== chunkIndex || (chunkIndex !== 0 && (params['mode'] !== 'append' || params['parent'] !== undefined))) { + throw new RpcError(RpcErrorCode.InvalidRequest, 'write chunks must arrive once and in order for the same file'); + } + transfer.nextIndex += 1; + transfer.final = final; + transfer.timer.refresh(); + const run = transfer.tail.then(() => this.writeChunk(params)); + transfer.tail = run; + try { + return await run; + } finally { + if (final) { + clearTimeout(transfer.timer); + this.writes.delete(transferId); + } + } + } + + private async writeChunk(rawParams: unknown): Promise { const params = requireParams(rawParams); const path = requireAbsolutePath(params, 'path'); const dataBase64 = params['dataBase64']; @@ -139,6 +190,12 @@ export class FsHandler { throw new RpcError(RpcErrorCode.InvalidParams, 'mode must be truncate, append or exclusive'); } const data = Buffer.from(dataBase64, 'base64'); + if (data.byteLength > FS_WRITE_FILE_CHUNK_BYTES) throw new RpcError(RpcErrorCode.InvalidParams, 'write chunk exceeds the byte limit'); + if (params['parent'] !== undefined) { + const parent = requireAbsolutePath(params, 'parent'); + const error = await ensureParentDirectory(this.fs, parent); + if (error !== undefined) throw new RpcError(RpcErrorCode.InvalidRequest, error); + } try { if (mode === 'truncate') { await this.fs.writeBytes(path, data); diff --git a/packages/agent-core-v2/src/remote/server/searchHandler.ts b/packages/agent-core-v2/src/remote/server/searchHandler.ts new file mode 100644 index 00000000000..b8b9c302039 --- /dev/null +++ b/packages/agent-core-v2/src/remote/server/searchHandler.ts @@ -0,0 +1,76 @@ +import { PROCESS_MAX_ACTIVE_SEARCHES, PROCESS_SEARCH_MAX_ROOTS } from '#/remote/protocol/methods'; +import type { IHostProcessService } from '#/os/interface/hostProcess'; +import type { SearchRequest, SearchResult } from '#/os/interface/searchOperations'; +import { searchGlob } from '#/os/operations/searchGlob'; +import { searchGrep } from '#/os/operations/searchGrep'; +import { RpcError, RpcErrorCode } from '#/remote/protocol/errors'; +import { optionalBoolean, optionalInteger, requireAbsolutePath, requireParams, requireString } from './fsHandler'; + +export class SearchHandler { + private readonly active = new Map }>(); + + constructor(private readonly process: IHostProcessService) {} + + async search(rawParams: unknown): Promise { + const params = requireParams(rawParams); + const searchId = requireString(params, 'searchId'); + const kind = requireString(params, 'kind'); + if (kind !== 'glob' && kind !== 'grep') throw new RpcError(RpcErrorCode.InvalidParams, 'invalid search kind'); + const rawArgs = requireParams(params['args']); + const roots = params['roots']; + if (!Array.isArray(roots) || roots.length === 0 || roots.length > PROCESS_SEARCH_MAX_ROOTS) throw new RpcError(RpcErrorCode.InvalidParams, 'invalid search roots'); + const pattern = rawArgs['pattern']; + if (typeof pattern !== 'string') throw new RpcError(RpcErrorCode.InvalidParams, 'pattern must be a string'); + const outputMode = rawArgs['output_mode']; + if (outputMode !== undefined && outputMode !== 'content' && outputMode !== 'files_with_matches' && outputMode !== 'count_matches') throw new RpcError(RpcErrorCode.InvalidParams, 'invalid search output mode'); + const pathClass = params['pathClass']; + if (pathClass !== 'posix' && pathClass !== 'win32') throw new RpcError(RpcErrorCode.InvalidParams, 'invalid search path class'); + const optionalString = (key: string): string | undefined => { + const value = rawArgs[key]; + if (value !== undefined && typeof value !== 'string') throw new RpcError(RpcErrorCode.InvalidParams, `${key} must be a string`); + return value; + }; + const request: SearchRequest = { + kind, + roots: roots.map((root: unknown) => requireAbsolutePath({ root }, 'root')), + workspaceDir: requireAbsolutePath(params, 'workspaceDir'), + rgPath: requireString(params, 'rgPath'), + pathClass, + args: { + pattern, + glob: optionalString('glob'), + type: optionalString('type'), + output_mode: outputMode, + offset: optionalInteger(rawArgs, 'offset', 0, Number.MAX_SAFE_INTEGER), + head_limit: optionalInteger(rawArgs, 'head_limit', 0, Number.MAX_SAFE_INTEGER), + include_ignored: optionalBoolean(rawArgs, 'include_ignored'), + multiline: optionalBoolean(rawArgs, 'multiline'), + '-i': optionalBoolean(rawArgs, '-i'), + '-n': optionalBoolean(rawArgs, '-n'), + '-A': optionalInteger(rawArgs, '-A', 0, Number.MAX_SAFE_INTEGER), + '-B': optionalInteger(rawArgs, '-B', 0, Number.MAX_SAFE_INTEGER), + '-C': optionalInteger(rawArgs, '-C', 0, Number.MAX_SAFE_INTEGER), + }, + }; + if (this.active.has(searchId) || this.active.size >= PROCESS_MAX_ACTIVE_SEARCHES) throw new RpcError(RpcErrorCode.InvalidRequest, 'search ID is active or too many active searches'); + const controller = new AbortController(); + const done = kind === 'glob' ? searchGlob(this.process, request, controller.signal) : searchGrep(this.process, request, controller.signal); + this.active.set(searchId, { controller, done }); + try { + return await done; + } finally { + this.active.delete(searchId); + } + } + + cancel(rawParams: unknown): void { + const params = requireParams(rawParams); + this.active.get(requireString(params, 'searchId'))?.controller.abort(); + } + + async dispose(): Promise { + const pending = [...this.active.values()]; + for (const search of pending) search.controller.abort(); + await Promise.allSettled(pending.map((search) => search.done)); + } +} diff --git a/packages/agent-core-v2/src/remote/server/stdioHost.ts b/packages/agent-core-v2/src/remote/server/stdioHost.ts index 78515f49214..8936cf5c33a 100644 --- a/packages/agent-core-v2/src/remote/server/stdioHost.ts +++ b/packages/agent-core-v2/src/remote/server/stdioHost.ts @@ -17,6 +17,13 @@ import { FS_GET_METADATA_METHOD, FS_READ_DIRECTORY_METHOD, FS_READ_FILE_METHOD, + FS_READ_TEXT_PAGE_METHOD, + FS_CANCEL_READ_METHOD, + FS_SNAPSHOT_FILES_METHOD, + FS_MATCH_FILES_METHOD, + FS_FIND_GIT_WORK_TREE_METHOD, + FS_DISCOVER_INSTRUCTIONS_METHOD, + FS_EDIT_TEXT_METHOD, FS_REMOVE_METHOD, FS_RENAME_METHOD, FS_WALK_METHOD, @@ -28,6 +35,8 @@ import { PROCESS_FLOW_METHOD, PROCESS_SIGNAL_METHOD, PROCESS_START_METHOD, + PROCESS_SEARCH_METHOD, + PROCESS_CANCEL_SEARCH_METHOD, PROCESS_WRITE_METHOD, type InitializeResult, type RemoteEnvironmentInfo, @@ -35,6 +44,8 @@ import { import { HostFileSystem } from '#/os/backends/node-local/hostFsService'; import { HostProcessService } from '#/os/backends/node-local/hostProcessService'; +import { FileOperationsHandler } from './fileOperationsHandler'; +import { SearchHandler } from './searchHandler'; import { FsHandler } from './fsHandler'; import { ProcessManager } from './processManager'; @@ -52,6 +63,8 @@ type Handler = (params: unknown) => Promise; const CONTROL_METHODS: ReadonlySet = new Set([ PING_METHOD, + FS_READ_TEXT_PAGE_METHOD, + PROCESS_SEARCH_METHOD, PROCESS_WRITE_METHOD, PROCESS_SIGNAL_METHOD, ]); @@ -133,7 +146,10 @@ export class StdioHost { private readonly decoder = new LineFrameDecoder(); private readonly writer: OutboundWriter; private readonly processManager: ProcessManager; - private readonly fsHandler = new FsHandler(new HostFileSystem()); + private readonly searchHandler = new SearchHandler(new HostProcessService()); + private readonly fs = new HostFileSystem(); + private readonly fsHandler = new FsHandler(this.fs); + private readonly fileOperations = new FileOperationsHandler(this.fs); private readonly handlers: ReadonlyMap; private state: 'pre-init' | 'ready' = 'pre-init'; private inFlight = 0; @@ -161,6 +177,12 @@ export class StdioHost { const pm = this.processManager; this.handlers = new Map([ [PING_METHOD, () => Promise.resolve({})], + [FS_FIND_GIT_WORK_TREE_METHOD, (p) => this.fileOperations.findGitWorkTree(p)], + [FS_DISCOVER_INSTRUCTIONS_METHOD, (p) => this.fileOperations.discoverInstructions(p)], + [FS_MATCH_FILES_METHOD, (p) => this.fileOperations.matchFiles(p)], + [FS_SNAPSHOT_FILES_METHOD, (p) => this.fileOperations.snapshotFiles(p)], + [FS_READ_TEXT_PAGE_METHOD, (p) => this.fileOperations.readTextPage(p)], + [FS_EDIT_TEXT_METHOD, (p) => this.fileOperations.editText(p)], [FS_READ_FILE_METHOD, (p) => fs.readFile(p)], [FS_WRITE_FILE_METHOD, (p) => fs.writeFile(p)], [FS_CREATE_DIRECTORY_METHOD, (p) => fs.createDirectory(p)], @@ -170,6 +192,7 @@ export class StdioHost { [FS_WALK_METHOD, (p) => fs.walk(p)], [FS_REMOVE_METHOD, (p) => fs.remove(p)], [FS_RENAME_METHOD, (p) => fs.rename(p)], + [PROCESS_SEARCH_METHOD, (p) => this.searchHandler.search(p)], [PROCESS_START_METHOD, (p) => pm.start(p)], [PROCESS_WRITE_METHOD, (p) => pm.write(p)], [PROCESS_SIGNAL_METHOD, (p) => pm.signal(p)], @@ -234,6 +257,14 @@ export class StdioHost { } if (isNotification(message)) { + if (message.method === FS_CANCEL_READ_METHOD) { + try { this.fileOperations.cancelRead(message.params); } catch (error) { this.violation(error); } + return; + } + if (message.method === PROCESS_CANCEL_SEARCH_METHOD) { + try { this.searchHandler.cancel(message.params); } catch (error) { this.violation(error); } + return; + } if (message.method === PROCESS_FLOW_METHOD) { this.onProcessFlow(message.params); return; @@ -285,7 +316,7 @@ export class StdioHost { ); return; } - const lane: Lane = message.method === FS_READ_FILE_METHOD || message.method === FS_WALK_METHOD + const lane: Lane = message.method === FS_READ_FILE_METHOD || message.method === FS_WALK_METHOD || message.method === FS_READ_TEXT_PAGE_METHOD || message.method === FS_SNAPSHOT_FILES_METHOD || message.method === FS_MATCH_FILES_METHOD || message.method === PROCESS_SEARCH_METHOD ? 'data' : 'control'; if (CONTROL_METHODS.has(message.method)) { @@ -366,7 +397,8 @@ export class StdioHost { this.shuttingDown = true; this.options.log(`shutdown: ${reason}`); this.waiting.length = 0; - await this.processManager.terminateAll(); + this.fsHandler.dispose(); + await Promise.all([this.processManager.terminateAll(), this.searchHandler.dispose(), this.fileOperations.dispose()]); this.writer.destroy(); this.options.output.end(); this.resolveDone(); diff --git a/packages/agent-core-v2/src/tool/output-accumulator.ts b/packages/agent-core-v2/src/tool/output-accumulator.ts index bb619e5652d..4211def91a9 100644 --- a/packages/agent-core-v2/src/tool/output-accumulator.ts +++ b/packages/agent-core-v2/src/tool/output-accumulator.ts @@ -1,97 +1 @@ -import { sliceOnCodePointBoundary } from '#/_base/text/code-points'; -import { - DEFAULT_TOOL_RESULT_MAX_CHARS, - DEFAULT_TOOL_RESULT_MAX_RETAINED_CHARS, - type ExecutableToolErrorResult, - type ExecutableToolSuccessResult, - type ToolResultSpill, -} from './toolContract'; - -export type ToolOutputAccumulatorResult = ( - | ExecutableToolErrorResult - | ExecutableToolSuccessResult -) & { - readonly output: string; - readonly brief?: string; -}; - -export class ToolOutputAccumulator { - private readonly buffer: string[] = []; - private retainedChars = 0; - private totalCharsValue = 0; - private saturated = false; - - get nChars(): number { - return this.retainedChars; - } - - get totalChars(): number { - return this.totalCharsValue; - } - - write(text: string): void { - this.totalCharsValue += text.length; - if (this.saturated) return; - const remainingRetention = DEFAULT_TOOL_RESULT_MAX_RETAINED_CHARS - this.retainedChars; - const kept = - text.length <= remainingRetention - ? text - : sliceOnCodePointBoundary(text, remainingRetention); - if (kept.length !== text.length) this.saturated = true; - this.buffer.push(kept); - this.retainedChars += kept.length; - } - - ok(message = '', options: { readonly brief?: string } = {}): ToolOutputAccumulatorResult { - let finalMessage = message; - if (finalMessage.length > 0 && !finalMessage.endsWith('.')) { - finalMessage += '.'; - } - const output = this.buffer.join(''); - return { - isError: false, - output: output.length === 0 ? finalMessage : output, - brief: options.brief, - spill: this.completionSpill(finalMessage), - }; - } - - error( - message: string, - options: { readonly brief?: string } = {}, - ): ToolOutputAccumulatorResult { - const output = this.buffer.join(''); - return { - isError: true, - output: - message.length === 0 - ? output - : output.length === 0 - ? message - : output.endsWith('\n') - ? `${output}${message}` - : `${output}\n${message}`, - brief: options.brief, - spill: this.retentionSpill(message), - }; - } - - private retentionSpill(suffix?: string): ToolResultSpill | undefined { - if (this.totalCharsValue <= this.retainedChars) return undefined; - return { - totalChars: this.totalCharsValue, - suffix: suffix !== undefined && suffix.length > 0 ? suffix : undefined, - }; - } - - private completionSpill(suffix: string): ToolResultSpill | undefined { - const retentionSpill = this.retentionSpill(); - if (retentionSpill !== undefined) { - return suffix.length > 0 ? { ...retentionSpill, suffix } : retentionSpill; - } - if (suffix.length === 0 || this.totalCharsValue <= DEFAULT_TOOL_RESULT_MAX_CHARS) { - return undefined; - } - return { suffix }; - } -} +export * from '#/_base/text/outputAccumulator'; diff --git a/packages/agent-core-v2/src/tool/path-access.ts b/packages/agent-core-v2/src/tool/path-access.ts index 7e11c46313f..3bdf69cba72 100644 --- a/packages/agent-core-v2/src/tool/path-access.ts +++ b/packages/agent-core-v2/src/tool/path-access.ts @@ -1,3 +1,4 @@ +import { isSensitiveFile, isWithinDirectory } from '#/_base/fs/pathFilters'; import * as pathe from 'pathe'; import { @@ -7,81 +8,13 @@ import { } from '#/_base/execEnv/shellPathBridge'; import type { IHostEnvironment } from '#/os/interface/hostEnvironment'; +export { isSensitiveFile, SENSITIVE_DOT_VARIANT_SUFFIXES, isWithinDirectory } from '#/_base/fs/pathFilters'; + export interface WorkspaceConfig { readonly workspaceDir: string; readonly additionalDirs: readonly string[]; } -const SENSITIVE_BASENAMES = new Set([ - '.env', - 'id_rsa', - 'id_ed25519', - 'id_ecdsa', - 'credentials', -]); - -const SENSITIVE_PATH_SUFFIXES = [ - ['.aws', 'credentials'], - ['.gcp', 'credentials'], -]; - -const ENV_PREFIX = '.env.'; -const ENV_EXEMPTIONS = new Set(['.env.example', '.env.sample', '.env.template']); - -const SENSITIVE_BASENAME_PREFIXES = ['id_rsa', 'id_ed25519', 'id_ecdsa', 'credentials']; -const PUBLIC_KEY_BASENAMES = new Set(['id_rsa.pub', 'id_ed25519.pub', 'id_ecdsa.pub']); -export const SENSITIVE_DOT_VARIANT_SUFFIXES = [ - '.bak', - '.backup', - '.copy', - '.disabled', - '.key', - '.old', - '.orig', - '.pem', - '.save', - '.tmp', -] as const; -const SENSITIVE_DOT_VARIANT_SUFFIX_SET = new Set(SENSITIVE_DOT_VARIANT_SUFFIXES); - -function comparable(path: string): string { - return path.toLowerCase(); -} - -export function isSensitiveFile(path: string): boolean { - const name = pathe.basename(path); - const comparableName = comparable(name); - const comparablePath = comparable(path); - - if (ENV_EXEMPTIONS.has(comparableName)) return false; - if (PUBLIC_KEY_BASENAMES.has(comparableName)) return false; - if (SENSITIVE_BASENAMES.has(comparableName)) return true; - if (comparableName.startsWith(ENV_PREFIX)) return true; - - for (const prefix of SENSITIVE_BASENAME_PREFIXES) { - if (comparableName === prefix) return true; - if (comparableName.length > prefix.length && comparableName.startsWith(prefix)) { - const suffix = comparableName.slice(prefix.length); - const next = suffix[0]; - if (next === '-' || next === '_') return true; - if (next === '.' && SENSITIVE_DOT_VARIANT_SUFFIX_SET.has(suffix)) return true; - } - } - - for (const suffixParts of SENSITIVE_PATH_SUFFIXES) { - const suffix = suffixParts.join('/'); - const comparableSuffix = comparable(suffix); - if ( - comparablePath.endsWith(`/${comparableSuffix}`) || - comparablePath.includes(`/${comparableSuffix}/`) - ) { - return true; - } - } - - return false; -} - export type PathClass = 'posix' | 'win32'; export type PathSecurityCode = 'PATH_OUTSIDE_WORKSPACE' | 'PATH_SENSITIVE' | 'PATH_INVALID'; export type PathAccessOperation = 'read' | 'write' | 'search'; @@ -164,20 +97,6 @@ export function canonicalizePath( return pathe.normalize(abs); } -export function isWithinDirectory( - candidate: string, - base: string, - pathClass: PathClass = DEFAULT_PATH_CLASS, -): boolean { - const nc = pathe.normalize(candidate); - const nb = pathe.normalize(base); - const comparableCandidate = pathClass === 'win32' ? nc.toLowerCase() : nc; - const comparableBase = pathClass === 'win32' ? nb.toLowerCase() : nb; - if (comparableCandidate === comparableBase) return true; - const prefix = comparableBase.endsWith('/') ? comparableBase : comparableBase + '/'; - return comparableCandidate.startsWith(prefix); -} - export function isWithinWorkspace( candidate: string, config: WorkspaceConfig, diff --git a/packages/agent-core-v2/src/tool/toolContract.ts b/packages/agent-core-v2/src/tool/toolContract.ts index e319c4fa1c8..002782ba5f1 100644 --- a/packages/agent-core-v2/src/tool/toolContract.ts +++ b/packages/agent-core-v2/src/tool/toolContract.ts @@ -1,18 +1,11 @@ +import type { ToolResultSpill } from '#/_base/text/outputLimits'; import type { ContentPart, ToolCall, ToolDescription as Tool } from '#human/llm/message'; import type { LLMRequestTrace } from '#/llm-adapter/contract/request-trace'; import type { ToolInputDisplay } from '#/tool/toolInputDisplay'; export type ExecutableToolOutput = string | ContentPart[]; -export const DEFAULT_TOOL_RESULT_MAX_CHARS = 50_000; - -export const DEFAULT_TOOL_RESULT_MAX_RETAINED_CHARS = 10_000_000; - -export interface ToolResultSpill { - readonly outputPath?: string; - readonly totalChars?: number; - readonly suffix?: string; -} +export { DEFAULT_TOOL_RESULT_MAX_CHARS, DEFAULT_TOOL_RESULT_MAX_RETAINED_CHARS, type ToolResultSpill } from '#/_base/text/outputLimits'; export type ToolDeliveryKind = 'steer'; diff --git a/packages/agent-core-v2/src/workspace/workspaceFs/fsService.ts b/packages/agent-core-v2/src/workspace/workspaceFs/fsService.ts index 6f837a0c569..5d17786b0fc 100644 --- a/packages/agent-core-v2/src/workspace/workspaceFs/fsService.ts +++ b/packages/agent-core-v2/src/workspace/workspaceFs/fsService.ts @@ -1,3 +1,4 @@ +import { matchFiles } from '#/os/operations/matchFiles'; import { type FsDiffRequest, type FsDiffResponse, @@ -1178,7 +1179,7 @@ export class WorkspaceFsService implements IWorkspaceFsService { signal: AbortSignal, startedAt: number, ): Promise { - const re = compileGrepPattern(req); + compileGrepPattern(req); const files: FsGrepFileHit[] = []; let filesScanned = 0; @@ -1210,49 +1211,25 @@ export class WorkspaceFsService implements IWorkspaceFsService { throw new Error2(ErrorCodes.FS_GREP_TIMEOUT, `grep timed out after ${Date.now() - startedAt}ms`); } - for (const file of filePaths) { - if ((signal.aborted || Date.now() >= deadline) && filesScanned > 0) { + for (let offset = 0; offset < filePaths.length; offset += 128) { + const expired = signal.aborted || Date.now() >= deadline; + if ((expired && filesScanned > 0) || filesScanned >= req.max_files) { truncated = true; break; } - if (filesScanned >= req.max_files) { - truncated = true; - break; - } - filesScanned += 1; - let content: string; - try { - content = await this.hostFs.readText(file.absolute); - } catch { - continue; - } - const lines = content.split(/\r?\n/); - const matches: FsGrepMatch[] = []; - for (let i = 0; i < lines.length; i++) { - const line = lines[i]!; - re.lastIndex = 0; - const m = re.exec(line); - if (m === null) continue; - if (matches.length >= req.max_matches_per_file) break; - const before: string[] = []; - for (let k = Math.max(0, i - req.context_lines); k < i; k++) { - before.push(lines[k] ?? ''); - } - const after: string[] = []; - for (let k = i + 1; k < Math.min(lines.length, i + 1 + req.context_lines); k++) { - after.push(lines[k] ?? ''); - } - matches.push({ line: i + 1, col: m.index + 1, text: line, before, after }); - totalMatches += 1; - if (totalMatches >= req.max_total_matches) { - truncated = true; - break; - } - } - if (matches.length > 0) { - files.push({ path: file.path, matches }); - } - if (totalMatches >= req.max_total_matches) break; + const options = { ...req, max_files: req.max_files - filesScanned, max_total_matches: req.max_total_matches - totalMatches }; + const batch = filePaths.slice(offset, offset + (expired ? 1 : 128)); + truncated ||= expired; + const remainingMs = Math.max(1, deadline - Date.now()); + let result = this.hostFs.operations === undefined + ? await matchFiles(this.hostFs, batch, options, remainingMs, signal) + : await this.hostFs.operations.matchFiles(batch, options, remainingMs); + if (result.responseTooLarge) result = await matchFiles(this.hostFs, batch, options, Math.max(1, deadline - Date.now()), signal); + files.push(...result.files); + filesScanned += result.files_scanned; + totalMatches += result.files.reduce((sum, file) => sum + file.matches.length, 0); + truncated ||= result.truncated; + if (result.truncated || totalMatches >= req.max_total_matches) break; } return { files, files_scanned: filesScanned, truncated, elapsed_ms: Date.now() - startedAt }; diff --git a/packages/agent-core-v2/src/workspace/workspaceFs/internal/fsSearch.ts b/packages/agent-core-v2/src/workspace/workspaceFs/internal/fsSearch.ts index 1444ca48165..755048e9191 100644 --- a/packages/agent-core-v2/src/workspace/workspaceFs/internal/fsSearch.ts +++ b/packages/agent-core-v2/src/workspace/workspaceFs/internal/fsSearch.ts @@ -1,6 +1,6 @@ import picomatch from 'picomatch'; -import type { FsGrepRequest } from '../fs'; +export { compileGrepPattern } from '#/os/operations/matchFiles'; export function computeFuzzyScore(name: string, queryLower: string): number { if (queryLower.length === 0) return 0; @@ -83,16 +83,6 @@ function globSegmentsMatchPrefix(globSegments: readonly string[], relSegments: r return dp[0] === 1; } -export function compileGrepPattern(req: FsGrepRequest): RegExp { - const flags = req.case_sensitive ? 'g' : 'gi'; - const body = req.regex ? req.pattern : escapeRegExp(req.pattern); - return new RegExp(body, flags); -} - -function escapeRegExp(s: string): string { - return s.replaceAll(/[.*+?^${}()|[\]\\]/g, '\\$&'); -} - export function stripTrailingNewline(s: string): string { if (s.endsWith('\r\n')) return s.slice(0, -2); if (s.endsWith('\n')) return s.slice(0, -1); diff --git a/packages/agent-core-v2/test/features/fileHistory/fileHistory.test.ts b/packages/agent-core-v2/test/features/fileHistory/fileHistory.test.ts index 3ff45c97299..3c7382412fe 100644 --- a/packages/agent-core-v2/test/features/fileHistory/fileHistory.test.ts +++ b/packages/agent-core-v2/test/features/fileHistory/fileHistory.test.ts @@ -91,9 +91,9 @@ describe('AgentFileHistoryService', () => { disposables.dispose(); }); - function stubEnvironment(remoteShape = false): IAgentEnvironmentService { + function stubEnvironment(remoteShape = false, fs: IHostFileSystem = hostFs(remoteShape)): IAgentEnvironmentService { const environment = { - fs: hostFs(remoteShape), + fs, path: posix, workspace: { mapRoots: (roots: unknown) => roots }, }; @@ -127,7 +127,7 @@ describe('AgentFileHistoryService', () => { }); } - function createService(agentId = 'main', remoteShape = false): AgentFileHistoryService { + function createService(agentId = 'main', remoteShape = false, fs?: IHostFileSystem): AgentFileHistoryService { const ctx = agentId === scopeCtx.agentId ? scopeCtx @@ -151,7 +151,7 @@ describe('AgentFileHistoryService', () => { executorEvents.executor, eventBus, ix.get(IEventDispatcher), - stubEnvironment(remoteShape), + stubEnvironment(remoteShape, fs), blobs, workspace, { @@ -216,6 +216,17 @@ describe('AgentFileHistoryService', () => { return bytes === undefined ? undefined : decoder.decode(bytes); } + it('treats a failed snapshot RPC as unreadable without recording a deletion', async () => { + const fs = hostFs(); + Object.defineProperty(fs, 'operations', { value: { snapshotFiles: async () => { throw new Error('connection closed'); } } }); + const service = createService('main', true, fs); + setFile('/ws/a.txt', 'original'); + startTurn(1); + await fireEdit(service, '/ws/a.txt', 1); + expect(service.history().checkpoints).toEqual([]); + expect(await blobs.list(scopeCtx.scope(), 'file-history/')).toEqual([]); + }); + it('backs up pre-edit content on first touch and versions changes at the next turn boundary', async () => { const service = createService(); setFile('/ws/a.txt', 'one\ntwo\n'); diff --git a/packages/agent-core-v2/test/lint/import-boundaries.test.ts b/packages/agent-core-v2/test/lint/import-boundaries.test.ts index bf3165fcf8e..d2ef1016e5c 100644 --- a/packages/agent-core-v2/test/lint/import-boundaries.test.ts +++ b/packages/agent-core-v2/test/lint/import-boundaries.test.ts @@ -227,6 +227,19 @@ import { HostEnvironmentService } from '#/os/backends/node-local/hostEnvironment expect(violations.every((violation) => violation.message.includes('remote executor stays light'))).toBe(true); }); + it('keeps shared host operations free of engine services and transport', () => { + const violations = checkSource( + `import type { IHostFileSystem } from '#/os/interface/hostFileSystem'; +import { decodeTextWithErrors } from '#/_base/execEnv/decodeText'; +import { IFileEditService } from '#/app/edit/fileEdit'; +import { RemoteExecConnection } from '#/remote/client/connection'; +import { IAgentProfileService } from '#/agent/profile/profile';`, + at('os/operations', 'readTextPage.ts'), + ); + expect(violations).toHaveLength(3); + expect(violations.every((violation) => violation.message.includes('shared host operations stay DI-free'))).toBe(true); + }); + it('keeps the remote os backend free of the remote transport and engine domains', () => { const violations = checkSource( `import type { ExecChannel } from '#/remote/protocol/channel'; diff --git a/packages/agent-core-v2/test/remote/fs.test.ts b/packages/agent-core-v2/test/remote/fs.test.ts index b2a067f1255..49cc44a3de8 100644 --- a/packages/agent-core-v2/test/remote/fs.test.ts +++ b/packages/agent-core-v2/test/remote/fs.test.ts @@ -1,8 +1,12 @@ -import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'; +import { createHash } from 'node:crypto'; +import { HostFileSystem } from '#/os/backends/node-local/hostFsService'; +import { readTextPage } from '#/os/operations/readTextPage'; +import { TEXT_PAGE_MAX_REMOTE_CHARS, type TextPageOptions } from '#/os/interface/fileOperations'; +import { mkdir, mkdtemp, readFile, rm, writeFile, utimes } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; -import { afterAll, beforeAll, describe, expect, it } from 'vitest'; +import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest'; import { HostFsError } from '#/os/interface/hostFsErrors'; import { HOST_WALK_MAX_IGNORE_BYTES } from '#/os/interface/hostFsWalk'; @@ -42,6 +46,243 @@ describe('fs group over a subprocess loopback', () => { await rm(workDir, { recursive: true, force: true }); }); + it('fetches exactly the requested sniff bytes', async () => { + const path = join(workDir, 'sniff.txt'); + await writeFile(path, 'x'.repeat(2 * 1024 * 1024)); + const calls = vi.spyOn(connection, 'call'); + try { + expect((await fs.readBytes(path, 512)).byteLength).toBe(512); + expect(calls.mock.calls).toEqual([['fs/readFile', { path, offset: 0, maxBytes: 512 }]]); + const reply = await calls.mock.results[0]!.value; + expect(Buffer.byteLength(JSON.stringify(reply))).toBeLessThan(1024); + } finally { calls.mockRestore(); } + }); + + it('reads a bounded page of a large file in one RPC', async () => { + const path = join(workDir, 'page.txt'); + await writeFile(path, 'hello world\n'.repeat(1_000_000)); + const calls = vi.spyOn(connection, 'call'); + try { + const result = await fs.operations.readTextPage(path, pageOptions(path, { n_lines: 3 })); + expect(result.output).toBe('1\thello world\n2\thello world\n3\thello world'); + expect(result.note).toContain('Total lines in file: 1000000'); + expect(calls.mock.calls.map(([method]) => method)).toEqual(['fs/readTextPage']); + expect(Buffer.byteLength(JSON.stringify(await calls.mock.results[0]!.value))).toBeLessThan(1024); + } finally { calls.mockRestore(); } + }); + + it.each([ + ['crlf', Buffer.from('alpha\r\nbeta\r\ngamma\r\n')], + ['mixed', Buffer.from('alpha\r\nbeta\ngam\rma')], + ['utf16', Buffer.from('\uFEFFalpha\r\nbeta😀\r\ngamma', 'utf16le')], + ['binary', Buffer.from('alpha\n\0beta')], + ['long', Buffer.from('😀'.repeat(4000) + '\nlast\n')], + ])('preserves local Read semantics for %s', async (name, bytes) => { + const path = join(workDir, `${name}.txt`); + await writeFile(path, bytes); + const local = new HostFileSystem(); + const source = { name: path, stat: () => local.stat(path), readBytes: (n?: number) => local.readBytes(path, n), readLines: () => local.readLines(path, { errors: 'strict' }) }; + for (const args of [{ n_lines: 1 }, { line_offset: -2, n_lines: 1 }, { line_offset: -3 }, { column_offset: 2, n_lines: 1 }]) { + const options = pageOptions(path, args); + expect(await fs.operations.readTextPage(path, options)).toEqual(await readTextPage(source, options)); + } + }); + + it('preserves configured Read budgets larger than the remote page limit', async () => { + const path = join(workDir, 'large-read-budget.txt'); + await writeFile(path, 'first\nsecond\n'); + const calls = vi.spyOn(connection, 'call'); + try { + const options = { ...pageOptions(path), maxChars: TEXT_PAGE_MAX_REMOTE_CHARS + 1, maxCharsLimit: TEXT_PAGE_MAX_REMOTE_CHARS + 1 }; + expect((await fs.operations.readTextPage(path, options)).output).toBe('1\tfirst\n2\tsecond'); + expect(calls.mock.calls.some(([method]) => method === 'fs/readTextPage')).toBe(false); + expect(calls.mock.calls.some(([method]) => method === 'fs/readFile')).toBe(true); + } finally { calls.mockRestore(); } + }); + + it('keeps concurrent Read calls working above the executor scan limit', async () => { + const path = join(workDir, 'concurrent-read.txt'); + await writeFile(path, 'first\nsecond\n'); + const calls = vi.spyOn(connection, 'call'); + try { + const results = await Promise.all(Array.from({ length: 12 }, () => fs.operations.readTextPage(path, pageOptions(path)))); + expect(results.every((result) => !result.isError && result.output === '1\tfirst\n2\tsecond')).toBe(true); + expect(calls.mock.calls.filter(([method]) => method === 'fs/readTextPage')).toHaveLength(8); + expect(calls.mock.calls.some(([method]) => method === 'fs/readFile')).toBe(true); + } finally { calls.mockRestore(); } + }); + + it('preserves Read and Edit error results when an operation RPC fails', async () => { + const path = join(workDir, 'rpc-failure.txt'); + const calls = vi.spyOn(connection, 'call').mockRejectedValue(new Error('connection closed')); + try { + expect(await fs.operations.readTextPage(path, pageOptions(path))).toEqual({ isError: true, output: 'connection closed' }); + expect(await fs.operations.editText({ path, displayPath: 'rpc-failure.txt', old_string: 'old', new_string: 'new', replace_all: false })).toEqual({ ok: false, error: 'connection closed' }); + } finally { calls.mockRestore(); } + }); + + it('edits on the execution host without downloading or uploading the full file', async () => { + const path = join(workDir, 'edit-page.txt'); + const before = 'a'.repeat(3 * 1024 * 1024) + '\r\nold value\r\n'; + await writeFile(path, before); + const calls = vi.spyOn(connection, 'call'); + try { + expect(await fs.operations.editText({ path, displayPath: 'edit-page.txt', old_string: 'old value', new_string: 'new value\nnext', replace_all: false })).toEqual({ ok: true, count: 1 }); + expect(calls.mock.calls.map(([method]) => method)).toEqual(['fs/editText']); + expect(Buffer.byteLength(JSON.stringify(calls.mock.calls))).toBeLessThan(1024); + expect(await readFile(path, 'utf8')).toBe(before.replace('old value', 'new value\r\nnext')); + const failure = await fs.operations.editText({ path, displayPath: 'edit-page.txt', old_string: 'absent', new_string: '', replace_all: false }); + expect(failure.ok).toBe(false); + } finally { calls.mockRestore(); } + }); + + it('creates the parent and writes text in a single RPC', async () => { + const path = join(workDir, 'created', 'nested', 'out.txt'); + const calls = vi.spyOn(connection, 'call'); + try { + const result = await fs.operations.writeText({ path, parent: join(workDir, 'created', 'nested'), displayPath: 'out.txt', content: 'hello', mode: 'overwrite' }); + expect(result).toEqual({ output: 'Wrote 5 bytes to out.txt' }); + expect(calls.mock.calls.map(([method]) => method)).toEqual(['fs/writeFile']); + expect(await readFile(path, 'utf8')).toBe('hello'); + } finally { calls.mockRestore(); } + }); + + it('batches snapshots and skips bytes only for a matching content hash', async () => { + const path = join(workDir, 'snapshot.txt'); + const bytes = Buffer.from('before'); + await writeFile(path, bytes); + const knownHash = createHash('sha256').update(bytes).digest('hex'); + const calls = vi.spyOn(connection, 'call'); + try { + const snapshots = await fs.operations.snapshotFiles([{ path, knownHash }, { path: join(workDir, 'missing-snapshot') }], 1024); + expect(snapshots).toEqual([{ kind: 'file', contentHash: knownHash, bytes: undefined }, { kind: 'missing' }]); + expect(calls.mock.calls.map(([method]) => method)).toEqual(['fs/snapshotFiles']); + const stat = await fs.stat(path); + await writeFile(path, 'after!'); + await utimes(path, new Date(), new Date(stat.mtimeMs!)); + const changed = (await fs.operations.snapshotFiles([{ path, knownHash }], 1024))[0]!; + expect(changed.kind).toBe('file'); + if (changed.kind === 'file') expect(Buffer.from(changed.bytes!).toString()).toBe('after!'); + expect(await fs.operations.snapshotFiles([{ path }], 2)).toMatchObject([{ kind: 'oversize', size: 6 }]); + } finally { calls.mockRestore(); } + }); + + it('discovers Git and instruction files without per-directory RPCs or stale negative caching', async () => { + const root = join(workDir, 'probe'); + const nested = join(root, 'a', 'b', 'c'); + await mkdir(nested, { recursive: true }); + await mkdir(join(root, '.git')); + await writeFile(join(root, 'AGENTS.md'), 'root instructions'); + const calls = vi.spyOn(connection, 'call'); + try { + expect((await fs.operations.findGitWorkTree(nested))?.root).toBe(root); + expect(await fs.operations.discoverInstructions([nested, join(nested, 'missing')], [])).toEqual([join(root, 'AGENTS.md')]); + expect(calls.mock.calls.map(([method]) => method)).toEqual(['fs/findGitWorkTree', 'fs/discoverInstructions']); + await writeFile(join(nested, 'AGENTS.md'), 'new instructions'); + expect(await fs.operations.discoverInstructions([nested], [join(root, 'AGENTS.md')])).toContain(join(nested, 'AGENTS.md')); + expect(await fs.operations.findGitWorkTree('/')).toBeNull(); + } finally { calls.mockRestore(); } + }); + + it('pipelines a bounded window of write chunks in order', async () => { + const path = join(workDir, 'pipelined.bin'); + const original = connection.call.bind(connection); + let active = 0; + let peak = 0; + const calls = vi.spyOn(connection, 'call').mockImplementation(async (method, params) => { + if (method !== 'fs/writeFile') return original(method, params); + active += 1; + peak = Math.max(peak, active); + try { + await new Promise((resolve) => { setTimeout(resolve, 10); }); + return await original(method, params); + } finally { active -= 1; } + }); + try { + const bytes = Buffer.concat(Array.from({ length: 9 }, (_, index) => Buffer.alloc(1024 * 1024, index))); + await fs.writeBytes(path, bytes); + expect(peak).toBe(4); + expect((await readFile(path)).equals(bytes)).toBe(true); + } finally { calls.mockRestore(); } + }); + + it('does not append later chunks after an earlier exclusive write fails', async () => { + const path = join(workDir, 'failed-pipeline.txt'); + await writeFile(path, 'original'); + const results = await Promise.allSettled([0, 1, 2].map((chunkIndex) => connection.call('fs/writeFile', { + path, transferId: 'failed-write', chunkIndex, final: chunkIndex === 2, + mode: chunkIndex === 0 ? 'exclusive' : 'append', dataBase64: Buffer.from('changed').toString('base64'), + }))); + expect(results.every((result) => result.status === 'rejected')).toBe(true); + expect(await readFile(path, 'utf8')).toBe('original'); + }); + + it('rejects duplicate and out-of-order chunks without repeating an append', async () => { + const path = join(workDir, 'ordered-write.txt'); + const first = { path, transferId: 'ordered-write', chunkIndex: 0, mode: 'append', dataBase64: Buffer.from('one').toString('base64') }; + await connection.call('fs/writeFile', first); + await expect(connection.call('fs/writeFile', first)).rejects.toThrow('once and in order'); + await expect(connection.call('fs/writeFile', { ...first, chunkIndex: 2 })).rejects.toThrow('once and in order'); + await connection.call('fs/writeFile', { ...first, chunkIndex: 1, final: true, dataBase64: Buffer.from('two').toString('base64') }); + expect(await readFile(path, 'utf8')).toBe('onetwo'); + }); + + it('cancels a remote text scan', async () => { + const path = join(workDir, 'cancel-read.txt'); + await writeFile(path, 'line\n'.repeat(1_000_000)); + const controller = new AbortController(); + const pending = fs.operations.readTextPage(path, pageOptions(path), controller.signal); + const timer = setTimeout(() => { controller.abort(); }, 20); + try { + expect(await pending).toEqual({ isError: true, output: 'Read aborted' }); + } finally { clearTimeout(timer); } + }); + + it('matches a batch of files without transferring nonmatching contents', async () => { + const paths = []; + for (let index = 0; index < 12; index += 1) { + const absolute = join(workDir, `match-${index}.txt`); + await writeFile(absolute, 'no match here\n'.repeat(10_000) + 'needle\n'); + paths.push({ path: `match-${index}.txt`, absolute }); + } + const calls = vi.spyOn(connection, 'call'); + try { + const result = await fs.operations.matchFiles(paths, { pattern: 'needle', regex: false, case_sensitive: true, context_lines: 0, max_files: 100, max_matches_per_file: 10, max_total_matches: 100 }, 30_000); + expect(result.files_scanned).toBe(12); + expect(result.files).toHaveLength(12); + expect(result.files[0]?.matches).toEqual([{ line: 10001, col: 1, text: 'needle', before: [], after: [] }]); + expect(calls.mock.calls.map(([method]) => method)).toEqual(['fs/matchFiles']); + expect(Buffer.byteLength(JSON.stringify(await calls.mock.results[0]!.value))).toBeLessThan(4096); + } finally { calls.mockRestore(); } + }); + + it('requests legacy matching when a remote result would exceed the response budget', async () => { + const path = join(workDir, 'long-match.txt'); + await writeFile(path, 'needle' + 'x'.repeat(8 * 1024 * 1024)); + const result = await fs.operations.matchFiles([{ path: 'long-match.txt', absolute: path }], { pattern: 'needle', regex: false, case_sensitive: true, context_lines: 0, max_files: 1, max_matches_per_file: 1, max_total_matches: 1 }, 30_000); + expect(result).toEqual({ files: [], files_scanned: 0, truncated: false, responseTooLarge: true }); + }); + + it('reuses written and edited bytes only after the executor verifies their hash', async () => { + const path = join(workDir, 'reuse.txt'); + await fs.operations.writeText({ path, parent: workDir, displayPath: 'reuse.txt', content: 'old value', mode: 'overwrite' }); + const calls = vi.spyOn(connection, 'call'); + try { + const written = (await fs.operations.snapshotFiles([{ path }], 1024))[0]!; + expect(written.kind).toBe('file'); + if (written.kind === 'file') expect(Buffer.from(written.bytes!).toString()).toBe('old value'); + expect(JSON.stringify(await calls.mock.results[0]!.value)).not.toContain('dataBase64'); + calls.mockClear(); + await fs.operations.editText({ path, displayPath: 'reuse.txt', old_string: 'old', new_string: 'new', replace_all: false }); + const edited = (await fs.operations.snapshotFiles([{ path }], 1024))[0]!; + if (edited.kind === 'file') expect(Buffer.from(edited.bytes!).toString()).toBe('new value'); + expect(JSON.stringify(await calls.mock.results[1]!.value)).not.toContain('dataBase64'); + await writeFile(path, 'external!'); + const changed = (await fs.operations.snapshotFiles([{ path }], 1024))[0]!; + if (changed.kind === 'file') expect(Buffer.from(changed.bytes!).toString()).toBe('external!'); + } finally { calls.mockRestore(); } + }); + it('writes a byte stream progressively without retaining the complete input', async () => { const path = join(workDir, 'stream.bin'); await fs.writeText(path, 'old content'); @@ -318,3 +559,7 @@ describe('fs protocol semantics', () => { } }, 60_000); }); + +function pageOptions(path: string, args: Partial = {}): TextPageOptions { + return { args: { path, ...args }, maxChars: 2000, maxCharsLimit: 500_000, eventLog: false, imageSupported: false, videoSupported: false, mediaToolAvailable: false }; +} diff --git a/packages/agent-core-v2/test/remote/process.test.ts b/packages/agent-core-v2/test/remote/process.test.ts index 6dfc6125ce0..1474225239b 100644 --- a/packages/agent-core-v2/test/remote/process.test.ts +++ b/packages/agent-core-v2/test/remote/process.test.ts @@ -1,5 +1,8 @@ +import type { SearchRequest } from '#/os/interface/searchOperations'; +import { searchGrep } from '#/os/operations/searchGrep'; +import { searchGlob } from '#/os/operations/searchGlob'; import { randomUUID } from 'node:crypto'; -import { mkdtemp, realpath, rm, writeFile } from 'node:fs/promises'; +import { mkdtemp, readFile, realpath, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; @@ -563,3 +566,96 @@ describe('finished process-group ownership', () => { } finally { manager.dispose(); kill.mockRestore(); vi.useRealTimers(); } }); }); + +describe('search processing on the execution host', () => { + let directory: string; + let connection: RemoteExecConnection; + let processes: RemoteProcessService; + let loopback: Awaited>['loopback']; + + beforeAll(async () => { + directory = await mkdtemp(join(tmpdir(), 'remote-search-')); + ({ connection, loopback } = await connectInProcess()); + processes = new RemoteProcessService(connection, directory, '/bin/bash'); + }); + + afterAll(async () => { + connection.close(); + await loopback.host.done; + await rm(directory, { recursive: true, force: true }); + }); + + it.each(['grep', 'glob'] as const)('returns only the selected %s page and exact total over one RPC', async (kind) => { + const script = join(directory, `large-${kind}`); + const separator = kind === 'grep' ? '\0' : '\n'; + const program = `#!/usr/bin/env node\nprocess.stdout.write(Array.from({length:50000},(_,i)=>${JSON.stringify(directory + '/')}+'file-'+i+'.txt'+${JSON.stringify(separator)}).join(''));`; + await writeFile(script, program, { mode: 0o755 }); + const request: SearchRequest = { kind, args: { pattern: '*', head_limit: 3, offset: 2 }, rgPath: script, roots: [directory], workspaceDir: directory, pathClass: 'posix' }; + const calls = vi.spyOn(connection, 'call'); + let receivedBytes = 0; + const receive = (chunk: Buffer): void => { receivedBytes += chunk.byteLength; }; + loopback.serverOutput.on('data', receive); + try { + const remote = await processes.search(request, new AbortController().signal); + const local = await (kind === 'grep' ? searchGrep : searchGlob)(new HostProcessService(), request, new AbortController().signal); + expect(remote).toEqual(local); + expect(remote.output).toContain('50000'); + expect(remote.output).toContain('file-2.txt'); + expect(remote.output).toContain('file-4.txt'); + expect(remote.output).not.toContain('file-5.txt'); + expect(receivedBytes).toBeLessThan(2048); + expect(calls.mock.calls.map(([method]) => method)).toEqual(['process/search']); + } finally { + calls.mockRestore(); + loopback.serverOutput.off('data', receive); + } + }); + + it.each(['content', 'count_matches', 'files_with_matches'] as const)('preserves real ripgrep %s output including filtering and counts', async (output_mode) => { + const file = join(directory, 'sample.txt'); + await writeFile(file, 'before\nmatch here\nafter\nmatch again\n'); + const request: SearchRequest = { kind: 'grep', args: { pattern: 'match', output_mode, '-C': 1, head_limit: 2 }, rgPath: 'rg', roots: [file], workspaceDir: directory, pathClass: 'posix' }; + const signal = new AbortController().signal; + expect(await processes.search(request, signal)).toEqual(await searchGrep(new HostProcessService(), request, signal)); + }); + + it('keeps search working with more roots than fit the remote operation', async () => { + const script = join(directory, 'many-roots'); + await writeFile(script, `#!/usr/bin/env node\nprocess.stdout.write(${JSON.stringify(directory + '/found.txt\0')});`, { mode: 0o755 }); + const request: SearchRequest = { kind: 'grep', args: { pattern: 'match' }, rgPath: script, roots: Array.from({ length: 65 }, () => directory), workspaceDir: directory, pathClass: 'posix' }; + const calls = vi.spyOn(connection, 'call'); + try { + const result = await processes.search(request, new AbortController().signal); + expect(result.isError).not.toBe(true); + expect(result.output).toContain('found.txt'); + expect(calls.mock.calls.some(([method]) => method === 'process/search')).toBe(false); + expect(calls.mock.calls.some(([method]) => method === 'process/start')).toBe(true); + } finally { calls.mockRestore(); } + }); + + it('keeps concurrent searches working above the executor search limit', async () => { + const script = join(directory, 'concurrent-search'); + await writeFile(script, `#!/usr/bin/env node\nprocess.stdout.write(${JSON.stringify(directory + '/found.txt\0')});`, { mode: 0o755 }); + const request: SearchRequest = { kind: 'grep', args: { pattern: 'match' }, rgPath: script, roots: [directory], workspaceDir: directory, pathClass: 'posix' }; + const calls = vi.spyOn(connection, 'call'); + try { + const results = await Promise.all(Array.from({ length: 12 }, () => processes.search(request, new AbortController().signal))); + expect(results.every((result) => !result.isError && result.output.includes('found.txt'))).toBe(true); + expect(calls.mock.calls.filter(([method]) => method === 'process/search')).toHaveLength(8); + expect(calls.mock.calls.some(([method]) => method === 'process/start')).toBe(true); + } finally { calls.mockRestore(); } + }); + + it('cancels and reaps a running search without streaming output to the controller', async () => { + const script = join(directory, 'slow-search'); + const pidFile = join(directory, 'search.pid'); + await writeFile(script, `#!/usr/bin/env node\nrequire('node:fs').writeFileSync(${JSON.stringify(pidFile)},String(process.pid));setInterval(()=>{},1000);`, { mode: 0o755 }); + const controller = new AbortController(); + const pending = processes.search({ kind: 'grep', args: { pattern: 'match' }, rgPath: script, roots: [directory], workspaceDir: directory, pathClass: 'posix' }, controller.signal); + let pid: number | undefined; + await vi.waitFor(async () => { pid = Number(await readFile(pidFile, 'utf8')); }, { timeout: 5000 }); + controller.abort(); + expect(await pending).toEqual({ isError: true, output: 'Grep aborted' }); + await waitForDeath(pid!); + }); +}); diff --git a/packages/agent-core-v2/test/remote/protocol.test.ts b/packages/agent-core-v2/test/remote/protocol.test.ts index b9a3826207e..605114b585e 100644 --- a/packages/agent-core-v2/test/remote/protocol.test.ts +++ b/packages/agent-core-v2/test/remote/protocol.test.ts @@ -144,6 +144,17 @@ describe('request call timeout', () => { connection.close(); }); + it('lets cancellable text scans outlive the per-call transport timeout', async () => { + const pipe = createScriptedServer((frame, reply) => { + if (frame.method === INITIALIZE_METHOD) reply({ id: frame.id, result: testInitializeResult() }); + else if (frame.method === 'fs/readTextPage') setTimeout(() => { reply({ id: frame.id, result: { output: 'complete' } }); }, 80); + }); + const connection = await RemoteExecConnection.connect(pipe, { clientName: 'test', clientVersion: '0.0.0', requestCallTimeoutMs: 20 }); + try { + await expect(connection.call('fs/readTextPage', {})).resolves.toEqual({ output: 'complete' }); + } finally { connection.close(); } + }); + it('cancels a process start that timed out, so the late spawn cannot go orphan', async () => { const seen: ScriptedFrame[] = []; const pipe = createScriptedServer((frame, reply) => { diff --git a/packages/agent-core-v2/test/workspace/workspaceFs/fsService.test.ts b/packages/agent-core-v2/test/workspace/workspaceFs/fsService.test.ts index 76b4761478c..451f776b84a 100644 --- a/packages/agent-core-v2/test/workspace/workspaceFs/fsService.test.ts +++ b/packages/agent-core-v2/test/workspace/workspaceFs/fsService.test.ts @@ -1503,6 +1503,21 @@ describe('WorkspaceFsService.grep', () => { }); }); + it('keeps the legacy matches when the remote matching response is too large', async () => { + const hostFs = fakeFs({ 'src/a.ts': 'hello world\nfoo bar\nhello again\n' }); + const matchFiles = vi.fn(async () => ({ files: [], files_scanned: 0, truncated: false, responseTooLarge: true })); + Object.defineProperty(hostFs, 'operations', { value: { matchFiles } }); + const fs = makeRemoteSession({}, () => ({ stdout: '', exitCode: 1 }), [], 'ssh-dev', '/home/target', hostFs); + const result = await fs.grep({ pattern: 'hello', regex: false, case_sensitive: true, follow_gitignore: false, max_files: 200, max_matches_per_file: 50, max_total_matches: 5000, context_lines: 0 }); + expect(matchFiles).toHaveBeenCalledOnce(); + expect(result.files_scanned).toBe(1); + expect(result.truncated).toBe(false); + expect(result.files).toEqual([{ path: 'src/a.ts', matches: [ + { line: 1, col: 1, text: 'hello world', before: [], after: [] }, + { line: 3, col: 1, text: 'hello again', before: [], after: [] }, + ] }]); + }); + it('uses rg when available and parses its JSON output', async () => { const rgJson = [ JSON.stringify({ type: 'begin', data: { path: { text: 'src/a.ts' } } }), From 27e4977eadeb50ae0774a80d8114732f5b942af9 Mon Sep 17 00:00:00 2001 From: 7Sageer <125936732+7Sageer@users.noreply.github.com> Date: Sat, 31 Oct 2026 00:31:45 +0800 Subject: [PATCH 8/8] refactor: simplify remote file operation optimizations --- .changeset/batch-remote-filesystem-walks.md | 2 +- AGENTS.md | 2 +- .../scripts/check-import-boundaries.mjs | 14 +- .../agent-core-v2/src/_base/fs/pathFilters.ts | 85 --- .../agent-core-v2/src/_base/media/fileType.ts | 368 ------------- .../src/_base/media/mimeTypes.ts | 44 -- .../src/_base/text/outputAccumulator.ts | 94 ---- .../src/_base/text/outputLimits.ts | 9 - .../src/_base/text/resultStatus.ts | 12 - .../agentsMdReminderService.ts | 49 +- .../agent/contextMemory/toolResultRender.ts | 17 +- .../src/agent/media/file-type.ts | 369 ++++++++++++- .../agent-core-v2/src/agent/media/mediaRef.ts | 48 +- .../src/agent/profile/context.ts | 59 +- .../src/agent/tools/fileReadSource.ts | 14 +- .../src/agent/tools/os/glob/globTool.ts | 218 +++++++- .../src/agent/tools/os/grep/grepTool.ts | 508 ++++++++++++++++- .../src/agent/tools/os/read/read.ts | 2 +- .../src/agent/tools/os/read/readTool.ts | 511 +++++++++++++++++- .../src/agent/tools/os/write/writeTool.ts | 58 +- .../agent-core-v2/src/app/edit/editService.ts | 42 +- .../agent-core-v2/src/app/edit/fileEdit.ts | 13 +- .../src/app/edit/fileEditService.ts | 35 +- .../agent-core-v2/src/app/edit/textModel.ts | 44 +- .../agent-core-v2/src/app/git/workTree.ts | 55 +- .../fileHistory/fileHistoryService.ts | 123 ++--- .../src/os/backends/node-local/tools/runRg.ts | 197 ++++++- .../os/backends/remote/remoteFileSystem.ts | 230 ++------ .../src/os/backends/remote/remoteProcess.ts | 30 - .../src/os/interface/fileOperations.ts | 115 ---- .../src/os/interface/hostFileSystem.ts | 10 +- .../src/os/interface/hostProcess.ts | 4 - .../src/os/interface/searchOperations.ts | 33 -- .../src/os/operations/editService.ts | 41 -- .../src/os/operations/editText.ts | 32 -- .../src/os/operations/gitWorkTree.ts | 50 -- .../src/os/operations/instructions.ts | 107 ---- .../src/os/operations/matchFiles.ts | 72 --- .../src/os/operations/readTextPage.ts | 481 ----------------- .../agent-core-v2/src/os/operations/runRg.ts | 202 ------- .../src/os/operations/searchGlob.ts | 208 ------- .../src/os/operations/searchGrep.ts | 502 ----------------- .../src/os/operations/snapshotFiles.ts | 39 -- .../src/os/operations/textModel.ts | 43 -- .../src/os/operations/writeText.ts | 53 -- packages/agent-core-v2/src/remote/README.md | 54 +- .../src/remote/client/connection.ts | 3 +- .../client/remoteEnvironmentProvider.ts | 12 +- .../src/remote/protocol/methods.ts | 22 - .../remote/server/fileOperationsHandler.ts | 135 ----- .../src/remote/server/fsHandler.ts | 73 +-- .../src/remote/server/searchHandler.ts | 76 --- .../src/remote/server/stdioHost.ts | 38 +- .../src/tool/output-accumulator.ts | 98 +++- .../agent-core-v2/src/tool/path-access.ts | 87 ++- .../agent-core-v2/src/tool/toolContract.ts | 11 +- .../src/workspace/workspaceFs/fsService.ts | 59 +- .../workspaceFs/internal/fsSearch.ts | 12 +- .../features/fileHistory/fileHistory.test.ts | 19 +- .../test/lint/import-boundaries.test.ts | 13 - .../os/backends/node-local/tools/glob.test.ts | 7 +- .../os/backends/node-local/tools/read.test.ts | 19 + .../backends/node-local/tools/write.test.ts | 13 + packages/agent-core-v2/test/remote/fs.test.ts | 283 ++-------- .../agent-core-v2/test/remote/process.test.ts | 98 +--- .../test/remote/protocol.test.ts | 11 - .../workspace/workspaceFs/fsService.test.ts | 15 - 67 files changed, 2650 insertions(+), 3752 deletions(-) delete mode 100644 packages/agent-core-v2/src/_base/fs/pathFilters.ts delete mode 100644 packages/agent-core-v2/src/_base/media/fileType.ts delete mode 100644 packages/agent-core-v2/src/_base/media/mimeTypes.ts delete mode 100644 packages/agent-core-v2/src/_base/text/outputAccumulator.ts delete mode 100644 packages/agent-core-v2/src/_base/text/outputLimits.ts delete mode 100644 packages/agent-core-v2/src/_base/text/resultStatus.ts delete mode 100644 packages/agent-core-v2/src/os/interface/fileOperations.ts delete mode 100644 packages/agent-core-v2/src/os/interface/searchOperations.ts delete mode 100644 packages/agent-core-v2/src/os/operations/editService.ts delete mode 100644 packages/agent-core-v2/src/os/operations/editText.ts delete mode 100644 packages/agent-core-v2/src/os/operations/gitWorkTree.ts delete mode 100644 packages/agent-core-v2/src/os/operations/instructions.ts delete mode 100644 packages/agent-core-v2/src/os/operations/matchFiles.ts delete mode 100644 packages/agent-core-v2/src/os/operations/readTextPage.ts delete mode 100644 packages/agent-core-v2/src/os/operations/runRg.ts delete mode 100644 packages/agent-core-v2/src/os/operations/searchGlob.ts delete mode 100644 packages/agent-core-v2/src/os/operations/searchGrep.ts delete mode 100644 packages/agent-core-v2/src/os/operations/snapshotFiles.ts delete mode 100644 packages/agent-core-v2/src/os/operations/textModel.ts delete mode 100644 packages/agent-core-v2/src/os/operations/writeText.ts delete mode 100644 packages/agent-core-v2/src/remote/server/fileOperationsHandler.ts delete mode 100644 packages/agent-core-v2/src/remote/server/searchHandler.ts diff --git a/.changeset/batch-remote-filesystem-walks.md b/.changeset/batch-remote-filesystem-walks.md index 69bdd20b7e1..9a16b8ec629 100644 --- a/.changeset/batch-remote-filesystem-walks.md +++ b/.changeset/batch-remote-filesystem-walks.md @@ -2,4 +2,4 @@ "@moonshot-ai/kimi-code": patch --- -Reduce latency for file operations and searches in remote environments. +Speed up remote file reads, edits, writes, search, and completion. diff --git a/AGENTS.md b/AGENTS.md index b80b90dd160..463158d0ad4 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -26,7 +26,7 @@ This is a TypeScript monorepo built for agent-assisted development. Keep the roo - `packages/session-import`: standalone Node.js conversion of external sessions into a single user message that carries the conversation, full effective-history attachments, and image references. It has no engine or UI dependency; callers provide the destination session attachment directory and context budget. - `packages/kap-server`: the Kimi Code server, backed by `@moonshot-ai/agent-core-v2`; exposes sessions over REST + WebSocket (`/api/v1` + `/api/v1/ws`), plus the `/api/v1/debug/*` reflection RPC surface (`--debug-endpoints`, loopback bind + bearer auth). - `packages/remote-control`: the Kimi Remote Control tunnel client — registers this machine with the relay and forwards HTTP/WebSocket traffic to the local server, with a machine-wide single-instance lock; consumed by kap-server (the `/api/v1/remote-control` toggle) and by the CLI (`kimi web --remote-control`). -- `packages/agent-core-v2/src/remote`: the remote execution stack inside the engine — self-contained NDJSON line-framed RPC (codex exec-server dialect) between the local `ExecBridge` (ssh/docker/`command` launchers) and the light `kimi exec-server` executor, plus the provider that registers declared targets as `Environment`s (the fs/process RPC stubs are an OS backend under `src/os/backends/remote`, sibling of `node-local`); exported as `@moonshot-ai/agent-core-v2/remote` and `/remote/server`. The executor stays light — `src/remote/server` may import only node builtins, relative modules, `#/remote/protocol`, `#/os/interface`, the DI-free shared `#/os/operations`, `#/_base/execEnv` and the DI-free node-local fs/process backends it adapts (enforced by the package's `lint:imports`); the CLI reaches it only through the SDK's light `@moonshot-ai/kimi-code-sdk/exec-server` subpath, loaded dynamically in `src/cli/exec-server.ts`. +- `packages/agent-core-v2/src/remote`: the remote execution stack inside the engine — self-contained NDJSON line-framed RPC (codex exec-server dialect) between the local `ExecBridge` (ssh/docker/`command` launchers) and the light `kimi exec-server` executor, plus the provider that registers declared targets as `Environment`s (the fs/process RPC stubs are an OS backend under `src/os/backends/remote`, sibling of `node-local`); exported as `@moonshot-ai/agent-core-v2/remote` and `/remote/server`. The executor stays light — `src/remote/server` may import only node builtins, relative modules, `#/remote/protocol`, `#/os/interface`, `#/_base/execEnv` and the DI-free node-local fs/process backends it adapts (enforced by the package's `lint:imports`); the CLI reaches it only through the SDK's light `@moonshot-ai/kimi-code-sdk/exec-server` subpath, loaded dynamically in `src/cli/exec-server.ts`. - `packages/klient`: the client SDK — a contract-driven facade over agent-core-v2 (`global.*` / `session(id).*` / `agent(id).*`, zod-validated); transport via subpath entry (`@moonshot-ai/klient/ipc|memory`, both return the same `Klient`); also hosts the e2e suites. See `packages/klient/AGENTS.md`. - `packages/tree-sitter-bash`: a pure-TypeScript bash parser (no runtime deps, no wasm); `parse(source, { timeoutMs, maxNodes })` runs under a deterministic budget and returns a discriminated `ParseResult` — callers must treat aborted/hasError trees as "cannot analyze" and degrade. Parser only, no safety judgments; see the package README's "Known differences" section. - `packages/minidb`: the embedded JSON document store (`MiniDb`) behind kap-server's search index — snapshot + WAL persistence with an exclusive write lock, a larger-than-RAM full-text layer, and persistent index generations. See `packages/minidb/AGENTS.md`. diff --git a/packages/agent-core-v2/scripts/check-import-boundaries.mjs b/packages/agent-core-v2/scripts/check-import-boundaries.mjs index 11c7035c346..7893fe96ccb 100644 --- a/packages/agent-core-v2/scripts/check-import-boundaries.mjs +++ b/packages/agent-core-v2/scripts/check-import-boundaries.mjs @@ -12,13 +12,11 @@ const LOOP_ADAPTER_ROOT = join(SRC_ROOT, 'agent/loop'); const SESSION_LIFECYCLE_ADAPTER_ROOT = join(SRC_ROOT, 'session/agentLifecycle'); const REMOTE_SERVER_ROOT = join(SRC_ROOT, 'remote/server'); const REMOTE_SERVER_ALLOWED_RE = - /^#\/(?:(?:remote\/protocol|os\/interface|os\/operations|_base\/execEnv)(?:\/|$)|os\/backends\/node-local\/(?:hostFsService|hostProcessService)$)/; + /^#\/(?:(?:remote\/protocol|os\/interface|_base\/execEnv)(?:\/|$)|os\/backends\/node-local\/(?:hostFsService|hostProcessService)$)/; const REMOTE_SERVER_MODULE_RE = /\/remote\/server(?:\/|$)/; const REMOTE_ASKPASS_FILE = join(SRC_ROOT, 'remote/askpass.ts'); -const OS_OPERATIONS_ROOT = join(SRC_ROOT, 'os/operations'); -const OS_OPERATIONS_ALLOWED_RE = /^#\/(?:os\/(?:interface|operations)|_base\/(?:execEnv|errors|text|media|fs))(?:\/|$)/; const OS_REMOTE_BACKEND_ROOT = join(SRC_ROOT, 'os/backends/remote'); -const OS_REMOTE_BACKEND_ALLOWED_RE = /^#\/(?:remote\/protocol|os\/interface|os\/operations|_base)(?:\/|$)/; +const OS_REMOTE_BACKEND_ALLOWED_RE = /^#\/(?:remote\/protocol|os\/interface|_base)(?:\/|$)/; const ENVIRONMENT_ROOT = join(SRC_ROOT, 'environment'); const REMOTE_ROOT = join(SRC_ROOT, 'remote'); const REMOTE_DECLARATION_ROOT = join(SRC_ROOT, 'remote/declaration'); @@ -41,7 +39,7 @@ function remoteServerViolation(absFile, specifier) { if (!isInside(REMOTE_SERVER_ROOT, absFile)) return undefined; if (specifier.startsWith('.') || specifier.startsWith('node:')) return undefined; if (REMOTE_SERVER_ALLOWED_RE.test(specifier)) return undefined; - return `the remote executor stays light: src/remote/server may import only node builtins, relative modules, #/remote/protocol, #/os/interface, #/os/operations, #/_base/execEnv and the node-local hostFsService / hostProcessService implementations ('${specifier}')`; + return `the remote executor stays light: src/remote/server may import only node builtins, relative modules, #/remote/protocol, #/os/interface, #/_base/execEnv and the node-local hostFsService / hostProcessService implementations ('${specifier}')`; } function remoteAskpassViolation(absFile, specifier) { @@ -53,7 +51,7 @@ function osRemoteBackendViolation(absFile, specifier) { if (!isInside(OS_REMOTE_BACKEND_ROOT, absFile)) return undefined; if (specifier.startsWith('.') || specifier.startsWith('node:')) return undefined; if (OS_REMOTE_BACKEND_ALLOWED_RE.test(specifier)) return undefined; - return `the remote os backend is a sibling of node-local: src/os/backends/remote may import only node builtins, relative modules, #/remote/protocol, #/os/interface, #/os/operations and #/_base ('${specifier}') — the transport and environment wiring live in src/remote/client`; + return `the remote os backend is a sibling of node-local: src/os/backends/remote may import only node builtins, relative modules, #/remote/protocol, #/os/interface and #/_base ('${specifier}') — the transport and environment wiring live in src/remote/client`; } function remoteServerConsumerViolation(absFile, specifier) { @@ -184,10 +182,6 @@ export function checkSource(source, absFile) { if (!inSrc) continue; - if (isInside(OS_OPERATIONS_ROOT, absFile) && !specifier.startsWith('node:') && !specifier.startsWith('./') && specifier !== 'pathe' && !OS_OPERATIONS_ALLOWED_RE.test(specifier)) { - violations.push({ file: absFile, line, message: `shared host operations stay DI-free and cannot import engine or transport modules ('${specifier}')` }); - continue; - } const remoteServer = remoteServerViolation(absFile, specifier); if (remoteServer !== undefined) { violations.push({ file: absFile, line, message: remoteServer }); diff --git a/packages/agent-core-v2/src/_base/fs/pathFilters.ts b/packages/agent-core-v2/src/_base/fs/pathFilters.ts deleted file mode 100644 index adaff55d15a..00000000000 --- a/packages/agent-core-v2/src/_base/fs/pathFilters.ts +++ /dev/null @@ -1,85 +0,0 @@ -import * as pathe from 'pathe'; - -const SENSITIVE_BASENAMES = new Set([ - '.env', - 'id_rsa', - 'id_ed25519', - 'id_ecdsa', - 'credentials', -]); - -const SENSITIVE_PATH_SUFFIXES = [ - ['.aws', 'credentials'], - ['.gcp', 'credentials'], -]; - -const ENV_PREFIX = '.env.'; -const ENV_EXEMPTIONS = new Set(['.env.example', '.env.sample', '.env.template']); - -const SENSITIVE_BASENAME_PREFIXES = ['id_rsa', 'id_ed25519', 'id_ecdsa', 'credentials']; -const PUBLIC_KEY_BASENAMES = new Set(['id_rsa.pub', 'id_ed25519.pub', 'id_ecdsa.pub']); -export const SENSITIVE_DOT_VARIANT_SUFFIXES = [ - '.bak', - '.backup', - '.copy', - '.disabled', - '.key', - '.old', - '.orig', - '.pem', - '.save', - '.tmp', -] as const; -const SENSITIVE_DOT_VARIANT_SUFFIX_SET = new Set(SENSITIVE_DOT_VARIANT_SUFFIXES); - -function comparable(path: string): string { - return path.toLowerCase(); -} - -export function isSensitiveFile(path: string): boolean { - const name = pathe.basename(path); - const comparableName = comparable(name); - const comparablePath = comparable(path); - - if (ENV_EXEMPTIONS.has(comparableName)) return false; - if (PUBLIC_KEY_BASENAMES.has(comparableName)) return false; - if (SENSITIVE_BASENAMES.has(comparableName)) return true; - if (comparableName.startsWith(ENV_PREFIX)) return true; - - for (const prefix of SENSITIVE_BASENAME_PREFIXES) { - if (comparableName === prefix) return true; - if (comparableName.length > prefix.length && comparableName.startsWith(prefix)) { - const suffix = comparableName.slice(prefix.length); - const next = suffix[0]; - if (next === '-' || next === '_') return true; - if (next === '.' && SENSITIVE_DOT_VARIANT_SUFFIX_SET.has(suffix)) return true; - } - } - - for (const suffixParts of SENSITIVE_PATH_SUFFIXES) { - const suffix = suffixParts.join('/'); - const comparableSuffix = comparable(suffix); - if ( - comparablePath.endsWith(`/${comparableSuffix}`) || - comparablePath.includes(`/${comparableSuffix}/`) - ) { - return true; - } - } - - return false; -} - -export function isWithinDirectory( - candidate: string, - base: string, - pathClass: 'posix' | 'win32' = process.platform === 'win32' ? 'win32' : 'posix', -): boolean { - const nc = pathe.normalize(candidate); - const nb = pathe.normalize(base); - const comparableCandidate = pathClass === 'win32' ? nc.toLowerCase() : nc; - const comparableBase = pathClass === 'win32' ? nb.toLowerCase() : nb; - if (comparableCandidate === comparableBase) return true; - const prefix = comparableBase.endsWith('/') ? comparableBase : comparableBase + '/'; - return comparableCandidate.startsWith(prefix); -} diff --git a/packages/agent-core-v2/src/_base/media/fileType.ts b/packages/agent-core-v2/src/_base/media/fileType.ts deleted file mode 100644 index 21e626c43cc..00000000000 --- a/packages/agent-core-v2/src/_base/media/fileType.ts +++ /dev/null @@ -1,368 +0,0 @@ -import { - AUDIO_MIME_BY_SUFFIX, - IMAGE_MIME_BY_SUFFIX, - VIDEO_MIME_BY_SUFFIX, -} from './mimeTypes'; - -export { AUDIO_MIME_BY_SUFFIX, IMAGE_MIME_BY_SUFFIX, VIDEO_MIME_BY_SUFFIX }; - -export const MEDIA_SNIFF_BYTES = 512; - -export interface FileType { - readonly kind: 'text' | 'image' | 'video' | 'unknown'; - readonly mimeType: string; -} - -export type DetectFileTypeMode = 'text' | 'media'; - -const TEXT_MIME_BY_SUFFIX: Readonly> = Object.freeze({ - '.svg': 'image/svg+xml', -}); - -export const NON_TEXT_SUFFIXES: ReadonlySet = new Set([ - '.icns', - '.psd', - '.ai', - '.eps', - '.pdf', - '.doc', - '.docx', - '.dot', - '.dotx', - '.rtf', - '.odt', - '.xls', - '.xlsx', - '.xlsm', - '.xlt', - '.xltx', - '.xltm', - '.ods', - '.ppt', - '.pptx', - '.pptm', - '.pps', - '.ppsx', - '.odp', - '.pages', - '.numbers', - '.key', - '.zip', - '.rar', - '.7z', - '.tar', - '.gz', - '.tgz', - '.bz2', - '.xz', - '.zst', - '.lz', - '.lz4', - '.br', - '.cab', - '.ar', - '.deb', - '.rpm', - '.mp3', - '.wav', - '.flac', - '.ogg', - '.oga', - '.opus', - '.aac', - '.m4a', - '.wma', - '.ttf', - '.otf', - '.woff', - '.woff2', - '.exe', - '.dll', - '.so', - '.dylib', - '.bin', - '.apk', - '.ipa', - '.jar', - '.class', - '.pyc', - '.pyo', - '.wasm', - '.dmg', - '.iso', - '.img', - '.sqlite', - '.sqlite3', - '.db', - '.db3', -]); - -const ASF_HEADER = Buffer.from([ - 0x30, 0x26, 0xb2, 0x75, 0x8e, 0x66, 0xcf, 0x11, 0xa6, 0xd9, 0x00, 0xaa, 0x00, 0x62, 0xce, 0x6c, -]); - -const FTYP_IMAGE_BRANDS: Readonly> = Object.freeze({ - avif: 'image/avif', - avis: 'image/avif', - heic: 'image/heic', - heif: 'image/heif', - heix: 'image/heif', - hevc: 'image/heic', - mif1: 'image/heif', - msf1: 'image/heif', -}); - -const FTYP_VIDEO_BRANDS: Readonly> = Object.freeze({ - isom: 'video/mp4', - iso2: 'video/mp4', - iso5: 'video/mp4', - mp41: 'video/mp4', - mp42: 'video/mp4', - avc1: 'video/mp4', - mp4v: 'video/mp4', - m4v: 'video/x-m4v', - qt: 'video/quicktime', - '3gp4': 'video/3gpp', - '3gp5': 'video/3gpp', - '3gp6': 'video/3gpp', - '3gp7': 'video/3gpp', - '3g2': 'video/3gpp2', -}); - -function toBuffer(data: Buffer | Uint8Array): Buffer { - return Buffer.isBuffer(data) ? data : Buffer.from(data.buffer, data.byteOffset, data.byteLength); -} - -function startsWith(buf: Buffer, prefix: Buffer | readonly number[]): boolean { - const needle = Buffer.isBuffer(prefix) ? prefix : Buffer.from(prefix); - if (buf.length < needle.length) return false; - for (let i = 0; i < needle.length; i += 1) { - if (buf[i] !== needle[i]) return false; - } - return true; -} - -function sniffFtypBrand(header: Buffer): string | null { - if (header.length < 12) return null; - if (header.subarray(4, 8).toString('latin1') !== 'ftyp') return null; - const raw = header.subarray(8, 12).toString('latin1').toLowerCase(); - return raw.replaceAll(/[\s\u0000]+$/g, '').trim(); -} - -export function sniffMediaFromMagic(data: Buffer | Uint8Array): FileType | null { - const buf = toBuffer(data); - const header = buf.length > MEDIA_SNIFF_BYTES ? buf.subarray(0, MEDIA_SNIFF_BYTES) : buf; - - if (startsWith(header, [0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a])) { - return { kind: 'image', mimeType: 'image/png' }; - } - if (startsWith(header, [0xff, 0xd8, 0xff])) { - return { kind: 'image', mimeType: 'image/jpeg' }; - } - if (startsWith(header, Buffer.from('GIF87a')) || startsWith(header, Buffer.from('GIF89a'))) { - return { kind: 'image', mimeType: 'image/gif' }; - } - if (startsWith(header, Buffer.from('BM'))) { - return { kind: 'image', mimeType: 'image/bmp' }; - } - if ( - startsWith(header, [0x49, 0x49, 0x2a, 0x00]) || - startsWith(header, [0x4d, 0x4d, 0x00, 0x2a]) - ) { - return { kind: 'image', mimeType: 'image/tiff' }; - } - if (startsWith(header, [0x00, 0x00, 0x01, 0x00])) { - return { kind: 'image', mimeType: 'image/x-icon' }; - } - if (startsWith(header, Buffer.from('RIFF')) && header.length >= 12) { - const chunk = header.subarray(8, 12).toString('latin1'); - if (chunk === 'WEBP') return { kind: 'image', mimeType: 'image/webp' }; - if (chunk === 'AVI ') return { kind: 'video', mimeType: 'video/x-msvideo' }; - } - if (startsWith(header, Buffer.from('FLV'))) { - return { kind: 'video', mimeType: 'video/x-flv' }; - } - if (startsWith(header, ASF_HEADER)) { - return { kind: 'video', mimeType: 'video/x-ms-wmv' }; - } - if (startsWith(header, [0x1a, 0x45, 0xdf, 0xa3])) { - const lowered = header.toString('latin1').toLowerCase(); - if (lowered.includes('webm')) return { kind: 'video', mimeType: 'video/webm' }; - if (lowered.includes('matroska')) return { kind: 'video', mimeType: 'video/x-matroska' }; - } - const brand = sniffFtypBrand(header); - if (brand !== null && brand !== '') { - if (brand in FTYP_IMAGE_BRANDS) { - return { kind: 'image', mimeType: FTYP_IMAGE_BRANDS[brand]! }; - } - if (brand in FTYP_VIDEO_BRANDS) { - return { kind: 'video', mimeType: FTYP_VIDEO_BRANDS[brand]! }; - } - } - return null; -} - -export interface ImageDimensions { - readonly width: number; - readonly height: number; - readonly transposed?: boolean; -} - -export function sniffImageDimensions(data: Buffer | Uint8Array): ImageDimensions | null { - const buf = toBuffer(data); - - if (startsWith(buf, [0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]) && buf.length >= 24) { - return { width: buf.readUInt32BE(16), height: buf.readUInt32BE(20) }; - } - - if ( - (startsWith(buf, Buffer.from('GIF87a')) || startsWith(buf, Buffer.from('GIF89a'))) && - buf.length >= 10 - ) { - return { width: buf.readUInt16LE(6), height: buf.readUInt16LE(8) }; - } - - if (startsWith(buf, Buffer.from('BM')) && buf.length >= 26) { - return { width: buf.readInt32LE(18), height: Math.abs(buf.readInt32LE(22)) }; - } - - if (startsWith(buf, Buffer.from('RIFF')) && buf.length >= 30) { - const fourCc = buf.subarray(12, 16).toString('latin1'); - if (fourCc === 'VP8 ') { - return { - width: buf.readUInt16LE(26) & 0x3fff, - height: buf.readUInt16LE(28) & 0x3fff, - }; - } - if (fourCc === 'VP8L' && buf.length >= 25) { - const bits = buf.readUInt32LE(21); - return { - width: (bits & 0x3fff) + 1, - height: ((bits >> 14) & 0x3fff) + 1, - }; - } - if (fourCc === 'VP8X') { - const width = 1 + (buf[24]! | (buf[25]! << 8) | (buf[26]! << 16)); - const height = 1 + (buf[27]! | (buf[28]! << 8) | (buf[29]! << 16)); - return { width, height }; - } - } - - if (startsWith(buf, [0xff, 0xd8])) { - let orientation: number | null = null; - let offset = 2; - while (offset + 9 < buf.length) { - if (buf[offset] !== 0xff) { - offset += 1; - continue; - } - const marker = buf[offset + 1]!; - if ( - marker >= 0xc0 && - marker <= 0xcf && - marker !== 0xc4 && - marker !== 0xc8 && - marker !== 0xcc - ) { - const height = buf.readUInt16BE(offset + 5); - const width = buf.readUInt16BE(offset + 7); - return orientation !== null && orientation >= 5 - ? { width: height, height: width, transposed: true } - : { width, height }; - } - if (marker === 0xd8 || marker === 0xd9 || (marker >= 0xd0 && marker <= 0xd7)) { - offset += 2; - continue; - } - const segmentLength = buf.readUInt16BE(offset + 2); - if (segmentLength < 2) break; - if (marker === 0xe1 && orientation === null) { - orientation = readExifOrientation(buf, offset + 4, offset + 2 + segmentLength); - } - offset += 2 + segmentLength; - } - } - - return null; -} - -function readExifOrientation(buf: Buffer, start: number, end: number): number | null { - const boundedEnd = Math.min(end, buf.length); - if (start + 6 > boundedEnd || buf.toString('latin1', start, start + 6) !== 'Exif\0\0') { - return null; - } - const tiff = start + 6; - if (tiff + 8 > boundedEnd) return null; - const byteOrder = buf.toString('latin1', tiff, tiff + 2); - const le = byteOrder === 'II'; - if (!le && byteOrder !== 'MM') return null; - const u16 = (offset: number): number => (le ? buf.readUInt16LE(offset) : buf.readUInt16BE(offset)); - const u32 = (offset: number): number => (le ? buf.readUInt32LE(offset) : buf.readUInt32BE(offset)); - if (u16(tiff + 2) !== 42) return null; - const ifd = tiff + u32(tiff + 4); - if (ifd + 2 > boundedEnd) return null; - const entryCount = u16(ifd); - for (let i = 0; i < entryCount; i += 1) { - const entry = ifd + 2 + i * 12; - if (entry + 12 > boundedEnd) return null; - if (u16(entry) === 0x0112) { - const value = u16(entry + 8); - return value >= 1 && value <= 8 ? value : null; - } - } - return null; -} - -function getSuffix(path: string): string { - const idx = path.lastIndexOf('.'); - if (idx === -1) return ''; - const lastSep = Math.max(path.lastIndexOf('/'), path.lastIndexOf('\\')); - if (idx <= lastSep + 1) return ''; - return path.slice(idx).toLowerCase(); -} - -export function detectFileType( - path: string, - header?: Buffer | Uint8Array, - type: DetectFileTypeMode = 'text', -): FileType { - const suffix = getSuffix(path); - let mediaHint: FileType | null = null; - if (suffix in TEXT_MIME_BY_SUFFIX) { - mediaHint = { kind: 'text', mimeType: TEXT_MIME_BY_SUFFIX[suffix]! }; - } else if (suffix in IMAGE_MIME_BY_SUFFIX) { - mediaHint = { kind: 'image', mimeType: IMAGE_MIME_BY_SUFFIX[suffix]! }; - } else if (suffix in VIDEO_MIME_BY_SUFFIX) { - mediaHint = { kind: 'video', mimeType: VIDEO_MIME_BY_SUFFIX[suffix]! }; - } - - if (header !== undefined) { - const buf = toBuffer(header); - const sniffed = sniffMediaFromMagic(buf); - if (sniffed) { - if (type === 'media') return sniffed; - if (mediaHint) { - if (sniffed.kind !== mediaHint.kind) { - return { kind: 'unknown', mimeType: '' }; - } - return mediaHint; - } - return sniffed; - } - if (mediaHint?.kind === 'image') { - return { kind: 'unknown', mimeType: '' }; - } - if (type === 'media' && mediaHint?.kind === 'video') { - return mediaHint; - } - if (buf.includes(0x00)) { - return { kind: 'unknown', mimeType: '' }; - } - } - - if (mediaHint) return mediaHint; - if (NON_TEXT_SUFFIXES.has(suffix)) { - return { kind: 'unknown', mimeType: '' }; - } - return { kind: 'text', mimeType: 'text/plain' }; -} diff --git a/packages/agent-core-v2/src/_base/media/mimeTypes.ts b/packages/agent-core-v2/src/_base/media/mimeTypes.ts deleted file mode 100644 index d1bb267008e..00000000000 --- a/packages/agent-core-v2/src/_base/media/mimeTypes.ts +++ /dev/null @@ -1,44 +0,0 @@ -export const IMAGE_MIME_BY_SUFFIX: Readonly> = Object.freeze({ - '.png': 'image/png', - '.jpg': 'image/jpeg', - '.jpeg': 'image/jpeg', - '.gif': 'image/gif', - '.bmp': 'image/bmp', - '.tif': 'image/tiff', - '.tiff': 'image/tiff', - '.webp': 'image/webp', - '.ico': 'image/x-icon', - '.heic': 'image/heic', - '.heif': 'image/heif', - '.avif': 'image/avif', - '.svgz': 'image/svg+xml', -}); - -export const VIDEO_MIME_BY_SUFFIX: Readonly> = Object.freeze({ - '.mp4': 'video/mp4', - '.mpg': 'video/mpeg', - '.mpeg': 'video/mpeg', - '.mkv': 'video/x-matroska', - '.avi': 'video/x-msvideo', - '.mov': 'video/quicktime', - '.ogv': 'video/ogg', - '.wmv': 'video/x-ms-wmv', - '.webm': 'video/webm', - '.m4v': 'video/x-m4v', - '.flv': 'video/x-flv', - '.3gp': 'video/3gpp', - '.3g2': 'video/3gpp2', -}); - -export const AUDIO_MIME_BY_SUFFIX: Readonly> = Object.freeze({ - '.mp3': 'audio/mpeg', - '.wav': 'audio/wav', - '.m4a': 'audio/mp4', - '.ogg': 'audio/ogg', - '.oga': 'audio/ogg', - '.flac': 'audio/flac', - '.aac': 'audio/aac', - '.opus': 'audio/opus', - '.weba': 'audio/webm', - '.wma': 'audio/x-ms-wma', -}); diff --git a/packages/agent-core-v2/src/_base/text/outputAccumulator.ts b/packages/agent-core-v2/src/_base/text/outputAccumulator.ts deleted file mode 100644 index 16406631ba3..00000000000 --- a/packages/agent-core-v2/src/_base/text/outputAccumulator.ts +++ /dev/null @@ -1,94 +0,0 @@ -import { sliceOnCodePointBoundary } from '#/_base/text/code-points'; -import { - DEFAULT_TOOL_RESULT_MAX_CHARS, - DEFAULT_TOOL_RESULT_MAX_RETAINED_CHARS, - type ToolResultSpill, -} from './outputLimits'; - -export interface ToolOutputAccumulatorResult { - readonly output: string; - readonly isError: boolean; - readonly spill?: ToolResultSpill; - readonly brief?: string; -} - -export class ToolOutputAccumulator { - private readonly buffer: string[] = []; - private retainedChars = 0; - private totalCharsValue = 0; - private saturated = false; - - get nChars(): number { - return this.retainedChars; - } - - get totalChars(): number { - return this.totalCharsValue; - } - - write(text: string): void { - this.totalCharsValue += text.length; - if (this.saturated) return; - const remainingRetention = DEFAULT_TOOL_RESULT_MAX_RETAINED_CHARS - this.retainedChars; - const kept = - text.length <= remainingRetention - ? text - : sliceOnCodePointBoundary(text, remainingRetention); - if (kept.length !== text.length) this.saturated = true; - this.buffer.push(kept); - this.retainedChars += kept.length; - } - - ok(message = '', options: { readonly brief?: string } = {}): ToolOutputAccumulatorResult { - let finalMessage = message; - if (finalMessage.length > 0 && !finalMessage.endsWith('.')) { - finalMessage += '.'; - } - const output = this.buffer.join(''); - return { - isError: false, - output: output.length === 0 ? finalMessage : output, - brief: options.brief, - spill: this.completionSpill(finalMessage), - }; - } - - error( - message: string, - options: { readonly brief?: string } = {}, - ): ToolOutputAccumulatorResult { - const output = this.buffer.join(''); - return { - isError: true, - output: - message.length === 0 - ? output - : output.length === 0 - ? message - : output.endsWith('\n') - ? `${output}${message}` - : `${output}\n${message}`, - brief: options.brief, - spill: this.retentionSpill(message), - }; - } - - private retentionSpill(suffix?: string): ToolResultSpill | undefined { - if (this.totalCharsValue <= this.retainedChars) return undefined; - return { - totalChars: this.totalCharsValue, - suffix: suffix !== undefined && suffix.length > 0 ? suffix : undefined, - }; - } - - private completionSpill(suffix: string): ToolResultSpill | undefined { - const retentionSpill = this.retentionSpill(); - if (retentionSpill !== undefined) { - return suffix.length > 0 ? { ...retentionSpill, suffix } : retentionSpill; - } - if (suffix.length === 0 || this.totalCharsValue <= DEFAULT_TOOL_RESULT_MAX_CHARS) { - return undefined; - } - return { suffix }; - } -} diff --git a/packages/agent-core-v2/src/_base/text/outputLimits.ts b/packages/agent-core-v2/src/_base/text/outputLimits.ts deleted file mode 100644 index 430e1f951b3..00000000000 --- a/packages/agent-core-v2/src/_base/text/outputLimits.ts +++ /dev/null @@ -1,9 +0,0 @@ -export const DEFAULT_TOOL_RESULT_MAX_CHARS = 50_000; - -export const DEFAULT_TOOL_RESULT_MAX_RETAINED_CHARS = 10_000_000; - -export interface ToolResultSpill { - readonly outputPath?: string; - readonly totalChars?: number; - readonly suffix?: string; -} diff --git a/packages/agent-core-v2/src/_base/text/resultStatus.ts b/packages/agent-core-v2/src/_base/text/resultStatus.ts deleted file mode 100644 index 6a1bdc51a22..00000000000 --- a/packages/agent-core-v2/src/_base/text/resultStatus.ts +++ /dev/null @@ -1,12 +0,0 @@ -export const TOOL_ERROR_STATUS = 'ERROR: Tool execution failed.'; -export const TOOL_EMPTY_STATUS = 'Tool output is empty.'; -export const TOOL_EMPTY_ERROR_STATUS = - 'ERROR: Tool execution failed. Tool output is empty.'; -export const TOOL_OUTPUT_EMPTY_TEXT = 'Tool output is empty.'; - -export function renderTextStatus(output: string, isError?: boolean): string { - if (isError === true) { - return output.length === 0 ? TOOL_EMPTY_ERROR_STATUS : TOOL_ERROR_STATUS + '\n' + output; - } - return output.trim().length === 0 || output.trim() === TOOL_OUTPUT_EMPTY_TEXT ? TOOL_EMPTY_STATUS : output; -} diff --git a/packages/agent-core-v2/src/agent/agentsMdReminder/agentsMdReminderService.ts b/packages/agent-core-v2/src/agent/agentsMdReminder/agentsMdReminderService.ts index d98862569f4..24512eb4150 100644 --- a/packages/agent-core-v2/src/agent/agentsMdReminder/agentsMdReminderService.ts +++ b/packages/agent-core-v2/src/agent/agentsMdReminder/agentsMdReminderService.ts @@ -9,7 +9,7 @@ import { IBashParserService } from '#/app/bashParser/bashParser'; import { IBootstrapService } from '#/app/bootstrap/bootstrap'; import type { AgentsMdReminderShownEvent } from '#/app/telemetry/events'; import { ITelemetryService } from '#/app/telemetry/telemetry'; -import { discoverInstructions } from '#/os/operations/instructions'; +import type { IHostFileSystem } from '#/os/interface/hostFileSystem'; import type { WatchChange } from '#human/utils/watch'; import { IAgentEnvironmentService } from '#/agent/environmentBinding/agentEnvironment'; import { acquireEnvironmentLease } from '#/agent/permissionPolicy/policies/environment-lease'; @@ -19,6 +19,10 @@ import { ISessionInstructionsProvider } from '#/session/sessionInstructions/inst import { normalizeUserPath } from '#/tool/path-access'; import { AGENTS_MD_PLAIN_NAMES, + agentsMdCandidatePaths, + dirsRootToLeaf, + findAgentsMdInDir, + findProjectRoot, extractAgentsMdPathsFromSystemPrompt, loadAgentsMdDetailed, } from '#/agent/profile/context'; @@ -186,9 +190,13 @@ export class AgentAgentsMdReminderService const { dirs, selfKnown } = this.targetDirs(ctx); const selfKnownSet = new Set(selfKnown); const discovered: string[] = []; - for (const path of await this.probeDirs(dirs)) { - if (this.known.has(path) || this.remindQueue.has(path) || selfKnownSet.has(path)) continue; - discovered.push(path); + for (const dir of dirs) { + for (const path of await this.probeDir(dir)) { + if (this.known.has(path) || this.remindQueue.has(path) || selfKnownSet.has(path)) { + continue; + } + discovered.push(path); + } } for (const path of selfKnown) { this.remindQueue.delete(path); @@ -302,20 +310,43 @@ export class AgentAgentsMdReminderService return { dirs: [...new Set(dirs)], selfKnown: [...new Set(selfKnown)] }; } - private async probeDirs(dirs: readonly string[]): Promise { - if (dirs.length === 0) return []; + private async probeDir(dir: string): Promise { const lease = acquireEnvironmentLease(this.environment, ['fs']); if (lease === undefined) return []; try { const fs = lease.environment.fs!; - return await (fs.operations === undefined - ? discoverInstructions(fs, dirs, [...this.known]) - : fs.operations.discoverInstructions(dirs, [...this.known])); + const anchor = await this.nearestExistingDir(fs, dir); + if (anchor === undefined) return []; + const deps = { fs }; + const projectRoot = await findProjectRoot(deps, anchor); + const chain = dirsRootToLeaf(anchor, projectRoot); + const found: string[] = []; + for (const chainDir of chain) { + const candidates = agentsMdCandidatePaths(chainDir); + if (candidates.every((candidate) => this.known.has(normalize(candidate)))) continue; + for (const path of await findAgentsMdInDir(deps, chainDir)) { + found.push(normalize(path)); + } + } + return found; } finally { lease.dispose(); } } + private async nearestExistingDir(fs: IHostFileSystem, path: string): Promise { + let current = path; + for (;;) { + const stat = await fs.stat(current).catch(() => undefined); + if (stat?.isDirectory === true) return current; + const parent = dirname(current); + if (parent === current) { + this.log.debug('agentsMdReminder probe found no existing anchor directory', { path }); + return undefined; + } + current = parent; + } + } } function hostPath(path: string, pathClass: 'posix' | 'win32'): string { diff --git a/packages/agent-core-v2/src/agent/contextMemory/toolResultRender.ts b/packages/agent-core-v2/src/agent/contextMemory/toolResultRender.ts index f936f9ea417..2bf13ed32a0 100644 --- a/packages/agent-core-v2/src/agent/contextMemory/toolResultRender.ts +++ b/packages/agent-core-v2/src/agent/contextMemory/toolResultRender.ts @@ -1,8 +1,13 @@ -import { renderTextStatus, TOOL_EMPTY_STATUS, TOOL_EMPTY_ERROR_STATUS, TOOL_ERROR_STATUS } from '#/_base/text/resultStatus'; import type { ContentPart } from '#human/llm/message'; import { isMcpToolName } from '#/tool/toolContract'; +const TOOL_ERROR_STATUS = 'ERROR: Tool execution failed.'; +const TOOL_EMPTY_STATUS = 'Tool output is empty.'; +const TOOL_EMPTY_ERROR_STATUS = + 'ERROR: Tool execution failed. Tool output is empty.'; +const TOOL_OUTPUT_EMPTY_TEXT = 'Tool output is empty.'; + export interface RenderableToolResult { readonly output: string | readonly ContentPart[]; readonly note?: string; @@ -47,7 +52,11 @@ function renderStatus(result: RenderableToolResult): ContentPart[] { const output = result.output; const single = typeof output === 'string' ? output : singleTextPart(output); if (single !== undefined) { - return [textPart(renderTextStatus(single, result.isError))]; + if (result.isError === true) { + if (single.length === 0) return [textPart(TOOL_EMPTY_ERROR_STATUS)]; + return [textPart(TOOL_ERROR_STATUS + '\n' + single)]; + } + return isEmptyOutputText(single) ? [textPart(TOOL_EMPTY_STATUS)] : [textPart(single)]; } const parts = output as readonly ContentPart[]; @@ -67,6 +76,10 @@ function textPart(text: string): ContentPart { return { type: 'text', text }; } +function isEmptyOutputText(output: string): boolean { + return output.trim().length === 0 || output.trim() === TOOL_OUTPUT_EMPTY_TEXT; +} + function isEmptyEquivalentContentArray(output: readonly ContentPart[]): boolean { return output.every((part) => part.type === 'text' && part.text.trim().length === 0); } diff --git a/packages/agent-core-v2/src/agent/media/file-type.ts b/packages/agent-core-v2/src/agent/media/file-type.ts index 459e0d5db11..d729c68231c 100644 --- a/packages/agent-core-v2/src/agent/media/file-type.ts +++ b/packages/agent-core-v2/src/agent/media/file-type.ts @@ -1 +1,368 @@ -export * from '#/_base/media/fileType'; +import { + AUDIO_MIME_BY_SUFFIX, + IMAGE_MIME_BY_SUFFIX, + VIDEO_MIME_BY_SUFFIX, +} from './mediaRef'; + +export { AUDIO_MIME_BY_SUFFIX, IMAGE_MIME_BY_SUFFIX, VIDEO_MIME_BY_SUFFIX }; + +export const MEDIA_SNIFF_BYTES = 512; + +export interface FileType { + readonly kind: 'text' | 'image' | 'video' | 'unknown'; + readonly mimeType: string; +} + +export type DetectFileTypeMode = 'text' | 'media'; + +const TEXT_MIME_BY_SUFFIX: Readonly> = Object.freeze({ + '.svg': 'image/svg+xml', +}); + +export const NON_TEXT_SUFFIXES: ReadonlySet = new Set([ + '.icns', + '.psd', + '.ai', + '.eps', + '.pdf', + '.doc', + '.docx', + '.dot', + '.dotx', + '.rtf', + '.odt', + '.xls', + '.xlsx', + '.xlsm', + '.xlt', + '.xltx', + '.xltm', + '.ods', + '.ppt', + '.pptx', + '.pptm', + '.pps', + '.ppsx', + '.odp', + '.pages', + '.numbers', + '.key', + '.zip', + '.rar', + '.7z', + '.tar', + '.gz', + '.tgz', + '.bz2', + '.xz', + '.zst', + '.lz', + '.lz4', + '.br', + '.cab', + '.ar', + '.deb', + '.rpm', + '.mp3', + '.wav', + '.flac', + '.ogg', + '.oga', + '.opus', + '.aac', + '.m4a', + '.wma', + '.ttf', + '.otf', + '.woff', + '.woff2', + '.exe', + '.dll', + '.so', + '.dylib', + '.bin', + '.apk', + '.ipa', + '.jar', + '.class', + '.pyc', + '.pyo', + '.wasm', + '.dmg', + '.iso', + '.img', + '.sqlite', + '.sqlite3', + '.db', + '.db3', +]); + +const ASF_HEADER = Buffer.from([ + 0x30, 0x26, 0xb2, 0x75, 0x8e, 0x66, 0xcf, 0x11, 0xa6, 0xd9, 0x00, 0xaa, 0x00, 0x62, 0xce, 0x6c, +]); + +const FTYP_IMAGE_BRANDS: Readonly> = Object.freeze({ + avif: 'image/avif', + avis: 'image/avif', + heic: 'image/heic', + heif: 'image/heif', + heix: 'image/heif', + hevc: 'image/heic', + mif1: 'image/heif', + msf1: 'image/heif', +}); + +const FTYP_VIDEO_BRANDS: Readonly> = Object.freeze({ + isom: 'video/mp4', + iso2: 'video/mp4', + iso5: 'video/mp4', + mp41: 'video/mp4', + mp42: 'video/mp4', + avc1: 'video/mp4', + mp4v: 'video/mp4', + m4v: 'video/x-m4v', + qt: 'video/quicktime', + '3gp4': 'video/3gpp', + '3gp5': 'video/3gpp', + '3gp6': 'video/3gpp', + '3gp7': 'video/3gpp', + '3g2': 'video/3gpp2', +}); + +function toBuffer(data: Buffer | Uint8Array): Buffer { + return Buffer.isBuffer(data) ? data : Buffer.from(data.buffer, data.byteOffset, data.byteLength); +} + +function startsWith(buf: Buffer, prefix: Buffer | readonly number[]): boolean { + const needle = Buffer.isBuffer(prefix) ? prefix : Buffer.from(prefix); + if (buf.length < needle.length) return false; + for (let i = 0; i < needle.length; i += 1) { + if (buf[i] !== needle[i]) return false; + } + return true; +} + +function sniffFtypBrand(header: Buffer): string | null { + if (header.length < 12) return null; + if (header.subarray(4, 8).toString('latin1') !== 'ftyp') return null; + const raw = header.subarray(8, 12).toString('latin1').toLowerCase(); + return raw.replaceAll(/[\s\u0000]+$/g, '').trim(); +} + +export function sniffMediaFromMagic(data: Buffer | Uint8Array): FileType | null { + const buf = toBuffer(data); + const header = buf.length > MEDIA_SNIFF_BYTES ? buf.subarray(0, MEDIA_SNIFF_BYTES) : buf; + + if (startsWith(header, [0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a])) { + return { kind: 'image', mimeType: 'image/png' }; + } + if (startsWith(header, [0xff, 0xd8, 0xff])) { + return { kind: 'image', mimeType: 'image/jpeg' }; + } + if (startsWith(header, Buffer.from('GIF87a')) || startsWith(header, Buffer.from('GIF89a'))) { + return { kind: 'image', mimeType: 'image/gif' }; + } + if (startsWith(header, Buffer.from('BM'))) { + return { kind: 'image', mimeType: 'image/bmp' }; + } + if ( + startsWith(header, [0x49, 0x49, 0x2a, 0x00]) || + startsWith(header, [0x4d, 0x4d, 0x00, 0x2a]) + ) { + return { kind: 'image', mimeType: 'image/tiff' }; + } + if (startsWith(header, [0x00, 0x00, 0x01, 0x00])) { + return { kind: 'image', mimeType: 'image/x-icon' }; + } + if (startsWith(header, Buffer.from('RIFF')) && header.length >= 12) { + const chunk = header.subarray(8, 12).toString('latin1'); + if (chunk === 'WEBP') return { kind: 'image', mimeType: 'image/webp' }; + if (chunk === 'AVI ') return { kind: 'video', mimeType: 'video/x-msvideo' }; + } + if (startsWith(header, Buffer.from('FLV'))) { + return { kind: 'video', mimeType: 'video/x-flv' }; + } + if (startsWith(header, ASF_HEADER)) { + return { kind: 'video', mimeType: 'video/x-ms-wmv' }; + } + if (startsWith(header, [0x1a, 0x45, 0xdf, 0xa3])) { + const lowered = header.toString('latin1').toLowerCase(); + if (lowered.includes('webm')) return { kind: 'video', mimeType: 'video/webm' }; + if (lowered.includes('matroska')) return { kind: 'video', mimeType: 'video/x-matroska' }; + } + const brand = sniffFtypBrand(header); + if (brand !== null && brand !== '') { + if (brand in FTYP_IMAGE_BRANDS) { + return { kind: 'image', mimeType: FTYP_IMAGE_BRANDS[brand]! }; + } + if (brand in FTYP_VIDEO_BRANDS) { + return { kind: 'video', mimeType: FTYP_VIDEO_BRANDS[brand]! }; + } + } + return null; +} + +export interface ImageDimensions { + readonly width: number; + readonly height: number; + readonly transposed?: boolean; +} + +export function sniffImageDimensions(data: Buffer | Uint8Array): ImageDimensions | null { + const buf = toBuffer(data); + + if (startsWith(buf, [0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]) && buf.length >= 24) { + return { width: buf.readUInt32BE(16), height: buf.readUInt32BE(20) }; + } + + if ( + (startsWith(buf, Buffer.from('GIF87a')) || startsWith(buf, Buffer.from('GIF89a'))) && + buf.length >= 10 + ) { + return { width: buf.readUInt16LE(6), height: buf.readUInt16LE(8) }; + } + + if (startsWith(buf, Buffer.from('BM')) && buf.length >= 26) { + return { width: buf.readInt32LE(18), height: Math.abs(buf.readInt32LE(22)) }; + } + + if (startsWith(buf, Buffer.from('RIFF')) && buf.length >= 30) { + const fourCc = buf.subarray(12, 16).toString('latin1'); + if (fourCc === 'VP8 ') { + return { + width: buf.readUInt16LE(26) & 0x3fff, + height: buf.readUInt16LE(28) & 0x3fff, + }; + } + if (fourCc === 'VP8L' && buf.length >= 25) { + const bits = buf.readUInt32LE(21); + return { + width: (bits & 0x3fff) + 1, + height: ((bits >> 14) & 0x3fff) + 1, + }; + } + if (fourCc === 'VP8X') { + const width = 1 + (buf[24]! | (buf[25]! << 8) | (buf[26]! << 16)); + const height = 1 + (buf[27]! | (buf[28]! << 8) | (buf[29]! << 16)); + return { width, height }; + } + } + + if (startsWith(buf, [0xff, 0xd8])) { + let orientation: number | null = null; + let offset = 2; + while (offset + 9 < buf.length) { + if (buf[offset] !== 0xff) { + offset += 1; + continue; + } + const marker = buf[offset + 1]!; + if ( + marker >= 0xc0 && + marker <= 0xcf && + marker !== 0xc4 && + marker !== 0xc8 && + marker !== 0xcc + ) { + const height = buf.readUInt16BE(offset + 5); + const width = buf.readUInt16BE(offset + 7); + return orientation !== null && orientation >= 5 + ? { width: height, height: width, transposed: true } + : { width, height }; + } + if (marker === 0xd8 || marker === 0xd9 || (marker >= 0xd0 && marker <= 0xd7)) { + offset += 2; + continue; + } + const segmentLength = buf.readUInt16BE(offset + 2); + if (segmentLength < 2) break; + if (marker === 0xe1 && orientation === null) { + orientation = readExifOrientation(buf, offset + 4, offset + 2 + segmentLength); + } + offset += 2 + segmentLength; + } + } + + return null; +} + +function readExifOrientation(buf: Buffer, start: number, end: number): number | null { + const boundedEnd = Math.min(end, buf.length); + if (start + 6 > boundedEnd || buf.toString('latin1', start, start + 6) !== 'Exif\0\0') { + return null; + } + const tiff = start + 6; + if (tiff + 8 > boundedEnd) return null; + const byteOrder = buf.toString('latin1', tiff, tiff + 2); + const le = byteOrder === 'II'; + if (!le && byteOrder !== 'MM') return null; + const u16 = (offset: number): number => (le ? buf.readUInt16LE(offset) : buf.readUInt16BE(offset)); + const u32 = (offset: number): number => (le ? buf.readUInt32LE(offset) : buf.readUInt32BE(offset)); + if (u16(tiff + 2) !== 42) return null; + const ifd = tiff + u32(tiff + 4); + if (ifd + 2 > boundedEnd) return null; + const entryCount = u16(ifd); + for (let i = 0; i < entryCount; i += 1) { + const entry = ifd + 2 + i * 12; + if (entry + 12 > boundedEnd) return null; + if (u16(entry) === 0x0112) { + const value = u16(entry + 8); + return value >= 1 && value <= 8 ? value : null; + } + } + return null; +} + +function getSuffix(path: string): string { + const idx = path.lastIndexOf('.'); + if (idx === -1) return ''; + const lastSep = Math.max(path.lastIndexOf('/'), path.lastIndexOf('\\')); + if (idx <= lastSep + 1) return ''; + return path.slice(idx).toLowerCase(); +} + +export function detectFileType( + path: string, + header?: Buffer | Uint8Array, + type: DetectFileTypeMode = 'text', +): FileType { + const suffix = getSuffix(path); + let mediaHint: FileType | null = null; + if (suffix in TEXT_MIME_BY_SUFFIX) { + mediaHint = { kind: 'text', mimeType: TEXT_MIME_BY_SUFFIX[suffix]! }; + } else if (suffix in IMAGE_MIME_BY_SUFFIX) { + mediaHint = { kind: 'image', mimeType: IMAGE_MIME_BY_SUFFIX[suffix]! }; + } else if (suffix in VIDEO_MIME_BY_SUFFIX) { + mediaHint = { kind: 'video', mimeType: VIDEO_MIME_BY_SUFFIX[suffix]! }; + } + + if (header !== undefined) { + const buf = toBuffer(header); + const sniffed = sniffMediaFromMagic(buf); + if (sniffed) { + if (type === 'media') return sniffed; + if (mediaHint) { + if (sniffed.kind !== mediaHint.kind) { + return { kind: 'unknown', mimeType: '' }; + } + return mediaHint; + } + return sniffed; + } + if (mediaHint?.kind === 'image') { + return { kind: 'unknown', mimeType: '' }; + } + if (type === 'media' && mediaHint?.kind === 'video') { + return mediaHint; + } + if (buf.includes(0x00)) { + return { kind: 'unknown', mimeType: '' }; + } + } + + if (mediaHint) return mediaHint; + if (NON_TEXT_SUFFIXES.has(suffix)) { + return { kind: 'unknown', mimeType: '' }; + } + return { kind: 'text', mimeType: 'text/plain' }; +} diff --git a/packages/agent-core-v2/src/agent/media/mediaRef.ts b/packages/agent-core-v2/src/agent/media/mediaRef.ts index 3f2bc71f3b4..d1ea3e2fd02 100644 --- a/packages/agent-core-v2/src/agent/media/mediaRef.ts +++ b/packages/agent-core-v2/src/agent/media/mediaRef.ts @@ -1,12 +1,54 @@ -import { AUDIO_MIME_BY_SUFFIX, IMAGE_MIME_BY_SUFFIX, VIDEO_MIME_BY_SUFFIX } from '#/_base/media/mimeTypes'; import { join } from 'node:path'; import type { ContentPart } from '#human/llm/message'; -export { AUDIO_MIME_BY_SUFFIX, IMAGE_MIME_BY_SUFFIX, VIDEO_MIME_BY_SUFFIX } from '#/_base/media/mimeTypes'; - export type MediaKind = 'image' | 'video' | 'audio' | 'file'; +export const IMAGE_MIME_BY_SUFFIX: Readonly> = Object.freeze({ + '.png': 'image/png', + '.jpg': 'image/jpeg', + '.jpeg': 'image/jpeg', + '.gif': 'image/gif', + '.bmp': 'image/bmp', + '.tif': 'image/tiff', + '.tiff': 'image/tiff', + '.webp': 'image/webp', + '.ico': 'image/x-icon', + '.heic': 'image/heic', + '.heif': 'image/heif', + '.avif': 'image/avif', + '.svgz': 'image/svg+xml', +}); + +export const VIDEO_MIME_BY_SUFFIX: Readonly> = Object.freeze({ + '.mp4': 'video/mp4', + '.mpg': 'video/mpeg', + '.mpeg': 'video/mpeg', + '.mkv': 'video/x-matroska', + '.avi': 'video/x-msvideo', + '.mov': 'video/quicktime', + '.ogv': 'video/ogg', + '.wmv': 'video/x-ms-wmv', + '.webm': 'video/webm', + '.m4v': 'video/x-m4v', + '.flv': 'video/x-flv', + '.3gp': 'video/3gpp', + '.3g2': 'video/3gpp2', +}); + +export const AUDIO_MIME_BY_SUFFIX: Readonly> = Object.freeze({ + '.mp3': 'audio/mpeg', + '.wav': 'audio/wav', + '.m4a': 'audio/mp4', + '.ogg': 'audio/ogg', + '.oga': 'audio/ogg', + '.flac': 'audio/flac', + '.aac': 'audio/aac', + '.opus': 'audio/opus', + '.weba': 'audio/webm', + '.wma': 'audio/x-ms-wma', +}); + const IMAGE_EXT_BY_MIME = invertMimeBySuffix(IMAGE_MIME_BY_SUFFIX); const VIDEO_EXT_BY_MIME = invertMimeBySuffix(VIDEO_MIME_BY_SUFFIX); const AUDIO_EXT_BY_MIME = invertMimeBySuffix(AUDIO_MIME_BY_SUFFIX); diff --git a/packages/agent-core-v2/src/agent/profile/context.ts b/packages/agent-core-v2/src/agent/profile/context.ts index 5fb57a63efb..2075e71ec37 100644 --- a/packages/agent-core-v2/src/agent/profile/context.ts +++ b/packages/agent-core-v2/src/agent/profile/context.ts @@ -1,13 +1,10 @@ -import { AGENTS_MD_PLAIN_NAMES, dotKimiAgentsMdPath, dirsRootToLeaf } from '#/os/operations/instructions'; -import { basename, join, normalize } from 'pathe'; +import { basename, dirname, join, normalize } from 'pathe'; import { findGitWorkTree } from '#/app/git/workTree'; import type { IHostFileSystem } from '#/os/interface/hostFileSystem'; import type { SystemPromptContext } from './profile'; -export { AGENTS_MD_PLAIN_NAMES, dotKimiAgentsMdPath, agentsMdCandidatePaths, dirsRootToLeaf, findAgentsMdInDir } from '#/os/operations/instructions'; - export const AGENTS_MD_RECOMMENDED_MAX_BYTES = 32 * 1024; export const LIST_DIR_ROOT_WIDTH = 30; @@ -103,6 +100,16 @@ export function splitAgentsMdFiles( return files; } +export const AGENTS_MD_PLAIN_NAMES = ['AGENTS.md', 'agents.md'] as const; + +export function dotKimiAgentsMdPath(dir: string): string { + return join(dir, '.kimi-code', 'AGENTS.md'); +} + +export function agentsMdCandidatePaths(dir: string): string[] { + return [dotKimiAgentsMdPath(dir), ...AGENTS_MD_PLAIN_NAMES.map((name) => join(dir, name))]; +} + export function extractAgentsMdPathsFromSystemPrompt(systemPrompt: string): string[] { const paths: string[] = []; const seen = new Set(); @@ -122,6 +129,35 @@ export function extractAgentsMdPathsFromSystemPrompt(systemPrompt: string): stri return paths; } +export async function findAgentsMdInDir( + deps: { readonly fs: IHostFileSystem }, + dir: string, +): Promise { + const found: string[] = []; + const dotKimi = dotKimiAgentsMdPath(dir); + if (await isNonEmptyFile(deps, dotKimi)) found.push(dotKimi); + for (const fileName of AGENTS_MD_PLAIN_NAMES) { + const candidate = join(dir, fileName); + if (await isNonEmptyFile(deps, candidate)) { + found.push(candidate); + break; + } + } + return found; +} + +async function isNonEmptyFile( + deps: { readonly fs: IHostFileSystem }, + path: string, +): Promise { + try { + const content = await deps.fs.readText(path, { errors: 'ignore' }); + return content.trim().length > 0; + } catch { + return false; + } +} + export async function loadAgentsMdForRoots( deps: ProfileContextDeps, brandHome: string | undefined, @@ -241,6 +277,21 @@ export async function findProjectRoot( return (await findGitWorkTree(deps.fs, rootWorkDir))?.root ?? rootWorkDir; } +export function dirsRootToLeaf(workDir: string, projectRoot: string): string[] { + const dirs: string[] = []; + let current = normalize(workDir); + + while (true) { + dirs.push(current); + if (current === projectRoot) break; + const parent = dirname(current); + if (parent === current) break; + current = parent; + } + + return dirs.toReversed(); +} + interface AgentFile { readonly path: string; readonly content: string; diff --git a/packages/agent-core-v2/src/agent/tools/fileReadSource.ts b/packages/agent-core-v2/src/agent/tools/fileReadSource.ts index dad55321224..0f3c9ceb6eb 100644 --- a/packages/agent-core-v2/src/agent/tools/fileReadSource.ts +++ b/packages/agent-core-v2/src/agent/tools/fileReadSource.ts @@ -1,11 +1,17 @@ -import type { FileReadSource } from '#/os/interface/fileOperations'; import { readUtf8Lines } from '#/_base/execEnv/decodeText'; -import type { IHostFileSystem } from '#/os/interface/hostFileSystem'; +import type { HostFileStat, HostWholeFile, IHostFileSystem } from '#/os/interface/hostFileSystem'; import { parseDaemonFileUrl } from '#/agent/media/mediaRef'; import type { ISessionMediaStore } from '#/agent/media/sessionMediaStore'; import type { ExecutableToolResult } from '#/tool/toolContract'; -export type { FileReadSource } from '#/os/interface/fileOperations'; +export interface FileReadSource { + readonly name: string; + readonly localPath?: string; + stat(): Promise; + readWholeFile?(): Promise; + readBytes(n?: number): Promise; + readLines(): AsyncIterable; +} export function withAttachmentLocation(result: ExecutableToolResult, source: FileReadSource): ExecutableToolResult { if (!result.isError || source.localPath === undefined || typeof result.output !== 'string') return result; @@ -15,8 +21,8 @@ export function withAttachmentLocation(result: ExecutableToolResult, source: Fil export function environmentFileSource(fs: IHostFileSystem, path: string): FileReadSource { return { name: path, - readTextPage: fs.operations === undefined ? undefined : (options, signal) => fs.operations!.readTextPage(path, options, signal), stat: () => fs.stat(path), + readWholeFile: fs.readWholeFile === undefined ? undefined : () => fs.readWholeFile!(path), readBytes: (n) => fs.readBytes(path, n), readLines: () => fs.readLines(path, { errors: 'strict' }), }; diff --git a/packages/agent-core-v2/src/agent/tools/os/glob/globTool.ts b/packages/agent-core-v2/src/agent/tools/os/glob/globTool.ts index 2ed6b4c00a5..d34c9e7017d 100644 --- a/packages/agent-core-v2/src/agent/tools/os/glob/globTool.ts +++ b/packages/agent-core-v2/src/agent/tools/os/glob/globTool.ts @@ -1,8 +1,13 @@ -import { searchGlob } from '#/os/operations/searchGlob'; +import { normalize, resolve } from 'pathe'; import { ensureRgPath, rgUnavailableMessage, type RgProbe } from '#/os/backends/node-local/tools/rgLocator'; import type { Environment } from '#/environment/environment'; - +import { + DEFAULT_TIMEOUT_MS, + MAX_OUTPUT_BYTES, + runRgOnce, + shouldRetryRipgrepEagain, +} from '#/os/backends/node-local/tools/runRg'; import type { IHostEnvironment } from '#/os/interface/hostEnvironment'; import type { IHostFileSystem } from '#/os/interface/hostFileSystem'; import { isHostFsNotFound } from '#/os/interface/hostFsErrors'; @@ -14,13 +19,18 @@ import { ISessionWorkspaceContext } from '#/session/workspaceContext/workspaceCo import { ITelemetryService } from '#/app/telemetry/telemetry'; import { IOAuthService } from '#/app/auth/auth'; import { + DEFAULT_TOOL_RESULT_MAX_RETAINED_CHARS, ToolAccesses, type ExecutableToolResult, type ToolExecution, } from '#/tool/toolContract'; import { registerAgentToolService } from '#/agent/toolRegistry/toolContribution'; import { + isWithinDirectory, resolvePathAccessPath, + type PathClass, + isSensitiveFile, + SENSITIVE_DOT_VARIANT_SUFFIXES, type WorkspaceConfig, } from '#/tool/path-access'; import { toInputJsonSchema } from '#/tool/input-schema'; @@ -30,9 +40,26 @@ import { type GlobInput, GlobInputSchema, IGlobTool, + DEFAULT_HEAD_LIMIT, WINDOWS_PATH_HINT, } from './glob'; +const VCS_DIRECTORIES_TO_EXCLUDE = ['.git', '.svn', '.hg', '.bzr', '.jj', '.sl'] as const; + +const SENSITIVE_KEY_BASENAMES = ['id_rsa', 'id_ed25519', 'id_ecdsa'] as const; +const SENSITIVE_GLOBS_TO_EXCLUDE: readonly string[] = [ + '**/.env', + ...SENSITIVE_KEY_BASENAMES.flatMap((name) => [ + `**/${name}`, + `**/${name}[-_]*`, + ...SENSITIVE_DOT_VARIANT_SUFFIXES.map((suffix) => `**/${name}${suffix}`), + ]), + '**/.aws/credentials', + '**/.aws/credentials/**', + '**/.gcp/credentials', + '**/.gcp/credentials/**', +]; + export class GlobTool implements IGlobTool { declare readonly _serviceBrand: undefined; readonly name = 'Glob' as const; @@ -176,16 +203,140 @@ export class GlobTool implements IGlobTool { return { isError: true, output: rgUnavailableMessage(error, environment) }; } + let run; try { - const request = { kind: 'glob' as const, args, rgPath, roots: searchRoots, workspaceDir: workspace.workspaceDir, pathClass: env.pathClass }; - return await (processService.search === undefined - ? searchGlob(processService, request, signal) - : processService.search(request, signal)); + run = await runRgOnce(processService, buildRgArgs(rgPath, args), signal, { cwd: searchRoot }); } catch (error) { return { isError: true, output: formatSpawnError(error, environment) }; } - } + if (run.kind === 'aborted') { + return { isError: true, output: 'Glob aborted' }; + } + + if (shouldRetryRipgrepEagain(run)) { + try { + run = await runRgOnce(processService, buildRgArgs(rgPath, args, true), signal, { cwd: searchRoot }); + } catch (error) { + return { isError: true, output: formatSpawnError(error, environment) }; + } + if (run.kind === 'aborted') { + return { isError: true, output: 'Glob aborted' }; + } + } + + const { exitCode, stdoutText, stderrText, bufferTruncated, timedOut } = run; + + let traversalWarning: string | undefined; + if (exitCode !== 0 && exitCode !== 1 && !timedOut) { + const rawPathsBeforeError = splitCompletePaths(stdoutText, true); + if (rawPathsBeforeError.length === 0) { + return { isError: true, output: formatGlobError(searchRoot, stderrText) }; + } + traversalWarning = formatGlobWarning(stderrText); + } + if (signal.aborted) { + return { isError: true, output: 'Glob aborted' }; + } + + const rawPaths = splitCompletePaths(stdoutText, bufferTruncated || timedOut).map((p) => + resolve(searchRoot, p), + ); + + const kept: string[] = []; + let filteredSensitive = 0; + for (const p of rawPaths) { + if (isSensitiveFile(p)) { + filteredSensitive++; + } else { + kept.push(p); + } + } + + const offset = args.offset ?? 0; + const headLimit = args.head_limit ?? DEFAULT_HEAD_LIMIT; + const limited = headLimit === 0 ? kept.slice(offset) : kept.slice(offset, offset + headLimit); + const partial = bufferTruncated || timedOut || traversalWarning !== undefined; + + const pathClass = env.pathClass; + const shouldRelativize = isWithinDirectory(searchRoot, workspace.workspaceDir, pathClass); + const candidates = limited.map((p) => + shouldRelativize ? relativizeIfUnder(p, searchRoot, pathClass) : p, + ); + const warnings: string[] = []; + if (timedOut) { + warnings.push( + `Glob timed out after ${String(DEFAULT_TIMEOUT_MS / 1000)}s; partial results returned.`, + ); + } + if (bufferTruncated) { + warnings.push( + `[stdout truncated at ${String(MAX_OUTPUT_BYTES)} bytes; results may be incomplete — use a more specific pattern]`, + ); + } + if (traversalWarning !== undefined) { + warnings.push(traversalWarning); + } + const pageNotices = (count: number, characterLimited: boolean) => { + const lines = [...warnings]; + const footer: string[] = []; + const truncated = characterLimited || offset + count < kept.length; + if (count === 0) { + if (kept.length > 0) { + const resultSet = partial ? 'collected partial result set' : 'current result set'; + lines.push( + `No more matches at offset=${String(offset)} in the ${resultSet} (${String(kept.length)} matches).`, + ); + } else if (partial) { + lines.push('No matches collected; search incomplete.'); + } else if (filteredSensitive > 0) { + lines.push( + `No non-sensitive matches found (${String(filteredSensitive)} sensitive file(s) filtered).`, + ); + } else { + lines.push('No matches found'); + } + } else if (truncated || offset > 0 || partial) { + const total = partial + ? `${String(kept.length)} collected matches (partial result set)` + : String(kept.length); + lines.push(`Showing matches ${String(offset + 1)}–${String(offset + count)} of ${total}.`); + } + if (characterLimited) lines.push('Character limit reached; only complete paths are returned.'); + if (truncated) { + lines.push( + `Continue with the same search arguments and offset=${String(offset + count)}.`, + ); + if (!characterLimited) lines.push('To remove the match-count limit, omit offset and use head_limit=0.'); + } + if (filteredSensitive > 0 && (kept.length > 0 || partial)) { + footer.push(`Filtered ${String(filteredSensitive)} sensitive file(s).`); + } + if (!truncated && !partial && offset === 0 && headLimit > 0 && count === headLimit) { + footer.push(`Found ${String(count)} matches`); + } + return { lines, footer }; + }; + const noticeChars = Math.max(...[false, true].map((characterLimited) => { + const { lines, footer } = pageNotices(candidates.length, characterLimited); + return [...lines, ...footer].join('\n').length + 2; + })); + let remaining = DEFAULT_TOOL_RESULT_MAX_RETAINED_CHARS - noticeChars; + const displayLines: string[] = []; + for (const path of candidates) { + if (path.length + 1 > remaining) break; + displayLines.push(path); + remaining -= path.length + 1; + } + if (candidates.length > 0 && displayLines.length === 0) { + return { + isError: true, + output: 'Glob cannot fit a complete path and its diagnostics within the output limit. Narrow the search path or pattern.', + }; + } + const notices = pageNotices(displayLines.length, displayLines.length < candidates.length); + return { output: [...notices.lines, ...displayLines, ...notices.footer].join('\n') }; + } } registerAgentToolService(IGlobTool, GlobTool, { @@ -216,6 +367,37 @@ function createRgProbe(processService: IHostProcessService, cwd: string): RgProb }; } +function buildRgArgs(rgPath: string, args: GlobInput, singleThreaded = false): string[] { + const cmd: string[] = [rgPath]; + if (singleThreaded) cmd.push('-j', '1'); + cmd.push('--files', '--hidden', '--sortr=modified'); + for (const dir of VCS_DIRECTORIES_TO_EXCLUDE) { + cmd.push('--glob', `!${dir}`); + } + cmd.push('--glob', args.pattern); + for (const glob of SENSITIVE_GLOBS_TO_EXCLUDE) { + cmd.push('--glob', `!${glob}`); + } + if (args.include_ignored) cmd.push('--no-ignore'); + cmd.push('.'); + return cmd; +} + +function formatGlobError(searchRoot: string, stderr: string): string { + const trimmed = stderr.trim(); + if (/no such file or directory/i.test(trimmed)) { + return `${searchRoot} does not exist`; + } + return trimmed.length > 0 ? `Glob failed: ${trimmed}` : 'Glob failed'; +} + +function formatGlobWarning(stderr: string): string { + const trimmed = stderr.trim(); + return trimmed.length > 0 + ? `Glob completed with warnings; some directories could not be read: ${trimmed}` + : 'Glob completed with warnings; some directories could not be read.'; +} + function formatSpawnError(error: unknown, environment: Environment): string { return isHostFsNotFound(error) ? rgUnavailableMessage(error, environment) @@ -224,4 +406,24 @@ function formatSpawnError(error: unknown, environment: Environment): string { : String(error); } -export { splitCompletePaths } from '#/os/operations/searchGlob'; +export function splitCompletePaths(stdoutText: string, truncatedOutput: boolean): string[] { + let text = stdoutText; + if (truncatedOutput && !text.endsWith('\n')) { + const lastNewline = text.lastIndexOf('\n'); + text = lastNewline >= 0 ? text.slice(0, lastNewline + 1) : ''; + } + return text.split('\n').filter((p) => p.length > 0); +} + +function relativizeIfUnder(candidate: string, base: string, pathClass: PathClass): string { + const normCandidate = normalize(candidate); + const normBase = normalize(base); + const comparableCandidate = pathClass === 'win32' ? normCandidate.toLowerCase() : normCandidate; + const comparableBase = pathClass === 'win32' ? normBase.toLowerCase() : normBase; + if (comparableCandidate === comparableBase) return '.'; + const prefix = comparableBase.endsWith('/') ? comparableBase : comparableBase + '/'; + if (comparableCandidate.startsWith(prefix)) { + return normCandidate.slice(prefix.length); + } + return normCandidate; +} diff --git a/packages/agent-core-v2/src/agent/tools/os/grep/grepTool.ts b/packages/agent-core-v2/src/agent/tools/os/grep/grepTool.ts index b3f4f7a0841..d02dae75765 100644 --- a/packages/agent-core-v2/src/agent/tools/os/grep/grepTool.ts +++ b/packages/agent-core-v2/src/agent/tools/os/grep/grepTool.ts @@ -1,5 +1,6 @@ -import { searchGrep } from '#/os/operations/searchGrep'; +import { normalize } from 'pathe'; +import { ToolOutputAccumulator } from '#/tool/output-accumulator'; import { ToolAccesses, type ExecutableToolResult, @@ -19,6 +20,9 @@ import { ISessionSkillCatalog } from '#/features/skill/session/skillCatalog'; import { ISessionWorkspaceContext } from '#/session/workspaceContext/workspaceContext'; import { resolvePathAccessPath, + type PathClass, + isSensitiveFile, + SENSITIVE_DOT_VARIANT_SUFFIXES, type WorkspaceConfig, } from '#/tool/path-access'; import { toInputJsonSchema } from '#/tool/input-schema'; @@ -28,10 +32,37 @@ import { rgUnavailableMessage, type RgProbe, } from '#/os/backends/node-local/tools/rgLocator'; - +import { + DEFAULT_TIMEOUT_MS, + MAX_OUTPUT_BYTES, + runRgOnce, + shouldRetryRipgrepEagain, + type RunRgResult, +} from '#/os/backends/node-local/tools/runRg'; import GREP_DESCRIPTION from './grep.md?raw'; import { type GrepInput, GrepInputSchema, IGrepTool } from './grep'; +const RG_MAX_COLUMNS = 500; +const DEFAULT_HEAD_LIMIT = 250; + +const VCS_DIRECTORIES_TO_EXCLUDE = ['.git', '.svn', '.hg', '.bzr', '.jj', '.sl'] as const; +const SENSITIVE_KEY_BASENAMES = ['id_rsa', 'id_ed25519', 'id_ecdsa'] as const; +const SENSITIVE_KEY_GLOBS_TO_EXCLUDE = SENSITIVE_KEY_BASENAMES.flatMap((name) => [ + `**/${name}`, + `**/${name}[-_]*`, + ...SENSITIVE_DOT_VARIANT_SUFFIXES.map((suffix) => `**/${name}${suffix}`), +]); +const SENSITIVE_GLOBS_TO_EXCLUDE = [ + '**/.env', + ...SENSITIVE_KEY_GLOBS_TO_EXCLUDE, + '**/.aws/credentials', + '**/.aws/credentials/**', + '**/.gcp/credentials', + '**/.gcp/credentials/**', +] as const; + +const CONTENT_LINE_RE = /^(.*?)([:-])(\d+)\2/; + export class GrepTool implements IGrepTool { declare readonly _serviceBrand: undefined; readonly name = 'Grep' as const; @@ -111,6 +142,7 @@ export class GrepTool implements IGrepTool { return { isError: true, output: 'Aborted before search started' }; } + const pathClass = env.pathClass; let rgPath: string; try { const resolution = await ensureRgPath(this.createRgProbe(processService, workspace.workspaceDir), { @@ -134,15 +166,135 @@ export class GrepTool implements IGrepTool { return { isError: true, output: rgUnavailableMessage(error, environment) }; } + let runResult: RunRgResult; try { - const request = { kind: 'grep' as const, args, rgPath, roots: searchPaths, workspaceDir: workspace.workspaceDir, pathClass: env.pathClass }; - return await (processService.search === undefined - ? searchGrep(processService, request, signal) - : processService.search(request, signal)); + const firstRun = await runRgOnce( + processService, + buildRgArgs(rgPath, args, searchPaths), + signal, + ); + if (firstRun.kind === 'aborted') { + return { isError: true, output: 'Grep aborted' }; + } + runResult = firstRun; + + if (shouldRetryRipgrepEagain(runResult)) { + const retryRun = await runRgOnce( + processService, + buildRgArgs(rgPath, args, searchPaths, true), + signal, + ); + if (retryRun.kind === 'aborted') { + return { isError: true, output: 'Grep aborted' }; + } + runResult = retryRun; + } } catch (error) { return { isError: true, output: formatSpawnError(error, environment) }; } + + const { exitCode, stderrText, bufferTruncated, stderrTruncated, timedOut } = runResult; + let { stdoutText } = runResult; + + if (exitCode !== 0 && exitCode !== 1 && !timedOut) { + return { + isError: true, + output: formatRipgrepError(exitCode, stderrText, stderrTruncated), + }; + } + + const mode = args.output_mode ?? 'files_with_matches'; + if (bufferTruncated || timedOut) { + stdoutText = omitIncompleteTrailingRecord(stdoutText, mode); + } + if (timedOut && stdoutText.trim() === '') { + return { + isError: true, + output: `Grep timed out after ${String(DEFAULT_TIMEOUT_MS / 1000)}s. Try a more specific path or pattern.`, + }; + } + if (signal.aborted) { + return { isError: true, output: 'Grep aborted' }; + } + + const rawLines = parseRipgrepOutput(stdoutText, mode); + + const filteredSensitive = new Set(); + const keptLines = filterSensitiveLines(rawLines, mode, filteredSensitive, pathClass); + const orderedLines = keptLines; + + const offset = args.offset ?? 0; + const headLimit = args.head_limit ?? DEFAULT_HEAD_LIMIT; + const afterOffset = offset > 0 ? orderedLines.slice(offset) : orderedLines; + const limitActive = headLimit > 0; + const limited = limitActive ? afterOffset.slice(0, headLimit) : afterOffset; + const paginationTruncated = limitActive && afterOffset.length > headLimit; + + const headerLines: string[] = []; + const messages: string[] = []; + if (filteredSensitive.size > 0) { + const displayedFilteredPaths = [...filteredSensitive].map((path) => + relativizeIfUnder(path, workspace.workspaceDir, pathClass), + ); + messages.push( + `Filtered ${String(filteredSensitive.size)} sensitive file(s): ${displayedFilteredPaths.join(', ')}`, + ); + } + if (mode === 'count_matches' && orderedLines.length > 0) { + headerLines.push(formatCountSummary(orderedLines, filteredSensitive.size > 0)); + } + if (paginationTruncated) { + const total = afterOffset.length + offset; + const nextOffset = offset + headLimit; + const paginationNotice = + bufferTruncated || timedOut + ? `Results truncated to ${String(headLimit)} lines (total: ${String(total)} of a partial result set). Use offset=${String(nextOffset)} to see more.` + : `Results truncated to ${String(headLimit)} lines (total: ${String(total)}). Use offset=${String(nextOffset)} to see more.`; + if (mode === 'count_matches') { + headerLines.push(paginationNotice); + } else { + messages.push(paginationNotice); + } + } + if (bufferTruncated) { + messages.push( + `[Output truncated at ${String(MAX_OUTPUT_BYTES)} bytes of rg output — the result set is incomplete. Narrow the pattern, path, or glob filters and re-run to recover complete results.]`, + ); + } + if (timedOut) { + messages.push( + `Grep timed out after ${String(DEFAULT_TIMEOUT_MS / 1000)}s; partial results returned. Narrow the path, glob, or pattern and retry for complete results.`, + ); + } + + const contentIncludesLineNumbers = mode === 'content' && args['-n'] !== false; + const displayedLines = limited.map((line) => + formatDisplayLine( + line, + mode, + workspace.workspaceDir, + pathClass, + contentIncludesLineNumbers, + ), + ); + const contentBody = displayedLines.join('\n'); + const visibleBody = + orderedLines.length === 0 && filteredSensitive.size > 0 + ? 'No non-sensitive matches found' + : contentBody; + const emptyResultMessage = + SENSITIVE_GLOBS_TO_EXCLUDE.length > 0 ? 'No non-sensitive matches found' : 'No matches found'; + const body = + visibleBody === '' && headerLines.length === 0 && messages.length === 0 + ? emptyResultMessage + : visibleBody; + const combined = [...headerLines, body, ...messages].filter((part) => part !== '').join('\n'); + + const builder = new ToolOutputAccumulator(); + builder.write(combined); + return builder.ok(); } + private createRgProbe(processService: IHostProcessService, cwd: string): RgProbe { return { exec: async (args) => { @@ -180,3 +332,347 @@ function formatSpawnError(error: unknown, environment: Environment): string { ? error.message : String(error); } + +type GrepMode = 'content' | 'files_with_matches' | 'count_matches'; + +type ParsedGrepLine = + | { + readonly kind: 'record'; + readonly filePath: string; + readonly payload: string; + } + | { + readonly kind: 'separator'; + } + | { + readonly kind: 'legacy'; + readonly text: string; + }; + +function buildRgArgs( + rgPath: string, + args: GrepInput, + searchPaths: readonly string[], + singleThreaded = false, +): string[] { + const cmd: string[] = [rgPath]; + if (singleThreaded) cmd.push('-j', '1'); + cmd.push('--hidden'); + const mode = args.output_mode ?? 'files_with_matches'; + if (mode !== 'content') { + cmd.push('--max-columns', String(RG_MAX_COLUMNS)); + } + cmd.push('--null'); + for (const dir of VCS_DIRECTORIES_TO_EXCLUDE) { + cmd.push('--glob', `!${dir}`); + } + + if (mode === 'files_with_matches') cmd.push('-l', '--sortr=modified'); + else if (mode === 'count_matches') { + cmd.push('--count-matches', '--with-filename'); + } + + if (args['-i']) cmd.push('-i'); + if (mode === 'content') { + cmd.push('--with-filename'); + if (args['-n'] !== false) { + cmd.push('-n'); + } else { + cmd.push('--field-context-separator', ':'); + } + if (args['-C'] !== undefined) { + cmd.push('-C', String(args['-C'])); + } else { + if (args['-A'] !== undefined) cmd.push('-A', String(args['-A'])); + if (args['-B'] !== undefined) cmd.push('-B', String(args['-B'])); + } + } + if (args.glob !== undefined) cmd.push('--glob', args.glob); + if (args.type !== undefined) cmd.push('--type', args.type); + if (args.multiline) cmd.push('-U', '--multiline-dotall'); + if (args.include_ignored) cmd.push('--no-ignore'); + for (const glob of SENSITIVE_GLOBS_TO_EXCLUDE) { + cmd.push('--glob', `!${glob}`); + } + + cmd.push('--', args.pattern, ...searchPaths); + return cmd; +} + +function splitRgLines(text: string): string[] { + if (text === '') return []; + const lines = text.split('\n'); + while (lines.length > 0 && lines.at(-1) === '') { + lines.pop(); + } + return lines.map((line) => stripTrailingCarriageReturn(line)); +} + +function parseRipgrepOutput(text: string, mode: GrepMode): ParsedGrepLine[] { + if (text === '') return []; + if (!text.includes('\0')) { + return splitRgLines(text).map((line) => + mode === 'content' && line === '--' ? { kind: 'separator' } : { kind: 'legacy', text: line }, + ); + } + + if (mode === 'files_with_matches') { + return text + .split('\0') + .map((filePath) => stripTrailingCarriageReturn(filePath)) + .filter((filePath) => filePath !== '') + .map((filePath) => ({ kind: 'record', filePath, payload: '' })); + } + + const records: ParsedGrepLine[] = []; + let cursor = 0; + while (cursor < text.length) { + if (text[cursor] === '\n') { + cursor += 1; + continue; + } + if (text.startsWith('--\r\n', cursor)) { + records.push({ kind: 'separator' }); + cursor += 4; + continue; + } + if (text.startsWith('--\n', cursor)) { + records.push({ kind: 'separator' }); + cursor += 3; + continue; + } + + const nulIndex = text.indexOf('\0', cursor); + if (nulIndex < 0) { + const tail = stripTrailingCarriageReturn(text.slice(cursor)); + if (tail !== '') records.push({ kind: 'legacy', text: tail }); + break; + } + + const lineEnd = text.indexOf('\n', nulIndex + 1); + const payloadEnd = lineEnd >= 0 ? lineEnd : text.length; + const filePath = text.slice(cursor, nulIndex); + const payload = stripTrailingCarriageReturn(text.slice(nulIndex + 1, payloadEnd)); + records.push({ kind: 'record', filePath, payload }); + cursor = lineEnd >= 0 ? lineEnd + 1 : text.length; + } + return records; +} + +function formatDisplayLine( + line: ParsedGrepLine, + mode: GrepMode, + workspaceDir: string, + pathClass: PathClass, + contentIncludesLineNumbers: boolean, +): string { + if (line.kind === 'separator') return '--'; + if (line.kind === 'record') { + const displayPath = relativizeIfUnder(line.filePath, workspaceDir, pathClass); + if (mode === 'files_with_matches') return displayPath; + if (mode === 'count_matches') return `${displayPath}:${line.payload}`; + const separator = contentIncludesLineNumbers ? contentPayloadPathSeparator(line.payload) : ':'; + return `${displayPath}${separator}${line.payload}`; + } + + const text = line.text; + if (mode === 'files_with_matches') { + return relativizeIfUnder(text, workspaceDir, pathClass); + } + if (mode === 'count_matches') { + const idx = text.lastIndexOf(':'); + if (idx <= 0) return text; + return relativizeIfUnder(text.slice(0, idx), workspaceDir, pathClass) + text.slice(idx); + } + + const filePath = extractContentFilePath(text, pathClass); + if (filePath !== undefined) { + return relativizeIfUnder(filePath, workspaceDir, pathClass) + text.slice(filePath.length); + } + return text; +} + +function relativizeIfUnder(candidate: string, base: string, pathClass: PathClass): string { + const normCandidate = normalize(candidate); + const normBase = normalize(base); + const comparableCandidate = pathClass === 'win32' ? normCandidate.toLowerCase() : normCandidate; + const comparableBase = pathClass === 'win32' ? normBase.toLowerCase() : normBase; + if (comparableCandidate === comparableBase) return '.'; + const prefix = comparableBase.endsWith('/') ? comparableBase : comparableBase + '/'; + if (comparableCandidate.startsWith(prefix)) { + return normCandidate.slice(prefix.length); + } + return normCandidate; +} + +function omitIncompleteTrailingRecord(text: string, mode: GrepMode): string { + if (!text.includes('\0')) return omitIncompleteTrailingLine(text); + if (mode === 'files_with_matches') { + const lastNul = text.lastIndexOf('\0'); + return lastNul >= 0 ? text.slice(0, lastNul + 1) : ''; + } + + let cursor = 0; + let lastCompleteEnd = 0; + while (cursor < text.length) { + if (text[cursor] === '\n') { + cursor += 1; + lastCompleteEnd = cursor; + continue; + } + if (text.startsWith('--\r\n', cursor)) { + cursor += 4; + lastCompleteEnd = cursor; + continue; + } + if (text.startsWith('--\n', cursor)) { + cursor += 3; + lastCompleteEnd = cursor; + continue; + } + + const nulIndex = text.indexOf('\0', cursor); + if (nulIndex < 0) break; + const lineEnd = text.indexOf('\n', nulIndex + 1); + if (lineEnd < 0) break; + cursor = lineEnd + 1; + lastCompleteEnd = cursor; + } + return text.slice(0, lastCompleteEnd); +} + +function omitIncompleteTrailingLine(text: string): string { + const lastNewline = text.lastIndexOf('\n'); + return lastNewline >= 0 ? text.slice(0, lastNewline) : ''; +} + +function formatRipgrepError( + exitCode: number, + stderrText: string, + stderrTruncated: boolean, +): string { + const stderr = stderrText.trim(); + if (stderr.length === 0) { + return `Failed to grep: ripgrep exited with code ${String(exitCode)}`; + } + + const summary = summarizeRipgrepStderr(stderr); + const lines = [`Failed to grep: ${summary}`, '', 'ripgrep stderr:', stderr]; + if (stderrTruncated) { + lines.push(`[stderr truncated at ${String(MAX_OUTPUT_BYTES)} bytes]`); + } + return lines.join('\n'); +} + +function summarizeRipgrepStderr(stderr: string): string { + const lines = splitRgLines(stderr) + .map((line) => line.trim()) + .filter((line) => line.length > 0); + const errorLine = lines.findLast((line) => line.toLowerCase().startsWith('error:')); + return errorLine ?? lines.at(-1) ?? 'ripgrep error'; +} + +function filterSensitiveLines( + lines: readonly ParsedGrepLine[], + mode: GrepMode, + filteredPaths: Set, + pathClass: PathClass, +): ParsedGrepLine[] { + const kept: ParsedGrepLine[] = []; + for (const line of lines) { + if (line.kind === 'separator') { + kept.push(line); + continue; + } + const filePath = parsedFilePath(line, mode, pathClass); + if (filePath !== undefined && isSensitiveFile(filePath)) { + filteredPaths.add(filePath); + continue; + } + kept.push(line); + } + return mode === 'content' ? normalizeContextSeparators(kept) : kept; +} + +function normalizeContextSeparators(lines: readonly ParsedGrepLine[]): ParsedGrepLine[] { + const normalized: ParsedGrepLine[] = []; + for (const line of lines) { + if ( + line.kind === 'separator' && + (normalized.length === 0 || normalized.at(-1)?.kind === 'separator') + ) { + continue; + } + normalized.push(line); + } + while (normalized.length > 0 && normalized.at(-1)?.kind === 'separator') { + normalized.pop(); + } + return normalized; +} + +function parsedFilePath( + line: ParsedGrepLine, + mode: GrepMode, + pathClass: PathClass, +): string | undefined { + if (line.kind === 'record') return normalize(line.filePath); + if (line.kind === 'separator') return undefined; + const text = line.text; + if (mode === 'files_with_matches') return normalize(text); + if (mode === 'count_matches') { + const idx = text.lastIndexOf(':'); + return idx > 0 ? normalize(text.slice(0, idx)) : normalize(text); + } + return extractContentFilePath(text, pathClass); +} + +function extractContentFilePath(line: string, pathClass: PathClass): string | undefined { + const m = CONTENT_LINE_RE.exec(line); + if (m?.[1] !== undefined) return normalize(m[1]); + + const separatorIndex = noLineNumberContentSeparatorIndex(line, pathClass); + return separatorIndex > 0 ? normalize(line.slice(0, separatorIndex)) : undefined; +} + +function noLineNumberContentSeparatorIndex(line: string, pathClass: PathClass): number { + const searchFrom = pathClass === 'win32' && /^[A-Za-z]:/.test(line) ? 2 : 0; + return line.indexOf(':', searchFrom); +} + +function contentPayloadPathSeparator(payload: string): ':' | '-' { + const m = /^(\d+)([:-])/.exec(payload); + return m?.[2] === '-' ? '-' : ':'; +} + +function stripTrailingCarriageReturn(value: string): string { + return value.endsWith('\r') ? value.slice(0, -1) : value; +} + +function formatCountSummary(lines: readonly ParsedGrepLine[], redactedSensitive: boolean): string { + let totalMatches = 0; + let totalFiles = 0; + for (const line of lines) { + const rawCount = + line.kind === 'record' + ? line.payload + : line.kind === 'legacy' + ? countPayloadFromLegacyLine(line.text) + : undefined; + if (rawCount === undefined) continue; + const count = Number(rawCount); + if (!Number.isSafeInteger(count) || count < 0) continue; + totalMatches += count; + totalFiles++; + } + + const occurrenceWord = totalMatches === 1 ? 'occurrence' : 'occurrences'; + const fileWord = totalFiles === 1 ? 'file' : 'files'; + const scope = redactedSensitive ? 'total non-sensitive' : 'total'; + return `Found ${String(totalMatches)} ${scope} ${occurrenceWord} across ${String(totalFiles)} ${fileWord}.`; +} + +function countPayloadFromLegacyLine(line: string): string | undefined { + const idx = line.lastIndexOf(':'); + return idx > 0 ? line.slice(idx + 1) : undefined; +} diff --git a/packages/agent-core-v2/src/agent/tools/os/read/read.ts b/packages/agent-core-v2/src/agent/tools/os/read/read.ts index 3650ebba17b..060f82571c8 100644 --- a/packages/agent-core-v2/src/agent/tools/os/read/read.ts +++ b/packages/agent-core-v2/src/agent/tools/os/read/read.ts @@ -6,7 +6,7 @@ import { type AgentTool } from '#/tool/toolContract'; export const DEFAULT_MAX_CHARS = 100_000; export const DEFAULT_MAX_CHARS_LIMIT = 500_000; -export { TRANSCODE_MAX_BYTES } from '#/os/operations/readTextPage'; +export const TRANSCODE_MAX_BYTES: number = 10 * 1024 * 1024; const PositiveLineOffsetSchema = z.number().int().min(1); const TailLineOffsetSchema = z.number().int().negative(); diff --git a/packages/agent-core-v2/src/agent/tools/os/read/readTool.ts b/packages/agent-core-v2/src/agent/tools/os/read/readTool.ts index 5a8d75258af..4d8797867fb 100644 --- a/packages/agent-core-v2/src/agent/tools/os/read/readTool.ts +++ b/packages/agent-core-v2/src/agent/tools/os/read/readTool.ts @@ -1,4 +1,3 @@ -import { readTextPage } from '#/os/operations/readTextPage'; import * as posixPath from 'node:path/posix'; import * as win32Path from 'node:path/win32'; @@ -7,6 +6,7 @@ import { ISessionContext } from '#/session/sessionContext/sessionContext'; import { IAgentScopeContext } from '#/agent/scopeContext/scopeContext'; import { isTaskOutputFilePath } from '#/agent/task/persist'; import { IHostFileSystem } from '#/os/interface/hostFileSystem'; +import { isHostFsNotDirectory, isHostFsNotFound } from '#/os/interface/hostFsErrors'; import { acquireOrWhenReady, IAgentEnvironmentService, pinnedGeneration } from '#/agent/environmentBinding/agentEnvironment'; import { ISessionMediaStore } from '#/agent/media/sessionMediaStore'; import { IAgentProfileService } from '#/agent/profile/profile'; @@ -16,9 +16,11 @@ import { IAgentToolRegistryService } from '#/agent/toolRegistry/toolRegistry'; import { isDaemonFileUrl } from '#/agent/media/mediaRef'; import { attachmentFileSource, environmentFileSource, withAttachmentLocation, type FileReadSource } from '#/agent/tools/fileReadSource'; import { EnvironmentWorkspaceView } from '#/environment/environmentWorkspaceView'; +import { unwrapErrorCause } from '#/_base/errors/errors'; import { ISessionSkillCatalog } from '#/features/skill/session/skillCatalog'; import { ISessionWorkspaceContext } from '#/session/workspaceContext/workspaceContext'; import { IConfigService } from '#/app/config/config'; +import { renderToolResultForModel } from '#/agent/contextMemory/toolResultRender'; import { ToolAccesses, type ExecutableToolResult, @@ -29,20 +31,145 @@ import { resolvePathAccessPath, type WorkspaceConfig, } from '#/tool/path-access'; +import { MEDIA_SNIFF_BYTES, detectFileType } from '#/agent/media/file-type'; import { toInputJsonSchema } from '#/tool/input-schema'; import { literalRulePattern, matchesGlobRuleSubject, matchesPathRuleSubject } from '#/tool/rule-match'; +import { makeCarriageReturnsVisible, splitLinesKeepingTerminator, type LineEndingStyle } from '#/_base/text/line-endings'; +import { detectTextEncoding, type UtfTextEncoding } from '#/_base/text/encoding'; import { renderPrompt } from '#/_base/utils/render-prompt'; import { DEFAULT_MAX_CHARS, DEFAULT_MAX_CHARS_LIMIT, IReadTool, ReadInputSchema, + TRANSCODE_MAX_BYTES, type ReadInput, } from './read'; import { IAgentToolResultTruncationService } from '#/agent/toolResultTruncation/toolResultTruncation'; import { READ_SECTION, type ReadConfig } from './configSection'; import readDescriptionTemplate from './read.md?raw'; +interface LineEndingFlags { + hasCrLf: boolean; + hasLf: boolean; + hasLoneCr: boolean; +} + +interface ReadLineEntry { + readonly lineNo: number; + readonly rawContent: string; +} + +interface ReadTailEntry extends ReadLineEntry { + readonly minChars: number; +} + +interface ReadRequest { + readonly args: ReadInput; + readonly maxChars: number; + readonly maxCharsLimit: number; + readonly detectedEncoding?: UtfTextEncoding; + readonly lossyDecoding: boolean; + readonly eventLog: boolean; +} + +interface ReadPage { + readonly request: ReadRequest; + readonly renderedLines: readonly string[]; + readonly startLine: number; + readonly rangeStart: number; + readonly rangeEnd: number; + readonly totalLines: number; + readonly fromTail: boolean; + readonly lineEndingStyle: LineEndingStyle; +} + +function stripTrailingLf(line: string): string { + return line.endsWith('\n') ? line.slice(0, -1) : line; +} + +function splitsSurrogatePair(text: string, offset: number): boolean { + // oxlint-disable-next-line unicorn/prefer-code-point -- raw UTF-16 halves are the point here + const previous = text.charCodeAt(offset - 1); + // oxlint-disable-next-line unicorn/prefer-code-point -- raw UTF-16 halves are the point here + const next = text.charCodeAt(offset); + return previous >= 0xd800 && previous <= 0xdbff && next >= 0xdc00 && next <= 0xdfff; +} + +function updateLineEndingFlags(flags: LineEndingFlags, text: string): void { + for (let i = 0; i < text.length; i += 1) { + const code = text.codePointAt(i); + if (code === 13) { + if (text.codePointAt(i + 1) === 10) { + flags.hasCrLf = true; + i += 1; + } else { + flags.hasLoneCr = true; + } + } else if (code === 10) { + flags.hasLf = true; + } + } +} + +function lineEndingStyleFromFlags(flags: LineEndingFlags): LineEndingStyle { + if (flags.hasLoneCr || (flags.hasCrLf && flags.hasLf)) return 'mixed'; + if (flags.hasCrLf) return 'crlf'; + return 'lf'; +} + +function renderLine(entry: ReadLineEntry, lineEndingStyle: LineEndingStyle): string { + const modelContent = + lineEndingStyle === 'crlf' && entry.rawContent.endsWith('\r') + ? entry.rawContent.slice(0, -1) + : entry.rawContent; + const renderedContent = + lineEndingStyle === 'mixed' ? makeCarriageReturnsVisible(modelContent) : modelContent; + return `${String(entry.lineNo)}\t${renderedContent}`; +} + +function isTextDecodeError(error: unknown): boolean { + const unwrapped = unwrapErrorCause(error); + if (typeof unwrapped !== 'object' || unwrapped === null) return false; + const code = (unwrapped as { code?: unknown })['code']; + if (code === 'ERR_ENCODING_INVALID_ENCODED_DATA') return true; + if (!(unwrapped instanceof Error)) return false; + return /encoded data was not valid|invalid.*encoding|invalid.*utf-?8/i.test(unwrapped.message); +} + +function containsNulByte(text: string): boolean { + return text.includes('\u0000'); +} + +function encodingDisplayName(encoding: UtfTextEncoding): string { + switch (encoding) { + case 'utf-16le': + return 'UTF-16 LE'; + case 'utf-16be': + return 'UTF-16 BE'; + default: + return 'UTF-8'; + } +} + +async function* decodedLines(lines: readonly string[]): AsyncGenerator { + yield* lines; +} + +const READ_MEDIA_FILE_TOOL_NAME = 'ReadMediaFile'; + +function notReadableFileOutput(path: string): string { + return `"${path}" is not readable as UTF-8 text. Only text files can be read.`; +} + +function notUtf8DecodableFileOutput(path: string): string { + return ( + `"${path}" is not valid UTF-8 or UTF-16 text. ` + + 'Only UTF-8 and UTF-16 text files can be read; ' + + 'for other encodings (e.g. GBK), convert the file to UTF-8 first (e.g. with `iconv`).' + ); +} + export class ReadTool implements IReadTool { declare readonly _serviceBrand: undefined; readonly name = 'Read' as const; @@ -119,14 +246,14 @@ export class ReadTool implements IReadTool { pathClass: env.pathClass, homeDir: env.homeDir, }), - execute: async ({ signal }) => { + execute: async () => { const lease = await acquireOrWhenReady(this.environment, ['fs']); try { if (expectedGeneration !== undefined && lease.environment.identity.generation !== expectedGeneration) { return { isError: true, output: 'Environment changed before execution. Retry the tool call.' }; } const eventLog = this.resultTruncation.isWireJournalPath(path); - const result = await this.execution(environmentFileSource(lease.environment.fs!, path), args, eventLog, signal); + const result = await this.execution(environmentFileSource(lease.environment.fs!, path), args, eventLog); return { ...result, spillExempt: true }; } finally { lease.dispose(); @@ -161,11 +288,11 @@ export class ReadTool implements IReadTool { matchesRule: (ruleArgs) => matchesPathRuleSubject(ruleArgs, path, { cwd: target.environmentPath.dirname(path), pathClass: target.pathClass, }), - execute: async ({ signal }) => { + execute: async () => { if (planFile && (await this.planService?.resolveFile(path, 'read'))?.path !== path) { return { isError: true, output: 'The active plan changed before execution. Retry the tool call.' }; } - return { ...await this.execution(environmentFileSource(target.fs, path), args, false, signal), spillExempt: true }; + return { ...await this.execution(environmentFileSource(target.fs, path), args, false), spillExempt: true }; }, }; } @@ -178,26 +305,372 @@ export class ReadTool implements IReadTool { display: { kind: 'file_io', operation: 'read', path: source.localPath ?? args.path }, approvalRule: literalRulePattern(this.name, args.path), matchesRule: (ruleArgs) => matchesGlobRuleSubject(ruleArgs, args.path), - execute: async ({ signal }) => ({ - ...withAttachmentLocation(await this.execution(source, args, false, signal), source), + execute: async () => ({ + ...withAttachmentLocation(await this.execution(source, args, false), source), spillExempt: true, }), }; } - private async execution(source: FileReadSource, args: ReadInput, eventLog: boolean, signal: AbortSignal): Promise { - const limits = this.limits(); - const capabilities = this.profile.getModelCapabilities(); - const options = { - args, - maxChars: Math.min(args.max_chars ?? limits.defaultMaxChars, limits.maxChars), - maxCharsLimit: limits.maxChars, - eventLog, - imageSupported: capabilities.image_in, - videoSupported: capabilities.video_in, - mediaToolAvailable: this.toolRegistry.resolve('ReadMediaFile') !== undefined && this.toolPolicy.isToolActive('ReadMediaFile'), + private async execution( + source: FileReadSource, + args: ReadInput, + eventLog: boolean, + ): Promise { + try { + let stat: Awaited>; + let whole: Awaited>> | undefined; + try { + if ((args.line_offset ?? 1) >= 0) whole = await source.readWholeFile?.(); + stat = whole?.stat ?? await source.stat(); + } catch (error) { + if (isHostFsNotFound(error) || isHostFsNotDirectory(error)) { + return { isError: true, output: `"${args.path}" does not exist.` }; + } + throw error; + } + if (!stat.isFile) { + return { isError: true, output: `"${args.path}" is not a file.` }; + } + + const header = whole?.bytes.subarray(0, MEDIA_SNIFF_BYTES) ?? await source.readBytes(MEDIA_SNIFF_BYTES); + const fileType = detectFileType(source.name, header); + if (fileType.kind === 'image' || fileType.kind === 'video') { + const kind = fileType.kind; + const article = kind === 'image' ? 'an' : 'a'; + const capabilities = this.profile.getModelCapabilities(); + const supported = kind === 'image' ? capabilities.image_in : capabilities.video_in; + if (!supported) { + return { + isError: true, + output: `"${args.path}" is ${article} ${kind} file. The current model does not support ${kind} input (missing ${kind}_in capability), so this agent cannot view it. Only text files can be read.`, + }; + } + const mediaToolAvailable = + this.toolRegistry.resolve(READ_MEDIA_FILE_TOOL_NAME) !== undefined && + this.toolPolicy.isToolActive(READ_MEDIA_FILE_TOOL_NAME); + return { + isError: true, + output: mediaToolAvailable + ? `"${args.path}" is ${article} ${kind} file. Only text files can be read; use ReadMediaFile for image and video files.` + : `"${args.path}" is ${article} ${kind} file. Only text files can be read.`, + }; + } + + const detection = detectTextEncoding(header); + let readLines: () => AsyncIterable; + let detectedEncoding: UtfTextEncoding | undefined; + let lossyDecoding = false; + if (!detection.seemsBinary && detection.encoding !== 'utf-8') { + if (stat.size > TRANSCODE_MAX_BYTES) { + return { + isError: true, + output: + `"${args.path}" is ${encodingDisplayName(detection.encoding)} text but too large to transcode ` + + `(${String(stat.size)} bytes > ${String(TRANSCODE_MAX_BYTES)}). ` + + 'Convert it to UTF-8 first (e.g. with `iconv`).', + }; + } + const bytes = whole?.bytes ?? await source.readBytes(); + let decoded: string; + try { + decoded = new TextDecoder(detection.encoding, { fatal: true }).decode(bytes); + } catch (error) { + if (!isTextDecodeError(error)) throw error; + decoded = new TextDecoder(detection.encoding, { fatal: false }).decode(bytes); + lossyDecoding = true; + } + detectedEncoding = detection.encoding; + const decodedContent = splitLinesKeepingTerminator(decoded); + readLines = () => decodedLines(decodedContent); + } else if (fileType.kind === 'unknown') { + return { + isError: true, + output: notReadableFileOutput(args.path), + }; + } else if (whole !== undefined) { + const decoded = new TextDecoder('utf-8', { fatal: true }).decode(whole.bytes); + const content = splitLinesKeepingTerminator(decoded); + readLines = () => decodedLines(content); + } else { + readLines = () => source.readLines(); + } + + const limits = this.limits(); + const request: ReadRequest = { + args, + maxChars: Math.min(args.max_chars ?? limits.defaultMaxChars, limits.maxChars), + maxCharsLimit: limits.maxChars, + detectedEncoding, + lossyDecoding, + eventLog, + }; + const lineOffset = args.line_offset ?? 1; + if (lineOffset >= 0) return await this.readForward(readLines(), request); + const rereadsFile = detectedEncoding === undefined && (args.n_lines ?? Infinity) < -lineOffset; + const result = await this.readTail(readLines, request); + if (!result.isError && rereadsFile) { + const currentStat = await source.stat(); + if (!currentStat.isFile || currentStat.size !== stat.size || + currentStat.mtimeMs !== stat.mtimeMs || currentStat.ino !== stat.ino) { + return { isError: true, output: 'File changed while reading its tail. Retry Read with the updated file.' }; + } + } + return result; + } catch (error) { + if (isTextDecodeError(error)) { + return { isError: true, output: notUtf8DecodableFileOutput(args.path) }; + } + return { + isError: true, + output: error instanceof Error ? error.message : String(error), + }; + } + } + + private async readForward( + lines: AsyncIterable, + request: ReadRequest, + ): Promise { + const { args, maxChars } = request; + const lineOffset = args.line_offset ?? 1; + const columnOffset = args.column_offset ?? 0; + const requestedLines = args.n_lines ?? Infinity; + const selectedEntries: ReadLineEntry[] = []; + const flags: LineEndingFlags = { hasCrLf: false, hasLf: false, hasLoneCr: false }; + let currentLineNo = 0; + let collectionClosed = false; + let minimumChars = 0; + + for await (const rawLine of lines) { + if (containsNulByte(rawLine)) { + return { isError: true, output: notReadableFileOutput(args.path) }; + } + currentLineNo += 1; + updateLineEndingFlags(flags, rawLine); + if (collectionClosed) continue; + if (currentLineNo < lineOffset) continue; + if (selectedEntries.length >= requestedLines) { + collectionClosed = true; + continue; + } + const rawContent = stripTrailingLf(rawLine); + const lineChars = String(currentLineNo).length + 1 + Math.max( + 0, + rawContent.length - (rawContent.endsWith('\r') ? 1 : 0) - + (currentLineNo === lineOffset ? columnOffset : 0), + ) + (selectedEntries.length === 0 ? 0 : 1); + if (minimumChars + lineChars > maxChars && selectedEntries.length > 0) { + collectionClosed = true; + continue; + } + selectedEntries.push({ + lineNo: currentLineNo, + rawContent, + }); + minimumChars += lineChars; + if (selectedEntries.length >= requestedLines || minimumChars >= maxChars) { + collectionClosed = true; + } + } + + const lineEndingStyle = lineEndingStyleFromFlags(flags); + const renderedLines = selectedEntries.map((entry) => renderLine(entry, lineEndingStyle)); + const firstLine = renderedLines[0]; + if (columnOffset > 0) { + const prefix = `${String(lineOffset)}\t`; + const text = firstLine?.slice(prefix.length); + if (text === undefined || columnOffset > text.length) { + return { isError: true, output: `column_offset=${String(columnOffset)} is past the end of the starting line ${String(lineOffset)}. Read the line from column 0 to inspect its current contents.` }; + } + if (splitsSurrogatePair(text, columnOffset)) { + return { isError: true, output: `column_offset=${String(columnOffset)} splits a Unicode character in line ${String(lineOffset)}. Use a character boundary or the Next Read arguments.` }; + } + renderedLines[0] = prefix + text.slice(columnOffset); + } + return this.finishPage({ + request, + renderedLines, + startLine: lineOffset, + rangeStart: lineOffset, + rangeEnd: Math.min(currentLineNo, lineOffset + requestedLines - 1), + totalLines: currentLineNo, + fromTail: false, + lineEndingStyle, + }); + } + + private finishPage(page: ReadPage): ExecutableToolResult { + const { args, maxChars, maxCharsLimit, eventLog, detectedEncoding, lossyDecoding } = page.request; + let first = 0; + let end = page.renderedLines.length; + let contentChars = page.renderedLines.reduce((sum, line) => sum + line.length + 1, -1); + const firstColumn = page.fromTail ? 0 : args.column_offset ?? 0; + const firstPrefix = `${String(page.startLine)}\t`; + const firstText = page.renderedLines[0]?.slice(firstPrefix.length) ?? ''; + let fragmentEnd: number | undefined; + + while (true) { + const count = end - first; + const startLine = page.startLine + first; + const endLine = page.startLine + end - 1; + const lineIncomplete = fragmentEnd !== undefined; + const complete = page.rangeStart > page.rangeEnd || + (count > 0 && startLine === page.rangeStart && endLine === page.rangeEnd && !lineIncomplete); + const parts = [ + count > 0 + ? `${String(count)} ${count === 1 ? 'line' : 'lines'} read from file starting from line ${String(startLine)}.` + : 'No lines read from file.', + `Total lines in file: ${String(page.totalLines)}.`, + complete ? 'Requested range complete.' : 'Character limit reached.', + `Effective max_chars: ${String(maxChars)}.`, + ]; + if (!lineIncomplete && ((count > 0 && endLine === page.totalLines) || page.rangeStart > page.totalLines)) { + parts.push('End of file reached.'); + } + if (count > 0 && !page.fromTail && (firstColumn > 0 || fragmentEnd !== undefined)) { + parts.push(`Line ${String(startLine)} fragment: columns [${String(firstColumn)}, ${String(firstColumn + (fragmentEnd ?? firstText.length))}) of ${String(firstColumn + firstText.length)}. ${lineIncomplete ? 'Line continues.' : 'Line complete.'}`); + } + if (args.max_chars !== undefined && args.max_chars > maxCharsLimit) { + parts.push(`Requested max_chars=${String(args.max_chars)} was capped at the configured maximum ${String(maxCharsLimit)}.`); + } + if (!complete && (count > 0 || page.fromTail)) { + const nextStart = page.fromTail ? page.rangeStart : lineIncomplete ? startLine : endLine + 1; + const nextEnd = page.fromTail && count > 0 ? startLine - 1 : page.rangeEnd; + const next = { + path: args.path, + line_offset: nextStart, + column_offset: fragmentEnd !== undefined ? firstColumn + fragmentEnd : undefined, + n_lines: page.fromTail || args.n_lines !== undefined ? nextEnd - nextStart + 1 : undefined, + max_chars: maxChars, + }; + parts.push(`Next Read: ${JSON.stringify(next)}`); + } + if (eventLog) { + parts.push('Kimi Code agent event log: read one record at a time (n_lines=1); increase max_chars for a longer record or extract fields with Bash.'); + } + if (page.lineEndingStyle === 'mixed') { + parts.push('Mixed or lone carriage-return line endings are shown as \\r. Use exact \\r\\n or \\r escapes in Edit.old_string for those lines.'); + } + if (detectedEncoding !== undefined) { + parts.push(`Detected file encoding: ${encodingDisplayName(detectedEncoding)}; content transcoded to UTF-8 for display. Edit and Write expect UTF-8 — convert the file's encoding first (e.g. \`iconv\` via Bash).`); + } + if (lossyDecoding) { + parts.push('Lossy UTF-16 decoding: malformed sequences were replaced with U+FFFD. The decoded text may differ from the original file.'); + } + const note = `${parts.join(' ')}`; + const renderedChars = count === 0 + ? renderToolResultForModel({ output: '', note }).reduce( + (sum, part) => sum + (part.type === 'text' ? part.text.length : 0), + 0, + ) + : contentChars + 1 + note.length; + if (renderedChars <= maxChars && (complete || count > 0)) { + return { + output: fragmentEnd !== undefined + ? firstPrefix + firstText.slice(0, fragmentEnd) + : page.renderedLines.slice(first, end).join('\n'), + note, + truncated: complete ? undefined : true, + }; + } + if (count === 1 && !page.fromTail) { + const previousEnd = fragmentEnd ?? firstText.length; + fragmentEnd = Math.min(previousEnd - 1, previousEnd - (renderedChars - maxChars)); + if (splitsSurrogatePair(firstText, fragmentEnd)) fragmentEnd -= 1; + if (fragmentEnd <= 0) { + return { isError: true, output: `max_chars=${String(maxChars)} is too small for file text and the Read status. Increase max_chars.` }; + } + contentChars = firstPrefix.length + fragmentEnd; + continue; + } + if (count === 0) { + if (!complete) { + const recovery: ExecutableToolResult = { + isError: true, + output: 'No complete line fits. Continue with the forward Next Read.', + note, + truncated: true, + }; + const recoveryChars = renderToolResultForModel(recovery).reduce( + (sum, part) => sum + (part.type === 'text' ? part.text.length : 0), + 0, + ); + if (recoveryChars <= maxChars) return recovery; + } + return { isError: true, output: `max_chars=${String(maxChars)} is too small for the Read status. Increase max_chars.` }; + } + const dropped = page.fromTail ? first++ : --end; + contentChars -= page.renderedLines[dropped]!.length + 1; + } + } + + private async readTail( + readLines: () => AsyncIterable, + request: ReadRequest, + ): Promise { + const { args, maxChars } = request; + const lineOffset = args.line_offset ?? 1; + const requestedLines = args.n_lines ?? Infinity; + const tailCount = -lineOffset; + const singlePass = requestedLines >= tailCount; + let entries: (ReadTailEntry | undefined)[] = []; + let first = 0; + let chars = 0; + const retainLine = (rawLine: string, lineNo: number): void => { + const rawContent = stripTrailingLf(rawLine); + const minChars = String(lineNo).length + 2 + rawContent.length - (rawContent.endsWith('\r') ? 1 : 0); + entries.push({ lineNo, rawContent, minChars }); + chars += minChars; + while (first < entries.length && (chars - 1 > maxChars || entries.length - first > tailCount)) { + chars -= entries[first]!.minChars; + entries[first++] = undefined; + } + if (first > 1024 && first >= entries.length / 2) { + entries = entries.slice(first); + first = 0; + } }; - return source.readTextPage === undefined ? readTextPage(source, options, signal) : source.readTextPage(options, signal); + const flags: LineEndingFlags = { hasCrLf: false, hasLf: false, hasLoneCr: false }; + let totalLines = 0; + for await (const rawLine of readLines()) { + if (containsNulByte(rawLine)) { + return { isError: true, output: notReadableFileOutput(args.path) }; + } + totalLines += 1; + updateLineEndingFlags(flags, rawLine); + if (singlePass) retainLine(rawLine, totalLines); + } + + const rangeStart = Math.max(1, totalLines + lineOffset + 1); + const rangeEnd = Math.min(totalLines, rangeStart + requestedLines - 1); + const lineEndingStyle = lineEndingStyleFromFlags(flags); + if (!singlePass) { + let currentLine = 0; + for await (const rawLine of readLines()) { + currentLine += 1; + if (currentLine > rangeEnd) break; + if (currentLine < rangeStart) continue; + if (containsNulByte(rawLine)) { + return { isError: true, output: notReadableFileOutput(args.path) }; + } + retainLine(rawLine, currentLine); + } + if (currentLine < rangeEnd || currentLine > totalLines) { + return { isError: true, output: 'File changed while reading its tail. Retry Read with the updated file.' }; + } + } + const selected = entries.slice(first).map((entry) => renderLine(entry!, lineEndingStyle)); + return this.finishPage({ + request, + renderedLines: selected, + startLine: rangeEnd - selected.length + 1, + rangeStart, + rangeEnd, + totalLines, + fromTail: true, + lineEndingStyle, + }); } } diff --git a/packages/agent-core-v2/src/agent/tools/os/write/writeTool.ts b/packages/agent-core-v2/src/agent/tools/os/write/writeTool.ts index 5b3a73bdb7d..24b0e6d0fbe 100644 --- a/packages/agent-core-v2/src/agent/tools/os/write/writeTool.ts +++ b/packages/agent-core-v2/src/agent/tools/os/write/writeTool.ts @@ -1,7 +1,7 @@ import { dirname } from 'pathe'; -import { type IHostFileSystem } from '#/os/interface/hostFileSystem'; -import { writeText } from '#/os/operations/writeText'; +import { type HostFileStat, type IHostFileSystem } from '#/os/interface/hostFileSystem'; +import { isHostFsNotFound } from '#/os/interface/hostFsErrors'; import { acquireOrWhenReady, IAgentEnvironmentService, pinnedGeneration } from '#/agent/environmentBinding/agentEnvironment'; import { IAgentPlanService } from '#/features/plan/plan'; import { EnvironmentWorkspaceView } from '#/environment/environmentWorkspaceView'; @@ -105,10 +105,60 @@ export class WriteTool implements IWriteTool { } private async execution(fs: IHostFileSystem, args: WriteInput, safePath: string, parent = dirname(safePath)): Promise { - const input = { path: safePath, displayPath: args.path, parent, content: args.content, mode: args.mode ?? 'overwrite' as const }; - return fs.operations === undefined ? writeText(fs, input) : fs.operations.writeText(input); + const mode = args.mode ?? 'overwrite'; + if (fs.writeTextWithParent === undefined) { + const parentError = await this.ensureParentDirectory(fs, parent); + if (parentError !== undefined) { + return { isError: true, output: parentError }; + } + } + + try { + if (fs.writeTextWithParent !== undefined) { + await fs.writeTextWithParent(safePath, args.content, mode, parent); + } else if (mode === 'append') { + await fs.appendText(safePath, args.content); + } else { + await fs.writeText(safePath, args.content); + } + const bytesWritten = Buffer.byteLength(args.content, 'utf8'); + return { + output: `${mode === 'append' ? 'Appended' : 'Wrote'} ${String(bytesWritten)} bytes to ${args.path}`, + }; + } catch (error) { + if (isHostFsNotFound(error)) { + return { + isError: true, + output: `Failed to write ${args.path}: parent directory does not exist.`, + }; + } + return { + isError: true, + output: error instanceof Error ? error.message : String(error), + }; + } } + private async ensureParentDirectory(fs: IHostFileSystem, parent: string): Promise { + let stat: HostFileStat; + try { + stat = await fs.stat(parent); + } catch (error) { + if (isHostFsNotFound(error)) { + try { + await fs.mkdir(parent, { recursive: true }); + return undefined; + } catch (mkdirError) { + return mkdirError instanceof Error ? mkdirError.message : String(mkdirError); + } + } + return undefined; + } + if (!stat.isDirectory) { + return `Parent path is not a directory: ${parent}.`; + } + return undefined; + } } registerAgentToolService(IWriteTool, WriteTool, { diff --git a/packages/agent-core-v2/src/app/edit/editService.ts b/packages/agent-core-v2/src/app/edit/editService.ts index 2108d09ab0c..3c261d4c9c6 100644 --- a/packages/agent-core-v2/src/app/edit/editService.ts +++ b/packages/agent-core-v2/src/app/edit/editService.ts @@ -1 +1,41 @@ -export * from '#/os/operations/editService'; +import type { TextModel } from './textModel'; + +export interface EditApplyInput { + readonly path: string; + readonly old_string: string; + readonly new_string: string; + readonly replace_all: boolean; +} + +export type EditApplyResult = + | { readonly ok: true; readonly rawContent: string; readonly count: number } + | { readonly ok: false; readonly error: string }; + +function notFoundMessage(path: string): string { + return `old_string not found in ${path}, the file contents may be out of date. Please use the Read Tool to reload the content. +`; +} + +function notUniqueMessage(path: string, count: number): string { + return ( + `old_string is not unique in ${path} (found ${String(count)} occurrences). ` + + 'To replace every occurrence, set replace_all=true. To replace only one occurrence, include more surrounding context in old_string.' + ); +} + +export class EditService { + apply(model: TextModel, input: EditApplyInput): EditApplyResult { + if (input.replace_all) { + const { text, count } = model.replaceAll(input.old_string, input.new_string); + if (count === 0) return { ok: false, error: notFoundMessage(input.path) }; + return { ok: true, rawContent: model.materialize(text), count }; + } + + const count = model.countOccurrences(input.old_string); + if (count === 0) return { ok: false, error: notFoundMessage(input.path) }; + if (count > 1) return { ok: false, error: notUniqueMessage(input.path, count) }; + + const text = model.replaceOnce(input.old_string, input.new_string); + return { ok: true, rawContent: model.materialize(text), count: 1 }; + } +} diff --git a/packages/agent-core-v2/src/app/edit/fileEdit.ts b/packages/agent-core-v2/src/app/edit/fileEdit.ts index a572bb98c0e..b2b10cc7274 100644 --- a/packages/agent-core-v2/src/app/edit/fileEdit.ts +++ b/packages/agent-core-v2/src/app/edit/fileEdit.ts @@ -1,8 +1,17 @@ -import type { TextEditInput as FileEditInput, TextEditResult as FileEditResult } from '#/os/interface/fileOperations'; import { createDecorator, type ServiceIdentifier } from '#/_base/di/instantiation'; import type { IHostFileSystem } from '#/os/interface/hostFileSystem'; -export type { TextEditInput as FileEditInput, TextEditResult as FileEditResult } from '#/os/interface/fileOperations'; +export interface FileEditInput { + readonly path: string; + readonly displayPath: string; + readonly old_string: string; + readonly new_string: string; + readonly replace_all: boolean; +} + +export type FileEditResult = + | { readonly ok: true; readonly count: number } + | { readonly ok: false; readonly error: string }; export interface IFileEditService { readonly _serviceBrand: undefined; diff --git a/packages/agent-core-v2/src/app/edit/fileEditService.ts b/packages/agent-core-v2/src/app/edit/fileEditService.ts index c8904a4a71c..79e8510248e 100644 --- a/packages/agent-core-v2/src/app/edit/fileEditService.ts +++ b/packages/agent-core-v2/src/app/edit/fileEditService.ts @@ -1,18 +1,47 @@ import { LifecycleScope } from '#/app/scopes'; import { ScopeActivation, registerScopedService } from '#/_base/di/scope'; +import { unwrapErrorCause } from '#/_base/errors/errors'; import { IHostFileSystem } from '#/os/interface/hostFileSystem'; -import { editText } from '#/os/operations/editText'; +import { EditService } from './editService'; import { type FileEditInput, type FileEditResult, IFileEditService } from './fileEdit'; +import { TextModel } from './textModel'; export class FileEditService implements IFileEditService { declare readonly _serviceBrand: undefined; - constructor(@IHostFileSystem private readonly fs: IHostFileSystem) {} + private readonly editor: EditService; + + constructor(@IHostFileSystem private readonly fs: IHostFileSystem) { + this.editor = new EditService(); + } async edit(input: FileEditInput, fs: IHostFileSystem = this.fs): Promise { - return fs.operations === undefined ? editText(fs, input) : fs.operations.editText(input); + try { + const raw = await fs.readText(input.path, { errors: 'strict' }); + const model = new TextModel(raw); + const result = this.editor.apply(model, { + path: input.displayPath, + old_string: input.old_string, + new_string: input.new_string, + replace_all: input.replace_all, + }); + if (!result.ok) { + return { ok: false, error: result.error }; + } + await fs.writeText(input.path, result.rawContent); + return { ok: true, count: result.count }; + } catch (error) { + const code = (unwrapErrorCause(error) as { code?: unknown } | null)?.code; + if (code === 'EISDIR') { + return { ok: false, error: `${input.displayPath} is not a file.` }; + } + return { + ok: false, + error: error instanceof Error ? error.message : String(error), + }; + } } } diff --git a/packages/agent-core-v2/src/app/edit/textModel.ts b/packages/agent-core-v2/src/app/edit/textModel.ts index 05211bd786a..ab6611d738f 100644 --- a/packages/agent-core-v2/src/app/edit/textModel.ts +++ b/packages/agent-core-v2/src/app/edit/textModel.ts @@ -1 +1,43 @@ -export * from '#/os/operations/textModel'; +import { + type LineEndingStyle, + materializeModelText, + toModelTextView, +} from '#/_base/text/line-endings'; + +export class TextModel { + readonly lineEndingStyle: LineEndingStyle; + readonly text: string; + + constructor(raw: string) { + const view = toModelTextView(raw); + this.text = view.text; + this.lineEndingStyle = view.lineEndingStyle; + } + + countOccurrences(needle: string): number { + let count = 0; + let pos = 0; + while (pos < this.text.length) { + const idx = this.text.indexOf(needle, pos); + if (idx === -1) break; + count += 1; + pos = idx + needle.length; + } + return count; + } + + replaceOnce(needle: string, replacement: string): string { + const index = this.text.indexOf(needle); + if (index === -1) return this.text; + return this.text.slice(0, index) + replacement + this.text.slice(index + needle.length); + } + + replaceAll(needle: string, replacement: string): { text: string; count: number } { + const parts = this.text.split(needle); + return { text: parts.join(replacement), count: parts.length - 1 }; + } + + materialize(modelText: string): string { + return materializeModelText(modelText, this.lineEndingStyle); + } +} diff --git a/packages/agent-core-v2/src/app/git/workTree.ts b/packages/agent-core-v2/src/app/git/workTree.ts index 2868783c05e..42b30154d8a 100644 --- a/packages/agent-core-v2/src/app/git/workTree.ts +++ b/packages/agent-core-v2/src/app/git/workTree.ts @@ -1,9 +1,54 @@ +import { dirname, isAbsolute, join, normalize } from 'pathe'; + import type { IHostFileSystem } from '#/os/interface/hostFileSystem'; -import type { GitWorkTree } from '#/os/interface/fileOperations'; -import { findGitWorkTree as findLocalGitWorkTree } from '#/os/operations/gitWorkTree'; -export type { GitWorkTree } from '#/os/interface/fileOperations'; +export interface GitWorkTree { + readonly root: string; + readonly dotGitPath: string; + readonly controlDirPath: string; +} + +export async function findGitWorkTree( + fs: IHostFileSystem, + cwd: string, +): Promise { + if (cwd.length === 0 || !isAbsolute(cwd)) return null; + + let current = normalize(cwd); + while (true) { + const dotGitPath = join(current, '.git'); + const controlDirPath = await probeGitControlDir(fs, dotGitPath, current); + if (controlDirPath !== null) return { root: current, dotGitPath, controlDirPath }; + + const parent = dirname(current); + if (parent === current) return null; + current = parent; + } +} + +async function probeGitControlDir( + fs: IHostFileSystem, + dotGitPath: string, + markerParent: string, +): Promise { + try { + const stat = await fs.stat(dotGitPath); + if (stat.isDirectory) return dotGitPath; + if (!stat.isFile) return null; + + const content = await fs.readText(dotGitPath); + return parseGitDirPointer(content, markerParent) ?? null; + } catch { + return null; + } +} + +function parseGitDirPointer(content: string, markerParent: string): string | undefined { + const stripped = content.codePointAt(0) === 0xfeff ? content.slice(1) : content; + const line = stripped.trimStart().split(/\r?\n/, 1)[0]?.trim(); + if (line === undefined || !line.startsWith('gitdir:')) return undefined; -export function findGitWorkTree(fs: IHostFileSystem, cwd: string): Promise { - return fs.operations === undefined ? findLocalGitWorkTree(fs, cwd) : fs.operations.findGitWorkTree(cwd); + const rawPath = line.slice('gitdir:'.length).trim(); + if (rawPath.length === 0) return undefined; + return normalize(isAbsolute(rawPath) ? rawPath : join(markerParent, rawPath)); } diff --git a/packages/agent-core-v2/src/features/fileHistory/fileHistoryService.ts b/packages/agent-core-v2/src/features/fileHistory/fileHistoryService.ts index a141275a640..d343b45b504 100644 --- a/packages/agent-core-v2/src/features/fileHistory/fileHistoryService.ts +++ b/packages/agent-core-v2/src/features/fileHistory/fileHistoryService.ts @@ -1,4 +1,3 @@ -import { snapshotFiles } from '#/os/operations/snapshotFiles'; import { createHash } from 'node:crypto'; import { Service } from '#/_base/di/service'; @@ -11,6 +10,7 @@ import type { WillExecuteToolEvent } from '#/agent/toolExecutor/toolHooks'; import { TurnStarted } from '#/agent/loop/turnEvents'; import { ISessionContext } from '#/session/sessionContext/sessionContext'; import { IHostFileSystem } from '#/os/interface/hostFileSystem'; +import { isHostFsNotFound } from '#/os/interface/hostFsErrors'; import { IAtomicDocumentStore } from '#/persistence/interface/atomicDocumentStore'; import { TurnEnded } from '#/agent/loop/turnOps'; import { IEventBus } from '#/app/event/eventBus'; @@ -49,8 +49,6 @@ import { touchFileHistorySession } from './fileHistoryRetention'; export const FILE_HISTORY_MAX_FILE_BYTES = 4 * 1024 * 1024; export { FILE_HISTORY_BLOB_PREFIX } from './fileHistory'; -type CurrentFile = Uint8Array | 'missing' | 'unreadable' | { oversizeBytes: number; mtimeMs?: number }; - export class AgentFileHistoryService extends Service implements IAgentFileHistoryService { declare readonly _serviceBrand: undefined; @@ -301,9 +299,9 @@ export class AgentFileHistoryService extends Service implements IAgentFileHistor ); if (startCheckpoint !== undefined && Object.hasOwn(startCheckpoint.entries, pathKey)) return; - const latest = latestEntry(state.checkpoints, pathKey); - const current = await this.readCurrent(pathKey, latest); + const current = await this.readCurrent(pathKey); if (current === 'unreadable') return; + const latest = latestEntry(state.checkpoints, pathKey); const nextVersion = maxVersion(state.checkpoints, pathKey) + 1; let entry: FileBackupEntry; if (current === 'missing') { @@ -358,40 +356,35 @@ export class AgentFileHistoryService extends Service implements IAgentFileHistor string, FileBackupEntry >; - const tracked = Object.entries(start.entries); - for (let offset = 0; offset < tracked.length; offset += 4) { - const batch = tracked.slice(offset, offset + 4); - const currentFiles = await this.readCurrents(batch.map(([pathKey, previous]) => ({ pathKey, previous }))); - for (const [pathKey, before] of batch) { - const nextVersion = maxVersion(state.checkpoints, pathKey) + 1; - const current = currentFiles.get(pathKey) ?? 'unreadable'; - if (current === 'unreadable') continue; - if (current === 'missing') { - if (before.key !== null || before.oversize === true) { - entries[pathKey] = { key: null, version: nextVersion }; - } - continue; + for (const [pathKey, before] of Object.entries(start.entries)) { + const nextVersion = maxVersion(state.checkpoints, pathKey) + 1; + const current = await this.readCurrent(pathKey); + if (current === 'unreadable') continue; + if (current === 'missing') { + if (before.key !== null || before.oversize === true) { + entries[pathKey] = { key: null, version: nextVersion }; } - if (!(current instanceof Uint8Array)) { - if ( - before.oversize !== true || - before.size !== current.oversizeBytes || - before.mtimeMs !== current.mtimeMs - ) { - entries[pathKey] = { - key: null, - version: nextVersion, - oversize: true, - size: current.oversizeBytes, - mtimeMs: current.mtimeMs, - }; - } - continue; + continue; + } + if (!(current instanceof Uint8Array)) { + if ( + before.oversize !== true || + before.size !== current.oversizeBytes || + before.mtimeMs !== current.mtimeMs + ) { + entries[pathKey] = { + key: null, + version: nextVersion, + oversize: true, + size: current.oversizeBytes, + mtimeMs: current.mtimeMs, + }; } - const contentHash = sha256(current); - if (before.contentHash === contentHash) continue; - entries[pathKey] = await this.backup(pathKey, nextVersion, current, contentHash); + continue; } + const contentHash = sha256(current); + if (before.contentHash === contentHash) continue; + entries[pathKey] = await this.backup(pathKey, nextVersion, current, contentHash); } const evictable = displacedCheckpoints(state.checkpoints, turnId); @@ -471,44 +464,48 @@ export class AgentFileHistoryService extends Service implements IAgentFileHistor return this.blobs.get(this.agentCtx.scope(), entry.key); } - private async readCurrent(pathKey: string, previous?: FileBackupEntry): Promise { - return (await this.readCurrents([{ pathKey, previous }])).get(pathKey) ?? 'unreadable'; - } - - private async readCurrents(files: readonly { pathKey: string; previous?: FileBackupEntry }[]): Promise> { - const results = new Map(); + private async readCurrent( + pathKey: string, + ): Promise< + Uint8Array | 'missing' | 'unreadable' | { oversizeBytes: number; mtimeMs?: number } + > { let lease: EnvironmentLease; try { lease = this.environment.acquire(['fs']); } catch (error) { - if (error instanceof EnvironmentError) return results; + if (error instanceof EnvironmentError) return 'unreadable'; throw error; } try { const fs = lease.environment.fs; + if (fs === undefined) return 'unreadable'; const path = lease.environment.path; - if (fs === undefined || path === undefined) return results; - for (let offset = 0; offset < files.length; offset += 4) { - const batch = files.slice(offset, offset + 4); - const known = await Promise.all(batch.map((file) => this.entryBytes(file.previous))); - const requests = batch.map((file, index) => ({ - path: resolveWorkspacePath(path, this.workspaceCtx.workDir, file.pathKey), - knownHash: known[index] === undefined ? undefined : file.previous?.contentHash, - })); - const snapshots = await (fs.operations === undefined - ? snapshotFiles(fs, requests, FILE_HISTORY_MAX_FILE_BYTES) - : fs.operations.snapshotFiles(requests, FILE_HISTORY_MAX_FILE_BYTES)).catch(() => []); - for (let index = 0; index < batch.length; index += 1) { - const snapshot = snapshots[index]; - const current = snapshot?.kind === 'file' - ? snapshot.bytes ?? known[index] ?? 'unreadable' - : snapshot?.kind === 'oversize' - ? { oversizeBytes: snapshot.size, mtimeMs: snapshot.mtimeMs } - : snapshot?.kind ?? 'unreadable'; - results.set(batch[index]!.pathKey, current); + if (path === undefined) return 'unreadable'; + const absolute = resolveWorkspacePath(path, this.workspaceCtx.workDir, pathKey); + let info; + try { + info = await fs.stat(absolute); + } catch (error) { + return isHostFsNotFound(error) ? 'missing' : 'unreadable'; + } + if (!info.isFile) return 'unreadable'; + if (info.size > FILE_HISTORY_MAX_FILE_BYTES) { + return { oversizeBytes: info.size, mtimeMs: info.mtimeMs }; + } + try { + const bytes = await fs.readBytes(absolute, info.size + 1); + if (bytes.byteLength > FILE_HISTORY_MAX_FILE_BYTES) { + const grown = await fs.stat(absolute).catch(() => undefined); + return { + oversizeBytes: grown?.size ?? bytes.byteLength, + mtimeMs: grown?.mtimeMs, + }; } + if (bytes.byteLength !== info.size) return 'unreadable'; + return bytes; + } catch { + return 'unreadable'; } - return results; } finally { lease.dispose(); } diff --git a/packages/agent-core-v2/src/os/backends/node-local/tools/runRg.ts b/packages/agent-core-v2/src/os/backends/node-local/tools/runRg.ts index 957040d2cba..e607da9adef 100644 --- a/packages/agent-core-v2/src/os/backends/node-local/tools/runRg.ts +++ b/packages/agent-core-v2/src/os/backends/node-local/tools/runRg.ts @@ -1 +1,196 @@ -export * from '#/os/operations/runRg'; +import type { Readable } from 'node:stream'; + +import { BugIndicatingError } from '#/errors'; +import type { IHostProcess, IHostProcessService } from '#/os/interface/hostProcess'; + +export const DEFAULT_TIMEOUT_MS = 20_000; +export const SIGTERM_GRACE_MS = 5_000; +export const MAX_OUTPUT_BYTES = 10 * 1024 * 1024; + +export interface RunRgResult { + readonly kind: 'result'; + readonly exitCode: number; + readonly stdoutText: string; + readonly stderrText: string; + readonly bufferTruncated: boolean; + readonly stderrTruncated: boolean; + readonly timedOut: boolean; +} + +export type RunRgOutcome = RunRgResult | { readonly kind: 'aborted' }; + +function disposeProcess(proc: IHostProcess): void { + try { + void proc.dispose(); + } catch { + } +} + +export async function runRgOnce( + processService: IHostProcessService, + rgArgs: readonly string[], + signal: AbortSignal, + options?: { readonly cwd?: string }, +): Promise { + if (signal.aborted) { + return { kind: 'aborted' }; + } + + const [command, ...args] = rgArgs; + if (command === undefined) { + throw new BugIndicatingError('runRgOnce: rgArgs must not be empty'); + } + const proc: IHostProcess = await processService.spawn(command, args, { cwd: options?.cwd }); + + try { + proc.stdin.end(); + } catch { + } + + let timedOut = false; + let aborted = false; + let killed = false; + + const killProc = async (): Promise => { + if (killed) return; + killed = true; + try { + await proc.kill('SIGTERM'); + } catch { + } + const exited = proc + .wait() + .then(() => true) + .catch(() => true); + const raced = await Promise.race([ + exited, + new Promise((resolve) => { + setTimeout(() => { + resolve(false); + }, SIGTERM_GRACE_MS); + }), + ]); + if (!raced && proc.exitCode === null) { + try { + await proc.kill('SIGKILL'); + } catch { + } + } + disposeProcess(proc); + }; + + const onAbort = (): void => { + aborted = true; + void killProc(); + }; + signal.addEventListener('abort', onAbort); + if (signal.aborted) onAbort(); + + const timeoutHandle = setTimeout(() => { + timedOut = true; + void killProc(); + }, DEFAULT_TIMEOUT_MS); + + let exitCode = 0; + let stdoutText = ''; + let stderrText = ''; + let bufferTruncated = false; + let stderrTruncated = false; + + try { + const isTerminating = (): boolean => timedOut || aborted || killed; + const [stdoutResult, stderrResult, code] = await Promise.all([ + readStreamWithCap(proc.stdout, MAX_OUTPUT_BYTES, isTerminating, () => { + void killProc(); + }), + readStreamWithCap(proc.stderr, MAX_OUTPUT_BYTES, isTerminating), + proc.wait(), + ]); + stdoutText = stdoutResult.text; + stderrText = stderrResult.text; + bufferTruncated = stdoutResult.truncated; + stderrTruncated = stderrResult.truncated; + exitCode = code; + } catch (error) { + if (!(isPrematureCloseError(error) && (timedOut || aborted || killed))) { + throw error; + } + } finally { + clearTimeout(timeoutHandle); + signal.removeEventListener('abort', onAbort); + disposeProcess(proc); + } + + if (aborted) { + return { kind: 'aborted' }; + } + + return { + kind: 'result', + exitCode, + stdoutText, + stderrText, + bufferTruncated, + stderrTruncated, + timedOut, + }; +} + +export function shouldRetryRipgrepEagain(result: RunRgResult): boolean { + return ( + result.exitCode !== 0 && + result.exitCode !== 1 && + !result.timedOut && + !result.bufferTruncated && + isEagainRipgrepError(result.stderrText) + ); +} + +function isEagainRipgrepError(stderr: string): boolean { + return stderr.includes('os error 11') || stderr.includes('Resource temporarily unavailable'); +} + +function isPrematureCloseError(error: unknown): boolean { + return ( + error instanceof Error && + (error as NodeJS.ErrnoException).code === 'ERR_STREAM_PREMATURE_CLOSE' + ); +} + +interface CappedStreamResult { + readonly text: string; + readonly truncated: boolean; +} + +async function readStreamWithCap( + stream: Readable, + maxBytes: number, + suppressPrematureClose?: () => boolean, + onLimit?: () => void, +): Promise { + const chunks: Buffer[] = []; + let total = 0; + let truncated = false; + try { + for await (const chunk of stream) { + const buf: Buffer = + typeof chunk === 'string' ? Buffer.from(chunk, 'utf8') : (chunk as Buffer); + if (truncated) continue; + if (total + buf.length > maxBytes) { + const remaining = maxBytes - total; + if (remaining > 0) chunks.push(buf.subarray(0, remaining)); + total = maxBytes; + truncated = true; + onLimit?.(); + continue; + } + chunks.push(buf); + total += buf.length; + } + } catch (error) { + if (!isPrematureCloseError(error) || suppressPrematureClose?.() !== true) { + throw error; + } + } + return { text: Buffer.concat(chunks).toString('utf8'), truncated }; +} diff --git a/packages/agent-core-v2/src/os/backends/remote/remoteFileSystem.ts b/packages/agent-core-v2/src/os/backends/remote/remoteFileSystem.ts index 6ac913048bd..b9fb1931ceb 100644 --- a/packages/agent-core-v2/src/os/backends/remote/remoteFileSystem.ts +++ b/packages/agent-core-v2/src/os/backends/remote/remoteFileSystem.ts @@ -1,9 +1,3 @@ -import { readTextPage } from '#/os/operations/readTextPage'; -import { EditService } from '#/os/operations/editService'; -import { TextModel } from '#/os/operations/textModel'; -import { createHash, randomUUID } from 'node:crypto'; -import { writeTextResult } from '#/os/operations/writeText'; -import { TEXT_PAGE_MAX_REMOTE_CHARS, type GitWorkTree, type FileSnapshot, type HostFileOperations } from '#/os/interface/fileOperations'; import * as posixPath from 'node:path/posix'; import { @@ -41,15 +35,6 @@ import { FS_READ_DIRECTORY_MAX_ENTRIES, FS_READ_FILE_MAX_BYTES, FS_READ_FILE_METHOD, - FS_READ_TEXT_PAGE_METHOD, - FS_CANCEL_READ_METHOD, - FS_MAX_ACTIVE_READS, - FS_SNAPSHOT_FILES_METHOD, - FS_MATCH_FILES_METHOD, - FS_FIND_GIT_WORK_TREE_METHOD, - FS_DISCOVER_INSTRUCTIONS_METHOD, - FS_SNAPSHOT_MAX_FILES, - FS_EDIT_TEXT_METHOD, FS_REMOVE_METHOD, FS_RENAME_METHOD, FS_WALK_METHOD, @@ -94,115 +79,7 @@ function isUtf8Encoding(encoding: BufferEncoding): boolean { export class RemoteFileSystem implements IHostFileSystem { declare readonly _serviceBrand: undefined; - private readonly contents = new Map(); - private contentBytes = 0; - private activeReads = 0; - - constructor(private readonly connection: ExecChannel) { - connection.onDidClose(() => { - this.contents.clear(); - this.contentBytes = 0; - }); - } - - private rememberContent(path: string, data: Uint8Array): void { - const previous = this.contents.get(path); - if (previous !== undefined) { - this.contentBytes -= previous.bytes.byteLength; - this.contents.delete(path); - } - if (data.byteLength > 4 * 1024 * 1024) return; - const bytes = Uint8Array.from(data); - this.contents.set(path, { bytes, hash: createHash('sha256').update(bytes).digest('hex') }); - this.contentBytes += bytes.byteLength; - while (this.contentBytes > 16 * 1024 * 1024 || this.contents.size > 16) { - const first = this.contents.entries().next().value!; - this.contents.delete(first[0]); - this.contentBytes -= first[1].bytes.byteLength; - } - } - - - readonly operations: HostFileOperations = { - matchFiles: (files, options, timeoutMs) => this.call(FS_MATCH_FILES_METHOD, { files, options, timeoutMs }), - findGitWorkTree: async (cwd) => posixPath.isAbsolute(cwd) ? (await this.call<{ workTree: GitWorkTree | null }>(FS_FIND_GIT_WORK_TREE_METHOD, { cwd })).workTree : null, - discoverInstructions: async (dirs, known) => { - const paths = new Set(); - for (let offset = 0; offset < dirs.length; offset += 64) { - for (const path of await this.call(FS_DISCOVER_INSTRUCTIONS_METHOD, { dirs: dirs.slice(offset, offset + 64), known })) paths.add(path); - } - return [...paths]; - }, - snapshotFiles: async (files, maxFileBytes) => { - const results: FileSnapshot[] = []; - for (let offset = 0; offset < files.length; offset += FS_SNAPSHOT_MAX_FILES) { - const requests = files.slice(offset, offset + FS_SNAPSHOT_MAX_FILES); - const cached = requests.map((file) => this.contents.get(file.path)); - const batch = await this.call(FS_SNAPSHOT_FILES_METHOD, { - files: requests.map((file, index) => ({ path: file.path, knownHash: cached[index]?.hash ?? file.knownHash })), maxFileBytes, - }); - for (let index = 0; index < batch.length; index += 1) { - const result = batch[index]!; - const request = requests[index]!; - if (result.kind !== 'file') { - results.push(result); - continue; - } - const bytes = result.dataBase64 === undefined - ? cached[index]?.hash === result.contentHash ? cached[index]!.bytes : undefined - : decodeBase64(result.dataBase64); - if (bytes !== undefined) this.rememberContent(request.path, bytes); - results.push({ kind: 'file', contentHash: result.contentHash, bytes: request.knownHash === result.contentHash ? undefined : bytes }); - } - } - return results; - }, - writeText: (input) => writeTextResult(input, async () => { - const bytes = Buffer.from(input.content, 'utf8'); - const previous = input.mode === 'append' ? this.contents.get(input.path) : undefined; - await this.writeChunked(input.path, bytes, input.mode === 'append' ? 'append' : 'truncate', input.parent); - if (input.mode === 'overwrite') this.rememberContent(input.path, bytes); - else if (previous !== undefined) this.rememberContent(input.path, Buffer.concat([previous.bytes, bytes])); - }), - readTextPage: async (path, options, signal) => { - if (signal?.aborted === true) return { isError: true, output: 'Read aborted' }; - if (options.maxChars > TEXT_PAGE_MAX_REMOTE_CHARS || this.activeReads >= FS_MAX_ACTIVE_READS) { - return readTextPage({ name: path, stat: () => this.stat(path), readBytes: (n) => this.readBytes(path, n), readLines: () => this.readLines(path) }, options, signal); - } - this.activeReads += 1; - const readId = randomUUID(); - const cancel = (): void => { - try { this.connection.notify(FS_CANCEL_READ_METHOD, { readId }); } catch {} - }; - signal?.addEventListener('abort', cancel, { once: true }); - try { - return await this.call(FS_READ_TEXT_PAGE_METHOD, { path, ...options, readId }); - } catch (error) { - cancel(); - return { isError: true, output: error instanceof Error ? error.message : String(error) }; - } finally { - this.activeReads -= 1; - signal?.removeEventListener('abort', cancel); - } - }, - editText: async (input) => { - const previous = this.contents.get(input.path); - let result: Awaited>; - try { - result = await this.call(FS_EDIT_TEXT_METHOD, input); - } catch (error) { - return { ok: false, error: error instanceof Error ? error.message : String(error) }; - } - if (result.ok && previous !== undefined) { - try { - const raw = decodeTextWithErrors(Buffer.from(previous.bytes), 'utf8', 'strict'); - const applied = new EditService().apply(new TextModel(raw), { ...input, path: input.displayPath }); - if (applied.ok) this.rememberContent(input.path, Buffer.from(applied.rawContent)); - } catch {} - } - return result; - }, - }; + constructor(private readonly connection: ExecChannel) {} private async call(method: string, params: unknown): Promise { try { @@ -216,31 +93,30 @@ export class RemoteFileSystem implements IHostFileSystem { return this.call(FS_READ_FILE_METHOD, { path, offset, maxBytes }); } + private async writeMode(path: string, data: Uint8Array, mode: FsWriteMode, parent?: string): Promise { + await this.call(FS_WRITE_FILE_METHOD, { path, dataBase64: encodeBase64(data), mode, parent }); + } + private async writeChunked(path: string, data: Uint8Array, mode: FsWriteMode, parent?: string): Promise { if (data.byteLength <= FS_WRITE_FILE_CHUNK_BYTES) { - await this.call(FS_WRITE_FILE_METHOD, { path, dataBase64: encodeBase64(data), mode, parent }); + await this.writeMode(path, data, mode, parent); return; } - const transferId = randomUUID(); - let chunkIndex = 0; - for (let offset = 0; offset < data.byteLength;) { - const pending: Promise[] = []; - for (let window = 0; window < 4 && offset < data.byteLength; window += 1) { - const end = Math.min(data.byteLength, offset + FS_WRITE_FILE_CHUNK_BYTES); - pending.push(this.call(FS_WRITE_FILE_METHOD, { - path, - dataBase64: encodeBase64(data.subarray(offset, end)), - mode: offset === 0 ? mode : 'append', - parent: offset === 0 ? parent : undefined, - transferId, - chunkIndex: chunkIndex++, - final: end === data.byteLength, - })); - offset = end; - } - const results = await Promise.allSettled(pending); - const failure = results.find((result) => result.status === 'rejected'); - if (failure?.status === 'rejected') throw failure.reason; + for (let offset = 0; offset < data.byteLength; offset += FS_WRITE_FILE_CHUNK_BYTES) { + await this.writeMode(path, data.subarray(offset, offset + FS_WRITE_FILE_CHUNK_BYTES), offset === 0 ? mode : 'append', offset === 0 ? parent : undefined); + } + } + + async readWholeFile(path: string) { + try { + const result = await this.readRange(path, 0); + const bytes = decodeBase64(result.dataBase64); + return { + bytes, + stat: { isFile: true, isDirectory: false, size: bytes.byteLength }, + }; + } catch { + return undefined; } } @@ -248,7 +124,8 @@ export class RemoteFileSystem implements IHostFileSystem { path: string, options?: { encoding?: BufferEncoding; errors?: TextDecodeErrors }, ): Promise { - const bytes = await this.readBytes(path); + const whole = await this.readWholeFile(path); + const bytes = whole === undefined ? await this.readBytes(path) : whole.bytes; const buffer = Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength); return decodeTextWithErrors(buffer, options?.encoding ?? 'utf-8', options?.errors ?? 'strict'); } @@ -257,6 +134,10 @@ export class RemoteFileSystem implements IHostFileSystem { await this.writeChunked(path, Buffer.from(data, 'utf8'), 'truncate'); } + async writeTextWithParent(path: string, data: string, mode: 'overwrite' | 'append', parent: string): Promise { + await this.writeChunked(path, Buffer.from(data, 'utf8'), mode === 'append' ? 'append' : 'truncate', parent); + } + async appendText(path: string, data: string): Promise { await this.writeChunked(path, Buffer.from(data, 'utf8'), 'append'); } @@ -265,7 +146,7 @@ export class RemoteFileSystem implements IHostFileSystem { if (n === 0) return new Uint8Array(0); const chunks: Uint8Array[] = []; let total = 0; - for await (const chunk of this.readChunks(path, offset, n)) { + for await (const chunk of this.readChunks(path, offset)) { const length = n === undefined ? chunk.byteLength : Math.min(chunk.byteLength, n - total); if (length === 0) break; chunks.push(chunk.subarray(0, length)); @@ -286,43 +167,25 @@ export class RemoteFileSystem implements IHostFileSystem { await this.writeChunked(path, data, 'truncate'); return; } - const transferId = randomUUID(); - let chunkIndex = 0; + let mode: FsWriteMode = 'truncate'; const pending = new Uint8Array(FS_WRITE_FILE_CHUNK_BYTES); let pendingBytes = 0; - let writes: Promise[] = []; - const send = (bytes: Uint8Array, final: boolean): void => { - writes.push(this.call(FS_WRITE_FILE_METHOD, { - path, dataBase64: encodeBase64(bytes), mode: chunkIndex === 0 ? 'truncate' : 'append', - transferId, chunkIndex: chunkIndex++, final, - })); - }; - const flush = async (): Promise => { - const results = await Promise.allSettled(writes); - writes = []; - const failure = results.find((result) => result.status === 'rejected'); - if (failure?.status === 'rejected') throw failure.reason; - }; - try { - for await (const chunk of data) { - let offset = 0; - while (offset < chunk.byteLength) { - const length = Math.min(pending.byteLength - pendingBytes, chunk.byteLength - offset); - pending.set(chunk.subarray(offset, offset + length), pendingBytes); - pendingBytes += length; - offset += length; - if (pendingBytes === pending.byteLength) { - send(pending, false); - pendingBytes = 0; - if (writes.length === 4) await flush(); - } + for await (const chunk of data) { + let offset = 0; + while (offset < chunk.byteLength) { + const length = Math.min(pending.byteLength - pendingBytes, chunk.byteLength - offset); + pending.set(chunk.subarray(offset, offset + length), pendingBytes); + pendingBytes += length; + offset += length; + if (pendingBytes === pending.byteLength) { + await this.writeMode(path, pending, mode); + mode = 'append'; + pendingBytes = 0; } - await flush(); } - send(pending.subarray(0, pendingBytes), true); - await flush(); - } finally { - await Promise.allSettled(writes); + } + if (pendingBytes > 0 || mode === 'truncate') { + await this.writeMode(path, pending.subarray(0, pendingBytes), mode); } } @@ -340,15 +203,14 @@ export class RemoteFileSystem implements IHostFileSystem { yield* readUtf8Lines(this.readChunks(path), errors); } - private async *readChunks(path: string, offset = 0, remaining?: number): AsyncGenerator { + private async *readChunks(path: string, offset = 0): AsyncGenerator { for (;;) { - const result = await this.readRange(path, offset, Math.min(remaining ?? FS_READ_FILE_MAX_BYTES, FS_READ_FILE_MAX_BYTES)); + const result = await this.readRange(path, offset, FS_READ_FILE_MAX_BYTES); if (result.dataBase64.length === 0) return; const chunk = decodeBase64(result.dataBase64); yield chunk; offset += chunk.byteLength; - if (remaining !== undefined) remaining -= chunk.byteLength; - if (result.eof || remaining === 0) return; + if (result.eof) return; } } diff --git a/packages/agent-core-v2/src/os/backends/remote/remoteProcess.ts b/packages/agent-core-v2/src/os/backends/remote/remoteProcess.ts index 3daad334307..5dc2a7c6e12 100644 --- a/packages/agent-core-v2/src/os/backends/remote/remoteProcess.ts +++ b/packages/agent-core-v2/src/os/backends/remote/remoteProcess.ts @@ -1,6 +1,3 @@ -import { searchGlob } from '#/os/operations/searchGlob'; -import { searchGrep } from '#/os/operations/searchGrep'; -import type { SearchRequest, SearchResult } from '#/os/interface/searchOperations'; import { randomUUID } from 'node:crypto'; import { Readable, Writable } from 'node:stream'; @@ -23,10 +20,6 @@ import { PROCESS_OUTPUT_METHOD, PROCESS_SIGNAL_METHOD, PROCESS_START_METHOD, - PROCESS_SEARCH_METHOD, - PROCESS_CANCEL_SEARCH_METHOD, - PROCESS_MAX_ACTIVE_SEARCHES, - PROCESS_SEARCH_MAX_ROOTS, PROCESS_WRITE_METHOD, type ProcessClosedNotification, type ProcessExitedNotification, @@ -240,7 +233,6 @@ export class RemoteProcessService implements IHostProcessService { declare readonly _serviceBrand: undefined; private readonly processes = new Map(); - private activeSearches = 0; constructor( private readonly connection: ExecChannel, @@ -271,28 +263,6 @@ export class RemoteProcessService implements IHostProcessService { }); } - async search(request: SearchRequest, signal: AbortSignal): Promise { - if (signal.aborted) return { isError: true, output: request.kind === 'glob' ? 'Glob aborted' : 'Grep aborted' }; - if (request.roots.length > PROCESS_SEARCH_MAX_ROOTS || this.activeSearches >= PROCESS_MAX_ACTIVE_SEARCHES) { - return request.kind === 'glob' ? searchGlob(this, request, signal) : searchGrep(this, request, signal); - } - this.activeSearches += 1; - const searchId = randomUUID(); - const cancel = (): void => { - try { this.connection.notify(PROCESS_CANCEL_SEARCH_METHOD, { searchId }); } catch {} - }; - signal.addEventListener('abort', cancel, { once: true }); - try { - return await this.connection.call(PROCESS_SEARCH_METHOD, { ...request, searchId }) as SearchResult; - } catch (error) { - cancel(); - throw toRemoteProcessError(error); - } finally { - this.activeSearches -= 1; - signal.removeEventListener('abort', cancel); - } - } - async spawn( command: string, args: readonly string[] = [], diff --git a/packages/agent-core-v2/src/os/interface/fileOperations.ts b/packages/agent-core-v2/src/os/interface/fileOperations.ts deleted file mode 100644 index cfa8479f111..00000000000 --- a/packages/agent-core-v2/src/os/interface/fileOperations.ts +++ /dev/null @@ -1,115 +0,0 @@ -import type { ToolResultSpill } from '#/_base/text/outputLimits'; -import type { HostFileStat } from './hostFileSystem'; - -export const TEXT_PAGE_MAX_REMOTE_CHARS = 4_000_000; - -export interface TextFileResult { - readonly output: string; - readonly note?: string; - readonly isError?: boolean; - readonly truncated?: boolean; - readonly spill?: ToolResultSpill; -} - -export interface TextPageInput { - readonly path: string; - readonly line_offset?: number; - readonly column_offset?: number; - readonly n_lines?: number; - readonly max_chars?: number; -} - -export interface TextPageOptions { - readonly args: TextPageInput; - readonly maxChars: number; - readonly maxCharsLimit: number; - readonly eventLog: boolean; - readonly imageSupported: boolean; - readonly videoSupported: boolean; - readonly mediaToolAvailable: boolean; -} - -export interface FileReadSource { - readonly name: string; - readonly localPath?: string; - stat(): Promise; - readBytes(n?: number): Promise; - readLines(): AsyncIterable; - readTextPage?(options: TextPageOptions, signal?: AbortSignal): Promise; -} - -export interface TextEditInput { - readonly path: string; - readonly displayPath: string; - readonly old_string: string; - readonly new_string: string; - readonly replace_all: boolean; -} - -export type TextEditResult = - | { readonly ok: true; readonly count: number } - | { readonly ok: false; readonly error: string }; - -export interface TextWriteInput { - readonly path: string; - readonly displayPath: string; - readonly parent: string; - readonly content: string; - readonly mode: 'overwrite' | 'append'; -} - -export interface FileSnapshotRequest { - readonly path: string; - readonly knownHash?: string; -} - -export type FileSnapshot = - | { readonly kind: 'missing' | 'unreadable' } - | { readonly kind: 'oversize'; readonly size: number; readonly mtimeMs?: number } - | { readonly kind: 'file'; readonly contentHash: string; readonly bytes?: Uint8Array }; - -export interface GitWorkTree { - readonly root: string; - readonly dotGitPath: string; - readonly controlDirPath: string; -} - -export interface FileMatchOptions { - readonly pattern: string; - readonly regex: boolean; - readonly case_sensitive: boolean; - readonly context_lines: number; - readonly max_files: number; - readonly max_matches_per_file: number; - readonly max_total_matches: number; -} - -export interface FileMatch { - readonly line: number; - readonly col: number; - readonly text: string; - readonly before: string[]; - readonly after: string[]; -} - -export interface FileMatches { - readonly responseTooLarge?: boolean; - readonly files: { path: string; matches: FileMatch[] }[]; - readonly files_scanned: number; - readonly truncated: boolean; -} - -export interface FileMatchPath { - readonly path: string; - readonly absolute: string; -} - -export interface HostFileOperations { - matchFiles(files: readonly FileMatchPath[], options: FileMatchOptions, timeoutMs: number): Promise; - findGitWorkTree(cwd: string): Promise; - discoverInstructions(dirs: readonly string[], known: readonly string[]): Promise; - snapshotFiles(files: readonly FileSnapshotRequest[], maxFileBytes: number): Promise; - writeText(input: TextWriteInput): Promise; - readTextPage(path: string, options: TextPageOptions, signal?: AbortSignal): Promise; - editText(input: TextEditInput): Promise; -} diff --git a/packages/agent-core-v2/src/os/interface/hostFileSystem.ts b/packages/agent-core-v2/src/os/interface/hostFileSystem.ts index 69b02812575..9327692e364 100644 --- a/packages/agent-core-v2/src/os/interface/hostFileSystem.ts +++ b/packages/agent-core-v2/src/os/interface/hostFileSystem.ts @@ -1,5 +1,4 @@ import { createDecorator, type ServiceIdentifier } from '#/_base/di/instantiation'; -import type { HostFileOperations } from './fileOperations'; import type { TextDecodeErrors } from '#/_base/execEnv/decodeText'; export interface HostFileStat { @@ -12,6 +11,11 @@ export interface HostFileStat { readonly mode?: number; } +export interface HostWholeFile { + readonly stat: HostFileStat; + readonly bytes: Uint8Array; +} + export interface HostDirEntry { readonly name: string; readonly isFile: boolean; @@ -43,13 +47,13 @@ export interface HostWalkResult { export interface IHostFileSystem { readonly _serviceBrand: undefined; - readonly operations?: HostFileOperations; - readText( path: string, options?: { encoding?: BufferEncoding; errors?: TextDecodeErrors }, ): Promise; + readWholeFile?(path: string): Promise; writeText(path: string, data: string): Promise; + writeTextWithParent?(path: string, data: string, mode: 'overwrite' | 'append', parent: string): Promise; appendText(path: string, data: string): Promise; readBytes(path: string, n?: number, offset?: number): Promise; writeBytes(path: string, data: Uint8Array | AsyncIterable): Promise; diff --git a/packages/agent-core-v2/src/os/interface/hostProcess.ts b/packages/agent-core-v2/src/os/interface/hostProcess.ts index 376c63ef45b..6197026f790 100644 --- a/packages/agent-core-v2/src/os/interface/hostProcess.ts +++ b/packages/agent-core-v2/src/os/interface/hostProcess.ts @@ -1,4 +1,3 @@ -import type { SearchRequest, SearchResult } from './searchOperations'; import type { Readable, Writable } from 'node:stream'; import { registerErrorDomain, type ErrorDomain } from '#/_base/errors/codes'; @@ -28,12 +27,9 @@ export interface IHostProcess { dispose(): void | Promise; } - export interface IHostProcessService { readonly _serviceBrand: undefined; - search?(request: SearchRequest, signal: AbortSignal): Promise; - spawn( command: string, args?: readonly string[], diff --git a/packages/agent-core-v2/src/os/interface/searchOperations.ts b/packages/agent-core-v2/src/os/interface/searchOperations.ts deleted file mode 100644 index f880e02d4b7..00000000000 --- a/packages/agent-core-v2/src/os/interface/searchOperations.ts +++ /dev/null @@ -1,33 +0,0 @@ -import type { TextFileResult } from './fileOperations'; - -export interface GlobSearchInput { - readonly pattern: string; - readonly head_limit?: number; - readonly offset?: number; - readonly path?: string; - readonly include_ignored?: boolean; - readonly include_dirs?: boolean; -} - -export interface GrepSearchInput extends GlobSearchInput { - readonly glob?: string; - readonly type?: string; - readonly output_mode?: 'content' | 'files_with_matches' | 'count_matches'; - readonly '-i'?: boolean; - readonly '-n'?: boolean; - readonly '-A'?: number; - readonly '-B'?: number; - readonly '-C'?: number; - readonly multiline?: boolean; -} - -export interface SearchRequest { - readonly kind: 'glob' | 'grep'; - readonly args: GrepSearchInput; - readonly rgPath: string; - readonly roots: readonly string[]; - readonly workspaceDir: string; - readonly pathClass: 'posix' | 'win32'; -} - -export type SearchResult = TextFileResult; diff --git a/packages/agent-core-v2/src/os/operations/editService.ts b/packages/agent-core-v2/src/os/operations/editService.ts deleted file mode 100644 index 3c261d4c9c6..00000000000 --- a/packages/agent-core-v2/src/os/operations/editService.ts +++ /dev/null @@ -1,41 +0,0 @@ -import type { TextModel } from './textModel'; - -export interface EditApplyInput { - readonly path: string; - readonly old_string: string; - readonly new_string: string; - readonly replace_all: boolean; -} - -export type EditApplyResult = - | { readonly ok: true; readonly rawContent: string; readonly count: number } - | { readonly ok: false; readonly error: string }; - -function notFoundMessage(path: string): string { - return `old_string not found in ${path}, the file contents may be out of date. Please use the Read Tool to reload the content. -`; -} - -function notUniqueMessage(path: string, count: number): string { - return ( - `old_string is not unique in ${path} (found ${String(count)} occurrences). ` + - 'To replace every occurrence, set replace_all=true. To replace only one occurrence, include more surrounding context in old_string.' - ); -} - -export class EditService { - apply(model: TextModel, input: EditApplyInput): EditApplyResult { - if (input.replace_all) { - const { text, count } = model.replaceAll(input.old_string, input.new_string); - if (count === 0) return { ok: false, error: notFoundMessage(input.path) }; - return { ok: true, rawContent: model.materialize(text), count }; - } - - const count = model.countOccurrences(input.old_string); - if (count === 0) return { ok: false, error: notFoundMessage(input.path) }; - if (count > 1) return { ok: false, error: notUniqueMessage(input.path, count) }; - - const text = model.replaceOnce(input.old_string, input.new_string); - return { ok: true, rawContent: model.materialize(text), count: 1 }; - } -} diff --git a/packages/agent-core-v2/src/os/operations/editText.ts b/packages/agent-core-v2/src/os/operations/editText.ts deleted file mode 100644 index faa24f57876..00000000000 --- a/packages/agent-core-v2/src/os/operations/editText.ts +++ /dev/null @@ -1,32 +0,0 @@ -import type { IHostFileSystem } from '#/os/interface/hostFileSystem'; -import type { TextEditInput, TextEditResult } from '#/os/interface/fileOperations'; -import { unwrapErrorCause } from '#/_base/errors/errors'; -import { EditService } from './editService'; -import { TextModel } from './textModel'; - -export async function editText(fs: IHostFileSystem, input: TextEditInput): Promise { - try { - const raw = await fs.readText(input.path, { errors: 'strict' }); - const model = new TextModel(raw); - const result = new EditService().apply(model, { - path: input.displayPath, - old_string: input.old_string, - new_string: input.new_string, - replace_all: input.replace_all, - }); - if (!result.ok) { - return { ok: false, error: result.error }; - } - await fs.writeText(input.path, result.rawContent); - return { ok: true, count: result.count }; - } catch (error) { - const code = (unwrapErrorCause(error) as { code?: unknown } | null)?.code; - if (code === 'EISDIR') { - return { ok: false, error: `${input.displayPath} is not a file.` }; - } - return { - ok: false, - error: error instanceof Error ? error.message : String(error), - }; - } -} diff --git a/packages/agent-core-v2/src/os/operations/gitWorkTree.ts b/packages/agent-core-v2/src/os/operations/gitWorkTree.ts deleted file mode 100644 index 40d0c5471ab..00000000000 --- a/packages/agent-core-v2/src/os/operations/gitWorkTree.ts +++ /dev/null @@ -1,50 +0,0 @@ -import { dirname, isAbsolute, join, normalize } from 'pathe'; - -import type { IHostFileSystem } from '#/os/interface/hostFileSystem'; - -import type { GitWorkTree } from '#/os/interface/fileOperations'; - -export async function findGitWorkTree( - fs: Pick, - cwd: string, -): Promise { - if (cwd.length === 0 || !isAbsolute(cwd)) return null; - - let current = normalize(cwd); - while (true) { - const dotGitPath = join(current, '.git'); - const controlDirPath = await probeGitControlDir(fs, dotGitPath, current); - if (controlDirPath !== null) return { root: current, dotGitPath, controlDirPath }; - - const parent = dirname(current); - if (parent === current) return null; - current = parent; - } -} - -async function probeGitControlDir( - fs: Pick, - dotGitPath: string, - markerParent: string, -): Promise { - try { - const stat = await fs.stat(dotGitPath); - if (stat.isDirectory) return dotGitPath; - if (!stat.isFile) return null; - - const content = await fs.readText(dotGitPath); - return parseGitDirPointer(content, markerParent) ?? null; - } catch { - return null; - } -} - -function parseGitDirPointer(content: string, markerParent: string): string | undefined { - const stripped = content.codePointAt(0) === 0xfeff ? content.slice(1) : content; - const line = stripped.trimStart().split(/\r?\n/, 1)[0]?.trim(); - if (line === undefined || !line.startsWith('gitdir:')) return undefined; - - const rawPath = line.slice('gitdir:'.length).trim(); - if (rawPath.length === 0) return undefined; - return normalize(isAbsolute(rawPath) ? rawPath : join(markerParent, rawPath)); -} diff --git a/packages/agent-core-v2/src/os/operations/instructions.ts b/packages/agent-core-v2/src/os/operations/instructions.ts deleted file mode 100644 index c5bf48ff044..00000000000 --- a/packages/agent-core-v2/src/os/operations/instructions.ts +++ /dev/null @@ -1,107 +0,0 @@ -import { dirname, join, normalize } from 'pathe'; - -import type { HostFileStat, IHostFileSystem } from '#/os/interface/hostFileSystem'; -import { findGitWorkTree } from './gitWorkTree'; - -export const AGENTS_MD_PLAIN_NAMES = ['AGENTS.md', 'agents.md'] as const; - -export function dotKimiAgentsMdPath(dir: string): string { - return join(dir, '.kimi-code', 'AGENTS.md'); -} - -export function agentsMdCandidatePaths(dir: string): string[] { - return [dotKimiAgentsMdPath(dir), ...AGENTS_MD_PLAIN_NAMES.map((name) => join(dir, name))]; -} - -export async function findAgentsMdInDir( - deps: { readonly fs: Pick }, - dir: string, -): Promise { - const found: string[] = []; - const dotKimi = dotKimiAgentsMdPath(dir); - if (await isNonEmptyFile(deps, dotKimi)) found.push(dotKimi); - for (const fileName of AGENTS_MD_PLAIN_NAMES) { - const candidate = join(dir, fileName); - if (await isNonEmptyFile(deps, candidate)) { - found.push(candidate); - break; - } - } - return found; -} - -async function isNonEmptyFile( - deps: { readonly fs: Pick }, - path: string, -): Promise { - try { - const content = await deps.fs.readText(path, { errors: 'ignore' }); - return content.trim().length > 0; - } catch { - return false; - } -} - -export function dirsRootToLeaf(workDir: string, projectRoot: string): string[] { - const dirs: string[] = []; - let current = normalize(workDir); - - while (true) { - dirs.push(current); - if (current === projectRoot) break; - const parent = dirname(current); - if (parent === current) break; - current = parent; - } - - return dirs.toReversed(); -} - -export async function discoverInstructions(fs: IHostFileSystem, dirs: readonly string[], knownPaths: readonly string[]): Promise { - const known = new Set(knownPaths.map(normalize)); - const stats = new Map>(); - const contents = new Map>(); - const probe: Pick = { - stat: (path) => { - let pending = stats.get(path); - if (pending === undefined) { - pending = fs.stat(path); - stats.set(path, pending); - } - return pending; - }, - readText: (path, options) => { - const key = JSON.stringify([path, options]); - let pending = contents.get(key); - if (pending === undefined) { - pending = fs.readText(path, options); - contents.set(key, pending); - } - return pending; - }, - }; - const found = new Set(); - const inspected = new Set(); - for (const dir of new Set(dirs)) { - let anchor = dir; - let exists = false; - for (;;) { - if ((await probe.stat(anchor).catch(() => undefined))?.isDirectory === true) { - exists = true; - break; - } - const parent = dirname(anchor); - if (parent === anchor) break; - anchor = parent; - } - if (!exists) continue; - const projectRoot = (await findGitWorkTree(probe, anchor))?.root ?? anchor; - for (const chainDir of dirsRootToLeaf(anchor, projectRoot)) { - if (inspected.has(chainDir)) continue; - inspected.add(chainDir); - if (agentsMdCandidatePaths(chainDir).every((path) => known.has(normalize(path)))) continue; - for (const path of await findAgentsMdInDir({ fs }, chainDir)) found.add(normalize(path)); - } - } - return [...found]; -} diff --git a/packages/agent-core-v2/src/os/operations/matchFiles.ts b/packages/agent-core-v2/src/os/operations/matchFiles.ts deleted file mode 100644 index 921047cb956..00000000000 --- a/packages/agent-core-v2/src/os/operations/matchFiles.ts +++ /dev/null @@ -1,72 +0,0 @@ -import type { IHostFileSystem } from '#/os/interface/hostFileSystem'; -import type { FileMatch, FileMatches, FileMatchOptions, FileMatchPath } from '#/os/interface/fileOperations'; - -export function compileGrepPattern>(req: T): RegExp { - const flags = req.case_sensitive ? 'g' : 'gi'; - const body = req.regex ? req.pattern : escapeRegExp(req.pattern); - return new RegExp(body, flags); -} - -function escapeRegExp(s: string): string { - return s.replaceAll(/[.*+?^${}()|[\]\\]/g, '\\$&'); -} - -export async function matchFiles(fs: IHostFileSystem, paths: readonly FileMatchPath[], options: FileMatchOptions, timeoutMs: number, signal?: AbortSignal, maxResponseBytes = Infinity): Promise { - const re = compileGrepPattern(options); - const files: FileMatches['files'] = []; - let filesScanned = 0; - let totalMatches = 0; - let truncated = false; - const deadline = Date.now() + timeoutMs; - let responseBytes = 0; - for (const file of paths) { - if ((signal?.aborted === true || Date.now() >= deadline) && filesScanned > 0) { - truncated = true; - break; - } - if (filesScanned >= options.max_files) { - truncated = true; - break; - } - filesScanned += 1; - let content: string; - try { - content = await fs.readText(file.absolute); - } catch { - continue; - } - const lines = content.split(/\r?\n/); - const matches: FileMatch[] = []; - for (let i = 0; i < lines.length; i++) { - const line = lines[i]!; - re.lastIndex = 0; - const m = re.exec(line); - if (m === null) continue; - if (matches.length >= options.max_matches_per_file) break; - const before: string[] = []; - for (let k = Math.max(0, i - options.context_lines); k < i; k++) { - before.push(lines[k] ?? ''); - } - const after: string[] = []; - for (let k = i + 1; k < Math.min(lines.length, i + 1 + options.context_lines); k++) { - after.push(lines[k] ?? ''); - } - matches.push({ line: i + 1, col: m.index + 1, text: line, before, after }); - totalMatches += 1; - if (totalMatches >= options.max_total_matches) { - truncated = true; - break; - } - } - if (matches.length > 0) { - responseBytes += Buffer.byteLength(JSON.stringify({ path: file.path, matches })); - if (responseBytes > maxResponseBytes) { - return { files: [], files_scanned: 0, truncated: false, responseTooLarge: true }; - } - files.push({ path: file.path, matches }); - } - if (totalMatches >= options.max_total_matches) break; - } - - return { files, files_scanned: filesScanned, truncated }; -} diff --git a/packages/agent-core-v2/src/os/operations/readTextPage.ts b/packages/agent-core-v2/src/os/operations/readTextPage.ts deleted file mode 100644 index ea59f8e9e48..00000000000 --- a/packages/agent-core-v2/src/os/operations/readTextPage.ts +++ /dev/null @@ -1,481 +0,0 @@ -import type { IHostFileSystem } from '#/os/interface/hostFileSystem'; -import type { FileReadSource, TextPageInput, TextPageOptions, TextFileResult } from '#/os/interface/fileOperations'; -import { isHostFsNotDirectory, isHostFsNotFound } from '#/os/interface/hostFsErrors'; -import { unwrapErrorCause } from '#/_base/errors/errors'; -import { MEDIA_SNIFF_BYTES, detectFileType } from '#/_base/media/fileType'; -import { makeCarriageReturnsVisible, splitLinesKeepingTerminator, type LineEndingStyle } from '#/_base/text/line-endings'; -import { detectTextEncoding, type UtfTextEncoding } from '#/_base/text/encoding'; -import { renderTextStatus } from '#/_base/text/resultStatus'; - -export const TRANSCODE_MAX_BYTES = 10 * 1024 * 1024; - -interface LineEndingFlags { - hasCrLf: boolean; - hasLf: boolean; - hasLoneCr: boolean; -} - -interface ReadLineEntry { - readonly lineNo: number; - readonly rawContent: string; -} - -interface ReadTailEntry extends ReadLineEntry { - readonly minChars: number; -} - -interface ReadRequest { - readonly args: TextPageInput; - readonly maxChars: number; - readonly maxCharsLimit: number; - readonly detectedEncoding?: UtfTextEncoding; - readonly lossyDecoding: boolean; - readonly eventLog: boolean; - readonly signal?: AbortSignal; -} - -interface ReadPage { - readonly request: ReadRequest; - readonly renderedLines: readonly string[]; - readonly startLine: number; - readonly rangeStart: number; - readonly rangeEnd: number; - readonly totalLines: number; - readonly fromTail: boolean; - readonly lineEndingStyle: LineEndingStyle; -} - -function stripTrailingLf(line: string): string { - return line.endsWith('\n') ? line.slice(0, -1) : line; -} - -function splitsSurrogatePair(text: string, offset: number): boolean { - // oxlint-disable-next-line unicorn/prefer-code-point -- raw UTF-16 halves are the point here - const previous = text.charCodeAt(offset - 1); - // oxlint-disable-next-line unicorn/prefer-code-point -- raw UTF-16 halves are the point here - const next = text.charCodeAt(offset); - return previous >= 0xd800 && previous <= 0xdbff && next >= 0xdc00 && next <= 0xdfff; -} - -function updateLineEndingFlags(flags: LineEndingFlags, text: string): void { - for (let i = 0; i < text.length; i += 1) { - const code = text.codePointAt(i); - if (code === 13) { - if (text.codePointAt(i + 1) === 10) { - flags.hasCrLf = true; - i += 1; - } else { - flags.hasLoneCr = true; - } - } else if (code === 10) { - flags.hasLf = true; - } - } -} - -function lineEndingStyleFromFlags(flags: LineEndingFlags): LineEndingStyle { - if (flags.hasLoneCr || (flags.hasCrLf && flags.hasLf)) return 'mixed'; - if (flags.hasCrLf) return 'crlf'; - return 'lf'; -} - -function renderLine(entry: ReadLineEntry, lineEndingStyle: LineEndingStyle): string { - const modelContent = - lineEndingStyle === 'crlf' && entry.rawContent.endsWith('\r') - ? entry.rawContent.slice(0, -1) - : entry.rawContent; - const renderedContent = - lineEndingStyle === 'mixed' ? makeCarriageReturnsVisible(modelContent) : modelContent; - return `${String(entry.lineNo)}\t${renderedContent}`; -} - -function isTextDecodeError(error: unknown): boolean { - const unwrapped = unwrapErrorCause(error); - if (typeof unwrapped !== 'object' || unwrapped === null) return false; - const code = (unwrapped as { code?: unknown })['code']; - if (code === 'ERR_ENCODING_INVALID_ENCODED_DATA') return true; - if (!(unwrapped instanceof Error)) return false; - return /encoded data was not valid|invalid.*encoding|invalid.*utf-?8/i.test(unwrapped.message); -} - -function containsNulByte(text: string): boolean { - return text.includes('\u0000'); -} - -function encodingDisplayName(encoding: UtfTextEncoding): string { - switch (encoding) { - case 'utf-16le': - return 'UTF-16 LE'; - case 'utf-16be': - return 'UTF-16 BE'; - case 'utf-8': - return 'UTF-8'; - } -} - -async function* decodedLines(lines: readonly string[]): AsyncGenerator { - yield* lines; -} - -function notReadableFileOutput(path: string): string { - return `"${path}" is not readable as UTF-8 text. Only text files can be read.`; -} - -function notUtf8DecodableFileOutput(path: string): string { - return ( - `"${path}" is not valid UTF-8 or UTF-16 text. ` + - 'Only UTF-8 and UTF-16 text files can be read; ' + - 'for other encodings (e.g. GBK), convert the file to UTF-8 first (e.g. with `iconv`).' - ); -} - -export async function readTextPage( - source: FileReadSource, - options: TextPageOptions, - signal?: AbortSignal, -): Promise { - const { args, eventLog } = options; - try { - signal?.throwIfAborted(); - let stat: Awaited>; - try { - stat = await source.stat(); - } catch (error) { - if (isHostFsNotFound(error) || isHostFsNotDirectory(error)) { - return { isError: true, output: `"${args.path}" does not exist.` }; - } - throw error; - } - if (!stat.isFile) { - return { isError: true, output: `"${args.path}" is not a file.` }; - } - - const header = await source.readBytes(MEDIA_SNIFF_BYTES); - const fileType = detectFileType(source.name, header); - if (fileType.kind === 'image' || fileType.kind === 'video') { - const kind = fileType.kind; - const article = kind === 'image' ? 'an' : 'a'; - const supported = kind === 'image' ? options.imageSupported : options.videoSupported; - if (!supported) { - return { - isError: true, - output: `"${args.path}" is ${article} ${kind} file. The current model does not support ${kind} input (missing ${kind}_in capability), so this agent cannot view it. Only text files can be read.`, - }; - } - const mediaToolAvailable = options.mediaToolAvailable; - return { - isError: true, - output: mediaToolAvailable - ? `"${args.path}" is ${article} ${kind} file. Only text files can be read; use ReadMediaFile for image and video files.` - : `"${args.path}" is ${article} ${kind} file. Only text files can be read.`, - }; - } - - const detection = detectTextEncoding(header); - let readLines: () => AsyncIterable; - let detectedEncoding: UtfTextEncoding | undefined; - let lossyDecoding = false; - if (!detection.seemsBinary && detection.encoding !== 'utf-8') { - if (stat.size > TRANSCODE_MAX_BYTES) { - return { - isError: true, - output: - `"${args.path}" is ${encodingDisplayName(detection.encoding)} text but too large to transcode ` + - `(${String(stat.size)} bytes > ${String(TRANSCODE_MAX_BYTES)}). ` + - 'Convert it to UTF-8 first (e.g. with `iconv`).', - }; - } - const bytes = await source.readBytes(); - let decoded: string; - try { - decoded = new TextDecoder(detection.encoding, { fatal: true }).decode(bytes); - } catch (error) { - if (!isTextDecodeError(error)) throw error; - decoded = new TextDecoder(detection.encoding, { fatal: false }).decode(bytes); - lossyDecoding = true; - } - detectedEncoding = detection.encoding; - const decodedContent = splitLinesKeepingTerminator(decoded); - readLines = () => decodedLines(decodedContent); - } else if (fileType.kind === 'unknown') { - return { - isError: true, - output: notReadableFileOutput(args.path), - }; - } else { - readLines = () => source.readLines(); - } - - const request: ReadRequest = { - args, - maxChars: options.maxChars, - maxCharsLimit: options.maxCharsLimit, - detectedEncoding, - lossyDecoding, - eventLog, - signal, - }; - const lineOffset = args.line_offset ?? 1; - if (lineOffset >= 0) return await readForward(readLines(), request); - const rereadsFile = detectedEncoding === undefined && (args.n_lines ?? Infinity) < -lineOffset; - const result = await readTail(readLines, request); - if (!result.isError && rereadsFile) { - const currentStat = await source.stat(); - if (!currentStat.isFile || currentStat.size !== stat.size || - currentStat.mtimeMs !== stat.mtimeMs || currentStat.ino !== stat.ino) { - return { isError: true, output: 'File changed while reading its tail. Retry Read with the updated file.' }; - } - } - return result; - } catch (error) { - if (isTextDecodeError(error)) { - return { isError: true, output: notUtf8DecodableFileOutput(args.path) }; - } - return { - isError: true, - output: error instanceof Error ? error.message : String(error), - }; - } -} - -async function readForward( - lines: AsyncIterable, - request: ReadRequest, -): Promise { - const { args, maxChars } = request; - const lineOffset = args.line_offset ?? 1; - const columnOffset = args.column_offset ?? 0; - const requestedLines = args.n_lines ?? Infinity; - const selectedEntries: ReadLineEntry[] = []; - const flags: LineEndingFlags = { hasCrLf: false, hasLf: false, hasLoneCr: false }; - let currentLineNo = 0; - let collectionClosed = false; - let minimumChars = 0; - - for await (const rawLine of lines) { - request.signal?.throwIfAborted(); - if (containsNulByte(rawLine)) { - return { isError: true, output: notReadableFileOutput(args.path) }; - } - currentLineNo += 1; - updateLineEndingFlags(flags, rawLine); - if (collectionClosed) continue; - if (currentLineNo < lineOffset) continue; - if (selectedEntries.length >= requestedLines) { - collectionClosed = true; - continue; - } - const rawContent = stripTrailingLf(rawLine); - const lineChars = String(currentLineNo).length + 1 + Math.max( - 0, - rawContent.length - (rawContent.endsWith('\r') ? 1 : 0) - - (currentLineNo === lineOffset ? columnOffset : 0), - ) + (selectedEntries.length === 0 ? 0 : 1); - if (minimumChars + lineChars > maxChars && selectedEntries.length > 0) { - collectionClosed = true; - continue; - } - selectedEntries.push({ - lineNo: currentLineNo, - rawContent, - }); - minimumChars += lineChars; - if (selectedEntries.length >= requestedLines || minimumChars >= maxChars) { - collectionClosed = true; - } - } - - const lineEndingStyle = lineEndingStyleFromFlags(flags); - const renderedLines = selectedEntries.map((entry) => renderLine(entry, lineEndingStyle)); - const firstLine = renderedLines[0]; - if (columnOffset > 0) { - const prefix = `${String(lineOffset)}\t`; - const text = firstLine?.slice(prefix.length); - if (text === undefined || columnOffset > text.length) { - return { isError: true, output: `column_offset=${String(columnOffset)} is past the end of the starting line ${String(lineOffset)}. Read the line from column 0 to inspect its current contents.` }; - } - if (splitsSurrogatePair(text, columnOffset)) { - return { isError: true, output: `column_offset=${String(columnOffset)} splits a Unicode character in line ${String(lineOffset)}. Use a character boundary or the Next Read arguments.` }; - } - renderedLines[0] = prefix + text.slice(columnOffset); - } - return finishPage({ - request, - renderedLines, - startLine: lineOffset, - rangeStart: lineOffset, - rangeEnd: Math.min(currentLineNo, lineOffset + requestedLines - 1), - totalLines: currentLineNo, - fromTail: false, - lineEndingStyle, - }); -} - -function finishPage(page: ReadPage): TextFileResult { - const { args, maxChars, maxCharsLimit, eventLog, detectedEncoding, lossyDecoding } = page.request; - let first = 0; - let end = page.renderedLines.length; - let contentChars = page.renderedLines.reduce((sum, line) => sum + line.length + 1, -1); - const firstColumn = page.fromTail ? 0 : args.column_offset ?? 0; - const firstPrefix = `${String(page.startLine)}\t`; - const firstText = page.renderedLines[0]?.slice(firstPrefix.length) ?? ''; - let fragmentEnd: number | undefined; - - while (true) { - const count = end - first; - const startLine = page.startLine + first; - const endLine = page.startLine + end - 1; - const lineIncomplete = fragmentEnd !== undefined; - const complete = page.rangeStart > page.rangeEnd || - (count > 0 && startLine === page.rangeStart && endLine === page.rangeEnd && !lineIncomplete); - const parts = [ - count > 0 - ? `${String(count)} ${count === 1 ? 'line' : 'lines'} read from file starting from line ${String(startLine)}.` - : 'No lines read from file.', - `Total lines in file: ${String(page.totalLines)}.`, - complete ? 'Requested range complete.' : 'Character limit reached.', - `Effective max_chars: ${String(maxChars)}.`, - ]; - if (!lineIncomplete && ((count > 0 && endLine === page.totalLines) || page.rangeStart > page.totalLines)) { - parts.push('End of file reached.'); - } - if (count > 0 && !page.fromTail && (firstColumn > 0 || fragmentEnd !== undefined)) { - parts.push(`Line ${String(startLine)} fragment: columns [${String(firstColumn)}, ${String(firstColumn + (fragmentEnd ?? firstText.length))}) of ${String(firstColumn + firstText.length)}. ${lineIncomplete ? 'Line continues.' : 'Line complete.'}`); - } - if (args.max_chars !== undefined && args.max_chars > maxCharsLimit) { - parts.push(`Requested max_chars=${String(args.max_chars)} was capped at the configured maximum ${String(maxCharsLimit)}.`); - } - if (!complete && (count > 0 || page.fromTail)) { - const nextStart = page.fromTail ? page.rangeStart : lineIncomplete ? startLine : endLine + 1; - const nextEnd = page.fromTail && count > 0 ? startLine - 1 : page.rangeEnd; - const next = { - path: args.path, - line_offset: nextStart, - column_offset: fragmentEnd !== undefined ? firstColumn + fragmentEnd : undefined, - n_lines: page.fromTail || args.n_lines !== undefined ? nextEnd - nextStart + 1 : undefined, - max_chars: maxChars, - }; - parts.push(`Next Read: ${JSON.stringify(next)}`); - } - if (eventLog) { - parts.push('Kimi Code agent event log: read one record at a time (n_lines=1); increase max_chars for a longer record or extract fields with Bash.'); - } - if (page.lineEndingStyle === 'mixed') { - parts.push('Mixed or lone carriage-return line endings are shown as \\r. Use exact \\r\\n or \\r escapes in Edit.old_string for those lines.'); - } - if (detectedEncoding !== undefined) { - parts.push(`Detected file encoding: ${encodingDisplayName(detectedEncoding)}; content transcoded to UTF-8 for display. Edit and Write expect UTF-8 — convert the file's encoding first (e.g. \`iconv\` via Bash).`); - } - if (lossyDecoding) { - parts.push('Lossy UTF-16 decoding: malformed sequences were replaced with U+FFFD. The decoded text may differ from the original file.'); - } - const note = `${parts.join(' ')}`; - const renderedChars = count === 0 - ? renderTextStatus('').length + 1 + note.length - : contentChars + 1 + note.length; - if (renderedChars <= maxChars && (complete || count > 0)) { - return { - output: fragmentEnd !== undefined - ? firstPrefix + firstText.slice(0, fragmentEnd) - : page.renderedLines.slice(first, end).join('\n'), - note, - truncated: complete ? undefined : true, - }; - } - if (count === 1 && !page.fromTail) { - const previousEnd = fragmentEnd ?? firstText.length; - fragmentEnd = Math.min(previousEnd - 1, previousEnd - (renderedChars - maxChars)); - if (splitsSurrogatePair(firstText, fragmentEnd)) fragmentEnd -= 1; - if (fragmentEnd <= 0) { - return { isError: true, output: `max_chars=${String(maxChars)} is too small for file text and the Read status. Increase max_chars.` }; - } - contentChars = firstPrefix.length + fragmentEnd; - continue; - } - if (count === 0) { - if (!complete) { - const recovery: TextFileResult = { - isError: true, - output: 'No complete line fits. Continue with the forward Next Read.', - note, - truncated: true, - }; - const recoveryChars = renderTextStatus(recovery.output, true).length + 1 + note.length; - if (recoveryChars <= maxChars) return recovery; - } - return { isError: true, output: `max_chars=${String(maxChars)} is too small for the Read status. Increase max_chars.` }; - } - const dropped = page.fromTail ? first++ : --end; - contentChars -= page.renderedLines[dropped]!.length + 1; - } -} - -async function readTail( - readLines: () => AsyncIterable, - request: ReadRequest, -): Promise { - const { args, maxChars } = request; - const lineOffset = args.line_offset ?? 1; - const requestedLines = args.n_lines ?? Infinity; - const tailCount = -lineOffset; - const singlePass = requestedLines >= tailCount; - let entries: (ReadTailEntry | undefined)[] = []; - let first = 0; - let chars = 0; - const retainLine = (rawLine: string, lineNo: number): void => { - const rawContent = stripTrailingLf(rawLine); - const minChars = String(lineNo).length + 2 + rawContent.length - (rawContent.endsWith('\r') ? 1 : 0); - entries.push({ lineNo, rawContent, minChars }); - chars += minChars; - while (first < entries.length && (chars - 1 > maxChars || entries.length - first > tailCount)) { - chars -= entries[first]!.minChars; - entries[first++] = undefined; - } - if (first > 1024 && first >= entries.length / 2) { - entries = entries.slice(first); - first = 0; - } - }; - const flags: LineEndingFlags = { hasCrLf: false, hasLf: false, hasLoneCr: false }; - let totalLines = 0; - for await (const rawLine of readLines()) { - request.signal?.throwIfAborted(); - if (containsNulByte(rawLine)) { - return { isError: true, output: notReadableFileOutput(args.path) }; - } - totalLines += 1; - updateLineEndingFlags(flags, rawLine); - if (singlePass) retainLine(rawLine, totalLines); - } - - const rangeStart = Math.max(1, totalLines + lineOffset + 1); - const rangeEnd = Math.min(totalLines, rangeStart + requestedLines - 1); - const lineEndingStyle = lineEndingStyleFromFlags(flags); - if (!singlePass) { - let currentLine = 0; - for await (const rawLine of readLines()) { - request.signal?.throwIfAborted(); - currentLine += 1; - if (currentLine > rangeEnd) break; - if (currentLine < rangeStart) continue; - if (containsNulByte(rawLine)) { - return { isError: true, output: notReadableFileOutput(args.path) }; - } - retainLine(rawLine, currentLine); - } - if (currentLine < rangeEnd || currentLine > totalLines) { - return { isError: true, output: 'File changed while reading its tail. Retry Read with the updated file.' }; - } - } - const selected = entries.slice(first).map((entry) => renderLine(entry!, lineEndingStyle)); - return finishPage({ - request, - renderedLines: selected, - startLine: rangeEnd - selected.length + 1, - rangeStart, - rangeEnd, - totalLines, - fromTail: true, - lineEndingStyle, - }); -} diff --git a/packages/agent-core-v2/src/os/operations/runRg.ts b/packages/agent-core-v2/src/os/operations/runRg.ts deleted file mode 100644 index bf48db4c448..00000000000 --- a/packages/agent-core-v2/src/os/operations/runRg.ts +++ /dev/null @@ -1,202 +0,0 @@ -import type { Readable } from 'node:stream'; - -import { BugIndicatingError } from '#/_base/errors/errors'; -import type { IHostProcess, IHostProcessService } from '#/os/interface/hostProcess'; - -export const DEFAULT_TIMEOUT_MS = 20_000; -export const SIGTERM_GRACE_MS = 5_000; -export const MAX_OUTPUT_BYTES = 10 * 1024 * 1024; - -export interface RunRgResult { - readonly kind: 'result'; - readonly exitCode: number; - readonly stdoutText: string; - readonly stderrText: string; - readonly bufferTruncated: boolean; - readonly stderrTruncated: boolean; - readonly timedOut: boolean; - readonly outputLimitReached?: boolean; -} - -export type RunRgOutcome = RunRgResult | { readonly kind: 'aborted' }; - -function disposeProcess(proc: IHostProcess): void { - try { - void proc.dispose(); - } catch { - } -} - -export async function runRgOnce( - processService: IHostProcessService, - rgArgs: readonly string[], - signal: AbortSignal, - options?: { readonly cwd?: string }, -): Promise { - if (signal.aborted) { - return { kind: 'aborted' }; - } - - const [command, ...args] = rgArgs; - if (command === undefined) { - throw new BugIndicatingError('runRgOnce: rgArgs must not be empty'); - } - const proc: IHostProcess = await processService.spawn(command, args, { cwd: options?.cwd }); - - try { - proc.stdin.end(); - } catch { - } - - let timedOut = false; - let aborted = false; - let killed = false; - let outputLimitReached = false; - - const killProc = async (): Promise => { - if (killed) return; - killed = true; - try { - await proc.kill('SIGTERM'); - } catch { - } - const exited = proc - .wait() - .then(() => true) - .catch(() => true); - let graceTimer: ReturnType | undefined; - const raced = await Promise.race([ - exited, - new Promise((resolve) => { - graceTimer = setTimeout(() => { - resolve(false); - }, SIGTERM_GRACE_MS); - }), - ]); - clearTimeout(graceTimer); - if (!raced && proc.exitCode === null) { - try { - await proc.kill('SIGKILL'); - } catch { - } - } - disposeProcess(proc); - }; - - const onAbort = (): void => { - aborted = true; - void killProc(); - }; - signal.addEventListener('abort', onAbort); - if (signal.aborted) onAbort(); - - const timeoutHandle = setTimeout(() => { - timedOut = true; - void killProc(); - }, DEFAULT_TIMEOUT_MS); - - let exitCode = 0; - let stdoutText = ''; - let stderrText = ''; - let bufferTruncated = false; - let stderrTruncated = false; - - try { - const isTerminating = (): boolean => timedOut || aborted || killed; - const [stdoutResult, stderrResult, code] = await Promise.all([ - readStreamWithCap(proc.stdout, MAX_OUTPUT_BYTES, isTerminating, () => { - outputLimitReached = true; - void killProc(); - }), - readStreamWithCap(proc.stderr, MAX_OUTPUT_BYTES, isTerminating), - proc.wait(), - ]); - stdoutText = stdoutResult.text; - stderrText = stderrResult.text; - bufferTruncated = stdoutResult.truncated; - stderrTruncated = stderrResult.truncated; - exitCode = code; - } catch (error) { - if (!(isPrematureCloseError(error) && (timedOut || aborted || killed))) { - throw error; - } - } finally { - clearTimeout(timeoutHandle); - signal.removeEventListener('abort', onAbort); - disposeProcess(proc); - } - - if (aborted) { - return { kind: 'aborted' }; - } - - return { - kind: 'result', - exitCode, - stdoutText, - stderrText, - bufferTruncated, - stderrTruncated, - timedOut, - outputLimitReached, - }; -} - -export function shouldRetryRipgrepEagain(result: RunRgResult): boolean { - return ( - result.exitCode !== 0 && - result.exitCode !== 1 && - !result.timedOut && - !result.outputLimitReached && - isEagainRipgrepError(result.stderrText) - ); -} - -function isEagainRipgrepError(stderr: string): boolean { - return stderr.includes('os error 11') || stderr.includes('Resource temporarily unavailable'); -} - -function isPrematureCloseError(error: unknown): boolean { - return ( - error instanceof Error && - (error as NodeJS.ErrnoException).code === 'ERR_STREAM_PREMATURE_CLOSE' - ); -} - -interface CappedStreamResult { - readonly text: string; - readonly truncated: boolean; -} - -async function readStreamWithCap( - stream: Readable, - maxBytes: number, - suppressPrematureClose?: () => boolean, - onLimit?: () => void, -): Promise { - const chunks: Buffer[] = []; - let total = 0; - let truncated = false; - try { - for await (const chunk of stream) { - const buf: Buffer = - typeof chunk === 'string' ? Buffer.from(chunk, 'utf8') : (chunk as Buffer); - if (truncated) continue; - if (total + buf.length > maxBytes) { - const remaining = maxBytes - total; - if (remaining > 0) chunks.push(buf.subarray(0, remaining)); - total = maxBytes; - truncated = true; - onLimit?.(); - continue; - } - chunks.push(buf); - total += buf.length; - } - } catch (error) { - if (!isPrematureCloseError(error) || suppressPrematureClose?.() !== true) { - throw error; - } - } - return { text: Buffer.concat(chunks).toString('utf8'), truncated }; -} diff --git a/packages/agent-core-v2/src/os/operations/searchGlob.ts b/packages/agent-core-v2/src/os/operations/searchGlob.ts deleted file mode 100644 index 3d914b2b3b6..00000000000 --- a/packages/agent-core-v2/src/os/operations/searchGlob.ts +++ /dev/null @@ -1,208 +0,0 @@ -import { normalize, resolve } from 'pathe'; -import type { IHostProcessService } from '#/os/interface/hostProcess'; -import type { SearchRequest, SearchResult, GlobSearchInput as GlobInput } from '#/os/interface/searchOperations'; -import { isSensitiveFile, SENSITIVE_DOT_VARIANT_SUFFIXES, isWithinDirectory } from '#/_base/fs/pathFilters'; -import { DEFAULT_TOOL_RESULT_MAX_RETAINED_CHARS } from '#/_base/text/outputLimits'; -import { DEFAULT_TIMEOUT_MS, MAX_OUTPUT_BYTES, runRgOnce, shouldRetryRipgrepEagain } from './runRg'; - -type PathClass = 'posix' | 'win32'; -const DEFAULT_HEAD_LIMIT = 100; - -const VCS_DIRECTORIES_TO_EXCLUDE = ['.git', '.svn', '.hg', '.bzr', '.jj', '.sl'] as const; - -const SENSITIVE_KEY_BASENAMES = ['id_rsa', 'id_ed25519', 'id_ecdsa'] as const; -const SENSITIVE_GLOBS_TO_EXCLUDE: readonly string[] = [ - '**/.env', - ...SENSITIVE_KEY_BASENAMES.flatMap((name) => [ - `**/${name}`, - `**/${name}[-_]*`, - ...SENSITIVE_DOT_VARIANT_SUFFIXES.map((suffix) => `**/${name}${suffix}`), - ]), - '**/.aws/credentials', - '**/.aws/credentials/**', - '**/.gcp/credentials', - '**/.gcp/credentials/**', -]; - -export async function searchGlob(processService: IHostProcessService, request: SearchRequest, signal: AbortSignal): Promise { - const { args, rgPath, roots: searchPaths, workspaceDir, pathClass } = request; - const searchRoot = searchPaths[0] ?? workspaceDir; - let run; - run = await runRgOnce(processService, buildRgArgs(rgPath, args), signal, { cwd: searchRoot }); - if (run.kind === 'aborted') { - return { isError: true, output: 'Glob aborted' }; - } - - if (shouldRetryRipgrepEagain(run)) { - run = await runRgOnce(processService, buildRgArgs(rgPath, args, true), signal, { cwd: searchRoot }); - if (run.kind === 'aborted') { - return { isError: true, output: 'Glob aborted' }; - } - } - - const { exitCode, stdoutText, stderrText, bufferTruncated, timedOut } = run; - - let traversalWarning: string | undefined; - if (exitCode !== 0 && exitCode !== 1 && !timedOut && !run.outputLimitReached) { - const rawPathsBeforeError = splitCompletePaths(stdoutText, true); - if (rawPathsBeforeError.length === 0) { - return { isError: true, output: formatGlobError(searchRoot, stderrText) }; - } - traversalWarning = formatGlobWarning(stderrText); - } - if (signal.aborted) { - return { isError: true, output: 'Glob aborted' }; - } - - const rawPaths = splitCompletePaths(stdoutText, bufferTruncated || timedOut).map((p) => - resolve(searchRoot, p), - ); - - const kept: string[] = []; - let filteredSensitive = 0; - for (const p of rawPaths) { - if (isSensitiveFile(p)) { - filteredSensitive++; - } else { - kept.push(p); - } - } - - const offset = args.offset ?? 0; - const headLimit = args.head_limit ?? DEFAULT_HEAD_LIMIT; - const limited = headLimit === 0 ? kept.slice(offset) : kept.slice(offset, offset + headLimit); - const partial = bufferTruncated || timedOut || traversalWarning !== undefined; - - const shouldRelativize = isWithinDirectory(searchRoot, workspaceDir, pathClass); - const candidates = limited.map((p) => - shouldRelativize ? relativizeIfUnder(p, searchRoot, pathClass) : p, - ); - - const warnings: string[] = []; - if (timedOut) { - warnings.push( - `Glob timed out after ${String(DEFAULT_TIMEOUT_MS / 1000)}s; partial results returned.`, - ); - } - if (bufferTruncated) { - warnings.push( - `[stdout truncated at ${String(MAX_OUTPUT_BYTES)} bytes; results may be incomplete — use a more specific pattern]`, - ); - } - if (traversalWarning !== undefined) { - warnings.push(traversalWarning); - } - const pageNotices = (count: number, characterLimited: boolean) => { - const lines = [...warnings]; - const footer: string[] = []; - const truncated = characterLimited || offset + count < kept.length; - if (count === 0) { - if (kept.length > 0) { - const resultSet = partial ? 'collected partial result set' : 'current result set'; - lines.push( - `No more matches at offset=${String(offset)} in the ${resultSet} (${String(kept.length)} matches).`, - ); - } else if (partial) { - lines.push('No matches collected; search incomplete.'); - } else if (filteredSensitive > 0) { - lines.push( - `No non-sensitive matches found (${String(filteredSensitive)} sensitive file(s) filtered).`, - ); - } else { - lines.push('No matches found'); - } - } else if (truncated || offset > 0 || partial) { - const total = partial - ? `${String(kept.length)} collected matches (partial result set)` - : String(kept.length); - lines.push(`Showing matches ${String(offset + 1)}–${String(offset + count)} of ${total}.`); - } - if (characterLimited) lines.push('Character limit reached; only complete paths are returned.'); - if (truncated) { - lines.push( - `Continue with the same search arguments and offset=${String(offset + count)}.`, - ); - if (!characterLimited) lines.push('To remove the match-count limit, omit offset and use head_limit=0.'); - } - if (filteredSensitive > 0 && (kept.length > 0 || partial)) { - footer.push(`Filtered ${String(filteredSensitive)} sensitive file(s).`); - } - if (!truncated && !partial && offset === 0 && headLimit > 0 && count === headLimit) { - footer.push(`Found ${String(count)} matches`); - } - return { lines, footer }; - }; - const noticeChars = Math.max(...[false, true].map((characterLimited) => { - const { lines, footer } = pageNotices(candidates.length, characterLimited); - return [...lines, ...footer].join('\n').length + 2; - })); - let remaining = DEFAULT_TOOL_RESULT_MAX_RETAINED_CHARS - noticeChars; - const displayLines: string[] = []; - for (const path of candidates) { - if (path.length + 1 > remaining) break; - displayLines.push(path); - remaining -= path.length + 1; - } - if (candidates.length > 0 && displayLines.length === 0) { - return { - isError: true, - output: 'Glob cannot fit a complete path and its diagnostics within the output limit. Narrow the search path or pattern.', - }; - } - const notices = pageNotices(displayLines.length, displayLines.length < candidates.length); - return { output: [...notices.lines, ...displayLines, ...notices.footer].join('\n') }; - -} - -function buildRgArgs(rgPath: string, args: GlobInput, singleThreaded = false): string[] { - const cmd: string[] = [rgPath]; - if (singleThreaded) cmd.push('-j', '1'); - cmd.push('--files', '--hidden', '--sortr=modified'); - for (const dir of VCS_DIRECTORIES_TO_EXCLUDE) { - cmd.push('--glob', `!${dir}`); - } - cmd.push('--glob', args.pattern); - for (const glob of SENSITIVE_GLOBS_TO_EXCLUDE) { - cmd.push('--glob', `!${glob}`); - } - if (args.include_ignored) cmd.push('--no-ignore'); - cmd.push('.'); - return cmd; -} - -function formatGlobError(searchRoot: string, stderr: string): string { - const trimmed = stderr.trim(); - if (/no such file or directory/i.test(trimmed)) { - return `${searchRoot} does not exist`; - } - return trimmed.length > 0 ? `Glob failed: ${trimmed}` : 'Glob failed'; -} - -function formatGlobWarning(stderr: string): string { - const trimmed = stderr.trim(); - return trimmed.length > 0 - ? `Glob completed with warnings; some directories could not be read: ${trimmed}` - : 'Glob completed with warnings; some directories could not be read.'; -} - -export function splitCompletePaths(stdoutText: string, truncatedOutput: boolean): string[] { - let text = stdoutText; - if (truncatedOutput && !text.endsWith('\n')) { - const lastNewline = text.lastIndexOf('\n'); - text = lastNewline >= 0 ? text.slice(0, lastNewline + 1) : ''; - } - return text.split('\n').filter((p) => p.length > 0); -} - -function relativizeIfUnder(candidate: string, base: string, pathClass: PathClass): string { - const normCandidate = normalize(candidate); - const normBase = normalize(base); - const comparableCandidate = pathClass === 'win32' ? normCandidate.toLowerCase() : normCandidate; - const comparableBase = pathClass === 'win32' ? normBase.toLowerCase() : normBase; - if (comparableCandidate === comparableBase) return '.'; - const prefix = comparableBase.endsWith('/') ? comparableBase : comparableBase + '/'; - if (comparableCandidate.startsWith(prefix)) { - return normCandidate.slice(prefix.length); - } - return normCandidate; -} diff --git a/packages/agent-core-v2/src/os/operations/searchGrep.ts b/packages/agent-core-v2/src/os/operations/searchGrep.ts deleted file mode 100644 index f78f28904b8..00000000000 --- a/packages/agent-core-v2/src/os/operations/searchGrep.ts +++ /dev/null @@ -1,502 +0,0 @@ -import { normalize } from 'pathe'; -import type { IHostProcessService } from '#/os/interface/hostProcess'; -import type { SearchRequest, SearchResult, GrepSearchInput as GrepInput } from '#/os/interface/searchOperations'; -import { isSensitiveFile, SENSITIVE_DOT_VARIANT_SUFFIXES } from '#/_base/fs/pathFilters'; -import { ToolOutputAccumulator } from '#/_base/text/outputAccumulator'; -import { DEFAULT_TIMEOUT_MS, MAX_OUTPUT_BYTES, runRgOnce, shouldRetryRipgrepEagain, type RunRgResult } from './runRg'; - -type PathClass = 'posix' | 'win32'; - -const RG_MAX_COLUMNS = 500; -const DEFAULT_HEAD_LIMIT = 250; - -const VCS_DIRECTORIES_TO_EXCLUDE = ['.git', '.svn', '.hg', '.bzr', '.jj', '.sl'] as const; -const SENSITIVE_KEY_BASENAMES = ['id_rsa', 'id_ed25519', 'id_ecdsa'] as const; -const SENSITIVE_KEY_GLOBS_TO_EXCLUDE = SENSITIVE_KEY_BASENAMES.flatMap((name) => [ - `**/${name}`, - `**/${name}[-_]*`, - ...SENSITIVE_DOT_VARIANT_SUFFIXES.map((suffix) => `**/${name}${suffix}`), -]); -const SENSITIVE_GLOBS_TO_EXCLUDE = [ - '**/.env', - ...SENSITIVE_KEY_GLOBS_TO_EXCLUDE, - '**/.aws/credentials', - '**/.aws/credentials/**', - '**/.gcp/credentials', - '**/.gcp/credentials/**', -] as const; - -const CONTENT_LINE_RE = /^(.*?)([:-])(\d+)\2/; - -export async function searchGrep(processService: IHostProcessService, request: SearchRequest, signal: AbortSignal): Promise { - const { args, rgPath, roots: searchPaths, workspaceDir, pathClass } = request; - let runResult: RunRgResult; - const firstRun = await runRgOnce( - processService, - buildRgArgs(rgPath, args, searchPaths), - signal, - ); - if (firstRun.kind === 'aborted') { - return { isError: true, output: 'Grep aborted' }; - } - runResult = firstRun; - - if (shouldRetryRipgrepEagain(runResult)) { - const retryRun = await runRgOnce( - processService, - buildRgArgs(rgPath, args, searchPaths, true), - signal, - ); - if (retryRun.kind === 'aborted') { - return { isError: true, output: 'Grep aborted' }; - } - runResult = retryRun; - } - - - const { exitCode, stderrText, bufferTruncated, stderrTruncated, timedOut } = runResult; - let { stdoutText } = runResult; - - if (exitCode !== 0 && exitCode !== 1 && !timedOut && !runResult.outputLimitReached) { - return { - isError: true, - output: formatRipgrepError(exitCode, stderrText, stderrTruncated), - }; - } - - const mode = args.output_mode ?? 'files_with_matches'; - if (bufferTruncated || timedOut) { - stdoutText = omitIncompleteTrailingRecord(stdoutText, mode); - } - if (timedOut && stdoutText.trim() === '') { - return { - isError: true, - output: `Grep timed out after ${String(DEFAULT_TIMEOUT_MS / 1000)}s. Try a more specific path or pattern.`, - }; - } - if (signal.aborted) { - return { isError: true, output: 'Grep aborted' }; - } - - const rawLines = parseRipgrepOutput(stdoutText, mode); - - const filteredSensitive = new Set(); - const keptLines = filterSensitiveLines(rawLines, mode, filteredSensitive, pathClass); - const orderedLines = keptLines; - - const offset = args.offset ?? 0; - const headLimit = args.head_limit ?? DEFAULT_HEAD_LIMIT; - const afterOffset = offset > 0 ? orderedLines.slice(offset) : orderedLines; - const limitActive = headLimit > 0; - const limited = limitActive ? afterOffset.slice(0, headLimit) : afterOffset; - const paginationTruncated = limitActive && afterOffset.length > headLimit; - - const headerLines: string[] = []; - const messages: string[] = []; - if (filteredSensitive.size > 0) { - const displayedFilteredPaths = [...filteredSensitive].map((path) => - relativizeIfUnder(path, workspaceDir, pathClass), - ); - messages.push( - `Filtered ${String(filteredSensitive.size)} sensitive file(s): ${displayedFilteredPaths.join(', ')}`, - ); - } - if (mode === 'count_matches' && orderedLines.length > 0) { - headerLines.push(formatCountSummary(orderedLines, filteredSensitive.size > 0)); - } - if (paginationTruncated) { - const total = afterOffset.length + offset; - const nextOffset = offset + headLimit; - const paginationNotice = - bufferTruncated || timedOut - ? `Results truncated to ${String(headLimit)} lines (total: ${String(total)} of a partial result set). Use offset=${String(nextOffset)} to see more.` - : `Results truncated to ${String(headLimit)} lines (total: ${String(total)}). Use offset=${String(nextOffset)} to see more.`; - if (mode === 'count_matches') { - headerLines.push(paginationNotice); - } else { - messages.push(paginationNotice); - } - } - if (bufferTruncated) { - messages.push( - `[Output truncated at ${String(MAX_OUTPUT_BYTES)} bytes of rg output — the result set is incomplete. Narrow the pattern, path, or glob filters and re-run to recover complete results.]`, - ); - } - if (timedOut) { - messages.push( - `Grep timed out after ${String(DEFAULT_TIMEOUT_MS / 1000)}s; partial results returned. Narrow the path, glob, or pattern and retry for complete results.`, - ); - } - - const contentIncludesLineNumbers = mode === 'content' && args['-n'] !== false; - const displayedLines = limited.map((line) => - formatDisplayLine( - line, - mode, - workspaceDir, - pathClass, - contentIncludesLineNumbers, - ), - ); - const contentBody = displayedLines.join('\n'); - const visibleBody = - orderedLines.length === 0 && filteredSensitive.size > 0 - ? 'No non-sensitive matches found' - : contentBody; - const emptyResultMessage = - SENSITIVE_GLOBS_TO_EXCLUDE.length > 0 ? 'No non-sensitive matches found' : 'No matches found'; - const body = - visibleBody === '' && headerLines.length === 0 && messages.length === 0 - ? emptyResultMessage - : visibleBody; - const combined = [...headerLines, body, ...messages].filter((part) => part !== '').join('\n'); - - const builder = new ToolOutputAccumulator(); - builder.write(combined); - return builder.ok(); - -} - -type GrepMode = 'content' | 'files_with_matches' | 'count_matches'; - -type ParsedGrepLine = - | { - readonly kind: 'record'; - readonly filePath: string; - readonly payload: string; - } - | { - readonly kind: 'separator'; - } - | { - readonly kind: 'legacy'; - readonly text: string; - }; - -function buildRgArgs( - rgPath: string, - args: GrepInput, - searchPaths: readonly string[], - singleThreaded = false, -): string[] { - const cmd: string[] = [rgPath]; - if (singleThreaded) cmd.push('-j', '1'); - cmd.push('--hidden'); - const mode = args.output_mode ?? 'files_with_matches'; - if (mode !== 'content') { - cmd.push('--max-columns', String(RG_MAX_COLUMNS)); - } - cmd.push('--null'); - for (const dir of VCS_DIRECTORIES_TO_EXCLUDE) { - cmd.push('--glob', `!${dir}`); - } - - if (mode === 'files_with_matches') cmd.push('-l', '--sortr=modified'); - else if (mode === 'count_matches') { - cmd.push('--count-matches', '--with-filename'); - } - - if (args['-i']) cmd.push('-i'); - if (mode === 'content') { - cmd.push('--with-filename'); - if (args['-n'] !== false) { - cmd.push('-n'); - } else { - cmd.push('--field-context-separator', ':'); - } - if (args['-C'] !== undefined) { - cmd.push('-C', String(args['-C'])); - } else { - if (args['-A'] !== undefined) cmd.push('-A', String(args['-A'])); - if (args['-B'] !== undefined) cmd.push('-B', String(args['-B'])); - } - } - if (args.glob !== undefined) cmd.push('--glob', args.glob); - if (args.type !== undefined) cmd.push('--type', args.type); - if (args.multiline) cmd.push('-U', '--multiline-dotall'); - if (args.include_ignored) cmd.push('--no-ignore'); - for (const glob of SENSITIVE_GLOBS_TO_EXCLUDE) { - cmd.push('--glob', `!${glob}`); - } - - cmd.push('--', args.pattern, ...searchPaths); - return cmd; -} - -function splitRgLines(text: string): string[] { - if (text === '') return []; - const lines = text.split('\n'); - while (lines.length > 0 && lines.at(-1) === '') { - lines.pop(); - } - return lines.map((line) => stripTrailingCarriageReturn(line)); -} - -function parseRipgrepOutput(text: string, mode: GrepMode): ParsedGrepLine[] { - if (text === '') return []; - if (!text.includes('\0')) { - return splitRgLines(text).map((line) => - mode === 'content' && line === '--' ? { kind: 'separator' } : { kind: 'legacy', text: line }, - ); - } - - if (mode === 'files_with_matches') { - return text - .split('\0') - .map((filePath) => stripTrailingCarriageReturn(filePath)) - .filter((filePath) => filePath !== '') - .map((filePath) => ({ kind: 'record', filePath, payload: '' })); - } - - const records: ParsedGrepLine[] = []; - let cursor = 0; - while (cursor < text.length) { - if (text[cursor] === '\n') { - cursor += 1; - continue; - } - if (text.startsWith('--\r\n', cursor)) { - records.push({ kind: 'separator' }); - cursor += 4; - continue; - } - if (text.startsWith('--\n', cursor)) { - records.push({ kind: 'separator' }); - cursor += 3; - continue; - } - - const nulIndex = text.indexOf('\0', cursor); - if (nulIndex < 0) { - const tail = stripTrailingCarriageReturn(text.slice(cursor)); - if (tail !== '') records.push({ kind: 'legacy', text: tail }); - break; - } - - const lineEnd = text.indexOf('\n', nulIndex + 1); - const payloadEnd = lineEnd >= 0 ? lineEnd : text.length; - const filePath = text.slice(cursor, nulIndex); - const payload = stripTrailingCarriageReturn(text.slice(nulIndex + 1, payloadEnd)); - records.push({ kind: 'record', filePath, payload }); - cursor = lineEnd >= 0 ? lineEnd + 1 : text.length; - } - return records; -} - -function formatDisplayLine( - line: ParsedGrepLine, - mode: GrepMode, - workspaceDir: string, - pathClass: PathClass, - contentIncludesLineNumbers: boolean, -): string { - if (line.kind === 'separator') return '--'; - if (line.kind === 'record') { - const displayPath = relativizeIfUnder(line.filePath, workspaceDir, pathClass); - if (mode === 'files_with_matches') return displayPath; - if (mode === 'count_matches') return `${displayPath}:${line.payload}`; - const separator = contentIncludesLineNumbers ? contentPayloadPathSeparator(line.payload) : ':'; - return `${displayPath}${separator}${line.payload}`; - } - - const text = line.text; - if (mode === 'files_with_matches') { - return relativizeIfUnder(text, workspaceDir, pathClass); - } - if (mode === 'count_matches') { - const idx = text.lastIndexOf(':'); - if (idx <= 0) return text; - return relativizeIfUnder(text.slice(0, idx), workspaceDir, pathClass) + text.slice(idx); - } - - const filePath = extractContentFilePath(text, pathClass); - if (filePath !== undefined) { - return relativizeIfUnder(filePath, workspaceDir, pathClass) + text.slice(filePath.length); - } - return text; -} - -function relativizeIfUnder(candidate: string, base: string, pathClass: PathClass): string { - const normCandidate = normalize(candidate); - const normBase = normalize(base); - const comparableCandidate = pathClass === 'win32' ? normCandidate.toLowerCase() : normCandidate; - const comparableBase = pathClass === 'win32' ? normBase.toLowerCase() : normBase; - if (comparableCandidate === comparableBase) return '.'; - const prefix = comparableBase.endsWith('/') ? comparableBase : comparableBase + '/'; - if (comparableCandidate.startsWith(prefix)) { - return normCandidate.slice(prefix.length); - } - return normCandidate; -} - -function omitIncompleteTrailingRecord(text: string, mode: GrepMode): string { - if (!text.includes('\0')) return omitIncompleteTrailingLine(text); - if (mode === 'files_with_matches') { - const lastNul = text.lastIndexOf('\0'); - return lastNul >= 0 ? text.slice(0, lastNul + 1) : ''; - } - - let cursor = 0; - let lastCompleteEnd = 0; - while (cursor < text.length) { - if (text[cursor] === '\n') { - cursor += 1; - lastCompleteEnd = cursor; - continue; - } - if (text.startsWith('--\r\n', cursor)) { - cursor += 4; - lastCompleteEnd = cursor; - continue; - } - if (text.startsWith('--\n', cursor)) { - cursor += 3; - lastCompleteEnd = cursor; - continue; - } - - const nulIndex = text.indexOf('\0', cursor); - if (nulIndex < 0) break; - const lineEnd = text.indexOf('\n', nulIndex + 1); - if (lineEnd < 0) break; - cursor = lineEnd + 1; - lastCompleteEnd = cursor; - } - return text.slice(0, lastCompleteEnd); -} - -function omitIncompleteTrailingLine(text: string): string { - const lastNewline = text.lastIndexOf('\n'); - return lastNewline >= 0 ? text.slice(0, lastNewline) : ''; -} - -function formatRipgrepError( - exitCode: number, - stderrText: string, - stderrTruncated: boolean, -): string { - const stderr = stderrText.trim(); - if (stderr.length === 0) { - return `Failed to grep: ripgrep exited with code ${String(exitCode)}`; - } - - const summary = summarizeRipgrepStderr(stderr); - const lines = [`Failed to grep: ${summary}`, '', 'ripgrep stderr:', stderr]; - if (stderrTruncated) { - lines.push(`[stderr truncated at ${String(MAX_OUTPUT_BYTES)} bytes]`); - } - return lines.join('\n'); -} - -function summarizeRipgrepStderr(stderr: string): string { - const lines = splitRgLines(stderr) - .map((line) => line.trim()) - .filter((line) => line.length > 0); - const errorLine = lines.findLast((line) => line.toLowerCase().startsWith('error:')); - return errorLine ?? lines.at(-1) ?? 'ripgrep error'; -} - -function filterSensitiveLines( - lines: readonly ParsedGrepLine[], - mode: GrepMode, - filteredPaths: Set, - pathClass: PathClass, -): ParsedGrepLine[] { - const kept: ParsedGrepLine[] = []; - for (const line of lines) { - if (line.kind === 'separator') { - kept.push(line); - continue; - } - const filePath = parsedFilePath(line, mode, pathClass); - if (filePath !== undefined && isSensitiveFile(filePath)) { - filteredPaths.add(filePath); - continue; - } - kept.push(line); - } - return mode === 'content' ? normalizeContextSeparators(kept) : kept; -} - -function normalizeContextSeparators(lines: readonly ParsedGrepLine[]): ParsedGrepLine[] { - const normalized: ParsedGrepLine[] = []; - for (const line of lines) { - if ( - line.kind === 'separator' && - (normalized.length === 0 || normalized.at(-1)?.kind === 'separator') - ) { - continue; - } - normalized.push(line); - } - while (normalized.length > 0 && normalized.at(-1)?.kind === 'separator') { - normalized.pop(); - } - return normalized; -} - -function parsedFilePath( - line: ParsedGrepLine, - mode: GrepMode, - pathClass: PathClass, -): string | undefined { - if (line.kind === 'record') return normalize(line.filePath); - if (line.kind === 'separator') return undefined; - const text = line.text; - if (mode === 'files_with_matches') return normalize(text); - if (mode === 'count_matches') { - const idx = text.lastIndexOf(':'); - return idx > 0 ? normalize(text.slice(0, idx)) : normalize(text); - } - return extractContentFilePath(text, pathClass); -} - -function extractContentFilePath(line: string, pathClass: PathClass): string | undefined { - const m = CONTENT_LINE_RE.exec(line); - if (m?.[1] !== undefined) return normalize(m[1]); - - const separatorIndex = noLineNumberContentSeparatorIndex(line, pathClass); - return separatorIndex > 0 ? normalize(line.slice(0, separatorIndex)) : undefined; -} - -function noLineNumberContentSeparatorIndex(line: string, pathClass: PathClass): number { - const searchFrom = pathClass === 'win32' && /^[A-Za-z]:/.test(line) ? 2 : 0; - return line.indexOf(':', searchFrom); -} - -function contentPayloadPathSeparator(payload: string): ':' | '-' { - const m = /^(\d+)([:-])/.exec(payload); - return m?.[2] === '-' ? '-' : ':'; -} - -function stripTrailingCarriageReturn(value: string): string { - return value.endsWith('\r') ? value.slice(0, -1) : value; -} - -function formatCountSummary(lines: readonly ParsedGrepLine[], redactedSensitive: boolean): string { - let totalMatches = 0; - let totalFiles = 0; - for (const line of lines) { - const rawCount = - line.kind === 'record' - ? line.payload - : line.kind === 'legacy' - ? countPayloadFromLegacyLine(line.text) - : undefined; - if (rawCount === undefined) continue; - const count = Number(rawCount); - if (!Number.isSafeInteger(count) || count < 0) continue; - totalMatches += count; - totalFiles++; - } - - const occurrenceWord = totalMatches === 1 ? 'occurrence' : 'occurrences'; - const fileWord = totalFiles === 1 ? 'file' : 'files'; - const scope = redactedSensitive ? 'total non-sensitive' : 'total'; - return `Found ${String(totalMatches)} ${scope} ${occurrenceWord} across ${String(totalFiles)} ${fileWord}.`; -} - -function countPayloadFromLegacyLine(line: string): string | undefined { - const idx = line.lastIndexOf(':'); - return idx > 0 ? line.slice(idx + 1) : undefined; -} diff --git a/packages/agent-core-v2/src/os/operations/snapshotFiles.ts b/packages/agent-core-v2/src/os/operations/snapshotFiles.ts deleted file mode 100644 index 5a0e5048179..00000000000 --- a/packages/agent-core-v2/src/os/operations/snapshotFiles.ts +++ /dev/null @@ -1,39 +0,0 @@ -import { createHash } from 'node:crypto'; - -import type { IHostFileSystem } from '#/os/interface/hostFileSystem'; -import type { FileSnapshotRequest, FileSnapshot } from '#/os/interface/fileOperations'; -import { isHostFsNotFound } from '#/os/interface/hostFsErrors'; - -export async function snapshotFiles(fs: IHostFileSystem, files: readonly FileSnapshotRequest[], maxFileBytes: number): Promise { - const results: FileSnapshot[] = []; - for (const file of files) { - let inspected = false; - try { - const before = await fs.stat(file.path); - inspected = true; - if (!before.isFile) { - results.push({ kind: 'unreadable' }); - continue; - } - if (before.size > maxFileBytes) { - results.push({ kind: 'oversize', size: before.size, mtimeMs: before.mtimeMs }); - continue; - } - const bytes = await fs.readBytes(file.path, before.size + 1); - const after = await fs.stat(file.path); - if (bytes.byteLength > maxFileBytes) { - results.push({ kind: 'oversize', size: after.size, mtimeMs: after.mtimeMs }); - continue; - } - if (!after.isFile || bytes.byteLength !== before.size || before.size !== after.size || before.mtimeMs !== after.mtimeMs || before.ino !== after.ino) { - results.push({ kind: 'unreadable' }); - continue; - } - const contentHash = createHash('sha256').update(bytes).digest('hex'); - results.push({ kind: 'file', contentHash, bytes: file.knownHash === contentHash ? undefined : bytes }); - } catch (error) { - results.push({ kind: !inspected && isHostFsNotFound(error) ? 'missing' : 'unreadable' }); - } - } - return results; -} diff --git a/packages/agent-core-v2/src/os/operations/textModel.ts b/packages/agent-core-v2/src/os/operations/textModel.ts deleted file mode 100644 index ab6611d738f..00000000000 --- a/packages/agent-core-v2/src/os/operations/textModel.ts +++ /dev/null @@ -1,43 +0,0 @@ -import { - type LineEndingStyle, - materializeModelText, - toModelTextView, -} from '#/_base/text/line-endings'; - -export class TextModel { - readonly lineEndingStyle: LineEndingStyle; - readonly text: string; - - constructor(raw: string) { - const view = toModelTextView(raw); - this.text = view.text; - this.lineEndingStyle = view.lineEndingStyle; - } - - countOccurrences(needle: string): number { - let count = 0; - let pos = 0; - while (pos < this.text.length) { - const idx = this.text.indexOf(needle, pos); - if (idx === -1) break; - count += 1; - pos = idx + needle.length; - } - return count; - } - - replaceOnce(needle: string, replacement: string): string { - const index = this.text.indexOf(needle); - if (index === -1) return this.text; - return this.text.slice(0, index) + replacement + this.text.slice(index + needle.length); - } - - replaceAll(needle: string, replacement: string): { text: string; count: number } { - const parts = this.text.split(needle); - return { text: parts.join(replacement), count: parts.length - 1 }; - } - - materialize(modelText: string): string { - return materializeModelText(modelText, this.lineEndingStyle); - } -} diff --git a/packages/agent-core-v2/src/os/operations/writeText.ts b/packages/agent-core-v2/src/os/operations/writeText.ts deleted file mode 100644 index 461562ee83d..00000000000 --- a/packages/agent-core-v2/src/os/operations/writeText.ts +++ /dev/null @@ -1,53 +0,0 @@ -import type { HostFileStat, IHostFileSystem } from '#/os/interface/hostFileSystem'; -import type { TextWriteInput, TextFileResult } from '#/os/interface/fileOperations'; -import { isHostFsNotFound } from '#/os/interface/hostFsErrors'; - -export async function writeText(fs: IHostFileSystem, input: TextWriteInput): Promise { - const parentError = await ensureParentDirectory(fs, input.parent); - if (parentError !== undefined) return { isError: true, output: parentError }; - return writeTextResult(input, () => input.mode === 'append' ? fs.appendText(input.path, input.content) : fs.writeText(input.path, input.content)); -} - -export async function writeTextResult(input: TextWriteInput, write: () => Promise): Promise { - try { - const mode = input.mode; - await write(); - const bytesWritten = Buffer.byteLength(input.content, 'utf8'); - return { - output: `${mode === 'append' ? 'Appended' : 'Wrote'} ${String(bytesWritten)} bytes to ${input.displayPath}`, - }; - } catch (error) { - if (isHostFsNotFound(error)) { - return { - isError: true, - output: `Failed to write ${input.displayPath}: parent directory does not exist.`, - }; - } - return { - isError: true, - output: error instanceof Error ? error.message : String(error), - }; - } - -} - -export async function ensureParentDirectory(fs: IHostFileSystem, parent: string): Promise { - let stat: HostFileStat; - try { - stat = await fs.stat(parent); - } catch (error) { - if (isHostFsNotFound(error)) { - try { - await fs.mkdir(parent, { recursive: true }); - return undefined; - } catch (mkdirError) { - return mkdirError instanceof Error ? mkdirError.message : String(mkdirError); - } - } - return undefined; - } - if (!stat.isDirectory) { - return `Parent path is not a directory: ${parent}.`; - } - return undefined; -} diff --git a/packages/agent-core-v2/src/remote/README.md b/packages/agent-core-v2/src/remote/README.md index fec6e4973e6..1cdc4a90467 100644 --- a/packages/agent-core-v2/src/remote/README.md +++ b/packages/agent-core-v2/src/remote/README.md @@ -12,7 +12,6 @@ node-local one rather than here: packages/agent-core-v2/src/ ├── os/ │ ├── interface/ IHostFileSystem / IHostProcessService contracts -│ ├── operations/ DI-free file and search algorithms shared with the controller │ └── backends/ │ ├── node-local/ direct node implementation (host, local environment, executor) │ └── remote/ RPC stubs over an ExecChannel (remoteFileSystem, remoteProcess) @@ -20,13 +19,13 @@ packages/agent-core-v2/src/ ├── protocol/ message types, error codes, NDJSON codec, ExecChannel (self-contained) ├── client/ execBridge, launchers, sshAskpass, connection, remoteSession, │ remoteEnvironmentProvider, connectGuidance (executor detection + guidance) - ├── server/ stdioHost, fsHandler, fileOperationsHandler, searchHandler, processManager + ├── server/ stdioHost, fsHandler, processManager, environment, entry ├── declaration/ [environments] config schema └── askpass.ts SSH_ASKPASS helper process (light entry, node builtins only) ``` -`os/backends/remote` may import only `#/remote/protocol`, `#/os/interface`, -`#/os/operations` and `#/_base`; the connection lifecycle that feeds it an `ExecChannel` stays in +`os/backends/remote` may import only `#/remote/protocol`, `#/os/interface` and +`#/_base`; the connection lifecycle that feeds it an `ExecChannel` stays in `remote/client`. ## Entries @@ -48,14 +47,14 @@ config/OAuth/telemetry/session services. Logs go to stderr; stdout carries protocol frames only. The light import graph is enforced by `scripts/check-import-boundaries.mjs`: `src/remote/server` may import only node builtins, relative modules, `#/remote/protocol`, `#/os/interface`, -`#/os/operations`, `#/_base/execEnv` and the DI-free node-local `hostFsService` / +`#/_base/execEnv` and the DI-free node-local `hostFsService` / `hostProcessService` modules (their App-scope registration lives in `os/backends/node-local/hostServices.ts`). ## Protocol notes (deviations from codex exec-server) Blueprint is codex exec-server (`codex-rs/exec-server*`, master `0265dd7b`); -deviations per the design spec and the shared host-operation extensions: +deviations per the design spec, plus one forced addition: - Bare absolute paths instead of `file:` URIs (our abstraction is bare-path end to end). @@ -65,13 +64,7 @@ deviations per the design spec and the shared host-operation extensions: - `fs/writeFile` takes `mode: truncate|append|exclusive` (codex only has truncate); `exclusive` maps EEXIST to `os.fs.already_exists`, which the client maps back to `createExclusive === false`. Optional `parent` folds - directory preparation into the first write. Large transfers attach - `transferId`/`chunkIndex`/`final`; the client pipelines at most four 1MiB - chunks, and the executor sequences each transfer and stops after an error. - It tracks at most 64 transfers, expires idle transfers after 60 seconds, - and rejects duplicate or out-of-order chunks while a transfer is live. - Writes are never automatically retried, and connection loss may leave a - partial write or an uncertain mutation outcome. + directory preparation into the first write while chunks remain sequential. - `fs/readDirectory` preserves the symlink marker on each entry. The client rejects a truncated listing with `os.fs.directory_too_large` rather than exposing an incomplete directory; the current limit is 50,000 entries. @@ -81,34 +74,6 @@ deviations per the design spec and the shared host-operation extensions: back to per-directory reads when the JSON-encoded ignore rules exceed 1MiB. Walks default to a 55-second budget so they can return a truncated result before the 60-second RPC deadline. -- `fs/readTextPage` performs sniffing, text decoding, line counting and page - rendering on the executor. It returns the same bounded output and exact - totals as local Read. Up to eight scans may run at once; `fs/cancelRead` - aborts an active scan. Requests above 4,000,000 output characters or above - the active-scan limit fall back to controller-side streaming, preserving - larger configured Read budgets and concurrent calls. -- `fs/editText` applies the shared text-edit algorithm next to the file and - returns the replacement count or original edit error. It does not add an - atomic compare-and-swap guarantee against external writers. -- `fs/snapshotFiles` reads and hashes up to four files of at most 4MiB each. - Matching known hashes omit the bytes; the controller retains durable - history blobs before a mutation. Content hints from writes and edits are - bounded to 16 files / 16MiB and reused only after a fresh remote content hash. -- `fs/findGitWorkTree` and `fs/discoverInstructions` perform directory-chain - probes on the executor. Probe reuse is scoped to one instruction batch, - so a newly created Git marker or instruction file is visible on the next call. -- `fs/matchFiles` scans batches of up to 128 already-selected files for the - workspace grep fallback. Responses above 8MiB request controller-side - matching so existing large match/context outputs remain available. -- `process/search` runs the shared Glob/Grep algorithm on the executor and - returns the formatted page. The controller still resolves ripgrep and - handles permissions and telemetry. Up to eight searches may be active; - `process/cancelSearch` and disconnect abort their processes. More than 64 - roots or eight concurrent searches use controller-side processing with the - original process RPC path. The existing - 20-second timeout, retry, sorting, filtering and exact count semantics - remain; reaching the 10MiB stdout capture limit stops ripgrep early and - returns the existing partial-output diagnostics. - Added `fs/rename` (no codex counterpart). - `fs/remove` takes only `path` and always removes recursively and ignores a missing path, matching `IHostFileSystem.remove` (codex has `recursive`/`force`). @@ -153,9 +118,7 @@ Client-surface notes beyond the wire protocol: and a late response is discarded — the connection survives. `process/write` is exempt: its response is deliberately suspended until the child's stdin drains, so it carries no timeout and is never retried — a hung write fails - only with the connection. `fs/readTextPage` is also exempt so long scans - retain Read's existing behavior; explicit cancellation and connection - liveness remain in force. + only with the connection. - Whole-file reads without `maxBytes` are rejected server-side above 32MiB (base64 of the response must fit the 64MiB frame cap); larger files are read through `offset`/`maxBytes` range reads. @@ -218,9 +181,6 @@ handshake completes: any byte or unknown notification disconnects. ## Operational notes -- Remote execution is experimental; client and executor must be upgraded - together. The new operation methods do not negotiate compatibility with - earlier development executors. - The executor is posix-only; the client refuses non-posix environments and executors below `MIN_EXECUTOR_VERSION` at the handshake gate. - `test/remote/e2e/` holds the real-machine (ssh/docker) acceptance driver and diff --git a/packages/agent-core-v2/src/remote/client/connection.ts b/packages/agent-core-v2/src/remote/client/connection.ts index 840fbff8d30..1c1375f2ad8 100644 --- a/packages/agent-core-v2/src/remote/client/connection.ts +++ b/packages/agent-core-v2/src/remote/client/connection.ts @@ -16,7 +16,6 @@ import { MIN_EXECUTOR_VERSION, PING_METHOD, PROCESS_WRITE_METHOD, - FS_READ_TEXT_PAGE_METHOD, SERVER_NOTIFICATION_METHODS, type InitializeResult, type RemoteEnvironmentInfo, @@ -259,7 +258,7 @@ export class RemoteExecConnection implements ExecChannel { } return new Promise((resolve, reject) => { const pending: PendingCall = { resolve, reject }; - if (method !== PROCESS_WRITE_METHOD && method !== FS_READ_TEXT_PAGE_METHOD) { + if (method !== PROCESS_WRITE_METHOD) { pending.timer = setTimeout(() => { this.expireRequest(id, method); }, this.requestCallTimeoutMs); diff --git a/packages/agent-core-v2/src/remote/client/remoteEnvironmentProvider.ts b/packages/agent-core-v2/src/remote/client/remoteEnvironmentProvider.ts index bcfa92777ad..b58c7ba4359 100644 --- a/packages/agent-core-v2/src/remote/client/remoteEnvironmentProvider.ts +++ b/packages/agent-core-v2/src/remote/client/remoteEnvironmentProvider.ts @@ -51,17 +51,10 @@ class DeclaredRemoteEnvironment implements Environment { private connectedAt = 0; readonly fs: IHostFileSystem = { _serviceBrand: undefined, - operations: { - matchFiles: async (files, options, timeoutMs) => this.connected().fs.operations.matchFiles(files, options, timeoutMs), - findGitWorkTree: async (cwd) => this.connected().fs.operations.findGitWorkTree(cwd), - discoverInstructions: async (dirs, known) => this.connected().fs.operations.discoverInstructions(dirs, known), - snapshotFiles: async (files, maxFileBytes) => this.connected().fs.operations.snapshotFiles(files, maxFileBytes), - writeText: async (input) => this.connected().fs.operations.writeText(input), - readTextPage: async (path, options, signal) => this.connected().fs.operations.readTextPage(path, options, signal), - editText: async (input) => this.connected().fs.operations.editText(input), - }, readText: async (path, options) => this.connected().fs.readText(path, options), + readWholeFile: async (path) => this.connected().fs.readWholeFile!(path), writeText: async (path, data) => this.connected().fs.writeText(path, data), + writeTextWithParent: async (path, data, mode, parent) => this.connected().fs.writeTextWithParent!(path, data, mode, parent), appendText: async (path, data) => this.connected().fs.appendText(path, data), readBytes: async (path, n, offset) => this.connected().fs.readBytes(path, n, offset), writeBytes: async (path, data) => this.connected().fs.writeBytes(path, data), @@ -78,7 +71,6 @@ class DeclaredRemoteEnvironment implements Environment { }; readonly process: IHostProcessService = { _serviceBrand: undefined, - search: async (request, signal) => this.connected().process.search(request, signal), spawn: async (command, args, options) => this.connected().process.spawn(command, args, options), }; diff --git a/packages/agent-core-v2/src/remote/protocol/methods.ts b/packages/agent-core-v2/src/remote/protocol/methods.ts index a25cc454b39..99f4bde03ad 100644 --- a/packages/agent-core-v2/src/remote/protocol/methods.ts +++ b/packages/agent-core-v2/src/remote/protocol/methods.ts @@ -1,22 +1,6 @@ export const INITIALIZE_METHOD = 'initialize'; export const PING_METHOD = 'ping'; -export const FS_FIND_GIT_WORK_TREE_METHOD = 'fs/findGitWorkTree'; -export const FS_DISCOVER_INSTRUCTIONS_METHOD = 'fs/discoverInstructions'; - -export const FS_MATCH_FILES_METHOD = 'fs/matchFiles'; -export const FS_SNAPSHOT_FILES_METHOD = 'fs/snapshotFiles'; -export const FS_SNAPSHOT_MAX_FILES = 4; -export const FS_SNAPSHOT_MAX_FILE_BYTES = 4 * 1024 * 1024; - -export const FS_MAX_ACTIVE_READS = 8; -export const PROCESS_MAX_ACTIVE_SEARCHES = 8; -export const PROCESS_SEARCH_MAX_ROOTS = 64; - -export const FS_CANCEL_READ_METHOD = 'fs/cancelRead'; -export const FS_READ_TEXT_PAGE_METHOD = 'fs/readTextPage'; -export const FS_EDIT_TEXT_METHOD = 'fs/editText'; - export const FS_READ_FILE_METHOD = 'fs/readFile'; export const FS_WRITE_FILE_METHOD = 'fs/writeFile'; export const FS_CREATE_DIRECTORY_METHOD = 'fs/createDirectory'; @@ -27,9 +11,6 @@ export const FS_WALK_METHOD = 'fs/walk'; export const FS_REMOVE_METHOD = 'fs/remove'; export const FS_RENAME_METHOD = 'fs/rename'; -export const PROCESS_SEARCH_METHOD = 'process/search'; -export const PROCESS_CANCEL_SEARCH_METHOD = 'process/cancelSearch'; - export const PROCESS_START_METHOD = 'process/start'; export const PROCESS_OUTPUT_METHOD = 'process/output'; export const PROCESS_EXITED_METHOD = 'process/exited'; @@ -84,9 +65,6 @@ export interface FsWriteFileParams { readonly dataBase64: string; readonly mode: FsWriteMode; readonly parent?: string; - readonly transferId?: string; - readonly chunkIndex?: number; - readonly final?: boolean; } export interface FsCreateDirectoryParams { diff --git a/packages/agent-core-v2/src/remote/server/fileOperationsHandler.ts b/packages/agent-core-v2/src/remote/server/fileOperationsHandler.ts deleted file mode 100644 index 62ff3f3c510..00000000000 --- a/packages/agent-core-v2/src/remote/server/fileOperationsHandler.ts +++ /dev/null @@ -1,135 +0,0 @@ -import type { HostFileSystem } from '#/os/backends/node-local/hostFsService'; -import { TEXT_PAGE_MAX_REMOTE_CHARS, type TextFileResult, type TextEditResult } from '#/os/interface/fileOperations'; -import { findGitWorkTree } from '#/os/operations/gitWorkTree'; -import { discoverInstructions } from '#/os/operations/instructions'; -import { matchFiles } from '#/os/operations/matchFiles'; -import { snapshotFiles } from '#/os/operations/snapshotFiles'; -import { FS_MAX_ACTIVE_READS, FS_SNAPSHOT_MAX_FILES, FS_SNAPSHOT_MAX_FILE_BYTES } from '#/remote/protocol/methods'; -import { encodeBase64 } from '#/remote/protocol/codec'; -import { readTextPage } from '#/os/operations/readTextPage'; -import { editText } from '#/os/operations/editText'; -import { RpcError, RpcErrorCode } from '#/remote/protocol/errors'; -import { optionalBoolean, optionalInteger, requireAbsolutePath, requireParams, requireString } from './fsHandler'; - -export class FileOperationsHandler { - private readonly reads = new Map }>(); - - cancelRead(rawParams: unknown): void { - const params = requireParams(rawParams); - this.reads.get(requireString(params, 'readId'))?.controller.abort(new Error('Read aborted')); - } - - async dispose(): Promise { - const pending = [...this.reads.values()]; - for (const read of pending) read.controller.abort(new Error('Read aborted')); - await Promise.allSettled(pending.map((read) => read.done)); - } - - constructor(private readonly fs: HostFileSystem) {} - - async findGitWorkTree(rawParams: unknown): Promise<{ workTree: Awaited> }> { - const params = requireParams(rawParams); - return { workTree: await findGitWorkTree(this.fs, requireAbsolutePath(params, 'cwd')) }; - } - - discoverInstructions(rawParams: unknown): Promise { - const params = requireParams(rawParams); - const dirs = params['dirs']; - const known = params['known']; - if (!Array.isArray(dirs) || dirs.length > 64 || !Array.isArray(known) || known.some((path) => typeof path !== 'string')) throw new RpcError(RpcErrorCode.InvalidParams, 'invalid instruction directories'); - const paths = dirs.map((dir: unknown) => requireAbsolutePath({ dir }, 'dir')); - return discoverInstructions(this.fs, paths, known); - } - - matchFiles(rawParams: unknown): ReturnType { - const params = requireParams(rawParams); - const files = params['files']; - if (!Array.isArray(files) || files.length > 128) throw new RpcError(RpcErrorCode.InvalidParams, 'too many match files'); - const paths = files.map((file: unknown) => { - const value = requireParams(file); - return { path: requireString(value, 'path'), absolute: requireAbsolutePath(value, 'absolute') }; - }); - const options = requireParams(params['options']); - return matchFiles(this.fs, paths, { - pattern: requireString(options, 'pattern'), - regex: optionalBoolean(options, 'regex') ?? false, - case_sensitive: optionalBoolean(options, 'case_sensitive') ?? true, - context_lines: optionalInteger(options, 'context_lines', 0, 10) ?? 2, - max_files: optionalInteger(options, 'max_files', 1, 10_000) ?? 200, - max_matches_per_file: optionalInteger(options, 'max_matches_per_file', 1, 10_000) ?? 50, - max_total_matches: optionalInteger(options, 'max_total_matches', 1, 100_000) ?? 5000, - }, optionalInteger(params, 'timeoutMs', 1, 30_000) ?? 30_000, undefined, 8 * 1024 * 1024); - } - - async snapshotFiles(rawParams: unknown): Promise { - const params = requireParams(rawParams); - const files = params['files']; - if (!Array.isArray(files) || files.length > FS_SNAPSHOT_MAX_FILES) throw new RpcError(RpcErrorCode.InvalidParams, 'too many snapshot files'); - const requests = files.map((file: unknown) => { - const entry = requireParams(file); - const knownHash = entry['knownHash']; - if (knownHash !== undefined && (typeof knownHash !== 'string' || !/^[a-f0-9]{64}$/.test(knownHash))) throw new RpcError(RpcErrorCode.InvalidParams, 'invalid snapshot hash'); - return { path: requireAbsolutePath(entry, 'path'), knownHash }; - }); - const maxFileBytes = optionalInteger(params, 'maxFileBytes', 0, FS_SNAPSHOT_MAX_FILE_BYTES) ?? FS_SNAPSHOT_MAX_FILE_BYTES; - const results = await snapshotFiles(this.fs, requests, maxFileBytes); - return results.map((result) => result.kind === 'file' ? { - kind: result.kind, - contentHash: result.contentHash, - dataBase64: result.bytes === undefined ? undefined : encodeBase64(result.bytes), - } : result); - } - - async readTextPage(rawParams: unknown): Promise { - const params = requireParams(rawParams); - const path = requireAbsolutePath(params, 'path'); - const args = requireParams(params['args']); - const lineOffset = optionalInteger(args, 'line_offset', -Number.MAX_SAFE_INTEGER, Number.MAX_SAFE_INTEGER); - if (lineOffset === 0) throw new RpcError(RpcErrorCode.InvalidParams, 'line_offset must be nonzero'); - const maxChars = optionalInteger(params, 'maxChars', 1, TEXT_PAGE_MAX_REMOTE_CHARS); - const maxCharsLimit = optionalInteger(params, 'maxCharsLimit', 1, Number.MAX_SAFE_INTEGER); - if (maxChars === undefined || maxCharsLimit === undefined) throw new RpcError(RpcErrorCode.InvalidParams, 'read limits are required'); - const readId = requireString(params, 'readId'); - if (this.reads.has(readId) || this.reads.size >= FS_MAX_ACTIVE_READS) throw new RpcError(RpcErrorCode.InvalidRequest, 'read ID is active or too many active reads'); - const controller = new AbortController(); - const done = readTextPage({ - name: path, - stat: () => this.fs.stat(path), - readBytes: (n) => this.fs.readBytes(path, n), - readLines: () => this.fs.readLines(path, { errors: 'strict' }), - }, { - args: { - path: requireString(args, 'path'), - line_offset: lineOffset, - column_offset: optionalInteger(args, 'column_offset', 0, Number.MAX_SAFE_INTEGER), - n_lines: optionalInteger(args, 'n_lines', 1, Number.MAX_SAFE_INTEGER), - max_chars: optionalInteger(args, 'max_chars', 1, Number.MAX_SAFE_INTEGER), - }, - maxChars, - maxCharsLimit, - eventLog: optionalBoolean(params, 'eventLog') ?? false, - imageSupported: optionalBoolean(params, 'imageSupported') ?? false, - videoSupported: optionalBoolean(params, 'videoSupported') ?? false, - mediaToolAvailable: optionalBoolean(params, 'mediaToolAvailable') ?? false, - }, controller.signal); - this.reads.set(readId, { controller, done }); - try { - return await done; - } finally { - this.reads.delete(readId); - } - } - - editText(rawParams: unknown): Promise { - const params = requireParams(rawParams); - const newString = params['new_string']; - if (typeof newString !== 'string') throw new RpcError(RpcErrorCode.InvalidParams, 'new_string must be a string'); - return editText(this.fs, { - path: requireAbsolutePath(params, 'path'), - displayPath: requireString(params, 'displayPath'), - old_string: requireString(params, 'old_string'), - new_string: newString, - replace_all: optionalBoolean(params, 'replace_all') ?? false, - }); - } -} diff --git a/packages/agent-core-v2/src/remote/server/fsHandler.ts b/packages/agent-core-v2/src/remote/server/fsHandler.ts index 462732a01d3..0125f3ca397 100644 --- a/packages/agent-core-v2/src/remote/server/fsHandler.ts +++ b/packages/agent-core-v2/src/remote/server/fsHandler.ts @@ -1,13 +1,11 @@ -import { ensureParentDirectory } from '#/os/operations/writeText'; import { isAbsolute } from 'node:path'; import type { HostFileSystem } from '#/os/backends/node-local/hostFsService'; -import { HostFsError, OsFsErrors, toHostFsError } from '#/os/interface/hostFsErrors'; +import { HostFsError, isHostFsNotFound, OsFsErrors, toHostFsError } from '#/os/interface/hostFsErrors'; import { RpcError, RpcErrorCode } from '#/remote/protocol/errors'; import { FS_READ_DIRECTORY_MAX_ENTRIES, - FS_WRITE_FILE_CHUNK_BYTES, FS_READ_FILE_MAX_BYTES, FS_READ_FILE_WHOLE_MAX_BYTES, type FsCanonicalizeResult, @@ -61,7 +59,7 @@ export function optionalBoolean(params: Params, name: string): boolean | undefin return value; } -export function optionalInteger( +function optionalInteger( params: Params, name: string, min: number, @@ -101,22 +99,7 @@ function fsDomainError(error: unknown, ctx: { path: string; op: string }): RpcEr const EMPTY: Record = {}; -interface WriteTransfer { - readonly path: string; - nextIndex: number; - final: boolean; - tail: Promise; - timer: ReturnType; -} - export class FsHandler { - private readonly writes = new Map(); - - dispose(): void { - for (const transfer of this.writes.values()) clearTimeout(transfer.timer); - this.writes.clear(); - } - constructor(private readonly fs: HostFileSystem) {} async readFile(rawParams: unknown): Promise { @@ -145,40 +128,6 @@ export class FsHandler { } async writeFile(rawParams: unknown): Promise { - const params = requireParams(rawParams); - const transferId = params['transferId']; - if (transferId === undefined) return this.writeChunk(params); - if (typeof transferId !== 'string' || transferId.length > 128 || transferId.length === 0) { - throw new RpcError(RpcErrorCode.InvalidParams, 'invalid write transfer ID'); - } - const path = requireAbsolutePath(params, 'path'); - const chunkIndex = optionalInteger(params, 'chunkIndex', 0, Number.MAX_SAFE_INTEGER); - const final = optionalBoolean(params, 'final') ?? false; - let transfer = this.writes.get(transferId); - if (transfer === undefined) { - if (chunkIndex !== 0 || this.writes.size >= 64) throw new RpcError(RpcErrorCode.InvalidRequest, 'unknown write transfer or too many active writes'); - transfer = { path, nextIndex: 0, final: false, tail: Promise.resolve(), timer: setTimeout(() => this.writes.delete(transferId), 60_000).unref() }; - this.writes.set(transferId, transfer); - } - if (transfer.final || transfer.path !== path || transfer.nextIndex !== chunkIndex || (chunkIndex !== 0 && (params['mode'] !== 'append' || params['parent'] !== undefined))) { - throw new RpcError(RpcErrorCode.InvalidRequest, 'write chunks must arrive once and in order for the same file'); - } - transfer.nextIndex += 1; - transfer.final = final; - transfer.timer.refresh(); - const run = transfer.tail.then(() => this.writeChunk(params)); - transfer.tail = run; - try { - return await run; - } finally { - if (final) { - clearTimeout(transfer.timer); - this.writes.delete(transferId); - } - } - } - - private async writeChunk(rawParams: unknown): Promise { const params = requireParams(rawParams); const path = requireAbsolutePath(params, 'path'); const dataBase64 = params['dataBase64']; @@ -190,11 +139,23 @@ export class FsHandler { throw new RpcError(RpcErrorCode.InvalidParams, 'mode must be truncate, append or exclusive'); } const data = Buffer.from(dataBase64, 'base64'); - if (data.byteLength > FS_WRITE_FILE_CHUNK_BYTES) throw new RpcError(RpcErrorCode.InvalidParams, 'write chunk exceeds the byte limit'); if (params['parent'] !== undefined) { const parent = requireAbsolutePath(params, 'parent'); - const error = await ensureParentDirectory(this.fs, parent); - if (error !== undefined) throw new RpcError(RpcErrorCode.InvalidRequest, error); + try { + const stat = await this.fs.stat(parent); + if (!stat.isDirectory) { + throw new RpcError(RpcErrorCode.InvalidRequest, `Parent path is not a directory: ${parent}.`); + } + } catch (error) { + if (error instanceof RpcError) throw error; + if (isHostFsNotFound(error)) { + try { + await this.fs.mkdir(parent, { recursive: true }); + } catch (mkdirError) { + throw new RpcError(RpcErrorCode.InvalidRequest, mkdirError instanceof Error ? mkdirError.message : String(mkdirError)); + } + } + } } try { if (mode === 'truncate') { diff --git a/packages/agent-core-v2/src/remote/server/searchHandler.ts b/packages/agent-core-v2/src/remote/server/searchHandler.ts deleted file mode 100644 index b8b9c302039..00000000000 --- a/packages/agent-core-v2/src/remote/server/searchHandler.ts +++ /dev/null @@ -1,76 +0,0 @@ -import { PROCESS_MAX_ACTIVE_SEARCHES, PROCESS_SEARCH_MAX_ROOTS } from '#/remote/protocol/methods'; -import type { IHostProcessService } from '#/os/interface/hostProcess'; -import type { SearchRequest, SearchResult } from '#/os/interface/searchOperations'; -import { searchGlob } from '#/os/operations/searchGlob'; -import { searchGrep } from '#/os/operations/searchGrep'; -import { RpcError, RpcErrorCode } from '#/remote/protocol/errors'; -import { optionalBoolean, optionalInteger, requireAbsolutePath, requireParams, requireString } from './fsHandler'; - -export class SearchHandler { - private readonly active = new Map }>(); - - constructor(private readonly process: IHostProcessService) {} - - async search(rawParams: unknown): Promise { - const params = requireParams(rawParams); - const searchId = requireString(params, 'searchId'); - const kind = requireString(params, 'kind'); - if (kind !== 'glob' && kind !== 'grep') throw new RpcError(RpcErrorCode.InvalidParams, 'invalid search kind'); - const rawArgs = requireParams(params['args']); - const roots = params['roots']; - if (!Array.isArray(roots) || roots.length === 0 || roots.length > PROCESS_SEARCH_MAX_ROOTS) throw new RpcError(RpcErrorCode.InvalidParams, 'invalid search roots'); - const pattern = rawArgs['pattern']; - if (typeof pattern !== 'string') throw new RpcError(RpcErrorCode.InvalidParams, 'pattern must be a string'); - const outputMode = rawArgs['output_mode']; - if (outputMode !== undefined && outputMode !== 'content' && outputMode !== 'files_with_matches' && outputMode !== 'count_matches') throw new RpcError(RpcErrorCode.InvalidParams, 'invalid search output mode'); - const pathClass = params['pathClass']; - if (pathClass !== 'posix' && pathClass !== 'win32') throw new RpcError(RpcErrorCode.InvalidParams, 'invalid search path class'); - const optionalString = (key: string): string | undefined => { - const value = rawArgs[key]; - if (value !== undefined && typeof value !== 'string') throw new RpcError(RpcErrorCode.InvalidParams, `${key} must be a string`); - return value; - }; - const request: SearchRequest = { - kind, - roots: roots.map((root: unknown) => requireAbsolutePath({ root }, 'root')), - workspaceDir: requireAbsolutePath(params, 'workspaceDir'), - rgPath: requireString(params, 'rgPath'), - pathClass, - args: { - pattern, - glob: optionalString('glob'), - type: optionalString('type'), - output_mode: outputMode, - offset: optionalInteger(rawArgs, 'offset', 0, Number.MAX_SAFE_INTEGER), - head_limit: optionalInteger(rawArgs, 'head_limit', 0, Number.MAX_SAFE_INTEGER), - include_ignored: optionalBoolean(rawArgs, 'include_ignored'), - multiline: optionalBoolean(rawArgs, 'multiline'), - '-i': optionalBoolean(rawArgs, '-i'), - '-n': optionalBoolean(rawArgs, '-n'), - '-A': optionalInteger(rawArgs, '-A', 0, Number.MAX_SAFE_INTEGER), - '-B': optionalInteger(rawArgs, '-B', 0, Number.MAX_SAFE_INTEGER), - '-C': optionalInteger(rawArgs, '-C', 0, Number.MAX_SAFE_INTEGER), - }, - }; - if (this.active.has(searchId) || this.active.size >= PROCESS_MAX_ACTIVE_SEARCHES) throw new RpcError(RpcErrorCode.InvalidRequest, 'search ID is active or too many active searches'); - const controller = new AbortController(); - const done = kind === 'glob' ? searchGlob(this.process, request, controller.signal) : searchGrep(this.process, request, controller.signal); - this.active.set(searchId, { controller, done }); - try { - return await done; - } finally { - this.active.delete(searchId); - } - } - - cancel(rawParams: unknown): void { - const params = requireParams(rawParams); - this.active.get(requireString(params, 'searchId'))?.controller.abort(); - } - - async dispose(): Promise { - const pending = [...this.active.values()]; - for (const search of pending) search.controller.abort(); - await Promise.allSettled(pending.map((search) => search.done)); - } -} diff --git a/packages/agent-core-v2/src/remote/server/stdioHost.ts b/packages/agent-core-v2/src/remote/server/stdioHost.ts index 8936cf5c33a..78515f49214 100644 --- a/packages/agent-core-v2/src/remote/server/stdioHost.ts +++ b/packages/agent-core-v2/src/remote/server/stdioHost.ts @@ -17,13 +17,6 @@ import { FS_GET_METADATA_METHOD, FS_READ_DIRECTORY_METHOD, FS_READ_FILE_METHOD, - FS_READ_TEXT_PAGE_METHOD, - FS_CANCEL_READ_METHOD, - FS_SNAPSHOT_FILES_METHOD, - FS_MATCH_FILES_METHOD, - FS_FIND_GIT_WORK_TREE_METHOD, - FS_DISCOVER_INSTRUCTIONS_METHOD, - FS_EDIT_TEXT_METHOD, FS_REMOVE_METHOD, FS_RENAME_METHOD, FS_WALK_METHOD, @@ -35,8 +28,6 @@ import { PROCESS_FLOW_METHOD, PROCESS_SIGNAL_METHOD, PROCESS_START_METHOD, - PROCESS_SEARCH_METHOD, - PROCESS_CANCEL_SEARCH_METHOD, PROCESS_WRITE_METHOD, type InitializeResult, type RemoteEnvironmentInfo, @@ -44,8 +35,6 @@ import { import { HostFileSystem } from '#/os/backends/node-local/hostFsService'; import { HostProcessService } from '#/os/backends/node-local/hostProcessService'; -import { FileOperationsHandler } from './fileOperationsHandler'; -import { SearchHandler } from './searchHandler'; import { FsHandler } from './fsHandler'; import { ProcessManager } from './processManager'; @@ -63,8 +52,6 @@ type Handler = (params: unknown) => Promise; const CONTROL_METHODS: ReadonlySet = new Set([ PING_METHOD, - FS_READ_TEXT_PAGE_METHOD, - PROCESS_SEARCH_METHOD, PROCESS_WRITE_METHOD, PROCESS_SIGNAL_METHOD, ]); @@ -146,10 +133,7 @@ export class StdioHost { private readonly decoder = new LineFrameDecoder(); private readonly writer: OutboundWriter; private readonly processManager: ProcessManager; - private readonly searchHandler = new SearchHandler(new HostProcessService()); - private readonly fs = new HostFileSystem(); - private readonly fsHandler = new FsHandler(this.fs); - private readonly fileOperations = new FileOperationsHandler(this.fs); + private readonly fsHandler = new FsHandler(new HostFileSystem()); private readonly handlers: ReadonlyMap; private state: 'pre-init' | 'ready' = 'pre-init'; private inFlight = 0; @@ -177,12 +161,6 @@ export class StdioHost { const pm = this.processManager; this.handlers = new Map([ [PING_METHOD, () => Promise.resolve({})], - [FS_FIND_GIT_WORK_TREE_METHOD, (p) => this.fileOperations.findGitWorkTree(p)], - [FS_DISCOVER_INSTRUCTIONS_METHOD, (p) => this.fileOperations.discoverInstructions(p)], - [FS_MATCH_FILES_METHOD, (p) => this.fileOperations.matchFiles(p)], - [FS_SNAPSHOT_FILES_METHOD, (p) => this.fileOperations.snapshotFiles(p)], - [FS_READ_TEXT_PAGE_METHOD, (p) => this.fileOperations.readTextPage(p)], - [FS_EDIT_TEXT_METHOD, (p) => this.fileOperations.editText(p)], [FS_READ_FILE_METHOD, (p) => fs.readFile(p)], [FS_WRITE_FILE_METHOD, (p) => fs.writeFile(p)], [FS_CREATE_DIRECTORY_METHOD, (p) => fs.createDirectory(p)], @@ -192,7 +170,6 @@ export class StdioHost { [FS_WALK_METHOD, (p) => fs.walk(p)], [FS_REMOVE_METHOD, (p) => fs.remove(p)], [FS_RENAME_METHOD, (p) => fs.rename(p)], - [PROCESS_SEARCH_METHOD, (p) => this.searchHandler.search(p)], [PROCESS_START_METHOD, (p) => pm.start(p)], [PROCESS_WRITE_METHOD, (p) => pm.write(p)], [PROCESS_SIGNAL_METHOD, (p) => pm.signal(p)], @@ -257,14 +234,6 @@ export class StdioHost { } if (isNotification(message)) { - if (message.method === FS_CANCEL_READ_METHOD) { - try { this.fileOperations.cancelRead(message.params); } catch (error) { this.violation(error); } - return; - } - if (message.method === PROCESS_CANCEL_SEARCH_METHOD) { - try { this.searchHandler.cancel(message.params); } catch (error) { this.violation(error); } - return; - } if (message.method === PROCESS_FLOW_METHOD) { this.onProcessFlow(message.params); return; @@ -316,7 +285,7 @@ export class StdioHost { ); return; } - const lane: Lane = message.method === FS_READ_FILE_METHOD || message.method === FS_WALK_METHOD || message.method === FS_READ_TEXT_PAGE_METHOD || message.method === FS_SNAPSHOT_FILES_METHOD || message.method === FS_MATCH_FILES_METHOD || message.method === PROCESS_SEARCH_METHOD + const lane: Lane = message.method === FS_READ_FILE_METHOD || message.method === FS_WALK_METHOD ? 'data' : 'control'; if (CONTROL_METHODS.has(message.method)) { @@ -397,8 +366,7 @@ export class StdioHost { this.shuttingDown = true; this.options.log(`shutdown: ${reason}`); this.waiting.length = 0; - this.fsHandler.dispose(); - await Promise.all([this.processManager.terminateAll(), this.searchHandler.dispose(), this.fileOperations.dispose()]); + await this.processManager.terminateAll(); this.writer.destroy(); this.options.output.end(); this.resolveDone(); diff --git a/packages/agent-core-v2/src/tool/output-accumulator.ts b/packages/agent-core-v2/src/tool/output-accumulator.ts index 4211def91a9..bb619e5652d 100644 --- a/packages/agent-core-v2/src/tool/output-accumulator.ts +++ b/packages/agent-core-v2/src/tool/output-accumulator.ts @@ -1 +1,97 @@ -export * from '#/_base/text/outputAccumulator'; +import { sliceOnCodePointBoundary } from '#/_base/text/code-points'; +import { + DEFAULT_TOOL_RESULT_MAX_CHARS, + DEFAULT_TOOL_RESULT_MAX_RETAINED_CHARS, + type ExecutableToolErrorResult, + type ExecutableToolSuccessResult, + type ToolResultSpill, +} from './toolContract'; + +export type ToolOutputAccumulatorResult = ( + | ExecutableToolErrorResult + | ExecutableToolSuccessResult +) & { + readonly output: string; + readonly brief?: string; +}; + +export class ToolOutputAccumulator { + private readonly buffer: string[] = []; + private retainedChars = 0; + private totalCharsValue = 0; + private saturated = false; + + get nChars(): number { + return this.retainedChars; + } + + get totalChars(): number { + return this.totalCharsValue; + } + + write(text: string): void { + this.totalCharsValue += text.length; + if (this.saturated) return; + const remainingRetention = DEFAULT_TOOL_RESULT_MAX_RETAINED_CHARS - this.retainedChars; + const kept = + text.length <= remainingRetention + ? text + : sliceOnCodePointBoundary(text, remainingRetention); + if (kept.length !== text.length) this.saturated = true; + this.buffer.push(kept); + this.retainedChars += kept.length; + } + + ok(message = '', options: { readonly brief?: string } = {}): ToolOutputAccumulatorResult { + let finalMessage = message; + if (finalMessage.length > 0 && !finalMessage.endsWith('.')) { + finalMessage += '.'; + } + const output = this.buffer.join(''); + return { + isError: false, + output: output.length === 0 ? finalMessage : output, + brief: options.brief, + spill: this.completionSpill(finalMessage), + }; + } + + error( + message: string, + options: { readonly brief?: string } = {}, + ): ToolOutputAccumulatorResult { + const output = this.buffer.join(''); + return { + isError: true, + output: + message.length === 0 + ? output + : output.length === 0 + ? message + : output.endsWith('\n') + ? `${output}${message}` + : `${output}\n${message}`, + brief: options.brief, + spill: this.retentionSpill(message), + }; + } + + private retentionSpill(suffix?: string): ToolResultSpill | undefined { + if (this.totalCharsValue <= this.retainedChars) return undefined; + return { + totalChars: this.totalCharsValue, + suffix: suffix !== undefined && suffix.length > 0 ? suffix : undefined, + }; + } + + private completionSpill(suffix: string): ToolResultSpill | undefined { + const retentionSpill = this.retentionSpill(); + if (retentionSpill !== undefined) { + return suffix.length > 0 ? { ...retentionSpill, suffix } : retentionSpill; + } + if (suffix.length === 0 || this.totalCharsValue <= DEFAULT_TOOL_RESULT_MAX_CHARS) { + return undefined; + } + return { suffix }; + } +} diff --git a/packages/agent-core-v2/src/tool/path-access.ts b/packages/agent-core-v2/src/tool/path-access.ts index 3bdf69cba72..7e11c46313f 100644 --- a/packages/agent-core-v2/src/tool/path-access.ts +++ b/packages/agent-core-v2/src/tool/path-access.ts @@ -1,4 +1,3 @@ -import { isSensitiveFile, isWithinDirectory } from '#/_base/fs/pathFilters'; import * as pathe from 'pathe'; import { @@ -8,13 +7,81 @@ import { } from '#/_base/execEnv/shellPathBridge'; import type { IHostEnvironment } from '#/os/interface/hostEnvironment'; -export { isSensitiveFile, SENSITIVE_DOT_VARIANT_SUFFIXES, isWithinDirectory } from '#/_base/fs/pathFilters'; - export interface WorkspaceConfig { readonly workspaceDir: string; readonly additionalDirs: readonly string[]; } +const SENSITIVE_BASENAMES = new Set([ + '.env', + 'id_rsa', + 'id_ed25519', + 'id_ecdsa', + 'credentials', +]); + +const SENSITIVE_PATH_SUFFIXES = [ + ['.aws', 'credentials'], + ['.gcp', 'credentials'], +]; + +const ENV_PREFIX = '.env.'; +const ENV_EXEMPTIONS = new Set(['.env.example', '.env.sample', '.env.template']); + +const SENSITIVE_BASENAME_PREFIXES = ['id_rsa', 'id_ed25519', 'id_ecdsa', 'credentials']; +const PUBLIC_KEY_BASENAMES = new Set(['id_rsa.pub', 'id_ed25519.pub', 'id_ecdsa.pub']); +export const SENSITIVE_DOT_VARIANT_SUFFIXES = [ + '.bak', + '.backup', + '.copy', + '.disabled', + '.key', + '.old', + '.orig', + '.pem', + '.save', + '.tmp', +] as const; +const SENSITIVE_DOT_VARIANT_SUFFIX_SET = new Set(SENSITIVE_DOT_VARIANT_SUFFIXES); + +function comparable(path: string): string { + return path.toLowerCase(); +} + +export function isSensitiveFile(path: string): boolean { + const name = pathe.basename(path); + const comparableName = comparable(name); + const comparablePath = comparable(path); + + if (ENV_EXEMPTIONS.has(comparableName)) return false; + if (PUBLIC_KEY_BASENAMES.has(comparableName)) return false; + if (SENSITIVE_BASENAMES.has(comparableName)) return true; + if (comparableName.startsWith(ENV_PREFIX)) return true; + + for (const prefix of SENSITIVE_BASENAME_PREFIXES) { + if (comparableName === prefix) return true; + if (comparableName.length > prefix.length && comparableName.startsWith(prefix)) { + const suffix = comparableName.slice(prefix.length); + const next = suffix[0]; + if (next === '-' || next === '_') return true; + if (next === '.' && SENSITIVE_DOT_VARIANT_SUFFIX_SET.has(suffix)) return true; + } + } + + for (const suffixParts of SENSITIVE_PATH_SUFFIXES) { + const suffix = suffixParts.join('/'); + const comparableSuffix = comparable(suffix); + if ( + comparablePath.endsWith(`/${comparableSuffix}`) || + comparablePath.includes(`/${comparableSuffix}/`) + ) { + return true; + } + } + + return false; +} + export type PathClass = 'posix' | 'win32'; export type PathSecurityCode = 'PATH_OUTSIDE_WORKSPACE' | 'PATH_SENSITIVE' | 'PATH_INVALID'; export type PathAccessOperation = 'read' | 'write' | 'search'; @@ -97,6 +164,20 @@ export function canonicalizePath( return pathe.normalize(abs); } +export function isWithinDirectory( + candidate: string, + base: string, + pathClass: PathClass = DEFAULT_PATH_CLASS, +): boolean { + const nc = pathe.normalize(candidate); + const nb = pathe.normalize(base); + const comparableCandidate = pathClass === 'win32' ? nc.toLowerCase() : nc; + const comparableBase = pathClass === 'win32' ? nb.toLowerCase() : nb; + if (comparableCandidate === comparableBase) return true; + const prefix = comparableBase.endsWith('/') ? comparableBase : comparableBase + '/'; + return comparableCandidate.startsWith(prefix); +} + export function isWithinWorkspace( candidate: string, config: WorkspaceConfig, diff --git a/packages/agent-core-v2/src/tool/toolContract.ts b/packages/agent-core-v2/src/tool/toolContract.ts index 9beae270208..d8c29fe543d 100644 --- a/packages/agent-core-v2/src/tool/toolContract.ts +++ b/packages/agent-core-v2/src/tool/toolContract.ts @@ -1,11 +1,18 @@ -import type { ToolResultSpill } from '#/_base/text/outputLimits'; import type { ContentPart, SystemMessage, ToolCall, ToolDescription as Tool, UserMessage } from '#human/llm/message'; import type { LLMRequestTrace } from '#/llm-adapter/contract/request-trace'; import type { ToolInputDisplay } from '#/tool/toolInputDisplay'; export type ExecutableToolOutput = string | ContentPart[]; -export { DEFAULT_TOOL_RESULT_MAX_CHARS, DEFAULT_TOOL_RESULT_MAX_RETAINED_CHARS, type ToolResultSpill } from '#/_base/text/outputLimits'; +export const DEFAULT_TOOL_RESULT_MAX_CHARS = 50_000; + +export const DEFAULT_TOOL_RESULT_MAX_RETAINED_CHARS = 10_000_000; + +export interface ToolResultSpill { + readonly outputPath?: string; + readonly totalChars?: number; + readonly suffix?: string; +} export type ToolDeliveryKind = 'steer'; diff --git a/packages/agent-core-v2/src/workspace/workspaceFs/fsService.ts b/packages/agent-core-v2/src/workspace/workspaceFs/fsService.ts index 39a9e4c5583..7a8e142e35c 100644 --- a/packages/agent-core-v2/src/workspace/workspaceFs/fsService.ts +++ b/packages/agent-core-v2/src/workspace/workspaceFs/fsService.ts @@ -1,4 +1,3 @@ -import { matchFiles } from '#/os/operations/matchFiles'; import { type FsDiffRequest, type FsDiffResponse, @@ -1261,7 +1260,7 @@ export class WorkspaceFsService implements IWorkspaceFsService { signal: AbortSignal, startedAt: number, ): Promise { - compileGrepPattern(req); + const re = compileGrepPattern(req); const files: FsGrepFileHit[] = []; let filesScanned = 0; @@ -1293,25 +1292,49 @@ export class WorkspaceFsService implements IWorkspaceFsService { throw new Error2(ErrorCodes.FS_GREP_TIMEOUT, `grep timed out after ${Date.now() - startedAt}ms`); } - for (let offset = 0; offset < filePaths.length; offset += 128) { - const expired = signal.aborted || Date.now() >= deadline; - if ((expired && filesScanned > 0) || filesScanned >= req.max_files) { + for (const file of filePaths) { + if ((signal.aborted || Date.now() >= deadline) && filesScanned > 0) { truncated = true; break; } - const options = { ...req, max_files: req.max_files - filesScanned, max_total_matches: req.max_total_matches - totalMatches }; - const batch = filePaths.slice(offset, offset + (expired ? 1 : 128)); - truncated ||= expired; - const remainingMs = Math.max(1, deadline - Date.now()); - let result = this.hostFs.operations === undefined - ? await matchFiles(this.hostFs, batch, options, remainingMs, signal) - : await this.hostFs.operations.matchFiles(batch, options, remainingMs); - if (result.responseTooLarge) result = await matchFiles(this.hostFs, batch, options, Math.max(1, deadline - Date.now()), signal); - files.push(...result.files); - filesScanned += result.files_scanned; - totalMatches += result.files.reduce((sum, file) => sum + file.matches.length, 0); - truncated ||= result.truncated; - if (result.truncated || totalMatches >= req.max_total_matches) break; + if (filesScanned >= req.max_files) { + truncated = true; + break; + } + filesScanned += 1; + let content: string; + try { + content = await this.hostFs.readText(file.absolute); + } catch { + continue; + } + const lines = content.split(/\r?\n/); + const matches: FsGrepMatch[] = []; + for (let i = 0; i < lines.length; i++) { + const line = lines[i]!; + re.lastIndex = 0; + const m = re.exec(line); + if (m === null) continue; + if (matches.length >= req.max_matches_per_file) break; + const before: string[] = []; + for (let k = Math.max(0, i - req.context_lines); k < i; k++) { + before.push(lines[k] ?? ''); + } + const after: string[] = []; + for (let k = i + 1; k < Math.min(lines.length, i + 1 + req.context_lines); k++) { + after.push(lines[k] ?? ''); + } + matches.push({ line: i + 1, col: m.index + 1, text: line, before, after }); + totalMatches += 1; + if (totalMatches >= req.max_total_matches) { + truncated = true; + break; + } + } + if (matches.length > 0) { + files.push({ path: file.path, matches }); + } + if (totalMatches >= req.max_total_matches) break; } return { files, files_scanned: filesScanned, truncated, elapsed_ms: Date.now() - startedAt }; diff --git a/packages/agent-core-v2/src/workspace/workspaceFs/internal/fsSearch.ts b/packages/agent-core-v2/src/workspace/workspaceFs/internal/fsSearch.ts index 755048e9191..1444ca48165 100644 --- a/packages/agent-core-v2/src/workspace/workspaceFs/internal/fsSearch.ts +++ b/packages/agent-core-v2/src/workspace/workspaceFs/internal/fsSearch.ts @@ -1,6 +1,6 @@ import picomatch from 'picomatch'; -export { compileGrepPattern } from '#/os/operations/matchFiles'; +import type { FsGrepRequest } from '../fs'; export function computeFuzzyScore(name: string, queryLower: string): number { if (queryLower.length === 0) return 0; @@ -83,6 +83,16 @@ function globSegmentsMatchPrefix(globSegments: readonly string[], relSegments: r return dp[0] === 1; } +export function compileGrepPattern(req: FsGrepRequest): RegExp { + const flags = req.case_sensitive ? 'g' : 'gi'; + const body = req.regex ? req.pattern : escapeRegExp(req.pattern); + return new RegExp(body, flags); +} + +function escapeRegExp(s: string): string { + return s.replaceAll(/[.*+?^${}()|[\]\\]/g, '\\$&'); +} + export function stripTrailingNewline(s: string): string { if (s.endsWith('\r\n')) return s.slice(0, -2); if (s.endsWith('\n')) return s.slice(0, -1); diff --git a/packages/agent-core-v2/test/features/fileHistory/fileHistory.test.ts b/packages/agent-core-v2/test/features/fileHistory/fileHistory.test.ts index 3c7382412fe..3ff45c97299 100644 --- a/packages/agent-core-v2/test/features/fileHistory/fileHistory.test.ts +++ b/packages/agent-core-v2/test/features/fileHistory/fileHistory.test.ts @@ -91,9 +91,9 @@ describe('AgentFileHistoryService', () => { disposables.dispose(); }); - function stubEnvironment(remoteShape = false, fs: IHostFileSystem = hostFs(remoteShape)): IAgentEnvironmentService { + function stubEnvironment(remoteShape = false): IAgentEnvironmentService { const environment = { - fs, + fs: hostFs(remoteShape), path: posix, workspace: { mapRoots: (roots: unknown) => roots }, }; @@ -127,7 +127,7 @@ describe('AgentFileHistoryService', () => { }); } - function createService(agentId = 'main', remoteShape = false, fs?: IHostFileSystem): AgentFileHistoryService { + function createService(agentId = 'main', remoteShape = false): AgentFileHistoryService { const ctx = agentId === scopeCtx.agentId ? scopeCtx @@ -151,7 +151,7 @@ describe('AgentFileHistoryService', () => { executorEvents.executor, eventBus, ix.get(IEventDispatcher), - stubEnvironment(remoteShape, fs), + stubEnvironment(remoteShape), blobs, workspace, { @@ -216,17 +216,6 @@ describe('AgentFileHistoryService', () => { return bytes === undefined ? undefined : decoder.decode(bytes); } - it('treats a failed snapshot RPC as unreadable without recording a deletion', async () => { - const fs = hostFs(); - Object.defineProperty(fs, 'operations', { value: { snapshotFiles: async () => { throw new Error('connection closed'); } } }); - const service = createService('main', true, fs); - setFile('/ws/a.txt', 'original'); - startTurn(1); - await fireEdit(service, '/ws/a.txt', 1); - expect(service.history().checkpoints).toEqual([]); - expect(await blobs.list(scopeCtx.scope(), 'file-history/')).toEqual([]); - }); - it('backs up pre-edit content on first touch and versions changes at the next turn boundary', async () => { const service = createService(); setFile('/ws/a.txt', 'one\ntwo\n'); diff --git a/packages/agent-core-v2/test/lint/import-boundaries.test.ts b/packages/agent-core-v2/test/lint/import-boundaries.test.ts index 7f060af0105..b7c14b5ec7b 100644 --- a/packages/agent-core-v2/test/lint/import-boundaries.test.ts +++ b/packages/agent-core-v2/test/lint/import-boundaries.test.ts @@ -227,19 +227,6 @@ import { HostEnvironmentService } from '#/os/backends/node-local/hostEnvironment expect(violations.every((violation) => violation.message.includes('remote executor stays light'))).toBe(true); }); - it('keeps shared host operations free of engine services and transport', () => { - const violations = checkSource( - `import type { IHostFileSystem } from '#/os/interface/hostFileSystem'; -import { decodeTextWithErrors } from '#/_base/execEnv/decodeText'; -import { IFileEditService } from '#/app/edit/fileEdit'; -import { RemoteExecConnection } from '#/remote/client/connection'; -import { IAgentProfileService } from '#/agent/profile/profile';`, - at('os/operations', 'readTextPage.ts'), - ); - expect(violations).toHaveLength(3); - expect(violations.every((violation) => violation.message.includes('shared host operations stay DI-free'))).toBe(true); - }); - it('keeps the remote os backend free of the remote transport and engine domains', () => { const violations = checkSource( `import type { ExecChannel } from '#/remote/protocol/channel'; diff --git a/packages/agent-core-v2/test/os/backends/node-local/tools/glob.test.ts b/packages/agent-core-v2/test/os/backends/node-local/tools/glob.test.ts index dd7099e4f2c..1cac4a398fe 100644 --- a/packages/agent-core-v2/test/os/backends/node-local/tools/glob.test.ts +++ b/packages/agent-core-v2/test/os/backends/node-local/tools/glob.test.ts @@ -459,13 +459,14 @@ describe('GlobTool', () => { }); it('does not present a capped capture as a complete search when returning all matches', async () => { - const { tool } = makeTool(workspace, { - exec: execReturning(`/workspace/${'x'.repeat(200)}.ts\n`.repeat(50_000)), - }); + const exec = vi.fn(async () => fakeProcess(`/workspace/${'x'.repeat(200)}.ts\n`.repeat(50_000))); + const { tool } = makeTool(workspace, { exec }); const result = await execute(tool, GlobInputSchema.parse({ pattern: '*.ts', head_limit: 0 })); expect(result.output).toContain('stdout truncated'); expect(result.output).toContain('collected matches (partial result set)'); expect(result.output).not.toContain('Continue with'); + const process = await exec.mock.results[0]!.value as IHostProcess; + expect(process.kill).toHaveBeenCalledWith('SIGTERM'); }); it('keeps the timeout warning when the requested page has no collected matches', async () => { diff --git a/packages/agent-core-v2/test/os/backends/node-local/tools/read.test.ts b/packages/agent-core-v2/test/os/backends/node-local/tools/read.test.ts index 87c84919874..4769a8a6253 100644 --- a/packages/agent-core-v2/test/os/backends/node-local/tools/read.test.ts +++ b/packages/agent-core-v2/test/os/backends/node-local/tools/read.test.ts @@ -455,6 +455,25 @@ describe('ReadTool', () => { }); }); + it('uses a whole-file capability without separate stat, sniff and line reads', async () => { + const { fs, stat, readBytes, readLines } = createSpiedFs('alpha\nbeta\n'); + const bytes = Buffer.from('alpha\nbeta\n'); + const readWholeFile = vi.fn(async () => ({ + stat: { isFile: true, isDirectory: false, size: bytes.length }, + bytes, + })); + fs.readWholeFile = readWholeFile; + const tool = createReadTool(fs, createTestEnv(), PERMISSIVE_WORKSPACE); + + const result = await execute(tool, { path: '/tmp/a.txt' }); + + expect(result.output).toBe('1\talpha\n2\tbeta'); + expect(readWholeFile).toHaveBeenCalledWith('/tmp/a.txt'); + expect(stat).not.toHaveBeenCalled(); + expect(readBytes).not.toHaveBeenCalled(); + expect(readLines).not.toHaveBeenCalled(); + }); + it('stats the resolved target so symlinked files stay readable', async () => { const { fs, stat } = createSpiedFs('alpha\n'); const tool = createReadTool(fs, createTestEnv(), PERMISSIVE_WORKSPACE); diff --git a/packages/agent-core-v2/test/os/backends/node-local/tools/write.test.ts b/packages/agent-core-v2/test/os/backends/node-local/tools/write.test.ts index e1565adc2ff..52a8d5392c1 100644 --- a/packages/agent-core-v2/test/os/backends/node-local/tools/write.test.ts +++ b/packages/agent-core-v2/test/os/backends/node-local/tools/write.test.ts @@ -257,6 +257,19 @@ describe('WriteTool', () => { expect(writeText).toHaveBeenCalledWith('/tmp/missing-dir/file.txt', 'data'); }); + it('uses the combined parent-and-write operation when available', async () => { + const { tool, fs, stat, writeText } = makeTool(); + const writeTextWithParent = vi.fn(async () => {}); + fs.writeTextWithParent = writeTextWithParent; + + const result = await execute(tool, { path: '/tmp/missing-dir/file.txt', content: 'data' }); + + expect(result.isError).toBeUndefined(); + expect(writeTextWithParent).toHaveBeenCalledWith('/tmp/missing-dir/file.txt', 'data', 'overwrite', '/tmp/missing-dir'); + expect(stat).not.toHaveBeenCalled(); + expect(writeText).not.toHaveBeenCalled(); + }); + it('surfaces mkdir failures when a missing parent cannot be created', async () => { const enoent = Object.assign(new Error('ENOENT: no such file or directory'), { code: 'ENOENT', diff --git a/packages/agent-core-v2/test/remote/fs.test.ts b/packages/agent-core-v2/test/remote/fs.test.ts index 49cc44a3de8..f5689d2af95 100644 --- a/packages/agent-core-v2/test/remote/fs.test.ts +++ b/packages/agent-core-v2/test/remote/fs.test.ts @@ -1,13 +1,10 @@ -import { createHash } from 'node:crypto'; -import { HostFileSystem } from '#/os/backends/node-local/hostFsService'; -import { readTextPage } from '#/os/operations/readTextPage'; -import { TEXT_PAGE_MAX_REMOTE_CHARS, type TextPageOptions } from '#/os/interface/fileOperations'; -import { mkdir, mkdtemp, readFile, rm, writeFile, utimes } from 'node:fs/promises'; +import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest'; +import { FileEditService } from '#/app/edit/fileEditService'; import { HostFsError } from '#/os/interface/hostFsErrors'; import { HOST_WALK_MAX_IGNORE_BYTES } from '#/os/interface/hostFsWalk'; @@ -46,241 +43,52 @@ describe('fs group over a subprocess loopback', () => { await rm(workDir, { recursive: true, force: true }); }); - it('fetches exactly the requested sniff bytes', async () => { - const path = join(workDir, 'sniff.txt'); - await writeFile(path, 'x'.repeat(2 * 1024 * 1024)); + it('reads a small whole file in one RPC', async () => { + const path = join(workDir, 'whole.txt'); + await writeFile(path, 'alpha\nbeta\n'); const calls = vi.spyOn(connection, 'call'); try { - expect((await fs.readBytes(path, 512)).byteLength).toBe(512); - expect(calls.mock.calls).toEqual([['fs/readFile', { path, offset: 0, maxBytes: 512 }]]); - const reply = await calls.mock.results[0]!.value; - expect(Buffer.byteLength(JSON.stringify(reply))).toBeLessThan(1024); - } finally { calls.mockRestore(); } - }); - - it('reads a bounded page of a large file in one RPC', async () => { - const path = join(workDir, 'page.txt'); - await writeFile(path, 'hello world\n'.repeat(1_000_000)); - const calls = vi.spyOn(connection, 'call'); - try { - const result = await fs.operations.readTextPage(path, pageOptions(path, { n_lines: 3 })); - expect(result.output).toBe('1\thello world\n2\thello world\n3\thello world'); - expect(result.note).toContain('Total lines in file: 1000000'); - expect(calls.mock.calls.map(([method]) => method)).toEqual(['fs/readTextPage']); - expect(Buffer.byteLength(JSON.stringify(await calls.mock.results[0]!.value))).toBeLessThan(1024); - } finally { calls.mockRestore(); } - }); - - it.each([ - ['crlf', Buffer.from('alpha\r\nbeta\r\ngamma\r\n')], - ['mixed', Buffer.from('alpha\r\nbeta\ngam\rma')], - ['utf16', Buffer.from('\uFEFFalpha\r\nbeta😀\r\ngamma', 'utf16le')], - ['binary', Buffer.from('alpha\n\0beta')], - ['long', Buffer.from('😀'.repeat(4000) + '\nlast\n')], - ])('preserves local Read semantics for %s', async (name, bytes) => { - const path = join(workDir, `${name}.txt`); - await writeFile(path, bytes); - const local = new HostFileSystem(); - const source = { name: path, stat: () => local.stat(path), readBytes: (n?: number) => local.readBytes(path, n), readLines: () => local.readLines(path, { errors: 'strict' }) }; - for (const args of [{ n_lines: 1 }, { line_offset: -2, n_lines: 1 }, { line_offset: -3 }, { column_offset: 2, n_lines: 1 }]) { - const options = pageOptions(path, args); - expect(await fs.operations.readTextPage(path, options)).toEqual(await readTextPage(source, options)); + const result = await fs.readWholeFile(path); + expect(Buffer.from(result?.bytes ?? []).toString('utf8')).toBe('alpha\nbeta\n'); + expect(result?.stat).toEqual({ isFile: true, isDirectory: false, size: 11 }); + expect(calls.mock.calls.map(([method]) => method)).toEqual(['fs/readFile']); + } finally { + calls.mockRestore(); } }); - it('preserves configured Read budgets larger than the remote page limit', async () => { - const path = join(workDir, 'large-read-budget.txt'); - await writeFile(path, 'first\nsecond\n'); - const calls = vi.spyOn(connection, 'call'); - try { - const options = { ...pageOptions(path), maxChars: TEXT_PAGE_MAX_REMOTE_CHARS + 1, maxCharsLimit: TEXT_PAGE_MAX_REMOTE_CHARS + 1 }; - expect((await fs.operations.readTextPage(path, options)).output).toBe('1\tfirst\n2\tsecond'); - expect(calls.mock.calls.some(([method]) => method === 'fs/readTextPage')).toBe(false); - expect(calls.mock.calls.some(([method]) => method === 'fs/readFile')).toBe(true); - } finally { calls.mockRestore(); } - }); - - it('keeps concurrent Read calls working above the executor scan limit', async () => { - const path = join(workDir, 'concurrent-read.txt'); - await writeFile(path, 'first\nsecond\n'); - const calls = vi.spyOn(connection, 'call'); - try { - const results = await Promise.all(Array.from({ length: 12 }, () => fs.operations.readTextPage(path, pageOptions(path)))); - expect(results.every((result) => !result.isError && result.output === '1\tfirst\n2\tsecond')).toBe(true); - expect(calls.mock.calls.filter(([method]) => method === 'fs/readTextPage')).toHaveLength(8); - expect(calls.mock.calls.some(([method]) => method === 'fs/readFile')).toBe(true); - } finally { calls.mockRestore(); } - }); - - it('preserves Read and Edit error results when an operation RPC fails', async () => { - const path = join(workDir, 'rpc-failure.txt'); - const calls = vi.spyOn(connection, 'call').mockRejectedValue(new Error('connection closed')); - try { - expect(await fs.operations.readTextPage(path, pageOptions(path))).toEqual({ isError: true, output: 'connection closed' }); - expect(await fs.operations.editText({ path, displayPath: 'rpc-failure.txt', old_string: 'old', new_string: 'new', replace_all: false })).toEqual({ ok: false, error: 'connection closed' }); - } finally { calls.mockRestore(); } - }); - - it('edits on the execution host without downloading or uploading the full file', async () => { - const path = join(workDir, 'edit-page.txt'); - const before = 'a'.repeat(3 * 1024 * 1024) + '\r\nold value\r\n'; - await writeFile(path, before); + it('edits a small file with one read and one write RPC', async () => { + const path = join(workDir, 'edit.txt'); + await writeFile(path, 'before\nold value\n'); + const editor = new FileEditService(fs); const calls = vi.spyOn(connection, 'call'); try { - expect(await fs.operations.editText({ path, displayPath: 'edit-page.txt', old_string: 'old value', new_string: 'new value\nnext', replace_all: false })).toEqual({ ok: true, count: 1 }); - expect(calls.mock.calls.map(([method]) => method)).toEqual(['fs/editText']); - expect(Buffer.byteLength(JSON.stringify(calls.mock.calls))).toBeLessThan(1024); - expect(await readFile(path, 'utf8')).toBe(before.replace('old value', 'new value\r\nnext')); - const failure = await fs.operations.editText({ path, displayPath: 'edit-page.txt', old_string: 'absent', new_string: '', replace_all: false }); - expect(failure.ok).toBe(false); - } finally { calls.mockRestore(); } + const result = await editor.edit({ + path, + displayPath: 'edit.txt', + old_string: 'old value', + new_string: 'new value', + replace_all: false, + }); + expect(result).toEqual({ ok: true, count: 1 }); + expect(calls.mock.calls.map(([method]) => method)).toEqual(['fs/readFile', 'fs/writeFile']); + expect(await readFile(path, 'utf8')).toBe('before\nnew value\n'); + } finally { + calls.mockRestore(); + } }); - it('creates the parent and writes text in a single RPC', async () => { - const path = join(workDir, 'created', 'nested', 'out.txt'); + it('creates the parent and writes text in one RPC', async () => { + const parent = join(workDir, 'created', 'nested'); + const path = join(parent, 'out.txt'); const calls = vi.spyOn(connection, 'call'); try { - const result = await fs.operations.writeText({ path, parent: join(workDir, 'created', 'nested'), displayPath: 'out.txt', content: 'hello', mode: 'overwrite' }); - expect(result).toEqual({ output: 'Wrote 5 bytes to out.txt' }); + await fs.writeTextWithParent(path, 'hello', 'overwrite', parent); expect(calls.mock.calls.map(([method]) => method)).toEqual(['fs/writeFile']); expect(await readFile(path, 'utf8')).toBe('hello'); - } finally { calls.mockRestore(); } - }); - - it('batches snapshots and skips bytes only for a matching content hash', async () => { - const path = join(workDir, 'snapshot.txt'); - const bytes = Buffer.from('before'); - await writeFile(path, bytes); - const knownHash = createHash('sha256').update(bytes).digest('hex'); - const calls = vi.spyOn(connection, 'call'); - try { - const snapshots = await fs.operations.snapshotFiles([{ path, knownHash }, { path: join(workDir, 'missing-snapshot') }], 1024); - expect(snapshots).toEqual([{ kind: 'file', contentHash: knownHash, bytes: undefined }, { kind: 'missing' }]); - expect(calls.mock.calls.map(([method]) => method)).toEqual(['fs/snapshotFiles']); - const stat = await fs.stat(path); - await writeFile(path, 'after!'); - await utimes(path, new Date(), new Date(stat.mtimeMs!)); - const changed = (await fs.operations.snapshotFiles([{ path, knownHash }], 1024))[0]!; - expect(changed.kind).toBe('file'); - if (changed.kind === 'file') expect(Buffer.from(changed.bytes!).toString()).toBe('after!'); - expect(await fs.operations.snapshotFiles([{ path }], 2)).toMatchObject([{ kind: 'oversize', size: 6 }]); - } finally { calls.mockRestore(); } - }); - - it('discovers Git and instruction files without per-directory RPCs or stale negative caching', async () => { - const root = join(workDir, 'probe'); - const nested = join(root, 'a', 'b', 'c'); - await mkdir(nested, { recursive: true }); - await mkdir(join(root, '.git')); - await writeFile(join(root, 'AGENTS.md'), 'root instructions'); - const calls = vi.spyOn(connection, 'call'); - try { - expect((await fs.operations.findGitWorkTree(nested))?.root).toBe(root); - expect(await fs.operations.discoverInstructions([nested, join(nested, 'missing')], [])).toEqual([join(root, 'AGENTS.md')]); - expect(calls.mock.calls.map(([method]) => method)).toEqual(['fs/findGitWorkTree', 'fs/discoverInstructions']); - await writeFile(join(nested, 'AGENTS.md'), 'new instructions'); - expect(await fs.operations.discoverInstructions([nested], [join(root, 'AGENTS.md')])).toContain(join(nested, 'AGENTS.md')); - expect(await fs.operations.findGitWorkTree('/')).toBeNull(); - } finally { calls.mockRestore(); } - }); - - it('pipelines a bounded window of write chunks in order', async () => { - const path = join(workDir, 'pipelined.bin'); - const original = connection.call.bind(connection); - let active = 0; - let peak = 0; - const calls = vi.spyOn(connection, 'call').mockImplementation(async (method, params) => { - if (method !== 'fs/writeFile') return original(method, params); - active += 1; - peak = Math.max(peak, active); - try { - await new Promise((resolve) => { setTimeout(resolve, 10); }); - return await original(method, params); - } finally { active -= 1; } - }); - try { - const bytes = Buffer.concat(Array.from({ length: 9 }, (_, index) => Buffer.alloc(1024 * 1024, index))); - await fs.writeBytes(path, bytes); - expect(peak).toBe(4); - expect((await readFile(path)).equals(bytes)).toBe(true); - } finally { calls.mockRestore(); } - }); - - it('does not append later chunks after an earlier exclusive write fails', async () => { - const path = join(workDir, 'failed-pipeline.txt'); - await writeFile(path, 'original'); - const results = await Promise.allSettled([0, 1, 2].map((chunkIndex) => connection.call('fs/writeFile', { - path, transferId: 'failed-write', chunkIndex, final: chunkIndex === 2, - mode: chunkIndex === 0 ? 'exclusive' : 'append', dataBase64: Buffer.from('changed').toString('base64'), - }))); - expect(results.every((result) => result.status === 'rejected')).toBe(true); - expect(await readFile(path, 'utf8')).toBe('original'); - }); - - it('rejects duplicate and out-of-order chunks without repeating an append', async () => { - const path = join(workDir, 'ordered-write.txt'); - const first = { path, transferId: 'ordered-write', chunkIndex: 0, mode: 'append', dataBase64: Buffer.from('one').toString('base64') }; - await connection.call('fs/writeFile', first); - await expect(connection.call('fs/writeFile', first)).rejects.toThrow('once and in order'); - await expect(connection.call('fs/writeFile', { ...first, chunkIndex: 2 })).rejects.toThrow('once and in order'); - await connection.call('fs/writeFile', { ...first, chunkIndex: 1, final: true, dataBase64: Buffer.from('two').toString('base64') }); - expect(await readFile(path, 'utf8')).toBe('onetwo'); - }); - - it('cancels a remote text scan', async () => { - const path = join(workDir, 'cancel-read.txt'); - await writeFile(path, 'line\n'.repeat(1_000_000)); - const controller = new AbortController(); - const pending = fs.operations.readTextPage(path, pageOptions(path), controller.signal); - const timer = setTimeout(() => { controller.abort(); }, 20); - try { - expect(await pending).toEqual({ isError: true, output: 'Read aborted' }); - } finally { clearTimeout(timer); } - }); - - it('matches a batch of files without transferring nonmatching contents', async () => { - const paths = []; - for (let index = 0; index < 12; index += 1) { - const absolute = join(workDir, `match-${index}.txt`); - await writeFile(absolute, 'no match here\n'.repeat(10_000) + 'needle\n'); - paths.push({ path: `match-${index}.txt`, absolute }); + } finally { + calls.mockRestore(); } - const calls = vi.spyOn(connection, 'call'); - try { - const result = await fs.operations.matchFiles(paths, { pattern: 'needle', regex: false, case_sensitive: true, context_lines: 0, max_files: 100, max_matches_per_file: 10, max_total_matches: 100 }, 30_000); - expect(result.files_scanned).toBe(12); - expect(result.files).toHaveLength(12); - expect(result.files[0]?.matches).toEqual([{ line: 10001, col: 1, text: 'needle', before: [], after: [] }]); - expect(calls.mock.calls.map(([method]) => method)).toEqual(['fs/matchFiles']); - expect(Buffer.byteLength(JSON.stringify(await calls.mock.results[0]!.value))).toBeLessThan(4096); - } finally { calls.mockRestore(); } - }); - - it('requests legacy matching when a remote result would exceed the response budget', async () => { - const path = join(workDir, 'long-match.txt'); - await writeFile(path, 'needle' + 'x'.repeat(8 * 1024 * 1024)); - const result = await fs.operations.matchFiles([{ path: 'long-match.txt', absolute: path }], { pattern: 'needle', regex: false, case_sensitive: true, context_lines: 0, max_files: 1, max_matches_per_file: 1, max_total_matches: 1 }, 30_000); - expect(result).toEqual({ files: [], files_scanned: 0, truncated: false, responseTooLarge: true }); - }); - - it('reuses written and edited bytes only after the executor verifies their hash', async () => { - const path = join(workDir, 'reuse.txt'); - await fs.operations.writeText({ path, parent: workDir, displayPath: 'reuse.txt', content: 'old value', mode: 'overwrite' }); - const calls = vi.spyOn(connection, 'call'); - try { - const written = (await fs.operations.snapshotFiles([{ path }], 1024))[0]!; - expect(written.kind).toBe('file'); - if (written.kind === 'file') expect(Buffer.from(written.bytes!).toString()).toBe('old value'); - expect(JSON.stringify(await calls.mock.results[0]!.value)).not.toContain('dataBase64'); - calls.mockClear(); - await fs.operations.editText({ path, displayPath: 'reuse.txt', old_string: 'old', new_string: 'new', replace_all: false }); - const edited = (await fs.operations.snapshotFiles([{ path }], 1024))[0]!; - if (edited.kind === 'file') expect(Buffer.from(edited.bytes!).toString()).toBe('new value'); - expect(JSON.stringify(await calls.mock.results[1]!.value)).not.toContain('dataBase64'); - await writeFile(path, 'external!'); - const changed = (await fs.operations.snapshotFiles([{ path }], 1024))[0]!; - if (changed.kind === 'file') expect(Buffer.from(changed.bytes!).toString()).toBe('external!'); - } finally { calls.mockRestore(); } }); it('writes a byte stream progressively without retaining the complete input', async () => { @@ -382,6 +190,25 @@ describe('fs protocol semantics', () => { maxResponseBytes: 8 * 1024 * 1024, }; + it('falls back to bounded reads when a whole-file read is rejected', async () => { + const calls: Array<{ maxBytes?: number }> = []; + const fs = new RemoteFileSystem({ + call: async (method, params) => { + if (method !== 'fs/readFile') throw new Error(`unexpected method: ${method}`); + const maxBytes = (params as { maxBytes?: number }).maxBytes; + calls.push({ maxBytes }); + if (maxBytes === undefined) throw new Error('whole read rejected'); + return { dataBase64: Buffer.from('fallback').toString('base64'), eof: true }; + }, + notify: () => {}, + onNotification: () => () => {}, + onDidClose: () => {}, + }); + + expect(await fs.readText('/tmp/file.txt')).toBe('fallback'); + expect(calls).toEqual([{ maxBytes: undefined }, { maxBytes: 1024 * 1024 }]); + }); + it('honors the caller deadline before and during a remote walk', async () => { const calls: Array<{ method: string; timeoutMs?: number }> = []; const fs = new RemoteFileSystem({ @@ -559,7 +386,3 @@ describe('fs protocol semantics', () => { } }, 60_000); }); - -function pageOptions(path: string, args: Partial = {}): TextPageOptions { - return { args: { path, ...args }, maxChars: 2000, maxCharsLimit: 500_000, eventLog: false, imageSupported: false, videoSupported: false, mediaToolAvailable: false }; -} diff --git a/packages/agent-core-v2/test/remote/process.test.ts b/packages/agent-core-v2/test/remote/process.test.ts index 1474225239b..6dfc6125ce0 100644 --- a/packages/agent-core-v2/test/remote/process.test.ts +++ b/packages/agent-core-v2/test/remote/process.test.ts @@ -1,8 +1,5 @@ -import type { SearchRequest } from '#/os/interface/searchOperations'; -import { searchGrep } from '#/os/operations/searchGrep'; -import { searchGlob } from '#/os/operations/searchGlob'; import { randomUUID } from 'node:crypto'; -import { mkdtemp, readFile, realpath, rm, writeFile } from 'node:fs/promises'; +import { mkdtemp, realpath, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; @@ -566,96 +563,3 @@ describe('finished process-group ownership', () => { } finally { manager.dispose(); kill.mockRestore(); vi.useRealTimers(); } }); }); - -describe('search processing on the execution host', () => { - let directory: string; - let connection: RemoteExecConnection; - let processes: RemoteProcessService; - let loopback: Awaited>['loopback']; - - beforeAll(async () => { - directory = await mkdtemp(join(tmpdir(), 'remote-search-')); - ({ connection, loopback } = await connectInProcess()); - processes = new RemoteProcessService(connection, directory, '/bin/bash'); - }); - - afterAll(async () => { - connection.close(); - await loopback.host.done; - await rm(directory, { recursive: true, force: true }); - }); - - it.each(['grep', 'glob'] as const)('returns only the selected %s page and exact total over one RPC', async (kind) => { - const script = join(directory, `large-${kind}`); - const separator = kind === 'grep' ? '\0' : '\n'; - const program = `#!/usr/bin/env node\nprocess.stdout.write(Array.from({length:50000},(_,i)=>${JSON.stringify(directory + '/')}+'file-'+i+'.txt'+${JSON.stringify(separator)}).join(''));`; - await writeFile(script, program, { mode: 0o755 }); - const request: SearchRequest = { kind, args: { pattern: '*', head_limit: 3, offset: 2 }, rgPath: script, roots: [directory], workspaceDir: directory, pathClass: 'posix' }; - const calls = vi.spyOn(connection, 'call'); - let receivedBytes = 0; - const receive = (chunk: Buffer): void => { receivedBytes += chunk.byteLength; }; - loopback.serverOutput.on('data', receive); - try { - const remote = await processes.search(request, new AbortController().signal); - const local = await (kind === 'grep' ? searchGrep : searchGlob)(new HostProcessService(), request, new AbortController().signal); - expect(remote).toEqual(local); - expect(remote.output).toContain('50000'); - expect(remote.output).toContain('file-2.txt'); - expect(remote.output).toContain('file-4.txt'); - expect(remote.output).not.toContain('file-5.txt'); - expect(receivedBytes).toBeLessThan(2048); - expect(calls.mock.calls.map(([method]) => method)).toEqual(['process/search']); - } finally { - calls.mockRestore(); - loopback.serverOutput.off('data', receive); - } - }); - - it.each(['content', 'count_matches', 'files_with_matches'] as const)('preserves real ripgrep %s output including filtering and counts', async (output_mode) => { - const file = join(directory, 'sample.txt'); - await writeFile(file, 'before\nmatch here\nafter\nmatch again\n'); - const request: SearchRequest = { kind: 'grep', args: { pattern: 'match', output_mode, '-C': 1, head_limit: 2 }, rgPath: 'rg', roots: [file], workspaceDir: directory, pathClass: 'posix' }; - const signal = new AbortController().signal; - expect(await processes.search(request, signal)).toEqual(await searchGrep(new HostProcessService(), request, signal)); - }); - - it('keeps search working with more roots than fit the remote operation', async () => { - const script = join(directory, 'many-roots'); - await writeFile(script, `#!/usr/bin/env node\nprocess.stdout.write(${JSON.stringify(directory + '/found.txt\0')});`, { mode: 0o755 }); - const request: SearchRequest = { kind: 'grep', args: { pattern: 'match' }, rgPath: script, roots: Array.from({ length: 65 }, () => directory), workspaceDir: directory, pathClass: 'posix' }; - const calls = vi.spyOn(connection, 'call'); - try { - const result = await processes.search(request, new AbortController().signal); - expect(result.isError).not.toBe(true); - expect(result.output).toContain('found.txt'); - expect(calls.mock.calls.some(([method]) => method === 'process/search')).toBe(false); - expect(calls.mock.calls.some(([method]) => method === 'process/start')).toBe(true); - } finally { calls.mockRestore(); } - }); - - it('keeps concurrent searches working above the executor search limit', async () => { - const script = join(directory, 'concurrent-search'); - await writeFile(script, `#!/usr/bin/env node\nprocess.stdout.write(${JSON.stringify(directory + '/found.txt\0')});`, { mode: 0o755 }); - const request: SearchRequest = { kind: 'grep', args: { pattern: 'match' }, rgPath: script, roots: [directory], workspaceDir: directory, pathClass: 'posix' }; - const calls = vi.spyOn(connection, 'call'); - try { - const results = await Promise.all(Array.from({ length: 12 }, () => processes.search(request, new AbortController().signal))); - expect(results.every((result) => !result.isError && result.output.includes('found.txt'))).toBe(true); - expect(calls.mock.calls.filter(([method]) => method === 'process/search')).toHaveLength(8); - expect(calls.mock.calls.some(([method]) => method === 'process/start')).toBe(true); - } finally { calls.mockRestore(); } - }); - - it('cancels and reaps a running search without streaming output to the controller', async () => { - const script = join(directory, 'slow-search'); - const pidFile = join(directory, 'search.pid'); - await writeFile(script, `#!/usr/bin/env node\nrequire('node:fs').writeFileSync(${JSON.stringify(pidFile)},String(process.pid));setInterval(()=>{},1000);`, { mode: 0o755 }); - const controller = new AbortController(); - const pending = processes.search({ kind: 'grep', args: { pattern: 'match' }, rgPath: script, roots: [directory], workspaceDir: directory, pathClass: 'posix' }, controller.signal); - let pid: number | undefined; - await vi.waitFor(async () => { pid = Number(await readFile(pidFile, 'utf8')); }, { timeout: 5000 }); - controller.abort(); - expect(await pending).toEqual({ isError: true, output: 'Grep aborted' }); - await waitForDeath(pid!); - }); -}); diff --git a/packages/agent-core-v2/test/remote/protocol.test.ts b/packages/agent-core-v2/test/remote/protocol.test.ts index 605114b585e..b9a3826207e 100644 --- a/packages/agent-core-v2/test/remote/protocol.test.ts +++ b/packages/agent-core-v2/test/remote/protocol.test.ts @@ -144,17 +144,6 @@ describe('request call timeout', () => { connection.close(); }); - it('lets cancellable text scans outlive the per-call transport timeout', async () => { - const pipe = createScriptedServer((frame, reply) => { - if (frame.method === INITIALIZE_METHOD) reply({ id: frame.id, result: testInitializeResult() }); - else if (frame.method === 'fs/readTextPage') setTimeout(() => { reply({ id: frame.id, result: { output: 'complete' } }); }, 80); - }); - const connection = await RemoteExecConnection.connect(pipe, { clientName: 'test', clientVersion: '0.0.0', requestCallTimeoutMs: 20 }); - try { - await expect(connection.call('fs/readTextPage', {})).resolves.toEqual({ output: 'complete' }); - } finally { connection.close(); } - }); - it('cancels a process start that timed out, so the late spawn cannot go orphan', async () => { const seen: ScriptedFrame[] = []; const pipe = createScriptedServer((frame, reply) => { diff --git a/packages/agent-core-v2/test/workspace/workspaceFs/fsService.test.ts b/packages/agent-core-v2/test/workspace/workspaceFs/fsService.test.ts index f6baeb8719d..c92d197e568 100644 --- a/packages/agent-core-v2/test/workspace/workspaceFs/fsService.test.ts +++ b/packages/agent-core-v2/test/workspace/workspaceFs/fsService.test.ts @@ -1728,21 +1728,6 @@ describe('WorkspaceFsService.grep', () => { }); }); - it('keeps the legacy matches when the remote matching response is too large', async () => { - const hostFs = fakeFs({ 'src/a.ts': 'hello world\nfoo bar\nhello again\n' }); - const matchFiles = vi.fn(async () => ({ files: [], files_scanned: 0, truncated: false, responseTooLarge: true })); - Object.defineProperty(hostFs, 'operations', { value: { matchFiles } }); - const fs = makeRemoteSession({}, () => ({ stdout: '', exitCode: 1 }), [], 'ssh-dev', '/home/target', hostFs); - const result = await fs.grep({ pattern: 'hello', regex: false, case_sensitive: true, follow_gitignore: false, max_files: 200, max_matches_per_file: 50, max_total_matches: 5000, context_lines: 0 }); - expect(matchFiles).toHaveBeenCalledOnce(); - expect(result.files_scanned).toBe(1); - expect(result.truncated).toBe(false); - expect(result.files).toEqual([{ path: 'src/a.ts', matches: [ - { line: 1, col: 1, text: 'hello world', before: [], after: [] }, - { line: 3, col: 1, text: 'hello again', before: [], after: [] }, - ] }]); - }); - it('uses rg when available and parses its JSON output', async () => { const rgJson = [ JSON.stringify({ type: 'begin', data: { path: { text: 'src/a.ts' } } }),