diff --git a/.github/workflows/contained-source.yml b/.github/workflows/contained-source.yml new file mode 100644 index 0000000000..9bda865e96 --- /dev/null +++ b/.github/workflows/contained-source.yml @@ -0,0 +1,52 @@ +name: Contained source reads +on: + pull_request: + branches: [fork/consolidated] + paths: + - 'codegraph-kernel/src/contained_source.rs' + - 'codegraph-kernel/src/lib.rs' + - 'src/extraction/kernel/loader.ts' + - 'src/mcp/source-scan.ts' + - '__tests__/source-scan.test.ts' + - '__tests__/security.test.ts' + - '__tests__/explore-unmatched-names.test.ts' + - '.github/workflows/contained-source.yml' +permissions: + contents: read +jobs: + source-reads: + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, windows-latest, macos-latest] + runs-on: ${{ matrix.os }} + timeout-minutes: 20 + env: + CARGO_BUILD_JOBS: 2 + CARGO_PROFILE_RELEASE_DEBUG: 0 + steps: + - name: Check out the tested commit + shell: bash + run: | + git init . + git config core.autocrlf false + git remote add origin "https://github.com/$GITHUB_REPOSITORY.git" + git fetch --no-tags --depth=1 origin "$GITHUB_SHA" + git checkout --detach FETCH_HEAD + - name: Use the runner's Node 24 installation + shell: pwsh + run: | + $versions = Get-ChildItem "$env:RUNNER_TOOL_CACHE/node" -Directory + $version = $versions | Where-Object Name -like '24.*' | Sort-Object { [version]$_.Name } -Descending | Select-Object -First 1 + if (!$version) { throw 'Node 24 is missing from the runner tool cache' } + $arch = [System.Runtime.InteropServices.RuntimeInformation]::OSArchitecture.ToString().ToLowerInvariant() + $nodeDir = Join-Path $version.FullName $arch + if (!$IsWindows) { $nodeDir = Join-Path $nodeDir 'bin' } + $nodeExe = if ($IsWindows) { 'node.exe' } else { 'node' } + if (!(Test-Path (Join-Path $nodeDir $nodeExe))) { throw 'Node 24 executable is missing' } + Add-Content $env:GITHUB_PATH $nodeDir + - run: node -e "if (process.versions.node.split('.')[0] !== '24') process.exit(1)" + - run: npm ci --ignore-scripts + - run: npm run build:kernel + - run: cargo test --manifest-path codegraph-kernel/Cargo.toml --release contained_source::tests -- --test-threads=1 + - run: npx vitest run __tests__/source-scan.test.ts __tests__/security.test.ts __tests__/explore-unmatched-names.test.ts -t "scanIndexedSource|native source reader|unmatched names" --maxWorkers=2 --minWorkers=1 diff --git a/CHANGELOG.md b/CHANGELOG.md index 080e20d8c3..062c315e29 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,6 +30,8 @@ and adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ### New Features +- After a complete scan of indexed source without skipped files, `codegraph_explore` lists query names it cannot find, so an agent can catch a guessed name. + - RedwoodSDK registered routes now link to their page or API handlers, preserving prefixes and method tables while excluding middleware from page roots. - Analog default file pages now link to their component classes, preserving its directory layouts, dotted paths and dynamic segments when the platform plugin and file router are registered. diff --git a/README.md b/README.md index 0286c41aed..5d3ae07628 100644 --- a/README.md +++ b/README.md @@ -80,6 +80,7 @@ Compared with upstream `main` at `6560052` (v1.6.2). Each item here and in the d |---|:-:|:-:|---| | Session search (`codegraph sessions`, `codegraph_sessions`) | — | ✓ | Searches this project's earlier Claude Code, Codex, Cursor/T3, OpenCode, AGY, Devin and Grok transcripts and its git commit messages, so an agent can find what a past session decided. `codegraph_explore` also names the sessions that mentioned the symbols it returns. `"sessions": false` in `codegraph.json` turns it off. | | Markdown indexing | — | ✓ | Headings, sections, tables and links become graph nodes; a documentation question gets the matching section. | +| Missing names reported | — | ✓ | After a complete scan of indexed source without skipped files, `codegraph_explore` lists requested names it cannot find, helping an agent catch a guessed name. | | Near-duplicate functions | — | ✓ | `codegraph_explore` and `codegraph_node` name the near-identical copies of a function, so a fix made in one copy is not forgotten in the others. | | External HTTP endpoints | — | ✓ | JavaScript and TypeScript calls through `fetch`, axios, ky, got and similar clients appear as endpoint nodes such as `GET https://api.github.com/…`. | | Change questions | — | ✓ | "What did my changes touch?" or `main..HEAD` is answered from the diff: the changed functions, their callers and their tests. | diff --git a/__tests__/explore-unmatched-names.test.ts b/__tests__/explore-unmatched-names.test.ts new file mode 100644 index 0000000000..702aeca6a5 --- /dev/null +++ b/__tests__/explore-unmatched-names.test.ts @@ -0,0 +1,154 @@ +/** + * Explore lists the symbol-shaped names it could not find. A query that mixes + * real names with a guessed one (`finalHistoryReconciled`) used to come back + * with the real names' code and no word about the guess, so the agent read + * on as if the returned code held it. The summary now names the misses, and + * stays quiet for words that only look precise (a sentence-initial `How`), + * quoted literals, pinned files, camel infixes that seed by definer, and names + * that appear in source without a node of their own (imports, parameters, + * object keys, module stems). + */ +import { describe, it, expect, beforeAll, afterAll } from 'vitest'; +import * as fs from 'fs'; +import * as path from 'path'; +import * as os from 'os'; +import CodeGraph from '../src/index'; +import { ToolHandler } from '../src/mcp/tools'; + +const FILES: Record = { + 'src/snapshot.ts': [ + 'export function verifySnapshot(rows: string[]): boolean {', + ' const key = getSnapshotKey(rows);', + " return key.length > 0 && localStorage.getItem('draft.pick_state') !== null;", + '}', + '', + 'export function getSnapshotKey(rows: string[]): string {', + " return rows.join(':');", + '}', + '', + ].join('\n'), + 'src/draftStoreState.ts': [ + "import { spawnSync } from 'child_process';", + '', + 'export function runDraft(projectPath: string): number {', + " const events = { DRAFT_STATE_UPDATED: 'updated', maxBuffer: 1024 };", + " const out = spawnSync('draft', [projectPath], { maxBuffer: events.maxBuffer });", + ' process.exitCode = out.status ?? 1;', + ' return out.status ?? 1;', + '}', + '', + ].join('\n'), + // A stem whose extension the explore token cleanup does not strip. + 'config/draft_rules.yaml': 'a: 1\n', + // Prose that names the guess: docs are not code, so this must not hide it. + 'docs/notes.md': '# Notes\n\nWe planned `finalHistoryReconciled` but never wrote it.\n', + 'src/pickLedger.ts': [ + "import { verifySnapshot } from './snapshot';", + '', + 'export function recordPick(rows: string[]): void {', + ' if (!verifySnapshot(rows)) throw new Error("bad snapshot");', + '}', + '', + ].join('\n'), +}; + +let dir: string; +let cg: CodeGraph; + +async function explore(query: string): Promise { + const res = await new ToolHandler(cg).execute('codegraph_explore', { query }); + return res.content?.[0]?.text ?? ''; +} + +beforeAll(async () => { + dir = fs.mkdtempSync(path.join(os.tmpdir(), 'codegraph-unmatched-')); + for (const [rel, text] of Object.entries(FILES)) { + fs.mkdirSync(path.dirname(path.join(dir, rel)), { recursive: true }); + fs.writeFileSync(path.join(dir, rel), text); + } + cg = CodeGraph.initSync(dir); + await cg.indexAll(); +}, 180_000); + +afterAll(() => { + cg?.destroy(); + if (dir && fs.existsSync(dir)) fs.rmSync(dir, { recursive: true, force: true }); +}); + +describe('unmatched names in the explore summary', () => { + it('names a guessed symbol beside the real one it still answers', async () => { + const out = await explore('verifySnapshot finalHistoryReconciled'); + expect(out).toContain('**`src/snapshot.ts`'); + expect(out).toContain('Not found in the index: `finalHistoryReconciled`.'); + }); + + // An apostrophe is not a quote: the guess in a plain sentence is still a + // guess, and a double-quoted phrase stays a phrase around one. + it('reads apostrophes as part of words, not as quotes', async () => { + const out = await explore("verifySnapshot don't call finalHistoryReconciled if it's gone"); + expect(out).toContain('Not found in the index: `finalHistoryReconciled`.'); + for (const query of [ + 'verifySnapshot "we don\'t call finalHistoryReconciled here"', + "verifySnapshot 'we don't call finalHistoryReconciled here'", + ]) { + expect(await explore(query), query).not.toContain('Not found in the index'); + } + }); + + // Quotes pair left to right, and a word quoted once but also written bare + // is still the agent's own guess. + it('reads the words between and outside quoted spans as unquoted', async () => { + for (const query of [ + 'verifySnapshot "draft" finalHistoryReconciled "state"', + 'verifySnapshot "we skip finalHistoryReconciled here" then finalHistoryReconciled', + 'verifySnapshot "we skip finalHistoryReconciled here" then "finalHistoryReconciled"', + ]) { + expect(await explore(query), query).toContain('Not found in the index: `finalHistoryReconciled`.'); + } + }); + + // A quoted lone name is a name the agent expects in code, unlike a quoted + // literal that some file holds. + it('lists a quoted name that neither the graph nor any literal holds', async () => { + expect(await explore('verifySnapshot "finalHistoryReconciled"')).toContain('Not found in the index: `finalHistoryReconciled`.'); + }); + + it('lists qualified and snake_case guesses too', async () => { + const out = await explore('verifySnapshot Ledger.replayAll draft_state_guard'); + expect(out).toContain('`Ledger.replayAll`'); + expect(out).toContain('`draft_state_guard`'); + }); + + it('stays quiet for prose, quoted literals, pinned files and camel infixes', async () => { + for (const query of [ + 'How does verifySnapshot work', + 'verifySnapshot "draft.pick_state"', + 'pickLedger.ts recordPick', + 'verifySnapshot snapshotKey', + 'verifySnapshot "snapshotKey"', + // Real code the node table does not hold: an import, a parameter, a + // module stem, object keys, a global member, a version string. + 'runDraft spawnSync projectPath', + 'runDraft draftStoreState', + 'runDraft DRAFT_STATE_UPDATED maxBuffer', + 'runDraft process.exitCode', + 'runDraft v1.0.352', + 'verifySnapshot draft_rules', + 'verifySnapshot "some finalHistoryReconciled phrase"', + ]) { + expect(await explore(query), query).not.toContain('Not found in the index'); + } + }); + + // A file that cannot be read leaves the absence unproven: the guess may be + // an object key in exactly that file. + it.runIf(process.platform !== 'win32' && process.getuid?.() !== 0)('lists nothing when a source file cannot be read', async () => { + const file = path.join(dir, 'src/draftStoreState.ts'); + fs.chmodSync(file, 0o000); + try { + expect(await explore('verifySnapshot finalHistoryReconciled')).not.toContain('Not found in the index'); + } finally { + fs.chmodSync(file, 0o644); + } + }); +}); diff --git a/__tests__/security.test.ts b/__tests__/security.test.ts index b16c5b4fff..63c0e31ea1 100644 --- a/__tests__/security.test.ts +++ b/__tests__/security.test.ts @@ -14,6 +14,7 @@ import * as path from 'path'; import * as os from 'os'; import { FileLock, validateProjectPath, validatePathWithinRoot } from '../src/utils'; import CodeGraph from '../src/index'; +import { requireKernel } from '../src/extraction/kernel/loader'; import { ToolHandler, tools } from '../src/mcp/tools'; import { scanDirectory, isSourceFile } from '../src/extraction'; import { DatabaseConnection, getDatabasePath } from '../src/db'; @@ -217,9 +218,13 @@ describe('Symlink escape prevention (#527)', () => { fs.rmSync(outside, { recursive: true, force: true }); }); - // Symlink creation needs privileges on Windows; skip gracefully if it fails. + // Legacy checks allow missing Windows symlink privileges. Native reader + // checks require link creation, so platform CI cannot skip their assertions. const link = (linkPath: string, target: string): boolean => { - try { fs.symlinkSync(target, linkPath); return true; } catch { return false; } + try { + fs.symlinkSync(target, linkPath, fs.statSync(target).isDirectory() ? 'dir' : 'file'); + return true; + } catch { return false; } }; it('allows a real file inside the root (and realpaths consistently)', () => { @@ -249,6 +254,37 @@ describe('Symlink escape prevention (#527)', () => { expect(validatePathWithinRoot(root, 'src/inlink.ts')).not.toBeNull(); }); + // The batch form a source scan uses checks the file it opened, so each + // escape route must be caught there too, including a swap after a first read. + function containedReader(rootPath: string): (relative: string) => boolean { + const Reader = requireKernel().ContainedSourceReader; + expect(Reader).toBeTypeOf('function'); + const reader = new Reader!(rootPath); + return (relative) => reader.read(relative, 4096, 4096).content != null; + } + + it('native source reader agrees: in-root files read, every escape is refused', () => { + const read = containedReader(root); + expect(read('src/in.ts')).toBe(true); + expect(read(`../${path.basename(outside)}/pkg/secret.txt`)).toBe(false); + expect(read('src/missing.ts')).toBe(false); + expect(link(path.join(root, 'escape'), path.join(outside, 'pkg', 'secret.txt'))).toBe(true); + expect(read('escape')).toBe(false); + expect(link(path.join(root, 'escapedir'), path.join(outside, 'pkg'))).toBe(true); + expect(read('escapedir/secret.txt')).toBe(false); + expect(link(path.join(root, 'src', 'inlink.ts'), path.join(root, 'src', 'in.ts'))).toBe(true); + expect(read('src/inlink.ts')).toBe(true); + }); + + it('native source reader refuses a directory moved out of the root and linked back in', () => { + const read = containedReader(root); + expect(read('src/in.ts')).toBe(true); + // The same directory, same inode, now reached through a symlink. + fs.renameSync(path.join(root, 'src'), path.join(outside, 'moved-src')); + expect(link(path.join(root, 'src'), path.join(outside, 'moved-src'))).toBe(true); + expect(read('src/in.ts')).toBe(false); + }); + // The INDEXING read path opts into following in-root symlinks the directory // walk already descended into — discovery and the reader must agree, or files // reached via an in-root symlink-to-outside fail to index (#935). The lexical diff --git a/__tests__/source-scan.test.ts b/__tests__/source-scan.test.ts new file mode 100644 index 0000000000..819ca27b22 --- /dev/null +++ b/__tests__/source-scan.test.ts @@ -0,0 +1,73 @@ +/** + * The source scan's limits are checked on the file it opened, so the answer + * "this name appears nowhere" is given only after every file was read whole. + */ +import { describe, it, expect, beforeAll, afterAll } from 'vitest'; +import * as fs from 'fs'; +import * as path from 'path'; +import * as os from 'os'; +import { execFileSync } from 'child_process'; +import { scanIndexedSource } from '../src/mcp/source-scan'; + +let dir: string; +const LIMITS = { budgetMs: 10_000, maxFileBytes: 1024, maxTotalBytes: 4096 }; + +beforeAll(() => { + dir = fs.mkdtempSync(path.join(os.tmpdir(), 'codegraph-source-scan-')); + fs.writeFileSync(path.join(dir, 'a.ts'), 'const alpha = 1;\n'); + fs.writeFileSync(path.join(dir, 'b.ts'), 'const beta = 2;\n'); + fs.writeFileSync(path.join(dir, 'big.ts'), `const gamma = '${'x'.repeat(2000)}';\n`); +}); + +afterAll(() => { + if (dir) fs.rmSync(dir, { recursive: true, force: true }); +}); + +function words(files: string[], limits = LIMITS) { + const seen: string[] = []; + // Index-time sizes are stale on purpose: the scan must use the opened file's. + const result = scanIndexedSource(dir, files.map((p) => ({ path: p, size: 1 })), /\b(?:alpha|beta|gamma)\b/g, limits, (_f, _o, text) => { + seen.push(text); + return false; + }); + return { ...result, seen }; +} + +describe('scanIndexedSource', () => { + it('reads every file and reports a complete scan', () => { + expect(words(['a.ts', 'b.ts'])).toEqual({ complete: true, skipped: 0, seen: ['alpha', 'beta'] }); + }); + + it('skips a file over the per-file limit, which leaves absence unproven', () => { + expect(words(['a.ts', 'big.ts'])).toEqual({ complete: true, skipped: 1, seen: ['alpha'] }); + }); + + it('stops incomplete when the bytes read would pass the total limit', () => { + const limits = { ...LIMITS, maxFileBytes: 4096, maxTotalBytes: 1024 }; + expect(words(['a.ts', 'big.ts', 'b.ts'], limits)).toEqual({ complete: false, skipped: 0, seen: ['alpha'] }); + }); + + // A FIFO with no writer would block a plain open forever. + it.runIf(process.platform !== 'win32')('skips a FIFO without waiting for a writer', () => { + execFileSync('mkfifo', [path.join(dir, 'pipe.ts')]); + // A child deadline makes a blocking-open regression fail without hanging + // the test runner's own event loop. + const script = ` + const { scanIndexedSource } = require(${JSON.stringify(path.resolve(__dirname, '../dist/mcp/source-scan.js'))}); + const seen = []; + const result = scanIndexedSource(${JSON.stringify(dir)}, + ['a.ts', 'pipe.ts', 'b.ts'].map(path => ({ path, size: 1 })), + /alpha|beta/g, ${JSON.stringify(LIMITS)}, + (_f, _o, word) => { seen.push(word); return false; }); + console.log(JSON.stringify({ ...result, seen })); + `; + const result = execFileSync(process.execPath, ['-e', script], { + encoding: 'utf8', timeout: 3000, killSignal: 'SIGKILL', + }); + expect(JSON.parse(result)).toEqual({ complete: true, skipped: 1, seen: ['alpha', 'beta'] }); + }); + + it('counts a missing or escaping path as skipped', () => { + expect(words(['a.ts', 'gone.ts', '../outside.ts'])).toEqual({ complete: true, skipped: 2, seen: ['alpha'] }); + }); +}); diff --git a/codegraph-kernel/src/contained_source.rs b/codegraph-kernel/src/contained_source.rs new file mode 100644 index 0000000000..69376cf91f --- /dev/null +++ b/codegraph-kernel/src/contained_source.rs @@ -0,0 +1,327 @@ +//! Read source only after the OS identifies the file actually opened. +//! Pathname checks cannot establish this when a symlink changes between calls. + +use std::fs::{self, File, OpenOptions}; +use std::io::{self, Read}; +use std::path::{Component, Path, PathBuf}; + +const MAX_READ_BYTES: u64 = 64 * 1024 * 1024; + +struct SourceReader { + root: PathBuf, +} + +#[derive(Default)] +struct ReadOutcome { + content: Option>, + bytes_read: u32, + limit_reached: bool, +} + +impl SourceReader { + fn new(root: &Path) -> io::Result { + let root = fs::canonicalize(root)?; + if !root.is_dir() { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "source root is not a directory", + )); + } + Ok(Self { root }) + } + + fn read(&self, relative: &Path, max_file_bytes: u32, remaining_bytes: u32) -> ReadOutcome { + if relative.as_os_str().is_empty() + || relative + .components() + .any(|c| !matches!(c, Component::Normal(_) | Component::CurDir)) + { + return ReadOutcome::default(); + } + let file = match open_source(&self.root.join(relative)) { + Ok(file) => file, + Err(_) => return ReadOutcome::default(), + }; + // Inspect the open handle, never a second lookup of the supplied name. + if !opened_path(&file).is_ok_and(|p| p.starts_with(&self.root)) { + return ReadOutcome::default(); + } + let metadata = match file.metadata() { + Ok(m) if m.is_file() => m, + _ => return ReadOutcome::default(), + }; + let size = metadata.len(); + if size > u64::from(max_file_bytes).min(MAX_READ_BYTES) { + return ReadOutcome::default(); + } + if size > u64::from(remaining_bytes).min(MAX_READ_BYTES) { + return ReadOutcome { + limit_reached: true, + ..ReadOutcome::default() + }; + } + read_open_file(file, size) + } +} + +fn read_open_file(file: File, size: u64) -> ReadOutcome { + // One extra byte detects growth after metadata. Even discarded or failed + // reads count against the caller's total scan budget. + let mut content = Vec::with_capacity(size as usize + 1); + let result = file.take(size + 1).read_to_end(&mut content); + let bytes_read = content.len() as u32; + let content = (result.is_ok() && u64::from(bytes_read) <= size).then_some(content); + ReadOutcome { + content, + bytes_read, + limit_reached: false, + } +} + +fn open_source(path: &Path) -> io::Result { + let mut options = OpenOptions::new(); + options.read(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + options.custom_flags(libc::O_NONBLOCK); + } + options.open(path) +} + +#[cfg(target_os = "linux")] +fn opened_path(file: &File) -> io::Result { + use std::os::fd::AsRawFd; + fs::read_link(format!("/proc/self/fd/{}", file.as_raw_fd())) +} + +#[cfg(target_os = "macos")] +fn opened_path(file: &File) -> io::Result { + use std::os::fd::AsRawFd; + use std::os::unix::ffi::OsStringExt; + let mut buffer = vec![0u8; libc::PATH_MAX as usize]; + // SAFETY: the descriptor remains owned by file; F_GETPATH writes at most + // PATH_MAX bytes into the allocated buffer and does not take ownership. + if unsafe { libc::fcntl(file.as_raw_fd(), libc::F_GETPATH, buffer.as_mut_ptr()) } == -1 { + return Err(io::Error::last_os_error()); + } + let end = buffer + .iter() + .position(|b| *b == 0) + .ok_or_else(|| io::Error::other("unterminated descriptor path"))?; + buffer.truncate(end); + Ok(std::ffi::OsString::from_vec(buffer).into()) +} + +#[cfg(windows)] +fn opened_path(file: &File) -> io::Result { + use std::os::windows::ffi::OsStringExt; + use std::os::windows::io::AsRawHandle; + #[link(name = "kernel32")] + extern "system" { + fn GetFinalPathNameByHandleW( + handle: *mut std::ffi::c_void, + path: *mut u16, + size: u32, + flags: u32, + ) -> u32; + fn GetFileType(handle: *mut std::ffi::c_void) -> u32; + } + // SAFETY: file owns a live handle. Only disk files have source content; + // refuse pipes/devices before asking for their normalized path. + if unsafe { GetFileType(file.as_raw_handle()) } != 1 { + return Err(io::Error::other("source is not a disk file")); + } + let mut buffer = vec![0u16; 512]; + for _ in 0..2 { + // SAFETY: the live handle is borrowed and the buffer size is exact. + // Flags 0 request a normalized DOS path, matching fs::canonicalize. + let size = unsafe { + GetFinalPathNameByHandleW( + file.as_raw_handle(), + buffer.as_mut_ptr(), + buffer.len() as u32, + 0, + ) + }; + if size == 0 { + return Err(io::Error::last_os_error()); + } + if (size as usize) < buffer.len() { + buffer.truncate(size as usize); + return Ok(std::ffi::OsString::from_wide(&buffer).into()); + } + if size > 32_768 { + break; + } + buffer.resize(size as usize + 1, 0); + } + Err(io::Error::other("descriptor path exceeds its buffer")) +} + +#[cfg(not(any(target_os = "linux", target_os = "macos", windows)))] +fn opened_path(_file: &File) -> io::Result { + Err(io::Error::new( + io::ErrorKind::Unsupported, + "no descriptor path lookup", + )) +} + +#[napi_derive::napi(object)] +pub struct ContainedSourceRead { + pub content: Option, + pub bytes_read: u32, + pub limit_reached: bool, +} + +#[napi_derive::napi] +pub struct ContainedSourceReader { + inner: SourceReader, +} + +#[napi_derive::napi] +impl ContainedSourceReader { + #[napi(constructor)] + pub fn new(root: String) -> napi::Result { + let inner = SourceReader::new(Path::new(&root)) + .map_err(|e| napi::Error::from_reason(e.to_string()))?; + Ok(Self { inner }) + } + + #[napi] + pub fn read( + &self, + relative: String, + max_file_bytes: u32, + remaining_bytes: u32, + ) -> ContainedSourceRead { + let read = self + .inner + .read(Path::new(&relative), max_file_bytes, remaining_bytes); + ContainedSourceRead { + content: read.content.map(Into::into), + bytes_read: read.bytes_read, + limit_reached: read.limit_reached, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::atomic::{AtomicUsize, Ordering}; + static NEXT: AtomicUsize = AtomicUsize::new(0); + + struct Fixture(PathBuf); + impl Fixture { + fn new() -> Self { + let path = std::env::temp_dir().join(format!( + "cg-contained-{}-{}", + std::process::id(), + NEXT.fetch_add(1, Ordering::Relaxed) + )); + fs::create_dir_all(path.join("root")).unwrap(); + fs::write(path.join("root/source.ts"), b"alpha").unwrap(); + fs::write(path.join("outside.ts"), b"secret").unwrap(); + Self(path) + } + fn reader(&self) -> SourceReader { + SourceReader::new(&self.0.join("root")).unwrap() + } + } + impl Drop for Fixture { + fn drop(&mut self) { + let _ = fs::remove_dir_all(&self.0); + } + } + + #[test] + fn reads_only_regular_in_root_files_and_enforces_both_byte_limits() { + let f = Fixture::new(); + let reader = f.reader(); + assert_eq!( + reader.read(Path::new("source.ts"), 5, 5).content.unwrap(), + b"alpha" + ); + assert!(reader.read(Path::new("source.ts"), 4, 5).content.is_none()); + assert!(reader.read(Path::new("source.ts"), 5, 4).limit_reached); + for path in ["../outside.ts", "/outside.ts", ".", "missing.ts"] { + assert!( + reader.read(Path::new(path), 100, 100).content.is_none(), + "{path}" + ); + } + } + + #[test] + fn charges_growth_even_when_content_is_discarded() { + let f = Fixture::new(); + let path = f.0.join("root/source.ts"); + let file = open_source(&path).unwrap(); + let size = file.metadata().unwrap().len(); + fs::write(&path, b"alphabet").unwrap(); + let result = read_open_file(file, size); + assert!(result.content.is_none()); + assert_eq!(result.bytes_read, 6); + } + + #[test] + fn descriptor_path_follows_the_open_file_after_its_name_is_replaced() { + let f = Fixture::new(); + let original = f.0.join("root/source.ts"); + let file = open_source(&original).unwrap(); + fs::rename(&original, f.0.join("moved-outside.ts")).unwrap(); + fs::write(&original, b"replacement").unwrap(); + assert!(!opened_path(&file).unwrap().starts_with(&f.reader().root)); + assert_eq!( + f.reader() + .read(Path::new("source.ts"), 100, 100) + .content + .unwrap(), + b"replacement" + ); + } + + #[test] + fn supports_unicode_root_and_source_names() { + let f = Fixture::new(); + let root = f.0.join("root/Évidence"); + fs::create_dir(&root).unwrap(); + fs::write(root.join("résumé.ts"), b"unicode").unwrap(); + let reader = SourceReader::new(&root).unwrap(); + assert_eq!( + reader + .read(Path::new("résumé.ts"), 100, 100) + .content + .unwrap(), + b"unicode" + ); + } + + #[cfg(unix)] + #[test] + fn checks_open_file_after_a_symlink_is_replaced() { + use std::os::unix::fs::symlink; + let f = Fixture::new(); + let alias = f.0.join("root/alias.ts"); + symlink(f.0.join("outside.ts"), &alias).unwrap(); + let outside = open_source(&alias).unwrap(); + fs::remove_file(&alias).unwrap(); + symlink(f.0.join("root/source.ts"), &alias).unwrap(); + assert!(!opened_path(&outside).unwrap().starts_with(&f.reader().root)); + assert_eq!( + f.reader() + .read(Path::new("alias.ts"), 100, 100) + .content + .unwrap(), + b"alpha" + ); + fs::remove_file(&alias).unwrap(); + symlink(f.0.join("outside.ts"), &alias).unwrap(); + assert!(f + .reader() + .read(Path::new("alias.ts"), 100, 100) + .content + .is_none()); + } +} diff --git a/codegraph-kernel/src/lib.rs b/codegraph-kernel/src/lib.rs index d4631e2e68..a076d903b3 100644 --- a/codegraph-kernel/src/lib.rs +++ b/codegraph-kernel/src/lib.rs @@ -394,6 +394,7 @@ mod rlang; mod ruby; mod rustlang; mod scala; +mod contained_source; mod stack; mod swift; mod textutil; diff --git a/site/src/content/docs/reference/mcp-server.md b/site/src/content/docs/reference/mcp-server.md index bf54d25a3a..1727331c36 100644 --- a/site/src/content/docs/reference/mcp-server.md +++ b/site/src/content/docs/reference/mcp-server.md @@ -15,7 +15,7 @@ The tools below are listed in every workspace. Where the project has no `.codegr By default the server exposes `codegraph_explore` for code and `codegraph_sessions` for project history. -`codegraph_explore` It's Read-equivalent: give it a natural-language question or a bag of symbol and file names, and it returns the **verbatim, line-numbered source** of the relevant symbols grouped by file — the same shape the `Read` tool gives you — plus the call paths between them (including dynamic-dispatch hops like callbacks, React re-render, and JSX children that grep can't follow) and a blast-radius summary of what depends on them. One call usually answers the whole question. +`codegraph_explore` It's Read-equivalent: give it a natural-language question or a bag of symbol and file names, and it returns the **verbatim, line-numbered source** of the relevant symbols grouped by file — the same shape the `Read` tool gives you — plus the call paths between them (including dynamic-dispatch hops like callbacks, React re-render, and JSX children that grep can't follow) and a blast-radius summary of what depends on them. One call usually answers the whole question. When a complete scan of indexed source finishes without skipped files, the summary lists requested names it could not find, helping catch a guessed name. `codegraph_sessions` searches this project's earlier agent sessions (Claude Code, Codex, Cursor/T3, OpenCode, AGY, Devin and Grok transcripts, plus git commit messages) for what a previous session asked, decided or tried. It answers "why is X like this" questions, which the code graph cannot. Set `"sessions": false` in `codegraph.json` to turn it off. diff --git a/src/codegraph.ts b/src/codegraph.ts index bd90da96f3..f32bee8825 100644 --- a/src/codegraph.ts +++ b/src/codegraph.ts @@ -2292,6 +2292,22 @@ export class CodeGraph { return this.queries.getAllFiles(); } + /** + * The files whose text a source scan reads: neither generated nor markdown, + * with the size recorded at index time. + */ + getTextScanFiles(): Array<{ path: string; size: number }> { + return this.queries.getTextScanFiles(); + } + + /** + * Whether any file binds `name` — an import, parameter, local or declaration + * the node table does not hold. False on a database without bindings. + */ + hasBindingNamed(name: string): boolean { + try { return this.queries.hasBindingNamed(name); } catch { return false; } + } + /** Every tracked file path, sorted; reads one column instead of whole file records. */ getFilePaths(): string[] { return this.queries.getAllFilePaths(); diff --git a/src/db/queries.ts b/src/db/queries.ts index 444ffbe1d3..98de150136 100644 --- a/src/db/queries.ts +++ b/src/db/queries.ts @@ -333,6 +333,8 @@ export class QueryBuilder { deleteLiteralsByFile?: SqliteStatement; deleteBindingsByFile?: SqliteStatement; getBindingsByFile?: SqliteStatement; + hasBindingNamed?: SqliteStatement; + getTextScanFiles?: SqliteStatement; getNodeById?: SqliteStatement; getNodesByFile?: SqliteStatement; getNodesByKind?: SqliteStatement; @@ -592,6 +594,24 @@ export class QueryBuilder { for (const value of node.literals ?? []) rows.push([value, node.id, node.filePath]); } + /** Hand-written, non-markdown files with their indexed sizes, sorted by path. */ + getTextScanFiles(): Array<{ path: string; size: number }> { + if (!this.stmts.getTextScanFiles) { + this.stmts.getTextScanFiles = this.db.prepare( + "SELECT path, size FROM files WHERE generated = 0 AND language <> 'markdown' ORDER BY path", + ); + } + return this.stmts.getTextScanFiles.all() as Array<{ path: string; size: number }>; + } + + /** Whether any file binds `name` (an import, parameter, local or declaration). */ + hasBindingNamed(name: string): boolean { + if (!this.stmts.hasBindingNamed) { + this.stmts.hasBindingNamed = this.db.prepare('SELECT 1 FROM bindings WHERE name = ? LIMIT 1'); + } + return this.stmts.hasBindingNamed.get(name) !== undefined; + } + /** The file's `bindings` rows (see schema.sql), in emission order. */ getBindingsByFile(filePath: string): Binding[] { if (!this.stmts.getBindingsByFile) { diff --git a/src/extraction/kernel/loader.ts b/src/extraction/kernel/loader.ts index 5be270b890..c0f54917d2 100644 --- a/src/extraction/kernel/loader.ts +++ b/src/extraction/kernel/loader.ts @@ -279,7 +279,20 @@ export interface KernelResolverLike { close(): void; } +export interface ContainedSourceRead { + content?: Buffer | null; + bytesRead: number; + limitReached: boolean; +} + +export interface ContainedSourceReaderLike { + read(relative: string, maxFileBytes: number, remainingBytes: number): ContainedSourceRead; +} + export interface KernelModule { + /** Descriptor-verified source reads. Older released kernels omit this API; + * scans then leave absence unproven. */ + ContainedSourceReader?: new (root: string) => ContainedSourceReaderLike; extractFile(filePath: string, content: string, language: string): KernelBuffers; /** Binding rows only, from the AST, for a TS/JS-family or ArkTS file the * generic extractor extracts (resolution-binding-model-plan.md §2.4). diff --git a/src/mcp/server-instructions.ts b/src/mcp/server-instructions.ts index 69cd5bac32..3ea1de34f3 100644 --- a/src/mcp/server-instructions.ts +++ b/src/mcp/server-instructions.ts @@ -66,6 +66,7 @@ against dozens of greps and reads. - **Read/Grep are for what the index lacks**: a detail a codegraph answer didn't cover, or files codegraph doesn't index (such as configs). Markdown is indexed — every \`.md\` file's headings, sections, tables and links — so a documentation question (a rule, a runbook, a plan row, a research finding) goes to \`codegraph_explore\` too; it returns the section body, not just its heading. - **A question about the working changes** ("what do my changes affect", "this branch", \`main..HEAD\`) goes to \`codegraph_explore\` as asked: it reads the diff itself (merge base with the default branch plus uncommitted edits, or the named range) and leads with the changed symbols, their callers and their tests. - **Near-duplicates (update together)** in a blast radius or trail names other function bodies nearly identical to the one you are reading. A bug fixed in one copy is usually still in the others: check them, and tell the user if you leave them unchanged. +- **"Not found in the index: \`name\`"** in the summary means that symbol-shaped name has no symbol and appears in no indexed code file (Markdown is not checked). Check its spelling or ask whether it exists. Grepping indexed code finds nothing new; a search outside the index may still find the name. - **Don't reconstruct a flow by hand** — name the endpoints in one \`codegraph_explore\` and it surfaces the path between them, dynamic-dispatch hops included. - **After editing, check the staleness banner.** When a tool response starts with "⚠️ Some files referenced below were edited since the last index sync…", the listed files are pending re-index — Read those specific files for accurate content. Every file NOT in that banner is fresh, so still trust codegraph. A different, rarer banner — "⚠️ CodeGraph auto-sync is DISABLED…" or "⚠️ CodeGraph auto-sync is RECOVERING…" — means the graph may lag behind recent edits until a sync completes; source is still read from disk. A response that lists files which changed after their last sync still answers, but symbols, callers and callees taken from those files may be out of date; if they matter, retry after \`codegraph sync\`. - **A file flagged "⚠ changed on disk after the last index sync" drifted from its index** (most common on projects queried via \`projectPath\`, which have no live watcher). Codegraph never serves a possibly-mis-sliced body from such a file — it either shows the file's full CURRENT source (trust it as a Read) or omits the source with this flag. When the source was omitted, Read that specific file; line numbers referencing it elsewhere in the response may be shifted until that project's next sync. All unflagged files remain trustworthy. diff --git a/src/mcp/source-scan.ts b/src/mcp/source-scan.ts new file mode 100644 index 0000000000..bb092e61eb --- /dev/null +++ b/src/mcp/source-scan.ts @@ -0,0 +1,100 @@ +/** + * A time-capped pass over the source text of indexed files, for the questions + * the graph cannot answer: whether a name appears anywhere (an object key, a + * template binding) and where a quoted phrase sits. Reads only files the index + * tracks. The native reader verifies the opened descriptor before reading content. + */ +import { getKernel } from '../extraction/kernel/loader'; + +/** + * Limits on what a warm scan reads inside a 300 ms budget, with room to spare. + * Cost is about 12 microseconds per file plus 1 ms per MB: 13,609 TypeScript + * files (39.5 MB) took 175 ms, 2,174 C files (47.4 MB) took 55 ms. A larger + * project skips the scan, because a scan that runs out of time costs the whole + * budget and answers nothing. + */ +export const MAX_SCAN_FILES = 16_000; +export const MAX_SCAN_TOTAL_BYTES = 64 * 1024 * 1024; + +/** Whether a scan of `files` should fit its budget; see {@link MAX_SCAN_FILES}. */ +export function scanFits(files: readonly ScanFile[]): boolean { + if (files.length > MAX_SCAN_FILES) return false; + let total = 0; + for (const f of files) total += f.size; + return total <= MAX_SCAN_TOTAL_BYTES; +} + +export interface ScanFile { + path: string; + /** Size recorded at index time. */ + size: number; +} + +export interface ScanLimits { + budgetMs: number; + /** A file larger than this when opened is skipped. */ + maxFileBytes: number; + /** + * The scan stops, incomplete, before a file that would take it past this + * many bytes. Every byte read counts, including a skipped file's. A read + * may use one extra byte to detect growth, then the scan stops. + */ + maxTotalBytes: number; +} + +export interface ScanResult { + /** + * True when every file was read, or `onMatch` stopped the scan. The budget + * is checked before each file, so the last read can end past it; its text + * is still matched, because the answer is complete and the time is spent. + */ + complete: boolean; + /** Files not read: outside the root, unreadable, over `maxFileBytes`, or grown while read. */ + skipped: number; +} + +/** + * Calls `onMatch` for every match of the global `pattern` in `files`, in + * order, until it returns true or a limit runs out. Sizes are checked on the + * opened file, and the read stops one byte past that size, so a file that grew + * since indexing, or grows while it is read, cannot overrun them. + */ +export function scanIndexedSource( + projectRoot: string, + files: readonly ScanFile[], + pattern: RegExp, + limits: ScanLimits, + onMatch: (filePath: string, offset: number, text: string) => boolean, +): ScanResult { + if (!pattern.global) throw new Error('scanIndexedSource needs a global pattern'); + const deadline = Date.now() + limits.budgetMs; + const Reader = getKernel()?.ContainedSourceReader; + if (!Reader) return { complete: false, skipped: 0 }; + let reader: InstanceType>; + try { reader = new Reader(projectRoot); } + catch { return { complete: false, skipped: files.length }; } + let skipped = 0; + let total = 0; + for (const file of files) { + if (total > limits.maxTotalBytes) return { complete: false, skipped }; + if (Date.now() > deadline) return { complete: false, skipped }; + let text: string; + try { + const result = reader.read(file.path, + Math.min(limits.maxFileBytes, MAX_SCAN_TOTAL_BYTES), + Math.max(0, Math.min(limits.maxTotalBytes - total, MAX_SCAN_TOTAL_BYTES))); + total += result.bytesRead; + if (result.limitReached) return { complete: false, skipped }; + if (result.content == null) { skipped++; continue; } + text = result.content.toString('utf8'); + } catch { + skipped++; + continue; + } + pattern.lastIndex = 0; + for (const m of text.matchAll(pattern)) { + if (onMatch(file.path, m.index ?? 0, m[0])) return { complete: true, skipped }; + } + } + return { complete: true, skipped }; +} diff --git a/src/mcp/tools.ts b/src/mcp/tools.ts index a6da25e00c..1432bf88c3 100644 --- a/src/mcp/tools.ts +++ b/src/mcp/tools.ts @@ -52,6 +52,7 @@ import { statSync, } from 'fs'; import { createHash } from 'crypto'; +import { MAX_SCAN_TOTAL_BYTES, scanFits, scanIndexedSource } from './source-scan'; import { clamp, validatePathWithinRoot, validateProjectPath, isConfigLeafNode, CONFIG_LEAF_LANGUAGES } from '../utils'; import { guardLabel, guardsForFileSync, siteKey, supportsBranchGuards, warmBranchGuardGrammars } from '../graph/branch-guards'; import { findDynamicBoundaries, type BoundarySite } from '../graph/dynamic-boundary-report'; @@ -244,6 +245,59 @@ function filesDefiningSymbol(cg: CodeGraph, symbol: string): string[] { } } +/** Time the not-found check may spend reading indexed source text. */ +const UNMATCHED_SCAN_BUDGET_MS = 300; +/** + * One quoted span of a query: double quotes, backticks, or single quotes that + * sit away from a word, so the apostrophes in "don't" and "it's" are letters. + * Matched left to right, so each closing quote ends its own span. + */ +const QUOTED_SPAN = /"([^"\n]*)"|`([^`\n]*)`|(?(); + for (const p of cg.getFilePaths()) { + const base = p.slice(p.lastIndexOf('/') + 1); + stems.add(base.replace(/\.[^.]*$/, '')); + stems.add(base.replace(/\..*$/, '')); + } + const words = new Map(); // candidate → the word the scan looks for + for (const name of candidates) { + const word = isQualifiedSymbol(name) ? lastQualifierPart(name) : name; + if (stems.has(name) || stems.has(word) || cg.hasBindingNamed(word)) continue; + words.set(name, word); + } + if (words.size === 0) return []; + const wanted = new Set(words.values()); + const escaped = [...wanted].map((w) => w.replace(/[$]/g, '\\$')); + const pattern = new RegExp(`(?(); + const scan = scanIndexedSource(cg.getProjectRoot(), files, pattern, { + budgetMs: UNMATCHED_SCAN_BUDGET_MS, + maxFileBytes: MAX_SCAN_TOTAL_BYTES, + maxTotalBytes: MAX_SCAN_TOTAL_BYTES, + }, (_path, _offset, text) => { + seen.add(text); + return seen.size === wanted.size; + }); + if (seen.size === wanted.size || !scan.complete || scan.skipped > 0) return []; + return [...words].filter(([, word]) => !seen.has(word)).map(([name]) => name).slice(0, 4); +} + /** * Calculate the recommended number of codegraph_explore calls based on project size. * Larger codebases need more exploration calls to cover their surface area, @@ -4830,7 +4884,34 @@ export class ToolHandler { // Files declaring a TYPE the query named by name — the counter-case guard // for the declaration-only penalty (CG-28). Populated in the token loop. const namedTypeFiles = new Set(); + // Symbol-shaped names the query gave that match nothing in the index — a + // guessed `finalHistoryReconciled` — listed in the summary so the agent + // learns the name is wrong instead of reading the other code returned as + // if it were the answer. Populated in the token loop. + const unmatchedNames: string[] = []; { + // Misses the node table alone reports; filtered against bindings, file + // stems and source text after the loop before any reaches the summary. + const missCandidates: string[] = []; + // Words inside a quoted phrase are text the agent copied, not names it + // guessed, unless the query also names them outside every quote. A + // quoted single name stays a candidate: agents quote names. + const phraseWords = new Set(); + const quotedNames: string[] = []; + for (const m of matchQuery.matchAll(QUOTED_SPAN)) { + const inner = (m[1] ?? m[2] ?? m[3] ?? '').trim(); + if (/\s/.test(inner)) { + for (const w of inner.match(NAME_SHAPE) ?? []) phraseWords.add(w); + } else if (inner.length >= 3 && inner.match(NAME_SHAPE)?.[0] === inner) { + quotedNames.push(inner); + } + } + const unquotedWords = new Set(matchQuery.replace(QUOTED_SPAN, ' ').match(NAME_SHAPE) ?? []); + const copiedWord = (t: string) => phraseWords.has(t) && !unquotedWords.has(t) && !quotedNames.includes(t); + // Only shapes no English word takes (a hump, `_`, `$`, a qualifier); a + // digit after a qualifier is a version (`v1.0.352`), not a member. + const missShape = (t: string, isQual: boolean) => + (/[a-z][A-Z]|[_$]|::/.test(t) || (isQual && /\./.test(t))) && !/(?:\.|::)\d/.test(t); const FILE_EXT = /\.(?:java|kt|kts|ts|tsx|js|jsx|mjs|cjs|cs|py|go|rb|php|swift|rs|cpp|cc|cxx|c|h|hpp|scala|lua|dart|vue|svelte|astro|erl|hrl)$/i; const CALLABLE = new Set(['method', 'function', 'component', 'constructor']); // Variables/constants seed too: in Svelte/React a `$state` variable @@ -4844,7 +4925,7 @@ export class ToolHandler { const bodyLines = (n: Node) => Math.max(0, (n.endLine ?? n.startLine) - n.startLine); const callerCount = (n: Node) => { try { return cg.getCallers(n.id).length; } catch { return 0; } }; const tokens = [...new Set( - matchQuery.split(/[\s,()[\]]+/) + [...matchQuery.split(/[\s,()[\]]+/), ...quotedNames] .map((t) => t.replace(/(? t.length >= 3 && /^[A-Za-z_$][\w$]*(?:(?:::|\.)[\w$]+)*$/.test(t)) )].slice(0, 16); @@ -4976,6 +5057,14 @@ export class ToolHandler { if (!isPreciseToken(t)) { cands = cands.filter((n) => coNamedCount(t, n.filePath) > 0); } + // A sentence-initial `How` is precise to the seeding but names nothing. + // A quoted literal or a pinned file's stem is matched by its own channel. + if (raw.length === 0 && cands.length === 0 && missCandidates.length < 8 + && missShape(t, isQual) && !copiedWord(t) + && !pinnedFiles.some((p) => p.slice(p.lastIndexOf('/') + 1).replace(FILE_EXT, '') === t) + && cg.findLiteralSeedIds(`"${t}"`).length === 0) { + missCandidates.push(t); + } // A specific name (<=3 defs) injects all its defs. An overloaded name // (`validate` = 10, `request` = 44) would flood the subgraph, so inject // only: the overloads whose file/class the query ALSO names (the agent @@ -5037,6 +5126,7 @@ export class ToolHandler { if (!isInterfaceOwnedMethod(n)) tierSeedIds.add(n.id); } } + unmatchedNames.push(...confirmUnmatchedNames(cg, missCandidates)); // Path-vocabulary seeds: a bare query token that names a DIRECTORY // segment ("command" → `commands/`, "handler" → `handlers/`) is // location evidence the name/FTS channels can't express: the answer @@ -5851,13 +5941,16 @@ export class ToolHandler { return b[1].nodes.length - a[1].nodes.length; }); + const unmatchedNamesNote = unmatchedNames.length > 0 + ? ` Not found in the index: ${unmatchedNames.map((n) => `\`${n}\``).join(', ')}.` + : ''; const formatSummary = (symbols: number, files: number, pins: number): string => { let text = `Found ${symbols} symbol${symbols === 1 ? '' : 's'} across ${files} file${files === 1 ? '' : 's'}.`; if (pins > 0) text += ` ${pins} file${pins === 1 ? '' : 's'} pinned from the query.`; if (unresolvedPathSpans.length > 0) { text += ` No indexed file uniquely matches ${unresolvedPathSpans.map(s => `\`${s}\``).join(', ')}.`; } - return text + setAsideNote; + return text + unmatchedNamesNote + setAsideNote; }; // Reserve the summary's maximum actual size before spending the source // envelope. Replacing a short sentinel after fitting could exceed the cap. @@ -6183,6 +6276,7 @@ export class ToolHandler { + (unresolvedPathSpans.length > 0 ? ` No indexed file uniquely matches ${unresolvedPathSpans.map((sp) => `\`${sp}\``).join(', ')}.`.length : 0) + + unmatchedNamesNote.length + setAsideNote.length; const epilogueFloor = Math.max( EXPLORE_FALLBACK_NOTES.lost.complete.length, EXPLORE_FALLBACK_NOTES.lost.trimmed.length,