diff --git a/.github/workflows/daemon-lifecycle.yml b/.github/workflows/daemon-lifecycle.yml index 1394dc48d..33e2081c2 100644 --- a/.github/workflows/daemon-lifecycle.yml +++ b/.github/workflows/daemon-lifecycle.yml @@ -58,4 +58,4 @@ jobs: - run: npm ci --ignore-scripts - run: npm run build:kernel - run: npm run build - - run: npx vitest run __tests__/runtime-control.test.ts __tests__/daemon-install-check.test.ts __tests__/daemon-older-version.test.ts __tests__/cli-version.test.ts __tests__/daemon-registry.test.ts __tests__/daemon-pid-reuse.test.ts __tests__/daemon-lock-refresh.test.ts __tests__/mcp-daemon.test.ts __tests__/daemon-client-liveness.test.ts __tests__/daemon-bind-failure.test.ts __tests__/daemon-socket-fallback.test.ts __tests__/daemon-rendezvous-case.test.ts __tests__/writer-lock.test.ts __tests__/writer-handover-ordering.test.ts __tests__/mcp-writer-lock.test.ts __tests__/mcp-projectpath-lifecycle.test.ts __tests__/query-pool.test.ts __tests__/status-index-health.test.ts __tests__/vbnet-project-ties.test.ts __tests__/vbnet-receiver-types.test.ts __tests__/upstream-native-scope-regressions.test.ts --maxWorkers=2 --minWorkers=1 + - run: npx vitest run __tests__/runtime-control.test.ts __tests__/daemon-install-check.test.ts __tests__/daemon-older-version.test.ts __tests__/cli-version.test.ts __tests__/daemon-registry.test.ts __tests__/daemon-pid-reuse.test.ts __tests__/daemon-lock-refresh.test.ts __tests__/mcp-daemon.test.ts __tests__/daemon-client-liveness.test.ts __tests__/daemon-bind-failure.test.ts __tests__/daemon-socket-fallback.test.ts __tests__/daemon-rendezvous-case.test.ts __tests__/writer-lock.test.ts __tests__/writer-handover-ordering.test.ts __tests__/mcp-writer-lock.test.ts __tests__/mcp-projectpath-lifecycle.test.ts __tests__/query-pool.test.ts __tests__/status-index-health.test.ts __tests__/vbnet-project-ties.test.ts __tests__/vbnet-receiver-types.test.ts __tests__/upstream-native-scope-regressions.test.ts --maxWorkers=${{ runner.os == 'Windows' && '1' || '2' }} --minWorkers=1 diff --git a/CHANGELOG.md b/CHANGELOG.md index 7459234e1..631800b6c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -95,6 +95,8 @@ and adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ### Fixes +- `codegraph_explore` reports requested-file limits and uncovered indexed continuation ranges so partial answers have a clear next query. + - Explore retains explicitly pinned code in mixed documentation/code requests, returns parameterized test callbacks, and preserves quoted source terms that match the filename stem. - Python closures no longer link calls or method values to a captured receiver type when later replacement makes that type uncertain. diff --git a/README.md b/README.md index 987eba01b..be0041f61 100644 --- a/README.md +++ b/README.md @@ -83,6 +83,7 @@ Compared with upstream `main` at `aeb8f955` (after v1.6.2). Each item here and i | Session search (`codegraph sessions`, `codegraph_sessions`) | — | ✓ | Searches this project's earlier Claude Code, Codex, Cursor/T3, OpenCode, AGY, Devin and Grok transcripts and its git commit messages, so an agent can find what a past session decided. `codegraph_explore` also names the sessions that mentioned the symbols it returns. `"sessions": false` in `codegraph.json` turns it off. | | Markdown indexing | — | ✓ | Headings, sections, tables and links become graph nodes; a documentation question gets the matching section. Explicit code-file pins remain eligible in mixed documentation/code requests. | | Missing names reported | — | ✓ | After a complete scan of indexed source without skipped files, `codegraph_explore` lists requested names it cannot find, helping an agent catch a guessed name. | +| Requested-source limits reported | — | ✓ | Explore reports requested files that did not receive pin priority within the file limit, unexamined path-like references beyond its bounded scan, and uncovered indexed continuation ranges after a capped file gather. Existing output limits still apply. | | Local callee source with file pins | — | ✓ | When a query names functions in pinned files, `codegraph_explore` prioritizes local callee bodies through two call or callback hops, within the file and output budgets. | | Quoted prose source | — | ✓ | Quoted spans of three or more words find script strings and template text, ignoring case and punctuation, through a capped source scan. No re-index needed. | | Requested test source | — | ✓ | Pinned JavaScript and TypeScript test files return matching `it`/`test` callbacks, including `.each` parameterized tests. Questions about tests include source from the nearest test callers of named symbols, within the file and output budgets. | diff --git a/__tests__/explore-cap-notices.test.ts b/__tests__/explore-cap-notices.test.ts new file mode 100644 index 000000000..46d085f70 --- /dev/null +++ b/__tests__/explore-cap-notices.test.ts @@ -0,0 +1,198 @@ +import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest'; +import * as fs from 'fs'; +import * as os from 'os'; +import * as path from 'path'; +import CodeGraph from '../src/index'; +import { ToolHandler } from '../src/mcp/tools'; +import { ExploreSessionState } from '../src/mcp/explore-session-state'; +import { fileFingerprint } from '../src/mcp/explore-dedup'; + +let dir: string; +let cg: CodeGraph; +const bigSource = Array.from({ length: 360 }, (_, i) => [ + `export function work${String(i).padStart(4, '0')}() {`, + ` // independent source ${'x'.repeat(160)}`, + ` return ${i};`, + '}', +].join('\n')).join('\n'); + +async function explore(query: string, extra: Record = {}): Promise { + const result = await new ToolHandler(cg).execute('codegraph_explore', { query, ...extra }); + return result.content?.[0]?.text ?? ''; +} + +beforeAll(async () => { + dir = fs.mkdtempSync(path.join(os.tmpdir(), 'codegraph-cap-notices-')); + fs.writeFileSync(path.join(dir, 'big.ts'), bigSource); + fs.writeFileSync(path.join(dir, 'config.yaml'), 'flag: true\n'); + fs.writeFileSync(path.join(dir, '1.ts'), 'export const numericFile = 1;\n'); + fs.writeFileSync(path.join(dir, 'padding.ts'), 'export function padding() {\n' + Array.from({ length: 100 }, (_, i) => ` // padding row ${i} ${'p'.repeat(45)}`).join('\n') + '\n return 7;\n}'); + fs.writeFileSync(path.join(dir, 'partial.md'), Array.from({ length: 300 }, (_, i) => `## Tiny${i}\n\nSmall.\n\n\n`).join('\n') + '## TailSection\n' + Array.from({ length: 1900 }, () => 'x').join('\n') + '\n'); + fs.writeFileSync(path.join(dir, 'compact.ts'), Array.from({ length: 360 }, (_, i) => `export const val${i}=${i};`).join(' ')); + fs.writeFileSync(path.join(dir, 'manual.md'), Array.from({ length: 301 }, (_, i) => `## Chapter${String(i).padStart(3, '0')}\n\nParagraph ${i}.\n`).join('\n')); + for (let j = 0; j < 3; j++) fs.writeFileSync(path.join(dir, `large-doc${j}.md`), + Array.from({ length: 301 }, (_, i) => `## Section${i}\n\n${i === 300 ? Array.from({ length: 110 }, (_, k) => `Last section row ${k} ${'q'.repeat(45)}`).join('\n') : 'Small paragraph.'}\n`).join('\n')); + if (process.platform !== 'win32') { + const nested = path.join(dir, ...Array.from({ length: 14 }, () => 'd'.repeat(200))); + fs.mkdirSync(nested, { recursive: true }); + for (let i = 0; i < 7; i++) fs.writeFileSync(path.join(nested, `extra${i}.ts`), `export const extraValue${i} = ${i};`); + fs.writeFileSync(path.join(nested, 'enormous.ts'), bigSource); + } + for (let i = 0; i < 10; i++) fs.writeFileSync(path.join(dir, `unit${i}.ts`), `export function uniqueUnit${i}() { return ${i}; }`); + cg = CodeGraph.initSync(dir); + await cg.indexAll(); +}, 180_000); + +afterAll(() => { + cg?.destroy(); + if (dir) fs.rmSync(dir, { recursive: true, force: true }); +}); + +describe('bounded explore exclusion notices', () => { + it('offers a usable continuation for a pure-path gather tail', async () => { + const out = await explore('big.ts'); + const pointer = /Additional indexed source not included: `big\.ts:(\d+)-(\d+)`/.exec(out); + expect(pointer).not.toBeNull(); + expect(Number(pointer![1])).toBeGreaterThan(1000); + expect(out.length).toBeLessThanOrEqual(19500); + const continued = await explore(`big.ts:${pointer![1]}-${pointer![2]}`); + const firstLine = bigSource.split('\n')[Number(pointer![1]) - 1]!; + expect(continued).toContain(firstLine.trim()); + }); + + it('suppresses gather warnings when the complete file covers more than300 nodes', async () => { + expect(cg.getNodesInFile('compact.ts').filter(n => !['file', 'import', 'export'].includes(n.kind)).length).toBeGreaterThan(300); + const out = await explore('compact.ts'); + expect(out).toContain('export const val359=359;'); + expect(out).not.toContain('Additional indexed source not included'); + }); + + it('retains named source beyond the gather cap', async () => { + const out = await explore('big.ts work0359'); + expect(out).toContain('return 359;'); + }); + + it('names overflow references without adding source file slots', async () => { + const out = await explore('unit0.ts unit1.ts unit1.ts', { maxFiles: 1 }); + expect(out).toContain("Requested files not pinned within this call's initial file limit: `unit1.ts`"); + expect(out.match(/\*\*`unit[01]\.ts`/g)?.length).toBe(1); + expect(out).not.toContain('No indexed file uniquely matches'); + }); + + it('reports an overflow file when the selected file has no code nodes', async () => { + const out = await explore('config.yaml 1.ts', { maxFiles: 1 }); + expect(out).toContain('No relevant code found'); + expect(out).toContain("Requested files not pinned within this call's initial file limit: `1.ts`"); + }); + + it.runIf(process.platform !== 'win32' && process.getuid?.() !== 0)('does not promise a continuation for unreadable source', async () => { + const file = path.join(dir, 'big.ts'); + const mode = fs.statSync(file).mode; + fs.chmodSync(file, 0); + try { + expect(() => fs.readFileSync(file, 'utf8')).toThrow(/EACCES/); + const out = await explore('big.ts'); + expect(out).not.toContain('Additional indexed source not included'); + expect(out).not.toContain('independent source'); + } finally { fs.chmodSync(file, mode); } + }); + + it('offers the Markdown tail cut inside a surviving section', async () => { + const state = new ExploreSessionState(); + const result = await new ToolHandler(cg).execute('codegraph_explore', { + query: 'padding.ts partial.md TailSection', maxFiles: 2, + }, state); + const out = result.content?.[0]?.text ?? ''; + expect(out).toContain('output truncated to budget'); + expect(out).toContain('**`partial.md`'); + const pointer = /Additional indexed source not included: `partial\.md:(\d+)-(\d+)`/.exec(out); + expect(pointer).not.toBeNull(); + const source = fs.readFileSync(path.join(dir, 'partial.md'), 'utf8').split('\n'); + const firstMissing = Number(pointer![1]); + expect(out).not.toContain(`${firstMissing}\tx`); + const held = state.view().projects[0]!.calls[0]!.files.find(file => file.path === 'partial.md'); + expect(held).toBeDefined(); + expect(held!.ranges.every(range => range.end < firstMissing)).toBe(true); + expect(out.length).toBeLessThanOrEqual(19500); + const continued = await explore(`partial.md:${pointer![1]}-${pointer![2]}`); + expect(continued).toContain(`${firstMissing}\t${source[firstMissing - 1]}`); + }); + + it('keeps capacity notices off at the exact file limit', async () => { + const out = await explore('unit0.ts unit1.ts', { maxFiles: 2 }); + expect(out).toContain('uniqueUnit0'); + expect(out).toContain('uniqueUnit1'); + expect(out).not.toContain('Requested files not pinned'); + }); + + it('reports unexamined references beyond the scan budget', async () => { + const out = await explore(Array.from({ length: 10 }, (_, i) => `unit${i}.ts`).join(' '), { maxFiles: 1 }); + expect(out).toContain('Further path-like references were not examined'); + const notice = out.split("Requested files not pinned within this call's initial file limit: ")[1]!.split('. Explore')[0]!; + expect(notice).toContain('unit7.ts'); + expect(notice).not.toContain('unit8.ts'); + expect(out.length).toBeLessThanOrEqual(19500); + }); + + it('recognizes the selected Markdown section as delivered past the gather cap', async () => { + const out = await explore('manual.md Chapter300'); + expect(out).toContain('Paragraph 300.'); + expect(out).not.toContain('Additional indexed source not included'); + }); + + it('reports an unexamined path even when no relevant source was found', async () => { + const out = await explore(Array.from({ length: 9 }, (_, i) => `zzmissing${i}.zzz`).join(' ')); + expect(out).toContain('No relevant code found'); + expect(out).toContain('Further path-like references were not examined'); + }); + + it.runIf(process.platform !== 'win32')('bounds long path notices while retaining pinned source', async () => { + // POSIX allows this fixture below its4096-byte path limit. Windows long-path + // configuration is outside this retrieval test's contract. + const out = await explore('unit0.ts ' + Array.from({ length: 7 }, (_, i) => `extra${i}.ts`).join(' '), { maxFiles: 1 }); + expect(out).toContain('uniqueUnit0'); + expect(out).toContain("Requested files not pinned within this call's initial file limit:"); + expect(out).toContain('(7 more)'); + expect(out.length).toBeLessThanOrEqual(19500); + }); + + it.runIf(process.platform !== 'win32')('keeps gather guidance usable when its full path label cannot fit', async () => { + const out = await explore('enormous.ts'); + expect(out).toContain('Additional indexed source not included: (1 more)'); + expect(out).toContain('Explore fewer requested files or narrower line ranges'); + expect(out).not.toContain('Explore these ranges to continue'); + expect(out.length).toBeLessThanOrEqual(19500); + }); + + it('reports a capped section dropped at the final output boundary', async () => { + const out = await explore('large-doc0.md large-doc1.md large-doc2.md Section300', { maxFiles: 3 }); + expect(out).toContain('output truncated to budget'); + const missing = /Additional indexed source not included: `([^`]+):(\d+)-(\d+)`/.exec(out); + expect(missing).not.toBeNull(); + expect(out).not.toContain('**`' + missing![1] + '`'); + expect(out).toContain('Last section row'); + expect(out.length).toBeLessThanOrEqual(19500); + }); + + it('counts verified source held earlier in the same conversation', async () => { + vi.stubEnv('CODEGRAPH_EXPLORE_DEDUP', '1'); + const state = new ExploreSessionState(); + state.record({ projectRoot: dir, query: 'prior', sourceBytes: bigSource.length, responseBytes: bigSource.length, + files: [{ path: 'big.ts', fingerprint: fileFingerprint(bigSource), bytes: bigSource.length, + ranges: [{ start: 1, end: bigSource.split('\n').length }] }] }); + try { + const result = await new ToolHandler(cg).execute('codegraph_explore', { query: 'big.ts' }, state); + const out = result.content?.[0]?.text ?? ''; + expect(out).not.toContain('Additional indexed source not included'); + } finally { vi.unstubAllEnvs(); } + }); + + it('does not offer stale indexed continuation ranges', async () => { + fs.appendFileSync(path.join(dir, 'big.ts'), '\n// disk drift'); + try { + const out = await explore('big.ts'); + expect(out).toContain('changed on disk'); + expect(out).not.toContain('Additional indexed source not included'); + } finally { fs.writeFileSync(path.join(dir, 'big.ts'), bigSource); } + }); +}); diff --git a/__tests__/query-paths.test.ts b/__tests__/query-paths.test.ts index ed3640e7b..a0d9dbffe 100644 --- a/__tests__/query-paths.test.ts +++ b/__tests__/query-paths.test.ts @@ -159,7 +159,7 @@ describe('extractQueryPaths — resolution and stripping', () => { it('passes through untouched when nothing resolves', () => { const q = 'plain prose question about scrolling'; - const out = extractQueryPaths(q, INDEX); + const { capacityFiles, unexaminedPathSpans, ...out } = extractQueryPaths(q, INDEX); expect(out).toEqual({ strippedQuery: q, pinnedFiles: [], unresolvedPathSpans: [], lineAnchors: [], setAsideMatches: [] }); }); }); @@ -199,13 +199,13 @@ describe('extractQueryPaths — extension-less kebab basenames', () => { it('leaves kebab prose that names no indexed file untouched — and unreported', () => { const q = 'how does cross-call dedup make explore non-blocking'; - const out = extractQueryPaths(q, INDEX); + const { capacityFiles, unexaminedPathSpans, ...out } = extractQueryPaths(q, INDEX); expect(out).toEqual({ strippedQuery: q, pinnedFiles: [], unresolvedPathSpans: [], lineAnchors: [], setAsideMatches: [] }); }); it('leaves a stem shared by too many files alone — one hot name must not pin half the repo', () => { const q = 'refactor the user-profile rendering'; - const out = extractQueryPaths(q, INDEX); + const { capacityFiles, unexaminedPathSpans, ...out } = extractQueryPaths(q, INDEX); expect(out).toEqual({ strippedQuery: q, pinnedFiles: [], unresolvedPathSpans: [], lineAnchors: [], setAsideMatches: [] }); }); @@ -511,3 +511,45 @@ describe('extractQueryPaths — same-named files, narrowed by named symbols', () expect(l.asked).toEqual([]); }); }); + + +describe('explicit reference capacity reporting', () => { + it('reports overflow once while preserving the original matching query', () => { + const out = extractQueryPaths('src/lib/chat-manager.ts src/lib/task-runner-manager.ts src/lib/task-runner-manager.ts', INDEX, { maxPins: 1 }); + expect(out.pinnedFiles).toEqual(['src/lib/chat-manager.ts']); + expect(out.capacityFiles).toEqual(['src/lib/task-runner-manager.ts']); + expect(out.strippedQuery).toBe('src/lib/task-runner-manager.ts src/lib/task-runner-manager.ts'); + expect(out.unresolvedPathSpans).toEqual([]); + }); + + it('reports partial shared-basename capacity separately from ambiguity and missing files', () => { + const out = extractQueryPaths('generic-modal.tsx src/missing.ts and/or', INDEX, { maxPins: 1 }); + expect(out.pinnedFiles).toEqual(['src/x/generic-modal.tsx']); + expect(out.capacityFiles).toEqual(['src/y/generic-modal.tsx']); + expect(out.setAsideMatches).toEqual([]); + expect(out.strippedQuery).toBe('src/missing.ts and/or'); + }); + + it('does not count duplicate pins, ambiguous references or prose as capacity overflow', () => { + const out = extractQueryPaths('src/lib/chat-manager.ts src/lib/chat-manager.ts user-profile.tsx and/or', INDEX, { maxPins: 1 }); + expect(out.capacityFiles).toEqual([]); + expect(out.unexaminedPathSpans).toBe(false); + }); + + it('marks the bounded scan incomplete without inventing overflow matches', () => { + const files = Array.from({ length: 10 }, (_, i) => `src/unit${i}.ts`); + const out = extractQueryPaths(files.join(' '), files, { maxPins: 1 }); + expect(out.pinnedFiles).toEqual([files[0]]); + expect(out.capacityFiles).toEqual(files.slice(1, 8)); + expect(out.unexaminedPathSpans).toBe(true); + expect(out.strippedQuery).toBe(files.slice(1).join(' ')); + }); +}); + + +it('reports extensionless overflow after explicit paths fill pin capacity', () => { + const out = extractQueryPaths('background-image-table then src/lib/chat-manager.ts', INDEX, { maxPins: 1 }); + expect(out.pinnedFiles).toEqual(['src/lib/chat-manager.ts']); + expect(out.capacityFiles).toEqual(['src/components/training-set-page/background-image-table.tsx']); + expect(out.strippedQuery).toBe('background-image-table then'); +}); diff --git a/site/src/content/docs/reference/mcp-server.md b/site/src/content/docs/reference/mcp-server.md index 29b8fe985..fb1766552 100644 --- a/site/src/content/docs/reference/mcp-server.md +++ b/site/src/content/docs/reference/mcp-server.md @@ -21,6 +21,8 @@ By default the server exposes `codegraph_explore` for code and `codegraph_sessio Exposing one strong code tool is deliberate. Measured agent behavior showed that one well-aimed tool steers agents to a direct answer better than a menu of narrower ones — fewer mis-picks — and agents reach for it both when answering questions and while editing code. +Explore reports requested files that did not receive pin priority within the file limit. Those files may still appear through other matches. References beyond the bounded path scan are marked unexamined. When a capped file gather leaves indexed source uncovered, the summary offers continuation ranges for another explore call. Existing file and output limits still apply. + Explicit code-file pins remain eligible when the query also names documentation. Quoted terms used to select template or style source remain meaningful even when they match the filename stem. File and output budgets still apply. When a query names functions in pinned files, `codegraph_explore` prioritizes their local callee bodies through two call or callback hops. It selects at most sixteen local helpers per file, each at most 200 lines. File and output budgets still apply. diff --git a/src/mcp/server-instructions.ts b/src/mcp/server-instructions.ts index 18a7711c9..c79467b3f 100644 --- a/src/mcp/server-instructions.ts +++ b/src/mcp/server-instructions.ts @@ -72,6 +72,7 @@ against dozens of greps and reads. - **Read/Grep are for what the index lacks**: a detail a codegraph answer didn't cover, or files codegraph doesn't index (such as configs). Markdown is indexed — every \`.md\` file's headings, sections, tables and links — so a documentation question (a rule, a runbook, a plan row, a research finding) goes to \`codegraph_explore\` too; it returns the section body, not just its heading. - **A question about the working changes** ("what do my changes affect", "this branch", \`main..HEAD\`) goes to \`codegraph_explore\` as asked: it reads the diff itself (merge base with the default branch plus uncommitted edits, or the named range) and leads with the changed symbols, their callers and their tests. - **Near-duplicates (update together)** in a blast radius or trail names other function bodies nearly identical to the one you are reading. A bug fixed in one copy is usually still in the others: check them, and tell the user if you leave them unchanged. +- **Requested-source limit notices** distinguish files not pinned within the file limit from unexamined path-like references. A file without pin priority may still appear through other matches. If the summary names uncovered indexed source ranges, explore those ranges to continue; the file and output limits still apply. - **"Not found in the index: \`name\`"** in the summary means that symbol-shaped name has no symbol and appears in no indexed code file (Markdown is not checked). Check its spelling or ask whether it exists. Grepping indexed code finds nothing new; a search outside the index may still find the name. - **Don't reconstruct a flow by hand** — name the endpoints in one \`codegraph_explore\` and it surfaces the path between them, dynamic-dispatch hops included. - **After editing, check the staleness banner.** When a tool response starts with "⚠️ Some files referenced below were edited since the last index sync…", the listed files are pending re-index — Read those specific files for accurate content. Every file NOT in that banner is fresh, so still trust codegraph. A different, rarer banner — "⚠️ CodeGraph auto-sync is DISABLED…" or "⚠️ CodeGraph auto-sync is RECOVERING…" — means the graph may lag behind recent edits until a sync completes; source is still read from disk. A response that lists files which changed after their last sync still answers, but symbols, callers and callees taken from those files may be out of date; if they matter, retry after \`codegraph sync\`. diff --git a/src/mcp/tools.ts b/src/mcp/tools.ts index 6beb33c71..899eb9a4a 100644 --- a/src/mcp/tools.ts +++ b/src/mcp/tools.ts @@ -88,6 +88,7 @@ import { fileFingerprint, formatBackReference, mergeRanges, + subtractRange, servedRangesForFile, symbolsInSpans, } from './explore-dedup'; @@ -4631,6 +4632,8 @@ export class ToolHandler { let unresolvedPathSpans: string[] = []; let lineAnchors: QueryLineAnchor[] = []; let setAsideMatches: QuerySetAsideMatch[] = []; + let capacityFiles: string[] = []; + let unexaminedPathSpans = false; let matchQuery = changes ? normalizeQuerySpelling(rawMatch) : query; if (queryMightContainPaths(rawMatch)) { try { @@ -4643,15 +4646,21 @@ export class ToolHandler { symbolFiles: (symbol) => filesDefiningSymbol(cg, symbol), }, ); - if (extraction.pinnedFiles.length > 0 || extraction.unresolvedPathSpans.length > 0) { + if (extraction.pinnedFiles.length > 0 || extraction.unresolvedPathSpans.length > 0 + || extraction.unexaminedPathSpans) { pinnedFiles = extraction.pinnedFiles; unresolvedPathSpans = extraction.unresolvedPathSpans; lineAnchors = extraction.lineAnchors; setAsideMatches = extraction.setAsideMatches; + capacityFiles = extraction.capacityFiles; + unexaminedPathSpans = extraction.unexaminedPathSpans; matchQuery = normalizeQuerySpelling(extraction.strippedQuery); } } catch { /* path pinning must never fail an explore call */ } } + const scanNote = unexaminedPathSpans + ? ' Further path-like references were not examined within the bounded path scan; explore them in a separate call.' + : ''; // A COBOL copybook the query names (`CVACT01Y`) names its FILE as surely as // `app/cpy/CVACT01Y.cpy` would (#2342), so it is pinned the same way: the // copybook's own source is admitted, ranked first and funded first. Looked @@ -4743,6 +4752,7 @@ export class ToolHandler { string, { ranges: ExploreLineRange[]; bytes: number; fingerprint?: string } >(); + const docRenderedLines = new Map(); const noteEmitted = ( fp: string, ranges: ExploreLineRange[], @@ -4787,6 +4797,7 @@ export class ToolHandler { // the file itself, so its contents ARE the answer regardless of what the // stripped query text matched (which, for a pure-path query, is nothing). const PINNED_FILE_NODE_CAP = 300; + const gatherExcluded = new Map(); for (const fp of pinnedFiles) { let fileNodes: Node[] = []; try { fileNodes = cg.getNodesInFile(fp); } catch { continue; } @@ -4797,9 +4808,13 @@ export class ToolHandler { const file = fileNodes.find(n => n.kind === 'file' && !CONFIG_LEAF_LANGUAGES.has(n.language)); if (file) seeds.push(file); } - seeds - .sort((a, b) => a.startLine - b.startLine) - .slice(0, PINNED_FILE_NODE_CAP) + seeds.sort((a, b) => a.startLine - b.startLine); + if (seeds.length > PINNED_FILE_NODE_CAP) { + gatherExcluded.set(fp, mergeRanges(seeds.slice(PINNED_FILE_NODE_CAP) + .filter(n => n.startLine > 0 && n.endLine >= n.startLine) + .map(n => ({ start: n.startLine, end: n.endLine })))); + } + seeds.slice(0, PINNED_FILE_NODE_CAP) .forEach((n) => { if (!subgraph.nodes.has(n.id)) subgraph.nodes.set(n.id, n); }); } @@ -4859,6 +4874,23 @@ export class ToolHandler { anchorSpans.set(anchor.file, spans); } + // Capacity notices describe pin priority, not whether another channel + // happened to return the same file. Keep the list bounded and deduplicated. + const NOTICE_LABEL_CHARS = 600; + const boundedNotice = (prefix: string, entries: string[], suffix: string, maxEntries: number): string => { + if (entries.length === 0) return ''; + const labels: string[] = []; + for (const entry of entries) { + if (labels.length >= maxEntries || [...labels, entry].join(', ').length > NOTICE_LABEL_CHARS) break; + labels.push(entry); + } + const omitted = entries.length - labels.length; + return prefix + labels.join(', ') + (omitted > 0 ? `${labels.length > 0 ? ' ' : ''}(${omitted} more)` : '') + + (labels.length === 0 ? '. Explore fewer requested files or narrower line ranges in a separate call.' : suffix); + }; + const capacityNote = boundedNotice(" Requested files not pinned within this call's initial file limit: ", + capacityFiles.map(fp => `\`${fp}\``), '. Explore fewer requested paths in a separate call.', POINTER_MAX_FILES); + if (subgraph.nodes.size === 0 && proseFiles.size === 0) { diag?.finishEmpty('no relevant code found — empty subgraph'); const missNote = unresolvedPathSpans.length > 0 @@ -4891,8 +4923,8 @@ export class ToolHandler { : '\nNo shared-word symbol candidates found; retry codegraph_explore with literal symbol/file names or code terms.'; } const empty = changes - ? `${this.buildChangesSection(cg, changes, changedNodes)}\nNo other relevant code found for "${query}"${missNote}${explanation}` - : `No relevant code found for "${query}"${missNote}${explanation}`; + ? `${this.buildChangesSection(cg, changes, changedNodes)}\nNo other relevant code found for "${query}"${missNote}${explanation}${capacityNote}${scanNote}` + : `No relevant code found for "${query}"${missNote}${explanation}${capacityNote}${scanNote}`; // Still an explore call, so it is still recorded: an empty answer spends a // call against the tier budget even though it emits no source. return { ...this.textResult(empty), [EXPLORE_EMISSION_KEY]: { @@ -6080,18 +6112,32 @@ export class ToolHandler { const unmatchedNamesNote = unmatchedNames.length > 0 ? ` Not found in the index: ${unmatchedNames.map((n) => `\`${n}\``).join(', ')}.` : ''; - const formatSummary = (symbols: number, files: number, pins: number): string => { + const gatherPrefix = ' Additional indexed source not included: '; + const gatherSuffix = '. Explore these ranges to continue.'; + const gatherNotice = (ranges: QueryLineAnchor[]): string => boundedNotice(gatherPrefix, + ranges.map(r => `\`${r.file}:${r.start}-${r.end}\``), gatherSuffix, TRIMMED_FILES_NAMED); + // Bound every possible subset of final uncovered files. Long path labels + // cannot consume the source envelope, even when no complete label fits. + const longestLabels = [...gatherExcluded].map(([file, ranges]) => { + const end = ranges.reduce((max, range) => Math.max(max, range.end), 0); + return `\`${file}:${end}-${end}\``.length; + }).sort((a, b) => b - a).slice(0, TRIMMED_FILES_NAMED); + const gatherNoticeReserve = gatherExcluded.size === 0 ? '' : ' '.repeat( + gatherPrefix.length + gatherSuffix.length + + Math.min(NOTICE_LABEL_CHARS, longestLabels.reduce((n, size) => n + size + 2, 0)) + + ` (${gatherExcluded.size} more)`.length); + const formatSummary = (symbols: number, files: number, pins: number, gatherNote = ''): string => { let text = `Found ${symbols} symbol${symbols === 1 ? '' : 's'} across ${files} file${files === 1 ? '' : 's'}.`; if (pins > 0) text += ` ${pins} file${pins === 1 ? '' : 's'} pinned from the query.`; if (unresolvedPathSpans.length > 0) { text += ` No indexed file uniquely matches ${unresolvedPathSpans.map(s => `\`${s}\``).join(', ')}.`; } - return text + unmatchedNamesNote + setAsideNote; + return text + unmatchedNamesNote + setAsideNote + capacityNote + scanNote + gatherNote; }; // Reserve the summary's maximum actual size before spending the source // envelope. Replacing a short sentinel after fitting could exceed the cap. const summaryPlaceholder = SUMMARY_SENTINEL.padEnd( - formatSummary(subgraph.nodes.size, fileGroups.size, pinnedFiles.length).length, + formatSummary(subgraph.nodes.size, fileGroups.size, pinnedFiles.length, gatherNoticeReserve).length, ); // Step 3: Build relationship map @@ -6450,7 +6496,7 @@ export class ToolHandler { ? ` No indexed file uniquely matches ${unresolvedPathSpans.map((sp) => `\`${sp}\``).join(', ')}.`.length : 0) + unmatchedNamesNote.length - + setAsideNote.length; + + setAsideNote.length + capacityNote.length + scanNote.length + gatherNoticeReserve.length; const epilogueFloor = Math.max( EXPLORE_FALLBACK_NOTES.lost.complete.length, EXPLORE_FALLBACK_NOTES.lost.trimmed.length, ) + 2 + cliffPointerFloor + summaryReserve; @@ -7135,7 +7181,7 @@ export class ToolHandler { // an answer that is a list or a table only reads as one when every row // is present, and its rows rarely carry the question's words. const docSecs = docSections.get(filePath); - if (docSecs) { + if (docSecs && !anchorSpans.has(filePath)) { const DOC_FILE_CAP = 8000; const numbered = (ln: number) => exploreLineNumbersEnabled() ? `${ln}\t${fileLines[ln - 1] ?? ''}` : (fileLines[ln - 1] ?? ''); @@ -7171,16 +7217,30 @@ export class ToolHandler { } const parts: string[] = []; const shown: string[] = []; + const renderedLines: { line: number; end: number; bytes: number }[] = []; + let bodyOffset = 0; for (const { node: s } of docSecs) { const chosen = picked.get(s.id); if (!chosen) continue; - parts.push(chosen.map(numbered).join('\n')); + const rows = chosen.map(ln => { + const text = numbered(ln); + bodyOffset += text.length; + renderedLines.push({ line: ln, end: bodyOffset, bytes: (fileLines[ln - 1] ?? '').length }); + bodyOffset += 1; + return text; + }); + parts.push(rows.join('\n')); + bodyOffset += 1; // Two newlines separate selected sections. shown.push(s.name); } if (parts.length > 0) { const suffix = `sections: ${shown.join(' · ')}${elided ? ' (long sections show only the rows that match the question)' : ''} — the rest of the file is not shown`; - lines.push(fileSectionHeader(filePath, suffix), '', '```' + lang, parts.join('\n\n'), '```', ''); + const header = fileSectionHeader(filePath, suffix); + lines.push(header, '', '```' + lang, parts.join('\n\n'), '```', ''); + docRenderedLines.set(filePath, { marker: header + '\n\n```' + lang + '\n', lines: renderedLines }); totalChars += used + 120; + noteEmitted(filePath, mergeRanges([...picked.values()].flat() + .map(ln => ({ start: ln, end: ln }))), used, fingerprint); renderedFilePaths.push(filePath); filesIncluded++; if (elided) anyFileTrimmed = true; @@ -8690,6 +8750,7 @@ export class ToolHandler { const output = flow.text + lines.join('\n'); let finalText: string; + let retainedTextLength = output.length; // The epilogue costs less than a file section, so it is cut FIRST (CG-31). // Dropping a trailing section throws away source the render loop had already // set that file's reservation aside for — the exact starvation the @@ -8706,6 +8767,7 @@ export class ToolHandler { && epilogueOnlyCut !== null && epilogueOnlyCut.length + EPILOGUE_CUT_NOTE.length <= hardCeiling) { finalText = epilogueOnlyCut + EPILOGUE_CUT_NOTE; + retainedTextLength = epilogueOnlyCut.length; } else if (output.length > hardCeiling) { // Still over with the epilogue gone: cut at a FILE-SECTION boundary (the // last ``**` `` file header before the ceiling) so we drop whole trailing @@ -8713,11 +8775,16 @@ export class ToolHandler { // through a method body — a half-rendered method just forces the Read this // tool exists to prevent. Fall back to a line boundary only if no section // header sits in the back half (degenerate single-giant-section case). - const cut = output.slice(0, hardCeiling); + const cutNoteReserve = Math.max( + EXPLORE_FALLBACK_NOTES.truncated.complete.length, + EXPLORE_FALLBACK_NOTES.truncated.trimmed.length, + ); + const cut = output.slice(0, Math.max(0, hardCeiling - cutNoteReserve)); const lastSection = cut.lastIndexOf('\n' + FILE_SECTION_PREFIX); const boundary = lastSection > hardCeiling * 0.5 ? lastSection : cut.lastIndexOf('\n'); const safe = boundary > 0 ? cut.slice(0, boundary) : cut; finalText = safe + EXPLORE_FALLBACK_NOTES.truncated[trimmedIn(safe) ? 'trimmed' : 'complete']; + retainedTextLength = safe.length; } else { finalText = output; } @@ -8731,6 +8798,17 @@ export class ToolHandler { // shown above" + the budget note, so nothing is silently dropped. const survivors = renderedFilePaths.filter((fp) => finalText.includes(`${FILE_SECTION_PREFIX}${fp}\``)); + // A surviving header can precede a cut inside a Markdown section. Record + // only complete source lines kept by that cut, with or without numbering. + for (const [file, rendered] of docRenderedLines) { + const emitted = emittedByFile.get(file); + if (!emitted) continue; + const markerAt = finalText.indexOf(rendered.marker); + const bodyAt = markerAt + rendered.marker.length; + const kept = markerAt < 0 ? [] : rendered.lines.filter(row => bodyAt + row.end <= retainedTextLength); + emitted.ranges = mergeRanges(kept.map(row => ({ start: row.line, end: row.line }))); + emitted.bytes = kept.reduce((sum, row) => sum + row.bytes + 1, 0); + } const shownSymbols = survivors.reduce((sum, fp) => { const g = fileGroups.get(fp); if (!g) return sum; @@ -8742,10 +8820,37 @@ export class ToolHandler { // honored, and which path spans matched nothing so the agent can correct // them instead of trusting a response that quietly ignored the path. const pinnedShown = pinnedFiles.filter((fp) => survivors.includes(fp)).length; + const uncoveredGather: QueryLineAnchor[] = []; + for (const [file, excluded] of gatherExcluded) { + // Indexed line pointers are unsafe after drift, and unreadable files have + // no verified continuation. The existing drift guard owns their notice. + const absolute = validatePathWithinRoot(projectRoot, file); + if (!absolute || !existsSync(absolute) || this.isFileStaleOnDisk(cg, file)) continue; + let content: string; + try { content = readFileSync(absolute, 'utf8'); } catch { continue; } + const emission = emittedByFile.get(file); + let held: ExploreLineRange[] = []; + if (dedupEnabled && priorCalls?.calls.some(call => call.files.some(entry => entry.path === file))) { + try { + const fingerprint = emission?.fingerprint ?? fileFingerprint(content); + held = servedRangesForFile(priorCalls, file, fingerprint); + } catch { continue; } + } + const delivered = mergeRanges([ + ...(survivors.includes(file) ? emission?.ranges ?? [] : []), ...held, + ]); + let missing: ExploreLineRange | undefined; + for (const range of excluded) { + missing = subtractRange(range, delivered)[0]; + if (missing) break; + } + if (missing) uncoveredGather.push({ file, ...missing }); + } const summaryLine = formatSummary( survivors.length > 0 ? shownSymbols : subgraph.nodes.size, survivors.length > 0 ? survivors.length : fileGroups.size, pinnedShown, + gatherNotice(uncoveredGather), ); finalText = finalText.replace(summaryPlaceholder, summaryLine); diff --git a/src/search/query-paths.ts b/src/search/query-paths.ts index 0673b8daf..a372c7c92 100644 --- a/src/search/query-paths.ts +++ b/src/search/query-paths.ts @@ -60,6 +60,10 @@ export interface QueryPathExtraction { * Surfaced so a set-aside file is visible, not silently dropped. */ setAsideMatches: QuerySetAsideMatch[]; + /** Resolved references that did not receive pin priority because maxPins was full. */ + capacityFiles: string[]; + /** Further path-like references remain beyond the bounded candidate scan. */ + unexaminedPathSpans: boolean; } export interface QuerySetAsideMatch { @@ -326,6 +330,8 @@ export function extractQueryPaths( unresolvedPathSpans: [], lineAnchors: [], setAsideMatches: [], + capacityFiles: [], + unexaminedPathSpans: false, }; if (!query.trim() || indexedPaths.length === 0) return passthrough; @@ -339,6 +345,8 @@ export function extractQueryPaths( const pinned: string[] = []; const pinnedSeen = new Set(); const unresolved: string[] = []; + const capacity = new Set(); + let unexaminedPathSpans = false; const anchors: QueryLineAnchor[] = []; const setAside: QuerySetAsideMatch[] = []; /** Token index → the ONE file it pinned, for binding prose line ranges. */ @@ -395,8 +403,14 @@ export function extractQueryPaths( const collectLimit = opts.symbolFiles ? Number.POSITIVE_INFINITY : maxMatchesPerSpan; for (let i = 0; i < tokens.length; i++) { - if (pinned.length >= maxPins) break; - if (candidatesExamined >= MAX_CANDIDATE_SPANS) break; + if (candidatesExamined >= MAX_CANDIDATE_SPANS) { + unexaminedPathSpans = tokens.slice(i).some(token => { + const span = stripWrapping(token).path; + return span.length >= 4 && (/[/\\]/.test(span) || DOTTED_BASENAME.test(span)); + }); + break; + } + const atCapacity = pinned.length >= maxPins; const { path: stripped, lines } = stripWrapping(tokens[i]!); if (stripped.length < 4) continue; const hasSlash = /[/\\]/.test(stripped); @@ -407,13 +421,21 @@ export function extractQueryPaths( candidatesExamined++; const resolved = resolveSpan(normalized.toLowerCase(), lowerToOriginal, collectLimit); + const asideCount = setAside.length; const pinnable = resolved.matches.length > 0 ? pinnableMatches(normalized, resolved.matches) : null; const matches = pinnable ?? []; const ambiguous = resolved.ambiguous || (resolved.matches.length > 0 && pinnable === null); + if (atCapacity) { + // Observe overflow without changing the matching query or disambiguation. + setAside.length = asideCount; + for (const m of matches) if (!pinnedSeen.has(m)) capacity.add(m); + continue; + } if (matches.length > 0) { consumed.add(i); for (const m of matches) { - if (pinnedSeen.has(m) || pinned.length >= maxPins) continue; + if (pinnedSeen.has(m)) continue; + if (pinned.length >= maxPins) { capacity.add(m); continue; } pinnedSeen.add(m); pinned.push(m); } @@ -458,13 +480,19 @@ export function extractQueryPaths( // examines every remaining token: lookups are O(1) map hits, so the // scan-cost rationale behind MAX_CANDIDATE_SPANS doesn't apply. let basenameStems: Map | null = null; - for (let i = 0; i < tokens.length && pinned.length < maxPins; i++) { + for (let i = 0; i < tokens.length; i++) { if (consumed.has(i)) continue; const { path: stripped, lines } = stripWrapping(tokens[i]!); if (stripped.length < 4 || !KEBAB_BASENAME.test(stripped)) continue; basenameStems ??= buildBasenameStems(indexedPaths); const stemMatches = basenameStems.get(stripped.toLowerCase()); + const asideCount = setAside.length; const matches = stemMatches ? pinnableMatches(stripped, stemMatches) : null; + if (pinned.length >= maxPins) { + setAside.length = asideCount; + for (const m of matches ?? []) if (!pinnedSeen.has(m)) capacity.add(m); + continue; + } if (!matches) continue; // A quoted duplicate stem asks for source evidence inside an already // pinned file. Keep it in the matching query rather than consuming it again. @@ -472,7 +500,8 @@ export function extractQueryPaths( && matches.some(m => pinnedSeen.has(m)); if (!keepQuotedStem) consumed.add(i); for (const m of matches) { - if (pinnedSeen.has(m) || pinned.length >= maxPins) continue; + if (pinnedSeen.has(m)) continue; + if (pinned.length >= maxPins) { capacity.add(m); continue; } pinnedSeen.add(m); pinned.push(m); } @@ -523,11 +552,13 @@ export function extractQueryPaths( } } - if (consumed.size === 0) return passthrough; + if (consumed.size === 0 && capacity.size === 0 && !unexaminedPathSpans) return passthrough; const seenAnchor = new Set(); return { strippedQuery: tokens.filter((_, i) => !consumed.has(i)).join(' '), pinnedFiles: pinned, + capacityFiles: [...capacity], + unexaminedPathSpans, unresolvedPathSpans: unresolved, lineAnchors: anchors.filter((a) => { const key = `${a.file}:${a.start}-${a.end}`;