diff --git a/CHANGELOG.md b/CHANGELOG.md index 83d8e34229..8006dbbcab 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,10 @@ and adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +### New Features + +- `codegraph_explore` answers sooner from a long-running MCP server or agent session. Each call used to recount the whole graph, reload every symbol name for typo-tolerant search, and look for a `codegraph.json` once per candidate it ranked; the first two are now kept until the index changes, and the config check is cheap when there is no config file. Answers are unchanged. + ### Fixes - Laravel routes are now named by the path a request takes: a leading `/` is added where the routes file leaves it out, `Route::prefix()` and `Route::group(['prefix' => …])` groups are applied, and routes in `routes/api.php` carry the `/api` prefix Laravel serves them under, read from your `RouteServiceProvider`, `bootstrap/app.php` or any file that mounts a routes file. A front-end `fetch('/api/…')` now connects to the Laravel route that serves it. Re-index Laravel projects after upgrading. diff --git a/__tests__/explore-repeat-work-cache.test.ts b/__tests__/explore-repeat-work-cache.test.ts new file mode 100644 index 0000000000..018aaf9584 --- /dev/null +++ b/__tests__/explore-repeat-work-cache.test.ts @@ -0,0 +1,110 @@ +/** + * Work every `codegraph_explore` repeated although its answer depends only on + * the index: the graph's stats counts (explore sizes its output from them), + * the distinct name list fuzzy search walks, and the `codegraph.json` probe + * search ranking makes for each candidate path. The first two are now held per + * database change stamp, like the dominant file (#1864). + */ + +import { describe, it, expect, afterEach, vi } from 'vitest'; +import * as fs from 'fs'; +import * as path from 'path'; +import * as os from 'os'; +import CodeGraph from '../src'; +import { QueryBuilder } from '../src/db/queries'; + +function queriesOf(cg: CodeGraph): QueryBuilder { + return (cg as unknown as { queries: QueryBuilder }).queries; +} + +function dbOf(cg: CodeGraph): any { + return (cg as any).db.getDb(); +} + +/** Count statements run through this CodeGraph's connection whose SQL matches. */ +function countStatements(cg: CodeGraph, pattern: RegExp): () => number { + const db = dbOf(cg); + const prepare = db.prepare.bind(db); + let n = 0; + vi.spyOn(db, 'prepare').mockImplementation((sql: unknown) => { + const stmt = prepare(sql as string); + if (!pattern.test(String(sql))) return stmt; + for (const method of ['all', 'get'] as const) { + const run = stmt[method].bind(stmt); + stmt[method] = (...args: unknown[]) => { n++; return run(...args); }; + } + return stmt; + }); + // Statements prepared before the spy (lazily cached ones) must be re-prepared. + (queriesOf(cg) as any).stmts = {}; + return () => n; +} + +describe('explore repeat work — held per index state', () => { + let dir: string; + const open: CodeGraph[] = []; + + afterEach(() => { + for (const cg of open.splice(0)) cg.close(); + vi.restoreAllMocks(); + if (dir) fs.rmSync(dir, { recursive: true, force: true }); + }); + + async function setup(): Promise { + dir = fs.mkdtempSync(path.join(os.tmpdir(), 'cg-repeat-')); + fs.writeFileSync(path.join(dir, 'engine.ts'), 'export function engineStart(): void { engineRun(); }\nexport function engineRun(): void {}\n'); + const cg = await CodeGraph.init(dir, { index: true }); + open.push(cg); + return cg; + } + + it('counts the graph once across explores while the index is unchanged', async () => { + const cg = await setup(); + const counted = countStatements(cg, /GROUP BY kind/); + for (const q of ['engine start', 'how does the engine run', 'engineRun']) { + await cg.findRelevantContext(q); + cg.getStats(); + } + expect(counted()).toBe(2); // nodes by kind + edges by kind, once + }); + + it('hands every caller its own stats object', async () => { + const cg = await setup(); + const first = cg.getStats(); + first.nodesByKind.function = -1; + first.fileCount = -1; + const second = cg.getStats(); + expect(second.nodesByKind.function).toBe(2); + expect(second.fileCount).toBe(1); + }); + + it('reuses the name list across fuzzy searches while the index is unchanged', async () => { + const cg = await setup(); + const counted = countStatements(cg, /SELECT DISTINCT name FROM nodes$/); + const q = queriesOf(cg); + for (let i = 0; i < 3; i++) q.getAllNodeNames(); + expect(counted()).toBe(1); + }); + + it('sees new counts and names after a sync', async () => { + const cg = await setup(); + expect(cg.getStats().fileCount).toBe(1); + expect(queriesOf(cg).getAllNodeNames()).not.toContain('pluginLoad'); + fs.writeFileSync(path.join(dir, 'plugin.ts'), 'export function pluginLoad(): void {}\n'); + await cg.sync(); + expect(cg.getStats().fileCount).toBe(2); + expect(queriesOf(cg).getAllNodeNames()).toContain('pluginLoad'); + }); + + it('sees a sync made through another connection (another process)', async () => { + const writer = await setup(); + const reader = await CodeGraph.open(dir); + open.push(reader); + expect(reader.getStats().fileCount).toBe(1); + fs.writeFileSync(path.join(dir, 'plugin.ts'), 'export function pluginLoad(): void {}\n'); + await writer.sync(); + expect(reader.getStats().fileCount).toBe(2); + expect(queriesOf(reader).getAllNodeNames()).toContain('pluginLoad'); + }); +}); + diff --git a/src/db/queries.ts b/src/db/queries.ts index 443b851372..b1ca33cdf7 100644 --- a/src/db/queries.ts +++ b/src/db/queries.ts @@ -234,6 +234,9 @@ function rowToFileRecord(row: FileRow): FileRecord { /** * Query builder for the knowledge graph database */ +/** Distinct-name lists longer than this are recomputed rather than held. */ +const MAX_MEMOIZED_NODE_NAMES = 250_000; + export class QueryBuilder { private db: SqliteDatabase; @@ -251,12 +254,11 @@ export class QueryBuilder { private nodeCache: Map = new Map(); private readonly maxCacheSize = 1000; - // getDominantFile()'s answer, tagged with the database change stamp it was - // computed under (see getChangeStamp). Query-independent, so one value + // Whole-graph answers that do not depend on the query (the dominant file, + // the distinct name list, the stats counts), each tagged with the database + // change stamp it was computed under (see memoizeByChangeStamp). One value // serves every explore until the database changes (#1864). - private dominantFileMemo: - | { stamp: string; value: { filePath: string; edgeCount: number; nextEdgeCount: number } | null } - | undefined; + private stampMemos = new Map(); // Prepared statements (lazily initialized) private stmts: { @@ -390,7 +392,7 @@ export class QueryBuilder { // The change stamp is per connection, and fresh connections to two // different databases report the same one — the memo goes with the old // connection, or a worker following a rebuilt index keeps its answer (#1864). - this.dominantFileMemo = undefined; + this.stampMemos.clear(); } private edgeKindStmt(sql: string): SqliteStatement { @@ -1074,18 +1076,28 @@ export class QueryBuilder { * stamp: recomputed only after something wrote to the database. */ getDominantFile(): { filePath: string; edgeCount: number; nextEdgeCount: number } | null { + return this.memoizeByChangeStamp('dominantFile', () => this.computeDominantFile()); + } + + /** + * `compute()`'s value, reused until the database changes (see getChangeStamp). + * `keep` can decline to hold a value (e.g. one too large to pin in memory). + */ + private memoizeByChangeStamp(key: string, compute: () => T, keep: (value: T) => boolean = () => true): T { // total_changes() counts writes that are later rolled back, so a result // read inside a transaction could outlive a ROLLBACK under an unchanged // stamp. Never keep one; `undefined` (a runtime without the getter) is // treated the same way. if (this.db.inTransaction !== false) { - this.dominantFileMemo = undefined; - return this.computeDominantFile(); + this.stampMemos.delete(key); + return compute(); } const stamp = this.getChangeStamp(); - if (this.dominantFileMemo?.stamp === stamp) return this.dominantFileMemo.value; - const value = this.computeDominantFile(); - this.dominantFileMemo = { stamp, value }; + const memo = this.stampMemos.get(key); + if (memo?.stamp === stamp) return memo.value as T; + const value = compute(); + if (keep(value)) this.stampMemos.set(key, { stamp, value }); + else this.stampMemos.delete(key); return value; } @@ -3459,12 +3471,21 @@ export class QueryBuilder { /** * Get all distinct node names (lightweight — just name strings for pre-filtering) */ - getAllNodeNames(): string[] { - if (!this.stmts.getAllNodeNames) { - this.stmts.getAllNodeNames = this.db.prepare('SELECT DISTINCT name FROM nodes'); - } - const rows = this.stmts.getAllNodeNames.all() as Array<{ name: string }>; - return rows.map((r) => r.name); + getAllNodeNames(): readonly string[] { + // Fuzzy search walks this list on every call. Held until the database + // changes, unless it is large enough that pinning it would cost real + // memory for the life of a long-running server. + return this.memoizeByChangeStamp( + 'allNodeNames', + () => { + if (!this.stmts.getAllNodeNames) { + this.stmts.getAllNodeNames = this.db.prepare('SELECT DISTINCT name FROM nodes'); + } + const rows = this.stmts.getAllNodeNames.all() as Array<{ name: string }>; + return Object.freeze(rows.map((r) => r.name)); + }, + (names) => names.length <= MAX_MEMOIZED_NODE_NAMES + ); } /** @@ -3878,6 +3899,19 @@ export class QueryBuilder { * Get graph statistics */ getStats(): GraphStats { + // Explore sizes its output from these counts on every call. Callers fill + // in the size fields, so each gets its own copy of the memoized counts. + const stats = this.memoizeByChangeStamp('stats', () => this.computeStats()); + return { + ...stats, + nodesByKind: { ...stats.nodesByKind }, + edgesByKind: { ...stats.edgesByKind }, + filesByLanguage: { ...stats.filesByLanguage }, + lastUpdated: Date.now(), + }; + } + + private computeStats(): GraphStats { // Single query for all three aggregate counts const counts = this.db.prepare(` SELECT diff --git a/src/project-config.ts b/src/project-config.ts index 56c5debe18..e1fd4121df 100644 --- a/src/project-config.ts +++ b/src/project-config.ts @@ -314,14 +314,21 @@ function extractInclude(parsed: object, file: string): string[] { function loadParsedConfig(rootDir: string): ParsedConfig { const file = path.join(rootDir, PROJECT_CONFIG_FILENAME); - let mtimeMs: number; + // Search ranking calls this once per candidate path, and most projects + // have no config file: a throwing stat would build an ENOENT error for + // every one of them. + let stat: fs.Stats | undefined; try { - mtimeMs = fs.statSync(file).mtimeMs; + stat = fs.statSync(file, { throwIfNoEntry: false }); } catch { + stat = undefined; + } + if (!stat) { // No config file — drop any stale cache entry and return the default. cache.delete(rootDir); return EMPTY_CONFIG; } + const mtimeMs = stat.mtimeMs; const entry = cache.get(rootDir); if (entry && entry.mtimeMs === mtimeMs) return entry.config;