diff --git a/CHANGELOG.md b/CHANGELOG.md index 68d7cc062b..f0d9d3ae1a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,10 @@ and adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +### New Features + +- Searching for part of a symbol name is much faster on large projects. Your index is upgraded automatically the first time it is opened. + ### Fixes - In VB.NET, every member of a `Structure` is now indexed, including its fields, properties, methods, constructors and nested enums. Before, only the first member was, so the rest could not be found and their callers looked empty. diff --git a/__tests__/fts5-fallback.test.ts b/__tests__/fts5-fallback.test.ts index 19d6022b53..47f6f43bcc 100644 --- a/__tests__/fts5-fallback.test.ts +++ b/__tests__/fts5-fallback.test.ts @@ -108,6 +108,7 @@ describe('FTS5 fallback (#1532)', () => { const nonFtsSchema = (connection: DatabaseConnection) => connection.getDb().prepare(` SELECT type, name, sql FROM sqlite_master WHERE name NOT LIKE 'nodes_fts%' + AND name NOT LIKE 'nodes_tri%' AND name NOT IN ('nodes_ai', 'nodes_ad', 'nodes_au') ORDER BY type, name `).all(); @@ -121,6 +122,30 @@ describe('FTS5 fallback (#1532)', () => { expect(nonFtsSchema(fallback)).toEqual(expected); }); + it('opens a pre-trigram database without FTS5 and keeps substring search on LIKE', () => { + const connection = initialize(); + const db = connection.getDb(); + for (const sql of [ + 'DROP TRIGGER nodes_tri_ai', 'DROP TRIGGER nodes_tri_ad', 'DROP TRIGGER nodes_tri_au', + 'DROP TABLE nodes_tri', + 'DROP TRIGGER nodes_ai', 'DROP TRIGGER nodes_ad', 'DROP TRIGGER nodes_au', + 'DROP TABLE nodes_fts', + 'DELETE FROM schema_versions WHERE version = 12', + "INSERT INTO schema_versions (version, applied_at, description) VALUES (11, 0, 'test')", + ]) db.exec(sql); + new QueryBuilder(db).insertNodes([makeNode('getUserProfile'), makeNode('saveOrder')]); + connection.close(); + + const reopened = DatabaseConnection.open(path.join(dir, 'test.db')); + connections.push(reopened); + + expect(reopened.fts5Available).toBe(false); + expect(reopened.getDb().prepare("SELECT name FROM sqlite_master WHERE name LIKE 'nodes_tri%'").all()).toEqual([]); + const queries = new QueryBuilder(reopened.getDb()); + expect(queries.findNodesByNameSubstring('Profile').map(r => r.node.name)).toEqual(['getUserProfile']); + expect(queries.searchNodes('serPro').map(r => r.node.name)).toEqual(['getUserProfile']); + }); + it.each(['initialization', 'reopening'])('uses real FTS5 after %s', (state) => { const warn = vi.spyOn(console, 'warn').mockImplementation(() => {}); let connection = initialize(); diff --git a/__tests__/trigram-substring.test.ts b/__tests__/trigram-substring.test.ts new file mode 100644 index 0000000000..d00d1a6a2f --- /dev/null +++ b/__tests__/trigram-substring.test.ts @@ -0,0 +1,78 @@ +/** + * Substring name search goes through the nodes_tri trigram index but must + * return exactly what a plain LIKE scan returns, and the index must follow + * node inserts, updates and deletes. + */ + +import { describe, it, expect, afterEach } from 'vitest'; +import * as fs from 'fs'; +import * as path from 'path'; +import * as os from 'os'; +import CodeGraph from '../src'; +import { DatabaseConnection, getDatabasePath } from '../src/db'; + +const SOURCE = [ + 'export function clusterTopics(): void {}', + 'export function ReconcileCluster(): void {}', + 'export function a_b(): void {}', + 'export function userPercent(): void {}', + 'export class ClusterWorker { reconcile(): void {} }', +].join('\n'); + +// Expected names come from reading the source, not from the index: a token +// matches when it occurs in the name, ignoring ASCII case, like SQL LIKE. +const NAMES = ['clusterTopics', 'ReconcileCluster', 'a_b', 'userPercent', 'ClusterWorker', 'reconcile']; + +function expected(token: string): string[] { + return NAMES.filter(n => n.toLowerCase().includes(token.toLowerCase())).sort(); +} + +describe('nodes_tri substring search', () => { + let dir: string; + let cg: CodeGraph | undefined; + + afterEach(() => { + cg?.close(); + if (dir) fs.rmSync(dir, { recursive: true, force: true }); + }); + + it('matches a LIKE scan for long, short and wildcard tokens, and tracks edits', async () => { + dir = fs.mkdtempSync(path.join(os.tmpdir(), 'cg-tri-')); + fs.writeFileSync(path.join(dir, 'a.ts'), SOURCE); + cg = await CodeGraph.init(dir, { index: true }); + + // `a_b` and `ab` take the LIKE path (wildcard / under 3 chars); the rest use nodes_tri. + for (const token of ['Cluster', 'CLUSTER', 'reconcile', 'Rec', 'ncile', 'a_b', 'ab']) { + const found = cg.getNodesByNameSubstring(token, { limit: 100 }).map(n => n.name).sort(); + expect(found, token).toEqual(expected(token)); + } + // No FTS token matches an infix, so this reaches the LIKE search (searchNodesLike). + expect(cg.searchNodes('econcileClus').map(r => r.node.name)).toEqual(['ReconcileCluster']); + + fs.writeFileSync(path.join(dir, 'a.ts'), 'export function renamedThing(): void {}\n'); + await cg.sync(); + expect(cg.getNodesByNameSubstring('Cluster')).toEqual([]); + expect(cg.getNodesByNameSubstring('namedTh').map(n => n.name)).toEqual(['renamedThing']); + }); + it('consults nodes_tri for long tokens only', async () => { + dir = fs.mkdtempSync(path.join(os.tmpdir(), 'cg-tri-')); + fs.writeFileSync(path.join(dir, 'a.ts'), SOURCE); + cg = await CodeGraph.init(dir, { index: true }); + + // Drop one node from the trigram index only: a query that uses the index misses it. + const conn = DatabaseConnection.open(getDatabasePath(dir)); + try { + conn.getDb().exec(`INSERT INTO nodes_tri(nodes_tri, rowid, name, qualified_name) + SELECT 'delete', rowid, name, qualified_name FROM nodes WHERE name = 'clusterTopics'`); + } finally { + conn.close(); + } + + const names = (token: string) => cg!.getNodesByNameSubstring(token, { limit: 100 }).map(n => n.name); + expect(names('Cluster')).not.toContain('clusterTopics'); + expect(names('Cluster')).toContain('ClusterWorker'); + // Under 3 chars or with a wildcard: plain LIKE, still sees it. + expect(names('te')).toContain('clusterTopics'); + expect(names('cluster_opics')).toContain('clusterTopics'); + }); +}); diff --git a/src/db/index.ts b/src/db/index.ts index cfe7c10a72..a53322e09f 100644 --- a/src/db/index.ts +++ b/src/db/index.ts @@ -204,36 +204,35 @@ export class DatabaseConnection { } /** - * FTS maintenance triggers dropped/recreated around a bulk load. - * Names must match schema.sql. - */ - private static readonly FTS_TRIGGER_NAMES = ['nodes_ai', 'nodes_ad', 'nodes_au'] as const; - - /** - * Enter bulk-load mode: drop the per-row FTS sync triggers so mass node - * inserts skip per-row tokenization. MUST be paired with endBulkNodeLoad() - * (use try/finally); a crash inside the window is healed on the next open(). - * The window is DB-wide (triggers are schema objects), which is safe because - * endBulkNodeLoad() rebuilds nodes_fts from the nodes table wholesale — any - * row written by anyone during the window is captured by the rebuild. + * Enter bulk-load mode: drop the per-row FTS and trigram sync triggers so + * mass node inserts skip per-row tokenization. MUST be paired with + * endBulkNodeLoad() (use try/finally); a crash inside the window is healed on + * the next open(). The window is DB-wide (triggers are schema objects), which + * is safe because endBulkNodeLoad() rebuilds both indexes from the nodes + * table wholesale — any row written by anyone during the window is captured + * by the rebuild. */ beginBulkNodeLoad(): void { if (!this.fts5Available) return; - for (const t of DatabaseConnection.FTS_TRIGGER_NAMES) { - this.db.exec(`DROP TRIGGER IF EXISTS ${t}`); + for (const { name } of DatabaseConnection.schemaTriggers()) { + this.db.exec(`DROP TRIGGER IF EXISTS ${name}`); } } /** - * Leave bulk-load mode: rebuild the whole FTS index from the nodes table in - * one pass (far cheaper than per-row trigger firings), then recreate the - * triggers by re-running schema.sql (idempotent — everything in it is - * IF NOT EXISTS). + * Leave bulk-load mode: rebuild the FTS and trigram indexes from the nodes + * table in one pass (far cheaper than per-row trigger firings), then + * recreate the triggers from schema.sql. */ endBulkNodeLoad(): void { if (!this.fts5Available) return; this.db.exec(`INSERT INTO nodes_fts(nodes_fts) VALUES('rebuild')`); - this.recreateFtsTriggers(); + // A database migrated on a Node build without FTS5 never got nodes_tri. + this.db.exec(DatabaseConnection.schemaDdl(/CREATE VIRTUAL TABLE IF NOT EXISTS nodes_tri[\s\S]*?\);/)); + this.db.exec(`INSERT INTO nodes_tri(nodes_tri) VALUES('rebuild')`); + for (const { ddl } of DatabaseConnection.schemaTriggers()) { + this.db.exec(ddl); + } } /** @@ -425,12 +424,12 @@ export class DatabaseConnection { /** Recreate the FTS triggers + rebuild if a bulk-load window never closed. */ private healBulkNodeLoad(): void { if (!this.fts5Available) return; + const expected = DatabaseConnection.schemaTriggers().map(t => t.name); + const placeholders = expected.map(() => '?').join(','); const row = this.db - .prepare( - `SELECT count(*) AS c FROM sqlite_master WHERE type = 'trigger' AND name IN ('nodes_ai','nodes_ad','nodes_au')` - ) - .get() as { c: number } | undefined; - if ((row?.c ?? 0) >= DatabaseConnection.FTS_TRIGGER_NAMES.length) return; + .prepare(`SELECT count(*) AS c FROM sqlite_master WHERE type = 'trigger' AND name IN (${placeholders})`) + .get(...expected) as { c: number } | undefined; + if ((row?.c ?? 0) >= expected.length) return; this.endBulkNodeLoad(); } @@ -456,26 +455,21 @@ export class DatabaseConnection { } } - /** - * Recreate the FTS sync triggers from schema.sql — extracted from the file - * rather than duplicated here so the DDL cannot drift from the schema. - * (Re-execing the whole schema is not an option: it contains data INSERTs - * that are not idempotent, e.g. schema_versions.) - */ - private recreateFtsTriggers(): void { - const schemaPath = path.join(__dirname, 'schema.sql'); - const schema = fs.readFileSync(schemaPath, 'utf-8'); - const triggerDdls = schema.match( - /CREATE TRIGGER IF NOT EXISTS nodes_a[idu]\b[\s\S]*?END;/g - ); - if (!triggerDdls || triggerDdls.length !== DatabaseConnection.FTS_TRIGGER_NAMES.length) { - throw new Error( - `schema.sql: expected ${DatabaseConnection.FTS_TRIGGER_NAMES.length} nodes FTS triggers, found ${triggerDdls?.length ?? 0}` - ); - } - for (const ddl of triggerDdls) { - this.db.exec(ddl); - } + /** First schema.sql statement matching `pattern`; the DDL cannot drift from the schema. */ + private static schemaDdl(pattern: RegExp): string { + const schema = fs.readFileSync(path.join(__dirname, 'schema.sql'), 'utf-8'); + const ddl = schema.match(pattern)?.[0]; + if (!ddl) throw new Error(`schema.sql: ${pattern} not found`); + return ddl; + } + + /** Every trigger in schema.sql (all keep nodes_fts / nodes_tri in sync). */ + private static schemaTriggers(): Array<{ name: string; ddl: string }> { + const schema = fs.readFileSync(path.join(__dirname, 'schema.sql'), 'utf-8'); + return [...schema.matchAll(/CREATE TRIGGER IF NOT EXISTS (\w+)\b[\s\S]*?END;/g)].map(m => ({ + name: m[1]!, + ddl: m[0], + })); } /** diff --git a/src/db/migrations.ts b/src/db/migrations.ts index bb7b84bf53..62357ef368 100644 --- a/src/db/migrations.ts +++ b/src/db/migrations.ts @@ -9,7 +9,7 @@ import { SqliteDatabase } from './sqlite-adapter'; /** * Current schema version */ -export const CURRENT_SCHEMA_VERSION = 11; +export const CURRENT_SCHEMA_VERSION = 12; /** * Migration definition @@ -216,6 +216,40 @@ const migrations: Migration[] = [ `); }, }, + { + version: 12, + description: 'Add nodes_tri trigram index for substring name search', + up: (db) => { + // Same optional-FTS5 contract as open(): a Node build without FTS5 + // cannot read nodes_fts, so it gets no trigram index either (#1532). + // The next bulk load on an FTS5 build creates it. + try { + db.exec('SELECT * FROM nodes_fts LIMIT 0'); + } catch { + return; + } + db.exec(` + CREATE VIRTUAL TABLE IF NOT EXISTS nodes_tri USING fts5( + name, qualified_name, content='nodes', content_rowid='rowid', tokenize='trigram' + ); + CREATE TRIGGER IF NOT EXISTS nodes_tri_ai AFTER INSERT ON nodes BEGIN + INSERT INTO nodes_tri(rowid, name, qualified_name) + VALUES (NEW.rowid, NEW.name, NEW.qualified_name); + END; + CREATE TRIGGER IF NOT EXISTS nodes_tri_ad AFTER DELETE ON nodes BEGIN + INSERT INTO nodes_tri(nodes_tri, rowid, name, qualified_name) + VALUES ('delete', OLD.rowid, OLD.name, OLD.qualified_name); + END; + CREATE TRIGGER IF NOT EXISTS nodes_tri_au AFTER UPDATE ON nodes BEGIN + INSERT INTO nodes_tri(nodes_tri, rowid, name, qualified_name) + VALUES ('delete', OLD.rowid, OLD.name, OLD.qualified_name); + INSERT INTO nodes_tri(rowid, name, qualified_name) + VALUES (NEW.rowid, NEW.name, NEW.qualified_name); + END; + INSERT INTO nodes_tri(nodes_tri) VALUES('rebuild'); + `); + }, + }, ]; /** diff --git a/src/db/queries.ts b/src/db/queries.ts index 704cfd5bcd..54c83ed091 100644 --- a/src/db/queries.ts +++ b/src/db/queries.ts @@ -267,6 +267,9 @@ export class QueryBuilder { | { stamp: string; value: { filePath: string; edgeCount: number; nextEdgeCount: number } | null } | undefined; + // nodes_tri (trigram index) presence; databases migrated without FTS5 lack it. + private triAvailable: boolean | undefined; + // Prepared statements (lazily initialized) private stmts: { insertNode?: SqliteStatement; @@ -402,6 +405,7 @@ export class QueryBuilder { // different databases report the same one — the memo goes with the old // connection, or a worker following a rebuilt index keeps its answer (#1864). this.dominantFileMemo = undefined; + this.triAvailable = undefined; } private edgeKindStmt(sql: string): SqliteStatement { @@ -1867,6 +1871,7 @@ export class QueryBuilder { name LIKE ? ) `; + const tri = this.triCandidates(query, true); // Pattern variants for better matching const exactMatch = query; @@ -1883,6 +1888,11 @@ export class QueryBuilder { startsWith, // WHERE: name starts with ]; + if (tri) { + sql += ` AND ${tri.clause}`; + params.push(...tri.params); + } + if (kinds && kinds.length > 0) { sql += ` AND kind IN (${kinds.map(() => '?').join(',')})`; params.push(...kinds); @@ -2002,6 +2012,30 @@ export class QueryBuilder { return allResults.slice(0, limit); } + /** + * Trigram prefilter for a `%token%` LIKE: a rowid clause that narrows the + * scan to nodes_tri candidates. The caller keeps its own LIKE, so a coarser + * trigram fold can only add candidates, never results. Null (keep the plain + * scan) when the token is under 3 chars, has a LIKE wildcard, or the + * database has no nodes_tri. + */ + private triCandidates(token: string, withQualifiedName: boolean): { clause: string; params: string[] } | null { + if ([...token].length < 3 || token.includes('_') || token.includes('%')) return null; + if (this.triAvailable === undefined) { + this.triAvailable = !!this.db.prepare("SELECT 1 FROM sqlite_master WHERE name = 'nodes_tri'").get(); + } + if (!this.triAvailable) return null; + const pattern = `%${token}%`; + if (!withQualifiedName) { + return { clause: 'nodes.rowid IN (SELECT rowid FROM nodes_tri WHERE name LIKE ?)', params: [pattern] }; + } + return { + clause: `nodes.rowid IN (SELECT rowid FROM nodes_tri WHERE name LIKE ? + UNION SELECT rowid FROM nodes_tri WHERE qualified_name LIKE ?)`, + params: [pattern, pattern], + }; + } + /** * Find nodes whose name contains a substring (LIKE-based). * Useful for CamelCase-part matching where FTS fails because @@ -2022,6 +2056,12 @@ export class QueryBuilder { `; const params: (string | number)[] = [`%${substring}%`]; + const tri = this.triCandidates(substring, false); + if (tri) { + sql += ` AND ${tri.clause}`; + params.push(...tri.params); + } + // Exclude prefix matches (handled by FTS-based prefix search in Step 2b) if (excludePrefix) { sql += ` AND name NOT LIKE ?`; diff --git a/src/db/schema.sql b/src/db/schema.sql index 91ae4805ba..fb0b609874 100644 --- a/src/db/schema.sql +++ b/src/db/schema.sql @@ -124,6 +124,34 @@ CREATE VIRTUAL TABLE IF NOT EXISTS nodes_fts USING fts5( content_rowid='rowid' ); +-- Trigram index for substring search: `name LIKE '%x%'` is a full table scan +-- on a large index; with this table it is an index lookup (see +-- QueryBuilder.triCandidates). Kept in sync by the same write path as nodes_fts. +CREATE VIRTUAL TABLE IF NOT EXISTS nodes_tri USING fts5( + name, + qualified_name, + content='nodes', + content_rowid='rowid', + tokenize='trigram' +); + +CREATE TRIGGER IF NOT EXISTS nodes_tri_ai AFTER INSERT ON nodes BEGIN + INSERT INTO nodes_tri(rowid, name, qualified_name) + VALUES (NEW.rowid, NEW.name, NEW.qualified_name); +END; + +CREATE TRIGGER IF NOT EXISTS nodes_tri_ad AFTER DELETE ON nodes BEGIN + INSERT INTO nodes_tri(nodes_tri, rowid, name, qualified_name) + VALUES ('delete', OLD.rowid, OLD.name, OLD.qualified_name); +END; + +CREATE TRIGGER IF NOT EXISTS nodes_tri_au AFTER UPDATE ON nodes BEGIN + INSERT INTO nodes_tri(nodes_tri, rowid, name, qualified_name) + VALUES ('delete', OLD.rowid, OLD.name, OLD.qualified_name); + INSERT INTO nodes_tri(rowid, name, qualified_name) + VALUES (NEW.rowid, NEW.name, NEW.qualified_name); +END; + -- Triggers to keep FTS index in sync CREATE TRIGGER IF NOT EXISTS nodes_ai AFTER INSERT ON nodes BEGIN INSERT INTO nodes_fts(rowid, id, name, qualified_name, docstring, signature)