perf(search): trigram index for substring name search

CHANGELOG.md

@@ -12,6 +12,10 @@ and adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 ## [Unreleased]
 
+### New Features
+
+- Searching for part of a symbol name is much faster on large projects. Your index is upgraded automatically the first time it is opened.
+
 ### Fixes
 
 - In VB.NET, every member of a `Structure` is now indexed, including its fields, properties, methods, constructors and nested enums. Before, only the first member was, so the rest could not be found and their callers looked empty.

__tests__/fts5-fallback.test.ts

@@ -108,6 +108,7 @@ describe('FTS5 fallback (#1532)', () => {
     const nonFtsSchema = (connection: DatabaseConnection) => connection.getDb().prepare(`
       SELECT type, name, sql FROM sqlite_master
       WHERE name NOT LIKE 'nodes_fts%'
+        AND name NOT LIKE 'nodes_tri%'
         AND name NOT IN ('nodes_ai', 'nodes_ad', 'nodes_au')
       ORDER BY type, name
     `).all();
@@ -121,6 +122,30 @@ describe('FTS5 fallback (#1532)', () => {
     expect(nonFtsSchema(fallback)).toEqual(expected);
   });
 
+  it('opens a pre-trigram database without FTS5 and keeps substring search on LIKE', () => {
+    const connection = initialize();
+    const db = connection.getDb();
+    for (const sql of [
+      'DROP TRIGGER nodes_tri_ai', 'DROP TRIGGER nodes_tri_ad', 'DROP TRIGGER nodes_tri_au',
+      'DROP TABLE nodes_tri',
+      'DROP TRIGGER nodes_ai', 'DROP TRIGGER nodes_ad', 'DROP TRIGGER nodes_au',
+      'DROP TABLE nodes_fts',
+      'DELETE FROM schema_versions WHERE version = 12',
+      "INSERT INTO schema_versions (version, applied_at, description) VALUES (11, 0, 'test')",
+    ]) db.exec(sql);
+    new QueryBuilder(db).insertNodes([makeNode('getUserProfile'), makeNode('saveOrder')]);
+    connection.close();
+
+    const reopened = DatabaseConnection.open(path.join(dir, 'test.db'));
+    connections.push(reopened);
+
+    expect(reopened.fts5Available).toBe(false);
+    expect(reopened.getDb().prepare("SELECT name FROM sqlite_master WHERE name LIKE 'nodes_tri%'").all()).toEqual([]);
+    const queries = new QueryBuilder(reopened.getDb());
+    expect(queries.findNodesByNameSubstring('Profile').map(r => r.node.name)).toEqual(['getUserProfile']);
+    expect(queries.searchNodes('serPro').map(r => r.node.name)).toEqual(['getUserProfile']);
+  });
+
   it.each(['initialization', 'reopening'])('uses real FTS5 after %s', (state) => {
     const warn = vi.spyOn(console, 'warn').mockImplementation(() => {});
     let connection = initialize();

__tests__/trigram-substring.test.ts

@@ -0,0 +1,78 @@
+/**
+ * Substring name search goes through the nodes_tri trigram index but must
+ * return exactly what a plain LIKE scan returns, and the index must follow
+ * node inserts, updates and deletes.
+ */
+
+import { describe, it, expect, afterEach } from 'vitest';
+import * as fs from 'fs';
+import * as path from 'path';
+import * as os from 'os';
+import CodeGraph from '../src';
+import { DatabaseConnection, getDatabasePath } from '../src/db';
+
+const SOURCE = [
+  'export function clusterTopics(): void {}',
+  'export function ReconcileCluster(): void {}',
+  'export function a_b(): void {}',
+  'export function userPercent(): void {}',
+  'export class ClusterWorker { reconcile(): void {} }',
+].join('\n');
+
+// Expected names come from reading the source, not from the index: a token
+// matches when it occurs in the name, ignoring ASCII case, like SQL LIKE.
+const NAMES = ['clusterTopics', 'ReconcileCluster', 'a_b', 'userPercent', 'ClusterWorker', 'reconcile'];
+
+function expected(token: string): string[] {
+  return NAMES.filter(n => n.toLowerCase().includes(token.toLowerCase())).sort();
+}
+
+describe('nodes_tri substring search', () => {
+  let dir: string;
+  let cg: CodeGraph | undefined;
+
+  afterEach(() => {
+    cg?.close();
+    if (dir) fs.rmSync(dir, { recursive: true, force: true });
+  });
+
+  it('matches a LIKE scan for long, short and wildcard tokens, and tracks edits', async () => {
+    dir = fs.mkdtempSync(path.join(os.tmpdir(), 'cg-tri-'));
+    fs.writeFileSync(path.join(dir, 'a.ts'), SOURCE);
+    cg = await CodeGraph.init(dir, { index: true });
+
+    // `a_b` and `ab` take the LIKE path (wildcard / under 3 chars); the rest use nodes_tri.
+    for (const token of ['Cluster', 'CLUSTER', 'reconcile', 'Rec', 'ncile', 'a_b', 'ab']) {
+      const found = cg.getNodesByNameSubstring(token, { limit: 100 }).map(n => n.name).sort();
+      expect(found, token).toEqual(expected(token));
+    }
+    // No FTS token matches an infix, so this reaches the LIKE search (searchNodesLike).
+    expect(cg.searchNodes('econcileClus').map(r => r.node.name)).toEqual(['ReconcileCluster']);
+
+    fs.writeFileSync(path.join(dir, 'a.ts'), 'export function renamedThing(): void {}\n');
+    await cg.sync();
+    expect(cg.getNodesByNameSubstring('Cluster')).toEqual([]);
+    expect(cg.getNodesByNameSubstring('namedTh').map(n => n.name)).toEqual(['renamedThing']);
+  });
+  it('consults nodes_tri for long tokens only', async () => {
+    dir = fs.mkdtempSync(path.join(os.tmpdir(), 'cg-tri-'));
+    fs.writeFileSync(path.join(dir, 'a.ts'), SOURCE);
+    cg = await CodeGraph.init(dir, { index: true });
+
+    // Drop one node from the trigram index only: a query that uses the index misses it.
+    const conn = DatabaseConnection.open(getDatabasePath(dir));
+    try {
+      conn.getDb().exec(`INSERT INTO nodes_tri(nodes_tri, rowid, name, qualified_name)
+        SELECT 'delete', rowid, name, qualified_name FROM nodes WHERE name = 'clusterTopics'`);
+    } finally {
+      conn.close();
+    }
+
+    const names = (token: string) => cg!.getNodesByNameSubstring(token, { limit: 100 }).map(n => n.name);
+    expect(names('Cluster')).not.toContain('clusterTopics');
+    expect(names('Cluster')).toContain('ClusterWorker');
+    // Under 3 chars or with a wildcard: plain LIKE, still sees it.
+    expect(names('te')).toContain('clusterTopics');
+    expect(names('cluster_opics')).toContain('clusterTopics');
+  });
+});

src/db/index.ts

@@ -204,36 +204,35 @@ export class DatabaseConnection {
   }
 
   /**
-   * FTS maintenance triggers dropped/recreated around a bulk load.
-   * Names must match schema.sql.
-   */
-  private static readonly FTS_TRIGGER_NAMES = ['nodes_ai', 'nodes_ad', 'nodes_au'] as const;
-
-  /**
-   * Enter bulk-load mode: drop the per-row FTS sync triggers so mass node
-   * inserts skip per-row tokenization. MUST be paired with endBulkNodeLoad()
-   * (use try/finally); a crash inside the window is healed on the next open().
-   * The window is DB-wide (triggers are schema objects), which is safe because
-   * endBulkNodeLoad() rebuilds nodes_fts from the nodes table wholesale — any
-   * row written by anyone during the window is captured by the rebuild.
+   * Enter bulk-load mode: drop the per-row FTS and trigram sync triggers so
+   * mass node inserts skip per-row tokenization. MUST be paired with
+   * endBulkNodeLoad() (use try/finally); a crash inside the window is healed on
+   * the next open(). The window is DB-wide (triggers are schema objects), which
+   * is safe because endBulkNodeLoad() rebuilds both indexes from the nodes
+   * table wholesale — any row written by anyone during the window is captured
+   * by the rebuild.
    */
   beginBulkNodeLoad(): void {
     if (!this.fts5Available) return;
-    for (const t of DatabaseConnection.FTS_TRIGGER_NAMES) {
-      this.db.exec(`DROP TRIGGER IF EXISTS ${t}`);
+    for (const { name } of DatabaseConnection.schemaTriggers()) {
+      this.db.exec(`DROP TRIGGER IF EXISTS ${name}`);
     }
   }
 
   /**
-   * Leave bulk-load mode: rebuild the whole FTS index from the nodes table in
-   * one pass (far cheaper than per-row trigger firings), then recreate the
-   * triggers by re-running schema.sql (idempotent — everything in it is
-   * IF NOT EXISTS).
+   * Leave bulk-load mode: rebuild the FTS and trigram indexes from the nodes
+   * table in one pass (far cheaper than per-row trigger firings), then
+   * recreate the triggers from schema.sql.
    */
   endBulkNodeLoad(): void {
     if (!this.fts5Available) return;
     this.db.exec(`INSERT INTO nodes_fts(nodes_fts) VALUES('rebuild')`);
-    this.recreateFtsTriggers();
+    // A database migrated on a Node build without FTS5 never got nodes_tri.
+    this.db.exec(DatabaseConnection.schemaDdl(/CREATE VIRTUAL TABLE IF NOT EXISTS nodes_tri[\s\S]*?\);/));
+    this.db.exec(`INSERT INTO nodes_tri(nodes_tri) VALUES('rebuild')`);
+    for (const { ddl } of DatabaseConnection.schemaTriggers()) {
+      this.db.exec(ddl);
+    }
   }
 
   /**
@@ -425,12 +424,12 @@ export class DatabaseConnection {
   /** Recreate the FTS triggers + rebuild if a bulk-load window never closed. */
   private healBulkNodeLoad(): void {
     if (!this.fts5Available) return;
+    const expected = DatabaseConnection.schemaTriggers().map(t => t.name);
+    const placeholders = expected.map(() => '?').join(',');
     const row = this.db
-      .prepare(
-        `SELECT count(*) AS c FROM sqlite_master WHERE type = 'trigger' AND name IN ('nodes_ai','nodes_ad','nodes_au')`
-      )
-      .get() as { c: number } | undefined;
-    if ((row?.c ?? 0) >= DatabaseConnection.FTS_TRIGGER_NAMES.length) return;
+      .prepare(`SELECT count(*) AS c FROM sqlite_master WHERE type = 'trigger' AND name IN (${placeholders})`)
+      .get(...expected) as { c: number } | undefined;
+    if ((row?.c ?? 0) >= expected.length) return;
     this.endBulkNodeLoad();
   }
 
@@ -456,26 +455,21 @@ export class DatabaseConnection {
     }
   }
 
-  /**
-   * Recreate the FTS sync triggers from schema.sql — extracted from the file
-   * rather than duplicated here so the DDL cannot drift from the schema.
-   * (Re-execing the whole schema is not an option: it contains data INSERTs
-   * that are not idempotent, e.g. schema_versions.)
-   */
-  private recreateFtsTriggers(): void {
-    const schemaPath = path.join(__dirname, 'schema.sql');
-    const schema = fs.readFileSync(schemaPath, 'utf-8');
-    const triggerDdls = schema.match(
-      /CREATE TRIGGER IF NOT EXISTS nodes_a[idu]\b[\s\S]*?END;/g
-    );
-    if (!triggerDdls || triggerDdls.length !== DatabaseConnection.FTS_TRIGGER_NAMES.length) {
-      throw new Error(
-        `schema.sql: expected ${DatabaseConnection.FTS_TRIGGER_NAMES.length} nodes FTS triggers, found ${triggerDdls?.length ?? 0}`
-      );
-    }
-    for (const ddl of triggerDdls) {
-      this.db.exec(ddl);
-    }
+  /** First schema.sql statement matching `pattern`; the DDL cannot drift from the schema. */
+  private static schemaDdl(pattern: RegExp): string {
+    const schema = fs.readFileSync(path.join(__dirname, 'schema.sql'), 'utf-8');
+    const ddl = schema.match(pattern)?.[0];
+    if (!ddl) throw new Error(`schema.sql: ${pattern} not found`);
+    return ddl;
+  }
+
+  /** Every trigger in schema.sql (all keep nodes_fts / nodes_tri in sync). */
+  private static schemaTriggers(): Array<{ name: string; ddl: string }> {
+    const schema = fs.readFileSync(path.join(__dirname, 'schema.sql'), 'utf-8');
+    return [...schema.matchAll(/CREATE TRIGGER IF NOT EXISTS (\w+)\b[\s\S]*?END;/g)].map(m => ({
+      name: m[1]!,
+      ddl: m[0],
+    }));
   }
 
   /**

src/db/migrations.ts

@@ -9,7 +9,7 @@ import { SqliteDatabase } from './sqlite-adapter';
 /**
  * Current schema version
  */
-export const CURRENT_SCHEMA_VERSION = 11;
+export const CURRENT_SCHEMA_VERSION = 12;
 
 /**
  * Migration definition
@@ -216,6 +216,40 @@ const migrations: Migration[] = [
       `);
     },
   },
+  {
+    version: 12,
+    description: 'Add nodes_tri trigram index for substring name search',
+    up: (db) => {
+      // Same optional-FTS5 contract as open(): a Node build without FTS5
+      // cannot read nodes_fts, so it gets no trigram index either (#1532).
+      // The next bulk load on an FTS5 build creates it.
+      try {
+        db.exec('SELECT * FROM nodes_fts LIMIT 0');
+      } catch {
+        return;
+      }
+      db.exec(`
+        CREATE VIRTUAL TABLE IF NOT EXISTS nodes_tri USING fts5(
+          name, qualified_name, content='nodes', content_rowid='rowid', tokenize='trigram'
+        );
+        CREATE TRIGGER IF NOT EXISTS nodes_tri_ai AFTER INSERT ON nodes BEGIN
+          INSERT INTO nodes_tri(rowid, name, qualified_name)
+          VALUES (NEW.rowid, NEW.name, NEW.qualified_name);
+        END;
+        CREATE TRIGGER IF NOT EXISTS nodes_tri_ad AFTER DELETE ON nodes BEGIN
+          INSERT INTO nodes_tri(nodes_tri, rowid, name, qualified_name)
+          VALUES ('delete', OLD.rowid, OLD.name, OLD.qualified_name);
+        END;
+        CREATE TRIGGER IF NOT EXISTS nodes_tri_au AFTER UPDATE ON nodes BEGIN
+          INSERT INTO nodes_tri(nodes_tri, rowid, name, qualified_name)
+          VALUES ('delete', OLD.rowid, OLD.name, OLD.qualified_name);
+          INSERT INTO nodes_tri(rowid, name, qualified_name)
+          VALUES (NEW.rowid, NEW.name, NEW.qualified_name);
+        END;
+        INSERT INTO nodes_tri(nodes_tri) VALUES('rebuild');
+      `);
+    },
+  },
 ];
 
 /**

src/db/queries.ts

@@ -267,6 +267,9 @@ export class QueryBuilder {
     | { stamp: string; value: { filePath: string; edgeCount: number; nextEdgeCount: number } | null }
     | undefined;
 
+  // nodes_tri (trigram index) presence; databases migrated without FTS5 lack it.
+  private triAvailable: boolean | undefined;
+
   // Prepared statements (lazily initialized)
   private stmts: {
     insertNode?: SqliteStatement;
@@ -402,6 +405,7 @@ export class QueryBuilder {
     // different databases report the same one — the memo goes with the old
     // connection, or a worker following a rebuilt index keeps its answer (#1864).
     this.dominantFileMemo = undefined;
+    this.triAvailable = undefined;
   }
 
   private edgeKindStmt(sql: string): SqliteStatement {
@@ -1867,6 +1871,7 @@ export class QueryBuilder {
         name LIKE ?
       )
     `;
+    const tri = this.triCandidates(query, true);
 
     // Pattern variants for better matching
     const exactMatch = query;
@@ -1883,6 +1888,11 @@ export class QueryBuilder {
       startsWith,     // WHERE: name starts with
     ];
 
+    if (tri) {
+      sql += ` AND ${tri.clause}`;
+      params.push(...tri.params);
+    }
+
     if (kinds && kinds.length > 0) {
       sql += ` AND kind IN (${kinds.map(() => '?').join(',')})`;
       params.push(...kinds);
@@ -2002,6 +2012,30 @@ export class QueryBuilder {
     return allResults.slice(0, limit);
   }
 
+  /**
+   * Trigram prefilter for a `%token%` LIKE: a rowid clause that narrows the
+   * scan to nodes_tri candidates. The caller keeps its own LIKE, so a coarser
+   * trigram fold can only add candidates, never results. Null (keep the plain
+   * scan) when the token is under 3 chars, has a LIKE wildcard, or the
+   * database has no nodes_tri.
+   */
+  private triCandidates(token: string, withQualifiedName: boolean): { clause: string; params: string[] } | null {
+    if ([...token].length < 3 || token.includes('_') || token.includes('%')) return null;
+    if (this.triAvailable === undefined) {
+      this.triAvailable = !!this.db.prepare("SELECT 1 FROM sqlite_master WHERE name = 'nodes_tri'").get();
+    }
+    if (!this.triAvailable) return null;
+    const pattern = `%${token}%`;
+    if (!withQualifiedName) {
+      return { clause: 'nodes.rowid IN (SELECT rowid FROM nodes_tri WHERE name LIKE ?)', params: [pattern] };
+    }
+    return {
+      clause: `nodes.rowid IN (SELECT rowid FROM nodes_tri WHERE name LIKE ?
+        UNION SELECT rowid FROM nodes_tri WHERE qualified_name LIKE ?)`,
+      params: [pattern, pattern],
+    };
+  }
+
   /**
    * Find nodes whose name contains a substring (LIKE-based).
    * Useful for CamelCase-part matching where FTS fails because
@@ -2022,6 +2056,12 @@ export class QueryBuilder {
     `;
     const params: (string | number)[] = [`%${substring}%`];
 
+    const tri = this.triCandidates(substring, false);
+    if (tri) {
+      sql += ` AND ${tri.clause}`;
+      params.push(...tri.params);
+    }
+
     // Exclude prefix matches (handled by FTS-based prefix search in Step 2b)
     if (excludePrefix) {
       sql += ` AND name NOT LIKE ?`;