feat: FTS5 search lane — BM25 full-text search

Add SQLite FTS5 virtual table as second search lane. Populated
during indexing alongside vector embeddings. Supports exact keyword
matches for ticket IDs, names, and dates that semantic search misses.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

Author: devwhodevs

src/fts.rs

@@ -0,0 +1,107 @@
+// FTS5 search support.
+//
+// The `FtsResult` struct and `fts_search` method live on `Store` (in store.rs)
+// since the store owns the database connection. We re-export `FtsResult` here
+// so downstream code can import it from either location.
+
+pub use crate::store::FtsResult;
+
+#[cfg(test)]
+mod tests {
+    use crate::store::Store;
+    use crate::docid::generate_docid;
+
+    fn setup_store() -> Store {
+        let store = Store::open_memory().unwrap();
+        store.ensure_fts_table().unwrap();
+        store
+    }
+
+    #[test]
+    fn test_fts_exact_match() {
+        let store = setup_store();
+        let file_id = store
+            .insert_file("notes/ticket.md", "hash1", 100, &[], &generate_docid("notes/ticket.md"))
+            .unwrap();
+
+        store
+            .insert_fts_chunk(file_id, 0, "BRE-2579 delivery date extension for checkout")
+            .unwrap();
+
+        let results = store.fts_search("BRE-2579", 10).unwrap();
+        assert_eq!(results.len(), 1);
+        assert_eq!(results[0].file_id, file_id);
+        assert_eq!(results[0].chunk_seq, 0);
+        assert!(results[0].score > 0.0, "score should be positive (negated BM25)");
+    }
+
+    #[test]
+    fn test_fts_no_match() {
+        let store = setup_store();
+        let file_id = store
+            .insert_file("notes/note.md", "hash1", 100, &[], &generate_docid("notes/note.md"))
+            .unwrap();
+
+        store
+            .insert_fts_chunk(file_id, 0, "Rust programming language guide")
+            .unwrap();
+
+        let results = store.fts_search("kubernetes", 10).unwrap();
+        assert_eq!(results.len(), 0);
+    }
+
+    #[test]
+    fn test_fts_multiple_results() {
+        let store = setup_store();
+
+        let file_id1 = store
+            .insert_file("notes/a.md", "h1", 100, &[], &generate_docid("notes/a.md"))
+            .unwrap();
+        let file_id2 = store
+            .insert_file("notes/b.md", "h2", 100, &[], &generate_docid("notes/b.md"))
+            .unwrap();
+        let file_id3 = store
+            .insert_file("notes/c.md", "h3", 100, &[], &generate_docid("notes/c.md"))
+            .unwrap();
+
+        // Chunk with "delivery" appearing multiple times should rank higher.
+        store
+            .insert_fts_chunk(file_id1, 0, "delivery date delivery schedule delivery tracking")
+            .unwrap();
+        store
+            .insert_fts_chunk(file_id2, 0, "delivery date for the checkout page")
+            .unwrap();
+        store
+            .insert_fts_chunk(file_id3, 0, "unrelated content about Rust and WebAssembly")
+            .unwrap();
+
+        let results = store.fts_search("delivery", 10).unwrap();
+        assert_eq!(results.len(), 2, "only 2 chunks mention 'delivery'");
+
+        // Results should be sorted by score descending.
+        assert!(
+            results[0].score >= results[1].score,
+            "results should be ranked by relevance"
+        );
+    }
+
+    #[test]
+    fn test_fts_delete_chunks_for_file() {
+        let store = setup_store();
+        let file_id = store
+            .insert_file("notes/del.md", "hash1", 100, &[], &generate_docid("notes/del.md"))
+            .unwrap();
+
+        store.insert_fts_chunk(file_id, 0, "first chunk content").unwrap();
+        store.insert_fts_chunk(file_id, 1, "second chunk content").unwrap();
+
+        // Verify they exist.
+        let results = store.fts_search("chunk", 10).unwrap();
+        assert_eq!(results.len(), 2);
+
+        // Delete and verify gone.
+        store.delete_fts_chunks_for_file(file_id).unwrap();
+        let results = store.fts_search("chunk", 10).unwrap();
+        assert_eq!(results.len(), 0);
+    }
+}

src/indexer.rs

@@ -167,6 +167,7 @@ pub fn run_index(vault_path: &Path, config: &Config, rebuild: bool) -> Result<In
         if !vector_ids.is_empty() {
             store.add_tombstones(&vector_ids)?;
         }
+        store.delete_fts_chunks_for_file(record.id)?;
         store.delete_file(record.id)?;
     }
 
@@ -180,6 +181,7 @@ pub fn run_index(vault_path: &Path, config: &Config, rebuild: bool) -> Result<In
             if !vector_ids.is_empty() {
                 store.add_tombstones(&vector_ids)?;
             }
+            store.delete_fts_chunks_for_file(record.id)?;
             store.delete_file(record.id)?;
         }
         files_to_index.push(file_path.clone());
@@ -295,7 +297,7 @@ pub fn run_index(vault_path: &Path, config: &Config, rebuild: bool) -> Result<In
         let file_id =
             store.insert_file(&result.rel_path, &result.hash, result.mtime, &result.tags, &docid)?;
 
-        for (heading, snippet, vector, token_count) in &result.chunks {
+        for (chunk_seq, (heading, snippet, vector, token_count)) in result.chunks.iter().enumerate() {
             let vector_id = next_vector_id;
             next_vector_id += 1;
             store.insert_chunk_with_vector(
@@ -306,6 +308,7 @@ pub fn run_index(vault_path: &Path, config: &Config, rebuild: bool) -> Result<In
                 *token_count as i64,
                 vector,
             )?;
+            store.insert_fts_chunk(file_id, chunk_seq as i64, snippet)?;
         }
     }
 

src/lib.rs

@@ -2,6 +2,7 @@ pub mod chunker;
 pub mod config;
 pub mod docid;
 pub mod embedder;
+pub mod fts;
 pub mod hnsw;
 pub mod indexer;
 pub mod search;

src/store.rs

@@ -26,6 +26,15 @@ pub struct ChunkRecord {
     pub token_count: i64,
 }
 
+/// A single result from an FTS5 full-text search.
+#[derive(Debug, Clone)]
+pub struct FtsResult {
+    pub file_id: i64,
+    pub chunk_seq: i64,
+    pub score: f64,
+    pub snippet: String,
+}
+
 /// Summary statistics for the store.
 #[derive(Debug)]
 pub struct StoreStats {
@@ -100,6 +109,7 @@ impl Store {
             .execute_batch(SCHEMA)
             .context("failed to initialize schema")?;
         self.migrate()?;
+        self.ensure_fts_table()?;
         Ok(())
     }
 
@@ -439,6 +449,78 @@ impl Store {
         }
     }
 
+    // ── FTS5 ──────────────────────────────────────────────────
+
+    /// Ensure the FTS5 virtual table exists. Called during init.
+    pub fn ensure_fts_table(&self) -> Result<()> {
+        self.conn.execute_batch(
+            "CREATE VIRTUAL TABLE IF NOT EXISTS chunks_fts USING fts5(
+                content,
+                file_id UNINDEXED,
+                chunk_seq UNINDEXED
+            );",
+        ).context("failed to create FTS5 virtual table")?;
+        Ok(())
+    }
+
+    /// Insert a chunk's text into the FTS5 table.
+    pub fn insert_fts_chunk(&self, file_id: i64, chunk_seq: i64, text: &str) -> Result<()> {
+        self.conn.execute(
+            "INSERT INTO chunks_fts (content, file_id, chunk_seq) VALUES (?1, ?2, ?3)",
+            params![text, file_id, chunk_seq],
+        )?;
+        Ok(())
+    }
+
+    /// Delete all FTS5 entries for a file.
+    pub fn delete_fts_chunks_for_file(&self, file_id: i64) -> Result<()> {
+        self.conn.execute(
+            "DELETE FROM chunks_fts WHERE file_id = ?1",
+            params![file_id],
+        )?;
+        Ok(())
+    }
+
+    /// Search the FTS5 index. Returns results ranked by BM25 score.
+    /// BM25 in SQLite returns negative values (more negative = better match),
+    /// so we negate them to get positive scores where higher = better.
+    ///
+    /// The query is wrapped in double quotes so that FTS5 treats it as a
+    /// phrase/literal rather than interpreting operators like `-`.
+    pub fn fts_search(&self, query: &str, limit: usize) -> Result<Vec<FtsResult>> {
+        // Escape any double quotes in the query, then wrap in double quotes
+        // so FTS5 treats hyphens etc. as literal characters.
+        let escaped = query.replace('"', "\"\"");
+        let fts_query = format!("\"{}\"", escaped);
+
+        let mut stmt = self.conn.prepare(
+            "SELECT file_id, chunk_seq, bm25(chunks_fts) as score,
+                    snippet(chunks_fts, 0, '<b>', '</b>', '...', 64)
+             FROM chunks_fts
+             WHERE chunks_fts MATCH ?1
+             ORDER BY score
+             LIMIT ?2",
+        )?;
+
+        let rows = stmt.query_map(params![fts_query, limit as i64], |row| {
+            Ok(FtsResult {
+                file_id: row.get(0)?,
+                chunk_seq: row.get(1)?,
+                score: {
+                    let raw: f64 = row.get(2)?;
+                    -raw // negate: SQLite BM25 returns negative, more negative = better
+                },
+                snippet: row.get(3)?,
+            })
+        })?;
+
+        let mut results = Vec::new();
+        for row in rows {
+            results.push(row?);
+        }
+        Ok(results)
+    }
+
     /// Return vector_ids for all chunks belonging to a file.
     /// Useful for tombstoning before re-indexing a changed file.
     pub fn get_vector_ids_for_file(&self, file_id: i64) -> Result<Vec<u64>> {