Add semantic_query to search_graph: vector search via cbm_cosine_i8 SQL function

Vector search pipeline (end-to-end):
- store.c: cbm_cosine_i8 custom SQLite function (int8 dot product with
  sqrt magnitude), registered at store open time
- store.c: cbm_store_vector_search() builds merged query vector from
  keywords via base RI vectors (xxHash), int8-quantizes, runs SQL query
  with JOIN nodes ON node_id = id for metadata
- mcp.c: search_graph gains semantic_query param (array of strings),
  returns semantic_results alongside regular results
- sqlite_writer.c: fixed build_vector_record to include node_id in
  record body (matches build_node_record pattern)
- mcp.c: use yyjson_mut_obj_add_strcpy (not add_str) to avoid
  use-after-free when vector results are freed before JSON serialization

Quality note: base RI vectors at query time don't capture co-occurrence
enrichment — results are moderate quality. Storing enriched token vectors
would improve this significantly (future work).

Author: DeusData

internal/cbm/sqlite_writer.c

@@ -107,6 +107,9 @@ enum {
 // SQLite text serial type offset: serial_type = len*2 + TEXT_SERIAL_BASE.
 #define TEXT_SERIAL_BASE 13
 
+// SQLite blob serial type offset: serial_type = len*2 + BLOB_SERIAL_BASE.
+#define BLOB_SERIAL_BASE 12
+
 // SQLite integer storage range limits.
 #define INT8_MAX_VAL 127
 #define INT16_MAX_VAL 32767
@@ -360,6 +363,16 @@ static void rec_add_text(RecordBuilder *r, const char *s) {
     }
 }
 
+static void rec_add_blob(RecordBuilder *r, const uint8_t *data, int len) {
+    int64_t st = len > 0 ? (int64_t)len * 2 + BLOB_SERIAL_BASE : 0;
+    uint8_t vbuf[VARINT_MAX_BYTES];
+    int vlen = put_varint(vbuf, st);
+    dynbuf_append(&r->header, vbuf, vlen);
+    if (len > 0 && data) {
+        dynbuf_append(&r->body, data, len);
+    }
+}
+
 // Finalize: returns the complete record bytes (header_len + header + body).
 // Caller must free the returned buffer.
 static uint8_t *rec_finalize(RecordBuilder *r, int *out_len) {
@@ -718,6 +731,21 @@ static uint8_t *build_edge_record(const CBMDumpEdge *e, int *out_len) {
     return data;
 }
 
+// Build a node_vectors table record: (node_id, project, vector)
+// Includes node_id in the record body (same pattern as build_node_record).
+static uint8_t *build_vector_record(const CBMDumpVector *v, int *out_len) {
+    RecordBuilder r;
+    rec_init(&r);
+
+    rec_add_int(&r, v->node_id);
+    rec_add_text(&r, v->project);
+    rec_add_blob(&r, v->vector, v->vector_len);
+
+    uint8_t *data = rec_finalize(&r, out_len);
+    rec_free(&r);
+    return data;
+}
+
 // Build a projects table record: (name, indexed_at, root_path)
 static uint8_t *build_project_record(const char *name, const char *indexed_at,
                                      const char *root_path, int *out_len) {
@@ -1462,10 +1490,13 @@ typedef struct {
     int node_count;
     CBMDumpEdge *edges;
     int edge_count;
+    CBMDumpVector *vectors;
+    int vector_count;
 } write_db_ctx_t;
 
-/* Phase 1: Write node + edge data tables (streaming). */
-static int write_data_tables(write_db_ctx_t *w, uint32_t *nodes_root, uint32_t *edges_root) {
+/* Phase 1: Write node + edge + vector data tables (streaming). */
+static int write_data_tables(write_db_ctx_t *w, uint32_t *nodes_root, uint32_t *edges_root,
+                             uint32_t *vectors_root) {
     if (w->node_count > 0) {
         PageBuilder pb;
         pb_init(&pb, w->fp, w->next_page, false);
@@ -1501,6 +1532,26 @@ static int write_data_tables(write_db_ctx_t *w, uint32_t *nodes_root, uint32_t *
     } else {
         *edges_root = write_table_btree(w->fp, &w->next_page, NULL, NULL, NULL, 0, false);
     }
+
+    /* node_vectors table — uses node_id as rowid (not AUTOINCREMENT) */
+    if (w->vector_count > 0 && w->vectors) {
+        PageBuilder pb;
+        pb_init(&pb, w->fp, w->next_page, false);
+        for (int i = 0; i < w->vector_count; i++) {
+            int rec_len;
+            uint8_t *rec = build_vector_record(&w->vectors[i], &rec_len);
+            if (!rec) {
+                return ERR_WRITE_FAILED;
+            }
+            pb_add_table_cell_with_flush(&pb, w->vectors[i].node_id, rec, rec_len,
+                                         i > 0 ? w->vectors[i - SKIP_ONE].node_id : 0);
+            free(rec);
+        }
+        *vectors_root =
+            pb_finalize_table(&pb, &w->next_page, w->vectors[w->vector_count - SKIP_ONE].node_id);
+    } else {
+        *vectors_root = write_table_btree(w->fp, &w->next_page, NULL, NULL, NULL, 0, false);
+    }
     return 0;
 }
 
@@ -1548,7 +1599,7 @@ static void write_metadata_tables(write_db_ctx_t *w, uint32_t *projects_root,
 
 int cbm_write_db(const char *path, const char *project, const char *root_path,
                  const char *indexed_at, CBMDumpNode *nodes, int node_count, CBMDumpEdge *edges,
-                 int edge_count) {
+                 int edge_count, CBMDumpVector *vectors, int vector_count) {
     FILE *fp = fopen(path, "wb");
     if (!fp) {
         return CBM_NOT_FOUND;
@@ -1562,12 +1613,15 @@ int cbm_write_db(const char *path, const char *project, const char *root_path,
                         .nodes = nodes,
                         .node_count = node_count,
                         .edges = edges,
-                        .edge_count = edge_count};
+                        .edge_count = edge_count,
+                        .vectors = vectors,
+                        .vector_count = vector_count};
 
-    // Phase 1: Data tables (streaming node + edge records)
+    // Phase 1: Data tables (streaming node + edge + vector records)
     uint32_t nodes_root;
     uint32_t edges_root;
-    int rc = write_data_tables(&w, &nodes_root, &edges_root);
+    uint32_t vectors_root;
+    int rc = write_data_tables(&w, &nodes_root, &edges_root, &vectors_root);
     if (rc != 0) {
         (void)fclose(fp);
         return rc;
@@ -1753,6 +1807,9 @@ int cbm_write_db(const char *path, const char *project, const char *root_path,
          "NULL,\n\t\t\tupdated_at TEXT NOT NULL\n\t\t)"},
         {"index", "sqlite_autoindex_project_summaries_1", "project_summaries",
          autoindex_summaries_root, NULL},
+        {"table", "node_vectors", "node_vectors", vectors_root,
+         "CREATE TABLE node_vectors (\n\t\tnode_id INTEGER PRIMARY KEY,\n\t\tproject TEXT NOT "
+         "NULL,\n\t\tvector BLOB NOT NULL\n\t)"},
         {"table", "sqlite_sequence", "sqlite_sequence", sqlite_seq_root,
          "CREATE TABLE sqlite_sequence(name,seq)"},
     };

internal/cbm/sqlite_writer.h

@@ -27,13 +27,21 @@ typedef struct {
     const char *url_path;   // extracted from properties by Go (for idx_edges_url_path)
 } CBMDumpEdge;
 
+typedef struct {
+    int64_t node_id;       // final sequential ID (matches nodes.id)
+    const char *project;
+    const uint8_t *vector; // int8-quantized vector blob
+    int vector_len;        // length in bytes (e.g. 256 for d=256)
+} CBMDumpVector;
+
 // --- Public API ---
 
 // Write a complete SQLite .db file from sorted in-memory data.
 // Constructs B-tree pages directly — no SQL parser, no INSERTs.
 // Returns 0 on success, non-zero on error.
+// vectors/vector_count may be NULL/0 if no vectors are available.
 int cbm_write_db(const char *path, const char *project, const char *root_path,
                  const char *indexed_at, CBMDumpNode *nodes, int node_count, CBMDumpEdge *edges,
-                 int edge_count);
+                 int edge_count, CBMDumpVector *vectors, int vector_count);
 
 #endif // CBM_SQLITE_WRITER_H

src/graph_buffer/graph_buffer.c

@@ -82,6 +82,12 @@ struct cbm_gbuf {
     CBMHashTable *edges_by_source_type; /* "srcID:type" → edge_ptr_array_t* */
     CBMHashTable *edges_by_target_type; /* "tgtID:type" → edge_ptr_array_t* */
     CBMHashTable *edges_by_type;        /* "type" → edge_ptr_array_t* */
+
+    /* Vector storage for semantic embeddings (filled by pass_semantic_edges,
+     * consumed by cbm_write_db during dump). */
+    CBMDumpVector *dump_vectors;
+    int dump_vector_count;
+    int dump_vector_cap;
 };
 
 /* ── Helpers ─────────────────────────────────────────────────────── */
@@ -419,11 +425,50 @@ void cbm_gbuf_free(cbm_gbuf_t *gb) {
         cbm_ht_free(gb->edges_by_type);
     }
 
+    /* Free vector storage */
+    for (int i = 0; i < gb->dump_vector_count; i++) {
+        free((void *)gb->dump_vectors[i].vector);
+    }
+    free(gb->dump_vectors);
+
     free(gb->project);
     free(gb->root_path);
     free(gb);
 }
 
+/* ── Vector storage ──────────────────────────────────────────────── */
+
+int cbm_gbuf_store_vector(cbm_gbuf_t *gb, int64_t node_id, const uint8_t *vector, int vector_len) {
+    if (!gb || !vector || vector_len <= 0) {
+        return -1;
+    }
+    enum { VEC_INIT_CAP = 1024, VEC_GROW = 2 };
+    if (gb->dump_vector_count >= gb->dump_vector_cap) {
+        int new_cap =
+            gb->dump_vector_cap < VEC_INIT_CAP ? VEC_INIT_CAP : gb->dump_vector_cap * VEC_GROW;
+        CBMDumpVector *grown = realloc(gb->dump_vectors, (size_t)new_cap * sizeof(CBMDumpVector));
+        if (!grown) {
+            return -1;
+        }
+        gb->dump_vectors = grown;
+        gb->dump_vector_cap = new_cap;
+    }
+    /* Copy vector data */
+    uint8_t *vec_copy = malloc((size_t)vector_len);
+    if (!vec_copy) {
+        return -1;
+    }
+    memcpy(vec_copy, vector, (size_t)vector_len);
+
+    gb->dump_vectors[gb->dump_vector_count++] = (CBMDumpVector){
+        .node_id = node_id,
+        .project = gb->project, /* borrowed — valid until gbuf_free */
+        .vector = vec_copy,
+        .vector_len = vector_len,
+    };
+    return 0;
+}
+
 /* ── ID accessors ────────────────────────────────────────────────── */
 
 int64_t cbm_gbuf_next_id(const cbm_gbuf_t *gb) {
@@ -1197,7 +1242,7 @@ int cbm_gbuf_dump_to_sqlite(cbm_gbuf_t *gb, const char *path) {
      * Callers must delete the old .db before calling this (reindex)
      * or ensure no file exists (first index). */
     int rc = cbm_write_db(path, gb->project, gb->root_path, indexed_at, dump_nodes, node_idx,
-                          dump_edges, edge_idx);
+                          dump_edges, edge_idx, gb->dump_vectors, gb->dump_vector_count);
 
     {
         char b1[CBM_SZ_16];

src/graph_buffer/graph_buffer.h

@@ -148,6 +148,13 @@ int cbm_gbuf_edge_count_by_type(const cbm_gbuf_t *gb, const char *type);
 /* Delete all edges of a type. */
 int cbm_gbuf_delete_edges_by_type(cbm_gbuf_t *gb, const char *type);
 
+/* ── Vector storage (for semantic embeddings) ───────────────────── */
+
+/* Store an int8-quantized vector for a node. The vector data is copied.
+ * Called by pass_semantic_edges after computing RI vectors.
+ * Vectors are carried through to cbm_write_db during the dump phase. */
+int cbm_gbuf_store_vector(cbm_gbuf_t *gb, int64_t node_id, const uint8_t *vector, int vector_len);
+
 /* ── Dump to SQLite ──────────────────────────────────────────────── */
 
 /* Dump the entire buffer to a SQLite file using the direct page writer.

src/mcp/mcp.c

@@ -269,7 +269,9 @@ static const tool_def_t TOOLS[] = {
      "\"string\"},\"name_pattern\":{\"type\":\"string\"},\"qn_pattern\":{\"type\":\"string\"},"
      "\"file_pattern\":{\"type\":\"string\"},\"relationship\":{\"type\":\"string\"},\"min_degree\":"
      "{\"type\":\"integer\"},\"max_degree\":{\"type\":\"integer\"},\"exclude_entry_points\":{"
-     "\"type\":\"boolean\"},\"include_connected\":{\"type\":\"boolean\"},\"limit\":{\"type\":"
+     "\"type\":\"boolean\"},\"include_connected\":{\"type\":\"boolean\"},\"semantic_query\":{"
+     "\"type\":\"array\",\"items\":{\"type\":\"string\"},\"description\":\"Keywords for semantic "
+     "vector search (requires moderate/full index mode)\"},\"limit\":{\"type\":"
      "\"integer\",\"description\":\"Max results. Default: "
      "unlimited\"},\"offset\":{\"type\":\"integer\",\"default\":0}},\"required\":[\"project\"]}"},
 
@@ -1099,6 +1101,56 @@ static char *handle_search_graph(cbm_mcp_server_t *srv, const char *args) {
     yyjson_mut_obj_add_val(doc, root, "results", results);
     yyjson_mut_obj_add_bool(doc, root, "has_more", out.total > offset + out.count);
 
+    /* Semantic vector search: parse semantic_query array and run vector search */
+    {
+        yyjson_doc *args_doc = yyjson_read(args, strlen(args), 0);
+        yyjson_val *args_root = args_doc ? yyjson_doc_get_root(args_doc) : NULL;
+        yyjson_val *sq_arr = args_root ? yyjson_obj_get(args_root, "semantic_query") : NULL;
+        if (sq_arr && yyjson_is_arr(sq_arr)) {
+            int kw_count = (int)yyjson_arr_size(sq_arr);
+            if (kw_count > 0) {
+                enum { MAX_KW = 32 };
+                const char *keywords[MAX_KW];
+                if (kw_count > MAX_KW) {
+                    kw_count = MAX_KW;
+                }
+                size_t kw_idx = 0;
+                size_t kw_max = 0;
+                yyjson_val *kw_val;
+                int ki = 0;
+                yyjson_arr_foreach(sq_arr, kw_idx, kw_max, kw_val) {
+                    if (ki < kw_count && yyjson_is_str(kw_val)) {
+                        keywords[ki++] = yyjson_get_str(kw_val);
+                    }
+                }
+
+                cbm_vector_result_t *vresults = NULL;
+                int vcount = 0;
+                int sem_limit = limit > 0 ? limit : CBM_SZ_16;
+                if (cbm_store_vector_search(store, project, keywords, ki, sem_limit, &vresults,
+                                            &vcount) == CBM_STORE_OK &&
+                    vcount > 0) {
+                    yyjson_mut_val *sem_results = yyjson_mut_arr(doc);
+                    for (int v = 0; v < vcount; v++) {
+                        yyjson_mut_val *vitem = yyjson_mut_obj(doc);
+                        yyjson_mut_obj_add_strcpy(doc, vitem, "name", vresults[v].name);
+                        yyjson_mut_obj_add_strcpy(doc, vitem, "qualified_name",
+                                                  vresults[v].qualified_name);
+                        yyjson_mut_obj_add_strcpy(doc, vitem, "label", vresults[v].label);
+                        yyjson_mut_obj_add_strcpy(doc, vitem, "file_path", vresults[v].file_path);
+                        yyjson_mut_obj_add_real(doc, vitem, "score", vresults[v].score);
+                        yyjson_mut_arr_add_val(sem_results, vitem);
+                    }
+                    yyjson_mut_obj_add_val(doc, root, "semantic_results", sem_results);
+                    cbm_store_free_vector_results(vresults, vcount);
+                }
+            }
+        }
+        if (args_doc) {
+            yyjson_doc_free(args_doc);
+        }
+    }
+
     char *json = yy_doc_to_str(doc);
     yyjson_mut_doc_free(doc);
     cbm_store_search_free(&out);

src/pipeline/pass_semantic_edges.c

@@ -383,8 +383,24 @@ int cbm_pipeline_pass_semantic_edges(cbm_pipeline_ctx_t *ctx) {
             }
         }
         cbm_sem_normalize(&funcs[f].ri_vec);
+
+        /* Int8-quantize and store in graph buffer for vector search at query time */
+        uint8_t qvec[CBM_SEM_DIM];
+        for (int d = 0; d < CBM_SEM_DIM; d++) {
+            float clamped = funcs[f].ri_vec.v[d];
+            if (clamped > 1.0f) {
+                clamped = 1.0f;
+            }
+            if (clamped < -1.0f) {
+                clamped = -1.0f;
+            }
+            qvec[d] = (uint8_t)(int8_t)(clamped * 127.0f);
+        }
+        cbm_gbuf_store_vector(gbuf, funcs[f].node_id, qvec, CBM_SEM_DIM);
     }
 
+    cbm_log_info("pass.semantic.vectors_stored", "count", itoa_log(func_count));
+
     /* Phase 5: Build cosine-LSH index using random hyperplanes on RI vectors.
      * This gives O(n) candidate generation instead of O(n²) brute-force.
      *