WIP: Algorithmic semantic embeddings + indexing modes (checkpoint before vector storage)

Author: DeusData

Makefile.cbm

@@ -186,11 +186,15 @@ PIPELINE_SRCS = \
     src/pipeline/pass_compile_commands.c \
     src/pipeline/pass_infrascan.c \
     src/pipeline/pass_k8s.c \
-    src/pipeline/pass_similarity.c
+    src/pipeline/pass_similarity.c \
+    src/pipeline/pass_semantic_edges.c
 
 # SimHash / MinHash module
 SIMHASH_SRCS = src/simhash/minhash.c
 
+# Semantic embedding module
+SEMANTIC_SRCS = src/semantic/semantic.c src/semantic/ast_profile.c
+
 # Traces module (new)
 TRACES_SRCS = src/traces/traces.c
 
@@ -237,7 +241,7 @@ TRE_CFLAGS = -std=c11 -g -O1 -w -Ivendored/tre
 YYJSON_SRC = vendored/yyjson/yyjson.c
 
 # All production sources
-PROD_SRCS = $(FOUNDATION_SRCS) $(STORE_SRCS) $(CYPHER_SRCS) $(MCP_SRCS) $(DISCOVER_SRCS) $(GRAPH_BUFFER_SRCS) $(PIPELINE_SRCS) $(SIMHASH_SRCS) $(TRACES_SRCS) $(WATCHER_SRCS) $(CLI_SRCS) $(UI_SRCS) $(YYJSON_SRC)
+PROD_SRCS = $(FOUNDATION_SRCS) $(STORE_SRCS) $(CYPHER_SRCS) $(MCP_SRCS) $(DISCOVER_SRCS) $(GRAPH_BUFFER_SRCS) $(PIPELINE_SRCS) $(SIMHASH_SRCS) $(SEMANTIC_SRCS) $(TRACES_SRCS) $(WATCHER_SRCS) $(CLI_SRCS) $(UI_SRCS) $(YYJSON_SRC)
 EXISTING_C_SRCS = $(EXTRACTION_SRCS) $(LSP_SRCS) $(TS_RUNTIME_SRC) \
                   $(GRAMMAR_SRCS) $(AC_LZ4_SRCS) $(SQLITE_WRITER_SRC)
 
@@ -511,7 +515,7 @@ SYSROOT_FLAG = $(if $(SYSROOT),-isysroot $(SYSROOT),)
 
 # Our source files (excluding vendored, grammars, tree-sitter runtime)
 LINT_SRCS = $(FOUNDATION_SRCS) $(STORE_SRCS) $(CYPHER_SRCS) $(MCP_SRCS) \
-            $(DISCOVER_SRCS) $(GRAPH_BUFFER_SRCS) $(PIPELINE_SRCS) $(SIMHASH_SRCS) \
+            $(DISCOVER_SRCS) $(GRAPH_BUFFER_SRCS) $(PIPELINE_SRCS) $(SIMHASH_SRCS) $(SEMANTIC_SRCS) \
             $(TRACES_SRCS) $(WATCHER_SRCS) $(CLI_SRCS) $(EXTRACTION_SRCS) $(AC_LZ4_SRCS) \
             $(SQLITE_WRITER_SRC) $(MAIN_SRC)
 LINT_HDRS = $(wildcard src/**/*.h src/*.h $(CBM_DIR)/*.h)

internal/cbm/cbm.h

@@ -109,6 +109,7 @@ typedef struct {
     bool is_abstract;
     bool is_test;
     bool is_entry_point;
+    const char *structural_profile; // AST structural profile (arena-allocated) or NULL
 } CBMDefinition;
 
 /* Argument captured from a call expression */

internal/cbm/extract_defs.c

@@ -4,6 +4,7 @@
 #include "lang_specs.h"
 #include "foundation/constants.h"
 #include "simhash/minhash.h"
+#include "semantic/ast_profile.h"
 #include "tree_sitter/api.h" // TSNode, ts_node_*
 #include <stdint.h>          // uint32_t
 #include <string.h>
@@ -50,6 +51,21 @@ static void compute_fingerprint(CBMExtractCtx *ctx, CBMDefinition *def, TSNode f
     memcpy(fp, result.values, CBM_MINHASH_K * sizeof(uint32_t));
     def->fingerprint = fp;
     def->fingerprint_k = CBM_MINHASH_K;
+
+    /* AST structural profile (signals 8, 9, 11) — rides the same body node */
+    cbm_ast_profile_t profile;
+    int pc = 0;
+    if (def->param_names) {
+        while (def->param_names[pc]) {
+            pc++;
+        }
+    }
+    if (cbm_ast_profile_compute(body, ctx->source, def->param_names, pc, &profile)) {
+        profile.body_lines = (uint16_t)def->lines;
+        char sp_buf[CBM_AST_PROFILE_BUF];
+        cbm_ast_profile_to_str(&profile, sp_buf, sizeof(sp_buf));
+        def->structural_profile = cbm_arena_strdup(ctx->arena, sp_buf);
+    }
 }
 
 // Tree-sitter row is 0-based; lines are 1-based.

src/mcp/mcp.c

@@ -1678,6 +1678,8 @@ static char *handle_index_repository(cbm_mcp_server_t *srv, const char *args) {
     cbm_index_mode_t mode = CBM_MODE_FULL;
     if (mode_str && strcmp(mode_str, "fast") == 0) {
         mode = CBM_MODE_FAST;
+    } else if (mode_str && strcmp(mode_str, "moderate") == 0) {
+        mode = CBM_MODE_MODERATE;
     }
     free(mode_str);
 

src/pipeline/pass_definitions.c

@@ -21,6 +21,7 @@ enum { PD_RING = 4, PD_RING_MASK = 3, PD_JSON_MARGIN = 10, PD_ESC_MARGIN = 3, PD
 #include "foundation/compat.h"
 #include "cbm.h"
 #include "simhash/minhash.h"
+#include "semantic/ast_profile.h"
 
 #include <stdio.h>
 #include <stdlib.h>
@@ -200,6 +201,11 @@ static void build_def_props(char *buf, size_t bufsize, const CBMDefinition *def)
         append_json_string(buf, bufsize, &pos, "fp", fp_hex);
     }
 
+    /* AST structural profile */
+    if (def->structural_profile && pos + CBM_AST_PROFILE_BUF < bufsize) {
+        append_json_string(buf, bufsize, &pos, "sp", def->structural_profile);
+    }
+
     if (pos < bufsize - SKIP_ONE) {
         buf[pos] = '}';
         buf[pos + SKIP_ONE] = '\0';

src/pipeline/pass_parallel.c

@@ -40,6 +40,7 @@ enum {
 #include "foundation/compat_regex.h"
 #include "cbm.h"
 #include "simhash/minhash.h"
+#include "semantic/ast_profile.h"
 
 #include <stdatomic.h>
 #include <stdint.h>
@@ -223,6 +224,11 @@ static void build_def_props(char *buf, size_t bufsize, const CBMDefinition *def)
         append_json_string(buf, bufsize, &pos, "fp", fp_hex);
     }
 
+    /* AST structural profile — append if present and buffer has room. */
+    if (def->structural_profile && pos + CBM_AST_PROFILE_BUF < bufsize) {
+        append_json_string(buf, bufsize, &pos, "sp", def->structural_profile);
+    }
+
     if (pos < bufsize - SKIP_ONE) {
         buf[pos] = '}';
         buf[pos + SKIP_ONE] = '\0';