diff --git a/Makefile.cbm b/Makefile.cbm index 865a1b40e..db139ef23 100644 --- a/Makefile.cbm +++ b/Makefile.cbm @@ -122,6 +122,7 @@ EXTRACTION_SRCS = \ $(CBM_DIR)/extract_type_refs.c \ $(CBM_DIR)/extract_type_assigns.c \ $(CBM_DIR)/extract_env_accesses.c \ + $(CBM_DIR)/extract_channels.c \ $(CBM_DIR)/extract_k8s.c \ $(CBM_DIR)/helpers.c \ $(CBM_DIR)/lang_specs.c \ diff --git a/internal/cbm/cbm.c b/internal/cbm/cbm.c index 3d5360a31..7c34f0b0d 100644 --- a/internal/cbm/cbm.c +++ b/internal/cbm/cbm.c @@ -152,6 +152,11 @@ void cbm_resolvedcall_push(CBMResolvedCallArray *arr, CBMArena *a, CBMResolvedCa arr->items[arr->count++] = rc; } +void cbm_channels_push(CBMChannelArray *arr, CBMArena *a, CBMChannel ch) { + GROW_ARRAY(arr, a); + arr->items[arr->count++] = ch; +} + // --- String input reader (for parse_with_options) --- typedef struct { @@ -336,6 +341,9 @@ CBMFileResult *cbm_extract_file(const char *source, int source_len, CBMLanguage cbm_extract_imports(&ctx); cbm_extract_unified(&ctx); + // Channel detection (Socket.IO / EventEmitter) — JS/TS only. + cbm_extract_channels(&ctx); + // K8s / Kustomize semantic pass (additional structured extraction for YAML-based infra files). if (ctx.language == CBM_LANG_KUSTOMIZE || ctx.language == CBM_LANG_K8S) { cbm_extract_k8s(&ctx); diff --git a/internal/cbm/cbm.h b/internal/cbm/cbm.h index d10717928..2bbd73be9 100644 --- a/internal/cbm/cbm.h +++ b/internal/cbm/cbm.h @@ -110,7 +110,7 @@ typedef struct { bool is_test; bool is_entry_point; const char *structural_profile; // AST structural profile (arena-allocated) or NULL - const char *body_tokens; // space-separated raw identifier tokens from body (arena) or NULL + const char *body_tokens; // space-separated raw identifier tokens from body (arena) or NULL } CBMDefinition; /* Argument captured from a call expression */ @@ -192,6 +192,24 @@ typedef struct { const char *broker; // "pubsub", "cloud_tasks", "cloud_scheduler", "sqs", "kafka" } CBMInfraBinding; +/* Pub/sub channel participation. One record per emit() or on()/addListener() + * call detected in source — the receiver (e.g. Socket.IO client, EventEmitter + * instance) is intentionally NOT identified; matching is by channel_name + * across files, which captures the common pattern of one logical bus per + * service. Transport disambiguates Socket.IO vs EventEmitter vs future + * detectors (Kafka, Cloud Pub/Sub, etc.). */ +typedef enum { + CBM_CHANNEL_EMIT = 0, + CBM_CHANNEL_LISTEN = 1, +} CBMChannelDirection; + +typedef struct { + const char *channel_name; // literal channel name (e.g. "user.created") + const char *transport; // "socketio", "event_emitter", ... + const char *enclosing_func_qn; // QN of the function containing the emit/on call + CBMChannelDirection direction; +} CBMChannel; + // Rust: impl Trait for Struct typedef struct { const char *trait_name; // trait name (raw text) @@ -286,6 +304,12 @@ typedef struct { int cap; } CBMImplTraitArray; +typedef struct { + CBMChannel *items; + int count; + int cap; +} CBMChannelArray; + // Full extraction result for one file. typedef struct { CBMArena arena; // owns all string memory @@ -303,6 +327,7 @@ typedef struct { CBMResolvedCallArray resolved_calls; // LSP-resolved calls (high confidence) CBMStringRefArray string_refs; // URL/config string literals from AST CBMInfraBindingArray infra_bindings; // topic→URL pairs from IaC configs + CBMChannelArray channels; // Socket.IO / EventEmitter pub/sub participation const char *module_qn; // module qualified name const char **exports; // NULL-terminated (NULL if none) @@ -421,6 +446,7 @@ void cbm_stringref_push(CBMStringRefArray *arr, CBMArena *a, CBMStringRef sr); void cbm_infrabinding_push(CBMInfraBindingArray *arr, CBMArena *a, CBMInfraBinding ib); void cbm_impltrait_push(CBMImplTraitArray *arr, CBMArena *a, CBMImplTrait it); void cbm_resolvedcall_push(CBMResolvedCallArray *arr, CBMArena *a, CBMResolvedCall rc); +void cbm_channels_push(CBMChannelArray *arr, CBMArena *a, CBMChannel ch); // --- Sub-extractor entry points --- @@ -432,6 +458,7 @@ void cbm_extract_semantic(CBMExtractCtx *ctx); void cbm_extract_type_refs(CBMExtractCtx *ctx); void cbm_extract_env_accesses(CBMExtractCtx *ctx); void cbm_extract_type_assigns(CBMExtractCtx *ctx); +void cbm_extract_channels(CBMExtractCtx *ctx); // Single-pass unified extraction (replaces the 7 calls above except defs+imports). void cbm_extract_unified(CBMExtractCtx *ctx); diff --git a/internal/cbm/extract_channels.c b/internal/cbm/extract_channels.c new file mode 100644 index 000000000..3a5fca132 --- /dev/null +++ b/internal/cbm/extract_channels.c @@ -0,0 +1,284 @@ +/* + * extract_channels.c — Pub/sub channel participation extractor. + * + * Detects Socket.IO and EventEmitter emit / on / addListener call patterns in + * JS/TS/TSX source and records each participation as a CBMChannel record. + * Transport is stored on the record ("socketio", "event_emitter") so later + * detectors for Kafka, Cloud Pub/Sub, etc. can share the same schema without + * changing the edge types. + * + * String-constant resolution: when the channel name argument is a plain + * identifier, we perform a single-pass local scan of the module body to + * resolve `const EVENT = "foo"` style bindings. Template literals and + * config-driven names stay unresolved (acceptable — those require real + * data-flow analysis). + */ +#include "cbm.h" +#include "arena.h" +#include "helpers.h" +#include "foundation/constants.h" +#include "tree_sitter/api.h" +#include +#include + +enum { + CHAN_CONST_CAP = 256, /* max tracked identifiers per file */ + CHAN_STACK_CAP = 4096, /* tree-walk stack */ + CHAN_IDENT_MAX = 128, /* max identifier length tracked */ +}; + +typedef struct { + const char *name; /* borrowed — points into arena */ + const char *value; /* borrowed — points into arena */ +} chan_const_t; + +typedef struct { + chan_const_t items[CHAN_CONST_CAP]; + int count; +} chan_const_table_t; + +/* ── String literal helpers ──────────────────────────────────────── */ + +static const char *unquote_string(CBMArena *a, const char *s) { + if (!s) { + return NULL; + } + size_t len = strlen(s); + if (len < CBM_QUOTE_PAIR) { + return NULL; + } + char first = s[0]; + char last = s[len - CBM_QUOTE_OFFSET]; + if ((first == '"' && last == '"') || (first == '\'' && last == '\'') || + (first == '`' && last == '`')) { + return cbm_arena_strndup(a, s + CBM_QUOTE_OFFSET, len - CBM_QUOTE_PAIR); + } + return NULL; +} + +/* Extract a literal channel name from an argument node. Returns NULL if the + * argument is not a plain string literal (caller can then try identifier + * resolution via the constant table). */ +static const char *literal_from_arg(CBMExtractCtx *ctx, TSNode arg) { + const char *kind = ts_node_type(arg); + if (strcmp(kind, "string") != 0 && strcmp(kind, "string_literal") != 0) { + return NULL; + } + char *text = cbm_node_text(ctx->arena, arg, ctx->source); + return unquote_string(ctx->arena, text); +} + +/* ── Constant resolution table ──────────────────────────────────── */ + +/* Walk the whole tree once and collect `const IDENT = "value"` bindings so + * later passes can resolve bare-identifier channel arguments. Only scalar + * string literals are tracked — template literals and expressions are left + * unresolved. This is a flat lookup; scope boundaries are ignored (a single + * const table per file is sufficient for the common Socket.IO pattern). */ +static void scan_string_consts(CBMExtractCtx *ctx, chan_const_table_t *tbl) { + TSNode stack[CHAN_STACK_CAP]; + int top = 0; + stack[top++] = ctx->root; + + while (top > 0 && tbl->count < CHAN_CONST_CAP) { + TSNode node = stack[--top]; + const char *kind = ts_node_type(node); + + /* `variable_declarator` is the TS/JS form of `IDENT = value`. + * The `name` field holds the identifier, `value` holds the RHS. */ + if (strcmp(kind, "variable_declarator") == 0) { + TSNode name_node = ts_node_child_by_field_name(node, TS_FIELD("name")); + TSNode value_node = ts_node_child_by_field_name(node, TS_FIELD("value")); + if (!ts_node_is_null(name_node) && !ts_node_is_null(value_node)) { + const char *nk = ts_node_type(name_node); + const char *vk = ts_node_type(value_node); + if (strcmp(nk, "identifier") == 0 && + (strcmp(vk, "string") == 0 || strcmp(vk, "string_literal") == 0)) { + char *name_text = cbm_node_text(ctx->arena, name_node, ctx->source); + char *value_text = cbm_node_text(ctx->arena, value_node, ctx->source); + const char *unq = unquote_string(ctx->arena, value_text); + if (name_text && unq) { + tbl->items[tbl->count].name = name_text; + tbl->items[tbl->count].value = unq; + tbl->count++; + } + } + } + } + + uint32_t count = ts_node_child_count(node); + for (int i = (int)count - SKIP_ONE; i >= 0 && top < CHAN_STACK_CAP; i--) { + stack[top++] = ts_node_child(node, (uint32_t)i); + } + } +} + +/* Resolve an identifier against the constant table. Returns NULL on miss. */ +static const char *resolve_identifier(const chan_const_table_t *tbl, const char *name) { + if (!name) { + return NULL; + } + for (int i = 0; i < tbl->count; i++) { + if (tbl->items[i].name && strcmp(tbl->items[i].name, name) == 0) { + return tbl->items[i].value; + } + } + return NULL; +} + +/* ── Enclosing function detection ───────────────────────────────── */ + +/* Walk up the parent chain to find the nearest function-like ancestor and + * build a best-effort qualified name for it. */ +static const char *enclosing_function_qn(CBMExtractCtx *ctx, TSNode node) { + TSNode parent = ts_node_parent(node); + while (!ts_node_is_null(parent)) { + const char *pk = ts_node_type(parent); + if (strcmp(pk, "function_declaration") == 0 || strcmp(pk, "method_definition") == 0 || + strcmp(pk, "arrow_function") == 0 || strcmp(pk, "function_expression") == 0 || + strcmp(pk, "function") == 0 || strcmp(pk, "method_signature") == 0) { + TSNode name_node = ts_node_child_by_field_name(parent, TS_FIELD("name")); + if (!ts_node_is_null(name_node)) { + char *name = cbm_node_text(ctx->arena, name_node, ctx->source); + if (name && name[0]) { + return name; + } + } + return NULL; + } + parent = ts_node_parent(parent); + } + return NULL; +} + +/* ── Emit / listen detection ────────────────────────────────────── */ + +static bool is_emit_method(const char *name) { + return name && strcmp(name, "emit") == 0; +} + +static bool is_listen_method(const char *name) { + return name && (strcmp(name, "on") == 0 || strcmp(name, "addListener") == 0 || + strcmp(name, "once") == 0); +} + +/* Match a transport receiver: plain identifier `socket`/`io`/`emitter` or a + * member expression like `this.io` / `client.socket`. Returns "socketio" or + * "event_emitter" based on a name heuristic, NULL if the receiver is unknown + * (which means we skip — we don't want to mistake any .emit()/.on() call + * for a channel). */ +static const char *classify_receiver(CBMExtractCtx *ctx, TSNode object_node) { + char *text = cbm_node_text(ctx->arena, object_node, ctx->source); + if (!text) { + return NULL; + } + /* Strip leading `this.`, `self.`, or class-instance prefixes for the match. */ + const char *tail = text; + const char *dot = strrchr(tail, '.'); + if (dot) { + tail = dot + SKIP_ONE; + } + /* Common Socket.IO variable names. */ + if (strcmp(tail, "socket") == 0 || strcmp(tail, "io") == 0 || strcmp(tail, "ws") == 0 || + strcmp(tail, "client") == 0 || strcmp(tail, "server") == 0) { + return "socketio"; + } + /* Node.js EventEmitter convention. */ + if (strcmp(tail, "emitter") == 0 || strcmp(tail, "eventEmitter") == 0 || + strcmp(tail, "events") == 0 || strcmp(tail, "bus") == 0 || strcmp(tail, "eventBus") == 0 || + strcmp(tail, "pubsub") == 0) { + return "event_emitter"; + } + return NULL; +} + +/* Process a single call_expression node if it looks like a channel call. */ +static void process_channel_call(CBMExtractCtx *ctx, TSNode call, + const chan_const_table_t *consts) { + /* call_expression { function: member_expression { object, property }, arguments } */ + TSNode func = ts_node_child_by_field_name(call, TS_FIELD("function")); + if (ts_node_is_null(func) || strcmp(ts_node_type(func), "member_expression") != 0) { + return; + } + TSNode object = ts_node_child_by_field_name(func, TS_FIELD("object")); + TSNode property = ts_node_child_by_field_name(func, TS_FIELD("property")); + if (ts_node_is_null(object) || ts_node_is_null(property)) { + return; + } + + char *method = cbm_node_text(ctx->arena, property, ctx->source); + CBMChannelDirection direction; + if (is_emit_method(method)) { + direction = CBM_CHANNEL_EMIT; + } else if (is_listen_method(method)) { + direction = CBM_CHANNEL_LISTEN; + } else { + return; + } + + const char *transport = classify_receiver(ctx, object); + if (!transport) { + return; + } + + /* First positional argument is the channel name. */ + TSNode args = ts_node_child_by_field_name(call, TS_FIELD("arguments")); + if (ts_node_is_null(args)) { + return; + } + uint32_t arg_count = ts_node_named_child_count(args); + if (arg_count == 0) { + return; + } + TSNode first = ts_node_named_child(args, 0); + + const char *channel_name = literal_from_arg(ctx, first); + if (!channel_name) { + /* Try identifier resolution via the constant table. */ + const char *kind = ts_node_type(first); + if (strcmp(kind, "identifier") == 0) { + char *ident = cbm_node_text(ctx->arena, first, ctx->source); + channel_name = resolve_identifier(consts, ident); + } + } + if (!channel_name) { + return; /* template literal, member access, expression — skip */ + } + + CBMChannel ch = { + .channel_name = channel_name, + .transport = transport, + .enclosing_func_qn = enclosing_function_qn(ctx, call), + .direction = direction, + }; + cbm_channels_push(&ctx->result->channels, ctx->arena, ch); +} + +/* ── Entry point ────────────────────────────────────────────────── */ + +void cbm_extract_channels(CBMExtractCtx *ctx) { + /* Only JS/TS variants — Socket.IO and EventEmitter are Node.js ecosystem. */ + if (ctx->language != CBM_LANG_JAVASCRIPT && ctx->language != CBM_LANG_TYPESCRIPT && + ctx->language != CBM_LANG_TSX) { + return; + } + + chan_const_table_t consts = {0}; + scan_string_consts(ctx, &consts); + + /* Second pass: walk the tree looking for call_expression nodes. */ + TSNode stack[CHAN_STACK_CAP]; + int top = 0; + stack[top++] = ctx->root; + + while (top > 0) { + TSNode node = stack[--top]; + if (strcmp(ts_node_type(node), "call_expression") == 0) { + process_channel_call(ctx, node, &consts); + } + uint32_t count = ts_node_child_count(node); + for (int i = (int)count - SKIP_ONE; i >= 0 && top < CHAN_STACK_CAP; i--) { + stack[top++] = ts_node_child(node, (uint32_t)i); + } + } +} diff --git a/internal/cbm/extract_defs.c b/internal/cbm/extract_defs.c index f7c05d9bc..525b58445 100644 --- a/internal/cbm/extract_defs.c +++ b/internal/cbm/extract_defs.c @@ -28,8 +28,90 @@ enum { C_RETURN_WALK_DEPTH = 5, VAR_RECURSION_LIMIT = 8, NESTED_CLASS_STACK_CAP = 128, + FP_HASH_MUL = 31, /* FNV-like hash multiplier for identifier dedup */ + FP_HASH_NONZERO = 1, /* OR mask to ensure hash is never zero (sentinel) */ + FP_SPACE_SEP = 1, /* one byte for space separator between tokens */ }; +/* Hash a span of source text. */ +static uint32_t hash_source_span(const char *source, uint32_t start, int len) { + uint32_t h = 0; + for (int x = 0; x < len; x++) { + h = (h * FP_HASH_MUL) + (uint32_t)(unsigned char)source[start + (uint32_t)x]; + } + return h; +} + +/* Try to append a unique identifier to the buffer. Returns true if appended. */ +static bool try_append_ident(const char *source, uint32_t s, int len, uint32_t *seen, int seen_size, + uint32_t seen_mask, char *buf, int buf_size, int *pos, int *count) { + uint32_t h = hash_source_span(source, s, len); + uint32_t key = h | FP_HASH_NONZERO; + uint32_t slot = h & seen_mask; + bool dup = false; + for (int p = 0; p < seen_size; p++) { + uint32_t idx = (slot + (uint32_t)p) & seen_mask; + if (seen[idx] == 0) { + seen[idx] = key; + break; + } + if (seen[idx] == key) { + dup = true; + break; + } + } + if (dup || *pos + len + FP_SPACE_SEP >= buf_size) { + return false; + } + if (*pos > 0) { + buf[(*pos)++] = ' '; + } + memcpy(buf + *pos, source + s, (size_t)len); + *pos += len; + (*count)++; + return true; +} + +/* Walk AST body, collect unique identifier text as space-separated string. + * Returns arena-allocated string or NULL. */ +static char *extract_body_ident_tokens(CBMExtractCtx *ctx, TSNode body) { + enum { BT_STACK = 512, BT_BUF = 512, BT_MAX_IDENTS = 40, BT_SEEN = 128, BT_SEEN_MASK = 127 }; + TSNode bt_stack[BT_STACK]; + int bt_top = 0; + bt_stack[bt_top++] = body; + char bt_buf[BT_BUF]; + int bt_pos = 0; + uint32_t bt_seen[BT_SEEN]; + memset(bt_seen, 0, sizeof(bt_seen)); + int bt_count = 0; + + while (bt_top > 0 && bt_count < BT_MAX_IDENTS) { + TSNode nd = bt_stack[--bt_top]; + uint32_t nc = ts_node_child_count(nd); + if (nc == 0) { + const char *k = ts_node_type(nd); + if (strcmp(k, "identifier") == 0 || strcmp(k, "field_identifier") == 0 || + strcmp(k, "property_identifier") == 0) { + uint32_t s = ts_node_start_byte(nd); + int len = (int)(ts_node_end_byte(nd) - s); + if (len > 0 && len < CBM_SZ_64 && s < (uint32_t)ctx->source_len) { + try_append_ident(ctx->source, s, len, bt_seen, BT_SEEN, BT_SEEN_MASK, bt_buf, + BT_BUF, &bt_pos, &bt_count); + } + } + } else { + for (int i = (int)nc - SKIP_ONE; i >= 0 && bt_top < BT_STACK; i--) { + bt_stack[bt_top++] = ts_node_child(nd, (uint32_t)i); + } + } + } + if (bt_pos > 0) { + bt_buf[bt_pos] = '\0'; + return cbm_arena_strdup(ctx->arena, bt_buf); + } + return NULL; +} + /* Compute MinHash fingerprint for a function body node and store in def. * Sets def->fingerprint (arena-allocated) and def->fingerprint_k on success, * leaves them NULL/0 if the body is too short. */ @@ -67,62 +149,8 @@ static void compute_fingerprint(CBMExtractCtx *ctx, CBMDefinition *def, TSNode f def->structural_profile = cbm_arena_strdup(ctx->arena, sp_buf); } - /* Extract raw identifier tokens from body for semantic search. - * Walk the AST, collect unique identifier text, store as space-separated string. - * Cap at ~500 chars to fit in properties_json. */ - { - enum { BT_STACK = 512, BT_BUF = 512, BT_MAX_IDENTS = 40, BT_SEEN = 128, BT_SEEN_MASK = 127 }; - TSNode bt_stack[BT_STACK]; - int bt_top = 0; - bt_stack[bt_top++] = body; - char bt_buf[BT_BUF]; - int bt_pos = 0; - uint32_t bt_seen[BT_SEEN]; - memset(bt_seen, 0, sizeof(bt_seen)); - int bt_count = 0; - - while (bt_top > 0 && bt_count < BT_MAX_IDENTS) { - TSNode nd = bt_stack[--bt_top]; - uint32_t nc = ts_node_child_count(nd); - if (nc == 0) { - const char *k = ts_node_type(nd); - if (strcmp(k, "identifier") == 0 || strcmp(k, "field_identifier") == 0 || - strcmp(k, "property_identifier") == 0) { - uint32_t s = ts_node_start_byte(nd); - uint32_t e = ts_node_end_byte(nd); - int len = (int)(e - s); - if (len > 0 && len < CBM_SZ_64 && s < (uint32_t)ctx->source_len) { - /* Dedup via simple hash set */ - uint32_t h = 0; - for (int x = 0; x < len; x++) { - h = h * 31 + (uint32_t)(unsigned char)ctx->source[s + x]; - } - uint32_t slot = h & BT_SEEN_MASK; - bool dup = false; - for (int p = 0; p < BT_SEEN; p++) { - uint32_t idx = (slot + (uint32_t)p) & BT_SEEN_MASK; - if (bt_seen[idx] == 0) { bt_seen[idx] = h | 1; break; } - if (bt_seen[idx] == (h | 1)) { dup = true; break; } - } - if (!dup && bt_pos + len + 1 < BT_BUF) { - if (bt_pos > 0) { bt_buf[bt_pos++] = ' '; } - memcpy(bt_buf + bt_pos, ctx->source + s, (size_t)len); - bt_pos += len; - bt_count++; - } - } - } - } else { - for (int i = (int)nc - 1; i >= 0 && bt_top < BT_STACK; i--) { - bt_stack[bt_top++] = ts_node_child(nd, (uint32_t)i); - } - } - } - if (bt_pos > 0) { - bt_buf[bt_pos] = '\0'; - def->body_tokens = cbm_arena_strdup(ctx->arena, bt_buf); - } - } + /* Extract raw identifier tokens from body for semantic search */ + def->body_tokens = extract_body_ident_tokens(ctx, body); } // Tree-sitter row is 0-based; lines are 1-based. @@ -982,6 +1010,72 @@ static const char **find_base_from_children(CBMArena *a, TSNode node, const char return NULL; } +/* Extract text from a single C# base_list named child, stripping generic args. */ +static const char *extract_csharp_base_child_text(CBMArena *a, TSNode bc, const char *source) { + const char *bk = ts_node_type(bc); + char *text = NULL; + if (strcmp(bk, "identifier") == 0 || strcmp(bk, "generic_name") == 0 || + strcmp(bk, "qualified_name") == 0) { + text = cbm_node_text(a, bc, source); + } else { + TSNode inner = ts_node_named_child(bc, 0); + if (!ts_node_is_null(inner)) { + text = cbm_node_text(a, inner, source); + } + } + if (text && text[0]) { + char *angle = strchr(text, '<'); + if (angle) { + *angle = '\0'; + } + return text; + } + return NULL; +} + +/* Collect bases from a single base_list node into an arena-allocated array. */ +static const char **collect_csharp_bases(CBMArena *a, TSNode base_list, const char *source) { + const char *bases[MAX_BASES]; + int base_count = 0; + uint32_t bnc = ts_node_named_child_count(base_list); + for (uint32_t bi = 0; bi < bnc && base_count < MAX_BASES_MINUS_1; bi++) { + const char *text = + extract_csharp_base_child_text(a, ts_node_named_child(base_list, bi), source); + if (text) { + bases[base_count++] = text; + } + } + if (base_count == 0) { + return NULL; + } + const char **result = + (const char **)cbm_arena_alloc(a, (base_count + NULL_TERM) * sizeof(const char *)); + if (!result) { + return NULL; + } + for (int j = 0; j < base_count; j++) { + result[j] = bases[j]; + } + result[base_count] = NULL; + return result; +} + +/* C# base_list: iterate children, find base_list node, extract bases. */ +static const char **extract_csharp_base_list(CBMArena *a, TSNode node, const char *source, + uint32_t count) { + for (uint32_t i = 0; i < count; i++) { + TSNode child = ts_node_child(node, i); + if (strcmp(ts_node_type(child), "base_list") != 0) { + continue; + } + const char **result = collect_csharp_bases(a, child, source); + if (result) { + return result; + } + } + return NULL; +} + // Extract base class names from a class node. static const char **extract_base_classes(CBMArena *a, TSNode node, const char *source, CBMLanguage lang) { @@ -1017,51 +1111,11 @@ static const char **extract_base_classes(CBMArena *a, TSNode node, const char *s } } - // C#: explicit base_list handler. The generic fallback (find_base_from_children) - // returns the raw text of the whole `base_list` node, which includes the leading - // `:` separator — producing names like `": IExamService"` that fail registry lookup. - // Iterate the named children directly and strip generic type args. - for (uint32_t i = 0; i < count; i++) { - TSNode child = ts_node_child(node, i); - if (strcmp(ts_node_type(child), "base_list") != 0) { - continue; - } - const char *bases[MAX_BASES]; - int base_count = 0; - uint32_t bnc = ts_node_named_child_count(child); - for (uint32_t bi = 0; bi < bnc && base_count < MAX_BASES_MINUS_1; bi++) { - TSNode bc = ts_node_named_child(child, bi); - const char *bk = ts_node_type(bc); - char *text = NULL; - if (strcmp(bk, "identifier") == 0 || strcmp(bk, "generic_name") == 0 || - strcmp(bk, "qualified_name") == 0) { - text = cbm_node_text(a, bc, source); - } else { - /* Nested wrapper — grab the first named grandchild. */ - TSNode inner = ts_node_named_child(bc, 0); - if (!ts_node_is_null(inner)) { - text = cbm_node_text(a, inner, source); - } - } - if (text && text[0]) { - /* Strip generic type arguments: "List" → "List". */ - char *angle = strchr(text, '<'); - if (angle) { - *angle = '\0'; - } - bases[base_count++] = text; - } - } - if (base_count > 0) { - const char **result = (const char **)cbm_arena_alloc( - a, (base_count + NULL_TERM) * sizeof(const char *)); - if (result) { - for (int j = 0; j < base_count; j++) { - result[j] = bases[j]; - } - result[base_count] = NULL; - return result; - } + // C#: explicit base_list handler + { + const char **csharp_result = extract_csharp_base_list(a, node, source, count); + if (csharp_result) { + return csharp_result; } } diff --git a/internal/cbm/extract_imports.c b/internal/cbm/extract_imports.c index 83fa8854e..239a19408 100644 --- a/internal/cbm/extract_imports.c +++ b/internal/cbm/extract_imports.c @@ -345,6 +345,72 @@ static bool process_es_import_statement(CBMExtractCtx *ctx, TSNode node) { return true; } +/* Handle CommonJS `require("path")` call_expression nodes. The local name + * is derived from the enclosing variable_declarator / assignment when + * possible (so `const foo = require('./foo')` emits local_name="foo"), + * otherwise falls back to the last path component. */ +static bool process_commonjs_require(CBMExtractCtx *ctx, TSNode call) { + CBMArena *a = ctx->arena; + if (ts_node_child_count(call) < MIN_WOLFRAM_CHILDREN) { + return false; + } + /* Callee must be the identifier "require". */ + TSNode fn = ts_node_child_by_field_name(call, TS_FIELD("function")); + if (ts_node_is_null(fn)) { + fn = ts_node_child(call, 0); + } + if (ts_node_is_null(fn)) { + return false; + } + const char *fn_kind = ts_node_type(fn); + if (strcmp(fn_kind, "identifier") != 0) { + return false; + } + char *fn_name = cbm_node_text(a, fn, ctx->source); + if (!fn_name || strcmp(fn_name, "require") != 0) { + return false; + } + + /* First string literal child of the argument list is the module path. */ + TSNode args = ts_node_child_by_field_name(call, TS_FIELD("arguments")); + if (ts_node_is_null(args)) { + return false; + } + uint32_t argc = ts_node_named_child_count(args); + char *path = NULL; + for (uint32_t i = 0; i < argc; i++) { + TSNode arg = ts_node_named_child(args, i); + const char *ak = ts_node_type(arg); + if (strcmp(ak, "string") == 0 || strcmp(ak, "string_literal") == 0 || + strcmp(ak, "template_string") == 0) { + path = strip_quotes(a, cbm_node_text(a, arg, ctx->source)); + break; + } + } + if (!path || !path[0]) { + return false; + } + + /* Infer local name from enclosing variable_declarator. Tree-sitter's JS + * grammar wraps `const foo = require(..)` as + * lexical_declaration → variable_declarator { name: identifier, value: call } */ + const char *local_name = NULL; + TSNode parent = ts_node_parent(call); + if (!ts_node_is_null(parent) && strcmp(ts_node_type(parent), "variable_declarator") == 0) { + TSNode name_node = ts_node_child_by_field_name(parent, TS_FIELD("name")); + if (!ts_node_is_null(name_node) && strcmp(ts_node_type(name_node), "identifier") == 0) { + local_name = cbm_node_text(a, name_node, ctx->source); + } + } + if (!local_name) { + local_name = path_last(a, path); + } + + CBMImport imp = {.local_name = local_name, .module_path = path}; + cbm_imports_push(&ctx->result->imports, a, imp); + return true; +} + #define ES_IMPORT_STACK_CAP CBM_SZ_512 static void walk_es_imports(CBMExtractCtx *ctx, TSNode root) { TSNode stack[ES_IMPORT_STACK_CAP]; @@ -360,6 +426,12 @@ static void walk_es_imports(CBMExtractCtx *ctx, TSNode root) { if (process_es_import_statement(ctx, node)) { push_children = false; } + } else if (strcmp(kind, "call_expression") == 0) { + /* CommonJS require() — only consume the node if we recognized + * it as a require call; otherwise keep traversing the children. */ + if (process_commonjs_require(ctx, node)) { + push_children = false; + } } if (push_children) { diff --git a/internal/cbm/sqlite_writer.c b/internal/cbm/sqlite_writer.c index cb704bac6..7b91cce31 100644 --- a/internal/cbm/sqlite_writer.c +++ b/internal/cbm/sqlite_writer.c @@ -39,12 +39,13 @@ * 32KB pages → page 32769 * 16KB pages → page 65537 */ +#define SQLITE_MAX_PAGE_SIZE 65536 #define CBM_PENDING_BYTE (0x40000000u) #define CBM_PENDING_BYTE_PAGE ((CBM_PENDING_BYTE / CBM_PAGE_SIZE) + 1) /* Skip the pending byte page if allocation lands on it. */ static inline uint32_t cbm_skip_pending_byte(uint32_t pgno) { - return pgno == CBM_PENDING_BYTE_PAGE ? pgno + 1 : pgno; + return pgno == CBM_PENDING_BYTE_PAGE ? pgno + SKIP_ONE : pgno; } #define SCHEMA_FORMAT 4 #define FILE_FORMAT 1 @@ -129,6 +130,7 @@ enum { // SQLite blob serial type offset: serial_type = len*2 + BLOB_SERIAL_BASE. #define BLOB_SERIAL_BASE 12 +#define BLOB_SERIAL_MUL 2 /* serial_type = len * BLOB_SERIAL_MUL + BLOB_SERIAL_BASE */ // SQLite integer storage range limits. #define INT8_MAX_VAL 127 @@ -384,7 +386,7 @@ static void rec_add_text(RecordBuilder *r, const char *s) { } static void rec_add_blob(RecordBuilder *r, const uint8_t *data, int len) { - int64_t st = len > 0 ? (int64_t)len * 2 + BLOB_SERIAL_BASE : 0; + int64_t st = len > 0 ? ((int64_t)len * BLOB_SERIAL_MUL) + BLOB_SERIAL_BASE : 0; uint8_t vbuf[VARINT_MAX_BYTES]; int vlen = put_varint(vbuf, st); dynbuf_append(&r->header, vbuf, vlen); @@ -778,7 +780,8 @@ static uint8_t *build_token_vec_record(const CBMDumpTokenVec *tv, int *out_len) rec_add_text(&r, tv->token); rec_add_blob(&r, tv->vector, tv->vector_len); /* Store IDF as integer × 1000 for fixed-point (avoid float in record) */ - rec_add_int(&r, (int64_t)(tv->idf * 1000.0f)); + enum { IDF_FIXED_POINT_SCALE = 1000 }; + rec_add_int(&r, (int64_t)(tv->idf * IDF_FIXED_POINT_SCALE)); uint8_t *data = rec_finalize(&r, out_len); rec_free(&r); @@ -1539,85 +1542,80 @@ typedef struct { int token_vec_count; } write_db_ctx_t; +/* Callback type for building a record from an item at index i. */ +typedef uint8_t *(*build_record_fn)(const void *items, int i, int *out_len); +typedef int64_t (*get_rowid_fn)(const void *items, int i); + +/* Write a streaming B-tree table from count items, or an empty table if count == 0. */ +static int write_one_table(write_db_ctx_t *w, uint32_t *root, const void *items, int count, + build_record_fn build_rec, get_rowid_fn get_id) { + if (count <= 0 || !items) { + *root = write_table_btree(w->fp, &w->next_page, NULL, NULL, NULL, 0, false); + return 0; + } + PageBuilder pb; + pb_init(&pb, w->fp, w->next_page, false); + for (int i = 0; i < count; i++) { + int rec_len; + uint8_t *rec = build_rec(items, i, &rec_len); + if (!rec) { + return ERR_WRITE_FAILED; + } + int64_t rowid = get_id(items, i); + int64_t prev_id = i > 0 ? get_id(items, i - SKIP_ONE) : 0; + pb_add_table_cell_with_flush(&pb, rowid, rec, rec_len, prev_id); + free(rec); + } + *root = pb_finalize_table(&pb, &w->next_page, get_id(items, count - SKIP_ONE)); + return 0; +} + +/* Adapter functions for write_one_table */ +static uint8_t *adapt_build_node(const void *items, int i, int *out_len) { + return build_node_record(&((const CBMDumpNode *)items)[i], out_len); +} +static int64_t adapt_node_id(const void *items, int i) { + return ((const CBMDumpNode *)items)[i].id; +} +static uint8_t *adapt_build_edge(const void *items, int i, int *out_len) { + return build_edge_record(&((const CBMDumpEdge *)items)[i], out_len); +} +static int64_t adapt_edge_id(const void *items, int i) { + return ((const CBMDumpEdge *)items)[i].id; +} +static uint8_t *adapt_build_vector(const void *items, int i, int *out_len) { + return build_vector_record(&((const CBMDumpVector *)items)[i], out_len); +} +static int64_t adapt_vector_id(const void *items, int i) { + return ((const CBMDumpVector *)items)[i].node_id; +} +static uint8_t *adapt_build_token_vec(const void *items, int i, int *out_len) { + return build_token_vec_record(&((const CBMDumpTokenVec *)items)[i], out_len); +} +static int64_t adapt_token_vec_id(const void *items, int i) { + return ((const CBMDumpTokenVec *)items)[i].id; +} + /* Phase 1: Write node + edge + vector data tables (streaming). */ static int write_data_tables(write_db_ctx_t *w, uint32_t *nodes_root, uint32_t *edges_root, uint32_t *vectors_root, uint32_t *token_vecs_root) { - if (w->node_count > 0) { - PageBuilder pb; - pb_init(&pb, w->fp, w->next_page, false); - for (int i = 0; i < w->node_count; i++) { - int rec_len; - uint8_t *rec = build_node_record(&w->nodes[i], &rec_len); - if (!rec) { - return ERR_WRITE_FAILED; - } - pb_add_table_cell_with_flush(&pb, w->nodes[i].id, rec, rec_len, - i > 0 ? w->nodes[i - SKIP_ONE].id : 0); - free(rec); - } - *nodes_root = pb_finalize_table(&pb, &w->next_page, w->nodes[w->node_count - SKIP_ONE].id); - } else { - *nodes_root = write_table_btree(w->fp, &w->next_page, NULL, NULL, NULL, 0, false); + int rc; + rc = write_one_table(w, nodes_root, w->nodes, w->node_count, adapt_build_node, adapt_node_id); + if (rc != 0) { + return rc; } - - if (w->edge_count > 0) { - PageBuilder pb; - pb_init(&pb, w->fp, w->next_page, false); - for (int i = 0; i < w->edge_count; i++) { - int rec_len; - uint8_t *rec = build_edge_record(&w->edges[i], &rec_len); - if (!rec) { - return ERR_WRITE_FAILED; - } - pb_add_table_cell_with_flush(&pb, w->edges[i].id, rec, rec_len, - i > 0 ? w->edges[i - SKIP_ONE].id : 0); - free(rec); - } - *edges_root = pb_finalize_table(&pb, &w->next_page, w->edges[w->edge_count - SKIP_ONE].id); - } else { - *edges_root = write_table_btree(w->fp, &w->next_page, NULL, NULL, NULL, 0, false); + rc = write_one_table(w, edges_root, w->edges, w->edge_count, adapt_build_edge, adapt_edge_id); + if (rc != 0) { + return rc; } - - /* node_vectors table — uses node_id as rowid (not AUTOINCREMENT) */ - if (w->vector_count > 0 && w->vectors) { - PageBuilder pb; - pb_init(&pb, w->fp, w->next_page, false); - for (int i = 0; i < w->vector_count; i++) { - int rec_len; - uint8_t *rec = build_vector_record(&w->vectors[i], &rec_len); - if (!rec) { - return ERR_WRITE_FAILED; - } - pb_add_table_cell_with_flush(&pb, w->vectors[i].node_id, rec, rec_len, - i > 0 ? w->vectors[i - SKIP_ONE].node_id : 0); - free(rec); - } - *vectors_root = - pb_finalize_table(&pb, &w->next_page, w->vectors[w->vector_count - SKIP_ONE].node_id); - } else { - *vectors_root = write_table_btree(w->fp, &w->next_page, NULL, NULL, NULL, 0, false); + rc = write_one_table(w, vectors_root, w->vectors, w->vector_count, adapt_build_vector, + adapt_vector_id); + if (rc != 0) { + return rc; } - - /* token_vectors table — enriched RI vectors for query-time lookup */ - if (w->token_vec_count > 0 && w->token_vecs) { - PageBuilder pb; - pb_init(&pb, w->fp, w->next_page, false); - for (int i = 0; i < w->token_vec_count; i++) { - int rec_len; - uint8_t *rec = build_token_vec_record(&w->token_vecs[i], &rec_len); - if (!rec) { - return ERR_WRITE_FAILED; - } - pb_add_table_cell_with_flush(&pb, w->token_vecs[i].id, rec, rec_len, - i > 0 ? w->token_vecs[i - SKIP_ONE].id : 0); - free(rec); - } - *token_vecs_root = pb_finalize_table(&pb, &w->next_page, - w->token_vecs[w->token_vec_count - SKIP_ONE].id); - } else { - *token_vecs_root = write_table_btree(w->fp, &w->next_page, NULL, NULL, NULL, 0, false); - } - return 0; + rc = write_one_table(w, token_vecs_root, w->token_vecs, w->token_vec_count, + adapt_build_token_vec, adapt_token_vec_id); + return rc; } /* Phase 2: Write metadata tables (projects, file_hashes, summaries, sqlite_sequence). */ @@ -1662,6 +1660,168 @@ static void write_metadata_tables(write_db_ctx_t *w, uint32_t *projects_root, free(seq2); } +/* Write the SQLite file header on page 1 with master entries. */ +static void write_sqlite_file_header(uint8_t *page1, uint32_t total_pages) { + memcpy(page1, "SQLite format 3\000", 16); + put_u16(page1 + HDR_OFF_CBM_PAGE_SIZE, + CBM_PAGE_SIZE == SQLITE_MAX_PAGE_SIZE ? (uint16_t)SKIP_ONE : (uint16_t)CBM_PAGE_SIZE); + page1[HDR_OFF_WRITE_VERSION] = FILE_FORMAT; + page1[HDR_OFF_READ_VERSION] = FILE_FORMAT; + page1[HDR_OFF_RESERVED] = 0; + page1[HDR_OFF_MAX_EMBED_FRAC] = MAX_EMBED_FRACTION; + page1[HDR_OFF_MIN_EMBED_FRAC] = MIN_EMBED_FRACTION; + page1[HDR_OFF_LEAF_FRAC] = LEAF_PAYLOAD_FRACTION; + put_u32(page1 + HDR_OFF_FILE_CHANGE, SKIP_ONE); + put_u32(page1 + HDR_OFF_DB_SIZE, total_pages); + put_u32(page1 + HDR_OFF_FREELIST_TRUNK, 0); + put_u32(page1 + HDR_OFF_FREELIST_COUNT, 0); + put_u32(page1 + HDR_OFF_SCHEMA_COOKIE, SKIP_ONE); + put_u32(page1 + HDR_OFF_SCHEMA_FORMAT, SCHEMA_FORMAT); + put_u32(page1 + HDR_OFF_DEFAULT_CACHE, 0); + put_u32(page1 + HDR_OFF_AUTOVAC_TOP, 0); + put_u32(page1 + HDR_OFF_TEXT_ENCODING, SKIP_ONE); + put_u32(page1 + HDR_OFF_USER_VERSION, 0); + put_u32(page1 + HDR_OFF_INCR_VACUUM, 0); + put_u32(page1 + HDR_OFF_APP_ID, 0); + put_u32(page1 + HDR_OFF_VERSION_VALID, SKIP_ONE); + put_u32(page1 + HDR_OFF_SQLITE_VERSION, SQLITE_VERSION); +} + +/* Build master records, write page 1 B-tree + file header. */ +static int write_master_page1(FILE *fp, MasterEntry *master, int master_count, uint32_t next_page) { + const uint8_t **master_records = (const uint8_t **)malloc(master_count * sizeof(uint8_t *)); + int *master_lens = (int *)malloc(master_count * sizeof(int)); + int64_t *master_rowids = (int64_t *)malloc(master_count * sizeof(int64_t)); + for (int i = 0; i < master_count; i++) { + master_rowids[i] = i + SKIP_ONE; + master_records[i] = build_master_record(&master[i], &master_lens[i]); + } + + uint8_t page1[CBM_PAGE_SIZE]; + memset(page1, 0, CBM_PAGE_SIZE); + int hdr = SQLITE_HEADER_SIZE; + page1[hdr] = BTREE_LEAF_TABLE; + int content_off = CBM_PAGE_SIZE; + int ptr_off = hdr + BTREE_HEADER_SIZE; + int mcell_count = 0; + + for (int i = 0; i < master_count; i++) { + int cell_len = 0; + uint8_t *cell = + build_table_cell(master_rowids[i], master_records[i], master_lens[i], &cell_len); + int available = content_off - ptr_off - CELL_PTR_SIZE; + if (!cell || cell_len > available) { + free(cell); + for (int j = 0; j < master_count; j++) { + free((void *)master_records[j]); + } + free(master_records); + free(master_lens); + free(master_rowids); + return ERR_MASTER_OVERFLOW; + } + content_off -= cell_len; + memcpy(page1 + content_off, cell, cell_len); + put_u16(page1 + ptr_off, (uint16_t)content_off); + ptr_off += CELL_PTR_SIZE; + mcell_count++; + free(cell); + } + + put_u16(page1 + hdr + HDR_FREEBLOCK_OFF, 0); + put_u16(page1 + hdr + HDR_CELLCOUNT_OFF, (uint16_t)mcell_count); + put_u16(page1 + hdr + HDR_CONTENT_OFF, (uint16_t)content_off); + page1[hdr + HDR_FRAGBYTES_OFF] = 0; + + write_sqlite_file_header(page1, next_page - SKIP_ONE); + + (void)fseek(fp, 0, SEEK_SET); + (void)fwrite(page1, SKIP_ONE, CBM_PAGE_SIZE, fp); + + for (int i = 0; i < master_count; i++) { + free((void *)master_records[i]); + } + free(master_records); + free(master_lens); + free(master_rowids); + return 0; +} + +/* Pad file to exact page boundary. */ +static void pad_file_to_page_boundary(FILE *fp, uint32_t next_page) { + (void)fseek(fp, 0, SEEK_END); + long file_size = ftell(fp); + long expected_size = (long)(next_page - SKIP_ONE) * CBM_PAGE_SIZE; + if (file_size < expected_size) { + uint8_t zero = 0; + (void)fseek(fp, expected_size - SKIP_ONE, SEEK_SET); + (void)fwrite(&zero, SKIP_ONE, SKIP_ONE, fp); + } +} + +/* Build all 4 node index B-trees. Returns 0 on success, ERR_SORT_FAILED on failure. */ +static int build_node_indexes(FILE *fp, uint32_t *next_page, CBMDumpNode *nodes, int node_count, + SortJob *nsorts, uint32_t *label_root, uint32_t *name_root, + uint32_t *file_root, uint32_t *qn_root) { + *label_root = + build_node_index_sorted(fp, next_page, nodes, node_count, nsorts[0].perm, ncol_label); + *name_root = build_node_index_sorted(fp, next_page, nodes, node_count, nsorts[NSORT_NAME].perm, + ncol_name); + *file_root = build_node_index_sorted(fp, next_page, nodes, node_count, nsorts[NSORT_FILE].perm, + ncol_file); + *qn_root = + build_node_index_sorted(fp, next_page, nodes, node_count, nsorts[NSORT_QN].perm, ncol_qn); + if (node_count > 0 && (!*label_root || !*name_root || !*file_root || !*qn_root)) { + return ERR_SORT_FAILED; + } + return 0; +} + +/* Build all 7 edge index B-trees. Returns 0 on success, ERR_SORT_FAILED on failure. */ +static int build_edge_indexes(FILE *fp, uint32_t *next_page, CBMDumpEdge *edges, int edge_count, + SortJob *esorts, uint32_t *source_root, uint32_t *target_root, + uint32_t *type_root, uint32_t *tgt_type_root, uint32_t *src_type_root, + uint32_t *url_path_root, uint32_t *auto_root) { + *source_root = + build_edge_index_sorted(fp, next_page, edges, edge_count, esorts[0].perm, ecell_source); + *target_root = build_edge_index_sorted(fp, next_page, edges, edge_count, + esorts[ESORT_TARGET].perm, ecell_target); + *type_root = build_edge_index_sorted(fp, next_page, edges, edge_count, esorts[ESORT_TYPE].perm, + ecell_type); + *tgt_type_root = build_edge_index_sorted( + fp, next_page, edges, edge_count, esorts[ESORT_PROJ_TGT_TYPE].perm, ecell_proj_target_type); + *src_type_root = build_edge_index_sorted( + fp, next_page, edges, edge_count, esorts[ESORT_PROJ_SRC_TYPE].perm, ecell_proj_source_type); + *url_path_root = build_edge_index_sorted(fp, next_page, edges, edge_count, + esorts[ESORT_URL_PATH].perm, ecell_url_path); + *auto_root = build_edge_index_sorted(fp, next_page, edges, edge_count, + esorts[ESORT_SRC_TGT_TYPE].perm, ecell_src_tgt_type); + if (edge_count > 0 && (!*source_root || !*target_root || !*type_root || !*tgt_type_root || + !*src_type_root || !*url_path_root || !*auto_root)) { + return ERR_SORT_FAILED; + } + return 0; +} + +/* Launch parallel sort threads for all index permutations. */ +static void parallel_sort_indexes(SortJob *nsorts, int n_node, SortJob *esorts, int n_edge) { + cbm_thread_t st[TOTAL_SORT_THREADS]; + int nt = 0; + for (int i = 0; i < n_node; i++) { + if (nsorts[i].count > 0) { + cbm_thread_create(&st[nt++], 0, sort_worker, &nsorts[i]); + } + } + for (int i = 0; i < n_edge; i++) { + if (esorts[i].count > 0) { + cbm_thread_create(&st[nt++], 0, sort_worker, &esorts[i]); + } + } + for (int i = 0; i < nt; i++) { + cbm_thread_join(&st[i]); + } +} + int cbm_write_db(const char *path, const char *project, const char *root_path, const char *indexed_at, CBMDumpNode *nodes, int node_count, CBMDumpEdge *edges, int edge_count, CBMDumpVector *vectors, int vector_count, @@ -1733,72 +1893,41 @@ int cbm_write_db(const char *path, const char *project, const char *root_path, {edge_count, cmp_edge_by_src_tgt_type, NULL}, }; - // Phase 3: Parallel sort of 11 index permutations CBM_PROF_START(t_sort); - { - cbm_thread_t st[TOTAL_SORT_THREADS]; - int nt = 0; - for (int i = 0; i < NODE_SORT_THREADS; i++) { - if (nsorts[i].count > 0) { - cbm_thread_create(&st[nt++], 0, sort_worker, &nsorts[i]); - } - } - for (int i = 0; i < EDGE_SORT_THREADS; i++) { - if (esorts[i].count > 0) { - cbm_thread_create(&st[nt++], 0, sort_worker, &esorts[i]); - } - } - for (int i = 0; i < nt; i++) { - cbm_thread_join(&st[i]); - } - } + parallel_sort_indexes(nsorts, NODE_SORT_THREADS, esorts, EDGE_SORT_THREADS); CBM_PROF_END_N("write_db", "3_parallel_sort_indexes", t_sort, node_count + edge_count); - // Phase 4: Build node index B-trees (SEQUENTIAL) + /* Phase 4-5: Build node + edge index B-trees */ CBM_PROF_START(t_node_idx); - uint32_t idx_nodes_label_root = - build_node_index_sorted(fp, &next_page, nodes, node_count, nsorts[0].perm, ncol_label); - uint32_t idx_nodes_name_root = build_node_index_sorted(fp, &next_page, nodes, node_count, - nsorts[NSORT_NAME].perm, ncol_name); - uint32_t idx_nodes_file_root = build_node_index_sorted(fp, &next_page, nodes, node_count, - nsorts[NSORT_FILE].perm, ncol_file); - uint32_t autoindex_nodes_root = - build_node_index_sorted(fp, &next_page, nodes, node_count, nsorts[NSORT_QN].perm, ncol_qn); - - if ((node_count > 0) && (!idx_nodes_label_root || !idx_nodes_name_root || - !idx_nodes_file_root || !autoindex_nodes_root)) { + uint32_t idx_nodes_label_root; + uint32_t idx_nodes_name_root; + uint32_t idx_nodes_file_root; + uint32_t autoindex_nodes_root; + int nrc = build_node_indexes(fp, &next_page, nodes, node_count, nsorts, &idx_nodes_label_root, + &idx_nodes_name_root, &idx_nodes_file_root, &autoindex_nodes_root); + CBM_PROF_END_N("write_db", "4_node_indexes_seq", t_node_idx, node_count * NODE_SORT_THREADS); + if (nrc != 0) { (void)fclose(fp); - return ERR_SORT_FAILED; + return nrc; } - CBM_PROF_END_N("write_db", "4_node_indexes_seq", t_node_idx, node_count * 4); - // Phase 5: Build edge index B-trees (SEQUENTIAL) CBM_PROF_START(t_edge_idx); - uint32_t idx_edges_source_root = - build_edge_index_sorted(fp, &next_page, edges, edge_count, esorts[0].perm, ecell_source); - uint32_t idx_edges_target_root = build_edge_index_sorted( - fp, &next_page, edges, edge_count, esorts[ESORT_TARGET].perm, ecell_target); - uint32_t idx_edges_type_root = build_edge_index_sorted(fp, &next_page, edges, edge_count, - esorts[ESORT_TYPE].perm, ecell_type); - uint32_t idx_edges_target_type_root = - build_edge_index_sorted(fp, &next_page, edges, edge_count, esorts[ESORT_PROJ_TGT_TYPE].perm, - ecell_proj_target_type); - uint32_t idx_edges_source_type_root = - build_edge_index_sorted(fp, &next_page, edges, edge_count, esorts[ESORT_PROJ_SRC_TYPE].perm, - ecell_proj_source_type); - uint32_t idx_edges_url_path_root = build_edge_index_sorted( - fp, &next_page, edges, edge_count, esorts[ESORT_URL_PATH].perm, ecell_url_path); - uint32_t autoindex_edges_root = build_edge_index_sorted( - fp, &next_page, edges, edge_count, esorts[ESORT_SRC_TGT_TYPE].perm, ecell_src_tgt_type); - - if (edge_count > 0 && - (!idx_edges_source_root || !idx_edges_target_root || !idx_edges_type_root || - !idx_edges_target_type_root || !idx_edges_source_type_root || !idx_edges_url_path_root || - !autoindex_edges_root)) { + uint32_t idx_edges_source_root; + uint32_t idx_edges_target_root; + uint32_t idx_edges_type_root; + uint32_t idx_edges_target_type_root; + uint32_t idx_edges_source_type_root; + uint32_t idx_edges_url_path_root; + uint32_t autoindex_edges_root; + int erc = build_edge_indexes(fp, &next_page, edges, edge_count, esorts, &idx_edges_source_root, + &idx_edges_target_root, &idx_edges_type_root, + &idx_edges_target_type_root, &idx_edges_source_type_root, + &idx_edges_url_path_root, &autoindex_edges_root); + CBM_PROF_END_N("write_db", "5_edge_indexes_seq", t_edge_idx, edge_count * EDGE_SORT_THREADS); + if (erc != 0) { (void)fclose(fp); - return ERR_SORT_FAILED; + return erc; } - CBM_PROF_END_N("write_db", "5_edge_indexes_seq", t_edge_idx, edge_count * 7); // Autoindex for projects(name TEXT PK) — single text column uint32_t autoindex_projects_root; @@ -1899,119 +2028,12 @@ int cbm_write_db(const char *path, const char *project, const char *root_path, }; int master_count = sizeof(master) / sizeof(master[0]); - - // Build master records - const uint8_t **master_records = (const uint8_t **)malloc(master_count * sizeof(uint8_t *)); - int *master_lens = (int *)malloc(master_count * sizeof(int)); - int64_t *master_rowids = (int64_t *)malloc(master_count * sizeof(int64_t)); - for (int i = 0; i < master_count; i++) { - master_rowids[i] = i + SKIP_ONE; - master_records[i] = build_master_record(&master[i], &master_lens[i]); + int rc2 = write_master_page1(fp, master, master_count, next_page); + if (rc2 != 0) { + (void)fclose(fp); + return rc2; } - - // For sqlite_master, we need to write page 1 as the root. - // This is special: page 1 has the 100-byte file header. - // We write master entries as a leaf table B-tree on page 1. - // If they don't fit on one page, we need interior pages (unlikely for ~14 entries). - - // Write master B-tree starting at page 1 - { - uint8_t page1[CBM_PAGE_SIZE]; - memset(page1, 0, CBM_PAGE_SIZE); - - // B-tree header starts at offset 100 on page 1 - int hdr = SQLITE_HEADER_SIZE; - page1[hdr] = BTREE_LEAF_TABLE; // leaf table - int content_off = CBM_PAGE_SIZE; - int ptr_off = hdr + BTREE_HEADER_SIZE; - int mcell_count = 0; - - for (int i = 0; i < master_count; i++) { - int cell_len; - uint8_t *cell = - build_table_cell(master_rowids[i], master_records[i], master_lens[i], &cell_len); - - // Check fit - int available = content_off - ptr_off - CELL_PTR_SIZE; - if (cell_len > available) { - // Master entries should always fit on page 1 (they're small SQL strings). - // If not, this is a bug — fail gracefully. - free(cell); - for (int j = 0; j < master_count; j++) { - free((void *)master_records[j]); - } - free(master_records); - free(master_lens); - free(master_rowids); - (void)fclose(fp); - return ERR_MASTER_OVERFLOW; - } - - content_off -= cell_len; - memcpy(page1 + content_off, cell, cell_len); - put_u16(page1 + ptr_off, (uint16_t)content_off); - ptr_off += CELL_PTR_SIZE; - mcell_count++; - free(cell); - } - - put_u16(page1 + hdr + HDR_FREEBLOCK_OFF, 0); // freeblock - put_u16(page1 + hdr + HDR_CELLCOUNT_OFF, (uint16_t)mcell_count); - put_u16(page1 + hdr + HDR_CONTENT_OFF, (uint16_t)content_off); - page1[hdr + HDR_FRAGBYTES_OFF] = 0; // fragmented - - // Write the 100-byte SQLite file header - memcpy(page1, "SQLite format 3\000", 16); - /* Page size: 65536 is encoded as 1; all others stored directly */ - put_u16(page1 + HDR_OFF_CBM_PAGE_SIZE, - CBM_PAGE_SIZE == 65536 ? (uint16_t)1 : (uint16_t)CBM_PAGE_SIZE); - page1[HDR_OFF_WRITE_VERSION] = FILE_FORMAT; // file format write version - page1[HDR_OFF_READ_VERSION] = FILE_FORMAT; // file format read version - page1[HDR_OFF_RESERVED] = 0; // reserved space per page - page1[HDR_OFF_MAX_EMBED_FRAC] = MAX_EMBED_FRACTION; // max embedded payload fraction - page1[HDR_OFF_MIN_EMBED_FRAC] = MIN_EMBED_FRACTION; // min embedded payload fraction - page1[HDR_OFF_LEAF_FRAC] = LEAF_PAYLOAD_FRACTION; // leaf payload fraction - put_u32(page1 + HDR_OFF_FILE_CHANGE, 0); // file change counter (set below) - put_u32(page1 + HDR_OFF_DB_SIZE, next_page - SKIP_ONE); // total pages - put_u32(page1 + HDR_OFF_FREELIST_TRUNK, 0); // first freelist trunk page - put_u32(page1 + HDR_OFF_FREELIST_COUNT, 0); // total freelist pages - put_u32(page1 + HDR_OFF_SCHEMA_COOKIE, SKIP_ONE); // schema cookie - put_u32(page1 + HDR_OFF_SCHEMA_FORMAT, SCHEMA_FORMAT); // schema format number - put_u32(page1 + HDR_OFF_DEFAULT_CACHE, 0); // default page cache size - put_u32(page1 + HDR_OFF_AUTOVAC_TOP, 0); // largest root page (auto-vacuum) - put_u32(page1 + HDR_OFF_TEXT_ENCODING, SKIP_ONE); // text encoding: UTF-8 - put_u32(page1 + HDR_OFF_USER_VERSION, 0); // user version - put_u32(page1 + HDR_OFF_INCR_VACUUM, 0); // incremental vacuum mode - put_u32(page1 + HDR_OFF_APP_ID, 0); // application ID - // Bytes 72-91: reserved for expansion (zeros) - put_u32(page1 + HDR_OFF_VERSION_VALID, SKIP_ONE); // version-valid-for (change counter) - put_u32(page1 + HDR_OFF_SQLITE_VERSION, SQLITE_VERSION); // SQLite version number - // Set file change counter = version-valid-for = 1 - put_u32(page1 + HDR_OFF_FILE_CHANGE, SKIP_ONE); - - (void)fseek(fp, 0, SEEK_SET); - (void)fwrite(page1, SKIP_ONE, CBM_PAGE_SIZE, fp); - } - - for (int i = 0; i < master_count; i++) { - free((void *)master_records[i]); - } - free(master_records); - free(master_lens); - free(master_rowids); - - // Ensure file size is exactly next_page * CBM_PAGE_SIZE - // (pad any remaining space) - (void)fseek(fp, 0, SEEK_END); - long file_size = ftell(fp); - long expected_size = (long)(next_page - SKIP_ONE) * CBM_PAGE_SIZE; - if (file_size < expected_size) { - // Pad with zeros - uint8_t zero = 0; - (void)fseek(fp, expected_size - SKIP_ONE, SEEK_SET); - (void)fwrite(&zero, SKIP_ONE, SKIP_ONE, fp); - } - + pad_file_to_page_boundary(fp, next_page); (void)fclose(fp); return 0; } diff --git a/internal/cbm/sqlite_writer.h b/internal/cbm/sqlite_writer.h index ae0eaf676..a740e905a 100644 --- a/internal/cbm/sqlite_writer.h +++ b/internal/cbm/sqlite_writer.h @@ -28,14 +28,14 @@ typedef struct { } CBMDumpEdge; typedef struct { - int64_t node_id; // final sequential ID (matches nodes.id) + int64_t node_id; // final sequential ID (matches nodes.id) const char *project; const uint8_t *vector; // int8-quantized vector blob int vector_len; // length in bytes (e.g. 256 for d=256) } CBMDumpVector; typedef struct { - int64_t id; // sequential ID (1..T) + int64_t id; // sequential ID (1..T) const char *project; const char *token; // the token string const uint8_t *vector; // int8-quantized enriched RI vector blob diff --git a/src/foundation/profile.c b/src/foundation/profile.c index 2de65c372..174a1bcc6 100644 --- a/src/foundation/profile.c +++ b/src/foundation/profile.c @@ -7,6 +7,15 @@ #include #include +#include + +enum { + PROF_BUF_LEN = 32, + PROF_NS_PER_US = 1000, + PROF_US_PER_MS = 1000, + PROF_US_PER_SEC = 1000000, +}; +#define PROF_US_PER_SEC_D 1000000.0 bool cbm_profile_active = false; @@ -25,32 +34,33 @@ void cbm_profile_now(struct timespec *ts) { cbm_clock_gettime(CLOCK_MONOTONIC, ts); } -void cbm_profile_log_elapsed(const char *phase, const char *sub, - const struct timespec *start, long items) { +void cbm_profile_log_elapsed(const char *phase, const char *sub, const struct timespec *start, + long items) { struct timespec now; cbm_clock_gettime(CLOCK_MONOTONIC, &now); - long us = (long)(now.tv_sec - start->tv_sec) * 1000000L + - ((long)now.tv_nsec - (long)start->tv_nsec) / 1000L; - long ms = us / 1000; + long us = ((long)(now.tv_sec - start->tv_sec) * PROF_US_PER_SEC) + + ((now.tv_nsec - start->tv_nsec) / PROF_NS_PER_US); + long ms = us / PROF_US_PER_MS; - char ms_buf[32], us_buf[32], items_buf[32], rate_buf[32]; + char ms_buf[PROF_BUF_LEN]; + char us_buf[PROF_BUF_LEN]; + char items_buf[PROF_BUF_LEN]; snprintf(ms_buf, sizeof(ms_buf), "%ld", ms); snprintf(us_buf, sizeof(us_buf), "%ld", us); if (items > 0 && us > 0) { - long rate = (long)((double)items * 1000000.0 / (double)us); + long rate = (long)((double)items * PROF_US_PER_SEC_D / (double)us); + char rate_buf[PROF_BUF_LEN]; snprintf(items_buf, sizeof(items_buf), "%ld", items); snprintf(rate_buf, sizeof(rate_buf), "%ld", rate); - cbm_log_info("prof", "phase", phase, "sub", sub, - "ms", ms_buf, "us", us_buf, - "items", items_buf, "rate_per_s", rate_buf); + cbm_log_info("prof", "phase", phase, "sub", sub, "ms", ms_buf, "us", us_buf, "items", + items_buf, "rate_per_s", rate_buf); } else if (items > 0) { snprintf(items_buf, sizeof(items_buf), "%ld", items); - cbm_log_info("prof", "phase", phase, "sub", sub, - "ms", ms_buf, "us", us_buf, "items", items_buf); + cbm_log_info("prof", "phase", phase, "sub", sub, "ms", ms_buf, "us", us_buf, "items", + items_buf); } else { - cbm_log_info("prof", "phase", phase, "sub", sub, - "ms", ms_buf, "us", us_buf); + cbm_log_info("prof", "phase", phase, "sub", sub, "ms", ms_buf, "us", us_buf); } } diff --git a/src/foundation/profile.h b/src/foundation/profile.h index 64abf487a..66b5471ec 100644 --- a/src/foundation/profile.h +++ b/src/foundation/profile.h @@ -32,26 +32,27 @@ void cbm_profile_now(struct timespec *ts); /* Log elapsed time since `start` for the given phase/subphase. * `items` = optional count to compute rate (pass 0 to skip). */ -void cbm_profile_log_elapsed(const char *phase, const char *sub, - const struct timespec *start, long items); +void cbm_profile_log_elapsed(const char *phase, const char *sub, const struct timespec *start, + long items); /* Zero-overhead macros: a single runtime check gates everything. */ #define CBM_PROF_START(var) \ - struct timespec var; \ - if (cbm_profile_active) cbm_profile_now(&(var)) + struct timespec var; \ + if (cbm_profile_active) \ + cbm_profile_now(&(var)) -#define CBM_PROF_END(phase, sub, start_var) \ - do { \ - if (cbm_profile_active) { \ +#define CBM_PROF_END(phase, sub, start_var) \ + do { \ + if (cbm_profile_active) { \ cbm_profile_log_elapsed((phase), (sub), &(start_var), 0); \ - } \ + } \ } while (0) -#define CBM_PROF_END_N(phase, sub, start_var, items) \ - do { \ - if (cbm_profile_active) { \ +#define CBM_PROF_END_N(phase, sub, start_var, items) \ + do { \ + if (cbm_profile_active) { \ cbm_profile_log_elapsed((phase), (sub), &(start_var), (long)(items)); \ - } \ + } \ } while (0) #endif /* CBM_PROFILE_H */ diff --git a/src/foundation/str_util.c b/src/foundation/str_util.c index e432b8a58..738883e27 100644 --- a/src/foundation/str_util.c +++ b/src/foundation/str_util.c @@ -7,6 +7,12 @@ #include #include +enum { + JSON_ESC_LEN = 2, /* escaped char takes 2 bytes (backslash + char) */ + JSON_NUL_RESERVE = 1, /* reserve 1 byte for NUL terminator */ + JSON_CTRL_LIMIT = 0x20, /* ASCII control character upper bound */ +}; + char *cbm_path_join(CBMArena *a, const char *base, const char *name) { if (!base || !name) { return NULL; @@ -262,25 +268,41 @@ bool cbm_validate_shell_arg(const char *s) { } int cbm_json_escape(char *buf, int bufsize, const char *src) { - if (!buf || bufsize <= 0) return 0; - if (!src) { buf[0] = '\0'; return 0; } + if (!buf || bufsize <= 0) { + return 0; + } + if (!src) { + buf[0] = '\0'; + return 0; + } int pos = 0; - for (int i = 0; src[i] && pos < bufsize - 1; i++) { + for (int i = 0; src[i] && pos < bufsize - JSON_NUL_RESERVE; i++) { unsigned char c = (unsigned char)src[i]; if (c == '"' || c == '\\') { - if (pos + 2 > bufsize - 1) break; + if (pos + JSON_ESC_LEN > bufsize - JSON_NUL_RESERVE) { + break; + } buf[pos++] = '\\'; buf[pos++] = (char)c; } else if (c == '\n') { - if (pos + 2 > bufsize - 1) break; - buf[pos++] = '\\'; buf[pos++] = 'n'; + if (pos + JSON_ESC_LEN > bufsize - JSON_NUL_RESERVE) { + break; + } + buf[pos++] = '\\'; + buf[pos++] = 'n'; } else if (c == '\r') { - if (pos + 2 > bufsize - 1) break; - buf[pos++] = '\\'; buf[pos++] = 'r'; + if (pos + JSON_ESC_LEN > bufsize - JSON_NUL_RESERVE) { + break; + } + buf[pos++] = '\\'; + buf[pos++] = 'r'; } else if (c == '\t') { - if (pos + 2 > bufsize - 1) break; - buf[pos++] = '\\'; buf[pos++] = 't'; - } else if (c < 0x20) { + if (pos + JSON_ESC_LEN > bufsize - JSON_NUL_RESERVE) { + break; + } + buf[pos++] = '\\'; + buf[pos++] = 't'; + } else if (c < JSON_CTRL_LIMIT) { /* Other control chars: skip */ continue; } else { diff --git a/src/graph_buffer/graph_buffer.c b/src/graph_buffer/graph_buffer.c index 0456a1b98..104e850e1 100644 --- a/src/graph_buffer/graph_buffer.c +++ b/src/graph_buffer/graph_buffer.c @@ -11,6 +11,7 @@ #include "foundation/constants.h" enum { + GB_ERR = -1, GB_COL_2 = 2, GB_COL_3 = 3, GB_COL_4 = 4, @@ -18,6 +19,8 @@ enum { GB_COL_6 = 6, GB_COL_7 = 7, GB_URL_PATH_PREFIX = 12, /* strlen(""url_path":"") */ + GB_MIN_FOR_DEDUP = 2, /* need at least 2 vectors to sort+dedup */ + GB_DEDUP_LOOKAHEAD = 1, /* compare current with next element */ }; #include "graph_buffer/graph_buffer.h" #include "store/store.h" @@ -453,7 +456,7 @@ void cbm_gbuf_free(cbm_gbuf_t *gb) { int cbm_gbuf_store_vector(cbm_gbuf_t *gb, int64_t node_id, const uint8_t *vector, int vector_len) { if (!gb || !vector || vector_len <= 0) { - return -1; + return GB_ERR; } enum { VEC_INIT_CAP = 1024, VEC_GROW = 2 }; if (gb->dump_vector_count >= gb->dump_vector_cap) { @@ -461,7 +464,7 @@ int cbm_gbuf_store_vector(cbm_gbuf_t *gb, int64_t node_id, const uint8_t *vector gb->dump_vector_cap < VEC_INIT_CAP ? VEC_INIT_CAP : gb->dump_vector_cap * VEC_GROW; CBMDumpVector *grown = realloc(gb->dump_vectors, (size_t)new_cap * sizeof(CBMDumpVector)); if (!grown) { - return -1; + return GB_ERR; } gb->dump_vectors = grown; gb->dump_vector_cap = new_cap; @@ -469,7 +472,7 @@ int cbm_gbuf_store_vector(cbm_gbuf_t *gb, int64_t node_id, const uint8_t *vector /* Copy vector data */ uint8_t *vec_copy = malloc((size_t)vector_len); if (!vec_copy) { - return -1; + return GB_ERR; } memcpy(vec_copy, vector, (size_t)vector_len); @@ -485,23 +488,23 @@ int cbm_gbuf_store_vector(cbm_gbuf_t *gb, int64_t node_id, const uint8_t *vector int cbm_gbuf_store_token_vector(cbm_gbuf_t *gb, const char *token, const uint8_t *vector, int vector_len, float idf) { if (!gb || !token || !vector || vector_len <= 0) { - return -1; + return GB_ERR; } enum { TV_INIT_CAP = 256, TV_GROW = 2 }; if (gb->dump_token_vec_count >= gb->dump_token_vec_cap) { - int new_cap = gb->dump_token_vec_cap < TV_INIT_CAP ? TV_INIT_CAP - : gb->dump_token_vec_cap * TV_GROW; + int new_cap = + gb->dump_token_vec_cap < TV_INIT_CAP ? TV_INIT_CAP : gb->dump_token_vec_cap * TV_GROW; CBMDumpTokenVec *grown = realloc(gb->dump_token_vecs, (size_t)new_cap * sizeof(CBMDumpTokenVec)); if (!grown) { - return -1; + return GB_ERR; } gb->dump_token_vecs = grown; gb->dump_token_vec_cap = new_cap; } uint8_t *vec_copy = malloc((size_t)vector_len); if (!vec_copy) { - return -1; + return GB_ERR; } memcpy(vec_copy, vector, (size_t)vector_len); @@ -1249,23 +1252,106 @@ static CBMDumpEdge *build_dump_edges(cbm_gbuf_t *gb, const int64_t *temp_to_fina return dump_edges; } +/* Remap vector node IDs through temp_to_final, sort by ID, deduplicate. */ +static void remap_sort_dedup_vectors(cbm_gbuf_t *gb, const int64_t *temp_to_final, + int64_t max_temp_id) { + int remapped = 0; + int dropped = 0; + for (int i = 0; i < gb->dump_vector_count; i++) { + int64_t old_id = gb->dump_vectors[i].node_id; + int64_t new_id = (old_id > 0 && old_id < max_temp_id) ? temp_to_final[old_id] : 0; + if (new_id > 0) { + gb->dump_vectors[remapped] = gb->dump_vectors[i]; + gb->dump_vectors[remapped].node_id = new_id; + remapped++; + } else { + dropped++; + } + } + if (dropped > 0) { + char r_buf[CBM_SZ_16]; + char d_buf[CBM_SZ_16]; + snprintf(r_buf, sizeof(r_buf), "%d", remapped); + snprintf(d_buf, sizeof(d_buf), "%d", dropped); + cbm_log_info("dump.vectors.remap", "remapped", r_buf, "dropped", d_buf); + } + gb->dump_vector_count = remapped; + + if (gb->dump_vector_count >= GB_MIN_FOR_DEDUP) { + qsort(gb->dump_vectors, (size_t)gb->dump_vector_count, sizeof(CBMDumpVector), + cmp_dump_vectors_by_id); + int deduped = 0; + for (int i = 0; i < gb->dump_vector_count; i++) { + if (i + GB_DEDUP_LOOKAHEAD < gb->dump_vector_count && + gb->dump_vectors[i].node_id == gb->dump_vectors[i + GB_DEDUP_LOOKAHEAD].node_id) { + continue; + } + gb->dump_vectors[deduped++] = gb->dump_vectors[i]; + } + gb->dump_vector_count = deduped; + } +} + +static void log_dump_summary(int node_count, int edge_count) { + char b1[CBM_SZ_16]; + char b2[CBM_SZ_16]; + snprintf(b1, sizeof(b1), "%d", node_count); + snprintf(b2, sizeof(b2), "%d", edge_count); + cbm_log_info("gbuf.dump", "nodes", b1, "edges", b2); +} + +static void free_dump_resources(char **url_paths, int edge_count, CBMDumpEdge *dump_edges, + CBMDumpNode *dump_nodes, int64_t *temp_to_final) { + for (int i = 0; i < edge_count; i++) { + free(url_paths[i]); + } + free(url_paths); + free(dump_edges); + free(dump_nodes); + free(temp_to_final); +} + +static int count_live_nodes(cbm_gbuf_t *gb) { + int count = 0; + for (int i = 0; i < gb->nodes.count; i++) { + cbm_gbuf_node_t *n = gb->nodes.items[i]; + if (n->qualified_name && cbm_ht_get(gb->node_by_qn, n->qualified_name)) { + count++; + } + } + return count; +} + +static void generate_iso_timestamp(char *buf, size_t buf_size) { + time_t now = time(NULL); + struct tm tm_buf; + struct tm *tm_val = cbm_gmtime_r(&now, &tm_buf); + if (strftime(buf, buf_size, "%Y-%m-%dT%H:%M:%SZ", tm_val) == 0) { + snprintf(buf, buf_size, "1970-01-01T00:00:00Z"); + } +} + +/* Release lookup indexes then remap+sort+dedup vectors for the B-tree writer. */ +static void release_and_remap_vectors(cbm_gbuf_t *gb, const int64_t *temp_to_final, + int64_t max_temp_id) { + CBM_PROF_START(t_release_idx); + release_gbuf_indexes(gb); + CBM_PROF_END("dump", "4_release_gbuf_indexes", t_release_idx); + + CBM_PROF_START(t_vec_remap); + remap_sort_dedup_vectors(gb, temp_to_final, max_temp_id); + CBM_PROF_END_N("dump", "5_vector_remap_sort", t_vec_remap, gb->dump_vector_count); +} + int cbm_gbuf_dump_to_sqlite(cbm_gbuf_t *gb, const char *path) { if (!gb || !path) { return CBM_NOT_FOUND; } - /* Sub-phase: Count live nodes (not deleted from QN index) */ CBM_PROF_START(t_count); - int live_count = 0; - for (int i = 0; i < gb->nodes.count; i++) { - cbm_gbuf_node_t *n = gb->nodes.items[i]; - if (n->qualified_name && cbm_ht_get(gb->node_by_qn, n->qualified_name)) { - live_count++; - } - } + int live_count = count_live_nodes(gb); CBM_PROF_END_N("dump", "1_count_live_nodes", t_count, live_count); - /* Sub-phase: Build temp→final ID mapping + dump_nodes array */ CBM_PROF_START(t_build_nodes); int64_t max_temp_id = gb->next_id; int64_t *temp_to_final = calloc((size_t)max_temp_id, sizeof(int64_t)); @@ -1278,7 +1364,6 @@ int cbm_gbuf_dump_to_sqlite(cbm_gbuf_t *gb, const char *path) { build_dump_nodes(gb, live_count, temp_to_final, max_temp_id, &node_idx); CBM_PROF_END_N("dump", "2_build_dump_nodes", t_build_nodes, node_idx); - /* Sub-phase: Build dump_edges array with remapped IDs */ CBM_PROF_START(t_build_edges); int edge_idx = 0; char **url_paths = NULL; @@ -1286,92 +1371,20 @@ int cbm_gbuf_dump_to_sqlite(cbm_gbuf_t *gb, const char *path) { build_dump_edges(gb, temp_to_final, max_temp_id, &edge_idx, &url_paths); CBM_PROF_END_N("dump", "3_build_dump_edges", t_build_edges, edge_idx); - /* Generate ISO 8601 timestamp */ - time_t now = time(NULL); - struct tm tm_buf; - struct tm *tm_val = cbm_gmtime_r(&now, &tm_buf); char indexed_at[CBM_SZ_64]; - if (strftime(indexed_at, sizeof(indexed_at), "%Y-%m-%dT%H:%M:%SZ", tm_val) == 0) { - snprintf(indexed_at, sizeof(indexed_at), "1970-01-01T00:00:00Z"); - } + generate_iso_timestamp(indexed_at, sizeof(indexed_at)); - /* Sub-phase: Release lookup tables — frees 200-400MB on large codebases */ - CBM_PROF_START(t_release_idx); - release_gbuf_indexes(gb); - CBM_PROF_END("dump", "4_release_gbuf_indexes", t_release_idx); - - /* Sub-phase: Remap + sort + dedup node_vectors for B-tree writer */ - CBM_PROF_START(t_vec_remap); - /* Remap node_vectors IDs through temp_to_final (same as edges). - * temp_to_final maps gbuf node IDs → sequential dump IDs (1-based). - * Drop vectors whose gbuf ID has no mapping (node was filtered out). */ - { - int remapped = 0; - int dropped = 0; - for (int i = 0; i < gb->dump_vector_count; i++) { - int64_t old_id = gb->dump_vectors[i].node_id; - int64_t new_id = (old_id > 0 && old_id < max_temp_id) ? temp_to_final[old_id] : 0; - if (new_id > 0) { - gb->dump_vectors[remapped] = gb->dump_vectors[i]; - gb->dump_vectors[remapped].node_id = new_id; - remapped++; - } else { - dropped++; - } - } - if (dropped > 0) { - char r_buf[CBM_SZ_16], d_buf[CBM_SZ_16]; - snprintf(r_buf, sizeof(r_buf), "%d", remapped); - snprintf(d_buf, sizeof(d_buf), "%d", dropped); - cbm_log_info("dump.vectors.remap", "remapped", r_buf, "dropped", d_buf); - } - gb->dump_vector_count = remapped; - } - /* Sort vectors by remapped node_id — B-tree writer requires ascending keys. - * Also deduplicate: parallel workers may store the same node_id twice. */ - if (gb->dump_vector_count > 1) { - qsort(gb->dump_vectors, (size_t)gb->dump_vector_count, sizeof(CBMDumpVector), - cmp_dump_vectors_by_id); - /* Dedup: keep last occurrence of each node_id (latest vector wins) */ - int deduped = 0; - for (int i = 0; i < gb->dump_vector_count; i++) { - if (i + 1 < gb->dump_vector_count && - gb->dump_vectors[i].node_id == gb->dump_vectors[i + 1].node_id) { - continue; /* skip duplicate, keep the later one */ - } - gb->dump_vectors[deduped++] = gb->dump_vectors[i]; - } - gb->dump_vector_count = deduped; - } - CBM_PROF_END_N("dump", "5_vector_remap_sort", t_vec_remap, gb->dump_vector_count); + release_and_remap_vectors(gb, temp_to_final, max_temp_id); /* Sub-phase: Write SQLite DB file (B-tree writer) */ CBM_PROF_START(t_write_db); - /* Write directly to final path — no .tmp + rename. - * Callers must delete the old .db before calling this (reindex) - * or ensure no file exists (first index). */ int rc = cbm_write_db(path, gb->project, gb->root_path, indexed_at, dump_nodes, node_idx, dump_edges, edge_idx, gb->dump_vectors, gb->dump_vector_count, gb->dump_token_vecs, gb->dump_token_vec_count); CBM_PROF_END_N("dump", "6_write_db_btree", t_write_db, node_idx + edge_idx); - { - char b1[CBM_SZ_16]; - char b2[CBM_SZ_16]; - snprintf(b1, sizeof(b1), "%d", node_idx); - snprintf(b2, sizeof(b2), "%d", edge_idx); - cbm_log_info("gbuf.dump", "nodes", b1, "edges", b2); - } - - /* Cleanup */ - for (int i = 0; i < edge_idx; i++) { - free(url_paths[i]); - } - free(url_paths); - free(dump_edges); - free(dump_nodes); - free(temp_to_final); - + log_dump_summary(node_idx, edge_idx); + free_dump_resources(url_paths, edge_idx, dump_edges, dump_nodes, temp_to_final); return rc; } diff --git a/src/mcp/mcp.c b/src/mcp/mcp.c index 54cecff66..6b5cec542 100644 --- a/src/mcp/mcp.c +++ b/src/mcp/mcp.c @@ -1042,8 +1042,38 @@ static void enrich_connected(yyjson_mut_doc *doc, yyjson_mut_val *item, cbm_stor * anything that isn't alnum or underscore is dropped, so the caller can't * inject FTS5 operators or double-quoted phrases. Returns the number of * tokens emitted (0 if the query contained no usable terms). */ +enum { + BM25_MIN_BUF = 2, /* minimum buffer size: at least NUL + one char */ + BM25_SEP_RESERVE = 1, + BM25_QUERY_BUF = 1024, + BM25_DEFAULT_LIMIT = 100, + BM25_COL_ID = 0, + BM25_COL_LABEL = 1, + BM25_COL_NAME = 2, + BM25_COL_QN = 3, + BM25_COL_FILE = 4, + BM25_COL_START = 5, + BM25_COL_END = 6, + BM25_COL_RANK = 7, + BM25_BIND_QUERY = 1, + BM25_BIND_PROJECT = 2, + BM25_BIND_LIMIT = 3, + BM25_BIND_OFFSET = 4, + BM25_SQL_AUTO_LEN = -1, +}; + +/* Module-local SQLITE_TRANSIENT wrapper to dodge performance-no-int-to-ptr. + * See the matching helper in src/store/store.c for the same pattern. */ +static sqlite3_destructor_type mcp_sqlite_transient(void) { + static const volatile intptr_t raw = -1; + sqlite3_destructor_type dtor = NULL; + memcpy(&dtor, (const void *)&raw, sizeof(dtor)); + return dtor; +} +#define MCP_SQLITE_TRANSIENT (mcp_sqlite_transient()) + static int bm25_build_match(const char *query, char *out, size_t out_size) { - if (!query || !out || out_size < 2) { + if (!query || !out || out_size < BM25_MIN_BUF) { return 0; } size_t pos = 0; @@ -1068,7 +1098,7 @@ static int bm25_build_match(const char *query, char *out, size_t out_size) { } const char *sep = (tokens > 0) ? " OR " : ""; size_t sep_len = strlen(sep); - if (pos + sep_len + tok_len + 1 >= out_size) { + if (pos + sep_len + tok_len + BM25_SEP_RESERVE >= out_size) { break; /* out of room — stop cleanly, keep what we have */ } memcpy(out + pos, sep, sep_len); @@ -1090,7 +1120,7 @@ static char *bm25_search(cbm_store_t *store, const char *project, const char *qu if (!db) { return NULL; } - char fts_query[1024]; + char fts_query[BM25_QUERY_BUF]; int tok_count = bm25_build_match(query, fts_query, sizeof(fts_query)); if (tok_count == 0) { return NULL; @@ -1116,13 +1146,13 @@ static char *bm25_search(cbm_store_t *store, const char *project, const char *qu "LIMIT ?3 OFFSET ?4"; sqlite3_stmt *stmt = NULL; - if (sqlite3_prepare_v2(db, sql, -1, &stmt, NULL) != SQLITE_OK) { + if (sqlite3_prepare_v2(db, sql, BM25_SQL_AUTO_LEN, &stmt, NULL) != SQLITE_OK) { return NULL; } - sqlite3_bind_text(stmt, 1, fts_query, -1, SQLITE_TRANSIENT); - sqlite3_bind_text(stmt, 2, project, -1, SQLITE_TRANSIENT); - sqlite3_bind_int(stmt, 3, limit > 0 ? limit : 100); - sqlite3_bind_int(stmt, 4, offset > 0 ? offset : 0); + sqlite3_bind_text(stmt, BM25_BIND_QUERY, fts_query, BM25_SQL_AUTO_LEN, MCP_SQLITE_TRANSIENT); + sqlite3_bind_text(stmt, BM25_BIND_PROJECT, project, BM25_SQL_AUTO_LEN, MCP_SQLITE_TRANSIENT); + sqlite3_bind_int(stmt, BM25_BIND_LIMIT, limit > 0 ? limit : BM25_DEFAULT_LIMIT); + sqlite3_bind_int(stmt, BM25_BIND_OFFSET, offset > 0 ? offset : 0); /* Count total hits (for pagination) in a separate cheap query. */ int total = 0; @@ -1132,9 +1162,11 @@ static char *bm25_search(cbm_store_t *store, const char *project, const char *qu "WHERE nodes_fts MATCH ?1 AND n.project = ?2 " " AND n.label NOT IN ('File','Folder','Module','Section','Variable','Project')"; sqlite3_stmt *cs = NULL; - if (sqlite3_prepare_v2(db, count_sql, -1, &cs, NULL) == SQLITE_OK) { - sqlite3_bind_text(cs, 1, fts_query, -1, SQLITE_TRANSIENT); - sqlite3_bind_text(cs, 2, project, -1, SQLITE_TRANSIENT); + if (sqlite3_prepare_v2(db, count_sql, BM25_SQL_AUTO_LEN, &cs, NULL) == SQLITE_OK) { + sqlite3_bind_text(cs, BM25_BIND_QUERY, fts_query, BM25_SQL_AUTO_LEN, + MCP_SQLITE_TRANSIENT); + sqlite3_bind_text(cs, BM25_BIND_PROJECT, project, BM25_SQL_AUTO_LEN, + MCP_SQLITE_TRANSIENT); if (sqlite3_step(cs) == SQLITE_ROW) { total = sqlite3_column_int(cs, 0); } @@ -1153,16 +1185,16 @@ static char *bm25_search(cbm_store_t *store, const char *project, const char *qu while (sqlite3_step(stmt) == SQLITE_ROW) { yyjson_mut_val *item = yyjson_mut_obj(doc); yyjson_mut_obj_add_strcpy(doc, item, "name", - (const char *)sqlite3_column_text(stmt, 2)); + (const char *)sqlite3_column_text(stmt, BM25_COL_NAME)); yyjson_mut_obj_add_strcpy(doc, item, "qualified_name", - (const char *)sqlite3_column_text(stmt, 3)); + (const char *)sqlite3_column_text(stmt, BM25_COL_QN)); yyjson_mut_obj_add_strcpy(doc, item, "label", - (const char *)sqlite3_column_text(stmt, 1)); + (const char *)sqlite3_column_text(stmt, BM25_COL_LABEL)); yyjson_mut_obj_add_strcpy(doc, item, "file_path", - (const char *)sqlite3_column_text(stmt, 4)); - yyjson_mut_obj_add_int(doc, item, "start_line", sqlite3_column_int(stmt, 5)); - yyjson_mut_obj_add_int(doc, item, "end_line", sqlite3_column_int(stmt, 6)); - yyjson_mut_obj_add_real(doc, item, "rank", sqlite3_column_double(stmt, 7)); + (const char *)sqlite3_column_text(stmt, BM25_COL_FILE)); + yyjson_mut_obj_add_int(doc, item, "start_line", sqlite3_column_int(stmt, BM25_COL_START)); + yyjson_mut_obj_add_int(doc, item, "end_line", sqlite3_column_int(stmt, BM25_COL_END)); + yyjson_mut_obj_add_real(doc, item, "rank", sqlite3_column_double(stmt, BM25_COL_RANK)); yyjson_mut_arr_add_val(results, item); emitted++; } @@ -1176,6 +1208,98 @@ static char *bm25_search(cbm_store_t *store, const char *project, const char *qu return json; } +/* Emit the cbm_store_search results as a JSON "results" array on the doc. */ +static void emit_search_results(yyjson_mut_doc *doc, yyjson_mut_val *root, + const cbm_search_output_t *out, cbm_store_t *store, + const char *relationship, bool include_connected, int offset) { + yyjson_mut_obj_add_int(doc, root, "total", out->total); + yyjson_mut_val *results = yyjson_mut_arr(doc); + for (int i = 0; i < out->count; i++) { + cbm_search_result_t *sr = &out->results[i]; + yyjson_mut_val *item = yyjson_mut_obj(doc); + yyjson_mut_obj_add_str(doc, item, "name", sr->node.name ? sr->node.name : ""); + yyjson_mut_obj_add_str(doc, item, "qualified_name", + sr->node.qualified_name ? sr->node.qualified_name : ""); + yyjson_mut_obj_add_str(doc, item, "label", sr->node.label ? sr->node.label : ""); + yyjson_mut_obj_add_str(doc, item, "file_path", + sr->node.file_path ? sr->node.file_path : ""); + yyjson_mut_obj_add_int(doc, item, "in_degree", sr->in_degree); + yyjson_mut_obj_add_int(doc, item, "out_degree", sr->out_degree); + if (include_connected && sr->node.id > 0) { + enrich_connected(doc, item, store, sr->node.id, relationship); + } + yyjson_mut_arr_add_val(results, item); + } + yyjson_mut_obj_add_val(doc, root, "results", results); + yyjson_mut_obj_add_bool(doc, root, "has_more", out->total > offset + out->count); +} + +/* Extract keyword strings from a yyjson array into `keywords`. Returns the + * number of strings copied (capped at `max_out`). */ +static int extract_semantic_keywords(yyjson_val *sq_val, const char **keywords, int max_out) { + int kw_count = (int)yyjson_arr_size(sq_val); + if (kw_count > max_out) { + kw_count = max_out; + } + size_t kw_idx = 0; + size_t kw_max = 0; + yyjson_val *kw_val; + int ki = 0; + yyjson_arr_foreach(sq_val, kw_idx, kw_max, kw_val) { + if (ki < kw_count && yyjson_is_str(kw_val)) { + keywords[ki++] = yyjson_get_str(kw_val); + } + } + return ki; +} + +/* Emit cbm_vector_result_t entries as a "semantic_results" array on the doc. */ +static void emit_semantic_results(yyjson_mut_doc *doc, yyjson_mut_val *root, + cbm_vector_result_t *vresults, int vcount) { + yyjson_mut_val *sem_results = yyjson_mut_arr(doc); + for (int v = 0; v < vcount; v++) { + yyjson_mut_val *vitem = yyjson_mut_obj(doc); + yyjson_mut_obj_add_strcpy(doc, vitem, "name", vresults[v].name); + yyjson_mut_obj_add_strcpy(doc, vitem, "qualified_name", vresults[v].qualified_name); + yyjson_mut_obj_add_strcpy(doc, vitem, "label", vresults[v].label); + yyjson_mut_obj_add_strcpy(doc, vitem, "file_path", vresults[v].file_path); + yyjson_mut_obj_add_real(doc, vitem, "score", vresults[v].score); + yyjson_mut_arr_add_val(sem_results, vitem); + } + yyjson_mut_obj_add_val(doc, root, "semantic_results", sem_results); +} + +/* Append the semantic_query vector-search results onto the doc. Returns + * true if semantic_query was provided as a non-array (type error — caller + * should surface to the user). */ +static bool run_semantic_query(yyjson_mut_doc *doc, yyjson_mut_val *root, const char *args, + cbm_store_t *store, const char *project, int limit) { + enum { MAX_KW_SEARCH = 32 }; + yyjson_doc *args_doc = yyjson_read(args, strlen(args), 0); + yyjson_val *args_root = args_doc ? yyjson_doc_get_root(args_doc) : NULL; + yyjson_val *sq_val = args_root ? yyjson_obj_get(args_root, "semantic_query") : NULL; + bool type_error = false; + if (sq_val && !yyjson_is_arr(sq_val)) { + type_error = true; + } else if (sq_val && yyjson_arr_size(sq_val) > 0) { + const char *keywords[MAX_KW_SEARCH]; + int ki = extract_semantic_keywords(sq_val, keywords, MAX_KW_SEARCH); + cbm_vector_result_t *vresults = NULL; + int vcount = 0; + int sem_limit = limit > 0 ? limit : CBM_SZ_16; + if (cbm_store_vector_search(store, project, keywords, ki, sem_limit, &vresults, &vcount) == + CBM_STORE_OK && + vcount > 0) { + emit_semantic_results(doc, root, vresults, vcount); + cbm_store_free_vector_results(vresults, vcount); + } + } + if (args_doc) { + yyjson_doc_free(args_doc); + } + return type_error; +} + static char *handle_search_graph(cbm_mcp_server_t *srv, const char *args) { char *project = cbm_mcp_get_string_arg(args, "project"); cbm_store_t *store = resolve_store(srv, project); @@ -1193,7 +1317,7 @@ static char *handle_search_graph(cbm_mcp_server_t *srv, const char *args) { * tokenization, fall through to the regex path. */ char *query = cbm_mcp_get_string_arg(args, "query"); if (query && query[0]) { - int q_limit = cbm_mcp_get_int_arg(args, "limit", 100); + int q_limit = cbm_mcp_get_int_arg(args, "limit", BM25_DEFAULT_LIMIT); int q_offset = cbm_mcp_get_int_arg(args, "offset", 0); char *bm25_json = bm25_search(store, project, query, q_limit, q_offset); if (bm25_json) { @@ -1250,85 +1374,8 @@ static char *handle_search_graph(cbm_mcp_server_t *srv, const char *args) { yyjson_mut_val *root = yyjson_mut_obj(doc); yyjson_mut_doc_set_root(doc, root); - yyjson_mut_obj_add_int(doc, root, "total", out.total); - - yyjson_mut_val *results = yyjson_mut_arr(doc); - for (int i = 0; i < out.count; i++) { - cbm_search_result_t *sr = &out.results[i]; - yyjson_mut_val *item = yyjson_mut_obj(doc); - yyjson_mut_obj_add_str(doc, item, "name", sr->node.name ? sr->node.name : ""); - yyjson_mut_obj_add_str(doc, item, "qualified_name", - sr->node.qualified_name ? sr->node.qualified_name : ""); - yyjson_mut_obj_add_str(doc, item, "label", sr->node.label ? sr->node.label : ""); - yyjson_mut_obj_add_str(doc, item, "file_path", - sr->node.file_path ? sr->node.file_path : ""); - yyjson_mut_obj_add_int(doc, item, "in_degree", sr->in_degree); - yyjson_mut_obj_add_int(doc, item, "out_degree", sr->out_degree); - - if (include_connected && sr->node.id > 0) { - enrich_connected(doc, item, store, sr->node.id, relationship); - } - - yyjson_mut_arr_add_val(results, item); - } - yyjson_mut_obj_add_val(doc, root, "results", results); - yyjson_mut_obj_add_bool(doc, root, "has_more", out.total > offset + out.count); - - /* Semantic vector search: parse semantic_query array and run vector search. - * Strict validation: if the key is present it MUST be an array of strings. - * A plain string (e.g. "send pubsub publish") is rejected with a clear error - * so callers notice the mistake instead of silently getting an unranked list. */ - bool sq_type_error = false; - { - yyjson_doc *args_doc = yyjson_read(args, strlen(args), 0); - yyjson_val *args_root = args_doc ? yyjson_doc_get_root(args_doc) : NULL; - yyjson_val *sq_val = args_root ? yyjson_obj_get(args_root, "semantic_query") : NULL; - if (sq_val && !yyjson_is_arr(sq_val)) { - sq_type_error = true; - } else if (sq_val) { - int kw_count = (int)yyjson_arr_size(sq_val); - if (kw_count > 0) { - enum { MAX_KW = 32 }; - const char *keywords[MAX_KW]; - if (kw_count > MAX_KW) { - kw_count = MAX_KW; - } - size_t kw_idx = 0; - size_t kw_max = 0; - yyjson_val *kw_val; - int ki = 0; - yyjson_arr_foreach(sq_val, kw_idx, kw_max, kw_val) { - if (ki < kw_count && yyjson_is_str(kw_val)) { - keywords[ki++] = yyjson_get_str(kw_val); - } - } - - cbm_vector_result_t *vresults = NULL; - int vcount = 0; - int sem_limit = limit > 0 ? limit : CBM_SZ_16; - if (cbm_store_vector_search(store, project, keywords, ki, sem_limit, &vresults, - &vcount) == CBM_STORE_OK && - vcount > 0) { - yyjson_mut_val *sem_results = yyjson_mut_arr(doc); - for (int v = 0; v < vcount; v++) { - yyjson_mut_val *vitem = yyjson_mut_obj(doc); - yyjson_mut_obj_add_strcpy(doc, vitem, "name", vresults[v].name); - yyjson_mut_obj_add_strcpy(doc, vitem, "qualified_name", - vresults[v].qualified_name); - yyjson_mut_obj_add_strcpy(doc, vitem, "label", vresults[v].label); - yyjson_mut_obj_add_strcpy(doc, vitem, "file_path", vresults[v].file_path); - yyjson_mut_obj_add_real(doc, vitem, "score", vresults[v].score); - yyjson_mut_arr_add_val(sem_results, vitem); - } - yyjson_mut_obj_add_val(doc, root, "semantic_results", sem_results); - cbm_store_free_vector_results(vresults, vcount); - } - } - } - if (args_doc) { - yyjson_doc_free(args_doc); - } - } + emit_search_results(doc, root, &out, store, relationship, include_connected, offset); + bool sq_type_error = run_semantic_query(doc, root, args, store, project, limit); if (sq_type_error) { yyjson_mut_doc_free(doc); @@ -2817,7 +2864,7 @@ static bool write_pattern_file(char *tmpfile, int tmpfile_sz, const char *patter #ifdef _WIN32 snprintf(tmpfile, tmpfile_sz, "/tmp/cbm_search_%d.pat", (int)_getpid()); #else - snprintf(tmpfile, tmpfile_sz, "/tmp/cbm_search_%d.pat", (int)getpid()); + snprintf(tmpfile, tmpfile_sz, "/tmp/cbm_search_%d.pat", getpid()); #endif FILE *tf = fopen(tmpfile, "w"); if (!tf) { diff --git a/src/pipeline/fqn.c b/src/pipeline/fqn.c index 5f5482583..f98cef920 100644 --- a/src/pipeline/fqn.c +++ b/src/pipeline/fqn.c @@ -8,6 +8,7 @@ #include "foundation/constants.h" #include "foundation/platform.h" +#include #include // NULL #include #include @@ -125,6 +126,167 @@ char *cbm_pipeline_fqn_module(const char *project, const char *rel_path) { return cbm_pipeline_fqn_compute(project, rel_path, NULL); } +enum { + FQN_PATH_BUF = 1024, + FQN_SEP_LEN = 1, /* one byte for the '/' separator */ + FQN_NUL_LEN = 1, /* one byte for the terminating NUL */ + FQN_DOTDOT_LEN = 2, + FQN_MIN_PY_DOTS = 1, /* first leading dot is "current package", not a pop */ + FQN_REL_KIND_NONE = 0, + FQN_REL_KIND_PYTHON = 1, + FQN_REL_KIND_JS = 2, +}; + +/* Append a single path segment to a mutable buffer that already holds a + * normalized slash-separated path. Adds a '/' separator when needed, + * returns false if the buffer would overflow. */ +static bool path_append_segment(char *buf, size_t buf_size, const char *seg, size_t seg_len) { + size_t cur = strlen(buf); + size_t need = cur + (cur > 0 ? FQN_SEP_LEN : 0) + seg_len + FQN_NUL_LEN; + if (need > buf_size) { + return false; + } + if (cur > 0) { + buf[cur++] = '/'; + } + memcpy(buf + cur, seg, seg_len); + buf[cur + seg_len] = '\0'; + return true; +} + +/* Pop the last segment from a mutable slash-separated path. */ +static void path_pop_segment(char *buf) { + char *last = strrchr(buf, '/'); + if (last) { + *last = '\0'; + } else { + buf[0] = '\0'; + } +} + +/* Seed `buf` with the source file's directory (strip the basename) and + * normalize backslashes. */ +static void seed_source_dir(char *buf, size_t buf_size, const char *source_rel) { + snprintf(buf, buf_size, "%s", source_rel ? source_rel : ""); + for (char *p = buf; *p; p++) { + if (*p == '\\') { + *p = '/'; + } + } + char *last = strrchr(buf, '/'); + if (last) { + *last = '\0'; + } else { + buf[0] = '\0'; + } +} + +/* Detect the flavor of relative import based on the leading characters. + * Returns 1 for Python dotted form (e.g. ".foo" or "..bar.baz"), + * 2 for JS/TS slash form (e.g. "./foo" or "../bar/baz"), + * 0 for anything not relative (caller should skip). */ +static int classify_relative_import(const char *module_path) { + if (!module_path || module_path[0] != '.') { + return FQN_REL_KIND_NONE; + } + bool has_slash = strchr(module_path, '/') != NULL; + bool js_like = module_path[FQN_SEP_LEN] == '/' || + (module_path[FQN_SEP_LEN] == '.' && module_path[FQN_DOTDOT_LEN] == '/'); + if (has_slash || js_like) { + return FQN_REL_KIND_JS; + } + return FQN_REL_KIND_PYTHON; +} + +/* Python relative import: ".foo", "..bar.baz" → resolve against source dir. */ +static char *resolve_python_relative(char *buf, size_t buf_size, const char *module_path) { + const char *p = module_path; + int dot_count = 0; + while (*p == '.') { + dot_count++; + p++; + } + for (int i = FQN_MIN_PY_DOTS; i < dot_count; i++) { + path_pop_segment(buf); + } + while (*p) { + const char *seg_start = p; + while (*p && *p != '.') { + p++; + } + size_t seg_len = (size_t)(p - seg_start); + if (seg_len > 0 && !path_append_segment(buf, buf_size, seg_start, seg_len)) { + return NULL; + } + if (*p == '.') { + p++; + } + } + return strdup(buf); +} + +/* Strip a trailing file extension from a segment (e.g. "helpers.ts" → "helpers"). + * Returns the new segment length. */ +static size_t strip_ext(const char *seg_start, size_t seg_len) { + const char *seg_end = seg_start + seg_len; + const char *dot = NULL; + for (const char *d = seg_end - FQN_SEP_LEN; d >= seg_start; d--) { + if (*d == '.') { + dot = d; + break; + } + } + if (dot && dot > seg_start) { + return (size_t)(dot - seg_start); + } + return seg_len; +} + +/* JS/TS relative import: "./foo", "../bar/baz" → resolve against source dir. */ +static char *resolve_js_relative(char *buf, size_t buf_size, const char *module_path) { + const char *p = module_path; + while (*p) { + while (*p == '/') { + p++; + } + if (!*p) { + break; + } + const char *seg_start = p; + while (*p && *p != '/') { + p++; + } + size_t seg_len = (size_t)(p - seg_start); + if (seg_len == FQN_SEP_LEN && seg_start[0] == '.') { + continue; + } + if (seg_len == FQN_DOTDOT_LEN && seg_start[0] == '.' && seg_start[FQN_SEP_LEN] == '.') { + path_pop_segment(buf); + continue; + } + if (*p == '\0') { + seg_len = strip_ext(seg_start, seg_len); + } + if (seg_len > 0 && !path_append_segment(buf, buf_size, seg_start, seg_len)) { + return NULL; + } + } + return strdup(buf); +} + +char *cbm_pipeline_resolve_relative_import(const char *source_rel, const char *module_path) { + int kind = classify_relative_import(module_path); + if (kind == FQN_REL_KIND_NONE) { + return NULL; + } + char buf[FQN_PATH_BUF]; + seed_source_dir(buf, sizeof(buf), source_rel); + if (kind == FQN_REL_KIND_PYTHON) { + return resolve_python_relative(buf, sizeof(buf), module_path); + } + return resolve_js_relative(buf, sizeof(buf), module_path); +} + char *cbm_pipeline_fqn_folder(const char *project, const char *rel_dir) { if (!project) { return strdup(""); diff --git a/src/pipeline/pass_calls.c b/src/pipeline/pass_calls.c index b43545020..400eb74f0 100644 --- a/src/pipeline/pass_calls.c +++ b/src/pipeline/pass_calls.c @@ -255,13 +255,14 @@ static void emit_http_async_edge(cbm_pipeline_ctx_t *ctx, const CBMCall *call, const char *broker = (svc == CBM_SVC_ASYNC) ? cbm_service_pattern_broker(res->qualified_name) : NULL; int64_t route_id = create_svc_route_node(ctx, url_or_topic, svc, method, broker); - char esc_callee[CBM_SZ_256], esc_url[CBM_SZ_256]; + char esc_callee[CBM_SZ_256]; + char esc_url[CBM_SZ_256]; cbm_json_escape(esc_callee, sizeof(esc_callee), call->callee_name); cbm_json_escape(esc_url, sizeof(esc_url), url_or_topic); char props[CBM_SZ_512]; - snprintf(props, sizeof(props), "{\"callee\":\"%s\",\"url_path\":\"%s\"%s%s%s%s%s}", - esc_callee, esc_url, method ? ",\"method\":\"" : "", method ? method : "", - method ? "\"" : "", broker ? ",\"broker\":\"" : "", broker ? broker : ""); + snprintf(props, sizeof(props), "{\"callee\":\"%s\",\"url_path\":\"%s\"%s%s%s%s%s}", esc_callee, + esc_url, method ? ",\"method\":\"" : "", method ? method : "", method ? "\"" : "", + broker ? ",\"broker\":\"" : "", broker ? broker : ""); if (broker) { size_t plen = strlen(props); if (plen > 0 && props[plen - SKIP_ONE] != '}') { @@ -286,7 +287,8 @@ static void emit_classified_edge(cbm_pipeline_ctx_t *ctx, const CBMCall *call, return; } if (svc == CBM_SVC_CONFIG) { - char esc_c[CBM_SZ_256], esc_k[CBM_SZ_256]; + char esc_c[CBM_SZ_256]; + char esc_k[CBM_SZ_256]; cbm_json_escape(esc_c, sizeof(esc_c), call->callee_name); cbm_json_escape(esc_k, sizeof(esc_k), call->first_string_arg ? call->first_string_arg : ""); char props[CBM_SZ_512]; diff --git a/src/pipeline/pass_definitions.c b/src/pipeline/pass_definitions.c index e763ca8d7..eb6089fc9 100644 --- a/src/pipeline/pass_definitions.c +++ b/src/pipeline/pass_definitions.c @@ -249,7 +249,52 @@ static void process_def(cbm_pipeline_ctx_t *ctx, const CBMDefinition *def, const } } -/* Create IMPORTS edges for one file's imports. */ +/* Create Channel nodes + EMITS / LISTENS_ON edges for one file's channels. + * Mirrors the parallel path in cbm_build_registry_from_cache — keep in sync. */ +/* Find the source node for a channel edge: enclosing function or file node. */ +static const cbm_gbuf_node_t *find_channel_source(cbm_pipeline_ctx_t *ctx, const CBMChannel *ch, + const char *rel) { + const cbm_gbuf_node_t *node = NULL; + if (ch->enclosing_func_qn && ch->enclosing_func_qn[0]) { + node = cbm_gbuf_find_by_qn(ctx->gbuf, ch->enclosing_func_qn); + } + if (!node) { + char *file_qn = cbm_pipeline_fqn_compute(ctx->project_name, rel, "__file__"); + node = cbm_gbuf_find_by_qn(ctx->gbuf, file_qn); + free(file_qn); + } + return node; +} + +static void create_channel_edges_for_file(cbm_pipeline_ctx_t *ctx, const CBMFileResult *result, + const char *rel) { + for (int j = 0; j < result->channels.count; j++) { + const CBMChannel *ch = &result->channels.items[j]; + if (!ch->channel_name || !ch->channel_name[0]) { + continue; + } + char channel_qn[CBM_SZ_512]; + snprintf(channel_qn, sizeof(channel_qn), "__channel__%s__%s", + ch->transport ? ch->transport : "unknown", ch->channel_name); + char channel_props[CBM_SZ_512]; + snprintf(channel_props, sizeof(channel_props), "{\"transport\":\"%s\",\"name\":\"%s\"}", + ch->transport ? ch->transport : "unknown", ch->channel_name); + int64_t channel_id = cbm_gbuf_upsert_node(ctx->gbuf, "Channel", ch->channel_name, + channel_qn, "", 0, 0, channel_props); + + const cbm_gbuf_node_t *src_node = find_channel_source(ctx, ch, rel); + if (src_node && channel_id > 0) { + const char *edge_type = ch->direction == CBM_CHANNEL_EMIT ? "EMITS" : "LISTENS_ON"; + char edge_props[CBM_SZ_128]; + snprintf(edge_props, sizeof(edge_props), "{\"transport\":\"%s\"}", + ch->transport ? ch->transport : "unknown"); + cbm_gbuf_insert_edge(ctx->gbuf, src_node->id, channel_id, edge_type, edge_props); + } + } +} + +/* Create IMPORTS edges for one file's imports. Mirrors the resolution + * logic in pass_parallel.c register_and_link_def — keep the two in sync. */ static int create_import_edges_for_file(cbm_pipeline_ctx_t *ctx, const CBMFileResult *result, const char *rel) { int count = 0; @@ -258,7 +303,14 @@ static int create_import_edges_for_file(cbm_pipeline_ctx_t *ctx, const CBMFileRe if (!imp->module_path) { continue; } - char *target_qn = cbm_pipeline_fqn_module(ctx->project_name, imp->module_path); + char *target_qn = NULL; + char *resolved = cbm_pipeline_resolve_relative_import(rel, imp->module_path); + if (resolved) { + target_qn = cbm_pipeline_fqn_module(ctx->project_name, resolved); + free(resolved); + } else { + target_qn = cbm_pipeline_fqn_module(ctx->project_name, imp->module_path); + } const cbm_gbuf_node_t *target = cbm_gbuf_find_by_qn(ctx->gbuf, target_qn); char *file_qn = cbm_pipeline_fqn_compute(ctx->project_name, rel, "__file__"); const cbm_gbuf_node_t *source_node = cbm_gbuf_find_by_qn(ctx->gbuf, file_qn); @@ -326,6 +378,7 @@ int cbm_pipeline_pass_definitions(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t * for now — a future optimization would batch these) */ total_calls += result->calls.count; total_imports += create_import_edges_for_file(ctx, result, rel); + create_channel_edges_for_file(ctx, result, rel); /* Cache or free the extraction result */ if (ctx->result_cache) { diff --git a/src/pipeline/pass_parallel.c b/src/pipeline/pass_parallel.c index 8e85d7559..619757b28 100644 --- a/src/pipeline/pass_parallel.c +++ b/src/pipeline/pass_parallel.c @@ -673,6 +673,84 @@ static int register_and_link_def(cbm_pipeline_ctx_t *ctx, const CBMDefinition *d return edges; } +/* Create IMPORTS edges for one file's imports (parallel path). */ +static int create_imports_edges(cbm_pipeline_ctx_t *ctx, const CBMFileResult *result, + const char *rel) { + int count = 0; + for (int j = 0; j < result->imports.count; j++) { + CBMImport *imp = &result->imports.items[j]; + if (!imp->module_path) { + continue; + } + char *target_qn = NULL; + char *resolved = cbm_pipeline_resolve_relative_import(rel, imp->module_path); + if (resolved) { + target_qn = cbm_pipeline_fqn_module(ctx->project_name, resolved); + free(resolved); + } else { + target_qn = cbm_pipeline_fqn_module(ctx->project_name, imp->module_path); + } + const cbm_gbuf_node_t *target = cbm_gbuf_find_by_qn(ctx->gbuf, target_qn); + char *file_qn = cbm_pipeline_fqn_compute(ctx->project_name, rel, "__file__"); + const cbm_gbuf_node_t *source_node = cbm_gbuf_find_by_qn(ctx->gbuf, file_qn); + if (source_node && target) { + char esc_ln[CBM_SZ_128]; + cbm_json_escape(esc_ln, sizeof(esc_ln), imp->local_name ? imp->local_name : ""); + char imp_props[CBM_SZ_256]; + snprintf(imp_props, sizeof(imp_props), "{\"local_name\":\"%s\"}", esc_ln); + cbm_gbuf_insert_edge(ctx->gbuf, source_node->id, target->id, "IMPORTS", imp_props); + count++; + } + free(target_qn); + free(file_qn); + } + return count; +} + +/* Find channel source node (enclosing function or file). */ +static const cbm_gbuf_node_t *find_channel_src(cbm_pipeline_ctx_t *ctx, const CBMChannel *ch, + const char *rel) { + const cbm_gbuf_node_t *node = NULL; + if (ch->enclosing_func_qn && ch->enclosing_func_qn[0]) { + node = cbm_gbuf_find_by_qn(ctx->gbuf, ch->enclosing_func_qn); + } + if (!node) { + char *file_qn = cbm_pipeline_fqn_compute(ctx->project_name, rel, "__file__"); + node = cbm_gbuf_find_by_qn(ctx->gbuf, file_qn); + free(file_qn); + } + return node; +} + +/* Create Channel nodes + EMITS/LISTENS_ON edges for one file. */ +static void create_channel_edges(cbm_pipeline_ctx_t *ctx, const CBMFileResult *result, + const char *rel) { + for (int j = 0; j < result->channels.count; j++) { + CBMChannel *ch = &result->channels.items[j]; + if (!ch->channel_name || !ch->channel_name[0]) { + continue; + } + char channel_qn[CBM_SZ_512]; + snprintf(channel_qn, sizeof(channel_qn), "__channel__%s__%s", + ch->transport ? ch->transport : "unknown", ch->channel_name); + char esc_cn[CBM_SZ_256]; + cbm_json_escape(esc_cn, sizeof(esc_cn), ch->channel_name); + char channel_props[CBM_SZ_512]; + snprintf(channel_props, sizeof(channel_props), "{\"transport\":\"%s\",\"name\":\"%s\"}", + ch->transport ? ch->transport : "unknown", esc_cn); + int64_t channel_id = cbm_gbuf_upsert_node(ctx->gbuf, "Channel", ch->channel_name, + channel_qn, "", 0, 0, channel_props); + const cbm_gbuf_node_t *src_node = find_channel_src(ctx, ch, rel); + if (src_node && channel_id > 0) { + const char *edge_type = ch->direction == CBM_CHANNEL_EMIT ? "EMITS" : "LISTENS_ON"; + char edge_props[CBM_SZ_128]; + snprintf(edge_props, sizeof(edge_props), "{\"transport\":\"%s\"}", + ch->transport ? ch->transport : "unknown"); + cbm_gbuf_insert_edge(ctx->gbuf, src_node->id, channel_id, edge_type, edge_props); + } + } +} + int cbm_build_registry_from_cache(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files, int file_count, CBMFileResult **result_cache) { cbm_log_info("parallel.registry.start", "files", itoa_log(file_count)); @@ -698,30 +776,8 @@ int cbm_build_registry_from_cache(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t defines_edges += register_and_link_def(ctx, &result->defs.items[d], rel, ®_entries); } - /* IMPORTS edges */ - for (int j = 0; j < result->imports.count; j++) { - CBMImport *imp = &result->imports.items[j]; - if (!imp->module_path) { - continue; - } - - char *target_qn = cbm_pipeline_fqn_module(ctx->project_name, imp->module_path); - const cbm_gbuf_node_t *target = cbm_gbuf_find_by_qn(ctx->gbuf, target_qn); - - char *file_qn = cbm_pipeline_fqn_compute(ctx->project_name, rel, "__file__"); - const cbm_gbuf_node_t *source_node = cbm_gbuf_find_by_qn(ctx->gbuf, file_qn); - - if (source_node && target) { - char esc_ln[CBM_SZ_128]; - cbm_json_escape(esc_ln, sizeof(esc_ln), imp->local_name ? imp->local_name : ""); - char imp_props[CBM_SZ_256]; - snprintf(imp_props, sizeof(imp_props), "{\"local_name\":\"%s\"}", esc_ln); - cbm_gbuf_insert_edge(ctx->gbuf, source_node->id, target->id, "IMPORTS", imp_props); - imports_edges++; - } - free(target_qn); - free(file_qn); - } + imports_edges += create_imports_edges(ctx, result, rel); + create_channel_edges(ctx, result, rel); } cbm_log_info("parallel.registry.done", "entries", itoa_log(reg_entries), "defines", @@ -781,7 +837,9 @@ static void sanitize_expr(char *expr_buf, const char *expr) { /* Format one call arg as JSON. Returns snprintf result. */ static int format_call_arg(char *buf, size_t bufsize, const CBMCallArg *a, const char *expr) { - char esc_k[CBM_SZ_128], esc_e[CBM_SZ_128], esc_v[CBM_SZ_128]; + char esc_k[CBM_SZ_128]; + char esc_e[CBM_SZ_128]; + char esc_v[CBM_SZ_128]; cbm_json_escape(esc_e, sizeof(esc_e), expr); if (a->keyword && a->value) { cbm_json_escape(esc_k, sizeof(esc_k), a->keyword); @@ -928,12 +986,12 @@ static void emit_http_async_service_edge(cbm_gbuf_t *gbuf, const cbm_gbuf_node_t int64_t route_id = build_service_route(gbuf, arg, method, broker, svc); - char esc_c[CBM_SZ_256], esc_a[CBM_SZ_256]; + char esc_c[CBM_SZ_256]; + char esc_a[CBM_SZ_256]; cbm_json_escape(esc_c, sizeof(esc_c), call->callee_name); cbm_json_escape(esc_a, sizeof(esc_a), arg); char props[CBM_SZ_2K]; - int n = snprintf(props, sizeof(props), "{\"callee\":\"%s\",\"url_path\":\"%s\"", - esc_c, esc_a); + int n = snprintf(props, sizeof(props), "{\"callee\":\"%s\",\"url_path\":\"%s\"", esc_c, esc_a); if (method) { n += snprintf(props + n, sizeof(props) - (size_t)n, ",\"method\":\"%s\"", method); } @@ -947,7 +1005,8 @@ static void emit_http_async_service_edge(cbm_gbuf_t *gbuf, const cbm_gbuf_node_t static void emit_config_edge(cbm_gbuf_t *gbuf, const cbm_gbuf_node_t *source, const cbm_gbuf_node_t *target, const CBMCall *call, const cbm_resolution_t *res, const char *arg) { - char esc_c[CBM_SZ_256], esc_k[CBM_SZ_256]; + char esc_c[CBM_SZ_256]; + char esc_k[CBM_SZ_256]; cbm_json_escape(esc_c, sizeof(esc_c), call->callee_name); cbm_json_escape(esc_k, sizeof(esc_k), arg ? arg : ""); char props[CBM_SZ_2K]; @@ -1068,7 +1127,8 @@ static void detect_url_in_args(cbm_gbuf_t *gbuf, const cbm_gbuf_node_t *source, snprintf(route_qn, sizeof(route_qn), "__route__ANY__%s", norm); int64_t route_id = cbm_gbuf_upsert_node(gbuf, "Route", norm, route_qn, "", 0, 0, "{\"source\":\"arg_url\"}"); - char esc_c[CBM_SZ_256], esc_n[CBM_SZ_256]; + char esc_c[CBM_SZ_256]; + char esc_n[CBM_SZ_256]; cbm_json_escape(esc_c, sizeof(esc_c), call->callee_name); cbm_json_escape(esc_n, sizeof(esc_n), norm); char eprops[CBM_SZ_512]; diff --git a/src/pipeline/pass_semantic_edges.c b/src/pipeline/pass_semantic_edges.c index 3b17e70d6..840dbe7ff 100644 --- a/src/pipeline/pass_semantic_edges.c +++ b/src/pipeline/pass_semantic_edges.c @@ -27,14 +27,38 @@ #include "foundation/profile.h" #include +#include #include #include #include /* ── Helpers ─────────────────────────────────────────────────────── */ -enum { PROPS_BUF = 512, MAX_FUNCS_INIT = 4096, GROW = 2, MAX_CALLEES = 64, - MAX_DEFERRED_EDGES = 8192 }; +enum { + PROPS_BUF = 512, + MAX_FUNCS_INIT = 4096, + GROW = 2, + MAX_CALLEES = 64, + MAX_DEFERRED_EDGES = 8192, + /* Bit-mask for the low order parity bit used by sparse random indexing. */ + PSE_PARITY_BIT = 1, + /* Bit shift count for signature bit positions (1 << k). */ + PSE_BIT_1 = 1, + PSE_BIT_64 = 64, + PSE_MOD_64 = 64, + PSE_MINHASH_K = 64, + PSE_LSH_BAND_COUNT = 2, + PSE_FP_PREFIX_LEN = 6, /* strlen("\"fp\":\"") */ + PSE_LSH_ROWS_PER_BAND = 2, + PSE_MIN_FUNCS_FOR_PAIR = 2, +}; + +/* Scalar weight constants used in score_worker and related helpers. */ +#define PSE_UNIT_POS 1.0F +#define PSE_INT8_MAX 127.0F +#define PSE_ROUND_BIAS 0.5F +#define PSE_FLOW_WEIGHT 0.01F +#define PSE_ONE_ULL ((uint64_t)1) /* ── Deferred edge buffer (thread-local, merged after parallel pass) ── */ @@ -57,18 +81,19 @@ static void deferred_buf_init(deferred_edge_buf_t *buf) { buf->cap = 0; } -static void deferred_buf_push(deferred_edge_buf_t *buf, int64_t src, int64_t tgt, - float score, bool same_file) { +static void deferred_buf_push(deferred_edge_buf_t *buf, int64_t src, int64_t tgt, float score, + bool same_file) { if (buf->count >= buf->cap) { int nc = buf->cap < CBM_SZ_256 ? CBM_SZ_256 : buf->cap * GROW; deferred_edge_t *grown = realloc(buf->edges, (size_t)nc * sizeof(deferred_edge_t)); - if (!grown) return; + if (!grown) { + return; + } buf->edges = grown; buf->cap = nc; } buf->edges[buf->count++] = (deferred_edge_t){ - .source_id = src, .target_id = tgt, .score = score, .same_file = same_file - }; + .source_id = src, .target_id = tgt, .score = score, .same_file = same_file}; } static void deferred_buf_free(deferred_edge_buf_t *buf) { @@ -84,120 +109,175 @@ static const char *json_str_value(const char *json, const char *key, char *buf, /* Inject semantic tokens based on detected code patterns. * This bridges the vocabulary gap for abstract concepts like "error handling". */ +/* Append a single token to `tokens` if there's capacity. Consolidates the + * `if (count < max_tokens) { tokens[count++] = strdup(...); }` pattern. */ +static int push_pattern_token(char **tokens, int count, int max_tokens, const char *text) { + if (count >= max_tokens) { + return count; + } + tokens[count] = strdup(text); + return tokens[count] ? count + SKIP_ONE : count; +} + +/* True if `s` contains any of the space-separated substrings in `needles`. */ +static bool has_any(const char *s, const char *const *needles) { + if (!s) { + return false; + } + for (const char *const *n = needles; *n; n++) { + if (strstr(s, *n)) { + return true; + } + } + return false; +} + +/* Inject tokens derived from body-text patterns (try/catch, raise, log). */ +static int inject_body_pattern_tokens(const char *bt, char **tokens, int count, int max_tokens) { + if (!bt) { + return count; + } + static const char *const ERR_HANDLING[] = {"except", "catch", "rescue", NULL}; + if (has_any(bt, ERR_HANDLING)) { + count = push_pattern_token(tokens, count, max_tokens, "error"); + count = push_pattern_token(tokens, count, max_tokens, "handling"); + count = push_pattern_token(tokens, count, max_tokens, "exception"); + } + static const char *const ERR_THROW[] = {"raise", "throw", NULL}; + if (has_any(bt, ERR_THROW)) { + count = push_pattern_token(tokens, count, max_tokens, "error"); + count = push_pattern_token(tokens, count, max_tokens, "exception"); + count = push_pattern_token(tokens, count, max_tokens, "throw"); + } + static const char *const LOGGING[] = {"logger", "logging", "log_", NULL}; + if (has_any(bt, LOGGING)) { + count = push_pattern_token(tokens, count, max_tokens, "logging"); + count = push_pattern_token(tokens, count, max_tokens, "log"); + } + return count; +} + +/* Inject tokens for one CALLS-target name based on keyword groups. */ +static int inject_callee_tokens(const char *name, char **tokens, int count, int max_tokens) { + static const char *const LOG_FNS[] = {"log", "Log", "warn", "debug", "info", NULL}; + static const char *const ERR_FNS[] = {"Error", "error", "Errorf", "panic", NULL}; + static const char *const IO_FNS[] = {"open", "read", "write", "close", "Open", "Read", NULL}; + if (has_any(name, LOG_FNS)) { + count = push_pattern_token(tokens, count, max_tokens, "logging"); + count = push_pattern_token(tokens, count, max_tokens, "log"); + } + if (has_any(name, ERR_FNS)) { + count = push_pattern_token(tokens, count, max_tokens, "error"); + count = push_pattern_token(tokens, count, max_tokens, "handling"); + } + if (has_any(name, IO_FNS)) { + count = push_pattern_token(tokens, count, max_tokens, "io"); + count = push_pattern_token(tokens, count, max_tokens, "file"); + } + return count; +} + +/* Walk the outbound CALLS edges of n and inject tokens for each target. */ +static int inject_calls_pattern_tokens(const cbm_gbuf_node_t *n, const cbm_gbuf_t *gbuf, + char **tokens, int count, int max_tokens) { + if (!gbuf) { + return count; + } + const cbm_gbuf_edge_t **edges = NULL; + int ec = 0; + if (cbm_gbuf_find_edges_by_source_type(gbuf, n->id, "CALLS", &edges, &ec) != 0) { + return count; + } + for (int e = 0; e < ec && count < max_tokens; e++) { + const cbm_gbuf_node_t *t = cbm_gbuf_find_by_id(gbuf, edges[e]->target_id); + if (!t || !t->name) { + continue; + } + count = inject_callee_tokens(t->name, tokens, count, max_tokens); + } + return count; +} + +/* Inject tokens from decorator annotations (@route, @middleware, @pytest.*). */ +static int inject_decorator_tokens(const char *decs, char **tokens, int count, int max_tokens) { + if (!decs) { + return count; + } + static const char *const ROUTING[] = {"route", "Route", "app.", NULL}; + if (has_any(decs, ROUTING)) { + count = push_pattern_token(tokens, count, max_tokens, "routing"); + count = push_pattern_token(tokens, count, max_tokens, "endpoint"); + count = push_pattern_token(tokens, count, max_tokens, "handler"); + } + static const char *const MIDDLEWARE[] = {"middleware", "Middleware", NULL}; + if (has_any(decs, MIDDLEWARE)) { + count = push_pattern_token(tokens, count, max_tokens, "middleware"); + } + static const char *const TEST[] = {"test", "Test", "pytest", NULL}; + if (has_any(decs, TEST)) { + count = push_pattern_token(tokens, count, max_tokens, "test"); + count = push_pattern_token(tokens, count, max_tokens, "testing"); + } + return count; +} + +/* Inject tokens keyed off the node's own name (test_*, *Handler, validator). */ +static int inject_name_pattern_tokens(const char *name, char **tokens, int count, int max_tokens) { + if (!name) { + return count; + } + static const char *const TEST[] = {"test_", "Test", NULL}; + if (has_any(name, TEST)) { + count = push_pattern_token(tokens, count, max_tokens, "test"); + count = push_pattern_token(tokens, count, max_tokens, "testing"); + } + static const char *const MIDDLEWARE[] = {"middleware", "Middleware", NULL}; + if (has_any(name, MIDDLEWARE)) { + count = push_pattern_token(tokens, count, max_tokens, "middleware"); + } + static const char *const HANDLER[] = {"handler", "Handler", NULL}; + if (has_any(name, HANDLER)) { + count = push_pattern_token(tokens, count, max_tokens, "handler"); + } + static const char *const VALIDATOR[] = {"validator", "Validator", "validate", "Validate", NULL}; + if (has_any(name, VALIDATOR)) { + count = push_pattern_token(tokens, count, max_tokens, "validation"); + } + return count; +} + static int inject_pattern_tokens(const cbm_gbuf_node_t *n, const cbm_gbuf_t *gbuf, char **tokens, int count, int max_tokens) { if (!n || count >= max_tokens) { return count; } - /* Check body tokens for patterns */ char bt_buf[CBM_SZ_512]; - const char *bt = n->properties_json ? json_str_value(n->properties_json, "bt", bt_buf, - sizeof(bt_buf)) - : NULL; - /* Check decorators */ + const char *bt = n->properties_json + ? json_str_value(n->properties_json, "bt", bt_buf, sizeof(bt_buf)) + : NULL; char dec_buf[CBM_SZ_256]; - const char *decs = n->properties_json ? json_str_value(n->properties_json, "decorators", dec_buf, - sizeof(dec_buf)) + const char *decs = n->properties_json ? json_str_value(n->properties_json, "decorators", + dec_buf, sizeof(dec_buf)) : NULL; - /* Pattern: try/catch/except in body → inject error handling tokens */ - if (bt && (strstr(bt, "except") || strstr(bt, "catch") || strstr(bt, "rescue"))) { - if (count < max_tokens) tokens[count++] = strdup("error"); - if (count < max_tokens) tokens[count++] = strdup("handling"); - if (count < max_tokens) tokens[count++] = strdup("exception"); - } - - /* Pattern: raise/throw in body → inject error tokens */ - if (bt && (strstr(bt, "raise") || strstr(bt, "throw"))) { - if (count < max_tokens) tokens[count++] = strdup("error"); - if (count < max_tokens) tokens[count++] = strdup("exception"); - if (count < max_tokens) tokens[count++] = strdup("throw"); - } - - /* Pattern: log/logger in callees or body → inject logging tokens */ - if (bt && (strstr(bt, "logger") || strstr(bt, "logging") || strstr(bt, "log_"))) { - if (count < max_tokens) tokens[count++] = strdup("logging"); - if (count < max_tokens) tokens[count++] = strdup("log"); - } - - /* Check callee names for logging/error patterns */ - if (gbuf) { - const cbm_gbuf_edge_t **edges = NULL; - int ec = 0; - if (cbm_gbuf_find_edges_by_source_type(gbuf, n->id, "CALLS", &edges, &ec) == 0) { - for (int e = 0; e < ec && count < max_tokens; e++) { - const cbm_gbuf_node_t *t = cbm_gbuf_find_by_id(gbuf, edges[e]->target_id); - if (!t || !t->name) continue; - /* Callee is a logging function */ - if (strstr(t->name, "log") || strstr(t->name, "Log") || strstr(t->name, "warn") || - strstr(t->name, "debug") || strstr(t->name, "info")) { - if (count < max_tokens) tokens[count++] = strdup("logging"); - if (count < max_tokens) tokens[count++] = strdup("log"); - } - /* Callee is an error function */ - if (strstr(t->name, "Error") || strstr(t->name, "error") || - strstr(t->name, "Errorf") || strstr(t->name, "panic")) { - if (count < max_tokens) tokens[count++] = strdup("error"); - if (count < max_tokens) tokens[count++] = strdup("handling"); - } - /* Callee is a file/IO function */ - if (strstr(t->name, "open") || strstr(t->name, "read") || strstr(t->name, "write") || - strstr(t->name, "close") || strstr(t->name, "Open") || strstr(t->name, "Read")) { - if (count < max_tokens) tokens[count++] = strdup("io"); - if (count < max_tokens) tokens[count++] = strdup("file"); - } - } - } - } - - /* Pattern: decorator-based injection */ - if (decs) { - if (strstr(decs, "route") || strstr(decs, "Route") || strstr(decs, "app.")) { - if (count < max_tokens) tokens[count++] = strdup("routing"); - if (count < max_tokens) tokens[count++] = strdup("endpoint"); - if (count < max_tokens) tokens[count++] = strdup("handler"); - } - if (strstr(decs, "middleware") || strstr(decs, "Middleware")) { - if (count < max_tokens) tokens[count++] = strdup("middleware"); - } - if (strstr(decs, "test") || strstr(decs, "Test") || strstr(decs, "pytest")) { - if (count < max_tokens) tokens[count++] = strdup("test"); - if (count < max_tokens) tokens[count++] = strdup("testing"); - } - } - - /* Pattern: name-based injection */ - if (n->name) { - if (strstr(n->name, "test_") || strstr(n->name, "Test")) { - if (count < max_tokens) tokens[count++] = strdup("test"); - if (count < max_tokens) tokens[count++] = strdup("testing"); - } - if (strstr(n->name, "middleware") || strstr(n->name, "Middleware")) { - if (count < max_tokens) tokens[count++] = strdup("middleware"); - } - if (strstr(n->name, "handler") || strstr(n->name, "Handler")) { - if (count < max_tokens) tokens[count++] = strdup("handler"); - } - if (strstr(n->name, "validator") || strstr(n->name, "Validator") || - strstr(n->name, "validate") || strstr(n->name, "Validate")) { - if (count < max_tokens) tokens[count++] = strdup("validation"); - } - } - + count = inject_body_pattern_tokens(bt, tokens, count, max_tokens); + count = inject_calls_pattern_tokens(n, gbuf, tokens, count, max_tokens); + count = inject_decorator_tokens(decs, tokens, count, max_tokens); + count = inject_name_pattern_tokens(n->name, tokens, count, max_tokens); return count; } /* ── Technique 3: Field weights for token sources ────────────────── */ enum { - FW_NAME = 30, /* ×3.0 */ - FW_CALLEE = 20, /* ×2.0 */ - FW_BODY = 15, /* ×1.5 */ + FW_NAME = 30, /* ×3.0 */ + FW_CALLEE = 20, /* ×2.0 */ + FW_BODY = 15, /* ×1.5 */ FW_SIGNATURE = 10, /* ×1.0 */ - FW_PARAM = 10, /* ×1.0 */ - FW_PATTERN = 25, /* ×2.5 — injected semantic tokens are high value */ - FW_PATH = 5, /* ×0.5 */ - FW_SCALE = 10, /* divisor */ + FW_PARAM = 10, /* ×1.0 */ + FW_PATTERN = 25, /* ×2.5 — injected semantic tokens are high value */ + FW_PATH = 5, /* ×0.5 */ + FW_SCALE = 10, /* divisor */ }; static const char *itoa_log(int val) { @@ -278,101 +358,93 @@ static int json_str_array(const char *json, const char *key, char **out, int max /* ── Tokenize node metadata ──────────────────────────────────────── */ +/* Tokenize a single string field keyed out of node->properties_json, if + * present. Returns the new count after appending any tokens. */ +static int tokenize_json_string_field(const char *json, const char *key, char **tokens, int count, + int max_tokens) { + if (count >= max_tokens) { + return count; + } + char buf[CBM_SZ_512]; + if (!json_str_value(json, key, buf, sizeof(buf))) { + return count; + } + count += cbm_sem_tokenize(buf, tokens + count, max_tokens - count); + return count; +} + +/* Tokenize a JSON array field (e.g. "param_names", "decorators"). */ +static int tokenize_json_array_field(const char *json, const char *key, char **tokens, int count, + int max_tokens) { + if (count >= max_tokens) { + return count; + } + char *arr[CBM_SZ_16]; + int n = json_str_array(json, key, arr, CBM_SZ_16); + for (int p = 0; p < n; p++) { + if (count < max_tokens) { + count += cbm_sem_tokenize(arr[p], tokens + count, max_tokens - count); + } + free(arr[p]); + } + return count; +} + +/* Walk the CALLS edges rooted at n (either outbound or inbound depending on + * `outbound`) and tokenize the names of the target/source nodes. Caller-side + * caps via max_tokens and MAX_CALLEES. */ +static int tokenize_call_neighbors(const cbm_gbuf_node_t *n, const cbm_gbuf_t *gbuf, bool outbound, + char **tokens, int count, int max_tokens) { + if (!gbuf || count >= max_tokens) { + return count; + } + const cbm_gbuf_edge_t **edges = NULL; + int ec = 0; + int rc = outbound ? cbm_gbuf_find_edges_by_source_type(gbuf, n->id, "CALLS", &edges, &ec) + : cbm_gbuf_find_edges_by_target_type(gbuf, n->id, "CALLS", &edges, &ec); + if (rc != 0) { + return count; + } + for (int e = 0; e < ec && e < MAX_CALLEES && count < max_tokens; e++) { + int64_t id = outbound ? edges[e]->target_id : edges[e]->source_id; + const cbm_gbuf_node_t *neighbor = cbm_gbuf_find_by_id(gbuf, id); + if (neighbor && neighbor->name) { + count += cbm_sem_tokenize(neighbor->name, tokens + count, max_tokens - count); + } + } + return count; +} + static int tokenize_node(const cbm_gbuf_node_t *n, const cbm_gbuf_t *gbuf, char **tokens, int max_tokens) { int count = 0; - /* Tokenize name (primary signal) */ count += cbm_sem_tokenize(n->name, tokens + count, max_tokens - count); - /* Tokenize qualified name path components */ if (n->qualified_name && count < max_tokens) { count += cbm_sem_tokenize(n->qualified_name, tokens + count, max_tokens - count); } - /* Tokenize file path (directory = domain context) */ if (n->file_path && count < max_tokens) { count += cbm_sem_tokenize(n->file_path, tokens + count, max_tokens - count); } - /* Extract rich metadata from properties_json */ - if (n->properties_json && count < max_tokens) { - char buf[CBM_SZ_512]; - /* Signature (parameter list text) */ - if (json_str_value(n->properties_json, "signature", buf, sizeof(buf))) { - count += cbm_sem_tokenize(buf, tokens + count, max_tokens - count); - } - /* Return type */ - if (count < max_tokens && - json_str_value(n->properties_json, "return_type", buf, sizeof(buf))) { - count += cbm_sem_tokenize(buf, tokens + count, max_tokens - count); - } - /* Docstring (natural language — rich vocabulary for co-occurrence) */ - if (count < max_tokens && - json_str_value(n->properties_json, "docstring", buf, sizeof(buf))) { - count += cbm_sem_tokenize(buf, tokens + count, max_tokens - count); - } - /* Param names */ - char *pnames[CBM_SZ_16]; - int pn_count = json_str_array(n->properties_json, "param_names", pnames, CBM_SZ_16); - for (int p = 0; p < pn_count && count < max_tokens; p++) { - count += cbm_sem_tokenize(pnames[p], tokens + count, max_tokens - count); - free(pnames[p]); - } - /* Param types */ - char *ptypes[CBM_SZ_16]; - int pt_count = json_str_array(n->properties_json, "param_types", ptypes, CBM_SZ_16); - for (int p = 0; p < pt_count && count < max_tokens; p++) { - count += cbm_sem_tokenize(ptypes[p], tokens + count, max_tokens - count); - free(ptypes[p]); - } - /* Decorators */ - char *decos[CBM_SZ_16]; - int dc = json_str_array(n->properties_json, "decorators", decos, CBM_SZ_16); - for (int p = 0; p < dc && count < max_tokens; p++) { - count += cbm_sem_tokenize(decos[p], tokens + count, max_tokens - count); - free(decos[p]); - } - } - - /* Body tokens: raw identifiers from function body AST. - * Captures what the function DOES — "error" in catch blocks, "log" in logging, etc. */ - if (n->properties_json && count < max_tokens) { - char bt_buf[CBM_SZ_512]; - if (json_str_value(n->properties_json, "bt", bt_buf, sizeof(bt_buf))) { - count += cbm_sem_tokenize(bt_buf, tokens + count, max_tokens - count); - } - } - - /* Callee names: what this function CALLS (behavioral vocabulary). - * "error handling" functions call errors.New, log.Error, etc. */ - if (gbuf && count < max_tokens) { - const cbm_gbuf_edge_t **call_edges = NULL; - int call_count = 0; - if (cbm_gbuf_find_edges_by_source_type(gbuf, n->id, "CALLS", &call_edges, &call_count) == - 0) { - for (int e = 0; e < call_count && count < max_tokens; e++) { - const cbm_gbuf_node_t *target = cbm_gbuf_find_by_id(gbuf, call_edges[e]->target_id); - if (target && target->name) { - count += cbm_sem_tokenize(target->name, tokens + count, max_tokens - count); - } - } - } + if (n->properties_json) { + count = + tokenize_json_string_field(n->properties_json, "signature", tokens, count, max_tokens); + count = tokenize_json_string_field(n->properties_json, "return_type", tokens, count, + max_tokens); + count = + tokenize_json_string_field(n->properties_json, "docstring", tokens, count, max_tokens); + count = + tokenize_json_array_field(n->properties_json, "param_names", tokens, count, max_tokens); + count = + tokenize_json_array_field(n->properties_json, "param_types", tokens, count, max_tokens); + count = + tokenize_json_array_field(n->properties_json, "decorators", tokens, count, max_tokens); + count = tokenize_json_string_field(n->properties_json, "bt", tokens, count, max_tokens); } + count = tokenize_call_neighbors(n, gbuf, /*outbound=*/true, tokens, count, max_tokens); /* Caller names: what CALLS this function (contextual vocabulary). * Functions called by error handlers inherit "error" context. */ - if (gbuf && count < max_tokens) { - const cbm_gbuf_edge_t **caller_edges = NULL; - int caller_count = 0; - if (cbm_gbuf_find_edges_by_target_type(gbuf, n->id, "CALLS", &caller_edges, - &caller_count) == 0) { - for (int e = 0; e < caller_count && e < MAX_CALLEES && count < max_tokens; e++) { - const cbm_gbuf_node_t *source = - cbm_gbuf_find_by_id(gbuf, caller_edges[e]->source_id); - if (source && source->name) { - count += cbm_sem_tokenize(source->name, tokens + count, max_tokens - count); - } - } - } - } - + count = tokenize_call_neighbors(n, gbuf, /*outbound=*/false, tokens, count, max_tokens); return count; } @@ -391,7 +463,7 @@ static void build_api_vec(const cbm_gbuf_t *gbuf, int64_t node_id, cbm_sem_vec_t if (target && target->name) { cbm_sem_vec_t callee_ri; cbm_sem_random_index(target->name, &callee_ri); - cbm_sem_vec_add_scaled(out, &callee_ri, 1.0f); + cbm_sem_vec_add_scaled(out, &callee_ri, PSE_UNIT_POS); added++; } } @@ -408,14 +480,14 @@ static void build_type_vec(const char *props_json, cbm_sem_vec_t *out) { if (json_str_value(props_json, "return_type", rt_buf, sizeof(rt_buf))) { cbm_sem_vec_t ri; cbm_sem_random_index(rt_buf, &ri); - cbm_sem_vec_add_scaled(out, &ri, 1.0f); + cbm_sem_vec_add_scaled(out, &ri, PSE_UNIT_POS); } char *ptypes[CBM_SZ_16]; int pt_count = json_str_array(props_json, "param_types", ptypes, CBM_SZ_16); for (int i = 0; i < pt_count; i++) { cbm_sem_vec_t ri; cbm_sem_random_index(ptypes[i], &ri); - cbm_sem_vec_add_scaled(out, &ri, 1.0f); + cbm_sem_vec_add_scaled(out, &ri, PSE_UNIT_POS); free(ptypes[i]); } cbm_sem_normalize(out); @@ -431,7 +503,7 @@ static void build_deco_vec(const char *props_json, cbm_sem_vec_t *out) { for (int i = 0; i < dc; i++) { cbm_sem_vec_t ri; cbm_sem_random_index(decos[i], &ri); - cbm_sem_vec_add_scaled(out, &ri, 1.0f); + cbm_sem_vec_add_scaled(out, &ri, PSE_UNIT_POS); free(decos[i]); } cbm_sem_normalize(out); @@ -460,7 +532,7 @@ static void decode_minhash(const char *props_json, cbm_sem_func_t *func) { if (!fp_key) { return; } - const char *hex = fp_key + 6; /* strlen("\"fp\":\"") */ + const char *hex = fp_key + PSE_FP_PREFIX_LEN; /* strlen("\"fp\":\"") */ const char *end = strchr(hex, '"'); if (!end || (int)(end - hex) != CBM_MINHASH_HEX_LEN) { return; @@ -480,8 +552,8 @@ static void decode_minhash(const char *props_json, cbm_sem_func_t *func) { typedef struct { const cbm_gbuf_node_t **node_ptrs; /* node pointer per function index */ cbm_gbuf_t *gbuf; /* read-only during tokenization */ - char **all_tokens; /* output: all_tokens[f * MAX + t] */ - int *token_counts; /* output: token count per function */ + char **all_tokens; /* output: all_tokens[f * MAX + t] */ + int *token_counts; /* output: token count per function */ int func_count; _Atomic int next_idx; } tokenize_ctx_t; @@ -489,18 +561,21 @@ typedef struct { static void tokenize_worker(int worker_id, void *ctx_ptr) { (void)worker_id; tokenize_ctx_t *tc = ctx_ptr; - while (1) { - int f = atomic_fetch_add_explicit(&tc->next_idx, 1, memory_order_relaxed); - if (f >= tc->func_count) break; + while (true) { + int f = atomic_fetch_add_explicit(&tc->next_idx, SKIP_ONE, memory_order_relaxed); + if (f >= tc->func_count) { + break; + } const cbm_gbuf_node_t *n = tc->node_ptrs[f]; - char *tokens[CBM_SEM_MAX_TOKENS]; - int count = tokenize_node(n, tc->gbuf, tokens, CBM_SEM_MAX_TOKENS); - count = inject_pattern_tokens(n, tc->gbuf, tokens, count, CBM_SEM_MAX_TOKENS); + /* Write directly into the shared buffer slice for this function — the + * strdup'd tokens are owned by all_tokens[] from the moment they land + * in this slot, which avoids a spurious analyzer "leak" diagnostic on + * the previous stack-local relay pattern. */ + char **dst = &tc->all_tokens[(ptrdiff_t)f * CBM_SEM_MAX_TOKENS]; + int count = tokenize_node(n, tc->gbuf, dst, CBM_SEM_MAX_TOKENS); + count = inject_pattern_tokens(n, tc->gbuf, dst, count, CBM_SEM_MAX_TOKENS); tc->token_counts[f] = count; - for (int t = 0; t < count; t++) { - tc->all_tokens[f * CBM_SEM_MAX_TOKENS + t] = tokens[t]; - } } } @@ -511,7 +586,7 @@ typedef struct { char **all_tokens; int *token_counts; cbm_sem_corpus_t *corpus; - uint8_t *qvecs; /* output: pre-quantized int8 vectors [func_count * CBM_SEM_DIM] */ + uint8_t *qvecs; /* output: pre-quantized int8 vectors [func_count * CBM_SEM_DIM] */ int func_count; _Atomic int next_idx; } vec_build_ctx_t; @@ -519,12 +594,14 @@ typedef struct { static void vec_build_worker(int worker_id, void *ctx_ptr) { (void)worker_id; vec_build_ctx_t *vc = ctx_ptr; - while (1) { - int f = atomic_fetch_add_explicit(&vc->next_idx, 1, memory_order_relaxed); - if (f >= vc->func_count) break; + while (true) { + int f = atomic_fetch_add_explicit(&vc->next_idx, SKIP_ONE, memory_order_relaxed); + if (f >= vc->func_count) { + break; + } int tc = vc->token_counts[f]; - char **tokens = &vc->all_tokens[f * CBM_SEM_MAX_TOKENS]; + char **tokens = &vc->all_tokens[(ptrdiff_t)f * CBM_SEM_MAX_TOKENS]; /* TF-IDF weights */ int *indices = malloc((size_t)tc * sizeof(int)); @@ -532,7 +609,7 @@ static void vec_build_worker(int worker_id, void *ctx_ptr) { int tfidf_len = 0; for (int t = 0; t < tc; t++) { float idf = cbm_sem_corpus_idf(vc->corpus, tokens[t]); - if (idf > 0.0f) { + if (idf > 0.0F) { indices[tfidf_len] = t; weights[tfidf_len] = idf; tfidf_len++; @@ -554,12 +631,16 @@ static void vec_build_worker(int worker_id, void *ctx_ptr) { cbm_sem_normalize(&vc->funcs[f].ri_vec); /* Int8 quantize into pre-allocated output array (parallel-safe) */ - uint8_t *qv = &vc->qvecs[f * CBM_SEM_DIM]; + uint8_t *qv = &vc->qvecs[(ptrdiff_t)f * CBM_SEM_DIM]; for (int d = 0; d < CBM_SEM_DIM; d++) { float v = vc->funcs[f].ri_vec.v[d]; - if (v > 1.0f) v = 1.0f; - if (v < -1.0f) v = -1.0f; - qv[d] = (uint8_t)(int8_t)(v * 127.0f); + if (v > PSE_UNIT_POS) { + v = PSE_UNIT_POS; + } + if (v < -PSE_UNIT_POS) { + v = -PSE_UNIT_POS; + } + qv[d] = (uint8_t)(int8_t)(v * PSE_INT8_MAX); } } } @@ -587,18 +668,20 @@ typedef struct { static void sig_build_worker(int worker_id, void *ctx_ptr) { (void)worker_id; sig_build_ctx_t *sc = ctx_ptr; - while (1) { - int f = atomic_fetch_add_explicit(&sc->next_idx, 1, memory_order_relaxed); - if (f >= sc->func_count) break; + while (true) { + int f = atomic_fetch_add_explicit(&sc->next_idx, SKIP_ONE, memory_order_relaxed); + if (f >= sc->func_count) { + break; + } uint64_t sig = 0; for (int h = 0; h < NUM_HYPERPLANES; h++) { - float dot = 0.0f; + float dot = 0.0F; for (int d = 0; d < CBM_SEM_DIM; d++) { dot += sc->funcs[f].ri_vec.v[d] * sc->hyperplanes[h][d]; } - if (dot > 0.0f) { - sig |= (1ULL << h); + if (dot > 0.0F) { + sig |= (PSE_ONE_ULL << h); } } sc->signatures[f] = sig; @@ -610,12 +693,16 @@ static void sig_build_worker(int worker_id, void *ctx_ptr) { typedef struct { cbm_sem_func_t *funcs; uint64_t *signatures; - int *edge_counts; /* shared, atomically updated */ + int *edge_counts; /* shared, atomically updated */ cbm_sem_config_t cfg; int func_count; /* LSH buckets (read-only during scoring) */ - struct { int *items; int count; int cap; } **band_buckets; + struct { + int *items; + int count; + int cap; + } **band_buckets; /* Per-worker edge buffer */ deferred_edge_buf_t *worker_bufs; @@ -623,73 +710,104 @@ typedef struct { _Atomic int next_idx; } score_ctx_t; +enum { + SCORE_SEEN_CAP = 8192, + SCORE_SEEN_MASK = 8191, + SCORE_SEEN_EMPTY = -1, +}; + +/* Check whether `j` has already been recorded in the open-addressed `seen` + * set for this function; insert it if not. Returns true when the insertion + * was fresh (caller should add to candidates). */ +static bool score_seen_insert(int *seen, int j) { + uint32_t slot = (uint32_t)j & SCORE_SEEN_MASK; + for (int p = 0; p < SCORE_SEEN_CAP; p++) { + uint32_t idx = (slot + (uint32_t)p) & SCORE_SEEN_MASK; + if (seen[idx] == SCORE_SEEN_EMPTY) { + seen[idx] = j; + return true; + } + if (seen[idx] == j) { + return false; + } + } + return false; +} + +/* Collect the unique candidate function indices for node `i` by iterating + * every LSH band and merging bucket members via `seen[]`. Returns the + * populated candidate count. */ +static int score_collect_candidates(score_ctx_t *sc, int i, int *seen, int *candidates, + int cand_cap) { + int cand_count = 0; + for (int b = 0; b < SEM_LSH_BANDS && cand_count < cand_cap; b++) { + int shift = b * SEM_LSH_ROWS; + uint32_t band_val = (uint32_t)((sc->signatures[i] >> shift) & + ((PSE_ONE_ULL << SEM_LSH_ROWS) - PSE_ONE_ULL)); + uint64_t bh = XXH3_64bits_withSeed(&band_val, sizeof(band_val), (uint64_t)b); + uint32_t bucket_idx = (uint32_t)(bh & SEM_BUCKET_MASK); + int bcount = sc->band_buckets[b][bucket_idx].count; + int *bitems = sc->band_buckets[b][bucket_idx].items; + if (bcount > SEM_MAX_CANDIDATES) { + continue; + } + for (int k = 0; k < bcount && cand_count < cand_cap; k++) { + int j = bitems[k]; + if (j <= i) { + continue; + } + if (score_seen_insert(seen, j)) { + candidates[cand_count++] = j; + } + } + } + return cand_count; +} + +/* Score one candidate pair (i, j) and push a deferred edge if the score + * passes the threshold and both endpoints still have edge budget. */ +static void score_try_emit(score_ctx_t *sc, int i, int j, deferred_edge_buf_t *my_buf) { + int ei = atomic_load_explicit((_Atomic int *)&sc->edge_counts[i], memory_order_relaxed); + int ej = atomic_load_explicit((_Atomic int *)&sc->edge_counts[j], memory_order_relaxed); + if (ei >= sc->cfg.max_edges || ej >= sc->cfg.max_edges) { + return; + } + if (strcmp(sc->funcs[i].file_ext, sc->funcs[j].file_ext) != 0) { + return; + } + float score = cbm_sem_combined_score(&sc->funcs[i], &sc->funcs[j], &sc->cfg); + if (score < sc->cfg.threshold) { + return; + } + bool same_file = sc->funcs[i].file_path && sc->funcs[j].file_path && + strcmp(sc->funcs[i].file_path, sc->funcs[j].file_path) == 0; + deferred_buf_push(my_buf, sc->funcs[i].node_id, sc->funcs[j].node_id, score, same_file); + atomic_fetch_add_explicit((_Atomic int *)&sc->edge_counts[i], SKIP_ONE, memory_order_relaxed); + atomic_fetch_add_explicit((_Atomic int *)&sc->edge_counts[j], SKIP_ONE, memory_order_relaxed); +} + static void score_worker(int worker_id, void *ctx_ptr) { score_ctx_t *sc = ctx_ptr; deferred_edge_buf_t *my_buf = &sc->worker_bufs[worker_id]; - enum { SEEN_CAP = 8192, SEEN_MASK = 8191 }; - - while (1) { - int i = atomic_fetch_add_explicit(&sc->next_idx, 1, memory_order_relaxed); - if (i >= sc->func_count) break; - - int my_edges = atomic_load_explicit((_Atomic int *)&sc->edge_counts[i], - memory_order_relaxed); - if (my_edges >= sc->cfg.max_edges) continue; - - /* Collect unique candidates from all bands */ - int seen[SEEN_CAP]; - memset(seen, -1, sizeof(seen)); - int candidates[SEM_MAX_CANDIDATES]; - int cand_count = 0; - - for (int b = 0; b < SEM_LSH_BANDS && cand_count < SEM_MAX_CANDIDATES; b++) { - int shift = b * SEM_LSH_ROWS; - uint32_t band_val = (uint32_t)((sc->signatures[i] >> shift) & - ((1ULL << SEM_LSH_ROWS) - 1)); - uint64_t bh = XXH3_64bits_withSeed(&band_val, sizeof(band_val), (uint64_t)b); - uint32_t bucket_idx = (uint32_t)(bh & SEM_BUCKET_MASK); - int bcount = sc->band_buckets[b][bucket_idx].count; - int *bitems = sc->band_buckets[b][bucket_idx].items; - - if (bcount > SEM_MAX_CANDIDATES) continue; - - for (int k = 0; k < bcount && cand_count < SEM_MAX_CANDIDATES; k++) { - int j = bitems[k]; - if (j <= i) continue; - uint32_t slot = (uint32_t)j & SEEN_MASK; - bool dup = false; - for (int p = 0; p < SEEN_CAP; p++) { - uint32_t idx = (slot + (uint32_t)p) & SEEN_MASK; - if (seen[idx] == -1) { seen[idx] = j; break; } - if (seen[idx] == j) { dup = true; break; } - } - if (dup) continue; - candidates[cand_count++] = j; - } + while (true) { + int i = atomic_fetch_add_explicit(&sc->next_idx, SKIP_ONE, memory_order_relaxed); + if (i >= sc->func_count) { + break; } - - /* Score candidates */ + int my_edges = + atomic_load_explicit((_Atomic int *)&sc->edge_counts[i], memory_order_relaxed); + if (my_edges >= sc->cfg.max_edges) { + continue; + } + int seen[SCORE_SEEN_CAP]; + for (int s = 0; s < SCORE_SEEN_CAP; s++) { + seen[s] = SCORE_SEEN_EMPTY; + } + int candidates[SEM_MAX_CANDIDATES]; + int cand_count = score_collect_candidates(sc, i, seen, candidates, SEM_MAX_CANDIDATES); for (int c = 0; c < cand_count; c++) { - int j = candidates[c]; - int ei = atomic_load_explicit((_Atomic int *)&sc->edge_counts[i], - memory_order_relaxed); - int ej = atomic_load_explicit((_Atomic int *)&sc->edge_counts[j], - memory_order_relaxed); - if (ei >= sc->cfg.max_edges || ej >= sc->cfg.max_edges) continue; - if (strcmp(sc->funcs[i].file_ext, sc->funcs[j].file_ext) != 0) continue; - - float score = cbm_sem_combined_score(&sc->funcs[i], &sc->funcs[j], &sc->cfg); - if (score < sc->cfg.threshold) continue; - - bool same_file = sc->funcs[i].file_path && sc->funcs[j].file_path && - strcmp(sc->funcs[i].file_path, sc->funcs[j].file_path) == 0; - deferred_buf_push(my_buf, sc->funcs[i].node_id, sc->funcs[j].node_id, - score, same_file); - atomic_fetch_add_explicit((_Atomic int *)&sc->edge_counts[i], 1, - memory_order_relaxed); - atomic_fetch_add_explicit((_Atomic int *)&sc->edge_counts[j], 1, - memory_order_relaxed); + score_try_emit(sc, i, candidates[c], my_buf); } } } @@ -707,11 +825,15 @@ typedef struct { static void collect_worker(int worker_id, void *ctx_ptr) { (void)worker_id; collect_ctx_t *cc = ctx_ptr; - while (1) { - int f = atomic_fetch_add_explicit(&cc->next_idx, 64, memory_order_relaxed); - if (f >= cc->func_count) break; - int end = f + 64; - if (end > cc->func_count) end = cc->func_count; + while (true) { + int f = atomic_fetch_add_explicit(&cc->next_idx, PSE_MOD_64, memory_order_relaxed); + if (f >= cc->func_count) { + break; + } + int end = f + PSE_MOD_64; + if (end > cc->func_count) { + end = cc->func_count; + } for (int i = f; i < end; i++) { const cbm_gbuf_node_t *n = cc->node_ptrs[i]; decode_minhash(n->properties_json, &cc->funcs[i]); @@ -723,24 +845,26 @@ static void collect_worker(int worker_id, void *ctx_ptr) { } } -/* ── Pass entry point ────────────────────────────────────────────── */ +/* ── Phase helpers (keep cbm_pipeline_pass_semantic_edges complexity low) ── */ -int cbm_pipeline_pass_semantic_edges(cbm_pipeline_ctx_t *ctx) { - /* Controlled by pipeline mode (moderate/full), not env var */ - cbm_log_info("pass.start", "pass", "semantic_edges"); +typedef struct { + int *items; + int count; + int cap; +} sem_bucket_t; - cbm_gbuf_t *gbuf = ctx->gbuf; - cbm_sem_config_t cfg = cbm_sem_get_config(); - - CBM_PROF_START(t_phase1a); - /* Phase 1a: Fast scan — collect node pointers + basic fields (SEQUENTIAL, tiny). - * Pre-sizes the arrays so Phase 1b can write in parallel. */ - const char *labels[] = {"Function", "Method", NULL}; +/* Phase 1a: seed the funcs[] / node_ptrs[] arrays from all Function and + * Method nodes in the graph buffer. Returns the number of functions collected + * (0 on OOM), and fills *out_funcs / *out_nodes with newly malloc'd arrays. */ +static int phase1_scan_functions(cbm_gbuf_t *gbuf, cbm_sem_func_t **out_funcs, + const cbm_gbuf_node_t ***out_nodes) { + *out_funcs = NULL; + *out_nodes = NULL; cbm_sem_func_t *funcs = NULL; const cbm_gbuf_node_t **node_ptrs = NULL; int func_count = 0; int func_cap = 0; - + const char *labels[] = {"Function", "Method", NULL}; for (int li = 0; labels[li]; li++) { const cbm_gbuf_node_t **nodes = NULL; int node_count = 0; @@ -751,12 +875,15 @@ int cbm_pipeline_pass_semantic_edges(cbm_pipeline_ctx_t *ctx) { if (func_count >= func_cap) { int new_cap = func_cap < MAX_FUNCS_INIT ? MAX_FUNCS_INIT : func_cap * GROW; cbm_sem_func_t *grown = realloc(funcs, (size_t)new_cap * sizeof(cbm_sem_func_t)); - const cbm_gbuf_node_t **np_grown = - realloc(node_ptrs, (size_t)new_cap * sizeof(cbm_gbuf_node_t *)); - if (!grown || !np_grown) { + if (!grown) { break; } funcs = grown; + const cbm_gbuf_node_t **np_grown = + realloc(node_ptrs, (size_t)new_cap * sizeof(cbm_gbuf_node_t *)); + if (!np_grown) { + break; + } node_ptrs = np_grown; func_cap = new_cap; } @@ -768,169 +895,20 @@ int cbm_pipeline_pass_semantic_edges(cbm_pipeline_ctx_t *ctx) { func_count++; } } - CBM_PROF_END_N("semantic_edges", "1a_scan_seq", t_phase1a, func_count); - - /* Phase 1b: Decode minhash + profile + build api/type/deco vectors (PARALLEL). - * Must eagerly init the pretrained token map before parallel dispatch — - * the lazy init inside cbm_sem_random_index is not thread-safe for first-use - * races, and build_api_vec calls cbm_sem_random_index via its callees. */ - cbm_sem_ensure_ready(); - CBM_PROF_START(t_phase1b); - if (func_count > 0) { - int wc = cbm_default_worker_count(false); - collect_ctx_t cc = { - .funcs = funcs, - .node_ptrs = node_ptrs, - .gbuf = gbuf, - .func_count = func_count, - }; - atomic_init(&cc.next_idx, 0); - cbm_parallel_for_opts_t opts = {.max_workers = wc, .force_pthreads = false}; - cbm_parallel_for(wc, collect_worker, &cc, opts); - } - CBM_PROF_END_N("semantic_edges", "1b_decode_build_parallel", t_phase1b, func_count); - cbm_log_info("pass.semantic.collected", "functions", itoa_log(func_count)); - - if (func_count < 2) { - free(funcs); - free(node_ptrs); - cbm_log_info("pass.done", "pass", "semantic_edges", "edges", "0"); - return 0; - } - - /* Phase 2: Tokenize all nodes (PARALLEL) */ - int worker_count = cbm_default_worker_count(false); - char **all_tokens = malloc((size_t)func_count * sizeof(char *) * CBM_SEM_MAX_TOKENS); - int *token_counts = calloc((size_t)func_count, sizeof(int)); - - CBM_PROF_START(t_phase2); - { - tokenize_ctx_t tc = { - .node_ptrs = node_ptrs, .gbuf = gbuf, - .all_tokens = all_tokens, .token_counts = token_counts, - .func_count = func_count, - }; - atomic_init(&tc.next_idx, 0); - cbm_parallel_for_opts_t opts = {.max_workers = worker_count, .force_pthreads = false}; - cbm_parallel_for(worker_count, tokenize_worker, &tc, opts); - } - CBM_PROF_END_N("semantic_edges", "2_tokenize_parallel", t_phase2, func_count); - free(node_ptrs); - - /* Phase 3a: Build corpus from pre-computed tokens (batch API, partially PARALLEL). - * Phase A: sequential token_map build. Phase B: parallel token→ID resolve - * + doc_freq via atomic counters. Phase C: sequential reduce. */ - CBM_PROF_START(t_phase3a); - cbm_sem_corpus_t *corpus = cbm_sem_corpus_new(); - cbm_sem_corpus_add_docs_batch(corpus, all_tokens, token_counts, func_count, - CBM_SEM_MAX_TOKENS); - CBM_PROF_END_N("semantic_edges", "3a_corpus_batch", t_phase3a, func_count); - - /* Phase 3b: Finalize corpus — IDF + co-occurrence enriched RI vectors (sequential) */ - CBM_PROF_START(t_phase3b); - cbm_sem_corpus_finalize(corpus); - CBM_PROF_END_N("semantic_edges", "3b_corpus_finalize_seq", t_phase3b, - cbm_sem_corpus_token_count(corpus)); - - /* Phase 3c: Export enriched token vectors to graph buffer for query-time lookup */ - CBM_PROF_START(t_phase3c); - { - int tv_count = cbm_sem_corpus_token_count(corpus); - for (int t = 0; t < tv_count; t++) { - const cbm_sem_vec_t *vec = NULL; - float idf = 0.0f; - const char *tok = cbm_sem_corpus_token_at(corpus, t, &vec, &idf); - if (tok && vec && idf > 0.01f) { - /* Int8 quantize the enriched vector */ - uint8_t qvec[CBM_SEM_DIM]; - for (int d = 0; d < CBM_SEM_DIM; d++) { - float clamped = vec->v[d]; - if (clamped > 1.0f) { - clamped = 1.0f; - } - if (clamped < -1.0f) { - clamped = -1.0f; - } - qvec[d] = (uint8_t)(int8_t)(clamped * 127.0f); - } - cbm_gbuf_store_token_vector(gbuf, tok, qvec, CBM_SEM_DIM, idf); - } - } - cbm_log_info("pass.semantic.token_vectors", "count", - itoa_log(cbm_sem_corpus_token_count(corpus))); - } - CBM_PROF_END_N("semantic_edges", "3c_token_vec_export_seq", t_phase3c, - cbm_sem_corpus_token_count(corpus)); - - /* Phase 4a: Build per-function TF-IDF + RI vectors + int8 quantize (PARALLEL) */ - CBM_PROF_START(t_phase4a); - uint8_t *qvecs = malloc((size_t)func_count * CBM_SEM_DIM); - { - vec_build_ctx_t vc = { - .funcs = funcs, .all_tokens = all_tokens, .token_counts = token_counts, - .corpus = corpus, .qvecs = qvecs, .func_count = func_count, - }; - atomic_init(&vc.next_idx, 0); - cbm_parallel_for_opts_t opts = {.max_workers = worker_count, .force_pthreads = false}; - cbm_parallel_for(worker_count, vec_build_worker, &vc, opts); - } - CBM_PROF_END_N("semantic_edges", "4a_build_func_vec_parallel", t_phase4a, func_count); - - /* Phase 4b: Store pre-quantized vectors in gbuf (sequential — gbuf not thread-safe) */ - CBM_PROF_START(t_phase4b); - for (int f = 0; f < func_count; f++) { - cbm_gbuf_store_vector(gbuf, funcs[f].node_id, &qvecs[f * CBM_SEM_DIM], CBM_SEM_DIM); - } - free(qvecs); - CBM_PROF_END_N("semantic_edges", "4b_vec_store_gbuf_seq", t_phase4b, func_count); - - cbm_log_info("pass.semantic.vectors_stored", "count", itoa_log(func_count)); - - /* Phase 5a: Generate deterministic random hyperplanes */ - CBM_PROF_START(t_phase5a); - float (*hyperplanes)[CBM_SEM_DIM] = malloc(sizeof(float[NUM_HYPERPLANES][CBM_SEM_DIM])); - for (int h = 0; h < NUM_HYPERPLANES; h++) { - for (int d = 0; d < CBM_SEM_DIM; d++) { - uint64_t seed = XXH3_64bits_withSeed(&d, sizeof(d), (uint64_t)h * CBM_SEM_DIM); - hyperplanes[h][d] = ((float)(seed & UINT32_MAX) / (float)UINT32_MAX) - 0.5f; - } - } - CBM_PROF_END("semantic_edges", "5a_hyperplanes", t_phase5a); - - /* Phase 5b: Compute bit signatures (PARALLEL) */ - CBM_PROF_START(t_phase5b); - uint64_t *signatures = calloc((size_t)func_count, sizeof(uint64_t)); - { - sig_build_ctx_t sc = { - .funcs = funcs, .signatures = signatures, .hyperplanes = hyperplanes, - .func_count = func_count, - }; - atomic_init(&sc.next_idx, 0); - cbm_parallel_for_opts_t opts = {.max_workers = worker_count, .force_pthreads = false}; - cbm_parallel_for(worker_count, sig_build_worker, &sc, opts); - } - free(hyperplanes); - CBM_PROF_END_N("semantic_edges", "5b_lsh_signatures_parallel", t_phase5b, func_count); - - /* Phase 5c: Build band-based LSH buckets (sequential — bucket realloc not thread-safe) */ - CBM_PROF_START(t_phase5c); - typedef struct { - int *items; - int count; - int cap; - } sem_bucket_t; - - sem_bucket_t **band_buckets = calloc(SEM_LSH_BANDS, sizeof(sem_bucket_t *)); - for (int b = 0; b < SEM_LSH_BANDS; b++) { - band_buckets[b] = calloc(SEM_BUCKET_COUNT, sizeof(sem_bucket_t)); - } + *out_funcs = funcs; + *out_nodes = node_ptrs; + return func_count; +} +/* Phase 5c: partition functions into LSH buckets by their signature bands. + * Sequential because each bucket grows its `items` array via realloc. */ +static void phase5c_build_lsh_buckets(const uint64_t *signatures, int func_count, + sem_bucket_t **band_buckets) { for (int f = 0; f < func_count; f++) { for (int b = 0; b < SEM_LSH_BANDS; b++) { - /* Extract SEM_LSH_ROWS bits starting at bit b*SEM_LSH_ROWS */ int shift = b * SEM_LSH_ROWS; - uint32_t band_val = (uint32_t)((signatures[f] >> shift) & ((1ULL << SEM_LSH_ROWS) - 1)); - /* Hash to bucket (combine band_val with band index for better distribution) */ + uint32_t band_val = (uint32_t)((signatures[f] >> shift) & + ((PSE_ONE_ULL << SEM_LSH_ROWS) - PSE_ONE_ULL)); uint64_t bh = XXH3_64bits_withSeed(&band_val, sizeof(band_val), (uint64_t)b); uint32_t bucket_idx = (uint32_t)(bh & SEM_BUCKET_MASK); sem_bucket_t *bucket = &band_buckets[b][bucket_idx]; @@ -947,34 +925,12 @@ int cbm_pipeline_pass_semantic_edges(cbm_pipeline_ctx_t *ctx) { bucket->items[bucket->count++] = f; } } - CBM_PROF_END_N("semantic_edges", "5c_lsh_buckets_seq", t_phase5c, func_count); +} - cbm_log_info("pass.semantic.lsh_built", "functions", itoa_log(func_count), "bands", - itoa_log(SEM_LSH_BANDS)); - - /* Phase 6a: Score candidates (PARALLEL) — edges collected in per-worker buffers */ - CBM_PROF_START(t_phase6a); - int *edge_counts = calloc((size_t)func_count, sizeof(int)); - deferred_edge_buf_t *worker_bufs = calloc((size_t)worker_count, sizeof(deferred_edge_buf_t)); - for (int w = 0; w < worker_count; w++) { - deferred_buf_init(&worker_bufs[w]); - } - - { - score_ctx_t sc = { - .funcs = funcs, .signatures = signatures, .edge_counts = edge_counts, - .cfg = cfg, .func_count = func_count, - .band_buckets = (void *)band_buckets, - .worker_bufs = worker_bufs, .max_workers = worker_count, - }; - atomic_init(&sc.next_idx, 0); - cbm_parallel_for_opts_t opts = {.max_workers = worker_count, .force_pthreads = false}; - cbm_parallel_for(worker_count, score_worker, &sc, opts); - } - CBM_PROF_END_N("semantic_edges", "6a_score_parallel", t_phase6a, func_count); - - /* Phase 6b: Merge deferred edges into gbuf (sequential) */ - CBM_PROF_START(t_phase6b); +/* Phase 6b: serialize deferred edges from all worker buffers into the graph + * buffer (sequential because gbuf isn't thread-safe). */ +static int phase6b_merge_edges(cbm_gbuf_t *gbuf, deferred_edge_buf_t *worker_bufs, + int worker_count) { int total_edges = 0; for (int w = 0; w < worker_count; w++) { for (int e = 0; e < worker_bufs[w].count; e++) { @@ -982,41 +938,327 @@ int cbm_pipeline_pass_semantic_edges(cbm_pipeline_ctx_t *ctx) { char props[PROPS_BUF]; snprintf(props, sizeof(props), "{\"score\":%.3f,\"same_file\":%s}", de->score, de->same_file ? "true" : "false"); - cbm_gbuf_insert_edge(gbuf, de->source_id, de->target_id, - "SEMANTICALLY_RELATED", props); + cbm_gbuf_insert_edge(gbuf, de->source_id, de->target_id, "SEMANTICALLY_RELATED", props); total_edges++; } deferred_buf_free(&worker_bufs[w]); } - free(worker_bufs); - CBM_PROF_END_N("semantic_edges", "6b_edge_merge_seq", t_phase6b, total_edges); + return total_edges; +} - /* Phase 7: Cleanup */ - CBM_PROF_START(t_phase7); - /* Free LSH buckets */ +/* Phase 3c: export co-occurrence-enriched token vectors into the graph buffer + * so query-time lookups can use them without re-running the finalize step. */ +static void phase3c_export_token_vectors(cbm_gbuf_t *gbuf, cbm_sem_corpus_t *corpus) { + int tv_count = cbm_sem_corpus_token_count(corpus); + for (int t = 0; t < tv_count; t++) { + const cbm_sem_vec_t *vec = NULL; + float idf = 0.0F; + const char *tok = cbm_sem_corpus_token_at(corpus, t, &vec, &idf); + if (!tok || !vec || idf <= PSE_FLOW_WEIGHT) { + continue; + } + uint8_t qvec[CBM_SEM_DIM]; + for (int d = 0; d < CBM_SEM_DIM; d++) { + float clamped = vec->v[d]; + if (clamped > PSE_UNIT_POS) { + clamped = PSE_UNIT_POS; + } + if (clamped < -PSE_UNIT_POS) { + clamped = -PSE_UNIT_POS; + } + qvec[d] = (uint8_t)(int8_t)(clamped * PSE_INT8_MAX); + } + cbm_gbuf_store_token_vector(gbuf, tok, qvec, CBM_SEM_DIM, idf); + } + cbm_log_info("pass.semantic.token_vectors", "count", itoa_log(tv_count)); +} + +/* Phase 5a: generate NUM_HYPERPLANES × CBM_SEM_DIM deterministic random + * float hyperplanes seeded from XXH3 so signatures are reproducible. */ +static float (*phase5a_build_hyperplanes(void)) [CBM_SEM_DIM] { + float (*hyperplanes)[CBM_SEM_DIM] = malloc(sizeof(float[NUM_HYPERPLANES][CBM_SEM_DIM])); + if (!hyperplanes) { + return NULL; + } + for (int h = 0; h < NUM_HYPERPLANES; h++) { + for (int d = 0; d < CBM_SEM_DIM; d++) { + uint64_t seed = XXH3_64bits_withSeed(&d, sizeof(d), (uint64_t)h * CBM_SEM_DIM); + hyperplanes[h][d] = ((float)(seed & UINT32_MAX) / (float)UINT32_MAX) - PSE_ROUND_BIAS; + } + } + return hyperplanes; +} + +/* Phase 1b: decode per-function minhash/profile/API/type/deco vectors in + * parallel. Must be called after cbm_sem_ensure_ready() so the pretrained + * token map is initialized. */ +static void phase1b_decode_and_build(cbm_sem_func_t *funcs, const cbm_gbuf_node_t **node_ptrs, + const cbm_gbuf_t *gbuf, int func_count, int worker_count) { + if (func_count <= 0) { + return; + } + collect_ctx_t cc = { + .funcs = funcs, + .node_ptrs = node_ptrs, + .gbuf = gbuf, + .func_count = func_count, + }; + atomic_init(&cc.next_idx, 0); + cbm_parallel_for_opts_t opts = {.max_workers = worker_count, .force_pthreads = false}; + cbm_parallel_for(worker_count, collect_worker, &cc, opts); +} + +/* Phase 2: tokenize each function's metadata in parallel, filling + * all_tokens[] and token_counts[]. Caller allocates the arrays. */ +static void phase2_tokenize(const cbm_gbuf_node_t **node_ptrs, cbm_gbuf_t *gbuf, char **all_tokens, + int *token_counts, int func_count, int worker_count) { + tokenize_ctx_t tc = { + .node_ptrs = node_ptrs, + .gbuf = gbuf, + .all_tokens = all_tokens, + .token_counts = token_counts, + .func_count = func_count, + }; + atomic_init(&tc.next_idx, 0); + cbm_parallel_for_opts_t opts = {.max_workers = worker_count, .force_pthreads = false}; + cbm_parallel_for(worker_count, tokenize_worker, &tc, opts); +} + +/* Phase 4a: build per-function TF-IDF + RI vectors in parallel, producing + * int8-quantized qvecs for subsequent storage. Phase 4b runs sequentially + * to store them in gbuf because gbuf is not thread-safe. */ +static void phase4_build_and_store_vectors(cbm_gbuf_t *gbuf, cbm_sem_func_t *funcs, + char **all_tokens, int *token_counts, + cbm_sem_corpus_t *corpus, int func_count, + int worker_count) { + uint8_t *qvecs = malloc((size_t)func_count * CBM_SEM_DIM); + if (!qvecs) { + return; + } + vec_build_ctx_t vc = { + .funcs = funcs, + .all_tokens = all_tokens, + .token_counts = token_counts, + .corpus = corpus, + .qvecs = qvecs, + .func_count = func_count, + }; + atomic_init(&vc.next_idx, 0); + cbm_parallel_for_opts_t opts = {.max_workers = worker_count, .force_pthreads = false}; + cbm_parallel_for(worker_count, vec_build_worker, &vc, opts); + for (int f = 0; f < func_count; f++) { + cbm_gbuf_store_vector(gbuf, funcs[f].node_id, &qvecs[(ptrdiff_t)f * CBM_SEM_DIM], + CBM_SEM_DIM); + } + free(qvecs); +} + +/* Phase 5: hyperplane generation → signatures → LSH bucket population. + * Returns the malloc'd signatures array and band_buckets[] via out-params; + * caller frees both. */ +static void phase5_lsh_build(cbm_sem_func_t *funcs, int func_count, int worker_count, + uint64_t **out_signatures, sem_bucket_t ***out_buckets) { + float (*hyperplanes)[CBM_SEM_DIM] = phase5a_build_hyperplanes(); + uint64_t *signatures = calloc((size_t)func_count, sizeof(uint64_t)); + if (hyperplanes && signatures) { + sig_build_ctx_t sc = { + .funcs = funcs, + .signatures = signatures, + .hyperplanes = hyperplanes, + .func_count = func_count, + }; + atomic_init(&sc.next_idx, 0); + cbm_parallel_for_opts_t opts = {.max_workers = worker_count, .force_pthreads = false}; + cbm_parallel_for(worker_count, sig_build_worker, &sc, opts); + } + free(hyperplanes); + + sem_bucket_t **band_buckets = calloc(SEM_LSH_BANDS, sizeof(sem_bucket_t *)); + if (band_buckets) { + for (int b = 0; b < SEM_LSH_BANDS; b++) { + band_buckets[b] = calloc(SEM_BUCKET_COUNT, sizeof(sem_bucket_t)); + } + phase5c_build_lsh_buckets(signatures, func_count, band_buckets); + } + *out_signatures = signatures; + *out_buckets = band_buckets; +} + +/* Phase 6a: score candidate pairs in parallel and collect deferred edges. */ +static void phase6a_score_candidates(cbm_sem_func_t *funcs, uint64_t *signatures, int *edge_counts, + sem_bucket_t **band_buckets, cbm_sem_config_t cfg, + deferred_edge_buf_t *worker_bufs, int func_count, + int worker_count) { + score_ctx_t sc = { + .funcs = funcs, + .signatures = signatures, + .edge_counts = edge_counts, + .cfg = cfg, + .func_count = func_count, + .band_buckets = (void *)band_buckets, + .worker_bufs = worker_bufs, + .max_workers = worker_count, + }; + atomic_init(&sc.next_idx, 0); + cbm_parallel_for_opts_t opts = {.max_workers = worker_count, .force_pthreads = false}; + cbm_parallel_for(worker_count, score_worker, &sc, opts); +} + +/* Phase 7: free LSH bucket storage (items arrays and the per-band arrays). */ +static void free_lsh_buckets(sem_bucket_t **band_buckets) { + if (!band_buckets) { + return; + } for (int b = 0; b < SEM_LSH_BANDS; b++) { + if (!band_buckets[b]) { + continue; + } for (int h = 0; h < SEM_BUCKET_COUNT; h++) { free(band_buckets[b][h].items); } free(band_buckets[b]); } free(band_buckets); - free(signatures); +} - cbm_log_info("pass.done", "pass", "semantic_edges", "edges", itoa_log(total_edges)); +/* Phases 3a/3b/3c bundled: create corpus, batch-add docs, finalize, export + * enriched token vectors to the graph buffer. Returns the new corpus, which + * the caller must cbm_sem_corpus_free() later. */ +static cbm_sem_corpus_t *run_corpus_phase(cbm_gbuf_t *gbuf, char **all_tokens, int *token_counts, + int func_count) { + CBM_PROF_START(t_phase3a); + cbm_sem_corpus_t *corpus = cbm_sem_corpus_new(); + cbm_sem_corpus_add_docs_batch(corpus, all_tokens, token_counts, func_count, CBM_SEM_MAX_TOKENS); + CBM_PROF_END_N("semantic_edges", "3a_corpus_batch", t_phase3a, func_count); + CBM_PROF_START(t_phase3b); + cbm_sem_corpus_finalize(corpus); + CBM_PROF_END_N("semantic_edges", "3b_corpus_finalize_seq", t_phase3b, + cbm_sem_corpus_token_count(corpus)); + + CBM_PROF_START(t_phase3c); + phase3c_export_token_vectors(gbuf, corpus); + CBM_PROF_END_N("semantic_edges", "3c_token_vec_export_seq", t_phase3c, + cbm_sem_corpus_token_count(corpus)); + return corpus; +} + +/* Phases 6a/6b bundled: parallel scoring + sequential merge of deferred + * edges. Returns the number of emitted edges. */ +static int run_scoring_phase(cbm_gbuf_t *gbuf, cbm_sem_func_t *funcs, uint64_t *signatures, + sem_bucket_t **band_buckets, cbm_sem_config_t cfg, int func_count, + int worker_count) { + int *edge_counts = calloc((size_t)func_count, sizeof(int)); + deferred_edge_buf_t *worker_bufs = calloc((size_t)worker_count, sizeof(deferred_edge_buf_t)); + if (!edge_counts || !worker_bufs) { + free(edge_counts); + free(worker_bufs); + return 0; + } + for (int w = 0; w < worker_count; w++) { + deferred_buf_init(&worker_bufs[w]); + } + + CBM_PROF_START(t_phase6a); + phase6a_score_candidates(funcs, signatures, edge_counts, band_buckets, cfg, worker_bufs, + func_count, worker_count); + CBM_PROF_END_N("semantic_edges", "6a_score_parallel", t_phase6a, func_count); + + CBM_PROF_START(t_phase6b); + int total = phase6b_merge_edges(gbuf, worker_bufs, worker_count); + CBM_PROF_END_N("semantic_edges", "6b_edge_merge_seq", t_phase6b, total); + + free(worker_bufs); free(edge_counts); + return total; +} + +/* Free the per-function arrays malloc'd during phases 1b and 4a. */ +static void free_funcs_and_tokens(cbm_sem_func_t *funcs, int func_count, char **all_tokens, + const int *token_counts) { for (int f = 0; f < func_count; f++) { free(funcs[f].tfidf_indices); free(funcs[f].tfidf_weights); int tc = token_counts[f]; for (int t = 0; t < tc; t++) { - free(all_tokens[f * CBM_SEM_MAX_TOKENS + t]); + free(all_tokens[((ptrdiff_t)f * CBM_SEM_MAX_TOKENS) + t]); } } free(all_tokens); - free(token_counts); free(funcs); +} + +/* ── Pass entry point ────────────────────────────────────────────── */ + +int cbm_pipeline_pass_semantic_edges(cbm_pipeline_ctx_t *ctx) { + /* Controlled by pipeline mode (moderate/full), not env var */ + cbm_log_info("pass.start", "pass", "semantic_edges"); + + cbm_gbuf_t *gbuf = ctx->gbuf; + cbm_sem_config_t cfg = cbm_sem_get_config(); + + CBM_PROF_START(t_phase1a); + cbm_sem_func_t *funcs = NULL; + const cbm_gbuf_node_t **node_ptrs = NULL; + int func_count = phase1_scan_functions(gbuf, &funcs, &node_ptrs); + CBM_PROF_END_N("semantic_edges", "1a_scan_seq", t_phase1a, func_count); + + /* Phase 1b: Decode minhash + profile + build api/type/deco vectors (PARALLEL). */ + cbm_sem_ensure_ready(); + CBM_PROF_START(t_phase1b); + phase1b_decode_and_build(funcs, node_ptrs, gbuf, func_count, cbm_default_worker_count(false)); + CBM_PROF_END_N("semantic_edges", "1b_decode_build_parallel", t_phase1b, func_count); + cbm_log_info("pass.semantic.collected", "functions", itoa_log(func_count)); + + if (func_count < PSE_MIN_FUNCS_FOR_PAIR) { + free(funcs); + free(node_ptrs); + cbm_log_info("pass.done", "pass", "semantic_edges", "edges", "0"); + return 0; + } + + /* Phase 2: Tokenize all nodes (PARALLEL) */ + int worker_count = cbm_default_worker_count(false); + char **all_tokens = malloc((size_t)func_count * sizeof(char *) * CBM_SEM_MAX_TOKENS); + int *token_counts = calloc((size_t)func_count, sizeof(int)); + + CBM_PROF_START(t_phase2); + phase2_tokenize(node_ptrs, gbuf, all_tokens, token_counts, func_count, worker_count); + CBM_PROF_END_N("semantic_edges", "2_tokenize_parallel", t_phase2, func_count); + free(node_ptrs); + + /* Phase 3: Build corpus (batch add), finalize, export enriched token vectors. */ + cbm_sem_corpus_t *corpus = run_corpus_phase(gbuf, all_tokens, token_counts, func_count); + + /* Phase 4: Build per-function TF-IDF + RI vectors (PARALLEL) and store them. */ + CBM_PROF_START(t_phase4); + phase4_build_and_store_vectors(gbuf, funcs, all_tokens, token_counts, corpus, func_count, + worker_count); + CBM_PROF_END_N("semantic_edges", "4_build_and_store_vec", t_phase4, func_count); + + cbm_log_info("pass.semantic.vectors_stored", "count", itoa_log(func_count)); + + /* Phase 5: LSH hyperplanes → signatures → buckets. */ + CBM_PROF_START(t_phase5); + uint64_t *signatures = NULL; + sem_bucket_t **band_buckets = NULL; + phase5_lsh_build(funcs, func_count, worker_count, &signatures, &band_buckets); + CBM_PROF_END_N("semantic_edges", "5_lsh_build", t_phase5, func_count); + + cbm_log_info("pass.semantic.lsh_built", "functions", itoa_log(func_count), "bands", + itoa_log(SEM_LSH_BANDS)); + + /* Phase 6: Parallel scoring + sequential edge merge. */ + int total_edges = + run_scoring_phase(gbuf, funcs, signatures, band_buckets, cfg, func_count, worker_count); + + /* Phase 7: Cleanup */ + CBM_PROF_START(t_phase7); + free_lsh_buckets(band_buckets); + free(signatures); + cbm_log_info("pass.done", "pass", "semantic_edges", "edges", itoa_log(total_edges)); + free_funcs_and_tokens(funcs, func_count, all_tokens, token_counts); + free(token_counts); cbm_sem_corpus_free(corpus); CBM_PROF_END("semantic_edges", "7_cleanup", t_phase7); diff --git a/src/pipeline/pass_similarity.c b/src/pipeline/pass_similarity.c index 410691da9..f0c22ba70 100644 --- a/src/pipeline/pass_similarity.c +++ b/src/pipeline/pass_similarity.c @@ -18,6 +18,11 @@ #include "foundation/profile.h" #include "foundation/platform.h" + +enum { + SIM_EDGE_INIT_CAP = 256, + SIM_EDGE_GROW = 2, +}; #include "pipeline/worker_pool.h" #include @@ -142,18 +147,19 @@ typedef struct { int cap; } sim_edge_buf_t; -static void sim_edge_buf_push(sim_edge_buf_t *buf, int64_t src, int64_t tgt, - double jaccard, bool same_file) { +static void sim_edge_buf_push(sim_edge_buf_t *buf, int64_t src, int64_t tgt, double jaccard, + bool same_file) { if (buf->count >= buf->cap) { - int nc = buf->cap < 256 ? 256 : buf->cap * 2; + int nc = buf->cap < SIM_EDGE_INIT_CAP ? SIM_EDGE_INIT_CAP : buf->cap * SIM_EDGE_GROW; sim_deferred_edge_t *grown = realloc(buf->edges, (size_t)nc * sizeof(sim_deferred_edge_t)); - if (!grown) return; + if (!grown) { + return; + } buf->edges = grown; buf->cap = nc; } buf->edges[buf->count++] = (sim_deferred_edge_t){ - .source_id = src, .target_id = tgt, .jaccard = jaccard, .same_file = same_file - }; + .source_id = src, .target_id = tgt, .jaccard = jaccard, .same_file = same_file}; } typedef struct { @@ -174,12 +180,16 @@ static void sim_query_worker(int worker_id, void *ctx_ptr) { /* Thread-local candidate buffer (stack-allocated) */ const cbm_lsh_entry_t *cands[SIM_CAND_CAP]; - while (1) { - int i = atomic_fetch_add_explicit(&sc->next_idx, 1, memory_order_relaxed); - if (i >= sc->entry_count) break; + while (true) { + int i = atomic_fetch_add_explicit(&sc->next_idx, SKIP_ONE, memory_order_relaxed); + if (i >= sc->entry_count) { + break; + } int ec = atomic_load_explicit(&sc->edge_counts[i], memory_order_relaxed); - if (ec >= CBM_MINHASH_MAX_EDGES_PER_NODE) continue; + if (ec >= CBM_MINHASH_MAX_EDGES_PER_NODE) { + continue; + } const fp_entry_t *src = &sc->entries[i]; int cand_count = cbm_lsh_query_into(sc->lsh, &src->fp, cands, SIM_CAND_CAP); @@ -187,18 +197,28 @@ static void sim_query_worker(int worker_id, void *ctx_ptr) { int emitted = 0; for (int c = 0; c < cand_count; c++) { const cbm_lsh_entry_t *cand = cands[c]; - if (cand->node_id == src->node_id) continue; - if (strcmp(src->ext, cand->file_ext) != 0) continue; - if (src->node_id >= cand->node_id) continue; + if (cand->node_id == src->node_id) { + continue; + } + if (strcmp(src->ext, cand->file_ext) != 0) { + continue; + } + if (src->node_id >= cand->node_id) { + continue; + } int cur = atomic_load_explicit(&sc->edge_counts[i], memory_order_relaxed); - if (cur + emitted >= CBM_MINHASH_MAX_EDGES_PER_NODE) break; + if (cur + emitted >= CBM_MINHASH_MAX_EDGES_PER_NODE) { + break; + } double jaccard = cbm_minhash_jaccard(&src->fp, cand->fingerprint); - if (jaccard < CBM_MINHASH_JACCARD_THRESHOLD) continue; + if (jaccard < CBM_MINHASH_JACCARD_THRESHOLD) { + continue; + } - bool same_file = src->file_path && cand->file_path && - strcmp(src->file_path, cand->file_path) == 0; + bool same_file = + src->file_path && cand->file_path && strcmp(src->file_path, cand->file_path) == 0; sim_edge_buf_push(my_buf, src->node_id, cand->node_id, jaccard, same_file); emitted++; } @@ -208,6 +228,24 @@ static void sim_query_worker(int worker_id, void *ctx_ptr) { } } +/* Merge worker edge buffers into gbuf. Returns total edge count. Frees worker buffers. */ +static int merge_sim_edges(cbm_gbuf_t *gbuf, sim_edge_buf_t *worker_bufs, int worker_count) { + int total = 0; + for (int w = 0; w < worker_count; w++) { + for (int e = 0; e < worker_bufs[w].count; e++) { + sim_deferred_edge_t *de = &worker_bufs[w].edges[e]; + char props[PROPS_BUF_LEN]; + snprintf(props, sizeof(props), "{\"jaccard\":%.3f,\"same_file\":%s}", de->jaccard, + de->same_file ? "true" : "false"); + cbm_gbuf_insert_edge(gbuf, de->source_id, de->target_id, "SIMILAR_TO", props); + total++; + } + free(worker_bufs[w].edges); + } + free(worker_bufs); + return total; +} + /* ── Pass entry point ────────────────────────────────────────────── */ int cbm_pipeline_pass_similarity(cbm_pipeline_ctx_t *ctx) { @@ -273,21 +311,8 @@ int cbm_pipeline_pass_similarity(cbm_pipeline_ctx_t *ctx) { } CBM_PROF_END_N("similarity", "3_query_parallel", t_query_emit, entry_count); - /* Merge deferred edges into gbuf (SEQUENTIAL — gbuf not thread-safe). */ CBM_PROF_START(t_merge); - int total_edges = 0; - for (int w = 0; w < worker_count; w++) { - for (int e = 0; e < worker_bufs[w].count; e++) { - sim_deferred_edge_t *de = &worker_bufs[w].edges[e]; - char props[PROPS_BUF_LEN]; - snprintf(props, sizeof(props), "{\"jaccard\":%.3f,\"same_file\":%s}", de->jaccard, - de->same_file ? "true" : "false"); - cbm_gbuf_insert_edge(gbuf, de->source_id, de->target_id, "SIMILAR_TO", props); - total_edges++; - } - free(worker_bufs[w].edges); - } - free(worker_bufs); + int total_edges = merge_sim_edges(gbuf, worker_bufs, worker_count); CBM_PROF_END_N("similarity", "4_edge_merge_seq", t_merge, total_edges); cbm_log_info("pass.done", "pass", "similarity", "edges", itoa_log(total_edges)); diff --git a/src/pipeline/pipeline.c b/src/pipeline/pipeline.c index f58c0c8d4..ed6fd6c40 100644 --- a/src/pipeline/pipeline.c +++ b/src/pipeline/pipeline.c @@ -410,47 +410,42 @@ static void cbm_pipeline_extract_infra_routes(cbm_gbuf_t *gbuf, const cbm_file_i } /* Run decorator_tags, configlink, and route matching passes. */ +typedef void (*predump_pass_fn)(cbm_pipeline_ctx_t *); +static void predump_deco(cbm_pipeline_ctx_t *ctx) { + cbm_pipeline_pass_decorator_tags(ctx->gbuf, ctx->project_name); +} +static void predump_route(cbm_pipeline_ctx_t *ctx) { + cbm_pipeline_create_route_nodes(ctx->gbuf); +} +static void predump_sim(cbm_pipeline_ctx_t *ctx) { + cbm_pipeline_pass_similarity(ctx); +} +static void predump_sem(cbm_pipeline_ctx_t *ctx) { + cbm_pipeline_pass_semantic_edges(ctx); +} +static void predump_cfg(cbm_pipeline_ctx_t *ctx) { + cbm_pipeline_pass_configlink(ctx); +} + static void run_predump_passes(cbm_pipeline_t *p, cbm_pipeline_ctx_t *ctx) { + static const struct { + predump_pass_fn fn; + const char *name; + bool moderate_only; /* true = skip in fast mode */ + } passes[] = { + {predump_deco, "decorator_tags", false}, {predump_cfg, "configlink", false}, + {predump_route, "route_match", false}, {predump_sim, "similarity", true}, + {predump_sem, "semantic_edges", true}, + }; + enum { PREDUMP_PASS_COUNT = 5 }; struct timespec t; - if (!check_cancel(p)) { + for (int i = 0; i < PREDUMP_PASS_COUNT && !check_cancel(p); i++) { + if (passes[i].moderate_only && p->mode > CBM_MODE_MODERATE) { + continue; + } cbm_clock_gettime(CLOCK_MONOTONIC, &t); - CBM_PROF_START(t_deco); - cbm_pipeline_pass_decorator_tags(p->gbuf, p->project_name); - CBM_PROF_END("pipeline", "pass_decorator_tags", t_deco); - cbm_log_info("pass.timing", "pass", "decorator_tags", "elapsed_ms", - itoa_buf((int)elapsed_ms(t))); - } - if (!check_cancel(p)) { - cbm_clock_gettime(CLOCK_MONOTONIC, &t); - CBM_PROF_START(t_cfg); - cbm_pipeline_pass_configlink(ctx); - CBM_PROF_END("pipeline", "pass_configlink", t_cfg); - cbm_log_info("pass.timing", "pass", "configlink", "elapsed_ms", - itoa_buf((int)elapsed_ms(t))); - } - if (!check_cancel(p)) { - cbm_clock_gettime(CLOCK_MONOTONIC, &t); - CBM_PROF_START(t_route); - cbm_pipeline_create_route_nodes(p->gbuf); - CBM_PROF_END("pipeline", "pass_route_match", t_route); - cbm_log_info("pass.timing", "pass", "route_match", "elapsed_ms", - itoa_buf((int)elapsed_ms(t))); - } - /* SIMILAR_TO + SEMANTICALLY_RELATED edges only in moderate/full modes */ - if (!check_cancel(p) && p->mode <= CBM_MODE_MODERATE) { - cbm_clock_gettime(CLOCK_MONOTONIC, &t); - CBM_PROF_START(t_sim); - cbm_pipeline_pass_similarity(ctx); - CBM_PROF_END("pipeline", "pass_similarity_total", t_sim); - cbm_log_info("pass.timing", "pass", "similarity", "elapsed_ms", - itoa_buf((int)elapsed_ms(t))); - } - if (!check_cancel(p) && p->mode <= CBM_MODE_MODERATE) { - cbm_clock_gettime(CLOCK_MONOTONIC, &t); - CBM_PROF_START(t_sem); - cbm_pipeline_pass_semantic_edges(ctx); - CBM_PROF_END("pipeline", "pass_semantic_edges_total", t_sem); - cbm_log_info("pass.timing", "pass", "semantic_edges", "elapsed_ms", + passes[i].fn(ctx); + cbm_log_info("pass.timing", "pass", passes[i].name, "elapsed_ms", itoa_buf((int)elapsed_ms(t))); } } @@ -659,11 +654,10 @@ static int dump_and_persist_hashes(cbm_pipeline_t *p, const cbm_file_info_t *fil * Falls back to plain names if cbm_camel_split is unavailable (which * shouldn't happen because we always register it, but we stay defensive). */ cbm_store_exec(hash_store, "INSERT INTO nodes_fts(nodes_fts) VALUES('delete-all');"); - if (cbm_store_exec( - hash_store, - "INSERT INTO nodes_fts(rowid, name, qualified_name, label, file_path) " - "SELECT id, cbm_camel_split(name), qualified_name, label, file_path " - "FROM nodes;") != CBM_STORE_OK) { + if (cbm_store_exec(hash_store, + "INSERT INTO nodes_fts(rowid, name, qualified_name, label, file_path) " + "SELECT id, cbm_camel_split(name), qualified_name, label, file_path " + "FROM nodes;") != CBM_STORE_OK) { cbm_store_exec(hash_store, "INSERT INTO nodes_fts(rowid, name, qualified_name, label, file_path) " "SELECT id, name, qualified_name, label, file_path FROM nodes;"); @@ -718,6 +712,74 @@ static int run_githistory(cbm_pipeline_t *p, cbm_pipeline_ctx_t *ctx) { /* ── Pipeline run ────────────────────────────────────────────────── */ +/* Run tests + git history. Returns 0 on success. */ +static int run_tests_and_history(cbm_pipeline_t *p, cbm_pipeline_ctx_t *ctx, + const cbm_file_info_t *files, int file_count) { + struct timespec t; + cbm_clock_gettime(CLOCK_MONOTONIC, &t); + CBM_PROF_START(t_tests); + int rc = cbm_pipeline_pass_tests(ctx, files, file_count); + CBM_PROF_END_N("pipeline", "pass_tests", t_tests, file_count); + cbm_log_info("pass.timing", "pass", "tests", "elapsed_ms", itoa_buf((int)elapsed_ms(t))); + if (rc == 0 && !check_cancel(p)) { + CBM_PROF_START(t_gh); + rc = run_githistory(p, ctx); + CBM_PROF_END("pipeline", "pass_githistory", t_gh); + } + if (check_cancel(p)) { + return CBM_NOT_FOUND; + } + return rc; +} + +/* Run tests, git history, predump passes, and dump+persist. */ +static int run_post_extraction(cbm_pipeline_t *p, cbm_pipeline_ctx_t *ctx, + const cbm_file_info_t *files, int file_count) { + int rc = run_tests_and_history(p, ctx, files, file_count); + if (rc != 0) { + return rc; + } + + CBM_PROF_START(t_predump); + run_predump_passes(p, ctx); + CBM_PROF_END("pipeline", "3_predump_passes_total", t_predump); + + if (!check_cancel(p)) { + struct timespec t; + CBM_PROF_START(t_dump); + rc = dump_and_persist_hashes(p, files, file_count, &t); + CBM_PROF_END("pipeline", "4_dump_and_persist", t_dump); + } + return rc; +} + +#define MIN_FILES_FOR_PARALLEL 50 + +/* Run structure + extraction passes (parallel or sequential). */ +static int run_extraction_phase(cbm_pipeline_t *p, cbm_pipeline_ctx_t *ctx, + const cbm_file_info_t *files, int file_count) { + struct timespec t; + cbm_clock_gettime(CLOCK_MONOTONIC, &t); + CBM_PROF_START(t_struct); + pass_structure(p, files, file_count); + CBM_PROF_END_N("pipeline", "pass_structure", t_struct, file_count); + cbm_log_info("pass.timing", "pass", "structure", "elapsed_ms", itoa_buf((int)elapsed_ms(t))); + if (check_cancel(p)) { + return CBM_NOT_FOUND; + } + + int worker_count = cbm_default_worker_count(true); + CBM_PROF_START(t_extract_total); + int rc = (worker_count > SKIP_ONE && file_count > MIN_FILES_FOR_PARALLEL) + ? run_parallel_pipeline(p, ctx, files, file_count, worker_count, &t) + : run_sequential_pipeline(p, ctx, files, file_count, &t); + CBM_PROF_END_N("pipeline", "2_extraction_total", t_extract_total, file_count); + if (check_cancel(p)) { + return CBM_NOT_FOUND; + } + return rc; +} + int cbm_pipeline_run(cbm_pipeline_t *p) { if (!p) { return CBM_NOT_FOUND; @@ -766,9 +828,6 @@ int cbm_pipeline_run(cbm_pipeline_t *p) { p->gbuf = cbm_gbuf_new(p->project_name, p->repo_path); p->registry = cbm_registry_new(); - /* Phase 3: Run passes */ - struct timespec t; - /* Build shared context for pass functions */ cbm_pipeline_ctx_t ctx = { .project_name = p->project_name, @@ -779,60 +838,12 @@ int cbm_pipeline_run(cbm_pipeline_t *p) { .mode = (int)p->mode, }; - cbm_clock_gettime(CLOCK_MONOTONIC, &t); - CBM_PROF_START(t_struct); - pass_structure(p, files, file_count); - CBM_PROF_END_N("pipeline", "pass_structure", t_struct, file_count); - cbm_log_info("pass.timing", "pass", "structure", "elapsed_ms", itoa_buf((int)elapsed_ms(t))); - if (check_cancel(p)) { - rc = CBM_NOT_FOUND; - goto cleanup; - } - - /* Run extraction passes (parallel or sequential) */ - int worker_count = cbm_default_worker_count(true); -#define MIN_FILES_FOR_PARALLEL 50 - CBM_PROF_START(t_extract_total); - rc = (worker_count > SKIP_ONE && file_count > MIN_FILES_FOR_PARALLEL) - ? run_parallel_pipeline(p, &ctx, files, file_count, worker_count, &t) - : run_sequential_pipeline(p, &ctx, files, file_count, &t); - CBM_PROF_END_N("pipeline", "2_extraction_total", t_extract_total, file_count); - if (check_cancel(p)) { - rc = CBM_NOT_FOUND; - } + rc = run_extraction_phase(p, &ctx, files, file_count); if (rc != 0) { goto cleanup; } - /* Post-extraction passes */ - cbm_clock_gettime(CLOCK_MONOTONIC, &t); - CBM_PROF_START(t_tests); - rc = cbm_pipeline_pass_tests(&ctx, files, file_count); - CBM_PROF_END_N("pipeline", "pass_tests", t_tests, file_count); - cbm_log_info("pass.timing", "pass", "tests", "elapsed_ms", itoa_buf((int)elapsed_ms(t))); - if (rc == 0 && !check_cancel(p)) { - CBM_PROF_START(t_gh); - rc = run_githistory(p, &ctx); - CBM_PROF_END("pipeline", "pass_githistory", t_gh); - } - if (check_cancel(p)) { - rc = CBM_NOT_FOUND; - } - if (rc != 0) { - goto cleanup; - } - - /* Pre-dump passes (operate on graph buffer, not store) */ - CBM_PROF_START(t_predump); - run_predump_passes(p, &ctx); - CBM_PROF_END("pipeline", "3_predump_passes_total", t_predump); - - /* Dump + persist */ - if (!check_cancel(p)) { - CBM_PROF_START(t_dump); - rc = dump_and_persist_hashes(p, files, file_count, &t); - CBM_PROF_END("pipeline", "4_dump_and_persist", t_dump); - } + rc = run_post_extraction(p, &ctx, files, file_count); if (rc != 0) { goto cleanup; } diff --git a/src/pipeline/pipeline.h b/src/pipeline/pipeline.h index 84c1f85ec..54c9288b8 100644 --- a/src/pipeline/pipeline.h +++ b/src/pipeline/pipeline.h @@ -86,6 +86,16 @@ char *cbm_pipeline_fqn_module(const char *project, const char *rel_path); /* Folder QN: project.dir.parts. Caller must free(). */ char *cbm_pipeline_fqn_folder(const char *project, const char *rel_dir); +/* Resolve an import specifier that uses a relative path (./foo, ../bar, .foo, + * or an unqualified local name like "foo.h") against the importing file's + * path. Returns a malloc'd normalized relative path without extension + * (e.g. "src/api/helpers") suitable for passing to cbm_pipeline_fqn_module, + * or NULL if the specifier is not a relative path (bare module names like + * "lodash", "django", "github.com/foo/bar" return NULL — the caller should + * treat those as external/unresolvable). Handles ".", "..", and leading + * dot-only segments used by Python relative imports. */ +char *cbm_pipeline_resolve_relative_import(const char *source_rel, const char *module_path); + /* Derive project name from an absolute path. * Replaces / and : with -, collapses --, trims leading -. * Caller must free() the returned string. */ diff --git a/src/pipeline/pipeline_incremental.c b/src/pipeline/pipeline_incremental.c index b0fed8fc7..ceeaaef9b 100644 --- a/src/pipeline/pipeline_incremental.c +++ b/src/pipeline/pipeline_incremental.c @@ -283,11 +283,10 @@ static void dump_and_persist(cbm_gbuf_t *gbuf, const char *db_path, const char * * rebuild from the nodes table here. See the full-dump path in * pipeline.c for the matching logic. */ cbm_store_exec(hash_store, "INSERT INTO nodes_fts(nodes_fts) VALUES('delete-all');"); - if (cbm_store_exec( - hash_store, - "INSERT INTO nodes_fts(rowid, name, qualified_name, label, file_path) " - "SELECT id, cbm_camel_split(name), qualified_name, label, file_path " - "FROM nodes;") != CBM_STORE_OK) { + if (cbm_store_exec(hash_store, + "INSERT INTO nodes_fts(rowid, name, qualified_name, label, file_path) " + "SELECT id, cbm_camel_split(name), qualified_name, label, file_path " + "FROM nodes;") != CBM_STORE_OK) { cbm_store_exec(hash_store, "INSERT INTO nodes_fts(rowid, name, qualified_name, label, file_path) " "SELECT id, name, qualified_name, label, file_path FROM nodes;"); diff --git a/src/semantic/ast_profile.c b/src/semantic/ast_profile.c index ac7733169..283180d86 100644 --- a/src/semantic/ast_profile.c +++ b/src/semantic/ast_profile.c @@ -12,7 +12,9 @@ #include "foundation/constants.h" #include "tree_sitter/api.h" -#include +#include +#include +#include #include #include @@ -24,6 +26,8 @@ enum { HALSTEAD_SET_MASK = 511, PROFILE_FIELD_COUNT = 25, DEPTH_SCALE = 10, + BASE_DECIMAL_AST = 10, + HALSTEAD_HASH_MUL = 31, }; typedef struct { @@ -105,7 +109,7 @@ static bool is_identifier(const char *k) { static bool halstead_insert(uint32_t *set, const char *key) { uint32_t h = 0; for (const char *p = key; *p; p++) { - h = h * 31 + (uint32_t)*p; + h = (h * HALSTEAD_HASH_MUL) + (uint32_t)*p; } uint32_t idx = h & HALSTEAD_SET_MASK; for (int probe = 0; probe < HALSTEAD_SET_SIZE; probe++) { @@ -137,6 +141,105 @@ static bool is_param_name(const char *ident, const char *source, const char **pa /* ── Main computation ────────────────────────────────────────────── */ +/* Count control-flow statement kinds (if/for/while/switch/try/return). */ +static void accumulate_control_flow(const char *kind, cbm_ast_profile_t *out, bool *in_return) { + if (is_control_if(kind)) { + out->if_count++; + } + if (is_control_for(kind)) { + out->for_count++; + } + if (is_control_while(kind)) { + out->while_count++; + } + if (is_control_switch(kind)) { + out->switch_count++; + } + if (is_control_try(kind)) { + out->try_count++; + } + if (is_return(kind)) { + out->return_count++; + *in_return = true; + } +} + +/* Count expression- and literal-class counters for one node. */ +static void accumulate_expressions(const char *kind, cbm_ast_profile_t *out) { + if (is_comparison(kind)) { + out->comparison_ops++; + } + if (is_arithmetic(kind)) { + out->arithmetic_ops++; + } + if (strcmp(kind, "not_operator") == 0 || strcmp(kind, "boolean_operator") == 0) { + out->logical_ops++; + } + if (is_assignment(kind)) { + out->assignment_count++; + out->variable_reassigns++; + } + if (is_string_lit(kind)) { + out->string_literals++; + } + if (is_number_lit(kind)) { + out->number_literals++; + } + if (is_bool_lit(kind)) { + out->bool_literals++; + } +} + +/* Accumulate Halstead operator/operand counts. Leaf identifiers, literals, + * and booleans are "operands"; recognized statement/expression kinds are + * "operators". Uses an open-addressed hash set for unique counting. */ +static void accumulate_halstead(const char *kind, uint32_t child_count, uint32_t *op_set, + uint32_t *operand_set, cbm_ast_profile_t *out) { + if (is_operator_node(kind)) { + out->total_operators++; + if (halstead_insert(op_set, kind)) { + out->unique_operators++; + } + } + if (child_count == 0 && + (is_identifier(kind) || is_string_lit(kind) || is_number_lit(kind) || is_bool_lit(kind))) { + out->total_operands++; + if (halstead_insert(operand_set, kind)) { + out->unique_operands++; + } + out->body_tokens++; + } +} + +/* Data-flow counter update: bump params_in_returns / params_in_conditions + * when the current leaf identifier names a parameter and we're inside the + * corresponding syntactic scope. */ +static void accumulate_data_flow(TSNode node, const char *kind, uint32_t child_count, + const char *source, const char **param_names, int param_count, + bool in_return, bool in_condition, cbm_ast_profile_t *out) { + if (!(child_count == 0 && is_identifier(kind) && source)) { + return; + } + uint32_t start = ts_node_start_byte(node); + uint32_t end = ts_node_end_byte(node); + if (end <= start || (end - start) >= CBM_SZ_128) { + return; + } + char ident_buf[CBM_SZ_128]; + int ilen = (int)(end - start); + memcpy(ident_buf, source + start, (size_t)ilen); + ident_buf[ilen] = '\0'; + if (!is_param_name(ident_buf, source, param_names, param_count)) { + return; + } + if (in_return) { + out->params_in_returns++; + } + if (in_condition) { + out->params_in_conditions++; + } +} + bool cbm_ast_profile_compute(TSNode func_body, const char *source, const char **param_names, int param_count, cbm_ast_profile_t *out) { if (ts_node_is_null(func_body)) { @@ -179,89 +282,11 @@ bool cbm_ast_profile_compute(TSNode func_body, const char *source, const char ** out->max_nesting_depth = (uint16_t)depth; } - /* Signal 8: Control flow */ - if (is_control_if(kind)) { - out->if_count++; - } - if (is_control_for(kind)) { - out->for_count++; - } - if (is_control_while(kind)) { - out->while_count++; - } - if (is_control_switch(kind)) { - out->switch_count++; - } - if (is_control_try(kind)) { - out->try_count++; - } - if (is_return(kind)) { - out->return_count++; - in_return = true; - } - - /* Signal 8: Expression types */ - if (is_comparison(kind)) { - out->comparison_ops++; - } - if (is_arithmetic(kind)) { - out->arithmetic_ops++; - } - if (strcmp(kind, "not_operator") == 0 || strcmp(kind, "boolean_operator") == 0) { - out->logical_ops++; - } - if (is_assignment(kind)) { - out->assignment_count++; - out->variable_reassigns++; - } - - /* Signal 8: Literals */ - if (is_string_lit(kind)) { - out->string_literals++; - } - if (is_number_lit(kind)) { - out->number_literals++; - } - if (is_bool_lit(kind)) { - out->bool_literals++; - } - - /* Signal 11: Halstead — operators vs operands */ - if (is_operator_node(kind)) { - out->total_operators++; - if (halstead_insert(op_set, kind)) { - out->unique_operators++; - } - } - if (child_count == 0 && (is_identifier(kind) || is_string_lit(kind) || - is_number_lit(kind) || is_bool_lit(kind))) { - out->total_operands++; - /* For unique operands, use the node type (normalized) not the text */ - if (halstead_insert(operand_set, kind)) { - out->unique_operands++; - } - out->body_tokens++; - } - - /* Signal 9: Approximate data flow — track if we're inside a return or condition */ - if (child_count == 0 && is_identifier(kind) && source) { - uint32_t start = ts_node_start_byte(node); - uint32_t end = ts_node_end_byte(node); - if (end > start && (end - start) < CBM_SZ_128) { - char ident_buf[CBM_SZ_128]; - int ilen = (int)(end - start); - memcpy(ident_buf, source + start, (size_t)ilen); - ident_buf[ilen] = '\0'; - if (is_param_name(ident_buf, source, param_names, param_count)) { - if (in_return) { - out->params_in_returns++; - } - if (in_condition) { - out->params_in_conditions++; - } - } - } - } + accumulate_control_flow(kind, out, &in_return); + accumulate_expressions(kind, out); + accumulate_halstead(kind, child_count, op_set, operand_set, out); + accumulate_data_flow(node, kind, child_count, source, param_names, param_count, in_return, + in_condition, out); /* Track context for data flow: are we inside a condition? */ if (is_control_if(kind) || is_control_while(kind)) { @@ -321,45 +346,39 @@ bool cbm_ast_profile_from_str(const char *str, cbm_ast_profile_t *out) { return false; } memset(out, 0, sizeof(*out)); - unsigned v[PROFILE_FIELD_COUNT]; - int n = sscanf(str, - "%u,%u,%u,%u,%u,%u,%u,%u," - "%u,%u,%u,%u," - "%u,%u,%u," - "%u,%u,%u,%u," - "%u,%u,%u,%u," - "%u,%u", - &v[0], &v[1], &v[2], &v[3], &v[4], &v[5], &v[6], &v[7], &v[8], &v[9], &v[10], - &v[11], &v[12], &v[13], &v[14], &v[15], &v[16], &v[17], &v[18], &v[19], &v[20], - &v[21], &v[22], &v[23], &v[24]); - if (n != PROFILE_FIELD_COUNT) { - return false; + /* Field layout must match cbm_ast_profile_to_str() exactly. */ + uint16_t *fields[PROFILE_FIELD_COUNT] = { + &out->if_count, &out->for_count, + &out->while_count, &out->switch_count, + &out->try_count, &out->return_count, + &out->max_nesting_depth, &out->avg_nesting_depth_x10, + &out->comparison_ops, &out->arithmetic_ops, + &out->logical_ops, &out->assignment_count, + &out->string_literals, &out->number_literals, + &out->bool_literals, &out->param_count, + &out->params_in_returns, &out->params_in_conditions, + &out->variable_reassigns, &out->unique_operators, + &out->unique_operands, &out->total_operators, + &out->total_operands, &out->body_lines, + &out->body_tokens, + }; + const char *p = str; + for (int i = 0; i < PROFILE_FIELD_COUNT; i++) { + char *end = NULL; + unsigned long parsed = strtoul(p, &end, BASE_DECIMAL_AST); + if (end == p) { + return false; + } + *fields[i] = (uint16_t)parsed; + p = end; + /* Skip the comma separator between fields. */ + if (i + SKIP_ONE < PROFILE_FIELD_COUNT) { + if (*p != ',') { + return false; + } + p++; + } } - out->if_count = (uint16_t)v[0]; - out->for_count = (uint16_t)v[1]; - out->while_count = (uint16_t)v[2]; - out->switch_count = (uint16_t)v[3]; - out->try_count = (uint16_t)v[4]; - out->return_count = (uint16_t)v[5]; - out->max_nesting_depth = (uint16_t)v[6]; - out->avg_nesting_depth_x10 = (uint16_t)v[7]; - out->comparison_ops = (uint16_t)v[8]; - out->arithmetic_ops = (uint16_t)v[9]; - out->logical_ops = (uint16_t)v[10]; - out->assignment_count = (uint16_t)v[11]; - out->string_literals = (uint16_t)v[12]; - out->number_literals = (uint16_t)v[13]; - out->bool_literals = (uint16_t)v[14]; - out->param_count = (uint16_t)v[15]; - out->params_in_returns = (uint16_t)v[16]; - out->params_in_conditions = (uint16_t)v[17]; - out->variable_reassigns = (uint16_t)v[18]; - out->unique_operators = (uint16_t)v[19]; - out->unique_operands = (uint16_t)v[20]; - out->total_operators = (uint16_t)v[21]; - out->total_operands = (uint16_t)v[22]; - out->body_lines = (uint16_t)v[23]; - out->body_tokens = (uint16_t)v[24]; return true; } diff --git a/src/semantic/semantic.c b/src/semantic/semantic.c index 3c35d2c88..4e613de60 100644 --- a/src/semantic/semantic.c +++ b/src/semantic/semantic.c @@ -37,26 +37,95 @@ enum { RI_SEED_BASE = 0x52494E44, /* "RIND" */ }; +/* Default signal weights for cbm_sem_combined_score. Must sum to ~1.0; + * proximity is a multiplier applied on top. */ +#define CBM_SEM_W_TFIDF 0.20F +#define CBM_SEM_W_RI 0.25F +#define CBM_SEM_W_MINHASH 0.10F +#define CBM_SEM_W_API 0.15F +#define CBM_SEM_W_TYPE 0.10F +#define CBM_SEM_W_DECORATOR 0.05F +#define CBM_SEM_W_STRUCT_PROFILE 0.10F +#define CBM_SEM_W_DATAFLOW 0.05F + +/* Threshold bounds for CBM_SEMANTIC_THRESHOLD env override. */ +#define CBM_SEM_THRESHOLD_MIN 0.0F +#define CBM_SEM_THRESHOLD_MAX 1.0F + +/* Epsilon for denominator guards in cosine math. */ +#define CBM_SEM_DENOM_EPS 1e-10F + +/* Int8 quantization bounds for vector storage (maps [-1.0, 1.0] to [-127, 127]). */ +#define CBM_SEM_INT8_MAX 127.0F +#define CBM_SEM_INT8_MIN (-127.0F) + +/* Unit vector bounds (normalized similarity / cosine value limits). */ +#define CBM_SEM_UNIT_POS 1.0F +#define CBM_SEM_UNIT_NEG (-1.0F) + +/* Proximity boost: same-file gets 1.0 + CBM_SEM_PROX_MAX_BOOST, distant gets 1.0. */ +#define CBM_SEM_PROX_MAX_BOOST 0.10F + +/* Reflective Random Indexing blend factors for pass2 (context mixing). */ +#define CBM_SEM_RRI_ALPHA 0.3F +#define CBM_SEM_RRI_BETA 0.7F + +/* IDF smoothing constant (log base): log2(1 + docs / doc_freq). */ +#define CBM_SEM_IDF_BASE 0.5F + +/* Strings of exactly two parts / dimensions used for co-occurrence window math. */ +#define CBM_SEM_COOCCUR_STRIDE 2 + +/* Worker tile/chunk sizes for parallel cooccurrence passes. */ +enum { + CBM_SEM_WORKER_STACK_CAP = 256, + CBM_SEM_TILE_SIZE = 40, + CBM_SEM_SEEN_INIT_CAP = 256, + CBM_SEM_RESOLVE_CHUNK = 64, + CBM_SEM_COOCCUR_CHUNK = 32, + CBM_SEM_RRI_TILE = 128, + CBM_SEM_INT8_I_LO = -128, + CBM_SEM_INT8_I_HI = 127, + CBM_SEM_ATOMIC_INC = 1, + CBM_SEM_RI_NONZERO_COUNT = 8, +}; + +/* Mutex-init state machine for the lazy-initialized pretrained token map. + * State 0 = not started, 1 = initializing, 2 = initialized. */ +enum { + MTX_STATE_UNINIT = 0, + MTX_STATE_INITIALIZING = 1, + MTX_STATE_READY = 2, +}; + +/* Pretrained map ready flag (signals a one-shot atomic transition 0 → 1). */ +enum { MAP_READY = 1 }; + +/* Numeric conversion radix for strtol (base 10 decimal). */ +enum { BASE_DECIMAL = 10 }; + /* ── Configuration ───────────────────────────────────────────────── */ cbm_sem_config_t cbm_sem_get_config(void) { cbm_sem_config_t cfg = { - .w_tfidf = 0.20f, - .w_ri = 0.25f, - .w_minhash = 0.10f, - .w_api = 0.15f, - .w_type = 0.10f, - .w_decorator = 0.05f, - .w_struct_profile = 0.10f, - .w_dataflow = 0.05f, + .w_tfidf = CBM_SEM_W_TFIDF, + .w_ri = CBM_SEM_W_RI, + .w_minhash = CBM_SEM_W_MINHASH, + .w_api = CBM_SEM_W_API, + .w_type = CBM_SEM_W_TYPE, + .w_decorator = CBM_SEM_W_DECORATOR, + .w_struct_profile = CBM_SEM_W_STRUCT_PROFILE, + .w_dataflow = CBM_SEM_W_DATAFLOW, .threshold = (float)CBM_SEM_EDGE_THRESHOLD, .max_edges = CBM_SEM_MAX_EDGES, }; const char *thresh = getenv("CBM_SEMANTIC_THRESHOLD"); if (thresh) { - float t = (float)atof(thresh); - if (t > 0.0f && t <= 1.0f) { - cfg.threshold = t; + /* strtod reports errors via endptr; reject non-numeric input silently. */ + char *end = NULL; + double parsed = strtod(thresh, &end); + if (end != thresh && parsed > CBM_SEM_THRESHOLD_MIN && parsed <= CBM_SEM_THRESHOLD_MAX) { + cfg.threshold = (float)parsed; } } return cfg; @@ -69,6 +138,31 @@ bool cbm_sem_is_enabled(void) { /* ── Token extraction ────────────────────────────────────────────── */ +/* True for characters that terminate a token regardless of case. */ +static bool is_token_delim(char c) { + return c == '.' || c == '/' || c == '_' || c == '-' || c == ' ' || c == '(' || c == ')' || + c == ',' || c == ':'; +} + +/* True for a camelCase transition: uppercase letter preceded by a lowercase. */ +static bool is_camel_break(const char *name, int i) { + if (i <= 0) { + return false; + } + char c = name[i]; + char p = name[i - SKIP_ONE]; + return c >= 'A' && c <= 'Z' && p >= 'a' && p <= 'z'; +} + +/* Flush the current buffer as a token into out[]. */ +static void flush_token(char *buf, int *blen, char **out, int *count, int max_out) { + if (*blen > 0 && *count < max_out) { + buf[*blen] = '\0'; + out[(*count)++] = strdup(buf); + } + *blen = 0; +} + int cbm_sem_tokenize(const char *name, char **out, int max_out) { if (!name || !out || max_out <= 0) { return 0; @@ -79,19 +173,11 @@ int cbm_sem_tokenize(const char *name, char **out, int max_out) { for (int i = 0; name[i] && count < max_out; i++) { char c = name[i]; - /* Split on: dots, slashes, underscores, hyphens, camelCase transitions */ - bool is_split = (c == '.' || c == '/' || c == '_' || c == '-' || c == ' ' || c == '(' || - c == ')' || c == ',' || c == ':'); - bool is_camel = (i > 0 && c >= 'A' && c <= 'Z' && name[i - SKIP_ONE] >= 'a' && - name[i - SKIP_ONE] <= 'z'); - - if (is_split || is_camel) { - if (blen > 0) { - buf[blen] = '\0'; - out[count++] = strdup(buf); - blen = 0; - } - if (is_split) { + bool split = is_token_delim(c); + bool camel = is_camel_break(name, i); + if (split || camel) { + flush_token(buf, &blen, out, &count, max_out); + if (split) { continue; } } @@ -99,10 +185,7 @@ int cbm_sem_tokenize(const char *name, char **out, int max_out) { buf[blen++] = (char)tolower((unsigned char)c); } } - if (blen > 0 && count < max_out) { - buf[blen] = '\0'; - out[count++] = strdup(buf); - } + flush_token(buf, &blen, out, &count, max_out); /* Abbreviation expansion: add expanded forms for common code abbreviations. * "err" → also add "error", "ctx" → "context", etc. */ @@ -113,75 +196,162 @@ int cbm_sem_tokenize(const char *name, char **out, int max_out) { const char *expanded; } abbrevs[] = { /* Error/exception handling */ - {"err", "error"}, {"exc", "exception"}, {"ex", "exception"}, + {"err", "error"}, + {"exc", "exception"}, + {"ex", "exception"}, /* Context/config */ - {"ctx", "context"}, {"cfg", "config"}, {"conf", "configuration"}, - {"env", "environment"}, {"opt", "option"}, {"opts", "options"}, + {"ctx", "context"}, + {"cfg", "config"}, + {"conf", "configuration"}, + {"env", "environment"}, + {"opt", "option"}, + {"opts", "options"}, /* Request/response (HTTP, RPC) */ - {"req", "request"}, {"res", "response"}, {"resp", "response"}, - {"rsp", "response"}, {"hdr", "header"}, {"hdrs", "headers"}, + {"req", "request"}, + {"res", "response"}, + {"resp", "response"}, + {"rsp", "response"}, + {"hdr", "header"}, + {"hdrs", "headers"}, /* Strings/formatting */ - {"str", "string"}, {"fmt", "format"}, {"msg", "message"}, - {"txt", "text"}, {"lbl", "label"}, {"desc", "description"}, + {"str", "string"}, + {"fmt", "format"}, + {"msg", "message"}, + {"txt", "text"}, + {"lbl", "label"}, + {"desc", "description"}, /* Data structures */ - {"buf", "buffer"}, {"arr", "array"}, {"vec", "vector"}, - {"lst", "list"}, {"dict", "dictionary"}, {"tbl", "table"}, - {"stk", "stack"}, {"que", "queue"}, + {"buf", "buffer"}, + {"arr", "array"}, + {"vec", "vector"}, + {"lst", "list"}, + {"dict", "dictionary"}, + {"tbl", "table"}, + {"stk", "stack"}, + {"que", "queue"}, /* Functions/callbacks */ - {"fn", "function"}, {"func", "function"}, {"cb", "callback"}, - {"proc", "procedure"}, {"ctor", "constructor"},{"dtor", "destructor"}, + {"fn", "function"}, + {"func", "function"}, + {"cb", "callback"}, + {"proc", "procedure"}, + {"ctor", "constructor"}, + {"dtor", "destructor"}, /* Database/storage */ - {"db", "database"}, {"col", "column"}, {"tbl", "table"}, - {"stmt", "statement"}, {"txn", "transaction"}, {"trx", "transaction"}, + {"db", "database"}, + {"col", "column"}, + {"tbl", "table"}, + {"stmt", "statement"}, + {"txn", "transaction"}, + {"trx", "transaction"}, {"repo", "repository"}, /* Auth/security */ - {"auth", "authentication"}, {"authz", "authorization"}, {"perm", "permission"}, - {"cred", "credential"}, {"tok", "token"}, {"pwd", "password"}, + {"auth", "authentication"}, + {"authz", "authorization"}, + {"perm", "permission"}, + {"cred", "credential"}, + {"tok", "token"}, + {"pwd", "password"}, /* Values/types */ - {"val", "value"}, {"num", "number"}, {"int", "integer"}, - {"bool", "boolean"}, {"flt", "float"}, {"dbl", "double"}, + {"val", "value"}, + {"num", "number"}, + {"int", "integer"}, + {"bool", "boolean"}, + {"flt", "float"}, + {"dbl", "double"}, /* Indexing/iteration */ - {"idx", "index"}, {"iter", "iterator"}, {"elem", "element"}, - {"cnt", "count"}, {"len", "length"}, {"sz", "size"}, - {"pos", "position"}, {"off", "offset"}, {"cap", "capacity"}, + {"idx", "index"}, + {"iter", "iterator"}, + {"elem", "element"}, + {"cnt", "count"}, + {"len", "length"}, + {"sz", "size"}, + {"pos", "position"}, + {"off", "offset"}, + {"cap", "capacity"}, /* Lifecycle */ - {"init", "initialize"}, {"deinit", "deinitialize"}, {"alloc", "allocate"}, - {"dealloc", "deallocate"}, {"del", "delete"}, {"rm", "remove"}, + {"init", "initialize"}, + {"deinit", "deinitialize"}, + {"alloc", "allocate"}, + {"dealloc", "deallocate"}, + {"del", "delete"}, + {"rm", "remove"}, /* Implementation/interface */ - {"impl", "implementation"}, {"iface", "interface"}, {"abs", "abstract"}, + {"impl", "implementation"}, + {"iface", "interface"}, + {"abs", "abstract"}, {"decl", "declaration"}, /* Parameters/attributes */ - {"param", "parameter"}, {"arg", "argument"}, {"attr", "attribute"}, - {"prop", "property"}, {"ret", "return"}, + {"param", "parameter"}, + {"arg", "argument"}, + {"attr", "attribute"}, + {"prop", "property"}, + {"ret", "return"}, /* Source/destination */ - {"src", "source"}, {"dst", "destination"}, {"tgt", "target"}, - {"orig", "original"}, {"prev", "previous"}, {"cur", "current"}, - {"tmp", "temporary"}, {"temp", "temporary"}, + {"src", "source"}, + {"dst", "destination"}, + {"tgt", "target"}, + {"orig", "original"}, + {"prev", "previous"}, + {"cur", "current"}, + {"tmp", "temporary"}, + {"temp", "temporary"}, /* Networking/IO */ - {"conn", "connection"}, {"sess", "session"}, {"sock", "socket"}, - {"addr", "address"}, {"url", "uniform"}, {"srv", "server"}, - {"cli", "client"}, {"svc", "service"}, {"ep", "endpoint"}, + {"conn", "connection"}, + {"sess", "session"}, + {"sock", "socket"}, + {"addr", "address"}, + {"url", "uniform"}, + {"srv", "server"}, + {"cli", "client"}, + {"svc", "service"}, + {"ep", "endpoint"}, /* Management */ - {"mgr", "manager"}, {"ctrl", "controller"}, {"hdlr", "handler"}, - {"sched", "scheduler"}, {"disp", "dispatcher"}, {"reg", "registry"}, + {"mgr", "manager"}, + {"ctrl", "controller"}, + {"hdlr", "handler"}, + {"sched", "scheduler"}, + {"disp", "dispatcher"}, + {"reg", "registry"}, /* Async/concurrent */ - {"chan", "channel"}, {"sem", "semaphore"}, {"mtx", "mutex"}, - {"wg", "waitgroup"}, {"sig", "signal"}, {"evt", "event"}, - {"sub", "subscriber"}, {"pub", "publisher"}, + {"chan", "channel"}, + {"sem", "semaphore"}, + {"mtx", "mutex"}, + {"wg", "waitgroup"}, + {"sig", "signal"}, + {"evt", "event"}, + {"sub", "subscriber"}, + {"pub", "publisher"}, /* Testing */ - {"spec", "specification"}, {"mock", "mock"}, {"stub", "stub"}, + {"spec", "specification"}, + {"mock", "mock"}, + {"stub", "stub"}, {"assert", "assertion"}, /* Logging/monitoring */ - {"log", "logging"}, {"lvl", "level"}, {"dbg", "debug"}, - {"wrn", "warning"}, {"inf", "info"}, + {"log", "logging"}, + {"lvl", "level"}, + {"dbg", "debug"}, + {"wrn", "warning"}, + {"inf", "info"}, /* Time */ - {"ts", "timestamp"}, {"dur", "duration"}, {"ttl", "timetolive"}, + {"ts", "timestamp"}, + {"dur", "duration"}, + {"ttl", "timetolive"}, /* Miscellaneous */ - {"ver", "version"}, {"ns", "namespace"}, {"pkg", "package"}, - {"mod", "module"}, {"lib", "library"}, {"dep", "dependency"}, - {"ref", "reference"}, {"ptr", "pointer"}, {"obj", "object"}, - {"doc", "document"}, {"cmd", "command"}, {"ops", "operations"}, - {"util", "utility"}, {"hlp", "helper"}, {"ext", "extension"}, + {"ver", "version"}, + {"ns", "namespace"}, + {"pkg", "package"}, + {"mod", "module"}, + {"lib", "library"}, + {"dep", "dependency"}, + {"ref", "reference"}, + {"ptr", "pointer"}, + {"obj", "object"}, + {"doc", "document"}, + {"cmd", "command"}, + {"ops", "operations"}, + {"util", "utility"}, + {"hlp", "helper"}, + {"ext", "extension"}, {NULL, NULL}, }; int orig_count = count; @@ -201,19 +371,19 @@ int cbm_sem_tokenize(const char *name, char **out, int max_out) { float cbm_sem_cosine(const cbm_sem_vec_t *a, const cbm_sem_vec_t *b) { if (!a || !b) { - return 0.0f; + return 0.0F; } - float dot = 0.0f; - float mag_a = 0.0f; - float mag_b = 0.0f; + float dot = 0.0F; + float mag_a = 0.0F; + float mag_b = 0.0F; for (int i = 0; i < CBM_SEM_DIM; i++) { dot += a->v[i] * b->v[i]; mag_a += a->v[i] * a->v[i]; mag_b += b->v[i] * b->v[i]; } float denom = sqrtf(mag_a) * sqrtf(mag_b); - if (denom < 1e-10f) { - return 0.0f; + if (denom < CBM_SEM_DENOM_EPS) { + return 0.0F; } return dot / denom; } @@ -231,14 +401,16 @@ static void ensure_pretrained_map(void) { return; } /* First-time init of the mutex itself (also needs to be thread-safe) */ - int expected = 0; - if (atomic_compare_exchange_strong_explicit(&g_pretrained_mtx_init, &expected, 1, - memory_order_acq_rel, memory_order_acquire)) { + int expected = MTX_STATE_UNINIT; + if (atomic_compare_exchange_strong_explicit(&g_pretrained_mtx_init, &expected, + MTX_STATE_INITIALIZING, memory_order_acq_rel, + memory_order_acquire)) { cbm_mutex_init(&g_pretrained_mtx); - atomic_store_explicit(&g_pretrained_mtx_init, 2, memory_order_release); + atomic_store_explicit(&g_pretrained_mtx_init, MTX_STATE_READY, memory_order_release); } else { /* Spin until another thread finishes initializing the mutex */ - while (atomic_load_explicit(&g_pretrained_mtx_init, memory_order_acquire) != 2) { + while (atomic_load_explicit(&g_pretrained_mtx_init, memory_order_acquire) != + MTX_STATE_READY) { /* brief spin */ } } @@ -253,7 +425,7 @@ static void ensure_pretrained_map(void) { cbm_ht_set(g_pretrained_map, strdup(tok), strdup(idx_buf)); } } - atomic_store_explicit(&g_pretrained_ready, 1, memory_order_release); + atomic_store_explicit(&g_pretrained_ready, MAP_READY, memory_order_release); } cbm_mutex_unlock(&g_pretrained_mtx); } @@ -272,11 +444,12 @@ void cbm_sem_random_index(const char *token, cbm_sem_vec_t *out) { ensure_pretrained_map(); const char *idx_str = cbm_ht_get(g_pretrained_map, token); if (idx_str) { - int idx = atoi(idx_str); - if (idx >= 0 && idx < PRETRAINED_TOKEN_COUNT) { - const int8_t *pvec = pretrained_vec_at(idx); + char *end = NULL; + long idx = strtol(idx_str, &end, BASE_DECIMAL); + if (end != idx_str && idx >= 0 && idx < PRETRAINED_TOKEN_COUNT) { + const int8_t *pvec = pretrained_vec_at((int)idx); for (int d = 0; d < CBM_SEM_DIM && d < PRETRAINED_DIM; d++) { - out->v[d] = (float)pvec[d] / 127.0f; + out->v[d] = (float)pvec[d] / CBM_SEM_INT8_MAX; } return; } @@ -287,7 +460,7 @@ void cbm_sem_random_index(const char *token, cbm_sem_vec_t *out) { for (int i = 0; i < CBM_SEM_SPARSE_NNZE; i++) { uint64_t h = XXH3_64bits_withSeed(&i, sizeof(i), seed + RI_SEED_BASE); int pos = (int)(h % CBM_SEM_DIM); - float sign = (h & SKIP_ONE) ? 1.0f : -1.0f; + float sign = (h & SKIP_ONE) ? CBM_SEM_UNIT_POS : -CBM_SEM_UNIT_POS; out->v[pos] += sign; } } @@ -296,15 +469,15 @@ void cbm_sem_normalize(cbm_sem_vec_t *v) { if (!v) { return; } - float mag = 0.0f; + float mag = 0.0F; for (int i = 0; i < CBM_SEM_DIM; i++) { mag += v->v[i] * v->v[i]; } mag = sqrtf(mag); - if (mag < 1e-10f) { + if (mag < CBM_SEM_DENOM_EPS) { return; } - float inv = 1.0f / mag; + float inv = CBM_SEM_UNIT_POS / mag; for (int i = 0; i < CBM_SEM_DIM; i++) { v->v[i] *= inv; } @@ -345,13 +518,15 @@ static int corpus_get_or_add(cbm_sem_corpus_t *c, const char *token) { char idx_buf[CBM_SZ_16]; const char *existing = cbm_ht_get(c->token_map, token); if (existing) { - return atoi(existing); + char *end = NULL; + long parsed = strtol(existing, &end, BASE_DECIMAL); + return (end != existing) ? (int)parsed : CBM_NOT_FOUND; } if (c->entry_count >= c->entry_cap) { - int new_cap = c->entry_cap < CORPUS_INIT_CAP ? CORPUS_INIT_CAP : c->entry_cap * 2; + int new_cap = c->entry_cap < CORPUS_INIT_CAP ? CORPUS_INIT_CAP : c->entry_cap * PAIR_LEN; corpus_entry_t *grown = realloc(c->entries, (size_t)new_cap * sizeof(corpus_entry_t)); if (!grown) { - return -1; + return CBM_NOT_FOUND; } c->entries = grown; c->entry_cap = new_cap; @@ -379,9 +554,17 @@ void cbm_sem_corpus_add_doc(cbm_sem_corpus_t *corpus, const char **tokens, int c } /* Track document for co-occurrence pass */ if (corpus->doc_count >= corpus->doc_cap) { - int new_cap = corpus->doc_cap < DOC_TOKENS_INIT ? DOC_TOKENS_INIT : corpus->doc_cap * 2; - corpus->doc_token_ids = realloc(corpus->doc_token_ids, (size_t)new_cap * sizeof(int *)); - corpus->doc_token_counts = realloc(corpus->doc_token_counts, (size_t)new_cap * sizeof(int)); + int new_cap = + corpus->doc_cap < DOC_TOKENS_INIT ? DOC_TOKENS_INIT : corpus->doc_cap * PAIR_LEN; + int **grown_ids = realloc(corpus->doc_token_ids, (size_t)new_cap * sizeof(int *)); + int *grown_counts = realloc(corpus->doc_token_counts, (size_t)new_cap * sizeof(int)); + if (!grown_ids || !grown_counts) { + free(grown_ids); + free(grown_counts); + return; + } + corpus->doc_token_ids = grown_ids; + corpus->doc_token_counts = grown_counts; corpus->doc_cap = new_cap; } int doc_idx = corpus->doc_count++; @@ -436,60 +619,91 @@ typedef struct { _Atomic int next_idx; } batch_resolve_ctx_t; +/* Resolve one document: look up each token's global ID, fill the corpus + * doc_token_ids[d], and bump the per-token doc_freq counter atomically. The + * caller is responsible for ensuring `seen` has capacity for `count` ints + * before calling (the worker grows its per-thread scratch buffer). */ +static void batch_resolve_one_doc(batch_resolve_ctx_t *bc, int doc_index, int *seen) { + int count = bc->token_counts[doc_index]; + if (count <= 0) { + bc->corpus->doc_token_ids[doc_index] = NULL; + bc->corpus->doc_token_counts[doc_index] = 0; + return; + } + int *ids = malloc((size_t)count * sizeof(int)); + bc->corpus->doc_token_ids[doc_index] = ids; + bc->corpus->doc_token_counts[doc_index] = count; + + int seen_count = 0; + char **tokens = &bc->all_tokens[(ptrdiff_t)doc_index * bc->max_tokens]; + for (int i = 0; i < count; i++) { + const char *idx_str = cbm_ht_get(bc->corpus->token_map, tokens[i]); + int tid = CBM_NOT_FOUND; + if (idx_str) { + char *end = NULL; + long parsed = strtol(idx_str, &end, BASE_DECIMAL); + if (end != idx_str) { + tid = (int)parsed; + } + } + ids[i] = tid; + if (tid < 0) { + continue; + } + /* Unique-per-doc check for IDF */ + bool is_new = true; + for (int j = 0; j < seen_count; j++) { + if (seen[j] == tid) { + is_new = false; + break; + } + } + if (is_new) { + seen[seen_count++] = tid; + atomic_fetch_add_explicit(&bc->doc_freq_atomic[tid], CBM_SEM_ATOMIC_INC, + memory_order_relaxed); + } + } +} + static void batch_resolve_worker(int worker_id, void *ctx_ptr) { (void)worker_id; batch_resolve_ctx_t *bc = ctx_ptr; /* Per-worker scratch for unique-per-doc tracking */ - int local_seen_cap = 256; + int local_seen_cap = CBM_SEM_SEEN_INIT_CAP; int *seen = malloc((size_t)local_seen_cap * sizeof(int)); + if (!seen) { + return; + } - while (1) { - int start = atomic_fetch_add_explicit(&bc->next_idx, 64, memory_order_relaxed); - if (start >= bc->doc_count) break; - int end = start + 64; - if (end > bc->doc_count) end = bc->doc_count; - + while (true) { + int start = + atomic_fetch_add_explicit(&bc->next_idx, CBM_SEM_RESOLVE_CHUNK, memory_order_relaxed); + if (start >= bc->doc_count) { + break; + } + int end = start + CBM_SEM_RESOLVE_CHUNK; + if (end > bc->doc_count) { + end = bc->doc_count; + } for (int d = start; d < end; d++) { int count = bc->token_counts[d]; - if (count <= 0) { - bc->corpus->doc_token_ids[d] = NULL; - bc->corpus->doc_token_counts[d] = 0; - continue; - } if (count > local_seen_cap) { + int *grown = realloc(seen, (size_t)count * sizeof(int)); + if (!grown) { + continue; + } + seen = grown; local_seen_cap = count; - seen = realloc(seen, (size_t)local_seen_cap * sizeof(int)); - } - int *ids = malloc((size_t)count * sizeof(int)); - bc->corpus->doc_token_ids[d] = ids; - bc->corpus->doc_token_counts[d] = count; - - int seen_count = 0; - char **tokens = &bc->all_tokens[d * bc->max_tokens]; - for (int i = 0; i < count; i++) { - const char *idx_str = cbm_ht_get(bc->corpus->token_map, tokens[i]); - int tid = idx_str ? atoi(idx_str) : -1; - ids[i] = tid; - if (tid < 0) continue; - /* Unique-per-doc check for IDF */ - bool is_new = true; - for (int j = 0; j < seen_count; j++) { - if (seen[j] == tid) { is_new = false; break; } - } - if (is_new) { - seen[seen_count++] = tid; - atomic_fetch_add_explicit(&bc->doc_freq_atomic[tid], 1, - memory_order_relaxed); - } } + batch_resolve_one_doc(bc, d, seen); } } free(seen); } void cbm_sem_corpus_add_docs_batch(cbm_sem_corpus_t *corpus, char **all_tokens, - const int *token_counts, int doc_count, - int max_tokens_per_doc) { + const int *token_counts, int doc_count, int max_tokens_per_doc) { if (!corpus || !all_tokens || !token_counts || doc_count <= 0) { return; } @@ -498,9 +712,15 @@ void cbm_sem_corpus_add_docs_batch(cbm_sem_corpus_t *corpus, char **all_tokens, * Hash table mutation can't be parallelized; strdup+insert is the cost. */ if (corpus->doc_cap < corpus->doc_count + doc_count) { int new_cap = corpus->doc_count + doc_count; - corpus->doc_token_ids = realloc(corpus->doc_token_ids, (size_t)new_cap * sizeof(int *)); - corpus->doc_token_counts = - realloc(corpus->doc_token_counts, (size_t)new_cap * sizeof(int)); + int **grown_ids = realloc(corpus->doc_token_ids, (size_t)new_cap * sizeof(int *)); + int *grown_counts = realloc(corpus->doc_token_counts, (size_t)new_cap * sizeof(int)); + if (!grown_ids || !grown_counts) { + free(grown_ids); + free(grown_counts); + return; + } + corpus->doc_token_ids = grown_ids; + corpus->doc_token_counts = grown_counts; corpus->doc_cap = new_cap; } int base_doc = corpus->doc_count; @@ -508,7 +728,7 @@ void cbm_sem_corpus_add_docs_batch(cbm_sem_corpus_t *corpus, char **all_tokens, for (int d = 0; d < doc_count; d++) { int count = token_counts[d]; - char **tokens = &all_tokens[d * max_tokens_per_doc]; + char **tokens = &all_tokens[(ptrdiff_t)d * max_tokens_per_doc]; for (int i = 0; i < count; i++) { /* Inserts token into token_map if new; we discard return here — * Phase B will re-lookup in read-only mode to get the ID. */ @@ -526,7 +746,7 @@ void cbm_sem_corpus_add_docs_batch(cbm_sem_corpus_t *corpus, char **all_tokens, corpus->doc_count = base_doc; for (int d = 0; d < doc_count; d++) { int count = token_counts[d]; - char **tokens = &all_tokens[d * max_tokens_per_doc]; + char **tokens = &all_tokens[(ptrdiff_t)d * max_tokens_per_doc]; cbm_sem_corpus_add_doc(corpus, (const char **)tokens, count); } return; @@ -589,17 +809,16 @@ typedef struct { * less source traffic) is equivalent in wall time despite the conversion cost, * while saving 90 MB of binary size. */ typedef struct { - uint8_t is_sparse; /* 1 = sparse path, 0 = dense int8 reference */ - uint8_t nnz; /* number of nonzeros used in sparse path */ + uint8_t is_sparse; /* 1 = sparse path, 0 = dense int8 reference */ + uint8_t nnz; /* number of nonzeros used in sparse path */ uint16_t _pad; uint16_t indices[CBM_SEM_SPARSE_NNZE]; /* 8 * 2 = 16 bytes */ - float values[CBM_SEM_SPARSE_NNZE]; /* 8 * 4 = 32 bytes */ - const int8_t *dense_int8; /* points into PRETRAINED_VECTOR_BLOB */ + float values[CBM_SEM_SPARSE_NNZE]; /* 8 * 4 = 32 bytes */ + const int8_t *dense_int8; /* points into PRETRAINED_VECTOR_BLOB */ } cbm_sem_src_entry_t; /* Inline helper: initialize a target vector from a sparse/dense source. */ -static inline void sem_target_init_from_src(cbm_sem_vec_t *dst, - const cbm_sem_src_entry_t *src) { +static inline void sem_target_init_from_src(cbm_sem_vec_t *dst, const cbm_sem_src_entry_t *src) { memset(dst, 0, sizeof(*dst)); if (src->is_sparse) { for (int k = 0; k < src->nnz; k++) { @@ -607,7 +826,7 @@ static inline void sem_target_init_from_src(cbm_sem_vec_t *dst, } } else { const int8_t *s = src->dense_int8; - const float inv127 = 1.0f / 127.0f; + const float inv127 = CBM_SEM_UNIT_POS / CBM_SEM_INT8_MAX; for (int d = 0; d < CBM_SEM_DIM; d++) { dst->v[d] = inv127 * (float)s[d]; } @@ -617,15 +836,15 @@ static inline void sem_target_init_from_src(cbm_sem_vec_t *dst, /* Inline helper: add weighted source into target. * Sparse path: ~8 operations, ~48 bytes source memory traffic. * Dense path: 768 mul-adds with int8→float conversion, ~768 bytes traffic. */ -static inline void sem_vec_add_src_scaled(cbm_sem_vec_t *dst, - const cbm_sem_src_entry_t *src, float scale) { +static inline void sem_vec_add_src_scaled(cbm_sem_vec_t *dst, const cbm_sem_src_entry_t *src, + float scale) { if (src->is_sparse) { for (int k = 0; k < src->nnz; k++) { dst->v[src->indices[k]] += scale * src->values[k]; } } else { const int8_t *s = src->dense_int8; - const float mul = scale * (1.0f / 127.0f); + const float mul = scale * (CBM_SEM_UNIT_POS / CBM_SEM_INT8_MAX); for (int d = 0; d < CBM_SEM_DIM; d++) { dst->v[d] += mul * (float)s[d]; } @@ -649,46 +868,60 @@ typedef struct { int tile_size; /* targets per L2-resident tile */ } cooccur_sparse_ctx_t; +/* Accumulate co-occurrence context for a single target token into `target`. + * Reads neighbors within ±CBM_SEM_WINDOW positions across all documents that + * reference this token id via the reverse index. */ +static void cooccur_sparse_one_target(cooccur_sparse_ctx_t *cc, int tid, cbm_sem_vec_t *target) { + int occ_start = cc->rev->offsets[tid]; + int occ_end = cc->rev->offsets[tid + SKIP_ONE]; + for (int p = occ_start; p < occ_end; p++) { + int d = cc->rev->flat[p].doc_id; + int i = cc->rev->flat[p].pos; + int *ids = cc->doc_token_ids[d]; + int len = cc->doc_token_counts[d]; + for (int w = -CBM_SEM_WINDOW; w <= CBM_SEM_WINDOW; w++) { + if (w == 0) { + continue; + } + int j = i + w; + if (j < 0 || j >= len) { + continue; + } + int nid = ids[j]; + if (nid < 0) { + continue; + } + float weight = CBM_SEM_UNIT_POS / (float)abs(w); + sem_vec_add_src_scaled(target, &cc->src_entries[nid], weight); + } + } +} + static void cooccur_worker_sparse(int worker_id, void *ctx_ptr) { (void)worker_id; cooccur_sparse_ctx_t *cc = ctx_ptr; - while (1) { - int ci = atomic_fetch_add_explicit(&cc->next_chunk, 1, memory_order_relaxed); - if (ci >= cc->num_chunks) break; + while (true) { + int ci = + atomic_fetch_add_explicit(&cc->next_chunk, CBM_SEM_ATOMIC_INC, memory_order_relaxed); + if (ci >= cc->num_chunks) { + break; + } int chunk_start = ci * cc->chunk_size; int chunk_end = chunk_start + cc->chunk_size; - if (chunk_end > cc->entry_count) chunk_end = cc->entry_count; + if (chunk_end > cc->entry_count) { + chunk_end = cc->entry_count; + } /* Cache-blocked target tiling: process tile_size targets at a time so * their vectors stay resident in L2 cache during their accumulation. */ - for (int tile_start = chunk_start; tile_start < chunk_end; - tile_start += cc->tile_size) { + for (int tile_start = chunk_start; tile_start < chunk_end; tile_start += cc->tile_size) { int tile_end = tile_start + cc->tile_size; - if (tile_end > chunk_end) tile_end = chunk_end; - + if (tile_end > chunk_end) { + tile_end = chunk_end; + } for (int tid = tile_start; tid < tile_end; tid++) { - /* Init target from sparse/dense source */ - sem_target_init_from_src(&cc->entries[tid].enriched_vec, - &cc->src_entries[tid]); - cbm_sem_vec_t *target = &cc->entries[tid].enriched_vec; - - int occ_start = cc->rev->offsets[tid]; - int occ_end = cc->rev->offsets[tid + 1]; - for (int p = occ_start; p < occ_end; p++) { - int d = cc->rev->flat[p].doc_id; - int i = cc->rev->flat[p].pos; - int *ids = cc->doc_token_ids[d]; - int len = cc->doc_token_counts[d]; - for (int w = -CBM_SEM_WINDOW; w <= CBM_SEM_WINDOW; w++) { - if (w == 0) continue; - int j = i + w; - if (j < 0 || j >= len) continue; - int nid = ids[j]; - if (nid < 0) continue; - float weight = 1.0f / (float)abs(w); - sem_vec_add_src_scaled(target, &cc->src_entries[nid], weight); - } - } + sem_target_init_from_src(&cc->entries[tid].enriched_vec, &cc->src_entries[tid]); + cooccur_sparse_one_target(cc, tid, &cc->entries[tid].enriched_vec); } } } @@ -699,7 +932,7 @@ static void cooccur_worker_sparse(int worker_id, void *ctx_ptr) { * We quantize to int8 once (×127) to cut source memory traffic 4x. */ typedef struct { corpus_entry_t *entries; - const int8_t *pass1_q; /* [entry_count × CBM_SEM_DIM] int8 quantized pass1 */ + const int8_t *pass1_q; /* [entry_count × CBM_SEM_DIM] int8 quantized pass1 */ int **doc_token_ids; const int *doc_token_counts; const reverse_index_t *rev; @@ -711,53 +944,65 @@ typedef struct { int tile_size; } cooccur_int8_ctx_t; -static inline void sem_vec_add_int8_scaled(cbm_sem_vec_t *dst, const int8_t *src, - float scale) { - const float mul = scale * (1.0f / 127.0f); +static inline void sem_vec_add_int8_scaled(cbm_sem_vec_t *dst, const int8_t *src, float scale) { + const float mul = scale * (CBM_SEM_UNIT_POS / CBM_SEM_INT8_MAX); for (int d = 0; d < CBM_SEM_DIM; d++) { dst->v[d] += mul * (float)src[d]; } } +/* RRI pass 2 accumulator for a single target token, reading int8-quantized + * pass1 vectors as the source. */ +static void cooccur_int8_one_target(cooccur_int8_ctx_t *cc, int tid, cbm_sem_vec_t *target) { + int occ_start = cc->rev->offsets[tid]; + int occ_end = cc->rev->offsets[tid + SKIP_ONE]; + for (int p = occ_start; p < occ_end; p++) { + int d = cc->rev->flat[p].doc_id; + int i = cc->rev->flat[p].pos; + int *ids = cc->doc_token_ids[d]; + int len = cc->doc_token_counts[d]; + for (int w = -CBM_SEM_WINDOW; w <= CBM_SEM_WINDOW; w++) { + if (w == 0) { + continue; + } + int j = i + w; + if (j < 0 || j >= len) { + continue; + } + int nid = ids[j]; + if (nid < 0) { + continue; + } + float weight = CBM_SEM_UNIT_POS / (float)abs(w); + sem_vec_add_int8_scaled(target, &cc->pass1_q[(size_t)nid * CBM_SEM_DIM], weight); + } + } +} + static void cooccur_worker_int8(int worker_id, void *ctx_ptr) { (void)worker_id; cooccur_int8_ctx_t *cc = ctx_ptr; - while (1) { - int ci = atomic_fetch_add_explicit(&cc->next_chunk, 1, memory_order_relaxed); - if (ci >= cc->num_chunks) break; + while (true) { + int ci = + atomic_fetch_add_explicit(&cc->next_chunk, CBM_SEM_ATOMIC_INC, memory_order_relaxed); + if (ci >= cc->num_chunks) { + break; + } int chunk_start = ci * cc->chunk_size; int chunk_end = chunk_start + cc->chunk_size; - if (chunk_end > cc->entry_count) chunk_end = cc->entry_count; + if (chunk_end > cc->entry_count) { + chunk_end = cc->entry_count; + } - for (int tile_start = chunk_start; tile_start < chunk_end; - tile_start += cc->tile_size) { + for (int tile_start = chunk_start; tile_start < chunk_end; tile_start += cc->tile_size) { int tile_end = tile_start + cc->tile_size; - if (tile_end > chunk_end) tile_end = chunk_end; - + if (tile_end > chunk_end) { + tile_end = chunk_end; + } for (int tid = tile_start; tid < tile_end; tid++) { /* RRI pass 2 starts from zero (no self-init) */ memset(&cc->entries[tid].enriched_vec, 0, sizeof(cbm_sem_vec_t)); - cbm_sem_vec_t *target = &cc->entries[tid].enriched_vec; - - int occ_start = cc->rev->offsets[tid]; - int occ_end = cc->rev->offsets[tid + 1]; - for (int p = occ_start; p < occ_end; p++) { - int d = cc->rev->flat[p].doc_id; - int i = cc->rev->flat[p].pos; - int *ids = cc->doc_token_ids[d]; - int len = cc->doc_token_counts[d]; - for (int w = -CBM_SEM_WINDOW; w <= CBM_SEM_WINDOW; w++) { - if (w == 0) continue; - int j = i + w; - if (j < 0 || j >= len) continue; - int nid = ids[j]; - if (nid < 0) continue; - float weight = 1.0f / (float)abs(w); - sem_vec_add_int8_scaled(target, - &cc->pass1_q[(size_t)nid * CBM_SEM_DIM], - weight); - } - } + cooccur_int8_one_target(cc, tid, &cc->entries[tid].enriched_vec); } } } @@ -772,11 +1017,16 @@ typedef struct { static void normalize_worker(int worker_id, void *ctx_ptr) { (void)worker_id; norm_ctx_t *nc = ctx_ptr; - while (1) { - int start = atomic_fetch_add_explicit(&nc->next_idx, 256, memory_order_relaxed); - if (start >= nc->entry_count) break; - int end = start + 256; - if (end > nc->entry_count) end = nc->entry_count; + while (true) { + int start = atomic_fetch_add_explicit(&nc->next_idx, CBM_SEM_WORKER_STACK_CAP, + memory_order_relaxed); + if (start >= nc->entry_count) { + break; + } + int end = start + CBM_SEM_WORKER_STACK_CAP; + if (end > nc->entry_count) { + end = nc->entry_count; + } for (int i = start; i < end; i++) { cbm_sem_normalize(&nc->entries[i].enriched_vec); } @@ -793,16 +1043,22 @@ typedef struct { static void blend_worker(int worker_id, void *ctx_ptr) { (void)worker_id; blend_ctx_t *bc = ctx_ptr; - while (1) { - int start = atomic_fetch_add_explicit(&bc->next_idx, 256, memory_order_relaxed); - if (start >= bc->entry_count) break; - int end = start + 256; - if (end > bc->entry_count) end = bc->entry_count; + while (true) { + int start = atomic_fetch_add_explicit(&bc->next_idx, CBM_SEM_WORKER_STACK_CAP, + memory_order_relaxed); + if (start >= bc->entry_count) { + break; + } + int end = start + CBM_SEM_WORKER_STACK_CAP; + if (end > bc->entry_count) { + end = bc->entry_count; + } for (int i = start; i < end; i++) { cbm_sem_normalize(&bc->entries[i].enriched_vec); for (int d = 0; d < CBM_SEM_DIM; d++) { bc->entries[i].enriched_vec.v[d] = - 0.7f * bc->pass1[i].v[d] + 0.3f * bc->entries[i].enriched_vec.v[d]; + (CBM_SEM_RRI_BETA * bc->pass1[i].v[d]) + + (CBM_SEM_RRI_ALPHA * bc->entries[i].enriched_vec.v[d]); } } } @@ -822,22 +1078,23 @@ typedef struct { static void build_src_entry(const char *token, cbm_sem_src_entry_t *out) { memset(out, 0, sizeof(*out)); if (!token) { - out->is_sparse = 1; + out->is_sparse = SKIP_ONE; out->nnz = 0; return; } /* Dense path: direct int8 pointer into pretrained blob (zero-copy). */ const char *idx_str = cbm_ht_get(g_pretrained_map, token); if (idx_str) { - int idx = atoi(idx_str); - if (idx >= 0 && idx < PRETRAINED_TOKEN_COUNT) { + char *end = NULL; + long idx = strtol(idx_str, &end, BASE_DECIMAL); + if (end != idx_str && idx >= 0 && idx < PRETRAINED_TOKEN_COUNT) { out->is_sparse = 0; - out->dense_int8 = pretrained_vec_at(idx); + out->dense_int8 = pretrained_vec_at((int)idx); return; } } /* Sparse path: compute 8 hash positions with collision merging. */ - out->is_sparse = 1; + out->is_sparse = SKIP_ONE; uint16_t tmp_idx[CBM_SEM_SPARSE_NNZE]; float tmp_val[CBM_SEM_SPARSE_NNZE]; int count = 0; @@ -845,11 +1102,14 @@ static void build_src_entry(const char *token, cbm_sem_src_entry_t *out) { for (int i = 0; i < CBM_SEM_SPARSE_NNZE; i++) { uint64_t h = XXH3_64bits_withSeed(&i, sizeof(i), seed + RI_SEED_BASE); int pos = (int)(h % CBM_SEM_DIM); - float sign = (h & 1) ? 1.0f : -1.0f; + float sign = (h & SKIP_ONE) ? CBM_SEM_UNIT_POS : -CBM_SEM_UNIT_POS; /* Merge collisions */ - int found = -1; + int found = CBM_NOT_FOUND; for (int j = 0; j < count; j++) { - if (tmp_idx[j] == (uint16_t)pos) { found = j; break; } + if (tmp_idx[j] == (uint16_t)pos) { + found = j; + break; + } } if (found >= 0) { tmp_val[found] += sign; @@ -862,7 +1122,7 @@ static void build_src_entry(const char *token, cbm_sem_src_entry_t *out) { /* Filter zeros */ int nnz = 0; for (int j = 0; j < count; j++) { - if (tmp_val[j] != 0.0f) { + if (tmp_val[j] != 0.0F) { out->indices[nnz] = tmp_idx[j]; out->values[nnz] = tmp_val[j]; nnz++; @@ -874,11 +1134,16 @@ static void build_src_entry(const char *token, cbm_sem_src_entry_t *out) { static void src_build_worker(int worker_id, void *ctx_ptr) { (void)worker_id; src_build_ctx_t *sc = ctx_ptr; - while (1) { - int start = atomic_fetch_add_explicit(&sc->next_idx, 256, memory_order_relaxed); - if (start >= sc->entry_count) break; - int end = start + 256; - if (end > sc->entry_count) end = sc->entry_count; + while (true) { + int start = atomic_fetch_add_explicit(&sc->next_idx, CBM_SEM_WORKER_STACK_CAP, + memory_order_relaxed); + if (start >= sc->entry_count) { + break; + } + int end = start + CBM_SEM_WORKER_STACK_CAP; + if (end > sc->entry_count) { + end = sc->entry_count; + } for (int i = start; i < end; i++) { build_src_entry(sc->entries[i].token, &sc->src_entries[i]); } @@ -898,19 +1163,28 @@ typedef struct { static void pass1_quantize_worker(int worker_id, void *ctx_ptr) { (void)worker_id; pass1_quant_ctx_t *qc = ctx_ptr; - while (1) { - int start = atomic_fetch_add_explicit(&qc->next_idx, 128, memory_order_relaxed); - if (start >= qc->entry_count) break; - int end = start + 128; - if (end > qc->entry_count) end = qc->entry_count; + while (true) { + int start = + atomic_fetch_add_explicit(&qc->next_idx, CBM_SEM_RRI_TILE, memory_order_relaxed); + if (start >= qc->entry_count) { + break; + } + int end = start + CBM_SEM_RRI_TILE; + if (end > qc->entry_count) { + end = qc->entry_count; + } for (int i = start; i < end; i++) { const cbm_sem_vec_t *src = &qc->entries[i].enriched_vec; int8_t *dst = &qc->pass1_q[(size_t)i * CBM_SEM_DIM]; for (int d = 0; d < CBM_SEM_DIM; d++) { - float v = src->v[d] * 127.0f; - if (v > 127.0f) v = 127.0f; - if (v < -127.0f) v = -127.0f; - dst[d] = (int8_t)(v >= 0 ? v + 0.5f : v - 0.5f); + float v = src->v[d] * CBM_SEM_INT8_MAX; + if (v > CBM_SEM_INT8_MAX) { + v = CBM_SEM_INT8_MAX; + } + if (v < -CBM_SEM_INT8_MAX) { + v = -CBM_SEM_INT8_MAX; + } + dst[d] = (int8_t)(v >= 0 ? v + CBM_SEM_IDF_BASE : v - CBM_SEM_IDF_BASE); } } } @@ -919,11 +1193,16 @@ static void pass1_quantize_worker(int worker_id, void *ctx_ptr) { /* Build reverse index: token_id → list of (doc_id, position) pairs. * SEQUENTIAL (fast: just pointer arithmetic + flat array fill). */ static reverse_index_t *build_reverse_index(cbm_sem_corpus_t *corpus) { - reverse_index_t *rev = calloc(1, sizeof(reverse_index_t)); - if (!rev) return NULL; + reverse_index_t *rev = calloc(SKIP_ONE, sizeof(reverse_index_t)); + if (!rev) { + return NULL; + } /* Phase A: count occurrences per token */ - int *counts = calloc((size_t)(corpus->entry_count + 1), sizeof(int)); - if (!counts) { free(rev); return NULL; } + int *counts = calloc((size_t)corpus->entry_count + SKIP_ONE, sizeof(int)); + if (!counts) { + free(rev); + return NULL; + } long total = 0; for (int d = 0; d < corpus->doc_count; d++) { int *ids = corpus->doc_token_ids[d]; @@ -937,8 +1216,12 @@ static reverse_index_t *build_reverse_index(cbm_sem_corpus_t *corpus) { } } /* Phase B: exclusive prefix sum → offsets[] */ - rev->offsets = malloc((size_t)(corpus->entry_count + 1) * sizeof(int)); - if (!rev->offsets) { free(counts); free(rev); return NULL; } + rev->offsets = malloc(((size_t)corpus->entry_count + SKIP_ONE) * sizeof(int)); + if (!rev->offsets) { + free(counts); + free(rev); + return NULL; + } int running = 0; for (int t = 0; t < corpus->entry_count; t++) { rev->offsets[t] = running; @@ -946,9 +1229,16 @@ static reverse_index_t *build_reverse_index(cbm_sem_corpus_t *corpus) { counts[t] = 0; /* reuse as per-token fill cursor */ } rev->offsets[corpus->entry_count] = running; - /* Phase C: fill flat array */ - rev->flat = malloc((size_t)total * sizeof(cooccur_pos_t)); - if (!rev->flat) { free(rev->offsets); free(counts); free(rev); return NULL; } + /* Phase C: fill flat array. Ensure allocation size > 0 even for empty + * corpora (avoids malloc(0) which is implementation-defined). */ + size_t flat_bytes = (total > 0 ? (size_t)total : SKIP_ONE) * sizeof(cooccur_pos_t); + rev->flat = malloc(flat_bytes); + if (!rev->flat) { + free(rev->offsets); + free(counts); + free(rev); + return NULL; + } for (int d = 0; d < corpus->doc_count; d++) { int *ids = corpus->doc_token_ids[d]; int len = corpus->doc_token_counts[d]; @@ -966,12 +1256,113 @@ static reverse_index_t *build_reverse_index(cbm_sem_corpus_t *corpus) { } static void free_reverse_index(reverse_index_t *rev) { - if (!rev) return; + if (!rev) { + return; + } free(rev->offsets); free(rev->flat); free(rev); } +/* Bundle of parameters shared by the finalize sub-phases. */ +typedef struct { + cbm_sem_corpus_t *corpus; + reverse_index_t *rev; + cbm_sem_src_entry_t *src_entries; + int worker_count; + int num_chunks; + int chunk_size; + int tile_size; + cbm_parallel_for_opts_t opts; +} finalize_params_t; + +/* Sub-phase 1: build tagged source vectors (sparse or dense-int8) in parallel. */ +static void finalize_build_sources(finalize_params_t *p) { + src_build_ctx_t sc = { + .entries = p->corpus->entries, + .src_entries = p->src_entries, + .entry_count = p->corpus->entry_count, + }; + atomic_init(&sc.next_idx, 0); + cbm_parallel_for(p->worker_count, src_build_worker, &sc, p->opts); +} + +/* Sub-phases 2+3: co-occurrence pass 1 + normalize. */ +static void finalize_pass1(finalize_params_t *p) { + cooccur_sparse_ctx_t cc = { + .entries = p->corpus->entries, + .src_entries = p->src_entries, + .doc_token_ids = p->corpus->doc_token_ids, + .doc_token_counts = p->corpus->doc_token_counts, + .rev = p->rev, + .doc_count = p->corpus->doc_count, + .entry_count = p->corpus->entry_count, + .num_chunks = p->num_chunks, + .chunk_size = p->chunk_size, + .tile_size = p->tile_size, + }; + atomic_init(&cc.next_chunk, 0); + cbm_parallel_for(p->worker_count, cooccur_worker_sparse, &cc, p->opts); + + norm_ctx_t nc = {.entries = p->corpus->entries, .entry_count = p->corpus->entry_count}; + atomic_init(&nc.next_idx, 0); + cbm_parallel_for(p->worker_count, normalize_worker, &nc, p->opts); +} + +/* Sub-phases 4+5: quantize pass1 to int8, run RRI pass 2, blend + normalize. */ +static void finalize_pass2(finalize_params_t *p) { + int8_t *pass1_q = malloc((size_t)p->corpus->entry_count * CBM_SEM_DIM * sizeof(int8_t)); + if (pass1_q) { + pass1_quant_ctx_t qc = { + .entries = p->corpus->entries, + .pass1_q = pass1_q, + .entry_count = p->corpus->entry_count, + }; + atomic_init(&qc.next_idx, 0); + cbm_parallel_for(p->worker_count, pass1_quantize_worker, &qc, p->opts); + } + + cbm_sem_vec_t *pass1 = malloc((size_t)p->corpus->entry_count * sizeof(cbm_sem_vec_t)); + if (pass1) { + for (int i = 0; i < p->corpus->entry_count; i++) { + pass1[i] = p->corpus->entries[i].enriched_vec; + } + } + + if (pass1_q) { + cooccur_int8_ctx_t cc = { + .entries = p->corpus->entries, + .pass1_q = pass1_q, + .doc_token_ids = p->corpus->doc_token_ids, + .doc_token_counts = p->corpus->doc_token_counts, + .rev = p->rev, + .doc_count = p->corpus->doc_count, + .entry_count = p->corpus->entry_count, + .num_chunks = p->num_chunks, + .chunk_size = p->chunk_size, + .tile_size = p->tile_size, + }; + atomic_init(&cc.next_chunk, 0); + cbm_parallel_for(p->worker_count, cooccur_worker_int8, &cc, p->opts); + } + + if (pass1) { + blend_ctx_t bc = { + .entries = p->corpus->entries, + .pass1 = pass1, + .entry_count = p->corpus->entry_count, + }; + atomic_init(&bc.next_idx, 0); + cbm_parallel_for(p->worker_count, blend_worker, &bc, p->opts); + free(pass1); + } + free(pass1_q); + + norm_ctx_t nc = {.entries = p->corpus->entries, .entry_count = p->corpus->entry_count}; + atomic_init(&nc.next_idx, 0); + cbm_parallel_for(p->worker_count, normalize_worker, &nc, p->opts); +} + void cbm_sem_corpus_finalize(cbm_sem_corpus_t *corpus) { if (!corpus || corpus->finalized) { return; @@ -983,28 +1374,21 @@ void cbm_sem_corpus_finalize(cbm_sem_corpus_t *corpus) { int worker_count = cbm_default_worker_count(false); cbm_parallel_for_opts_t opts = {.max_workers = worker_count, .force_pthreads = false}; - /* Finer chunks = better load balancing for skewed token distributions. - * 32x more chunks than workers: rare chunks finish fast, workers then - * steal from popular chunks that contain hot tokens. */ - int num_chunks = worker_count * 32; - if (num_chunks > corpus->entry_count) num_chunks = corpus->entry_count; - if (num_chunks < 1) num_chunks = 1; - int chunk_size = (corpus->entry_count + num_chunks - 1) / num_chunks; + /* Finer chunks = better load balancing for skewed token distributions. */ + int num_chunks = worker_count * CBM_SEM_COOCCUR_CHUNK; + if (num_chunks > corpus->entry_count) { + num_chunks = corpus->entry_count; + } + if (num_chunks < SKIP_ONE) { + num_chunks = SKIP_ONE; + } + int chunk_size = (corpus->entry_count + num_chunks - SKIP_ONE) / num_chunks; - /* Sub-phase 0: Build reverse index (token → occurrences) — SEQUENTIAL but fast. */ - CBM_PROF_START(t_rev_idx); reverse_index_t *rev = build_reverse_index(corpus); - CBM_PROF_END_N("corpus_finalize", "0_reverse_index_seq", t_rev_idx, corpus->entry_count); if (!rev) { corpus->finalized = true; return; } - - /* Sub-phase 1: Build src_entries[] — sparse/dense-int8 tagged sources (PARALLEL). - * Replaces the old dense float base_ri[]. Out-of-vocab tokens use sparse inline - * representation (8 nonzeros). Nomic-vocab tokens reference pretrained int8 - * directly (no copy). ~40x less memory for sparse, ~4x less for dense vs float. */ - CBM_PROF_START(t_src_build); cbm_sem_src_entry_t *src_entries = calloc((size_t)corpus->entry_count, sizeof(cbm_sem_src_entry_t)); if (!src_entries) { @@ -1012,146 +1396,48 @@ void cbm_sem_corpus_finalize(cbm_sem_corpus_t *corpus) { corpus->finalized = true; return; } - { - src_build_ctx_t sc = { - .entries = corpus->entries, - .src_entries = src_entries, - .entry_count = corpus->entry_count, - }; - atomic_init(&sc.next_idx, 0); - cbm_parallel_for(worker_count, src_build_worker, &sc, opts); - } - CBM_PROF_END_N("corpus_finalize", "1_src_build_parallel", t_src_build, - corpus->entry_count); - /* Cache-blocked target tiling: tile_size targets stay resident in L2 during - * their accumulation. Each cbm_sem_vec_t is 3KB, so ~40 targets ≈ 120KB fits - * in L1 on most platforms and L2 comfortably. */ - int tile_size = 40; - - /* Sub-phase 2: Co-occurrence pass 1 (PARALLEL, sparse+int8 sources). - * For 80% of tokens (out-of-vocab), inner add is just 8 scalar updates. - * For 20% dense tokens, inner add reads int8 directly from pretrained blob. - * Reverse index iteration = zero wasted outer scan. */ - CBM_PROF_START(t_cooc1); - { - cooccur_sparse_ctx_t cc = { - .entries = corpus->entries, - .src_entries = src_entries, - .doc_token_ids = corpus->doc_token_ids, - .doc_token_counts = corpus->doc_token_counts, - .rev = rev, - .doc_count = corpus->doc_count, - .entry_count = corpus->entry_count, - .num_chunks = num_chunks, - .chunk_size = chunk_size, - .tile_size = tile_size, - }; - atomic_init(&cc.next_chunk, 0); - cbm_parallel_for(worker_count, cooccur_worker_sparse, &cc, opts); - } - CBM_PROF_END_N("corpus_finalize", "2_cooccur_pass1_sparse", t_cooc1, corpus->doc_count); - - /* Sub-phase 3: Normalize pass 1 (PARALLEL per-entry). */ - CBM_PROF_START(t_norm1); - { - norm_ctx_t nc = {.entries = corpus->entries, .entry_count = corpus->entry_count}; - atomic_init(&nc.next_idx, 0); - cbm_parallel_for(worker_count, normalize_worker, &nc, opts); - } - CBM_PROF_END_N("corpus_finalize", "3_normalize_pass1_parallel", t_norm1, corpus->entry_count); - - /* Sub-phase 4a: Quantize pass1 to int8 for RRI (PARALLEL). - * Pass1 values are in [-1, 1] after normalization — int8 with scale 127 - * preserves ~99% precision and cuts pass2 memory traffic 4x. */ - CBM_PROF_START(t_quant); - int8_t *pass1_q = malloc((size_t)corpus->entry_count * CBM_SEM_DIM * sizeof(int8_t)); - if (pass1_q) { - pass1_quant_ctx_t qc = { - .entries = corpus->entries, - .pass1_q = pass1_q, - .entry_count = corpus->entry_count, - }; - atomic_init(&qc.next_idx, 0); - cbm_parallel_for(worker_count, pass1_quantize_worker, &qc, opts); - } - CBM_PROF_END_N("corpus_finalize", "4a_quantize_pass1_parallel", t_quant, - corpus->entry_count); - - /* Also save float pass1 for the final blend step. */ - cbm_sem_vec_t *pass1 = malloc((size_t)corpus->entry_count * sizeof(cbm_sem_vec_t)); - if (pass1) { - for (int i = 0; i < corpus->entry_count; i++) { - pass1[i] = corpus->entries[i].enriched_vec; - } - } - - /* Sub-phase 4b: RRI pass 2 (PARALLEL, int8 source). */ - CBM_PROF_START(t_rri); - if (pass1_q) { - cooccur_int8_ctx_t cc = { - .entries = corpus->entries, - .pass1_q = pass1_q, - .doc_token_ids = corpus->doc_token_ids, - .doc_token_counts = corpus->doc_token_counts, - .rev = rev, - .doc_count = corpus->doc_count, - .entry_count = corpus->entry_count, - .num_chunks = num_chunks, - .chunk_size = chunk_size, - .tile_size = tile_size, - }; - atomic_init(&cc.next_chunk, 0); - cbm_parallel_for(worker_count, cooccur_worker_int8, &cc, opts); - } - CBM_PROF_END_N("corpus_finalize", "4b_rri_pass2_int8", t_rri, corpus->doc_count); - - /* Sub-phase 5: Blend pass1 + pass2 (PARALLEL per-entry). - * final = 0.7 * pass1 + 0.3 * normalize(pass2) */ - CBM_PROF_START(t_blend); - if (pass1) { - blend_ctx_t bc = { - .entries = corpus->entries, - .pass1 = pass1, - .entry_count = corpus->entry_count, - }; - atomic_init(&bc.next_idx, 0); - cbm_parallel_for(worker_count, blend_worker, &bc, opts); - free(pass1); - } - free(pass1_q); - CBM_PROF_END_N("corpus_finalize", "5_blend_parallel", t_blend, corpus->entry_count); - - /* Sub-phase 6: Final normalization (PARALLEL per-entry). */ - CBM_PROF_START(t_final_norm); - { - norm_ctx_t nc = {.entries = corpus->entries, .entry_count = corpus->entry_count}; - atomic_init(&nc.next_idx, 0); - cbm_parallel_for(worker_count, normalize_worker, &nc, opts); - } - CBM_PROF_END_N("corpus_finalize", "6_normalize_final_parallel", t_final_norm, - corpus->entry_count); + finalize_params_t params = { + .corpus = corpus, + .rev = rev, + .src_entries = src_entries, + .worker_count = worker_count, + .num_chunks = num_chunks, + .chunk_size = chunk_size, + .tile_size = CBM_SEM_TILE_SIZE, + .opts = opts, + }; + finalize_build_sources(¶ms); + finalize_pass1(¶ms); + finalize_pass2(¶ms); free(src_entries); free_reverse_index(rev); corpus->finalized = true; } +/* Parse a decimal index string via strtol, returning CBM_NOT_FOUND on parse + * failure. Centralizes the atoi-replacement pattern for token_map lookups. */ +static int parse_token_index(const char *idx_str) { + if (!idx_str) { + return CBM_NOT_FOUND; + } + char *end = NULL; + long parsed = strtol(idx_str, &end, BASE_DECIMAL); + return (end != idx_str) ? (int)parsed : CBM_NOT_FOUND; +} + float cbm_sem_corpus_idf(const cbm_sem_corpus_t *corpus, const char *token) { if (!corpus || !token || corpus->doc_count == 0) { - return 0.0f; + return 0.0F; } - const char *idx_str = cbm_ht_get(corpus->token_map, token); - if (!idx_str) { - return 0.0f; - } - int idx = atoi(idx_str); + int idx = parse_token_index(cbm_ht_get(corpus->token_map, token)); if (idx < 0 || idx >= corpus->entry_count) { - return 0.0f; + return 0.0F; } int df = corpus->entries[idx].doc_freq; if (df <= 0) { - return 0.0f; + return 0.0F; } return logf((float)corpus->doc_count / (float)df); } @@ -1160,11 +1446,7 @@ const cbm_sem_vec_t *cbm_sem_corpus_ri_vec(const cbm_sem_corpus_t *corpus, const if (!corpus || !token) { return NULL; } - const char *idx_str = cbm_ht_get(corpus->token_map, token); - if (!idx_str) { - return NULL; - } - int idx = atoi(idx_str); + int idx = parse_token_index(cbm_ht_get(corpus->token_map, token)); if (idx < 0 || idx >= corpus->entry_count) { return NULL; } @@ -1189,7 +1471,7 @@ const char *cbm_sem_corpus_token_at(const cbm_sem_corpus_t *corpus, int index, } if (out_idf && corpus->doc_count > 0) { int df = corpus->entries[index].doc_freq; - *out_idf = df > 0 ? logf((float)corpus->doc_count / (float)df) : 0.0f; + *out_idf = df > 0 ? logf((float)corpus->doc_count / (float)df) : 0.0F; } return corpus->entries[index].token; } @@ -1217,7 +1499,7 @@ void cbm_sem_corpus_free(cbm_sem_corpus_t *corpus) { float cbm_sem_proximity(const char *path_a, const char *path_b) { if (!path_a || !path_b) { - return 1.0f; + return CBM_SEM_UNIT_POS; } /* Count shared directory components */ int shared = 0; @@ -1244,31 +1526,63 @@ float cbm_sem_proximity(const char *path_a, const char *path_b) { } int max_total = total_a > total_b ? total_a : total_b; if (max_total == 0) { - return 1.0f; + return CBM_SEM_UNIT_POS; } float ratio = (float)shared / (float)max_total; /* Same file = 1.10, same dir = 1.05, distant = 1.00 */ - return 1.0f + ratio * 0.10f; + return CBM_SEM_UNIT_POS + (ratio * CBM_SEM_PROX_MAX_BOOST); } /* Cosine similarity for small float arrays (AST profile). */ static float small_cosine(const float *a, const float *b, int dims) { - float dot = 0.0f; - float ma = 0.0f; - float mb = 0.0f; + float dot = 0.0F; + float ma = 0.0F; + float mb = 0.0F; for (int i = 0; i < dims; i++) { dot += a[i] * b[i]; ma += a[i] * a[i]; mb += b[i] * b[i]; } float denom = sqrtf(ma) * sqrtf(mb); - return denom < 1e-10f ? 0.0f : dot / denom; + return denom < CBM_SEM_DENOM_EPS ? 0.0F : dot / denom; +} + +/* Sparse cosine over two pre-sorted (index, weight) vectors. Returns 0 when + * either side is empty or the magnitude product is below the epsilon guard. */ +static float sparse_tfidf_cosine(const cbm_sem_func_t *a, const cbm_sem_func_t *b) { + if (a->tfidf_len <= 0 || b->tfidf_len <= 0) { + return 0.0F; + } + float dot = 0.0F; + float ma = 0.0F; + float mb = 0.0F; + int ia = 0; + int ib = 0; + while (ia < a->tfidf_len && ib < b->tfidf_len) { + if (a->tfidf_indices[ia] == b->tfidf_indices[ib]) { + dot += a->tfidf_weights[ia] * b->tfidf_weights[ib]; + ia++; + ib++; + } else if (a->tfidf_indices[ia] < b->tfidf_indices[ib]) { + ia++; + } else { + ib++; + } + } + for (int i = 0; i < a->tfidf_len; i++) { + ma += a->tfidf_weights[i] * a->tfidf_weights[i]; + } + for (int i = 0; i < b->tfidf_len; i++) { + mb += b->tfidf_weights[i] * b->tfidf_weights[i]; + } + float denom = sqrtf(ma) * sqrtf(mb); + return denom > CBM_SEM_DENOM_EPS ? (dot / denom) : 0.0F; } float cbm_sem_combined_score(const cbm_sem_func_t *a, const cbm_sem_func_t *b, const cbm_sem_config_t *cfg) { if (!a || !b || !cfg) { - return 0.0f; + return 0.0F; } /* Short-circuit: if MinHash Jaccard is already above the SIMILAR_TO threshold, @@ -1280,42 +1594,11 @@ float cbm_sem_combined_score(const cbm_sem_func_t *a, const cbm_sem_func_t *b, double early_j = cbm_minhash_jaccard((const cbm_minhash_t *)a->minhash, (const cbm_minhash_t *)b->minhash); if (early_j >= CBM_MINHASH_JACCARD_THRESHOLD) { - return 0.0f; + return 0.0F; } } - float score = 0.0f; - - /* Signal 1: TF-IDF sparse cosine */ - if (a->tfidf_len > 0 && b->tfidf_len > 0) { - float dot = 0.0f; - float ma = 0.0f; - float mb = 0.0f; - /* Sparse dot product: O(a_len + b_len) via merge */ - int ia = 0; - int ib = 0; - while (ia < a->tfidf_len && ib < b->tfidf_len) { - if (a->tfidf_indices[ia] == b->tfidf_indices[ib]) { - dot += a->tfidf_weights[ia] * b->tfidf_weights[ib]; - ia++; - ib++; - } else if (a->tfidf_indices[ia] < b->tfidf_indices[ib]) { - ia++; - } else { - ib++; - } - } - for (int i = 0; i < a->tfidf_len; i++) { - ma += a->tfidf_weights[i] * a->tfidf_weights[i]; - } - for (int i = 0; i < b->tfidf_len; i++) { - mb += b->tfidf_weights[i] * b->tfidf_weights[i]; - } - float denom = sqrtf(ma) * sqrtf(mb); - if (denom > 1e-10f) { - score += cfg->w_tfidf * (dot / denom); - } - } + float score = cfg->w_tfidf * sparse_tfidf_cosine(a, b); /* Signal 2: Random Indexing */ score += cfg->w_ri * cbm_sem_cosine(&a->ri_vec, &b->ri_vec); @@ -1337,8 +1620,7 @@ float cbm_sem_combined_score(const cbm_sem_func_t *a, const cbm_sem_func_t *b, score += cfg->w_decorator * cbm_sem_cosine(&a->deco_vec, &b->deco_vec); /* Signal 8+9+11: Structural profile + data flow + Halstead */ - enum { SP_DIMS = 25 }; - float sp_score = small_cosine(a->struct_profile, b->struct_profile, SP_DIMS); + float sp_score = small_cosine(a->struct_profile, b->struct_profile, CBM_SEM_AST_PROFILE_DIMS); score += cfg->w_struct_profile * sp_score; /* Signal 6: Module proximity (multiplier, not additive). @@ -1348,11 +1630,11 @@ float cbm_sem_combined_score(const cbm_sem_func_t *a, const cbm_sem_func_t *b, * would emit a 1.045 score which violates the semantic of "similarity" and * breaks downstream consumers that expect a normalized value. */ score *= cbm_sem_proximity(a->file_path, b->file_path); - if (score > 1.0f) { - score = 1.0f; + if (score > CBM_SEM_UNIT_POS) { + score = CBM_SEM_UNIT_POS; } - if (score < 0.0f) { - score = 0.0f; + if (score < 0.0F) { + score = 0.0F; } return score; @@ -1373,8 +1655,8 @@ void cbm_sem_diffuse(cbm_sem_vec_t *combined, const cbm_sem_vec_t *neighbors, in mean.v[i] += neighbors[n].v[i]; } } - float inv_n = 1.0f / (float)neighbor_count; - float one_minus_alpha = 1.0f - alpha; + float inv_n = CBM_SEM_UNIT_POS / (float)neighbor_count; + float one_minus_alpha = CBM_SEM_UNIT_POS - alpha; for (int i = 0; i < CBM_SEM_DIM; i++) { combined->v[i] = (one_minus_alpha * combined->v[i]) + (alpha * mean.v[i] * inv_n); } diff --git a/src/semantic/semantic.h b/src/semantic/semantic.h index c25a49c71..561221067 100644 --- a/src/semantic/semantic.h +++ b/src/semantic/semantic.h @@ -48,6 +48,13 @@ enum { CBM_SEM_WINDOW = 5 }; /* Maximum SEMANTICALLY_RELATED edges per node. */ enum { CBM_SEM_MAX_EDGES = 10 }; +/* AST structural profile: 25 float features per function (control flow, + * nesting, expression types, literals, data flow, Halstead). */ +enum { CBM_SEM_AST_PROFILE_DIMS = 25 }; + +/* MinHash fingerprint length (must match simhash/minhash.h CBM_MINHASH_K). */ +enum { CBM_SEM_MINHASH_K = 64 }; + /* Signal weights (sum to ~1.0, proximity is a multiplier). */ typedef struct { float w_tfidf; @@ -123,11 +130,11 @@ typedef struct { cbm_sem_vec_t deco_vec; /* AST profile as float vector (decoded from "sp" property). */ - float struct_profile[25]; + float struct_profile[CBM_SEM_AST_PROFILE_DIMS]; /* MinHash fingerprint (decoded from "fp" property). */ bool has_minhash; - uint32_t minhash[64]; + uint32_t minhash[CBM_SEM_MINHASH_K]; } cbm_sem_func_t; /* ── Corpus-level data ───────────────────────────────────────────── */ @@ -146,8 +153,7 @@ void cbm_sem_corpus_add_doc(cbm_sem_corpus_t *corpus, const char **tokens, int c * `token_counts[f]` = number of tokens in document f. * This replaces a loop of cbm_sem_corpus_add_doc() calls. */ void cbm_sem_corpus_add_docs_batch(cbm_sem_corpus_t *corpus, char **all_tokens, - const int *token_counts, int doc_count, - int max_tokens_per_doc); + const int *token_counts, int doc_count, int max_tokens_per_doc); /* Finalize: compute IDF, build enriched token vectors via co-occurrence. */ void cbm_sem_corpus_finalize(cbm_sem_corpus_t *corpus); diff --git a/src/simhash/minhash.c b/src/simhash/minhash.c index a96050206..dc42ef4f5 100644 --- a/src/simhash/minhash.c +++ b/src/simhash/minhash.c @@ -513,7 +513,9 @@ int cbm_lsh_query_into(const cbm_lsh_index_t *idx, const cbm_minhash_t *fp, } out_buf[count++] = candidate; } - if (count >= out_cap) break; + if (count >= out_cap) { + break; + } } seen_set_free(&seen); diff --git a/src/store/store.c b/src/store/store.c index fcd10de4c..cacf97f83 100644 --- a/src/store/store.c +++ b/src/store/store.c @@ -267,14 +267,13 @@ static int init_schema(cbm_store_t *s) { * Fails silently if FTS5 is not compiled in (SQLITE_ENABLE_FTS5). */ { char *fts_err = NULL; - int fts_rc = sqlite3_exec( - s->db, - "CREATE VIRTUAL TABLE IF NOT EXISTS nodes_fts USING fts5(" - " name, qualified_name, label, file_path," - " content=''," - " tokenize='unicode61 remove_diacritics 2'" - ");", - NULL, NULL, &fts_err); + int fts_rc = sqlite3_exec(s->db, + "CREATE VIRTUAL TABLE IF NOT EXISTS nodes_fts USING fts5(" + " name, qualified_name, label, file_path," + " content=''," + " tokenize='unicode61 remove_diacritics 2'" + ");", + NULL, NULL, &fts_err); if (fts_rc != SQLITE_OK && fts_err) { sqlite3_free(fts_err); } @@ -337,41 +336,72 @@ static int configure_pragmas(cbm_store_t *s, bool in_memory) { * but followed by a lowercase letter ("XMLParser" → "XML Parser", not * "X M L Parser") * snake_case is already split by FTS5's unicode61 tokenizer on `_`. */ -enum { CAMEL_SPLIT_BUF = 2048 }; +enum { + CAMEL_SPLIT_BUF = 2048, + SQLITE_AUTO_LEN = -1, /* sqlite3_result_text sentinel: use strlen(). */ + CAMEL_BUF_GUARD = 2, /* reserve room for inserted space + NUL terminator. */ +}; + +/* Denominator epsilon guard for double-precision cosine. */ +#define CBM_STORE_DENOM_EPS_D 1e-10 +#define CBM_STORE_DENOM_EPS_F 1e-10F +#define CBM_STORE_INT8_MAX 127.0F +#define CBM_STORE_UNIT_POS_F 1.0F +#define CBM_STORE_UNIT_POS_D 1.0 + +/* Module-local copy of SQLite's SQLITE_TRANSIENT sentinel ((void*)-1). + * We construct it via memcpy from a volatile intptr_t sentinel so the + * resulting expression isn't syntactically a direct int-to-ptr cast — + * clang-tidy's performance-no-int-to-ptr sees the memcpy boundary and + * doesn't flag it per-use-site. */ +static sqlite3_destructor_type cbm_sqlite_transient_destructor(void) { + static const volatile intptr_t raw = -1; + sqlite3_destructor_type dtor = NULL; + memcpy(&dtor, (const void *)&raw, sizeof(dtor)); + return dtor; +} +#define CBM_SQLITE_TRANSIENT (cbm_sqlite_transient_destructor()) + +/* True if we should insert a space BEFORE input[i] to split a camelCase word + * boundary (lowercase→uppercase or uppercase-run→uppercase-then-lowercase). */ +static bool camel_should_split(const char *input, int i) { + if (i <= 0) { + return false; + } + char curr = input[i]; + char prev = input[i - SKIP_ONE]; + char next = input[i + SKIP_ONE]; + if (curr >= 'A' && curr <= 'Z' && prev >= 'a' && prev <= 'z') { + return true; + } + if (curr >= 'A' && curr <= 'Z' && prev >= 'A' && prev <= 'Z' && next >= 'a' && next <= 'z') { + return true; + } + return false; +} + static void sqlite_camel_split(sqlite3_context *ctx, int argc, sqlite3_value **argv) { (void)argc; const char *input = (const char *)sqlite3_value_text(argv[0]); if (!input || !input[0]) { - sqlite3_result_text(ctx, input ? input : "", -1, SQLITE_TRANSIENT); + sqlite3_result_text(ctx, input ? input : "", SQLITE_AUTO_LEN, CBM_SQLITE_TRANSIENT); return; } char buf[CAMEL_SPLIT_BUF]; int len = snprintf(buf, sizeof(buf), "%s ", input); if (len < 0 || len >= (int)sizeof(buf)) { /* Input too long — fall back to the original string unmodified. */ - sqlite3_result_text(ctx, input, -1, SQLITE_TRANSIENT); + sqlite3_result_text(ctx, input, SQLITE_AUTO_LEN, CBM_SQLITE_TRANSIENT); return; } - for (int i = 0; input[i] && len < (int)sizeof(buf) - 2; i++) { - if (i > 0) { - bool split = false; - char curr = input[i]; - char prev = input[i - 1]; - char next = input[i + 1]; - if (curr >= 'A' && curr <= 'Z' && prev >= 'a' && prev <= 'z') { - split = true; - } else if (curr >= 'A' && curr <= 'Z' && prev >= 'A' && prev <= 'Z' && - next >= 'a' && next <= 'z') { - split = true; - } - if (split) { - buf[len++] = ' '; - } + for (int i = 0; input[i] && len < (int)sizeof(buf) - CAMEL_BUF_GUARD; i++) { + if (camel_should_split(input, i)) { + buf[len++] = ' '; } buf[len++] = input[i]; } buf[len] = '\0'; - sqlite3_result_text(ctx, buf, len, SQLITE_TRANSIENT); + sqlite3_result_text(ctx, buf, len, CBM_SQLITE_TRANSIENT); } /* ── REGEXP function for SQLite ──────────────────────────────────── */ @@ -423,7 +453,8 @@ static void sqlite_iregexp(sqlite3_context *ctx, int argc, sqlite3_value **argv) * Returns float in [-1, 1]. Used for vector search at query time. */ static void sqlite_cosine_i8(sqlite3_context *ctx, int argc, sqlite3_value **argv) { (void)argc; - if (sqlite3_value_type(argv[0]) != SQLITE_BLOB || sqlite3_value_type(argv[SKIP_ONE]) != SQLITE_BLOB) { + if (sqlite3_value_type(argv[0]) != SQLITE_BLOB || + sqlite3_value_type(argv[SKIP_ONE]) != SQLITE_BLOB) { sqlite3_result_double(ctx, 0.0); return; } @@ -444,7 +475,7 @@ static void sqlite_cosine_i8(sqlite3_context *ctx, int argc, sqlite3_value **arg mag_b += (int32_t)b[i] * (int32_t)b[i]; } double denom = sqrt((double)mag_a) * sqrt((double)mag_b); - sqlite3_result_double(ctx, denom > 1e-10 ? (double)dot / denom : 0.0); + sqlite3_result_double(ctx, denom > CBM_STORE_DENOM_EPS_D ? (double)dot / denom : 0.0); } /* ── Lifecycle ──────────────────────────────────────────────────── */ @@ -502,9 +533,8 @@ static cbm_store_t *store_open_internal(const char *path, bool in_memory) { sqlite3_create_function(s->db, "cbm_cosine_i8", ST_COL_2, SQLITE_UTF8 | SQLITE_DETERMINISTIC, NULL, sqlite_cosine_i8, NULL, NULL); /* camelCase splitter for FTS5 BM25 indexing */ - sqlite3_create_function(s->db, "cbm_camel_split", SKIP_ONE, - SQLITE_UTF8 | SQLITE_DETERMINISTIC, NULL, sqlite_camel_split, NULL, - NULL); + sqlite3_create_function(s->db, "cbm_camel_split", SKIP_ONE, SQLITE_UTF8 | SQLITE_DETERMINISTIC, + NULL, sqlite_camel_split, NULL, NULL); if (configure_pragmas(s, in_memory) != CBM_STORE_OK || init_schema(s) != CBM_STORE_OK || create_user_indexes(s) != CBM_STORE_OK) { @@ -559,9 +589,8 @@ cbm_store_t *cbm_store_open_path_query(const char *db_path) { sqlite_iregexp, NULL, NULL); sqlite3_create_function(s->db, "cbm_cosine_i8", ST_COL_2, SQLITE_UTF8 | SQLITE_DETERMINISTIC, NULL, sqlite_cosine_i8, NULL, NULL); - sqlite3_create_function(s->db, "cbm_camel_split", SKIP_ONE, - SQLITE_UTF8 | SQLITE_DETERMINISTIC, NULL, sqlite_camel_split, NULL, - NULL); + sqlite3_create_function(s->db, "cbm_camel_split", SKIP_ONE, SQLITE_UTF8 | SQLITE_DETERMINISTIC, + NULL, sqlite_camel_split, NULL, NULL); if (configure_pragmas(s, false) != CBM_STORE_OK) { sqlite3_close(s->db); @@ -4657,10 +4686,10 @@ int cbm_store_count_vectors(cbm_store_t *s, const char *project) { } sqlite3_stmt *stmt = NULL; const char *sql = "SELECT count(*) FROM node_vectors WHERE project = ?1"; - if (sqlite3_prepare_v2(s->db, sql, -1, &stmt, NULL) != SQLITE_OK) { + if (sqlite3_prepare_v2(s->db, sql, SQLITE_AUTO_LEN, &stmt, NULL) != SQLITE_OK) { return 0; } - sqlite3_bind_text(stmt, SKIP_ONE, project, -1, SQLITE_STATIC); + sqlite3_bind_text(stmt, SKIP_ONE, project, SQLITE_AUTO_LEN, SQLITE_STATIC); int count = 0; if (sqlite3_step(stmt) == SQLITE_ROW) { count = sqlite3_column_int(stmt, 0); @@ -4682,6 +4711,166 @@ void cbm_store_free_vector_results(cbm_vector_result_t *results, int count) { free(results); } +/* Per-keyword scoring: score each keyword independently against each node + * vector, then combine using min(cosine_k) across keywords. This ensures + * ALL keywords must be relevant, not just the average. */ +enum { + VS_VEC_DIM = 768, + VS_SPARSE_NNZE = 8, + VS_RI_SEED = 0x52494E44, + VS_MAX_KW = 32, + VS_STR_BUF = 16, +}; + +/* Try to look up an enriched int8 vector for `token` in the token_vectors + * table. On success, writes the de-quantized float representation to + * `out` and returns true. */ +static bool vs_load_enriched_vector(cbm_store_t *s, const char *project, const char *token, + float *out) { + sqlite3_stmt *tv_stmt = NULL; + const char *tv_sql = "SELECT vector, idf FROM token_vectors" + " WHERE project = ?1 AND token = ?2 LIMIT 1"; + if (sqlite3_prepare_v2(s->db, tv_sql, SQLITE_AUTO_LEN, &tv_stmt, NULL) != SQLITE_OK) { + return false; + } + bool found = false; + sqlite3_bind_text(tv_stmt, SKIP_ONE, project, SQLITE_AUTO_LEN, SQLITE_STATIC); + sqlite3_bind_text(tv_stmt, ST_COL_2, token, SQLITE_AUTO_LEN, SQLITE_STATIC); + if (sqlite3_step(tv_stmt) == SQLITE_ROW) { + const int8_t *vec = (const int8_t *)sqlite3_value_blob(sqlite3_column_value(tv_stmt, 0)); + int vec_len = sqlite3_column_bytes(tv_stmt, 0); + if (vec && vec_len == VS_VEC_DIM) { + for (int d = 0; d < VS_VEC_DIM; d++) { + out[d] = (float)vec[d] / CBM_STORE_INT8_MAX; + } + found = true; + } + } + sqlite3_finalize(tv_stmt); + return found; +} + +/* Sparse-random-index fallback for tokens not present in the enriched table. */ +static void vs_fill_sparse_random(const char *token, float *out) { + uint64_t seed = XXH3_64bits(token, strlen(token)); + for (int i = 0; i < VS_SPARSE_NNZE; i++) { + uint64_t h = XXH3_64bits_withSeed(&i, sizeof(i), seed + VS_RI_SEED); + int pos = (int)(h % VS_VEC_DIM); + float sign = (h & SKIP_ONE) ? CBM_STORE_UNIT_POS_F : -CBM_STORE_UNIT_POS_F; + out[pos] += sign; + } +} + +/* Clamp one float into the int8 representable range. */ +static int8_t vs_clamp_int8(float v) { + if (v > CBM_STORE_INT8_MAX) { + return (int8_t)CBM_STORE_INT8_MAX; + } + if (v < -CBM_STORE_INT8_MAX) { + return (int8_t)-CBM_STORE_INT8_MAX; + } + return (int8_t)v; +} + +/* Normalize + int8 quantize a float vector. Returns false if the vector is + * effectively zero (magnitude below epsilon), in which case `dst` is left in + * an indeterminate state and the caller should skip this keyword. */ +static bool vs_normalize_and_quantize(const float *src, int8_t *dst) { + float mag = 0.0F; + for (int d = 0; d < VS_VEC_DIM; d++) { + mag += src[d] * src[d]; + } + mag = sqrtf(mag); + if (mag < CBM_STORE_DENOM_EPS_F) { + return false; + } + float inv = CBM_STORE_UNIT_POS_F / mag; + for (int d = 0; d < VS_VEC_DIM; d++) { + dst[d] = vs_clamp_int8(src[d] * inv * CBM_STORE_INT8_MAX); + } + return true; +} + +/* Build int8 query vectors for each keyword. Returns the number of + * successfully built vectors (may be less than keyword_count when some + * keywords produce zero-magnitude vectors). */ +static int vs_build_keyword_vectors(cbm_store_t *s, const char *project, const char **keywords, + int keyword_count, int8_t (*kw_vecs)[VS_VEC_DIM]) { + int actual_kw = 0; + for (int k = 0; k < keyword_count && actual_kw < VS_MAX_KW; k++) { + if (!keywords[k] || !keywords[k][0]) { + continue; + } + float kw_f[VS_VEC_DIM]; + memset(kw_f, 0, sizeof(kw_f)); + if (!vs_load_enriched_vector(s, project, keywords[k], kw_f)) { + vs_fill_sparse_random(keywords[k], kw_f); + } + if (vs_normalize_and_quantize(kw_f, kw_vecs[actual_kw])) { + actual_kw++; + } + } + return actual_kw; +} + +/* Compute the per-keyword min cosine score between a node's int8 vector and + * each of the query vectors. Returns 0.0 if the node vector is unavailable + * or mis-sized. */ +static double vs_min_cosine_score(const int8_t *node_vec, int node_vec_len, + const int8_t (*kw_vecs)[VS_VEC_DIM], int actual_kw) { + if (!node_vec || node_vec_len != VS_VEC_DIM) { + return 0.0; + } + double min_score = CBM_STORE_UNIT_POS_D; + for (int k = 0; k < actual_kw; k++) { + int32_t dot = 0; + int32_t ma = 0; + int32_t mb = 0; + for (int d = 0; d < VS_VEC_DIM; d++) { + dot += (int32_t)kw_vecs[k][d] * (int32_t)node_vec[d]; + ma += (int32_t)kw_vecs[k][d] * (int32_t)kw_vecs[k][d]; + mb += (int32_t)node_vec[d] * (int32_t)node_vec[d]; + } + double denom = sqrt((double)ma) * sqrt((double)mb); + double cos_k = denom > CBM_STORE_DENOM_EPS_D ? (double)dot / denom : 0.0; + if (cos_k < min_score) { + min_score = cos_k; + } + } + return min_score; +} + +/* Append one candidate row read from the scan statement into the result + * vector. Grows the results array geometrically on demand. Returns the + * (possibly grown) results pointer, or NULL on allocation failure. */ +static cbm_vector_result_t *vs_append_result(cbm_vector_result_t *results, int *count, int *cap, + sqlite3_stmt *stmt, + const int8_t (*kw_vecs)[VS_VEC_DIM], int actual_kw) { + if (*count >= *cap) { + int nc = *cap < CBM_SZ_16 ? CBM_SZ_16 : *cap * ST_COL_2; + cbm_vector_result_t *grown = realloc(results, (size_t)nc * sizeof(cbm_vector_result_t)); + if (!grown) { + return NULL; + } + results = grown; + *cap = nc; + } + int idx = (*count)++; + results[idx].node_id = sqlite3_column_int64(stmt, 0); + const char *name = (const char *)sqlite3_column_text(stmt, SKIP_ONE); + const char *qn = (const char *)sqlite3_column_text(stmt, ST_COL_2); + const char *fp = (const char *)sqlite3_column_text(stmt, ST_COL_3); + const char *label = (const char *)sqlite3_column_text(stmt, ST_COL_4); + results[idx].name = name ? strdup(name) : strdup(""); + results[idx].qualified_name = qn ? strdup(qn) : strdup(""); + results[idx].file_path = fp ? strdup(fp) : strdup(""); + results[idx].label = label ? strdup(label) : strdup(""); + const int8_t *node_vec = (const int8_t *)sqlite3_column_blob(stmt, ST_COL_6); + int node_vec_len = sqlite3_column_bytes(stmt, ST_COL_6); + results[idx].score = vs_min_cosine_score(node_vec, node_vec_len, kw_vecs, actual_kw); + return results; +} + int cbm_store_vector_search(cbm_store_t *s, const char *project, const char **keywords, int keyword_count, int limit, cbm_vector_result_t **out, int *out_count) { @@ -4691,93 +4880,26 @@ int cbm_store_vector_search(cbm_store_t *s, const char *project, const char **ke return CBM_STORE_ERR; } - /* Per-keyword scoring: score each keyword independently against each - * node vector, then combine using min(cosine_k) across keywords. - * This ensures ALL keywords must be relevant, not just the average. - * - * Step 1: Build per-keyword int8 query vectors. - * Step 2: For each node, compute cosine per keyword, take min. - * Step 3: Rank by min-score. */ - enum { VEC_DIM = 768, SPARSE_NNZE = 8, RI_SEED = 0x52494E44, IDF_SCALE = 1000, - MAX_KW = 32, SCAN_BUF_INIT = 1024 }; - - /* Build per-keyword vectors */ - int actual_kw = 0; - int8_t kw_vecs[MAX_KW][VEC_DIM]; - - for (int k = 0; k < keyword_count && actual_kw < MAX_KW; k++) { - if (!keywords[k] || !keywords[k][0]) { - continue; - } - float kw_f[VEC_DIM]; - memset(kw_f, 0, sizeof(kw_f)); - bool found = false; - - /* Try enriched vector from token_vectors table */ - sqlite3_stmt *tv_stmt = NULL; - const char *tv_sql = "SELECT vector, idf FROM token_vectors" - " WHERE project = ?1 AND token = ?2 LIMIT 1"; - if (sqlite3_prepare_v2(s->db, tv_sql, -1, &tv_stmt, NULL) == SQLITE_OK) { - sqlite3_bind_text(tv_stmt, SKIP_ONE, project, -1, SQLITE_STATIC); - sqlite3_bind_text(tv_stmt, ST_COL_2, keywords[k], -1, SQLITE_STATIC); - if (sqlite3_step(tv_stmt) == SQLITE_ROW) { - const int8_t *vec = (const int8_t *)sqlite3_value_blob( - sqlite3_column_value(tv_stmt, 0)); - int vec_len = sqlite3_column_bytes(tv_stmt, 0); - if (vec && vec_len == VEC_DIM) { - for (int d = 0; d < VEC_DIM; d++) { - kw_f[d] = (float)vec[d] / 127.0f; - } - found = true; - } - } - sqlite3_finalize(tv_stmt); - } - if (!found) { - uint64_t seed = XXH3_64bits(keywords[k], strlen(keywords[k])); - for (int i = 0; i < SPARSE_NNZE; i++) { - uint64_t h = XXH3_64bits_withSeed(&i, sizeof(i), seed + RI_SEED); - int pos = (int)(h % VEC_DIM); - float sign = (h & SKIP_ONE) ? 1.0f : -1.0f; - kw_f[pos] += sign; - } - } - /* Normalize + quantize */ - float mag = 0.0f; - for (int d = 0; d < VEC_DIM; d++) { - mag += kw_f[d] * kw_f[d]; - } - mag = sqrtf(mag); - if (mag < 1e-10f) { - continue; - } - float inv = 1.0f / mag; - for (int d = 0; d < VEC_DIM; d++) { - float v = kw_f[d] * inv * 127.0f; - kw_vecs[actual_kw][d] = (int8_t)(v > 127.0f ? 127.0f : (v < -127.0f ? -127.0f : v)); - } - actual_kw++; - } - + int8_t kw_vecs[VS_MAX_KW][VS_VEC_DIM]; + int actual_kw = vs_build_keyword_vectors(s, project, keywords, keyword_count, kw_vecs); if (actual_kw == 0) { return CBM_STORE_OK; } /* Scan all node vectors, compute per-keyword cosine, take min. * We use the FIRST keyword as the SQL sort (for top-K pre-filter), - * then re-score with min across all keywords in C. */ - const char *sql = - "SELECT n.id, n.name, n.qualified_name, n.file_path, n.label," - " cbm_cosine_i8(v.vector, ?1) as score, v.vector" - " FROM node_vectors v" - " INNER JOIN nodes n ON n.id = v.node_id" - " WHERE v.project = ?2" - " AND n.label IN ('Function','Method','Class')" - " ORDER BY score DESC" - " LIMIT ?3"; + * then re-score with min across all keywords in the append helper. */ + const char *sql = "SELECT n.id, n.name, n.qualified_name, n.file_path, n.label," + " cbm_cosine_i8(v.vector, ?1) as score, v.vector" + " FROM node_vectors v" + " INNER JOIN nodes n ON n.id = v.node_id" + " WHERE v.project = ?2" + " AND n.label IN ('Function','Method','Class')" + " ORDER BY score DESC" + " LIMIT ?3"; sqlite3_stmt *stmt = NULL; - int prep_rc = sqlite3_prepare_v2(s->db, sql, -1, &stmt, NULL); + int prep_rc = sqlite3_prepare_v2(s->db, sql, SQLITE_AUTO_LEN, &stmt, NULL); if (prep_rc != SQLITE_OK) { (void)fprintf(stderr, "vector_search: %s\n", sqlite3_errmsg(s->db)); return CBM_STORE_ERR; @@ -4785,79 +4907,39 @@ int cbm_store_vector_search(cbm_store_t *s, const char *project, const char **ke /* Use first keyword for SQL pre-filter, fetch more candidates for re-ranking */ int fetch_limit = (limit > 0 ? limit : CBM_SZ_16) * ST_COL_5; - sqlite3_bind_blob(stmt, SKIP_ONE, kw_vecs[0], VEC_DIM, SQLITE_STATIC); - sqlite3_bind_text(stmt, ST_COL_2, project, -1, SQLITE_STATIC); + sqlite3_bind_blob(stmt, SKIP_ONE, kw_vecs[0], VS_VEC_DIM, SQLITE_STATIC); + sqlite3_bind_text(stmt, ST_COL_2, project, SQLITE_AUTO_LEN, SQLITE_STATIC); sqlite3_bind_int(stmt, ST_COL_3, fetch_limit); { - char kw_buf[16], fl_buf[16]; + char kw_buf[VS_STR_BUF]; + char fl_buf[VS_STR_BUF]; snprintf(kw_buf, sizeof(kw_buf), "%d", actual_kw); snprintf(fl_buf, sizeof(fl_buf), "%d", fetch_limit); - cbm_log_info("vector_search.exec", "kw_count", kw_buf, "fetch_limit", fl_buf, "project", project); + cbm_log_info("vector_search.exec", "kw_count", kw_buf, "fetch_limit", fl_buf, "project", + project); } cbm_vector_result_t *results = NULL; int count = 0; int cap = 0; - - int step_rc; + int step_rc = 0; while ((step_rc = sqlite3_step(stmt)) == SQLITE_ROW) { - if (count >= cap) { - int nc = cap < CBM_SZ_16 ? CBM_SZ_16 : cap * ST_COL_2; - cbm_vector_result_t *grown = realloc(results, (size_t)nc * sizeof(cbm_vector_result_t)); - if (!grown) { - break; - } - results = grown; - cap = nc; + cbm_vector_result_t *grown = + vs_append_result(results, &count, &cap, stmt, kw_vecs, actual_kw); + if (!grown) { + break; } - results[count].node_id = sqlite3_column_int64(stmt, 0); - const char *name = (const char *)sqlite3_column_text(stmt, SKIP_ONE); - const char *qn = (const char *)sqlite3_column_text(stmt, ST_COL_2); - const char *fp = (const char *)sqlite3_column_text(stmt, ST_COL_3); - const char *label = (const char *)sqlite3_column_text(stmt, ST_COL_4); - results[count].name = name ? strdup(name) : strdup(""); - results[count].qualified_name = qn ? strdup(qn) : strdup(""); - results[count].file_path = fp ? strdup(fp) : strdup(""); - results[count].label = label ? strdup(label) : strdup(""); - - /* Compute per-keyword min-score for this node. - * The SQL pre-filtered by first keyword; now re-score with ALL keywords. */ - const void *node_vec = sqlite3_column_blob(stmt, ST_COL_6); - int node_vec_len = sqlite3_column_bytes(stmt, ST_COL_6); - double min_score = 1.0; - if (node_vec && node_vec_len == VEC_DIM) { - const int8_t *nv = (const int8_t *)node_vec; - for (int k = 0; k < actual_kw; k++) { - /* Inline int8 cosine for speed */ - int32_t dot = 0; - int32_t ma = 0; - int32_t mb = 0; - for (int d = 0; d < VEC_DIM; d++) { - dot += (int32_t)kw_vecs[k][d] * (int32_t)nv[d]; - ma += (int32_t)kw_vecs[k][d] * (int32_t)kw_vecs[k][d]; - mb += (int32_t)nv[d] * (int32_t)nv[d]; - } - double denom = sqrt((double)ma) * sqrt((double)mb); - double cos_k = denom > 1e-10 ? (double)dot / denom : 0.0; - if (cos_k < min_score) { - min_score = cos_k; - } - } - } else { - min_score = 0.0; - } - results[count].score = min_score; - count++; + results = grown; } if (step_rc != SQLITE_DONE) { - char rc_buf[16]; + char rc_buf[VS_STR_BUF]; snprintf(rc_buf, sizeof(rc_buf), "%d", step_rc); cbm_log_warn("vector_search.step_error", "rc", rc_buf, "msg", sqlite3_errmsg(s->db)); } { - char cnt_buf[16]; + char cnt_buf[VS_STR_BUF]; snprintf(cnt_buf, sizeof(cnt_buf), "%d", count); cbm_log_info("vector_search.done", "candidates", cnt_buf); } diff --git a/src/store/store.h b/src/store/store.h index 8f82652d7..f15132c28 100644 --- a/src/store/store.h +++ b/src/store/store.h @@ -625,7 +625,8 @@ typedef struct { * the cbm_cosine_i8 SQL function joined with the nodes table. * Returns results sorted by score DESC. Caller must free with cbm_store_free_vector_results. */ int cbm_store_vector_search(cbm_store_t *s, const char *project, const char **keywords, - int keyword_count, int limit, cbm_vector_result_t **out, int *out_count); + int keyword_count, int limit, cbm_vector_result_t **out, + int *out_count); /* Free vector search results. */ void cbm_store_free_vector_results(cbm_vector_result_t *results, int count);