mirror of
https://github.com/DeusData/codebase-memory-mcp.git
synced 2026-11-11 09:25:35 +08:00
Add JS/TS IMPORTS resolution, Channel schema, lint cleanup
Features: - Relative import resolution for JS/TS/Python/Ruby (./foo, ../bar, leading dots) via cbm_pipeline_resolve_relative_import in fqn.c - Generic Channel detection: Socket.IO emit/on, EventEmitter patterns produce Channel nodes with EMITS/LISTENS_ON edges and transport property (extract_channels.c) - Constant resolution trick: `const EVENT = "foo"; emit(EVENT)` resolves the channel name through a per-file constant table Lint fixes (337 errors across 16 files): - Named constants replacing all magic numbers (enums/defines) - Cognitive complexity reduced via helper extraction (table-driven sqlite_writer, pipeline predump passes, graph_buffer dump phases) - SQLITE_TRANSIENT int-to-ptr workaround in store.c + mcp.c - clang-format + cppcheck clean Co-Authored-By: Koolerx <tommy@koolerx.com>
This commit is contained in:
Martin Vogel
+107
-94
@@ -11,6 +11,7 @@
|
||||
#include "foundation/constants.h"
|
||||
|
||||
enum {
|
||||
GB_ERR = -1,
|
||||
GB_COL_2 = 2,
|
||||
GB_COL_3 = 3,
|
||||
GB_COL_4 = 4,
|
||||
@@ -18,6 +19,8 @@ enum {
|
||||
GB_COL_6 = 6,
|
||||
GB_COL_7 = 7,
|
||||
GB_URL_PATH_PREFIX = 12, /* strlen(""url_path":"") */
|
||||
GB_MIN_FOR_DEDUP = 2, /* need at least 2 vectors to sort+dedup */
|
||||
GB_DEDUP_LOOKAHEAD = 1, /* compare current with next element */
|
||||
};
|
||||
#include "graph_buffer/graph_buffer.h"
|
||||
#include "store/store.h"
|
||||
@@ -453,7 +456,7 @@ void cbm_gbuf_free(cbm_gbuf_t *gb) {
|
||||
|
||||
int cbm_gbuf_store_vector(cbm_gbuf_t *gb, int64_t node_id, const uint8_t *vector, int vector_len) {
|
||||
if (!gb || !vector || vector_len <= 0) {
|
||||
return -1;
|
||||
return GB_ERR;
|
||||
}
|
||||
enum { VEC_INIT_CAP = 1024, VEC_GROW = 2 };
|
||||
if (gb->dump_vector_count >= gb->dump_vector_cap) {
|
||||
@@ -461,7 +464,7 @@ int cbm_gbuf_store_vector(cbm_gbuf_t *gb, int64_t node_id, const uint8_t *vector
|
||||
gb->dump_vector_cap < VEC_INIT_CAP ? VEC_INIT_CAP : gb->dump_vector_cap * VEC_GROW;
|
||||
CBMDumpVector *grown = realloc(gb->dump_vectors, (size_t)new_cap * sizeof(CBMDumpVector));
|
||||
if (!grown) {
|
||||
return -1;
|
||||
return GB_ERR;
|
||||
}
|
||||
gb->dump_vectors = grown;
|
||||
gb->dump_vector_cap = new_cap;
|
||||
@@ -469,7 +472,7 @@ int cbm_gbuf_store_vector(cbm_gbuf_t *gb, int64_t node_id, const uint8_t *vector
|
||||
/* Copy vector data */
|
||||
uint8_t *vec_copy = malloc((size_t)vector_len);
|
||||
if (!vec_copy) {
|
||||
return -1;
|
||||
return GB_ERR;
|
||||
}
|
||||
memcpy(vec_copy, vector, (size_t)vector_len);
|
||||
|
||||
@@ -485,23 +488,23 @@ int cbm_gbuf_store_vector(cbm_gbuf_t *gb, int64_t node_id, const uint8_t *vector
|
||||
int cbm_gbuf_store_token_vector(cbm_gbuf_t *gb, const char *token, const uint8_t *vector,
|
||||
int vector_len, float idf) {
|
||||
if (!gb || !token || !vector || vector_len <= 0) {
|
||||
return -1;
|
||||
return GB_ERR;
|
||||
}
|
||||
enum { TV_INIT_CAP = 256, TV_GROW = 2 };
|
||||
if (gb->dump_token_vec_count >= gb->dump_token_vec_cap) {
|
||||
int new_cap = gb->dump_token_vec_cap < TV_INIT_CAP ? TV_INIT_CAP
|
||||
: gb->dump_token_vec_cap * TV_GROW;
|
||||
int new_cap =
|
||||
gb->dump_token_vec_cap < TV_INIT_CAP ? TV_INIT_CAP : gb->dump_token_vec_cap * TV_GROW;
|
||||
CBMDumpTokenVec *grown =
|
||||
realloc(gb->dump_token_vecs, (size_t)new_cap * sizeof(CBMDumpTokenVec));
|
||||
if (!grown) {
|
||||
return -1;
|
||||
return GB_ERR;
|
||||
}
|
||||
gb->dump_token_vecs = grown;
|
||||
gb->dump_token_vec_cap = new_cap;
|
||||
}
|
||||
uint8_t *vec_copy = malloc((size_t)vector_len);
|
||||
if (!vec_copy) {
|
||||
return -1;
|
||||
return GB_ERR;
|
||||
}
|
||||
memcpy(vec_copy, vector, (size_t)vector_len);
|
||||
|
||||
@@ -1249,23 +1252,106 @@ static CBMDumpEdge *build_dump_edges(cbm_gbuf_t *gb, const int64_t *temp_to_fina
|
||||
return dump_edges;
|
||||
}
|
||||
|
||||
/* Remap vector node IDs through temp_to_final, sort by ID, deduplicate. */
|
||||
static void remap_sort_dedup_vectors(cbm_gbuf_t *gb, const int64_t *temp_to_final,
|
||||
int64_t max_temp_id) {
|
||||
int remapped = 0;
|
||||
int dropped = 0;
|
||||
for (int i = 0; i < gb->dump_vector_count; i++) {
|
||||
int64_t old_id = gb->dump_vectors[i].node_id;
|
||||
int64_t new_id = (old_id > 0 && old_id < max_temp_id) ? temp_to_final[old_id] : 0;
|
||||
if (new_id > 0) {
|
||||
gb->dump_vectors[remapped] = gb->dump_vectors[i];
|
||||
gb->dump_vectors[remapped].node_id = new_id;
|
||||
remapped++;
|
||||
} else {
|
||||
dropped++;
|
||||
}
|
||||
}
|
||||
if (dropped > 0) {
|
||||
char r_buf[CBM_SZ_16];
|
||||
char d_buf[CBM_SZ_16];
|
||||
snprintf(r_buf, sizeof(r_buf), "%d", remapped);
|
||||
snprintf(d_buf, sizeof(d_buf), "%d", dropped);
|
||||
cbm_log_info("dump.vectors.remap", "remapped", r_buf, "dropped", d_buf);
|
||||
}
|
||||
gb->dump_vector_count = remapped;
|
||||
|
||||
if (gb->dump_vector_count >= GB_MIN_FOR_DEDUP) {
|
||||
qsort(gb->dump_vectors, (size_t)gb->dump_vector_count, sizeof(CBMDumpVector),
|
||||
cmp_dump_vectors_by_id);
|
||||
int deduped = 0;
|
||||
for (int i = 0; i < gb->dump_vector_count; i++) {
|
||||
if (i + GB_DEDUP_LOOKAHEAD < gb->dump_vector_count &&
|
||||
gb->dump_vectors[i].node_id == gb->dump_vectors[i + GB_DEDUP_LOOKAHEAD].node_id) {
|
||||
continue;
|
||||
}
|
||||
gb->dump_vectors[deduped++] = gb->dump_vectors[i];
|
||||
}
|
||||
gb->dump_vector_count = deduped;
|
||||
}
|
||||
}
|
||||
|
||||
static void log_dump_summary(int node_count, int edge_count) {
|
||||
char b1[CBM_SZ_16];
|
||||
char b2[CBM_SZ_16];
|
||||
snprintf(b1, sizeof(b1), "%d", node_count);
|
||||
snprintf(b2, sizeof(b2), "%d", edge_count);
|
||||
cbm_log_info("gbuf.dump", "nodes", b1, "edges", b2);
|
||||
}
|
||||
|
||||
static void free_dump_resources(char **url_paths, int edge_count, CBMDumpEdge *dump_edges,
|
||||
CBMDumpNode *dump_nodes, int64_t *temp_to_final) {
|
||||
for (int i = 0; i < edge_count; i++) {
|
||||
free(url_paths[i]);
|
||||
}
|
||||
free(url_paths);
|
||||
free(dump_edges);
|
||||
free(dump_nodes);
|
||||
free(temp_to_final);
|
||||
}
|
||||
|
||||
static int count_live_nodes(cbm_gbuf_t *gb) {
|
||||
int count = 0;
|
||||
for (int i = 0; i < gb->nodes.count; i++) {
|
||||
cbm_gbuf_node_t *n = gb->nodes.items[i];
|
||||
if (n->qualified_name && cbm_ht_get(gb->node_by_qn, n->qualified_name)) {
|
||||
count++;
|
||||
}
|
||||
}
|
||||
return count;
|
||||
}
|
||||
|
||||
static void generate_iso_timestamp(char *buf, size_t buf_size) {
|
||||
time_t now = time(NULL);
|
||||
struct tm tm_buf;
|
||||
struct tm *tm_val = cbm_gmtime_r(&now, &tm_buf);
|
||||
if (strftime(buf, buf_size, "%Y-%m-%dT%H:%M:%SZ", tm_val) == 0) {
|
||||
snprintf(buf, buf_size, "1970-01-01T00:00:00Z");
|
||||
}
|
||||
}
|
||||
|
||||
/* Release lookup indexes then remap+sort+dedup vectors for the B-tree writer. */
|
||||
static void release_and_remap_vectors(cbm_gbuf_t *gb, const int64_t *temp_to_final,
|
||||
int64_t max_temp_id) {
|
||||
CBM_PROF_START(t_release_idx);
|
||||
release_gbuf_indexes(gb);
|
||||
CBM_PROF_END("dump", "4_release_gbuf_indexes", t_release_idx);
|
||||
|
||||
CBM_PROF_START(t_vec_remap);
|
||||
remap_sort_dedup_vectors(gb, temp_to_final, max_temp_id);
|
||||
CBM_PROF_END_N("dump", "5_vector_remap_sort", t_vec_remap, gb->dump_vector_count);
|
||||
}
|
||||
|
||||
int cbm_gbuf_dump_to_sqlite(cbm_gbuf_t *gb, const char *path) {
|
||||
if (!gb || !path) {
|
||||
return CBM_NOT_FOUND;
|
||||
}
|
||||
|
||||
/* Sub-phase: Count live nodes (not deleted from QN index) */
|
||||
CBM_PROF_START(t_count);
|
||||
int live_count = 0;
|
||||
for (int i = 0; i < gb->nodes.count; i++) {
|
||||
cbm_gbuf_node_t *n = gb->nodes.items[i];
|
||||
if (n->qualified_name && cbm_ht_get(gb->node_by_qn, n->qualified_name)) {
|
||||
live_count++;
|
||||
}
|
||||
}
|
||||
int live_count = count_live_nodes(gb);
|
||||
CBM_PROF_END_N("dump", "1_count_live_nodes", t_count, live_count);
|
||||
|
||||
/* Sub-phase: Build temp→final ID mapping + dump_nodes array */
|
||||
CBM_PROF_START(t_build_nodes);
|
||||
int64_t max_temp_id = gb->next_id;
|
||||
int64_t *temp_to_final = calloc((size_t)max_temp_id, sizeof(int64_t));
|
||||
@@ -1278,7 +1364,6 @@ int cbm_gbuf_dump_to_sqlite(cbm_gbuf_t *gb, const char *path) {
|
||||
build_dump_nodes(gb, live_count, temp_to_final, max_temp_id, &node_idx);
|
||||
CBM_PROF_END_N("dump", "2_build_dump_nodes", t_build_nodes, node_idx);
|
||||
|
||||
/* Sub-phase: Build dump_edges array with remapped IDs */
|
||||
CBM_PROF_START(t_build_edges);
|
||||
int edge_idx = 0;
|
||||
char **url_paths = NULL;
|
||||
@@ -1286,92 +1371,20 @@ int cbm_gbuf_dump_to_sqlite(cbm_gbuf_t *gb, const char *path) {
|
||||
build_dump_edges(gb, temp_to_final, max_temp_id, &edge_idx, &url_paths);
|
||||
CBM_PROF_END_N("dump", "3_build_dump_edges", t_build_edges, edge_idx);
|
||||
|
||||
/* Generate ISO 8601 timestamp */
|
||||
time_t now = time(NULL);
|
||||
struct tm tm_buf;
|
||||
struct tm *tm_val = cbm_gmtime_r(&now, &tm_buf);
|
||||
char indexed_at[CBM_SZ_64];
|
||||
if (strftime(indexed_at, sizeof(indexed_at), "%Y-%m-%dT%H:%M:%SZ", tm_val) == 0) {
|
||||
snprintf(indexed_at, sizeof(indexed_at), "1970-01-01T00:00:00Z");
|
||||
}
|
||||
generate_iso_timestamp(indexed_at, sizeof(indexed_at));
|
||||
|
||||
/* Sub-phase: Release lookup tables — frees 200-400MB on large codebases */
|
||||
CBM_PROF_START(t_release_idx);
|
||||
release_gbuf_indexes(gb);
|
||||
CBM_PROF_END("dump", "4_release_gbuf_indexes", t_release_idx);
|
||||
|
||||
/* Sub-phase: Remap + sort + dedup node_vectors for B-tree writer */
|
||||
CBM_PROF_START(t_vec_remap);
|
||||
/* Remap node_vectors IDs through temp_to_final (same as edges).
|
||||
* temp_to_final maps gbuf node IDs → sequential dump IDs (1-based).
|
||||
* Drop vectors whose gbuf ID has no mapping (node was filtered out). */
|
||||
{
|
||||
int remapped = 0;
|
||||
int dropped = 0;
|
||||
for (int i = 0; i < gb->dump_vector_count; i++) {
|
||||
int64_t old_id = gb->dump_vectors[i].node_id;
|
||||
int64_t new_id = (old_id > 0 && old_id < max_temp_id) ? temp_to_final[old_id] : 0;
|
||||
if (new_id > 0) {
|
||||
gb->dump_vectors[remapped] = gb->dump_vectors[i];
|
||||
gb->dump_vectors[remapped].node_id = new_id;
|
||||
remapped++;
|
||||
} else {
|
||||
dropped++;
|
||||
}
|
||||
}
|
||||
if (dropped > 0) {
|
||||
char r_buf[CBM_SZ_16], d_buf[CBM_SZ_16];
|
||||
snprintf(r_buf, sizeof(r_buf), "%d", remapped);
|
||||
snprintf(d_buf, sizeof(d_buf), "%d", dropped);
|
||||
cbm_log_info("dump.vectors.remap", "remapped", r_buf, "dropped", d_buf);
|
||||
}
|
||||
gb->dump_vector_count = remapped;
|
||||
}
|
||||
/* Sort vectors by remapped node_id — B-tree writer requires ascending keys.
|
||||
* Also deduplicate: parallel workers may store the same node_id twice. */
|
||||
if (gb->dump_vector_count > 1) {
|
||||
qsort(gb->dump_vectors, (size_t)gb->dump_vector_count, sizeof(CBMDumpVector),
|
||||
cmp_dump_vectors_by_id);
|
||||
/* Dedup: keep last occurrence of each node_id (latest vector wins) */
|
||||
int deduped = 0;
|
||||
for (int i = 0; i < gb->dump_vector_count; i++) {
|
||||
if (i + 1 < gb->dump_vector_count &&
|
||||
gb->dump_vectors[i].node_id == gb->dump_vectors[i + 1].node_id) {
|
||||
continue; /* skip duplicate, keep the later one */
|
||||
}
|
||||
gb->dump_vectors[deduped++] = gb->dump_vectors[i];
|
||||
}
|
||||
gb->dump_vector_count = deduped;
|
||||
}
|
||||
CBM_PROF_END_N("dump", "5_vector_remap_sort", t_vec_remap, gb->dump_vector_count);
|
||||
release_and_remap_vectors(gb, temp_to_final, max_temp_id);
|
||||
|
||||
/* Sub-phase: Write SQLite DB file (B-tree writer) */
|
||||
CBM_PROF_START(t_write_db);
|
||||
/* Write directly to final path — no .tmp + rename.
|
||||
* Callers must delete the old .db before calling this (reindex)
|
||||
* or ensure no file exists (first index). */
|
||||
int rc = cbm_write_db(path, gb->project, gb->root_path, indexed_at, dump_nodes, node_idx,
|
||||
dump_edges, edge_idx, gb->dump_vectors, gb->dump_vector_count,
|
||||
gb->dump_token_vecs, gb->dump_token_vec_count);
|
||||
CBM_PROF_END_N("dump", "6_write_db_btree", t_write_db, node_idx + edge_idx);
|
||||
|
||||
{
|
||||
char b1[CBM_SZ_16];
|
||||
char b2[CBM_SZ_16];
|
||||
snprintf(b1, sizeof(b1), "%d", node_idx);
|
||||
snprintf(b2, sizeof(b2), "%d", edge_idx);
|
||||
cbm_log_info("gbuf.dump", "nodes", b1, "edges", b2);
|
||||
}
|
||||
|
||||
/* Cleanup */
|
||||
for (int i = 0; i < edge_idx; i++) {
|
||||
free(url_paths[i]);
|
||||
}
|
||||
free(url_paths);
|
||||
free(dump_edges);
|
||||
free(dump_nodes);
|
||||
free(temp_to_final);
|
||||
|
||||
log_dump_summary(node_idx, edge_idx);
|
||||
free_dump_resources(url_paths, edge_idx, dump_edges, dump_nodes, temp_to_final);
|
||||
return rc;
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user