trending/codebase-memory-mcp
1
0
Fork 0
mirror of https://github.com/DeusData/codebase-memory-mcp.git synced 2026-11-07 08:43:35 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(pipeline): add advanced index mode — gate ALL LSP behind it

Browse Source 
New CBM_MODE_ADVANCED=3. Both per-file LSP (type-aware resolution
during extract) and the cross-file LSP pass (during resolve) are now
OPT-IN: FULL and below run tree-sitter extraction + registry textual
resolution only; ADVANCED adds the full LSP layer.

Rationale: profiling showed LSP changes the kubernetes graph by only
~0.01% of edges (it mostly re-confirms textual resolution) while
costing significant memory + time. Making it opt-in keeps the default
FULL path lean (3.0 GB vs 5.6 GB peak RSS on K8s) without meaningfully
worse output, and lets users pay for LSP precision only when wanted.

Mechanism:
- cbm.c: g_lsp_enabled atomic toggle wraps the entire per-file LSP
  block (Go/C/C++/PHP/Python/TS/JS/C#) + the C/C++ preprocessor-pass
  LSP call. Default true so ad-hoc extraction / tests keep historical
  behaviour. cbm_set_lsp_enabled / cbm_get_lsp_enabled in cbm.h.
- pipeline.c: sets the toggle (save→set→restore so it never leaks to
  other callers) from p->mode before extraction; gates the cross-LSP
  all_defs collection + registry build on advanced; gates the
  sequential lsp_cross dispatch. Fixed the moderate_only comparison
  (== FAST, not > MODERATE) so ADVANCED is not treated as lighter.
- pipeline_incremental.c: same toggle save/set/restore.
- discover.h/.c: added CBM_MODE_ADVANCED to discover's enum + all 4
  filter comparisons treat ADVANCED like FULL (full discovery, no
  fast-filtering). Without this, ADVANCED (numerically 3) was
  mis-detected as fast-mode and dropped ~half the files.
- mcp.c: parse mode "advanced" + tool schema enum/description.

Dual-mode K8s bench: FULL 46.7s (lsp_overrides=0, 3.0 GB) vs
ADVANCED 48.8s (lsp_overrides=80,818, 5.6 GB). Graphs within 0.01%.
All 3,615 tests pass.
This commit is contained in:
Martin Vogel
2026-05-26 16:48:58 +02:00
parent 2ee0c68d4a
commit b61861eb5a
9 changed files with 125 additions and 47 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/cbm/cbm.c
+54 -33
View File
@@ -28,6 +28,17 @@ static _Atomic uint64_t total_preprocess_ns = 0;
static _Atomic uint64_t total_files_preprocessed = 0;
static _Atomic uint64_t total_files = 0;
/* Per-file LSP toggle. The pipeline sets this ONCE before extraction
* based on index mode: only CBM_MODE_ADVANCED enables the per-file
* type-aware LSP resolution (the slowest part of extraction). Default
* true so callers that don't set it (tests, ad-hoc extraction) keep
* the historical behaviour. Set-once-before-parallel-extract +
* read-concurrently makes the atomic sufficient (no per-file variance). */
static _Atomic bool g_lsp_enabled = true;
void cbm_set_lsp_enabled(bool enabled) { atomic_store(&g_lsp_enabled, enabled); }
bool cbm_get_lsp_enabled(void) { return atomic_load(&g_lsp_enabled); }
#define NSEC_PER_SEC 1000000000ULL
#define USEC_TO_NSEC 1000ULL
/* Use compat.h's cbm_clock_gettime which accepts CLOCK_MONOTONIC (value
@@ -353,39 +364,44 @@ CBMFileResult *cbm_extract_file(const char *source, int source_len, CBMLanguage
cbm_extract_k8s(&ctx);
}
// LSP type-aware call resolution
// LSP type-aware call resolution — gated behind advanced mode.
// When disabled (FULL and below), tree-sitter extraction +
// registry-based textual resolution still produce the graph; only
// the type-aware LSP refinement is skipped.
uint64_t lsp_start = now_ns();
if (language == CBM_LANG_GO) {
cbm_run_go_lsp(a, result, source, source_len, root);
}
if (language == CBM_LANG_C || language == CBM_LANG_CPP || language == CBM_LANG_CUDA) {
cbm_run_c_lsp(a, result, source, source_len, root, language != CBM_LANG_C);
}
if (language == CBM_LANG_PHP) {
cbm_run_php_lsp(a, result, source, source_len, root);
}
if (language == CBM_LANG_PYTHON) {
cbm_run_py_lsp(a, result, source, source_len, root);
}
if (language == CBM_LANG_JAVASCRIPT || language == CBM_LANG_TYPESCRIPT ||
language == CBM_LANG_TSX) {
bool js_mode = (language == CBM_LANG_JAVASCRIPT);
// jsx_mode: TSX always; .jsx in the JS bucket also enables it.
bool jsx_mode = (language == CBM_LANG_TSX);
if (language == CBM_LANG_JAVASCRIPT && rel_path) {
size_t rl = strlen(rel_path);
if (rl >= 4 && strcmp(rel_path + rl - 4, ".jsx") == 0) jsx_mode = true;
if (atomic_load(&g_lsp_enabled)) {
if (language == CBM_LANG_GO) {
cbm_run_go_lsp(a, result, source, source_len, root);
}
// dts_mode: ".d.ts" suffix (TypeScript only).
bool dts_mode = false;
if (language == CBM_LANG_TYPESCRIPT && rel_path) {
size_t rl = strlen(rel_path);
if (rl >= 5 && strcmp(rel_path + rl - 5, ".d.ts") == 0) dts_mode = true;
if (language == CBM_LANG_C || language == CBM_LANG_CPP || language == CBM_LANG_CUDA) {
cbm_run_c_lsp(a, result, source, source_len, root, language != CBM_LANG_C);
}
if (language == CBM_LANG_PHP) {
cbm_run_php_lsp(a, result, source, source_len, root);
}
if (language == CBM_LANG_PYTHON) {
cbm_run_py_lsp(a, result, source, source_len, root);
}
if (language == CBM_LANG_JAVASCRIPT || language == CBM_LANG_TYPESCRIPT ||
language == CBM_LANG_TSX) {
bool js_mode = (language == CBM_LANG_JAVASCRIPT);
// jsx_mode: TSX always; .jsx in the JS bucket also enables it.
bool jsx_mode = (language == CBM_LANG_TSX);
if (language == CBM_LANG_JAVASCRIPT && rel_path) {
size_t rl = strlen(rel_path);
if (rl >= 4 && strcmp(rel_path + rl - 4, ".jsx") == 0) jsx_mode = true;
}
// dts_mode: ".d.ts" suffix (TypeScript only).
bool dts_mode = false;
if (language == CBM_LANG_TYPESCRIPT && rel_path) {
size_t rl = strlen(rel_path);
if (rl >= 5 && strcmp(rel_path + rl - 5, ".d.ts") == 0) dts_mode = true;
}
cbm_run_ts_lsp(a, result, source, source_len, root, js_mode, jsx_mode, dts_mode);
}
if (language == CBM_LANG_CSHARP) {
cbm_run_cs_lsp(a, result, source, source_len, root);
}
cbm_run_ts_lsp(a, result, source, source_len, root, js_mode, jsx_mode, dts_mode);
}
if (language == CBM_LANG_CSHARP) {
cbm_run_cs_lsp(a, result, source, source_len, root);
}
atomic_fetch_add(&total_lsp_ns, now_ns() - lsp_start);
@@ -435,9 +451,14 @@ CBMFileResult *cbm_extract_file(const char *source, int source_len, CBMLanguage
cbm_extract_unified(&pp_ctx);
// Also run LSP on expanded source for additional type-resolved calls
// (language is already C/C++/CUDA — checked in enclosing block)
cbm_run_c_lsp(a, result, expanded, expanded_len, pp_root,
language != CBM_LANG_C);
// (language is already C/C++/CUDA — checked in enclosing block).
// Gated behind advanced mode like the first-pass LSP above;
// the macro-expanded call extraction (cbm_extract_unified) is
// base extraction and always runs.
if (atomic_load(&g_lsp_enabled)) {
cbm_run_c_lsp(a, result, expanded, expanded_len, pp_root,
language != CBM_LANG_C);
}
ts_tree_delete(pp_tree);
}
internal/cbm/cbm.h
+7
View File
@@ -529,6 +529,13 @@ uint64_t cbm_get_preprocess_ns(void);
uint64_t cbm_get_files_preprocessed(void);
void cbm_reset_profile(void);
/* Per-file LSP toggle. The pipeline sets this before extraction based
* on index mode (only CBM_MODE_ADVANCED enables LSP). Defaults to true
* so ad-hoc extraction and tests keep historical behaviour. Set once
* before parallel extraction; read concurrently by extract workers. */
void cbm_set_lsp_enabled(bool enabled);
bool cbm_get_lsp_enabled(void);
// --- Internal helpers used by extractors ---
// Growable array push functions (arena-allocated, no individual free needed).
src/discover/discover.c
+4 -4
View File
@@ -140,7 +140,7 @@ bool cbm_should_skip_dir(const char *dirname, cbm_index_mode_t mode) {
}
/* Fast discovery applies to both MODERATE and FAST — only FULL keeps everything. */
if (mode != CBM_MODE_FULL) {
if (mode != CBM_MODE_FULL && mode != CBM_MODE_ADVANCED) {
if (str_in_list(dirname, FAST_SKIP_DIRS)) {
return true;
}
@@ -160,7 +160,7 @@ bool cbm_has_ignored_suffix(const char *filename, cbm_index_mode_t mode) {
}
}
if (mode != CBM_MODE_FULL) {
if (mode != CBM_MODE_FULL && mode != CBM_MODE_ADVANCED) {
for (int i = 0; FAST_IGNORED_SUFFIXES[i]; i++) {
if (ends_with(filename, FAST_IGNORED_SUFFIXES[i])) {
return true;
@@ -176,7 +176,7 @@ bool cbm_should_skip_filename(const char *filename, cbm_index_mode_t mode) {
return false;
}
if (mode != CBM_MODE_FULL) {
if (mode != CBM_MODE_FULL && mode != CBM_MODE_ADVANCED) {
if (str_in_list(filename, FAST_SKIP_FILENAMES)) {
return true;
}
@@ -186,7 +186,7 @@ bool cbm_should_skip_filename(const char *filename, cbm_index_mode_t mode) {
}
bool cbm_matches_fast_pattern(const char *filename, cbm_index_mode_t mode) {
if (!filename || mode == CBM_MODE_FULL) {
if (!filename || mode == CBM_MODE_FULL || mode == CBM_MODE_ADVANCED) {
return false;
}
src/discover/discover.h
+1
View File
@@ -72,6 +72,7 @@ typedef enum {
CBM_MODE_FULL = 0, /* parse everything supported */
CBM_MODE_MODERATE = 1, /* aggressive filtering + similarity/semantic edges */
CBM_MODE_FAST = 2, /* aggressive filtering + no similarity/semantic edges */
CBM_MODE_ADVANCED = 3, /* FULL discovery + all LSP (per-file + cross-file) */
} cbm_index_mode_t;
#endif
src/mcp/mcp.c
+8 -3
View File
@@ -268,9 +268,11 @@ static const tool_def_t TOOLS[] = {
"{\"type\":\"object\",\"properties\":{\"repo_path\":{\"type\":\"string\",\"description\":"
"\"Path to the repository\"},"
"\"mode\":{\"type\":\"string\","
"\"enum\":[\"full\",\"moderate\",\"fast\",\"cross-repo-intelligence\"],"
"\"default\":\"full\",\"description\":\"full: all passes. moderate: fast + semantic. "
"fast: structure only. cross-repo-intelligence: match Routes/Channels across projects.\"},"
"\"enum\":[\"full\",\"advanced\",\"moderate\",\"fast\",\"cross-repo-intelligence\"],"
"\"default\":\"full\",\"description\":\"full: all passes EXCEPT cross-file LSP. "
"advanced: full + cross-file type-aware call/usage resolution (slowest, most precise). "
"moderate: fast + semantic. fast: structure only. "
"cross-repo-intelligence: match Routes/Channels across projects.\"},"
"\"target_projects\":{\"type\":\"array\",\"items\":{\"type\":\"string\"},"
"\"description\":\"Projects to search for cross-repo links (cross-repo-intelligence mode). "
"Use [\\\"*\\\"] for all indexed projects. Run list_projects to see available projects.\"},"
@@ -2523,6 +2525,9 @@ static char *handle_index_repository(cbm_mcp_server_t *srv, const char *args) {
mode = CBM_MODE_FAST;
} else if (mode_str && strcmp(mode_str, "moderate") == 0) {
mode = CBM_MODE_MODERATE;
} else if (mode_str && strcmp(mode_str, "advanced") == 0) {
/* Advanced = full + the cross-file LSP pass (opt-in, slowest). */
mode = CBM_MODE_ADVANCED;
}
free(mode_str);
src/pipeline/pipeline.c
+35 -6
View File
@@ -462,7 +462,11 @@ static void run_predump_passes(cbm_pipeline_t *p, cbm_pipeline_ctx_t *ctx) {
enum { PREDUMP_PASS_COUNT = 5 };
struct timespec t;
for (int i = 0; i < PREDUMP_PASS_COUNT && !check_cancel(p); i++) {
if (passes[i].moderate_only && p->mode > CBM_MODE_MODERATE) {
/* "moderate_only" passes (similarity/semantic edges) run in FULL,
* MODERATE and ADVANCED — they are skipped only in FAST. Compare
* explicitly against FAST rather than `> MODERATE` so ADVANCED
* (numerically 3) is not mistaken for a lighter mode than FULL. */
if (passes[i].moderate_only && p->mode == CBM_MODE_FAST) {
continue;
}
cbm_clock_gettime(CLOCK_MONOTONIC, &t);
@@ -482,6 +486,9 @@ static void run_predump_passes(cbm_pipeline_t *p, cbm_pipeline_ctx_t *ctx) {
static int seq_pass_lsp_cross_dispatch(cbm_pipeline_ctx_t *ctx,
const cbm_file_info_t *files, int file_count) {
if (!ctx || !ctx->result_cache) return 0;
/* Cross-file LSP is advanced-mode only (matches the parallel path
* and the per-file LSP gate). FULL and below skip it. */
if (ctx->mode != CBM_MODE_ADVANCED) return 0;
return cbm_pipeline_pass_lsp_cross(ctx, files, file_count, ctx->result_cache);
}
@@ -583,12 +590,24 @@ static int run_parallel_pipeline(cbm_pipeline_t *p, cbm_pipeline_ctx_t *ctx,
* mean cross-file LSP no-ops; per-file LSP already ran during
* extract. */
cbm_clock_gettime(CLOCK_MONOTONIC, t);
char **def_modules = (char **)calloc((size_t)file_count, sizeof(char *));
/* Cross-file LSP is the most expensive phase of the pipeline
* (type-aware call/usage resolution across files). As of the
* mode rework it is OPT-IN via CBM_MODE_ADVANCED — FULL and below
* skip it entirely. When skipped, all_defs stays NULL, which makes
* cross_lsp_eligible false for every file in the resolve worker, so
* the per-file cross-LSP step no-ops. Per-file LSP (run during
* extract) is unaffected; only the cross-FILE refinement is gated. */
const bool run_cross_lsp = (p->mode == CBM_MODE_ADVANCED);
char **def_modules = NULL;
int def_count = 0;
CBMLSPDef *all_defs = def_modules
? cbm_pxc_collect_all_defs(cache, files, file_count, ctx->project_name,
def_modules, &def_count)
: NULL;
CBMLSPDef *all_defs = NULL;
if (run_cross_lsp) {
def_modules = (char **)calloc((size_t)file_count, sizeof(char *));
all_defs = def_modules
? cbm_pxc_collect_all_defs(cache, files, file_count, ctx->project_name,
def_modules, &def_count)
: NULL;
}
/* Build inverted index: module_qn → defs. The fused resolve_worker
* uses this to filter the global all_defs[] down to just the defs
* each file actually needs (own_module + imported modules) — the
@@ -610,6 +629,8 @@ static int run_parallel_pipeline(cbm_pipeline_t *p, cbm_pipeline_ctx_t *ctx,
cbm_go_build_cross_registry(&cross_lsp_arena, all_defs, def_count);
cross_registries.python =
cbm_py_build_cross_registry(&cross_lsp_arena, all_defs, def_count);
cross_registries.c =
cbm_c_build_cross_registry(&cross_lsp_arena, all_defs, def_count);
}
cbm_log_info("pass.timing", "pass", "lsp_cross_prepare", "elapsed_ms",
itoa_buf((int)elapsed_ms(*t)));
@@ -870,10 +891,18 @@ static int run_extraction_phase(cbm_pipeline_t *p, cbm_pipeline_ctx_t *ctx,
int worker_count = cbm_default_worker_count(true);
CBM_PROF_START(t_extract_total);
/* Gate ALL LSP (per-file type-aware resolution during extract +
* the cross-file LSP pass during resolve) behind ADVANCED mode.
* Set once here before any extraction worker starts; extract
* workers read it concurrently. FULL and below get tree-sitter
* extraction + registry textual resolution only. */
bool saved_lsp = cbm_get_lsp_enabled();
cbm_set_lsp_enabled(p->mode == CBM_MODE_ADVANCED);
int rc = (worker_count > SKIP_ONE && file_count > MIN_FILES_FOR_PARALLEL)
? run_parallel_pipeline(p, ctx, files, file_count, worker_count, &t)
: run_sequential_pipeline(p, ctx, files, file_count, &t);
CBM_PROF_END_N("pipeline", "2_extraction_total", t_extract_total, file_count);
cbm_set_lsp_enabled(saved_lsp); /* restore: pipeline must not pollute g_lsp_enabled for callers */
if (check_cancel(p)) {
return CBM_NOT_FOUND;
}
src/pipeline/pipeline.h
+4
View File
@@ -34,6 +34,10 @@ typedef enum {
CBM_MODE_FULL = 0, /* Full: everything including SIMILAR_TO + SEMANTICALLY_RELATED */
CBM_MODE_MODERATE = 1, /* Moderate: fast discovery + SIMILAR_TO + SEMANTICALLY_RELATED */
CBM_MODE_FAST = 2, /* Fast: skip non-essential files, no similarity/semantic edges */
CBM_MODE_ADVANCED = 3, /* Advanced: everything in FULL + the cross-file LSP pass
* (type-aware call/usage resolution across files). The
* cross-LSP pass is the most expensive phase, so it is
* opt-in: FULL no longer runs it, ADVANCED does. */
} cbm_index_mode_t;
#endif
src/pipeline/pipeline_incremental.c
+11
View File
@@ -379,6 +379,16 @@ static void registry_visitor(const cbm_gbuf_node_t *node, void *userdata) {
static void run_extract_resolve(cbm_pipeline_ctx_t *ctx, cbm_file_info_t *changed_files, int ci) {
struct timespec t;
/* Per-file LSP gate (same as the full pipeline): advanced mode only.
* Without this, the global toggle would carry over from whatever
* index ran last in this process. Cross-file LSP stays disabled in
* incremental regardless (cbm_parallel_resolve is called with NULL
* cross_registries below). Save and restore so this run does not
* pollute g_lsp_enabled for subsequent callers (e.g. LSP unit tests
* sharing the same process). */
bool saved_lsp = cbm_get_lsp_enabled();
cbm_set_lsp_enabled(ctx->mode == CBM_MODE_ADVANCED);
#define MIN_FILES_FOR_PARALLEL_INCR 50
int worker_count = cbm_default_worker_count(true);
bool use_parallel = (worker_count > SKIP_ONE && ci > MIN_FILES_FOR_PARALLEL_INCR);
@@ -432,6 +442,7 @@ static void run_extract_resolve(cbm_pipeline_ctx_t *ctx, cbm_file_info_t *change
cbm_pipeline_pass_usages(ctx, changed_files, ci);
cbm_pipeline_pass_semantic(ctx, changed_files, ci);
}
cbm_set_lsp_enabled(saved_lsp); /* restore: incremental must not pollute g_lsp_enabled */
}
/* Run post-extraction passes (tests, decorator tags, configlink). */
src/pipeline/pipeline_internal.h
+1 -1
View File
@@ -57,7 +57,7 @@ typedef struct {
cbm_gbuf_t *gbuf; /* owned by pipeline */
cbm_registry_t *registry; /* owned by pipeline */
atomic_int *cancelled; /* pointer to pipeline's cancelled flag */
int mode; /* cbm_index_mode_t (0=full, 1=moderate, 2=fast) */
int mode; /* cbm_index_mode_t (0=full, 1=moderate, 2=fast, 3=advanced) */
/* Extraction result cache (sequential pipeline optimization).
* When non-NULL, pass_definitions stores results here instead of freeing,