From bb1411620ac84b031880d782541f16396e925e08 Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Thu, 28 May 2026 19:06:09 +0200 Subject: [PATCH] refactor(pipeline): always run LSP; remove opt-in advanced mode MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The per-file and cross-file LSP type-aware call/usage resolution now runs in every index mode (full/moderate/fast) instead of being gated behind a separate CBM_MODE_ADVANCED. Remove the mode from both index-mode enums, the g_lsp_enabled toggle and its save/restore dance, and the per-mode gating in discover, pipeline and the incremental path. Now that the LSP walkers are O(n), the callee-attribution accuracy (e.g. ~91k call overrides on roslyn, ~60k on kubernetes) is worth the latency in every mode. Fast mode still filters files and skips similarity/semantic edges, so it stays faster than full while now also benefiting from LSP resolution. A 'mode':'advanced' request falls back to the full default — identical behaviour to the old advanced mode, so existing callers are unaffected. This also fixes the test_incremental.c failures, which were asserting LSP-resolved results that the gated incremental path never produced. --- internal/cbm/cbm.c | 33 +++++++---------------------- internal/cbm/cbm.h | 7 ------ src/discover/discover.c | 8 +++---- src/discover/discover.h | 1 - src/mcp/mcp.c | 13 +++++------- src/pipeline/pipeline.c | 23 ++++---------------- src/pipeline/pipeline.h | 7 +++--- src/pipeline/pipeline_incremental.c | 11 ++-------- 8 files changed, 26 insertions(+), 77 deletions(-) diff --git a/internal/cbm/cbm.c b/internal/cbm/cbm.c index ef32fa05a..f46e87ceb 100644 --- a/internal/cbm/cbm.c +++ b/internal/cbm/cbm.c @@ -28,17 +28,6 @@ static _Atomic uint64_t total_preprocess_ns = 0; static _Atomic uint64_t total_files_preprocessed = 0; static _Atomic uint64_t total_files = 0; -/* Per-file LSP toggle. The pipeline sets this ONCE before extraction - * based on index mode: only CBM_MODE_ADVANCED enables the per-file - * type-aware LSP resolution (the slowest part of extraction). Default - * true so callers that don't set it (tests, ad-hoc extraction) keep - * the historical behaviour. Set-once-before-parallel-extract + - * read-concurrently makes the atomic sufficient (no per-file variance). */ -static _Atomic bool g_lsp_enabled = true; - -void cbm_set_lsp_enabled(bool enabled) { atomic_store(&g_lsp_enabled, enabled); } -bool cbm_get_lsp_enabled(void) { return atomic_load(&g_lsp_enabled); } - #define NSEC_PER_SEC 1000000000ULL #define USEC_TO_NSEC 1000ULL /* Use compat.h's cbm_clock_gettime which accepts CLOCK_MONOTONIC (value @@ -364,12 +353,10 @@ CBMFileResult *cbm_extract_file(const char *source, int source_len, CBMLanguage cbm_extract_k8s(&ctx); } - // LSP type-aware call resolution — gated behind advanced mode. - // When disabled (FULL and below), tree-sitter extraction + - // registry-based textual resolution still produce the graph; only - // the type-aware LSP refinement is skipped. + // LSP type-aware call/usage resolution (per-file). Runs in every mode; + // refines the tree-sitter + textual-resolution graph with type info. uint64_t lsp_start = now_ns(); - if (atomic_load(&g_lsp_enabled)) { + { if (language == CBM_LANG_GO) { cbm_run_go_lsp(a, result, source, source_len, root); } @@ -450,15 +437,11 @@ CBMFileResult *cbm_extract_file(const char *source, int source_len, CBMLanguage // harmless (pipeline deduplicates by caller+callee). cbm_extract_unified(&pp_ctx); - // Also run LSP on expanded source for additional type-resolved calls - // (language is already C/C++/CUDA — checked in enclosing block). - // Gated behind advanced mode like the first-pass LSP above; - // the macro-expanded call extraction (cbm_extract_unified) is - // base extraction and always runs. - if (atomic_load(&g_lsp_enabled)) { - cbm_run_c_lsp(a, result, expanded, expanded_len, pp_root, - language != CBM_LANG_C); - } + // Also run LSP on expanded source for additional type-resolved + // calls (language is already C/C++/CUDA — checked in enclosing + // block). Runs in every mode. + cbm_run_c_lsp(a, result, expanded, expanded_len, pp_root, + language != CBM_LANG_C); ts_tree_delete(pp_tree); } diff --git a/internal/cbm/cbm.h b/internal/cbm/cbm.h index 23d2ec3ad..a6d18e8bf 100644 --- a/internal/cbm/cbm.h +++ b/internal/cbm/cbm.h @@ -529,13 +529,6 @@ uint64_t cbm_get_preprocess_ns(void); uint64_t cbm_get_files_preprocessed(void); void cbm_reset_profile(void); -/* Per-file LSP toggle. The pipeline sets this before extraction based - * on index mode (only CBM_MODE_ADVANCED enables LSP). Defaults to true - * so ad-hoc extraction and tests keep historical behaviour. Set once - * before parallel extraction; read concurrently by extract workers. */ -void cbm_set_lsp_enabled(bool enabled); -bool cbm_get_lsp_enabled(void); - // --- Internal helpers used by extractors --- // Growable array push functions (arena-allocated, no individual free needed). diff --git a/src/discover/discover.c b/src/discover/discover.c index 8baa193f9..b1e49ae89 100644 --- a/src/discover/discover.c +++ b/src/discover/discover.c @@ -140,7 +140,7 @@ bool cbm_should_skip_dir(const char *dirname, cbm_index_mode_t mode) { } /* Fast discovery applies to both MODERATE and FAST — only FULL keeps everything. */ - if (mode != CBM_MODE_FULL && mode != CBM_MODE_ADVANCED) { + if (mode != CBM_MODE_FULL) { if (str_in_list(dirname, FAST_SKIP_DIRS)) { return true; } @@ -160,7 +160,7 @@ bool cbm_has_ignored_suffix(const char *filename, cbm_index_mode_t mode) { } } - if (mode != CBM_MODE_FULL && mode != CBM_MODE_ADVANCED) { + if (mode != CBM_MODE_FULL) { for (int i = 0; FAST_IGNORED_SUFFIXES[i]; i++) { if (ends_with(filename, FAST_IGNORED_SUFFIXES[i])) { return true; @@ -176,7 +176,7 @@ bool cbm_should_skip_filename(const char *filename, cbm_index_mode_t mode) { return false; } - if (mode != CBM_MODE_FULL && mode != CBM_MODE_ADVANCED) { + if (mode != CBM_MODE_FULL) { if (str_in_list(filename, FAST_SKIP_FILENAMES)) { return true; } @@ -186,7 +186,7 @@ bool cbm_should_skip_filename(const char *filename, cbm_index_mode_t mode) { } bool cbm_matches_fast_pattern(const char *filename, cbm_index_mode_t mode) { - if (!filename || mode == CBM_MODE_FULL || mode == CBM_MODE_ADVANCED) { + if (!filename || mode == CBM_MODE_FULL) { return false; } diff --git a/src/discover/discover.h b/src/discover/discover.h index 85b870d75..c604e6f83 100644 --- a/src/discover/discover.h +++ b/src/discover/discover.h @@ -72,7 +72,6 @@ typedef enum { CBM_MODE_FULL = 0, /* parse everything supported */ CBM_MODE_MODERATE = 1, /* aggressive filtering + similarity/semantic edges */ CBM_MODE_FAST = 2, /* aggressive filtering + no similarity/semantic edges */ - CBM_MODE_ADVANCED = 3, /* FULL discovery + all LSP (per-file + cross-file) */ } cbm_index_mode_t; #endif diff --git a/src/mcp/mcp.c b/src/mcp/mcp.c index a82174e32..ac4ddb80e 100644 --- a/src/mcp/mcp.c +++ b/src/mcp/mcp.c @@ -268,11 +268,11 @@ static const tool_def_t TOOLS[] = { "{\"type\":\"object\",\"properties\":{\"repo_path\":{\"type\":\"string\",\"description\":" "\"Path to the repository\"}," "\"mode\":{\"type\":\"string\"," - "\"enum\":[\"full\",\"advanced\",\"moderate\",\"fast\",\"cross-repo-intelligence\"]," - "\"default\":\"full\",\"description\":\"full: all passes EXCEPT cross-file LSP. " - "advanced: full + cross-file type-aware call/usage resolution (slowest, most precise). " - "moderate: fast + semantic. fast: structure only. " - "cross-repo-intelligence: match Routes/Channels across projects.\"}," + "\"enum\":[\"full\",\"moderate\",\"fast\",\"cross-repo-intelligence\"]," + "\"default\":\"full\",\"description\":\"All modes run type-aware LSP call/usage " + "resolution (per-file + cross-file). full: all files + similarity/semantic edges. " + "moderate: filtered files + similarity/semantic. fast: filtered files, no " + "similarity/semantic. cross-repo-intelligence: match Routes/Channels across projects.\"}," "\"target_projects\":{\"type\":\"array\",\"items\":{\"type\":\"string\"}," "\"description\":\"Projects to search for cross-repo links (cross-repo-intelligence mode). " "Use [\\\"*\\\"] for all indexed projects. Run list_projects to see available projects.\"}," @@ -2525,9 +2525,6 @@ static char *handle_index_repository(cbm_mcp_server_t *srv, const char *args) { mode = CBM_MODE_FAST; } else if (mode_str && strcmp(mode_str, "moderate") == 0) { mode = CBM_MODE_MODERATE; - } else if (mode_str && strcmp(mode_str, "advanced") == 0) { - /* Advanced = full + the cross-file LSP pass (opt-in, slowest). */ - mode = CBM_MODE_ADVANCED; } free(mode_str); diff --git a/src/pipeline/pipeline.c b/src/pipeline/pipeline.c index aa0c60559..cd3da9b49 100644 --- a/src/pipeline/pipeline.c +++ b/src/pipeline/pipeline.c @@ -486,9 +486,7 @@ static void run_predump_passes(cbm_pipeline_t *p, cbm_pipeline_ctx_t *ctx) { static int seq_pass_lsp_cross_dispatch(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files, int file_count) { if (!ctx || !ctx->result_cache) return 0; - /* Cross-file LSP is advanced-mode only (matches the parallel path - * and the per-file LSP gate). FULL and below skip it. */ - if (ctx->mode != CBM_MODE_ADVANCED) return 0; + /* Cross-file LSP runs in every mode. */ return cbm_pipeline_pass_lsp_cross(ctx, files, file_count, ctx->result_cache); } @@ -590,14 +588,9 @@ static int run_parallel_pipeline(cbm_pipeline_t *p, cbm_pipeline_ctx_t *ctx, * mean cross-file LSP no-ops; per-file LSP already ran during * extract. */ cbm_clock_gettime(CLOCK_MONOTONIC, t); - /* Cross-file LSP is the most expensive phase of the pipeline - * (type-aware call/usage resolution across files). As of the - * mode rework it is OPT-IN via CBM_MODE_ADVANCED — FULL and below - * skip it entirely. When skipped, all_defs stays NULL, which makes - * cross_lsp_eligible false for every file in the resolve worker, so - * the per-file cross-LSP step no-ops. Per-file LSP (run during - * extract) is unaffected; only the cross-FILE refinement is gated. */ - const bool run_cross_lsp = (p->mode == CBM_MODE_ADVANCED); + /* Cross-file LSP (type-aware call/usage resolution across files) — the + * most expensive phase — runs in every mode. */ + const bool run_cross_lsp = true; char **def_modules = NULL; int def_count = 0; CBMLSPDef *all_defs = NULL; @@ -895,18 +888,10 @@ static int run_extraction_phase(cbm_pipeline_t *p, cbm_pipeline_ctx_t *ctx, int worker_count = cbm_default_worker_count(true); CBM_PROF_START(t_extract_total); - /* Gate ALL LSP (per-file type-aware resolution during extract + - * the cross-file LSP pass during resolve) behind ADVANCED mode. - * Set once here before any extraction worker starts; extract - * workers read it concurrently. FULL and below get tree-sitter - * extraction + registry textual resolution only. */ - bool saved_lsp = cbm_get_lsp_enabled(); - cbm_set_lsp_enabled(p->mode == CBM_MODE_ADVANCED); int rc = (worker_count > SKIP_ONE && file_count > MIN_FILES_FOR_PARALLEL) ? run_parallel_pipeline(p, ctx, files, file_count, worker_count, &t) : run_sequential_pipeline(p, ctx, files, file_count, &t); CBM_PROF_END_N("pipeline", "2_extraction_total", t_extract_total, file_count); - cbm_set_lsp_enabled(saved_lsp); /* restore: pipeline must not pollute g_lsp_enabled for callers */ if (check_cancel(p)) { return CBM_NOT_FOUND; } diff --git a/src/pipeline/pipeline.h b/src/pipeline/pipeline.h index 7ffb16edf..d48857554 100644 --- a/src/pipeline/pipeline.h +++ b/src/pipeline/pipeline.h @@ -31,13 +31,12 @@ typedef struct cbm_pipeline cbm_pipeline_t; #ifndef CBM_INDEX_MODE_T_DEFINED #define CBM_INDEX_MODE_T_DEFINED typedef enum { + /* All modes run the LSP type-aware call/usage resolution (per-file + + * cross-file). The mode only controls file discovery breadth and whether + * SIMILAR_TO / SEMANTICALLY_RELATED edges are computed. */ CBM_MODE_FULL = 0, /* Full: everything including SIMILAR_TO + SEMANTICALLY_RELATED */ CBM_MODE_MODERATE = 1, /* Moderate: fast discovery + SIMILAR_TO + SEMANTICALLY_RELATED */ CBM_MODE_FAST = 2, /* Fast: skip non-essential files, no similarity/semantic edges */ - CBM_MODE_ADVANCED = 3, /* Advanced: everything in FULL + the cross-file LSP pass - * (type-aware call/usage resolution across files). The - * cross-LSP pass is the most expensive phase, so it is - * opt-in: FULL no longer runs it, ADVANCED does. */ } cbm_index_mode_t; #endif diff --git a/src/pipeline/pipeline_incremental.c b/src/pipeline/pipeline_incremental.c index 76e6a15a7..d274b47e4 100644 --- a/src/pipeline/pipeline_incremental.c +++ b/src/pipeline/pipeline_incremental.c @@ -379,15 +379,9 @@ static void registry_visitor(const cbm_gbuf_node_t *node, void *userdata) { static void run_extract_resolve(cbm_pipeline_ctx_t *ctx, cbm_file_info_t *changed_files, int ci) { struct timespec t; - /* Per-file LSP gate (same as the full pipeline): advanced mode only. - * Without this, the global toggle would carry over from whatever - * index ran last in this process. Cross-file LSP stays disabled in + /* Per-file LSP always runs (every mode). Cross-file LSP stays disabled in * incremental regardless (cbm_parallel_resolve is called with NULL - * cross_registries below). Save and restore so this run does not - * pollute g_lsp_enabled for subsequent callers (e.g. LSP unit tests - * sharing the same process). */ - bool saved_lsp = cbm_get_lsp_enabled(); - cbm_set_lsp_enabled(ctx->mode == CBM_MODE_ADVANCED); + * cross_registries below). */ #define MIN_FILES_FOR_PARALLEL_INCR 50 int worker_count = cbm_default_worker_count(true); @@ -442,7 +436,6 @@ static void run_extract_resolve(cbm_pipeline_ctx_t *ctx, cbm_file_info_t *change cbm_pipeline_pass_usages(ctx, changed_files, ci); cbm_pipeline_pass_semantic(ctx, changed_files, ci); } - cbm_set_lsp_enabled(saved_lsp); /* restore: incremental must not pollute g_lsp_enabled */ } /* Run post-extraction passes (tests, decorator tags, configlink). */