From e24f0c63282caa2f861900b446fc2db2e4ae12cf Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Sun, 16 Aug 2026 18:43:44 +0200 Subject: [PATCH] perf(lsp): shared Java cross-registry + own-file overlay (#1669) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cross-file LSP for Java rebuilt a type registry for EVERY file from a def set the module filter reduces by a constant factor, not to a constant size: the JVM filter branch includes every def sharing the file's namespace, so per-file work tracks the corpus (measured defs_per_file 1,292 -> 5,031 in lockstep as defs_total went 179k -> 689k). That makes the pass O(files x corpus_defs) — 87% of a Java index and the bulk of the v0.9.0 -> v0.10.x slowdown. Three changes, one architecture (the pattern Go/Python/C/C#/TS already use): 1. cbm_java_build_cross_registry — the JVM def universe (Java + Kotlin, for mixed source roots) built ONCE per run, sealed read-only, shared across resolve workers. Wired into both pipeline.c and pipeline_incremental.c. 2. Two-phase registration inside that build: all TYPES first, finalize (hash buckets exist), then FUNCS. Func registration parses signatures, and type-name qualification via cbm_registry_lookup_type is a LINEAR scan until finalize — a single mixed pass measured 0.44 ms/def at 689k defs, a ~300 s sequential build that erased the sharing win. Two-phase: 306 s -> 2.8 s. Stable partition order because overload ties resolve to the first registered QN match. 3. cbm_run_java_lsp_cross_with_registry resolves each file against the shared base through an overlay holding exactly THIS FILE's defs (register_local_func_or_type_from_file). Own-file scope is the load- bearing choice: patch_one_method refines signatures from the AST and must write a private copy (its types live in the per-file arena, the base is sealed), and any wider scope re-imports the quadratic — a module/namespace-scoped overlay measured ratio 4.00 on the growing- package corpus, own-file measures 2.00. Elasticsearch corpus (46,477 files), same host, CBM_PROFILE=1: v0.10.5 this change v0.9.0 wall 419.9 s 85.6 s (4.9x) 61.5 s cross-LSP CPU 6,195,780ms 52,513ms (118x) 110,344 ms us_per_file 207,448 1,770 3,722 us_per_file_per_kdef 300 2 — nodes 693,100 693,100 — edges 5,646,235 5,649,949 — Cross-LSP CPU now beats v0.9.0. Nodes are identical; edges +0.066%, consistent with the shared base resolving cross-package targets the old per-file namespace/import filter could not see, plus source-order independence from the two-phase build. Guarded by the complexity suite's shared-package gate (RED at ratio 4.00 on the pre-change tree, 2.00 after — see the suite commit). Refs #1669. Signed-off-by: Martin Vogel --- internal/cbm/lsp/java_lsp.c | 144 ++++++++++++++++++++++++++++ internal/cbm/lsp/java_lsp.h | 10 ++ src/pipeline/pass_lsp_cross.c | 17 ++++ src/pipeline/pass_lsp_cross.h | 7 ++ src/pipeline/pipeline.c | 4 +- src/pipeline/pipeline_incremental.c | 2 + 6 files changed, 183 insertions(+), 1 deletion(-) diff --git a/internal/cbm/lsp/java_lsp.c b/internal/cbm/lsp/java_lsp.c index a24a93cc5..6f5d40dd8 100644 --- a/internal/cbm/lsp/java_lsp.c +++ b/internal/cbm/lsp/java_lsp.c @@ -3806,6 +3806,150 @@ void cbm_java_register_lsp_defs(CBMArena *arena, CBMTypeRegistry *reg, const CBM } } +/* ── Tier 2: shared cross-registry + per-file overlay (#1669) ────────── + * + * Without this, every Java file rebuilt a whole registry from its filtered def + * set. That set is reduced by a constant FACTOR, not to a constant SIZE, so it + * grew with the corpus (measured 1,292 -> 5,031 defs/file as the tree went + * 179k -> 689k defs) and cross-file LSP became O(files x corpus_defs): 3.7 + * ms/file on v0.9.0 versus 208.9 ms/file on v0.10.5. + * + * Build the JVM def universe ONCE, seal it read-only, and share it across + * workers. Kotlin defs are included deliberately: a mixed source root resolves + * Java->Kotlin calls, and dropping them here would silently lose those edges. */ +/* Declared in src/pipeline/pass_lsp_cross.h; extern'd here to keep the + * internal/cbm layer free of src/pipeline includes. */ +extern void cbm_pxc_count_perfile_defs(uint64_t defs); + +CBMTypeRegistry *cbm_java_build_cross_registry(CBMArena *arena, CBMLSPDef *defs, int def_count) { + if (!arena) { + return NULL; + } + CBMTypeRegistry *reg = (CBMTypeRegistry *)cbm_arena_alloc(arena, sizeof(*reg)); + if (!reg) { + return NULL; + } + cbm_registry_init(reg, arena); + cbm_java_stdlib_register(reg, arena); + /* Two-phase registration, same idiom as the per-file Pass 1a/1b split. + * + * Func registration parses signatures, and parse_param_text_full qualifies + * bare type names via cbm_registry_lookup_type. Before finalize that lookup + * is a LINEAR scan over every type registered so far, so one mixed pass is + * O(defs x types) — measured 0.44 ms/def at 689k defs, i.e. a ~300 s + * sequential build that erased the win of sharing the registry at all. + * + * Register all TYPES first (their registration does no lookups), finalize + * once so the type-QN buckets exist, then register FUNCS with O(1) type + * lookups, and finalize again to index the funcs. parse_param_text_full + * takes the registry const — no stub types appear post-finalize, so the + * post-finalize tail stays empty. Bonus over the old one-pass order: a + * signature can now resolve a type that appears LATER in defs[], the same + * source-order independence the per-file path already guarantees. */ + CBMLSPDef *jvm = (CBMLSPDef *)cbm_arena_alloc(arena, (size_t)def_count * sizeof(*jvm)); + if (!jvm) { + return NULL; + } + /* Stable two-pass partition: types in defs[] order, then funcs in defs[] + * order. Func registration order is load-bearing — overload ties resolve + * to the FIRST registered QN match — so no swap-based partition. */ + int total = 0; + for (int i = 0; i < def_count; i++) { + const char *lb = defs[i].label; + if ((defs[i].lang == CBM_LANG_JAVA || defs[i].lang == CBM_LANG_KOTLIN) && lb && + (strcmp(lb, "Class") == 0 || strcmp(lb, "Interface") == 0 || strcmp(lb, "Enum") == 0 || + strcmp(lb, "Type") == 0)) { + jvm[total++] = defs[i]; + } + } + int type_count = total; + for (int i = 0; i < def_count; i++) { + const char *lb = defs[i].label; + if ((defs[i].lang == CBM_LANG_JAVA || defs[i].lang == CBM_LANG_KOTLIN) && + !(lb && (strcmp(lb, "Class") == 0 || strcmp(lb, "Interface") == 0 || + strcmp(lb, "Enum") == 0 || strcmp(lb, "Type") == 0))) { + jvm[total++] = defs[i]; + } + } + cbm_java_register_lsp_defs(arena, reg, jvm, type_count); + cbm_registry_finalize(reg); + cbm_java_register_lsp_defs(arena, reg, jvm + type_count, total - type_count); + cbm_registry_finalize(reg); + reg->read_only = true; /* seal: shared Tier-2 registry is read-only during resolve */ + return reg; +} + +/* Per-file resolve against the shared base. The overlay holds exactly THIS + * FILE's own definitions (from the extract result), because patch_one_method() + * refines signatures from the AST and those writes must land on a private + * copy: its types live in the per-file arena, and the shared base is sealed + * read-only. Everything else — same-package siblings included — resolves + * through overlay.fallback into the shared registry. + * + * Scope matters for complexity, not just safety: a MODULE- or NAMESPACE-scoped + * overlay pays package-size per file, which is quadratic on repos that + * concentrate files in large packages (the #1669 growth pattern; pinned by + * test_complexity.c's shared-package gate). Own-file scope is O(file). */ +void cbm_run_java_lsp_cross_with_registry(CBMArena *arena, CBMFileResult *result, + const char *source, int source_len, const char *module_qn, + CBMTypeRegistry *reg, const char **import_names, + const char **import_qns, int import_count, + TSTree *cached_tree, CBMResolvedCallArray *out) { + if (!arena || !result || !source || !reg || !out) { + return; + } + + CBMTypeRegistry overlay; + cbm_registry_init(&overlay, arena); + overlay.fallback = reg; + + JavaLSPContext ctx; + java_lsp_init(&ctx, arena, source, source_len, &overlay, NULL, module_qn, out); + + register_local_func_or_type_from_file(&ctx, &overlay, result); + cbm_pxc_count_perfile_defs((uint64_t)result->defs.count); + + /* Index the overlay only — the base is already finalized. Scratch arena so + * per-file bucket allocations do not accumulate in the pipeline-lifetime + * arena across a large repo. */ + CBMArena idx_arena; + cbm_arena_init(&idx_arena); + cbm_registry_finalize_into(&overlay, &idx_arena); + + TSTree *tree = cached_tree; + bool owns_tree = false; + if (!tree) { + TSParser *parser = ts_parser_new(); + if (!parser) { + cbm_arena_destroy(&idx_arena); + return; + } + ts_parser_set_language(parser, tree_sitter_java()); + tree = ts_parser_parse_string(parser, NULL, source, (uint32_t)source_len); + ts_parser_delete(parser); + owns_tree = true; + } + if (!tree) { + cbm_arena_destroy(&idx_arena); + return; + } + TSNode root = ts_tree_root_node(tree); + + for (int i = 0; i < import_count; i++) { + if (!import_names[i] || !import_qns[i]) { + continue; + } + java_lsp_add_import(&ctx, import_names[i], import_qns[i], CBM_JAVA_IMPORT_TYPE); + } + + java_lsp_process_file(&ctx, root); + cbm_arena_destroy(&idx_arena); + + if (owns_tree) { + ts_tree_delete(tree); + } +} + void cbm_run_java_lsp_cross(CBMArena *arena, const char *source, int source_len, const char *module_qn, CBMLSPDef *defs, int def_count, const char **import_names, const char **import_qns, int import_count, diff --git a/internal/cbm/lsp/java_lsp.h b/internal/cbm/lsp/java_lsp.h index fd1a16c75..ea304c64b 100644 --- a/internal/cbm/lsp/java_lsp.h +++ b/internal/cbm/lsp/java_lsp.h @@ -142,6 +142,16 @@ void cbm_run_java_lsp(CBMArena *arena, CBMFileResult *result, const char *source void cbm_java_register_lsp_defs(CBMArena *arena, CBMTypeRegistry *reg, const CBMLSPDef *defs, int def_count); +/* Tier 2 (#1669): build the shared JVM cross registry ONCE per run, then + * resolve each file against it with a small own-module overlay. Skips the + * per-file registry build that made cross-file LSP O(files x corpus_defs). */ +CBMTypeRegistry *cbm_java_build_cross_registry(CBMArena *arena, CBMLSPDef *defs, int def_count); +void cbm_run_java_lsp_cross_with_registry(CBMArena *arena, CBMFileResult *result, + const char *source, int source_len, const char *module_qn, + CBMTypeRegistry *reg, const char **import_names, + const char **import_qns, int import_count, + TSTree *cached_tree, CBMResolvedCallArray *out); + void cbm_run_java_lsp_cross(CBMArena *arena, const char *source, int source_len, const char *module_qn, CBMLSPDef *defs, int def_count, const char **import_names, const char **import_qns, int import_count, diff --git a/src/pipeline/pass_lsp_cross.c b/src/pipeline/pass_lsp_cross.c index ac1ddf4ef..17491a9ea 100644 --- a/src/pipeline/pass_lsp_cross.c +++ b/src/pipeline/pass_lsp_cross.c @@ -1009,6 +1009,15 @@ _Atomic uint64_t g_pxc_build_files = 0; _Atomic uint64_t g_pxc_filter_files = 0; _Atomic uint64_t g_pxc_filter_failed = 0; +/* Overlay registrations count as per-file registry work too: the complexity + * gate (test_complexity.c) sums ALL defs registered per file, whichever path + * built them. Without this the shared-registry languages would report zero and + * the linearity gate would pass vacuously. */ +void cbm_pxc_count_perfile_defs(uint64_t defs) { + atomic_fetch_add_explicit(&g_pxc_defs_registered, defs, memory_order_relaxed); + atomic_fetch_add_explicit(&g_pxc_build_files, 1, memory_order_relaxed); +} + void cbm_pxc_filter_stats(uint64_t *defs_registered, uint64_t *build_files, uint64_t *filter_files, uint64_t *filter_failed) { if (defs_registered) @@ -1070,6 +1079,14 @@ void cbm_pxc_dispatch_file(CBMLanguage lang, CBMFileResult *result, const char * &result->resolved_calls); used_prebuilt = true; break; + case CBM_LANG_JAVA: + /* Own-module defs go into a per-file overlay; imports and stdlib + * resolve through the shared base (#1669). */ + cbm_run_java_lsp_cross_with_registry( + &result->arena, result, source, source_len, def_module, prebuilt, imp_keys, + imp_vals, imp_count, result->cached_tree, &result->resolved_calls); + used_prebuilt = true; + break; case CBM_LANG_JAVASCRIPT: case CBM_LANG_TYPESCRIPT: case CBM_LANG_TSX: { diff --git a/src/pipeline/pass_lsp_cross.h b/src/pipeline/pass_lsp_cross.h index d60d8fa16..149959bce 100644 --- a/src/pipeline/pass_lsp_cross.h +++ b/src/pipeline/pass_lsp_cross.h @@ -35,6 +35,7 @@ #include "lsp/c_lsp.h" /* cbm_c_build_cross_registry / cbm_run_c_lsp_cross_with_registry */ #include "lsp/cs_lsp.h" /* cbm_cs_build_cross_registry / cbm_run_cs_lsp_cross_with_registry */ #include "lsp/ts_lsp.h" /* cbm_ts_build_cross_registry / cbm_run_ts_lsp_cross_with_registry */ +#include "lsp/java_lsp.h" /* cbm_java_build_cross_registry / cbm_run_java_lsp_cross_with_registry */ #include "lsp/rust_lsp.h" /* cbm_rust_build_cross_registry / cbm_run_rust_lsp_cross_with_registry */ #include "pipeline/pipeline_internal.h" #include @@ -125,6 +126,7 @@ typedef struct { CBMTypeRegistry *ts; /* CBM_LANG_JAVASCRIPT, TYPESCRIPT, TSX */ CBMTypeRegistry *php; /* CBM_LANG_PHP */ CBMTypeRegistry *cs; /* CBM_LANG_CSHARP */ + CBMTypeRegistry *java; /* CBM_LANG_JAVA (JVM def universe incl. Kotlin defs) */ /* CBM_LANG_RUST: intentionally absent — the shared rust registry is built * LAZILY inside cbm_parallel_resolve (first NULL-filter rust file), not eagerly. */ } CBMCrossLspRegistries; @@ -133,6 +135,9 @@ typedef struct { * if none was built (or language has no cross-LSP entrypoint). */ /* Per-file registry-build cost (#1669): how many defs the per-file cross-LSP * path actually registered, and how often the module filter failed. */ +/* Count defs an overlay registered for one file (complexity-gate telemetry). */ +void cbm_pxc_count_perfile_defs(uint64_t defs); + void cbm_pxc_filter_stats(uint64_t *defs_registered, uint64_t *build_files, uint64_t *filter_files, uint64_t *filter_failed); @@ -157,6 +162,8 @@ static inline CBMTypeRegistry *cbm_pxc_registry_for_lang(const CBMCrossLspRegist return r->php; case CBM_LANG_CSHARP: return r->cs; + case CBM_LANG_JAVA: + return r->java; default: return NULL; /* incl. CBM_LANG_RUST — its shared registry is built lazily */ } diff --git a/src/pipeline/pipeline.c b/src/pipeline/pipeline.c index f38f4151f..0f2892afe 100644 --- a/src/pipeline/pipeline.c +++ b/src/pipeline/pipeline.c @@ -1254,7 +1254,7 @@ static int run_parallel_pipeline(cbm_pipeline_t *p, cbm_pipeline_ctx_t *ctx, * Built ONCE here; shared READ-ONLY across all files of that language * during resolve. Per-file work is then: parse + AST walk + O(1) lookups * — no registry build, no Phase 1b mutations. Languages added so far: - * Go, Python. Others (C/C++, TS/JS, PHP, C#) fall back to per-file. */ + * Go, Python, C/C++, C#, TS/JS, Java. Others (Kotlin, PHP) fall back to per-file. */ CBMArena cross_lsp_arena; cbm_arena_init(&cross_lsp_arena); CBMCrossLspRegistries cross_registries = {0}; @@ -1265,6 +1265,8 @@ static int run_parallel_pipeline(cbm_pipeline_t *p, cbm_pipeline_ctx_t *ctx, cross_registries.c = cbm_c_build_cross_registry(&cross_lsp_arena, all_defs, def_count); cross_registries.cs = cbm_cs_build_cross_registry(&cross_lsp_arena, all_defs, def_count); cross_registries.ts = cbm_ts_build_cross_registry(&cross_lsp_arena, all_defs, def_count); + cross_registries.java = + cbm_java_build_cross_registry(&cross_lsp_arena, all_defs, def_count); /* Rust: NOT built here. The shared all_defs registry is built LAZILY on the * first NULL-filter rust file (the amplifier files) inside cbm_parallel_resolve * — repos whose rust files all filter to subsets never pay the build/RSS. */ diff --git a/src/pipeline/pipeline_incremental.c b/src/pipeline/pipeline_incremental.c index 3e4610677..26cfbc16a 100644 --- a/src/pipeline/pipeline_incremental.c +++ b/src/pipeline/pipeline_incremental.c @@ -1334,6 +1334,8 @@ static int run_extract_resolve(cbm_pipeline_ctx_t *ctx, cbm_file_info_t *changed cross_registries.c = cbm_c_build_cross_registry(xa, all_defs, all_def_count); cross_registries.cs = cbm_cs_build_cross_registry(xa, all_defs, all_def_count); cross_registries.ts = cbm_ts_build_cross_registry(xa, all_defs, all_def_count); + cross_registries.java = + cbm_java_build_cross_registry(xa, all_defs, all_def_count); registries_arg = &cross_registries; } } else {