mirror of
https://github.com/DeusData/codebase-memory-mcp.git
synced 2026-11-12 09:47:20 +08:00
Merge pull request #1681 from DeusData/feat/scaling-probe
perf(lsp): eliminate the cross-LSP O(n²) — shared Java registry, own-file overlay, complexity guard (#1669)
This commit is contained in:
Martin Vogel
+3
-2
@@ -505,6 +505,7 @@ TEST_FOUNDATION_SRCS = \
|
||||
tests/test_workspace.c \
|
||||
tests/test_platform.c \
|
||||
tests/test_diagnostics.c \
|
||||
tests/test_complexity.c \
|
||||
tests/test_dump_verify.c \
|
||||
tests/test_subprocess.c \
|
||||
tests/test_private_file_lock.c \
|
||||
@@ -712,9 +713,9 @@ BUILD_DIR = build/c
|
||||
# Grammar + tree-sitter runtime: compiled without -Werror (upstream code has warnings)
|
||||
GRAMMAR_CFLAGS = -std=c11 -D_DEFAULT_SOURCE -O2 -w -I$(CBM_DIR) -I$(TS_INCLUDE) -I$(TS_SRC)
|
||||
GRAMMAR_CFLAGS_TEST = -std=c11 -D_DEFAULT_SOURCE -g -O1 -w -I$(CBM_DIR) -I$(TS_INCLUDE) -I$(TS_SRC) \
|
||||
$(SANITIZED_DEFINE) $(SANITIZE)
|
||||
-DCBM_ENABLE_TEST_SEAMS=1 $(SANITIZED_DEFINE) $(SANITIZE)
|
||||
GRAMMAR_CFLAGS_TSAN = -std=c11 -D_DEFAULT_SOURCE -g -O1 -w -I$(CBM_DIR) -I$(TS_INCLUDE) -I$(TS_SRC) \
|
||||
-DCBM_SANITIZED_BUILD=1 $(TSAN_SANITIZE)
|
||||
-DCBM_ENABLE_TEST_SEAMS=1 -DCBM_SANITIZED_BUILD=1 $(TSAN_SANITIZE)
|
||||
|
||||
# Object files for grammars + ts_runtime + lsp_all + preprocessor
|
||||
GRAMMAR_OBJS_TEST = $(patsubst $(CBM_DIR)/%.c,$(BUILD_DIR)/%.o,$(GRAMMAR_SRCS))
|
||||
|
||||
@@ -1646,6 +1646,39 @@ static bool is_actual_import_boundary(CBMExtractCtx *ctx, TSNode node, const CBM
|
||||
char *name = ts_node_is_null(head) ? NULL : cbm_node_text(ctx->arena, head, ctx->source);
|
||||
|
||||
switch (ctx->language) {
|
||||
case CBM_LANG_JAVASCRIPT:
|
||||
case CBM_LANG_TYPESCRIPT:
|
||||
case CBM_LANG_TSX:
|
||||
if (strcmp(kind, "export_statement") == 0) {
|
||||
/* An export is an import CONTEXT only in its re-export forms:
|
||||
* `export ... from 'mod'` (source field) or a bare specifier list
|
||||
* `export { a, b }` with no declaration. An export OF a declaration
|
||||
* must not put the declaration's body behind inside_import — the
|
||||
* old kind-blacklist (is_export_of_declaration) missed the TS-only
|
||||
* forms (ambient_declaration, function_signature,
|
||||
* module_declaration), so declare-heavy code (.d.ts, baselines,
|
||||
* `export namespace`) ran whole subtrees as import context:
|
||||
* suppressed usages plus per-identifier ancestor walks. Positive
|
||||
* detection replaces the blacklist. */
|
||||
if (!ts_node_is_null(ts_node_child_by_field_name(node, TS_FIELD("source")))) {
|
||||
return true;
|
||||
}
|
||||
return !ts_node_is_null(cbm_find_child_by_kind(node, "export_clause")) &&
|
||||
ts_node_is_null(ts_node_child_by_field_name(node, TS_FIELD("declaration")));
|
||||
}
|
||||
return true; /* import_statement / import / require / extends: unchanged */
|
||||
case CBM_LANG_CSHARP:
|
||||
/* cs_import_types also lists namespace_declaration (so the import pass
|
||||
* can map namespace names) and using_statement (C#'s RAII block, a
|
||||
* grammar-name collision with using_directive). Neither is an import
|
||||
* CONTEXT: treating them as one put every namespaced C# file's whole
|
||||
* body behind inside_import, which both suppressed ordinary usage
|
||||
* extraction there and sent every identifier through the ancestor-
|
||||
* walking import-binding check — 92% of extract time on wide files
|
||||
* (dotnet/runtime JIT torture tests, 490 s for one 147 KB file). Only
|
||||
* the using DIRECTIVE opens an import scope. */
|
||||
return strcmp(kind, "using_directive") == 0 ||
|
||||
strcmp(kind, "namespace_use_declaration") == 0;
|
||||
case CBM_LANG_ELIXIR:
|
||||
return strcmp(kind, "call") != 0 ||
|
||||
(name && (strcmp(name, "import") == 0 || strcmp(name, "alias") == 0 ||
|
||||
|
||||
@@ -49,36 +49,6 @@ static void walk_usages(CBMExtractCtx *ctx, TSNode root, const CBMLangSpec *spec
|
||||
static bool is_direct_argument_value(TSNode node);
|
||||
static TSNode python_direct_callable_attribute_site(TSNode node);
|
||||
|
||||
// Check if a node is inside a call expression (to avoid double-counting as usage)
|
||||
static bool is_inside_call(TSNode node, const CBMLangSpec *spec) {
|
||||
TSNode cur = ts_node_parent(node);
|
||||
while (!ts_node_is_null(cur)) {
|
||||
if (cbm_kind_in_set(cur, spec->call_node_types)) {
|
||||
return true;
|
||||
}
|
||||
cur = ts_node_parent(cur);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
// Check if a node is inside an import statement
|
||||
static bool is_inside_import(TSNode node, const CBMLangSpec *spec) {
|
||||
bool has_imports = spec->import_node_types && spec->import_node_types[0];
|
||||
bool has_from_imports = spec->import_from_types && spec->import_from_types[0];
|
||||
if (!has_imports && !has_from_imports) {
|
||||
return false;
|
||||
}
|
||||
TSNode cur = ts_node_parent(node);
|
||||
while (!ts_node_is_null(cur)) {
|
||||
if ((has_imports && cbm_kind_in_set(cur, spec->import_node_types)) ||
|
||||
(has_from_imports && cbm_kind_in_set(cur, spec->import_from_types))) {
|
||||
return true;
|
||||
}
|
||||
cur = ts_node_parent(cur);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
// Is this an identifier-like node that represents a reference?
|
||||
static bool is_reference_node(TSNode node, CBMLanguage lang) {
|
||||
const char *kind = ts_node_type(node);
|
||||
@@ -2426,7 +2396,8 @@ static bool emit_direct_perl_coderef_usage(CBMExtractCtx *ctx, TSNode node,
|
||||
}
|
||||
|
||||
// Try to emit a usage for a reference node. Returns early if the node should be skipped.
|
||||
static void try_emit_usage(CBMExtractCtx *ctx, TSNode node, const CBMLangSpec *spec) {
|
||||
static void try_emit_usage(CBMExtractCtx *ctx, TSNode node, const CBMLangSpec *spec,
|
||||
bool inside_call, bool inside_import) {
|
||||
if (emit_direct_perl_coderef_usage(ctx, node, cbm_enclosing_func_qn_cached(ctx, node), 0)) {
|
||||
return;
|
||||
}
|
||||
@@ -2439,7 +2410,7 @@ static void try_emit_usage(CBMExtractCtx *ctx, TSNode node, const CBMLangSpec *s
|
||||
if (is_call_argument_label(node)) {
|
||||
return;
|
||||
}
|
||||
if (is_inside_call(node, spec) || is_inside_import(node, spec)) {
|
||||
if (inside_call || inside_import) {
|
||||
return;
|
||||
}
|
||||
if (is_binding_occurrence(ctx, node, spec, NULL) ||
|
||||
@@ -2456,18 +2427,79 @@ static void try_emit_usage(CBMExtractCtx *ctx, TSNode node, const CBMLangSpec *s
|
||||
}
|
||||
}
|
||||
|
||||
// Iterative usage walker — explicit stack
|
||||
// Iterative usage walker — explicit stack.
|
||||
//
|
||||
// The call/import ancestry that gates usage emission is maintained as ENTER/
|
||||
// EXIT counters on the walk instead of per-node ancestor re-walks: the old
|
||||
// is_inside_call/is_inside_import helpers climbed every ancestor via
|
||||
// ts_node_parent, and tree-sitter's ts_node_parent RE-DESCENDS from the root
|
||||
// scanning siblings — O(depth x sibling-position) per node, which went
|
||||
// quadratic on wide nodes (a 1,536-argument call in dotnet/runtime's JIT
|
||||
// torture tests put 92% of extract time into these walks; 490 s for one
|
||||
// 147 KB file). Counter semantics match the helpers exactly: strict ancestors
|
||||
// only — a node is emitted BEFORE its own kind increments the counters, so a
|
||||
// call node itself does not count as "inside a call".
|
||||
static void walk_usages(CBMExtractCtx *ctx, TSNode root, const CBMLangSpec *spec) {
|
||||
TSNodeStack stack;
|
||||
ts_nstack_init(&stack, ctx->arena, 4096);
|
||||
ts_nstack_push(&stack, ctx->arena, root);
|
||||
typedef struct {
|
||||
TSNode node;
|
||||
uint32_t next_child;
|
||||
bool counts_call;
|
||||
bool counts_import;
|
||||
} UsageFrame;
|
||||
int cap = 256;
|
||||
UsageFrame *frames = (UsageFrame *)cbm_arena_alloc(ctx->arena, (size_t)cap * sizeof(*frames));
|
||||
if (!frames) {
|
||||
return;
|
||||
}
|
||||
bool has_imports = spec->import_node_types && spec->import_node_types[0];
|
||||
bool has_from_imports = spec->import_from_types && spec->import_from_types[0];
|
||||
int call_depth = 0;
|
||||
int import_depth = 0;
|
||||
int top = 0;
|
||||
frames[top++] = (UsageFrame){root, 0, false, false};
|
||||
bool entering = true;
|
||||
|
||||
while (stack.count > 0) {
|
||||
TSNode node = ts_nstack_pop(&stack);
|
||||
try_emit_usage(ctx, node, spec);
|
||||
uint32_t count = ts_node_child_count(node);
|
||||
for (int i = (int)count - LAST_IDX; i >= 0; i--) {
|
||||
ts_nstack_push(&stack, ctx->arena, ts_node_child(node, (uint32_t)i));
|
||||
while (top > 0) {
|
||||
UsageFrame *f = &frames[top - 1];
|
||||
if (entering) {
|
||||
try_emit_usage(ctx, f->node, spec, call_depth > 0, import_depth > 0);
|
||||
f->counts_call = cbm_kind_in_set(f->node, spec->call_node_types);
|
||||
f->counts_import =
|
||||
(has_imports && cbm_kind_in_set(f->node, spec->import_node_types)) ||
|
||||
(has_from_imports && cbm_kind_in_set(f->node, spec->import_from_types));
|
||||
if (f->counts_call) {
|
||||
call_depth++;
|
||||
}
|
||||
if (f->counts_import) {
|
||||
import_depth++;
|
||||
}
|
||||
}
|
||||
uint32_t count = ts_node_child_count(f->node);
|
||||
if (f->next_child < count) {
|
||||
TSNode child = ts_node_child(f->node, f->next_child);
|
||||
f->next_child++;
|
||||
if (top == cap) {
|
||||
int new_cap = cap * 2;
|
||||
UsageFrame *grown =
|
||||
(UsageFrame *)cbm_arena_alloc(ctx->arena, (size_t)new_cap * sizeof(*grown));
|
||||
if (!grown) {
|
||||
return;
|
||||
}
|
||||
memcpy(grown, frames, (size_t)cap * sizeof(*frames));
|
||||
frames = grown;
|
||||
cap = new_cap;
|
||||
}
|
||||
frames[top++] = (UsageFrame){child, 0, false, false};
|
||||
entering = true;
|
||||
} else {
|
||||
if (f->counts_call) {
|
||||
call_depth--;
|
||||
}
|
||||
if (f->counts_import) {
|
||||
import_depth--;
|
||||
}
|
||||
top--;
|
||||
entering = false;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+893
-514
File diff suppressed because it is too large
Load Diff
@@ -3806,6 +3806,156 @@ void cbm_java_register_lsp_defs(CBMArena *arena, CBMTypeRegistry *reg, const CBM
|
||||
}
|
||||
}
|
||||
|
||||
/* ── Tier 2: shared cross-registry + per-file overlay (#1669) ──────────
|
||||
*
|
||||
* Without this, every Java file rebuilt a whole registry from its filtered def
|
||||
* set. That set is reduced by a constant FACTOR, not to a constant SIZE, so it
|
||||
* grew with the corpus (measured 1,292 -> 5,031 defs/file as the tree went
|
||||
* 179k -> 689k defs) and cross-file LSP became O(files x corpus_defs): 3.7
|
||||
* ms/file on v0.9.0 versus 208.9 ms/file on v0.10.5.
|
||||
*
|
||||
* Build the JVM def universe ONCE, seal it read-only, and share it across
|
||||
* workers. Kotlin defs are included deliberately: a mixed source root resolves
|
||||
* Java->Kotlin calls, and dropping them here would silently lose those edges. */
|
||||
/* Declared in src/pipeline/pass_lsp_cross.h; extern'd here to keep the
|
||||
* internal/cbm layer free of src/pipeline includes. */
|
||||
extern void cbm_pxc_count_perfile_defs(uint64_t defs);
|
||||
|
||||
CBMTypeRegistry *cbm_java_build_cross_registry(CBMArena *arena, CBMLSPDef *defs, int def_count) {
|
||||
if (!arena) {
|
||||
return NULL;
|
||||
}
|
||||
CBMTypeRegistry *reg = (CBMTypeRegistry *)cbm_arena_alloc(arena, sizeof(*reg));
|
||||
if (!reg) {
|
||||
return NULL;
|
||||
}
|
||||
cbm_registry_init(reg, arena);
|
||||
cbm_java_stdlib_register(reg, arena);
|
||||
/* Two-phase registration, same idiom as the per-file Pass 1a/1b split.
|
||||
*
|
||||
* Func registration parses signatures, and parse_param_text_full qualifies
|
||||
* bare type names via cbm_registry_lookup_type. Before finalize that lookup
|
||||
* is a LINEAR scan over every type registered so far, so one mixed pass is
|
||||
* O(defs x types) — measured 0.44 ms/def at 689k defs, i.e. a ~300 s
|
||||
* sequential build that erased the win of sharing the registry at all.
|
||||
*
|
||||
* Register all TYPES first (their registration does no lookups), finalize
|
||||
* once so the type-QN buckets exist, then register FUNCS with O(1) type
|
||||
* lookups, and finalize again to index the funcs. parse_param_text_full
|
||||
* takes the registry const — no stub types appear post-finalize, so the
|
||||
* post-finalize tail stays empty. Bonus over the old one-pass order: a
|
||||
* signature can now resolve a type that appears LATER in defs[], the same
|
||||
* source-order independence the per-file path already guarantees. */
|
||||
/* def_count == 0 is a valid corpus (no Java files): arena_alloc(0) returns
|
||||
* NULL, which must not be mistaken for OOM — the empty registry is still
|
||||
* built, finalized, and shared. */
|
||||
CBMLSPDef *jvm = NULL;
|
||||
if (def_count > 0) {
|
||||
jvm = (CBMLSPDef *)cbm_arena_alloc(arena, (size_t)def_count * sizeof(*jvm));
|
||||
if (!jvm) {
|
||||
return NULL;
|
||||
}
|
||||
}
|
||||
/* Stable two-pass partition: types in defs[] order, then funcs in defs[]
|
||||
* order. Func registration order is load-bearing — overload ties resolve
|
||||
* to the FIRST registered QN match — so no swap-based partition. */
|
||||
int total = 0;
|
||||
for (int i = 0; i < def_count; i++) {
|
||||
const char *lb = defs[i].label;
|
||||
if ((defs[i].lang == CBM_LANG_JAVA || defs[i].lang == CBM_LANG_KOTLIN) && lb &&
|
||||
(strcmp(lb, "Class") == 0 || strcmp(lb, "Interface") == 0 || strcmp(lb, "Enum") == 0 ||
|
||||
strcmp(lb, "Type") == 0)) {
|
||||
jvm[total++] = defs[i];
|
||||
}
|
||||
}
|
||||
int type_count = total;
|
||||
for (int i = 0; i < def_count; i++) {
|
||||
const char *lb = defs[i].label;
|
||||
if ((defs[i].lang == CBM_LANG_JAVA || defs[i].lang == CBM_LANG_KOTLIN) &&
|
||||
!(lb && (strcmp(lb, "Class") == 0 || strcmp(lb, "Interface") == 0 ||
|
||||
strcmp(lb, "Enum") == 0 || strcmp(lb, "Type") == 0))) {
|
||||
jvm[total++] = defs[i];
|
||||
}
|
||||
}
|
||||
cbm_java_register_lsp_defs(arena, reg, jvm, type_count);
|
||||
cbm_registry_finalize(reg);
|
||||
cbm_java_register_lsp_defs(arena, reg, jvm + type_count, total - type_count);
|
||||
cbm_registry_finalize(reg);
|
||||
reg->read_only = true; /* seal: shared Tier-2 registry is read-only during resolve */
|
||||
return reg;
|
||||
}
|
||||
|
||||
/* Per-file resolve against the shared base. The overlay holds exactly THIS
|
||||
* FILE's own definitions (from the extract result), because patch_one_method()
|
||||
* refines signatures from the AST and those writes must land on a private
|
||||
* copy: its types live in the per-file arena, and the shared base is sealed
|
||||
* read-only. Everything else — same-package siblings included — resolves
|
||||
* through overlay.fallback into the shared registry.
|
||||
*
|
||||
* Scope matters for complexity, not just safety: a MODULE- or NAMESPACE-scoped
|
||||
* overlay pays package-size per file, which is quadratic on repos that
|
||||
* concentrate files in large packages (the #1669 growth pattern; pinned by
|
||||
* test_complexity.c's shared-package gate). Own-file scope is O(file). */
|
||||
void cbm_run_java_lsp_cross_with_registry(CBMArena *arena, CBMFileResult *result,
|
||||
const char *source, int source_len, const char *module_qn,
|
||||
CBMTypeRegistry *reg, const char **import_names,
|
||||
const char **import_qns, int import_count,
|
||||
TSTree *cached_tree, CBMResolvedCallArray *out) {
|
||||
if (!arena || !result || !source || !reg || !out) {
|
||||
return;
|
||||
}
|
||||
|
||||
CBMTypeRegistry overlay;
|
||||
cbm_registry_init(&overlay, arena);
|
||||
overlay.fallback = reg;
|
||||
|
||||
JavaLSPContext ctx;
|
||||
java_lsp_init(&ctx, arena, source, source_len, &overlay, NULL, module_qn, out);
|
||||
|
||||
register_local_func_or_type_from_file(&ctx, &overlay, result);
|
||||
cbm_pxc_count_perfile_defs((uint64_t)result->defs.count);
|
||||
|
||||
/* Index the overlay only — the base is already finalized. Scratch arena so
|
||||
* per-file bucket allocations do not accumulate in the pipeline-lifetime
|
||||
* arena across a large repo. */
|
||||
CBMArena idx_arena;
|
||||
cbm_arena_init(&idx_arena);
|
||||
cbm_registry_finalize_into(&overlay, &idx_arena);
|
||||
|
||||
TSTree *tree = cached_tree;
|
||||
bool owns_tree = false;
|
||||
if (!tree) {
|
||||
TSParser *parser = ts_parser_new();
|
||||
if (!parser) {
|
||||
cbm_arena_destroy(&idx_arena);
|
||||
return;
|
||||
}
|
||||
ts_parser_set_language(parser, tree_sitter_java());
|
||||
tree = ts_parser_parse_string(parser, NULL, source, (uint32_t)source_len);
|
||||
ts_parser_delete(parser);
|
||||
owns_tree = true;
|
||||
}
|
||||
if (!tree) {
|
||||
cbm_arena_destroy(&idx_arena);
|
||||
return;
|
||||
}
|
||||
TSNode root = ts_tree_root_node(tree);
|
||||
|
||||
for (int i = 0; i < import_count; i++) {
|
||||
if (!import_names[i] || !import_qns[i]) {
|
||||
continue;
|
||||
}
|
||||
java_lsp_add_import(&ctx, import_names[i], import_qns[i], CBM_JAVA_IMPORT_TYPE);
|
||||
}
|
||||
|
||||
java_lsp_process_file(&ctx, root);
|
||||
cbm_arena_destroy(&idx_arena);
|
||||
|
||||
if (owns_tree) {
|
||||
ts_tree_delete(tree);
|
||||
}
|
||||
}
|
||||
|
||||
void cbm_run_java_lsp_cross(CBMArena *arena, const char *source, int source_len,
|
||||
const char *module_qn, CBMLSPDef *defs, int def_count,
|
||||
const char **import_names, const char **import_qns, int import_count,
|
||||
|
||||
@@ -142,6 +142,16 @@ void cbm_run_java_lsp(CBMArena *arena, CBMFileResult *result, const char *source
|
||||
void cbm_java_register_lsp_defs(CBMArena *arena, CBMTypeRegistry *reg, const CBMLSPDef *defs,
|
||||
int def_count);
|
||||
|
||||
/* Tier 2 (#1669): build the shared JVM cross registry ONCE per run, then
|
||||
* resolve each file against it with a small own-module overlay. Skips the
|
||||
* per-file registry build that made cross-file LSP O(files x corpus_defs). */
|
||||
CBMTypeRegistry *cbm_java_build_cross_registry(CBMArena *arena, CBMLSPDef *defs, int def_count);
|
||||
void cbm_run_java_lsp_cross_with_registry(CBMArena *arena, CBMFileResult *result,
|
||||
const char *source, int source_len, const char *module_qn,
|
||||
CBMTypeRegistry *reg, const char **import_names,
|
||||
const char **import_qns, int import_count,
|
||||
TSTree *cached_tree, CBMResolvedCallArray *out);
|
||||
|
||||
void cbm_run_java_lsp_cross(CBMArena *arena, const char *source, int source_len,
|
||||
const char *module_qn, CBMLSPDef *defs, int def_count,
|
||||
const char **import_names, const char **import_qns, int import_count,
|
||||
|
||||
+141
-1
@@ -70,6 +70,107 @@ static void ts_type_budget_reset(size_t source_len) {
|
||||
g_ts_type_budget_warned = false;
|
||||
}
|
||||
|
||||
/* Bumped on every degraded eval return (depth cap or budget exhaustion). An
|
||||
* evaluation whose subtree bumped this counter must NOT be memoized: its value
|
||||
* embeds the degradation, and first-eval-wins would freeze it for call sites
|
||||
* that could have evaluated the node fully. */
|
||||
static _Thread_local unsigned long g_ts_eval_degraded;
|
||||
|
||||
/* Expression-type memo (per file): node.id -> evaluated CBMType.
|
||||
*
|
||||
* ts_signature_for_call re-evaluates a call's argument expressions while
|
||||
* resolving overloads, and boolean/spread chains re-enter the same
|
||||
* subexpression once per enclosing alternative — the TS suite's
|
||||
* objectSpreadRepeatedComplexity.js builds a union of 2^(n-1) members and
|
||||
* measured 11+ s for a 3.6 KB file while emitting the same 5-node graph
|
||||
* v0.9.0 emits in 0.14 s. Expression types are position-pure within a file
|
||||
* pass (one node = one scope path, and the per-file walk is single-threaded
|
||||
* and deterministic), so one eval per node is the correct semantics, not a
|
||||
* cache trade-off. Degraded results are never stored (see
|
||||
* g_ts_eval_degraded). Arena-backed, linear-probed, power-of-two capacity;
|
||||
* grows by rehash and dies with the file pass. */
|
||||
typedef struct TsEvalMemo {
|
||||
const void **keys; /* TSNode.id; NULL = empty slot */
|
||||
const CBMType **vals;
|
||||
uint32_t cap;
|
||||
uint32_t count;
|
||||
} TsEvalMemo;
|
||||
|
||||
static uint32_t ts_memo_slot(const TsEvalMemo *m, const void *id) {
|
||||
uint64_t x = (uint64_t)(uintptr_t)id;
|
||||
x ^= x >> 33;
|
||||
x *= 0xff51afd7ed558ccdULL;
|
||||
x ^= x >> 33;
|
||||
uint32_t i = (uint32_t)x & (m->cap - 1);
|
||||
while (m->keys[i] && m->keys[i] != id)
|
||||
i = (i + 1) & (m->cap - 1);
|
||||
return i;
|
||||
}
|
||||
|
||||
static const CBMType *ts_memo_get(const TsEvalMemo *m, const void *id) {
|
||||
if (!m || !m->cap)
|
||||
return NULL;
|
||||
uint32_t i = ts_memo_slot(m, id);
|
||||
return m->keys[i] ? m->vals[i] : NULL;
|
||||
}
|
||||
|
||||
static void ts_memo_put(TSLSPContext *ctx, const void *id, const CBMType *t) {
|
||||
TsEvalMemo *m = ctx->eval_memo;
|
||||
if (!m) {
|
||||
m = (TsEvalMemo *)cbm_arena_alloc(ctx->arena, sizeof(*m));
|
||||
if (!m)
|
||||
return;
|
||||
m->cap = 1024;
|
||||
m->count = 0;
|
||||
m->keys = (const void **)cbm_arena_alloc(ctx->arena, m->cap * sizeof(*m->keys));
|
||||
m->vals = (const CBMType **)cbm_arena_alloc(ctx->arena, m->cap * sizeof(*m->vals));
|
||||
if (!m->keys || !m->vals)
|
||||
return;
|
||||
memset(m->keys, 0, m->cap * sizeof(*m->keys));
|
||||
memset(m->vals, 0, m->cap * sizeof(*m->vals));
|
||||
ctx->eval_memo = m;
|
||||
}
|
||||
if (m->count * 10 >= m->cap * 7) { /* grow at 70% load */
|
||||
TsEvalMemo bigger = {0};
|
||||
bigger.cap = m->cap * 2;
|
||||
bigger.keys = (const void **)cbm_arena_alloc(ctx->arena, bigger.cap * sizeof(*bigger.keys));
|
||||
bigger.vals =
|
||||
(const CBMType **)cbm_arena_alloc(ctx->arena, bigger.cap * sizeof(*bigger.vals));
|
||||
if (!bigger.keys || !bigger.vals)
|
||||
return; /* keep serving from the full table; stores stop */
|
||||
memset(bigger.keys, 0, bigger.cap * sizeof(*bigger.keys));
|
||||
memset(bigger.vals, 0, bigger.cap * sizeof(*bigger.vals));
|
||||
for (uint32_t i = 0; i < m->cap; i++) {
|
||||
if (!m->keys[i])
|
||||
continue;
|
||||
uint32_t j = ts_memo_slot(&bigger, m->keys[i]);
|
||||
bigger.keys[j] = m->keys[i];
|
||||
bigger.vals[j] = m->vals[i];
|
||||
}
|
||||
bigger.count = m->count;
|
||||
*m = bigger; /* old arrays stay in the arena */
|
||||
}
|
||||
uint32_t i = ts_memo_slot(m, id);
|
||||
if (!m->keys[i]) {
|
||||
m->keys[i] = id;
|
||||
m->vals[i] = t;
|
||||
m->count++;
|
||||
}
|
||||
}
|
||||
|
||||
#ifdef CBM_ENABLE_TEST_SEAMS
|
||||
/* Complexity-regression seam: expose the thread's remaining eval budget and
|
||||
* the warned flag after a cbm_run_ts_lsp call. Budget units consumed are a
|
||||
* deterministic work counter — the memo turns the spread-bomb class from
|
||||
* O(2^n) evals into O(nodes), which tests assert without wall-clock. */
|
||||
long cbm_ts_lsp_test_budget_remaining(void) {
|
||||
return g_ts_type_budget;
|
||||
}
|
||||
bool cbm_ts_lsp_test_budget_warned(void) {
|
||||
return g_ts_type_budget_warned;
|
||||
}
|
||||
#endif
|
||||
|
||||
#define TS_LSP_MAX_EVAL_DEPTH 64
|
||||
#define TS_LSP_FIELD_LEN(s) ((uint32_t)(sizeof(s) - 1))
|
||||
|
||||
@@ -1978,9 +2079,43 @@ static const CBMType *return_type_of(CBMArena *arena, const CBMType *sig) {
|
||||
const CBMType *ts_eval_expr_type(TSLSPContext *ctx, TSNode node) {
|
||||
if (!ctx || ts_node_is_null(node))
|
||||
return cbm_type_unknown();
|
||||
if (ctx->eval_depth > TS_LSP_MAX_EVAL_DEPTH)
|
||||
/* Memo hit: O(1), charges no budget, ignores depth — the value came from a
|
||||
* completed, non-degraded evaluation of this exact node. */
|
||||
{
|
||||
const CBMType *memo = ts_memo_get(ctx->eval_memo, node.id);
|
||||
if (memo)
|
||||
return memo;
|
||||
}
|
||||
if (ctx->eval_depth > TS_LSP_MAX_EVAL_DEPTH) {
|
||||
g_ts_eval_degraded++;
|
||||
return cbm_type_unknown();
|
||||
}
|
||||
/* Depth alone does not bound WORK: crafted expressions (e.g. the TS test
|
||||
* suite's repeated object spreads) stay under the depth cap while fanning
|
||||
* out exponentially — one 3.6 KB baseline file measured 11 s here. Charge
|
||||
* the same per-file budget the type-text parser uses and degrade to
|
||||
* UNKNOWN on exhaustion, exactly like that path. */
|
||||
if (g_ts_type_budget >= 0) {
|
||||
/* An eval entry does ~two orders of magnitude more work than one
|
||||
* text-parse unit (allocations, lookups, recursion bookkeeping), so it
|
||||
* charges accordingly — otherwise the default per-file budget still
|
||||
* permits ~12 s of evaluation on a crafted file. Normal files run
|
||||
* hundreds-to-thousands of entries, far under any budget. */
|
||||
g_ts_type_budget -= 16;
|
||||
if (g_ts_type_budget < 0) {
|
||||
if (!g_ts_type_budget_warned) {
|
||||
g_ts_type_budget_warned = true;
|
||||
fprintf(stderr, " [tslsp] expression-eval budget exhausted; degrading to "
|
||||
"unknown\n");
|
||||
}
|
||||
g_ts_eval_degraded++;
|
||||
return cbm_type_unknown();
|
||||
}
|
||||
}
|
||||
ctx->eval_depth++;
|
||||
/* Snapshot for the memo-store guard: any degraded return in this subtree
|
||||
* bumps the counter and blocks the store below. */
|
||||
unsigned long degraded_before = g_ts_eval_degraded;
|
||||
|
||||
const CBMType *result = cbm_type_unknown();
|
||||
const char *kind = ts_node_type(node);
|
||||
@@ -2240,6 +2375,11 @@ const CBMType *ts_eval_expr_type(TSLSPContext *ctx, TSNode node) {
|
||||
}
|
||||
|
||||
ctx->eval_depth--;
|
||||
/* Store only completed, non-degraded evaluations (see g_ts_eval_degraded);
|
||||
* result is never NULL here (initialized to unknown), so NULL stays the
|
||||
* empty-slot sentinel. */
|
||||
if (result && g_ts_eval_degraded == degraded_before)
|
||||
ts_memo_put(ctx, node.id, result);
|
||||
return result;
|
||||
}
|
||||
|
||||
|
||||
@@ -61,12 +61,23 @@ typedef struct {
|
||||
|
||||
// Recursion guard for ts_eval_expr_type (mirrors c_lsp).
|
||||
int eval_depth;
|
||||
// Expression-type memo: node.id -> evaluated type (lazily created on the
|
||||
// first completed eval; see TsEvalMemo in ts_lsp.c). Kills the exponential
|
||||
// re-evaluation of shared subexpressions under overload resolution.
|
||||
struct TsEvalMemo *eval_memo;
|
||||
// Recursion guard for lookup_member_type: cyclic type graphs (mutually
|
||||
// recursive unions/wrappers across registered types) otherwise recurse
|
||||
// without bound — stack overflow on real repos.
|
||||
int member_depth;
|
||||
} TSLSPContext;
|
||||
|
||||
#ifdef CBM_ENABLE_TEST_SEAMS
|
||||
// Complexity-regression seam: remaining expression-eval budget / warned flag
|
||||
// for the calling thread, valid after a cbm_run_ts_lsp on the same thread.
|
||||
long cbm_ts_lsp_test_budget_remaining(void);
|
||||
bool cbm_ts_lsp_test_budget_warned(void);
|
||||
#endif
|
||||
|
||||
// --- Initialization ---
|
||||
|
||||
// Initialise a TSLSPContext for processing one file. Mode flags select dialect.
|
||||
|
||||
@@ -169,6 +169,46 @@ static void build_type_embed_index(CBMTypeRegistry *reg, CBMArena *idx_arena) {
|
||||
|
||||
/* Index: short_name -> chain of FREE-function (receiver_type==NULL) indices.
|
||||
* Descending-iterate + prepend for ascending chain order (as above). */
|
||||
static void build_type_short_index(CBMTypeRegistry *reg, CBMArena *idx_arena) {
|
||||
int tcount = 0;
|
||||
for (int i = 0; i < reg->type_count; i++) {
|
||||
if (reg->types[i].short_name)
|
||||
tcount++;
|
||||
}
|
||||
if (tcount == 0)
|
||||
return;
|
||||
int bucket_count = next_pow2(tcount * 2);
|
||||
if (bucket_count < 16)
|
||||
bucket_count = 16;
|
||||
int *buckets = (int *)cbm_arena_alloc(idx_arena, (size_t)bucket_count * sizeof(int));
|
||||
CBMRegistryHashEntry *entries = (CBMRegistryHashEntry *)cbm_arena_alloc(
|
||||
idx_arena, (size_t)tcount * sizeof(CBMRegistryHashEntry));
|
||||
if (!buckets || !entries)
|
||||
return;
|
||||
for (int i = 0; i < bucket_count; i++)
|
||||
buckets[i] = -1;
|
||||
int idx = 0;
|
||||
/* Reverse insertion so each chain yields ASCENDING types[] order — callers
|
||||
* that used first-match-in-registration-order keep their tie-breaks. */
|
||||
for (int i = reg->type_count - 1; i >= 0; i--) {
|
||||
const CBMRegisteredType *t = ®->types[i];
|
||||
if (!t->short_name)
|
||||
continue;
|
||||
uint64_t h = fnv1a(t->short_name);
|
||||
int slot = (int)(h & (uint64_t)(bucket_count - 1));
|
||||
entries[idx].hash = h;
|
||||
entries[idx].payload_index = i;
|
||||
entries[idx].next_index = buckets[slot];
|
||||
entries[idx].slot = slot;
|
||||
buckets[slot] = idx;
|
||||
idx++;
|
||||
}
|
||||
reg->type_short_buckets = buckets;
|
||||
reg->type_short_entries = entries;
|
||||
reg->type_short_bucket_count = bucket_count;
|
||||
reg->type_short_entry_count = idx;
|
||||
}
|
||||
|
||||
static void build_ffunc_short_index(CBMTypeRegistry *reg, CBMArena *idx_arena) {
|
||||
int fcount = 0;
|
||||
for (int i = 0; i < reg->func_count; i++) {
|
||||
@@ -286,6 +326,42 @@ int cbm_free_func_iter_next(CBMFreeFuncIter *it) {
|
||||
return -1;
|
||||
}
|
||||
|
||||
void cbm_registry_types_by_short_name(const CBMTypeRegistry *reg, const char *short_name,
|
||||
CBMTypeShortIter *out) {
|
||||
out->reg = reg;
|
||||
out->hash = fnv1a(short_name);
|
||||
if (reg->type_qn_buckets && reg->type_qn_bucket_count > 0) {
|
||||
if (reg->type_short_buckets && reg->type_short_bucket_count > 0) {
|
||||
int slot = (int)(out->hash & (uint64_t)(reg->type_short_bucket_count - 1));
|
||||
out->chain_idx = reg->type_short_buckets[slot];
|
||||
} else {
|
||||
out->chain_idx = -1;
|
||||
}
|
||||
out->tail_i = reg->type_qn_entry_count;
|
||||
out->tail_end = reg->type_count;
|
||||
} else {
|
||||
out->chain_idx = -1;
|
||||
out->tail_i = 0;
|
||||
out->tail_end = reg->type_count;
|
||||
}
|
||||
}
|
||||
|
||||
int cbm_type_short_iter_next(CBMTypeShortIter *it) {
|
||||
const CBMTypeRegistry *reg = it->reg;
|
||||
while (it->chain_idx >= 0) {
|
||||
const CBMRegistryHashEntry *e = ®->type_short_entries[it->chain_idx];
|
||||
int p = e->payload_index;
|
||||
uint64_t h = e->hash;
|
||||
it->chain_idx = e->next_index;
|
||||
if (h != it->hash)
|
||||
continue;
|
||||
return p;
|
||||
}
|
||||
if (it->tail_i < it->tail_end)
|
||||
return it->tail_i++;
|
||||
return -1;
|
||||
}
|
||||
|
||||
void cbm_registry_methods(const CBMTypeRegistry *reg, const char *receiver_qn,
|
||||
const char *method_name, CBMMethodIter *out) {
|
||||
memset(out, 0, sizeof(*out));
|
||||
@@ -346,6 +422,7 @@ void cbm_registry_finalize_into(CBMTypeRegistry *reg, CBMArena *idx_arena) {
|
||||
build_method_index(reg, idx_arena);
|
||||
build_type_embed_index(reg, idx_arena);
|
||||
build_ffunc_short_index(reg, idx_arena);
|
||||
build_type_short_index(reg, idx_arena);
|
||||
}
|
||||
|
||||
void cbm_registry_finalize(CBMTypeRegistry *reg) {
|
||||
|
||||
@@ -123,6 +123,10 @@ typedef struct CBMTypeRegistry {
|
||||
int type_embed_entry_count;
|
||||
// Free-function short-name index: fnv1a(short_name) -> chain of FREE-function
|
||||
// (receiver_type==NULL) indices. payload_index = func index.
|
||||
int *type_short_buckets;
|
||||
CBMRegistryHashEntry *type_short_entries;
|
||||
int type_short_bucket_count;
|
||||
int type_short_entry_count;
|
||||
int *ffunc_short_buckets;
|
||||
CBMRegistryHashEntry *ffunc_short_entries;
|
||||
int ffunc_short_bucket_count;
|
||||
@@ -248,6 +252,22 @@ typedef struct {
|
||||
int tail_i;
|
||||
int tail_end;
|
||||
} CBMFreeFuncIter;
|
||||
/* Iterate TYPE indices sharing a short name — the type-side twin of the free-
|
||||
* function iterator. Built by finalize into type_short_buckets; degrades to a
|
||||
* full types[] scan on an unfinalized registry (same correctness, old cost).
|
||||
* Added for cs_resolve_type_name's step-9 fallback, which scanned type_count
|
||||
* per unresolved name — quadratic against the shared Tier-2 registry. */
|
||||
typedef struct {
|
||||
const CBMTypeRegistry *reg;
|
||||
uint64_t hash;
|
||||
int chain_idx;
|
||||
int tail_i;
|
||||
int tail_end;
|
||||
} CBMTypeShortIter;
|
||||
void cbm_registry_types_by_short_name(const CBMTypeRegistry *reg, const char *short_name,
|
||||
CBMTypeShortIter *out);
|
||||
int cbm_type_short_iter_next(CBMTypeShortIter *it);
|
||||
|
||||
void cbm_registry_free_funcs_by_short_name(const CBMTypeRegistry *reg, const char *short_name,
|
||||
CBMFreeFuncIter *out);
|
||||
int cbm_free_func_iter_next(CBMFreeFuncIter *it);
|
||||
|
||||
@@ -5,6 +5,7 @@
|
||||
#include "foundation/log.h"
|
||||
#include "foundation/compat.h"
|
||||
|
||||
#include <math.h>
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <time.h>
|
||||
@@ -64,3 +65,122 @@ void cbm_profile_log_elapsed(const char *phase, const char *sub, const struct ti
|
||||
cbm_log_info("prof", "phase", phase, "sub", sub, "ms", ms_buf, "us", us_buf);
|
||||
}
|
||||
}
|
||||
|
||||
/* ── Scaling probe ──────────────────────────────────────────────────── */
|
||||
|
||||
enum {
|
||||
/* Below this, scheduling noise swamps the fit and every short pass would
|
||||
* cry wolf. A pass with fewer items than this is not where an O(n^2)
|
||||
* regression hurts anyone. */
|
||||
SCALE_MIN_ITEMS = 512,
|
||||
/* Likewise in time: a first checkpoint under a millisecond is measuring the
|
||||
* clock, not the workload. */
|
||||
SCALE_MIN_FIRST_US = 1000,
|
||||
};
|
||||
|
||||
static long scale_elapsed_us(const struct timespec *start) {
|
||||
struct timespec now;
|
||||
cbm_clock_gettime(CLOCK_MONOTONIC, &now);
|
||||
return ((long)(now.tv_sec - start->tv_sec) * PROF_US_PER_SEC) +
|
||||
((now.tv_nsec - start->tv_nsec) / PROF_NS_PER_US);
|
||||
}
|
||||
|
||||
void cbm_scale_begin(cbm_scale_probe_t *probe, const char *phase, long total) {
|
||||
if (!probe) {
|
||||
return;
|
||||
}
|
||||
probe->phase = phase;
|
||||
probe->total = total;
|
||||
atomic_init(&probe->next_cp, 0);
|
||||
for (int i = 0; i < CBM_SCALE_CHECKPOINTS; i++) {
|
||||
probe->cp_us[i] = 0;
|
||||
probe->cp_items[i] = 0;
|
||||
}
|
||||
cbm_clock_gettime(CLOCK_MONOTONIC, &probe->start);
|
||||
}
|
||||
|
||||
void cbm_scale_tick(cbm_scale_probe_t *probe, long done) {
|
||||
if (!probe || probe->total < SCALE_MIN_ITEMS) {
|
||||
return;
|
||||
}
|
||||
int cp = atomic_load_explicit(&probe->next_cp, memory_order_relaxed);
|
||||
if (cp >= CBM_SCALE_CHECKPOINTS) {
|
||||
return;
|
||||
}
|
||||
/* cp 0..3 -> total/8, total/4, total/2, total */
|
||||
long threshold = probe->total >> (CBM_SCALE_CHECKPOINTS - 1 - cp);
|
||||
if (done < threshold) {
|
||||
return;
|
||||
}
|
||||
/* Exactly one thread records each checkpoint; a loser simply moves on and
|
||||
* will re-evaluate against the next threshold on its following tick. */
|
||||
if (!atomic_compare_exchange_strong_explicit(&probe->next_cp, &cp, cp + 1, memory_order_relaxed,
|
||||
memory_order_relaxed)) {
|
||||
return;
|
||||
}
|
||||
probe->cp_us[cp] = scale_elapsed_us(&probe->start);
|
||||
probe->cp_items[cp] = done;
|
||||
}
|
||||
|
||||
double cbm_scale_fit_k(long first_n, long first_us, long last_n, long last_us) {
|
||||
if (first_n <= 0 || first_us <= 0 || last_us <= 0 || last_n <= first_n) {
|
||||
return -1.0;
|
||||
}
|
||||
/* k = log(T_last / T_first) / log(n_last / n_first) */
|
||||
return log((double)last_us / (double)first_us) / log((double)last_n / (double)first_n);
|
||||
}
|
||||
|
||||
void cbm_scale_end(cbm_scale_probe_t *probe) {
|
||||
if (!probe || probe->total < SCALE_MIN_ITEMS) {
|
||||
return;
|
||||
}
|
||||
int recorded = atomic_load_explicit(&probe->next_cp, memory_order_relaxed);
|
||||
if (recorded < 2) {
|
||||
return; /* need two points to speak about growth at all */
|
||||
}
|
||||
long first_us = probe->cp_us[0];
|
||||
long first_n = probe->cp_items[0];
|
||||
long last_us = probe->cp_us[recorded - 1];
|
||||
long last_n = probe->cp_items[recorded - 1];
|
||||
if (first_us < SCALE_MIN_FIRST_US || first_n <= 0 || last_n <= first_n || last_us <= 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
double k = cbm_scale_fit_k(first_n, first_us, last_n, last_us);
|
||||
if (k < 0.0) {
|
||||
return;
|
||||
}
|
||||
long us_per_item = last_us / last_n;
|
||||
|
||||
char k_buf[PROF_BUF_LEN];
|
||||
char n_buf[PROF_BUF_LEN];
|
||||
char per_buf[PROF_BUF_LEN];
|
||||
char ms_buf[PROF_BUF_LEN];
|
||||
snprintf(k_buf, sizeof(k_buf), "%.2f", k);
|
||||
snprintf(n_buf, sizeof(n_buf), "%ld", last_n);
|
||||
snprintf(per_buf, sizeof(per_buf), "%ld", us_per_item);
|
||||
snprintf(ms_buf, sizeof(ms_buf), "%ld", last_us / PROF_US_PER_MS);
|
||||
|
||||
if (k >= CBM_SCALE_WARN_K) {
|
||||
/* Unflagged on purpose: this is the alarm the next accidental O(n^2)
|
||||
* should trip in an ordinary user's log. */
|
||||
cbm_log_warn("scaling.superlinear", "phase", probe->phase, "k", k_buf, "items", n_buf,
|
||||
"elapsed_ms", ms_buf, "us_per_item", per_buf);
|
||||
}
|
||||
if (!cbm_profile_active) {
|
||||
return;
|
||||
}
|
||||
char curve[PROF_BUF_LEN * CBM_SCALE_CHECKPOINTS];
|
||||
int used = 0;
|
||||
for (int i = 0; i < recorded && used < (int)sizeof(curve) - 1; i++) {
|
||||
int wrote =
|
||||
snprintf(curve + used, sizeof(curve) - (size_t)used, "%s%ld:%ld", i == 0 ? "" : ",",
|
||||
probe->cp_items[i], probe->cp_us[i] / PROF_US_PER_MS);
|
||||
if (wrote < 0) {
|
||||
break;
|
||||
}
|
||||
used += wrote;
|
||||
}
|
||||
cbm_log_info("scaling", "phase", probe->phase, "k", k_buf, "items", n_buf, "elapsed_ms", ms_buf,
|
||||
"us_per_item", per_buf, "curve_items_ms", curve);
|
||||
}
|
||||
|
||||
@@ -15,12 +15,88 @@
|
||||
#ifndef CBM_PROFILE_H
|
||||
#define CBM_PROFILE_H
|
||||
|
||||
#include <stdatomic.h>
|
||||
#include <stdbool.h>
|
||||
#include <time.h>
|
||||
|
||||
/* Runtime-active flag. Set once by cbm_profile_init() from CBM_PROFILE env. */
|
||||
extern bool cbm_profile_active;
|
||||
|
||||
/* ── Scaling probe — single-run superlinearity detector ──────────────
|
||||
*
|
||||
* Per-pass elapsed time (pass.timing) tells you a pass is slow. It cannot tell
|
||||
* you WHY: big corpus, or O(n^2)? Establishing that for #1669 took an 11-corpus
|
||||
* A/B across two release binaries, where parallel_resolve turned out to be 87%
|
||||
* of a Java index.
|
||||
*
|
||||
* This makes the same question answerable from ONE run. A pass processing
|
||||
* `total` items records cumulative elapsed at 1/8, 1/4, 1/2 and 1 of them, then
|
||||
* fits the exponent k in T ~ n^k:
|
||||
*
|
||||
* k ~ 1.0 linear — cost per item is flat
|
||||
* k ~ 1.5 superlinear — e.g. a candidate set that grows with the corpus
|
||||
* k ~ 2.0 quadratic — every item compared against every other
|
||||
*
|
||||
* A pass whose k reaches CBM_SCALE_WARN_K logs `scaling.superlinear` at WARN in
|
||||
* SHIPPED builds with no flag — that is the point: the next accidental O(n^2)
|
||||
* should announce itself in an ordinary user's log rather than needing a
|
||||
* two-binary bench to find. The full checkpoint curve is emitted only under
|
||||
* CBM_PROFILE / --profile, so normal runs stay quiet.
|
||||
*
|
||||
* KNOW WHAT THIS DOES NOT CATCH. It measures growth WITHIN one run, so it sees
|
||||
* a pass whose per-item cost rises as the run proceeds. It does NOT see the
|
||||
* other quadratic shape — per-item work proportional to the WHOLE corpus, where
|
||||
* every item costs the same large amount and the in-run curve is therefore
|
||||
* flat. #1669 was exactly that shape: cross-file LSP rebuilt a registry from a
|
||||
* corpus-scaled def set for every file, measured k=1.86 across corpus SIZES
|
||||
* while this probe reported a harmless-looking 1.26 within a single run.
|
||||
*
|
||||
* For that shape the comparable number is `us_per_item`, which is emitted on
|
||||
* every line: it is a property of the corpus, not the machine, so the same
|
||||
* pass measured on a small and a large repo tells you immediately whether
|
||||
* per-item cost tracks repo size. Two runs, no rebuild, no A/B of binaries.
|
||||
*
|
||||
* Cost when nothing is wrong: four clock reads per pass, plus one relaxed
|
||||
* atomic load and an integer compare per tick. Tick from a site that is already
|
||||
* throttled (a progress-log point), never from the innermost loop. */
|
||||
|
||||
/* Checkpoints at total/8, total/4, total/2, total. */
|
||||
enum { CBM_SCALE_CHECKPOINTS = 4 };
|
||||
|
||||
/* k at or above this is reported. 1.35 sits clear of measured linear passes
|
||||
* (parallel_extract runs ~1.0-1.15 with normal scheduling jitter) while still
|
||||
* catching the mild-but-real 1.4-1.5 cases early. */
|
||||
#define CBM_SCALE_WARN_K 1.35
|
||||
|
||||
typedef struct {
|
||||
const char *phase;
|
||||
long total;
|
||||
struct timespec start;
|
||||
_Atomic int next_cp;
|
||||
long cp_us[CBM_SCALE_CHECKPOINTS];
|
||||
long cp_items[CBM_SCALE_CHECKPOINTS];
|
||||
} cbm_scale_probe_t;
|
||||
|
||||
/* Arm a probe for a pass that will process `total` items. Safe with total <= 0
|
||||
* (the probe then no-ops). */
|
||||
void cbm_scale_begin(cbm_scale_probe_t *probe, const char *phase, long total);
|
||||
|
||||
/* Record progress. Thread-safe and cheap: only the thread that crosses a
|
||||
* checkpoint boundary does any work, and only CBM_SCALE_CHECKPOINTS times. */
|
||||
void cbm_scale_tick(cbm_scale_probe_t *probe, long done);
|
||||
|
||||
/* Fit the exponent and report. Emits `scaling.superlinear` at WARN when
|
||||
* k >= CBM_SCALE_WARN_K (always), and `scaling` at INFO with the full curve
|
||||
* when profiling is active. */
|
||||
void cbm_scale_end(cbm_scale_probe_t *probe);
|
||||
|
||||
/* The fit itself, as a pure function of two (items, microseconds) points:
|
||||
* k such that T ~ n^k. Separated from the probe so the arithmetic is testable
|
||||
* without depending on wall-clock timing — a test that had to *produce* a
|
||||
* quadratic workload to check this would be measuring the scheduler.
|
||||
* Returns -1.0 for degenerate input (non-positive, or no growth in n). */
|
||||
double cbm_scale_fit_k(long first_n, long first_us, long last_n, long last_us);
|
||||
|
||||
/* Initialize profiling — reads CBM_PROFILE env var. Call once at startup. */
|
||||
void cbm_profile_init(void);
|
||||
|
||||
|
||||
@@ -28,6 +28,7 @@
|
||||
#include <ctype.h>
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <stdatomic.h>
|
||||
#include <string.h>
|
||||
|
||||
/* Confidence floor below which LSP-resolved calls are ignored and the
|
||||
@@ -856,6 +857,18 @@ static inline const CBMResolvedCall *cbm_pipeline_find_lsp_reference(
|
||||
* graph's file-shaped QN, accept one globally unique callable short name.
|
||||
*
|
||||
* Returns the matching node, or NULL if neither lookup hits. */
|
||||
/* Tail-match cost counters (#1669).
|
||||
*
|
||||
* The scan below is the one candidate loop in the resolve path with NO cap —
|
||||
* cbm_registry_resolve bails at REG_MAX_CANDIDATES=256, this does not. Its
|
||||
* candidate set is every node sharing a short name, which grows with the
|
||||
* corpus, so `candidates scanned` is the quantity that turns resolve from O(n)
|
||||
* into O(n^2). Counted always (two relaxed adds), surfaced by pass_parallel at
|
||||
* end of resolve: a candidates/lookup ratio that grows with corpus size IS the
|
||||
* regression, visible in one run. */
|
||||
extern _Atomic uint64_t g_lsp_tail_lookups;
|
||||
extern _Atomic uint64_t g_lsp_tail_candidates;
|
||||
|
||||
static inline const cbm_gbuf_node_t *cbm_pipeline_lsp_target_node_policy(
|
||||
const cbm_gbuf_t *gbuf, const char *project_name, const char *callee_qn, bool allow_tail_match,
|
||||
bool allow_unique_callable_fallback) {
|
||||
@@ -901,6 +914,8 @@ static inline const cbm_gbuf_node_t *cbm_pipeline_lsp_target_node_policy(
|
||||
if (cbm_gbuf_find_by_name(gbuf, short_name, &hits, &hit_count) != 0 || hit_count == 0) {
|
||||
return NULL;
|
||||
}
|
||||
atomic_fetch_add_explicit(&g_lsp_tail_lookups, 1, memory_order_relaxed);
|
||||
atomic_fetch_add_explicit(&g_lsp_tail_candidates, (uint64_t)hit_count, memory_order_relaxed);
|
||||
|
||||
const cbm_gbuf_node_t *match = NULL;
|
||||
const cbm_gbuf_node_t *unique_callable = NULL;
|
||||
|
||||
@@ -36,6 +36,7 @@
|
||||
#include <stdio.h>
|
||||
#include <stdint.h>
|
||||
#include <stdlib.h>
|
||||
#include <stdatomic.h>
|
||||
#include <string.h>
|
||||
|
||||
/* ── Constants ─────────────────────────────────────────────────── */
|
||||
@@ -1001,6 +1002,34 @@ void cbm_pxc_run_one_ts(CBMFileResult *r, const char *source, int source_len, co
|
||||
* `rust_shared_get` supplies the lazily-built shared Rust all-defs registry
|
||||
* (the parallel resolver owns its once-guard); NULL means "no shared rust
|
||||
* registry available" and rust NULL-filter files take the per-file build. */
|
||||
/* Per-file registry-build cost counters (#1669). Surfaced by pass_parallel at
|
||||
* end of resolve. */
|
||||
_Atomic uint64_t g_pxc_defs_registered = 0;
|
||||
_Atomic uint64_t g_pxc_build_files = 0;
|
||||
_Atomic uint64_t g_pxc_filter_files = 0;
|
||||
_Atomic uint64_t g_pxc_filter_failed = 0;
|
||||
|
||||
/* Overlay registrations count as per-file registry work too: the complexity
|
||||
* gate (test_complexity.c) sums ALL defs registered per file, whichever path
|
||||
* built them. Without this the shared-registry languages would report zero and
|
||||
* the linearity gate would pass vacuously. */
|
||||
void cbm_pxc_count_perfile_defs(uint64_t defs) {
|
||||
atomic_fetch_add_explicit(&g_pxc_defs_registered, defs, memory_order_relaxed);
|
||||
atomic_fetch_add_explicit(&g_pxc_build_files, 1, memory_order_relaxed);
|
||||
}
|
||||
|
||||
void cbm_pxc_filter_stats(uint64_t *defs_registered, uint64_t *build_files, uint64_t *filter_files,
|
||||
uint64_t *filter_failed) {
|
||||
if (defs_registered)
|
||||
*defs_registered = atomic_load_explicit(&g_pxc_defs_registered, memory_order_relaxed);
|
||||
if (build_files)
|
||||
*build_files = atomic_load_explicit(&g_pxc_build_files, memory_order_relaxed);
|
||||
if (filter_files)
|
||||
*filter_files = atomic_load_explicit(&g_pxc_filter_files, memory_order_relaxed);
|
||||
if (filter_failed)
|
||||
*filter_failed = atomic_load_explicit(&g_pxc_filter_failed, memory_order_relaxed);
|
||||
}
|
||||
|
||||
void cbm_pxc_dispatch_file(CBMLanguage lang, CBMFileResult *result, const char *source,
|
||||
int source_len, const char *rel, const char *def_module,
|
||||
const CBMCrossLspRegistries *cross_registries,
|
||||
@@ -1050,6 +1079,14 @@ void cbm_pxc_dispatch_file(CBMLanguage lang, CBMFileResult *result, const char *
|
||||
&result->resolved_calls);
|
||||
used_prebuilt = true;
|
||||
break;
|
||||
case CBM_LANG_JAVA:
|
||||
/* Own-module defs go into a per-file overlay; imports and stdlib
|
||||
* resolve through the shared base (#1669). */
|
||||
cbm_run_java_lsp_cross_with_registry(
|
||||
&result->arena, result, source, source_len, def_module, prebuilt, imp_keys,
|
||||
imp_vals, imp_count, result->cached_tree, &result->resolved_calls);
|
||||
used_prebuilt = true;
|
||||
break;
|
||||
case CBM_LANG_JAVASCRIPT:
|
||||
case CBM_LANG_TYPESCRIPT:
|
||||
case CBM_LANG_TSX: {
|
||||
@@ -1112,7 +1149,17 @@ void cbm_pxc_dispatch_file(CBMLanguage lang, CBMFileResult *result, const char *
|
||||
file_defs = filtered;
|
||||
file_def_count = filtered_count;
|
||||
}
|
||||
atomic_fetch_add_explicit(&g_pxc_filter_files, 1, memory_order_relaxed);
|
||||
if (!filter_succeeded) {
|
||||
atomic_fetch_add_explicit(&g_pxc_filter_failed, 1, memory_order_relaxed);
|
||||
}
|
||||
}
|
||||
/* Per-file registry build cost is driven by THIS number. If it tracks the
|
||||
* corpus instead of the file's own module + imports, cross-file LSP is
|
||||
* O(files x corpus_defs) — see #1669. */
|
||||
atomic_fetch_add_explicit(&g_pxc_defs_registered, (uint64_t)file_def_count,
|
||||
memory_order_relaxed);
|
||||
atomic_fetch_add_explicit(&g_pxc_build_files, 1, memory_order_relaxed);
|
||||
if (lang == CBM_LANG_RUST) {
|
||||
CBMTypeRegistry *shared = rust_shared_get ? rust_shared_get(rust_shared_ctx) : NULL;
|
||||
if (shared) {
|
||||
|
||||
@@ -35,6 +35,7 @@
|
||||
#include "lsp/c_lsp.h" /* cbm_c_build_cross_registry / cbm_run_c_lsp_cross_with_registry */
|
||||
#include "lsp/cs_lsp.h" /* cbm_cs_build_cross_registry / cbm_run_cs_lsp_cross_with_registry */
|
||||
#include "lsp/ts_lsp.h" /* cbm_ts_build_cross_registry / cbm_run_ts_lsp_cross_with_registry */
|
||||
#include "lsp/java_lsp.h" /* cbm_java_build_cross_registry / cbm_run_java_lsp_cross_with_registry */
|
||||
#include "lsp/rust_lsp.h" /* cbm_rust_build_cross_registry / cbm_run_rust_lsp_cross_with_registry */
|
||||
#include "pipeline/pipeline_internal.h"
|
||||
#include <stdbool.h>
|
||||
@@ -125,12 +126,21 @@ typedef struct {
|
||||
CBMTypeRegistry *ts; /* CBM_LANG_JAVASCRIPT, TYPESCRIPT, TSX */
|
||||
CBMTypeRegistry *php; /* CBM_LANG_PHP */
|
||||
CBMTypeRegistry *cs; /* CBM_LANG_CSHARP */
|
||||
CBMTypeRegistry *java; /* CBM_LANG_JAVA (JVM def universe incl. Kotlin defs) */
|
||||
/* CBM_LANG_RUST: intentionally absent — the shared rust registry is built
|
||||
* LAZILY inside cbm_parallel_resolve (first NULL-filter rust file), not eagerly. */
|
||||
} CBMCrossLspRegistries;
|
||||
|
||||
/* Return the appropriate pre-built registry for a language, or NULL
|
||||
* if none was built (or language has no cross-LSP entrypoint). */
|
||||
/* Per-file registry-build cost (#1669): how many defs the per-file cross-LSP
|
||||
* path actually registered, and how often the module filter failed. */
|
||||
/* Count defs an overlay registered for one file (complexity-gate telemetry). */
|
||||
void cbm_pxc_count_perfile_defs(uint64_t defs);
|
||||
|
||||
void cbm_pxc_filter_stats(uint64_t *defs_registered, uint64_t *build_files, uint64_t *filter_files,
|
||||
uint64_t *filter_failed);
|
||||
|
||||
static inline CBMTypeRegistry *cbm_pxc_registry_for_lang(const CBMCrossLspRegistries *r,
|
||||
CBMLanguage lang) {
|
||||
if (!r)
|
||||
@@ -152,6 +162,8 @@ static inline CBMTypeRegistry *cbm_pxc_registry_for_lang(const CBMCrossLspRegist
|
||||
return r->php;
|
||||
case CBM_LANG_CSHARP:
|
||||
return r->cs;
|
||||
case CBM_LANG_JAVA:
|
||||
return r->java;
|
||||
default:
|
||||
return NULL; /* incl. CBM_LANG_RUST — its shared registry is built lazily */
|
||||
}
|
||||
|
||||
@@ -106,6 +106,10 @@ enum { PP_CSHARP_M_PREFIX_LEN = 2 };
|
||||
* memory (the resident floor, not in-flight transients, holds the budget). */
|
||||
static _Atomic long g_bp_nap_cycles = 0;
|
||||
static _Atomic uint64_t g_lsp_linear_fallback_rows = 0;
|
||||
/* Defined here, declared in lsp_resolve.h — the uncapped tail-match scan's
|
||||
* cost, surfaced at end of resolve (#1669). */
|
||||
_Atomic uint64_t g_lsp_tail_lookups = 0;
|
||||
_Atomic uint64_t g_lsp_tail_candidates = 0;
|
||||
|
||||
long cbm_pp_bp_nap_cycles(void) {
|
||||
return atomic_load_explicit(&g_bp_nap_cycles, memory_order_relaxed);
|
||||
@@ -121,6 +125,8 @@ uint64_t cbm_pp_lsp_linear_fallback_rows(void) {
|
||||
|
||||
void cbm_pp_lsp_linear_fallback_rows_reset(void) {
|
||||
atomic_store_explicit(&g_lsp_linear_fallback_rows, 0, memory_order_relaxed);
|
||||
atomic_store_explicit(&g_lsp_tail_lookups, 0, memory_order_relaxed);
|
||||
atomic_store_explicit(&g_lsp_tail_candidates, 0, memory_order_relaxed);
|
||||
}
|
||||
|
||||
/* Parse a positive MB-valued retention env knob (CBM_RETAIN_*_MB) into bytes.
|
||||
@@ -663,6 +669,9 @@ typedef struct {
|
||||
|
||||
const CBMMacroTable *macro_table; /* ObjectScript $$$macros (NULL if none) */
|
||||
const CBMReturnTypeTable *return_type_table; /* ObjectScript return types (NULL if none) */
|
||||
|
||||
/* Superlinearity probe — see profile.h. Ticked on each file claim below. */
|
||||
cbm_scale_probe_t scale;
|
||||
} extract_ctx_t;
|
||||
|
||||
/* Cap on the number of index.file_oversized WARN lines (the full list still goes
|
||||
@@ -728,6 +737,7 @@ static void extract_worker(int worker_id, void *ctx_ptr) {
|
||||
if (sort_pos >= ec->file_count) {
|
||||
break;
|
||||
}
|
||||
cbm_scale_tick(&ec->scale, sort_pos);
|
||||
if (atomic_load_explicit(ec->cancelled, memory_order_relaxed)) {
|
||||
break;
|
||||
}
|
||||
@@ -1120,7 +1130,9 @@ int cbm_parallel_extract_ex(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *file
|
||||
/* Sub-phase: Dispatch workers (parse + extract per file, PARALLEL) */
|
||||
CBM_PROF_START(t_dispatch);
|
||||
cbm_parallel_for_opts_t parallel_opts = {.max_workers = worker_count, .force_pthreads = false};
|
||||
cbm_scale_begin(&ec.scale, "parallel_extract", (long)file_count);
|
||||
cbm_parallel_for(worker_count, extract_worker, &ec, parallel_opts);
|
||||
cbm_scale_end(&ec.scale);
|
||||
CBM_PROF_END_N("parallel_extract", "3_dispatch_workers_parallel", t_dispatch, file_count);
|
||||
|
||||
/* Sub-phase: Merge all local gbufs into main gbuf (SEQUENTIAL, gbuf not thread-safe) */
|
||||
@@ -1432,6 +1444,11 @@ typedef struct {
|
||||
_Atomic uint64_t time_ns_rc_target; /* gbuf_find_by_qn for target */
|
||||
_Atomic uint64_t time_ns_rc_emit; /* emit_service_edge */
|
||||
_Atomic uint64_t time_ns_rc_source; /* find_source_node */
|
||||
|
||||
/* Superlinearity probe — see profile.h. This is the pass that made it
|
||||
* necessary (#1669: 87% of a Java index), so it is the one that must never
|
||||
* again grow superlinear without saying so. */
|
||||
cbm_scale_probe_t scale;
|
||||
} resolve_ctx_t;
|
||||
|
||||
/* Minimum buffer space needed per arg JSON object */
|
||||
@@ -2934,6 +2951,7 @@ static void resolve_worker(int worker_id, void *ctx_ptr) {
|
||||
if (file_idx >= rc->file_count) {
|
||||
break;
|
||||
}
|
||||
cbm_scale_tick(&rc->scale, file_idx);
|
||||
if (atomic_load_explicit(rc->cancelled, memory_order_relaxed)) {
|
||||
break;
|
||||
}
|
||||
@@ -3212,7 +3230,9 @@ int cbm_parallel_resolve(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files,
|
||||
/* Sub-phase: Dispatch resolve workers (per-file call/usage resolution, PARALLEL) */
|
||||
CBM_PROF_START(t_resolve_dispatch);
|
||||
cbm_parallel_for_opts_t opts = {.max_workers = worker_count, .force_pthreads = false};
|
||||
cbm_scale_begin(&rc.scale, "parallel_resolve", (long)file_count);
|
||||
cbm_parallel_for(worker_count, resolve_worker, &rc, opts);
|
||||
cbm_scale_end(&rc.scale);
|
||||
CBM_PROF_END_N("parallel_resolve", "1_dispatch_workers_parallel", t_resolve_dispatch,
|
||||
file_count);
|
||||
/* Workers joined: the shared Rust registry (if built) is no longer read.
|
||||
@@ -3267,6 +3287,56 @@ int cbm_parallel_resolve(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files,
|
||||
itoa_log(atomic_load_explicit(&rc.lsp_cross_skipped_no_source, memory_order_relaxed)),
|
||||
"defs_total", itoa_log(def_count));
|
||||
|
||||
/* Cross-LSP cost, NORMALISED (#1669). Wall time alone cannot distinguish
|
||||
* "big repo" from "superlinear pass"; us_per_file can. For a pass that is
|
||||
* linear in file count this number is roughly CONSTANT across repo sizes.
|
||||
* When cross-file LSP rebuilds its registry from a corpus-scaled def set,
|
||||
* per-file cost tracks defs_total instead — measured 35ms/file at 5.8k
|
||||
* files and 209ms/file at 46k on the same tree, which is the O(n^2) that
|
||||
* cost a 6x java regression and took an 11-corpus two-binary A/B to find.
|
||||
* Emitted next to defs_total so one grep on two differently sized repos
|
||||
* answers it. */
|
||||
int cross_files = atomic_load_explicit(&rc.lsp_cross_processed, memory_order_relaxed);
|
||||
uint64_t cross_us = atomic_load_explicit(&rc.time_ns_cross_lsp, memory_order_relaxed) / 1000ULL;
|
||||
if (cross_files > 0) {
|
||||
char cf_buf[CBM_SZ_32];
|
||||
char nf_buf[CBM_SZ_32];
|
||||
char cu_buf[CBM_SZ_32];
|
||||
char pk_buf[CBM_SZ_32];
|
||||
snprintf(cf_buf, sizeof(cf_buf), "%llu",
|
||||
(unsigned long long)(cross_us / (uint64_t)cross_files));
|
||||
snprintf(nf_buf, sizeof(nf_buf), "%d", cross_files);
|
||||
snprintf(cu_buf, sizeof(cu_buf), "%llu", (unsigned long long)(cross_us / 1000ULL));
|
||||
snprintf(pk_buf, sizeof(pk_buf), "%llu",
|
||||
(unsigned long long)(def_count > 0
|
||||
? (cross_us * 1000ULL) /
|
||||
((uint64_t)cross_files * (uint64_t)def_count)
|
||||
: 0ULL));
|
||||
cbm_log_info("parallel.resolve.cross_lsp_cost", "cross_lsp_ms", cu_buf, "files", nf_buf,
|
||||
"us_per_file", cf_buf, "defs_total", itoa_log(def_count),
|
||||
"us_per_file_per_kdef", pk_buf);
|
||||
|
||||
/* What the per-file registry build actually cost. defs_per_file is the
|
||||
* lever: if it tracks defs_total rather than the file's own module plus
|
||||
* imports, the module filter is not containing the work and cross-file
|
||||
* LSP is O(files x corpus_defs). */
|
||||
uint64_t reg_defs = 0;
|
||||
uint64_t reg_files = 0;
|
||||
uint64_t flt_files = 0;
|
||||
uint64_t flt_failed = 0;
|
||||
cbm_pxc_filter_stats(®_defs, ®_files, &flt_files, &flt_failed);
|
||||
if (reg_files > 0) {
|
||||
char rd_buf[CBM_SZ_32];
|
||||
char ff_buf[CBM_SZ_32];
|
||||
char fp_buf[CBM_SZ_32];
|
||||
snprintf(rd_buf, sizeof(rd_buf), "%llu", (unsigned long long)(reg_defs / reg_files));
|
||||
snprintf(ff_buf, sizeof(ff_buf), "%llu", (unsigned long long)flt_failed);
|
||||
snprintf(fp_buf, sizeof(fp_buf), "%llu", (unsigned long long)flt_files);
|
||||
cbm_log_info("parallel.resolve.perfile_registry", "defs_per_file", rd_buf, "defs_total",
|
||||
itoa_log(def_count), "filtered_files", fp_buf, "filter_failed", ff_buf);
|
||||
}
|
||||
}
|
||||
|
||||
cbm_log_info("parallel.resolve.done", "calls", itoa_log(total_calls), "usages",
|
||||
itoa_log(total_usages), "semantic", itoa_log(total_semantic + go_impl),
|
||||
"lsp_overrides", itoa_log(total_lsp_overrides));
|
||||
@@ -3344,5 +3414,27 @@ int cbm_parallel_resolve(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files,
|
||||
"resolve", rsv_buf);
|
||||
cbm_log_info("parallel.resolve.calls_breakdown2", "field_hint", hnt_buf, "find_target", tgt_buf,
|
||||
"emit_edge", emt_buf);
|
||||
|
||||
/* Candidate-scan cost (#1669). `per_lookup` is the diagnostic that matters:
|
||||
* it is a property of the CORPUS, not of the machine, so it is comparable
|
||||
* across runs and versions. If it grows with repo size, the tail-match scan
|
||||
* is turning resolve superlinear — which is precisely what took an
|
||||
* 11-corpus two-binary A/B to establish the first time. `fallback_rows`
|
||||
* was already counted but, until now, readable only from a test. */
|
||||
uint64_t tail_lookups = atomic_load_explicit(&g_lsp_tail_lookups, memory_order_relaxed);
|
||||
uint64_t tail_cands = atomic_load_explicit(&g_lsp_tail_candidates, memory_order_relaxed);
|
||||
char tl_buf[CBM_SZ_32];
|
||||
char tc_buf[CBM_SZ_32];
|
||||
char tp_buf[CBM_SZ_32];
|
||||
char fb_buf[CBM_SZ_32];
|
||||
snprintf(tl_buf, sizeof(tl_buf), "%llu", (unsigned long long)tail_lookups);
|
||||
snprintf(tc_buf, sizeof(tc_buf), "%llu", (unsigned long long)tail_cands);
|
||||
snprintf(tp_buf, sizeof(tp_buf), "%llu",
|
||||
(unsigned long long)(tail_lookups ? tail_cands / tail_lookups : 0ULL));
|
||||
snprintf(fb_buf, sizeof(fb_buf), "%llu",
|
||||
(unsigned long long)atomic_load_explicit(&g_lsp_linear_fallback_rows,
|
||||
memory_order_relaxed));
|
||||
cbm_log_info("parallel.resolve.scan_cost", "tail_lookups", tl_buf, "tail_candidates", tc_buf,
|
||||
"per_lookup", tp_buf, "fallback_rows", fb_buf);
|
||||
return 0;
|
||||
}
|
||||
|
||||
+28
-1
@@ -1254,17 +1254,44 @@ static int run_parallel_pipeline(cbm_pipeline_t *p, cbm_pipeline_ctx_t *ctx,
|
||||
* Built ONCE here; shared READ-ONLY across all files of that language
|
||||
* during resolve. Per-file work is then: parse + AST walk + O(1) lookups
|
||||
* — no registry build, no Phase 1b mutations. Languages added so far:
|
||||
* Go, Python. Others (C/C++, TS/JS, PHP, C#) fall back to per-file. */
|
||||
* Go, Python, C/C++, C#, TS/JS, Java. Others (Kotlin, PHP) fall back to per-file. */
|
||||
CBMArena cross_lsp_arena;
|
||||
cbm_arena_init(&cross_lsp_arena);
|
||||
CBMCrossLspRegistries cross_registries = {0};
|
||||
if (all_defs) {
|
||||
/* Per-builder split of lsp_cross_prepare — attributes a slow prepare to
|
||||
* ONE language instead of re-diagnosing the whole pass (the cs builder
|
||||
* hid ~140 s behind the pass total, #1669 follow-up). */
|
||||
struct timespec t_b;
|
||||
long b_ms[6];
|
||||
cbm_clock_gettime(CLOCK_MONOTONIC, &t_b);
|
||||
cross_registries.go = cbm_go_build_cross_registry(&cross_lsp_arena, all_defs, def_count);
|
||||
b_ms[0] = (long)elapsed_ms(t_b);
|
||||
cbm_clock_gettime(CLOCK_MONOTONIC, &t_b);
|
||||
cross_registries.python =
|
||||
cbm_py_build_cross_registry(&cross_lsp_arena, all_defs, def_count);
|
||||
b_ms[1] = (long)elapsed_ms(t_b);
|
||||
cbm_clock_gettime(CLOCK_MONOTONIC, &t_b);
|
||||
cross_registries.c = cbm_c_build_cross_registry(&cross_lsp_arena, all_defs, def_count);
|
||||
b_ms[2] = (long)elapsed_ms(t_b);
|
||||
cbm_clock_gettime(CLOCK_MONOTONIC, &t_b);
|
||||
cross_registries.cs = cbm_cs_build_cross_registry(&cross_lsp_arena, all_defs, def_count);
|
||||
b_ms[3] = (long)elapsed_ms(t_b);
|
||||
cbm_clock_gettime(CLOCK_MONOTONIC, &t_b);
|
||||
cross_registries.ts = cbm_ts_build_cross_registry(&cross_lsp_arena, all_defs, def_count);
|
||||
b_ms[4] = (long)elapsed_ms(t_b);
|
||||
cbm_clock_gettime(CLOCK_MONOTONIC, &t_b);
|
||||
cross_registries.java =
|
||||
cbm_java_build_cross_registry(&cross_lsp_arena, all_defs, def_count);
|
||||
b_ms[5] = (long)elapsed_ms(t_b);
|
||||
char b_buf[6][CBM_SZ_16];
|
||||
const char *b_name[6] = {"go", "python", "c", "cs", "ts", "java"};
|
||||
for (int bi = 0; bi < 6; bi++) {
|
||||
snprintf(b_buf[bi], sizeof(b_buf[bi]), "%ld", b_ms[bi]);
|
||||
}
|
||||
cbm_log_info("lsp_cross_prepare.builders", b_name[0], b_buf[0], b_name[1], b_buf[1],
|
||||
b_name[2], b_buf[2], b_name[3], b_buf[3], b_name[4], b_buf[4], b_name[5],
|
||||
b_buf[5]);
|
||||
/* Rust: NOT built here. The shared all_defs registry is built LAZILY on the
|
||||
* first NULL-filter rust file (the amplifier files) inside cbm_parallel_resolve
|
||||
* — repos whose rust files all filter to subsets never pay the build/RSS. */
|
||||
|
||||
@@ -1334,6 +1334,8 @@ static int run_extract_resolve(cbm_pipeline_ctx_t *ctx, cbm_file_info_t *changed
|
||||
cross_registries.c = cbm_c_build_cross_registry(xa, all_defs, all_def_count);
|
||||
cross_registries.cs = cbm_cs_build_cross_registry(xa, all_defs, all_def_count);
|
||||
cross_registries.ts = cbm_ts_build_cross_registry(xa, all_defs, all_def_count);
|
||||
cross_registries.java =
|
||||
cbm_java_build_cross_registry(xa, all_defs, all_def_count);
|
||||
registries_arg = &cross_registries;
|
||||
}
|
||||
} else {
|
||||
|
||||
+95
-2
@@ -2668,6 +2668,20 @@ static void cov_failure_fingerprint(const cbm_coverage_row_t *rows, int count,
|
||||
out[CBM_SHA256_HEX_LEN] = 0;
|
||||
}
|
||||
|
||||
static void cov_dir_ids_free_entry(const char *key, void *val, void *ud) {
|
||||
(void)ud;
|
||||
free((char *)key);
|
||||
free(val);
|
||||
}
|
||||
|
||||
static void cov_dir_ids_free(CBMHashTable *ht) {
|
||||
if (!ht) {
|
||||
return;
|
||||
}
|
||||
cbm_ht_foreach(ht, cov_dir_ids_free_entry, NULL);
|
||||
cbm_ht_free(ht);
|
||||
}
|
||||
|
||||
static int cov_rebuild_shadow_graph(cbm_store_t *s, const char *project) {
|
||||
char covproj[CBM_SZ_512];
|
||||
cbm_store_coverage_shadow_project(covproj, sizeof(covproj), project);
|
||||
@@ -2765,6 +2779,12 @@ static int cov_rebuild_shadow_graph(cbm_store_t *s, const char *project) {
|
||||
return CBM_STORE_ERR;
|
||||
}
|
||||
|
||||
/* Directory nodes repeat massively across failure rows (13k baseline files
|
||||
* under one tests/ subtree meant ~80k redundant upsert/edge round-trips —
|
||||
* 9.1 s of a 9.2 s coverage_replace on the TypeScript corpus). Dedup them
|
||||
* in-memory for the rebuild: first sight of a directory creates its node
|
||||
* and containment edge; every later file under it is a hash hit. */
|
||||
CBMHashTable *dir_ids = cbm_ht_create(CBM_SZ_256);
|
||||
for (int i = 0; i < count; i++) {
|
||||
const char *rel = rows[i].rel_path;
|
||||
if (!rel || !rel[0]) {
|
||||
@@ -2787,6 +2807,12 @@ static int cov_rebuild_shadow_graph(cbm_store_t *s, const char *project) {
|
||||
/* Truncate at this slash → pathbuf is the directory prefix; the
|
||||
* upsert binds copies, so restore the slash right after. */
|
||||
*p = '\0';
|
||||
int64_t *cached = dir_ids ? (int64_t *)cbm_ht_get(dir_ids, pathbuf) : NULL;
|
||||
if (cached) {
|
||||
parent = *cached;
|
||||
*p = '/';
|
||||
continue;
|
||||
}
|
||||
const char *seg = strrchr(pathbuf, '/');
|
||||
cbm_node_t folder = {.project = covproj,
|
||||
.label = "Folder",
|
||||
@@ -2795,8 +2821,9 @@ static int cov_rebuild_shadow_graph(cbm_store_t *s, const char *project) {
|
||||
.file_path = pathbuf,
|
||||
.properties_json = "{}"};
|
||||
int64_t fid = cbm_store_upsert_node(s, &folder);
|
||||
*p = '/';
|
||||
if (fid <= 0) {
|
||||
*p = '/';
|
||||
cov_dir_ids_free(dir_ids);
|
||||
cbm_store_free_coverage(rows, count);
|
||||
return CBM_STORE_ERR;
|
||||
}
|
||||
@@ -2806,9 +2833,24 @@ static int cov_rebuild_shadow_graph(cbm_store_t *s, const char *project) {
|
||||
.type = "CONTAINS_FOLDER",
|
||||
.properties_json = "{}"};
|
||||
if (cbm_store_insert_edge(s, &e) <= 0) {
|
||||
*p = '/';
|
||||
cov_dir_ids_free(dir_ids);
|
||||
cbm_store_free_coverage(rows, count);
|
||||
return CBM_STORE_ERR;
|
||||
}
|
||||
if (dir_ids) {
|
||||
int64_t *idv = (int64_t *)malloc(sizeof(*idv));
|
||||
if (idv) {
|
||||
*idv = fid;
|
||||
char *kdup = strdup(pathbuf);
|
||||
if (kdup) {
|
||||
cbm_ht_set(dir_ids, kdup, idv);
|
||||
} else {
|
||||
free(idv);
|
||||
}
|
||||
}
|
||||
}
|
||||
*p = '/';
|
||||
parent = fid;
|
||||
}
|
||||
|
||||
@@ -2826,6 +2868,7 @@ static int cov_rebuild_shadow_graph(cbm_store_t *s, const char *project) {
|
||||
.properties_json = props};
|
||||
int64_t file_id = cbm_store_upsert_node(s, &file);
|
||||
if (file_id <= 0) {
|
||||
cov_dir_ids_free(dir_ids);
|
||||
cbm_store_free_coverage(rows, count);
|
||||
return CBM_STORE_ERR;
|
||||
}
|
||||
@@ -2835,10 +2878,12 @@ static int cov_rebuild_shadow_graph(cbm_store_t *s, const char *project) {
|
||||
.type = "CONTAINS_FILE",
|
||||
.properties_json = "{}"};
|
||||
if (cbm_store_insert_edge(s, &e) <= 0) {
|
||||
cov_dir_ids_free(dir_ids);
|
||||
cbm_store_free_coverage(rows, count);
|
||||
return CBM_STORE_ERR;
|
||||
}
|
||||
}
|
||||
cov_dir_ids_free(dir_ids);
|
||||
cbm_store_free_coverage(rows, count);
|
||||
{
|
||||
sqlite3_stmt *set = NULL;
|
||||
@@ -2862,6 +2907,15 @@ int cbm_store_coverage_replace_ex(cbm_store_t *s, const char *project,
|
||||
if (exec_sql(s, "BEGIN;") != CBM_STORE_OK) {
|
||||
return CBM_STORE_ERR;
|
||||
}
|
||||
/* Sub-block timings (publish.timing style): coverage_replace measured 9 s
|
||||
* on the TypeScript corpus and the caller-level block could not say WHY —
|
||||
* delete, 13k row inserts with large error_ranges payloads, the NOT-IN
|
||||
* prune, meta, and COMMIT are very different suspects. */
|
||||
struct timespec cov_t0;
|
||||
struct timespec cov_t1;
|
||||
long cov_ms[5] = {0, 0, 0, 0, 0};
|
||||
long long cov_detail_bytes = 0;
|
||||
cbm_clock_gettime(CLOCK_MONOTONIC, &cov_t0);
|
||||
sqlite3_stmt *del = NULL;
|
||||
if (sqlite3_prepare_v2(s->db, "DELETE FROM index_coverage WHERE project = ?1;", CBM_NOT_FOUND,
|
||||
&del, NULL) != SQLITE_OK) {
|
||||
@@ -2877,6 +2931,10 @@ int cbm_store_coverage_replace_ex(cbm_store_t *s, const char *project,
|
||||
(void)exec_sql(s, "ROLLBACK;");
|
||||
return CBM_STORE_ERR;
|
||||
}
|
||||
cbm_clock_gettime(CLOCK_MONOTONIC, &cov_t1);
|
||||
cov_ms[0] =
|
||||
(cov_t1.tv_sec - cov_t0.tv_sec) * 1000 + (cov_t1.tv_nsec - cov_t0.tv_nsec) / 1000000;
|
||||
cov_t0 = cov_t1;
|
||||
sqlite3_stmt *ins = NULL;
|
||||
if (sqlite3_prepare_v2(
|
||||
s->db,
|
||||
@@ -2894,6 +2952,7 @@ int cbm_store_coverage_replace_ex(cbm_store_t *s, const char *project,
|
||||
bind_text(ins, SKIP_ONE, project);
|
||||
bind_text(ins, ST_COL_2, rows[i].rel_path);
|
||||
bind_text(ins, ST_COL_3, rows[i].kind);
|
||||
cov_detail_bytes += rows[i].detail ? (long long)strlen(rows[i].detail) : 0;
|
||||
bind_text(ins, CBM_SZ_4, rows[i].detail ? rows[i].detail : "");
|
||||
if (sqlite3_step(ins) != SQLITE_DONE) {
|
||||
store_set_error_sqlite(s, "coverage insert");
|
||||
@@ -2904,6 +2963,10 @@ int cbm_store_coverage_replace_ex(cbm_store_t *s, const char *project,
|
||||
sqlite3_reset(ins);
|
||||
}
|
||||
sqlite3_finalize(ins);
|
||||
cbm_clock_gettime(CLOCK_MONOTONIC, &cov_t1);
|
||||
cov_ms[1] =
|
||||
(cov_t1.tv_sec - cov_t0.tv_sec) * 1000 + (cov_t1.tv_nsec - cov_t0.tv_nsec) / 1000000;
|
||||
cov_t0 = cov_t1;
|
||||
/* Prune FAILURE rows for files no longer known to the index (deleted from
|
||||
* the repo): file_hashes is the authoritative live-file set after
|
||||
* persist. By-design not_indexed_* rows are exempt — deliberately
|
||||
@@ -2927,6 +2990,10 @@ int cbm_store_coverage_replace_ex(cbm_store_t *s, const char *project,
|
||||
(void)exec_sql(s, "ROLLBACK;");
|
||||
return CBM_STORE_ERR;
|
||||
}
|
||||
cbm_clock_gettime(CLOCK_MONOTONIC, &cov_t1);
|
||||
cov_ms[2] =
|
||||
(cov_t1.tv_sec - cov_t0.tv_sec) * 1000 + (cov_t1.tv_nsec - cov_t0.tv_nsec) / 1000000;
|
||||
cov_t0 = cov_t1;
|
||||
|
||||
if (meta) {
|
||||
char recorded_at[CBM_SZ_64];
|
||||
@@ -3002,7 +3069,33 @@ int cbm_store_coverage_replace_ex(cbm_store_t *s, const char *project,
|
||||
(void)exec_sql(s, "ROLLBACK;");
|
||||
return CBM_STORE_ERR;
|
||||
}
|
||||
return exec_sql(s, "COMMIT;");
|
||||
cbm_clock_gettime(CLOCK_MONOTONIC, &cov_t1);
|
||||
cov_ms[3] =
|
||||
(cov_t1.tv_sec - cov_t0.tv_sec) * 1000 + (cov_t1.tv_nsec - cov_t0.tv_nsec) / 1000000;
|
||||
cov_t0 = cov_t1;
|
||||
int commit_rc = exec_sql(s, "COMMIT;");
|
||||
cbm_clock_gettime(CLOCK_MONOTONIC, &cov_t1);
|
||||
cov_ms[4] =
|
||||
(cov_t1.tv_sec - cov_t0.tv_sec) * 1000 + (cov_t1.tv_nsec - cov_t0.tv_nsec) / 1000000;
|
||||
{
|
||||
char b0[ST_BUF_64];
|
||||
char b1[ST_BUF_64];
|
||||
char b2[ST_BUF_64];
|
||||
char b3[ST_BUF_64];
|
||||
char b4[ST_BUF_64];
|
||||
char bn[ST_BUF_64];
|
||||
char bb[ST_BUF_64];
|
||||
snprintf(b0, sizeof(b0), "%ld", cov_ms[0]);
|
||||
snprintf(b1, sizeof(b1), "%ld", cov_ms[1]);
|
||||
snprintf(b2, sizeof(b2), "%ld", cov_ms[2]);
|
||||
snprintf(b3, sizeof(b3), "%ld", cov_ms[3]);
|
||||
snprintf(b4, sizeof(b4), "%ld", cov_ms[4]);
|
||||
snprintf(bn, sizeof(bn), "%d", count);
|
||||
snprintf(bb, sizeof(bb), "%lld", cov_detail_bytes);
|
||||
cbm_log_info("publish.timing.coverage", "del", b0, "rows", b1, "prune", b2, "meta", b3,
|
||||
"commit", b4, "row_count", bn, "detail_bytes", bb);
|
||||
}
|
||||
return commit_rc;
|
||||
}
|
||||
|
||||
int cbm_store_coverage_replace(cbm_store_t *s, const char *project, const cbm_coverage_row_t *rows,
|
||||
|
||||
@@ -0,0 +1,725 @@
|
||||
/*
|
||||
* test_complexity.c — Complexity guard: superlinearity detection at tiny input
|
||||
* sizes, gated on DETERMINISTIC work counters — never time.
|
||||
*
|
||||
* Why this suite exists: the v0.9.0→v0.10.x indexing regression was a
|
||||
* files×corpus_defs coupling in cross-file LSP (#1669). Finding it took an
|
||||
* 11-corpus A/B across two release binaries, because nothing in CI could see
|
||||
* an O(n^2) forming. This suite makes that class of bug fail a unit test.
|
||||
*
|
||||
* Method — replicated independent modules:
|
||||
* Build a synthetic corpus of k INDEPENDENT module copies per language
|
||||
* (module i never references module j). Then every extensive quantity —
|
||||
* nodes, edges, and Σ per-file work — MUST grow linearly in k. Run the full
|
||||
* in-process pipeline at k and 2k and assert counter RATIOS:
|
||||
*
|
||||
* linear pipeline ratio ≈ 2 (gate: within [lo, hi])
|
||||
* files×corpus bug ratio ≈ 4 (per-file work itself grows with k)
|
||||
*
|
||||
* Ratios expose the exponent at TINY sizes (dozens of files, seconds of
|
||||
* runtime): a counter that counts the quadratic term directly doubles its
|
||||
* growth per doubling regardless of absolute scale. No large corpora needed.
|
||||
*
|
||||
* Determinism doctrine (O9): a verdict must be a pure function of
|
||||
* (code, input). Work counters are sums over per-file work and do not depend
|
||||
* on scheduling; wall time does. Therefore ONLY counters gate. Throughput
|
||||
* (nodes/s, edges/s) is measured and written to a LOCAL report under private/
|
||||
* as information for trend comparison — it never gates, and the report step is
|
||||
* skipped entirely under CBM_SKIP_PERF (starved fidelity legs would record
|
||||
* meaningless rates).
|
||||
*
|
||||
* Dynamic coverage: languages come from two providers, iterated over the full
|
||||
* CBM_LANG_COUNT enum —
|
||||
* 1. embedded templates below (the LSP-hybrid languages, where cross-file
|
||||
* machinery — and therefore files×corpus coupling risk — lives);
|
||||
* 2. auto-discovered fixture dirs tests/fixtures/complexity/<lang-name>/
|
||||
* (drop files there when adding a language; no test edit needed).
|
||||
* Languages with neither provider are recorded in the report as skipped with
|
||||
* the reason "no complexity template": their extractors are per-file by
|
||||
* construction (grammar-only, no cross-file resolution), so the coupling this
|
||||
* suite hunts cannot arise from them; the shared passes they feed (registry,
|
||||
* similarity, semantic) are exercised by the template corpus.
|
||||
*/
|
||||
#include "test_framework.h"
|
||||
#include "test_helpers.h"
|
||||
|
||||
#include "../src/foundation/compat.h"
|
||||
#include "../src/foundation/compat_fs.h"
|
||||
#include "../src/foundation/log.h"
|
||||
#include "../src/foundation/profile.h"
|
||||
#include "cbm.h"
|
||||
#include "discover/discover.h"
|
||||
#include "pipeline/pass_lsp_cross.h"
|
||||
#include "pipeline/pipeline.h"
|
||||
#include "pipeline/pipeline_internal.h"
|
||||
#include "store/store.h"
|
||||
|
||||
#include <stdatomic.h>
|
||||
#include <stdint.h>
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <time.h>
|
||||
|
||||
/* Tail-match scan counters (defined in pass_parallel.c, declared in
|
||||
* lsp_resolve.h — re-declared here to avoid pulling that header's statics). */
|
||||
extern _Atomic uint64_t g_lsp_tail_lookups;
|
||||
extern _Atomic uint64_t g_lsp_tail_candidates;
|
||||
|
||||
/* ── Corpus scale ──────────────────────────────────────────────────────
|
||||
* Small on purpose: the gate reads exponents from ratios, not magnitudes.
|
||||
* K_BASE modules vs 2*K_BASE modules, CX_FILES_PER_MOD files each, per
|
||||
* provider language. Total runtime target for the whole suite: seconds. */
|
||||
enum {
|
||||
CX_K_BASE = 3,
|
||||
CX_FILES_PER_MOD = 5,
|
||||
/* Gate only when the base run produced enough work for the ratio to be
|
||||
* meaningful — a near-zero denominator would make the gate noise. The
|
||||
* false-guard audit rule applies: a separate assertion below proves the
|
||||
* counter is genuinely nonzero so this gate can never pass vacuously. */
|
||||
CX_MIN_BASE_WORK = 40,
|
||||
/* The pipeline resolves sequentially below MIN_FILES_FOR_PARALLEL (50
|
||||
* files, pipeline.c) and that path builds no shared registries — so both
|
||||
* legs of every corpus pair must exceed it or the gates measure the wrong
|
||||
* code path. The sequential path's own per-file registry work is bounded
|
||||
* by the 50-file ceiling and is deliberately out of scope here. */
|
||||
CX_BIGPKG_FILES_PER_K = 20,
|
||||
};
|
||||
|
||||
/* Linear growth bounds for a 2x input doubling. Fixed structural overhead
|
||||
* (Project/root nodes) pulls node ratios slightly under 2; template boundary
|
||||
* effects push edge ratios slightly around 2. A files×corpus coupling lands
|
||||
* at ~4 — far outside. */
|
||||
#define CX_RATIO_LO 1.45
|
||||
#define CX_RATIO_HI 2.75
|
||||
|
||||
/* ── Language templates ────────────────────────────────────────────────
|
||||
* Each emitter writes file j of a module. Files reference file j-1 of the
|
||||
* SAME module (cross-file resolution work); modules never reference each
|
||||
* other (independence — the property the linearity gate rests on). */
|
||||
|
||||
static void cx_emit_java(FILE *f, int m, int j) {
|
||||
fprintf(f, "package mod%d;\n\npublic class C%d {\n", m, j);
|
||||
fprintf(f, " public int val%d(int x) {\n return x + %d;\n }\n", j, j);
|
||||
if (j > 0) {
|
||||
fprintf(f, " public int chain() {\n");
|
||||
fprintf(f, " C%d prev = new C%d();\n", j - 1, j - 1);
|
||||
fprintf(f, " return prev.val%d(1) + val%d(2);\n }\n", j - 1, j);
|
||||
} else {
|
||||
fprintf(f, " public int chain() {\n return val0(2);\n }\n");
|
||||
}
|
||||
fprintf(f, "}\n");
|
||||
}
|
||||
|
||||
static void cx_emit_python(FILE *f, int m, int j) {
|
||||
(void)m;
|
||||
if (j > 0) {
|
||||
fprintf(f, "import f%d\n\n", j - 1);
|
||||
}
|
||||
fprintf(f, "def val%d(x):\n return x + %d\n\n", j, j);
|
||||
if (j > 0) {
|
||||
fprintf(f, "def chain%d():\n return f%d.val%d(1) + val%d(2)\n", j, j - 1, j - 1, j);
|
||||
} else {
|
||||
fprintf(f, "def chain0():\n return val0(2)\n");
|
||||
}
|
||||
}
|
||||
|
||||
static void cx_emit_go(FILE *f, int m, int j) {
|
||||
fprintf(f, "package mod%d\n\n", m);
|
||||
fprintf(f, "func Val%d(x int) int {\n\treturn x + %d\n}\n\n", j, j);
|
||||
if (j > 0) {
|
||||
fprintf(f, "func Chain%d() int {\n\treturn Val%d(1) + Val%d(2)\n}\n", j, j - 1, j);
|
||||
} else {
|
||||
fprintf(f, "func Chain0() int {\n\treturn Val0(2)\n}\n");
|
||||
}
|
||||
}
|
||||
|
||||
static void cx_emit_ts(FILE *f, int m, int j) {
|
||||
(void)m;
|
||||
if (j > 0) {
|
||||
fprintf(f, "import { val%d } from \"./f%d\";\n\n", j - 1, j - 1);
|
||||
}
|
||||
fprintf(f, "export function val%d(x: number): number {\n return x + %d;\n}\n\n", j, j);
|
||||
if (j > 0) {
|
||||
fprintf(f, "export function chain%d(): number {\n return val%d(1) + val%d(2);\n}\n", j,
|
||||
j - 1, j);
|
||||
} else {
|
||||
fprintf(f, "export function chain0(): number {\n return val0(2);\n}\n");
|
||||
}
|
||||
}
|
||||
|
||||
typedef struct {
|
||||
CBMLanguage lang;
|
||||
const char *dirname; /* corpus subdir, doubles as the per-lang scope */
|
||||
const char *file_prefix;
|
||||
const char *ext;
|
||||
void (*emit)(FILE *f, int m, int j);
|
||||
} CxTemplate;
|
||||
|
||||
static const CxTemplate CX_TEMPLATES[] = {
|
||||
{CBM_LANG_JAVA, "javasrc", "C", ".java", cx_emit_java},
|
||||
{CBM_LANG_PYTHON, "pysrc", "f", ".py", cx_emit_python},
|
||||
{CBM_LANG_GO, "gosrc", "f", ".go", cx_emit_go},
|
||||
{CBM_LANG_TYPESCRIPT, "tssrc", "f", ".ts", cx_emit_ts},
|
||||
};
|
||||
enum { CX_TEMPLATE_COUNT = sizeof(CX_TEMPLATES) / sizeof(CX_TEMPLATES[0]) };
|
||||
|
||||
/* Fixture-dir provider: tests/fixtures/complexity/<lang-name>/ — every file in
|
||||
* it is copied verbatim into each module dir. Lets a new language join the
|
||||
* guard by dropping fixtures, with no edit to this suite. */
|
||||
static bool cx_fixture_dir_for(CBMLanguage lang, char *out, size_t cap) {
|
||||
const char *name = cbm_language_name(lang);
|
||||
if (!name || !name[0]) {
|
||||
return false;
|
||||
}
|
||||
snprintf(out, cap, "tests/fixtures/complexity/%s", name);
|
||||
cbm_dir_t *d = cbm_opendir(out);
|
||||
if (!d) {
|
||||
return false;
|
||||
}
|
||||
cbm_closedir(d);
|
||||
return true;
|
||||
}
|
||||
|
||||
static int cx_copy_file(const char *src, const char *dst) {
|
||||
FILE *in = fopen(src, "rb");
|
||||
if (!in) {
|
||||
return -1;
|
||||
}
|
||||
FILE *out = fopen(dst, "wb");
|
||||
if (!out) {
|
||||
fclose(in);
|
||||
return -1;
|
||||
}
|
||||
char buf[4096];
|
||||
size_t n;
|
||||
while ((n = fread(buf, 1, sizeof(buf), in)) > 0) {
|
||||
fwrite(buf, 1, n, out);
|
||||
}
|
||||
fclose(in);
|
||||
fclose(out);
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* ── Corpus builder ──────────────────────────────────────────────────── */
|
||||
|
||||
static int cx_build_corpus(const char *root, int k_modules) {
|
||||
char dir[1024];
|
||||
char path[1200];
|
||||
for (int t = 0; t < CX_TEMPLATE_COUNT; t++) {
|
||||
const CxTemplate *tp = &CX_TEMPLATES[t];
|
||||
for (int m = 0; m < k_modules; m++) {
|
||||
snprintf(dir, sizeof(dir), "%s/%s/mod%d", root, tp->dirname, m);
|
||||
if (th_mkdir_p(dir) != 0) {
|
||||
return -1;
|
||||
}
|
||||
for (int j = 0; j < CX_FILES_PER_MOD; j++) {
|
||||
snprintf(path, sizeof(path), "%s/%s%d%s", dir, tp->file_prefix, j, tp->ext);
|
||||
FILE *f = fopen(path, "w");
|
||||
if (!f) {
|
||||
return -1;
|
||||
}
|
||||
tp->emit(f, m, j);
|
||||
fclose(f);
|
||||
}
|
||||
}
|
||||
}
|
||||
/* Fixture-dir providers: replicate each discovered language dir into
|
||||
* k module copies. Distinct paths make distinct modules/QNs, which is all
|
||||
* the independence argument needs. */
|
||||
for (int lang = 0; lang < CBM_LANG_COUNT; lang++) {
|
||||
bool templated = false;
|
||||
for (int t = 0; t < CX_TEMPLATE_COUNT; t++) {
|
||||
if (CX_TEMPLATES[t].lang == (CBMLanguage)lang) {
|
||||
templated = true;
|
||||
}
|
||||
}
|
||||
if (templated) {
|
||||
continue;
|
||||
}
|
||||
char fixdir[512];
|
||||
if (!cx_fixture_dir_for((CBMLanguage)lang, fixdir, sizeof(fixdir))) {
|
||||
continue;
|
||||
}
|
||||
cbm_dir_t *d = cbm_opendir(fixdir);
|
||||
if (!d) {
|
||||
continue;
|
||||
}
|
||||
cbm_dirent_t *entry;
|
||||
while ((entry = cbm_readdir(d)) != NULL) {
|
||||
if (entry->name[0] == '.') {
|
||||
continue;
|
||||
}
|
||||
for (int m = 0; m < k_modules; m++) {
|
||||
snprintf(dir, sizeof(dir), "%s/fx_%s/mod%d", root,
|
||||
cbm_language_name((CBMLanguage)lang), m);
|
||||
if (th_mkdir_p(dir) != 0) {
|
||||
continue;
|
||||
}
|
||||
char src[1024];
|
||||
snprintf(src, sizeof(src), "%s/%s", fixdir, entry->name);
|
||||
snprintf(path, sizeof(path), "%s/%s", dir, entry->name);
|
||||
(void)cx_copy_file(src, path);
|
||||
}
|
||||
}
|
||||
cbm_closedir(d);
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* Corpus shape #2 — the growing shared package. Real monorepos concentrate
|
||||
* files in a few large packages (org.<org>.common, …), and the JVM filter
|
||||
* branch includes every def sharing the file's namespace — so per-file work
|
||||
* tracks PACKAGE size. A package whose file count scales with the corpus is
|
||||
* therefore the honest reproducer for the #1669 growth pattern that fully
|
||||
* independent modules cannot show: here every extensive quantity must STILL
|
||||
* be linear in k, while a namespace/module-scoped per-file registry build
|
||||
* goes quadratic. */
|
||||
static int cx_build_bigpkg(const char *root, int k) {
|
||||
char dir[1024];
|
||||
char path[1200];
|
||||
snprintf(dir, sizeof(dir), "%s/bigsrc/bigpkg", root);
|
||||
if (th_mkdir_p(dir) != 0) {
|
||||
return -1;
|
||||
}
|
||||
int files = k * CX_BIGPKG_FILES_PER_K;
|
||||
for (int j = 0; j < files; j++) {
|
||||
snprintf(path, sizeof(path), "%s/B%d.java", dir, j);
|
||||
FILE *f = fopen(path, "w");
|
||||
if (!f) {
|
||||
return -1;
|
||||
}
|
||||
fprintf(f, "package bigpkg;\n\npublic class B%d {\n", j);
|
||||
fprintf(f, " public int val%d(int x) {\n return x + %d;\n }\n", j, j);
|
||||
if (j > 0) {
|
||||
fprintf(f, " public int chain() {\n");
|
||||
fprintf(f, " B%d prev = new B%d();\n", j - 1, j - 1);
|
||||
fprintf(f, " return prev.val%d(1) + val%d(2);\n }\n", j - 1, j);
|
||||
} else {
|
||||
fprintf(f, " public int chain() {\n return val0(2);\n }\n");
|
||||
}
|
||||
fprintf(f, "}\n");
|
||||
fclose(f);
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* ── Metrics ─────────────────────────────────────────────────────────── */
|
||||
|
||||
/* Per-pass timing capture: a TEE log sink parses `pass.timing` lines during a
|
||||
* run. Information for the report only — pass timings are wall-clock and never
|
||||
* gate (O9). */
|
||||
enum { CX_MAX_PASSES = 48 };
|
||||
typedef struct {
|
||||
char name[64];
|
||||
long ms;
|
||||
} CxPassMs;
|
||||
static CxPassMs g_cx_passes[CX_MAX_PASSES];
|
||||
static _Atomic int g_cx_pass_count = 0;
|
||||
|
||||
static void cx_pass_sink(const char *line) {
|
||||
if (!line || !strstr(line, "pass.timing")) {
|
||||
return;
|
||||
}
|
||||
const char *pp = strstr(line, "pass=");
|
||||
const char *ee = strstr(line, "elapsed_ms=");
|
||||
if (!pp || !ee) {
|
||||
return;
|
||||
}
|
||||
int n = atomic_fetch_add_explicit(&g_cx_pass_count, 1, memory_order_relaxed);
|
||||
if (n >= CX_MAX_PASSES) {
|
||||
return;
|
||||
}
|
||||
size_t i = 0;
|
||||
pp += 5;
|
||||
while (pp[i] && pp[i] != ' ' && i < sizeof(g_cx_passes[n].name) - 1) {
|
||||
g_cx_passes[n].name[i] = pp[i];
|
||||
i++;
|
||||
}
|
||||
g_cx_passes[n].name[i] = '\0';
|
||||
g_cx_passes[n].ms = atol(ee + 11);
|
||||
}
|
||||
|
||||
typedef struct {
|
||||
int nodes;
|
||||
int edges;
|
||||
int lang_nodes[CX_TEMPLATE_COUNT];
|
||||
int lang_edges[CX_TEMPLATE_COUNT];
|
||||
CxPassMs passes[CX_MAX_PASSES];
|
||||
int pass_count;
|
||||
uint64_t perfile_defs; /* Σ defs registered by per-file/overlay registry builds */
|
||||
uint64_t build_files;
|
||||
uint64_t filter_failed;
|
||||
uint64_t tail_lookups;
|
||||
uint64_t tail_candidates;
|
||||
uint64_t fallback_rows;
|
||||
double wall_s;
|
||||
} CxMetrics;
|
||||
|
||||
static double cx_now_s(void) {
|
||||
struct timespec ts;
|
||||
cbm_profile_now(&ts);
|
||||
return (double)ts.tv_sec + (double)ts.tv_nsec / 1e9;
|
||||
}
|
||||
|
||||
static int cx_run(const char *root, const char *db_path, CxMetrics *out) {
|
||||
memset(out, 0, sizeof(*out));
|
||||
|
||||
uint64_t d0;
|
||||
uint64_t b0;
|
||||
uint64_t f0;
|
||||
uint64_t x0;
|
||||
cbm_pxc_filter_stats(&d0, &b0, &f0, &x0);
|
||||
uint64_t tl0 = atomic_load_explicit(&g_lsp_tail_lookups, memory_order_relaxed);
|
||||
uint64_t tc0 = atomic_load_explicit(&g_lsp_tail_candidates, memory_order_relaxed);
|
||||
uint64_t fb0 = cbm_pp_lsp_linear_fallback_rows();
|
||||
|
||||
atomic_store_explicit(&g_cx_pass_count, 0, memory_order_relaxed);
|
||||
cbm_log_set_sink_ex(cx_pass_sink, CBM_LOG_SINK_TEE);
|
||||
|
||||
double t0 = cx_now_s();
|
||||
cbm_pipeline_t *p = cbm_pipeline_new(root, db_path, CBM_MODE_FULL);
|
||||
if (!p) {
|
||||
return -1;
|
||||
}
|
||||
int rc = cbm_pipeline_run(p);
|
||||
out->wall_s = cx_now_s() - t0;
|
||||
cbm_log_set_sink(NULL);
|
||||
int captured = atomic_load_explicit(&g_cx_pass_count, memory_order_relaxed);
|
||||
out->pass_count = captured < CX_MAX_PASSES ? captured : CX_MAX_PASSES;
|
||||
memcpy(out->passes, g_cx_passes, (size_t)out->pass_count * sizeof(CxPassMs));
|
||||
|
||||
char project[512];
|
||||
snprintf(project, sizeof(project), "%s", cbm_pipeline_project_name(p));
|
||||
cbm_pipeline_free(p);
|
||||
if (rc != 0) {
|
||||
return rc;
|
||||
}
|
||||
|
||||
uint64_t d1;
|
||||
uint64_t b1;
|
||||
uint64_t f1;
|
||||
uint64_t x1;
|
||||
cbm_pxc_filter_stats(&d1, &b1, &f1, &x1);
|
||||
out->perfile_defs = d1 - d0;
|
||||
out->build_files = b1 - b0;
|
||||
out->filter_failed = x1 - x0;
|
||||
out->tail_lookups = atomic_load_explicit(&g_lsp_tail_lookups, memory_order_relaxed) - tl0;
|
||||
out->tail_candidates = atomic_load_explicit(&g_lsp_tail_candidates, memory_order_relaxed) - tc0;
|
||||
out->fallback_rows = cbm_pp_lsp_linear_fallback_rows() - fb0;
|
||||
|
||||
cbm_store_t *s = cbm_store_open_path(db_path);
|
||||
if (!s) {
|
||||
return -1;
|
||||
}
|
||||
out->nodes = cbm_store_count_nodes(s, project);
|
||||
out->edges = cbm_store_count_edges(s, project);
|
||||
for (int t = 0; t < CX_TEMPLATE_COUNT; t++) {
|
||||
out->lang_nodes[t] = cbm_store_count_nodes_scoped(s, project, CX_TEMPLATES[t].dirname);
|
||||
out->lang_edges[t] = cbm_store_count_edges_scoped(s, project, CX_TEMPLATES[t].dirname);
|
||||
}
|
||||
cbm_store_close(s);
|
||||
return 0;
|
||||
}
|
||||
|
||||
static double cx_ratio(double num, double den) {
|
||||
return den > 0.0 ? num / den : 0.0;
|
||||
}
|
||||
|
||||
/* Shared across the suite so the report test reuses the measured pair instead
|
||||
* of paying two more pipeline runs. */
|
||||
static CxMetrics g_cx_base;
|
||||
static CxMetrics g_cx_doubled;
|
||||
static bool g_cx_measured = false;
|
||||
static char g_cx_root_base[512];
|
||||
static char g_cx_root_doubled[512];
|
||||
|
||||
static int cx_measure_pair(void) {
|
||||
if (g_cx_measured) {
|
||||
return 0;
|
||||
}
|
||||
const char *tmp = cbm_tmpdir();
|
||||
snprintf(g_cx_root_base, sizeof(g_cx_root_base), "%s/cbm_cx_base_XXXXXX", tmp);
|
||||
snprintf(g_cx_root_doubled, sizeof(g_cx_root_doubled), "%s/cbm_cx_dbl_XXXXXX", tmp);
|
||||
if (!cbm_mkdtemp(g_cx_root_base) || !cbm_mkdtemp(g_cx_root_doubled)) {
|
||||
return -1;
|
||||
}
|
||||
if (cx_build_corpus(g_cx_root_base, CX_K_BASE) != 0 ||
|
||||
cx_build_corpus(g_cx_root_doubled, CX_K_BASE * 2) != 0) {
|
||||
return -1;
|
||||
}
|
||||
char db1[600];
|
||||
char db2[600];
|
||||
snprintf(db1, sizeof(db1), "%s/cx.db", g_cx_root_base);
|
||||
snprintf(db2, sizeof(db2), "%s/cx.db", g_cx_root_doubled);
|
||||
if (cx_run(g_cx_root_base, db1, &g_cx_base) != 0) {
|
||||
return -1;
|
||||
}
|
||||
if (cx_run(g_cx_root_doubled, db2, &g_cx_doubled) != 0) {
|
||||
return -1;
|
||||
}
|
||||
g_cx_measured = true;
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* ── Tests ───────────────────────────────────────────────────────────── */
|
||||
|
||||
/* The flagship gate. Independent module copies ⇒ every extensive quantity must
|
||||
* scale linearly in the copy count. A files×corpus coupling shows up as a
|
||||
* ratio near 4 on the counter that sums per-file work. */
|
||||
TEST(complexity_replicated_modules_scale_linearly) {
|
||||
if (cx_measure_pair() != 0) {
|
||||
FAIL("failed to build/run the complexity corpus pair");
|
||||
}
|
||||
const CxMetrics *a = &g_cx_base;
|
||||
const CxMetrics *b = &g_cx_doubled;
|
||||
|
||||
/* Sanity: the corpus is real. */
|
||||
ASSERT_GT(a->nodes, 50);
|
||||
ASSERT_GT(a->edges, 20);
|
||||
|
||||
double node_r = cx_ratio(b->nodes, a->nodes);
|
||||
double edge_r = cx_ratio(b->edges, a->edges);
|
||||
printf(" nodes %d -> %d (ratio %.2f) edges %d -> %d (ratio %.2f)\n", a->nodes, b->nodes,
|
||||
node_r, a->edges, b->edges, edge_r);
|
||||
ASSERT_TRUE(node_r >= CX_RATIO_LO && node_r <= CX_RATIO_HI);
|
||||
ASSERT_TRUE(edge_r >= CX_RATIO_LO && edge_r <= CX_RATIO_HI);
|
||||
|
||||
/* Per-language linearity, from the same run pair (scoped by subtree). If
|
||||
* scoped counting returns 0 for the base run the scope semantics changed —
|
||||
* surface that rather than silently skipping. */
|
||||
for (int t = 0; t < CX_TEMPLATE_COUNT; t++) {
|
||||
ASSERT_GT(a->lang_nodes[t], 0);
|
||||
double lr = cx_ratio(b->lang_nodes[t], a->lang_nodes[t]);
|
||||
printf(" %-8s nodes %d -> %d (ratio %.2f)\n", CX_TEMPLATES[t].dirname, a->lang_nodes[t],
|
||||
b->lang_nodes[t], lr);
|
||||
ASSERT_TRUE(lr >= CX_RATIO_LO && lr <= CX_RATIO_HI);
|
||||
}
|
||||
PASS();
|
||||
}
|
||||
|
||||
/* Σ per-file registry work must be linear in independent copies. This is the
|
||||
* counter that measured ~4x/doubling on the v0.10.x Java path (defs_per_file
|
||||
* tracked defs_total — #1669). Guarded against vacuous passes: the base run
|
||||
* must have produced real registry work, so an accidental zeroing of the
|
||||
* counter fails loudly instead of green-washing the gate. */
|
||||
TEST(complexity_perfile_registry_work_is_linear) {
|
||||
if (cx_measure_pair() != 0) {
|
||||
FAIL("failed to build/run the complexity corpus pair");
|
||||
}
|
||||
const CxMetrics *a = &g_cx_base;
|
||||
const CxMetrics *b = &g_cx_doubled;
|
||||
|
||||
printf(" perfile_defs %llu -> %llu build_files %llu -> %llu filter_failed %llu\n",
|
||||
(unsigned long long)a->perfile_defs, (unsigned long long)b->perfile_defs,
|
||||
(unsigned long long)a->build_files, (unsigned long long)b->build_files,
|
||||
(unsigned long long)(a->filter_failed + b->filter_failed));
|
||||
|
||||
/* Non-vacuous: the corpus includes languages that register per-file defs
|
||||
* (Java at minimum). If this is ever 0 the counter wiring broke — that is
|
||||
* a test defect to fix, not a pass. */
|
||||
ASSERT_GT((long long)a->perfile_defs, (long long)CX_MIN_BASE_WORK);
|
||||
|
||||
double defs_r = cx_ratio((double)b->perfile_defs, (double)a->perfile_defs);
|
||||
printf(" perfile_defs ratio %.2f (linear ~2, files x corpus ~4)\n", defs_r);
|
||||
ASSERT_TRUE(defs_r <= CX_RATIO_HI);
|
||||
|
||||
/* Recorded, deliberately NOT gated (the O10 note): tail_candidates and
|
||||
* fallback_rows are legitimately superlinear under replication TODAY —
|
||||
* same-short-name candidate sets grow with the copy count by design of
|
||||
* the current tail scan. Both measured ~1 ns/unit (#1669: indexing the
|
||||
* scan away cut 242M visits for 0% wall). Gate them only after those
|
||||
* scans are bounded; until then they are trend data for the report. */
|
||||
printf(" [info] tail_lookups %llu -> %llu tail_candidates %llu -> %llu fallback_rows "
|
||||
"%llu -> %llu\n",
|
||||
(unsigned long long)a->tail_lookups, (unsigned long long)b->tail_lookups,
|
||||
(unsigned long long)a->tail_candidates, (unsigned long long)b->tail_candidates,
|
||||
(unsigned long long)a->fallback_rows, (unsigned long long)b->fallback_rows);
|
||||
PASS();
|
||||
}
|
||||
|
||||
static CxMetrics g_cx_big_base;
|
||||
static CxMetrics g_cx_big_doubled;
|
||||
static bool g_cx_big_measured = false;
|
||||
|
||||
static int cx_measure_bigpkg_pair(void) {
|
||||
if (g_cx_big_measured) {
|
||||
return 0;
|
||||
}
|
||||
const char *tmp = cbm_tmpdir();
|
||||
char ra[512];
|
||||
char rb[512];
|
||||
snprintf(ra, sizeof(ra), "%s/cbm_cxbig_a_XXXXXX", tmp);
|
||||
snprintf(rb, sizeof(rb), "%s/cbm_cxbig_b_XXXXXX", tmp);
|
||||
if (!cbm_mkdtemp(ra) || !cbm_mkdtemp(rb)) {
|
||||
return -1;
|
||||
}
|
||||
if (cx_build_bigpkg(ra, CX_K_BASE) != 0 || cx_build_bigpkg(rb, CX_K_BASE * 2) != 0) {
|
||||
return -1;
|
||||
}
|
||||
char db1[600];
|
||||
char db2[600];
|
||||
snprintf(db1, sizeof(db1), "%s/cx.db", ra);
|
||||
snprintf(db2, sizeof(db2), "%s/cx.db", rb);
|
||||
if (cx_run(ra, db1, &g_cx_big_base) != 0) {
|
||||
return -1;
|
||||
}
|
||||
if (cx_run(rb, db2, &g_cx_big_doubled) != 0) {
|
||||
return -1;
|
||||
}
|
||||
g_cx_big_measured = true;
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* One growing package instead of independent modules. Nodes and edges must
|
||||
* still be linear in file count — and so must Σ per-file registry work: a
|
||||
* build scoped to the file's MODULE or NAMESPACE pays package-size per file
|
||||
* and lands at ratio ~4 here. Only a per-FILE-scoped build stays at ~2. This
|
||||
* is the exact #1669 growth pattern (per-file work tracking corpus share). */
|
||||
TEST(complexity_shared_package_growth_stays_linear) {
|
||||
if (cx_measure_bigpkg_pair() != 0) {
|
||||
FAIL("failed to build/run the big-package corpus pair");
|
||||
}
|
||||
const CxMetrics *a = &g_cx_big_base;
|
||||
const CxMetrics *b = &g_cx_big_doubled;
|
||||
|
||||
double node_r = cx_ratio(b->nodes, a->nodes);
|
||||
double edge_r = cx_ratio(b->edges, a->edges);
|
||||
printf(" nodes %d -> %d (ratio %.2f) edges %d -> %d (ratio %.2f)\n", a->nodes, b->nodes,
|
||||
node_r, a->edges, b->edges, edge_r);
|
||||
ASSERT_GT(a->nodes, 30);
|
||||
ASSERT_TRUE(node_r >= CX_RATIO_LO && node_r <= CX_RATIO_HI);
|
||||
ASSERT_TRUE(edge_r >= CX_RATIO_LO && edge_r <= CX_RATIO_HI);
|
||||
|
||||
printf(" perfile_defs %llu -> %llu (ratio %.2f; linear ~2, namespace/module-scoped ~4)\n",
|
||||
(unsigned long long)a->perfile_defs, (unsigned long long)b->perfile_defs,
|
||||
cx_ratio((double)b->perfile_defs, (double)a->perfile_defs));
|
||||
/* Non-vacuous floor (false-guard audit): the package produces real
|
||||
* registry work; zero means the counter wiring broke. */
|
||||
ASSERT_GT((long long)a->perfile_defs, (long long)CX_MIN_BASE_WORK);
|
||||
double defs_r = cx_ratio((double)b->perfile_defs, (double)a->perfile_defs);
|
||||
ASSERT_TRUE(defs_r <= CX_RATIO_HI);
|
||||
PASS();
|
||||
}
|
||||
|
||||
/* Throughput report — information only, never a gate (CI-determinism rule:
|
||||
* rates depend on the machine and scheduler, so a threshold would be a
|
||||
* lottery). Written locally under private/ (gitignored); CBM_COMPLEXITY_
|
||||
* REPORT_DIR overrides. Skipped on starved legs where rates are meaningless. */
|
||||
TEST(complexity_throughput_report_written) {
|
||||
const char *skip_perf = getenv("CBM_SKIP_PERF");
|
||||
if (skip_perf && skip_perf[0] == '1') {
|
||||
/* Deliberate operator config, not a hidden environment failure
|
||||
* (no-skips policy): rates measured under CBM_SKIP_PERF starvation
|
||||
* would only mislead, so reporting is OFF and there is nothing left
|
||||
* for this test to assert. */
|
||||
fprintf(stderr, " [complexity] CBM_SKIP_PERF=1: throughput report disabled by config\n");
|
||||
PASS();
|
||||
}
|
||||
if (cx_measure_pair() != 0) {
|
||||
FAIL("failed to build/run the complexity corpus pair");
|
||||
}
|
||||
const char *dir = getenv("CBM_COMPLEXITY_REPORT_DIR");
|
||||
if (!dir || !dir[0]) {
|
||||
dir = "private/benchmarks";
|
||||
}
|
||||
if (th_mkdir_p(dir) != 0) {
|
||||
FAIL("report dir not creatable (set CBM_COMPLEXITY_REPORT_DIR to a writable path)");
|
||||
}
|
||||
char path[1024];
|
||||
snprintf(path, sizeof(path), "%s/complexity-%lld.json", dir, (long long)time(NULL));
|
||||
FILE *f = fopen(path, "w");
|
||||
ASSERT_NOT_NULL(f);
|
||||
|
||||
const CxMetrics *a = &g_cx_base;
|
||||
const CxMetrics *b = &g_cx_doubled;
|
||||
#if defined(__APPLE__)
|
||||
const char *plat = "darwin";
|
||||
#elif defined(_WIN32)
|
||||
const char *plat = "windows";
|
||||
#else
|
||||
const char *plat = "linux";
|
||||
#endif
|
||||
#if defined(__aarch64__) || defined(_M_ARM64)
|
||||
const char *arch = "arm64";
|
||||
#else
|
||||
const char *arch = "x86_64";
|
||||
#endif
|
||||
fprintf(f, "{\n \"schema\": 1,\n \"suite\": \"complexity\",\n");
|
||||
fprintf(f, " \"timestamp\": %lld,\n \"platform\": \"%s\",\n \"arch\": \"%s\",\n",
|
||||
(long long)time(NULL), plat, arch);
|
||||
fprintf(f, " \"k_base\": %d,\n \"files_per_module\": %d,\n", CX_K_BASE, CX_FILES_PER_MOD);
|
||||
fprintf(f, " \"runs\": [\n");
|
||||
const CxMetrics *runs[2] = {a, b};
|
||||
for (int i = 0; i < 2; i++) {
|
||||
const CxMetrics *m = runs[i];
|
||||
fprintf(f,
|
||||
" {\"k\": %d, \"nodes\": %d, \"edges\": %d, \"wall_s\": %.3f,\n"
|
||||
" \"nodes_per_s\": %.0f, \"edges_per_s\": %.0f,\n"
|
||||
" \"perfile_defs\": %llu, \"tail_candidates\": %llu, \"fallback_rows\": "
|
||||
"%llu}%s\n",
|
||||
i == 0 ? CX_K_BASE : CX_K_BASE * 2, m->nodes, m->edges, m->wall_s,
|
||||
m->wall_s > 0 ? (double)m->nodes / m->wall_s : 0.0,
|
||||
m->wall_s > 0 ? (double)m->edges / m->wall_s : 0.0,
|
||||
(unsigned long long)m->perfile_defs, (unsigned long long)m->tail_candidates,
|
||||
(unsigned long long)m->fallback_rows, i == 0 ? "," : "");
|
||||
}
|
||||
fprintf(f, " ],\n");
|
||||
fprintf(f, " \"languages\": [\n");
|
||||
for (int t = 0; t < CX_TEMPLATE_COUNT; t++) {
|
||||
fprintf(f,
|
||||
" {\"name\": \"%s\", \"nodes\": [%d, %d], \"edges\": [%d, %d], "
|
||||
"\"node_ratio\": %.3f}%s\n",
|
||||
CX_TEMPLATES[t].dirname, a->lang_nodes[t], b->lang_nodes[t], a->lang_edges[t],
|
||||
b->lang_edges[t], cx_ratio(b->lang_nodes[t], a->lang_nodes[t]),
|
||||
t + 1 < CX_TEMPLATE_COUNT ? "," : "");
|
||||
}
|
||||
fprintf(f, " ],\n");
|
||||
fprintf(f, " \"passes_ms\": {\n");
|
||||
for (int i = 0; i < 2; i++) {
|
||||
const CxMetrics *m = runs[i];
|
||||
fprintf(f, " \"k%d\": {", i == 0 ? CX_K_BASE : CX_K_BASE * 2);
|
||||
for (int j = 0; j < m->pass_count; j++) {
|
||||
fprintf(f, "%s\"%s\": %ld", j == 0 ? "" : ", ", m->passes[j].name, m->passes[j].ms);
|
||||
}
|
||||
fprintf(f, "}%s\n", i == 0 ? "," : "");
|
||||
}
|
||||
fprintf(f, " },\n");
|
||||
fprintf(f, " \"ratios\": {\"nodes\": %.3f, \"edges\": %.3f, \"perfile_defs\": %.3f},\n",
|
||||
cx_ratio(b->nodes, a->nodes), cx_ratio(b->edges, a->edges),
|
||||
cx_ratio((double)b->perfile_defs, (double)a->perfile_defs));
|
||||
/* Languages without a provider, so the coverage boundary is explicit in
|
||||
* the artifact rather than implied. */
|
||||
fprintf(f, " \"skipped_languages\": [");
|
||||
bool first = true;
|
||||
for (int lang = 0; lang < CBM_LANG_COUNT; lang++) {
|
||||
bool covered = false;
|
||||
for (int t = 0; t < CX_TEMPLATE_COUNT; t++) {
|
||||
if (CX_TEMPLATES[t].lang == (CBMLanguage)lang) {
|
||||
covered = true;
|
||||
}
|
||||
}
|
||||
char fixdir[512];
|
||||
if (!covered && cx_fixture_dir_for((CBMLanguage)lang, fixdir, sizeof(fixdir))) {
|
||||
covered = true;
|
||||
}
|
||||
if (!covered) {
|
||||
const char *name = cbm_language_name((CBMLanguage)lang);
|
||||
if (name && name[0]) {
|
||||
fprintf(f, "%s\"%s\"", first ? "" : ", ", name);
|
||||
first = false;
|
||||
}
|
||||
}
|
||||
}
|
||||
fprintf(f, "]\n}\n");
|
||||
fclose(f);
|
||||
printf(" report: %s\n", path);
|
||||
PASS();
|
||||
}
|
||||
|
||||
SUITE(complexity) {
|
||||
RUN_TEST(complexity_replicated_modules_scale_linearly);
|
||||
RUN_TEST(complexity_perfile_registry_work_is_linear);
|
||||
RUN_TEST(complexity_shared_package_growth_stays_linear);
|
||||
RUN_TEST(complexity_throughput_report_written);
|
||||
}
|
||||
@@ -9,6 +9,7 @@
|
||||
#include "../src/foundation/diagnostics.h"
|
||||
#include "../src/foundation/log.h"
|
||||
#include "../src/foundation/platform.h"
|
||||
#include "../src/foundation/profile.h"
|
||||
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
@@ -568,7 +569,88 @@ TEST(diagnostics_outputs_are_owner_private_windows) {
|
||||
}
|
||||
#endif
|
||||
|
||||
|
||||
/* ── scaling probe (profile.h) ──────────────────────────────────────
|
||||
*
|
||||
* These pin the arithmetic and the checkpoint bookkeeping, deliberately WITHOUT
|
||||
* timing anything. A test that tried to prove the detector works by generating
|
||||
* a genuinely quadratic workload would be asserting on the scheduler, and would
|
||||
* go flaky on a loaded CI box — so the fit is a pure function and gets fed
|
||||
* synthetic points instead. */
|
||||
|
||||
TEST(scale_fit_k_recognises_linear_and_quadratic_growth) {
|
||||
/* 8x the items, 8x the time => k = 1 */
|
||||
ASSERT_TRUE(cbm_scale_fit_k(1000, 1000, 8000, 8000) > 0.99);
|
||||
ASSERT_TRUE(cbm_scale_fit_k(1000, 1000, 8000, 8000) < 1.01);
|
||||
|
||||
/* 8x the items, 64x the time => k = 2 */
|
||||
ASSERT_TRUE(cbm_scale_fit_k(1000, 1000, 8000, 64000) > 1.99);
|
||||
ASSERT_TRUE(cbm_scale_fit_k(1000, 1000, 8000, 64000) < 2.01);
|
||||
|
||||
/* n log n sits between, and must NOT be mistaken for quadratic */
|
||||
double k_nlogn = cbm_scale_fit_k(1000, 1000, 8000, 8000 * 13 / 10);
|
||||
ASSERT_TRUE(k_nlogn > 1.0);
|
||||
ASSERT_TRUE(k_nlogn < CBM_SCALE_WARN_K);
|
||||
PASS();
|
||||
}
|
||||
|
||||
TEST(scale_fit_k_rejects_degenerate_input) {
|
||||
ASSERT_TRUE(cbm_scale_fit_k(0, 1000, 8000, 8000) < 0.0); /* no first point */
|
||||
ASSERT_TRUE(cbm_scale_fit_k(1000, 0, 8000, 8000) < 0.0); /* zero elapsed */
|
||||
ASSERT_TRUE(cbm_scale_fit_k(1000, 1000, 1000, 8000) < 0.0); /* n did not grow */
|
||||
ASSERT_TRUE(cbm_scale_fit_k(1000, 1000, 500, 8000) < 0.0); /* n went backwards */
|
||||
ASSERT_TRUE(cbm_scale_fit_k(1000, 1000, 8000, 0) < 0.0); /* zero total */
|
||||
PASS();
|
||||
}
|
||||
|
||||
TEST(scale_probe_records_checkpoints_at_eighths) {
|
||||
cbm_scale_probe_t p;
|
||||
cbm_scale_begin(&p, "unit", 1024);
|
||||
|
||||
/* Below the first threshold (1024/8 = 128), nothing is claimed. */
|
||||
for (long i = 0; i < 128; i++) {
|
||||
cbm_scale_tick(&p, i);
|
||||
}
|
||||
ASSERT_EQ(atomic_load(&p.next_cp), 0);
|
||||
|
||||
cbm_scale_tick(&p, 128); /* 1/8 */
|
||||
ASSERT_EQ(atomic_load(&p.next_cp), 1);
|
||||
cbm_scale_tick(&p, 200); /* still short of 1/4 */
|
||||
ASSERT_EQ(atomic_load(&p.next_cp), 1);
|
||||
cbm_scale_tick(&p, 256); /* 1/4 */
|
||||
ASSERT_EQ(atomic_load(&p.next_cp), 2);
|
||||
cbm_scale_tick(&p, 512); /* 1/2 */
|
||||
ASSERT_EQ(atomic_load(&p.next_cp), 3);
|
||||
cbm_scale_tick(&p, 1024); /* all */
|
||||
ASSERT_EQ(atomic_load(&p.next_cp), 4);
|
||||
|
||||
/* Saturates rather than overrunning the array. */
|
||||
cbm_scale_tick(&p, 2048);
|
||||
ASSERT_EQ(atomic_load(&p.next_cp), 4);
|
||||
ASSERT_EQ(p.cp_items[0], 128);
|
||||
ASSERT_EQ(p.cp_items[3], 1024);
|
||||
cbm_scale_end(&p);
|
||||
PASS();
|
||||
}
|
||||
|
||||
TEST(scale_probe_ignores_runs_too_small_to_judge) {
|
||||
/* Under SCALE_MIN_ITEMS a pass is not where an O(n^2) hurts anyone, and the
|
||||
* fit would be measuring noise — so the probe must stay silent. */
|
||||
cbm_scale_probe_t p;
|
||||
cbm_scale_begin(&p, "tiny", 64);
|
||||
for (long i = 0; i <= 64; i++) {
|
||||
cbm_scale_tick(&p, i);
|
||||
}
|
||||
ASSERT_EQ(atomic_load(&p.next_cp), 0);
|
||||
cbm_scale_end(&p); /* must not emit, must not crash */
|
||||
PASS();
|
||||
}
|
||||
|
||||
SUITE(diagnostics) {
|
||||
RUN_TEST(scale_fit_k_recognises_linear_and_quadratic_growth);
|
||||
RUN_TEST(scale_fit_k_rejects_degenerate_input);
|
||||
RUN_TEST(scale_probe_records_checkpoints_at_eighths);
|
||||
RUN_TEST(scale_probe_ignores_runs_too_small_to_judge);
|
||||
#ifndef _WIN32
|
||||
RUN_TEST(diagnostics_rejects_predictable_tmp_symlinks);
|
||||
#endif
|
||||
|
||||
@@ -680,6 +680,7 @@ extern void suite_str_util(void);
|
||||
extern void suite_workspace(void);
|
||||
extern void suite_platform(void);
|
||||
extern void suite_diagnostics(void);
|
||||
extern void suite_complexity(void);
|
||||
extern void suite_subprocess(void);
|
||||
extern void suite_private_file_lock(void);
|
||||
extern void suite_lock_registry(void);
|
||||
@@ -926,6 +927,7 @@ int main(int argc, char **argv) {
|
||||
RUN_SELECTED_SUITE(workspace);
|
||||
RUN_SELECTED_SUITE(platform);
|
||||
RUN_SELECTED_SUITE(diagnostics);
|
||||
RUN_SELECTED_SUITE(complexity);
|
||||
RUN_SELECTED_SUITE(subprocess);
|
||||
RUN_SELECTED_SUITE(private_file_lock);
|
||||
RUN_SELECTED_SUITE(lock_registry);
|
||||
|
||||
@@ -4131,6 +4131,49 @@ TEST(tslsp_nested_class_resolution) {
|
||||
/* JavaScript relies on local `new` inference here, while TypeScript and TSX use
|
||||
* explicit parameter types below. All three parser modes must preserve the
|
||||
* full ordinary call-expression span, independent of JSX-specific behavior. */
|
||||
/* ── Expression-eval complexity guard ─────────────────────────────────────────
|
||||
*
|
||||
* The objectSpreadRepeatedComplexity class (TS conformance suite): a chain of
|
||||
* `...(c[i] && c[j] && {...})` spreads makes the union fan out 2^(n-1) ways,
|
||||
* and overload resolution re-evaluates shared subexpressions once per
|
||||
* alternative. The per-node memo turns that into one eval per node. Guarded
|
||||
* via the deterministic budget counter, not wall-clock: without the memo this
|
||||
* source burns the entire eval budget (warned=true); with it, consumption
|
||||
* stays around the node count. */
|
||||
TEST(tslsp_eval_memo_bounds_spread_bomb_work) {
|
||||
/* The tsc-compiled form of that file: nested Object.assign(...) member
|
||||
* calls. Resolving the stdlib method evaluates every argument expression
|
||||
* once per lookup path (method dispatch + namespace fallback), and the
|
||||
* first argument is itself the next nested call — without the memo the
|
||||
* shared subtree re-walks once per enclosing level: 2^n evals. */
|
||||
char src[16384];
|
||||
int off = snprintf(src, sizeof(src), "\"use strict\";\nfunction f(cnd) {\n return ");
|
||||
for (int i = 18; i >= 0; i--) {
|
||||
off += snprintf(src + off, sizeof(src) - (size_t)off, "Object.assign(");
|
||||
}
|
||||
off += snprintf(src + off, sizeof(src) - (size_t)off, "{}");
|
||||
for (int i = 0; i < 19; i++) {
|
||||
off += snprintf(src + off, sizeof(src) - (size_t)off,
|
||||
", (cnd[%d] && cnd[%d] && {\n prop%da: 1,\n prop%db: 1,\n"
|
||||
" }))",
|
||||
2 * i + 1, 2 * i + 2, i, i);
|
||||
}
|
||||
snprintf(src + off, sizeof(src) - (size_t)off, ";\n}\n");
|
||||
|
||||
CBMFileResult *r = extract_js(src);
|
||||
ASSERT_NOT_NULL(r);
|
||||
ASSERT_GTE(r->defs.count, 1); /* f extracted; the graph itself is tiny */
|
||||
|
||||
/* Same thread ran the eval: the seam reads its final budget state. */
|
||||
ASSERT_FALSE(cbm_ts_lsp_test_budget_warned());
|
||||
long budget0 = 1000000 + (long)strlen(src) * 64;
|
||||
long consumed = budget0 - cbm_ts_lsp_test_budget_remaining();
|
||||
ASSERT_LT(consumed, 100000);
|
||||
|
||||
cbm_free_result(r);
|
||||
PASS();
|
||||
}
|
||||
|
||||
TEST(tslsp_js_ordinary_same_leaf_calls_join_by_exact_site) {
|
||||
static const char source[] = "class Alpha { render() {} }\n"
|
||||
"class Beta { render() {} }\n"
|
||||
@@ -4641,6 +4684,9 @@ SUITE(ts_lsp) {
|
||||
RUN_TEST(tslsp_void_returning_method);
|
||||
RUN_TEST(tslsp_nested_class_resolution);
|
||||
|
||||
/* Expression-eval complexity guard. */
|
||||
RUN_TEST(tslsp_eval_memo_bounds_spread_bomb_work);
|
||||
|
||||
/* Ordinary same-leaf occurrence identity (JS / TS / TSX). */
|
||||
RUN_TEST(tslsp_js_ordinary_same_leaf_calls_join_by_exact_site);
|
||||
RUN_TEST(tslsp_ts_ordinary_same_leaf_calls_join_by_exact_site);
|
||||
|
||||
Reference in New Issue
Block a user