trending/codebase-memory-mcp
1
0
Fork 0
mirror of https://github.com/DeusData/codebase-memory-mcp.git synced 2026-10-31 07:33:32 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #1883 from DeusData/feat/879-importance-distill

Browse Source 
feat(pipeline): index-time importance scoring (weighted-degree)
This commit is contained in:
Martin Vogel
2026-08-29 11:32:15 +02:00
committed by GitHub
parent d8f8f45fda 396ac348b5
commit 1130f577ee
8 changed files with 1635 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Makefile.cbm
+2 -1
View File
@@ -354,6 +354,7 @@ PIPELINE_SRCS = \
src/pipeline/pass_similarity.c \
src/pipeline/pass_semantic_edges.c \
src/pipeline/pass_complexity.c \
src/pipeline/pass_importance.c \
src/pipeline/pass_cross_repo.c \
src/pipeline/artifact.c \
src/pipeline/pass_ensemble_routing.c \
@@ -560,7 +561,7 @@ TEST_DISCOVER_SRCS = \
TEST_GRAPH_BUFFER_SRCS = tests/test_graph_buffer.c
TEST_PIPELINE_SRCS = tests/test_registry.c tests/test_pipeline.c tests/test_cross_repo.c tests/test_fqn.c tests/test_route_canon.c tests/test_path_alias.c tests/test_configlink.c tests/test_infrascan.c tests/test_worker_pool.c tests/test_parallel.c tests/test_index_resilience.c tests/test_index_format.c tests/test_call_reference_contract.c tests/repro/repro_call_scope_usages.c tests/repro/repro_call_argument_usages.c tests/repro/repro_reference_precision.c tests/repro/repro_lexical_binding_precision.c tests/repro/repro_call_argument_matrix_a.c tests/repro/repro_call_argument_matrix_b.c tests/repro/repro_call_node_behaviors.c tests/repro/repro_language_registry.c tests/repro/repro_call_node_manifest.c tests/repro/repro_lsp_ordered_signatures.c tests/repro/repro_lsp_ordered_local.c tests/repro/repro_ts_overload_return_chains.c tests/repro/repro_harness_cleanup.c tests/repro/repro_runner_filter.c
TEST_PIPELINE_SRCS = tests/test_registry.c tests/test_pipeline.c tests/test_importance.c tests/test_cross_repo.c tests/test_fqn.c tests/test_route_canon.c tests/test_path_alias.c tests/test_configlink.c tests/test_infrascan.c tests/test_worker_pool.c tests/test_parallel.c tests/test_index_resilience.c tests/test_index_format.c tests/test_call_reference_contract.c tests/repro/repro_call_scope_usages.c tests/repro/repro_call_argument_usages.c tests/repro/repro_reference_precision.c tests/repro/repro_lexical_binding_precision.c tests/repro/repro_call_argument_matrix_a.c tests/repro/repro_call_argument_matrix_b.c tests/repro/repro_call_node_behaviors.c tests/repro/repro_language_registry.c tests/repro/repro_call_node_manifest.c tests/repro/repro_lsp_ordered_signatures.c tests/repro/repro_lsp_ordered_local.c tests/repro/repro_ts_overload_return_chains.c tests/repro/repro_harness_cleanup.c tests/repro/repro_runner_filter.c
TEST_WATCHER_SRCS = tests/test_watcher.c
src/pipeline/pass_importance.c
+556
View File
@@ -0,0 +1,556 @@
/*
* pass_importance.c — Index-time per-symbol importance score (weighted degree).
*
* Predump pass. Computes an importance score for every Function/Method/Class
* node and stores it as a numeric "importance" key inside the node's EXISTING
* properties_json blob — no new column, no schema change, and therefore no
* CBM_INDEX_FORMAT_VERSION bump. Indexes written by older builds simply lack
* the key; every consumer must tolerate its absence.
*
* Model (weighted degree, Aider-repomap shaped):
*
* importance = sqrt(num_refs) * priv * generic * distinct * test_penalty
*
* num_refs incoming CALLS + USAGE edges. 0 -> sqrt(0) = 0.
* priv 0.1 name is private (leading underscore)
* generic 0.1 the name is defined in >= 5 DISTINCT files
* distinct 10 name is snake_case or camelCase AND len >= 8
* test_penalty
* 0.1 the symbol is test scaffolding — either the target of an
* incoming TESTS edge (a production helper exercised by a
* test), or it lives in a test file per the graph's own
* canonical cbm_is_test_path() classifier (the same one
* pass_tests uses).
*
* Transitive importance (PageRank) is deliberately NOT built here: weighted
* degree is what this pass claims, and a transitive refinement has to earn
* itself against a fixed judgment set before it ships.
*
* ── Cost ─────────────────────────────────────────────────────────────
* The generic-name multiplier needs |{ file a name is defined in }|. The
* obvious shape — for each node, walk its whole same-name group and
* pairwise-compare file paths — is O(k^2) per node and therefore O(k^3) per
* same-name group of size k. That is not a theoretical concern: on a large
* Java corpus a single name group (`toString`, k≈4.6k) costs ~2 minutes, and
* a handful of such names alone multiplies total index time several-fold.
*
* This implementation is linear in the graph instead:
* - the distinct-file count is computed ONCE PER DISTINCT NAME and memoized
* (g_imp_name_counts), so a group of size k is scanned once, not k times;
* - within a group, file paths are deduplicated through a hash set rather
* than pairwise strcmp.
* Total distinct-file work is therefore Σ over distinct names of k = O(N).
* Both gbuf lookups it rests on (find_by_name, find_edges_by_target_type) are
* hash-indexed, so the per-node loop is O(1) amortised per node.
*
* g_importance_name_visits counts same-name-group member visits and is GATED
* in the complexity suite: it must stay linear (ratio ~2) under a corpus
* doubling. A regression to the per-node scan lands at ~4 and fails the gate.
*
* ── Ordering (load-bearing) ──────────────────────────────────────────
* Registered LAST among the predump passes, and last in the incremental
* post-pass sequence, so every edge type the score reads (CALLS, USAGE from
* extraction; TESTS from pass_tests) already exists. A pass ordered earlier
* would silently score everything at num_refs = 0 with no test penalty.
*/
#include "foundation/constants.h"
#include "pipeline/pipeline.h"
#include "pipeline/pipeline_internal.h"
#include "graph_buffer/graph_buffer.h"
#include "foundation/hash_table.h"
#include "foundation/log.h"
#include "foundation/compat.h"
#include "store/store.h"
#include "cbm.h"
#include "sqlite3.h"
#include <ctype.h>
#include <math.h>
#include <stdatomic.h>
#include <stdint.h>
#include <stdlib.h>
#include <string.h>
#include <stdio.h>
#include <stdbool.h>
/* Work counters — deltas are read by the complexity suite's linearity gate.
* Never reset here: callers snapshot before/after so nested runs compose. */
_Atomic uint64_t g_importance_nodes = 0;
_Atomic uint64_t g_importance_name_visits = 0;
_Atomic uint64_t g_importance_store_rows = 0;
enum {
CBM_IMPORTANCE_GENERIC_MIN_FILES = 5, /* name defined in >= N files -> generic */
CBM_IMPORTANCE_DISTINCT_MIN_LEN = 8, /* distinctive-identifier length floor */
};
static const double CBM_IMPORTANCE_PRIV_MUL = 0.1;
static const double CBM_IMPORTANCE_GENERIC_MUL = 0.1;
static const double CBM_IMPORTANCE_DISTINCT_MUL = 10.0;
static const double CBM_IMPORTANCE_TEST_MUL = 0.1;
/* The node labels that carry an importance score. File/Module and other
* non-symbol nodes are deliberately excluded. */
static const char *const CBM_IMPORTANCE_LABELS[] = {"Function", "Method", "Class"};
enum { CBM_IMPORTANCE_LABEL_COUNT = 3 };
/* Int → string for structured logging (thread-safe ring buffer). */
static const char *itoa_imp(uint64_t val) {
enum { RING = 2, MASK = 1 };
static CBM_TLS char bufs[RING][CBM_SZ_32];
static CBM_TLS int idx = 0;
int i = idx;
idx = (idx + 1) & MASK;
snprintf(bufs[i], sizeof(bufs[i]), "%llu", (unsigned long long)val);
return bufs[i];
}
static bool name_is_private(const char *name) {
return name != NULL && name[0] == '_';
}
/* Distinctive = (snake_case OR camelCase) AND length >= 8. snake_case is an
* embedded '_'; camelCase is a lower->upper "hump". A plain len>=8 lowercase
* word (no '_', no hump) is NOT distinctive. */
static bool name_is_distinctive(const char *name) {
if (!name) {
return false;
}
size_t len = strlen(name);
if (len < (size_t)CBM_IMPORTANCE_DISTINCT_MIN_LEN) {
return false;
}
if (strchr(name, '_') != NULL) {
return true; /* snake_case */
}
for (size_t i = 1; i < len; i++) {
if (islower((unsigned char)name[i - 1]) && isupper((unsigned char)name[i])) {
return true; /* camelCase hump */
}
}
return false;
}
/* ── JSON property write-back ─────────────────────────────────────────
*
* properties_json is a flat object here (same assumption the sibling passes'
* json_get_int/json_get_bool helpers make), but the value scanner below is
* written to survive nested values so a future richer blob cannot be
* truncated by an in-place overwrite. */
/* Return a pointer just past the JSON value beginning at p (first non-space
* char of the value). NULL if the value is unterminated/malformed. */
static const char *imp_value_end(const char *p) {
if (*p == '"') {
p++;
while (*p) {
if (*p == '\\' && p[1]) {
p += 2;
continue;
}
if (*p == '"') {
return p + 1;
}
p++;
}
return NULL;
}
if (*p == '{' || *p == '[') {
int depth = 0;
bool in_str = false;
while (*p) {
if (in_str) {
if (*p == '\\' && p[1]) {
p += 2;
continue;
}
if (*p == '"') {
in_str = false;
}
} else if (*p == '"') {
in_str = true;
} else if (*p == '{' || *p == '[') {
depth++;
} else if (*p == '}' || *p == ']') {
depth--;
if (depth == 0) {
return p + 1;
}
}
p++;
}
return NULL;
}
while (*p && *p != ',' && *p != '}' && *p != ']') {
p++;
}
return p;
}
/* Find `"key"` where it is genuinely in KEY position: preceded (modulo
* whitespace) by '{' or ',' and followed (modulo whitespace) by ':'. A bare
* strstr would also match the text inside a string VALUE. Returns a pointer
* to the opening quote, or NULL. */
static const char *imp_find_key(const char *json, const char *key) {
char pat[CBM_SZ_64];
int n = snprintf(pat, sizeof(pat), "\"%s\"", key);
if (n < 0 || (size_t)n >= sizeof(pat)) {
return NULL;
}
const char *p = json;
while ((p = strstr(p, pat)) != NULL) {
const char *before = p;
while (before > json && isspace((unsigned char)before[-1])) {
before--;
}
char prev = (before > json) ? before[-1] : '\0';
const char *after = p + n;
while (isspace((unsigned char)*after)) {
after++;
}
if ((prev == '{' || prev == ',') && *after == ':') {
return p;
}
p += n;
}
return NULL;
}
/* Produce a copy of `json` carrying the numeric "importance" key set to
* `score`, or NULL if `json` is not a JSON object (caller leaves it alone) or
* on allocation failure. Caller owns the result.
*
* IDEMPOTENT BY CONTRACT. Nodes reach this function already carrying an
* "importance" key on every re-scoring route: the legacy-partial incremental
* path rehydrates them via cbm_gbuf_load_from_db, and the store-level
* recompute reads them straight out of SQLite. A pure append would emit
* {"importance":1.0,...,"importance":2.0} — silent property corruption that no
* build or schema check would catch. So: overwrite an existing key in place,
* append only when the key is genuinely absent.
*
* SINGLE DEFINITION. Both scoring routes (the in-memory gbuf pass and the
* SQL-level store recompute) write through this one function, so the JSON
* shape cannot drift between them. */
char *cbm_pipeline_importance_set_prop(const char *json, double score) {
const char *old = json ? json : "{}";
size_t olen = strlen(old);
if (olen < 2 || old[olen - 1] != '}') {
return NULL; /* not a JSON object — leave untouched */
}
char val[CBM_SZ_32];
int vn = snprintf(val, sizeof(val), "%.6f", score);
if (vn < 0 || (size_t)vn >= sizeof(val)) {
return NULL;
}
const char *k = imp_find_key(old, "importance");
if (k) {
const char *colon = strchr(k, ':');
if (!colon) {
return NULL; /* imp_find_key guarantees one, but never trust a raw scan */
}
const char *v = colon + 1;
while (isspace((unsigned char)*v)) {
v++;
}
const char *vend = imp_value_end(v);
if (!vend) {
return NULL;
}
size_t head = (size_t)(v - old);
size_t tail = strlen(vend);
char *neu = malloc(head + (size_t)vn + tail + 1);
if (!neu) {
return NULL;
}
memcpy(neu, old, head);
memcpy(neu + head, val, (size_t)vn);
memcpy(neu + head + (size_t)vn, vend, tail + 1);
return neu;
}
char frag[CBM_SZ_64];
int fn = snprintf(frag, sizeof(frag), "%s\"importance\":%s}", (olen == 2) ? "" : ",", val);
if (fn < 0 || (size_t)fn >= sizeof(frag)) {
return NULL;
}
char *neu = malloc(olen - 1 + (size_t)fn + 1);
if (!neu) {
return NULL;
}
memcpy(neu, old, olen - 1); /* copy without the trailing '}' */
memcpy(neu + olen - 1, frag, (size_t)fn + 1);
return neu;
}
/* gbuf-node convenience wrapper around cbm_pipeline_importance_set_prop. */
void cbm_pipeline_importance_append_prop(cbm_gbuf_node_t *node, double score) {
if (!node) {
return;
}
char *neu = cbm_pipeline_importance_set_prop(node->properties_json, score);
if (!neu) {
return;
}
free(node->properties_json);
node->properties_json = neu;
}
/* ── The scoring rule ─────────────────────────────────────────────────
*
* SINGLE DEFINITION, deliberately. There are two routes that score nodes —
* the in-memory gbuf pass (full index, legacy-partial incremental) and the
* SQL-level recompute over the staging store (closure-delta incremental) —
* and they gather their inputs very differently. Only the GATHERING differs;
* the rule itself lives here alone, so the two routes cannot drift apart in
* what a score means. The equivalence test in tests/test_importance.c guards
* the remaining freedom: that both gatherings produce the same inputs for the
* same graph. */
double cbm_pipeline_importance_score(const cbm_importance_inputs_t *in) {
if (!in) {
return 0.0;
}
double score = sqrt((double)(in->num_refs > 0 ? in->num_refs : 0));
if (name_is_private(in->name)) {
score *= CBM_IMPORTANCE_PRIV_MUL;
}
if (in->name_distinct_files >= CBM_IMPORTANCE_GENERIC_MIN_FILES) {
score *= CBM_IMPORTANCE_GENERIC_MUL;
}
if (name_is_distinctive(in->name)) {
score *= CBM_IMPORTANCE_DISTINCT_MUL;
}
if (cbm_is_test_path(in->file_path) || in->tests_target) {
score *= CBM_IMPORTANCE_TEST_MUL;
}
return score;
}
/* ── Distinct-file counting (the linear core) ─────────────────────────── */
typedef struct {
CBMHashTable *counts; /* name -> (void *)(intptr_t)(distinct_files + 1) */
CBMHashTable *paths; /* scratch file-path set, cleared between names */
} imp_name_index_t;
/* Distinct files a name is DEFINED in. Counts distinct file paths, not
* distinct nodes — two defs of one name in the same file count once.
* Computed once per distinct name; every later node in the group is a hash
* hit. Name keys are borrowed from the gbuf nodes, which outlive the pass. */
static int imp_distinct_file_count(const cbm_gbuf_t *gb, const char *name, imp_name_index_t *ix) {
if (!name || !*name || !ix || !ix->counts) {
return 0;
}
void *cached = cbm_ht_get(ix->counts, name);
if (cached) {
return (int)((intptr_t)cached - 1);
}
const cbm_gbuf_node_t **nodes = NULL;
int count = 0;
int distinct = 0;
if (cbm_gbuf_find_by_name(gb, name, &nodes, &count) == 0 && count > 0) {
atomic_fetch_add_explicit(&g_importance_name_visits, (uint64_t)count, memory_order_relaxed);
if (ix->paths) {
cbm_ht_clear(ix->paths);
for (int i = 0; i < count; i++) {
const char *fp = nodes[i]->file_path;
if (!fp || cbm_ht_has(ix->paths, fp)) {
continue;
}
cbm_ht_set(ix->paths, fp, (void *)(intptr_t)1);
distinct++;
}
} else {
/* Scratch set unavailable (allocation failure): the generic-name
* multiplier degrades to "never generic" rather than falling back
* to a pairwise scan — a silent quadratic is worse than a missing
* 0.1 multiplier. */
distinct = 0;
}
}
cbm_ht_set(ix->counts, name, (void *)(intptr_t)(distinct + 1));
return distinct;
}
static int incoming_edge_count(const cbm_gbuf_t *gb, int64_t id, const char *type) {
const cbm_gbuf_edge_t **edges = NULL;
int ne = 0;
if (cbm_gbuf_find_edges_by_target_type(gb, id, type, &edges, &ne) != 0) {
return 0;
}
return ne;
}
void cbm_pipeline_pass_importance(cbm_pipeline_ctx_t *ctx) {
if (!ctx || !ctx->gbuf) {
return;
}
cbm_gbuf_t *gb = ctx->gbuf;
/* Modest reservations, deliberately not sized from node_count: the memo
* holds one entry per DISTINCT name, far fewer than nodes, and reserving
* buckets for every node would cost hundreds of MB of transient peak on a
* multi-million-node graph to save a handful of amortised rehashes. */
imp_name_index_t ix = {
.counts = cbm_ht_create(CBM_SZ_4K),
.paths = cbm_ht_create(CBM_SZ_256),
};
if (!ix.counts) {
cbm_ht_free(ix.paths);
cbm_log_error("pass.importance", "msg", "name_index_alloc_failed");
return;
}
uint64_t updated = 0;
for (int li = 0; li < CBM_IMPORTANCE_LABEL_COUNT; li++) {
const cbm_gbuf_node_t **nodes = NULL;
int count = 0;
if (cbm_gbuf_find_by_label(gb, CBM_IMPORTANCE_LABELS[li], &nodes, &count) != 0) {
continue;
}
for (int i = 0; i < count; i++) {
cbm_gbuf_node_t *n = (cbm_gbuf_node_t *)nodes[i];
/* Gather only — the rule itself lives in one place. */
cbm_importance_inputs_t in = {
.name = n->name,
.file_path = n->file_path,
.num_refs = incoming_edge_count(gb, n->id, "CALLS") +
incoming_edge_count(gb, n->id, "USAGE"),
.name_distinct_files = imp_distinct_file_count(gb, n->name, &ix),
.tests_target = incoming_edge_count(gb, n->id, "TESTS") > 0,
};
cbm_pipeline_importance_append_prop(n, cbm_pipeline_importance_score(&in));
updated++;
}
}
cbm_ht_free(ix.paths);
cbm_ht_free(ix.counts);
atomic_fetch_add_explicit(&g_importance_nodes, updated, memory_order_relaxed);
cbm_log_info("pass.importance", "symbols", itoa_imp(updated));
}
/* ── Store-level recompute (closure-delta incremental) ────────────────
*
* WHY THIS EXISTS. The closure-delta route never materialises the project
* graph in RAM: cbm_delta_preseed fills the delta gbuf with *proxy* nodes
* (id, label, name, qn, file_path — no edges, empty properties), and only the
* re-extracted files' symbols carry real edges. Scoring there reads a
* near-zero in-degree for a symbol referenced hundreds of times project-wide,
* and cbm_delta_patch then persists exactly those fresh rows. Loading the full
* graph to fix that would give back the point of the delta route, so the
* recompute runs where the complete graph already exists: in SQL, over the
* staging store, AFTER cbm_delta_patch has inserted the new nodes and
* re-linked the snapshotted inbound edges.
*
* WHY IT CANNOT DRIFT FROM THE IN-MEMORY PASS. The scoring rule and the JSON
* write-back are not reimplemented here. They are registered as SQLite
* user-defined functions that call cbm_pipeline_importance_score() and
* cbm_pipeline_importance_set_prop() — the same C code the gbuf pass runs.
* SQL supplies only the three aggregates (in-degree, TESTS presence, distinct
* files per name), each one indexed GROUP BY. What remains free is the
* GATHERING, and the full-vs-incremental equivalence test guards that.
*
* The whole project is rescored, not just the changed files: distinct-files-
* per-name is a global input, so editing one file can legitimately move a name
* across the generic-name threshold for every other definition of that name. A
* scoped recompute would leave those stale — and would fail the equivalence
* test, correctly. */
/* UDF: cbm_imp_score(name, file_path, num_refs, name_files, tests_target). */
static void sql_importance_score(sqlite3_context *c, int argc, sqlite3_value **argv) {
if (argc != 5) {
sqlite3_result_error(c, "cbm_imp_score arity", -1);
return;
}
cbm_importance_inputs_t in = {
.name = (const char *)sqlite3_value_text(argv[0]),
.file_path = (const char *)sqlite3_value_text(argv[1]),
.num_refs = sqlite3_value_int(argv[2]),
.name_distinct_files = sqlite3_value_int(argv[3]),
.tests_target = sqlite3_value_int(argv[4]) != 0,
};
sqlite3_result_double(c, cbm_pipeline_importance_score(&in));
}
/* UDF: cbm_imp_set(properties, score) → properties with "importance" set.
* A non-object blob yields NULL; the UPDATE's COALESCE then leaves that row
* untouched — the same tolerance the in-memory writer has. */
static void sql_importance_set(sqlite3_context *c, int argc, sqlite3_value **argv) {
if (argc != 2) {
sqlite3_result_error(c, "cbm_imp_set arity", -1);
return;
}
const char *props = (const char *)sqlite3_value_text(argv[0]);
char *neu = cbm_pipeline_importance_set_prop(props, sqlite3_value_double(argv[1]));
if (!neu) {
sqlite3_result_null(c);
return;
}
sqlite3_result_text(c, neu, -1, SQLITE_TRANSIENT);
free(neu);
}
/* The aggregate CTE is MATERIALIZED on purpose: it reads `nodes` while the
* UPDATE writes `nodes`, and materialising settles that ordering explicitly
* rather than relying on the planner to choose it. */
static const char *const kImportanceRecomputeSQL =
"WITH agg AS MATERIALIZED ("
" SELECT n.id AS id,"
" cbm_imp_score(n.name, n.file_path, COALESCE(d.refs, 0),"
" COALESCE(f.files, 0), COALESCE(d.tests, 0)) AS score"
" FROM nodes n"
" LEFT JOIN (SELECT target_id AS id,"
" SUM(CASE WHEN type IN ('CALLS','USAGE') THEN 1 ELSE 0 END) AS refs,"
" SUM(CASE WHEN type = 'TESTS' THEN 1 ELSE 0 END) AS tests"
" FROM edges WHERE project = ?1 GROUP BY target_id) d ON d.id = n.id"
" LEFT JOIN (SELECT name, COUNT(DISTINCT file_path) AS files"
" FROM nodes WHERE project = ?1 GROUP BY name) f ON f.name = n.name"
" WHERE n.project = ?1 AND n.label IN ('Function','Method','Class')"
")"
"UPDATE nodes SET properties = COALESCE(cbm_imp_set(properties, agg.score), properties)"
" FROM agg WHERE nodes.id = agg.id";
int cbm_pipeline_importance_recompute_store(cbm_store_t *store, const char *project) {
if (!store || !project) {
return -1;
}
sqlite3 *db = cbm_store_get_db(store);
if (!db) {
return -1;
}
if (sqlite3_create_function(db, "cbm_imp_score", 5, SQLITE_UTF8 | SQLITE_DETERMINISTIC, NULL,
sql_importance_score, NULL, NULL) != SQLITE_OK ||
sqlite3_create_function(db, "cbm_imp_set", 2, SQLITE_UTF8 | SQLITE_DETERMINISTIC, NULL,
sql_importance_set, NULL, NULL) != SQLITE_OK) {
cbm_log_error("importance.store_udf_failed", "err", sqlite3_errmsg(db));
return -1;
}
if (cbm_store_begin(store) != CBM_STORE_OK) {
return -1;
}
sqlite3_stmt *st = NULL;
if (sqlite3_prepare_v2(db, kImportanceRecomputeSQL, -1, &st, NULL) != SQLITE_OK) {
cbm_log_error("importance.store_prepare_failed", "err", sqlite3_errmsg(db));
cbm_store_rollback(store);
return -1;
}
sqlite3_bind_text(st, 1, project, -1, SQLITE_TRANSIENT);
int step_rc = sqlite3_step(st);
sqlite3_finalize(st);
if (step_rc != SQLITE_DONE) {
cbm_log_error("importance.store_step_failed", "err", sqlite3_errmsg(db));
cbm_store_rollback(store);
return -1;
}
int changed = sqlite3_changes(db);
if (cbm_store_commit(store) != CBM_STORE_OK) {
return -1;
}
uint64_t rows = (uint64_t)(changed > 0 ? changed : 0);
atomic_fetch_add_explicit(&g_importance_store_rows, rows, memory_order_relaxed);
cbm_log_info("pass.importance_store", "symbols", itoa_imp(rows));
return 0;
}
src/pipeline/pipeline.c
+18 -4
View File
@@ -946,6 +946,9 @@ static void predump_complexity(cbm_pipeline_ctx_t *ctx) {
static void predump_ensemble(cbm_pipeline_ctx_t *ctx) {
cbm_pipeline_pass_ensemble_routing(ctx);
}
static void predump_importance(cbm_pipeline_ctx_t *ctx) {
cbm_pipeline_pass_importance(ctx);
}
static void run_predump_passes(cbm_pipeline_t *p, cbm_pipeline_ctx_t *ctx) {
static const struct {
@@ -953,12 +956,23 @@ static void run_predump_passes(cbm_pipeline_t *p, cbm_pipeline_ctx_t *ctx) {
const char *name;
bool moderate_only; /* true = skip in fast mode */
} passes[] = {
{predump_deco, "decorator_tags", false}, {predump_cfg, "configlink", false},
{predump_route, "route_match", false}, {predump_ensemble, "ensemble_routing", false},
{predump_sim, "similarity", true}, {predump_sem, "semantic_edges", true},
{predump_deco, "decorator_tags", false},
{predump_cfg, "configlink", false},
{predump_route, "route_match", false},
{predump_ensemble, "ensemble_routing", false},
{predump_sim, "similarity", true},
{predump_sem, "semantic_edges", true},
{predump_complexity, "complexity", false},
/* Importance runs LAST: it reads CALLS/USAGE (extraction) and TESTS
* (pass_tests, which run_post_extraction runs before this loop), so
* every edge type its score depends on already exists here. */
{predump_importance, "importance", false},
};
enum { PREDUMP_PASS_COUNT = 7 };
/* Derived from the table, never hand-written. A hand-written count that
* lags a newly appended entry silently skips the LAST pass while every
* test stays green — exactly the failure this expression makes
* impossible. */
enum { PREDUMP_PASS_COUNT = (int)(sizeof(passes) / sizeof(passes[0])) };
struct timespec t;
for (int i = 0; i < PREDUMP_PASS_COUNT && !check_cancel(p); i++) {
/* "moderate_only" passes (similarity/semantic edges) run in FULL,
src/pipeline/pipeline_incremental.c
+47 -4
View File
@@ -1410,9 +1410,15 @@ static int run_extract_resolve(cbm_pipeline_ctx_t *ctx, cbm_file_info_t *changed
}
}
/* Run post-extraction passes (tests, decorator tags, configlink). */
/* Run post-extraction passes (tests, decorator tags, configlink).
*
* score_importance: false on the closure-delta route, whose ctx->gbuf is a
* PROXY buffer — cbm_delta_preseed fills it with id/label/name/qn/file_path
* and no project-wide edges, so an in-memory score there would read a
* near-zero in-degree and cbm_delta_patch would persist it. That route scores
* in SQL instead, after the patch; see cbm_pipeline_importance_recompute_store. */
static int run_postpasses(cbm_pipeline_ctx_t *ctx, cbm_file_info_t *changed_files, int ci,
const char *project) {
const char *project, bool score_importance) {
struct timespec t;
cbm_clock_gettime(CLOCK_MONOTONIC, &t);
@@ -1456,6 +1462,26 @@ static int run_postpasses(cbm_pipeline_ctx_t *ctx, cbm_file_info_t *changed_file
return rc < 0 ? rc : CBM_NOT_FOUND;
}
}
/* Importance last, in EVERY mode — same ordering rule as the full path: it
* reads CALLS/USAGE (re-extraction, above) and TESTS (pass_tests, first in
* this function).
*
* ONLY on the legacy-partial route, where ctx->gbuf really is the whole
* project rehydrated by cbm_gbuf_load_from_db — those nodes already carry
* an "importance" key from the previous run, and the write-back overwrites
* it in place rather than appending a duplicate. The closure-delta route
* passes score_importance = false and rescores in SQL after its patch,
* because its buffer holds proxies without project-wide edges. */
if (score_importance) {
cbm_clock_gettime(CLOCK_MONOTONIC, &t);
cbm_pipeline_pass_importance(ctx);
cbm_log_info("pass.timing", "pass", "incr_importance", "elapsed_ms",
itoa_buf((int)elapsed_ms(t)));
}
if (cbm_pipeline_check_cancel(ctx)) {
return CBM_NOT_FOUND;
}
return 0;
}
/* Publish the test-only legacy partial result through the same atomic
@@ -2144,7 +2170,7 @@ static int run_closure_delta(cbm_pipeline_t *p, const char *db_path, const char
phase_rc = cbm_pipeline_check_cancel(&ctx);
}
if (phase_rc == 0) {
phase_rc = run_postpasses(&ctx, changed_files, ci, project);
phase_rc = run_postpasses(&ctx, changed_files, ci, project, false);
}
if (ctx.return_type_table) {
for (int i = 0; i < ctx.return_type_table->count; i++) {
@@ -2169,6 +2195,23 @@ static int run_closure_delta(cbm_pipeline_t *p, const char *db_path, const char
}
cbm_log_info("delta.patch_done", "elapsed_ms", itoa_buf((int)elapsed_ms(t)));
/* Importance, in SQL, over the now-complete staging graph. This is the
* FIRST point on this route where the whole project is queryable: the
* patch has just inserted the repaired files' nodes AND re-linked the
* snapshotted inbound edges, so in-degree is finally correct. Scoring
* earlier — in run_postpasses, off the proxy buffer — would persist a
* near-zero score for any heavily-referenced symbol in a changed file.
* A failure here is a delta-route failure: `result` still holds
* FORCE_FULL_REINDEX, so the run degrades to a correct full rebuild
* rather than publishing wrong scores. */
cbm_clock_gettime(CLOCK_MONOTONIC, &t);
if (cbm_pipeline_importance_recompute_store(staging, project) != 0) {
cbm_log_error("delta.err", "phase", "importance_recompute");
goto out;
}
cbm_log_info("pass.timing", "pass", "delta_importance_store", "elapsed_ms",
itoa_buf((int)elapsed_ms(t)));
/* Coverage: previous failure rows for files not re-extracted + this
* run's fresh entries — same merge the legacy tail performs. */
cbm_file_error_t *run_errs = NULL;
@@ -2699,7 +2742,7 @@ int cbm_pipeline_run_incremental(cbm_pipeline_t *p, const char *db_path, cbm_fil
phase_rc = cbm_pipeline_check_cancel(&ctx);
}
if (phase_rc == 0) {
phase_rc = run_postpasses(&ctx, changed_files, ci, project);
phase_rc = run_postpasses(&ctx, changed_files, ci, project, true);
}
/* Free ObjectScript tables built by pass_calls during run_extract_resolve. */
src/pipeline/pipeline_internal.h
+55
View File
@@ -630,6 +630,61 @@ int cbm_pipeline_pass_semantic_edges(cbm_pipeline_ctx_t *ctx);
* cycles (recursive). Runs on the graph buffer before the dump. */
void cbm_pipeline_pass_complexity(cbm_pipeline_ctx_t *ctx);
/* Pre-dump pass: per-symbol importance score (weighted degree).
* importance = sqrt(num_refs) * priv * generic * distinct * test_penalty
* Stored as a numeric "importance" key inside the node's EXISTING
* properties_json — no schema change and no index-format bump; indexes written
* by older builds simply lack the key and consumers must tolerate its absence.
* MUST run after pass_tests and after CALLS/USAGE extraction — a pass ordered
* earlier would see zero TESTS edges and num_refs = 0 everywhere. It is
* therefore registered last in run_predump_passes and last in the incremental
* post-pass sequence. */
void cbm_pipeline_pass_importance(cbm_pipeline_ctx_t *ctx);
/* Gathered inputs for one symbol. Each scoring route fills this its own way
* (gbuf lookups, or SQL aggregates) and then calls the ONE rule below. */
typedef struct {
const char *name;
const char *file_path;
int num_refs; /* incoming CALLS + USAGE */
int name_distinct_files; /* distinct files the NAME is defined in */
bool tests_target; /* has an incoming TESTS edge */
} cbm_importance_inputs_t;
/* THE scoring rule — single definition, shared by every route, so the two
* gathering strategies cannot drift apart in what a score means. */
double cbm_pipeline_importance_score(const cbm_importance_inputs_t *in);
/* Produce a copy of `json` with the numeric "importance" key set to `score`,
* or NULL when `json` is not a JSON object or allocation fails. Caller owns
* the result. IDEMPOTENT: an existing key is overwritten in place, never
* appended twice — every re-scoring route sees nodes that already carry it.
* Single definition, shared by the in-memory and SQL writers alike. */
char *cbm_pipeline_importance_set_prop(const char *json, double score);
/* gbuf-node convenience wrapper around cbm_pipeline_importance_set_prop. */
void cbm_pipeline_importance_append_prop(cbm_gbuf_node_t *node, double score);
/* Recompute importance for an ENTIRE project directly in a store, in SQL.
* Used by the closure-delta incremental route, whose in-RAM graph is a proxy
* buffer with no project-wide edges — scoring there would persist a near-zero
* in-degree for heavily-referenced symbols in the changed files. Must be
* called on the STAGING store AFTER cbm_delta_patch has merged nodes and
* re-linked inbound edges. Returns 0 on success; the caller treats failure as
* a delta-route failure and falls back to a full rebuild. */
int cbm_pipeline_importance_recompute_store(cbm_store_t *store, const char *project);
/* Work counters for the importance pass (pass_importance.c). Deltas are read
* by the complexity suite's linearity gate; never reset by the pass itself, so
* nested/repeated runs compose. g_importance_name_visits counts same-name-group
* member visits during distinct-file counting — the quantity that goes
* superlinear if the per-distinct-name memoization is ever lost. */
extern _Atomic uint64_t g_importance_nodes;
extern _Atomic uint64_t g_importance_name_visits;
/* Rows rescored by the SQL-level store recompute. Lets a test prove the
* closure-delta route actually took that path instead of passing vacuously. */
extern _Atomic uint64_t g_importance_store_rows;
/* ── Env URL scanner (pass_envscan.c) ────────────────────────────── */
typedef struct {
tests/test_complexity.c
+44
View File
@@ -353,6 +353,8 @@ typedef struct {
uint64_t tail_lookups;
uint64_t tail_candidates;
uint64_t fallback_rows;
uint64_t imp_nodes; /* symbols scored by the importance pass */
uint64_t imp_name_visits; /* same-name-group members visited by that pass */
double wall_s;
} CxMetrics;
@@ -373,6 +375,8 @@ static int cx_run(const char *root, const char *db_path, CxMetrics *out) {
uint64_t tl0 = atomic_load_explicit(&g_lsp_tail_lookups, memory_order_relaxed);
uint64_t tc0 = atomic_load_explicit(&g_lsp_tail_candidates, memory_order_relaxed);
uint64_t fb0 = cbm_pp_lsp_linear_fallback_rows();
uint64_t in0 = atomic_load_explicit(&g_importance_nodes, memory_order_relaxed);
uint64_t iv0 = atomic_load_explicit(&g_importance_name_visits, memory_order_relaxed);
atomic_store_explicit(&g_cx_pass_count, 0, memory_order_relaxed);
cbm_log_set_sink_ex(cx_pass_sink, CBM_LOG_SINK_TEE);
@@ -407,6 +411,9 @@ static int cx_run(const char *root, const char *db_path, CxMetrics *out) {
out->tail_lookups = atomic_load_explicit(&g_lsp_tail_lookups, memory_order_relaxed) - tl0;
out->tail_candidates = atomic_load_explicit(&g_lsp_tail_candidates, memory_order_relaxed) - tc0;
out->fallback_rows = cbm_pp_lsp_linear_fallback_rows() - fb0;
out->imp_nodes = atomic_load_explicit(&g_importance_nodes, memory_order_relaxed) - in0;
out->imp_name_visits =
atomic_load_explicit(&g_importance_name_visits, memory_order_relaxed) - iv0;
cbm_store_t *s = cbm_store_open_path(db_path);
if (!s) {
@@ -717,9 +724,46 @@ TEST(complexity_throughput_report_written) {
PASS();
}
/* Importance scoring must stay linear in the graph. Its generic-name
* multiplier needs |{files a name is defined in}|; computing that per NODE
* instead of once per distinct NAME makes a same-name group of size k cost
* O(k^3) — measured on a large Java corpus as a multi-minute cost for a
* handful of names. Under replication the same-name groups grow with the copy
* count, so the per-node shape lands at ~4x per doubling here while the
* memoized shape stays at ~2x. Gated on the counter ratio, never on wall time
* (O9). This counter is GATED, not merely recorded: the whole point of the
* work is that this quantity must not go superlinear. */
TEST(complexity_importance_scoring_is_linear) {
if (cx_measure_pair() != 0) {
FAIL("failed to build/run the complexity corpus pair");
}
const CxMetrics *a = &g_cx_base;
const CxMetrics *b = &g_cx_doubled;
printf(" imp_nodes %llu -> %llu imp_name_visits %llu -> %llu\n",
(unsigned long long)a->imp_nodes, (unsigned long long)b->imp_nodes,
(unsigned long long)a->imp_name_visits, (unsigned long long)b->imp_name_visits);
/* Non-vacuous, twice over: the pass must have run at all (a miscounted
* PREDUMP_PASS_COUNT would silently skip it and zero both counters), and
* the base leg must have produced enough work for a ratio to mean
* anything. A zero here is a wiring defect to fix, never a pass. */
ASSERT_GT((long long)a->imp_nodes, 0);
ASSERT_GT((long long)a->imp_name_visits, (long long)CX_MIN_BASE_WORK);
double node_r = cx_ratio((double)b->imp_nodes, (double)a->imp_nodes);
double visit_r = cx_ratio((double)b->imp_name_visits, (double)a->imp_name_visits);
printf(" imp_nodes ratio %.2f imp_name_visits ratio %.2f (linear ~2, per-node ~4)\n",
node_r, visit_r);
ASSERT_TRUE(node_r >= CX_RATIO_LO && node_r <= CX_RATIO_HI);
ASSERT_TRUE(visit_r <= CX_RATIO_HI);
PASS();
}
SUITE(complexity) {
RUN_TEST(complexity_replicated_modules_scale_linearly);
RUN_TEST(complexity_perfile_registry_work_is_linear);
RUN_TEST(complexity_importance_scoring_is_linear);
RUN_TEST(complexity_shared_package_growth_stays_linear);
RUN_TEST(complexity_throughput_report_written);
}
tests/test_importance.c
+911
View File
@@ -0,0 +1,911 @@
/*
* test_importance.c — index-time per-symbol importance score (pass_importance.c).
*
* Formula under test:
* importance = sqrt(num_refs) * priv * generic * distinct * test_penalty
*
* Three of these tests exist to catch failure modes that are SILENT — they
* produce a wrong index with a fully green build:
*
* - importance_pass_actually_runs_in_full_pipeline
* binds the predump registration. A hand-written PREDUMP_PASS_COUNT that
* lags the pass table skips the LAST-registered pass and nothing else
* notices. This test fails loudly if the score never lands.
* - importance_rehydrated_node_keeps_exactly_one_key
* - importance_second_index_run_keeps_exactly_one_key
* bind the idempotent write-back. The incremental path rehydrates nodes
* that already carry "importance"; a plain append yields
* {"importance":1.0,...,"importance":2.0} — property corruption, not a
* build failure.
* - importance_distinct_file_count_is_linear_in_group_size
* binds the cost shape: the distinct-file count must be computed once
* per distinct NAME, not once per node. The per-node shape is O(k^3) in
* a same-name group and is measurably disqualifying on real corpora.
*/
#include "test_framework.h"
#include "test_helpers.h"
#include "pipeline/pipeline.h"
#include "pipeline/pipeline_internal.h"
#include "graph_buffer/graph_buffer.h"
#include "store/store.h"
#include "foundation/compat.h"
#include "foundation/log.h"
#include <math.h>
#include <stdatomic.h>
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
/* ── Shared helpers ───────────────────────────────────────────────────── */
/* Parse the numeric value of "importance" out of a properties_json blob.
* Returns a sentinel (-999.0) when the key is absent — callers that care about
* presence assert on imp_key_count() instead. */
static double imp_value(const char *json) {
if (!json) {
return -999.0;
}
const char *p = strstr(json, "\"importance\":");
if (!p) {
return -999.0;
}
return strtod(p + strlen("\"importance\":"), NULL);
}
/* How many times the key appears. The duplicate-key corruption this suite
* guards against shows up here as 2. */
static int imp_key_count(const char *json) {
if (!json) {
return 0;
}
int n = 0;
const char *p = json;
while ((p = strstr(p, "\"importance\":")) != NULL) {
n++;
p += strlen("\"importance\":");
}
return n;
}
static void imp_run_pass(cbm_gbuf_t *gb) {
atomic_int cancelled = 0;
cbm_pipeline_ctx_t ctx = {
.project_name = "test-proj",
.repo_path = "/tmp/test",
.gbuf = gb,
.cancelled = &cancelled,
};
cbm_pipeline_pass_importance(&ctx);
}
/* Every stored node of the given label must carry exactly `expect` copies of
* the key. Returns the number of nodes checked, or -1 on a store error. */
static int imp_check_label(cbm_store_t *s, const char *project, const char *label, int expect) {
cbm_node_t *nodes = NULL;
int count = 0;
if (cbm_store_find_nodes_by_label(s, project, label, &nodes, &count) != CBM_STORE_OK) {
return -1;
}
int bad = 0;
for (int i = 0; i < count; i++) {
if (imp_key_count(nodes[i].properties_json) != expect) {
bad++;
printf(" %s node '%s' props=%s\n", label, nodes[i].name ? nodes[i].name : "?",
nodes[i].properties_json ? nodes[i].properties_json : "(null)");
}
}
cbm_store_free_nodes(nodes, count);
return bad == 0 ? count : -1;
}
/* ── Store-level score readout (for the closure-delta tests) ──────────── */
typedef struct {
char **qns;
double *scores;
int count;
} ImpScoreSet;
static void imp_scores_free(ImpScoreSet *set) {
for (int i = 0; i < set->count; i++) {
free(set->qns[i]);
}
free(set->qns);
free(set->scores);
set->qns = NULL;
set->scores = NULL;
set->count = 0;
}
/* Collect (qualified_name -> importance) for every scored label in a store. */
static int imp_collect_scores(const char *db_path, const char *project, ImpScoreSet *out) {
static const char *const kLabels[] = {"Function", "Method", "Class"};
memset(out, 0, sizeof(*out));
cbm_store_t *s = cbm_store_open_path(db_path);
if (!s) {
return -1;
}
int rc = 0;
for (int li = 0; li < 3; li++) {
cbm_node_t *nodes = NULL;
int n = 0;
if (cbm_store_find_nodes_by_label(s, project, kLabels[li], &nodes, &n) != CBM_STORE_OK) {
rc = -1;
break;
}
char **qns = realloc(out->qns, (size_t)(out->count + n) * sizeof(*qns));
double *sc = realloc(out->scores, (size_t)(out->count + n) * sizeof(*sc));
if (n > 0 && (!qns || !sc)) {
cbm_store_free_nodes(nodes, n);
rc = -1;
break;
}
if (qns) {
out->qns = qns;
}
if (sc) {
out->scores = sc;
}
for (int i = 0; i < n; i++) {
out->qns[out->count] = strdup(nodes[i].qualified_name ? nodes[i].qualified_name : "");
out->scores[out->count] = imp_value(nodes[i].properties_json);
out->count++;
}
cbm_store_free_nodes(nodes, n);
}
cbm_store_close(s);
return rc;
}
static double imp_score_of(const ImpScoreSet *set, const char *qn_suffix) {
for (int i = 0; i < set->count; i++) {
size_t l = strlen(set->qns[i]);
size_t k = strlen(qn_suffix);
if (l >= k && strcmp(set->qns[i] + (l - k), qn_suffix) == 0) {
return set->scores[i];
}
}
return -999.0;
}
/* Write a fixture whose `helper` in hot.py is called from `callers` other
* files — the inbound edges that only exist project-wide, never in a delta
* buffer. `tag` lets the caller perturb hot.py to force a re-parse. */
static int imp_write_hot_fixture(const char *root, int callers, const char *tag) {
char body[256];
snprintf(body, sizeof(body), "%sdef helper():\n return 1\n", tag);
if (th_write_file(TH_PATH(root, "hot.py"), body) != 0) {
return -1;
}
for (int i = 0; i < callers; i++) {
char rel[64];
char src[192];
snprintf(rel, sizeof(rel), "c%d.py", i);
snprintf(src, sizeof(src), "import hot\n\ndef use%d():\n return hot.helper()\n", i);
if (th_write_file(TH_PATH(root, rel), src) != 0) {
return -1;
}
}
return 0;
}
/* ── Registration: the pass must actually execute ─────────────────────── */
/* A miscounted PREDUMP_PASS_COUNT silently drops the last-registered pass.
* The score is the only observable, so assert on the score. */
TEST(importance_pass_actually_runs_in_full_pipeline) {
char *tmp = th_mktempdir("cbm_imp_run");
if (!tmp) {
FAIL("tmpdir");
}
char root[512];
snprintf(root, sizeof(root), "%s", tmp); /* th_mktempdir returns a static buffer */
ASSERT_EQ(th_write_file(TH_PATH(root, "main.py"),
"def helper():\n return 1\n\n"
"def caller():\n return helper() + helper()\n\n"
"class Widget:\n def render(self):\n return helper()\n"),
0);
char db_path[600];
snprintf(db_path, sizeof(db_path), "%s/graph.db", root);
cbm_pipeline_t *p = cbm_pipeline_new(root, db_path, CBM_MODE_FULL);
ASSERT_NOT_NULL(p);
ASSERT_EQ(cbm_pipeline_run(p), 0);
char project[256];
snprintf(project, sizeof(project), "%s", cbm_pipeline_project_name(p));
cbm_pipeline_free(p);
cbm_store_t *s = cbm_store_open_path(db_path);
ASSERT_NOT_NULL(s);
/* Non-vacuous: the corpus really produced symbols of each kind. */
int fc = imp_check_label(s, project, "Function", 1);
int mc = imp_check_label(s, project, "Method", 1);
int cc = imp_check_label(s, project, "Class", 1);
printf(" scored Function=%d Method=%d Class=%d\n", fc, mc, cc);
ASSERT_GT(fc, 0);
ASSERT_GT(mc, 0);
ASSERT_GT(cc, 0);
/* File nodes are deliberately excluded from scoring. */
cbm_node_t *files = NULL;
int filec = 0;
ASSERT_EQ(cbm_store_find_nodes_by_label(s, project, "File", &files, &filec), CBM_STORE_OK);
for (int i = 0; i < filec; i++) {
ASSERT_EQ(imp_key_count(files[i].properties_json), 0);
}
cbm_store_free_nodes(files, filec);
cbm_store_close(s);
th_rmtree(root);
PASS();
}
/* ── Formula ──────────────────────────────────────────────────────────── */
TEST(importance_base_is_sqrt_of_incoming_refs) {
cbm_gbuf_t *gb = cbm_gbuf_new("test-proj", "/tmp/test");
ASSERT_NOT_NULL(gb);
int64_t target =
cbm_gbuf_upsert_node(gb, "Function", "target", "pkg.target", "pkg/main.go", 1, 1, "{}");
int64_t c1 = cbm_gbuf_upsert_node(gb, "Function", "c1", "pkg.c1", "pkg/main.go", 2, 2, "{}");
int64_t c2 = cbm_gbuf_upsert_node(gb, "Function", "c2", "pkg.c2", "pkg/main.go", 3, 3, "{}");
int64_t c3 = cbm_gbuf_upsert_node(gb, "Function", "c3", "pkg.c3", "pkg/main.go", 4, 4, "{}");
int64_t lonely =
cbm_gbuf_upsert_node(gb, "Function", "lonely", "pkg.lonely", "pkg/main.go", 5, 5, "{}");
ASSERT_GT(target, 0);
ASSERT_GT(lonely, 0);
cbm_gbuf_insert_edge(gb, c1, target, "CALLS", "{}");
cbm_gbuf_insert_edge(gb, c2, target, "CALLS", "{}");
cbm_gbuf_insert_edge(gb, c3, target, "USAGE", "{}"); /* spans BOTH edge types */
imp_run_pass(gb);
const cbm_gbuf_node_t *tn = cbm_gbuf_find_by_id(gb, target);
ASSERT_NOT_NULL(tn);
ASSERT_EQ(imp_key_count(tn->properties_json), 1);
ASSERT_FLOAT_EQ(imp_value(tn->properties_json), sqrt(3.0), 1e-6);
/* num_refs == 0 -> sqrt(0) == 0, still written. */
const cbm_gbuf_node_t *ln = cbm_gbuf_find_by_id(gb, lonely);
ASSERT_NOT_NULL(ln);
ASSERT_EQ(imp_key_count(ln->properties_json), 1);
ASSERT_FLOAT_EQ(imp_value(ln->properties_json), 0.0, 1e-9);
cbm_gbuf_free(gb);
PASS();
}
TEST(importance_private_names_are_demoted) {
cbm_gbuf_t *gb = cbm_gbuf_new("test-proj", "/tmp/test");
ASSERT_NOT_NULL(gb);
int64_t priv =
cbm_gbuf_upsert_node(gb, "Function", "_run", "pkg._run", "pkg/main.go", 1, 1, "{}");
int64_t pub = cbm_gbuf_upsert_node(gb, "Function", "run", "pkg.run", "pkg/main.go", 2, 2, "{}");
for (int i = 0; i < 4; i++) {
char name[CBM_SZ_32];
char qn[CBM_SZ_64];
snprintf(name, sizeof(name), "p%d", i);
snprintf(qn, sizeof(qn), "pkg.p%d", i);
int64_t c =
cbm_gbuf_upsert_node(gb, "Function", name, qn, "pkg/main.go", 10 + i, 10 + i, "{}");
cbm_gbuf_insert_edge(gb, c, priv, "CALLS", "{}");
snprintf(name, sizeof(name), "q%d", i);
snprintf(qn, sizeof(qn), "pkg.q%d", i);
c = cbm_gbuf_upsert_node(gb, "Function", name, qn, "pkg/main.go", 20 + i, 20 + i, "{}");
cbm_gbuf_insert_edge(gb, c, pub, "CALLS", "{}");
}
imp_run_pass(gb);
ASSERT_FLOAT_EQ(imp_value(cbm_gbuf_find_by_id(gb, priv)->properties_json), sqrt(4.0) * 0.1,
1e-6);
ASSERT_FLOAT_EQ(imp_value(cbm_gbuf_find_by_id(gb, pub)->properties_json), sqrt(4.0), 1e-6);
cbm_gbuf_free(gb);
PASS();
}
/* Generic = defined in >= 5 DISTINCT FILES. Two defs of one name in the same
* file count once, so the boundary is tested on files, not on node count. */
TEST(importance_generic_names_are_demoted_on_distinct_files) {
cbm_gbuf_t *gb = cbm_gbuf_new("test-proj", "/tmp/test");
ASSERT_NOT_NULL(gb);
/* "generic": 5 distinct files -> demoted. */
int64_t generic = 0;
for (int i = 0; i < 5; i++) {
char qn[CBM_SZ_64];
char file[CBM_SZ_64];
snprintf(qn, sizeof(qn), "pkg%d.generic", i);
snprintf(file, sizeof(file), "pkg%d/main.go", i);
int64_t id = cbm_gbuf_upsert_node(gb, "Function", "generic", qn, file, 1, 1, "{}");
if (i == 0) {
generic = id;
}
}
/* "narrow": 4 distinct files, but 6 nodes — two duplicates share a file,
* so the DISTINCT count is 4 and it stays undemoted. */
int64_t narrow = 0;
for (int i = 0; i < 4; i++) {
char qn[CBM_SZ_64];
char file[CBM_SZ_64];
snprintf(qn, sizeof(qn), "n%d.narrow", i);
snprintf(file, sizeof(file), "n%d/main.go", i);
int64_t id = cbm_gbuf_upsert_node(gb, "Function", "narrow", qn, file, 1, 1, "{}");
if (i == 0) {
narrow = id;
}
}
cbm_gbuf_upsert_node(gb, "Function", "narrow", "n0.narrow_b", "n0/main.go", 9, 9, "{}");
cbm_gbuf_upsert_node(gb, "Function", "narrow", "n1.narrow_b", "n1/main.go", 9, 9, "{}");
for (int i = 0; i < 9; i++) {
char name[CBM_SZ_32];
char qn[CBM_SZ_64];
snprintf(name, sizeof(name), "z%d", i);
snprintf(qn, sizeof(qn), "z.z%d", i);
int64_t c = cbm_gbuf_upsert_node(gb, "Function", name, qn, "z/main.go", i, i, "{}");
cbm_gbuf_insert_edge(gb, c, i < 4 ? generic : narrow, "CALLS", "{}");
}
imp_run_pass(gb);
/* generic: sqrt(4) * 0.1; narrow: sqrt(5), undemoted. */
ASSERT_FLOAT_EQ(imp_value(cbm_gbuf_find_by_id(gb, generic)->properties_json), sqrt(4.0) * 0.1,
1e-6);
ASSERT_FLOAT_EQ(imp_value(cbm_gbuf_find_by_id(gb, narrow)->properties_json), sqrt(5.0), 1e-6);
cbm_gbuf_free(gb);
PASS();
}
TEST(importance_distinctive_names_are_promoted) {
cbm_gbuf_t *gb = cbm_gbuf_new("test-proj", "/tmp/test");
ASSERT_NOT_NULL(gb);
/* snake_case >= 8, camelCase >= 8, and a plain long lowercase word that is
* NOT distinctive (no '_', no hump). */
int64_t snake = cbm_gbuf_upsert_node(gb, "Function", "load_user_profile",
"pkg.load_user_profile", "pkg/a.go", 1, 1, "{}");
int64_t camel = cbm_gbuf_upsert_node(gb, "Function", "loadUserProfile", "pkg.loadUserProfile",
"pkg/b.go", 1, 1, "{}");
int64_t plain =
cbm_gbuf_upsert_node(gb, "Function", "aggregate", "pkg.aggregate", "pkg/c.go", 1, 1, "{}");
int64_t shortnm =
cbm_gbuf_upsert_node(gb, "Function", "get_x", "pkg.get_x", "pkg/d.go", 1, 1, "{}");
const int64_t targets[] = {snake, camel, plain, shortnm};
for (int t = 0; t < 4; t++) {
char name[CBM_SZ_32];
char qn[CBM_SZ_64];
snprintf(name, sizeof(name), "cc%d", t);
snprintf(qn, sizeof(qn), "z.cc%d", t);
int64_t c = cbm_gbuf_upsert_node(gb, "Function", name, qn, "z/main.go", t, t, "{}");
cbm_gbuf_insert_edge(gb, c, targets[t], "CALLS", "{}");
}
imp_run_pass(gb);
ASSERT_FLOAT_EQ(imp_value(cbm_gbuf_find_by_id(gb, snake)->properties_json), 1.0 * 10.0, 1e-6);
ASSERT_FLOAT_EQ(imp_value(cbm_gbuf_find_by_id(gb, camel)->properties_json), 1.0 * 10.0, 1e-6);
ASSERT_FLOAT_EQ(imp_value(cbm_gbuf_find_by_id(gb, plain)->properties_json), 1.0, 1e-6);
ASSERT_FLOAT_EQ(imp_value(cbm_gbuf_find_by_id(gb, shortnm)->properties_json), 1.0, 1e-6);
cbm_gbuf_free(gb);
PASS();
}
/* Test scaffolding is demoted two ways: living in a test file (the graph's own
* cbm_is_test_path classifier) or being the target of a TESTS edge. */
TEST(importance_test_scaffolding_is_demoted) {
cbm_gbuf_t *gb = cbm_gbuf_new("test-proj", "/tmp/test");
ASSERT_NOT_NULL(gb);
int64_t in_test_file = cbm_gbuf_upsert_node(gb, "Function", "fixture", "t.fixture",
"pkg/thing_test.go", 1, 1, "{}");
int64_t tests_target =
cbm_gbuf_upsert_node(gb, "Function", "helper", "pkg.helper", "pkg/util.go", 1, 1, "{}");
int64_t plain =
cbm_gbuf_upsert_node(gb, "Function", "worker", "pkg.worker", "pkg/util.go", 2, 2, "{}");
/* A production file whose NAME merely contains "test" must NOT be demoted. */
int64_t testutil = cbm_gbuf_upsert_node(gb, "Function", "spawn", "pkg.spawn",
"pkg/testutil_helpers.go", 1, 1, "{}");
const int64_t targets[] = {in_test_file, tests_target, plain, testutil};
for (int t = 0; t < 4; t++) {
char name[CBM_SZ_32];
char qn[CBM_SZ_64];
snprintf(name, sizeof(name), "d%d", t);
snprintf(qn, sizeof(qn), "z.d%d", t);
int64_t c = cbm_gbuf_upsert_node(gb, "Function", name, qn, "z/main.go", t, t, "{}");
cbm_gbuf_insert_edge(gb, c, targets[t], "CALLS", "{}");
}
int64_t tfn = cbm_gbuf_upsert_node(gb, "Function", "TestHelper", "t.TestHelper",
"pkg/util_test.go", 5, 5, "{}");
cbm_gbuf_insert_edge(gb, tfn, tests_target, "TESTS", "{}");
imp_run_pass(gb);
ASSERT_FLOAT_EQ(imp_value(cbm_gbuf_find_by_id(gb, in_test_file)->properties_json), 1.0 * 0.1,
1e-6);
ASSERT_FLOAT_EQ(imp_value(cbm_gbuf_find_by_id(gb, tests_target)->properties_json), 1.0 * 0.1,
1e-6);
ASSERT_FLOAT_EQ(imp_value(cbm_gbuf_find_by_id(gb, plain)->properties_json), 1.0, 1e-6);
ASSERT_FLOAT_EQ(imp_value(cbm_gbuf_find_by_id(gb, testutil)->properties_json), 1.0, 1e-6);
cbm_gbuf_free(gb);
PASS();
}
/* ── Idempotent write-back ────────────────────────────────────────────── */
TEST(importance_append_prop_overwrites_instead_of_duplicating) {
cbm_gbuf_t *gb = cbm_gbuf_new("test-proj", "/tmp/test");
ASSERT_NOT_NULL(gb);
int64_t id = cbm_gbuf_upsert_node(gb, "Function", "f", "p.f", "p/a.go", 1, 1,
"{\"loop_depth\":2,\"recursive\":false}");
cbm_gbuf_node_t *n = (cbm_gbuf_node_t *)cbm_gbuf_find_by_id(gb, id);
ASSERT_NOT_NULL(n);
cbm_pipeline_importance_append_prop(n, 1.5);
ASSERT_EQ(imp_key_count(n->properties_json), 1);
ASSERT_FLOAT_EQ(imp_value(n->properties_json), 1.5, 1e-6);
cbm_pipeline_importance_append_prop(n, 42.25);
printf(" after second write: %s\n", n->properties_json);
ASSERT_EQ(imp_key_count(n->properties_json), 1);
ASSERT_FLOAT_EQ(imp_value(n->properties_json), 42.25, 1e-6);
/* Sibling keys survive the in-place overwrite, on both sides of the key. */
ASSERT_TRUE(strstr(n->properties_json, "\"loop_depth\":2") != NULL);
ASSERT_TRUE(strstr(n->properties_json, "\"recursive\":false") != NULL);
/* A key that is NOT last: overwrite must not truncate the tail. */
free(n->properties_json);
n->properties_json = strdup("{\"importance\":0.000000,\"tail\":7}");
ASSERT_NOT_NULL(n->properties_json);
cbm_pipeline_importance_append_prop(n, 3.0);
ASSERT_EQ(imp_key_count(n->properties_json), 1);
ASSERT_FLOAT_EQ(imp_value(n->properties_json), 3.0, 1e-6);
ASSERT_TRUE(strstr(n->properties_json, "\"tail\":7") != NULL);
/* A string VALUE that merely contains the key text is not a key. */
free(n->properties_json);
n->properties_json = strdup("{\"doc\":\"see \\\"importance\\\": below\"}");
ASSERT_NOT_NULL(n->properties_json);
cbm_pipeline_importance_append_prop(n, 2.0);
ASSERT_FLOAT_EQ(imp_value(n->properties_json), 2.0, 1e-6);
ASSERT_TRUE(strstr(n->properties_json, "\"doc\":") != NULL);
/* Empty object and a non-object blob. */
free(n->properties_json);
n->properties_json = strdup("{}");
ASSERT_NOT_NULL(n->properties_json);
cbm_pipeline_importance_append_prop(n, 0.5);
ASSERT_STR_EQ(n->properties_json, "{\"importance\":0.500000}");
free(n->properties_json);
n->properties_json = strdup("not json");
ASSERT_NOT_NULL(n->properties_json);
cbm_pipeline_importance_append_prop(n, 9.0);
ASSERT_STR_EQ(n->properties_json, "not json");
cbm_gbuf_free(gb);
PASS();
}
/* Running the pass twice over the same graph — the shape the incremental path
* produces when it re-scores an already-scored rehydrated buffer. */
TEST(importance_pass_is_idempotent_across_reruns) {
cbm_gbuf_t *gb = cbm_gbuf_new("test-proj", "/tmp/test");
ASSERT_NOT_NULL(gb);
int64_t t =
cbm_gbuf_upsert_node(gb, "Function", "target", "pkg.target", "pkg/main.go", 1, 1, "{}");
int64_t c =
cbm_gbuf_upsert_node(gb, "Function", "caller", "pkg.caller", "pkg/main.go", 2, 2, "{}");
cbm_gbuf_insert_edge(gb, c, t, "CALLS", "{}");
imp_run_pass(gb);
char first[CBM_SZ_512];
snprintf(first, sizeof(first), "%s", cbm_gbuf_find_by_id(gb, t)->properties_json);
imp_run_pass(gb);
const char *second = cbm_gbuf_find_by_id(gb, t)->properties_json;
printf(" run1=%s run2=%s\n", first, second);
ASSERT_EQ(imp_key_count(second), 1);
ASSERT_STR_EQ(first, second); /* re-scoring is a fixed point */
cbm_gbuf_free(gb);
PASS();
}
/* The literal incremental shape: index, rehydrate the stored graph exactly as
* pipeline_incremental.c does, re-run the pass. Nodes arrive already carrying
* the key — an append-only write-back yields two. */
TEST(importance_rehydrated_node_keeps_exactly_one_key) {
char *tmp = th_mktempdir("cbm_imp_rehy");
if (!tmp) {
FAIL("tmpdir");
}
char root[512];
snprintf(root, sizeof(root), "%s", tmp); /* th_mktempdir returns a static buffer */
ASSERT_EQ(th_write_file(TH_PATH(root, "main.py"),
"def helper():\n return 1\n\n"
"def caller():\n return helper() + helper()\n"),
0);
char db_path[600];
snprintf(db_path, sizeof(db_path), "%s/graph.db", root);
cbm_pipeline_t *p = cbm_pipeline_new(root, db_path, CBM_MODE_FULL);
ASSERT_NOT_NULL(p);
ASSERT_EQ(cbm_pipeline_run(p), 0);
char project[256];
snprintf(project, sizeof(project), "%s", cbm_pipeline_project_name(p));
cbm_pipeline_free(p);
cbm_gbuf_t *gb = cbm_gbuf_new(project, root);
ASSERT_NOT_NULL(gb);
ASSERT_EQ(cbm_gbuf_load_from_db(gb, db_path, project), 0);
/* Precondition: the rehydrated nodes already carry the key. Without this
* the test could pass while proving nothing. */
const cbm_gbuf_node_t **fns = NULL;
int fc = 0;
ASSERT_EQ(cbm_gbuf_find_by_label(gb, "Function", &fns, &fc), 0);
ASSERT_GT(fc, 0);
for (int i = 0; i < fc; i++) {
ASSERT_EQ(imp_key_count(fns[i]->properties_json), 1);
}
imp_run_pass(gb);
ASSERT_EQ(cbm_gbuf_find_by_label(gb, "Function", &fns, &fc), 0);
ASSERT_GT(fc, 0);
for (int i = 0; i < fc; i++) {
if (imp_key_count(fns[i]->properties_json) != 1) {
printf(" duplicate key on '%s': %s\n", fns[i]->name, fns[i]->properties_json);
}
ASSERT_EQ(imp_key_count(fns[i]->properties_json), 1);
}
cbm_gbuf_free(gb);
th_rmtree(root);
PASS();
}
/* Pass-timing observer: which importance-pass call sites fired. */
static atomic_int g_saw_incr_importance = 0;
static atomic_int g_saw_full_importance = 0;
static void imp_pass_sink(const char *line) {
if (!line || !strstr(line, "pass.timing")) {
return;
}
if (strstr(line, "pass=incr_importance")) {
atomic_fetch_add_explicit(&g_saw_incr_importance, 1, memory_order_relaxed);
} else if (strstr(line, "pass=importance")) {
atomic_fetch_add_explicit(&g_saw_full_importance, 1, memory_order_relaxed);
}
}
/* End-to-end, on the incremental route that actually rehydrates the stored
* graph (LEGACY_PARTIAL — production's other incremental route, closure
* repair, works on a delta buffer and never sees pre-scored nodes). Index,
* edit a file, index again: the run must be observed on that route, the pass
* must have run on it, and every stored symbol must carry exactly one key. An
* append-only write-back produces two here. */
TEST(importance_second_index_run_keeps_exactly_one_key) {
char *tmp = th_mktempdir("cbm_imp_incr");
if (!tmp) {
FAIL("tmpdir");
}
char root[512];
snprintf(root, sizeof(root), "%s", tmp); /* th_mktempdir returns a static buffer */
ASSERT_EQ(th_write_file(TH_PATH(root, "a.py"), "def helper():\n return 1\n"), 0);
ASSERT_EQ(
th_write_file(TH_PATH(root, "b.py"), "import a\n\ndef caller():\n return a.helper()\n"),
0);
char db_path[600];
snprintf(db_path, sizeof(db_path), "%s/graph.db", root);
atomic_store(&g_saw_incr_importance, 0);
atomic_store(&g_saw_full_importance, 0);
cbm_log_set_sink_ex(imp_pass_sink, CBM_LOG_SINK_TEE);
cbm_pipeline_t *p1 = cbm_pipeline_new(root, db_path, CBM_MODE_FULL);
ASSERT_NOT_NULL(p1);
ASSERT_EQ(cbm_pipeline_run(p1), 0);
char project[256];
snprintf(project, sizeof(project), "%s", cbm_pipeline_project_name(p1));
cbm_pipeline_free(p1);
/* Edit one file so the second run has real work but stays incremental. */
ASSERT_EQ(th_write_file(TH_PATH(root, "b.py"),
"import a\n\ndef caller():\n return a.helper() + a.helper()\n"),
0);
cbm_pipeline_incremental_test_force_legacy_partial_once();
cbm_pipeline_t *p2 = cbm_pipeline_new(root, db_path, CBM_MODE_FULL);
ASSERT_NOT_NULL(p2);
ASSERT_EQ(cbm_pipeline_run(p2), 0);
cbm_incremental_route_t route = cbm_pipeline_incremental_test_last_route();
cbm_pipeline_free(p2);
cbm_log_set_sink(NULL);
/* Route proof: without it a silent fall-back to a full reindex would make
* the duplicate-key assertion below vacuous. */
ASSERT_EQ((int)route, (int)CBM_INCREMENTAL_ROUTE_LEGACY_PARTIAL);
int incr = atomic_load(&g_saw_incr_importance);
int full = atomic_load(&g_saw_full_importance);
printf(" importance pass runs: full=%d incremental=%d\n", full, incr);
ASSERT_GT(full, 0); /* first run scored via the predump table */
/* The second run must have re-scored too — through whichever path it took.
* If it was NOT incremental the duplicate-key trap is untested, so say so
* rather than passing quietly. */
if (incr == 0) {
th_rmtree(root);
FAIL("second run did not re-score on the incremental path — trap untested");
}
cbm_store_t *s = cbm_store_open_path(db_path);
ASSERT_NOT_NULL(s);
int fc = imp_check_label(s, project, "Function", 1);
printf(" functions with exactly one importance key: %d\n", fc);
ASSERT_GT(fc, 0);
cbm_store_close(s);
th_rmtree(root);
PASS();
}
/* ── Closure-delta route: the graph is only complete in the store ─────── */
/* REGRESSION. `helper` lives in hot.py and is called from 25 other files. Edit
* hot.py and re-index: the closure-delta route purges hot.py's rows, rebuilds
* them from a PROXY buffer that holds no project-wide edges, and persists what
* it computed. Scoring off that buffer sees in-degree 0 and stores 0.0 for a
* symbol with 25 real callers. The fix rescores in SQL after cbm_delta_patch
* has re-linked the snapshotted inbound edges. */
TEST(importance_closure_delta_keeps_referenced_symbol_sane) {
enum { CALLERS = 25 };
char *tmp = th_mktempdir("cbm_imp_delta");
if (!tmp) {
FAIL("tmpdir");
}
char root[512];
snprintf(root, sizeof(root), "%s", tmp);
char *tmp2 = th_mktempdir("cbm_imp_deltadb");
if (!tmp2) {
th_rmtree(root);
FAIL("tmpdir2");
}
char dbdir[512];
snprintf(dbdir, sizeof(dbdir), "%s", tmp2);
ASSERT_EQ(imp_write_hot_fixture(root, CALLERS, ""), 0);
char db_path[600];
snprintf(db_path, sizeof(db_path), "%s/graph.db", dbdir);
cbm_pipeline_t *p1 = cbm_pipeline_new(root, db_path, CBM_MODE_FULL);
ASSERT_NOT_NULL(p1);
ASSERT_EQ(cbm_pipeline_run(p1), 0);
char project[256];
snprintf(project, sizeof(project), "%s", cbm_pipeline_project_name(p1));
cbm_pipeline_free(p1);
ImpScoreSet before = {0};
ASSERT_EQ(imp_collect_scores(db_path, project, &before), 0);
double full_score = imp_score_of(&before, "helper");
printf(" full-index helper score %.6f (expect sqrt(%d) = %.6f)\n", full_score, CALLERS,
sqrt((double)CALLERS));
/* Non-vacuous: the callers really produced edges. If this is 0 the fixture
* stopped exercising the bug and the test below would prove nothing. */
ASSERT_TRUE(full_score > 1.0);
imp_scores_free(&before);
/* Touch hot.py so the delta route has exactly one file to repair. */
ASSERT_EQ(imp_write_hot_fixture(root, CALLERS, "# touched\n"), 0);
uint64_t sql0 = atomic_load_explicit(&g_importance_store_rows, memory_order_relaxed);
cbm_pipeline_t *p2 = cbm_pipeline_new(root, db_path, CBM_MODE_FULL);
ASSERT_NOT_NULL(p2);
ASSERT_EQ(cbm_pipeline_run(p2), 0);
cbm_incremental_route_t route = cbm_pipeline_incremental_test_last_route();
cbm_pipeline_free(p2);
uint64_t sql_rows = atomic_load_explicit(&g_importance_store_rows, memory_order_relaxed) - sql0;
/* Route proof — without it a fall-back to a full reindex would repair the
* score for the wrong reason and the test would pass vacuously. */
printf(" route=%d (closure_repair=%d) sql_rescored_rows=%llu\n", (int)route,
(int)CBM_INCREMENTAL_ROUTE_CLOSURE_REPAIR, (unsigned long long)sql_rows);
ASSERT_EQ((int)route, (int)CBM_INCREMENTAL_ROUTE_CLOSURE_REPAIR);
ImpScoreSet after = {0};
ASSERT_EQ(imp_collect_scores(db_path, project, &after), 0);
double delta_score = imp_score_of(&after, "helper");
printf(" after closure-delta helper score %.6f\n", delta_score);
imp_scores_free(&after);
th_rmtree(root);
th_rmtree(dbdir);
/* THE SYMPTOM, asserted before any mechanism guard: the graph is unchanged,
* so the score must be unchanged. Scoring off the proxy buffer yields 0.0. */
ASSERT_FLOAT_EQ(delta_score, full_score, 1e-6);
/* Mechanism guard second, so a regression fails on the wrong score rather
* than on bookkeeping. */
ASSERT_GT((long long)sql_rows, 0);
PASS();
}
/* EQUIVALENCE. A full index and an incremental index that converge on the same
* tree must produce the same scores — for every symbol, not just the hot one.
* Same directory and therefore the same project name and qualified names, so
* the two score maps are directly comparable. */
TEST(importance_full_and_incremental_converge_on_same_scores) {
enum { CALLERS = 12 };
char *tmp = th_mktempdir("cbm_imp_equiv");
if (!tmp) {
FAIL("tmpdir");
}
char root[512];
snprintf(root, sizeof(root), "%s", tmp);
char *tmp2 = th_mktempdir("cbm_imp_equivdb");
if (!tmp2) {
th_rmtree(root);
FAIL("tmpdir2");
}
char dbdir[512];
snprintf(dbdir, sizeof(dbdir), "%s", tmp2);
/* v1: every caller but c0 calls helper. c0 imports hot but does not. */
ASSERT_EQ(imp_write_hot_fixture(root, CALLERS, ""), 0);
ASSERT_EQ(th_write_file(TH_PATH(root, "c0.py"), "import hot\n\ndef use0():\n return 1\n"),
0);
ASSERT_EQ(th_write_file(TH_PATH(root, "extra.py"),
"import hot\n\ndef load_user_profile():\n return hot.helper()\n"),
0);
char db_incr[600];
snprintf(db_incr, sizeof(db_incr), "%s/incr.db", dbdir);
cbm_pipeline_t *p1 = cbm_pipeline_new(root, db_incr, CBM_MODE_FULL);
ASSERT_NOT_NULL(p1);
ASSERT_EQ(cbm_pipeline_run(p1), 0);
char project[256];
snprintf(project, sizeof(project), "%s", cbm_pipeline_project_name(p1));
cbm_pipeline_free(p1);
/* v2: c0's BODY starts calling helper. Body-only, so c0's surface is
* unchanged and the closure planner accepts the delta — but the in-degree
* of `helper`, which lives in the UNCHANGED hot.py, goes up by one. That
* is the case a changed-files-only rescore gets wrong: the symbol whose
* score moved is not in the changed set at all. */
ASSERT_EQ(th_write_file(TH_PATH(root, "c0.py"),
"import hot\n\ndef use0():\n return hot.helper()\n"),
0);
uint64_t sql0 = atomic_load_explicit(&g_importance_store_rows, memory_order_relaxed);
cbm_pipeline_t *p2 = cbm_pipeline_new(root, db_incr, CBM_MODE_FULL);
ASSERT_NOT_NULL(p2);
ASSERT_EQ(cbm_pipeline_run(p2), 0);
cbm_incremental_route_t route = cbm_pipeline_incremental_test_last_route();
cbm_pipeline_free(p2);
uint64_t sql_rows = atomic_load_explicit(&g_importance_store_rows, memory_order_relaxed) - sql0;
printf(" route=%d (closure_repair=%d) sql_rescored_rows=%llu\n", (int)route,
(int)CBM_INCREMENTAL_ROUTE_CLOSURE_REPAIR, (unsigned long long)sql_rows);
ASSERT_EQ((int)route, (int)CBM_INCREMENTAL_ROUTE_CLOSURE_REPAIR);
/* Same tree, indexed cold into a separate database. */
char db_full[600];
snprintf(db_full, sizeof(db_full), "%s/full.db", dbdir);
cbm_pipeline_t *p3 = cbm_pipeline_new(root, db_full, CBM_MODE_FULL);
ASSERT_NOT_NULL(p3);
ASSERT_EQ(cbm_pipeline_run(p3), 0);
cbm_pipeline_free(p3);
ImpScoreSet incr = {0};
ImpScoreSet full = {0};
ASSERT_EQ(imp_collect_scores(db_incr, project, &incr), 0);
ASSERT_EQ(imp_collect_scores(db_full, project, &full), 0);
printf(" scored symbols: incremental=%d full=%d\n", incr.count, full.count);
int mismatches = 0;
for (int i = 0; i < full.count; i++) {
double a = full.scores[i];
double b = imp_score_of(&incr, full.qns[i]);
if (!(fabs(a - b) < 1e-6)) {
mismatches++;
if (mismatches <= 5) {
printf(" MISMATCH %s: full=%.6f incremental=%.6f\n", full.qns[i], a, b);
}
}
}
int count_full = full.count;
int count_incr = incr.count;
imp_scores_free(&incr);
imp_scores_free(&full);
th_rmtree(root);
th_rmtree(dbdir);
ASSERT_GT(count_full, 5); /* non-vacuous: there really are symbols to compare */
ASSERT_EQ(count_incr, count_full);
ASSERT_EQ(mismatches, 0); /* the symptom */
ASSERT_GT((long long)sql_rows, 0); /* mechanism guard, asserted second */
PASS();
}
/* ── Cost shape ───────────────────────────────────────────────────────── */
/* The distinct-file count must be computed once per distinct NAME. Computing
* it per NODE makes a same-name group of size k cost O(k^3) — measurably
* disqualifying on real corpora. Assert on the pass's own work counter, not
* on wall time: doubling a same-name group must roughly double the work. */
TEST(importance_distinct_file_count_is_linear_in_group_size) {
enum { K = 200 };
uint64_t visits[2];
for (int leg = 0; leg < 2; leg++) {
int k = (leg == 0) ? K : 2 * K;
cbm_gbuf_t *gb = cbm_gbuf_new("test-proj", "/tmp/test");
ASSERT_NOT_NULL(gb);
for (int i = 0; i < k; i++) {
char qn[CBM_SZ_64];
char file[CBM_SZ_64];
snprintf(qn, sizeof(qn), "pkg%d.toString", i);
snprintf(file, sizeof(file), "pkg%d/a.java", i);
ASSERT_GT(cbm_gbuf_upsert_node(gb, "Method", "toString", qn, file, 1, 1, "{}"), 0);
}
uint64_t v0 = atomic_load_explicit(&g_importance_name_visits, memory_order_relaxed);
imp_run_pass(gb);
visits[leg] = atomic_load_explicit(&g_importance_name_visits, memory_order_relaxed) - v0;
cbm_gbuf_free(gb);
}
double ratio = (visits[0] > 0) ? (double)visits[1] / (double)visits[0] : 0.0;
printf(" name visits k=%d -> %llu, k=%d -> %llu (ratio %.2f; linear ~2, per-node ~4)\n", K,
(unsigned long long)visits[0], 2 * K, (unsigned long long)visits[1], ratio);
/* Non-vacuous: the group really was scanned. */
ASSERT_GT((long long)visits[0], 0);
/* Memoized: one scan of the whole group, so visits == k exactly. */
ASSERT_EQ((long long)visits[0], (long long)K);
ASSERT_EQ((long long)visits[1], (long long)(2 * K));
ASSERT_TRUE(ratio <= 2.75);
PASS();
}
/* ── Suite ────────────────────────────────────────────────────────────── */
SUITE(importance) {
/* Registration — catches a miscounted PREDUMP_PASS_COUNT. */
RUN_TEST(importance_pass_actually_runs_in_full_pipeline);
/* Formula. */
RUN_TEST(importance_base_is_sqrt_of_incoming_refs);
RUN_TEST(importance_private_names_are_demoted);
RUN_TEST(importance_generic_names_are_demoted_on_distinct_files);
RUN_TEST(importance_distinctive_names_are_promoted);
RUN_TEST(importance_test_scaffolding_is_demoted);
/* Idempotent write-back — the incremental duplicate-key trap. */
RUN_TEST(importance_append_prop_overwrites_instead_of_duplicating);
RUN_TEST(importance_pass_is_idempotent_across_reruns);
RUN_TEST(importance_rehydrated_node_keeps_exactly_one_key);
RUN_TEST(importance_second_index_run_keeps_exactly_one_key);
/* Closure-delta route — the store-level recompute. */
RUN_TEST(importance_closure_delta_keeps_referenced_symbol_sane);
RUN_TEST(importance_full_and_incremental_converge_on_same_scores);
/* Cost shape. */
RUN_TEST(importance_distinct_file_count_is_linear_in_group_size);
}
tests/test_main.c
+2
View File
@@ -768,6 +768,7 @@ extern void suite_discover(void);
extern void suite_graph_buffer(void);
extern void suite_registry(void);
extern void suite_pipeline(void);
extern void suite_importance(void);
extern void suite_pipeline_semantic_manifest_repro(void);
extern void suite_cross_repo(void);
extern void suite_index_resilience(void);
@@ -1041,6 +1042,7 @@ int main(int argc, char **argv) {
/* Pipeline (M8) */
RUN_SELECTED_SUITE(registry);
RUN_SELECTED_SUITE(pipeline);
RUN_SELECTED_SUITE(importance);
RUN_SELECTED_SUITE(index_format);
RUN_SELECTED_SUITE(pipeline_semantic_manifest_repro);
RUN_SELECTED_SUITE(call_reference_contract);