trending/codebase-memory-mcp
1
0
Fork 0
mirror of https://github.com/DeusData/codebase-memory-mcp.git synced 2026-11-07 08:43:35 +08:00
Code Issues Packages Projects Releases Wiki Activity

perf(pipeline): pin dependency-query join order; gate the coverage shadow rebuild

Browse Source 
Two measured pathologies out of the delta path, both shared-code fixes
that help every route:

Join order. The inbound-edge snapshot and the dependent-files lookup
let the planner start from EDGES, walking every project edge through
the url_path index prefix — 14.6s for a one-file closure against the
kernel's 16.5M edges. CROSS JOIN pins nodes-first (idx_nodes_file →
idx_edges_target → primary key): measured 4ms for the same query, 23ms
for the full snapshot+purge step.

Coverage shadow graph. cbm_store_coverage_replace_ex rebuilt the
miss-graph shadow view wholesale inside every publish — wipe plus tens
of thousands of node/edge upserts probing the full-size nodes index,
23.4s at kernel scale — even when the failure-row set it derives from
was byte-identical. The rebuild is now gated on a sha256 fingerprint of
the failure rows persisted in store_meta: unchanged set, provable
no-op, skipped. Measured 84ms steady-state; the rebuild still fires
whenever the set actually changes, and the shadow output is untouched.

The benchmark probe also stops appending a trailing comment that
happened to break bootp.c's parse — a probe that mutates the failure
set on every run measures the shadow rebuild, not the repair. It now
edits inside the license-header comment.

Kernel one-file warm, steady state: 111.6s -> 85.3s wall (71.1s
worker). Remaining measured blocks: base-def rehydration ~35s, preseed
18.8s, repair 10.8s — the parallelization targets. 554 store/pipeline/
incremental tests green.

Signed-off-by: Martin Vogel <martin.vogel.tech@gmail.com>
This commit is contained in:
Martin Vogel
2026-08-03 22:08:17 +02:00
parent 6ec9e944e5
commit 87eafefe73
3 changed files with 111 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/pipeline/pipeline.c
+6
View File
@@ -1722,6 +1722,9 @@ int cbm_pipeline_publish_staged(char *stage_path, const cbm_pipeline_generation_
cbm_project_t project_info = {0};
bool have_project_info =
cbm_store_get_project(store, generation->project, &project_info) == CBM_STORE_OK;
cbm_log_info("publish.timing", "block", "get_project", "elapsed_ms",
itoa_buf((int)elapsed_ms(t_pub)));
cbm_clock_gettime(CLOCK_MONOTONIC, &t_pub);
cbm_coverage_meta_t meta = generation->coverage_meta;
meta.generation = have_project_info ? project_info.indexed_at : NULL;
meta.coverage_version = CBM_SEMANTIC_INDEX_VERSION;
@@ -1734,6 +1737,9 @@ int cbm_pipeline_publish_staged(char *stage_path, const cbm_pipeline_generation_
if (have_project_info) {
cbm_project_free_fields(&project_info);
}
cbm_log_info("publish.timing", "block", "coverage_replace", "elapsed_ms",
itoa_buf((int)elapsed_ms(t_pub)));
cbm_clock_gettime(CLOCK_MONOTONIC, &t_pub);
if (fts_wholesale && generation_rebuild_fts(store) != CBM_STORE_OK) {
ok = false;
}
src/pipeline/pipeline_delta.c
+7 -4
View File
@@ -98,11 +98,14 @@ int cbm_delta_snapshot_inbound(cbm_store_t *store, const char *project, const ch
delta_placeholders(ph, chunk);
char sql[CBM_SZ_4K];
int n = snprintf(sql, sizeof(sql),
/* CROSS JOIN pins nodes-first: the planner otherwise walks
* EVERY project edge through the url_path index prefix
* (measured 14.6s vs 4ms at kernel scale). */
"SELECT src.qualified_name, tgt.qualified_name, e.type, e.properties"
" FROM edges e"
" JOIN nodes tgt ON e.target_id = tgt.id"
" JOIN nodes src ON e.source_id = src.id"
" WHERE e.project = ?1 AND tgt.file_path IN (%s)"
" FROM nodes tgt"
" CROSS JOIN edges e ON e.target_id = tgt.id"
" CROSS JOIN nodes src ON e.source_id = src.id"
" WHERE tgt.project = ?1 AND tgt.file_path IN (%s)"
" AND src.file_path NOT IN (%s)"
" AND src.file_path <> '' AND src.file_path IS NOT NULL",
ph, ph);
src/store/store.c
+98 -4
View File
@@ -11,6 +11,7 @@
#include <stdint.h>
#include "foundation/constants.h"
#include "foundation/hash_table.h"
#include "foundation/sha256.h"
#include <math.h>
@@ -2511,10 +2512,73 @@ static void cov_json_escape(char *dst, size_t dstsz, const char *src) {
* {kind, detail}. Queryable via query_graph(graph="missed") with zero
* cypher-engine changes; the real project's graph gains no rows. Derived
* data — rebuilt from the authoritative (post-prune) table contents. */
/* Fingerprint of the FAILURE rows the shadow graph derives from. When the
* set is unchanged since the last rebuild, the wipe-and-rebuild (tens of
* thousands of node/edge upserts probing the full-size nodes index -- a
* measured 23s at kernel scale) is provably a no-op and is skipped. The
* fingerprint lives in store_meta keyed per project and is cleared
* implicitly by any rebuild from a different row set. */
static void cov_failure_fingerprint(const cbm_coverage_row_t *rows, int count,
char out[CBM_SHA256_HEX_LEN + 1]) {
cbm_sha256_ctx sha;
cbm_sha256_init(&sha);
for (int i = 0; i < count; i++) {
if (!rows[i].kind || strncmp(rows[i].kind, "not_indexed", 11) == 0) {
continue;
}
const char *rel = rows[i].rel_path ? rows[i].rel_path : "";
const char *kind = rows[i].kind;
const char *detail = rows[i].detail ? rows[i].detail : "";
cbm_sha256_update(&sha, rel, strlen(rel) + 1);
cbm_sha256_update(&sha, kind, strlen(kind) + 1);
cbm_sha256_update(&sha, detail, strlen(detail) + 1);
}
uint8_t digest[CBM_SHA256_DIGEST_LEN];
cbm_sha256_final(&sha, digest);
static const char hex[] = "0123456789abcdef";
for (int i = 0; i < CBM_SHA256_DIGEST_LEN; i++) {
out[i * 2] = hex[digest[i] >> 4];
out[i * 2 + 1] = hex[digest[i] & 0x0f];
}
out[CBM_SHA256_HEX_LEN] = 0;
}
static int cov_rebuild_shadow_graph(cbm_store_t *s, const char *project) {
char covproj[CBM_SZ_512];
cbm_store_coverage_shadow_project(covproj, sizeof(covproj), project);
/* Fingerprint gate: rows first, so an unchanged failure set skips the
* wipe entirely and the existing shadow stays byte-identical. */
cbm_coverage_row_t *fp_rows = NULL;
int fp_count = 0;
if (cbm_store_coverage_get(s, project, &fp_rows, &fp_count) != CBM_STORE_OK) {
cbm_store_free_coverage(fp_rows, fp_count);
return CBM_STORE_ERR;
}
char fp[CBM_SHA256_HEX_LEN + 1];
cov_failure_fingerprint(fp_rows, fp_count, fp);
cbm_store_free_coverage(fp_rows, fp_count);
char fp_key[CBM_SZ_512];
snprintf(fp_key, sizeof(fp_key), "coverage_shadow_fp:%s", project);
char prev_fp[CBM_SHA256_HEX_LEN + 1] = {0};
{
sqlite3_stmt *get = NULL;
if (sqlite3_prepare_v2(s->db, "SELECT v FROM store_meta WHERE k = ?1;", CBM_NOT_FOUND, &get,
NULL) == SQLITE_OK) {
bind_text(get, SKIP_ONE, fp_key);
if (sqlite3_step(get) == SQLITE_ROW) {
const char *v = (const char *)sqlite3_column_text(get, 0);
if (v) {
snprintf(prev_fp, sizeof(prev_fp), "%s", v);
}
}
sqlite3_finalize(get);
}
}
if (prev_fp[0] && strcmp(prev_fp, fp) == 0) {
return CBM_STORE_OK;
}
/* Wipe the previous view (edges first: no FK pragma guarantee). */
static const char *wipes[] = {"DELETE FROM edges WHERE project = ?1;",
"DELETE FROM nodes WHERE project = ?1;"};
@@ -2651,18 +2715,42 @@ static int cov_rebuild_shadow_graph(cbm_store_t *s, const char *project) {
}
}
cbm_store_free_coverage(rows, count);
{
sqlite3_stmt *set = NULL;
if (sqlite3_prepare_v2(s->db, "INSERT OR REPLACE INTO store_meta (k, v) VALUES (?1, ?2);",
CBM_NOT_FOUND, &set, NULL) == SQLITE_OK) {
bind_text(set, SKIP_ONE, fp_key);
bind_text(set, ST_COL_2, fp);
(void)sqlite3_step(set);
sqlite3_finalize(set);
}
}
return CBM_STORE_OK;
}
static void cov_timing_mark(struct timespec *t, const char *what) {
struct timespec now;
cbm_clock_gettime(CLOCK_MONOTONIC, &now);
double ms =
(double)(now.tv_sec - t->tv_sec) * 1000.0 + (double)(now.tv_nsec - t->tv_nsec) / 1000000.0;
char buf[32];
snprintf(buf, sizeof(buf), "%.0f", ms);
cbm_log_info("coverage.timing", "step", what, "elapsed_ms", buf);
*t = now;
}
int cbm_store_coverage_replace_ex(cbm_store_t *s, const char *project,
const cbm_coverage_row_t *rows, int count,
const cbm_coverage_meta_t *meta) {
if (!s || !s->db || !project || count < 0 || (count > 0 && !rows)) {
return CBM_STORE_ERR;
}
struct timespec cov_t0;
cbm_clock_gettime(CLOCK_MONOTONIC, &cov_t0);
if (exec_sql(s, "BEGIN;") != CBM_STORE_OK) {
return CBM_STORE_ERR;
}
cov_timing_mark(&cov_t0, "begin");
sqlite3_stmt *del = NULL;
if (sqlite3_prepare_v2(s->db, "DELETE FROM index_coverage WHERE project = ?1;", CBM_NOT_FOUND,
&del, NULL) != SQLITE_OK) {
@@ -2678,6 +2766,7 @@ int cbm_store_coverage_replace_ex(cbm_store_t *s, const char *project,
(void)exec_sql(s, "ROLLBACK;");
return CBM_STORE_ERR;
}
cov_timing_mark(&cov_t0, "delete");
sqlite3_stmt *ins = NULL;
if (sqlite3_prepare_v2(
s->db,
@@ -2705,6 +2794,7 @@ int cbm_store_coverage_replace_ex(cbm_store_t *s, const char *project,
sqlite3_reset(ins);
}
sqlite3_finalize(ins);
cov_timing_mark(&cov_t0, "insert_loop");
/* Prune FAILURE rows for files no longer known to the index (deleted from
* the repo): file_hashes is the authoritative live-file set after
* persist. By-design not_indexed_* rows are exempt — deliberately
@@ -2728,6 +2818,7 @@ int cbm_store_coverage_replace_ex(cbm_store_t *s, const char *project,
(void)exec_sql(s, "ROLLBACK;");
return CBM_STORE_ERR;
}
cov_timing_mark(&cov_t0, "prune");
if (meta) {
char recorded_at[CBM_SZ_64];
@@ -2804,6 +2895,7 @@ int cbm_store_coverage_replace_ex(cbm_store_t *s, const char *project,
return CBM_STORE_ERR;
}
return exec_sql(s, "COMMIT;");
cov_timing_mark(&cov_t0, "meta_and_commit");
}
int cbm_store_coverage_replace(cbm_store_t *s, const char *project, const cbm_coverage_row_t *rows,
@@ -3555,10 +3647,12 @@ static int dependent_files_chunk(cbm_store_t *s, const char *project,
CBMHashTable *seen, char ***out, int *out_count, int *out_cap) {
char sql[CBM_SZ_4K];
int pos = snprintf(sql, sizeof(sql),
"SELECT DISTINCT src.file_path FROM edges e"
" JOIN nodes tgt ON e.target_id = tgt.id"
" JOIN nodes src ON e.source_id = src.id"
" WHERE e.project = ?1 AND tgt.file_path IN (");
/* CROSS JOIN pins nodes-first (see the delta snapshot query:
* the free planner walks every project edge instead). */
"SELECT DISTINCT src.file_path FROM nodes tgt"
" CROSS JOIN edges e ON e.target_id = tgt.id"
" CROSS JOIN nodes src ON e.source_id = src.id"
" WHERE tgt.project = ?1 AND tgt.file_path IN (");
for (int i = 0; i < chunk_count; i++) {
pos += snprintf(sql + pos, sizeof(sql) - (size_t)pos, "%s?%d", i ? "," : "", i + ST_COL_2);
if ((size_t)pos >= sizeof(sql)) {