trending/codebase-memory-mcp
1
0
Fork 0
mirror of https://github.com/DeusData/codebase-memory-mcp.git synced 2026-11-03 08:03:38 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(pipeline): publish additive header overlays

Browse Source 
Add a pipeline wrapper for non-tombstone overlay publishes and reuse the store active overlay CTE for file-node reads and summary counts. This lets header/type overlay rows refresh active read views without hiding canonical source-owned rows.

Add focused tests for duplicate-QN additive overlay ranking, single header type/implementation additive overlay behavior, and exact-frontier overlay dirty state.

Validation: bash scripts/check-source-safety.sh; bash scripts/test-source-safety.sh; git diff --check; make -f Makefile.cbm -j8 cbm; CBM_ONLY_SUITE=store_nodes build/c/test-runner; CBM_ONLY_SUITE=pipeline build/c/test-runner.

Scope: this is not the store_pipeline_batch default speed fix. Current benchmark evidence still shows safe full fallback around 2.6x, so narrower dependency-slice or active-read algorithm work remains open.
Signed-off-by: Andrew Hundt <ATHundt@gmail.com>
This commit is contained in:
Andrew Hundt
2026-07-04 12:53:55 -04:00
parent 170aa7d830
commit af4022b91f
6 changed files with 366 additions and 213 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/pipeline/pipeline_delta.c
+92 -37
View File
@@ -277,6 +277,48 @@ static void delta_edge_free_fields(cbm_store_delta_edge_t *edge) {
*edge = (cbm_store_delta_edge_t){0};
}
int cbm_pipeline_copy_delta_node(const cbm_node_t *src, cbm_node_t *dst) {
if (!src || !dst) {
return CBM_STORE_ERR;
}
*dst = (cbm_node_t){
.id = CBM_STORE_NO_NODE_ID,
.project = delta_strdup(src->project),
.label = delta_strdup(src->label),
.name = delta_strdup(src->name),
.qualified_name = delta_strdup(src->qualified_name),
.file_path = delta_strdup(src->file_path),
.start_line = src->start_line,
.end_line = src->end_line,
.properties_json = delta_strdup(src->properties_json ? src->properties_json : "{}"),
};
if (!dst->project || !dst->label || !dst->name || !dst->qualified_name ||
!dst->file_path || !dst->properties_json) {
cbm_node_free_fields(dst);
return CBM_STORE_ERR;
}
return CBM_STORE_OK;
}
int cbm_pipeline_copy_delta_edge(const cbm_store_delta_edge_t *src,
cbm_store_delta_edge_t *dst) {
if (!src || !dst) {
return CBM_STORE_ERR;
}
*dst = (cbm_store_delta_edge_t){
.source_qn = delta_strdup(src->source_qn),
.target_qn = delta_strdup(src->target_qn),
.type = delta_strdup(src->type),
.properties_json = delta_strdup(src->properties_json ? src->properties_json : "{}"),
.derived_kind = src->derived_kind ? src->derived_kind : CBM_STORE_DERIVED_KIND_DIRECT,
};
if (!dst->source_qn || !dst->target_qn || !dst->type || !dst->properties_json) {
delta_edge_free_fields(dst);
return CBM_STORE_ERR;
}
return CBM_STORE_OK;
}
static void delta_import_free_fields(cbm_store_import_ref_t *import) {
if (!import) {
return;
@@ -293,23 +335,19 @@ static int delta_append_node(cbm_delta_build_ctx_t *ctx, const cbm_gbuf_node_t *
CBM_STORE_OK) {
return CBM_STORE_ERR;
}
cbm_node_t row = {
.id = CBM_STORE_NO_NODE_ID,
.project = delta_strdup(ctx->project),
.label = delta_strdup(node->label),
.name = delta_strdup(node->name),
.qualified_name = delta_strdup(node->qualified_name),
.file_path = delta_strdup(node->file_path),
.start_line = node->start_line,
.end_line = node->end_line,
.properties_json = delta_strdup(node->properties_json ? node->properties_json : "{}"),
};
if (!row.project || !row.label || !row.name || !row.qualified_name || !row.file_path ||
!row.properties_json) {
cbm_node_free_fields(&row);
cbm_node_t row = {.project = ctx->project,
.label = node->label,
.name = node->name,
.qualified_name = node->qualified_name,
.file_path = node->file_path,
.start_line = node->start_line,
.end_line = node->end_line,
.properties_json = node->properties_json};
if (cbm_pipeline_copy_delta_node(&row, &ctx->out->nodes[ctx->out->delta.node_count]) !=
CBM_STORE_OK) {
return CBM_STORE_ERR;
}
ctx->out->nodes[ctx->out->delta.node_count++] = row;
ctx->out->delta.node_count++;
return CBM_STORE_OK;
}
@@ -338,23 +376,20 @@ static int delta_append_context_node(cbm_delta_build_ctx_t *ctx,
sizeof(*ctx->out->context_nodes)) != CBM_STORE_OK) {
return CBM_STORE_ERR;
}
cbm_node_t row = {
.id = CBM_STORE_NO_NODE_ID,
.project = delta_strdup(ctx->project),
.label = delta_strdup(node->label),
.name = delta_strdup(node->name),
.qualified_name = delta_strdup(node->qualified_name),
.file_path = delta_strdup(node->file_path),
.start_line = node->start_line,
.end_line = node->end_line,
.properties_json = delta_strdup(node->properties_json ? node->properties_json : "{}"),
};
if (!row.project || !row.label || !row.name || !row.qualified_name || !row.file_path ||
!row.properties_json) {
cbm_node_free_fields(&row);
cbm_node_t row = {.project = ctx->project,
.label = node->label,
.name = node->name,
.qualified_name = node->qualified_name,
.file_path = node->file_path,
.start_line = node->start_line,
.end_line = node->end_line,
.properties_json = node->properties_json};
if (cbm_pipeline_copy_delta_node(
&row, &ctx->out->context_nodes[ctx->out->delta.context_node_count]) !=
CBM_STORE_OK) {
return CBM_STORE_ERR;
}
ctx->out->context_nodes[ctx->out->delta.context_node_count++] = row;
ctx->out->delta.context_node_count++;
return CBM_STORE_OK;
}
@@ -1621,11 +1656,12 @@ int cbm_pipeline_apply_file_delta_batch(cbm_store_t *store,
return CBM_STORE_OK;
}
int cbm_pipeline_publish_overlay_file_delta_batch(cbm_store_t *store,
const cbm_pipeline_file_delta_t *const *deltas,
int delta_count, int64_t base_generation,
const char *dirty_source,
int64_t *out_overlay_generation) {
static int pipeline_publish_overlay_file_delta_batch(cbm_store_t *store,
const cbm_pipeline_file_delta_t *const *deltas,
int delta_count, int64_t base_generation,
const char *dirty_source,
int64_t *out_overlay_generation,
bool replace_files) {
if (out_overlay_generation) {
*out_overlay_generation = 0;
}
@@ -1675,8 +1711,11 @@ int cbm_pipeline_publish_overlay_file_delta_batch(cbm_store_t *store,
store_deltas[i] = &deltas[i]->delta;
}
rc = cbm_store_publish_overlay_file_delta_batch(store, store_deltas, delta_count,
overlay_generation);
rc = replace_files ? cbm_store_publish_overlay_file_delta_batch(store, store_deltas,
delta_count,
overlay_generation)
: cbm_store_publish_overlay_file_delta_additions_batch(
store, store_deltas, delta_count, overlay_generation);
free(store_deltas);
if (rc != CBM_STORE_OK) {
(void)cbm_store_set_overlay_generation_status(store, project, overlay_generation,
@@ -1713,6 +1752,22 @@ int cbm_pipeline_publish_overlay_file_delta_batch(cbm_store_t *store,
return CBM_STORE_OK;
}
int cbm_pipeline_publish_overlay_file_delta_batch(cbm_store_t *store,
const cbm_pipeline_file_delta_t *const *deltas,
int delta_count, int64_t base_generation,
const char *dirty_source,
int64_t *out_overlay_generation) {
return pipeline_publish_overlay_file_delta_batch(store, deltas, delta_count, base_generation,
dirty_source, out_overlay_generation, true);
}
int cbm_pipeline_publish_overlay_file_delta_additions_batch(
cbm_store_t *store, const cbm_pipeline_file_delta_t *const *deltas, int delta_count,
int64_t base_generation, const char *dirty_source, int64_t *out_overlay_generation) {
return pipeline_publish_overlay_file_delta_batch(store, deltas, delta_count, base_generation,
dirty_source, out_overlay_generation, false);
}
void cbm_pipeline_file_delta_plan_free(cbm_pipeline_file_delta_plan_t *plan) {
if (!plan) {
return;
src/pipeline/pipeline_incremental.c
+158 -34
View File
@@ -133,6 +133,103 @@ static bool incr_header_overlay_has_type_impl_pair(const cbm_file_info_t *file,
return incr_file_delta_has_type_like_node(delta) && incr_same_stem_impl_exists(file->path);
}
static bool incr_delta_node_qn_present(const cbm_node_t *nodes, int count, const char *qn) {
if (!nodes || count <= 0 || !qn) {
return false;
}
for (int i = 0; i < count; i++) {
if (nodes[i].qualified_name && strcmp(nodes[i].qualified_name, qn) == 0) {
return true;
}
}
return false;
}
static void incr_free_additive_delta(cbm_pipeline_file_delta_t *delta) {
cbm_pipeline_file_delta_free(delta);
}
static int incr_build_additive_delta(const cbm_pipeline_file_delta_t *src,
cbm_pipeline_file_delta_t *out) {
if (!src || !out || src->change_kind != CBM_PIPELINE_DELTA_CHANGE_UPSERT) {
return CBM_STORE_ERR;
}
memset(out, 0, sizeof(*out));
out->delta = (cbm_store_file_delta_t){
.project = src->delta.project,
.rel_path = src->delta.rel_path,
.generation = src->delta.generation,
.file_hash = src->delta.file_hash,
.file_state = src->delta.file_state,
.derived_view_name = src->delta.derived_view_name,
.derived_status = src->delta.derived_status,
};
out->change_kind = src->change_kind;
if (src->delta.node_count > 0) {
out->nodes = calloc((size_t)src->delta.node_count, sizeof(*out->nodes));
if (!out->nodes) {
return CBM_STORE_ERR;
}
}
for (int i = 0; i < src->delta.node_count; i++) {
const cbm_node_t *node = &src->delta.nodes[i];
int rc = cbm_pipeline_copy_delta_node(node, &out->nodes[out->delta.node_count]);
if (rc != CBM_STORE_OK) {
incr_free_additive_delta(out);
return rc;
}
out->delta.node_count++;
}
if (out->delta.node_count == 0) {
incr_free_additive_delta(out);
return CBM_STORE_NOT_FOUND;
}
if (src->delta.edge_count > 0) {
out->edges = calloc((size_t)src->delta.edge_count, sizeof(*out->edges));
if (!out->edges) {
incr_free_additive_delta(out);
return CBM_STORE_ERR;
}
}
for (int i = 0; i < src->delta.edge_count; i++) {
const cbm_store_delta_edge_t *edge = &src->delta.edges[i];
if (!incr_delta_node_qn_present(out->nodes, out->delta.node_count, edge->source_qn) &&
!incr_delta_node_qn_present(out->nodes, out->delta.node_count, edge->target_qn)) {
continue;
}
int rc = cbm_pipeline_copy_delta_edge(edge, &out->edges[out->delta.edge_count]);
if (rc != CBM_STORE_OK) {
incr_free_additive_delta(out);
return rc;
}
out->delta.edge_count++;
}
if (src->delta.export_count > 0) {
out->exports = calloc((size_t)src->delta.export_count, sizeof(*out->exports));
if (!out->exports) {
incr_free_additive_delta(out);
return CBM_STORE_ERR;
}
}
for (int i = 0; i < src->delta.export_count; i++) {
const char *qn = src->delta.exports[i].qualified_name;
out->exports[out->delta.export_count].qualified_name = cbm_strdup(qn ? qn : "");
if (!out->exports[out->delta.export_count].qualified_name) {
incr_free_additive_delta(out);
return CBM_STORE_ERR;
}
out->delta.export_count++;
}
out->delta.nodes = out->nodes;
out->delta.edges = out->edges;
out->delta.exports = out->exports;
return CBM_STORE_OK;
}
/* ── File classification ─────────────────────────────────────────── */
/* Classify discovered files against stored metadata.
@@ -1483,10 +1580,8 @@ static int incr_try_overlay_upsert_route(cbm_pipeline_t *p, cbm_store_t *store,
cbm_pipeline_get_mode(p) < CBM_MODE_FAST || changed_count > max_affected_paths) {
return CBM_STORE_OK;
}
bool additive_header_overlay = false;
if (changed_count > 1 && incr_changed_contains_c_family_header(changed_files, changed_count)) {
/* Multi-file header batches can require importer and second-level
* caller expansion. Keep those on the exact/full path; single-header
* overlays are validated through the active overlay graph gate. */
return CBM_STORE_OK;
}
int rc = CBM_STORE_OK;
@@ -1495,18 +1590,23 @@ static int incr_try_overlay_upsert_route(cbm_pipeline_t *p, cbm_store_t *store,
cbm_registry_t *registry = NULL;
cbm_path_alias_collection_t *path_aliases = NULL;
cbm_pipeline_file_delta_t *deltas = NULL;
cbm_pipeline_file_delta_t *additive_deltas = NULL;
const cbm_pipeline_file_delta_t **delta_ptrs = NULL;
const cbm_pipeline_file_delta_t **additive_delta_ptrs = NULL;
CBMFileResult **result_cache = NULL;
int64_t base_generation = 0;
int64_t overlay_generation = 0;
changed_paths = malloc((size_t)changed_count * sizeof(*changed_paths));
deltas = calloc((size_t)changed_count, sizeof(*deltas));
additive_deltas = calloc((size_t)changed_count, sizeof(*additive_deltas));
delta_ptrs = malloc((size_t)changed_count * sizeof(*delta_ptrs));
additive_delta_ptrs = malloc((size_t)changed_count * sizeof(*additive_delta_ptrs));
result_cache = calloc((size_t)changed_count, sizeof(*result_cache));
scratch = cbm_gbuf_new(project, cbm_pipeline_repo_path(p));
registry = cbm_registry_new();
if (!changed_paths || !deltas || !delta_ptrs || !result_cache || !scratch || !registry) {
if (!changed_paths || !deltas || !additive_deltas || !delta_ptrs || !additive_delta_ptrs ||
!result_cache || !scratch || !registry) {
cbm_pipeline_set_publish_reason(p, "overlay_alloc");
cbm_log_info("incremental.overlay.fallback", "reason", "alloc");
goto cleanup;
@@ -1612,43 +1712,61 @@ static int incr_try_overlay_upsert_route(cbm_pipeline_t *p, cbm_store_t *store,
goto cleanup;
}
if (incr_header_overlay_has_type_impl_pair(&changed_files[i], &deltas[i])) {
cbm_pipeline_set_publish_reason(p, CBM_PIPELINE_DELTA_REASON_HEADER_TYPE_IMPL_PAIR);
cbm_log_info("incremental.overlay.fallback", "reason",
CBM_PIPELINE_DELTA_REASON_HEADER_TYPE_IMPL_PAIR);
goto cleanup;
additive_header_overlay = true;
}
int preserved = 0;
rc = cbm_pipeline_file_delta_add_preserved_inbound_edges(store, &deltas[i], &preserved);
if (rc != CBM_STORE_OK) {
cbm_pipeline_set_publish_reason(p, "overlay_preserve_inbound");
cbm_log_info("incremental.overlay.fallback", "reason", "preserve_inbound", "rc",
itoa_buf_incr(rc));
goto cleanup;
if (!additive_header_overlay) {
int preserved = 0;
rc = cbm_pipeline_file_delta_add_preserved_inbound_edges(store, &deltas[i], &preserved);
if (rc != CBM_STORE_OK) {
cbm_pipeline_set_publish_reason(p, "overlay_preserve_inbound");
cbm_log_info("incremental.overlay.fallback", "reason", "preserve_inbound", "rc",
itoa_buf_incr(rc));
goto cleanup;
}
}
delta_ptrs[i] = &deltas[i];
}
for (int i = 0; i < changed_count; i++) {
bool collision = false;
rc = cbm_pipeline_file_delta_has_cross_file_node_qn_collision(store, &deltas[i],
&collision);
if (rc != CBM_STORE_OK || collision) {
const char *reason =
collision ? CBM_PIPELINE_DELTA_REASON_CROSS_FILE_NODE_QN_COLLISION
: CBM_PIPELINE_DELTA_REASON_PREFLIGHT_ERROR;
cbm_pipeline_set_publish_reason(p, reason);
cbm_log_info("incremental.overlay.fallback", "reason", reason, "rc",
itoa_buf_incr(rc));
goto cleanup;
if (additive_header_overlay) {
for (int i = 0; i < changed_count; i++) {
rc = incr_build_additive_delta(&deltas[i], &additive_deltas[i]);
if (rc != CBM_STORE_OK) {
const char *reason = rc == CBM_STORE_NOT_FOUND ? "overlay_no_additive_facts"
: "overlay_additive_filter";
cbm_pipeline_set_publish_reason(p, reason);
cbm_log_info("incremental.overlay.fallback", "reason", reason, "rc",
itoa_buf_incr(rc));
goto cleanup;
}
additive_delta_ptrs[i] = &additive_deltas[i];
}
} else {
for (int i = 0; i < changed_count; i++) {
bool collision = false;
rc = cbm_pipeline_file_delta_has_cross_file_node_qn_collision(store, &deltas[i],
&collision);
if (rc != CBM_STORE_OK || collision) {
const char *reason =
collision ? CBM_PIPELINE_DELTA_REASON_CROSS_FILE_NODE_QN_COLLISION
: CBM_PIPELINE_DELTA_REASON_PREFLIGHT_ERROR;
cbm_pipeline_set_publish_reason(p, reason);
cbm_log_info("incremental.overlay.fallback", "reason", reason, "rc",
itoa_buf_incr(rc));
goto cleanup;
}
}
}
rc = cbm_store_latest_complete_index_generation(store, project, &base_generation);
if (rc == CBM_STORE_OK) {
CBM_PROF_START(t_overlay_publish);
rc = cbm_pipeline_publish_overlay_file_delta_batch(
store, delta_ptrs, changed_count, base_generation,
CBM_STORE_DIRTY_SOURCE_EXPLICIT_REINDEX, &overlay_generation);
rc = additive_header_overlay
? cbm_pipeline_publish_overlay_file_delta_additions_batch(
store, additive_delta_ptrs, changed_count, base_generation,
CBM_STORE_DIRTY_SOURCE_EXPLICIT_REINDEX, &overlay_generation)
: cbm_pipeline_publish_overlay_file_delta_batch(
store, delta_ptrs, changed_count, base_generation,
CBM_STORE_DIRTY_SOURCE_EXPLICIT_REINDEX, &overlay_generation);
CBM_PROF_END_N("incremental_overlay", "2_publish_overlay", t_overlay_publish,
changed_count);
}
@@ -1668,7 +1786,9 @@ static int incr_try_overlay_upsert_route(cbm_pipeline_t *p, cbm_store_t *store,
itoa_buf_incr(rc));
cleanup:
free(additive_delta_ptrs);
free(delta_ptrs);
incr_free_file_deltas(additive_deltas, changed_count);
incr_free_file_deltas(deltas, changed_count);
incr_free_result_cache(result_cache, changed_count);
cbm_path_alias_collection_free(path_aliases);
@@ -1937,7 +2057,7 @@ static int incr_try_exact_upsert_route(cbm_pipeline_t *p, cbm_store_t *store, co
bool overlay_publish_already_failed =
prior_reason && strcmp(prior_reason, "overlay_publish_error") == 0;
bool header_overlay_unsafe = incr_changed_contains_c_family_header(changed_files, changed_count);
if (!graph_noop_candidate && !header_overlay_unsafe &&
if (!graph_noop_candidate &&
cbm_pipeline_overlay_publish_small_deltas(p) &&
!overlay_publish_already_failed) {
int64_t base_generation = 0;
@@ -1945,9 +2065,13 @@ static int incr_try_exact_upsert_route(cbm_pipeline_t *p, cbm_store_t *store, co
rc = cbm_store_latest_complete_index_generation(store, project, &base_generation);
if (rc == CBM_STORE_OK) {
CBM_PROF_START(t_overlay_publish);
rc = cbm_pipeline_publish_overlay_file_delta_batch(
store, delta_ptrs, delta_count, base_generation,
CBM_STORE_DIRTY_SOURCE_EXPLICIT_REINDEX, &overlay_generation);
rc = header_overlay_unsafe
? cbm_pipeline_publish_overlay_file_delta_additions_batch(
store, delta_ptrs, delta_count, base_generation,
CBM_STORE_DIRTY_SOURCE_EXPLICIT_REINDEX, &overlay_generation)
: cbm_pipeline_publish_overlay_file_delta_batch(
store, delta_ptrs, delta_count, base_generation,
CBM_STORE_DIRTY_SOURCE_EXPLICIT_REINDEX, &overlay_generation);
CBM_PROF_END_N("incremental_exact", "11_publish_overlay", t_overlay_publish,
delta_count);
}
src/pipeline/pipeline_internal.h
+6
View File
@@ -442,6 +442,9 @@ int cbm_pipeline_build_file_delta_from_gbuf(const cbm_gbuf_t *gbuf, const char *
const char *rel_path, int64_t generation,
cbm_pipeline_file_delta_t *out);
bool cbm_pipeline_delta_edge_type_is_recomputed(const char *type);
int cbm_pipeline_copy_delta_node(const cbm_node_t *src, cbm_node_t *dst);
int cbm_pipeline_copy_delta_edge(const cbm_store_delta_edge_t *src,
cbm_store_delta_edge_t *dst);
int cbm_pipeline_file_delta_has_cross_file_node_qn_collision(
cbm_store_t *store, const cbm_pipeline_file_delta_t *delta, bool *out_collision);
int cbm_pipeline_file_delta_add_preserved_inbound_edges(cbm_store_t *store,
@@ -473,6 +476,9 @@ int cbm_pipeline_publish_overlay_file_delta_batch(cbm_store_t *store,
int delta_count, int64_t base_generation,
const char *dirty_source,
int64_t *out_overlay_generation);
int cbm_pipeline_publish_overlay_file_delta_additions_batch(
cbm_store_t *store, const cbm_pipeline_file_delta_t *const *deltas, int delta_count,
int64_t base_generation, const char *dirty_source, int64_t *out_overlay_generation);
void cbm_pipeline_file_delta_plan_free(cbm_pipeline_file_delta_plan_t *plan);
/* Seed a scratch graph with persisted unchanged nodes needed by import and
src/store/store.c
+49 -109
View File
@@ -6518,60 +6518,39 @@ int cbm_store_get_overlay_node_view_summary(cbm_store_t *s, const char *project,
return CBM_STORE_ERR;
}
static const char sql[] =
"WITH active_overlay_files AS ("
" SELECT rel_path, MAX(overlay_generation) AS overlay_generation"
" FROM ("
" SELECT n.rel_path, n.overlay_generation"
" FROM overlay_nodes n"
" JOIN overlay_generations g"
" ON g.project = n.project AND g.overlay_generation = n.overlay_generation"
" WHERE n.project = ?1 AND g.status = ?2"
" UNION"
" SELECT e.rel_path, e.overlay_generation"
" FROM overlay_edges e"
" JOIN overlay_generations g"
" ON g.project = e.project AND g.overlay_generation = e.overlay_generation"
" WHERE e.project = ?1 AND g.status = ?2"
" UNION"
" SELECT t.rel_path, t.overlay_generation"
" FROM overlay_tombstones t"
" JOIN overlay_generations g"
" ON g.project = t.project AND g.overlay_generation = t.overlay_generation"
" WHERE t.project = ?1 AND g.status = ?2 AND t.active = ?4"
" ) overlay_files"
" GROUP BY rel_path"
"), active_file_tombstones AS ("
" SELECT t.rel_path, MAX(t.overlay_generation) AS overlay_generation"
" FROM overlay_tombstones t"
" JOIN overlay_generations g"
" ON g.project = t.project AND g.overlay_generation = t.overlay_generation"
" WHERE t.project = ?1 AND g.status = ?2 AND t.entity_kind = ?3 AND t.active = ?4"
" GROUP BY t.rel_path"
")"
"SELECT "
" (SELECT COUNT(*) FROM overlay_generations g"
" WHERE g.project = ?1 AND g.status = ?2) AS ready_generations,"
" (SELECT COUNT(*) FROM active_file_tombstones) AS active_files,"
" (SELECT COUNT(*) FROM nodes n"
" WHERE n.project = ?1"
" AND NOT EXISTS (SELECT 1 FROM active_file_tombstones af"
" WHERE af.rel_path = n.file_path)) AS canonical_visible,"
" (SELECT COUNT(*) FROM overlay_nodes n"
" JOIN active_overlay_files af"
" ON af.rel_path = n.rel_path AND af.overlay_generation = n.overlay_generation"
" WHERE n.project = ?1 AND n.owned = ?5) AS overlay_visible;";
char active_cte[ST_SQL_BUF];
if (cbm_store_build_active_overlay_cte(active_cte, sizeof(active_cte), false, false) !=
CBM_STORE_OK) {
store_set_error(s, "overlay_node_view_summary active CTE SQL truncated");
return CBM_STORE_ERR;
}
char sql[ST_SQL_BUF];
int n = snprintf(sql, sizeof(sql),
"%s"
"SELECT "
" (SELECT COUNT(*) FROM overlay_generations g"
" WHERE g.project = ?3 AND g.status = ?1) AS ready_generations,"
" (SELECT COUNT(*) FROM active_file_tombstones af"
" WHERE af.project = ?3) AS active_files,"
" (SELECT COUNT(*) FROM active_nodes n"
" WHERE n.project = ?3 AND n.id <> ?4) AS canonical_visible,"
" (SELECT COUNT(*) FROM active_nodes n"
" WHERE n.project = ?3 AND n.id = ?4) AS overlay_visible;",
active_cte);
if (n < 0 || (size_t)n >= sizeof(sql)) {
store_set_error(s, "overlay_node_view_summary SQL truncated");
return CBM_STORE_ERR;
}
sqlite3_stmt *stmt = NULL;
if (sqlite3_prepare_v2(s->db, sql, CBM_NOT_FOUND, &stmt, NULL) != SQLITE_OK) {
store_set_error_sqlite(s, "overlay_node_view_summary prepare");
return CBM_STORE_ERR;
}
bind_text(stmt, ST_COL_1, project);
bind_text(stmt, ST_COL_2, CBM_STORE_OVERLAY_STATUS_READY);
bind_text(stmt, ST_COL_3, CBM_STORE_OVERLAY_TOMBSTONE_FILE);
sqlite3_bind_int(stmt, ST_COL_4, STORE_OVERLAY_TOMBSTONE_ACTIVE);
sqlite3_bind_int(stmt, ST_COL_5, STORE_OVERLAY_ROW_OWNED);
bind_text(stmt, ST_COL_1, CBM_STORE_OVERLAY_STATUS_READY);
bind_text(stmt, ST_COL_2, CBM_STORE_OVERLAY_TOMBSTONE_FILE);
bind_text(stmt, ST_COL_3, project);
sqlite3_bind_int64(stmt, ST_COL_4, CBM_STORE_NO_NODE_ID);
int rc = sqlite3_step(stmt);
if (rc != SQLITE_ROW) {
@@ -6590,49 +6569,6 @@ int cbm_store_get_overlay_node_view_summary(cbm_store_t *s, const char *project,
return CBM_STORE_OK;
}
static int store_latest_ready_overlay_for_file(cbm_store_t *s, const char *project,
const char *file_path,
int64_t *out_overlay_generation) {
if (out_overlay_generation) {
*out_overlay_generation = 0;
}
if (!s || !s->db || !project || !project[0] || !file_path || !file_path[0] ||
!out_overlay_generation) {
if (s) {
store_set_error(s, "latest_ready_overlay_for_file: invalid argument");
}
return CBM_STORE_ERR;
}
static const char sql[] =
"SELECT MAX(t.overlay_generation) "
"FROM overlay_tombstones t "
"JOIN overlay_generations g "
" ON g.project = t.project AND g.overlay_generation = t.overlay_generation "
"WHERE t.project = ?1 AND t.rel_path = ?2 AND g.status = ?3 "
" AND t.entity_kind = ?4 AND t.active = ?5;";
sqlite3_stmt *stmt = NULL;
if (sqlite3_prepare_v2(s->db, sql, CBM_NOT_FOUND, &stmt, NULL) != SQLITE_OK) {
store_set_error_sqlite(s, "latest_ready_overlay_for_file prepare");
return CBM_STORE_ERR;
}
bind_text(stmt, ST_COL_1, project);
bind_text(stmt, ST_COL_2, file_path);
bind_text(stmt, ST_COL_3, CBM_STORE_OVERLAY_STATUS_READY);
bind_text(stmt, ST_COL_4, CBM_STORE_OVERLAY_TOMBSTONE_FILE);
sqlite3_bind_int(stmt, ST_COL_5, STORE_OVERLAY_TOMBSTONE_ACTIVE);
int rc = sqlite3_step(stmt);
if (rc == SQLITE_ROW) {
if (sqlite3_column_type(stmt, 0) != SQLITE_NULL) {
*out_overlay_generation = sqlite3_column_int64(stmt, 0);
}
sqlite3_finalize(stmt);
return CBM_STORE_OK;
}
sqlite3_finalize(stmt);
store_set_error_sqlite(s, "latest_ready_overlay_for_file");
return CBM_STORE_ERR;
}
int cbm_store_find_nodes_by_file_overlay_view(cbm_store_t *s, const char *project,
const char *file_path, cbm_node_t **out,
int *count) {
@@ -6650,31 +6586,35 @@ int cbm_store_find_nodes_by_file_overlay_view(cbm_store_t *s, const char *projec
return CBM_STORE_ERR;
}
int64_t overlay_generation = 0;
int rc = store_latest_ready_overlay_for_file(s, project, file_path, &overlay_generation);
if (rc != CBM_STORE_OK) {
return rc;
char active_cte[ST_SQL_BUF];
if (cbm_store_build_active_overlay_cte(active_cte, sizeof(active_cte), false, false) !=
CBM_STORE_OK) {
store_set_error(s, "find_nodes_by_file_overlay active CTE SQL truncated");
return CBM_STORE_ERR;
}
if (overlay_generation <= 0) {
return cbm_store_find_nodes_by_file(s, project, file_path, out, count);
char sql[ST_SQL_BUF];
int n = snprintf(sql, sizeof(sql),
"%s"
"SELECT n.id, n.project, n.label, n.name, n.qualified_name, "
"n.file_path, n.start_line, n.end_line, n.properties "
"FROM active_nodes n "
"WHERE n.project = ?3 AND n.file_path = ?4 "
"ORDER BY n.name, n.qualified_name;",
active_cte);
if (n < 0 || (size_t)n >= sizeof(sql)) {
store_set_error(s, "find_nodes_by_file_overlay SQL truncated");
return CBM_STORE_ERR;
}
static const char sql[] =
"SELECT ?4 AS id, project, label, name, qualified_name, file_path, start_line, end_line, "
"properties FROM overlay_nodes "
"WHERE project = ?1 AND overlay_generation = ?2 AND rel_path = ?3 AND owned = ?5 "
"ORDER BY name, qualified_name;";
sqlite3_stmt *stmt = NULL;
if (sqlite3_prepare_v2(s->db, sql, CBM_NOT_FOUND, &stmt, NULL) != SQLITE_OK) {
store_set_error_sqlite(s, "find_nodes_by_file_overlay_view prepare");
return CBM_STORE_ERR;
}
bind_text(stmt, ST_COL_1, project);
sqlite3_bind_int64(stmt, ST_COL_2, overlay_generation);
bind_text(stmt, ST_COL_3, file_path);
sqlite3_bind_int64(stmt, ST_COL_4, CBM_STORE_NO_NODE_ID);
sqlite3_bind_int(stmt, ST_COL_5, STORE_OVERLAY_ROW_OWNED);
rc = collect_nodes_from_stmt(s, stmt, "find_nodes_by_file_overlay_view", out, count);
bind_text(stmt, ST_COL_1, CBM_STORE_OVERLAY_STATUS_READY);
bind_text(stmt, ST_COL_2, CBM_STORE_OVERLAY_TOMBSTONE_FILE);
bind_text(stmt, ST_COL_3, project);
bind_text(stmt, ST_COL_4, file_path);
int rc = collect_nodes_from_stmt(s, stmt, "find_nodes_by_file_overlay_view", out, count);
sqlite3_finalize(stmt);
return rc;
}
tests/test_pipeline.c
+37 -20
View File
@@ -10212,7 +10212,7 @@ TEST(incremental_overlay_publish_single_c_header_uses_active_overlay) {
PASS();
}
TEST(incremental_overlay_single_c_header_type_impl_pair_falls_back) {
TEST(incremental_overlay_single_c_header_type_impl_pair_keeps_canonical_rows_visible) {
if (setup_incremental_repo() != 0) {
FAIL("setup failed");
}
@@ -10249,7 +10249,9 @@ TEST(incremental_overlay_single_c_header_type_impl_pair_falls_back) {
ASSERT_NOT_NULL(p);
cbm_pipeline_apply_config(p, cfg);
ASSERT_EQ(cbm_pipeline_run(p), 0);
char *project = cbm_strdup(cbm_pipeline_project_name(p));
cbm_pipeline_free(p);
ASSERT_NOT_NULL(project);
ASSERT_EQ(th_write_file(header_path,
"#ifndef PAIRED_H\n"
@@ -10269,17 +10271,30 @@ TEST(incremental_overlay_single_c_header_type_impl_pair_falls_back) {
int run_rc = cbm_pipeline_run(p);
const char *logs = pipeline_capture_logs_end();
ASSERT_EQ(run_rc, 0);
ASSERT(strstr(logs, "msg=incremental.overlay.fallback reason="
CBM_PIPELINE_DELTA_REASON_HEADER_TYPE_IMPL_PAIR) != NULL);
ASSERT_EQ(cbm_pipeline_publish_kind(p), CBM_PIPELINE_PUBLISH_INCREMENTAL_EXACT);
ASSERT(strstr(logs, "msg=incremental.overlay.done files=1") != NULL);
ASSERT(strstr(logs, CBM_PIPELINE_DELTA_REASON_HEADER_TYPE_IMPL_PAIR) == NULL);
ASSERT_EQ(cbm_pipeline_publish_kind(p), CBM_PIPELINE_PUBLISH_INCREMENTAL_OVERLAY);
cbm_pipeline_free(p);
ASSERT(pipeline_store_has_function_name(g_incr_dbpath, project, "paired_value"));
ASSERT(!pipeline_store_has_function_name(g_incr_dbpath, project, "paired_extra"));
ASSERT(pipeline_store_overlay_file_has_function(g_incr_dbpath, project, "paired.h",
"paired_extra"));
int dirty_pending = -1;
int dirty_overlay_ready = -1;
ASSERT_EQ(pipeline_store_dirty_counts(g_incr_dbpath, project, &dirty_pending,
&dirty_overlay_ready),
CBM_STORE_OK);
ASSERT_EQ(dirty_pending, 0);
ASSERT_EQ(dirty_overlay_ready, 1);
free(project);
cbm_config_close(cfg);
cleanup_incremental_repo();
PASS();
}
TEST(incremental_c_header_uses_exact_not_overlay_until_active_edges_are_exact) {
TEST(incremental_c_header_uses_exact_frontier_overlay_for_active_edges) {
enum {
PIPELINE_C_HEADER_OVERLAY_MAX_AFFECTED = CBM_SZ_16,
PIPELINE_C_HEADER_AFFECTED_FRONTIER =
@@ -10322,9 +10337,6 @@ TEST(incremental_c_header_uses_exact_not_overlay_until_active_edges_are_exact) {
int run_rc = cbm_pipeline_run(p);
const char *logs = pipeline_capture_logs_end();
ASSERT_EQ(run_rc, 0);
if (strstr(logs, "msg=incremental.overlay.done files=") != NULL) {
FAIL(logs);
}
ASSERT(strstr(logs, "msg=incremental.classify changed=2") != NULL);
char frontier_log[CBM_SZ_128];
int log_n = snprintf(frontier_log, sizeof(frontier_log),
@@ -10332,22 +10344,27 @@ TEST(incremental_c_header_uses_exact_not_overlay_until_active_edges_are_exact) {
PIPELINE_C_HEADER_AFFECTED_FRONTIER);
ASSERT(log_n >= 0 && (size_t)log_n < sizeof(frontier_log));
ASSERT(strstr(logs, frontier_log) != NULL);
ASSERT(strstr(logs, "msg=incremental.exact.done files=") != NULL);
ASSERT_EQ(cbm_pipeline_publish_kind(p), CBM_PIPELINE_PUBLISH_INCREMENTAL_EXACT);
ASSERT(cbm_pipeline_graph_changed(p));
char overlay_log[CBM_SZ_128];
log_n = snprintf(overlay_log, sizeof(overlay_log),
"msg=incremental.overlay.done files=%d",
PIPELINE_C_HEADER_AFFECTED_FRONTIER);
ASSERT(log_n >= 0 && (size_t)log_n < sizeof(overlay_log));
ASSERT(strstr(logs, overlay_log) != NULL);
ASSERT_EQ(cbm_pipeline_publish_kind(p), CBM_PIPELINE_PUBLISH_INCREMENTAL_OVERLAY);
ASSERT(!cbm_pipeline_graph_changed(p));
cbm_pipeline_exact_delta_stats_t stats = cbm_pipeline_exact_delta_stats(p);
ASSERT_EQ(stats.changed_paths, CBM_SZ_2);
ASSERT_EQ(stats.affected_paths, PIPELINE_C_HEADER_AFFECTED_FRONTIER);
ASSERT_EQ(stats.published_paths, PIPELINE_C_HEADER_AFFECTED_FRONTIER);
cbm_pipeline_free(p);
char diff_err[CBM_SZ_8K] = {0};
int diff_rc = pipeline_compare_current_db_to_fresh_fast_rebuild(
g_incr_tmpdir, g_incr_dbpath, project, cfg, diff_err, sizeof(diff_err));
if (diff_rc != 0) {
FAIL(diff_err[0] ? diff_err : "C header exact update differed from fresh rebuild");
}
ASSERT_EQ(diff_rc, 0);
int dirty_pending = -1;
int dirty_overlay_ready = -1;
ASSERT_EQ(pipeline_store_dirty_counts(g_incr_dbpath, project, &dirty_pending,
&dirty_overlay_ready),
CBM_STORE_OK);
ASSERT_EQ(dirty_pending, 0);
ASSERT_EQ(dirty_overlay_ready, PIPELINE_C_HEADER_AFFECTED_FRONTIER);
free(project);
cbm_config_close(cfg);
@@ -13828,8 +13845,8 @@ SUITE(pipeline) {
RUN_TEST(incremental_fast_falls_back_for_oversized_inbound_frontier_and_matches_full);
RUN_TEST(incremental_fast_c_header_frontier_too_large_uses_full_rebuild);
RUN_TEST(incremental_overlay_publish_single_c_header_uses_active_overlay);
RUN_TEST(incremental_overlay_single_c_header_type_impl_pair_falls_back);
RUN_TEST(incremental_c_header_uses_exact_not_overlay_until_active_edges_are_exact);
RUN_TEST(incremental_overlay_single_c_header_type_impl_pair_keeps_canonical_rows_visible);
RUN_TEST(incremental_c_header_uses_exact_frontier_overlay_for_active_edges);
RUN_TEST(incremental_fast_configured_frontier_cap_allows_bounded_exact);
RUN_TEST(incremental_fast_mixed_unowned_edge_frontier_falls_back_before_exact_build);
RUN_TEST(incremental_fast_expands_small_inbound_frontier_and_matches_full);
tests/test_store_nodes.c
+24 -13
View File
@@ -1897,19 +1897,29 @@ TEST(store_overlay_additions_keep_canonical_file_rows_visible) {
ASSERT_EQ(cbm_store_reserve_overlay_generation(s, "test", BASE_GENERATION,
&overlay_generation),
CBM_STORE_OK);
cbm_node_t added = {.project = "test",
.label = "Function",
.name = "added",
.qualified_name = "test.added",
.file_path = "main.h",
.start_line = 5,
.end_line = 7,
.properties_json = "{}"};
cbm_node_t overlay_nodes[] = {
{.project = "test",
.label = "Function",
.name = "stable",
.qualified_name = "test.stable",
.file_path = "main.h",
.start_line = 1,
.end_line = 6,
.properties_json = "{}"},
{.project = "test",
.label = "Function",
.name = "added",
.qualified_name = "test.added",
.file_path = "main.h",
.start_line = 8,
.end_line = 10,
.properties_json = "{}"},
};
cbm_store_file_delta_t delta = {.project = "test",
.rel_path = "main.h",
.generation = BASE_GENERATION,
.nodes = &added,
.node_count = 1};
.nodes = overlay_nodes,
.node_count = 2};
const cbm_store_file_delta_t *deltas[] = {&delta};
ASSERT_EQ(cbm_store_publish_overlay_file_delta_additions_batch(s, deltas, 1,
overlay_generation),
@@ -1922,19 +1932,20 @@ TEST(store_overlay_additions_keep_canonical_file_rows_visible) {
ASSERT_EQ(cbm_store_get_overlay_node_view_summary(s, "test", &summary), CBM_STORE_OK);
ASSERT_EQ(summary.overlay_ready_generations, 1);
ASSERT_EQ(summary.active_file_tombstones, 0);
ASSERT_EQ(summary.canonical_nodes_visible, 1);
ASSERT_EQ(summary.overlay_owned_nodes_visible, 1);
ASSERT_EQ(summary.canonical_nodes_visible, 0);
ASSERT_EQ(summary.overlay_owned_nodes_visible, 2);
ASSERT_EQ(summary.total_nodes_visible, 2);
cbm_node_t found = {0};
ASSERT_EQ(cbm_store_find_node_by_qn_overlay_view(s, "test", "test.stable", &found),
CBM_STORE_OK);
ASSERT_STR_EQ(found.file_path, "main.h");
ASSERT_EQ(found.end_line, 6);
cbm_node_free_fields(&found);
ASSERT_EQ(cbm_store_find_node_by_qn_overlay_view(s, "test", "test.added", &found),
CBM_STORE_OK);
ASSERT_STR_EQ(found.file_path, "main.h");
ASSERT_EQ(found.start_line, 5);
ASSERT_EQ(found.start_line, 8);
cbm_node_free_fields(&found);
cbm_store_close(s);