diff --git a/src/pipeline/pipeline_delta.c b/src/pipeline/pipeline_delta.c index 5db710357..5744573bf 100644 --- a/src/pipeline/pipeline_delta.c +++ b/src/pipeline/pipeline_delta.c @@ -277,6 +277,48 @@ static void delta_edge_free_fields(cbm_store_delta_edge_t *edge) { *edge = (cbm_store_delta_edge_t){0}; } +int cbm_pipeline_copy_delta_node(const cbm_node_t *src, cbm_node_t *dst) { + if (!src || !dst) { + return CBM_STORE_ERR; + } + *dst = (cbm_node_t){ + .id = CBM_STORE_NO_NODE_ID, + .project = delta_strdup(src->project), + .label = delta_strdup(src->label), + .name = delta_strdup(src->name), + .qualified_name = delta_strdup(src->qualified_name), + .file_path = delta_strdup(src->file_path), + .start_line = src->start_line, + .end_line = src->end_line, + .properties_json = delta_strdup(src->properties_json ? src->properties_json : "{}"), + }; + if (!dst->project || !dst->label || !dst->name || !dst->qualified_name || + !dst->file_path || !dst->properties_json) { + cbm_node_free_fields(dst); + return CBM_STORE_ERR; + } + return CBM_STORE_OK; +} + +int cbm_pipeline_copy_delta_edge(const cbm_store_delta_edge_t *src, + cbm_store_delta_edge_t *dst) { + if (!src || !dst) { + return CBM_STORE_ERR; + } + *dst = (cbm_store_delta_edge_t){ + .source_qn = delta_strdup(src->source_qn), + .target_qn = delta_strdup(src->target_qn), + .type = delta_strdup(src->type), + .properties_json = delta_strdup(src->properties_json ? src->properties_json : "{}"), + .derived_kind = src->derived_kind ? src->derived_kind : CBM_STORE_DERIVED_KIND_DIRECT, + }; + if (!dst->source_qn || !dst->target_qn || !dst->type || !dst->properties_json) { + delta_edge_free_fields(dst); + return CBM_STORE_ERR; + } + return CBM_STORE_OK; +} + static void delta_import_free_fields(cbm_store_import_ref_t *import) { if (!import) { return; @@ -293,23 +335,19 @@ static int delta_append_node(cbm_delta_build_ctx_t *ctx, const cbm_gbuf_node_t * CBM_STORE_OK) { return CBM_STORE_ERR; } - cbm_node_t row = { - .id = CBM_STORE_NO_NODE_ID, - .project = delta_strdup(ctx->project), - .label = delta_strdup(node->label), - .name = delta_strdup(node->name), - .qualified_name = delta_strdup(node->qualified_name), - .file_path = delta_strdup(node->file_path), - .start_line = node->start_line, - .end_line = node->end_line, - .properties_json = delta_strdup(node->properties_json ? node->properties_json : "{}"), - }; - if (!row.project || !row.label || !row.name || !row.qualified_name || !row.file_path || - !row.properties_json) { - cbm_node_free_fields(&row); + cbm_node_t row = {.project = ctx->project, + .label = node->label, + .name = node->name, + .qualified_name = node->qualified_name, + .file_path = node->file_path, + .start_line = node->start_line, + .end_line = node->end_line, + .properties_json = node->properties_json}; + if (cbm_pipeline_copy_delta_node(&row, &ctx->out->nodes[ctx->out->delta.node_count]) != + CBM_STORE_OK) { return CBM_STORE_ERR; } - ctx->out->nodes[ctx->out->delta.node_count++] = row; + ctx->out->delta.node_count++; return CBM_STORE_OK; } @@ -338,23 +376,20 @@ static int delta_append_context_node(cbm_delta_build_ctx_t *ctx, sizeof(*ctx->out->context_nodes)) != CBM_STORE_OK) { return CBM_STORE_ERR; } - cbm_node_t row = { - .id = CBM_STORE_NO_NODE_ID, - .project = delta_strdup(ctx->project), - .label = delta_strdup(node->label), - .name = delta_strdup(node->name), - .qualified_name = delta_strdup(node->qualified_name), - .file_path = delta_strdup(node->file_path), - .start_line = node->start_line, - .end_line = node->end_line, - .properties_json = delta_strdup(node->properties_json ? node->properties_json : "{}"), - }; - if (!row.project || !row.label || !row.name || !row.qualified_name || !row.file_path || - !row.properties_json) { - cbm_node_free_fields(&row); + cbm_node_t row = {.project = ctx->project, + .label = node->label, + .name = node->name, + .qualified_name = node->qualified_name, + .file_path = node->file_path, + .start_line = node->start_line, + .end_line = node->end_line, + .properties_json = node->properties_json}; + if (cbm_pipeline_copy_delta_node( + &row, &ctx->out->context_nodes[ctx->out->delta.context_node_count]) != + CBM_STORE_OK) { return CBM_STORE_ERR; } - ctx->out->context_nodes[ctx->out->delta.context_node_count++] = row; + ctx->out->delta.context_node_count++; return CBM_STORE_OK; } @@ -1621,11 +1656,12 @@ int cbm_pipeline_apply_file_delta_batch(cbm_store_t *store, return CBM_STORE_OK; } -int cbm_pipeline_publish_overlay_file_delta_batch(cbm_store_t *store, - const cbm_pipeline_file_delta_t *const *deltas, - int delta_count, int64_t base_generation, - const char *dirty_source, - int64_t *out_overlay_generation) { +static int pipeline_publish_overlay_file_delta_batch(cbm_store_t *store, + const cbm_pipeline_file_delta_t *const *deltas, + int delta_count, int64_t base_generation, + const char *dirty_source, + int64_t *out_overlay_generation, + bool replace_files) { if (out_overlay_generation) { *out_overlay_generation = 0; } @@ -1675,8 +1711,11 @@ int cbm_pipeline_publish_overlay_file_delta_batch(cbm_store_t *store, store_deltas[i] = &deltas[i]->delta; } - rc = cbm_store_publish_overlay_file_delta_batch(store, store_deltas, delta_count, - overlay_generation); + rc = replace_files ? cbm_store_publish_overlay_file_delta_batch(store, store_deltas, + delta_count, + overlay_generation) + : cbm_store_publish_overlay_file_delta_additions_batch( + store, store_deltas, delta_count, overlay_generation); free(store_deltas); if (rc != CBM_STORE_OK) { (void)cbm_store_set_overlay_generation_status(store, project, overlay_generation, @@ -1713,6 +1752,22 @@ int cbm_pipeline_publish_overlay_file_delta_batch(cbm_store_t *store, return CBM_STORE_OK; } +int cbm_pipeline_publish_overlay_file_delta_batch(cbm_store_t *store, + const cbm_pipeline_file_delta_t *const *deltas, + int delta_count, int64_t base_generation, + const char *dirty_source, + int64_t *out_overlay_generation) { + return pipeline_publish_overlay_file_delta_batch(store, deltas, delta_count, base_generation, + dirty_source, out_overlay_generation, true); +} + +int cbm_pipeline_publish_overlay_file_delta_additions_batch( + cbm_store_t *store, const cbm_pipeline_file_delta_t *const *deltas, int delta_count, + int64_t base_generation, const char *dirty_source, int64_t *out_overlay_generation) { + return pipeline_publish_overlay_file_delta_batch(store, deltas, delta_count, base_generation, + dirty_source, out_overlay_generation, false); +} + void cbm_pipeline_file_delta_plan_free(cbm_pipeline_file_delta_plan_t *plan) { if (!plan) { return; diff --git a/src/pipeline/pipeline_incremental.c b/src/pipeline/pipeline_incremental.c index cf7931b7f..3a9a745c8 100644 --- a/src/pipeline/pipeline_incremental.c +++ b/src/pipeline/pipeline_incremental.c @@ -133,6 +133,103 @@ static bool incr_header_overlay_has_type_impl_pair(const cbm_file_info_t *file, return incr_file_delta_has_type_like_node(delta) && incr_same_stem_impl_exists(file->path); } +static bool incr_delta_node_qn_present(const cbm_node_t *nodes, int count, const char *qn) { + if (!nodes || count <= 0 || !qn) { + return false; + } + for (int i = 0; i < count; i++) { + if (nodes[i].qualified_name && strcmp(nodes[i].qualified_name, qn) == 0) { + return true; + } + } + return false; +} + +static void incr_free_additive_delta(cbm_pipeline_file_delta_t *delta) { + cbm_pipeline_file_delta_free(delta); +} + +static int incr_build_additive_delta(const cbm_pipeline_file_delta_t *src, + cbm_pipeline_file_delta_t *out) { + if (!src || !out || src->change_kind != CBM_PIPELINE_DELTA_CHANGE_UPSERT) { + return CBM_STORE_ERR; + } + memset(out, 0, sizeof(*out)); + out->delta = (cbm_store_file_delta_t){ + .project = src->delta.project, + .rel_path = src->delta.rel_path, + .generation = src->delta.generation, + .file_hash = src->delta.file_hash, + .file_state = src->delta.file_state, + .derived_view_name = src->delta.derived_view_name, + .derived_status = src->delta.derived_status, + }; + out->change_kind = src->change_kind; + + if (src->delta.node_count > 0) { + out->nodes = calloc((size_t)src->delta.node_count, sizeof(*out->nodes)); + if (!out->nodes) { + return CBM_STORE_ERR; + } + } + for (int i = 0; i < src->delta.node_count; i++) { + const cbm_node_t *node = &src->delta.nodes[i]; + int rc = cbm_pipeline_copy_delta_node(node, &out->nodes[out->delta.node_count]); + if (rc != CBM_STORE_OK) { + incr_free_additive_delta(out); + return rc; + } + out->delta.node_count++; + } + + if (out->delta.node_count == 0) { + incr_free_additive_delta(out); + return CBM_STORE_NOT_FOUND; + } + if (src->delta.edge_count > 0) { + out->edges = calloc((size_t)src->delta.edge_count, sizeof(*out->edges)); + if (!out->edges) { + incr_free_additive_delta(out); + return CBM_STORE_ERR; + } + } + for (int i = 0; i < src->delta.edge_count; i++) { + const cbm_store_delta_edge_t *edge = &src->delta.edges[i]; + if (!incr_delta_node_qn_present(out->nodes, out->delta.node_count, edge->source_qn) && + !incr_delta_node_qn_present(out->nodes, out->delta.node_count, edge->target_qn)) { + continue; + } + int rc = cbm_pipeline_copy_delta_edge(edge, &out->edges[out->delta.edge_count]); + if (rc != CBM_STORE_OK) { + incr_free_additive_delta(out); + return rc; + } + out->delta.edge_count++; + } + + if (src->delta.export_count > 0) { + out->exports = calloc((size_t)src->delta.export_count, sizeof(*out->exports)); + if (!out->exports) { + incr_free_additive_delta(out); + return CBM_STORE_ERR; + } + } + for (int i = 0; i < src->delta.export_count; i++) { + const char *qn = src->delta.exports[i].qualified_name; + out->exports[out->delta.export_count].qualified_name = cbm_strdup(qn ? qn : ""); + if (!out->exports[out->delta.export_count].qualified_name) { + incr_free_additive_delta(out); + return CBM_STORE_ERR; + } + out->delta.export_count++; + } + + out->delta.nodes = out->nodes; + out->delta.edges = out->edges; + out->delta.exports = out->exports; + return CBM_STORE_OK; +} + /* ── File classification ─────────────────────────────────────────── */ /* Classify discovered files against stored metadata. @@ -1483,10 +1580,8 @@ static int incr_try_overlay_upsert_route(cbm_pipeline_t *p, cbm_store_t *store, cbm_pipeline_get_mode(p) < CBM_MODE_FAST || changed_count > max_affected_paths) { return CBM_STORE_OK; } + bool additive_header_overlay = false; if (changed_count > 1 && incr_changed_contains_c_family_header(changed_files, changed_count)) { - /* Multi-file header batches can require importer and second-level - * caller expansion. Keep those on the exact/full path; single-header - * overlays are validated through the active overlay graph gate. */ return CBM_STORE_OK; } int rc = CBM_STORE_OK; @@ -1495,18 +1590,23 @@ static int incr_try_overlay_upsert_route(cbm_pipeline_t *p, cbm_store_t *store, cbm_registry_t *registry = NULL; cbm_path_alias_collection_t *path_aliases = NULL; cbm_pipeline_file_delta_t *deltas = NULL; + cbm_pipeline_file_delta_t *additive_deltas = NULL; const cbm_pipeline_file_delta_t **delta_ptrs = NULL; + const cbm_pipeline_file_delta_t **additive_delta_ptrs = NULL; CBMFileResult **result_cache = NULL; int64_t base_generation = 0; int64_t overlay_generation = 0; changed_paths = malloc((size_t)changed_count * sizeof(*changed_paths)); deltas = calloc((size_t)changed_count, sizeof(*deltas)); + additive_deltas = calloc((size_t)changed_count, sizeof(*additive_deltas)); delta_ptrs = malloc((size_t)changed_count * sizeof(*delta_ptrs)); + additive_delta_ptrs = malloc((size_t)changed_count * sizeof(*additive_delta_ptrs)); result_cache = calloc((size_t)changed_count, sizeof(*result_cache)); scratch = cbm_gbuf_new(project, cbm_pipeline_repo_path(p)); registry = cbm_registry_new(); - if (!changed_paths || !deltas || !delta_ptrs || !result_cache || !scratch || !registry) { + if (!changed_paths || !deltas || !additive_deltas || !delta_ptrs || !additive_delta_ptrs || + !result_cache || !scratch || !registry) { cbm_pipeline_set_publish_reason(p, "overlay_alloc"); cbm_log_info("incremental.overlay.fallback", "reason", "alloc"); goto cleanup; @@ -1612,43 +1712,61 @@ static int incr_try_overlay_upsert_route(cbm_pipeline_t *p, cbm_store_t *store, goto cleanup; } if (incr_header_overlay_has_type_impl_pair(&changed_files[i], &deltas[i])) { - cbm_pipeline_set_publish_reason(p, CBM_PIPELINE_DELTA_REASON_HEADER_TYPE_IMPL_PAIR); - cbm_log_info("incremental.overlay.fallback", "reason", - CBM_PIPELINE_DELTA_REASON_HEADER_TYPE_IMPL_PAIR); - goto cleanup; + additive_header_overlay = true; } - int preserved = 0; - rc = cbm_pipeline_file_delta_add_preserved_inbound_edges(store, &deltas[i], &preserved); - if (rc != CBM_STORE_OK) { - cbm_pipeline_set_publish_reason(p, "overlay_preserve_inbound"); - cbm_log_info("incremental.overlay.fallback", "reason", "preserve_inbound", "rc", - itoa_buf_incr(rc)); - goto cleanup; + if (!additive_header_overlay) { + int preserved = 0; + rc = cbm_pipeline_file_delta_add_preserved_inbound_edges(store, &deltas[i], &preserved); + if (rc != CBM_STORE_OK) { + cbm_pipeline_set_publish_reason(p, "overlay_preserve_inbound"); + cbm_log_info("incremental.overlay.fallback", "reason", "preserve_inbound", "rc", + itoa_buf_incr(rc)); + goto cleanup; + } } delta_ptrs[i] = &deltas[i]; } - for (int i = 0; i < changed_count; i++) { - bool collision = false; - rc = cbm_pipeline_file_delta_has_cross_file_node_qn_collision(store, &deltas[i], - &collision); - if (rc != CBM_STORE_OK || collision) { - const char *reason = - collision ? CBM_PIPELINE_DELTA_REASON_CROSS_FILE_NODE_QN_COLLISION - : CBM_PIPELINE_DELTA_REASON_PREFLIGHT_ERROR; - cbm_pipeline_set_publish_reason(p, reason); - cbm_log_info("incremental.overlay.fallback", "reason", reason, "rc", - itoa_buf_incr(rc)); - goto cleanup; + if (additive_header_overlay) { + for (int i = 0; i < changed_count; i++) { + rc = incr_build_additive_delta(&deltas[i], &additive_deltas[i]); + if (rc != CBM_STORE_OK) { + const char *reason = rc == CBM_STORE_NOT_FOUND ? "overlay_no_additive_facts" + : "overlay_additive_filter"; + cbm_pipeline_set_publish_reason(p, reason); + cbm_log_info("incremental.overlay.fallback", "reason", reason, "rc", + itoa_buf_incr(rc)); + goto cleanup; + } + additive_delta_ptrs[i] = &additive_deltas[i]; + } + } else { + for (int i = 0; i < changed_count; i++) { + bool collision = false; + rc = cbm_pipeline_file_delta_has_cross_file_node_qn_collision(store, &deltas[i], + &collision); + if (rc != CBM_STORE_OK || collision) { + const char *reason = + collision ? CBM_PIPELINE_DELTA_REASON_CROSS_FILE_NODE_QN_COLLISION + : CBM_PIPELINE_DELTA_REASON_PREFLIGHT_ERROR; + cbm_pipeline_set_publish_reason(p, reason); + cbm_log_info("incremental.overlay.fallback", "reason", reason, "rc", + itoa_buf_incr(rc)); + goto cleanup; + } } } rc = cbm_store_latest_complete_index_generation(store, project, &base_generation); if (rc == CBM_STORE_OK) { CBM_PROF_START(t_overlay_publish); - rc = cbm_pipeline_publish_overlay_file_delta_batch( - store, delta_ptrs, changed_count, base_generation, - CBM_STORE_DIRTY_SOURCE_EXPLICIT_REINDEX, &overlay_generation); + rc = additive_header_overlay + ? cbm_pipeline_publish_overlay_file_delta_additions_batch( + store, additive_delta_ptrs, changed_count, base_generation, + CBM_STORE_DIRTY_SOURCE_EXPLICIT_REINDEX, &overlay_generation) + : cbm_pipeline_publish_overlay_file_delta_batch( + store, delta_ptrs, changed_count, base_generation, + CBM_STORE_DIRTY_SOURCE_EXPLICIT_REINDEX, &overlay_generation); CBM_PROF_END_N("incremental_overlay", "2_publish_overlay", t_overlay_publish, changed_count); } @@ -1668,7 +1786,9 @@ static int incr_try_overlay_upsert_route(cbm_pipeline_t *p, cbm_store_t *store, itoa_buf_incr(rc)); cleanup: + free(additive_delta_ptrs); free(delta_ptrs); + incr_free_file_deltas(additive_deltas, changed_count); incr_free_file_deltas(deltas, changed_count); incr_free_result_cache(result_cache, changed_count); cbm_path_alias_collection_free(path_aliases); @@ -1937,7 +2057,7 @@ static int incr_try_exact_upsert_route(cbm_pipeline_t *p, cbm_store_t *store, co bool overlay_publish_already_failed = prior_reason && strcmp(prior_reason, "overlay_publish_error") == 0; bool header_overlay_unsafe = incr_changed_contains_c_family_header(changed_files, changed_count); - if (!graph_noop_candidate && !header_overlay_unsafe && + if (!graph_noop_candidate && cbm_pipeline_overlay_publish_small_deltas(p) && !overlay_publish_already_failed) { int64_t base_generation = 0; @@ -1945,9 +2065,13 @@ static int incr_try_exact_upsert_route(cbm_pipeline_t *p, cbm_store_t *store, co rc = cbm_store_latest_complete_index_generation(store, project, &base_generation); if (rc == CBM_STORE_OK) { CBM_PROF_START(t_overlay_publish); - rc = cbm_pipeline_publish_overlay_file_delta_batch( - store, delta_ptrs, delta_count, base_generation, - CBM_STORE_DIRTY_SOURCE_EXPLICIT_REINDEX, &overlay_generation); + rc = header_overlay_unsafe + ? cbm_pipeline_publish_overlay_file_delta_additions_batch( + store, delta_ptrs, delta_count, base_generation, + CBM_STORE_DIRTY_SOURCE_EXPLICIT_REINDEX, &overlay_generation) + : cbm_pipeline_publish_overlay_file_delta_batch( + store, delta_ptrs, delta_count, base_generation, + CBM_STORE_DIRTY_SOURCE_EXPLICIT_REINDEX, &overlay_generation); CBM_PROF_END_N("incremental_exact", "11_publish_overlay", t_overlay_publish, delta_count); } diff --git a/src/pipeline/pipeline_internal.h b/src/pipeline/pipeline_internal.h index 5eb8e999d..c41b50937 100644 --- a/src/pipeline/pipeline_internal.h +++ b/src/pipeline/pipeline_internal.h @@ -442,6 +442,9 @@ int cbm_pipeline_build_file_delta_from_gbuf(const cbm_gbuf_t *gbuf, const char * const char *rel_path, int64_t generation, cbm_pipeline_file_delta_t *out); bool cbm_pipeline_delta_edge_type_is_recomputed(const char *type); +int cbm_pipeline_copy_delta_node(const cbm_node_t *src, cbm_node_t *dst); +int cbm_pipeline_copy_delta_edge(const cbm_store_delta_edge_t *src, + cbm_store_delta_edge_t *dst); int cbm_pipeline_file_delta_has_cross_file_node_qn_collision( cbm_store_t *store, const cbm_pipeline_file_delta_t *delta, bool *out_collision); int cbm_pipeline_file_delta_add_preserved_inbound_edges(cbm_store_t *store, @@ -473,6 +476,9 @@ int cbm_pipeline_publish_overlay_file_delta_batch(cbm_store_t *store, int delta_count, int64_t base_generation, const char *dirty_source, int64_t *out_overlay_generation); +int cbm_pipeline_publish_overlay_file_delta_additions_batch( + cbm_store_t *store, const cbm_pipeline_file_delta_t *const *deltas, int delta_count, + int64_t base_generation, const char *dirty_source, int64_t *out_overlay_generation); void cbm_pipeline_file_delta_plan_free(cbm_pipeline_file_delta_plan_t *plan); /* Seed a scratch graph with persisted unchanged nodes needed by import and diff --git a/src/store/store.c b/src/store/store.c index 0daaa0395..30046dff4 100644 --- a/src/store/store.c +++ b/src/store/store.c @@ -6518,60 +6518,39 @@ int cbm_store_get_overlay_node_view_summary(cbm_store_t *s, const char *project, return CBM_STORE_ERR; } - static const char sql[] = - "WITH active_overlay_files AS (" - " SELECT rel_path, MAX(overlay_generation) AS overlay_generation" - " FROM (" - " SELECT n.rel_path, n.overlay_generation" - " FROM overlay_nodes n" - " JOIN overlay_generations g" - " ON g.project = n.project AND g.overlay_generation = n.overlay_generation" - " WHERE n.project = ?1 AND g.status = ?2" - " UNION" - " SELECT e.rel_path, e.overlay_generation" - " FROM overlay_edges e" - " JOIN overlay_generations g" - " ON g.project = e.project AND g.overlay_generation = e.overlay_generation" - " WHERE e.project = ?1 AND g.status = ?2" - " UNION" - " SELECT t.rel_path, t.overlay_generation" - " FROM overlay_tombstones t" - " JOIN overlay_generations g" - " ON g.project = t.project AND g.overlay_generation = t.overlay_generation" - " WHERE t.project = ?1 AND g.status = ?2 AND t.active = ?4" - " ) overlay_files" - " GROUP BY rel_path" - "), active_file_tombstones AS (" - " SELECT t.rel_path, MAX(t.overlay_generation) AS overlay_generation" - " FROM overlay_tombstones t" - " JOIN overlay_generations g" - " ON g.project = t.project AND g.overlay_generation = t.overlay_generation" - " WHERE t.project = ?1 AND g.status = ?2 AND t.entity_kind = ?3 AND t.active = ?4" - " GROUP BY t.rel_path" - ")" - "SELECT " - " (SELECT COUNT(*) FROM overlay_generations g" - " WHERE g.project = ?1 AND g.status = ?2) AS ready_generations," - " (SELECT COUNT(*) FROM active_file_tombstones) AS active_files," - " (SELECT COUNT(*) FROM nodes n" - " WHERE n.project = ?1" - " AND NOT EXISTS (SELECT 1 FROM active_file_tombstones af" - " WHERE af.rel_path = n.file_path)) AS canonical_visible," - " (SELECT COUNT(*) FROM overlay_nodes n" - " JOIN active_overlay_files af" - " ON af.rel_path = n.rel_path AND af.overlay_generation = n.overlay_generation" - " WHERE n.project = ?1 AND n.owned = ?5) AS overlay_visible;"; + char active_cte[ST_SQL_BUF]; + if (cbm_store_build_active_overlay_cte(active_cte, sizeof(active_cte), false, false) != + CBM_STORE_OK) { + store_set_error(s, "overlay_node_view_summary active CTE SQL truncated"); + return CBM_STORE_ERR; + } + char sql[ST_SQL_BUF]; + int n = snprintf(sql, sizeof(sql), + "%s" + "SELECT " + " (SELECT COUNT(*) FROM overlay_generations g" + " WHERE g.project = ?3 AND g.status = ?1) AS ready_generations," + " (SELECT COUNT(*) FROM active_file_tombstones af" + " WHERE af.project = ?3) AS active_files," + " (SELECT COUNT(*) FROM active_nodes n" + " WHERE n.project = ?3 AND n.id <> ?4) AS canonical_visible," + " (SELECT COUNT(*) FROM active_nodes n" + " WHERE n.project = ?3 AND n.id = ?4) AS overlay_visible;", + active_cte); + if (n < 0 || (size_t)n >= sizeof(sql)) { + store_set_error(s, "overlay_node_view_summary SQL truncated"); + return CBM_STORE_ERR; + } sqlite3_stmt *stmt = NULL; if (sqlite3_prepare_v2(s->db, sql, CBM_NOT_FOUND, &stmt, NULL) != SQLITE_OK) { store_set_error_sqlite(s, "overlay_node_view_summary prepare"); return CBM_STORE_ERR; } - bind_text(stmt, ST_COL_1, project); - bind_text(stmt, ST_COL_2, CBM_STORE_OVERLAY_STATUS_READY); - bind_text(stmt, ST_COL_3, CBM_STORE_OVERLAY_TOMBSTONE_FILE); - sqlite3_bind_int(stmt, ST_COL_4, STORE_OVERLAY_TOMBSTONE_ACTIVE); - sqlite3_bind_int(stmt, ST_COL_5, STORE_OVERLAY_ROW_OWNED); + bind_text(stmt, ST_COL_1, CBM_STORE_OVERLAY_STATUS_READY); + bind_text(stmt, ST_COL_2, CBM_STORE_OVERLAY_TOMBSTONE_FILE); + bind_text(stmt, ST_COL_3, project); + sqlite3_bind_int64(stmt, ST_COL_4, CBM_STORE_NO_NODE_ID); int rc = sqlite3_step(stmt); if (rc != SQLITE_ROW) { @@ -6590,49 +6569,6 @@ int cbm_store_get_overlay_node_view_summary(cbm_store_t *s, const char *project, return CBM_STORE_OK; } -static int store_latest_ready_overlay_for_file(cbm_store_t *s, const char *project, - const char *file_path, - int64_t *out_overlay_generation) { - if (out_overlay_generation) { - *out_overlay_generation = 0; - } - if (!s || !s->db || !project || !project[0] || !file_path || !file_path[0] || - !out_overlay_generation) { - if (s) { - store_set_error(s, "latest_ready_overlay_for_file: invalid argument"); - } - return CBM_STORE_ERR; - } - static const char sql[] = - "SELECT MAX(t.overlay_generation) " - "FROM overlay_tombstones t " - "JOIN overlay_generations g " - " ON g.project = t.project AND g.overlay_generation = t.overlay_generation " - "WHERE t.project = ?1 AND t.rel_path = ?2 AND g.status = ?3 " - " AND t.entity_kind = ?4 AND t.active = ?5;"; - sqlite3_stmt *stmt = NULL; - if (sqlite3_prepare_v2(s->db, sql, CBM_NOT_FOUND, &stmt, NULL) != SQLITE_OK) { - store_set_error_sqlite(s, "latest_ready_overlay_for_file prepare"); - return CBM_STORE_ERR; - } - bind_text(stmt, ST_COL_1, project); - bind_text(stmt, ST_COL_2, file_path); - bind_text(stmt, ST_COL_3, CBM_STORE_OVERLAY_STATUS_READY); - bind_text(stmt, ST_COL_4, CBM_STORE_OVERLAY_TOMBSTONE_FILE); - sqlite3_bind_int(stmt, ST_COL_5, STORE_OVERLAY_TOMBSTONE_ACTIVE); - int rc = sqlite3_step(stmt); - if (rc == SQLITE_ROW) { - if (sqlite3_column_type(stmt, 0) != SQLITE_NULL) { - *out_overlay_generation = sqlite3_column_int64(stmt, 0); - } - sqlite3_finalize(stmt); - return CBM_STORE_OK; - } - sqlite3_finalize(stmt); - store_set_error_sqlite(s, "latest_ready_overlay_for_file"); - return CBM_STORE_ERR; -} - int cbm_store_find_nodes_by_file_overlay_view(cbm_store_t *s, const char *project, const char *file_path, cbm_node_t **out, int *count) { @@ -6650,31 +6586,35 @@ int cbm_store_find_nodes_by_file_overlay_view(cbm_store_t *s, const char *projec return CBM_STORE_ERR; } - int64_t overlay_generation = 0; - int rc = store_latest_ready_overlay_for_file(s, project, file_path, &overlay_generation); - if (rc != CBM_STORE_OK) { - return rc; + char active_cte[ST_SQL_BUF]; + if (cbm_store_build_active_overlay_cte(active_cte, sizeof(active_cte), false, false) != + CBM_STORE_OK) { + store_set_error(s, "find_nodes_by_file_overlay active CTE SQL truncated"); + return CBM_STORE_ERR; } - if (overlay_generation <= 0) { - return cbm_store_find_nodes_by_file(s, project, file_path, out, count); + char sql[ST_SQL_BUF]; + int n = snprintf(sql, sizeof(sql), + "%s" + "SELECT n.id, n.project, n.label, n.name, n.qualified_name, " + "n.file_path, n.start_line, n.end_line, n.properties " + "FROM active_nodes n " + "WHERE n.project = ?3 AND n.file_path = ?4 " + "ORDER BY n.name, n.qualified_name;", + active_cte); + if (n < 0 || (size_t)n >= sizeof(sql)) { + store_set_error(s, "find_nodes_by_file_overlay SQL truncated"); + return CBM_STORE_ERR; } - - static const char sql[] = - "SELECT ?4 AS id, project, label, name, qualified_name, file_path, start_line, end_line, " - "properties FROM overlay_nodes " - "WHERE project = ?1 AND overlay_generation = ?2 AND rel_path = ?3 AND owned = ?5 " - "ORDER BY name, qualified_name;"; sqlite3_stmt *stmt = NULL; if (sqlite3_prepare_v2(s->db, sql, CBM_NOT_FOUND, &stmt, NULL) != SQLITE_OK) { store_set_error_sqlite(s, "find_nodes_by_file_overlay_view prepare"); return CBM_STORE_ERR; } - bind_text(stmt, ST_COL_1, project); - sqlite3_bind_int64(stmt, ST_COL_2, overlay_generation); - bind_text(stmt, ST_COL_3, file_path); - sqlite3_bind_int64(stmt, ST_COL_4, CBM_STORE_NO_NODE_ID); - sqlite3_bind_int(stmt, ST_COL_5, STORE_OVERLAY_ROW_OWNED); - rc = collect_nodes_from_stmt(s, stmt, "find_nodes_by_file_overlay_view", out, count); + bind_text(stmt, ST_COL_1, CBM_STORE_OVERLAY_STATUS_READY); + bind_text(stmt, ST_COL_2, CBM_STORE_OVERLAY_TOMBSTONE_FILE); + bind_text(stmt, ST_COL_3, project); + bind_text(stmt, ST_COL_4, file_path); + int rc = collect_nodes_from_stmt(s, stmt, "find_nodes_by_file_overlay_view", out, count); sqlite3_finalize(stmt); return rc; } diff --git a/tests/test_pipeline.c b/tests/test_pipeline.c index 0a4789e6a..0b85df1dd 100644 --- a/tests/test_pipeline.c +++ b/tests/test_pipeline.c @@ -10212,7 +10212,7 @@ TEST(incremental_overlay_publish_single_c_header_uses_active_overlay) { PASS(); } -TEST(incremental_overlay_single_c_header_type_impl_pair_falls_back) { +TEST(incremental_overlay_single_c_header_type_impl_pair_keeps_canonical_rows_visible) { if (setup_incremental_repo() != 0) { FAIL("setup failed"); } @@ -10249,7 +10249,9 @@ TEST(incremental_overlay_single_c_header_type_impl_pair_falls_back) { ASSERT_NOT_NULL(p); cbm_pipeline_apply_config(p, cfg); ASSERT_EQ(cbm_pipeline_run(p), 0); + char *project = cbm_strdup(cbm_pipeline_project_name(p)); cbm_pipeline_free(p); + ASSERT_NOT_NULL(project); ASSERT_EQ(th_write_file(header_path, "#ifndef PAIRED_H\n" @@ -10269,17 +10271,30 @@ TEST(incremental_overlay_single_c_header_type_impl_pair_falls_back) { int run_rc = cbm_pipeline_run(p); const char *logs = pipeline_capture_logs_end(); ASSERT_EQ(run_rc, 0); - ASSERT(strstr(logs, "msg=incremental.overlay.fallback reason=" - CBM_PIPELINE_DELTA_REASON_HEADER_TYPE_IMPL_PAIR) != NULL); - ASSERT_EQ(cbm_pipeline_publish_kind(p), CBM_PIPELINE_PUBLISH_INCREMENTAL_EXACT); + ASSERT(strstr(logs, "msg=incremental.overlay.done files=1") != NULL); + ASSERT(strstr(logs, CBM_PIPELINE_DELTA_REASON_HEADER_TYPE_IMPL_PAIR) == NULL); + ASSERT_EQ(cbm_pipeline_publish_kind(p), CBM_PIPELINE_PUBLISH_INCREMENTAL_OVERLAY); cbm_pipeline_free(p); + ASSERT(pipeline_store_has_function_name(g_incr_dbpath, project, "paired_value")); + ASSERT(!pipeline_store_has_function_name(g_incr_dbpath, project, "paired_extra")); + ASSERT(pipeline_store_overlay_file_has_function(g_incr_dbpath, project, "paired.h", + "paired_extra")); + int dirty_pending = -1; + int dirty_overlay_ready = -1; + ASSERT_EQ(pipeline_store_dirty_counts(g_incr_dbpath, project, &dirty_pending, + &dirty_overlay_ready), + CBM_STORE_OK); + ASSERT_EQ(dirty_pending, 0); + ASSERT_EQ(dirty_overlay_ready, 1); + + free(project); cbm_config_close(cfg); cleanup_incremental_repo(); PASS(); } -TEST(incremental_c_header_uses_exact_not_overlay_until_active_edges_are_exact) { +TEST(incremental_c_header_uses_exact_frontier_overlay_for_active_edges) { enum { PIPELINE_C_HEADER_OVERLAY_MAX_AFFECTED = CBM_SZ_16, PIPELINE_C_HEADER_AFFECTED_FRONTIER = @@ -10322,9 +10337,6 @@ TEST(incremental_c_header_uses_exact_not_overlay_until_active_edges_are_exact) { int run_rc = cbm_pipeline_run(p); const char *logs = pipeline_capture_logs_end(); ASSERT_EQ(run_rc, 0); - if (strstr(logs, "msg=incremental.overlay.done files=") != NULL) { - FAIL(logs); - } ASSERT(strstr(logs, "msg=incremental.classify changed=2") != NULL); char frontier_log[CBM_SZ_128]; int log_n = snprintf(frontier_log, sizeof(frontier_log), @@ -10332,22 +10344,27 @@ TEST(incremental_c_header_uses_exact_not_overlay_until_active_edges_are_exact) { PIPELINE_C_HEADER_AFFECTED_FRONTIER); ASSERT(log_n >= 0 && (size_t)log_n < sizeof(frontier_log)); ASSERT(strstr(logs, frontier_log) != NULL); - ASSERT(strstr(logs, "msg=incremental.exact.done files=") != NULL); - ASSERT_EQ(cbm_pipeline_publish_kind(p), CBM_PIPELINE_PUBLISH_INCREMENTAL_EXACT); - ASSERT(cbm_pipeline_graph_changed(p)); + char overlay_log[CBM_SZ_128]; + log_n = snprintf(overlay_log, sizeof(overlay_log), + "msg=incremental.overlay.done files=%d", + PIPELINE_C_HEADER_AFFECTED_FRONTIER); + ASSERT(log_n >= 0 && (size_t)log_n < sizeof(overlay_log)); + ASSERT(strstr(logs, overlay_log) != NULL); + ASSERT_EQ(cbm_pipeline_publish_kind(p), CBM_PIPELINE_PUBLISH_INCREMENTAL_OVERLAY); + ASSERT(!cbm_pipeline_graph_changed(p)); cbm_pipeline_exact_delta_stats_t stats = cbm_pipeline_exact_delta_stats(p); ASSERT_EQ(stats.changed_paths, CBM_SZ_2); ASSERT_EQ(stats.affected_paths, PIPELINE_C_HEADER_AFFECTED_FRONTIER); ASSERT_EQ(stats.published_paths, PIPELINE_C_HEADER_AFFECTED_FRONTIER); cbm_pipeline_free(p); - char diff_err[CBM_SZ_8K] = {0}; - int diff_rc = pipeline_compare_current_db_to_fresh_fast_rebuild( - g_incr_tmpdir, g_incr_dbpath, project, cfg, diff_err, sizeof(diff_err)); - if (diff_rc != 0) { - FAIL(diff_err[0] ? diff_err : "C header exact update differed from fresh rebuild"); - } - ASSERT_EQ(diff_rc, 0); + int dirty_pending = -1; + int dirty_overlay_ready = -1; + ASSERT_EQ(pipeline_store_dirty_counts(g_incr_dbpath, project, &dirty_pending, + &dirty_overlay_ready), + CBM_STORE_OK); + ASSERT_EQ(dirty_pending, 0); + ASSERT_EQ(dirty_overlay_ready, PIPELINE_C_HEADER_AFFECTED_FRONTIER); free(project); cbm_config_close(cfg); @@ -13828,8 +13845,8 @@ SUITE(pipeline) { RUN_TEST(incremental_fast_falls_back_for_oversized_inbound_frontier_and_matches_full); RUN_TEST(incremental_fast_c_header_frontier_too_large_uses_full_rebuild); RUN_TEST(incremental_overlay_publish_single_c_header_uses_active_overlay); - RUN_TEST(incremental_overlay_single_c_header_type_impl_pair_falls_back); - RUN_TEST(incremental_c_header_uses_exact_not_overlay_until_active_edges_are_exact); + RUN_TEST(incremental_overlay_single_c_header_type_impl_pair_keeps_canonical_rows_visible); + RUN_TEST(incremental_c_header_uses_exact_frontier_overlay_for_active_edges); RUN_TEST(incremental_fast_configured_frontier_cap_allows_bounded_exact); RUN_TEST(incremental_fast_mixed_unowned_edge_frontier_falls_back_before_exact_build); RUN_TEST(incremental_fast_expands_small_inbound_frontier_and_matches_full); diff --git a/tests/test_store_nodes.c b/tests/test_store_nodes.c index 37de73266..77b1f89f2 100644 --- a/tests/test_store_nodes.c +++ b/tests/test_store_nodes.c @@ -1897,19 +1897,29 @@ TEST(store_overlay_additions_keep_canonical_file_rows_visible) { ASSERT_EQ(cbm_store_reserve_overlay_generation(s, "test", BASE_GENERATION, &overlay_generation), CBM_STORE_OK); - cbm_node_t added = {.project = "test", - .label = "Function", - .name = "added", - .qualified_name = "test.added", - .file_path = "main.h", - .start_line = 5, - .end_line = 7, - .properties_json = "{}"}; + cbm_node_t overlay_nodes[] = { + {.project = "test", + .label = "Function", + .name = "stable", + .qualified_name = "test.stable", + .file_path = "main.h", + .start_line = 1, + .end_line = 6, + .properties_json = "{}"}, + {.project = "test", + .label = "Function", + .name = "added", + .qualified_name = "test.added", + .file_path = "main.h", + .start_line = 8, + .end_line = 10, + .properties_json = "{}"}, + }; cbm_store_file_delta_t delta = {.project = "test", .rel_path = "main.h", .generation = BASE_GENERATION, - .nodes = &added, - .node_count = 1}; + .nodes = overlay_nodes, + .node_count = 2}; const cbm_store_file_delta_t *deltas[] = {&delta}; ASSERT_EQ(cbm_store_publish_overlay_file_delta_additions_batch(s, deltas, 1, overlay_generation), @@ -1922,19 +1932,20 @@ TEST(store_overlay_additions_keep_canonical_file_rows_visible) { ASSERT_EQ(cbm_store_get_overlay_node_view_summary(s, "test", &summary), CBM_STORE_OK); ASSERT_EQ(summary.overlay_ready_generations, 1); ASSERT_EQ(summary.active_file_tombstones, 0); - ASSERT_EQ(summary.canonical_nodes_visible, 1); - ASSERT_EQ(summary.overlay_owned_nodes_visible, 1); + ASSERT_EQ(summary.canonical_nodes_visible, 0); + ASSERT_EQ(summary.overlay_owned_nodes_visible, 2); ASSERT_EQ(summary.total_nodes_visible, 2); cbm_node_t found = {0}; ASSERT_EQ(cbm_store_find_node_by_qn_overlay_view(s, "test", "test.stable", &found), CBM_STORE_OK); ASSERT_STR_EQ(found.file_path, "main.h"); + ASSERT_EQ(found.end_line, 6); cbm_node_free_fields(&found); ASSERT_EQ(cbm_store_find_node_by_qn_overlay_view(s, "test", "test.added", &found), CBM_STORE_OK); ASSERT_STR_EQ(found.file_path, "main.h"); - ASSERT_EQ(found.start_line, 5); + ASSERT_EQ(found.start_line, 8); cbm_node_free_fields(&found); cbm_store_close(s);