From 52b017f84bf64a975e38f07c1ac8a30290aa7a45 Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Sat, 4 Jul 2026 13:06:09 -0400 Subject: [PATCH] fix(pipeline): block unsafe broad header overlays High-cap diagnostics showed that publishing non-tombstone additive overlay rows for broad C-family header exact frontiers can leave stale canonical edges visible in the active overlay view and can be slower than a full rebuild. Keep the single-header overlay path separate, but require broad header batches to publish through the exact route until a strict additive-subset proof exists. Update the pipeline canary to require exact publish and fresh-rebuild equivalence for this case. Signed-off-by: Andrew Hundt --- src/pipeline/pipeline_incremental.c | 12 ++++------- tests/test_pipeline.c | 32 ++++++++++++++--------------- 2 files changed, 20 insertions(+), 24 deletions(-) diff --git a/src/pipeline/pipeline_incremental.c b/src/pipeline/pipeline_incremental.c index 3a9a745c8..03754ba8c 100644 --- a/src/pipeline/pipeline_incremental.c +++ b/src/pipeline/pipeline_incremental.c @@ -2057,7 +2057,7 @@ static int incr_try_exact_upsert_route(cbm_pipeline_t *p, cbm_store_t *store, co bool overlay_publish_already_failed = prior_reason && strcmp(prior_reason, "overlay_publish_error") == 0; bool header_overlay_unsafe = incr_changed_contains_c_family_header(changed_files, changed_count); - if (!graph_noop_candidate && + if (!graph_noop_candidate && !header_overlay_unsafe && cbm_pipeline_overlay_publish_small_deltas(p) && !overlay_publish_already_failed) { int64_t base_generation = 0; @@ -2065,13 +2065,9 @@ static int incr_try_exact_upsert_route(cbm_pipeline_t *p, cbm_store_t *store, co rc = cbm_store_latest_complete_index_generation(store, project, &base_generation); if (rc == CBM_STORE_OK) { CBM_PROF_START(t_overlay_publish); - rc = header_overlay_unsafe - ? cbm_pipeline_publish_overlay_file_delta_additions_batch( - store, delta_ptrs, delta_count, base_generation, - CBM_STORE_DIRTY_SOURCE_EXPLICIT_REINDEX, &overlay_generation) - : cbm_pipeline_publish_overlay_file_delta_batch( - store, delta_ptrs, delta_count, base_generation, - CBM_STORE_DIRTY_SOURCE_EXPLICIT_REINDEX, &overlay_generation); + rc = cbm_pipeline_publish_overlay_file_delta_batch( + store, delta_ptrs, delta_count, base_generation, + CBM_STORE_DIRTY_SOURCE_EXPLICIT_REINDEX, &overlay_generation); CBM_PROF_END_N("incremental_exact", "11_publish_overlay", t_overlay_publish, delta_count); } diff --git a/tests/test_pipeline.c b/tests/test_pipeline.c index 0b85df1dd..2a99cbd3e 100644 --- a/tests/test_pipeline.c +++ b/tests/test_pipeline.c @@ -10294,7 +10294,7 @@ TEST(incremental_overlay_single_c_header_type_impl_pair_keeps_canonical_rows_vis PASS(); } -TEST(incremental_c_header_uses_exact_frontier_overlay_for_active_edges) { +TEST(incremental_c_header_uses_exact_not_additive_overlay_without_subset_proof) { enum { PIPELINE_C_HEADER_OVERLAY_MAX_AFFECTED = CBM_SZ_16, PIPELINE_C_HEADER_AFFECTED_FRONTIER = @@ -10344,27 +10344,27 @@ TEST(incremental_c_header_uses_exact_frontier_overlay_for_active_edges) { PIPELINE_C_HEADER_AFFECTED_FRONTIER); ASSERT(log_n >= 0 && (size_t)log_n < sizeof(frontier_log)); ASSERT(strstr(logs, frontier_log) != NULL); - char overlay_log[CBM_SZ_128]; - log_n = snprintf(overlay_log, sizeof(overlay_log), - "msg=incremental.overlay.done files=%d", + ASSERT(strstr(logs, "msg=incremental.overlay.done files=") == NULL); + char done_log[CBM_SZ_128]; + log_n = snprintf(done_log, sizeof(done_log), "msg=incremental.exact.done files=%d", PIPELINE_C_HEADER_AFFECTED_FRONTIER); - ASSERT(log_n >= 0 && (size_t)log_n < sizeof(overlay_log)); - ASSERT(strstr(logs, overlay_log) != NULL); - ASSERT_EQ(cbm_pipeline_publish_kind(p), CBM_PIPELINE_PUBLISH_INCREMENTAL_OVERLAY); - ASSERT(!cbm_pipeline_graph_changed(p)); + ASSERT(log_n >= 0 && (size_t)log_n < sizeof(done_log)); + ASSERT(strstr(logs, done_log) != NULL); + ASSERT_EQ(cbm_pipeline_publish_kind(p), CBM_PIPELINE_PUBLISH_INCREMENTAL_EXACT); + ASSERT(cbm_pipeline_graph_changed(p)); cbm_pipeline_exact_delta_stats_t stats = cbm_pipeline_exact_delta_stats(p); ASSERT_EQ(stats.changed_paths, CBM_SZ_2); ASSERT_EQ(stats.affected_paths, PIPELINE_C_HEADER_AFFECTED_FRONTIER); ASSERT_EQ(stats.published_paths, PIPELINE_C_HEADER_AFFECTED_FRONTIER); cbm_pipeline_free(p); - int dirty_pending = -1; - int dirty_overlay_ready = -1; - ASSERT_EQ(pipeline_store_dirty_counts(g_incr_dbpath, project, &dirty_pending, - &dirty_overlay_ready), - CBM_STORE_OK); - ASSERT_EQ(dirty_pending, 0); - ASSERT_EQ(dirty_overlay_ready, PIPELINE_C_HEADER_AFFECTED_FRONTIER); + char diff_err[CBM_SZ_8K] = {0}; + int diff_rc = pipeline_compare_current_db_to_fresh_fast_rebuild( + g_incr_tmpdir, g_incr_dbpath, project, cfg, diff_err, sizeof(diff_err)); + if (diff_rc != 0) { + FAIL(diff_err[0] ? diff_err : "C header exact update differed from fresh rebuild"); + } + ASSERT_EQ(diff_rc, 0); free(project); cbm_config_close(cfg); @@ -13846,7 +13846,7 @@ SUITE(pipeline) { RUN_TEST(incremental_fast_c_header_frontier_too_large_uses_full_rebuild); RUN_TEST(incremental_overlay_publish_single_c_header_uses_active_overlay); RUN_TEST(incremental_overlay_single_c_header_type_impl_pair_keeps_canonical_rows_visible); - RUN_TEST(incremental_c_header_uses_exact_frontier_overlay_for_active_edges); + RUN_TEST(incremental_c_header_uses_exact_not_additive_overlay_without_subset_proof); RUN_TEST(incremental_fast_configured_frontier_cap_allows_bounded_exact); RUN_TEST(incremental_fast_mixed_unowned_edge_frontier_falls_back_before_exact_build); RUN_TEST(incremental_fast_expands_small_inbound_frontier_and_matches_full);