diff --git a/src/pipeline/pipeline_delta.c b/src/pipeline/pipeline_delta.c index d6258983e..2c7fce286 100644 --- a/src/pipeline/pipeline_delta.c +++ b/src/pipeline/pipeline_delta.c @@ -2,6 +2,7 @@ #include "foundation/compat.h" #include "foundation/constants.h" +#include "foundation/log.h" #include "foundation/str_util.h" #include @@ -44,6 +45,7 @@ static const char cbm_delta_reason_rename_requires_full[] = "rename_requires_ful static const char cbm_delta_reason_unresolved_edge_endpoint[] = "unresolved_edge_endpoint"; static const char cbm_delta_reason_unsupported_derived_view[] = "unsupported_derived_view"; static const char cbm_delta_reason_unsupported_edges[] = "unsupported_edges"; +static const char cbm_delta_debug_inbound_env[] = "CBM_DEBUG_DELTA_INBOUND"; static const char *const cbm_delta_scratch_graph_seed_labels[] = { "Project", "Branch", "Folder", "File", "Module", NULL, @@ -1035,6 +1037,26 @@ static bool delta_inbound_edge_is_regenerated_by_batch( edge->type); } +static void delta_inbound_debug_unsupported(const cbm_store_file_delta_t *delta, + const cbm_store_inbound_edge_t *edge, + int delta_count) { + char env[CBM_SZ_16]; + if (!delta || !edge || + cbm_safe_getenv(cbm_delta_debug_inbound_env, env, sizeof(env), NULL) == NULL || + env[0] == '\0' || env[0] == '0') { + return; + } + char delta_count_buf[CBM_SZ_16]; + if (snprintf(delta_count_buf, sizeof(delta_count_buf), "%d", delta_count) < 0) { + return; + } + cbm_log_debug("delta.inbound.unsupported", "project", delta->project, "rel_path", + delta->rel_path, "source_path", edge->source_rel_path, "edge_path", + edge->edge_rel_path, "target_path", edge->target_rel_path, "type", + edge->type, "source_qn", edge->source_qn, "target_qn", edge->target_qn, + "delta_count", delta_count_buf); +} + static bool delta_owned_inbound_edge_is_deleted(const cbm_store_inbound_edge_t *edge, const cbm_pipeline_file_delta_t *delta) { return edge && delta && delta->change_kind == CBM_PIPELINE_DELTA_CHANGE_DELETE && @@ -1061,6 +1083,7 @@ static bool delta_inbound_edges_supported(cbm_store_t *store, edges[i].target_qn, edges[i].type) && !delta_inbound_edge_is_regenerated_by_batch(&edges[i], deltas, delta_count) && !delta_owned_inbound_edge_is_deleted(&edges[i], delta)) { + delta_inbound_debug_unsupported(&delta->delta, &edges[i], delta_count); ok = false; break; } diff --git a/src/pipeline/pipeline_incremental.c b/src/pipeline/pipeline_incremental.c index f6faf7948..944a2161a 100644 --- a/src/pipeline/pipeline_incremental.c +++ b/src/pipeline/pipeline_incremental.c @@ -1631,6 +1631,7 @@ static int incr_try_overlay_upsert_route(cbm_pipeline_t *p, cbm_store_t *store, cbm_gbuf_t *scratch = NULL; cbm_registry_t *registry = NULL; cbm_path_alias_collection_t *path_aliases = NULL; + CBMHashTable *pkgmap = NULL; cbm_pipeline_file_delta_t *deltas = NULL; cbm_pipeline_file_delta_t *additive_deltas = NULL; const cbm_pipeline_file_delta_t **delta_ptrs = NULL; @@ -1677,6 +1678,9 @@ static int incr_try_overlay_upsert_route(cbm_pipeline_t *p, cbm_store_t *store, } path_aliases = cbm_load_path_aliases(cbm_pipeline_repo_path(p)); + pkgmap = cbm_pkgmap_build_from_repo(cbm_pipeline_repo_path(p), changed_files, changed_count, + project); + cbm_pipeline_set_pkgmap(pkgmap); cbm_pipeline_ctx_t ctx = { .project_name = project, .repo_path = cbm_pipeline_repo_path(p), @@ -1851,6 +1855,10 @@ cleanup: incr_free_file_deltas(deltas, changed_count); incr_free_result_cache(result_cache, changed_count); cbm_path_alias_collection_free(path_aliases); + if (cbm_pipeline_get_pkgmap() == pkgmap) { + cbm_pipeline_set_pkgmap(NULL); + } + cbm_pkgmap_free(pkgmap); cbm_registry_free(registry); cbm_gbuf_free(scratch); free(changed_paths); @@ -1897,6 +1905,14 @@ static int incr_try_exact_upsert_route(cbm_pipeline_t *p, cbm_store_t *store, co bool exact_deferred_global_derived = cbm_pipeline_get_mode(p) < CBM_MODE_FAST && cbm_pipeline_incremental_derived_refresh_stale_on_exact(p); + if (scoped_overlay_gap) { + cbm_pipeline_set_exact_delta_stats_with_limit( + p, input_path_count, input_path_count, -1, max_affected_paths, true); + cbm_pipeline_set_publish_reason(p, CBM_PIPELINE_DELTA_REASON_FRONTIER_TOO_LARGE); + cbm_log_info("incremental.exact.skip", "reason", "scoped_lsp_gap", "action", + "full_reindex"); + return CBM_STORE_OK; + } if (deleted_count < 0 || changed_count > max_changed_paths || (cbm_pipeline_get_mode(p) < CBM_MODE_FAST && !exact_deferred_global_derived) || input_path_count > max_affected_paths) { @@ -1967,6 +1983,7 @@ static int incr_try_exact_upsert_route(cbm_pipeline_t *p, cbm_store_t *store, co cbm_gbuf_t *scratch = NULL; cbm_registry_t *registry = NULL; cbm_path_alias_collection_t *path_aliases = NULL; + CBMHashTable *pkgmap = NULL; cbm_pipeline_file_delta_t *deltas = NULL; const cbm_pipeline_file_delta_t **delta_ptrs = NULL; const cbm_store_file_delta_t **store_delta_ptrs = NULL; @@ -2021,6 +2038,9 @@ static int incr_try_exact_upsert_route(cbm_pipeline_t *p, cbm_store_t *store, co } path_aliases = cbm_load_path_aliases(cbm_pipeline_repo_path(p)); + pkgmap = cbm_pkgmap_build_from_repo(cbm_pipeline_repo_path(p), exact_files, exact_count, + project); + cbm_pipeline_set_pkgmap(pkgmap); cbm_pipeline_ctx_t ctx = { .project_name = project, .repo_path = cbm_pipeline_repo_path(p), @@ -2302,6 +2322,10 @@ cleanup: incr_free_file_deltas(deltas, delta_count); incr_free_result_cache(result_cache, exact_count); cbm_path_alias_collection_free(path_aliases); + if (cbm_pipeline_get_pkgmap() == pkgmap) { + cbm_pipeline_set_pkgmap(NULL); + } + cbm_pkgmap_free(pkgmap); cbm_registry_free(registry); cbm_gbuf_free(scratch); free(changed_paths); diff --git a/tests/test_pipeline.c b/tests/test_pipeline.c index e15640153..6c2d749a7 100644 --- a/tests/test_pipeline.c +++ b/tests/test_pipeline.c @@ -11455,9 +11455,8 @@ TEST(incremental_fast_route_decorator_change_matches_full_rebuild) { ASSERT_NOT_NULL(p); cbm_pipeline_apply_config(p, cfg); ASSERT_EQ(cbm_pipeline_run(p), 0); - cbm_pipeline_publish_kind_t kind = cbm_pipeline_publish_kind(p); - ASSERT(kind == CBM_PIPELINE_PUBLISH_INCREMENTAL_EXACT || - kind == CBM_PIPELINE_PUBLISH_INCREMENTAL_CONTAINMENT); + ASSERT_EQ(cbm_pipeline_publish_kind(p), CBM_PIPELINE_PUBLISH_FULL); + ASSERT_STR_EQ(cbm_pipeline_publish_reason(p), "frontier_too_large"); cbm_pipeline_free(p); ASSERT(!pipeline_store_has_route_name(g_incr_dbpath, project, "/api/orders")); @@ -11847,8 +11846,8 @@ TEST(incremental_overlay_publish_small_deltas_keeps_canonical_base_visible) { PASS(); } -TEST(incremental_overlay_publish_python_scoped_lsp_gap_uses_direct_exact) { - enum { PIPELINE_EXACT_TWO_PATHS = 2 }; +TEST(incremental_overlay_publish_python_scoped_lsp_gap_uses_full_reindex) { + enum { PIPELINE_EXACT_ONE_PATH = 1 }; if (setup_incremental_repo() != 0) { FAIL("setup failed"); } @@ -11875,7 +11874,7 @@ TEST(incremental_overlay_publish_python_scoped_lsp_gap_uses_direct_exact) { CBM_CONFIG_OVERLAY_PUBLISH_SMALL_DELTAS), 0); char cap_value[CBM_SZ_32]; - n = snprintf(cap_value, sizeof(cap_value), "%d", PIPELINE_EXACT_TWO_PATHS); + n = snprintf(cap_value, sizeof(cap_value), "%d", PIPELINE_EXACT_ONE_PATH); ASSERT(n >= 0 && (size_t)n < sizeof(cap_value)); ASSERT_EQ(cbm_config_set(cfg, CBM_CONFIG_INCREMENTAL_EXACT_MAX_CHANGED_PATHS, cap_value), 0); @@ -11903,21 +11902,24 @@ TEST(incremental_overlay_publish_python_scoped_lsp_gap_uses_direct_exact) { int run_rc = cbm_pipeline_run(p); const char *logs = pipeline_capture_logs_end(); ASSERT_EQ(run_rc, 0); - ASSERT(strstr(logs, "msg=incremental.exact.done files=1") != NULL); - ASSERT(strstr(logs, "msg=incremental.fallback reason=scoped_lsp_gap") == NULL); + ASSERT(strstr(logs, "msg=incremental.exact.done files=1") == NULL); + ASSERT(strstr(logs, "msg=incremental.exact.frontier") == NULL); + ASSERT(strstr(logs, "msg=incremental.exact.skip reason=scoped_lsp_gap") != NULL); + ASSERT(strstr(logs, "msg=incremental.fallback reason=scoped_lsp_gap") != NULL); ASSERT(strstr(logs, "msg=incremental.overlay.done files=") == NULL); - ASSERT_EQ(cbm_pipeline_publish_kind(p), CBM_PIPELINE_PUBLISH_INCREMENTAL_EXACT); + ASSERT_EQ(cbm_pipeline_publish_kind(p), CBM_PIPELINE_PUBLISH_FULL); cbm_pipeline_exact_delta_stats_t stats = cbm_pipeline_exact_delta_stats(p); ASSERT_EQ(stats.changed_paths, 1); - ASSERT_EQ(stats.affected_paths, PIPELINE_EXACT_TWO_PATHS); - ASSERT_EQ(stats.published_paths, 1); + ASSERT_EQ(stats.affected_paths, PIPELINE_EXACT_ONE_PATH); + ASSERT_EQ(stats.published_paths, -1); cbm_pipeline_free(p); char diff_err[CBM_SZ_8K] = {0}; int diff_rc = pipeline_compare_current_db_to_fresh_fast_rebuild( g_incr_tmpdir, g_incr_dbpath, project, cfg, diff_err, sizeof(diff_err)); if (diff_rc != 0) { - FAIL(diff_err[0] ? diff_err : "Python scoped-LSP fallback differed from fresh rebuild"); + FAIL(diff_err[0] ? diff_err + : "Python scoped-LSP full reindex differed from fresh rebuild"); } ASSERT_EQ(diff_rc, 0); @@ -14491,7 +14493,7 @@ SUITE(pipeline) { RUN_TEST(incremental_fast_exact_batch_publish_matches_fresh_rebuild_for_two_file_go); RUN_TEST(incremental_overlay_producer_marks_dirty_ready_without_canonical_mutation); RUN_TEST(incremental_overlay_publish_small_deltas_keeps_canonical_base_visible); - RUN_TEST(incremental_overlay_publish_python_scoped_lsp_gap_uses_direct_exact); + RUN_TEST(incremental_overlay_publish_python_scoped_lsp_gap_uses_full_reindex); RUN_TEST(incremental_overlay_first_preserves_inbound_edges_past_exact_frontier_cap); RUN_TEST(incremental_overlay_publish_delete_keeps_canonical_base_visible); RUN_TEST(incremental_overlay_publish_repeated_update_keeps_active_view_idempotent);