From d2e8eee6e165f1062d14228c86bb239ee50d71e1 Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Wed, 1 Jul 2026 06:33:14 -0400 Subject: [PATCH] feat(pipeline): apply exact incremental deletes Use the existing exact-delta planner and store delete publisher for the narrow safe case of one deleted file, no changed files, and FAST-or-lighter indexing. Rejected delete shapes continue to fall back before graph mutation or full publish. Add a focused pipeline regression that deletes an isolated file, proves a completed positive generation was published, and compares the result against a fresh FAST rebuild. Validation: git diff --check; bash scripts/check-source-safety.sh; make -j8 -f Makefile.cbm build/c/test-runner; CBM_ONLY_SUITE=pipeline ./build/c/test-runner (283 passed); CBM_ONLY_SUITE=incremental ./build/c/test-runner (160 passed). Signed-off-by: Andrew Hundt --- src/pipeline/pipeline_incremental.c | 82 +++++++++++++++++++++++++++++ tests/test_pipeline.c | 78 +++++++++++++++++++++++++++ 2 files changed, 160 insertions(+) diff --git a/src/pipeline/pipeline_incremental.c b/src/pipeline/pipeline_incremental.c index bfe7b665a..16b2f9ab0 100644 --- a/src/pipeline/pipeline_incremental.c +++ b/src/pipeline/pipeline_incremental.c @@ -863,6 +863,79 @@ static void incr_mark_generation_failed(cbm_store_t *store, const char *project, } } +static int incr_try_exact_delete_route(cbm_pipeline_t *p, cbm_store_t *store, const char *db_path, + const char *project, char **deleted, int deleted_count, + int changed_count, int *applied) { + if (applied) { + *applied = 0; + } + if (!p || !store || !db_path || !project || !deleted || !applied) { + return CBM_STORE_OK; + } + if (changed_count != 0 || deleted_count != 1 || + cbm_pipeline_get_mode(p) < CBM_MODE_FAST) { + return CBM_STORE_OK; + } + + const char *rel_path = deleted[0]; + if (!rel_path || !rel_path[0]) { + cbm_log_info("incremental.exact.delete.fallback", "reason", "missing_rel_path"); + return CBM_STORE_OK; + } + + enum { + CBM_INCR_DELETE_DELTA_COUNT = 1, + CBM_INCR_DELETE_PREFLIGHT_GENERATION = CBM_PIPELINE_COMPAT_GENERATION + 1, + }; + cbm_pipeline_file_delta_t delta = { + .delta = {.project = project, + .rel_path = rel_path, + .generation = CBM_INCR_DELETE_PREFLIGHT_GENERATION}, + .change_kind = CBM_PIPELINE_DELTA_CHANGE_DELETE, + }; + const cbm_pipeline_file_delta_t *delta_ptrs[] = {&delta}; + cbm_pipeline_file_delta_plan_t plan = {0}; + int rc = cbm_pipeline_plan_file_delta_batch(store, delta_ptrs, CBM_INCR_DELETE_DELTA_COUNT, + CBM_PIPELINE_EXACT_DELTA_MAX_AFFECTED_PATHS, + &plan); + if (rc != CBM_STORE_OK || plan.route != CBM_PIPELINE_DELTA_ROUTE_EXACT_CANDIDATE) { + cbm_log_info("incremental.exact.delete.fallback", "reason", + plan.reason ? plan.reason : "plan_error"); + cbm_pipeline_file_delta_plan_free(&plan); + return CBM_STORE_OK; + } + cbm_pipeline_file_delta_plan_free(&plan); + + int64_t generation = 0; + rc = cbm_store_reserve_index_generation(store, project, NULL, NULL, &generation); + if (rc != CBM_STORE_OK || generation <= 0) { + cbm_log_info("incremental.exact.delete.fallback", "reason", "reserve_generation", "rc", + itoa_buf_incr(rc)); + return CBM_STORE_OK; + } + delta.delta.generation = generation; + + rc = cbm_pipeline_apply_file_delta_batch(store, delta_ptrs, CBM_INCR_DELETE_DELTA_COUNT, + CBM_PIPELINE_EXACT_DELTA_MAX_AFFECTED_PATHS, &plan); + if (rc != CBM_STORE_OK || plan.route != CBM_PIPELINE_DELTA_ROUTE_EXACT_CANDIDATE) { + incr_mark_generation_failed(store, project, generation); + cbm_log_info("incremental.exact.delete.fallback", "reason", + plan.reason ? plan.reason : "apply_error"); + cbm_pipeline_file_delta_plan_free(&plan); + return CBM_STORE_OK; + } + cbm_pipeline_file_delta_plan_free(&plan); + + cbm_pipeline_set_committed_counts(p, cbm_store_count_nodes(store, project), + cbm_store_count_edges(store, project)); + if (cbm_pipeline_repo_path(p) && cbm_artifact_exists(cbm_pipeline_repo_path(p))) { + (void)cbm_artifact_export(db_path, cbm_pipeline_repo_path(p), project, CBM_ARTIFACT_FAST); + } + cbm_log_info("incremental.exact.delete.done", "files", "1"); + *applied = 1; + return CBM_STORE_OK; +} + static int incr_try_exact_upsert_route(cbm_pipeline_t *p, cbm_store_t *store, const char *db_path, const char *project, cbm_file_info_t *changed_files, int changed_count, int deleted_count, @@ -1181,6 +1254,15 @@ int cbm_pipeline_run_incremental(cbm_pipeline_t *p, const char *db_path, cbm_fil cbm_file_info_t *changed_files = cls.changed_files; int ci = cls.changed_file_count; int exact_applied = 0; + (void)incr_try_exact_delete_route(p, store, db_path, project, cls.deleted, cls.deleted_count, + ci, &exact_applied); + if (exact_applied) { + incr_classification_free(&cls); + cbm_store_close(store); + cbm_log_info("incremental.done", "elapsed_ms", itoa_buf_incr((int)elapsed_ms_incr(t0))); + return 0; + } + (void)incr_try_exact_upsert_route(p, store, db_path, project, changed_files, ci, cls.deleted_count, pass_fingerprint, &exact_applied); if (exact_applied) { diff --git a/tests/test_pipeline.c b/tests/test_pipeline.c index 6af12f8ce..f2a5b770b 100644 --- a/tests/test_pipeline.c +++ b/tests/test_pipeline.c @@ -7618,6 +7618,30 @@ static int pipeline_store_file_state_generation(const char *db_path, const char return rc; } +static int pipeline_store_completed_generation_count(const char *db_path, const char *project) { + cbm_store_t *s = cbm_store_open_path_query(db_path); + if (!s) { + return CBM_STORE_ERR; + } + sqlite3 *db = cbm_store_get_db(s); + sqlite3_stmt *stmt = NULL; + const char *sql = "SELECT COUNT(*) FROM index_generations " + "WHERE project = ?1 AND status = ?2 AND generation > ?3;"; + int count = CBM_STORE_ERR; + if (db && sqlite3_prepare_v2(db, sql, CBM_NOT_FOUND, &stmt, NULL) == SQLITE_OK) { + sqlite3_bind_text(stmt, 1, project, CBM_NOT_FOUND, SQLITE_TRANSIENT); + sqlite3_bind_text(stmt, 2, CBM_STORE_INDEX_STATUS_COMPLETE, CBM_NOT_FOUND, + SQLITE_TRANSIENT); + sqlite3_bind_int64(stmt, 3, CBM_PIPELINE_COMPAT_GENERATION); + if (sqlite3_step(stmt) == SQLITE_ROW) { + count = sqlite3_column_int(stmt, 0); + } + } + sqlite3_finalize(stmt); + cbm_store_close(s); + return count; +} + static int pipeline_compare_current_db_to_fresh_fast_rebuild(const char *repo_path, const char *db_path, const char *project, @@ -8504,6 +8528,59 @@ TEST(incremental_fast_three_file_batch_falls_back_to_full_rebuild_parity) { PASS(); } +TEST(incremental_fast_single_delete_exact_matches_full_rebuild) { + if (setup_incremental_repo() != 0) { + FAIL("setup failed"); + } + + char path[CBM_PATH_MAX]; + int n = snprintf(path, sizeof(path), "%s/leaf.go", g_incr_tmpdir); + ASSERT(n >= 0 && (size_t)n < sizeof(path)); + ASSERT_EQ(th_write_file(path, + "package main\n\n" + "func Leaf() int {\n\treturn 1\n}\n"), + 0); + + cbm_config_t *cfg = incremental_test_config(g_incr_tmpdir); + ASSERT_NOT_NULL(cfg); + cbm_pipeline_t *p = cbm_pipeline_new(g_incr_tmpdir, g_incr_dbpath, CBM_MODE_FAST); + ASSERT_NOT_NULL(p); + cbm_pipeline_apply_config(p, cfg); + ASSERT_EQ(cbm_pipeline_run(p), 0); + char *project = cbm_strdup(cbm_pipeline_project_name(p)); + cbm_pipeline_free(p); + ASSERT_NOT_NULL(project); + ASSERT(pipeline_store_has_function_name(g_incr_dbpath, project, "Leaf")); + + ASSERT_EQ(cbm_unlink(path), 0); + + p = cbm_pipeline_new(g_incr_tmpdir, g_incr_dbpath, CBM_MODE_FAST); + ASSERT_NOT_NULL(p); + cbm_pipeline_apply_config(p, cfg); + ASSERT_EQ(cbm_pipeline_run(p), 0); + cbm_pipeline_free(p); + + ASSERT(!pipeline_store_has_function_name(g_incr_dbpath, project, "Leaf")); + int64_t leaf_generation = 0; + ASSERT_EQ(pipeline_store_file_state_generation(g_incr_dbpath, project, "leaf.go", + &leaf_generation), + CBM_STORE_NOT_FOUND); + ASSERT_GT(pipeline_store_completed_generation_count(g_incr_dbpath, project), 0); + + char diff_err[CBM_SZ_8K] = {0}; + int diff_rc = pipeline_compare_current_db_to_fresh_fast_rebuild( + g_incr_tmpdir, g_incr_dbpath, project, cfg, diff_err, sizeof(diff_err)); + if (diff_rc != 0) { + FAIL(diff_err[0] ? diff_err : "single-delete exact differed from fresh FAST rebuild"); + } + ASSERT_EQ(diff_rc, 0); + + free(project); + cbm_config_close(cfg); + cleanup_incremental_repo(); + PASS(); +} + TEST(incremental_fast_delete_falls_back_to_full_rebuild_parity) { if (setup_incremental_repo() != 0) { FAIL("setup failed"); @@ -10758,6 +10835,7 @@ SUITE(pipeline) { RUN_TEST(incremental_fast_exact_upsert_matches_full_rebuild); RUN_TEST(incremental_fast_two_file_batch_exact_upsert_matches_full_rebuild); RUN_TEST(incremental_fast_three_file_batch_falls_back_to_full_rebuild_parity); + RUN_TEST(incremental_fast_single_delete_exact_matches_full_rebuild); RUN_TEST(incremental_fast_delete_falls_back_to_full_rebuild_parity); RUN_TEST(incremental_fast_rename_like_batch_falls_back_to_full_rebuild_parity); RUN_TEST(incremental_fast_new_folder_falls_back_to_full_rebuild_parity);