From eb2bb03275ff99b264200817f7d8cbfeb8a3e69c Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Wed, 1 Jul 2026 03:19:32 -0400 Subject: [PATCH] feat(pipeline): allow two-file exact upserts Raise the live FAST exact-delta changed-file cap to two after the batch publish path gained same-generation node-before-edge ordering. Convert the former two-file fallback fixture into a production-route exact parity test, and add a three-file fixture that proves larger batches still fall back before publish and match a fresh FAST rebuild. Validation: git diff --check; bash scripts/check-source-safety.sh; make -j8 -f Makefile.cbm build/c/test-runner; CBM_ONLY_SUITE=pipeline ./build/c/test-runner (277 passed); CBM_ONLY_SUITE=incremental ./build/c/test-runner (160 passed). Signed-off-by: Andrew Hundt --- src/pipeline/pipeline_internal.h | 8 +-- tests/test_pipeline.c | 93 ++++++++++++++++++++++++++++++-- 2 files changed, 93 insertions(+), 8 deletions(-) diff --git a/src/pipeline/pipeline_internal.h b/src/pipeline/pipeline_internal.h index 95b16b43e..848bf8084 100644 --- a/src/pipeline/pipeline_internal.h +++ b/src/pipeline/pipeline_internal.h @@ -165,10 +165,10 @@ typedef struct { /* Conservative live exact-delta frontier cap. Larger affected sets fall back * to the existing containment reindex path until broader parity benchmarks pass. */ enum { CBM_PIPELINE_EXACT_DELTA_MAX_AFFECTED_PATHS = CBM_SZ_4 }; -/* Live exact upserts are currently limited to one changed file. Same-batch - * cross-file usage parity is still under design, so multi-file batches fall - * back before publish. */ -enum { CBM_PIPELINE_EXACT_DELTA_MAX_CHANGED_PATHS = CBM_ALLOC_ONE }; +/* Live exact upserts are currently limited to two changed files. Larger + * batches keep the existing fallback path until same-batch parity coverage + * includes deletes, renames, new folders, and derived-view freshness. */ +enum { CBM_PIPELINE_EXACT_DELTA_MAX_CHANGED_PATHS = CBM_SZ_2 }; /* Get the current pipeline's package map (NULL if none). */ CBMHashTable *cbm_pipeline_get_pkgmap(void); diff --git a/tests/test_pipeline.c b/tests/test_pipeline.c index f8f2d5ed2..b57d314d0 100644 --- a/tests/test_pipeline.c +++ b/tests/test_pipeline.c @@ -8255,7 +8255,7 @@ TEST(incremental_fast_exact_upsert_matches_full_rebuild) { PASS(); } -TEST(incremental_fast_multi_file_batch_falls_back_to_full_rebuild_parity) { +TEST(incremental_fast_two_file_batch_exact_upsert_matches_full_rebuild) { if (setup_incremental_repo() != 0) { FAIL("setup failed"); } @@ -8302,14 +8302,98 @@ TEST(incremental_fast_multi_file_batch_falls_back_to_full_rebuild_parity) { ASSERT_EQ(pipeline_store_file_state_generation(g_incr_dbpath, project, "helper.go", &helper_generation), CBM_STORE_OK); - ASSERT_EQ(main_generation, CBM_PIPELINE_COMPAT_GENERATION); + ASSERT_GT(main_generation, CBM_PIPELINE_COMPAT_GENERATION); ASSERT_EQ(main_generation, helper_generation); char diff_err[CBM_SZ_8K] = {0}; int diff_rc = pipeline_compare_current_db_to_fresh_fast_rebuild( g_incr_tmpdir, g_incr_dbpath, project, cfg, diff_err, sizeof(diff_err)); if (diff_rc != 0) { - printf(" [multi-file-fallback-diff] %s\n", diff_err); + FAIL(diff_err[0] ? diff_err : "two-file exact upsert differed from fresh FAST rebuild"); + } + ASSERT_EQ(diff_rc, 0); + + free(project); + cbm_config_close(cfg); + cleanup_incremental_repo(); + PASS(); +} + +TEST(incremental_fast_three_file_batch_falls_back_to_full_rebuild_parity) { + if (setup_incremental_repo() != 0) { + FAIL("setup failed"); + } + + char path[CBM_PATH_MAX]; + int n = snprintf(path, sizeof(path), "%s/leaf.go", g_incr_tmpdir); + ASSERT(n >= 0 && (size_t)n < sizeof(path)); + ASSERT_EQ(th_write_file(path, + "package main\n\n" + "func Leaf() int {\n\treturn 1\n}\n"), + 0); + + cbm_config_t *cfg = incremental_test_config(g_incr_tmpdir); + ASSERT_NOT_NULL(cfg); + cbm_pipeline_t *p = cbm_pipeline_new(g_incr_tmpdir, g_incr_dbpath, CBM_MODE_FAST); + ASSERT_NOT_NULL(p); + cbm_pipeline_apply_config(p, cfg); + ASSERT_EQ(cbm_pipeline_run(p), 0); + char *project = cbm_strdup(cbm_pipeline_project_name(p)); + cbm_pipeline_free(p); + ASSERT_NOT_NULL(project); + + n = snprintf(path, sizeof(path), "%s/main.go", g_incr_tmpdir); + ASSERT(n >= 0 && (size_t)n < sizeof(path)); + ASSERT_EQ(th_write_file(path, + "package main\n\n" + "func main() {\n\tHelper()\n\tNewHelper()\n\tLeaf()\n}\n\n" + "func NewMain() int {\n\treturn 11\n}\n"), + 0); + n = snprintf(path, sizeof(path), "%s/helper.go", g_incr_tmpdir); + ASSERT(n >= 0 && (size_t)n < sizeof(path)); + ASSERT_EQ(th_write_file(path, + "package main\n\n" + "func Helper() string {\n\treturn \"updated\"\n}\n\n" + "func NewHelper() int {\n\treturn 13\n}\n"), + 0); + n = snprintf(path, sizeof(path), "%s/leaf.go", g_incr_tmpdir); + ASSERT(n >= 0 && (size_t)n < sizeof(path)); + ASSERT_EQ(th_write_file(path, + "package main\n\n" + "func Leaf() int {\n\treturn 2\n}\n\n" + "func NewLeaf() int {\n\treturn 17\n}\n"), + 0); + + p = cbm_pipeline_new(g_incr_tmpdir, g_incr_dbpath, CBM_MODE_FAST); + ASSERT_NOT_NULL(p); + cbm_pipeline_apply_config(p, cfg); + ASSERT_EQ(cbm_pipeline_run(p), 0); + cbm_pipeline_free(p); + + ASSERT(pipeline_store_has_function_name(g_incr_dbpath, project, "NewMain")); + ASSERT(pipeline_store_has_function_name(g_incr_dbpath, project, "NewHelper")); + ASSERT(pipeline_store_has_function_name(g_incr_dbpath, project, "NewLeaf")); + int64_t main_generation = 0; + int64_t helper_generation = 0; + int64_t leaf_generation = 0; + ASSERT_EQ(pipeline_store_file_state_generation(g_incr_dbpath, project, "main.go", + &main_generation), + CBM_STORE_OK); + ASSERT_EQ(pipeline_store_file_state_generation(g_incr_dbpath, project, "helper.go", + &helper_generation), + CBM_STORE_OK); + ASSERT_EQ(pipeline_store_file_state_generation(g_incr_dbpath, project, "leaf.go", + &leaf_generation), + CBM_STORE_OK); + ASSERT_EQ(main_generation, CBM_PIPELINE_COMPAT_GENERATION); + ASSERT_EQ(main_generation, helper_generation); + ASSERT_EQ(main_generation, leaf_generation); + + char diff_err[CBM_SZ_8K] = {0}; + int diff_rc = pipeline_compare_current_db_to_fresh_fast_rebuild( + g_incr_tmpdir, g_incr_dbpath, project, cfg, diff_err, sizeof(diff_err)); + if (diff_rc != 0) { + FAIL(diff_err[0] ? diff_err : "three-file fallback differed from fresh FAST rebuild"); } ASSERT_EQ(diff_rc, 0); @@ -10378,7 +10462,8 @@ SUITE(pipeline) { RUN_TEST(incremental_full_then_noop); RUN_TEST(incremental_detects_changed_file); RUN_TEST(incremental_fast_exact_upsert_matches_full_rebuild); - RUN_TEST(incremental_fast_multi_file_batch_falls_back_to_full_rebuild_parity); + RUN_TEST(incremental_fast_two_file_batch_exact_upsert_matches_full_rebuild); + RUN_TEST(incremental_fast_three_file_batch_falls_back_to_full_rebuild_parity); RUN_TEST(incremental_fast_exact_scratch_multifile_usage_edges_match_fresh); RUN_TEST(incremental_fast_exact_batch_publish_matches_fresh_rebuild_for_two_file_go); RUN_TEST(incremental_full_mode_keeps_exact_upsert_disabled);