From 52cc13ff8383de151396d4bf3fa6eb9c05779357 Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Tue, 30 Jun 2026 22:35:01 -0400 Subject: [PATCH] fix(pipeline): fingerprint effective index config Include effective index mode and pipeline thresholds in the file_state pass fingerprint used by full and containment incremental indexing. Metadata-equal files now remain unchanged only when content hash and effective pass/config fingerprint both match, so mode or threshold changes fail closed instead of returning a stale no-op. The fingerprint encodes threshold doubles by raw bits rather than locale-sensitive decimal formatting. Missing file_state rows remain legacy-compatible. No live exact-delta route or incremental default behavior is enabled by this change. Validated with: git diff --check; bash scripts/check-source-safety.sh; make -j8 -f Makefile.cbm build/c/test-runner; CBM_ONLY_SUITE=pipeline ./build/c/test-runner; CBM_ONLY_SUITE=incremental ./build/c/test-runner. Signed-off-by: Andrew Hundt --- src/pipeline/pipeline.c | 19 ++++++- src/pipeline/pipeline_delta.c | 30 +++++++++++ src/pipeline/pipeline_incremental.c | 25 +++++++--- src/pipeline/pipeline_internal.h | 7 +++ tests/test_pipeline.c | 77 +++++++++++++++++++++++++++++ 5 files changed, 148 insertions(+), 10 deletions(-) diff --git a/src/pipeline/pipeline.c b/src/pipeline/pipeline.c index 5212dfa28..fdb7237fb 100644 --- a/src/pipeline/pipeline.c +++ b/src/pipeline/pipeline.c @@ -299,6 +299,15 @@ double cbm_pipeline_lsp_confidence_floor(const cbm_pipeline_t *p) { return p ? p->lsp_confidence_floor : 0.0; } +int cbm_pipeline_current_pass_fingerprint(const cbm_pipeline_t *p, char *out, size_t out_sz) { + if (!p) { + return CBM_STORE_ERR; + } + return cbm_pipeline_format_file_delta_pass_fingerprint( + out, out_sz, p->mode, p->similarity_threshold, p->httplink_min_confidence, + p->semantic_threshold, p->githistory_min_coupling, p->lsp_confidence_floor); +} + void cbm_pipeline_set_persistence(cbm_pipeline_t *p, bool enabled) { if (p) { p->persistence = enabled; @@ -1366,9 +1375,15 @@ int cbm_pipeline_run(cbm_pipeline_t *p) { goto cleanup; } } + char pass_fingerprint[CBM_SZ_256]; int state_rc = - cbm_pipeline_persist_file_states(hash_store, p->project_name, files, file_count, - CBM_PIPELINE_COMPAT_GENERATION, NULL); + cbm_pipeline_current_pass_fingerprint(p, pass_fingerprint, + sizeof(pass_fingerprint)); + if (state_rc == CBM_STORE_OK) { + state_rc = cbm_pipeline_persist_file_states( + hash_store, p->project_name, files, file_count, + CBM_PIPELINE_COMPAT_GENERATION, pass_fingerprint); + } if (state_rc != CBM_STORE_OK) { cbm_log_error("pipeline.err", "phase", "persist_file_state", "rc", itoa_buf(state_rc)); diff --git a/src/pipeline/pipeline_delta.c b/src/pipeline/pipeline_delta.c index fa220fc2a..df76acc28 100644 --- a/src/pipeline/pipeline_delta.c +++ b/src/pipeline/pipeline_delta.c @@ -66,6 +66,36 @@ const char *cbm_pipeline_file_delta_pass_fingerprint(void) { return cbm_delta_pass_fingerprint_v1; } +static uint64_t delta_double_bits(double value) { + uint64_t bits = 0; + memcpy(&bits, &value, sizeof(bits)); + return bits; +} + +int cbm_pipeline_format_file_delta_pass_fingerprint(char *out, size_t out_sz, int mode, + double similarity_threshold, + double httplink_min_confidence, + double semantic_threshold, + double githistory_min_coupling, + double lsp_confidence_floor) { + if (!out || out_sz == 0) { + return CBM_STORE_ERR; + } + int n = snprintf(out, out_sz, + "%s|mode=%d|sim=%016" PRIx64 "|http=%016" PRIx64 "|sem=%016" PRIx64 + "|gh=%016" PRIx64 "|lsp=%016" PRIx64, + cbm_delta_pass_fingerprint_v1, mode, delta_double_bits(similarity_threshold), + delta_double_bits(httplink_min_confidence), + delta_double_bits(semantic_threshold), + delta_double_bits(githistory_min_coupling), + delta_double_bits(lsp_confidence_floor)); + if (n < 0 || (size_t)n >= out_sz) { + out[0] = '\0'; + return CBM_STORE_ERR; + } + return CBM_STORE_OK; +} + static bool delta_same_path(const char *a, const char *b) { return a && b && strcmp(a, b) == 0; } diff --git a/src/pipeline/pipeline_incremental.c b/src/pipeline/pipeline_incremental.c index ab2a7e85e..4a7157c2e 100644 --- a/src/pipeline/pipeline_incremental.c +++ b/src/pipeline/pipeline_incremental.c @@ -73,7 +73,7 @@ static bool incr_test_fail_phase_enabled(const char *phase) { * Caller must free the returned array. */ static bool *classify_files(cbm_store_t *store, const char *project, cbm_file_info_t *files, int file_count, cbm_file_hash_t *stored, int stored_count, - int *out_changed, int *out_unchanged) { + const char *pass_fingerprint, int *out_changed, int *out_unchanged) { bool *changed = calloc((size_t)file_count, sizeof(bool)); if (!changed) { return NULL; @@ -113,7 +113,7 @@ static bool *classify_files(cbm_store_t *store, const char *project, cbm_file_in changed[i] = true; n_changed++; } else if (!cbm_pipeline_file_state_is_current_or_legacy( - store, project, &files[i], cbm_pipeline_file_delta_pass_fingerprint())) { + store, project, &files[i], pass_fingerprint)) { changed[i] = true; n_changed++; } else { @@ -347,6 +347,7 @@ static void incr_classification_free(cbm_incr_classification_t *c) { static int incr_classification_build(cbm_pipeline_t *p, cbm_store_t *store, const char *project, cbm_file_info_t *files, int file_count, cbm_file_hash_t *stored, int stored_count, + const char *pass_fingerprint, cbm_incr_classification_t *out) { if (!p || !out) { return CBM_NOT_FOUND; @@ -354,8 +355,8 @@ static int incr_classification_build(cbm_pipeline_t *p, cbm_store_t *store, cons memset(out, 0, sizeof(*out)); out->is_changed = - classify_files(store, project, files, file_count, stored, stored_count, &out->n_changed, - &out->n_unchanged); + classify_files(store, project, files, file_count, stored, stored_count, pass_fingerprint, + &out->n_changed, &out->n_unchanged); if (!out->is_changed) { cbm_log_error("incremental.err", "msg", "classify_files_oom"); return CBM_NOT_FOUND; @@ -825,7 +826,7 @@ static const char *incremental_structure_root_qn(cbm_gbuf_t *gbuf, const char *p static int dump_and_persist(cbm_gbuf_t *gbuf, const char *db_path, const char *project, cbm_file_info_t *files, int file_count, const cbm_file_hash_t *mode_skipped, int mode_skipped_count, - const char *repo_path) { + const char *repo_path, const char *pass_fingerprint) { struct timespec t; cbm_clock_gettime(CLOCK_MONOTONIC, &t); @@ -844,7 +845,8 @@ static int dump_and_persist(cbm_gbuf_t *gbuf, const char *db_path, const char *p int state_rc = CBM_STORE_OK; if (hash_rc == CBM_STORE_OK) { state_rc = cbm_pipeline_persist_file_states(hash_store, project, files, file_count, - CBM_PIPELINE_COMPAT_GENERATION, NULL); + CBM_PIPELINE_COMPAT_GENERATION, + pass_fingerprint); } /* FTS5 rebuild after incremental dump. The btree dump path bypasses @@ -890,6 +892,12 @@ int cbm_pipeline_run_incremental(cbm_pipeline_t *p, const char *db_path, cbm_fil cbm_clock_gettime(CLOCK_MONOTONIC, &t0); const char *project = cbm_pipeline_project_name(p); + char pass_fingerprint[CBM_SZ_256]; + if (cbm_pipeline_current_pass_fingerprint(p, pass_fingerprint, sizeof(pass_fingerprint)) != + CBM_STORE_OK) { + cbm_log_error("incremental.err", "phase", "pass_fingerprint"); + return CBM_NOT_FOUND; + } /* Open existing disk DB */ cbm_store_t *store = cbm_store_open_path(db_path); @@ -907,7 +915,7 @@ int cbm_pipeline_run_incremental(cbm_pipeline_t *p, const char *db_path, cbm_fil * route-decision boundary for exact delta and the existing containment path. */ cbm_incr_classification_t cls = {0}; if (incr_classification_build(p, store, project, files, file_count, stored, stored_count, - &cls) != 0) { + pass_fingerprint, &cls) != 0) { cbm_store_free_file_hashes(stored, stored_count); cbm_store_close(store); return CBM_NOT_FOUND; @@ -1136,7 +1144,8 @@ int cbm_pipeline_run_incremental(cbm_pipeline_t *p, const char *db_path, cbm_fil cbm_pipeline_set_committed_counts(p, cbm_gbuf_node_count(existing), cbm_gbuf_edge_count(existing)); int persist_rc = dump_and_persist(existing, db_path, project, files, file_count, cls.mode_skipped, - cls.mode_skipped_count, cbm_pipeline_repo_path(p)); + cls.mode_skipped_count, cbm_pipeline_repo_path(p), + pass_fingerprint); incr_classification_free(&cls); cbm_gbuf_free(existing); if (persist_rc != 0) { diff --git a/src/pipeline/pipeline_internal.h b/src/pipeline/pipeline_internal.h index 7f02d493c..15dd38983 100644 --- a/src/pipeline/pipeline_internal.h +++ b/src/pipeline/pipeline_internal.h @@ -221,6 +221,13 @@ void cbm_pipeline_free_import_map(const char **keys, const char **vals, int coun * back instead of publishing when unsupported edges are present. */ int64_t cbm_pipeline_stat_mtime_ns(const struct stat *st); const char *cbm_pipeline_file_delta_pass_fingerprint(void); +int cbm_pipeline_format_file_delta_pass_fingerprint(char *out, size_t out_sz, int mode, + double similarity_threshold, + double httplink_min_confidence, + double semantic_threshold, + double githistory_min_coupling, + double lsp_confidence_floor); +int cbm_pipeline_current_pass_fingerprint(const cbm_pipeline_t *p, char *out, size_t out_sz); int cbm_pipeline_content_hash_file(const char *path, char *out, size_t out_sz); bool cbm_pipeline_file_state_is_current_or_legacy(cbm_store_t *store, const char *project, const cbm_file_info_t *file, diff --git a/tests/test_pipeline.c b/tests/test_pipeline.c index de77d6e25..fef4140e8 100644 --- a/tests/test_pipeline.c +++ b/tests/test_pipeline.c @@ -3701,6 +3701,81 @@ TEST(pipeline_file_state_current_check_rejects_stale_pass_fingerprint) { PASS(); } +TEST(pipeline_pass_fingerprint_includes_effective_mode_and_thresholds) { + char full_default[CBM_SZ_256]; + char full_tuned[CBM_SZ_256]; + char full_tuned_again[CBM_SZ_256]; + char fast_default[CBM_SZ_256]; + + cbm_pipeline_t *full = cbm_pipeline_new("/tmp/nonexistent", NULL, CBM_MODE_FULL); + cbm_pipeline_t *fast = cbm_pipeline_new("/tmp/nonexistent", NULL, CBM_MODE_FAST); + ASSERT_NOT_NULL(full); + ASSERT_NOT_NULL(fast); + + ASSERT_EQ(cbm_pipeline_current_pass_fingerprint(full, full_default, sizeof(full_default)), + CBM_STORE_OK); + cbm_pipeline_set_similarity_threshold(full, 0.7); + cbm_pipeline_set_httplink_min_confidence(full, 0.25); + cbm_pipeline_set_semantic_threshold(full, 0.75); + cbm_pipeline_set_githistory_min_coupling(full, 0.3); + cbm_pipeline_set_lsp_confidence_floor(full, 0.6); + ASSERT_EQ(cbm_pipeline_current_pass_fingerprint(full, full_tuned, sizeof(full_tuned)), + CBM_STORE_OK); + ASSERT_EQ(cbm_pipeline_current_pass_fingerprint(full, full_tuned_again, + sizeof(full_tuned_again)), + CBM_STORE_OK); + ASSERT_EQ(cbm_pipeline_current_pass_fingerprint(fast, fast_default, sizeof(fast_default)), + CBM_STORE_OK); + + ASSERT_NEQ(strcmp(full_default, full_tuned), 0); + ASSERT_STR_EQ(full_tuned, full_tuned_again); + ASSERT_NEQ(strcmp(full_default, fast_default), 0); + + cbm_pipeline_free(full); + cbm_pipeline_free(fast); + PASS(); +} + +TEST(pipeline_file_state_current_check_rejects_stale_config_fingerprint) { + enum { PIPELINE_FILE_STATE_GENERATION = 16 }; + char *tmp = th_mktempdir("cbm_file_state_current_config"); + ASSERT_NOT_NULL(tmp); + const char *path = TH_PATH(tmp, "main.go"); + const char *content = "package main\nfunc Run() { println(\"config\") }\n"; + ASSERT_EQ(th_write_file(path, content), 0); + + char old_fingerprint[CBM_SZ_256]; + char current_fingerprint[CBM_SZ_256]; + ASSERT_EQ(cbm_pipeline_format_file_delta_pass_fingerprint( + old_fingerprint, sizeof(old_fingerprint), CBM_MODE_FULL, 0.7, 0.25, 0.75, + 0.3, 0.6), + CBM_STORE_OK); + ASSERT_EQ(cbm_pipeline_format_file_delta_pass_fingerprint( + current_fingerprint, sizeof(current_fingerprint), CBM_MODE_FULL, 0.8, 0.25, + 0.75, 0.3, 0.6), + CBM_STORE_OK); + ASSERT_NEQ(strcmp(old_fingerprint, current_fingerprint), 0); + + cbm_store_t *s = cbm_store_open_memory(); + ASSERT_NOT_NULL(s); + ASSERT_EQ(cbm_store_upsert_project(s, "test", tmp), CBM_STORE_OK); + + cbm_file_info_t file = {.path = (char *)path, + .rel_path = "main.go", + .language = CBM_LANG_GO, + .size = (int64_t)strlen(content)}; + ASSERT_EQ(cbm_pipeline_persist_file_states(s, "test", &file, 1, + PIPELINE_FILE_STATE_GENERATION, old_fingerprint), + CBM_STORE_OK); + ASSERT_FALSE(cbm_pipeline_file_state_is_current_or_legacy( + s, "test", &file, current_fingerprint)); + ASSERT_TRUE(cbm_pipeline_file_state_is_current_or_legacy(s, "test", &file, old_fingerprint)); + + cbm_store_close(s); + th_cleanup(tmp); + PASS(); +} + TEST(pipeline_file_state_persist_helper_rolls_back_on_failure) { char *tmp = th_mktempdir("cbm_file_state_persist_fail"); ASSERT_NOT_NULL(tmp); @@ -8886,6 +8961,8 @@ SUITE(pipeline) { RUN_TEST(pipeline_content_hash_helper_matches_file_delta_metadata); RUN_TEST(pipeline_file_state_persist_helper_writes_hash_metadata); RUN_TEST(pipeline_file_state_current_check_rejects_stale_pass_fingerprint); + RUN_TEST(pipeline_pass_fingerprint_includes_effective_mode_and_thresholds); + RUN_TEST(pipeline_file_state_current_check_rejects_stale_config_fingerprint); RUN_TEST(pipeline_file_state_persist_helper_rolls_back_on_failure); RUN_TEST(pipeline_file_delta_plan_candidate_from_frontier); RUN_TEST(pipeline_file_delta_apply_falls_back_on_publish_error);