diff --git a/src/cli/cli.c b/src/cli/cli.c index 323c94549..59bfdea6f 100644 --- a/src/cli/cli.c +++ b/src/cli/cli.c @@ -2706,6 +2706,12 @@ const cbm_config_entry_t CBM_CONFIG_REGISTRY[] = { "Re-index if DB is older than N seconds (0=disabled)", "0-2592000", "0=disabled. 3600=hourly, 86400=daily, 604800=weekly. Runs on startup if stale."}, + {"incremental_reindex", "fast", NULL, "Indexing", + "When to use the disk incremental reindex path", + "fast|always|off", + "'fast' uses incremental only for fast-mode indexes. Full/moderate modes rebuild atomically because " + "their current incremental path still performs full-graph quality work. 'always' preserves the legacy " + "route for benchmarking; 'off' always rebuilds from scratch."}, /* ── Search ── */ {"search_limit", "50", NULL, "Search", "Default max results for search_graph/search_code", diff --git a/src/pipeline/pipeline.c b/src/pipeline/pipeline.c index 005c24fae..7ce5e4172 100644 --- a/src/pipeline/pipeline.c +++ b/src/pipeline/pipeline.c @@ -13,7 +13,6 @@ #include "foundation/constants.h" enum { CBM_DIR_PERMS = 0755, PL_RING = 4, PL_RING_MASK = 3, PL_SEQ_PASSES = 6 }; -#define PL_NSEC_PER_SEC 1000000000LL #include "cli/cli.h" #include "pipeline/pipeline.h" #include "pipeline/artifact.h" @@ -63,6 +62,12 @@ bool cbm_pipeline_try_lock(void) { #define LOCK_SPIN_NS 100000000 /* 100ms between lock retries */ +typedef enum { + CBM_INCREMENTAL_REINDEX_FAST = 0, + CBM_INCREMENTAL_REINDEX_ALWAYS, + CBM_INCREMENTAL_REINDEX_OFF, +} cbm_incremental_reindex_policy_t; + void cbm_pipeline_lock(void) { while (atomic_exchange(&g_pipeline_busy, 1) != 0) { struct timespec ts = {0, LOCK_SPIN_NS}; @@ -88,6 +93,7 @@ struct cbm_pipeline { double semantic_threshold; double githistory_min_coupling; double lsp_confidence_floor; + cbm_incremental_reindex_policy_t incremental_reindex; atomic_int cancelled; cbm_store_t *flush_store; /* when set, use flush_to_store instead of dump_to_sqlite */ bool persistence; /* write .codebase-memory/graph.db.zst after indexing */ @@ -173,6 +179,7 @@ cbm_pipeline_t *cbm_pipeline_new(const char *repo_path, const char *db_path, p->semantic_threshold = 0.0; p->githistory_min_coupling = 0.0; p->lsp_confidence_floor = 0.0; + p->incremental_reindex = CBM_INCREMENTAL_REINDEX_FAST; p->persistence = false; p->committed_nodes = -1; p->committed_edges = -1; @@ -259,6 +266,15 @@ void cbm_pipeline_apply_config(cbm_pipeline_t *p, cbm_config_t *cfg) { if (lsp_floor > 0.0) { cbm_pipeline_set_lsp_confidence_floor(p, lsp_floor); } + + const char *incremental = cbm_config_get(cfg, CBM_CONFIG_INCREMENTAL_REINDEX, "fast"); + if (incremental && strcmp(incremental, "always") == 0) { + p->incremental_reindex = CBM_INCREMENTAL_REINDEX_ALWAYS; + } else if (incremental && strcmp(incremental, "off") == 0) { + p->incremental_reindex = CBM_INCREMENTAL_REINDEX_OFF; + } else { + p->incremental_reindex = CBM_INCREMENTAL_REINDEX_FAST; + } } double cbm_pipeline_httplink_min_confidence(const cbm_pipeline_t *p) { @@ -914,6 +930,16 @@ static int try_incremental_or_reindex(cbm_pipeline_t *p, cbm_file_info_t *files, free(db_path); return CBM_NOT_FOUND; } + if (p->incremental_reindex == CBM_INCREMENTAL_REINDEX_OFF || + (p->incremental_reindex == CBM_INCREMENTAL_REINDEX_FAST && + p->mode != CBM_MODE_FAST)) { + cbm_log_info("pipeline.route", "path", "full", "reason", + p->incremental_reindex == CBM_INCREMENTAL_REINDEX_OFF + ? "incremental_reindex=off" + : "incremental_reindex=fast_requires_fast_mode"); + free(db_path); + return CBM_NOT_FOUND; + } cbm_store_t *check_store = cbm_store_open_path(db_path); if (check_store && cbm_store_check_integrity(check_store)) { cbm_file_hash_t *hashes = NULL; @@ -1248,12 +1274,12 @@ int cbm_pipeline_run(cbm_pipeline_t *p) { if (stat(files[i].path, &fst) == 0) { int64_t mtime_ns; #ifdef __APPLE__ - mtime_ns = ((int64_t)fst.st_mtimespec.tv_sec * 1000000000LL) + + mtime_ns = ((int64_t)fst.st_mtimespec.tv_sec * CBM_NS_PER_SEC) + (int64_t)fst.st_mtimespec.tv_nsec; #elif defined(_WIN32) - mtime_ns = (int64_t)fst.st_mtime * 1000000000LL; + mtime_ns = (int64_t)fst.st_mtime * CBM_NS_PER_SEC; #else - mtime_ns = ((int64_t)fst.st_mtim.tv_sec * 1000000000LL) + + mtime_ns = ((int64_t)fst.st_mtim.tv_sec * CBM_NS_PER_SEC) + (int64_t)fst.st_mtim.tv_nsec; #endif cbm_store_upsert_file_hash(hash_store, p->project_name, diff --git a/src/pipeline/pipeline.h b/src/pipeline/pipeline.h index e34221abc..e13983b49 100644 --- a/src/pipeline/pipeline.h +++ b/src/pipeline/pipeline.h @@ -84,6 +84,7 @@ void cbm_pipeline_set_flush_store(cbm_pipeline_t *p, cbm_store_t *store); #define CBM_CONFIG_SEMANTIC_THRESHOLD "semantic_threshold" #define CBM_CONFIG_GITHISTORY_MIN_COUPLING "githistory_min_coupling" #define CBM_CONFIG_LSP_CONFIDENCE_FLOOR "lsp_confidence_floor" +#define CBM_CONFIG_INCREMENTAL_REINDEX "incremental_reindex" /* Set the Jaccard similarity threshold for SIMILAR-edge creation (pass_similarity). * <=0 (or unset) uses the CBM_MINHASH_JACCARD_THRESHOLD default. Before run(). */ diff --git a/src/pipeline/pipeline_incremental.c b/src/pipeline/pipeline_incremental.c index 411d03ef7..2fa341cfd 100644 --- a/src/pipeline/pipeline_incremental.c +++ b/src/pipeline/pipeline_incremental.c @@ -37,7 +37,6 @@ enum { INCR_RING_BUF = 4, INCR_RING_MASK = 3, INCR_TS_BUF = 24 }; #define CBM_MS_PER_SEC 1000.0 #define CBM_NS_PER_MS 1000000.0 -#define CBM_NS_PER_SEC 1000000000LL /* ── Timing helper (same as pipeline.c) ──────────────────────────── */ @@ -184,6 +183,10 @@ static int find_deleted_files(const char *repo_path, cbm_file_info_t *files, int } CBMHashTable *current = cbm_ht_create((size_t)file_count * PAIR_LEN); + if (!current) { + cbm_log_error("incremental.err", "msg", "find_deleted_files_current_oom"); + return 0; + } for (int i = 0; i < file_count; i++) { cbm_ht_set(current, files[i].rel_path, &files[i]); } @@ -278,7 +281,13 @@ static int find_deleted_files(const char *repo_path, cbm_file_info_t *files, int } deleted = tmp; } - deleted[del_count++] = strdup(stored[i].rel_path); + char *rp = strdup(stored[i].rel_path); + if (!rp) { + cbm_log_error("incremental.err", "msg", "find_deleted_files_strdup_oom", "rel_path", + stored[i].rel_path); + break; + } + deleted[del_count++] = rp; } cbm_ht_free(current); @@ -300,6 +309,16 @@ static void free_mode_skipped(cbm_file_hash_t *ms, int count) { free(ms); } +static void free_deleted_paths(char **deleted, int count) { + if (!deleted) { + return; + } + for (int i = 0; i < count; i++) { + free(deleted[i]); + } + free(deleted); +} + /* ── Inbound cross-file edge preservation (incremental correctness) ── * * The purge step (cbm_gbuf_delete_by_file) removes a changed file's nodes, @@ -702,6 +721,12 @@ int cbm_pipeline_run_incremental(cbm_pipeline_t *p, const char *db_path, cbm_fil int n_unchanged = 0; bool *is_changed = classify_files(files, file_count, stored, stored_count, &n_changed, &n_unchanged); + if (!is_changed) { + cbm_log_error("incremental.err", "msg", "classify_files_oom"); + cbm_store_free_file_hashes(stored, stored_count); + cbm_store_close(store); + return CBM_NOT_FOUND; + } /* Classify stored files absent from current discovery: truly-deleted * (purge) vs mode-skipped (preserve nodes AND hash rows). */ @@ -722,7 +747,7 @@ int cbm_pipeline_run_incremental(cbm_pipeline_t *p, const char *db_path, cbm_fil if (n_changed == 0 && deleted_count == 0) { cbm_log_info("incremental.noop", "reason", "no_changes"); free(is_changed); - free(deleted); + free_deleted_paths(deleted, deleted_count); free_mode_skipped(mode_skipped, mode_skipped_count); cbm_store_free_file_hashes(stored, stored_count); cbm_store_close(store); @@ -734,6 +759,14 @@ int cbm_pipeline_run_incremental(cbm_pipeline_t *p, const char *db_path, cbm_fil /* Build list of changed files */ cbm_file_info_t *changed_files = (n_changed > 0) ? malloc((size_t)n_changed * sizeof(cbm_file_info_t)) : NULL; + if (n_changed > 0 && !changed_files) { + cbm_log_error("incremental.err", "msg", "changed_files_oom"); + free(is_changed); + free_deleted_paths(deleted, deleted_count); + free_mode_skipped(mode_skipped, mode_skipped_count); + cbm_store_close(store); + return CBM_NOT_FOUND; + } int ci = 0; for (int i = 0; i < file_count; i++) { if (is_changed[i]) { @@ -749,6 +782,14 @@ int cbm_pipeline_run_incremental(cbm_pipeline_t *p, const char *db_path, cbm_fil /* Step 1: Load existing graph into RAM */ cbm_clock_gettime(CLOCK_MONOTONIC, &t); cbm_gbuf_t *existing = cbm_gbuf_new(project, cbm_pipeline_repo_path(p)); + if (!existing) { + cbm_log_error("incremental.err", "msg", "gbuf_new_oom"); + free(changed_files); + free_deleted_paths(deleted, deleted_count); + free_mode_skipped(mode_skipped, mode_skipped_count); + cbm_store_close(store); + return CBM_NOT_FOUND; + } int load_rc = cbm_gbuf_load_from_db(existing, db_path, project); cbm_log_info("incremental.load_db", "rc", itoa_buf_incr(load_rc), "nodes", itoa_buf_incr(cbm_gbuf_node_count(existing)), "edges", @@ -759,10 +800,7 @@ int cbm_pipeline_run_incremental(cbm_pipeline_t *p, const char *db_path, cbm_fil cbm_log_error("incremental.err", "msg", "load_db_failed"); cbm_gbuf_free(existing); free(changed_files); - for (int i = 0; i < deleted_count; i++) { - free(deleted[i]); - } - free(deleted); + free_deleted_paths(deleted, deleted_count); free_mode_skipped(mode_skipped, mode_skipped_count); cbm_store_close(store); return CBM_NOT_FOUND; @@ -777,6 +815,15 @@ int cbm_pipeline_run_incremental(cbm_pipeline_t *p, const char *db_path, cbm_fil edge_cap.gbuf = existing; { CBMHashTable *changed_paths = cbm_ht_create(ci > 0 ? (size_t)ci * PAIR_LEN : CBM_SZ_64); + if (!changed_paths) { + cbm_log_error("incremental.err", "msg", "changed_paths_oom"); + incr_free_edge_capture(&edge_cap); + cbm_gbuf_free(existing); + free(changed_files); + free_deleted_paths(deleted, deleted_count); + free_mode_skipped(mode_skipped, mode_skipped_count); + return CBM_NOT_FOUND; + } for (int i = 0; i < ci; i++) { cbm_ht_set(changed_paths, changed_files[i].rel_path, &changed_files[i]); } @@ -812,10 +859,7 @@ int cbm_pipeline_run_incremental(cbm_pipeline_t *p, const char *db_path, cbm_fil } } } - for (int i = 0; i < deleted_count; i++) { - free(deleted[i]); - } - free(deleted); + free_deleted_paths(deleted, deleted_count); cbm_log_info("incremental.purge", "elapsed_ms", itoa_buf_incr((int)elapsed_ms_incr(t))); /* Step 3-5: Registry + extract + resolve */ diff --git a/tests/test_incremental.c b/tests/test_incremental.c index bb43d1b36..020b6768e 100644 --- a/tests/test_incremental.c +++ b/tests/test_incremental.c @@ -13,6 +13,7 @@ #include "../src/foundation/compat.h" #include "test_framework.h" #include "test_helpers.h" +#include #include #include #include @@ -36,6 +37,7 @@ static char g_tmpdir[256]; static char g_repodir[512]; static char g_dbpath[512]; static cbm_mcp_server_t *g_srv = NULL; +static cbm_config_t *g_cfg = NULL; static char *g_project = NULL; /* Baseline counts after full index */ @@ -249,6 +251,14 @@ static int incremental_setup(void) { g_srv = cbm_mcp_server_new(NULL); if (!g_srv) return -1; + g_cfg = cbm_config_open(cdir); + if (!g_cfg) { + cbm_mcp_server_free(g_srv); + g_srv = NULL; + return -1; + } + cbm_config_set(g_cfg, CBM_CONFIG_INCREMENTAL_REINDEX, "always"); + cbm_mcp_server_set_config(g_srv, g_cfg); g_rss_before_full = cbm_mem_rss(); @@ -260,6 +270,8 @@ static void incremental_teardown(void) { cbm_mcp_server_free(g_srv); g_srv = NULL; } + cbm_config_close(g_cfg); + g_cfg = NULL; if (g_project) { unlink(g_dbpath); char wal[520], shm[520]; diff --git a/tests/test_pipeline.c b/tests/test_pipeline.c index c874227db..c014c1dbd 100644 --- a/tests/test_pipeline.c +++ b/tests/test_pipeline.c @@ -5665,6 +5665,15 @@ TEST(config_registry_includes_mcp_timeout_knobs) { PASS(); } +TEST(config_registry_includes_incremental_reindex_policy) { + const cbm_config_entry_t *entry = find_config_entry(CBM_CONFIG_INCREMENTAL_REINDEX); + ASSERT_NOT_NULL(entry); + ASSERT_STR_EQ(entry->default_val, "fast"); + ASSERT_STR_EQ(entry->category, "Indexing"); + ASSERT_STR_EQ(entry->range, "fast|always|off"); + PASS(); +} + TEST(trackable_source_files) { /* Common source extensions are trackable */ ASSERT_TRUE(cbm_is_trackable_file("main.go")); @@ -6187,6 +6196,7 @@ SUITE(pipeline) { RUN_TEST(pipeline_unit_threshold_setters_clamp_invalid_values); RUN_TEST(pipeline_apply_config_sets_all_thresholds); RUN_TEST(config_registry_includes_mcp_timeout_knobs); + RUN_TEST(config_registry_includes_incremental_reindex_policy); /* File persistence */ RUN_TEST(store_file_persistence); RUN_TEST(store_bulk_persistence);