fix: guard slow incremental reindex routing

Add a config-backed incremental_reindex policy with default fast so full and moderate indexes rebuild atomically instead of entering the current disk incremental path, which still performs full-graph quality work and can be slower than a clean rebuild. Keep always for benchmarking and off for deterministic full rebuilds.

Harden the existing incremental path around allocation failures before mutation: classify, changed-file list, graph buffer creation, changed-path set, and deleted-path copies now fail closed or preserve existing nodes rather than dereferencing NULL or recording corrupt path arrays.

Reuse the existing pipeline nanosecond constant for file-hash mtimes and keep the dedicated incremental suite opted into incremental_reindex=always so its canaries continue exercising the legacy disk incremental implementation.

Validation: make -f Makefile.cbm build/c/test-runner cbm; CBM_ONLY_SUITE=pipeline build/c/test-runner: 212 passed; CBM_ONLY_SUITE=cli build/c/test-runner: 102 passed; CBM_ONLY_SUITE=mcp build/c/test-runner: 113 passed; CBM_ONLY_SUITE=tool_consolidation build/c/test-runner: 94 passed; git diff --check.
Signed-off-by: Andrew Hundt <ATHundt@gmail.com>

src/cli/cli.c

@@ -2706,6 +2706,12 @@ const cbm_config_entry_t CBM_CONFIG_REGISTRY[] = {
      "Re-index if DB is older than N seconds (0=disabled)",
      "0-2592000",
      "0=disabled. 3600=hourly, 86400=daily, 604800=weekly. Runs on startup if stale."},
+    {"incremental_reindex", "fast", NULL, "Indexing",
+     "When to use the disk incremental reindex path",
+     "fast|always|off",
+     "'fast' uses incremental only for fast-mode indexes. Full/moderate modes rebuild atomically because "
+     "their current incremental path still performs full-graph quality work. 'always' preserves the legacy "
+     "route for benchmarking; 'off' always rebuilds from scratch."},
     /* ── Search ── */
     {"search_limit", "50", NULL, "Search",
      "Default max results for search_graph/search_code",

src/pipeline/pipeline.c

@@ -13,7 +13,6 @@
 #include "foundation/constants.h"
 
 enum { CBM_DIR_PERMS = 0755, PL_RING = 4, PL_RING_MASK = 3, PL_SEQ_PASSES = 6 };
-#define PL_NSEC_PER_SEC 1000000000LL
 #include "cli/cli.h"
 #include "pipeline/pipeline.h"
 #include "pipeline/artifact.h"
@@ -63,6 +62,12 @@ bool cbm_pipeline_try_lock(void) {
 
 #define LOCK_SPIN_NS 100000000 /* 100ms between lock retries */
 
+typedef enum {
+    CBM_INCREMENTAL_REINDEX_FAST = 0,
+    CBM_INCREMENTAL_REINDEX_ALWAYS,
+    CBM_INCREMENTAL_REINDEX_OFF,
+} cbm_incremental_reindex_policy_t;
+
 void cbm_pipeline_lock(void) {
     while (atomic_exchange(&g_pipeline_busy, 1) != 0) {
         struct timespec ts = {0, LOCK_SPIN_NS};
@@ -88,6 +93,7 @@ struct cbm_pipeline {
     double semantic_threshold;
     double githistory_min_coupling;
     double lsp_confidence_floor;
+    cbm_incremental_reindex_policy_t incremental_reindex;
     atomic_int cancelled;
     cbm_store_t *flush_store; /* when set, use flush_to_store instead of dump_to_sqlite */
     bool persistence; /* write .codebase-memory/graph.db.zst after indexing */
@@ -173,6 +179,7 @@ cbm_pipeline_t *cbm_pipeline_new(const char *repo_path, const char *db_path,
     p->semantic_threshold = 0.0;
     p->githistory_min_coupling = 0.0;
     p->lsp_confidence_floor = 0.0;
+    p->incremental_reindex = CBM_INCREMENTAL_REINDEX_FAST;
     p->persistence = false;
     p->committed_nodes = -1;
     p->committed_edges = -1;
@@ -259,6 +266,15 @@ void cbm_pipeline_apply_config(cbm_pipeline_t *p, cbm_config_t *cfg) {
     if (lsp_floor > 0.0) {
         cbm_pipeline_set_lsp_confidence_floor(p, lsp_floor);
     }
+
+    const char *incremental = cbm_config_get(cfg, CBM_CONFIG_INCREMENTAL_REINDEX, "fast");
+    if (incremental && strcmp(incremental, "always") == 0) {
+        p->incremental_reindex = CBM_INCREMENTAL_REINDEX_ALWAYS;
+    } else if (incremental && strcmp(incremental, "off") == 0) {
+        p->incremental_reindex = CBM_INCREMENTAL_REINDEX_OFF;
+    } else {
+        p->incremental_reindex = CBM_INCREMENTAL_REINDEX_FAST;
+    }
 }
 
 double cbm_pipeline_httplink_min_confidence(const cbm_pipeline_t *p) {
@@ -914,6 +930,16 @@ static int try_incremental_or_reindex(cbm_pipeline_t *p, cbm_file_info_t *files,
         free(db_path);
         return CBM_NOT_FOUND;
     }
+    if (p->incremental_reindex == CBM_INCREMENTAL_REINDEX_OFF ||
+        (p->incremental_reindex == CBM_INCREMENTAL_REINDEX_FAST &&
+         p->mode != CBM_MODE_FAST)) {
+        cbm_log_info("pipeline.route", "path", "full", "reason",
+                     p->incremental_reindex == CBM_INCREMENTAL_REINDEX_OFF
+                         ? "incremental_reindex=off"
+                         : "incremental_reindex=fast_requires_fast_mode");
+        free(db_path);
+        return CBM_NOT_FOUND;
+    }
     cbm_store_t *check_store = cbm_store_open_path(db_path);
     if (check_store && cbm_store_check_integrity(check_store)) {
         cbm_file_hash_t *hashes = NULL;
@@ -1248,12 +1274,12 @@ int cbm_pipeline_run(cbm_pipeline_t *p) {
                     if (stat(files[i].path, &fst) == 0) {
                         int64_t mtime_ns;
 #ifdef __APPLE__
-                        mtime_ns = ((int64_t)fst.st_mtimespec.tv_sec * 1000000000LL) +
+                        mtime_ns = ((int64_t)fst.st_mtimespec.tv_sec * CBM_NS_PER_SEC) +
                                    (int64_t)fst.st_mtimespec.tv_nsec;
 #elif defined(_WIN32)
-                        mtime_ns = (int64_t)fst.st_mtime * 1000000000LL;
+                        mtime_ns = (int64_t)fst.st_mtime * CBM_NS_PER_SEC;
 #else
-                        mtime_ns = ((int64_t)fst.st_mtim.tv_sec * 1000000000LL) +
+                        mtime_ns = ((int64_t)fst.st_mtim.tv_sec * CBM_NS_PER_SEC) +
                                    (int64_t)fst.st_mtim.tv_nsec;
 #endif
                         cbm_store_upsert_file_hash(hash_store, p->project_name,

src/pipeline/pipeline.h

@@ -84,6 +84,7 @@ void cbm_pipeline_set_flush_store(cbm_pipeline_t *p, cbm_store_t *store);
 #define CBM_CONFIG_SEMANTIC_THRESHOLD "semantic_threshold"
 #define CBM_CONFIG_GITHISTORY_MIN_COUPLING "githistory_min_coupling"
 #define CBM_CONFIG_LSP_CONFIDENCE_FLOOR "lsp_confidence_floor"
+#define CBM_CONFIG_INCREMENTAL_REINDEX "incremental_reindex"
 
 /* Set the Jaccard similarity threshold for SIMILAR-edge creation (pass_similarity).
  * <=0 (or unset) uses the CBM_MINHASH_JACCARD_THRESHOLD default. Before run(). */

src/pipeline/pipeline_incremental.c

@@ -37,7 +37,6 @@ enum { INCR_RING_BUF = 4, INCR_RING_MASK = 3, INCR_TS_BUF = 24 };
 
 #define CBM_MS_PER_SEC 1000.0
 #define CBM_NS_PER_MS 1000000.0
-#define CBM_NS_PER_SEC 1000000000LL
 
 /* ── Timing helper (same as pipeline.c) ──────────────────────────── */
 
@@ -184,6 +183,10 @@ static int find_deleted_files(const char *repo_path, cbm_file_info_t *files, int
     }
 
     CBMHashTable *current = cbm_ht_create((size_t)file_count * PAIR_LEN);
+    if (!current) {
+        cbm_log_error("incremental.err", "msg", "find_deleted_files_current_oom");
+        return 0;
+    }
     for (int i = 0; i < file_count; i++) {
         cbm_ht_set(current, files[i].rel_path, &files[i]);
     }
@@ -278,7 +281,13 @@ static int find_deleted_files(const char *repo_path, cbm_file_info_t *files, int
             }
             deleted = tmp;
         }
-        deleted[del_count++] = strdup(stored[i].rel_path);
+        char *rp = strdup(stored[i].rel_path);
+        if (!rp) {
+            cbm_log_error("incremental.err", "msg", "find_deleted_files_strdup_oom", "rel_path",
+                          stored[i].rel_path);
+            break;
+        }
+        deleted[del_count++] = rp;
     }
 
     cbm_ht_free(current);
@@ -300,6 +309,16 @@ static void free_mode_skipped(cbm_file_hash_t *ms, int count) {
     free(ms);
 }
 
+static void free_deleted_paths(char **deleted, int count) {
+    if (!deleted) {
+        return;
+    }
+    for (int i = 0; i < count; i++) {
+        free(deleted[i]);
+    }
+    free(deleted);
+}
+
 /* ── Inbound cross-file edge preservation (incremental correctness) ──
  *
  * The purge step (cbm_gbuf_delete_by_file) removes a changed file's nodes,
@@ -702,6 +721,12 @@ int cbm_pipeline_run_incremental(cbm_pipeline_t *p, const char *db_path, cbm_fil
     int n_unchanged = 0;
     bool *is_changed =
         classify_files(files, file_count, stored, stored_count, &n_changed, &n_unchanged);
+    if (!is_changed) {
+        cbm_log_error("incremental.err", "msg", "classify_files_oom");
+        cbm_store_free_file_hashes(stored, stored_count);
+        cbm_store_close(store);
+        return CBM_NOT_FOUND;
+    }
 
     /* Classify stored files absent from current discovery: truly-deleted
      * (purge) vs mode-skipped (preserve nodes AND hash rows). */
@@ -722,7 +747,7 @@ int cbm_pipeline_run_incremental(cbm_pipeline_t *p, const char *db_path, cbm_fil
     if (n_changed == 0 && deleted_count == 0) {
         cbm_log_info("incremental.noop", "reason", "no_changes");
         free(is_changed);
-        free(deleted);
+        free_deleted_paths(deleted, deleted_count);
         free_mode_skipped(mode_skipped, mode_skipped_count);
         cbm_store_free_file_hashes(stored, stored_count);
         cbm_store_close(store);
@@ -734,6 +759,14 @@ int cbm_pipeline_run_incremental(cbm_pipeline_t *p, const char *db_path, cbm_fil
     /* Build list of changed files */
     cbm_file_info_t *changed_files =
         (n_changed > 0) ? malloc((size_t)n_changed * sizeof(cbm_file_info_t)) : NULL;
+    if (n_changed > 0 && !changed_files) {
+        cbm_log_error("incremental.err", "msg", "changed_files_oom");
+        free(is_changed);
+        free_deleted_paths(deleted, deleted_count);
+        free_mode_skipped(mode_skipped, mode_skipped_count);
+        cbm_store_close(store);
+        return CBM_NOT_FOUND;
+    }
     int ci = 0;
     for (int i = 0; i < file_count; i++) {
         if (is_changed[i]) {
@@ -749,6 +782,14 @@ int cbm_pipeline_run_incremental(cbm_pipeline_t *p, const char *db_path, cbm_fil
     /* Step 1: Load existing graph into RAM */
     cbm_clock_gettime(CLOCK_MONOTONIC, &t);
     cbm_gbuf_t *existing = cbm_gbuf_new(project, cbm_pipeline_repo_path(p));
+    if (!existing) {
+        cbm_log_error("incremental.err", "msg", "gbuf_new_oom");
+        free(changed_files);
+        free_deleted_paths(deleted, deleted_count);
+        free_mode_skipped(mode_skipped, mode_skipped_count);
+        cbm_store_close(store);
+        return CBM_NOT_FOUND;
+    }
     int load_rc = cbm_gbuf_load_from_db(existing, db_path, project);
     cbm_log_info("incremental.load_db", "rc", itoa_buf_incr(load_rc), "nodes",
                  itoa_buf_incr(cbm_gbuf_node_count(existing)), "edges",
@@ -759,10 +800,7 @@ int cbm_pipeline_run_incremental(cbm_pipeline_t *p, const char *db_path, cbm_fil
         cbm_log_error("incremental.err", "msg", "load_db_failed");
         cbm_gbuf_free(existing);
         free(changed_files);
-        for (int i = 0; i < deleted_count; i++) {
-            free(deleted[i]);
-        }
-        free(deleted);
+        free_deleted_paths(deleted, deleted_count);
         free_mode_skipped(mode_skipped, mode_skipped_count);
         cbm_store_close(store);
         return CBM_NOT_FOUND;
@@ -777,6 +815,15 @@ int cbm_pipeline_run_incremental(cbm_pipeline_t *p, const char *db_path, cbm_fil
     edge_cap.gbuf = existing;
     {
         CBMHashTable *changed_paths = cbm_ht_create(ci > 0 ? (size_t)ci * PAIR_LEN : CBM_SZ_64);
+        if (!changed_paths) {
+            cbm_log_error("incremental.err", "msg", "changed_paths_oom");
+            incr_free_edge_capture(&edge_cap);
+            cbm_gbuf_free(existing);
+            free(changed_files);
+            free_deleted_paths(deleted, deleted_count);
+            free_mode_skipped(mode_skipped, mode_skipped_count);
+            return CBM_NOT_FOUND;
+        }
         for (int i = 0; i < ci; i++) {
             cbm_ht_set(changed_paths, changed_files[i].rel_path, &changed_files[i]);
         }
@@ -812,10 +859,7 @@ int cbm_pipeline_run_incremental(cbm_pipeline_t *p, const char *db_path, cbm_fil
             }
         }
     }
-    for (int i = 0; i < deleted_count; i++) {
-        free(deleted[i]);
-    }
-    free(deleted);
+    free_deleted_paths(deleted, deleted_count);
     cbm_log_info("incremental.purge", "elapsed_ms", itoa_buf_incr((int)elapsed_ms_incr(t)));
 
     /* Step 3-5: Registry + extract + resolve */

tests/test_incremental.c

@@ -13,6 +13,7 @@
 #include "../src/foundation/compat.h"
 #include "test_framework.h"
 #include "test_helpers.h"
+#include <cli/cli.h>
 #include <mcp/mcp.h>
 #include <store/store.h>
 #include <pipeline/pipeline.h>
@@ -36,6 +37,7 @@ static char g_tmpdir[256];
 static char g_repodir[512];
 static char g_dbpath[512];
 static cbm_mcp_server_t *g_srv = NULL;
+static cbm_config_t *g_cfg = NULL;
 static char *g_project = NULL;
 
 /* Baseline counts after full index */
@@ -249,6 +251,14 @@ static int incremental_setup(void) {
     g_srv = cbm_mcp_server_new(NULL);
     if (!g_srv)
         return -1;
+    g_cfg = cbm_config_open(cdir);
+    if (!g_cfg) {
+        cbm_mcp_server_free(g_srv);
+        g_srv = NULL;
+        return -1;
+    }
+    cbm_config_set(g_cfg, CBM_CONFIG_INCREMENTAL_REINDEX, "always");
+    cbm_mcp_server_set_config(g_srv, g_cfg);
 
     g_rss_before_full = cbm_mem_rss();
 
@@ -260,6 +270,8 @@ static void incremental_teardown(void) {
         cbm_mcp_server_free(g_srv);
         g_srv = NULL;
     }
+    cbm_config_close(g_cfg);
+    g_cfg = NULL;
     if (g_project) {
         unlink(g_dbpath);
         char wal[520], shm[520];

tests/test_pipeline.c

@@ -5665,6 +5665,15 @@ TEST(config_registry_includes_mcp_timeout_knobs) {
     PASS();
 }
 
+TEST(config_registry_includes_incremental_reindex_policy) {
+    const cbm_config_entry_t *entry = find_config_entry(CBM_CONFIG_INCREMENTAL_REINDEX);
+    ASSERT_NOT_NULL(entry);
+    ASSERT_STR_EQ(entry->default_val, "fast");
+    ASSERT_STR_EQ(entry->category, "Indexing");
+    ASSERT_STR_EQ(entry->range, "fast|always|off");
+    PASS();
+}
+
 TEST(trackable_source_files) {
     /* Common source extensions are trackable */
     ASSERT_TRUE(cbm_is_trackable_file("main.go"));
@@ -6187,6 +6196,7 @@ SUITE(pipeline) {
     RUN_TEST(pipeline_unit_threshold_setters_clamp_invalid_values);
     RUN_TEST(pipeline_apply_config_sets_all_thresholds);
     RUN_TEST(config_registry_includes_mcp_timeout_knobs);
+    RUN_TEST(config_registry_includes_incremental_reindex_policy);
     /* File persistence */
     RUN_TEST(store_file_persistence);
     RUN_TEST(store_bulk_persistence);