From 68cc19ebf23c1edff0fa0cf1496491e95a5a7ba5 Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Tue, 24 Mar 2026 22:06:31 +0100 Subject: [PATCH] Auto-detect and clean corrupt index databases Add cbm_store_check_integrity() that validates: - projects table has <= 5 rows (1 project per .db file) - root_path starts with / or drive letter (not numeric garbage) Wire into resolve_store (MCP queries) and pipeline routing: - MCP: corrupt DB auto-deleted with ERROR log, returns "not indexed" - Pipeline: corrupt DB deleted, falls through to full reindex Root cause: user-reported corrupt DBs had 48K rows in projects table with node IDs in text columns. Exact write path TBD, but detection + auto-clean prevents the symptom from persisting. 5 unit tests: clean DB, empty DB, bad path, too many rows, NULL. --- src/mcp/mcp.c | 17 +++++++++++ src/pipeline/pipeline.c | 39 ++++++++++++++++-------- src/store/store.c | 52 +++++++++++++++++++++++++++++++ src/store/store.h | 8 +++++ tests/test_store_nodes.c | 66 ++++++++++++++++++++++++++++++++++++++++ 5 files changed, 170 insertions(+), 12 deletions(-) diff --git a/src/mcp/mcp.c b/src/mcp/mcp.c index 4cead8451..9e4060184 100644 --- a/src/mcp/mcp.c +++ b/src/mcp/mcp.c @@ -657,6 +657,23 @@ static cbm_store_t *resolve_store(cbm_mcp_server_t *srv, const char *project) { project_db_path(project, path, sizeof(path)); srv->store = cbm_store_open_path_query(path); if (srv->store) { + /* Check DB integrity — auto-clean corrupt databases */ + if (!cbm_store_check_integrity(srv->store)) { + cbm_log_error("store.auto_clean", "project", project, "path", path, "action", + "deleting corrupt db — re-index required"); + cbm_store_close(srv->store); + srv->store = NULL; + /* Delete the corrupt DB + WAL/SHM files */ + cbm_unlink(path); + char wal_path[1040]; + char shm_path[1040]; + snprintf(wal_path, sizeof(wal_path), "%s-wal", path); + snprintf(shm_path, sizeof(shm_path), "%s-shm", path); + cbm_unlink(wal_path); + cbm_unlink(shm_path); + return NULL; + } + /* Verify the project actually exists in this database. * A .db file may exist but be empty (e.g., after delete_project on * Linux where unlink defers actual removal). Opening an empty/deleted diff --git a/src/pipeline/pipeline.c b/src/pipeline/pipeline.c index 8b480d24d..95ba3120a 100644 --- a/src/pipeline/pipeline.c +++ b/src/pipeline/pipeline.c @@ -353,19 +353,34 @@ int cbm_pipeline_run(cbm_pipeline_t *p) { /* DB exists — check if it has file hashes */ cbm_store_t *check_store = cbm_store_open_path(db_path); if (check_store) { - cbm_file_hash_t *hashes = NULL; - int hash_count = 0; - cbm_store_get_file_hashes(check_store, p->project_name, &hashes, &hash_count); - cbm_store_free_file_hashes(hashes, hash_count); - cbm_store_close(check_store); + /* Integrity check — corrupt DB → delete and fall through to full reindex */ + if (!cbm_store_check_integrity(check_store)) { + cbm_log_error("pipeline.corrupt_db", "path", db_path, "action", + "deleting — will do full reindex"); + cbm_store_close(check_store); + cbm_unlink(db_path); + char wal[1040]; + char shm[1040]; + snprintf(wal, sizeof(wal), "%s-wal", db_path); + snprintf(shm, sizeof(shm), "%s-shm", db_path); + cbm_unlink(wal); + cbm_unlink(shm); + } else { + cbm_file_hash_t *hashes = NULL; + int hash_count = 0; + cbm_store_get_file_hashes(check_store, p->project_name, &hashes, + &hash_count); + cbm_store_free_file_hashes(hashes, hash_count); + cbm_store_close(check_store); - if (hash_count > 0) { - cbm_log_info("pipeline.route", "path", "incremental", "stored_hashes", - itoa_buf(hash_count)); - rc = cbm_pipeline_run_incremental(p, db_path, files, file_count); - cbm_discover_free(files, file_count); - free(db_path); - return rc; + if (hash_count > 0) { + cbm_log_info("pipeline.route", "path", "incremental", "stored_hashes", + itoa_buf(hash_count)); + rc = cbm_pipeline_run_incremental(p, db_path, files, file_count); + cbm_discover_free(files, file_count); + free(db_path); + return rc; + } } } } diff --git a/src/store/store.c b/src/store/store.c index ddff36ee2..669b8136b 100644 --- a/src/store/store.c +++ b/src/store/store.c @@ -404,6 +404,54 @@ cbm_store_t *cbm_store_open_path_query(const char *db_path) { return s; } +/* ── Integrity check ───────────────────────────────────────────── */ + +bool cbm_store_check_integrity(cbm_store_t *s) { + if (!s || !s->db) { + return false; + } + + /* Each project gets its own .db file, so the projects table should have + * exactly 1 row. More than 5 rows is definitely corrupt (allows some slack + * for edge cases). Also check that root_path looks like a real path. */ + sqlite3_stmt *stmt = NULL; + int rc = sqlite3_prepare_v2(s->db, "SELECT count(*) FROM projects;", -1, &stmt, NULL); + if (rc != SQLITE_OK) { + return false; + } + + bool ok = true; + if (sqlite3_step(stmt) == SQLITE_ROW) { + int row_count = sqlite3_column_int(stmt, 0); + if (row_count > 5) { + fprintf(stderr, "ERROR store.corrupt table=projects rows=%d (expected 1)\n", row_count); + ok = false; + } + } + sqlite3_finalize(stmt); + + if (ok) { + /* Check that root_path in projects table starts with '/' or a drive letter. + * Corrupt DBs often have numeric strings like "826" in root_path. */ + rc = sqlite3_prepare_v2( + s->db, + "SELECT root_path FROM projects WHERE root_path != '' " + "AND substr(root_path, 1, 1) NOT IN ('/', 'A','B','C','D','E','F','G','H') LIMIT 1;", + -1, &stmt, NULL); + if (rc == SQLITE_OK) { + if (sqlite3_step(stmt) == SQLITE_ROW) { + const char *bad_path = (const char *)sqlite3_column_text(stmt, 0); + fprintf(stderr, "ERROR store.corrupt table=projects bad_root_path=%s\n", + bad_path ? bad_path : "(null)"); + ok = false; + } + sqlite3_finalize(stmt); + } + } + + return ok; +} + cbm_store_t *cbm_store_open(const char *project) { if (!project) { return NULL; @@ -470,6 +518,10 @@ void cbm_store_close(cbm_store_t *s) { free(s); } +sqlite3 *cbm_store_get_db(cbm_store_t *s) { + return s ? s->db : NULL; +} + const char *cbm_store_error(cbm_store_t *s) { return s ? s->errbuf : "null store"; } diff --git a/src/store/store.h b/src/store/store.h index b40234138..17b0df116 100644 --- a/src/store/store.h +++ b/src/store/store.h @@ -198,12 +198,20 @@ cbm_store_t *cbm_store_open_path(const char *db_path); * Returns NULL if the file does not exist — never creates a new .db file. */ cbm_store_t *cbm_store_open_path_query(const char *db_path); +/* Check database integrity. Returns true if the DB passes basic sanity checks + * (projects table has correct types, no corruption indicators). + * Returns false if corruption is detected — caller should delete and re-index. */ +bool cbm_store_check_integrity(cbm_store_t *s); + /* Open database for a named project in the default cache dir. */ cbm_store_t *cbm_store_open(const char *project); /* Close the store and free all resources. NULL-safe. */ void cbm_store_close(cbm_store_t *s); +/* Get the underlying sqlite3 handle (for testing only). */ +struct sqlite3 *cbm_store_get_db(cbm_store_t *s); + /* Get the last error message (static string, valid until next call). */ const char *cbm_store_error(cbm_store_t *s); diff --git a/tests/test_store_nodes.c b/tests/test_store_nodes.c index 4d56f081d..4160a15aa 100644 --- a/tests/test_store_nodes.c +++ b/tests/test_store_nodes.c @@ -6,6 +6,7 @@ */ #include "test_framework.h" #include +#include #include #include #include @@ -895,10 +896,75 @@ TEST(store_find_node_ids_by_qns) { PASS(); } +/* ── Integrity check tests ──────────────────────────────────────── */ + +TEST(store_integrity_clean) { + /* A fresh store with correct data should pass integrity check */ + cbm_store_t *s = cbm_store_open_memory(); + ASSERT_NOT_NULL(s); + cbm_store_upsert_project(s, "test-proj", "/tmp/test"); + ASSERT_TRUE(cbm_store_check_integrity(s)); + cbm_store_close(s); + PASS(); +} + +TEST(store_integrity_empty) { + /* An empty store (no project rows) should pass — 0 rows is fine */ + cbm_store_t *s = cbm_store_open_memory(); + ASSERT_NOT_NULL(s); + ASSERT_TRUE(cbm_store_check_integrity(s)); + cbm_store_close(s); + PASS(); +} + +TEST(store_integrity_corrupt_bad_path) { + /* Simulate corruption: root_path is a numeric string (not a real path). + * This matches the real corruption where node IDs ended up in root_path. */ + cbm_store_t *s = cbm_store_open_memory(); + ASSERT_NOT_NULL(s); + sqlite3 *db = cbm_store_get_db(s); + sqlite3_exec(db, + "INSERT INTO projects (name, indexed_at, root_path) " + "VALUES ('some-project', '2024-01-01', '826');", + NULL, NULL, NULL); + ASSERT_FALSE(cbm_store_check_integrity(s)); + cbm_store_close(s); + PASS(); +} + +TEST(store_integrity_corrupt_too_many_rows) { + /* Simulate corruption: >5 rows in projects table */ + cbm_store_t *s = cbm_store_open_memory(); + ASSERT_NOT_NULL(s); + sqlite3 *db = cbm_store_get_db(s); + for (int i = 0; i < 10; i++) { + char sql[256]; + snprintf(sql, sizeof(sql), + "INSERT INTO projects (name, indexed_at, root_path) " + "VALUES ('proj-%d', '2024-01-01', '/tmp/%d');", + i, i); + sqlite3_exec(db, sql, NULL, NULL, NULL); + } + ASSERT_FALSE(cbm_store_check_integrity(s)); + cbm_store_close(s); + PASS(); +} + +TEST(store_integrity_null_check) { + /* NULL store should return false (not crash) */ + ASSERT_FALSE(cbm_store_check_integrity(NULL)); + PASS(); +} + SUITE(store_nodes) { RUN_TEST(store_open_memory); RUN_TEST(store_close_null); RUN_TEST(store_open_memory_twice); + RUN_TEST(store_integrity_clean); + RUN_TEST(store_integrity_empty); + RUN_TEST(store_integrity_corrupt_bad_path); + RUN_TEST(store_integrity_corrupt_too_many_rows); + RUN_TEST(store_integrity_null_check); RUN_TEST(store_project_crud); RUN_TEST(store_project_update); RUN_TEST(store_project_delete);