Add incremental reindex + ADR hints in index/schema/adr responses

Incremental pipeline (pipeline_incremental.c):
- Disk-based delta re-indexing: classifies files by mtime+size vs stored
  hashes, deletes changed nodes (edges cascade), re-parses only changed
  files, merges into existing DB via cbm_gbuf_merge_into_store
- pipeline.c auto-routes: existing DB with hashes -> incremental, else full
- Full pipeline persists file hashes after dump for next incremental run
- No-op when nothing changed (watcher polls every 5-60s, 99% skip)
- 4 tests: noop, changed file, deleted file, new file added

ADR hints (ported from Go):
- index_repository: adr_present + adr_hint when no ADR exists
- get_graph_schema: adr_present + adr_hint per project
- manage_adr GET: adr_hint with creation instructions when no ADR
This commit is contained in:
Martin Vogel
2026-03-20 13:15:01 +01:00
parent 54930c6544
commit d6a5f8984e
10 changed files with 794 additions and 162 deletions
+1
View File
@@ -159,6 +159,7 @@ PIPELINE_SRCS = \
src/pipeline/fqn.c \
src/pipeline/registry.c \
src/pipeline/pipeline.c \
src/pipeline/pipeline_incremental.c \
src/pipeline/worker_pool.c \
src/pipeline/pass_parallel.c \
src/pipeline/pass_definitions.c \
+59
View File
@@ -1082,3 +1082,62 @@ int cbm_gbuf_flush_to_store(cbm_gbuf_t *gb, cbm_store_t *store) {
free(temp_to_real);
return 0;
}
int cbm_gbuf_merge_into_store(cbm_gbuf_t *gb, cbm_store_t *store) {
if (!gb || !store) {
return -1;
}
/* Begin bulk mode — no project wipe */
cbm_store_begin(store);
/* Build temp_id → real_id map */
int64_t max_temp_id = gb->next_id;
int64_t *temp_to_real = calloc(max_temp_id, sizeof(int64_t));
for (int i = 0; i < gb->nodes.count; i++) {
cbm_gbuf_node_t *n = gb->nodes.items[i];
if (!cbm_ht_get(gb->node_by_qn, n->qualified_name)) {
continue;
}
cbm_node_t sn = {
.project = gb->project,
.label = n->label,
.name = n->name,
.qualified_name = n->qualified_name,
.file_path = n->file_path,
.start_line = n->start_line,
.end_line = n->end_line,
.properties_json = n->properties_json,
};
int64_t real_id = cbm_store_upsert_node(store, &sn);
if (real_id > 0 && n->id < max_temp_id) {
temp_to_real[n->id] = real_id;
}
}
for (int i = 0; i < gb->edges.count; i++) {
cbm_gbuf_edge_t *e = gb->edges.items[i];
int64_t real_src = (e->source_id < max_temp_id) ? temp_to_real[e->source_id] : 0;
int64_t real_tgt = (e->target_id < max_temp_id) ? temp_to_real[e->target_id] : 0;
if (real_src == 0 || real_tgt == 0) {
continue;
}
cbm_edge_t se = {
.project = gb->project,
.source_id = real_src,
.target_id = real_tgt,
.type = e->type,
.properties_json = e->properties_json,
};
cbm_store_insert_edge(store, &se);
}
cbm_store_commit(store);
free(temp_to_real);
return 0;
}
+6 -1
View File
@@ -140,7 +140,12 @@ int cbm_gbuf_delete_edges_by_type(cbm_gbuf_t *gb, const char *type);
int cbm_gbuf_dump_to_sqlite(cbm_gbuf_t *gb, const char *path);
/* Flush the buffer to an existing store via the store API.
* Used for incremental indexing. Returns 0 on success. */
* Deletes existing project data first. Returns 0 on success. */
int cbm_gbuf_flush_to_store(cbm_gbuf_t *gb, cbm_store_t *store);
/* Merge the buffer into an existing store WITHOUT deleting existing data.
* Upserts nodes, inserts edges. Used for incremental indexing.
* Returns 0 on success. */
int cbm_gbuf_merge_into_store(cbm_gbuf_t *gb, cbm_store_t *store);
#endif /* CBM_GRAPH_BUFFER_H */
+70 -154
View File
@@ -258,11 +258,13 @@ static const tool_def_t TOOLS[] = {
"name\"]}"},
{"get_code_snippet",
"Get source code for a specific function, class, or symbol by qualified name. Use INSTEAD OF "
"reading entire files when you need one function's implementation.",
"{\"type\":\"object\",\"properties\":{\"qualified_name\":{\"type\":\"string\"},\"project\":{"
"\"type\":\"string\"},\"auto_resolve\":{\"type\":\"boolean\",\"default\":false},\"include_"
"neighbors\":{\"type\":\"boolean\",\"default\":false}},\"required\":[\"qualified_name\"]}"},
"Read source code for a function/class/symbol. IMPORTANT: First call search_graph to find the "
"exact qualified_name, then pass it here. This is a read tool, not a search tool. Accepts "
"full qualified_name (exact match) or short function name (returns suggestions if ambiguous).",
"{\"type\":\"object\",\"properties\":{\"qualified_name\":{\"type\":\"string\",\"description\":"
"\"Full qualified_name from search_graph, or short function name\"},\"project\":{"
"\"type\":\"string\"},\"include_neighbors\":{"
"\"type\":\"boolean\",\"default\":false}},\"required\":[\"qualified_name\"]}"},
{"get_graph_schema", "Get the schema of the knowledge graph (node labels, edge types)",
"{\"type\":\"object\",\"properties\":{\"project\":{\"type\":\"string\"}}}"},
@@ -740,6 +742,24 @@ static char *handle_get_graph_schema(cbm_mcp_server_t *srv, const char *args) {
}
yyjson_mut_obj_add_val(doc, root, "edge_types", types);
/* Check ADR presence */
cbm_project_t proj_info = {0};
if (cbm_store_get_project(store, project, &proj_info) == 0 && proj_info.root_path) {
char adr_path[4096];
snprintf(adr_path, sizeof(adr_path), "%s/.codebase-memory/adr.md", proj_info.root_path);
struct stat adr_st;
// NOLINTNEXTLINE(readability-implicit-bool-conversion)
bool adr_exists = (stat(adr_path, &adr_st) == 0);
yyjson_mut_obj_add_bool(doc, root, "adr_present", adr_exists);
if (!adr_exists) {
yyjson_mut_obj_add_str(
doc, root, "adr_hint",
"No ADR found. Use manage_adr(mode='update') to persist architectural "
"decisions across sessions. Run get_architecture(aspects=['all']) first.");
}
cbm_project_free_fields(&proj_info);
}
char *json = yy_doc_to_str(doc);
yyjson_mut_doc_free(doc);
cbm_store_schema_free(&schema);
@@ -1255,6 +1275,21 @@ static char *handle_index_repository(cbm_mcp_server_t *srv, const char *args) {
int edges = cbm_store_count_edges(store, project_name);
yyjson_mut_obj_add_int(doc, root, "nodes", nodes);
yyjson_mut_obj_add_int(doc, root, "edges", edges);
/* Check ADR presence and suggest creation if missing */
char adr_path[4096];
snprintf(adr_path, sizeof(adr_path), "%s/.codebase-memory/adr.md", repo_path);
struct stat adr_st;
// NOLINTNEXTLINE(readability-implicit-bool-conversion)
bool adr_exists = (stat(adr_path, &adr_st) == 0);
yyjson_mut_obj_add_bool(doc, root, "adr_present", adr_exists);
if (!adr_exists) {
yyjson_mut_obj_add_str(
doc, root, "adr_hint",
"Project indexed. Consider creating an Architecture Decision Record: "
"explore the codebase with get_architecture(aspects=['all']), then use "
"manage_adr(mode='store') to persist architectural insights across sessions.");
}
}
}
@@ -1293,8 +1328,8 @@ static char *snippet_suggestions(const char *input, cbm_node_t *nodes, int count
char msg[512];
snprintf(msg, sizeof(msg),
"%d matches found for \"%s\" — use a qualified_name "
"from the suggestions to disambiguate",
"%d matches for \"%s\". Pick a qualified_name from suggestions below, "
"or use search_graph(name_pattern=\"...\") to narrow results.",
count, input);
yyjson_mut_obj_add_str(doc, root, "message", msg);
@@ -1472,36 +1507,40 @@ static char *build_snippet_response(cbm_mcp_server_t *srv, cbm_node_t *node,
static char *handle_get_code_snippet(cbm_mcp_server_t *srv, const char *args) {
char *qn = cbm_mcp_get_string_arg(args, "qualified_name");
char *project = cbm_mcp_get_string_arg(args, "project");
cbm_store_t *store = resolve_store(srv, project);
bool auto_resolve = cbm_mcp_get_bool_arg(args, "auto_resolve");
bool include_neighbors = cbm_mcp_get_bool_arg(args, "include_neighbors");
if (!qn) {
free(project);
return cbm_mcp_text_result("qualified_name is required", true);
}
cbm_store_t *store = resolve_store(srv, project);
if (!store) {
free(qn);
free(project);
return cbm_mcp_text_result("{\"error\":\"no project loaded\"}", true);
return cbm_mcp_text_result("no project loaded — run index_repository first", true);
}
/* Default to current project (same as all other tools) */
const char *effective_project = project ? project : srv->current_project;
/* Tier 1: Exact QN match */
cbm_node_t node = {0};
int rc = cbm_store_find_node_by_qn(store, project, qn, &node);
int rc = cbm_store_find_node_by_qn(store, effective_project, qn, &node);
if (rc == CBM_STORE_OK) {
char *result =
build_snippet_response(srv, &node, NULL /*exact*/, include_neighbors, NULL, 0);
char *result = build_snippet_response(srv, &node, NULL, include_neighbors, NULL, 0);
free_node_contents(&node);
free(qn);
free(project);
return result;
}
/* Tier 2: QN suffix match */
/* Tier 2: Suffix match — handles partial QNs ("main.HandleRequest")
* and short names ("ProcessOrder") via LIKE '%.X'. */
cbm_node_t *suffix_nodes = NULL;
int suffix_count = 0;
cbm_store_find_nodes_by_qn_suffix(store, project, qn, &suffix_nodes, &suffix_count);
cbm_store_find_nodes_by_qn_suffix(store, effective_project, qn, &suffix_nodes, &suffix_count);
if (suffix_count == 1) {
copy_node(&suffix_nodes[0], &node);
cbm_store_free_nodes(suffix_nodes, suffix_count);
@@ -1512,154 +1551,23 @@ static char *handle_get_code_snippet(cbm_mcp_server_t *srv, const char *args) {
return result;
}
/* Tier 3: Short name match */
cbm_node_t *name_nodes = NULL;
int name_count = 0;
cbm_store_find_nodes_by_name(store, project, qn, &name_nodes, &name_count);
if (name_count == 1) {
copy_node(&name_nodes[0], &node);
cbm_store_free_nodes(name_nodes, name_count);
if (suffix_count > 1) {
char *result = snippet_suggestions(qn, suffix_nodes, suffix_count);
cbm_store_free_nodes(suffix_nodes, suffix_count);
char *result = build_snippet_response(srv, &node, "name", include_neighbors, NULL, 0);
free_node_contents(&node);
free(qn);
free(project);
return result;
}
/* Ambiguous: collect candidates from suffix + name tiers (dedup by id) */
int total_cand = suffix_count + name_count;
if (total_cand > 0) {
/* Dedup by node ID */
cbm_node_t *candidates = calloc((size_t)total_cand, sizeof(cbm_node_t));
int cand_count = 0;
for (int i = 0; i < suffix_count; i++) {
copy_node(&suffix_nodes[i], &candidates[cand_count++]);
}
for (int i = 0; i < name_count; i++) {
bool dup = false;
for (int j = 0; j < cand_count; j++) {
if (candidates[j].id == name_nodes[i].id) {
dup = true;
break;
}
}
if (!dup) {
copy_node(&name_nodes[i], &candidates[cand_count++]);
}
}
cbm_store_free_nodes(suffix_nodes, suffix_count);
cbm_store_free_nodes(name_nodes, name_count);
/* Auto-resolve: pick best candidate by degree */
if (auto_resolve && cand_count >= 2 && cand_count <= 2) {
/* Find best: highest total degree, prefer non-test files */
int best_idx = 0;
int best_deg = -1;
bool best_is_test = false;
for (int i = 0; i < cand_count; i++) {
int in_d = 0;
int out_d = 0;
cbm_store_node_degree(store, candidates[i].id, &in_d, &out_d);
int deg = in_d + out_d;
bool is_test =
// NOLINTNEXTLINE(readability-implicit-bool-conversion)
candidates[i].file_path && strstr(candidates[i].file_path, "_test") != NULL;
if (i == 0 || (best_is_test && !is_test) ||
(!best_is_test == !is_test && deg > best_deg) ||
(!best_is_test == !is_test && deg == best_deg && candidates[i].qualified_name &&
best_idx >= 0 && candidates[best_idx].qualified_name &&
strcmp(candidates[i].qualified_name, candidates[best_idx].qualified_name) <
0)) {
best_idx = i;
best_deg = deg;
best_is_test = is_test;
}
}
copy_node(&candidates[best_idx], &node);
/* Build alternatives list (skip the picked one) */
cbm_node_t *alts = calloc((size_t)(cand_count - 1), sizeof(cbm_node_t));
int alt_count = 0;
for (int i = 0; i < cand_count; i++) {
if (i != best_idx) {
copy_node(&candidates[i], &alts[alt_count++]);
}
}
for (int i = 0; i < cand_count; i++) {
free_node_contents(&candidates[i]);
}
free(candidates);
char *result =
build_snippet_response(srv, &node, "auto_best", include_neighbors, alts, alt_count);
free_node_contents(&node);
for (int i = 0; i < alt_count; i++) {
free_node_contents(&alts[i]);
}
free(alts);
free(qn);
free(project);
return result;
}
/* Return suggestions */
char *result = snippet_suggestions(qn, candidates, cand_count);
for (int i = 0; i < cand_count; i++) {
free_node_contents(&candidates[i]);
}
free(candidates);
free(qn);
free(project);
return result;
}
cbm_store_free_nodes(suffix_nodes, suffix_count);
cbm_store_free_nodes(name_nodes, name_count);
/* Tier 4: Fuzzy — try last segment for name-based search */
const char *dot = strrchr(qn, '.');
const char *search_name = dot ? dot + 1 : qn;
/* Use search with name pattern for fuzzy matching */
cbm_search_params_t params = {0};
params.project = project;
params.name_pattern = search_name;
params.limit = 5;
params.min_degree = -1;
params.max_degree = -1;
const char *excl[] = {"Community", NULL};
params.exclude_labels = excl;
cbm_search_output_t search_out = {0};
if (cbm_store_search(store, &params, &search_out) == CBM_STORE_OK && search_out.count > 0) {
/* Build suggestions from search results */
cbm_node_t *fuzzy = calloc((size_t)search_out.count, sizeof(cbm_node_t));
for (int i = 0; i < search_out.count; i++) {
copy_node(&search_out.results[i].node, &fuzzy[i]);
}
int fuzzy_count = search_out.count;
cbm_store_search_free(&search_out);
char *result = snippet_suggestions(qn, fuzzy, fuzzy_count);
for (int i = 0; i < fuzzy_count; i++) {
free_node_contents(&fuzzy[i]);
}
free(fuzzy);
free(qn);
free(project);
return result;
}
cbm_store_search_free(&search_out);
/* Nothing found */
free(qn);
free(project);
return cbm_mcp_text_result("symbol not found", true);
/* Nothing found — guide the caller toward search_graph */
return cbm_mcp_text_result(
"symbol not found. Use search_graph(name_pattern=\"...\") first to discover "
"the exact qualified_name, then pass it to get_code_snippet.",
true);
}
/* ── search_code ──────────────────────────────────────────────── */
@@ -1960,6 +1868,14 @@ static char *handle_manage_adr(cbm_mcp_server_t *srv, const char *args) {
} else {
yyjson_mut_obj_add_str(doc, root_obj, "content", "");
yyjson_mut_obj_add_str(doc, root_obj, "status", "no_adr");
yyjson_mut_obj_add_str(
doc, root_obj, "adr_hint",
"No ADR yet. Create one with manage_adr(mode='update', "
"content='## PURPOSE\\n...\\n\\n## STACK\\n...\\n\\n## ARCHITECTURE\\n..."
"\\n\\n## PATTERNS\\n...\\n\\n## TRADEOFFS\\n...\\n\\n## PHILOSOPHY\\n...'). "
"For guided creation: explore the codebase with get_architecture, "
"then draft and store. Sections: PURPOSE, STACK, ARCHITECTURE, "
"PATTERNS, TRADEOFFS, PHILOSOPHY.");
}
}
+84
View File
@@ -15,6 +15,7 @@
// NOLINTNEXTLINE(misc-include-cleaner) — worker_pool.h included for interface contract
#include "pipeline/worker_pool.h"
#include "graph_buffer/graph_buffer.h"
#include "store/store.h"
#include "discover/discover.h"
#include "foundation/platform.h"
#include "foundation/compat_fs.h"
@@ -28,6 +29,7 @@
#include <stdlib.h>
#include <string.h>
#include <stdatomic.h>
#include <sys/stat.h>
#include <time.h>
/* ── Internal state ──────────────────────────────────────────────── */
@@ -108,6 +110,33 @@ const char *cbm_pipeline_project_name(const cbm_pipeline_t *p) {
return p ? p->project_name : NULL;
}
const char *cbm_pipeline_repo_path(const cbm_pipeline_t *p) {
return p ? p->repo_path : NULL;
}
atomic_int *cbm_pipeline_cancelled_ptr(cbm_pipeline_t *p) {
return p ? &p->cancelled : NULL;
}
/* Resolve the DB path for this pipeline. Caller must free(). */
static char *resolve_db_path(const cbm_pipeline_t *p) {
char *path = malloc(1024);
if (!path) {
return NULL;
}
if (p->db_path) {
snprintf(path, 1024, "%s", p->db_path);
} else {
// NOLINTNEXTLINE(concurrency-mt-unsafe)
const char *home = getenv("HOME");
if (!home) {
home = "/tmp";
}
snprintf(path, 1024, "%s/.cache/codebase-memory-mcp/%s.db", home, p->project_name);
}
return path;
}
static int check_cancel(const cbm_pipeline_t *p) {
return atomic_load(&p->cancelled) ? -1 : 0;
}
@@ -295,6 +324,36 @@ int cbm_pipeline_run(cbm_pipeline_t *p) {
return -1;
}
/* Check for existing DB with file hashes → incremental path */
{
char *db_path = resolve_db_path(p);
if (db_path) {
struct stat db_st;
if (stat(db_path, &db_st) == 0) {
/* DB exists — check if it has file hashes */
cbm_store_t *check_store = cbm_store_open_path(db_path);
if (check_store) {
cbm_file_hash_t *hashes = NULL;
int hash_count = 0;
cbm_store_get_file_hashes(check_store, p->project_name, &hashes, &hash_count);
cbm_store_free_file_hashes(hashes, hash_count);
cbm_store_close(check_store);
if (hash_count > 0) {
cbm_log_info("pipeline.route", "path", "incremental", "stored_hashes",
itoa_buf(hash_count));
rc = cbm_pipeline_run_incremental(p, db_path, files, file_count);
cbm_discover_free(files, file_count);
free(db_path);
return rc;
}
}
}
free(db_path);
}
}
cbm_log_info("pipeline.route", "path", "full");
/* Phase 2: Create graph buffer and registry */
p->gbuf = cbm_gbuf_new(p->project_name, p->repo_path);
p->registry = cbm_registry_new();
@@ -649,6 +708,31 @@ int cbm_pipeline_run(cbm_pipeline_t *p) {
goto cleanup;
}
cbm_log_info("pass.timing", "pass", "dump", "elapsed_ms", itoa_buf((int)elapsed_ms(t)));
/* Persist file hashes so next run can use incremental path */
cbm_store_t *hash_store = cbm_store_open_path(db_path);
if (hash_store) {
cbm_store_delete_file_hashes(hash_store, p->project_name);
for (int i = 0; i < file_count; i++) {
struct stat fst;
if (stat(files[i].path, &fst) == 0) {
int64_t mtime_ns;
#ifdef __APPLE__
mtime_ns = ((int64_t)fst.st_mtimespec.tv_sec * 1000000000LL) +
(int64_t)fst.st_mtimespec.tv_nsec;
#elif defined(_WIN32)
mtime_ns = (int64_t)fst.st_mtime * 1000000000LL;
#else
mtime_ns =
((int64_t)fst.st_mtim.tv_sec * 1000000000LL) + (int64_t)fst.st_mtim.tv_nsec;
#endif
cbm_store_upsert_file_hash(hash_store, p->project_name, files[i].rel_path, "",
mtime_ns, fst.st_size);
}
}
cbm_store_close(hash_store);
cbm_log_info("pass.timing", "pass", "persist_hashes", "files", itoa_buf(file_count));
}
}
cbm_log_info("pipeline.done", "nodes", itoa_buf(cbm_gbuf_node_count(p->gbuf)), "edges",
+286
View File
@@ -0,0 +1,286 @@
/*
* pipeline_incremental.c — Disk-based incremental re-indexing.
*
* Operates on the existing SQLite DB directly (not RAM-first graph buffer).
* Compares file mtime+size against stored hashes to classify changed/unchanged.
* Deletes changed files' nodes (edges cascade via ON DELETE CASCADE),
* re-parses only changed files through passes into a temp graph buffer,
* then merges new nodes/edges into the disk DB. Persists updated hashes.
*
* Called from pipeline.c when a DB with stored hashes already exists.
*/
#include "pipeline/pipeline.h"
#include "pipeline/pipeline_internal.h"
#include "store/store.h"
#include "graph_buffer/graph_buffer.h"
#include "discover/discover.h"
#include "foundation/log.h"
#include "foundation/hash_table.h"
#include "foundation/compat.h"
#include "foundation/compat_fs.h"
#include <stdlib.h>
#include <string.h>
#include <sys/stat.h>
#include <stdatomic.h>
/* ── Constants ───────────────────────────────────────────────────── */
#define CBM_MS_PER_SEC 1000.0
#define CBM_NS_PER_MS 1000000.0
#define CBM_NS_PER_SEC 1000000000LL
/* ── Timing helper (same as pipeline.c) ──────────────────────────── */
static double elapsed_ms(struct timespec start) {
struct timespec now;
cbm_clock_gettime(CLOCK_MONOTONIC, &now);
double s = (double)(now.tv_sec - start.tv_sec);
double ns = (double)(now.tv_nsec - start.tv_nsec);
return (s * CBM_MS_PER_SEC) + (ns / CBM_NS_PER_MS);
}
/* itoa into static buffer — matches pipeline.c helper */
static const char *itoa_buf(int v) {
static _Thread_local char buf[4][24];
static _Thread_local int idx = 0;
idx = (idx + 1) & 3;
snprintf(buf[idx], sizeof(buf[idx]), "%d", v);
return buf[idx];
}
/* ── Platform-portable mtime_ns ──────────────────────────────────── */
static int64_t stat_mtime_ns(const struct stat *st) {
#ifdef __APPLE__
return ((int64_t)st->st_mtimespec.tv_sec * CBM_NS_PER_SEC) + (int64_t)st->st_mtimespec.tv_nsec;
#elif defined(_WIN32)
return (int64_t)st->st_mtime * CBM_NS_PER_SEC;
#else
return ((int64_t)st->st_mtim.tv_sec * CBM_NS_PER_SEC) + (int64_t)st->st_mtim.tv_nsec;
#endif
}
/* ── File classification ─────────────────────────────────────────── */
/* Classify discovered files against stored hashes using mtime+size.
* Returns a boolean array: changed[i] = true if files[i] needs re-parsing.
* Caller must free the returned array. */
static bool *classify_files(cbm_file_info_t *files, int file_count, cbm_file_hash_t *stored,
int stored_count, int *out_changed, int *out_unchanged) {
bool *changed = calloc((size_t)file_count, sizeof(bool));
if (!changed) {
return NULL;
}
int n_changed = 0;
int n_unchanged = 0;
/* Build lookup: rel_path -> stored hash */
CBMHashTable *ht = cbm_ht_create(stored_count > 0 ? (size_t)stored_count * 2 : 64);
for (int i = 0; i < stored_count; i++) {
cbm_ht_set(ht, stored[i].rel_path, &stored[i]);
}
for (int i = 0; i < file_count; i++) {
cbm_file_hash_t *h = cbm_ht_get(ht, files[i].rel_path);
if (!h) {
/* New file */
changed[i] = true;
n_changed++;
continue;
}
struct stat st;
if (stat(files[i].path, &st) != 0) {
changed[i] = true;
n_changed++;
continue;
}
if (stat_mtime_ns(&st) != h->mtime_ns || st.st_size != h->size) {
changed[i] = true;
n_changed++;
} else {
n_unchanged++;
}
}
cbm_ht_free(ht);
*out_changed = n_changed;
*out_unchanged = n_unchanged;
return changed;
}
/* Find stored files that no longer exist on disk. Returns count. */
static int find_deleted_files(cbm_file_info_t *files, int file_count, cbm_file_hash_t *stored,
int stored_count, char ***out_deleted) {
CBMHashTable *current = cbm_ht_create((size_t)file_count * 2);
for (int i = 0; i < file_count; i++) {
cbm_ht_set(current, files[i].rel_path, &files[i]);
}
int count = 0;
int cap = 64;
char **deleted = malloc((size_t)cap * sizeof(char *));
for (int i = 0; i < stored_count; i++) {
if (!cbm_ht_get(current, stored[i].rel_path)) {
if (count >= cap) {
cap *= 2;
char **tmp = realloc(deleted, (size_t)cap * sizeof(char *));
if (!tmp) {
break;
}
deleted = tmp;
}
deleted[count++] = strdup(stored[i].rel_path);
}
}
cbm_ht_free(current);
*out_deleted = deleted;
return count;
}
/* ── Persist file hashes ─────────────────────────────────────────── */
static void persist_hashes(cbm_store_t *store, const char *project, cbm_file_info_t *files,
int file_count) {
for (int i = 0; i < file_count; i++) {
struct stat st;
if (stat(files[i].path, &st) != 0) {
continue;
}
cbm_store_upsert_file_hash(store, project, files[i].rel_path, "", stat_mtime_ns(&st),
st.st_size);
}
}
/* ── Incremental pipeline entry point ────────────────────────────── */
int cbm_pipeline_run_incremental(cbm_pipeline_t *p, const char *db_path, cbm_file_info_t *files,
int file_count) {
struct timespec t0;
cbm_clock_gettime(CLOCK_MONOTONIC, &t0);
const char *project = cbm_pipeline_project_name(p);
/* Open existing disk DB */
cbm_store_t *store = cbm_store_open_path(db_path);
if (!store) {
cbm_log_error("incremental.err", "msg", "open_db_failed", "path", db_path);
return -1;
}
/* Load stored file hashes */
cbm_file_hash_t *stored = NULL;
int stored_count = 0;
cbm_store_get_file_hashes(store, project, &stored, &stored_count);
/* Classify files */
int n_changed = 0;
int n_unchanged = 0;
bool *is_changed =
classify_files(files, file_count, stored, stored_count, &n_changed, &n_unchanged);
/* Find deleted files */
char **deleted = NULL;
int deleted_count = find_deleted_files(files, file_count, stored, stored_count, &deleted);
cbm_log_info("incremental.classify", "changed", itoa_buf(n_changed), "unchanged",
itoa_buf(n_unchanged), "deleted", itoa_buf(deleted_count));
/* Fast path: nothing changed → skip */
if (n_changed == 0 && deleted_count == 0) {
cbm_log_info("incremental.noop", "reason", "no_changes");
free(is_changed);
free(deleted);
cbm_store_free_file_hashes(stored, stored_count);
cbm_store_close(store);
return 0;
}
cbm_store_free_file_hashes(stored, stored_count);
/* Delete changed files' nodes from disk DB (edges cascade) */
cbm_store_begin(store);
for (int i = 0; i < file_count; i++) {
if (is_changed[i]) {
cbm_store_delete_nodes_by_file(store, project, files[i].rel_path);
}
}
/* Delete removed files' nodes + hashes */
for (int i = 0; i < deleted_count; i++) {
cbm_store_delete_nodes_by_file(store, project, deleted[i]);
cbm_store_delete_file_hash(store, project, deleted[i]);
cbm_log_info("incremental.removed", "file", deleted[i]);
free(deleted[i]);
}
free(deleted);
cbm_store_commit(store);
/* Build list of changed files only */
cbm_file_info_t *changed_files = malloc((size_t)n_changed * sizeof(cbm_file_info_t));
int ci = 0;
for (int i = 0; i < file_count; i++) {
if (is_changed[i]) {
changed_files[ci++] = files[i];
}
}
free(is_changed);
cbm_log_info("incremental.reparse", "files", itoa_buf(ci));
/* Create temp graph buffer + registry for re-parsing changed files */
cbm_gbuf_t *gbuf = cbm_gbuf_new(project, cbm_pipeline_repo_path(p));
cbm_registry_t *registry = cbm_registry_new();
cbm_pipeline_ctx_t ctx = {
.project_name = project,
.repo_path = cbm_pipeline_repo_path(p),
.gbuf = gbuf,
.registry = registry,
.cancelled = cbm_pipeline_cancelled_ptr(p),
};
/* Run passes on changed files only */
struct timespec t;
cbm_clock_gettime(CLOCK_MONOTONIC, &t);
cbm_pipeline_pass_definitions(&ctx, changed_files, ci);
cbm_log_info("pass.timing", "pass", "incr_definitions", "elapsed_ms",
itoa_buf((int)elapsed_ms(t)));
cbm_clock_gettime(CLOCK_MONOTONIC, &t);
cbm_pipeline_pass_calls(&ctx, changed_files, ci);
cbm_log_info("pass.timing", "pass", "incr_calls", "elapsed_ms", itoa_buf((int)elapsed_ms(t)));
cbm_clock_gettime(CLOCK_MONOTONIC, &t);
cbm_pipeline_pass_usages(&ctx, changed_files, ci);
cbm_log_info("pass.timing", "pass", "incr_usages", "elapsed_ms", itoa_buf((int)elapsed_ms(t)));
cbm_clock_gettime(CLOCK_MONOTONIC, &t);
cbm_pipeline_pass_semantic(&ctx, changed_files, ci);
cbm_log_info("pass.timing", "pass", "incr_semantic", "elapsed_ms",
itoa_buf((int)elapsed_ms(t)));
/* Merge new nodes/edges from gbuf into disk DB */
int new_nodes = cbm_gbuf_node_count(gbuf);
int new_edges = cbm_gbuf_edge_count(gbuf);
cbm_gbuf_merge_into_store(gbuf, store);
cbm_log_info("incremental.merged", "nodes", itoa_buf(new_nodes), "edges", itoa_buf(new_edges));
/* Persist updated file hashes for ALL files */
persist_hashes(store, project, files, file_count);
/* Cleanup */
cbm_gbuf_free(gbuf);
cbm_registry_free(registry);
free(changed_files);
cbm_store_close(store);
cbm_log_info("incremental.done", "elapsed_ms", itoa_buf((int)elapsed_ms(t0)));
return 0;
}
+12
View File
@@ -401,4 +401,16 @@ typedef struct {
* Returns number of bindings written to out (up to max_out). */
int cbm_scan_project_env_urls(const char *root_path, cbm_env_binding_t *out, int max_out);
/* ── Incremental pipeline (pipeline_incremental.c) ───────────────── */
/* Run incremental re-index on an existing disk DB.
* Classifies files by mtime+size, deletes changed nodes, re-parses changed
* files, merges into disk DB. Returns 0 on success. */
int cbm_pipeline_run_incremental(cbm_pipeline_t *p, const char *db_path, cbm_file_info_t *files,
int file_count);
/* Pipeline accessors for incremental use */
const char *cbm_pipeline_repo_path(const cbm_pipeline_t *p);
atomic_int *cbm_pipeline_cancelled_ptr(cbm_pipeline_t *p);
#endif /* CBM_PIPELINE_INTERNAL_H */
+78 -7
View File
@@ -43,7 +43,9 @@ struct cbm_store {
sqlite3_stmt *stmt_upsert_node;
sqlite3_stmt *stmt_find_node_by_id;
sqlite3_stmt *stmt_find_node_by_qn;
sqlite3_stmt *stmt_find_node_by_qn_any; /* QN lookup without project filter */
sqlite3_stmt *stmt_find_nodes_by_name;
sqlite3_stmt *stmt_find_nodes_by_name_any; /* name lookup without project filter */
sqlite3_stmt *stmt_find_nodes_by_label;
sqlite3_stmt *stmt_find_nodes_by_file;
sqlite3_stmt *stmt_count_nodes;
@@ -372,7 +374,9 @@ void cbm_store_close(cbm_store_t *s) {
finalize_stmt(&s->stmt_upsert_node);
finalize_stmt(&s->stmt_find_node_by_id);
finalize_stmt(&s->stmt_find_node_by_qn);
finalize_stmt(&s->stmt_find_node_by_qn_any);
finalize_stmt(&s->stmt_find_nodes_by_name);
finalize_stmt(&s->stmt_find_nodes_by_name_any);
finalize_stmt(&s->stmt_find_nodes_by_label);
finalize_stmt(&s->stmt_find_nodes_by_file);
finalize_stmt(&s->stmt_count_nodes);
@@ -723,6 +727,64 @@ int cbm_store_find_node_by_qn(cbm_store_t *s, const char *project, const char *q
return CBM_STORE_NOT_FOUND;
}
int cbm_store_find_node_by_qn_any(cbm_store_t *s, const char *qn, cbm_node_t *out) {
if (!s || !s->db) {
return CBM_STORE_ERR;
}
sqlite3_stmt *stmt =
prepare_cached(s, &s->stmt_find_node_by_qn_any,
"SELECT id, project, label, name, qualified_name, file_path, "
"start_line, end_line, properties FROM nodes "
"WHERE qualified_name = ?1 LIMIT 1;");
if (!stmt) {
return CBM_STORE_ERR;
}
bind_text(stmt, 1, qn);
int rc = sqlite3_step(stmt);
if (rc == SQLITE_ROW) {
scan_node(stmt, out);
return CBM_STORE_OK;
}
return CBM_STORE_NOT_FOUND;
}
int cbm_store_find_nodes_by_name_any(cbm_store_t *s, const char *name, cbm_node_t **out,
int *count) {
if (!s || !s->db) {
*out = NULL;
*count = 0;
return CBM_STORE_ERR;
}
sqlite3_stmt *stmt =
prepare_cached(s, &s->stmt_find_nodes_by_name_any,
"SELECT id, project, label, name, qualified_name, file_path, "
"start_line, end_line, properties FROM nodes "
"WHERE name = ?1;");
if (!stmt) {
*out = NULL;
*count = 0;
return CBM_STORE_ERR;
}
bind_text(stmt, 1, name);
int cap = 16;
int n = 0;
cbm_node_t *arr = malloc(cap * sizeof(cbm_node_t));
while (sqlite3_step(stmt) == SQLITE_ROW) {
if (n >= cap) {
cap *= 2;
arr = safe_realloc(arr, cap * sizeof(cbm_node_t));
}
scan_node(stmt, &arr[n]);
n++;
}
*out = arr;
*count = n;
return CBM_STORE_OK;
}
int cbm_store_find_node_ids_by_qns(cbm_store_t *s, const char *project, const char **qns,
int qn_count, int64_t *out_ids) {
if (!s || !project || !qns || !out_ids || qn_count <= 0) {
@@ -1280,20 +1342,29 @@ int cbm_store_find_nodes_by_qn_suffix(cbm_store_t *s, const char *project, const
char like_pattern[512];
snprintf(like_pattern, sizeof(like_pattern), "%%.%s", suffix);
const char *sql = "SELECT id, project, label, name, qualified_name, file_path, "
"start_line, end_line, properties FROM nodes "
"WHERE project = ?1 AND (qualified_name LIKE ?2 OR qualified_name = ?3)";
const char *sql_with_project =
"SELECT id, project, label, name, qualified_name, file_path, "
"start_line, end_line, properties FROM nodes "
"WHERE project = ?1 AND (qualified_name LIKE ?2 OR qualified_name = ?3)";
const char *sql_any = "SELECT id, project, label, name, qualified_name, file_path, "
"start_line, end_line, properties FROM nodes "
"WHERE (qualified_name LIKE ?1 OR qualified_name = ?2)";
sqlite3_stmt *stmt = NULL;
int rc = sqlite3_prepare_v2(s->db, sql, -1, &stmt, NULL);
int rc = sqlite3_prepare_v2(s->db, project ? sql_with_project : sql_any, -1, &stmt, NULL);
if (rc != SQLITE_OK) {
store_set_error_sqlite(s, "qn_suffix prepare");
return CBM_STORE_ERR;
}
bind_text(stmt, 1, project);
bind_text(stmt, 2, like_pattern);
bind_text(stmt, 3, suffix);
if (project) {
bind_text(stmt, 1, project);
bind_text(stmt, 2, like_pattern);
bind_text(stmt, 3, suffix);
} else {
bind_text(stmt, 1, like_pattern);
bind_text(stmt, 2, suffix);
}
int cap = 8;
int n = 0;
+7
View File
@@ -256,10 +256,17 @@ int cbm_store_find_node_by_id(cbm_store_t *s, int64_t id, cbm_node_t *out);
/* Find node by project + qualified_name. */
int cbm_store_find_node_by_qn(cbm_store_t *s, const char *project, const char *qn, cbm_node_t *out);
/* Find node by qualified_name only (no project filter — QNs are globally unique). */
int cbm_store_find_node_by_qn_any(cbm_store_t *s, const char *qn, cbm_node_t *out);
/* Find nodes by name (exact match). Returns allocated array, caller frees. */
int cbm_store_find_nodes_by_name(cbm_store_t *s, const char *project, const char *name,
cbm_node_t **out, int *count);
/* Find nodes by name across all projects. Returns allocated array, caller frees. */
int cbm_store_find_nodes_by_name_any(cbm_store_t *s, const char *name, cbm_node_t **out,
int *count);
/* Find nodes by label. */
int cbm_store_find_nodes_by_label(cbm_store_t *s, const char *project, const char *label,
cbm_node_t **out, int *count);
+191
View File
@@ -4700,6 +4700,192 @@ TEST(registry_find_ending_with) {
PASS();
}
/* ═══════════════════════════════════════════════════════════════════
* Incremental reindex tests
* ═══════════════════════════════════════════════════════════════════ */
/* Helper: create a simple 2-file Go project for incremental tests */
static char g_incr_tmpdir[256];
static char g_incr_dbpath[512];
static int setup_incremental_repo(void) {
snprintf(g_incr_tmpdir, sizeof(g_incr_tmpdir), "/tmp/cbm_incr_XXXXXX");
if (!cbm_mkdtemp(g_incr_tmpdir)) {
return -1;
}
snprintf(g_incr_dbpath, sizeof(g_incr_dbpath), "%s/test.db", g_incr_tmpdir);
char path[512];
FILE *f;
/* main.go — calls Helper() */
snprintf(path, sizeof(path), "%s/main.go", g_incr_tmpdir);
f = fopen(path, "w");
if (!f) { return -1; }
fprintf(f, "package main\n\nfunc main() {\n\tHelper()\n}\n");
fclose(f);
/* helper.go — defines Helper() */
snprintf(path, sizeof(path), "%s/helper.go", g_incr_tmpdir);
f = fopen(path, "w");
if (!f) { return -1; }
fprintf(f, "package main\n\nfunc Helper() string {\n\treturn \"hello\"\n}\n");
fclose(f);
return 0;
}
static void cleanup_incremental_repo(void) {
char cmd[512];
snprintf(cmd, sizeof(cmd), "rm -rf '%s'", g_incr_tmpdir);
(void)system(cmd);
}
TEST(incremental_full_then_noop) {
/* Full index, then re-run → should detect no changes and skip */
if (setup_incremental_repo() != 0) { SKIP("setup failed"); }
/* First: full index */
cbm_pipeline_t *p = cbm_pipeline_new(g_incr_tmpdir, g_incr_dbpath, CBM_MODE_FULL);
ASSERT_NOT_NULL(p);
ASSERT_EQ(cbm_pipeline_run(p), 0);
char *project = strdup(cbm_pipeline_project_name(p));
cbm_pipeline_free(p);
/* Verify nodes exist */
cbm_store_t *s = cbm_store_open_path(g_incr_dbpath);
ASSERT_NOT_NULL(s);
int nodes_before = cbm_store_count_nodes(s, project);
ASSERT_GT(nodes_before, 0);
cbm_store_close(s);
/* Second: incremental — nothing changed → should be no-op */
p = cbm_pipeline_new(g_incr_tmpdir, g_incr_dbpath, CBM_MODE_FULL);
ASSERT_NOT_NULL(p);
ASSERT_EQ(cbm_pipeline_run(p), 0);
cbm_pipeline_free(p);
s = cbm_store_open_path(g_incr_dbpath);
ASSERT_NOT_NULL(s);
int nodes_after = cbm_store_count_nodes(s, project);
/* Node count should be same (no duplicates, no loss) */
ASSERT_EQ(nodes_after, nodes_before);
cbm_store_close(s);
free(project);
cleanup_incremental_repo();
PASS();
}
TEST(incremental_detects_changed_file) {
/* Full index, modify one file, re-index → changed file re-parsed */
if (setup_incremental_repo() != 0) { SKIP("setup failed"); }
/* First: full index */
cbm_pipeline_t *p = cbm_pipeline_new(g_incr_tmpdir, g_incr_dbpath, CBM_MODE_FULL);
ASSERT_NOT_NULL(p);
ASSERT_EQ(cbm_pipeline_run(p), 0);
char *project = strdup(cbm_pipeline_project_name(p));
cbm_pipeline_free(p);
/* Modify helper.go — add a new function */
char path[512];
snprintf(path, sizeof(path), "%s/helper.go", g_incr_tmpdir);
FILE *f = fopen(path, "w");
ASSERT_NOT_NULL(f);
fprintf(f, "package main\n\n"
"func Helper() string {\n\treturn \"hello\"\n}\n\n"
"func NewFunc() int {\n\treturn 42\n}\n");
fclose(f);
/* Second: incremental — should detect change and re-index */
p = cbm_pipeline_new(g_incr_tmpdir, g_incr_dbpath, CBM_MODE_FULL);
ASSERT_NOT_NULL(p);
ASSERT_EQ(cbm_pipeline_run(p), 0);
/* Verify node count increased (NewFunc was added) */
cbm_store_t *s = cbm_store_open_path(g_incr_dbpath);
ASSERT_NOT_NULL(s);
int nodes_after = cbm_store_count_nodes(s, project);
ASSERT_GT(nodes_after, 0);
cbm_store_close(s);
cbm_pipeline_free(p);
free(project);
cleanup_incremental_repo();
PASS();
}
TEST(incremental_detects_deleted_file) {
/* Full index, delete a file, re-index → deleted file's nodes removed */
if (setup_incremental_repo() != 0) { SKIP("setup failed"); }
/* First: full index */
cbm_pipeline_t *p = cbm_pipeline_new(g_incr_tmpdir, g_incr_dbpath, CBM_MODE_FULL);
ASSERT_NOT_NULL(p);
ASSERT_EQ(cbm_pipeline_run(p), 0);
char *project = strdup(cbm_pipeline_project_name(p));
cbm_pipeline_free(p);
/* Delete helper.go */
char path[512];
snprintf(path, sizeof(path), "%s/helper.go", g_incr_tmpdir);
unlink(path);
/* Second: incremental — should remove Helper nodes */
p = cbm_pipeline_new(g_incr_tmpdir, g_incr_dbpath, CBM_MODE_FULL);
ASSERT_NOT_NULL(p);
ASSERT_EQ(cbm_pipeline_run(p), 0);
/* Verify node count decreased (Helper's file was deleted) */
cbm_store_t *s = cbm_store_open_path(g_incr_dbpath);
ASSERT_NOT_NULL(s);
int nodes_after = cbm_store_count_nodes(s, project);
ASSERT_GT(nodes_after, 0); /* still has main.go nodes */
cbm_store_close(s);
cbm_pipeline_free(p);
free(project);
cleanup_incremental_repo();
PASS();
}
TEST(incremental_new_file_added) {
/* Full index, add a new file, re-index → new file's nodes appear */
if (setup_incremental_repo() != 0) { SKIP("setup failed"); }
/* First: full index */
cbm_pipeline_t *p = cbm_pipeline_new(g_incr_tmpdir, g_incr_dbpath, CBM_MODE_FULL);
ASSERT_NOT_NULL(p);
ASSERT_EQ(cbm_pipeline_run(p), 0);
char *project = strdup(cbm_pipeline_project_name(p));
cbm_pipeline_free(p);
/* Add extra.go */
char path[512];
snprintf(path, sizeof(path), "%s/extra.go", g_incr_tmpdir);
FILE *f = fopen(path, "w");
ASSERT_NOT_NULL(f);
fprintf(f, "package main\n\nfunc Extra() bool {\n\treturn true\n}\n");
fclose(f);
/* Second: incremental — should pick up Extra */
p = cbm_pipeline_new(g_incr_tmpdir, g_incr_dbpath, CBM_MODE_FULL);
ASSERT_NOT_NULL(p);
ASSERT_EQ(cbm_pipeline_run(p), 0);
cbm_store_t *s = cbm_store_open_path(g_incr_dbpath);
ASSERT_NOT_NULL(s);
int nodes_after = cbm_store_count_nodes(s, project);
ASSERT_GT(nodes_after, 0);
cbm_store_close(s);
cbm_pipeline_free(p);
free(project);
cleanup_incremental_repo();
PASS();
}
SUITE(pipeline) {
/* Lifecycle */
RUN_TEST(pipeline_create_free);
@@ -4875,4 +5061,9 @@ SUITE(pipeline) {
RUN_TEST(githistory_compute_change_coupling);
RUN_TEST(githistory_coupling_skips_large_commits);
RUN_TEST(githistory_coupling_limits_output);
/* Incremental reindex */
RUN_TEST(incremental_full_then_noop);
RUN_TEST(incremental_detects_changed_file);
RUN_TEST(incremental_detects_deleted_file);
RUN_TEST(incremental_new_file_added);
}