diff --git a/Makefile.cbm b/Makefile.cbm index 666a94551..547827b6b 100644 --- a/Makefile.cbm +++ b/Makefile.cbm @@ -159,6 +159,7 @@ PIPELINE_SRCS = \ src/pipeline/fqn.c \ src/pipeline/registry.c \ src/pipeline/pipeline.c \ + src/pipeline/pipeline_incremental.c \ src/pipeline/worker_pool.c \ src/pipeline/pass_parallel.c \ src/pipeline/pass_definitions.c \ diff --git a/src/graph_buffer/graph_buffer.c b/src/graph_buffer/graph_buffer.c index 0013096de..0a04aea30 100644 --- a/src/graph_buffer/graph_buffer.c +++ b/src/graph_buffer/graph_buffer.c @@ -1082,3 +1082,62 @@ int cbm_gbuf_flush_to_store(cbm_gbuf_t *gb, cbm_store_t *store) { free(temp_to_real); return 0; } + +int cbm_gbuf_merge_into_store(cbm_gbuf_t *gb, cbm_store_t *store) { + if (!gb || !store) { + return -1; + } + + /* Begin bulk mode — no project wipe */ + cbm_store_begin(store); + + /* Build temp_id → real_id map */ + int64_t max_temp_id = gb->next_id; + int64_t *temp_to_real = calloc(max_temp_id, sizeof(int64_t)); + + for (int i = 0; i < gb->nodes.count; i++) { + cbm_gbuf_node_t *n = gb->nodes.items[i]; + + if (!cbm_ht_get(gb->node_by_qn, n->qualified_name)) { + continue; + } + + cbm_node_t sn = { + .project = gb->project, + .label = n->label, + .name = n->name, + .qualified_name = n->qualified_name, + .file_path = n->file_path, + .start_line = n->start_line, + .end_line = n->end_line, + .properties_json = n->properties_json, + }; + int64_t real_id = cbm_store_upsert_node(store, &sn); + if (real_id > 0 && n->id < max_temp_id) { + temp_to_real[n->id] = real_id; + } + } + + for (int i = 0; i < gb->edges.count; i++) { + cbm_gbuf_edge_t *e = gb->edges.items[i]; + int64_t real_src = (e->source_id < max_temp_id) ? temp_to_real[e->source_id] : 0; + int64_t real_tgt = (e->target_id < max_temp_id) ? temp_to_real[e->target_id] : 0; + if (real_src == 0 || real_tgt == 0) { + continue; + } + + cbm_edge_t se = { + .project = gb->project, + .source_id = real_src, + .target_id = real_tgt, + .type = e->type, + .properties_json = e->properties_json, + }; + cbm_store_insert_edge(store, &se); + } + + cbm_store_commit(store); + + free(temp_to_real); + return 0; +} diff --git a/src/graph_buffer/graph_buffer.h b/src/graph_buffer/graph_buffer.h index 50f525752..eabaf1265 100644 --- a/src/graph_buffer/graph_buffer.h +++ b/src/graph_buffer/graph_buffer.h @@ -140,7 +140,12 @@ int cbm_gbuf_delete_edges_by_type(cbm_gbuf_t *gb, const char *type); int cbm_gbuf_dump_to_sqlite(cbm_gbuf_t *gb, const char *path); /* Flush the buffer to an existing store via the store API. - * Used for incremental indexing. Returns 0 on success. */ + * Deletes existing project data first. Returns 0 on success. */ int cbm_gbuf_flush_to_store(cbm_gbuf_t *gb, cbm_store_t *store); +/* Merge the buffer into an existing store WITHOUT deleting existing data. + * Upserts nodes, inserts edges. Used for incremental indexing. + * Returns 0 on success. */ +int cbm_gbuf_merge_into_store(cbm_gbuf_t *gb, cbm_store_t *store); + #endif /* CBM_GRAPH_BUFFER_H */ diff --git a/src/mcp/mcp.c b/src/mcp/mcp.c index 3924b8683..55b99b6f3 100644 --- a/src/mcp/mcp.c +++ b/src/mcp/mcp.c @@ -258,11 +258,13 @@ static const tool_def_t TOOLS[] = { "name\"]}"}, {"get_code_snippet", - "Get source code for a specific function, class, or symbol by qualified name. Use INSTEAD OF " - "reading entire files when you need one function's implementation.", - "{\"type\":\"object\",\"properties\":{\"qualified_name\":{\"type\":\"string\"},\"project\":{" - "\"type\":\"string\"},\"auto_resolve\":{\"type\":\"boolean\",\"default\":false},\"include_" - "neighbors\":{\"type\":\"boolean\",\"default\":false}},\"required\":[\"qualified_name\"]}"}, + "Read source code for a function/class/symbol. IMPORTANT: First call search_graph to find the " + "exact qualified_name, then pass it here. This is a read tool, not a search tool. Accepts " + "full qualified_name (exact match) or short function name (returns suggestions if ambiguous).", + "{\"type\":\"object\",\"properties\":{\"qualified_name\":{\"type\":\"string\",\"description\":" + "\"Full qualified_name from search_graph, or short function name\"},\"project\":{" + "\"type\":\"string\"},\"include_neighbors\":{" + "\"type\":\"boolean\",\"default\":false}},\"required\":[\"qualified_name\"]}"}, {"get_graph_schema", "Get the schema of the knowledge graph (node labels, edge types)", "{\"type\":\"object\",\"properties\":{\"project\":{\"type\":\"string\"}}}"}, @@ -740,6 +742,24 @@ static char *handle_get_graph_schema(cbm_mcp_server_t *srv, const char *args) { } yyjson_mut_obj_add_val(doc, root, "edge_types", types); + /* Check ADR presence */ + cbm_project_t proj_info = {0}; + if (cbm_store_get_project(store, project, &proj_info) == 0 && proj_info.root_path) { + char adr_path[4096]; + snprintf(adr_path, sizeof(adr_path), "%s/.codebase-memory/adr.md", proj_info.root_path); + struct stat adr_st; + // NOLINTNEXTLINE(readability-implicit-bool-conversion) + bool adr_exists = (stat(adr_path, &adr_st) == 0); + yyjson_mut_obj_add_bool(doc, root, "adr_present", adr_exists); + if (!adr_exists) { + yyjson_mut_obj_add_str( + doc, root, "adr_hint", + "No ADR found. Use manage_adr(mode='update') to persist architectural " + "decisions across sessions. Run get_architecture(aspects=['all']) first."); + } + cbm_project_free_fields(&proj_info); + } + char *json = yy_doc_to_str(doc); yyjson_mut_doc_free(doc); cbm_store_schema_free(&schema); @@ -1255,6 +1275,21 @@ static char *handle_index_repository(cbm_mcp_server_t *srv, const char *args) { int edges = cbm_store_count_edges(store, project_name); yyjson_mut_obj_add_int(doc, root, "nodes", nodes); yyjson_mut_obj_add_int(doc, root, "edges", edges); + + /* Check ADR presence and suggest creation if missing */ + char adr_path[4096]; + snprintf(adr_path, sizeof(adr_path), "%s/.codebase-memory/adr.md", repo_path); + struct stat adr_st; + // NOLINTNEXTLINE(readability-implicit-bool-conversion) + bool adr_exists = (stat(adr_path, &adr_st) == 0); + yyjson_mut_obj_add_bool(doc, root, "adr_present", adr_exists); + if (!adr_exists) { + yyjson_mut_obj_add_str( + doc, root, "adr_hint", + "Project indexed. Consider creating an Architecture Decision Record: " + "explore the codebase with get_architecture(aspects=['all']), then use " + "manage_adr(mode='store') to persist architectural insights across sessions."); + } } } @@ -1293,8 +1328,8 @@ static char *snippet_suggestions(const char *input, cbm_node_t *nodes, int count char msg[512]; snprintf(msg, sizeof(msg), - "%d matches found for \"%s\" — use a qualified_name " - "from the suggestions to disambiguate", + "%d matches for \"%s\". Pick a qualified_name from suggestions below, " + "or use search_graph(name_pattern=\"...\") to narrow results.", count, input); yyjson_mut_obj_add_str(doc, root, "message", msg); @@ -1472,36 +1507,40 @@ static char *build_snippet_response(cbm_mcp_server_t *srv, cbm_node_t *node, static char *handle_get_code_snippet(cbm_mcp_server_t *srv, const char *args) { char *qn = cbm_mcp_get_string_arg(args, "qualified_name"); char *project = cbm_mcp_get_string_arg(args, "project"); - cbm_store_t *store = resolve_store(srv, project); - bool auto_resolve = cbm_mcp_get_bool_arg(args, "auto_resolve"); bool include_neighbors = cbm_mcp_get_bool_arg(args, "include_neighbors"); if (!qn) { free(project); return cbm_mcp_text_result("qualified_name is required", true); } + + cbm_store_t *store = resolve_store(srv, project); if (!store) { free(qn); free(project); - return cbm_mcp_text_result("{\"error\":\"no project loaded\"}", true); + return cbm_mcp_text_result("no project loaded — run index_repository first", true); } + /* Default to current project (same as all other tools) */ + const char *effective_project = project ? project : srv->current_project; + /* Tier 1: Exact QN match */ cbm_node_t node = {0}; - int rc = cbm_store_find_node_by_qn(store, project, qn, &node); + int rc = cbm_store_find_node_by_qn(store, effective_project, qn, &node); if (rc == CBM_STORE_OK) { - char *result = - build_snippet_response(srv, &node, NULL /*exact*/, include_neighbors, NULL, 0); + char *result = build_snippet_response(srv, &node, NULL, include_neighbors, NULL, 0); free_node_contents(&node); free(qn); free(project); return result; } - /* Tier 2: QN suffix match */ + /* Tier 2: Suffix match — handles partial QNs ("main.HandleRequest") + * and short names ("ProcessOrder") via LIKE '%.X'. */ cbm_node_t *suffix_nodes = NULL; int suffix_count = 0; - cbm_store_find_nodes_by_qn_suffix(store, project, qn, &suffix_nodes, &suffix_count); + cbm_store_find_nodes_by_qn_suffix(store, effective_project, qn, &suffix_nodes, &suffix_count); + if (suffix_count == 1) { copy_node(&suffix_nodes[0], &node); cbm_store_free_nodes(suffix_nodes, suffix_count); @@ -1512,154 +1551,23 @@ static char *handle_get_code_snippet(cbm_mcp_server_t *srv, const char *args) { return result; } - /* Tier 3: Short name match */ - cbm_node_t *name_nodes = NULL; - int name_count = 0; - cbm_store_find_nodes_by_name(store, project, qn, &name_nodes, &name_count); - if (name_count == 1) { - copy_node(&name_nodes[0], &node); - cbm_store_free_nodes(name_nodes, name_count); + if (suffix_count > 1) { + char *result = snippet_suggestions(qn, suffix_nodes, suffix_count); cbm_store_free_nodes(suffix_nodes, suffix_count); - char *result = build_snippet_response(srv, &node, "name", include_neighbors, NULL, 0); - free_node_contents(&node); - free(qn); - free(project); - return result; - } - - /* Ambiguous: collect candidates from suffix + name tiers (dedup by id) */ - int total_cand = suffix_count + name_count; - if (total_cand > 0) { - /* Dedup by node ID */ - cbm_node_t *candidates = calloc((size_t)total_cand, sizeof(cbm_node_t)); - int cand_count = 0; - - for (int i = 0; i < suffix_count; i++) { - copy_node(&suffix_nodes[i], &candidates[cand_count++]); - } - for (int i = 0; i < name_count; i++) { - bool dup = false; - for (int j = 0; j < cand_count; j++) { - if (candidates[j].id == name_nodes[i].id) { - dup = true; - break; - } - } - if (!dup) { - copy_node(&name_nodes[i], &candidates[cand_count++]); - } - } - - cbm_store_free_nodes(suffix_nodes, suffix_count); - cbm_store_free_nodes(name_nodes, name_count); - - /* Auto-resolve: pick best candidate by degree */ - if (auto_resolve && cand_count >= 2 && cand_count <= 2) { - /* Find best: highest total degree, prefer non-test files */ - int best_idx = 0; - int best_deg = -1; - bool best_is_test = false; - for (int i = 0; i < cand_count; i++) { - int in_d = 0; - int out_d = 0; - cbm_store_node_degree(store, candidates[i].id, &in_d, &out_d); - int deg = in_d + out_d; - bool is_test = - // NOLINTNEXTLINE(readability-implicit-bool-conversion) - candidates[i].file_path && strstr(candidates[i].file_path, "_test") != NULL; - if (i == 0 || (best_is_test && !is_test) || - (!best_is_test == !is_test && deg > best_deg) || - (!best_is_test == !is_test && deg == best_deg && candidates[i].qualified_name && - best_idx >= 0 && candidates[best_idx].qualified_name && - strcmp(candidates[i].qualified_name, candidates[best_idx].qualified_name) < - 0)) { - best_idx = i; - best_deg = deg; - best_is_test = is_test; - } - } - - copy_node(&candidates[best_idx], &node); - - /* Build alternatives list (skip the picked one) */ - cbm_node_t *alts = calloc((size_t)(cand_count - 1), sizeof(cbm_node_t)); - int alt_count = 0; - for (int i = 0; i < cand_count; i++) { - if (i != best_idx) { - copy_node(&candidates[i], &alts[alt_count++]); - } - } - - for (int i = 0; i < cand_count; i++) { - free_node_contents(&candidates[i]); - } - free(candidates); - - char *result = - build_snippet_response(srv, &node, "auto_best", include_neighbors, alts, alt_count); - free_node_contents(&node); - for (int i = 0; i < alt_count; i++) { - free_node_contents(&alts[i]); - } - free(alts); - free(qn); - free(project); - return result; - } - - /* Return suggestions */ - char *result = snippet_suggestions(qn, candidates, cand_count); - for (int i = 0; i < cand_count; i++) { - free_node_contents(&candidates[i]); - } - free(candidates); free(qn); free(project); return result; } cbm_store_free_nodes(suffix_nodes, suffix_count); - cbm_store_free_nodes(name_nodes, name_count); - - /* Tier 4: Fuzzy — try last segment for name-based search */ - const char *dot = strrchr(qn, '.'); - const char *search_name = dot ? dot + 1 : qn; - - /* Use search with name pattern for fuzzy matching */ - cbm_search_params_t params = {0}; - params.project = project; - params.name_pattern = search_name; - params.limit = 5; - params.min_degree = -1; - params.max_degree = -1; - const char *excl[] = {"Community", NULL}; - params.exclude_labels = excl; - - cbm_search_output_t search_out = {0}; - if (cbm_store_search(store, ¶ms, &search_out) == CBM_STORE_OK && search_out.count > 0) { - /* Build suggestions from search results */ - cbm_node_t *fuzzy = calloc((size_t)search_out.count, sizeof(cbm_node_t)); - for (int i = 0; i < search_out.count; i++) { - copy_node(&search_out.results[i].node, &fuzzy[i]); - } - int fuzzy_count = search_out.count; - cbm_store_search_free(&search_out); - - char *result = snippet_suggestions(qn, fuzzy, fuzzy_count); - for (int i = 0; i < fuzzy_count; i++) { - free_node_contents(&fuzzy[i]); - } - free(fuzzy); - free(qn); - free(project); - return result; - } - cbm_store_search_free(&search_out); - - /* Nothing found */ free(qn); free(project); - return cbm_mcp_text_result("symbol not found", true); + + /* Nothing found — guide the caller toward search_graph */ + return cbm_mcp_text_result( + "symbol not found. Use search_graph(name_pattern=\"...\") first to discover " + "the exact qualified_name, then pass it to get_code_snippet.", + true); } /* ── search_code ──────────────────────────────────────────────── */ @@ -1960,6 +1868,14 @@ static char *handle_manage_adr(cbm_mcp_server_t *srv, const char *args) { } else { yyjson_mut_obj_add_str(doc, root_obj, "content", ""); yyjson_mut_obj_add_str(doc, root_obj, "status", "no_adr"); + yyjson_mut_obj_add_str( + doc, root_obj, "adr_hint", + "No ADR yet. Create one with manage_adr(mode='update', " + "content='## PURPOSE\\n...\\n\\n## STACK\\n...\\n\\n## ARCHITECTURE\\n..." + "\\n\\n## PATTERNS\\n...\\n\\n## TRADEOFFS\\n...\\n\\n## PHILOSOPHY\\n...'). " + "For guided creation: explore the codebase with get_architecture, " + "then draft and store. Sections: PURPOSE, STACK, ARCHITECTURE, " + "PATTERNS, TRADEOFFS, PHILOSOPHY."); } } diff --git a/src/pipeline/pipeline.c b/src/pipeline/pipeline.c index bcb48c6fc..f5b75108d 100644 --- a/src/pipeline/pipeline.c +++ b/src/pipeline/pipeline.c @@ -15,6 +15,7 @@ // NOLINTNEXTLINE(misc-include-cleaner) — worker_pool.h included for interface contract #include "pipeline/worker_pool.h" #include "graph_buffer/graph_buffer.h" +#include "store/store.h" #include "discover/discover.h" #include "foundation/platform.h" #include "foundation/compat_fs.h" @@ -28,6 +29,7 @@ #include #include #include +#include #include /* ── Internal state ──────────────────────────────────────────────── */ @@ -108,6 +110,33 @@ const char *cbm_pipeline_project_name(const cbm_pipeline_t *p) { return p ? p->project_name : NULL; } +const char *cbm_pipeline_repo_path(const cbm_pipeline_t *p) { + return p ? p->repo_path : NULL; +} + +atomic_int *cbm_pipeline_cancelled_ptr(cbm_pipeline_t *p) { + return p ? &p->cancelled : NULL; +} + +/* Resolve the DB path for this pipeline. Caller must free(). */ +static char *resolve_db_path(const cbm_pipeline_t *p) { + char *path = malloc(1024); + if (!path) { + return NULL; + } + if (p->db_path) { + snprintf(path, 1024, "%s", p->db_path); + } else { + // NOLINTNEXTLINE(concurrency-mt-unsafe) + const char *home = getenv("HOME"); + if (!home) { + home = "/tmp"; + } + snprintf(path, 1024, "%s/.cache/codebase-memory-mcp/%s.db", home, p->project_name); + } + return path; +} + static int check_cancel(const cbm_pipeline_t *p) { return atomic_load(&p->cancelled) ? -1 : 0; } @@ -295,6 +324,36 @@ int cbm_pipeline_run(cbm_pipeline_t *p) { return -1; } + /* Check for existing DB with file hashes → incremental path */ + { + char *db_path = resolve_db_path(p); + if (db_path) { + struct stat db_st; + if (stat(db_path, &db_st) == 0) { + /* DB exists — check if it has file hashes */ + cbm_store_t *check_store = cbm_store_open_path(db_path); + if (check_store) { + cbm_file_hash_t *hashes = NULL; + int hash_count = 0; + cbm_store_get_file_hashes(check_store, p->project_name, &hashes, &hash_count); + cbm_store_free_file_hashes(hashes, hash_count); + cbm_store_close(check_store); + + if (hash_count > 0) { + cbm_log_info("pipeline.route", "path", "incremental", "stored_hashes", + itoa_buf(hash_count)); + rc = cbm_pipeline_run_incremental(p, db_path, files, file_count); + cbm_discover_free(files, file_count); + free(db_path); + return rc; + } + } + } + free(db_path); + } + } + cbm_log_info("pipeline.route", "path", "full"); + /* Phase 2: Create graph buffer and registry */ p->gbuf = cbm_gbuf_new(p->project_name, p->repo_path); p->registry = cbm_registry_new(); @@ -649,6 +708,31 @@ int cbm_pipeline_run(cbm_pipeline_t *p) { goto cleanup; } cbm_log_info("pass.timing", "pass", "dump", "elapsed_ms", itoa_buf((int)elapsed_ms(t))); + + /* Persist file hashes so next run can use incremental path */ + cbm_store_t *hash_store = cbm_store_open_path(db_path); + if (hash_store) { + cbm_store_delete_file_hashes(hash_store, p->project_name); + for (int i = 0; i < file_count; i++) { + struct stat fst; + if (stat(files[i].path, &fst) == 0) { + int64_t mtime_ns; +#ifdef __APPLE__ + mtime_ns = ((int64_t)fst.st_mtimespec.tv_sec * 1000000000LL) + + (int64_t)fst.st_mtimespec.tv_nsec; +#elif defined(_WIN32) + mtime_ns = (int64_t)fst.st_mtime * 1000000000LL; +#else + mtime_ns = + ((int64_t)fst.st_mtim.tv_sec * 1000000000LL) + (int64_t)fst.st_mtim.tv_nsec; +#endif + cbm_store_upsert_file_hash(hash_store, p->project_name, files[i].rel_path, "", + mtime_ns, fst.st_size); + } + } + cbm_store_close(hash_store); + cbm_log_info("pass.timing", "pass", "persist_hashes", "files", itoa_buf(file_count)); + } } cbm_log_info("pipeline.done", "nodes", itoa_buf(cbm_gbuf_node_count(p->gbuf)), "edges", diff --git a/src/pipeline/pipeline_incremental.c b/src/pipeline/pipeline_incremental.c new file mode 100644 index 000000000..78e2ffde6 --- /dev/null +++ b/src/pipeline/pipeline_incremental.c @@ -0,0 +1,286 @@ +/* + * pipeline_incremental.c — Disk-based incremental re-indexing. + * + * Operates on the existing SQLite DB directly (not RAM-first graph buffer). + * Compares file mtime+size against stored hashes to classify changed/unchanged. + * Deletes changed files' nodes (edges cascade via ON DELETE CASCADE), + * re-parses only changed files through passes into a temp graph buffer, + * then merges new nodes/edges into the disk DB. Persists updated hashes. + * + * Called from pipeline.c when a DB with stored hashes already exists. + */ +#include "pipeline/pipeline.h" +#include "pipeline/pipeline_internal.h" +#include "store/store.h" +#include "graph_buffer/graph_buffer.h" +#include "discover/discover.h" +#include "foundation/log.h" +#include "foundation/hash_table.h" +#include "foundation/compat.h" +#include "foundation/compat_fs.h" + +#include +#include +#include +#include + +/* ── Constants ───────────────────────────────────────────────────── */ + +#define CBM_MS_PER_SEC 1000.0 +#define CBM_NS_PER_MS 1000000.0 +#define CBM_NS_PER_SEC 1000000000LL + +/* ── Timing helper (same as pipeline.c) ──────────────────────────── */ + +static double elapsed_ms(struct timespec start) { + struct timespec now; + cbm_clock_gettime(CLOCK_MONOTONIC, &now); + double s = (double)(now.tv_sec - start.tv_sec); + double ns = (double)(now.tv_nsec - start.tv_nsec); + return (s * CBM_MS_PER_SEC) + (ns / CBM_NS_PER_MS); +} + +/* itoa into static buffer — matches pipeline.c helper */ +static const char *itoa_buf(int v) { + static _Thread_local char buf[4][24]; + static _Thread_local int idx = 0; + idx = (idx + 1) & 3; + snprintf(buf[idx], sizeof(buf[idx]), "%d", v); + return buf[idx]; +} + +/* ── Platform-portable mtime_ns ──────────────────────────────────── */ + +static int64_t stat_mtime_ns(const struct stat *st) { +#ifdef __APPLE__ + return ((int64_t)st->st_mtimespec.tv_sec * CBM_NS_PER_SEC) + (int64_t)st->st_mtimespec.tv_nsec; +#elif defined(_WIN32) + return (int64_t)st->st_mtime * CBM_NS_PER_SEC; +#else + return ((int64_t)st->st_mtim.tv_sec * CBM_NS_PER_SEC) + (int64_t)st->st_mtim.tv_nsec; +#endif +} + +/* ── File classification ─────────────────────────────────────────── */ + +/* Classify discovered files against stored hashes using mtime+size. + * Returns a boolean array: changed[i] = true if files[i] needs re-parsing. + * Caller must free the returned array. */ +static bool *classify_files(cbm_file_info_t *files, int file_count, cbm_file_hash_t *stored, + int stored_count, int *out_changed, int *out_unchanged) { + bool *changed = calloc((size_t)file_count, sizeof(bool)); + if (!changed) { + return NULL; + } + + int n_changed = 0; + int n_unchanged = 0; + + /* Build lookup: rel_path -> stored hash */ + CBMHashTable *ht = cbm_ht_create(stored_count > 0 ? (size_t)stored_count * 2 : 64); + for (int i = 0; i < stored_count; i++) { + cbm_ht_set(ht, stored[i].rel_path, &stored[i]); + } + + for (int i = 0; i < file_count; i++) { + cbm_file_hash_t *h = cbm_ht_get(ht, files[i].rel_path); + if (!h) { + /* New file */ + changed[i] = true; + n_changed++; + continue; + } + + struct stat st; + if (stat(files[i].path, &st) != 0) { + changed[i] = true; + n_changed++; + continue; + } + + if (stat_mtime_ns(&st) != h->mtime_ns || st.st_size != h->size) { + changed[i] = true; + n_changed++; + } else { + n_unchanged++; + } + } + + cbm_ht_free(ht); + *out_changed = n_changed; + *out_unchanged = n_unchanged; + return changed; +} + +/* Find stored files that no longer exist on disk. Returns count. */ +static int find_deleted_files(cbm_file_info_t *files, int file_count, cbm_file_hash_t *stored, + int stored_count, char ***out_deleted) { + CBMHashTable *current = cbm_ht_create((size_t)file_count * 2); + for (int i = 0; i < file_count; i++) { + cbm_ht_set(current, files[i].rel_path, &files[i]); + } + + int count = 0; + int cap = 64; + char **deleted = malloc((size_t)cap * sizeof(char *)); + + for (int i = 0; i < stored_count; i++) { + if (!cbm_ht_get(current, stored[i].rel_path)) { + if (count >= cap) { + cap *= 2; + char **tmp = realloc(deleted, (size_t)cap * sizeof(char *)); + if (!tmp) { + break; + } + deleted = tmp; + } + deleted[count++] = strdup(stored[i].rel_path); + } + } + + cbm_ht_free(current); + *out_deleted = deleted; + return count; +} + +/* ── Persist file hashes ─────────────────────────────────────────── */ + +static void persist_hashes(cbm_store_t *store, const char *project, cbm_file_info_t *files, + int file_count) { + for (int i = 0; i < file_count; i++) { + struct stat st; + if (stat(files[i].path, &st) != 0) { + continue; + } + cbm_store_upsert_file_hash(store, project, files[i].rel_path, "", stat_mtime_ns(&st), + st.st_size); + } +} + +/* ── Incremental pipeline entry point ────────────────────────────── */ + +int cbm_pipeline_run_incremental(cbm_pipeline_t *p, const char *db_path, cbm_file_info_t *files, + int file_count) { + struct timespec t0; + cbm_clock_gettime(CLOCK_MONOTONIC, &t0); + + const char *project = cbm_pipeline_project_name(p); + + /* Open existing disk DB */ + cbm_store_t *store = cbm_store_open_path(db_path); + if (!store) { + cbm_log_error("incremental.err", "msg", "open_db_failed", "path", db_path); + return -1; + } + + /* Load stored file hashes */ + cbm_file_hash_t *stored = NULL; + int stored_count = 0; + cbm_store_get_file_hashes(store, project, &stored, &stored_count); + + /* Classify files */ + int n_changed = 0; + int n_unchanged = 0; + bool *is_changed = + classify_files(files, file_count, stored, stored_count, &n_changed, &n_unchanged); + + /* Find deleted files */ + char **deleted = NULL; + int deleted_count = find_deleted_files(files, file_count, stored, stored_count, &deleted); + + cbm_log_info("incremental.classify", "changed", itoa_buf(n_changed), "unchanged", + itoa_buf(n_unchanged), "deleted", itoa_buf(deleted_count)); + + /* Fast path: nothing changed → skip */ + if (n_changed == 0 && deleted_count == 0) { + cbm_log_info("incremental.noop", "reason", "no_changes"); + free(is_changed); + free(deleted); + cbm_store_free_file_hashes(stored, stored_count); + cbm_store_close(store); + return 0; + } + + cbm_store_free_file_hashes(stored, stored_count); + + /* Delete changed files' nodes from disk DB (edges cascade) */ + cbm_store_begin(store); + for (int i = 0; i < file_count; i++) { + if (is_changed[i]) { + cbm_store_delete_nodes_by_file(store, project, files[i].rel_path); + } + } + + /* Delete removed files' nodes + hashes */ + for (int i = 0; i < deleted_count; i++) { + cbm_store_delete_nodes_by_file(store, project, deleted[i]); + cbm_store_delete_file_hash(store, project, deleted[i]); + cbm_log_info("incremental.removed", "file", deleted[i]); + free(deleted[i]); + } + free(deleted); + cbm_store_commit(store); + + /* Build list of changed files only */ + cbm_file_info_t *changed_files = malloc((size_t)n_changed * sizeof(cbm_file_info_t)); + int ci = 0; + for (int i = 0; i < file_count; i++) { + if (is_changed[i]) { + changed_files[ci++] = files[i]; + } + } + free(is_changed); + + cbm_log_info("incremental.reparse", "files", itoa_buf(ci)); + + /* Create temp graph buffer + registry for re-parsing changed files */ + cbm_gbuf_t *gbuf = cbm_gbuf_new(project, cbm_pipeline_repo_path(p)); + cbm_registry_t *registry = cbm_registry_new(); + + cbm_pipeline_ctx_t ctx = { + .project_name = project, + .repo_path = cbm_pipeline_repo_path(p), + .gbuf = gbuf, + .registry = registry, + .cancelled = cbm_pipeline_cancelled_ptr(p), + }; + + /* Run passes on changed files only */ + struct timespec t; + cbm_clock_gettime(CLOCK_MONOTONIC, &t); + cbm_pipeline_pass_definitions(&ctx, changed_files, ci); + cbm_log_info("pass.timing", "pass", "incr_definitions", "elapsed_ms", + itoa_buf((int)elapsed_ms(t))); + + cbm_clock_gettime(CLOCK_MONOTONIC, &t); + cbm_pipeline_pass_calls(&ctx, changed_files, ci); + cbm_log_info("pass.timing", "pass", "incr_calls", "elapsed_ms", itoa_buf((int)elapsed_ms(t))); + + cbm_clock_gettime(CLOCK_MONOTONIC, &t); + cbm_pipeline_pass_usages(&ctx, changed_files, ci); + cbm_log_info("pass.timing", "pass", "incr_usages", "elapsed_ms", itoa_buf((int)elapsed_ms(t))); + + cbm_clock_gettime(CLOCK_MONOTONIC, &t); + cbm_pipeline_pass_semantic(&ctx, changed_files, ci); + cbm_log_info("pass.timing", "pass", "incr_semantic", "elapsed_ms", + itoa_buf((int)elapsed_ms(t))); + + /* Merge new nodes/edges from gbuf into disk DB */ + int new_nodes = cbm_gbuf_node_count(gbuf); + int new_edges = cbm_gbuf_edge_count(gbuf); + cbm_gbuf_merge_into_store(gbuf, store); + + cbm_log_info("incremental.merged", "nodes", itoa_buf(new_nodes), "edges", itoa_buf(new_edges)); + + /* Persist updated file hashes for ALL files */ + persist_hashes(store, project, files, file_count); + + /* Cleanup */ + cbm_gbuf_free(gbuf); + cbm_registry_free(registry); + free(changed_files); + cbm_store_close(store); + + cbm_log_info("incremental.done", "elapsed_ms", itoa_buf((int)elapsed_ms(t0))); + return 0; +} diff --git a/src/pipeline/pipeline_internal.h b/src/pipeline/pipeline_internal.h index 450ecb6f7..38f35a45e 100644 --- a/src/pipeline/pipeline_internal.h +++ b/src/pipeline/pipeline_internal.h @@ -401,4 +401,16 @@ typedef struct { * Returns number of bindings written to out (up to max_out). */ int cbm_scan_project_env_urls(const char *root_path, cbm_env_binding_t *out, int max_out); +/* ── Incremental pipeline (pipeline_incremental.c) ───────────────── */ + +/* Run incremental re-index on an existing disk DB. + * Classifies files by mtime+size, deletes changed nodes, re-parses changed + * files, merges into disk DB. Returns 0 on success. */ +int cbm_pipeline_run_incremental(cbm_pipeline_t *p, const char *db_path, cbm_file_info_t *files, + int file_count); + +/* Pipeline accessors for incremental use */ +const char *cbm_pipeline_repo_path(const cbm_pipeline_t *p); +atomic_int *cbm_pipeline_cancelled_ptr(cbm_pipeline_t *p); + #endif /* CBM_PIPELINE_INTERNAL_H */ diff --git a/src/store/store.c b/src/store/store.c index 28e91ed8e..aa50955e1 100644 --- a/src/store/store.c +++ b/src/store/store.c @@ -43,7 +43,9 @@ struct cbm_store { sqlite3_stmt *stmt_upsert_node; sqlite3_stmt *stmt_find_node_by_id; sqlite3_stmt *stmt_find_node_by_qn; + sqlite3_stmt *stmt_find_node_by_qn_any; /* QN lookup without project filter */ sqlite3_stmt *stmt_find_nodes_by_name; + sqlite3_stmt *stmt_find_nodes_by_name_any; /* name lookup without project filter */ sqlite3_stmt *stmt_find_nodes_by_label; sqlite3_stmt *stmt_find_nodes_by_file; sqlite3_stmt *stmt_count_nodes; @@ -372,7 +374,9 @@ void cbm_store_close(cbm_store_t *s) { finalize_stmt(&s->stmt_upsert_node); finalize_stmt(&s->stmt_find_node_by_id); finalize_stmt(&s->stmt_find_node_by_qn); + finalize_stmt(&s->stmt_find_node_by_qn_any); finalize_stmt(&s->stmt_find_nodes_by_name); + finalize_stmt(&s->stmt_find_nodes_by_name_any); finalize_stmt(&s->stmt_find_nodes_by_label); finalize_stmt(&s->stmt_find_nodes_by_file); finalize_stmt(&s->stmt_count_nodes); @@ -723,6 +727,64 @@ int cbm_store_find_node_by_qn(cbm_store_t *s, const char *project, const char *q return CBM_STORE_NOT_FOUND; } +int cbm_store_find_node_by_qn_any(cbm_store_t *s, const char *qn, cbm_node_t *out) { + if (!s || !s->db) { + return CBM_STORE_ERR; + } + sqlite3_stmt *stmt = + prepare_cached(s, &s->stmt_find_node_by_qn_any, + "SELECT id, project, label, name, qualified_name, file_path, " + "start_line, end_line, properties FROM nodes " + "WHERE qualified_name = ?1 LIMIT 1;"); + if (!stmt) { + return CBM_STORE_ERR; + } + + bind_text(stmt, 1, qn); + int rc = sqlite3_step(stmt); + if (rc == SQLITE_ROW) { + scan_node(stmt, out); + return CBM_STORE_OK; + } + return CBM_STORE_NOT_FOUND; +} + +int cbm_store_find_nodes_by_name_any(cbm_store_t *s, const char *name, cbm_node_t **out, + int *count) { + if (!s || !s->db) { + *out = NULL; + *count = 0; + return CBM_STORE_ERR; + } + sqlite3_stmt *stmt = + prepare_cached(s, &s->stmt_find_nodes_by_name_any, + "SELECT id, project, label, name, qualified_name, file_path, " + "start_line, end_line, properties FROM nodes " + "WHERE name = ?1;"); + if (!stmt) { + *out = NULL; + *count = 0; + return CBM_STORE_ERR; + } + + bind_text(stmt, 1, name); + + int cap = 16; + int n = 0; + cbm_node_t *arr = malloc(cap * sizeof(cbm_node_t)); + while (sqlite3_step(stmt) == SQLITE_ROW) { + if (n >= cap) { + cap *= 2; + arr = safe_realloc(arr, cap * sizeof(cbm_node_t)); + } + scan_node(stmt, &arr[n]); + n++; + } + *out = arr; + *count = n; + return CBM_STORE_OK; +} + int cbm_store_find_node_ids_by_qns(cbm_store_t *s, const char *project, const char **qns, int qn_count, int64_t *out_ids) { if (!s || !project || !qns || !out_ids || qn_count <= 0) { @@ -1280,20 +1342,29 @@ int cbm_store_find_nodes_by_qn_suffix(cbm_store_t *s, const char *project, const char like_pattern[512]; snprintf(like_pattern, sizeof(like_pattern), "%%.%s", suffix); - const char *sql = "SELECT id, project, label, name, qualified_name, file_path, " - "start_line, end_line, properties FROM nodes " - "WHERE project = ?1 AND (qualified_name LIKE ?2 OR qualified_name = ?3)"; + const char *sql_with_project = + "SELECT id, project, label, name, qualified_name, file_path, " + "start_line, end_line, properties FROM nodes " + "WHERE project = ?1 AND (qualified_name LIKE ?2 OR qualified_name = ?3)"; + const char *sql_any = "SELECT id, project, label, name, qualified_name, file_path, " + "start_line, end_line, properties FROM nodes " + "WHERE (qualified_name LIKE ?1 OR qualified_name = ?2)"; sqlite3_stmt *stmt = NULL; - int rc = sqlite3_prepare_v2(s->db, sql, -1, &stmt, NULL); + int rc = sqlite3_prepare_v2(s->db, project ? sql_with_project : sql_any, -1, &stmt, NULL); if (rc != SQLITE_OK) { store_set_error_sqlite(s, "qn_suffix prepare"); return CBM_STORE_ERR; } - bind_text(stmt, 1, project); - bind_text(stmt, 2, like_pattern); - bind_text(stmt, 3, suffix); + if (project) { + bind_text(stmt, 1, project); + bind_text(stmt, 2, like_pattern); + bind_text(stmt, 3, suffix); + } else { + bind_text(stmt, 1, like_pattern); + bind_text(stmt, 2, suffix); + } int cap = 8; int n = 0; diff --git a/src/store/store.h b/src/store/store.h index 9864ac5f3..066f35363 100644 --- a/src/store/store.h +++ b/src/store/store.h @@ -256,10 +256,17 @@ int cbm_store_find_node_by_id(cbm_store_t *s, int64_t id, cbm_node_t *out); /* Find node by project + qualified_name. */ int cbm_store_find_node_by_qn(cbm_store_t *s, const char *project, const char *qn, cbm_node_t *out); +/* Find node by qualified_name only (no project filter — QNs are globally unique). */ +int cbm_store_find_node_by_qn_any(cbm_store_t *s, const char *qn, cbm_node_t *out); + /* Find nodes by name (exact match). Returns allocated array, caller frees. */ int cbm_store_find_nodes_by_name(cbm_store_t *s, const char *project, const char *name, cbm_node_t **out, int *count); +/* Find nodes by name across all projects. Returns allocated array, caller frees. */ +int cbm_store_find_nodes_by_name_any(cbm_store_t *s, const char *name, cbm_node_t **out, + int *count); + /* Find nodes by label. */ int cbm_store_find_nodes_by_label(cbm_store_t *s, const char *project, const char *label, cbm_node_t **out, int *count); diff --git a/tests/test_pipeline.c b/tests/test_pipeline.c index ca894254e..8f6a6f6ad 100644 --- a/tests/test_pipeline.c +++ b/tests/test_pipeline.c @@ -4700,6 +4700,192 @@ TEST(registry_find_ending_with) { PASS(); } +/* ═══════════════════════════════════════════════════════════════════ + * Incremental reindex tests + * ═══════════════════════════════════════════════════════════════════ */ + +/* Helper: create a simple 2-file Go project for incremental tests */ +static char g_incr_tmpdir[256]; +static char g_incr_dbpath[512]; + +static int setup_incremental_repo(void) { + snprintf(g_incr_tmpdir, sizeof(g_incr_tmpdir), "/tmp/cbm_incr_XXXXXX"); + if (!cbm_mkdtemp(g_incr_tmpdir)) { + return -1; + } + snprintf(g_incr_dbpath, sizeof(g_incr_dbpath), "%s/test.db", g_incr_tmpdir); + + char path[512]; + FILE *f; + + /* main.go — calls Helper() */ + snprintf(path, sizeof(path), "%s/main.go", g_incr_tmpdir); + f = fopen(path, "w"); + if (!f) { return -1; } + fprintf(f, "package main\n\nfunc main() {\n\tHelper()\n}\n"); + fclose(f); + + /* helper.go — defines Helper() */ + snprintf(path, sizeof(path), "%s/helper.go", g_incr_tmpdir); + f = fopen(path, "w"); + if (!f) { return -1; } + fprintf(f, "package main\n\nfunc Helper() string {\n\treturn \"hello\"\n}\n"); + fclose(f); + + return 0; +} + +static void cleanup_incremental_repo(void) { + char cmd[512]; + snprintf(cmd, sizeof(cmd), "rm -rf '%s'", g_incr_tmpdir); + (void)system(cmd); +} + +TEST(incremental_full_then_noop) { + /* Full index, then re-run → should detect no changes and skip */ + if (setup_incremental_repo() != 0) { SKIP("setup failed"); } + + /* First: full index */ + cbm_pipeline_t *p = cbm_pipeline_new(g_incr_tmpdir, g_incr_dbpath, CBM_MODE_FULL); + ASSERT_NOT_NULL(p); + ASSERT_EQ(cbm_pipeline_run(p), 0); + char *project = strdup(cbm_pipeline_project_name(p)); + cbm_pipeline_free(p); + + /* Verify nodes exist */ + cbm_store_t *s = cbm_store_open_path(g_incr_dbpath); + ASSERT_NOT_NULL(s); + int nodes_before = cbm_store_count_nodes(s, project); + ASSERT_GT(nodes_before, 0); + cbm_store_close(s); + + /* Second: incremental — nothing changed → should be no-op */ + p = cbm_pipeline_new(g_incr_tmpdir, g_incr_dbpath, CBM_MODE_FULL); + ASSERT_NOT_NULL(p); + ASSERT_EQ(cbm_pipeline_run(p), 0); + cbm_pipeline_free(p); + + s = cbm_store_open_path(g_incr_dbpath); + ASSERT_NOT_NULL(s); + int nodes_after = cbm_store_count_nodes(s, project); + /* Node count should be same (no duplicates, no loss) */ + ASSERT_EQ(nodes_after, nodes_before); + cbm_store_close(s); + free(project); + + cleanup_incremental_repo(); + PASS(); +} + +TEST(incremental_detects_changed_file) { + /* Full index, modify one file, re-index → changed file re-parsed */ + if (setup_incremental_repo() != 0) { SKIP("setup failed"); } + + /* First: full index */ + cbm_pipeline_t *p = cbm_pipeline_new(g_incr_tmpdir, g_incr_dbpath, CBM_MODE_FULL); + ASSERT_NOT_NULL(p); + ASSERT_EQ(cbm_pipeline_run(p), 0); + char *project = strdup(cbm_pipeline_project_name(p)); + cbm_pipeline_free(p); + + /* Modify helper.go — add a new function */ + char path[512]; + snprintf(path, sizeof(path), "%s/helper.go", g_incr_tmpdir); + FILE *f = fopen(path, "w"); + ASSERT_NOT_NULL(f); + fprintf(f, "package main\n\n" + "func Helper() string {\n\treturn \"hello\"\n}\n\n" + "func NewFunc() int {\n\treturn 42\n}\n"); + fclose(f); + + /* Second: incremental — should detect change and re-index */ + p = cbm_pipeline_new(g_incr_tmpdir, g_incr_dbpath, CBM_MODE_FULL); + ASSERT_NOT_NULL(p); + ASSERT_EQ(cbm_pipeline_run(p), 0); + + /* Verify node count increased (NewFunc was added) */ + cbm_store_t *s = cbm_store_open_path(g_incr_dbpath); + ASSERT_NOT_NULL(s); + int nodes_after = cbm_store_count_nodes(s, project); + ASSERT_GT(nodes_after, 0); + cbm_store_close(s); + cbm_pipeline_free(p); + free(project); + + cleanup_incremental_repo(); + PASS(); +} + +TEST(incremental_detects_deleted_file) { + /* Full index, delete a file, re-index → deleted file's nodes removed */ + if (setup_incremental_repo() != 0) { SKIP("setup failed"); } + + /* First: full index */ + cbm_pipeline_t *p = cbm_pipeline_new(g_incr_tmpdir, g_incr_dbpath, CBM_MODE_FULL); + ASSERT_NOT_NULL(p); + ASSERT_EQ(cbm_pipeline_run(p), 0); + char *project = strdup(cbm_pipeline_project_name(p)); + cbm_pipeline_free(p); + + /* Delete helper.go */ + char path[512]; + snprintf(path, sizeof(path), "%s/helper.go", g_incr_tmpdir); + unlink(path); + + /* Second: incremental — should remove Helper nodes */ + p = cbm_pipeline_new(g_incr_tmpdir, g_incr_dbpath, CBM_MODE_FULL); + ASSERT_NOT_NULL(p); + ASSERT_EQ(cbm_pipeline_run(p), 0); + + /* Verify node count decreased (Helper's file was deleted) */ + cbm_store_t *s = cbm_store_open_path(g_incr_dbpath); + ASSERT_NOT_NULL(s); + int nodes_after = cbm_store_count_nodes(s, project); + ASSERT_GT(nodes_after, 0); /* still has main.go nodes */ + cbm_store_close(s); + cbm_pipeline_free(p); + free(project); + + cleanup_incremental_repo(); + PASS(); +} + +TEST(incremental_new_file_added) { + /* Full index, add a new file, re-index → new file's nodes appear */ + if (setup_incremental_repo() != 0) { SKIP("setup failed"); } + + /* First: full index */ + cbm_pipeline_t *p = cbm_pipeline_new(g_incr_tmpdir, g_incr_dbpath, CBM_MODE_FULL); + ASSERT_NOT_NULL(p); + ASSERT_EQ(cbm_pipeline_run(p), 0); + char *project = strdup(cbm_pipeline_project_name(p)); + cbm_pipeline_free(p); + + /* Add extra.go */ + char path[512]; + snprintf(path, sizeof(path), "%s/extra.go", g_incr_tmpdir); + FILE *f = fopen(path, "w"); + ASSERT_NOT_NULL(f); + fprintf(f, "package main\n\nfunc Extra() bool {\n\treturn true\n}\n"); + fclose(f); + + /* Second: incremental — should pick up Extra */ + p = cbm_pipeline_new(g_incr_tmpdir, g_incr_dbpath, CBM_MODE_FULL); + ASSERT_NOT_NULL(p); + ASSERT_EQ(cbm_pipeline_run(p), 0); + + cbm_store_t *s = cbm_store_open_path(g_incr_dbpath); + ASSERT_NOT_NULL(s); + int nodes_after = cbm_store_count_nodes(s, project); + ASSERT_GT(nodes_after, 0); + cbm_store_close(s); + cbm_pipeline_free(p); + free(project); + + cleanup_incremental_repo(); + PASS(); +} + SUITE(pipeline) { /* Lifecycle */ RUN_TEST(pipeline_create_free); @@ -4875,4 +5061,9 @@ SUITE(pipeline) { RUN_TEST(githistory_compute_change_coupling); RUN_TEST(githistory_coupling_skips_large_commits); RUN_TEST(githistory_coupling_limits_output); + /* Incremental reindex */ + RUN_TEST(incremental_full_then_noop); + RUN_TEST(incremental_detects_changed_file); + RUN_TEST(incremental_detects_deleted_file); + RUN_TEST(incremental_new_file_added); }