merge reference-api-indexing: expand_project_param, result tagging, dep re-index

Merge latest reference-api-indexing (bd09623) with gap implementations:

Conflicts resolved (2 in mcp.c search_graph handler):
1. Param parsing: keep reduce-token-usage compact/summary/mode params,
   use fill_project_params() from reference-api-indexing for smart project
2. Result loop: keep reduce-token-usage compact mode + pagination_hint,
   add unconditional source/package/read_only tagging from ref-api-indexing

Superset features in merged handler:
- expand_project_param (self/dep/deps/glob/prefix resolution)
- compact mode (omit redundant name when suffix of QN)
- summary mode (aggregate counts by label/file)
- pagination_hint with offset/limit guidance
- session_project in all responses
- source:"project"/"dependency" on every result
- package + read_only:true on dep results
- dep auto-reindex in all 3 paths (handler, watcher, autoindex)
- index_status reports dependencies[] + detected_ecosystem

All 2081 tests pass (2042 base + 22 token-reduction + 17 dep-indexing).
This commit is contained in:
Andrew Hundt
2026-03-21 00:04:48 -04:00
4 changed files with 410 additions and 20 deletions
+14
View File
@@ -17,6 +17,7 @@
#include "watcher/watcher.h"
#include "pipeline/pipeline.h"
#include "store/store.h"
#include "depindex/depindex.h"
#include "cli/cli.h"
#include "foundation/log.h"
#include "foundation/compat_thread.h"
@@ -85,6 +86,19 @@ static int watcher_index_fn(const char *project_name, const char *root_path, voi
int rc = cbm_pipeline_run(p);
cbm_pipeline_free(p);
/* Re-index dependencies after fresh dump. Uses cbm_project_name_from_path
* for consistent naming (matches pipeline's project_name derivation). */
if (rc == 0) {
char *pname = cbm_project_name_from_path(root_path);
cbm_store_t *store = cbm_store_open(pname);
if (store) {
cbm_dep_auto_index(pname, root_path, store, CBM_DEFAULT_AUTO_DEP_LIMIT);
cbm_store_close(store);
}
free(pname);
}
cbm_mem_collect();
return rc;
}
+213 -20
View File
@@ -597,6 +597,11 @@ void cbm_mcp_server_set_project(cbm_mcp_server_t *srv, const char *project) {
srv->current_project = project ? heap_strdup(project) : NULL;
}
void cbm_mcp_server_set_session_project(cbm_mcp_server_t *srv, const char *name) {
if (!srv || !name) return;
snprintf(srv->session_project, sizeof(srv->session_project), "%s", name);
}
void cbm_mcp_server_set_watcher(cbm_mcp_server_t *srv, struct cbm_watcher *w) {
if (srv) {
srv->watcher = w;
@@ -724,6 +729,99 @@ static cbm_store_t *resolve_store(cbm_mcp_server_t *srv, const char *project) {
} \
} while (0)
/* ── Smart project param expansion ─────────────────────────────── */
typedef enum { MATCH_NONE, MATCH_EXACT, MATCH_PREFIX, MATCH_GLOB } match_mode_t;
typedef struct {
char *value; /* expanded project string (heap) or NULL. Caller must free. */
match_mode_t mode; /* how to match in SQL */
} project_expand_t;
/* Expand project param shorthands (self/dep/glob/prefix).
* Takes ownership of raw — caller must NOT free raw after this call.
* Returns expanded result. Caller must free(result.value).
* Runtime: O(1) — fixed number of string comparisons + one snprintf + strdup.
* Memory: one heap allocation for result.value. */
static project_expand_t expand_project_param(cbm_mcp_server_t *srv, char *raw) {
project_expand_t r = {.value = NULL, .mode = MATCH_NONE};
if (!raw) return r;
/* Guard: if session_project is empty, skip all expansion rules */
if (!srv->session_project[0]) {
r.value = raw;
r.mode = strchr(raw, '*') ? MATCH_GLOB : MATCH_PREFIX;
return r;
}
size_t sp_len = strlen(srv->session_project);
char buf[4096];
/* Rule 1: "self" prefix → replace with session project name */
if (strncmp(raw, "self", 4) == 0 && (raw[4] == '\0' || raw[4] == '.')) {
bool is_self_only = (raw[4] == '\0');
snprintf(buf, sizeof(buf), "%s%s", srv->session_project, raw + 4);
free(raw);
r.value = heap_strdup(buf);
r.mode = is_self_only ? MATCH_EXACT : MATCH_PREFIX;
if (r.mode == MATCH_PREFIX && strchr(r.value, '*')) r.mode = MATCH_GLOB;
return r;
}
/* Rule 2: "dep" / "deps" exactly → "{session}.dep" */
if (strcmp(raw, "dep") == 0 || strcmp(raw, "deps") == 0) {
snprintf(buf, sizeof(buf), "%s.dep", srv->session_project);
free(raw);
r.value = heap_strdup(buf);
r.mode = MATCH_PREFIX;
return r;
}
/* Rule 3: starts with "dep." → prepend session */
if (strncmp(raw, "dep.", 4) == 0) {
snprintf(buf, sizeof(buf), "%s.%s", srv->session_project, raw);
free(raw);
r.value = heap_strdup(buf);
r.mode = strchr(r.value, '*') ? MATCH_GLOB : MATCH_PREFIX;
return r;
}
/* Rule 4: starts with "{session}." but next segment isn't "dep" → insert .dep. */
if (strncmp(raw, srv->session_project, sp_len) == 0 && raw[sp_len] == '.' &&
!(strncmp(raw + sp_len + 1, "dep", 3) == 0 &&
(raw[sp_len + 4] == '.' || raw[sp_len + 4] == '\0'))) {
snprintf(buf, sizeof(buf), "%s.dep.%s", srv->session_project, raw + sp_len + 1);
free(raw);
r.value = heap_strdup(buf);
r.mode = strchr(r.value, '*') ? MATCH_GLOB : MATCH_PREFIX;
return r;
}
/* Rule 5: everything else — as-is (bare words are project names) */
r.value = raw;
r.mode = strchr(raw, '*') ? MATCH_GLOB : MATCH_PREFIX;
return r;
}
/* Fill cbm_search_params_t project fields from an expand result.
* Also translates * → % for SQL LIKE in glob mode. */
static void fill_project_params(const project_expand_t *pe, cbm_search_params_t *params) {
switch (pe->mode) {
case MATCH_GLOB:
params->project_pattern = pe->value;
break;
case MATCH_EXACT:
params->project = pe->value;
params->project_exact = true;
break;
case MATCH_PREFIX:
params->project = pe->value;
break;
case MATCH_NONE:
break;
}
}
/* ── Tool handler implementations ─────────────────────────────── */
/* list_projects: scan cache directory for .db files.
@@ -853,9 +951,24 @@ static char *handle_get_graph_schema(cbm_mcp_server_t *srv, const char *args) {
}
static char *handle_search_graph(cbm_mcp_server_t *srv, const char *args) {
char *project = cbm_mcp_get_string_arg(args, "project");
cbm_store_t *store = resolve_store(srv, project);
REQUIRE_STORE(store, project);
char *raw_project = cbm_mcp_get_string_arg(args, "project");
project_expand_t pe = expand_project_param(srv, raw_project);
/* DB selection: if session_project is set and expanded value starts with it,
* use session store. Otherwise pass expanded value to resolve_store (opens .db). */
const char *db_project = pe.value; /* default: pass through to resolve_store */
if (pe.value && srv->session_project[0] &&
strncmp(pe.value, srv->session_project, strlen(srv->session_project)) == 0) {
db_project = srv->session_project; /* deps are in session db */
}
cbm_store_t *store = resolve_store(srv, db_project);
if (!store) {
free(pe.value);
return cbm_mcp_text_result(
"{\"error\":\"no project loaded\","
"\"hint\":\"Run index_repository with repo_path to index the project first.\"}", true);
}
char *label = cbm_mcp_get_string_arg(args, "label");
char *name_pattern = cbm_mcp_get_string_arg(args, "name_pattern");
char *file_pattern = cbm_mcp_get_string_arg(args, "file_pattern");
@@ -865,7 +978,6 @@ static char *handle_search_graph(cbm_mcp_server_t *srv, const char *args) {
int offset = cbm_mcp_get_int_arg(args, "offset", 0);
bool compact = cbm_mcp_get_bool_arg(args, "compact");
char *search_mode = cbm_mcp_get_string_arg(args, "mode");
bool include_deps = cbm_mcp_get_bool_arg(args, "include_dependencies");
int min_degree = cbm_mcp_get_int_arg(args, "min_degree", -1);
int max_degree = cbm_mcp_get_int_arg(args, "max_degree", -1);
@@ -873,16 +985,15 @@ static char *handle_search_graph(cbm_mcp_server_t *srv, const char *args) {
bool is_summary = search_mode && strcmp(search_mode, "summary") == 0;
int effective_limit = is_summary ? 10000 : limit;
cbm_search_params_t params = {
.project = project,
.label = label,
.name_pattern = name_pattern,
.file_pattern = file_pattern,
.limit = effective_limit,
.offset = offset,
.min_degree = min_degree,
.max_degree = max_degree,
};
cbm_search_params_t params = {0};
fill_project_params(&pe, &params);
params.label = label;
params.name_pattern = name_pattern;
params.file_pattern = file_pattern;
params.limit = effective_limit;
params.offset = offset;
params.min_degree = min_degree;
params.max_degree = max_degree;
cbm_search_output_t out = {0};
cbm_store_search(store, &params, &out);
@@ -893,6 +1004,10 @@ static char *handle_search_graph(cbm_mcp_server_t *srv, const char *args) {
yyjson_mut_obj_add_int(doc, root, "total", out.total);
/* Always include session_project so AI knows the project name */
if (srv->session_project[0])
yyjson_mut_obj_add_str(doc, root, "session_project", srv->session_project);
if (is_summary) {
/* Summary mode: aggregate counts by label and file (top 20) */
yyjson_mut_val *by_label = yyjson_mut_obj(doc);
@@ -960,10 +1075,19 @@ static char *handle_search_graph(cbm_mcp_server_t *srv, const char *args) {
sr->node.file_path ? sr->node.file_path : "");
yyjson_mut_obj_add_int(doc, item, "in_degree", sr->in_degree);
yyjson_mut_obj_add_int(doc, item, "out_degree", sr->out_degree);
/* AI grounding: mark source provenance when dependencies are included */
if (include_deps) {
yyjson_mut_obj_add_str(doc, item, "source", "project");
/* Unconditional source tagging — critical for AI grounding.
* Every result tagged source:"project" or source:"dependency".
* Dep results also get package name and read_only:true. */
bool is_dep = cbm_is_dep_project(sr->node.project, srv->session_project);
yyjson_mut_obj_add_str(doc, item, "source", is_dep ? "dependency" : "project");
if (is_dep && sr->node.project) {
size_t sp_len2 = strlen(srv->session_project);
const char *pkg = sr->node.project + sp_len2 + CBM_DEP_SEPARATOR_LEN;
yyjson_mut_obj_add_strcpy(doc, item, "package", pkg);
yyjson_mut_obj_add_bool(doc, item, "read_only", true);
}
yyjson_mut_arr_add_val(results, item);
}
yyjson_mut_obj_add_val(doc, root, "results", results);
@@ -983,7 +1107,7 @@ static char *handle_search_graph(cbm_mcp_server_t *srv, const char *args) {
yyjson_mut_doc_free(doc);
cbm_store_search_free(&out);
free(project);
free(pe.value);
free(label);
free(name_pattern);
free(file_pattern);
@@ -1085,6 +1209,9 @@ static char *handle_index_status(cbm_mcp_server_t *srv, const char *args) {
yyjson_mut_val *root = yyjson_mut_obj(doc);
yyjson_mut_doc_set_root(doc, root);
if (srv->session_project[0])
yyjson_mut_obj_add_str(doc, root, "session_project", srv->session_project);
if (project) {
int nodes = cbm_store_count_nodes(store, project);
int edges = cbm_store_count_edges(store, project);
@@ -1092,6 +1219,51 @@ static char *handle_index_status(cbm_mcp_server_t *srv, const char *args) {
yyjson_mut_obj_add_int(doc, root, "nodes", nodes);
yyjson_mut_obj_add_int(doc, root, "edges", edges);
yyjson_mut_obj_add_str(doc, root, "status", nodes > 0 ? "ready" : "empty");
/* Report indexed dependencies by searching for {project}.dep.% nodes.
* Uses project_pattern for LIKE query to find all dep projects. */
char dep_like[4096];
snprintf(dep_like, sizeof(dep_like), "%s.dep.%%", project);
cbm_search_params_t dep_params = {0};
dep_params.project_pattern = dep_like;
dep_params.limit = 100;
cbm_search_output_t dep_out = {0};
if (cbm_store_search(store, &dep_params, &dep_out) == 0 && dep_out.count > 0) {
/* Collect unique dep project names */
yyjson_mut_val *dep_arr = yyjson_mut_arr(doc);
const char *last_dep_proj = "";
int dep_count = 0;
for (int i = 0; i < dep_out.count; i++) {
const char *proj = dep_out.results[i].node.project;
if (!proj || strcmp(proj, last_dep_proj) == 0) continue;
last_dep_proj = proj;
/* Extract package name from "myproj.dep.pandas" */
const char *dep_sep = strstr(proj, CBM_DEP_SEPARATOR);
if (!dep_sep) continue;
const char *pkg = dep_sep + CBM_DEP_SEPARATOR_LEN;
yyjson_mut_val *d = yyjson_mut_obj(doc);
yyjson_mut_obj_add_strcpy(doc, d, "package", pkg);
int dn = cbm_store_count_nodes(store, proj);
yyjson_mut_obj_add_int(doc, d, "nodes", dn);
yyjson_mut_arr_add_val(dep_arr, d);
dep_count++;
}
if (dep_count > 0) {
yyjson_mut_obj_add_val(doc, root, "dependencies", dep_arr);
yyjson_mut_obj_add_int(doc, root, "dependency_count", dep_count);
}
cbm_store_search_free(&dep_out);
}
/* Report detected ecosystem */
cbm_project_t proj_info;
if (cbm_store_get_project(store, project, &proj_info) == 0 && proj_info.root_path) {
cbm_pkg_manager_t eco = cbm_detect_ecosystem(proj_info.root_path);
if (eco != CBM_PKG_COUNT) {
yyjson_mut_obj_add_str(doc, root, "detected_ecosystem",
cbm_pkg_manager_str(eco));
}
}
} else {
yyjson_mut_obj_add_str(doc, root, "status", "no_project");
}
@@ -1503,13 +1675,28 @@ static char *handle_index_repository(cbm_mcp_server_t *srv, const char *args) {
if (rc == 0) {
cbm_store_t *store = resolve_store(srv, project_name);
if (store) {
/* Auto-detect ecosystem and index installed deps from fresh graph.
* Queries manifest files already indexed by pipeline step 1. */
int deps_reindexed = cbm_dep_auto_index(
project_name, repo_path, store, CBM_DEFAULT_AUTO_DEP_LIMIT);
int nodes = cbm_store_count_nodes(store, project_name);
int edges = cbm_store_count_edges(store, project_name);
yyjson_mut_obj_add_int(doc, root, "nodes", nodes);
yyjson_mut_obj_add_int(doc, root, "edges", edges);
if (deps_reindexed > 0)
yyjson_mut_obj_add_int(doc, root, "dependencies_indexed", deps_reindexed);
cbm_pkg_manager_t eco = cbm_detect_ecosystem(repo_path);
if (eco != CBM_PKG_COUNT)
yyjson_mut_obj_add_str(doc, root, "detected_ecosystem",
cbm_pkg_manager_str(eco));
}
}
if (srv->session_project[0])
yyjson_mut_obj_add_str(doc, root, "session_project", srv->session_project);
char *json = yy_doc_to_str(doc);
yyjson_mut_doc_free(doc);
free(project_name);
@@ -2591,17 +2778,23 @@ static void *autoindex_thread(void *arg) {
int rc = cbm_pipeline_run(p);
cbm_pipeline_free(p);
cbm_mem_collect(); /* return mimalloc pages to OS after indexing */
if (rc == 0) {
/* Re-index dependencies after fresh dump */
cbm_store_t *store = resolve_store(srv, srv->session_project);
if (store) {
cbm_dep_auto_index(srv->session_project, srv->session_root,
store, CBM_DEFAULT_AUTO_DEP_LIMIT);
}
cbm_log_info("autoindex.done", "project", srv->session_project);
/* Register with watcher for ongoing change detection */
if (srv->watcher) {
cbm_watcher_watch(srv->watcher, srv->session_project, srv->session_root);
}
} else {
cbm_log_warn("autoindex.err", "msg", "pipeline_run_failed");
}
cbm_mem_collect();
return NULL;
}
+3
View File
@@ -124,6 +124,9 @@ cbm_store_t *cbm_mcp_server_store(cbm_mcp_server_t *srv);
* This prevents resolve_store() from trying to open a .db file when tools specify a project. */
void cbm_mcp_server_set_project(cbm_mcp_server_t *srv, const char *project);
/* Set the session project name (for testing and manual override). */
void cbm_mcp_server_set_session_project(cbm_mcp_server_t *srv, const char *name);
/* ── URI helpers ───────────────────────────────────────────────── */
/* Parse a file:// URI and extract the filesystem path.
+180
View File
@@ -456,6 +456,78 @@ TEST(dep_discover_max_files_guard) {
PASS();
}
/* ══════════════════════════════════════════════════════════════════
* FIXTURE: Project + dep nodes in same db for integration tests
* ══════════════════════════════════════════════════════════════════ */
/* Create an MCP server with project AND dep nodes indexed. */
static cbm_mcp_server_t *setup_proj_with_deps(char *tmp_dir, size_t tmp_sz) {
snprintf(tmp_dir, tmp_sz, "/tmp/cbm_depfull_XXXXXX");
if (!cbm_mkdtemp(tmp_dir))
return NULL;
char proj_dir[512];
snprintf(proj_dir, sizeof(proj_dir), "%s/project", tmp_dir);
cbm_mkdir(proj_dir);
/* Write a source file */
char src_path[512];
snprintf(src_path, sizeof(src_path), "%s/app.py", proj_dir);
FILE *fp = fopen(src_path, "w");
if (!fp) return NULL;
fprintf(fp, "import pandas as pd\ndef process_data():\n return pd.DataFrame()\n");
fclose(fp);
cbm_mcp_server_t *srv = cbm_mcp_server_new(NULL);
if (!srv) return NULL;
cbm_store_t *st = cbm_mcp_server_store(srv);
if (!st) { cbm_mcp_server_free(srv); return NULL; }
const char *proj_name = "testproj";
cbm_mcp_server_set_project(srv, proj_name);
cbm_mcp_server_set_session_project(srv, proj_name);
cbm_store_upsert_project(st, proj_name, proj_dir);
/* Project node */
cbm_node_t n1 = {0};
n1.project = proj_name;
n1.label = "Function";
n1.name = "process_data";
n1.qualified_name = "testproj.app.process_data";
n1.file_path = "app.py";
n1.start_line = 2;
n1.end_line = 3;
n1.properties_json = "{\"is_exported\":true}";
cbm_store_upsert_node(st, &n1);
/* Dep nodes */
cbm_store_upsert_project(st, "testproj.dep.pandas", "/tmp/pandas");
cbm_node_t n_df = {0};
n_df.project = "testproj.dep.pandas";
n_df.label = "Class";
n_df.name = "DataFrame";
n_df.qualified_name = "testproj.dep.pandas.DataFrame";
n_df.file_path = "pandas/core/frame.py";
n_df.start_line = 100;
n_df.end_line = 500;
n_df.properties_json = "{\"is_exported\":true}";
cbm_store_upsert_node(st, &n_df);
cbm_node_t n_read = {0};
n_read.project = "testproj.dep.pandas";
n_read.label = "Function";
n_read.name = "read_csv";
n_read.qualified_name = "testproj.dep.pandas.read_csv";
n_read.file_path = "pandas/io/parsers.py";
n_read.start_line = 50;
n_read.end_line = 80;
n_read.properties_json = "{\"is_exported\":true}";
cbm_store_upsert_node(st, &n_read);
return srv;
}
/* ══════════════════════════════════════════════════════════════════
* DEPINDEX HELPER UNIT TESTS
* ══════════════════════════════════════════════════════════════════ */
@@ -641,6 +713,106 @@ TEST(test_dep_reindex_replaces) {
PASS();
}
/* ══════════════════════════════════════════════════════════════════
* RESULT TAGGING: source field on all search results
* ══════════════════════════════════════════════════════════════════ */
TEST(test_search_results_have_source_field) {
/* ALL search results must have source:"project" or source:"dependency" */
char tmp[256];
cbm_mcp_server_t *srv = setup_proj_with_deps(tmp, sizeof(tmp));
ASSERT_NOT_NULL(srv);
/* Search with no project filter — should return both project + dep nodes */
char *raw = cbm_mcp_handle_tool(srv, "search_graph",
"{\"project\":\"testproj\","
"\"label\":\"Function\"}");
char *resp = extract_text_content_di(raw);
free(raw);
ASSERT_NOT_NULL(resp);
/* Project results must have source:"project" */
ASSERT_NOT_NULL(strstr(resp, "\"source\":\"project\""));
free(resp);
cbm_mcp_server_free(srv);
cleanup_fixture_dir(tmp);
PASS();
}
TEST(test_search_dep_results_tagged_dependency) {
/* Dep results must have source:"dependency", package, read_only */
char tmp[256];
cbm_mcp_server_t *srv = setup_proj_with_deps(tmp, sizeof(tmp));
ASSERT_NOT_NULL(srv);
/* Search dep nodes via project_pattern */
char *raw = cbm_mcp_handle_tool(srv, "search_graph",
"{\"project\":\"testproj\","
"\"label\":\"Class\"}");
char *resp = extract_text_content_di(raw);
free(raw);
ASSERT_NOT_NULL(resp);
/* DataFrame is a dep node — should have source:dependency */
if (strstr(resp, "DataFrame")) {
ASSERT_NOT_NULL(strstr(resp, "\"source\":\"dependency\""));
ASSERT_NOT_NULL(strstr(resp, "\"read_only\":true"));
ASSERT_NOT_NULL(strstr(resp, "\"package\":\"pandas\""));
}
free(resp);
cbm_mcp_server_free(srv);
cleanup_fixture_dir(tmp);
PASS();
}
TEST(test_search_response_has_session_project) {
/* Every response must include session_project */
char tmp[256];
cbm_mcp_server_t *srv = setup_proj_with_deps(tmp, sizeof(tmp));
ASSERT_NOT_NULL(srv);
char *raw = cbm_mcp_handle_tool(srv, "search_graph",
"{\"project\":\"testproj\","
"\"label\":\"Function\"}");
char *resp = extract_text_content_di(raw);
free(raw);
ASSERT_NOT_NULL(resp);
ASSERT_NOT_NULL(strstr(resp, "\"session_project\":\"testproj\""));
free(resp);
cbm_mcp_server_free(srv);
cleanup_fixture_dir(tmp);
PASS();
}
/* ══════════════════════════════════════════════════════════════════
* INDEX STATUS: dep info in response
* ══════════════════════════════════════════════════════════════════ */
TEST(test_index_status_shows_deps) {
char tmp[256];
cbm_mcp_server_t *srv = setup_proj_with_deps(tmp, sizeof(tmp));
ASSERT_NOT_NULL(srv);
char *raw = cbm_mcp_handle_tool(srv, "index_status",
"{\"project\":\"testproj\"}");
char *resp = extract_text_content_di(raw);
free(raw);
ASSERT_NOT_NULL(resp);
/* Should include dependency info */
ASSERT_TRUE(strstr(resp, "\"dependencies\"") != NULL ||
strstr(resp, "\"dependency_count\"") != NULL);
free(resp);
cbm_mcp_server_free(srv);
cleanup_fixture_dir(tmp);
PASS();
}
/* ══════════════════════════════════════════════════════════════════
* SUITE
* ══════════════════════════════════════════════════════════════════ */
@@ -684,4 +856,12 @@ SUITE(depindex) {
RUN_TEST(test_resolve_npm_node_modules);
RUN_TEST(test_pipeline_set_project_name);
RUN_TEST(test_dep_reindex_replaces);
/* Result tagging */
RUN_TEST(test_search_results_have_source_field);
RUN_TEST(test_search_dep_results_tagged_dependency);
RUN_TEST(test_search_response_has_session_project);
/* Index status deps */
RUN_TEST(test_index_status_shows_deps);
}