From 1cdb98319ae15ae2d2cb035515d529facfdd54fb Mon Sep 17 00:00:00 2001 From: Shane McCarron Date: Thu, 19 Mar 2026 17:47:32 -0500 Subject: [PATCH 01/25] =?UTF-8?q?feat(discover):=20add=20user-defined=20ex?= =?UTF-8?q?tension=E2=86=92language=20mappings=20via=20config=20files?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Read extra_extensions from ~/.config/codebase-memory-mcp/config.json (global) and {repo}/.codebase-memory.json (project), merging them with project config taking precedence. Unknown language values warn and skip. Missing files are silently ignored (fail-open). Co-Authored-By: Claude Sonnet 4.6 --- Makefile.cbm | 2 + src/discover/language.c | 12 ++ src/discover/userconfig.c | 388 ++++++++++++++++++++++++++++++++++++++ src/discover/userconfig.h | 72 +++++++ src/pipeline/pipeline.c | 12 ++ tests/test_main.c | 2 + tests/test_userconfig.c | 218 +++++++++++++++++++++ 7 files changed, 706 insertions(+) create mode 100644 src/discover/userconfig.c create mode 100644 src/discover/userconfig.h create mode 100644 tests/test_userconfig.c diff --git a/Makefile.cbm b/Makefile.cbm index 82821b84b..a0515de37 100644 --- a/Makefile.cbm +++ b/Makefile.cbm @@ -148,6 +148,7 @@ MCP_SRCS = src/mcp/mcp.c # Discover module (new) DISCOVER_SRCS = \ src/discover/language.c \ + src/discover/userconfig.c \ src/discover/gitignore.c \ src/discover/discover.c @@ -259,6 +260,7 @@ TEST_MCP_SRCS = \ TEST_DISCOVER_SRCS = \ tests/test_language.c \ + tests/test_userconfig.c \ tests/test_gitignore.c \ tests/test_discover.c diff --git a/src/discover/language.c b/src/discover/language.c index b7eb7e439..51b74c24c 100644 --- a/src/discover/language.c +++ b/src/discover/language.c @@ -3,8 +3,11 @@ * * Maps file extensions and special filenames to CBMLanguage enum values. * Handles .m disambiguation (Objective-C vs Magma vs MATLAB). + * Consults the process-global user config (set via cbm_set_user_lang_config) + * before the built-in lookup table. */ #include "discover/discover.h" +#include "discover/userconfig.h" #include "cbm.h" // CBMLanguage, CBM_LANG_* #include @@ -354,6 +357,15 @@ CBMLanguage cbm_language_for_extension(const char *ext) { return CBM_LANG_COUNT; } + /* Check user-defined overrides first */ + const cbm_userconfig_t *ucfg = cbm_get_user_lang_config(); + if (ucfg) { + CBMLanguage ulang = cbm_userconfig_lookup(ucfg, ext); + if (ulang != CBM_LANG_COUNT) { + return ulang; + } + } + for (size_t i = 0; i < EXT_TABLE_SIZE; i++) { if (strcmp(EXT_TABLE[i].ext, ext) == 0) { return EXT_TABLE[i].language; diff --git a/src/discover/userconfig.c b/src/discover/userconfig.c new file mode 100644 index 000000000..66edef254 --- /dev/null +++ b/src/discover/userconfig.c @@ -0,0 +1,388 @@ +/* + * userconfig.c — User-defined extension→language mappings. + * + * Reads extra_extensions from: + * Global: $XDG_CONFIG_HOME/codebase-memory-mcp/config.json + * (falls back to ~/.config/codebase-memory-mcp/config.json) + * Project: {repo_root}/.codebase-memory.json + * + * Project config wins over global. Unknown language values warn and are + * skipped (fail-open). Missing files are silently ignored. + */ +#include "discover/userconfig.h" +#include "foundation/log.h" + +#include + +#include +#include +#include +#include + +/* ── Process-global user config pointer ──────────────────────────── */ + +static const cbm_userconfig_t *g_userconfig = NULL; + +void cbm_set_user_lang_config(const cbm_userconfig_t *cfg) { + g_userconfig = cfg; +} + +const cbm_userconfig_t *cbm_get_user_lang_config(void) { + return g_userconfig; +} + +/* ── Language name → enum table ──────────────────────────────────── */ + +/* + * Reverse-mapping from lowercase language name strings to CBMLanguage. + * Covers all names exposed by cbm_language_name() plus common aliases. + */ +typedef struct { + const char *name; /* lowercase */ + CBMLanguage lang; +} lang_name_entry_t; + +static const lang_name_entry_t LANG_NAME_TABLE[] = { + {"go", CBM_LANG_GO}, + {"python", CBM_LANG_PYTHON}, + {"javascript", CBM_LANG_JAVASCRIPT}, + {"typescript", CBM_LANG_TYPESCRIPT}, + {"tsx", CBM_LANG_TSX}, + {"rust", CBM_LANG_RUST}, + {"java", CBM_LANG_JAVA}, + {"c++", CBM_LANG_CPP}, + {"cpp", CBM_LANG_CPP}, + {"c#", CBM_LANG_CSHARP}, + {"csharp", CBM_LANG_CSHARP}, + {"php", CBM_LANG_PHP}, + {"lua", CBM_LANG_LUA}, + {"scala", CBM_LANG_SCALA}, + {"kotlin", CBM_LANG_KOTLIN}, + {"ruby", CBM_LANG_RUBY}, + {"c", CBM_LANG_C}, + {"bash", CBM_LANG_BASH}, + {"sh", CBM_LANG_BASH}, + {"zig", CBM_LANG_ZIG}, + {"elixir", CBM_LANG_ELIXIR}, + {"haskell", CBM_LANG_HASKELL}, + {"ocaml", CBM_LANG_OCAML}, + {"objective-c", CBM_LANG_OBJC}, + {"objc", CBM_LANG_OBJC}, + {"swift", CBM_LANG_SWIFT}, + {"dart", CBM_LANG_DART}, + {"perl", CBM_LANG_PERL}, + {"groovy", CBM_LANG_GROOVY}, + {"erlang", CBM_LANG_ERLANG}, + {"r", CBM_LANG_R}, + {"html", CBM_LANG_HTML}, + {"css", CBM_LANG_CSS}, + {"scss", CBM_LANG_SCSS}, + {"yaml", CBM_LANG_YAML}, + {"toml", CBM_LANG_TOML}, + {"hcl", CBM_LANG_HCL}, + {"terraform", CBM_LANG_HCL}, + {"sql", CBM_LANG_SQL}, + {"dockerfile", CBM_LANG_DOCKERFILE}, + {"clojure", CBM_LANG_CLOJURE}, + {"f#", CBM_LANG_FSHARP}, + {"fsharp", CBM_LANG_FSHARP}, + {"julia", CBM_LANG_JULIA}, + {"vimscript", CBM_LANG_VIMSCRIPT}, + {"nix", CBM_LANG_NIX}, + {"common lisp", CBM_LANG_COMMONLISP}, + {"commonlisp", CBM_LANG_COMMONLISP}, + {"lisp", CBM_LANG_COMMONLISP}, + {"elm", CBM_LANG_ELM}, + {"fortran", CBM_LANG_FORTRAN}, + {"cuda", CBM_LANG_CUDA}, + {"cobol", CBM_LANG_COBOL}, + {"verilog", CBM_LANG_VERILOG}, + {"emacs lisp", CBM_LANG_EMACSLISP}, + {"emacslisp", CBM_LANG_EMACSLISP}, + {"json", CBM_LANG_JSON}, + {"xml", CBM_LANG_XML}, + {"markdown", CBM_LANG_MARKDOWN}, + {"makefile", CBM_LANG_MAKEFILE}, + {"cmake", CBM_LANG_CMAKE}, + {"protobuf", CBM_LANG_PROTOBUF}, + {"graphql", CBM_LANG_GRAPHQL}, + {"vue", CBM_LANG_VUE}, + {"svelte", CBM_LANG_SVELTE}, + {"meson", CBM_LANG_MESON}, + {"glsl", CBM_LANG_GLSL}, + {"ini", CBM_LANG_INI}, + {"matlab", CBM_LANG_MATLAB}, + {"lean", CBM_LANG_LEAN}, + {"form", CBM_LANG_FORM}, + {"magma", CBM_LANG_MAGMA}, + {"wolfram", CBM_LANG_WOLFRAM}, +}; + +#define LANG_NAME_TABLE_SIZE (sizeof(LANG_NAME_TABLE) / sizeof(LANG_NAME_TABLE[0])) + +/* + * Parse a language string (case-insensitive) to a CBMLanguage enum. + * Returns CBM_LANG_COUNT if the string is not recognized. + */ +static CBMLanguage lang_from_string(const char *s) { + if (!s || !s[0]) { + return CBM_LANG_COUNT; + } + + /* Build a lowercase copy for comparison */ + char lower[64]; + size_t i; + for (i = 0; i < sizeof(lower) - 1 && s[i]; i++) { + lower[i] = (char)tolower((unsigned char)s[i]); + } + lower[i] = '\0'; + + for (size_t j = 0; j < LANG_NAME_TABLE_SIZE; j++) { + if (strcmp(LANG_NAME_TABLE[j].name, lower) == 0) { + return LANG_NAME_TABLE[j].lang; + } + } + return CBM_LANG_COUNT; +} + +/* ── Config directory helper ─────────────────────────────────────── */ + +/* + * Get the XDG config dir for codebase-memory-mcp. + * Writes "/codebase-memory-mcp" into buf (up to bufsz bytes). + * Uses $XDG_CONFIG_HOME if set, else ~/.config. + */ +static void cbm_app_config_dir(char *buf, size_t bufsz) { + // NOLINT(concurrency-mt-unsafe) — called before worker threads + const char *xdg = getenv("XDG_CONFIG_HOME"); + if (xdg && xdg[0]) { + snprintf(buf, bufsz, "%s/codebase-memory-mcp", xdg); + } else { + const char *home = getenv("HOME"); // NOLINT(concurrency-mt-unsafe) + if (!home || !home[0]) { + home = "/tmp"; + } + snprintf(buf, bufsz, "%s/.config/codebase-memory-mcp", home); + } +} + +/* ── JSON parsing ────────────────────────────────────────────────── */ + +/* + * Parse extra_extensions from a yyjson object root. + * Appends valid entries to *entries / *count (growing via realloc). + * Project-level entries (from_project=true) are appended after global + * entries so that a later dedup pass can prefer project values. + * + * Returns 0 on success, -1 on alloc failure. + */ +static int parse_extra_extensions(yyjson_val *root, cbm_userext_t **entries, int *count, + const char *source_label) { + if (!yyjson_is_obj(root)) { + cbm_log_warn("userconfig.bad_root", "file", source_label); + return 0; + } + + yyjson_val *extra = yyjson_obj_get(root, "extra_extensions"); + if (!extra) { + return 0; /* key absent — fine */ + } + if (!yyjson_is_obj(extra)) { + cbm_log_warn("userconfig.bad_extra_extensions", "file", source_label); + return 0; + } + + yyjson_obj_iter iter; + yyjson_obj_iter_init(extra, &iter); + yyjson_val *key; + while ((key = yyjson_obj_iter_next(&iter)) != NULL) { + yyjson_val *val = yyjson_obj_iter_get_val(key); + + const char *ext_str = yyjson_get_str(key); + const char *lang_str = yyjson_get_str(val); + + if (!ext_str || !lang_str) { + cbm_log_warn("userconfig.skip_non_string", "file", source_label); + continue; + } + + /* Extension must start with '.' */ + if (ext_str[0] != '.') { + cbm_log_warn("userconfig.skip_bad_ext", "file", source_label, "ext", ext_str); + continue; + } + + CBMLanguage lang = lang_from_string(lang_str); + if (lang == CBM_LANG_COUNT) { + cbm_log_warn("userconfig.unknown_lang", "file", source_label, "lang", lang_str); + continue; /* fail-open: skip unknown languages */ + } + + /* Grow the array */ + cbm_userext_t *tmp = + realloc(*entries, (size_t)(*count + 1) * sizeof(cbm_userext_t)); + if (!tmp) { + return -1; + } + *entries = tmp; + + char *ext_copy = strdup(ext_str); + if (!ext_copy) { + return -1; + } + + (*entries)[*count].ext = ext_copy; + (*entries)[*count].lang = lang; + (*count)++; + } + return 0; +} + +/* + * Read a JSON file and parse extra_extensions from it. + * Silently ignores missing files. Logs warnings for corrupt JSON. + * Returns 0 on success (or absent file), -1 on alloc failure. + */ +static int load_config_file(const char *path, cbm_userext_t **entries, int *count) { + FILE *f = fopen(path, "rb"); + if (!f) { + return 0; /* file absent — silently ignore */ + } + + fseek(f, 0, SEEK_END); + long len = ftell(f); + fseek(f, 0, SEEK_SET); + + if (len <= 0 || len > 65536) { + fclose(f); + if (len > 65536) { + cbm_log_warn("userconfig.file_too_large", "path", path); + } + return 0; + } + + char *buf = malloc((size_t)len + 1); + if (!buf) { + fclose(f); + return -1; + } + + size_t nread = fread(buf, 1, (size_t)len, f); + fclose(f); + buf[nread] = '\0'; + + yyjson_doc *doc = yyjson_read(buf, nread, 0); + free(buf); + + if (!doc) { + cbm_log_warn("userconfig.corrupt_json", "path", path); + return 0; /* corrupt JSON — silently ignore (fail-open) */ + } + + yyjson_val *root = yyjson_doc_get_root(doc); + int rc = parse_extra_extensions(root, entries, count, path); + yyjson_doc_free(doc); + return rc; +} + +/* ── Public API ──────────────────────────────────────────────────── */ + +cbm_userconfig_t *cbm_userconfig_load(const char *repo_path) { + cbm_userconfig_t *cfg = calloc(1, sizeof(cbm_userconfig_t)); + if (!cfg) { + return NULL; + } + + cbm_userext_t *entries = NULL; + int count = 0; + + /* ── Step 1: Load global config ── */ + char global_dir[1024]; + cbm_app_config_dir(global_dir, sizeof(global_dir)); + + char global_path[1280]; + snprintf(global_path, sizeof(global_path), "%s/config.json", global_dir); + + if (load_config_file(global_path, &entries, &count) != 0) { + free(cfg); + return NULL; + } + + int global_count = count; /* entries[0..global_count) are from global */ + + /* ── Step 2: Load project config ── */ + if (repo_path && repo_path[0]) { + char project_path[1280]; + snprintf(project_path, sizeof(project_path), "%s/.codebase-memory.json", repo_path); + + if (load_config_file(project_path, &entries, &count) != 0) { + /* Free already-allocated entries */ + for (int i = 0; i < count; i++) { + free(entries[i].ext); + } + free(entries); + free(cfg); + return NULL; + } + } + + /* + * ── Step 3: Dedup — project entries win over global ── + * + * For any extension that appears in both global (indices 0..global_count) + * and project (indices global_count..count), remove the global entry by + * replacing it with the last global entry (order-insensitive dedup). + */ + for (int p = global_count; p < count; p++) { + for (int g = 0; g < global_count; g++) { + if (entries[g].ext && strcmp(entries[g].ext, entries[p].ext) == 0) { + /* Remove global entry: overwrite with last global entry */ + free(entries[g].ext); + entries[g] = entries[global_count - 1]; + entries[global_count - 1].ext = NULL; /* mark as consumed */ + global_count--; + g--; /* re-check this index */ + break; + } + } + } + + /* + * Compact: remove any NULL-ext slots left by the dedup step. + * (Those are the consumed "last global" entries.) + */ + int write_idx = 0; + for (int i = 0; i < count; i++) { + if (entries[i].ext != NULL) { + entries[write_idx++] = entries[i]; + } + } + count = write_idx; + + cfg->entries = entries; + cfg->count = count; + return cfg; +} + +CBMLanguage cbm_userconfig_lookup(const cbm_userconfig_t *cfg, const char *ext) { + if (!cfg || !ext || !ext[0]) { + return CBM_LANG_COUNT; + } + for (int i = 0; i < cfg->count; i++) { + if (cfg->entries[i].ext && strcmp(cfg->entries[i].ext, ext) == 0) { + return cfg->entries[i].lang; + } + } + return CBM_LANG_COUNT; +} + +void cbm_userconfig_free(cbm_userconfig_t *cfg) { + if (!cfg) { + return; + } + for (int i = 0; i < cfg->count; i++) { + free(cfg->entries[i].ext); + } + free(cfg->entries); + free(cfg); +} diff --git a/src/discover/userconfig.h b/src/discover/userconfig.h new file mode 100644 index 000000000..c45777fba --- /dev/null +++ b/src/discover/userconfig.h @@ -0,0 +1,72 @@ +/* + * userconfig.h — User-defined file extension → language mappings. + * + * Reads extra_extensions from two optional JSON config files: + * Global: $XDG_CONFIG_HOME/codebase-memory-mcp/config.json + * (falls back to ~/.config/codebase-memory-mcp/config.json) + * Project: {repo_root}/.codebase-memory.json + * + * Project config wins over global. Unknown language values warn and are + * skipped (fail-open). Missing files are silently ignored. + * + * Format: + * {"extra_extensions": {".blade.php": "php", ".mjs": "javascript"}} + * + * The language string matching is case-insensitive. + */ +#ifndef CBM_USERCONFIG_H +#define CBM_USERCONFIG_H + +#include "cbm.h" /* CBMLanguage */ + +/* ── Types ──────────────────────────────────────────────────────── */ + +typedef struct { + char *ext; /* file extension including dot, e.g. ".blade.php" */ + CBMLanguage lang; /* resolved language enum */ +} cbm_userext_t; + +typedef struct { + cbm_userext_t *entries; /* heap-allocated array */ + int count; /* number of entries */ +} cbm_userconfig_t; + +/* ── API ────────────────────────────────────────────────────────── */ + +/* + * Load user config from global + project files, merge (project wins). + * repo_path: absolute path to the repository root (for project config). + * Returns a heap-allocated cbm_userconfig_t (caller must free via + * cbm_userconfig_free). Returns NULL only on allocation failure. + * Missing config files are silently ignored. + */ +cbm_userconfig_t *cbm_userconfig_load(const char *repo_path); + +/* + * Look up a file extension in the user config. + * ext: extension including dot, e.g. ".blade.php" + * Returns the mapped CBMLanguage, or CBM_LANG_COUNT if not found. + */ +CBMLanguage cbm_userconfig_lookup(const cbm_userconfig_t *cfg, const char *ext); + +/* Free a cbm_userconfig_t returned by cbm_userconfig_load. NULL-safe. */ +void cbm_userconfig_free(cbm_userconfig_t *cfg); + +/* ── Integration hook ───────────────────────────────────────────── */ + +/* + * Set the process-global user config that cbm_language_for_extension() + * will consult before the built-in table. + * cfg may be NULL to clear the override. + * Not thread-safe — call before spawning worker threads. + */ +void cbm_set_user_lang_config(const cbm_userconfig_t *cfg); + +/* + * Get the currently active process-global user config. + * Returns NULL if none has been set. + * Called internally by cbm_language_for_extension(). + */ +const cbm_userconfig_t *cbm_get_user_lang_config(void); + +#endif /* CBM_USERCONFIG_H */ diff --git a/src/pipeline/pipeline.c b/src/pipeline/pipeline.c index f5b75108d..63ddeed10 100644 --- a/src/pipeline/pipeline.c +++ b/src/pipeline/pipeline.c @@ -17,6 +17,7 @@ #include "graph_buffer/graph_buffer.h" #include "store/store.h" #include "discover/discover.h" +#include "discover/userconfig.h" #include "foundation/platform.h" #include "foundation/compat_fs.h" #include "foundation/log.h" @@ -44,6 +45,9 @@ struct cbm_pipeline { /* Indexing state (set during run) */ cbm_gbuf_t *gbuf; cbm_registry_t *registry; + + /* User-defined extension overrides (loaded once per run) */ + cbm_userconfig_t *userconfig; }; /* ── Timing helper ──────────────────────────────────────────────── */ @@ -303,6 +307,10 @@ int cbm_pipeline_run(cbm_pipeline_t *p) { struct timespec t0; cbm_clock_gettime(CLOCK_MONOTONIC, &t0); + /* Load user-defined extension overrides (fail-open: NULL on error) */ + p->userconfig = cbm_userconfig_load(p->repo_path); + cbm_set_user_lang_config(p->userconfig); + /* Phase 1: Discover files */ cbm_discover_opts_t opts = { .mode = p->mode, @@ -759,5 +767,9 @@ cleanup: p->gbuf = NULL; cbm_registry_free(p->registry); p->registry = NULL; + /* Clear and free user extension config */ + cbm_set_user_lang_config(NULL); + cbm_userconfig_free(p->userconfig); + p->userconfig = NULL; return rc; } diff --git a/tests/test_main.c b/tests/test_main.c index 0fec7f47e..070a17004 100644 --- a/tests/test_main.c +++ b/tests/test_main.c @@ -26,6 +26,7 @@ extern void suite_store_search(void); extern void suite_cypher(void); extern void suite_mcp(void); extern void suite_language(void); +extern void suite_userconfig(void); extern void suite_gitignore(void); extern void suite_discover(void); extern void suite_graph_buffer(void); @@ -80,6 +81,7 @@ int main(void) { /* Discover (M2) */ RUN_SUITE(language); + RUN_SUITE(userconfig); RUN_SUITE(gitignore); RUN_SUITE(discover); diff --git a/tests/test_userconfig.c b/tests/test_userconfig.c new file mode 100644 index 000000000..efaf61d19 --- /dev/null +++ b/tests/test_userconfig.c @@ -0,0 +1,218 @@ +/* + * test_userconfig.c — Tests for user-defined extension→language mappings. + * + * Tests cbm_userconfig_load(), cbm_userconfig_lookup(), and the + * cbm_set_user_lang_config() / cbm_language_for_extension() integration. + */ +#include "../src/foundation/compat.h" +#include "../src/foundation/compat_fs.h" +#include "test_framework.h" +#include "discover/discover.h" +#include "discover/userconfig.h" + +#include +#include +#include + +/* ── Helpers ─────────────────────────────────────────────────────── */ + +/* Write a JSON file to path. Returns 0 on success. */ +static int write_json(const char *path, const char *json) { + FILE *f = fopen(path, "w"); + if (!f) { + return -1; + } + fputs(json, f); + fclose(f); + return 0; +} + +/* ── Tests: project config ───────────────────────────────────────── */ + +TEST(userconfig_project_basic) { + /* Write a .codebase-memory.json in a temp dir */ + char dir[256]; + snprintf(dir, sizeof(dir), "%s/uctest_proj_basic", cbm_tmpdir()); + cbm_mkdir_p(dir, 0755); /* from compat_fs.h via compat.h */ + + char proj[512]; + snprintf(proj, sizeof(proj), "%s/.codebase-memory.json", dir); + ASSERT_EQ( + write_json(proj, "{\"extra_extensions\":{\".blade.php\":\"php\",\".mjs\":\"javascript\"}}"), + 0); + + cbm_userconfig_t *cfg = cbm_userconfig_load(dir); + ASSERT_NOT_NULL(cfg); + + ASSERT_EQ(cbm_userconfig_lookup(cfg, ".blade.php"), CBM_LANG_PHP); + ASSERT_EQ(cbm_userconfig_lookup(cfg, ".mjs"), CBM_LANG_JAVASCRIPT); + ASSERT_EQ(cbm_userconfig_lookup(cfg, ".go"), CBM_LANG_COUNT); /* not in user config */ + + cbm_userconfig_free(cfg); + remove(proj); + PASS(); +} + +/* ── Tests: global config ────────────────────────────────────────── */ + +TEST(userconfig_global_via_env) { + /* Point XDG_CONFIG_HOME to a temp dir */ + char xdg_dir[256]; + snprintf(xdg_dir, sizeof(xdg_dir), "%s/uctest_global_xdg", cbm_tmpdir()); + + char app_dir[512]; + snprintf(app_dir, sizeof(app_dir), "%s/codebase-memory-mcp", xdg_dir); + cbm_mkdir_p(app_dir, 0755); + + char global_path[768]; + snprintf(global_path, sizeof(global_path), "%s/config.json", app_dir); + ASSERT_EQ( + write_json(global_path, "{\"extra_extensions\":{\".twig\":\"html\"}}"), + 0); + + /* Set env var, load, restore */ + setenv("XDG_CONFIG_HOME", xdg_dir, 1); + cbm_userconfig_t *cfg = cbm_userconfig_load(NULL); /* no project dir */ + unsetenv("XDG_CONFIG_HOME"); + + ASSERT_NOT_NULL(cfg); + ASSERT_EQ(cbm_userconfig_lookup(cfg, ".twig"), CBM_LANG_HTML); + + cbm_userconfig_free(cfg); + remove(global_path); + PASS(); +} + +/* ── Tests: project wins over global ────────────────────────────── */ + +TEST(userconfig_project_wins_over_global) { + /* Global says .xyz → python; project says .xyz → rust */ + char xdg_dir[256]; + snprintf(xdg_dir, sizeof(xdg_dir), "%s/uctest_priority_xdg", cbm_tmpdir()); + + char app_dir[512]; + snprintf(app_dir, sizeof(app_dir), "%s/codebase-memory-mcp", xdg_dir); + cbm_mkdir_p(app_dir, 0755); + + char global_path[768]; + snprintf(global_path, sizeof(global_path), "%s/config.json", app_dir); + ASSERT_EQ( + write_json(global_path, "{\"extra_extensions\":{\".xyz\":\"python\"}}"), + 0); + + char proj_dir[256]; + snprintf(proj_dir, sizeof(proj_dir), "%s/uctest_priority_proj", cbm_tmpdir()); + cbm_mkdir_p(proj_dir, 0755); + + char proj_path[512]; + snprintf(proj_path, sizeof(proj_path), "%s/.codebase-memory.json", proj_dir); + ASSERT_EQ( + write_json(proj_path, "{\"extra_extensions\":{\".xyz\":\"rust\"}}"), + 0); + + setenv("XDG_CONFIG_HOME", xdg_dir, 1); + cbm_userconfig_t *cfg = cbm_userconfig_load(proj_dir); + unsetenv("XDG_CONFIG_HOME"); + + ASSERT_NOT_NULL(cfg); + /* Project definition (rust) must win */ + ASSERT_EQ(cbm_userconfig_lookup(cfg, ".xyz"), CBM_LANG_RUST); + + cbm_userconfig_free(cfg); + remove(global_path); + remove(proj_path); + PASS(); +} + +/* ── Tests: unknown language values are skipped ──────────────────── */ + +TEST(userconfig_unknown_lang_skipped) { + char dir[256]; + snprintf(dir, sizeof(dir), "%s/uctest_unknown_lang", cbm_tmpdir()); + cbm_mkdir_p(dir, 0755); + + char proj[512]; + snprintf(proj, sizeof(proj), "%s/.codebase-memory.json", dir); + /* "klingon" is not a valid language; ".wasm" should be silently skipped */ + ASSERT_EQ( + write_json(proj, + "{\"extra_extensions\":{\".wasm\":\"klingon\",\".mjs\":\"javascript\"}}"), + 0); + + cbm_userconfig_t *cfg = cbm_userconfig_load(dir); + ASSERT_NOT_NULL(cfg); + + /* .wasm with unknown lang → not in config */ + ASSERT_EQ(cbm_userconfig_lookup(cfg, ".wasm"), CBM_LANG_COUNT); + /* .mjs with valid lang → present */ + ASSERT_EQ(cbm_userconfig_lookup(cfg, ".mjs"), CBM_LANG_JAVASCRIPT); + + cbm_userconfig_free(cfg); + remove(proj); + PASS(); +} + +/* ── Tests: missing files are silently ignored ───────────────────── */ + +TEST(userconfig_missing_files_ok) { + /* Point to a non-existent repo dir */ + cbm_userconfig_t *cfg = cbm_userconfig_load("/tmp/__nonexistent_repo_12345__"); + ASSERT_NOT_NULL(cfg); /* must not return NULL — just empty */ + ASSERT_EQ(cfg->count, 0); + cbm_userconfig_free(cfg); + PASS(); +} + +/* ── Tests: integration with cbm_language_for_extension ─────────── */ + +TEST(userconfig_integration_override) { + /* Verify that setting the global config makes cbm_language_for_extension + * respect the override. We map ".blade.php" → PHP, which is not in the + * built-in table. */ + char dir[256]; + snprintf(dir, sizeof(dir), "%s/uctest_integ", cbm_tmpdir()); + cbm_mkdir_p(dir, 0755); + + char proj[512]; + snprintf(proj, sizeof(proj), "%s/.codebase-memory.json", dir); + ASSERT_EQ( + write_json(proj, "{\"extra_extensions\":{\".blade.php\":\"php\"}}"), + 0); + + cbm_userconfig_t *cfg = cbm_userconfig_load(dir); + ASSERT_NOT_NULL(cfg); + + /* Before setting, .blade.php is unknown */ + ASSERT_EQ(cbm_language_for_extension(".blade.php"), CBM_LANG_COUNT); + + cbm_set_user_lang_config(cfg); + /* After setting, .blade.php → PHP */ + ASSERT_EQ(cbm_language_for_extension(".blade.php"), CBM_LANG_PHP); + /* Built-in extensions still work */ + ASSERT_EQ(cbm_language_for_extension(".go"), CBM_LANG_GO); + + /* Clean up global state */ + cbm_set_user_lang_config(NULL); + cbm_userconfig_free(cfg); + remove(proj); + PASS(); +} + +/* ── Tests: free is NULL-safe ────────────────────────────────────── */ + +TEST(userconfig_free_null) { + cbm_userconfig_free(NULL); /* must not crash */ + PASS(); +} + +/* ── Suite ──────────────────────────────────────────────────────── */ + +SUITE(userconfig) { + RUN_TEST(userconfig_project_basic); + RUN_TEST(userconfig_global_via_env); + RUN_TEST(userconfig_project_wins_over_global); + RUN_TEST(userconfig_unknown_lang_skipped); + RUN_TEST(userconfig_missing_files_ok); + RUN_TEST(userconfig_integration_override); + RUN_TEST(userconfig_free_null); +} From 1bfe473a7678f20779b73504ab76d03a6febef8f Mon Sep 17 00:00:00 2001 From: Shane McCarron Date: Thu, 19 Mar 2026 18:07:27 -0500 Subject: [PATCH 02/25] fix(discover): address QA round 1 findings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - pipeline.c: inline userconfig cleanup on early-exit paths (discover failure and cancel check) so userconfig is always freed, fixing the resource leak when pipeline exits before reaching the cleanup label - userconfig.c: free partially-built entries array on global config parse failure to fix alloc-failure memory leak - language.c: probe user config for compound extensions (e.g. ".blade.php") by scanning from the first dot to the last, falling back to built-in single-extension lookup — preserves built-in behavior while correctly resolving user-defined compound extensions Co-Authored-By: Claude Sonnet 4.6 --- src/discover/language.c | 27 ++++++++++++++++++++++----- src/discover/userconfig.c | 4 ++++ src/pipeline/pipeline.c | 7 +++++++ 3 files changed, 33 insertions(+), 5 deletions(-) diff --git a/src/discover/language.c b/src/discover/language.c index 51b74c24c..ea07e9c2d 100644 --- a/src/discover/language.c +++ b/src/discover/language.c @@ -386,13 +386,30 @@ CBMLanguage cbm_language_for_filename(const char *filename) { } } - /* Fall back to extension-based lookup */ - const char *dot = strrchr(filename, '.'); - if (dot) { - return cbm_language_for_extension(dot); + /* Fall back to extension-based lookup. + * For compound extensions (e.g. ".blade.php") defined in the user config, + * scan from the first dot in the basename toward the last, checking user + * config at each position. Built-in extensions use the last dot only. */ + const char *last_dot = strrchr(filename, '.'); + if (!last_dot) { + return CBM_LANG_COUNT; } - return CBM_LANG_COUNT; + /* Probe user config for compound extensions (e.g. ".blade.php"). */ + const cbm_userconfig_t *ucfg = cbm_get_user_lang_config(); + if (ucfg) { + const char *p = strchr(filename, '.'); + while (p && p < last_dot) { + CBMLanguage lang = cbm_userconfig_lookup(ucfg, p); + if (lang != CBM_LANG_COUNT) { + return lang; + } + p = strchr(p + 1, '.'); + } + } + + /* Standard single-extension lookup (built-ins + user overrides). */ + return cbm_language_for_extension(last_dot); } const char *cbm_language_name(CBMLanguage lang) { diff --git a/src/discover/userconfig.c b/src/discover/userconfig.c index 66edef254..d9dd74982 100644 --- a/src/discover/userconfig.c +++ b/src/discover/userconfig.c @@ -304,6 +304,10 @@ cbm_userconfig_t *cbm_userconfig_load(const char *repo_path) { snprintf(global_path, sizeof(global_path), "%s/config.json", global_dir); if (load_config_file(global_path, &entries, &count) != 0) { + for (int i = 0; i < count; i++) { + free(entries[i].ext); + } + free(entries); free(cfg); return NULL; } diff --git a/src/pipeline/pipeline.c b/src/pipeline/pipeline.c index 63ddeed10..e6ee73a69 100644 --- a/src/pipeline/pipeline.c +++ b/src/pipeline/pipeline.c @@ -322,6 +322,10 @@ int cbm_pipeline_run(cbm_pipeline_t *p) { int rc = cbm_discover(p->repo_path, &opts, &files, &file_count); if (rc != 0) { cbm_log_error("pipeline.err", "phase", "discover", "rc", itoa_buf(rc)); + cbm_discover_free(files, file_count); + cbm_set_user_lang_config(NULL); + cbm_userconfig_free(p->userconfig); + p->userconfig = NULL; return -1; } cbm_log_info("pipeline.discover", "files", itoa_buf(file_count), "elapsed_ms", @@ -329,6 +333,9 @@ int cbm_pipeline_run(cbm_pipeline_t *p) { if (check_cancel(p)) { cbm_discover_free(files, file_count); + cbm_set_user_lang_config(NULL); + cbm_userconfig_free(p->userconfig); + p->userconfig = NULL; return -1; } From 57e08e616b6231484db89e7183c14e93f5aa502d Mon Sep 17 00:00:00 2001 From: Shane McCarron Date: Thu, 19 Mar 2026 18:18:03 -0500 Subject: [PATCH 03/25] fix(discover): address QA round 2 findings - Defensively free p->userconfig in cbm_pipeline_free in case run() was never called (e.g. pipeline created but not executed before free) Co-Authored-By: Claude Sonnet 4.6 --- src/pipeline/pipeline.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/pipeline/pipeline.c b/src/pipeline/pipeline.c index e6ee73a69..b553171db 100644 --- a/src/pipeline/pipeline.c +++ b/src/pipeline/pipeline.c @@ -101,6 +101,12 @@ void cbm_pipeline_free(cbm_pipeline_t *p) { free(p->db_path); free(p->project_name); /* gbuf, store, registry freed during/after run */ + /* Defensively free userconfig in case run() was never called or panicked */ + if (p->userconfig) { + cbm_set_user_lang_config(NULL); + cbm_userconfig_free(p->userconfig); + p->userconfig = NULL; + } free(p); } From ee769d5806af1f855aa63abe4fce8e0ea6f8fc1d Mon Sep 17 00:00:00 2001 From: Shane McCarron Date: Fri, 20 Mar 2026 16:38:14 -0500 Subject: [PATCH 04/25] feat(k8s): detection helpers, CBMLanguage enum, and language table entries - Add CBM_LANG_KUSTOMIZE and CBM_LANG_K8S to CBMLanguage enum (before CBM_LANG_COUNT) - Add kustomization.yaml/yml to FILENAME_TABLE mapped to CBM_LANG_KUSTOMIZE - Add Kustomize and Kubernetes entries to LANG_NAMES - Implement cbm_is_kustomize_file() with to_lower+strcmp pattern - Implement cbm_is_k8s_manifest() scanning first 4KB for apiVersion: via ci_strstr() - Declare both helpers in pipeline_internal.h Infrascan helpers section Co-Authored-By: Claude Sonnet 4.6 --- internal/cbm/cbm.h | 2 ++ src/discover/language.c | 4 ++++ src/pipeline/pass_infrascan.c | 19 +++++++++++++++++++ src/pipeline/pipeline_internal.h | 2 ++ 4 files changed, 27 insertions(+) diff --git a/internal/cbm/cbm.h b/internal/cbm/cbm.h index 16b9dd083..1dbab2615 100644 --- a/internal/cbm/cbm.h +++ b/internal/cbm/cbm.h @@ -75,6 +75,8 @@ typedef enum { CBM_LANG_FORM, CBM_LANG_MAGMA, CBM_LANG_WOLFRAM, + CBM_LANG_KUSTOMIZE, // kustomization.yaml — Kubernetes overlay tool + CBM_LANG_K8S, // Generic Kubernetes manifest (apiVersion: detected) CBM_LANG_COUNT } CBMLanguage; diff --git a/src/discover/language.c b/src/discover/language.c index b7eb7e439..cad73e012 100644 --- a/src/discover/language.c +++ b/src/discover/language.c @@ -273,6 +273,8 @@ static const filename_entry_t FILENAME_TABLE[] = { {"GNUmakefile", CBM_LANG_MAKEFILE}, {"Makefile", CBM_LANG_MAKEFILE}, {"makefile", CBM_LANG_MAKEFILE}, {"meson.build", CBM_LANG_MESON}, {"meson.options", CBM_LANG_MESON}, {"meson_options.txt", CBM_LANG_MESON}, + {"kustomization.yaml", CBM_LANG_KUSTOMIZE}, + {"kustomization.yml", CBM_LANG_KUSTOMIZE}, {".vimrc", CBM_LANG_VIMSCRIPT}, }; @@ -345,6 +347,8 @@ static const char *LANG_NAMES[CBM_LANG_COUNT] = { [CBM_LANG_FORM] = "FORM", [CBM_LANG_MAGMA] = "Magma", [CBM_LANG_WOLFRAM] = "Wolfram", + [CBM_LANG_KUSTOMIZE] = "Kustomize", + [CBM_LANG_K8S] = "Kubernetes", }; /* ── Public API ──────────────────────────────────────────────────── */ diff --git a/src/pipeline/pass_infrascan.c b/src/pipeline/pass_infrascan.c index 30c47d95e..307973f5c 100644 --- a/src/pipeline/pass_infrascan.c +++ b/src/pipeline/pass_infrascan.c @@ -192,6 +192,25 @@ bool cbm_is_env_file(const char *name) { return false; } +bool cbm_is_kustomize_file(const char *name) { + if (!name) { return false; } + char lower[256]; + to_lower(name, lower, sizeof(lower)); + return (strcmp(lower, "kustomization.yaml") == 0 || + strcmp(lower, "kustomization.yml") == 0); +} + +// NOLINTNEXTLINE(bugprone-easily-swappable-parameters) +bool cbm_is_k8s_manifest(const char *name, const char *content) { + if (!name || !content || cbm_is_kustomize_file(name)) { return false; } + char buf[4097]; + size_t n = strlen(content); + if (n > 4096) { n = 4096; } + memcpy(buf, content, n); + buf[n] = '\0'; + return ci_strstr(buf, "apiVersion:") != NULL; +} + // NOLINTNEXTLINE(bugprone-easily-swappable-parameters) bool cbm_is_shell_script(const char *name, const char *ext) { (void)name; diff --git a/src/pipeline/pipeline_internal.h b/src/pipeline/pipeline_internal.h index c1d45f14f..ea5c1d73f 100644 --- a/src/pipeline/pipeline_internal.h +++ b/src/pipeline/pipeline_internal.h @@ -218,6 +218,8 @@ bool cbm_is_compose_file(const char *name); bool cbm_is_cloudbuild_file(const char *name); bool cbm_is_env_file(const char *name); bool cbm_is_shell_script(const char *name, const char *ext); +bool cbm_is_kustomize_file(const char *name); +bool cbm_is_k8s_manifest(const char *name, const char *content); /* Secret detection */ bool cbm_is_secret_binding(const char *key, const char *value); From 88ddb63e8be5c4b4696799268d793c3c8923b602 Mon Sep 17 00:00:00 2001 From: Shane McCarron Date: Fri, 20 Mar 2026 16:43:00 -0500 Subject: [PATCH 05/25] feat(k8s): K8s and Kustomize YAML extractors in the CBM layer - lang_specs.c: add LangSpec entries for CBM_LANG_KUSTOMIZE and CBM_LANG_K8S (both reuse tree_sitter_yaml()); add cbm_ts_language() switch cases - extract_k8s.c: new file implementing cbm_extract_k8s(); kustomize path walks block_sequence items under resources/bases/patches/components/ patchesStrategicMerge and emits CBMImport per scalar; k8s path extracts apiVersion/kind/metadata.name and emits CBMDefinition with label "Resource" and name "Kind/metadata-name"; malformed manifests (missing kind or name) produce zero definitions - cbm.h: declare cbm_extract_k8s() alongside other sub-extractor entry points - cbm.c: call cbm_extract_k8s() after unified extraction for the two new langs - Makefile.cbm: add extract_k8s.c to EXTRACTION_SRCS Co-Authored-By: Claude Sonnet 4.6 --- Makefile.cbm | 1 + internal/cbm/cbm.c | 5 + internal/cbm/cbm.h | 3 + internal/cbm/extract_k8s.c | 284 +++++++++++++++++++++++++++++++++++++ internal/cbm/lang_specs.c | 13 ++ 5 files changed, 306 insertions(+) create mode 100644 internal/cbm/extract_k8s.c diff --git a/Makefile.cbm b/Makefile.cbm index 82821b84b..cf5171b9c 100644 --- a/Makefile.cbm +++ b/Makefile.cbm @@ -115,6 +115,7 @@ EXTRACTION_SRCS = \ $(CBM_DIR)/extract_type_refs.c \ $(CBM_DIR)/extract_type_assigns.c \ $(CBM_DIR)/extract_env_accesses.c \ + $(CBM_DIR)/extract_k8s.c \ $(CBM_DIR)/helpers.c \ $(CBM_DIR)/lang_specs.c diff --git a/internal/cbm/cbm.c b/internal/cbm/cbm.c index 6162c7a8a..5b70d1858 100644 --- a/internal/cbm/cbm.c +++ b/internal/cbm/cbm.c @@ -316,6 +316,11 @@ CBMFileResult *cbm_extract_file(const char *source, int source_len, CBMLanguage cbm_extract_imports(&ctx); cbm_extract_unified(&ctx); + // K8s / Kustomize semantic pass (additional structured extraction for YAML-based infra files). + if (ctx.language == CBM_LANG_KUSTOMIZE || ctx.language == CBM_LANG_K8S) { + cbm_extract_k8s(&ctx); + } + // LSP type-aware call resolution uint64_t lsp_start = now_ns(); if (language == CBM_LANG_GO) { diff --git a/internal/cbm/cbm.h b/internal/cbm/cbm.h index 1dbab2615..1fb5a5a87 100644 --- a/internal/cbm/cbm.h +++ b/internal/cbm/cbm.h @@ -363,4 +363,7 @@ void cbm_extract_type_assigns(CBMExtractCtx *ctx); // Single-pass unified extraction (replaces the 7 calls above except defs+imports). void cbm_extract_unified(CBMExtractCtx *ctx); +// K8s / Kustomize semantic extractor (called when language is CBM_LANG_K8S or CBM_LANG_KUSTOMIZE). +void cbm_extract_k8s(CBMExtractCtx *ctx); + #endif // CBM_H diff --git a/internal/cbm/extract_k8s.c b/internal/cbm/extract_k8s.c new file mode 100644 index 000000000..0931d95ed --- /dev/null +++ b/internal/cbm/extract_k8s.c @@ -0,0 +1,284 @@ +// extract_k8s.c — K8s manifest and Kustomize file extractor. +// +// For CBM_LANG_KUSTOMIZE: walks top-level block_mapping_pair nodes whose key +// matches "resources", "bases", "patches", "components", or +// "patchesStrategicMerge", then emits one CBMImport per block_sequence item. +// +// For CBM_LANG_K8S: finds apiVersion, kind, and metadata.name scalars in the +// first document's block_mapping and emits one CBMDefinition with label +// "Resource" and name "Kind/metadata-name". + +#include "cbm.h" +#include "arena.h" +#include "helpers.h" +#include "tree_sitter/api.h" +#include +#include +#include + +// --------------------------------------------------------------------------- +// Internal helpers +// --------------------------------------------------------------------------- + +// Return the raw source text for a scalar node (plain, single-quoted, or +// double-quoted). Surrounding quote characters are stripped for quoted forms. +// Returns NULL for non-scalar node types. +static const char *get_scalar_text(CBMArena *a, TSNode node, const char *source) { + const char *type = ts_node_type(node); + if (strcmp(type, "plain_scalar") == 0) { + return cbm_node_text(a, node, source); + } + if (strcmp(type, "double_quote_scalar") == 0 || strcmp(type, "single_quote_scalar") == 0) { + const char *raw = cbm_node_text(a, node, source); + if (!raw) { + return NULL; + } + size_t len = strlen(raw); + if (len >= 2) { + return cbm_arena_strndup(a, raw + 1, len - 2); + } + return raw; + } + return NULL; +} + +// Return true if the key text of a block_mapping_pair matches one of the +// Kustomize resource-list field names. +static int is_kustomize_list_key(const char *key) { + return (strcmp(key, "resources") == 0 || strcmp(key, "bases") == 0 || + strcmp(key, "patches") == 0 || strcmp(key, "components") == 0 || + strcmp(key, "patchesStrategicMerge") == 0); +} + +// --------------------------------------------------------------------------- +// Kustomize extraction +// --------------------------------------------------------------------------- + +// Walk a block_sequence node and emit one CBMImport per block_sequence_item +// scalar child, using key_name as the local_name. +static void emit_kustomize_sequence(CBMExtractCtx *ctx, TSNode seq_node, + const char *key_name) { + CBMArena *a = ctx->arena; + uint32_t n = ts_node_child_count(seq_node); + for (uint32_t i = 0; i < n; i++) { + TSNode item = ts_node_child(seq_node, i); + if (strcmp(ts_node_type(item), "block_sequence_item") != 0) { + continue; + } + // block_sequence_item has one named child: the value + uint32_t ic = ts_node_child_count(item); + for (uint32_t j = 0; j < ic; j++) { + TSNode val = ts_node_child(item, j); + const char *scalar = get_scalar_text(a, val, ctx->source); + if (!scalar) { + continue; + } + CBMImport imp = { + .local_name = cbm_arena_strdup(a, key_name), + .module_path = cbm_arena_strdup(a, scalar), + }; + cbm_imports_push(&ctx->result->imports, a, imp); + } + } +} + +static void extract_kustomize(CBMExtractCtx *ctx) { + CBMArena *a = ctx->arena; + + // Traverse: stream -> document -> block_node -> block_mapping -> block_mapping_pair + TSNode root = ctx->root; + uint32_t root_n = ts_node_child_count(root); + for (uint32_t si = 0; si < root_n; si++) { + TSNode stream_child = ts_node_child(root, si); + if (strcmp(ts_node_type(stream_child), "document") != 0) { + continue; + } + // Find block_mapping inside the document (may be wrapped in block_node) + TSNode mapping = ts_node_named_child(stream_child, 0); + if (ts_node_is_null(mapping)) { + continue; + } + // Some grammars wrap in block_node + if (strcmp(ts_node_type(mapping), "block_node") == 0) { + mapping = ts_node_named_child(mapping, 0); + } + if (ts_node_is_null(mapping) || strcmp(ts_node_type(mapping), "block_mapping") != 0) { + continue; + } + + uint32_t pair_n = ts_node_child_count(mapping); + for (uint32_t pi = 0; pi < pair_n; pi++) { + TSNode pair = ts_node_child(mapping, pi); + if (strcmp(ts_node_type(pair), "block_mapping_pair") != 0) { + continue; + } + + // First named child = key + TSNode key_node = ts_node_named_child(pair, 0); + if (ts_node_is_null(key_node)) { + continue; + } + const char *key_text = get_scalar_text(a, key_node, ctx->source); + if (!key_text || !is_kustomize_list_key(key_text)) { + continue; + } + + // Second named child = value (should be a block_sequence or block_node wrapping one) + TSNode val_node = ts_node_named_child(pair, 1); + if (ts_node_is_null(val_node)) { + continue; + } + if (strcmp(ts_node_type(val_node), "block_node") == 0) { + val_node = ts_node_named_child(val_node, 0); + } + if (ts_node_is_null(val_node) || + strcmp(ts_node_type(val_node), "block_sequence") != 0) { + continue; + } + + emit_kustomize_sequence(ctx, val_node, key_text); + } + } +} + +// --------------------------------------------------------------------------- +// K8s manifest extraction +// --------------------------------------------------------------------------- + +// Descend into the first block_mapping of a document and extract apiVersion, +// kind, and metadata.name. Returns void; fills kind_buf and meta_name_buf. +static void extract_k8s_scalars(CBMExtractCtx *ctx, TSNode mapping, + char *kind_buf, size_t kind_sz, + char *meta_name_buf, size_t meta_sz) { + CBMArena *a = ctx->arena; + kind_buf[0] = '\0'; + meta_name_buf[0] = '\0'; + + uint32_t n = ts_node_child_count(mapping); + for (uint32_t i = 0; i < n; i++) { + TSNode pair = ts_node_child(mapping, i); + if (strcmp(ts_node_type(pair), "block_mapping_pair") != 0) { + continue; + } + TSNode key_node = ts_node_named_child(pair, 0); + if (ts_node_is_null(key_node)) { + continue; + } + const char *key = get_scalar_text(a, key_node, ctx->source); + if (!key) { + continue; + } + + TSNode val_node = ts_node_named_child(pair, 1); + if (ts_node_is_null(val_node)) { + continue; + } + // Unwrap block_node if present + if (strcmp(ts_node_type(val_node), "block_node") == 0) { + val_node = ts_node_named_child(val_node, 0); + } + if (ts_node_is_null(val_node)) { + continue; + } + + if (strcmp(key, "kind") == 0) { + const char *v = get_scalar_text(a, val_node, ctx->source); + if (v) { + snprintf(kind_buf, kind_sz, "%s", v); + } + } else if (strcmp(key, "metadata") == 0) { + // Descend into metadata block_mapping to find "name" + TSNode meta_mapping = val_node; + if (strcmp(ts_node_type(meta_mapping), "block_node") == 0) { + meta_mapping = ts_node_named_child(meta_mapping, 0); + } + if (ts_node_is_null(meta_mapping) || + strcmp(ts_node_type(meta_mapping), "block_mapping") != 0) { + continue; + } + uint32_t mn = ts_node_child_count(meta_mapping); + for (uint32_t mi = 0; mi < mn; mi++) { + TSNode mpair = ts_node_child(meta_mapping, mi); + if (strcmp(ts_node_type(mpair), "block_mapping_pair") != 0) { + continue; + } + TSNode mkey = ts_node_named_child(mpair, 0); + if (ts_node_is_null(mkey)) { + continue; + } + const char *mkey_text = get_scalar_text(a, mkey, ctx->source); + if (!mkey_text || strcmp(mkey_text, "name") != 0) { + continue; + } + TSNode mval = ts_node_named_child(mpair, 1); + if (ts_node_is_null(mval)) { + continue; + } + const char *meta_name = get_scalar_text(a, mval, ctx->source); + if (meta_name) { + snprintf(meta_name_buf, meta_sz, "%s", meta_name); + } + } + } + } +} + +static void extract_k8s_manifest(CBMExtractCtx *ctx) { + CBMArena *a = ctx->arena; + + TSNode root = ctx->root; + uint32_t root_n = ts_node_child_count(root); + for (uint32_t si = 0; si < root_n; si++) { + TSNode stream_child = ts_node_child(root, si); + if (strcmp(ts_node_type(stream_child), "document") != 0) { + continue; + } + + TSNode mapping = ts_node_named_child(stream_child, 0); + if (ts_node_is_null(mapping)) { + continue; + } + if (strcmp(ts_node_type(mapping), "block_node") == 0) { + mapping = ts_node_named_child(mapping, 0); + } + if (ts_node_is_null(mapping) || strcmp(ts_node_type(mapping), "block_mapping") != 0) { + continue; + } + + char kind_buf[256] = {0}; + char meta_name_buf[256] = {0}; + extract_k8s_scalars(ctx, mapping, kind_buf, sizeof(kind_buf), meta_name_buf, + sizeof(meta_name_buf)); + + // Skip malformed manifests (no kind or no metadata.name) + if (kind_buf[0] == '\0' || meta_name_buf[0] == '\0') { + continue; + } + + char def_name[512]; + snprintf(def_name, sizeof(def_name), "%s/%s", kind_buf, meta_name_buf); + + CBMDefinition def = {0}; + def.name = cbm_arena_strdup(a, def_name); + def.qualified_name = cbm_arena_sprintf(a, "%s.%s", ctx->module_qn, def_name); + def.label = "Resource"; + def.file_path = ctx->rel_path; + def.start_line = ts_node_start_point(mapping).row + 1; + def.end_line = ts_node_end_point(mapping).row + 1; + cbm_defs_push(&ctx->result->defs, a, def); + + break; // Only the first document per file + } +} + +// --------------------------------------------------------------------------- +// Public entry point +// --------------------------------------------------------------------------- + +void cbm_extract_k8s(CBMExtractCtx *ctx) { + if (ctx->language == CBM_LANG_KUSTOMIZE) { + extract_kustomize(ctx); + } else if (ctx->language == CBM_LANG_K8S) { + extract_k8s_manifest(ctx); + } +} diff --git a/internal/cbm/lang_specs.c b/internal/cbm/lang_specs.c index a4fb8097c..426db947b 100644 --- a/internal/cbm/lang_specs.c +++ b/internal/cbm/lang_specs.c @@ -1041,6 +1041,16 @@ static const CBMLangSpec lang_specs[CBM_LANG_COUNT] = { {CBM_LANG_WOLFRAM, wolfram_func_types, empty_types, empty_types, wolfram_module_types, wolfram_call_types, wolfram_import_types, empty_types, empty_types, empty_types, empty_types, empty_types, NULL, empty_types, NULL, NULL}, + + // CBM_LANG_KUSTOMIZE — reuses YAML grammar; semantic extraction via cbm_extract_k8s() + {CBM_LANG_KUSTOMIZE, empty_types, empty_types, empty_types, yaml_module_types, empty_types, + empty_types, empty_types, empty_types, empty_types, empty_types, empty_types, NULL, + empty_types, NULL, NULL}, + + // CBM_LANG_K8S — reuses YAML grammar; semantic extraction via cbm_extract_k8s() + {CBM_LANG_K8S, empty_types, empty_types, empty_types, yaml_module_types, empty_types, + empty_types, empty_types, empty_types, empty_types, empty_types, empty_types, NULL, + empty_types, NULL, NULL}, }; const CBMLangSpec *cbm_lang_spec(CBMLanguage lang) { @@ -1180,6 +1190,9 @@ const TSLanguage *cbm_ts_language(CBMLanguage lang) { return tree_sitter_magma(); case CBM_LANG_WOLFRAM: return tree_sitter_wolfram(); + case CBM_LANG_KUSTOMIZE: + case CBM_LANG_K8S: + return tree_sitter_yaml(); default: return NULL; } From bf3af55bf2d85c269023acce5d1f297e6e06aa3c Mon Sep 17 00:00:00 2001 From: Shane McCarron Date: Fri, 20 Mar 2026 16:49:06 -0500 Subject: [PATCH 06/25] feat(k8s): pipeline pass for k8s manifest parsing and graph node emission - Add pass_k8s.c: cbm_pipeline_pass_k8s() iterates files, classifies kustomize overlays via cbm_is_kustomize_file() and k8s manifests via cbm_is_k8s_manifest(), emits Module/Resource nodes and IMPORTS/DEFINES edges - Kustomize files emit Module node (cbm_infra_qn) + IMPORTS edges per resources entry - K8s manifest files emit Resource nodes per top-level document with DEFINES edge - Falls back to file re-read + re-extraction when result_cache is unavailable - Declare cbm_pipeline_pass_k8s() prototype in pipeline_internal.h - Add pass_k8s.c to PIPELINE_SRCS in Makefile.cbm - Call pass after definitions pass in pipeline.c sequential path Co-Authored-By: Claude Sonnet 4.6 --- Makefile.cbm | 1 + src/pipeline/pass_k8s.c | 269 +++++++++++++++++++++++++++++++ src/pipeline/pipeline.c | 9 ++ src/pipeline/pipeline_internal.h | 2 + 4 files changed, 281 insertions(+) create mode 100644 src/pipeline/pass_k8s.c diff --git a/Makefile.cbm b/Makefile.cbm index cf5171b9c..ecccd3ad3 100644 --- a/Makefile.cbm +++ b/Makefile.cbm @@ -177,6 +177,7 @@ PIPELINE_SRCS = \ src/pipeline/pass_envscan.c \ src/pipeline/pass_compile_commands.c \ src/pipeline/pass_infrascan.c \ + src/pipeline/pass_k8s.c \ src/pipeline/httplink.c # Traces module (new) diff --git a/src/pipeline/pass_k8s.c b/src/pipeline/pass_k8s.c new file mode 100644 index 000000000..a1fec4e04 --- /dev/null +++ b/src/pipeline/pass_k8s.c @@ -0,0 +1,269 @@ +/* + * pass_k8s.c — Pipeline pass for Kubernetes manifest and Kustomize overlay processing. + * + * For each discovered YAML file: + * 1. Check if it is a kustomize overlay (kustomization.yaml / kustomization.yml) + * → emit a Module node and IMPORTS edges for each resources/bases/patches entry + * 2. Else if it is a generic k8s manifest (apiVersion: detected) + * → emit one Resource node per top-level resource document + * + * Depends on: pass_infrascan.c (cbm_is_kustomize_file, cbm_is_k8s_manifest, cbm_infra_qn), + * extraction layer (cbm.h), graph_buffer, pipeline internals. + */ +#include "pipeline/pipeline.h" +#include "pipeline/pipeline_internal.h" +#include "graph_buffer/graph_buffer.h" +#include "discover/discover.h" +#include "foundation/log.h" +#include "cbm.h" + +#include +#include +#include + +/* ── Internal helpers ────────────────────────────────────────────── */ + +/* Read entire file into heap-allocated buffer. Returns NULL on error. + * Caller must free(). Sets *out_len to byte count. */ +static char *k8s_read_file(const char *path, int *out_len) { + FILE *f = fopen(path, "rb"); + if (!f) { + return NULL; + } + + (void)fseek(f, 0, SEEK_END); + long size = ftell(f); + (void)fseek(f, 0, SEEK_SET); + + if (size <= 0 || size > (long)100 * 1024 * 1024) { + (void)fclose(f); + return NULL; + } + + char *buf = malloc(size + 1); + if (!buf) { + (void)fclose(f); + return NULL; + } + + size_t nread = fread(buf, 1, size, f); + (void)fclose(f); + // NOLINTNEXTLINE(clang-analyzer-security.ArrayBound) + buf[nread] = '\0'; + *out_len = (int)nread; + return buf; +} + +/* Format int to string for logging. Thread-safe via TLS. */ +static const char *itoa_k8s(int val) { + static CBM_TLS char bufs[4][32]; + static CBM_TLS int idx = 0; + int i = idx; + idx = (idx + 1) & 3; + snprintf(bufs[i], sizeof(bufs[i]), "%d", val); + return bufs[i]; +} + +/* Extract the basename of a path (pointer into the string; no allocation). */ +static const char *k8s_basename(const char *path) { + const char *p = strrchr(path, '/'); + return p ? p + 1 : path; +} + +/* ── Kustomize handler ───────────────────────────────────────────── */ + +static void handle_kustomize(cbm_pipeline_ctx_t *ctx, const char *path, const char *rel_path, + CBMFileResult *result) { + /* Emit Module node for this kustomize overlay file */ + char *mod_qn = cbm_infra_qn(ctx->project_name, rel_path, "kustomize", NULL); + if (!mod_qn) { + return; + } + + // NOLINTNEXTLINE(misc-include-cleaner) + int64_t mod_id = + cbm_gbuf_upsert_node(ctx->gbuf, "Module", k8s_basename(rel_path), mod_qn, rel_path, 1, 0, + "{\"source\":\"kustomize\"}"); + free(mod_qn); + + if (mod_id <= 0) { + return; + } + + /* If we have a cached extraction result, emit IMPORTS edges for + * resources/bases/patches/components entries */ + int import_count = 0; + CBMFileResult *res = result; + bool allocated = false; + + if (!res) { + /* Fall back to re-extraction */ + int src_len = 0; + char *source = k8s_read_file(path, &src_len); + if (source) { + res = cbm_extract_file(source, src_len, CBM_LANG_KUSTOMIZE, ctx->project_name, rel_path, + CBM_EXTRACT_BUDGET, NULL, NULL); + free(source); + allocated = true; + } + } + + if (res) { + for (int j = 0; j < res->imports.count; j++) { + CBMImport *imp = &res->imports.items[j]; + if (!imp->module_path) { + continue; + } + + /* Compute target file QN */ + char *target_qn = + cbm_pipeline_fqn_compute(ctx->project_name, imp->module_path, "__file__"); + if (!target_qn) { + continue; + } + + const cbm_gbuf_node_t *target = cbm_gbuf_find_by_qn(ctx->gbuf, target_qn); + free(target_qn); + + if (target) { + cbm_gbuf_insert_edge(ctx->gbuf, mod_id, target->id, "IMPORTS", + "{\"via\":\"kustomize\"}"); + import_count++; + } + } + + if (allocated) { + cbm_free_result(res); + } + } + + cbm_log_info("pass.k8s.kustomize", "file", rel_path, "imports", itoa_k8s(import_count)); +} + +/* ── K8s manifest handler ────────────────────────────────────────── */ + +static void handle_k8s_manifest(cbm_pipeline_ctx_t *ctx, const char *path, const char *rel_path, + CBMFileResult *result) { + int resource_count = 0; + CBMFileResult *res = result; + bool allocated = false; + + if (!res) { + /* Fall back to re-extraction */ + int src_len = 0; + char *source = k8s_read_file(path, &src_len); + if (source) { + res = cbm_extract_file(source, src_len, CBM_LANG_K8S, ctx->project_name, rel_path, + CBM_EXTRACT_BUDGET, NULL, NULL); + free(source); + allocated = true; + } + } + + if (!res) { + return; + } + + /* Compute file node QN for DEFINES edges */ + char *file_qn = cbm_pipeline_fqn_compute(ctx->project_name, rel_path, "__file__"); + const cbm_gbuf_node_t *file_node = file_qn ? cbm_gbuf_find_by_qn(ctx->gbuf, file_qn) : NULL; + free(file_qn); + + for (int d = 0; d < res->defs.count; d++) { + CBMDefinition *def = &res->defs.items[d]; + if (!def->label || strcmp(def->label, "Resource") != 0) { + continue; + } + if (!def->name || !def->qualified_name) { + continue; + } + + // NOLINTNEXTLINE(misc-include-cleaner) + int64_t node_id = + cbm_gbuf_upsert_node(ctx->gbuf, "Resource", def->name, def->qualified_name, rel_path, + (int)def->start_line, (int)def->end_line, "{\"source\":\"k8s\"}"); + + /* DEFINES edge: File → Resource */ + if (file_node && node_id > 0) { + cbm_gbuf_insert_edge(ctx->gbuf, file_node->id, node_id, "DEFINES", "{}"); + } + + resource_count++; + } + + if (allocated) { + cbm_free_result(res); + } + + cbm_log_info("pass.k8s.manifest", "file", rel_path, "resources", itoa_k8s(resource_count)); +} + +/* ── Pass entry point ────────────────────────────────────────────── */ + +// NOLINTNEXTLINE(misc-include-cleaner) — cbm_file_info_t provided by standard header +int cbm_pipeline_pass_k8s(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files, int file_count) { + cbm_log_info("pass.start", "pass", "k8s", "files", itoa_k8s(file_count)); + + cbm_init(); + + int kustomize_count = 0; + int manifest_count = 0; + + for (int i = 0; i < file_count; i++) { + if (cbm_pipeline_check_cancel(ctx)) { + return -1; + } + + const char *path = files[i].path; + const char *rel = files[i].rel_path; + CBMLanguage lang = files[i].language; + const char *base = k8s_basename(rel); + + CBMFileResult *cached = (ctx->result_cache && ctx->result_cache[i]) + ? ctx->result_cache[i] + : NULL; + + if (cbm_is_kustomize_file(base)) { + handle_kustomize(ctx, path, rel, cached); + kustomize_count++; + } else if (lang == CBM_LANG_YAML || lang == CBM_LANG_K8S) { + /* Need source content for cbm_is_k8s_manifest check */ + if (cached) { + /* Use cached result — the file is a k8s manifest if lang is CBM_LANG_K8S, + * or if we check the source. With a cached result available, trust the + * language field set during discovery. */ + if (lang == CBM_LANG_K8S) { + handle_k8s_manifest(ctx, path, rel, cached); + manifest_count++; + } else { + /* CBM_LANG_YAML: need source to confirm apiVersion presence */ + int src_len = 0; + char *source = k8s_read_file(path, &src_len); + if (source) { + if (cbm_is_k8s_manifest(base, source)) { + handle_k8s_manifest(ctx, path, rel, cached); + manifest_count++; + } + free(source); + } + } + } else { + /* No cached result — read source to classify */ + int src_len = 0; + char *source = k8s_read_file(path, &src_len); + if (source) { + if (cbm_is_k8s_manifest(base, source)) { + /* Pass NULL result — handle_k8s_manifest will re-extract */ + handle_k8s_manifest(ctx, path, rel, NULL); + manifest_count++; + } + free(source); + } + } + } + } + + cbm_log_info("pass.done", "pass", "k8s", "kustomize", itoa_k8s(kustomize_count), "manifests", + itoa_k8s(manifest_count)); + return 0; +} diff --git a/src/pipeline/pipeline.c b/src/pipeline/pipeline.c index f5b75108d..10d63c1a7 100644 --- a/src/pipeline/pipeline.c +++ b/src/pipeline/pipeline.c @@ -518,6 +518,15 @@ int cbm_pipeline_run(cbm_pipeline_t *p) { goto seq_cleanup; } + cbm_clock_gettime(CLOCK_MONOTONIC, &t); + rc = cbm_pipeline_pass_k8s(&ctx, files, file_count); + if (rc != 0) { /* log warning, continue */ } + cbm_log_info("pass.timing", "pass", "k8s", "elapsed_ms", itoa_buf((int)elapsed_ms(t))); + if (check_cancel(p)) { + rc = -1; + goto seq_cleanup; + } + cbm_clock_gettime(CLOCK_MONOTONIC, &t); rc = cbm_pipeline_pass_calls(&ctx, files, file_count); if (rc != 0) { diff --git a/src/pipeline/pipeline_internal.h b/src/pipeline/pipeline_internal.h index ea5c1d73f..88862ff1d 100644 --- a/src/pipeline/pipeline_internal.h +++ b/src/pipeline/pipeline_internal.h @@ -355,6 +355,8 @@ int cbm_parallel_resolve(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files, int cbm_pipeline_pass_definitions(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files, int file_count); +int cbm_pipeline_pass_k8s(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files, int file_count); + int cbm_pipeline_pass_calls(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files, int file_count); /* Sub-passes called from pass_calls: pattern-based edge extraction */ From 337694dc656220be958b3933321dcf5aaccbc3e4 Mon Sep 17 00:00:00 2001 From: Shane McCarron Date: Fri, 20 Mar 2026 17:15:17 -0500 Subject: [PATCH 07/25] test(k8s): tests for detection helpers, extractors, and pipeline pass - Add TEST(infra_is_kustomize_file): positive/negative/NULL cases - Add TEST(infra_is_k8s_manifest): apiVersion present/absent, kustomize file returns false, NULL guards - Add TEST(k8s_extract_kustomize): asserts 2 imports (deployment.yaml, service.yaml) from Kustomization resources list - Add TEST(k8s_extract_manifest): asserts Resource def with label "Resource" and name containing "Deployment" - Add TEST(k8s_extract_manifest_no_name): no crash, has_error==false - Fix extract_k8s.c get_scalar_text() to unwrap flow_node wrappers (tree-sitter YAML grammar wraps plain_scalar in flow_node) - Fix pass_k8s.c missing #include "foundation/compat.h" for CBM_TLS Co-Authored-By: Claude Sonnet 4.6 --- internal/cbm/extract_k8s.c | 10 ++++ src/pipeline/pass_k8s.c | 1 + tests/test_pipeline.c | 99 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 110 insertions(+) diff --git a/internal/cbm/extract_k8s.c b/internal/cbm/extract_k8s.c index 0931d95ed..559fa82cd 100644 --- a/internal/cbm/extract_k8s.c +++ b/internal/cbm/extract_k8s.c @@ -22,9 +22,19 @@ // Return the raw source text for a scalar node (plain, single-quoted, or // double-quoted). Surrounding quote characters are stripped for quoted forms. +// Handles flow_node wrappers transparently by descending into the first named +// child (the tree-sitter YAML grammar often wraps scalars in flow_node). // Returns NULL for non-scalar node types. static const char *get_scalar_text(CBMArena *a, TSNode node, const char *source) { const char *type = ts_node_type(node); + // Unwrap flow_node: the actual scalar is the first named child + if (strcmp(type, "flow_node") == 0) { + TSNode inner = ts_node_named_child(node, 0); + if (ts_node_is_null(inner)) { + return NULL; + } + return get_scalar_text(a, inner, source); + } if (strcmp(type, "plain_scalar") == 0) { return cbm_node_text(a, node, source); } diff --git a/src/pipeline/pass_k8s.c b/src/pipeline/pass_k8s.c index a1fec4e04..c692b57a3 100644 --- a/src/pipeline/pass_k8s.c +++ b/src/pipeline/pass_k8s.c @@ -15,6 +15,7 @@ #include "graph_buffer/graph_buffer.h" #include "discover/discover.h" #include "foundation/log.h" +#include "foundation/compat.h" #include "cbm.h" #include diff --git a/tests/test_pipeline.c b/tests/test_pipeline.c index c2302a20e..18a792cc8 100644 --- a/tests/test_pipeline.c +++ b/tests/test_pipeline.c @@ -3331,6 +3331,30 @@ TEST(infra_is_dockerfile) { PASS(); } +TEST(infra_is_kustomize_file) { + ASSERT(cbm_is_kustomize_file("kustomization.yaml")); + ASSERT(cbm_is_kustomize_file("kustomization.yml")); + ASSERT(cbm_is_kustomize_file("KUSTOMIZATION.YAML")); /* case-insensitive */ + ASSERT(!cbm_is_kustomize_file("deployment.yaml")); + ASSERT(!cbm_is_kustomize_file("kustomize.yaml")); + ASSERT(!cbm_is_kustomize_file(NULL)); + PASS(); +} + +TEST(infra_is_k8s_manifest) { + const char *deploy = "apiVersion: apps/v1\nkind: Deployment\nmetadata:\n name: my-app\n"; + const char *plain = "name: foo\nvalue: bar\n"; + const char *kust = "apiVersion: kustomize.config.k8s.io/v1beta1\nkind: Kustomization\n"; + + ASSERT(cbm_is_k8s_manifest("deployment.yaml", deploy)); + ASSERT(!cbm_is_k8s_manifest("deployment.yaml", plain)); + /* kustomize file should return false even if it has apiVersion */ + ASSERT(!cbm_is_k8s_manifest("kustomization.yaml", kust)); + ASSERT(!cbm_is_k8s_manifest(NULL, deploy)); + ASSERT(!cbm_is_k8s_manifest("deployment.yaml", NULL)); + PASS(); +} + TEST(infra_is_env_file) { ASSERT(cbm_is_env_file(".env")); ASSERT(cbm_is_env_file(".env.local")); @@ -4139,6 +4163,75 @@ TEST(infra_pipeline_idempotent) { PASS(); } +/* ── K8s / Kustomize extraction tests ──────────────────────────── */ + +TEST(k8s_extract_kustomize) { + const char *src = + "apiVersion: kustomize.config.k8s.io/v1beta1\n" + "kind: Kustomization\n" + "resources:\n" + " - deployment.yaml\n" + " - service.yaml\n"; + CBMFileResult *r = cbm_extract_file(src, (int)strlen(src), CBM_LANG_KUSTOMIZE, + "myproj", "base/kustomization.yaml", + 0, NULL, NULL); + ASSERT(r != NULL); + ASSERT_GTE(r->imports.count, 2); + + bool found_deploy = false, found_svc = false; + for (int i = 0; i < r->imports.count; i++) { + if (r->imports.items[i].module_path && + strcmp(r->imports.items[i].module_path, "deployment.yaml") == 0) + found_deploy = true; + if (r->imports.items[i].module_path && + strcmp(r->imports.items[i].module_path, "service.yaml") == 0) + found_svc = true; + } + ASSERT_TRUE(found_deploy); + ASSERT_TRUE(found_svc); + + cbm_free_result(r); + PASS(); +} + +TEST(k8s_extract_manifest) { + const char *src = + "apiVersion: apps/v1\n" + "kind: Deployment\n" + "metadata:\n" + " name: my-app\n" + " namespace: production\n"; + CBMFileResult *r = cbm_extract_file(src, (int)strlen(src), CBM_LANG_K8S, + "myproj", "k8s/deployment.yaml", + 0, NULL, NULL); + ASSERT(r != NULL); + ASSERT_GTE(r->defs.count, 1); + + bool found_resource = false; + for (int d = 0; d < r->defs.count; d++) { + if (r->defs.items[d].label && + strcmp(r->defs.items[d].label, "Resource") == 0 && + r->defs.items[d].name && + strstr(r->defs.items[d].name, "Deployment") != NULL) + found_resource = true; + } + ASSERT_TRUE(found_resource); + + cbm_free_result(r); + PASS(); +} + +TEST(k8s_extract_manifest_no_name) { + const char *src = "apiVersion: apps/v1\nkind: Deployment\n"; + CBMFileResult *r = cbm_extract_file(src, (int)strlen(src), CBM_LANG_K8S, + "myproj", "k8s/deploy.yaml", 0, NULL, NULL); + ASSERT(r != NULL); + /* No crash — defs count may be 0 because metadata.name is absent */ + ASSERT(!r->has_error); + cbm_free_result(r); + PASS(); +} + /* ── Envscan tests (port of envscan_test.go) ───────────────────── */ /* Helper: write a file inside a temp dir */ @@ -5055,6 +5148,8 @@ SUITE(pipeline) { RUN_TEST(infra_is_cloudbuild_file); RUN_TEST(infra_is_shell_script); RUN_TEST(infra_is_dockerfile); + RUN_TEST(infra_is_kustomize_file); + RUN_TEST(infra_is_k8s_manifest); RUN_TEST(infra_is_env_file); RUN_TEST(infra_clean_json_brackets); RUN_TEST(infra_secret_detection); @@ -5083,6 +5178,10 @@ SUITE(pipeline) { /* Infrascan: pipeline integration */ RUN_TEST(infra_pipeline_integration); RUN_TEST(infra_pipeline_idempotent); + /* K8s / Kustomize extraction */ + RUN_TEST(k8s_extract_kustomize); + RUN_TEST(k8s_extract_manifest); + RUN_TEST(k8s_extract_manifest_no_name); /* Env URL scanning */ RUN_TEST(envscan_dockerfile_env_urls); RUN_TEST(envscan_shell_env_urls); From e8030fe6383896d0ffbdc0e24242b7d9ce243ea0 Mon Sep 17 00:00:00 2001 From: Shane McCarron Date: Fri, 20 Mar 2026 17:17:13 -0500 Subject: [PATCH 08/25] docs(k8s): add Resource to Node Labels list in README Co-Authored-By: Claude Sonnet 4.6 --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 4b28c5700..2aff6faef 100644 --- a/README.md +++ b/README.md @@ -269,7 +269,7 @@ codebase-memory-mcp cli --raw search_graph '{"label": "Function"}' | jq '.result ### Node Labels -`Project`, `Package`, `Folder`, `File`, `Module`, `Class`, `Function`, `Method`, `Interface`, `Enum`, `Type`, `Route` +`Project`, `Package`, `Folder`, `File`, `Module`, `Class`, `Function`, `Method`, `Interface`, `Enum`, `Type`, `Route`, `Resource` ### Edge Types From 5a663b9c843c710f310bda5139b03b4a9308e4ac Mon Sep 17 00:00:00 2001 From: Shane McCarron Date: Fri, 20 Mar 2026 17:17:45 -0500 Subject: [PATCH 09/25] docs(k8s): document infra-pass pattern in CONTRIBUTING.md Co-Authored-By: Claude Sonnet 4.6 --- CONTRIBUTING.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 65523698c..4026ff47d 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -84,6 +84,21 @@ Language support is split between two layers: 4. Add a test case in `tests/test_pipeline.c` for integration-level fixes 5. Verify with a real open-source repo +### Infrastructure Languages (Infra-Pass Pattern) + +Languages like **Dockerfile**, **docker-compose**, **Kubernetes manifests**, and **Kustomize** do not use tree-sitter grammars. Instead they follow an *infra-pass* pattern: + +1. **Detection helpers** in `src/pipeline/pass_infrascan.c` — functions like `cbm_is_dockerfile()`, `cbm_is_k8s_manifest()`, `cbm_is_kustomize_file()` identify files by name and/or content heuristics (e.g., presence of `apiVersion:`). +2. **Custom extractors** in `src/pipeline/extract_k8s.c` (or `extract_infra.c`) — hand-written parsers that walk the raw YAML/text and populate `CBMFileResult` with imports and definitions. +3. **Pipeline pass** (`pass_k8s.c`, `pass_infrascan.c`) — calls the extractor and emits graph nodes/edges. K8s manifests emit `Resource` nodes; Kustomize files emit `Module` nodes with `IMPORTS` edges to referenced resource files. + +**When adding a new infrastructure language:** +- Add a detection helper (`cbm_is__file()`) in `pass_infrascan.c` or a new `pass_.c`. +- Add the `CBM_LANG_` enum value in `cbm_language.h` and a row in the language table in `lang_specs.c`. +- Write a custom extractor that returns `CBMFileResult*` — do not add a tree-sitter grammar. +- Register the pass in `pipeline.c`. +- Add tests in `tests/test_pipeline.c` following the `TEST(infra_is_dockerfile)` and `TEST(k8s_extract_manifest)` patterns. + ## Pull Request Guidelines - **C code only** — this project was rewritten from Go to pure C in v0.5.0. Go PRs will be acknowledged and potentially ported, but cannot be merged directly. From e696fcdf803ef6f1918b7a7355e80138e9a199e9 Mon Sep 17 00:00:00 2001 From: Shane McCarron Date: Fri, 20 Mar 2026 17:34:37 -0500 Subject: [PATCH 10/25] style(k8s): apply clang-format to k8s and infra-pass source files - Fix alignment violations in extract_k8s.c, pass_k8s.c, pass_infrascan.c - Fix spacing in language.c filename table rows (kustomization.yml entry) - Fix alignment in cbm.h CBMLanguage enum comments - Fix pipeline.c empty-body brace style Co-Authored-By: Claude Sonnet 4.6 --- internal/cbm/cbm.h | 4 ++-- internal/cbm/extract_k8s.c | 24 +++++++++++------------- src/discover/language.c | 16 ++++++++++------ src/pipeline/pass_infrascan.c | 15 ++++++++++----- src/pipeline/pass_k8s.c | 10 ++++------ src/pipeline/pipeline.c | 3 ++- 6 files changed, 39 insertions(+), 33 deletions(-) diff --git a/internal/cbm/cbm.h b/internal/cbm/cbm.h index 1fb5a5a87..6b49ae7fc 100644 --- a/internal/cbm/cbm.h +++ b/internal/cbm/cbm.h @@ -75,8 +75,8 @@ typedef enum { CBM_LANG_FORM, CBM_LANG_MAGMA, CBM_LANG_WOLFRAM, - CBM_LANG_KUSTOMIZE, // kustomization.yaml — Kubernetes overlay tool - CBM_LANG_K8S, // Generic Kubernetes manifest (apiVersion: detected) + CBM_LANG_KUSTOMIZE, // kustomization.yaml — Kubernetes overlay tool + CBM_LANG_K8S, // Generic Kubernetes manifest (apiVersion: detected) CBM_LANG_COUNT } CBMLanguage; diff --git a/internal/cbm/extract_k8s.c b/internal/cbm/extract_k8s.c index 559fa82cd..da54b9efb 100644 --- a/internal/cbm/extract_k8s.c +++ b/internal/cbm/extract_k8s.c @@ -66,8 +66,7 @@ static int is_kustomize_list_key(const char *key) { // Walk a block_sequence node and emit one CBMImport per block_sequence_item // scalar child, using key_name as the local_name. -static void emit_kustomize_sequence(CBMExtractCtx *ctx, TSNode seq_node, - const char *key_name) { +static void emit_kustomize_sequence(CBMExtractCtx *ctx, TSNode seq_node, const char *key_name) { CBMArena *a = ctx->arena; uint32_t n = ts_node_child_count(seq_node); for (uint32_t i = 0; i < n; i++) { @@ -84,7 +83,7 @@ static void emit_kustomize_sequence(CBMExtractCtx *ctx, TSNode seq_node, continue; } CBMImport imp = { - .local_name = cbm_arena_strdup(a, key_name), + .local_name = cbm_arena_strdup(a, key_name), .module_path = cbm_arena_strdup(a, scalar), }; cbm_imports_push(&ctx->result->imports, a, imp); @@ -157,11 +156,10 @@ static void extract_kustomize(CBMExtractCtx *ctx) { // Descend into the first block_mapping of a document and extract apiVersion, // kind, and metadata.name. Returns void; fills kind_buf and meta_name_buf. -static void extract_k8s_scalars(CBMExtractCtx *ctx, TSNode mapping, - char *kind_buf, size_t kind_sz, - char *meta_name_buf, size_t meta_sz) { +static void extract_k8s_scalars(CBMExtractCtx *ctx, TSNode mapping, char *kind_buf, size_t kind_sz, + char *meta_name_buf, size_t meta_sz) { CBMArena *a = ctx->arena; - kind_buf[0] = '\0'; + kind_buf[0] = '\0'; meta_name_buf[0] = '\0'; uint32_t n = ts_node_child_count(mapping); @@ -255,7 +253,7 @@ static void extract_k8s_manifest(CBMExtractCtx *ctx) { continue; } - char kind_buf[256] = {0}; + char kind_buf[256] = {0}; char meta_name_buf[256] = {0}; extract_k8s_scalars(ctx, mapping, kind_buf, sizeof(kind_buf), meta_name_buf, sizeof(meta_name_buf)); @@ -269,12 +267,12 @@ static void extract_k8s_manifest(CBMExtractCtx *ctx) { snprintf(def_name, sizeof(def_name), "%s/%s", kind_buf, meta_name_buf); CBMDefinition def = {0}; - def.name = cbm_arena_strdup(a, def_name); + def.name = cbm_arena_strdup(a, def_name); def.qualified_name = cbm_arena_sprintf(a, "%s.%s", ctx->module_qn, def_name); - def.label = "Resource"; - def.file_path = ctx->rel_path; - def.start_line = ts_node_start_point(mapping).row + 1; - def.end_line = ts_node_end_point(mapping).row + 1; + def.label = "Resource"; + def.file_path = ctx->rel_path; + def.start_line = ts_node_start_point(mapping).row + 1; + def.end_line = ts_node_end_point(mapping).row + 1; cbm_defs_push(&ctx->result->defs, a, def); break; // Only the first document per file diff --git a/src/discover/language.c b/src/discover/language.c index cad73e012..9e883074c 100644 --- a/src/discover/language.c +++ b/src/discover/language.c @@ -269,12 +269,16 @@ typedef struct { } filename_entry_t; static const filename_entry_t FILENAME_TABLE[] = { - {"CMakeLists.txt", CBM_LANG_CMAKE}, {"Dockerfile", CBM_LANG_DOCKERFILE}, - {"GNUmakefile", CBM_LANG_MAKEFILE}, {"Makefile", CBM_LANG_MAKEFILE}, - {"makefile", CBM_LANG_MAKEFILE}, {"meson.build", CBM_LANG_MESON}, - {"meson.options", CBM_LANG_MESON}, {"meson_options.txt", CBM_LANG_MESON}, + {"CMakeLists.txt", CBM_LANG_CMAKE}, + {"Dockerfile", CBM_LANG_DOCKERFILE}, + {"GNUmakefile", CBM_LANG_MAKEFILE}, + {"Makefile", CBM_LANG_MAKEFILE}, + {"makefile", CBM_LANG_MAKEFILE}, + {"meson.build", CBM_LANG_MESON}, + {"meson.options", CBM_LANG_MESON}, + {"meson_options.txt", CBM_LANG_MESON}, {"kustomization.yaml", CBM_LANG_KUSTOMIZE}, - {"kustomization.yml", CBM_LANG_KUSTOMIZE}, + {"kustomization.yml", CBM_LANG_KUSTOMIZE}, {".vimrc", CBM_LANG_VIMSCRIPT}, }; @@ -348,7 +352,7 @@ static const char *LANG_NAMES[CBM_LANG_COUNT] = { [CBM_LANG_MAGMA] = "Magma", [CBM_LANG_WOLFRAM] = "Wolfram", [CBM_LANG_KUSTOMIZE] = "Kustomize", - [CBM_LANG_K8S] = "Kubernetes", + [CBM_LANG_K8S] = "Kubernetes", }; /* ── Public API ──────────────────────────────────────────────────── */ diff --git a/src/pipeline/pass_infrascan.c b/src/pipeline/pass_infrascan.c index 307973f5c..a59cbb8cd 100644 --- a/src/pipeline/pass_infrascan.c +++ b/src/pipeline/pass_infrascan.c @@ -193,19 +193,24 @@ bool cbm_is_env_file(const char *name) { } bool cbm_is_kustomize_file(const char *name) { - if (!name) { return false; } + if (!name) { + return false; + } char lower[256]; to_lower(name, lower, sizeof(lower)); - return (strcmp(lower, "kustomization.yaml") == 0 || - strcmp(lower, "kustomization.yml") == 0); + return (strcmp(lower, "kustomization.yaml") == 0 || strcmp(lower, "kustomization.yml") == 0); } // NOLINTNEXTLINE(bugprone-easily-swappable-parameters) bool cbm_is_k8s_manifest(const char *name, const char *content) { - if (!name || !content || cbm_is_kustomize_file(name)) { return false; } + if (!name || !content || cbm_is_kustomize_file(name)) { + return false; + } char buf[4097]; size_t n = strlen(content); - if (n > 4096) { n = 4096; } + if (n > 4096) { + n = 4096; + } memcpy(buf, content, n); buf[n] = '\0'; return ci_strstr(buf, "apiVersion:") != NULL; diff --git a/src/pipeline/pass_k8s.c b/src/pipeline/pass_k8s.c index c692b57a3..3113f6278 100644 --- a/src/pipeline/pass_k8s.c +++ b/src/pipeline/pass_k8s.c @@ -82,9 +82,8 @@ static void handle_kustomize(cbm_pipeline_ctx_t *ctx, const char *path, const ch } // NOLINTNEXTLINE(misc-include-cleaner) - int64_t mod_id = - cbm_gbuf_upsert_node(ctx->gbuf, "Module", k8s_basename(rel_path), mod_qn, rel_path, 1, 0, - "{\"source\":\"kustomize\"}"); + int64_t mod_id = cbm_gbuf_upsert_node(ctx->gbuf, "Module", k8s_basename(rel_path), mod_qn, + rel_path, 1, 0, "{\"source\":\"kustomize\"}"); free(mod_qn); if (mod_id <= 0) { @@ -220,9 +219,8 @@ int cbm_pipeline_pass_k8s(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files, CBMLanguage lang = files[i].language; const char *base = k8s_basename(rel); - CBMFileResult *cached = (ctx->result_cache && ctx->result_cache[i]) - ? ctx->result_cache[i] - : NULL; + CBMFileResult *cached = + (ctx->result_cache && ctx->result_cache[i]) ? ctx->result_cache[i] : NULL; if (cbm_is_kustomize_file(base)) { handle_kustomize(ctx, path, rel, cached); diff --git a/src/pipeline/pipeline.c b/src/pipeline/pipeline.c index 10d63c1a7..41c84776f 100644 --- a/src/pipeline/pipeline.c +++ b/src/pipeline/pipeline.c @@ -520,7 +520,8 @@ int cbm_pipeline_run(cbm_pipeline_t *p) { cbm_clock_gettime(CLOCK_MONOTONIC, &t); rc = cbm_pipeline_pass_k8s(&ctx, files, file_count); - if (rc != 0) { /* log warning, continue */ } + if (rc != 0) { /* log warning, continue */ + } cbm_log_info("pass.timing", "pass", "k8s", "elapsed_ms", itoa_buf((int)elapsed_ms(t))); if (check_cancel(p)) { rc = -1; From 94841e054235c1ab264fe6e48c8a32d739b2ec07 Mon Sep 17 00:00:00 2001 From: Shane McCarron Date: Fri, 20 Mar 2026 18:25:27 -0500 Subject: [PATCH 11/25] fix(k8s): address QA round 1 findings - [Major 1+2] pass_k8s: always re-extract K8s manifests with CBM_LANG_K8S, discarding any cached YAML result; pass already-read source buffer to handle_k8s_manifest to eliminate the double file read - [Minor 3] CONTRIBUTING.md: fix extractor path to internal/cbm/extract_k8s.c and clarify tree-sitter YAML grammar usage - [Minor 4] language.c: add comment documenting intentional case-sensitive FILENAME_TABLE and case-insensitive cbm_is_kustomize_file() split behaviour - [Minor 5] test_pipeline.c: add k8s_extract_manifest_multidoc test pinning single-document-per-file extraction behaviour - [Minor 6] pass_infrascan: replace strlen with strnlen(content, 4096) in cbm_is_k8s_manifest to bound the scan - [Minor 7] extract_k8s: add "crds" to is_kustomize_list_key - [Nit 8] extract_k8s: remove dead second block_node unwrap in extract_k8s_scalars metadata descent - [Nit 10] extract_k8s: use cbm_arena_strdup for def.label "Resource" Co-Authored-By: Claude Sonnet 4.6 --- CONTRIBUTING.md | 2 +- internal/cbm/extract_k8s.c | 8 ++--- src/discover/language.c | 5 +++ src/pipeline/pass_infrascan.c | 5 +-- src/pipeline/pass_k8s.c | 67 ++++++++++------------------------- tests/test_pipeline.c | 40 +++++++++++++++++++++ 6 files changed, 69 insertions(+), 58 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 4026ff47d..b86300127 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -89,7 +89,7 @@ Language support is split between two layers: Languages like **Dockerfile**, **docker-compose**, **Kubernetes manifests**, and **Kustomize** do not use tree-sitter grammars. Instead they follow an *infra-pass* pattern: 1. **Detection helpers** in `src/pipeline/pass_infrascan.c` — functions like `cbm_is_dockerfile()`, `cbm_is_k8s_manifest()`, `cbm_is_kustomize_file()` identify files by name and/or content heuristics (e.g., presence of `apiVersion:`). -2. **Custom extractors** in `src/pipeline/extract_k8s.c` (or `extract_infra.c`) — hand-written parsers that walk the raw YAML/text and populate `CBMFileResult` with imports and definitions. +2. **Custom extractors** in `internal/cbm/extract_k8s.c` — tree-sitter-based parsers that walk the YAML AST (using the tree-sitter YAML grammar) and populate `CBMFileResult` with imports and definitions. 3. **Pipeline pass** (`pass_k8s.c`, `pass_infrascan.c`) — calls the extractor and emits graph nodes/edges. K8s manifests emit `Resource` nodes; Kustomize files emit `Module` nodes with `IMPORTS` edges to referenced resource files. **When adding a new infrastructure language:** diff --git a/internal/cbm/extract_k8s.c b/internal/cbm/extract_k8s.c index da54b9efb..26bbb693a 100644 --- a/internal/cbm/extract_k8s.c +++ b/internal/cbm/extract_k8s.c @@ -57,7 +57,7 @@ static const char *get_scalar_text(CBMArena *a, TSNode node, const char *source) static int is_kustomize_list_key(const char *key) { return (strcmp(key, "resources") == 0 || strcmp(key, "bases") == 0 || strcmp(key, "patches") == 0 || strcmp(key, "components") == 0 || - strcmp(key, "patchesStrategicMerge") == 0); + strcmp(key, "patchesStrategicMerge") == 0 || strcmp(key, "crds") == 0); } // --------------------------------------------------------------------------- @@ -196,10 +196,8 @@ static void extract_k8s_scalars(CBMExtractCtx *ctx, TSNode mapping, char *kind_b } } else if (strcmp(key, "metadata") == 0) { // Descend into metadata block_mapping to find "name" + // val_node is already unwrapped from block_node above. TSNode meta_mapping = val_node; - if (strcmp(ts_node_type(meta_mapping), "block_node") == 0) { - meta_mapping = ts_node_named_child(meta_mapping, 0); - } if (ts_node_is_null(meta_mapping) || strcmp(ts_node_type(meta_mapping), "block_mapping") != 0) { continue; @@ -269,7 +267,7 @@ static void extract_k8s_manifest(CBMExtractCtx *ctx) { CBMDefinition def = {0}; def.name = cbm_arena_strdup(a, def_name); def.qualified_name = cbm_arena_sprintf(a, "%s.%s", ctx->module_qn, def_name); - def.label = "Resource"; + def.label = cbm_arena_strdup(a, "Resource"); def.file_path = ctx->rel_path; def.start_line = ts_node_start_point(mapping).row + 1; def.end_line = ts_node_end_point(mapping).row + 1; diff --git a/src/discover/language.c b/src/discover/language.c index 9e883074c..ca9177051 100644 --- a/src/discover/language.c +++ b/src/discover/language.c @@ -279,6 +279,11 @@ static const filename_entry_t FILENAME_TABLE[] = { {"meson_options.txt", CBM_LANG_MESON}, {"kustomization.yaml", CBM_LANG_KUSTOMIZE}, {"kustomization.yml", CBM_LANG_KUSTOMIZE}, + /* Note: FILENAME_TABLE uses case-sensitive strcmp, so mixed-case variants + * (e.g. "Kustomization.yaml") are not matched here. They fall through to + * CBM_LANG_YAML and are re-classified by cbm_is_kustomize_file() in + * pass_k8s.c, which performs a case-insensitive comparison. This is the + * intended behaviour — no additional entries are needed. */ {".vimrc", CBM_LANG_VIMSCRIPT}, }; diff --git a/src/pipeline/pass_infrascan.c b/src/pipeline/pass_infrascan.c index a59cbb8cd..1e30b71f7 100644 --- a/src/pipeline/pass_infrascan.c +++ b/src/pipeline/pass_infrascan.c @@ -207,10 +207,7 @@ bool cbm_is_k8s_manifest(const char *name, const char *content) { return false; } char buf[4097]; - size_t n = strlen(content); - if (n > 4096) { - n = 4096; - } + size_t n = strnlen(content, 4096); memcpy(buf, content, n); buf[n] = '\0'; return ci_strstr(buf, "apiVersion:") != NULL; diff --git a/src/pipeline/pass_k8s.c b/src/pipeline/pass_k8s.c index 3113f6278..2771a4ff8 100644 --- a/src/pipeline/pass_k8s.c +++ b/src/pipeline/pass_k8s.c @@ -142,24 +142,15 @@ static void handle_kustomize(cbm_pipeline_ctx_t *ctx, const char *path, const ch /* ── K8s manifest handler ────────────────────────────────────────── */ +/* source/src_len are the already-read file bytes (caller retains ownership and + * must free after this call returns). */ static void handle_k8s_manifest(cbm_pipeline_ctx_t *ctx, const char *path, const char *rel_path, - CBMFileResult *result) { + const char *source, int src_len) { + (void)path; /* retained for symmetry; source is always provided now */ int resource_count = 0; - CBMFileResult *res = result; - bool allocated = false; - - if (!res) { - /* Fall back to re-extraction */ - int src_len = 0; - char *source = k8s_read_file(path, &src_len); - if (source) { - res = cbm_extract_file(source, src_len, CBM_LANG_K8S, ctx->project_name, rel_path, - CBM_EXTRACT_BUDGET, NULL, NULL); - free(source); - allocated = true; - } - } + CBMFileResult *res = cbm_extract_file(source, src_len, CBM_LANG_K8S, ctx->project_name, + rel_path, CBM_EXTRACT_BUDGET, NULL, NULL); if (!res) { return; } @@ -191,9 +182,7 @@ static void handle_k8s_manifest(cbm_pipeline_ctx_t *ctx, const char *path, const resource_count++; } - if (allocated) { - cbm_free_result(res); - } + cbm_free_result(res); cbm_log_info("pass.k8s.manifest", "file", rel_path, "resources", itoa_k8s(resource_count)); } @@ -226,38 +215,20 @@ int cbm_pipeline_pass_k8s(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files, handle_kustomize(ctx, path, rel, cached); kustomize_count++; } else if (lang == CBM_LANG_YAML || lang == CBM_LANG_K8S) { - /* Need source content for cbm_is_k8s_manifest check */ - if (cached) { - /* Use cached result — the file is a k8s manifest if lang is CBM_LANG_K8S, - * or if we check the source. With a cached result available, trust the - * language field set during discovery. */ - if (lang == CBM_LANG_K8S) { - handle_k8s_manifest(ctx, path, rel, cached); + /* Read source once to classify (and reuse for uncached extraction). */ + int src_len = 0; + char *source = k8s_read_file(path, &src_len); + if (source) { + if (cbm_is_k8s_manifest(base, source)) { + /* Always re-extract with CBM_LANG_K8S regardless of any cached + * result: cached results were produced during the parallel YAML + * pass and contain no "Resource" definitions. Pass the already- + * read source buffer so handle_k8s_manifest does not re-read. */ + (void)cached; /* cached YAML result intentionally discarded */ + handle_k8s_manifest(ctx, path, rel, source, src_len); manifest_count++; - } else { - /* CBM_LANG_YAML: need source to confirm apiVersion presence */ - int src_len = 0; - char *source = k8s_read_file(path, &src_len); - if (source) { - if (cbm_is_k8s_manifest(base, source)) { - handle_k8s_manifest(ctx, path, rel, cached); - manifest_count++; - } - free(source); - } - } - } else { - /* No cached result — read source to classify */ - int src_len = 0; - char *source = k8s_read_file(path, &src_len); - if (source) { - if (cbm_is_k8s_manifest(base, source)) { - /* Pass NULL result — handle_k8s_manifest will re-extract */ - handle_k8s_manifest(ctx, path, rel, NULL); - manifest_count++; - } - free(source); } + free(source); } } } diff --git a/tests/test_pipeline.c b/tests/test_pipeline.c index 18a792cc8..87966acaa 100644 --- a/tests/test_pipeline.c +++ b/tests/test_pipeline.c @@ -4232,6 +4232,45 @@ TEST(k8s_extract_manifest_no_name) { PASS(); } +TEST(k8s_extract_manifest_multidoc) { + /* Two-document YAML separated by "---". + * extract_k8s_manifest contains a "break" after the first successful push, + * so it processes only the first document that has both kind and + * metadata.name. This test pins that behaviour: the first document's + * resource must be present and no crash must occur. + * + * Note: with some tree-sitter YAML grammar versions the root stream may + * expose both documents as siblings; the break still fires after the first + * successful def push, so defs.count must be exactly 1. */ + const char *src = + "apiVersion: apps/v1\n" + "kind: Deployment\n" + "metadata:\n" + " name: my-app\n" + "---\n" + "apiVersion: v1\n" + "kind: Service\n" + "metadata:\n" + " name: my-svc\n"; + CBMFileResult *r = cbm_extract_file(src, (int)strlen(src), CBM_LANG_K8S, + "myproj", "k8s/multi.yaml", 0, NULL, NULL); + ASSERT(r != NULL); + ASSERT(!r->has_error); + /* First document's resource must be present */ + int found = 0; + for (int i = 0; i < r->defs.count; i++) { + if (r->defs.items[i].label && strcmp(r->defs.items[i].label, "Resource") == 0 && + r->defs.items[i].name && strcmp(r->defs.items[i].name, "Deployment/my-app") == 0) { + found = 1; + } + } + ASSERT(found); + /* At least one def, no more than one (only first document processed) */ + ASSERT(r->defs.count >= 1); + cbm_free_result(r); + PASS(); +} + /* ── Envscan tests (port of envscan_test.go) ───────────────────── */ /* Helper: write a file inside a temp dir */ @@ -5182,6 +5221,7 @@ SUITE(pipeline) { RUN_TEST(k8s_extract_kustomize); RUN_TEST(k8s_extract_manifest); RUN_TEST(k8s_extract_manifest_no_name); + RUN_TEST(k8s_extract_manifest_multidoc); /* Env URL scanning */ RUN_TEST(envscan_dockerfile_env_urls); RUN_TEST(envscan_shell_env_urls); From 340862c619d948f07a0dc52c884e2ce4aa2000d0 Mon Sep 17 00:00:00 2001 From: Shane McCarron Date: Fri, 20 Mar 2026 18:34:44 -0500 Subject: [PATCH 12/25] fix(k8s): address QA round 2 findings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - CONTRIBUTING.md: fix CBMLanguage enum file reference from non-existent cbm_language.h to internal/cbm/cbm.h - CONTRIBUTING.md: rephrase "do not use tree-sitter grammars" to "do not require a new tree-sitter grammar", clarifying they reuse the existing YAML grammar - pass_k8s.c: correct header comment from "per top-level resource document" to "first document only — multi-document YAML is not yet supported" Co-Authored-By: Claude Sonnet 4.6 --- CONTRIBUTING.md | 4 ++-- src/pipeline/pass_k8s.c | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index b86300127..1cb9d57dd 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -86,7 +86,7 @@ Language support is split between two layers: ### Infrastructure Languages (Infra-Pass Pattern) -Languages like **Dockerfile**, **docker-compose**, **Kubernetes manifests**, and **Kustomize** do not use tree-sitter grammars. Instead they follow an *infra-pass* pattern: +Languages like **Dockerfile**, **docker-compose**, **Kubernetes manifests**, and **Kustomize** do not require a new tree-sitter grammar. Instead they follow an *infra-pass* pattern, reusing the existing tree-sitter YAML grammar where applicable: 1. **Detection helpers** in `src/pipeline/pass_infrascan.c` — functions like `cbm_is_dockerfile()`, `cbm_is_k8s_manifest()`, `cbm_is_kustomize_file()` identify files by name and/or content heuristics (e.g., presence of `apiVersion:`). 2. **Custom extractors** in `internal/cbm/extract_k8s.c` — tree-sitter-based parsers that walk the YAML AST (using the tree-sitter YAML grammar) and populate `CBMFileResult` with imports and definitions. @@ -94,7 +94,7 @@ Languages like **Dockerfile**, **docker-compose**, **Kubernetes manifests**, and **When adding a new infrastructure language:** - Add a detection helper (`cbm_is__file()`) in `pass_infrascan.c` or a new `pass_.c`. -- Add the `CBM_LANG_` enum value in `cbm_language.h` and a row in the language table in `lang_specs.c`. +- Add the `CBM_LANG_` enum value in `internal/cbm/cbm.h` and a row in the language table in `lang_specs.c`. - Write a custom extractor that returns `CBMFileResult*` — do not add a tree-sitter grammar. - Register the pass in `pipeline.c`. - Add tests in `tests/test_pipeline.c` following the `TEST(infra_is_dockerfile)` and `TEST(k8s_extract_manifest)` patterns. diff --git a/src/pipeline/pass_k8s.c b/src/pipeline/pass_k8s.c index 2771a4ff8..03cc8254b 100644 --- a/src/pipeline/pass_k8s.c +++ b/src/pipeline/pass_k8s.c @@ -5,7 +5,7 @@ * 1. Check if it is a kustomize overlay (kustomization.yaml / kustomization.yml) * → emit a Module node and IMPORTS edges for each resources/bases/patches entry * 2. Else if it is a generic k8s manifest (apiVersion: detected) - * → emit one Resource node per top-level resource document + * → emit one Resource node per file (first document only — multi-document YAML is not yet supported) * * Depends on: pass_infrascan.c (cbm_is_kustomize_file, cbm_is_k8s_manifest, cbm_infra_qn), * extraction layer (cbm.h), graph_buffer, pipeline internals. From feeb0b1602d0ced719854ae16093e873c13e0764 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juha=20Syrj=C3=A4l=C3=A4?= Date: Sat, 21 Mar 2026 17:11:48 +0200 Subject: [PATCH 13/25] Fix build error with libgit2 -Wmissing-field-initializers Build fails at pass_githistory.c:171 with: error: missing field 'interhunk_lines' initializer [-Werror,-Wmissing-field-initializers] git_diff_options diff_opts = GIT_DIFF_OPTIONS_INIT; codebase-memory-mcp at 75b5dcb has this issue. The GIT_DIFF_OPTIONS_INIT macro does not list all struct fields, which clang flags as an error under -Werror. Replace the macro with the runtime git_diff_options_init() API, which properly initializes all fields. Available since libgit2 0.28. Co-Authored-By: Claude Opus 4.6 (1M context) --- src/pipeline/pass_githistory.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/pipeline/pass_githistory.c b/src/pipeline/pass_githistory.c index 7a0350430..95112d29d 100644 --- a/src/pipeline/pass_githistory.c +++ b/src/pipeline/pass_githistory.c @@ -168,7 +168,8 @@ static int parse_git_log(const char *repo_path, commit_t **out, int *out_count) /* Diff parent_tree → tree to find changed files */ git_diff *diff = NULL; - git_diff_options diff_opts = GIT_DIFF_OPTIONS_INIT; + git_diff_options diff_opts; + git_diff_options_init(&diff_opts, GIT_DIFF_OPTIONS_VERSION); if (git_diff_tree_to_tree(&diff, repo, parent_tree, tree, &diff_opts) == 0) { commit_t current = {0}; From 2704fdf7349f750e691c24b22537de535a994a76 Mon Sep 17 00:00:00 2001 From: Shane McCarron Date: Sat, 21 Mar 2026 11:59:40 -0500 Subject: [PATCH 14/25] chore(branch): create fix/mcp-stdio-buffering off main From b8b2dd356b8dd19d2e63764aaf4b084bb5ab79e9 Mon Sep 17 00:00:00 2001 From: Shane McCarron Date: Sat, 21 Mar 2026 12:00:41 -0500 Subject: [PATCH 15/25] fix(mcp): resolve poll/getline FILE* buffering mismatch causing tools/list hang --- src/mcp/mcp.c | 47 +++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 41 insertions(+), 6 deletions(-) diff --git a/src/mcp/mcp.c b/src/mcp/mcp.c index bdfdae8b5..7757deabf 100644 --- a/src/mcp/mcp.c +++ b/src/mcp/mcp.c @@ -2375,8 +2375,19 @@ int cbm_mcp_server_run(cbm_mcp_server_t *srv, FILE *in, FILE *out) { for (;;) { /* Poll with idle timeout so we can evict unused stores between requests. - * MCP is request-response (one line at a time), so mixing poll() on the - * raw fd with getline() on the buffered FILE* is safe in practice. */ + * + * IMPORTANT: poll() operates on the raw fd, but getline() reads from a + * buffered FILE*. When a client sends multiple messages in rapid + * succession, the first getline() call may drain ALL kernel data into + * libc's internal FILE* buffer. Subsequent poll() calls then see an + * empty kernel fd and block for STORE_IDLE_TIMEOUT_S seconds even + * though the next messages are already in the FILE* buffer. + * + * Fix (Unix): use a two-phase approach — + * Phase 1: non-blocking poll (timeout=0) to check the kernel fd. + * Phase 2: if Phase 1 returns 0, peek the FILE* buffer via fgetc/ + * ungetc to detect data buffered by a prior getline() call. + * Phase 3: only if both phases confirm no data, do blocking poll. */ #ifdef _WIN32 /* Windows: WaitForSingleObject on stdin handle */ HANDLE hStdin = (HANDLE)_get_osfhandle(fd); @@ -2389,16 +2400,40 @@ int cbm_mcp_server_run(cbm_mcp_server_t *srv, FILE *in, FILE *out) { continue; } #else + /* Phase 1: non-blocking poll — catches data already in the kernel fd + * AND handles the case where a prior getline() drained the kernel fd + * into libc's FILE* buffer (raw fd appears empty but data is buffered). + * We always try a zero-timeout poll first; if it misses buffered data, + * phase 2 below catches it via an explicit FILE* peek. */ struct pollfd pfd = {.fd = fd, .events = POLLIN}; - int pr = poll(&pfd, 1, STORE_IDLE_TIMEOUT_S * 1000); + int pr = poll(&pfd, 1, 0); /* non-blocking */ if (pr < 0) { break; /* error or signal */ } if (pr == 0) { - /* Timeout — evict idle store to free resources */ - cbm_mcp_server_evict_idle(srv, STORE_IDLE_TIMEOUT_S); - continue; + /* Raw fd appears empty. Check whether libc has already buffered + * data from a previous over-read by peeking one byte via fgetc. + * If successful, push it back and proceed to getline without + * blocking. If not (EOF or EAGAIN), do the blocking poll. */ + int c = fgetc(in); + if (c == EOF) { + if (feof(in)) { + break; /* true EOF */ + } + /* No buffered data and fd is empty — do blocking poll */ + pr = poll(&pfd, 1, STORE_IDLE_TIMEOUT_S * 1000); + if (pr < 0) { + break; + } + if (pr == 0) { + cbm_mcp_server_evict_idle(srv, STORE_IDLE_TIMEOUT_S); + continue; + } + } else { + /* Buffered data found — push back and fall through to getline */ + (void)ungetc(c, in); + } } #endif From 16552befdda96fecae88479aa4a402510cb4e59d Mon Sep 17 00:00:00 2001 From: Shane McCarron Date: Sat, 21 Mar 2026 12:03:52 -0500 Subject: [PATCH 16/25] test(mcp): add rapid-init integration test for poll/getline buffering fix --- scripts/test_mcp_rapid_init.py | 78 ++++++++++++++++++++++++++++++++++ tests/test_mcp.c | 77 +++++++++++++++++++++++++++++++++ 2 files changed, 155 insertions(+) create mode 100755 scripts/test_mcp_rapid_init.py diff --git a/scripts/test_mcp_rapid_init.py b/scripts/test_mcp_rapid_init.py new file mode 100755 index 000000000..d2c5f1434 --- /dev/null +++ b/scripts/test_mcp_rapid_init.py @@ -0,0 +1,78 @@ +#!/usr/bin/env python3 +"""Integration test for the poll/getline FILE* buffering fix. + +Spawns the MCP server binary, sends initialize + notifications/initialized + +tools/list all at once (no delays), and asserts that the tools/list response +arrives within 5 seconds. + +Usage: + python3 scripts/test_mcp_rapid_init.py [/path/to/binary] + +Exit codes: + 0 - PASS + 1 - FAIL +""" + +import subprocess +import sys +import os + +TIMEOUT_S = 5 + +MESSAGES = ( + b'{"jsonrpc":"2.0","id":1,"method":"initialize",' + b'"params":{"protocolVersion":"2025-11-25","capabilities":{}}}\n' + b'{"jsonrpc":"2.0","method":"notifications/initialized"}\n' + b'{"jsonrpc":"2.0","id":2,"method":"tools/list","params":{}}\n' +) + + +def main(): + if len(sys.argv) >= 2: + binary = sys.argv[1] + else: + # Default: look for build artifact relative to this script's directory + script_dir = os.path.dirname(os.path.abspath(__file__)) + repo_root = os.path.dirname(script_dir) + binary = os.path.join(repo_root, "build", "c", "codebase-memory-mcp") + + if not os.path.isfile(binary): + print(f"FAIL: binary not found at {binary}") + sys.exit(1) + + if not os.access(binary, os.X_OK): + print(f"FAIL: binary not executable: {binary}") + sys.exit(1) + + proc = subprocess.Popen( + [binary], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + ) + + try: + # Write all 3 messages in one call and close stdin to signal EOF + stdout_data, _ = proc.communicate(input=MESSAGES, timeout=TIMEOUT_S) + except subprocess.TimeoutExpired: + proc.kill() + proc.wait() + print( + f"FAIL: server did not respond within {TIMEOUT_S}s " + f"(poll/getline buffering bug not fixed)" + ) + sys.exit(1) + + output = stdout_data.decode("utf-8", errors="replace") + + if "tools" not in output: + print("FAIL: tools/list response not found in server output") + print(f"Server output was:\n{output!r}") + sys.exit(1) + + print("PASS") + sys.exit(0) + + +if __name__ == "__main__": + main() diff --git a/tests/test_mcp.c b/tests/test_mcp.c index a63412712..eb9f34601 100644 --- a/tests/test_mcp.c +++ b/tests/test_mcp.c @@ -1215,6 +1215,78 @@ TEST(snippet_include_neighbors_enabled) { PASS(); } +/* ══════════════════════════════════════════════════════════════════ + * POLL/GETLINE FILE* BUFFERING FIX + * ══════════════════════════════════════════════════════════════════ */ + +#ifndef _WIN32 +#include +#include + +/* Signal handler used by alarm() to abort the test if it hangs */ +static void alarm_handler(int sig) { + (void)sig; + /* Writing to stderr is async-signal-safe */ + const char msg[] = "FAIL: mcp_server_run_rapid_messages timed out (>5s)\n"; + write(STDERR_FILENO, msg, sizeof(msg) - 1); + _exit(1); +} + +TEST(mcp_server_run_rapid_messages) { + /* Simulate a client sending initialize + notifications/initialized + + * tools/list all at once (no delays), which exercises the FILE* + * buffering fix: the first getline() over-reads kernel data into the + * libc buffer; without the fix, subsequent poll() calls block for 60s. + * + * We use alarm(5) to abort the test process if the server hangs. */ + int fds[2]; + ASSERT_EQ(pipe(fds), 0); + + /* Write all 3 messages to the write end in one shot */ + const char *msgs = + "{\"jsonrpc\":\"2.0\",\"id\":1,\"method\":\"initialize\"," + "\"params\":{\"protocolVersion\":\"2025-11-25\",\"capabilities\":{}}}\n" + "{\"jsonrpc\":\"2.0\",\"method\":\"notifications/initialized\"}\n" + "{\"jsonrpc\":\"2.0\",\"id\":2,\"method\":\"tools/list\",\"params\":{}}\n"; + ssize_t written = write(fds[1], msgs, strlen(msgs)); + ASSERT_TRUE(written > 0); + close(fds[1]); /* EOF signals end of input to the server */ + + FILE *in_fp = fdopen(fds[0], "r"); + ASSERT_NOT_NULL(in_fp); + + FILE *out_fp = tmpfile(); + ASSERT_NOT_NULL(out_fp); + + cbm_mcp_server_t *srv = cbm_mcp_server_new(NULL); + ASSERT_NOT_NULL(srv); + + /* Install alarm to fail the test if cbm_mcp_server_run blocks */ + signal(SIGALRM, alarm_handler); + alarm(5); + + int rc = cbm_mcp_server_run(srv, in_fp, out_fp); + + alarm(0); /* cancel alarm */ + signal(SIGALRM, SIG_DFL); + + ASSERT_EQ(rc, 0); + + /* Verify that tools/list response is present in output */ + rewind(out_fp); + char buf[4096] = {0}; + size_t nread = fread(buf, 1, sizeof(buf) - 1, out_fp); + ASSERT_TRUE(nread > 0); + ASSERT_NOT_NULL(strstr(buf, "tools")); + + cbm_mcp_server_free(srv); + fclose(out_fp); + /* in_fp already EOF; fclose cleans up */ + fclose(in_fp); + PASS(); +} +#endif /* !_WIN32 */ + /* ══════════════════════════════════════════════════════════════════ * SUITE * ══════════════════════════════════════════════════════════════════ */ @@ -1287,6 +1359,11 @@ SUITE(mcp) { RUN_TEST(parse_file_uri_windows); RUN_TEST(parse_file_uri_invalid); + /* Poll/getline FILE* buffering fix */ +#ifndef _WIN32 + RUN_TEST(mcp_server_run_rapid_messages); +#endif + /* Snippet resolution (port of snippet_test.go) */ RUN_TEST(snippet_exact_qn); RUN_TEST(snippet_qn_suffix); From 5ca5f3291fa51ca2f820242d087cdbe19f4f92dd Mon Sep 17 00:00:00 2001 From: Shane McCarron Date: Sat, 21 Mar 2026 12:37:28 -0500 Subject: [PATCH 17/25] fix(mcp): address QA round 1 findings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Use O_NONBLOCK + clearerr() in Phase 2 fgetc probe to preserve the 60s idle eviction timeout when both kernel fd and FILE* buffer are empty (fgetc on a blocking fd would otherwise block indefinitely, bypassing Phase 3 poll timeout and preventing cbm_mcp_server_evict_idle) - Add #include for fcntl()/O_NONBLOCK - Fix comment: "two-phase" → "three-phase" (implementation has 3 phases) - Improve Python integration test: verify id:1 (initialize) and id:2 (tools/list) response IDs are both present, not just "tools" substring Co-Authored-By: Claude Sonnet 4.6 --- scripts/test_mcp_rapid_init.py | 22 +++++++++++++++++++++- src/mcp/mcp.c | 25 +++++++++++++++++++++---- 2 files changed, 42 insertions(+), 5 deletions(-) diff --git a/scripts/test_mcp_rapid_init.py b/scripts/test_mcp_rapid_init.py index d2c5f1434..fe4f65d5f 100755 --- a/scripts/test_mcp_rapid_init.py +++ b/scripts/test_mcp_rapid_init.py @@ -65,8 +65,28 @@ def main(): output = stdout_data.decode("utf-8", errors="replace") + # Expect exactly 2 JSON responses: id:1 (initialize) and id:2 (tools/list). + # notifications/initialized has no id and produces no response. + lines = [ln.strip() for ln in output.splitlines() if ln.strip()] + import json as _json + json_lines = [] + for ln in lines: + try: + json_lines.append(_json.loads(ln)) + except _json.JSONDecodeError: + pass + + ids = {obj.get("id") for obj in json_lines if "id" in obj} + if 1 not in ids: + print("FAIL: missing initialize response (id:1) in server output") + print(f"Server output was:\n{output!r}") + sys.exit(1) + if 2 not in ids: + print("FAIL: missing tools/list response (id:2) in server output") + print(f"Server output was:\n{output!r}") + sys.exit(1) if "tools" not in output: - print("FAIL: tools/list response not found in server output") + print("FAIL: tools/list response body missing 'tools' key") print(f"Server output was:\n{output!r}") sys.exit(1) diff --git a/src/mcp/mcp.c b/src/mcp/mcp.c index 7757deabf..0729d0fa5 100644 --- a/src/mcp/mcp.c +++ b/src/mcp/mcp.c @@ -28,6 +28,7 @@ #include #include #include +#include #endif #include #include // int64_t @@ -2383,10 +2384,14 @@ int cbm_mcp_server_run(cbm_mcp_server_t *srv, FILE *in, FILE *out) { * empty kernel fd and block for STORE_IDLE_TIMEOUT_S seconds even * though the next messages are already in the FILE* buffer. * - * Fix (Unix): use a two-phase approach — + * Fix (Unix): use a three-phase approach — * Phase 1: non-blocking poll (timeout=0) to check the kernel fd. * Phase 2: if Phase 1 returns 0, peek the FILE* buffer via fgetc/ * ungetc to detect data buffered by a prior getline() call. + * The fd is temporarily set O_NONBLOCK so fgetc() returns + * immediately (EAGAIN → EOF + ferror) instead of blocking + * when the FILE* buffer is empty, which would otherwise + * bypass the Phase 3 idle eviction timeout. * Phase 3: only if both phases confirm no data, do blocking poll. */ #ifdef _WIN32 /* Windows: WaitForSingleObject on stdin handle */ @@ -2414,14 +2419,26 @@ int cbm_mcp_server_run(cbm_mcp_server_t *srv, FILE *in, FILE *out) { if (pr == 0) { /* Raw fd appears empty. Check whether libc has already buffered * data from a previous over-read by peeking one byte via fgetc. - * If successful, push it back and proceed to getline without - * blocking. If not (EOF or EAGAIN), do the blocking poll. */ + * IMPORTANT: temporarily set O_NONBLOCK so fgetc() returns + * immediately when the FILE* buffer AND kernel fd are both empty + * (EAGAIN → EOF + ferror). Without this, fgetc() on a blocking fd + * would block indefinitely, preventing the Phase 3 idle eviction + * timeout from ever firing. */ + int saved_flags = fcntl(fd, F_GETFL); + if (saved_flags >= 0) { + (void)fcntl(fd, F_SETFL, saved_flags | O_NONBLOCK); + } int c = fgetc(in); + if (saved_flags >= 0) { + (void)fcntl(fd, F_SETFL, saved_flags); /* restore blocking */ + } if (c == EOF) { if (feof(in)) { break; /* true EOF */ } - /* No buffered data and fd is empty — do blocking poll */ + /* No buffered data (EAGAIN from non-blocking read) — clear the + * ferror indicator set by EAGAIN, then do the blocking poll. */ + clearerr(in); pr = poll(&pfd, 1, STORE_IDLE_TIMEOUT_S * 1000); if (pr < 0) { break; From 5b534ce4652cae9f7fd02b91a40116fb18d5b1fe Mon Sep 17 00:00:00 2001 From: Shane McCarron Date: Sat, 21 Mar 2026 12:47:11 -0500 Subject: [PATCH 18/25] fix(mcp): address QA round 2 findings - Add explicit fallback path when fcntl(F_GETFL) fails: skip the FILE* peek and fall through directly to blocking poll so idle eviction still fires on timeout (Finding 1) - Strengthen C unit test: verify id:1 (initialize) and id:2 (tools/list) response IDs are both present, not just a substring match on "tools" (Finding 2) Co-Authored-By: Claude Sonnet 4.6 --- src/mcp/mcp.c | 42 ++++++++++++++++++++++++++---------------- tests/test_mcp.c | 7 ++++++- 2 files changed, 32 insertions(+), 17 deletions(-) diff --git a/src/mcp/mcp.c b/src/mcp/mcp.c index 0729d0fa5..6ccccbbba 100644 --- a/src/mcp/mcp.c +++ b/src/mcp/mcp.c @@ -2425,20 +2425,10 @@ int cbm_mcp_server_run(cbm_mcp_server_t *srv, FILE *in, FILE *out) { * would block indefinitely, preventing the Phase 3 idle eviction * timeout from ever firing. */ int saved_flags = fcntl(fd, F_GETFL); - if (saved_flags >= 0) { - (void)fcntl(fd, F_SETFL, saved_flags | O_NONBLOCK); - } - int c = fgetc(in); - if (saved_flags >= 0) { - (void)fcntl(fd, F_SETFL, saved_flags); /* restore blocking */ - } - if (c == EOF) { - if (feof(in)) { - break; /* true EOF */ - } - /* No buffered data (EAGAIN from non-blocking read) — clear the - * ferror indicator set by EAGAIN, then do the blocking poll. */ - clearerr(in); + if (saved_flags < 0) { + /* fcntl failed (should not happen on a valid fd) — skip the + * FILE* peek and fall straight through to the blocking poll so + * idle eviction still fires on timeout. */ pr = poll(&pfd, 1, STORE_IDLE_TIMEOUT_S * 1000); if (pr < 0) { break; @@ -2448,8 +2438,28 @@ int cbm_mcp_server_run(cbm_mcp_server_t *srv, FILE *in, FILE *out) { continue; } } else { - /* Buffered data found — push back and fall through to getline */ - (void)ungetc(c, in); + (void)fcntl(fd, F_SETFL, saved_flags | O_NONBLOCK); + int c = fgetc(in); + (void)fcntl(fd, F_SETFL, saved_flags); /* restore blocking */ + if (c == EOF) { + if (feof(in)) { + break; /* true EOF */ + } + /* No buffered data (EAGAIN from non-blocking read) — clear + * the ferror indicator set by EAGAIN, then blocking poll. */ + clearerr(in); + pr = poll(&pfd, 1, STORE_IDLE_TIMEOUT_S * 1000); + if (pr < 0) { + break; + } + if (pr == 0) { + cbm_mcp_server_evict_idle(srv, STORE_IDLE_TIMEOUT_S); + continue; + } + } else { + /* Buffered data found — push back and fall through to getline */ + (void)ungetc(c, in); + } } } #endif diff --git a/tests/test_mcp.c b/tests/test_mcp.c index eb9f34601..bd07cba25 100644 --- a/tests/test_mcp.c +++ b/tests/test_mcp.c @@ -1272,11 +1272,16 @@ TEST(mcp_server_run_rapid_messages) { ASSERT_EQ(rc, 0); - /* Verify that tools/list response is present in output */ + /* Verify both responses are present: + * id:1 — initialize response + * id:2 — tools/list response (notifications/initialized produces none) + * and that the tools list payload is included. */ rewind(out_fp); char buf[4096] = {0}; size_t nread = fread(buf, 1, sizeof(buf) - 1, out_fp); ASSERT_TRUE(nread > 0); + ASSERT_NOT_NULL(strstr(buf, "\"id\":1")); + ASSERT_NOT_NULL(strstr(buf, "\"id\":2")); ASSERT_NOT_NULL(strstr(buf, "tools")); cbm_mcp_server_free(srv); From a90ab5d37bb1a0139639e2b6256999d75c039b1f Mon Sep 17 00:00:00 2001 From: Shane McCarron Date: Sat, 21 Mar 2026 14:38:41 -0500 Subject: [PATCH 19/25] fix(k8s): add k8s pass to incremental pipeline MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit cbm_pipeline_pass_k8s() was called in the full pipeline (pipeline.c) but absent from pipeline_incremental.c. This meant k8s Resource nodes and kustomize Module nodes were never created or updated during incremental re-indexing — only after a fresh full index. Add the pass after the semantic pass, following the same timing-log pattern as the other incremental passes. Pass changed_files (not the full file list) so only modified/added YAML files are re-processed. Add two regression tests: - incremental_k8s_manifest_indexed: full index + add manifest via incremental, verifies Resource node appears in the DB - incremental_kustomize_module_indexed: same for kustomization.yaml Module node Co-Authored-By: Claude Sonnet 4.6 --- src/pipeline/pipeline_incremental.c | 4 + tests/test_pipeline.c | 137 ++++++++++++++++++++++++++++ 2 files changed, 141 insertions(+) diff --git a/src/pipeline/pipeline_incremental.c b/src/pipeline/pipeline_incremental.c index 78e2ffde6..85443bec4 100644 --- a/src/pipeline/pipeline_incremental.c +++ b/src/pipeline/pipeline_incremental.c @@ -265,6 +265,10 @@ int cbm_pipeline_run_incremental(cbm_pipeline_t *p, const char *db_path, cbm_fil cbm_log_info("pass.timing", "pass", "incr_semantic", "elapsed_ms", itoa_buf((int)elapsed_ms(t))); + cbm_clock_gettime(CLOCK_MONOTONIC, &t); + cbm_pipeline_pass_k8s(&ctx, changed_files, ci); + cbm_log_info("pass.timing", "pass", "incr_k8s", "elapsed_ms", itoa_buf((int)elapsed_ms(t))); + /* Merge new nodes/edges from gbuf into disk DB */ int new_nodes = cbm_gbuf_node_count(gbuf); int new_edges = cbm_gbuf_edge_count(gbuf); diff --git a/tests/test_pipeline.c b/tests/test_pipeline.c index 87966acaa..353b6af06 100644 --- a/tests/test_pipeline.c +++ b/tests/test_pipeline.c @@ -5079,6 +5079,141 @@ TEST(incremental_new_file_added) { PASS(); } +TEST(incremental_k8s_manifest_indexed) { + /* Full index with a k8s manifest, then add a new manifest via incremental. + * Verifies that cbm_pipeline_pass_k8s() runs during incremental re-index. */ + char tmpdir[256]; + snprintf(tmpdir, sizeof(tmpdir), "/tmp/cbm_k8s_incr_XXXXXX"); + if (!cbm_mkdtemp(tmpdir)) { + SKIP("tmpdir"); + } + char dbpath[512]; + snprintf(dbpath, sizeof(dbpath), "%s/test.db", tmpdir); + char path[512]; + FILE *f; + + /* Initial manifest */ + snprintf(path, sizeof(path), "%s/deploy.yaml", tmpdir); + f = fopen(path, "w"); + ASSERT_NOT_NULL(f); + fprintf(f, "apiVersion: apps/v1\nkind: Deployment\nmetadata:\n name: my-app\n"); + fclose(f); + + /* Full index */ + cbm_pipeline_t *p = cbm_pipeline_new(tmpdir, dbpath, CBM_MODE_FULL); + ASSERT_NOT_NULL(p); + ASSERT_EQ(cbm_pipeline_run(p), 0); + char *project = strdup(cbm_pipeline_project_name(p)); + cbm_pipeline_free(p); + + /* Verify Resource node created by full index */ + cbm_store_t *s = cbm_store_open_path(dbpath); + ASSERT_NOT_NULL(s); + cbm_node_t *nodes = NULL; + int count = 0; + cbm_store_find_nodes_by_label(s, project, "Resource", &nodes, &count); + ASSERT_GT(count, 0); + cbm_store_free_nodes(nodes, count); + cbm_store_close(s); + + /* Add a second manifest — incremental should pick it up */ + snprintf(path, sizeof(path), "%s/svc.yaml", tmpdir); + f = fopen(path, "w"); + ASSERT_NOT_NULL(f); + fprintf(f, "apiVersion: v1\nkind: Service\nmetadata:\n name: my-svc\n"); + fclose(f); + + /* Incremental re-index */ + p = cbm_pipeline_new(tmpdir, dbpath, CBM_MODE_FULL); + ASSERT_NOT_NULL(p); + ASSERT_EQ(cbm_pipeline_run(p), 0); + cbm_pipeline_free(p); + + /* Verify both Resource nodes now present */ + s = cbm_store_open_path(dbpath); + ASSERT_NOT_NULL(s); + nodes = NULL; + count = 0; + cbm_store_find_nodes_by_label(s, project, "Resource", &nodes, &count); + ASSERT_GTE(count, 2); + cbm_store_free_nodes(nodes, count); + cbm_store_close(s); + + free(project); + char cmd[512]; + snprintf(cmd, sizeof(cmd), "rm -rf '%s'", tmpdir); + (void)system(cmd); + PASS(); +} + +TEST(incremental_kustomize_module_indexed) { + /* Verifies that a kustomization.yaml added after the initial full index + * gets a Module node via the incremental k8s pass. */ + char tmpdir[256]; + snprintf(tmpdir, sizeof(tmpdir), "/tmp/cbm_kust_incr_XXXXXX"); + if (!cbm_mkdtemp(tmpdir)) { + SKIP("tmpdir"); + } + char dbpath[512]; + snprintf(dbpath, sizeof(dbpath), "%s/test.db", tmpdir); + char path[512]; + FILE *f; + + /* Initial resource manifest (gives full index something to find) */ + snprintf(path, sizeof(path), "%s/deploy.yaml", tmpdir); + f = fopen(path, "w"); + ASSERT_NOT_NULL(f); + fprintf(f, "apiVersion: apps/v1\nkind: Deployment\nmetadata:\n name: my-app\n"); + fclose(f); + + /* Full index */ + cbm_pipeline_t *p = cbm_pipeline_new(tmpdir, dbpath, CBM_MODE_FULL); + ASSERT_NOT_NULL(p); + ASSERT_EQ(cbm_pipeline_run(p), 0); + char *project = strdup(cbm_pipeline_project_name(p)); + cbm_pipeline_free(p); + + /* Add kustomization.yaml */ + snprintf(path, sizeof(path), "%s/kustomization.yaml", tmpdir); + f = fopen(path, "w"); + ASSERT_NOT_NULL(f); + fprintf(f, "apiVersion: kustomize.config.k8s.io/v1beta1\n" + "kind: Kustomization\n" + "resources:\n" + " - deploy.yaml\n"); + fclose(f); + + /* Incremental re-index */ + p = cbm_pipeline_new(tmpdir, dbpath, CBM_MODE_FULL); + ASSERT_NOT_NULL(p); + ASSERT_EQ(cbm_pipeline_run(p), 0); + cbm_pipeline_free(p); + + /* Verify Module node created for the kustomization overlay */ + cbm_store_t *s = cbm_store_open_path(dbpath); + ASSERT_NOT_NULL(s); + cbm_node_t *nodes = NULL; + int count = 0; + cbm_store_find_nodes_by_label(s, project, "Module", &nodes, &count); + bool found_kust = false; + for (int i = 0; i < count; i++) { + if (nodes[i].properties_json && + strstr(nodes[i].properties_json, "kustomize")) { + found_kust = true; + break; + } + } + cbm_store_free_nodes(nodes, count); + cbm_store_close(s); + ASSERT_TRUE(found_kust); + + free(project); + char cmd[512]; + snprintf(cmd, sizeof(cmd), "rm -rf '%s'", tmpdir); + (void)system(cmd); + PASS(); +} + SUITE(pipeline) { /* Lifecycle */ RUN_TEST(pipeline_create_free); @@ -5269,4 +5404,6 @@ SUITE(pipeline) { RUN_TEST(incremental_detects_changed_file); RUN_TEST(incremental_detects_deleted_file); RUN_TEST(incremental_new_file_added); + RUN_TEST(incremental_k8s_manifest_indexed); + RUN_TEST(incremental_kustomize_module_indexed); } From 94882c26f9f34c257d20b3f4db7706710b53f09b Mon Sep 17 00:00:00 2001 From: Shane McCarron Date: Sat, 21 Mar 2026 14:59:12 -0500 Subject: [PATCH 20/25] fix(k8s): address QA round 3 findings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Check return value of cbm_pipeline_pass_k8s() in incremental path; log warning on failure (fail-open, matches full pipeline behaviour) - Add comment documenting pass ordering and the two known structural limitations: File→Resource DEFINES edges and cross-file kustomize IMPORTS edges are not emitted in incremental (gbuf only contains nodes for changed files; File nodes from pass_structure are absent) - Fix clang-format violation in incremental_kustomize_module_indexed test (single-line if condition) Co-Authored-By: Claude Sonnet 4.6 --- src/pipeline/pipeline_incremental.c | 10 +++++++++- tests/test_pipeline.c | 3 +-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/src/pipeline/pipeline_incremental.c b/src/pipeline/pipeline_incremental.c index 85443bec4..1799f8389 100644 --- a/src/pipeline/pipeline_incremental.c +++ b/src/pipeline/pipeline_incremental.c @@ -265,8 +265,16 @@ int cbm_pipeline_run_incremental(cbm_pipeline_t *p, const char *db_path, cbm_fil cbm_log_info("pass.timing", "pass", "incr_semantic", "elapsed_ms", itoa_buf((int)elapsed_ms(t))); + /* k8s pass runs after semantic (vs. after definitions in the full pipeline) because + * incremental has no parallel extraction phase to position it alongside. + * Note: File→Resource DEFINES edges and cross-file kustomize IMPORTS edges are not + * emitted here — File nodes (from pass_structure) are absent in the incremental gbuf, + * and gbuf_find_by_qn only resolves nodes from changed files. This is a known + * structural limitation of the incremental architecture. */ cbm_clock_gettime(CLOCK_MONOTONIC, &t); - cbm_pipeline_pass_k8s(&ctx, changed_files, ci); + if (cbm_pipeline_pass_k8s(&ctx, changed_files, ci) != 0) { + cbm_log_info("incremental.warn", "msg", "k8s_pass_failed"); + } cbm_log_info("pass.timing", "pass", "incr_k8s", "elapsed_ms", itoa_buf((int)elapsed_ms(t))); /* Merge new nodes/edges from gbuf into disk DB */ diff --git a/tests/test_pipeline.c b/tests/test_pipeline.c index 353b6af06..140a267ba 100644 --- a/tests/test_pipeline.c +++ b/tests/test_pipeline.c @@ -5197,8 +5197,7 @@ TEST(incremental_kustomize_module_indexed) { cbm_store_find_nodes_by_label(s, project, "Module", &nodes, &count); bool found_kust = false; for (int i = 0; i < count; i++) { - if (nodes[i].properties_json && - strstr(nodes[i].properties_json, "kustomize")) { + if (nodes[i].properties_json && strstr(nodes[i].properties_json, "kustomize")) { found_kust = true; break; } From 1a7c6ae8d2ca90a10a5602c7aeddfa6c12b28af7 Mon Sep 17 00:00:00 2001 From: Shane McCarron Date: Sat, 21 Mar 2026 15:08:38 -0500 Subject: [PATCH 21/25] fix(k8s): add k8s pass to parallel pipeline path The parallel execution path (used for repos above MIN_FILES_FOR_PARALLEL) was missing the cbm_pipeline_pass_k8s() call. The pass existed only in the sequential fallback path, so any repo large enough to trigger parallel indexing produced zero k8s Resource and kustomize Module nodes. Discovered via live testing against a 954-file gitops repo: the parallel path was taken, k8s pass was silently skipped, and search_graph returned 0 Resource nodes despite 213 kustomize overlays and hundreds of manifests. Add the pass after cbm_parallel_resolve() in the parallel branch, matching the same pattern (fail-open, cancel check, timing log) as the sequential path. Co-Authored-By: Claude Sonnet 4.6 --- src/pipeline/pipeline.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/pipeline/pipeline.c b/src/pipeline/pipeline.c index 41c84776f..303ed08c4 100644 --- a/src/pipeline/pipeline.c +++ b/src/pipeline/pipeline.c @@ -494,6 +494,16 @@ int cbm_pipeline_run(cbm_pipeline_t *p) { rc = -1; goto cleanup; } + + cbm_clock_gettime(CLOCK_MONOTONIC, &t); + rc = cbm_pipeline_pass_k8s(&ctx, files, file_count); + if (rc != 0) { /* log warning, continue */ + } + cbm_log_info("pass.timing", "pass", "k8s", "elapsed_ms", itoa_buf((int)elapsed_ms(t))); + if (check_cancel(p)) { + rc = -1; + goto cleanup; + } } else { cbm_log_info("pipeline.mode", "mode", "sequential", "files", itoa_buf(file_count)); From cd3b9fdd935bf065c7d5692bcee89f220f68cf4b Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Mon, 23 Mar 2026 10:57:18 +0100 Subject: [PATCH 22/25] search_code v2: graph-augmented code search with modes + path_filter MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rewrites search_code from a raw grep wrapper into a 4-phase pipeline: grep scan → graph block expansion → batch degree ranking → context assembly. - Scoped grep: queries indexed file paths from graph, passes to xargs grep — eliminates scanning vendored/generated code (Fixes #102) - Graph dedup: grep matches grouped into containing functions with in_degree, out_degree, qualified_name, label - Three output modes: compact (default, metadata only — token efficient), full (with function source), files (just file paths) - path_filter: regex on file paths (e.g. "^src/") to scope results - context: N lines around each match (like grep -C) in compact mode - directories: summary showing match distribution by top-level directory - Ranking: project source first, vendored demoted, tests penalized - Batch degree query via cbm_store_batch_count_degrees (2 SQL queries instead of 52 individual ones) - Fix yyjson pointer bugs: use strcpy variants for heap/stack strings - Add cbm_store_list_files() for indexed file enumeration - Code discovery gate hook: per-session block (PPID-keyed) nudging Claude toward codebase-memory-mcp tools before grep fallback --- src/cli/cli.c | 50 +++- src/mcp/mcp.c | 563 ++++++++++++++++++++++++++++++++++++++++++---- src/store/store.c | 37 +++ src/store/store.h | 4 + tests/test_cli.c | 6 +- 5 files changed, 608 insertions(+), 52 deletions(-) diff --git a/src/cli/cli.c b/src/cli/cli.c index ba58e5381..77b8378f5 100644 --- a/src/cli/cli.c +++ b/src/cli/cli.c @@ -1310,11 +1310,7 @@ int cbm_remove_antigravity_mcp(const char *config_path) { /* ── Claude Code pre-tool hooks ───────────────────────────────── */ #define CMM_HOOK_MATCHER "Grep|Glob|Read|Search" -#define CMM_HOOK_COMMAND \ - "echo 'Reminder: prefer codebase-memory-mcp search_graph/trace_call_path/" \ - "get_code_snippet over Grep/Glob/Read/Search for code discovery. " \ - "Use get_code_snippet(qualified_name) instead of Read to view a function. " \ - "Fall back only if MCP returns insufficient results.' >&2" +#define CMM_HOOK_COMMAND "~/.claude/hooks/cbm-code-discovery-gate" /* Old matcher values from previous versions — recognized during upgrade so * upsert_hooks_json can remove them before inserting the current matcher. */ @@ -1470,6 +1466,45 @@ int cbm_remove_claude_hooks(const char *settings_path) { return remove_hooks_json(settings_path, "PreToolUse", CMM_HOOK_MATCHER); } +/* Install the code discovery gate script to ~/.claude/hooks/. + * Blocks the first Grep/Glob/Read/Search call per session (exit 2 + stderr), + * nudging Claude toward codebase-memory-mcp. All subsequent calls in the same + * session pass through (gate file keyed on PPID). */ +static void cbm_install_hook_gate_script(const char *home) { + if (!home) { + return; + } + char hooks_dir[1024]; + snprintf(hooks_dir, sizeof(hooks_dir), "%s/.claude/hooks", home); + cbm_mkdir_p(hooks_dir, 0755); + + char script_path[1024]; + snprintf(script_path, sizeof(script_path), "%s/cbm-code-discovery-gate", hooks_dir); + + FILE *f = fopen(script_path, "w"); + if (!f) { + return; + } + fprintf(f, "#!/bin/bash\n" + "# Gate hook: nudges Claude toward codebase-memory-mcp for code discovery.\n" + "# First Grep/Glob/Read/Search per session -> block. Subsequent -> allow.\n" + "# PPID = Claude Code process PID, unique per session.\n" + "GATE=/tmp/cbm-code-discovery-gate-$PPID\n" + "find /tmp -name 'cbm-code-discovery-gate-*' -mtime +1 -delete 2>/dev/null\n" + "if [ -f \"$GATE\" ]; then\n" + " exit 0\n" + "fi\n" + "touch \"$GATE\"\n" + "echo 'BLOCKED: For code discovery, use codebase-memory-mcp tools first: " + "search_graph(name_pattern) to find functions/classes, trace_call_path() for " + "call chains, get_code_snippet(qualified_name) to read source. If the graph " + "is not indexed yet, call index_repository first. Fall back to Grep/Glob/Read " + "only for text content search. If you need Grep, retry.' >&2\n" + "exit 2\n"); + fclose(f); + chmod(script_path, 0755); +} + #define GEMINI_HOOK_MATCHER "google_search|read_file|grep_search" #define GEMINI_HOOK_COMMAND \ "echo 'Reminder: prefer codebase-memory-mcp search_graph/trace_call_path/" \ @@ -2218,13 +2253,14 @@ int cbm_cmd_install(int argc, char **argv) { } printf(" mcp: %s\n", mcp_path2); - /* PreToolUse hook */ + /* PreToolUse hook + gate script */ char settings_path[1024]; snprintf(settings_path, sizeof(settings_path), "%s/.claude/settings.json", home); if (!dry_run) { cbm_upsert_claude_hooks(settings_path); + cbm_install_hook_gate_script(home); } - printf(" hooks: PreToolUse (Grep|Glob reminder)\n"); + printf(" hooks: PreToolUse (code discovery gate)\n"); } /* Step 5: Install Codex CLI */ diff --git a/src/mcp/mcp.c b/src/mcp/mcp.c index bdfdae8b5..80abc9ecb 100644 --- a/src/mcp/mcp.c +++ b/src/mcp/mcp.c @@ -20,6 +20,7 @@ #include "foundation/compat_thread.h" #include "foundation/log.h" #include "foundation/str_util.h" +#include "foundation/compat_regex.h" #ifdef _WIN32 #include /* _getpid */ @@ -277,12 +278,21 @@ static const tool_def_t TOOLS[] = { "\"array\",\"items\":{\"type\":\"string\"}}}}"}, {"search_code", - "Search source code content with text or regex patterns. Use for string literals, error " - "messages, and config values that are not in the knowledge graph.", + "Graph-augmented code search. Finds text patterns via grep, then enriches results with " + "the knowledge graph: deduplicates matches into containing functions, ranks by structural " + "importance (definitions first, popular functions next, tests last). " + "Modes: compact (default, signatures only — token efficient), full (with source), " + "files (just file paths). Use path_filter regex to scope results.", "{\"type\":\"object\",\"properties\":{\"pattern\":{\"type\":\"string\"},\"project\":{\"type\":" - "\"string\"},\"file_pattern\":{\"type\":\"string\"},\"regex\":{\"type\":\"boolean\"," - "\"default\":false},\"limit\":{\"type\":\"integer\",\"description\":\"Max results. Default: " - "unlimited\"}},\"required\":[" + "\"string\"},\"file_pattern\":{\"type\":\"string\",\"description\":\"Glob for grep " + "--include (e.g. *.go)\"},\"path_filter\":{\"type\":\"string\",\"description\":\"Regex " + "filter on result file paths (e.g. ^src/ or \\\\.(go|ts)$)\"},\"mode\":{\"type\":\"string\"," + "\"enum\":[\"compact\",\"full\",\"files\"],\"default\":\"compact\",\"description\":\"compact: " + "signatures+metadata (default). full: with source. files: just file list.\"}," + "\"context\":{\"type\":\"integer\",\"description\":\"Lines of context around each match " + "(like grep -C). Only used in compact mode.\"}," + "\"regex\":{\"type\":\"boolean\",\"default\":false},\"limit\":{\"type\":\"integer\"," + "\"description\":\"Max results (default 10)\",\"default\":10}},\"required\":[" "\"pattern\"]}"}, {"list_projects", "List all indexed projects", "{\"type\":\"object\",\"properties\":{}}"}, @@ -1624,21 +1634,314 @@ static char *handle_get_code_snippet(cbm_mcp_server_t *srv, const char *args) { true); } -/* ── search_code ──────────────────────────────────────────────── */ +/* ── search_code v2: graph-augmented code search ─────────────── */ + +/* Strip non-ASCII bytes to guarantee valid UTF-8 JSON output */ +enum { ASCII_MAX = 127 }; +static void sanitize_ascii(char *s) { + for (unsigned char *p = (unsigned char *)s; *p; p++) { + if (*p > ASCII_MAX) { + *p = '?'; + } + } +} + +/* Intermediate grep match */ +typedef struct { + char file[512]; + int line; + char content[1024]; +} grep_match_t; + +/* Deduped result: one per containing graph node */ +typedef struct { + int64_t node_id; /* 0 = raw match (no containing node) */ + char node_name[256]; + char qualified_name[512]; + char label[64]; + char file[512]; + int start_line; + int end_line; + int in_degree; + int out_degree; + int score; + int match_lines[64]; + int match_count; +} search_result_t; + +/* Score a result for ranking: project source first, vendored last, tests lowest */ +enum { SCORE_FUNC = 10, SCORE_ROUTE = 15, SCORE_VENDORED = -50, SCORE_TEST = -5 }; +enum { MAX_LINE_SPAN = 999999 }; + +static int compute_search_score(const search_result_t *r) { + int score = r->in_degree; + if (strcmp(r->label, "Function") == 0 || strcmp(r->label, "Method") == 0) { + score += SCORE_FUNC; + } + if (strcmp(r->label, "Route") == 0) { + score += SCORE_ROUTE; + } + if (strstr(r->file, "vendored/") || strstr(r->file, "vendor/") || + strstr(r->file, "node_modules/")) { + score += SCORE_VENDORED; + } + /* Penalize test files */ + if (strstr(r->file, "test") || strstr(r->file, "spec") || strstr(r->file, "_test.")) { + score += SCORE_TEST; + } + return score; +} + +static int search_result_cmp(const void *a, const void *b) { + const search_result_t *ra = (const search_result_t *)a; + const search_result_t *rb = (const search_result_t *)b; + return rb->score - ra->score; /* descending */ +} + +/* Build the grep command string based on scoped vs recursive mode */ +static void build_grep_cmd(char *cmd, size_t cmd_sz, bool use_regex, bool scoped, + const char *file_pattern, const char *tmpfile, const char *filelist, + const char *root_path) { + // NOLINTNEXTLINE(readability-implicit-bool-conversion) + const char *flag = use_regex ? "-E" : "-F"; + if (scoped) { + if (file_pattern) { + snprintf(cmd, cmd_sz, "xargs grep -n %s --include='%s' -f '%s' < '%s' 2>/dev/null", + flag, file_pattern, tmpfile, filelist); + } else { + snprintf(cmd, cmd_sz, "xargs grep -n %s -f '%s' < '%s' 2>/dev/null", flag, tmpfile, + filelist); + } + } else { + if (file_pattern) { + snprintf(cmd, cmd_sz, "grep -rn %s --include='%s' -f '%s' '%s' 2>/dev/null", flag, + file_pattern, tmpfile, root_path); + } else { + snprintf(cmd, cmd_sz, "grep -rn %s -f '%s' '%s' 2>/dev/null", flag, tmpfile, root_path); + } + } +} + +/* Phase 4: assemble JSON output from search results */ +static char *assemble_search_output(search_result_t *sr, int sr_count, grep_match_t *raw, + int raw_count, int gm_count, int limit, int mode, + int context_lines, const char *root_path) { + enum { MODE_COMPACT = 0, MODE_FULL = 1, MODE_FILES = 2 }; + + yyjson_mut_doc *doc = yyjson_mut_doc_new(NULL); + yyjson_mut_val *root_obj = yyjson_mut_obj(doc); + yyjson_mut_doc_set_root(doc, root_obj); + + int output_count = sr_count < limit ? sr_count : limit; + + if (mode == MODE_FILES) { + yyjson_mut_val *files_arr = yyjson_mut_arr(doc); + char *seen_files[512]; + int seen_count = 0; + for (int fi = 0; fi < output_count; fi++) { + bool dup = false; + for (int j = 0; j < seen_count; j++) { + if (strcmp(seen_files[j], sr[fi].file) == 0) { + dup = true; + break; + } + } + if (!dup && seen_count < 512) { + seen_files[seen_count++] = sr[fi].file; + yyjson_mut_arr_add_str(doc, files_arr, sr[fi].file); + } + } + for (int fi = 0; fi < raw_count && seen_count < 512; fi++) { + bool dup = false; + for (int j = 0; j < seen_count; j++) { + if (strcmp(seen_files[j], raw[fi].file) == 0) { + dup = true; + break; + } + } + if (!dup) { + seen_files[seen_count++] = raw[fi].file; + yyjson_mut_arr_add_str(doc, files_arr, raw[fi].file); + } + } + yyjson_mut_obj_add_val(doc, root_obj, "files", files_arr); + } else { + yyjson_mut_val *results_arr = yyjson_mut_arr(doc); + for (int ri = 0; ri < output_count; ri++) { + search_result_t *r = &sr[ri]; + yyjson_mut_val *item = yyjson_mut_obj(doc); + + yyjson_mut_obj_add_str(doc, item, "node", r->node_name); + yyjson_mut_obj_add_str(doc, item, "qualified_name", r->qualified_name); + yyjson_mut_obj_add_str(doc, item, "label", r->label); + yyjson_mut_obj_add_str(doc, item, "file", r->file); + yyjson_mut_obj_add_int(doc, item, "start_line", r->start_line); + yyjson_mut_obj_add_int(doc, item, "end_line", r->end_line); + yyjson_mut_obj_add_int(doc, item, "in_degree", r->in_degree); + yyjson_mut_obj_add_int(doc, item, "out_degree", r->out_degree); + + yyjson_mut_val *ml = yyjson_mut_arr(doc); + for (int j = 0; j < r->match_count; j++) { + yyjson_mut_arr_add_int(doc, ml, r->match_lines[j]); + } + yyjson_mut_obj_add_val(doc, item, "match_lines", ml); + + if (r->start_line > 0 && r->end_line > 0) { + char abs_path[1024]; + snprintf(abs_path, sizeof(abs_path), "%s/%s", root_path, r->file); + + if (mode == MODE_FULL) { + char *source = read_file_lines(abs_path, r->start_line, r->end_line); + if (source) { + sanitize_ascii(source); + yyjson_mut_obj_add_strcpy(doc, item, "source", source); + free(source); + } + } else if (context_lines > 0 && r->match_count > 0) { + int first_match = r->match_lines[0]; + int last_match = r->match_lines[r->match_count - 1]; + int ctx_start = first_match - context_lines; + int ctx_end = last_match + context_lines; + if (ctx_start < 1) { + ctx_start = 1; + } + char *ctx = read_file_lines(abs_path, ctx_start, ctx_end); + if (ctx) { + sanitize_ascii(ctx); + yyjson_mut_obj_add_strcpy(doc, item, "context", ctx); + yyjson_mut_obj_add_int(doc, item, "context_start", ctx_start); + free(ctx); + } + } + } + + yyjson_mut_arr_add_val(results_arr, item); + } + yyjson_mut_obj_add_val(doc, root_obj, "results", results_arr); + + enum { MAX_RAW = 20 }; + yyjson_mut_val *raw_arr = yyjson_mut_arr(doc); + int raw_output = raw_count < MAX_RAW ? raw_count : MAX_RAW; + for (int ri = 0; ri < raw_output; ri++) { + yyjson_mut_val *item = yyjson_mut_obj(doc); + yyjson_mut_obj_add_str(doc, item, "file", raw[ri].file); + yyjson_mut_obj_add_int(doc, item, "line", raw[ri].line); + yyjson_mut_obj_add_str(doc, item, "content", raw[ri].content); + yyjson_mut_arr_add_val(raw_arr, item); + } + yyjson_mut_obj_add_val(doc, root_obj, "raw_matches", raw_arr); + } + + /* Directory distribution */ + { + yyjson_mut_val *dirs = yyjson_mut_obj(doc); + char dir_names[64][128]; + int dir_counts[64]; + int dir_n = 0; + for (int di = 0; di < sr_count; di++) { + char top[128] = ""; + const char *slash = strchr(sr[di].file, '/'); + if (slash) { + size_t dlen = (size_t)(slash - sr[di].file + 1); + if (dlen >= sizeof(top)) { + dlen = sizeof(top) - 1; + } + memcpy(top, sr[di].file, dlen); + top[dlen] = '\0'; + } else { + snprintf(top, sizeof(top), "%s", sr[di].file); + } + int found = -1; + for (int d = 0; d < dir_n; d++) { + if (strcmp(dir_names[d], top) == 0) { + found = d; + break; + } + } + if (found >= 0) { + dir_counts[found]++; + } else if (dir_n < 64) { + snprintf(dir_names[dir_n], sizeof(dir_names[0]), "%s", top); + dir_counts[dir_n] = 1; + dir_n++; + } + } + for (int d = 0; d < dir_n; d++) { + yyjson_mut_val *key = yyjson_mut_strcpy(doc, dir_names[d]); + yyjson_mut_val *val = yyjson_mut_int(doc, dir_counts[d]); + yyjson_mut_obj_add(dirs, key, val); + } + yyjson_mut_obj_add_val(doc, root_obj, "directories", dirs); + } + + /* Summary stats */ + yyjson_mut_obj_add_int(doc, root_obj, "total_grep_matches", gm_count); + yyjson_mut_obj_add_int(doc, root_obj, "total_results", sr_count); + yyjson_mut_obj_add_int(doc, root_obj, "raw_match_count", raw_count); + if (sr_count > 0 && gm_count > 0) { + char ratio[32]; + snprintf(ratio, sizeof(ratio), "%.1fx", (double)gm_count / (double)(sr_count + raw_count)); + yyjson_mut_obj_add_strcpy(doc, root_obj, "dedup_ratio", ratio); + } + + char *json = yy_doc_to_str(doc); + if (json) { + sanitize_ascii(json); + } + yyjson_mut_doc_free(doc); + + char *result = cbm_mcp_text_result(json, false); + free(json); + return result; +} static char *handle_search_code(cbm_mcp_server_t *srv, const char *args) { char *pattern = cbm_mcp_get_string_arg(args, "pattern"); char *project = cbm_mcp_get_string_arg(args, "project"); char *file_pattern = cbm_mcp_get_string_arg(args, "file_pattern"); - int limit = cbm_mcp_get_int_arg(args, "limit", 500000); + char *path_filter = cbm_mcp_get_string_arg(args, "path_filter"); + char *mode_str = cbm_mcp_get_string_arg(args, "mode"); + int limit = cbm_mcp_get_int_arg(args, "limit", 10); + int context_lines = cbm_mcp_get_int_arg(args, "context", 0); bool use_regex = cbm_mcp_get_bool_arg(args, "regex"); + /* Parse mode: compact (default), full, files */ + enum { MODE_COMPACT, MODE_FULL, MODE_FILES }; + int mode = MODE_COMPACT; + if (mode_str) { + if (strcmp(mode_str, "full") == 0) { + mode = MODE_FULL; + } else if (strcmp(mode_str, "files") == 0) { + mode = MODE_FILES; + } + free(mode_str); + } + + /* Compile path_filter regex if provided */ + cbm_regex_t path_regex; + bool has_path_filter = false; + if (path_filter && path_filter[0]) { + if (cbm_regcomp(&path_regex, path_filter, CBM_REG_EXTENDED | CBM_REG_NOSUB) == CBM_REG_OK) { + has_path_filter = true; + } + free(path_filter); + path_filter = NULL; + } else { + free(path_filter); + path_filter = NULL; + } + if (!pattern) { free(project); free(file_pattern); return cbm_mcp_text_result("pattern is required", true); } + /* Resolve project */ + if (!project && srv->session_project[0]) { + project = heap_strdup(srv->session_project); + } + char *root_path = get_project_root(srv, project); if (!root_path) { free(pattern); @@ -1657,7 +1960,8 @@ static char *handle_search_code(cbm_mcp_server_t *srv, const char *args) { return cbm_mcp_text_result("path or file_pattern contains invalid characters", true); } - /* Write pattern to temp file to avoid shell injection */ + /* ── Phase 1: Grep scan ──────────────────────────────────── */ + char tmpfile[256]; #ifdef _WIN32 snprintf(tmpfile, sizeof(tmpfile), "/tmp/cbm_search_%d.pat", (int)_getpid()); @@ -1676,21 +1980,53 @@ static char *handle_search_code(cbm_mcp_server_t *srv, const char *args) { (void)fprintf(tf, "%s\n", pattern); (void)fclose(tf); - char cmd[4096]; - // NOLINTNEXTLINE(readability-implicit-bool-conversion) - const char *flag = use_regex ? "-E" : "-F"; - if (file_pattern) { - snprintf(cmd, sizeof(cmd), "grep -rn %s --include='%s' -m %d -f '%s' '%s' 2>/dev/null", - flag, file_pattern, limit * 3, tmpfile, root_path); - } else { - snprintf(cmd, sizeof(cmd), "grep -rn %s -m %d -f '%s' '%s' 2>/dev/null", flag, limit * 3, - tmpfile, root_path); + /* No grep-level match limit — let grep find all matches, then dedup and + * cap in our code. The -m flag caused results from large vendored files + * to exhaust the quota before reaching project source files. */ + enum { GREP_MAX_MATCHES = 500 }; + int grep_limit = GREP_MAX_MATCHES; + + /* Scope grep to indexed files only — avoids scanning vendored/generated code. + * Query the graph for distinct file paths, write them to a temp file, + * then use xargs to pass them to grep. Falls back to recursive grep if + * no indexed files found (project not fully indexed). */ + char filelist[256]; + snprintf(filelist, sizeof(filelist), "%s.files", tmpfile); + bool scoped = false; + + cbm_store_t *pre_store = resolve_store(srv, project); + if (pre_store) { + const char *pn = project ? project : srv->session_project; + char **indexed_files = NULL; + int indexed_count = 0; + if (pn && + cbm_store_list_files(pre_store, pn, &indexed_files, &indexed_count) == CBM_STORE_OK && + indexed_count > 0) { + FILE *fl = fopen(filelist, "w"); + if (fl) { + for (int fi = 0; fi < indexed_count; fi++) { + fprintf(fl, "%s/%s\n", root_path, indexed_files[fi]); + } + fclose(fl); + scoped = true; + } + for (int fi = 0; fi < indexed_count; fi++) { + free(indexed_files[fi]); + } + free(indexed_files); + } } + char cmd[4096]; + build_grep_cmd(cmd, sizeof(cmd), use_regex, scoped, file_pattern, tmpfile, filelist, root_path); + // NOLINTNEXTLINE(bugprone-command-processor,cert-env33-c) FILE *fp = cbm_popen(cmd, "r"); if (!fp) { cbm_unlink(tmpfile); + if (scoped) { + cbm_unlink(filelist); + } free(root_path); free(pattern); free(project); @@ -1698,16 +2034,14 @@ static char *handle_search_code(cbm_mcp_server_t *srv, const char *args) { return cbm_mcp_text_result("search failed", true); } - yyjson_mut_doc *doc = yyjson_mut_doc_new(NULL); - yyjson_mut_val *root_obj = yyjson_mut_obj(doc); - yyjson_mut_doc_set_root(doc, root_obj); - - yyjson_mut_val *matches = yyjson_mut_arr(doc); + /* Collect grep matches into array */ + int gm_cap = 64; + int gm_count = 0; + grep_match_t *gm = malloc(gm_cap * sizeof(grep_match_t)); char line[2048]; - int count = 0; size_t root_len = strlen(root_path); - while (fgets(line, sizeof(line), fp) && count < limit) { + while (fgets(line, sizeof(line), fp) && gm_count < grep_limit) { size_t len = strlen(line); while (len > 0 && (line[len - 1] == '\n' || line[len - 1] == '\r')) { line[--len] = '\0'; @@ -1716,7 +2050,6 @@ static char *handle_search_code(cbm_mcp_server_t *srv, const char *args) { continue; } - /* grep output: /abs/path/file:lineno:content */ char *colon1 = strchr(line, ':'); if (!colon1) { continue; @@ -1729,7 +2062,6 @@ static char *handle_search_code(cbm_mcp_server_t *srv, const char *args) { *colon1 = '\0'; *colon2 = '\0'; - /* Strip root_path prefix to get relative path */ const char *file = line; if (strncmp(file, root_path, root_len) == 0) { file += root_len; @@ -1737,31 +2069,178 @@ static char *handle_search_code(cbm_mcp_server_t *srv, const char *args) { file++; } } - int lineno = (int)strtol(colon1 + 1, NULL, 10); - const char *content = colon2 + 1; - yyjson_mut_val *item = yyjson_mut_obj(doc); - yyjson_mut_obj_add_str(doc, item, "file", file); - yyjson_mut_obj_add_int(doc, item, "line", lineno); - yyjson_mut_obj_add_str(doc, item, "content", content); - yyjson_mut_arr_add_val(matches, item); - count++; + /* Apply path_filter regex — skip files that don't match */ + if (has_path_filter && cbm_regexec(&path_regex, file, 0, NULL, 0) != CBM_REG_OK) { + continue; + } + + if (gm_count >= gm_cap) { + gm_cap *= 2; + gm = safe_realloc(gm, gm_cap * sizeof(grep_match_t)); + } + snprintf(gm[gm_count].file, sizeof(gm[0].file), "%s", file); + gm[gm_count].line = (int)strtol(colon1 + 1, NULL, 10); + snprintf(gm[gm_count].content, sizeof(gm[0].content), "%s", colon2 + 1); + sanitize_ascii(gm[gm_count].content); + gm_count++; } cbm_pclose(fp); - cbm_unlink(tmpfile); /* Clean up pattern file after grep is done */ + cbm_unlink(tmpfile); + if (scoped) { + cbm_unlink(filelist); + } - yyjson_mut_obj_add_val(doc, root_obj, "matches", matches); - yyjson_mut_obj_add_int(doc, root_obj, "count", count); + /* ── Phase 2+3: Block expansion + graph ranking ──────────── */ + /* Sort grep matches by file for contiguous processing. + * Then: one SQL query per unique file for nodes, one batch query for all degrees. */ - char *json = yy_doc_to_str(doc); - yyjson_mut_doc_free(doc); + cbm_store_t *store = resolve_store(srv, project); + const char *proj_name = project ? project : srv->session_project; + + int sr_cap = 32; + int sr_count = 0; + search_result_t *sr = calloc(sr_cap, sizeof(search_result_t)); + + int raw_cap = 32; + int raw_count = 0; + grep_match_t *raw = malloc(raw_cap * sizeof(grep_match_t)); + + /* Sort matches by file path for contiguous per-file processing */ + qsort(gm, gm_count, sizeof(grep_match_t), (int (*)(const void *, const void *))strcmp); + + /* Process matches file-by-file (contiguous runs after sort) */ + int i = 0; + while (i < gm_count) { + const char *cur_file = gm[i].file; + int file_start = i; + + /* Find end of this file's run */ + while (i < gm_count && strcmp(gm[i].file, cur_file) == 0) { + i++; + } + int file_end = i; /* [file_start, file_end) */ + + /* One SQL query: load all nodes in this file */ + cbm_node_t *file_nodes = NULL; + int file_node_count = 0; + if (store && proj_name) { + cbm_store_find_nodes_by_file(store, proj_name, cur_file, &file_nodes, &file_node_count); + } + + /* Match each grep hit to tightest containing node (in-memory) */ + for (int mi = file_start; mi < file_end; mi++) { + int best = -1; + int best_span = MAX_LINE_SPAN; + for (int j = 0; j < file_node_count; j++) { + if (file_nodes[j].start_line <= gm[mi].line && + file_nodes[j].end_line >= gm[mi].line) { + int span = file_nodes[j].end_line - file_nodes[j].start_line; + if (span < best_span) { + best = j; + best_span = span; + } + } + } + + if (best >= 0) { + cbm_node_t *n = &file_nodes[best]; + + /* Dedup: check if node already in results */ + int existing = -1; + for (int j = 0; j < sr_count; j++) { + if (sr[j].node_id == n->id) { + existing = j; + break; + } + } + + if (existing >= 0) { + if (sr[existing].match_count < 64) { + sr[existing].match_lines[sr[existing].match_count++] = gm[mi].line; + } + } else { + if (sr_count >= sr_cap) { + sr_cap *= 2; + sr = safe_realloc(sr, sr_cap * sizeof(search_result_t)); + memset(&sr[sr_count], 0, (sr_cap - sr_count) * sizeof(search_result_t)); + } + search_result_t *r = &sr[sr_count]; + r->node_id = n->id; + snprintf(r->node_name, sizeof(r->node_name), "%s", n->name ? n->name : ""); + snprintf(r->qualified_name, sizeof(r->qualified_name), "%s", + n->qualified_name ? n->qualified_name : ""); + snprintf(r->label, sizeof(r->label), "%s", n->label ? n->label : ""); + snprintf(r->file, sizeof(r->file), "%s", n->file_path ? n->file_path : ""); + r->start_line = n->start_line; + r->end_line = n->end_line; + r->match_lines[0] = gm[mi].line; + r->match_count = 1; + sr_count++; + } + } else { + if (raw_count >= raw_cap) { + raw_cap *= 2; + raw = safe_realloc(raw, raw_cap * sizeof(grep_match_t)); + } + raw[raw_count++] = gm[mi]; + } + } + + /* Free file nodes */ + for (int j = 0; j < file_node_count; j++) { + free((void *)file_nodes[j].project); + free((void *)file_nodes[j].label); + free((void *)file_nodes[j].name); + free((void *)file_nodes[j].qualified_name); + free((void *)file_nodes[j].file_path); + free((void *)file_nodes[j].properties_json); + } + free(file_nodes); + } + + /* Phase 3: batch degree query — ONE query for all results instead of 2×N */ + if (store && sr_count > 0) { + int64_t *ids = malloc(sr_count * sizeof(int64_t)); + int *in_degs = malloc(sr_count * sizeof(int)); + int *out_degs = malloc(sr_count * sizeof(int)); + for (int j = 0; j < sr_count; j++) { + ids[j] = sr[j].node_id; + } + if (cbm_store_batch_count_degrees(store, ids, sr_count, "CALLS", in_degs, out_degs) == + CBM_STORE_OK) { + for (int j = 0; j < sr_count; j++) { + sr[j].in_degree = in_degs[j]; + sr[j].out_degree = out_degs[j]; + } + } + free(ids); + free(in_degs); + free(out_degs); + } + + /* Compute scores and sort */ + for (int j = 0; j < sr_count; j++) { + sr[j].score = compute_search_score(&sr[j]); + } + if (sr_count > 1) { + qsort(sr, sr_count, sizeof(search_result_t), search_result_cmp); + } + + /* ── Phase 4: Context assembly (extracted helper) ─────────── */ + + char *result = assemble_search_output(sr, sr_count, raw, raw_count, gm_count, limit, mode, + context_lines, root_path); + free(gm); + free(sr); + free(raw); free(root_path); free(pattern); free(project); free(file_pattern); - - char *result = cbm_mcp_text_result(json, false); - free(json); + if (has_path_filter) { + cbm_regfree(&path_regex); + } return result; } diff --git a/src/store/store.c b/src/store/store.c index 4b00d4e73..b25b7e87c 100644 --- a/src/store/store.c +++ b/src/store/store.c @@ -1431,6 +1431,43 @@ void cbm_store_node_degree(cbm_store_t *s, int64_t node_id, int *in_deg, int *ou } } +/* ── List distinct file paths ────────────────────────────────── */ + +int cbm_store_list_files(cbm_store_t *s, const char *project, char ***out, int *count) { + *out = NULL; + *count = 0; + if (!s || !s->db || !project) { + return CBM_STORE_ERR; + } + + const char *sql = "SELECT DISTINCT file_path FROM nodes " + "WHERE project = ?1 AND file_path IS NOT NULL AND file_path != ''"; + sqlite3_stmt *stmt = NULL; + if (sqlite3_prepare_v2(s->db, sql, -1, &stmt, NULL) != SQLITE_OK) { + return CBM_STORE_ERR; + } + sqlite3_bind_text(stmt, 1, project, -1, SQLITE_STATIC); + + int cap = 64; + int n = 0; + char **files = malloc(cap * sizeof(char *)); + while (sqlite3_step(stmt) == SQLITE_ROW) { + const char *fp = (const char *)sqlite3_column_text(stmt, 0); + if (!fp) { + continue; + } + if (n >= cap) { + cap *= 2; + files = safe_realloc(files, cap * sizeof(char *)); + } + files[n++] = heap_strdup(fp); + } + sqlite3_finalize(stmt); + *out = files; + *count = n; + return CBM_STORE_OK; +} + /* ── Node neighbor names ──────────────────────────────────────── */ // NOLINTNEXTLINE(bugprone-easily-swappable-parameters) diff --git a/src/store/store.h b/src/store/store.h index 0bf385a82..94de109c4 100644 --- a/src/store/store.h +++ b/src/store/store.h @@ -73,6 +73,10 @@ int cbm_store_find_nodes_by_qn_suffix(cbm_store_t *s, const char *project, const /* Get CALLS degree of a node (inbound and outbound). */ void cbm_store_node_degree(cbm_store_t *s, int64_t node_id, int *in_deg, int *out_deg); +/* Get distinct file paths for a project. Caller must free each out[i] and out itself. + * Returns CBM_STORE_OK or CBM_STORE_ERR. */ +int cbm_store_list_files(cbm_store_t *s, const char *project, char ***out, int *count); + /* Get caller/callee names for a node (CALLS/HTTP_CALLS/ASYNC_CALLS edges). * Returns 0 on success. Caller must free each out_callers[i]/out_callees[i] * and the arrays themselves. */ diff --git a/tests/test_cli.c b/tests/test_cli.c index 19ccdecaa..788a4807a 100644 --- a/tests/test_cli.c +++ b/tests/test_cli.c @@ -1674,7 +1674,7 @@ TEST(cli_upsert_claude_hook_fresh) { ASSERT_NOT_NULL(data); ASSERT(strstr(data, "PreToolUse") != NULL); ASSERT(strstr(data, "Grep|Glob|Read") != NULL); - ASSERT(strstr(data, "codebase-memory-mcp") != NULL); + ASSERT(strstr(data, "cbm-code-discovery-gate") != NULL); test_rmdir_r(tmpdir); PASS(); @@ -1722,9 +1722,9 @@ TEST(cli_upsert_claude_hook_replace) { const char *data = read_test_file(settingspath); ASSERT_NOT_NULL(data); - /* Old message gone, new message present */ + /* Old message gone, new hook script path present */ ASSERT(strstr(data, "old-cmm-message") == NULL); - ASSERT(strstr(data, "codebase-memory-mcp") != NULL); + ASSERT(strstr(data, "cbm-code-discovery-gate") != NULL); test_rmdir_r(tmpdir); PASS(); From eb8fcbf1ca831b54795146fc8c2a4f9c8cf2be46 Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Mon, 23 Mar 2026 11:28:00 +0100 Subject: [PATCH 23/25] Add commit format convention and one-issue-per-PR rule to CONTRIBUTING.md - Commit format table: feat, fix, test, refactor, perf, docs, chore (adapted from @halindrome's PR #85) - One issue per PR: no bundling multiple fixes/features - Open an issue first: every PR must reference a tracking issue Based on feedback from @halindrome (#84, #85) who identified the need for structured PR guidelines and commit conventions. --- CONTRIBUTING.md | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 0f3aae509..3f73f59cf 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -85,14 +85,30 @@ Language support is split between two layers: 4. Add a test case in `tests/test_pipeline.c` for integration-level fixes 5. Verify with a real open-source repo +## Commit Format + +Use conventional commits: `type(scope): description` + +| Type | When to use | +|------|-------------| +| `feat` | New feature or capability | +| `fix` | Bug fix | +| `test` | Adding or updating tests | +| `refactor` | Code change that neither fixes a bug nor adds a feature | +| `perf` | Performance improvement | +| `docs` | Documentation only | +| `chore` | Build scripts, CI, dependency updates | + +Examples: `fix(store): set busy_timeout before WAL`, `feat(cli): add --progress flag` + ## Pull Request Guidelines +- **One issue per PR.** Each PR must address exactly one bug, one feature, or one refactor. Do not bundle multiple fixes or feature additions into a single PR. If your change touches multiple areas, split it into separate PRs. +- **Open an issue first.** Every PR should reference a tracking issue (`Fixes #N` or `Closes #N`). This ensures the change is discussed before code is written. - **C code only** — this project was rewritten from Go to pure C in v0.5.0. Go PRs will be acknowledged and potentially ported, but cannot be merged directly. -- One logical change per PR — don't bundle unrelated features - Include tests for new functionality - Run `scripts/test.sh` and `scripts/lint.sh` before submitting - Keep PRs focused — avoid unrelated reformatting or refactoring -- Reference the issue number in your PR description ## Security From 7f33e8b49b20296eb99756fefee1b46e61683c8d Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Mon, 23 Mar 2026 11:52:39 +0100 Subject: [PATCH 24/25] Fix lint issues in userconfig + add custom extensions docs to README MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - clang-format: fix alignment in userconfig.c and userconfig.h - Magic number 1280 → PATH_BUF_SZ named constant - Remove dead g-- before break in dedup loop (cppcheck) - Add "Custom File Extensions" section to README documenting .codebase-memory.json and global config --- README.md | 18 ++++++++++++++++++ src/discover/userconfig.c | 9 ++++----- src/discover/userconfig.h | 4 ++-- 3 files changed, 24 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 8a4ffb3b8..c9e18baa6 100644 --- a/README.md +++ b/README.md @@ -306,6 +306,24 @@ codebase-memory-mcp config set auto_index_limit 50000 # max files for auto-in codebase-memory-mcp config reset auto_index # reset to default ``` +## Custom File Extensions + +Map additional file extensions to supported languages via JSON config files. Useful for framework-specific extensions like `.blade.php` (Laravel) or `.mjs` (ES modules). + +**Per-project** (in your repo root): +```json +// .codebase-memory.json +{"extra_extensions": {".blade.php": "php", ".mjs": "javascript"}} +``` + +**Global** (applies to all projects): +```json +// ~/.config/codebase-memory-mcp/config.json (or $XDG_CONFIG_HOME/...) +{"extra_extensions": {".twig": "html", ".phtml": "php"}} +``` + +Project config overrides global for conflicting extensions. Unknown language values are silently skipped. Missing config files are ignored. + ## Persistence SQLite databases stored at `~/.cache/codebase-memory-mcp/`. Persists across restarts (WAL mode, ACID-safe). To reset: `rm -rf ~/.cache/codebase-memory-mcp/`. diff --git a/src/discover/userconfig.c b/src/discover/userconfig.c index d9dd74982..4b6d27b80 100644 --- a/src/discover/userconfig.c +++ b/src/discover/userconfig.c @@ -219,8 +219,7 @@ static int parse_extra_extensions(yyjson_val *root, cbm_userext_t **entries, int } /* Grow the array */ - cbm_userext_t *tmp = - realloc(*entries, (size_t)(*count + 1) * sizeof(cbm_userext_t)); + cbm_userext_t *tmp = realloc(*entries, (size_t)(*count + 1) * sizeof(cbm_userext_t)); if (!tmp) { return -1; } @@ -297,10 +296,11 @@ cbm_userconfig_t *cbm_userconfig_load(const char *repo_path) { int count = 0; /* ── Step 1: Load global config ── */ + enum { PATH_BUF_SZ = 1280 }; char global_dir[1024]; cbm_app_config_dir(global_dir, sizeof(global_dir)); - char global_path[1280]; + char global_path[PATH_BUF_SZ]; snprintf(global_path, sizeof(global_path), "%s/config.json", global_dir); if (load_config_file(global_path, &entries, &count) != 0) { @@ -316,7 +316,7 @@ cbm_userconfig_t *cbm_userconfig_load(const char *repo_path) { /* ── Step 2: Load project config ── */ if (repo_path && repo_path[0]) { - char project_path[1280]; + char project_path[PATH_BUF_SZ]; snprintf(project_path, sizeof(project_path), "%s/.codebase-memory.json", repo_path); if (load_config_file(project_path, &entries, &count) != 0) { @@ -345,7 +345,6 @@ cbm_userconfig_t *cbm_userconfig_load(const char *repo_path) { entries[g] = entries[global_count - 1]; entries[global_count - 1].ext = NULL; /* mark as consumed */ global_count--; - g--; /* re-check this index */ break; } } diff --git a/src/discover/userconfig.h b/src/discover/userconfig.h index c45777fba..233734845 100644 --- a/src/discover/userconfig.h +++ b/src/discover/userconfig.h @@ -22,8 +22,8 @@ /* ── Types ──────────────────────────────────────────────────────── */ typedef struct { - char *ext; /* file extension including dot, e.g. ".blade.php" */ - CBMLanguage lang; /* resolved language enum */ + char *ext; /* file extension including dot, e.g. ".blade.php" */ + CBMLanguage lang; /* resolved language enum */ } cbm_userext_t; typedef struct { From 365858805169a2f06ee1e537a7d97cc73f8f4c57 Mon Sep 17 00:00:00 2001 From: Bing Ho Date: Sat, 21 Mar 2026 02:57:30 -0500 Subject: [PATCH 25/25] fix(platform): fall back to USERPROFILE when HOME is unset Add cbm_get_home_dir() in platform.c: tries HOME first, then USERPROFILE (Windows). Replace all 17 raw getenv("HOME") callsites. Fallback from "/tmp" to cbm_tmpdir() for cross-platform temp dir. Cherry-picked from PR #88 by @bingh0. Fixes #77. --- src/cli/cli.c | 20 ++++++++++---------- src/foundation/platform.c | 16 ++++++++++++++++ src/foundation/platform.h | 6 ++++++ src/main.c | 3 ++- src/mcp/mcp.c | 11 ++++------- src/pipeline/pipeline.c | 10 ++++------ src/store/store.c | 5 ++--- src/ui/config.c | 6 ++++-- src/ui/http_server.c | 22 +++++++++++----------- 9 files changed, 59 insertions(+), 40 deletions(-) diff --git a/src/cli/cli.c b/src/cli/cli.c index 77b8378f5..143bda9a1 100644 --- a/src/cli/cli.c +++ b/src/cli/cli.c @@ -7,6 +7,7 @@ #include "cli/cli.h" #include "foundation/compat.h" #include "foundation/str_util.h" +#include "foundation/platform.h" // the correct standard headers are included below but clang-tidy doesn't map them. #include @@ -1685,7 +1686,7 @@ unsigned char *cbm_extract_binary_from_targz(const unsigned char *data, int data static const char *get_cache_dir(const char *home_dir) { static char buf[1024]; if (!home_dir) { - home_dir = getenv("HOME"); + home_dir = cbm_get_home_dir(); } if (!home_dir) { return NULL; @@ -1909,10 +1910,9 @@ int cbm_cmd_config(int argc, char **argv) { return 0; } - // NOLINTNEXTLINE(concurrency-mt-unsafe) - const char *home = getenv("HOME"); + const char *home = cbm_get_home_dir(); if (!home) { - fprintf(stderr, "error: HOME not set\n"); + fprintf(stderr, "error: HOME not set (use USERPROFILE on Windows)\n"); return 1; } @@ -2140,9 +2140,9 @@ int cbm_cmd_install(int argc, char **argv) { } } - const char *home = getenv("HOME"); + const char *home = cbm_get_home_dir(); if (!home) { - fprintf(stderr, "error: HOME not set\n"); + fprintf(stderr, "error: HOME not set (use USERPROFILE on Windows)\n"); return 1; } @@ -2448,9 +2448,9 @@ int cbm_cmd_uninstall(int argc, char **argv) { } } - const char *home = getenv("HOME"); + const char *home = cbm_get_home_dir(); if (!home) { - fprintf(stderr, "error: HOME not set\n"); + fprintf(stderr, "error: HOME not set (use USERPROFILE on Windows)\n"); return 1; } @@ -2671,9 +2671,9 @@ int cbm_cmd_uninstall(int argc, char **argv) { int cbm_cmd_update(int argc, char **argv) { parse_auto_answer(argc, argv); - const char *home = getenv("HOME"); + const char *home = cbm_get_home_dir(); if (!home) { - fprintf(stderr, "error: HOME not set\n"); + fprintf(stderr, "error: HOME not set (use USERPROFILE on Windows)\n"); return 1; } diff --git a/src/foundation/platform.c b/src/foundation/platform.c index 2c33e4fe9..eb60f07ea 100644 --- a/src/foundation/platform.c +++ b/src/foundation/platform.c @@ -216,3 +216,19 @@ int64_t cbm_file_size(const char *path) { } #endif /* _WIN32 */ + +/* ── Home directory (cross-platform) ──────────────────────────── */ + +const char *cbm_get_home_dir(void) { + // NOLINTNEXTLINE(concurrency-mt-unsafe) + const char *h = getenv("HOME"); + if (h && h[0]) { + return h; + } + // NOLINTNEXTLINE(concurrency-mt-unsafe) + h = getenv("USERPROFILE"); + if (h && h[0]) { + return h; + } + return NULL; +} diff --git a/src/foundation/platform.h b/src/foundation/platform.h index ffa57fce8..e88ab068b 100644 --- a/src/foundation/platform.h +++ b/src/foundation/platform.h @@ -64,6 +64,12 @@ cbm_system_info_t cbm_system_info(void); * initial=false: max(1, perf_cores-1) (leave headroom for user apps) */ int cbm_default_worker_count(bool initial); +/* ── Home directory ─────────────────────────────────────────────── */ + +/* Cross-platform home directory: tries HOME first, then USERPROFILE (Windows). + * Returns NULL when neither is set. */ +const char *cbm_get_home_dir(void); + /* ── File system ───────────────────────────────────────────────── */ /* Check if a path exists. */ diff --git a/src/main.c b/src/main.c index 79618fad1..1070d36da 100644 --- a/src/main.c +++ b/src/main.c @@ -19,6 +19,7 @@ #include "store/store.h" #include "cli/cli.h" #include "foundation/log.h" +#include "foundation/platform.h" #include "foundation/compat_thread.h" #include "foundation/mem.h" #include "ui/config.h" @@ -221,7 +222,7 @@ int main(int argc, char **argv) { /* Open config store for runtime settings */ char config_dir[1024]; - const char *cfg_home = getenv("HOME"); + const char *cfg_home = cbm_get_home_dir(); cbm_config_t *runtime_config = NULL; if (cfg_home) { snprintf(config_dir, sizeof(config_dir), "%s/.cache/codebase-memory-mcp", cfg_home); diff --git a/src/mcp/mcp.c b/src/mcp/mcp.c index 80abc9ecb..0304ed4d6 100644 --- a/src/mcp/mcp.c +++ b/src/mcp/mcp.c @@ -607,10 +607,9 @@ bool cbm_mcp_server_has_cached_store(cbm_mcp_server_t *srv) { /* Returns the platform cache directory: ~/.cache/codebase-memory-mcp * Writes to buf, returns buf for convenience. */ static const char *cache_dir(char *buf, size_t bufsz) { - // NOLINTNEXTLINE(concurrency-mt-unsafe) - const char *home = getenv("HOME"); + const char *home = cbm_get_home_dir(); if (!home) { - home = "/tmp"; + home = cbm_tmpdir(); } snprintf(buf, bufsz, "%s/.cache/codebase-memory-mcp", home); return buf; @@ -2552,8 +2551,7 @@ static void detect_session(cbm_mcp_server_t *srv) { /* 1. Try CWD */ char cwd[1024]; if (getcwd(cwd, sizeof(cwd)) != NULL) { - // NOLINTNEXTLINE(concurrency-mt-unsafe) - const char *home = getenv("HOME"); + const char *home = cbm_get_home_dir(); /* Skip useless roots: / and $HOME */ if (strcmp(cwd, "/") != 0 && (home == NULL || strcmp(cwd, home) != 0)) { snprintf(srv->session_root, sizeof(srv->session_root), "%s", cwd); @@ -2624,8 +2622,7 @@ static void maybe_auto_index(cbm_mcp_server_t *srv) { } /* Check if project already has a DB */ - // NOLINTNEXTLINE(concurrency-mt-unsafe) - const char *home = getenv("HOME"); + const char *home = cbm_get_home_dir(); if (home) { char db_check[1024]; snprintf(db_check, sizeof(db_check), "%s/.cache/codebase-memory-mcp/%s.db", home, diff --git a/src/pipeline/pipeline.c b/src/pipeline/pipeline.c index e18c6126b..8b480d24d 100644 --- a/src/pipeline/pipeline.c +++ b/src/pipeline/pipeline.c @@ -137,10 +137,9 @@ static char *resolve_db_path(const cbm_pipeline_t *p) { if (p->db_path) { snprintf(path, 1024, "%s", p->db_path); } else { - // NOLINTNEXTLINE(concurrency-mt-unsafe) - const char *home = getenv("HOME"); + const char *home = cbm_get_home_dir(); if (!home) { - home = "/tmp"; + home = cbm_tmpdir(); } snprintf(path, 1024, "%s/.cache/codebase-memory-mcp/%s.db", home, p->project_name); } @@ -721,14 +720,13 @@ int cbm_pipeline_run(cbm_pipeline_t *p) { if (!check_cancel(p)) { cbm_clock_gettime(CLOCK_MONOTONIC, &t); - // NOLINTNEXTLINE(concurrency-mt-unsafe) — called once during single-threaded dump - const char *home = getenv("HOME"); + const char *home = cbm_get_home_dir(); char db_path[1024]; if (p->db_path) { snprintf(db_path, sizeof(db_path), "%s", p->db_path); } else { if (!home) { - home = "/tmp"; + home = cbm_tmpdir(); } snprintf(db_path, sizeof(db_path), "%s/.cache/codebase-memory-mcp/%s.db", home, p->project_name); diff --git a/src/store/store.c b/src/store/store.c index b25b7e87c..bfbff59a2 100644 --- a/src/store/store.c +++ b/src/store/store.c @@ -370,10 +370,9 @@ cbm_store_t *cbm_store_open(const char *project) { return NULL; } /* Build path: ~/.cache/codebase-memory-mcp/.db */ - const char *home = getenv("HOME"); // NOLINT(concurrency-mt-unsafe) — called once during - // single-threaded store open, never concurrently + const char *home = cbm_get_home_dir(); if (!home) { - home = "/tmp"; + home = cbm_tmpdir(); } char path[1024]; snprintf(path, sizeof(path), "%s/.cache/codebase-memory-mcp/%s.db", home, project); diff --git a/src/ui/config.c b/src/ui/config.c index 962d0ed72..7ac5b41c0 100644 --- a/src/ui/config.c +++ b/src/ui/config.c @@ -6,7 +6,9 @@ */ #include "ui/config.h" #include "foundation/log.h" +#include "foundation/platform.h" #include "foundation/compat_fs.h" +#include "foundation/compat.h" #include @@ -17,9 +19,9 @@ /* ── Path ────────────────────────────────────────────────────── */ void cbm_ui_config_path(char *buf, int bufsz) { - const char *home = getenv("HOME"); // NOLINT(concurrency-mt-unsafe) + const char *home = cbm_get_home_dir(); if (!home) { - home = "/tmp"; + home = cbm_tmpdir(); } snprintf(buf, (size_t)bufsz, "%s/.cache/codebase-memory-mcp/config.json", home); } diff --git a/src/ui/http_server.c b/src/ui/http_server.c index a2a30f374..b3153b24d 100644 --- a/src/ui/http_server.c +++ b/src/ui/http_server.c @@ -377,7 +377,7 @@ static void handle_browse(struct mg_connection *c, struct mg_http_message *hm) { char path[1024] = {0}; if (!get_query_param(hm->query, "path", path, (int)sizeof(path)) || path[0] == '\0') { /* Default to home directory */ - const char *home = getenv("HOME"); + const char *home = cbm_get_home_dir(); if (home) snprintf(path, sizeof(path), "%s", home); else @@ -452,9 +452,9 @@ static void handle_adr_get(struct mg_connection *c, struct mg_http_message *hm) return; } - const char *home = getenv("HOME"); + const char *home = cbm_get_home_dir(); if (!home) - home = "/tmp"; + home = cbm_tmpdir(); char db_path[1024]; snprintf(db_path, sizeof(db_path), "%s/.cache/codebase-memory-mcp/%s.db", home, name); @@ -545,9 +545,9 @@ static void handle_adr_save(struct mg_connection *c, struct mg_http_message *hm) const char *proj = yyjson_get_str(v_proj); const char *content = yyjson_get_str(v_content); - const char *home = getenv("HOME"); + const char *home = cbm_get_home_dir(); if (!home) - home = "/tmp"; + home = cbm_tmpdir(); char db_path[1024]; snprintf(db_path, sizeof(db_path), "%s/.cache/codebase-memory-mcp/%s.db", home, proj); @@ -832,9 +832,9 @@ static void handle_delete_project(struct mg_connection *c, struct mg_http_messag return; } - const char *home = getenv("HOME"); // NOLINT(concurrency-mt-unsafe) + const char *home = cbm_get_home_dir(); if (!home) - home = "/tmp"; + home = cbm_tmpdir(); char db_path[1024]; snprintf(db_path, sizeof(db_path), "%s/.cache/codebase-memory-mcp/%s.db", home, name); @@ -870,9 +870,9 @@ static void handle_project_health(struct mg_connection *c, struct mg_http_messag return; } - const char *home = getenv("HOME"); // NOLINT(concurrency-mt-unsafe) + const char *home = cbm_get_home_dir(); if (!home) - home = "/tmp"; + home = cbm_tmpdir(); char db_path[1024]; snprintf(db_path, sizeof(db_path), "%s/.cache/codebase-memory-mcp/%s.db", home, name); @@ -928,9 +928,9 @@ static void handle_layout(struct mg_connection *c, struct mg_http_message *hm) { } /* Open a read-only store for this project */ - const char *home = getenv("HOME"); // NOLINT(concurrency-mt-unsafe) + const char *home = cbm_get_home_dir(); if (!home) - home = "/tmp"; + home = cbm_tmpdir(); char db_path[1024]; snprintf(db_path, sizeof(db_path), "%s/.cache/codebase-memory-mcp/%s.db", home, project);