diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 0f3aae509..ac0981b3f 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -85,14 +85,45 @@ Language support is split between two layers: 4. Add a test case in `tests/test_pipeline.c` for integration-level fixes 5. Verify with a real open-source repo +### Infrastructure Languages (Infra-Pass Pattern) + +Languages like **Dockerfile**, **docker-compose**, **Kubernetes manifests**, and **Kustomize** do not require a new tree-sitter grammar. Instead they follow an *infra-pass* pattern, reusing the existing tree-sitter YAML grammar where applicable: + +1. **Detection helpers** in `src/pipeline/pass_infrascan.c` — functions like `cbm_is_dockerfile()`, `cbm_is_k8s_manifest()`, `cbm_is_kustomize_file()` identify files by name and/or content heuristics (e.g., presence of `apiVersion:`). +2. **Custom extractors** in `internal/cbm/extract_k8s.c` — tree-sitter-based parsers that walk the YAML AST (using the tree-sitter YAML grammar) and populate `CBMFileResult` with imports and definitions. +3. **Pipeline pass** (`pass_k8s.c`, `pass_infrascan.c`) — calls the extractor and emits graph nodes/edges. K8s manifests emit `Resource` nodes; Kustomize files emit `Module` nodes with `IMPORTS` edges to referenced resource files. + +**When adding a new infrastructure language:** +- Add a detection helper (`cbm_is__file()`) in `pass_infrascan.c` or a new `pass_.c`. +- Add the `CBM_LANG_` enum value in `internal/cbm/cbm.h` and a row in the language table in `lang_specs.c`. +- Write a custom extractor that returns `CBMFileResult*` — do not add a tree-sitter grammar. +- Register the pass in `pipeline.c`. +- Add tests in `tests/test_pipeline.c` following the `TEST(infra_is_dockerfile)` and `TEST(k8s_extract_manifest)` patterns. + +## Commit Format + +Use conventional commits: `type(scope): description` + +| Type | When to use | +|------|-------------| +| `feat` | New feature or capability | +| `fix` | Bug fix | +| `test` | Adding or updating tests | +| `refactor` | Code change that neither fixes a bug nor adds a feature | +| `perf` | Performance improvement | +| `docs` | Documentation only | +| `chore` | Build scripts, CI, dependency updates | + +Examples: `fix(store): set busy_timeout before WAL`, `feat(cli): add --progress flag` + ## Pull Request Guidelines +- **One issue per PR.** Each PR must address exactly one bug, one feature, or one refactor. Do not bundle multiple fixes or feature additions into a single PR. If your change touches multiple areas, split it into separate PRs. +- **Open an issue first.** Every PR should reference a tracking issue (`Fixes #N` or `Closes #N`). This ensures the change is discussed before code is written. - **C code only** — this project was rewritten from Go to pure C in v0.5.0. Go PRs will be acknowledged and potentially ported, but cannot be merged directly. -- One logical change per PR — don't bundle unrelated features - Include tests for new functionality - Run `scripts/test.sh` and `scripts/lint.sh` before submitting - Keep PRs focused — avoid unrelated reformatting or refactoring -- Reference the issue number in your PR description ## Security diff --git a/Makefile.cbm b/Makefile.cbm index 2bd843d73..d149f0a1f 100644 --- a/Makefile.cbm +++ b/Makefile.cbm @@ -115,6 +115,7 @@ EXTRACTION_SRCS = \ $(CBM_DIR)/extract_type_refs.c \ $(CBM_DIR)/extract_type_assigns.c \ $(CBM_DIR)/extract_env_accesses.c \ + $(CBM_DIR)/extract_k8s.c \ $(CBM_DIR)/helpers.c \ $(CBM_DIR)/lang_specs.c @@ -148,6 +149,7 @@ MCP_SRCS = src/mcp/mcp.c # Discover module (new) DISCOVER_SRCS = \ src/discover/language.c \ + src/discover/userconfig.c \ src/discover/gitignore.c \ src/discover/discover.c @@ -176,6 +178,7 @@ PIPELINE_SRCS = \ src/pipeline/pass_envscan.c \ src/pipeline/pass_compile_commands.c \ src/pipeline/pass_infrascan.c \ + src/pipeline/pass_k8s.c \ src/pipeline/httplink.c # Traces module (new) @@ -259,6 +262,7 @@ TEST_MCP_SRCS = \ TEST_DISCOVER_SRCS = \ tests/test_language.c \ + tests/test_userconfig.c \ tests/test_gitignore.c \ tests/test_discover.c diff --git a/README.md b/README.md index 8a4ffb3b8..4c3c566f3 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ [![License](https://img.shields.io/badge/license-MIT-green)](LICENSE) [![CI](https://img.shields.io/github/actions/workflow/status/DeusData/codebase-memory-mcp/dry-run.yml?label=CI)](https://github.com/DeusData/codebase-memory-mcp/actions/workflows/dry-run.yml) [![Tests](https://img.shields.io/badge/tests-2042_passing-brightgreen)](https://github.com/DeusData/codebase-memory-mcp) -[![Languages](https://img.shields.io/badge/languages-64-orange)](https://github.com/DeusData/codebase-memory-mcp) +[![Languages](https://img.shields.io/badge/languages-66-orange)](https://github.com/DeusData/codebase-memory-mcp) [![Agents](https://img.shields.io/badge/agents-10-purple)](https://github.com/DeusData/codebase-memory-mcp) [![Pure C](https://img.shields.io/badge/pure_C-zero_dependencies-blue)](https://github.com/DeusData/codebase-memory-mcp) [![Platform](https://img.shields.io/badge/macOS_%7C_Linux_%7C_Windows-supported-lightgrey)](https://github.com/DeusData/codebase-memory-mcp/releases/latest) @@ -12,7 +12,7 @@ **The fastest and most efficient code intelligence engine for AI coding agents.** Full-indexes an average repository in milliseconds, the Linux kernel (28M LOC, 75K files) in 3 minutes. Answers structural queries in under 1ms. Ships as a single static binary for macOS, Linux, and Windows — download, run `install`, done. -High-quality parsing through [tree-sitter](https://tree-sitter.github.io/tree-sitter/) AST analysis across all 64 languages, enhanced with LSP-style hybrid type resolution for Go, C, and C++ (more languages coming soon) — producing a persistent knowledge graph of functions, classes, call chains, HTTP routes, and cross-service links. 14 MCP tools. Zero dependencies. Plug and play across 10 coding agents. +High-quality parsing through [tree-sitter](https://tree-sitter.github.io/tree-sitter/) AST analysis across all 66 languages, enhanced with LSP-style hybrid type resolution for Go, C, and C++ (more languages coming soon) — producing a persistent knowledge graph of functions, classes, call chains, HTTP routes, and cross-service links. 14 MCP tools. Zero dependencies. Plug and play across 10 coding agents.

Graph visualization UI showing the codebase-memory-mcp knowledge graph @@ -24,10 +24,11 @@ High-quality parsing through [tree-sitter](https://tree-sitter.github.io/tree-si - **Extreme indexing speed** — Linux kernel (28M LOC, 75K files) in 3 minutes. RAM-first pipeline: LZ4 compression, in-memory SQLite, fused Aho-Corasick pattern matching. Memory released after indexing. - **Plug and play** — single static binary for macOS (arm64/amd64), Linux (arm64/amd64), and Windows (amd64). No Docker, no runtime dependencies, no API keys. Download → `install` → restart agent → done. -- **64 languages** — vendored tree-sitter grammars compiled into the binary. Nothing to install, nothing that breaks. +- **66 languages** — vendored tree-sitter grammars compiled into the binary. Nothing to install, nothing that breaks. - **120x fewer tokens** — 5 structural queries: ~3,400 tokens vs ~412,000 via file-by-file search. One graph query replaces dozens of grep/read cycles. - **10 agents, one command** — `install` auto-detects Claude Code, Codex CLI, Gemini CLI, Zed, OpenCode, Antigravity, Aider, KiloCode, VS Code, and OpenClaw — configures MCP entries, instruction files, and pre-tool hooks for each. - **Built-in graph visualization** — 3D interactive UI at `localhost:9749` (optional UI binary variant). +- **Infrastructure-as-code indexing** — Dockerfiles, Kubernetes manifests, and Kustomize overlays indexed as graph nodes with cross-references. `Resource` nodes for K8s kinds, `Module` nodes for Kustomize overlays with `IMPORTS` edges to referenced resources. - **14 MCP tools** — search, trace, architecture, impact analysis, Cypher queries, dead code detection, cross-service HTTP linking, ADR management, and more. ## Quick Start @@ -279,7 +280,7 @@ codebase-memory-mcp cli --raw search_graph '{"label": "Function"}' | jq '.result ### Node Labels -`Project`, `Package`, `Folder`, `File`, `Module`, `Class`, `Function`, `Method`, `Interface`, `Enum`, `Type`, `Route` +`Project`, `Package`, `Folder`, `File`, `Module`, `Class`, `Function`, `Method`, `Interface`, `Enum`, `Type`, `Route`, `Resource` ### Edge Types @@ -306,6 +307,24 @@ codebase-memory-mcp config set auto_index_limit 50000 # max files for auto-in codebase-memory-mcp config reset auto_index # reset to default ``` +## Custom File Extensions + +Map additional file extensions to supported languages via JSON config files. Useful for framework-specific extensions like `.blade.php` (Laravel) or `.mjs` (ES modules). + +**Per-project** (in your repo root): +```json +// .codebase-memory.json +{"extra_extensions": {".blade.php": "php", ".mjs": "javascript"}} +``` + +**Global** (applies to all projects): +```json +// ~/.config/codebase-memory-mcp/config.json (or $XDG_CONFIG_HOME/...) +{"extra_extensions": {".twig": "html", ".phtml": "php"}} +``` + +Project config overrides global for conflicting extensions. Unknown language values are silently skipped. Missing config files are ignored. + ## Persistence SQLite databases stored at `~/.cache/codebase-memory-mcp/`. Persists across restarts (WAL mode, ACID-safe). To reset: `rm -rf ~/.cache/codebase-memory-mcp/`. @@ -323,7 +342,7 @@ SQLite databases stored at `~/.cache/codebase-memory-mcp/`. Persists across rest ## Language Support -64 languages. Benchmarked against 64 real open-source repositories (78 to 49K nodes): +66 languages. Benchmarked against 64 real open-source repositories (78 to 49K nodes): | Tier | Score | Languages | |------|-------|-----------| @@ -348,7 +367,7 @@ src/ traces/ Runtime trace ingestion ui/ Embedded HTTP server + 3D graph visualization foundation/ Platform abstractions (threads, filesystem, logging, memory) -internal/cbm/ Vendored tree-sitter grammars (64 languages) + AST extraction engine +internal/cbm/ Vendored tree-sitter grammars (66 languages) + AST extraction engine ``` ## License diff --git a/internal/cbm/cbm.c b/internal/cbm/cbm.c index 6162c7a8a..5b70d1858 100644 --- a/internal/cbm/cbm.c +++ b/internal/cbm/cbm.c @@ -316,6 +316,11 @@ CBMFileResult *cbm_extract_file(const char *source, int source_len, CBMLanguage cbm_extract_imports(&ctx); cbm_extract_unified(&ctx); + // K8s / Kustomize semantic pass (additional structured extraction for YAML-based infra files). + if (ctx.language == CBM_LANG_KUSTOMIZE || ctx.language == CBM_LANG_K8S) { + cbm_extract_k8s(&ctx); + } + // LSP type-aware call resolution uint64_t lsp_start = now_ns(); if (language == CBM_LANG_GO) { diff --git a/internal/cbm/cbm.h b/internal/cbm/cbm.h index 16b9dd083..6b49ae7fc 100644 --- a/internal/cbm/cbm.h +++ b/internal/cbm/cbm.h @@ -75,6 +75,8 @@ typedef enum { CBM_LANG_FORM, CBM_LANG_MAGMA, CBM_LANG_WOLFRAM, + CBM_LANG_KUSTOMIZE, // kustomization.yaml — Kubernetes overlay tool + CBM_LANG_K8S, // Generic Kubernetes manifest (apiVersion: detected) CBM_LANG_COUNT } CBMLanguage; @@ -361,4 +363,7 @@ void cbm_extract_type_assigns(CBMExtractCtx *ctx); // Single-pass unified extraction (replaces the 7 calls above except defs+imports). void cbm_extract_unified(CBMExtractCtx *ctx); +// K8s / Kustomize semantic extractor (called when language is CBM_LANG_K8S or CBM_LANG_KUSTOMIZE). +void cbm_extract_k8s(CBMExtractCtx *ctx); + #endif // CBM_H diff --git a/internal/cbm/extract_k8s.c b/internal/cbm/extract_k8s.c new file mode 100644 index 000000000..26bbb693a --- /dev/null +++ b/internal/cbm/extract_k8s.c @@ -0,0 +1,290 @@ +// extract_k8s.c — K8s manifest and Kustomize file extractor. +// +// For CBM_LANG_KUSTOMIZE: walks top-level block_mapping_pair nodes whose key +// matches "resources", "bases", "patches", "components", or +// "patchesStrategicMerge", then emits one CBMImport per block_sequence item. +// +// For CBM_LANG_K8S: finds apiVersion, kind, and metadata.name scalars in the +// first document's block_mapping and emits one CBMDefinition with label +// "Resource" and name "Kind/metadata-name". + +#include "cbm.h" +#include "arena.h" +#include "helpers.h" +#include "tree_sitter/api.h" +#include +#include +#include + +// --------------------------------------------------------------------------- +// Internal helpers +// --------------------------------------------------------------------------- + +// Return the raw source text for a scalar node (plain, single-quoted, or +// double-quoted). Surrounding quote characters are stripped for quoted forms. +// Handles flow_node wrappers transparently by descending into the first named +// child (the tree-sitter YAML grammar often wraps scalars in flow_node). +// Returns NULL for non-scalar node types. +static const char *get_scalar_text(CBMArena *a, TSNode node, const char *source) { + const char *type = ts_node_type(node); + // Unwrap flow_node: the actual scalar is the first named child + if (strcmp(type, "flow_node") == 0) { + TSNode inner = ts_node_named_child(node, 0); + if (ts_node_is_null(inner)) { + return NULL; + } + return get_scalar_text(a, inner, source); + } + if (strcmp(type, "plain_scalar") == 0) { + return cbm_node_text(a, node, source); + } + if (strcmp(type, "double_quote_scalar") == 0 || strcmp(type, "single_quote_scalar") == 0) { + const char *raw = cbm_node_text(a, node, source); + if (!raw) { + return NULL; + } + size_t len = strlen(raw); + if (len >= 2) { + return cbm_arena_strndup(a, raw + 1, len - 2); + } + return raw; + } + return NULL; +} + +// Return true if the key text of a block_mapping_pair matches one of the +// Kustomize resource-list field names. +static int is_kustomize_list_key(const char *key) { + return (strcmp(key, "resources") == 0 || strcmp(key, "bases") == 0 || + strcmp(key, "patches") == 0 || strcmp(key, "components") == 0 || + strcmp(key, "patchesStrategicMerge") == 0 || strcmp(key, "crds") == 0); +} + +// --------------------------------------------------------------------------- +// Kustomize extraction +// --------------------------------------------------------------------------- + +// Walk a block_sequence node and emit one CBMImport per block_sequence_item +// scalar child, using key_name as the local_name. +static void emit_kustomize_sequence(CBMExtractCtx *ctx, TSNode seq_node, const char *key_name) { + CBMArena *a = ctx->arena; + uint32_t n = ts_node_child_count(seq_node); + for (uint32_t i = 0; i < n; i++) { + TSNode item = ts_node_child(seq_node, i); + if (strcmp(ts_node_type(item), "block_sequence_item") != 0) { + continue; + } + // block_sequence_item has one named child: the value + uint32_t ic = ts_node_child_count(item); + for (uint32_t j = 0; j < ic; j++) { + TSNode val = ts_node_child(item, j); + const char *scalar = get_scalar_text(a, val, ctx->source); + if (!scalar) { + continue; + } + CBMImport imp = { + .local_name = cbm_arena_strdup(a, key_name), + .module_path = cbm_arena_strdup(a, scalar), + }; + cbm_imports_push(&ctx->result->imports, a, imp); + } + } +} + +static void extract_kustomize(CBMExtractCtx *ctx) { + CBMArena *a = ctx->arena; + + // Traverse: stream -> document -> block_node -> block_mapping -> block_mapping_pair + TSNode root = ctx->root; + uint32_t root_n = ts_node_child_count(root); + for (uint32_t si = 0; si < root_n; si++) { + TSNode stream_child = ts_node_child(root, si); + if (strcmp(ts_node_type(stream_child), "document") != 0) { + continue; + } + // Find block_mapping inside the document (may be wrapped in block_node) + TSNode mapping = ts_node_named_child(stream_child, 0); + if (ts_node_is_null(mapping)) { + continue; + } + // Some grammars wrap in block_node + if (strcmp(ts_node_type(mapping), "block_node") == 0) { + mapping = ts_node_named_child(mapping, 0); + } + if (ts_node_is_null(mapping) || strcmp(ts_node_type(mapping), "block_mapping") != 0) { + continue; + } + + uint32_t pair_n = ts_node_child_count(mapping); + for (uint32_t pi = 0; pi < pair_n; pi++) { + TSNode pair = ts_node_child(mapping, pi); + if (strcmp(ts_node_type(pair), "block_mapping_pair") != 0) { + continue; + } + + // First named child = key + TSNode key_node = ts_node_named_child(pair, 0); + if (ts_node_is_null(key_node)) { + continue; + } + const char *key_text = get_scalar_text(a, key_node, ctx->source); + if (!key_text || !is_kustomize_list_key(key_text)) { + continue; + } + + // Second named child = value (should be a block_sequence or block_node wrapping one) + TSNode val_node = ts_node_named_child(pair, 1); + if (ts_node_is_null(val_node)) { + continue; + } + if (strcmp(ts_node_type(val_node), "block_node") == 0) { + val_node = ts_node_named_child(val_node, 0); + } + if (ts_node_is_null(val_node) || + strcmp(ts_node_type(val_node), "block_sequence") != 0) { + continue; + } + + emit_kustomize_sequence(ctx, val_node, key_text); + } + } +} + +// --------------------------------------------------------------------------- +// K8s manifest extraction +// --------------------------------------------------------------------------- + +// Descend into the first block_mapping of a document and extract apiVersion, +// kind, and metadata.name. Returns void; fills kind_buf and meta_name_buf. +static void extract_k8s_scalars(CBMExtractCtx *ctx, TSNode mapping, char *kind_buf, size_t kind_sz, + char *meta_name_buf, size_t meta_sz) { + CBMArena *a = ctx->arena; + kind_buf[0] = '\0'; + meta_name_buf[0] = '\0'; + + uint32_t n = ts_node_child_count(mapping); + for (uint32_t i = 0; i < n; i++) { + TSNode pair = ts_node_child(mapping, i); + if (strcmp(ts_node_type(pair), "block_mapping_pair") != 0) { + continue; + } + TSNode key_node = ts_node_named_child(pair, 0); + if (ts_node_is_null(key_node)) { + continue; + } + const char *key = get_scalar_text(a, key_node, ctx->source); + if (!key) { + continue; + } + + TSNode val_node = ts_node_named_child(pair, 1); + if (ts_node_is_null(val_node)) { + continue; + } + // Unwrap block_node if present + if (strcmp(ts_node_type(val_node), "block_node") == 0) { + val_node = ts_node_named_child(val_node, 0); + } + if (ts_node_is_null(val_node)) { + continue; + } + + if (strcmp(key, "kind") == 0) { + const char *v = get_scalar_text(a, val_node, ctx->source); + if (v) { + snprintf(kind_buf, kind_sz, "%s", v); + } + } else if (strcmp(key, "metadata") == 0) { + // Descend into metadata block_mapping to find "name" + // val_node is already unwrapped from block_node above. + TSNode meta_mapping = val_node; + if (ts_node_is_null(meta_mapping) || + strcmp(ts_node_type(meta_mapping), "block_mapping") != 0) { + continue; + } + uint32_t mn = ts_node_child_count(meta_mapping); + for (uint32_t mi = 0; mi < mn; mi++) { + TSNode mpair = ts_node_child(meta_mapping, mi); + if (strcmp(ts_node_type(mpair), "block_mapping_pair") != 0) { + continue; + } + TSNode mkey = ts_node_named_child(mpair, 0); + if (ts_node_is_null(mkey)) { + continue; + } + const char *mkey_text = get_scalar_text(a, mkey, ctx->source); + if (!mkey_text || strcmp(mkey_text, "name") != 0) { + continue; + } + TSNode mval = ts_node_named_child(mpair, 1); + if (ts_node_is_null(mval)) { + continue; + } + const char *meta_name = get_scalar_text(a, mval, ctx->source); + if (meta_name) { + snprintf(meta_name_buf, meta_sz, "%s", meta_name); + } + } + } + } +} + +static void extract_k8s_manifest(CBMExtractCtx *ctx) { + CBMArena *a = ctx->arena; + + TSNode root = ctx->root; + uint32_t root_n = ts_node_child_count(root); + for (uint32_t si = 0; si < root_n; si++) { + TSNode stream_child = ts_node_child(root, si); + if (strcmp(ts_node_type(stream_child), "document") != 0) { + continue; + } + + TSNode mapping = ts_node_named_child(stream_child, 0); + if (ts_node_is_null(mapping)) { + continue; + } + if (strcmp(ts_node_type(mapping), "block_node") == 0) { + mapping = ts_node_named_child(mapping, 0); + } + if (ts_node_is_null(mapping) || strcmp(ts_node_type(mapping), "block_mapping") != 0) { + continue; + } + + char kind_buf[256] = {0}; + char meta_name_buf[256] = {0}; + extract_k8s_scalars(ctx, mapping, kind_buf, sizeof(kind_buf), meta_name_buf, + sizeof(meta_name_buf)); + + // Skip malformed manifests (no kind or no metadata.name) + if (kind_buf[0] == '\0' || meta_name_buf[0] == '\0') { + continue; + } + + char def_name[512]; + snprintf(def_name, sizeof(def_name), "%s/%s", kind_buf, meta_name_buf); + + CBMDefinition def = {0}; + def.name = cbm_arena_strdup(a, def_name); + def.qualified_name = cbm_arena_sprintf(a, "%s.%s", ctx->module_qn, def_name); + def.label = cbm_arena_strdup(a, "Resource"); + def.file_path = ctx->rel_path; + def.start_line = ts_node_start_point(mapping).row + 1; + def.end_line = ts_node_end_point(mapping).row + 1; + cbm_defs_push(&ctx->result->defs, a, def); + + break; // Only the first document per file + } +} + +// --------------------------------------------------------------------------- +// Public entry point +// --------------------------------------------------------------------------- + +void cbm_extract_k8s(CBMExtractCtx *ctx) { + if (ctx->language == CBM_LANG_KUSTOMIZE) { + extract_kustomize(ctx); + } else if (ctx->language == CBM_LANG_K8S) { + extract_k8s_manifest(ctx); + } +} diff --git a/internal/cbm/lang_specs.c b/internal/cbm/lang_specs.c index a4fb8097c..426db947b 100644 --- a/internal/cbm/lang_specs.c +++ b/internal/cbm/lang_specs.c @@ -1041,6 +1041,16 @@ static const CBMLangSpec lang_specs[CBM_LANG_COUNT] = { {CBM_LANG_WOLFRAM, wolfram_func_types, empty_types, empty_types, wolfram_module_types, wolfram_call_types, wolfram_import_types, empty_types, empty_types, empty_types, empty_types, empty_types, NULL, empty_types, NULL, NULL}, + + // CBM_LANG_KUSTOMIZE — reuses YAML grammar; semantic extraction via cbm_extract_k8s() + {CBM_LANG_KUSTOMIZE, empty_types, empty_types, empty_types, yaml_module_types, empty_types, + empty_types, empty_types, empty_types, empty_types, empty_types, empty_types, NULL, + empty_types, NULL, NULL}, + + // CBM_LANG_K8S — reuses YAML grammar; semantic extraction via cbm_extract_k8s() + {CBM_LANG_K8S, empty_types, empty_types, empty_types, yaml_module_types, empty_types, + empty_types, empty_types, empty_types, empty_types, empty_types, empty_types, NULL, + empty_types, NULL, NULL}, }; const CBMLangSpec *cbm_lang_spec(CBMLanguage lang) { @@ -1180,6 +1190,9 @@ const TSLanguage *cbm_ts_language(CBMLanguage lang) { return tree_sitter_magma(); case CBM_LANG_WOLFRAM: return tree_sitter_wolfram(); + case CBM_LANG_KUSTOMIZE: + case CBM_LANG_K8S: + return tree_sitter_yaml(); default: return NULL; } diff --git a/scripts/test_mcp_rapid_init.py b/scripts/test_mcp_rapid_init.py new file mode 100755 index 000000000..fe4f65d5f --- /dev/null +++ b/scripts/test_mcp_rapid_init.py @@ -0,0 +1,98 @@ +#!/usr/bin/env python3 +"""Integration test for the poll/getline FILE* buffering fix. + +Spawns the MCP server binary, sends initialize + notifications/initialized + +tools/list all at once (no delays), and asserts that the tools/list response +arrives within 5 seconds. + +Usage: + python3 scripts/test_mcp_rapid_init.py [/path/to/binary] + +Exit codes: + 0 - PASS + 1 - FAIL +""" + +import subprocess +import sys +import os + +TIMEOUT_S = 5 + +MESSAGES = ( + b'{"jsonrpc":"2.0","id":1,"method":"initialize",' + b'"params":{"protocolVersion":"2025-11-25","capabilities":{}}}\n' + b'{"jsonrpc":"2.0","method":"notifications/initialized"}\n' + b'{"jsonrpc":"2.0","id":2,"method":"tools/list","params":{}}\n' +) + + +def main(): + if len(sys.argv) >= 2: + binary = sys.argv[1] + else: + # Default: look for build artifact relative to this script's directory + script_dir = os.path.dirname(os.path.abspath(__file__)) + repo_root = os.path.dirname(script_dir) + binary = os.path.join(repo_root, "build", "c", "codebase-memory-mcp") + + if not os.path.isfile(binary): + print(f"FAIL: binary not found at {binary}") + sys.exit(1) + + if not os.access(binary, os.X_OK): + print(f"FAIL: binary not executable: {binary}") + sys.exit(1) + + proc = subprocess.Popen( + [binary], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + ) + + try: + # Write all 3 messages in one call and close stdin to signal EOF + stdout_data, _ = proc.communicate(input=MESSAGES, timeout=TIMEOUT_S) + except subprocess.TimeoutExpired: + proc.kill() + proc.wait() + print( + f"FAIL: server did not respond within {TIMEOUT_S}s " + f"(poll/getline buffering bug not fixed)" + ) + sys.exit(1) + + output = stdout_data.decode("utf-8", errors="replace") + + # Expect exactly 2 JSON responses: id:1 (initialize) and id:2 (tools/list). + # notifications/initialized has no id and produces no response. + lines = [ln.strip() for ln in output.splitlines() if ln.strip()] + import json as _json + json_lines = [] + for ln in lines: + try: + json_lines.append(_json.loads(ln)) + except _json.JSONDecodeError: + pass + + ids = {obj.get("id") for obj in json_lines if "id" in obj} + if 1 not in ids: + print("FAIL: missing initialize response (id:1) in server output") + print(f"Server output was:\n{output!r}") + sys.exit(1) + if 2 not in ids: + print("FAIL: missing tools/list response (id:2) in server output") + print(f"Server output was:\n{output!r}") + sys.exit(1) + if "tools" not in output: + print("FAIL: tools/list response body missing 'tools' key") + print(f"Server output was:\n{output!r}") + sys.exit(1) + + print("PASS") + sys.exit(0) + + +if __name__ == "__main__": + main() diff --git a/server.json b/server.json index 2b6e81083..e85fc90e1 100644 --- a/server.json +++ b/server.json @@ -2,7 +2,7 @@ "$schema": "https://static.modelcontextprotocol.io/schemas/2025-12-11/server.schema.json", "name": "io.github.DeusData/codebase-memory-mcp", "title": "Codebase Memory", - "description": "Persistent codebase knowledge graph — 64 languages, sub-ms queries, 99% fewer tokens than grep. Survives session restarts and context compaction.", + "description": "Persistent codebase knowledge graph — 66 languages, sub-ms queries, 99% fewer tokens than grep. Survives session restarts and context compaction.", "repository": { "url": "https://github.com/DeusData/codebase-memory-mcp", "source": "github" diff --git a/src/cli/cli.c b/src/cli/cli.c index ba58e5381..143bda9a1 100644 --- a/src/cli/cli.c +++ b/src/cli/cli.c @@ -7,6 +7,7 @@ #include "cli/cli.h" #include "foundation/compat.h" #include "foundation/str_util.h" +#include "foundation/platform.h" // the correct standard headers are included below but clang-tidy doesn't map them. #include @@ -1310,11 +1311,7 @@ int cbm_remove_antigravity_mcp(const char *config_path) { /* ── Claude Code pre-tool hooks ───────────────────────────────── */ #define CMM_HOOK_MATCHER "Grep|Glob|Read|Search" -#define CMM_HOOK_COMMAND \ - "echo 'Reminder: prefer codebase-memory-mcp search_graph/trace_call_path/" \ - "get_code_snippet over Grep/Glob/Read/Search for code discovery. " \ - "Use get_code_snippet(qualified_name) instead of Read to view a function. " \ - "Fall back only if MCP returns insufficient results.' >&2" +#define CMM_HOOK_COMMAND "~/.claude/hooks/cbm-code-discovery-gate" /* Old matcher values from previous versions — recognized during upgrade so * upsert_hooks_json can remove them before inserting the current matcher. */ @@ -1470,6 +1467,45 @@ int cbm_remove_claude_hooks(const char *settings_path) { return remove_hooks_json(settings_path, "PreToolUse", CMM_HOOK_MATCHER); } +/* Install the code discovery gate script to ~/.claude/hooks/. + * Blocks the first Grep/Glob/Read/Search call per session (exit 2 + stderr), + * nudging Claude toward codebase-memory-mcp. All subsequent calls in the same + * session pass through (gate file keyed on PPID). */ +static void cbm_install_hook_gate_script(const char *home) { + if (!home) { + return; + } + char hooks_dir[1024]; + snprintf(hooks_dir, sizeof(hooks_dir), "%s/.claude/hooks", home); + cbm_mkdir_p(hooks_dir, 0755); + + char script_path[1024]; + snprintf(script_path, sizeof(script_path), "%s/cbm-code-discovery-gate", hooks_dir); + + FILE *f = fopen(script_path, "w"); + if (!f) { + return; + } + fprintf(f, "#!/bin/bash\n" + "# Gate hook: nudges Claude toward codebase-memory-mcp for code discovery.\n" + "# First Grep/Glob/Read/Search per session -> block. Subsequent -> allow.\n" + "# PPID = Claude Code process PID, unique per session.\n" + "GATE=/tmp/cbm-code-discovery-gate-$PPID\n" + "find /tmp -name 'cbm-code-discovery-gate-*' -mtime +1 -delete 2>/dev/null\n" + "if [ -f \"$GATE\" ]; then\n" + " exit 0\n" + "fi\n" + "touch \"$GATE\"\n" + "echo 'BLOCKED: For code discovery, use codebase-memory-mcp tools first: " + "search_graph(name_pattern) to find functions/classes, trace_call_path() for " + "call chains, get_code_snippet(qualified_name) to read source. If the graph " + "is not indexed yet, call index_repository first. Fall back to Grep/Glob/Read " + "only for text content search. If you need Grep, retry.' >&2\n" + "exit 2\n"); + fclose(f); + chmod(script_path, 0755); +} + #define GEMINI_HOOK_MATCHER "google_search|read_file|grep_search" #define GEMINI_HOOK_COMMAND \ "echo 'Reminder: prefer codebase-memory-mcp search_graph/trace_call_path/" \ @@ -1650,7 +1686,7 @@ unsigned char *cbm_extract_binary_from_targz(const unsigned char *data, int data static const char *get_cache_dir(const char *home_dir) { static char buf[1024]; if (!home_dir) { - home_dir = getenv("HOME"); + home_dir = cbm_get_home_dir(); } if (!home_dir) { return NULL; @@ -1874,10 +1910,9 @@ int cbm_cmd_config(int argc, char **argv) { return 0; } - // NOLINTNEXTLINE(concurrency-mt-unsafe) - const char *home = getenv("HOME"); + const char *home = cbm_get_home_dir(); if (!home) { - fprintf(stderr, "error: HOME not set\n"); + fprintf(stderr, "error: HOME not set (use USERPROFILE on Windows)\n"); return 1; } @@ -2105,9 +2140,9 @@ int cbm_cmd_install(int argc, char **argv) { } } - const char *home = getenv("HOME"); + const char *home = cbm_get_home_dir(); if (!home) { - fprintf(stderr, "error: HOME not set\n"); + fprintf(stderr, "error: HOME not set (use USERPROFILE on Windows)\n"); return 1; } @@ -2218,13 +2253,14 @@ int cbm_cmd_install(int argc, char **argv) { } printf(" mcp: %s\n", mcp_path2); - /* PreToolUse hook */ + /* PreToolUse hook + gate script */ char settings_path[1024]; snprintf(settings_path, sizeof(settings_path), "%s/.claude/settings.json", home); if (!dry_run) { cbm_upsert_claude_hooks(settings_path); + cbm_install_hook_gate_script(home); } - printf(" hooks: PreToolUse (Grep|Glob reminder)\n"); + printf(" hooks: PreToolUse (code discovery gate)\n"); } /* Step 5: Install Codex CLI */ @@ -2412,9 +2448,9 @@ int cbm_cmd_uninstall(int argc, char **argv) { } } - const char *home = getenv("HOME"); + const char *home = cbm_get_home_dir(); if (!home) { - fprintf(stderr, "error: HOME not set\n"); + fprintf(stderr, "error: HOME not set (use USERPROFILE on Windows)\n"); return 1; } @@ -2635,9 +2671,9 @@ int cbm_cmd_uninstall(int argc, char **argv) { int cbm_cmd_update(int argc, char **argv) { parse_auto_answer(argc, argv); - const char *home = getenv("HOME"); + const char *home = cbm_get_home_dir(); if (!home) { - fprintf(stderr, "error: HOME not set\n"); + fprintf(stderr, "error: HOME not set (use USERPROFILE on Windows)\n"); return 1; } diff --git a/src/discover/language.c b/src/discover/language.c index b7eb7e439..0efc28f65 100644 --- a/src/discover/language.c +++ b/src/discover/language.c @@ -3,8 +3,11 @@ * * Maps file extensions and special filenames to CBMLanguage enum values. * Handles .m disambiguation (Objective-C vs Magma vs MATLAB). + * Consults the process-global user config (set via cbm_set_user_lang_config) + * before the built-in lookup table. */ #include "discover/discover.h" +#include "discover/userconfig.h" #include "cbm.h" // CBMLanguage, CBM_LANG_* #include @@ -269,10 +272,21 @@ typedef struct { } filename_entry_t; static const filename_entry_t FILENAME_TABLE[] = { - {"CMakeLists.txt", CBM_LANG_CMAKE}, {"Dockerfile", CBM_LANG_DOCKERFILE}, - {"GNUmakefile", CBM_LANG_MAKEFILE}, {"Makefile", CBM_LANG_MAKEFILE}, - {"makefile", CBM_LANG_MAKEFILE}, {"meson.build", CBM_LANG_MESON}, - {"meson.options", CBM_LANG_MESON}, {"meson_options.txt", CBM_LANG_MESON}, + {"CMakeLists.txt", CBM_LANG_CMAKE}, + {"Dockerfile", CBM_LANG_DOCKERFILE}, + {"GNUmakefile", CBM_LANG_MAKEFILE}, + {"Makefile", CBM_LANG_MAKEFILE}, + {"makefile", CBM_LANG_MAKEFILE}, + {"meson.build", CBM_LANG_MESON}, + {"meson.options", CBM_LANG_MESON}, + {"meson_options.txt", CBM_LANG_MESON}, + {"kustomization.yaml", CBM_LANG_KUSTOMIZE}, + {"kustomization.yml", CBM_LANG_KUSTOMIZE}, + /* Note: FILENAME_TABLE uses case-sensitive strcmp, so mixed-case variants + * (e.g. "Kustomization.yaml") are not matched here. They fall through to + * CBM_LANG_YAML and are re-classified by cbm_is_kustomize_file() in + * pass_k8s.c, which performs a case-insensitive comparison. This is the + * intended behaviour — no additional entries are needed. */ {".vimrc", CBM_LANG_VIMSCRIPT}, }; @@ -345,6 +359,8 @@ static const char *LANG_NAMES[CBM_LANG_COUNT] = { [CBM_LANG_FORM] = "FORM", [CBM_LANG_MAGMA] = "Magma", [CBM_LANG_WOLFRAM] = "Wolfram", + [CBM_LANG_KUSTOMIZE] = "Kustomize", + [CBM_LANG_K8S] = "Kubernetes", }; /* ── Public API ──────────────────────────────────────────────────── */ @@ -354,6 +370,15 @@ CBMLanguage cbm_language_for_extension(const char *ext) { return CBM_LANG_COUNT; } + /* Check user-defined overrides first */ + const cbm_userconfig_t *ucfg = cbm_get_user_lang_config(); + if (ucfg) { + CBMLanguage ulang = cbm_userconfig_lookup(ucfg, ext); + if (ulang != CBM_LANG_COUNT) { + return ulang; + } + } + for (size_t i = 0; i < EXT_TABLE_SIZE; i++) { if (strcmp(EXT_TABLE[i].ext, ext) == 0) { return EXT_TABLE[i].language; @@ -374,13 +399,30 @@ CBMLanguage cbm_language_for_filename(const char *filename) { } } - /* Fall back to extension-based lookup */ - const char *dot = strrchr(filename, '.'); - if (dot) { - return cbm_language_for_extension(dot); + /* Fall back to extension-based lookup. + * For compound extensions (e.g. ".blade.php") defined in the user config, + * scan from the first dot in the basename toward the last, checking user + * config at each position. Built-in extensions use the last dot only. */ + const char *last_dot = strrchr(filename, '.'); + if (!last_dot) { + return CBM_LANG_COUNT; } - return CBM_LANG_COUNT; + /* Probe user config for compound extensions (e.g. ".blade.php"). */ + const cbm_userconfig_t *ucfg = cbm_get_user_lang_config(); + if (ucfg) { + const char *p = strchr(filename, '.'); + while (p && p < last_dot) { + CBMLanguage lang = cbm_userconfig_lookup(ucfg, p); + if (lang != CBM_LANG_COUNT) { + return lang; + } + p = strchr(p + 1, '.'); + } + } + + /* Standard single-extension lookup (built-ins + user overrides). */ + return cbm_language_for_extension(last_dot); } const char *cbm_language_name(CBMLanguage lang) { diff --git a/src/discover/userconfig.c b/src/discover/userconfig.c new file mode 100644 index 000000000..4b6d27b80 --- /dev/null +++ b/src/discover/userconfig.c @@ -0,0 +1,391 @@ +/* + * userconfig.c — User-defined extension→language mappings. + * + * Reads extra_extensions from: + * Global: $XDG_CONFIG_HOME/codebase-memory-mcp/config.json + * (falls back to ~/.config/codebase-memory-mcp/config.json) + * Project: {repo_root}/.codebase-memory.json + * + * Project config wins over global. Unknown language values warn and are + * skipped (fail-open). Missing files are silently ignored. + */ +#include "discover/userconfig.h" +#include "foundation/log.h" + +#include + +#include +#include +#include +#include + +/* ── Process-global user config pointer ──────────────────────────── */ + +static const cbm_userconfig_t *g_userconfig = NULL; + +void cbm_set_user_lang_config(const cbm_userconfig_t *cfg) { + g_userconfig = cfg; +} + +const cbm_userconfig_t *cbm_get_user_lang_config(void) { + return g_userconfig; +} + +/* ── Language name → enum table ──────────────────────────────────── */ + +/* + * Reverse-mapping from lowercase language name strings to CBMLanguage. + * Covers all names exposed by cbm_language_name() plus common aliases. + */ +typedef struct { + const char *name; /* lowercase */ + CBMLanguage lang; +} lang_name_entry_t; + +static const lang_name_entry_t LANG_NAME_TABLE[] = { + {"go", CBM_LANG_GO}, + {"python", CBM_LANG_PYTHON}, + {"javascript", CBM_LANG_JAVASCRIPT}, + {"typescript", CBM_LANG_TYPESCRIPT}, + {"tsx", CBM_LANG_TSX}, + {"rust", CBM_LANG_RUST}, + {"java", CBM_LANG_JAVA}, + {"c++", CBM_LANG_CPP}, + {"cpp", CBM_LANG_CPP}, + {"c#", CBM_LANG_CSHARP}, + {"csharp", CBM_LANG_CSHARP}, + {"php", CBM_LANG_PHP}, + {"lua", CBM_LANG_LUA}, + {"scala", CBM_LANG_SCALA}, + {"kotlin", CBM_LANG_KOTLIN}, + {"ruby", CBM_LANG_RUBY}, + {"c", CBM_LANG_C}, + {"bash", CBM_LANG_BASH}, + {"sh", CBM_LANG_BASH}, + {"zig", CBM_LANG_ZIG}, + {"elixir", CBM_LANG_ELIXIR}, + {"haskell", CBM_LANG_HASKELL}, + {"ocaml", CBM_LANG_OCAML}, + {"objective-c", CBM_LANG_OBJC}, + {"objc", CBM_LANG_OBJC}, + {"swift", CBM_LANG_SWIFT}, + {"dart", CBM_LANG_DART}, + {"perl", CBM_LANG_PERL}, + {"groovy", CBM_LANG_GROOVY}, + {"erlang", CBM_LANG_ERLANG}, + {"r", CBM_LANG_R}, + {"html", CBM_LANG_HTML}, + {"css", CBM_LANG_CSS}, + {"scss", CBM_LANG_SCSS}, + {"yaml", CBM_LANG_YAML}, + {"toml", CBM_LANG_TOML}, + {"hcl", CBM_LANG_HCL}, + {"terraform", CBM_LANG_HCL}, + {"sql", CBM_LANG_SQL}, + {"dockerfile", CBM_LANG_DOCKERFILE}, + {"clojure", CBM_LANG_CLOJURE}, + {"f#", CBM_LANG_FSHARP}, + {"fsharp", CBM_LANG_FSHARP}, + {"julia", CBM_LANG_JULIA}, + {"vimscript", CBM_LANG_VIMSCRIPT}, + {"nix", CBM_LANG_NIX}, + {"common lisp", CBM_LANG_COMMONLISP}, + {"commonlisp", CBM_LANG_COMMONLISP}, + {"lisp", CBM_LANG_COMMONLISP}, + {"elm", CBM_LANG_ELM}, + {"fortran", CBM_LANG_FORTRAN}, + {"cuda", CBM_LANG_CUDA}, + {"cobol", CBM_LANG_COBOL}, + {"verilog", CBM_LANG_VERILOG}, + {"emacs lisp", CBM_LANG_EMACSLISP}, + {"emacslisp", CBM_LANG_EMACSLISP}, + {"json", CBM_LANG_JSON}, + {"xml", CBM_LANG_XML}, + {"markdown", CBM_LANG_MARKDOWN}, + {"makefile", CBM_LANG_MAKEFILE}, + {"cmake", CBM_LANG_CMAKE}, + {"protobuf", CBM_LANG_PROTOBUF}, + {"graphql", CBM_LANG_GRAPHQL}, + {"vue", CBM_LANG_VUE}, + {"svelte", CBM_LANG_SVELTE}, + {"meson", CBM_LANG_MESON}, + {"glsl", CBM_LANG_GLSL}, + {"ini", CBM_LANG_INI}, + {"matlab", CBM_LANG_MATLAB}, + {"lean", CBM_LANG_LEAN}, + {"form", CBM_LANG_FORM}, + {"magma", CBM_LANG_MAGMA}, + {"wolfram", CBM_LANG_WOLFRAM}, +}; + +#define LANG_NAME_TABLE_SIZE (sizeof(LANG_NAME_TABLE) / sizeof(LANG_NAME_TABLE[0])) + +/* + * Parse a language string (case-insensitive) to a CBMLanguage enum. + * Returns CBM_LANG_COUNT if the string is not recognized. + */ +static CBMLanguage lang_from_string(const char *s) { + if (!s || !s[0]) { + return CBM_LANG_COUNT; + } + + /* Build a lowercase copy for comparison */ + char lower[64]; + size_t i; + for (i = 0; i < sizeof(lower) - 1 && s[i]; i++) { + lower[i] = (char)tolower((unsigned char)s[i]); + } + lower[i] = '\0'; + + for (size_t j = 0; j < LANG_NAME_TABLE_SIZE; j++) { + if (strcmp(LANG_NAME_TABLE[j].name, lower) == 0) { + return LANG_NAME_TABLE[j].lang; + } + } + return CBM_LANG_COUNT; +} + +/* ── Config directory helper ─────────────────────────────────────── */ + +/* + * Get the XDG config dir for codebase-memory-mcp. + * Writes "

/codebase-memory-mcp" into buf (up to bufsz bytes). + * Uses $XDG_CONFIG_HOME if set, else ~/.config. + */ +static void cbm_app_config_dir(char *buf, size_t bufsz) { + // NOLINT(concurrency-mt-unsafe) — called before worker threads + const char *xdg = getenv("XDG_CONFIG_HOME"); + if (xdg && xdg[0]) { + snprintf(buf, bufsz, "%s/codebase-memory-mcp", xdg); + } else { + const char *home = getenv("HOME"); // NOLINT(concurrency-mt-unsafe) + if (!home || !home[0]) { + home = "/tmp"; + } + snprintf(buf, bufsz, "%s/.config/codebase-memory-mcp", home); + } +} + +/* ── JSON parsing ────────────────────────────────────────────────── */ + +/* + * Parse extra_extensions from a yyjson object root. + * Appends valid entries to *entries / *count (growing via realloc). + * Project-level entries (from_project=true) are appended after global + * entries so that a later dedup pass can prefer project values. + * + * Returns 0 on success, -1 on alloc failure. + */ +static int parse_extra_extensions(yyjson_val *root, cbm_userext_t **entries, int *count, + const char *source_label) { + if (!yyjson_is_obj(root)) { + cbm_log_warn("userconfig.bad_root", "file", source_label); + return 0; + } + + yyjson_val *extra = yyjson_obj_get(root, "extra_extensions"); + if (!extra) { + return 0; /* key absent — fine */ + } + if (!yyjson_is_obj(extra)) { + cbm_log_warn("userconfig.bad_extra_extensions", "file", source_label); + return 0; + } + + yyjson_obj_iter iter; + yyjson_obj_iter_init(extra, &iter); + yyjson_val *key; + while ((key = yyjson_obj_iter_next(&iter)) != NULL) { + yyjson_val *val = yyjson_obj_iter_get_val(key); + + const char *ext_str = yyjson_get_str(key); + const char *lang_str = yyjson_get_str(val); + + if (!ext_str || !lang_str) { + cbm_log_warn("userconfig.skip_non_string", "file", source_label); + continue; + } + + /* Extension must start with '.' */ + if (ext_str[0] != '.') { + cbm_log_warn("userconfig.skip_bad_ext", "file", source_label, "ext", ext_str); + continue; + } + + CBMLanguage lang = lang_from_string(lang_str); + if (lang == CBM_LANG_COUNT) { + cbm_log_warn("userconfig.unknown_lang", "file", source_label, "lang", lang_str); + continue; /* fail-open: skip unknown languages */ + } + + /* Grow the array */ + cbm_userext_t *tmp = realloc(*entries, (size_t)(*count + 1) * sizeof(cbm_userext_t)); + if (!tmp) { + return -1; + } + *entries = tmp; + + char *ext_copy = strdup(ext_str); + if (!ext_copy) { + return -1; + } + + (*entries)[*count].ext = ext_copy; + (*entries)[*count].lang = lang; + (*count)++; + } + return 0; +} + +/* + * Read a JSON file and parse extra_extensions from it. + * Silently ignores missing files. Logs warnings for corrupt JSON. + * Returns 0 on success (or absent file), -1 on alloc failure. + */ +static int load_config_file(const char *path, cbm_userext_t **entries, int *count) { + FILE *f = fopen(path, "rb"); + if (!f) { + return 0; /* file absent — silently ignore */ + } + + fseek(f, 0, SEEK_END); + long len = ftell(f); + fseek(f, 0, SEEK_SET); + + if (len <= 0 || len > 65536) { + fclose(f); + if (len > 65536) { + cbm_log_warn("userconfig.file_too_large", "path", path); + } + return 0; + } + + char *buf = malloc((size_t)len + 1); + if (!buf) { + fclose(f); + return -1; + } + + size_t nread = fread(buf, 1, (size_t)len, f); + fclose(f); + buf[nread] = '\0'; + + yyjson_doc *doc = yyjson_read(buf, nread, 0); + free(buf); + + if (!doc) { + cbm_log_warn("userconfig.corrupt_json", "path", path); + return 0; /* corrupt JSON — silently ignore (fail-open) */ + } + + yyjson_val *root = yyjson_doc_get_root(doc); + int rc = parse_extra_extensions(root, entries, count, path); + yyjson_doc_free(doc); + return rc; +} + +/* ── Public API ──────────────────────────────────────────────────── */ + +cbm_userconfig_t *cbm_userconfig_load(const char *repo_path) { + cbm_userconfig_t *cfg = calloc(1, sizeof(cbm_userconfig_t)); + if (!cfg) { + return NULL; + } + + cbm_userext_t *entries = NULL; + int count = 0; + + /* ── Step 1: Load global config ── */ + enum { PATH_BUF_SZ = 1280 }; + char global_dir[1024]; + cbm_app_config_dir(global_dir, sizeof(global_dir)); + + char global_path[PATH_BUF_SZ]; + snprintf(global_path, sizeof(global_path), "%s/config.json", global_dir); + + if (load_config_file(global_path, &entries, &count) != 0) { + for (int i = 0; i < count; i++) { + free(entries[i].ext); + } + free(entries); + free(cfg); + return NULL; + } + + int global_count = count; /* entries[0..global_count) are from global */ + + /* ── Step 2: Load project config ── */ + if (repo_path && repo_path[0]) { + char project_path[PATH_BUF_SZ]; + snprintf(project_path, sizeof(project_path), "%s/.codebase-memory.json", repo_path); + + if (load_config_file(project_path, &entries, &count) != 0) { + /* Free already-allocated entries */ + for (int i = 0; i < count; i++) { + free(entries[i].ext); + } + free(entries); + free(cfg); + return NULL; + } + } + + /* + * ── Step 3: Dedup — project entries win over global ── + * + * For any extension that appears in both global (indices 0..global_count) + * and project (indices global_count..count), remove the global entry by + * replacing it with the last global entry (order-insensitive dedup). + */ + for (int p = global_count; p < count; p++) { + for (int g = 0; g < global_count; g++) { + if (entries[g].ext && strcmp(entries[g].ext, entries[p].ext) == 0) { + /* Remove global entry: overwrite with last global entry */ + free(entries[g].ext); + entries[g] = entries[global_count - 1]; + entries[global_count - 1].ext = NULL; /* mark as consumed */ + global_count--; + break; + } + } + } + + /* + * Compact: remove any NULL-ext slots left by the dedup step. + * (Those are the consumed "last global" entries.) + */ + int write_idx = 0; + for (int i = 0; i < count; i++) { + if (entries[i].ext != NULL) { + entries[write_idx++] = entries[i]; + } + } + count = write_idx; + + cfg->entries = entries; + cfg->count = count; + return cfg; +} + +CBMLanguage cbm_userconfig_lookup(const cbm_userconfig_t *cfg, const char *ext) { + if (!cfg || !ext || !ext[0]) { + return CBM_LANG_COUNT; + } + for (int i = 0; i < cfg->count; i++) { + if (cfg->entries[i].ext && strcmp(cfg->entries[i].ext, ext) == 0) { + return cfg->entries[i].lang; + } + } + return CBM_LANG_COUNT; +} + +void cbm_userconfig_free(cbm_userconfig_t *cfg) { + if (!cfg) { + return; + } + for (int i = 0; i < cfg->count; i++) { + free(cfg->entries[i].ext); + } + free(cfg->entries); + free(cfg); +} diff --git a/src/discover/userconfig.h b/src/discover/userconfig.h new file mode 100644 index 000000000..233734845 --- /dev/null +++ b/src/discover/userconfig.h @@ -0,0 +1,72 @@ +/* + * userconfig.h — User-defined file extension → language mappings. + * + * Reads extra_extensions from two optional JSON config files: + * Global: $XDG_CONFIG_HOME/codebase-memory-mcp/config.json + * (falls back to ~/.config/codebase-memory-mcp/config.json) + * Project: {repo_root}/.codebase-memory.json + * + * Project config wins over global. Unknown language values warn and are + * skipped (fail-open). Missing files are silently ignored. + * + * Format: + * {"extra_extensions": {".blade.php": "php", ".mjs": "javascript"}} + * + * The language string matching is case-insensitive. + */ +#ifndef CBM_USERCONFIG_H +#define CBM_USERCONFIG_H + +#include "cbm.h" /* CBMLanguage */ + +/* ── Types ──────────────────────────────────────────────────────── */ + +typedef struct { + char *ext; /* file extension including dot, e.g. ".blade.php" */ + CBMLanguage lang; /* resolved language enum */ +} cbm_userext_t; + +typedef struct { + cbm_userext_t *entries; /* heap-allocated array */ + int count; /* number of entries */ +} cbm_userconfig_t; + +/* ── API ────────────────────────────────────────────────────────── */ + +/* + * Load user config from global + project files, merge (project wins). + * repo_path: absolute path to the repository root (for project config). + * Returns a heap-allocated cbm_userconfig_t (caller must free via + * cbm_userconfig_free). Returns NULL only on allocation failure. + * Missing config files are silently ignored. + */ +cbm_userconfig_t *cbm_userconfig_load(const char *repo_path); + +/* + * Look up a file extension in the user config. + * ext: extension including dot, e.g. ".blade.php" + * Returns the mapped CBMLanguage, or CBM_LANG_COUNT if not found. + */ +CBMLanguage cbm_userconfig_lookup(const cbm_userconfig_t *cfg, const char *ext); + +/* Free a cbm_userconfig_t returned by cbm_userconfig_load. NULL-safe. */ +void cbm_userconfig_free(cbm_userconfig_t *cfg); + +/* ── Integration hook ───────────────────────────────────────────── */ + +/* + * Set the process-global user config that cbm_language_for_extension() + * will consult before the built-in table. + * cfg may be NULL to clear the override. + * Not thread-safe — call before spawning worker threads. + */ +void cbm_set_user_lang_config(const cbm_userconfig_t *cfg); + +/* + * Get the currently active process-global user config. + * Returns NULL if none has been set. + * Called internally by cbm_language_for_extension(). + */ +const cbm_userconfig_t *cbm_get_user_lang_config(void); + +#endif /* CBM_USERCONFIG_H */ diff --git a/src/foundation/platform.c b/src/foundation/platform.c index 2c33e4fe9..eb60f07ea 100644 --- a/src/foundation/platform.c +++ b/src/foundation/platform.c @@ -216,3 +216,19 @@ int64_t cbm_file_size(const char *path) { } #endif /* _WIN32 */ + +/* ── Home directory (cross-platform) ──────────────────────────── */ + +const char *cbm_get_home_dir(void) { + // NOLINTNEXTLINE(concurrency-mt-unsafe) + const char *h = getenv("HOME"); + if (h && h[0]) { + return h; + } + // NOLINTNEXTLINE(concurrency-mt-unsafe) + h = getenv("USERPROFILE"); + if (h && h[0]) { + return h; + } + return NULL; +} diff --git a/src/foundation/platform.h b/src/foundation/platform.h index ffa57fce8..e88ab068b 100644 --- a/src/foundation/platform.h +++ b/src/foundation/platform.h @@ -64,6 +64,12 @@ cbm_system_info_t cbm_system_info(void); * initial=false: max(1, perf_cores-1) (leave headroom for user apps) */ int cbm_default_worker_count(bool initial); +/* ── Home directory ─────────────────────────────────────────────── */ + +/* Cross-platform home directory: tries HOME first, then USERPROFILE (Windows). + * Returns NULL when neither is set. */ +const char *cbm_get_home_dir(void); + /* ── File system ───────────────────────────────────────────────── */ /* Check if a path exists. */ diff --git a/src/main.c b/src/main.c index 79618fad1..1070d36da 100644 --- a/src/main.c +++ b/src/main.c @@ -19,6 +19,7 @@ #include "store/store.h" #include "cli/cli.h" #include "foundation/log.h" +#include "foundation/platform.h" #include "foundation/compat_thread.h" #include "foundation/mem.h" #include "ui/config.h" @@ -221,7 +222,7 @@ int main(int argc, char **argv) { /* Open config store for runtime settings */ char config_dir[1024]; - const char *cfg_home = getenv("HOME"); + const char *cfg_home = cbm_get_home_dir(); cbm_config_t *runtime_config = NULL; if (cfg_home) { snprintf(config_dir, sizeof(config_dir), "%s/.cache/codebase-memory-mcp", cfg_home); diff --git a/src/mcp/mcp.c b/src/mcp/mcp.c index bdfdae8b5..0c5892817 100644 --- a/src/mcp/mcp.c +++ b/src/mcp/mcp.c @@ -20,6 +20,7 @@ #include "foundation/compat_thread.h" #include "foundation/log.h" #include "foundation/str_util.h" +#include "foundation/compat_regex.h" #ifdef _WIN32 #include /* _getpid */ @@ -28,6 +29,7 @@ #include #include #include +#include #endif #include #include // int64_t @@ -277,12 +279,21 @@ static const tool_def_t TOOLS[] = { "\"array\",\"items\":{\"type\":\"string\"}}}}"}, {"search_code", - "Search source code content with text or regex patterns. Use for string literals, error " - "messages, and config values that are not in the knowledge graph.", + "Graph-augmented code search. Finds text patterns via grep, then enriches results with " + "the knowledge graph: deduplicates matches into containing functions, ranks by structural " + "importance (definitions first, popular functions next, tests last). " + "Modes: compact (default, signatures only — token efficient), full (with source), " + "files (just file paths). Use path_filter regex to scope results.", "{\"type\":\"object\",\"properties\":{\"pattern\":{\"type\":\"string\"},\"project\":{\"type\":" - "\"string\"},\"file_pattern\":{\"type\":\"string\"},\"regex\":{\"type\":\"boolean\"," - "\"default\":false},\"limit\":{\"type\":\"integer\",\"description\":\"Max results. Default: " - "unlimited\"}},\"required\":[" + "\"string\"},\"file_pattern\":{\"type\":\"string\",\"description\":\"Glob for grep " + "--include (e.g. *.go)\"},\"path_filter\":{\"type\":\"string\",\"description\":\"Regex " + "filter on result file paths (e.g. ^src/ or \\\\.(go|ts)$)\"},\"mode\":{\"type\":\"string\"," + "\"enum\":[\"compact\",\"full\",\"files\"],\"default\":\"compact\",\"description\":\"compact: " + "signatures+metadata (default). full: with source. files: just file list.\"}," + "\"context\":{\"type\":\"integer\",\"description\":\"Lines of context around each match " + "(like grep -C). Only used in compact mode.\"}," + "\"regex\":{\"type\":\"boolean\",\"default\":false},\"limit\":{\"type\":\"integer\"," + "\"description\":\"Max results (default 10)\",\"default\":10}},\"required\":[" "\"pattern\"]}"}, {"list_projects", "List all indexed projects", "{\"type\":\"object\",\"properties\":{}}"}, @@ -597,10 +608,9 @@ bool cbm_mcp_server_has_cached_store(cbm_mcp_server_t *srv) { /* Returns the platform cache directory: ~/.cache/codebase-memory-mcp * Writes to buf, returns buf for convenience. */ static const char *cache_dir(char *buf, size_t bufsz) { - // NOLINTNEXTLINE(concurrency-mt-unsafe) - const char *home = getenv("HOME"); + const char *home = cbm_get_home_dir(); if (!home) { - home = "/tmp"; + home = cbm_tmpdir(); } snprintf(buf, bufsz, "%s/.cache/codebase-memory-mcp", home); return buf; @@ -1624,21 +1634,314 @@ static char *handle_get_code_snippet(cbm_mcp_server_t *srv, const char *args) { true); } -/* ── search_code ──────────────────────────────────────────────── */ +/* ── search_code v2: graph-augmented code search ─────────────── */ + +/* Strip non-ASCII bytes to guarantee valid UTF-8 JSON output */ +enum { ASCII_MAX = 127 }; +static void sanitize_ascii(char *s) { + for (unsigned char *p = (unsigned char *)s; *p; p++) { + if (*p > ASCII_MAX) { + *p = '?'; + } + } +} + +/* Intermediate grep match */ +typedef struct { + char file[512]; + int line; + char content[1024]; +} grep_match_t; + +/* Deduped result: one per containing graph node */ +typedef struct { + int64_t node_id; /* 0 = raw match (no containing node) */ + char node_name[256]; + char qualified_name[512]; + char label[64]; + char file[512]; + int start_line; + int end_line; + int in_degree; + int out_degree; + int score; + int match_lines[64]; + int match_count; +} search_result_t; + +/* Score a result for ranking: project source first, vendored last, tests lowest */ +enum { SCORE_FUNC = 10, SCORE_ROUTE = 15, SCORE_VENDORED = -50, SCORE_TEST = -5 }; +enum { MAX_LINE_SPAN = 999999 }; + +static int compute_search_score(const search_result_t *r) { + int score = r->in_degree; + if (strcmp(r->label, "Function") == 0 || strcmp(r->label, "Method") == 0) { + score += SCORE_FUNC; + } + if (strcmp(r->label, "Route") == 0) { + score += SCORE_ROUTE; + } + if (strstr(r->file, "vendored/") || strstr(r->file, "vendor/") || + strstr(r->file, "node_modules/")) { + score += SCORE_VENDORED; + } + /* Penalize test files */ + if (strstr(r->file, "test") || strstr(r->file, "spec") || strstr(r->file, "_test.")) { + score += SCORE_TEST; + } + return score; +} + +static int search_result_cmp(const void *a, const void *b) { + const search_result_t *ra = (const search_result_t *)a; + const search_result_t *rb = (const search_result_t *)b; + return rb->score - ra->score; /* descending */ +} + +/* Build the grep command string based on scoped vs recursive mode */ +static void build_grep_cmd(char *cmd, size_t cmd_sz, bool use_regex, bool scoped, + const char *file_pattern, const char *tmpfile, const char *filelist, + const char *root_path) { + // NOLINTNEXTLINE(readability-implicit-bool-conversion) + const char *flag = use_regex ? "-E" : "-F"; + if (scoped) { + if (file_pattern) { + snprintf(cmd, cmd_sz, "xargs grep -n %s --include='%s' -f '%s' < '%s' 2>/dev/null", + flag, file_pattern, tmpfile, filelist); + } else { + snprintf(cmd, cmd_sz, "xargs grep -n %s -f '%s' < '%s' 2>/dev/null", flag, tmpfile, + filelist); + } + } else { + if (file_pattern) { + snprintf(cmd, cmd_sz, "grep -rn %s --include='%s' -f '%s' '%s' 2>/dev/null", flag, + file_pattern, tmpfile, root_path); + } else { + snprintf(cmd, cmd_sz, "grep -rn %s -f '%s' '%s' 2>/dev/null", flag, tmpfile, root_path); + } + } +} + +/* Phase 4: assemble JSON output from search results */ +static char *assemble_search_output(search_result_t *sr, int sr_count, grep_match_t *raw, + int raw_count, int gm_count, int limit, int mode, + int context_lines, const char *root_path) { + enum { MODE_COMPACT = 0, MODE_FULL = 1, MODE_FILES = 2 }; + + yyjson_mut_doc *doc = yyjson_mut_doc_new(NULL); + yyjson_mut_val *root_obj = yyjson_mut_obj(doc); + yyjson_mut_doc_set_root(doc, root_obj); + + int output_count = sr_count < limit ? sr_count : limit; + + if (mode == MODE_FILES) { + yyjson_mut_val *files_arr = yyjson_mut_arr(doc); + char *seen_files[512]; + int seen_count = 0; + for (int fi = 0; fi < output_count; fi++) { + bool dup = false; + for (int j = 0; j < seen_count; j++) { + if (strcmp(seen_files[j], sr[fi].file) == 0) { + dup = true; + break; + } + } + if (!dup && seen_count < 512) { + seen_files[seen_count++] = sr[fi].file; + yyjson_mut_arr_add_str(doc, files_arr, sr[fi].file); + } + } + for (int fi = 0; fi < raw_count && seen_count < 512; fi++) { + bool dup = false; + for (int j = 0; j < seen_count; j++) { + if (strcmp(seen_files[j], raw[fi].file) == 0) { + dup = true; + break; + } + } + if (!dup) { + seen_files[seen_count++] = raw[fi].file; + yyjson_mut_arr_add_str(doc, files_arr, raw[fi].file); + } + } + yyjson_mut_obj_add_val(doc, root_obj, "files", files_arr); + } else { + yyjson_mut_val *results_arr = yyjson_mut_arr(doc); + for (int ri = 0; ri < output_count; ri++) { + search_result_t *r = &sr[ri]; + yyjson_mut_val *item = yyjson_mut_obj(doc); + + yyjson_mut_obj_add_str(doc, item, "node", r->node_name); + yyjson_mut_obj_add_str(doc, item, "qualified_name", r->qualified_name); + yyjson_mut_obj_add_str(doc, item, "label", r->label); + yyjson_mut_obj_add_str(doc, item, "file", r->file); + yyjson_mut_obj_add_int(doc, item, "start_line", r->start_line); + yyjson_mut_obj_add_int(doc, item, "end_line", r->end_line); + yyjson_mut_obj_add_int(doc, item, "in_degree", r->in_degree); + yyjson_mut_obj_add_int(doc, item, "out_degree", r->out_degree); + + yyjson_mut_val *ml = yyjson_mut_arr(doc); + for (int j = 0; j < r->match_count; j++) { + yyjson_mut_arr_add_int(doc, ml, r->match_lines[j]); + } + yyjson_mut_obj_add_val(doc, item, "match_lines", ml); + + if (r->start_line > 0 && r->end_line > 0) { + char abs_path[1024]; + snprintf(abs_path, sizeof(abs_path), "%s/%s", root_path, r->file); + + if (mode == MODE_FULL) { + char *source = read_file_lines(abs_path, r->start_line, r->end_line); + if (source) { + sanitize_ascii(source); + yyjson_mut_obj_add_strcpy(doc, item, "source", source); + free(source); + } + } else if (context_lines > 0 && r->match_count > 0) { + int first_match = r->match_lines[0]; + int last_match = r->match_lines[r->match_count - 1]; + int ctx_start = first_match - context_lines; + int ctx_end = last_match + context_lines; + if (ctx_start < 1) { + ctx_start = 1; + } + char *ctx = read_file_lines(abs_path, ctx_start, ctx_end); + if (ctx) { + sanitize_ascii(ctx); + yyjson_mut_obj_add_strcpy(doc, item, "context", ctx); + yyjson_mut_obj_add_int(doc, item, "context_start", ctx_start); + free(ctx); + } + } + } + + yyjson_mut_arr_add_val(results_arr, item); + } + yyjson_mut_obj_add_val(doc, root_obj, "results", results_arr); + + enum { MAX_RAW = 20 }; + yyjson_mut_val *raw_arr = yyjson_mut_arr(doc); + int raw_output = raw_count < MAX_RAW ? raw_count : MAX_RAW; + for (int ri = 0; ri < raw_output; ri++) { + yyjson_mut_val *item = yyjson_mut_obj(doc); + yyjson_mut_obj_add_str(doc, item, "file", raw[ri].file); + yyjson_mut_obj_add_int(doc, item, "line", raw[ri].line); + yyjson_mut_obj_add_str(doc, item, "content", raw[ri].content); + yyjson_mut_arr_add_val(raw_arr, item); + } + yyjson_mut_obj_add_val(doc, root_obj, "raw_matches", raw_arr); + } + + /* Directory distribution */ + { + yyjson_mut_val *dirs = yyjson_mut_obj(doc); + char dir_names[64][128]; + int dir_counts[64]; + int dir_n = 0; + for (int di = 0; di < sr_count; di++) { + char top[128] = ""; + const char *slash = strchr(sr[di].file, '/'); + if (slash) { + size_t dlen = (size_t)(slash - sr[di].file + 1); + if (dlen >= sizeof(top)) { + dlen = sizeof(top) - 1; + } + memcpy(top, sr[di].file, dlen); + top[dlen] = '\0'; + } else { + snprintf(top, sizeof(top), "%s", sr[di].file); + } + int found = -1; + for (int d = 0; d < dir_n; d++) { + if (strcmp(dir_names[d], top) == 0) { + found = d; + break; + } + } + if (found >= 0) { + dir_counts[found]++; + } else if (dir_n < 64) { + snprintf(dir_names[dir_n], sizeof(dir_names[0]), "%s", top); + dir_counts[dir_n] = 1; + dir_n++; + } + } + for (int d = 0; d < dir_n; d++) { + yyjson_mut_val *key = yyjson_mut_strcpy(doc, dir_names[d]); + yyjson_mut_val *val = yyjson_mut_int(doc, dir_counts[d]); + yyjson_mut_obj_add(dirs, key, val); + } + yyjson_mut_obj_add_val(doc, root_obj, "directories", dirs); + } + + /* Summary stats */ + yyjson_mut_obj_add_int(doc, root_obj, "total_grep_matches", gm_count); + yyjson_mut_obj_add_int(doc, root_obj, "total_results", sr_count); + yyjson_mut_obj_add_int(doc, root_obj, "raw_match_count", raw_count); + if (sr_count > 0 && gm_count > 0) { + char ratio[32]; + snprintf(ratio, sizeof(ratio), "%.1fx", (double)gm_count / (double)(sr_count + raw_count)); + yyjson_mut_obj_add_strcpy(doc, root_obj, "dedup_ratio", ratio); + } + + char *json = yy_doc_to_str(doc); + if (json) { + sanitize_ascii(json); + } + yyjson_mut_doc_free(doc); + + char *result = cbm_mcp_text_result(json, false); + free(json); + return result; +} static char *handle_search_code(cbm_mcp_server_t *srv, const char *args) { char *pattern = cbm_mcp_get_string_arg(args, "pattern"); char *project = cbm_mcp_get_string_arg(args, "project"); char *file_pattern = cbm_mcp_get_string_arg(args, "file_pattern"); - int limit = cbm_mcp_get_int_arg(args, "limit", 500000); + char *path_filter = cbm_mcp_get_string_arg(args, "path_filter"); + char *mode_str = cbm_mcp_get_string_arg(args, "mode"); + int limit = cbm_mcp_get_int_arg(args, "limit", 10); + int context_lines = cbm_mcp_get_int_arg(args, "context", 0); bool use_regex = cbm_mcp_get_bool_arg(args, "regex"); + /* Parse mode: compact (default), full, files */ + enum { MODE_COMPACT, MODE_FULL, MODE_FILES }; + int mode = MODE_COMPACT; + if (mode_str) { + if (strcmp(mode_str, "full") == 0) { + mode = MODE_FULL; + } else if (strcmp(mode_str, "files") == 0) { + mode = MODE_FILES; + } + free(mode_str); + } + + /* Compile path_filter regex if provided */ + cbm_regex_t path_regex; + bool has_path_filter = false; + if (path_filter && path_filter[0]) { + if (cbm_regcomp(&path_regex, path_filter, CBM_REG_EXTENDED | CBM_REG_NOSUB) == CBM_REG_OK) { + has_path_filter = true; + } + free(path_filter); + path_filter = NULL; + } else { + free(path_filter); + path_filter = NULL; + } + if (!pattern) { free(project); free(file_pattern); return cbm_mcp_text_result("pattern is required", true); } + /* Resolve project */ + if (!project && srv->session_project[0]) { + project = heap_strdup(srv->session_project); + } + char *root_path = get_project_root(srv, project); if (!root_path) { free(pattern); @@ -1657,7 +1960,8 @@ static char *handle_search_code(cbm_mcp_server_t *srv, const char *args) { return cbm_mcp_text_result("path or file_pattern contains invalid characters", true); } - /* Write pattern to temp file to avoid shell injection */ + /* ── Phase 1: Grep scan ──────────────────────────────────── */ + char tmpfile[256]; #ifdef _WIN32 snprintf(tmpfile, sizeof(tmpfile), "/tmp/cbm_search_%d.pat", (int)_getpid()); @@ -1676,21 +1980,53 @@ static char *handle_search_code(cbm_mcp_server_t *srv, const char *args) { (void)fprintf(tf, "%s\n", pattern); (void)fclose(tf); - char cmd[4096]; - // NOLINTNEXTLINE(readability-implicit-bool-conversion) - const char *flag = use_regex ? "-E" : "-F"; - if (file_pattern) { - snprintf(cmd, sizeof(cmd), "grep -rn %s --include='%s' -m %d -f '%s' '%s' 2>/dev/null", - flag, file_pattern, limit * 3, tmpfile, root_path); - } else { - snprintf(cmd, sizeof(cmd), "grep -rn %s -m %d -f '%s' '%s' 2>/dev/null", flag, limit * 3, - tmpfile, root_path); + /* No grep-level match limit — let grep find all matches, then dedup and + * cap in our code. The -m flag caused results from large vendored files + * to exhaust the quota before reaching project source files. */ + enum { GREP_MAX_MATCHES = 500 }; + int grep_limit = GREP_MAX_MATCHES; + + /* Scope grep to indexed files only — avoids scanning vendored/generated code. + * Query the graph for distinct file paths, write them to a temp file, + * then use xargs to pass them to grep. Falls back to recursive grep if + * no indexed files found (project not fully indexed). */ + char filelist[256]; + snprintf(filelist, sizeof(filelist), "%s.files", tmpfile); + bool scoped = false; + + cbm_store_t *pre_store = resolve_store(srv, project); + if (pre_store) { + const char *pn = project ? project : srv->session_project; + char **indexed_files = NULL; + int indexed_count = 0; + if (pn && + cbm_store_list_files(pre_store, pn, &indexed_files, &indexed_count) == CBM_STORE_OK && + indexed_count > 0) { + FILE *fl = fopen(filelist, "w"); + if (fl) { + for (int fi = 0; fi < indexed_count; fi++) { + fprintf(fl, "%s/%s\n", root_path, indexed_files[fi]); + } + fclose(fl); + scoped = true; + } + for (int fi = 0; fi < indexed_count; fi++) { + free(indexed_files[fi]); + } + free(indexed_files); + } } + char cmd[4096]; + build_grep_cmd(cmd, sizeof(cmd), use_regex, scoped, file_pattern, tmpfile, filelist, root_path); + // NOLINTNEXTLINE(bugprone-command-processor,cert-env33-c) FILE *fp = cbm_popen(cmd, "r"); if (!fp) { cbm_unlink(tmpfile); + if (scoped) { + cbm_unlink(filelist); + } free(root_path); free(pattern); free(project); @@ -1698,16 +2034,14 @@ static char *handle_search_code(cbm_mcp_server_t *srv, const char *args) { return cbm_mcp_text_result("search failed", true); } - yyjson_mut_doc *doc = yyjson_mut_doc_new(NULL); - yyjson_mut_val *root_obj = yyjson_mut_obj(doc); - yyjson_mut_doc_set_root(doc, root_obj); - - yyjson_mut_val *matches = yyjson_mut_arr(doc); + /* Collect grep matches into array */ + int gm_cap = 64; + int gm_count = 0; + grep_match_t *gm = malloc(gm_cap * sizeof(grep_match_t)); char line[2048]; - int count = 0; size_t root_len = strlen(root_path); - while (fgets(line, sizeof(line), fp) && count < limit) { + while (fgets(line, sizeof(line), fp) && gm_count < grep_limit) { size_t len = strlen(line); while (len > 0 && (line[len - 1] == '\n' || line[len - 1] == '\r')) { line[--len] = '\0'; @@ -1716,7 +2050,6 @@ static char *handle_search_code(cbm_mcp_server_t *srv, const char *args) { continue; } - /* grep output: /abs/path/file:lineno:content */ char *colon1 = strchr(line, ':'); if (!colon1) { continue; @@ -1729,7 +2062,6 @@ static char *handle_search_code(cbm_mcp_server_t *srv, const char *args) { *colon1 = '\0'; *colon2 = '\0'; - /* Strip root_path prefix to get relative path */ const char *file = line; if (strncmp(file, root_path, root_len) == 0) { file += root_len; @@ -1737,31 +2069,178 @@ static char *handle_search_code(cbm_mcp_server_t *srv, const char *args) { file++; } } - int lineno = (int)strtol(colon1 + 1, NULL, 10); - const char *content = colon2 + 1; - yyjson_mut_val *item = yyjson_mut_obj(doc); - yyjson_mut_obj_add_str(doc, item, "file", file); - yyjson_mut_obj_add_int(doc, item, "line", lineno); - yyjson_mut_obj_add_str(doc, item, "content", content); - yyjson_mut_arr_add_val(matches, item); - count++; + /* Apply path_filter regex — skip files that don't match */ + if (has_path_filter && cbm_regexec(&path_regex, file, 0, NULL, 0) != CBM_REG_OK) { + continue; + } + + if (gm_count >= gm_cap) { + gm_cap *= 2; + gm = safe_realloc(gm, gm_cap * sizeof(grep_match_t)); + } + snprintf(gm[gm_count].file, sizeof(gm[0].file), "%s", file); + gm[gm_count].line = (int)strtol(colon1 + 1, NULL, 10); + snprintf(gm[gm_count].content, sizeof(gm[0].content), "%s", colon2 + 1); + sanitize_ascii(gm[gm_count].content); + gm_count++; } cbm_pclose(fp); - cbm_unlink(tmpfile); /* Clean up pattern file after grep is done */ + cbm_unlink(tmpfile); + if (scoped) { + cbm_unlink(filelist); + } - yyjson_mut_obj_add_val(doc, root_obj, "matches", matches); - yyjson_mut_obj_add_int(doc, root_obj, "count", count); + /* ── Phase 2+3: Block expansion + graph ranking ──────────── */ + /* Sort grep matches by file for contiguous processing. + * Then: one SQL query per unique file for nodes, one batch query for all degrees. */ - char *json = yy_doc_to_str(doc); - yyjson_mut_doc_free(doc); + cbm_store_t *store = resolve_store(srv, project); + const char *proj_name = project ? project : srv->session_project; + + int sr_cap = 32; + int sr_count = 0; + search_result_t *sr = calloc(sr_cap, sizeof(search_result_t)); + + int raw_cap = 32; + int raw_count = 0; + grep_match_t *raw = malloc(raw_cap * sizeof(grep_match_t)); + + /* Sort matches by file path for contiguous per-file processing */ + qsort(gm, gm_count, sizeof(grep_match_t), (int (*)(const void *, const void *))strcmp); + + /* Process matches file-by-file (contiguous runs after sort) */ + int i = 0; + while (i < gm_count) { + const char *cur_file = gm[i].file; + int file_start = i; + + /* Find end of this file's run */ + while (i < gm_count && strcmp(gm[i].file, cur_file) == 0) { + i++; + } + int file_end = i; /* [file_start, file_end) */ + + /* One SQL query: load all nodes in this file */ + cbm_node_t *file_nodes = NULL; + int file_node_count = 0; + if (store && proj_name) { + cbm_store_find_nodes_by_file(store, proj_name, cur_file, &file_nodes, &file_node_count); + } + + /* Match each grep hit to tightest containing node (in-memory) */ + for (int mi = file_start; mi < file_end; mi++) { + int best = -1; + int best_span = MAX_LINE_SPAN; + for (int j = 0; j < file_node_count; j++) { + if (file_nodes[j].start_line <= gm[mi].line && + file_nodes[j].end_line >= gm[mi].line) { + int span = file_nodes[j].end_line - file_nodes[j].start_line; + if (span < best_span) { + best = j; + best_span = span; + } + } + } + + if (best >= 0) { + cbm_node_t *n = &file_nodes[best]; + + /* Dedup: check if node already in results */ + int existing = -1; + for (int j = 0; j < sr_count; j++) { + if (sr[j].node_id == n->id) { + existing = j; + break; + } + } + + if (existing >= 0) { + if (sr[existing].match_count < 64) { + sr[existing].match_lines[sr[existing].match_count++] = gm[mi].line; + } + } else { + if (sr_count >= sr_cap) { + sr_cap *= 2; + sr = safe_realloc(sr, sr_cap * sizeof(search_result_t)); + memset(&sr[sr_count], 0, (sr_cap - sr_count) * sizeof(search_result_t)); + } + search_result_t *r = &sr[sr_count]; + r->node_id = n->id; + snprintf(r->node_name, sizeof(r->node_name), "%s", n->name ? n->name : ""); + snprintf(r->qualified_name, sizeof(r->qualified_name), "%s", + n->qualified_name ? n->qualified_name : ""); + snprintf(r->label, sizeof(r->label), "%s", n->label ? n->label : ""); + snprintf(r->file, sizeof(r->file), "%s", n->file_path ? n->file_path : ""); + r->start_line = n->start_line; + r->end_line = n->end_line; + r->match_lines[0] = gm[mi].line; + r->match_count = 1; + sr_count++; + } + } else { + if (raw_count >= raw_cap) { + raw_cap *= 2; + raw = safe_realloc(raw, raw_cap * sizeof(grep_match_t)); + } + raw[raw_count++] = gm[mi]; + } + } + + /* Free file nodes */ + for (int j = 0; j < file_node_count; j++) { + free((void *)file_nodes[j].project); + free((void *)file_nodes[j].label); + free((void *)file_nodes[j].name); + free((void *)file_nodes[j].qualified_name); + free((void *)file_nodes[j].file_path); + free((void *)file_nodes[j].properties_json); + } + free(file_nodes); + } + + /* Phase 3: batch degree query — ONE query for all results instead of 2×N */ + if (store && sr_count > 0) { + int64_t *ids = malloc(sr_count * sizeof(int64_t)); + int *in_degs = malloc(sr_count * sizeof(int)); + int *out_degs = malloc(sr_count * sizeof(int)); + for (int j = 0; j < sr_count; j++) { + ids[j] = sr[j].node_id; + } + if (cbm_store_batch_count_degrees(store, ids, sr_count, "CALLS", in_degs, out_degs) == + CBM_STORE_OK) { + for (int j = 0; j < sr_count; j++) { + sr[j].in_degree = in_degs[j]; + sr[j].out_degree = out_degs[j]; + } + } + free(ids); + free(in_degs); + free(out_degs); + } + + /* Compute scores and sort */ + for (int j = 0; j < sr_count; j++) { + sr[j].score = compute_search_score(&sr[j]); + } + if (sr_count > 1) { + qsort(sr, sr_count, sizeof(search_result_t), search_result_cmp); + } + + /* ── Phase 4: Context assembly (extracted helper) ─────────── */ + + char *result = assemble_search_output(sr, sr_count, raw, raw_count, gm_count, limit, mode, + context_lines, root_path); + free(gm); + free(sr); + free(raw); free(root_path); free(pattern); free(project); free(file_pattern); - - char *result = cbm_mcp_text_result(json, false); - free(json); + if (has_path_filter) { + cbm_regfree(&path_regex); + } return result; } @@ -2073,8 +2552,7 @@ static void detect_session(cbm_mcp_server_t *srv) { /* 1. Try CWD */ char cwd[1024]; if (getcwd(cwd, sizeof(cwd)) != NULL) { - // NOLINTNEXTLINE(concurrency-mt-unsafe) - const char *home = getenv("HOME"); + const char *home = cbm_get_home_dir(); /* Skip useless roots: / and $HOME */ if (strcmp(cwd, "/") != 0 && (home == NULL || strcmp(cwd, home) != 0)) { snprintf(srv->session_root, sizeof(srv->session_root), "%s", cwd); @@ -2145,8 +2623,7 @@ static void maybe_auto_index(cbm_mcp_server_t *srv) { } /* Check if project already has a DB */ - // NOLINTNEXTLINE(concurrency-mt-unsafe) - const char *home = getenv("HOME"); + const char *home = cbm_get_home_dir(); if (home) { char db_check[1024]; snprintf(db_check, sizeof(db_check), "%s/.cache/codebase-memory-mcp/%s.db", home, @@ -2375,8 +2852,23 @@ int cbm_mcp_server_run(cbm_mcp_server_t *srv, FILE *in, FILE *out) { for (;;) { /* Poll with idle timeout so we can evict unused stores between requests. - * MCP is request-response (one line at a time), so mixing poll() on the - * raw fd with getline() on the buffered FILE* is safe in practice. */ + * + * IMPORTANT: poll() operates on the raw fd, but getline() reads from a + * buffered FILE*. When a client sends multiple messages in rapid + * succession, the first getline() call may drain ALL kernel data into + * libc's internal FILE* buffer. Subsequent poll() calls then see an + * empty kernel fd and block for STORE_IDLE_TIMEOUT_S seconds even + * though the next messages are already in the FILE* buffer. + * + * Fix (Unix): use a three-phase approach — + * Phase 1: non-blocking poll (timeout=0) to check the kernel fd. + * Phase 2: if Phase 1 returns 0, peek the FILE* buffer via fgetc/ + * ungetc to detect data buffered by a prior getline() call. + * The fd is temporarily set O_NONBLOCK so fgetc() returns + * immediately (EAGAIN → EOF + ferror) instead of blocking + * when the FILE* buffer is empty, which would otherwise + * bypass the Phase 3 idle eviction timeout. + * Phase 3: only if both phases confirm no data, do blocking poll. */ #ifdef _WIN32 /* Windows: WaitForSingleObject on stdin handle */ HANDLE hStdin = (HANDLE)_get_osfhandle(fd); @@ -2389,16 +2881,62 @@ int cbm_mcp_server_run(cbm_mcp_server_t *srv, FILE *in, FILE *out) { continue; } #else + /* Phase 1: non-blocking poll — catches data already in the kernel fd + * AND handles the case where a prior getline() drained the kernel fd + * into libc's FILE* buffer (raw fd appears empty but data is buffered). + * We always try a zero-timeout poll first; if it misses buffered data, + * phase 2 below catches it via an explicit FILE* peek. */ struct pollfd pfd = {.fd = fd, .events = POLLIN}; - int pr = poll(&pfd, 1, STORE_IDLE_TIMEOUT_S * 1000); + int pr = poll(&pfd, 1, 0); /* non-blocking */ if (pr < 0) { break; /* error or signal */ } if (pr == 0) { - /* Timeout — evict idle store to free resources */ - cbm_mcp_server_evict_idle(srv, STORE_IDLE_TIMEOUT_S); - continue; + /* Raw fd appears empty. Check whether libc has already buffered + * data from a previous over-read by peeking one byte via fgetc. + * IMPORTANT: temporarily set O_NONBLOCK so fgetc() returns + * immediately when the FILE* buffer AND kernel fd are both empty + * (EAGAIN → EOF + ferror). Without this, fgetc() on a blocking fd + * would block indefinitely, preventing the Phase 3 idle eviction + * timeout from ever firing. */ + int saved_flags = fcntl(fd, F_GETFL); + if (saved_flags < 0) { + /* fcntl failed (should not happen on a valid fd) — skip the + * FILE* peek and fall straight through to the blocking poll so + * idle eviction still fires on timeout. */ + pr = poll(&pfd, 1, STORE_IDLE_TIMEOUT_S * 1000); + if (pr < 0) { + break; + } + if (pr == 0) { + cbm_mcp_server_evict_idle(srv, STORE_IDLE_TIMEOUT_S); + continue; + } + } else { + (void)fcntl(fd, F_SETFL, saved_flags | O_NONBLOCK); + int c = fgetc(in); + (void)fcntl(fd, F_SETFL, saved_flags); /* restore blocking */ + if (c == EOF) { + if (feof(in)) { + break; /* true EOF */ + } + /* No buffered data (EAGAIN from non-blocking read) — clear + * the ferror indicator set by EAGAIN, then blocking poll. */ + clearerr(in); + pr = poll(&pfd, 1, STORE_IDLE_TIMEOUT_S * 1000); + if (pr < 0) { + break; + } + if (pr == 0) { + cbm_mcp_server_evict_idle(srv, STORE_IDLE_TIMEOUT_S); + continue; + } + } else { + /* Buffered data found — push back and fall through to getline */ + (void)ungetc(c, in); + } + } } #endif diff --git a/src/pipeline/pass_githistory.c b/src/pipeline/pass_githistory.c index 7a0350430..95112d29d 100644 --- a/src/pipeline/pass_githistory.c +++ b/src/pipeline/pass_githistory.c @@ -168,7 +168,8 @@ static int parse_git_log(const char *repo_path, commit_t **out, int *out_count) /* Diff parent_tree → tree to find changed files */ git_diff *diff = NULL; - git_diff_options diff_opts = GIT_DIFF_OPTIONS_INIT; + git_diff_options diff_opts; + git_diff_options_init(&diff_opts, GIT_DIFF_OPTIONS_VERSION); if (git_diff_tree_to_tree(&diff, repo, parent_tree, tree, &diff_opts) == 0) { commit_t current = {0}; diff --git a/src/pipeline/pass_infrascan.c b/src/pipeline/pass_infrascan.c index 30c47d95e..d19439e9e 100644 --- a/src/pipeline/pass_infrascan.c +++ b/src/pipeline/pass_infrascan.c @@ -192,6 +192,31 @@ bool cbm_is_env_file(const char *name) { return false; } +bool cbm_is_kustomize_file(const char *name) { + if (!name) { + return false; + } + char lower[256]; + to_lower(name, lower, sizeof(lower)); + if (strcmp(lower, "kustomization.yaml") == 0) { + return true; + } + return strcmp(lower, "kustomization.yml") == 0; +} + +// NOLINTNEXTLINE(bugprone-easily-swappable-parameters) +bool cbm_is_k8s_manifest(const char *name, const char *content) { + if (!name || !content || cbm_is_kustomize_file(name)) { + return false; + } + enum { K8S_PEEK_SZ = 4097 }; + char buf[K8S_PEEK_SZ]; + size_t n = strnlen(content, 4096); + memcpy(buf, content, n); + buf[n] = '\0'; + return ci_strstr(buf, "apiVersion:") != NULL; +} + // NOLINTNEXTLINE(bugprone-easily-swappable-parameters) bool cbm_is_shell_script(const char *name, const char *ext) { (void)name; diff --git a/src/pipeline/pass_k8s.c b/src/pipeline/pass_k8s.c new file mode 100644 index 000000000..b0a6aa5c3 --- /dev/null +++ b/src/pipeline/pass_k8s.c @@ -0,0 +1,240 @@ +/* + * pass_k8s.c — Pipeline pass for Kubernetes manifest and Kustomize overlay processing. + * + * For each discovered YAML file: + * 1. Check if it is a kustomize overlay (kustomization.yaml / kustomization.yml) + * → emit a Module node and IMPORTS edges for each resources/bases/patches entry + * 2. Else if it is a generic k8s manifest (apiVersion: detected) + * → emit one Resource node per file (first document only — multi-document YAML is not yet + * supported) + * + * Depends on: pass_infrascan.c (cbm_is_kustomize_file, cbm_is_k8s_manifest, cbm_infra_qn), + * extraction layer (cbm.h), graph_buffer, pipeline internals. + */ +#include "pipeline/pipeline.h" +#include "pipeline/pipeline_internal.h" +#include "graph_buffer/graph_buffer.h" +#include "discover/discover.h" +#include "foundation/log.h" +#include "foundation/compat.h" +#include "cbm.h" + +#include +#include +#include + +/* ── Internal helpers ────────────────────────────────────────────── */ + +/* Read entire file into heap-allocated buffer. Returns NULL on error. + * Caller must free(). Sets *out_len to byte count. */ +static char *k8s_read_file(const char *path, int *out_len) { + FILE *f = fopen(path, "rb"); + if (!f) { + return NULL; + } + + (void)fseek(f, 0, SEEK_END); + long size = ftell(f); + (void)fseek(f, 0, SEEK_SET); + + if (size <= 0 || size > (long)100 * 1024 * 1024) { + (void)fclose(f); + return NULL; + } + + char *buf = malloc(size + 1); + if (!buf) { + (void)fclose(f); + return NULL; + } + + size_t nread = fread(buf, 1, size, f); + (void)fclose(f); + // NOLINTNEXTLINE(clang-analyzer-security.ArrayBound) + buf[nread] = '\0'; + *out_len = (int)nread; + return buf; +} + +/* Format int to string for logging. Thread-safe via TLS. */ +static const char *itoa_k8s(int val) { + static CBM_TLS char bufs[4][32]; + static CBM_TLS int idx = 0; + int i = idx; + idx = (idx + 1) & 3; + snprintf(bufs[i], sizeof(bufs[i]), "%d", val); + return bufs[i]; +} + +/* Extract the basename of a path (pointer into the string; no allocation). */ +static const char *k8s_basename(const char *path) { + const char *p = strrchr(path, '/'); + return p ? p + 1 : path; +} + +/* ── Kustomize handler ───────────────────────────────────────────── */ + +static void handle_kustomize(cbm_pipeline_ctx_t *ctx, const char *path, const char *rel_path, + CBMFileResult *result) { + /* Emit Module node for this kustomize overlay file */ + char *mod_qn = cbm_infra_qn(ctx->project_name, rel_path, "kustomize", NULL); + if (!mod_qn) { + return; + } + + // NOLINTNEXTLINE(misc-include-cleaner) + int64_t mod_id = cbm_gbuf_upsert_node(ctx->gbuf, "Module", k8s_basename(rel_path), mod_qn, + rel_path, 1, 0, "{\"source\":\"kustomize\"}"); + free(mod_qn); + + if (mod_id <= 0) { + return; + } + + /* If we have a cached extraction result, emit IMPORTS edges for + * resources/bases/patches/components entries */ + int import_count = 0; + CBMFileResult *res = result; + bool allocated = false; + + if (!res) { + /* Fall back to re-extraction */ + int src_len = 0; + char *source = k8s_read_file(path, &src_len); + if (source) { + res = cbm_extract_file(source, src_len, CBM_LANG_KUSTOMIZE, ctx->project_name, rel_path, + CBM_EXTRACT_BUDGET, NULL, NULL); + free(source); + allocated = true; + } + } + + if (res) { + for (int j = 0; j < res->imports.count; j++) { + CBMImport *imp = &res->imports.items[j]; + if (!imp->module_path) { + continue; + } + + /* Compute target file QN */ + char *target_qn = + cbm_pipeline_fqn_compute(ctx->project_name, imp->module_path, "__file__"); + if (!target_qn) { + continue; + } + + const cbm_gbuf_node_t *target = cbm_gbuf_find_by_qn(ctx->gbuf, target_qn); + free(target_qn); + + if (target) { + cbm_gbuf_insert_edge(ctx->gbuf, mod_id, target->id, "IMPORTS", + "{\"via\":\"kustomize\"}"); + import_count++; + } + } + + if (allocated) { + cbm_free_result(res); + } + } + + cbm_log_info("pass.k8s.kustomize", "file", rel_path, "imports", itoa_k8s(import_count)); +} + +/* ── K8s manifest handler ────────────────────────────────────────── */ + +/* source/src_len are the already-read file bytes (caller retains ownership and + * must free after this call returns). */ +static void handle_k8s_manifest(cbm_pipeline_ctx_t *ctx, const char *path, const char *rel_path, + const char *source, int src_len) { + (void)path; /* retained for symmetry; source is always provided now */ + int resource_count = 0; + + CBMFileResult *res = cbm_extract_file(source, src_len, CBM_LANG_K8S, ctx->project_name, + rel_path, CBM_EXTRACT_BUDGET, NULL, NULL); + if (!res) { + return; + } + + /* Compute file node QN for DEFINES edges */ + char *file_qn = cbm_pipeline_fqn_compute(ctx->project_name, rel_path, "__file__"); + const cbm_gbuf_node_t *file_node = file_qn ? cbm_gbuf_find_by_qn(ctx->gbuf, file_qn) : NULL; + free(file_qn); + + for (int d = 0; d < res->defs.count; d++) { + CBMDefinition *def = &res->defs.items[d]; + if (!def->label || strcmp(def->label, "Resource") != 0) { + continue; + } + if (!def->name || !def->qualified_name) { + continue; + } + + // NOLINTNEXTLINE(misc-include-cleaner) + int64_t node_id = + cbm_gbuf_upsert_node(ctx->gbuf, "Resource", def->name, def->qualified_name, rel_path, + (int)def->start_line, (int)def->end_line, "{\"source\":\"k8s\"}"); + + /* DEFINES edge: File → Resource */ + if (file_node && node_id > 0) { + cbm_gbuf_insert_edge(ctx->gbuf, file_node->id, node_id, "DEFINES", "{}"); + } + + resource_count++; + } + + cbm_free_result(res); + + cbm_log_info("pass.k8s.manifest", "file", rel_path, "resources", itoa_k8s(resource_count)); +} + +/* ── Pass entry point ────────────────────────────────────────────── */ + +// NOLINTNEXTLINE(misc-include-cleaner) — cbm_file_info_t provided by standard header +int cbm_pipeline_pass_k8s(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files, int file_count) { + cbm_log_info("pass.start", "pass", "k8s", "files", itoa_k8s(file_count)); + + cbm_init(); + + int kustomize_count = 0; + int manifest_count = 0; + + for (int i = 0; i < file_count; i++) { + if (cbm_pipeline_check_cancel(ctx)) { + return -1; + } + + const char *path = files[i].path; + const char *rel = files[i].rel_path; + CBMLanguage lang = files[i].language; + const char *base = k8s_basename(rel); + + CBMFileResult *cached = + (ctx->result_cache && ctx->result_cache[i]) ? ctx->result_cache[i] : NULL; + + if (cbm_is_kustomize_file(base)) { + handle_kustomize(ctx, path, rel, cached); + kustomize_count++; + } else if (lang == CBM_LANG_YAML || lang == CBM_LANG_K8S) { + /* Read source once to classify (and reuse for uncached extraction). */ + int src_len = 0; + char *source = k8s_read_file(path, &src_len); + if (source) { + if (cbm_is_k8s_manifest(base, source)) { + /* Always re-extract with CBM_LANG_K8S regardless of any cached + * result: cached results were produced during the parallel YAML + * pass and contain no "Resource" definitions. Pass the already- + * read source buffer so handle_k8s_manifest does not re-read. */ + (void)cached; /* cached YAML result intentionally discarded */ + handle_k8s_manifest(ctx, path, rel, source, src_len); + manifest_count++; + } + free(source); + } + } + } + + cbm_log_info("pass.done", "pass", "k8s", "kustomize", itoa_k8s(kustomize_count), "manifests", + itoa_k8s(manifest_count)); + return 0; +} diff --git a/src/pipeline/pipeline.c b/src/pipeline/pipeline.c index f5b75108d..8b480d24d 100644 --- a/src/pipeline/pipeline.c +++ b/src/pipeline/pipeline.c @@ -17,6 +17,7 @@ #include "graph_buffer/graph_buffer.h" #include "store/store.h" #include "discover/discover.h" +#include "discover/userconfig.h" #include "foundation/platform.h" #include "foundation/compat_fs.h" #include "foundation/log.h" @@ -44,6 +45,9 @@ struct cbm_pipeline { /* Indexing state (set during run) */ cbm_gbuf_t *gbuf; cbm_registry_t *registry; + + /* User-defined extension overrides (loaded once per run) */ + cbm_userconfig_t *userconfig; }; /* ── Timing helper ──────────────────────────────────────────────── */ @@ -97,6 +101,12 @@ void cbm_pipeline_free(cbm_pipeline_t *p) { free(p->db_path); free(p->project_name); /* gbuf, store, registry freed during/after run */ + /* Defensively free userconfig in case run() was never called or panicked */ + if (p->userconfig) { + cbm_set_user_lang_config(NULL); + cbm_userconfig_free(p->userconfig); + p->userconfig = NULL; + } free(p); } @@ -127,10 +137,9 @@ static char *resolve_db_path(const cbm_pipeline_t *p) { if (p->db_path) { snprintf(path, 1024, "%s", p->db_path); } else { - // NOLINTNEXTLINE(concurrency-mt-unsafe) - const char *home = getenv("HOME"); + const char *home = cbm_get_home_dir(); if (!home) { - home = "/tmp"; + home = cbm_tmpdir(); } snprintf(path, 1024, "%s/.cache/codebase-memory-mcp/%s.db", home, p->project_name); } @@ -303,6 +312,10 @@ int cbm_pipeline_run(cbm_pipeline_t *p) { struct timespec t0; cbm_clock_gettime(CLOCK_MONOTONIC, &t0); + /* Load user-defined extension overrides (fail-open: NULL on error) */ + p->userconfig = cbm_userconfig_load(p->repo_path); + cbm_set_user_lang_config(p->userconfig); + /* Phase 1: Discover files */ cbm_discover_opts_t opts = { .mode = p->mode, @@ -314,6 +327,10 @@ int cbm_pipeline_run(cbm_pipeline_t *p) { int rc = cbm_discover(p->repo_path, &opts, &files, &file_count); if (rc != 0) { cbm_log_error("pipeline.err", "phase", "discover", "rc", itoa_buf(rc)); + cbm_discover_free(files, file_count); + cbm_set_user_lang_config(NULL); + cbm_userconfig_free(p->userconfig); + p->userconfig = NULL; return -1; } cbm_log_info("pipeline.discover", "files", itoa_buf(file_count), "elapsed_ms", @@ -321,6 +338,9 @@ int cbm_pipeline_run(cbm_pipeline_t *p) { if (check_cancel(p)) { cbm_discover_free(files, file_count); + cbm_set_user_lang_config(NULL); + cbm_userconfig_free(p->userconfig); + p->userconfig = NULL; return -1; } @@ -494,6 +514,16 @@ int cbm_pipeline_run(cbm_pipeline_t *p) { rc = -1; goto cleanup; } + + cbm_clock_gettime(CLOCK_MONOTONIC, &t); + rc = cbm_pipeline_pass_k8s(&ctx, files, file_count); + if (rc != 0) { /* log warning, continue */ + } + cbm_log_info("pass.timing", "pass", "k8s", "elapsed_ms", itoa_buf((int)elapsed_ms(t))); + if (check_cancel(p)) { + rc = -1; + goto cleanup; + } } else { cbm_log_info("pipeline.mode", "mode", "sequential", "files", itoa_buf(file_count)); @@ -518,6 +548,16 @@ int cbm_pipeline_run(cbm_pipeline_t *p) { goto seq_cleanup; } + cbm_clock_gettime(CLOCK_MONOTONIC, &t); + rc = cbm_pipeline_pass_k8s(&ctx, files, file_count); + if (rc != 0) { /* log warning, continue */ + } + cbm_log_info("pass.timing", "pass", "k8s", "elapsed_ms", itoa_buf((int)elapsed_ms(t))); + if (check_cancel(p)) { + rc = -1; + goto seq_cleanup; + } + cbm_clock_gettime(CLOCK_MONOTONIC, &t); rc = cbm_pipeline_pass_calls(&ctx, files, file_count); if (rc != 0) { @@ -680,14 +720,13 @@ int cbm_pipeline_run(cbm_pipeline_t *p) { if (!check_cancel(p)) { cbm_clock_gettime(CLOCK_MONOTONIC, &t); - // NOLINTNEXTLINE(concurrency-mt-unsafe) — called once during single-threaded dump - const char *home = getenv("HOME"); + const char *home = cbm_get_home_dir(); char db_path[1024]; if (p->db_path) { snprintf(db_path, sizeof(db_path), "%s", p->db_path); } else { if (!home) { - home = "/tmp"; + home = cbm_tmpdir(); } snprintf(db_path, sizeof(db_path), "%s/.cache/codebase-memory-mcp/%s.db", home, p->project_name); @@ -759,5 +798,9 @@ cleanup: p->gbuf = NULL; cbm_registry_free(p->registry); p->registry = NULL; + /* Clear and free user extension config */ + cbm_set_user_lang_config(NULL); + cbm_userconfig_free(p->userconfig); + p->userconfig = NULL; return rc; } diff --git a/src/pipeline/pipeline_incremental.c b/src/pipeline/pipeline_incremental.c index 78e2ffde6..1799f8389 100644 --- a/src/pipeline/pipeline_incremental.c +++ b/src/pipeline/pipeline_incremental.c @@ -265,6 +265,18 @@ int cbm_pipeline_run_incremental(cbm_pipeline_t *p, const char *db_path, cbm_fil cbm_log_info("pass.timing", "pass", "incr_semantic", "elapsed_ms", itoa_buf((int)elapsed_ms(t))); + /* k8s pass runs after semantic (vs. after definitions in the full pipeline) because + * incremental has no parallel extraction phase to position it alongside. + * Note: File→Resource DEFINES edges and cross-file kustomize IMPORTS edges are not + * emitted here — File nodes (from pass_structure) are absent in the incremental gbuf, + * and gbuf_find_by_qn only resolves nodes from changed files. This is a known + * structural limitation of the incremental architecture. */ + cbm_clock_gettime(CLOCK_MONOTONIC, &t); + if (cbm_pipeline_pass_k8s(&ctx, changed_files, ci) != 0) { + cbm_log_info("incremental.warn", "msg", "k8s_pass_failed"); + } + cbm_log_info("pass.timing", "pass", "incr_k8s", "elapsed_ms", itoa_buf((int)elapsed_ms(t))); + /* Merge new nodes/edges from gbuf into disk DB */ int new_nodes = cbm_gbuf_node_count(gbuf); int new_edges = cbm_gbuf_edge_count(gbuf); diff --git a/src/pipeline/pipeline_internal.h b/src/pipeline/pipeline_internal.h index c1d45f14f..88862ff1d 100644 --- a/src/pipeline/pipeline_internal.h +++ b/src/pipeline/pipeline_internal.h @@ -218,6 +218,8 @@ bool cbm_is_compose_file(const char *name); bool cbm_is_cloudbuild_file(const char *name); bool cbm_is_env_file(const char *name); bool cbm_is_shell_script(const char *name, const char *ext); +bool cbm_is_kustomize_file(const char *name); +bool cbm_is_k8s_manifest(const char *name, const char *content); /* Secret detection */ bool cbm_is_secret_binding(const char *key, const char *value); @@ -353,6 +355,8 @@ int cbm_parallel_resolve(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files, int cbm_pipeline_pass_definitions(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files, int file_count); +int cbm_pipeline_pass_k8s(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files, int file_count); + int cbm_pipeline_pass_calls(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files, int file_count); /* Sub-passes called from pass_calls: pattern-based edge extraction */ diff --git a/src/store/store.c b/src/store/store.c index 4b00d4e73..bfbff59a2 100644 --- a/src/store/store.c +++ b/src/store/store.c @@ -370,10 +370,9 @@ cbm_store_t *cbm_store_open(const char *project) { return NULL; } /* Build path: ~/.cache/codebase-memory-mcp/.db */ - const char *home = getenv("HOME"); // NOLINT(concurrency-mt-unsafe) — called once during - // single-threaded store open, never concurrently + const char *home = cbm_get_home_dir(); if (!home) { - home = "/tmp"; + home = cbm_tmpdir(); } char path[1024]; snprintf(path, sizeof(path), "%s/.cache/codebase-memory-mcp/%s.db", home, project); @@ -1431,6 +1430,43 @@ void cbm_store_node_degree(cbm_store_t *s, int64_t node_id, int *in_deg, int *ou } } +/* ── List distinct file paths ────────────────────────────────── */ + +int cbm_store_list_files(cbm_store_t *s, const char *project, char ***out, int *count) { + *out = NULL; + *count = 0; + if (!s || !s->db || !project) { + return CBM_STORE_ERR; + } + + const char *sql = "SELECT DISTINCT file_path FROM nodes " + "WHERE project = ?1 AND file_path IS NOT NULL AND file_path != ''"; + sqlite3_stmt *stmt = NULL; + if (sqlite3_prepare_v2(s->db, sql, -1, &stmt, NULL) != SQLITE_OK) { + return CBM_STORE_ERR; + } + sqlite3_bind_text(stmt, 1, project, -1, SQLITE_STATIC); + + int cap = 64; + int n = 0; + char **files = malloc(cap * sizeof(char *)); + while (sqlite3_step(stmt) == SQLITE_ROW) { + const char *fp = (const char *)sqlite3_column_text(stmt, 0); + if (!fp) { + continue; + } + if (n >= cap) { + cap *= 2; + files = safe_realloc(files, cap * sizeof(char *)); + } + files[n++] = heap_strdup(fp); + } + sqlite3_finalize(stmt); + *out = files; + *count = n; + return CBM_STORE_OK; +} + /* ── Node neighbor names ──────────────────────────────────────── */ // NOLINTNEXTLINE(bugprone-easily-swappable-parameters) diff --git a/src/store/store.h b/src/store/store.h index 0bf385a82..94de109c4 100644 --- a/src/store/store.h +++ b/src/store/store.h @@ -73,6 +73,10 @@ int cbm_store_find_nodes_by_qn_suffix(cbm_store_t *s, const char *project, const /* Get CALLS degree of a node (inbound and outbound). */ void cbm_store_node_degree(cbm_store_t *s, int64_t node_id, int *in_deg, int *out_deg); +/* Get distinct file paths for a project. Caller must free each out[i] and out itself. + * Returns CBM_STORE_OK or CBM_STORE_ERR. */ +int cbm_store_list_files(cbm_store_t *s, const char *project, char ***out, int *count); + /* Get caller/callee names for a node (CALLS/HTTP_CALLS/ASYNC_CALLS edges). * Returns 0 on success. Caller must free each out_callers[i]/out_callees[i] * and the arrays themselves. */ diff --git a/src/ui/config.c b/src/ui/config.c index 962d0ed72..7ac5b41c0 100644 --- a/src/ui/config.c +++ b/src/ui/config.c @@ -6,7 +6,9 @@ */ #include "ui/config.h" #include "foundation/log.h" +#include "foundation/platform.h" #include "foundation/compat_fs.h" +#include "foundation/compat.h" #include @@ -17,9 +19,9 @@ /* ── Path ────────────────────────────────────────────────────── */ void cbm_ui_config_path(char *buf, int bufsz) { - const char *home = getenv("HOME"); // NOLINT(concurrency-mt-unsafe) + const char *home = cbm_get_home_dir(); if (!home) { - home = "/tmp"; + home = cbm_tmpdir(); } snprintf(buf, (size_t)bufsz, "%s/.cache/codebase-memory-mcp/config.json", home); } diff --git a/src/ui/http_server.c b/src/ui/http_server.c index a2a30f374..b3153b24d 100644 --- a/src/ui/http_server.c +++ b/src/ui/http_server.c @@ -377,7 +377,7 @@ static void handle_browse(struct mg_connection *c, struct mg_http_message *hm) { char path[1024] = {0}; if (!get_query_param(hm->query, "path", path, (int)sizeof(path)) || path[0] == '\0') { /* Default to home directory */ - const char *home = getenv("HOME"); + const char *home = cbm_get_home_dir(); if (home) snprintf(path, sizeof(path), "%s", home); else @@ -452,9 +452,9 @@ static void handle_adr_get(struct mg_connection *c, struct mg_http_message *hm) return; } - const char *home = getenv("HOME"); + const char *home = cbm_get_home_dir(); if (!home) - home = "/tmp"; + home = cbm_tmpdir(); char db_path[1024]; snprintf(db_path, sizeof(db_path), "%s/.cache/codebase-memory-mcp/%s.db", home, name); @@ -545,9 +545,9 @@ static void handle_adr_save(struct mg_connection *c, struct mg_http_message *hm) const char *proj = yyjson_get_str(v_proj); const char *content = yyjson_get_str(v_content); - const char *home = getenv("HOME"); + const char *home = cbm_get_home_dir(); if (!home) - home = "/tmp"; + home = cbm_tmpdir(); char db_path[1024]; snprintf(db_path, sizeof(db_path), "%s/.cache/codebase-memory-mcp/%s.db", home, proj); @@ -832,9 +832,9 @@ static void handle_delete_project(struct mg_connection *c, struct mg_http_messag return; } - const char *home = getenv("HOME"); // NOLINT(concurrency-mt-unsafe) + const char *home = cbm_get_home_dir(); if (!home) - home = "/tmp"; + home = cbm_tmpdir(); char db_path[1024]; snprintf(db_path, sizeof(db_path), "%s/.cache/codebase-memory-mcp/%s.db", home, name); @@ -870,9 +870,9 @@ static void handle_project_health(struct mg_connection *c, struct mg_http_messag return; } - const char *home = getenv("HOME"); // NOLINT(concurrency-mt-unsafe) + const char *home = cbm_get_home_dir(); if (!home) - home = "/tmp"; + home = cbm_tmpdir(); char db_path[1024]; snprintf(db_path, sizeof(db_path), "%s/.cache/codebase-memory-mcp/%s.db", home, name); @@ -928,9 +928,9 @@ static void handle_layout(struct mg_connection *c, struct mg_http_message *hm) { } /* Open a read-only store for this project */ - const char *home = getenv("HOME"); // NOLINT(concurrency-mt-unsafe) + const char *home = cbm_get_home_dir(); if (!home) - home = "/tmp"; + home = cbm_tmpdir(); char db_path[1024]; snprintf(db_path, sizeof(db_path), "%s/.cache/codebase-memory-mcp/%s.db", home, project); diff --git a/tests/test_cli.c b/tests/test_cli.c index 19ccdecaa..788a4807a 100644 --- a/tests/test_cli.c +++ b/tests/test_cli.c @@ -1674,7 +1674,7 @@ TEST(cli_upsert_claude_hook_fresh) { ASSERT_NOT_NULL(data); ASSERT(strstr(data, "PreToolUse") != NULL); ASSERT(strstr(data, "Grep|Glob|Read") != NULL); - ASSERT(strstr(data, "codebase-memory-mcp") != NULL); + ASSERT(strstr(data, "cbm-code-discovery-gate") != NULL); test_rmdir_r(tmpdir); PASS(); @@ -1722,9 +1722,9 @@ TEST(cli_upsert_claude_hook_replace) { const char *data = read_test_file(settingspath); ASSERT_NOT_NULL(data); - /* Old message gone, new message present */ + /* Old message gone, new hook script path present */ ASSERT(strstr(data, "old-cmm-message") == NULL); - ASSERT(strstr(data, "codebase-memory-mcp") != NULL); + ASSERT(strstr(data, "cbm-code-discovery-gate") != NULL); test_rmdir_r(tmpdir); PASS(); diff --git a/tests/test_main.c b/tests/test_main.c index c6163bc93..c6f1f05ef 100644 --- a/tests/test_main.c +++ b/tests/test_main.c @@ -26,6 +26,7 @@ extern void suite_store_search(void); extern void suite_cypher(void); extern void suite_mcp(void); extern void suite_language(void); +extern void suite_userconfig(void); extern void suite_gitignore(void); extern void suite_discover(void); extern void suite_graph_buffer(void); @@ -81,6 +82,7 @@ int main(void) { /* Discover (M2) */ RUN_SUITE(language); + RUN_SUITE(userconfig); RUN_SUITE(gitignore); RUN_SUITE(discover); diff --git a/tests/test_mcp.c b/tests/test_mcp.c index a63412712..bd07cba25 100644 --- a/tests/test_mcp.c +++ b/tests/test_mcp.c @@ -1215,6 +1215,83 @@ TEST(snippet_include_neighbors_enabled) { PASS(); } +/* ══════════════════════════════════════════════════════════════════ + * POLL/GETLINE FILE* BUFFERING FIX + * ══════════════════════════════════════════════════════════════════ */ + +#ifndef _WIN32 +#include +#include + +/* Signal handler used by alarm() to abort the test if it hangs */ +static void alarm_handler(int sig) { + (void)sig; + /* Writing to stderr is async-signal-safe */ + const char msg[] = "FAIL: mcp_server_run_rapid_messages timed out (>5s)\n"; + write(STDERR_FILENO, msg, sizeof(msg) - 1); + _exit(1); +} + +TEST(mcp_server_run_rapid_messages) { + /* Simulate a client sending initialize + notifications/initialized + + * tools/list all at once (no delays), which exercises the FILE* + * buffering fix: the first getline() over-reads kernel data into the + * libc buffer; without the fix, subsequent poll() calls block for 60s. + * + * We use alarm(5) to abort the test process if the server hangs. */ + int fds[2]; + ASSERT_EQ(pipe(fds), 0); + + /* Write all 3 messages to the write end in one shot */ + const char *msgs = + "{\"jsonrpc\":\"2.0\",\"id\":1,\"method\":\"initialize\"," + "\"params\":{\"protocolVersion\":\"2025-11-25\",\"capabilities\":{}}}\n" + "{\"jsonrpc\":\"2.0\",\"method\":\"notifications/initialized\"}\n" + "{\"jsonrpc\":\"2.0\",\"id\":2,\"method\":\"tools/list\",\"params\":{}}\n"; + ssize_t written = write(fds[1], msgs, strlen(msgs)); + ASSERT_TRUE(written > 0); + close(fds[1]); /* EOF signals end of input to the server */ + + FILE *in_fp = fdopen(fds[0], "r"); + ASSERT_NOT_NULL(in_fp); + + FILE *out_fp = tmpfile(); + ASSERT_NOT_NULL(out_fp); + + cbm_mcp_server_t *srv = cbm_mcp_server_new(NULL); + ASSERT_NOT_NULL(srv); + + /* Install alarm to fail the test if cbm_mcp_server_run blocks */ + signal(SIGALRM, alarm_handler); + alarm(5); + + int rc = cbm_mcp_server_run(srv, in_fp, out_fp); + + alarm(0); /* cancel alarm */ + signal(SIGALRM, SIG_DFL); + + ASSERT_EQ(rc, 0); + + /* Verify both responses are present: + * id:1 — initialize response + * id:2 — tools/list response (notifications/initialized produces none) + * and that the tools list payload is included. */ + rewind(out_fp); + char buf[4096] = {0}; + size_t nread = fread(buf, 1, sizeof(buf) - 1, out_fp); + ASSERT_TRUE(nread > 0); + ASSERT_NOT_NULL(strstr(buf, "\"id\":1")); + ASSERT_NOT_NULL(strstr(buf, "\"id\":2")); + ASSERT_NOT_NULL(strstr(buf, "tools")); + + cbm_mcp_server_free(srv); + fclose(out_fp); + /* in_fp already EOF; fclose cleans up */ + fclose(in_fp); + PASS(); +} +#endif /* !_WIN32 */ + /* ══════════════════════════════════════════════════════════════════ * SUITE * ══════════════════════════════════════════════════════════════════ */ @@ -1287,6 +1364,11 @@ SUITE(mcp) { RUN_TEST(parse_file_uri_windows); RUN_TEST(parse_file_uri_invalid); + /* Poll/getline FILE* buffering fix */ +#ifndef _WIN32 + RUN_TEST(mcp_server_run_rapid_messages); +#endif + /* Snippet resolution (port of snippet_test.go) */ RUN_TEST(snippet_exact_qn); RUN_TEST(snippet_qn_suffix); diff --git a/tests/test_pipeline.c b/tests/test_pipeline.c index c2302a20e..140a267ba 100644 --- a/tests/test_pipeline.c +++ b/tests/test_pipeline.c @@ -3331,6 +3331,30 @@ TEST(infra_is_dockerfile) { PASS(); } +TEST(infra_is_kustomize_file) { + ASSERT(cbm_is_kustomize_file("kustomization.yaml")); + ASSERT(cbm_is_kustomize_file("kustomization.yml")); + ASSERT(cbm_is_kustomize_file("KUSTOMIZATION.YAML")); /* case-insensitive */ + ASSERT(!cbm_is_kustomize_file("deployment.yaml")); + ASSERT(!cbm_is_kustomize_file("kustomize.yaml")); + ASSERT(!cbm_is_kustomize_file(NULL)); + PASS(); +} + +TEST(infra_is_k8s_manifest) { + const char *deploy = "apiVersion: apps/v1\nkind: Deployment\nmetadata:\n name: my-app\n"; + const char *plain = "name: foo\nvalue: bar\n"; + const char *kust = "apiVersion: kustomize.config.k8s.io/v1beta1\nkind: Kustomization\n"; + + ASSERT(cbm_is_k8s_manifest("deployment.yaml", deploy)); + ASSERT(!cbm_is_k8s_manifest("deployment.yaml", plain)); + /* kustomize file should return false even if it has apiVersion */ + ASSERT(!cbm_is_k8s_manifest("kustomization.yaml", kust)); + ASSERT(!cbm_is_k8s_manifest(NULL, deploy)); + ASSERT(!cbm_is_k8s_manifest("deployment.yaml", NULL)); + PASS(); +} + TEST(infra_is_env_file) { ASSERT(cbm_is_env_file(".env")); ASSERT(cbm_is_env_file(".env.local")); @@ -4139,6 +4163,114 @@ TEST(infra_pipeline_idempotent) { PASS(); } +/* ── K8s / Kustomize extraction tests ──────────────────────────── */ + +TEST(k8s_extract_kustomize) { + const char *src = + "apiVersion: kustomize.config.k8s.io/v1beta1\n" + "kind: Kustomization\n" + "resources:\n" + " - deployment.yaml\n" + " - service.yaml\n"; + CBMFileResult *r = cbm_extract_file(src, (int)strlen(src), CBM_LANG_KUSTOMIZE, + "myproj", "base/kustomization.yaml", + 0, NULL, NULL); + ASSERT(r != NULL); + ASSERT_GTE(r->imports.count, 2); + + bool found_deploy = false, found_svc = false; + for (int i = 0; i < r->imports.count; i++) { + if (r->imports.items[i].module_path && + strcmp(r->imports.items[i].module_path, "deployment.yaml") == 0) + found_deploy = true; + if (r->imports.items[i].module_path && + strcmp(r->imports.items[i].module_path, "service.yaml") == 0) + found_svc = true; + } + ASSERT_TRUE(found_deploy); + ASSERT_TRUE(found_svc); + + cbm_free_result(r); + PASS(); +} + +TEST(k8s_extract_manifest) { + const char *src = + "apiVersion: apps/v1\n" + "kind: Deployment\n" + "metadata:\n" + " name: my-app\n" + " namespace: production\n"; + CBMFileResult *r = cbm_extract_file(src, (int)strlen(src), CBM_LANG_K8S, + "myproj", "k8s/deployment.yaml", + 0, NULL, NULL); + ASSERT(r != NULL); + ASSERT_GTE(r->defs.count, 1); + + bool found_resource = false; + for (int d = 0; d < r->defs.count; d++) { + if (r->defs.items[d].label && + strcmp(r->defs.items[d].label, "Resource") == 0 && + r->defs.items[d].name && + strstr(r->defs.items[d].name, "Deployment") != NULL) + found_resource = true; + } + ASSERT_TRUE(found_resource); + + cbm_free_result(r); + PASS(); +} + +TEST(k8s_extract_manifest_no_name) { + const char *src = "apiVersion: apps/v1\nkind: Deployment\n"; + CBMFileResult *r = cbm_extract_file(src, (int)strlen(src), CBM_LANG_K8S, + "myproj", "k8s/deploy.yaml", 0, NULL, NULL); + ASSERT(r != NULL); + /* No crash — defs count may be 0 because metadata.name is absent */ + ASSERT(!r->has_error); + cbm_free_result(r); + PASS(); +} + +TEST(k8s_extract_manifest_multidoc) { + /* Two-document YAML separated by "---". + * extract_k8s_manifest contains a "break" after the first successful push, + * so it processes only the first document that has both kind and + * metadata.name. This test pins that behaviour: the first document's + * resource must be present and no crash must occur. + * + * Note: with some tree-sitter YAML grammar versions the root stream may + * expose both documents as siblings; the break still fires after the first + * successful def push, so defs.count must be exactly 1. */ + const char *src = + "apiVersion: apps/v1\n" + "kind: Deployment\n" + "metadata:\n" + " name: my-app\n" + "---\n" + "apiVersion: v1\n" + "kind: Service\n" + "metadata:\n" + " name: my-svc\n"; + CBMFileResult *r = cbm_extract_file(src, (int)strlen(src), CBM_LANG_K8S, + "myproj", "k8s/multi.yaml", 0, NULL, NULL); + ASSERT(r != NULL); + ASSERT(!r->has_error); + /* First document's resource must be present */ + int found = 0; + for (int i = 0; i < r->defs.count; i++) { + if (r->defs.items[i].label && strcmp(r->defs.items[i].label, "Resource") == 0 && + r->defs.items[i].name && strcmp(r->defs.items[i].name, "Deployment/my-app") == 0) { + found = 1; + } + } + ASSERT(found); + /* At least one def, no more than one (only first document processed) */ + ASSERT(r->defs.count >= 1); + cbm_free_result(r); + PASS(); +} + /* ── Envscan tests (port of envscan_test.go) ───────────────────── */ /* Helper: write a file inside a temp dir */ @@ -4947,6 +5079,140 @@ TEST(incremental_new_file_added) { PASS(); } +TEST(incremental_k8s_manifest_indexed) { + /* Full index with a k8s manifest, then add a new manifest via incremental. + * Verifies that cbm_pipeline_pass_k8s() runs during incremental re-index. */ + char tmpdir[256]; + snprintf(tmpdir, sizeof(tmpdir), "/tmp/cbm_k8s_incr_XXXXXX"); + if (!cbm_mkdtemp(tmpdir)) { + SKIP("tmpdir"); + } + char dbpath[512]; + snprintf(dbpath, sizeof(dbpath), "%s/test.db", tmpdir); + char path[512]; + FILE *f; + + /* Initial manifest */ + snprintf(path, sizeof(path), "%s/deploy.yaml", tmpdir); + f = fopen(path, "w"); + ASSERT_NOT_NULL(f); + fprintf(f, "apiVersion: apps/v1\nkind: Deployment\nmetadata:\n name: my-app\n"); + fclose(f); + + /* Full index */ + cbm_pipeline_t *p = cbm_pipeline_new(tmpdir, dbpath, CBM_MODE_FULL); + ASSERT_NOT_NULL(p); + ASSERT_EQ(cbm_pipeline_run(p), 0); + char *project = strdup(cbm_pipeline_project_name(p)); + cbm_pipeline_free(p); + + /* Verify Resource node created by full index */ + cbm_store_t *s = cbm_store_open_path(dbpath); + ASSERT_NOT_NULL(s); + cbm_node_t *nodes = NULL; + int count = 0; + cbm_store_find_nodes_by_label(s, project, "Resource", &nodes, &count); + ASSERT_GT(count, 0); + cbm_store_free_nodes(nodes, count); + cbm_store_close(s); + + /* Add a second manifest — incremental should pick it up */ + snprintf(path, sizeof(path), "%s/svc.yaml", tmpdir); + f = fopen(path, "w"); + ASSERT_NOT_NULL(f); + fprintf(f, "apiVersion: v1\nkind: Service\nmetadata:\n name: my-svc\n"); + fclose(f); + + /* Incremental re-index */ + p = cbm_pipeline_new(tmpdir, dbpath, CBM_MODE_FULL); + ASSERT_NOT_NULL(p); + ASSERT_EQ(cbm_pipeline_run(p), 0); + cbm_pipeline_free(p); + + /* Verify both Resource nodes now present */ + s = cbm_store_open_path(dbpath); + ASSERT_NOT_NULL(s); + nodes = NULL; + count = 0; + cbm_store_find_nodes_by_label(s, project, "Resource", &nodes, &count); + ASSERT_GTE(count, 2); + cbm_store_free_nodes(nodes, count); + cbm_store_close(s); + + free(project); + char cmd[512]; + snprintf(cmd, sizeof(cmd), "rm -rf '%s'", tmpdir); + (void)system(cmd); + PASS(); +} + +TEST(incremental_kustomize_module_indexed) { + /* Verifies that a kustomization.yaml added after the initial full index + * gets a Module node via the incremental k8s pass. */ + char tmpdir[256]; + snprintf(tmpdir, sizeof(tmpdir), "/tmp/cbm_kust_incr_XXXXXX"); + if (!cbm_mkdtemp(tmpdir)) { + SKIP("tmpdir"); + } + char dbpath[512]; + snprintf(dbpath, sizeof(dbpath), "%s/test.db", tmpdir); + char path[512]; + FILE *f; + + /* Initial resource manifest (gives full index something to find) */ + snprintf(path, sizeof(path), "%s/deploy.yaml", tmpdir); + f = fopen(path, "w"); + ASSERT_NOT_NULL(f); + fprintf(f, "apiVersion: apps/v1\nkind: Deployment\nmetadata:\n name: my-app\n"); + fclose(f); + + /* Full index */ + cbm_pipeline_t *p = cbm_pipeline_new(tmpdir, dbpath, CBM_MODE_FULL); + ASSERT_NOT_NULL(p); + ASSERT_EQ(cbm_pipeline_run(p), 0); + char *project = strdup(cbm_pipeline_project_name(p)); + cbm_pipeline_free(p); + + /* Add kustomization.yaml */ + snprintf(path, sizeof(path), "%s/kustomization.yaml", tmpdir); + f = fopen(path, "w"); + ASSERT_NOT_NULL(f); + fprintf(f, "apiVersion: kustomize.config.k8s.io/v1beta1\n" + "kind: Kustomization\n" + "resources:\n" + " - deploy.yaml\n"); + fclose(f); + + /* Incremental re-index */ + p = cbm_pipeline_new(tmpdir, dbpath, CBM_MODE_FULL); + ASSERT_NOT_NULL(p); + ASSERT_EQ(cbm_pipeline_run(p), 0); + cbm_pipeline_free(p); + + /* Verify Module node created for the kustomization overlay */ + cbm_store_t *s = cbm_store_open_path(dbpath); + ASSERT_NOT_NULL(s); + cbm_node_t *nodes = NULL; + int count = 0; + cbm_store_find_nodes_by_label(s, project, "Module", &nodes, &count); + bool found_kust = false; + for (int i = 0; i < count; i++) { + if (nodes[i].properties_json && strstr(nodes[i].properties_json, "kustomize")) { + found_kust = true; + break; + } + } + cbm_store_free_nodes(nodes, count); + cbm_store_close(s); + ASSERT_TRUE(found_kust); + + free(project); + char cmd[512]; + snprintf(cmd, sizeof(cmd), "rm -rf '%s'", tmpdir); + (void)system(cmd); + PASS(); +} + SUITE(pipeline) { /* Lifecycle */ RUN_TEST(pipeline_create_free); @@ -5055,6 +5321,8 @@ SUITE(pipeline) { RUN_TEST(infra_is_cloudbuild_file); RUN_TEST(infra_is_shell_script); RUN_TEST(infra_is_dockerfile); + RUN_TEST(infra_is_kustomize_file); + RUN_TEST(infra_is_k8s_manifest); RUN_TEST(infra_is_env_file); RUN_TEST(infra_clean_json_brackets); RUN_TEST(infra_secret_detection); @@ -5083,6 +5351,11 @@ SUITE(pipeline) { /* Infrascan: pipeline integration */ RUN_TEST(infra_pipeline_integration); RUN_TEST(infra_pipeline_idempotent); + /* K8s / Kustomize extraction */ + RUN_TEST(k8s_extract_kustomize); + RUN_TEST(k8s_extract_manifest); + RUN_TEST(k8s_extract_manifest_no_name); + RUN_TEST(k8s_extract_manifest_multidoc); /* Env URL scanning */ RUN_TEST(envscan_dockerfile_env_urls); RUN_TEST(envscan_shell_env_urls); @@ -5130,4 +5403,6 @@ SUITE(pipeline) { RUN_TEST(incremental_detects_changed_file); RUN_TEST(incremental_detects_deleted_file); RUN_TEST(incremental_new_file_added); + RUN_TEST(incremental_k8s_manifest_indexed); + RUN_TEST(incremental_kustomize_module_indexed); } diff --git a/tests/test_userconfig.c b/tests/test_userconfig.c new file mode 100644 index 000000000..efaf61d19 --- /dev/null +++ b/tests/test_userconfig.c @@ -0,0 +1,218 @@ +/* + * test_userconfig.c — Tests for user-defined extension→language mappings. + * + * Tests cbm_userconfig_load(), cbm_userconfig_lookup(), and the + * cbm_set_user_lang_config() / cbm_language_for_extension() integration. + */ +#include "../src/foundation/compat.h" +#include "../src/foundation/compat_fs.h" +#include "test_framework.h" +#include "discover/discover.h" +#include "discover/userconfig.h" + +#include +#include +#include + +/* ── Helpers ─────────────────────────────────────────────────────── */ + +/* Write a JSON file to path. Returns 0 on success. */ +static int write_json(const char *path, const char *json) { + FILE *f = fopen(path, "w"); + if (!f) { + return -1; + } + fputs(json, f); + fclose(f); + return 0; +} + +/* ── Tests: project config ───────────────────────────────────────── */ + +TEST(userconfig_project_basic) { + /* Write a .codebase-memory.json in a temp dir */ + char dir[256]; + snprintf(dir, sizeof(dir), "%s/uctest_proj_basic", cbm_tmpdir()); + cbm_mkdir_p(dir, 0755); /* from compat_fs.h via compat.h */ + + char proj[512]; + snprintf(proj, sizeof(proj), "%s/.codebase-memory.json", dir); + ASSERT_EQ( + write_json(proj, "{\"extra_extensions\":{\".blade.php\":\"php\",\".mjs\":\"javascript\"}}"), + 0); + + cbm_userconfig_t *cfg = cbm_userconfig_load(dir); + ASSERT_NOT_NULL(cfg); + + ASSERT_EQ(cbm_userconfig_lookup(cfg, ".blade.php"), CBM_LANG_PHP); + ASSERT_EQ(cbm_userconfig_lookup(cfg, ".mjs"), CBM_LANG_JAVASCRIPT); + ASSERT_EQ(cbm_userconfig_lookup(cfg, ".go"), CBM_LANG_COUNT); /* not in user config */ + + cbm_userconfig_free(cfg); + remove(proj); + PASS(); +} + +/* ── Tests: global config ────────────────────────────────────────── */ + +TEST(userconfig_global_via_env) { + /* Point XDG_CONFIG_HOME to a temp dir */ + char xdg_dir[256]; + snprintf(xdg_dir, sizeof(xdg_dir), "%s/uctest_global_xdg", cbm_tmpdir()); + + char app_dir[512]; + snprintf(app_dir, sizeof(app_dir), "%s/codebase-memory-mcp", xdg_dir); + cbm_mkdir_p(app_dir, 0755); + + char global_path[768]; + snprintf(global_path, sizeof(global_path), "%s/config.json", app_dir); + ASSERT_EQ( + write_json(global_path, "{\"extra_extensions\":{\".twig\":\"html\"}}"), + 0); + + /* Set env var, load, restore */ + setenv("XDG_CONFIG_HOME", xdg_dir, 1); + cbm_userconfig_t *cfg = cbm_userconfig_load(NULL); /* no project dir */ + unsetenv("XDG_CONFIG_HOME"); + + ASSERT_NOT_NULL(cfg); + ASSERT_EQ(cbm_userconfig_lookup(cfg, ".twig"), CBM_LANG_HTML); + + cbm_userconfig_free(cfg); + remove(global_path); + PASS(); +} + +/* ── Tests: project wins over global ────────────────────────────── */ + +TEST(userconfig_project_wins_over_global) { + /* Global says .xyz → python; project says .xyz → rust */ + char xdg_dir[256]; + snprintf(xdg_dir, sizeof(xdg_dir), "%s/uctest_priority_xdg", cbm_tmpdir()); + + char app_dir[512]; + snprintf(app_dir, sizeof(app_dir), "%s/codebase-memory-mcp", xdg_dir); + cbm_mkdir_p(app_dir, 0755); + + char global_path[768]; + snprintf(global_path, sizeof(global_path), "%s/config.json", app_dir); + ASSERT_EQ( + write_json(global_path, "{\"extra_extensions\":{\".xyz\":\"python\"}}"), + 0); + + char proj_dir[256]; + snprintf(proj_dir, sizeof(proj_dir), "%s/uctest_priority_proj", cbm_tmpdir()); + cbm_mkdir_p(proj_dir, 0755); + + char proj_path[512]; + snprintf(proj_path, sizeof(proj_path), "%s/.codebase-memory.json", proj_dir); + ASSERT_EQ( + write_json(proj_path, "{\"extra_extensions\":{\".xyz\":\"rust\"}}"), + 0); + + setenv("XDG_CONFIG_HOME", xdg_dir, 1); + cbm_userconfig_t *cfg = cbm_userconfig_load(proj_dir); + unsetenv("XDG_CONFIG_HOME"); + + ASSERT_NOT_NULL(cfg); + /* Project definition (rust) must win */ + ASSERT_EQ(cbm_userconfig_lookup(cfg, ".xyz"), CBM_LANG_RUST); + + cbm_userconfig_free(cfg); + remove(global_path); + remove(proj_path); + PASS(); +} + +/* ── Tests: unknown language values are skipped ──────────────────── */ + +TEST(userconfig_unknown_lang_skipped) { + char dir[256]; + snprintf(dir, sizeof(dir), "%s/uctest_unknown_lang", cbm_tmpdir()); + cbm_mkdir_p(dir, 0755); + + char proj[512]; + snprintf(proj, sizeof(proj), "%s/.codebase-memory.json", dir); + /* "klingon" is not a valid language; ".wasm" should be silently skipped */ + ASSERT_EQ( + write_json(proj, + "{\"extra_extensions\":{\".wasm\":\"klingon\",\".mjs\":\"javascript\"}}"), + 0); + + cbm_userconfig_t *cfg = cbm_userconfig_load(dir); + ASSERT_NOT_NULL(cfg); + + /* .wasm with unknown lang → not in config */ + ASSERT_EQ(cbm_userconfig_lookup(cfg, ".wasm"), CBM_LANG_COUNT); + /* .mjs with valid lang → present */ + ASSERT_EQ(cbm_userconfig_lookup(cfg, ".mjs"), CBM_LANG_JAVASCRIPT); + + cbm_userconfig_free(cfg); + remove(proj); + PASS(); +} + +/* ── Tests: missing files are silently ignored ───────────────────── */ + +TEST(userconfig_missing_files_ok) { + /* Point to a non-existent repo dir */ + cbm_userconfig_t *cfg = cbm_userconfig_load("/tmp/__nonexistent_repo_12345__"); + ASSERT_NOT_NULL(cfg); /* must not return NULL — just empty */ + ASSERT_EQ(cfg->count, 0); + cbm_userconfig_free(cfg); + PASS(); +} + +/* ── Tests: integration with cbm_language_for_extension ─────────── */ + +TEST(userconfig_integration_override) { + /* Verify that setting the global config makes cbm_language_for_extension + * respect the override. We map ".blade.php" → PHP, which is not in the + * built-in table. */ + char dir[256]; + snprintf(dir, sizeof(dir), "%s/uctest_integ", cbm_tmpdir()); + cbm_mkdir_p(dir, 0755); + + char proj[512]; + snprintf(proj, sizeof(proj), "%s/.codebase-memory.json", dir); + ASSERT_EQ( + write_json(proj, "{\"extra_extensions\":{\".blade.php\":\"php\"}}"), + 0); + + cbm_userconfig_t *cfg = cbm_userconfig_load(dir); + ASSERT_NOT_NULL(cfg); + + /* Before setting, .blade.php is unknown */ + ASSERT_EQ(cbm_language_for_extension(".blade.php"), CBM_LANG_COUNT); + + cbm_set_user_lang_config(cfg); + /* After setting, .blade.php → PHP */ + ASSERT_EQ(cbm_language_for_extension(".blade.php"), CBM_LANG_PHP); + /* Built-in extensions still work */ + ASSERT_EQ(cbm_language_for_extension(".go"), CBM_LANG_GO); + + /* Clean up global state */ + cbm_set_user_lang_config(NULL); + cbm_userconfig_free(cfg); + remove(proj); + PASS(); +} + +/* ── Tests: free is NULL-safe ────────────────────────────────────── */ + +TEST(userconfig_free_null) { + cbm_userconfig_free(NULL); /* must not crash */ + PASS(); +} + +/* ── Suite ──────────────────────────────────────────────────────── */ + +SUITE(userconfig) { + RUN_TEST(userconfig_project_basic); + RUN_TEST(userconfig_global_via_env); + RUN_TEST(userconfig_project_wins_over_global); + RUN_TEST(userconfig_unknown_lang_skipped); + RUN_TEST(userconfig_missing_files_ok); + RUN_TEST(userconfig_integration_override); + RUN_TEST(userconfig_free_null); +}