From 440c84501e30d5becabef7d900e740df8019a2e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=91=A8=E5=87=AF?= <851708184@qq.com> Date: Wed, 8 Jul 2026 18:16:50 +0800 Subject: [PATCH 01/21] fix(ui): enumerate codebase-memory-mcp processes on Windows MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Windows handle_processes previously returned an empty processes array because it only queried the current process via GetProcessMemoryInfo/ GetProcessTimes. POSIX uses popen("ps") to enumerate all instances. Add process enumeration via CreateToolhelp32Snapshot + Process32First/ Process32Next to find all codebase-memory-mcp.exe processes, query per- process CPU/memory/elapsed time via OpenProcess + GetProcessTimes + GetProcessMemoryInfo, and return them in the JSON response. Closes #950 Signed-off-by: 周凯 <851708184@qq.com> --- src/ui/http_server.c | 78 ++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 76 insertions(+), 2 deletions(-) diff --git a/src/ui/http_server.c b/src/ui/http_server.c index e08ed8dbe..1583fec91 100644 --- a/src/ui/http_server.c +++ b/src/ui/http_server.c @@ -50,6 +50,7 @@ #include #include #include /* GetProcessMemoryInfo */ +#include /* CreateToolhelp32Snapshot, Process32First/Next */ #else #include #include @@ -501,7 +502,7 @@ static void handle_processes(cbm_http_conn_t *c) { int pos = 0; #ifdef _WIN32 - /* Windows: GetProcessMemoryInfo + GetProcessTimes */ + /* Windows: GetProcessMemoryInfo + GetProcessTimes for the current process */ PROCESS_MEMORY_COUNTERS pmc; FILETIME ft_create, ft_exit, ft_kernel, ft_user; double user_s = 0, sys_s = 0; @@ -519,8 +520,81 @@ static void handle_processes(cbm_http_conn_t *c) { } http_appendf(buf, sizeof(buf), &pos, "{\"self_pid\":%d,\"self_rss_mb\":%.1f," - "\"self_user_cpu_s\":%.1f,\"self_sys_cpu_s\":%.1f,\"processes\":[]}", + "\"self_user_cpu_s\":%.1f,\"self_sys_cpu_s\":%.1f,\"processes\":[", (int)_getpid(), (double)rss_bytes / (1024.0 * 1024.0), user_s, sys_s); + + /* Enumerate all codebase-memory-mcp.exe processes via toolhelp snapshot */ + int proc_count = 0; + HANDLE hSnap = CreateToolhelp32Snapshot(TH32CS_SNAPPROCESS, 0); + if (hSnap != INVALID_HANDLE_VALUE) { + PROCESSENTRY32 pe; + pe.dwSize = sizeof(pe); + for (BOOL ok = Process32First(hSnap, &pe); ok; ok = Process32Next(hSnap, &pe)) { + if (_stricmp(pe.szExeFile, "codebase-memory-mcp.exe") == 0) { + HANDLE hProc = OpenProcess( + PROCESS_QUERY_INFORMATION | PROCESS_VM_READ, + FALSE, pe.th32ProcessID); + if (hProc) { + PROCESS_MEMORY_COUNTERS ppmc; + FILETIME ftc, fte, ftk, ftu; + double cpu_user = 0, cpu_sys = 0; + size_t proc_rss = 0; + DWORD elapsed_sec = 0; + + if (GetProcessMemoryInfo(hProc, &ppmc, sizeof(ppmc))) + proc_rss = ppmc.WorkingSetSize; + + if (GetProcessTimes(hProc, &ftc, &fte, &ftk, &ftu)) { + ULARGE_INTEGER pu, pk; + pu.LowPart = ftu.dwLowDateTime; + pu.HighPart = ftu.dwHighDateTime; + pk.LowPart = ftk.dwLowDateTime; + pk.HighPart = ftk.dwHighDateTime; + cpu_user = (double)pu.QuadPart / 1e7; + cpu_sys = (double)pk.QuadPart / 1e7; + + /* 进程创建以来的运行时间 */ + FILETIME now_ft; + GetSystemTimeAsFileTime(&now_ft); + ULARGE_INTEGER now_uli, start_uli; + now_uli.LowPart = now_ft.dwLowDateTime; + now_uli.HighPart = now_ft.dwHighDateTime; + start_uli.LowPart = ftc.dwLowDateTime; + start_uli.HighPart = ftc.dwHighDateTime; + /* 时钟偏移保护：防止创建时间微偏未来导致 unsigned 下溢 */ + if (now_uli.QuadPart > start_uli.QuadPart) { + ULONGLONG elapsed_100ns = now_uli.QuadPart - start_uli.QuadPart; + elapsed_sec = (DWORD)(elapsed_100ns / 10000000ULL); + } + } + + if (proc_count > 0) + buf[pos++] = ','; + http_appendf(buf, sizeof(buf), &pos, + "{\"pid\":%lu,\"cpu\":%.1f,\"rss_mb\":%.1f," + "\"elapsed\":\"%lu-%02lu:%02lu:%02lu\"," + "\"command\":\"codebase-memory-mcp\"," + "\"is_self\":%s}", + pe.th32ProcessID, cpu_user + cpu_sys, + (double)proc_rss / (1024.0 * 1024.0), + elapsed_sec / 86400, + (elapsed_sec % 86400) / 3600, + (elapsed_sec % 3600) / 60, + elapsed_sec % 60, + pe.th32ProcessID == (DWORD)_getpid() + ? "true" : "false"); + if (pos >= (int)sizeof(buf)) { + pos = (int)sizeof(buf) - 1; + } + proc_count++; + CloseHandle(hProc); + } + } + } + CloseHandle(hSnap); + } + + http_appendf(buf, sizeof(buf), &pos, "]}"); #else struct rusage ru; getrusage(RUSAGE_SELF, &ru); From afbafa1b1c7e0d48f54a01c002375564c48f7d71 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 8 Aug 2026 05:42:37 +0000 Subject: [PATCH 02/21] build(deps): bump actions/stale from 10.4.0 to 11.0.0 Bumps [actions/stale](https://github.com/actions/stale) from 10.4.0 to 11.0.0. - [Release notes](https://github.com/actions/stale/releases) - [Changelog](https://github.com/actions/stale/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/stale/compare/1e223db275d687790206a7acac4d1a11bd6fe629...4391f3da665fdf50b6810c1a66712fb9ba21aa93) --- updated-dependencies: - dependency-name: actions/stale dependency-version: 11.0.0 dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/stale.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index d34e5944d..05a95e9e2 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -34,7 +34,7 @@ jobs: permissions: issues: write steps: - - uses: actions/stale@1e223db275d687790206a7acac4d1a11bd6fe629 # v10.4.0 + - uses: actions/stale@4391f3da665fdf50b6810c1a66712fb9ba21aa93 # v11.0.0 with: only-labels: 'awaiting-reporter' stale-issue-label: 'stale' From bd6b572f09102d807bf88ba6b3a87535591f6ec4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 8 Aug 2026 05:45:39 +0000 Subject: [PATCH 03/21] build(deps): bump github/codeql-action/upload-sarif Bumps [github/codeql-action/upload-sarif](https://github.com/github/codeql-action) from 4.37.1 to 4.37.6. - [Release notes](https://github.com/github/codeql-action/releases) - [Changelog](https://github.com/github/codeql-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/github/codeql-action/compare/7188fc363630916deb702c7fdcf4e481b751f97a...5595ccaf912efad79be6eef63a5619ff05969be3) --- updated-dependencies: - dependency-name: github/codeql-action/upload-sarif dependency-version: 4.37.6 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- .github/workflows/scorecard.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/scorecard.yml b/.github/workflows/scorecard.yml index e84de9601..5008c3835 100644 --- a/.github/workflows/scorecard.yml +++ b/.github/workflows/scorecard.yml @@ -27,6 +27,6 @@ jobs: publish_results: true - name: Upload SARIF results - uses: github/codeql-action/upload-sarif@7188fc363630916deb702c7fdcf4e481b751f97a # v4 + uses: github/codeql-action/upload-sarif@5595ccaf912efad79be6eef63a5619ff05969be3 # v4 with: sarif_file: results.sarif From a5c456cb7527f3f7117706f82143ed25a7b771cd Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 8 Aug 2026 05:46:56 +0000 Subject: [PATCH 04/21] build(deps): bump ossf/scorecard-action from 2.4.3 to 2.4.4 Bumps [ossf/scorecard-action](https://github.com/ossf/scorecard-action) from 2.4.3 to 2.4.4. - [Release notes](https://github.com/ossf/scorecard-action/releases) - [Changelog](https://github.com/ossf/scorecard-action/blob/main/RELEASE.md) - [Commits](https://github.com/ossf/scorecard-action/compare/4eaacf0543bb3f2c246792bd56e8cdeffafb205a...2d1146689b8cda280b9bc96326124645441f03bc) --- updated-dependencies: - dependency-name: ossf/scorecard-action dependency-version: 2.4.4 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- .github/workflows/scorecard.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/scorecard.yml b/.github/workflows/scorecard.yml index e84de9601..0757cd28e 100644 --- a/.github/workflows/scorecard.yml +++ b/.github/workflows/scorecard.yml @@ -20,7 +20,7 @@ jobs: persist-credentials: false - name: Run OpenSSF Scorecard - uses: ossf/scorecard-action@4eaacf0543bb3f2c246792bd56e8cdeffafb205a # v2.4.3 + uses: ossf/scorecard-action@2d1146689b8cda280b9bc96326124645441f03bc # v2.4.4 with: results_file: results.sarif results_format: sarif From 283321f07fab4a0b26262d3f92e84a156fee6ad4 Mon Sep 17 00:00:00 2001 From: tmonestudio Date: Sun, 12 Jul 2026 19:46:02 -0300 Subject: [PATCH 05/21] fix(pipeline): escape dynamic edge properties as JSON Escape import aliases and GraphQL operation strings before serializing edge properties, preventing raw control characters and quotes from producing malformed JSON. Add a reachable single-line GraphQL regression fixture and assert that emitted edge properties remain valid JSON. Co-Authored-By: Claude Opus 4.8 Signed-off-by: tmonestudio --- src/pipeline/pass_definitions.c | 6 ++++-- src/pipeline/pass_parallel.c | 4 +++- tests/test_lang_contract.c | 19 ++++++++++++++++++- 3 files changed, 25 insertions(+), 4 deletions(-) diff --git a/src/pipeline/pass_definitions.c b/src/pipeline/pass_definitions.c index 747f7e16b..dc414ee75 100644 --- a/src/pipeline/pass_definitions.c +++ b/src/pipeline/pass_definitions.c @@ -24,6 +24,7 @@ enum { PD_JSON_FIELD_OVERHEAD = 6 }; #include "foundation/compat.h" #include "foundation/compat_fs.h" #include "foundation/limits.h" +#include "foundation/str_util.h" #include "cbm.h" #include "arena.h" #include "iris_export_xml.h" @@ -470,9 +471,10 @@ static int create_import_edges_for_file(cbm_pipeline_ctx_t *ctx, const CBMFileRe const cbm_gbuf_node_t *target = cbm_pipeline_resolve_import_node(ctx, rel, file_qn, imp, namespace_map); if (target && target->id != source_node->id) { + char esc_ln[CBM_SZ_128]; + cbm_json_escape(esc_ln, sizeof(esc_ln), imp->local_name ? imp->local_name : ""); char imp_props[CBM_SZ_256]; - snprintf(imp_props, sizeof(imp_props), "{\"local_name\":\"%s\"}", - imp->local_name ? imp->local_name : ""); + snprintf(imp_props, sizeof(imp_props), "{\"local_name\":\"%s\"}", esc_ln); cbm_gbuf_insert_edge(ctx->gbuf, source_node->id, target->id, "IMPORTS", imp_props); count++; } diff --git a/src/pipeline/pass_parallel.c b/src/pipeline/pass_parallel.c index a34b45eb9..3abed9e00 100644 --- a/src/pipeline/pass_parallel.c +++ b/src/pipeline/pass_parallel.c @@ -1921,10 +1921,12 @@ static void emit_graphql_edge(cbm_gbuf_t *gbuf, const cbm_gbuf_node_t *source, c cbm_gbuf_upsert_node(gbuf, "Route", p, route_qn, "", 0, 0, "{\"source\":\"graphql\"}"); char esc_c[CBM_SZ_256]; + char esc_op[CBM_SZ_512]; cbm_json_escape(esc_c, sizeof(esc_c), call->callee_name); + cbm_json_escape(esc_op, sizeof(esc_op), p); char props[CBM_SZ_1K]; snprintf(props, sizeof(props), "{\"callee\":\"%s\",\"operation\":\"%s\",\"confidence\":%.2f}", - esc_c, p, res->confidence); + esc_c, esc_op, res->confidence); cbm_gbuf_insert_edge(gbuf, source->id, route_id, "GRAPHQL_CALLS", props); } diff --git a/tests/test_lang_contract.c b/tests/test_lang_contract.c index c8abfd118..bbc5651cd 100644 --- a/tests/test_lang_contract.c +++ b/tests/test_lang_contract.c @@ -24,6 +24,7 @@ #include #include #include +#include #include #include @@ -1518,7 +1519,9 @@ TEST(contract_edge_parallel_service_edges) { {"gql.py", "def gql(query_string):\n return query_string\n"}, {"client.py", "from gql import gql\n\n\ndef fetch_user():\n" - " return gql(\"query GetUser { user { id name } }\")\n\n\n" + /* The embedded double quote survives extraction from this single-quoted + * Python string and breaks GRAPHQL_CALLS JSON unless operation is escaped. */ + " return gql('query GetUser { user(name: \"quoted\") { id name } }')\n\n\n" "def create_user():\n" " return gql(\"mutation CreateUser { addUser(name: \\\"x\\\") { id } }\")\n"}, /* TRPC_CALLS: local createTRPCProxyClient (same-module resolution). */ @@ -1559,6 +1562,19 @@ TEST(contract_edge_parallel_service_edges) { int grpc = store ? cbm_store_count_edges_by_type(store, lp.project, "GRPC_CALLS") : -1; int trpc = store ? cbm_store_count_edges_by_type(store, lp.project, "TRPC_CALLS") : -1; int infra = store ? cbm_store_count_edges_by_type(store, lp.project, "INFRA_MAPS") : -1; + int invalid_props = -1; + if (store) { + sqlite3_stmt *stmt = NULL; + sqlite3 *db = cbm_store_get_db(store); + if (db && sqlite3_prepare_v2(db, + "SELECT count(*) FROM edges WHERE properties IS NOT NULL " + "AND properties != '' AND json_valid(properties)=0;", + -1, &stmt, NULL) == SQLITE_OK && + sqlite3_step(stmt) == SQLITE_ROW) { + invalid_props = sqlite3_column_int(stmt, 0); + } + sqlite3_finalize(stmt); + } if (graphql < 1 || grpc < 1 || trpc < 1 || infra < 1) { fprintf(stderr, " [EDGE] parallel-service: GRAPHQL_CALLS=%d GRPC_CALLS=%d TRPC_CALLS=%d " @@ -1571,6 +1587,7 @@ TEST(contract_edge_parallel_service_edges) { ASSERT_TRUE(grpc >= 1); ASSERT_TRUE(trpc >= 1); ASSERT_TRUE(infra >= 1); + ASSERT_EQ(invalid_props, 0); PASS(); } From 263cf22b9c400b512617fd7e404ae6c74a09a908 Mon Sep 17 00:00:00 2001 From: Alex Musichen Date: Sun, 16 Aug 2026 16:41:09 +0200 Subject: [PATCH 06/21] fix(pi): emit parameters and execute in the generated pi adapter The generated Pi extension registered each MCP tool as { name, run }, but Pi's ToolDefinition requires label, description, parameters, and execute. Tools registered that way carried no parameter schema, so strict providers such as xAI/Grok reject the request with a 422 'missing field parameters', and the tools were uncallable because Pi invokes execute, never run. Emit the full tool shape from the registry: label/description via new accessors, the input_schema embedded directly as a JSON object literal, and execute instead of run. Signed-off-by: Alex Musichen --- src/cli/client_adapter.c | 56 ++++++++++++++++++++++++++++++++++---- src/mcp/mcp.c | 24 ++++++++++++++++ src/mcp/mcp.h | 6 ++++ tests/test_agent_clients.c | 18 ++++++++++++ 4 files changed, 99 insertions(+), 5 deletions(-) diff --git a/src/cli/client_adapter.c b/src/cli/client_adapter.c index 839b3d87b..dd7e62570 100644 --- a/src/cli/client_adapter.c +++ b/src/cli/client_adapter.c @@ -49,6 +49,39 @@ static void sb_append(adapter_sb_t *sb, const char *s) { sb->buf[sb->len] = '\0'; } +/* Append a single-quoted JavaScript string literal, escaping the characters + * that could terminate or corrupt it. Avoids the fixed buffer sizing that + * cbm_client_adapter_escape_js imposes, so long registry descriptions fit. */ +static void sb_append_js_string(adapter_sb_t *sb, const char *s) { + if (!s) { + sb_append(sb, "''"); + return; + } + sb_append(sb, "'"); + for (const char *p = s; *p; p++) { + switch (*p) { + case '\\': + sb_append(sb, "\\\\"); + break; + case '\'': + sb_append(sb, "\\'"); + break; + case '\n': + sb_append(sb, "\\n"); + break; + case '\r': + sb_append(sb, "\\r"); + break; + default: { + char ch[2] = { *p, '\0' }; + sb_append(sb, ch); + break; + } + } + } + sb_append(sb, "'"); +} + bool cbm_client_adapter_escape_js(const char *in, char *out, size_t out_sz) { if (!in || !out || out_sz == 0) { return false; @@ -170,11 +203,24 @@ char *cbm_client_adapter_pi(const char *binary_path) { if (!name || !name[0]) { continue; } - sb_append(&sb, " pi.registerTool({ name: '"); - sb_append(&sb, name); - sb_append(&sb, "', run: (args, ctx) => call('"); - sb_append(&sb, name); - sb_append(&sb, "', args, ctx?.signal) });\n"); + const char *title = cbm_mcp_tool_title(name); + const char *description = cbm_mcp_tool_description(name); + const char *schema = cbm_mcp_tool_input_schema(name); + sb_append(&sb, " pi.registerTool({\n"); + sb_append(&sb, " name: "); + sb_append_js_string(&sb, name); + sb_append(&sb, ",\n label: "); + sb_append_js_string(&sb, title ? title : name); + sb_append(&sb, ",\n description: "); + sb_append_js_string(&sb, description ? description : ""); + sb_append(&sb, ",\n parameters: "); + /* input_schema is compact JSON, which is a valid JavaScript object + * literal; embedding it directly keeps the generated module free of a + * JSON.parse indirection and of any escaping drift. */ + sb_append(&sb, schema ? schema : "{}"); + sb_append(&sb, ",\n execute: (args, ctx) => call("); + sb_append_js_string(&sb, name); + sb_append(&sb, ", args, ctx?.signal),\n });\n"); } sb_append(&sb, "}\n"); diff --git a/src/mcp/mcp.c b/src/mcp/mcp.c index 411c15ebd..7af4fe159 100644 --- a/src/mcp/mcp.c +++ b/src/mcp/mcp.c @@ -939,6 +939,30 @@ const char *cbm_mcp_tool_name(int index) { return TOOLS[index].name; } +const char *cbm_mcp_tool_title(const char *tool_name) { + if (!tool_name) { + return NULL; + } + for (int i = 0; i < TOOL_COUNT; i++) { + if (strcmp(TOOLS[i].name, tool_name) == 0) { + return TOOLS[i].title; + } + } + return NULL; +} + +const char *cbm_mcp_tool_description(const char *tool_name) { + if (!tool_name) { + return NULL; + } + for (int i = 0; i < TOOL_COUNT; i++) { + if (strcmp(TOOLS[i].name, tool_name) == 0) { + return TOOLS[i].description; + } + } + return NULL; +} + /* Render the top-level --help "Tools:" block from the registry tools/list * serves. The list used to be hand-maintained in the help text and drifted * when check_index_coverage was added (#1361); deriving it here makes that diff --git a/src/mcp/mcp.h b/src/mcp/mcp.h index a70b6372b..d55d58f3d 100644 --- a/src/mcp/mcp.h +++ b/src/mcp/mcp.h @@ -72,6 +72,12 @@ const char *cbm_mcp_tool_input_schema(const char *tool_name); int cbm_mcp_tool_count(void); const char *cbm_mcp_tool_name(int index); +/* Tool metadata by name (static; do not free; NULL when unknown). Used by the + * generated client adapters so they can emit descriptions and parameter + * schemas without drifting from tools/list. */ +const char *cbm_mcp_tool_title(const char *tool_name); +const char *cbm_mcp_tool_description(const char *tool_name); + /* Render the top-level --help "Tools:" block from the registry so the help * text cannot drift from tools/list (#1361). Heap-allocated; caller frees. */ char *cbm_mcp_tools_help_list(void); diff --git a/tests/test_agent_clients.c b/tests/test_agent_clients.c index 15794c2b1..d8dd7026e 100644 --- a/tests/test_agent_clients.c +++ b/tests/test_agent_clients.c @@ -1114,6 +1114,23 @@ TEST(client_adapter_pi_default_exports_its_factory_issue1550) { PASS(); } +/* Pi's ToolDefinition requires `parameters` and `execute`. The adapter used to + * emit `{ name, run }`, which Pi accepted but treated as a tool with no + * parameter schema — strict providers (xAI/Grok) then reject the request with + * a 422 "missing field parameters", and the tool is uncallable because Pi + * invokes `execute`, never `run`. Pin the corrected shape. */ +TEST(client_adapter_pi_emits_parameters_and_execute) { + char *js = cbm_client_adapter_pi("/usr/local/bin/codebase-memory-mcp"); + ASSERT_NOT_NULL(js); + ASSERT_NOT_NULL(strstr(js, "execute: (args, ctx) => call(")); + ASSERT_NULL(strstr(js, "run: (args, ctx)")); + ASSERT_NOT_NULL(strstr(js, "parameters:")); + /* The registry input_schema is embedded as a JSON object literal. */ + ASSERT_NOT_NULL(strstr(js, "\"type\":\"object\"")); + free(js); + PASS(); +} + /* #616 rejected `"` in its path template but not `\`, so a Windows home like * C:\Users\urs\bin produced `\u` — an invalid unicode escape — and the whole * auto-loaded plugin failed to parse. A broken plugin in an auto-load directory @@ -1204,6 +1221,7 @@ SUITE(agent_clients) { RUN_TEST(agent_clients_continue_refuses_foreign_same_name_and_nonsequence_section); RUN_TEST(client_adapter_pi_registers_every_registry_tool); RUN_TEST(client_adapter_pi_default_exports_its_factory_issue1550); + RUN_TEST(client_adapter_pi_emits_parameters_and_execute); RUN_TEST(client_adapter_escapes_windows_paths_and_quotes); RUN_TEST(client_adapter_opencode_sends_the_required_hook_event); RUN_TEST(client_adapter_rejects_missing_binary_path); From cf5eb61c20d073ca0116be7365288664befa37f3 Mon Sep 17 00:00:00 2001 From: Alex Musichen Date: Sun, 16 Aug 2026 22:26:34 +0200 Subject: [PATCH 07/21] fix(pi): return a valid AgentToolResult and request raw JSON MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The generated execute forwarded the raw MCP JSON directly, but pi's ToolDefinition.execute must return { content: [{ type: 'text', text }], details } — a result without a content array crashes the TUI's getTextOutput on result.content.filter(...). Request raw JSON from the CLI ('--json') so the bridge parses the MCP result instead of the human-readable text, then wrap it: pass the content array through, throw on transport errors, and stringify anything else. Adds coverage asserting the corrected execute shape and the --json flag. Signed-off-by: Alex Musichen --- src/cli/client_adapter.c | 17 ++++++++++++++--- tests/test_agent_clients.c | 6 +++++- 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/src/cli/client_adapter.c b/src/cli/client_adapter.c index dd7e62570..f0c92a3fa 100644 --- a/src/cli/client_adapter.c +++ b/src/cli/client_adapter.c @@ -164,7 +164,7 @@ char *cbm_client_adapter_pi(const char *binary_path) { sb_append( &sb, "async function call(tool, args, signal) {\n" " return new Promise((resolve) => {\n" - " const child = spawn(BIN, ['cli', tool, JSON.stringify(args ?? {})], {\n" + " const child = spawn(BIN, ['cli', '--json', tool, JSON.stringify(args ?? {})], {\n" " stdio: ['ignore', 'pipe', 'pipe'],\n" " env: { ...process.env, CBM_LOG_LEVEL: 'error' },\n" " });\n" @@ -218,9 +218,20 @@ char *cbm_client_adapter_pi(const char *binary_path) { * literal; embedding it directly keeps the generated module free of a * JSON.parse indirection and of any escaping drift. */ sb_append(&sb, schema ? schema : "{}"); - sb_append(&sb, ",\n execute: (args, ctx) => call("); + sb_append(&sb, ",\n execute: async (args, ctx) => {\n"); + sb_append(&sb, " const result = await call("); sb_append_js_string(&sb, name); - sb_append(&sb, ", args, ctx?.signal),\n });\n"); + sb_append(&sb, + ", args, ctx?.signal);\n" + " if (result && typeof result === 'object' && result.error) {\n" + " throw new Error(String(result.error));\n" + " }\n" + " const content = result && Array.isArray(result.content)\n" + " ? result.content\n" + " : [{ type: 'text', text: JSON.stringify(result ?? null, null, 2) }];\n" + " return { content, details: result ?? {} };\n" + " },\n" + " });\n"); } sb_append(&sb, "}\n"); diff --git a/tests/test_agent_clients.c b/tests/test_agent_clients.c index d8dd7026e..f15e69a66 100644 --- a/tests/test_agent_clients.c +++ b/tests/test_agent_clients.c @@ -1122,11 +1122,15 @@ TEST(client_adapter_pi_default_exports_its_factory_issue1550) { TEST(client_adapter_pi_emits_parameters_and_execute) { char *js = cbm_client_adapter_pi("/usr/local/bin/codebase-memory-mcp"); ASSERT_NOT_NULL(js); - ASSERT_NOT_NULL(strstr(js, "execute: (args, ctx) => call(")); + ASSERT_NOT_NULL(strstr(js, "execute: async (args, ctx) => {")); + ASSERT_NOT_NULL(strstr(js, "result.content")); ASSERT_NULL(strstr(js, "run: (args, ctx)")); ASSERT_NOT_NULL(strstr(js, "parameters:")); /* The registry input_schema is embedded as a JSON object literal. */ ASSERT_NOT_NULL(strstr(js, "\"type\":\"object\"")); + /* Raw JSON output is required so the bridge can parse the MCP result; the + * human-readable path would leave `call` with nothing to JSON.parse. */ + ASSERT_NOT_NULL(strstr(js, "'cli', '--json'")); free(js); PASS(); } From d918fce2872e0679fabcb3d9f9423f4ef2ccde2a Mon Sep 17 00:00:00 2001 From: Alex Musichen Date: Sun, 16 Aug 2026 22:58:58 +0200 Subject: [PATCH 08/21] style: fix clang-format violation in pi adapter clang-format wants no spaces inside a braced initializer. Signed-off-by: Alex Musichen --- src/cli/client_adapter.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cli/client_adapter.c b/src/cli/client_adapter.c index f0c92a3fa..73cd9f112 100644 --- a/src/cli/client_adapter.c +++ b/src/cli/client_adapter.c @@ -73,7 +73,7 @@ static void sb_append_js_string(adapter_sb_t *sb, const char *s) { sb_append(sb, "\\r"); break; default: { - char ch[2] = { *p, '\0' }; + char ch[2] = {*p, '\0'}; sb_append(sb, ch); break; } From 109299f26296181000a8491d8ee1d1e6977911c4 Mon Sep 17 00:00:00 2001 From: Alex Musichen Date: Tue, 18 Aug 2026 04:11:49 +0200 Subject: [PATCH 09/21] fix(pi): match ToolDefinition.execute arity on 0.84.2 Pi 0.84.2 calls execute(toolCallId, params, signal, onUpdate, ctx). The generated (args, ctx) shape bound the call id as the MCP arguments. Forward params and signal, pin the @earendil-works/pi-coding-agent contract in the generated header, and lock the 5-arg form in tests. Signed-off-by: Alex Musichen --- src/cli/client_adapter.c | 10 ++++++++-- tests/test_agent_clients.c | 6 +++++- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/src/cli/client_adapter.c b/src/cli/client_adapter.c index 73cd9f112..cc7454793 100644 --- a/src/cli/client_adapter.c +++ b/src/cli/client_adapter.c @@ -153,6 +153,10 @@ char *cbm_client_adapter_pi(const char *binary_path) { adapter_sb_t sb = {0}; emit_header(&sb, "pi"); + /* Pin the coding-agent contract so a probe against @mariozechner/pi + * (pods CLI) or an old AgentTool arity cannot be mistaken for this file. */ + sb_append(&sb, "// Target: @earendil-works/pi-coding-agent >= 0.74.0 (verified 0.84.2)\n" + "// ToolDefinition.execute(toolCallId, params, signal, onUpdate, ctx)\n"); sb_append(&sb, "import { spawn } from 'node:child_process';\n\n"); sb_append(&sb, "const BIN = '"); sb_append(&sb, bin); @@ -218,11 +222,13 @@ char *cbm_client_adapter_pi(const char *binary_path) { * literal; embedding it directly keeps the generated module free of a * JSON.parse indirection and of any escaping drift. */ sb_append(&sb, schema ? schema : "{}"); - sb_append(&sb, ",\n execute: async (args, ctx) => {\n"); + /* 0.84.2 calls execute(toolCallId, params, signal, onUpdate, ctx). + * The previous (args, ctx) shape bound the call id as the MCP args. */ + sb_append(&sb, ",\n execute: async (toolCallId, params, signal, _onUpdate, ctx) => {\n"); sb_append(&sb, " const result = await call("); sb_append_js_string(&sb, name); sb_append(&sb, - ", args, ctx?.signal);\n" + ", params, signal ?? ctx?.signal);\n" " if (result && typeof result === 'object' && result.error) {\n" " throw new Error(String(result.error));\n" " }\n" diff --git a/tests/test_agent_clients.c b/tests/test_agent_clients.c index f15e69a66..89dda28ba 100644 --- a/tests/test_agent_clients.c +++ b/tests/test_agent_clients.c @@ -1122,7 +1122,11 @@ TEST(client_adapter_pi_default_exports_its_factory_issue1550) { TEST(client_adapter_pi_emits_parameters_and_execute) { char *js = cbm_client_adapter_pi("/usr/local/bin/codebase-memory-mcp"); ASSERT_NOT_NULL(js); - ASSERT_NOT_NULL(strstr(js, "execute: async (args, ctx) => {")); + ASSERT_NOT_NULL(strstr(js, "execute: async (toolCallId, params, signal, _onUpdate, ctx) => {")); + ASSERT_NOT_NULL(strstr(js, ", params, signal ?? ctx?.signal)")); + ASSERT_NULL(strstr(js, "execute: async (args, ctx) => {")); + ASSERT_NULL(strstr(js, ", args, ctx?.signal)")); + ASSERT_NOT_NULL(strstr(js, "@earendil-works/pi-coding-agent")); ASSERT_NOT_NULL(strstr(js, "result.content")); ASSERT_NULL(strstr(js, "run: (args, ctx)")); ASSERT_NOT_NULL(strstr(js, "parameters:")); From 7ba84e37548f70c31eadc48fafa731432176e06c Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Thu, 20 Aug 2026 00:12:28 +0200 Subject: [PATCH 10/21] feat(sql): first-class Table/View nodes with FROM/JOIN lineage CREATE TABLE / CREATE VIEW / CREATE MATERIALIZED VIEW now produce Table and View nodes (previously generic Variable), CREATE PROCEDURE produces a Function, and schema-qualified DDL names (schema.table) are named by the table identifier instead of the schema. A view's FROM/JOIN relations are emitted as usages and resolve into view -> table USAGE lineage edges. Relations join the cross-file name registry so lineage can resolve, with two structural safeguards: - Registry membership is defined once by cbm_label_is_registry_symbol (helpers.c); the full, parallel and incremental seed sites all call it, ending the KEEP-IN-SYNC copies the old label lists required. - The default cbm_registry_resolve vetoes relation-labeled results: common table names (users, orders, config) collide with code identifiers in every language, and no CALLS/USAGE/READS/WRITES/THROWS/ handler/decorator consumer may bind them. The SQL lineage path opts in through the new cbm_registry_resolve_lineage. Table/View also join the registry-only per-file LSP surface labels so a table rename invalidates dependent SQL files on incremental (no stale lineage edges), rank with the type tier in BM25 search, and appear in the architecture boundary/package/cluster queries via the pinned CBM_SQL_RELATION_LABELS fragment. Tests: extraction trio (labels, lineage usages, schema-qualified names), grammar golden + probe updates, relation-label contract pin, and two pipeline tests: cross-language isolation (binding: fails without the veto) and incremental table-rename stale-lineage. Closes #574. Co-authored-by: alexisperinger-ux Signed-off-by: Martin Vogel --- internal/cbm/cbm.h | 13 ++++ internal/cbm/extract_defs.c | 87 ++++++++++++++++++++++++++- internal/cbm/helpers.c | 25 ++++++++ internal/cbm/lang_specs.c | 12 ++-- src/foundation/constants.h | 3 + src/mcp/mcp.c | 3 + src/pipeline/lsp_surface.c | 12 ++-- src/pipeline/pass_definitions.c | 18 ++---- src/pipeline/pass_parallel.c | 24 ++++---- src/pipeline/pass_usages.c | 12 +++- src/pipeline/pipeline.h | 13 +++- src/pipeline/pipeline_incremental.c | 21 +++---- src/pipeline/registry.c | 74 +++++++++++++++++------ src/store/store.c | 9 ++- tests/test_extraction.c | 55 +++++++++++++++++ tests/test_grammar_labels.c | 2 +- tests/test_grammar_probe_g.c | 31 ++++++---- tests/test_pipeline.c | 92 +++++++++++++++++++++++++++++ tests/test_store_nodes.c | 31 ++++++++++ 19 files changed, 452 insertions(+), 85 deletions(-) diff --git a/internal/cbm/cbm.h b/internal/cbm/cbm.h index 440d80fce..63849a3c9 100644 --- a/internal/cbm/cbm.h +++ b/internal/cbm/cbm.h @@ -753,4 +753,17 @@ void cbm_extract_k8s(CBMExtractCtx *ctx); // `label` may be NULL (returns false). Defined in helpers.c. bool cbm_label_is_type_like(const char *label); +// True for data-relation labels (Table, View — SQL DDL). Relations resolve as +// lineage targets only: registry members, but never type-like and never valid +// CALLS/THROWS/READS/WRITES targets. `label` may be NULL. Defined in helpers.c. +bool cbm_label_is_relation(const char *label); + +// True for labels admitted to the cross-file name registry: Function, Method, +// every type-like container, Variable, Field, and the relation labels. Single +// source of truth for registry seeding — the full (pass_definitions.c), +// parallel (pass_parallel.c) and incremental (pipeline_incremental.c) pipelines +// all seed through this predicate so their registries never diverge. +// `label` may be NULL (returns false). Defined in helpers.c. +bool cbm_label_is_registry_symbol(const char *label); + #endif // CBM_H diff --git a/internal/cbm/extract_defs.c b/internal/cbm/extract_defs.c index f9854b03e..772adcee9 100644 --- a/internal/cbm/extract_defs.c +++ b/internal/cbm/extract_defs.c @@ -275,12 +275,31 @@ static TSNode resolve_ocaml_func_name(TSNode node) { return null_node; } -// SQL: resolve create_function name from object_reference→identifier or direct identifier. +// Last identifier (DFS pre-order) under `node`. For a schema-qualified +// object_reference (schema.table) this is the table name; the schema prefix is +// ignored. Leaves *found false and returns `best` unchanged if none is present. +static TSNode sql_last_identifier(TSNode node, TSNode best, bool *found) { + if (strcmp(ts_node_type(node), "identifier") == 0) { + best = node; + *found = true; + } + uint32_t cc = ts_node_child_count(node); + for (uint32_t i = 0; i < cc; i++) { + best = sql_last_identifier(ts_node_child(node, i), best, found); + } + return best; +} + +// SQL: resolve create_function / create_table / create_view name. The name sits +// on an object_reference; for a schema-qualified name (schema.table) take the +// last identifier (the table), not the first (the schema). static TSNode resolve_sql_func_name(TSNode node) { TSNode obj_ref = cbm_find_child_by_kind(node, "object_reference"); if (!ts_node_is_null(obj_ref)) { - TSNode id = cbm_find_child_by_kind(obj_ref, "identifier"); - if (!ts_node_is_null(id)) { + bool found = false; + TSNode empty = {0}; + TSNode id = sql_last_identifier(obj_ref, empty, &found); + if (found) { return id; } } @@ -4024,6 +4043,65 @@ static bool extract_config_class_def(CBMExtractCtx *ctx, TSNode node, const char return true; } +// Collect FROM/JOIN table references (tree-sitter-sql `relation` nodes) anywhere +// under `node` and emit them as usages scoped to enclosing_qn. pass_usages then +// resolves each ref_name to the referenced Table/View def and creates a USAGE +// lineage edge (e.g. a view -> the tables it selects from). Emitting them here +// (rather than via the generic identifier walker) sets the correct enclosing +// scope and bypasses the is_definition_name suppression that drops them. +static void collect_sql_relation_usages(CBMExtractCtx *ctx, TSNode node, const char *enclosing_qn) { + if (strcmp(ts_node_type(node), "relation") == 0) { + TSNode nm = resolve_sql_func_name(node); // object_reference -> identifier + if (!ts_node_is_null(nm)) { + char *tname = cbm_node_text(ctx->arena, nm, ctx->source); + if (tname && tname[0]) { + CBMUsage usage = {0}; + usage.ref_name = tname; + usage.enclosing_func_qn = enclosing_qn; + cbm_usages_push(&ctx->result->usages, ctx->arena, usage); + } + } + } + uint32_t n = ts_node_child_count(node); + for (uint32_t i = 0; i < n; i++) { + collect_sql_relation_usages(ctx, ts_node_child(node, i), enclosing_qn); + } +} + +// Handle SQL DDL relation defs: CREATE TABLE / VIEW / MATERIALIZED VIEW become +// first-class Table/View nodes rather than generic Variable nodes. The relation +// name sits on an object_reference child (the same shape create_function uses), +// so resolve_sql_func_name locates it. Also emits FROM/JOIN dependencies as +// usages so lineage edges form. Returns true if handled. +static bool extract_sql_ddl_class_def(CBMExtractCtx *ctx, TSNode node, const char *kind) { + if (ctx->language != CBM_LANG_SQL) { + return false; + } + const char *label; + if (strcmp(kind, "create_table") == 0) { + label = "Table"; + } else if (strcmp(kind, "create_view") == 0 || strcmp(kind, "create_materialized_view") == 0) { + label = "View"; + } else { + return false; + } + TSNode name_node = resolve_sql_func_name(node); + if (ts_node_is_null(name_node)) { + return false; + } + char *name = cbm_node_text(ctx->arena, name_node, ctx->source); + if (!name || !name[0]) { + return false; + } + push_simple_class_def(ctx, node, name, label); + // Must match push_simple_class_def's QN exactly (qn_safe_segment included) + // or pass_usages cannot find the enclosing def for the lineage source. + const char *qn = + cbm_fqn_compute(ctx->arena, ctx->project, ctx->rel_path, qn_safe_segment(ctx->arena, name)); + collect_sql_relation_usages(ctx, node, qn); + return true; +} + static void extract_class_def(CBMExtractCtx *ctx, TSNode node, const CBMLangSpec *spec) { CBMArena *a = ctx->arena; const char *kind = ts_node_type(node); @@ -4031,6 +4109,9 @@ static void extract_class_def(CBMExtractCtx *ctx, TSNode node, const CBMLangSpec if (extract_config_class_def(ctx, node, kind)) { return; } + if (extract_sql_ddl_class_def(ctx, node, kind)) { + return; + } TSNode name_node = ts_node_child_by_field_name(node, TS_FIELD("name")); // ObjC: class name is first identifier child diff --git a/internal/cbm/helpers.c b/internal/cbm/helpers.c index f139ff61e..cae8164ae 100644 --- a/internal/cbm/helpers.c +++ b/internal/cbm/helpers.c @@ -166,6 +166,31 @@ bool cbm_label_is_type_like(const char *label) { strcmp(label, "Type") == 0 || strcmp(label, "Trait") == 0; } +// True when `label` names a data relation (SQL CREATE TABLE / CREATE VIEW). +// Relations live in the registry so FROM/JOIN lineage can resolve, but they are +// deliberately NOT type-like: they must never satisfy inheritance, impl-receiver, +// semantic-type, or LSP-registrar lookups, and resolver fallbacks treat them as +// lineage-only targets (see cbm_label_is_registry_symbol call sites). +bool cbm_label_is_relation(const char *label) { + if (!label) { + return false; + } + return strcmp(label, "Table") == 0 || strcmp(label, "View") == 0; +} + +// True when `label` belongs in the cross-file name registry (see cbm.h). Single +// source of truth for every registry-seeding site — full, parallel and +// incremental pipelines MUST admit the same set or an incremental re-resolve +// diverges from a clean full reindex. +bool cbm_label_is_registry_symbol(const char *label) { + if (!label) { + return false; + } + return strcmp(label, "Function") == 0 || strcmp(label, "Method") == 0 || + cbm_label_is_type_like(label) || strcmp(label, "Variable") == 0 || + strcmp(label, "Field") == 0 || cbm_label_is_relation(label); +} + bool cbm_is_keyword(const char *name, CBMLanguage lang) { if (!name || !name[0]) { return true; diff --git a/internal/cbm/lang_specs.c b/internal/cbm/lang_specs.c index 2bc1fe1e7..53b1ca728 100644 --- a/internal/cbm/lang_specs.c +++ b/internal/cbm/lang_specs.c @@ -656,13 +656,17 @@ static const char *hcl_call_types[] = {"function_call", NULL}; static const char *hcl_var_types[] = {"attribute", NULL}; // ==================== SQL ==================== -static const char *sql_func_types[] = {"create_function", "function_declaration", NULL}; +static const char *sql_func_types[] = {"create_function", "function_declaration", + "create_procedure", NULL}; static const char *sql_field_types[] = {"column_definition", NULL}; -static const char *sql_class_types[] = {"custom_type", NULL}; +// create_table/create_view route through the class-def path where +// extract_sql_ddl_class_def turns them into first-class Table/View nodes +// (previously they were generic Variable nodes via sql_var_types). +static const char *sql_class_types[] = {"custom_type", "create_table", "create_view", + "create_materialized_view", NULL}; static const char *sql_module_types[] = {"program", NULL}; static const char *sql_call_types[] = {"invocation", NULL}; static const char *sql_branch_types[] = {"if_statement", "case_expression", NULL}; -static const char *sql_var_types[] = {"create_table", "create_view", NULL}; // ==================== DOCKERFILE ==================== static const char *dockerfile_module_types[] = {"source_file", NULL}; @@ -1840,7 +1844,7 @@ static const CBMLangSpec lang_specs[CBM_LANG_COUNT] = { // CBM_LANG_SQL [CBM_LANG_SQL] = {CBM_LANG_SQL, sql_func_types, sql_class_types, sql_field_types, sql_module_types, sql_call_types, empty_types, empty_types, sql_branch_types, - sql_var_types, empty_types, empty_types, NULL, empty_types, NULL, NULL, + empty_types, empty_types, empty_types, NULL, empty_types, NULL, NULL, tree_sitter_sql, NULL}, // CBM_LANG_DOCKERFILE diff --git a/src/foundation/constants.h b/src/foundation/constants.h index 373ea1a51..c9f3a9d70 100644 --- a/src/foundation/constants.h +++ b/src/foundation/constants.h @@ -108,5 +108,8 @@ enum { SKIP_ONE = 1, PAIR_LEN = 2 }; #define CBM_SQL_TYPE_LIKE_LABELS "'Class','Struct','Interface','Enum','Type','Trait'" #define CBM_SQL_CALLABLE_LABELS "'Function','Method'" #define CBM_SQL_CALLABLE_OR_TYPE_LABELS CBM_SQL_CALLABLE_LABELS "," CBM_SQL_TYPE_LIKE_LABELS +/* SQL mirror of cbm_label_is_relation() (Table/View — data-lineage nodes), + * pinned by tests/test_store_nodes.c the same way as the sets above. */ +#define CBM_SQL_RELATION_LABELS "'Table','View'" #endif /* CBM_CONSTANTS_H */ diff --git a/src/mcp/mcp.c b/src/mcp/mcp.c index 618d04469..42bd33d7d 100644 --- a/src/mcp/mcp.c +++ b/src/mcp/mcp.c @@ -3017,6 +3017,9 @@ static char *bm25_search(cbm_store_t *store, const char *project, const char *qu " - CASE WHEN n.label IN ('Function','Method') THEN 10.0 " " WHEN n.label = 'Route' THEN 8.0 " " WHEN n.label IN (" CBM_SQL_TYPE_LIKE_LABELS ") THEN 5.0 " + /* Relations rank with the type tier: a table IS the schema container + * a data question is looking for (findability-first). */ + " WHEN n.label IN (" CBM_SQL_RELATION_LABELS ") THEN 5.0 " " ELSE 0.0 END) AS rank " "FROM (" " SELECT rowid, bm25(nodes_fts) AS base_rank" diff --git a/src/pipeline/lsp_surface.c b/src/pipeline/lsp_surface.c index 92a3c67ca..369f7f7e5 100644 --- a/src/pipeline/lsp_surface.c +++ b/src/pipeline/lsp_surface.c @@ -21,6 +21,7 @@ #include #include +#include "cbm.h" /* cbm_label_is_relation — reg-only surface membership */ #include "foundation/log.h" #include "foundation/sha256.h" #include "yyjson/yyjson.h" @@ -30,12 +31,13 @@ enum { SURFACE_CODEC_VERSION = 1 }; /* Labels the incremental name registry serves that pxc_map_label does NOT * carry into the CBMLSPDef set. Their (name, qn, label) triple must still * participate in the surface hash, or renaming one would slip past the - * early cutoff while stale references to it survive in dependent files. - * KEEP IN SYNC with pxc_map_label (pass_lsp_cross.c) and - * incr_label_is_registry_symbol (pipeline_incremental.c); the codec unit - * test cross-checks the three. */ + * early cutoff while stale references to it survive in dependent files — + * for Table/View that means a renamed table keeping stale FROM/JOIN lineage + * edges from dependent SQL files. KEEP IN SYNC with pxc_map_label + * (pass_lsp_cross.c) and incr_label_is_registry_symbol + * (pipeline_incremental.c); the codec unit test cross-checks the three. */ static bool surface_reg_only_label(const char *label) { - return label && strcmp(label, "Field") == 0; + return label && (strcmp(label, "Field") == 0 || cbm_label_is_relation(label)); } static void add_str_or_null(yyjson_mut_doc *doc, yyjson_mut_val *obj, const char *key, diff --git a/src/pipeline/pass_definitions.c b/src/pipeline/pass_definitions.c index 747f7e16b..b9be382fa 100644 --- a/src/pipeline/pass_definitions.c +++ b/src/pipeline/pass_definitions.c @@ -327,18 +327,12 @@ static void process_def(cbm_pipeline_ctx_t *ctx, const CBMDefinition *def, const int64_t node_id = cbm_gbuf_upsert_node( ctx->gbuf, def->label ? def->label : "Function", def->name, def->qualified_name, def->file_path ? def->file_path : rel, (int)def->start_line, (int)def->end_line, props); - /* Register callable symbols + every type-like container (Class/Struct/ - * Interface/Enum/Type/Trait). Type-like defs must be in the registry so - * `class Foo : IBar` (INHERITS), `impl Trait for S` (IMPLEMENTS), and method/ - * field resolution can reach them — Struct included so Rust/Go/Swift/D structs - * resolve as type targets just as a Class did. Variable/Field defs are also - * registered so pass_usages.c can resolve READS/WRITES accesses (rw->var_name) - * to a Variable/Field node QN. - * KEEP IN SYNC with pass_parallel.c and pipeline_incremental.c's seed sets. */ - if (node_id > 0 && def->label && - (strcmp(def->label, "Function") == 0 || strcmp(def->label, "Method") == 0 || - cbm_label_is_type_like(def->label) || strcmp(def->label, "Variable") == 0 || - strcmp(def->label, "Field") == 0)) { + /* Registry membership is defined ONCE by cbm_label_is_registry_symbol + * (helpers.c): callables + type-like containers (INHERITS/IMPLEMENTS/method/ + * field resolution), Variable/Field (READS/WRITES resolution), and Table/View + * (SQL FROM/JOIN lineage). pass_parallel.c and pipeline_incremental.c seed + * through the same predicate, so the three registries cannot diverge. */ + if (node_id > 0 && cbm_label_is_registry_symbol(def->label)) { cbm_registry_add(ctx->registry, def->name, def->qualified_name, def->label); } char *file_qn = cbm_pipeline_fqn_compute(ctx->project_name, rel, "__file__"); diff --git a/src/pipeline/pass_parallel.c b/src/pipeline/pass_parallel.c index 0c91db09a..4e6183337 100644 --- a/src/pipeline/pass_parallel.c +++ b/src/pipeline/pass_parallel.c @@ -1200,14 +1200,9 @@ static int register_and_link_def(cbm_pipeline_ctx_t *ctx, const CBMDefinition *d if (!def->name || !def->qualified_name || !def->label) { return 0; } - /* Register callable symbols + every type-like container (Class/Struct/ - * Interface/Enum/Type/Trait) — see pass_definitions.c for rationale. Struct - * included so Rust/Go/Swift/D structs resolve as type targets. Variable/Field - * defs are registered too so READS/WRITES can resolve. - * KEEP IN SYNC with pass_definitions.c and pipeline_incremental.c. */ - if (strcmp(def->label, "Function") == 0 || strcmp(def->label, "Method") == 0 || - cbm_label_is_type_like(def->label) || strcmp(def->label, "Variable") == 0 || - strcmp(def->label, "Field") == 0) { + /* Registry membership is defined ONCE by cbm_label_is_registry_symbol + * (helpers.c) — see pass_definitions.c for the per-label rationale. */ + if (cbm_label_is_registry_symbol(def->label)) { cbm_registry_add(ctx->registry, def->name, def->qualified_name, def->label); (*reg_entries)++; } @@ -2626,8 +2621,17 @@ static void resolve_file_usages(resolve_ctx_t *rc, resolve_worker_state_t *ws, if (semantic_reference) { continue; } - cbm_resolution_t res = cbm_registry_resolve(rc->registry, usage->ref_name, module_qn, - imp_keys, imp_vals, imp_count); + /* SQL usages are FROM/JOIN lineage refs and may bind Table/View + * targets (cbm_registry_resolve_lineage); every other language + * resolves through the default variant, whose central relation + * veto keeps same-named code identifiers out of the lineage layer. + * Must mirror the sequential twin (pass_usages.c) exactly. */ + cbm_resolution_t res = + (lang == CBM_LANG_SQL) + ? cbm_registry_resolve_lineage(rc->registry, usage->ref_name, module_qn, + imp_keys, imp_vals, imp_count) + : cbm_registry_resolve(rc->registry, usage->ref_name, module_qn, imp_keys, + imp_vals, imp_count); if (!res.qualified_name || res.qualified_name[0] == '\0') { continue; } diff --git a/src/pipeline/pass_usages.c b/src/pipeline/pass_usages.c index 498cf849c..31219ba68 100644 --- a/src/pipeline/pass_usages.c +++ b/src/pipeline/pass_usages.c @@ -192,8 +192,16 @@ static int resolve_usage_edges(cbm_pipeline_ctx_t *ctx, const CBMFileResult *res if (semantic_reference) { continue; } - cbm_resolution_t res = cbm_registry_resolve(ctx->registry, usage->ref_name, module_qn, - imp_keys, imp_vals, imp_count); + /* SQL usages are FROM/JOIN lineage refs and may bind Table/View + * targets (cbm_registry_resolve_lineage); every other language + * resolves through the default variant, whose central relation + * veto keeps same-named code identifiers out of the lineage layer. */ + cbm_resolution_t res = + (lang == CBM_LANG_SQL) + ? cbm_registry_resolve_lineage(ctx->registry, usage->ref_name, module_qn, + imp_keys, imp_vals, imp_count) + : cbm_registry_resolve(ctx->registry, usage->ref_name, module_qn, imp_keys, + imp_vals, imp_count); if (!res.qualified_name || res.qualified_name[0] == '\0') { continue; } diff --git a/src/pipeline/pipeline.h b/src/pipeline/pipeline.h index 5cde0c0f0..df684ede5 100644 --- a/src/pipeline/pipeline.h +++ b/src/pipeline/pipeline.h @@ -211,11 +211,22 @@ void cbm_registry_add(cbm_registry_t *r, const char *name, const char *qualified /* Resolve a callee name using prioritized strategies. * import_map: NULL-terminated array of {local_name, resolved_qn} pairs, or NULL. - * Returns result with qualified_name="" if unresolved. */ + * Returns result with qualified_name="" if unresolved. + * Never returns a data relation (Table/View): relations are lineage-only + * registry members and common table names (users, orders, config) collide with + * code identifiers in every language, so the default resolve vetoes them + * centrally instead of relying on per-consumer label checks. */ cbm_resolution_t cbm_registry_resolve(const cbm_registry_t *r, const char *callee_name, const char *module_qn, const char **import_map_keys, const char **import_map_vals, int import_map_count); +/* Relation-permitting resolve for SQL FROM/JOIN lineage usages ONLY — the one + * consumer allowed to bind Table/View targets. Uncached (the per-file resolve + * cache stores the default variant's relation-vetoed answers). */ +cbm_resolution_t cbm_registry_resolve_lineage(const cbm_registry_t *r, const char *callee_name, + const char *module_qn, const char **import_map_keys, + const char **import_map_vals, int import_map_count); + /* Per-file memoization cache for is_import_reachable. Thread-local — * each resolve worker owns its own cache. Call _begin at the start * of resolve_file_calls (or any per-file resolve loop) and _end at diff --git a/src/pipeline/pipeline_incremental.c b/src/pipeline/pipeline_incremental.c index d3c5247f5..fca1c8f06 100644 --- a/src/pipeline/pipeline_incremental.c +++ b/src/pipeline/pipeline_incremental.c @@ -1020,20 +1020,15 @@ static void incr_free_edge_capture(cbm_edge_capture_t *cap) { /* ── Registry seed visitor ────────────────────────────────────────── */ -/* Labels the full-index definition pass seeds into the registry - * (pass_definitions.c — KEEP IN SYNC). Incremental re-resolution must see the - * SAME symbol set, or it diverges from a clean full reindex: seeding extra - * container nodes (File / Module / Folder / ...) lets a type usage like `Word` - * resolve to the same-named Module node instead of the Class node. Only - * callable / declared symbols belong in the registry. */ +/* Labels the full-index definition pass seeds into the registry. Incremental + * re-resolution must see the SAME symbol set, or it diverges from a clean full + * reindex: seeding extra container nodes (File / Module / Folder / ...) lets a + * type usage like `Word` resolve to the same-named Module node instead of the + * Class node. Membership is defined once by cbm_label_is_registry_symbol + * (helpers.c) — the same predicate pass_definitions.c / pass_parallel.c seed + * through, so divergence is impossible by construction. */ static bool incr_label_is_registry_symbol(const char *label) { - /* Mirror pass_definitions.c / pass_parallel.c registry seeding EXACTLY: - * callables + every type-like container (Class/Struct/Interface/Enum/Type/ - * Trait) + Variable/Field. Struct included so an incremental re-resolve seeds - * the same struct type nodes a full reindex would. */ - return label && (strcmp(label, "Function") == 0 || strcmp(label, "Method") == 0 || - cbm_label_is_type_like(label) || strcmp(label, "Variable") == 0 || - strcmp(label, "Field") == 0); + return cbm_label_is_registry_symbol(label); } /* Callback for cbm_gbuf_foreach_node: seed the registry with the existing diff --git a/src/pipeline/registry.c b/src/pipeline/registry.c index 447183a37..67d6c2fe3 100644 --- a/src/pipeline/registry.c +++ b/src/pipeline/registry.c @@ -25,6 +25,7 @@ enum { REG_MAX_CANDIDATES = 256 }; #define DEFAULT_CONFIDENCE 0.5 #include "pipeline/pipeline.h" +#include "cbm.h" /* cbm_label_is_relation — the resolve-time relation veto */ #include "foundation/compat.h" /* CBM_TLS */ #include "foundation/hash_table.h" #include "foundation/dyn_array.h" @@ -868,24 +869,11 @@ static cbm_resolution_t resolve_name_lookup(const cbm_registry_t *r, const char return empty_result(); } -cbm_resolution_t cbm_registry_resolve(const cbm_registry_t *r, const char *callee_name, - const char *module_qn, const char **import_map_keys, - const char **import_map_vals, int import_map_count) { - if (!r || !callee_name) { - return empty_result(); - } - - /* Per-file cache: same callee_name in N call sites → 1 chain walk - * + N-1 O(1) hash hits. module_qn is constant per file so the - * cache key only needs callee_name. */ - if (_resolve_cache) { - resolve_cache_entry_t *cached = - (resolve_cache_entry_t *)cbm_ht_get(_resolve_cache, callee_name); - if (cached) { - return cached->res; - } - } - +/* The strategy chain shared by both public resolve variants (no caching here — + * cbm_registry_resolve owns the per-file cache). */ +static cbm_resolution_t registry_resolve_chain(const cbm_registry_t *r, const char *callee_name, + const char *module_qn, const char **import_map_keys, + const char **import_map_vals, int import_map_count) { /* Split callee at the first path separator: "pkg.Func" → prefix="pkg", * suffix="Func". Rust/C++ use "::" rather than ".", so honor whichever * separator appears first ("lib::square" → prefix="lib", suffix="square"). @@ -924,6 +912,41 @@ cbm_resolution_t cbm_registry_resolve(const cbm_registry_t *r, const char *calle /* Strategy 3+4: name lookup */ res = resolve_name_lookup(r, callee_name, module_qn, import_map_vals, import_map_count); } + return res; +} + +cbm_resolution_t cbm_registry_resolve(const cbm_registry_t *r, const char *callee_name, + const char *module_qn, const char **import_map_keys, + const char **import_map_vals, int import_map_count) { + if (!r || !callee_name) { + return empty_result(); + } + + /* Per-file cache: same callee_name in N call sites → 1 chain walk + * + N-1 O(1) hash hits. module_qn is constant per file so the + * cache key only needs callee_name. */ + if (_resolve_cache) { + resolve_cache_entry_t *cached = + (resolve_cache_entry_t *)cbm_ht_get(_resolve_cache, callee_name); + if (cached) { + return cached->res; + } + } + + cbm_resolution_t res = registry_resolve_chain(r, callee_name, module_qn, import_map_keys, + import_map_vals, import_map_count); + + /* Data relations (Table/View) are lineage-only registry members: common + * table names (users, orders, config) collide with code identifiers across + * every language, so the DEFAULT resolve never returns them — a veto, not a + * re-route, so a name-collision does not fall through to a weaker strategy. + * Every consumer (CALLS/USAGE/READS/WRITES/THROWS/handlers/decorators, + * present and future) is thereby relation-safe by construction. The SQL + * lineage path opts in via cbm_registry_resolve_lineage. */ + if (res.qualified_name && res.qualified_name[0] && + cbm_label_is_relation(cbm_registry_label_of(r, res.qualified_name))) { + res = empty_result(); + } /* Cache the result (including empty — caching the negative answer * is just as valuable; same name asks the same question). */ @@ -942,6 +965,21 @@ cbm_resolution_t cbm_registry_resolve(const cbm_registry_t *r, const char *calle return res; } +cbm_resolution_t cbm_registry_resolve_lineage(const cbm_registry_t *r, const char *callee_name, + const char *module_qn, const char **import_map_keys, + const char **import_map_vals, int import_map_count) { + if (!r || !callee_name) { + return empty_result(); + } + /* Relation-permitting variant for SQL FROM/JOIN lineage usages ONLY. + * Deliberately uncached: the per-file cache is keyed by bare callee_name + * and stores the relation-vetoed answer of the default variant — sharing + * it would poison one variant with the other's semantics. SQL files hold + * few distinct relation refs, so the chain walk stays cheap. */ + return registry_resolve_chain(r, callee_name, module_qn, import_map_keys, import_map_vals, + import_map_count); +} + /* ── Fuzzy Resolve ──────────────────────────────────────────────── */ /* Filter candidates by import reachability. Returns count of reachable. */ diff --git a/src/store/store.c b/src/store/store.c index aca9f7641..015e50ca2 100644 --- a/src/store/store.c +++ b/src/store/store.c @@ -6026,9 +6026,11 @@ static int arch_boundaries(cbm_store_t *s, const char *project, const char *path char like[CBM_SZ_512]; bool scoped = arch_path_prepare(path, norm, sizeof(norm), like, sizeof(like)); char nsqlbuf[ST_SQL_BUF]; + /* Relations included: a view->table USAGE across package boundaries is + * data-lineage coupling and belongs in the architecture picture. */ const char *nbase = "SELECT id, qualified_name, file_path FROM nodes WHERE project=?1 AND label IN " - "(" CBM_SQL_CALLABLE_OR_TYPE_LABELS ")"; + "(" CBM_SQL_CALLABLE_OR_TYPE_LABELS "," CBM_SQL_RELATION_LABELS ")"; if (scoped) { snprintf(nsqlbuf, sizeof(nsqlbuf), "%s%s ORDER BY id", nbase, arch_path_scope_sql()); } else { @@ -6181,7 +6183,7 @@ static int arch_packages_from_qn(cbm_store_t *s, const char *project, const char bool scoped = arch_path_prepare(path, norm, sizeof(norm), like, sizeof(like)); char qsqlbuf[ST_SQL_BUF]; const char *qbase = "SELECT qualified_name FROM nodes WHERE project=?1 AND label IN " - "(" CBM_SQL_CALLABLE_OR_TYPE_LABELS ")"; + "(" CBM_SQL_CALLABLE_OR_TYPE_LABELS "," CBM_SQL_RELATION_LABELS ")"; if (scoped) { snprintf(qsqlbuf, sizeof(qsqlbuf), "%s%s", qbase, arch_path_scope_sql()); } else { @@ -7457,7 +7459,8 @@ static int arch_clusters(cbm_store_t *s, const char *project, const char *path, bool scoped = arch_path_prepare(path, norm, sizeof(norm), like, sizeof(like)); char nsqlbuf[ST_SQL_BUF]; const char *nbase = "SELECT id, name, qualified_name, file_path FROM nodes " - "WHERE project=?1 AND label IN (" CBM_SQL_CALLABLE_OR_TYPE_LABELS ")"; + "WHERE project=?1 AND label IN (" CBM_SQL_CALLABLE_OR_TYPE_LABELS + "," CBM_SQL_RELATION_LABELS ")"; if (scoped) { snprintf(nsqlbuf, sizeof(nsqlbuf), "%s%s ORDER BY id LIMIT ?4", nbase, arch_path_scope_sql()); diff --git a/tests/test_extraction.c b/tests/test_extraction.c index 447e7c104..506973350 100644 --- a/tests/test_extraction.c +++ b/tests/test_extraction.c @@ -1816,6 +1816,58 @@ TEST(sql_function) { PASS(); } +TEST(sql_ddl_node_labels) { + CBMFileResult *r = extract("CREATE TABLE users (id INTEGER PRIMARY KEY, name TEXT);\n" + "CREATE VIEW active_users AS SELECT * FROM users;\n", + CBM_LANG_SQL, "t", "schema.sql"); + ASSERT_NOT_NULL(r); + ASSERT_FALSE(r->has_error); + ASSERT(has_def(r, "Table", "users")); + ASSERT(has_def(r, "View", "active_users")); + cbm_free_result(r); + PASS(); +} + +TEST(sql_view_lineage_usages) { + /* A view's FROM/JOIN relations are emitted as usages (ref_name = table), + * which pass_usages later resolves into view -> table USAGE lineage edges. */ + CBMFileResult *r = extract("CREATE TABLE users (id INTEGER);\n" + "CREATE VIEW active_users AS SELECT * FROM users;\n", + CBM_LANG_SQL, "t", "schema.sql"); + ASSERT_NOT_NULL(r); + ASSERT_FALSE(r->has_error); + int found_users = 0; + for (int i = 0; i < r->usages.count; i++) { + if (r->usages.items[i].ref_name && strcmp(r->usages.items[i].ref_name, "users") == 0) { + found_users = 1; + } + } + ASSERT(found_users); + cbm_free_result(r); + PASS(); +} + +TEST(sql_schema_qualified_name) { + /* schema-qualified DDL (schema.table) is named by the table, not the schema, + * and FROM schema.table resolves to that table for lineage. */ + CBMFileResult *r = extract("CREATE TABLE app.users (id INTEGER);\n" + "CREATE VIEW app.active AS SELECT * FROM app.users;\n", + CBM_LANG_SQL, "t", "schema.sql"); + ASSERT_NOT_NULL(r); + ASSERT_FALSE(r->has_error); + ASSERT(has_def(r, "Table", "users")); + ASSERT(has_def(r, "View", "active")); + int found_users = 0; + for (int i = 0; i < r->usages.count; i++) { + if (r->usages.items[i].ref_name && strcmp(r->usages.items[i].ref_name, "users") == 0) { + found_users = 1; + } + } + ASSERT(found_users); + cbm_free_result(r); + PASS(); +} + /* --- Meson project --- */ TEST(meson_project) { CBMFileResult *r = extract( @@ -5500,6 +5552,9 @@ SUITE(extraction) { /* Config/Markup */ RUN_TEST(html_elements); RUN_TEST(sql_function); + RUN_TEST(sql_ddl_node_labels); + RUN_TEST(sql_view_lineage_usages); + RUN_TEST(sql_schema_qualified_name); RUN_TEST(meson_project); RUN_TEST(css_rules); RUN_TEST(scss_rules); diff --git a/tests/test_grammar_labels.c b/tests/test_grammar_labels.c index 78df28ad0..74284bf51 100644 --- a/tests/test_grammar_labels.c +++ b/tests/test_grammar_labels.c @@ -186,7 +186,7 @@ static const LabelGolden LABEL_GOLDENS[] = { {"toml", "Class:1,Module:1,Variable:1"}, {"ini", "Class:1,Module:1,Variable:1"}, {"csv", "Module:1"}, - {"sql", "Module:1,Variable:1"}, + {"sql", "Module:1,Table:1"}, {"xml", "Class:2,Module:1"}, {"html", "Module:1"}, {"css", "Module:1"}, diff --git a/tests/test_grammar_probe_g.c b/tests/test_grammar_probe_g.c index 3a95612a2..d0f99d4e0 100644 --- a/tests/test_grammar_probe_g.c +++ b/tests/test_grammar_probe_g.c @@ -157,6 +157,7 @@ typedef struct { int modules; int classes; int variables; + int tables; int sections; int imports; /* IMPORTS edges */ int depends; /* DEPENDS_ON edges */ @@ -172,6 +173,7 @@ static GpgMetrics gpg_metrics_files(const GpgFile *files, int nfiles) { m.modules = gpg_count_label(store, lp.project, "Module"); m.classes = gpg_count_label(store, lp.project, "Class"); m.variables = gpg_count_label(store, lp.project, "Variable"); + m.tables = gpg_count_label(store, lp.project, "Table"); m.sections = gpg_count_label(store, lp.project, "Section"); m.imports = cbm_store_count_edges_by_type(store, lp.project, "IMPORTS"); m.depends = cbm_store_count_edges_by_type(store, lp.project, "DEPENDS_ON"); @@ -605,12 +607,12 @@ TEST(probe_csv_module_only) { /* ══════════════════════════════════════════════════════════════════ * GROUP 13 — SQL (.sql) * - * SQL golden histogram: Module:1, Variable:1 - * Table references (e.g. CREATE TABLE / SELECT FROM) produce Variable nodes. + * SQL golden histogram: Module:1, Table:1 + * CREATE TABLE / CREATE VIEW produce first-class Table / View nodes. * ══════════════════════════════════════════════════════════════════ */ -/* SQL: CREATE TABLE + SELECT → at least 1 Variable node. */ -TEST(probe_sql_variable_node) { +/* SQL: CREATE TABLE → a first-class Table node (was Variable). */ +TEST(probe_sql_table_node) { GpgMetrics m = gpg_metrics("schema.sql", "CREATE TABLE users (\n" " id INTEGER PRIMARY KEY,\n" " name TEXT NOT NULL\n" @@ -618,23 +620,26 @@ TEST(probe_sql_variable_node) { "\n" "SELECT id, name FROM users WHERE id = 1;\n"); ASSERT_TRUE(m.ok); - /* GREEN: SQL table reference produces at least 1 Variable node. */ - ASSERT_TRUE(m.variables >= 1); + /* GREEN: CREATE TABLE produces a first-class Table node, and no longer + * a generic Variable. */ + ASSERT_TRUE(m.tables >= 1); + ASSERT_TRUE(m.variables == 0); ASSERT_TRUE(m.modules >= 1); PASS(); } -/* SQL: multiple statements → still at least 1 Variable. */ +/* SQL: DML-only file → no relation defs at all. */ TEST(probe_sql_insert_select) { GpgMetrics m = gpg_metrics( "queries.sql", "INSERT INTO orders (user_id, total) VALUES (1, 99.99);\n" "SELECT o.id, u.name FROM orders o JOIN users u ON o.user_id = u.id;\n"); ASSERT_TRUE(m.ok); - /* GREEN (fixture fix): SQL Variable nodes come ONLY from DDL - * create_table/create_view (lang_specs.c sql_var_types), NOT from - * DML table *references* in INSERT/SELECT. A DML-only file correctly - * yields 0 Variable nodes; the original `>= 1` asserted a non-feature. - * Assert the true contract: no table definitions → 0 Variables. */ + /* GREEN (fixture fix): SQL Table/View nodes come ONLY from DDL + * create_table/create_view (extract_sql_ddl_class_def), NOT from DML + * table *references* in INSERT/SELECT. A DML-only file correctly yields + * 0 relation defs; the original `>= 1 Variable` asserted a non-feature. + * Assert the true contract: no table definitions → 0 Tables, 0 Variables. */ + ASSERT_TRUE(m.tables == 0); ASSERT_TRUE(m.variables == 0); PASS(); } @@ -1084,7 +1089,7 @@ SUITE(grammar_probe_g) { RUN_TEST(probe_csv_module_only); /* SQL */ - RUN_TEST(probe_sql_variable_node); + RUN_TEST(probe_sql_table_node); RUN_TEST(probe_sql_insert_select); /* SOQL */ diff --git a/tests/test_pipeline.c b/tests/test_pipeline.c index 25878a8b3..8e70d86a1 100644 --- a/tests/test_pipeline.c +++ b/tests/test_pipeline.c @@ -2204,6 +2204,96 @@ TEST(pipeline_incremental_repoints_call_reference_without_stale_edge) { PASS(); } +/* SQL DDL becomes first-class Table/View nodes wired into FROM/JOIN lineage, + * while the shared name registry must NOT leak those relations into other + * languages' textual resolution: a Python call or identifier sharing the + * table's name (`users`) would otherwise unique-name-bind a false CALLS/USAGE + * edge into the lineage layer. Pins the resolve-time relation veto + * (cbm_registry_resolve) together with the lineage opt-in + * (cbm_registry_resolve_lineage). */ +TEST(pipeline_sql_lineage_and_relation_isolation) { + char tmp[256]; + snprintf(tmp, sizeof(tmp), "/tmp/cbm_sql_lineage_XXXXXX"); + if (!cbm_mkdtemp(tmp)) { + FAIL("tmpdir"); + } + write_temp_file(tmp, "schema.sql", + "CREATE TABLE users (id INTEGER PRIMARY KEY, name TEXT);\n" + "CREATE VIEW active_users AS SELECT * FROM users;\n"); + /* `users` exists project-wide ONLY as the SQL table, so without the + * relation veto the cross-file unique-name fallback would bind both the + * call and the bare reference below straight to the Table node. */ + write_temp_file(tmp, "app.py", + "def load_users():\n" + " return users()\n" + "\n" + "def show_users():\n" + " return users\n"); + char db_path[512]; + snprintf(db_path, sizeof(db_path), "%s/sql_lineage.db", tmp); + cbm_pipeline_t *p = cbm_pipeline_new(tmp, db_path, CBM_MODE_FULL); + ASSERT_NOT_NULL(p); + ASSERT_EQ(cbm_pipeline_run(p), 0); + const char *project = cbm_pipeline_project_name(p); + cbm_store_t *s = cbm_store_open_path(db_path); + ASSERT_NOT_NULL(s); + /* Positive control: the view's FROM emits real lineage. */ + ASSERT_EQ(named_edge_count(s, project, "USAGE", "active_users", "users"), 1); + /* Isolation: no Python edge of any kind reaches the Table. */ + ASSERT_EQ(named_edge_count(s, project, "CALLS", "load_users", "users"), 0); + ASSERT_EQ(named_edge_count(s, project, "USAGE", "load_users", "users"), 0); + ASSERT_EQ(named_edge_count(s, project, "USAGE", "show_users", "users"), 0); + ASSERT_EQ(named_edge_count(s, project, "READS", "show_users", "users"), 0); + cbm_store_close(s); + cbm_pipeline_free(p); + th_rmtree(tmp); + PASS(); +} + +/* Renaming a table must drop lineage from DEPENDENT (unchanged) SQL files on + * the incremental path. Table/View participate in the per-file LSP surface + * hash as registry-only labels (lsp_surface.c), so tables.sql's def change + * invalidates views.sql's resolution instead of slipping the early cutoff and + * leaving a stale view -> old-table USAGE edge. */ +TEST(pipeline_incremental_sql_table_rename_drops_stale_lineage) { + char tmp[256]; + snprintf(tmp, sizeof(tmp), "/tmp/cbm_sql_rename_XXXXXX"); + if (!cbm_mkdtemp(tmp)) { + FAIL("tmpdir"); + } + write_temp_file(tmp, "tables.sql", "CREATE TABLE users (id INTEGER PRIMARY KEY);\n"); + write_temp_file(tmp, "views.sql", "CREATE VIEW active AS SELECT * FROM users;\n"); + char db_path[512]; + snprintf(db_path, sizeof(db_path), "%s/sql_rename.db", tmp); + cbm_pipeline_t *first = cbm_pipeline_new(tmp, db_path, CBM_MODE_FULL); + ASSERT_NOT_NULL(first); + ASSERT_EQ(cbm_pipeline_run(first), 0); + const char *first_project = cbm_pipeline_project_name(first); + cbm_store_t *first_store = cbm_store_open_path(db_path); + ASSERT_NOT_NULL(first_store); + ASSERT_EQ(named_edge_count(first_store, first_project, "USAGE", "active", "users"), 1); + cbm_store_close(first_store); + cbm_pipeline_free(first); + + write_temp_file(tmp, "tables.sql", "CREATE TABLE people (id INTEGER PRIMARY KEY);\n"); + cbm_pipeline_incremental_test_reset_faults(); + cbm_pipeline_t *second = cbm_pipeline_new(tmp, db_path, CBM_MODE_FULL); + ASSERT_NOT_NULL(second); + ASSERT_EQ(cbm_pipeline_run(second), 0); + const char *second_project = cbm_pipeline_project_name(second); + cbm_store_t *second_store = cbm_store_open_path(db_path); + ASSERT_NOT_NULL(second_store); + /* The view's FROM still says `users`, which no longer exists: the old + * edge must be gone (no stale lineage), and the unchanged dependent must + * not have been rebound to the renamed table either. */ + ASSERT_EQ(named_edge_count(second_store, second_project, "USAGE", "active", "users"), 0); + ASSERT_EQ(named_edge_count(second_store, second_project, "USAGE", "active", "people"), 0); + cbm_store_close(second_store); + cbm_pipeline_free(second); + th_rmtree(tmp); + PASS(); +} + /* Re-indexing only the caller must retain cross-file semantic proof for a * callable value whose definition lives in an unchanged file. A fresh full * index of the edited sources is the convergence oracle: incremental output @@ -12256,6 +12346,8 @@ SUITE(pipeline) { SUITE(pipeline_semantic_manifest_repro) { RUN_TEST(incremental_downgrade_preserves_scope_and_artifact_across_change_noop_delete); RUN_TEST(pipeline_incremental_repoints_call_reference_without_stale_edge); + RUN_TEST(pipeline_sql_lineage_and_relation_isolation); + RUN_TEST(pipeline_incremental_sql_table_rename_drops_stale_lineage); RUN_TEST(pipeline_parallel_manifest_is_byte_stable_above_threshold); RUN_TEST(pipeline_closure_repair_body_edit_converges_with_fresh_full); RUN_TEST(pipeline_closure_repair_removed_def_drops_dependent_edge); diff --git a/tests/test_store_nodes.c b/tests/test_store_nodes.c index d063ea748..b8c8305bf 100644 --- a/tests/test_store_nodes.c +++ b/tests/test_store_nodes.c @@ -52,6 +52,36 @@ TEST(sql_label_allowlists_match_cbm_label_is_type_like) { PASS(); } +/* Same drift guard for the relation labels (Table/View — SQL data lineage). + * Relations are registry symbols but deliberately NOT type-like: the default + * cbm_registry_resolve vetoes them, so a code identifier sharing a table's + * name never binds into the lineage layer. */ +TEST(sql_relation_labels_match_cbm_label_is_relation) { + static const char *const relations[] = {"Table", "View"}; + for (size_t i = 0; i < sizeof(relations) / sizeof(relations[0]); i++) { + ASSERT_TRUE(cbm_label_is_relation(relations[i])); + ASSERT_TRUE(cbm_label_is_registry_symbol(relations[i])); + ASSERT_FALSE(cbm_label_is_type_like(relations[i])); + char quoted[64]; + snprintf(quoted, sizeof(quoted), "'%s'", relations[i]); + ASSERT_NOT_NULL(strstr(CBM_SQL_RELATION_LABELS, quoted)); + /* Relations must NOT ride in the callable/type fragments — the arch + * queries opt in explicitly by appending CBM_SQL_RELATION_LABELS. */ + ASSERT_NULL(strstr(CBM_SQL_CALLABLE_OR_TYPE_LABELS, quoted)); + } + /* cbm_label_is_registry_symbol covers exactly the seeded families. */ + ASSERT_TRUE(cbm_label_is_registry_symbol("Function")); + ASSERT_TRUE(cbm_label_is_registry_symbol("Method")); + ASSERT_TRUE(cbm_label_is_registry_symbol("Class")); + ASSERT_TRUE(cbm_label_is_registry_symbol("Variable")); + ASSERT_TRUE(cbm_label_is_registry_symbol("Field")); + ASSERT_FALSE(cbm_label_is_registry_symbol("Module")); + ASSERT_FALSE(cbm_label_is_registry_symbol("File")); + ASSERT_FALSE(cbm_label_is_relation("Class")); + ASSERT_FALSE(cbm_label_is_relation(NULL)); + PASS(); +} + /* ── Schema / Open / Close ──────────────────────────────────────── */ TEST(store_open_memory) { @@ -2177,6 +2207,7 @@ SUITE(store_nodes) { RUN_TEST(store_coverage_replace_rejects_invalid_row_arguments); RUN_TEST(store_coverage_replace_rolls_back_when_shadow_rebuild_fails); RUN_TEST(sql_label_allowlists_match_cbm_label_is_type_like); + RUN_TEST(sql_relation_labels_match_cbm_label_is_relation); RUN_TEST(store_open_memory); RUN_TEST(store_close_null); RUN_TEST(store_open_memory_twice); From 43ecc0988c045f939cb96813123bee86e53a10bc Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Thu, 20 Aug 2026 10:23:48 +0200 Subject: [PATCH 11/21] feat(dbt): extract dbt model lineage from Jinja-templated SQL A dbt model is an ordinary .sql file whose dependencies are written {{ ref('other_model') }} or {{ source('group','table') }}, never as literal table names. The SQL grammar cannot read those -- FROM {{ ref( 'x') }} is a parse error to it -- so the dependency structure of an entire dbt project was invisible to the graph. extract_dbt.c runs as a sub-extractor inside the normal indexing pipeline. Per qualifying file it emits one Model definition (named by the file stem, dbt's own model identity) plus one usage per ref()/ source() call; pass_usages resolves those into model -> relation lineage edges like any other reference. Model is a relation label alongside Table and View, which is what makes the rest work without new plumbing: registry seeding, the central relation veto, the incremental surface hash, search ranking and the architecture queries all pick it up from cbm_label_is_relation. Sharing one label class also means a model's source('raw','customers') resolves onto a Table declared in a plain DDL migration in the same repository, so dbt lineage and SQL lineage form one graph rather than two, while the veto keeps model names out of every non-lineage consumer. The pass is self-gating: SQL files only, and only those carrying a real ref()/source() call. The dbt builtins are themselves the evidence, which is cheaper than a dbt_project.yml lookup and more precise -- templated SQL that is not dbt (an Airflow {{ ds }} parameter) produces no Model node and no usages even inside a dbt repository. {% macro %} definitions are deliberately excluded: the vendored tree-sitter-jinja2 grammar has no node types for {% %} statements, so they can only be recovered by a hand-written scanner that cannot see comments or find {% endmacro %} for a correct span. Filed as follow-up rather than approximated. Tests cover the lineage, the last-string-argument semantics of both builtins, the gate (against a plain-SQL control extraction), plain DDL staying untouched, and an end-to-end pipeline case asserting model -> model across files, model -> Table onto plain DDL, and cross-language isolation. Disabling the pass reddens three of them; removing Model from the relation set breaks lineage outright. Implements the lineage half of #575. Co-authored-by: alexisperinger-ux Signed-off-by: Martin Vogel --- Makefile.cbm | 1 + internal/cbm/cbm.c | 7 ++ internal/cbm/cbm.h | 4 + internal/cbm/extract_dbt.c | 219 +++++++++++++++++++++++++++++++++++++ internal/cbm/helpers.c | 16 ++- src/foundation/constants.h | 6 +- tests/test_extraction.c | 98 +++++++++++++++++ tests/test_pipeline.c | 43 ++++++++ tests/test_store_nodes.c | 2 +- 9 files changed, 386 insertions(+), 10 deletions(-) create mode 100644 internal/cbm/extract_dbt.c diff --git a/Makefile.cbm b/Makefile.cbm index bf51a5a8c..28eb21e27 100644 --- a/Makefile.cbm +++ b/Makefile.cbm @@ -245,6 +245,7 @@ EXTRACTION_SRCS = \ $(CBM_DIR)/extract_env_accesses.c \ $(CBM_DIR)/extract_channels.c \ $(CBM_DIR)/extract_k8s.c \ + $(CBM_DIR)/extract_dbt.c \ $(CBM_DIR)/helpers.c \ $(CBM_DIR)/lang_specs.c \ $(CBM_DIR)/macro_table.c \ diff --git a/internal/cbm/cbm.c b/internal/cbm/cbm.c index d7468af5a..6c46d7c91 100644 --- a/internal/cbm/cbm.c +++ b/internal/cbm/cbm.c @@ -1303,6 +1303,13 @@ CBMFileResult *cbm_extract_file_ex(const char *source, int source_len, CBMLangua cbm_extract_k8s(&ctx); } + // dbt lineage pass: a dbt model's dependencies live in Jinja ({{ ref(...) }}), + // which the SQL grammar cannot read. Self-gated — SQL files only, and only + // those carrying a real dbt builtin call. + if (ctx.language == CBM_LANG_SQL) { + cbm_extract_dbt(&ctx); + } + // LSP type-aware call/usage resolution (per-file). Runs in every mode; // refines the tree-sitter + textual-resolution graph with type info. uint64_t lsp_start = now_ns(); diff --git a/internal/cbm/cbm.h b/internal/cbm/cbm.h index 63849a3c9..aebd12291 100644 --- a/internal/cbm/cbm.h +++ b/internal/cbm/cbm.h @@ -726,6 +726,10 @@ void cbm_channels_push(CBMChannelArray *arr, CBMArena *a, CBMChannel ch); // --- Sub-extractor entry points --- void cbm_extract_definitions(CBMExtractCtx *ctx); +// dbt lineage for Jinja-templated SQL models: emits a Model def plus one usage +// per ref()/source() call. No-op unless the file parses as SQL and actually +// contains a dbt builtin call. Defined in extract_dbt.c. +void cbm_extract_dbt(CBMExtractCtx *ctx); void cbm_extract_imports(CBMExtractCtx *ctx); void cbm_extract_usages(CBMExtractCtx *ctx); void cbm_extract_semantic(CBMExtractCtx *ctx); diff --git a/internal/cbm/extract_dbt.c b/internal/cbm/extract_dbt.c new file mode 100644 index 000000000..99b50df57 --- /dev/null +++ b/internal/cbm/extract_dbt.c @@ -0,0 +1,219 @@ +// extract_dbt.c — dbt lineage extractor for Jinja-templated SQL models. +// +// A dbt model is an ordinary .sql file in the repository whose SELECT is +// templated with Jinja: dependencies are written `{{ ref('other_model') }}` +// (another model in the project) or `{{ source('group', 'table') }}` (a raw +// warehouse table), never as literal table names. The SQL grammar cannot read +// those — `FROM {{ ref('x') }}` is a parse error to it — so the dependency +// structure of an entire dbt project is invisible to the code graph without +// this pass. +// +// The vendored tree-sitter-jinja2 grammar models `{{ ... }}` expressions +// (jinja_expression / fn_call / lit_string) but has no node types at all for +// `{% ... %}` statements, so this extractor covers lineage only. `{% macro %}` +// definitions would need a hand-written scanner and are deliberately left out +// rather than recovered approximately. +// +// What it emits, per qualifying file: +// - one Model definition, named by the file stem (dbt's own model identity) +// - one usage per ref()/source() call, scoped to that Model +// pass_usages then resolves each usage against the shared definition registry +// and emits a `model -USAGE-> relation` lineage edge. Model is a relation label +// (cbm_label_is_relation), so lineage joins dbt models to Table/View nodes +// declared in plain DDL elsewhere in the same repository, and the registry's +// relation veto keeps these names out of every non-lineage consumer. +// +// Gate: the file must parse as SQL, contain a Jinja expression delimiter, and +// contain at least one real ref()/source() call. Generic templated SQL (an +// Airflow `{{ ds }}` parameter, say) therefore produces nothing at all — the +// dbt builtins are the evidence that this is a dbt model, so no dbt_project.yml +// lookup is needed and no non-dbt repository pays for a Model node it did not +// ask for. + +#include "cbm.h" +#include "arena.h" +#include "helpers.h" +#include "lang_specs.h" +#include "tree_sitter/api.h" +#include "foundation/constants.h" +#include +#include +#include + +/* Local constants. */ +enum { DBT_FIRST_LINE = 1 }; + +/* Cheap pre-filter: does the source contain a Jinja expression opener? Files + * without one cannot hold a ref()/source() call, and skipping them keeps the + * second parse off every ordinary .sql file in the repository. */ +static bool source_has_jinja_expr(const char *s, int len) { + for (int i = 0; i + 1 < len; i++) { + if (s[i] == '{' && s[i + 1] == '{') { + return true; + } + } + return false; +} + +/* Strip one pair of surrounding quotes from a jinja lit_string. The grammar + * hands back the token with its quotes attached. */ +static char *dbt_unquote(CBMArena *a, char *s) { + if (!s) { + return NULL; + } + size_t n = strlen(s); + if (n >= 2 && (s[0] == '\'' || s[0] == '"') && s[n - 1] == s[0]) { + char *inner = cbm_arena_strdup(a, s + 1); + if (!inner) { + return s; + } + size_t m = strlen(inner); + if (m > 0) { + inner[m - 1] = '\0'; + } + return inner; + } + return s; +} + +/* Rightmost lit_string under `node` in DFS order. Both dbt builtins put the + * referenced relation last: ref('model'), ref('package', 'model') and + * source('group', 'table') all name the relation in their final string + * argument. The callee itself is an identifier, never a lit_string, so it + * cannot be mistaken for one. */ +static TSNode dbt_last_lit_string(TSNode node, TSNode best, bool *found) { + if (strcmp(ts_node_type(node), "lit_string") == 0) { + best = node; + *found = true; + } + uint32_t cc = ts_node_child_count(node); + for (uint32_t i = 0; i < cc; i++) { + best = dbt_last_lit_string(ts_node_child(node, i), best, found); + } + return best; +} + +/* Collect ref()/source() targets from a jinja2 parse tree into `out`, which is + * the file's usage array. Usages are scoped to enclosing_qn (the Model). */ +static void collect_dbt_refs(CBMExtractCtx *ctx, TSNode node, const char *enclosing_qn, + CBMUsageArray *out) { + if (strcmp(ts_node_type(node), "fn_call") == 0) { + TSNode fn = ts_node_child_by_field_name(node, "fn_name", (uint32_t)strlen("fn_name")); + if (ts_node_is_null(fn)) { + fn = cbm_find_child_by_kind(node, "identifier"); + } + if (!ts_node_is_null(fn)) { + char *fname = cbm_node_text(ctx->arena, fn, ctx->source); + if (fname && (strcmp(fname, "ref") == 0 || strcmp(fname, "source") == 0)) { + bool found = false; + TSNode empty = {0}; + TSNode strn = dbt_last_lit_string(node, empty, &found); + if (found) { + char *name = + dbt_unquote(ctx->arena, cbm_node_text(ctx->arena, strn, ctx->source)); + if (name && name[0]) { + CBMUsage usage = {0}; + usage.ref_name = name; + usage.enclosing_func_qn = enclosing_qn; + usage.site_start_byte = ts_node_start_byte(strn); + usage.site_end_byte = ts_node_end_byte(strn); + cbm_usages_push(out, ctx->arena, usage); + } + } + } + } + } + uint32_t cc = ts_node_child_count(node); + for (uint32_t i = 0; i < cc; i++) { + collect_dbt_refs(ctx, ts_node_child(node, i), enclosing_qn, out); + } +} + +/* dbt model identity is the file stem: models/staging/stg_users.sql is the + * model `stg_users`, and `{{ ref('stg_users') }}` anywhere in the project + * addresses it by that bare name (dbt requires model names to be unique across + * a project, so the directory is deliberately not part of the identity). */ +static char *dbt_name_from_path(CBMArena *a, const char *rel_path) { + if (!rel_path) { + return NULL; + } + const char *base = rel_path; + for (const char *p = rel_path; *p; p++) { + if (*p == '/' || *p == '\\') { + base = p + 1; + } + } + const char *dot = NULL; + for (const char *p = base; *p; p++) { + if (*p == '.') { + dot = p; + } + } + size_t n = dot ? (size_t)(dot - base) : strlen(base); + if (n == 0) { + return NULL; + } + char *out = cbm_arena_strdup(a, base); + if (!out) { + return NULL; + } + out[n] = '\0'; + return out; +} + +void cbm_extract_dbt(CBMExtractCtx *ctx) { + if (!ctx || ctx->language != CBM_LANG_SQL || !ctx->source || ctx->source_len <= 0) { + return; + } + if (!source_has_jinja_expr(ctx->source, ctx->source_len)) { + return; + } + const TSLanguage *jl = cbm_ts_language(CBM_LANG_JINJA2); + if (!jl) { + return; + } + char *model_name = dbt_name_from_path(ctx->arena, ctx->rel_path); + if (!model_name || !model_name[0]) { + return; + } + const char *model_qn = cbm_fqn_compute(ctx->arena, ctx->project, ctx->rel_path, model_name); + if (!model_qn) { + return; + } + + /* A fresh parser: the primary SQL pass owns the thread-local one, and this + * runs inside its walk. */ + TSParser *parser = ts_parser_new(); + if (!parser) { + return; + } + /* Refs are staged locally so a file with Jinja but no dbt builtins commits + * nothing at all — neither usages nor a Model node. */ + CBMUsageArray staged = {0}; + if (ts_parser_set_language(parser, jl)) { + TSTree *tree = ts_parser_parse_string(parser, NULL, ctx->source, (uint32_t)ctx->source_len); + if (tree) { + collect_dbt_refs(ctx, ts_tree_root_node(tree), model_qn, &staged); + ts_tree_delete(tree); + } + } + ts_parser_delete(parser); + if (staged.count == 0) { + return; /* templated SQL, but not dbt — emit nothing */ + } + + CBMDefinition def; + memset(&def, 0, sizeof(def)); + def.name = model_name; + def.qualified_name = model_qn; + def.label = "Model"; + def.file_path = ctx->rel_path; + def.start_line = DBT_FIRST_LINE; + def.end_line = ts_node_end_point(ctx->root).row + TS_LINE_OFFSET; + def.is_exported = true; + cbm_defs_push(&ctx->result->defs, ctx->arena, def); + + for (int i = 0; i < staged.count; i++) { + cbm_usages_push(&ctx->result->usages, ctx->arena, staged.items[i]); + } +} diff --git a/internal/cbm/helpers.c b/internal/cbm/helpers.c index cae8164ae..26fa55820 100644 --- a/internal/cbm/helpers.c +++ b/internal/cbm/helpers.c @@ -166,16 +166,20 @@ bool cbm_label_is_type_like(const char *label) { strcmp(label, "Type") == 0 || strcmp(label, "Trait") == 0; } -// True when `label` names a data relation (SQL CREATE TABLE / CREATE VIEW). -// Relations live in the registry so FROM/JOIN lineage can resolve, but they are -// deliberately NOT type-like: they must never satisfy inheritance, impl-receiver, -// semantic-type, or LSP-registrar lookups, and resolver fallbacks treat them as -// lineage-only targets (see cbm_label_is_registry_symbol call sites). +// True when `label` names a data relation: SQL CREATE TABLE / CREATE VIEW, and +// a dbt Model (a Jinja-templated .sql file, which materializes as a warehouse +// table or view). Relations live in the registry so FROM/JOIN and dbt ref() +// lineage can resolve, and sharing one label class is what lets a dbt model's +// ref() reach a Table declared in plain DDL elsewhere in the same repository. +// They are deliberately NOT type-like: they must never satisfy inheritance, +// impl-receiver, semantic-type, or LSP-registrar lookups, and resolver +// fallbacks treat them as lineage-only targets (see cbm_label_is_registry_symbol +// call sites). bool cbm_label_is_relation(const char *label) { if (!label) { return false; } - return strcmp(label, "Table") == 0 || strcmp(label, "View") == 0; + return strcmp(label, "Table") == 0 || strcmp(label, "View") == 0 || strcmp(label, "Model") == 0; } // True when `label` belongs in the cross-file name registry (see cbm.h). Single diff --git a/src/foundation/constants.h b/src/foundation/constants.h index c9f3a9d70..9af0d4001 100644 --- a/src/foundation/constants.h +++ b/src/foundation/constants.h @@ -108,8 +108,8 @@ enum { SKIP_ONE = 1, PAIR_LEN = 2 }; #define CBM_SQL_TYPE_LIKE_LABELS "'Class','Struct','Interface','Enum','Type','Trait'" #define CBM_SQL_CALLABLE_LABELS "'Function','Method'" #define CBM_SQL_CALLABLE_OR_TYPE_LABELS CBM_SQL_CALLABLE_LABELS "," CBM_SQL_TYPE_LIKE_LABELS -/* SQL mirror of cbm_label_is_relation() (Table/View — data-lineage nodes), - * pinned by tests/test_store_nodes.c the same way as the sets above. */ -#define CBM_SQL_RELATION_LABELS "'Table','View'" +/* SQL mirror of cbm_label_is_relation() (Table/View/Model — data-lineage + * nodes), pinned by tests/test_store_nodes.c the same way as the sets above. */ +#define CBM_SQL_RELATION_LABELS "'Table','View','Model'" #endif /* CBM_CONSTANTS_H */ diff --git a/tests/test_extraction.c b/tests/test_extraction.c index 506973350..89b5414ed 100644 --- a/tests/test_extraction.c +++ b/tests/test_extraction.c @@ -1868,6 +1868,100 @@ TEST(sql_schema_qualified_name) { PASS(); } +/* --- dbt Jinja lineage --- */ + +/* Helper: does the file's usage list carry `name`? */ +static int has_usage(CBMFileResult *r, const char *name) { + for (int i = 0; i < r->usages.count; i++) { + if (r->usages.items[i].ref_name && strcmp(r->usages.items[i].ref_name, name) == 0) { + return 1; + } + } + return 0; +} + +TEST(dbt_model_and_ref_lineage) { + /* A dbt model: the file stem is the model identity, and each ref() is a + * dependency on another model. The SQL grammar cannot read `{{ ref(..) }}` + * at all, so without the dbt pass this file yields no lineage whatsoever. */ + CBMFileResult *r = extract("SELECT o.id, c.name\n" + "FROM {{ ref('stg_orders') }} o\n" + "JOIN {{ ref('stg_customers') }} c ON c.id = o.customer_id\n", + CBM_LANG_SQL, "t", "models/marts/orders_enriched.sql"); + ASSERT_NOT_NULL(r); + ASSERT(has_def(r, "Model", "orders_enriched")); + ASSERT(has_usage(r, "stg_orders")); + ASSERT(has_usage(r, "stg_customers")); + cbm_free_result(r); + PASS(); +} + +TEST(dbt_source_and_two_arg_ref) { + /* Both dbt builtins name the relation in their LAST string argument: + * source('group','table') -> table, and the two-argument + * ref('package','model') form -> model. */ + CBMFileResult *r = extract("SELECT * FROM {{ source('raw', 'customers') }}\n" + "UNION ALL SELECT * FROM {{ ref('analytics', 'legacy_customers') }}\n", + CBM_LANG_SQL, "t", "models/stg_customers.sql"); + ASSERT_NOT_NULL(r); + ASSERT(has_def(r, "Model", "stg_customers")); + ASSERT(has_usage(r, "customers")); + ASSERT(has_usage(r, "legacy_customers")); + /* the group/package argument is not the relation */ + ASSERT_FALSE(has_usage(r, "raw")); + ASSERT_FALSE(has_usage(r, "analytics")); + cbm_free_result(r); + PASS(); +} + +TEST(dbt_ignores_non_dbt_jinja) { + /* Templated SQL is not dbt SQL. An Airflow-style parameter substitution has + * Jinja but no dbt builtin, so the dbt pass must contribute NOTHING — no + * Model node named after the file, and no usage minted from the template + * variables. This is the gate that keeps every non-dbt repository free of + * fabricated data-lineage vocabulary. + * + * The ordinary SQL identifier path is unaffected and still sees the literal + * `FROM events`; the second extraction below is the control proving that + * usage is pre-existing SQL behaviour rather than anything dbt added. */ + CBMFileResult *r = extract("SELECT * FROM events WHERE day = '{{ ds }}'\n" + " AND region = '{{ params.region_code }}'\n", + CBM_LANG_SQL, "t", "queries/daily_events.sql"); + ASSERT_NOT_NULL(r); + ASSERT_FALSE(has_def(r, "Model", "daily_events")); + + /* Control: the same statement with the templates replaced by plain string + * literals. Both parse as SQL identically, so an equal usage count is the + * precise statement of "the dbt pass contributed nothing here" — stronger + * than naming individual identifiers, and immune to how SQL happens to + * tokenize the template text. */ + CBMFileResult *plain = extract("SELECT * FROM events WHERE day = '2026-01-01'\n" + " AND region = 'eu-west'\n", + CBM_LANG_SQL, "t", "queries/daily_events.sql"); + ASSERT_NOT_NULL(plain); + ASSERT_FALSE(has_def(plain, "Model", "daily_events")); + ASSERT_EQ(r->usages.count, plain->usages.count); + ASSERT_EQ(r->defs.count, plain->defs.count); + cbm_free_result(plain); + cbm_free_result(r); + PASS(); +} + +TEST(dbt_plain_sql_untouched) { + /* Plain DDL keeps producing exactly the Table/View relations it did before + * the dbt pass existed — no Model node, and the FROM lineage is unchanged. */ + CBMFileResult *r = extract("CREATE TABLE users (id INTEGER);\n" + "CREATE VIEW active_users AS SELECT * FROM users;\n", + CBM_LANG_SQL, "t", "schema.sql"); + ASSERT_NOT_NULL(r); + ASSERT_FALSE(r->has_error); + ASSERT(has_def(r, "Table", "users")); + ASSERT(has_def(r, "View", "active_users")); + ASSERT_FALSE(has_def(r, "Model", "schema")); + cbm_free_result(r); + PASS(); +} + /* --- Meson project --- */ TEST(meson_project) { CBMFileResult *r = extract( @@ -5555,6 +5649,10 @@ SUITE(extraction) { RUN_TEST(sql_ddl_node_labels); RUN_TEST(sql_view_lineage_usages); RUN_TEST(sql_schema_qualified_name); + RUN_TEST(dbt_model_and_ref_lineage); + RUN_TEST(dbt_source_and_two_arg_ref); + RUN_TEST(dbt_ignores_non_dbt_jinja); + RUN_TEST(dbt_plain_sql_untouched); RUN_TEST(meson_project); RUN_TEST(css_rules); RUN_TEST(scss_rules); diff --git a/tests/test_pipeline.c b/tests/test_pipeline.c index 8e70d86a1..fb33bcc72 100644 --- a/tests/test_pipeline.c +++ b/tests/test_pipeline.c @@ -2250,6 +2250,48 @@ TEST(pipeline_sql_lineage_and_relation_isolation) { PASS(); } +/* dbt lineage end-to-end. A dbt project's dependency structure lives entirely + * in Jinja ({{ ref('x') }}), which the SQL grammar cannot read, so this is the + * whole value: model -> model edges across files, plus the join onto a Table + * declared in ordinary DDL — Model and Table are both relation labels, so one + * lineage layer spans both. The Python file is the isolation control: `stg_orders` + * exists project-wide only as a dbt model, and the registry's relation veto must + * keep a same-named call out of the lineage layer. */ +TEST(pipeline_dbt_jinja_lineage) { + char tmp[256]; + snprintf(tmp, sizeof(tmp), "/tmp/cbm_dbt_lineage_XXXXXX"); + if (!cbm_mkdtemp(tmp)) { + FAIL("tmpdir"); + } + write_temp_file(tmp, "raw_schema.sql", "CREATE TABLE customers (id INTEGER, name TEXT);\n"); + write_temp_file(tmp, "stg_orders.sql", + "SELECT id, customer_id FROM {{ source('raw', 'customers') }}\n"); + write_temp_file(tmp, "orders_enriched.sql", + "SELECT o.id, c.name\n" + "FROM {{ ref('stg_orders') }} o\n" + "JOIN {{ ref('stg_orders') }} c ON c.id = o.customer_id\n"); + write_temp_file(tmp, "app.py", "def load():\n return stg_orders()\n"); + char db_path[512]; + snprintf(db_path, sizeof(db_path), "%s/dbt.db", tmp); + cbm_pipeline_t *p = cbm_pipeline_new(tmp, db_path, CBM_MODE_FULL); + ASSERT_NOT_NULL(p); + ASSERT_EQ(cbm_pipeline_run(p), 0); + const char *project = cbm_pipeline_project_name(p); + cbm_store_t *s = cbm_store_open_path(db_path); + ASSERT_NOT_NULL(s); + /* model -> model: the ref() lineage the SQL grammar cannot see */ + ASSERT_TRUE(named_edge_count(s, project, "USAGE", "orders_enriched", "stg_orders") >= 1); + /* model -> table: source() joining dbt onto plain DDL in the same repo */ + ASSERT_EQ(named_edge_count(s, project, "USAGE", "stg_orders", "customers"), 1); + /* isolation: the Python call must not reach the model */ + ASSERT_EQ(named_edge_count(s, project, "CALLS", "load", "stg_orders"), 0); + ASSERT_EQ(named_edge_count(s, project, "USAGE", "load", "stg_orders"), 0); + cbm_store_close(s); + cbm_pipeline_free(p); + th_rmtree(tmp); + PASS(); +} + /* Renaming a table must drop lineage from DEPENDENT (unchanged) SQL files on * the incremental path. Table/View participate in the per-file LSP surface * hash as registry-only labels (lsp_surface.c), so tables.sql's def change @@ -12348,6 +12390,7 @@ SUITE(pipeline_semantic_manifest_repro) { RUN_TEST(pipeline_incremental_repoints_call_reference_without_stale_edge); RUN_TEST(pipeline_sql_lineage_and_relation_isolation); RUN_TEST(pipeline_incremental_sql_table_rename_drops_stale_lineage); + RUN_TEST(pipeline_dbt_jinja_lineage); RUN_TEST(pipeline_parallel_manifest_is_byte_stable_above_threshold); RUN_TEST(pipeline_closure_repair_body_edit_converges_with_fresh_full); RUN_TEST(pipeline_closure_repair_removed_def_drops_dependent_edge); diff --git a/tests/test_store_nodes.c b/tests/test_store_nodes.c index b8c8305bf..4e8991ba5 100644 --- a/tests/test_store_nodes.c +++ b/tests/test_store_nodes.c @@ -57,7 +57,7 @@ TEST(sql_label_allowlists_match_cbm_label_is_type_like) { * cbm_registry_resolve vetoes them, so a code identifier sharing a table's * name never binds into the lineage layer. */ TEST(sql_relation_labels_match_cbm_label_is_relation) { - static const char *const relations[] = {"Table", "View"}; + static const char *const relations[] = {"Table", "View", "Model"}; for (size_t i = 0; i < sizeof(relations) / sizeof(relations[0]); i++) { ASSERT_TRUE(cbm_label_is_relation(relations[i])); ASSERT_TRUE(cbm_label_is_registry_symbol(relations[i])); From 3308f36080803af0122f54ff0b1c6b8bd6c5646e Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Thu, 20 Aug 2026 13:08:21 +0200 Subject: [PATCH 12/21] feat(build): add a macOS cross-architecture Nix dev shell MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `nix develop .#cross` on macOS exposes the OTHER darwin architecture's zlib, so `scripts/build.sh --arch ` can link an x86_64 binary on Apple Silicon and vice versa. The native clang already cross-emits object code via -arch; zlib is the product's single link-time dependency, and the host-arch copy cannot satisfy a cross link, which is the whole gap. Darwin-gated and opt-in: the attribute set is empty on Linux, the default shell is untouched, flake.lock is unchanged and no new flake input is introduced. The shell deliberately does not use inputsFrom, since that would put the host-arch zlib back on the link path. Distilled from #724 by Kris Williams. That version also wired libgit2 via pkg-config, which is now dead weight: libgit2 was removed project-wide for GPL-licensing reasons and appears nowhere in the tree, so both it and pkg-config are dropped here — leaving zlib as the only library the cross shell needs to place. Validated by parsing the flake with nix-instantiate in a nixos/nix container (no Nix toolchain on the maintainer host), which also confirms the shellHook's ''${...} escaping resolves to literal shell expansions. Evaluating or entering the shell needs a macOS host with Nix; no CI leg exercises it, as noted on the original PR. Addresses #705. Co-authored-by: Kris Williams <115474+kriswill@users.noreply.github.com> Signed-off-by: Martin Vogel --- flake.nix | 53 ++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 52 insertions(+), 1 deletion(-) diff --git a/flake.nix b/flake.nix index 4a6a04576..2c0929702 100644 --- a/flake.nix +++ b/flake.nix @@ -7,6 +7,57 @@ let systems = [ "aarch64-darwin" "x86_64-darwin" "aarch64-linux" "x86_64-linux" ]; forAllSystems = f: nixpkgs.lib.genAttrs systems (system: f nixpkgs.legacyPackages.${system}); + + # Cross dev shell (macOS only): build the *other* darwin arch locally. + # The native clang already cross-emits object code via -arch (scripts/env.sh + # exports ARCHFLAGS), so the only thing missing for a cross link is zlib: + # it is the product's single link-time dependency (`-lz`), and the host-arch + # copy cannot satisfy an x86_64 link on arm64 or vice versa. This shell puts + # the TARGET-arch zlib on the include and link paths, and nothing else. + # + # Building needs no Rosetta; *running* an x86_64 binary on Apple Silicon + # does. The target-arch libs come from prebuilt substitutes, so entering the + # shell is a download rather than a cross-build of the dependency tree. + # + # Returns {} off macOS, where there is no second darwin arch to target. + crossDevShells = pkgs: + let + inherit (nixpkgs) lib; + crossTarget = + { + "aarch64-darwin" = "x86_64-darwin"; + "x86_64-darwin" = "aarch64-darwin"; + } + .${pkgs.system} or null; + mkCrossShell = + targetSystem: + let + tpkgs = nixpkgs.legacyPackages.${targetSystem}; + targetArch = if targetSystem == "x86_64-darwin" then "x86_64" else "arm64"; + in + pkgs.mkShell { + # Host toolchain only — clang comes from the shell stdenv. Deliberately + # NOT inputsFrom the default package: that would put the HOST-arch zlib + # on the link path, which is exactly the wrong-arch copy this shell + # exists to displace. + nativeBuildInputs = [ pkgs.gnumake ]; + shellHook = '' + # zlib is consumed as a bare `-lz` / `#include `, so its + # include dir (the header is arch-independent) and lib dir are + # supplied by hand. NIX_LDFLAGS is searched ahead of the link's own + # -L, so the target-arch slice wins; a host-arch copy that still + # reaches ld is skipped as "wrong architecture" rather than + # producing a silently mislinked binary. + export NIX_CFLAGS_COMPILE="-I${lib.getDev tpkgs.zlib}/include ''${NIX_CFLAGS_COMPILE:-}" + export NIX_LDFLAGS="-L${lib.getLib tpkgs.zlib}/lib ''${NIX_LDFLAGS:-}" + echo "[cross] target ${targetSystem} — build with: scripts/build.sh --arch ${targetArch}" + ''; + }; + in + lib.optionalAttrs (crossTarget != null) { + # `nix develop .#cross` then `scripts/build.sh --arch `. + cross = mkCrossShell crossTarget; + }; in { packages = forAllSystems (pkgs: { @@ -45,6 +96,6 @@ default = pkgs.mkShell { inputsFrom = [ self.packages.${pkgs.system}.default ]; }; - }); + } // crossDevShells pkgs); }; } From 5cabeb4b10a5c56b991cf80d7dd2dc754c7a8953 Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Thu, 20 Aug 2026 13:39:49 +0200 Subject: [PATCH 13/21] fix(swift): index protocol method requirements as Method nodes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A Swift protocol requirement — a bodyless `func generate() -> String` inside a protocol — was absent from the graph entirely. Swift codebases are heavily protocol-driven, so the requirement is very often the declaration a reader is looking for, and "who declares generate()" had no answer. Three edits deliver it: protocol_body joins the class-body types so a protocol's members are walked at all; protocol_function_declaration joins swift_func_types, because extract_class_methods gates on that set and would otherwise walk the requirement and discard it; and both name resolvers accept the node, which has the same simple_identifier shape as function_declaration. Distilled from #613 by xbsjason. That PR also carried an enum/struct half whose two assertions fail on CI: tree-sitter-swift has no struct_declaration or enum_declaration node type (it models both as class_declaration), so the corresponding swift_class_types entries are inert and a bare enum is still labeled Class. That is a real pre-existing modelling gap, left untouched here and now documented at the dead entries rather than silently deleted. Also dropped from the original: a find_first_descendant_by_kind rescue in extract_class_methods. Removing it changes no test outcome once the function-type entry is present, so it was carrying nothing. Advances #43. Co-authored-by: xbsjason Signed-off-by: Martin Vogel --- internal/cbm/extract_defs.c | 13 ++++++++++--- internal/cbm/lang_specs.c | 11 ++++++++++- tests/test_extraction.c | 16 ++++++++++++++++ 3 files changed, 36 insertions(+), 4 deletions(-) diff --git a/internal/cbm/extract_defs.c b/internal/cbm/extract_defs.c index 772adcee9..4994bd7b8 100644 --- a/internal/cbm/extract_defs.c +++ b/internal/cbm/extract_defs.c @@ -745,9 +745,14 @@ TSNode cbm_resolve_func_name(TSNode node, CBMLanguage lang) { } /* Swift and newer tree-sitter-kotlin: function_declaration has no `name` - * field; the function name is a `simple_identifier` child. */ + * field; the function name is a `simple_identifier` child. A Swift + * protocol requirement (a bodyless `func` inside a protocol) is a + * separate node type with the same shape; it is named here as well as in + * resolve_method_name because swift_func_types now admits it, so it can + * reach the free-function path too and would otherwise land unnamed. */ if ((lang == CBM_LANG_SWIFT || lang == CBM_LANG_KOTLIN) && - strcmp(kind, "function_declaration") == 0) { + (strcmp(kind, "function_declaration") == 0 || + strcmp(kind, "protocol_function_declaration") == 0)) { TSNode si = cbm_find_child_by_kind(node, "simple_identifier"); if (!ts_node_is_null(si)) { return si; @@ -4579,6 +4584,7 @@ static TSNode find_class_body(TSNode class_node, CBMLanguage lang) { static const char *body_types[] = {"class_body", "interface_body", "enum_body", + "protocol_body", "template_body", "interface_type", "struct_type", @@ -4699,7 +4705,8 @@ static TSNode resolve_method_name(TSNode child, CBMLanguage lang) { } if ((lang == CBM_LANG_SWIFT || lang == CBM_LANG_KOTLIN) && - strcmp(ck, "function_declaration") == 0) { + (strcmp(ck, "function_declaration") == 0 || + strcmp(ck, "protocol_function_declaration") == 0)) { return cbm_find_child_by_kind(child, "simple_identifier"); } diff --git a/internal/cbm/lang_specs.c b/internal/cbm/lang_specs.c index 53b1ca728..9dd1cdb37 100644 --- a/internal/cbm/lang_specs.c +++ b/internal/cbm/lang_specs.c @@ -547,7 +547,16 @@ static const char *objc_var_types[] = {"declaration", NULL}; static const char *objc_assign_types[] = {"assignment_expression", NULL}; // ==================== SWIFT ==================== -static const char *swift_func_types[] = {"function_declaration", "macro_declaration", NULL}; +// protocol_function_declaration: a protocol's method requirements. Needed here as +// well as in the name resolvers — extract_class_methods gates on this set, so +// without the entry the requirement is walked and then discarded. +static const char *swift_func_types[] = {"function_declaration", "protocol_function_declaration", + "macro_declaration", NULL}; +// KNOWN GAP: struct_declaration and enum_declaration are not node types in the +// vendored tree-sitter-swift grammar — it models both as class_declaration — so +// these two entries never match anything, and a bare Swift `enum` is labeled +// Class rather than Enum. Left in place deliberately: they are the only marker +// of that modelling gap, and deleting them would hide it. Tracked separately. static const char *swift_class_types[] = {"class_declaration", "protocol_declaration", "struct_declaration", "enum_declaration", NULL}; static const char *swift_field_types[] = {"property_declaration", NULL}; diff --git a/tests/test_extraction.c b/tests/test_extraction.c index 506973350..687f901d0 100644 --- a/tests/test_extraction.c +++ b/tests/test_extraction.c @@ -655,6 +655,21 @@ TEST(swift_class) { PASS(); } +TEST(swift_protocol) { + /* A protocol requirement is a bodyless func inside a protocol body. Swift + * codebases are heavily protocol-driven, so the requirement is very often + * the declaration a reader is actually looking for — before this it was + * absent from the graph entirely. */ + CBMFileResult *r = extract("protocol StudyRunning {\n func generate() -> String\n}\n", + CBM_LANG_SWIFT, "t", "StudyRunning.swift"); + ASSERT_NOT_NULL(r); + ASSERT_FALSE(r->has_error); + ASSERT(has_def(r, "Interface", "StudyRunning")); + ASSERT(has_def(r, "Method", "generate")); + cbm_free_result(r); + PASS(); +} + /* --- Kotlin --- */ TEST(kotlin_function) { CBMFileResult *r = extract("fun greet(name: String): String = \"Hello $name\"\nfun main() { " @@ -5451,6 +5466,7 @@ SUITE(extraction) { RUN_TEST(csharp_class); RUN_TEST(csharp_interface); RUN_TEST(swift_class); + RUN_TEST(swift_protocol); RUN_TEST(kotlin_function); RUN_TEST(kotlin_class); RUN_TEST(scala_function); From 9eba0279090e0ccf24a736026591ccb36ae31ac3 Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Thu, 20 Aug 2026 15:33:44 +0200 Subject: [PATCH 14/21] docs(npm): align README language and MCP tool counts with the root README pkg/npm/README.md claimed 159 languages and 14 MCP tools while the root README says 158 languages and 15 MCP tools. The tool count is verifiably wrong: the TOOLS[] table in src/mcp/mcp.c has 15 entries. The language count is aligned to the root README, which states 158 consistently in five places. Each stale figure appeared twice; both occurrences of each are fixed, so the package README no longer contradicts itself or the root README. Distilled from two PRs that each caught one half: #967 by ketpatil77 (language count) and #1268 by nhtkylc91 (tool count). Neither fixed all four occurrences, and both touched the same file, so they are combined here with credit to both. Note for whoever refreshes these next: 159 grammar directories exist under internal/cbm/vendored/grammars/, which does not reconcile trivially with either figure (some languages share a grammar; some enums have no vendored grammar). This change deliberately does not re-derive the number, only stops the two READMEs disagreeing. Co-authored-by: ketpatil77 Co-authored-by: nhtkylc91 Signed-off-by: Martin Vogel --- pkg/npm/README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/npm/README.md b/pkg/npm/README.md index e6efc83a1..0121a3635 100644 --- a/pkg/npm/README.md +++ b/pkg/npm/README.md @@ -7,7 +7,7 @@ **The fastest and most efficient code intelligence engine for AI coding agents.** Full-indexes an average repository in milliseconds, the Linux kernel (28M LOC, 75K files) in 3 minutes. Answers structural queries in under 1ms. This npm wrapper downloads, verifies, and caches the selected native runtime set: the executable, its authenticated integration asset, and—when requested—the content-addressed UI pack. -High-quality parsing through [tree-sitter](https://tree-sitter.github.io/tree-sitter/) AST analysis across 159 languages — producing a persistent knowledge graph of functions, classes, call chains, HTTP routes, and cross-service links. 14 MCP tools. No hosted service or API key; this package requires Node.js to manage and launch the cached native runtime set. Plug and play across 43 automatic/conditional client surfaces. +High-quality parsing through [tree-sitter](https://tree-sitter.github.io/tree-sitter/) AST analysis across 158 languages — producing a persistent knowledge graph of functions, classes, call chains, HTTP routes, and cross-service links. 15 MCP tools. No hosted service or API key; this package requires Node.js to manage and launch the cached native runtime set. Plug and play across 43 automatic/conditional client surfaces. ## Installation @@ -29,7 +29,7 @@ Restart your agent. Say **"Index this project"** — done. - **Extreme indexing speed** — Linux kernel (28M LOC, 75K files) in 3 minutes. RAM-first pipeline with LZ4 compression and in-memory SQLite. - **Plug and play** — verified native runtime sets for macOS (arm64/amd64), Linux (arm64/amd64), and Windows (arm64/amd64). No Docker or API keys; Node.js owns package download, cache repair, and launch. -- **159 languages** — vendored tree-sitter grammars compiled into the binary. Nothing to install, nothing that breaks. +- **158 languages** — vendored tree-sitter grammars compiled into the binary. Nothing to install, nothing that breaks. - **120x fewer tokens** — 5 structural queries: ~3,400 tokens vs ~412,000 via file-by-file search. - **43 supported automatic/conditional client surfaces** — `install` configures the appropriate MCP, durable-context, and documented hook surfaces without widening client permissions. - **Detected automatically (37)** — Claude Code, Codex CLI, Gemini CLI, Zed, OpenCode, Antigravity, Aider, KiloCode, VS Code, Cursor, Windsurf, Augment / Auggie, OpenClaw, Kiro, Junie, Hermes, OpenHands, Cline, Warp, Qwen Code, GitHub Copilot CLI, Factory Droid, Crush, Goose, Mistral Vibe, Qoder CLI, Kimi Code CLI, GitLab Duo CLI, Rovo Dev CLI, Amp, Devin CLI / Local, Tabnine, Amazon Q Developer IDE, CodeBuddy Code CLI, IBM Bob Shell, Pochi, and Pi. @@ -38,7 +38,7 @@ Restart your agent. Say **"Index this project"** — done. - **Lifecycle hooks stay conservative** — Kimi uses `UserPromptSubmit`; on macOS/Linux, GitLab Duo gets a fail-open user `SessionStart`, while Devin gets `UserPromptSubmit`, `PostCompaction`, and a deduplicated `SessionStart` when Claude does not already provide it. Qoder, GitLab Duo, Devin, and Factory hooks are withheld on Windows without a documented shell/executor contract. Cline's auto-activating file hooks are withheld because their context output is not reliably consumed, CodeBuddy beta hooks are not auto-installed, and Cursor context hooks remain withheld. - **Subagent access is explicit** — Claude, Gemini, Kiro, Qwen, CodeBuddy, KiloCode, Mistral Vibe, Qoder, Junie, and Factory get documented graph profiles with the narrowest tool/server filters their schemas support. KiloCode and Vibe enumerate read-only query tools rather than using server wildcards. Cursor, Rovo, Pochi, and Cline use explicit parent handoff where child MCP is unavailable or unsafe; IBM Bob receives no invented hook or agent. - **Manual, UI, cloud, or repository-managed (not counted)** — Qodo, Warp MCP, JetBrains AI/ACP, GitHub Copilot coding agent, Jules, CodeRabbit, Replit, BLACKBOX AI, Plandex, and SWE-agent. Warp is counted above for its detected skill installation; its MCP connection remains manual. -- **14 MCP tools** — search, trace, architecture, impact analysis, Cypher queries, dead code detection, cross-service HTTP linking, ADR management, and more. +- **15 MCP tools** — search, trace, architecture, impact analysis, Cypher queries, dead code detection, cross-service HTTP linking, ADR management, and more. ## Supported Platforms From 1af49dfe584d9bc146d6de836bdf45c7a9060d07 Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Thu, 20 Aug 2026 16:43:38 +0200 Subject: [PATCH 15/21] fix(watcher): stop nested non-git dirs and monorepo siblings retriggering Three related defects in how the watcher decides what counts as a change. 1. `git rev-parse --git-dir` walks UP the tree, so an ordinary folder that merely sits under an unrelated repository answered yes. The watcher then inherited that ancestor's dirty state -- permanently non-empty and about entirely different files -- and reindexed on every poll forever. A directory is now git-managed only if it carries its own .git (directory or gitlink file) or the ancestor actually tracks something inside it, so a real monorepo sub-package still qualifies while a scratch or ignored folder does not. 2. `git status` reports the whole repository regardless of -C, so every package in a monorepo reindexed whenever any sibling was edited. The status call is now scoped with a `-- .` pathspec. 3. Porcelain paths are REPOSITORY-relative but were stat'ed against root_path. For a project watched at the repository root the two coincide and the bug is invisible; for a subdirectory project every stat missed, silently degrading the dirty signature to text-only and losing the size/mtime component that makes an edit to an already-dirty file detectable. Paths now resolve through `rev-parse --show-cdup`. --show-cdup rather than --show-toplevel because MSYS/Cygwin git returns a translated absolute path from the latter that will not join onto the native path we hold; a relative hop composes on every platform. Distilled from #1001 by bethzyy, whose diagnosis identified all three. Main independently fixed that PR's two headline problems (the unbounded non-git file count, and per-poll churn via a dirty-state signature) while it waited, and the watcher was rewritten onto a supervised argv-spawn harness in between, so this is a reimplementation against the new code rather than a rebase of theirs. Both new tests are verified binding: reverting the classification guard reddens the nested-directory test, and removing the pathspec reddens the monorepo one, each without disturbing the other. Addresses the remaining parts of #713, #841 and #937. Co-authored-by: bethzyy Signed-off-by: Martin Vogel --- src/watcher/watcher.c | 139 ++++++++++++++++++++++++++++++++++++++++-- tests/test_watcher.c | 130 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 263 insertions(+), 6 deletions(-) diff --git a/src/watcher/watcher.c b/src/watcher/watcher.c index 38c1b0163..ec186c05a 100644 --- a/src/watcher/watcher.c +++ b/src/watcher/watcher.c @@ -75,6 +75,10 @@ typedef struct { uint64_t last_dirty_sig; /* committed dirty-state signature */ uint64_t pending_dirty_sig; /* observed at check time */ char pending_head[CBM_SZ_128]; /* HEAD observed at check time */ + /* Hop from root_path up to the repository root ("" when they are the same), + * from `rev-parse --show-cdup`. Porcelain paths are repository-relative, so + * the signature needs this to stat them. Resolved once at baseline. */ + char repo_cdup[CBM_SZ_4K]; } project_state_t; /* ── Watcher struct ─────────────────────────────────────────────── */ @@ -452,6 +456,86 @@ static watcher_git_status_t git_repo_status(cbm_watcher_t *w, project_state_t *s return watcher_git_run(w, state, argv, 0, NULL); } +/* True when root_path carries its OWN repository marker: a `.git` directory, or + * a `.git` file (the gitlink form used by linked worktrees and initialized + * submodules). Deliberately a filesystem check, not a git invocation — the whole + * point is to learn something `rev-parse` cannot tell us, because it walks up. */ +static bool git_has_own_dot_git(const char *root_path) { + char path[CBM_SZ_4K]; + int written = snprintf(path, sizeof(path), "%s/.git", root_path); + if (written <= 0 || (size_t)written >= sizeof(path)) { + return false; + } + struct stat st; + return stat(path, &st) == 0 && (S_ISDIR(st.st_mode) || S_ISREG(st.st_mode)); +} + +/* Does the ancestor repository actually track anything inside this directory? + * Emptiness distinguishes "a scratch folder that merely sits under a repo" from + * "a genuine subdirectory of one". Output is capped: we only care whether the + * first byte exists, never what it is. */ +static watcher_git_status_t git_tracks_anything_here(cbm_watcher_t *w, project_state_t *state, + bool *tracked_out) { + *tracked_out = false; + const char *argv[] = {"git", "-C", state->root_path, "ls-files", "-z", "--", ".", NULL}; + watcher_git_output_t output; + watcher_git_status_t status = watcher_git_run(w, state, argv, WATCHER_GIT_HEAD_MAX, &output); + if (status != WATCHER_GIT_OK) { + return status; + } + FILE *fp = cbm_fopen(output.path, "rb"); + if (fp) { + *tracked_out = fgetc(fp) != EOF; + (void)fclose(fp); + } + watcher_git_output_cleanup(&output); + return fp ? WATCHER_GIT_OK : WATCHER_GIT_SUPERVISION_FAILED; +} + +/* Relative hop from root_path up to the repository root, as `git rev-parse + * --show-cdup` reports it ("" at the root, "../" one level down, and so on). + * + * This matters because `git status --porcelain` prints paths relative to the + * REPOSITORY root, while the signature stats them relative to root_path. For a + * project watched at the repository root the two coincide and the bug is + * invisible; for a subdirectory project every stat silently misses, and the + * signature quietly degrades to text-only — losing the size/mtime component + * that makes an edit to an already-dirty file detectable. + * + * --show-cdup rather than --show-toplevel: under MSYS/Cygwin git, --show-toplevel + * returns a translated absolute path (/c/... or a drive-letter form) that does + * not join cleanly onto the native root_path we hold. A relative hop composes + * correctly on every platform because it never leaves our own path space. */ +static watcher_git_status_t git_repo_cdup(cbm_watcher_t *w, project_state_t *state, char *out, + size_t out_size) { + if (!out || out_size < 2) { + return WATCHER_GIT_SUPERVISION_FAILED; + } + out[0] = '\0'; + const char *argv[] = {"git", "-C", state->root_path, "rev-parse", "--show-cdup", NULL}; + watcher_git_output_t output; + watcher_git_status_t status = watcher_git_run(w, state, argv, WATCHER_GIT_HEAD_MAX, &output); + if (status != WATCHER_GIT_OK) { + return status; + } + FILE *file = cbm_fopen(output.path, "rb"); + bool read = file && fgets(out, (int)out_size, file) != NULL; + if (file) { + (void)fclose(file); + } + watcher_git_output_cleanup(&output); + if (!read) { + /* At the repository root git prints an empty line; that is success. */ + out[0] = '\0'; + return WATCHER_GIT_OK; + } + size_t len = strlen(out); + while (len > 0 && (out[len - 1] == '\n' || out[len - 1] == '\r')) { + out[--len] = '\0'; + } + return WATCHER_GIT_OK; +} + static watcher_git_status_t git_head(cbm_watcher_t *w, project_state_t *state, char *out, size_t out_size) { if (!out || out_size < 2) { @@ -511,9 +595,14 @@ static int64_t sig_stat_mtime_ns(const struct stat *st) { * of an already-dirty file still produces a new signature. A failed stat * (deleted file, quoting artifact) degrades to the entry text alone — the * deletion itself is represented by the porcelain status. */ -static uint64_t sig_fold_path_stat(uint64_t h, const char *root_path, const char *rel) { +/* `rel` is repository-relative (that is what porcelain prints), so it is joined + * through `cdup` — the hop from root_path up to the repository root — rather than + * onto root_path directly. cdup is "" when the project IS the repository root, + * which reduces this to the original join. */ +static uint64_t sig_fold_path_stat(uint64_t h, const char *root_path, const char *cdup, + const char *rel) { char abs[CBM_SZ_4K]; - snprintf(abs, sizeof(abs), "%s/%s", root_path, rel); + snprintf(abs, sizeof(abs), "%s/%s%s", root_path, cdup ? cdup : "", rel); struct stat st; if (stat(abs, &st) == 0) { int64_t mt = sig_stat_mtime_ns(&st); @@ -538,8 +627,16 @@ static watcher_git_status_t git_dirty_signature(cbm_watcher_t *w, project_state_ return WATCHER_GIT_SUPERVISION_FAILED; } *signature_out = 0; - const char *status_argv[] = {"git", "--no-optional-locks", "-C", state->root_path, - "status", "--porcelain", "-uall", "-z", + /* `-- .` scopes the report to the watched directory. Without it a project + * watched at a sub-package of a monorepo reindexes whenever any SIBLING + * package changes, because git reports the whole repository's dirty state + * regardless of -C. Paths stay repository-relative either way, which is + * what repo_cdup is for. */ + const char *status_argv[] = {"git", "--no-optional-locks", + "-C", state->root_path, + "status", "--porcelain", + "-uall", "-z", + "--", ".", NULL}; watcher_git_output_t output; watcher_git_status_t status = @@ -585,7 +682,7 @@ static watcher_git_status_t git_dirty_signature(cbm_watcher_t *w, project_state_ if (entry[0] == 'R' || entry[0] == 'C') { origin_token = true; } - h = sig_fold_path_stat(h, state->root_path, entry + 3); + h = sig_fold_path_stat(h, state->root_path, state->repo_cdup, entry + 3); } } elen = 0; @@ -638,7 +735,7 @@ static watcher_git_status_t git_dirty_signature(cbm_watcher_t *w, project_state_ h = sig_fold(h, line, len); h = sig_fold(h, "", 1); if (len > 3 && line[2] == ' ') { - h = sig_fold_path_stat(h, state->root_path, line + 3); + h = sig_fold_path_stat(h, state->root_path, state->repo_cdup, line + 3); } } parsed = !ferror(fp) && fclose(fp) == 0; @@ -1060,9 +1157,39 @@ static bool init_baseline(cbm_watcher_t *w, project_state_t *s) { return false; } s->is_git = repository_status == WATCHER_GIT_OK; + + /* `rev-parse --git-dir` walks UP, so an ordinary folder that merely happens + * to live under some unrelated repository answers yes. Treating it as a git + * project is what produced the runaway churn: it inherits the ancestor's + * dirty state, which is permanently non-empty and has nothing to do with + * this directory, so every poll looked like a change. + * + * A directory is only really git-managed here if it carries its own .git, + * or the ancestor repository actually tracks something inside it. A + * genuine sub-package of a monorepo passes the second test; a scratch or + * gitignored folder sitting under a repo fails both and is polled as a + * plain directory instead. */ + if (s->is_git && !git_has_own_dot_git(s->root_path)) { + bool tracked = false; + watcher_git_status_t tracked_status = git_tracks_anything_here(w, s, &tracked); + if (tracked_status != WATCHER_GIT_OK && tracked_status != WATCHER_GIT_COMMAND_FAILED) { + return false; + } + if (!tracked) { + s->is_git = false; + cbm_log_info("watcher.nested_non_git", "project", s->project_name, "path", + s->root_path); + } + } + s->baseline_done = true; if (s->is_git) { + watcher_git_status_t cdup_status = git_repo_cdup(w, s, s->repo_cdup, sizeof(s->repo_cdup)); + if (cdup_status != WATCHER_GIT_OK && cdup_status != WATCHER_GIT_COMMAND_FAILED) { + s->baseline_done = false; + return false; + } watcher_git_status_t head_status = git_head(w, s, s->last_head, sizeof(s->last_head)); if (head_status != WATCHER_GIT_OK && head_status != WATCHER_GIT_COMMAND_FAILED) { s->baseline_done = false; diff --git a/tests/test_watcher.c b/tests/test_watcher.c index cd4144f3e..14b602be1 100644 --- a/tests/test_watcher.c +++ b/tests/test_watcher.c @@ -1226,6 +1226,134 @@ TEST(watcher_detects_git_commit) { PASS(); } +/* A plain directory that merely SITS UNDER an unrelated repository is not a git + * project. `git rev-parse --git-dir` walks up, so it answers yes for such a + * folder, and the watcher then inherited the ancestor's dirty state — which is + * permanently non-empty and has nothing to do with this directory — and + * reindexed on every single poll forever (#841/#937: reporters measured this in + * hundreds of GB of writes per day). */ +TEST(watcher_nested_non_git_dir_does_not_inherit_ancestor_dirt) { + char tmpdir[256]; + snprintf(tmpdir, sizeof(tmpdir), "/tmp/cbm_watcher_nested_XXXXXX"); + if (!cbm_mkdtemp(tmpdir)) + FAIL("cbm_mkdtemp failed"); + + if (wt_git(tmpdir, "init -q") != 0) { + th_rmtree(tmpdir); + FAIL("git init failed"); + } + { + char p[300]; + th_write_file(wt_path(p, sizeof(p), tmpdir, "tracked.txt"), "hello\n"); + } + wt_git(tmpdir, "add tracked.txt"); + wt_git(tmpdir, "commit -q -m init"); + + /* Leave the ancestor permanently dirty — this is the condition that used to + * retrigger indexing on every poll of the nested directory. */ + { + char p[300]; + th_write_file(wt_path(p, sizeof(p), tmpdir, "dirty.txt"), "uncommitted\n"); + } + + /* A scratch directory inside it, tracked by nothing. */ + char nested[400]; + snprintf(nested, sizeof(nested), "%s/scratch", tmpdir); + if (!cbm_mkdir_p(nested, 0755)) { + th_rmtree(tmpdir); + FAIL("mkdir nested failed"); + } + { + char p[500]; + th_write_file(wt_path(p, sizeof(p), nested, "notes.md"), "scratch\n"); + } + + cbm_store_t *store = cbm_store_open_memory(); + cbm_watcher_t *w = cbm_watcher_new(store, index_callback, NULL); + cbm_watcher_watch(w, "nested-scratch", nested); + index_call_count = 0; + + cbm_watcher_poll_once(w); /* baseline */ + int after_baseline = index_call_count; + + /* Three polls with the ancestor still dirty and the nested dir untouched. + * Under the old classification each of these reindexed. */ + for (int i = 0; i < 3; i++) { + cbm_watcher_touch(w, "nested-scratch"); + cbm_watcher_poll_once(w); + } + ASSERT_EQ(index_call_count, after_baseline); + + cbm_watcher_free(w); + cbm_store_close(store); + th_rmtree(tmpdir); + PASS(); +} + +/* A genuine sub-package of a monorepo IS git-managed, and must still be watched + * as such — the nested-directory guard above must not disqualify it. It also + * must not react to a sibling package's changes: `git status` reports the whole + * repository regardless of -C, so without a `-- .` pathspec every package in a + * monorepo reindexes whenever any other one is edited. */ +TEST(watcher_monorepo_subdir_ignores_sibling_changes) { + char tmpdir[256]; + snprintf(tmpdir, sizeof(tmpdir), "/tmp/cbm_watcher_mono_XXXXXX"); + if (!cbm_mkdtemp(tmpdir)) + FAIL("cbm_mkdtemp failed"); + + if (wt_git(tmpdir, "init -q") != 0) { + th_rmtree(tmpdir); + FAIL("git init failed"); + } + char pkg_a[400]; + char pkg_b[400]; + snprintf(pkg_a, sizeof(pkg_a), "%s/pkg-a", tmpdir); + snprintf(pkg_b, sizeof(pkg_b), "%s/pkg-b", tmpdir); + if (!cbm_mkdir_p(pkg_a, 0755) || !cbm_mkdir_p(pkg_b, 0755)) { + th_rmtree(tmpdir); + FAIL("mkdir packages failed"); + } + { + char p[500]; + th_write_file(wt_path(p, sizeof(p), pkg_a, "a.txt"), "a\n"); + th_write_file(wt_path(p, sizeof(p), pkg_b, "b.txt"), "b\n"); + } + wt_git(tmpdir, "add -A"); + wt_git(tmpdir, "commit -q -m init"); + + cbm_store_t *store = cbm_store_open_memory(); + cbm_watcher_t *w = cbm_watcher_new(store, index_callback, NULL); + cbm_watcher_watch(w, "pkg-a", pkg_a); + index_call_count = 0; + + cbm_watcher_poll_once(w); /* baseline */ + int after_baseline = index_call_count; + + /* Edit the SIBLING package only. pkg-a is untouched. */ + { + char p[500]; + th_append_file(wt_path(p, sizeof(p), pkg_b, "b.txt"), "sibling edit\n"); + } + cbm_watcher_touch(w, "pkg-a"); + cbm_watcher_poll_once(w); + ASSERT_EQ(index_call_count, after_baseline); + + /* Editing pkg-a itself must still be seen — the scoping must not have + * silenced real changes. */ + { + char p[500]; + th_append_file(wt_path(p, sizeof(p), pkg_a, "a.txt"), "own edit\n"); + } + cbm_watcher_touch(w, "pkg-a"); + cbm_watcher_poll_once(w); + ASSERT_EQ(index_call_count, after_baseline + 1); + + cbm_watcher_free(w); + cbm_store_close(store); + th_rmtree(tmpdir); + PASS(); +} + /* SHA-256 repositories emit a 64-hex-character HEAD. The watcher must retain * the complete object ID (plus line terminator/NUL while reading it), otherwise * baseline initialization silently retries forever and auto-refresh never runs. */ @@ -3041,6 +3169,8 @@ SUITE(watcher) { /* Git change detection */ RUN_TEST(watcher_detects_git_commit); + RUN_TEST(watcher_nested_non_git_dir_does_not_inherit_ancestor_dirt); + RUN_TEST(watcher_monorepo_subdir_ignores_sibling_changes); RUN_TEST(watcher_detects_sha256_git_commit); RUN_TEST(watcher_detects_dirty_worktree); RUN_TEST(watcher_identical_watch_preserves_dirty_baseline); From 4a29f0da5967bd355a07143bd3b1e5675fda7d5b Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Thu, 20 Aug 2026 18:17:28 +0200 Subject: [PATCH 16/21] fix(cli): wait out a held startup transition instead of failing at 10s A busy startup transition always resolves, by one of two routes: the live peer holding it finishes its command, or the holder is already dead and the operating system has not finished reclaiming the lock. Waiting was capped at 10s regardless, so a clock decided a user-visible outcome and commands were refused with "CLI startup coordination remained busy" while nothing was actually wrong. The second route is Windows-specific and is what makes this reachable in practice. On POSIX the kernel drops flock the instant an owner dies. Windows byte-range locks do not: Microsoft documents that after a process terminates holding one, "the time it takes for the operating system to unlock these locks depends upon available system resources", and that until then "access to these files may be denied". A loaded CI runner is exactly where those resources are scarce, and tests/windows/test_daemon_stability.py creates the condition deliberately by hard-killing daemons with `taskkill /F`, including a crash-recovery section -- so the next client meets a lock whose owner no longer exists and is refused for a reason that no longer applies. The cap becomes a backstop against a peer that never finishes rather than a budget for healthy contention, and the message it prints now names both explanations instead of just "busy", which sent reporters looking for a CBM session that had already exited. Clean exits already release via main_local_transition_close, so no new release path is needed; this only affects what happens after an abrupt termination or under genuine concurrency. VERIFICATION LIMIT, stated plainly: this cannot be reproduced or verified on macOS -- the mechanism requires Windows lock-reclaim semantics, and a local 12-client storm passes even under the old cap. 458 daemon/ipc/runtime/cli/ watcher tests pass and lint is clean, but whether this removes the CI flake can only be established on Windows CI. Signed-off-by: Martin Vogel --- src/main.c | 68 +++++++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 60 insertions(+), 8 deletions(-) diff --git a/src/main.c b/src/main.c index 288dec357..c5c936e49 100644 --- a/src/main.c +++ b/src/main.c @@ -44,6 +44,12 @@ enum { MAIN_PATH_CAP = 4096, MAIN_CONNECT_TIMEOUT_MS = 1000, MAIN_STARTUP_TIMEOUT_MS = 10000, + /* Backstop for waiting out a held startup transition — see + * main_local_transition_acquire. Not a budget for healthy contention: a busy + * lock always resolves, either because the live holder finishes or because + * the OS finishes reclaiming a dead holder's lock. This only bounds a peer + * that never finishes, so a command cannot hang indefinitely. */ + MAIN_STARTUP_CONTENTION_CEILING_MS = 120000, MAIN_MCP_STARTUP_TIMEOUT_MS = 30000, MAIN_REQUEST_TIMEOUT_MS = 24 * 60 * 60 * 1000, MAIN_HOOK_CONNECT_TIMEOUT_MS = 250, @@ -1288,14 +1294,47 @@ static bool main_local_cli_feedback_enabled(int argc, char **argv) { return cbm_cli_progress_enabled(requested, cli_isatty(2) != 0); } +/* Acquire the exclusive startup transition, waiting out whatever currently holds it. + * + * try_acquire answers one of three things, and they must not be conflated: + * 1 acquired + * -1 coordination is unsafe or unverifiable — fail now, waiting cannot help + * 0 BUSY: the lock is held + * + * A busy lock always resolves, by one of two routes: + * - a live peer holds it, and releases when its command finishes; + * - the holder is already dead and the operating system has not finished + * reclaiming the lock yet. + * + * The second route is Windows-specific and is why this wait needs room. On POSIX + * the kernel drops flock the instant the owner dies. Windows byte-range locks do + * not work that way: Microsoft documents that after a process terminates holding + * one, "the time it takes for the operating system to unlock these locks depends + * upon available system resources", and that until then "access to these files + * may be denied". A loaded CI runner is precisely where those resources are + * scarce, and `tests/windows/test_daemon_stability.py` manufactures the situation + * deliberately — it hard-kills daemons with `taskkill /F`, including a crash + * recovery section, so the next client meets a lock whose owner no longer exists. + * + * Waiting was previously capped at MAIN_STARTUP_TIMEOUT_MS (10s), which let a + * clock decide a user-visible outcome: a command was refused with "coordination + * remained busy" while the only thing wrong was that the OS had not yet swept up + * after a killed process. Since both routes above terminate, the correct response + * to busy is to keep waiting; the ceiling below is a backstop against a peer that + * never finishes, not a budget for healthy contention. Clean exits already + * release through main_local_transition_close, so this path is only reached after + * an abrupt termination or under genuine concurrency. */ static int main_local_transition_acquire(const cbm_daemon_ipc_endpoint_t *endpoint, FILE *feedback, cbm_daemon_ipc_local_transition_t **transition_out) { - uint64_t deadline = main_deadline_after(MAIN_STARTUP_TIMEOUT_MS); + uint64_t ceiling = main_deadline_after(MAIN_STARTUP_CONTENTION_CEILING_MS); bool waiting_reported = false; for (;;) { int status = cbm_daemon_ipc_local_transition_try_acquire(endpoint, transition_out); - if (status != 0 || cbm_now_ms() >= deadline) { - return status; + if (status != 0) { + return status; /* acquired, or a genuine coordination failure */ + } + if (cbm_now_ms() >= ceiling) { + return 0; /* still busy after the backstop */ } if (feedback && !waiting_reported) { (void)fputs("Waiting for CBM startup coordination...\n", feedback); @@ -2542,11 +2581,24 @@ int main(int argc, char **argv) { int transition_status = main_local_transition_acquire(local_endpoint, feedback, &local_transition); if (transition_status != 1 || !local_transition) { - (void)fprintf(stderr, - "codebase-memory-mcp: CLI startup coordination %s; retry after the " - "active CBM transition exits\n", - transition_status == 0 ? "remained busy" - : "could not be verified safely"); + if (transition_status == 0) { + /* The backstop fired. Name both explanations: after this much + * waiting the likely causes are a peer that is genuinely stuck, + * or (on Windows) a lock left behind by a force-killed process + * that the OS has not reclaimed. "Busy" alone sent reporters + * hunting for a CBM session that had already exited. */ + (void)fprintf(stderr, + "codebase-memory-mcp: CLI startup coordination stayed busy for " + "%d seconds. Either another CBM command is still running, or a " + "previous one was force-killed and the operating system has not " + "released its lock yet. Check for running CBM processes; if there " + "are none, retry shortly.\n", + MAIN_STARTUP_CONTENTION_CEILING_MS / 1000); + } else { + (void)fprintf(stderr, "codebase-memory-mcp: CLI startup coordination could not " + "be verified safely; retry after active CBM sessions " + "exit\n"); + } goto local_cli_cleanup; } int seal_status = cbm_daemon_ipc_local_transition_seal_legacy(local_transition); From 1acd87912fe46416022eae146e7b13f7796b65f9 Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Thu, 20 Aug 2026 15:59:58 +0200 Subject: [PATCH 17/21] test(java): pin enum constants surviving alongside enum methods MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Java enum methods already resolve on main: find_class_member_body descends into enum_body_declarations (Java-gated) while find_class_body keeps returning enum_body, which is what extract_enum_members needs to reach the constants — they are siblings of enum_body_declarations, not children. That distinction was unguarded. java_enum_dedup_preserves_calls_issue1234 asserted the methods and the absence of duplicate Function defs, but never asserted the constants, so collapsing the two lookups would have passed the suite while silently dropping every enum constant from the graph. PR #984 proposed exactly that collapse — redirecting the shared find_class_body — which is what surfaced the gap. Verified binding: pointing extract_enum_members at find_class_member_body reddens the new Variable assertions and nothing else. Signed-off-by: Martin Vogel Co-authored-by: sahil-mangla --- tests/test_extraction.c | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/test_extraction.c b/tests/test_extraction.c index 21899b1dc..88796c70d 100644 --- a/tests/test_extraction.c +++ b/tests/test_extraction.c @@ -472,6 +472,15 @@ TEST(java_enum_dedup_preserves_calls_issue1234) { ASSERT(has_def(r, "Enum", "Day")); ASSERT(has_def(r, "Method", "isWeekend")); ASSERT(has_def(r, "Method", "label")); + /* The enum CONSTANTS must survive alongside the methods. Reaching the + * methods means descending into enum_body_declarations, and the tempting + * way to do that -- redirecting the shared find_class_body -- also makes + * the constants unreachable, because they are siblings of that node rather + * than children. find_class_member_body exists to descend for members only + * and leave find_class_body (which extract_enum_members uses) alone; these + * two assertions are what stop that distinction being collapsed again. */ + ASSERT(has_def(r, "Variable", "MON")); + ASSERT(has_def(r, "Variable", "SUN")); ASSERT(has_def(r, "Class", "DayUtil")); ASSERT(has_def(r, "Method", "describe")); ASSERT_EQ(count_defs_with_label(r, "Function"), 0); From 18edfa0023ac2e1bb9f6638e32cd6823c40c4411 Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Thu, 20 Aug 2026 21:20:44 +0200 Subject: [PATCH 18/21] fix(build): stop blob regeneration from reverting the ELF stack hardening vendored/nomic/code_vectors_blob.S is a GENERATED file that was hand-edited. 1f674c80 added the ELF .note.GNU-stack section to it after finding that every Linux binary we had ever shipped requested an executable stack: an object carrying no such note tells GNU ld nothing about its stack requirement, and ld then assumes the worst for the entire link. That fix went into the artifact only. scripts/extract_nomic_vectors.py still emitted the Mach-O branch alone, so the next regeneration would have overwritten the tracked .S, dropped both the ELF and COFF branches, and put GNU_STACK RWE back into every release -- silently, because nothing compared the generator against its own output. write_blob_s() now emits the tracked wrapper verbatim, and the new contract test fails if the two ever drift again. The template is compared rather than executed so the test does not need the torch/transformers import the extraction script pulls in. The write is pinned to UTF-8 because the WHY comment carries an em dash and the default encoding is locale-dependent. check-binary-composition.sh already fails a release whose binary has an executable stack; this closes the same hole at edit time, where the diff is still small enough to read. Reported and originally fixed by Anand Aiyer in #1151, which also carried the COFF branch. That PR sat unreviewed for a month while the .S half was rediscovered and landed independently without credit. Co-authored-by: Anand Aiyer Signed-off-by: Martin Vogel --- scripts/extract_nomic_vectors.py | 55 ++++++++++++++++- scripts/test.sh | 3 + tests/test_nomic_blob_generator_contract.sh | 66 +++++++++++++++++++++ 3 files changed, 121 insertions(+), 3 deletions(-) create mode 100644 tests/test_nomic_blob_generator_contract.sh diff --git a/scripts/extract_nomic_vectors.py b/scripts/extract_nomic_vectors.py index ef38c2ca1..7b1c2e799 100755 --- a/scripts/extract_nomic_vectors.py +++ b/scripts/extract_nomic_vectors.py @@ -332,9 +332,18 @@ static inline const int8_t *pretrained_vec_at(int i) {{ def write_blob_s(path: str, incbin_path: str): - """Write assembler .incbin directive.""" - with open(path, "w") as f: - f.write(f"""/* nomic-embed-code vector blob embedded via assembler. */ + """Write the assembler .incbin wrapper for every target object format. + + Must stay byte-identical to the tracked vendored/nomic/code_vectors_blob.S: + tests/test_nomic_blob_generator_contract.sh fails if the two drift. The ELF + branch's .note.GNU-stack is load-bearing -- regenerating without it puts an + executable stack back into every Linux release binary. + """ + with open(path, "w", encoding="utf-8") as f: + f.write(f"""/* nomic-embed-code vector blob embedded via assembler. + * Cross-platform: macOS (Mach-O) vs Linux (ELF) vs Windows (COFF). */ + +#if defined(__APPLE__) .section __DATA,__const .globl _PRETRAINED_VECTOR_BLOB .globl _PRETRAINED_VECTOR_BLOB_LEN @@ -347,6 +356,46 @@ _PRETRAINED_VECTOR_BLOB_END: .p2align 2 _PRETRAINED_VECTOR_BLOB_LEN: .long _PRETRAINED_VECTOR_BLOB_END - _PRETRAINED_VECTOR_BLOB + +#elif defined(_WIN32) || defined(__CYGWIN__) || defined(__MINGW32__) + .section .rdata,"dr" + .globl PRETRAINED_VECTOR_BLOB + .globl PRETRAINED_VECTOR_BLOB_LEN + .p2align 4 +PRETRAINED_VECTOR_BLOB: + .incbin "{incbin_path}" +PRETRAINED_VECTOR_BLOB_END: + + .section .rdata,"dr" + .p2align 2 +PRETRAINED_VECTOR_BLOB_LEN: + .long PRETRAINED_VECTOR_BLOB_END - PRETRAINED_VECTOR_BLOB + +#else + /* WHY: an ELF object that carries no .note.GNU-stack tells the linker + * nothing about its stack requirement, and GNU ld then assumes the WORST + * for the whole link — every Linux release binary shipped GNU_STACK RWE + * because of this one omission. This is the only assembly source in the + * build, so it alone decided that property. The note must stay even though + * a blob of constant data obviously never executes: absence is the signal, + * not the contents. -Wl,-z,noexecstack in the link flags enforces the + * outcome, and scripts/ci/check-binary-composition.sh fails the release if + * an executable stack ever comes back. */ + .section .note.GNU-stack,"",@progbits + + .section .rodata,"a",@progbits + .globl PRETRAINED_VECTOR_BLOB + .globl PRETRAINED_VECTOR_BLOB_LEN + .p2align 4 +PRETRAINED_VECTOR_BLOB: + .incbin "{incbin_path}" +PRETRAINED_VECTOR_BLOB_END: + + .section .rodata,"a",@progbits + .p2align 2 +PRETRAINED_VECTOR_BLOB_LEN: + .long PRETRAINED_VECTOR_BLOB_END - PRETRAINED_VECTOR_BLOB +#endif """) print(f" {path}: written") diff --git a/scripts/test.sh b/scripts/test.sh index 0dc575b04..06b9ead6f 100755 --- a/scripts/test.sh +++ b/scripts/test.sh @@ -218,6 +218,9 @@ bash "$ROOT/tests/test_windows_bundle_contract.sh" echo "=== Step 0f: tree-sitter runtime Makefile dependencies ===" bash "$ROOT/tests/test_makefile_ts_runtime_dependencies.sh" +echo "=== Step 0f2: nomic blob generator contract ===" +bash "$ROOT/tests/test_nomic_blob_generator_contract.sh" + echo "=== Step 0g: security fuzz harness self-test ===" bash "$ROOT/tests/test_security_fuzz_harness.sh" diff --git a/tests/test_nomic_blob_generator_contract.sh b/tests/test_nomic_blob_generator_contract.sh new file mode 100644 index 000000000..8960bbc31 --- /dev/null +++ b/tests/test_nomic_blob_generator_contract.sh @@ -0,0 +1,66 @@ +#!/usr/bin/env bash +# Contract: scripts/extract_nomic_vectors.py must still emit the tracked +# vendored/nomic/code_vectors_blob.S byte for byte. +# +# WHY this exists: the .S is a GENERATED file that was hand-edited to add the +# ELF .note.GNU-stack section. Without that note GNU ld assumes the whole link +# needs an executable stack, and every Linux release binary shipped GNU_STACK +# RWE because of the omission. The generator kept emitting only the Mach-O +# branch, so the next regeneration would have silently reverted the hardening +# and reintroduced the defect. check-binary-composition.sh catches that at +# release time; this catches it at edit time. +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +GEN="$ROOT/scripts/extract_nomic_vectors.py" +TRACKED="$ROOT/vendored/nomic/code_vectors_blob.S" + +for f in "$GEN" "$TRACKED"; do + if [ ! -f "$f" ]; then + echo "FAIL: missing $f" >&2 + exit 1 + fi +done + +# The generator holds the .S as an f-string whose only placeholder is the +# incbin path, so the tracked file with that path re-parameterized must appear +# in it verbatim. Comparing the template rather than running the generator +# keeps this test free of the torch/transformers import the script needs. +python3 - "$GEN" "$TRACKED" <<'PYEOF' +import sys + +gen_path, tracked_path = sys.argv[1], sys.argv[2] + +with open(gen_path, encoding="utf-8") as fh: + generator = fh.read() +with open(tracked_path, encoding="utf-8") as fh: + tracked = fh.read() + +INCBIN = "vendored/nomic/code_vectors.bin" +if INCBIN not in tracked: + sys.exit("FAIL: %s no longer references %s" % (tracked_path, INCBIN)) +if "{" in tracked or "}" in tracked: + sys.exit("FAIL: %s gained a brace; the generator f-string would need " + "escaping and this contract can no longer compare them directly" + % tracked_path) + +template = tracked.replace(INCBIN, "{incbin_path}") + +if template not in generator: + sys.exit( + "FAIL: scripts/extract_nomic_vectors.py no longer emits the tracked\n" + " vendored/nomic/code_vectors_blob.S.\n" + " Regenerating the blob wrapper would overwrite the tracked file\n" + " with different content. Update write_blob_s() so its f-string\n" + " matches the tracked .S with the incbin path parameterized.") + +# The note is the whole point; assert it explicitly so a future edit that keeps +# the files in sync while dropping the note still fails loudly. +NOTE = '.section .note.GNU-stack,"",@progbits' +for label, blob in (("tracked .S", tracked), ("generator template", generator)): + if NOTE not in blob: + sys.exit("FAIL: %s lost the ELF %s section -- Linux binaries would ship " + "an executable stack again" % (label, NOTE)) + +print("ok: generator template is byte-identical to the tracked blob wrapper") +PYEOF From 978ac7d7455c90497dee5299824b7e67a0259632 Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Thu, 20 Aug 2026 21:31:24 +0200 Subject: [PATCH 19/21] test: keep the step-0 contract sequence linear scripts/test.sh labels its pre-build contracts as a running 0a..0u sequence. The new blob-generator contract was slotted in as "0f2" to sit next to the other build-tooling contract, which breaks the one convention the list has. Append it as 0v instead; these steps all run before the build, so position carries no meaning beyond the label. Signed-off-by: Martin Vogel --- scripts/test.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/test.sh b/scripts/test.sh index 06b9ead6f..f8a3d0c3e 100755 --- a/scripts/test.sh +++ b/scripts/test.sh @@ -218,9 +218,6 @@ bash "$ROOT/tests/test_windows_bundle_contract.sh" echo "=== Step 0f: tree-sitter runtime Makefile dependencies ===" bash "$ROOT/tests/test_makefile_ts_runtime_dependencies.sh" -echo "=== Step 0f2: nomic blob generator contract ===" -bash "$ROOT/tests/test_nomic_blob_generator_contract.sh" - echo "=== Step 0g: security fuzz harness self-test ===" bash "$ROOT/tests/test_security_fuzz_harness.sh" @@ -266,6 +263,9 @@ bash "$ROOT/tests/test_runtime_isolation_contract.sh" echo "=== Step 0u: shell line-ending contract ===" bash "$ROOT/tests/test_shell_line_endings.sh" +echo "=== Step 0v: nomic blob generator contract ===" +bash "$ROOT/tests/test_nomic_blob_generator_contract.sh" + # Verify compiler supports target arch verify_compiler "$CC" From f3b356afed18bd81125b7cdd12ab0ca9d9666a50 Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Thu, 20 Aug 2026 21:36:57 +0200 Subject: [PATCH 20/21] test(cli): clean up before failing the install-plan receipt fixture cli_install_plan_receipt_no_mutation_issue388 asserted inline, so any failure returned before free(json) and test_rmdir_r(tmpdir). A red run therefore leaked the receipt and left a stray /tmp/cli-plan-* directory behind -- the failure report was accompanied by exactly the noise that makes the next debugging session harder. Record what went wrong, release everything, then fail. The message now names the specific missing marker rather than reporting that some marker was absent, which is the difference between reading a failure and bisecting one. The env-isolation half of the original PR is not carried over: main's tf_setup_cache_sentinel already unsets CODEX_HOME along with every other client home override, so a per-test save/unset/restore would be redundant now. Distilled from #1149 by Anand Aiyer, which predates that sentinel and could not land as written once its companion PR closed unmerged. Co-authored-by: Anand Aiyer Signed-off-by: Martin Vogel --- tests/test_cli.c | 49 +++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 40 insertions(+), 9 deletions(-) diff --git a/tests/test_cli.c b/tests/test_cli.c index 95fff9d76..6295479f2 100644 --- a/tests/test_cli.c +++ b/tests/test_cli.c @@ -4447,24 +4447,55 @@ TEST(cli_install_plan_receipt_no_mutation_issue388) { test_mkdirp(dir); char *json = cbm_build_install_plan_json(tmpdir, "/usr/local/bin/codebase-memory-mcp"); - ASSERT_NOT_NULL(json); - ASSERT(strstr(json, "agent.install.plan.v1") != NULL); - ASSERT(strstr(json, "writes_started") != NULL); - ASSERT(strstr(json, "next_safe_command") != NULL); - ASSERT(strstr(json, "cursor") != NULL); - ASSERT(strstr(json, ".cursor/mcp.json") != NULL); - ASSERT(strstr(json, ".codex/config.toml") != NULL); + + /* WHY the failures are deferred: asserting inline returns before the free + * and the rmdir below, so every red run leaked the receipt and left a + * stray /tmp/cli-plan-* directory behind. That makes the next debugging + * session harder than the failure it is reporting. Record what went wrong, + * release everything, then fail -- and name the specific marker, because + * "a marker was missing" costs a reader a bisect that "next_safe_command + * was missing" does not. */ + const char *missing = NULL; + if (!json) { + missing = "receipt was NULL"; + } else { + static const char *const required[] = { + "agent.install.plan.v1", "writes_started", "next_safe_command", "cursor", + ".cursor/mcp.json", ".codex/config.toml", + }; + for (size_t i = 0; i < sizeof(required) / sizeof(required[0]); i++) { + if (!strstr(json, required[i])) { + missing = required[i]; + break; + } + } + } free(json); /* Critical: building the plan must NOT have created any config file. */ char cfg[512]; struct stat st; + const char *created = NULL; snprintf(cfg, sizeof(cfg), "%s/.cursor/mcp.json", tmpdir); - ASSERT(stat(cfg, &st) != 0); /* must not exist */ + if (stat(cfg, &st) == 0) { + created = ".cursor/mcp.json"; + } snprintf(cfg, sizeof(cfg), "%s/.codex/config.toml", tmpdir); - ASSERT(stat(cfg, &st) != 0); /* must not exist */ + if (!created && stat(cfg, &st) == 0) { + created = ".codex/config.toml"; + } test_rmdir_r(tmpdir); + + char reason[256]; + if (missing) { + snprintf(reason, sizeof(reason), "install plan receipt is missing %s", missing); + FAIL(reason); + } + if (created) { + snprintf(reason, sizeof(reason), "building an install plan created %s", created); + FAIL(reason); + } PASS(); } From 2ae84e7f73b0d5d8fd59b62441b1cf900a05f204 Mon Sep 17 00:00:00 2001 From: Alex Musichen Date: Fri, 21 Aug 2026 04:11:51 +0200 Subject: [PATCH 21/21] test(pi): pin result wrap, error throw, and abort in generated adapter Maintainer asked for in-tree coverage of {content, details} wrapping and how tool errors are surfaced, plus a cheap abort-signal line. String-shape only; no live Pi process. Signed-off-by: Alex Musichen --- tests/test_agent_clients.c | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/tests/test_agent_clients.c b/tests/test_agent_clients.c index 89dda28ba..ceb717e18 100644 --- a/tests/test_agent_clients.c +++ b/tests/test_agent_clients.c @@ -1139,6 +1139,24 @@ TEST(client_adapter_pi_emits_parameters_and_execute) { PASS(); } +/* Result wrap + abort: Pi's TUI reads result.content; a spawn/parse failure + * must throw rather than return a result without content; AbortSignal must + * kill the cli child. String-shape only; a live Pi process is not gated. */ +TEST(client_adapter_pi_wraps_result_and_honors_abort) { + char *js = cbm_client_adapter_pi("/usr/local/bin/codebase-memory-mcp"); + ASSERT_NOT_NULL(js); + ASSERT_NOT_NULL(strstr(js, "return { content, details: result ?? {} };")); + ASSERT_NOT_NULL(strstr(js, "if (result && typeof result === 'object' && result.error)")); + ASSERT_NOT_NULL(strstr(js, "throw new Error(String(result.error));")); + ASSERT_NOT_NULL(strstr(js, "Array.isArray(result.content)")); + ASSERT_NOT_NULL( + strstr(js, "[{ type: 'text', text: JSON.stringify(result ?? null, null, 2) }]")); + ASSERT_NOT_NULL(strstr(js, "signal?.addEventListener('abort', onAbort, { once: true })")); + ASSERT_NOT_NULL(strstr(js, "if (!child.killed) child.kill();")); + free(js); + PASS(); +} + /* #616 rejected `"` in its path template but not `\`, so a Windows home like * C:\Users\urs\bin produced `\u` — an invalid unicode escape — and the whole * auto-loaded plugin failed to parse. A broken plugin in an auto-load directory @@ -1230,6 +1248,7 @@ SUITE(agent_clients) { RUN_TEST(client_adapter_pi_registers_every_registry_tool); RUN_TEST(client_adapter_pi_default_exports_its_factory_issue1550); RUN_TEST(client_adapter_pi_emits_parameters_and_execute); + RUN_TEST(client_adapter_pi_wraps_result_and_honors_abort); RUN_TEST(client_adapter_escapes_windows_paths_and_quotes); RUN_TEST(client_adapter_opencode_sends_the_required_hook_event); RUN_TEST(client_adapter_rejects_missing_binary_path);